summaryrefslogtreecommitdiffstats
path: root/libnetdata
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-05 11:19:16 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-07-24 09:53:24 +0000
commitb5f8ee61a7f7e9bd291dd26b0585d03eb686c941 (patch)
treed4d31289c39fc00da064a825df13a0b98ce95b10 /libnetdata
parentAdding upstream version 1.44.3. (diff)
downloadnetdata-b5f8ee61a7f7e9bd291dd26b0585d03eb686c941.tar.xz
netdata-b5f8ee61a7f7e9bd291dd26b0585d03eb686c941.zip
Adding upstream version 1.46.3.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--libnetdata/Makefile.am45
-rw-r--r--libnetdata/README.md14
-rw-r--r--libnetdata/adaptive_resortable_list/Makefile.am8
-rw-r--r--libnetdata/adaptive_resortable_list/README.md103
-rw-r--r--libnetdata/aral/Makefile.am8
-rw-r--r--libnetdata/aral/README.md173
-rw-r--r--libnetdata/avl/Makefile.am8
-rw-r--r--libnetdata/avl/README.md21
-rw-r--r--libnetdata/buffer/Makefile.am8
-rw-r--r--libnetdata/buffer/README.md20
-rw-r--r--libnetdata/buffer/buffer.c503
-rw-r--r--libnetdata/buffer/buffer.h1228
-rw-r--r--libnetdata/buffered_reader/Makefile.am8
-rw-r--r--libnetdata/buffered_reader/buffered_reader.h146
-rw-r--r--libnetdata/circular_buffer/Makefile.am8
-rw-r--r--libnetdata/circular_buffer/README.md14
-rw-r--r--libnetdata/clocks/Makefile.am8
-rw-r--r--libnetdata/clocks/README.md10
-rw-r--r--libnetdata/completion/Makefile.am4
-rw-r--r--libnetdata/completion/completion.c64
-rw-r--r--libnetdata/config/Makefile.am8
-rw-r--r--libnetdata/config/README.md58
-rw-r--r--libnetdata/datetime/Makefile.am8
-rw-r--r--libnetdata/datetime/README.md11
-rw-r--r--libnetdata/dictionary/Makefile.am8
-rw-r--r--libnetdata/dictionary/README.md235
-rw-r--r--libnetdata/dictionary/dictionary.c3799
-rw-r--r--libnetdata/dictionary/dictionary.h335
-rw-r--r--libnetdata/dyn_conf/README.md188
-rw-r--r--libnetdata/dyn_conf/dyn_conf.c1140
-rw-r--r--libnetdata/dyn_conf/dyn_conf.h237
-rw-r--r--libnetdata/dyn_conf/tests/sample_test_config.json22
-rw-r--r--libnetdata/dyn_conf/tests/sub_tests/test_parent_child.rb192
-rwxr-xr-xlibnetdata/dyn_conf/tests/test_dyncfg.rb266
-rwxr-xr-xlibnetdata/dyn_conf/tests/test_plugin/test.plugin250
-rw-r--r--libnetdata/ebpf/Makefile.am8
-rw-r--r--libnetdata/ebpf/README.md13
-rw-r--r--libnetdata/ebpf/ebpf.c1618
-rw-r--r--libnetdata/ebpf/ebpf.h487
-rw-r--r--libnetdata/endian.h32
-rw-r--r--libnetdata/eval/Makefile.am8
-rw-r--r--libnetdata/eval/eval.c1201
-rw-r--r--libnetdata/eval/eval.h87
-rw-r--r--libnetdata/facets/Makefile.am8
-rw-r--r--libnetdata/functions_evloop/Makefile.am8
-rw-r--r--libnetdata/functions_evloop/functions_evloop.c223
-rw-r--r--libnetdata/functions_evloop/functions_evloop.h101
-rw-r--r--libnetdata/health/Makefile.am7
-rw-r--r--libnetdata/health/health.c169
-rw-r--r--libnetdata/health/health.h53
-rw-r--r--libnetdata/http/http_defs.h47
-rw-r--r--libnetdata/json/Makefile.am8
-rw-r--r--libnetdata/json/README.md14
-rw-r--r--libnetdata/json/json.c557
-rw-r--r--libnetdata/json/json.h77
-rw-r--r--libnetdata/july/Makefile.am8
-rw-r--r--libnetdata/july/README.md14
-rw-r--r--libnetdata/libnetdata.h916
-rw-r--r--libnetdata/line_splitter/Makefile.am8
-rw-r--r--libnetdata/line_splitter/README.md14
-rw-r--r--libnetdata/locks/Makefile.am8
-rw-r--r--libnetdata/locks/README.md107
-rw-r--r--libnetdata/log/Makefile.am9
-rw-r--r--libnetdata/log/README.md204
-rw-r--r--libnetdata/log/log.c2431
-rw-r--r--libnetdata/log/log.h301
-rw-r--r--libnetdata/onewayalloc/Makefile.am8
-rw-r--r--libnetdata/onewayalloc/README.md75
-rw-r--r--libnetdata/os.c300
-rw-r--r--libnetdata/os.h70
-rw-r--r--libnetdata/popen/Makefile.am8
-rw-r--r--libnetdata/popen/README.md15
-rw-r--r--libnetdata/procfile/Makefile.am8
-rw-r--r--libnetdata/procfile/README.md71
-rw-r--r--libnetdata/required_dummies.h45
-rw-r--r--libnetdata/simple_hashtable.h396
-rw-r--r--libnetdata/simple_pattern/Makefile.am8
-rw-r--r--libnetdata/simple_pattern/README.md47
-rw-r--r--libnetdata/socket/Makefile.am8
-rw-r--r--libnetdata/socket/README.md8
-rw-r--r--libnetdata/socket/socket.c2130
-rw-r--r--libnetdata/socket/socket.h248
-rw-r--r--libnetdata/statistical/Makefile.am8
-rw-r--r--libnetdata/statistical/README.md12
-rw-r--r--libnetdata/storage_number/Makefile.am12
-rw-r--r--libnetdata/storage_number/README.md21
-rw-r--r--libnetdata/storage_number/tests/Makefile.am4
-rw-r--r--libnetdata/string/Makefile.am8
-rw-r--r--libnetdata/string/README.md25
-rw-r--r--libnetdata/string/string.c688
-rw-r--r--libnetdata/string/string.h34
-rw-r--r--libnetdata/tests/Makefile.am4
-rw-r--r--libnetdata/threads/Makefile.am8
-rw-r--r--libnetdata/threads/README.md12
-rw-r--r--libnetdata/threads/threads.c340
-rw-r--r--libnetdata/threads/threads.h87
-rw-r--r--libnetdata/url/Makefile.am8
-rw-r--r--libnetdata/url/README.md14
-rw-r--r--libnetdata/url/url.c299
-rw-r--r--libnetdata/url/url.h31
-rw-r--r--libnetdata/uuid/Makefile.am8
-rw-r--r--libnetdata/uuid/README.md13
-rw-r--r--libnetdata/uuid/uuid.c179
-rw-r--r--libnetdata/uuid/uuid.h29
-rw-r--r--libnetdata/worker_utilization/Makefile.am8
-rw-r--r--libnetdata/worker_utilization/README.md94
-rw-r--r--src/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c (renamed from libnetdata/adaptive_resortable_list/adaptive_resortable_list.c)0
-rw-r--r--src/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h (renamed from libnetdata/adaptive_resortable_list/adaptive_resortable_list.h)0
-rw-r--r--src/libnetdata/aral/aral.c (renamed from libnetdata/aral/aral.c)25
-rw-r--r--src/libnetdata/aral/aral.h (renamed from libnetdata/aral/aral.h)4
-rw-r--r--src/libnetdata/avl/avl.c (renamed from libnetdata/avl/avl.c)4
-rw-r--r--src/libnetdata/avl/avl.h (renamed from libnetdata/avl/avl.h)0
-rw-r--r--src/libnetdata/buffered_reader/README.md (renamed from libnetdata/buffered_reader/README.md)0
-rw-r--r--src/libnetdata/buffered_reader/buffered_reader.c (renamed from libnetdata/buffered_reader/buffered_reader.c)0
-rw-r--r--src/libnetdata/circular_buffer/circular_buffer.c (renamed from libnetdata/circular_buffer/circular_buffer.c)0
-rw-r--r--src/libnetdata/circular_buffer/circular_buffer.h (renamed from libnetdata/circular_buffer/circular_buffer.h)0
-rw-r--r--src/libnetdata/clocks/clocks.c (renamed from libnetdata/clocks/clocks.c)10
-rw-r--r--src/libnetdata/clocks/clocks.h (renamed from libnetdata/clocks/clocks.h)7
-rw-r--r--src/libnetdata/completion/completion.h (renamed from libnetdata/completion/completion.h)4
-rw-r--r--src/libnetdata/config/appconfig.c (renamed from libnetdata/config/appconfig.c)4
-rw-r--r--src/libnetdata/config/appconfig.h (renamed from libnetdata/config/appconfig.h)0
-rw-r--r--src/libnetdata/datetime/iso8601.c (renamed from libnetdata/datetime/iso8601.c)0
-rw-r--r--src/libnetdata/datetime/iso8601.h (renamed from libnetdata/datetime/iso8601.h)0
-rw-r--r--src/libnetdata/datetime/rfc3339.c (renamed from libnetdata/datetime/rfc3339.c)6
-rw-r--r--src/libnetdata/datetime/rfc3339.h (renamed from libnetdata/datetime/rfc3339.h)0
-rw-r--r--src/libnetdata/datetime/rfc7231.c (renamed from libnetdata/datetime/rfc7231.c)0
-rw-r--r--src/libnetdata/datetime/rfc7231.h (renamed from libnetdata/datetime/rfc7231.h)0
-rw-r--r--src/libnetdata/eval/README.md (renamed from libnetdata/eval/README.md)0
-rw-r--r--src/libnetdata/facets/README.md (renamed from libnetdata/facets/README.md)0
-rw-r--r--src/libnetdata/facets/facets.c (renamed from libnetdata/facets/facets.c)28
-rw-r--r--src/libnetdata/facets/facets.h (renamed from libnetdata/facets/facets.h)0
-rw-r--r--src/libnetdata/functions_evloop/README.md (renamed from libnetdata/functions_evloop/README.md)0
-rw-r--r--src/libnetdata/gorilla/README.md (renamed from libnetdata/gorilla/README.md)0
-rwxr-xr-xsrc/libnetdata/gorilla/benchmark.sh (renamed from libnetdata/gorilla/benchmark.sh)0
-rwxr-xr-xsrc/libnetdata/gorilla/fuzzer.sh (renamed from libnetdata/gorilla/fuzzer.sh)0
-rw-r--r--src/libnetdata/gorilla/gorilla.cc (renamed from libnetdata/gorilla/gorilla.cc)4
-rw-r--r--src/libnetdata/gorilla/gorilla.h (renamed from libnetdata/gorilla/gorilla.h)4
-rw-r--r--src/libnetdata/inlined.h (renamed from libnetdata/inlined.h)125
-rw-r--r--src/libnetdata/json/jsmn.c (renamed from libnetdata/json/jsmn.c)0
-rw-r--r--src/libnetdata/json/jsmn.h (renamed from libnetdata/json/jsmn.h)0
-rw-r--r--src/libnetdata/july/july.c (renamed from libnetdata/july/july.c)0
-rw-r--r--src/libnetdata/july/july.h (renamed from libnetdata/july/july.h)0
-rw-r--r--src/libnetdata/libjudy/src/Judy.h (renamed from libnetdata/libjudy/src/Judy.h)0
-rw-r--r--src/libnetdata/libjudy/src/JudyCommon/JudyMalloc.c (renamed from libnetdata/libjudy/src/JudyCommon/JudyMalloc.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyCommon/JudyPrivate.h (renamed from libnetdata/libjudy/src/JudyCommon/JudyPrivate.h)0
-rw-r--r--src/libnetdata/libjudy/src/JudyCommon/JudyPrivate1L.h (renamed from libnetdata/libjudy/src/JudyCommon/JudyPrivate1L.h)0
-rw-r--r--src/libnetdata/libjudy/src/JudyCommon/JudyPrivateBranch.h (renamed from libnetdata/libjudy/src/JudyCommon/JudyPrivateBranch.h)0
-rw-r--r--src/libnetdata/libjudy/src/JudyHS/JudyHS.c (renamed from libnetdata/libjudy/src/JudyHS/JudyHS.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyL.h (renamed from libnetdata/libjudy/src/JudyL/JudyL.h)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLByCount.c (renamed from libnetdata/libjudy/src/JudyL/JudyLByCount.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLCascade.c (renamed from libnetdata/libjudy/src/JudyL/JudyLCascade.c)1
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLCount.c (renamed from libnetdata/libjudy/src/JudyL/JudyLCount.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLCreateBranch.c (renamed from libnetdata/libjudy/src/JudyL/JudyLCreateBranch.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLDecascade.c (renamed from libnetdata/libjudy/src/JudyL/JudyLDecascade.c)2
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLDel.c (renamed from libnetdata/libjudy/src/JudyL/JudyLDel.c)1
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLFirst.c (renamed from libnetdata/libjudy/src/JudyL/JudyLFirst.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLFreeArray.c (renamed from libnetdata/libjudy/src/JudyL/JudyLFreeArray.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLGet.c (renamed from libnetdata/libjudy/src/JudyL/JudyLGet.c)2
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLIns.c (renamed from libnetdata/libjudy/src/JudyL/JudyLIns.c)1
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLInsArray.c (renamed from libnetdata/libjudy/src/JudyL/JudyLInsArray.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLInsertBranch.c (renamed from libnetdata/libjudy/src/JudyL/JudyLInsertBranch.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLMallocIF.c (renamed from libnetdata/libjudy/src/JudyL/JudyLMallocIF.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLMemActive.c (renamed from libnetdata/libjudy/src/JudyL/JudyLMemActive.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLMemUsed.c (renamed from libnetdata/libjudy/src/JudyL/JudyLMemUsed.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLNext.c (renamed from libnetdata/libjudy/src/JudyL/JudyLNext.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLNextEmpty.c (renamed from libnetdata/libjudy/src/JudyL/JudyLNextEmpty.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLPrev.c (renamed from libnetdata/libjudy/src/JudyL/JudyLPrev.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLPrevEmpty.c (renamed from libnetdata/libjudy/src/JudyL/JudyLPrevEmpty.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/JudyLTables.c (renamed from libnetdata/libjudy/src/JudyL/JudyLTables.c)0
-rw-r--r--src/libnetdata/libjudy/src/JudyL/j__udyLGet.c (renamed from libnetdata/libjudy/src/JudyL/j__udyLGet.c)0
-rw-r--r--src/libnetdata/libnetdata.c (renamed from libnetdata/libnetdata.c)209
-rw-r--r--src/libnetdata/line_splitter/line_splitter.c (renamed from libnetdata/line_splitter/line_splitter.c)9
-rw-r--r--src/libnetdata/line_splitter/line_splitter.h (renamed from libnetdata/line_splitter/line_splitter.h)5
-rw-r--r--src/libnetdata/locks/locks.c (renamed from libnetdata/locks/locks.c)213
-rw-r--r--src/libnetdata/locks/locks.h (renamed from libnetdata/locks/locks.h)19
-rw-r--r--src/libnetdata/log/journal.c (renamed from libnetdata/log/journal.c)16
-rw-r--r--src/libnetdata/log/journal.h (renamed from libnetdata/log/journal.h)0
-rw-r--r--src/libnetdata/log/systemd-cat-native.c (renamed from libnetdata/log/systemd-cat-native.c)10
-rw-r--r--src/libnetdata/log/systemd-cat-native.h (renamed from libnetdata/log/systemd-cat-native.h)0
-rw-r--r--src/libnetdata/log/systemd-cat-native.md (renamed from libnetdata/log/systemd-cat-native.md)0
-rw-r--r--src/libnetdata/onewayalloc/onewayalloc.c (renamed from libnetdata/onewayalloc/onewayalloc.c)2
-rw-r--r--src/libnetdata/onewayalloc/onewayalloc.h (renamed from libnetdata/onewayalloc/onewayalloc.h)0
-rw-r--r--src/libnetdata/popen/popen.c (renamed from libnetdata/popen/popen.c)2
-rw-r--r--src/libnetdata/popen/popen.h (renamed from libnetdata/popen/popen.h)4
-rw-r--r--src/libnetdata/procfile/procfile.c (renamed from libnetdata/procfile/procfile.c)2
-rw-r--r--src/libnetdata/procfile/procfile.h (renamed from libnetdata/procfile/procfile.h)0
-rw-r--r--src/libnetdata/simple_pattern/simple_pattern.c (renamed from libnetdata/simple_pattern/simple_pattern.c)23
-rw-r--r--src/libnetdata/simple_pattern/simple_pattern.h (renamed from libnetdata/simple_pattern/simple_pattern.h)7
-rw-r--r--src/libnetdata/socket/security.c (renamed from libnetdata/socket/security.c)13
-rw-r--r--src/libnetdata/socket/security.h (renamed from libnetdata/socket/security.h)5
-rw-r--r--src/libnetdata/statistical/statistical.c (renamed from libnetdata/statistical/statistical.c)0
-rw-r--r--src/libnetdata/statistical/statistical.h (renamed from libnetdata/statistical/statistical.h)0
-rw-r--r--src/libnetdata/storage_number/storage_number.c (renamed from libnetdata/storage_number/storage_number.c)63
-rw-r--r--src/libnetdata/storage_number/storage_number.h (renamed from libnetdata/storage_number/storage_number.h)12
-rw-r--r--src/libnetdata/storage_number/tests/test_storage_number.c (renamed from libnetdata/storage_number/tests/test_storage_number.c)0
-rw-r--r--src/libnetdata/string/utf8.h (renamed from libnetdata/string/utf8.h)0
-rw-r--r--src/libnetdata/tests/test_str2ld.c (renamed from libnetdata/tests/test_str2ld.c)0
-rw-r--r--src/libnetdata/worker_utilization/worker_utilization.c (renamed from libnetdata/worker_utilization/worker_utilization.c)4
-rw-r--r--src/libnetdata/worker_utilization/worker_utilization.h (renamed from libnetdata/worker_utilization/worker_utilization.h)0
-rw-r--r--src/libnetdata/xxhash.h (renamed from libnetdata/xxhash.h)0
200 files changed, 437 insertions, 23708 deletions
diff --git a/libnetdata/Makefile.am b/libnetdata/Makefile.am
deleted file mode 100644
index 333155aab..000000000
--- a/libnetdata/Makefile.am
+++ /dev/null
@@ -1,45 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-SUBDIRS = \
- adaptive_resortable_list \
- aral \
- avl \
- buffer \
- buffered_reader \
- clocks \
- completion \
- config \
- datetime \
- dictionary \
- ebpf \
- eval \
- facets \
- functions_evloop \
- json \
- july \
- health \
- line_splitter \
- locks \
- log \
- onewayalloc \
- popen \
- procfile \
- simple_pattern \
- socket \
- statistical \
- storage_number \
- string \
- threads \
- url \
- uuid \
- worker_utilization \
- tests \
- $(NULL)
-
-dist_noinst_DATA = \
- README.md \
- gorilla/README.md \
- $(NULL)
diff --git a/libnetdata/README.md b/libnetdata/README.md
deleted file mode 100644
index 1424faf6c..000000000
--- a/libnetdata/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-<!--
-title: "libnetdata"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/README.md
-sidebar_label: "libnetdata"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# libnetdata
-
-`libnetdata` is a collection of library code that is used by all Netdata `C` programs.
-
-
diff --git a/libnetdata/adaptive_resortable_list/Makefile.am b/libnetdata/adaptive_resortable_list/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/adaptive_resortable_list/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/adaptive_resortable_list/README.md b/libnetdata/adaptive_resortable_list/README.md
deleted file mode 100644
index ceed467d6..000000000
--- a/libnetdata/adaptive_resortable_list/README.md
+++ /dev/null
@@ -1,103 +0,0 @@
-<!--
-title: "Adaptive Re-sortable List (ARL)"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/adaptive_resortable_list/README.md
-sidebar_label: "Adaptive Re-sortable List (ARL)"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Adaptive Re-sortable List (ARL)
-
-This library allows Netdata to read a series of `name - value` pairs
-in the **fastest possible way**.
-
-ARLs are used all over Netdata, as they are the most
-CPU utilization efficient way to process `/proc` files. They are used to
-process both vertical (csv like) and horizontal (one pair per line) `name - value` pairs.
-
-## How ARL works
-
-It maintains a linked list of all `NAME` (keywords), sorted in the
-order found in the data source. The linked list is kept
-sorted at all times - the data source may change at any time, the
-linked list will adapt at the next iteration.
-
-### Initialization
-
-During initialization (just once), the caller:
-
-- calls `arl_create()` to create the ARL
-
-- calls `arl_expect()` multiple times to register the expected keywords
-
-The library will call the `processor()` function (given to
-`arl_create()`), for each expected keyword found.
-The default `processor()` expects `dst` to be an `unsigned long long *`.
-
-Each `name` keyword may have a different `processor()` (by calling
-`arl_expect_custom()` instead of `arl_expect()`).
-
-### Data collection iterations
-
-For each iteration through the data source, the caller:
-
-- calls `arl_begin()` to initiate a data collection iteration.
- This is to be called just ONCE every time the source is re-evaluated.
-
-- calls `arl_check()` for each entry read from the file.
-
-### Cleanup
-
-When the caller exits:
-
-- calls `arl_free()` to destroy this and free all memory.
-
-### Performance
-
-ARL maintains a list of `name` keywords found in the data source (even the ones
-that are not useful for data collection).
-
-If the data source maintains the same order on the `name-value` pairs, for each
-each call to `arl_check()` only an `strcmp()` is executed to verify the
-expected order has not changed, a counter is incremented and a pointer is changed.
-So, if the data source has 100 `name-value` pairs, and their order remains constant
-over time, 100 successful `strcmp()` are executed.
-
-In the unlikely event that an iteration sees the data source with a different order,
-for each out-of-order keyword, a full search of the remaining keywords is made. But
-this search uses 32bit hashes, not string comparisons, so it should also be fast.
-
-When all expectations are satisfied (even in the middle of an iteration),
-the call to `arl_check()` will return 1, to signal the caller to stop the loop,
-saving valuable CPU resources for the rest of the data source.
-
-In the following test we used alternative methods to process, **1M times**,
-a data source like `/proc/meminfo`, already tokenized, in memory,
-to extract the same number of expected metrics:
-
-|test|code|string comparison|number parsing|duration|
-|:--:|:--:|:---------------:|:------------:|:------:|
-|1|if-else-if-else-if|`strcmp()`|`strtoull()`|4630.337 ms|
-|2|nested loops|inline `simple_hash()` and `strcmp()`|`strtoull()`|1597.481 ms|
-|3|nested loops|inline `simple_hash()` and `strcmp()`|`str2ull()`|923.523 ms|
-|4|if-else-if-else-if|inline `simple_hash()` and `strcmp()`|`strtoull()`|854.574 ms|
-|5|if-else-if-else-if|statement expression `simple_hash()` and `strcmp()`|`strtoull()`|912.013 ms|
-|6|if-continue|inline `simple_hash()` and `strcmp()`|`strtoull()`|842.279 ms|
-|7|if-else-if-else-if|inline `simple_hash()` and `strcmp()`|`str2ull()`|602.837 ms|
-|8|ARL|ARL|`strtoull()`|350.360 ms|
-|9|ARL|ARL|`str2ull()`|157.237 ms|
-
-Compared to unoptimized code (test No 1: 4.6sec):
-
-- before ARL Netdata was using test No **7** with hashing and a custom `str2ull()` to achieve 602ms.
-- the current ARL implementation is test No **9** that needs only 157ms (29 times faster vs unoptimized code, about 4 times faster vs optimized code).
-
-[Check the source code of this test](https://raw.githubusercontent.com/netdata/netdata/master/tests/profile/benchmark-value-pairs.c).
-
-## Limitations
-
-Do not use ARL if the a name/keyword may appear more than once in the
-source data.
-
-
diff --git a/libnetdata/aral/Makefile.am b/libnetdata/aral/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/aral/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/aral/README.md b/libnetdata/aral/README.md
deleted file mode 100644
index e556144ba..000000000
--- a/libnetdata/aral/README.md
+++ /dev/null
@@ -1,173 +0,0 @@
-<!--
-title: "Array Allocator"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/aral/README.md
-sidebar_label: "Array allocator"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Array Allocator
-
-Come on! Array allocators are embedded in libc! Why do we need such a thing in Netdata?
-
-Well, we have a couple of problems to solve:
-
-1. **Fragmentation** - It is important for Netdata to keeps its overall memory footprint as low as possible. libc does an amazing job when the same thread allocates and frees some memory. But it simply cannot do better without knowing the specifics of the application when memory is allocated and freed randomly between threads.
-2. **Speed** - Especially when allocations and de-allocations happen across threads, the speed penalty is tremendous.
-
-In Netdata we have a few moments that are very tough. Imagine collecting 1 million metrics per second. You have a buffer for each metric and put append new points there. This works beautifully, of course! But then, when the buffers get full, imagine the situation. You suddenly need 1 million buffers, at once!
-
-To solve this problem we first spread out the buffers. So, the first time each metric asks for a buffer, it gets a smaller one. We added logic there to spread them as evenly as possible across time. Solved? Not exactly!
-
-We have 3 tiers for each metric. For the metrics of tier 0 (per second resolution) we have a max buffer for 1024 points and every new metrics gets a random size between 3 points and 1024. So they are distributed across time. For 1 million metrics, we have about 1000 buffers beings created every second.
-
-But at some point, the end of the minute will come, and suddenly all the metrics will need a new buffer for tier 1 (per minute). Oops! We will spread tier 1 buffers across time too, but the first minute is a tough one. We really need 1 million buffers instantly.
-
-And if that minute happens to also be the beginning of an hour... tier 2 (per hour) kicks in. For that instant we are going to need 2 million buffers instantly.
-
-The problem becomes even bigger when we collect 2, or even 10 million metrics...
-
-So solve it, Netdata uses a special implementation of an array allocator that is tightly integrated with the structures we need.
-
-## Features
-
-1. Malloc, or MMAP modes. File based MMAP is also supported to put the data in file backed up shared memory.
-2. Fully asynchronous operations. There are just a couple of points where spin-locks protect a few counters and pointers.
-3. Optional defragmenter, that once enabled it will make free operation slower while trying to maintain a sorted list of fragments to offer first during allocations. The defragmenter can be enabled / disabled at run time. The defragmenter can hurt performance on application with intense turn-around of allocation, like Netdata dbengine caches. So, it is disabled by default.
-4. Without the defragmenter enabled, ARAL still tries to keep pages full, but the depth of the search is limited to 3 pages (so, a page with a free slot will either become 1st, 2nd, or 3rd). At the same time, during allocations, ARAL will evaluate the first 2 pages to find the one that is more full than the other, to use it for the new allocation.
-
-## How it works
-
-Allocations are organized in pages. Pages have a minimum size (a system page, usually 4KB) and a maximum defined by for each different kind of object.
-
-Initially every page is free. When an allocation request is made, the free space is split, and the first element is reserved. Free space is now considered there rest.
-
-This continuous until the page gets full, where a new page is allocated and the process is repeated.
-
-Each allocation returned has a pointer appended to it. The pointer points to the page the allocation belongs to.
-
-When a pointer is freed, the page it belongs is identified, its space is marked free, and it is prepended in a single linked list that resides in the page itself. So, each page has its own list of free slots to use.
-
-Pages are then on another linked list. This is a double linked list and at its beginning has the pages with free space and at the end the pages that are full.
-
-When the defragmenter is enabled the pages double linked list is also sorted, like this: the fewer the free slots on a page, the earlier in the linked list the page will be, except if it does not have any free slot, in which case it will be at the end. So, the defragmenter tries to have pages full.
-
-When a page is entirerly free, it is given back to the system immediately. There is no caching of free pages.
-
-
-Parallelism is achieved like this:
-
-When some threads are waiting for a page to be allocated, free operations are allowed. If a free operation happens before a new page is allocated, any waiting thread will get the slot that is freed on another page.
-
-Free operations happen in parallel, even for the same page. There is a spin-lock on each page to protect the base pointer of the page's free slots single linked list. But, this is instant. All preparative work happens lockless, then to add the free slot to the page, the page spinlock is acquired, the free slot is prepended to the linked list on the page, the spinlock is released. Such free operations on different pages are totally parallel.
-
-Once the free operation on a page has finished, the pages double linked list spinlock is acquired to put the page first on that linked list. If the defragmenter is enabled, the spinlock is retained for a little longer, to find the exact position of the page in the linked list.
-
-During allocations, the reverse order is used. First get the pages double linked list spinlock, get the first page and decrement its free slots counter, then release the spinlock. If the first page does not have any free slots, a page allocation is spawn, without any locks acquired. All threads are spinning waiting for a page with free slots, either from the newly allocated one or from a free operation that may happen in parallel.
-
-Once a page is acquired, each thread locks its own page to get the first free slot and releases the lock immediately. This is guaranteed to succeed, because when the page was given to that thread its free slots counter was decremented. So, there is a free slot for every thread that got that page. All preparative work to return a pointer to the caller is done lock free. Allocations on different pages are done in parallel, without any intervention between them.
-
-
-## What to expect
-
-Systems not designed for parallelism achieve their top performance single threaded. The single threaded speed is the baseline. Adding more threads makes them slower.
-
-The baseline for ARAL is the following, the included stress test when running single threaded:
-
-```
-Running stress test of 1 threads, with 10000 elements each, for 5 seconds...
-2023-01-29 17:04:50: netdata INFO : TH[0] : set name of thread 1314983 to TH[0]
-ARAL executes 12.27 M malloc and 12.26 M free operations/s
-ARAL executes 12.29 M malloc and 12.29 M free operations/s
-ARAL executes 12.30 M malloc and 12.30 M free operations/s
-ARAL executes 12.30 M malloc and 12.29 M free operations/s
-ARAL executes 12.29 M malloc and 12.29 M free operations/s
-Waiting the threads to finish...
-2023-01-29 17:04:55: netdata INFO : MAIN : ARAL: did 61487356 malloc, 61487356 free, using 1 threads, in 5003808 usecs
-```
-
-The same test with 2 threads, both threads on the same ARAL of course. As you see performance improved:
-
-```
-Running stress test of 2 threads, with 10000 elements each, for 5 seconds...
-2023-01-29 17:05:25: netdata INFO : TH[0] : set name of thread 1315537 to TH[0]
-2023-01-29 17:05:25: netdata INFO : TH[1] : set name of thread 1315538 to TH[1]
-ARAL executes 17.75 M malloc and 17.73 M free operations/s
-ARAL executes 17.93 M malloc and 17.93 M free operations/s
-ARAL executes 18.17 M malloc and 18.18 M free operations/s
-ARAL executes 18.33 M malloc and 18.32 M free operations/s
-ARAL executes 18.36 M malloc and 18.36 M free operations/s
-Waiting the threads to finish...
-2023-01-29 17:05:30: netdata INFO : MAIN : ARAL: did 90976190 malloc, 90976190 free, using 2 threads, in 5029462 usecs
-```
-
-The same test with 4 threads:
-
-```
-Running stress test of 4 threads, with 10000 elements each, for 5 seconds...
-2023-01-29 17:10:12: netdata INFO : TH[0] : set name of thread 1319552 to TH[0]
-2023-01-29 17:10:12: netdata INFO : TH[1] : set name of thread 1319553 to TH[1]
-2023-01-29 17:10:12: netdata INFO : TH[2] : set name of thread 1319554 to TH[2]
-2023-01-29 17:10:12: netdata INFO : TH[3] : set name of thread 1319555 to TH[3]
-ARAL executes 19.95 M malloc and 19.91 M free operations/s
-ARAL executes 20.08 M malloc and 20.08 M free operations/s
-ARAL executes 20.85 M malloc and 20.85 M free operations/s
-ARAL executes 20.84 M malloc and 20.84 M free operations/s
-ARAL executes 21.37 M malloc and 21.37 M free operations/s
-Waiting the threads to finish...
-2023-01-29 17:10:17: netdata INFO : MAIN : ARAL: did 103549747 malloc, 103549747 free, using 4 threads, in 5023325 usecs
-```
-
-The same with 8 threads:
-
-```
-Running stress test of 8 threads, with 10000 elements each, for 5 seconds...
-2023-01-29 17:07:06: netdata INFO : TH[0] : set name of thread 1317608 to TH[0]
-2023-01-29 17:07:06: netdata INFO : TH[1] : set name of thread 1317609 to TH[1]
-2023-01-29 17:07:06: netdata INFO : TH[2] : set name of thread 1317610 to TH[2]
-2023-01-29 17:07:06: netdata INFO : TH[3] : set name of thread 1317611 to TH[3]
-2023-01-29 17:07:06: netdata INFO : TH[4] : set name of thread 1317612 to TH[4]
-2023-01-29 17:07:06: netdata INFO : TH[5] : set name of thread 1317613 to TH[5]
-2023-01-29 17:07:06: netdata INFO : TH[6] : set name of thread 1317614 to TH[6]
-2023-01-29 17:07:06: netdata INFO : TH[7] : set name of thread 1317615 to TH[7]
-ARAL executes 15.73 M malloc and 15.66 M free operations/s
-ARAL executes 13.95 M malloc and 13.94 M free operations/s
-ARAL executes 15.59 M malloc and 15.58 M free operations/s
-ARAL executes 15.49 M malloc and 15.49 M free operations/s
-ARAL executes 16.16 M malloc and 16.16 M free operations/s
-Waiting the threads to finish...
-2023-01-29 17:07:11: netdata INFO : MAIN : ARAL: did 78427750 malloc, 78427750 free, using 8 threads, in 5088591 usecs
-```
-
-The same with 16 threads:
-
-```
-Running stress test of 16 threads, with 10000 elements each, for 5 seconds...
-2023-01-29 17:08:04: netdata INFO : TH[0] : set name of thread 1318663 to TH[0]
-2023-01-29 17:08:04: netdata INFO : TH[1] : set name of thread 1318664 to TH[1]
-2023-01-29 17:08:04: netdata INFO : TH[2] : set name of thread 1318665 to TH[2]
-2023-01-29 17:08:04: netdata INFO : TH[3] : set name of thread 1318666 to TH[3]
-2023-01-29 17:08:04: netdata INFO : TH[4] : set name of thread 1318667 to TH[4]
-2023-01-29 17:08:04: netdata INFO : TH[5] : set name of thread 1318668 to TH[5]
-2023-01-29 17:08:04: netdata INFO : TH[6] : set name of thread 1318669 to TH[6]
-2023-01-29 17:08:04: netdata INFO : TH[7] : set name of thread 1318670 to TH[7]
-2023-01-29 17:08:04: netdata INFO : TH[8] : set name of thread 1318671 to TH[8]
-2023-01-29 17:08:04: netdata INFO : TH[9] : set name of thread 1318672 to TH[9]
-2023-01-29 17:08:04: netdata INFO : TH[10] : set name of thread 1318673 to TH[10]
-2023-01-29 17:08:04: netdata INFO : TH[11] : set name of thread 1318674 to TH[11]
-2023-01-29 17:08:04: netdata INFO : TH[12] : set name of thread 1318675 to TH[12]
-2023-01-29 17:08:04: netdata INFO : TH[13] : set name of thread 1318676 to TH[13]
-2023-01-29 17:08:04: netdata INFO : TH[14] : set name of thread 1318677 to TH[14]
-2023-01-29 17:08:04: netdata INFO : TH[15] : set name of thread 1318678 to TH[15]
-ARAL executes 11.77 M malloc and 11.62 M free operations/s
-ARAL executes 12.80 M malloc and 12.81 M free operations/s
-ARAL executes 13.26 M malloc and 13.25 M free operations/s
-ARAL executes 13.30 M malloc and 13.29 M free operations/s
-ARAL executes 13.23 M malloc and 13.25 M free operations/s
-Waiting the threads to finish...
-2023-01-29 17:08:09: netdata INFO : MAIN : ARAL: did 65302122 malloc, 65302122 free, using 16 threads, in 5066009 usecs
-```
-
-As you can see, the top performance is with 4 threads, almost double the single thread speed.
-16 threads performance is still better than single threaded, despite the intense concurrency.
diff --git a/libnetdata/avl/Makefile.am b/libnetdata/avl/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/avl/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/avl/README.md b/libnetdata/avl/README.md
deleted file mode 100644
index 94c0f634f..000000000
--- a/libnetdata/avl/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-<!--
-title: "AVL"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/avl/README.md
-sidebar_label: "AVL"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# AVL
-
-AVL is a library indexing objects in B-Trees.
-
-`avl_insert()`, `avl_remove()` and `avl_search()` are adaptations
-of the AVL algorithm found in `libavl` v2.0.3, so that they do not
-use any memory allocations and their memory footprint is optimized
-(by eliminating non-necessary data members).
-
-In addition to the above, this version of AVL, provides versions using locks
-and traversal functions.
-
diff --git a/libnetdata/buffer/Makefile.am b/libnetdata/buffer/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/buffer/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/buffer/README.md b/libnetdata/buffer/README.md
deleted file mode 100644
index 2937ae141..000000000
--- a/libnetdata/buffer/README.md
+++ /dev/null
@@ -1,20 +0,0 @@
-<!--
-title: "BUFFER"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/buffer/README.md
-sidebar_label: "BUFFER library"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# BUFFER
-
-`BUFFER` is a convenience library for working with strings in `C`.
-Mainly, `BUFFER`s eliminate the need for tracking the string length, thus providing
-a safe alternative for string operations.
-
-Also, they are super fast in printing and appending data to the string and its `buffer_strlen()`
-is just a lookup (it does not traverse the string).
-
-Netdata uses `BUFFER`s for preparing web responses and buffering data to be sent upstream or
-to external databases.
diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c
deleted file mode 100644
index 64f9cce47..000000000
--- a/libnetdata/buffer/buffer.c
+++ /dev/null
@@ -1,503 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "../libnetdata.h"
-
-static inline void buffer_overflow_init(BUFFER *b)
-{
- b->buffer[b->size] = '\0';
- strcpy(&b->buffer[b->size + 1], BUFFER_OVERFLOW_EOF);
-}
-
-void buffer_reset(BUFFER *wb) {
- buffer_flush(wb);
-
- wb->content_type = CT_TEXT_PLAIN;
- wb->options = 0;
- wb->date = 0;
- wb->expires = 0;
- buffer_no_cacheable(wb);
-
- buffer_overflow_check(wb);
-}
-
-void buffer_char_replace(BUFFER *wb, char from, char to) {
- char *s = wb->buffer, *end = &wb->buffer[wb->len];
-
- while(s != end) {
- if(*s == from) *s = to;
- s++;
- }
-
- buffer_overflow_check(wb);
-}
-
-void buffer_print_sn_flags(BUFFER *wb, SN_FLAGS flags, bool send_anomaly_bit) {
- if(unlikely(flags == SN_EMPTY_SLOT)) {
- buffer_fast_strcat(wb, "E", 1);
- return;
- }
-
- size_t printed = 0;
- if(likely(send_anomaly_bit && (flags & SN_FLAG_NOT_ANOMALOUS))) {
- buffer_fast_strcat(wb, "A", 1);
- printed++;
- }
-
- if(unlikely(flags & SN_FLAG_RESET)) {
- buffer_fast_strcat(wb, "R", 1);
- printed++;
- }
-
- if(!printed)
- buffer_fast_strcat(wb, "''", 2);
-}
-
-void buffer_strcat_htmlescape(BUFFER *wb, const char *txt)
-{
- while(*txt) {
- switch(*txt) {
- case '&': buffer_strcat(wb, "&amp;"); break;
- case '<': buffer_strcat(wb, "&lt;"); break;
- case '>': buffer_strcat(wb, "&gt;"); break;
- case '"': buffer_strcat(wb, "&quot;"); break;
- case '/': buffer_strcat(wb, "&#x2F;"); break;
- case '\'': buffer_strcat(wb, "&#x27;"); break;
- default: {
- buffer_need_bytes(wb, 1);
- wb->buffer[wb->len++] = *txt;
- }
- }
- txt++;
- }
-
- buffer_overflow_check(wb);
-}
-
-void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...)
-{
- if(unlikely(!fmt || !*fmt)) return;
-
- buffer_need_bytes(wb, len + 1);
-
- va_list args;
- va_start(args, fmt);
- // vsnprintfz() returns the number of bytes actually written - after possible truncation
- wb->len += vsnprintfz(&wb->buffer[wb->len], len, fmt, args);
- va_end(args);
-
- buffer_overflow_check(wb);
-
- // the buffer is \0 terminated by vsnprintfz
-}
-
-inline void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args) {
- if(unlikely(!fmt || !*fmt)) return;
-
- size_t full_size_bytes = 0, need = 2, space_remaining = 0;
-
- do {
- need += full_size_bytes + 2;
-
- buffer_need_bytes(wb, need);
-
- space_remaining = wb->size - wb->len - 1;
-
- // Use the copy of va_list for vsnprintf
- va_list args_copy;
- va_copy(args_copy, args);
- // vsnprintf() returns the number of bytes required, even if bigger than the buffer provided
- full_size_bytes = (size_t) vsnprintf(&wb->buffer[wb->len], space_remaining, fmt, args_copy);
- va_end(args_copy);
-
- } while(full_size_bytes >= space_remaining);
-
- wb->len += full_size_bytes;
-
- wb->buffer[wb->len] = '\0';
- buffer_overflow_check(wb);
-}
-
-void buffer_sprintf(BUFFER *wb, const char *fmt, ...)
-{
- va_list args;
- va_start(args, fmt);
- buffer_vsprintf(wb, fmt, args);
- va_end(args);
-}
-
-// generate a javascript date, the fastest possible way...
-void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds)
-{
- // 10 20 30 = 35
- // 01234567890123456789012345678901234
- // Date(2014,04,01,03,28,20)
-
- buffer_need_bytes(wb, 30);
-
- char *b = &wb->buffer[wb->len], *p;
- unsigned int *q = (unsigned int *)b;
-
- #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- *q++ = 0x65746144; // "Date" backwards.
- #else
- *q++ = 0x44617465; // "Date"
- #endif
- p = (char *)q;
-
- *p++ = '(';
- *p++ = '0' + year / 1000; year %= 1000;
- *p++ = '0' + year / 100; year %= 100;
- *p++ = '0' + year / 10;
- *p++ = '0' + year % 10;
- *p++ = ',';
- *p = '0' + month / 10; if (*p != '0') p++;
- *p++ = '0' + month % 10;
- *p++ = ',';
- *p = '0' + day / 10; if (*p != '0') p++;
- *p++ = '0' + day % 10;
- *p++ = ',';
- *p = '0' + hours / 10; if (*p != '0') p++;
- *p++ = '0' + hours % 10;
- *p++ = ',';
- *p = '0' + minutes / 10; if (*p != '0') p++;
- *p++ = '0' + minutes % 10;
- *p++ = ',';
- *p = '0' + seconds / 10; if (*p != '0') p++;
- *p++ = '0' + seconds % 10;
-
- unsigned short *r = (unsigned short *)p;
-
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
- *r++ = 0x0029; // ")\0" backwards.
- #else
- *r++ = 0x2900; // ")\0"
- #endif
-
- wb->len += (size_t)((char *)r - b - 1);
-
- // terminate it
- wb->buffer[wb->len] = '\0';
- buffer_overflow_check(wb);
-}
-
-// generate a date, the fastest possible way...
-void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds)
-{
- // 10 20 30 = 35
- // 01234567890123456789012345678901234
- // 2014-04-01 03:28:20
-
- buffer_need_bytes(wb, 36);
-
- char *b = &wb->buffer[wb->len];
- char *p = b;
-
- *p++ = '0' + year / 1000; year %= 1000;
- *p++ = '0' + year / 100; year %= 100;
- *p++ = '0' + year / 10;
- *p++ = '0' + year % 10;
- *p++ = '-';
- *p++ = '0' + month / 10;
- *p++ = '0' + month % 10;
- *p++ = '-';
- *p++ = '0' + day / 10;
- *p++ = '0' + day % 10;
- *p++ = ' ';
- *p++ = '0' + hours / 10;
- *p++ = '0' + hours % 10;
- *p++ = ':';
- *p++ = '0' + minutes / 10;
- *p++ = '0' + minutes % 10;
- *p++ = ':';
- *p++ = '0' + seconds / 10;
- *p++ = '0' + seconds % 10;
- *p = '\0';
-
- wb->len += (size_t)(p - b);
-
- // terminate it
- wb->buffer[wb->len] = '\0';
- buffer_overflow_check(wb);
-}
-
-BUFFER *buffer_create(size_t size, size_t *statistics)
-{
- BUFFER *b;
-
- netdata_log_debug(D_WEB_BUFFER, "Creating new web buffer of size %zu.", size);
-
- b = callocz(1, sizeof(BUFFER));
- b->buffer = mallocz(size + sizeof(BUFFER_OVERFLOW_EOF) + 2);
- b->buffer[0] = '\0';
- b->size = size;
- b->content_type = CT_TEXT_PLAIN;
- b->statistics = statistics;
- buffer_no_cacheable(b);
- buffer_overflow_init(b);
- buffer_overflow_check(b);
-
- if(b->statistics)
- __atomic_add_fetch(b->statistics, b->size + sizeof(BUFFER) + sizeof(BUFFER_OVERFLOW_EOF) + 2, __ATOMIC_RELAXED);
-
- return(b);
-}
-
-void buffer_free(BUFFER *b) {
- if(unlikely(!b)) return;
-
- buffer_overflow_check(b);
-
- netdata_log_debug(D_WEB_BUFFER, "Freeing web buffer of size %zu.", b->size);
-
- if(b->statistics)
- __atomic_sub_fetch(b->statistics, b->size + sizeof(BUFFER) + sizeof(BUFFER_OVERFLOW_EOF) + 2, __ATOMIC_RELAXED);
-
- freez(b->buffer);
- freez(b);
-}
-
-void buffer_increase(BUFFER *b, size_t free_size_required) {
- buffer_overflow_check(b);
-
- size_t left = b->size - b->len;
- if(left >= free_size_required) return;
-
- size_t wanted = free_size_required - left;
- size_t minimum = WEB_DATA_LENGTH_INCREASE_STEP;
- if(minimum > wanted) wanted = minimum;
-
- size_t optimal = (b->size > 5*1024*1024) ? b->size / 2 : b->size;
- if(optimal > wanted) wanted = optimal;
-
- netdata_log_debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + wanted);
-
- b->buffer = reallocz(b->buffer, b->size + wanted + sizeof(BUFFER_OVERFLOW_EOF) + 2);
- b->size += wanted;
-
- if(b->statistics)
- __atomic_add_fetch(b->statistics, wanted, __ATOMIC_RELAXED);
-
- buffer_overflow_init(b);
- buffer_overflow_check(b);
-}
-
-// ----------------------------------------------------------------------------
-
-void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth,
- bool add_anonymous_object, BUFFER_JSON_OPTIONS options) {
- strncpyz(wb->json.key_quote, key_quote, BUFFER_QUOTE_MAX_SIZE);
- strncpyz(wb->json.value_quote, value_quote, BUFFER_QUOTE_MAX_SIZE);
-
- wb->json.depth = (int8_t)(depth - 1);
- _buffer_json_depth_push(wb, BUFFER_JSON_OBJECT);
-
- if(add_anonymous_object)
- buffer_fast_strcat(wb, "{", 1);
- else
- options |= BUFFER_JSON_OPTIONS_NON_ANONYMOUS;
-
- wb->json.options = options;
-
- wb->content_type = CT_APPLICATION_JSON;
- buffer_no_cacheable(wb);
-}
-
-void buffer_json_finalize(BUFFER *wb) {
- while(wb->json.depth >= 0) {
- switch(wb->json.stack[wb->json.depth].type) {
- case BUFFER_JSON_OBJECT:
- if (wb->json.depth == 0)
- if (!(wb->json.options & BUFFER_JSON_OPTIONS_NON_ANONYMOUS))
- buffer_json_object_close(wb);
- else
- _buffer_json_depth_pop(wb);
- else
- buffer_json_object_close(wb);
- break;
- case BUFFER_JSON_ARRAY:
- buffer_json_array_close(wb);
- break;
-
- default:
- internal_fatal(true, "BUFFER: unknown json member type in stack");
- break;
- }
- }
-
- if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY))
- buffer_fast_strcat(wb, "\n", 1);
-}
-
-// ----------------------------------------------------------------------------
-
-const char hex_digits[16] = "0123456789ABCDEF";
-const char base64_digits[64] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-unsigned char hex_value_from_ascii[256];
-unsigned char base64_value_from_ascii[256];
-
-__attribute__((constructor)) void initialize_ascii_maps(void) {
- for(size_t i = 0 ; i < 256 ; i++) {
- hex_value_from_ascii[i] = 255;
- base64_value_from_ascii[i] = 255;
- }
-
- for(size_t i = 0; i < 16 ; i++)
- hex_value_from_ascii[(int)hex_digits[i]] = i;
-
- for(size_t i = 0; i < 64 ; i++)
- base64_value_from_ascii[(int)base64_digits[i]] = i;
-}
-
-// ----------------------------------------------------------------------------
-// unit test
-
-static int buffer_expect(BUFFER *wb, const char *expected) {
- const char *generated = buffer_tostring(wb);
-
- if(strcmp(generated, expected) != 0) {
- netdata_log_error("BUFFER: mismatch.\nGenerated:\n%s\nExpected:\n%s\n",
- generated, expected);
- return 1;
- }
-
- return 0;
-}
-
-static int buffer_uint64_roundtrip(BUFFER *wb, NUMBER_ENCODING encoding, uint64_t value, const char *expected) {
- int errors = 0;
- buffer_flush(wb);
- buffer_print_uint64_encoded(wb, encoding, value);
-
- if(expected)
- errors += buffer_expect(wb, expected);
-
- uint64_t v = str2ull_encoded(buffer_tostring(wb));
- if(v != value) {
- netdata_log_error("BUFFER: string '%s' does resolves to %llu, expected %llu",
- buffer_tostring(wb), (unsigned long long)v, (unsigned long long)value);
- errors++;
- }
- buffer_flush(wb);
- return errors;
-}
-
-static int buffer_int64_roundtrip(BUFFER *wb, NUMBER_ENCODING encoding, int64_t value, const char *expected) {
- int errors = 0;
- buffer_flush(wb);
- buffer_print_int64_encoded(wb, encoding, value);
-
- if(expected)
- errors += buffer_expect(wb, expected);
-
- int64_t v = str2ll_encoded(buffer_tostring(wb));
- if(v != value) {
- netdata_log_error("BUFFER: string '%s' does resolves to %lld, expected %lld",
- buffer_tostring(wb), (long long)v, (long long)value);
- errors++;
- }
- buffer_flush(wb);
- return errors;
-}
-
-static int buffer_double_roundtrip(BUFFER *wb, NUMBER_ENCODING encoding, NETDATA_DOUBLE value, const char *expected) {
- int errors = 0;
- buffer_flush(wb);
- buffer_print_netdata_double_encoded(wb, encoding, value);
-
- if(expected)
- errors += buffer_expect(wb, expected);
-
- NETDATA_DOUBLE v = str2ndd_encoded(buffer_tostring(wb), NULL);
- if(v != value) {
- netdata_log_error("BUFFER: string '%s' does resolves to %.12f, expected %.12f",
- buffer_tostring(wb), v, value);
- errors++;
- }
- buffer_flush(wb);
- return errors;
-}
-
-int buffer_unittest(void) {
- int errors = 0;
- BUFFER *wb = buffer_create(0, NULL);
-
- buffer_uint64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 0, "0");
- buffer_uint64_roundtrip(wb, NUMBER_ENCODING_HEX, 0, "0x0");
- buffer_uint64_roundtrip(wb, NUMBER_ENCODING_BASE64, 0, "#A");
-
- buffer_uint64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 1676071986, "1676071986");
- buffer_uint64_roundtrip(wb, NUMBER_ENCODING_HEX, 1676071986, "0x63E6D432");
- buffer_uint64_roundtrip(wb, NUMBER_ENCODING_BASE64, 1676071986, "#Bj5tQy");
-
- buffer_uint64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 18446744073709551615ULL, "18446744073709551615");
- buffer_uint64_roundtrip(wb, NUMBER_ENCODING_HEX, 18446744073709551615ULL, "0xFFFFFFFFFFFFFFFF");
- buffer_uint64_roundtrip(wb, NUMBER_ENCODING_BASE64, 18446744073709551615ULL, "#P//////////");
-
- buffer_int64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 0, "0");
- buffer_int64_roundtrip(wb, NUMBER_ENCODING_HEX, 0, "0x0");
- buffer_int64_roundtrip(wb, NUMBER_ENCODING_BASE64, 0, "#A");
-
- buffer_int64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, -1676071986, "-1676071986");
- buffer_int64_roundtrip(wb, NUMBER_ENCODING_HEX, -1676071986, "-0x63E6D432");
- buffer_int64_roundtrip(wb, NUMBER_ENCODING_BASE64, -1676071986, "-#Bj5tQy");
-
- buffer_int64_roundtrip(wb, NUMBER_ENCODING_DECIMAL, (int64_t)-9223372036854775807ULL, "-9223372036854775807");
- buffer_int64_roundtrip(wb, NUMBER_ENCODING_HEX, (int64_t)-9223372036854775807ULL, "-0x7FFFFFFFFFFFFFFF");
- buffer_int64_roundtrip(wb, NUMBER_ENCODING_BASE64, (int64_t)-9223372036854775807ULL, "-#H//////////");
-
- buffer_double_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 0, "0");
- buffer_double_roundtrip(wb, NUMBER_ENCODING_HEX, 0, "%0");
- buffer_double_roundtrip(wb, NUMBER_ENCODING_BASE64, 0, "@A");
-
- buffer_double_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 1.5, "1.5");
- buffer_double_roundtrip(wb, NUMBER_ENCODING_HEX, 1.5, "%3FF8000000000000");
- buffer_double_roundtrip(wb, NUMBER_ENCODING_BASE64, 1.5, "@D/4AAAAAAAA");
-
- buffer_double_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 1.23e+14, "123000000000000");
- buffer_double_roundtrip(wb, NUMBER_ENCODING_HEX, 1.23e+14, "%42DBF78AD3AC0000");
- buffer_double_roundtrip(wb, NUMBER_ENCODING_BASE64, 1.23e+14, "@ELb94rTrAAA");
-
- buffer_double_roundtrip(wb, NUMBER_ENCODING_DECIMAL, 9.12345678901234567890123456789e+45, "9.123456789012346128e+45");
- buffer_double_roundtrip(wb, NUMBER_ENCODING_HEX, 9.12345678901234567890123456789e+45, "%497991C25C9E4309");
- buffer_double_roundtrip(wb, NUMBER_ENCODING_BASE64, 9.12345678901234567890123456789e+45, "@El5kcJcnkMJ");
-
- buffer_flush(wb);
-
- {
- char buf[1024 + 1];
- for(size_t i = 0; i < 1024 ;i++)
- buf[i] = (char)(i % 26) + 'A';
- buf[1024] = '\0';
-
- buffer_strcat(wb, buf);
- errors += buffer_expect(wb, buf);
- }
-
- buffer_flush(wb);
-
- buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
- buffer_json_finalize(wb);
- errors += buffer_expect(wb, "{\n}\n");
-
- buffer_flush(wb);
-
- buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
- buffer_json_member_add_string(wb, "hello", "world");
- buffer_json_member_add_string(wb, "alpha", "this: \" is a double quote");
- buffer_json_member_add_object(wb, "object1");
- buffer_json_member_add_string(wb, "hello", "world");
- buffer_json_finalize(wb);
- errors += buffer_expect(wb, "{\n \"hello\":\"world\",\n \"alpha\":\"this: \\\" is a double quote\",\n \"object1\":{\n \"hello\":\"world\"\n }\n}\n");
-
- buffer_free(wb);
- return errors;
-}
-
-#ifdef ENABLE_H2O
-h2o_iovec_t buffer_to_h2o_iovec(BUFFER *wb) {
- h2o_iovec_t ret;
- ret.base = wb->buffer;
- ret.len = wb->len;
- return ret;
-}
-#endif
diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h
deleted file mode 100644
index 88d3f0282..000000000
--- a/libnetdata/buffer/buffer.h
+++ /dev/null
@@ -1,1228 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_WEB_BUFFER_H
-#define NETDATA_WEB_BUFFER_H 1
-
-#include "../string/utf8.h"
-#include "../libnetdata.h"
-
-#ifdef ENABLE_H2O
-#include "h2o/memory.h"
-#endif
-
-#define WEB_DATA_LENGTH_INCREASE_STEP 1024
-
-#define BUFFER_JSON_MAX_DEPTH 32 // max is 255
-
-extern const char hex_digits[16];
-extern const char base64_digits[64];
-extern unsigned char hex_value_from_ascii[256];
-extern unsigned char base64_value_from_ascii[256];
-
-typedef enum __attribute__ ((__packed__)) {
- BUFFER_JSON_EMPTY = 0,
- BUFFER_JSON_OBJECT,
- BUFFER_JSON_ARRAY,
-} BUFFER_JSON_NODE_TYPE;
-
-typedef struct web_buffer_json_node {
- BUFFER_JSON_NODE_TYPE type;
- uint32_t count:24;
-} BUFFER_JSON_NODE;
-
-#define BUFFER_QUOTE_MAX_SIZE 7
-
-typedef enum __attribute__ ((__packed__)) {
- WB_CONTENT_CACHEABLE = (1 << 0),
- WB_CONTENT_NO_CACHEABLE = (1 << 1),
-} BUFFER_OPTIONS;
-
-typedef enum __attribute__ ((__packed__)) {
- CT_NONE = 0,
- CT_APPLICATION_JSON,
- CT_TEXT_PLAIN,
- CT_TEXT_HTML,
- CT_APPLICATION_X_JAVASCRIPT,
- CT_TEXT_CSS,
- CT_TEXT_XML,
- CT_APPLICATION_XML,
- CT_TEXT_XSL,
- CT_APPLICATION_OCTET_STREAM,
- CT_APPLICATION_X_FONT_TRUETYPE,
- CT_APPLICATION_X_FONT_OPENTYPE,
- CT_APPLICATION_FONT_WOFF,
- CT_APPLICATION_FONT_WOFF2,
- CT_APPLICATION_VND_MS_FONTOBJ,
- CT_IMAGE_SVG_XML,
- CT_IMAGE_PNG,
- CT_IMAGE_JPG,
- CT_IMAGE_GIF,
- CT_IMAGE_XICON,
- CT_IMAGE_ICNS,
- CT_IMAGE_BMP,
- CT_PROMETHEUS,
- CT_AUDIO_MPEG,
- CT_AUDIO_OGG,
- CT_VIDEO_MP4,
- CT_APPLICATION_PDF,
- CT_APPLICATION_ZIP,
-} HTTP_CONTENT_TYPE;
-
-typedef enum __attribute__ ((__packed__)) {
- BUFFER_JSON_OPTIONS_DEFAULT = 0,
- BUFFER_JSON_OPTIONS_MINIFY = (1 << 0),
- BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS = (1 << 1),
- BUFFER_JSON_OPTIONS_NON_ANONYMOUS = (1 << 2),
-} BUFFER_JSON_OPTIONS;
-
-typedef struct web_buffer {
- size_t size; // allocation size of buffer, in bytes
- size_t len; // current data length in buffer, in bytes
- char *buffer; // the buffer itself
- HTTP_CONTENT_TYPE content_type; // the content type of the data in the buffer
- BUFFER_OPTIONS options; // options related to the content
- time_t date; // the timestamp this content has been generated
- time_t expires; // the timestamp this content expires
- size_t *statistics;
-
- struct {
- char key_quote[BUFFER_QUOTE_MAX_SIZE + 1];
- char value_quote[BUFFER_QUOTE_MAX_SIZE + 1];
- int8_t depth;
- BUFFER_JSON_OPTIONS options;
- BUFFER_JSON_NODE stack[BUFFER_JSON_MAX_DEPTH];
- } json;
-} BUFFER;
-
-#define CLEAN_BUFFER _cleanup_(buffer_freep) BUFFER
-
-#define buffer_cacheable(wb) do { (wb)->options |= WB_CONTENT_CACHEABLE; if((wb)->options & WB_CONTENT_NO_CACHEABLE) (wb)->options &= ~WB_CONTENT_NO_CACHEABLE; } while(0)
-#define buffer_no_cacheable(wb) do { (wb)->options |= WB_CONTENT_NO_CACHEABLE; if((wb)->options & WB_CONTENT_CACHEABLE) (wb)->options &= ~WB_CONTENT_CACHEABLE; (wb)->expires = 0; } while(0)
-
-#define buffer_strlen(wb) ((wb)->len)
-
-#define BUFFER_OVERFLOW_EOF "EOF"
-
-#ifdef NETDATA_INTERNAL_CHECKS
-#define buffer_overflow_check(b) _buffer_overflow_check(b)
-#else
-#define buffer_overflow_check(b)
-#endif
-
-static inline void _buffer_overflow_check(BUFFER *b) {
- assert(b->len <= b->size &&
- "BUFFER: length is above buffer size.");
-
- assert(!(b->buffer && (b->buffer[b->size] != '\0' || strcmp(&b->buffer[b->size + 1], BUFFER_OVERFLOW_EOF) != 0)) &&
- "BUFFER: detected overflow.");
-}
-
-static inline void buffer_flush(BUFFER *wb) {
- wb->len = 0;
-
- wb->json.depth = 0;
- wb->json.stack[0].type = BUFFER_JSON_EMPTY;
- wb->json.stack[0].count = 0;
-
- if(wb->buffer)
- wb->buffer[0] = '\0';
-}
-
-void buffer_reset(BUFFER *wb);
-
-void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
-void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds);
-
-BUFFER *buffer_create(size_t size, size_t *statistics);
-void buffer_free(BUFFER *b);
-void buffer_increase(BUFFER *b, size_t free_size_required);
-
-static inline void buffer_freep(BUFFER **bp) {
- if(bp) buffer_free(*bp);
-}
-
-void buffer_snprintf(BUFFER *wb, size_t len, const char *fmt, ...) PRINTFLIKE(3, 4);
-void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args);
-void buffer_sprintf(BUFFER *wb, const char *fmt, ...) PRINTFLIKE(2,3);
-void buffer_strcat_htmlescape(BUFFER *wb, const char *txt);
-
-void buffer_char_replace(BUFFER *wb, char from, char to);
-
-void buffer_print_sn_flags(BUFFER *wb, SN_FLAGS flags, bool send_anomaly_bit);
-
-#ifdef ENABLE_H2O
-h2o_iovec_t buffer_to_h2o_iovec(BUFFER *wb);
-#endif
-
-static inline void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) {
- if(unlikely(buffer->len + needed_free_size >= buffer->size))
- buffer_increase(buffer, needed_free_size + 1);
-}
-
-void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value_quote, int depth,
- bool add_anonymous_object, BUFFER_JSON_OPTIONS options);
-
-void buffer_json_finalize(BUFFER *wb);
-
-static const char *buffer_tostring(BUFFER *wb)
-{
- buffer_need_bytes(wb, 1);
- wb->buffer[wb->len] = '\0';
-
- buffer_overflow_check(wb);
-
- return(wb->buffer);
-}
-
-static inline void _buffer_json_depth_push(BUFFER *wb, BUFFER_JSON_NODE_TYPE type) {
-#ifdef NETDATA_INTERNAL_CHECKS
- assert(wb->json.depth <= BUFFER_JSON_MAX_DEPTH && "BUFFER JSON: max nesting reached");
-#endif
- wb->json.depth++;
- wb->json.stack[wb->json.depth].count = 0;
- wb->json.stack[wb->json.depth].type = type;
-}
-
-static inline void _buffer_json_depth_pop(BUFFER *wb) {
- wb->json.depth--;
-}
-
-static inline void buffer_fast_charcat(BUFFER *wb, const char c) {
-
- buffer_need_bytes(wb, 2);
- *(&wb->buffer[wb->len]) = c;
- wb->len += 1;
- wb->buffer[wb->len] = '\0';
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_fast_rawcat(BUFFER *wb, const char *txt, size_t len) {
- if(unlikely(!txt || !*txt || !len)) return;
-
- buffer_need_bytes(wb, len + 1);
-
- const char *t = txt;
- const char *e = &txt[len];
-
- char *d = &wb->buffer[wb->len];
-
- while(t != e)
- *d++ = *t++;
-
- wb->len += len;
- wb->buffer[wb->len] = '\0';
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_putc(BUFFER *wb, char c) {
- buffer_need_bytes(wb, 2);
- wb->buffer[wb->len++] = c;
- wb->buffer[wb->len] = '\0';
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len) {
- if(unlikely(!txt || !*txt || !len)) return;
-
- buffer_need_bytes(wb, len + 1);
-
- const char *t = txt;
- const char *e = &txt[len];
-
- char *d = &wb->buffer[wb->len];
-
- while(t != e
-#ifdef NETDATA_INTERNAL_CHECKS
- && *t
-#endif
- )
- *d++ = *t++;
-
-#ifdef NETDATA_INTERNAL_CHECKS
- assert(!(t != e && !*t) && "BUFFER: source string is shorter than the length given.");
-#endif
-
- wb->len += len;
- wb->buffer[wb->len] = '\0';
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_strcat(BUFFER *wb, const char *txt) {
- if(unlikely(!txt || !*txt)) return;
-
- const char *t = txt;
- while(*t) {
- buffer_need_bytes(wb, 100);
- char *s = &wb->buffer[wb->len];
- char *d = s;
- const char *e = &wb->buffer[wb->size];
-
- while(*t && d < e)
- *d++ = *t++;
-
- wb->len += d - s;
- }
-
- buffer_need_bytes(wb, 1);
- wb->buffer[wb->len] = '\0';
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_contents_replace(BUFFER *wb, const char *txt, size_t len) {
- wb->len = 0;
- buffer_need_bytes(wb, len + 1);
-
- memcpy(wb->buffer, txt, len);
- wb->len = len;
- wb->buffer[wb->len] = '\0';
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_strncat(BUFFER *wb, const char *txt, size_t len) {
- if(unlikely(!txt || !*txt)) return;
-
- buffer_need_bytes(wb, len + 1);
-
- memcpy(&wb->buffer[wb->len], txt, len);
-
- wb->len += len;
- wb->buffer[wb->len] = '\0';
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_memcat(BUFFER *wb, const void *mem, size_t bytes) {
- if(unlikely(!mem)) return;
-
- buffer_need_bytes(wb, bytes + 1);
-
- memcpy(&wb->buffer[wb->len], mem, bytes);
-
- wb->len += bytes;
- wb->buffer[wb->len] = '\0';
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_json_strcat(BUFFER *wb, const char *txt) {
- if(unlikely(!txt || !*txt)) return;
-
- const unsigned char *t = (const unsigned char *)txt;
- while(*t) {
- buffer_need_bytes(wb, 110);
- unsigned char *s = (unsigned char *)&wb->buffer[wb->len];
- unsigned char *d = s;
- const unsigned char *e = (unsigned char *)&wb->buffer[wb->size - 10]; // make room for the max escape sequence
-
- while(*t && d < e) {
-#ifdef BUFFER_JSON_ESCAPE_UTF
- if(unlikely(IS_UTF8_STARTBYTE(*t) && IS_UTF8_BYTE(t[1]))) {
- // UTF-8 multi-byte encoded character
-
- // find how big this character is (2-4 bytes)
- size_t utf_character_size = 2;
- while(utf_character_size < 4 && t[utf_character_size] && IS_UTF8_BYTE(t[utf_character_size]) && !IS_UTF8_STARTBYTE(t[utf_character_size]))
- utf_character_size++;
-
- uint32_t code_point = 0;
- for (size_t i = 0; i < utf_character_size; i++) {
- code_point <<= 6;
- code_point |= (t[i] & 0x3F);
- }
-
- t += utf_character_size;
-
- // encode as \u escape sequence
- *d++ = '\\';
- *d++ = 'u';
- *d++ = hex_digits[(code_point >> 12) & 0xf];
- *d++ = hex_digits[(code_point >> 8) & 0xf];
- *d++ = hex_digits[(code_point >> 4) & 0xf];
- *d++ = hex_digits[code_point & 0xf];
- }
- else
-#endif
- if(unlikely(*t < ' ')) {
- uint32_t v = *t++;
- *d++ = '\\';
- *d++ = 'u';
- *d++ = hex_digits[(v >> 12) & 0xf];
- *d++ = hex_digits[(v >> 8) & 0xf];
- *d++ = hex_digits[(v >> 4) & 0xf];
- *d++ = hex_digits[v & 0xf];
- }
- else {
- if (unlikely(*t == '\\' || *t == '\"'))
- *d++ = '\\';
-
- *d++ = *t++;
- }
- }
-
- wb->len += d - s;
- }
-
- buffer_need_bytes(wb, 1);
- wb->buffer[wb->len] = '\0';
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_json_quoted_strcat(BUFFER *wb, const char *txt) {
- if(unlikely(!txt || !*txt)) return;
-
- if(*txt == '"')
- txt++;
-
- const char *t = txt;
- while(*t) {
- buffer_need_bytes(wb, 100);
- char *s = &wb->buffer[wb->len];
- char *d = s;
- const char *e = &wb->buffer[wb->size - 1]; // remove 1 to make room for the escape character
-
- while(*t && d < e) {
- if(unlikely(*t == '"' && !t[1])) {
- t++;
- continue;
- }
-
- if(unlikely(*t == '\\' || *t == '"'))
- *d++ = '\\';
-
- *d++ = *t++;
- }
-
- wb->len += d - s;
- }
-
- buffer_need_bytes(wb, 1);
- wb->buffer[wb->len] = '\0';
-
- buffer_overflow_check(wb);
-}
-
-// This trick seems to give an 80% speed increase in 32bit systems
-// print_number_llu_r() will just print the digits up to the
-// point the remaining value fits in 32 bits, and then calls
-// print_number_lu_r() to print the rest with 32 bit arithmetic.
-
-static inline char *print_uint32_reversed(char *dst, uint32_t value) {
- char *d = dst;
- do *d++ = (char)('0' + (value % 10)); while((value /= 10));
- return d;
-}
-
-static inline char *print_uint64_reversed(char *dst, uint64_t value) {
-#ifdef ENV32BIT
- if(value <= (uint64_t)0xffffffff)
- return print_uint32_reversed(dst, value);
-
- char *d = dst;
- do *d++ = (char)('0' + (value % 10)); while((value /= 10) && value > (uint64_t)0xffffffff);
- if(value) return print_uint32_reversed(d, value);
- return d;
-#else
- char *d = dst;
- do *d++ = (char)('0' + (value % 10)); while((value /= 10));
- return d;
-#endif
-}
-
-static inline char *print_uint32_hex_reversed(char *dst, uint32_t value) {
- static const char *digits = "0123456789ABCDEF";
- char *d = dst;
- do *d++ = digits[value & 0xf]; while((value >>= 4));
- return d;
-}
-
-static inline char *print_uint64_hex_reversed(char *dst, uint64_t value) {
-#ifdef ENV32BIT
- if(value <= (uint64_t)0xffffffff)
- return print_uint32_hex_reversed(dst, value);
-
- char *d = dst;
- do *d++ = hex_digits[value & 0xf]; while((value >>= 4) && value > (uint64_t)0xffffffff);
- if(value) return print_uint32_hex_reversed(d, value);
- return d;
-#else
- char *d = dst;
- do *d++ = hex_digits[value & 0xf]; while((value >>= 4));
- return d;
-#endif
-}
-
-static inline char *print_uint64_base64_reversed(char *dst, uint64_t value) {
- char *d = dst;
- do *d++ = base64_digits[value & 63]; while ((value >>= 6));
- return d;
-}
-
-static inline void char_array_reverse(char *from, char *to) {
- // from and to are inclusive
- char *begin = from, *end = to, aux;
- while (end > begin) aux = *end, *end-- = *begin, *begin++ = aux;
-}
-
-static inline int print_netdata_double(char *dst, NETDATA_DOUBLE value) {
- char *s = dst;
-
- if(unlikely(value < 0)) {
- *s++ = '-';
- value = fabsndd(value);
- }
-
- uint64_t fractional_precision = 10000000ULL; // fractional part 7 digits
- int fractional_wanted_digits = 7;
- int exponent = 0;
- if(unlikely(value >= (NETDATA_DOUBLE)(UINT64_MAX / 10))) {
- // the number is too big to print using 64bit numbers
- // so, let's convert it to exponential notation
- exponent = (int)(floorndd(log10ndd(value)));
- value /= powndd(10, exponent);
-
- // the max precision we can support is 18 digits
- // (UINT64_MAX is 20, but the first is 1)
- fractional_precision = 1000000000000000000ULL; // fractional part 18 digits
- fractional_wanted_digits = 18;
- }
-
- char *d = s;
- NETDATA_DOUBLE integral_d, fractional_d;
- fractional_d = modfndd(value, &integral_d);
-
- // get the integral and the fractional parts as 64-bit integers
- uint64_t integral = (uint64_t)integral_d;
- uint64_t fractional = (uint64_t)llrintndd(fractional_d * (NETDATA_DOUBLE)fractional_precision);
- if(unlikely(fractional >= fractional_precision)) {
- integral++;
- fractional -= fractional_precision;
- }
-
- // convert the integral part to string (reversed)
- d = print_uint64_reversed(d, integral);
- char_array_reverse(s, d - 1); // copy reversed the integral string
-
- if(likely(fractional != 0)) {
- *d++ = '.'; // add the dot
-
- // convert the fractional part to string (reversed)
- d = print_uint64_reversed(s = d, fractional);
-
- while(d - s < fractional_wanted_digits) *d++ = '0'; // prepend zeros to reach precision
- char_array_reverse(s, d - 1); // copy reversed the fractional string
-
- // remove trailing zeros from the fractional part
- while(*(d - 1) == '0') d--;
- }
-
- if(unlikely(exponent != 0)) {
- *d++ = 'e';
- *d++ = '+';
- d = print_uint32_reversed(s = d, exponent);
- char_array_reverse(s, d - 1);
- }
-
- *d = '\0';
- return (int)(d - dst);
-}
-
-static inline void buffer_print_uint64(BUFFER *wb, uint64_t value) {
- buffer_need_bytes(wb, 50);
-
- char *s = &wb->buffer[wb->len];
- char *d = print_uint64_reversed(s, value);
- char_array_reverse(s, d - 1);
- *d = '\0';
- wb->len += d - s;
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_print_int64(BUFFER *wb, int64_t value) {
- buffer_need_bytes(wb, 50);
-
- if(value < 0) {
- buffer_fast_strcat(wb, "-", 1);
- value = -value;
- }
-
- buffer_print_uint64(wb, (uint64_t)value);
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_print_uint64_hex(BUFFER *wb, uint64_t value) {
- buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1);
-
- buffer_fast_strcat(wb, HEX_PREFIX, sizeof(HEX_PREFIX) - 1);
-
- char *s = &wb->buffer[wb->len];
- char *d = print_uint64_hex_reversed(s, value);
- char_array_reverse(s, d - 1);
- *d = '\0';
- wb->len += d - s;
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_print_uint64_base64(BUFFER *wb, uint64_t value) {
- buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1);
-
- buffer_fast_strcat(wb, IEEE754_UINT64_B64_PREFIX, sizeof(IEEE754_UINT64_B64_PREFIX) - 1);
-
- char *s = &wb->buffer[wb->len];
- char *d = print_uint64_base64_reversed(s, value);
- char_array_reverse(s, d - 1);
- *d = '\0';
- wb->len += d - s;
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_print_int64_hex(BUFFER *wb, int64_t value) {
- buffer_need_bytes(wb, 2);
-
- if(value < 0) {
- buffer_fast_strcat(wb, "-", 1);
- value = -value;
- }
-
- buffer_print_uint64_hex(wb, (uint64_t)value);
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_print_int64_base64(BUFFER *wb, int64_t value) {
- buffer_need_bytes(wb, 2);
-
- if(value < 0) {
- buffer_fast_strcat(wb, "-", 1);
- value = -value;
- }
-
- buffer_print_uint64_base64(wb, (uint64_t)value);
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_print_netdata_double(BUFFER *wb, NETDATA_DOUBLE value) {
- buffer_need_bytes(wb, 512 + 2);
-
- if(isnan(value) || isinf(value)) {
- buffer_fast_strcat(wb, "null", 4);
- return;
- }
- else
- wb->len += print_netdata_double(&wb->buffer[wb->len], value);
-
- // terminate it
- buffer_need_bytes(wb, 1);
- wb->buffer[wb->len] = '\0';
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_print_netdata_double_hex(BUFFER *wb, NETDATA_DOUBLE value) {
- buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1 + 1);
-
- uint64_t *ptr = (uint64_t *) (&value);
- buffer_fast_strcat(wb, IEEE754_DOUBLE_HEX_PREFIX, sizeof(IEEE754_DOUBLE_HEX_PREFIX) - 1);
-
- char *s = &wb->buffer[wb->len];
- char *d = print_uint64_hex_reversed(s, *ptr);
- char_array_reverse(s, d - 1);
- *d = '\0';
- wb->len += d - s;
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_print_netdata_double_base64(BUFFER *wb, NETDATA_DOUBLE value) {
- buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1 + 1);
-
- uint64_t *ptr = (uint64_t *) (&value);
- buffer_fast_strcat(wb, IEEE754_DOUBLE_B64_PREFIX, sizeof(IEEE754_DOUBLE_B64_PREFIX) - 1);
-
- char *s = &wb->buffer[wb->len];
- char *d = print_uint64_base64_reversed(s, *ptr);
- char_array_reverse(s, d - 1);
- *d = '\0';
- wb->len += d - s;
-
- buffer_overflow_check(wb);
-}
-
-typedef enum {
- NUMBER_ENCODING_DECIMAL,
- NUMBER_ENCODING_HEX,
- NUMBER_ENCODING_BASE64,
-} NUMBER_ENCODING;
-
-static inline void buffer_print_int64_encoded(BUFFER *wb, NUMBER_ENCODING encoding, int64_t value) {
- if(encoding == NUMBER_ENCODING_BASE64)
- return buffer_print_int64_base64(wb, value);
-
- if(encoding == NUMBER_ENCODING_HEX)
- return buffer_print_int64_hex(wb, value);
-
- return buffer_print_int64(wb, value);
-}
-
-static inline void buffer_print_uint64_encoded(BUFFER *wb, NUMBER_ENCODING encoding, uint64_t value) {
- if(encoding == NUMBER_ENCODING_BASE64)
- return buffer_print_uint64_base64(wb, value);
-
- if(encoding == NUMBER_ENCODING_HEX)
- return buffer_print_uint64_hex(wb, value);
-
- return buffer_print_uint64(wb, value);
-}
-
-static inline void buffer_print_netdata_double_encoded(BUFFER *wb, NUMBER_ENCODING encoding, NETDATA_DOUBLE value) {
- if(encoding == NUMBER_ENCODING_BASE64)
- return buffer_print_netdata_double_base64(wb, value);
-
- if(encoding == NUMBER_ENCODING_HEX)
- return buffer_print_netdata_double_hex(wb, value);
-
- return buffer_print_netdata_double(wb, value);
-}
-
-static inline void buffer_print_spaces(BUFFER *wb, size_t spaces) {
- buffer_need_bytes(wb, spaces * 4 + 1);
-
- char *d = &wb->buffer[wb->len];
- for(size_t i = 0; i < spaces; i++) {
- *d++ = ' ';
- *d++ = ' ';
- *d++ = ' ';
- *d++ = ' ';
- }
-
- *d = '\0';
- wb->len += spaces * 4;
-
- buffer_overflow_check(wb);
-}
-
-static inline void buffer_print_json_comma(BUFFER *wb) {
- if(wb->json.stack[wb->json.depth].count)
- buffer_fast_strcat(wb, ",", 1);
-}
-
-static inline void buffer_print_json_comma_newline_spacing(BUFFER *wb) {
- buffer_print_json_comma(wb);
-
- if((wb->json.options & BUFFER_JSON_OPTIONS_MINIFY) ||
- (wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && !(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS)))
- return;
-
- buffer_fast_strcat(wb, "\n", 1);
- buffer_print_spaces(wb, wb->json.depth + 1);
-}
-
-static inline void buffer_print_json_key(BUFFER *wb, const char *key) {
- buffer_strcat(wb, wb->json.key_quote);
- buffer_json_strcat(wb, key);
- buffer_strcat(wb, wb->json.key_quote);
-}
-
-static inline void buffer_json_add_string_value(BUFFER *wb, const char *value) {
- if(value) {
- buffer_strcat(wb, wb->json.value_quote);
- buffer_json_strcat(wb, value);
- buffer_strcat(wb, wb->json.value_quote);
- }
- else
- buffer_fast_strcat(wb, "null", 4);
-}
-
-static inline void buffer_json_add_quoted_string_value(BUFFER *wb, const char *value) {
- if(value) {
- buffer_strcat(wb, wb->json.value_quote);
- buffer_json_quoted_strcat(wb, value);
- buffer_strcat(wb, wb->json.value_quote);
- }
- else
- buffer_fast_strcat(wb, "null", 4);
-}
-
-static inline void buffer_json_member_add_object(BUFFER *wb, const char *key) {
- buffer_print_json_comma_newline_spacing(wb);
- buffer_print_json_key(wb, key);
- buffer_fast_strcat(wb, ":{", 2);
- wb->json.stack[wb->json.depth].count++;
-
- _buffer_json_depth_push(wb, BUFFER_JSON_OBJECT);
-}
-
-static inline void buffer_json_object_close(BUFFER *wb) {
-#ifdef NETDATA_INTERNAL_CHECKS
- assert(wb->json.depth >= 0 && "BUFFER JSON: nothing is open to close it");
- assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_OBJECT && "BUFFER JSON: an object is not open to close it");
-#endif
- if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY)) {
- buffer_fast_strcat(wb, "\n", 1);
- buffer_print_spaces(wb, wb->json.depth);
- }
- buffer_fast_strcat(wb, "}", 1);
- _buffer_json_depth_pop(wb);
-}
-
-static inline void buffer_json_member_add_string(BUFFER *wb, const char *key, const char *value) {
- buffer_print_json_comma_newline_spacing(wb);
- buffer_print_json_key(wb, key);
- buffer_fast_strcat(wb, ":", 1);
- buffer_json_add_string_value(wb, value);
-
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_member_add_string_or_omit(BUFFER *wb, const char *key, const char *value) {
- if(value && *value)
- buffer_json_member_add_string(wb, key, value);
-}
-
-static inline void buffer_json_member_add_string_or_empty(BUFFER *wb, const char *key, const char *value) {
- if(!value)
- value = "";
-
- buffer_json_member_add_string(wb, key, value);
-}
-
-static inline void buffer_json_member_add_quoted_string(BUFFER *wb, const char *key, const char *value) {
- buffer_print_json_comma_newline_spacing(wb);
- buffer_print_json_key(wb, key);
- buffer_fast_strcat(wb, ":", 1);
-
- if(!value || strcmp(value, "null") == 0)
- buffer_fast_strcat(wb, "null", 4);
- else
- buffer_json_add_quoted_string_value(wb, value);
-
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_member_add_uuid(BUFFER *wb, const char *key, uuid_t *value) {
- buffer_print_json_comma_newline_spacing(wb);
- buffer_print_json_key(wb, key);
- buffer_fast_strcat(wb, ":", 1);
-
- if(value && !uuid_is_null(*value)) {
- char uuid[GUID_LEN + 1];
- uuid_unparse_lower(*value, uuid);
- buffer_json_add_string_value(wb, uuid);
- }
- else
- buffer_json_add_string_value(wb, NULL);
-
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_member_add_boolean(BUFFER *wb, const char *key, bool value) {
- buffer_print_json_comma_newline_spacing(wb);
- buffer_print_json_key(wb, key);
- buffer_fast_strcat(wb, ":", 1);
- buffer_strcat(wb, value?"true":"false");
-
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_member_add_array(BUFFER *wb, const char *key) {
- buffer_print_json_comma_newline_spacing(wb);
- if (key) {
- buffer_print_json_key(wb, key);
- buffer_fast_strcat(wb, ":[", 2);
- }
- else
- buffer_fast_strcat(wb, "[", 1);
-
- wb->json.stack[wb->json.depth].count++;
-
- _buffer_json_depth_push(wb, BUFFER_JSON_ARRAY);
-}
-
-static inline void buffer_json_add_array_item_array(BUFFER *wb) {
- if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY) && wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY) {
- // an array inside another array
- buffer_print_json_comma(wb);
- buffer_fast_strcat(wb, "\n", 1);
- buffer_print_spaces(wb, wb->json.depth + 1);
- }
- else
- buffer_print_json_comma_newline_spacing(wb);
-
- buffer_fast_strcat(wb, "[", 1);
- wb->json.stack[wb->json.depth].count++;
-
- _buffer_json_depth_push(wb, BUFFER_JSON_ARRAY);
-}
-
-static inline void buffer_json_add_array_item_string(BUFFER *wb, const char *value) {
- buffer_print_json_comma_newline_spacing(wb);
-
- buffer_json_add_string_value(wb, value);
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_add_array_item_double(BUFFER *wb, NETDATA_DOUBLE value) {
- buffer_print_json_comma_newline_spacing(wb);
-
- buffer_print_netdata_double(wb, value);
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_add_array_item_int64(BUFFER *wb, int64_t value) {
- buffer_print_json_comma_newline_spacing(wb);
-
- buffer_print_int64(wb, value);
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_add_array_item_uint64(BUFFER *wb, uint64_t value) {
- buffer_print_json_comma_newline_spacing(wb);
-
- buffer_print_uint64(wb, value);
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_add_array_item_boolean(BUFFER *wb, bool value) {
- buffer_print_json_comma_newline_spacing(wb);
-
- buffer_strcat(wb, value ? "true" : "false");
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_add_array_item_time_t(BUFFER *wb, time_t value) {
- buffer_print_json_comma_newline_spacing(wb);
-
- buffer_print_int64(wb, value);
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_add_array_item_time_ms(BUFFER *wb, time_t value) {
- buffer_print_json_comma_newline_spacing(wb);
-
- buffer_print_int64(wb, value);
- buffer_fast_strcat(wb, "000", 3);
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_add_array_item_time_t2ms(BUFFER *wb, time_t value) {
- buffer_print_json_comma_newline_spacing(wb);
-
- buffer_print_int64(wb, value);
- buffer_fast_strcat(wb, "000", 3);
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_add_array_item_object(BUFFER *wb) {
- buffer_print_json_comma_newline_spacing(wb);
-
- buffer_fast_strcat(wb, "{", 1);
- wb->json.stack[wb->json.depth].count++;
-
- _buffer_json_depth_push(wb, BUFFER_JSON_OBJECT);
-}
-
-static inline void buffer_json_member_add_time_t(BUFFER *wb, const char *key, time_t value) {
- buffer_print_json_comma_newline_spacing(wb);
- buffer_print_json_key(wb, key);
- buffer_fast_strcat(wb, ":", 1);
- buffer_print_int64(wb, value);
-
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_member_add_time_t2ms(BUFFER *wb, const char *key, time_t value) {
- buffer_print_json_comma_newline_spacing(wb);
- buffer_print_json_key(wb, key);
- buffer_fast_strcat(wb, ":", 1);
- buffer_print_int64(wb, value);
- buffer_fast_strcat(wb, "000", 3);
-
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_member_add_uint64(BUFFER *wb, const char *key, uint64_t value) {
- buffer_print_json_comma_newline_spacing(wb);
- buffer_print_json_key(wb, key);
- buffer_fast_strcat(wb, ":", 1);
- buffer_print_uint64(wb, value);
-
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_member_add_int64(BUFFER *wb, const char *key, int64_t value) {
- buffer_print_json_comma_newline_spacing(wb);
- buffer_print_json_key(wb, key);
- buffer_fast_strcat(wb, ":", 1);
- buffer_print_int64(wb, value);
-
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_member_add_double(BUFFER *wb, const char *key, NETDATA_DOUBLE value) {
- buffer_print_json_comma_newline_spacing(wb);
- buffer_print_json_key(wb, key);
- buffer_fast_strcat(wb, ":", 1);
- buffer_print_netdata_double(wb, value);
-
- wb->json.stack[wb->json.depth].count++;
-}
-
-static inline void buffer_json_array_close(BUFFER *wb) {
-#ifdef NETDATA_INTERNAL_CHECKS
- assert(wb->json.depth >= 0 && "BUFFER JSON: nothing is open to close it");
- assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && "BUFFER JSON: an array is not open to close it");
-#endif
- if(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS) {
- buffer_fast_strcat(wb, "\n", 1);
- buffer_print_spaces(wb, wb->json.depth);
- }
-
- buffer_fast_strcat(wb, "]", 1);
- _buffer_json_depth_pop(wb);
-}
-
-typedef enum __attribute__((packed)) {
- RRDF_FIELD_OPTS_NONE = 0,
- RRDF_FIELD_OPTS_UNIQUE_KEY = (1 << 0), // the field is the unique key of the row
- RRDF_FIELD_OPTS_VISIBLE = (1 << 1), // the field should be visible by default
- RRDF_FIELD_OPTS_STICKY = (1 << 2), // the field should be sticky
- RRDF_FIELD_OPTS_FULL_WIDTH = (1 << 3), // the field should get full width
- RRDF_FIELD_OPTS_WRAP = (1 << 4), // the field should wrap
- RRDF_FIELD_OPTS_DUMMY = (1 << 5), // not a presentable field
- RRDF_FIELD_OPTS_EXPANDED_FILTER = (1 << 6), // show the filter expanded
-} RRDF_FIELD_OPTIONS;
-
-typedef enum __attribute__((packed)) {
- RRDF_FIELD_TYPE_NONE,
- RRDF_FIELD_TYPE_INTEGER,
- RRDF_FIELD_TYPE_BOOLEAN,
- RRDF_FIELD_TYPE_STRING,
- RRDF_FIELD_TYPE_DETAIL_STRING,
- RRDF_FIELD_TYPE_BAR_WITH_INTEGER,
- RRDF_FIELD_TYPE_DURATION,
- RRDF_FIELD_TYPE_TIMESTAMP,
- RRDF_FIELD_TYPE_ARRAY,
-} RRDF_FIELD_TYPE;
-
-static inline const char *rrdf_field_type_to_string(RRDF_FIELD_TYPE type) {
- switch(type) {
- default:
- case RRDF_FIELD_TYPE_NONE:
- return "none";
-
- case RRDF_FIELD_TYPE_INTEGER:
- return "integer";
-
- case RRDF_FIELD_TYPE_BOOLEAN:
- return "boolean";
-
- case RRDF_FIELD_TYPE_STRING:
- return "string";
-
- case RRDF_FIELD_TYPE_DETAIL_STRING:
- return "detail-string";
-
- case RRDF_FIELD_TYPE_BAR_WITH_INTEGER:
- return "bar-with-integer";
-
- case RRDF_FIELD_TYPE_DURATION:
- return "duration";
-
- case RRDF_FIELD_TYPE_TIMESTAMP:
- return "timestamp";
-
- case RRDF_FIELD_TYPE_ARRAY:
- return "array";
- }
-}
-
-typedef enum __attribute__((packed)) {
- RRDF_FIELD_VISUAL_VALUE, // show the value, possibly applying a transformation
- RRDF_FIELD_VISUAL_BAR, // show the value and a bar, respecting the max field to fill the bar at 100%
- RRDF_FIELD_VISUAL_PILL, //
- RRDF_FIELD_VISUAL_RICH, //
- RRDR_FIELD_VISUAL_ROW_OPTIONS, // this is a dummy column that is used for row options
-} RRDF_FIELD_VISUAL;
-
-static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual) {
- switch(visual) {
- default:
- case RRDF_FIELD_VISUAL_VALUE:
- return "value";
-
- case RRDF_FIELD_VISUAL_BAR:
- return "bar";
-
- case RRDF_FIELD_VISUAL_PILL:
- return "pill";
-
- case RRDF_FIELD_VISUAL_RICH:
- return "richValue";
-
- case RRDR_FIELD_VISUAL_ROW_OPTIONS:
- return "rowOptions";
- }
-}
-
-typedef enum __attribute__((packed)) {
- RRDF_FIELD_TRANSFORM_NONE, // show the value as-is
- RRDF_FIELD_TRANSFORM_NUMBER, // show the value respecting the decimal_points
- RRDF_FIELD_TRANSFORM_DURATION_S, // transform as duration in second to a human-readable duration
- RRDF_FIELD_TRANSFORM_DATETIME_MS, // UNIX epoch timestamp in ms
- RRDF_FIELD_TRANSFORM_DATETIME_USEC, // UNIX epoch timestamp in usec
-} RRDF_FIELD_TRANSFORM;
-
-static inline const char *rrdf_field_transform_to_string(RRDF_FIELD_TRANSFORM transform) {
- switch(transform) {
- default:
- case RRDF_FIELD_TRANSFORM_NONE:
- return "none";
-
- case RRDF_FIELD_TRANSFORM_NUMBER:
- return "number";
-
- case RRDF_FIELD_TRANSFORM_DURATION_S:
- return "duration";
-
- case RRDF_FIELD_TRANSFORM_DATETIME_MS:
- return "datetime";
-
- case RRDF_FIELD_TRANSFORM_DATETIME_USEC:
- return "datetime_usec";
- }
-}
-
-typedef enum __attribute__((packed)) {
- RRDF_FIELD_SORT_ASCENDING = (1 << 0),
- RRDF_FIELD_SORT_DESCENDING = (1 << 1),
-
- RRDF_FIELD_SORT_FIXED = (1 << 7),
-} RRDF_FIELD_SORT;
-
-static inline const char *rrdf_field_sort_to_string(RRDF_FIELD_SORT sort) {
- if(sort & RRDF_FIELD_SORT_DESCENDING)
- return "descending";
-
- else
- return "ascending";
-}
-
-typedef enum __attribute__((packed)) {
- RRDF_FIELD_SUMMARY_UNIQUECOUNT, // Finds the number of unique values of a group of rows
- RRDF_FIELD_SUMMARY_SUM, // Sums the values of a group of rows
- RRDF_FIELD_SUMMARY_MIN, // Finds the minimum value of a group of rows
- RRDF_FIELD_SUMMARY_MAX, // Finds the maximum value of a group of rows
- // RRDF_FIELD_SUMMARY_EXTENT, // Finds the minimum and maximum values of a group of rows
- RRDF_FIELD_SUMMARY_MEAN, // Finds the mean/average value of a group of rows
- RRDF_FIELD_SUMMARY_MEDIAN, // Finds the median value of a group of rows
- // RRDF_FIELD_SUMMARY_UNIQUE, // Finds the unique values of a group of rows
- RRDF_FIELD_SUMMARY_COUNT, // Calculates the number of rows in a group
-} RRDF_FIELD_SUMMARY;
-
-static inline const char *rrdf_field_summary_to_string(RRDF_FIELD_SUMMARY summary) {
- switch(summary) {
- default:
- case RRDF_FIELD_SUMMARY_COUNT:
- return "count";
-
- case RRDF_FIELD_SUMMARY_UNIQUECOUNT:
- return "uniqueCount";
-
- case RRDF_FIELD_SUMMARY_SUM:
- return "sum";
-
- case RRDF_FIELD_SUMMARY_MIN:
- return "min";
-
- case RRDF_FIELD_SUMMARY_MEAN:
- return "mean";
-
- case RRDF_FIELD_SUMMARY_MEDIAN:
- return "median";
-
- case RRDF_FIELD_SUMMARY_MAX:
- return "max";
- }
-}
-
-typedef enum __attribute__((packed)) {
- RRDF_FIELD_FILTER_NONE = 0,
- RRDF_FIELD_FILTER_RANGE,
- RRDF_FIELD_FILTER_MULTISELECT,
- RRDF_FIELD_FILTER_FACET,
-} RRDF_FIELD_FILTER;
-
-static inline const char *rrdf_field_filter_to_string(RRDF_FIELD_FILTER filter) {
- switch(filter) {
- case RRDF_FIELD_FILTER_RANGE:
- return "range";
-
- case RRDF_FIELD_FILTER_MULTISELECT:
- return "multiselect";
-
- case RRDF_FIELD_FILTER_FACET:
- return "facet";
-
- default:
- case RRDF_FIELD_FILTER_NONE:
- return "none";
- }
-}
-
-static inline void
-buffer_rrdf_table_add_field(BUFFER *wb, size_t field_id, const char *key, const char *name, RRDF_FIELD_TYPE type,
- RRDF_FIELD_VISUAL visual, RRDF_FIELD_TRANSFORM transform, size_t decimal_points,
- const char *units, NETDATA_DOUBLE max, RRDF_FIELD_SORT sort, const char *pointer_to,
- RRDF_FIELD_SUMMARY summary, RRDF_FIELD_FILTER filter, RRDF_FIELD_OPTIONS options,
- const char *default_value) {
-
- buffer_json_member_add_object(wb, key);
- {
- buffer_json_member_add_uint64(wb, "index", field_id);
- buffer_json_member_add_boolean(wb, "unique_key", options & RRDF_FIELD_OPTS_UNIQUE_KEY);
- buffer_json_member_add_string(wb, "name", name);
- buffer_json_member_add_boolean(wb, "visible", options & RRDF_FIELD_OPTS_VISIBLE);
- buffer_json_member_add_string(wb, "type", rrdf_field_type_to_string(type));
- buffer_json_member_add_string_or_omit(wb, "units", units);
- buffer_json_member_add_string(wb, "visualization", rrdf_field_visual_to_string(visual));
-
- buffer_json_member_add_object(wb, "value_options");
- {
- buffer_json_member_add_string_or_omit(wb, "units", units);
- buffer_json_member_add_string(wb, "transform", rrdf_field_transform_to_string(transform));
- buffer_json_member_add_uint64(wb, "decimal_points", decimal_points);
- buffer_json_member_add_string(wb, "default_value", default_value);
- }
- buffer_json_object_close(wb);
-
- if (!isnan((NETDATA_DOUBLE) (max)))
- buffer_json_member_add_double(wb, "max", (NETDATA_DOUBLE) (max));
-
- buffer_json_member_add_string_or_omit(wb, "pointer_to", pointer_to);
- buffer_json_member_add_string(wb, "sort", rrdf_field_sort_to_string(sort));
- buffer_json_member_add_boolean(wb, "sortable", !(sort & RRDF_FIELD_SORT_FIXED));
- buffer_json_member_add_boolean(wb, "sticky", options & RRDF_FIELD_OPTS_STICKY);
- buffer_json_member_add_string(wb, "summary", rrdf_field_summary_to_string(summary));
- buffer_json_member_add_string(wb, "filter", rrdf_field_filter_to_string(filter));
-
- buffer_json_member_add_boolean(wb, "full_width", options & RRDF_FIELD_OPTS_FULL_WIDTH);
- buffer_json_member_add_boolean(wb, "wrap", options & RRDF_FIELD_OPTS_WRAP);
- buffer_json_member_add_boolean(wb, "default_expanded_filter", options & RRDF_FIELD_OPTS_EXPANDED_FILTER);
-
- if(options & RRDF_FIELD_OPTS_DUMMY)
- buffer_json_member_add_boolean(wb, "dummy", true);
- }
- buffer_json_object_close(wb);
-}
-
-#endif /* NETDATA_WEB_BUFFER_H */
diff --git a/libnetdata/buffered_reader/Makefile.am b/libnetdata/buffered_reader/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/buffered_reader/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/buffered_reader/buffered_reader.h b/libnetdata/buffered_reader/buffered_reader.h
deleted file mode 100644
index 4db57cd29..000000000
--- a/libnetdata/buffered_reader/buffered_reader.h
+++ /dev/null
@@ -1,146 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "../libnetdata.h"
-
-#ifndef NETDATA_BUFFERED_READER_H
-#define NETDATA_BUFFERED_READER_H
-
-struct buffered_reader {
- ssize_t read_len;
- ssize_t pos;
- char read_buffer[PLUGINSD_LINE_MAX + 1];
-};
-
-static inline void buffered_reader_init(struct buffered_reader *reader) {
- reader->read_buffer[0] = '\0';
- reader->read_len = 0;
- reader->pos = 0;
-}
-
-typedef enum {
- BUFFERED_READER_READ_OK = 0,
- BUFFERED_READER_READ_FAILED = -1,
- BUFFERED_READER_READ_BUFFER_FULL = -2,
- BUFFERED_READER_READ_POLLERR = -3,
- BUFFERED_READER_READ_POLLHUP = -4,
- BUFFERED_READER_READ_POLLNVAL = -5,
- BUFFERED_READER_READ_POLL_UNKNOWN = -6,
- BUFFERED_READER_READ_POLL_TIMEOUT = -7,
- BUFFERED_READER_READ_POLL_FAILED = -8,
-} buffered_reader_ret_t;
-
-
-static inline buffered_reader_ret_t buffered_reader_read(struct buffered_reader *reader, int fd) {
-#ifdef NETDATA_INTERNAL_CHECKS
- if(reader->read_buffer[reader->read_len] != '\0')
- fatal("read_buffer does not start with zero");
-#endif
-
- char *read_at = reader->read_buffer + reader->read_len;
- ssize_t remaining = sizeof(reader->read_buffer) - reader->read_len - 1;
-
- if(unlikely(remaining <= 0))
- return BUFFERED_READER_READ_BUFFER_FULL;
-
- ssize_t bytes_read = read(fd, read_at, remaining);
- if(unlikely(bytes_read <= 0))
- return BUFFERED_READER_READ_FAILED;
-
- reader->read_len += bytes_read;
- reader->read_buffer[reader->read_len] = '\0';
-
- return BUFFERED_READER_READ_OK;
-}
-
-static inline buffered_reader_ret_t buffered_reader_read_timeout(struct buffered_reader *reader, int fd, int timeout_ms, bool log_error) {
- errno = 0;
- struct pollfd fds[1];
-
- fds[0].fd = fd;
- fds[0].events = POLLIN;
-
- int ret = poll(fds, 1, timeout_ms);
-
- if (ret > 0) {
- /* There is data to read */
- if (fds[0].revents & POLLIN)
- return buffered_reader_read(reader, fd);
-
- else if(fds[0].revents & POLLERR) {
- if(log_error)
- netdata_log_error("PARSER: read failed: POLLERR.");
- return BUFFERED_READER_READ_POLLERR;
- }
- else if(fds[0].revents & POLLHUP) {
- if(log_error)
- netdata_log_error("PARSER: read failed: POLLHUP.");
- return BUFFERED_READER_READ_POLLHUP;
- }
- else if(fds[0].revents & POLLNVAL) {
- if(log_error)
- netdata_log_error("PARSER: read failed: POLLNVAL.");
- return BUFFERED_READER_READ_POLLNVAL;
- }
-
- if(log_error)
- netdata_log_error("PARSER: poll() returned positive number, but POLLIN|POLLERR|POLLHUP|POLLNVAL are not set.");
- return BUFFERED_READER_READ_POLL_UNKNOWN;
- }
- else if (ret == 0) {
- if(log_error)
- netdata_log_error("PARSER: timeout while waiting for data.");
- return BUFFERED_READER_READ_POLL_TIMEOUT;
- }
-
- if(log_error)
- netdata_log_error("PARSER: poll() failed with code %d.", ret);
- return BUFFERED_READER_READ_POLL_FAILED;
-}
-
-/* Produce a full line if one exists, statefully return where we start next time.
- * When we hit the end of the buffer with a partial line move it to the beginning for the next fill.
- */
-static inline bool buffered_reader_next_line(struct buffered_reader *reader, BUFFER *dst) {
- buffer_need_bytes(dst, reader->read_len - reader->pos + 2);
-
- size_t start = reader->pos;
-
- char *ss = &reader->read_buffer[start];
- char *se = &reader->read_buffer[reader->read_len];
- char *ds = &dst->buffer[dst->len];
- char *de = &ds[dst->size - dst->len - 2];
-
- if(ss >= se) {
- *ds = '\0';
- reader->pos = 0;
- reader->read_len = 0;
- reader->read_buffer[reader->read_len] = '\0';
- return false;
- }
-
- // copy all bytes to buffer
- while(ss < se && ds < de && *ss != '\n') {
- *ds++ = *ss++;
- dst->len++;
- }
-
- // if we have a newline, return the buffer
- if(ss < se && ds < de && *ss == '\n') {
- // newline found in the r->read_buffer
-
- *ds++ = *ss++; // copy the newline too
- dst->len++;
-
- *ds = '\0';
-
- reader->pos = ss - reader->read_buffer;
- return true;
- }
-
- reader->pos = 0;
- reader->read_len = 0;
- reader->read_buffer[reader->read_len] = '\0';
- return false;
-}
-
-#endif //NETDATA_BUFFERED_READER_H
diff --git a/libnetdata/circular_buffer/Makefile.am b/libnetdata/circular_buffer/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/circular_buffer/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/circular_buffer/README.md b/libnetdata/circular_buffer/README.md
deleted file mode 100644
index c47386516..000000000
--- a/libnetdata/circular_buffer/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-<!--
-title: "Circular Buffer"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/circular_buffer/README.md
-sidebar_label: "Circular Buffer"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Circular Buffer
-
-`struct circular_buffer` is an adaptive circular buffer. It will start at an initial size
-and grow up to a maximum size as it fills. Two indices within the structure track the current
-`read` and `write` position for data.
diff --git a/libnetdata/clocks/Makefile.am b/libnetdata/clocks/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/clocks/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/clocks/README.md b/libnetdata/clocks/README.md
deleted file mode 100644
index 33b0f0e85..000000000
--- a/libnetdata/clocks/README.md
+++ /dev/null
@@ -1,10 +0,0 @@
-<!--
-custom_edit_url: "https://github.com/netdata/netdata/edit/master/libnetdata/clocks/README.md"
-title: "Clocks"
-sidebar_label: "Clocks"
-learn_status: "Published"
-learn_topic_type: "References"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Clocks \ No newline at end of file
diff --git a/libnetdata/completion/Makefile.am b/libnetdata/completion/Makefile.am
deleted file mode 100644
index babdcf0df..000000000
--- a/libnetdata/completion/Makefile.am
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
diff --git a/libnetdata/completion/completion.c b/libnetdata/completion/completion.c
deleted file mode 100644
index 6257e0299..000000000
--- a/libnetdata/completion/completion.c
+++ /dev/null
@@ -1,64 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "completion.h"
-
-void completion_init(struct completion *p)
-{
- p->completed = 0;
- p->completed_jobs = 0;
- fatal_assert(0 == uv_cond_init(&p->cond));
- fatal_assert(0 == uv_mutex_init(&p->mutex));
-}
-
-void completion_destroy(struct completion *p)
-{
- uv_cond_destroy(&p->cond);
- uv_mutex_destroy(&p->mutex);
-}
-
-void completion_wait_for(struct completion *p)
-{
- uv_mutex_lock(&p->mutex);
- while (0 == p->completed) {
- uv_cond_wait(&p->cond, &p->mutex);
- }
- fatal_assert(1 == p->completed);
- uv_mutex_unlock(&p->mutex);
-}
-
-void completion_mark_complete(struct completion *p)
-{
- uv_mutex_lock(&p->mutex);
- p->completed = 1;
- uv_cond_broadcast(&p->cond);
- uv_mutex_unlock(&p->mutex);
-}
-
-unsigned completion_wait_for_a_job(struct completion *p, unsigned completed_jobs)
-{
- uv_mutex_lock(&p->mutex);
- while (0 == p->completed && p->completed_jobs <= completed_jobs) {
- uv_cond_wait(&p->cond, &p->mutex);
- }
- completed_jobs = p->completed_jobs;
- uv_mutex_unlock(&p->mutex);
-
- return completed_jobs;
-}
-
-void completion_mark_complete_a_job(struct completion *p)
-{
- uv_mutex_lock(&p->mutex);
- p->completed_jobs++;
- uv_cond_broadcast(&p->cond);
- uv_mutex_unlock(&p->mutex);
-}
-
-bool completion_is_done(struct completion *p)
-{
- bool ret;
- uv_mutex_lock(&p->mutex);
- ret = p->completed;
- uv_mutex_unlock(&p->mutex);
- return ret;
-}
diff --git a/libnetdata/config/Makefile.am b/libnetdata/config/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/config/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/config/README.md b/libnetdata/config/README.md
deleted file mode 100644
index c3a9d147c..000000000
--- a/libnetdata/config/README.md
+++ /dev/null
@@ -1,58 +0,0 @@
-<!--
-title: "Netdata ini config files"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/config/README.md
-sidebar_label: "Netdata ini config files"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Netdata ini config files
-
-Configuration files `netdata.conf` and `stream.conf` are Netdata ini files.
-
-## Motivation
-
-The whole idea came up when we were evaluating the documentation involved
-in maintaining a complex configuration system. Our intention was to give
-configuration options for everything imaginable. But then, documenting all
-these options would require a tremendous amount of time, users would have
-to search through endless pages for the option they need, etc.
-
-We concluded then that **configuring software like that is a waste of time
-and effort**. Of course there must be plenty of configuration options, but
-the implementation itself should require a lot less effort for both the
-developers and the users.
-
-So, we did this:
-
-1. No configuration is required to run Netdata
-2. There are plenty of options to tweak
-3. There is minimal documentation (or no at all)
-
-## Why this works?
-
-The configuration file is a `name = value` dictionary with `[sections]`.
-Write whatever you like there as long as it follows this simple format.
-
-Netdata loads this dictionary and then when the code needs a value from
-it, it just looks up the `name` in the dictionary at the proper `section`.
-In all places, in the code, there are both the `names` and their
-`default values`, so if something is not found in the configuration
-file, the default is used. The lookup is made using B-Trees and hashes
-(no string comparisons), so they are super fast. Also the `names` of the
-settings can be `my super duper setting that once set to yes, will turn the world upside down = no`
-
-- so goodbye to most of the documentation involved.
-
-Next, Netdata can generate a valid configuration for the user to edit.
-No need to remember anything or copy and paste settings. Just get the
-configuration from the server (`/netdata.conf` on your Netdata server),
-edit it and save it.
-
-Last, what about options you believe you have set, but you misspelled?
-When you get the configuration file from the server, there will be a
-comment above all `name = value` pairs the server does not use.
-So you know that whatever you wrote there, is not used.
-
-
diff --git a/libnetdata/datetime/Makefile.am b/libnetdata/datetime/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/datetime/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/datetime/README.md b/libnetdata/datetime/README.md
deleted file mode 100644
index ba2366316..000000000
--- a/libnetdata/datetime/README.md
+++ /dev/null
@@ -1,11 +0,0 @@
-<!--
-title: "Datetime"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/datetime/README.md
-sidebar_label: "Datetime"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Datetime
-
-Formatting dates and timestamps.
diff --git a/libnetdata/dictionary/Makefile.am b/libnetdata/dictionary/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/dictionary/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/dictionary/README.md b/libnetdata/dictionary/README.md
deleted file mode 100644
index 3b54cf18f..000000000
--- a/libnetdata/dictionary/README.md
+++ /dev/null
@@ -1,235 +0,0 @@
-<!--
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/dictionary/README.md
-sidebar_label: "Dictionaries"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Dictionaries
-
-Netdata dictionaries associate a `name` with a `value`:
-
-- A `name` can be any string.
-- A `value` can be anything.
-
-Such a pair of a `name` and a `value` consists of an `item` or an `entry` in the dictionary.
-
-Dictionaries provide an interface to:
-
-- **Add** an item to the dictionary
-- **Get** an item from the dictionary (provided its `name`)
-- **Delete** an item from the dictionary (provided its `name`)
-- **Traverse** the list of items in the dictionary
-
-Dictionaries are **ordered**, meaning that the order they have been added, is preserved while traversing them. The caller may reverse this order by passing the flag `DICT_OPTION_ADD_IN_FRONT` when creating the dictionary.
-
-Dictionaries guarantee **uniqueness** of all items added to them, meaning that only one item with a given `name` can exist in the dictionary at any given time.
-
-Dictionaries are extremely fast in all operations. They are indexing the keys with `JudyHS` and they utilize a double-linked-list for the traversal operations. Deletion is the most expensive operation, usually somewhat slower than insertion.
-
-## Memory management
-
-Dictionaries come with 2 memory management options:
-
-- **Clone** (copy) the `name` and/or the `value` to memory allocated by the dictionary.
-- **Link** the `name` and/or the `value`, without allocating any memory about them.
-
-In **clone** mode, the dictionary guarantees that all operations on the dictionary items, will automatically take care of the memory used by the `name` and/or the `value`. In case the `value` is an object that needs to have user allocated memory, the following callback functions can be registered:
-
-1. `dictionary_register_insert_callback()` that can be called just after the insertion of an item to the dictionary, or after the replacement of the value of a dictionary item.
-2. `dictionary_register_delete_callback()` that will be called just prior to the deletion of an item from the dictionary, or prior to the replacement of the value of a dictionary item.
-3. `dictionary_register_conflict_callback()` that will be called when `DICT_OPTION_DONT_OVERWRITE_VALUE` is set, and another `value` is attempted to be inserted for the same key.
-4. `dictionary_register_react_callback()` that will be called after the the `insert` and the `conflict` callbacks. The `conflict` callback is called while the dictionary hash table is available for other threads.
-
-In **link** mode, the `name` and/or the `value` are just linked to the dictionary item, and it is the user's responsibility to free the memory they use after an item is deleted from the dictionary or when the dictionary is destroyed.
-
-By default, **clone** mode is used for both the name and the value.
-
-To use **link** mode for names, add `DICT_OPTION_NAME_LINK_DONT_CLONE` to the flags when creating the dictionary.
-
-To use **link** mode for values, add `DICT_OPTION_VALUE_LINK_DONT_CLONE` to the flags when creating the dictionary.
-
-## Locks
-
-The dictionary allows both **single-threaded** operation (no locks - faster) and **multi-threaded** operation utilizing a read-write lock.
-
-The default is **multi-threaded**. To enable **single-threaded** add `DICT_OPTION_SINGLE_THREADED` to the flags when creating the dictionary.
-
-When in **multi-threaded** mode, the dictionaries have 2 independent R/W locks. One for the linked list and one for the hash table (index). An insertion and a deletion will acquire both independently (one after another) for as long as they are needed, but a traversal may hold the the linked list for longer durations. The hash table (index) lock may be acquired while the linked list is acquired, but not the other way around (and the way the code is structured, it is not technically possible to hold and index lock and then lock the linked list one).
-
-These locks are R/W locks. They allow multiple readers, but only one writer.
-
-Unlike POSIX standards, the linked-list lock, allows one writer to lock it multiple times. This has been implemented in such a way, so that a traversal to the items of the dictionary in write-lock mode, allows the writing thread to call `dictionary_set()` or `dictionary_del()`, which alter the dictionary index and the linked list. Especially for the deletion of the currently working item, the dictionary support delayed removal, so it will remove it from the index immediately and mark it as deleted, so that it can be added to the dictionary again with a different value and the traversal will still proceed from the point it was.
-
-## Hash table operations
-
-The dictionary supports the following operations supported by the hash table:
-
-- `dictionary_set()` to add an item to the dictionary, or change its value.
-- `dictionary_get()` and `dictionary_get_and_acquire_item()` to get an item from the dictionary.
-- `dictionary_del()` to delete an item from the dictionary.
-
-For all the calls, there are also `*_advanced()` versions of them, that support more parameters. Check the header file for more information about them.
-
-## Creation and destruction
-
-Use `dictionary_create()` to create a dictionary.
-
-Use `dictionary_destroy()` to destroy a dictionary. When destroyed, a dictionary frees all the memory it has allocated on its own. This can be complemented by the registration of a deletion callback function that can be called upon deletion of each item in the dictionary, which may free additional resources linked to it.
-
-### dictionary_set()
-
-This call is used to:
-
-- **add** an item to the dictionary.
-- **reset** the value of an existing item in the dictionary.
-
-If **resetting** is not desired, add `DICT_OPTION_DONT_OVERWRITE_VALUE` to the flags when creating the dictionary. In this case, `dictionary_set()` will return the value of the original item found in the dictionary instead of resetting it and the value passed to the call will be ignored. Optionally a conflict callback function can be registered, to manipulate (probably merge or extend) the original value, based on the new value attempted to be added to the dictionary.
-
-The format is:
-
-```c
-value = dictionary_set(dict, name, value, value_len);
-```
-
-Where:
-
-* `dict` is a pointer to the dictionary previously created.
-* `name` is a pointer to a string to be used as the key of this item. The name must not be `NULL` and must not be an empty string `""`.
-* `value` is a pointer to the value associated with this item. In **clone** mode, if `value` is `NULL`, a new memory allocation will be made of `value_len` size and will be initialized to zero.
-* `value_len` is the size of the `value` data in bytes. If `value_len` is zero, no allocation will be done and the dictionary item will permanently have the `NULL` value.
-
-### dictionary_get()
-
-This call is used to get the `value` of an item, given its `name`. It utilizes the hash table (index) for making the lookup.
-
-For **multi-threaded** operation, the `dictionary_get()` call gets a shared read lock on the index lock (multiple readers are allowed). The linked-list lock is not used.
-
-In clone mode, the value returned is not guaranteed to be valid, as any other thread may delete the item from the dictionary at any time. To ensure the value will be available, use `dictionary_get_and_acquire_item()`, which uses a reference counter to defer deletes until the item is released with `dictionary_acquired_item_release()`.
-
-The format is:
-
-```c
-value = dictionary_get(dict, name);
-```
-
-Where:
-
-* `dict` is a pointer to the dictionary previously created.
-* `name` is a pointer to a string to be used as the key of this item. The name must not be `NULL` and must not be an empty string `""`.
-
-### dictionary_del()
-
-This call is used to delete an item from the dictionary, given its name.
-
-If there is a deletion callback registered to the dictionary (`dictionary_register_delete_callback()`), it is called prior to the actual deletion of the item.
-
-The format is:
-
-```c
-value = dictionary_del(dict, name);
-```
-
-Where:
-
-* `dict` is a pointer to the dictionary previously created.
-* `name` is a pointer to a string to be used as the key of this item. The name must not be `NULL` and must not be an empty string `""`.
-
-### dictionary_get_and_acquire_item()
-
-This call can be used to search and acquire a dictionary item, while ensuring that it will be available for use, until `dictionary_acquired_item_release()` is called.
-
-This call **does not return the value** of the dictionary item. It returns an internal pointer to a structure that maintains the reference counter used to protect the actual value. To get the value of the item (the same value as returned by `dictionary_get()`), the function `dictionary_acquired_item_value()` has to be called.
-
-Example:
-
-```c
-// create the dictionary
-DICTIONARY *dict = dictionary_create(DICT_OPTION_NONE);
-
-// add an item to it
-dictionary_set(dict, "name", "value", 6);
-
-// find the item we added and acquire it
-const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(dict, "name");
-
-// extract its value
-char *value = (char *)dictionary_acquired_item_value(dict, item);
-
-// now value points to the string "value"
-printf("I got value = '%s'\n", value);
-
-// release the item, so that it can deleted
-dictionary_acquired_item_release(dict, item);
-
-// destroy the dictionary
-dictionary_destroy(dict);
-```
-
-When items are acquired, a reference counter is maintained to keep track of how many users exist for it. If an item with a non-zero number of users is deleted, it is removed from the index, it can be added again to the index (without conflict), and although it exists in the linked-list, it is not offered during traversal. Garbage collection to actually delete the item happens every time another item is added or removed from the linked-list and items are deleted only if no users are using them.
-
-If any item is still acquired when the dictionary is destroyed, the destruction of the dictionary is also deferred until all the acquired items are released. When the dictionary is destroyed like that, all operations on the dictionary fail (traversals do not traverse, insertions do not insert, deletions do not delete, searches do not find any items, etc). Once the last item in the dictionary is released, the dictionary is automatically destroyed too.
-
-## Traversal
-
-Dictionaries offer 3 ways to traverse the entire dictionary:
-
-- **walkthrough**, implemented by setting a callback function to be called for every item.
-- **sorted walkthrough**, which first sorts the dictionary and then call a callback function for every item.
-- **foreach**, a way to traverse the dictionary with a for-next loop.
-
-All these methods are available in **read**, **write**, or **reentrant** mode. In **read** mode only lookups are allowed to the dictionary. In **write** lookups but also insertions and deletions are allowed, and in **reentrant** mode the dictionary is unlocked outside dictionary code.
-
-### walkthrough (callback)
-
-There are 4 calls:
-
-- `dictionary_walkthrough_read()` and `dictionary_sorted_walkthrough_read()` acquire a shared read lock on the linked-list, and they call a callback function for every item of the dictionary.
-- `dictionary_walkthrough_write()` and `dictionary_sorted_walkthrough_write()` acquire a write lock on the linked-list, and they call a callback function for every item of the dictionary. This is to be used when items need to be added to or removed from the dictionary. The `write` versions can be used to delete any or all the items from the dictionary, including the currently working one. For the `sorted` version, all items in the dictionary maintain a reference counter, so all deletions are deferred until the sorted walkthrough finishes.
-
-The non sorted versions traverse the items in the same order they have been added to the dictionary (or the reverse order if the flag `DICT_OPTION_ADD_IN_FRONT` is set during dictionary creation). The sorted versions sort alphabetically the items based on their name, and then they traverse them in the sorted order.
-
-The callback function returns an `int`. If this value is negative, traversal of the dictionary is stopped immediately and the negative value is returned to the caller. If the returned value of all callback calls is zero or positive, the walkthrough functions return the sum of the return values of all callbacks. So, if you are just interested to know how many items fall into some condition, write a callback function that returns 1 when the item satisfies that condition and 0 when it does not and the walkthrough function will return how many tested positive.
-
-### foreach (for-next loop)
-
-The following is a snippet of such a loop:
-
-```c
-MY_STRUCTURE *x;
-dfe_start_read(dict, x) {
- printf("hey, I got an item named '%s' with value ptr %08X", x_dfe.name, x);
-}
-dfe_done(x);
-```
-
-The `x` parameter gives the name of the pointer to be used while iterating the items. Any name is accepted. `x` points to the `value` of the item in the dictionary.
-
-The `x_dfe.name` is a variable that is automatically created, by concatenating whatever is given as `x` and `_dfe`. It is an object and it has a few members, including `x_dfe.counter` that counts the iterations made so far, `x_dfe.item` that provides the acquired item from the dictionary and which can be used to pass it over for further processing, etc. Check the header file for more info. So, if you call `dfe_start_read(dict, myvar)`, the name will be `myvar_dfe`.
-
-Both `dfe_start_read(dict, item)` and `dfe_done(item)` are together inside a `do { ... } while(0)` loop, so that the following will work:
-
-```c
-MY_ITEM *item;
-
-if(a = 1)
- // do {
- dfe_start_read(dict, x)
- printf("hey, I got an item named '%s' with value ptr %08X", x_dfe.name, x);
- dfe_done(x);
- // } while(0);
-else
- something else;
-```
-
-In the above, the `if(a == 1)` condition will work as expected. It will do the foreach loop when a is 1, otherwise it will run `something else`.
-
-There are 2 versions of `dfe_start`:
-
-- `dfe_start_read()` that acquires a shared read linked-list lock to the dictionary.
-- `dfe_start_write()` that acquires an exclusive write linked-list lock to the dictionary.
-
-While in the loop, depending on the read or write versions of `dfe_start`, the caller may lookup or manipulate the dictionary. The rules are the same with the unsorted walkthrough callback functions.
-
-PS: DFE is Dictionary For Each.
diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c
deleted file mode 100644
index 45922abf0..000000000
--- a/libnetdata/dictionary/dictionary.c
+++ /dev/null
@@ -1,3799 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#define DICTIONARY_INTERNALS
-
-#include "../libnetdata.h"
-
-// runtime flags of the dictionary - must be checked with atomics
-typedef enum __attribute__ ((__packed__)) {
- DICT_FLAG_NONE = 0,
- DICT_FLAG_DESTROYED = (1 << 0), // this dictionary has been destroyed
-} DICT_FLAGS;
-
-#define dict_flag_check(dict, flag) (__atomic_load_n(&((dict)->flags), __ATOMIC_RELAXED) & (flag))
-#define dict_flag_set(dict, flag) __atomic_or_fetch(&((dict)->flags), flag, __ATOMIC_RELAXED)
-#define dict_flag_clear(dict, flag) __atomic_and_fetch(&((dict)->flags), ~(flag), __ATOMIC_RELAXED)
-
-// flags macros
-#define is_dictionary_destroyed(dict) dict_flag_check(dict, DICT_FLAG_DESTROYED)
-
-// configuration options macros
-#define is_dictionary_single_threaded(dict) ((dict)->options & DICT_OPTION_SINGLE_THREADED)
-#define is_view_dictionary(dict) ((dict)->master)
-#define is_master_dictionary(dict) (!is_view_dictionary(dict))
-
-typedef enum __attribute__ ((__packed__)) item_options {
- ITEM_OPTION_NONE = 0,
- ITEM_OPTION_ALLOCATED_NAME = (1 << 0), // the name pointer is a STRING
-
- // IMPORTANT: This is 1-bit - to add more change ITEM_OPTIONS_BITS
-} ITEM_OPTIONS;
-
-typedef enum __attribute__ ((__packed__)) item_flags {
- ITEM_FLAG_NONE = 0,
- ITEM_FLAG_DELETED = (1 << 0), // this item is marked deleted, so it is not available for traversal (deleted from the index too)
- ITEM_FLAG_BEING_CREATED = (1 << 1), // this item is currently being created - this flag is removed when construction finishes
-
- // IMPORTANT: This is 8-bit
-} ITEM_FLAGS;
-
-#define item_flag_check(item, flag) (__atomic_load_n(&((item)->flags), __ATOMIC_RELAXED) & (flag))
-#define item_flag_set(item, flag) __atomic_or_fetch(&((item)->flags), flag, __ATOMIC_RELAXED)
-#define item_flag_clear(item, flag) __atomic_and_fetch(&((item)->flags), ~(flag), __ATOMIC_RELAXED)
-
-#define item_shared_flag_check(item, flag) (__atomic_load_n(&((item)->shared->flags), __ATOMIC_RELAXED) & (flag))
-#define item_shared_flag_set(item, flag) __atomic_or_fetch(&((item)->shared->flags), flag, __ATOMIC_RELAXED)
-#define item_shared_flag_clear(item, flag) __atomic_and_fetch(&((item)->shared->flags), ~(flag), __ATOMIC_RELAXED)
-
-#define REFCOUNT_DELETING (-100)
-
-#define ITEM_FLAGS_TYPE uint8_t
-#define KEY_LEN_TYPE uint32_t
-#define VALUE_LEN_TYPE uint32_t
-
-#define ITEM_OPTIONS_BITS 1
-#define KEY_LEN_BITS ((sizeof(KEY_LEN_TYPE) * 8) - (sizeof(ITEM_FLAGS_TYPE) * 8) - ITEM_OPTIONS_BITS)
-#define KEY_LEN_MAX ((1 << KEY_LEN_BITS) - 1)
-
-#define VALUE_LEN_BITS ((sizeof(VALUE_LEN_TYPE) * 8) - (sizeof(ITEM_FLAGS_TYPE) * 8))
-#define VALUE_LEN_MAX ((1 << VALUE_LEN_BITS) - 1)
-
-
-/*
- * Every item in the dictionary has the following structure.
- */
-
-typedef int32_t REFCOUNT;
-
-typedef struct dictionary_item_shared {
- void *value; // the value of the dictionary item
-
- // the order of the following items is important!
- // The total of their storage should be 64-bits
-
- REFCOUNT links; // how many links this item has
- VALUE_LEN_TYPE value_len:VALUE_LEN_BITS; // the size of the value
- ITEM_FLAGS_TYPE flags; // shared flags
-} DICTIONARY_ITEM_SHARED;
-
-struct dictionary_item {
-#ifdef NETDATA_INTERNAL_CHECKS
- DICTIONARY *dict;
- pid_t creator_pid;
- pid_t deleter_pid;
- pid_t ll_adder_pid;
- pid_t ll_remover_pid;
-#endif
-
- DICTIONARY_ITEM_SHARED *shared;
-
- struct dictionary_item *next; // a double linked list to allow fast insertions and deletions
- struct dictionary_item *prev;
-
- union {
- STRING *string_name; // the name of the dictionary item
- char *caller_name; // the user supplied string pointer
-// void *key_ptr; // binary key pointer
- };
-
- // the order of the following items is important!
- // The total of their storage should be 64-bits
-
- REFCOUNT refcount; // the private reference counter
-
- KEY_LEN_TYPE key_len:KEY_LEN_BITS; // the size of key indexed (for strings, including the null terminator)
- // this is (2^23 - 1) = 8.388.607 bytes max key length.
-
- ITEM_OPTIONS options:ITEM_OPTIONS_BITS; // permanent configuration options
- // (no atomic operations on this - they never change)
-
- ITEM_FLAGS_TYPE flags; // runtime changing flags for this item (atomic operations on this)
- // cannot be a bit field because of atomics.
-};
-
-struct dictionary_hooks {
- REFCOUNT links;
- usec_t last_master_deletion_us;
-
- void (*ins_callback)(const DICTIONARY_ITEM *item, void *value, void *data);
- void *ins_callback_data;
-
- bool (*conflict_callback)(const DICTIONARY_ITEM *item, void *old_value, void *new_value, void *data);
- void *conflict_callback_data;
-
- void (*react_callback)(const DICTIONARY_ITEM *item, void *value, void *data);
- void *react_callback_data;
-
- void (*del_callback)(const DICTIONARY_ITEM *item, void *value, void *data);
- void *del_callback_data;
-};
-
-struct dictionary_stats dictionary_stats_category_other = {
- .name = "other",
-};
-
-struct dictionary {
-#ifdef NETDATA_INTERNAL_CHECKS
- const char *creation_function;
- const char *creation_file;
- size_t creation_line;
- pid_t creation_tid;
-#endif
-
- usec_t last_gc_run_us;
- DICT_OPTIONS options; // the configuration flags of the dictionary (they never change - no atomics)
- DICT_FLAGS flags; // run time flags for the dictionary (they change all the time - atomics needed)
-
- ARAL *value_aral;
-
- struct { // support for multiple indexing engines
- Pvoid_t JudyHSArray; // the hash table
- RW_SPINLOCK rw_spinlock; // protect the index
- } index;
-
- struct {
- DICTIONARY_ITEM *list; // the double linked list of all items in the dictionary
- RW_SPINLOCK rw_spinlock; // protect the linked-list
- pid_t writer_pid; // the gettid() of the writer
- uint32_t writer_depth; // nesting of write locks
- } items;
-
- struct dictionary_hooks *hooks; // pointer to external function callbacks to be called at certain points
- struct dictionary_stats *stats; // statistics data, when DICT_OPTION_STATS is set
-
- DICTIONARY *master; // the master dictionary
- DICTIONARY *next; // linked list for delayed destruction (garbage collection of whole dictionaries)
-
- uint32_t version; // the current version of the dictionary
- // it is incremented when:
- // - item added
- // - item removed
- // - item value reset
- // - conflict callback returns true
- // - function dictionary_version_increment() is called
-
- int32_t entries; // how many items are currently in the index (the linked list may have more)
- int32_t referenced_items; // how many items of the dictionary are currently being used by 3rd parties
- int32_t pending_deletion_items; // how many items of the dictionary have been deleted, but have not been removed yet
-
-#ifdef NETDATA_DICTIONARY_VALIDATE_POINTERS
- netdata_mutex_t global_pointer_registry_mutex;
- Pvoid_t global_pointer_registry;
-#endif
-};
-
-// ----------------------------------------------------------------------------
-// forward definitions of functions used in reverse order in the code
-
-static void garbage_collect_pending_deletes(DICTIONARY *dict);
-static inline void item_linked_list_remove(DICTIONARY *dict, DICTIONARY_ITEM *item);
-static size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item);
-static inline const char *item_get_name(const DICTIONARY_ITEM *item);
-static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *item);
-static void item_release(DICTIONARY *dict, DICTIONARY_ITEM *item);
-static bool dict_item_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item);
-
-#define RC_ITEM_OK ( 0)
-#define RC_ITEM_MARKED_FOR_DELETION (-1) // the item is marked for deletion
-#define RC_ITEM_IS_CURRENTLY_BEING_DELETED (-2) // the item is currently being deleted
-#define RC_ITEM_IS_CURRENTLY_BEING_CREATED (-3) // the item is currently being deleted
-#define RC_ITEM_IS_REFERENCED (-4) // the item is currently referenced
-#define item_check_and_acquire(dict, item) (item_check_and_acquire_advanced(dict, item, false) == RC_ITEM_OK)
-static int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_ITEM *item, bool having_index_lock);
-#define item_is_not_referenced_and_can_be_removed(dict, item) (item_is_not_referenced_and_can_be_removed_advanced(dict, item) == RC_ITEM_OK)
-static inline int item_is_not_referenced_and_can_be_removed_advanced(DICTIONARY *dict, DICTIONARY_ITEM *item);
-
-// ----------------------------------------------------------------------------
-// validate each pointer is indexed once - internal checks only
-
-#ifdef NETDATA_DICTIONARY_VALIDATE_POINTERS
-static inline void pointer_index_init(DICTIONARY *dict __maybe_unused) {
- netdata_mutex_init(&dict->global_pointer_registry_mutex);
-}
-
-static inline void pointer_destroy_index(DICTIONARY *dict __maybe_unused) {
- netdata_mutex_lock(&dict->global_pointer_registry_mutex);
- JudyHSFreeArray(&dict->global_pointer_registry, PJE0);
- netdata_mutex_unlock(&dict->global_pointer_registry_mutex);
-}
-static inline void pointer_add(DICTIONARY *dict __maybe_unused, DICTIONARY_ITEM *item __maybe_unused) {
- netdata_mutex_lock(&dict->global_pointer_registry_mutex);
- Pvoid_t *PValue = JudyHSIns(&dict->global_pointer_registry, &item, sizeof(void *), PJE0);
- if(*PValue != NULL)
- fatal("pointer already exists in registry");
- *PValue = item;
- netdata_mutex_unlock(&dict->global_pointer_registry_mutex);
-}
-
-static inline void pointer_check(DICTIONARY *dict __maybe_unused, DICTIONARY_ITEM *item __maybe_unused) {
- netdata_mutex_lock(&dict->global_pointer_registry_mutex);
- Pvoid_t *PValue = JudyHSGet(dict->global_pointer_registry, &item, sizeof(void *));
- if(PValue == NULL)
- fatal("pointer is not found in registry");
- netdata_mutex_unlock(&dict->global_pointer_registry_mutex);
-}
-
-static inline void pointer_del(DICTIONARY *dict __maybe_unused, DICTIONARY_ITEM *item __maybe_unused) {
- netdata_mutex_lock(&dict->global_pointer_registry_mutex);
- int ret = JudyHSDel(&dict->global_pointer_registry, &item, sizeof(void *), PJE0);
- if(!ret)
- fatal("pointer to be deleted does not exist in registry");
- netdata_mutex_unlock(&dict->global_pointer_registry_mutex);
-}
-#else // !NETDATA_DICTIONARY_VALIDATE_POINTERS
-#define pointer_index_init(dict) debug_dummy()
-#define pointer_destroy_index(dict) debug_dummy()
-#define pointer_add(dict, item) debug_dummy()
-#define pointer_check(dict, item) debug_dummy()
-#define pointer_del(dict, item) debug_dummy()
-#endif // !NETDATA_DICTIONARY_VALIDATE_POINTERS
-
-// ----------------------------------------------------------------------------
-// memory statistics
-
-#ifdef DICT_WITH_STATS
-static inline void DICTIONARY_STATS_PLUS_MEMORY(DICTIONARY *dict, size_t key_size, size_t item_size, size_t value_size) {
- if(key_size)
- __atomic_fetch_add(&dict->stats->memory.index, (long)JUDYHS_INDEX_SIZE_ESTIMATE(key_size), __ATOMIC_RELAXED);
-
- if(item_size)
- __atomic_fetch_add(&dict->stats->memory.dict, (long)item_size, __ATOMIC_RELAXED);
-
- if(value_size)
- __atomic_fetch_add(&dict->stats->memory.values, (long)value_size, __ATOMIC_RELAXED);
-}
-
-static inline void DICTIONARY_STATS_MINUS_MEMORY(DICTIONARY *dict, size_t key_size, size_t item_size, size_t value_size) {
- if(key_size)
- __atomic_fetch_sub(&dict->stats->memory.index, (long)JUDYHS_INDEX_SIZE_ESTIMATE(key_size), __ATOMIC_RELAXED);
-
- if(item_size)
- __atomic_fetch_sub(&dict->stats->memory.dict, (long)item_size, __ATOMIC_RELAXED);
-
- if(value_size)
- __atomic_fetch_sub(&dict->stats->memory.values, (long)value_size, __ATOMIC_RELAXED);
-}
-#else
-#define DICTIONARY_STATS_PLUS_MEMORY(dict, key_size, item_size, value_size) do {;} while(0)
-#define DICTIONARY_STATS_MINUS_MEMORY(dict, key_size, item_size, value_size) do {;} while(0)
-#endif
-
-// ----------------------------------------------------------------------------
-// callbacks registration
-
-static inline void dictionary_hooks_allocate(DICTIONARY *dict) {
- if(dict->hooks) return;
-
- dict->hooks = callocz(1, sizeof(struct dictionary_hooks));
- dict->hooks->links = 1;
-
- DICTIONARY_STATS_PLUS_MEMORY(dict, 0, sizeof(struct dictionary_hooks), 0);
-}
-
-static inline size_t dictionary_hooks_free(DICTIONARY *dict) {
- if(!dict->hooks) return 0;
-
- REFCOUNT links = __atomic_sub_fetch(&dict->hooks->links, 1, __ATOMIC_ACQUIRE);
- if(links == 0) {
- freez(dict->hooks);
- dict->hooks = NULL;
-
- DICTIONARY_STATS_MINUS_MEMORY(dict, 0, sizeof(struct dictionary_hooks), 0);
- return sizeof(struct dictionary_hooks);
- }
-
- return 0;
-}
-
-void dictionary_register_insert_callback(DICTIONARY *dict, void (*ins_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data) {
- if(unlikely(is_view_dictionary(dict)))
- fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ );
-
- dictionary_hooks_allocate(dict);
- dict->hooks->ins_callback = ins_callback;
- dict->hooks->ins_callback_data = data;
-}
-
-void dictionary_register_conflict_callback(DICTIONARY *dict, bool (*conflict_callback)(const DICTIONARY_ITEM *item, void *old_value, void *new_value, void *data), void *data) {
- if(unlikely(is_view_dictionary(dict)))
- fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ );
-
- internal_error(!(dict->options & DICT_OPTION_DONT_OVERWRITE_VALUE), "DICTIONARY: registering conflict callback without DICT_OPTION_DONT_OVERWRITE_VALUE");
- dict->options |= DICT_OPTION_DONT_OVERWRITE_VALUE;
-
- dictionary_hooks_allocate(dict);
- dict->hooks->conflict_callback = conflict_callback;
- dict->hooks->conflict_callback_data = data;
-}
-
-void dictionary_register_react_callback(DICTIONARY *dict, void (*react_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data) {
- if(unlikely(is_view_dictionary(dict)))
- fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ );
-
- dictionary_hooks_allocate(dict);
- dict->hooks->react_callback = react_callback;
- dict->hooks->react_callback_data = data;
-}
-
-void dictionary_register_delete_callback(DICTIONARY *dict, void (*del_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data) {
- if(unlikely(is_view_dictionary(dict)))
- fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ );
-
- dictionary_hooks_allocate(dict);
- dict->hooks->del_callback = del_callback;
- dict->hooks->del_callback_data = data;
-}
-
-// ----------------------------------------------------------------------------
-// dictionary statistics API
-
-size_t dictionary_version(DICTIONARY *dict) {
- if(unlikely(!dict)) return 0;
-
- // this is required for views to return the right number
- // garbage_collect_pending_deletes(dict);
-
- return __atomic_load_n(&dict->version, __ATOMIC_RELAXED);
-}
-size_t dictionary_entries(DICTIONARY *dict) {
- if(unlikely(!dict)) return 0;
-
- // this is required for views to return the right number
- // garbage_collect_pending_deletes(dict);
-
- long int entries = __atomic_load_n(&dict->entries, __ATOMIC_RELAXED);
- internal_fatal(entries < 0, "DICTIONARY: entries is negative: %ld", entries);
-
- return entries;
-}
-size_t dictionary_referenced_items(DICTIONARY *dict) {
- if(unlikely(!dict)) return 0;
-
- long int referenced_items = __atomic_load_n(&dict->referenced_items, __ATOMIC_RELAXED);
- if(referenced_items < 0)
- fatal("DICTIONARY: referenced items is negative: %ld", referenced_items);
-
- return referenced_items;
-}
-
-void dictionary_version_increment(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED);
-}
-
-// ----------------------------------------------------------------------------
-// internal statistics API
-
-#ifdef DICT_WITH_STATS
-static inline void DICTIONARY_STATS_SEARCHES_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->ops.searches, 1, __ATOMIC_RELAXED);
-}
-#else
-#define DICTIONARY_STATS_SEARCHES_PLUS1(dict) do {;} while(0)
-#endif
-
-static inline void DICTIONARY_ENTRIES_PLUS1(DICTIONARY *dict) {
-#ifdef DICT_WITH_STATS
- // statistics
- __atomic_fetch_add(&dict->stats->items.entries, 1, __ATOMIC_RELAXED);
- __atomic_fetch_add(&dict->stats->items.referenced, 1, __ATOMIC_RELAXED);
- __atomic_fetch_add(&dict->stats->ops.inserts, 1, __ATOMIC_RELAXED);
-#endif
-
- if(unlikely(is_dictionary_single_threaded(dict))) {
- dict->version++;
- dict->entries++;
- dict->referenced_items++;
-
- }
- else {
- __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED);
- __atomic_fetch_add(&dict->entries, 1, __ATOMIC_RELAXED);
- __atomic_fetch_add(&dict->referenced_items, 1, __ATOMIC_RELAXED);
- }
-}
-
-static inline void DICTIONARY_ENTRIES_MINUS1(DICTIONARY *dict) {
-#ifdef DICT_WITH_STATS
- // statistics
- __atomic_fetch_add(&dict->stats->ops.deletes, 1, __ATOMIC_RELAXED);
- __atomic_fetch_sub(&dict->stats->items.entries, 1, __ATOMIC_RELAXED);
-#endif
-
- size_t entries; (void)entries;
- if(unlikely(is_dictionary_single_threaded(dict))) {
- dict->version++;
- entries = dict->entries--;
- }
- else {
- __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED);
- entries = __atomic_fetch_sub(&dict->entries, 1, __ATOMIC_RELAXED);
- }
-
- internal_fatal(entries == 0,
- "DICT: negative number of entries in dictionary created from %s() (%zu@%s)",
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
-}
-
-static inline void DICTIONARY_VALUE_RESETS_PLUS1(DICTIONARY *dict) {
-#ifdef DICT_WITH_STATS
- __atomic_fetch_add(&dict->stats->ops.resets, 1, __ATOMIC_RELAXED);
-#endif
-
- if(unlikely(is_dictionary_single_threaded(dict)))
- dict->version++;
- else
- __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED);
-}
-
-#ifdef DICT_WITH_STATS
-static inline void DICTIONARY_STATS_TRAVERSALS_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->ops.traversals, 1, __ATOMIC_RELAXED);
-}
-static inline void DICTIONARY_STATS_WALKTHROUGHS_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->ops.walkthroughs, 1, __ATOMIC_RELAXED);
-}
-static inline void DICTIONARY_STATS_CHECK_SPINS_PLUS(DICTIONARY *dict, size_t count) {
- __atomic_fetch_add(&dict->stats->spin_locks.use_spins, count, __ATOMIC_RELAXED);
-}
-static inline void DICTIONARY_STATS_INSERT_SPINS_PLUS(DICTIONARY *dict, size_t count) {
- __atomic_fetch_add(&dict->stats->spin_locks.insert_spins, count, __ATOMIC_RELAXED);
-}
-static inline void DICTIONARY_STATS_DELETE_SPINS_PLUS(DICTIONARY *dict, size_t count) {
- __atomic_fetch_add(&dict->stats->spin_locks.delete_spins, count, __ATOMIC_RELAXED);
-}
-static inline void DICTIONARY_STATS_SEARCH_IGNORES_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->spin_locks.search_spins, 1, __ATOMIC_RELAXED);
-}
-static inline void DICTIONARY_STATS_CALLBACK_INSERTS_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->callbacks.inserts, 1, __ATOMIC_RELEASE);
-}
-static inline void DICTIONARY_STATS_CALLBACK_CONFLICTS_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->callbacks.conflicts, 1, __ATOMIC_RELEASE);
-}
-static inline void DICTIONARY_STATS_CALLBACK_REACTS_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->callbacks.reacts, 1, __ATOMIC_RELEASE);
-}
-static inline void DICTIONARY_STATS_CALLBACK_DELETES_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->callbacks.deletes, 1, __ATOMIC_RELEASE);
-}
-static inline void DICTIONARY_STATS_GARBAGE_COLLECTIONS_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->ops.garbage_collections, 1, __ATOMIC_RELAXED);
-}
-static inline void DICTIONARY_STATS_DICT_CREATIONS_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->dictionaries.active, 1, __ATOMIC_RELAXED);
- __atomic_fetch_add(&dict->stats->ops.creations, 1, __ATOMIC_RELAXED);
-}
-static inline void DICTIONARY_STATS_DICT_DESTRUCTIONS_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_sub(&dict->stats->dictionaries.active, 1, __ATOMIC_RELAXED);
- __atomic_fetch_add(&dict->stats->ops.destructions, 1, __ATOMIC_RELAXED);
-}
-static inline void DICTIONARY_STATS_DICT_DESTROY_QUEUED_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->dictionaries.deleted, 1, __ATOMIC_RELAXED);
-}
-static inline void DICTIONARY_STATS_DICT_DESTROY_QUEUED_MINUS1(DICTIONARY *dict) {
- __atomic_fetch_sub(&dict->stats->dictionaries.deleted, 1, __ATOMIC_RELAXED);
-}
-static inline void DICTIONARY_STATS_DICT_FLUSHES_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->ops.flushes, 1, __ATOMIC_RELAXED);
-}
-#else
-#define DICTIONARY_STATS_TRAVERSALS_PLUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_CHECK_SPINS_PLUS(dict, count) do {;} while(0)
-#define DICTIONARY_STATS_INSERT_SPINS_PLUS(dict, count) do {;} while(0)
-#define DICTIONARY_STATS_DELETE_SPINS_PLUS(dict, count) do {;} while(0)
-#define DICTIONARY_STATS_SEARCH_IGNORES_PLUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_CALLBACK_INSERTS_PLUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_CALLBACK_CONFLICTS_PLUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_CALLBACK_REACTS_PLUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_CALLBACK_DELETES_PLUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_GARBAGE_COLLECTIONS_PLUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_DICT_CREATIONS_PLUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_DICT_DESTRUCTIONS_PLUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_DICT_DESTROY_QUEUED_PLUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_DICT_DESTROY_QUEUED_MINUS1(dict) do {;} while(0)
-#define DICTIONARY_STATS_DICT_FLUSHES_PLUS1(dict) do {;} while(0)
-#endif
-
-static inline void DICTIONARY_REFERENCED_ITEMS_PLUS1(DICTIONARY *dict) {
-#ifdef DICT_WITH_STATS
- __atomic_fetch_add(&dict->stats->items.referenced, 1, __ATOMIC_RELAXED);
-#endif
-
- if(unlikely(is_dictionary_single_threaded(dict)))
- ++dict->referenced_items;
- else
- __atomic_add_fetch(&dict->referenced_items, 1, __ATOMIC_RELAXED);
-}
-
-static inline void DICTIONARY_REFERENCED_ITEMS_MINUS1(DICTIONARY *dict) {
-#ifdef DICT_WITH_STATS
- __atomic_fetch_sub(&dict->stats->items.referenced, 1, __ATOMIC_RELAXED);
-#endif
-
- long int referenced_items; (void)referenced_items;
- if(unlikely(is_dictionary_single_threaded(dict)))
- referenced_items = --dict->referenced_items;
- else
- referenced_items = __atomic_sub_fetch(&dict->referenced_items, 1, __ATOMIC_SEQ_CST);
-
- internal_fatal(referenced_items < 0,
- "DICT: negative number of referenced items (%ld) in dictionary created from %s() (%zu@%s)",
- referenced_items,
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
-}
-
-static inline void DICTIONARY_PENDING_DELETES_PLUS1(DICTIONARY *dict) {
-#ifdef DICT_WITH_STATS
- __atomic_fetch_add(&dict->stats->items.pending_deletion, 1, __ATOMIC_RELAXED);
-#endif
-
- if(unlikely(is_dictionary_single_threaded(dict)))
- ++dict->pending_deletion_items;
- else
- __atomic_add_fetch(&dict->pending_deletion_items, 1, __ATOMIC_RELEASE);
-}
-
-static inline long int DICTIONARY_PENDING_DELETES_MINUS1(DICTIONARY *dict) {
-#ifdef DICT_WITH_STATS
- __atomic_fetch_sub(&dict->stats->items.pending_deletion, 1, __ATOMIC_RELEASE);
-#endif
-
- if(unlikely(is_dictionary_single_threaded(dict)))
- return --dict->pending_deletion_items;
- else
- return __atomic_sub_fetch(&dict->pending_deletion_items, 1, __ATOMIC_ACQUIRE);
-}
-
-static inline long int DICTIONARY_PENDING_DELETES_GET(DICTIONARY *dict) {
- if(unlikely(is_dictionary_single_threaded(dict)))
- return dict->pending_deletion_items;
- else
- return __atomic_load_n(&dict->pending_deletion_items, __ATOMIC_SEQ_CST);
-}
-
-static inline REFCOUNT DICTIONARY_ITEM_REFCOUNT_GET(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- if(unlikely(dict && is_dictionary_single_threaded(dict))) // this is an exception, dict can be null
- return item->refcount;
- else
- return (REFCOUNT)__atomic_load_n(&item->refcount, __ATOMIC_ACQUIRE);
-}
-
-static inline REFCOUNT DICTIONARY_ITEM_REFCOUNT_GET_SOLE(DICTIONARY_ITEM *item) {
- return (REFCOUNT)__atomic_load_n(&item->refcount, __ATOMIC_ACQUIRE);
-}
-
-// ----------------------------------------------------------------------------
-// callbacks execution
-
-static void dictionary_execute_insert_callback(DICTIONARY *dict, DICTIONARY_ITEM *item, void *constructor_data) {
- if(likely(!dict->hooks || !dict->hooks->ins_callback))
- return;
-
- if(unlikely(is_view_dictionary(dict)))
- fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ );
-
- internal_error(false,
- "DICTIONARY: Running insert callback on item '%s' of dictionary created from %s() %zu@%s.",
- item_get_name(item),
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
-
- dict->hooks->ins_callback(item, item->shared->value, constructor_data?constructor_data:dict->hooks->ins_callback_data);
- DICTIONARY_STATS_CALLBACK_INSERTS_PLUS1(dict);
-}
-
-static bool dictionary_execute_conflict_callback(DICTIONARY *dict, DICTIONARY_ITEM *item, void *new_value, void *constructor_data) {
- if(likely(!dict->hooks || !dict->hooks->conflict_callback))
- return false;
-
- if(unlikely(is_view_dictionary(dict)))
- fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ );
-
- internal_error(false,
- "DICTIONARY: Running conflict callback on item '%s' of dictionary created from %s() %zu@%s.",
- item_get_name(item),
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
-
- bool ret = dict->hooks->conflict_callback(
- item, item->shared->value, new_value,
- constructor_data ? constructor_data : dict->hooks->conflict_callback_data);
-
- DICTIONARY_STATS_CALLBACK_CONFLICTS_PLUS1(dict);
-
- return ret;
-}
-
-static void dictionary_execute_react_callback(DICTIONARY *dict, DICTIONARY_ITEM *item, void *constructor_data) {
- if(likely(!dict->hooks || !dict->hooks->react_callback))
- return;
-
- if(unlikely(is_view_dictionary(dict)))
- fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ );
-
- internal_error(false,
- "DICTIONARY: Running react callback on item '%s' of dictionary created from %s() %zu@%s.",
- item_get_name(item),
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
-
- dict->hooks->react_callback(item, item->shared->value,
- constructor_data?constructor_data:dict->hooks->react_callback_data);
-
- DICTIONARY_STATS_CALLBACK_REACTS_PLUS1(dict);
-}
-
-static void dictionary_execute_delete_callback(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- if(likely(!dict->hooks || !dict->hooks->del_callback))
- return;
-
- // We may execute delete callback on items deleted from a view,
- // because we may have references to it, after the master is gone
- // so, the shared structure will remain until the last reference is released.
-
- internal_error(false,
- "DICTIONARY: Running delete callback on item '%s' of dictionary created from %s() %zu@%s.",
- item_get_name(item),
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
-
- dict->hooks->del_callback(item, item->shared->value, dict->hooks->del_callback_data);
-
- DICTIONARY_STATS_CALLBACK_DELETES_PLUS1(dict);
-}
-
-// ----------------------------------------------------------------------------
-// dictionary locks
-
-static inline size_t dictionary_locks_init(DICTIONARY *dict) {
- if(likely(!is_dictionary_single_threaded(dict))) {
- rw_spinlock_init(&dict->index.rw_spinlock);
- rw_spinlock_init(&dict->items.rw_spinlock);
- }
-
- return 0;
-}
-
-static inline size_t dictionary_locks_destroy(DICTIONARY *dict __maybe_unused) {
- return 0;
-}
-
-static inline void ll_recursive_lock_set_thread_as_writer(DICTIONARY *dict) {
- pid_t expected = 0, desired = gettid();
- if(!__atomic_compare_exchange_n(&dict->items.writer_pid, &expected, desired, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED))
- fatal("DICTIONARY: Cannot set thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_RELAXED));
-}
-
-static inline void ll_recursive_unlock_unset_thread_writer(DICTIONARY *dict) {
- pid_t expected = gettid(), desired = 0;
- if(!__atomic_compare_exchange_n(&dict->items.writer_pid, &expected, desired, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED))
- fatal("DICTIONARY: Cannot unset thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_RELAXED));
-}
-
-static inline bool ll_recursive_lock_is_thread_the_writer(DICTIONARY *dict) {
- pid_t tid = gettid();
- return tid > 0 && tid == __atomic_load_n(&dict->items.writer_pid, __ATOMIC_RELAXED);
-}
-
-static inline void ll_recursive_lock(DICTIONARY *dict, char rw) {
- if(unlikely(is_dictionary_single_threaded(dict)))
- return;
-
- if(ll_recursive_lock_is_thread_the_writer(dict)) {
- dict->items.writer_depth++;
- return;
- }
-
- if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') {
- // read lock
- rw_spinlock_read_lock(&dict->items.rw_spinlock);
- }
- else {
- // write lock
- rw_spinlock_write_lock(&dict->items.rw_spinlock);
- ll_recursive_lock_set_thread_as_writer(dict);
- }
-}
-
-static inline void ll_recursive_unlock(DICTIONARY *dict, char rw) {
- if(unlikely(is_dictionary_single_threaded(dict)))
- return;
-
- if(ll_recursive_lock_is_thread_the_writer(dict) && dict->items.writer_depth > 0) {
- dict->items.writer_depth--;
- return;
- }
-
- if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') {
- // read unlock
-
- rw_spinlock_read_unlock(&dict->items.rw_spinlock);
- }
- else {
- // write unlock
-
- ll_recursive_unlock_unset_thread_writer(dict);
-
- rw_spinlock_write_unlock(&dict->items.rw_spinlock);
- }
-}
-
-inline void dictionary_write_lock(DICTIONARY *dict) {
- ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
-}
-inline void dictionary_write_unlock(DICTIONARY *dict) {
- ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
-}
-
-static inline void dictionary_index_lock_rdlock(DICTIONARY *dict) {
- if(unlikely(is_dictionary_single_threaded(dict)))
- return;
-
- rw_spinlock_read_lock(&dict->index.rw_spinlock);
-}
-
-static inline void dictionary_index_rdlock_unlock(DICTIONARY *dict) {
- if(unlikely(is_dictionary_single_threaded(dict)))
- return;
-
- rw_spinlock_read_unlock(&dict->index.rw_spinlock);
-}
-
-static inline void dictionary_index_lock_wrlock(DICTIONARY *dict) {
- if(unlikely(is_dictionary_single_threaded(dict)))
- return;
-
- rw_spinlock_write_lock(&dict->index.rw_spinlock);
-}
-static inline void dictionary_index_wrlock_unlock(DICTIONARY *dict) {
- if(unlikely(is_dictionary_single_threaded(dict)))
- return;
-
- rw_spinlock_write_unlock(&dict->index.rw_spinlock);
-}
-
-// ----------------------------------------------------------------------------
-// items garbage collector
-
-static void garbage_collect_pending_deletes(DICTIONARY *dict) {
- usec_t last_master_deletion_us = dict->hooks?__atomic_load_n(&dict->hooks->last_master_deletion_us, __ATOMIC_RELAXED):0;
- usec_t last_gc_run_us = __atomic_load_n(&dict->last_gc_run_us, __ATOMIC_RELAXED);
-
- bool is_view = is_view_dictionary(dict);
-
- if(likely(!(
- DICTIONARY_PENDING_DELETES_GET(dict) > 0 ||
- (is_view && last_master_deletion_us > last_gc_run_us)
- )))
- return;
-
- ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
-
- __atomic_store_n(&dict->last_gc_run_us, now_realtime_usec(), __ATOMIC_RELAXED);
-
- if(is_view)
- dictionary_index_lock_wrlock(dict);
-
- DICTIONARY_STATS_GARBAGE_COLLECTIONS_PLUS1(dict);
-
- size_t deleted = 0, pending = 0, examined = 0;
- DICTIONARY_ITEM *item = dict->items.list, *item_next;
- while(item) {
- examined++;
-
- // this will clean up
- item_next = item->next;
- int rc = item_check_and_acquire_advanced(dict, item, is_view);
-
- if(rc == RC_ITEM_MARKED_FOR_DELETION) {
- // we didn't get a reference
-
- if(item_is_not_referenced_and_can_be_removed(dict, item)) {
- DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(dict->items.list, item, prev, next);
- dict_item_free_with_hooks(dict, item);
- deleted++;
-
- pending = DICTIONARY_PENDING_DELETES_MINUS1(dict);
- if (!pending)
- break;
- }
- }
- else if(rc == RC_ITEM_IS_CURRENTLY_BEING_DELETED)
- ; // do not touch this item (we didn't get a reference)
-
- else if(rc == RC_ITEM_OK)
- item_release(dict, item);
-
- item = item_next;
- }
-
- if(is_view)
- dictionary_index_wrlock_unlock(dict);
-
- ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
-
- (void)deleted;
- (void)examined;
-
- internal_error(false, "DICTIONARY: garbage collected dictionary created by %s (%zu@%s), "
- "examined %zu items, deleted %zu items, still pending %zu items",
- dict->creation_function, dict->creation_line, dict->creation_file,
- examined, deleted, pending);
-}
-
-void dictionary_garbage_collect(DICTIONARY *dict) {
- if(!dict) return;
- garbage_collect_pending_deletes(dict);
-}
-
-// ----------------------------------------------------------------------------
-// reference counters
-
-static inline size_t reference_counter_init(DICTIONARY *dict __maybe_unused) {
- // allocate memory required for reference counters
- // return number of bytes
- return 0;
-}
-
-static inline size_t reference_counter_free(DICTIONARY *dict __maybe_unused) {
- // free memory required for reference counters
- // return number of bytes
- return 0;
-}
-
-static void item_acquire(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- REFCOUNT refcount;
-
- if(unlikely(is_dictionary_single_threaded(dict)))
- refcount = ++item->refcount;
-
- else
- // increment the refcount
- refcount = __atomic_add_fetch(&item->refcount, 1, __ATOMIC_SEQ_CST);
-
-
- if(refcount <= 0) {
- internal_error(
- true,
- "DICTIONARY: attempted to acquire item which is deleted (refcount = %d): "
- "'%s' on dictionary created by %s() (%zu@%s)",
- refcount - 1,
- item_get_name(item),
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
-
- fatal(
- "DICTIONARY: request to acquire item '%s', which is deleted (refcount = %d)!",
- item_get_name(item),
- refcount - 1);
- }
-
- if(refcount == 1) {
- // referenced items counts number of unique items referenced
- // so, we increase it only when refcount == 1
- DICTIONARY_REFERENCED_ITEMS_PLUS1(dict);
-
- // if this is a deleted item, but the counter increased to 1
- // we need to remove it from the pending items to delete
- if(item_flag_check(item, ITEM_FLAG_DELETED))
- DICTIONARY_PENDING_DELETES_MINUS1(dict);
- }
-}
-
-static void item_release(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- // this function may be called without any lock on the dictionary
- // or even when someone else has 'write' lock on the dictionary
-
- bool is_deleted;
- REFCOUNT refcount;
-
- if(unlikely(is_dictionary_single_threaded(dict))) {
- is_deleted = item->flags & ITEM_FLAG_DELETED;
- refcount = --item->refcount;
- }
- else {
- // get the flags before decrementing any reference counters
- // (the other way around may lead to use-after-free)
- is_deleted = item_flag_check(item, ITEM_FLAG_DELETED);
-
- // decrement the refcount
- refcount = __atomic_sub_fetch(&item->refcount, 1, __ATOMIC_RELEASE);
- }
-
- if(refcount < 0) {
- internal_error(
- true,
- "DICTIONARY: attempted to release item without references (refcount = %d): "
- "'%s' on dictionary created by %s() (%zu@%s)",
- refcount + 1,
- item_get_name(item),
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
-
- fatal(
- "DICTIONARY: attempted to release item '%s' without references (refcount = %d)",
- item_get_name(item),
- refcount + 1);
- }
-
- if(refcount == 0) {
-
- if(is_deleted)
- DICTIONARY_PENDING_DELETES_PLUS1(dict);
-
- // referenced items counts number of unique items referenced
- // so, we decrease it only when refcount == 0
- DICTIONARY_REFERENCED_ITEMS_MINUS1(dict);
- }
-}
-
-static int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_ITEM *item, bool having_index_lock) {
- size_t spins = 0;
- REFCOUNT refcount, desired;
-
- int ret = RC_ITEM_OK;
-
- refcount = DICTIONARY_ITEM_REFCOUNT_GET(dict, item);
-
- do {
- spins++;
-
- if(refcount < 0) {
- // we can't use this item
- ret = RC_ITEM_IS_CURRENTLY_BEING_DELETED;
- break;
- }
-
- if(item_flag_check(item, ITEM_FLAG_DELETED)) {
- // we can't use this item
- ret = RC_ITEM_MARKED_FOR_DELETION;
- break;
- }
-
- desired = refcount + 1;
-
- } while(!__atomic_compare_exchange_n(&item->refcount, &refcount, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED));
-
- // if ret == ITEM_OK, we acquired the item
-
- if(ret == RC_ITEM_OK) {
- if (unlikely(is_view_dictionary(dict) &&
- item_shared_flag_check(item, ITEM_FLAG_DELETED) &&
- !item_flag_check(item, ITEM_FLAG_DELETED))) {
- // but, we can't use this item
-
- if (having_index_lock) {
- // delete it from the hashtable
- if(hashtable_delete_unsafe(dict, item_get_name(item), item->key_len, item) == 0)
- netdata_log_error("DICTIONARY: INTERNAL ERROR VIEW: tried to delete item with name '%s', name_len %u that is not in the index", item_get_name(item), (KEY_LEN_TYPE)(item->key_len - 1));
- else
- pointer_del(dict, item);
-
- // mark it in our dictionary as deleted too,
- // this is safe to be done here, because we have got
- // a reference counter on item
- dict_item_set_deleted(dict, item);
-
- // decrement the refcount we incremented above
- if (__atomic_sub_fetch(&item->refcount, 1, __ATOMIC_RELEASE) == 0) {
- // this is a deleted item, and we are the last one
- DICTIONARY_PENDING_DELETES_PLUS1(dict);
- }
-
- // do not touch the item below this point
- } else {
- // this is traversal / walkthrough
- // decrement the refcount we incremented above
- __atomic_sub_fetch(&item->refcount, 1, __ATOMIC_RELEASE);
- }
-
- return RC_ITEM_MARKED_FOR_DELETION;
- }
-
- if(desired == 1)
- DICTIONARY_REFERENCED_ITEMS_PLUS1(dict);
- }
-
- if(unlikely(spins > 1))
- DICTIONARY_STATS_CHECK_SPINS_PLUS(dict, spins - 1);
-
- return ret;
-}
-
-// if a dictionary item can be deleted, return true, otherwise return false
-// we use the private reference counter
-static inline int item_is_not_referenced_and_can_be_removed_advanced(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- // if we can set refcount to REFCOUNT_DELETING, we can delete this item
-
- size_t spins = 0;
- REFCOUNT refcount, desired = REFCOUNT_DELETING;
-
- int ret = RC_ITEM_OK;
-
- refcount = DICTIONARY_ITEM_REFCOUNT_GET(dict, item);
-
- do {
- spins++;
-
- if(refcount < 0) {
- // we can't use this item
- ret = RC_ITEM_IS_CURRENTLY_BEING_DELETED;
- break;
- }
-
- if(refcount > 0) {
- // we can't delete this
- ret = RC_ITEM_IS_REFERENCED;
- break;
- }
-
- if(item_flag_check(item, ITEM_FLAG_BEING_CREATED)) {
- // we can't use this item
- ret = RC_ITEM_IS_CURRENTLY_BEING_CREATED;
- break;
- }
- } while(!__atomic_compare_exchange_n(&item->refcount, &refcount, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED));
-
-#ifdef NETDATA_INTERNAL_CHECKS
- if(ret == RC_ITEM_OK)
- item->deleter_pid = gettid();
-#endif
-
- if(unlikely(spins > 1))
- DICTIONARY_STATS_DELETE_SPINS_PLUS(dict, spins - 1);
-
- return ret;
-}
-
-// if a dictionary item can be freed, return true, otherwise return false
-// we use the shared reference counter
-static inline bool item_shared_release_and_check_if_it_can_be_freed(DICTIONARY *dict __maybe_unused, DICTIONARY_ITEM *item) {
- // if we can set refcount to REFCOUNT_DELETING, we can delete this item
-
- REFCOUNT links = __atomic_sub_fetch(&item->shared->links, 1, __ATOMIC_RELEASE);
- if(links == 0 && __atomic_compare_exchange_n(&item->shared->links, &links, REFCOUNT_DELETING, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
-
- // we can delete it
- return true;
- }
-
- // we can't delete it
- return false;
-}
-
-
-// ----------------------------------------------------------------------------
-// hash table operations
-
-static size_t hashtable_init_unsafe(DICTIONARY *dict) {
- dict->index.JudyHSArray = NULL;
- return 0;
-}
-
-static size_t hashtable_destroy_unsafe(DICTIONARY *dict) {
- if(unlikely(!dict->index.JudyHSArray)) return 0;
-
- pointer_destroy_index(dict);
-
- JError_t J_Error;
- Word_t ret = JudyHSFreeArray(&dict->index.JudyHSArray, &J_Error);
- if(unlikely(ret == (Word_t) JERR)) {
- netdata_log_error("DICTIONARY: Cannot destroy JudyHS, JU_ERRNO_* == %u, ID == %d",
- JU_ERRNO(&J_Error), JU_ERRID(&J_Error));
- }
-
- netdata_log_debug(D_DICTIONARY, "Dictionary: hash table freed %lu bytes", ret);
-
- dict->index.JudyHSArray = NULL;
- return (size_t)ret;
-}
-
-static inline void **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) {
- JError_t J_Error;
- Pvoid_t *Rc = JudyHSIns(&dict->index.JudyHSArray, (void *)name, name_len, &J_Error);
- if (unlikely(Rc == PJERR)) {
- netdata_log_error("DICTIONARY: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d",
- name, JU_ERRNO(&J_Error), JU_ERRID(&J_Error));
- }
-
- // if *Rc == 0, new item added to the array
- // otherwise the existing item value is returned in *Rc
-
- // we return a pointer to a pointer, so that the caller can
- // put anything needed at the value of the index.
- // The pointer to pointer we return has to be used before
- // any other operation that may change the index (insert/delete).
- return Rc;
-}
-
-static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *item) {
- (void)item;
- if(unlikely(!dict->index.JudyHSArray)) return 0;
-
- JError_t J_Error;
- int ret = JudyHSDel(&dict->index.JudyHSArray, (void *)name, name_len, &J_Error);
- if(unlikely(ret == JERR)) {
- netdata_log_error("DICTIONARY: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d",
- name,
- JU_ERRNO(&J_Error), JU_ERRID(&J_Error));
- return 0;
- }
-
- // Hey, this is problematic! We need the value back, not just an int with a status!
- // https://sourceforge.net/p/judy/feature-requests/23/
-
- if(unlikely(ret == 0)) {
- // not found in the dictionary
- return 0;
- }
- else {
- // found and deleted from the dictionary
- return 1;
- }
-}
-
-static inline DICTIONARY_ITEM *hashtable_get_unsafe(DICTIONARY *dict, const char *name, size_t name_len) {
- if(unlikely(!dict->index.JudyHSArray)) return NULL;
-
- DICTIONARY_STATS_SEARCHES_PLUS1(dict);
-
- Pvoid_t *Rc;
- Rc = JudyHSGet(dict->index.JudyHSArray, (void *)name, name_len);
- if(likely(Rc)) {
- // found in the hash table
- pointer_check(dict, (DICTIONARY_ITEM *)*Rc);
- return (DICTIONARY_ITEM *)*Rc;
- }
- else {
- // not found in the hash table
- return NULL;
- }
-}
-
-static inline void hashtable_inserted_item_unsafe(DICTIONARY *dict, void *item) {
- (void)dict;
- (void)item;
-
- // this is called just after an item is successfully inserted to the hashtable
- // we don't need this for judy, but we may need it if we integrate more hash tables
-
- ;
-}
-
-// ----------------------------------------------------------------------------
-// linked list management
-
-static inline void item_linked_list_add(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
-
- if(dict->options & DICT_OPTION_ADD_IN_FRONT)
- DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(dict->items.list, item, prev, next);
- else
- DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(dict->items.list, item, prev, next);
-
-#ifdef NETDATA_INTERNAL_CHECKS
- item->ll_adder_pid = gettid();
-#endif
-
- // clear the BEING created flag,
- // after it has been inserted into the linked list
- item_flag_clear(item, ITEM_FLAG_BEING_CREATED);
-
- garbage_collect_pending_deletes(dict);
- ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
-}
-
-static inline void item_linked_list_remove(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
-
- DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(dict->items.list, item, prev, next);
-
-#ifdef NETDATA_INTERNAL_CHECKS
- item->ll_remover_pid = gettid();
-#endif
-
- garbage_collect_pending_deletes(dict);
- ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
-}
-
-// ----------------------------------------------------------------------------
-// ITEM initialization and updates
-
-static inline size_t item_set_name(DICTIONARY *dict, DICTIONARY_ITEM *item, const char *name, size_t name_len) {
- if(likely(dict->options & DICT_OPTION_NAME_LINK_DONT_CLONE)) {
- item->caller_name = (char *)name;
- item->key_len = name_len;
- }
- else {
- item->string_name = string_strdupz(name);
- item->key_len = string_strlen(item->string_name) + 1;
- item->options |= ITEM_OPTION_ALLOCATED_NAME;
- }
-
- return item->key_len;
-}
-
-static inline size_t item_free_name(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- if(likely(!(dict->options & DICT_OPTION_NAME_LINK_DONT_CLONE)))
- string_freez(item->string_name);
-
- return item->key_len;
-}
-
-static inline const char *item_get_name(const DICTIONARY_ITEM *item) {
- if(item->options & ITEM_OPTION_ALLOCATED_NAME)
- return string2str(item->string_name);
- else
- return item->caller_name;
-}
-
-static inline size_t item_get_name_len(const DICTIONARY_ITEM *item) {
- if(item->options & ITEM_OPTION_ALLOCATED_NAME)
- return string_strlen(item->string_name);
- else
- return strlen(item->caller_name);
-}
-
-static ARAL *dict_items_aral = NULL;
-static ARAL *dict_shared_items_aral = NULL;
-
-void dictionary_static_items_aral_init(void) {
- static SPINLOCK spinlock;
-
- if(unlikely(!dict_items_aral || !dict_shared_items_aral)) {
- spinlock_lock(&spinlock);
-
- // we have to check again
- if(!dict_items_aral)
- dict_items_aral = aral_create(
- "dict-items",
- sizeof(DICTIONARY_ITEM),
- 0,
- 65536,
- aral_by_size_statistics(),
- NULL, NULL, false, false);
-
- // we have to check again
- if(!dict_shared_items_aral)
- dict_shared_items_aral = aral_create(
- "dict-shared-items",
- sizeof(DICTIONARY_ITEM_SHARED),
- 0,
- 65536,
- aral_by_size_statistics(),
- NULL, NULL, false, false);
-
- spinlock_unlock(&spinlock);
- }
-}
-
-static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t *allocated_bytes, DICTIONARY_ITEM *master_item) {
- DICTIONARY_ITEM *item;
-
- size_t size = sizeof(DICTIONARY_ITEM);
- item = aral_mallocz(dict_items_aral);
- memset(item, 0, sizeof(DICTIONARY_ITEM));
-
-#ifdef NETDATA_INTERNAL_CHECKS
- item->creator_pid = gettid();
-#endif
-
- item->refcount = 1;
- item->flags = ITEM_FLAG_BEING_CREATED;
-
- *allocated_bytes += size;
-
- if(master_item) {
- item->shared = master_item->shared;
-
- if(unlikely(__atomic_add_fetch(&item->shared->links, 1, __ATOMIC_ACQUIRE) <= 1))
- fatal("DICTIONARY: attempted to link to a shared item structure that had zero references");
- }
- else {
- size = sizeof(DICTIONARY_ITEM_SHARED);
- item->shared = aral_mallocz(dict_shared_items_aral);
- memset(item->shared, 0, sizeof(DICTIONARY_ITEM_SHARED));
-
- item->shared->links = 1;
- *allocated_bytes += size;
- }
-
-#ifdef NETDATA_INTERNAL_CHECKS
- item->dict = dict;
-#endif
- return item;
-}
-
-static inline void *dict_item_value_mallocz(DICTIONARY *dict, size_t value_len) {
- if(dict->value_aral) {
- internal_fatal(aral_element_size(dict->value_aral) != value_len,
- "DICTIONARY: item value size %zu does not match the configured fixed one %zu",
- value_len, aral_element_size(dict->value_aral));
- return aral_mallocz(dict->value_aral);
- }
- else
- return mallocz(value_len);
-}
-
-static inline void dict_item_value_freez(DICTIONARY *dict, void *ptr) {
- if(dict->value_aral)
- aral_freez(dict->value_aral, ptr);
- else
- freez(ptr);
-}
-
-static void *dict_item_value_create(DICTIONARY *dict, void *value, size_t value_len) {
- void *ptr = NULL;
-
- if(likely(value_len)) {
- if (likely(value)) {
- // a value has been supplied
- // copy it
- ptr = dict_item_value_mallocz(dict, value_len);
- memcpy(ptr, value, value_len);
- }
- else {
- // no value has been supplied
- // allocate a clear memory block
- ptr = dict_item_value_mallocz(dict, value_len);
- memset(ptr, 0, value_len);
- }
- }
- // else
- // the caller wants an item without any value
-
- return ptr;
-}
-
-static DICTIONARY_ITEM *dict_item_create_with_hooks(DICTIONARY *dict, const char *name, size_t name_len, void *value, size_t value_len, void *constructor_data, DICTIONARY_ITEM *master_item) {
-#ifdef NETDATA_INTERNAL_CHECKS
- if(unlikely(name_len > KEY_LEN_MAX))
- fatal("DICTIONARY: tried to index a key of size %zu, but the maximum acceptable is %zu", name_len, (size_t)KEY_LEN_MAX);
-
- if(unlikely(value_len > VALUE_LEN_MAX))
- fatal("DICTIONARY: tried to add an item of size %zu, but the maximum acceptable is %zu", value_len, (size_t)VALUE_LEN_MAX);
-#endif
-
- size_t item_size = 0, key_size = 0, value_size = 0;
-
- DICTIONARY_ITEM *item = dict_item_create(dict, &item_size, master_item);
- key_size += item_set_name(dict, item, name, name_len);
-
- if(unlikely(is_view_dictionary(dict))) {
- // we are on a view dictionary
- // do not touch the value
- ;
-
-#ifdef NETDATA_INTERNAL_CHECKS
- if(unlikely(!master_item))
- fatal("DICTIONARY: cannot add an item to a view without a master item.");
-#endif
- }
- else {
- // we are on the master dictionary
-
- if(unlikely(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE))
- item->shared->value = value;
- else
- item->shared->value = dict_item_value_create(dict, value, value_len);
-
- item->shared->value_len = value_len;
- value_size += value_len;
-
- dictionary_execute_insert_callback(dict, item, constructor_data);
- }
-
- DICTIONARY_ENTRIES_PLUS1(dict);
- DICTIONARY_STATS_PLUS_MEMORY(dict, key_size, item_size, value_size);
-
- return item;
-}
-
-static void dict_item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item, void *value, size_t value_len, void *constructor_data) {
- if(unlikely(is_view_dictionary(dict)))
- fatal("DICTIONARY: %s() should never be called on views.", __FUNCTION__ );
-
- netdata_log_debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", item_get_name(item));
-
- DICTIONARY_VALUE_RESETS_PLUS1(dict);
-
- if(item->shared->value_len != value_len) {
- DICTIONARY_STATS_PLUS_MEMORY(dict, 0, 0, value_len);
- DICTIONARY_STATS_MINUS_MEMORY(dict, 0, 0, item->shared->value_len);
- }
-
- dictionary_execute_delete_callback(dict, item);
-
- if(likely(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE)) {
- netdata_log_debug(D_DICTIONARY, "Dictionary: linking value to '%s'", item_get_name(item));
- item->shared->value = value;
- item->shared->value_len = value_len;
- }
- else {
- netdata_log_debug(D_DICTIONARY, "Dictionary: cloning value to '%s'", item_get_name(item));
-
- void *old_value = item->shared->value;
- void *new_value = NULL;
- if(value_len) {
- new_value = dict_item_value_mallocz(dict, value_len);
- if(value) memcpy(new_value, value, value_len);
- else memset(new_value, 0, value_len);
- }
- item->shared->value = new_value;
- item->shared->value_len = value_len;
-
- netdata_log_debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", item_get_name(item));
- dict_item_value_freez(dict, old_value);
- }
-
- dictionary_execute_insert_callback(dict, item, constructor_data);
-}
-
-static size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- netdata_log_debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", item_get_name(item));
-
- if(!item_flag_check(item, ITEM_FLAG_DELETED))
- DICTIONARY_ENTRIES_MINUS1(dict);
-
- size_t item_size = 0, key_size = 0, value_size = 0;
-
- key_size += item->key_len;
- if(unlikely(!(dict->options & DICT_OPTION_NAME_LINK_DONT_CLONE)))
- item_free_name(dict, item);
-
- if(item_shared_release_and_check_if_it_can_be_freed(dict, item)) {
- dictionary_execute_delete_callback(dict, item);
-
- if(unlikely(!(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE))) {
- netdata_log_debug(D_DICTIONARY, "Dictionary freeing value of '%s'", item_get_name(item));
- dict_item_value_freez(dict, item->shared->value);
- item->shared->value = NULL;
- }
- value_size += item->shared->value_len;
-
- aral_freez(dict_shared_items_aral, item->shared);
- item->shared = NULL;
- item_size += sizeof(DICTIONARY_ITEM_SHARED);
- }
-
- aral_freez(dict_items_aral, item);
-
- item_size += sizeof(DICTIONARY_ITEM);
-
- DICTIONARY_STATS_MINUS_MEMORY(dict, key_size, item_size, value_size);
-
- // we return the memory we actually freed
- return item_size + ((dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE) ? 0 : value_size);
-}
-
-// ----------------------------------------------------------------------------
-// item operations
-
-static void dict_item_shared_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- if(is_master_dictionary(dict)) {
- item_shared_flag_set(item, ITEM_FLAG_DELETED);
-
- if(dict->hooks)
- __atomic_store_n(&dict->hooks->last_master_deletion_us, now_realtime_usec(), __ATOMIC_RELAXED);
- }
-}
-
-// returns true if we set the deleted flag on this item
-static bool dict_item_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- ITEM_FLAGS expected, desired;
-
- expected = __atomic_load_n(&item->flags, __ATOMIC_RELAXED);
-
- do {
-
- if (expected & ITEM_FLAG_DELETED)
- return false;
-
- desired = expected | ITEM_FLAG_DELETED;
-
- } while(!__atomic_compare_exchange_n(&item->flags, &expected, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED));
-
- DICTIONARY_ENTRIES_MINUS1(dict);
- return true;
-}
-
-static inline void dict_item_free_or_mark_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
- int rc = item_is_not_referenced_and_can_be_removed_advanced(dict, item);
- switch(rc) {
- case RC_ITEM_OK:
- // the item is ours, refcount set to -100
- dict_item_shared_set_deleted(dict, item);
- item_linked_list_remove(dict, item);
- dict_item_free_with_hooks(dict, item);
- break;
-
- case RC_ITEM_IS_REFERENCED:
- case RC_ITEM_IS_CURRENTLY_BEING_CREATED:
- // the item is currently referenced by others
- dict_item_shared_set_deleted(dict, item);
- dict_item_set_deleted(dict, item);
- // after this point do not touch the item
- break;
-
- case RC_ITEM_IS_CURRENTLY_BEING_DELETED:
- // an item that is currently being deleted by someone else - don't touch it
- break;
-
- default:
- internal_error(true, "Hey dev! You forgot to add the new condition here!");
- break;
- }
-}
-
-// this is used by traversal functions to remove the current item
-// if it is deleted, and it has zero references. This will eliminate
-// the need for the garbage collector to kick-in later.
-// Most deletions happen during traversal, so this is a nice hack
-// to speed up everything!
-static inline void dict_item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(DICTIONARY *dict, DICTIONARY_ITEM *item, char rw) {
- if(rw == DICTIONARY_LOCK_WRITE) {
- bool should_be_deleted = item_flag_check(item, ITEM_FLAG_DELETED);
-
- item_release(dict, item);
-
- if(should_be_deleted && item_is_not_referenced_and_can_be_removed(dict, item)) {
- // this has to be before removing from the linked list,
- // otherwise the garbage collector will also kick in!
- DICTIONARY_PENDING_DELETES_MINUS1(dict);
-
- item_linked_list_remove(dict, item);
- dict_item_free_with_hooks(dict, item);
- }
- }
- else {
- // we can't do anything under this mode
- item_release(dict, item);
- }
-}
-
-static bool dict_item_del(DICTIONARY *dict, const char *name, ssize_t name_len) {
- if(name_len == -1)
- name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too
-
- netdata_log_debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name);
-
- // Unfortunately, the JudyHSDel() does not return the value of the
- // item that was deleted, so we have to find it before we delete it,
- // since we need to release our structures too.
-
- dictionary_index_lock_wrlock(dict);
-
- int ret;
- DICTIONARY_ITEM *item = hashtable_get_unsafe(dict, name, name_len);
- if(unlikely(!item)) {
- dictionary_index_wrlock_unlock(dict);
- ret = false;
- }
- else {
- if(hashtable_delete_unsafe(dict, name, name_len, item) == 0)
- netdata_log_error("DICTIONARY: INTERNAL ERROR: tried to delete item with name '%s', name_len %zd that is not in the index",
- name,
- name_len - 1);
- else
- pointer_del(dict, item);
-
- dictionary_index_wrlock_unlock(dict);
-
- dict_item_free_or_mark_deleted(dict, item);
- ret = true;
- }
-
- return ret;
-}
-
-static DICTIONARY_ITEM *dict_item_add_or_reset_value_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data, DICTIONARY_ITEM *master_item) {
- if(unlikely(!name || !*name)) {
- internal_error(
- true,
- "DICTIONARY: attempted to %s() without a name on a dictionary created from %s() %zu@%s.",
- __FUNCTION__,
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
- return NULL;
- }
-
- if(unlikely(is_dictionary_destroyed(dict))) {
- internal_error(true, "DICTIONARY: attempted to dictionary_set() on a destroyed dictionary");
- return NULL;
- }
-
- if(name_len == -1)
- name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too
-
- netdata_log_debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name);
-
- // DISCUSSION:
- // Is it better to gain a read-lock and do a hashtable_get_unsafe()
- // before we write lock to do hashtable_insert_unsafe()?
- //
- // Probably this depends on the use case.
- // For statsd for example that does dictionary_set() to update received values,
- // it could be beneficial to do a get() before we insert().
- //
- // But the caller has the option to do this on his/her own.
- // So, let's do the fastest here and let the caller decide the flow of calls.
-
- dictionary_index_lock_wrlock(dict);
-
- bool added_or_updated = false;
- size_t spins = 0;
- DICTIONARY_ITEM *item = NULL;
- do {
- DICTIONARY_ITEM **item_pptr = (DICTIONARY_ITEM **)hashtable_insert_unsafe(dict, name, name_len);
- if (likely(*item_pptr == NULL)) {
- // a new item added to the index
-
- // create the dictionary item
- item = *item_pptr =
- dict_item_create_with_hooks(dict, name, name_len, value, value_len, constructor_data, master_item);
-
- pointer_add(dict, item);
-
- // call the hashtable react
- hashtable_inserted_item_unsafe(dict, item);
-
- // unlock the index lock, before we add it to the linked list
- // DON'T DO IT THE OTHER WAY AROUND - DO NOT CROSS THE LOCKS!
- dictionary_index_wrlock_unlock(dict);
-
- item_linked_list_add(dict, item);
-
- added_or_updated = true;
- }
- else {
- pointer_check(dict, *item_pptr);
-
- if(item_check_and_acquire_advanced(dict, *item_pptr, true) != RC_ITEM_OK) {
- spins++;
- continue;
- }
-
- // the item is already in the index
- // so, either we will return the old one
- // or overwrite the value, depending on dictionary flags
-
- // We should not compare the values here!
- // even if they are the same, we have to do the whole job
- // so that the callbacks will be called.
-
- item = *item_pptr;
-
- if(is_view_dictionary(dict)) {
- // view dictionary
- // the item is already there and can be used
- if(item->shared != master_item->shared)
- netdata_log_error("DICTIONARY: changing the master item on a view is not supported. The previous item will remain. To change the key of an item in a view, delete it and add it again.");
- }
- else {
- // master dictionary
- // the user wants to reset its value
-
- if (!(dict->options & DICT_OPTION_DONT_OVERWRITE_VALUE)) {
- dict_item_reset_value_with_hooks(dict, item, value, value_len, constructor_data);
- added_or_updated = true;
- }
-
- else if (dictionary_execute_conflict_callback(dict, item, value, constructor_data)) {
- dictionary_version_increment(dict);
- added_or_updated = true;
- }
-
- else {
- // conflict callback returned false
- // we did really nothing!
- ;
- }
- }
-
- dictionary_index_wrlock_unlock(dict);
- }
- } while(!item);
-
-
- if(unlikely(spins > 0))
- DICTIONARY_STATS_INSERT_SPINS_PLUS(dict, spins);
-
- if(is_master_dictionary(dict) && added_or_updated)
- dictionary_execute_react_callback(dict, item, constructor_data);
-
- return item;
-}
-
-static DICTIONARY_ITEM *dict_item_find_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len) {
- if(unlikely(!name || !*name)) {
- internal_error(
- true,
- "DICTIONARY: attempted to %s() without a name on a dictionary created from %s() %zu@%s.",
- __FUNCTION__,
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
- return NULL;
- }
-
- if(unlikely(is_dictionary_destroyed(dict))) {
- internal_error(true, "DICTIONARY: attempted to dictionary_get() on a destroyed dictionary");
- return NULL;
- }
-
- if(name_len == -1)
- name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too
-
- netdata_log_debug(D_DICTIONARY, "GET dictionary entry with name '%s'.", name);
-
- dictionary_index_lock_rdlock(dict);
-
- DICTIONARY_ITEM *item = hashtable_get_unsafe(dict, name, name_len);
- if(unlikely(item && !item_check_and_acquire(dict, item))) {
- item = NULL;
- DICTIONARY_STATS_SEARCH_IGNORES_PLUS1(dict);
- }
-
- dictionary_index_rdlock_unlock(dict);
-
- return item;
-}
-
-// ----------------------------------------------------------------------------
-// delayed destruction of dictionaries
-
-static bool dictionary_free_all_resources(DICTIONARY *dict, size_t *mem, bool force) {
- if(mem)
- *mem = 0;
-
- if(!force && dictionary_referenced_items(dict))
- return false;
-
- size_t dict_size = 0, counted_items = 0, item_size = 0, index_size = 0;
- (void)counted_items;
-
-#ifdef NETDATA_INTERNAL_CHECKS
- long int entries = dict->entries;
- long int referenced_items = dict->referenced_items;
- long int pending_deletion_items = dict->pending_deletion_items;
- const char *creation_function = dict->creation_function;
- const char *creation_file = dict->creation_file;
- size_t creation_line = dict->creation_line;
-#endif
-
- // destroy the index
- dictionary_index_lock_wrlock(dict);
- index_size += hashtable_destroy_unsafe(dict);
- dictionary_index_wrlock_unlock(dict);
-
- ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
- DICTIONARY_ITEM *item = dict->items.list;
- while (item) {
- // cache item->next
- // because we are going to free item
- DICTIONARY_ITEM *item_next = item->next;
-
- item_size += dict_item_free_with_hooks(dict, item);
- item = item_next;
-
- // to speed up destruction, we don't
- // unlink item from the linked-list here
-
- counted_items++;
- }
- dict->items.list = NULL;
- ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
-
- dict_size += dictionary_locks_destroy(dict);
- dict_size += reference_counter_free(dict);
- dict_size += dictionary_hooks_free(dict);
- dict_size += sizeof(DICTIONARY);
- DICTIONARY_STATS_MINUS_MEMORY(dict, 0, sizeof(DICTIONARY), 0);
-
- if(dict->value_aral)
- aral_by_size_release(dict->value_aral);
-
- freez(dict);
-
- internal_error(
- false,
- "DICTIONARY: Freed dictionary created from %s() %zu@%s, having %ld (counted %zu) entries, %ld referenced, %ld pending deletion, total freed memory: %zu bytes (sizeof(dict) = %zu, sizeof(item) = %zu).",
- creation_function,
- creation_line,
- creation_file,
- entries, counted_items, referenced_items, pending_deletion_items,
- dict_size + item_size, sizeof(DICTIONARY), sizeof(DICTIONARY_ITEM) + sizeof(DICTIONARY_ITEM_SHARED));
-
- if(mem)
- *mem = dict_size + item_size + index_size;
-
- return true;
-}
-
-netdata_mutex_t dictionaries_waiting_to_be_destroyed_mutex = NETDATA_MUTEX_INITIALIZER;
-static DICTIONARY *dictionaries_waiting_to_be_destroyed = NULL;
-
-void dictionary_queue_for_destruction(DICTIONARY *dict) {
- if(is_dictionary_destroyed(dict))
- return;
-
- DICTIONARY_STATS_DICT_DESTROY_QUEUED_PLUS1(dict);
- dict_flag_set(dict, DICT_FLAG_DESTROYED);
-
- netdata_mutex_lock(&dictionaries_waiting_to_be_destroyed_mutex);
-
- dict->next = dictionaries_waiting_to_be_destroyed;
- dictionaries_waiting_to_be_destroyed = dict;
-
- netdata_mutex_unlock(&dictionaries_waiting_to_be_destroyed_mutex);
-}
-
-void cleanup_destroyed_dictionaries(void) {
- if(!dictionaries_waiting_to_be_destroyed)
- return;
-
- netdata_mutex_lock(&dictionaries_waiting_to_be_destroyed_mutex);
-
- DICTIONARY *dict, *last = NULL, *next = NULL;
- for(dict = dictionaries_waiting_to_be_destroyed; dict ; dict = next) {
- next = dict->next;
-
-#ifdef NETDATA_INTERNAL_CHECKS
- size_t line = dict->creation_line;
- const char *file = dict->creation_file;
- const char *function = dict->creation_function;
- pid_t pid = dict->creation_tid;
-#endif
-
- DICTIONARY_STATS_DICT_DESTROY_QUEUED_MINUS1(dict);
- if(dictionary_free_all_resources(dict, NULL, false)) {
-
- internal_error(
- true,
- "DICTIONARY: freed dictionary with delayed destruction, created from %s() %zu@%s pid %d.",
- function, line, file, pid);
-
- if(last) last->next = next;
- else dictionaries_waiting_to_be_destroyed = next;
- }
- else {
-
- internal_error(
- true,
- "DICTIONARY: cannot free dictionary with delayed destruction, created from %s() %zu@%s pid %d.",
- function, line, file, pid);
-
- DICTIONARY_STATS_DICT_DESTROY_QUEUED_PLUS1(dict);
- last = dict;
- }
- }
-
- netdata_mutex_unlock(&dictionaries_waiting_to_be_destroyed_mutex);
-}
-
-// ----------------------------------------------------------------------------
-// API internal checks
-
-#ifdef NETDATA_INTERNAL_CHECKS
-#define api_internal_check(dict, item, allow_null_dict, allow_null_item) api_internal_check_with_trace(dict, item, __FUNCTION__, allow_null_dict, allow_null_item)
-static inline void api_internal_check_with_trace(DICTIONARY *dict, DICTIONARY_ITEM *item, const char *function, bool allow_null_dict, bool allow_null_item) {
- if(!allow_null_dict && !dict) {
- internal_error(
- item,
- "DICTIONARY: attempted to %s() with a NULL dictionary, passing an item created from %s() %zu@%s.",
- function,
- item->dict->creation_function,
- item->dict->creation_line,
- item->dict->creation_file);
- fatal("DICTIONARY: attempted to %s() but dict is NULL", function);
- }
-
- if(!allow_null_item && !item) {
- internal_error(
- true,
- "DICTIONARY: attempted to %s() without an item on a dictionary created from %s() %zu@%s.",
- function,
- dict?dict->creation_function:"unknown",
- dict?dict->creation_line:0,
- dict?dict->creation_file:"unknown");
- fatal("DICTIONARY: attempted to %s() but item is NULL", function);
- }
-
- if(dict && item && dict != item->dict) {
- internal_error(
- true,
- "DICTIONARY: attempted to %s() an item on a dictionary created from %s() %zu@%s, but the item belongs to the dictionary created from %s() %zu@%s.",
- function,
- dict->creation_function,
- dict->creation_line,
- dict->creation_file,
- item->dict->creation_function,
- item->dict->creation_line,
- item->dict->creation_file
- );
- fatal("DICTIONARY: %s(): item does not belong to this dictionary.", function);
- }
-
- if(item) {
- REFCOUNT refcount = DICTIONARY_ITEM_REFCOUNT_GET(dict, item);
- if (unlikely(refcount <= 0)) {
- internal_error(
- true,
- "DICTIONARY: attempted to %s() of an item with reference counter = %d on a dictionary created from %s() %zu@%s",
- function,
- refcount,
- item->dict->creation_function,
- item->dict->creation_line,
- item->dict->creation_file);
- fatal("DICTIONARY: attempted to %s but item is having refcount = %d", function, refcount);
- }
- }
-}
-#else
-#define api_internal_check(dict, item, allow_null_dict, allow_null_item) debug_dummy()
-#endif
-
-#define api_is_name_good(dict, name, name_len) api_is_name_good_with_trace(dict, name, name_len, __FUNCTION__)
-static bool api_is_name_good_with_trace(DICTIONARY *dict __maybe_unused, const char *name, ssize_t name_len __maybe_unused, const char *function __maybe_unused) {
- if(unlikely(!name)) {
- internal_error(
- true,
- "DICTIONARY: attempted to %s() with name = NULL on a dictionary created from %s() %zu@%s.",
- function,
- dict?dict->creation_function:"unknown",
- dict?dict->creation_line:0,
- dict?dict->creation_file:"unknown");
- return false;
- }
-
- if(unlikely(!*name)) {
- internal_error(
- true,
- "DICTIONARY: attempted to %s() with empty name on a dictionary created from %s() %zu@%s.",
- function,
- dict?dict->creation_function:"unknown",
- dict?dict->creation_line:0,
- dict?dict->creation_file:"unknown");
- return false;
- }
-
- internal_error(
- name_len > 0 && name_len != (ssize_t)(strlen(name) + 1),
- "DICTIONARY: attempted to %s() with a name of '%s', having length of %zu (incl. '\\0'), but the supplied name_len = %ld, on a dictionary created from %s() %zu@%s.",
- function,
- name,
- strlen(name) + 1,
- (long int) name_len,
- dict?dict->creation_function:"unknown",
- dict?dict->creation_line:0,
- dict?dict->creation_file:"unknown");
-
- internal_error(
- name_len <= 0 && name_len != -1,
- "DICTIONARY: attempted to %s() with a name of '%s', having length of %zu (incl. '\\0'), but the supplied name_len = %ld, on a dictionary created from %s() %zu@%s.",
- function,
- name,
- strlen(name) + 1,
- (long int) name_len,
- dict?dict->creation_function:"unknown",
- dict?dict->creation_line:0,
- dict?dict->creation_file:"unknown");
-
- return true;
-}
-
-// ----------------------------------------------------------------------------
-// API - dictionary management
-
-static DICTIONARY *dictionary_create_internal(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size) {
- cleanup_destroyed_dictionaries();
-
- DICTIONARY *dict = callocz(1, sizeof(DICTIONARY));
- dict->options = options;
- dict->stats = stats;
-
- if((dict->options & DICT_OPTION_FIXED_SIZE) && !fixed_size) {
- dict->options &= ~DICT_OPTION_FIXED_SIZE;
- internal_fatal(true, "DICTIONARY: requested fixed size dictionary, without setting the size");
- }
- if(!(dict->options & DICT_OPTION_FIXED_SIZE) && fixed_size) {
- dict->options |= DICT_OPTION_FIXED_SIZE;
- internal_fatal(true, "DICTIONARY: set a fixed size for the items, without setting DICT_OPTION_FIXED_SIZE flag");
- }
-
- if(dict->options & DICT_OPTION_FIXED_SIZE)
- dict->value_aral = aral_by_size_acquire(fixed_size);
- else
- dict->value_aral = NULL;
-
- size_t dict_size = 0;
- dict_size += sizeof(DICTIONARY);
- dict_size += dictionary_locks_init(dict);
- dict_size += reference_counter_init(dict);
- dict_size += hashtable_init_unsafe(dict);
-
- dictionary_static_items_aral_init();
- pointer_index_init(dict);
-
- DICTIONARY_STATS_PLUS_MEMORY(dict, 0, dict_size, 0);
-
- return dict;
-}
-
-#ifdef NETDATA_INTERNAL_CHECKS
-DICTIONARY *dictionary_create_advanced_with_trace(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size, const char *function, size_t line, const char *file) {
-#else
-DICTIONARY *dictionary_create_advanced(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size) {
-#endif
-
- DICTIONARY *dict = dictionary_create_internal(options, stats?stats:&dictionary_stats_category_other, fixed_size);
-
-#ifdef NETDATA_INTERNAL_CHECKS
- dict->creation_function = function;
- dict->creation_file = file;
- dict->creation_line = line;
-#endif
-
- DICTIONARY_STATS_DICT_CREATIONS_PLUS1(dict);
- return dict;
-}
-
-#ifdef NETDATA_INTERNAL_CHECKS
-DICTIONARY *dictionary_create_view_with_trace(DICTIONARY *master, const char *function, size_t line, const char *file) {
-#else
-DICTIONARY *dictionary_create_view(DICTIONARY *master) {
-#endif
-
- DICTIONARY *dict = dictionary_create_internal(master->options, master->stats,
- master->value_aral ? aral_element_size(master->value_aral) : 0);
-
- dict->master = master;
-
- dictionary_hooks_allocate(master);
-
- if(unlikely(__atomic_load_n(&master->hooks->links, __ATOMIC_RELAXED)) < 1)
- fatal("DICTIONARY: attempted to create a view that has %d links", master->hooks->links);
-
- dict->hooks = master->hooks;
- __atomic_add_fetch(&master->hooks->links, 1, __ATOMIC_ACQUIRE);
-
-#ifdef NETDATA_INTERNAL_CHECKS
- dict->creation_function = function;
- dict->creation_file = file;
- dict->creation_line = line;
- dict->creation_tid = gettid();
-#endif
-
- DICTIONARY_STATS_DICT_CREATIONS_PLUS1(dict);
- return dict;
-}
-
-void dictionary_flush(DICTIONARY *dict) {
- if(unlikely(!dict))
- return;
-
- ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
-
- DICTIONARY_ITEM *item, *next = NULL;
- for(item = dict->items.list; item ;item = next) {
- next = item->next;
- dict_item_del(dict, item_get_name(item), (ssize_t) item_get_name_len(item) + 1);
- }
-
- ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
-
- DICTIONARY_STATS_DICT_FLUSHES_PLUS1(dict);
-}
-
-size_t dictionary_destroy(DICTIONARY *dict) {
- cleanup_destroyed_dictionaries();
-
- if(!dict) return 0;
-
- ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
-
- dict_flag_set(dict, DICT_FLAG_DESTROYED);
- DICTIONARY_STATS_DICT_DESTRUCTIONS_PLUS1(dict);
-
- size_t referenced_items = dictionary_referenced_items(dict);
- if(referenced_items) {
- dictionary_flush(dict);
- dictionary_queue_for_destruction(dict);
-
- internal_error(
- true,
- "DICTIONARY: delaying destruction of dictionary created from %s() %zu@%s, because it has %d referenced items in it (%d total).",
- dict->creation_function,
- dict->creation_line,
- dict->creation_file,
- dict->referenced_items,
- dict->entries);
-
- ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
- return 0;
- }
-
- ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
-
- size_t freed;
- dictionary_free_all_resources(dict, &freed, true);
-
- return freed;
-}
-
-// ----------------------------------------------------------------------------
-// SET an item to the dictionary
-
-DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_set_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data) {
- if(unlikely(!api_is_name_good(dict, name, name_len)))
- return NULL;
-
- api_internal_check(dict, NULL, false, true);
-
- if(unlikely(is_view_dictionary(dict)))
- fatal("DICTIONARY: this dictionary is a view, you cannot add items other than the ones from the master dictionary.");
-
- DICTIONARY_ITEM *item =
- dict_item_add_or_reset_value_and_acquire(dict, name, name_len, value, value_len, constructor_data, NULL);
- api_internal_check(dict, item, false, false);
- return item;
-}
-
-void *dictionary_set_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data) {
- DICTIONARY_ITEM *item = dictionary_set_and_acquire_item_advanced(dict, name, name_len, value, value_len, constructor_data);
-
- if(likely(item)) {
- void *v = item->shared->value;
- item_release(dict, item);
- return v;
- }
-
- return NULL;
-}
-
-DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_view_set_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, DICTIONARY_ITEM *master_item) {
- if(unlikely(!api_is_name_good(dict, name, name_len)))
- return NULL;
-
- api_internal_check(dict, NULL, false, true);
-
- if(unlikely(is_master_dictionary(dict)))
- fatal("DICTIONARY: this dictionary is a master, you cannot add items from other dictionaries.");
-
- garbage_collect_pending_deletes(dict);
-
- dictionary_acquired_item_dup(dict->master, master_item);
- DICTIONARY_ITEM *item = dict_item_add_or_reset_value_and_acquire(dict, name, name_len, NULL, 0, NULL, master_item);
- dictionary_acquired_item_release(dict->master, master_item);
-
- api_internal_check(dict, item, false, false);
- return item;
-}
-
-void *dictionary_view_set_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, DICTIONARY_ITEM *master_item) {
- DICTIONARY_ITEM *item = dictionary_view_set_and_acquire_item_advanced(dict, name, name_len, master_item);
-
- if(likely(item)) {
- void *v = item->shared->value;
- item_release(dict, item);
- return v;
- }
-
- return NULL;
-}
-
-// ----------------------------------------------------------------------------
-// GET an item from the dictionary
-
-DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_get_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len) {
- if(unlikely(!api_is_name_good(dict, name, name_len)))
- return NULL;
-
- api_internal_check(dict, NULL, false, true);
- DICTIONARY_ITEM *item = dict_item_find_and_acquire(dict, name, name_len);
- api_internal_check(dict, item, false, true);
- return item;
-}
-
-void *dictionary_get_advanced(DICTIONARY *dict, const char *name, ssize_t name_len) {
- DICTIONARY_ITEM *item = dictionary_get_and_acquire_item_advanced(dict, name, name_len);
-
- if(likely(item)) {
- void *v = item->shared->value;
- item_release(dict, item);
- return v;
- }
-
- return NULL;
-}
-
-// ----------------------------------------------------------------------------
-// DUP/REL an item (increase/decrease its reference counter)
-
-DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_acquired_item_dup(DICTIONARY *dict, DICT_ITEM_CONST DICTIONARY_ITEM *item) {
- // we allow the item to be NULL here
- api_internal_check(dict, item, false, true);
-
- if(likely(item)) {
- item_acquire(dict, item);
- api_internal_check(dict, item, false, false);
- }
-
- return item;
-}
-
-void dictionary_acquired_item_release(DICTIONARY *dict, DICT_ITEM_CONST DICTIONARY_ITEM *item) {
- // we allow the item to be NULL here
- api_internal_check(dict, item, false, true);
-
- // no need to get a lock here
- // we pass the last parameter to reference_counter_release() as true
- // so that the release may get a write-lock if required to clean up
-
- if(likely(item))
- item_release(dict, item);
-}
-
-// ----------------------------------------------------------------------------
-// get the name/value of an item
-
-const char *dictionary_acquired_item_name(DICT_ITEM_CONST DICTIONARY_ITEM *item) {
- return item_get_name(item);
-}
-
-void *dictionary_acquired_item_value(DICT_ITEM_CONST DICTIONARY_ITEM *item) {
- if(likely(item))
- return item->shared->value;
-
- return NULL;
-}
-
-size_t dictionary_acquired_item_references(DICT_ITEM_CONST DICTIONARY_ITEM *item) {
- if(likely(item))
- return DICTIONARY_ITEM_REFCOUNT_GET_SOLE(item);
-
- return 0;
-}
-
-// ----------------------------------------------------------------------------
-// DEL an item
-
-bool dictionary_del_advanced(DICTIONARY *dict, const char *name, ssize_t name_len) {
- if(unlikely(!api_is_name_good(dict, name, name_len)))
- return false;
-
- api_internal_check(dict, NULL, false, true);
-
- if(unlikely(is_dictionary_destroyed(dict))) {
- internal_error(true, "DICTIONARY: attempted to delete item on a destroyed dictionary");
- return false;
- }
-
- return dict_item_del(dict, name, name_len);
-}
-
-// ----------------------------------------------------------------------------
-// traversal with loop
-
-void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {
- if(unlikely(!dfe || !dict)) return NULL;
-
- DICTIONARY_STATS_TRAVERSALS_PLUS1(dict);
-
- if(unlikely(is_dictionary_destroyed(dict))) {
- internal_error(true, "DICTIONARY: attempted to dictionary_foreach_start_rw() on a destroyed dictionary");
- dfe->counter = 0;
- dfe->item = NULL;
- dfe->name = NULL;
- dfe->value = NULL;
- return NULL;
- }
-
- dfe->counter = 0;
- dfe->dict = dict;
- dfe->rw = rw;
- dfe->locked = true;
- ll_recursive_lock(dict, dfe->rw);
-
- // get the first item from the list
- DICTIONARY_ITEM *item = dict->items.list;
-
- // skip all the deleted items
- while(item && !item_check_and_acquire(dict, item))
- item = item->next;
-
- if(likely(item)) {
- dfe->item = item;
- dfe->name = (char *)item_get_name(item);
- dfe->value = item->shared->value;
- }
- else {
- dfe->item = NULL;
- dfe->name = NULL;
- dfe->value = NULL;
- }
-
- if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) {
- ll_recursive_unlock(dfe->dict, dfe->rw);
- dfe->locked = false;
- }
-
- return dfe->value;
-}
-
-void *dictionary_foreach_next(DICTFE *dfe) {
- if(unlikely(!dfe || !dfe->dict)) return NULL;
-
- if(unlikely(is_dictionary_destroyed(dfe->dict))) {
- internal_error(true, "DICTIONARY: attempted to dictionary_foreach_next() on a destroyed dictionary");
- dfe->item = NULL;
- dfe->name = NULL;
- dfe->value = NULL;
- return NULL;
- }
-
- if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT) || !dfe->locked) {
- ll_recursive_lock(dfe->dict, dfe->rw);
- dfe->locked = true;
- }
-
- // the item we just did
- DICTIONARY_ITEM *item = dfe->item;
-
- // get the next item from the list
- DICTIONARY_ITEM *item_next = (item) ? item->next : NULL;
-
- // skip all the deleted items until one that can be acquired is found
- while(item_next && !item_check_and_acquire(dfe->dict, item_next))
- item_next = item_next->next;
-
- if(likely(item)) {
- dict_item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dfe->dict, item, dfe->rw);
- // item_release(dfe->dict, item);
- }
-
- item = item_next;
- if(likely(item)) {
- dfe->item = item;
- dfe->name = (char *)item_get_name(item);
- dfe->value = item->shared->value;
- dfe->counter++;
- }
- else {
- dfe->item = NULL;
- dfe->name = NULL;
- dfe->value = NULL;
- }
-
- if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) {
- ll_recursive_unlock(dfe->dict, dfe->rw);
- dfe->locked = false;
- }
-
- return dfe->value;
-}
-
-void dictionary_foreach_unlock(DICTFE *dfe) {
- if(dfe->locked) {
- ll_recursive_unlock(dfe->dict, dfe->rw);
- dfe->locked = false;
- }
-}
-
-void dictionary_foreach_done(DICTFE *dfe) {
- if(unlikely(!dfe || !dfe->dict)) return;
-
- if(unlikely(is_dictionary_destroyed(dfe->dict))) {
- internal_error(true, "DICTIONARY: attempted to dictionary_foreach_next() on a destroyed dictionary");
- return;
- }
-
- // the item we just did
- DICTIONARY_ITEM *item = dfe->item;
-
- // release it, so that it can possibly be deleted
- if(likely(item)) {
- dict_item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dfe->dict, item, dfe->rw);
- // item_release(dfe->dict, item);
- }
-
- if(likely(dfe->rw != DICTIONARY_LOCK_REENTRANT) && dfe->locked) {
- ll_recursive_unlock(dfe->dict, dfe->rw);
- dfe->locked = false;
- }
-
- dfe->dict = NULL;
- dfe->item = NULL;
- dfe->name = NULL;
- dfe->value = NULL;
- dfe->counter = 0;
-}
-
-// ----------------------------------------------------------------------------
-// API - walk through the dictionary.
-// The dictionary is locked for reading while this happens
-// do not use other dictionary calls while walking the dictionary - deadlock!
-
-int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *entry, void *data), void *data) {
- if(unlikely(!dict || !callback)) return 0;
-
- if(unlikely(is_dictionary_destroyed(dict))) {
- internal_error(true, "DICTIONARY: attempted to dictionary_walkthrough_rw() on a destroyed dictionary");
- return 0;
- }
-
- ll_recursive_lock(dict, rw);
-
- DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict);
-
- // written in such a way, that the callback can delete the active element
-
- int ret = 0;
- DICTIONARY_ITEM *item = dict->items.list, *item_next;
- while(item) {
-
- // skip the deleted items
- if(unlikely(!item_check_and_acquire(dict, item))) {
- item = item->next;
- continue;
- }
-
- if(unlikely(rw == DICTIONARY_LOCK_REENTRANT))
- ll_recursive_unlock(dict, rw);
-
- int r = callback(item, item->shared->value, data);
-
- if(unlikely(rw == DICTIONARY_LOCK_REENTRANT))
- ll_recursive_lock(dict, rw);
-
- // since we have a reference counter, this item cannot be deleted
- // until we release the reference counter, so the pointers are there
- item_next = item->next;
-
- dict_item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dict, item, rw);
- // item_release(dict, item);
-
- if(unlikely(r < 0)) {
- ret = r;
- break;
- }
-
- ret += r;
-
- item = item_next;
- }
-
- ll_recursive_unlock(dict, rw);
-
- return ret;
-}
-
-// ----------------------------------------------------------------------------
-// sorted walkthrough
-
-typedef int (*qsort_compar)(const void *item1, const void *item2);
-
-static int dictionary_sort_compar(const void *item1, const void *item2) {
- return strcmp(item_get_name((*(DICTIONARY_ITEM **)item1)), item_get_name((*(DICTIONARY_ITEM **)item2)));
-}
-
-int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *entry, void *data), void *data, dictionary_sorted_compar compar) {
- if(unlikely(!dict || !callback)) return 0;
-
- if(unlikely(is_dictionary_destroyed(dict))) {
- internal_error(true, "DICTIONARY: attempted to dictionary_sorted_walkthrough_rw() on a destroyed dictionary");
- return 0;
- }
-
- DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict);
-
- ll_recursive_lock(dict, rw);
- size_t entries = __atomic_load_n(&dict->entries, __ATOMIC_RELAXED);
- DICTIONARY_ITEM **array = mallocz(sizeof(DICTIONARY_ITEM *) * entries);
-
- size_t i;
- DICTIONARY_ITEM *item;
- for(item = dict->items.list, i = 0; item && i < entries; item = item->next) {
- if(likely(item_check_and_acquire(dict, item)))
- array[i++] = item;
- }
- ll_recursive_unlock(dict, rw);
-
- if(unlikely(i != entries))
- entries = i;
-
- if(compar)
- qsort(array, entries, sizeof(DICTIONARY_ITEM *), (qsort_compar)compar);
- else
- qsort(array, entries, sizeof(DICTIONARY_ITEM *), dictionary_sort_compar);
-
- bool callit = true;
- int ret = 0, r;
- for(i = 0; i < entries ;i++) {
- item = array[i];
-
- if(callit)
- r = callback(item, item->shared->value, data);
-
- dict_item_release_and_check_if_it_is_deleted_and_can_be_removed_under_this_lock_mode(dict, item, rw);
- // item_release(dict, item);
-
- if(r < 0) {
- ret = r;
- r = 0;
-
- // stop calling the callback,
- // but we have to continue, to release all the reference counters
- callit = false;
- }
- else
- ret += r;
- }
-
- freez(array);
-
- return ret;
-}
-
-// ----------------------------------------------------------------------------
-// THREAD_CACHE
-
-static __thread Pvoid_t thread_cache_judy_array = NULL;
-
-void *thread_cache_entry_get_or_set(void *key,
- ssize_t key_length,
- void *value,
- void *(*transform_the_value_before_insert)(void *key, size_t key_length, void *value)
- ) {
- if(unlikely(!key || !key_length)) return NULL;
-
- if(key_length == -1)
- key_length = (ssize_t)strlen((char *)key) + 1;
-
- JError_t J_Error;
- Pvoid_t *Rc = JudyHSIns(&thread_cache_judy_array, key, key_length, &J_Error);
- if (unlikely(Rc == PJERR)) {
- fatal("THREAD_CACHE: Cannot insert entry to JudyHS, JU_ERRNO_* == %u, ID == %d",
- JU_ERRNO(&J_Error), JU_ERRID(&J_Error));
- }
-
- if(*Rc == 0) {
- // new item added
-
- *Rc = (transform_the_value_before_insert) ? transform_the_value_before_insert(key, key_length, value) : value;
- }
-
- return *Rc;
-}
-
-void thread_cache_destroy(void) {
- if(unlikely(!thread_cache_judy_array)) return;
-
- JError_t J_Error;
- Word_t ret = JudyHSFreeArray(&thread_cache_judy_array, &J_Error);
- if(unlikely(ret == (Word_t) JERR)) {
- netdata_log_error("THREAD_CACHE: Cannot destroy JudyHS, JU_ERRNO_* == %u, ID == %d",
- JU_ERRNO(&J_Error), JU_ERRID(&J_Error));
- }
-
- internal_error(true, "THREAD_CACHE: hash table freed %lu bytes", ret);
-
- thread_cache_judy_array = NULL;
-}
-
-// ----------------------------------------------------------------------------
-// unit test
-
-static void dictionary_unittest_free_char_pp(char **pp, size_t entries) {
- for(size_t i = 0; i < entries ;i++)
- freez(pp[i]);
-
- freez(pp);
-}
-
-static char **dictionary_unittest_generate_names(size_t entries) {
- char **names = mallocz(sizeof(char *) * entries);
- for(size_t i = 0; i < entries ;i++) {
- char buf[25 + 1] = "";
- snprintfz(buf, sizeof(buf) - 1, "name.%zu.0123456789.%zu!@#$%%^&*(),./[]{}\\|~`", i, entries / 2 + i);
- names[i] = strdupz(buf);
- }
- return names;
-}
-
-static char **dictionary_unittest_generate_values(size_t entries) {
- char **values = mallocz(sizeof(char *) * entries);
- for(size_t i = 0; i < entries ;i++) {
- char buf[25 + 1] = "";
- snprintfz(buf, sizeof(buf) - 1, "value-%zu-0987654321.%zu%%^&*(),. \t !@#$/[]{}\\|~`", i, entries / 2 + i);
- values[i] = strdupz(buf);
- }
- return values;
-}
-
-static size_t dictionary_unittest_set_clone(DICTIONARY *dict, char **names, char **values, size_t entries) {
- size_t errors = 0;
- for(size_t i = 0; i < entries ;i++) {
- size_t vallen = strlen(values[i]) + 1;
- char *val = (char *)dictionary_set(dict, names[i], values[i], vallen);
- if(val == values[i]) { fprintf(stderr, ">>> %s() returns reference to value\n", __FUNCTION__); errors++; }
- if(!val || memcmp(val, values[i], vallen) != 0) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
- }
- return errors;
-}
-
-static size_t dictionary_unittest_set_null(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)values;
- size_t errors = 0;
- size_t i = 0;
- for(; i < entries ;i++) {
- void *val = dictionary_set(dict, names[i], NULL, 0);
- if(val != NULL) { fprintf(stderr, ">>> %s() returns a non NULL value\n", __FUNCTION__); errors++; }
- }
- if(dictionary_entries(dict) != i) {
- fprintf(stderr, ">>> %s() dictionary items do not match\n", __FUNCTION__);
- errors++;
- }
- return errors;
-}
-
-
-static size_t dictionary_unittest_set_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries) {
- size_t errors = 0;
- for(size_t i = 0; i < entries ;i++) {
- size_t vallen = strlen(values[i]) + 1;
- char *val = (char *)dictionary_set(dict, names[i], values[i], vallen);
- if(val != values[i]) { fprintf(stderr, ">>> %s() returns invalid pointer to value\n", __FUNCTION__); errors++; }
- }
- return errors;
-}
-
-static size_t dictionary_unittest_get_clone(DICTIONARY *dict, char **names, char **values, size_t entries) {
- size_t errors = 0;
- for(size_t i = 0; i < entries ;i++) {
- size_t vallen = strlen(values[i]) + 1;
- char *val = (char *)dictionary_get(dict, names[i]);
- if(val == values[i]) { fprintf(stderr, ">>> %s() returns reference to value\n", __FUNCTION__); errors++; }
- if(!val || memcmp(val, values[i], vallen) != 0) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
- }
- return errors;
-}
-
-static size_t dictionary_unittest_get_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries) {
- size_t errors = 0;
- for(size_t i = 0; i < entries ;i++) {
- char *val = (char *)dictionary_get(dict, names[i]);
- if(val != values[i]) { fprintf(stderr, ">>> %s() returns invalid pointer to value\n", __FUNCTION__); errors++; }
- }
- return errors;
-}
-
-static size_t dictionary_unittest_get_nonexisting(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)names;
- size_t errors = 0;
- for(size_t i = 0; i < entries ;i++) {
- char *val = (char *)dictionary_get(dict, values[i]);
- if(val) { fprintf(stderr, ">>> %s() returns non-existing item\n", __FUNCTION__); errors++; }
- }
- return errors;
-}
-
-static size_t dictionary_unittest_del_nonexisting(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)names;
- size_t errors = 0;
- for(size_t i = 0; i < entries ;i++) {
- bool ret = dictionary_del(dict, values[i]);
- if(ret) { fprintf(stderr, ">>> %s() deleted non-existing item\n", __FUNCTION__); errors++; }
- }
- return errors;
-}
-
-static size_t dictionary_unittest_del_existing(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)values;
- size_t errors = 0;
-
- size_t forward_from = 0, forward_to = entries / 3;
- size_t middle_from = forward_to, middle_to = entries * 2 / 3;
- size_t backward_from = middle_to, backward_to = entries;
-
- for(size_t i = forward_from; i < forward_to ;i++) {
- bool ret = dictionary_del(dict, names[i]);
- if(!ret) { fprintf(stderr, ">>> %s() didn't delete (forward) existing item\n", __FUNCTION__); errors++; }
- }
-
- for(size_t i = middle_to - 1; i >= middle_from ;i--) {
- bool ret = dictionary_del(dict, names[i]);
- if(!ret) { fprintf(stderr, ">>> %s() didn't delete (middle) existing item\n", __FUNCTION__); errors++; }
- }
-
- for(size_t i = backward_to - 1; i >= backward_from ;i--) {
- bool ret = dictionary_del(dict, names[i]);
- if(!ret) { fprintf(stderr, ">>> %s() didn't delete (backward) existing item\n", __FUNCTION__); errors++; }
- }
-
- return errors;
-}
-
-static size_t dictionary_unittest_reset_clone(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)values;
- // set the name as value too
- size_t errors = 0;
- for(size_t i = 0; i < entries ;i++) {
- size_t vallen = strlen(names[i]) + 1;
- char *val = (char *)dictionary_set(dict, names[i], names[i], vallen);
- if(val == names[i]) { fprintf(stderr, ">>> %s() returns reference to value\n", __FUNCTION__); errors++; }
- if(!val || memcmp(val, names[i], vallen) != 0) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
- }
- return errors;
-}
-
-static size_t dictionary_unittest_reset_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)values;
- // set the name as value too
- size_t errors = 0;
- for(size_t i = 0; i < entries ;i++) {
- size_t vallen = strlen(names[i]) + 1;
- char *val = (char *)dictionary_set(dict, names[i], names[i], vallen);
- if(val != names[i]) { fprintf(stderr, ">>> %s() returns invalid pointer to value\n", __FUNCTION__); errors++; }
- if(!val) { fprintf(stderr, ">>> %s() returns invalid value\n", __FUNCTION__); errors++; }
- }
- return errors;
-}
-
-static size_t dictionary_unittest_reset_dont_overwrite_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries) {
- // set the name as value too
- size_t errors = 0;
- for(size_t i = 0; i < entries ;i++) {
- size_t vallen = strlen(names[i]) + 1;
- char *val = (char *)dictionary_set(dict, names[i], names[i], vallen);
- if(val != values[i]) { fprintf(stderr, ">>> %s() returns invalid pointer to value\n", __FUNCTION__); errors++; }
- }
- return errors;
-}
-
-static int dictionary_unittest_walkthrough_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value __maybe_unused, void *data __maybe_unused) {
- return 1;
-}
-
-static size_t dictionary_unittest_walkthrough(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)names;
- (void)values;
- int sum = dictionary_walkthrough_read(dict, dictionary_unittest_walkthrough_callback, NULL);
- if(sum < (int)entries) return entries - sum;
- else return sum - entries;
-}
-
-static int dictionary_unittest_walkthrough_delete_this_callback(const DICTIONARY_ITEM *item, void *value __maybe_unused, void *data) {
- const char *name = dictionary_acquired_item_name((DICTIONARY_ITEM *)item);
-
- if(!dictionary_del((DICTIONARY *)data, name))
- return 0;
-
- return 1;
-}
-
-static size_t dictionary_unittest_walkthrough_delete_this(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)names;
- (void)values;
- int sum = dictionary_walkthrough_write(dict, dictionary_unittest_walkthrough_delete_this_callback, dict);
- if(sum < (int)entries) return entries - sum;
- else return sum - entries;
-}
-
-static int dictionary_unittest_walkthrough_stop_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value __maybe_unused, void *data __maybe_unused) {
- return -1;
-}
-
-static size_t dictionary_unittest_walkthrough_stop(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)names;
- (void)values;
- (void)entries;
- int sum = dictionary_walkthrough_read(dict, dictionary_unittest_walkthrough_stop_callback, NULL);
- if(sum != -1) return 1;
- return 0;
-}
-
-static size_t dictionary_unittest_foreach(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)names;
- (void)values;
- (void)entries;
- size_t count = 0;
- char *item;
- dfe_start_read(dict, item)
- count++;
- dfe_done(item);
-
- if(count > entries) return count - entries;
- return entries - count;
-}
-
-static size_t dictionary_unittest_foreach_delete_this(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)names;
- (void)values;
- (void)entries;
- size_t count = 0;
- char *item;
- dfe_start_write(dict, item)
- if(dictionary_del(dict, item_dfe.name)) count++;
- dfe_done(item);
-
- if(count > entries) return count - entries;
- return entries - count;
-}
-
-static size_t dictionary_unittest_destroy(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)names;
- (void)values;
- (void)entries;
- size_t bytes = dictionary_destroy(dict);
- fprintf(stderr, " %s() freed %zu bytes,", __FUNCTION__, bytes);
- return 0;
-}
-
-static usec_t dictionary_unittest_run_and_measure_time(DICTIONARY *dict, char *message, char **names, char **values, size_t entries, size_t *errors, size_t (*callback)(DICTIONARY *dict, char **names, char **values, size_t entries)) {
- fprintf(stderr, "%40s ... ", message);
-
- usec_t started = now_realtime_usec();
- size_t errs = callback(dict, names, values, entries);
- usec_t ended = now_realtime_usec();
- usec_t dt = ended - started;
-
- if(callback == dictionary_unittest_destroy) dict = NULL;
-
- long int found_ok = 0, found_deleted = 0, found_referenced = 0;
- if(dict) {
- DICTIONARY_ITEM *item;
- DOUBLE_LINKED_LIST_FOREACH_FORWARD(dict->items.list, item, prev, next) {
- if(item->refcount >= 0 && !(item ->flags & ITEM_FLAG_DELETED))
- found_ok++;
- else
- found_deleted++;
-
- if(item->refcount > 0)
- found_referenced++;
- }
- }
-
- fprintf(stderr, " %zu errors, %d (found %ld) items in dictionary, %d (found %ld) referenced, %d (found %ld) deleted, %"PRIu64" usec \n",
- errs, dict?dict->entries:0, found_ok, dict?dict->referenced_items:0, found_referenced, dict?dict->pending_deletion_items:0, found_deleted, dt);
- *errors += errs;
- return dt;
-}
-
-static void dictionary_unittest_clone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
- dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_clone);
- dictionary_unittest_run_and_measure_time(dict, "getting entries", names, values, entries, errors, dictionary_unittest_get_clone);
- dictionary_unittest_run_and_measure_time(dict, "getting non-existing entries", names, values, entries, errors, dictionary_unittest_get_nonexisting);
- dictionary_unittest_run_and_measure_time(dict, "resetting entries", names, values, entries, errors, dictionary_unittest_reset_clone);
- dictionary_unittest_run_and_measure_time(dict, "deleting non-existing entries", names, values, entries, errors, dictionary_unittest_del_nonexisting);
- dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, errors, dictionary_unittest_foreach);
- dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback", names, values, entries, errors, dictionary_unittest_walkthrough);
- dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback stop", names, values, entries, errors, dictionary_unittest_walkthrough_stop);
- dictionary_unittest_run_and_measure_time(dict, "deleting existing entries", names, values, entries, errors, dictionary_unittest_del_existing);
- dictionary_unittest_run_and_measure_time(dict, "walking through empty", names, values, 0, errors, dictionary_unittest_walkthrough);
- dictionary_unittest_run_and_measure_time(dict, "traverse foreach empty", names, values, 0, errors, dictionary_unittest_foreach);
- dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, errors, dictionary_unittest_destroy);
-}
-
-static void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
- dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_nonclone);
- dictionary_unittest_run_and_measure_time(dict, "getting entries", names, values, entries, errors, dictionary_unittest_get_nonclone);
- dictionary_unittest_run_and_measure_time(dict, "getting non-existing entries", names, values, entries, errors, dictionary_unittest_get_nonexisting);
- dictionary_unittest_run_and_measure_time(dict, "resetting entries", names, values, entries, errors, dictionary_unittest_reset_nonclone);
- dictionary_unittest_run_and_measure_time(dict, "deleting non-existing entries", names, values, entries, errors, dictionary_unittest_del_nonexisting);
- dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, errors, dictionary_unittest_foreach);
- dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback", names, values, entries, errors, dictionary_unittest_walkthrough);
- dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback stop", names, values, entries, errors, dictionary_unittest_walkthrough_stop);
- dictionary_unittest_run_and_measure_time(dict, "deleting existing entries", names, values, entries, errors, dictionary_unittest_del_existing);
- dictionary_unittest_run_and_measure_time(dict, "walking through empty", names, values, 0, errors, dictionary_unittest_walkthrough);
- dictionary_unittest_run_and_measure_time(dict, "traverse foreach empty", names, values, 0, errors, dictionary_unittest_foreach);
- dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, errors, dictionary_unittest_destroy);
-}
-
-struct dictionary_unittest_sorting {
- const char *old_name;
- const char *old_value;
- size_t count;
-};
-
-static int dictionary_unittest_sorting_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
- const char *name = dictionary_acquired_item_name((DICTIONARY_ITEM *)item);
- struct dictionary_unittest_sorting *t = (struct dictionary_unittest_sorting *)data;
- const char *v = (const char *)value;
-
- int ret = 0;
- if(t->old_name && strcmp(t->old_name, name) > 0) {
- fprintf(stderr, "name '%s' should be after '%s'\n", t->old_name, name);
- ret = 1;
- }
- t->count++;
- t->old_name = name;
- t->old_value = v;
-
- return ret;
-}
-
-static size_t dictionary_unittest_sorted_walkthrough(DICTIONARY *dict, char **names, char **values, size_t entries) {
- (void)names;
- (void)values;
- struct dictionary_unittest_sorting tmp = { .old_name = NULL, .old_value = NULL, .count = 0 };
- size_t errors;
- errors = dictionary_sorted_walkthrough_read(dict, dictionary_unittest_sorting_callback, &tmp);
-
- if(tmp.count != entries) {
- fprintf(stderr, "Expected %zu entries, counted %zu\n", entries, tmp.count);
- errors++;
- }
- return errors;
-}
-
-static void dictionary_unittest_sorting(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
- dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_clone);
- dictionary_unittest_run_and_measure_time(dict, "sorted walkthrough", names, values, entries, errors, dictionary_unittest_sorted_walkthrough);
-}
-
-static void dictionary_unittest_null_dfe(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
- dictionary_unittest_run_and_measure_time(dict, "adding null value entries", names, values, entries, errors, dictionary_unittest_set_null);
- dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, errors, dictionary_unittest_foreach);
-}
-
-
-static int unittest_check_dictionary_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value __maybe_unused, void *data __maybe_unused) {
- return 1;
-}
-
-static size_t unittest_check_dictionary(const char *label, DICTIONARY *dict, size_t traversable, size_t active_items, size_t deleted_items, size_t referenced_items, size_t pending_deletion) {
- size_t errors = 0;
-
- size_t ll = 0;
- void *t;
- dfe_start_read(dict, t)
- ll++;
- dfe_done(t);
-
- fprintf(stderr, "DICT %-20s: dictionary foreach entries %zu, expected %zu...\t\t\t\t\t",
- label, ll, traversable);
- if(ll != traversable) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- ll = dictionary_walkthrough_read(dict, unittest_check_dictionary_callback, NULL);
- fprintf(stderr, "DICT %-20s: dictionary walkthrough entries %zu, expected %zu...\t\t\t\t",
- label, ll, traversable);
- if(ll != traversable) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- ll = dictionary_sorted_walkthrough_read(dict, unittest_check_dictionary_callback, NULL);
- fprintf(stderr, "DICT %-20s: dictionary sorted walkthrough entries %zu, expected %zu...\t\t\t",
- label, ll, traversable);
- if(ll != traversable) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- DICTIONARY_ITEM *item;
- size_t active = 0, deleted = 0, referenced = 0, pending = 0;
- for(item = dict->items.list; item; item = item->next) {
- if(!(item->flags & ITEM_FLAG_DELETED) && !(item->shared->flags & ITEM_FLAG_DELETED))
- active++;
- else {
- deleted++;
-
- if(item->refcount == 0)
- pending++;
- }
-
- if(item->refcount > 0)
- referenced++;
- }
-
- fprintf(stderr, "DICT %-20s: dictionary active items reported %d, counted %zu, expected %zu...\t\t\t",
- label, dict->entries, active, active_items);
- if(active != active_items || active != (size_t)dict->entries) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- fprintf(stderr, "DICT %-20s: dictionary deleted items counted %zu, expected %zu...\t\t\t\t",
- label, deleted, deleted_items);
- if(deleted != deleted_items) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- fprintf(stderr, "DICT %-20s: dictionary referenced items reported %d, counted %zu, expected %zu...\t\t",
- label, dict->referenced_items, referenced, referenced_items);
- if(referenced != referenced_items || dict->referenced_items != (long int)referenced) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- fprintf(stderr, "DICT %-20s: dictionary pending deletion items reported %d, counted %zu, expected %zu...\t",
- label, dict->pending_deletion_items, pending, pending_deletion);
- if(pending != pending_deletion || pending != (size_t)dict->pending_deletion_items) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- return errors;
-}
-
-static int check_item_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
- return value == data;
-}
-
-static size_t unittest_check_item(const char *label, DICTIONARY *dict,
- DICTIONARY_ITEM *item, const char *name, const char *value, int refcount,
- ITEM_FLAGS deleted_flags, bool searchable, bool browsable, bool linked) {
- size_t errors = 0;
-
- fprintf(stderr, "ITEM %-20s: name is '%s', expected '%s'...\t\t\t\t\t\t", label, item_get_name(item), name);
- if(strcmp(item_get_name(item), name) != 0) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- fprintf(stderr, "ITEM %-20s: value is '%s', expected '%s'...\t\t\t\t\t", label, (const char *)item->shared->value, value);
- if(strcmp((const char *)item->shared->value, value) != 0) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- fprintf(stderr, "ITEM %-20s: refcount is %d, expected %d...\t\t\t\t\t\t\t", label, item->refcount, refcount);
- if (item->refcount != refcount) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- fprintf(stderr, "ITEM %-20s: deleted flag is %s, expected %s...\t\t\t\t\t", label,
- (item->flags & ITEM_FLAG_DELETED || item->shared->flags & ITEM_FLAG_DELETED)?"true":"false",
- (deleted_flags & ITEM_FLAG_DELETED)?"true":"false");
-
- if ((item->flags & ITEM_FLAG_DELETED || item->shared->flags & ITEM_FLAG_DELETED) != (deleted_flags & ITEM_FLAG_DELETED)) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- void *v = dictionary_get(dict, name);
- bool found = v == item->shared->value;
- fprintf(stderr, "ITEM %-20s: searchable %5s, expected %5s...\t\t\t\t\t\t", label,
- found?"true":"false", searchable?"true":"false");
- if(found != searchable) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- found = false;
- void *t;
- dfe_start_read(dict, t) {
- if(t == item->shared->value) found = true;
- }
- dfe_done(t);
-
- fprintf(stderr, "ITEM %-20s: dfe browsable %5s, expected %5s...\t\t\t\t\t", label,
- found?"true":"false", browsable?"true":"false");
- if(found != browsable) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- found = dictionary_walkthrough_read(dict, check_item_callback, item->shared->value);
- fprintf(stderr, "ITEM %-20s: walkthrough browsable %5s, expected %5s...\t\t\t\t", label,
- found?"true":"false", browsable?"true":"false");
- if(found != browsable) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- found = dictionary_sorted_walkthrough_read(dict, check_item_callback, item->shared->value);
- fprintf(stderr, "ITEM %-20s: sorted walkthrough browsable %5s, expected %5s...\t\t\t", label,
- found?"true":"false", browsable?"true":"false");
- if(found != browsable) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- found = false;
- DICTIONARY_ITEM *n;
- for(n = dict->items.list; n ;n = n->next)
- if(n == item) found = true;
-
- fprintf(stderr, "ITEM %-20s: linked %5s, expected %5s...\t\t\t\t\t\t", label,
- found?"true":"false", linked?"true":"false");
- if(found != linked) {
- fprintf(stderr, "FAILED\n");
- errors++;
- }
- else
- fprintf(stderr, "OK\n");
-
- return errors;
-}
-
-struct thread_unittest {
- int join;
- DICTIONARY *dict;
- int dups;
-
- netdata_thread_t thread;
- struct dictionary_stats stats;
-};
-
-static void *unittest_dict_thread(void *arg) {
- struct thread_unittest *tu = arg;
- for(; 1 ;) {
- if(__atomic_load_n(&tu->join, __ATOMIC_RELAXED))
- break;
-
- DICT_ITEM_CONST DICTIONARY_ITEM *item =
- dictionary_set_and_acquire_item_advanced(tu->dict, "dict thread checking 1234567890",
- -1, NULL, 0, NULL);
- tu->stats.ops.inserts++;
-
- dictionary_get(tu->dict, dictionary_acquired_item_name(item));
- tu->stats.ops.searches++;
-
- void *t1;
- dfe_start_write(tu->dict, t1) {
-
- // this should delete the referenced item
- dictionary_del(tu->dict, t1_dfe.name);
- tu->stats.ops.deletes++;
-
- void *t2;
- dfe_start_write(tu->dict, t2) {
- // this should add another
- dictionary_set(tu->dict, t2_dfe.name, NULL, 0);
- tu->stats.ops.inserts++;
-
- dictionary_get(tu->dict, dictionary_acquired_item_name(item));
- tu->stats.ops.searches++;
-
- // and this should delete it again
- dictionary_del(tu->dict, t2_dfe.name);
- tu->stats.ops.deletes++;
- }
- dfe_done(t2);
- tu->stats.ops.traversals++;
-
- // this should fail to add it
- dictionary_set(tu->dict, t1_dfe.name, NULL, 0);
- tu->stats.ops.inserts++;
-
- dictionary_del(tu->dict, t1_dfe.name);
- tu->stats.ops.deletes++;
- }
- dfe_done(t1);
- tu->stats.ops.traversals++;
-
- for(int i = 0; i < tu->dups ; i++) {
- dictionary_acquired_item_dup(tu->dict, item);
- dictionary_get(tu->dict, dictionary_acquired_item_name(item));
- tu->stats.ops.searches++;
- }
-
- for(int i = 0; i < tu->dups ; i++) {
- dictionary_acquired_item_release(tu->dict, item);
- dictionary_del(tu->dict, dictionary_acquired_item_name(item));
- tu->stats.ops.deletes++;
- }
-
- dictionary_acquired_item_release(tu->dict, item);
- dictionary_del(tu->dict, "dict thread checking 1234567890");
- tu->stats.ops.deletes++;
-
- // test concurrent deletions and flushes
- {
- if(gettid() % 2) {
- char buf [256 + 1];
-
- for (int i = 0; i < 1000; i++) {
- snprintfz(buf, sizeof(buf) - 1, "del/flush test %d", i);
- dictionary_set(tu->dict, buf, NULL, 0);
- tu->stats.ops.inserts++;
- }
-
- for (int i = 0; i < 1000; i++) {
- snprintfz(buf, sizeof(buf) - 1, "del/flush test %d", i);
- dictionary_del(tu->dict, buf);
- tu->stats.ops.deletes++;
- }
- }
- else {
- for (int i = 0; i < 10; i++) {
- dictionary_flush(tu->dict);
- tu->stats.ops.flushes++;
- }
- }
- }
- }
-
- return arg;
-}
-
-static int dictionary_unittest_threads() {
- time_t seconds_to_run = 5;
- int threads_to_create = 2;
-
- struct thread_unittest tu[threads_to_create];
- memset(tu, 0, sizeof(struct thread_unittest) * threads_to_create);
-
- fprintf(
- stderr,
- "\nChecking dictionary concurrency with %d threads for %lld seconds...\n",
- threads_to_create,
- (long long)seconds_to_run);
-
- // threads testing of dictionary
- struct dictionary_stats stats = {};
- tu[0].join = 0;
- tu[0].dups = 1;
- tu[0].dict = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &stats, 0);
-
- for (int i = 0; i < threads_to_create; i++) {
- if(i)
- tu[i] = tu[0];
-
- char buf[100 + 1];
- snprintf(buf, 100, "dict%d", i);
- netdata_thread_create(
- &tu[i].thread,
- buf,
- NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE,
- unittest_dict_thread,
- &tu[i]);
- }
-
- sleep_usec(seconds_to_run * USEC_PER_SEC);
-
- for (int i = 0; i < threads_to_create; i++) {
- __atomic_store_n(&tu[i].join, 1, __ATOMIC_RELAXED);
-
- void *retval;
- netdata_thread_join(tu[i].thread, &retval);
-
- if(i) {
- tu[0].stats.ops.inserts += tu[i].stats.ops.inserts;
- tu[0].stats.ops.deletes += tu[i].stats.ops.deletes;
- tu[0].stats.ops.searches += tu[i].stats.ops.searches;
- tu[0].stats.ops.flushes += tu[i].stats.ops.flushes;
- tu[0].stats.ops.traversals += tu[i].stats.ops.traversals;
- }
- }
-
- fprintf(stderr,
- "CALLS : inserts %zu"
- ", deletes %zu"
- ", searches %zu"
- ", traversals %zu"
- ", flushes %zu"
- "\n",
- tu[0].stats.ops.inserts,
- tu[0].stats.ops.deletes,
- tu[0].stats.ops.searches,
- tu[0].stats.ops.traversals,
- tu[0].stats.ops.flushes
- );
-
-#ifdef DICT_WITH_STATS
- fprintf(stderr,
- "ACTUAL: inserts %zu"
- ", deletes %zu"
- ", searches %zu"
- ", traversals %zu"
- ", resets %zu"
- ", flushes %zu"
- ", entries %d"
- ", referenced_items %d"
- ", pending deletions %d"
- ", check spins %zu"
- ", insert spins %zu"
- ", delete spins %zu"
- ", search ignores %zu"
- "\n",
- stats.ops.inserts,
- stats.ops.deletes,
- stats.ops.searches,
- stats.ops.traversals,
- stats.ops.resets,
- stats.ops.flushes,
- tu[0].dict->entries,
- tu[0].dict->referenced_items,
- tu[0].dict->pending_deletion_items,
- stats.spin_locks.use_spins,
- stats.spin_locks.insert_spins,
- stats.spin_locks.delete_spins,
- stats.spin_locks.search_spins
- );
-#endif
-
- dictionary_destroy(tu[0].dict);
- return 0;
-}
-
-struct thread_view_unittest {
- int join;
- DICTIONARY *master;
- DICTIONARY *view;
- DICTIONARY_ITEM *item_master;
- int dups;
-};
-
-static void *unittest_dict_master_thread(void *arg) {
- struct thread_view_unittest *tv = arg;
-
- DICTIONARY_ITEM *item = NULL;
- int loops = 0;
- while(!__atomic_load_n(&tv->join, __ATOMIC_RELAXED)) {
-
- if(!item)
- item = dictionary_set_and_acquire_item(tv->master, "ITEM1", "123", strlen("123") + 1);
-
- if(__atomic_load_n(&tv->item_master, __ATOMIC_RELAXED) != NULL) {
- dictionary_acquired_item_release(tv->master, item);
- dictionary_del(tv->master, "ITEM1");
- item = NULL;
- loops++;
- continue;
- }
-
- dictionary_acquired_item_dup(tv->master, item); // for the view thread
- __atomic_store_n(&tv->item_master, item, __ATOMIC_RELAXED);
- dictionary_del(tv->master, "ITEM1");
-
-
- for(int i = 0; i < tv->dups + loops ; i++) {
- dictionary_acquired_item_dup(tv->master, item);
- }
-
- for(int i = 0; i < tv->dups + loops ; i++) {
- dictionary_acquired_item_release(tv->master, item);
- }
-
- dictionary_acquired_item_release(tv->master, item);
-
- item = NULL;
- loops = 0;
- }
-
- return arg;
-}
-
-static void *unittest_dict_view_thread(void *arg) {
- struct thread_view_unittest *tv = arg;
-
- DICTIONARY_ITEM *m_item = NULL;
-
- while(!__atomic_load_n(&tv->join, __ATOMIC_RELAXED)) {
- if(!(m_item = __atomic_load_n(&tv->item_master, __ATOMIC_RELAXED)))
- continue;
-
- DICTIONARY_ITEM *v_item = dictionary_view_set_and_acquire_item(tv->view, "ITEM2", m_item);
- dictionary_acquired_item_release(tv->master, m_item);
- __atomic_store_n(&tv->item_master, NULL, __ATOMIC_RELAXED);
-
- for(int i = 0; i < tv->dups ; i++) {
- dictionary_acquired_item_dup(tv->view, v_item);
- }
-
- for(int i = 0; i < tv->dups ; i++) {
- dictionary_acquired_item_release(tv->view, v_item);
- }
-
- dictionary_del(tv->view, "ITEM2");
-
- while(!__atomic_load_n(&tv->join, __ATOMIC_RELAXED) && !(m_item = __atomic_load_n(&tv->item_master, __ATOMIC_RELAXED))) {
- dictionary_acquired_item_dup(tv->view, v_item);
- dictionary_acquired_item_release(tv->view, v_item);
- }
-
- dictionary_acquired_item_release(tv->view, v_item);
- }
-
- return arg;
-}
-
-static int dictionary_unittest_view_threads() {
-
- struct thread_view_unittest tv = {
- .join = 0,
- .master = NULL,
- .view = NULL,
- .item_master = NULL,
- .dups = 1,
- };
-
- // threads testing of dictionary
- struct dictionary_stats stats_master = {};
- struct dictionary_stats stats_view = {};
- tv.master = dictionary_create_advanced(DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE, &stats_master, 0);
- tv.view = dictionary_create_view(tv.master);
- tv.view->stats = &stats_view;
-
- time_t seconds_to_run = 5;
- fprintf(
- stderr,
- "\nChecking dictionary concurrency with 1 master and 1 view threads for %lld seconds...\n",
- (long long)seconds_to_run);
-
- netdata_thread_t master_thread, view_thread;
- tv.join = 0;
-
- netdata_thread_create(
- &master_thread,
- "master",
- NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE,
- unittest_dict_master_thread,
- &tv);
-
- netdata_thread_create(
- &view_thread,
- "view",
- NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE,
- unittest_dict_view_thread,
- &tv);
-
- sleep_usec(seconds_to_run * USEC_PER_SEC);
-
- __atomic_store_n(&tv.join, 1, __ATOMIC_RELAXED);
- void *retval;
- netdata_thread_join(view_thread, &retval);
- netdata_thread_join(master_thread, &retval);
-
-#ifdef DICT_WITH_STATS
- fprintf(stderr,
- "MASTER: inserts %zu"
- ", deletes %zu"
- ", searches %zu"
- ", resets %zu"
- ", entries %d"
- ", referenced_items %d"
- ", pending deletions %d"
- ", check spins %zu"
- ", insert spins %zu"
- ", delete spins %zu"
- ", search ignores %zu"
- "\n",
- stats_master.ops.inserts,
- stats_master.ops.deletes,
- stats_master.ops.searches,
- stats_master.ops.resets,
- tv.master->entries,
- tv.master->referenced_items,
- tv.master->pending_deletion_items,
- stats_master.spin_locks.use_spins,
- stats_master.spin_locks.insert_spins,
- stats_master.spin_locks.delete_spins,
- stats_master.spin_locks.search_spins
- );
- fprintf(stderr,
- "VIEW : inserts %zu"
- ", deletes %zu"
- ", searches %zu"
- ", resets %zu"
- ", entries %d"
- ", referenced_items %d"
- ", pending deletions %d"
- ", check spins %zu"
- ", insert spins %zu"
- ", delete spins %zu"
- ", search ignores %zu"
- "\n",
- stats_view.ops.inserts,
- stats_view.ops.deletes,
- stats_view.ops.searches,
- stats_view.ops.resets,
- tv.view->entries,
- tv.view->referenced_items,
- tv.view->pending_deletion_items,
- stats_view.spin_locks.use_spins,
- stats_view.spin_locks.insert_spins,
- stats_view.spin_locks.delete_spins,
- stats_view.spin_locks.search_spins
- );
-#endif
-
- dictionary_destroy(tv.master);
- dictionary_destroy(tv.view);
-
- return 0;
-}
-
-size_t dictionary_unittest_views(void) {
- size_t errors = 0;
- struct dictionary_stats stats = {};
- DICTIONARY *master = dictionary_create_advanced(DICT_OPTION_NONE, &stats, 0);
- DICTIONARY *view = dictionary_create_view(master);
-
- fprintf(stderr, "\n\nChecking dictionary views...\n");
-
- // Add an item to both master and view, then remove the view first and the master second
- fprintf(stderr, "\nPASS 1: Adding 1 item to master:\n");
- DICTIONARY_ITEM *item1_on_master = dictionary_set_and_acquire_item(master, "KEY 1", "VALUE1", strlen("VALUE1") + 1);
- errors += unittest_check_dictionary("master", master, 1, 1, 0, 1, 0);
- errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true);
-
- fprintf(stderr, "\nPASS 1: Adding master item to view:\n");
- DICTIONARY_ITEM *item1_on_view = dictionary_view_set_and_acquire_item(view, "KEY 1 ON VIEW", item1_on_master);
- errors += unittest_check_dictionary("view", view, 1, 1, 0, 1, 0);
- errors += unittest_check_item("view", view, item1_on_view, "KEY 1 ON VIEW", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true);
-
- fprintf(stderr, "\nPASS 1: Deleting view item:\n");
- dictionary_del(view, "KEY 1 ON VIEW");
- errors += unittest_check_dictionary("master", master, 1, 1, 0, 1, 0);
- errors += unittest_check_dictionary("view", view, 0, 0, 1, 1, 0);
- errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true);
- errors += unittest_check_item("view", view, item1_on_view, "KEY 1 ON VIEW", item1_on_master->shared->value, 1, ITEM_FLAG_DELETED, false, false, true);
-
- fprintf(stderr, "\nPASS 1: Releasing the deleted view item:\n");
- dictionary_acquired_item_release(view, item1_on_view);
- errors += unittest_check_dictionary("master", master, 1, 1, 0, 1, 0);
- errors += unittest_check_dictionary("view", view, 0, 0, 1, 0, 1);
- errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true);
-
- fprintf(stderr, "\nPASS 1: Releasing the acquired master item:\n");
- dictionary_acquired_item_release(master, item1_on_master);
- errors += unittest_check_dictionary("master", master, 1, 1, 0, 0, 0);
- errors += unittest_check_dictionary("view", view, 0, 0, 1, 0, 1);
- errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 0, ITEM_FLAG_NONE, true, true, true);
-
- fprintf(stderr, "\nPASS 1: Deleting the released master item:\n");
- dictionary_del(master, "KEY 1");
- errors += unittest_check_dictionary("master", master, 0, 0, 0, 0, 0);
- errors += unittest_check_dictionary("view", view, 0, 0, 1, 0, 1);
-
- // The other way now:
- // Add an item to both master and view, then remove the master first and verify it is deleted on the view also
- fprintf(stderr, "\nPASS 2: Adding 1 item to master:\n");
- item1_on_master = dictionary_set_and_acquire_item(master, "KEY 1", "VALUE1", strlen("VALUE1") + 1);
- errors += unittest_check_dictionary("master", master, 1, 1, 0, 1, 0);
- errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true);
-
- fprintf(stderr, "\nPASS 2: Adding master item to view:\n");
- item1_on_view = dictionary_view_set_and_acquire_item(view, "KEY 1 ON VIEW", item1_on_master);
- errors += unittest_check_dictionary("view", view, 1, 1, 0, 1, 0);
- errors += unittest_check_item("view", view, item1_on_view, "KEY 1 ON VIEW", item1_on_master->shared->value, 1, ITEM_FLAG_NONE, true, true, true);
-
- fprintf(stderr, "\nPASS 2: Deleting master item:\n");
- dictionary_del(master, "KEY 1");
- garbage_collect_pending_deletes(view);
- errors += unittest_check_dictionary("master", master, 0, 0, 1, 1, 0);
- errors += unittest_check_dictionary("view", view, 0, 0, 1, 1, 0);
- errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_DELETED, false, false, true);
- errors += unittest_check_item("view", view, item1_on_view, "KEY 1 ON VIEW", item1_on_master->shared->value, 1, ITEM_FLAG_DELETED, false, false, true);
-
- fprintf(stderr, "\nPASS 2: Releasing the acquired master item:\n");
- dictionary_acquired_item_release(master, item1_on_master);
- errors += unittest_check_dictionary("master", master, 0, 0, 1, 0, 1);
- errors += unittest_check_dictionary("view", view, 0, 0, 1, 1, 0);
- errors += unittest_check_item("view", view, item1_on_view, "KEY 1 ON VIEW", item1_on_master->shared->value, 1, ITEM_FLAG_DELETED, false, false, true);
-
- fprintf(stderr, "\nPASS 2: Releasing the deleted view item:\n");
- dictionary_acquired_item_release(view, item1_on_view);
- errors += unittest_check_dictionary("master", master, 0, 0, 1, 0, 1);
- errors += unittest_check_dictionary("view", view, 0, 0, 1, 0, 1);
-
- dictionary_destroy(master);
- dictionary_destroy(view);
- return errors;
-}
-
-int dictionary_unittest(size_t entries) {
- if(entries < 10) entries = 10;
-
- DICTIONARY *dict;
- size_t errors = 0;
-
- fprintf(stderr, "Generating %zu names and values...\n", entries);
- char **names = dictionary_unittest_generate_names(entries);
- char **values = dictionary_unittest_generate_values(entries);
-
- fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries);
- dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
- dictionary_unittest_clone(dict, names, values, entries, &errors);
-
- fprintf(stderr, "\nCreating dictionary multi threaded, clone, %zu items\n", entries);
- dict = dictionary_create(DICT_OPTION_NONE);
- dictionary_unittest_clone(dict, names, values, entries, &errors);
-
- fprintf(stderr, "\nCreating dictionary single threaded, non-clone, add-in-front options, %zu items\n", entries);
- dict = dictionary_create(
- DICT_OPTION_SINGLE_THREADED | DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE |
- DICT_OPTION_ADD_IN_FRONT);
- dictionary_unittest_nonclone(dict, names, values, entries, &errors);
-
- fprintf(stderr, "\nCreating dictionary multi threaded, non-clone, add-in-front options, %zu items\n", entries);
- dict = dictionary_create(
- DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_ADD_IN_FRONT);
- dictionary_unittest_nonclone(dict, names, values, entries, &errors);
-
- fprintf(stderr, "\nCreating dictionary single-threaded, non-clone, don't overwrite options, %zu items\n", entries);
- dict = dictionary_create(
- DICT_OPTION_SINGLE_THREADED | DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE |
- DICT_OPTION_DONT_OVERWRITE_VALUE);
- dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
- dictionary_unittest_run_and_measure_time(dict, "resetting non-overwrite entries", names, values, entries, &errors, dictionary_unittest_reset_dont_overwrite_nonclone);
- dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, &errors, dictionary_unittest_foreach);
- dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback", names, values, entries, &errors, dictionary_unittest_walkthrough);
- dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback stop", names, values, entries, &errors, dictionary_unittest_walkthrough_stop);
- dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
-
- fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries);
- dict = dictionary_create(
- DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE);
- dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
- dictionary_unittest_run_and_measure_time(dict, "walkthrough write delete this", names, values, entries, &errors, dictionary_unittest_walkthrough_delete_this);
- dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
-
- fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries);
- dict = dictionary_create(
- DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_VALUE_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE);
- dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
- dictionary_unittest_run_and_measure_time(dict, "foreach write delete this", names, values, entries, &errors, dictionary_unittest_foreach_delete_this);
- dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop empty", names, values, 0, &errors, dictionary_unittest_foreach);
- dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback empty", names, values, 0, &errors, dictionary_unittest_walkthrough);
- dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
-
- fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries);
- dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
- dictionary_unittest_sorting(dict, names, values, entries, &errors);
- dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
-
- fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries);
- dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
- dictionary_unittest_null_dfe(dict, names, values, entries, &errors);
- dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
-
- fprintf(stderr, "\nCreating dictionary single threaded, noclone, %zu items\n", entries);
- dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_VALUE_LINK_DONT_CLONE);
- dictionary_unittest_null_dfe(dict, names, values, entries, &errors);
- dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
-
- // check reference counters
- {
- fprintf(stderr, "\nTesting reference counters:\n");
- dict = dictionary_create(DICT_OPTION_NONE | DICT_OPTION_NAME_LINK_DONT_CLONE);
- errors += unittest_check_dictionary("", dict, 0, 0, 0, 0, 0);
-
- fprintf(stderr, "\nAdding test item to dictionary and acquiring it\n");
- dictionary_set(dict, "test", "ITEM1", 6);
- DICTIONARY_ITEM *item = (DICTIONARY_ITEM *)dictionary_get_and_acquire_item(dict, "test");
-
- errors += unittest_check_dictionary("", dict, 1, 1, 0, 1, 0);
- errors += unittest_check_item("ACQUIRED", dict, item, "test", "ITEM1", 1, ITEM_FLAG_NONE, true, true, true);
-
- fprintf(stderr, "\nChecking that reference counters are increased:\n");
- void *t;
- dfe_start_read(dict, t) {
- errors += unittest_check_dictionary("", dict, 1, 1, 0, 1, 0);
- errors += unittest_check_item("ACQUIRED TRAVERSAL", dict, item, "test", "ITEM1", 2, ITEM_FLAG_NONE, true, true, true);
- }
- dfe_done(t);
-
- fprintf(stderr, "\nChecking that reference counters are decreased:\n");
- errors += unittest_check_dictionary("", dict, 1, 1, 0, 1, 0);
- errors += unittest_check_item("ACQUIRED TRAVERSAL 2", dict, item, "test", "ITEM1", 1, ITEM_FLAG_NONE, true, true, true);
-
- fprintf(stderr, "\nDeleting the item we have acquired:\n");
- dictionary_del(dict, "test");
-
- errors += unittest_check_dictionary("", dict, 0, 0, 1, 1, 0);
- errors += unittest_check_item("DELETED", dict, item, "test", "ITEM1", 1, ITEM_FLAG_DELETED, false, false, true);
-
- fprintf(stderr, "\nAdding another item with the same name of the item we deleted, while being acquired:\n");
- dictionary_set(dict, "test", "ITEM2", 6);
- errors += unittest_check_dictionary("", dict, 1, 1, 1, 1, 0);
-
- fprintf(stderr, "\nAcquiring the second item:\n");
- DICTIONARY_ITEM *item2 = (DICTIONARY_ITEM *)dictionary_get_and_acquire_item(dict, "test");
- errors += unittest_check_item("FIRST", dict, item, "test", "ITEM1", 1, ITEM_FLAG_DELETED, false, false, true);
- errors += unittest_check_item("SECOND", dict, item2, "test", "ITEM2", 1, ITEM_FLAG_NONE, true, true, true);
- errors += unittest_check_dictionary("", dict, 1, 1, 1, 2, 0);
-
- fprintf(stderr, "\nReleasing the second item (the first is still acquired):\n");
- dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)item2);
- errors += unittest_check_dictionary("", dict, 1, 1, 1, 1, 0);
- errors += unittest_check_item("FIRST", dict, item, "test", "ITEM1", 1, ITEM_FLAG_DELETED, false, false, true);
- errors += unittest_check_item("SECOND RELEASED", dict, item2, "test", "ITEM2", 0, ITEM_FLAG_NONE, true, true, true);
-
- fprintf(stderr, "\nDeleting the second item (the first is still acquired):\n");
- dictionary_del(dict, "test");
- errors += unittest_check_dictionary("", dict, 0, 0, 1, 1, 0);
- errors += unittest_check_item("ACQUIRED DELETED", dict, item, "test", "ITEM1", 1, ITEM_FLAG_DELETED, false, false, true);
-
- fprintf(stderr, "\nReleasing the first item (which we have already deleted):\n");
- dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)item);
- dfe_start_write(dict, item) ; dfe_done(item);
- errors += unittest_check_dictionary("", dict, 0, 0, 1, 0, 1);
-
- fprintf(stderr, "\nAdding again the test item to dictionary and acquiring it\n");
- dictionary_set(dict, "test", "ITEM1", 6);
- item = (DICTIONARY_ITEM *)dictionary_get_and_acquire_item(dict, "test");
-
- errors += unittest_check_dictionary("", dict, 1, 1, 0, 1, 0);
- errors += unittest_check_item("RE-ADDITION", dict, item, "test", "ITEM1", 1, ITEM_FLAG_NONE, true, true, true);
-
- fprintf(stderr, "\nDestroying the dictionary while we have acquired an item\n");
- dictionary_destroy(dict);
-
- fprintf(stderr, "Releasing the item (on a destroyed dictionary)\n");
- dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)item);
- item = NULL;
- dict = NULL;
- }
-
- dictionary_unittest_free_char_pp(names, entries);
- dictionary_unittest_free_char_pp(values, entries);
-
- errors += dictionary_unittest_views();
- errors += dictionary_unittest_threads();
- errors += dictionary_unittest_view_threads();
-
- fprintf(stderr, "\n%zu errors found\n", errors);
- return errors ? 1 : 0;
-}
diff --git a/libnetdata/dictionary/dictionary.h b/libnetdata/dictionary/dictionary.h
deleted file mode 100644
index 391be4ee5..000000000
--- a/libnetdata/dictionary/dictionary.h
+++ /dev/null
@@ -1,335 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_DICTIONARY_H
-#define NETDATA_DICTIONARY_H 1
-
-#include "../libnetdata.h"
-
-
-/*
- * Netdata DICTIONARY features:
- *
- * CLONE or LINK
- * Names and Values in the dictionary can be cloned or linked.
- * In clone mode, the dictionary does all the memory management.
- * The default is clone for both names and values.
- * Set DICT_OPTION_NAME_LINK_DONT_CLONE to link names.
- * Set DICT_OPTION_VALUE_LINK_DONT_CLONE to link names.
- *
- * ORDERED
- * Items are ordered in the order they are added (new items are appended at the end).
- * You may reverse the order by setting the flag DICT_OPTION_ADD_IN_FRONT.
- *
- * LOOKUP
- * The dictionary uses JudyHS to maintain a very fast randomly accessible hash table.
- *
- * MULTI-THREADED and SINGLE-THREADED
- * Each dictionary may be single threaded (no locks), or multi-threaded (multiple readers or one writer).
- * The default is multi-threaded. Add the flag DICT_OPTION_SINGLE_THREADED for single-threaded.
- *
- * WALK-THROUGH and FOREACH traversal
- * The dictionary can be traversed on read or write mode, either with a callback (walkthrough) or with
- * a loop (foreach).
- *
- * In write mode traversal, the caller may delete only the current item, but may add as many items as needed.
- *
- */
-
-#ifdef NETDATA_INTERNAL_CHECKS
-#define DICT_WITH_STATS 1
-#endif
-
-#ifdef DICTIONARY_INTERNALS
-#define DICTFE_CONST
-#define DICT_ITEM_CONST
-#else
-#define DICTFE_CONST const
-#define DICT_ITEM_CONST const
-#endif
-
-typedef struct dictionary DICTIONARY;
-typedef struct dictionary_item DICTIONARY_ITEM;
-
-typedef enum __attribute__((packed)) dictionary_options {
- DICT_OPTION_NONE = 0, // the default is the opposite of all below
- DICT_OPTION_SINGLE_THREADED = (1 << 0), // don't use any locks (default: use locks)
- DICT_OPTION_VALUE_LINK_DONT_CLONE = (1 << 1), // don't copy the value, just point to the one provided (default: copy)
- DICT_OPTION_NAME_LINK_DONT_CLONE = (1 << 2), // don't copy the name, just point to the one provided (default: copy)
- DICT_OPTION_DONT_OVERWRITE_VALUE = (1 << 3), // don't overwrite values of dictionary items (default: overwrite)
- DICT_OPTION_ADD_IN_FRONT = (1 << 4), // add dictionary items at the front of the linked list (default: at the end)
- DICT_OPTION_FIXED_SIZE = (1 << 5), // the items of the dictionary have a fixed size
-} DICT_OPTIONS;
-
-struct dictionary_stats {
- const char *name; // the name of the category
-
- struct {
- size_t active; // the number of active dictionaries
- size_t deleted; // the number of dictionaries queued for destruction
- } dictionaries;
-
- struct {
- long entries; // active items in the dictionary
- long pending_deletion; // pending deletion items in the dictionary
- long referenced; // referenced items in the dictionary
- } items;
-
- struct {
- size_t creations; // dictionary creations
- size_t destructions; // dictionary destructions
- size_t flushes; // dictionary flushes
- size_t traversals; // dictionary foreach
- size_t walkthroughs; // dictionary walkthrough
- size_t garbage_collections; // dictionary garbage collections
- size_t searches; // item searches
- size_t inserts; // item inserts
- size_t resets; // item resets
- size_t deletes; // item deletes
- } ops;
-
- struct {
- size_t inserts; // number of times the insert callback is called
- size_t conflicts; // number of times the conflict callback is called
- size_t reacts; // number of times the react callback is called
- size_t deletes; // number of times the delete callback is called
- } callbacks;
-
- // memory
- struct {
- long index; // bytes of keys indexed (indication of the index size)
- long values; // bytes of caller structures
- long dict; // bytes of the structures dictionary needs
- } memory;
-
- // spin locks
- struct {
- size_t use_spins; // number of times a reference to item had to spin to acquire it or ignore it
- size_t search_spins; // number of times a successful search result had to be thrown away
- size_t insert_spins; // number of times an insertion to the hash table had to be repeated
- size_t delete_spins; // number of times a deletion had to spin to get a decision
- } spin_locks;
-};
-
-// Create a dictionary
-#ifdef NETDATA_INTERNAL_CHECKS
-#define dictionary_create(options) dictionary_create_advanced_with_trace(options, NULL, 0, __FUNCTION__, __LINE__, __FILE__)
-#define dictionary_create_advanced(options, stats, fixed_size) dictionary_create_advanced_with_trace(options, stats, fixed_size, __FUNCTION__, __LINE__, __FILE__)
-DICTIONARY *dictionary_create_advanced_with_trace(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size, const char *function, size_t line, const char *file);
-#else
-#define dictionary_create(options) dictionary_create_advanced(options, NULL, 0)
-DICTIONARY *dictionary_create_advanced(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size);
-#endif
-
-// Create a view on a dictionary
-#ifdef NETDATA_INTERNAL_CHECKS
-#define dictionary_create_view(master) dictionary_create_view_with_trace(master, __FUNCTION__, __LINE__, __FILE__)
-DICTIONARY *dictionary_create_view_with_trace(DICTIONARY *master, const char *function, size_t line, const char *file);
-#else
-DICTIONARY *dictionary_create_view(DICTIONARY *master);
-#endif
-
-// an insert callback to be called just after an item is added to the dictionary
-// this callback is called while the dictionary is write locked!
-void dictionary_register_insert_callback(DICTIONARY *dict, void (*ins_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data);
-
-// a delete callback to be called just before an item is deleted forever
-// this callback is called while the dictionary is write locked!
-void dictionary_register_delete_callback(DICTIONARY *dict, void (*del_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data);
-
-// a merge callback to be called when DICT_OPTION_DONT_OVERWRITE_VALUE
-// and an item is already found in the dictionary - the dictionary does nothing else in this case
-// the old_value will remain in the dictionary - the new_value is ignored
-// The callback should return true if the value has been updated (it increases the dictionary version).
-void dictionary_register_conflict_callback(DICTIONARY *dict, bool (*conflict_callback)(const DICTIONARY_ITEM *item, void *old_value, void *new_value, void *data), void *data);
-
-// a reaction callback to be called after every item insertion or conflict
-// after the constructors have finished and the items are fully available for use
-// and the dictionary is not write locked anymore
-void dictionary_register_react_callback(DICTIONARY *dict, void (*react_callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data);
-
-// Destroy a dictionary
-// Returns the number of bytes freed
-// The returned value will not include name/key sizes
-// Registered delete callbacks will be run for each item in the dictionary.
-size_t dictionary_destroy(DICTIONARY *dict);
-
-// Empties a dictionary
-// Referenced items will survive, but are not offered anymore.
-// Registered delete callbacks will be run for each item in the dictionary.
-void dictionary_flush(DICTIONARY *dict);
-
-void dictionary_version_increment(DICTIONARY *dict);
-
-void dictionary_garbage_collect(DICTIONARY *dict);
-
-void cleanup_destroyed_dictionaries(void);
-
-// ----------------------------------------------------------------------------
-// Set an item in the dictionary
-//
-// - if an item with the same name does not exist, create one
-// - if an item with the same name exists, then:
-// a) if DICT_OPTION_DONT_OVERWRITE_VALUE is set, just return the existing value (ignore the new value)
-// else b) reset the value to the new value passed at the call
-//
-// When DICT_OPTION_VALUE_LINK_DONT_CLONE is set, the value is linked, otherwise it is copied
-// When DICT_OPTION_NAME_LINK_DONT_CLONE is set, the name is linked, otherwise it is copied
-//
-// When neither DICT_OPTION_VALUE_LINK_DONT_CLONE nor DICT_OPTION_NAME_LINK_DONT_CLONE are set, all the
-// memory management for names and values is done by the dictionary.
-//
-// Passing NULL as value, the dictionary will callocz() the newly allocated value, otherwise it will copy it.
-// Passing 0 as value_len, the dictionary will set the value to NULL (no allocations for value will be made).
-#define dictionary_set(dict, name, value, value_len) dictionary_set_advanced(dict, name, -1, value, value_len, NULL)
-void *dictionary_set_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data);
-
-#define dictionary_set_and_acquire_item(dict, name, value, value_len) dictionary_set_and_acquire_item_advanced(dict, name, -1, value, value_len, NULL)
-DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_set_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data);
-
-// set an item in a dictionary view
-#define dictionary_view_set_and_acquire_item(dict, name, master_item) dictionary_view_set_and_acquire_item_advanced(dict, name, -1, master_item)
-DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_view_set_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, DICTIONARY_ITEM *master_item);
-#define dictionary_view_set(dict, name, master_item) dictionary_view_set_advanced(dict, name, -1, master_item)
-void *dictionary_view_set_advanced(DICTIONARY *dict, const char *name, ssize_t name_len, DICT_ITEM_CONST DICTIONARY_ITEM *master_item);
-
-// ----------------------------------------------------------------------------
-// Get an item from the dictionary
-// If it returns NULL, the item is not found
-
-#define dictionary_get(dict, name) dictionary_get_advanced(dict, name, -1)
-void *dictionary_get_advanced(DICTIONARY *dict, const char *name, ssize_t name_len);
-
-#define dictionary_get_and_acquire_item(dict, name) dictionary_get_and_acquire_item_advanced(dict, name, -1)
-DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_get_and_acquire_item_advanced(DICTIONARY *dict, const char *name, ssize_t name_len);
-
-
-// ----------------------------------------------------------------------------
-// Delete an item from the dictionary
-// returns true if the item was found and has been deleted
-// returns false if the item was not found in the index
-
-#define dictionary_del(dict, name) dictionary_del_advanced(dict, name, -1)
-bool dictionary_del_advanced(DICTIONARY *dict, const char *name, ssize_t name_len);
-
-// ----------------------------------------------------------------------------
-// reference counters management
-
-void dictionary_acquired_item_release(DICTIONARY *dict, DICT_ITEM_CONST DICTIONARY_ITEM *item);
-
-DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_acquired_item_dup(DICTIONARY *dict, DICT_ITEM_CONST DICTIONARY_ITEM *item);
-
-const char *dictionary_acquired_item_name(DICT_ITEM_CONST DICTIONARY_ITEM *item);
-void *dictionary_acquired_item_value(DICT_ITEM_CONST DICTIONARY_ITEM *item);
-
-size_t dictionary_acquired_item_references(DICT_ITEM_CONST DICTIONARY_ITEM *item);
-
-// ----------------------------------------------------------------------------
-// Traverse (walk through) the items of the dictionary.
-// The order of traversal is currently the order of insertion.
-//
-// The callback function may return a negative number to stop the traversal,
-// in which case that negative value is returned to the caller.
-//
-// If all callback calls return zero or positive numbers, the sum of all of
-// them is returned to the caller.
-//
-// You cannot alter the dictionary from inside a dictionary_walkthrough_read() - deadlock!
-// You can only delete the current item from inside a dictionary_walkthrough_write() - you can add as many as you want.
-//
-#define dictionary_walkthrough_read(dict, callback, data) dictionary_walkthrough_rw(dict, 'r', callback, data)
-#define dictionary_walkthrough_write(dict, callback, data) dictionary_walkthrough_rw(dict, 'w', callback, data)
-int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data);
-
-typedef int (*dictionary_sorted_compar)(const DICTIONARY_ITEM **item1, const DICTIONARY_ITEM **item2);
-
-#define dictionary_sorted_walkthrough_read(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'r', callback, data, NULL)
-#define dictionary_sorted_walkthrough_write(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'w', callback, data, NULL)
-int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *entry, void *data), void *data, dictionary_sorted_compar compar);
-
-// ----------------------------------------------------------------------------
-// Traverse with foreach
-//
-// Use like this:
-//
-// DICTFE dfe = {};
-// for(MY_ITEM *item = dfe_start_read(&dfe, dict); item ; item = dfe_next(&dfe)) {
-// // do things with the item and its dfe.name
-// }
-// dfe_done(&dfe);
-//
-// You cannot alter the dictionary from within a dfe_read_start() - deadlock!
-// You can only delete the current item from inside a dfe_start_write() - you can add as many as you want.
-//
-
-#define DICTIONARY_LOCK_READ 'r'
-#define DICTIONARY_LOCK_WRITE 'w'
-#define DICTIONARY_LOCK_REENTRANT 'z'
-
-void dictionary_write_lock(DICTIONARY *dict);
-void dictionary_write_unlock(DICTIONARY *dict);
-
-typedef DICTFE_CONST struct dictionary_foreach {
- DICTIONARY *dict; // the dictionary upon we work
-
- DICTIONARY_ITEM *item; // the item we work on, to remember the position we are at
- // this can be used with dictionary_acquired_item_dup() to
- // acquire the currently working item.
-
- const char *name; // the dictionary name of the last item used
- void *value; // the dictionary value of the last item used
- // same as the return value of dictfe_start() and dictfe_next()
-
- size_t counter; // counts the number of iterations made, starting from zero
-
- char rw; // the lock mode 'r' or 'w'
- bool locked; // true when the dictionary is locked
-} DICTFE;
-
-#define dfe_start_read(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_READ)
-#define dfe_start_write(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_WRITE)
-#define dfe_start_reentrant(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_REENTRANT)
-
-#define dfe_start_rw(dict, value, mode) \
- do { \
- DICTFE value ## _dfe = {}; \
- (void)(value); /* needed to avoid warning when looping without using this */ \
- for((value) = dictionary_foreach_start_rw(&value ## _dfe, (dict), (mode)); \
- (value ## _dfe.item) ; \
- (value) = dictionary_foreach_next(&value ## _dfe)) \
- {
-
-#define dfe_done(value) \
- } \
- dictionary_foreach_done(&value ## _dfe); \
- } while(0)
-
-#define dfe_unlock(value) dictionary_foreach_unlock(&value ## _dfe);
-
-void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw);
-void *dictionary_foreach_next(DICTFE *dfe);
-void dictionary_foreach_done(DICTFE *dfe);
-void dictionary_foreach_unlock(DICTFE *dfe);
-
-// ----------------------------------------------------------------------------
-// Get statistics about the dictionary
-
-size_t dictionary_version(DICTIONARY *dict);
-size_t dictionary_entries(DICTIONARY *dict);
-size_t dictionary_referenced_items(DICTIONARY *dict);
-
-// for all cases that the caller does not provide a stats structure, this is where they are accumulated.
-extern struct dictionary_stats dictionary_stats_category_other;
-
-int dictionary_unittest(size_t entries);
-
-// ----------------------------------------------------------------------------
-// THREAD CACHE
-
-void *thread_cache_entry_get_or_set(void *key,
- ssize_t key_length,
- void *value,
- void *(*transform_the_value_before_insert)(void *key, size_t key_length, void *value));
-
-void thread_cache_destroy(void);
-
-#endif /* NETDATA_DICTIONARY_H */
diff --git a/libnetdata/dyn_conf/README.md b/libnetdata/dyn_conf/README.md
deleted file mode 100644
index 17d059b02..000000000
--- a/libnetdata/dyn_conf/README.md
+++ /dev/null
@@ -1,188 +0,0 @@
-# Netdata Dynamic Configuration
-
-Purpose of Netdata Dynamic Configuration is to allow configuration of select Netdata plugins and options through the
-Netdata API and by extension by UI.
-
-## HTTP API documentation
-
-### Summary API
-
-For summary of all jobs and their statuses (for all children that stream to parent) use the following URL:
-
-| Method | Endpoint | Description |
-|:-------:|-------------------------------|------------------------------------------------------------|
-| **GET** | `api/v2/job_statuses` | list of Jobs |
-| **GET** | `api/v2/job_statuses?grouped` | list of Jobs (hierarchical, grouped by host/plugin/module) |
-
-### Dyncfg API
-
-### Top level
-
-| Method | Endpoint | Description |
-|:-------:|------------------|-----------------------------------------|
-| **GET** | `/api/v2/config` | registered Plugins (sent DYNCFG_ENABLE) |
-
-### Plugin level
-
-| Method | Endpoint | Description |
-|:-------:|-----------------------------------|------------------------------|
-| **GET** | `/api/v2/config/[plugin]` | Plugin config |
-| **PUT** | `/api/v2/config/[plugin]` | update Plugin config |
-| **GET** | `/api/v2/config/[plugin]/modules` | Modules registered by Plugin |
-| **GET** | `/api/v2/config/[plugin]/schema` | Plugin config schema |
-
-### Module level
-
-| Method | Endpoint | Description |
-|:-------:|-----------------------------------------------|---------------------------|
-| **GET** | `/api/v2/config/<plugin>/[module]` | Module config |
-| **PUT** | `/api/v2/config/[plugin]/[module]` | update Module config |
-| **GET** | `/api/v2/config/[plugin]/[module]/jobs` | Jobs registered by Module |
-| **GET** | `/api/v2/config/[plugin]/[module]/job_schema` | Job config schema |
-| **GET** | `/api/v2/config/[plugin]/[module]/schema` | Module config schema |
-
-### Job level - only for modules where `module_type == job_array`
-
-| Method | Endpoint | Description |
-|:----------:|------------------------------------------|--------------------------------|
-| **GET** | `/api/v2/config/[plugin]/[module]/[job]` | Job config |
-| **PUT** | `/api/v2/config/[plugin]/[module]/[job]` | update Job config |
-| **POST** | `/api/v2/config/[plugin]/[module]/[job]` | create Job |
-| **DELETE** | `/api/v2/config/[plugin]/[module]/[job]` | delete Job (created by Dyncfg) |
-
-## Internal Plugins API
-
-TBD
-
-## External Plugins API
-
-### Commands plugins can use
-
-#### DYNCFG_ENABLE
-
-Plugin signifies to agent its ability to use new dynamic config and the name it wishes to use by sending
-
-```
-DYNCFG_ENABLE [{PLUGIN_NAME}]
-```
-
-This can be sent only once per lifetime of the plugin (at startup or later) sending it multiple times is considered a
-protocol violation and plugin might get terminated.
-
-After this command is sent the plugin has to be ready to accept all the new commands/keywords related to dynamic
-configuration (this command lets agent know this plugin is dyncfg capable and wishes to use dyncfg functionality).
-
-#### DYNCFG_RESET
-
-Sending this, will reset the internal state of the agent, considering this a `DYNCFG_ENABLE`.
-
-```
-DYNCFG_RESET
-```
-
-
-#### DYNCFG_REGISTER_MODULE
-
-```
-DYNCFG_REGISTER_MODULE {MODULE_NAME} {MODULE_TYPE}
-```
-
-Module has to choose one of following types at registration:
-
-- `single` - module itself has configuration but does not accept any jobs *(this is useful mainly for internal netdata
- configurable things like webserver etc.)*
-
-- `job_array` - module itself **can** *(not must)* have configuration and it has an array of jobs which can be added,
- modified and deleted. **this is what plugin developer needs in most cases**
-
-After a module has been registered agent can call `set_module_config`, `get_module_config` and `get_module_config_schema`.
-
-When `MODULE_TYPE` is `job_array` the agent may also send `set_job_config`, `get_job_config` and `get_job_config_schema`.
-
-#### DYNCFG_REGISTER_JOB
-
-The plugin can use `DYNCFG_REGISTER_JOB` to register its own configuration jobs. It should not register jobs configured
-via DYNCFG (doing so, the agent will shutdown the plugin).
-
-
-```
-DYNCFG_REGISTER_JOB {MODULE_NAME} {JOB_NAME} {JOB_TYPE} {FLAGS}
-```
-
-Where:
-
-- `MODULE_NAME` is the name of the module.
-- `JOB_NAME` is the name of the job.
-- `JOB_TYPE` is either `stock` or `autodiscovered`.
-- `FLAGS`, just send zero.
-
-#### REPORT_JOB_STATUS
-
-```
-REPORT_JOB_STATUS {MODULE_NAME} {JOB_NAME} {STATUS} {STATE} ["REASON"]
-```
-
-Note the REASON parameter is optional and can be entirelly ommited (for example when state is OK there is no need to send any reason).
-
-Where:
-
-- `MODULE_NAME` is the name of the module.
-- `JOB_NAME` is the name of the job.
-- `STATUS` is one of `stopped`, `running`, or `error`.
-- `STATE`, just send zero.
-- `REASON` is a message describing the status. In case you don't want to send any reason string it is preferable to omit this parameter altogether (as opposed to sending empty string `""`).
-
-
-### Commands plugins must serve
-
-Once a plugin calls `DYNCFG_ENABLE`, the must be able to handle these calls.
-
-function|parameters|prerequisites|request payload|response payload|
-:---:|:---:|:---:|:---:|:---:|
-`set_plugin_config`|none|`DYNCFG_ENABLE`|plugin configuration|none|
-`get_plugin_config`|none|`DYNCFG_ENABLE`|none|plugin configuration|
-`get_plugin_config_schema`|none|`DYNCFG_ENABLE`|none|plugin configuration schema|
-`set_module_config`|`module_name`|`DYNCFG_REGISTER_MODULE`|module configuration|none|
-`get_module_config`|`module_name`|`DYNCFG_REGISTER_MODULE`|none|module configuration|
-`get_module_config_schema`|`module_name`|`DYNCFG_REGISTER_MODULE`|none|module configuration schema|
-`set_job_config`|`module_name`, `job_name`|`DYNCFG_REGISTER_MODULE`|job configuration|none|
-`get_job_config`|`module_name`, `job_name`|`DYNCFG_REGISTER_MODULE`|none|job configuration|
-`get_job_config_schema`|`module_name`, `job_name`|`DYNCFG_REGISTER_MODULE`|none|job configuration schema|
-
-All of them work like this:
-
-If the request payload is `none`, then the request looks like this:
-
-```bash
-FUNCTION {TRANSACTION_UUID} {TIMEOUT_SECONDS} "{function} {parameters}"
-```
-
-When there is payload, the request looks like this:
-
-```bash
-FUNCTION_PAYLOAD {TRANSACTION_UUID} {TIMEOUT_SECONDS} "{function} {parameters}"
-<payload>
-FUNCTION_PAYLOAD_END
-```
-
-In all cases, the response is like this:
-
-```bash
-FUNCTION_RESULT_BEGIN {TRANSACTION_UUID} {HTTP_RESPONSE_CODE} "{CONTENT_TYPE}" {EXPIRATION_TIMESTAMP}
-<payload>
-FUNCTION_RESULT_END
-```
-Where:
-- `TRANSACTION_UUID` is the same UUID received with the request.
-- `HTTP_RESPONSE_CODE` is either `0` (rejected) or `1` (accepted).
-- `CONTENT_TYPE` should reflect the `payload` returned.
-- `EXPIRATION_TIMESTAMP` can be zero.
-
-
-## DYNCFG with streaming
-
-When above commands are transferred trough streaming additionally `plugin_name` is prefixed as first parameter. This is
-done to allow routing to appropriate plugin @child.
-
-As a plugin developer you don't need to concern yourself with this detail as that parameter is stripped when sent to the
-plugin *(and added when sent trough streaming)* automagically.
diff --git a/libnetdata/dyn_conf/dyn_conf.c b/libnetdata/dyn_conf/dyn_conf.c
deleted file mode 100644
index ee4b4733a..000000000
--- a/libnetdata/dyn_conf/dyn_conf.c
+++ /dev/null
@@ -1,1140 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "dyn_conf.h"
-
-#define DYN_CONF_PATH_MAX (4096)
-#define DYN_CONF_DIR VARLIB_DIR "/dynconf"
-
-#define DYN_CONF_JOB_SCHEMA "job_schema"
-#define DYN_CONF_SCHEMA "schema"
-#define DYN_CONF_MODULE_LIST "modules"
-#define DYN_CONF_JOB_LIST "jobs"
-#define DYN_CONF_CFG_EXT ".cfg"
-
-void job_flags_wallkthrough(dyncfg_job_flg_t flags, void (*cb)(const char *str, void *data), void *data)
-{
- if (flags & JOB_FLG_PS_LOADED)
- cb("JOB_FLG_PS_LOADED", data);
- if (flags & JOB_FLG_PLUGIN_PUSHED)
- cb("JOB_FLG_PLUGIN_PUSHED", data);
- if (flags & JOB_FLG_STREAMING_PUSHED)
- cb("JOB_FLG_STREAMING_PUSHED", data);
- if (flags & JOB_FLG_USER_CREATED)
- cb("JOB_FLG_USER_CREATED", data);
-}
-
-struct deferred_cfg_send {
- DICTIONARY *plugins_dict;
- char *plugin_name;
- char *module_name;
- char *job_name;
- struct deferred_cfg_send *next;
-};
-
-bool dyncfg_shutdown = false;
-struct deferred_cfg_send *deferred_configs = NULL;
-pthread_mutex_t deferred_configs_lock = PTHREAD_MUTEX_INITIALIZER;
-pthread_cond_t deferred_configs_cond = PTHREAD_COND_INITIALIZER;
-
-static void deferred_config_free(struct deferred_cfg_send *dcs)
-{
- freez(dcs->plugin_name);
- freez(dcs->module_name);
- freez(dcs->job_name);
- freez(dcs);
-}
-
-static void deferred_config_push_back(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name)
-{
- struct deferred_cfg_send *deferred = callocz(1, sizeof(struct deferred_cfg_send));
- deferred->plugin_name = strdupz(plugin_name);
- if (module_name != NULL) {
- deferred->module_name = strdupz(module_name);
- if (job_name != NULL)
- deferred->job_name = strdupz(job_name);
- }
- deferred->plugins_dict = plugins_dict;
- pthread_mutex_lock(&deferred_configs_lock);
- if (dyncfg_shutdown) {
- pthread_mutex_unlock(&deferred_configs_lock);
- deferred_config_free(deferred);
- return;
- }
- struct deferred_cfg_send *last = deferred_configs;
- if (last == NULL)
- deferred_configs = deferred;
- else {
- while (last->next != NULL)
- last = last->next;
- last->next = deferred;
- }
- pthread_cond_signal(&deferred_configs_cond);
- pthread_mutex_unlock(&deferred_configs_lock);
-}
-
-static void deferred_configs_unlock()
-{
- dyncfg_shutdown = true;
- // if we get cancelled in pthread_cond_wait
- // we will arrive at cancelled cleanup handler
- // with mutex locked we need to unlock it
- pthread_mutex_unlock(&deferred_configs_lock);
-}
-
-static struct deferred_cfg_send *deferred_config_pop(void *ptr)
-{
- pthread_mutex_lock(&deferred_configs_lock);
- while (deferred_configs == NULL) {
- netdata_thread_cleanup_push(deferred_configs_unlock, ptr);
- pthread_cond_wait(&deferred_configs_cond, &deferred_configs_lock);
- netdata_thread_cleanup_pop(0);
- }
- struct deferred_cfg_send *deferred = deferred_configs;
- deferred_configs = deferred_configs->next;
- pthread_mutex_unlock(&deferred_configs_lock);
- return deferred;
-}
-
-static int _get_list_of_plugins_json_cb(const DICTIONARY_ITEM *item, void *entry, void *data)
-{
- UNUSED(item);
- json_object *obj = (json_object *)data;
- struct configurable_plugin *plugin = (struct configurable_plugin *)entry;
-
- json_object *plugin_name = json_object_new_string(plugin->name);
- json_object_array_add(obj, plugin_name);
-
- return 0;
-}
-
-json_object *get_list_of_plugins_json(DICTIONARY *plugins_dict)
-{
- json_object *obj = json_object_new_array();
-
- dictionary_walkthrough_read(plugins_dict, _get_list_of_plugins_json_cb, obj);
-
- return obj;
-}
-
-static int _get_list_of_modules_json_cb(const DICTIONARY_ITEM *item, void *entry, void *data)
-{
- UNUSED(item);
- json_object *obj = (json_object *)data;
- struct module *module = (struct module *)entry;
-
- json_object *json_module = json_object_new_object();
-
- json_object *json_item = json_object_new_string(module->name);
- json_object_object_add(json_module, "name", json_item);
- const char *module_type = module_type2str(module->type);
- json_item = json_object_new_string(module_type);
- json_object_object_add(json_module, "type", json_item);
-
- json_object_array_add(obj, json_module);
-
- return 0;
-}
-
-json_object *get_list_of_modules_json(struct configurable_plugin *plugin)
-{
- json_object *obj = json_object_new_array();
-
- pthread_mutex_lock(&plugin->lock);
-
- dictionary_walkthrough_read(plugin->modules, _get_list_of_modules_json_cb, obj);
-
- pthread_mutex_unlock(&plugin->lock);
-
- return obj;
-}
-
-const char *job_status2str(enum job_status status)
-{
- switch (status) {
- case JOB_STATUS_UNKNOWN:
- return "unknown";
- case JOB_STATUS_STOPPED:
- return "stopped";
- case JOB_STATUS_RUNNING:
- return "running";
- case JOB_STATUS_ERROR:
- return "error";
- default:
- return "unknown";
- }
-}
-
-static void _job_flags2str_cb(const char *str, void *data)
-{
- json_object *json_item = json_object_new_string(str);
- json_object_array_add((json_object *)data, json_item);
-}
-
-json_object *job2json(struct job *job) {
- json_object *json_job = json_object_new_object();
-
- json_object *json_item = json_object_new_string(job->name);
- json_object_object_add(json_job, "name", json_item);
-
- json_item = json_object_new_string(job_type2str(job->type));
- json_object_object_add(json_job, "type", json_item);
-
- netdata_mutex_lock(&job->lock);
- json_item = json_object_new_string(job_status2str(job->status));
- json_object_object_add(json_job, "status", json_item);
-
- json_item = json_object_new_int(job->state);
- json_object_object_add(json_job, "state", json_item);
-
- json_item = job->reason == NULL ? NULL : json_object_new_string(job->reason);
- json_object_object_add(json_job, "reason", json_item);
-
- int64_t last_state_update_s = job->last_state_update / USEC_PER_SEC;
- int64_t last_state_update_us = job->last_state_update % USEC_PER_SEC;
-
- json_item = json_object_new_int64(last_state_update_s);
- json_object_object_add(json_job, "last_state_update_s", json_item);
-
- json_item = json_object_new_int64(last_state_update_us);
- json_object_object_add(json_job, "last_state_update_us", json_item);
-
- json_item = json_object_new_array();
- job_flags_wallkthrough(job->flags, _job_flags2str_cb, json_item);
- json_object_object_add(json_job, "flags", json_item);
-
- netdata_mutex_unlock(&job->lock);
-
- return json_job;
-}
-
-static int _get_list_of_jobs_json_cb(const DICTIONARY_ITEM *item, void *entry, void *data)
-{
- UNUSED(item);
- json_object *obj = (json_object *)data;
-
- json_object *json_job = job2json((struct job *)entry);
-
- json_object_array_add(obj, json_job);
-
- return 0;
-}
-
-json_object *get_list_of_jobs_json(struct module *module)
-{
- json_object *obj = json_object_new_array();
-
- pthread_mutex_lock(&module->lock);
-
- dictionary_walkthrough_read(module->jobs, _get_list_of_jobs_json_cb, obj);
-
- pthread_mutex_unlock(&module->lock);
-
- return obj;
-}
-
-struct job *get_job_by_name(struct module *module, const char *job_name)
-{
- return dictionary_get(module->jobs, job_name);
-}
-
-void unlink_job(const char *plugin_name, const char *module_name, const char *job_name)
-{
- // as we are going to do unlink here we better make sure we have all to build proper path
- if (unlikely(job_name == NULL || module_name == NULL || plugin_name == NULL))
- return;
- BUFFER *buffer = buffer_create(DYN_CONF_PATH_MAX, NULL);
- buffer_sprintf(buffer, DYN_CONF_DIR "/%s/%s/%s" DYN_CONF_CFG_EXT, plugin_name, module_name, job_name);
- if (unlink(buffer_tostring(buffer)))
- netdata_log_error("Cannot remove file %s", buffer_tostring(buffer));
-
- buffer_free(buffer);
-}
-
-void delete_job(struct configurable_plugin *plugin, const char *module_name, const char *job_name)
-{
- struct module *module = get_module_by_name(plugin, module_name);
- if (module == NULL) {
- error_report("DYNCFG module \"%s\" not found", module_name);
- return;
- }
-
- struct job *job_item = get_job_by_name(module, job_name);
- if (job_item == NULL) {
- error_report("DYNCFG job \"%s\" not found", job_name);
- return;
- }
-
- dictionary_del(module->jobs, job_name);
-}
-
-void delete_job_pname(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name)
-{
- const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, plugin_name);
- if (plugin_item == NULL) {
- error_report("DYNCFG plugin \"%s\" not found", plugin_name);
- return;
- }
- struct configurable_plugin *plugin = dictionary_acquired_item_value(plugin_item);
-
- delete_job(plugin, module_name, job_name);
-
- dictionary_acquired_item_release(plugins_dict, plugin_item);
-}
-
-int remove_job(struct module *module, struct job *job)
-{
- enum set_config_result rc = module->delete_job_cb(module->job_config_cb_usr_ctx, module->plugin->name, module->name, job->name);
-
- if (rc != SET_CONFIG_ACCEPTED) {
- error_report("DYNCFG module \"%s\" rejected delete job for \"%s\"", module->name, job->name);
- return 0;
- }
- return 1;
-}
-
-struct module *get_module_by_name(struct configurable_plugin *plugin, const char *module_name)
-{
- return dictionary_get(plugin->modules, module_name);
-}
-
-inline struct configurable_plugin *get_plugin_by_name(DICTIONARY *plugins_dict, const char *name)
-{
- return dictionary_get(plugins_dict, name);
-}
-
-static int store_config(const char *module_name, const char *submodule_name, const char *cfg_idx, dyncfg_config_t cfg)
-{
- BUFFER *filename = buffer_create(DYN_CONF_PATH_MAX, NULL);
- buffer_sprintf(filename, DYN_CONF_DIR "/%s", module_name);
- if (mkdir(buffer_tostring(filename), 0755) == -1) {
- if (errno != EEXIST) {
- netdata_log_error("DYNCFG store_config: failed to create module directory %s", buffer_tostring(filename));
- buffer_free(filename);
- return 1;
- }
- }
-
- if (submodule_name != NULL) {
- buffer_sprintf(filename, "/%s", submodule_name);
- if (mkdir(buffer_tostring(filename), 0755) == -1) {
- if (errno != EEXIST) {
- netdata_log_error("DYNCFG store_config: failed to create submodule directory %s", buffer_tostring(filename));
- buffer_free(filename);
- return 1;
- }
- }
- }
-
- if (cfg_idx != NULL)
- buffer_sprintf(filename, "/%s", cfg_idx);
-
- buffer_strcat(filename, DYN_CONF_CFG_EXT);
-
-
- error_report("DYNCFG store_config: %s", buffer_tostring(filename));
-
- //write to file
- FILE *f = fopen(buffer_tostring(filename), "w");
- if (f == NULL) {
- error_report("DYNCFG store_config: failed to open %s for writing", buffer_tostring(filename));
- buffer_free(filename);
- return 1;
- }
-
- fwrite(cfg.data, cfg.data_size, 1, f);
- fclose(f);
-
- buffer_free(filename);
- return 0;
-}
-
-#ifdef NETDATA_DEV_MODE
-#define netdata_dev_fatal(...) fatal(__VA_ARGS__)
-#else
-#define netdata_dev_fatal(...) error_report(__VA_ARGS__)
-#endif
-
-void dyn_conf_store_config(const char *function, const char *payload, struct configurable_plugin *plugin) {
- dyncfg_config_t config = {
- .data = (char*)payload,
- .data_size = strlen(payload)
- };
-
- char *fnc = strdupz(function);
- // split fnc to words
- char *words[DYNCFG_MAX_WORDS];
- size_t words_c = quoted_strings_splitter(fnc, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd);
-
- const char *fnc_name = get_word(words, words_c, 0);
- if (fnc_name == NULL) {
- error_report("Function name expected \"%s\"", function);
- goto CLEANUP;
- }
- if (strncmp(fnc_name, FUNCTION_NAME_SET_PLUGIN_CONFIG, strlen(FUNCTION_NAME_SET_PLUGIN_CONFIG)) == 0) {
- store_config(plugin->name, NULL, NULL, config);
- goto CLEANUP;
- }
-
- if (words_c < 2) {
- error_report("Module name expected \"%s\"", function);
- goto CLEANUP;
- }
- const char *module_name = get_word(words, words_c, 1);
- if (strncmp(fnc_name, FUNCTION_NAME_SET_MODULE_CONFIG, strlen(FUNCTION_NAME_SET_MODULE_CONFIG)) == 0) {
- store_config(plugin->name, module_name, NULL, config);
- goto CLEANUP;
- }
-
- if (words_c < 3) {
- error_report("Job name expected \"%s\"", function);
- goto CLEANUP;
- }
- const char *job_name = get_word(words, words_c, 2);
- if (strncmp(fnc_name, FUNCTION_NAME_SET_JOB_CONFIG, strlen(FUNCTION_NAME_SET_JOB_CONFIG)) == 0) {
- store_config(plugin->name, module_name, job_name, config);
- goto CLEANUP;
- }
-
- netdata_dev_fatal("Unknown function \"%s\"", function);
-
-CLEANUP:
- freez(fnc);
-}
-
-dyncfg_config_t load_config(const char *plugin_name, const char *module_name, const char *job_id)
-{
- BUFFER *filename = buffer_create(DYN_CONF_PATH_MAX, NULL);
- buffer_sprintf(filename, DYN_CONF_DIR "/%s", plugin_name);
- if (module_name != NULL)
- buffer_sprintf(filename, "/%s", module_name);
-
- if (job_id != NULL)
- buffer_sprintf(filename, "/%s", job_id);
-
- buffer_strcat(filename, DYN_CONF_CFG_EXT);
-
- dyncfg_config_t config;
- long bytes;
- config.data = read_by_filename(buffer_tostring(filename), &bytes);
-
- if (config.data == NULL)
- error_report("DYNCFG load_config: failed to load config from %s", buffer_tostring(filename));
-
- config.data_size = bytes;
-
- buffer_free(filename);
-
- return config;
-}
-
-char *set_plugin_config(struct configurable_plugin *plugin, dyncfg_config_t cfg)
-{
- enum set_config_result rc = plugin->set_config_cb(plugin->cb_usr_ctx, plugin->name, &cfg);
- if (rc != SET_CONFIG_ACCEPTED) {
- error_report("DYNCFG plugin \"%s\" rejected config", plugin->name);
- return "plugin rejected config";
- }
-
- return NULL;
-}
-
-static char *set_module_config(struct module *mod, dyncfg_config_t cfg)
-{
- struct configurable_plugin *plugin = mod->plugin;
-
- enum set_config_result rc = mod->set_config_cb(mod->config_cb_usr_ctx, plugin->name, mod->name, &cfg);
- if (rc != SET_CONFIG_ACCEPTED) {
- error_report("DYNCFG module \"%s\" rejected config", plugin->name);
- return "module rejected config";
- }
-
- return NULL;
-}
-
-struct job *job_new(const char *job_id)
-{
- struct job *job = callocz(1, sizeof(struct job));
- job->state = JOB_STATUS_UNKNOWN;
- job->last_state_update = now_realtime_usec();
- job->name = strdupz(job_id);
- netdata_mutex_init(&job->lock);
- return job;
-}
-
-static inline void job_del(struct job *job)
-{
- netdata_mutex_destroy(&job->lock);
- freez(job->reason);
- freez((void*)job->name);
- freez(job);
-}
-
-void job_del_cb(const DICTIONARY_ITEM *item, void *value, void *data)
-{
- UNUSED(item);
- UNUSED(data);
- job_del((struct job *)value);
-}
-
-void module_del_cb(const DICTIONARY_ITEM *item, void *value, void *data)
-{
- UNUSED(item);
- UNUSED(data);
- struct module *mod = (struct module *)value;
- dictionary_destroy(mod->jobs);
- freez(mod->name);
- freez(mod);
-}
-
-const DICTIONARY_ITEM *register_plugin(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, bool localhost)
-{
- if (get_plugin_by_name(plugins_dict, plugin->name) != NULL) {
- error_report("DYNCFG plugin \"%s\" already registered", plugin->name);
- return NULL;
- }
-
- if (plugin->set_config_cb == NULL) {
- error_report("DYNCFG plugin \"%s\" has no set_config_cb", plugin->name);
- return NULL;
- }
-
- pthread_mutex_init(&plugin->lock, NULL);
-
- plugin->modules = dictionary_create(DICT_OPTION_VALUE_LINK_DONT_CLONE);
- dictionary_register_delete_callback(plugin->modules, module_del_cb, NULL);
-
- if (localhost)
- deferred_config_push_back(plugins_dict, plugin->name, NULL, NULL);
-
- dictionary_set(plugins_dict, plugin->name, plugin, sizeof(plugin));
-
- // the plugin keeps the pointer to the dictionary item, so we need to acquire it
- return dictionary_get_and_acquire_item(plugins_dict, plugin->name);
-}
-
-void unregister_plugin(DICTIONARY *plugins_dict, const DICTIONARY_ITEM *plugin)
-{
- struct configurable_plugin *plug = dictionary_acquired_item_value(plugin);
- dictionary_acquired_item_release(plugins_dict, plugin);
- dictionary_del(plugins_dict, plug->name);
-}
-
-int register_module(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, struct module *module, bool localhost)
-{
- if (get_module_by_name(plugin, module->name) != NULL) {
- error_report("DYNCFG module \"%s\" already registered", module->name);
- return 1;
- }
-
- pthread_mutex_init(&module->lock, NULL);
-
- if (localhost)
- deferred_config_push_back(plugins_dict, plugin->name, module->name, NULL);
-
- module->plugin = plugin;
-
- if (module->type == MOD_TYPE_ARRAY) {
- module->jobs = dictionary_create(DICT_OPTION_VALUE_LINK_DONT_CLONE);
- dictionary_register_delete_callback(module->jobs, job_del_cb, NULL);
-
- if (localhost) {
- // load all jobs from disk
- BUFFER *path = buffer_create(DYN_CONF_PATH_MAX, NULL);
- buffer_sprintf(path, "%s/%s/%s", DYN_CONF_DIR, plugin->name, module->name);
- DIR *dir = opendir(buffer_tostring(path));
- if (dir != NULL) {
- struct dirent *ent;
- while ((ent = readdir(dir)) != NULL) {
- if (ent->d_name[0] == '.')
- continue;
- if (ent->d_type != DT_REG)
- continue;
- size_t len = strnlen(ent->d_name, NAME_MAX);
- if (len <= strlen(DYN_CONF_CFG_EXT))
- continue;
- if (strcmp(ent->d_name + len - strlen(DYN_CONF_CFG_EXT), DYN_CONF_CFG_EXT) != 0)
- continue;
- ent->d_name[len - strlen(DYN_CONF_CFG_EXT)] = '\0';
-
- struct job *job = job_new(ent->d_name);
- job->module = module;
- job->flags = JOB_FLG_PS_LOADED;
- job->type = JOB_TYPE_USER;
-
- dictionary_set(module->jobs, job->name, job, sizeof(job));
-
- deferred_config_push_back(plugins_dict, plugin->name, module->name, ent->d_name);
- }
- closedir(dir);
- }
- buffer_free(path);
- }
- }
-
- dictionary_set(plugin->modules, module->name, module, sizeof(module));
-
- return 0;
-}
-
-int register_job(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_type job_type, dyncfg_job_flg_t flags, int ignore_existing)
-{
- int rc = 1;
- const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, plugin_name);
- if (plugin_item == NULL) {
- error_report("plugin \"%s\" not registered", plugin_name);
- return rc;
- }
- struct configurable_plugin *plugin = dictionary_acquired_item_value(plugin_item);
- struct module *mod = get_module_by_name(plugin, module_name);
- if (mod == NULL) {
- error_report("module \"%s\" not registered", module_name);
- goto ERR_EXIT;
- }
- if (mod->type != MOD_TYPE_ARRAY) {
- error_report("module \"%s\" is not an array", module_name);
- goto ERR_EXIT;
- }
- if (get_job_by_name(mod, job_name) != NULL) {
- if (!ignore_existing)
- error_report("job \"%s\" already registered", job_name);
- goto ERR_EXIT;
- }
-
- struct job *job = job_new(job_name);
- job->module = mod;
- job->flags = flags;
- job->type = job_type;
-
- dictionary_set(mod->jobs, job->name, job, sizeof(job));
-
- rc = 0;
-ERR_EXIT:
- dictionary_acquired_item_release(plugins_dict, plugin_item);
- return rc;
-}
-
-void freez_dyncfg(void *ptr) {
- freez(ptr);
-}
-
-#ifdef NETDATA_TEST_DYNCFG
-static void handle_dyncfg_root(DICTIONARY *plugins_dict, struct uni_http_response *resp, int method)
-{
- if (method != HTTP_METHOD_GET) {
- resp->content = "method not allowed";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
- return;
- }
- json_object *obj = get_list_of_plugins_json(plugins_dict);
- json_object *wrapper = json_object_new_object();
- json_object_object_add(wrapper, "configurable_plugins", obj);
- resp->content = strdupz(json_object_to_json_string_ext(wrapper, JSON_C_TO_STRING_PRETTY));
- json_object_put(wrapper);
- resp->status = HTTP_RESP_OK;
- resp->content_type = CT_APPLICATION_JSON;
- resp->content_free = freez_dyncfg;
- resp->content_length = strlen(resp->content);
-}
-
-static void handle_plugin_root(struct uni_http_response *resp, int method, struct configurable_plugin *plugin, void *post_payload, size_t post_payload_size)
-{
- switch(method) {
- case HTTP_METHOD_GET:
- {
- dyncfg_config_t cfg = plugin->get_config_cb(plugin->cb_usr_ctx, plugin->name);
- resp->content = mallocz(cfg.data_size);
- memcpy(resp->content, cfg.data, cfg.data_size);
- resp->status = HTTP_RESP_OK;
- resp->content_free = freez_dyncfg;
- resp->content_length = cfg.data_size;
- return;
- }
- case HTTP_METHOD_PUT:
- {
- char *response;
- if (post_payload == NULL) {
- resp->content = "no payload";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_BAD_REQUEST;
- return;
- }
- dyncfg_config_t cont = {
- .data = post_payload,
- .data_size = post_payload_size
- };
- response = set_plugin_config(plugin, cont);
- if (response == NULL) {
- resp->status = HTTP_RESP_OK;
- resp->content = "OK";
- resp->content_length = strlen(resp->content);
- } else {
- resp->status = HTTP_RESP_BAD_REQUEST;
- resp->content = response;
- resp->content_length = strlen(resp->content);
- }
- return;
- }
- default:
- resp->content = "method not allowed";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
- return;
- }
-}
-#endif
-
-void handle_module_root(struct uni_http_response *resp, int method, struct configurable_plugin *plugin, const char *module, void *post_payload, size_t post_payload_size)
-{
- if (strncmp(module, DYN_CONF_SCHEMA, sizeof(DYN_CONF_SCHEMA)) == 0) {
- dyncfg_config_t cfg = plugin->get_config_schema_cb(plugin->cb_usr_ctx, plugin->name);
- resp->content = mallocz(cfg.data_size);
- memcpy(resp->content, cfg.data, cfg.data_size);
- resp->status = HTTP_RESP_OK;
- resp->content_free = freez_dyncfg;
- resp->content_length = cfg.data_size;
- return;
- }
- if (strncmp(module, DYN_CONF_MODULE_LIST, sizeof(DYN_CONF_MODULE_LIST)) == 0) {
- if (method != HTTP_METHOD_GET) {
- resp->content = "method not allowed (only GET)";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
- return;
- }
- json_object *obj = get_list_of_modules_json(plugin);
- json_object *wrapper = json_object_new_object();
- json_object_object_add(wrapper, "modules", obj);
- resp->content = strdupz(json_object_to_json_string_ext(wrapper, JSON_C_TO_STRING_PRETTY));
- json_object_put(wrapper);
- resp->status = HTTP_RESP_OK;
- resp->content_type = CT_APPLICATION_JSON;
- resp->content_free = freez_dyncfg;
- resp->content_length = strlen(resp->content);
- return;
- }
- struct module *mod = get_module_by_name(plugin, module);
- if (mod == NULL) {
- resp->content = "module not found";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_NOT_FOUND;
- return;
- }
- if (method == HTTP_METHOD_GET) {
- dyncfg_config_t cfg = mod->get_config_cb(mod->config_cb_usr_ctx, plugin->name, mod->name);
- resp->content = mallocz(cfg.data_size);
- memcpy(resp->content, cfg.data, cfg.data_size);
- resp->status = HTTP_RESP_OK;
- resp->content_free = freez_dyncfg;
- resp->content_length = cfg.data_size;
- return;
- } else if (method == HTTP_METHOD_PUT) {
- char *response;
- if (post_payload == NULL) {
- resp->content = "no payload";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_BAD_REQUEST;
- return;
- }
- dyncfg_config_t cont = {
- .data = post_payload,
- .data_size = post_payload_size
- };
- response = set_module_config(mod, cont);
- if (response == NULL) {
- resp->status = HTTP_RESP_OK;
- resp->content = "OK";
- resp->content_length = strlen(resp->content);
- } else {
- resp->status = HTTP_RESP_BAD_REQUEST;
- resp->content = response;
- resp->content_length = strlen(resp->content);
- }
- return;
- }
- resp->content = "method not allowed";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
-}
-
-static inline void _handle_job_root(struct uni_http_response *resp, int method, struct module *mod, const char *job_id, void *post_payload, size_t post_payload_size, struct job *job)
-{
- if (method == HTTP_METHOD_POST) {
- if (job != NULL) {
- resp->content = "can't POST, job already exists (use PUT to update?)";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_BAD_REQUEST;
- return;
- }
- if (post_payload == NULL) {
- resp->content = "no payload";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_BAD_REQUEST;
- return;
- }
- dyncfg_config_t cont = {
- .data = post_payload,
- .data_size = post_payload_size
- };
- if (mod->set_job_config_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name, job_id, &cont)) {
- resp->content = "failed to add job";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_INTERNAL_SERVER_ERROR;
- return;
- }
- resp->status = HTTP_RESP_OK;
- resp->content = "OK";
- resp->content_length = strlen(resp->content);
- return;
- }
- if (job == NULL) {
- resp->content = "job not found";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_NOT_FOUND;
- return;
- }
- switch (method) {
- case HTTP_METHOD_GET:
- {
- dyncfg_config_t cfg = mod->get_job_config_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name, job->name);
- resp->content = mallocz(cfg.data_size);
- memcpy(resp->content, cfg.data, cfg.data_size);
- resp->status = HTTP_RESP_OK;
- resp->content_free = freez_dyncfg;
- resp->content_length = cfg.data_size;
- return;
- }
- case HTTP_METHOD_PUT:
- {
- if (post_payload == NULL) {
- resp->content = "missing payload";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_BAD_REQUEST;
- return;
- }
- dyncfg_config_t cont = {
- .data = post_payload,
- .data_size = post_payload_size
- };
- if (mod->set_job_config_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name, job->name, &cont) != SET_CONFIG_ACCEPTED) {
- error_report("DYNCFG module \"%s\" rejected config for job \"%s\"", mod->name, job->name);
- resp->content = "failed to set job config";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_INTERNAL_SERVER_ERROR;
- return;
- }
- resp->status = HTTP_RESP_OK;
- resp->content = "OK";
- resp->content_length = strlen(resp->content);
- return;
- }
- case HTTP_METHOD_DELETE:
- {
- if (!remove_job(mod, job)) {
- resp->content = "failed to remove job";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_INTERNAL_SERVER_ERROR;
- return;
- }
- resp->status = HTTP_RESP_OK;
- resp->content = "OK";
- resp->content_length = strlen(resp->content);
- return;
- }
- default:
- resp->content = "method not allowed (only GET, PUT, DELETE)";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
- return;
- }
-}
-
-void handle_job_root(struct uni_http_response *resp, int method, struct module *mod, const char *job_id, void *post_payload, size_t post_payload_size)
-{
- if (strncmp(job_id, DYN_CONF_SCHEMA, sizeof(DYN_CONF_SCHEMA)) == 0) {
- dyncfg_config_t cfg = mod->get_config_schema_cb(mod->config_cb_usr_ctx, mod->plugin->name, mod->name);
- resp->content = mallocz(cfg.data_size);
- memcpy(resp->content, cfg.data, cfg.data_size);
- resp->status = HTTP_RESP_OK;
- resp->content_free = freez_dyncfg;
- resp->content_length = cfg.data_size;
- return;
- }
- if (strncmp(job_id, DYN_CONF_JOB_SCHEMA, sizeof(DYN_CONF_JOB_SCHEMA)) == 0) {
- dyncfg_config_t cfg = mod->get_job_config_schema_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name);
- resp->content = mallocz(cfg.data_size);
- memcpy(resp->content, cfg.data, cfg.data_size);
- resp->status = HTTP_RESP_OK;
- resp->content_free = freez_dyncfg;
- resp->content_length = cfg.data_size;
- return;
- }
- if (strncmp(job_id, DYN_CONF_JOB_LIST, sizeof(DYN_CONF_JOB_LIST)) == 0) {
- if (mod->type != MOD_TYPE_ARRAY) {
- resp->content = "module type is not job_array (can't get the list of jobs)";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_NOT_FOUND;
- return;
- }
- if (method != HTTP_METHOD_GET) {
- resp->content = "method not allowed (only GET)";
- resp->content_length = strlen(resp->content);
- resp->status = HTTP_RESP_METHOD_NOT_ALLOWED;
- return;
- }
- json_object *obj = get_list_of_jobs_json(mod);
- json_object *wrapper = json_object_new_object();
- json_object_object_add(wrapper, "jobs", obj);
- resp->content = strdupz(json_object_to_json_string_ext(wrapper, JSON_C_TO_STRING_PRETTY));
- json_object_put(wrapper);
- resp->status = HTTP_RESP_OK;
- resp->content_type = CT_APPLICATION_JSON;
- resp->content_free = freez_dyncfg;
- resp->content_length = strlen(resp->content);
- return;
- }
- const DICTIONARY_ITEM *job_item = dictionary_get_and_acquire_item(mod->jobs, job_id);
- struct job *job = dictionary_acquired_item_value(job_item);
-
- _handle_job_root(resp, method, mod, job_id, post_payload, post_payload_size, job);
-
- dictionary_acquired_item_release(mod->jobs, job_item);
-}
-
-struct uni_http_response dyn_conf_process_http_request(
- DICTIONARY *plugins_dict __maybe_unused,
- int method __maybe_unused,
- const char *plugin __maybe_unused,
- const char *module __maybe_unused,
- const char *job_id __maybe_unused,
- void *post_payload __maybe_unused,
- size_t post_payload_size __maybe_unused)
-{
- struct uni_http_response resp = {
- .status = HTTP_RESP_INTERNAL_SERVER_ERROR,
- .content_type = CT_TEXT_PLAIN,
- .content = HTTP_RESP_INTERNAL_SERVER_ERROR_STR,
- .content_free = NULL,
- .content_length = 0
- };
-#ifndef NETDATA_TEST_DYNCFG
- resp.content = "DYNCFG is disabled (as it is for now developer only feature). This will be enabled by default when ready for technical preview.";
- resp.content_length = strlen(resp.content);
- resp.status = HTTP_RESP_PRECOND_FAIL;
- return resp;
-#else
- if (plugin == NULL) {
- handle_dyncfg_root(plugins_dict, &resp, method);
- return resp;
- }
- const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, plugin);
- if (plugin_item == NULL) {
- resp.content = "plugin not found";
- resp.content_length = strlen(resp.content);
- resp.status = HTTP_RESP_NOT_FOUND;
- return resp;
- }
- struct configurable_plugin *plug = dictionary_acquired_item_value(plugin_item);
- if (module == NULL) {
- handle_plugin_root(&resp, method, plug, post_payload, post_payload_size);
- goto EXIT_PLUGIN;
- }
- if (job_id == NULL) {
- handle_module_root(&resp, method, plug, module, post_payload, post_payload_size);
- goto EXIT_PLUGIN;
- }
- // for modules we do not do get_and_acquire as modules are never removed (only together with the plugin)
- struct module *mod = get_module_by_name(plug, module);
- if (mod == NULL) {
- resp.content = "module not found";
- resp.content_length = strlen(resp.content);
- resp.status = HTTP_RESP_NOT_FOUND;
- goto EXIT_PLUGIN;
- }
- if (mod->type != MOD_TYPE_ARRAY) {
- resp.content = "400 - this module is not array type";
- resp.content_length = strlen(resp.content);
- resp.status = HTTP_RESP_BAD_REQUEST;
- goto EXIT_PLUGIN;
- }
- handle_job_root(&resp, method, mod, job_id, post_payload, post_payload_size);
-
-EXIT_PLUGIN:
- dictionary_acquired_item_release(plugins_dict, plugin_item);
- return resp;
-#endif
-}
-
-void plugin_del_cb(const DICTIONARY_ITEM *item, void *value, void *data)
-{
- UNUSED(item);
- UNUSED(data);
- struct configurable_plugin *plugin = (struct configurable_plugin *)value;
- dictionary_destroy(plugin->modules);
- freez(plugin->name);
- freez(plugin);
-}
-
-// on failure - return NULL - all unlocked, nothing acquired
-// on success - return pointer to job item - keep job and plugin acquired and locked!!!
-// for caller convenience (to prevent another lock and races)
-// caller is responsible to unlock the job and release it when not needed anymore
-// this also avoids dependency creep
-const DICTIONARY_ITEM *report_job_status_acq_lock(DICTIONARY *plugins_dict, const DICTIONARY_ITEM **plugin_acq_item, DICTIONARY **job_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_status status, int status_code, char *reason)
-{
- *plugin_acq_item = dictionary_get_and_acquire_item(plugins_dict, plugin_name);
- if (*plugin_acq_item == NULL) {
- netdata_log_error("plugin %s not found", plugin_name);
- return NULL;
- }
-
- struct configurable_plugin *plug = dictionary_acquired_item_value(*plugin_acq_item);
- struct module *mod = get_module_by_name(plug, module_name);
- if (mod == NULL) {
- netdata_log_error("module %s not found", module_name);
- dictionary_acquired_item_release(plugins_dict, *plugin_acq_item);
- return NULL;
- }
- if (mod->type != MOD_TYPE_ARRAY) {
- netdata_log_error("module %s is not array", module_name);
- dictionary_acquired_item_release(plugins_dict, *plugin_acq_item);
- return NULL;
- }
- *job_dict = mod->jobs;
- const DICTIONARY_ITEM *job_item = dictionary_get_and_acquire_item(mod->jobs, job_name);
- if (job_item == NULL) {
- netdata_log_error("job %s not found", job_name);
- dictionary_acquired_item_release(plugins_dict, *plugin_acq_item);
- return NULL;
- }
- struct job *job = dictionary_acquired_item_value(job_item);
-
- pthread_mutex_lock(&job->lock);
- job->status = status;
- job->state = status_code;
- if (job->reason != NULL) {
- freez(job->reason);
- }
- job->reason = reason != NULL ? strdupz(reason) : NULL; // reason is optional
- job->last_state_update = now_realtime_usec();
-
- job->dirty = true;
-
- // no unlock and acquired_item_release on success on purpose
- return job_item;
-}
-
-int dyn_conf_init(void)
-{
- if (mkdir(DYN_CONF_DIR, 0755) == -1) {
- if (errno != EEXIST) {
- netdata_log_error("failed to create directory for dynamic configuration");
- return 1;
- }
- }
-
- return 0;
-}
-
-static void dyncfg_cleanup(void *ptr) {
- struct netdata_static_thread *static_thread = (struct netdata_static_thread *) ptr;
- static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
-
- netdata_log_info("cleaning up...");
-
- pthread_mutex_lock(&deferred_configs_lock);
- dyncfg_shutdown = true;
- while (deferred_configs != NULL) {
- struct deferred_cfg_send *dcs = deferred_configs;
- deferred_configs = dcs->next;
- deferred_config_free(dcs);
- }
- pthread_mutex_unlock(&deferred_configs_lock);
-
- static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
-}
-
-void *dyncfg_main(void *ptr)
-{
- netdata_thread_cleanup_push(dyncfg_cleanup, ptr);
-
- while (!netdata_exit) {
- struct deferred_cfg_send *dcs = deferred_config_pop(ptr);
- DICTIONARY *plugins_dict = dcs->plugins_dict;
-#ifdef NETDATA_INTERNAL_CHECKS
- if (plugins_dict == NULL) {
- fatal("DYNCFG, plugins_dict is NULL");
- deferred_config_free(dcs);
- continue;
- }
-#endif
-
- const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, dcs->plugin_name);
- if (plugin_item == NULL) {
- error_report("DYNCFG, plugin %s not found", dcs->plugin_name);
- deferred_config_free(dcs);
- continue;
- }
- struct configurable_plugin *plugin = dictionary_acquired_item_value(plugin_item);
- if (dcs->module_name == NULL) {
- dyncfg_config_t cfg = load_config(dcs->plugin_name, NULL, NULL);
- if (cfg.data != NULL) {
- plugin->set_config_cb(plugin->cb_usr_ctx, plugin->name, &cfg);
- freez(cfg.data);
- }
- } else if (dcs->job_name == NULL) {
- dyncfg_config_t cfg = load_config(dcs->plugin_name, dcs->module_name, NULL);
- if (cfg.data != NULL) {
- struct module *mod = get_module_by_name(plugin, dcs->module_name);
- mod->set_config_cb(mod->config_cb_usr_ctx, plugin->name, mod->name, &cfg);
- freez(cfg.data);
- }
- } else {
- dyncfg_config_t cfg = load_config(dcs->plugin_name, dcs->module_name, dcs->job_name);
- if (cfg.data != NULL) {
- struct module *mod = get_module_by_name(plugin, dcs->module_name);
- mod->set_job_config_cb(mod->job_config_cb_usr_ctx, plugin->name, mod->name, dcs->job_name, &cfg);
- freez(cfg.data);
- }
- }
- deferred_config_free(dcs);
- dictionary_acquired_item_release(plugins_dict, plugin_item);
- }
-
- netdata_thread_cleanup_pop(1);
- return NULL;
-}
-
-bool is_dyncfg_function(const char *function_name, uint8_t type) {
- // TODO add hash to speed things up
- if (type & (DYNCFG_FUNCTION_TYPE_GET | DYNCFG_FUNCTION_TYPE_REGULAR)) {
- if (strncmp(function_name, FUNCTION_NAME_GET_PLUGIN_CONFIG, strlen(FUNCTION_NAME_GET_PLUGIN_CONFIG)) == 0)
- return true;
- if (strncmp(function_name, FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA, strlen(FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA)) == 0)
- return true;
- if (strncmp(function_name, FUNCTION_NAME_GET_MODULE_CONFIG, strlen(FUNCTION_NAME_GET_MODULE_CONFIG)) == 0)
- return true;
- if (strncmp(function_name, FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA, strlen(FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA)) == 0)
- return true;
- if (strncmp(function_name, FUNCTION_NAME_GET_JOB_CONFIG, strlen(FUNCTION_NAME_GET_JOB_CONFIG)) == 0)
- return true;
- if (strncmp(function_name, FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA, strlen(FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA)) == 0)
- return true;
- }
-
- if (type & (DYNCFG_FUNCTION_TYPE_SET | DYNCFG_FUNCTION_TYPE_PAYLOAD)) {
- if (strncmp(function_name, FUNCTION_NAME_SET_PLUGIN_CONFIG, strlen(FUNCTION_NAME_SET_PLUGIN_CONFIG)) == 0)
- return true;
- if (strncmp(function_name, FUNCTION_NAME_SET_MODULE_CONFIG, strlen(FUNCTION_NAME_SET_MODULE_CONFIG)) == 0)
- return true;
- if (strncmp(function_name, FUNCTION_NAME_SET_JOB_CONFIG, strlen(FUNCTION_NAME_SET_JOB_CONFIG)) == 0)
- return true;
- }
-
- if (type & (DYNCFG_FUNCTION_TYPE_DELETE | DYNCFG_FUNCTION_TYPE_REGULAR)) {
- if (strncmp(function_name, FUNCTION_NAME_DELETE_JOB, strlen(FUNCTION_NAME_DELETE_JOB)) == 0)
- return true;
- }
-
- return false;
-}
diff --git a/libnetdata/dyn_conf/dyn_conf.h b/libnetdata/dyn_conf/dyn_conf.h
deleted file mode 100644
index d584343b2..000000000
--- a/libnetdata/dyn_conf/dyn_conf.h
+++ /dev/null
@@ -1,237 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef DYN_CONF_H
-#define DYN_CONF_H
-
-#include "../libnetdata.h"
-
-#define FUNCTION_NAME_GET_PLUGIN_CONFIG "get_plugin_config"
-#define FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA "get_plugin_config_schema"
-#define FUNCTION_NAME_GET_MODULE_CONFIG "get_module_config"
-#define FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA "get_module_config_schema"
-#define FUNCTION_NAME_GET_JOB_CONFIG "get_job_config"
-#define FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA "get_job_config_schema"
-#define FUNCTION_NAME_SET_PLUGIN_CONFIG "set_plugin_config"
-#define FUNCTION_NAME_SET_MODULE_CONFIG "set_module_config"
-#define FUNCTION_NAME_SET_JOB_CONFIG "set_job_config"
-#define FUNCTION_NAME_DELETE_JOB "delete_job"
-
-#define DYNCFG_MAX_WORDS 5
-
-#define DYNCFG_VFNC_RET_CFG_ACCEPTED (1)
-
-enum module_type {
- MOD_TYPE_UNKNOWN = 0,
- MOD_TYPE_ARRAY,
- MOD_TYPE_SINGLE
-};
-
-static inline enum module_type str2_module_type(const char *type_name)
-{
- if (strcmp(type_name, "job_array") == 0)
- return MOD_TYPE_ARRAY;
- else if (strcmp(type_name, "single") == 0)
- return MOD_TYPE_SINGLE;
- return MOD_TYPE_UNKNOWN;
-}
-
-static inline const char *module_type2str(enum module_type type)
-{
- switch (type) {
- case MOD_TYPE_ARRAY:
- return "job_array";
- case MOD_TYPE_SINGLE:
- return "single";
- default:
- return "unknown";
- }
-}
-
-struct dyncfg_config {
- void *data;
- size_t data_size;
-};
-
-typedef struct dyncfg_config dyncfg_config_t;
-
-struct configurable_plugin;
-struct module;
-
-enum job_status {
- JOB_STATUS_UNKNOWN = 0, // State used until plugin reports first status
- JOB_STATUS_STOPPED,
- JOB_STATUS_RUNNING,
- JOB_STATUS_ERROR
-};
-
-static inline enum job_status str2job_state(const char *state_name) {
- if (strcmp(state_name, "stopped") == 0)
- return JOB_STATUS_STOPPED;
- else if (strcmp(state_name, "running") == 0)
- return JOB_STATUS_RUNNING;
- else if (strcmp(state_name, "error") == 0)
- return JOB_STATUS_ERROR;
- return JOB_STATUS_UNKNOWN;
-}
-
-const char *job_status2str(enum job_status status);
-
-enum set_config_result {
- SET_CONFIG_ACCEPTED = 0,
- SET_CONFIG_REJECTED,
- SET_CONFIG_DEFFER
-};
-
-typedef uint32_t dyncfg_job_flg_t;
-enum job_flags {
- JOB_FLG_PS_LOADED = 1 << 0, // PS abbr. Persistent Storage
- JOB_FLG_PLUGIN_PUSHED = 1 << 1, // got it from plugin (e.g. autodiscovered job)
- JOB_FLG_STREAMING_PUSHED = 1 << 2, // got it through streaming
- JOB_FLG_USER_CREATED = 1 << 3, // user created this job during agent runtime
-};
-
-enum job_type {
- JOB_TYPE_UNKNOWN = 0,
- JOB_TYPE_STOCK = 1,
- JOB_TYPE_USER = 2,
- JOB_TYPE_AUTODISCOVERED = 3,
-};
-
-static inline const char* job_type2str(enum job_type type)
-{
- switch (type) {
- case JOB_TYPE_STOCK:
- return "stock";
- case JOB_TYPE_USER:
- return "user";
- case JOB_TYPE_AUTODISCOVERED:
- return "autodiscovered";
- case JOB_TYPE_UNKNOWN:
- default:
- return "unknown";
- }
-}
-
-static inline enum job_type dyncfg_str2job_type(const char *type_name)
-{
- if (strcmp(type_name, "stock") == 0)
- return JOB_TYPE_STOCK;
- else if (strcmp(type_name, "user") == 0)
- return JOB_TYPE_USER;
- else if (strcmp(type_name, "autodiscovered") == 0)
- return JOB_TYPE_AUTODISCOVERED;
- error_report("Unknown job type: %s", type_name);
- return JOB_TYPE_UNKNOWN;
-}
-
-struct job
-{
- const char *name;
- enum job_type type;
- struct module *module;
-
- pthread_mutex_t lock;
- // lock protexts only fields below (which are modified during job existence)
- // others are static during lifetime of job
-
- int dirty; // this relates to rrdpush, true if parent has different data than us
-
- // state reported by plugin
- usec_t last_state_update;
- enum job_status status; // reported by plugin, enum as this has to be interpreted by UI
- int state; // code reported by plugin which can mean anything plugin wants
- char *reason; // reported by plugin, can be NULL (optional)
-
- dyncfg_job_flg_t flags;
-};
-
-struct module
-{
- pthread_mutex_t lock;
- char *name;
- enum module_type type;
-
- struct configurable_plugin *plugin;
-
- // module config
- enum set_config_result (*set_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, dyncfg_config_t *cfg);
- dyncfg_config_t (*get_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name);
- dyncfg_config_t (*get_config_schema_cb)(void *usr_ctx, const char *plugin_name, const char *module_name);
- void *config_cb_usr_ctx;
-
- DICTIONARY *jobs;
-
- // jobs config
- dyncfg_config_t (*get_job_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name);
- dyncfg_config_t (*get_job_config_schema_cb)(void *usr_ctx, const char *plugin_name, const char *module_name);
- enum set_config_result (*set_job_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name, dyncfg_config_t *cfg);
- enum set_config_result (*delete_job_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name);
- void *job_config_cb_usr_ctx;
-};
-
-struct configurable_plugin {
- pthread_mutex_t lock;
- char *name;
- DICTIONARY *modules;
- const char *schema;
-
- dyncfg_config_t (*get_config_cb)(void *usr_ctx, const char *plugin_name);
- dyncfg_config_t (*get_config_schema_cb)(void *usr_ctx, const char *plugin_name);
- enum set_config_result (*set_config_cb)(void *usr_ctx, const char *plugin_name, dyncfg_config_t *cfg);
- void *cb_usr_ctx; // context for all callbacks (split if needed in future)
-};
-
-// API to be used by plugins
-const DICTIONARY_ITEM *register_plugin(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, bool localhost);
-void unregister_plugin(DICTIONARY *plugins_dict, const DICTIONARY_ITEM *plugin);
-int register_module(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, struct module *module, bool localhost);
-int register_job(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_type job_type, dyncfg_job_flg_t flags, int ignore_existing);
-
-const DICTIONARY_ITEM *report_job_status_acq_lock(DICTIONARY *plugins_dict, const DICTIONARY_ITEM **plugin_acq_item, DICTIONARY **job_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_status status, int status_code, char *reason);
-
-void dyn_conf_store_config(const char *function, const char *payload, struct configurable_plugin *plugin);
-void unlink_job(const char *plugin_name, const char *module_name, const char *job_name);
-void delete_job(struct configurable_plugin *plugin, const char *module_name, const char *job_name);
-void delete_job_pname(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name);
-
-// API to be used by the web server(s)
-json_object *get_list_of_plugins_json(DICTIONARY *plugins_dict);
-struct configurable_plugin *get_plugin_by_name(DICTIONARY *plugins_dict, const char *name);
-
-json_object *get_list_of_modules_json(struct configurable_plugin *plugin);
-struct module *get_module_by_name(struct configurable_plugin *plugin, const char *module_name);
-
-json_object *job2json(struct job *job);
-
-// helper struct to make interface between internal webserver and h2o same
-struct uni_http_response {
- int status;
- char *content;
- size_t content_length;
- HTTP_CONTENT_TYPE content_type;
- void (*content_free)(void *);
-};
-
-struct uni_http_response dyn_conf_process_http_request(DICTIONARY *plugins_dict, int method, const char *plugin, const char *module, const char *job_id, void *payload, size_t payload_size);
-
-// API to be used by main netdata process, initialization and destruction etc.
-int dyn_conf_init(void);
-void freez_dyncfg(void *ptr);
-
-#define dyncfg_dictionary_create() dictionary_create(DICT_OPTION_VALUE_LINK_DONT_CLONE)
-
-void plugin_del_cb(const DICTIONARY_ITEM *item, void *value, void *data);
-
-void *dyncfg_main(void *in);
-
-#define DYNCFG_FUNCTION_TYPE_REGULAR (1 << 0)
-#define DYNCFG_FUNCTION_TYPE_PAYLOAD (1 << 1)
-#define DYNCFG_FUNCTION_TYPE_GET (1 << 2)
-#define DYNCFG_FUNCTION_TYPE_SET (1 << 3)
-#define DYNCFG_FUNCTION_TYPE_DELETE (1 << 4)
-#define DYNCFG_FUNCTION_TYPE_ALL \
- (DYNCFG_FUNCTION_TYPE_REGULAR | DYNCFG_FUNCTION_TYPE_PAYLOAD | DYNCFG_FUNCTION_TYPE_GET | DYNCFG_FUNCTION_TYPE_SET | DYNCFG_FUNCTION_TYPE_DELETE)
-
-bool is_dyncfg_function(const char *function_name, uint8_t type);
-
-#endif //DYN_CONF_H
diff --git a/libnetdata/dyn_conf/tests/sample_test_config.json b/libnetdata/dyn_conf/tests/sample_test_config.json
deleted file mode 100644
index a6595f124..000000000
--- a/libnetdata/dyn_conf/tests/sample_test_config.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "http_endpoints": {
- "parent": {
- "host": "127.0.0.1",
- "mguid": null,
- "port": 20001,
- "ssl": false
- },
- "child": {
- "host": "127.0.0.1",
- "mguid": "3bc2f7de-1445-11ee-9ed7-3c7c3f21784c",
- "port": 19999,
- "ssl": false
- }
- },
- "global": {
- "test_plugin_name": "external_plugin",
- "test_array_module_name": "module_of_the_future",
- "test_single_module_name": "module_of_the_future_single_type",
- "test_job_name": "fixed_job"
- }
-}
diff --git a/libnetdata/dyn_conf/tests/sub_tests/test_parent_child.rb b/libnetdata/dyn_conf/tests/sub_tests/test_parent_child.rb
deleted file mode 100644
index 820db77f8..000000000
--- a/libnetdata/dyn_conf/tests/sub_tests/test_parent_child.rb
+++ /dev/null
@@ -1,192 +0,0 @@
-class ParentChildTest
- @@plugin_cfg = <<~HEREDOC
-{ "test" : "true" }
-HEREDOC
- @@plugin_cfg2 = <<~HEREDOC
-{ "asdfgh" : "asdfgh" }
-HEREDOC
-
- @@job_cfg = <<~HEREDOC
-{ "i am newly created job" : "true" }
-HEREDOC
-
- def initialize
- @parent = $config[:http_endpoints][:parent]
- @child = $config[:http_endpoints][:child]
- @plugin = $config[:global][:test_plugin_name]
- @arry_mod = $config[:global][:test_array_module_name]
- @single_mod = $config[:global][:test_single_module_name]
- @test_job = $config[:global][:test_job_name]
- end
- def check_test_plugin_modules_list(host, child = nil)
- rc = DynCfgHttpClient.get_plugin_module_list(host, @plugin, child)
- assert_eq(rc.code, 200, "as HTTP code for get_module_list request on plugin \"#{@plugin}\"")
- modules = nil
- assert_nothing_raised do
- modules = JSON.parse(rc.parsed_response, symbolize_names: true)
- end
- assert_has_key?(modules, :modules)
- assert_eq(modules[:modules].count, 2, "as number of modules in plugin \"#{@plugin}\"")
- modules[:modules].each do |m|
- assert_has_key?(m, :name)
- assert_has_key?(m, :type)
- assert_is_one_of(m[:type], "job_array", "single")
- end
- assert_eq_str(modules[:modules][0][:name], @arry_mod, "name of first module in plugin \"#{@plugin}\"")
- assert_eq_str(modules[:modules][1][:name], @single_mod, "name of second module in plugin \"#{@plugin}\"")
- end
- def run
- TEST_SUITE("Parent/Child plugin config")
-
- TEST("parent/child/get_plugin_list", "Get child (hops:1) plugin list trough parent")
- plugins = DynCfgHttpClient.get_plugin_list(@parent, @child)
- assert_eq(plugins.code, 200, "as HTTP code for get_plugin_list request")
- assert_nothing_raised do
- plugins = JSON.parse(plugins.parsed_response, symbolize_names: true)
- end
- assert_has_key?(plugins, :configurable_plugins)
- assert_array_include?(plugins[:configurable_plugins], @plugin)
- PASS()
-
- TEST("parent/child/(set/get)plugin_config", "Set then get and compare child (hops:1) plugin config trough parent")
- rc = DynCfgHttpClient.set_plugin_config(@parent, @plugin, @@plugin_cfg, @child)
- assert_eq(rc.code, 200, "as HTTP code for set_plugin_config request")
-
- rc = DynCfgHttpClient.get_plugin_config(@parent, @plugin, @child)
- assert_eq(rc.code, 200, "as HTTP code for get_plugin_config request")
- assert_eq_str(rc.parsed_response.chomp!, @@plugin_cfg, "as plugin config")
-
- # We do this twice with different configs to ensure first config was not loaded from persistent storage (from previous tests)
- rc = DynCfgHttpClient.set_plugin_config(@parent, @plugin, @@plugin_cfg2, @child)
- assert_eq(rc.code, 200, "as HTTP code for set_plugin_config request 2")
-
- rc = DynCfgHttpClient.get_plugin_config(@parent, @plugin, @child)
- assert_eq(rc.code, 200, "as HTTP code for get_plugin_config request 2")
- assert_eq_str(rc.parsed_response.chomp!, @@plugin_cfg2, "set/get plugin config 2")
- PASS()
-
- TEST("child/get_plugin_config", "Get child (hops:0) plugin config and compare with what we got trough parent (set_plugin_config from previous test)")
- rc = DynCfgHttpClient.get_plugin_config(@child, @plugin, nil)
- assert_eq(rc.code, 200, "as HTTP code for get_plugin_config request")
- assert_eq_str(rc.parsed_response.chomp!, @@plugin_cfg2.chomp, "as plugin config")
- PASS()
-
- TEST("parent/child/plugin_module_list", "Get child (hops:1) plugin module list trough parent and check its contents")
- check_test_plugin_modules_list(@parent, @child)
- PASS()
-
- TEST("child/plugin_module_list", "Get child (hops:0) plugin module list directly and check its contents")
- check_test_plugin_modules_list(@child, nil)
- PASS()
-
- TEST("parent/child/module/jobs", "Get list of jobs from child (hops:1) trough parent and check its contents, check job updates")
- rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child)
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
- jobs = nil
- assert_nothing_raised do
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
- end
- assert_has_key?(jobs, :jobs)
- new_job = jobs[:jobs].find {|i| i[:name] == @test_job}
- assert_not_nil(new_job)
- assert_has_key?(new_job, :status)
- assert_not_eq_str(new_job[:status], "unknown", "job status is other than unknown")
- assert_has_key?(new_job, :flags)
- assert_array_include?(new_job[:flags], "JOB_FLG_STREAMING_PUSHED")
- PASS()
-
- TEST("child/module/jobs", "Get list of jobs direct from child (hops:0) and check its contents, check job updates")
- rc = DynCfgHttpClient.get_job_list(@child, @plugin, @arry_mod, nil)
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
- jobs = nil
- assert_nothing_raised do
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
- end
- assert_has_key?(jobs, :jobs)
- new_job = jobs[:jobs].find {|i| i[:name] == @test_job}
- assert_not_nil(new_job)
- assert_has_key?(new_job, :status)
- assert_not_eq_str(new_job[:status], "unknown", "job status is other than unknown")
- assert_has_key?(new_job, :flags)
-
- assert_array_not_include?(new_job[:flags], "JOB_FLG_STREAMING_PUSHED") # this is plugin directly at child so it should not show this flag
- PASS()
-
- TEST("parent/child/single_module/jobs", "Attempt getting list of jobs from child (hops:1) trough parent on single module. Check it fails properly")
- rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @single_mod, @child)
- assert_eq(rc.code, 400, "as HTTP code for get_jobs request")
- assert_eq_str(rc.parsed_response, '400 - this module is not array type', "as HTTP code for get_jobs request on single module")
- PASS()
-
- created_job = SecureRandom.uuid
- TEST("parent/child/module/cr_del_job", "Create and delete job on child (hops:1) trough parent")
- # create new job
- rc = DynCfgHttpClient.create_job(@parent, @plugin, @arry_mod, created_job, @@job_cfg, @child)
- assert_eq_http_code(rc, 200, "as HTTP code for create_job request")
- # check this job is in job list @parent
- rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child)
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
- jobs = nil
- assert_nothing_raised do
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
- end
- assert_has_key?(jobs, :jobs)
- new_job = jobs[:jobs].find {|i| i[:name] == created_job}
- assert_not_nil(new_job)
- # check this job is in job list @child
- rc = DynCfgHttpClient.get_job_list(@child, @plugin, @arry_mod, nil)
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
- jobs = nil
- assert_nothing_raised do
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
- end
- assert_has_key?(jobs, :jobs)
- new_job = jobs[:jobs].find {|i| i[:name] == created_job}
- assert_not_nil(new_job)
- # check we can get job config back
- rc = DynCfgHttpClient.get_job_config(@parent, @plugin, @arry_mod, created_job, @child)
- assert_eq(rc.code, 200, "as HTTP code for get_job_config request")
- assert_eq_str(rc.parsed_response.chomp!, @@job_cfg, "as job config")
- # delete job
- rc = DynCfgHttpClient.delete_job(@parent, @plugin, @arry_mod, created_job, @child)
- assert_eq(rc.code, 200, "as HTTP code for delete_job request")
- # Check it is not in parents job list anymore
- rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child)
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
- jobs = nil
- assert_nothing_raised do
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
- end
- assert_has_key?(jobs, :jobs)
- new_job = jobs[:jobs].find {|i| i[:name] == created_job}
- assert_nil(new_job)
- # Check it is not in childs job list anymore
- rc = DynCfgHttpClient.get_job_list(@child, @plugin, @arry_mod, nil)
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
- jobs = nil
- assert_nothing_raised do
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
- end
- assert_has_key?(jobs, :jobs)
- new_job = jobs[:jobs].find {|i| i[:name] == created_job}
- assert_nil(new_job)
- PASS()
-
- TEST("parent/child/module/del_undeletable_job", "Try delete job on child (child rejects), check failure case works (hops:1)")
- # test if plugin rejects job deletion the job still remains in list as it should
- rc = DynCfgHttpClient.delete_job(@parent, @plugin, @arry_mod, @test_job, @child)
- assert_eq(rc.code, 500, "as HTTP code for delete_job request")
- rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child)
- assert_eq(rc.code, 200, "as HTTP code for get_jobs request")
- jobs = nil
- assert_nothing_raised do
- jobs = JSON.parse(rc.parsed_response, symbolize_names: true)
- end
- assert_has_key?(jobs, :jobs)
- job = jobs[:jobs].find {|i| i[:name] == @test_job}
- assert_not_nil(job)
- PASS()
- end
-end
-
-ParentChildTest.new.run()
diff --git a/libnetdata/dyn_conf/tests/test_dyncfg.rb b/libnetdata/dyn_conf/tests/test_dyncfg.rb
deleted file mode 100755
index 1b4b3a068..000000000
--- a/libnetdata/dyn_conf/tests/test_dyncfg.rb
+++ /dev/null
@@ -1,266 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'json'
-require 'httparty'
-require 'pastel'
-require 'securerandom'
-
-ARGV.length == 1 or raise "Usage: #{$0} <config file>"
-config_file = ARGV[0]
-
-File.exist?(config_file) or raise "File not found: #{config_file}"
-
-$config = JSON.parse(File.read(config_file), symbolize_names: true)
-
-$plugin_name = $config[:global][:test_plugin_name]
-$pastel = Pastel.new
-
-class TestRunner
- attr_reader :stats
- def initialize
- @stats = {
- :suites => 0,
- :tests => 0,
- :assertions => 0
- }
- @test = nil
- end
- def add_assertion()
- @stats[:assertions] += 1
- end
- def FAIL(msg, exception = nil, loc = nil)
- puts $pastel.red.bold(" ✕ FAIL")
- STDERR.print " "
- if loc
- STDERR.print $pastel.yellow("@#{loc.path}:#{loc.lineno}: ")
- else
- STDERR.print $pastel.yellow("@#{caller_locations(1, 1).first.path}:#{caller_locations(1, 1).first.lineno}: ")
- end
- STDERR.puts msg
- STDERR.puts exception.full_message(:highlight => true) if exception
- STDERR.puts $pastel.yellow(" Backtrace:")
- caller.each do |line|
- STDERR.puts " #{line}"
- end
- exit 1
- end
- def PASS()
- STDERR.puts $pastel.green.bold(" ✓ PASS")
- @stats[:tests] += 1
- @test = nil
- end
- def TEST_SUITE(name)
- puts $pastel.bold("• TEST SUITE: \"#{name}\"")
- @stats[:suites] += 1
- end
- def assert_no_test_running()
- unless @test.nil?
- STDERR.puts $pastel.red("\nFATAL: Test \"#{@test}\" did not call PASS() or FAIL()!")
- exit 1
- end
- end
- def TEST(name, description = nil)
- assert_no_test_running()
- @test = name
- col = 0
- txt = " ├─ T: #{name} "
- col += txt.length
- print $pastel.bold(txt)
-
- tab = 50
- rem = tab - (col % tab)
- rem.times do putc ' ' end
- col += rem
-
- if (description)
- txt = " - #{description} "
- col += txt.length
- print txt
-
- tab = 180
- rem = tab - (col % tab)
- rem.times do putc '.' end
- end
- end
- def FINALIZE()
- assert_no_test_running()
- end
-end
-
-$test_runner = TestRunner.new
-def FAIL(msg, exception = nil, loc = nil)
- $test_runner.FAIL(msg, exception, loc)
-end
-def PASS()
- $test_runner.PASS()
-end
-def TEST_SUITE(name)
- $test_runner.TEST_SUITE(name)
-end
-def TEST(name, description = nil)
- $test_runner.TEST(name, description)
-end
-
-def assert_eq(got, expected, msg = nil)
- unless got == expected
- FAIL("Expected #{expected}, got #{got} #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first)
- end
- $test_runner.add_assertion()
-end
-def assert_eq_http_code(got, expected, msg = nil)
- unless got.code == expected
- FAIL("Expected #{expected}, got #{got}. Server \"#{got.parsed_response}\" #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first)
- end
- $test_runner.add_assertion()
-end
-def assert_eq_str(got, expected, msg = nil)
- unless got == expected
- FAIL("Strings do not match #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first)
- end
- $test_runner.add_assertion()
-end
-def assert_not_eq_str(got, expected, msg = nil)
- unless got != expected
- FAIL("Strings shoud not match #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first)
- end
- $test_runner.add_assertion()
-end
-def assert_nothing_raised()
- begin
- yield
- rescue Exception => e
- FAIL("Unexpected exception of type #{e.class} raised. Msg: \"#{e.message}\"", e, caller_locations(1, 1).first)
- end
- $test_runner.add_assertion()
-end
-def assert_has_key?(hash, key)
- unless hash.has_key?(key)
- FAIL("Expected key \"#{key}\" in hash", nil, caller_locations(1, 1).first)
- end
- $test_runner.add_assertion()
-end
-def assert_array_include?(array, value)
- unless array.include?(value)
- FAIL("Expected array to include \"#{value}\"", nil, caller_locations(1, 1).first)
- end
- $test_runner.add_assertion()
-end
-def assert_array_not_include?(array, value)
- if array.include?(value)
- FAIL("Expected array to not include \"#{value}\"", nil, caller_locations(1, 1).first)
- end
- $test_runner.add_assertion()
-end
-def assert_is_one_of(value, *values)
- unless values.include?(value)
- FAIL("Expected value to be one of #{values.join(", ")}", nil, caller_locations(1, 1).first)
- end
- $test_runner.add_assertion()
-end
-def assert_not_nil(value)
- if value.nil?
- FAIL("Expected value to not be nil", nil, caller_locations(1, 1).first)
- end
- $test_runner.add_assertion()
-end
-def assert_nil(value)
- unless value.nil?
- FAIL("Expected value to be nil", nil, caller_locations(1, 1).first)
- end
- $test_runner.add_assertion()
-end
-
-
-class DynCfgHttpClient
- def self.protocol(cfg)
- return cfg[:ssl] ? 'https://' : 'http://'
- end
- def self.url_base(host)
- return "#{protocol(host)}#{host[:host]}:#{host[:port]}"
- end
- def self.get_url_cfg_base(host, child = nil)
- url = url_base(host)
- url += "/host/#{child[:mguid]}" if child
- url += "/api/v2/config"
- return url
- end
- def self.get_url_cfg_plugin(host, plugin, child = nil)
- return get_url_cfg_base(host, child) + '/' + plugin
- end
- def self.get_url_cfg_module(host, plugin, mod, child = nil)
- return get_url_cfg_plugin(host, plugin, child) + '/' + mod
- end
- def self.get_url_cfg_job(host, plugin, mod, job_id, child = nil)
- return get_url_cfg_module(host, plugin, mod, child) + "/#{job_id}"
- end
- def self.get_plugin_list(host, child = nil)
- begin
- return HTTParty.get(get_url_cfg_base(host, child), verify: false, format: :plain)
- rescue => e
- FAIL(e.message, e)
- end
- end
- def self.get_plugin_config(host, plugin, child = nil)
- begin
- return HTTParty.get(get_url_cfg_plugin(host, plugin, child), verify: false)
- rescue => e
- FAIL(e.message, e)
- end
- end
- def self.set_plugin_config(host, plugin, cfg, child = nil)
- begin
- return HTTParty.put(get_url_cfg_plugin(host, plugin, child), verify: false, body: cfg)
- rescue => e
- FAIL(e.message, e)
- end
- end
- def self.get_plugin_module_list(host, plugin, child = nil)
- begin
- return HTTParty.get(get_url_cfg_plugin(host, plugin, child) + "/modules", verify: false, format: :plain)
- rescue => e
- FAIL(e.message, e)
- end
- end
- def self.get_job_list(host, plugin, mod, child = nil)
- begin
- return HTTParty.get(get_url_cfg_module(host, plugin, mod, child) + "/jobs", verify: false, format: :plain)
- rescue => e
- FAIL(e.message, e)
- end
- end
- def self.create_job(host, plugin, mod, job_id, job_cfg, child = nil)
- begin
- return HTTParty.post(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false, body: job_cfg)
- rescue => e
- FAIL(e.message, e)
- end
- end
- def self.delete_job(host, plugin, mod, job_id, child = nil)
- begin
- return HTTParty.delete(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false)
- rescue => e
- FAIL(e.message, e)
- end
- end
- def self.get_job_config(host, plugin, mod, job_id, child = nil)
- begin
- return HTTParty.get(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false, format: :plain)
- rescue => e
- FAIL(e.message, e)
- end
- end
- def self.set_job_config(host, plugin, mod, job_id, job_cfg, child = nil)
- begin
- return HTTParty.put(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false, body: job_cfg)
- rescue => e
- FAIL(e.message, e)
- end
- end
-end
-
-require_relative 'sub_tests/test_parent_child.rb'
-
-$test_runner.FINALIZE()
-puts $pastel.green.bold("All tests passed!")
-puts ("Total #{$test_runner.stats[:assertions]} assertions, #{$test_runner.stats[:tests]} tests in #{$test_runner.stats[:suites]} suites")
-exit 0
diff --git a/libnetdata/dyn_conf/tests/test_plugin/test.plugin b/libnetdata/dyn_conf/tests/test_plugin/test.plugin
deleted file mode 100755
index b890ab314..000000000
--- a/libnetdata/dyn_conf/tests/test_plugin/test.plugin
+++ /dev/null
@@ -1,250 +0,0 @@
-#!/usr/bin/env ruby
-
-# bogus chart that we create just so there is at least one chart
-CHART_TYPE = 'lines'
-UPDATE_EVERY = 1
-PRIORITY = 100000
-CHART_NAME = 'number_of_processes'
-DIMENSION_NAME = 'running'
-
-$plugin_name = "external_plugin"
-$plugin_version = "0.0.1"
-$plugin_config = <<-HEREDOC
-test_plugin_config
-hableba hableba hableba
-HEREDOC
-
-$array_module_name = 'module_of_the_future'
-$fixed_job_name = 'fixed_job'
-
-$modules = {
- $array_module_name => {
- :type => :job_array,
- :jobs => {
- $fixed_job_name => {
- :type => :fixed,
- :config => <<-HEREDOC
-fixed_job_config
-HEREDOC
- },
- },
- :config => <<-HEREDOC
-module_of_the_future_config
-HEREDOC
- },
- "module_of_the_future_single_type" => {
- :type => :single,
- :jobs => {},
- :config => <<-HEREDOC
-module_of_the_future_single_type_config
-HEREDOC
- }
-}
-
-def out(str)
- $log.puts "2 NETDATA> #{str}"
- $stdout.puts str
- $stdout.flush
- $log.flush
-end
-
-def log(str)
- $log.puts "LOG > #{str}"
- $log.flush
-end
-
-#TODO this is AI code, verify
-def split_with_quotes(str)
- result = []
- current_word = ""
- in_quotes = false
- escaped = false
-
- str.each_char do |char|
- if char == '\\' && !escaped
- escaped = true
- next
- end
-
- if char == '"' && !escaped
- in_quotes = !in_quotes
- current_word << char
- elsif char == ' ' && !in_quotes
- result << current_word unless current_word.empty?
- current_word = ""
- else
- current_word << char
- end
-
- escaped = false
- end
-
- result << current_word unless current_word.empty?
-
- result
-end
-
-
-def print_startup_messages
- out "DYNCFG_ENABLE #{$plugin_name}"
- $modules.each do |name, module_config|
- out "DYNCFG_REGISTER_MODULE #{name} #{module_config[:type]}"
- end
- out "CHART system.#{CHART_NAME} '' 'Number of running processes' 'processes' processes processes.#{CHART_NAME} #{CHART_TYPE} #{PRIORITY} #{UPDATE_EVERY}"
- out "DIMENSION #{DIMENSION_NAME} '' absolute 1 1"
-
- $modules.each do |mod_name, mod|
- next unless mod[:type] == :job_array
- mod[:jobs].each do |job_name, job|
- next unless job[:type] == :fixed
- out "DYNCFG_REGISTER_JOB #{mod_name} #{job_name} stock 0"
- out "REPORT_JOB_STATUS #{$array_module_name} #{$fixed_job_name} running 0"
- end
- end
-end
-
-def function_result(txid, msg, result)
- out "FUNCTION_RESULT_BEGIN #{txid} #{result} text/plain 5"
- out msg
- out "FUNCTION_RESULT_END"
-end
-
-def process_payload_function(params)
- log "payload function #{params[:fncname]}, #{params[:fncparams]}"
- fnc_name, mod_name, job_name = params[:fncparams]
- case fnc_name
- when 'set_plugin_config'
- $plugin_config = params[:payload]
- function_result(params[:txid], "plugin config set", 1)
- when 'set_module_config'
- mod = $modules[mod_name]
- return function_result(params[:txid], "no such module", 0) if mod.nil?
- mod[:config] = params[:payload]
- function_result(params[:txid], "module config set", 1)
- when 'set_job_config'
- mod = $modules[mod_name]
- return function_result(params[:txid], "no such module", 0) if mod.nil?
- job = mod[:jobs][job_name]
- if job.nil?
- job = Hash.new if job.nil?
- job[:type] = :dynamic
- mod[:jobs][job_name] = job
- end
- job[:config] = params[:payload]
- function_result(params[:txid], "job config set", 1)
- end
-end
-
-def process_function(params)
- log "normal function #{params[:fncname]}, #{params[:fncparams]}"
- fnc_name, mod_name, job_name = params[:fncparams]
- case fnc_name
- when 'get_plugin_config'
- function_result(params[:txid], $plugin_config, 1)
- when 'get_module_config'
- return function_result(params[:txid], "no such module", 0) unless $modules.has_key?(mod_name)
- function_result(params[:txid], $modules[mod_name][:config], 1)
- when 'get_job_config'
- mod = $modules[mod_name]
- return function_result(params[:txid], "no such module", 0) if mod.nil?
- job = mod[:jobs][job_name]
- return function_result(params[:txid], "no such job", 0) if job.nil?
- function_result(params[:txid], job[:config], 1)
- when 'delete_job'
- mod = $modules[mod_name]
- return function_result(params[:txid], "no such module", 0) if mod.nil?
- job = mod[:jobs][job_name]
- return function_result(params[:txid], "no such job", 0) if job.nil?
- if job[:type] == :fixed
- return function_result(params[:txid], "this job can't be deleted", 0)
- else
- mod[:jobs].delete(job_name)
- function_result(params[:txid], "job deleted", 1)
- end
- end
-end
-
-$inflight_incoming = nil
-def process_input(input)
- words = split_with_quotes(input)
-
- unless $inflight_incoming.nil?
- if input == "FUNCTION_PAYLOAD_END"
- log $inflight_incoming[:payload]
- process_payload_function($inflight_incoming)
- $inflight_incoming = nil
- else
- $inflight_incoming[:payload] << input
- $inflight_incoming[:payload] << "\n"
- end
- return
- end
-
- case words[0]
- when "FUNCTION", "FUNCTION_PAYLOAD"
- params = {}
- params[:command] = words[0]
- params[:txid] = words[1]
- params[:timeout] = words[2].to_i
- params[:fncname] = words[3]
- params[:fncname] = params[:fncname][1..-2] if params[:fncname].start_with?('"') && params[:fncname].end_with?('"')
- if params[:command] == "FUNCTION_PAYLOAD"
- $inflight_incoming = Hash.new
- params[:fncparams] = split_with_quotes(params[:fncname])
- params[:fncname] = params[:fncparams][0]
- $inflight_incoming[:txid] = params[:txid]
- $inflight_incoming[:fncname] = params[:fncname]
- $inflight_incoming[:params] = params
- $inflight_incoming[:fncparams] = params[:fncparams]
- $inflight_incoming[:payload] = ""
- else
- params[:fncparams] = split_with_quotes(params[:fncname])
- params[:fncname] = params[:fncparams][0]
- process_function(params)
- end
- end
-end
-
-def read_and_output_metric
- processes = `ps -e | wc -l`.to_i - 1 # -1 to exclude the header line
- timestamp = Time.now.to_i
-
- puts "BEGIN system.#{CHART_NAME}"
- puts "SET #{DIMENSION_NAME} = #{processes}"
- puts "END"
-end
-
-def the_main
- $stderr.reopen("/tmp/test_plugin_err.log", "w")
- $log = File.open("/tmp/test_plugin.log", "w")
- $log.puts "Starting plugin"
- print_startup_messages
- $log.puts "init done"
- $log.flush
-
- last_metric_time = Time.now
-
- loop do
- time_since_last_metric = Time.now - last_metric_time
-
- # If it's been more than 1 second since we collected metrics, collect them now
- if time_since_last_metric >= 1
- read_and_output_metric
- last_metric_time = Time.now
- end
-
- # Use select to wait for input, but only wait up to the time remaining until we need to collect metrics again
- remaining_time = [1 - time_since_last_metric, 0].max
- if select([$stdin], nil, nil, remaining_time)
- input = $stdin.gets
- next if input.class != String
- input.chomp!
- $log.puts "RAW INPUT< #{input}"
- $log.flush
- process_input(input)
- end
- end
-end
-
-
-the_main if __FILE__ == $PROGRAM_NAME
diff --git a/libnetdata/ebpf/Makefile.am b/libnetdata/ebpf/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/ebpf/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/ebpf/README.md b/libnetdata/ebpf/README.md
deleted file mode 100644
index bf2c6ff35..000000000
--- a/libnetdata/ebpf/README.md
+++ /dev/null
@@ -1,13 +0,0 @@
-<!--
-title: "eBPF"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/ebpf/README.md
-sidebar_label: "eBPF"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# eBPF library
-
-Netdata's eBPF library supports the [eBPF collector](https://github.com/netdata/netdata/blob/master/collectors/ebpf.plugin/README.md).
-
diff --git a/libnetdata/ebpf/ebpf.c b/libnetdata/ebpf/ebpf.c
deleted file mode 100644
index 99f23b9c0..000000000
--- a/libnetdata/ebpf/ebpf.c
+++ /dev/null
@@ -1,1618 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <dlfcn.h>
-#include <sys/utsname.h>
-
-#include "../libnetdata.h"
-
-char *ebpf_user_config_dir = CONFIG_DIR;
-char *ebpf_stock_config_dir = LIBCONFIG_DIR;
-
-/*
-static int clean_kprobe_event(FILE *out, char *filename, char *father_pid, netdata_ebpf_events_t *ptr)
-{
- int fd = open(filename, O_WRONLY | O_APPEND, 0);
- if (fd < 0) {
- if (out) {
- fprintf(out, "Cannot open %s : %s\n", filename, strerror(errno));
- }
- return 1;
- }
-
- char cmd[1024];
- int length = snprintf(cmd, 1023, "-:kprobes/%c_netdata_%s_%s", ptr->type, ptr->name, father_pid);
- int ret = 0;
- if (length > 0) {
- ssize_t written = write(fd, cmd, strlen(cmd));
- if (written < 0) {
- if (out) {
- fprintf(
- out, "Cannot remove the event (%d, %d) '%s' from %s : %s\n", getppid(), getpid(), cmd, filename,
- strerror((int)errno));
- }
- ret = 1;
- }
- }
-
- close(fd);
-
- return ret;
-}
-
-int clean_kprobe_events(FILE *out, int pid, netdata_ebpf_events_t *ptr)
-{
- char filename[FILENAME_MAX + 1];
- snprintf(filename, FILENAME_MAX, "%s%s", NETDATA_DEBUGFS, "kprobe_events");
-
- char removeme[16];
- snprintf(removeme, 15, "%d", pid);
-
- int i;
- for (i = 0; ptr[i].name; i++) {
- if (clean_kprobe_event(out, filename, removeme, &ptr[i])) {
- break;
- }
- }
-
- return 0;
-}
-*/
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/**
- * Get Kernel version
- *
- * Get the current kernel from /proc and returns an integer value representing it
- *
- * @return it returns a value representing the kernel version.
- */
-int ebpf_get_kernel_version()
-{
- char major[16], minor[16], patch[16];
- char ver[VERSION_STRING_LEN];
- char *version = ver;
-
- int fd = open("/proc/sys/kernel/osrelease", O_RDONLY);
- if (fd < 0)
- return -1;
-
- ssize_t len = read(fd, ver, sizeof(ver));
- if (len < 0) {
- close(fd);
- return -1;
- }
-
- close(fd);
-
- char *move = major;
- while (*version && *version != '.')
- *move++ = *version++;
- *move = '\0';
-
- version++;
- move = minor;
- while (*version && *version != '.')
- *move++ = *version++;
- *move = '\0';
-
- if (*version)
- version++;
- else
- return -1;
-
- move = patch;
- while (*version && *version != '\n' && *version != '-')
- *move++ = *version++;
- *move = '\0';
-
- // This new rule is fixing kernel version according the formula:
- // KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
- // that was extracted from /usr/include/linux/version.h
- int ipatch = (int)str2l(patch);
- if (ipatch > 255)
- ipatch = 255;
-
- return ((int)(str2l(major) * 65536) + (int)(str2l(minor) * 256) + ipatch);
-}
-
-/**
- * Get RH release
- *
- * Read Red Hat release from /etc/redhat-release
- *
- * @return It returns RH release on success and -1 otherwise
- */
-int get_redhat_release()
-{
- char buffer[VERSION_STRING_LEN + 1];
- int major, minor;
- FILE *fp = fopen("/etc/redhat-release", "r");
-
- if (fp) {
- major = 0;
- minor = -1;
- size_t length = fread(buffer, sizeof(char), VERSION_STRING_LEN, fp);
- if (length > 4) {
- buffer[length] = '\0';
- char *end = strchr(buffer, '.');
- char *start;
- if (end) {
- *end = 0x0;
-
- if (end > buffer) {
- start = end - 1;
-
- major = strtol(start, NULL, 10);
- start = ++end;
-
- end++;
- if (end) {
- end = 0x00;
- minor = strtol(start, NULL, 10);
- } else {
- minor = -1;
- }
- }
- }
- }
-
- fclose(fp);
- return ((major * 256) + minor);
- } else {
- return -1;
- }
-}
-
-/**
- * Check if the kernel is in a list of rejected ones
- *
- * @return Returns 1 if the kernel is rejected, 0 otherwise.
- */
-static int kernel_is_rejected()
-{
- // Get kernel version from system
- char version_string[VERSION_STRING_LEN + 1];
- int version_string_len = 0;
-
- if (read_file("/proc/version_signature", version_string, VERSION_STRING_LEN)) {
- if (read_file("/proc/version", version_string, VERSION_STRING_LEN)) {
- struct utsname uname_buf;
- if (!uname(&uname_buf)) {
- netdata_log_info("Cannot check kernel version");
- return 0;
- }
- version_string_len =
- snprintfz(version_string, VERSION_STRING_LEN, "%s %s", uname_buf.release, uname_buf.version);
- }
- }
-
- if (!version_string_len)
- version_string_len = strlen(version_string);
-
- // Open a file with a list of rejected kernels
- char *config_dir = getenv("NETDATA_USER_CONFIG_DIR");
- if (config_dir == NULL) {
- config_dir = CONFIG_DIR;
- }
-
- char filename[FILENAME_MAX + 1];
- snprintfz(filename, FILENAME_MAX, "%s/ebpf.d/%s", config_dir, EBPF_KERNEL_REJECT_LIST_FILE);
- FILE *kernel_reject_list = fopen(filename, "r");
-
- if (!kernel_reject_list) {
- // Keep this to have compatibility with old versions
- snprintfz(filename, FILENAME_MAX, "%s/%s", config_dir, EBPF_KERNEL_REJECT_LIST_FILE);
- kernel_reject_list = fopen(filename, "r");
-
- if (!kernel_reject_list) {
- config_dir = getenv("NETDATA_STOCK_CONFIG_DIR");
- if (config_dir == NULL) {
- config_dir = LIBCONFIG_DIR;
- }
-
- snprintfz(filename, FILENAME_MAX, "%s/ebpf.d/%s", config_dir, EBPF_KERNEL_REJECT_LIST_FILE);
- kernel_reject_list = fopen(filename, "r");
-
- if (!kernel_reject_list)
- return 0;
- }
- }
-
- // Find if the kernel is in the reject list
- char *reject_string = NULL;
- size_t buf_len = 0;
- ssize_t reject_string_len;
- while ((reject_string_len = getline(&reject_string, &buf_len, kernel_reject_list) - 1) > 0) {
- if (version_string_len >= reject_string_len) {
- if (!strncmp(version_string, reject_string, reject_string_len)) {
- netdata_log_info("A buggy kernel is detected");
- fclose(kernel_reject_list);
- freez(reject_string);
- return 1;
- }
- }
- }
-
- fclose(kernel_reject_list);
- free(reject_string);
-
- return 0;
-}
-
-static int has_ebpf_kernel_version(int version)
-{
- if (kernel_is_rejected())
- return 0;
-
- // Kernel 4.11.0 or RH > 7.5
- return (version >= NETDATA_MINIMUM_EBPF_KERNEL || get_redhat_release() >= NETDATA_MINIMUM_RH_VERSION);
-}
-
-int has_condition_to_run(int version)
-{
- if (!has_ebpf_kernel_version(version))
- return 0;
-
- return 1;
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/**
- * Kernel Name
- *
- * Select kernel name used by eBPF programs
- *
- * Netdata delivers for users eBPF programs with specific suffixes that represent the kernels they were
- * compiled, when we load the eBPF program, the suffix must be the nereast possible of the kernel running.
- *
- * @param selector select the kernel version.
- *
- * @return It returns the string to load kernel.
- */
-static char *ebpf_select_kernel_name(uint32_t selector)
-{
- static char *kernel_names[] = { NETDATA_IDX_STR_V3_10, NETDATA_IDX_STR_V4_14, NETDATA_IDX_STR_V4_16,
- NETDATA_IDX_STR_V4_18, NETDATA_IDX_STR_V5_4, NETDATA_IDX_STR_V5_10,
- NETDATA_IDX_STR_V5_11, NETDATA_IDX_STR_V5_14, NETDATA_IDX_STR_V5_15,
- NETDATA_IDX_STR_V5_16
- };
-
- return kernel_names[selector];
-}
-
-/**
- * Select Max Index
- *
- * Select last index that will be tested on host.
- *
- * @param is_rhf is Red Hat fammily?
- * @param kver the kernel version
- *
- * @return it returns the index to access kernel string.
- */
-static int ebpf_select_max_index(int is_rhf, uint32_t kver)
-{
- if (is_rhf > 0) { // Is Red Hat family
- if (kver >= NETDATA_EBPF_KERNEL_5_14)
- return NETDATA_IDX_V5_14;
- else if (kver >= NETDATA_EBPF_KERNEL_5_4 && kver < NETDATA_EBPF_KERNEL_5_5) // For Oracle Linux
- return NETDATA_IDX_V5_4;
- else if (kver >= NETDATA_EBPF_KERNEL_4_11)
- return NETDATA_IDX_V4_18;
- } else { // Kernels from kernel.org
- if (kver >= NETDATA_EBPF_KERNEL_5_16)
- return NETDATA_IDX_V5_16;
- else if (kver >= NETDATA_EBPF_KERNEL_5_15)
- return NETDATA_IDX_V5_15;
- else if (kver >= NETDATA_EBPF_KERNEL_5_11)
- return NETDATA_IDX_V5_11;
- else if (kver >= NETDATA_EBPF_KERNEL_5_10)
- return NETDATA_IDX_V5_10;
- else if (kver >= NETDATA_EBPF_KERNEL_4_17)
- return NETDATA_IDX_V5_4;
- else if (kver >= NETDATA_EBPF_KERNEL_4_15)
- return NETDATA_IDX_V4_16;
- else if (kver >= NETDATA_EBPF_KERNEL_4_11)
- return NETDATA_IDX_V4_14;
- }
-
- return NETDATA_IDX_V3_10;
-}
-
-/**
- * Select Index
- *
- * Select index to load data.
- *
- * @param kernels is the variable with kernel versions.
- * @param is_rhf is Red Hat fammily?
- * param kver the kernel version
- */
-static uint32_t ebpf_select_index(uint32_t kernels, int is_rhf, uint32_t kver)
-{
- uint32_t start = ebpf_select_max_index(is_rhf, kver);
- uint32_t idx;
-
- if (is_rhf == -1)
- kernels &= ~NETDATA_V5_14;
-
- for (idx = start; idx; idx--) {
- if (kernels & 1 << idx)
- break;
- }
-
- return idx;
-}
-
-/**
- * Mount Name
- *
- * Mount name of eBPF program to be loaded.
- *
- * Netdata eBPF programs has the following format:
- *
- * Tnetdata_ebpf_N.V.o
- *
- * where:
- * T - Is the eBPF type. When starts with 'p', this means we are only adding probes,
- * and when they start with 'r' we are using retprobes.
- * N - The eBPF program name.
- * V - The kernel version in string format.
- *
- * @param out the vector where the name will be stored
- * @param len the size of the out vector.
- * @param path where the binaries are stored
- * @param kver the kernel version
- * @param name the eBPF program name.
- * @param is_return is return or entry ?
- */
-static void ebpf_mount_name(char *out, size_t len, char *path, uint32_t kver, const char *name,
- int is_return, int is_rhf)
-{
- char *version = ebpf_select_kernel_name(kver);
- snprintfz(out, len, "%s/ebpf.d/%cnetdata_ebpf_%s.%s%s.o",
- path,
- (is_return) ? 'r' : 'p',
- name,
- version,
- (is_rhf != -1) ? ".rhf" : "");
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/**
- * Statistics from targets
- *
- * Count the information from targets.
- *
- * @param report the output structure
- * @param targets vector with information about the eBPF plugin.
- * @param value factor used to update calculation
- */
-static void ebpf_stats_targets(ebpf_plugin_stats_t *report, netdata_ebpf_targets_t *targets, int value)
-{
- if (!targets) {
- report->probes = report->tracepoints = report->trampolines = 0;
- return;
- }
-
- int i = 0;
- while (targets[i].name) {
- switch (targets[i].mode) {
- case EBPF_LOAD_PROBE: {
- report->probes += value;
- break;
- }
- case EBPF_LOAD_RETPROBE: {
- report->retprobes += value;
- break;
- }
- case EBPF_LOAD_TRACEPOINT: {
- report->tracepoints += value;
- break;
- }
- case EBPF_LOAD_TRAMPOLINE: {
- report->trampolines += value;
- break;
- }
- }
-
- i++;
- }
-}
-
-/**
- * Update General stats
- *
- * Update eBPF plugin statistics that has relationship with the thread.
- *
- * This function must be called with mutex associated to charts is locked.
- *
- * @param report the output structure
- * @param em the structure with information about how the module/thread is working.
- */
-void ebpf_update_stats(ebpf_plugin_stats_t *report, ebpf_module_t *em)
-{
- int value;
-
- // It is not necessary to report more information.
- if (em->enabled > NETDATA_THREAD_EBPF_FUNCTION_RUNNING)
- value = -1;
- else
- value = 1;
-
- report->threads += value;
- report->running += value;
-
- // In theory the `else if` is useless, because when this function is called, the module should not stay in
- // EBPF_LOAD_PLAY_DICE. We have this additional condition to detect errors from developers.
- if (em->load & EBPF_LOAD_LEGACY)
- report->legacy += value;
- else if (em->load & EBPF_LOAD_CORE)
- report->core += value;
-
- if (em->maps_per_core)
- report->hash_percpu += value;
- else
- report->hash_unique += value;
-
- ebpf_stats_targets(report, em->targets, value);
-}
-
-/**
- * Update Kernel memory with memory
- *
- * This algorithm is an adaptation of https://elixir.bootlin.com/linux/v6.1.14/source/tools/bpf/bpftool/common.c#L402
- * to get 'memlock' data and update report.
- *
- * @param report the output structure
- * @param map pointer to a map.
- * @param action What action will be done with this map.
- */
-void ebpf_update_kernel_memory(ebpf_plugin_stats_t *report, ebpf_local_maps_t *map, ebpf_stats_action_t action)
-{
- char filename[FILENAME_MAX+1];
- snprintfz(filename, FILENAME_MAX, "/proc/self/fdinfo/%d", map->map_fd);
- procfile *ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT);
- if(unlikely(!ff)) {
- netdata_log_error("Cannot open %s", filename);
- return;
- }
-
- ff = procfile_readall(ff);
- if(unlikely(!ff))
- return;
-
- unsigned long j, lines = procfile_lines(ff);
- char *memlock = { "memlock" };
- for (j = 0; j < lines ; j++) {
- char *cmp = procfile_lineword(ff, j,0);
- if (!strncmp(memlock, cmp, 7)) {
- uint64_t memsize = (uint64_t) str2l(procfile_lineword(ff, j,1));
- switch (action) {
- case EBPF_ACTION_STAT_ADD: {
- report->memlock_kern += memsize;
- report->hash_tables += 1;
-#ifdef NETDATA_DEV_MODE
- netdata_log_info("Hash table %u: %s (FD = %d) is consuming %lu bytes totalizing %lu bytes",
- report->hash_tables, map->name, map->map_fd, memsize, report->memlock_kern);
-#endif
- break;
- }
- case EBPF_ACTION_STAT_REMOVE: {
- report->memlock_kern -= memsize;
- report->hash_tables -= 1;
-#ifdef NETDATA_DEV_MODE
- netdata_log_info("Hash table %s (FD = %d) was removed releasing %lu bytes, now we have %u tables loaded totalizing %lu bytes.",
- map->name, map->map_fd, memsize, report->hash_tables, report->memlock_kern);
-#endif
- break;
- }
- default: {
- break;
- }
- }
- break;
- }
- }
-
- procfile_close(ff);
-}
-
-/**
- * Update Kernel memory with memory
- *
- * This algorithm is an adaptation of https://elixir.bootlin.com/linux/v6.1.14/source/tools/bpf/bpftool/common.c#L402
- * to get 'memlock' data and update report.
- *
- * @param report the output structure
- * @param map pointer to a map. Last map must fish with name = NULL
- * @param action should plugin add or remove values from amount.
- */
-void ebpf_update_kernel_memory_with_vector(ebpf_plugin_stats_t *report,
- ebpf_local_maps_t *maps,
- ebpf_stats_action_t action)
-{
- if (!maps)
- return;
-
- ebpf_local_maps_t *map;
- int i = 0;
- for (map = &maps[i]; maps[i].name; i++, map = &maps[i]) {
- int fd = map->map_fd;
- if (fd == ND_EBPF_MAP_FD_NOT_INITIALIZED)
- continue;
-
- ebpf_update_kernel_memory(report, map, action);
- }
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-void ebpf_update_pid_table(ebpf_local_maps_t *pid, ebpf_module_t *em)
-{
- pid->user_input = em->pid_map_size;
-}
-
-/**
- * Update map size
- *
- * Update map size with information read from configuration files.
- *
- * @param map the structure with file descriptor to update.
- * @param lmap the structure with information from configuration files.
- * @param em the structure with information about how the module/thread is working.
- * @param map_name the name of the file used to log.
- */
-void ebpf_update_map_size(struct bpf_map *map, ebpf_local_maps_t *lmap, ebpf_module_t *em, const char *map_name __maybe_unused)
-{
- uint32_t define_size = 0;
- uint32_t apps_type = NETDATA_EBPF_MAP_PID | NETDATA_EBPF_MAP_RESIZABLE;
- if (lmap->user_input && lmap->user_input != lmap->internal_input) {
- define_size = lmap->internal_input;
-#ifdef NETDATA_INTERNAL_CHECKS
- netdata_log_info("Changing map %s from size %u to %u ", map_name, lmap->internal_input, lmap->user_input);
-#endif
- } else if (((lmap->type & apps_type) == apps_type) && (!em->apps_charts) && (!em->cgroup_charts)) {
- lmap->user_input = ND_EBPF_DEFAULT_MIN_PID;
- } else if (((em->apps_charts) || (em->cgroup_charts)) && (em->apps_level != NETDATA_APPS_NOT_SET)) {
- switch (em->apps_level) {
- case NETDATA_APPS_LEVEL_ALL: {
- define_size = lmap->user_input;
- break;
- }
- case NETDATA_APPS_LEVEL_PARENT: {
- define_size = ND_EBPF_DEFAULT_PID_SIZE / 2;
- break;
- }
- case NETDATA_APPS_LEVEL_REAL_PARENT:
- default: {
- define_size = ND_EBPF_DEFAULT_PID_SIZE / 3;
- }
- }
- }
-
- if (!define_size)
- return;
-
-#ifdef LIBBPF_MAJOR_VERSION
- bpf_map__set_max_entries(map, define_size);
-#else
- bpf_map__resize(map, define_size);
-#endif
-}
-
-#ifdef LIBBPF_MAJOR_VERSION
-/**
- * Update map type
- *
- * Update map type with information given.
- *
- * @param map the map we want to modify
- * @param w a structure with user input
- */
-void ebpf_update_map_type(struct bpf_map *map, ebpf_local_maps_t *w)
-{
- if (bpf_map__set_type(map, w->map_type)) {
- netdata_log_error("Cannot modify map type for %s", w->name);
- }
-}
-
-/**
- * Define map type
- *
- * This PR defines the type used by hash tables according user input.
- *
- * @param maps the list of maps used with a hash table.
- * @param maps_per_core define if map type according user specification.
- * @param kver kernel version host is running.
- */
-void ebpf_define_map_type(ebpf_local_maps_t *maps, int maps_per_core, int kver)
-{
- if (!maps)
- return;
-
- // Before kernel 4.06 there was not percpu hash tables
- if (kver < NETDATA_EBPF_KERNEL_4_06)
- maps_per_core = CONFIG_BOOLEAN_NO;
-
- int i = 0;
- while (maps[i].name) {
- ebpf_local_maps_t *map = &maps[i];
- // maps_per_core is a boolean value in configuration files.
- if (maps_per_core) {
- if (map->map_type == BPF_MAP_TYPE_HASH)
- map->map_type = BPF_MAP_TYPE_PERCPU_HASH;
- else if (map->map_type == BPF_MAP_TYPE_ARRAY)
- map->map_type = BPF_MAP_TYPE_PERCPU_ARRAY;
- } else {
- if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH)
- map->map_type = BPF_MAP_TYPE_HASH;
- else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
- map->map_type = BPF_MAP_TYPE_ARRAY;
- }
-
- i++;
- }
-}
-#endif
-
-/**
- * Update Legacy map
- *
- * Update map for eBPF legacy code.
- *
- * @param program the structure with values read from binary.
- * @param em the structure with information about how the module/thread is working.
- */
-static void ebpf_update_legacy_map(struct bpf_object *program, ebpf_module_t *em)
-{
- struct bpf_map *map;
- ebpf_local_maps_t *maps = em->maps;
- if (!maps)
- return;
-
- bpf_map__for_each(map, program)
- {
- const char *map_name = bpf_map__name(map);
- int i = 0;
- while (maps[i].name) {
- ebpf_local_maps_t *w = &maps[i];
-
- if (!strcmp(w->name, map_name)) {
- // Modify size
- if (w->type & NETDATA_EBPF_MAP_RESIZABLE) {
- ebpf_update_map_size(map, w, em, map_name);
- }
-
-#ifdef LIBBPF_MAJOR_VERSION
- ebpf_update_map_type(map, w);
-#endif
- }
-
- i++;
- }
- }
-}
-
-size_t ebpf_count_programs(struct bpf_object *obj)
-{
- size_t tot = 0;
- struct bpf_program *prog;
- bpf_object__for_each_program(prog, obj)
- {
- tot++;
- }
-
- return tot;
-}
-
-static ebpf_specify_name_t *ebpf_find_names(ebpf_specify_name_t *names, const char *prog_name)
-{
- size_t i = 0;
- while (names[i].program_name) {
- if (!strcmp(prog_name, names[i].program_name))
- return &names[i];
-
- i++;
- }
-
- return NULL;
-}
-
-static struct bpf_link **ebpf_attach_programs(struct bpf_object *obj, size_t length, ebpf_specify_name_t *names)
-{
- struct bpf_link **links = callocz(length , sizeof(struct bpf_link *));
- size_t i = 0;
- struct bpf_program *prog;
- ebpf_specify_name_t *w;
- bpf_object__for_each_program(prog, obj)
- {
- if (names) {
- const char *name = bpf_program__name(prog);
- w = ebpf_find_names(names, name);
- } else
- w = NULL;
-
- if (w) {
- enum bpf_prog_type type = bpf_program__get_type(prog);
- if (type == BPF_PROG_TYPE_KPROBE)
- links[i] = bpf_program__attach_kprobe(prog, w->retprobe, w->optional);
- } else
- links[i] = bpf_program__attach(prog);
-
- if (libbpf_get_error(links[i])) {
- links[i] = NULL;
- }
-
- i++;
- }
-
- return links;
-}
-
-static void ebpf_update_maps(ebpf_module_t *em, struct bpf_object *obj)
-{
- if (!em->maps)
- return;
-
- ebpf_local_maps_t *maps = em->maps;
- struct bpf_map *map;
- bpf_map__for_each(map, obj)
- {
- int fd = bpf_map__fd(map);
- if (maps) {
- const char *map_name = bpf_map__name(map);
- int j = 0; ;
- while (maps[j].name) {
- ebpf_local_maps_t *w = &maps[j];
- if (w->map_fd == ND_EBPF_MAP_FD_NOT_INITIALIZED && !strcmp(map_name, w->name))
- w->map_fd = fd;
-
- j++;
- }
- }
- }
-}
-
-/**
- * Update Controller
- *
- * Update controller value with user input.
- *
- * @param fd the table file descriptor
- * @param em structure with information about eBPF program we will load.
- */
-void ebpf_update_controller(int fd, ebpf_module_t *em)
-{
- uint32_t values[NETDATA_CONTROLLER_END] = {
- (em->apps_charts & NETDATA_EBPF_APPS_FLAG_YES) | em->cgroup_charts,
- em->apps_level, 0, 0, 0, 0
- };
- uint32_t key;
- uint32_t end = NETDATA_CONTROLLER_PID_TABLE_ADD;
-
- for (key = NETDATA_CONTROLLER_APPS_ENABLED; key < end; key++) {
- int ret = bpf_map_update_elem(fd, &key, &values[key], BPF_ANY);
- if (ret)
- netdata_log_error("Add key(%u) for controller table failed.", key);
- }
-}
-
-/**
- * Update Legacy controller
- *
- * Update legacy controller table when eBPF program has it.
- *
- * @param em structure with information about eBPF program we will load.
- * @param obj bpf object with tables.
- */
-static void ebpf_update_legacy_controller(ebpf_module_t *em, struct bpf_object *obj)
-{
- ebpf_local_maps_t *maps = em->maps;
- if (!maps)
- return;
-
- struct bpf_map *map;
- bpf_map__for_each(map, obj)
- {
- size_t i = 0;
- while (maps[i].name) {
- ebpf_local_maps_t *w = &maps[i];
- if (w->map_fd != ND_EBPF_MAP_FD_NOT_INITIALIZED && (w->type & NETDATA_EBPF_MAP_CONTROLLER)) {
- w->type &= ~NETDATA_EBPF_MAP_CONTROLLER;
- w->type |= NETDATA_EBPF_MAP_CONTROLLER_UPDATED;
-
- ebpf_update_controller(w->map_fd, em);
- }
- i++;
- }
- }
-}
-
-/**
- * Load Program
- *
- * Load eBPF program into kernel
- *
- * @param plugins_dir directory where binary are stored
- * @param em structure with information about eBPF program we will load.
- * @param kver the kernel version according /usr/include/linux/version.h
- * @param is_rhf is a kernel from Red Hat Family?
- * @param obj structure where we will store object loaded.
- *
- * @return it returns a link for each target we associated an eBPF program.
- */
-struct bpf_link **ebpf_load_program(char *plugins_dir, ebpf_module_t *em, int kver, int is_rhf,
- struct bpf_object **obj)
-{
- char lpath[4096];
-
- uint32_t idx = ebpf_select_index(em->kernels, is_rhf, kver);
-
- ebpf_mount_name(lpath, 4095, plugins_dir, idx, em->info.thread_name, em->mode, is_rhf);
-
- // When this function is called ebpf.plugin is using legacy code, so we should reset the variable
- em->load &= ~ NETDATA_EBPF_LOAD_METHODS;
- em->load |= EBPF_LOAD_LEGACY;
-
- *obj = bpf_object__open_file(lpath, NULL);
- if (!*obj)
- return NULL;
-
- if (libbpf_get_error(obj)) {
- bpf_object__close(*obj);
- return NULL;
- }
-
- ebpf_update_legacy_map(*obj, em);
-
- if (bpf_object__load(*obj)) {
- netdata_log_error("ERROR: loading BPF object file failed %s\n", lpath);
- bpf_object__close(*obj);
- return NULL;
- }
-
- ebpf_update_maps(em, *obj);
- ebpf_update_legacy_controller(em, *obj);
-
- size_t count_programs = ebpf_count_programs(*obj);
-
-#ifdef NETDATA_INTERNAL_CHECKS
- netdata_log_info("eBPF program %s loaded with success!", lpath);
-#endif
-
- return ebpf_attach_programs(*obj, count_programs, em->names);
-}
-
-char *ebpf_find_symbol(char *search)
-{
- char filename[FILENAME_MAX + 1];
- char *ret = NULL;
- snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, NETDATA_KALLSYMS);
- procfile *ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT);
- if(unlikely(!ff)) {
- netdata_log_error("Cannot open %s%s", netdata_configured_host_prefix, NETDATA_KALLSYMS);
- return ret;
- }
-
- ff = procfile_readall(ff);
- if(unlikely(!ff))
- return ret;
-
- unsigned long i, lines = procfile_lines(ff);
- size_t length = strlen(search);
- for(i = 0; i < lines ; i++) {
- char *cmp = procfile_lineword(ff, i,2);;
- if (!strncmp(search, cmp, length)) {
- ret = strdupz(cmp);
- break;
- }
- }
-
- procfile_close(ff);
-
- return ret;
-}
-
-void ebpf_update_names(ebpf_specify_name_t *opt, ebpf_module_t *em)
-{
- int mode = em->mode;
- em->names = opt;
-
- size_t i = 0;
- while (opt[i].program_name) {
- opt[i].retprobe = (mode == MODE_RETURN);
- opt[i].optional = ebpf_find_symbol(opt[i].function_to_attach);
-
- i++;
- }
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-void ebpf_mount_config_name(char *filename, size_t length, char *path, const char *config)
-{
- snprintf(filename, length, "%s/ebpf.d/%s", path, config);
-}
-
-int ebpf_load_config(struct config *config, char *filename)
-{
- return appconfig_load(config, filename, 0, NULL);
-}
-
-
-static netdata_run_mode_t ebpf_select_mode(char *mode)
-{
- if (!strcasecmp(mode,EBPF_CFG_LOAD_MODE_RETURN ))
- return MODE_RETURN;
- else if (!strcasecmp(mode, "dev"))
- return MODE_DEVMODE;
-
- return MODE_ENTRY;
-}
-
-static void ebpf_select_mode_string(char *output, size_t len, netdata_run_mode_t sel)
-{
- if (sel == MODE_RETURN)
- strncpyz(output, EBPF_CFG_LOAD_MODE_RETURN, len);
- else
- strncpyz(output, EBPF_CFG_LOAD_MODE_DEFAULT, len);
-}
-
-/**
- * Convert string to load mode
- *
- * Convert the string given as argument to value present in enum.
- *
- * @param str value read from configuration file.
- *
- * @return It returns the value to be used.
- */
-netdata_ebpf_load_mode_t epbf_convert_string_to_load_mode(char *str)
-{
- if (!strcasecmp(str, EBPF_CFG_CORE_PROGRAM))
- return EBPF_LOAD_CORE;
- else if (!strcasecmp(str, EBPF_CFG_LEGACY_PROGRAM))
- return EBPF_LOAD_LEGACY;
-
- return EBPF_LOAD_PLAY_DICE;
-}
-
-/**
- * Convert load mode to string
- *
- * @param mode value that will select the string
- *
- * @return It returns the string associated to mode.
- */
-static char *ebpf_convert_load_mode_to_string(netdata_ebpf_load_mode_t mode)
-{
- if (mode & EBPF_LOAD_CORE)
- return EBPF_CFG_CORE_PROGRAM;
- else if (mode & EBPF_LOAD_LEGACY)
- return EBPF_CFG_LEGACY_PROGRAM;
-
- return EBPF_CFG_DEFAULT_PROGRAM;
-}
-
-/**
- * Convert collect pid to string
- *
- * @param level value that will select the string
- *
- * @return It returns the string associated to level.
- */
-static char *ebpf_convert_collect_pid_to_string(netdata_apps_level_t level)
-{
- if (level == NETDATA_APPS_LEVEL_REAL_PARENT)
- return EBPF_CFG_PID_REAL_PARENT;
- else if (level == NETDATA_APPS_LEVEL_PARENT)
- return EBPF_CFG_PID_PARENT;
- else if (level == NETDATA_APPS_LEVEL_ALL)
- return EBPF_CFG_PID_ALL;
-
- return EBPF_CFG_PID_INTERNAL_USAGE;
-}
-
-/**
- * Convert string to apps level
- *
- * @param str the argument read from config files
- *
- * @return it returns the level associated to the string or default when it is a wrong value
- */
-netdata_apps_level_t ebpf_convert_string_to_apps_level(char *str)
-{
- if (!strcasecmp(str, EBPF_CFG_PID_REAL_PARENT))
- return NETDATA_APPS_LEVEL_REAL_PARENT;
- else if (!strcasecmp(str, EBPF_CFG_PID_PARENT))
- return NETDATA_APPS_LEVEL_PARENT;
- else if (!strcasecmp(str, EBPF_CFG_PID_ALL))
- return NETDATA_APPS_LEVEL_ALL;
-
- return NETDATA_APPS_NOT_SET;
-}
-
-/**
- * CO-RE type
- *
- * Select the preferential type of CO-RE
- *
- * @param str value read from configuration file.
- * @param lmode load mode used by collector.
- */
-netdata_ebpf_program_loaded_t ebpf_convert_core_type(char *str, netdata_run_mode_t lmode)
-{
- if (!strcasecmp(str, EBPF_CFG_ATTACH_TRACEPOINT))
- return EBPF_LOAD_TRACEPOINT;
- else if (!strcasecmp(str, EBPF_CFG_ATTACH_PROBE)) {
- return (lmode == MODE_ENTRY) ? EBPF_LOAD_PROBE : EBPF_LOAD_RETPROBE;
- }
-
- return EBPF_LOAD_TRAMPOLINE;
-}
-
-#ifdef LIBBPF_MAJOR_VERSION
-/**
- * Adjust Thread Load
- *
- * Adjust thread configuration according specified load.
- *
- * @param mod the main structure that will be adjusted.
- * @param file the btf file used with thread.
- */
-void ebpf_adjust_thread_load(ebpf_module_t *mod, struct btf *file)
-{
- if (!file) {
- mod->load &= ~EBPF_LOAD_CORE;
- mod->load |= EBPF_LOAD_LEGACY;
- } else if (mod->load == EBPF_LOAD_PLAY_DICE && file) {
- mod->load &= ~EBPF_LOAD_LEGACY;
- mod->load |= EBPF_LOAD_CORE;
- }
-}
-
-/**
- * Parse BTF file
- *
- * Parse a specific BTF file present on filesystem
- *
- * @param filename the file that will be parsed.
- *
- * @return It returns a pointer for the file on success and NULL otherwise.
- */
-struct btf *ebpf_parse_btf_file(const char *filename)
-{
- struct btf *bf = btf__parse(filename, NULL);
- if (libbpf_get_error(bf)) {
- fprintf(stderr, "Cannot parse btf file");
- btf__free(bf);
- return NULL;
- }
-
- return bf;
-}
-
-/**
- * Load default btf file
- *
- * Load the default BTF file on environment.
- *
- * @param path is the fullpath
- * @param filename is the file inside BTF path.
- */
-struct btf *ebpf_load_btf_file(char *path, char *filename)
-{
- char fullpath[PATH_MAX + 1];
- snprintfz(fullpath, PATH_MAX, "%s/%s", path, filename);
- struct btf *ret = ebpf_parse_btf_file(fullpath);
- if (!ret)
- netdata_log_info("Your environment does not have BTF file %s/%s. The plugin will work with 'legacy' code.",
- path, filename);
-
- return ret;
-}
-
-/**
- * Find BTF attach type
- *
- * Search type fr current btf file.
- *
- * @param file is the structure for the btf file already parsed.
- */
-static inline const struct btf_type *ebpf_find_btf_attach_type(struct btf *file)
-{
- int id = btf__find_by_name_kind(file, "bpf_attach_type", BTF_KIND_ENUM);
- if (id < 0) {
- fprintf(stderr, "Cannot find 'bpf_attach_type'");
-
- return NULL;
- }
-
- return btf__type_by_id(file, id);
-}
-
-/**
- * Is function inside BTF
- *
- * Look for a specific function inside the given BTF file.
- *
- * @param file is the structure for the btf file already parsed.
- * @param function is the function that we want to find.
- */
-int ebpf_is_function_inside_btf(struct btf *file, char *function)
-{
- const struct btf_type *type = ebpf_find_btf_attach_type(file);
- if (!type)
- return -1;
-
- const struct btf_enum *e = btf_enum(type);
- int i, id;
- for (id = -1, i = 0; i < btf_vlen(type); i++, e++) {
- if (!strcmp(btf__name_by_offset(file, e->name_off), "BPF_TRACE_FENTRY")) {
- id = btf__find_by_name_kind(file, function, BTF_KIND_FUNC);
- break;
- }
- }
-
- return (id > 0) ? 1 : 0;
-}
-#endif
-
-/**
- * Update target with configuration
- *
- * Update target load mode with value.
- *
- * @param em the module structure
- * @param value value used to update.
- */
-static void ebpf_update_target_with_conf(ebpf_module_t *em, netdata_ebpf_program_loaded_t value)
-{
- netdata_ebpf_targets_t *targets = em->targets;
- if (!targets) {
- return;
- }
-
- int i = 0;
- while (targets[i].name) {
- targets[i].mode = value;
- i++;
- }
-}
-
-/**
- * Select Load Mode
- *
- * Select the load mode according the given inputs.
- *
- * @param btf_file a pointer to the loaded btf file.
- * @parma load current value.
- * @param btf_file a pointer to the loaded btf file.
- * @param is_rhf is Red Hat family?
- *
- * @return it returns the new load mode.
- */
-static netdata_ebpf_load_mode_t ebpf_select_load_mode(struct btf *btf_file, netdata_ebpf_load_mode_t load,
- int kver, int is_rh)
-{
-#ifdef LIBBPF_MAJOR_VERSION
- if ((load & EBPF_LOAD_CORE) || (load & EBPF_LOAD_PLAY_DICE)) {
- // Quick fix for Oracle linux 8.x
- load = (!btf_file || (is_rh && (kver >= NETDATA_EBPF_KERNEL_5_4 && kver < NETDATA_EBPF_KERNEL_5_5))) ?
- EBPF_LOAD_LEGACY : EBPF_LOAD_CORE;
- }
-#else
- load = EBPF_LOAD_LEGACY;
-#endif
-
- return load;
-}
-
-/**
- * Update Module using config
- *
- * Update configuration for a specific thread.
- *
- * @param modules structure that will be updated
- * @param origin specify the configuration file loaded
- * @param btf_file a pointer to the loaded btf file.
- * @param is_rhf is Red Hat family?
- */
-void ebpf_update_module_using_config(ebpf_module_t *modules, netdata_ebpf_load_mode_t origin, struct btf *btf_file,
- int kver, int is_rh)
-{
- char default_value[EBPF_MAX_MODE_LENGTH + 1];
- ebpf_select_mode_string(default_value, EBPF_MAX_MODE_LENGTH, modules->mode);
- char *load_mode = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_LOAD_MODE, default_value);
- modules->mode = ebpf_select_mode(load_mode);
-
- modules->update_every = (int)appconfig_get_number(modules->cfg, EBPF_GLOBAL_SECTION,
- EBPF_CFG_UPDATE_EVERY, modules->update_every);
-
- modules->apps_charts = appconfig_get_boolean(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_APPLICATION,
- (int) (modules->apps_charts & NETDATA_EBPF_APPS_FLAG_YES));
-
- modules->cgroup_charts = appconfig_get_boolean(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_CGROUP,
- modules->cgroup_charts);
-
- modules->pid_map_size = (uint32_t)appconfig_get_number(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_PID_SIZE,
- modules->pid_map_size);
-
- modules->lifetime = (uint32_t) appconfig_get_number(modules->cfg, EBPF_GLOBAL_SECTION,
- EBPF_CFG_LIFETIME, EBPF_DEFAULT_LIFETIME);
-
- char *value = ebpf_convert_load_mode_to_string(modules->load & NETDATA_EBPF_LOAD_METHODS);
- char *type_format = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_TYPE_FORMAT, value);
- netdata_ebpf_load_mode_t load = epbf_convert_string_to_load_mode(type_format);
- load = ebpf_select_load_mode(btf_file, load, kver, is_rh);
- modules->load = origin | load;
-
- char *core_attach = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_CORE_ATTACH, EBPF_CFG_ATTACH_TRAMPOLINE);
- netdata_ebpf_program_loaded_t fill_lm = ebpf_convert_core_type(core_attach, modules->mode);
- ebpf_update_target_with_conf(modules, fill_lm);
-
- value = ebpf_convert_collect_pid_to_string(modules->apps_level);
- char *collect_pid = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_COLLECT_PID, value);
- modules->apps_level = ebpf_convert_string_to_apps_level(collect_pid);
-
- modules->maps_per_core = appconfig_get_boolean(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_MAPS_PER_CORE,
- modules->maps_per_core);
- if (kver < NETDATA_EBPF_KERNEL_4_06)
- modules->maps_per_core = CONFIG_BOOLEAN_NO;
-
-#ifdef NETDATA_DEV_MODE
- netdata_log_info("The thread %s was configured with: mode = %s; update every = %d; apps = %s; cgroup = %s; ebpf type format = %s; ebpf co-re tracing = %s; collect pid = %s; maps per core = %s, lifetime=%u",
- modules->info.thread_name,
- load_mode,
- modules->update_every,
- (modules->apps_charts)?"enabled":"disabled",
- (modules->cgroup_charts)?"enabled":"disabled",
- type_format,
- core_attach,
- collect_pid,
- (modules->maps_per_core)?"enabled":"disabled",
- modules->lifetime
- );
-#endif
-}
-
-/**
- * Update module
- *
- * When this function is called, it will load the configuration file and after this
- * it updates the global information of ebpf_module.
- * If the module has specific configuration, this function will load it, but it will not
- * update the variables.
- *
- * @param em the module structure
- * @param btf_file a pointer to the loaded btf file.
- * @param is_rhf is Red Hat family?
- * @param kver the kernel version
- */
-void ebpf_update_module(ebpf_module_t *em, struct btf *btf_file, int kver, int is_rh)
-{
- char filename[FILENAME_MAX+1];
- netdata_ebpf_load_mode_t origin;
-
- ebpf_mount_config_name(filename, FILENAME_MAX, ebpf_user_config_dir, em->config_file);
- if (!ebpf_load_config(em->cfg, filename)) {
- ebpf_mount_config_name(filename, FILENAME_MAX, ebpf_stock_config_dir, em->config_file);
- if (!ebpf_load_config(em->cfg, filename)) {
- netdata_log_error("Cannot load the ebpf configuration file %s", em->config_file);
- return;
- }
- // If user defined data globally, we will have here EBPF_LOADED_FROM_USER, we need to consider this, to avoid
- // forcing users to configure thread by thread.
- origin = (!(em->load & NETDATA_EBPF_LOAD_SOURCE)) ? EBPF_LOADED_FROM_STOCK : em->load & NETDATA_EBPF_LOAD_SOURCE;
- } else
- origin = EBPF_LOADED_FROM_USER;
-
- ebpf_update_module_using_config(em, origin, btf_file, kver, is_rh);
-}
-
-/**
- * Adjust Apps Cgroup
- *
- * Apps and cgroup has internal cleanup that needs attaching tracers to release_task, to avoid overload the function
- * we will enable this integration by default, if and only if, we are running with trampolines.
- *
- * @param em a pointer to the main thread structure.
- * @param mode is the mode used with different
- */
-void ebpf_adjust_apps_cgroup(ebpf_module_t *em, netdata_ebpf_program_loaded_t mode)
-{
- if ((em->load & EBPF_LOADED_FROM_STOCK) &&
- (em->apps_charts || em->cgroup_charts) &&
- mode != EBPF_LOAD_TRAMPOLINE) {
- em->apps_charts = NETDATA_EBPF_APPS_FLAG_NO;
- em->cgroup_charts = 0;
- }
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/**
- * Load Address
- *
- * Helper used to get address from /proc/kallsym
- *
- * @param fa address structure
- * @param fd file descriptor loaded inside kernel. If a negative value is given
- * the function will load address and it won't update hash table.
- */
-void ebpf_load_addresses(ebpf_addresses_t *fa, int fd)
-{
- if (fa->addr)
- return ;
-
- procfile *ff = procfile_open("/proc/kallsyms", " \t:", PROCFILE_FLAG_DEFAULT);
- if (!ff)
- return;
-
- ff = procfile_readall(ff);
- if (!ff)
- return;
-
- fa->hash = simple_hash(fa->function);
-
- size_t lines = procfile_lines(ff), l;
- for(l = 0; l < lines ;l++) {
- char *fcnt = procfile_lineword(ff, l, 2);
- uint32_t hash = simple_hash(fcnt);
- if (fa->hash == hash && !strcmp(fcnt, fa->function)) {
- char *type = procfile_lineword(ff, l, 2);
- fa->type = type[0];
- if (fd > 0) {
- char addr[128];
- snprintf(addr, 127, "0x%s", procfile_lineword(ff, l, 0));
- fa->addr = (unsigned long) strtoul(addr, NULL, 16);
- uint32_t key = 0;
- bpf_map_update_elem(fd, &key, &fa->addr, BPF_ANY);
- } else
- fa->addr = 1;
- break;
- }
- }
-
- procfile_close(ff);
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/**
- * Fill Algorithms
- *
- * Set one unique dimension for all vector position.
- *
- * @param algorithms the output vector
- * @param length number of elements of algorithms vector
- * @param algorithm algorithm used on charts.
-*/
-void ebpf_fill_algorithms(int *algorithms, size_t length, int algorithm)
-{
- size_t i;
- for (i = 0; i < length; i++) {
- algorithms[i] = algorithm;
- }
-}
-
-/**
- * Fill Histogram dimension
- *
- * Fill the histogram dimension with the specified ranges
- */
-char **ebpf_fill_histogram_dimension(size_t maximum)
-{
- char *dimensions[] = { "us", "ms", "s"};
- int previous_dim = 0, current_dim = 0;
- uint32_t previous_level = 1000, current_level = 1000;
- uint32_t previous_divisor = 1, current_divisor = 1;
- uint32_t current = 1, previous = 0;
- uint32_t selector;
- char **out = callocz(maximum, sizeof(char *));
- char range[128];
- size_t end = maximum - 1;
- for (selector = 0; selector < end; selector++) {
- snprintf(range, 127, "%u%s->%u%s", previous/previous_divisor, dimensions[previous_dim],
- current/current_divisor, dimensions[current_dim]);
- out[selector] = strdupz(range);
- previous = current;
- current <<= 1;
-
- if (previous_dim != 2 && previous > previous_level) {
- previous_dim++;
-
- previous_divisor *= 1000;
- previous_level *= 1000;
- }
-
- if (current_dim != 2 && current > current_level) {
- current_dim++;
-
- current_divisor *= 1000;
- current_level *= 1000;
- }
- }
- snprintf(range, 127, "%u%s->+Inf", previous/previous_divisor, dimensions[previous_dim]);
- out[selector] = strdupz(range);
-
- return out;
-}
-
-/**
- * Histogram dimension cleanup
- *
- * Cleanup dimensions allocated with function ebpf_fill_histogram_dimension
- *
- * @param ptr
- * @param length
- */
-void ebpf_histogram_dimension_cleanup(char **ptr, size_t length)
-{
- size_t i;
- for (i = 0; i < length; i++) {
- freez(ptr[i]);
- }
- freez(ptr);
-}
-
-//----------------------------------------------------------------------------------------------------------------------
-
-/**
- * Open tracepoint path
- *
- * @param filename pointer to store the path
- * @param length file length
- * @param subsys is the name of your subsystem.
- * @param eventname is the name of the event to trace.
- * @param flags flags used with syscall open
- *
- * @return it returns a positive value on success and a negative otherwise.
- */
-static inline int ebpf_open_tracepoint_path(char *filename, size_t length, char *subsys, char *eventname, int flags)
-{
- snprintfz(filename, length, "%s/events/%s/%s/enable", NETDATA_DEBUGFS, subsys, eventname);
- return open(filename, flags, 0);
-}
-
-/**
- * Is tracepoint enabled
- *
- * Check whether the tracepoint is enabled.
- *
- * @param subsys is the name of your subsystem.
- * @param eventname is the name of the event to trace.
- *
- * @return it returns 1 when it is enabled, 0 when it is disabled and -1 on error.
- */
-int ebpf_is_tracepoint_enabled(char *subsys, char *eventname)
-{
- char text[FILENAME_MAX + 1];
- int fd = ebpf_open_tracepoint_path(text, FILENAME_MAX, subsys, eventname, O_RDONLY);
- if (fd < 0) {
- return -1;
- }
-
- ssize_t length = read(fd, text, 1);
- if (length != 1) {
- close(fd);
- return -1;
- }
- close(fd);
-
- return (text[0] == '1') ? CONFIG_BOOLEAN_YES : CONFIG_BOOLEAN_NO;
-}
-
-/**
- * Change Tracing values
- *
- * Change value for specific tracepoint enabling or disabling it according value given.
- *
- * @param subsys is the name of your subsystem.
- * @param eventname is the name of the event to trace.
- * @param value a value to enable (1) or disable (0) a tracepoint.
- *
- * @return It returns 0 on success and -1 otherwise
- */
-static int ebpf_change_tracing_values(char *subsys, char *eventname, char *value)
-{
- if (strcmp("0", value) && strcmp("1", value)) {
- netdata_log_error("Invalid value given to either enable or disable a tracepoint.");
- return -1;
- }
-
- char filename[1024];
- int fd = ebpf_open_tracepoint_path(filename, 1023, subsys, eventname, O_WRONLY);
- if (fd < 0) {
- return -1;
- }
-
- ssize_t written = write(fd, value, strlen(value));
- if (written < 0) {
- close(fd);
- return -1;
- }
-
- close(fd);
- return 0;
-}
-
-/**
- * Enable tracing values
- *
- * Enable a tracepoint on a system
- *
- * @param subsys is the name of your subsystem.
- * @param eventname is the name of the event to trace.
- *
- * @return It returns 0 on success and -1 otherwise
- */
-int ebpf_enable_tracing_values(char *subsys, char *eventname)
-{
- return ebpf_change_tracing_values(subsys, eventname, "1");
-}
-
-/**
- * Disable tracing values
- *
- * Disable tracing points enabled by collector
- *
- * @param subsys is the name of your subsystem.
- * @param eventname is the name of the event to trace.
- *
- * @return It returns 0 on success and -1 otherwise
- */
-int ebpf_disable_tracing_values(char *subsys, char *eventname)
-{
- return ebpf_change_tracing_values(subsys, eventname, "0");
-}
-
-/**
- * Select PC prefix
- *
- * Identify the prefix to run on PC architecture.
- *
- * @return It returns 32 or 64 according to host arch.
- */
-static uint32_t ebpf_select_pc_prefix()
-{
- long counter = 1;
- uint32_t i;
- for (i = 0; i < 128; i++) {
- counter <<= 1;
- if (counter < 0)
- break;
- }
-
- return counter;
-}
-
-/**
- * Select Host Prefix
- *
- * Select prefix to syscall when host is running a kernel newer than 4.17.0
- *
- * @param output the vector to store data.
- * @param length length of output vector.
- * @param syscall the syscall that prefix will be attached;
- * @param kver the current kernel version in format MAJOR*65536 + MINOR*256 + PATCH
- */
-void ebpf_select_host_prefix(char *output, size_t length, char *syscall, int kver)
-{
- if (kver < NETDATA_EBPF_KERNEL_4_17)
- snprintfz(output, length, "sys_%s", syscall);
- else {
- uint32_t arch = ebpf_select_pc_prefix();
- // Prefix selected according https://www.kernel.org/doc/html/latest/process/adding-syscalls.html
- char *prefix = (arch == 32) ? "__ia32" : "__x64";
- snprintfz(output, length, "%s_sys_%s", prefix, syscall);
- }
-}
-
diff --git a/libnetdata/ebpf/ebpf.h b/libnetdata/ebpf/ebpf.h
deleted file mode 100644
index 9218cb9d7..000000000
--- a/libnetdata/ebpf/ebpf.h
+++ /dev/null
@@ -1,487 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_EBPF_H
-#define NETDATA_EBPF_H 1
-
-#include <bpf/bpf.h>
-#include <bpf/libbpf.h>
-#ifdef LIBBPF_DEPRECATED
-#include <bpf/btf.h>
-#include <linux/btf.h>
-#endif
-#include <stdlib.h> // Necessary for stdtoul
-#include "libnetdata/aral/aral.h"
-
-#define NETDATA_DEBUGFS "/sys/kernel/debug/tracing/"
-#define NETDATA_KALLSYMS "/proc/kallsyms"
-
-// Config files
-#define EBPF_GLOBAL_SECTION "global"
-#define EBPF_CFG_LOAD_MODE "ebpf load mode"
-#define EBPF_CFG_LOAD_MODE_DEFAULT "entry"
-#define EBPF_CFG_LOAD_MODE_RETURN "return"
-#define EBPF_MAX_MODE_LENGTH 6
-
-#define EBPF_CFG_TYPE_FORMAT "ebpf type format"
-#define EBPF_CFG_DEFAULT_PROGRAM "auto"
-#define EBPF_CFG_CORE_PROGRAM "CO-RE"
-#define EBPF_CFG_LEGACY_PROGRAM "legacy"
-
-#define EBPF_CFG_COLLECT_PID "collect pid"
-#define EBPF_CFG_PID_REAL_PARENT "real parent"
-#define EBPF_CFG_PID_PARENT "parent"
-#define EBPF_CFG_PID_ALL "all"
-#define EBPF_CFG_PID_INTERNAL_USAGE "not used"
-
-#define EBPF_CFG_CORE_ATTACH "ebpf co-re tracing"
-#define EBPF_CFG_ATTACH_TRAMPOLINE "trampoline"
-#define EBPF_CFG_ATTACH_TRACEPOINT "tracepoint"
-#define EBPF_CFG_ATTACH_PROBE "probe"
-
-#define EBPF_CFG_PROGRAM_PATH "btf path"
-
-#define EBPF_CFG_MAPS_PER_CORE "maps per core"
-
-#define EBPF_CFG_UPDATE_EVERY "update every"
-#define EBPF_CFG_LIFETIME "lifetime"
-#define EBPF_CFG_UPDATE_APPS_EVERY_DEFAULT 10
-#define EBPF_CFG_PID_SIZE "pid table size"
-#define EBPF_CFG_APPLICATION "apps"
-#define EBPF_CFG_CGROUP "cgroups"
-
-#define EBPF_COMMON_FNCT_CLEAN_UP "release_task"
-
-/**
- * The RedHat magic number was got doing:
- *
- * 1797 = 7*256 + 5
- *
- * For more details, please, read /usr/include/linux/version.h
- * in any Red Hat installation.
- */
-#define NETDATA_MINIMUM_RH_VERSION 1797
-
-/**
- * 2048 = 8*256 + 0
- */
-#define NETDATA_RH_8 2048
-
-/**
- * Kernel Version
- *
- * Kernel versions are calculated using the following formula:
- *
- * VERSION = LINUX_VERSION_MAJOR*65536 + LINUX_VERSION_PATCHLEVEL*256 + LINUX_VERSION_SUBLEVEL
- *
- * Where LINUX_VERSION_MAJOR, LINUX_VERSION_PATCHLEVEL, and LINUX_VERSION_SUBLEVEL are extracted
- * from /usr/include/linux/version.h.
- *
- * LINUX_VERSION_SUBLEVEL has the maximum value 255, but linux can have more SUBLEVELS.
- *
- */
-enum netdata_ebpf_kernel_versions {
- NETDATA_EBPF_KERNEL_4_06 = 263680, // 264960 = 4 * 65536 + 6 * 256
- NETDATA_EBPF_KERNEL_4_11 = 264960, // 264960 = 4 * 65536 + 15 * 256
- NETDATA_EBPF_KERNEL_4_14 = 265728, // 264960 = 4 * 65536 + 14 * 256
- NETDATA_EBPF_KERNEL_4_15 = 265984, // 265984 = 4 * 65536 + 15 * 256
- NETDATA_EBPF_KERNEL_4_17 = 266496, // 266496 = 4 * 65536 + 17 * 256
- NETDATA_EBPF_KERNEL_5_0 = 327680, // 327680 = 5 * 65536 + 0 * 256
- NETDATA_EBPF_KERNEL_5_4 = 328704, // 327680 = 5 * 65536 + 4 * 256
- NETDATA_EBPF_KERNEL_5_5 = 328960, // 327680 = 5 * 65536 + 5 * 256
- NETDATA_EBPF_KERNEL_5_10 = 330240, // 330240 = 5 * 65536 + 10 * 256
- NETDATA_EBPF_KERNEL_5_11 = 330496, // 330240 = 5 * 65536 + 11 * 256
- NETDATA_EBPF_KERNEL_5_14 = 331264, // 331264 = 5 * 65536 + 14 * 256
- NETDATA_EBPF_KERNEL_5_15 = 331520, // 331520 = 5 * 65536 + 15 * 256
- NETDATA_EBPF_KERNEL_5_16 = 331776 // 331776 = 5 * 65536 + 16 * 256
-};
-
-enum netdata_kernel_flag {
- NETDATA_V3_10 = 1 << 0,
- NETDATA_V4_14 = 1 << 1,
- NETDATA_V4_16 = 1 << 2,
- NETDATA_V4_18 = 1 << 3,
- NETDATA_V5_4 = 1 << 4,
- NETDATA_V5_10 = 1 << 5,
- NETDATA_V5_11 = 1 << 6,
- NETDATA_V5_14 = 1 << 7,
- NETDATA_V5_15 = 1 << 8,
- NETDATA_V5_16 = 1 << 9
-};
-
-enum netdata_kernel_idx {
- NETDATA_IDX_V3_10,
- NETDATA_IDX_V4_14,
- NETDATA_IDX_V4_16,
- NETDATA_IDX_V4_18,
- NETDATA_IDX_V5_4 ,
- NETDATA_IDX_V5_10,
- NETDATA_IDX_V5_11,
- NETDATA_IDX_V5_14,
- NETDATA_IDX_V5_15,
- NETDATA_IDX_V5_16
-};
-
-#define NETDATA_IDX_STR_V3_10 "3.10"
-#define NETDATA_IDX_STR_V4_14 "4.14"
-#define NETDATA_IDX_STR_V4_16 "4.16"
-#define NETDATA_IDX_STR_V4_18 "4.18"
-#define NETDATA_IDX_STR_V5_4 "5.4"
-#define NETDATA_IDX_STR_V5_10 "5.10"
-#define NETDATA_IDX_STR_V5_11 "5.11"
-#define NETDATA_IDX_STR_V5_14 "5.14"
-#define NETDATA_IDX_STR_V5_15 "5.15"
-#define NETDATA_IDX_STR_V5_16 "5.16"
-
-/**
- * Minimum value has relationship with libbpf support.
- */
-#define NETDATA_MINIMUM_EBPF_KERNEL NETDATA_EBPF_KERNEL_4_11
-
-#define VERSION_STRING_LEN 256
-#define EBPF_KERNEL_REJECT_LIST_FILE "ebpf_kernel_reject_list.txt"
-
-#define ND_EBPF_DEFAULT_MIN_PID 1U
-#define ND_EBPF_MAP_FD_NOT_INITIALIZED (int)-1
-
-typedef struct ebpf_addresses {
- char *function;
- uint32_t hash;
- // We use long as address, because it matches system length
- unsigned long addr;
- uint32_t type;
-} ebpf_addresses_t;
-
-extern char *ebpf_user_config_dir;
-extern char *ebpf_stock_config_dir;
-
-typedef struct ebpf_data {
- int *map_fd;
-
- char *kernel_string;
- uint32_t running_on_kernel;
- int isrh;
-} ebpf_data_t;
-
-typedef enum {
- MODE_RETURN = 0, // This attaches kprobe when the function returns
- MODE_DEVMODE, // This stores log given description about the errors raised
- MODE_ENTRY // This attaches kprobe when the function is called
-} netdata_run_mode_t;
-
-#define ND_EBPF_DEFAULT_PID_SIZE 32768U
-
-enum netdata_ebpf_map_type {
- NETDATA_EBPF_MAP_STATIC = 0,
- NETDATA_EBPF_MAP_RESIZABLE = 1,
- NETDATA_EBPF_MAP_CONTROLLER = 2,
- NETDATA_EBPF_MAP_CONTROLLER_UPDATED = 4,
- NETDATA_EBPF_MAP_PID = 8
-};
-
-enum netdata_controller {
- NETDATA_CONTROLLER_APPS_ENABLED,
- NETDATA_CONTROLLER_APPS_LEVEL,
-
- // These index show the number of elements
- // stored inside hash tables.
- //
- // We have indexes to count increase and
- // decrease events, because __sync_fetch_and_sub
- // generates compilation errors.
- NETDATA_CONTROLLER_PID_TABLE_ADD,
- NETDATA_CONTROLLER_PID_TABLE_DEL,
- NETDATA_CONTROLLER_TEMP_TABLE_ADD,
- NETDATA_CONTROLLER_TEMP_TABLE_DEL,
-
- NETDATA_CONTROLLER_END
-};
-
-// Control how Netdata will monitor PIDs (apps and cgroups)
-typedef enum netdata_apps_level {
- NETDATA_APPS_LEVEL_REAL_PARENT,
- NETDATA_APPS_LEVEL_PARENT,
- NETDATA_APPS_LEVEL_ALL,
-
- // Present only in user ring
- NETDATA_APPS_NOT_SET
-} netdata_apps_level_t;
-
-typedef struct ebpf_local_maps {
- char *name;
- uint32_t internal_input;
- uint32_t user_input;
- uint32_t type;
- int map_fd;
-#ifdef LIBBPF_MAJOR_VERSION
- enum bpf_map_type map_type;
-#endif
-} ebpf_local_maps_t;
-
-typedef struct ebpf_specify_name {
- char *program_name;
- char *function_to_attach;
- char *optional;
- bool retprobe;
-} ebpf_specify_name_t;
-
-typedef enum netdata_ebpf_load_mode {
- EBPF_LOAD_LEGACY = 1<<0, // Select legacy mode, this means we will load binaries
- EBPF_LOAD_CORE = 1<<1, // When CO-RE is used, it is necessary to use the source code
- EBPF_LOAD_PLAY_DICE = 1<<2, // Take a look on environment and choose the best option
- EBPF_LOADED_FROM_STOCK = 1<<3, // Configuration loaded from Stock file
- EBPF_LOADED_FROM_USER = 1<<4 // Configuration loaded from user
-} netdata_ebpf_load_mode_t;
-#define NETDATA_EBPF_LOAD_METHODS (EBPF_LOAD_LEGACY|EBPF_LOAD_CORE|EBPF_LOAD_PLAY_DICE)
-#define NETDATA_EBPF_LOAD_SOURCE (EBPF_LOADED_FROM_STOCK|EBPF_LOADED_FROM_USER)
-
-typedef enum netdata_ebpf_program_loaded {
- EBPF_LOAD_PROBE, // Attach probes on targets
- EBPF_LOAD_RETPROBE, // Attach retprobes on targets
- EBPF_LOAD_TRACEPOINT, // This stores log given description about the errors raised
- EBPF_LOAD_TRAMPOLINE, // This attaches kprobe when the function is called
-} netdata_ebpf_program_loaded_t;
-
-typedef struct netdata_ebpf_targets {
- char *name;
- netdata_ebpf_program_loaded_t mode;
-} netdata_ebpf_targets_t;
-
-typedef struct ebpf_plugin_stats {
- // Load options
- uint32_t legacy; // Legacy codes
- uint32_t core; // CO-RE codes, this means we are using source code compiled.
-
- uint32_t threads; // Total number of threads
- uint32_t running; // total number of threads running
-
- uint32_t probes; // Number of kprobes loaded
- uint32_t retprobes; // Number of kretprobes loaded
- uint32_t tracepoints; // Number of tracepoints used
- uint32_t trampolines; // Number of trampolines used
-
- uint64_t memlock_kern; // The same information reported by bpftool, but it is not accurated
- // https://lore.kernel.org/linux-mm/20230112155326.26902-5-laoar.shao@gmail.com/T/
- uint32_t hash_tables; // Number of hash tables used on the system.
-
- uint32_t hash_percpu; // Number of threads running per cpu maps
- uint32_t hash_unique; // Number of threads running an unique map for all cores.
-} ebpf_plugin_stats_t;
-
-typedef enum ebpf_stats_action {
- EBPF_ACTION_STAT_ADD,
- EBPF_ACTION_STAT_REMOVE,
-} ebpf_stats_action_t;
-
-typedef enum netdata_apps_integration_flags {
- NETDATA_EBPF_APPS_FLAG_NO,
- NETDATA_EBPF_APPS_FLAG_YES,
- NETDATA_EBPF_APPS_FLAG_CHART_CREATED
-} netdata_apps_integration_flags_t;
-
-#define NETDATA_EBPF_CHART_MEM_LENGTH 48
-#define NETDATA_EBPF_STAT_DIMENSION_MEMORY "memory"
-#define NETDATA_EBPF_STAT_DIMENSION_ARAL "aral"
-
-enum ebpf_threads_status {
- NETDATA_THREAD_EBPF_RUNNING, // started by plugin
- NETDATA_THREAD_EBPF_FUNCTION_RUNNING, // started by function
- NETDATA_THREAD_EBPF_STOPPING, // stopping thread
- NETDATA_THREAD_EBPF_STOPPED, // thread stopped
- NETDATA_THREAD_EBPF_NOT_RUNNING // thread was never started
-};
-
-enum ebpf_global_table_values {
- NETDATA_EBPF_GLOBAL_TABLE_PID_TABLE_ADD, // Count elements added inside PID table
- NETDATA_EBPF_GLOBAL_TABLE_PID_TABLE_DEL, // Count elements removed from PID table
- NETDATA_EBPF_GLOBAL_TABLE_TEMP_TABLE_ADD, // Count elements added inside TEMP table
- NETDATA_EBPF_GLOBAL_TABLE_TEMP_TABLE_DEL, // Count elements removed from TEMP table
-
- NETDATA_EBPF_GLOBAL_TABLE_STATUS_END
-};
-
-typedef uint64_t netdata_idx_t;
-
-typedef struct ebpf_module {
- // Constants used with module
- struct {
- const char *thread_name;
- const char *config_name;
- const char *thread_description;
- } info;
-
- // Helpers used with plugin
- struct {
- void *(*start_routine)(void *); // the thread function
- void (*apps_routine)(struct ebpf_module *em, void *ptr); // the apps charts
- void (*fnct_routine)(BUFFER *bf, struct ebpf_module *em); // the function used for exteernal requests
- const char *fcnt_name; // name given to cloud
- const char *fcnt_desc; // description given about function
- const char *fcnt_thread_chart_name;
- int order_thread_chart;
- const char *fcnt_thread_lifetime_name;
- int order_thread_lifetime;
- } functions;
-
- enum ebpf_threads_status enabled;
- int update_every;
- int global_charts;
- netdata_apps_integration_flags_t apps_charts;
- netdata_apps_level_t apps_level;
- int cgroup_charts;
- netdata_run_mode_t mode;
- uint32_t thread_id;
- int optional;
- ebpf_local_maps_t *maps;
- ebpf_specify_name_t *names;
- uint32_t pid_map_size;
- struct config *cfg;
- const char *config_file;
- uint64_t kernels;
- netdata_ebpf_load_mode_t load;
- netdata_ebpf_targets_t *targets;
- struct bpf_link **probe_links;
- struct bpf_object *objects;
- struct netdata_static_thread *thread;
-
- // charts
- char memory_usage[NETDATA_EBPF_CHART_MEM_LENGTH];
- char memory_allocations[NETDATA_EBPF_CHART_MEM_LENGTH];
- int maps_per_core;
-
- // period to run
- uint32_t running_time; // internal usage, this is used to reset a value when a new request happens.
- uint32_t lifetime;
-
- netdata_idx_t hash_table_stats[NETDATA_EBPF_GLOBAL_TABLE_STATUS_END];
-} ebpf_module_t;
-
-#define EBPF_DEFAULT_LIFETIME 300
-// This will be present until all functions are merged. The deadline is planned for 68 years since plugin start
-#define EBPF_NON_FUNCTION_LIFE_TIME UINT_MAX
-
-int ebpf_get_kernel_version();
-int get_redhat_release();
-int has_condition_to_run(int version);
-char *ebpf_kernel_suffix(int version, int isrh);
-struct bpf_link **ebpf_load_program(char *plugins_dir, ebpf_module_t *em, int kver, int is_rhf,
- struct bpf_object **obj);
-
-void ebpf_mount_config_name(char *filename, size_t length, char *path, const char *config);
-int ebpf_load_config(struct config *config, char *filename);
-void ebpf_update_module(ebpf_module_t *em, struct btf *btf_file, int kver, int is_rh);
-void ebpf_update_names(ebpf_specify_name_t *opt, ebpf_module_t *em);
-void ebpf_adjust_apps_cgroup(ebpf_module_t *em, netdata_ebpf_program_loaded_t mode);
-char *ebpf_find_symbol(char *search);
-void ebpf_load_addresses(ebpf_addresses_t *fa, int fd);
-void ebpf_fill_algorithms(int *algorithms, size_t length, int algorithm);
-char **ebpf_fill_histogram_dimension(size_t maximum);
-void ebpf_update_stats(ebpf_plugin_stats_t *report, ebpf_module_t *em);
-void ebpf_update_controller(int fd, ebpf_module_t *em);
-void ebpf_update_map_size(struct bpf_map *map, ebpf_local_maps_t *lmap, ebpf_module_t *em, const char *map_name);
-
-// Histogram
-#define NETDATA_EBPF_HIST_MAX_BINS 24UL
-#define NETDATA_DISK_MAX 256U
-#define NETDATA_DISK_HISTOGRAM_LENGTH (NETDATA_DISK_MAX * NETDATA_EBPF_HIST_MAX_BINS)
-
-typedef struct netdata_ebpf_histogram {
- char *name;
- char *title;
- char *ctx;
- int order;
- uint64_t histogram[NETDATA_EBPF_HIST_MAX_BINS];
-} netdata_ebpf_histogram_t;
-
-enum fs_btf_counters {
- NETDATA_KEY_BTF_READ,
- NETDATA_KEY_BTF_WRITE,
- NETDATA_KEY_BTF_OPEN,
- NETDATA_KEY_BTF_SYNC_ATTR,
- NETDATA_KEY_BTF_OPEN2,
-
- NETDATA_FS_BTF_END
-};
-
-typedef struct ebpf_filesystem_partitions {
- char *filesystem;
- char *optional_filesystem;
- char *family;
- char *family_name;
- struct bpf_object *objects;
- struct bpf_link **probe_links;
-
- netdata_ebpf_histogram_t hread;
- netdata_ebpf_histogram_t hwrite;
- netdata_ebpf_histogram_t hopen;
- netdata_ebpf_histogram_t hadditional;
-
- uint32_t flags;
- uint32_t enabled;
-
- ebpf_addresses_t addresses;
- uint64_t kernels;
- ebpf_local_maps_t *fs_maps;
-
- // BPF structure
-#ifdef LIBBPF_MAJOR_VERSION
- struct filesystem_bpf *fs_obj;
-#else
- void *fs_obj;
-#endif
- const char *functions[NETDATA_FS_BTF_END];
-} ebpf_filesystem_partitions_t;
-
-typedef struct ebpf_sync_syscalls {
- char *syscall;
- int enabled;
- uint32_t flags;
-
- // BTF structure
- struct bpf_object *objects;
- struct bpf_link **probe_links;
-
- // BPF structure
-#ifdef LIBBPF_MAJOR_VERSION
- struct sync_bpf *sync_obj;
-#else
- void *sync_obj;
-#endif
- ebpf_local_maps_t *sync_maps;
-} ebpf_sync_syscalls_t;
-
-void ebpf_histogram_dimension_cleanup(char **ptr, size_t length);
-
-// Tracepoint helpers
-// For more information related to tracepoints read https://www.kernel.org/doc/html/latest/trace/tracepoints.html
-int ebpf_is_tracepoint_enabled(char *subsys, char *eventname);
-int ebpf_enable_tracing_values(char *subsys, char *eventname);
-int ebpf_disable_tracing_values(char *subsys, char *eventname);
-
-// BTF Section
-#define EBPF_DEFAULT_BTF_FILE "vmlinux"
-#define EBPF_DEFAULT_BTF_PATH "/sys/kernel/btf"
-#define EBPF_DEFAULT_ERROR_MSG "Cannot open or load BPF file for thread"
-
-// BTF helpers
-#define NETDATA_EBPF_MAX_SYSCALL_LENGTH 255
-
-netdata_ebpf_load_mode_t epbf_convert_string_to_load_mode(char *str);
-netdata_ebpf_program_loaded_t ebpf_convert_core_type(char *str, netdata_run_mode_t lmode);
-void ebpf_select_host_prefix(char *output, size_t length, char *syscall, int kver);
-#ifdef LIBBPF_MAJOR_VERSION
-void ebpf_adjust_thread_load(ebpf_module_t *mod, struct btf *file);
-struct btf *ebpf_parse_btf_file(const char *filename);
-struct btf *ebpf_load_btf_file(char *path, char *filename);
-int ebpf_is_function_inside_btf(struct btf *file, char *function);
-void ebpf_update_map_type(struct bpf_map *map, ebpf_local_maps_t *w);
-void ebpf_define_map_type(ebpf_local_maps_t *maps, int maps_per_core, int kver);
-#endif
-
-void ebpf_update_kernel_memory_with_vector(ebpf_plugin_stats_t *report, ebpf_local_maps_t *maps,
- ebpf_stats_action_t action);
-void ebpf_update_kernel_memory(ebpf_plugin_stats_t *report, ebpf_local_maps_t *map, ebpf_stats_action_t action);
-int ebpf_statistic_create_aral_chart(char *name, ebpf_module_t *em);
-void ebpf_statistic_obsolete_aral_chart(ebpf_module_t *em, int prio);
-void ebpf_send_data_aral_chart(ARAL *memory, ebpf_module_t *em);
-
-#endif /* NETDATA_EBPF_H */
diff --git a/libnetdata/endian.h b/libnetdata/endian.h
deleted file mode 100644
index 66f75cb1a..000000000
--- a/libnetdata/endian.h
+++ /dev/null
@@ -1,32 +0,0 @@
-#ifndef LIBNETDATA_ENDIAN_H
-#define LIBNETDATA_ENDIAN_H
-
-/** compatibility header for endian.h
- * This is a simple compatibility shim to convert
- * BSD/Linux endian macros to the Mac OS X equivalents.
- * It is public domain.
- * */
-
-#ifndef __APPLE__
-#error "This header file (endian.h) is MacOS X specific.\n"
-#endif /* __APPLE__ */
-
-
-#include <libkern/OSByteOrder.h>
-
-#define htobe16(x) OSSwapHostToBigInt16(x)
-#define htole16(x) OSSwapHostToLittleInt16(x)
-#define be16toh(x) OSSwapBigToHostInt16(x)
-#define le16toh(x) OSSwapLittleToHostInt16(x)
-
-#define htobe32(x) OSSwapHostToBigInt32(x)
-#define htole32(x) OSSwapHostToLittleInt32(x)
-#define be32toh(x) OSSwapBigToHostInt32(x)
-#define le32toh(x) OSSwapLittleToHostInt32(x)
-
-#define htobe64(x) OSSwapHostToBigInt64(x)
-#define htole64(x) OSSwapHostToLittleInt64(x)
-#define be64toh(x) OSSwapBigToHostInt64(x)
-#define le64toh(x) OSSwapLittleToHostInt64(x)
-
-#endif /* LIBNETDATA_ENDIAN_H */
diff --git a/libnetdata/eval/Makefile.am b/libnetdata/eval/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/eval/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/eval/eval.c b/libnetdata/eval/eval.c
deleted file mode 100644
index a1ac4483c..000000000
--- a/libnetdata/eval/eval.c
+++ /dev/null
@@ -1,1201 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "../libnetdata.h"
-
-// ----------------------------------------------------------------------------
-// data structures for storing the parsed expression in memory
-
-typedef struct eval_value {
- int type;
-
- union {
- NETDATA_DOUBLE number;
- EVAL_VARIABLE *variable;
- struct eval_node *expression;
- };
-} EVAL_VALUE;
-
-typedef struct eval_node {
- int id;
- unsigned char operator;
- int precedence;
-
- int count;
- EVAL_VALUE ops[];
-} EVAL_NODE;
-
-// these are used for EVAL_NODE.operator
-// they are used as internal IDs to identify an operator
-// THEY ARE NOT USED FOR PARSING OPERATORS LIKE THAT
-#define EVAL_OPERATOR_NOP '\0'
-#define EVAL_OPERATOR_EXPRESSION_OPEN '('
-#define EVAL_OPERATOR_EXPRESSION_CLOSE ')'
-#define EVAL_OPERATOR_NOT '!'
-#define EVAL_OPERATOR_PLUS '+'
-#define EVAL_OPERATOR_MINUS '-'
-#define EVAL_OPERATOR_AND '&'
-#define EVAL_OPERATOR_OR '|'
-#define EVAL_OPERATOR_GREATER_THAN_OR_EQUAL 'G'
-#define EVAL_OPERATOR_LESS_THAN_OR_EQUAL 'L'
-#define EVAL_OPERATOR_NOT_EQUAL '~'
-#define EVAL_OPERATOR_EQUAL '='
-#define EVAL_OPERATOR_LESS '<'
-#define EVAL_OPERATOR_GREATER '>'
-#define EVAL_OPERATOR_MULTIPLY '*'
-#define EVAL_OPERATOR_DIVIDE '/'
-#define EVAL_OPERATOR_SIGN_PLUS 'P'
-#define EVAL_OPERATOR_SIGN_MINUS 'M'
-#define EVAL_OPERATOR_ABS 'A'
-#define EVAL_OPERATOR_IF_THEN_ELSE '?'
-
-// ----------------------------------------------------------------------------
-// forward function definitions
-
-static inline void eval_node_free(EVAL_NODE *op);
-static inline EVAL_NODE *parse_full_expression(const char **string, int *error);
-static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error);
-static inline NETDATA_DOUBLE eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error);
-static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error);
-static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n);
-
-// ----------------------------------------------------------------------------
-// evaluation of expressions
-
-static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) {
- static STRING
- *this_string = NULL,
- *now_string = NULL,
- *after_string = NULL,
- *before_string = NULL,
- *status_string = NULL,
- *removed_string = NULL,
- *uninitialized_string = NULL,
- *undefined_string = NULL,
- *clear_string = NULL,
- *warning_string = NULL,
- *critical_string = NULL;
-
- NETDATA_DOUBLE n;
-
- if(unlikely(this_string == NULL)) {
- this_string = string_strdupz("this");
- now_string = string_strdupz("now");
- after_string = string_strdupz("after");
- before_string = string_strdupz("before");
- status_string = string_strdupz("status");
- removed_string = string_strdupz("REMOVED");
- uninitialized_string = string_strdupz("UNINITIALIZED");
- undefined_string = string_strdupz("UNDEFINED");
- clear_string = string_strdupz("CLEAR");
- warning_string = string_strdupz("WARNING");
- critical_string = string_strdupz("CRITICAL");
- }
-
- if(unlikely(v->name == this_string)) {
- n = (exp->myself)?*exp->myself:NAN;
- buffer_strcat(exp->error_msg, "[ $this = ");
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- if(unlikely(v->name == after_string)) {
- n = (exp->after && *exp->after)?*exp->after:NAN;
- buffer_strcat(exp->error_msg, "[ $after = ");
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- if(unlikely(v->name == before_string)) {
- n = (exp->before && *exp->before)?*exp->before:NAN;
- buffer_strcat(exp->error_msg, "[ $before = ");
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- if(unlikely(v->name == now_string)) {
- n = (NETDATA_DOUBLE)now_realtime_sec();
- buffer_strcat(exp->error_msg, "[ $now = ");
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- if(unlikely(v->name == status_string)) {
- n = (exp->status)?*exp->status:RRDCALC_STATUS_UNINITIALIZED;
- buffer_strcat(exp->error_msg, "[ $status = ");
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- if(unlikely(v->name == removed_string)) {
- n = RRDCALC_STATUS_REMOVED;
- buffer_strcat(exp->error_msg, "[ $REMOVED = ");
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- if(unlikely(v->name == uninitialized_string)) {
- n = RRDCALC_STATUS_UNINITIALIZED;
- buffer_strcat(exp->error_msg, "[ $UNINITIALIZED = ");
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- if(unlikely(v->name == undefined_string)) {
- n = RRDCALC_STATUS_UNDEFINED;
- buffer_strcat(exp->error_msg, "[ $UNDEFINED = ");
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- if(unlikely(v->name == clear_string)) {
- n = RRDCALC_STATUS_CLEAR;
- buffer_strcat(exp->error_msg, "[ $CLEAR = ");
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- if(unlikely(v->name == warning_string)) {
- n = RRDCALC_STATUS_WARNING;
- buffer_strcat(exp->error_msg, "[ $WARNING = ");
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- if(unlikely(v->name == critical_string)) {
- n = RRDCALC_STATUS_CRITICAL;
- buffer_strcat(exp->error_msg, "[ $CRITICAL = ");
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- if(exp->rrdcalc && health_variable_lookup(v->name, exp->rrdcalc, &n)) {
- buffer_sprintf(exp->error_msg, "[ ${%s} = ", string2str(v->name));
- print_parsed_as_constant(exp->error_msg, n);
- buffer_strcat(exp->error_msg, " ] ");
- return n;
- }
-
- *error = EVAL_ERROR_UNKNOWN_VARIABLE;
- buffer_sprintf(exp->error_msg, "[ undefined variable '%s' ] ", string2str(v->name));
- return NAN;
-}
-
-static inline NETDATA_DOUBLE eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v, int *error) {
- NETDATA_DOUBLE n;
-
- switch(v->type) {
- case EVAL_VALUE_EXPRESSION:
- n = eval_node(exp, v->expression, error);
- break;
-
- case EVAL_VALUE_NUMBER:
- n = v->number;
- break;
-
- case EVAL_VALUE_VARIABLE:
- n = eval_variable(exp, v->variable, error);
- break;
-
- default:
- *error = EVAL_ERROR_INVALID_VALUE;
- n = 0;
- break;
- }
-
- return n;
-}
-
-static inline int is_true(NETDATA_DOUBLE n) {
- if(isnan(n)) return 0;
- if(isinf(n)) return 1;
- if(n == 0) return 0;
- return 1;
-}
-
-NETDATA_DOUBLE eval_and(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- return is_true(eval_value(exp, &op->ops[0], error)) && is_true(eval_value(exp, &op->ops[1], error));
-}
-NETDATA_DOUBLE eval_or(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- return is_true(eval_value(exp, &op->ops[0], error)) || is_true(eval_value(exp, &op->ops[1], error));
-}
-NETDATA_DOUBLE eval_greater_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
- NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
- return isgreaterequal(n1, n2);
-}
-NETDATA_DOUBLE eval_less_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
- NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
- return islessequal(n1, n2);
-}
-NETDATA_DOUBLE eval_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
- NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
- if(isnan(n1) && isnan(n2)) return 1;
- if(isinf(n1) && isinf(n2)) return 1;
- if(isnan(n1) || isnan(n2)) return 0;
- if(isinf(n1) || isinf(n2)) return 0;
- return considered_equal_ndd(n1, n2);
-}
-NETDATA_DOUBLE eval_not_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- return !eval_equal(exp, op, error);
-}
-NETDATA_DOUBLE eval_less(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
- NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
- return isless(n1, n2);
-}
-NETDATA_DOUBLE eval_greater(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
- NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
- return isgreater(n1, n2);
-}
-NETDATA_DOUBLE eval_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
- NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
- if(isnan(n1) || isnan(n2)) return NAN;
- if(isinf(n1) || isinf(n2)) return INFINITY;
- return n1 + n2;
-}
-NETDATA_DOUBLE eval_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
- NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
- if(isnan(n1) || isnan(n2)) return NAN;
- if(isinf(n1) || isinf(n2)) return INFINITY;
- return n1 - n2;
-}
-NETDATA_DOUBLE eval_multiply(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
- NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
- if(isnan(n1) || isnan(n2)) return NAN;
- if(isinf(n1) || isinf(n2)) return INFINITY;
- return n1 * n2;
-}
-NETDATA_DOUBLE eval_divide(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
- NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error);
- if(isnan(n1) || isnan(n2)) return NAN;
- if(isinf(n1) || isinf(n2)) return INFINITY;
- return n1 / n2;
-}
-NETDATA_DOUBLE eval_nop(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- return eval_value(exp, &op->ops[0], error);
-}
-NETDATA_DOUBLE eval_not(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- return !is_true(eval_value(exp, &op->ops[0], error));
-}
-NETDATA_DOUBLE eval_sign_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- return eval_value(exp, &op->ops[0], error);
-}
-NETDATA_DOUBLE eval_sign_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
- if(isnan(n1)) return NAN;
- if(isinf(n1)) return INFINITY;
- return -n1;
-}
-NETDATA_DOUBLE eval_abs(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error);
- if(isnan(n1)) return NAN;
- if(isinf(n1)) return INFINITY;
- return ABS(n1);
-}
-NETDATA_DOUBLE eval_if_then_else(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- if(is_true(eval_value(exp, &op->ops[0], error)))
- return eval_value(exp, &op->ops[1], error);
- else
- return eval_value(exp, &op->ops[2], error);
-}
-
-static struct operator {
- const char *print_as;
- char precedence;
- char parameters;
- char isfunction;
- NETDATA_DOUBLE (*eval)(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error);
-} operators[256] = {
- // this is a random access array
- // we always access it with a known EVAL_OPERATOR_X
-
- [EVAL_OPERATOR_AND] = { "&&", 2, 2, 0, eval_and },
- [EVAL_OPERATOR_OR] = { "||", 2, 2, 0, eval_or },
- [EVAL_OPERATOR_GREATER_THAN_OR_EQUAL] = { ">=", 3, 2, 0, eval_greater_than_or_equal },
- [EVAL_OPERATOR_LESS_THAN_OR_EQUAL] = { "<=", 3, 2, 0, eval_less_than_or_equal },
- [EVAL_OPERATOR_NOT_EQUAL] = { "!=", 3, 2, 0, eval_not_equal },
- [EVAL_OPERATOR_EQUAL] = { "==", 3, 2, 0, eval_equal },
- [EVAL_OPERATOR_LESS] = { "<", 3, 2, 0, eval_less },
- [EVAL_OPERATOR_GREATER] = { ">", 3, 2, 0, eval_greater },
- [EVAL_OPERATOR_PLUS] = { "+", 4, 2, 0, eval_plus },
- [EVAL_OPERATOR_MINUS] = { "-", 4, 2, 0, eval_minus },
- [EVAL_OPERATOR_MULTIPLY] = { "*", 5, 2, 0, eval_multiply },
- [EVAL_OPERATOR_DIVIDE] = { "/", 5, 2, 0, eval_divide },
- [EVAL_OPERATOR_NOT] = { "!", 6, 1, 0, eval_not },
- [EVAL_OPERATOR_SIGN_PLUS] = { "+", 6, 1, 0, eval_sign_plus },
- [EVAL_OPERATOR_SIGN_MINUS] = { "-", 6, 1, 0, eval_sign_minus },
- [EVAL_OPERATOR_ABS] = { "abs(",6,1, 1, eval_abs },
- [EVAL_OPERATOR_IF_THEN_ELSE] = { "?", 7, 3, 0, eval_if_then_else },
- [EVAL_OPERATOR_NOP] = { NULL, 8, 1, 0, eval_nop },
- [EVAL_OPERATOR_EXPRESSION_OPEN] = { NULL, 8, 1, 0, eval_nop },
-
- // this should exist in our evaluation list
- [EVAL_OPERATOR_EXPRESSION_CLOSE] = { NULL, 99, 1, 0, eval_nop }
-};
-
-#define eval_precedence(operator) (operators[(unsigned char)(operator)].precedence)
-
-static inline NETDATA_DOUBLE eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) {
- if(unlikely(op->count != operators[op->operator].parameters)) {
- *error = EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS;
- return 0;
- }
-
- NETDATA_DOUBLE n = operators[op->operator].eval(exp, op, error);
-
- return n;
-}
-
-// ----------------------------------------------------------------------------
-// parsed-as generation
-
-static inline void print_parsed_as_variable(BUFFER *out, EVAL_VARIABLE *v, int *error) {
- (void)error;
- buffer_sprintf(out, "${%s}", string2str(v->name));
-}
-
-static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n) {
- if(unlikely(isnan(n))) {
- buffer_strcat(out, "nan");
- return;
- }
-
- if(unlikely(isinf(n))) {
- buffer_strcat(out, "inf");
- return;
- }
-
- char b[100+1], *s;
- snprintfz(b, sizeof(b) - 1, NETDATA_DOUBLE_FORMAT, n);
-
- s = &b[strlen(b) - 1];
- while(s > b && *s == '0') {
- *s ='\0';
- s--;
- }
-
- if(s > b && *s == '.')
- *s = '\0';
-
- buffer_strcat(out, b);
-}
-
-static inline void print_parsed_as_value(BUFFER *out, EVAL_VALUE *v, int *error) {
- switch(v->type) {
- case EVAL_VALUE_EXPRESSION:
- print_parsed_as_node(out, v->expression, error);
- break;
-
- case EVAL_VALUE_NUMBER:
- print_parsed_as_constant(out, v->number);
- break;
-
- case EVAL_VALUE_VARIABLE:
- print_parsed_as_variable(out, v->variable, error);
- break;
-
- default:
- *error = EVAL_ERROR_INVALID_VALUE;
- break;
- }
-}
-
-static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error) {
- if(unlikely(op->count != operators[op->operator].parameters)) {
- *error = EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS;
- return;
- }
-
- if(operators[op->operator].parameters == 1) {
-
- if(operators[op->operator].print_as)
- buffer_sprintf(out, "%s", operators[op->operator].print_as);
-
- //if(op->operator == EVAL_OPERATOR_EXPRESSION_OPEN)
- // buffer_strcat(out, "(");
-
- print_parsed_as_value(out, &op->ops[0], error);
-
- //if(op->operator == EVAL_OPERATOR_EXPRESSION_OPEN)
- // buffer_strcat(out, ")");
- }
-
- else if(operators[op->operator].parameters == 2) {
- buffer_strcat(out, "(");
- print_parsed_as_value(out, &op->ops[0], error);
-
- if(operators[op->operator].print_as)
- buffer_sprintf(out, " %s ", operators[op->operator].print_as);
-
- print_parsed_as_value(out, &op->ops[1], error);
- buffer_strcat(out, ")");
- }
- else if(op->operator == EVAL_OPERATOR_IF_THEN_ELSE && operators[op->operator].parameters == 3) {
- buffer_strcat(out, "(");
- print_parsed_as_value(out, &op->ops[0], error);
-
- if(operators[op->operator].print_as)
- buffer_sprintf(out, " %s ", operators[op->operator].print_as);
-
- print_parsed_as_value(out, &op->ops[1], error);
- buffer_strcat(out, " : ");
- print_parsed_as_value(out, &op->ops[2], error);
- buffer_strcat(out, ")");
- }
-
- if(operators[op->operator].isfunction)
- buffer_strcat(out, ")");
-}
-
-// ----------------------------------------------------------------------------
-// parsing expressions
-
-// skip spaces
-static inline void skip_spaces(const char **string) {
- const char *s = *string;
- while(isspace(*s)) s++;
- *string = s;
-}
-
-// what character can appear just after an operator keyword
-// like NOT AND OR ?
-static inline int isoperatorterm_word(const char s) {
- if(isspace(s) || s == '(' || s == '$' || s == '!' || s == '-' || s == '+' || isdigit(s) || !s)
- return 1;
-
- return 0;
-}
-
-// what character can appear just after an operator symbol?
-static inline int isoperatorterm_symbol(const char s) {
- if(isoperatorterm_word(s) || isalpha(s))
- return 1;
-
- return 0;
-}
-
-// return 1 if the character should never appear in a variable
-static inline int isvariableterm(const char s) {
- if(isalnum(s) || s == '.' || s == '_')
- return 0;
-
- return 1;
-}
-
-// ----------------------------------------------------------------------------
-// parse operators
-
-static inline int parse_and(const char **string) {
- const char *s = *string;
-
- // AND
- if((s[0] == 'A' || s[0] == 'a') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'D' || s[2] == 'd') && isoperatorterm_word(s[3])) {
- *string = &s[4];
- return 1;
- }
-
- // &&
- if(s[0] == '&' && s[1] == '&' && isoperatorterm_symbol(s[2])) {
- *string = &s[2];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_or(const char **string) {
- const char *s = *string;
-
- // OR
- if((s[0] == 'O' || s[0] == 'o') && (s[1] == 'R' || s[1] == 'r') && isoperatorterm_word(s[2])) {
- *string = &s[3];
- return 1;
- }
-
- // ||
- if(s[0] == '|' && s[1] == '|' && isoperatorterm_symbol(s[2])) {
- *string = &s[2];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_greater_than_or_equal(const char **string) {
- const char *s = *string;
-
- // >=
- if(s[0] == '>' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
- *string = &s[2];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_less_than_or_equal(const char **string) {
- const char *s = *string;
-
- // <=
- if (s[0] == '<' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
- *string = &s[2];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_greater(const char **string) {
- const char *s = *string;
-
- // >
- if(s[0] == '>' && isoperatorterm_symbol(s[1])) {
- *string = &s[1];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_less(const char **string) {
- const char *s = *string;
-
- // <
- if(s[0] == '<' && isoperatorterm_symbol(s[1])) {
- *string = &s[1];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_equal(const char **string) {
- const char *s = *string;
-
- // ==
- if(s[0] == '=' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
- *string = &s[2];
- return 1;
- }
-
- // =
- if(s[0] == '=' && isoperatorterm_symbol(s[1])) {
- *string = &s[1];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_not_equal(const char **string) {
- const char *s = *string;
-
- // !=
- if(s[0] == '!' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
- *string = &s[2];
- return 1;
- }
-
- // <>
- if(s[0] == '<' && s[1] == '>' && isoperatorterm_symbol(s[2])) {
- *string = &s[2];
- }
-
- return 0;
-}
-
-static inline int parse_not(const char **string) {
- const char *s = *string;
-
- // NOT
- if((s[0] == 'N' || s[0] == 'n') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'T' || s[2] == 't') && isoperatorterm_word(s[3])) {
- *string = &s[3];
- return 1;
- }
-
- if(s[0] == '!') {
- *string = &s[1];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_multiply(const char **string) {
- const char *s = *string;
-
- // *
- if(s[0] == '*' && isoperatorterm_symbol(s[1])) {
- *string = &s[1];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_divide(const char **string) {
- const char *s = *string;
-
- // /
- if(s[0] == '/' && isoperatorterm_symbol(s[1])) {
- *string = &s[1];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_minus(const char **string) {
- const char *s = *string;
-
- // -
- if(s[0] == '-' && isoperatorterm_symbol(s[1])) {
- *string = &s[1];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_plus(const char **string) {
- const char *s = *string;
-
- // +
- if(s[0] == '+' && isoperatorterm_symbol(s[1])) {
- *string = &s[1];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_open_subexpression(const char **string) {
- const char *s = *string;
-
- // (
- if(s[0] == '(') {
- *string = &s[1];
- return 1;
- }
-
- return 0;
-}
-
-#define parse_close_function(x) parse_close_subexpression(x)
-
-static inline int parse_close_subexpression(const char **string) {
- const char *s = *string;
-
- // )
- if(s[0] == ')') {
- *string = &s[1];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_variable(const char **string, char *buffer, size_t len) {
- const char *s = *string;
-
- // $
- if(*s == '$') {
- size_t i = 0;
- s++;
-
- if(*s == '{') {
- // ${variable_name}
-
- s++;
- while (*s && *s != '}' && i < len)
- buffer[i++] = *s++;
-
- if(*s == '}')
- s++;
- }
- else {
- // $variable_name
-
- while (*s && !isvariableterm(*s) && i < len)
- buffer[i++] = *s++;
- }
-
- buffer[i] = '\0';
-
- if (buffer[0]) {
- *string = s;
- return 1;
- }
- }
-
- return 0;
-}
-
-static inline int parse_constant(const char **string, NETDATA_DOUBLE *number) {
- char *end = NULL;
- NETDATA_DOUBLE n = str2ndd(*string, &end);
- if(unlikely(!end || *string == end)) {
- *number = 0;
- return 0;
- }
- *number = n;
- *string = end;
- return 1;
-}
-
-static inline int parse_abs(const char **string) {
- const char *s = *string;
-
- // ABS
- if((s[0] == 'A' || s[0] == 'a') && (s[1] == 'B' || s[1] == 'b') && (s[2] == 'S' || s[2] == 's') && s[3] == '(') {
- *string = &s[3];
- return 1;
- }
-
- return 0;
-}
-
-static inline int parse_if_then_else(const char **string) {
- const char *s = *string;
-
- // ?
- if(s[0] == '?') {
- *string = &s[1];
- return 1;
- }
-
- return 0;
-}
-
-static struct operator_parser {
- unsigned char id;
- int (*parse)(const char **);
-} operator_parsers[] = {
- // the order in this list is important!
- // the first matching will be used
- // so place the longer of overlapping ones
- // at the top
-
- { EVAL_OPERATOR_AND, parse_and },
- { EVAL_OPERATOR_OR, parse_or },
- { EVAL_OPERATOR_GREATER_THAN_OR_EQUAL, parse_greater_than_or_equal },
- { EVAL_OPERATOR_LESS_THAN_OR_EQUAL, parse_less_than_or_equal },
- { EVAL_OPERATOR_NOT_EQUAL, parse_not_equal },
- { EVAL_OPERATOR_EQUAL, parse_equal },
- { EVAL_OPERATOR_LESS, parse_less },
- { EVAL_OPERATOR_GREATER, parse_greater },
- { EVAL_OPERATOR_PLUS, parse_plus },
- { EVAL_OPERATOR_MINUS, parse_minus },
- { EVAL_OPERATOR_MULTIPLY, parse_multiply },
- { EVAL_OPERATOR_DIVIDE, parse_divide },
- { EVAL_OPERATOR_IF_THEN_ELSE, parse_if_then_else },
-
- /* we should not put in this list the following:
- *
- * - NOT
- * - (
- * - )
- *
- * these are handled in code
- */
-
- // termination
- { EVAL_OPERATOR_NOP, NULL }
-};
-
-static inline unsigned char parse_operator(const char **string, int *precedence) {
- skip_spaces(string);
-
- int i;
- for(i = 0 ; operator_parsers[i].parse != NULL ; i++)
- if(operator_parsers[i].parse(string)) {
- if(precedence) *precedence = eval_precedence(operator_parsers[i].id);
- return operator_parsers[i].id;
- }
-
- return EVAL_OPERATOR_NOP;
-}
-
-// ----------------------------------------------------------------------------
-// memory management
-
-static inline EVAL_NODE *eval_node_alloc(int count) {
- static int id = 1;
-
- EVAL_NODE *op = callocz(1, sizeof(EVAL_NODE) + (sizeof(EVAL_VALUE) * count));
-
- op->id = id++;
- op->operator = EVAL_OPERATOR_NOP;
- op->precedence = eval_precedence(EVAL_OPERATOR_NOP);
- op->count = count;
- return op;
-}
-
-static inline void eval_node_set_value_to_node(EVAL_NODE *op, int pos, EVAL_NODE *value) {
- if(pos >= op->count)
- fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1);
-
- op->ops[pos].type = EVAL_VALUE_EXPRESSION;
- op->ops[pos].expression = value;
-}
-
-static inline void eval_node_set_value_to_constant(EVAL_NODE *op, int pos, NETDATA_DOUBLE value) {
- if(pos >= op->count)
- fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1);
-
- op->ops[pos].type = EVAL_VALUE_NUMBER;
- op->ops[pos].number = value;
-}
-
-static inline void eval_node_set_value_to_variable(EVAL_NODE *op, int pos, const char *variable) {
- if(pos >= op->count)
- fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1);
-
- op->ops[pos].type = EVAL_VALUE_VARIABLE;
- op->ops[pos].variable = callocz(1, sizeof(EVAL_VARIABLE));
- op->ops[pos].variable->name = string_strdupz(variable);
-}
-
-static inline void eval_variable_free(EVAL_VARIABLE *v) {
- string_freez(v->name);
- freez(v);
-}
-
-static inline void eval_value_free(EVAL_VALUE *v) {
- switch(v->type) {
- case EVAL_VALUE_EXPRESSION:
- eval_node_free(v->expression);
- break;
-
- case EVAL_VALUE_VARIABLE:
- eval_variable_free(v->variable);
- break;
-
- default:
- break;
- }
-}
-
-static inline void eval_node_free(EVAL_NODE *op) {
- if(op->count) {
- int i;
- for(i = op->count - 1; i >= 0 ;i--)
- eval_value_free(&op->ops[i]);
- }
-
- freez(op);
-}
-
-// ----------------------------------------------------------------------------
-// the parsing logic
-
-// helper function to avoid allocations all over the place
-static inline EVAL_NODE *parse_next_operand_given_its_operator(const char **string, unsigned char operator_type, int *error) {
- EVAL_NODE *sub = parse_one_full_operand(string, error);
- if(!sub) return NULL;
-
- EVAL_NODE *op = eval_node_alloc(1);
- op->operator = operator_type;
- eval_node_set_value_to_node(op, 0, sub);
- return op;
-}
-
-// parse a full operand, including its sign or other associative operator (e.g. NOT)
-static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error) {
- char variable_buffer[EVAL_MAX_VARIABLE_NAME_LENGTH + 1];
- EVAL_NODE *op1 = NULL;
- NETDATA_DOUBLE number;
-
- *error = EVAL_ERROR_OK;
-
- skip_spaces(string);
- if(!(**string)) {
- *error = EVAL_ERROR_MISSING_OPERAND;
- return NULL;
- }
-
- if(parse_not(string)) {
- op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_NOT, error);
- op1->precedence = eval_precedence(EVAL_OPERATOR_NOT);
- }
- else if(parse_plus(string)) {
- op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_SIGN_PLUS, error);
- op1->precedence = eval_precedence(EVAL_OPERATOR_SIGN_PLUS);
- }
- else if(parse_minus(string)) {
- op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_SIGN_MINUS, error);
- op1->precedence = eval_precedence(EVAL_OPERATOR_SIGN_MINUS);
- }
- else if(parse_abs(string)) {
- op1 = parse_next_operand_given_its_operator(string, EVAL_OPERATOR_ABS, error);
- op1->precedence = eval_precedence(EVAL_OPERATOR_ABS);
- }
- else if(parse_open_subexpression(string)) {
- EVAL_NODE *sub = parse_full_expression(string, error);
- if(sub) {
- op1 = eval_node_alloc(1);
- op1->operator = EVAL_OPERATOR_EXPRESSION_OPEN;
- op1->precedence = eval_precedence(EVAL_OPERATOR_EXPRESSION_OPEN);
- eval_node_set_value_to_node(op1, 0, sub);
- if(!parse_close_subexpression(string)) {
- *error = EVAL_ERROR_MISSING_CLOSE_SUBEXPRESSION;
- eval_node_free(op1);
- return NULL;
- }
- }
- }
- else if(parse_variable(string, variable_buffer, EVAL_MAX_VARIABLE_NAME_LENGTH)) {
- op1 = eval_node_alloc(1);
- op1->operator = EVAL_OPERATOR_NOP;
- eval_node_set_value_to_variable(op1, 0, variable_buffer);
- }
- else if(parse_constant(string, &number)) {
- op1 = eval_node_alloc(1);
- op1->operator = EVAL_OPERATOR_NOP;
- eval_node_set_value_to_constant(op1, 0, number);
- }
- else if(**string)
- *error = EVAL_ERROR_UNKNOWN_OPERAND;
- else
- *error = EVAL_ERROR_MISSING_OPERAND;
-
- return op1;
-}
-
-// parse an operator and the rest of the expression
-// precedence processing is handled here
-static inline EVAL_NODE *parse_rest_of_expression(const char **string, int *error, EVAL_NODE *op1) {
- EVAL_NODE *op2 = NULL;
- unsigned char operator;
- int precedence;
-
- operator = parse_operator(string, &precedence);
- skip_spaces(string);
-
- if(operator != EVAL_OPERATOR_NOP) {
- op2 = parse_one_full_operand(string, error);
- if(!op2) {
- // error is already reported
- eval_node_free(op1);
- return NULL;
- }
-
- EVAL_NODE *op = eval_node_alloc(operators[operator].parameters);
- op->operator = operator;
- op->precedence = precedence;
-
- if(operator == EVAL_OPERATOR_IF_THEN_ELSE && op->count == 3) {
- skip_spaces(string);
-
- if(**string != ':') {
- eval_node_free(op);
- eval_node_free(op1);
- eval_node_free(op2);
- *error = EVAL_ERROR_IF_THEN_ELSE_MISSING_ELSE;
- return NULL;
- }
- (*string)++;
-
- skip_spaces(string);
-
- EVAL_NODE *op3 = parse_one_full_operand(string, error);
- if(!op3) {
- eval_node_free(op);
- eval_node_free(op1);
- eval_node_free(op2);
- // error is already reported
- return NULL;
- }
-
- eval_node_set_value_to_node(op, 2, op3);
- }
-
- eval_node_set_value_to_node(op, 1, op2);
-
- // precedence processing
- // if this operator has a higher precedence compared to its next
- // put the next operator on top of us (top = evaluated later)
- // function recursion does the rest...
- if(op->precedence > op1->precedence && op1->count == 2 && op1->operator != '(' && op1->ops[1].type == EVAL_VALUE_EXPRESSION) {
- eval_node_set_value_to_node(op, 0, op1->ops[1].expression);
- op1->ops[1].expression = op;
- op = op1;
- }
- else
- eval_node_set_value_to_node(op, 0, op1);
-
- return parse_rest_of_expression(string, error, op);
- }
- else if(**string == ')') {
- ;
- }
- else if(**string) {
- eval_node_free(op1);
- op1 = NULL;
- *error = EVAL_ERROR_MISSING_OPERATOR;
- }
-
- return op1;
-}
-
-// high level function to parse an expression or a sub-expression
-static inline EVAL_NODE *parse_full_expression(const char **string, int *error) {
- EVAL_NODE *op1 = parse_one_full_operand(string, error);
- if(!op1) {
- *error = EVAL_ERROR_MISSING_OPERAND;
- return NULL;
- }
-
- return parse_rest_of_expression(string, error, op1);
-}
-
-// ----------------------------------------------------------------------------
-// public API
-
-int expression_evaluate(EVAL_EXPRESSION *expression) {
- expression->error = EVAL_ERROR_OK;
-
- buffer_reset(expression->error_msg);
- expression->result = eval_node(expression, (EVAL_NODE *)expression->nodes, &expression->error);
-
- if(unlikely(isnan(expression->result))) {
- if(expression->error == EVAL_ERROR_OK)
- expression->error = EVAL_ERROR_VALUE_IS_NAN;
- }
- else if(unlikely(isinf(expression->result))) {
- if(expression->error == EVAL_ERROR_OK)
- expression->error = EVAL_ERROR_VALUE_IS_INFINITE;
- }
- else if(unlikely(expression->error == EVAL_ERROR_UNKNOWN_VARIABLE)) {
- // although there is an unknown variable
- // the expression was evaluated successfully
- expression->error = EVAL_ERROR_OK;
- }
-
- if(expression->error != EVAL_ERROR_OK) {
- expression->result = NAN;
-
- if(buffer_strlen(expression->error_msg))
- buffer_strcat(expression->error_msg, "; ");
-
- buffer_sprintf(expression->error_msg, "failed to evaluate expression with error %d (%s)", expression->error, expression_strerror(expression->error));
- return 0;
- }
-
- return 1;
-}
-
-EVAL_EXPRESSION *expression_parse(const char *string, const char **failed_at, int *error) {
- const char *s = string;
- int err = EVAL_ERROR_OK;
-
- EVAL_NODE *op = parse_full_expression(&s, &err);
-
- if(*s) {
- if(op) {
- eval_node_free(op);
- op = NULL;
- }
- err = EVAL_ERROR_REMAINING_GARBAGE;
- }
-
- if (failed_at) *failed_at = s;
- if (error) *error = err;
-
- if(!op) {
- unsigned long pos = s - string + 1;
- netdata_log_error("failed to parse expression '%s': %s at character %lu (i.e.: '%s').", string, expression_strerror(err), pos, s);
- return NULL;
- }
-
- BUFFER *out = buffer_create(1024, NULL);
- print_parsed_as_node(out, op, &err);
- if(err != EVAL_ERROR_OK) {
- netdata_log_error("failed to re-generate expression '%s' with reason: %s", string, expression_strerror(err));
- eval_node_free(op);
- buffer_free(out);
- return NULL;
- }
-
- EVAL_EXPRESSION *exp = callocz(1, sizeof(EVAL_EXPRESSION));
-
- exp->source = strdupz(string);
- exp->parsed_as = strdupz(buffer_tostring(out));
- buffer_free(out);
-
- exp->error_msg = buffer_create(100, NULL);
- exp->nodes = (void *)op;
-
- return exp;
-}
-
-void expression_free(EVAL_EXPRESSION *expression) {
- if(!expression) return;
-
- if(expression->nodes) eval_node_free((EVAL_NODE *)expression->nodes);
- freez((void *)expression->source);
- freez((void *)expression->parsed_as);
- buffer_free(expression->error_msg);
- freez(expression);
-}
-
-const char *expression_strerror(int error) {
- switch(error) {
- case EVAL_ERROR_OK:
- return "success";
-
- case EVAL_ERROR_MISSING_CLOSE_SUBEXPRESSION:
- return "missing closing parenthesis";
-
- case EVAL_ERROR_UNKNOWN_OPERAND:
- return "unknown operand";
-
- case EVAL_ERROR_MISSING_OPERAND:
- return "expected operand";
-
- case EVAL_ERROR_MISSING_OPERATOR:
- return "expected operator";
-
- case EVAL_ERROR_REMAINING_GARBAGE:
- return "remaining characters after expression";
-
- case EVAL_ERROR_INVALID_VALUE:
- return "invalid value structure - internal error";
-
- case EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS:
- return "wrong number of operands for operation - internal error";
-
- case EVAL_ERROR_VALUE_IS_NAN:
- return "value is unset";
-
- case EVAL_ERROR_VALUE_IS_INFINITE:
- return "computed value is infinite";
-
- case EVAL_ERROR_UNKNOWN_VARIABLE:
- return "undefined variable";
-
- case EVAL_ERROR_IF_THEN_ELSE_MISSING_ELSE:
- return "missing second sub-expression of inline conditional";
-
- default:
- return "unknown error";
- }
-}
diff --git a/libnetdata/eval/eval.h b/libnetdata/eval/eval.h
deleted file mode 100644
index 05a26936b..000000000
--- a/libnetdata/eval/eval.h
+++ /dev/null
@@ -1,87 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_EVAL_H
-#define NETDATA_EVAL_H 1
-
-#include "../libnetdata.h"
-
-#define EVAL_MAX_VARIABLE_NAME_LENGTH 300
-
-typedef enum rrdcalc_status {
- RRDCALC_STATUS_REMOVED = -2,
- RRDCALC_STATUS_UNDEFINED = -1,
- RRDCALC_STATUS_UNINITIALIZED = 0,
- RRDCALC_STATUS_CLEAR = 1,
- RRDCALC_STATUS_RAISED = 2, // DO NOT CHANGE THESE NUMBERS
- RRDCALC_STATUS_WARNING = 3, // DO NOT CHANGE THESE NUMBERS
- RRDCALC_STATUS_CRITICAL = 4, // DO NOT CHANGE THESE NUMBERS
-} RRDCALC_STATUS;
-
-typedef struct eval_variable {
- STRING *name;
- struct eval_variable *next;
-} EVAL_VARIABLE;
-
-typedef struct eval_expression {
- const char *source;
- const char *parsed_as;
-
- RRDCALC_STATUS *status;
- NETDATA_DOUBLE *myself;
- time_t *after;
- time_t *before;
-
- NETDATA_DOUBLE result;
-
- int error;
- BUFFER *error_msg;
-
- // hidden EVAL_NODE *
- void *nodes;
-
- // custom data to be used for looking up variables
- struct rrdcalc *rrdcalc;
-} EVAL_EXPRESSION;
-
-#define EVAL_VALUE_INVALID 0
-#define EVAL_VALUE_NUMBER 1
-#define EVAL_VALUE_VARIABLE 2
-#define EVAL_VALUE_EXPRESSION 3
-
-// parsing and evaluation
-#define EVAL_ERROR_OK 0
-
-// parsing errors
-#define EVAL_ERROR_MISSING_CLOSE_SUBEXPRESSION 1
-#define EVAL_ERROR_UNKNOWN_OPERAND 2
-#define EVAL_ERROR_MISSING_OPERAND 3
-#define EVAL_ERROR_MISSING_OPERATOR 4
-#define EVAL_ERROR_REMAINING_GARBAGE 5
-#define EVAL_ERROR_IF_THEN_ELSE_MISSING_ELSE 6
-
-// evaluation errors
-#define EVAL_ERROR_INVALID_VALUE 101
-#define EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS 102
-#define EVAL_ERROR_VALUE_IS_NAN 103
-#define EVAL_ERROR_VALUE_IS_INFINITE 104
-#define EVAL_ERROR_UNKNOWN_VARIABLE 105
-
-// parse the given string as an expression and return:
-// a pointer to an expression if it parsed OK
-// NULL in which case the pointer to error has the error code
-EVAL_EXPRESSION *expression_parse(const char *string, const char **failed_at, int *error);
-
-// free all resources allocated for an expression
-void expression_free(EVAL_EXPRESSION *expression);
-
-// convert an error code to a message
-const char *expression_strerror(int error);
-
-// evaluate an expression and return
-// 1 = OK, the result is in: expression->result
-// 2 = FAILED, the error message is in: buffer_tostring(expression->error_msg)
-int expression_evaluate(EVAL_EXPRESSION *expression);
-
-int health_variable_lookup(STRING *variable, struct rrdcalc *rc, NETDATA_DOUBLE *result);
-
-#endif //NETDATA_EVAL_H
diff --git a/libnetdata/facets/Makefile.am b/libnetdata/facets/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/facets/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/functions_evloop/Makefile.am b/libnetdata/functions_evloop/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/functions_evloop/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/functions_evloop/functions_evloop.c b/libnetdata/functions_evloop/functions_evloop.c
deleted file mode 100644
index 044556ac6..000000000
--- a/libnetdata/functions_evloop/functions_evloop.c
+++ /dev/null
@@ -1,223 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "functions_evloop.h"
-
-#define MAX_FUNCTION_PARAMETERS 1024
-
-struct functions_evloop_worker_job {
- bool used;
- bool running;
- bool cancelled;
- char *cmd;
- const char *transaction;
- time_t timeout;
- functions_evloop_worker_execute_t cb;
-};
-
-struct rrd_functions_expectation {
- const char *function;
- size_t function_length;
- functions_evloop_worker_execute_t cb;
- time_t default_timeout;
- struct rrd_functions_expectation *prev, *next;
-};
-
-struct functions_evloop_globals {
- const char *tag;
-
- DICTIONARY *worker_queue;
- pthread_mutex_t worker_mutex;
- pthread_cond_t worker_cond_var;
- size_t workers;
-
- netdata_mutex_t *stdout_mutex;
- bool *plugin_should_exit;
-
- netdata_thread_t reader_thread;
- netdata_thread_t *worker_threads;
-
- struct rrd_functions_expectation *expectations;
-};
-
-static void *rrd_functions_worker_globals_worker_main(void *arg) {
- struct functions_evloop_globals *wg = arg;
-
- bool last_acquired = true;
- while (true) {
- pthread_mutex_lock(&wg->worker_mutex);
-
- if(dictionary_entries(wg->worker_queue) == 0 || !last_acquired)
- pthread_cond_wait(&wg->worker_cond_var, &wg->worker_mutex);
-
- const DICTIONARY_ITEM *acquired = NULL;
- struct functions_evloop_worker_job *j;
- dfe_start_write(wg->worker_queue, j) {
- if(j->running || j->cancelled)
- continue;
-
- acquired = dictionary_acquired_item_dup(wg->worker_queue, j_dfe.item);
- j->running = true;
- break;
- }
- dfe_done(j);
-
- pthread_mutex_unlock(&wg->worker_mutex);
-
- if(acquired) {
- ND_LOG_STACK lgs[] = {
- ND_LOG_FIELD_TXT(NDF_REQUEST, j->cmd),
- ND_LOG_FIELD_END(),
- };
- ND_LOG_STACK_PUSH(lgs);
-
- last_acquired = true;
- j = dictionary_acquired_item_value(acquired);
- j->cb(j->transaction, j->cmd, j->timeout, &j->cancelled);
- dictionary_del(wg->worker_queue, j->transaction);
- dictionary_acquired_item_release(wg->worker_queue, acquired);
- dictionary_garbage_collect(wg->worker_queue);
- }
- else
- last_acquired = false;
- }
- return NULL;
-}
-
-static void *rrd_functions_worker_globals_reader_main(void *arg) {
- struct functions_evloop_globals *wg = arg;
-
- char buffer[PLUGINSD_LINE_MAX + 1];
-
- char *s = NULL;
- while(!(*wg->plugin_should_exit) && (s = fgets(buffer, PLUGINSD_LINE_MAX, stdin))) {
-
- char *words[MAX_FUNCTION_PARAMETERS] = { NULL };
- size_t num_words = quoted_strings_splitter_pluginsd(buffer, words, MAX_FUNCTION_PARAMETERS);
-
- const char *keyword = get_word(words, num_words, 0);
-
- if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION) == 0) {
- char *transaction = get_word(words, num_words, 1);
- char *timeout_s = get_word(words, num_words, 2);
- char *function = get_word(words, num_words, 3);
-
- if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) {
- netdata_log_error("Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.",
- keyword,
- transaction?transaction:"(unset)",
- timeout_s?timeout_s:"(unset)",
- function?function:"(unset)");
- }
- else {
- int timeout = str2i(timeout_s);
-
- bool found = false;
- struct rrd_functions_expectation *we;
- for(we = wg->expectations; we ;we = we->next) {
- if(strncmp(function, we->function, we->function_length) == 0) {
- struct functions_evloop_worker_job t = {
- .cmd = strdupz(function),
- .transaction = strdupz(transaction),
- .running = false,
- .cancelled = false,
- .timeout = timeout > 0 ? timeout : we->default_timeout,
- .used = false,
- .cb = we->cb,
- };
- struct functions_evloop_worker_job *j = dictionary_set(wg->worker_queue, transaction, &t, sizeof(t));
- if(j->used) {
- netdata_log_error("Received duplicate function transaction '%s'", transaction);
- freez((void *)t.cmd);
- freez((void *)t.transaction);
- }
- else {
- found = true;
- j->used = true;
- pthread_cond_signal(&wg->worker_cond_var);
- }
- }
- }
-
- if(!found) {
- netdata_mutex_lock(wg->stdout_mutex);
- pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND,
- "No function with this name found.");
- netdata_mutex_unlock(wg->stdout_mutex);
- }
- }
- }
- else if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION_CANCEL) == 0) {
- char *transaction = get_word(words, num_words, 1);
- const DICTIONARY_ITEM *acquired = dictionary_get_and_acquire_item(wg->worker_queue, transaction);
- if(acquired) {
- struct functions_evloop_worker_job *j = dictionary_acquired_item_value(acquired);
- __atomic_store_n(&j->cancelled, true, __ATOMIC_RELAXED);
- dictionary_acquired_item_release(wg->worker_queue, acquired);
- dictionary_del(wg->worker_queue, transaction);
- dictionary_garbage_collect(wg->worker_queue);
- }
- else
- netdata_log_error("Received CANCEL for transaction '%s', but it not available here", transaction);
- }
- else
- netdata_log_error("Received unknown command: %s", keyword?keyword:"(unset)");
- }
-
- if(!s || feof(stdin) || ferror(stdin)) {
- *wg->plugin_should_exit = true;
- netdata_log_error("Received error on stdin.");
- }
-
- exit(1);
-}
-
-void worker_queue_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
- struct functions_evloop_worker_job *j = value;
- freez((void *)j->cmd);
- freez((void *)j->transaction);
-}
-
-struct functions_evloop_globals *functions_evloop_init(size_t worker_threads, const char *tag, netdata_mutex_t *stdout_mutex, bool *plugin_should_exit) {
- struct functions_evloop_globals *wg = callocz(1, sizeof(struct functions_evloop_globals));
-
- wg->worker_queue = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
- dictionary_register_delete_callback(wg->worker_queue, worker_queue_delete_cb, NULL);
-
- pthread_mutex_init(&wg->worker_mutex, NULL);
- pthread_cond_init(&wg->worker_cond_var, NULL);
-
- wg->plugin_should_exit = plugin_should_exit;
- wg->stdout_mutex = stdout_mutex;
- wg->workers = worker_threads;
- wg->worker_threads = callocz(wg->workers, sizeof(netdata_thread_t ));
- wg->tag = tag;
-
- char tag_buffer[NETDATA_THREAD_TAG_MAX + 1];
- snprintfz(tag_buffer, NETDATA_THREAD_TAG_MAX, "%s_READER", wg->tag);
- netdata_thread_create(&wg->reader_thread, tag_buffer, NETDATA_THREAD_OPTION_DONT_LOG,
- rrd_functions_worker_globals_reader_main, wg);
-
- for(size_t i = 0; i < wg->workers ; i++) {
- snprintfz(tag_buffer, NETDATA_THREAD_TAG_MAX, "%s_WORK[%zu]", wg->tag, i+1);
- netdata_thread_create(&wg->worker_threads[i], tag_buffer, NETDATA_THREAD_OPTION_DONT_LOG,
- rrd_functions_worker_globals_worker_main, wg);
- }
-
- return wg;
-}
-
-void functions_evloop_add_function(struct functions_evloop_globals *wg, const char *function, functions_evloop_worker_execute_t cb, time_t default_timeout) {
- struct rrd_functions_expectation *we = callocz(1, sizeof(*we));
- we->function = function;
- we->function_length = strlen(we->function);
- we->cb = cb;
- we->default_timeout = default_timeout;
- DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(wg->expectations, we, prev, next);
-}
-
-void functions_evloop_cancel_threads(struct functions_evloop_globals *wg){
- for(size_t i = 0; i < wg->workers ; i++)
- netdata_thread_cancel(wg->worker_threads[i]);
-
- netdata_thread_cancel(wg->reader_thread);
-}
diff --git a/libnetdata/functions_evloop/functions_evloop.h b/libnetdata/functions_evloop/functions_evloop.h
deleted file mode 100644
index e5e83e95e..000000000
--- a/libnetdata/functions_evloop/functions_evloop.h
+++ /dev/null
@@ -1,101 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_FUNCTIONS_EVLOOP_H
-#define NETDATA_FUNCTIONS_EVLOOP_H
-
-#include "../libnetdata.h"
-
-#define PLUGINSD_KEYWORD_CHART "CHART"
-#define PLUGINSD_KEYWORD_CHART_DEFINITION_END "CHART_DEFINITION_END"
-#define PLUGINSD_KEYWORD_DIMENSION "DIMENSION"
-#define PLUGINSD_KEYWORD_BEGIN "BEGIN"
-#define PLUGINSD_KEYWORD_SET "SET"
-#define PLUGINSD_KEYWORD_END "END"
-#define PLUGINSD_KEYWORD_FLUSH "FLUSH"
-#define PLUGINSD_KEYWORD_DISABLE "DISABLE"
-#define PLUGINSD_KEYWORD_VARIABLE "VARIABLE"
-#define PLUGINSD_KEYWORD_LABEL "LABEL"
-#define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE"
-#define PLUGINSD_KEYWORD_CLABEL "CLABEL"
-#define PLUGINSD_KEYWORD_CLABEL_COMMIT "CLABEL_COMMIT"
-#define PLUGINSD_KEYWORD_FUNCTION "FUNCTION"
-#define PLUGINSD_KEYWORD_FUNCTION_CANCEL "FUNCTION_CANCEL"
-#define PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN "FUNCTION_RESULT_BEGIN"
-#define PLUGINSD_KEYWORD_FUNCTION_RESULT_END "FUNCTION_RESULT_END"
-
-#define PLUGINSD_KEYWORD_REPLAY_CHART "REPLAY_CHART"
-#define PLUGINSD_KEYWORD_REPLAY_BEGIN "RBEGIN"
-#define PLUGINSD_KEYWORD_REPLAY_SET "RSET"
-#define PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE "RDSTATE"
-#define PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE "RSSTATE"
-#define PLUGINSD_KEYWORD_REPLAY_END "REND"
-
-#define PLUGINSD_KEYWORD_BEGIN_V2 "BEGIN2"
-#define PLUGINSD_KEYWORD_SET_V2 "SET2"
-#define PLUGINSD_KEYWORD_END_V2 "END2"
-
-#define PLUGINSD_KEYWORD_HOST_DEFINE "HOST_DEFINE"
-#define PLUGINSD_KEYWORD_HOST_DEFINE_END "HOST_DEFINE_END"
-#define PLUGINSD_KEYWORD_HOST_LABEL "HOST_LABEL"
-#define PLUGINSD_KEYWORD_HOST "HOST"
-
-#define PLUGINSD_KEYWORD_DYNCFG_ENABLE "DYNCFG_ENABLE"
-#define PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE "DYNCFG_REGISTER_MODULE"
-
-#define PLUGINSD_KEYWORD_REPORT_JOB_STATUS "REPORT_JOB_STATUS"
-
-#define PLUGINSD_KEYWORD_EXIT "EXIT"
-
-#define PLUGINSD_KEYWORD_SLOT "SLOT" // to change the length of this, update pluginsd_extract_chart_slot() too
-
-#define PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT 10 // seconds
-
-typedef void (*functions_evloop_worker_execute_t)(const char *transaction, char *function, int timeout, bool *cancelled);
-struct functions_evloop_worker_job;
-struct functions_evloop_globals *functions_evloop_init(size_t worker_threads, const char *tag, netdata_mutex_t *stdout_mutex, bool *plugin_should_exit);
-void functions_evloop_add_function(struct functions_evloop_globals *wg, const char *function, functions_evloop_worker_execute_t cb, time_t default_timeout);
-void functions_evloop_cancel_threads(struct functions_evloop_globals *wg);
-
-
-#define pluginsd_function_result_begin_to_buffer(wb, transaction, code, content_type, expires) \
- buffer_sprintf(wb \
- , PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " \"%s\" %d \"%s\" %ld\n" \
- , (transaction) ? (transaction) : "" \
- , (int)(code) \
- , (content_type) ? (content_type) : "" \
- , (long int)(expires) \
- )
-
-#define pluginsd_function_result_end_to_buffer(wb) \
- buffer_strcat(wb, "\n" PLUGINSD_KEYWORD_FUNCTION_RESULT_END "\n")
-
-#define pluginsd_function_result_begin_to_stdout(transaction, code, content_type, expires) \
- fprintf(stdout \
- , PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " \"%s\" %d \"%s\" %ld\n" \
- , (transaction) ? (transaction) : "" \
- , (int)(code) \
- , (content_type) ? (content_type) : "" \
- , (long int)(expires) \
- )
-
-#define pluginsd_function_result_end_to_stdout() \
- fprintf(stdout, "\n" PLUGINSD_KEYWORD_FUNCTION_RESULT_END "\n")
-
-static inline void pluginsd_function_json_error_to_stdout(const char *transaction, int code, const char *msg) {
- char buffer[PLUGINSD_LINE_MAX + 1];
- json_escape_string(buffer, msg, PLUGINSD_LINE_MAX);
-
- pluginsd_function_result_begin_to_stdout(transaction, code, "application/json", now_realtime_sec());
- fprintf(stdout, "{\"status\":%d,\"error_message\":\"%s\"}", code, buffer);
- pluginsd_function_result_end_to_stdout();
- fflush(stdout);
-}
-
-static inline void pluginsd_function_result_to_stdout(const char *transaction, int code, const char *content_type, time_t expires, BUFFER *result) {
- pluginsd_function_result_begin_to_stdout(transaction, code, content_type, expires);
- fwrite(buffer_tostring(result), buffer_strlen(result), 1, stdout);
- pluginsd_function_result_end_to_stdout();
- fflush(stdout);
-}
-
-#endif //NETDATA_FUNCTIONS_EVLOOP_H
diff --git a/libnetdata/health/Makefile.am b/libnetdata/health/Makefile.am
deleted file mode 100644
index 643458b74..000000000
--- a/libnetdata/health/Makefile.am
+++ /dev/null
@@ -1,7 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- $(NULL)
diff --git a/libnetdata/health/health.c b/libnetdata/health/health.c
deleted file mode 100644
index 2b3b8f7d6..000000000
--- a/libnetdata/health/health.c
+++ /dev/null
@@ -1,169 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "health.h"
-
-SILENCERS *silencers;
-
-/**
- * Create Silencer
- *
- * Allocate a new silencer to Netdata.
- *
- * @return It returns the address off the silencer on success and NULL otherwise
- */
-SILENCER *create_silencer(void) {
- SILENCER *t = callocz(1, sizeof(SILENCER));
- netdata_log_debug(D_HEALTH, "HEALTH command API: Created empty silencer");
-
- return t;
-}
-
-/**
- * Health Silencers add
- *
- * Add more one silencer to the list of silencers.
- *
- * @param silencer
- */
-void health_silencers_add(SILENCER *silencer) {
- // Add the created instance to the linked list in silencers
- silencer->next = silencers->silencers;
- silencers->silencers = silencer;
- netdata_log_debug(
- D_HEALTH,
- "HEALTH command API: Added silencer %s:%s:%s:%s",
- silencer->alarms,
- silencer->charts,
- silencer->contexts,
- silencer->hosts);
-}
-
-/**
- * Silencers Add Parameter
- *
- * Create a new silencer and adjust the variables
- *
- * @param silencer a pointer to the silencer that will be adjusted
- * @param key the key value sent by client
- * @param value the value sent to the key
- *
- * @return It returns the silencer configured on success and NULL otherwise
- */
-SILENCER *health_silencers_addparam(SILENCER *silencer, char *key, char *value) {
- static uint32_t
- hash_alarm = 0,
- hash_template = 0,
- hash_chart = 0,
- hash_context = 0,
- hash_host = 0;
-
- if (unlikely(!hash_alarm)) {
- hash_alarm = simple_uhash(HEALTH_ALARM_KEY);
- hash_template = simple_uhash(HEALTH_TEMPLATE_KEY);
- hash_chart = simple_uhash(HEALTH_CHART_KEY);
- hash_context = simple_uhash(HEALTH_CONTEXT_KEY);
- hash_host = simple_uhash(HEALTH_HOST_KEY);
- }
-
- uint32_t hash = simple_uhash(key);
- if (unlikely(silencer == NULL)) {
- if (
- (hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) ||
- (hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) ||
- (hash == hash_chart && !strcasecmp(key, HEALTH_CHART_KEY)) ||
- (hash == hash_context && !strcasecmp(key, HEALTH_CONTEXT_KEY)) ||
- (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY))
- ) {
- silencer = create_silencer();
- }
- }
-
- if (hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
- silencer->alarms = strdupz(value);
- silencer->alarms_pattern = simple_pattern_create(silencer->alarms, NULL, SIMPLE_PATTERN_EXACT, true);
- } else if (hash == hash_chart && !strcasecmp(key, HEALTH_CHART_KEY)) {
- silencer->charts = strdupz(value);
- silencer->charts_pattern = simple_pattern_create(silencer->charts, NULL, SIMPLE_PATTERN_EXACT, true);
- } else if (hash == hash_context && !strcasecmp(key, HEALTH_CONTEXT_KEY)) {
- silencer->contexts = strdupz(value);
- silencer->contexts_pattern = simple_pattern_create(silencer->contexts, NULL, SIMPLE_PATTERN_EXACT, true);
- } else if (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) {
- silencer->hosts = strdupz(value);
- silencer->hosts_pattern = simple_pattern_create(silencer->hosts, NULL, SIMPLE_PATTERN_EXACT, true);
- }
-
- return silencer;
-}
-
-/**
- * JSON Read Callback
- *
- * Callback called by netdata to create the silencer.
- *
- * @param e the main json structure
- *
- * @return It always return 0.
- */
-int health_silencers_json_read_callback(JSON_ENTRY *e)
-{
- switch(e->type) {
- case JSON_OBJECT:
-#ifndef ENABLE_JSONC
- e->callback_function = health_silencers_json_read_callback;
- if(strcmp(e->name,"")) {
- // init silencer
- netdata_log_debug(D_HEALTH, "JSON: Got object with a name, initializing new silencer for %s",e->name);
-#endif
- e->callback_data = create_silencer();
- if(e->callback_data) {
- health_silencers_add(e->callback_data);
- }
-#ifndef ENABLE_JSONC
- }
-#endif
- break;
-
- case JSON_ARRAY:
- e->callback_function = health_silencers_json_read_callback;
- break;
-
- case JSON_STRING:
- if(!strcmp(e->name,"type")) {
- netdata_log_debug(D_HEALTH, "JSON: Processing type=%s",e->data.string);
- if (!strcmp(e->data.string,"SILENCE")) silencers->stype = STYPE_SILENCE_NOTIFICATIONS;
- else if (!strcmp(e->data.string,"DISABLE")) silencers->stype = STYPE_DISABLE_ALARMS;
- } else {
- netdata_log_debug(D_HEALTH, "JSON: Adding %s=%s", e->name, e->data.string);
- if (e->callback_data)
- (void)health_silencers_addparam(e->callback_data, e->name, e->data.string);
- }
- break;
-
- case JSON_BOOLEAN:
- netdata_log_debug(D_HEALTH, "JSON: Processing all_alarms");
- silencers->all_alarms=e->data.boolean?1:0;
- break;
-
- case JSON_NUMBER:
- case JSON_NULL:
- break;
- }
-
- return 0;
-}
-
-/**
- * Initialize Global Silencers
- *
- * Initialize the silencer for the whole netdata system.
- *
- * @return It returns 0 on success and -1 otherwise
- */
-int health_initialize_global_silencers() {
- silencers = mallocz(sizeof(SILENCERS));
- silencers->all_alarms = 0;
- silencers->stype = STYPE_NONE;
- silencers->silencers = NULL;
-
- return 0;
-}
diff --git a/libnetdata/health/health.h b/libnetdata/health/health.h
deleted file mode 100644
index 31173fe61..000000000
--- a/libnetdata/health/health.h
+++ /dev/null
@@ -1,53 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_HEALTH_LIB
-# define NETDATA_HEALTH_LIB 1
-
-# include "../libnetdata.h"
-
-#define HEALTH_ALARM_KEY "alarm"
-#define HEALTH_TEMPLATE_KEY "template"
-#define HEALTH_CONTEXT_KEY "context"
-#define HEALTH_CHART_KEY "chart"
-#define HEALTH_HOST_KEY "hosts"
-#define HEALTH_OS_KEY "os"
-#define HEALTH_LOOKUP_KEY "lookup"
-#define HEALTH_CALC_KEY "calc"
-
-typedef struct silencer {
- char *alarms;
- SIMPLE_PATTERN *alarms_pattern;
-
- char *hosts;
- SIMPLE_PATTERN *hosts_pattern;
-
- char *contexts;
- SIMPLE_PATTERN *contexts_pattern;
-
- char *charts;
- SIMPLE_PATTERN *charts_pattern;
-
- struct silencer *next;
-} SILENCER;
-
-typedef enum silence_type {
- STYPE_NONE,
- STYPE_DISABLE_ALARMS,
- STYPE_SILENCE_NOTIFICATIONS
-} SILENCE_TYPE;
-
-typedef struct silencers {
- int all_alarms;
- SILENCE_TYPE stype;
- SILENCER *silencers;
-} SILENCERS;
-
-extern SILENCERS *silencers;
-
-SILENCER *create_silencer(void);
-int health_silencers_json_read_callback(JSON_ENTRY *e);
-void health_silencers_add(SILENCER *silencer);
-SILENCER * health_silencers_addparam(SILENCER *silencer, char *key, char *value);
-int health_initialize_global_silencers();
-
-#endif
diff --git a/libnetdata/http/http_defs.h b/libnetdata/http/http_defs.h
deleted file mode 100644
index 635f703e4..000000000
--- a/libnetdata/http/http_defs.h
+++ /dev/null
@@ -1,47 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_HTTP_DEFS_H
-#define NETDATA_HTTP_DEFS_H
-
-#define HTTP_1_1 " HTTP/1.1"
-#define HTTP_HDR_END "\r\n\r\n"
-#define HTTP_ENDL "\r\n"
-
-// HTTP_CODES 1XX
-#define HTTP_RESP_SWITCH_PROTO 101
-
-// HTTP_CODES 2XX Success
-#define HTTP_RESP_OK 200
-
-// HTTP_CODES 3XX Redirections
-#define HTTP_RESP_MOVED_PERM 301
-#define HTTP_RESP_NOT_MODIFIED 304
-#define HTTP_RESP_REDIR_TEMP 307
-#define HTTP_RESP_REDIR_PERM 308
-#define HTTP_RESP_HTTPS_UPGRADE 399
-
-// HTTP_CODES 4XX Client Errors
-#define HTTP_RESP_BAD_REQUEST 400
-#define HTTP_RESP_UNAUTHORIZED 401
-#define HTTP_RESP_FORBIDDEN 403
-#define HTTP_RESP_NOT_FOUND 404
-#define HTTP_RESP_METHOD_NOT_ALLOWED 405
-#define HTTP_RESP_METHOD_NOT_ALLOWED_STR "Method Not Allowed"
-#define HTTP_RESP_CONFLICT 409
-#define HTTP_RESP_PRECOND_FAIL 412
-#define HTTP_RESP_CONTENT_TOO_LONG 413
-#define HTTP_RESP_CLIENT_CLOSED_REQUEST 499 // nginx's enxtension to the standard
-
-// HTTP_CODES 5XX Server Errors
-#define HTTP_RESP_INTERNAL_SERVER_ERROR 500
-#define HTTP_RESP_INTERNAL_SERVER_ERROR_STR "Internal Server Error"
-#define HTTP_RESP_SERVICE_UNAVAILABLE 503
-#define HTTP_RESP_GATEWAY_TIMEOUT 504
-#define HTTP_RESP_BACKEND_RESPONSE_INVALID 591
-
-#define HTTP_METHOD_GET (1)
-#define HTTP_METHOD_POST (2)
-#define HTTP_METHOD_PUT (3)
-#define HTTP_METHOD_DELETE (4)
-
-#endif /* NETDATA_HTTP_DEFS_H */
diff --git a/libnetdata/json/Makefile.am b/libnetdata/json/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/json/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/json/README.md b/libnetdata/json/README.md
deleted file mode 100644
index 1c49739be..000000000
--- a/libnetdata/json/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-<!--
-title: "json"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/json/README.md
-sidebar_label: "json"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# json
-
-`json` contains a parser for json strings, based on `jsmn` (<https://github.com/zserge/jsmn>), but case you have installed the JSON-C library, the installation script will prefer it, you can also force its use with `--enable-jsonc` in the compilation time.
-
-
diff --git a/libnetdata/json/json.c b/libnetdata/json/json.c
deleted file mode 100644
index ec1452911..000000000
--- a/libnetdata/json/json.c
+++ /dev/null
@@ -1,557 +0,0 @@
-#include "jsmn.h"
-#include "../libnetdata.h"
-#include "json.h"
-#include "libnetdata/libnetdata.h"
-#include "../../health/health.h"
-
-#define JSON_TOKENS 1024
-
-int json_tokens = JSON_TOKENS;
-
-/**
- * Json Tokenise
- *
- * Map the string given inside tokens.
- *
- * @param js is the string used to create the tokens
- * @param len is the string length
- * @param count the number of tokens present in the string
- *
- * @return it returns the json parsed in tokens
- */
-#ifdef ENABLE_JSONC
-json_object *json_tokenise(char *js) {
- if(!js) {
- netdata_log_error("JSON: json string is empty.");
- return NULL;
- }
-
- json_object *token = json_tokener_parse(js);
- if(!token) {
- netdata_log_error("JSON: Invalid json string.");
- return NULL;
- }
-
- return token;
-}
-#else
-jsmntok_t *json_tokenise(char *js, size_t len, size_t *count)
-{
- int n = json_tokens;
- if(!js || !len) {
- netdata_log_error("JSON: json string is empty.");
- return NULL;
- }
-
- jsmn_parser parser;
- jsmn_init(&parser);
-
- jsmntok_t *tokens = mallocz(sizeof(jsmntok_t) * n);
- if(!tokens) return NULL;
-
- int ret = jsmn_parse(&parser, js, len, tokens, n);
- while (ret == JSMN_ERROR_NOMEM) {
- n *= 2;
- jsmntok_t *new = reallocz(tokens, sizeof(jsmntok_t) * n);
- if(!new) {
- freez(tokens);
- return NULL;
- }
- tokens = new;
- ret = jsmn_parse(&parser, js, len, tokens, n);
- }
-
- if (ret == JSMN_ERROR_INVAL) {
- netdata_log_error("JSON: Invalid json string.");
- freez(tokens);
- return NULL;
- }
- else if (ret == JSMN_ERROR_PART) {
- netdata_log_error("JSON: Truncated JSON string.");
- freez(tokens);
- return NULL;
- }
-
- if(count) *count = (size_t)ret;
-
- if(json_tokens < n) json_tokens = n;
- return tokens;
-}
-#endif
-
-/**
- * Callback Print
- *
- * Set callback print case necessary and wrinte an information inside a buffer to write in the log.
- *
- * @param e a pointer for a structure that has the complete information about json structure.
- *
- * @return It always return 0
- */
-int json_callback_print(JSON_ENTRY *e)
-{
- BUFFER *wb=buffer_create(300, NULL);
-
- buffer_sprintf(wb,"%s = ", e->name);
- char txt[50];
- switch(e->type) {
- case JSON_OBJECT:
- e->callback_function = json_callback_print;
- buffer_strcat(wb,"OBJECT");
- break;
-
- case JSON_ARRAY:
- e->callback_function = json_callback_print;
- sprintf(txt,"ARRAY[%lu]", (long unsigned int) e->data.items);
- buffer_strcat(wb, txt);
- break;
-
- case JSON_STRING:
- buffer_strcat(wb, e->data.string);
- break;
-
- case JSON_NUMBER:
- sprintf(txt, NETDATA_DOUBLE_FORMAT_AUTO, e->data.number);
- buffer_strcat(wb,txt);
-
- break;
-
- case JSON_BOOLEAN:
- buffer_strcat(wb, e->data.boolean?"TRUE":"FALSE");
- break;
-
- case JSON_NULL:
- buffer_strcat(wb,"NULL");
- break;
- }
- netdata_log_info("JSON: %s", buffer_tostring(wb));
- buffer_free(wb);
- return 0;
-}
-
-/**
- * JSONC Set String
- *
- * Set the string value of the structure JSON_ENTRY.
- *
- * @param e the output structure
- */
-static inline void json_jsonc_set_string(JSON_ENTRY *e,char *key,const char *value) {
- size_t len = strlen(key);
- if(len > JSON_NAME_LEN)
- len = JSON_NAME_LEN;
- e->type = JSON_STRING;
- memcpy(e->name,key,len);
- e->name[len] = 0x00;
- e->data.string = (char *) value;
-}
-
-
-#ifdef ENABLE_JSONC
-/**
- * JSONC set Boolean
- *
- * Set the boolean value of the structure JSON_ENTRY
- *
- * @param e the output structure
- * @param value the input value
- */
-static inline void json_jsonc_set_boolean(JSON_ENTRY *e,int value) {
- e->type = JSON_BOOLEAN;
- e->data.boolean = value;
-}
-
-static inline void json_jsonc_set_integer(JSON_ENTRY *e, char *key, int64_t value) {
- size_t len = strlen(key);
- if(len > JSON_NAME_LEN)
- len = JSON_NAME_LEN;
- e->type = JSON_NUMBER;
- memcpy(e->name, key, len);
- e->name[len] = 0;
- e->data.number = (NETDATA_DOUBLE)value;
-}
-
-/**
- * Parse Array
- *
- * Parse the array object.
- *
- * @param ptr the pointer for the object that we will parse.
- * @param callback_data additional data to be used together the callback function
- * @param callback_function function used to create a silencer.
- */
-static inline void json_jsonc_parse_array(json_object *ptr, void *callback_data,int (*callback_function)(struct json_entry *)) {
- int end = json_object_array_length(ptr);
- JSON_ENTRY e;
-
- if(end) {
- int i;
- i = 0;
-
- enum json_type type;
- do {
- json_object *jvalue = json_object_array_get_idx(ptr, i);
- if(jvalue) {
- e.callback_data = callback_data;
- e.type = JSON_OBJECT;
- callback_function(&e);
- json_object_object_foreach(jvalue, key, val) {
- type = json_object_get_type(val);
- if (type == json_type_array) {
- e.type = JSON_ARRAY;
- json_jsonc_parse_array(val, callback_data, callback_function);
- } else if (type == json_type_object) {
- json_walk(val,callback_data,callback_function);
- } else if (type == json_type_string) {
- json_jsonc_set_string(&e,key,json_object_get_string(val));
- callback_function(&e);
- } else if (type == json_type_boolean) {
- json_jsonc_set_boolean(&e,json_object_get_boolean(val));
- callback_function(&e);
- }
- }
- }
-
- } while (++i < end);
- }
-}
-#else
-
-/**
- * Walk string
- *
- * Set JSON_ENTRY to string and map the values from jsmntok_t.
- *
- * @param js the original string
- * @param t the tokens
- * @param start the first position
- * @param e the output structure.
- *
- * @return It always return 1
- */
-size_t json_walk_string(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e)
-{
- char old = js[t[start].end];
- js[t[start].end] = '\0';
- e->original_string = &js[t[start].start];
-
- e->type = JSON_STRING;
- e->data.string = e->original_string;
- if(e->callback_function) e->callback_function(e);
- js[t[start].end] = old;
- return 1;
-}
-
-/**
- * Walk Primitive
- *
- * Define the data type of the string
- *
- * @param js the original string
- * @param t the tokens
- * @param start the first position
- * @param e the output structure.
- *
- * @return It always return 1
- */
-size_t json_walk_primitive(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e)
-{
- char old = js[t[start].end];
- js[t[start].end] = '\0';
- e->original_string = &js[t[start].start];
-
- switch(e->original_string[0]) {
- case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7':
- case '8': case '9': case '-': case '.':
- e->type = JSON_NUMBER;
- e->data.number = strtold(e->original_string, NULL);
- break;
-
- case 't': case 'T':
- e->type = JSON_BOOLEAN;
- e->data.boolean = 1;
- break;
-
- case 'f': case 'F':
- e->type = JSON_BOOLEAN;
- e->data.boolean = 0;
- break;
-
- case 'n': case 'N':
- default:
- e->type = JSON_NULL;
- break;
- }
- if(e->callback_function) e->callback_function(e);
- js[t[start].end] = old;
- return 1;
-}
-
-/**
- * Array
- *
- * Measure the array length
- *
- * @param js the original string
- * @param t the tokens
- * @param nest the length of structure t
- * @param start the first position
- * @param e the structure with values and callback to be used inside the function.
- *
- * @return It returns the array length
- */
-size_t json_walk_array(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e)
-{
- JSON_ENTRY ne;
-
- char old = js[t[start].end];
- js[t[start].end] = '\0';
- ne.original_string = &js[t[start].start];
-
- memcpy(&ne, e, sizeof(JSON_ENTRY));
- ne.type = JSON_ARRAY;
- ne.data.items = t[start].size;
- ne.callback_function = e->callback_function;
- ne.name[0]='\0';
- ne.fullname[0]='\0';
- if(e->callback_function) e->callback_function(&ne);
- js[t[start].end] = old;
-
- size_t i, init = start, size = t[start].size;
-
- start++;
- for(i = 0; i < size ; i++) {
- ne.pos = i;
- if (strlen(e->name) > JSON_NAME_LEN - 24 || strlen(e->fullname) > JSON_FULLNAME_LEN -24) {
- netdata_log_info("JSON: JSON walk_array ignoring element with name:%s fullname:%s",e->name, e->fullname);
- continue;
- }
- snprintfz(ne.name, JSON_NAME_LEN, "%s[%lu]", e->name, i);
- snprintfz(ne.fullname, JSON_FULLNAME_LEN, "%s[%lu]", e->fullname, i);
-
- switch(t[start].type) {
- case JSMN_PRIMITIVE:
- start += json_walk_primitive(js, t, start, &ne);
- break;
-
- case JSMN_OBJECT:
- start += json_walk_object(js, t, nest + 1, start, &ne);
- break;
-
- case JSMN_ARRAY:
- start += json_walk_array(js, t, nest + 1, start, &ne);
- break;
-
- case JSMN_STRING:
- start += json_walk_string(js, t, start, &ne);
- break;
- }
- }
- return start - init;
-}
-
-/**
- * Object
- *
- * Measure the Object length
- *
- * @param js the original string
- * @param t the tokens
- * @param nest the length of structure t
- * @param start the first position
- * @param e the output structure.
- *
- * @return It returns the Object length
- */
-size_t json_walk_object(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e)
-{
- JSON_ENTRY ne = {
- .name = "",
- .fullname = "",
- .callback_data = NULL,
- .callback_function = NULL
- };
-
- char old = js[t[start].end];
- js[t[start].end] = '\0';
- ne.original_string = &js[t[start].start];
- memcpy(&ne, e, sizeof(JSON_ENTRY));
- ne.type = JSON_OBJECT;
- ne.callback_function = e->callback_function;
- if(e->callback_function) e->callback_function(&ne);
- js[t[start].end] = old;
-
- int key = 1;
- size_t i, init = start, size = t[start].size;
-
- start++;
- for(i = 0; i < size ; i++) {
- switch(t[start].type) {
- case JSMN_PRIMITIVE:
- start += json_walk_primitive(js, t, start, &ne);
- key = 1;
- break;
-
- case JSMN_OBJECT:
- start += json_walk_object(js, t, nest + 1, start, &ne);
- key = 1;
- break;
-
- case JSMN_ARRAY:
- start += json_walk_array(js, t, nest + 1, start, &ne);
- key = 1;
- break;
-
- case JSMN_STRING:
- default:
- if(key) {
- int len = t[start].end - t[start].start;
- if (unlikely(len>JSON_NAME_LEN)) len=JSON_NAME_LEN;
- strncpy(ne.name, &js[t[start].start], len);
- ne.name[len] = '\0';
- len=strlen(e->fullname) + strlen(e->fullname[0]?".":"") + strlen(ne.name);
- char *c = mallocz((len+1)*sizeof(char));
- sprintf(c,"%s%s%s", e->fullname, e->fullname[0]?".":"", ne.name);
- if (unlikely(len>JSON_FULLNAME_LEN)) len=JSON_FULLNAME_LEN;
- strncpy(ne.fullname, c, len);
- freez(c);
- start++;
- key = 0;
- }
- else {
- start += json_walk_string(js, t, start, &ne);
- key = 1;
- }
- break;
- }
- }
- return start - init;
-}
-#endif
-
-/**
- * Tree
- *
- * Call the correct walk function according its type.
- *
- * @param t the json object to work
- * @param callback_data additional data to be used together the callback function
- * @param callback_function function used to create a silencer.
- *
- * @return It always return 1
- */
-#ifdef ENABLE_JSONC
-size_t json_walk(json_object *t, void *callback_data, int (*callback_function)(struct json_entry *)) {
- JSON_ENTRY e;
-
- e.callback_data = callback_data;
- enum json_type type;
- json_object_object_foreach(t, key, val) {
- type = json_object_get_type(val);
- if (type == json_type_array) {
- e.type = JSON_ARRAY;
- json_jsonc_parse_array(val,NULL,health_silencers_json_read_callback);
- } else if (type == json_type_object) {
- e.type = JSON_OBJECT;
- } else if (type == json_type_string) {
- json_jsonc_set_string(&e,key,json_object_get_string(val));
- callback_function(&e);
- } else if (type == json_type_boolean) {
- json_jsonc_set_boolean(&e,json_object_get_boolean(val));
- callback_function(&e);
- } else if (type == json_type_int) {
- json_jsonc_set_integer(&e,key,json_object_get_int64(val));
- callback_function(&e);
- }
- }
-
- return 1;
-}
-#else
-/**
- * Tree
- *
- * Call the correct walk function according its type.
- *
- * @param js the original string
- * @param t the tokens
- * @param callback_data additional data to be used together the callback function
- * @param callback_function function used to create a silencer.
- *
- * @return It always return 1
- */
-size_t json_walk_tree(char *js, jsmntok_t *t, void *callback_data, int (*callback_function)(struct json_entry *))
-{
- JSON_ENTRY e = {
- .name = "",
- .fullname = "",
- .callback_data = callback_data,
- .callback_function = callback_function
- };
-
- switch (t[0].type) {
- case JSMN_OBJECT:
- e.type = JSON_OBJECT;
- json_walk_object(js, t, 0, 0, &e);
- break;
-
- case JSMN_ARRAY:
- e.type = JSON_ARRAY;
- json_walk_array(js, t, 0, 0, &e);
- break;
-
- case JSMN_PRIMITIVE:
- case JSMN_STRING:
- break;
- }
-
- return 1;
-}
-#endif
-
-/**
- * JSON Parse
- *
- * Parse the json message with the callback function
- *
- * @param js the string that the callback function will parse
- * @param callback_data additional data to be used together the callback function
- * @param callback_function function used to create a silencer.
- *
- * @return JSON_OK case everything happened as expected, JSON_CANNOT_PARSE case there were errors in the
- * parsing process and JSON_CANNOT_DOWNLOAD case the string given(js) is NULL.
- */
-int json_parse(char *js, void *callback_data, int (*callback_function)(JSON_ENTRY *))
-{
- if(js) {
-#ifdef ENABLE_JSONC
- json_object *tokens = json_tokenise(js);
-#else
- size_t count;
- jsmntok_t *tokens = json_tokenise(js, strlen(js), &count);
-#endif
-
- if(tokens) {
-#ifdef ENABLE_JSONC
- json_walk(tokens, callback_data, callback_function);
- json_object_put(tokens);
-#else
- json_walk_tree(js, tokens, callback_data, callback_function);
- freez(tokens);
-#endif
- return JSON_OK;
- }
-
- return JSON_CANNOT_PARSE;
- }
-
- return JSON_CANNOT_DOWNLOAD;
-}
-
-/*
-int json_test(char *str)
-{
- return json_parse(str, NULL, json_callback_print);
-}
- */
-
diff --git a/libnetdata/json/json.h b/libnetdata/json/json.h
deleted file mode 100644
index b43f06b50..000000000
--- a/libnetdata/json/json.h
+++ /dev/null
@@ -1,77 +0,0 @@
-#ifndef CHECKIN_JSON_H
-#define CHECKIN_JSON_H 1
-
-
-#if ENABLE_JSONC
-#include <json-c/json.h>
-// fix an older json-c bug
-// https://github.com/json-c/json-c/issues/135
-#ifdef error_description
-#undef error_description
-#endif // error_description
-#endif // ENABLE_JSONC
-
-#include "jsmn.h"
-
-//https://www.ibm.com/support/knowledgecenter/en/SS9H2Y_7.6.0/com.ibm.dp.doc/json_parserlimits.html
-#define JSON_NAME_LEN 256
-#define JSON_FULLNAME_LEN 1024
-
-typedef enum {
- JSON_OBJECT = 0,
- JSON_ARRAY = 1,
- JSON_STRING = 2,
- JSON_NUMBER = 3,
- JSON_BOOLEAN = 4,
- JSON_NULL = 5,
-} JSON_ENTRY_TYPE;
-
-typedef struct json_entry {
- JSON_ENTRY_TYPE type;
- char name[JSON_NAME_LEN + 1];
- char fullname[JSON_FULLNAME_LEN + 1];
- union {
- char *string; // type == JSON_STRING
- NETDATA_DOUBLE number; // type == JSON_NUMBER
- int boolean; // type == JSON_BOOLEAN
- size_t items; // type == JSON_ARRAY
- } data;
- size_t pos; // the position of this item in its parent
-
- char *original_string;
-
- void *callback_data;
- int (*callback_function)(struct json_entry *);
-} JSON_ENTRY;
-
-// ----------------------------------------------------------------------------
-// public functions
-
-#define JSON_OK 0
-#define JSON_CANNOT_DOWNLOAD 1
-#define JSON_CANNOT_PARSE 2
-
-int json_parse(char *js, void *callback_data, int (*callback_function)(JSON_ENTRY *));
-
-
-// ----------------------------------------------------------------------------
-// private functions
-
-#ifdef ENABLE_JSONC
-json_object *json_tokenise(char *js);
-size_t json_walk(json_object *t, void *callback_data, int (*callback_function)(struct json_entry *));
-#else
-jsmntok_t *json_tokenise(char *js, size_t len, size_t *count);
-size_t json_walk_tree(char *js, jsmntok_t *t, void *callback_data, int (*callback_function)(struct json_entry *));
-#endif
-
-size_t json_walk_object(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e);
-size_t json_walk_array(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_ENTRY *e);
-size_t json_walk_string(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e);
-size_t json_walk_primitive(char *js, jsmntok_t *t, size_t start, JSON_ENTRY *e);
-
-int json_callback_print(JSON_ENTRY *e);
-
-
-
-#endif \ No newline at end of file
diff --git a/libnetdata/july/Makefile.am b/libnetdata/july/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/july/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/july/README.md b/libnetdata/july/README.md
deleted file mode 100644
index 995849128..000000000
--- a/libnetdata/july/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-<!--
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/july/README.md
-sidebar_label: "July interface"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-
-# July
-
-An interface similar to `Judy` that uses minimal allocations (that can be cached)
-for items that are mainly appended (just a few insertions in the middle)
-
diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h
deleted file mode 100644
index d2ecfa156..000000000
--- a/libnetdata/libnetdata.h
+++ /dev/null
@@ -1,916 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_LIB_H
-#define NETDATA_LIB_H 1
-
-# ifdef __cplusplus
-extern "C" {
-# endif
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#if defined(ENABLE_BROTLIENC) && defined(ENABLE_BROTLIDEC)
-#define ENABLE_BROTLI 1
-#endif
-
-#ifdef ENABLE_OPENSSL
-#define ENABLE_HTTPS 1
-#endif
-
-#ifdef HAVE_LIBDATACHANNEL
-#define ENABLE_WEBRTC 1
-#endif
-
-#define STRINGIFY(x) #x
-#define TOSTRING(x) STRINGIFY(x)
-
-#define JUDYHS_INDEX_SIZE_ESTIMATE(key_bytes) (((key_bytes) + sizeof(Word_t) - 1) / sizeof(Word_t) * 4)
-
-#if defined(NETDATA_DEV_MODE) && !defined(NETDATA_INTERNAL_CHECKS)
-#define NETDATA_INTERNAL_CHECKS 1
-#endif
-
-#if SIZEOF_VOID_P == 4
-#define ENV32BIT 1
-#else
-#define ENV64BIT 1
-#endif
-
-// NETDATA_TRACE_ALLOCATIONS does not work under musl libc, so don't enable it
-//#if defined(NETDATA_INTERNAL_CHECKS) && !defined(NETDATA_TRACE_ALLOCATIONS)
-//#define NETDATA_TRACE_ALLOCATIONS 1
-//#endif
-
-#define OS_LINUX 1
-#define OS_FREEBSD 2
-#define OS_MACOS 3
-
-#define MALLOC_ALIGNMENT (sizeof(uintptr_t) * 2)
-#define size_t_atomic_count(op, var, size) __atomic_## op ##_fetch(&(var), size, __ATOMIC_RELAXED)
-#define size_t_atomic_bytes(op, var, size) __atomic_## op ##_fetch(&(var), ((size) % MALLOC_ALIGNMENT)?((size) + MALLOC_ALIGNMENT - ((size) % MALLOC_ALIGNMENT)):(size), __ATOMIC_RELAXED)
-
-// ----------------------------------------------------------------------------
-// system include files for all netdata C programs
-
-/* select the memory allocator, based on autoconf findings */
-#if defined(ENABLE_JEMALLOC)
-
-#if defined(HAVE_JEMALLOC_JEMALLOC_H)
-#include <jemalloc/jemalloc.h>
-#else // !defined(HAVE_JEMALLOC_JEMALLOC_H)
-#include <malloc.h>
-#endif // !defined(HAVE_JEMALLOC_JEMALLOC_H)
-
-#elif defined(ENABLE_TCMALLOC)
-
-#include <google/tcmalloc.h>
-
-#else /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */
-
-#if !(defined(__FreeBSD__) || defined(__APPLE__))
-#include <malloc.h>
-#endif /* __FreeBSD__ || __APPLE__ */
-
-#endif /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */
-
-// ----------------------------------------------------------------------------
-
-#if defined(__FreeBSD__)
-#include <pthread_np.h>
-#define NETDATA_OS_TYPE "freebsd"
-#elif defined(__APPLE__)
-#define NETDATA_OS_TYPE "macos"
-#else
-#define NETDATA_OS_TYPE "linux"
-#endif /* __FreeBSD__, __APPLE__*/
-
-#include <pthread.h>
-#include <errno.h>
-#include <stdbool.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <stddef.h>
-#include <ctype.h>
-#include <string.h>
-#include <strings.h>
-#include <arpa/inet.h>
-#include <netinet/tcp.h>
-#include <sys/ioctl.h>
-#include <libgen.h>
-#include <dirent.h>
-#include <fcntl.h>
-#include <getopt.h>
-#include <grp.h>
-#include <pwd.h>
-#include <limits.h>
-#include <locale.h>
-#include <net/if.h>
-#include <poll.h>
-#include <signal.h>
-#include <syslog.h>
-#include <sys/mman.h>
-#include <sys/resource.h>
-#include <sys/socket.h>
-#include <sys/syscall.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <sys/un.h>
-#include <time.h>
-#include <unistd.h>
-#include <uuid/uuid.h>
-#include <spawn.h>
-#include <uv.h>
-#include <assert.h>
-
-// CentOS 7 has older version that doesn't define this
-// same goes for MacOS
-#ifndef UUID_STR_LEN
-#define UUID_STR_LEN (37)
-#endif
-
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-
-#ifdef HAVE_RESOLV_H
-#include <resolv.h>
-#endif
-
-#ifdef HAVE_NETDB_H
-#include <netdb.h>
-#endif
-
-#ifdef HAVE_SYS_PRCTL_H
-#include <sys/prctl.h>
-#endif
-
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-
-#ifdef HAVE_SYS_VFS_H
-#include <sys/vfs.h>
-#endif
-
-#ifdef HAVE_SYS_STATFS_H
-#include <sys/statfs.h>
-#endif
-
-#ifdef HAVE_LINUX_MAGIC_H
-#include <linux/magic.h>
-#endif
-
-#ifdef HAVE_SYS_MOUNT_H
-#include <sys/mount.h>
-#endif
-
-#ifdef HAVE_SYS_STATVFS_H
-#include <sys/statvfs.h>
-#endif
-
-// #1408
-#ifdef MAJOR_IN_MKDEV
-#include <sys/mkdev.h>
-#endif
-#ifdef MAJOR_IN_SYSMACROS
-#include <sys/sysmacros.h>
-#endif
-
-#ifdef STORAGE_WITH_MATH
-#include <math.h>
-#include <float.h>
-#endif
-
-#if defined(HAVE_INTTYPES_H)
-#include <inttypes.h>
-#elif defined(HAVE_STDINT_H)
-#include <stdint.h>
-#endif
-
-#include <zlib.h>
-
-#ifdef HAVE_CAPABILITY
-#include <sys/capability.h>
-#endif
-
-
-// ----------------------------------------------------------------------------
-// netdata common definitions
-
-#define _cleanup_(x) __attribute__((__cleanup__(x)))
-
-#ifdef __GNUC__
-#define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
-#endif // __GNUC__
-
-#ifdef HAVE_FUNC_ATTRIBUTE_RETURNS_NONNULL
-#define NEVERNULL __attribute__((returns_nonnull))
-#else
-#define NEVERNULL
-#endif
-
-#ifdef HAVE_FUNC_ATTRIBUTE_NOINLINE
-#define NOINLINE __attribute__((noinline))
-#else
-#define NOINLINE
-#endif
-
-#ifdef HAVE_FUNC_ATTRIBUTE_MALLOC
-#define MALLOCLIKE __attribute__((malloc))
-#else
-#define MALLOCLIKE
-#endif
-
-#ifdef HAVE_FUNC_ATTRIBUTE_FORMAT
-#define PRINTFLIKE(f, a) __attribute__ ((format(__printf__, f, a)))
-#else
-#define PRINTFLIKE(f, a)
-#endif
-
-#ifdef HAVE_FUNC_ATTRIBUTE_NORETURN
-#define NORETURN __attribute__ ((noreturn))
-#else
-#define NORETURN
-#endif
-
-#ifdef HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT
-#define WARNUNUSED __attribute__ ((warn_unused_result))
-#else
-#define WARNUNUSED
-#endif
-
-void aral_judy_init(void);
-size_t judy_aral_overhead(void);
-size_t judy_aral_structures(void);
-
-#define ABS(x) (((x) < 0)? (-(x)) : (x))
-#define MIN(a,b) (((a)<(b))?(a):(b))
-#define MAX(a,b) (((a)>(b))?(a):(b))
-
-#define GUID_LEN 36
-
-// ---------------------------------------------------------------------------------------------
-// double linked list management
-// inspired by https://github.com/troydhanson/uthash/blob/master/src/utlist.h
-
-#define DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(head, item, prev, next) \
- do { \
- (item)->next = (head); \
- \
- if(likely(head)) { \
- (item)->prev = (head)->prev; \
- (head)->prev = (item); \
- } \
- else \
- (item)->prev = (item); \
- \
- (head) = (item); \
- } while (0)
-
-#define DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(head, item, prev, next) \
- do { \
- \
- (item)->next = NULL; \
- \
- if(likely(head)) { \
- (item)->prev = (head)->prev; \
- (head)->prev->next = (item); \
- (head)->prev = (item); \
- } \
- else { \
- (item)->prev = (item); \
- (head) = (item); \
- } \
- \
- } while (0)
-
-#define DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(head, item, prev, next) \
- do { \
- fatal_assert((head) != NULL); \
- fatal_assert((item)->prev != NULL); \
- \
- if((item)->prev == (item)) \
- /* it is the only item in the list */ \
- (head) = NULL; \
- \
- else if((item) == (head)) { \
- /* it is the first item */ \
- fatal_assert((item)->next != NULL); \
- (item)->next->prev = (item)->prev; \
- (head) = (item)->next; \
- } \
- else { \
- /* it is any other item */ \
- (item)->prev->next = (item)->next; \
- \
- if ((item)->next) \
- (item)->next->prev = (item)->prev; \
- else \
- (head)->prev = (item)->prev; \
- } \
- \
- (item)->next = NULL; \
- (item)->prev = NULL; \
- } while (0)
-
-#define DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(head, existing, item, prev, next) \
- do { \
- if (existing) { \
- fatal_assert((head) != NULL); \
- fatal_assert((item) != NULL); \
- \
- (item)->next = (existing); \
- (item)->prev = (existing)->prev; \
- (existing)->prev = (item); \
- \
- if ((head) == (existing)) \
- (head) = (item); \
- else \
- (item)->prev->next = (item); \
- \
- } \
- else \
- DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(head, item, prev, next); \
- \
- } while (0)
-
-#define DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(head, existing, item, prev, next) \
- do { \
- if (existing) { \
- fatal_assert((head) != NULL); \
- fatal_assert((item) != NULL); \
- \
- (item)->next = (existing)->next; \
- (item)->prev = (existing); \
- (existing)->next = (item); \
- \
- if ((item)->next) \
- (item)->next->prev = (item); \
- else \
- (head)->prev = (item); \
- } \
- else \
- DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(head, item, prev, next); \
- \
- } while (0)
-
-#define DOUBLE_LINKED_LIST_APPEND_LIST_UNSAFE(head, head2, prev, next) \
- do { \
- if (head2) { \
- if (head) { \
- __typeof(head2) _head2_last_item = (head2)->prev; \
- \
- (head2)->prev = (head)->prev; \
- (head)->prev->next = (head2); \
- \
- (head)->prev = _head2_last_item; \
- } \
- else \
- (head) = (head2); \
- } \
- } while (0)
-
-#define DOUBLE_LINKED_LIST_FOREACH_FORWARD(head, var, prev, next) \
- for ((var) = (head); (var) ; (var) = (var)->next)
-
-#define DOUBLE_LINKED_LIST_FOREACH_BACKWARD(head, var, prev, next) \
- for ((var) = (head) ? (head)->prev : NULL ; (var) ; (var) = ((var) == (head)) ? NULL : (var)->prev)
-
-// ---------------------------------------------------------------------------------------------
-
-#include "storage_number/storage_number.h"
-
-typedef struct storage_point {
- NETDATA_DOUBLE min; // when count > 1, this is the minimum among them
- NETDATA_DOUBLE max; // when count > 1, this is the maximum among them
- NETDATA_DOUBLE sum; // the point sum - divided by count gives the average
-
- // end_time - start_time = point duration
- time_t start_time_s; // the time the point starts
- time_t end_time_s; // the time the point ends
-
- uint32_t count; // the number of original points aggregated
- uint32_t anomaly_count; // the number of original points found anomalous
-
- SN_FLAGS flags; // flags stored with the point
-} STORAGE_POINT;
-
-#define storage_point_unset(x) do { \
- (x).min = (x).max = (x).sum = NAN; \
- (x).count = 0; \
- (x).anomaly_count = 0; \
- (x).flags = SN_FLAG_NONE; \
- (x).start_time_s = 0; \
- (x).end_time_s = 0; \
- } while(0)
-
-#define storage_point_empty(x, start_s, end_s) do { \
- (x).min = (x).max = (x).sum = NAN; \
- (x).count = 1; \
- (x).anomaly_count = 0; \
- (x).flags = SN_FLAG_NONE; \
- (x).start_time_s = start_s; \
- (x).end_time_s = end_s; \
- } while(0)
-
-#define STORAGE_POINT_UNSET (STORAGE_POINT){ .min = NAN, .max = NAN, .sum = NAN, .count = 0, .anomaly_count = 0, .flags = SN_FLAG_NONE, .start_time_s = 0, .end_time_s = 0 }
-
-#define storage_point_is_unset(x) (!(x).count)
-#define storage_point_is_gap(x) (!netdata_double_isnumber((x).sum))
-#define storage_point_is_zero(x) (!(x).count || (netdata_double_is_zero((x).min) && netdata_double_is_zero((x).max) && netdata_double_is_zero((x).sum) && (x).anomaly_count == 0))
-
-#define storage_point_merge_to(dst, src) do { \
- if(storage_point_is_unset(dst)) \
- (dst) = (src); \
- \
- else if(!storage_point_is_unset(src) && \
- !storage_point_is_gap(src)) { \
- \
- if((src).start_time_s < (dst).start_time_s) \
- (dst).start_time_s = (src).start_time_s;\
- \
- if((src).end_time_s > (dst).end_time_s) \
- (dst).end_time_s = (src).end_time_s; \
- \
- if((src).min < (dst).min) \
- (dst).min = (src).min; \
- \
- if((src).max > (dst).max) \
- (dst).max = (src).max; \
- \
- (dst).sum += (src).sum; \
- \
- (dst).count += (src).count; \
- (dst).anomaly_count += (src).anomaly_count; \
- \
- (dst).flags |= (src).flags & SN_FLAG_RESET; \
- } \
-} while(0)
-
-#define storage_point_add_to(dst, src) do { \
- if(storage_point_is_unset(dst)) \
- (dst) = (src); \
- \
- else if(!storage_point_is_unset(src) && \
- !storage_point_is_gap(src)) { \
- \
- if((src).start_time_s < (dst).start_time_s) \
- (dst).start_time_s = (src).start_time_s;\
- \
- if((src).end_time_s > (dst).end_time_s) \
- (dst).end_time_s = (src).end_time_s; \
- \
- (dst).min += (src).min; \
- (dst).max += (src).max; \
- (dst).sum += (src).sum; \
- \
- (dst).count += (src).count; \
- (dst).anomaly_count += (src).anomaly_count; \
- \
- (dst).flags |= (src).flags & SN_FLAG_RESET; \
- } \
-} while(0)
-
-#define storage_point_make_positive(sp) do { \
- if(!storage_point_is_unset(sp) && \
- !storage_point_is_gap(sp)) { \
- \
- if(unlikely(signbit((sp).sum))) \
- (sp).sum = -(sp).sum; \
- \
- if(unlikely(signbit((sp).min))) \
- (sp).min = -(sp).min; \
- \
- if(unlikely(signbit((sp).max))) \
- (sp).max = -(sp).max; \
- \
- if(unlikely((sp).min > (sp).max)) { \
- NETDATA_DOUBLE t = (sp).min; \
- (sp).min = (sp).max; \
- (sp).max = t; \
- } \
- } \
-} while(0)
-
-#define storage_point_anomaly_rate(sp) \
- (NETDATA_DOUBLE)(storage_point_is_unset(sp) ? 0.0 : (NETDATA_DOUBLE)((sp).anomaly_count) * 100.0 / (NETDATA_DOUBLE)((sp).count))
-
-#define storage_point_average_value(sp) \
- ((sp).count ? (sp).sum / (NETDATA_DOUBLE)((sp).count) : 0.0)
-
-// ---------------------------------------------------------------------------------------------
-
-void netdata_fix_chart_id(char *s);
-void netdata_fix_chart_name(char *s);
-
-int madvise_sequential(void *mem, size_t len);
-int madvise_random(void *mem, size_t len);
-int madvise_dontfork(void *mem, size_t len);
-int madvise_willneed(void *mem, size_t len);
-int madvise_dontneed(void *mem, size_t len);
-int madvise_dontdump(void *mem, size_t len);
-int madvise_mergeable(void *mem, size_t len);
-
-int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args);
-int snprintfz(char *dst, size_t n, const char *fmt, ...) PRINTFLIKE(3, 4);
-
-// memory allocation functions that handle failures
-#ifdef NETDATA_TRACE_ALLOCATIONS
-int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data);
-
-#define strdupz(s) strdupz_int(s, __FILE__, __FUNCTION__, __LINE__)
-#define callocz(nmemb, size) callocz_int(nmemb, size, __FILE__, __FUNCTION__, __LINE__)
-#define mallocz(size) mallocz_int(size, __FILE__, __FUNCTION__, __LINE__)
-#define reallocz(ptr, size) reallocz_int(ptr, size, __FILE__, __FUNCTION__, __LINE__)
-#define freez(ptr) freez_int(ptr, __FILE__, __FUNCTION__, __LINE__)
-#define mallocz_usable_size(ptr) mallocz_usable_size_int(ptr, __FILE__, __FUNCTION__, __LINE__)
-
-char *strdupz_int(const char *s, const char *file, const char *function, size_t line);
-void *callocz_int(size_t nmemb, size_t size, const char *file, const char *function, size_t line);
-void *mallocz_int(size_t size, const char *file, const char *function, size_t line);
-void *reallocz_int(void *ptr, size_t size, const char *file, const char *function, size_t line);
-void freez_int(void *ptr, const char *file, const char *function, size_t line);
-size_t mallocz_usable_size_int(void *ptr, const char *file, const char *function, size_t line);
-
-#else // NETDATA_TRACE_ALLOCATIONS
-char *strdupz(const char *s) MALLOCLIKE NEVERNULL;
-void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL;
-void *mallocz(size_t size) MALLOCLIKE NEVERNULL;
-void *reallocz(void *ptr, size_t size) MALLOCLIKE NEVERNULL;
-void freez(void *ptr);
-#endif // NETDATA_TRACE_ALLOCATIONS
-
-void posix_memfree(void *ptr);
-
-void json_escape_string(char *dst, const char *src, size_t size);
-void json_fix_string(char *s);
-
-void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool read_only, int *open_fd);
-int netdata_munmap(void *ptr, size_t size);
-int memory_file_save(const char *filename, void *mem, size_t size);
-
-int fd_is_valid(int fd);
-
-extern struct rlimit rlimit_nofile;
-
-extern int enable_ksm;
-
-char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len);
-
-int verify_netdata_host_prefix(bool log_msg);
-
-int recursively_delete_dir(const char *path, const char *reason);
-
-extern volatile sig_atomic_t netdata_exit;
-
-extern const char *program_version;
-
-char *strdupz_path_subpath(const char *path, const char *subpath);
-int path_is_dir(const char *path, const char *subpath);
-int path_is_file(const char *path, const char *subpath);
-void recursive_config_double_dir_load(
- const char *user_path
- , const char *stock_path
- , const char *subpath
- , int (*callback)(const char *filename, void *data)
- , void *data
- , size_t depth
-);
-char *read_by_filename(const char *filename, long *file_size);
-char *find_and_replace(const char *src, const char *find, const char *replace, const char *where);
-
-/* fix for alpine linux */
-#ifndef RUSAGE_THREAD
-#ifdef RUSAGE_CHILDREN
-#define RUSAGE_THREAD RUSAGE_CHILDREN
-#endif
-#endif
-
-#define BITS_IN_A_KILOBIT 1000
-#define KILOBITS_IN_A_MEGABIT 1000
-
-/* misc. */
-
-#define UNUSED(x) (void)(x)
-
-#ifdef __GNUC__
-#define UNUSED_FUNCTION(x) __attribute__((unused)) UNUSED_##x
-#else
-#define UNUSED_FUNCTION(x) UNUSED_##x
-#endif
-
-#define error_report(x, args...) do { errno = 0; netdata_log_error(x, ##args); } while(0)
-
-// Taken from linux kernel
-#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)]))
-
-#ifdef ENV32BIT
-
-typedef struct bitmapX {
- uint32_t bits;
- uint32_t data[];
-} BITMAPX;
-
-typedef struct bitmap256 {
- uint32_t bits;
- uint32_t data[256 / 32];
-} BITMAP256;
-
-typedef struct bitmap1024 {
- uint32_t bits;
- uint32_t data[1024 / 32];
-} BITMAP1024;
-
-static inline BITMAPX *bitmapX_create(uint32_t bits) {
- BITMAPX *bmp = (BITMAPX *)callocz(1, sizeof(BITMAPX) + sizeof(uint32_t) * ((bits + 31) / 32));
- uint32_t *p = (uint32_t *)&bmp->bits;
- *p = bits;
- return bmp;
-}
-
-#define bitmapX_get_bit(ptr, idx) ((ptr)->data[(idx) >> 5] & (1U << ((idx) & 31)))
-#define bitmapX_set_bit(ptr, idx, value) do { \
- register uint32_t _bitmask = 1U << ((idx) & 31); \
- if (value) \
- (ptr)->data[(idx) >> 5] |= _bitmask; \
- else \
- (ptr)->data[(idx) >> 5] &= ~_bitmask; \
-} while(0)
-
-#else // 64bit version of bitmaps
-
-typedef struct bitmapX {
- uint32_t bits;
- uint64_t data[];
-} BITMAPX;
-
-typedef struct bitmap256 {
- uint32_t bits;
- uint64_t data[256 / 64];
-} BITMAP256;
-
-typedef struct bitmap1024 {
- uint32_t bits;
- uint64_t data[1024 / 64];
-} BITMAP1024;
-
-static inline BITMAPX *bitmapX_create(uint32_t bits) {
- BITMAPX *bmp = (BITMAPX *)callocz(1, sizeof(BITMAPX) + sizeof(uint64_t) * ((bits + 63) / 64));
- bmp->bits = bits;
- return bmp;
-}
-
-#define bitmapX_get_bit(ptr, idx) ((ptr)->data[(idx) >> 6] & (1ULL << ((idx) & 63)))
-#define bitmapX_set_bit(ptr, idx, value) do { \
- register uint64_t _bitmask = 1ULL << ((idx) & 63); \
- if (value) \
- (ptr)->data[(idx) >> 6] |= _bitmask; \
- else \
- (ptr)->data[(idx) >> 6] &= ~_bitmask; \
-} while(0)
-
-#endif // 64bit version of bitmaps
-
-#define BITMAPX_INITIALIZER(wanted_bits) { .bits = (wanted_bits), .data = {0} }
-#define BITMAP256_INITIALIZER (BITMAP256)BITMAPX_INITIALIZER(256)
-#define BITMAP1024_INITIALIZER (BITMAP1024)BITMAPX_INITIALIZER(1024)
-#define bitmap256_get_bit(ptr, idx) bitmapX_get_bit((BITMAPX *)ptr, idx)
-#define bitmap256_set_bit(ptr, idx, value) bitmapX_set_bit((BITMAPX *)ptr, idx, value)
-#define bitmap1024_get_bit(ptr, idx) bitmapX_get_bit((BITMAPX *)ptr, idx)
-#define bitmap1024_set_bit(ptr, idx, value) bitmapX_set_bit((BITMAPX *)ptr, idx, value)
-
-#define COMPRESSION_MAX_CHUNK 0x4000
-#define COMPRESSION_MAX_OVERHEAD 128
-#define COMPRESSION_MAX_MSG_SIZE (COMPRESSION_MAX_CHUNK - COMPRESSION_MAX_OVERHEAD - 1)
-#define PLUGINSD_LINE_MAX (COMPRESSION_MAX_MSG_SIZE - 768)
-
-bool run_command_and_copy_output_to_stdout(const char *command, int max_line_length);
-struct web_buffer *run_command_and_get_output_to_buffer(const char *command, int max_line_length);
-
-typedef enum {
- OPEN_FD_ACTION_CLOSE,
- OPEN_FD_ACTION_FD_CLOEXEC
-} OPEN_FD_ACTION;
-typedef enum {
- OPEN_FD_EXCLUDE_STDIN = 0x01,
- OPEN_FD_EXCLUDE_STDOUT = 0x02,
- OPEN_FD_EXCLUDE_STDERR = 0x04
-} OPEN_FD_EXCLUDE;
-void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds);
-
-void netdata_cleanup_and_exit(int ret) NORETURN;
-void send_statistics(const char *action, const char *action_result, const char *action_data);
-extern char *netdata_configured_host_prefix;
-
-#define XXH_INLINE_ALL
-#include "xxhash.h"
-
-#include "uuid/uuid.h"
-
-#include "libjudy/src/Judy.h"
-#include "july/july.h"
-#include "os.h"
-#include "threads/threads.h"
-#include "buffer/buffer.h"
-#include "locks/locks.h"
-#include "circular_buffer/circular_buffer.h"
-#include "avl/avl.h"
-#include "inlined.h"
-#include "line_splitter/line_splitter.h"
-#include "clocks/clocks.h"
-#include "datetime/iso8601.h"
-#include "datetime/rfc3339.h"
-#include "datetime/rfc7231.h"
-#include "completion/completion.h"
-#include "popen/popen.h"
-#include "simple_pattern/simple_pattern.h"
-#ifdef ENABLE_HTTPS
-# include "socket/security.h"
-#endif
-#include "socket/socket.h"
-#include "config/appconfig.h"
-#include "log/journal.h"
-#include "log/log.h"
-#include "buffered_reader/buffered_reader.h"
-#include "procfile/procfile.h"
-#include "string/string.h"
-#include "dictionary/dictionary.h"
-#if defined(HAVE_LIBBPF) && !defined(__cplusplus)
-#include "ebpf/ebpf.h"
-#endif
-#include "eval/eval.h"
-#include "statistical/statistical.h"
-#include "adaptive_resortable_list/adaptive_resortable_list.h"
-#include "url/url.h"
-#include "json/json.h"
-#include "health/health.h"
-#include "string/utf8.h"
-#include "libnetdata/aral/aral.h"
-#include "onewayalloc/onewayalloc.h"
-#include "worker_utilization/worker_utilization.h"
-#include "yaml.h"
-#include "http/http_defs.h"
-#include "gorilla/gorilla.h"
-#include "facets/facets.h"
-#include "dyn_conf/dyn_conf.h"
-#include "functions_evloop/functions_evloop.h"
-
-// BEWARE: this exists in alarm-notify.sh
-#define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"
-
-#define RRD_STORAGE_TIERS 5
-
-static inline size_t struct_natural_alignment(size_t size) __attribute__((const));
-
-#define STRUCT_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2)
-static inline size_t struct_natural_alignment(size_t size) {
- if(unlikely(size % STRUCT_NATURAL_ALIGNMENT))
- size = size + STRUCT_NATURAL_ALIGNMENT - (size % STRUCT_NATURAL_ALIGNMENT);
-
- return size;
-}
-
-#ifdef NETDATA_TRACE_ALLOCATIONS
-struct malloc_trace {
- avl_t avl;
-
- const char *function;
- const char *file;
- size_t line;
-
- size_t malloc_calls;
- size_t calloc_calls;
- size_t realloc_calls;
- size_t strdup_calls;
- size_t free_calls;
-
- size_t mmap_calls;
- size_t munmap_calls;
-
- size_t allocations;
- size_t bytes;
-
- struct rrddim *rd_bytes;
- struct rrddim *rd_allocations;
- struct rrddim *rd_avg_alloc;
- struct rrddim *rd_ops;
-};
-#endif // NETDATA_TRACE_ALLOCATIONS
-
-static inline PPvoid_t JudyLFirstThenNext(Pcvoid_t PArray, Word_t * PIndex, bool *first) {
- if(unlikely(*first)) {
- *first = false;
- return JudyLFirst(PArray, PIndex, PJE0);
- }
-
- return JudyLNext(PArray, PIndex, PJE0);
-}
-
-static inline PPvoid_t JudyLLastThenPrev(Pcvoid_t PArray, Word_t * PIndex, bool *first) {
- if(unlikely(*first)) {
- *first = false;
- return JudyLLast(PArray, PIndex, PJE0);
- }
-
- return JudyLPrev(PArray, PIndex, PJE0);
-}
-
-typedef enum {
- TIMING_STEP_INTERNAL = 0,
-
- TIMING_STEP_BEGIN2_PREPARE,
- TIMING_STEP_BEGIN2_FIND_CHART,
- TIMING_STEP_BEGIN2_PARSE,
- TIMING_STEP_BEGIN2_ML,
- TIMING_STEP_BEGIN2_PROPAGATE,
- TIMING_STEP_BEGIN2_STORE,
-
- TIMING_STEP_SET2_PREPARE,
- TIMING_STEP_SET2_LOOKUP_DIMENSION,
- TIMING_STEP_SET2_PARSE,
- TIMING_STEP_SET2_ML,
- TIMING_STEP_SET2_PROPAGATE,
- TIMING_STEP_RRDSET_STORE_METRIC,
- TIMING_STEP_DBENGINE_FIRST_CHECK,
- TIMING_STEP_DBENGINE_CHECK_DATA,
- TIMING_STEP_DBENGINE_PACK,
- TIMING_STEP_DBENGINE_PAGE_FIN,
- TIMING_STEP_DBENGINE_MRG_UPDATE,
- TIMING_STEP_DBENGINE_PAGE_ALLOC,
- TIMING_STEP_DBENGINE_CREATE_NEW_PAGE,
- TIMING_STEP_DBENGINE_FLUSH_PAGE,
- TIMING_STEP_SET2_STORE,
-
- TIMING_STEP_END2_PREPARE,
- TIMING_STEP_END2_PUSH_V1,
- TIMING_STEP_END2_ML,
- TIMING_STEP_END2_RRDSET,
- TIMING_STEP_END2_PROPAGATE,
- TIMING_STEP_END2_STORE,
-
- TIMING_STEP_FREEIPMI_CTX_CREATE,
- TIMING_STEP_FREEIPMI_DSR_CACHE_DIR,
- TIMING_STEP_FREEIPMI_SENSOR_CONFIG_FILE,
- TIMING_STEP_FREEIPMI_SENSOR_READINGS_BY_X,
- TIMING_STEP_FREEIPMI_READ_record_id,
- TIMING_STEP_FREEIPMI_READ_sensor_number,
- TIMING_STEP_FREEIPMI_READ_sensor_type,
- TIMING_STEP_FREEIPMI_READ_sensor_name,
- TIMING_STEP_FREEIPMI_READ_sensor_state,
- TIMING_STEP_FREEIPMI_READ_sensor_units,
- TIMING_STEP_FREEIPMI_READ_sensor_bitmask_type,
- TIMING_STEP_FREEIPMI_READ_sensor_bitmask,
- TIMING_STEP_FREEIPMI_READ_sensor_bitmask_strings,
- TIMING_STEP_FREEIPMI_READ_sensor_reading_type,
- TIMING_STEP_FREEIPMI_READ_sensor_reading,
- TIMING_STEP_FREEIPMI_READ_event_reading_type_code,
- TIMING_STEP_FREEIPMI_READ_record_type,
- TIMING_STEP_FREEIPMI_READ_record_type_class,
- TIMING_STEP_FREEIPMI_READ_sel_state,
- TIMING_STEP_FREEIPMI_READ_event_direction,
- TIMING_STEP_FREEIPMI_READ_event_type_code,
- TIMING_STEP_FREEIPMI_READ_event_offset_type,
- TIMING_STEP_FREEIPMI_READ_event_offset,
- TIMING_STEP_FREEIPMI_READ_event_offset_string,
- TIMING_STEP_FREEIPMI_READ_manufacturer_id,
-
- // terminator
- TIMING_STEP_MAX,
-} TIMING_STEP;
-
-typedef enum {
- TIMING_ACTION_INIT,
- TIMING_ACTION_STEP,
- TIMING_ACTION_FINISH,
-} TIMING_ACTION;
-
-#ifdef NETDATA_TIMING_REPORT
-#define timing_init() timing_action(TIMING_ACTION_INIT, TIMING_STEP_INTERNAL)
-#define timing_step(step) timing_action(TIMING_ACTION_STEP, step)
-#define timing_report() timing_action(TIMING_ACTION_FINISH, TIMING_STEP_INTERNAL)
-#else
-#define timing_init() debug_dummy()
-#define timing_step(step) debug_dummy()
-#define timing_report() debug_dummy()
-#endif
-void timing_action(TIMING_ACTION action, TIMING_STEP step);
-
-int hash256_string(const unsigned char *string, size_t size, char *hash);
-
-extern bool unittest_running;
-#define API_RELATIVE_TIME_MAX (3 * 365 * 86400)
-
-bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t now);
-bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running);
-
-int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size);
-
-# ifdef __cplusplus
-}
-# endif
-
-#endif // NETDATA_LIB_H
diff --git a/libnetdata/line_splitter/Makefile.am b/libnetdata/line_splitter/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/line_splitter/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/line_splitter/README.md b/libnetdata/line_splitter/README.md
deleted file mode 100644
index f811bb4b3..000000000
--- a/libnetdata/line_splitter/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-<!--
-title: "Log"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/log/README.md
-sidebar_label: "Log"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Log
-
-The netdata log library supports debug, info, error and fatal error logging.
-By default we have an access log, an error log and a collectors log.
-
diff --git a/libnetdata/locks/Makefile.am b/libnetdata/locks/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/locks/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/locks/README.md b/libnetdata/locks/README.md
deleted file mode 100644
index 5560832b7..000000000
--- a/libnetdata/locks/README.md
+++ /dev/null
@@ -1,107 +0,0 @@
-<!--
-title: "Locks"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/locks/README.md
-sidebar_label: "Locks"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Locks
-
-## How to trace netdata locks
-
-To enable tracing rwlocks in netdata, compile netdata by setting `CFLAGS="-DNETDATA_TRACE_RWLOCKS=1"`, like this:
-
-```
-CFLAGS="-O1 -ggdb -DNETDATA_TRACE_RWLOCKS=1" ./netdata-installer.sh
-```
-
-During compilation, the compiler will log:
-
-```
-libnetdata/locks/locks.c:105:2: warning: #warning NETDATA_TRACE_RWLOCKS ENABLED - EXPECT A LOT OF OUTPUT [-Wcpp]
- 105 | #warning NETDATA_TRACE_RWLOCKS ENABLED - EXPECT A LOT OF OUTPUT
- | ^~~~~~~
-```
-
-Once compiled, netdata will do the following:
-
-Every call to `netdata_rwlock_*()` is now measured in time.
-
-### logging of slow locks/unlocks
-
-If any call takes more than 10 usec, it will be logged like this:
-
-```
-RW_LOCK ON LOCK 0x0x7fbe1f2e5190: 4157038, 'ACLK_Query_2' (function build_context_param_list() 99@web/api/formatters/rrd2json.c) WAITED to UNLOCK for 29 usec.
-```
-
-The time can be changed by setting this `-DNETDATA_TRACE_RWLOCKS_WAIT_TIME_TO_IGNORE_USEC=20` (or whatever number) to the CFLAGS.
-
-### logging of long hold times
-
-If any lock is holded for more than 10000 usec, it will be logged like this:
-
-```
-RW_LOCK ON LOCK 0x0x55a20afc1b20: 4187198, 'ANALYTICS' (function analytics_gather_mutable_meta_data() 532@daemon/analytics.c) holded a 'R' for 13232 usec.
-```
-
-The time can be changed by setting this `-DNETDATA_TRACE_RWLOCKS_HOLD_TIME_TO_IGNORE_USEC=20000` (or whatever number) to the CFLAGS.
-
-### logging for probable pauses (predictive)
-
-The library maintains a linked-list of all the lock holders (one entry per thread). For this linked-list a mutex is used. So every call to the r/w locks now also has a mutex lock.
-
-If any call is expected to pause the caller (ie the caller is attempting a read lock while there is a write lock in place and vice versa), the library will log something like this:
-
-```
-RW_LOCK ON LOCK 0x0x5651c9fcce20: 4190039 'HEALTH' (function health_execute_pending_updates() 661@health/health.c) WANTS a 'W' lock (while holding 1 rwlocks and 1 mutexes).
-There are 7 readers and 0 writers are holding the lock:
- => 1: RW_LOCK: process 4190091 'WEB_SERVER[static14]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 709847 usec.
- => 2: RW_LOCK: process 4190079 'WEB_SERVER[static6]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 709869 usec.
- => 3: RW_LOCK: process 4190084 'WEB_SERVER[static10]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 709948 usec.
- => 4: RW_LOCK: process 4190076 'WEB_SERVER[static3]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710190 usec.
- => 5: RW_LOCK: process 4190092 'WEB_SERVER[static15]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710195 usec.
- => 6: RW_LOCK: process 4190077 'WEB_SERVER[static4]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710208 usec.
- => 7: RW_LOCK: process 4190044 'WEB_SERVER[static1]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710221 usec.
-```
-
-And each of the above is paired with a `GOT` log, like this:
-
-```
-RW_LOCK ON LOCK 0x0x5651c9fcce20: 4190039 'HEALTH' (function health_execute_pending_updates() 661@health/health.c) GOT a 'W' lock (while holding 2 rwlocks and 1 mutexes).
-There are 0 readers and 1 writers are holding the lock:
- => 1: RW_LOCK: process 4190039 'HEALTH' (function health_execute_pending_updates() 661@health/health.c) is having 1 'W' lock for 36 usec.
-```
-
-Keep in mind that the lock and log are not atomic. The list of callers is indicative (and sometimes just empty because the original holders of the lock, unlocked it until we had the chance to print their names).
-
-### POSIX compliance check
-
-The library may also log messages about POSIX unsupported cases, like this:
-
-```
-RW_LOCK FATAL ON LOCK 0x0x622000109290: 3609368 'PLUGIN[proc]' (function __rrdset_check_rdlock() 10@database/rrdset.c) attempts to acquire a 'W' lock.
-But it is not supported by POSIX because: ALREADY HAS THIS LOCK
-At this attempt, the task is holding 1 rwlocks and 1 mutexes.
-There are 1 readers and 0 writers are holding the lock requested now:
- => 1: RW_LOCK: process 3609368 'PLUGIN[proc]' (function rrdset_done() 1398@database/rrdset.c) is having 1 'R' lock for 0 usec.
-```
-
-### nested read locks
-
-When compiled with `-DNETDATA_TRACE_RWLOCKS_LOG_NESTED=1` the library will also detect nested read locks and print them like this:
-
-```
-RW_LOCK ON LOCK 0x0x7ff6ea46d190: 4140225 'WEB_SERVER[static14]' (function rrdr_json_wrapper_begin() 34@web/api/formatters/json_wrapper.c) NESTED READ LOCK REQUEST a 'R' lock (while holding 1 rwlocks and 1 mutexes).
-There are 5 readers and 0 writers are holding the lock:
- => 1: RW_LOCK: process 4140225 'WEB_SERVER[static14]' (function rrdr_lock_rrdset() 70@web/api/queries/rrdr.c) is having 1 'R' lock for 216667 usec.
- => 2: RW_LOCK: process 4140211 'WEB_SERVER[static6]' (function rrdr_lock_rrdset() 70@web/api/queries/rrdr.c) is having 1 'R' lock for 220001 usec.
- => 3: RW_LOCK: process 4140218 'WEB_SERVER[static8]' (function rrdr_lock_rrdset() 70@web/api/queries/rrdr.c) is having 1 'R' lock for 220001 usec.
- => 4: RW_LOCK: process 4140224 'WEB_SERVER[static13]' (function rrdr_lock_rrdset() 70@web/api/queries/rrdr.c) is having 1 'R' lock for 220001 usec.
- => 5: RW_LOCK: process 4140227 'WEB_SERVER[static16]' (function rrdr_lock_rrdset() 70@web/api/queries/rrdr.c) is having 1 'R' lock for 220001 usec.
-```
-
-
-
diff --git a/libnetdata/log/Makefile.am b/libnetdata/log/Makefile.am
deleted file mode 100644
index a02b8ebd2..000000000
--- a/libnetdata/log/Makefile.am
+++ /dev/null
@@ -1,9 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- systemd-cat-native.md \
- $(NULL)
diff --git a/libnetdata/log/README.md b/libnetdata/log/README.md
deleted file mode 100644
index d9ed64374..000000000
--- a/libnetdata/log/README.md
+++ /dev/null
@@ -1,204 +0,0 @@
-<!--
-title: "Log"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/log/README.md
-sidebar_label: "Log"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Netdata Logging
-
-This document describes how Netdata generates its own logs, not how Netdata manages and queries logs databases.
-
-## Log sources
-
-Netdata supports the following log sources:
-
-1. **daemon**, logs generated by Netdata daemon.
-2. **collector**, logs generated by Netdata collectors, including internal and external ones.
-3. **access**, API requests received by Netdata
-4. **health**, all alert transitions and notifications
-
-## Log outputs
-
-For each log source, Netdata supports the following output methods:
-
-- **off**, to disable this log source
-- **journal**, to send the logs to systemd-journal.
-- **syslog**, to send the logs to syslog.
-- **system**, to send the output to `stderr` or `stdout` depending on the log source.
-- **stdout**, to write the logs to Netdata's `stdout`.
-- **stderr**, to write the logs to Netdata's `stderr`.
-- **filename**, to send the logs to a file.
-
-For `daemon` and `collector` the default is `journal` when systemd-journal is available.
-To decide if systemd-journal is available, Netdata checks:
-
-1. `stderr` is connected to systemd-journald
-2. `/run/systemd/journal/socket` exists
-3. `/host/run/systemd/journal/socket` exists (`/host` is configurable in containers)
-
-If any of the above is detected, Netdata will select `journal` for `daemon` and `collector` sources.
-
-All other sources default to a file.
-
-## Log formats
-
-| Format | Description |
-|---------|--------------------------------------------------------------------------------------------------------|
-| journal | journald-specific log format. Automatically selected when logging to systemd-journal. |
-| logfmt | logs data as a series of key/value pairs. The default when logging to any output other than `journal`. |
-| json | logs data in JSON format. |
-
-## Log levels
-
-Each time Netdata logs, it assigns a priority to the log. It can be one of this (in order of importance):
-
-| Level | Description |
-|-----------|----------------------------------------------------------------------------------------|
-| emergency | a fatal condition, Netdata will most likely exit immediately after. |
-| alert | a very important issue that may affect how Netdata operates. |
-| critical | a very important issue the user should know which, Netdata thinks it can survive. |
-| error | an error condition indicating that Netdata is trying to do something, but it fails. |
-| warning | something unexpected has happened that may or may not affect the operation of Netdata. |
-| notice | something that does not affect the operation of Netdata, but the user should notice. |
-| info | the default log level about information the user should know. |
-| debug | these are more verbose logs that can be ignored. |
-
-## Logs Configuration
-
-In `netdata.conf`, there are the following settings:
-
-```
-[logs]
- # logs to trigger flood protection = 1000
- # logs flood protection period = 60
- # facility = daemon
- # level = info
- # daemon = journal
- # collector = journal
- # access = /var/log/netdata/access.log
- # health = /var/log/netdata/health.log
-```
-
-- `logs to trigger flood protection` and `logs flood protection period` enable logs flood protection for `daemon` and `collector` sources. It can also be configured per log source.
-- `facility` is used only when Netdata logs to syslog.
-- `level` defines the minimum [log level](#log-levels) of logs that will be logged. This setting is applied only to `daemon` and `collector` sources. It can also be configured per source.
-
-### Configuring log sources
-
-Each for the sources (`daemon`, `collector`, `access`, `health`), accepts the following:
-
-```
-source = {FORMAT},level={LEVEL},protection={LOG}/{PERIOD}@{OUTPUT}
-```
-
-Where:
-
-- `{FORMAT}`, is one of the [log formats](#log-formats),
-- `{LEVEL}`, is the minimum [log level](#log-levels) to be logged,
-- `{LOGS}` is the number of `logs to trigger flood protection` configured per output,
-- `{PERIOD}` is the equivalent of `logs flood protection period` configured per output,
-- `{OUTPUT}` is one of the `[log outputs](#log-outputs),
-
-All parameters can be omitted, except `{OUTPUT}`. If `{OUTPUT}` is the only given parameter, `@` can be omitted.
-
-### Logs rotation
-
-Netdata comes with `logrotate` configuration to rotate its log files periodically.
-
-The default is usually found in `/etc/logrotate.d/netdata`.
-
-Sending a `SIGHUP` to Netdata, will instruct it to re-open all its log files.
-
-## Log Fields
-
-Netdata exposes the following fields to its logs:
-
-| journal | logfmt | json | Description |
-|:--------------------------------------:|:------------------------------:|:------------------------------:|:---------------------------------------------------------------------------------------------------------:|
-| `_SOURCE_REALTIME_TIMESTAMP` | `time` | `time` | the timestamp of the event |
-| `SYSLOG_IDENTIFIER` | `comm` | `comm` | the program logging the event |
-| `ND_LOG_SOURCE` | `source` | `source` | one of the [log sources](#log-sources) |
-| `PRIORITY`<br/>numeric | `level`<br/>text | `level`<br/>numeric | one of the [log levels](#log-levels) |
-| `ERRNO` | `errno` | `errno` | the numeric value of `errno` |
-| `INVOCATION_ID` | - | - | a unique UUID of the Netdata session, reset on every Netdata restart, inherited by systemd when available |
-| `CODE_LINE` | - | - | the line number of of the source code logging this event |
-| `CODE_FILE` | - | - | the filename of the source code logging this event |
-| `CODE_FUNCTION` | - | - | the function name of the source code logging this event |
-| `TID` | `tid` | `tid` | the thread id of the thread logging this event |
-| `THREAD_TAG` | `thread` | `thread` | the name of the thread logging this event |
-| `MESSAGE_ID` | `msg_id` | `msg_id` | see [message IDs](#message-ids) |
-| `ND_MODULE` | `module` | `module` | the Netdata module logging this event |
-| `ND_NIDL_NODE` | `node` | `node` | the hostname of the node the event is related to |
-| `ND_NIDL_INSTANCE` | `instance` | `instance` | the instance of the node the event is related to |
-| `ND_NIDL_CONTEXT` | `context` | `context` | the context the event is related to (this is usually the chart name, as shown on netdata dashboards |
-| `ND_NIDL_DIMENSION` | `dimension` | `dimension` | the dimension the event is related to |
-| `ND_SRC_TRANSPORT` | `src_transport` | `src_transport` | when the event happened during a request, this is the request transport |
-| `ND_SRC_IP` | `src_ip` | `src_ip` | when the event happened during an inbound request, this is the IP the request came from |
-| `ND_SRC_PORT` | `src_port` | `src_port` | when the event happened during an inbound request, this is the port the request came from |
-| `ND_SRC_CAPABILITIES` | `src_capabilities` | `src_capabilities` | when the request came from a child, this is the communication capabilities of the child |
-| `ND_DST_TRANSPORT` | `dst_transport` | `dst_transport` | when the event happened during an outbound request, this is the outbound request transport |
-| `ND_DST_IP` | `dst_ip` | `dst_ip` | when the event happened during an outbound request, this is the IP the request destination |
-| `ND_DST_PORT` | `dst_port` | `dst_port` | when the event happened during an outbound request, this is the port the request destination |
-| `ND_DST_CAPABILITIES` | `dst_capabilities` | `dst_capabilities` | when the request goes to a parent, this is the communication capabilities of the parent |
-| `ND_REQUEST_METHOD` | `req_method` | `req_method` | when the event happened during an inbound request, this is the method the request was received |
-| `ND_RESPONSE_CODE` | `code` | `code` | when responding to a request, this this the response code |
-| `ND_CONNECTION_ID` | `conn` | `conn` | when there is a connection id for an inbound connection, this is the connection id |
-| `ND_TRANSACTION_ID` | `transaction` | `transaction` | the transaction id (UUID) of all API requests |
-| `ND_RESPONSE_SENT_BYTES` | `sent_bytes` | `sent_bytes` | the bytes we sent to API responses |
-| `ND_RESPONSE_SIZE_BYTES` | `size_bytes` | `size_bytes` | the uncompressed bytes of the API responses |
-| `ND_RESPONSE_PREP_TIME_USEC` | `prep_ut` | `prep_ut` | the time needed to prepare a response |
-| `ND_RESPONSE_SENT_TIME_USEC` | `sent_ut` | `sent_ut` | the time needed to send a response |
-| `ND_RESPONSE_TOTAL_TIME_USEC` | `total_ut` | `total_ut` | the total time needed to complete a response |
-| `ND_ALERT_ID` | `alert_id` | `alert_id` | the alert id this event is related to |
-| `ND_ALERT_EVENT_ID` | `alert_event_id` | `alert_event_id` | a sequential number of the alert transition (per host) |
-| `ND_ALERT_UNIQUE_ID` | `alert_unique_id` | `alert_unique_id` | a sequential number of the alert transition (per alert) |
-| `ND_ALERT_TRANSITION_ID` | `alert_transition_id` | `alert_transition_id` | the unique UUID of this alert transition |
-| `ND_ALERT_CONFIG` | `alert_config` | `alert_config` | the alert configuration hash (UUID) |
-| `ND_ALERT_NAME` | `alert` | `alert` | the alert name |
-| `ND_ALERT_CLASS` | `alert_class` | `alert_class` | the alert classification |
-| `ND_ALERT_COMPONENT` | `alert_component` | `alert_component` | the alert component |
-| `ND_ALERT_TYPE` | `alert_type` | `alert_type` | the alert type |
-| `ND_ALERT_EXEC` | `alert_exec` | `alert_exec` | the alert notification program |
-| `ND_ALERT_RECIPIENT` | `alert_recipient` | `alert_recipient` | the alert recipient(s) |
-| `ND_ALERT_VALUE` | `alert_value` | `alert_value` | the current alert value |
-| `ND_ALERT_VALUE_OLD` | `alert_value_old` | `alert_value_old` | the previous alert value |
-| `ND_ALERT_STATUS` | `alert_status` | `alert_status` | the current alert status |
-| `ND_ALERT_STATUS_OLD` | `alert_value_old` | `alert_value_old` | the previous alert value |
-| `ND_ALERT_UNITS` | `alert_units` | `alert_units` | the units of the alert |
-| `ND_ALERT_SUMMARY` | `alert_summary` | `alert_summary` | the summary text of the alert |
-| `ND_ALERT_INFO` | `alert_info` | `alert_info` | the info text of the alert |
-| `ND_ALERT_DURATION` | `alert_duration` | `alert_duration` | the duration the alert was in its previous state |
-| `ND_ALERT_NOTIFICATION_TIMESTAMP_USEC` | `alert_notification_timestamp` | `alert_notification_timestamp` | the timestamp the notification delivery is scheduled |
-| `ND_REQUEST` | `request` | `request` | the full request during which the event happened |
-| `MESSAGE` | `msg` | `msg` | the event message |
-
-
-### Message IDs
-
-Netdata assigns specific message IDs to certain events:
-
-- `ed4cdb8f1beb4ad3b57cb3cae2d162fa` when a Netdata child connects to this Netdata
-- `6e2e3839067648968b646045dbf28d66` when this Netdata connects to a Netdata parent
-- `9ce0cb58ab8b44df82c4bf1ad9ee22de` when alerts change state
-- `6db0018e83e34320ae2a659d78019fb7` when notifications are sent
-
-You can view these events using the Netdata systemd-journal.plugin at the `MESSAGE_ID` filter,
-or using `journalctl` like this:
-
-```bash
-# query children connection
-journalctl MESSAGE_ID=ed4cdb8f1beb4ad3b57cb3cae2d162fa
-
-# query parent connection
-journalctl MESSAGE_ID=6e2e3839067648968b646045dbf28d66
-
-# query alert transitions
-journalctl MESSAGE_ID=9ce0cb58ab8b44df82c4bf1ad9ee22de
-
-# query alert notifications
-journalctl MESSAGE_ID=6db0018e83e34320ae2a659d78019fb7
-```
-
diff --git a/libnetdata/log/log.c b/libnetdata/log/log.c
deleted file mode 100644
index c805716ce..000000000
--- a/libnetdata/log/log.c
+++ /dev/null
@@ -1,2431 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#define SD_JOURNAL_SUPPRESS_LOCATION
-
-#include "../libnetdata.h"
-#include <daemon/main.h>
-
-#ifdef __FreeBSD__
-#include <sys/endian.h>
-#endif
-
-#ifdef __APPLE__
-#include <machine/endian.h>
-#endif
-
-#ifdef HAVE_BACKTRACE
-#include <execinfo.h>
-#endif
-
-#ifdef HAVE_SYSTEMD
-#include <systemd/sd-journal.h>
-#endif
-
-#include <syslog.h>
-
-const char *program_name = "";
-
-uint64_t debug_flags = 0;
-
-#ifdef ENABLE_ACLK
-int aclklog_enabled = 0;
-#endif
-
-// ----------------------------------------------------------------------------
-
-struct nd_log_source;
-static bool nd_log_limit_reached(struct nd_log_source *source);
-
-// ----------------------------------------------------------------------------
-// logging method
-
-typedef enum __attribute__((__packed__)) {
- NDLM_DISABLED = 0,
- NDLM_DEVNULL,
- NDLM_DEFAULT,
- NDLM_JOURNAL,
- NDLM_SYSLOG,
- NDLM_STDOUT,
- NDLM_STDERR,
- NDLM_FILE,
-} ND_LOG_METHOD;
-
-static struct {
- ND_LOG_METHOD method;
- const char *name;
-} nd_log_methods[] = {
- { .method = NDLM_DISABLED, .name = "none" },
- { .method = NDLM_DEVNULL, .name = "/dev/null" },
- { .method = NDLM_DEFAULT, .name = "default" },
- { .method = NDLM_JOURNAL, .name = "journal" },
- { .method = NDLM_SYSLOG, .name = "syslog" },
- { .method = NDLM_STDOUT, .name = "stdout" },
- { .method = NDLM_STDERR, .name = "stderr" },
- { .method = NDLM_FILE, .name = "file" },
-};
-
-static ND_LOG_METHOD nd_log_method2id(const char *method) {
- if(!method || !*method)
- return NDLM_DEFAULT;
-
- size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(strcmp(nd_log_methods[i].name, method) == 0)
- return nd_log_methods[i].method;
- }
-
- return NDLM_FILE;
-}
-
-static const char *nd_log_id2method(ND_LOG_METHOD method) {
- size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(method == nd_log_methods[i].method)
- return nd_log_methods[i].name;
- }
-
- return "unknown";
-}
-
-#define IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ndlo) ((ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || (ndlo) == NDLM_STDERR)
-
-const char *nd_log_method_for_external_plugins(const char *s) {
- if(s && *s) {
- ND_LOG_METHOD method = nd_log_method2id(s);
- if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method))
- return nd_log_id2method(method);
- }
-
- return nd_log_id2method(NDLM_STDERR);
-}
-
-// ----------------------------------------------------------------------------
-// workaround strerror_r()
-
-#if defined(STRERROR_R_CHAR_P)
-// GLIBC version of strerror_r
-static const char *strerror_result(const char *a, const char *b) { (void)b; return a; }
-#elif defined(HAVE_STRERROR_R)
-// POSIX version of strerror_r
-static const char *strerror_result(int a, const char *b) { (void)a; return b; }
-#elif defined(HAVE_C__GENERIC)
-
-// what a trick!
-// http://stackoverflow.com/questions/479207/function-overloading-in-c
-static const char *strerror_result_int(int a, const char *b) { (void)a; return b; }
-static const char *strerror_result_string(const char *a, const char *b) { (void)b; return a; }
-
-#define strerror_result(a, b) _Generic((a), \
- int: strerror_result_int, \
- char *: strerror_result_string \
- )(a, b)
-
-#else
-#error "cannot detect the format of function strerror_r()"
-#endif
-
-static const char *errno2str(int errnum, char *buf, size_t size) {
- return strerror_result(strerror_r(errnum, buf, size), buf);
-}
-
-// ----------------------------------------------------------------------------
-// facilities
-//
-// sys/syslog.h (Linux)
-// sys/sys/syslog.h (FreeBSD)
-// bsd/sys/syslog.h (darwin-xnu)
-
-static struct {
- int facility;
- const char *name;
-} nd_log_facilities[] = {
- { LOG_AUTH, "auth" },
- { LOG_AUTHPRIV, "authpriv" },
- { LOG_CRON, "cron" },
- { LOG_DAEMON, "daemon" },
- { LOG_FTP, "ftp" },
- { LOG_KERN, "kern" },
- { LOG_LPR, "lpr" },
- { LOG_MAIL, "mail" },
- { LOG_NEWS, "news" },
- { LOG_SYSLOG, "syslog" },
- { LOG_USER, "user" },
- { LOG_UUCP, "uucp" },
- { LOG_LOCAL0, "local0" },
- { LOG_LOCAL1, "local1" },
- { LOG_LOCAL2, "local2" },
- { LOG_LOCAL3, "local3" },
- { LOG_LOCAL4, "local4" },
- { LOG_LOCAL5, "local5" },
- { LOG_LOCAL6, "local6" },
- { LOG_LOCAL7, "local7" },
-
-#ifdef __FreeBSD__
- { LOG_CONSOLE, "console" },
- { LOG_NTP, "ntp" },
-
- // FreeBSD does not consider 'security' as deprecated.
- { LOG_SECURITY, "security" },
-#else
- // For all other O/S 'security' is mapped to 'auth'.
- { LOG_AUTH, "security" },
-#endif
-
-#ifdef __APPLE__
- { LOG_INSTALL, "install" },
- { LOG_NETINFO, "netinfo" },
- { LOG_RAS, "ras" },
- { LOG_REMOTEAUTH, "remoteauth" },
- { LOG_LAUNCHD, "launchd" },
-
-#endif
-};
-
-static int nd_log_facility2id(const char *facility) {
- size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(strcmp(nd_log_facilities[i].name, facility) == 0)
- return nd_log_facilities[i].facility;
- }
-
- return LOG_DAEMON;
-}
-
-static const char *nd_log_id2facility(int facility) {
- size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(nd_log_facilities[i].facility == facility)
- return nd_log_facilities[i].name;
- }
-
- return "daemon";
-}
-
-// ----------------------------------------------------------------------------
-// priorities
-
-static struct {
- ND_LOG_FIELD_PRIORITY priority;
- const char *name;
-} nd_log_priorities[] = {
- { .priority = NDLP_EMERG, .name = "emergency" },
- { .priority = NDLP_EMERG, .name = "emerg" },
- { .priority = NDLP_ALERT, .name = "alert" },
- { .priority = NDLP_CRIT, .name = "critical" },
- { .priority = NDLP_CRIT, .name = "crit" },
- { .priority = NDLP_ERR, .name = "error" },
- { .priority = NDLP_ERR, .name = "err" },
- { .priority = NDLP_WARNING, .name = "warning" },
- { .priority = NDLP_WARNING, .name = "warn" },
- { .priority = NDLP_NOTICE, .name = "notice" },
- { .priority = NDLP_INFO, .name = NDLP_INFO_STR },
- { .priority = NDLP_DEBUG, .name = "debug" },
-};
-
-int nd_log_priority2id(const char *priority) {
- size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(strcmp(nd_log_priorities[i].name, priority) == 0)
- return nd_log_priorities[i].priority;
- }
-
- return NDLP_INFO;
-}
-
-const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority) {
- size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(priority == nd_log_priorities[i].priority)
- return nd_log_priorities[i].name;
- }
-
- return NDLP_INFO_STR;
-}
-
-// ----------------------------------------------------------------------------
-// log sources
-
-const char *nd_log_sources[] = {
- [NDLS_UNSET] = "UNSET",
- [NDLS_ACCESS] = "access",
- [NDLS_ACLK] = "aclk",
- [NDLS_COLLECTORS] = "collector",
- [NDLS_DAEMON] = "daemon",
- [NDLS_HEALTH] = "health",
- [NDLS_DEBUG] = "debug",
-};
-
-size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def) {
- size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(strcmp(nd_log_sources[i], source) == 0)
- return i;
- }
-
- return def;
-}
-
-
-static const char *nd_log_id2source(ND_LOG_SOURCES source) {
- size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]);
- if(source < entries)
- return nd_log_sources[source];
-
- return nd_log_sources[NDLS_COLLECTORS];
-}
-
-// ----------------------------------------------------------------------------
-// log output formats
-
-typedef enum __attribute__((__packed__)) {
- NDLF_JOURNAL,
- NDLF_LOGFMT,
- NDLF_JSON,
-} ND_LOG_FORMAT;
-
-static struct {
- ND_LOG_FORMAT format;
- const char *name;
-} nd_log_formats[] = {
- { .format = NDLF_JOURNAL, .name = "journal" },
- { .format = NDLF_LOGFMT, .name = "logfmt" },
- { .format = NDLF_JSON, .name = "json" },
-};
-
-static ND_LOG_FORMAT nd_log_format2id(const char *format) {
- if(!format || !*format)
- return NDLF_LOGFMT;
-
- size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(strcmp(nd_log_formats[i].name, format) == 0)
- return nd_log_formats[i].format;
- }
-
- return NDLF_LOGFMT;
-}
-
-static const char *nd_log_id2format(ND_LOG_FORMAT format) {
- size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]);
- for(size_t i = 0; i < entries ;i++) {
- if(format == nd_log_formats[i].format)
- return nd_log_formats[i].name;
- }
-
- return "logfmt";
-}
-
-// ----------------------------------------------------------------------------
-// format dates
-
-void log_date(char *buffer, size_t len, time_t now) {
- if(unlikely(!buffer || !len))
- return;
-
- time_t t = now;
- struct tm *tmp, tmbuf;
-
- tmp = localtime_r(&t, &tmbuf);
-
- if (unlikely(!tmp)) {
- buffer[0] = '\0';
- return;
- }
-
- if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0))
- buffer[0] = '\0';
-
- buffer[len - 1] = '\0';
-}
-
-// ----------------------------------------------------------------------------
-
-struct nd_log_limit {
- usec_t started_monotonic_ut;
- uint32_t counter;
- uint32_t prevented;
-
- uint32_t throttle_period;
- uint32_t logs_per_period;
- uint32_t logs_per_period_backup;
-};
-
-#define ND_LOG_LIMITS_DEFAULT (struct nd_log_limit){ .logs_per_period = ND_LOG_DEFAULT_THROTTLE_LOGS, .logs_per_period_backup = ND_LOG_DEFAULT_THROTTLE_LOGS, .throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD, }
-#define ND_LOG_LIMITS_UNLIMITED (struct nd_log_limit){ .logs_per_period = 0, .logs_per_period_backup = 0, .throttle_period = 0, }
-
-struct nd_log_source {
- SPINLOCK spinlock;
- ND_LOG_METHOD method;
- ND_LOG_FORMAT format;
- const char *filename;
- int fd;
- FILE *fp;
-
- ND_LOG_FIELD_PRIORITY min_priority;
- const char *pending_msg;
- struct nd_log_limit limits;
-};
-
-static __thread ND_LOG_SOURCES overwrite_thread_source = 0;
-
-void nd_log_set_thread_source(ND_LOG_SOURCES source) {
- overwrite_thread_source = source;
-}
-
-static struct {
- uuid_t invocation_id;
-
- ND_LOG_SOURCES overwrite_process_source;
-
- struct nd_log_source sources[_NDLS_MAX];
-
- struct {
- bool initialized;
- } journal;
-
- struct {
- bool initialized;
- int fd;
- char filename[FILENAME_MAX + 1];
- } journal_direct;
-
- struct {
- bool initialized;
- int facility;
- } syslog;
-
- struct {
- SPINLOCK spinlock;
- bool initialized;
- } std_output;
-
- struct {
- SPINLOCK spinlock;
- bool initialized;
- } std_error;
-
-} nd_log = {
- .overwrite_process_source = 0,
- .journal = {
- .initialized = false,
- },
- .journal_direct = {
- .initialized = false,
- .fd = -1,
- },
- .syslog = {
- .initialized = false,
- .facility = LOG_DAEMON,
- },
- .std_output = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .initialized = false,
- },
- .std_error = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .initialized = false,
- },
- .sources = {
- [NDLS_UNSET] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DISABLED,
- .format = NDLF_JOURNAL,
- .filename = NULL,
- .fd = -1,
- .fp = NULL,
- .min_priority = NDLP_EMERG,
- .limits = ND_LOG_LIMITS_UNLIMITED,
- },
- [NDLS_ACCESS] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DEFAULT,
- .format = NDLF_LOGFMT,
- .filename = LOG_DIR "/access.log",
- .fd = -1,
- .fp = NULL,
- .min_priority = NDLP_DEBUG,
- .limits = ND_LOG_LIMITS_UNLIMITED,
- },
- [NDLS_ACLK] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_FILE,
- .format = NDLF_LOGFMT,
- .filename = LOG_DIR "/aclk.log",
- .fd = -1,
- .fp = NULL,
- .min_priority = NDLP_DEBUG,
- .limits = ND_LOG_LIMITS_UNLIMITED,
- },
- [NDLS_COLLECTORS] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DEFAULT,
- .format = NDLF_LOGFMT,
- .filename = LOG_DIR "/collectors.log",
- .fd = STDERR_FILENO,
- .fp = NULL,
- .min_priority = NDLP_INFO,
- .limits = ND_LOG_LIMITS_DEFAULT,
- },
- [NDLS_DEBUG] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DISABLED,
- .format = NDLF_LOGFMT,
- .filename = LOG_DIR "/debug.log",
- .fd = STDOUT_FILENO,
- .fp = NULL,
- .min_priority = NDLP_DEBUG,
- .limits = ND_LOG_LIMITS_UNLIMITED,
- },
- [NDLS_DAEMON] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DEFAULT,
- .filename = LOG_DIR "/daemon.log",
- .format = NDLF_LOGFMT,
- .fd = -1,
- .fp = NULL,
- .min_priority = NDLP_INFO,
- .limits = ND_LOG_LIMITS_DEFAULT,
- },
- [NDLS_HEALTH] = {
- .spinlock = NETDATA_SPINLOCK_INITIALIZER,
- .method = NDLM_DEFAULT,
- .format = NDLF_LOGFMT,
- .filename = LOG_DIR "/health.log",
- .fd = -1,
- .fp = NULL,
- .min_priority = NDLP_DEBUG,
- .limits = ND_LOG_LIMITS_UNLIMITED,
- },
- },
-};
-
-__attribute__((constructor)) void initialize_invocation_id(void) {
- // check for a NETDATA_INVOCATION_ID
- if(uuid_parse_flexi(getenv("NETDATA_INVOCATION_ID"), nd_log.invocation_id) != 0) {
- // not found, check for systemd set INVOCATION_ID
- if(uuid_parse_flexi(getenv("INVOCATION_ID"), nd_log.invocation_id) != 0) {
- // not found, generate a new one
- uuid_generate_random(nd_log.invocation_id);
- }
- }
-
- char uuid[UUID_COMPACT_STR_LEN];
- uuid_unparse_lower_compact(nd_log.invocation_id, uuid);
- setenv("NETDATA_INVOCATION_ID", uuid, 1);
-}
-
-int nd_log_health_fd(void) {
- if(nd_log.sources[NDLS_HEALTH].method == NDLM_FILE && nd_log.sources[NDLS_HEALTH].fd != -1)
- return nd_log.sources[NDLS_HEALTH].fd;
-
- return STDERR_FILENO;
-}
-
-void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting) {
- char buf[FILENAME_MAX + 100];
- if(setting && *setting)
- strncpyz(buf, setting, sizeof(buf) - 1);
- else
- buf[0] = '\0';
-
- struct nd_log_source *ls = &nd_log.sources[source];
- char *output = strrchr(buf, '@');
-
- if(!output)
- // all of it is the output
- output = buf;
- else {
- // we found an '@', the next char is the output
- *output = '\0';
- output++;
-
- // parse the other params
- char *remaining = buf;
- while(remaining) {
- char *value = strsep_skip_consecutive_separators(&remaining, ",");
- if (!value || !*value) continue;
-
- char *name = strsep_skip_consecutive_separators(&value, "=");
- if (!name || !*name) continue;
-
- if(strcmp(name, "logfmt") == 0)
- ls->format = NDLF_LOGFMT;
- else if(strcmp(name, "json") == 0)
- ls->format = NDLF_JSON;
- else if(strcmp(name, "journal") == 0)
- ls->format = NDLF_JOURNAL;
- else if(strcmp(name, "level") == 0 && value && *value)
- ls->min_priority = nd_log_priority2id(value);
- else if(strcmp(name, "protection") == 0 && value && *value) {
- if(strcmp(value, "off") == 0 || strcmp(value, "none") == 0) {
- ls->limits = ND_LOG_LIMITS_UNLIMITED;
- ls->limits.counter = 0;
- ls->limits.prevented = 0;
- }
- else {
- ls->limits = ND_LOG_LIMITS_DEFAULT;
-
- char *slash = strchr(value, '/');
- if(slash) {
- *slash = '\0';
- slash++;
- ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
- ls->limits.throttle_period = str2u(slash);
- }
- else {
- ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value);
- ls->limits.throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD;
- }
- }
- }
- else
- nd_log(NDLS_DAEMON, NDLP_ERR, "Error while parsing configuration of log source '%s'. "
- "In config '%s', '%s' is not understood.",
- nd_log_id2source(source), setting, name);
- }
- }
-
- if(!output || !*output || strcmp(output, "none") == 0 || strcmp(output, "off") == 0) {
- ls->method = NDLM_DISABLED;
- ls->filename = "/dev/null";
- }
- else if(strcmp(output, "journal") == 0) {
- ls->method = NDLM_JOURNAL;
- ls->filename = NULL;
- }
- else if(strcmp(output, "syslog") == 0) {
- ls->method = NDLM_SYSLOG;
- ls->filename = NULL;
- }
- else if(strcmp(output, "/dev/null") == 0) {
- ls->method = NDLM_DEVNULL;
- ls->filename = "/dev/null";
- }
- else if(strcmp(output, "system") == 0) {
- if(ls->fd == STDERR_FILENO) {
- ls->method = NDLM_STDERR;
- ls->filename = NULL;
- ls->fd = STDERR_FILENO;
- }
- else {
- ls->method = NDLM_STDOUT;
- ls->filename = NULL;
- ls->fd = STDOUT_FILENO;
- }
- }
- else if(strcmp(output, "stderr") == 0) {
- ls->method = NDLM_STDERR;
- ls->filename = NULL;
- ls->fd = STDERR_FILENO;
- }
- else if(strcmp(output, "stdout") == 0) {
- ls->method = NDLM_STDOUT;
- ls->filename = NULL;
- ls->fd = STDOUT_FILENO;
- }
- else {
- ls->method = NDLM_FILE;
- ls->filename = strdupz(output);
- }
-
-#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
- ls->min_priority = NDLP_DEBUG;
-#endif
-
- if(source == NDLS_COLLECTORS) {
- // set the method for the collector processes we will spawn
-
- ND_LOG_METHOD method;
- ND_LOG_FORMAT format = ls->format;
- ND_LOG_FIELD_PRIORITY priority = ls->min_priority;
-
- if(ls->method == NDLM_SYSLOG || ls->method == NDLM_JOURNAL)
- method = ls->method;
- else
- method = NDLM_STDERR;
-
- setenv("NETDATA_LOG_METHOD", nd_log_id2method(method), 1);
- setenv("NETDATA_LOG_FORMAT", nd_log_id2format(format), 1);
- setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
- }
-}
-
-void nd_log_set_priority_level(const char *setting) {
- if(!setting || !*setting)
- setting = "info";
-
- ND_LOG_FIELD_PRIORITY priority = nd_log_priority2id(setting);
-
-#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE)
- priority = NDLP_DEBUG;
-#endif
-
- for (size_t i = 0; i < _NDLS_MAX; i++) {
- if (i != NDLS_DEBUG)
- nd_log.sources[i].min_priority = priority;
- }
-
- // the right one
- setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1);
-}
-
-void nd_log_set_facility(const char *facility) {
- if(!facility || !*facility)
- facility = "daemon";
-
- nd_log.syslog.facility = nd_log_facility2id(facility);
- setenv("NETDATA_SYSLOG_FACILITY", nd_log_id2facility(nd_log.syslog.facility), 1);
-}
-
-void nd_log_set_flood_protection(size_t logs, time_t period) {
- nd_log.sources[NDLS_DAEMON].limits.logs_per_period =
- nd_log.sources[NDLS_DAEMON].limits.logs_per_period_backup;
- nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period =
- nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period_backup = logs;
-
- nd_log.sources[NDLS_DAEMON].limits.throttle_period =
- nd_log.sources[NDLS_COLLECTORS].limits.throttle_period = period;
-
- char buf[100];
- snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )period);
- setenv("NETDATA_ERRORS_THROTTLE_PERIOD", buf, 1);
- snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )logs);
- setenv("NETDATA_ERRORS_PER_PERIOD", buf, 1);
-}
-
-static bool nd_log_journal_systemd_init(void) {
-#ifdef HAVE_SYSTEMD
- nd_log.journal.initialized = true;
-#else
- nd_log.journal.initialized = false;
-#endif
-
- return nd_log.journal.initialized;
-}
-
-static void nd_log_journal_direct_set_env(void) {
- if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_JOURNAL)
- setenv("NETDATA_SYSTEMD_JOURNAL_PATH", nd_log.journal_direct.filename, 1);
-}
-
-static bool nd_log_journal_direct_init(const char *path) {
- if(nd_log.journal_direct.initialized) {
- nd_log_journal_direct_set_env();
- return true;
- }
-
- int fd;
- char filename[FILENAME_MAX + 1];
- if(!is_path_unix_socket(path)) {
-
- journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, "netdata");
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
-
- journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, NULL);
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
-
- journal_construct_path(filename, sizeof(filename), NULL, "netdata");
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) {
-
- journal_construct_path(filename, sizeof(filename), NULL, NULL);
- if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1)
- return false;
- }
- }
- }
- }
- else {
- snprintfz(filename, sizeof(filename), "%s", path);
- fd = journal_direct_fd(filename);
- }
-
- if(fd < 0)
- return false;
-
- nd_log.journal_direct.fd = fd;
- nd_log.journal_direct.initialized = true;
-
- strncpyz(nd_log.journal_direct.filename, filename, sizeof(nd_log.journal_direct.filename) - 1);
- nd_log_journal_direct_set_env();
-
- return true;
-}
-
-static void nd_log_syslog_init() {
- if(nd_log.syslog.initialized)
- return;
-
- openlog(program_name, LOG_PID, nd_log.syslog.facility);
- nd_log.syslog.initialized = true;
-}
-
-void nd_log_initialize_for_external_plugins(const char *name) {
- // if we don't run under Netdata, log to stderr,
- // otherwise, use the logging method Netdata wants us to use.
- setenv("NETDATA_LOG_METHOD", "stderr", 0);
- setenv("NETDATA_LOG_FORMAT", "logfmt", 0);
-
- nd_log.overwrite_process_source = NDLS_COLLECTORS;
- program_name = name;
-
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
- nd_log.sources[i].method = STDERR_FILENO;
- nd_log.sources[i].fd = -1;
- nd_log.sources[i].fp = NULL;
- }
-
- nd_log_set_priority_level(getenv("NETDATA_LOG_LEVEL"));
- nd_log_set_facility(getenv("NETDATA_SYSLOG_FACILITY"));
-
- time_t period = 1200;
- size_t logs = 200;
- const char *s = getenv("NETDATA_ERRORS_THROTTLE_PERIOD");
- if(s && *s >= '0' && *s <= '9') {
- period = str2l(s);
- if(period < 0) period = 0;
- }
-
- s = getenv("NETDATA_ERRORS_PER_PERIOD");
- if(s && *s >= '0' && *s <= '9')
- logs = str2u(s);
-
- nd_log_set_flood_protection(logs, period);
-
- if(!netdata_configured_host_prefix) {
- s = getenv("NETDATA_HOST_PREFIX");
- if(s && *s)
- netdata_configured_host_prefix = (char *)s;
- }
-
- ND_LOG_METHOD method = nd_log_method2id(getenv("NETDATA_LOG_METHOD"));
- ND_LOG_FORMAT format = nd_log_format2id(getenv("NETDATA_LOG_FORMAT"));
-
- if(!IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method)) {
- if(is_stderr_connected_to_journal()) {
- nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using journal.");
- method = NDLM_JOURNAL;
- }
- else {
- nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using stderr.");
- method = NDLM_STDERR;
- }
- }
-
- switch(method) {
- case NDLM_JOURNAL:
- if(!nd_log_journal_direct_init(getenv("NETDATA_SYSTEMD_JOURNAL_PATH")) ||
- !nd_log_journal_direct_init(NULL) || !nd_log_journal_systemd_init()) {
- nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize journal. Using stderr.");
- method = NDLM_STDERR;
- }
- break;
-
- case NDLM_SYSLOG:
- nd_log_syslog_init();
- break;
-
- default:
- method = NDLM_STDERR;
- break;
- }
-
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
- nd_log.sources[i].method = method;
- nd_log.sources[i].format = format;
- nd_log.sources[i].fd = -1;
- nd_log.sources[i].fp = NULL;
- }
-
-// nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "FINAL_LOG_METHOD: %s", nd_log_id2method(method));
-}
-
-static bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd) {
- if(new_fd == -1 || e->fd == -1 ||
- (e->fd == STDOUT_FILENO && nd_log.std_output.initialized) ||
- (e->fd == STDERR_FILENO && nd_log.std_error.initialized))
- return false;
-
- if(new_fd != e->fd) {
- int t = dup2(new_fd, e->fd);
-
- bool ret = true;
- if (t == -1) {
- netdata_log_error("Cannot dup2() new fd %d to old fd %d for '%s'", new_fd, e->fd, e->filename);
- ret = false;
- }
- else
- close(new_fd);
-
- if(e->fd == STDOUT_FILENO)
- nd_log.std_output.initialized = true;
- else if(e->fd == STDERR_FILENO)
- nd_log.std_error.initialized = true;
-
- return ret;
- }
-
- return false;
-}
-
-static void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source) {
- if(e->method == NDLM_DEFAULT)
- nd_log_set_user_settings(source, e->filename);
-
- if((e->method == NDLM_FILE && !e->filename) ||
- (e->method == NDLM_DEVNULL && e->fd == -1))
- e->method = NDLM_DISABLED;
-
- if(e->fp)
- fflush(e->fp);
-
- switch(e->method) {
- case NDLM_SYSLOG:
- nd_log_syslog_init();
- break;
-
- case NDLM_JOURNAL:
- nd_log_journal_direct_init(NULL);
- nd_log_journal_systemd_init();
- break;
-
- case NDLM_STDOUT:
- e->fp = stdout;
- e->fd = STDOUT_FILENO;
- break;
-
- case NDLM_DISABLED:
- break;
-
- case NDLM_DEFAULT:
- case NDLM_STDERR:
- e->method = NDLM_STDERR;
- e->fp = stderr;
- e->fd = STDERR_FILENO;
- break;
-
- case NDLM_DEVNULL:
- case NDLM_FILE: {
- int fd = open(e->filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
- if(fd == -1) {
- if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO) {
- e->fd = STDERR_FILENO;
- e->method = NDLM_STDERR;
- netdata_log_error("Cannot open log file '%s'. Falling back to stderr.", e->filename);
- }
- else
- netdata_log_error("Cannot open log file '%s'. Leaving fd %d as-is.", e->filename, e->fd);
- }
- else {
- if (!nd_log_replace_existing_fd(e, fd)) {
- if(e->fd == STDOUT_FILENO || e->fd == STDERR_FILENO) {
- if(e->fd == STDOUT_FILENO)
- e->method = NDLM_STDOUT;
- else if(e->fd == STDERR_FILENO)
- e->method = NDLM_STDERR;
-
- // we have dup2() fd, so we can close the one we opened
- if(fd != STDOUT_FILENO && fd != STDERR_FILENO)
- close(fd);
- }
- else
- e->fd = fd;
- }
- }
-
- // at this point we have e->fd set properly
-
- if(e->fd == STDOUT_FILENO)
- e->fp = stdout;
- else if(e->fd == STDERR_FILENO)
- e->fp = stderr;
-
- if(!e->fp) {
- e->fp = fdopen(e->fd, "a");
- if (!e->fp) {
- netdata_log_error("Cannot fdopen() fd %d ('%s')", e->fd, e->filename);
-
- if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO)
- close(e->fd);
-
- e->fp = stderr;
- e->fd = STDERR_FILENO;
- }
- }
- else {
- if (setvbuf(e->fp, NULL, _IOLBF, 0) != 0)
- netdata_log_error("Cannot set line buffering on fd %d ('%s')", e->fd, e->filename);
- }
- }
- break;
- }
-}
-
-static void nd_log_stdin_init(int fd, const char *filename) {
- int f = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0664);
- if(f == -1)
- return;
-
- if(f != fd) {
- dup2(f, fd);
- close(f);
- }
-}
-
-void nd_log_initialize(void) {
- nd_log_stdin_init(STDIN_FILENO, "/dev/null");
-
- for(size_t i = 0 ; i < _NDLS_MAX ; i++)
- nd_log_open(&nd_log.sources[i], i);
-}
-
-void nd_log_reopen_log_files(void) {
- netdata_log_info("Reopening all log files.");
-
- nd_log.std_output.initialized = false;
- nd_log.std_error.initialized = false;
- nd_log_initialize();
-
- netdata_log_info("Log files re-opened.");
-}
-
-void chown_open_file(int fd, uid_t uid, gid_t gid) {
- if(fd == -1) return;
-
- struct stat buf;
-
- if(fstat(fd, &buf) == -1) {
- netdata_log_error("Cannot fstat() fd %d", fd);
- return;
- }
-
- if((buf.st_uid != uid || buf.st_gid != gid) && S_ISREG(buf.st_mode)) {
- if(fchown(fd, uid, gid) == -1)
- netdata_log_error("Cannot fchown() fd %d.", fd);
- }
-}
-
-void nd_log_chown_log_files(uid_t uid, gid_t gid) {
- for(size_t i = 0 ; i < _NDLS_MAX ; i++) {
- if(nd_log.sources[i].fd != -1 && nd_log.sources[i].fd != STDIN_FILENO)
- chown_open_file(nd_log.sources[i].fd, uid, gid);
- }
-}
-
-// ----------------------------------------------------------------------------
-// annotators
-struct log_field;
-static void errno_annotator(BUFFER *wb, const char *key, struct log_field *lf);
-static void priority_annotator(BUFFER *wb, const char *key, struct log_field *lf);
-static void timestamp_usec_annotator(BUFFER *wb, const char *key, struct log_field *lf);
-
-// ----------------------------------------------------------------------------
-
-typedef void (*annotator_t)(BUFFER *wb, const char *key, struct log_field *lf);
-
-struct log_field {
- const char *journal;
- const char *logfmt;
- annotator_t logfmt_annotator;
- struct log_stack_entry entry;
-};
-
-#define THREAD_LOG_STACK_MAX 50
-
-static __thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX];
-static __thread size_t thread_log_stack_next = 0;
-
-static __thread struct log_field thread_log_fields[_NDF_MAX] = {
- // THE ORDER DEFINES THE ORDER FIELDS WILL APPEAR IN logfmt
-
- [NDF_STOP] = { // processing will not stop on this - so it is ok to be first
- .journal = NULL,
- .logfmt = NULL,
- .logfmt_annotator = NULL,
- },
- [NDF_TIMESTAMP_REALTIME_USEC] = {
- .journal = NULL,
- .logfmt = "time",
- .logfmt_annotator = timestamp_usec_annotator,
- },
- [NDF_SYSLOG_IDENTIFIER] = {
- .journal = "SYSLOG_IDENTIFIER", // standard journald field
- .logfmt = "comm",
- },
- [NDF_LOG_SOURCE] = {
- .journal = "ND_LOG_SOURCE",
- .logfmt = "source",
- },
- [NDF_PRIORITY] = {
- .journal = "PRIORITY", // standard journald field
- .logfmt = "level",
- .logfmt_annotator = priority_annotator,
- },
- [NDF_ERRNO] = {
- .journal = "ERRNO", // standard journald field
- .logfmt = "errno",
- .logfmt_annotator = errno_annotator,
- },
- [NDF_INVOCATION_ID] = {
- .journal = "INVOCATION_ID", // standard journald field
- .logfmt = NULL,
- },
- [NDF_LINE] = {
- .journal = "CODE_LINE", // standard journald field
- .logfmt = NULL,
- },
- [NDF_FILE] = {
- .journal = "CODE_FILE", // standard journald field
- .logfmt = NULL,
- },
- [NDF_FUNC] = {
- .journal = "CODE_FUNC", // standard journald field
- .logfmt = NULL,
- },
- [NDF_TID] = {
- .journal = "TID", // standard journald field
- .logfmt = "tid",
- },
- [NDF_THREAD_TAG] = {
- .journal = "THREAD_TAG",
- .logfmt = "thread",
- },
- [NDF_MESSAGE_ID] = {
- .journal = "MESSAGE_ID",
- .logfmt = "msg_id",
- },
- [NDF_MODULE] = {
- .journal = "ND_MODULE",
- .logfmt = "module",
- },
- [NDF_NIDL_NODE] = {
- .journal = "ND_NIDL_NODE",
- .logfmt = "node",
- },
- [NDF_NIDL_INSTANCE] = {
- .journal = "ND_NIDL_INSTANCE",
- .logfmt = "instance",
- },
- [NDF_NIDL_CONTEXT] = {
- .journal = "ND_NIDL_CONTEXT",
- .logfmt = "context",
- },
- [NDF_NIDL_DIMENSION] = {
- .journal = "ND_NIDL_DIMENSION",
- .logfmt = "dimension",
- },
- [NDF_SRC_TRANSPORT] = {
- .journal = "ND_SRC_TRANSPORT",
- .logfmt = "src_transport",
- },
- [NDF_SRC_IP] = {
- .journal = "ND_SRC_IP",
- .logfmt = "src_ip",
- },
- [NDF_SRC_PORT] = {
- .journal = "ND_SRC_PORT",
- .logfmt = "src_port",
- },
- [NDF_SRC_CAPABILITIES] = {
- .journal = "ND_SRC_CAPABILITIES",
- .logfmt = "src_capabilities",
- },
- [NDF_DST_TRANSPORT] = {
- .journal = "ND_DST_TRANSPORT",
- .logfmt = "dst_transport",
- },
- [NDF_DST_IP] = {
- .journal = "ND_DST_IP",
- .logfmt = "dst_ip",
- },
- [NDF_DST_PORT] = {
- .journal = "ND_DST_PORT",
- .logfmt = "dst_port",
- },
- [NDF_DST_CAPABILITIES] = {
- .journal = "ND_DST_CAPABILITIES",
- .logfmt = "dst_capabilities",
- },
- [NDF_REQUEST_METHOD] = {
- .journal = "ND_REQUEST_METHOD",
- .logfmt = "req_method",
- },
- [NDF_RESPONSE_CODE] = {
- .journal = "ND_RESPONSE_CODE",
- .logfmt = "code",
- },
- [NDF_CONNECTION_ID] = {
- .journal = "ND_CONNECTION_ID",
- .logfmt = "conn",
- },
- [NDF_TRANSACTION_ID] = {
- .journal = "ND_TRANSACTION_ID",
- .logfmt = "transaction",
- },
- [NDF_RESPONSE_SENT_BYTES] = {
- .journal = "ND_RESPONSE_SENT_BYTES",
- .logfmt = "sent_bytes",
- },
- [NDF_RESPONSE_SIZE_BYTES] = {
- .journal = "ND_RESPONSE_SIZE_BYTES",
- .logfmt = "size_bytes",
- },
- [NDF_RESPONSE_PREPARATION_TIME_USEC] = {
- .journal = "ND_RESPONSE_PREP_TIME_USEC",
- .logfmt = "prep_ut",
- },
- [NDF_RESPONSE_SENT_TIME_USEC] = {
- .journal = "ND_RESPONSE_SENT_TIME_USEC",
- .logfmt = "sent_ut",
- },
- [NDF_RESPONSE_TOTAL_TIME_USEC] = {
- .journal = "ND_RESPONSE_TOTAL_TIME_USEC",
- .logfmt = "total_ut",
- },
- [NDF_ALERT_ID] = {
- .journal = "ND_ALERT_ID",
- .logfmt = "alert_id",
- },
- [NDF_ALERT_UNIQUE_ID] = {
- .journal = "ND_ALERT_UNIQUE_ID",
- .logfmt = "alert_unique_id",
- },
- [NDF_ALERT_TRANSITION_ID] = {
- .journal = "ND_ALERT_TRANSITION_ID",
- .logfmt = "alert_transition_id",
- },
- [NDF_ALERT_EVENT_ID] = {
- .journal = "ND_ALERT_EVENT_ID",
- .logfmt = "alert_event_id",
- },
- [NDF_ALERT_CONFIG_HASH] = {
- .journal = "ND_ALERT_CONFIG",
- .logfmt = "alert_config",
- },
- [NDF_ALERT_NAME] = {
- .journal = "ND_ALERT_NAME",
- .logfmt = "alert",
- },
- [NDF_ALERT_CLASS] = {
- .journal = "ND_ALERT_CLASS",
- .logfmt = "alert_class",
- },
- [NDF_ALERT_COMPONENT] = {
- .journal = "ND_ALERT_COMPONENT",
- .logfmt = "alert_component",
- },
- [NDF_ALERT_TYPE] = {
- .journal = "ND_ALERT_TYPE",
- .logfmt = "alert_type",
- },
- [NDF_ALERT_EXEC] = {
- .journal = "ND_ALERT_EXEC",
- .logfmt = "alert_exec",
- },
- [NDF_ALERT_RECIPIENT] = {
- .journal = "ND_ALERT_RECIPIENT",
- .logfmt = "alert_recipient",
- },
- [NDF_ALERT_VALUE] = {
- .journal = "ND_ALERT_VALUE",
- .logfmt = "alert_value",
- },
- [NDF_ALERT_VALUE_OLD] = {
- .journal = "ND_ALERT_VALUE_OLD",
- .logfmt = "alert_value_old",
- },
- [NDF_ALERT_STATUS] = {
- .journal = "ND_ALERT_STATUS",
- .logfmt = "alert_status",
- },
- [NDF_ALERT_STATUS_OLD] = {
- .journal = "ND_ALERT_STATUS_OLD",
- .logfmt = "alert_value_old",
- },
- [NDF_ALERT_UNITS] = {
- .journal = "ND_ALERT_UNITS",
- .logfmt = "alert_units",
- },
- [NDF_ALERT_SUMMARY] = {
- .journal = "ND_ALERT_SUMMARY",
- .logfmt = "alert_summary",
- },
- [NDF_ALERT_INFO] = {
- .journal = "ND_ALERT_INFO",
- .logfmt = "alert_info",
- },
- [NDF_ALERT_DURATION] = {
- .journal = "ND_ALERT_DURATION",
- .logfmt = "alert_duration",
- },
- [NDF_ALERT_NOTIFICATION_REALTIME_USEC] = {
- .journal = "ND_ALERT_NOTIFICATION_TIMESTAMP_USEC",
- .logfmt = "alert_notification_timestamp",
- .logfmt_annotator = timestamp_usec_annotator,
- },
-
- // put new items here
- // leave the request URL and the message last
-
- [NDF_REQUEST] = {
- .journal = "ND_REQUEST",
- .logfmt = "request",
- },
- [NDF_MESSAGE] = {
- .journal = "MESSAGE",
- .logfmt = "msg",
- },
-};
-
-#define THREAD_FIELDS_MAX (sizeof(thread_log_fields) / sizeof(thread_log_fields[0]))
-
-ND_LOG_FIELD_ID nd_log_field_id_by_name(const char *field, size_t len) {
- for(size_t i = 0; i < THREAD_FIELDS_MAX ;i++) {
- if(thread_log_fields[i].journal && strlen(thread_log_fields[i].journal) == len && strncmp(field, thread_log_fields[i].journal, len) == 0)
- return i;
- }
-
- return NDF_STOP;
-}
-
-void log_stack_pop(void *ptr) {
- if(!ptr) return;
-
- struct log_stack_entry *lgs = *(struct log_stack_entry (*)[])ptr;
-
- if(unlikely(!thread_log_stack_next || lgs != thread_log_stack_base[thread_log_stack_next - 1])) {
- fatal("You cannot pop in the middle of the stack, or an item not in the stack");
- return;
- }
-
- thread_log_stack_next--;
-}
-
-void log_stack_push(struct log_stack_entry *lgs) {
- if(!lgs || thread_log_stack_next >= THREAD_LOG_STACK_MAX) return;
- thread_log_stack_base[thread_log_stack_next++] = lgs;
-}
-
-// ----------------------------------------------------------------------------
-// json formatter
-
-static void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max) {
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
- CLEAN_BUFFER *tmp = NULL;
-
- for (size_t i = 0; i < fields_max; i++) {
- if (!fields[i].entry.set || !fields[i].logfmt)
- continue;
-
- const char *key = fields[i].logfmt;
-
- const char *s = NULL;
- switch(fields[i].entry.type) {
- case NDFT_TXT:
- s = fields[i].entry.txt;
- break;
- case NDFT_STR:
- s = string2str(fields[i].entry.str);
- break;
- case NDFT_BFR:
- s = buffer_tostring(fields[i].entry.bfr);
- break;
- case NDFT_U64:
- buffer_json_member_add_uint64(wb, key, fields[i].entry.u64);
- break;
- case NDFT_I64:
- buffer_json_member_add_int64(wb, key, fields[i].entry.i64);
- break;
- case NDFT_DBL:
- buffer_json_member_add_double(wb, key, fields[i].entry.dbl);
- break;
- case NDFT_UUID:{
- char u[UUID_COMPACT_STR_LEN];
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
- buffer_json_member_add_string(wb, key, u);
- }
- break;
- case NDFT_CALLBACK: {
- if(!tmp)
- tmp = buffer_create(1024, NULL);
- else
- buffer_flush(tmp);
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
- s = buffer_tostring(tmp);
- else
- s = NULL;
- }
- break;
- default:
- s = "UNHANDLED";
- break;
- }
-
- if(s && *s)
- buffer_json_member_add_string(wb, key, s);
- }
-
- buffer_json_finalize(wb);
-}
-
-// ----------------------------------------------------------------------------
-// logfmt formatter
-
-
-static int64_t log_field_to_int64(struct log_field *lf) {
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- CLEAN_BUFFER *tmp = NULL;
- const char *s = NULL;
-
- switch(lf->entry.type) {
- case NDFT_UUID:
- case NDFT_UNSET:
- return 0;
-
- case NDFT_TXT:
- s = lf->entry.txt;
- break;
-
- case NDFT_STR:
- s = string2str(lf->entry.str);
- break;
-
- case NDFT_BFR:
- s = buffer_tostring(lf->entry.bfr);
- break;
-
- case NDFT_CALLBACK:
- if(!tmp)
- tmp = buffer_create(0, NULL);
- else
- buffer_flush(tmp);
-
- if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
- s = buffer_tostring(tmp);
- else
- s = NULL;
- break;
-
- case NDFT_U64:
- return lf->entry.u64;
-
- case NDFT_I64:
- return lf->entry.i64;
-
- case NDFT_DBL:
- return lf->entry.dbl;
- }
-
- if(s && *s)
- return str2ll(s, NULL);
-
- return 0;
-}
-
-static uint64_t log_field_to_uint64(struct log_field *lf) {
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- CLEAN_BUFFER *tmp = NULL;
- const char *s = NULL;
-
- switch(lf->entry.type) {
- case NDFT_UUID:
- case NDFT_UNSET:
- return 0;
-
- case NDFT_TXT:
- s = lf->entry.txt;
- break;
-
- case NDFT_STR:
- s = string2str(lf->entry.str);
- break;
-
- case NDFT_BFR:
- s = buffer_tostring(lf->entry.bfr);
- break;
-
- case NDFT_CALLBACK:
- if(!tmp)
- tmp = buffer_create(0, NULL);
- else
- buffer_flush(tmp);
-
- if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))
- s = buffer_tostring(tmp);
- else
- s = NULL;
- break;
-
- case NDFT_U64:
- return lf->entry.u64;
-
- case NDFT_I64:
- return lf->entry.i64;
-
- case NDFT_DBL:
- return lf->entry.dbl;
- }
-
- if(s && *s)
- return str2uint64_t(s, NULL);
-
- return 0;
-}
-
-static void timestamp_usec_annotator(BUFFER *wb, const char *key, struct log_field *lf) {
- usec_t ut = log_field_to_uint64(lf);
-
- if(!ut)
- return;
-
- char datetime[RFC3339_MAX_LENGTH];
- rfc3339_datetime_ut(datetime, sizeof(datetime), ut, 3, false);
-
- if(buffer_strlen(wb))
- buffer_fast_strcat(wb, " ", 1);
-
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_json_strcat(wb, datetime);
-}
-
-static void errno_annotator(BUFFER *wb, const char *key, struct log_field *lf) {
- int64_t errnum = log_field_to_int64(lf);
-
- if(errnum == 0)
- return;
-
- char buf[1024];
- const char *s = errno2str(errnum, buf, sizeof(buf));
-
- if(buffer_strlen(wb))
- buffer_fast_strcat(wb, " ", 1);
-
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=\"", 2);
- buffer_print_int64(wb, errnum);
- buffer_fast_strcat(wb, ", ", 2);
- buffer_json_strcat(wb, s);
- buffer_fast_strcat(wb, "\"", 1);
-}
-
-static void priority_annotator(BUFFER *wb, const char *key, struct log_field *lf) {
- uint64_t pri = log_field_to_uint64(lf);
-
- if(buffer_strlen(wb))
- buffer_fast_strcat(wb, " ", 1);
-
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_strcat(wb, nd_log_id2priority(pri));
-}
-
-static bool needs_quotes_for_logfmt(const char *s) {
- static bool safe_for_logfmt[256] = {
- [' '] = true, ['!'] = true, ['"'] = false, ['#'] = true, ['$'] = true, ['%'] = true, ['&'] = true,
- ['\''] = true, ['('] = true, [')'] = true, ['*'] = true, ['+'] = true, [','] = true, ['-'] = true,
- ['.'] = true, ['/'] = true, ['0'] = true, ['1'] = true, ['2'] = true, ['3'] = true, ['4'] = true,
- ['5'] = true, ['6'] = true, ['7'] = true, ['8'] = true, ['9'] = true, [':'] = true, [';'] = true,
- ['<'] = true, ['='] = true, ['>'] = true, ['?'] = true, ['@'] = true, ['A'] = true, ['B'] = true,
- ['C'] = true, ['D'] = true, ['E'] = true, ['F'] = true, ['G'] = true, ['H'] = true, ['I'] = true,
- ['J'] = true, ['K'] = true, ['L'] = true, ['M'] = true, ['N'] = true, ['O'] = true, ['P'] = true,
- ['Q'] = true, ['R'] = true, ['S'] = true, ['T'] = true, ['U'] = true, ['V'] = true, ['W'] = true,
- ['X'] = true, ['Y'] = true, ['Z'] = true, ['['] = true, ['\\'] = false, [']'] = true, ['^'] = true,
- ['_'] = true, ['`'] = true, ['a'] = true, ['b'] = true, ['c'] = true, ['d'] = true, ['e'] = true,
- ['f'] = true, ['g'] = true, ['h'] = true, ['i'] = true, ['j'] = true, ['k'] = true, ['l'] = true,
- ['m'] = true, ['n'] = true, ['o'] = true, ['p'] = true, ['q'] = true, ['r'] = true, ['s'] = true,
- ['t'] = true, ['u'] = true, ['v'] = true, ['w'] = true, ['x'] = true, ['y'] = true, ['z'] = true,
- ['{'] = true, ['|'] = true, ['}'] = true, ['~'] = true, [0x7f] = true,
- };
-
- if(!*s)
- return true;
-
- while(*s) {
- if(*s == '=' || isspace(*s) || !safe_for_logfmt[(uint8_t)*s])
- return true;
-
- s++;
- }
-
- return false;
-}
-
-static void string_to_logfmt(BUFFER *wb, const char *s) {
- bool spaces = needs_quotes_for_logfmt(s);
-
- if(spaces)
- buffer_fast_strcat(wb, "\"", 1);
-
- buffer_json_strcat(wb, s);
-
- if(spaces)
- buffer_fast_strcat(wb, "\"", 1);
-}
-
-static void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max) {
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- CLEAN_BUFFER *tmp = NULL;
-
- for (size_t i = 0; i < fields_max; i++) {
- if (!fields[i].entry.set || !fields[i].logfmt)
- continue;
-
- const char *key = fields[i].logfmt;
-
- if(fields[i].logfmt_annotator)
- fields[i].logfmt_annotator(wb, key, &fields[i]);
- else {
- if(buffer_strlen(wb))
- buffer_fast_strcat(wb, " ", 1);
-
- switch(fields[i].entry.type) {
- case NDFT_TXT:
- if(*fields[i].entry.txt) {
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- string_to_logfmt(wb, fields[i].entry.txt);
- }
- break;
- case NDFT_STR:
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- string_to_logfmt(wb, string2str(fields[i].entry.str));
- break;
- case NDFT_BFR:
- if(buffer_strlen(fields[i].entry.bfr)) {
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- string_to_logfmt(wb, buffer_tostring(fields[i].entry.bfr));
- }
- break;
- case NDFT_U64:
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_print_uint64(wb, fields[i].entry.u64);
- break;
- case NDFT_I64:
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_print_int64(wb, fields[i].entry.i64);
- break;
- case NDFT_DBL:
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_print_netdata_double(wb, fields[i].entry.dbl);
- break;
- case NDFT_UUID: {
- char u[UUID_COMPACT_STR_LEN];
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- buffer_fast_strcat(wb, u, sizeof(u) - 1);
- }
- break;
- case NDFT_CALLBACK: {
- if(!tmp)
- tmp = buffer_create(1024, NULL);
- else
- buffer_flush(tmp);
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) {
- buffer_strcat(wb, key);
- buffer_fast_strcat(wb, "=", 1);
- string_to_logfmt(wb, buffer_tostring(tmp));
- }
- }
- break;
- default:
- buffer_strcat(wb, "UNHANDLED");
- break;
- }
- }
- }
-}
-
-// ----------------------------------------------------------------------------
-// journal logger
-
-bool nd_log_journal_socket_available(void) {
- if(netdata_configured_host_prefix && *netdata_configured_host_prefix) {
- char filename[FILENAME_MAX + 1];
-
- snprintfz(filename, sizeof(filename), "%s%s",
- netdata_configured_host_prefix, "/run/systemd/journal/socket");
-
- if(is_path_unix_socket(filename))
- return true;
- }
-
- return is_path_unix_socket("/run/systemd/journal/socket");
-}
-
-static bool nd_logger_journal_libsystemd(struct log_field *fields, size_t fields_max) {
-#ifdef HAVE_SYSTEMD
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- struct iovec iov[fields_max];
- int iov_count = 0;
-
- memset(iov, 0, sizeof(iov));
-
- CLEAN_BUFFER *tmp = NULL;
-
- for (size_t i = 0; i < fields_max; i++) {
- if (!fields[i].entry.set || !fields[i].journal)
- continue;
-
- const char *key = fields[i].journal;
- char *value = NULL;
- switch (fields[i].entry.type) {
- case NDFT_TXT:
- if(*fields[i].entry.txt)
- asprintf(&value, "%s=%s", key, fields[i].entry.txt);
- break;
- case NDFT_STR:
- asprintf(&value, "%s=%s", key, string2str(fields[i].entry.str));
- break;
- case NDFT_BFR:
- if(buffer_strlen(fields[i].entry.bfr))
- asprintf(&value, "%s=%s", key, buffer_tostring(fields[i].entry.bfr));
- break;
- case NDFT_U64:
- asprintf(&value, "%s=%" PRIu64, key, fields[i].entry.u64);
- break;
- case NDFT_I64:
- asprintf(&value, "%s=%" PRId64, key, fields[i].entry.i64);
- break;
- case NDFT_DBL:
- asprintf(&value, "%s=%f", key, fields[i].entry.dbl);
- break;
- case NDFT_UUID: {
- char u[UUID_COMPACT_STR_LEN];
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
- asprintf(&value, "%s=%s", key, u);
- }
- break;
- case NDFT_CALLBACK: {
- if(!tmp)
- tmp = buffer_create(1024, NULL);
- else
- buffer_flush(tmp);
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
- asprintf(&value, "%s=%s", key, buffer_tostring(tmp));
- }
- break;
- default:
- asprintf(&value, "%s=%s", key, "UNHANDLED");
- break;
- }
-
- if (value) {
- iov[iov_count].iov_base = value;
- iov[iov_count].iov_len = strlen(value);
- iov_count++;
- }
- }
-
- int r = sd_journal_sendv(iov, iov_count);
-
- // Clean up allocated memory
- for (int i = 0; i < iov_count; i++) {
- if (iov[i].iov_base != NULL) {
- free(iov[i].iov_base);
- }
- }
-
- return r == 0;
-#else
- return false;
-#endif
-}
-
-static bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max) {
- if(!nd_log.journal_direct.initialized)
- return false;
-
- // --- FIELD_PARSER_VERSIONS ---
- //
- // IMPORTANT:
- // THERE ARE 6 VERSIONS OF THIS CODE
- //
- // 1. journal (direct socket API),
- // 2. journal (libsystemd API),
- // 3. logfmt,
- // 4. json,
- // 5. convert to uint64
- // 6. convert to int64
- //
- // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES
-
- CLEAN_BUFFER *wb = buffer_create(4096, NULL);
- CLEAN_BUFFER *tmp = NULL;
-
- for (size_t i = 0; i < fields_max; i++) {
- if (!fields[i].entry.set || !fields[i].journal)
- continue;
-
- const char *key = fields[i].journal;
-
- const char *s = NULL;
- switch(fields[i].entry.type) {
- case NDFT_TXT:
- s = fields[i].entry.txt;
- break;
- case NDFT_STR:
- s = string2str(fields[i].entry.str);
- break;
- case NDFT_BFR:
- s = buffer_tostring(fields[i].entry.bfr);
- break;
- case NDFT_U64:
- buffer_strcat(wb, key);
- buffer_putc(wb, '=');
- buffer_print_uint64(wb, fields[i].entry.u64);
- buffer_putc(wb, '\n');
- break;
- case NDFT_I64:
- buffer_strcat(wb, key);
- buffer_putc(wb, '=');
- buffer_print_int64(wb, fields[i].entry.i64);
- buffer_putc(wb, '\n');
- break;
- case NDFT_DBL:
- buffer_strcat(wb, key);
- buffer_putc(wb, '=');
- buffer_print_netdata_double(wb, fields[i].entry.dbl);
- buffer_putc(wb, '\n');
- break;
- case NDFT_UUID:{
- char u[UUID_COMPACT_STR_LEN];
- uuid_unparse_lower_compact(*fields[i].entry.uuid, u);
- buffer_strcat(wb, key);
- buffer_putc(wb, '=');
- buffer_fast_strcat(wb, u, sizeof(u) - 1);
- buffer_putc(wb, '\n');
- }
- break;
- case NDFT_CALLBACK: {
- if(!tmp)
- tmp = buffer_create(1024, NULL);
- else
- buffer_flush(tmp);
- if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data))
- s = buffer_tostring(tmp);
- else
- s = NULL;
- }
- break;
- default:
- s = "UNHANDLED";
- break;
- }
-
- if(s && *s) {
- buffer_strcat(wb, key);
- if(!strchr(s, '\n')) {
- buffer_putc(wb, '=');
- buffer_strcat(wb, s);
- buffer_putc(wb, '\n');
- }
- else {
- buffer_putc(wb, '\n');
- size_t size = strlen(s);
- uint64_t le_size = htole64(size);
- buffer_memcat(wb, &le_size, sizeof(le_size));
- buffer_memcat(wb, s, size);
- buffer_putc(wb, '\n');
- }
- }
- }
-
- return journal_direct_send(nd_log.journal_direct.fd, buffer_tostring(wb), buffer_strlen(wb));
-}
-
-// ----------------------------------------------------------------------------
-// syslog logger - uses logfmt
-
-static bool nd_logger_syslog(int priority, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max) {
- CLEAN_BUFFER *wb = buffer_create(1024, NULL);
-
- nd_logger_logfmt(wb, fields, fields_max);
- syslog(priority, "%s", buffer_tostring(wb));
-
- return true;
-}
-
-// ----------------------------------------------------------------------------
-// file logger - uses logfmt
-
-static bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max) {
- BUFFER *wb = buffer_create(1024, NULL);
-
- if(format == NDLF_JSON)
- nd_logger_json(wb, fields, fields_max);
- else
- nd_logger_logfmt(wb, fields, fields_max);
-
- int r = fprintf(fp, "%s\n", buffer_tostring(wb));
- fflush(fp);
-
- buffer_free(wb);
- return r > 0;
-}
-
-// ----------------------------------------------------------------------------
-// logger router
-
-static ND_LOG_METHOD nd_logger_select_output(ND_LOG_SOURCES source, FILE **fpp, SPINLOCK **spinlock) {
- *spinlock = NULL;
- ND_LOG_METHOD output = nd_log.sources[source].method;
-
- switch(output) {
- case NDLM_JOURNAL:
- if(unlikely(!nd_log.journal_direct.initialized && !nd_log.journal.initialized)) {
- output = NDLM_FILE;
- *fpp = stderr;
- *spinlock = &nd_log.std_error.spinlock;
- }
- else {
- *fpp = NULL;
- *spinlock = NULL;
- }
- break;
-
- case NDLM_SYSLOG:
- if(unlikely(!nd_log.syslog.initialized)) {
- output = NDLM_FILE;
- *spinlock = &nd_log.std_error.spinlock;
- *fpp = stderr;
- }
- else {
- *spinlock = NULL;
- *fpp = NULL;
- }
- break;
-
- case NDLM_FILE:
- if(!nd_log.sources[source].fp) {
- *fpp = stderr;
- *spinlock = &nd_log.std_error.spinlock;
- }
- else {
- *fpp = nd_log.sources[source].fp;
- *spinlock = &nd_log.sources[source].spinlock;
- }
- break;
-
- case NDLM_STDOUT:
- output = NDLM_FILE;
- *fpp = stdout;
- *spinlock = &nd_log.std_output.spinlock;
- break;
-
- default:
- case NDLM_DEFAULT:
- case NDLM_STDERR:
- output = NDLM_FILE;
- *fpp = stderr;
- *spinlock = &nd_log.std_error.spinlock;
- break;
-
- case NDLM_DISABLED:
- case NDLM_DEVNULL:
- output = NDLM_DISABLED;
- *fpp = NULL;
- *spinlock = NULL;
- break;
- }
-
- return output;
-}
-
-// ----------------------------------------------------------------------------
-// high level logger
-
-static void nd_logger_log_fields(SPINLOCK *spinlock, FILE *fp, bool limit, ND_LOG_FIELD_PRIORITY priority,
- ND_LOG_METHOD output, struct nd_log_source *source,
- struct log_field *fields, size_t fields_max) {
- if(spinlock)
- spinlock_lock(spinlock);
-
- // check the limits
- if(limit && nd_log_limit_reached(source))
- goto cleanup;
-
- if(output == NDLM_JOURNAL) {
- if(!nd_logger_journal_direct(fields, fields_max) && !nd_logger_journal_libsystemd(fields, fields_max)) {
- // we can't log to journal, let's log to stderr
- if(spinlock)
- spinlock_unlock(spinlock);
-
- output = NDLM_FILE;
- spinlock = &nd_log.std_error.spinlock;
- fp = stderr;
-
- if(spinlock)
- spinlock_lock(spinlock);
- }
- }
-
- if(output == NDLM_SYSLOG)
- nd_logger_syslog(priority, source->format, fields, fields_max);
-
- if(output == NDLM_FILE)
- nd_logger_file(fp, source->format, fields, fields_max);
-
-
-cleanup:
- if(spinlock)
- spinlock_unlock(spinlock);
-}
-
-static void nd_logger_unset_all_thread_fields(void) {
- size_t fields_max = THREAD_FIELDS_MAX;
- for(size_t i = 0; i < fields_max ; i++)
- thread_log_fields[i].entry.set = false;
-}
-
-static void nd_logger_merge_log_stack_to_thread_fields(void) {
- for(size_t c = 0; c < thread_log_stack_next ;c++) {
- struct log_stack_entry *lgs = thread_log_stack_base[c];
-
- for(size_t i = 0; lgs[i].id != NDF_STOP ; i++) {
- if(lgs[i].id >= _NDF_MAX || !lgs[i].set)
- continue;
-
- struct log_stack_entry *e = &lgs[i];
- ND_LOG_STACK_FIELD_TYPE type = lgs[i].type;
-
- // do not add empty / unset fields
- if((type == NDFT_TXT && (!e->txt || !*e->txt)) ||
- (type == NDFT_BFR && (!e->bfr || !buffer_strlen(e->bfr))) ||
- (type == NDFT_STR && !e->str) ||
- (type == NDFT_UUID && !e->uuid) ||
- (type == NDFT_CALLBACK && !e->cb.formatter) ||
- type == NDFT_UNSET)
- continue;
-
- thread_log_fields[lgs[i].id].entry = *e;
- }
- }
-}
-
-static void nd_logger(const char *file, const char *function, const unsigned long line,
- ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, bool limit, int saved_errno,
- const char *fmt, va_list ap) {
-
- SPINLOCK *spinlock;
- FILE *fp;
- ND_LOG_METHOD output = nd_logger_select_output(source, &fp, &spinlock);
- if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG)
- return;
-
- // mark all fields as unset
- nd_logger_unset_all_thread_fields();
-
- // flatten the log stack into the fields
- nd_logger_merge_log_stack_to_thread_fields();
-
- // set the common fields that are automatically set by the logging subsystem
-
- if(likely(!thread_log_fields[NDF_INVOCATION_ID].entry.set))
- thread_log_fields[NDF_INVOCATION_ID].entry = ND_LOG_FIELD_UUID(NDF_INVOCATION_ID, &nd_log.invocation_id);
-
- if(likely(!thread_log_fields[NDF_LOG_SOURCE].entry.set))
- thread_log_fields[NDF_LOG_SOURCE].entry = ND_LOG_FIELD_TXT(NDF_LOG_SOURCE, nd_log_id2source(source));
- else {
- ND_LOG_SOURCES src = source;
-
- if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_TXT)
- src = nd_log_source2id(thread_log_fields[NDF_LOG_SOURCE].entry.txt, source);
- else if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_U64)
- src = thread_log_fields[NDF_LOG_SOURCE].entry.u64;
-
- if(src != source && src >= 0 && src < _NDLS_MAX) {
- source = src;
- output = nd_logger_select_output(source, &fp, &spinlock);
- if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG)
- return;
- }
- }
-
- if(likely(!thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry.set))
- thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, program_name);
-
- if(likely(!thread_log_fields[NDF_LINE].entry.set)) {
- thread_log_fields[NDF_LINE].entry = ND_LOG_FIELD_U64(NDF_LINE, line);
- thread_log_fields[NDF_FILE].entry = ND_LOG_FIELD_TXT(NDF_FILE, file);
- thread_log_fields[NDF_FUNC].entry = ND_LOG_FIELD_TXT(NDF_FUNC, function);
- }
-
- if(likely(!thread_log_fields[NDF_PRIORITY].entry.set)) {
- thread_log_fields[NDF_PRIORITY].entry = ND_LOG_FIELD_U64(NDF_PRIORITY, priority);
- }
-
- if(likely(!thread_log_fields[NDF_TID].entry.set))
- thread_log_fields[NDF_TID].entry = ND_LOG_FIELD_U64(NDF_TID, gettid());
-
- char os_threadname[NETDATA_THREAD_NAME_MAX + 1];
- if(likely(!thread_log_fields[NDF_THREAD_TAG].entry.set)) {
- const char *thread_tag = netdata_thread_tag();
- if(!netdata_thread_tag_exists()) {
- if (!netdata_thread_tag_exists()) {
- os_thread_get_current_name_np(os_threadname);
- if ('\0' != os_threadname[0])
- /* If it is not an empty string replace "MAIN" thread_tag */
- thread_tag = os_threadname;
- }
- }
- thread_log_fields[NDF_THREAD_TAG].entry = ND_LOG_FIELD_TXT(NDF_THREAD_TAG, thread_tag);
-
- // TODO: fix the ND_MODULE in logging by setting proper module name in threads
-// if(!thread_log_fields[NDF_MODULE].entry.set)
-// thread_log_fields[NDF_MODULE].entry = ND_LOG_FIELD_CB(NDF_MODULE, thread_tag_to_module, (void *)thread_tag);
- }
-
- if(likely(!thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry.set))
- thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = ND_LOG_FIELD_U64(NDF_TIMESTAMP_REALTIME_USEC, now_realtime_usec());
-
- if(saved_errno != 0 && !thread_log_fields[NDF_ERRNO].entry.set)
- thread_log_fields[NDF_ERRNO].entry = ND_LOG_FIELD_I64(NDF_ERRNO, saved_errno);
-
- CLEAN_BUFFER *wb = NULL;
- if(fmt && !thread_log_fields[NDF_MESSAGE].entry.set) {
- wb = buffer_create(1024, NULL);
- buffer_vsprintf(wb, fmt, ap);
- thread_log_fields[NDF_MESSAGE].entry = ND_LOG_FIELD_TXT(NDF_MESSAGE, buffer_tostring(wb));
- }
-
- nd_logger_log_fields(spinlock, fp, limit, priority, output, &nd_log.sources[source],
- thread_log_fields, THREAD_FIELDS_MAX);
-
- if(nd_log.sources[source].pending_msg) {
- // log a pending message
-
- nd_logger_unset_all_thread_fields();
-
- thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = (struct log_stack_entry){
- .set = true,
- .type = NDFT_U64,
- .u64 = now_realtime_usec(),
- };
-
- thread_log_fields[NDF_LOG_SOURCE].entry = (struct log_stack_entry){
- .set = true,
- .type = NDFT_TXT,
- .txt = nd_log_id2source(source),
- };
-
- thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = (struct log_stack_entry){
- .set = true,
- .type = NDFT_TXT,
- .txt = program_name,
- };
-
- thread_log_fields[NDF_MESSAGE].entry = (struct log_stack_entry){
- .set = true,
- .type = NDFT_TXT,
- .txt = nd_log.sources[source].pending_msg,
- };
-
- nd_logger_log_fields(spinlock, fp, false, priority, output,
- &nd_log.sources[source],
- thread_log_fields, THREAD_FIELDS_MAX);
-
- freez((void *)nd_log.sources[source].pending_msg);
- nd_log.sources[source].pending_msg = NULL;
- }
-
- errno = 0;
-}
-
-static ND_LOG_SOURCES nd_log_validate_source(ND_LOG_SOURCES source) {
- if(source >= _NDLS_MAX)
- source = NDLS_DAEMON;
-
- if(overwrite_thread_source)
- source = overwrite_thread_source;
-
- if(nd_log.overwrite_process_source)
- source = nd_log.overwrite_process_source;
-
- return source;
-}
-
-// ----------------------------------------------------------------------------
-// public API for loggers
-
-void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... ) {
- int saved_errno = errno;
- source = nd_log_validate_source(source);
-
- if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
- return;
-
- va_list args;
- va_start(args, fmt);
- nd_logger(file, function, line, source, priority,
- source == NDLS_DAEMON || source == NDLS_COLLECTORS,
- saved_errno, fmt, args);
- va_end(args);
-}
-
-void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) {
- int saved_errno = errno;
- source = nd_log_validate_source(source);
-
- if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority)
- return;
-
- if(erl->sleep_ut)
- sleep_usec(erl->sleep_ut);
-
- spinlock_lock(&erl->spinlock);
-
- erl->count++;
- time_t now = now_boottime_sec();
- if(now - erl->last_logged < erl->log_every) {
- spinlock_unlock(&erl->spinlock);
- return;
- }
-
- spinlock_unlock(&erl->spinlock);
-
- va_list args;
- va_start(args, fmt);
- nd_logger(file, function, line, source, priority,
- source == NDLS_DAEMON || source == NDLS_COLLECTORS,
- saved_errno, fmt, args);
- va_end(args);
- erl->last_logged = now;
- erl->count = 0;
-}
-
-void netdata_logger_fatal( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
- int saved_errno = errno;
- ND_LOG_SOURCES source = NDLS_DAEMON;
- source = nd_log_validate_source(source);
-
- va_list args;
- va_start(args, fmt);
- nd_logger(file, function, line, source, NDLP_ALERT, true, saved_errno, fmt, args);
- va_end(args);
-
- char date[LOG_DATE_LENGTH];
- log_date(date, LOG_DATE_LENGTH, now_realtime_sec());
-
- char action_data[70+1];
- snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, saved_errno);
- char action_result[60+1];
-
- char os_threadname[NETDATA_THREAD_NAME_MAX + 1];
- const char *thread_tag = netdata_thread_tag();
- if(!netdata_thread_tag_exists()) {
- if (!netdata_thread_tag_exists()) {
- os_thread_get_current_name_np(os_threadname);
- if ('\0' != os_threadname[0])
- /* If it is not an empty string replace "MAIN" thread_tag */
- thread_tag = os_threadname;
- }
- }
- if(!thread_tag)
- thread_tag = "UNKNOWN";
-
- const char *tag_to_send = thread_tag;
-
- // anonymize thread names
- if(strncmp(thread_tag, THREAD_TAG_STREAM_RECEIVER, strlen(THREAD_TAG_STREAM_RECEIVER)) == 0)
- tag_to_send = THREAD_TAG_STREAM_RECEIVER;
- if(strncmp(thread_tag, THREAD_TAG_STREAM_SENDER, strlen(THREAD_TAG_STREAM_SENDER)) == 0)
- tag_to_send = THREAD_TAG_STREAM_SENDER;
-
- snprintfz(action_result, 60, "%s:%s", program_name, tag_to_send);
- send_statistics("FATAL", action_result, action_data);
-
-#ifdef HAVE_BACKTRACE
- int fd = nd_log.sources[NDLS_DAEMON].fd;
- if(fd == -1)
- fd = STDERR_FILENO;
-
- int nptrs;
- void *buffer[10000];
-
- nptrs = backtrace(buffer, sizeof(buffer));
- if(nptrs)
- backtrace_symbols_fd(buffer, nptrs, fd);
-#endif
-
-#ifdef NETDATA_INTERNAL_CHECKS
- abort();
-#endif
-
- netdata_cleanup_and_exit(1);
-}
-
-// ----------------------------------------------------------------------------
-// log limits
-
-void nd_log_limits_reset(void) {
- usec_t now_ut = now_monotonic_usec();
-
- spinlock_lock(&nd_log.std_output.spinlock);
- spinlock_lock(&nd_log.std_error.spinlock);
-
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
- spinlock_lock(&nd_log.sources[i].spinlock);
- nd_log.sources[i].limits.prevented = 0;
- nd_log.sources[i].limits.counter = 0;
- nd_log.sources[i].limits.started_monotonic_ut = now_ut;
- nd_log.sources[i].limits.logs_per_period = nd_log.sources[i].limits.logs_per_period_backup;
- spinlock_unlock(&nd_log.sources[i].spinlock);
- }
-
- spinlock_unlock(&nd_log.std_output.spinlock);
- spinlock_unlock(&nd_log.std_error.spinlock);
-}
-
-void nd_log_limits_unlimited(void) {
- nd_log_limits_reset();
- for(size_t i = 0; i < _NDLS_MAX ;i++) {
- nd_log.sources[i].limits.logs_per_period = 0;
- }
-}
-
-static bool nd_log_limit_reached(struct nd_log_source *source) {
- if(source->limits.throttle_period == 0 || source->limits.logs_per_period == 0)
- return false;
-
- usec_t now_ut = now_monotonic_usec();
- if(!source->limits.started_monotonic_ut)
- source->limits.started_monotonic_ut = now_ut;
-
- source->limits.counter++;
-
- if(now_ut - source->limits.started_monotonic_ut > (usec_t)source->limits.throttle_period) {
- if(source->limits.prevented) {
- BUFFER *wb = buffer_create(1024, NULL);
- buffer_sprintf(wb,
- "LOG FLOOD PROTECTION: resuming logging "
- "(prevented %"PRIu32" logs in the last %"PRIu32" seconds).",
- source->limits.prevented,
- source->limits.throttle_period);
-
- if(source->pending_msg)
- freez((void *)source->pending_msg);
-
- source->pending_msg = strdupz(buffer_tostring(wb));
-
- buffer_free(wb);
- }
-
- // restart the period accounting
- source->limits.started_monotonic_ut = now_ut;
- source->limits.counter = 1;
- source->limits.prevented = 0;
-
- // log this error
- return false;
- }
-
- if(source->limits.counter > source->limits.logs_per_period) {
- if(!source->limits.prevented) {
- BUFFER *wb = buffer_create(1024, NULL);
- buffer_sprintf(wb,
- "LOG FLOOD PROTECTION: too many logs (%"PRIu32" logs in %"PRId64" seconds, threshold is set to %"PRIu32" logs "
- "in %"PRIu32" seconds). Preventing more logs from process '%s' for %"PRId64" seconds.",
- source->limits.counter,
- (int64_t)((now_ut - source->limits.started_monotonic_ut) / USEC_PER_SEC),
- source->limits.logs_per_period,
- source->limits.throttle_period,
- program_name,
- (int64_t)((source->limits.started_monotonic_ut + (source->limits.throttle_period * USEC_PER_SEC) - now_ut)) / USEC_PER_SEC);
-
- if(source->pending_msg)
- freez((void *)source->pending_msg);
-
- source->pending_msg = strdupz(buffer_tostring(wb));
-
- buffer_free(wb);
- }
-
- source->limits.prevented++;
-
- // prevent logging this error
-#ifdef NETDATA_INTERNAL_CHECKS
- return false;
-#else
- return true;
-#endif
- }
-
- return false;
-}
diff --git a/libnetdata/log/log.h b/libnetdata/log/log.h
deleted file mode 100644
index ad634693c..000000000
--- a/libnetdata/log/log.h
+++ /dev/null
@@ -1,301 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_LOG_H
-#define NETDATA_LOG_H 1
-
-# ifdef __cplusplus
-extern "C" {
-# endif
-
-#include "../libnetdata.h"
-
-#define ND_LOG_DEFAULT_THROTTLE_LOGS 1000
-#define ND_LOG_DEFAULT_THROTTLE_PERIOD 60
-
-typedef enum __attribute__((__packed__)) {
- NDLS_UNSET = 0, // internal use only
- NDLS_ACCESS, // access.log
- NDLS_ACLK, // aclk.log
- NDLS_COLLECTORS, // collectors.log
- NDLS_DAEMON, // error.log
- NDLS_HEALTH, // health.log
- NDLS_DEBUG, // debug.log
-
- // terminator
- _NDLS_MAX,
-} ND_LOG_SOURCES;
-
-typedef enum __attribute__((__packed__)) {
- NDLP_EMERG = LOG_EMERG,
- NDLP_ALERT = LOG_ALERT,
- NDLP_CRIT = LOG_CRIT,
- NDLP_ERR = LOG_ERR,
- NDLP_WARNING = LOG_WARNING,
- NDLP_NOTICE = LOG_NOTICE,
- NDLP_INFO = LOG_INFO,
- NDLP_DEBUG = LOG_DEBUG,
-} ND_LOG_FIELD_PRIORITY;
-
-typedef enum __attribute__((__packed__)) {
- // KEEP THESE IN THE SAME ORDER AS in thread_log_fields (log.c)
- // so that it easy to audit for missing fields
-
- NDF_STOP = 0,
- NDF_TIMESTAMP_REALTIME_USEC, // the timestamp of the log message - added automatically
- NDF_SYSLOG_IDENTIFIER, // the syslog identifier of the application - added automatically
- NDF_LOG_SOURCE, // DAEMON, COLLECTORS, HEALTH, ACCESS, ACLK - set at the log call
- NDF_PRIORITY, // the syslog priority (severity) - set at the log call
- NDF_ERRNO, // the ERRNO at the time of the log call - added automatically
- NDF_INVOCATION_ID, // the INVOCATION_ID of Netdata - added automatically
- NDF_LINE, // the source code file line number - added automatically
- NDF_FILE, // the source code filename - added automatically
- NDF_FUNC, // the source code function - added automatically
- NDF_TID, // the thread ID of the thread logging - added automatically
- NDF_THREAD_TAG, // the thread tag of the thread logging - added automatically
- NDF_MESSAGE_ID, // for specific events
- NDF_MODULE, // for internal plugin module, all other get the NDF_THREAD_TAG
-
- NDF_NIDL_NODE, // the node / rrdhost currently being worked
- NDF_NIDL_INSTANCE, // the instance / rrdset currently being worked
- NDF_NIDL_CONTEXT, // the context of the instance currently being worked
- NDF_NIDL_DIMENSION, // the dimension / rrddim currently being worked
-
- // web server, aclk and stream receiver
- NDF_SRC_TRANSPORT, // the transport we received the request, one of: http, https, pluginsd
-
- // web server and stream receiver
- NDF_SRC_IP, // the streaming / web server source IP
- NDF_SRC_PORT, // the streaming / web server source Port
- NDF_SRC_CAPABILITIES, // the stream receiver capabilities
-
- // stream sender (established links)
- NDF_DST_TRANSPORT, // the transport we send the request, one of: http, https
- NDF_DST_IP, // the destination streaming IP
- NDF_DST_PORT, // the destination streaming Port
- NDF_DST_CAPABILITIES, // the destination streaming capabilities
-
- // web server, aclk and stream receiver
- NDF_REQUEST_METHOD, // for http like requests, the http request method
- NDF_RESPONSE_CODE, // for http like requests, the http response code, otherwise a status string
-
- // web server (all), aclk (queries)
- NDF_CONNECTION_ID, // the web server connection ID
- NDF_TRANSACTION_ID, // the web server and API transaction ID
- NDF_RESPONSE_SENT_BYTES, // for http like requests, the response bytes
- NDF_RESPONSE_SIZE_BYTES, // for http like requests, the uncompressed response size
- NDF_RESPONSE_PREPARATION_TIME_USEC, // for http like requests, the preparation time
- NDF_RESPONSE_SENT_TIME_USEC, // for http like requests, the time to send the response back
- NDF_RESPONSE_TOTAL_TIME_USEC, // for http like requests, the total time to complete the response
-
- // health alerts
- NDF_ALERT_ID,
- NDF_ALERT_UNIQUE_ID,
- NDF_ALERT_EVENT_ID,
- NDF_ALERT_TRANSITION_ID,
- NDF_ALERT_CONFIG_HASH,
- NDF_ALERT_NAME,
- NDF_ALERT_CLASS,
- NDF_ALERT_COMPONENT,
- NDF_ALERT_TYPE,
- NDF_ALERT_EXEC,
- NDF_ALERT_RECIPIENT,
- NDF_ALERT_DURATION,
- NDF_ALERT_VALUE,
- NDF_ALERT_VALUE_OLD,
- NDF_ALERT_STATUS,
- NDF_ALERT_STATUS_OLD,
- NDF_ALERT_SOURCE,
- NDF_ALERT_UNITS,
- NDF_ALERT_SUMMARY,
- NDF_ALERT_INFO,
- NDF_ALERT_NOTIFICATION_REALTIME_USEC,
- // NDF_ALERT_FLAGS,
-
- // put new items here
- // leave the request URL and the message last
-
- NDF_REQUEST, // the request we are currently working on
- NDF_MESSAGE, // the log message, if any
-
- // terminator
- _NDF_MAX,
-} ND_LOG_FIELD_ID;
-
-typedef enum __attribute__((__packed__)) {
- NDFT_UNSET = 0,
- NDFT_TXT,
- NDFT_STR,
- NDFT_BFR,
- NDFT_U64,
- NDFT_I64,
- NDFT_DBL,
- NDFT_UUID,
- NDFT_CALLBACK,
-} ND_LOG_STACK_FIELD_TYPE;
-
-void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting);
-void nd_log_set_facility(const char *facility);
-void nd_log_set_priority_level(const char *setting);
-void nd_log_initialize(void);
-void nd_log_reopen_log_files(void);
-void chown_open_file(int fd, uid_t uid, gid_t gid);
-void nd_log_chown_log_files(uid_t uid, gid_t gid);
-void nd_log_set_flood_protection(size_t logs, time_t period);
-void nd_log_initialize_for_external_plugins(const char *name);
-void nd_log_set_thread_source(ND_LOG_SOURCES source);
-bool nd_log_journal_socket_available(void);
-ND_LOG_FIELD_ID nd_log_field_id_by_name(const char *field, size_t len);
-int nd_log_priority2id(const char *priority);
-const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);
-const char *nd_log_method_for_external_plugins(const char *s);
-
-int nd_log_health_fd(void);
-typedef bool (*log_formatter_callback_t)(BUFFER *wb, void *data);
-
-struct log_stack_entry {
- ND_LOG_FIELD_ID id;
- ND_LOG_STACK_FIELD_TYPE type;
- bool set;
- union {
- const char *txt;
- struct netdata_string *str;
- BUFFER *bfr;
- uint64_t u64;
- int64_t i64;
- double dbl;
- const uuid_t *uuid;
- struct {
- log_formatter_callback_t formatter;
- void *formatter_data;
- } cb;
- };
-};
-
-#define ND_LOG_STACK _cleanup_(log_stack_pop) struct log_stack_entry
-#define ND_LOG_STACK_PUSH(lgs) log_stack_push(lgs)
-
-#define ND_LOG_FIELD_TXT(field, value) (struct log_stack_entry){ .id = (field), .type = NDFT_TXT, .txt = (value), .set = true, }
-#define ND_LOG_FIELD_STR(field, value) (struct log_stack_entry){ .id = (field), .type = NDFT_STR, .str = (value), .set = true, }
-#define ND_LOG_FIELD_BFR(field, value) (struct log_stack_entry){ .id = (field), .type = NDFT_BFR, .bfr = (value), .set = true, }
-#define ND_LOG_FIELD_U64(field, value) (struct log_stack_entry){ .id = (field), .type = NDFT_U64, .u64 = (value), .set = true, }
-#define ND_LOG_FIELD_I64(field, value) (struct log_stack_entry){ .id = (field), .type = NDFT_I64, .i64 = (value), .set = true, }
-#define ND_LOG_FIELD_DBL(field, value) (struct log_stack_entry){ .id = (field), .type = NDFT_DBL, .dbl = (value), .set = true, }
-#define ND_LOG_FIELD_CB(field, func, data) (struct log_stack_entry){ .id = (field), .type = NDFT_CALLBACK, .cb = { .formatter = (func), .formatter_data = (data) }, .set = true, }
-#define ND_LOG_FIELD_UUID(field, value) (struct log_stack_entry){ .id = (field), .type = NDFT_UUID, .uuid = (value), .set = true, }
-#define ND_LOG_FIELD_END() (struct log_stack_entry){ .id = NDF_STOP, .type = NDFT_UNSET, .set = false, }
-
-void log_stack_pop(void *ptr);
-void log_stack_push(struct log_stack_entry *lgs);
-
-#define D_WEB_BUFFER 0x0000000000000001
-#define D_WEB_CLIENT 0x0000000000000002
-#define D_LISTENER 0x0000000000000004
-#define D_WEB_DATA 0x0000000000000008
-#define D_OPTIONS 0x0000000000000010
-#define D_PROCNETDEV_LOOP 0x0000000000000020
-#define D_RRD_STATS 0x0000000000000040
-#define D_WEB_CLIENT_ACCESS 0x0000000000000080
-#define D_TC_LOOP 0x0000000000000100
-#define D_DEFLATE 0x0000000000000200
-#define D_CONFIG 0x0000000000000400
-#define D_PLUGINSD 0x0000000000000800
-#define D_CHILDS 0x0000000000001000
-#define D_EXIT 0x0000000000002000
-#define D_CHECKS 0x0000000000004000
-#define D_NFACCT_LOOP 0x0000000000008000
-#define D_PROCFILE 0x0000000000010000
-#define D_RRD_CALLS 0x0000000000020000
-#define D_DICTIONARY 0x0000000000040000
-#define D_MEMORY 0x0000000000080000
-#define D_CGROUP 0x0000000000100000
-#define D_REGISTRY 0x0000000000200000
-#define D_VARIABLES 0x0000000000400000
-#define D_HEALTH 0x0000000000800000
-#define D_CONNECT_TO 0x0000000001000000
-#define D_RRDHOST 0x0000000002000000
-#define D_LOCKS 0x0000000004000000
-#define D_EXPORTING 0x0000000008000000
-#define D_STATSD 0x0000000010000000
-#define D_POLLFD 0x0000000020000000
-#define D_STREAM 0x0000000040000000
-#define D_ANALYTICS 0x0000000080000000
-#define D_RRDENGINE 0x0000000100000000
-#define D_ACLK 0x0000000200000000
-#define D_REPLICATION 0x0000002000000000
-#define D_SYSTEM 0x8000000000000000
-
-extern uint64_t debug_flags;
-
-extern const char *program_name;
-
-#ifdef ENABLE_ACLK
-extern int aclklog_enabled;
-#endif
-
-#define LOG_DATE_LENGTH 26
-void log_date(char *buffer, size_t len, time_t now);
-
-static inline void debug_dummy(void) {}
-
-void nd_log_limits_reset(void);
-void nd_log_limits_unlimited(void);
-
-#define NDLP_INFO_STR "info"
-
-#ifdef NETDATA_INTERNAL_CHECKS
-#define netdata_log_debug(type, args...) do { if(unlikely(debug_flags & type)) netdata_logger(NDLS_DEBUG, NDLP_DEBUG, __FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
-#define internal_error(condition, args...) do { if(unlikely(condition)) netdata_logger(NDLS_DAEMON, NDLP_DEBUG, __FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
-#define internal_fatal(condition, args...) do { if(unlikely(condition)) netdata_logger_fatal(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
-#else
-#define netdata_log_debug(type, args...) debug_dummy()
-#define internal_error(args...) debug_dummy()
-#define internal_fatal(args...) debug_dummy()
-#endif
-
-#define fatal(args...) netdata_logger_fatal(__FILE__, __FUNCTION__, __LINE__, ##args)
-#define fatal_assert(expr) ((expr) ? (void)(0) : netdata_logger_fatal(__FILE__, __FUNCTION__, __LINE__, "Assertion `%s' failed", #expr))
-
-// ----------------------------------------------------------------------------
-// normal logging
-
-void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... ) PRINTFLIKE(6, 7);
-#define nd_log(NDLS, NDLP, args...) netdata_logger(NDLS, NDLP, __FILE__, __FUNCTION__, __LINE__, ##args)
-#define nd_log_daemon(NDLP, args...) netdata_logger(NDLS_DAEMON, NDLP, __FILE__, __FUNCTION__, __LINE__, ##args)
-#define nd_log_collector(NDLP, args...) netdata_logger(NDLS_COLLECTORS, NDLP, __FILE__, __FUNCTION__, __LINE__, ##args)
-
-#define netdata_log_info(args...) netdata_logger(NDLS_DAEMON, NDLP_INFO, __FILE__, __FUNCTION__, __LINE__, ##args)
-#define netdata_log_error(args...) netdata_logger(NDLS_DAEMON, NDLP_ERR, __FILE__, __FUNCTION__, __LINE__, ##args)
-#define collector_info(args...) netdata_logger(NDLS_COLLECTORS, NDLP_INFO, __FILE__, __FUNCTION__, __LINE__, ##args)
-#define collector_error(args...) netdata_logger(NDLS_COLLECTORS, NDLP_ERR, __FILE__, __FUNCTION__, __LINE__, ##args)
-
-#define log_aclk_message_bin(__data, __data_len, __tx, __mqtt_topic, __message_name) \
- nd_log(NDLS_ACLK, NDLP_INFO, \
- "direction:%s message:'%s' topic:'%s' json:'%.*s'", \
- (__tx) ? "OUTGOING" : "INCOMING", __message_name, __mqtt_topic, (int)(__data_len), __data)
-
-// ----------------------------------------------------------------------------
-// logging with limits
-
-typedef struct error_with_limit {
- SPINLOCK spinlock;
- time_t log_every;
- size_t count;
- time_t last_logged;
- usec_t sleep_ut;
-} ERROR_LIMIT;
-
-#define nd_log_limit_static_global_var(var, log_every_secs, sleep_usecs) static ERROR_LIMIT var = { .last_logged = 0, .count = 0, .log_every = (log_every_secs), .sleep_ut = (sleep_usecs) }
-#define nd_log_limit_static_thread_var(var, log_every_secs, sleep_usecs) static __thread ERROR_LIMIT var = { .last_logged = 0, .count = 0, .log_every = (log_every_secs), .sleep_ut = (sleep_usecs) }
-void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... ) PRINTFLIKE(7, 8);;
-#define nd_log_limit(erl, NDLS, NDLP, args...) netdata_logger_with_limit(erl, NDLS, NDLP, __FILE__, __FUNCTION__, __LINE__, ##args)
-
-// ----------------------------------------------------------------------------
-
-void send_statistics(const char *action, const char *action_result, const char *action_data);
-void netdata_logger_fatal( const char *file, const char *function, unsigned long line, const char *fmt, ... ) NORETURN PRINTFLIKE(4, 5);
-
-# ifdef __cplusplus
-}
-# endif
-
-#endif /* NETDATA_LOG_H */
diff --git a/libnetdata/onewayalloc/Makefile.am b/libnetdata/onewayalloc/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/onewayalloc/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/onewayalloc/README.md b/libnetdata/onewayalloc/README.md
deleted file mode 100644
index 38d92cea6..000000000
--- a/libnetdata/onewayalloc/README.md
+++ /dev/null
@@ -1,75 +0,0 @@
-<!--
-title: "One Way Allocator"
-custom_edit_url: "https://github.com/netdata/netdata/edit/master/libnetdata/onewayalloc/README.md"
-sidebar_label: "One way allocator"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# One Way Allocator
-
-This is a very fast single-threaded-only memory allocator, that minimized system calls
-when a lot of memory allocations needs to be made to perform a task, which all of them
-can be freed together when the task finishes.
-
-It has been designed to be used for netdata context queries.
-
-For netdata to perform a context query, it builds a virtual chart, a chart that contains
-all the dimensions of the charts having the same context. This process requires allocating
-several structures for each of the dimensions to attach them to the virtual chart. All
-these data can be freed immediately after the query finishes.
-
-## How it works
-
-1. The caller calls `ONEWAYALLOC *owa = onewayalloc_create(sizehint)` to create an OWA.
- Internally this allocates the first memory buffer with size >= `sizehint`.
- If `sizehint` is zero, it will allocate 1 hardware page (usually 4kb).
- No need to check for success or failure. As with `mallocz()` in netdata, a `fatal()`
- will be called if the allocation fails - although this will never fail, since Linux
- does not really check if there is memory available for `mmap()` calls.
-
-2. The caller can then perform any number of the following calls to acquire memory:
- - `onewayalloc_mallocz(owa, size)`, similar to `mallocz()`
- - `onewayalloc_callocz(owa, nmemb, size)`, similar to `callocz()`
- - `onewayalloc_strdupz(owa, string)`, similar to `strdupz()`
- - `onewayalloc_memdupz(owa, ptr, size)`, similar to `mallocz()` and then `memcpy()`
-
-3. Once the caller has done all the work with the allocated buffers, all memory allocated
- can be freed with `onewayalloc_destroy(owa)`.
-
-## How faster it is?
-
-On modern hardware, for any single query the performance improvement is marginal and not
-noticeable at all.
-
-We performed the following tests using the same huge context query (1000 charts,
-100 dimensions each = 100k dimensions)
-
-1. using `mallocz()`, 1 caller, 256 queries (sequential)
-2. using `mallocz()`, 256 callers, 1 query each (parallel)
-3. using `OWA`, 1 caller, 256 queries (sequential)
-4. using `OWA`, 256 callers, 1 query each (parallel)
-
-Netdata was configured to use 24 web threads on the 24 core server we used.
-
-The results are as follows:
-
-### sequential test
-
-branch|transactions|time to complete|transaction rate|average response time|min response time|max response time
-:---:|:---:|:---:|:---:|:---:|:---:|:---:|
-`malloc()`|256|322.35s|0.79/sec|1.26s|1.01s|1.87s
-`OWA`|256|310.19s|0.83/sec|1.21s|1.04s|1.63s
-
-For a single query, the improvement is just marginal and not noticeable at all.
-
-### parallel test
-
-branch|transactions|time to complete|transaction rate|average response time|min response time|max response time
-:---:|:---:|:---:|:---:|:---:|:---:|:---:|
-`malloc()`|256|84.72s|3.02/sec|68.43s|50.20s|84.71s
-`OWA`|256|39.35s|6.51/sec|34.48s|20.55s|39.34s
-
-For parallel workload, like the one executed by netdata.cloud, `OWA` provides a 54% overall speed improvement (more than double the overall
-user-experienced speed, including the data query itself).
diff --git a/libnetdata/os.c b/libnetdata/os.c
deleted file mode 100644
index e4624be69..000000000
--- a/libnetdata/os.c
+++ /dev/null
@@ -1,300 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "os.h"
-
-// ----------------------------------------------------------------------------
-// system functions
-// to retrieve settings of the system
-
-#define CPUS_FOR_COLLECTORS 0
-#define CPUS_FOR_NETDATA 1
-
-long get_system_cpus_with_cache(bool cache, bool for_netdata) {
- static long processors[2] = { 0, 0 };
-
- int index = for_netdata ? CPUS_FOR_NETDATA : CPUS_FOR_COLLECTORS;
-
- if(likely(cache && processors[index] > 0))
- return processors[index];
-
-#if defined(__APPLE__) || defined(__FreeBSD__)
-#if defined(__APPLE__)
-#define HW_CPU_NAME "hw.logicalcpu"
-#else
-#define HW_CPU_NAME "hw.ncpu"
-#endif
-
- int32_t tmp_processors;
- bool error = false;
-
- if (unlikely(GETSYSCTL_BY_NAME(HW_CPU_NAME, tmp_processors)))
- error = true;
- else
- processors[index] = tmp_processors;
-
- if(processors[index] < 1) {
- processors[index] = 1;
-
- if(error)
- netdata_log_error("Assuming system has %d processors.", processors[index]);
- }
-
- return processors[index];
-#else
-
- char filename[FILENAME_MAX + 1];
- snprintfz(filename, FILENAME_MAX, "%s/proc/stat",
- (!for_netdata && netdata_configured_host_prefix) ? netdata_configured_host_prefix : "");
-
- procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT);
- if(!ff) {
- processors[index] = 1;
- netdata_log_error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]);
- return processors[index];
- }
-
- ff = procfile_readall(ff);
- if(!ff) {
- processors[index] = 1;
- netdata_log_error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]);
- return processors[index];
- }
-
- long tmp_processors = 0;
- unsigned int i;
- for(i = 0; i < procfile_lines(ff); i++) {
- if(!procfile_linewords(ff, i)) continue;
-
- if(strncmp(procfile_lineword(ff, i, 0), "cpu", 3) == 0)
- tmp_processors++;
- }
- procfile_close(ff);
-
- processors[index] = --tmp_processors;
-
- if(processors[index] < 1)
- processors[index] = 1;
-
- netdata_log_debug(D_SYSTEM, "System has %ld processors.", processors[index]);
- return processors[index];
-
-#endif /* __APPLE__, __FreeBSD__ */
-}
-
-pid_t pid_max = 32768;
-pid_t get_system_pid_max(void) {
-#ifdef __APPLE__
- // As we currently do not know a solution to query pid_max from the os
- // we use the number defined in bsd/sys/proc_internal.h in XNU sources
- pid_max = 99999;
- return pid_max;
-#elif __FreeBSD__
- int32_t tmp_pid_max;
-
- if (unlikely(GETSYSCTL_BY_NAME("kern.pid_max", tmp_pid_max))) {
- pid_max = 99999;
- netdata_log_error("Assuming system's maximum pid is %d.", pid_max);
- } else {
- pid_max = tmp_pid_max;
- }
-
- return pid_max;
-#else
-
- static char read = 0;
- if(unlikely(read)) return pid_max;
- read = 1;
-
- char filename[FILENAME_MAX + 1];
- snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", netdata_configured_host_prefix?netdata_configured_host_prefix:"");
-
- unsigned long long max = 0;
- if(read_single_number_file(filename, &max) != 0) {
- netdata_log_error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max);
- return pid_max;
- }
-
- if(!max) {
- netdata_log_error("Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max);
- return pid_max;
- }
-
- pid_max = (pid_t) max;
- return pid_max;
-
-#endif /* __APPLE__, __FreeBSD__ */
-}
-
-unsigned int system_hz;
-void get_system_HZ(void) {
- long ticks;
-
- if ((ticks = sysconf(_SC_CLK_TCK)) == -1) {
- netdata_log_error("Cannot get system clock ticks");
- }
-
- system_hz = (unsigned int) ticks;
-}
-
-static inline unsigned long cpuset_str2ul(char **s) {
- unsigned long n = 0;
- char c;
- for(c = **s; c >= '0' && c <= '9' ; c = *(++*s)) {
- n *= 10;
- n += c - '0';
- }
- return n;
-}
-
-unsigned long read_cpuset_cpus(const char *filename, long system_cpus) {
- static char *buf = NULL;
- static size_t buf_size = 0;
-
- if(!buf) {
- buf_size = 100U + 6 * system_cpus; // taken from kernel/cgroup/cpuset.c
- buf = mallocz(buf_size + 1);
- }
-
- int ret = read_file(filename, buf, buf_size);
-
- if(!ret) {
- char *s = buf;
- unsigned long ncpus = 0;
-
- // parse the cpuset string and calculate the number of cpus the cgroup is allowed to use
- while (*s) {
- if (isspace(*s)) {
- s++;
- continue;
- }
- unsigned long n = cpuset_str2ul(&s);
- ncpus++;
- if(*s == ',') {
- s++;
- continue;
- }
- if(*s == '-') {
- s++;
- unsigned long m = cpuset_str2ul(&s);
- ncpus += m - n; // calculate the number of cpus in the region
- }
- s++;
- }
-
- if(!ncpus)
- return 0;
-
- return ncpus;
- }
-
- return 0;
-}
-
-// =====================================================================================================================
-// FreeBSD
-
-#if __FreeBSD__
-
-const char *os_type = "freebsd";
-
-int getsysctl_by_name(const char *name, void *ptr, size_t len) {
- size_t nlen = len;
-
- if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) {
- netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno));
- return 1;
- }
- if (unlikely(nlen != len)) {
- netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen);
- return 1;
- }
- return 0;
-}
-
-int getsysctl_simple(const char *name, int *mib, size_t miblen, void *ptr, size_t len) {
- size_t nlen = len;
-
- if (unlikely(!mib[0]))
- if (unlikely(getsysctl_mib(name, mib, miblen)))
- return 1;
-
- if (unlikely(sysctl(mib, miblen, ptr, &nlen, NULL, 0) == -1)) {
- netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno));
- return 1;
- }
- if (unlikely(nlen != len)) {
- netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen);
- return 1;
- }
-
- return 0;
-}
-
-int getsysctl(const char *name, int *mib, size_t miblen, void *ptr, size_t *len) {
- size_t nlen = *len;
-
- if (unlikely(!mib[0]))
- if (unlikely(getsysctl_mib(name, mib, miblen)))
- return 1;
-
- if (unlikely(sysctl(mib, miblen, ptr, len, NULL, 0) == -1)) {
- netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno));
- return 1;
- }
- if (unlikely(ptr != NULL && nlen != *len)) {
- netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)*len, (unsigned long)nlen);
- return 1;
- }
-
- return 0;
-}
-
-int getsysctl_mib(const char *name, int *mib, size_t len) {
- size_t nlen = len;
-
- if (unlikely(sysctlnametomib(name, mib, &nlen) == -1)) {
- netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno));
- return 1;
- }
- if (unlikely(nlen != len)) {
- netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen);
- return 1;
- }
- return 0;
-}
-
-
-#endif
-
-
-// =====================================================================================================================
-// MacOS
-
-#if __APPLE__
-
-const char *os_type = "macos";
-
-int getsysctl_by_name(const char *name, void *ptr, size_t len) {
- size_t nlen = len;
-
- if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) {
- netdata_log_error("MACOS: sysctl(%s...) failed: %s", name, strerror(errno));
- return 1;
- }
- if (unlikely(nlen != len)) {
- netdata_log_error("MACOS: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen);
- return 1;
- }
- return 0;
-}
-
-#endif
-
-// =====================================================================================================================
-// Linux
-
-#if __linux__
-
-const char *os_type = "linux";
-
-#endif
diff --git a/libnetdata/os.h b/libnetdata/os.h
deleted file mode 100644
index 197548b0d..000000000
--- a/libnetdata/os.h
+++ /dev/null
@@ -1,70 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_OS_H
-#define NETDATA_OS_H
-
-#include "libnetdata.h"
-
-// =====================================================================================================================
-// FreeBSD
-
-#if __FreeBSD__
-
-#include <sys/sysctl.h>
-
-#define GETSYSCTL_BY_NAME(name, var) getsysctl_by_name(name, &(var), sizeof(var))
-int getsysctl_by_name(const char *name, void *ptr, size_t len);
-
-#define GETSYSCTL_MIB(name, mib) getsysctl_mib(name, mib, sizeof(mib)/sizeof(int))
-
-int getsysctl_mib(const char *name, int *mib, size_t len);
-
-#define GETSYSCTL_SIMPLE(name, mib, var) getsysctl_simple(name, mib, sizeof(mib)/sizeof(int), &(var), sizeof(var))
-#define GETSYSCTL_WSIZE(name, mib, var, size) getsysctl_simple(name, mib, sizeof(mib)/sizeof(int), var, size)
-
-int getsysctl_simple(const char *name, int *mib, size_t miblen, void *ptr, size_t len);
-
-#define GETSYSCTL_SIZE(name, mib, size) getsysctl(name, mib, sizeof(mib)/sizeof(int), NULL, &(size))
-#define GETSYSCTL(name, mib, var, size) getsysctl(name, mib, sizeof(mib)/sizeof(int), &(var), &(size))
-
-int getsysctl(const char *name, int *mib, size_t miblen, void *ptr, size_t *len);
-
-#endif
-
-// =====================================================================================================================
-// MacOS
-
-#if __APPLE__
-
-#include <sys/sysctl.h>
-#include "endian.h"
-
-#define GETSYSCTL_BY_NAME(name, var) getsysctl_by_name(name, &(var), sizeof(var))
-int getsysctl_by_name(const char *name, void *ptr, size_t len);
-
-#endif
-
-// =====================================================================================================================
-// common defs for Apple/FreeBSD/Linux
-
-extern const char *os_type;
-
-#define get_system_cpus() get_system_cpus_with_cache(true, false)
-#define get_system_cpus_uncached() get_system_cpus_with_cache(false, false)
-long get_system_cpus_with_cache(bool cache, bool for_netdata);
-unsigned long read_cpuset_cpus(const char *filename, long system_cpus);
-
-extern pid_t pid_max;
-pid_t get_system_pid_max(void);
-
-extern unsigned int system_hz;
-void get_system_HZ(void);
-
-#include <sys/timex.h>
-#if defined(__FreeBSD__) || defined(__APPLE__)
-#define ADJUST_TIMEX(x) ntp_adjtime(x)
-#else
-#define ADJUST_TIMEX(x) adjtimex(x)
-#endif
-
-#endif //NETDATA_OS_H
diff --git a/libnetdata/popen/Makefile.am b/libnetdata/popen/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/popen/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/popen/README.md b/libnetdata/popen/README.md
deleted file mode 100644
index 7bd2bd3dd..000000000
--- a/libnetdata/popen/README.md
+++ /dev/null
@@ -1,15 +0,0 @@
-<!--
-title: "popen"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/popen/README.md
-sidebar_label: "popen"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# popen
-
-Process management library
-
-
-
diff --git a/libnetdata/procfile/Makefile.am b/libnetdata/procfile/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/procfile/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/procfile/README.md b/libnetdata/procfile/README.md
deleted file mode 100644
index 37138bd19..000000000
--- a/libnetdata/procfile/README.md
+++ /dev/null
@@ -1,71 +0,0 @@
-<!--
-title: "PROCFILE"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/procfile/README.md
-sidebar_label: "Procfile"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# PROCFILE
-
-procfile is a library for reading text data files (i.e `/proc` files) in the fastest possible way.
-
-## How it works
-
-The library automatically adapts (through the iterations) its memory so that each file
-is read with single `read()` call.
-
-Then the library splits the file into words, using the supplied separators.
-The library also supported quoted words (i.e. strings within of which the separators are ignored).
-
-### Initialization
-
-Initially the caller:
-
-- calls `procfile_open()` to open the file and allocate the structures needed.
-
-### Iterations
-
-For each iteration, the caller:
-
-- calls `procfile_readall()` to read updated contents.
- This call also rewinds (`lseek()` to 0) before reading it.
-
- For every file, a [BUFFER](https://github.com/netdata/netdata/blob/master/libnetdata/buffer/README.md) is used that is automatically adjusted to fit the entire
- file contents of the file. So the file is read with a single `read()` call (providing atomicity / consistency when
- the data are read from the kernel).
-
- Once the data are read, 2 arrays of pointers are updated:
-
- - a `words` array, pointing to each word in the data read
- - a `lines` array, pointing to the first word for each line
-
- This is highly optimized. Both arrays are automatically adjusted to
- fit all contents and are updated in a single pass on the data.
-
- The library provides a number of macros:
-
- - `procfile_lines()` returns the # of lines read
- - `procfile_linewords()` returns the # of words in the given line
- - `procfile_word()` returns a pointer the given word #
- - `procfile_line()` returns a pointer to the first word of the given line #
- - `procfile_lineword()` returns a pointer to the given word # of the given line #
-
-### Cleanup
-
-When the caller exits:
-
-- calls `procfile_free()` to close the file and free all memory used.
-
-### Performance
-
-- a **raspberry Pi 1** (the oldest single core one) can process 5.000+ `/proc` files per second.
-- a **J1900 Celeron** processor can process 23.000+ `/proc` files per second per core.
-
-To achieve this kind of performance, the library tries to work in batches so that the code
-and the data are inside the processor's caches.
-
-This library is extensively used in Netdata and its plugins.
-
-
diff --git a/libnetdata/required_dummies.h b/libnetdata/required_dummies.h
deleted file mode 100644
index 1ffe1e9e5..000000000
--- a/libnetdata/required_dummies.h
+++ /dev/null
@@ -1,45 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_LIB_DUMMIES_H
-#define NETDATA_LIB_DUMMIES_H 1
-
-// callback required by fatal()
-void netdata_cleanup_and_exit(int ret)
-{
- exit(ret);
-}
-
-void send_statistics(const char *action, const char *action_result, const char *action_data)
-{
- (void)action;
- (void)action_result;
- (void)action_data;
- return;
-}
-
-// callbacks required by popen()
-void signals_block(void){};
-void signals_unblock(void){};
-void signals_reset(void){};
-
-#ifndef UNIT_TESTING
-// callback required by eval()
-int health_variable_lookup(STRING *variable, struct rrdcalc *rc, NETDATA_DOUBLE *result)
-{
- (void)variable;
- (void)rc;
- (void)result;
- return 0;
-};
-#endif
-
-void rrdset_thread_rda_free(void){};
-void sender_thread_buffer_free(void){};
-void query_target_free(void){};
-void service_exits(void){};
-void rrd_collector_finished(void){};
-
-// required by get_system_cpus()
-char *netdata_configured_host_prefix = "";
-
-#endif // NETDATA_LIB_DUMMIES_H
diff --git a/libnetdata/simple_hashtable.h b/libnetdata/simple_hashtable.h
deleted file mode 100644
index f6b6db906..000000000
--- a/libnetdata/simple_hashtable.h
+++ /dev/null
@@ -1,396 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_SIMPLE_HASHTABLE_H
-#define NETDATA_SIMPLE_HASHTABLE_H
-
-#ifndef XXH_INLINE_ALL
-#define XXH_INLINE_ALL
-#endif
-#include "xxhash.h"
-
-typedef uint64_t SIMPLE_HASHTABLE_HASH;
-#define SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS 32
-
-#ifndef SIMPLE_HASHTABLE_NAME
-#define SIMPLE_HASHTABLE_NAME
-#endif
-
-#ifndef SIMPLE_HASHTABLE_VALUE_TYPE
-#define SIMPLE_HASHTABLE_VALUE_TYPE void
-#endif
-
-// First layer of macro for token concatenation
-#define CONCAT_INTERNAL(a, b) a ## b
-// Second layer of macro, which ensures proper expansion
-#define CONCAT(a, b) CONCAT_INTERNAL(a, b)
-
-// define names for all structures and structures
-#define simple_hashtable_init_named CONCAT(simple_hashtable_init, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_destroy_named CONCAT(simple_hashtable_destroy, SIMPLE_HASHTABLE_NAME)
-
-#define simple_hashtable_slot_named CONCAT(simple_hashtable_slot, SIMPLE_HASHTABLE_NAME)
-#define SIMPLE_HASHTABLE_SLOT_NAMED CONCAT(SIMPLE_HASHTABLE_SLOT, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_named CONCAT(simple_hashtable, SIMPLE_HASHTABLE_NAME)
-#define SIMPLE_HASHTABLE_NAMED CONCAT(SIMPLE_HASHTABLE, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_resize_named CONCAT(simple_hashtable_resize, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_get_slot_named CONCAT(simple_hashtable_get_slot, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_del_slot_named CONCAT(simple_hashtable_del_slot, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_set_slot_named CONCAT(simple_hashtable_set_slot, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_first_read_only_named CONCAT(simple_hashtable_first_read_only, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_next_read_only_named CONCAT(simple_hashtable_next_read_only, SIMPLE_HASHTABLE_NAME)
-
-#define simple_hashtable_sorted_binary_search_named CONCAT(simple_hashtable_sorted_binary_search, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_add_value_sorted_named CONCAT(simple_hashtable_add_value_sorted, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_del_value_sorted_named CONCAT(simple_hashtable_del_value_sorted, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_replace_value_sorted_named CONCAT(simple_hashtable_replace_value_sorted, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_sorted_array_first_read_only_named CONCAT(simple_hashtable_sorted_array_first_read_only, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_sorted_array_next_read_only_named CONCAT(simple_hashtable_sorted_array_next_read_only, SIMPLE_HASHTABLE_NAME)
-
-typedef struct simple_hashtable_slot_named {
- SIMPLE_HASHTABLE_HASH hash;
- SIMPLE_HASHTABLE_VALUE_TYPE *data;
-} SIMPLE_HASHTABLE_SLOT_NAMED;
-
-typedef struct simple_hashtable_named {
- size_t resizes;
- size_t searches;
- size_t collisions;
- size_t deletions;
- size_t deleted;
- size_t used;
- size_t size;
- SIMPLE_HASHTABLE_SLOT_NAMED *hashtable;
-
-#ifdef SIMPLE_HASHTABLE_SORT_FUNCTION
- struct {
- size_t used;
- size_t size;
- SIMPLE_HASHTABLE_VALUE_TYPE **array;
- } sorted;
-#endif
-} SIMPLE_HASHTABLE_NAMED;
-
-#ifdef SIMPLE_HASHTABLE_SORT_FUNCTION
-static inline size_t simple_hashtable_sorted_binary_search_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_VALUE_TYPE *value) {
- size_t left = 0, right = ht->sorted.used;
-
- while (left < right) {
- size_t mid = left + (right - left) / 2;
- if (SIMPLE_HASHTABLE_SORT_FUNCTION(ht->sorted.array[mid], value) < 0)
- left = mid + 1;
- else
- right = mid;
- }
-
- return left;
-}
-
-static inline void simple_hashtable_add_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_VALUE_TYPE *value) {
- size_t index = simple_hashtable_sorted_binary_search_named(ht, value);
-
- // Ensure there's enough space in the sorted array
- if (ht->sorted.used >= ht->sorted.size) {
- size_t size = ht->sorted.size ? ht->sorted.size * 2 : 64;
- SIMPLE_HASHTABLE_VALUE_TYPE **array = mallocz(size * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
- if(ht->sorted.array) {
- memcpy(array, ht->sorted.array, ht->sorted.size * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
- freez(ht->sorted.array);
- }
- ht->sorted.array = array;
- ht->sorted.size = size;
- }
-
- // Use memmove to shift elements and create space for the new element
- memmove(&ht->sorted.array[index + 1], &ht->sorted.array[index], (ht->sorted.used - index) * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
-
- ht->sorted.array[index] = value;
- ht->sorted.used++;
-}
-
-static inline void simple_hashtable_del_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_VALUE_TYPE *value) {
- size_t index = simple_hashtable_sorted_binary_search_named(ht, value);
-
- // Check if the value exists at the found index
- assert(index < ht->sorted.used && ht->sorted.array[index] == value);
-
- // Use memmove to shift elements and close the gap
- memmove(&ht->sorted.array[index], &ht->sorted.array[index + 1], (ht->sorted.used - index - 1) * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
- ht->sorted.used--;
-}
-
-static inline void simple_hashtable_replace_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_VALUE_TYPE *old_value, SIMPLE_HASHTABLE_VALUE_TYPE *new_value) {
- if(new_value == old_value)
- return;
-
- size_t old_value_index = simple_hashtable_sorted_binary_search_named(ht, old_value);
- assert(old_value_index < ht->sorted.used && ht->sorted.array[old_value_index] == old_value);
-
- int r = SIMPLE_HASHTABLE_SORT_FUNCTION(old_value, new_value);
- if(r == 0) {
- // Same value, so use the same index
- ht->sorted.array[old_value_index] = new_value;
- return;
- }
-
- size_t new_value_index = simple_hashtable_sorted_binary_search_named(ht, new_value);
- if(old_value_index == new_value_index) {
- // Not the same value, but still at the same index
- ht->sorted.array[old_value_index] = new_value;
- return;
- }
- else if (old_value_index < new_value_index) {
- // The old value is before the new value
- size_t shift_start = old_value_index + 1;
- size_t shift_end = new_value_index - 1;
- size_t shift_size = shift_end - old_value_index;
-
- memmove(&ht->sorted.array[old_value_index], &ht->sorted.array[shift_start], shift_size * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
- ht->sorted.array[shift_end] = new_value;
- }
- else {
- // The old value is after the new value
- size_t shift_start = new_value_index;
- size_t shift_end = old_value_index;
- size_t shift_size = shift_end - new_value_index;
-
- memmove(&ht->sorted.array[new_value_index + 1], &ht->sorted.array[shift_start], shift_size * sizeof(SIMPLE_HASHTABLE_VALUE_TYPE *));
- ht->sorted.array[new_value_index] = new_value;
- }
-}
-
-static inline SIMPLE_HASHTABLE_VALUE_TYPE **simple_hashtable_sorted_array_first_read_only_named(SIMPLE_HASHTABLE_NAMED *ht) {
- if (ht->sorted.used > 0) {
- return &ht->sorted.array[0];
- }
- return NULL;
-}
-
-static inline SIMPLE_HASHTABLE_VALUE_TYPE **simple_hashtable_sorted_array_next_read_only_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_VALUE_TYPE **last) {
- if (!last) return NULL;
-
- // Calculate the current position in the sorted array
- size_t currentIndex = last - ht->sorted.array;
-
- // Proceed to the next element if it exists
- if (currentIndex + 1 < ht->sorted.used) {
- return &ht->sorted.array[currentIndex + 1];
- }
-
- // If no more elements, return NULL
- return NULL;
-}
-
-#define SIMPLE_HASHTABLE_SORTED_FOREACH_READ_ONLY(ht, var, type, name) \
- for (type **(var) = simple_hashtable_sorted_array_first_read_only ## name(ht); \
- var; \
- (var) = simple_hashtable_sorted_array_next_read_only ## name(ht, var))
-
-#define SIMPLE_HASHTABLE_SORTED_FOREACH_READ_ONLY_VALUE(var) (*(var))
-
-#else
-static inline void simple_hashtable_add_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht __maybe_unused, SIMPLE_HASHTABLE_VALUE_TYPE *value __maybe_unused) { ; }
-static inline void simple_hashtable_del_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht __maybe_unused, SIMPLE_HASHTABLE_VALUE_TYPE *value __maybe_unused) { ; }
-static inline void simple_hashtable_replace_value_sorted_named(SIMPLE_HASHTABLE_NAMED *ht __maybe_unused, SIMPLE_HASHTABLE_VALUE_TYPE *old_value __maybe_unused, SIMPLE_HASHTABLE_VALUE_TYPE *new_value __maybe_unused) { ; }
-#endif
-
-static void simple_hashtable_init_named(SIMPLE_HASHTABLE_NAMED *ht, size_t size) {
- memset(ht, 0, sizeof(*ht));
- ht->size = size;
- ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable));
-}
-
-static void simple_hashtable_destroy_named(SIMPLE_HASHTABLE_NAMED *ht) {
-#ifdef SIMPLE_HASHTABLE_SORT_FUNCTION
- freez(ht->sorted.array);
-#endif
-
- freez(ht->hashtable);
- memset(ht, 0, sizeof(*ht));
-}
-
-static inline void simple_hashtable_resize_named(SIMPLE_HASHTABLE_NAMED *ht);
-
-#define SHTS_DATA_UNSET ((void *)NULL)
-#define SHTS_DATA_DELETED ((void *)0x01)
-#define SHTS_DATA_USERNULL ((void *)0x02)
-#define SHTS_IS_UNSET(sl) ((sl)->data == SHTS_DATA_UNSET)
-#define SHTS_IS_DELETED(sl) ((sl)->data == SHTS_DATA_DELETED)
-#define SHTS_IS_USERNULL(sl) ((sl)->data == SHTS_DATA_USERNULL)
-#define SIMPLE_HASHTABLE_SLOT_DATA(sl) ((SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl) || SHTS_IS_USERNULL(sl)) ? NULL : (sl)->data)
-#define SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) ((SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl)) ? NULL : (sl)->data)
-
-// IMPORTANT
-// The pointer returned by this call is valid up to the next call of this function (or the resize one)
-// If you need to cache something, cache the hash, not the slot pointer.
-static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_get_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_HASH hash, bool resize) {
- ht->searches++;
-
- size_t slot;
- SIMPLE_HASHTABLE_SLOT_NAMED *sl;
- SIMPLE_HASHTABLE_SLOT_NAMED *deleted;
-
- slot = hash % ht->size;
- sl = &ht->hashtable[slot];
- deleted = SHTS_IS_DELETED(sl) ? sl : NULL;
- if(likely(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) || sl->hash == hash))
- return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl;
-
- ht->collisions++;
-
- if(unlikely(resize && (ht->size <= (ht->used << 1) || ht->used >= ht->size))) {
- simple_hashtable_resize_named(ht);
-
- slot = hash % ht->size;
- sl = &ht->hashtable[slot];
- deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted;
- if(likely(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) || sl->hash == hash))
- return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl;
-
- ht->collisions++;
- }
-
- slot = ((hash >> SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS) + 1) % ht->size;
- sl = &ht->hashtable[slot];
- deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted;
-
- // Linear probing until we find it
- while (SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl) && sl->hash != hash) {
- slot = (slot + 1) % ht->size; // Wrap around if necessary
- sl = &ht->hashtable[slot];
- deleted = (!deleted && SHTS_IS_DELETED(sl)) ? sl : deleted;
- ht->collisions++;
- }
-
- return (SHTS_IS_UNSET(sl) && deleted) ? deleted : sl;
-}
-
-static inline bool simple_hashtable_del_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *sl) {
- if(SHTS_IS_UNSET(sl) || SHTS_IS_DELETED(sl))
- return false;
-
- ht->deletions++;
- ht->deleted++;
-
- simple_hashtable_del_value_sorted_named(ht, SIMPLE_HASHTABLE_SLOT_DATA(sl));
-
- sl->data = SHTS_DATA_DELETED;
- return true;
-}
-
-static inline void simple_hashtable_set_slot_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *sl, SIMPLE_HASHTABLE_HASH hash, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
- if(data == NULL)
- data = SHTS_DATA_USERNULL;
-
- if(unlikely(data == SHTS_DATA_UNSET || data == SHTS_DATA_DELETED)) {
- simple_hashtable_del_slot_named(ht, sl);
- return;
- }
-
- if(likely(SHTS_IS_UNSET(sl))) {
- simple_hashtable_add_value_sorted_named(ht, data);
- ht->used++;
- }
-
- else if(unlikely(SHTS_IS_DELETED(sl))) {
- ht->deleted--;
- }
-
- else
- simple_hashtable_replace_value_sorted_named(ht, SIMPLE_HASHTABLE_SLOT_DATA(sl), data);
-
- sl->hash = hash;
- sl->data = data;
-}
-
-// IMPORTANT
-// this call invalidates all SIMPLE_HASHTABLE_SLOT_NAMED pointers
-static inline void simple_hashtable_resize_named(SIMPLE_HASHTABLE_NAMED *ht) {
- SIMPLE_HASHTABLE_SLOT_NAMED *old = ht->hashtable;
- size_t old_size = ht->size;
-
- ht->resizes++;
- ht->size = (ht->size << 1) - ((ht->size > 16) ? 1 : 0);
- ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable));
- ht->used = ht->deleted = 0;
- for(size_t i = 0 ; i < old_size ; i++) {
- if(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(&old[i]))
- continue;
-
- SIMPLE_HASHTABLE_SLOT_NAMED *slot = simple_hashtable_get_slot_named(ht, old[i].hash, false);
- *slot = old[i];
- ht->used++;
- }
-
- freez(old);
-}
-
-// ----------------------------------------------------------------------------
-// hashtable traversal, in read-only mode
-// the hashtable should not be modified while the traversal is taking place
-
-static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_first_read_only_named(SIMPLE_HASHTABLE_NAMED *ht) {
- for(size_t i = 0; i < ht->used ;i++) {
- SIMPLE_HASHTABLE_SLOT_NAMED *sl = &ht->hashtable[i];
- if(!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl))
- return sl;
- }
-
- return NULL;
-}
-
-static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_next_read_only_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_SLOT_NAMED *last) {
- if (!last) return NULL;
-
- // Calculate the current position in the array
- size_t currentIndex = last - ht->hashtable;
-
- // Iterate over the hashtable starting from the next element
- for (size_t i = currentIndex + 1; i < ht->size; i++) {
- SIMPLE_HASHTABLE_SLOT_NAMED *sl = &ht->hashtable[i];
- if (!SIMPLE_HASHTABLE_SLOT_UNSET_OR_DELETED(sl)) {
- return sl;
- }
- }
-
- // If no more data slots are found, return NULL
- return NULL;
-}
-
-#define SIMPLE_HASHTABLE_FOREACH_READ_ONLY(ht, var, name) \
- for(struct simple_hashtable_slot ## name *(var) = simple_hashtable_first_read_only ## name(ht); \
- var; \
- (var) = simple_hashtable_next_read_only ## name(ht, var))
-
-#define SIMPLE_HASHTABLE_FOREACH_READ_ONLY_VALUE(var) SIMPLE_HASHTABLE_SLOT_DATA(var)
-
-// ----------------------------------------------------------------------------
-// high level implementation
-
-#ifdef SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION
-
-#define simple_hashtable_set_named CONCAT(simple_hashtable_set, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_get_named CONCAT(simple_hashtable_get, SIMPLE_HASHTABLE_NAME)
-#define simple_hashtable_del_named CONCAT(simple_hashtable_del, SIMPLE_HASHTABLE_NAME)
-
-static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_set_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
- XXH64_hash_t hash = XXH3_64bits(key, key_len);
- SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true);
- simple_hashtable_set_slot_named(ht, sl, hash, data);
- return SIMPLE_HASHTABLE_SLOT_DATA(sl);
-}
-
-static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_get_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
- XXH64_hash_t hash = XXH3_64bits(key, key_len);
- SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true);
- return SIMPLE_HASHTABLE_SLOT_DATA(sl);
-}
-
-static inline bool simple_hashtable_del_named(SIMPLE_HASHTABLE_NAMED *ht, void *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) {
- XXH64_hash_t hash = XXH3_64bits(key, key_len);
- SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, true);
- return simple_hashtable_del_slot_named(ht, sl);
-}
-
-#endif // SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION
-
-#endif //NETDATA_SIMPLE_HASHTABLE_H
diff --git a/libnetdata/simple_pattern/Makefile.am b/libnetdata/simple_pattern/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/simple_pattern/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/simple_pattern/README.md b/libnetdata/simple_pattern/README.md
deleted file mode 100644
index 5f56a3af7..000000000
--- a/libnetdata/simple_pattern/README.md
+++ /dev/null
@@ -1,47 +0,0 @@
-<!--
-title: "Simple patterns"
-description: "Netdata supports simple patterns, which are less cryptic versions of regular expressions. Use familiar notation for powerful results."
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/simple_pattern/README.md
-sidebar_label: "Simple patterns"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Simple patterns
-
-Unix prefers regular expressions. But they are just too hard, too cryptic
-to use, write and understand.
-
-So, Netdata supports **simple patterns**.
-
-Simple patterns are a space separated list of words, that can have `*`
-as a wildcard. Each word may use any number of `*`. Simple patterns
-allow **negative** matches by prefixing a word with `!`.
-
-So, `pattern = !*bad* *` will match anything, except all those that
-contain the word `bad`.
-
-Simple patterns are quite powerful: `pattern = *foobar* !foo* !*bar *`
-matches everything containing `foobar`, except strings that start
-with `foo` or end with `bar`.
-
-You can use the Netdata command line to check simple patterns,
-like this:
-
-```sh
-# netdata -W simple-pattern '*foobar* !foo* !*bar *' 'hello world'
-RESULT: MATCHED - pattern '*foobar* !foo* !*bar *' matches 'hello world'
-
-# netdata -W simple-pattern '*foobar* !foo* !*bar *' 'hello world bar'
-RESULT: NOT MATCHED - pattern '*foobar* !foo* !*bar *' does not match 'hello world bar'
-
-# netdata -W simple-pattern '*foobar* !foo* !*bar *' 'hello world foobar'
-RESULT: MATCHED - pattern '*foobar* !foo* !*bar *' matches 'hello world foobar'
-```
-
-Netdata stops processing to the first positive or negative match
-(left to right). If it is not matched by either positive or negative
-patterns, it is denied at the end.
-
-
diff --git a/libnetdata/socket/Makefile.am b/libnetdata/socket/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/socket/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/socket/README.md b/libnetdata/socket/README.md
deleted file mode 100644
index e339a071c..000000000
--- a/libnetdata/socket/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-<!--
-Title: "Socket"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/socket/README.md
-sidebar_label: "Socket"
-learn_status: "Published"
-learn_topic_type: "References"
-learn_rel_path: "Developers/libnetdata"
--->
diff --git a/libnetdata/socket/socket.c b/libnetdata/socket/socket.c
deleted file mode 100644
index 605e85635..000000000
--- a/libnetdata/socket/socket.c
+++ /dev/null
@@ -1,2130 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE // for POLLRDHUP
-#endif
-
-#ifndef __BSD_VISIBLE
-#define __BSD_VISIBLE // for POLLRDHUP
-#endif
-
-#include "../libnetdata.h"
-
-bool ip_to_hostname(const char *ip, char *dst, size_t dst_len) {
- if(!dst || !dst_len)
- return false;
-
- struct sockaddr_in sa;
- struct sockaddr_in6 sa6;
- struct sockaddr *sa_ptr;
- int sa_len;
-
- // Try to convert the IP address to sockaddr_in (IPv4)
- if (inet_pton(AF_INET, ip, &(sa.sin_addr)) == 1) {
- sa.sin_family = AF_INET;
- sa_ptr = (struct sockaddr *)&sa;
- sa_len = sizeof(sa);
- }
- // Try to convert the IP address to sockaddr_in6 (IPv6)
- else if (inet_pton(AF_INET6, ip, &(sa6.sin6_addr)) == 1) {
- sa6.sin6_family = AF_INET6;
- sa_ptr = (struct sockaddr *)&sa6;
- sa_len = sizeof(sa6);
- }
-
- else {
- dst[0] = '\0';
- return false;
- }
-
- // Perform the reverse lookup
- int res = getnameinfo(sa_ptr, sa_len, dst, dst_len, NULL, 0, NI_NAMEREQD);
- if(res != 0)
- return false;
-
- return true;
-}
-
-SOCKET_PEERS socket_peers(int sock_fd) {
- SOCKET_PEERS peers;
-
- if(sock_fd < 0) {
- strncpyz(peers.peer.ip, "not connected", sizeof(peers.peer.ip) - 1);
- peers.peer.port = 0;
-
- strncpyz(peers.local.ip, "not connected", sizeof(peers.local.ip) - 1);
- peers.local.port = 0;
-
- return peers;
- }
-
- struct sockaddr_storage addr;
- socklen_t addr_len = sizeof(addr);
-
- // Get peer info
- if (getpeername(sock_fd, (struct sockaddr *)&addr, &addr_len) == 0) {
- if (addr.ss_family == AF_INET) { // IPv4
- struct sockaddr_in *s = (struct sockaddr_in *)&addr;
- inet_ntop(AF_INET, &s->sin_addr, peers.peer.ip, sizeof(peers.peer.ip));
- peers.peer.port = ntohs(s->sin_port);
- }
- else { // IPv6
- struct sockaddr_in6 *s = (struct sockaddr_in6 *)&addr;
- inet_ntop(AF_INET6, &s->sin6_addr, peers.peer.ip, sizeof(peers.peer.ip));
- peers.peer.port = ntohs(s->sin6_port);
- }
- }
- else {
- strncpyz(peers.peer.ip, "unknown", sizeof(peers.peer.ip) - 1);
- peers.peer.port = 0;
- }
-
- // Get local info
- addr_len = sizeof(addr);
- if (getsockname(sock_fd, (struct sockaddr *)&addr, &addr_len) == 0) {
- if (addr.ss_family == AF_INET) { // IPv4
- struct sockaddr_in *s = (struct sockaddr_in *) &addr;
- inet_ntop(AF_INET, &s->sin_addr, peers.local.ip, sizeof(peers.local.ip));
- peers.local.port = ntohs(s->sin_port);
- } else { // IPv6
- struct sockaddr_in6 *s = (struct sockaddr_in6 *) &addr;
- inet_ntop(AF_INET6, &s->sin6_addr, peers.local.ip, sizeof(peers.local.ip));
- peers.local.port = ntohs(s->sin6_port);
- }
- }
- else {
- strncpyz(peers.local.ip, "unknown", sizeof(peers.local.ip) - 1);
- peers.local.port = 0;
- }
-
- return peers;
-}
-
-
-// --------------------------------------------------------------------------------------------------------------------
-// various library calls
-
-#ifdef __gnu_linux__
-#define LARGE_SOCK_SIZE 33554431 // don't ask why - I found it at brubeck source - I guess it is just a large number
-#else
-#define LARGE_SOCK_SIZE 4096
-#endif
-
-bool fd_is_socket(int fd) {
- int type;
- socklen_t len = sizeof(type);
- if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &type, &len) == -1)
- return false;
-
- return true;
-}
-
-bool sock_has_output_error(int fd) {
- if(fd < 0) {
- //internal_error(true, "invalid socket %d", fd);
- return false;
- }
-
-// if(!fd_is_socket(fd)) {
-// //internal_error(true, "fd %d is not a socket", fd);
-// return false;
-// }
-
- short int errors = POLLERR | POLLHUP | POLLNVAL;
-
-#ifdef POLLRDHUP
- errors |= POLLRDHUP;
-#endif
-
- struct pollfd pfd = {
- .fd = fd,
- .events = POLLOUT | errors,
- .revents = 0,
- };
-
- if(poll(&pfd, 1, 0) == -1) {
- //internal_error(true, "poll() failed");
- return false;
- }
-
- return ((pfd.revents & errors) || !(pfd.revents & POLLOUT));
-}
-
-int sock_setnonblock(int fd) {
- int flags;
-
- flags = fcntl(fd, F_GETFL);
- flags |= O_NONBLOCK;
-
- int ret = fcntl(fd, F_SETFL, flags);
- if(ret < 0)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to set O_NONBLOCK on socket %d",
- fd);
-
- return ret;
-}
-
-int sock_delnonblock(int fd) {
- int flags;
-
- flags = fcntl(fd, F_GETFL);
- flags &= ~O_NONBLOCK;
-
- int ret = fcntl(fd, F_SETFL, flags);
- if(ret < 0)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to remove O_NONBLOCK on socket %d",
- fd);
-
- return ret;
-}
-
-int sock_setreuse(int fd, int reuse) {
- int ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));
-
- if(ret == -1)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to set SO_REUSEADDR on socket %d",
- fd);
-
- return ret;
-}
-
-int sock_setreuse_port(int fd, int reuse) {
- int ret;
-
-#ifdef SO_REUSEPORT
- ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &reuse, sizeof(reuse));
- if(ret == -1 && errno != ENOPROTOOPT)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "failed to set SO_REUSEPORT on socket %d",
- fd);
-#else
- ret = -1;
-#endif
-
- return ret;
-}
-
-int sock_enlarge_in(int fd) {
- int ret, bs = LARGE_SOCK_SIZE;
-
- ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &bs, sizeof(bs));
-
- if(ret == -1)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to set SO_RCVBUF on socket %d",
- fd);
-
- return ret;
-}
-
-int sock_enlarge_out(int fd) {
- int ret, bs = LARGE_SOCK_SIZE;
- ret = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &bs, sizeof(bs));
-
- if(ret == -1)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to set SO_SNDBUF on socket %d",
- fd);
-
- return ret;
-}
-
-
-// --------------------------------------------------------------------------------------------------------------------
-
-char *strdup_client_description(int family, const char *protocol, const char *ip, uint16_t port) {
- char buffer[100 + 1];
-
- switch(family) {
- case AF_INET:
- snprintfz(buffer, sizeof(buffer) - 1, "%s:%s:%d", protocol, ip, port);
- break;
-
- case AF_INET6:
- default:
- snprintfz(buffer, sizeof(buffer) - 1, "%s:[%s]:%d", protocol, ip, port);
- break;
-
- case AF_UNIX:
- snprintfz(buffer, sizeof(buffer) - 1, "%s:%s", protocol, ip);
- break;
- }
-
- return strdupz(buffer);
-}
-
-// --------------------------------------------------------------------------------------------------------------------
-// listening sockets
-
-int create_listen_socket_unix(const char *path, int listen_backlog) {
- int sock;
-
- sock = socket(AF_UNIX, SOCK_STREAM, 0);
- if(sock < 0) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: UNIX socket() on path '%s' failed.",
- path);
-
- return -1;
- }
-
- sock_setnonblock(sock);
- sock_enlarge_in(sock);
-
- struct sockaddr_un name;
- memset(&name, 0, sizeof(struct sockaddr_un));
- name.sun_family = AF_UNIX;
- strncpy(name.sun_path, path, sizeof(name.sun_path)-1);
-
- errno = 0;
- if (unlink(path) == -1 && errno != ENOENT)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: failed to remove existing (probably obsolete or left-over) file on UNIX socket path '%s'.",
- path);
-
- if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
- close(sock);
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: UNIX bind() on path '%s' failed.",
- path);
-
- return -1;
- }
-
- // we have to chmod this to 0777 so that the client will be able
- // to read from and write to this socket.
- if(chmod(path, 0777) == -1)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: failed to chmod() socket file '%s'.",
- path);
-
- if(listen(sock, listen_backlog) < 0) {
- close(sock);
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: UNIX listen() on path '%s' failed.",
- path);
-
- return -1;
- }
-
- return sock;
-}
-
-int create_listen_socket4(int socktype, const char *ip, uint16_t port, int listen_backlog) {
- int sock;
-
- sock = socket(AF_INET, socktype, 0);
- if(sock < 0) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: IPv4 socket() on ip '%s' port %d, socktype %d failed.",
- ip, port, socktype);
-
- return -1;
- }
-
- sock_setreuse(sock, 1);
- sock_setreuse_port(sock, 0);
- sock_setnonblock(sock);
- sock_enlarge_in(sock);
-
- struct sockaddr_in name;
- memset(&name, 0, sizeof(struct sockaddr_in));
- name.sin_family = AF_INET;
- name.sin_port = htons (port);
-
- int ret = inet_pton(AF_INET, ip, (void *)&name.sin_addr.s_addr);
- if(ret != 1) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: Failed to convert IP '%s' to a valid IPv4 address.",
- ip);
-
- close(sock);
- return -1;
- }
-
- if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
- close(sock);
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: IPv4 bind() on ip '%s' port %d, socktype %d failed.",
- ip, port, socktype);
-
- return -1;
- }
-
- if(socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) {
- close(sock);
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: IPv4 listen() on ip '%s' port %d, socktype %d failed.",
- ip, port, socktype);
-
- return -1;
- }
-
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "LISTENER: Listening on IPv4 ip '%s' port %d, socktype %d",
- ip, port, socktype);
-
- return sock;
-}
-
-int create_listen_socket6(int socktype, uint32_t scope_id, const char *ip, int port, int listen_backlog) {
- int sock;
- int ipv6only = 1;
-
- sock = socket(AF_INET6, socktype, 0);
- if (sock < 0) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: IPv6 socket() on ip '%s' port %d, socktype %d, failed.",
- ip, port, socktype);
-
- return -1;
- }
-
- sock_setreuse(sock, 1);
- sock_setreuse_port(sock, 0);
- sock_setnonblock(sock);
- sock_enlarge_in(sock);
-
- /* IPv6 only */
- if(setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (void*)&ipv6only, sizeof(ipv6only)) != 0)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: Cannot set IPV6_V6ONLY on ip '%s' port %d, socktype %d.",
- ip, port, socktype);
-
- struct sockaddr_in6 name;
- memset(&name, 0, sizeof(struct sockaddr_in6));
- name.sin6_family = AF_INET6;
- name.sin6_port = htons ((uint16_t) port);
- name.sin6_scope_id = scope_id;
-
- int ret = inet_pton(AF_INET6, ip, (void *)&name.sin6_addr.s6_addr);
- if(ret != 1) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: Failed to convert IP '%s' to a valid IPv6 address.",
- ip);
-
- close(sock);
- return -1;
- }
-
- name.sin6_scope_id = scope_id;
-
- if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
- close(sock);
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: IPv6 bind() on ip '%s' port %d, socktype %d failed.",
- ip, port, socktype);
-
- return -1;
- }
-
- if (socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) {
- close(sock);
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: IPv6 listen() on ip '%s' port %d, socktype %d failed.",
- ip, port, socktype);
-
- return -1;
- }
-
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "LISTENER: Listening on IPv6 ip '%s' port %d, socktype %d",
- ip, port, socktype);
-
- return sock;
-}
-
-static inline int listen_sockets_add(LISTEN_SOCKETS *sockets, int fd, int family, int socktype, const char *protocol, const char *ip, uint16_t port, int acl_flags) {
- if(sockets->opened >= MAX_LISTEN_FDS) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: Too many listening sockets. Failed to add listening %s socket at ip '%s' port %d, protocol %s, socktype %d",
- protocol, ip, port, protocol, socktype);
-
- close(fd);
- return -1;
- }
-
- sockets->fds[sockets->opened] = fd;
- sockets->fds_types[sockets->opened] = socktype;
- sockets->fds_families[sockets->opened] = family;
- sockets->fds_names[sockets->opened] = strdup_client_description(family, protocol, ip, port);
- sockets->fds_acl_flags[sockets->opened] = acl_flags;
-
- sockets->opened++;
- return 0;
-}
-
-int listen_sockets_check_is_member(LISTEN_SOCKETS *sockets, int fd) {
- size_t i;
- for(i = 0; i < sockets->opened ;i++)
- if(sockets->fds[i] == fd) return 1;
-
- return 0;
-}
-
-static inline void listen_sockets_init(LISTEN_SOCKETS *sockets) {
- size_t i;
- for(i = 0; i < MAX_LISTEN_FDS ;i++) {
- sockets->fds[i] = -1;
- sockets->fds_names[i] = NULL;
- sockets->fds_types[i] = -1;
- }
-
- sockets->opened = 0;
- sockets->failed = 0;
-}
-
-void listen_sockets_close(LISTEN_SOCKETS *sockets) {
- size_t i;
- for(i = 0; i < sockets->opened ;i++) {
- close(sockets->fds[i]);
- sockets->fds[i] = -1;
-
- freez(sockets->fds_names[i]);
- sockets->fds_names[i] = NULL;
-
- sockets->fds_types[i] = -1;
- }
-
- sockets->opened = 0;
- sockets->failed = 0;
-}
-
-/*
- * SSL ACL
- *
- * Search the SSL acl and apply it case it is set.
- *
- * @param acl is the acl given by the user.
- */
-WEB_CLIENT_ACL socket_ssl_acl(char *acl) {
- char *ssl = strchr(acl,'^');
- if(ssl) {
- //Due the format of the SSL command it is always the last command,
- //we finish it here to avoid problems with the ACLs
- *ssl = '\0';
-#ifdef ENABLE_HTTPS
- ssl++;
- if (!strncmp("SSL=",ssl,4)) {
- ssl += 4;
- if (!strcmp(ssl,"optional")) {
- return WEB_CLIENT_ACL_SSL_OPTIONAL;
- }
- else if (!strcmp(ssl,"force")) {
- return WEB_CLIENT_ACL_SSL_FORCE;
- }
- }
-#endif
- }
-
- return WEB_CLIENT_ACL_NONE;
-}
-
-WEB_CLIENT_ACL read_acl(char *st) {
- WEB_CLIENT_ACL ret = socket_ssl_acl(st);
-
- if (!strcmp(st,"dashboard")) ret |= WEB_CLIENT_ACL_DASHBOARD;
- if (!strcmp(st,"registry")) ret |= WEB_CLIENT_ACL_REGISTRY;
- if (!strcmp(st,"badges")) ret |= WEB_CLIENT_ACL_BADGE;
- if (!strcmp(st,"management")) ret |= WEB_CLIENT_ACL_MGMT;
- if (!strcmp(st,"streaming")) ret |= WEB_CLIENT_ACL_STREAMING;
- if (!strcmp(st,"netdata.conf")) ret |= WEB_CLIENT_ACL_NETDATACONF;
-
- return ret;
-}
-
-static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, uint16_t default_port, int listen_backlog) {
- int added = 0;
- WEB_CLIENT_ACL acl_flags = WEB_CLIENT_ACL_NONE;
-
- struct addrinfo hints;
- struct addrinfo *result = NULL, *rp = NULL;
-
- char buffer[strlen(definition) + 1];
- strcpy(buffer, definition);
-
- char buffer2[10 + 1];
- snprintfz(buffer2, 10, "%d", default_port);
-
- char *ip = buffer, *port = buffer2, *interface = "", *portconfig;;
-
- int protocol = IPPROTO_TCP, socktype = SOCK_STREAM;
- const char *protocol_str = "tcp";
-
- if(strncmp(ip, "tcp:", 4) == 0) {
- ip += 4;
- protocol = IPPROTO_TCP;
- socktype = SOCK_STREAM;
- protocol_str = "tcp";
- }
- else if(strncmp(ip, "udp:", 4) == 0) {
- ip += 4;
- protocol = IPPROTO_UDP;
- socktype = SOCK_DGRAM;
- protocol_str = "udp";
- }
- else if(strncmp(ip, "unix:", 5) == 0) {
- char *path = ip + 5;
- socktype = SOCK_STREAM;
- protocol_str = "unix";
- int fd = create_listen_socket_unix(path, listen_backlog);
- if (fd == -1) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: Cannot create unix socket '%s'",
- path);
-
- sockets->failed++;
- } else {
- acl_flags = WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_REGISTRY | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_NETDATACONF | WEB_CLIENT_ACL_STREAMING | WEB_CLIENT_ACL_SSL_DEFAULT;
- listen_sockets_add(sockets, fd, AF_UNIX, socktype, protocol_str, path, 0, acl_flags);
- added++;
- }
- return added;
- }
-
- char *e = ip;
- if(*e == '[') {
- e = ++ip;
- while(*e && *e != ']') e++;
- if(*e == ']') {
- *e = '\0';
- e++;
- }
- }
- else {
- while(*e && *e != ':' && *e != '%' && *e != '=') e++;
- }
-
- if(*e == '%') {
- *e = '\0';
- e++;
- interface = e;
- while(*e && *e != ':' && *e != '=') e++;
- }
-
- if(*e == ':') {
- port = e + 1;
- *e = '\0';
- e++;
- while(*e && *e != '=') e++;
- }
-
- if(*e == '=') {
- *e='\0';
- e++;
- portconfig = e;
- while (*e != '\0') {
- if (*e == '|') {
- *e = '\0';
- acl_flags |= read_acl(portconfig);
- e++;
- portconfig = e;
- continue;
- }
- e++;
- }
- acl_flags |= read_acl(portconfig);
- } else {
- acl_flags = WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_REGISTRY | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_NETDATACONF | WEB_CLIENT_ACL_STREAMING | WEB_CLIENT_ACL_SSL_DEFAULT;
- }
-
- //Case the user does not set the option SSL in the "bind to", but he has
- //the certificates, I must redirect, so I am assuming here the default option
- if(!(acl_flags & WEB_CLIENT_ACL_SSL_OPTIONAL) && !(acl_flags & WEB_CLIENT_ACL_SSL_FORCE)) {
- acl_flags |= WEB_CLIENT_ACL_SSL_DEFAULT;
- }
-
- uint32_t scope_id = 0;
- if(*interface) {
- scope_id = if_nametoindex(interface);
- if(!scope_id)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: Cannot find a network interface named '%s'. "
- "Continuing with limiting the network interface",
- interface);
- }
-
- if(!*ip || *ip == '*' || !strcmp(ip, "any") || !strcmp(ip, "all"))
- ip = NULL;
-
- if(!*port)
- port = buffer2;
-
- memset(&hints, 0, sizeof(hints));
- hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
- hints.ai_socktype = socktype;
- hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */
- hints.ai_protocol = protocol;
- hints.ai_canonname = NULL;
- hints.ai_addr = NULL;
- hints.ai_next = NULL;
-
- int r = getaddrinfo(ip, port, &hints, &result);
- if (r != 0) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: getaddrinfo('%s', '%s'): %s\n",
- ip, port, gai_strerror(r));
-
- return -1;
- }
-
- for (rp = result; rp != NULL; rp = rp->ai_next) {
- int fd = -1;
- int family;
-
- char rip[INET_ADDRSTRLEN + INET6_ADDRSTRLEN] = "INVALID";
- uint16_t rport = default_port;
-
- family = rp->ai_addr->sa_family;
- switch (family) {
- case AF_INET: {
- struct sockaddr_in *sin = (struct sockaddr_in *) rp->ai_addr;
- inet_ntop(AF_INET, &sin->sin_addr, rip, INET_ADDRSTRLEN);
- rport = ntohs(sin->sin_port);
- fd = create_listen_socket4(socktype, rip, rport, listen_backlog);
- break;
- }
-
- case AF_INET6: {
- struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) rp->ai_addr;
- inet_ntop(AF_INET6, &sin6->sin6_addr, rip, INET6_ADDRSTRLEN);
- rport = ntohs(sin6->sin6_port);
- fd = create_listen_socket6(socktype, scope_id, rip, rport, listen_backlog);
- break;
- }
-
- default:
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "LISTENER: Unknown socket family %d",
- family);
-
- break;
- }
-
- if (fd == -1) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: Cannot bind to ip '%s', port %d",
- rip, rport);
-
- sockets->failed++;
- }
- else {
- listen_sockets_add(sockets, fd, family, socktype, protocol_str, rip, rport, acl_flags);
- added++;
- }
- }
-
- freeaddrinfo(result);
-
- return added;
-}
-
-int listen_sockets_setup(LISTEN_SOCKETS *sockets) {
- listen_sockets_init(sockets);
-
- sockets->backlog = (int) appconfig_get_number(sockets->config, sockets->config_section, "listen backlog", sockets->backlog);
-
- long long int old_port = sockets->default_port;
- long long int new_port = appconfig_get_number(sockets->config, sockets->config_section, "default port", sockets->default_port);
- if(new_port < 1 || new_port > 65535) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: Invalid listen port %lld given. Defaulting to %lld.",
- new_port, old_port);
-
- sockets->default_port = (uint16_t) appconfig_set_number(sockets->config, sockets->config_section, "default port", old_port);
- }
- else sockets->default_port = (uint16_t)new_port;
-
- char *s = appconfig_get(sockets->config, sockets->config_section, "bind to", sockets->default_bind_to);
- while(*s) {
- char *e = s;
-
- // skip separators, moving both s(tart) and e(nd)
- while(isspace(*e) || *e == ',') s = ++e;
-
- // move e(nd) to the first separator
- while(*e && !isspace(*e) && *e != ',') e++;
-
- // is there anything?
- if(!*s || s == e) break;
-
- char buf[e - s + 1];
- strncpyz(buf, s, e - s);
- bind_to_this(sockets, buf, sockets->default_port, sockets->backlog);
-
- s = e;
- }
-
- if(sockets->failed) {
- size_t i;
- for(i = 0; i < sockets->opened ;i++)
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "LISTENER: Listen socket %s opened successfully.",
- sockets->fds_names[i]);
- }
-
- return (int)sockets->opened;
-}
-
-
-// --------------------------------------------------------------------------------------------------------------------
-// connect to another host/port
-
-// connect_to_this_unix()
-// path the path of the unix socket
-// timeout the timeout for establishing a connection
-
-static inline int connect_to_unix(const char *path, struct timeval *timeout) {
- int fd = socket(AF_UNIX, SOCK_STREAM, 0);
- if(fd == -1) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to create UNIX socket() for '%s'",
- path);
-
- return -1;
- }
-
- if(timeout) {
- if(setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char *) timeout, sizeof(struct timeval)) < 0)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to set timeout on UNIX socket '%s'",
- path);
- }
-
- struct sockaddr_un addr;
- memset(&addr, 0, sizeof(addr));
- addr.sun_family = AF_UNIX;
- strncpy(addr.sun_path, path, sizeof(addr.sun_path)-1);
-
- if (connect(fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Cannot connect to UNIX socket on path '%s'.",
- path);
-
- close(fd);
- return -1;
- }
-
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "Connected to UNIX socket on path '%s'.",
- path);
-
- return fd;
-}
-
-// connect_to_this_ip46()
-// protocol IPPROTO_TCP, IPPROTO_UDP
-// socktype SOCK_STREAM, SOCK_DGRAM
-// host the destination hostname or IP address (IPv4 or IPv6) to connect to
-// if it resolves to many IPs, all are tried (IPv4 and IPv6)
-// scope_id the if_index id of the interface to use for connecting (0 = any)
-// (used only under IPv6)
-// service the service name or port to connect to
-// timeout the timeout for establishing a connection
-
-int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t scope_id, const char *service, struct timeval *timeout) {
- struct addrinfo hints;
- struct addrinfo *ai_head = NULL, *ai = NULL;
-
- memset(&hints, 0, sizeof(hints));
- hints.ai_family = PF_UNSPEC; /* Allow IPv4 or IPv6 */
- hints.ai_socktype = socktype;
- hints.ai_protocol = protocol;
-
- int ai_err = getaddrinfo(host, service, &hints, &ai_head);
- if (ai_err != 0) {
-
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Cannot resolve host '%s', port '%s': %s",
- host, service, gai_strerror(ai_err));
-
- return -1;
- }
-
- char hostBfr[NI_MAXHOST + 1];
- char servBfr[NI_MAXSERV + 1];
-
- ND_LOG_STACK lgs[] = {
- ND_LOG_FIELD_TXT(NDF_DST_IP, hostBfr),
- ND_LOG_FIELD_TXT(NDF_DST_PORT, servBfr),
- ND_LOG_FIELD_END(),
- };
- ND_LOG_STACK_PUSH(lgs);
-
- int fd = -1;
- for (ai = ai_head; ai != NULL && fd == -1; ai = ai->ai_next) {
-
- if (ai->ai_family == PF_INET6) {
- struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr;
- if(pSadrIn6->sin6_scope_id == 0) {
- pSadrIn6->sin6_scope_id = scope_id;
- }
- }
-
- getnameinfo(ai->ai_addr,
- ai->ai_addrlen,
- hostBfr,
- sizeof(hostBfr),
- servBfr,
- sizeof(servBfr),
- NI_NUMERICHOST | NI_NUMERICSERV);
-
- switch (ai->ai_addr->sa_family) {
- case PF_INET: {
- struct sockaddr_in *pSadrIn = (struct sockaddr_in *)ai->ai_addr;
- (void)pSadrIn;
- break;
- }
-
- case PF_INET6: {
- struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr;
- (void)pSadrIn6;
- break;
- }
-
- default: {
- // Unknown protocol family
- continue;
- }
- }
-
- fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
- if(fd != -1) {
- if(timeout) {
- if(setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char *) timeout, sizeof(struct timeval)) < 0)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to set timeout on the socket to ip '%s' port '%s'",
- hostBfr, servBfr);
- }
-
- errno = 0;
- if(connect(fd, ai->ai_addr, ai->ai_addrlen) < 0) {
- if(errno == EALREADY || errno == EINPROGRESS) {
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "Waiting for connection to ip %s port %s to be established",
- hostBfr, servBfr);
-
- // Convert 'struct timeval' to milliseconds for poll():
- int timeout_milliseconds = timeout->tv_sec * 1000 + timeout->tv_usec / 1000;
-
- struct pollfd fds[1];
- fds[0].fd = fd;
- fds[0].events = POLLOUT; // We are looking for the ability to write to the socket
-
- int ret = poll(fds, 1, timeout_milliseconds);
- if (ret > 0) {
- // poll() completed normally. We can check the revents to see what happened
- if (fds[0].revents & POLLOUT) {
- // connect() completed successfully, socket is writable.
-
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "connect() to ip %s port %s completed successfully",
- hostBfr, servBfr);
-
- }
- else {
- // This means that the socket is in error. We will close it and set fd to -1
-
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to connect to '%s', port '%s'.",
- hostBfr, servBfr);
-
- close(fd);
- fd = -1;
- }
- }
- else if (ret == 0) {
- // poll() timed out, the connection is not established within the specified timeout.
- errno = 0;
-
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Timed out while connecting to '%s', port '%s'.",
- hostBfr, servBfr);
-
- close(fd);
- fd = -1;
- }
- else { // ret < 0
- // poll() returned an error.
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to connect to '%s', port '%s'. poll() returned %d",
- hostBfr, servBfr, ret);
-
- close(fd);
- fd = -1;
- }
- }
- else {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to connect to '%s', port '%s'",
- hostBfr, servBfr);
-
- close(fd);
- fd = -1;
- }
- }
- }
- else
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to socket() to '%s', port '%s'",
- hostBfr, servBfr);
- }
-
- freeaddrinfo(ai_head);
-
- return fd;
-}
-
-// connect_to_this()
-//
-// definition format:
-//
-// [PROTOCOL:]IP[%INTERFACE][:PORT]
-//
-// PROTOCOL = tcp or udp
-// IP = IPv4 or IPv6 IP or hostname, optionally enclosed in [] (required for IPv6)
-// INTERFACE = for IPv6 only, the network interface to use
-// PORT = port number or service name
-
-int connect_to_this(const char *definition, int default_port, struct timeval *timeout) {
- char buffer[strlen(definition) + 1];
- strcpy(buffer, definition);
-
- char default_service[10 + 1];
- snprintfz(default_service, 10, "%d", default_port);
-
- char *host = buffer, *service = default_service, *interface = "";
- int protocol = IPPROTO_TCP, socktype = SOCK_STREAM;
- uint32_t scope_id = 0;
-
- if(strncmp(host, "tcp:", 4) == 0) {
- host += 4;
- protocol = IPPROTO_TCP;
- socktype = SOCK_STREAM;
- }
- else if(strncmp(host, "udp:", 4) == 0) {
- host += 4;
- protocol = IPPROTO_UDP;
- socktype = SOCK_DGRAM;
- }
- else if(strncmp(host, "unix:", 5) == 0) {
- char *path = host + 5;
- return connect_to_unix(path, timeout);
- }
- else if(*host == '/') {
- char *path = host;
- return connect_to_unix(path, timeout);
- }
-
- char *e = host;
- if(*e == '[') {
- e = ++host;
- while(*e && *e != ']') e++;
- if(*e == ']') {
- *e = '\0';
- e++;
- }
- }
- else {
- while(*e && *e != ':' && *e != '%') e++;
- }
-
- if(*e == '%') {
- *e = '\0';
- e++;
- interface = e;
- while(*e && *e != ':') e++;
- }
-
- if(*e == ':') {
- *e = '\0';
- e++;
- service = e;
- }
-
- if(!*host) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Definition '%s' does not specify a host.",
- definition);
-
- return -1;
- }
-
- if(*interface) {
- scope_id = if_nametoindex(interface);
- if(!scope_id)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Cannot find a network interface named '%s'. Continuing with limiting the network interface",
- interface);
- }
-
- if(!*service)
- service = default_service;
-
-
- return connect_to_this_ip46(protocol, socktype, host, scope_id, service, timeout);
-}
-
-void foreach_entry_in_connection_string(const char *destination, bool (*callback)(char *entry, void *data), void *data) {
- const char *s = destination;
- while(*s) {
- const char *e = s;
-
- // skip separators, moving both s(tart) and e(nd)
- while(isspace(*e) || *e == ',') s = ++e;
-
- // move e(nd) to the first separator
- while(*e && !isspace(*e) && *e != ',') e++;
-
- // is there anything?
- if(!*s || s == e) break;
-
- char buf[e - s + 1];
- strncpyz(buf, s, e - s);
-
- if(callback(buf, data)) break;
-
- s = e;
- }
-}
-
-struct connect_to_one_of_data {
- int default_port;
- struct timeval *timeout;
- size_t *reconnects_counter;
- char *connected_to;
- size_t connected_to_size;
- int sock;
-};
-
-static bool connect_to_one_of_callback(char *entry, void *data) {
- struct connect_to_one_of_data *t = data;
-
- if(t->reconnects_counter)
- t->reconnects_counter++;
-
- t->sock = connect_to_this(entry, t->default_port, t->timeout);
- if(t->sock != -1) {
- if(t->connected_to && t->connected_to_size) {
- strncpyz(t->connected_to, entry, t->connected_to_size);
- t->connected_to[t->connected_to_size - 1] = '\0';
- }
-
- return true;
- }
-
- return false;
-}
-
-int connect_to_one_of(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size) {
- struct connect_to_one_of_data t = {
- .default_port = default_port,
- .timeout = timeout,
- .reconnects_counter = reconnects_counter,
- .connected_to = connected_to,
- .connected_to_size = connected_to_size,
- .sock = -1,
- };
-
- foreach_entry_in_connection_string(destination, connect_to_one_of_callback, &t);
-
- return t.sock;
-}
-
-static bool connect_to_one_of_urls_callback(char *entry, void *data) {
- char *s = strchr(entry, '/');
- if(s) *s = '\0';
-
- return connect_to_one_of_callback(entry, data);
-}
-
-int connect_to_one_of_urls(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size) {
- struct connect_to_one_of_data t = {
- .default_port = default_port,
- .timeout = timeout,
- .reconnects_counter = reconnects_counter,
- .connected_to = connected_to,
- .connected_to_size = connected_to_size,
- .sock = -1,
- };
-
- foreach_entry_in_connection_string(destination, connect_to_one_of_urls_callback, &t);
-
- return t.sock;
-}
-
-
-// --------------------------------------------------------------------------------------------------------------------
-// helpers to send/receive data in one call, in blocking mode, with a timeout
-
-#ifdef ENABLE_HTTPS
-ssize_t recv_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) {
-#else
-ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) {
-#endif
-
- for(;;) {
- struct pollfd fd = {
- .fd = sockfd,
- .events = POLLIN,
- .revents = 0
- };
-
- errno = 0;
- int retval = poll(&fd, 1, timeout * 1000);
-
- if(retval == -1) {
- // failed
-
- if(errno == EINTR || errno == EAGAIN)
- continue;
-
- return -1;
- }
-
- if(!retval) {
- // timeout
- return 0;
- }
-
- if(fd.revents & POLLIN)
- break;
- }
-
-#ifdef ENABLE_HTTPS
- if (SSL_connection(ssl)) {
- return netdata_ssl_read(ssl, buf, len);
- }
-#endif
-
- return recv(sockfd, buf, len, flags);
-}
-
-#ifdef ENABLE_HTTPS
-ssize_t send_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) {
-#else
-ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) {
-#endif
-
- for(;;) {
- struct pollfd fd = {
- .fd = sockfd,
- .events = POLLOUT,
- .revents = 0
- };
-
- errno = 0;
- int retval = poll(&fd, 1, timeout * 1000);
-
- if(retval == -1) {
- // failed
-
- if(errno == EINTR || errno == EAGAIN)
- continue;
-
- return -1;
- }
-
- if(!retval) {
- // timeout
- return 0;
- }
-
- if(fd.revents & POLLOUT) break;
- }
-
-#ifdef ENABLE_HTTPS
- if(ssl->conn) {
- if (SSL_connection(ssl)) {
- return netdata_ssl_write(ssl, buf, len);
- }
- else {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "cannot write to SSL connection - connection is not ready.");
-
- return -1;
- }
- }
-#endif
- return send(sockfd, buf, len, flags);
-}
-
-
-// --------------------------------------------------------------------------------------------------------------------
-// accept4() replacement for systems that do not have one
-
-#ifndef HAVE_ACCEPT4
-int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) {
- int fd = accept(sock, addr, addrlen);
- int newflags = 0;
-
- if (fd < 0) return fd;
-
- if (flags & SOCK_NONBLOCK) {
- newflags |= O_NONBLOCK;
- flags &= ~SOCK_NONBLOCK;
- }
-
-#ifdef SOCK_CLOEXEC
-#ifdef O_CLOEXEC
- if (flags & SOCK_CLOEXEC) {
- newflags |= O_CLOEXEC;
- flags &= ~SOCK_CLOEXEC;
- }
-#endif
-#endif
-
- if (flags) {
- close(fd);
- errno = EINVAL;
- return -1;
- }
-
- if (fcntl(fd, F_SETFL, newflags) < 0) {
- int saved_errno = errno;
- close(fd);
- errno = saved_errno;
- return -1;
- }
-
- return fd;
-}
-#endif
-
-/*
- * ---------------------------------------------------------------------------------------------------------------------
- * connection_allowed() - if there is an access list then check the connection matches a pattern.
- * Numeric patterns are checked against the IP address first, only if they
- * do not match is the hostname resolved (reverse-DNS) and checked. If the
- * hostname matches then we perform forward DNS resolution to check the IP
- * is really associated with the DNS record. This call is repeatable: the
- * web server may check more refined matches against the connection. Will
- * update the client_host if uninitialized - ensure the hostsize is the number
- * of *writable* bytes (i.e. be aware of the strdup used to compact the pollinfo).
- */
-int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list,
- const char *patname, int allow_dns)
-{
- if (!access_list)
- return 1;
- if (simple_pattern_matches(access_list, client_ip))
- return 1;
- // If the hostname is unresolved (and needed) then attempt the DNS lookups.
- //if (client_host[0]==0 && simple_pattern_is_potential_name(access_list))
- if (client_host[0]==0 && allow_dns)
- {
- struct sockaddr_storage sadr;
- socklen_t addrlen = sizeof(sadr);
- int err = getpeername(fd, (struct sockaddr*)&sadr, &addrlen);
- if (err != 0 ||
- (err = getnameinfo((struct sockaddr *)&sadr, addrlen, client_host, (socklen_t)hostsize,
- NULL, 0, NI_NAMEREQD)) != 0) {
-
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Incoming %s on '%s' does not match a numeric pattern, and host could not be resolved (err=%s)",
- patname, client_ip, gai_strerror(err));
-
- if (hostsize >= 8)
- strcpy(client_host,"UNKNOWN");
- return 0;
- }
- struct addrinfo *addr_infos = NULL;
- if (getaddrinfo(client_host, NULL, NULL, &addr_infos) !=0 ) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: cannot validate hostname '%s' from '%s' by resolving it",
- client_host, client_ip);
-
- if (hostsize >= 8)
- strcpy(client_host,"UNKNOWN");
- return 0;
- }
- struct addrinfo *scan = addr_infos;
- int validated = 0;
- while (scan) {
- char address[INET6_ADDRSTRLEN];
- address[0] = 0;
- switch (scan->ai_addr->sa_family) {
- case AF_INET:
- inet_ntop(AF_INET, &((struct sockaddr_in*)(scan->ai_addr))->sin_addr, address, INET6_ADDRSTRLEN);
- break;
- case AF_INET6:
- inet_ntop(AF_INET6, &((struct sockaddr_in6*)(scan->ai_addr))->sin6_addr, address, INET6_ADDRSTRLEN);
- break;
- }
- if (!strcmp(client_ip, address)) {
- validated = 1;
- break;
- }
- scan = scan->ai_next;
- }
- if (!validated) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: Cannot validate '%s' as ip of '%s', not listed in DNS",
- client_ip, client_host);
-
- if (hostsize >= 8)
- strcpy(client_host,"UNKNOWN");
- }
- if (addr_infos!=NULL)
- freeaddrinfo(addr_infos);
- }
- if (!simple_pattern_matches(access_list, client_host))
- return 0;
-
- return 1;
-}
-
-// --------------------------------------------------------------------------------------------------------------------
-// accept_socket() - accept a socket and store client IP and port
-int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize,
- char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list, int allow_dns) {
- struct sockaddr_storage sadr;
- socklen_t addrlen = sizeof(sadr);
-
- int nfd = accept4(fd, (struct sockaddr *)&sadr, &addrlen, flags);
- if (likely(nfd >= 0)) {
- if (getnameinfo((struct sockaddr *)&sadr, addrlen, client_ip, (socklen_t)ipsize,
- client_port, (socklen_t)portsize, NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
-
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "LISTENER: cannot getnameinfo() on received client connection.");
-
- strncpyz(client_ip, "UNKNOWN", ipsize);
- strncpyz(client_port, "UNKNOWN", portsize);
- }
- if (!strcmp(client_ip, "127.0.0.1") || !strcmp(client_ip, "::1")) {
- strncpyz(client_ip, "localhost", ipsize);
- }
-
-#ifdef __FreeBSD__
- if(((struct sockaddr *)&sadr)->sa_family == AF_LOCAL)
- strncpyz(client_ip, "localhost", ipsize);
-#endif
-
- client_ip[ipsize - 1] = '\0';
- client_port[portsize - 1] = '\0';
-
- switch (((struct sockaddr *)&sadr)->sa_family) {
- case AF_UNIX:
- // netdata_log_debug(D_LISTENER, "New UNIX domain web client from %s on socket %d.", client_ip, fd);
- // set the port - certain versions of libc return garbage on unix sockets
- strncpyz(client_port, "UNIX", portsize);
- break;
-
- case AF_INET:
- // netdata_log_debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd);
- break;
-
- case AF_INET6:
- if (strncmp(client_ip, "::ffff:", 7) == 0) {
- memmove(client_ip, &client_ip[7], strlen(&client_ip[7]) + 1);
- // netdata_log_debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd);
- }
- // else
- // netdata_log_debug(D_LISTENER, "New IPv6 web client from %s port %s on socket %d.", client_ip, client_port, fd);
- break;
-
- default:
- // netdata_log_debug(D_LISTENER, "New UNKNOWN web client from %s port %s on socket %d.", client_ip, client_port, fd);
- break;
- }
- if (!connection_allowed(nfd, client_ip, client_host, hostsize, access_list, "connection", allow_dns)) {
- errno = 0;
- nd_log(NDLS_DAEMON, NDLP_WARNING,
- "Permission denied for client '%s', port '%s'",
- client_ip, client_port);
-
- close(nfd);
- nfd = -1;
- errno = EPERM;
- }
- }
-#ifdef HAVE_ACCEPT4
- else if (errno == ENOSYS)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Netdata has been compiled with the assumption that the system has the accept4() call, but it is not here. "
- "Recompile netdata like this: ./configure --disable-accept4 ...");
-#endif
-
- return nfd;
-}
-
-
-// --------------------------------------------------------------------------------------------------------------------
-// poll() based listener
-// this should be the fastest possible listener for up to 100 sockets
-// above 100, an epoll() interface is needed on Linux
-
-#define POLL_FDS_INCREASE_STEP 10
-
-inline POLLINFO *poll_add_fd(POLLJOB *p
- , int fd
- , int socktype
- , WEB_CLIENT_ACL port_acl
- , uint32_t flags
- , const char *client_ip
- , const char *client_port
- , const char *client_host
- , void *(*add_callback)(POLLINFO * /*pi*/, short int * /*events*/, void * /*data*/)
- , void (*del_callback)(POLLINFO * /*pi*/)
- , int (*rcv_callback)(POLLINFO * /*pi*/, short int * /*events*/)
- , int (*snd_callback)(POLLINFO * /*pi*/, short int * /*events*/)
- , void *data
-) {
- if(unlikely(fd < 0)) return NULL;
-
- //if(p->limit && p->used >= p->limit) {
- // nd_log(NDLS_DAEMON, NDLP_WARNING, "Max sockets limit reached (%zu sockets), dropping connection", p->used);
- // close(fd);
- // return NULL;
- //}
-
- if(unlikely(!p->first_free)) {
- size_t new_slots = p->slots + POLL_FDS_INCREASE_STEP;
-
- p->fds = reallocz(p->fds, sizeof(struct pollfd) * new_slots);
- p->inf = reallocz(p->inf, sizeof(POLLINFO) * new_slots);
-
- // reset all the newly added slots
- ssize_t i;
- for(i = new_slots - 1; i >= (ssize_t)p->slots ; i--) {
- p->fds[i].fd = -1;
- p->fds[i].events = 0;
- p->fds[i].revents = 0;
-
- p->inf[i].p = p;
- p->inf[i].slot = (size_t)i;
- p->inf[i].flags = 0;
- p->inf[i].socktype = -1;
- p->inf[i].port_acl = -1;
-
- p->inf[i].client_ip = NULL;
- p->inf[i].client_port = NULL;
- p->inf[i].client_host = NULL;
- p->inf[i].del_callback = p->del_callback;
- p->inf[i].rcv_callback = p->rcv_callback;
- p->inf[i].snd_callback = p->snd_callback;
- p->inf[i].data = NULL;
-
- // link them so that the first free will be earlier in the array
- // (we loop decrementing i)
- p->inf[i].next = p->first_free;
- p->first_free = &p->inf[i];
- }
-
- p->slots = new_slots;
- }
-
- POLLINFO *pi = p->first_free;
- p->first_free = p->first_free->next;
-
- struct pollfd *pf = &p->fds[pi->slot];
- pf->fd = fd;
- pf->events = POLLIN;
- pf->revents = 0;
-
- pi->fd = fd;
- pi->p = p;
- pi->socktype = socktype;
- pi->port_acl = port_acl;
- pi->flags = flags;
- pi->next = NULL;
- pi->client_ip = strdupz(client_ip);
- pi->client_port = strdupz(client_port);
- pi->client_host = strdupz(client_host);
-
- pi->del_callback = del_callback;
- pi->rcv_callback = rcv_callback;
- pi->snd_callback = snd_callback;
-
- pi->connected_t = now_boottime_sec();
- pi->last_received_t = 0;
- pi->last_sent_t = 0;
- pi->last_sent_t = 0;
- pi->recv_count = 0;
- pi->send_count = 0;
-
- netdata_thread_disable_cancelability();
- p->used++;
- if(unlikely(pi->slot > p->max))
- p->max = pi->slot;
-
- if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) {
- pi->data = add_callback(pi, &pf->events, data);
- }
-
- if(pi->flags & POLLINFO_FLAG_SERVER_SOCKET) {
- p->min = pi->slot;
- }
- netdata_thread_enable_cancelability();
-
- return pi;
-}
-
-inline void poll_close_fd(POLLINFO *pi) {
- POLLJOB *p = pi->p;
-
- struct pollfd *pf = &p->fds[pi->slot];
-
- if(unlikely(pf->fd == -1)) return;
-
- netdata_thread_disable_cancelability();
-
- if(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) {
- pi->del_callback(pi);
-
- if(likely(!(pi->flags & POLLINFO_FLAG_DONT_CLOSE))) {
- if(close(pf->fd) == -1)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "Failed to close() poll_events() socket %d",
- pf->fd);
- }
- }
-
- pf->fd = -1;
- pf->events = 0;
- pf->revents = 0;
-
- pi->fd = -1;
- pi->socktype = -1;
- pi->flags = 0;
- pi->data = NULL;
-
- pi->del_callback = NULL;
- pi->rcv_callback = NULL;
- pi->snd_callback = NULL;
-
- freez(pi->client_ip);
- pi->client_ip = NULL;
-
- freez(pi->client_port);
- pi->client_port = NULL;
-
- freez(pi->client_host);
- pi->client_host = NULL;
-
- pi->next = p->first_free;
- p->first_free = pi;
-
- p->used--;
- if(unlikely(p->max == pi->slot)) {
- p->max = p->min;
- ssize_t i;
- for(i = (ssize_t)pi->slot; i > (ssize_t)p->min ;i--) {
- if (unlikely(p->fds[i].fd != -1)) {
- p->max = (size_t)i;
- break;
- }
- }
- }
- netdata_thread_enable_cancelability();
-}
-
-void *poll_default_add_callback(POLLINFO *pi, short int *events, void *data) {
- (void)pi;
- (void)events;
- (void)data;
-
- return NULL;
-}
-
-void poll_default_del_callback(POLLINFO *pi) {
- if(pi->data)
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "POLLFD: internal error: del_callback_default() called with data pointer - possible memory leak");
-}
-
-int poll_default_rcv_callback(POLLINFO *pi, short int *events) {
- *events |= POLLIN;
-
- char buffer[1024 + 1];
-
- ssize_t rc;
- do {
- rc = recv(pi->fd, buffer, 1024, MSG_DONTWAIT);
- if (rc < 0) {
- // read failed
- if (errno != EWOULDBLOCK && errno != EAGAIN) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "POLLFD: poll_default_rcv_callback(): recv() failed with %zd.",
- rc);
-
- return -1;
- }
- } else if (rc) {
- // data received
- nd_log(NDLS_DAEMON, NDLP_WARNING,
- "POLLFD: internal error: poll_default_rcv_callback() is discarding %zd bytes received on socket %d",
- rc, pi->fd);
- }
- } while (rc != -1);
-
- return 0;
-}
-
-int poll_default_snd_callback(POLLINFO *pi, short int *events) {
- *events &= ~POLLOUT;
-
- nd_log(NDLS_DAEMON, NDLP_WARNING,
- "POLLFD: internal error: poll_default_snd_callback(): nothing to send on socket %d",
- pi->fd);
-
- return 0;
-}
-
-void poll_default_tmr_callback(void *timer_data) {
- (void)timer_data;
-}
-
-static void poll_events_cleanup(void *data) {
- POLLJOB *p = (POLLJOB *)data;
-
- size_t i;
- for(i = 0 ; i <= p->max ; i++) {
- POLLINFO *pi = &p->inf[i];
- poll_close_fd(pi);
- }
-
- freez(p->fds);
- freez(p->inf);
-}
-
-static int poll_process_error(POLLINFO *pi, struct pollfd *pf, short int revents) {
- ND_LOG_STACK lgs[] = {
- ND_LOG_FIELD_TXT(NDF_SRC_IP, pi->client_ip),
- ND_LOG_FIELD_TXT(NDF_SRC_PORT, pi->client_port),
- ND_LOG_FIELD_END(),
- };
- ND_LOG_STACK_PUSH(lgs);
-
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "POLLFD: LISTENER: received %s %s %s on socket at slot %zu (fd %d) client '%s' port '%s' expecting %s %s %s, having %s %s %s"
- , revents & POLLERR ? "POLLERR" : ""
- , revents & POLLHUP ? "POLLHUP" : ""
- , revents & POLLNVAL ? "POLLNVAL" : ""
- , pi->slot
- , pi->fd
- , pi->client_ip ? pi->client_ip : "<undefined-ip>"
- , pi->client_port ? pi->client_port : "<undefined-port>"
- , pf->events & POLLIN ? "POLLIN" : "", pf->events & POLLOUT ? "POLLOUT" : "", pf->events & POLLPRI ? "POLLPRI" : ""
- , revents & POLLIN ? "POLLIN" : "", revents & POLLOUT ? "POLLOUT" : "", revents & POLLPRI ? "POLLPRI" : ""
- );
-
- pf->events = 0;
- poll_close_fd(pi);
- return 1;
-}
-
-static inline int poll_process_send(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, time_t now) {
- pi->last_sent_t = now;
- pi->send_count++;
-
- pf->events = 0;
-
- // remember the slot, in case we need to close it later
- // the callback may manipulate the socket list and our pf and pi pointers may be invalid after that call
- size_t slot = pi->slot;
-
- if (unlikely(pi->snd_callback(pi, &pf->events) == -1))
- poll_close_fd(&p->inf[slot]);
-
- // IMPORTANT:
- // pf and pi may be invalid below this point, they may have been reallocated.
-
- return 1;
-}
-
-static inline int poll_process_tcp_read(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, time_t now) {
- pi->last_received_t = now;
- pi->recv_count++;
-
- pf->events = 0;
-
- // remember the slot, in case we need to close it later
- // the callback may manipulate the socket list and our pf and pi pointers may be invalid after that call
- size_t slot = pi->slot;
-
- if (pi->rcv_callback(pi, &pf->events) == -1)
- poll_close_fd(&p->inf[slot]);
-
- // IMPORTANT:
- // pf and pi may be invalid below this point, they may have been reallocated.
-
- return 1;
-}
-
-static inline int poll_process_udp_read(POLLINFO *pi, struct pollfd *pf, time_t now __maybe_unused) {
- pi->last_received_t = now;
- pi->recv_count++;
-
- // TODO: access_list is not applied to UDP
- // but checking the access list on every UDP packet will destroy
- // performance, especially for statsd.
-
- pf->events = 0;
- if(pi->rcv_callback(pi, &pf->events) == -1)
- return 0;
-
- // IMPORTANT:
- // pf and pi may be invalid below this point, they may have been reallocated.
-
- return 1;
-}
-
-static int poll_process_new_tcp_connection(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, time_t now) {
- pi->last_received_t = now;
- pi->recv_count++;
-
- char client_ip[INET6_ADDRSTRLEN] = "";
- char client_port[NI_MAXSERV] = "";
- char client_host[NI_MAXHOST] = "";
-
- int nfd = accept_socket(
- pf->fd,SOCK_NONBLOCK,
- client_ip, INET6_ADDRSTRLEN, client_port,NI_MAXSERV, client_host, NI_MAXHOST,
- p->access_list, p->allow_dns
- );
-
- if (unlikely(nfd < 0)) {
- // accept failed
-
- if(unlikely(errno == EMFILE)) {
- nd_log_limit_static_global_var(erl, 10, 1000);
- nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR,
- "POLLFD: LISTENER: too many open files - used by this thread %zu, max for this thread %zu",
- p->used, p->limit);
- }
- else if(unlikely(errno != EWOULDBLOCK && errno != EAGAIN))
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "POLLFD: LISTENER: accept() failed.");
-
- }
- else {
- // accept ok
-
- poll_add_fd(p
- , nfd
- , SOCK_STREAM
- , pi->port_acl
- , POLLINFO_FLAG_CLIENT_SOCKET
- , client_ip
- , client_port
- , client_host
- , p->add_callback
- , p->del_callback
- , p->rcv_callback
- , p->snd_callback
- , NULL
- );
-
- // IMPORTANT:
- // pf and pi may be invalid below this point, they may have been reallocated.
-
- return 1;
- }
-
- return 0;
-}
-
-void poll_events(LISTEN_SOCKETS *sockets
- , void *(*add_callback)(POLLINFO * /*pi*/, short int * /*events*/, void * /*data*/)
- , void (*del_callback)(POLLINFO * /*pi*/)
- , int (*rcv_callback)(POLLINFO * /*pi*/, short int * /*events*/)
- , int (*snd_callback)(POLLINFO * /*pi*/, short int * /*events*/)
- , void (*tmr_callback)(void * /*timer_data*/)
- , bool (*check_to_stop_callback)(void)
- , SIMPLE_PATTERN *access_list
- , int allow_dns
- , void *data
- , time_t tcp_request_timeout_seconds
- , time_t tcp_idle_timeout_seconds
- , time_t timer_milliseconds
- , void *timer_data
- , size_t max_tcp_sockets
-) {
- if(!sockets || !sockets->opened) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "POLLFD: internal error: no listening sockets are opened");
- return;
- }
-
- if(timer_milliseconds <= 0) timer_milliseconds = 0;
-
- int retval;
-
- POLLJOB p = {
- .slots = 0,
- .used = 0,
- .max = 0,
- .limit = max_tcp_sockets,
- .fds = NULL,
- .inf = NULL,
- .first_free = NULL,
-
- .complete_request_timeout = tcp_request_timeout_seconds,
- .idle_timeout = tcp_idle_timeout_seconds,
- .checks_every = (tcp_idle_timeout_seconds / 3) + 1,
-
- .access_list = access_list,
- .allow_dns = allow_dns,
-
- .timer_milliseconds = timer_milliseconds,
- .timer_data = timer_data,
-
- .add_callback = add_callback?add_callback:poll_default_add_callback,
- .del_callback = del_callback?del_callback:poll_default_del_callback,
- .rcv_callback = rcv_callback?rcv_callback:poll_default_rcv_callback,
- .snd_callback = snd_callback?snd_callback:poll_default_snd_callback,
- .tmr_callback = tmr_callback?tmr_callback:poll_default_tmr_callback
- };
-
- size_t i;
- for(i = 0; i < sockets->opened ;i++) {
-
- POLLINFO *pi = poll_add_fd(&p
- , sockets->fds[i]
- , sockets->fds_types[i]
- , sockets->fds_acl_flags[i]
- , POLLINFO_FLAG_SERVER_SOCKET
- , (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN"
- , ""
- , ""
- , p.add_callback
- , p.del_callback
- , p.rcv_callback
- , p.snd_callback
- , NULL
- );
-
- pi->data = data;
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "POLLFD: LISTENER: listening on '%s'",
- (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN");
- }
-
- int listen_sockets_active = 1;
-
- int timeout_ms = 1000; // in milliseconds
- time_t last_check = now_boottime_sec();
-
- usec_t timer_usec = timer_milliseconds * USEC_PER_MS;
- usec_t now_usec = 0, next_timer_usec = 0, last_timer_usec = 0;
- (void)last_timer_usec;
-
- if(unlikely(timer_usec)) {
- now_usec = now_boottime_usec();
- next_timer_usec = now_usec - (now_usec % timer_usec) + timer_usec;
- }
-
- netdata_thread_cleanup_push(poll_events_cleanup, &p);
-
- while(!check_to_stop_callback()) {
- if(unlikely(timer_usec)) {
- now_usec = now_boottime_usec();
-
- if(unlikely(timer_usec && now_usec >= next_timer_usec)) {
- last_timer_usec = now_usec;
- p.tmr_callback(p.timer_data);
- now_usec = now_boottime_usec();
- next_timer_usec = now_usec - (now_usec % timer_usec) + timer_usec;
- }
-
- usec_t dt_usec = next_timer_usec - now_usec;
- if(dt_usec < 1000 * USEC_PER_MS)
- timeout_ms = 1000;
- else
- timeout_ms = (int)(dt_usec / USEC_PER_MS);
- }
-
- // enable or disable the TCP listening sockets, based on the current number of sockets used and the limit set
- if((listen_sockets_active && (p.limit && p.used >= p.limit)) || (!listen_sockets_active && (!p.limit || p.used < p.limit))) {
- listen_sockets_active = !listen_sockets_active;
-
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "%s listening sockets (used TCP sockets %zu, max allowed for this worker %zu)",
- (listen_sockets_active)?"ENABLING":"DISABLING", p.used, p.limit);
-
- for (i = 0; i <= p.max; i++) {
- if(p.inf[i].flags & POLLINFO_FLAG_SERVER_SOCKET && p.inf[i].socktype == SOCK_STREAM) {
- p.fds[i].events = (short int) ((listen_sockets_active) ? POLLIN : 0);
- }
- }
- }
-
- retval = poll(p.fds, p.max + 1, timeout_ms);
- time_t now = now_boottime_sec();
-
- if(unlikely(retval == -1)) {
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "POLLFD: LISTENER: poll() failed while waiting on %zu sockets.",
- p.max + 1);
-
- break;
- }
- else if(unlikely(!retval)) {
- // timeout
- ;
- }
- else {
- POLLINFO *pi;
- struct pollfd *pf;
- size_t idx, processed = 0;
- short int revents;
-
- // keep fast lookup arrays per function
- // to avoid looping through the entire list every time
- size_t sends[p.max + 1], sends_max = 0;
- size_t reads[p.max + 1], reads_max = 0;
- size_t conns[p.max + 1], conns_max = 0;
- size_t udprd[p.max + 1], udprd_max = 0;
-
- for (i = 0; i <= p.max; i++) {
- pi = &p.inf[i];
- pf = &p.fds[i];
- revents = pf->revents;
-
- if(unlikely(revents == 0 || pf->fd == -1))
- continue;
-
- if (unlikely(revents & (POLLERR|POLLHUP|POLLNVAL))) {
- // something is wrong to one of our sockets
-
- pf->revents = 0;
- processed += poll_process_error(pi, pf, revents);
- }
- else if (likely(revents & POLLOUT)) {
- // a client is ready to receive data
-
- sends[sends_max++] = i;
- }
- else if (likely(revents & (POLLIN|POLLPRI))) {
- if (pi->flags & POLLINFO_FLAG_CLIENT_SOCKET) {
- // a client sent data to us
-
- reads[reads_max++] = i;
- }
- else if (pi->flags & POLLINFO_FLAG_SERVER_SOCKET) {
- // something is coming to our server sockets
-
- if(pi->socktype == SOCK_DGRAM) {
- // UDP receive, directly on our listening socket
-
- udprd[udprd_max++] = i;
- }
- else if(pi->socktype == SOCK_STREAM) {
- // new TCP connection
-
- conns[conns_max++] = i;
- }
- else
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "POLLFD: LISTENER: server slot %zu (fd %d) connection from %s port %s using unhandled socket type %d."
- , i
- , pi->fd
- , pi->client_ip ? pi->client_ip : "<undefined-ip>"
- , pi->client_port ? pi->client_port : "<undefined-port>"
- , pi->socktype
- );
- }
- else
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "POLLFD: LISTENER: client slot %zu (fd %d) data from %s port %s using flags %08X is neither client nor server."
- , i
- , pi->fd
- , pi->client_ip ? pi->client_ip : "<undefined-ip>"
- , pi->client_port ? pi->client_port : "<undefined-port>"
- , pi->flags
- );
- }
- else
- nd_log(NDLS_DAEMON, NDLP_ERR,
- "POLLFD: LISTENER: socket slot %zu (fd %d) client %s port %s unhandled event id %d."
- , i
- , pi->fd
- , pi->client_ip ? pi->client_ip : "<undefined-ip>"
- , pi->client_port ? pi->client_port : "<undefined-port>"
- , revents
- );
- }
-
- // process sends
- for (idx = 0; idx < sends_max; idx++) {
- i = sends[idx];
- pi = &p.inf[i];
- pf = &p.fds[i];
- pf->revents = 0;
- processed += poll_process_send(&p, pi, pf, now);
- }
-
- // process UDP reads
- for (idx = 0; idx < udprd_max; idx++) {
- i = udprd[idx];
- pi = &p.inf[i];
- pf = &p.fds[i];
- pf->revents = 0;
- processed += poll_process_udp_read(pi, pf, now);
- }
-
- // process TCP reads
- for (idx = 0; idx < reads_max; idx++) {
- i = reads[idx];
- pi = &p.inf[i];
- pf = &p.fds[i];
- pf->revents = 0;
- processed += poll_process_tcp_read(&p, pi, pf, now);
- }
-
- if(!processed && (!p.limit || p.used < p.limit)) {
- // nothing processed above (rcv, snd) and we have room for another TCP connection
- // so, accept one TCP connection
- for (idx = 0; idx < conns_max; idx++) {
- i = conns[idx];
- pi = &p.inf[i];
- pf = &p.fds[i];
- pf->revents = 0;
- if (poll_process_new_tcp_connection(&p, pi, pf, now))
- break;
- }
- }
- }
-
- if(unlikely(p.checks_every > 0 && now - last_check > p.checks_every)) {
- last_check = now;
-
- // cleanup old sockets
- for(i = 0; i <= p.max; i++) {
- POLLINFO *pi = &p.inf[i];
-
- if(likely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET)) {
- if (unlikely(pi->send_count == 0 && p.complete_request_timeout > 0 && (now - pi->connected_t) >= p.complete_request_timeout)) {
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "POLLFD: LISTENER: client slot %zu (fd %d) from %s port %s has not sent a complete request in %zu seconds - closing it. "
- , i
- , pi->fd
- , pi->client_ip ? pi->client_ip : "<undefined-ip>"
- , pi->client_port ? pi->client_port : "<undefined-port>"
- , (size_t) p.complete_request_timeout
- );
- poll_close_fd(pi);
- }
- else if(unlikely(pi->recv_count && p.idle_timeout > 0 && now - ((pi->last_received_t > pi->last_sent_t) ? pi->last_received_t : pi->last_sent_t) >= p.idle_timeout )) {
- nd_log(NDLS_DAEMON, NDLP_DEBUG,
- "POLLFD: LISTENER: client slot %zu (fd %d) from %s port %s is idle for more than %zu seconds - closing it. "
- , i
- , pi->fd
- , pi->client_ip ? pi->client_ip : "<undefined-ip>"
- , pi->client_port ? pi->client_port : "<undefined-port>"
- , (size_t) p.idle_timeout
- );
- poll_close_fd(pi);
- }
- }
- }
- }
- }
-
- netdata_thread_cleanup_pop(1);
-}
diff --git a/libnetdata/socket/socket.h b/libnetdata/socket/socket.h
deleted file mode 100644
index e4ca08d47..000000000
--- a/libnetdata/socket/socket.h
+++ /dev/null
@@ -1,248 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_SOCKET_H
-#define NETDATA_SOCKET_H
-
-#include "../libnetdata.h"
-
-#ifndef MAX_LISTEN_FDS
-#define MAX_LISTEN_FDS 50
-#endif
-
-typedef enum web_client_acl {
- WEB_CLIENT_ACL_NONE = (0),
- WEB_CLIENT_ACL_NOCHECK = (1 << 0), // Don't check anything - this should work on all channels
- WEB_CLIENT_ACL_DASHBOARD = (1 << 1),
- WEB_CLIENT_ACL_REGISTRY = (1 << 2),
- WEB_CLIENT_ACL_BADGE = (1 << 3),
- WEB_CLIENT_ACL_MGMT = (1 << 4),
- WEB_CLIENT_ACL_STREAMING = (1 << 5),
- WEB_CLIENT_ACL_NETDATACONF = (1 << 6),
- WEB_CLIENT_ACL_SSL_OPTIONAL = (1 << 7),
- WEB_CLIENT_ACL_SSL_FORCE = (1 << 8),
- WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 9),
- WEB_CLIENT_ACL_ACLK = (1 << 10),
- WEB_CLIENT_ACL_WEBRTC = (1 << 11),
- WEB_CLIENT_ACL_BEARER_OPTIONAL = (1 << 12), // allow unprotected access if bearer is not enabled in netdata
- WEB_CLIENT_ACL_BEARER_REQUIRED = (1 << 13), // allow access only if a valid bearer is used
-} WEB_CLIENT_ACL;
-
-#define WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_OPTIONAL)
-#define WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_REQUIRED)
-
-#ifdef NETDATA_DEV_MODE
-#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_NOCHECK
-#else
-#define ACL_DEV_OPEN_ACCESS 0
-#endif
-
-#define WEB_CLIENT_ACL_ALL 0xFFFF
-
-#define web_client_can_access_dashboard(w) ((w)->acl & WEB_CLIENT_ACL_DASHBOARD)
-#define web_client_can_access_registry(w) ((w)->acl & WEB_CLIENT_ACL_REGISTRY)
-#define web_client_can_access_badges(w) ((w)->acl & WEB_CLIENT_ACL_BADGE)
-#define web_client_can_access_mgmt(w) ((w)->acl & WEB_CLIENT_ACL_MGMT)
-#define web_client_can_access_stream(w) ((w)->acl & WEB_CLIENT_ACL_STREAMING)
-#define web_client_can_access_netdataconf(w) ((w)->acl & WEB_CLIENT_ACL_NETDATACONF)
-#define web_client_is_using_ssl_optional(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_OPTIONAL)
-#define web_client_is_using_ssl_force(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_FORCE)
-#define web_client_is_using_ssl_default(w) ((w)->port_acl & WEB_CLIENT_ACL_SSL_DEFAULT)
-
-typedef struct listen_sockets {
- struct config *config; // the config file to use
- const char *config_section; // the netdata configuration section to read settings from
- const char *default_bind_to; // the default bind to configuration string
- uint16_t default_port; // the default port to use
- int backlog; // the default listen backlog to use
-
- size_t opened; // the number of sockets opened
- size_t failed; // the number of sockets attempted to open, but failed
- int fds[MAX_LISTEN_FDS]; // the open sockets
- char *fds_names[MAX_LISTEN_FDS]; // descriptions for the open sockets
- int fds_types[MAX_LISTEN_FDS]; // the socktype for the open sockets (SOCK_STREAM, SOCK_DGRAM)
- int fds_families[MAX_LISTEN_FDS]; // the family of the open sockets (AF_UNIX, AF_INET, AF_INET6)
- WEB_CLIENT_ACL fds_acl_flags[MAX_LISTEN_FDS]; // the acl to apply to the open sockets (dashboard, badges, streaming, netdata.conf, management)
-} LISTEN_SOCKETS;
-
-char *strdup_client_description(int family, const char *protocol, const char *ip, uint16_t port);
-
-int listen_sockets_setup(LISTEN_SOCKETS *sockets);
-void listen_sockets_close(LISTEN_SOCKETS *sockets);
-
-void foreach_entry_in_connection_string(const char *destination, bool (*callback)(char *entry, void *data), void *data);
-int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t scope_id, const char *service, struct timeval *timeout);
-int connect_to_this(const char *definition, int default_port, struct timeval *timeout);
-int connect_to_one_of(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size);
-int connect_to_one_of_urls(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size);
-
-
-#ifdef ENABLE_HTTPS
-ssize_t recv_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout);
-ssize_t send_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout);
-#else
-ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout);
-ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout);
-#endif
-
-bool fd_is_socket(int fd);
-bool sock_has_output_error(int fd);
-
-int sock_setnonblock(int fd);
-int sock_delnonblock(int fd);
-int sock_setreuse(int fd, int reuse);
-int sock_setreuse_port(int fd, int reuse);
-int sock_enlarge_in(int fd);
-int sock_enlarge_out(int fd);
-
-int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsize,
- SIMPLE_PATTERN *access_list, const char *patname, int allow_dns);
-int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *client_port, size_t portsize,
- char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list, int allow_dns);
-
-#ifndef HAVE_ACCEPT4
-int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags);
-
-#ifndef SOCK_NONBLOCK
-#define SOCK_NONBLOCK 00004000
-#endif /* #ifndef SOCK_NONBLOCK */
-
-#ifndef SOCK_CLOEXEC
-#define SOCK_CLOEXEC 02000000
-#endif /* #ifndef SOCK_CLOEXEC */
-
-#endif /* #ifndef HAVE_ACCEPT4 */
-
-
-// ----------------------------------------------------------------------------
-// poll() based listener
-
-#define POLLINFO_FLAG_SERVER_SOCKET 0x00000001
-#define POLLINFO_FLAG_CLIENT_SOCKET 0x00000002
-#define POLLINFO_FLAG_DONT_CLOSE 0x00000004
-
-typedef struct poll POLLJOB;
-
-typedef struct pollinfo {
- POLLJOB *p; // the parent
- size_t slot; // the slot id
-
- int fd; // the file descriptor
- int socktype; // the client socket type
- WEB_CLIENT_ACL port_acl; // the access lists permitted on this web server port (it's -1 for client sockets)
- char *client_ip; // Max INET6_ADDRSTRLEN bytes
- char *client_port; // Max NI_MAXSERV bytes
- char *client_host; // Max NI_MAXHOST bytes
-
- time_t connected_t; // the time the socket connected
- time_t last_received_t; // the time the socket last received data
- time_t last_sent_t; // the time the socket last sent data
-
- size_t recv_count; // the number of times the socket was ready for inbound traffic
- size_t send_count; // the number of times the socket was ready for outbound traffic
-
- uint32_t flags; // internal flags
-
- // callbacks for this socket
- void (*del_callback)(struct pollinfo *pi);
- int (*rcv_callback)(struct pollinfo *pi, short int *events);
- int (*snd_callback)(struct pollinfo *pi, short int *events);
-
- // the user data
- void *data;
-
- // linking of free pollinfo structures
- // for quickly finding the next available
- // this is like a stack, it grows and shrinks
- // (with gaps - lower empty slots are preferred)
- struct pollinfo *next;
-} POLLINFO;
-
-struct poll {
- size_t slots;
- size_t used;
- size_t min;
- size_t max;
-
- size_t limit;
-
- time_t complete_request_timeout;
- time_t idle_timeout;
- time_t checks_every;
-
- time_t timer_milliseconds;
- void *timer_data;
-
- struct pollfd *fds;
- struct pollinfo *inf;
- struct pollinfo *first_free;
-
- SIMPLE_PATTERN *access_list;
- int allow_dns;
-
- void *(*add_callback)(POLLINFO *pi, short int *events, void *data);
- void (*del_callback)(POLLINFO *pi);
- int (*rcv_callback)(POLLINFO *pi, short int *events);
- int (*snd_callback)(POLLINFO *pi, short int *events);
- void (*tmr_callback)(void *timer_data);
-};
-
-#define pollinfo_from_slot(p, slot) (&((p)->inf[(slot)]))
-
-int poll_default_snd_callback(POLLINFO *pi, short int *events);
-int poll_default_rcv_callback(POLLINFO *pi, short int *events);
-void poll_default_del_callback(POLLINFO *pi);
-void *poll_default_add_callback(POLLINFO *pi, short int *events, void *data);
-
-POLLINFO *poll_add_fd(POLLJOB *p
- , int fd
- , int socktype
- , WEB_CLIENT_ACL port_acl
- , uint32_t flags
- , const char *client_ip
- , const char *client_port
- , const char *client_host
- , void *(*add_callback)(POLLINFO *pi, short int *events, void *data)
- , void (*del_callback)(POLLINFO *pi)
- , int (*rcv_callback)(POLLINFO *pi, short int *events)
- , int (*snd_callback)(POLLINFO *pi, short int *events)
- , void *data
-);
-void poll_close_fd(POLLINFO *pi);
-
-void poll_events(LISTEN_SOCKETS *sockets
- , void *(*add_callback)(POLLINFO *pi, short int *events, void *data)
- , void (*del_callback)(POLLINFO *pi)
- , int (*rcv_callback)(POLLINFO *pi, short int *events)
- , int (*snd_callback)(POLLINFO *pi, short int *events)
- , void (*tmr_callback)(void *timer_data)
- , bool (*check_to_stop_callback)(void)
- , SIMPLE_PATTERN *access_list
- , int allow_dns
- , void *data
- , time_t tcp_request_timeout_seconds
- , time_t tcp_idle_timeout_seconds
- , time_t timer_milliseconds
- , void *timer_data
- , size_t max_tcp_sockets
-);
-
-#ifndef INET6_ADDRSTRLEN
-#define INET6_ADDRSTRLEN 46
-#endif
-
-typedef struct socket_peers {
- struct {
- char ip[INET6_ADDRSTRLEN];
- int port;
- } local;
-
- struct {
- char ip[INET6_ADDRSTRLEN];
- int port;
- } peer;
-} SOCKET_PEERS;
-
-SOCKET_PEERS socket_peers(int sock_fd);
-bool ip_to_hostname(const char *ip, char *dst, size_t dst_len);
-
-#endif //NETDATA_SOCKET_H
diff --git a/libnetdata/statistical/Makefile.am b/libnetdata/statistical/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/statistical/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/statistical/README.md b/libnetdata/statistical/README.md
deleted file mode 100644
index 937c26ac6..000000000
--- a/libnetdata/statistical/README.md
+++ /dev/null
@@ -1,12 +0,0 @@
-<!--
-title: "Statistical functions"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/statistical/README.md
-sidebar_label: "Statistical functions"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Statistical functions
-
-A library for easy and fast calculations of statistical measurements like average, median etc.
diff --git a/libnetdata/storage_number/Makefile.am b/libnetdata/storage_number/Makefile.am
deleted file mode 100644
index c5f84504f..000000000
--- a/libnetdata/storage_number/Makefile.am
+++ /dev/null
@@ -1,12 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-SUBDIRS = \
- tests \
- $(NULL)
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/storage_number/README.md b/libnetdata/storage_number/README.md
deleted file mode 100644
index 2dd6df6cf..000000000
--- a/libnetdata/storage_number/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-<!--
-title: "Netdata storage number"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/storage_number/README.md
-sidebar_label: "Storage number"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Netdata storage number
-
-Although `netdata` does all its calculations using `long double`, it stores all values using
-a **custom-made 32-bit number**.
-
-This custom-made number can store in 29 bits values from `-167772150000000.0` to `167772150000000.0`
-with a precision of 0.00001 (yes, it's a floating point number, meaning that higher integer values
-have less decimal precision) and 3 bits for flags.
-
-This provides an extremely optimized memory footprint with just 0.0001% max accuracy loss.
-
-
diff --git a/libnetdata/storage_number/tests/Makefile.am b/libnetdata/storage_number/tests/Makefile.am
deleted file mode 100644
index babdcf0df..000000000
--- a/libnetdata/storage_number/tests/Makefile.am
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
diff --git a/libnetdata/string/Makefile.am b/libnetdata/string/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/string/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/string/README.md b/libnetdata/string/README.md
deleted file mode 100644
index 4fd442507..000000000
--- a/libnetdata/string/README.md
+++ /dev/null
@@ -1,25 +0,0 @@
-<!--
-title: "String"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/string/README.md
-sidebar_label: "String"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# STRING
-
-STRING provides a way to allocate and free text strings, while de-duplicating them.
-
-It can be used similarly to libc string functions:
-
- - `strdup()` and `strdupz()` become `string_strdupz()`.
- - `strlen()` becomes `string_strlen()` (and it does not walkthrough the bytes of the string).
- - `free()` and `freez()` become `string_freez()`.
-
-There is also a special `string_dup()` function that increases the reference counter of a STRING, avoiding the
-index lookup to find it.
-
-Once there is a `STRING *`, the actual `const char *` can be accessed with `string2str()`.
-
-All STRING should be constant. Changing the contents of a `const char *` that has been acquired by `string2str()` should never happen.
diff --git a/libnetdata/string/string.c b/libnetdata/string/string.c
deleted file mode 100644
index e1c8352a5..000000000
--- a/libnetdata/string/string.c
+++ /dev/null
@@ -1,688 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "../libnetdata.h"
-#include <Judy.h>
-
-typedef int32_t REFCOUNT;
-
-// ----------------------------------------------------------------------------
-// STRING implementation - dedup all STRING
-
-#define STRING_PARTITION_SHIFTS (0)
-#define STRING_PARTITIONS (256 >> STRING_PARTITION_SHIFTS)
-#define string_partition_str(str) ((uint8_t)((str)[0]) >> STRING_PARTITION_SHIFTS)
-#define string_partition(string) (string_partition_str((string)->str))
-
-struct netdata_string {
- uint32_t length; // the string length including the terminating '\0'
-
- REFCOUNT refcount; // how many times this string is used
- // We use a signed number to be able to detect duplicate frees of a string.
- // If at any point this goes below zero, we have a duplicate free.
-
- const char str[]; // the string itself, is appended to this structure
-};
-
-static struct string_partition {
- RW_SPINLOCK spinlock; // the R/W spinlock to protect the Judy array
-
- Pvoid_t JudyHSArray; // the Judy array - hashtable
-
- size_t inserts; // the number of successful inserts to the index
- size_t deletes; // the number of successful deleted from the index
-
- long int entries; // the number of entries in the index
- long int memory; // the memory used, without the JudyHS index
-
-#ifdef NETDATA_INTERNAL_CHECKS
- // internal statistics
-
- struct {
- size_t searches; // the number of successful searches in the index
- size_t releases; // when a string is unreferenced
- size_t duplications; // when a string is referenced
- long int active_references; // the number of active references alive
- } atomic;
-
- size_t found_deleted_on_search;
- size_t found_available_on_search;
- size_t found_deleted_on_insert;
- size_t found_available_on_insert;
- size_t spins;
-#endif
-
-} string_base[STRING_PARTITIONS] = { 0 };
-
-#ifdef NETDATA_INTERNAL_CHECKS
-#define string_stats_atomic_increment(partition, var) __atomic_add_fetch(&string_base[partition].atomic.var, 1, __ATOMIC_RELAXED)
-#define string_stats_atomic_decrement(partition, var) __atomic_sub_fetch(&string_base[partition].atomic.var, 1, __ATOMIC_RELAXED)
-#define string_internal_stats_add(partition, var, val) __atomic_add_fetch(&string_base[partition].var, val, __ATOMIC_RELAXED)
-#else
-#define string_stats_atomic_increment(partition, var) do {;} while(0)
-#define string_stats_atomic_decrement(partition, var) do {;} while(0)
-#define string_internal_stats_add(partition, var, val) do {;} while(0)
-#endif
-
-void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases) {
- if (inserts) *inserts = 0;
- if (deletes) *deletes = 0;
- if (searches) *searches = 0;
- if (entries) *entries = 0;
- if (references) *references = 0;
- if (memory) *memory = 0;
- if (duplications) *duplications = 0;
- if (releases) *releases = 0;
-
- for(size_t i = 0; i < STRING_PARTITIONS ;i++) {
- if (inserts) *inserts += string_base[i].inserts;
- if (deletes) *deletes += string_base[i].deletes;
- if (entries) *entries += (size_t) string_base[i].entries;
- if (memory) *memory += (size_t) string_base[i].memory;
-
-#ifdef NETDATA_INTERNAL_CHECKS
- if (searches) *searches += string_base[i].atomic.searches;
- if (references) *references += (size_t) string_base[i].atomic.active_references;
- if (duplications) *duplications += string_base[i].atomic.duplications;
- if (releases) *releases += string_base[i].atomic.releases;
-#endif
- }
-}
-
-#define string_entry_acquire(se) __atomic_add_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST);
-#define string_entry_release(se) __atomic_sub_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST);
-
-static inline bool string_entry_check_and_acquire(STRING *se) {
-#ifdef NETDATA_INTERNAL_CHECKS
- uint8_t partition = string_partition(se);
-#endif
-
- REFCOUNT expected, desired, count = 0;
-
- expected = __atomic_load_n(&se->refcount, __ATOMIC_SEQ_CST);
-
- do {
- count++;
-
- if(expected <= 0) {
- // We cannot use this.
- // The reference counter reached value zero,
- // so another thread is deleting this.
- string_internal_stats_add(partition, spins, count - 1);
- return false;
- }
-
- desired = expected + 1;
-
- } while(!__atomic_compare_exchange_n(&se->refcount, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST));
-
- string_internal_stats_add(partition, spins, count - 1);
-
- // statistics
- // string_base.active_references is altered at the in string_strdupz() and string_freez()
- string_stats_atomic_increment(partition, duplications);
-
- return true;
-}
-
-STRING *string_dup(STRING *string) {
- if(unlikely(!string)) return NULL;
-
-#ifdef NETDATA_INTERNAL_CHECKS
- if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) <= 0))
- fatal("STRING: tried to %s() a string that is freed (it has %d references).", __FUNCTION__, string->refcount);
-#endif
-
- string_entry_acquire(string);
-
-#ifdef NETDATA_INTERNAL_CHECKS
- uint8_t partition = string_partition(string);
-#endif
-
- // statistics
- string_stats_atomic_increment(partition, active_references);
- string_stats_atomic_increment(partition, duplications);
-
- return string;
-}
-
-// Search the index and return an ACQUIRED string entry, or NULL
-static inline STRING *string_index_search(const char *str, size_t length) {
- STRING *string;
-
- uint8_t partition = string_partition_str(str);
-
- // Find the string in the index
- // With a read-lock so that multiple readers can use the index concurrently.
-
- rw_spinlock_read_lock(&string_base[partition].spinlock);
-
- Pvoid_t *Rc;
- Rc = JudyHSGet(string_base[partition].JudyHSArray, (void *)str, length - 1);
- if(likely(Rc)) {
- // found in the hash table
- string = *Rc;
-
- if(string_entry_check_and_acquire(string)) {
- // we can use this entry
- string_internal_stats_add(partition, found_available_on_search, 1);
- }
- else {
- // this entry is about to be deleted by another thread
- // do not touch it, let it go...
- string = NULL;
- string_internal_stats_add(partition, found_deleted_on_search, 1);
- }
- }
- else {
- // not found in the hash table
- string = NULL;
- }
-
- string_stats_atomic_increment(partition, searches);
- rw_spinlock_read_unlock(&string_base[partition].spinlock);
-
- return string;
-}
-
-// Insert a string to the index and return an ACQUIRED string entry,
-// or NULL if the call needs to be retried (a deleted entry with the same key is still in the index)
-// The returned entry is ACQUIRED, and it can either be:
-// 1. a new item inserted, or
-// 2. an item found in the index that is not currently deleted
-static inline STRING *string_index_insert(const char *str, size_t length) {
- STRING *string;
-
- uint8_t partition = string_partition_str(str);
-
- rw_spinlock_write_lock(&string_base[partition].spinlock);
-
- STRING **ptr;
- {
- JError_t J_Error;
- Pvoid_t *Rc = JudyHSIns(&string_base[partition].JudyHSArray, (void *)str, length - 1, &J_Error);
- if (unlikely(Rc == PJERR)) {
- fatal(
- "STRING: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d",
- str,
- JU_ERRNO(&J_Error),
- JU_ERRID(&J_Error));
- }
- ptr = (STRING **)Rc;
- }
-
- if (likely(*ptr == 0)) {
- // a new item added to the index
- size_t mem_size = sizeof(STRING) + length;
- string = mallocz(mem_size);
- strcpy((char *)string->str, str);
- string->length = length;
- string->refcount = 1;
- *ptr = string;
- string_base[partition].inserts++;
- string_base[partition].entries++;
- string_base[partition].memory += (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(length));
- }
- else {
- // the item is already in the index
- string = *ptr;
-
- if(string_entry_check_and_acquire(string)) {
- // we can use this entry
- string_internal_stats_add(partition, found_available_on_insert, 1);
- }
- else {
- // this entry is about to be deleted by another thread
- // do not touch it, let it go...
- string = NULL;
- string_internal_stats_add(partition, found_deleted_on_insert, 1);
- }
-
- string_stats_atomic_increment(partition, searches);
- }
-
- rw_spinlock_write_unlock(&string_base[partition].spinlock);
- return string;
-}
-
-// delete an entry from the index
-static inline void string_index_delete(STRING *string) {
- uint8_t partition = string_partition(string);
-
- rw_spinlock_write_lock(&string_base[partition].spinlock);
-
-#ifdef NETDATA_INTERNAL_CHECKS
- if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) != 0))
- fatal("STRING: tried to delete a string at %s() that is already freed (it has %d references).", __FUNCTION__, string->refcount);
-#endif
-
- bool deleted = false;
-
- if (likely(string_base[partition].JudyHSArray)) {
- JError_t J_Error;
- int ret = JudyHSDel(&string_base[partition].JudyHSArray, (void *)string->str, string->length - 1, &J_Error);
- if (unlikely(ret == JERR)) {
- netdata_log_error(
- "STRING: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d",
- string->str,
- JU_ERRNO(&J_Error),
- JU_ERRID(&J_Error));
- } else
- deleted = true;
- }
-
- if (unlikely(!deleted))
- netdata_log_error("STRING: tried to delete '%s' that is not in the index. Ignoring it.", string->str);
- else {
- size_t mem_size = sizeof(STRING) + string->length;
- string_base[partition].deletes++;
- string_base[partition].entries--;
- string_base[partition].memory -= (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(string->length));
- freez(string);
- }
-
- rw_spinlock_write_unlock(&string_base[partition].spinlock);
-}
-
-STRING *string_strdupz(const char *str) {
- if(unlikely(!str || !*str)) return NULL;
-
-#ifdef NETDATA_INTERNAL_CHECKS
- uint8_t partition = string_partition_str(str);
-#endif
-
- size_t length = strlen(str) + 1;
- STRING *string = string_index_search(str, length);
-
- while(!string) {
- // The search above did not find anything,
- // We loop here, because during insert we may find an entry that is being deleted by another thread.
- // So, we have to let it go and retry to insert it again.
-
- string = string_index_insert(str, length);
- }
-
- // statistics
- string_stats_atomic_increment(partition, active_references);
-
- return string;
-}
-
-void string_freez(STRING *string) {
- if(unlikely(!string)) return;
-
-#ifdef NETDATA_INTERNAL_CHECKS
- uint8_t partition = string_partition(string);
-#endif
- REFCOUNT refcount = string_entry_release(string);
-
-#ifdef NETDATA_INTERNAL_CHECKS
- if(unlikely(refcount < 0))
- fatal("STRING: tried to %s() a string that is already freed (it has %d references).", __FUNCTION__, string->refcount);
-#endif
-
- if(unlikely(refcount == 0))
- string_index_delete(string);
-
- // statistics
- string_stats_atomic_decrement(partition, active_references);
- string_stats_atomic_increment(partition, releases);
-}
-
-inline size_t string_strlen(STRING *string) {
- if(unlikely(!string)) return 0;
- return string->length - 1;
-}
-
-inline const char *string2str(STRING *string) {
- if(unlikely(!string)) return "";
- return string->str;
-}
-
-STRING *string_2way_merge(STRING *a, STRING *b) {
- static STRING *X = NULL;
-
- if(unlikely(!X)) {
- X = string_strdupz("[x]");
- }
-
- if(unlikely(a == b)) return string_dup(a);
- if(unlikely(a == X)) return string_dup(a);
- if(unlikely(b == X)) return string_dup(b);
- if(unlikely(!a)) return string_dup(X);
- if(unlikely(!b)) return string_dup(X);
-
- size_t alen = string_strlen(a);
- size_t blen = string_strlen(b);
- size_t length = alen + blen + string_strlen(X) + 1;
- char buf1[length + 1], buf2[length + 1], *dst1;
- const char *s1, *s2;
-
- s1 = string2str(a);
- s2 = string2str(b);
- dst1 = buf1;
- for( ; *s1 && *s2 && *s1 == *s2 ;s1++, s2++)
- *dst1++ = *s1;
-
- *dst1 = '\0';
-
- if(*s1 != '\0' || *s2 != '\0') {
- *dst1++ = '[';
- *dst1++ = 'x';
- *dst1++ = ']';
-
- s1 = &(string2str(a))[alen - 1];
- s2 = &(string2str(b))[blen - 1];
- char *dst2 = &buf2[length];
- *dst2 = '\0';
- for (; *s1 && *s2 && *s1 == *s2; s1--, s2--)
- *(--dst2) = *s1;
-
- strcpy(dst1, dst2);
- }
-
- return string_strdupz(buf1);
-}
-
-// ----------------------------------------------------------------------------
-// STRING unit test
-
-struct thread_unittest {
- int join;
- int dups;
-};
-
-static void *string_thread(void *arg) {
- struct thread_unittest *tu = arg;
-
- for(; 1 ;) {
- if(__atomic_load_n(&tu->join, __ATOMIC_RELAXED))
- break;
-
- STRING *s = string_strdupz("string thread checking 1234567890");
-
- for(int i = 0; i < tu->dups ; i++)
- string_dup(s);
-
- for(int i = 0; i < tu->dups ; i++)
- string_freez(s);
-
- string_freez(s);
- }
-
- return arg;
-}
-
-static char **string_unittest_generate_names(size_t entries) {
- char **names = mallocz(sizeof(char *) * entries);
- for(size_t i = 0; i < entries ;i++) {
- char buf[25 + 1] = "";
- snprintfz(buf, sizeof(buf) - 1, "name.%zu.0123456789.%zu \t !@#$%%^&*(),./[]{}\\|~`", i, entries / 2 + i);
- names[i] = strdupz(buf);
- }
- return names;
-}
-
-static void string_unittest_free_char_pp(char **pp, size_t entries) {
- for(size_t i = 0; i < entries ;i++)
- freez(pp[i]);
-
- freez(pp);
-}
-
-static long unittest_string_entries(void) {
- long entries = 0;
- for(size_t p = 0; p < STRING_PARTITIONS ;p++)
- entries += string_base[p].entries;
-
- return entries;
-}
-
-#ifdef NETDATA_INTERNAL_CHECKS
-
-static size_t unittest_string_found_deleted_on_search(void) {
- size_t entries = 0;
- for(size_t p = 0; p < STRING_PARTITIONS ;p++)
- entries += string_base[p].found_deleted_on_search;
-
- return entries;
-}
-static size_t unittest_string_found_available_on_search(void) {
- size_t entries = 0;
- for(size_t p = 0; p < STRING_PARTITIONS ;p++)
- entries += string_base[p].found_available_on_search;
-
- return entries;
-}
-static size_t unittest_string_found_deleted_on_insert(void) {
- size_t entries = 0;
- for(size_t p = 0; p < STRING_PARTITIONS ;p++)
- entries += string_base[p].found_deleted_on_insert;
-
- return entries;
-}
-static size_t unittest_string_found_available_on_insert(void) {
- size_t entries = 0;
- for(size_t p = 0; p < STRING_PARTITIONS ;p++)
- entries += string_base[p].found_available_on_insert;
-
- return entries;
-}
-static size_t unittest_string_spins(void) {
- size_t entries = 0;
- for(size_t p = 0; p < STRING_PARTITIONS ;p++)
- entries += string_base[p].spins;
-
- return entries;
-}
-
-#endif // NETDATA_INTERNAL_CHECKS
-
-int string_unittest(size_t entries) {
- size_t errors = 0;
-
- fprintf(stderr, "Generating %zu names and values...\n", entries);
- char **names = string_unittest_generate_names(entries);
-
- // check string
- {
- long entries_starting = unittest_string_entries();
-
- fprintf(stderr, "\nChecking strings...\n");
-
- STRING *s1 = string_strdupz("hello unittest");
- STRING *s2 = string_strdupz("hello unittest");
- if(s1 != s2) {
- errors++;
- fprintf(stderr, "ERROR: duplicating strings are not deduplicated\n");
- }
- else
- fprintf(stderr, "OK: duplicating string are deduplicated\n");
-
- STRING *s3 = string_dup(s1);
- if(s3 != s1) {
- errors++;
- fprintf(stderr, "ERROR: cloning strings are not deduplicated\n");
- }
- else
- fprintf(stderr, "OK: cloning string are deduplicated\n");
-
- if(s1->refcount != 3) {
- errors++;
- fprintf(stderr, "ERROR: string refcount is not 3\n");
- }
- else
- fprintf(stderr, "OK: string refcount is 3\n");
-
- STRING *s4 = string_strdupz("world unittest");
- if(s4 == s1) {
- errors++;
- fprintf(stderr, "ERROR: string is sharing pointers on different strings\n");
- }
- else
- fprintf(stderr, "OK: string is properly handling different strings\n");
-
- usec_t start_ut, end_ut;
- STRING **strings = mallocz(entries * sizeof(STRING *));
-
- start_ut = now_realtime_usec();
- for(size_t i = 0; i < entries ;i++) {
- strings[i] = string_strdupz(names[i]);
- }
- end_ut = now_realtime_usec();
- fprintf(stderr, "Created %zu strings in %"PRIu64" usecs\n", entries, end_ut - start_ut);
-
- start_ut = now_realtime_usec();
- for(size_t i = 0; i < entries ;i++) {
- strings[i] = string_dup(strings[i]);
- }
- end_ut = now_realtime_usec();
- fprintf(stderr, "Cloned %zu strings in %"PRIu64" usecs\n", entries, end_ut - start_ut);
-
- start_ut = now_realtime_usec();
- for(size_t i = 0; i < entries ;i++) {
- strings[i] = string_strdupz(string2str(strings[i]));
- }
- end_ut = now_realtime_usec();
- fprintf(stderr, "Found %zu existing strings in %"PRIu64" usecs\n", entries, end_ut - start_ut);
-
- start_ut = now_realtime_usec();
- for(size_t i = 0; i < entries ;i++) {
- string_freez(strings[i]);
- }
- end_ut = now_realtime_usec();
- fprintf(stderr, "Released %zu referenced strings in %"PRIu64" usecs\n", entries, end_ut - start_ut);
-
- start_ut = now_realtime_usec();
- for(size_t i = 0; i < entries ;i++) {
- string_freez(strings[i]);
- }
- end_ut = now_realtime_usec();
- fprintf(stderr, "Released (again) %zu referenced strings in %"PRIu64" usecs\n", entries, end_ut - start_ut);
-
- start_ut = now_realtime_usec();
- for(size_t i = 0; i < entries ;i++) {
- string_freez(strings[i]);
- }
- end_ut = now_realtime_usec();
- fprintf(stderr, "Freed %zu strings in %"PRIu64" usecs\n", entries, end_ut - start_ut);
-
- freez(strings);
-
- if(unittest_string_entries() != entries_starting + 2) {
- errors++;
- fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n",
- entries_starting + 2, unittest_string_entries());
- }
- else
- fprintf(stderr, "OK: strings dictionary has 2 items\n");
- }
-
- // check 2-way merge
- {
- struct testcase {
- char *src1; char *src2; char *expected;
- } tests[] = {
- { "", "", ""},
- { "a", "", "[x]"},
- { "", "a", "[x]"},
- { "a", "a", "a"},
- { "abcd", "abcd", "abcd"},
- { "foo_cs", "bar_cs", "[x]_cs"},
- { "cp_UNIQUE_INFIX_cs", "cp_unique_infix_cs", "cp_[x]_cs"},
- { "cp_UNIQUE_INFIX_ci_unique_infix_cs", "cp_unique_infix_ci_UNIQUE_INFIX_cs", "cp_[x]_cs"},
- { "foo[1234]", "foo[4321]", "foo[[x]]"},
- { NULL, NULL, NULL },
- };
-
- for (struct testcase *tc = &tests[0]; tc->expected != NULL; tc++) {
- STRING *src1 = string_strdupz(tc->src1);
- STRING *src2 = string_strdupz(tc->src2);
- STRING *expected = string_strdupz(tc->expected);
-
- STRING *result = string_2way_merge(src1, src2);
- if (string_cmp(result, expected) != 0) {
- fprintf(stderr, "string_2way_merge(\"%s\", \"%s\") -> \"%s\" (expected=\"%s\")\n",
- string2str(src1),
- string2str(src2),
- string2str(result),
- string2str(expected));
- errors++;
- }
-
- string_freez(src1);
- string_freez(src2);
- string_freez(expected);
- string_freez(result);
- }
- }
-
- // threads testing of string
- {
- struct thread_unittest tu = {
- .dups = 1,
- .join = 0,
- };
-
-#ifdef NETDATA_INTERNAL_CHECKS
- size_t ofound_deleted_on_search = unittest_string_found_deleted_on_search(),
- ofound_available_on_search = unittest_string_found_available_on_search(),
- ofound_deleted_on_insert = unittest_string_found_deleted_on_insert(),
- ofound_available_on_insert = unittest_string_found_available_on_insert(),
- ospins = unittest_string_spins();
-#endif
-
- size_t oinserts, odeletes, osearches, oentries, oreferences, omemory, oduplications, oreleases;
- string_statistics(&oinserts, &odeletes, &osearches, &oentries, &oreferences, &omemory, &oduplications, &oreleases);
-
- time_t seconds_to_run = 5;
- int threads_to_create = 2;
- fprintf(
- stderr,
- "Checking string concurrency with %d threads for %lld seconds...\n",
- threads_to_create,
- (long long)seconds_to_run);
- // check string concurrency
- netdata_thread_t threads[threads_to_create];
- tu.join = 0;
- for (int i = 0; i < threads_to_create; i++) {
- char buf[100 + 1];
- snprintf(buf, 100, "string%d", i);
- netdata_thread_create(
- &threads[i], buf, NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, string_thread, &tu);
- }
- sleep_usec(seconds_to_run * USEC_PER_SEC);
-
- __atomic_store_n(&tu.join, 1, __ATOMIC_RELAXED);
- for (int i = 0; i < threads_to_create; i++) {
- void *retval;
- netdata_thread_join(threads[i], &retval);
- }
-
- size_t inserts, deletes, searches, sentries, references, memory, duplications, releases;
- string_statistics(&inserts, &deletes, &searches, &sentries, &references, &memory, &duplications, &releases);
-
- fprintf(stderr, "inserts %zu, deletes %zu, searches %zu, entries %zu, references %zu, memory %zu, duplications %zu, releases %zu\n",
- inserts - oinserts, deletes - odeletes, searches - osearches, sentries - oentries, references - oreferences, memory - omemory, duplications - oduplications, releases - oreleases);
-
-#ifdef NETDATA_INTERNAL_CHECKS
- size_t found_deleted_on_search = unittest_string_found_deleted_on_search(),
- found_available_on_search = unittest_string_found_available_on_search(),
- found_deleted_on_insert = unittest_string_found_deleted_on_insert(),
- found_available_on_insert = unittest_string_found_available_on_insert(),
- spins = unittest_string_spins();
-
- fprintf(stderr, "on insert: %zu ok + %zu deleted\non search: %zu ok + %zu deleted\nspins: %zu\n",
- found_available_on_insert - ofound_available_on_insert,
- found_deleted_on_insert - ofound_deleted_on_insert,
- found_available_on_search - ofound_available_on_search,
- found_deleted_on_search - ofound_deleted_on_search,
- spins - ospins
- );
-#endif
- }
-
- string_unittest_free_char_pp(names, entries);
-
- fprintf(stderr, "\n%zu errors found\n", errors);
- return errors ? 1 : 0;
-}
diff --git a/libnetdata/string/string.h b/libnetdata/string/string.h
deleted file mode 100644
index ba0e3876b..000000000
--- a/libnetdata/string/string.h
+++ /dev/null
@@ -1,34 +0,0 @@
-
-#ifndef NETDATA_STRING_H
-#define NETDATA_STRING_H 1
-
-#include "../libnetdata.h"
-
-// ----------------------------------------------------------------------------
-// STRING implementation
-
-typedef struct netdata_string STRING;
-STRING *string_strdupz(const char *str);
-STRING *string_dup(STRING *string);
-void string_freez(STRING *string);
-size_t string_strlen(STRING *string);
-const char *string2str(STRING *string) NEVERNULL;
-
-// keep common prefix/suffix and replace everything else with [x]
-STRING *string_2way_merge(STRING *a, STRING *b);
-
-static inline int string_cmp(STRING *s1, STRING *s2) {
- // STRINGs are deduplicated, so the same strings have the same pointer
- // when they differ, we do the typical strcmp() comparison
- return (s1 == s2)?0:strcmp(string2str(s1), string2str(s2));
-}
-
-static inline int string_strcmp(STRING *string, const char *s) {
- return strcmp(string2str(string), s);
-}
-
-void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases);
-
-int string_unittest(size_t entries);
-
-#endif
diff --git a/libnetdata/tests/Makefile.am b/libnetdata/tests/Makefile.am
deleted file mode 100644
index babdcf0df..000000000
--- a/libnetdata/tests/Makefile.am
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
diff --git a/libnetdata/threads/Makefile.am b/libnetdata/threads/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/threads/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/threads/README.md b/libnetdata/threads/README.md
deleted file mode 100644
index 7e9e493f4..000000000
--- a/libnetdata/threads/README.md
+++ /dev/null
@@ -1,12 +0,0 @@
-<!--
-title: Threads
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/threads/README.md
-sidebar_label: "Threads"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Threads
-
-Netdata uses a custom threads library
diff --git a/libnetdata/threads/threads.c b/libnetdata/threads/threads.c
deleted file mode 100644
index c14f9a7eb..000000000
--- a/libnetdata/threads/threads.c
+++ /dev/null
@@ -1,340 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "../libnetdata.h"
-
-static pthread_attr_t *netdata_threads_attr = NULL;
-
-// ----------------------------------------------------------------------------
-// per thread data
-
-typedef struct {
- void *arg;
- char tag[NETDATA_THREAD_NAME_MAX + 1];
- void *(*start_routine) (void *);
- NETDATA_THREAD_OPTIONS options;
-} NETDATA_THREAD;
-
-static __thread NETDATA_THREAD *netdata_thread = NULL;
-
-inline int netdata_thread_tag_exists(void) {
- return (netdata_thread && *netdata_thread->tag);
-}
-
-static const char *thread_name_get(bool recheck) {
- static __thread char threadname[NETDATA_THREAD_NAME_MAX + 1] = "";
-
- if(netdata_thread_tag_exists())
- strncpyz(threadname, netdata_thread->tag, NETDATA_THREAD_NAME_MAX);
- else {
- if(!recheck && threadname[0])
- return threadname;
-
-#if defined(__FreeBSD__)
- pthread_get_name_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1);
- if(strcmp(threadname, "netdata") == 0)
- strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX);
-#elif defined(__APPLE__)
- strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX);
-#elif defined(HAVE_PTHREAD_GETNAME_NP)
- pthread_getname_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1);
- if(strcmp(threadname, "netdata") == 0)
- strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX);
-#else
- strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX);
-#endif
- }
-
- return threadname;
-}
-
-const char *netdata_thread_tag(void) {
- return thread_name_get(false);
-}
-
-static size_t webrtc_id = 0;
-static __thread bool webrtc_name_set = false;
-void webrtc_set_thread_name(void) {
- if(!netdata_thread && !webrtc_name_set) {
- webrtc_name_set = true;
- char threadname[NETDATA_THREAD_NAME_MAX + 1];
-
-#if defined(__FreeBSD__)
- snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED));
- pthread_set_name_np(pthread_self(), threadname);
-#elif defined(__APPLE__)
- snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED));
- pthread_setname_np(threadname);
-#elif defined(HAVE_PTHREAD_GETNAME_NP)
- pthread_getname_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX+1);
- if(strcmp(threadname, "netdata") == 0) {
- snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED));
- pthread_setname_np(pthread_self(), threadname);
- }
-#else
- snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED));
- pthread_setname_np(pthread_self(), threadname);
-#endif
-
- thread_name_get(true);
- }
-}
-
-// ----------------------------------------------------------------------------
-// compatibility library functions
-
-static __thread pid_t gettid_cached_tid = 0;
-pid_t gettid(void) {
- pid_t tid = 0;
-
- if(likely(gettid_cached_tid > 0))
- return gettid_cached_tid;
-
-#ifdef __FreeBSD__
-
- tid = (pid_t)pthread_getthreadid_np();
-
-#elif defined(__APPLE__)
-
- #if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060)
- uint64_t curthreadid;
- pthread_threadid_np(NULL, &curthreadid);
- tid = (pid_t)curthreadid;
- #else /* __MAC_OS_X_VERSION_MIN_REQUIRED */
- tid = (pid_t)pthread_self;
- #endif /* __MAC_OS_X_VERSION_MIN_REQUIRED */
-
-#else /* __APPLE__*/
-
- tid = (pid_t)syscall(SYS_gettid);
-
-#endif /* __FreeBSD__, __APPLE__*/
-
- gettid_cached_tid = tid;
- return tid;
-}
-
-// ----------------------------------------------------------------------------
-// early initialization
-
-size_t netdata_threads_init(void) {
- int i;
-
- // --------------------------------------------------------------------
- // get the required stack size of the threads of netdata
-
- if(!netdata_threads_attr) {
- netdata_threads_attr = callocz(1, sizeof(pthread_attr_t));
- i = pthread_attr_init(netdata_threads_attr);
- if (i != 0)
- fatal("pthread_attr_init() failed with code %d.", i);
- }
-
- size_t stacksize = 0;
- i = pthread_attr_getstacksize(netdata_threads_attr, &stacksize);
- if(i != 0)
- fatal("pthread_attr_getstacksize() failed with code %d.", i);
-
- return stacksize;
-}
-
-// ----------------------------------------------------------------------------
-// late initialization
-
-void netdata_threads_init_after_fork(size_t stacksize) {
- int i;
-
- // ------------------------------------------------------------------------
- // set pthread stack size
-
- if(netdata_threads_attr && stacksize > (size_t)PTHREAD_STACK_MIN) {
- i = pthread_attr_setstacksize(netdata_threads_attr, stacksize);
- if(i != 0)
- nd_log(NDLS_DAEMON, NDLP_WARNING, "pthread_attr_setstacksize() to %zu bytes, failed with code %d.", stacksize, i);
- else
- nd_log(NDLS_DAEMON, NDLP_DEBUG, "Set threads stack size to %zu bytes", stacksize);
- }
- else
- nd_log(NDLS_DAEMON, NDLP_WARNING, "Invalid pthread stacksize %zu", stacksize);
-}
-
-// ----------------------------------------------------------------------------
-// threads init for external plugins
-
-void netdata_threads_init_for_external_plugins(size_t stacksize) {
- size_t default_stacksize = netdata_threads_init();
- if(default_stacksize < 1 * 1024 * 1024)
- default_stacksize = 1 * 1024 * 1024;
-
- netdata_threads_init_after_fork(stacksize ? stacksize : default_stacksize);
-}
-
-// ----------------------------------------------------------------------------
-// netdata_thread_create
-
-void rrdset_thread_rda_free(void);
-void sender_thread_buffer_free(void);
-void query_target_free(void);
-void service_exits(void);
-void rrd_collector_finished(void);
-
-static void thread_cleanup(void *ptr) {
- if(netdata_thread != ptr) {
- NETDATA_THREAD *info = (NETDATA_THREAD *)ptr;
- nd_log(NDLS_DAEMON, NDLP_ERR, "THREADS: internal error - thread local variable does not match the one passed to this function. Expected thread '%s', passed thread '%s'", netdata_thread->tag, info->tag);
- }
-
- if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP))
- nd_log(NDLS_DAEMON, NDLP_DEBUG, "thread with task id %d finished", gettid());
-
- rrd_collector_finished();
- sender_thread_buffer_free();
- rrdset_thread_rda_free();
- query_target_free();
- thread_cache_destroy();
- service_exits();
- worker_unregister();
-
- netdata_thread->tag[0] = '\0';
-
- freez(netdata_thread);
- netdata_thread = NULL;
-}
-
-void netdata_thread_set_tag(const char *tag) {
- if(!tag || !*tag)
- return;
-
- int ret = 0;
-
- char threadname[NETDATA_THREAD_NAME_MAX+1];
- strncpyz(threadname, tag, NETDATA_THREAD_NAME_MAX);
-
-#if defined(__FreeBSD__)
- pthread_set_name_np(pthread_self(), threadname);
-#elif defined(__APPLE__)
- ret = pthread_setname_np(threadname);
-#else
- ret = pthread_setname_np(pthread_self(), threadname);
-#endif
-
- if (ret != 0)
- nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot set pthread name of %d to %s. ErrCode: %d", gettid(), threadname, ret);
- else
- nd_log(NDLS_DAEMON, NDLP_DEBUG, "set name of thread %d to %s", gettid(), threadname);
-
- if(netdata_thread) {
- strncpyz(netdata_thread->tag, threadname, sizeof(netdata_thread->tag) - 1);
- }
-}
-
-void uv_thread_set_name_np(uv_thread_t ut, const char* name) {
- int ret = 0;
-
- char threadname[NETDATA_THREAD_NAME_MAX+1];
- strncpyz(threadname, name, NETDATA_THREAD_NAME_MAX);
-
-#if defined(__FreeBSD__)
- pthread_set_name_np(ut ? ut : pthread_self(), threadname);
-#elif defined(__APPLE__)
- // Apple can only set its own name
- UNUSED(ut);
-#else
- ret = pthread_setname_np(ut ? ut : pthread_self(), threadname);
-#endif
-
- thread_name_get(true);
-
- if (ret)
- nd_log(NDLS_DAEMON, NDLP_NOTICE, "cannot set libuv thread name to %s. Err: %d", threadname, ret);
-}
-
-void os_thread_get_current_name_np(char threadname[NETDATA_THREAD_NAME_MAX + 1])
-{
- threadname[0] = '\0';
-#if defined(__FreeBSD__)
- pthread_get_name_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1);
-#elif defined(HAVE_PTHREAD_GETNAME_NP) /* Linux & macOS */
- (void)pthread_getname_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1);
-#endif
-}
-
-static void *netdata_thread_init(void *ptr) {
- netdata_thread = (NETDATA_THREAD *)ptr;
-
- if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_STARTUP))
- nd_log(NDLS_DAEMON, NDLP_DEBUG, "thread created with task id %d", gettid());
-
- if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
- nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot set pthread cancel type to DEFERRED.");
-
- if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
- nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot set pthread cancel state to ENABLE.");
-
- netdata_thread_set_tag(netdata_thread->tag);
-
- if (!(netdata_thread->options & NETDATA_THREAD_OPTION_JOINABLE)) {
- int rc = pthread_detach(pthread_self());
- if (rc != 0)
- nd_log(NDLS_DAEMON, NDLP_WARNING,
- "cannot request detach of newly created %s thread. pthread_detach() failed with code %d",
- netdata_thread->tag, rc);
- }
-
- void *ret = NULL;
- pthread_cleanup_push(thread_cleanup, ptr);
- ret = netdata_thread->start_routine(netdata_thread->arg);
- pthread_cleanup_pop(1);
-
- return ret;
-}
-
-int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg) {
- NETDATA_THREAD *info = callocz(1, sizeof(NETDATA_THREAD));
- info->arg = arg;
- info->start_routine = start_routine;
- info->options = options;
- strncpyz(info->tag, tag, NETDATA_THREAD_NAME_MAX);
-
- int ret = pthread_create(thread, netdata_threads_attr, netdata_thread_init, info);
- if(ret != 0)
- nd_log(NDLS_DAEMON, NDLP_ERR, "failed to create new thread for %s. pthread_create() failed with code %d", tag, ret);
-
- return ret;
-}
-
-// ----------------------------------------------------------------------------
-// netdata_thread_cancel
-#ifdef NETDATA_INTERNAL_CHECKS
-int netdata_thread_cancel_with_trace(netdata_thread_t thread, int line, const char *file, const char *function) {
-#else
-int netdata_thread_cancel(netdata_thread_t thread) {
-#endif
- int ret = pthread_cancel(thread);
- if(ret != 0)
-#ifdef NETDATA_INTERNAL_CHECKS
- nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot cancel thread. pthread_cancel() failed with code %d at %d@%s, function %s()", ret, line, file, function);
-#else
- nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot cancel thread. pthread_cancel() failed with code %d.", ret);
-#endif
-
- return ret;
-}
-
-// ----------------------------------------------------------------------------
-// netdata_thread_join
-
-int netdata_thread_join(netdata_thread_t thread, void **retval) {
- int ret = pthread_join(thread, retval);
- if(ret != 0)
- nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot join thread. pthread_join() failed with code %d.", ret);
-
- return ret;
-}
-
-int netdata_thread_detach(pthread_t thread) {
- int ret = pthread_detach(thread);
- if(ret != 0)
- nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot detach thread. pthread_detach() failed with code %d.", ret);
-
- return ret;
-}
diff --git a/libnetdata/threads/threads.h b/libnetdata/threads/threads.h
deleted file mode 100644
index 97c3c8036..000000000
--- a/libnetdata/threads/threads.h
+++ /dev/null
@@ -1,87 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_THREADS_H
-#define NETDATA_THREADS_H 1
-
-#include "../libnetdata.h"
-
-pid_t gettid(void);
-
-typedef enum {
- NETDATA_THREAD_OPTION_DEFAULT = 0 << 0,
- NETDATA_THREAD_OPTION_JOINABLE = 1 << 0,
- NETDATA_THREAD_OPTION_DONT_LOG_STARTUP = 1 << 1,
- NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP = 1 << 2,
- NETDATA_THREAD_OPTION_DONT_LOG = NETDATA_THREAD_OPTION_DONT_LOG_STARTUP|NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP,
-} NETDATA_THREAD_OPTIONS;
-
-#define netdata_thread_cleanup_push(func, arg) pthread_cleanup_push(func, arg)
-#define netdata_thread_cleanup_pop(execute) pthread_cleanup_pop(execute)
-
-void netdata_thread_set_tag(const char *tag);
-
-typedef pthread_t netdata_thread_t;
-
-struct netdata_static_thread {
- // the name of the thread as it should appear in the logs
- char *name;
-
- // the section of netdata.conf to check if this is enabled or not
- char *config_section;
-
- // the name of the config option to check if it is true or false
- char *config_name;
-
- // the current status of the thread
- volatile sig_atomic_t enabled;
-
- // internal use, to maintain a pointer to the created thread
- netdata_thread_t *thread;
-
- // an initialization function to run before spawning the thread
- void (*init_routine) (void);
-
- // the threaded worker
- void *(*start_routine) (void *);
-
- // the environment variable to create
- char *env_name;
-
- // global variable
- bool *global_variable;
-};
-
-#define NETDATA_MAIN_THREAD_RUNNING CONFIG_BOOLEAN_YES
-#define NETDATA_MAIN_THREAD_EXITING (CONFIG_BOOLEAN_YES + 1)
-#define NETDATA_MAIN_THREAD_EXITED CONFIG_BOOLEAN_NO
-
-#define NETDATA_THREAD_TAG_MAX 100
-const char *netdata_thread_tag(void);
-int netdata_thread_tag_exists(void);
-
-size_t netdata_threads_init(void);
-void netdata_threads_init_after_fork(size_t stacksize);
-void netdata_threads_init_for_external_plugins(size_t stacksize);
-
-int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg);
-
-#ifdef NETDATA_INTERNAL_CHECKS
-#define netdata_thread_cancel(thread) netdata_thread_cancel_with_trace(thread, __LINE__, __FILE__, __FUNCTION__)
-int netdata_thread_cancel_with_trace(netdata_thread_t thread, int line, const char *file, const char *function);
-#else
-int netdata_thread_cancel(netdata_thread_t thread);
-#endif
-
-int netdata_thread_join(netdata_thread_t thread, void **retval);
-int netdata_thread_detach(pthread_t thread);
-
-#define NETDATA_THREAD_NAME_MAX 15
-void uv_thread_set_name_np(uv_thread_t ut, const char* name);
-void os_thread_get_current_name_np(char threadname[NETDATA_THREAD_NAME_MAX + 1]);
-
-void webrtc_set_thread_name(void);
-
-#define netdata_thread_self pthread_self
-#define netdata_thread_testcancel pthread_testcancel
-
-#endif //NETDATA_THREADS_H
diff --git a/libnetdata/url/Makefile.am b/libnetdata/url/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/url/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/url/README.md b/libnetdata/url/README.md
deleted file mode 100644
index 1ba51e2ea..000000000
--- a/libnetdata/url/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-<!--
-title: "URL"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/url/README.md
-sidebar_label: "URL"
-learn_status: "Published"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# URL
-
-The URL library contains common functions useful for URLs, like conversion from/to hex,
-URL encode/decode and query string parsing.
-
diff --git a/libnetdata/url/url.c b/libnetdata/url/url.c
deleted file mode 100644
index 39366cbe8..000000000
--- a/libnetdata/url/url.c
+++ /dev/null
@@ -1,299 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "../libnetdata.h"
-
-// ----------------------------------------------------------------------------
-// URL encode / decode
-// code from: http://www.geekhideout.com/urlcode.shtml
-
-/* Converts a hex character to its integer value */
-char from_hex(char ch) {
- return (char)(isdigit(ch) ? ch - '0' : tolower(ch) - 'a' + 10);
-}
-
-/* Converts an integer value to its hex character*/
-char to_hex(char code) {
- static char hex[] = "0123456789abcdef";
- return hex[code & 15];
-}
-
-/* Returns an url-encoded version of str */
-/* IMPORTANT: be sure to free() the returned string after use */
-char *url_encode(char *str) {
- char *buf, *pbuf;
-
- pbuf = buf = mallocz(strlen(str) * 3 + 1);
-
- while (*str) {
- if (isalnum(*str) || *str == '-' || *str == '_' || *str == '.' || *str == '~')
- *pbuf++ = *str;
-
- else if (*str == ' ')
- *pbuf++ = '+';
-
- else{
- *pbuf++ = '%';
- *pbuf++ = to_hex((char)(*str >> 4));
- *pbuf++ = to_hex((char)(*str & 15));
- }
-
- str++;
- }
- *pbuf = '\0';
-
- pbuf = strdupz(buf);
- freez(buf);
- return pbuf;
-}
-
-/**
- * Percentage escape decode
- *
- * Decode %XX character or return 0 if cannot
- *
- * @param s the string to decode
- *
- * @return The character decoded on success and 0 otherwise
- */
-char url_percent_escape_decode(const char *s) {
- if(likely(s[1] && s[2]))
- return (char)(from_hex(s[1]) << 4 | from_hex(s[2]));
- return 0;
-}
-
-/**
- * Get byte length
- *
- * This (utf8 string related) should be moved in separate file in future
- *
- * @param c is the utf8 character
- * *
- * @return It returns the length of the specific character.
- */
-char url_utf8_get_byte_length(char c) {
- if(!IS_UTF8_BYTE(c))
- return 1;
-
- char length = 0;
- while(likely(c & 0x80)) {
- length++;
- c <<= 1;
- }
- //4 byte is max size for UTF-8 char
- //10XX XXXX is not valid character -> check length == 1
- if(length > 4 || length == 1)
- return -1;
-
- return length;
-}
-
-/**
- * Decode Multibyte UTF8
- *
- * Decode % encoded UTF-8 characters and copy them to *d
- *
- * @param s first address
- * @param d
- * @param d_end last address
- *
- * @return count of bytes written to *d
- */
-char url_decode_multibyte_utf8(const char *s, char *d, const char *d_end) {
- char first_byte = url_percent_escape_decode(s);
-
- if(unlikely(!first_byte || !IS_UTF8_STARTBYTE(first_byte)))
- return 0;
-
- char byte_length = url_utf8_get_byte_length(first_byte);
-
- if(unlikely(byte_length <= 0 || d+byte_length >= d_end))
- return 0;
-
- char to_read = byte_length;
- while(to_read > 0) {
- char c = url_percent_escape_decode(s);
-
- if(unlikely( !IS_UTF8_BYTE(c) ))
- return 0;
- if((to_read != byte_length) && IS_UTF8_STARTBYTE(c))
- return 0;
-
- *d++ = c;
- s+=3;
- to_read--;
- }
-
- return byte_length;
-}
-
-/*
- * The utf8_check() function scans the '\0'-terminated string starting
- * at s. It returns a pointer to the first byte of the first malformed
- * or overlong UTF-8 sequence found, or NULL if the string contains
- * only correct UTF-8. It also spots UTF-8 sequences that could cause
- * trouble if converted to UTF-16, namely surrogate characters
- * (U+D800..U+DFFF) and non-Unicode positions (U+FFFE..U+FFFF). This
- * routine is very likely to find a malformed sequence if the input
- * uses any other encoding than UTF-8. It therefore can be used as a
- * very effective heuristic for distinguishing between UTF-8 and other
- * encodings.
- *
- * Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> -- 2005-03-30
- * License: http://www.cl.cam.ac.uk/~mgk25/short-license.html
- */
-unsigned char *utf8_check(unsigned char *s)
-{
- while (*s)
- {
- if (*s < 0x80)
- /* 0xxxxxxx */
- s++;
- else if ((s[0] & 0xe0) == 0xc0)
- {
- /* 110XXXXx 10xxxxxx */
- if ((s[1] & 0xc0) != 0x80 ||
- (s[0] & 0xfe) == 0xc0) /* overlong? */
- return s;
- else
- s += 2;
- }
- else if ((s[0] & 0xf0) == 0xe0)
- {
- /* 1110XXXX 10Xxxxxx 10xxxxxx */
- if ((s[1] & 0xc0) != 0x80 ||
- (s[2] & 0xc0) != 0x80 ||
- (s[0] == 0xe0 && (s[1] & 0xe0) == 0x80) || /* overlong? */
- (s[0] == 0xed && (s[1] & 0xe0) == 0xa0) || /* surrogate? */
- (s[0] == 0xef && s[1] == 0xbf &&
- (s[2] & 0xfe) == 0xbe)) /* U+FFFE or U+FFFF? */
- return s;
- else
- s += 3;
- }
- else if ((s[0] & 0xf8) == 0xf0)
- {
- /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */
- if ((s[1] & 0xc0) != 0x80 ||
- (s[2] & 0xc0) != 0x80 ||
- (s[3] & 0xc0) != 0x80 ||
- (s[0] == 0xf0 && (s[1] & 0xf0) == 0x80) || /* overlong? */
- (s[0] == 0xf4 && s[1] > 0x8f) || s[0] > 0xf4) /* > U+10FFFF? */
- return s;
- else
- s += 4;
- }
- else
- return s;
- }
-
- return NULL;
-}
-
-char *url_decode_r(char *to, const char *url, size_t size) {
- const char *s = url; // source
- char *d = to, // destination
- *e = &to[size - 1]; // destination end
-
- while(*s && d < e) {
- if(unlikely(*s == '%')) {
- char t = url_percent_escape_decode(s);
- if(IS_UTF8_BYTE(t)) {
- char bytes_written = url_decode_multibyte_utf8(s, d, e);
- if(likely(bytes_written)){
- d += bytes_written;
- s += (bytes_written * 3)-1;
- }
- else {
- goto fail_cleanup;
- }
- }
- else if(likely(t) && isprint(t)) {
- // avoid HTTP header injection
- *d++ = t;
- s += 2;
- }
- else
- goto fail_cleanup;
- }
- else if(unlikely(*s == '+'))
- *d++ = ' ';
-
- else
- *d++ = *s;
-
- s++;
- }
-
- *d = '\0';
-
- if(unlikely( utf8_check((unsigned char *)to) )) //NULL means success here
- return NULL;
-
- return to;
-
-fail_cleanup:
- *d = '\0';
- return NULL;
-}
-
-inline bool url_is_request_complete(char *begin, char *end, size_t length, char **post_payload, size_t *post_payload_size) {
- if (begin == end || length < 4)
- return false;
-
- if(likely(strncmp(begin, "GET ", 4)) == 0) {
- return strstr(end - 4, "\r\n\r\n");
- }
- else if(unlikely(strncmp(begin, "POST ", 5) == 0 || strncmp(begin, "PUT ", 4) == 0)) {
- char *cl = strstr(begin, "Content-Length: ");
- if(!cl) return false;
- cl = &cl[16];
-
- size_t content_length = str2ul(cl);
-
- char *payload = strstr(cl, "\r\n\r\n");
- if(!payload) return false;
- payload += 4;
-
- size_t payload_length = length - (payload - begin);
-
- if(payload_length == content_length) {
- if(post_payload && post_payload_size) {
- if (*post_payload)
- freez(*post_payload);
-
- *post_payload = mallocz(payload_length + 1);
- memcpy(*post_payload, payload, payload_length);
- (*post_payload)[payload_length] = '\0';
-
- *post_payload_size = payload_length;
- }
- return true;
- }
-
- return false;
- }
- else {
- return strstr(end - 4, "\r\n\r\n");
- }
-}
-
-/**
- * Find protocol
- *
- * Search for the string ' HTTP/' in the message given.
- *
- * @param s is the start of the user request.
- * @return
- */
-inline char *url_find_protocol(char *s) {
- while(*s) {
- // find the next space
- while (*s && *s != ' ') s++;
-
- // is it SPACE + "HTTP/" ?
- if(*s && !strncmp(s, " HTTP/", 6)) break;
- else s++;
- }
-
- return s;
-}
diff --git a/libnetdata/url/url.h b/libnetdata/url/url.h
deleted file mode 100644
index 9db018f0b..000000000
--- a/libnetdata/url/url.h
+++ /dev/null
@@ -1,31 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_URL_H
-#define NETDATA_URL_H 1
-
-#include "../libnetdata.h"
-
-// ----------------------------------------------------------------------------
-// URL encode / decode
-// code from: http://www.geekhideout.com/urlcode.shtml
-
-/* Converts a hex character to its integer value */
-char from_hex(char ch);
-
-/* Converts an integer value to its hex character*/
-char to_hex(char code);
-
-/* Returns a url-encoded version of str */
-/* IMPORTANT: be sure to free() the returned string after use */
-char *url_encode(char *str);
-
-/* Returns a url-decoded version of str */
-/* IMPORTANT: be sure to free() the returned string after use */
-char *url_decode(char *str);
-
-char *url_decode_r(char *to, const char *url, size_t size);
-
-bool url_is_request_complete(char *begin, char *end, size_t length, char **post_payload, size_t *post_payload_length);
-char *url_find_protocol(char *s);
-
-#endif /* NETDATA_URL_H */
diff --git a/libnetdata/uuid/Makefile.am b/libnetdata/uuid/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/uuid/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/uuid/README.md b/libnetdata/uuid/README.md
deleted file mode 100644
index fbadfd63a..000000000
--- a/libnetdata/uuid/README.md
+++ /dev/null
@@ -1,13 +0,0 @@
-<!--
-title: "UUID"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/uuid/README.md
-sidebar_label: "UUID"
-learn_topic_type: "Tasks"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# UUID
-
-Netdata uses libuuid for managing UUIDs.
-
-In this folder are a few custom helpers. \ No newline at end of file
diff --git a/libnetdata/uuid/uuid.c b/libnetdata/uuid/uuid.c
deleted file mode 100644
index 55b66db9b..000000000
--- a/libnetdata/uuid/uuid.c
+++ /dev/null
@@ -1,179 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#include "../libnetdata.h"
-
-void uuid_unparse_lower_compact(const uuid_t uuid, char *out) {
- static const char *hex_chars = "0123456789abcdef";
- for (int i = 0; i < 16; i++) {
- out[i * 2] = hex_chars[(uuid[i] >> 4) & 0x0F];
- out[i * 2 + 1] = hex_chars[uuid[i] & 0x0F];
- }
- out[32] = '\0'; // Null-terminate the string
-}
-
-inline int uuid_parse_compact(const char *in, uuid_t uuid) {
- if (strlen(in) != 32)
- return -1; // Invalid input length
-
- for (int i = 0; i < 16; i++) {
- int high = hex_char_to_int(in[i * 2]);
- int low = hex_char_to_int(in[i * 2 + 1]);
-
- if (high < 0 || low < 0)
- return -1; // Invalid hexadecimal character
-
- uuid[i] = (high << 4) | low;
- }
-
- return 0; // Success
-}
-
-int uuid_parse_flexi(const char *in, uuid_t uu) {
- if(!in || !*in)
- return -1;
-
- size_t hexCharCount = 0;
- size_t hyphenCount = 0;
- const char *s = in;
- int byteIndex = 0;
- uuid_t uuid; // work on a temporary place, to not corrupt the previous value of uu if we fail
-
- while (*s && byteIndex < 16) {
- if (*s == '-') {
- s++;
- hyphenCount++;
-
- if (unlikely(hyphenCount > 4))
- // Too many hyphens
- return -2;
- }
-
- if (likely(isxdigit(*s))) {
- int high = hex_char_to_int(*s++);
- hexCharCount++;
-
- if (likely(isxdigit(*s))) {
- int low = hex_char_to_int(*s++);
- hexCharCount++;
-
- uuid[byteIndex++] = (high << 4) | low;
- }
- else
- // Not a valid UUID (expected a pair of hex digits)
- return -3;
- }
- else
- // Not a valid UUID
- return -4;
- }
-
- if (unlikely(byteIndex < 16))
- // Not enough data to form a UUID
- return -5;
-
- if (unlikely(hexCharCount != 32))
- // wrong number of hex digits
- return -6;
-
- if(unlikely(hyphenCount != 0 && hyphenCount != 4))
- // wrong number of hyphens
- return -7;
-
- // copy the final value
- memcpy(uu, uuid, sizeof(uuid_t));
-
- return 0;
-}
-
-
-// ----------------------------------------------------------------------------
-// unit test
-
-static inline void remove_hyphens(const char *uuid_with_hyphens, char *uuid_without_hyphens) {
- while (*uuid_with_hyphens) {
- if (*uuid_with_hyphens != '-') {
- *uuid_without_hyphens++ = *uuid_with_hyphens;
- }
- uuid_with_hyphens++;
- }
- *uuid_without_hyphens = '\0';
-}
-
-int uuid_unittest(void) {
- const int num_tests = 100000;
- int failed_tests = 0;
-
- int i;
- for (i = 0; i < num_tests; i++) {
- uuid_t original_uuid, parsed_uuid;
- char uuid_str_with_hyphens[UUID_STR_LEN], uuid_str_without_hyphens[UUID_COMPACT_STR_LEN];
-
- // Generate a random UUID
- switch(i % 2) {
- case 0:
- uuid_generate(original_uuid);
- break;
-
- case 1:
- uuid_generate_random(original_uuid);
- break;
- }
-
- // Unparse it with hyphens
- bool lower = false;
- switch(i % 3) {
- case 0:
- uuid_unparse_lower(original_uuid, uuid_str_with_hyphens);
- lower = true;
- break;
-
- case 1:
- uuid_unparse(original_uuid, uuid_str_with_hyphens);
- break;
-
- case 2:
- uuid_unparse_upper(original_uuid, uuid_str_with_hyphens);
- break;
- }
-
- // Remove the hyphens
- remove_hyphens(uuid_str_with_hyphens, uuid_str_without_hyphens);
-
- if(lower) {
- char test[UUID_COMPACT_STR_LEN];
- uuid_unparse_lower_compact(original_uuid, test);
- if(strcmp(test, uuid_str_without_hyphens) != 0) {
- printf("uuid_unparse_lower_compact() failed, expected '%s', got '%s'\n",
- uuid_str_without_hyphens, test);
- failed_tests++;
- }
- }
-
- // Parse the UUID string with hyphens
- int parse_result = uuid_parse_flexi(uuid_str_with_hyphens, parsed_uuid);
- if (parse_result != 0) {
- printf("uuid_parse_flexi() returned -1 (parsing error) for UUID with hyphens: %s\n", uuid_str_with_hyphens);
- failed_tests++;
- } else if (uuid_compare(original_uuid, parsed_uuid) != 0) {
- printf("uuid_parse_flexi() parsed value mismatch for UUID with hyphens: %s\n", uuid_str_with_hyphens);
- failed_tests++;
- }
-
- // Parse the UUID string without hyphens
- parse_result = uuid_parse_flexi(uuid_str_without_hyphens, parsed_uuid);
- if (parse_result != 0) {
- printf("uuid_parse_flexi() returned -1 (parsing error) for UUID without hyphens: %s\n", uuid_str_without_hyphens);
- failed_tests++;
- }
- else if(uuid_compare(original_uuid, parsed_uuid) != 0) {
- printf("uuid_parse_flexi() parsed value mismatch for UUID without hyphens: %s\n", uuid_str_without_hyphens);
- failed_tests++;
- }
-
- if(failed_tests)
- break;
- }
-
- printf("UUID: failed %d out of %d tests.\n", failed_tests, i);
- return failed_tests;
-}
diff --git a/libnetdata/uuid/uuid.h b/libnetdata/uuid/uuid.h
deleted file mode 100644
index 567646846..000000000
--- a/libnetdata/uuid/uuid.h
+++ /dev/null
@@ -1,29 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_UUID_H
-#define NETDATA_UUID_H
-
-UUID_DEFINE(streaming_from_child_msgid, 0xed,0x4c,0xdb, 0x8f, 0x1b, 0xeb, 0x4a, 0xd3, 0xb5, 0x7c, 0xb3, 0xca, 0xe2, 0xd1, 0x62, 0xfa);
-UUID_DEFINE(streaming_to_parent_msgid, 0x6e, 0x2e, 0x38, 0x39, 0x06, 0x76, 0x48, 0x96, 0x8b, 0x64, 0x60, 0x45, 0xdb, 0xf2, 0x8d, 0x66);
-UUID_DEFINE(health_alert_transition_msgid, 0x9c, 0xe0, 0xcb, 0x58, 0xab, 0x8b, 0x44, 0xdf, 0x82, 0xc4, 0xbf, 0x1a, 0xd9, 0xee, 0x22, 0xde);
-
-// this is also defined in alarm-notify.sh.in
-UUID_DEFINE(health_alert_notification_msgid, 0x6d, 0xb0, 0x01, 0x8e, 0x83, 0xe3, 0x43, 0x20, 0xae, 0x2a, 0x65, 0x9d, 0x78, 0x01, 0x9f, 0xb7);
-
-#define UUID_COMPACT_STR_LEN 33
-void uuid_unparse_lower_compact(const uuid_t uuid, char *out);
-int uuid_parse_compact(const char *in, uuid_t uuid);
-int uuid_parse_flexi(const char *in, uuid_t uuid);
-
-static inline int uuid_memcmp(const uuid_t *uu1, const uuid_t *uu2) {
- return memcmp(uu1, uu2, sizeof(uuid_t));
-}
-
-static inline int hex_char_to_int(char c) {
- if (c >= '0' && c <= '9') return c - '0';
- if (c >= 'a' && c <= 'f') return c - 'a' + 10;
- if (c >= 'A' && c <= 'F') return c - 'A' + 10;
- return -1; // Invalid hexadecimal character
-}
-
-#endif //NETDATA_UUID_H
diff --git a/libnetdata/worker_utilization/Makefile.am b/libnetdata/worker_utilization/Makefile.am
deleted file mode 100644
index 161784b8f..000000000
--- a/libnetdata/worker_utilization/Makefile.am
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
- README.md \
- $(NULL)
diff --git a/libnetdata/worker_utilization/README.md b/libnetdata/worker_utilization/README.md
deleted file mode 100644
index 04dfb6215..000000000
--- a/libnetdata/worker_utilization/README.md
+++ /dev/null
@@ -1,94 +0,0 @@
-<!--
-title: "Worker Utilization"
-custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/worker_utilization/README.md
-sidebar_label: "Worker Utilization"
-learn_status: "Published"
-learn_topic_type: "References"
-learn_rel_path: "Developers/libnetdata"
--->
-
-# Worker Utilization
-
-This library is to be used when there are 1 or more worker threads accepting requests
-of some kind and servicing them. The goal is to provide a very simple way to monitor
-worker threads utilization, as a percentage of the time they are busy and the amount
-of requests served.
-
-## Design goals
-
-1. Minimal, if any, impact on the performance of the workers
-2. Easy to be integrated into any kind of worker
-3. No state of any kind at the worker side
-
-## How to use
-
-When a working thread starts, call:
-
-```c
-void worker_register(const char *name);
-```
-
-This will create the necessary structures for the library to work.
-No need to keep a pointer to them. They are allocated as `__thread` variables.
-
-Then job types need to be defined. Job types are anything a worker does that can be
-counted and their execution time needs to be reported. The library is fast enough to
-be integrated even on workers that perform hundreds of thousands of actions per second.
-
-Job types are defined like this:
-
-```c
-void worker_register_job_type(size_t id, const char *name);
-```
-
-`id` is a number starting from zero. The library is compiled with a fixed size of 50
-ids (0 to 49). More can be allocated by setting `WORKER_UTILIZATION_MAX_JOB_TYPES` in
-`worker_utilization.h`. `name` can be any string up to 22 characters. This can be
-changed by setting `WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH` in `worker_utilization.h`.
-
-Each thread that calls `worker_register(name)` will allocate about 3kB for maintaining
-the information required.
-
-When the thread stops, call:
-
-```c
-void worker_unregister(void);
-```
-
-Again, no parameters, or return values.
-
-> IMPORTANT: cancellable threads need to add a call to `worker_unregister()` to the
-> `pop` function that cleans up the thread. Failure to do so, will result in about
-> 3kB of memory leak for every thread that is stopped.
-
-When you are about to do some work in the working thread, call:
-
-```c
-void worker_is_busy(size_t id);
-```
-
-When you finish doing the job, call:
-
-```c
-void worker_is_idle(void);
-```
-
-Calls to `worker_is_busy(id)` can be made one after another (without calling
-`worker_is_idle()` between them) to switch jobs without losing any time between
-them and eliminating one of the 2 clock calls involved.
-
-## Implementation details
-
-Totally lockless, extremely fast, it should not introduce any kind of problems to the
-workers. Every time `worker_is_busy(id)` or `worker_is_idle()` are called, a call to
-`now_realtime_usec()` is done and a couple of variables are updated. That's it!
-
-The worker does not need to update the variables regularly. Based on the last status
-of the worker, the statistics collector of netdata will calculate if the thread is
-busy or idle all the time or part of the time. Works well for both thousands of jobs
-per second and unlimited working time (being totally busy with a single request for
-ages).
-
-The statistics collector is called by the global statistics thread of netdata. So,
-even if the workers are extremely busy with their jobs, netdata will be able to know
-how busy they are.
diff --git a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c b/src/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c
index b645927d4..b645927d4 100644
--- a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c
+++ b/src/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c
diff --git a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h b/src/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h
index bca0ff272..bca0ff272 100644
--- a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h
+++ b/src/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h
diff --git a/libnetdata/aral/aral.c b/src/libnetdata/aral/aral.c
index 7223ee359..64b63d8e0 100644
--- a/libnetdata/aral/aral.c
+++ b/src/libnetdata/aral/aral.c
@@ -464,6 +464,12 @@ static inline ARAL_PAGE *aral_acquire_a_free_slot(ARAL *ar TRACE_ALLOCATIONS_FUN
return page;
}
+void *aral_callocz_internal(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
+ void *r = aral_mallocz_internal(ar TRACE_ALLOCATIONS_FUNCTION_CALL_PARAMS);
+ memset(r, 0, ar->config.requested_element_size);
+ return r;
+}
+
void *aral_mallocz_internal(ARAL *ar TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {
#ifdef FSANITIZE_ADDRESS
return mallocz(ar->config.requested_element_size);
@@ -714,6 +720,7 @@ ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_ele
ar->config.mmap.enabled = mmap;
strncpyz(ar->config.name, name, ARAL_MAX_NAME);
spinlock_init(&ar->aral_lock.spinlock);
+ spinlock_init(&ar->adders.spinlock);
if(stats) {
ar->stats = stats;
@@ -1012,14 +1019,16 @@ int aral_stress_test(size_t threads, size_t elements, size_t seconds) {
};
usec_t started_ut = now_monotonic_usec();
- netdata_thread_t thread_ptrs[threads];
+ ND_THREAD *thread_ptrs[threads];
for(size_t i = 0; i < threads ; i++) {
- char tag[NETDATA_THREAD_NAME_MAX + 1];
- snprintfz(tag, NETDATA_THREAD_NAME_MAX, "TH[%zu]", i);
- netdata_thread_create(&thread_ptrs[i], tag,
- NETDATA_THREAD_OPTION_JOINABLE | NETDATA_THREAD_OPTION_DONT_LOG,
- aral_test_thread, &auc);
+ char tag[ND_THREAD_TAG_MAX + 1];
+ snprintfz(tag, ND_THREAD_TAG_MAX, "TH[%zu]", i);
+ thread_ptrs[i] = nd_thread_create(
+ tag,
+ NETDATA_THREAD_OPTION_JOINABLE | NETDATA_THREAD_OPTION_DONT_LOG,
+ aral_test_thread,
+ &auc);
}
size_t malloc_done = 0;
@@ -1041,12 +1050,12 @@ int aral_stress_test(size_t threads, size_t elements, size_t seconds) {
// fprintf(stderr, "Cancelling the threads...\n");
// for(size_t i = 0; i < threads ; i++) {
-// netdata_thread_cancel(thread_ptrs[i]);
+// nd_thread_signal_cancel(thread_ptrs[i]);
// }
fprintf(stderr, "Waiting the threads to finish...\n");
for(size_t i = 0; i < threads ; i++) {
- netdata_thread_join(thread_ptrs[i], NULL);
+ nd_thread_join(thread_ptrs[i]);
}
usec_t ended_ut = now_monotonic_usec();
diff --git a/libnetdata/aral/aral.h b/src/libnetdata/aral/aral.h
index 96f5a9c44..2e749bc4c 100644
--- a/libnetdata/aral/aral.h
+++ b/src/libnetdata/aral/aral.h
@@ -46,10 +46,12 @@ int aral_unittest(size_t elements);
#ifdef NETDATA_TRACE_ALLOCATIONS
+#define aral_callocz(ar) aral_callocz_internal(ar, __FILE__, __FUNCTION__, __LINE__)
#define aral_mallocz(ar) aral_mallocz_internal(ar, __FILE__, __FUNCTION__, __LINE__)
#define aral_freez(ar, ptr) aral_freez_internal(ar, ptr, __FILE__, __FUNCTION__, __LINE__)
#define aral_destroy(ar) aral_destroy_internal(ar, __FILE__, __FUNCTION__, __LINE__)
+void *aral_callocz_internal(ARAL *ar, const char *file, const char *function, size_t line);
void *aral_mallocz_internal(ARAL *ar, const char *file, const char *function, size_t line);
void aral_freez_internal(ARAL *ar, void *ptr, const char *file, const char *function, size_t line);
void aral_destroy_internal(ARAL *ar, const char *file, const char *function, size_t line);
@@ -57,9 +59,11 @@ void aral_destroy_internal(ARAL *ar, const char *file, const char *function, siz
#else // NETDATA_TRACE_ALLOCATIONS
#define aral_mallocz(ar) aral_mallocz_internal(ar)
+#define aral_callocz(ar) aral_callocz_internal(ar)
#define aral_freez(ar, ptr) aral_freez_internal(ar, ptr)
#define aral_destroy(ar) aral_destroy_internal(ar)
+void *aral_callocz_internal(ARAL *ar);
void *aral_mallocz_internal(ARAL *ar);
void aral_freez_internal(ARAL *ar, void *ptr);
void aral_destroy_internal(ARAL *ar);
diff --git a/libnetdata/avl/avl.c b/src/libnetdata/avl/avl.c
index eef4c3116..e1d4064dc 100644
--- a/libnetdata/avl/avl.c
+++ b/src/libnetdata/avl/avl.c
@@ -334,7 +334,7 @@ static inline void avl_write_lock(avl_tree_lock *t) {
static inline void avl_read_unlock(avl_tree_lock *t) {
#if defined(AVL_LOCK_WITH_RWLOCK)
- netdata_rwlock_unlock(&t->rwlock);
+ netdata_rwlock_rdunlock(&t->rwlock);
#else
rw_spinlock_read_unlock(&t->rwlock);
#endif
@@ -342,7 +342,7 @@ static inline void avl_read_unlock(avl_tree_lock *t) {
static inline void avl_write_unlock(avl_tree_lock *t) {
#if defined(AVL_LOCK_WITH_RWLOCK)
- netdata_rwlock_unlock(&t->rwlock);
+ netdata_rwlock_wrunlock(&t->rwlock);
#else
rw_spinlock_write_unlock(&t->rwlock);
#endif
diff --git a/libnetdata/avl/avl.h b/src/libnetdata/avl/avl.h
index 595d6ec6c..595d6ec6c 100644
--- a/libnetdata/avl/avl.h
+++ b/src/libnetdata/avl/avl.h
diff --git a/libnetdata/buffered_reader/README.md b/src/libnetdata/buffered_reader/README.md
index e69de29bb..e69de29bb 100644
--- a/libnetdata/buffered_reader/README.md
+++ b/src/libnetdata/buffered_reader/README.md
diff --git a/libnetdata/buffered_reader/buffered_reader.c b/src/libnetdata/buffered_reader/buffered_reader.c
index 7cd17abfe..7cd17abfe 100644
--- a/libnetdata/buffered_reader/buffered_reader.c
+++ b/src/libnetdata/buffered_reader/buffered_reader.c
diff --git a/libnetdata/circular_buffer/circular_buffer.c b/src/libnetdata/circular_buffer/circular_buffer.c
index 7ffe6b8bc..7ffe6b8bc 100644
--- a/libnetdata/circular_buffer/circular_buffer.c
+++ b/src/libnetdata/circular_buffer/circular_buffer.c
diff --git a/libnetdata/circular_buffer/circular_buffer.h b/src/libnetdata/circular_buffer/circular_buffer.h
index 9d29a84d7..9d29a84d7 100644
--- a/libnetdata/circular_buffer/circular_buffer.h
+++ b/src/libnetdata/circular_buffer/circular_buffer.h
diff --git a/libnetdata/clocks/clocks.c b/src/libnetdata/clocks/clocks.c
index adbad045f..e1a3e64cb 100644
--- a/libnetdata/clocks/clocks.c
+++ b/src/libnetdata/clocks/clocks.c
@@ -60,7 +60,7 @@ static usec_t get_clock_resolution(clockid_t clock) {
if(clock_getres(clock, &ts) == 0) {
usec_t ret = (usec_t)ts.tv_sec * USEC_PER_SEC + (usec_t)ts.tv_nsec / NSEC_PER_USEC;
- if(!ret && ts.tv_nsec > 0 && ts.tv_nsec < NSEC_PER_USEC)
+ if(!ret && ts.tv_nsec > 0 && ts.tv_nsec < (long int)NSEC_PER_USEC)
return (usec_t)1;
else if(ret > MAX_CLOCK_RESOLUTION_UT) {
@@ -92,7 +92,7 @@ void clocks_init(void) {
inline time_t now_sec(clockid_t clk_id) {
struct timespec ts;
if(unlikely(clock_gettime(clk_id, &ts) == -1)) {
- netdata_log_error("clock_gettime(%d, &timespec) failed.", clk_id);
+ netdata_log_error("clock_gettime(%ld, &timespec) failed.", (long int)clk_id);
return 0;
}
return ts.tv_sec;
@@ -101,7 +101,7 @@ inline time_t now_sec(clockid_t clk_id) {
inline usec_t now_usec(clockid_t clk_id) {
struct timespec ts;
if(unlikely(clock_gettime(clk_id, &ts) == -1)) {
- netdata_log_error("clock_gettime(%d, &timespec) failed.", clk_id);
+ netdata_log_error("clock_gettime(%ld, &timespec) failed.", (long int)clk_id);
return 0;
}
return (usec_t)ts.tv_sec * USEC_PER_SEC + (usec_t)(ts.tv_nsec % NSEC_PER_SEC) / NSEC_PER_USEC;
@@ -111,7 +111,7 @@ inline int now_timeval(clockid_t clk_id, struct timeval *tv) {
struct timespec ts;
if(unlikely(clock_gettime(clk_id, &ts) == -1)) {
- netdata_log_error("clock_gettime(%d, &timespec) failed.", clk_id);
+ netdata_log_error("clock_gettime(%ld, &timespec) failed.", (long int)clk_id);
tv->tv_sec = 0;
tv->tv_usec = 0;
return -1;
@@ -243,7 +243,7 @@ void sleep_to_absolute_time(usec_t usec) {
sleep_usec(usec);
}
}
-};
+}
#endif
#define HEARTBEAT_ALIGNMENT_STATISTICS_SIZE 10
diff --git a/libnetdata/clocks/clocks.h b/src/libnetdata/clocks/clocks.h
index 2beb14ed9..f989fd6b8 100644
--- a/libnetdata/clocks/clocks.h
+++ b/src/libnetdata/clocks/clocks.h
@@ -5,15 +5,14 @@
#include "../libnetdata.h"
-#ifndef HAVE_STRUCT_TIMESPEC
+#ifndef HAVE_CLOCK_GETTIME
struct timespec {
time_t tv_sec; /* seconds */
long tv_nsec; /* nanoseconds */
};
#endif
-#ifndef HAVE_CLOCKID_T
-typedef int clockid_t;
+#ifndef HAVE_CLOCK_GETTIME
#endif
typedef uint64_t nsec_t;
@@ -143,7 +142,7 @@ usec_t heartbeat_next(heartbeat_t *hb, usec_t tick);
void heartbeat_statistics(usec_t *min_ptr, usec_t *max_ptr, usec_t *average_ptr, size_t *count_ptr);
void sleep_usec_with_now(usec_t usec, usec_t started_ut);
-#define sleep_usec(usec) sleep_usec_with_now(usec, 0);
+#define sleep_usec(usec) sleep_usec_with_now(usec, 0)
void clocks_init(void);
diff --git a/libnetdata/completion/completion.h b/src/libnetdata/completion/completion.h
index 723f73688..908ccfaf6 100644
--- a/libnetdata/completion/completion.h
+++ b/src/libnetdata/completion/completion.h
@@ -18,6 +18,10 @@ void completion_destroy(struct completion *p);
void completion_wait_for(struct completion *p);
+// Wait for at most `timeout` seconds. Return true on success, false on
+// error or timeout.
+bool completion_timedwait_for(struct completion *p, uint64_t timeout);
+
void completion_mark_complete(struct completion *p);
unsigned completion_wait_for_a_job(struct completion *p, unsigned completed_jobs);
diff --git a/libnetdata/config/appconfig.c b/src/libnetdata/config/appconfig.c
index fe4c1222d..81946b594 100644
--- a/libnetdata/config/appconfig.c
+++ b/src/libnetdata/config/appconfig.c
@@ -904,7 +904,7 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)
* @return It returns 1 on success and 0 otherwise
*/
int config_parse_duration(const char* string, int* result) {
- while(*string && isspace(*string)) string++;
+ while(*string && isspace((uint8_t)*string)) string++;
if(unlikely(!*string)) goto fallback;
@@ -915,7 +915,7 @@ int config_parse_duration(const char* string, int* result) {
}
// make sure it is a number
- if(!(isdigit(*string) || *string == '+' || *string == '-')) goto fallback;
+ if(!(isdigit((uint8_t)*string) || *string == '+' || *string == '-')) goto fallback;
char *e = NULL;
NETDATA_DOUBLE n = str2ndd(string, &e);
diff --git a/libnetdata/config/appconfig.h b/src/libnetdata/config/appconfig.h
index 214a15edd..214a15edd 100644
--- a/libnetdata/config/appconfig.h
+++ b/src/libnetdata/config/appconfig.h
diff --git a/libnetdata/datetime/iso8601.c b/src/libnetdata/datetime/iso8601.c
index 8e3f4e027..8e3f4e027 100644
--- a/libnetdata/datetime/iso8601.c
+++ b/src/libnetdata/datetime/iso8601.c
diff --git a/libnetdata/datetime/iso8601.h b/src/libnetdata/datetime/iso8601.h
index ce4800963..ce4800963 100644
--- a/libnetdata/datetime/iso8601.h
+++ b/src/libnetdata/datetime/iso8601.h
diff --git a/libnetdata/datetime/rfc3339.c b/src/libnetdata/datetime/rfc3339.c
index 157e340d3..5c4e990dd 100644
--- a/libnetdata/datetime/rfc3339.c
+++ b/src/libnetdata/datetime/rfc3339.c
@@ -27,7 +27,7 @@ size_t rfc3339_datetime_ut(char *buffer, size_t len, usec_t now_ut, size_t fract
return 0;
}
- if (fractional_digits >= 0 && fractional_digits <= 9) {
+ if (fractional_digits >= 1 && fractional_digits <= 9) {
int fractional_part = (int)(now_ut % USEC_PER_SEC);
if (fractional_part && len - used_length > fractional_digits + 1) {
char format[] = ".%01d";
@@ -102,8 +102,8 @@ usec_t rfc3339_parse_ut(const char *rfc3339, char **endptr) {
if (*s == '+' || *s == '-') {
// Parse the hours:mins part of the timezone
- if (!isdigit(s[1]) || !isdigit(s[2]) || s[3] != ':' ||
- !isdigit(s[4]) || !isdigit(s[5]))
+ if (!isdigit((uint8_t)s[1]) || !isdigit((uint8_t)s[2]) || s[3] != ':' ||
+ !isdigit((uint8_t)s[4]) || !isdigit((uint8_t)s[5]))
return 0; // Parsing error
char tz_sign = *s;
diff --git a/libnetdata/datetime/rfc3339.h b/src/libnetdata/datetime/rfc3339.h
index 88ebb3ec8..88ebb3ec8 100644
--- a/libnetdata/datetime/rfc3339.h
+++ b/src/libnetdata/datetime/rfc3339.h
diff --git a/libnetdata/datetime/rfc7231.c b/src/libnetdata/datetime/rfc7231.c
index 4925ed2c9..4925ed2c9 100644
--- a/libnetdata/datetime/rfc7231.c
+++ b/src/libnetdata/datetime/rfc7231.c
diff --git a/libnetdata/datetime/rfc7231.h b/src/libnetdata/datetime/rfc7231.h
index 5ba93053f..5ba93053f 100644
--- a/libnetdata/datetime/rfc7231.h
+++ b/src/libnetdata/datetime/rfc7231.h
diff --git a/libnetdata/eval/README.md b/src/libnetdata/eval/README.md
index 8b1378917..8b1378917 100644
--- a/libnetdata/eval/README.md
+++ b/src/libnetdata/eval/README.md
diff --git a/libnetdata/facets/README.md b/src/libnetdata/facets/README.md
index e69de29bb..e69de29bb 100644
--- a/libnetdata/facets/README.md
+++ b/src/libnetdata/facets/README.md
diff --git a/libnetdata/facets/facets.c b/src/libnetdata/facets/facets.c
index 4a5f5442b..3c746cbc3 100644
--- a/libnetdata/facets/facets.c
+++ b/src/libnetdata/facets/facets.c
@@ -102,10 +102,7 @@ static inline bool is_valid_string_hash(const char *s) {
// hashtable for FACET_VALUE
// cleanup hashtable defines
-#undef SIMPLE_HASHTABLE_SORT_FUNCTION
-#undef SIMPLE_HASHTABLE_VALUE_TYPE
-#undef SIMPLE_HASHTABLE_NAME
-#undef NETDATA_SIMPLE_HASHTABLE_H
+#include "../../libnetdata/simple_hashtable_undef.h"
struct facet_value;
// #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_value
@@ -117,10 +114,7 @@ struct facet_value;
// hashtable for FACET_KEY
// cleanup hashtable defines
-#undef SIMPLE_HASHTABLE_SORT_FUNCTION
-#undef SIMPLE_HASHTABLE_VALUE_TYPE
-#undef SIMPLE_HASHTABLE_NAME
-#undef NETDATA_SIMPLE_HASHTABLE_H
+#include "../../libnetdata/simple_hashtable_undef.h"
struct facet_key;
// #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_key
@@ -439,12 +433,12 @@ static inline void FACET_VALUE_ADD_CONFLICT(FACET_KEY *k, FACET_VALUE *v, const
}
static inline FACET_VALUE *FACET_VALUE_GET_FROM_INDEX(FACET_KEY *k, FACETS_HASH hash) {
- SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, hash, true);
+ SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, hash, NULL, true);
return SIMPLE_HASHTABLE_SLOT_DATA(slot);
}
static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VALUE * const tv) {
- SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, tv->hash, true);
+ SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, tv->hash, NULL, true);
if(SIMPLE_HASHTABLE_SLOT_DATA(slot)) {
// already exists
@@ -634,7 +628,7 @@ static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) {
}
static inline FACET_KEY *FACETS_KEY_GET_FROM_INDEX(FACETS *facets, FACETS_HASH hash) {
- SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true);
+ SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, NULL, true);
return SIMPLE_HASHTABLE_SLOT_DATA(slot);
}
@@ -714,7 +708,7 @@ static inline FACET_KEY *FACETS_KEY_CREATE(FACETS *facets, FACETS_HASH hash, con
static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) {
facets->operations.keys.registered++;
- SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true);
+ SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, NULL, true);
if(unlikely(!SIMPLE_HASHTABLE_SLOT_DATA(slot))) {
// we have to add it
@@ -2235,8 +2229,8 @@ static int facets_keys_reorder_compar(const void *a, const void *b) {
if(!an) an = "0";
if(!bn) bn = "0";
- while(*an && ispunct(*an)) an++;
- while(*bn && ispunct(*bn)) bn++;
+ while(*an && ispunct((uint8_t)*an)) an++;
+ while(*bn && ispunct((uint8_t)*bn)) bn++;
return strcasecmp(an, bn);
}
@@ -2262,8 +2256,8 @@ static int facets_key_values_reorder_by_name_compar(const void *a, const void *b
const char *an = (av->name && av->name_len) ? av->name : "0";
const char *bn = (bv->name && bv->name_len) ? bv->name : "0";
- while(*an && ispunct(*an)) an++;
- while(*bn && ispunct(*bn)) bn++;
+ while(*an && ispunct((uint8_t)*an)) an++;
+ while(*bn && ispunct((uint8_t)*bn)) bn++;
int ret = strcasecmp(an, bn);
return ret;
@@ -2315,7 +2309,7 @@ static uint32_t facets_sort_and_reorder_values_internal(FACET_KEY *k) {
if(all_values_numeric && !v->empty && v->name && v->name_len) {
const char *s = v->name;
- while(isdigit(*s)) s++;
+ while(isdigit((uint8_t)*s)) s++;
if(*s != '\0')
all_values_numeric = false;
}
diff --git a/libnetdata/facets/facets.h b/src/libnetdata/facets/facets.h
index 8364d8612..8364d8612 100644
--- a/libnetdata/facets/facets.h
+++ b/src/libnetdata/facets/facets.h
diff --git a/libnetdata/functions_evloop/README.md b/src/libnetdata/functions_evloop/README.md
index e69de29bb..e69de29bb 100644
--- a/libnetdata/functions_evloop/README.md
+++ b/src/libnetdata/functions_evloop/README.md
diff --git a/libnetdata/gorilla/README.md b/src/libnetdata/gorilla/README.md
index dc3718d13..dc3718d13 100644
--- a/libnetdata/gorilla/README.md
+++ b/src/libnetdata/gorilla/README.md
diff --git a/libnetdata/gorilla/benchmark.sh b/src/libnetdata/gorilla/benchmark.sh
index a5d111435..a5d111435 100755
--- a/libnetdata/gorilla/benchmark.sh
+++ b/src/libnetdata/gorilla/benchmark.sh
diff --git a/libnetdata/gorilla/fuzzer.sh b/src/libnetdata/gorilla/fuzzer.sh
index 19098a615..19098a615 100755
--- a/libnetdata/gorilla/fuzzer.sh
+++ b/src/libnetdata/gorilla/fuzzer.sh
diff --git a/libnetdata/gorilla/gorilla.cc b/src/libnetdata/gorilla/gorilla.cc
index e6138ce38..c76018365 100644
--- a/libnetdata/gorilla/gorilla.cc
+++ b/src/libnetdata/gorilla/gorilla.cc
@@ -212,7 +212,7 @@ bool gorilla_writer_serialize(const gorilla_writer_t *gw, uint8_t *dst, uint32_t
do {
const gorilla_buffer_t *next_gbuf = curr_gbuf->header.next;
- size_t bytes = GORILLA_BUFFER_SIZE;
+ size_t bytes = RRDENG_GORILLA_32BIT_BUFFER_SIZE;
if (bytes > dst_size)
return false;
@@ -232,7 +232,7 @@ uint32_t gorilla_buffer_patch(gorilla_buffer_t *gbuf) {
while (curr_gbuf->header.next) {
uint32_t *buf = reinterpret_cast<uint32_t *>(gbuf);
- gbuf = reinterpret_cast<gorilla_buffer_t *>(&buf[GORILLA_BUFFER_SLOTS]);
+ gbuf = reinterpret_cast<gorilla_buffer_t *>(&buf[RRDENG_GORILLA_32BIT_BUFFER_SLOTS]);
assert(((uintptr_t) (gbuf) % sizeof(uintptr_t)) == 0 &&
"Gorilla buffer not aligned to uintptr_t");
diff --git a/libnetdata/gorilla/gorilla.h b/src/libnetdata/gorilla/gorilla.h
index d57c07cf0..7975d85ee 100644
--- a/libnetdata/gorilla/gorilla.h
+++ b/src/libnetdata/gorilla/gorilla.h
@@ -67,8 +67,8 @@ uint32_t gorilla_buffer_patch(gorilla_buffer_t *buf);
gorilla_reader_t gorilla_reader_init(gorilla_buffer_t *buf);
bool gorilla_reader_read(gorilla_reader_t *gr, uint32_t *number);
-#define GORILLA_BUFFER_SLOTS 128
-#define GORILLA_BUFFER_SIZE (GORILLA_BUFFER_SLOTS * sizeof(uint32_t))
+#define RRDENG_GORILLA_32BIT_BUFFER_SLOTS 128
+#define RRDENG_GORILLA_32BIT_BUFFER_SIZE (RRDENG_GORILLA_32BIT_BUFFER_SLOTS * sizeof(uint32_t))
#ifdef __cplusplus
}
diff --git a/libnetdata/inlined.h b/src/libnetdata/inlined.h
index 535b791e3..6b71590c9 100644
--- a/libnetdata/inlined.h
+++ b/src/libnetdata/inlined.h
@@ -204,12 +204,28 @@ static inline long long str2ll(const char *s, char **endptr) {
}
}
+static inline uint32_t str2uint32_hex(const char *src, char **endptr) {
+ uint32_t num = 0;
+ const unsigned char *s = (const unsigned char *)src;
+ unsigned char c;
+
+ while ((c = hex_value_from_ascii[(uint8_t)*s]) != 255) {
+ num = (num << 4) | c;
+ s++;
+ }
+
+ if(endptr)
+ *endptr = (char *)s;
+
+ return num;
+}
+
static inline uint64_t str2uint64_hex(const char *src, char **endptr) {
uint64_t num = 0;
const unsigned char *s = (const unsigned char *)src;
unsigned char c;
- while ((c = hex_value_from_ascii[toupper(*s)]) != 255) {
+ while ((c = hex_value_from_ascii[(uint8_t)*s]) != 255) {
num = (num << 4) | c;
s++;
}
@@ -437,7 +453,7 @@ static inline bool sanitize_command_argument_string(char *dst, const char *src,
if (dst_size < 1)
return false;
- if (iscntrl(*src) || *src == '$') {
+ if (iscntrl((uint8_t)*src) || *src == '$') {
// remove control characters and characters that are expanded by bash
*dst++ = '_';
dst_size--;
@@ -469,16 +485,16 @@ static inline bool sanitize_command_argument_string(char *dst, const char *src,
return true;
}
-static inline int read_file(const char *filename, char *buffer, size_t size) {
+static inline int read_txt_file(const char *filename, char *buffer, size_t size) {
if(unlikely(!size)) return 3;
- int fd = open(filename, O_RDONLY, 0666);
+ int fd = open(filename, O_RDONLY | O_CLOEXEC, 0666);
if(unlikely(fd == -1)) {
buffer[0] = '\0';
return 1;
}
- ssize_t r = read(fd, buffer, size);
+ ssize_t r = read(fd, buffer, size - 1); // leave space of the final zero
if(unlikely(r == -1)) {
buffer[0] = '\0';
close(fd);
@@ -490,10 +506,43 @@ static inline int read_file(const char *filename, char *buffer, size_t size) {
return 0;
}
+static inline int read_proc_cmdline(const char *filename, char *buffer, size_t size) {
+ if (unlikely(!size)) return 3;
+
+ int fd = open(filename, O_RDONLY | O_CLOEXEC, 0666);
+ if (unlikely(fd == -1)) {
+ buffer[0] = '\0';
+ return 1;
+ }
+
+ ssize_t r = read(fd, buffer, size - 1); // Leave space for final null character
+ if (unlikely(r == -1)) {
+ buffer[0] = '\0';
+ close(fd);
+ return 2;
+ }
+
+ if (r > 0) {
+ // Replace null characters with spaces, except for the last one
+ for (ssize_t i = 0; i < r - 1; i++) {
+ if (buffer[i] == '\0') {
+ buffer[i] = ' ';
+ }
+ }
+ buffer[r] = '\0'; // Null-terminate the string
+ }
+ else {
+ buffer[0] = '\0'; // Empty cmdline
+ }
+
+ close(fd);
+ return 0;
+}
+
static inline int read_single_number_file(const char *filename, unsigned long long *result) {
char buffer[30 + 1];
- int ret = read_file(filename, buffer, 30);
+ int ret = read_txt_file(filename, buffer, sizeof(buffer));
if(unlikely(ret)) {
*result = 0;
return ret;
@@ -507,7 +556,7 @@ static inline int read_single_number_file(const char *filename, unsigned long lo
static inline int read_single_signed_number_file(const char *filename, long long *result) {
char buffer[30 + 1];
- int ret = read_file(filename, buffer, 30);
+ int ret = read_txt_file(filename, buffer, sizeof(buffer));
if(unlikely(ret)) {
*result = 0;
return ret;
@@ -521,7 +570,7 @@ static inline int read_single_signed_number_file(const char *filename, long long
static inline int read_single_base64_or_hex_number_file(const char *filename, unsigned long long *result) {
char buffer[30 + 1];
- int ret = read_file(filename, buffer, 30);
+ int ret = read_txt_file(filename, buffer, sizeof(buffer));
if(unlikely(ret)) {
*result = 0;
return ret;
@@ -548,7 +597,7 @@ static inline char *strsep_skip_consecutive_separators(char **ptr, char *s) {
// remove leading and trailing spaces; may return NULL
static inline char *trim(char *s) {
// skip leading spaces
- while (*s && isspace(*s)) s++;
+ while (*s && isspace((uint8_t)*s)) s++;
if (!*s) return NULL;
// skip tailing spaces
@@ -556,7 +605,7 @@ static inline char *trim(char *s) {
ssize_t l = (ssize_t)strlen(s);
if (--l >= 0) {
char *p = s + l;
- while (p > s && isspace(*p)) p--;
+ while (p > s && isspace((uint8_t)*p)) p--;
*++p = '\0';
}
@@ -570,31 +619,77 @@ static inline char *trim_all(char *buffer) {
char *d = buffer, *s = buffer;
// skip spaces
- while(isspace(*s)) s++;
+ while(isspace((uint8_t)*s)) s++;
while(*s) {
// copy the non-space part
- while(*s && !isspace(*s)) *d++ = *s++;
+ while(*s && !isspace((uint8_t)*s)) *d++ = *s++;
// add a space if we have to
- if(*s && isspace(*s)) {
+ if(*s && isspace((uint8_t)*s)) {
*d++ = ' ';
s++;
}
// skip spaces
- while(isspace(*s)) s++;
+ while(isspace((uint8_t)*s)) s++;
}
*d = '\0';
if(d > buffer) {
d--;
- if(isspace(*d)) *d = '\0';
+ if(isspace((uint8_t)*d)) *d = '\0';
}
if(!buffer[0]) return NULL;
return buffer;
}
+static inline bool streq(const char *a, const char *b) {
+ if (a == b)
+ return true;
+
+ if (a == NULL || b == NULL)
+ return false;
+
+ return strcmp(a, b) == 0;
+}
+
+static inline bool strstartswith(const char *string, const char *prefix) {
+ if (string == NULL || prefix == NULL)
+ return false;
+
+ size_t string_len = strlen(string);
+ size_t prefix_len = strlen(prefix);
+
+ if (prefix_len > string_len)
+ return false;
+
+ return strncmp(string, prefix, prefix_len) == 0;
+}
+
+static inline bool strendswith(const char *string, const char *suffix) {
+ if (string == NULL || suffix == NULL)
+ return false;
+
+ size_t string_len = strlen(string);
+ size_t suffix_len = strlen(suffix);
+
+ if (suffix_len > string_len)
+ return false;
+
+ return strcmp(string + string_len - suffix_len, suffix) == 0;
+}
+
+static inline bool strendswith_lengths(const char *string, size_t string_len, const char *suffix, size_t suffix_len) {
+ if (string == NULL || suffix == NULL)
+ return false;
+
+ if (suffix_len > string_len)
+ return false;
+
+ return strcmp(string + string_len - suffix_len, suffix) == 0;
+}
+
#endif //NETDATA_INLINED_H
diff --git a/libnetdata/json/jsmn.c b/src/libnetdata/json/jsmn.c
index 2f48bd65a..2f48bd65a 100644
--- a/libnetdata/json/jsmn.c
+++ b/src/libnetdata/json/jsmn.c
diff --git a/libnetdata/json/jsmn.h b/src/libnetdata/json/jsmn.h
index beff586c6..beff586c6 100644
--- a/libnetdata/json/jsmn.h
+++ b/src/libnetdata/json/jsmn.h
diff --git a/libnetdata/july/july.c b/src/libnetdata/july/july.c
index 56b8494b3..56b8494b3 100644
--- a/libnetdata/july/july.c
+++ b/src/libnetdata/july/july.c
diff --git a/libnetdata/july/july.h b/src/libnetdata/july/july.h
index 672ed44e4..672ed44e4 100644
--- a/libnetdata/july/july.h
+++ b/src/libnetdata/july/july.h
diff --git a/libnetdata/libjudy/src/Judy.h b/src/libnetdata/libjudy/src/Judy.h
index adfb5b53b..adfb5b53b 100644
--- a/libnetdata/libjudy/src/Judy.h
+++ b/src/libnetdata/libjudy/src/Judy.h
diff --git a/libnetdata/libjudy/src/JudyCommon/JudyMalloc.c b/src/libnetdata/libjudy/src/JudyCommon/JudyMalloc.c
index 09a20e399..09a20e399 100644
--- a/libnetdata/libjudy/src/JudyCommon/JudyMalloc.c
+++ b/src/libnetdata/libjudy/src/JudyCommon/JudyMalloc.c
diff --git a/libnetdata/libjudy/src/JudyCommon/JudyPrivate.h b/src/libnetdata/libjudy/src/JudyCommon/JudyPrivate.h
index 350631f01..350631f01 100644
--- a/libnetdata/libjudy/src/JudyCommon/JudyPrivate.h
+++ b/src/libnetdata/libjudy/src/JudyCommon/JudyPrivate.h
diff --git a/libnetdata/libjudy/src/JudyCommon/JudyPrivate1L.h b/src/libnetdata/libjudy/src/JudyCommon/JudyPrivate1L.h
index 5b4704899..5b4704899 100644
--- a/libnetdata/libjudy/src/JudyCommon/JudyPrivate1L.h
+++ b/src/libnetdata/libjudy/src/JudyCommon/JudyPrivate1L.h
diff --git a/libnetdata/libjudy/src/JudyCommon/JudyPrivateBranch.h b/src/libnetdata/libjudy/src/JudyCommon/JudyPrivateBranch.h
index 10295ba95..10295ba95 100644
--- a/libnetdata/libjudy/src/JudyCommon/JudyPrivateBranch.h
+++ b/src/libnetdata/libjudy/src/JudyCommon/JudyPrivateBranch.h
diff --git a/libnetdata/libjudy/src/JudyHS/JudyHS.c b/src/libnetdata/libjudy/src/JudyHS/JudyHS.c
index 21191babc..21191babc 100644
--- a/libnetdata/libjudy/src/JudyHS/JudyHS.c
+++ b/src/libnetdata/libjudy/src/JudyHS/JudyHS.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyL.h b/src/libnetdata/libjudy/src/JudyL/JudyL.h
index d901969d6..d901969d6 100644
--- a/libnetdata/libjudy/src/JudyL/JudyL.h
+++ b/src/libnetdata/libjudy/src/JudyL/JudyL.h
diff --git a/libnetdata/libjudy/src/JudyL/JudyLByCount.c b/src/libnetdata/libjudy/src/JudyL/JudyLByCount.c
index c5a004796..c5a004796 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLByCount.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLByCount.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLCascade.c b/src/libnetdata/libjudy/src/JudyL/JudyLCascade.c
index 6b52ddf5f..c1a26f413 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLCascade.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLCascade.c
@@ -311,6 +311,7 @@ static int j__udyStageJBBtoJBB(
//
// NOTE: Caller must release the Leaf2 that was passed in.
+__attribute__((no_sanitize("shift")))
FUNCTION static Pjlb_t j__udyJLL2toJLB1(
uint16_t * Pjll, // array of 16-bit indexes.
#ifdef JUDYL
diff --git a/libnetdata/libjudy/src/JudyL/JudyLCount.c b/src/libnetdata/libjudy/src/JudyL/JudyLCount.c
index 179757f0a..179757f0a 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLCount.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLCount.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLCreateBranch.c b/src/libnetdata/libjudy/src/JudyL/JudyLCreateBranch.c
index ffe6b3bde..ffe6b3bde 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLCreateBranch.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLCreateBranch.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLDecascade.c b/src/libnetdata/libjudy/src/JudyL/JudyLDecascade.c
index 39a89eff1..c2bf81ea1 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLDecascade.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLDecascade.c
@@ -345,6 +345,7 @@ FUNCTION int j__udyBranchUToBranchB(
// allocation and free, in order to allow the caller to continue with a LeafB1
// if allocation fails.
+__attribute__((no_sanitize("shift")))
FUNCTION int j__udyLeafB1ToLeaf1(
Pjp_t Pjp, // points to LeafB1 to shrink.
Pvoid_t Pjpm) // for global accounting.
@@ -431,6 +432,7 @@ FUNCTION int j__udyLeafB1ToLeaf1(
// TBD: In this and all following functions, the caller should already be able
// to compute the Pop1 return value, so why return it?
+__attribute__((no_sanitize("shift")))
FUNCTION Word_t j__udyLeaf1ToLeaf2(
uint16_t * PLeaf2, // destination uint16_t * Index portion of leaf.
#ifdef JUDYL
diff --git a/libnetdata/libjudy/src/JudyL/JudyLDel.c b/src/libnetdata/libjudy/src/JudyL/JudyLDel.c
index ced4b5fb3..7c3d9108b 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLDel.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLDel.c
@@ -147,6 +147,7 @@ extern Word_t j__udyLLeaf7ToLeafW(Pjlw_t, Pjv_t, Pjp_t, Word_t, Pvoid_t);
DBGCODE(uint8_t parentJPtype;) // parent branch JP type.
+__attribute__((no_sanitize("shift")))
FUNCTION static int j__udyDelWalk(
Pjp_t Pjp, // current JP under which to delete.
Word_t Index, // to delete.
diff --git a/libnetdata/libjudy/src/JudyL/JudyLFirst.c b/src/libnetdata/libjudy/src/JudyL/JudyLFirst.c
index aaf6639cf..aaf6639cf 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLFirst.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLFirst.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLFreeArray.c b/src/libnetdata/libjudy/src/JudyL/JudyLFreeArray.c
index 34fac509e..34fac509e 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLFreeArray.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLFreeArray.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLGet.c b/src/libnetdata/libjudy/src/JudyL/JudyLGet.c
index 0bb9971cc..e6853939b 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLGet.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLGet.c
@@ -44,6 +44,8 @@
// See the manual entry for details. Note support for "shortcut" entries to
// trees known to start with a JPM.
+__attribute__((no_sanitize("shift")))
+
#ifdef JUDY1
#ifdef JUDYGETINLINE
diff --git a/libnetdata/libjudy/src/JudyL/JudyLIns.c b/src/libnetdata/libjudy/src/JudyL/JudyLIns.c
index f96df4101..256a1ef3b 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLIns.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLIns.c
@@ -152,6 +152,7 @@ extern int j__udyLInsertBranch(Pjp_t Pjp, Word_t Index, Word_t Btype, Pjpm_t);
// Return -1 for error (details in JPM), 0 for Index already inserted, 1 for
// new Index inserted.
+__attribute__((no_sanitize("shift")))
FUNCTION static int j__udyInsWalk(
Pjp_t Pjp, // current JP to descend.
Word_t Index, // to insert.
diff --git a/libnetdata/libjudy/src/JudyL/JudyLInsArray.c b/src/libnetdata/libjudy/src/JudyL/JudyLInsArray.c
index f8e361f27..f8e361f27 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLInsArray.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLInsArray.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLInsertBranch.c b/src/libnetdata/libjudy/src/JudyL/JudyLInsertBranch.c
index cfa16bd6d..cfa16bd6d 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLInsertBranch.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLInsertBranch.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLMallocIF.c b/src/libnetdata/libjudy/src/JudyL/JudyLMallocIF.c
index 9a7d02f21..9a7d02f21 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLMallocIF.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLMallocIF.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLMemActive.c b/src/libnetdata/libjudy/src/JudyL/JudyLMemActive.c
index fb58d0e25..fb58d0e25 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLMemActive.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLMemActive.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLMemUsed.c b/src/libnetdata/libjudy/src/JudyL/JudyLMemUsed.c
index 81e3a79ce..81e3a79ce 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLMemUsed.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLMemUsed.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLNext.c b/src/libnetdata/libjudy/src/JudyL/JudyLNext.c
index 4bcdccf10..4bcdccf10 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLNext.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLNext.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLNextEmpty.c b/src/libnetdata/libjudy/src/JudyL/JudyLNextEmpty.c
index 4da43565d..4da43565d 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLNextEmpty.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLNextEmpty.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLPrev.c b/src/libnetdata/libjudy/src/JudyL/JudyLPrev.c
index 4bcdccf10..4bcdccf10 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLPrev.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLPrev.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLPrevEmpty.c b/src/libnetdata/libjudy/src/JudyL/JudyLPrevEmpty.c
index 4da43565d..4da43565d 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLPrevEmpty.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLPrevEmpty.c
diff --git a/libnetdata/libjudy/src/JudyL/JudyLTables.c b/src/libnetdata/libjudy/src/JudyL/JudyLTables.c
index 21c974986..21c974986 100644
--- a/libnetdata/libjudy/src/JudyL/JudyLTables.c
+++ b/src/libnetdata/libjudy/src/JudyL/JudyLTables.c
diff --git a/libnetdata/libjudy/src/JudyL/j__udyLGet.c b/src/libnetdata/libjudy/src/JudyL/j__udyLGet.c
index 0bb9971cc..0bb9971cc 100644
--- a/libnetdata/libjudy/src/JudyL/j__udyLGet.c
+++ b/src/libnetdata/libjudy/src/JudyL/j__udyLGet.c
diff --git a/libnetdata/libnetdata.c b/src/libnetdata/libnetdata.c
index 37319a884..909bb71d0 100644
--- a/libnetdata/libnetdata.c
+++ b/src/libnetdata/libnetdata.c
@@ -2,24 +2,23 @@
#include "libnetdata.h"
-#ifdef __APPLE__
-#define INHERIT_NONE 0
-#endif /* __APPLE__ */
-#if defined(__FreeBSD__) || defined(__APPLE__)
-# define O_NOATIME 0
-# define MADV_DONTFORK INHERIT_NONE
-#endif /* __FreeBSD__ || __APPLE__*/
+#if !defined(MADV_DONTFORK)
+#define MADV_DONTFORK 0
+#endif
+
+#if !defined(O_NOATIME)
+#define O_NOATIME 0
+#endif
struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 };
-#ifdef MADV_MERGEABLE
+#if defined(MADV_MERGEABLE)
int enable_ksm = 1;
#else
int enable_ksm = 0;
#endif
volatile sig_atomic_t netdata_exit = 0;
-const char *program_version = VERSION;
#define MAX_JUDY_SIZE_TO_ARAL 24
static bool judy_sizes_config[MAX_JUDY_SIZE_TO_ARAL + 1] = {
@@ -129,6 +128,9 @@ static void (*libc_free)(void *) = free_first_run;
static char *strdup_first_run(const char *s);
static char *(*libc_strdup)(const char *) = strdup_first_run;
+static char *strndup_first_run(const char *s, size_t len);
+static char *(*libc_strndup)(const char *, size_t) = strndup_first_run;
+
static size_t malloc_usable_size_first_run(void *ptr);
#ifdef HAVE_MALLOC_USABLE_SIZE
static size_t (*libc_malloc_usable_size)(void *) = malloc_usable_size_first_run;
@@ -169,6 +171,11 @@ static char *strdup_first_run(const char *s) {
return libc_strdup(s);
}
+static char *strndup_first_run(const char *s, size_t len) {
+ link_system_library_function((libc_function_t *) &libc_strndup, "strndup", true);
+ return libc_strndup(s, len);
+}
+
static size_t malloc_usable_size_first_run(void *ptr) {
link_system_library_function((libc_function_t *) &libc_malloc_usable_size, "malloc_usable_size", false);
@@ -202,6 +209,10 @@ char *strdup(const char *s) {
return strdupz(s);
}
+char *strndup(const char *s, size_t len) {
+ return strndupz(s, len);
+}
+
size_t malloc_usable_size(void *ptr) {
return mallocz_usable_size(ptr);
}
@@ -365,6 +376,30 @@ char *strdupz_int(const char *s, const char *file, const char *function, size_t
return (char *)&t->data;
}
+char *strndupz_int(const char *s, size_t len, const char *file, const char *function, size_t line) {
+ struct malloc_trace *p = malloc_trace_find_or_create(file, function, line);
+ size_t size = len + 1;
+
+ size_t_atomic_count(add, p->strdup_calls, 1);
+ size_t_atomic_count(add, p->allocations, 1);
+ size_t_atomic_bytes(add, p->bytes, size);
+
+ struct malloc_header *t = (struct malloc_header *)libc_malloc(malloc_header_size + size);
+ if (unlikely(!t)) fatal("strndupz() cannot allocate %zu bytes of memory (%zu with header).", size, malloc_header_size + size);
+ t->signature.magic = 0x0BADCAFE;
+ t->signature.trace = p;
+ t->signature.size = size;
+
+#ifdef NETDATA_INTERNAL_CHECKS
+ for(ssize_t i = 0; i < (ssize_t)sizeof(t->padding) ;i++) // signed to avoid compiler warning when zero-padded
+ t->padding[i] = 0xFF;
+#endif
+
+ memcpy(&t->data, s, size);
+ t->data[len] = '\0';
+ return (char *)&t->data;
+}
+
static struct malloc_header *malloc_get_header(void *ptr, const char *caller, const char *file, const char *function, size_t line) {
uint8_t *ret = (uint8_t *)ptr - malloc_header_size;
struct malloc_header *t = (struct malloc_header *)ret;
@@ -450,6 +485,12 @@ char *strdupz(const char *s) {
return t;
}
+char *strndupz(const char *s, size_t len) {
+ char *t = strndup(s, len);
+ if (unlikely(!t)) fatal("Cannot strndup() string '%s' of len %zu", s, len);
+ return t;
+}
+
// If ptr is NULL, no operation is performed.
void freez(void *ptr) {
free(ptr);
@@ -1045,7 +1086,7 @@ void netdata_fix_chart_id(char *s) {
static int memory_file_open(const char *filename, size_t size) {
// netdata_log_info("memory_file_open('%s', %zu", filename, size);
- int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME, 0664);
+ int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME | O_CLOEXEC, 0664);
if (fd != -1) {
if (lseek(fd, size, SEEK_SET) == (off_t) size) {
if (write(fd, "", 1) == 1) {
@@ -1218,37 +1259,6 @@ int netdata_munmap(void *ptr, size_t size) {
return munmap(ptr, size);
}
-int memory_file_save(const char *filename, void *mem, size_t size) {
- char tmpfilename[FILENAME_MAX + 1];
-
- snprintfz(tmpfilename, FILENAME_MAX, "%s.%ld.tmp", filename, (long) getpid());
-
- int fd = open(tmpfilename, O_RDWR | O_CREAT | O_NOATIME, 0664);
- if (fd < 0) {
- netdata_log_error("Cannot create/open file '%s'.", filename);
- return -1;
- }
-
- if (write(fd, mem, size) != (ssize_t) size) {
- netdata_log_error("Cannot write to file '%s' %ld bytes.", filename, (long) size);
- close(fd);
- return -1;
- }
-
- close(fd);
-
- if (rename(tmpfilename, filename)) {
- netdata_log_error("Cannot rename '%s' to '%s'", tmpfilename, filename);
- return -1;
- }
-
- return 0;
-}
-
-int fd_is_valid(int fd) {
- return fcntl(fd, F_GETFD) != -1 || errno != EBADF;
-}
-
char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) {
char *s = fgets(buf, (int)buf_size, fp);
if (!s) return NULL;
@@ -1292,88 +1302,55 @@ int snprintfz(char *dst, size_t n, const char *fmt, ...) {
return ret;
}
-/*
-// poor man cycle counting
-static unsigned long tsc;
-void begin_tsc(void) {
- unsigned long a, d;
- asm volatile ("cpuid\nrdtsc" : "=a" (a), "=d" (d) : "0" (0) : "ebx", "ecx");
- tsc = ((unsigned long)d << 32) | (unsigned long)a;
-}
-unsigned long end_tsc(void) {
- unsigned long a, d;
- asm volatile ("rdtscp" : "=a" (a), "=d" (d) : : "ecx");
- return (((unsigned long)d << 32) | (unsigned long)a) - tsc;
-}
-*/
+static int is_procfs(const char *path, char **reason) {
+#if defined(__APPLE__) || defined(__FreeBSD__)
+ (void)path;
+ (void)reason;
+#else
+ struct statfs stat;
-int recursively_delete_dir(const char *path, const char *reason) {
- DIR *dir = opendir(path);
- if(!dir) {
- netdata_log_error("Cannot read %s directory to be deleted '%s'", reason?reason:"", path);
+ if (statfs(path, &stat) == -1) {
+ if (reason)
+ *reason = "failed to statfs()";
return -1;
}
- int ret = 0;
- struct dirent *de = NULL;
- while((de = readdir(dir))) {
- if(de->d_type == DT_DIR
- && (
- (de->d_name[0] == '.' && de->d_name[1] == '\0')
- || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
- ))
- continue;
-
- char fullpath[FILENAME_MAX + 1];
- snprintfz(fullpath, FILENAME_MAX, "%s/%s", path, de->d_name);
-
- if(de->d_type == DT_DIR) {
- int r = recursively_delete_dir(fullpath, reason);
- if(r > 0) ret += r;
- continue;
- }
-
- netdata_log_info("Deleting %s file '%s'", reason?reason:"", fullpath);
- if(unlikely(unlink(fullpath) == -1))
- netdata_log_error("Cannot delete %s file '%s'", reason?reason:"", fullpath);
- else
- ret++;
+#if defined PROC_SUPER_MAGIC
+ if (stat.f_type != PROC_SUPER_MAGIC) {
+ if (reason)
+ *reason = "type is not procfs";
+ return -1;
}
+#endif
- netdata_log_info("Deleting empty directory '%s'", path);
- if(unlikely(rmdir(path) == -1))
- netdata_log_error("Cannot delete empty directory '%s'", path);
- else
- ret++;
-
- closedir(dir);
+#endif
- return ret;
+ return 0;
}
-static int is_virtual_filesystem(const char *path, char **reason) {
-
+static int is_sysfs(const char *path, char **reason) {
#if defined(__APPLE__) || defined(__FreeBSD__)
(void)path;
(void)reason;
#else
struct statfs stat;
- // stat.f_fsid.__val[0] is a file system id
- // stat.f_fsid.__val[1] is the inode
- // so their combination uniquely identifies the file/dir
if (statfs(path, &stat) == -1) {
- if(reason) *reason = "failed to statfs()";
+ if (reason)
+ *reason = "failed to statfs()";
return -1;
}
- if(stat.f_fsid.__val[0] != 0 || stat.f_fsid.__val[1] != 0) {
- errno = EINVAL;
- if(reason) *reason = "is not a virtual file system";
+#if defined SYSFS_MAGIC
+ if (stat.f_type != SYSFS_MAGIC) {
+ if (reason)
+ *reason = "type is not sysfs";
return -1;
}
#endif
+#endif
+
return 0;
}
@@ -1403,11 +1380,11 @@ int verify_netdata_host_prefix(bool log_msg) {
path = buffer;
snprintfz(path, FILENAME_MAX, "%s/proc", netdata_configured_host_prefix);
- if(is_virtual_filesystem(path, &reason) == -1)
+ if(is_procfs(path, &reason) == -1)
goto failed;
snprintfz(path, FILENAME_MAX, "%s/sys", netdata_configured_host_prefix);
- if(is_virtual_filesystem(path, &reason) == -1)
+ if(is_sysfs(path, &reason) == -1)
goto failed;
if (netdata_configured_host_prefix && *netdata_configured_host_prefix) {
@@ -1525,12 +1502,15 @@ int path_is_file(const char *path, const char *subpath) {
return is_file;
}
-void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *subpath, int (*callback)(const char *filename, void *data), void *data, size_t depth) {
+void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *subpath, int (*callback)(const char *filename, void *data, bool stock_config), void *data, size_t depth) {
if(depth > 3) {
netdata_log_error("CONFIG: Max directory depth reached while reading user path '%s', stock path '%s', subpath '%s'", user_path, stock_path, subpath);
return;
}
+ if(!stock_path)
+ stock_path = user_path;
+
char *udir = strdupz_path_subpath(user_path, subpath);
char *sdir = strdupz_path_subpath(stock_path, subpath);
@@ -1564,7 +1544,7 @@ void recursive_config_double_dir_load(const char *user_path, const char *stock_p
len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
char *filename = strdupz_path_subpath(udir, de->d_name);
netdata_log_debug(D_HEALTH, "CONFIG calling callback for user file '%s'", filename);
- callback(filename, data);
+ callback(filename, data, false);
freez(filename);
continue;
}
@@ -1612,7 +1592,7 @@ void recursive_config_double_dir_load(const char *user_path, const char *stock_p
len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
char *filename = strdupz_path_subpath(sdir, de->d_name);
netdata_log_debug(D_HEALTH, "CONFIG calling callback for stock file '%s'", filename);
- callback(filename, data);
+ callback(filename, data, true);
freez(filename);
continue;
}
@@ -1741,6 +1721,11 @@ bool run_command_and_copy_output_to_stdout(const char *command, int max_line_len
return true;
}
+
+static int fd_is_valid(int fd) {
+ return fcntl(fd, F_GETFD) != -1 || errno != EBADF;
+}
+
void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds){
int fd;
@@ -1751,7 +1736,7 @@ void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds){
if(!(excluded_fds & OPEN_FD_EXCLUDE_STDERR)) (void)close(STDERR_FILENO);
#if defined(HAVE_CLOSE_RANGE)
if(close_range(STDERR_FILENO + 1, ~0U, 0) == 0) return;
- netdata_log_error("close_range() failed, will try to close fds one by one");
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "close_range() failed, will try to close fds one by one");
#endif
break;
case OPEN_FD_ACTION_FD_CLOEXEC:
@@ -1760,7 +1745,7 @@ void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds){
if(!(excluded_fds & OPEN_FD_EXCLUDE_STDERR)) (void)fcntl(STDERR_FILENO, F_SETFD, FD_CLOEXEC);
#if defined(HAVE_CLOSE_RANGE) && defined(CLOSE_RANGE_CLOEXEC) // Linux >= 5.11, FreeBSD >= 13.1
if(close_range(STDERR_FILENO + 1, ~0U, CLOSE_RANGE_CLOEXEC) == 0) return;
- netdata_log_error("close_range() failed, will try to mark fds for closing one by one");
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "close_range() failed, will try to mark fds for closing one by one");
#endif
break;
default:
@@ -2014,7 +1999,7 @@ bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t now)
}
// Returns 1 if an absolute period was requested or 0 if it was a relative period
-bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running) {
+bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_t *now_ptr, bool unittest) {
time_t now = now_realtime_sec() - 1;
if(now_ptr)
@@ -2028,16 +2013,16 @@ bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_
time_t absolute_minimum_time = now - (10 * 365 * 86400);
time_t absolute_maximum_time = now + (1 * 365 * 86400);
- if (after_requested < absolute_minimum_time && !unittest_running)
+ if (after_requested < absolute_minimum_time && !unittest)
after_requested = absolute_minimum_time;
- if (after_requested > absolute_maximum_time && !unittest_running)
+ if (after_requested > absolute_maximum_time && !unittest)
after_requested = absolute_maximum_time;
- if (before_requested < absolute_minimum_time && !unittest_running)
+ if (before_requested < absolute_minimum_time && !unittest)
before_requested = absolute_minimum_time;
- if (before_requested > absolute_maximum_time && !unittest_running)
+ if (before_requested > absolute_maximum_time && !unittest)
before_requested = absolute_maximum_time;
*before = before_requested;
diff --git a/libnetdata/line_splitter/line_splitter.c b/src/libnetdata/line_splitter/line_splitter.c
index a459d3347..6726d9096 100644
--- a/libnetdata/line_splitter/line_splitter.c
+++ b/src/libnetdata/line_splitter/line_splitter.c
@@ -56,14 +56,23 @@ inline int group_by_label_isspace(char c) {
return 0;
}
+inline int dyncfg_id_isspace(char c) {
+ if(c == ':')
+ return 1;
+
+ return 0;
+}
+
bool isspace_map_pluginsd[256] = {};
bool isspace_map_config[256] = {};
bool isspace_map_group_by_label[256] = {};
+bool isspace_dyncfg_id_map[256] = {};
__attribute__((constructor)) void initialize_is_space_arrays(void) {
for(int c = 0; c < 256 ; c++) {
isspace_map_pluginsd[c] = pluginsd_isspace((char) c);
isspace_map_config[c] = config_isspace((char) c);
isspace_map_group_by_label[c] = group_by_label_isspace((char) c);
+ isspace_dyncfg_id_map[c] = dyncfg_id_isspace((char)c);
}
}
diff --git a/libnetdata/line_splitter/line_splitter.h b/src/libnetdata/line_splitter/line_splitter.h
index b5a59ad3a..968930410 100644
--- a/libnetdata/line_splitter/line_splitter.h
+++ b/src/libnetdata/line_splitter/line_splitter.h
@@ -22,10 +22,12 @@ static inline void line_splitter_reset(struct line_splitter *line) {
int pluginsd_isspace(char c);
int config_isspace(char c);
int group_by_label_isspace(char c);
+int dyncfg_id_isspace(char c);
extern bool isspace_map_pluginsd[256];
extern bool isspace_map_config[256];
extern bool isspace_map_group_by_label[256];
+extern bool isspace_dyncfg_id_map[256];
static inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words, bool *isspace_map) {
char *s = str, quote = 0;
@@ -110,6 +112,9 @@ static inline size_t quoted_strings_splitter(char *str, char **words, size_t max
#define quoted_strings_splitter_pluginsd(str, words, max_words) \
quoted_strings_splitter(str, words, max_words, isspace_map_pluginsd)
+#define quoted_strings_splitter_dyncfg_id(str, words, max_words) \
+ quoted_strings_splitter(str, words, max_words, isspace_dyncfg_id_map)
+
static inline char *get_word(char **words, size_t num_words, size_t index) {
if (unlikely(index >= num_words))
return NULL;
diff --git a/libnetdata/locks/locks.c b/src/libnetdata/locks/locks.c
index 625dd052c..d01ee29f1 100644
--- a/libnetdata/locks/locks.c
+++ b/src/libnetdata/locks/locks.c
@@ -19,64 +19,6 @@
#endif // NETDATA_TRACE_RWLOCKS
// ----------------------------------------------------------------------------
-// automatic thread cancelability management, based on locks
-
-static __thread int netdata_thread_first_cancelability = 0;
-static __thread int netdata_thread_nested_disables = 0;
-
-static __thread size_t netdata_locks_acquired_rwlocks = 0;
-static __thread size_t netdata_locks_acquired_mutexes = 0;
-
-inline void netdata_thread_disable_cancelability(void) {
- if(!netdata_thread_nested_disables) {
- int old;
- int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old);
-
- if(ret != 0)
- netdata_log_error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d",
- netdata_thread_tag(), ret);
-
- netdata_thread_first_cancelability = old;
- }
-
- netdata_thread_nested_disables++;
-}
-
-inline void netdata_thread_enable_cancelability(void) {
- if(unlikely(netdata_thread_nested_disables < 1)) {
- internal_fatal(true, "THREAD_CANCELABILITY: trying to enable cancelability, but it was not not disabled");
-
- netdata_log_error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): invalid thread cancelability count %d "
- "on thread %s - results will be undefined - please report this!",
- netdata_thread_nested_disables, netdata_thread_tag());
-
- netdata_thread_nested_disables = 1;
- }
-
- if(netdata_thread_nested_disables == 1) {
- int old = 1;
- int ret = pthread_setcancelstate(netdata_thread_first_cancelability, &old);
- if(ret != 0)
- netdata_log_error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d",
- netdata_thread_tag(),
- ret);
- else {
- if(old != PTHREAD_CANCEL_DISABLE) {
- internal_fatal(true, "THREAD_CANCELABILITY: invalid old state cancelability");
-
- netdata_log_error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): old thread cancelability "
- "on thread %s was changed, expected DISABLED (%d), found %s (%d) - please report this!",
- netdata_thread_tag(), PTHREAD_CANCEL_DISABLE,
- (old == PTHREAD_CANCEL_ENABLE) ? "ENABLED" : "UNKNOWN",
- old);
- }
- }
- }
-
- netdata_thread_nested_disables--;
-}
-
-// ----------------------------------------------------------------------------
// mutex
int __netdata_mutex_init(netdata_mutex_t *mutex) {
@@ -94,27 +36,22 @@ int __netdata_mutex_destroy(netdata_mutex_t *mutex) {
}
int __netdata_mutex_lock(netdata_mutex_t *mutex) {
- netdata_thread_disable_cancelability();
-
int ret = pthread_mutex_lock(mutex);
if(unlikely(ret != 0)) {
- netdata_thread_enable_cancelability();
netdata_log_error("MUTEX_LOCK: failed to get lock (code %d)", ret);
}
else
- netdata_locks_acquired_mutexes++;
+ nd_thread_mutex_locked();
return ret;
}
int __netdata_mutex_trylock(netdata_mutex_t *mutex) {
- netdata_thread_disable_cancelability();
-
int ret = pthread_mutex_trylock(mutex);
if(ret != 0)
- netdata_thread_enable_cancelability();
+ ;
else
- netdata_locks_acquired_mutexes++;
+ nd_thread_mutex_locked();
return ret;
}
@@ -123,10 +60,8 @@ int __netdata_mutex_unlock(netdata_mutex_t *mutex) {
int ret = pthread_mutex_unlock(mutex);
if(unlikely(ret != 0))
netdata_log_error("MUTEX_LOCK: failed to unlock (code %d).", ret);
- else {
- netdata_locks_acquired_mutexes--;
- netdata_thread_enable_cancelability();
- }
+ else
+ nd_thread_mutex_unlocked();
return ret;
}
@@ -226,65 +161,61 @@ int __netdata_rwlock_init(netdata_rwlock_t *rwlock) {
}
int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) {
- netdata_thread_disable_cancelability();
-
int ret = pthread_rwlock_rdlock(&rwlock->rwlock_t);
- if(unlikely(ret != 0)) {
- netdata_thread_enable_cancelability();
+ if(unlikely(ret != 0))
netdata_log_error("RW_LOCK: failed to obtain read lock (code %d)", ret);
- }
else
- netdata_locks_acquired_rwlocks++;
+ nd_thread_rwlock_read_locked();
return ret;
}
int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) {
- netdata_thread_disable_cancelability();
-
int ret = pthread_rwlock_wrlock(&rwlock->rwlock_t);
- if(unlikely(ret != 0)) {
+ if(unlikely(ret != 0))
netdata_log_error("RW_LOCK: failed to obtain write lock (code %d)", ret);
- netdata_thread_enable_cancelability();
- }
else
- netdata_locks_acquired_rwlocks++;
+ nd_thread_rwlock_write_locked();
return ret;
}
-int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock) {
+int __netdata_rwlock_rdunlock(netdata_rwlock_t *rwlock) {
int ret = pthread_rwlock_unlock(&rwlock->rwlock_t);
if(unlikely(ret != 0))
netdata_log_error("RW_LOCK: failed to release lock (code %d)", ret);
- else {
- netdata_thread_enable_cancelability();
- netdata_locks_acquired_rwlocks--;
- }
+ else
+ nd_thread_rwlock_read_unlocked();
return ret;
}
-int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) {
- netdata_thread_disable_cancelability();
+int __netdata_rwlock_wrunlock(netdata_rwlock_t *rwlock) {
+ int ret = pthread_rwlock_unlock(&rwlock->rwlock_t);
+ if(unlikely(ret != 0))
+ netdata_log_error("RW_LOCK: failed to release lock (code %d)", ret);
+ else
+ nd_thread_rwlock_write_unlocked();
+ return ret;
+}
+
+int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) {
int ret = pthread_rwlock_tryrdlock(&rwlock->rwlock_t);
if(ret != 0)
- netdata_thread_enable_cancelability();
+ ;
else
- netdata_locks_acquired_rwlocks++;
+ nd_thread_rwlock_read_locked();
return ret;
}
int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) {
- netdata_thread_disable_cancelability();
-
int ret = pthread_rwlock_trywrlock(&rwlock->rwlock_t);
if(ret != 0)
- netdata_thread_enable_cancelability();
+ ;
else
- netdata_locks_acquired_rwlocks++;
+ nd_thread_rwlock_write_locked();
return ret;
}
@@ -297,15 +228,11 @@ void spinlock_init(SPINLOCK *spinlock) {
memset(spinlock, 0, sizeof(SPINLOCK));
}
-void spinlock_lock(SPINLOCK *spinlock) {
- static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };
-
+static inline void spinlock_lock_internal(SPINLOCK *spinlock) {
#ifdef NETDATA_INTERNAL_CHECKS
size_t spins = 0;
#endif
- netdata_thread_disable_cancelability();
-
for(int i = 1;
__atomic_load_n(&spinlock->locked, __ATOMIC_RELAXED) ||
__atomic_test_and_set(&spinlock->locked, __ATOMIC_ACQUIRE)
@@ -317,7 +244,7 @@ void spinlock_lock(SPINLOCK *spinlock) {
#endif
if(unlikely(i == 8)) {
i = 0;
- nanosleep(&ns, NULL);
+ tinysleep();
}
}
@@ -325,31 +252,62 @@ void spinlock_lock(SPINLOCK *spinlock) {
#ifdef NETDATA_INTERNAL_CHECKS
spinlock->spins += spins;
- spinlock->locker_pid = gettid();
+ spinlock->locker_pid = gettid_cached();
#endif
+
+ nd_thread_spinlock_locked();
}
-void spinlock_unlock(SPINLOCK *spinlock) {
+static inline void spinlock_unlock_internal(SPINLOCK *spinlock) {
#ifdef NETDATA_INTERNAL_CHECKS
spinlock->locker_pid = 0;
#endif
__atomic_clear(&spinlock->locked, __ATOMIC_RELEASE);
- netdata_thread_enable_cancelability();
-}
-bool spinlock_trylock(SPINLOCK *spinlock) {
- netdata_thread_disable_cancelability();
+ nd_thread_spinlock_unlocked();
+}
+static inline bool spinlock_trylock_internal(SPINLOCK *spinlock) {
if(!__atomic_load_n(&spinlock->locked, __ATOMIC_RELAXED) &&
- !__atomic_test_and_set(&spinlock->locked, __ATOMIC_ACQUIRE))
+ !__atomic_test_and_set(&spinlock->locked, __ATOMIC_ACQUIRE)) {
// we got the lock
+ nd_thread_spinlock_locked();
return true;
+ }
- // we didn't get the lock
- netdata_thread_enable_cancelability();
return false;
}
+void spinlock_lock(SPINLOCK *spinlock)
+{
+ spinlock_lock_internal(spinlock);
+}
+
+void spinlock_unlock(SPINLOCK *spinlock)
+{
+ spinlock_unlock_internal(spinlock);
+}
+
+bool spinlock_trylock(SPINLOCK *spinlock)
+{
+ return spinlock_trylock_internal(spinlock);
+}
+
+void spinlock_lock_cancelable(SPINLOCK *spinlock)
+{
+ spinlock_lock_internal(spinlock);
+}
+
+void spinlock_unlock_cancelable(SPINLOCK *spinlock)
+{
+ spinlock_unlock_internal(spinlock);
+}
+
+bool spinlock_trylock_cancelable(SPINLOCK *spinlock)
+{
+ return spinlock_trylock_internal(spinlock);
+}
+
// ----------------------------------------------------------------------------
// rw_spinlock implementation
@@ -359,11 +317,11 @@ void rw_spinlock_init(RW_SPINLOCK *rw_spinlock) {
}
void rw_spinlock_read_lock(RW_SPINLOCK *rw_spinlock) {
- netdata_thread_disable_cancelability();
-
spinlock_lock(&rw_spinlock->spinlock);
__atomic_add_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED);
spinlock_unlock(&rw_spinlock->spinlock);
+
+ nd_thread_rwspinlock_read_locked();
}
void rw_spinlock_read_unlock(RW_SPINLOCK *rw_spinlock) {
@@ -375,12 +333,10 @@ void rw_spinlock_read_unlock(RW_SPINLOCK *rw_spinlock) {
fatal("RW_SPINLOCK: readers is negative %d", x);
#endif
- netdata_thread_enable_cancelability();
+ nd_thread_rwspinlock_read_unlocked();
}
void rw_spinlock_write_lock(RW_SPINLOCK *rw_spinlock) {
- static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };
-
size_t spins = 0;
while(1) {
spins++;
@@ -391,21 +347,24 @@ void rw_spinlock_write_lock(RW_SPINLOCK *rw_spinlock) {
// Busy wait until all readers have released their locks.
spinlock_unlock(&rw_spinlock->spinlock);
- nanosleep(&ns, NULL);
+ tinysleep();
}
(void)spins;
+
+ nd_thread_rwspinlock_write_locked();
}
void rw_spinlock_write_unlock(RW_SPINLOCK *rw_spinlock) {
spinlock_unlock(&rw_spinlock->spinlock);
+ nd_thread_rwspinlock_write_unlocked();
}
bool rw_spinlock_tryread_lock(RW_SPINLOCK *rw_spinlock) {
if(spinlock_trylock(&rw_spinlock->spinlock)) {
__atomic_add_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED);
spinlock_unlock(&rw_spinlock->spinlock);
- netdata_thread_disable_cancelability();
+ nd_thread_rwspinlock_read_locked();
return true;
}
@@ -416,6 +375,7 @@ bool rw_spinlock_trywrite_lock(RW_SPINLOCK *rw_spinlock) {
if(spinlock_trylock(&rw_spinlock->spinlock)) {
if (__atomic_load_n(&rw_spinlock->readers, __ATOMIC_RELAXED) == 0) {
// No readers, we've successfully acquired the write lock
+ nd_thread_rwspinlock_write_locked();
return true;
}
else {
@@ -548,7 +508,22 @@ int netdata_rwlock_wrlock_debug(const char *file __maybe_unused, const char *fun
return ret;
}
-int netdata_rwlock_unlock_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+int netdata_rwlock_rdunlock_debug(const char *file __maybe_unused, const char *function __maybe_unused,
+ const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) {
+
+ netdata_rwlock_locker *locker = find_rwlock_locker(file, function, line, rwlock);
+
+ if(unlikely(!locker))
+ fatal("UNLOCK WITHOUT LOCK");
+
+ int ret = __netdata_rwlock_rdunlock(rwlock);
+ if(likely(!ret))
+ remove_rwlock_locker(file, function, line, rwlock, locker);
+
+ return ret;
+}
+
+int netdata_rwlock_wrunlock_debug(const char *file __maybe_unused, const char *function __maybe_unused,
const unsigned long line __maybe_unused, netdata_rwlock_t *rwlock) {
netdata_rwlock_locker *locker = find_rwlock_locker(file, function, line, rwlock);
@@ -556,7 +531,7 @@ int netdata_rwlock_unlock_debug(const char *file __maybe_unused, const char *fun
if(unlikely(!locker))
fatal("UNLOCK WITHOUT LOCK");
- int ret = __netdata_rwlock_unlock(rwlock);
+ int ret = __netdata_rwlock_wrunlock(rwlock);
if(likely(!ret))
remove_rwlock_locker(file, function, line, rwlock, locker);
diff --git a/libnetdata/locks/locks.h b/src/libnetdata/locks/locks.h
index 6b492ae47..d3873c295 100644
--- a/libnetdata/locks/locks.h
+++ b/src/libnetdata/locks/locks.h
@@ -25,6 +25,10 @@ void spinlock_lock(SPINLOCK *spinlock);
void spinlock_unlock(SPINLOCK *spinlock);
bool spinlock_trylock(SPINLOCK *spinlock);
+void spinlock_lock_cancelable(SPINLOCK *spinlock);
+void spinlock_unlock_cancelable(SPINLOCK *spinlock);
+bool spinlock_trylock_cancelable(SPINLOCK *spinlock);
+
typedef struct netdata_rw_spinlock {
int32_t readers;
SPINLOCK spinlock;
@@ -102,13 +106,11 @@ int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock);
int __netdata_rwlock_init(netdata_rwlock_t *rwlock);
int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock);
int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock);
-int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock);
+int __netdata_rwlock_rdunlock(netdata_rwlock_t *rwlock);
+int __netdata_rwlock_wrunlock(netdata_rwlock_t *rwlock);
int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock);
int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock);
-void netdata_thread_disable_cancelability(void);
-void netdata_thread_enable_cancelability(void);
-
#ifdef NETDATA_TRACE_RWLOCKS
int netdata_mutex_init_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex);
@@ -121,7 +123,8 @@ int netdata_rwlock_destroy_debug( const char *file, const char *function, const
int netdata_rwlock_init_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
int netdata_rwlock_rdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
int netdata_rwlock_wrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
-int netdata_rwlock_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+int netdata_rwlock_rdunlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
+int netdata_rwlock_wrunlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
int netdata_rwlock_tryrdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
int netdata_rwlock_trywrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock);
@@ -135,7 +138,8 @@ int netdata_rwlock_trywrlock_debug( const char *file, const char *function, cons
#define netdata_rwlock_init(rwlock) netdata_rwlock_init_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
#define netdata_rwlock_rdlock(rwlock) netdata_rwlock_rdlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
#define netdata_rwlock_wrlock(rwlock) netdata_rwlock_wrlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
-#define netdata_rwlock_unlock(rwlock) netdata_rwlock_unlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
+#define netdata_rwlock_rdunlock(rwlock) netdata_rwlock_rdunlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
+#define netdata_rwlock_wrunlock(rwlock) netdata_rwlock_wrunlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
#define netdata_rwlock_tryrdlock(rwlock) netdata_rwlock_tryrdlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
#define netdata_rwlock_trywrlock(rwlock) netdata_rwlock_trywrlock_debug(__FILE__, __FUNCTION__, __LINE__, rwlock)
@@ -151,7 +155,8 @@ int netdata_rwlock_trywrlock_debug( const char *file, const char *function, cons
#define netdata_rwlock_init(rwlock) __netdata_rwlock_init(rwlock)
#define netdata_rwlock_rdlock(rwlock) __netdata_rwlock_rdlock(rwlock)
#define netdata_rwlock_wrlock(rwlock) __netdata_rwlock_wrlock(rwlock)
-#define netdata_rwlock_unlock(rwlock) __netdata_rwlock_unlock(rwlock)
+#define netdata_rwlock_rdunlock(rwlock) __netdata_rwlock_rdunlock(rwlock)
+#define netdata_rwlock_wrunlock(rwlock) __netdata_rwlock_wrunlock(rwlock)
#define netdata_rwlock_tryrdlock(rwlock) __netdata_rwlock_tryrdlock(rwlock)
#define netdata_rwlock_trywrlock(rwlock) __netdata_rwlock_trywrlock(rwlock)
diff --git a/libnetdata/log/journal.c b/src/libnetdata/log/journal.c
index 21978cf5f..2182212f6 100644
--- a/libnetdata/log/journal.c
+++ b/src/libnetdata/log/journal.c
@@ -3,15 +3,12 @@
#include "journal.h"
bool is_path_unix_socket(const char *path) {
+ // Check if the path is valid
if(!path || !*path)
return false;
struct stat statbuf;
- // Check if the path is valid
- if (!path || !*path)
- return false;
-
// Use stat to check if the file exists and is a socket
if (stat(path, &statbuf) == -1)
// The file does not exist or cannot be accessed
@@ -51,9 +48,11 @@ int journal_direct_fd(const char *path) {
if(!is_path_unix_socket(path))
return -1;
- int fd = socket(AF_UNIX, SOCK_DGRAM, 0);
+ int fd = socket(AF_UNIX, SOCK_DGRAM| DEFAULT_SOCKET_FLAGS, 0);
if (fd < 0) return -1;
+ sock_setcloexec(fd);
+
struct sockaddr_un addr;
memset(&addr, 0, sizeof(struct sockaddr_un));
addr.sun_family = AF_UNIX;
@@ -68,7 +67,7 @@ int journal_direct_fd(const char *path) {
return fd;
}
-static inline bool journal_send_with_memfd(int fd, const char *msg, size_t msg_len) {
+static inline bool journal_send_with_memfd(int fd __maybe_unused, const char *msg __maybe_unused, size_t msg_len __maybe_unused) {
#if defined(__NR_memfd_create) && defined(MFD_ALLOW_SEALING) && defined(F_ADD_SEALS) && defined(F_SEAL_SHRINK) && defined(F_SEAL_GROW) && defined(F_SEAL_WRITE)
// Create a memory file descriptor
int memfd = (int)syscall(__NR_memfd_create, "journald", MFD_ALLOW_SEALING);
@@ -97,6 +96,11 @@ static inline bool journal_send_with_memfd(int fd, const char *msg, size_t msg_l
msghdr.msg_controllen = sizeof(cmsgbuf);
cmsghdr = CMSG_FIRSTHDR(&msghdr);
+ if(!cmsghdr) {
+ close(memfd);
+ return false;
+ }
+
cmsghdr->cmsg_level = SOL_SOCKET;
cmsghdr->cmsg_type = SCM_RIGHTS;
cmsghdr->cmsg_len = CMSG_LEN(sizeof(int));
diff --git a/libnetdata/log/journal.h b/src/libnetdata/log/journal.h
index df8ece18b..df8ece18b 100644
--- a/libnetdata/log/journal.h
+++ b/src/libnetdata/log/journal.h
diff --git a/libnetdata/log/systemd-cat-native.c b/src/libnetdata/log/systemd-cat-native.c
index de6211cc0..74d3728a3 100644
--- a/libnetdata/log/systemd-cat-native.c
+++ b/src/libnetdata/log/systemd-cat-native.c
@@ -11,7 +11,7 @@
#include <machine/endian.h>
#endif
-static void log_message_to_stderr(BUFFER *msg) {
+static inline void log_message_to_stderr(BUFFER *msg) {
CLEAN_BUFFER *tmp = buffer_create(0, NULL);
for(size_t i = 0; i < msg->len ;i++) {
@@ -436,7 +436,7 @@ cleanup:
static int help(void) {
fprintf(stderr,
"\n"
- "Netdata systemd-cat-native " PACKAGE_VERSION "\n"
+ "Netdata systemd-cat-native " NETDATA_VERSION "\n"
"\n"
"This program reads from its standard input, lines in the format:\n"
"\n"
@@ -594,7 +594,7 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) {
// an empty line - we are done for this message
nd_log(NDLS_HEALTH, priority,
- "added %d fields", // if the user supplied a MESSAGE, this will be ignored
+ "added %zu fields", // if the user supplied a MESSAGE, this will be ignored
fields_added);
lgs_reset(lgs);
@@ -627,7 +627,7 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) {
nd_log(NDLS_COLLECTORS, NDLP_ERR,
"Field '%.*s' is not a Netdata field. Ignoring it.",
- field_len, field);
+ (int)field_len, field);
lgs[NDF_MESSAGE] = backup;
}
@@ -648,7 +648,7 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) {
}
if(fields_added) {
- nd_log(NDLS_HEALTH, priority, "added %d fields", fields_added);
+ nd_log(NDLS_HEALTH, priority, "added %zu fields", fields_added);
messages_logged++;
}
diff --git a/libnetdata/log/systemd-cat-native.h b/src/libnetdata/log/systemd-cat-native.h
index 34e7a3615..34e7a3615 100644
--- a/libnetdata/log/systemd-cat-native.h
+++ b/src/libnetdata/log/systemd-cat-native.h
diff --git a/libnetdata/log/systemd-cat-native.md b/src/libnetdata/log/systemd-cat-native.md
index b0b15f403..b0b15f403 100644
--- a/libnetdata/log/systemd-cat-native.md
+++ b/src/libnetdata/log/systemd-cat-native.md
diff --git a/libnetdata/onewayalloc/onewayalloc.c b/src/libnetdata/onewayalloc/onewayalloc.c
index 05c9f2a9d..98ab1835b 100644
--- a/libnetdata/onewayalloc/onewayalloc.c
+++ b/src/libnetdata/onewayalloc/onewayalloc.c
@@ -178,8 +178,6 @@ void onewayalloc_freez(ONEWAYALLOC *owa __maybe_unused, const void *ptr __maybe_
// let's free it with the system allocator
netdata_log_error("ONEWAYALLOC: request to free address 0x%p that is not allocated by this OWA", ptr);
#endif
-
- return;
}
void *onewayalloc_doublesize(ONEWAYALLOC *owa, const void *src, size_t oldsize) {
diff --git a/libnetdata/onewayalloc/onewayalloc.h b/src/libnetdata/onewayalloc/onewayalloc.h
index a415b063b..a415b063b 100644
--- a/libnetdata/onewayalloc/onewayalloc.h
+++ b/src/libnetdata/onewayalloc/onewayalloc.h
diff --git a/libnetdata/popen/popen.c b/src/libnetdata/popen/popen.c
index 5f8bd2b4a..c1721e9b4 100644
--- a/libnetdata/popen/popen.c
+++ b/src/libnetdata/popen/popen.c
@@ -98,7 +98,7 @@ int netdata_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options) {
}
else {
// we haven't reaped this child yet
- ret = waitid(idtype, id, infop, options);
+ ret = os_waitid(idtype, id, infop, options);
if(mp && !mp->reaped) {
mp->reaped = true;
diff --git a/libnetdata/popen/popen.h b/src/libnetdata/popen/popen.h
index 4f86158bc..8f46abbc8 100644
--- a/libnetdata/popen/popen.h
+++ b/src/libnetdata/popen/popen.h
@@ -3,6 +3,9 @@
#ifndef NETDATA_POPEN_H
#define NETDATA_POPEN_H 1
+#include "../os/waitid.h"
+int netdata_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options);
+
#include "../libnetdata.h"
#define PIPE_READ 0
@@ -28,6 +31,5 @@ int netdata_popene_variadic_internal_dont_use_directly(volatile pid_t *pidptr, c
int netdata_pclose(FILE *fp_child_input, FILE *fp_child_output, pid_t pid);
int netdata_spawn(const char *command, volatile pid_t *pidptr);
-int netdata_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options);
#endif /* NETDATA_POPEN_H */
diff --git a/libnetdata/procfile/procfile.c b/src/libnetdata/procfile/procfile.c
index 0bc731d68..d9ebf4c93 100644
--- a/libnetdata/procfile/procfile.c
+++ b/src/libnetdata/procfile/procfile.c
@@ -8,7 +8,7 @@
#define PFLINES_INCREASE_STEP 200
#define PROCFILE_INCREMENT_BUFFER 4096
-int procfile_open_flags = O_RDONLY;
+int procfile_open_flags = O_RDONLY | O_CLOEXEC;
int procfile_adaptive_initial_allocation = 0;
diff --git a/libnetdata/procfile/procfile.h b/src/libnetdata/procfile/procfile.h
index 8db5b45f4..8db5b45f4 100644
--- a/libnetdata/procfile/procfile.h
+++ b/src/libnetdata/procfile/procfile.h
diff --git a/libnetdata/simple_pattern/simple_pattern.c b/src/libnetdata/simple_pattern/simple_pattern.c
index a0051e8f0..7a7f41b1c 100644
--- a/libnetdata/simple_pattern/simple_pattern.c
+++ b/src/libnetdata/simple_pattern/simple_pattern.c
@@ -147,7 +147,7 @@ SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators,
if(default_mode == SIMPLE_PATTERN_SUBSTRING) {
m->mode = SIMPLE_PATTERN_SUBSTRING;
- struct simple_pattern *tm = m;
+ struct simple_pattern *tm;
for(tm = m; tm->child ; tm = tm->child) ;
tm->mode = SIMPLE_PATTERN_SUBSTRING;
}
@@ -396,27 +396,6 @@ extern int simple_pattern_is_potential_name(SIMPLE_PATTERN *p)
return (alpha || wildcards) && !colon;
}
-char *simple_pattern_trim_around_equal(char *src) {
- char *store = mallocz(strlen(src) + 1);
-
- char *dst = store;
- while (*src) {
- if (*src == '=') {
- if (*(dst -1) == ' ')
- dst--;
-
- *dst++ = *src++;
- if (*src == ' ')
- src++;
- }
-
- *dst++ = *src++;
- }
- *dst = 0x00;
-
- return store;
-}
-
char *simple_pattern_iterate(SIMPLE_PATTERN **p)
{
struct simple_pattern *root = (struct simple_pattern *) *p;
diff --git a/libnetdata/simple_pattern/simple_pattern.h b/src/libnetdata/simple_pattern/simple_pattern.h
index 1a8d8f7d6..1af0f87b9 100644
--- a/libnetdata/simple_pattern/simple_pattern.h
+++ b/src/libnetdata/simple_pattern/simple_pattern.h
@@ -5,7 +5,6 @@
#include "../libnetdata.h"
-
typedef enum __attribute__ ((__packed__)) {
SIMPLE_PATTERN_EXACT,
SIMPLE_PATTERN_PREFIX,
@@ -19,7 +18,8 @@ typedef enum __attribute__ ((__packed__)) {
SP_MATCHED_POSITIVE,
} SIMPLE_PATTERN_RESULT;
-typedef void SIMPLE_PATTERN;
+struct simple_pattern;
+typedef struct simple_pattern SIMPLE_PATTERN;
// create a simple_pattern from the string given
// default_mode is used in cases where EXACT matches, without an asterisk,
@@ -47,9 +47,6 @@ void simple_pattern_dump(uint64_t debug_type, SIMPLE_PATTERN *p) ;
int simple_pattern_is_potential_name(SIMPLE_PATTERN *p) ;
char *simple_pattern_iterate(SIMPLE_PATTERN **p);
-// Auxiliary function to create a pattern
-char *simple_pattern_trim_around_equal(char *src);
-
#define SIMPLE_PATTERN_DEFAULT_WEB_SEPARATORS ",|\t\r\n\f\v"
#define is_valid_sp(x) ((x) && *(x) && !((x)[0] == '*' && (x)[1] == '\0'))
diff --git a/libnetdata/socket/security.c b/src/libnetdata/socket/security.c
index 4deb76623..502998b79 100644
--- a/libnetdata/socket/security.c
+++ b/src/libnetdata/socket/security.c
@@ -233,6 +233,19 @@ static inline bool is_handshake_complete(NETDATA_SSL *ssl, const char *op) {
* (These are often the same value, but can be different on some systems.)
*/
+ssize_t netdata_ssl_pending(NETDATA_SSL *ssl) {
+ return SSL_pending(ssl->conn);
+}
+
+bool netdata_ssl_has_pending(NETDATA_SSL *ssl) {
+ // this call was added on OpenSSL 1.1.0
+ // however, it is more accurate than SSL_pending()
+ // unfortunately it does not exists in libressl.
+ // return SSL_has_pending(ssl->conn);
+
+ return SSL_pending(ssl->conn) > 0;
+}
+
ssize_t netdata_ssl_read(NETDATA_SSL *ssl, void *buf, size_t num) {
errno = 0;
ssl->ssl_errno = 0;
diff --git a/libnetdata/socket/security.h b/src/libnetdata/socket/security.h
index fd17b6f3f..283d81db8 100644
--- a/libnetdata/socket/security.h
+++ b/src/libnetdata/socket/security.h
@@ -39,7 +39,7 @@ typedef struct netdata_ssl {
unsigned long ssl_errno; // The SSL errno of the last SSL call
} NETDATA_SSL;
-#define NETDATA_SSL_UNSET_CONNECTION (NETDATA_SSL){ .conn = NULL, .state = NETDATA_SSL_STATE_NOT_SSL }
+#define NETDATA_SSL_UNSET_CONNECTION (NETDATA_SSL){ .conn = NULL, .state = NETDATA_SSL_STATE_NOT_SSL, .ssl_errno = 0 }
#define SSL_connection(ssl) ((ssl)->conn && (ssl)->state != NETDATA_SSL_STATE_NOT_SSL)
@@ -70,5 +70,8 @@ void netdata_ssl_close(NETDATA_SSL *ssl);
ssize_t netdata_ssl_read(NETDATA_SSL *ssl, void *buf, size_t num);
ssize_t netdata_ssl_write(NETDATA_SSL *ssl, const void *buf, size_t num);
+ssize_t netdata_ssl_pending(NETDATA_SSL *ssl);
+bool netdata_ssl_has_pending(NETDATA_SSL *ssl);
+
# endif //ENABLE_HTTPS
#endif //NETDATA_SECURITY_H
diff --git a/libnetdata/statistical/statistical.c b/src/libnetdata/statistical/statistical.c
index ef9fe4e56..ef9fe4e56 100644
--- a/libnetdata/statistical/statistical.c
+++ b/src/libnetdata/statistical/statistical.c
diff --git a/libnetdata/statistical/statistical.h b/src/libnetdata/statistical/statistical.h
index f3ecfadb4..f3ecfadb4 100644
--- a/libnetdata/statistical/statistical.h
+++ b/src/libnetdata/statistical/statistical.h
diff --git a/libnetdata/storage_number/storage_number.c b/src/libnetdata/storage_number/storage_number.c
index 6468951bd..89a67a532 100644
--- a/libnetdata/storage_number/storage_number.c
+++ b/src/libnetdata/storage_number/storage_number.c
@@ -147,13 +147,7 @@ storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) {
r += (m << 27); // the divider m
}
-#ifdef STORAGE_WITH_MATH
- // without this there are rounding problems
- // example: 0.9 becomes 0.89
r += lrint((double) n);
-#else
- r += (storage_number)n;
-#endif
return r;
}
@@ -174,60 +168,3 @@ __attribute__((constructor)) void initialize_lut(void) {
unpack_storage_number_lut10x[3 * 8 + i] = pow(100, i); // exp = 1
}
}
-
-/*
-int print_netdata_double(char *str, NETDATA_DOUBLE value)
-{
- char *wstr = str;
-
- int sign = (value < 0) ? 1 : 0;
- if(sign) value = -value;
-
-#ifdef STORAGE_WITH_MATH
- // without llrintl() there are rounding problems
- // for example 0.9 becomes 0.89
- unsigned long long uvalue = (unsigned long long int) llrintl(value * (NETDATA_DOUBLE)100000);
-#else
- unsigned long long uvalue = value * (NETDATA_DOUBLE)100000;
-#endif
-
- wstr = print_number_llu_r_smart(str, uvalue);
-
- // make sure we have 6 bytes at least
- while((wstr - str) < 6) *wstr++ = '0';
-
- // put the sign back
- if(sign) *wstr++ = '-';
-
- // reverse it
- char *begin = str, *end = --wstr, aux;
- while (end > begin) aux = *end, *end-- = *begin, *begin++ = aux;
- // wstr--;
- // strreverse(str, wstr);
-
- // remove trailing zeros
- int decimal = 5;
- while(decimal > 0 && *wstr == '0') {
- *wstr-- = '\0';
- decimal--;
- }
-
- // terminate it, one position to the right
- // to let space for a dot
- wstr[2] = '\0';
-
- // make space for the dot
- int i;
- for(i = 0; i < decimal ;i++) {
- wstr[1] = wstr[0];
- wstr--;
- }
-
- // put the dot
- if(wstr[2] == '\0') { wstr[1] = '\0'; decimal--; }
- else wstr[1] = '.';
-
- // return the buffer length
- return (int) ((wstr - str) + 2 + decimal );
-}
-*/
diff --git a/libnetdata/storage_number/storage_number.h b/src/libnetdata/storage_number/storage_number.h
index 82c870d69..9a95203cd 100644
--- a/libnetdata/storage_number/storage_number.h
+++ b/src/libnetdata/storage_number/storage_number.h
@@ -116,10 +116,10 @@ storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) __attri
static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) __attribute__((const));
// sign div/mul <--- multiplier / divider ---> 10/100 RESET EXISTS VALUE
-#define STORAGE_NUMBER_POSITIVE_MAX_RAW (storage_number)( (0 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1 << 27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff )
-#define STORAGE_NUMBER_POSITIVE_MIN_RAW (storage_number)( (0 << 31) | (0 << 30) | (1 << 29) | (1 << 28) | (1 << 27) | (0 << 26) | (0 << 25) | (1 << 24) | 0x00000001 )
-#define STORAGE_NUMBER_NEGATIVE_MAX_RAW (storage_number)( (1 << 31) | (0 << 30) | (1 << 29) | (1 << 28) | (1 << 27) | (0 << 26) | (0 << 25) | (1 << 24) | 0x00000001 )
-#define STORAGE_NUMBER_NEGATIVE_MIN_RAW (storage_number)( (1 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1 << 27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff )
+#define STORAGE_NUMBER_POSITIVE_MAX_RAW (storage_number)( (0U << 31) | (1U << 30) | (1U << 29) | (1U << 28) | (1U << 27) | (1U << 26) | (0U << 25) | (1U << 24) | 0x00ffffff )
+#define STORAGE_NUMBER_POSITIVE_MIN_RAW (storage_number)( (0U << 31) | (0U << 30) | (1U << 29) | (1U << 28) | (1U << 27) | (0U << 26) | (0U << 25) | (1U << 24) | 0x00000001 )
+#define STORAGE_NUMBER_NEGATIVE_MAX_RAW (storage_number)( (1U << 31) | (0U << 30) | (1U << 29) | (1U << 28) | (1U << 27) | (0U << 26) | (0U << 25) | (1U << 24) | 0x00000001 )
+#define STORAGE_NUMBER_NEGATIVE_MIN_RAW (storage_number)( (1U << 31) | (1U << 30) | (1U << 29) | (1U << 28) | (1U << 27) | (1U << 26) | (0U << 25) | (1U << 24) | 0x00ffffff )
// accepted accuracy loss
#define ACCURACY_LOSS_ACCEPTED_PERCENT 0.0001
@@ -155,10 +155,10 @@ static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) {
// bit 25 SN_FLAG_NOT_ANOMALOUS
// bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
- int mul = (int)((value & ((1<<29)|(1<<28)|(1<<27))) >> 27);
+ int mul = (int)((value & ((1U<<29)|(1U<<28)|(1U<<27))) >> 27);
// bit 24 to bit 1 = the value, so remove all other bits
- value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24));
+ value ^= value & ((1U <<31)|(1U <<30)|(1U <<29)|(1U <<28)|(1U <<27)|(1U <<26)|(1U <<25)|(1U<<24));
NETDATA_DOUBLE n = value;
diff --git a/libnetdata/storage_number/tests/test_storage_number.c b/src/libnetdata/storage_number/tests/test_storage_number.c
index 19309e5c2..19309e5c2 100644
--- a/libnetdata/storage_number/tests/test_storage_number.c
+++ b/src/libnetdata/storage_number/tests/test_storage_number.c
diff --git a/libnetdata/string/utf8.h b/src/libnetdata/string/utf8.h
index 3e6c8c288..3e6c8c288 100644
--- a/libnetdata/string/utf8.h
+++ b/src/libnetdata/string/utf8.h
diff --git a/libnetdata/tests/test_str2ld.c b/src/libnetdata/tests/test_str2ld.c
index 8b97a70f8..8b97a70f8 100644
--- a/libnetdata/tests/test_str2ld.c
+++ b/src/libnetdata/tests/test_str2ld.c
diff --git a/libnetdata/worker_utilization/worker_utilization.c b/src/libnetdata/worker_utilization/worker_utilization.c
index f39cea8a0..4c61ea921 100644
--- a/libnetdata/worker_utilization/worker_utilization.c
+++ b/src/libnetdata/worker_utilization/worker_utilization.c
@@ -92,8 +92,8 @@ void worker_register(const char *name) {
return;
worker = callocz(1, sizeof(struct worker));
- worker->pid = gettid();
- worker->tag = strdupz(netdata_thread_tag());
+ worker->pid = gettid_cached();
+ worker->tag = strdupz(nd_thread_tag());
worker->workname = strdupz(name);
usec_t now = worker_now_monotonic_usec();
diff --git a/libnetdata/worker_utilization/worker_utilization.h b/src/libnetdata/worker_utilization/worker_utilization.h
index e2f46c5a6..e2f46c5a6 100644
--- a/libnetdata/worker_utilization/worker_utilization.h
+++ b/src/libnetdata/worker_utilization/worker_utilization.h
diff --git a/libnetdata/xxhash.h b/src/libnetdata/xxhash.h
index 5e2c0ed24..5e2c0ed24 100644
--- a/libnetdata/xxhash.h
+++ b/src/libnetdata/xxhash.h