From 6cf8f2d5174a53f582e61d715edbb88d6e3367cc Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 14 Jun 2023 21:20:33 +0200 Subject: Adding upstream version 1.40.0. Signed-off-by: Daniel Baumann --- libnetdata/buffer/buffer.c | 8 + libnetdata/buffer/buffer.h | 8 + libnetdata/ebpf/ebpf.c | 121 ++++- libnetdata/ebpf/ebpf.h | 14 + libnetdata/gorilla/benchmark.sh | 14 + libnetdata/gorilla/fuzzer.sh | 14 + libnetdata/gorilla/gorilla.cc | 620 ++++++++++++++++++++++++++ libnetdata/gorilla/gorilla.h | 60 +++ libnetdata/http/http_defs.h | 30 ++ libnetdata/libjudy/src/JudyL/JudyLTables.c | 338 ++++++++++++++ libnetdata/libjudy/src/JudyL/JudyLTablesGen.c | 301 ------------- libnetdata/libnetdata.h | 4 +- libnetdata/parser/parser.c | 81 +++- libnetdata/parser/parser.h | 2 +- libnetdata/popen/popen.c | 106 ++--- libnetdata/popen/popen.h | 9 +- libnetdata/simple_pattern/README.md | 2 +- libnetdata/socket/security.c | 562 ++++++++++++++++++----- libnetdata/socket/security.h | 63 +-- libnetdata/socket/socket.c | 115 +++-- libnetdata/socket/socket.h | 24 +- 21 files changed, 1897 insertions(+), 599 deletions(-) create mode 100755 libnetdata/gorilla/benchmark.sh create mode 100755 libnetdata/gorilla/fuzzer.sh create mode 100644 libnetdata/gorilla/gorilla.cc create mode 100644 libnetdata/gorilla/gorilla.h create mode 100644 libnetdata/http/http_defs.h create mode 100644 libnetdata/libjudy/src/JudyL/JudyLTables.c delete mode 100644 libnetdata/libjudy/src/JudyL/JudyLTablesGen.c (limited to 'libnetdata') diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c index 142fbca14..91bc4dd60 100644 --- a/libnetdata/buffer/buffer.c +++ b/libnetdata/buffer/buffer.c @@ -503,3 +503,11 @@ int buffer_unittest(void) { return errors; } +#ifdef ENABLE_HTTPD +h2o_iovec_t buffer_to_h2o_iovec(BUFFER *wb) { + h2o_iovec_t ret; + ret.base = wb->buffer; + ret.len = wb->len; + return ret; +} +#endif diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h index f5f83bc2a..22686a5a1 100644 --- a/libnetdata/buffer/buffer.h +++ b/libnetdata/buffer/buffer.h @@ -6,6 +6,10 @@ #include "../string/utf8.h" #include "../libnetdata.h" +#ifdef ENABLE_HTTPD +#include "h2o/memory.h" +#endif + #define WEB_DATA_LENGTH_INCREASE_STEP 1024 #define BUFFER_JSON_MAX_DEPTH 32 // max is 255 @@ -129,6 +133,10 @@ void buffer_char_replace(BUFFER *wb, char from, char to); void buffer_print_sn_flags(BUFFER *wb, SN_FLAGS flags, bool send_anomaly_bit); +#ifdef ENABLE_HTTPD +h2o_iovec_t buffer_to_h2o_iovec(BUFFER *wb); +#endif + static inline void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) { if(unlikely(buffer->len + needed_free_size >= buffer->size)) buffer_increase(buffer, needed_free_size + 1); diff --git a/libnetdata/ebpf/ebpf.c b/libnetdata/ebpf/ebpf.c index 61833dd73..b980d09ed 100644 --- a/libnetdata/ebpf/ebpf.c +++ b/libnetdata/ebpf/ebpf.c @@ -453,6 +453,11 @@ void ebpf_update_stats(ebpf_plugin_stats_t *report, ebpf_module_t *em) else if (em->load & EBPF_LOAD_CORE) report->core++; + if (em->maps_per_core) + report->hash_percpu++; + else + report->hash_unique++; + ebpf_stats_targets(report, em->targets); } @@ -596,15 +601,70 @@ void ebpf_update_map_size(struct bpf_map *map, ebpf_local_maps_t *lmap, ebpf_mod #endif } +#ifdef LIBBPF_MAJOR_VERSION +/** + * Update map type + * + * Update map type with information given. + * + * @param map the map we want to modify + * @param w a structure with user input + */ +void ebpf_update_map_type(struct bpf_map *map, ebpf_local_maps_t *w) +{ + if (bpf_map__set_type(map, w->map_type)) { + error("Cannot modify map type for %s", w->name); + } +} + +/** + * Define map type + * + * This PR defines the type used by hash tables according user input. + * + * @param maps the list of maps used with a hash table. + * @param maps_per_core define if map type according user specification. + * @param kver kernel version host is running. + */ +void ebpf_define_map_type(ebpf_local_maps_t *maps, int maps_per_core, int kver) +{ + if (!maps) + return; + + // Before kernel 4.06 there was not percpu hash tables + if (kver < NETDATA_EBPF_KERNEL_4_06) + maps_per_core = CONFIG_BOOLEAN_NO; + + int i = 0; + while (maps[i].name) { + ebpf_local_maps_t *map = &maps[i]; + // maps_per_core is a boolean value in configuration files. + if (maps_per_core) { + if (map->map_type == BPF_MAP_TYPE_HASH) + map->map_type = BPF_MAP_TYPE_PERCPU_HASH; + else if (map->map_type == BPF_MAP_TYPE_ARRAY) + map->map_type = BPF_MAP_TYPE_PERCPU_ARRAY; + } else { + if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH) + map->map_type = BPF_MAP_TYPE_HASH; + else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) + map->map_type = BPF_MAP_TYPE_ARRAY; + } + + i++; + } +} +#endif + /** - * Update Legacy map sizes + * Update Legacy map * - * Update map size for eBPF legacy code. + * Update map for eBPF legacy code. * * @param program the structure with values read from binary. * @param em the structure with information about how the module/thread is working. */ -static void ebpf_update_legacy_map_sizes(struct bpf_object *program, ebpf_module_t *em) +static void ebpf_update_legacy_map(struct bpf_object *program, ebpf_module_t *em) { struct bpf_map *map; ebpf_local_maps_t *maps = em->maps; @@ -614,13 +674,19 @@ static void ebpf_update_legacy_map_sizes(struct bpf_object *program, ebpf_module bpf_map__for_each(map, program) { const char *map_name = bpf_map__name(map); - int i = 0; ; + int i = 0; while (maps[i].name) { ebpf_local_maps_t *w = &maps[i]; - if (w->type & NETDATA_EBPF_MAP_RESIZABLE) { - if (!strcmp(w->name, map_name)) { + + if (!strcmp(w->name, map_name)) { + // Modify size + if (w->type & NETDATA_EBPF_MAP_RESIZABLE) { ebpf_update_map_size(map, w, em, map_name); } + +#ifdef LIBBPF_MAJOR_VERSION + ebpf_update_map_type(map, w); +#endif } i++; @@ -790,13 +856,15 @@ struct bpf_link **ebpf_load_program(char *plugins_dir, ebpf_module_t *em, int kv em->load |= EBPF_LOAD_LEGACY; *obj = bpf_object__open_file(lpath, NULL); + if (!*obj) + return NULL; + if (libbpf_get_error(obj)) { - error("Cannot open BPF object %s", lpath); bpf_object__close(*obj); return NULL; } - ebpf_update_legacy_map_sizes(*obj, em); + ebpf_update_legacy_map(*obj, em); if (bpf_object__load(*obj)) { error("ERROR: loading BPF object file failed %s\n", lpath); @@ -1156,8 +1224,8 @@ void ebpf_update_module_using_config(ebpf_module_t *modules, netdata_ebpf_load_m { char default_value[EBPF_MAX_MODE_LENGTH + 1]; ebpf_select_mode_string(default_value, EBPF_MAX_MODE_LENGTH, modules->mode); - char *value = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_LOAD_MODE, default_value); - modules->mode = ebpf_select_mode(value); + char *load_mode = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_LOAD_MODE, default_value); + modules->mode = ebpf_select_mode(load_mode); modules->update_every = (int)appconfig_get_number(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_UPDATE_EVERY, modules->update_every); @@ -1171,19 +1239,38 @@ void ebpf_update_module_using_config(ebpf_module_t *modules, netdata_ebpf_load_m modules->pid_map_size = (uint32_t)appconfig_get_number(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_PID_SIZE, modules->pid_map_size); - value = ebpf_convert_load_mode_to_string(modules->load & NETDATA_EBPF_LOAD_METHODS); - value = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_TYPE_FORMAT, value); - netdata_ebpf_load_mode_t load = epbf_convert_string_to_load_mode(value); + char *value = ebpf_convert_load_mode_to_string(modules->load & NETDATA_EBPF_LOAD_METHODS); + char *type_format = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_TYPE_FORMAT, value); + netdata_ebpf_load_mode_t load = epbf_convert_string_to_load_mode(type_format); load = ebpf_select_load_mode(btf_file, load, kver, is_rh); modules->load = origin | load; - value = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_CORE_ATTACH, EBPF_CFG_ATTACH_TRAMPOLINE); - netdata_ebpf_program_loaded_t fill_lm = ebpf_convert_core_type(value, modules->mode); + char *core_attach = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_CORE_ATTACH, EBPF_CFG_ATTACH_TRAMPOLINE); + netdata_ebpf_program_loaded_t fill_lm = ebpf_convert_core_type(core_attach, modules->mode); ebpf_update_target_with_conf(modules, fill_lm); value = ebpf_convert_collect_pid_to_string(modules->apps_level); - value = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_COLLECT_PID, value); - modules->apps_level = ebpf_convert_string_to_apps_level(value); + char *collect_pid = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_COLLECT_PID, value); + modules->apps_level = ebpf_convert_string_to_apps_level(collect_pid); + + modules->maps_per_core = appconfig_get_boolean(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_MAPS_PER_CORE, + modules->maps_per_core); + if (kver < NETDATA_EBPF_KERNEL_4_06) + modules->maps_per_core = CONFIG_BOOLEAN_NO; + +#ifdef NETDATA_DEV_MODE + info("The thread %s was configured with: mode = %s; update every = %d; apps = %s; cgroup = %s; ebpf type format = %s; ebpf co-re tracing = %s; collect pid = %s; maps per core = %s", + modules->thread_name, + load_mode, + modules->update_every, + (modules->apps_charts)?"enabled":"disabled", + (modules->cgroup_charts)?"enabled":"disabled", + type_format, + core_attach, + collect_pid, + (modules->maps_per_core)?"enabled":"disabled" + ); +#endif } /** diff --git a/libnetdata/ebpf/ebpf.h b/libnetdata/ebpf/ebpf.h index bf5fdc33d..e82aaedd4 100644 --- a/libnetdata/ebpf/ebpf.h +++ b/libnetdata/ebpf/ebpf.h @@ -40,6 +40,8 @@ #define EBPF_CFG_PROGRAM_PATH "btf path" +#define EBPF_CFG_MAPS_PER_CORE "maps per core" + #define EBPF_CFG_UPDATE_EVERY "update every" #define EBPF_CFG_UPDATE_APPS_EVERY_DEFAULT 10 #define EBPF_CFG_PID_SIZE "pid table size" @@ -77,6 +79,7 @@ * */ enum netdata_ebpf_kernel_versions { + NETDATA_EBPF_KERNEL_4_06 = 263680, // 264960 = 4 * 65536 + 6 * 256 NETDATA_EBPF_KERNEL_4_11 = 264960, // 264960 = 4 * 65536 + 15 * 256 NETDATA_EBPF_KERNEL_4_14 = 265728, // 264960 = 4 * 65536 + 14 * 256 NETDATA_EBPF_KERNEL_4_15 = 265984, // 265984 = 4 * 65536 + 15 * 256 @@ -196,6 +199,9 @@ typedef struct ebpf_local_maps { uint32_t user_input; uint32_t type; int map_fd; +#ifdef LIBBPF_MAJOR_VERSION + enum bpf_map_type map_type; +#endif } ebpf_local_maps_t; typedef struct ebpf_specify_name { @@ -243,6 +249,9 @@ typedef struct ebpf_plugin_stats { uint64_t memlock_kern; // The same information reported by bpftool, but it is not accurated // https://lore.kernel.org/linux-mm/20230112155326.26902-5-laoar.shao@gmail.com/T/ uint32_t hash_tables; // Number of hash tables used on the system. + + uint32_t hash_percpu; // Number of threads running per cpu maps + uint32_t hash_unique; // Number of threads running an unique map for all cores. } ebpf_plugin_stats_t; typedef enum ebpf_stats_action { @@ -296,6 +305,7 @@ typedef struct ebpf_module { // charts char memory_usage[NETDATA_EBPF_CHART_MEM_LENGTH]; char memory_allocations[NETDATA_EBPF_CHART_MEM_LENGTH]; + int maps_per_core; } ebpf_module_t; int ebpf_get_kernel_version(); @@ -348,6 +358,7 @@ typedef struct ebpf_filesystem_partitions { ebpf_addresses_t addresses; uint64_t kernels; + ebpf_local_maps_t *fs_maps; } ebpf_filesystem_partitions_t; typedef struct ebpf_sync_syscalls { @@ -365,6 +376,7 @@ typedef struct ebpf_sync_syscalls { #else void *sync_obj; #endif + ebpf_local_maps_t *sync_maps; } ebpf_sync_syscalls_t; void ebpf_histogram_dimension_cleanup(char **ptr, size_t length); @@ -391,6 +403,8 @@ void ebpf_adjust_thread_load(ebpf_module_t *mod, struct btf *file); struct btf *ebpf_parse_btf_file(const char *filename); struct btf *ebpf_load_btf_file(char *path, char *filename); int ebpf_is_function_inside_btf(struct btf *file, char *function); +void ebpf_update_map_type(struct bpf_map *map, ebpf_local_maps_t *w); +void ebpf_define_map_type(ebpf_local_maps_t *maps, int maps_per_core, int kver); #endif void ebpf_update_kernel_memory_with_vector(ebpf_plugin_stats_t *report, ebpf_local_maps_t *maps); diff --git a/libnetdata/gorilla/benchmark.sh b/libnetdata/gorilla/benchmark.sh new file mode 100755 index 000000000..a5d111435 --- /dev/null +++ b/libnetdata/gorilla/benchmark.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# SPDX-License-Identifier: GPL-3.0-or-later +# + +set -exu -o pipefail + +clang++ \ + -std=c++11 -Wall -Wextra \ + -DENABLE_BENCHMARK -O2 -g \ + -lbenchmark -lbenchmark_main \ + -o gorilla_benchmark gorilla.cc + +./gorilla_benchmark diff --git a/libnetdata/gorilla/fuzzer.sh b/libnetdata/gorilla/fuzzer.sh new file mode 100755 index 000000000..9dfdec055 --- /dev/null +++ b/libnetdata/gorilla/fuzzer.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# SPDX-License-Identifier: GPL-3.0-or-later +# + +set -exu -o pipefail + +clang++ \ + -std=c++11 -Wall -Wextra \ + -DENABLE_FUZZER -O2 -g \ + -fsanitize=fuzzer \ + -o gorilla_fuzzer gorilla.cc + +./gorilla_fuzzer -workers=8 -jobs=8 diff --git a/libnetdata/gorilla/gorilla.cc b/libnetdata/gorilla/gorilla.cc new file mode 100644 index 000000000..af2f74007 --- /dev/null +++ b/libnetdata/gorilla/gorilla.cc @@ -0,0 +1,620 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "gorilla.h" + +#include +#include +#include +#include + +using std::size_t; + +template +static constexpr size_t bit_size() noexcept +{ + static_assert((sizeof(T) * CHAR_BIT) == 32 || (sizeof(T) * CHAR_BIT) == 64, + "Word size should be 32 or 64 bits."); + return (sizeof(T) * CHAR_BIT); +} + +/* + * Low-level bitstream operations, allowing us to read/write individual bits. +*/ + +template +struct bit_stream_t { + Word *buffer; + size_t capacity; + size_t position; +}; + +template +static bit_stream_t bit_stream_new(Word *buffer, Word capacity) { + bit_stream_t bs; + + bs.buffer = buffer; + bs.capacity = capacity * bit_size(); + bs.position = 0; + + return bs; +} + +template +static bool bit_stream_write(bit_stream_t *bs, Word value, size_t nbits) { + assert(nbits > 0 && nbits <= bit_size()); + assert(bs->capacity >= (bs->position + nbits)); + + if (bs->position + nbits > bs->capacity) { + return false; + } + + const size_t index = bs->position / bit_size(); + const size_t offset = bs->position % bit_size(); + bs->position += nbits; + + if (offset == 0) { + bs->buffer[index] = value; + } else { + const size_t remaining_bits = bit_size() - offset; + + // write the lower part of the value + const Word low_bits_mask = ((Word) 1 << remaining_bits) - 1; + const Word lowest_bits_in_value = value & low_bits_mask; + bs->buffer[index] |= (lowest_bits_in_value << offset); + + if (nbits > remaining_bits) { + // write the upper part of the value + const Word high_bits_mask = ~low_bits_mask; + const Word highest_bits_in_value = (value & high_bits_mask) >> (remaining_bits); + bs->buffer[index + 1] = highest_bits_in_value; + } + } + + return true; +} + +template +static bool bit_stream_read(bit_stream_t *bs, Word *value, size_t nbits) { + assert(nbits > 0 && nbits <= bit_size()); + assert(bs->capacity >= (bs->position + nbits)); + + if (bs->position + nbits > bs->capacity) { + return false; + } + + const size_t index = bs->position / bit_size(); + const size_t offset = bs->position % bit_size(); + bs->position += nbits; + + if (offset == 0) { + *value = (nbits == bit_size()) ? + bs->buffer[index] : + bs->buffer[index] & (((Word) 1 << nbits) - 1); + } else { + const size_t remaining_bits = bit_size() - offset; + + // extract the lower part of the value + if (nbits < remaining_bits) { + *value = (bs->buffer[index] >> offset) & (((Word) 1 << nbits) - 1); + } else { + *value = (bs->buffer[index] >> offset) & (((Word) 1 << remaining_bits) - 1); + nbits -= remaining_bits; + *value |= (bs->buffer[index + 1] & (((Word) 1 << nbits) - 1)) << remaining_bits; + } + } + + return true; +} + +/* + * High-level Gorilla codec implementation +*/ + +template +struct bit_code_t { + bit_stream_t bs; + Word entries; + Word prev_number; + Word prev_xor; + Word prev_xor_lzc; +}; + +template +static void bit_code_init(bit_code_t *bc, Word *buffer, Word capacity) { + bc->bs = bit_stream_new(buffer, capacity); + + bc->entries = 0; + bc->prev_number = 0; + bc->prev_xor = 0; + bc->prev_xor_lzc = 0; + + // reserved two words: + // Buffer[0] -> number of entries written + // Buffer[1] -> number of bits written + + bc->bs.position += 2 * bit_size(); +} + +template +static bool bit_code_read(bit_code_t *bc, Word *number) { + bit_stream_t *bs = &bc->bs; + + bc->entries++; + + // read the first number + if (bc->entries == 1) { + bool ok = bit_stream_read(bs, number, bit_size()); + bc->prev_number = *number; + return ok; + } + + // process same-number bit + Word is_same_number; + if (!bit_stream_read(bs, &is_same_number, 1)) { + return false; + } + + if (is_same_number) { + *number = bc->prev_number; + return true; + } + + // proceess same-xor-lzc bit + Word xor_lzc = bc->prev_xor_lzc; + + Word same_xor_lzc; + if (!bit_stream_read(bs, &same_xor_lzc, 1)) { + return false; + } + + if (!same_xor_lzc) { + if (!bit_stream_read(bs, &xor_lzc, (bit_size() == 32) ? 5 : 6)) { + return false; + } + } + + // process the non-lzc suffix + Word xor_value = 0; + if (!bit_stream_read(bs, &xor_value, bit_size() - xor_lzc)) { + return false; + } + + *number = (bc->prev_number ^ xor_value); + + bc->prev_number = *number; + bc->prev_xor_lzc = xor_lzc; + bc->prev_xor = xor_value; + + return true; +} + +template +static bool bit_code_write(bit_code_t *bc, const Word number) { + bit_stream_t *bs = &bc->bs; + Word position = bs->position; + + bc->entries++; + + // this is the first number we are writing + if (bc->entries == 1) { + bc->prev_number = number; + return bit_stream_write(bs, number, bit_size()); + } + + // write true/false based on whether we got the same number or not. + if (number == bc->prev_number) { + return bit_stream_write(bs, static_cast(1), 1); + } else { + if (bit_stream_write(bs, static_cast(0), 1) == false) { + return false; + } + } + + // otherwise: + // - compute the non-zero xor + // - find its leading-zero count + + Word xor_value = bc->prev_number ^ number; + // FIXME: Use SFINAE + Word xor_lzc = (bit_size() == 32) ? __builtin_clz(xor_value) : __builtin_clzll(xor_value); + Word is_xor_lzc_same = (xor_lzc == bc->prev_xor_lzc) ? 1 : 0; + + if (is_xor_lzc_same) { + // xor-lzc is same + if (bit_stream_write(bs, static_cast(1), 1) == false) { + goto RET_FALSE; + } + } else { + // xor-lzc is different + if (bit_stream_write(bs, static_cast(0), 1) == false) { + goto RET_FALSE; + } + + if (bit_stream_write(bs, xor_lzc, (bit_size() == 32) ? 5 : 6) == false) { + goto RET_FALSE; + } + } + + // write the bits of the XOR value without the LZC prefix + if (bit_stream_write(bs, xor_value, bit_size() - xor_lzc) == false) { + goto RET_FALSE; + } + + bc->prev_number = number; + bc->prev_xor_lzc = xor_lzc; + return true; + +RET_FALSE: + bc->bs.position = position; + return false; +} + +// only valid for writers +template +static bool bit_code_flush(bit_code_t *bc) { + bit_stream_t *bs = &bc->bs; + + Word num_entries_written = bc->entries; + Word num_bits_written = bs->position; + + // we want to write these at the beginning + bs->position = 0; + + if (!bit_stream_write(bs, num_entries_written, bit_size())) { + return false; + } + + if (!bit_stream_write(bs, num_bits_written, bit_size())) { + return false; + } + + bs->position = num_bits_written; + return true; +} + +// only valid for readers +template +static bool bit_code_info(bit_code_t *bc, Word *num_entries_written, + Word *num_bits_written) { + bit_stream_t *bs = &bc->bs; + + assert(bs->position == 2 * bit_size()); + if (bs->capacity < (2 * bit_size())) { + return false; + } + + if (num_entries_written) { + *num_entries_written = bs->buffer[0]; + } + if (num_bits_written) { + *num_bits_written = bs->buffer[1]; + } + + return true; +} + +template +static size_t gorilla_encode(Word *dst, Word dst_len, const Word *src, Word src_len) { + bit_code_t bcw; + + bit_code_init(&bcw, dst, dst_len); + + for (size_t i = 0; i != src_len; i++) { + if (!bit_code_write(&bcw, src[i])) + return 0; + } + + if (!bit_code_flush(&bcw)) + return 0; + + return src_len; +} + +template +static size_t gorilla_decode(Word *dst, Word dst_len, const Word *src, Word src_len) { + bit_code_t bcr; + + bit_code_init(&bcr, (Word *) src, src_len); + + Word num_entries; + if (!bit_code_info(&bcr, &num_entries, (Word *) NULL)) { + return 0; + } + if (num_entries > dst_len) { + return 0; + } + + for (size_t i = 0; i != num_entries; i++) { + if (!bit_code_read(&bcr, &dst[i])) + return 0; + } + + return num_entries; +} + +/* + * Low-level public API +*/ + +// 32-bit API + +void bit_code_writer_u32_init(bit_code_writer_u32_t *bcw, uint32_t *buffer, uint32_t capacity) { + bit_code_t *bc = (bit_code_t *) bcw; + bit_code_init(bc, buffer, capacity); +} + +bool bit_code_writer_u32_write(bit_code_writer_u32_t *bcw, const uint32_t number) { + bit_code_t *bc = (bit_code_t *) bcw; + return bit_code_write(bc, number); +} + +bool bit_code_writer_u32_flush(bit_code_writer_u32_t *bcw) { + bit_code_t *bc = (bit_code_t *) bcw; + return bit_code_flush(bc); +} + +void bit_code_reader_u32_init(bit_code_reader_u32_t *bcr, uint32_t *buffer, uint32_t capacity) { + bit_code_t *bc = (bit_code_t *) bcr; + bit_code_init(bc, buffer, capacity); +} + +bool bit_code_reader_u32_read(bit_code_reader_u32_t *bcr, uint32_t *number) { + bit_code_t *bc = (bit_code_t *) bcr; + return bit_code_read(bc, number); +} + +bool bit_code_reader_u32_info(bit_code_reader_u32_t *bcr, uint32_t *num_entries_written, + uint32_t *num_bits_written) { + bit_code_t *bc = (bit_code_t *) bcr; + return bit_code_info(bc, num_entries_written, num_bits_written); +} + +// 64-bit API + +void bit_code_writer_u64_init(bit_code_writer_u64_t *bcw, uint64_t *buffer, uint64_t capacity) { + bit_code_t *bc = (bit_code_t *) bcw; + bit_code_init(bc, buffer, capacity); +} + +bool bit_code_writer_u64_write(bit_code_writer_u64_t *bcw, const uint64_t number) { + bit_code_t *bc = (bit_code_t *) bcw; + return bit_code_write(bc, number); +} + +bool bit_code_writer_u64_flush(bit_code_writer_u64_t *bcw) { + bit_code_t *bc = (bit_code_t *) bcw; + return bit_code_flush(bc); +} + +void bit_code_reader_u64_init(bit_code_reader_u64_t *bcr, uint64_t *buffer, uint64_t capacity) { + bit_code_t *bc = (bit_code_t *) bcr; + bit_code_init(bc, buffer, capacity); +} + +bool bit_code_reader_u64_read(bit_code_reader_u64_t *bcr, uint64_t *number) { + bit_code_t *bc = (bit_code_t *) bcr; + return bit_code_read(bc, number); +} + +bool bit_code_reader_u64_info(bit_code_reader_u64_t *bcr, uint64_t *num_entries_written, + uint64_t *num_bits_written) { + bit_code_t *bc = (bit_code_t *) bcr; + return bit_code_info(bc, num_entries_written, num_bits_written); +} + +/* + * High-level public API +*/ + +// 32-bit API + +size_t gorilla_encode_u32(uint32_t *dst, size_t dst_len, const uint32_t *src, size_t src_len) { + return gorilla_encode(dst, (uint32_t) dst_len, src, (uint32_t) src_len); +} + +size_t gorilla_decode_u32(uint32_t *dst, size_t dst_len, const uint32_t *src, size_t src_len) { + return gorilla_decode(dst, (uint32_t) dst_len, src, (uint32_t) src_len); +} + +// 64-bit API + +size_t gorilla_encode_u64(uint64_t *dst, size_t dst_len, const uint64_t *src, size_t src_len) { + return gorilla_encode(dst, (uint64_t) dst_len, src, (uint64_t) src_len); +} + +size_t gorilla_decode_u64(uint64_t *dst, size_t dst_len, const uint64_t *src, size_t src_len) { + return gorilla_decode(dst, (uint64_t) dst_len, src, (uint64_t) src_len); +} + +/* + * Internal code used for fuzzing the library +*/ + +#ifdef ENABLE_FUZZER + +#include + +template +static std::vector random_vector(const uint8_t *data, size_t size) { + std::vector V; + + V.reserve(1024); + + while (size >= sizeof(Word)) { + size -= sizeof(Word); + + Word w; + memcpy(&w, &data[size], sizeof(Word)); + V.push_back(w); + } + + return V; +} + +template +static void check_equal_buffers(Word *lhs, Word lhs_size, Word *rhs, Word rhs_size) { + assert((lhs_size == rhs_size) && "Buffers have different size."); + + for (size_t i = 0; i != lhs_size; i++) { + assert((lhs[i] == rhs[i]) && "Buffers differ"); + } +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { + // 32-bit tests + { + if (Size < 4) + return 0; + + std::vector RandomData = random_vector(Data, Size); + std::vector EncodedData(10 * RandomData.capacity(), 0); + std::vector DecodedData(10 * RandomData.capacity(), 0); + + size_t num_entries_written = gorilla_encode_u32(EncodedData.data(), EncodedData.size(), + RandomData.data(), RandomData.size()); + size_t num_entries_read = gorilla_decode_u32(DecodedData.data(), DecodedData.size(), + EncodedData.data(), EncodedData.size()); + + assert(num_entries_written == num_entries_read); + check_equal_buffers(RandomData.data(), (uint32_t) RandomData.size(), + DecodedData.data(), (uint32_t) RandomData.size()); + } + + // 64-bit tests + { + if (Size < 8) + return 0; + + std::vector RandomData = random_vector(Data, Size); + std::vector EncodedData(10 * RandomData.capacity(), 0); + std::vector DecodedData(10 * RandomData.capacity(), 0); + + size_t num_entries_written = gorilla_encode_u64(EncodedData.data(), EncodedData.size(), + RandomData.data(), RandomData.size()); + size_t num_entries_read = gorilla_decode_u64(DecodedData.data(), DecodedData.size(), + EncodedData.data(), EncodedData.size()); + + assert(num_entries_written == num_entries_read); + check_equal_buffers(RandomData.data(), (uint64_t) RandomData.size(), + DecodedData.data(), (uint64_t) RandomData.size()); + } + + return 0; +} + +#endif /* ENABLE_FUZZER */ + +#ifdef ENABLE_BENCHMARK + +#include +#include + +static size_t NumItems = 1024; + +static void BM_EncodeU32Numbers(benchmark::State& state) { + std::random_device rd; + std::mt19937 mt(rd()); + std::uniform_int_distribution dist(0x0, 0x0000FFFF); + + std::vector RandomData; + for (size_t idx = 0; idx != NumItems; idx++) { + RandomData.push_back(dist(mt)); + } + std::vector EncodedData(10 * RandomData.capacity(), 0); + + for (auto _ : state) { + benchmark::DoNotOptimize( + gorilla_encode_u32(EncodedData.data(), EncodedData.size(), + RandomData.data(), RandomData.size()) + ); + benchmark::ClobberMemory(); + } + + state.SetItemsProcessed(NumItems * state.iterations()); + state.SetBytesProcessed(NumItems * state.iterations() * sizeof(uint32_t)); +} +BENCHMARK(BM_EncodeU32Numbers); + +static void BM_DecodeU32Numbers(benchmark::State& state) { + std::random_device rd; + std::mt19937 mt(rd()); + std::uniform_int_distribution dist(0x0, 0xFFFFFFFF); + + std::vector RandomData; + for (size_t idx = 0; idx != NumItems; idx++) { + RandomData.push_back(dist(mt)); + } + std::vector EncodedData(10 * RandomData.capacity(), 0); + std::vector DecodedData(10 * RandomData.capacity(), 0); + + gorilla_encode_u32(EncodedData.data(), EncodedData.size(), + RandomData.data(), RandomData.size()); + + for (auto _ : state) { + benchmark::DoNotOptimize( + gorilla_decode_u32(DecodedData.data(), DecodedData.size(), + EncodedData.data(), EncodedData.size()) + ); + benchmark::ClobberMemory(); + } + + state.SetItemsProcessed(NumItems * state.iterations()); + state.SetBytesProcessed(NumItems * state.iterations() * sizeof(uint32_t)); +} +// Register the function as a benchmark +BENCHMARK(BM_DecodeU32Numbers); + +static void BM_EncodeU64Numbers(benchmark::State& state) { + std::random_device rd; + std::mt19937 mt(rd()); + std::uniform_int_distribution dist(0x0, 0x0000FFFF); + + std::vector RandomData; + for (size_t idx = 0; idx != 1024; idx++) { + RandomData.push_back(dist(mt)); + } + std::vector EncodedData(10 * RandomData.capacity(), 0); + + for (auto _ : state) { + benchmark::DoNotOptimize( + gorilla_encode_u64(EncodedData.data(), EncodedData.size(), + RandomData.data(), RandomData.size()) + ); + benchmark::ClobberMemory(); + } + + state.SetItemsProcessed(NumItems * state.iterations()); + state.SetBytesProcessed(NumItems * state.iterations() * sizeof(uint64_t)); +} +BENCHMARK(BM_EncodeU64Numbers); + +static void BM_DecodeU64Numbers(benchmark::State& state) { + std::random_device rd; + std::mt19937 mt(rd()); + std::uniform_int_distribution dist(0x0, 0xFFFFFFFF); + + std::vector RandomData; + for (size_t idx = 0; idx != 1024; idx++) { + RandomData.push_back(dist(mt)); + } + std::vector EncodedData(10 * RandomData.capacity(), 0); + std::vector DecodedData(10 * RandomData.capacity(), 0); + + gorilla_encode_u64(EncodedData.data(), EncodedData.size(), + RandomData.data(), RandomData.size()); + + for (auto _ : state) { + benchmark::DoNotOptimize( + gorilla_decode_u64(DecodedData.data(), DecodedData.size(), + EncodedData.data(), EncodedData.size()) + ); + benchmark::ClobberMemory(); + } + + state.SetItemsProcessed(NumItems * state.iterations()); + state.SetBytesProcessed(NumItems * state.iterations() * sizeof(uint64_t)); +} +// Register the function as a benchmark +BENCHMARK(BM_DecodeU64Numbers); + +#endif /* ENABLE_BENCHMARK */ diff --git a/libnetdata/gorilla/gorilla.h b/libnetdata/gorilla/gorilla.h new file mode 100644 index 000000000..12bec42c0 --- /dev/null +++ b/libnetdata/gorilla/gorilla.h @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef GORILLA_H +#define GORILLA_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Low-level public API +*/ + +// 32-bit API + +typedef struct bit_code_writer_u32 bit_code_writer_u32_t; +typedef struct bit_code_reader_u32 bit_code_reader_u32_t; + +void bit_code_writer_u32_init(bit_code_writer_u32_t *bcw, uint32_t *buffer, uint32_t capacity); +bool bit_code_writer_u32_write(bit_code_writer_u32_t *bcw, const uint32_t number); +bool bit_code_writer_u32_flush(bit_code_writer_u32_t *bcw); + +void bit_code_reader_u32_init(bit_code_reader_u32_t *bcr, uint32_t *buffer, uint32_t capacity); +bool bit_code_reader_u32_read(bit_code_reader_u32_t *bcr, uint32_t *number); +bool bit_code_reader_u32_info(bit_code_reader_u32_t *bcr, uint32_t *num_entries_written, + uint64_t *num_bits_written); + +// 64-bit API + +typedef struct bit_code_writer_u64 bit_code_writer_u64_t; +typedef struct bit_code_reader_u64 bit_code_reader_u64_t; + +void bit_code_writer_u64_init(bit_code_writer_u64_t *bcw, uint64_t *buffer, uint64_t capacity); +bool bit_code_writer_u64_write(bit_code_writer_u64_t *bcw, const uint64_t number); +bool bit_code_writer_u64_flush(bit_code_writer_u64_t *bcw); + +void bit_code_reader_u64_init(bit_code_reader_u64_t *bcr, uint64_t *buffer, uint64_t capacity); +bool bit_code_reader_u64_read(bit_code_reader_u64_t *bcr, uint64_t *number); +bool bit_code_reader_u64_info(bit_code_reader_u64_t *bcr, uint64_t *num_entries_written, + uint64_t *num_bits_written); + +/* + * High-level public API +*/ + +size_t gorilla_encode_u32(uint32_t *dst, size_t dst_len, const uint32_t *src, size_t src_len); +size_t gorilla_decode_u32(uint32_t *dst, size_t dst_len, const uint32_t *src, size_t src_len); + +size_t gorilla_encode_u64(uint64_t *dst, size_t dst_len, const uint64_t *src, size_t src_len); +size_t gorilla_decode_u64(uint64_t *dst, size_t dst_len, const uint64_t *src, size_t src_len); + +#ifdef __cplusplus +} +#endif + +#endif /* GORILLA_H */ diff --git a/libnetdata/http/http_defs.h b/libnetdata/http/http_defs.h new file mode 100644 index 000000000..774ea0b71 --- /dev/null +++ b/libnetdata/http/http_defs.h @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_HTTP_DEFS_H +#define NETDATA_HTTP_DEFS_H + +// HTTP_CODES 2XX Success +#define HTTP_RESP_OK 200 + +// HTTP_CODES 3XX Redirections +#define HTTP_RESP_MOVED_PERM 301 +#define HTTP_RESP_REDIR_TEMP 307 +#define HTTP_RESP_REDIR_PERM 308 + +// HTTP_CODES 4XX Client Errors +#define HTTP_RESP_BAD_REQUEST 400 +#define HTTP_RESP_UNAUTHORIZED 401 +#define HTTP_RESP_FORBIDDEN 403 +#define HTTP_RESP_NOT_FOUND 404 +#define HTTP_RESP_CONFLICT 409 +#define HTTP_RESP_PRECOND_FAIL 412 +#define HTTP_RESP_CONTENT_TOO_LONG 413 + +// HTTP_CODES 5XX Server Errors +#define HTTP_RESP_INTERNAL_SERVER_ERROR 500 +#define HTTP_RESP_BACKEND_FETCH_FAILED 503 // 503 is right +#define HTTP_RESP_SERVICE_UNAVAILABLE 503 // 503 is right +#define HTTP_RESP_GATEWAY_TIMEOUT 504 +#define HTTP_RESP_BACKEND_RESPONSE_INVALID 591 + +#endif /* NETDATA_HTTP_DEFS_H */ diff --git a/libnetdata/libjudy/src/JudyL/JudyLTables.c b/libnetdata/libjudy/src/JudyL/JudyLTables.c new file mode 100644 index 000000000..21c974986 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLTables.c @@ -0,0 +1,338 @@ +// @(#) From generation tool: $Revision: 4.37 $ $Source: /judy/src/JudyCommon/JudyTables.c $ +// Pregenerated and modified by hand. Do not overwrite! + +#include "JudyL.h" +// Leave the malloc() sizes readable in the binary (via strings(1)): +#ifdef JU_64BIT +const char * JudyLMallocSizes = "JudyLMallocSizes = 3, 5, 7, 11, 15, 23, 32, 47, 64, Leaf1 = 13"; +#else // JU_32BIT +const char * JudyLMallocSizes = "JudyLMallocSizes = 3, 5, 7, 11, 15, 23, 32, 47, 64, Leaf1 = 25"; +#endif // JU_64BIT + +#ifdef JU_64BIT +// object uses 64 words +// cJU_BITSPERSUBEXPB = 32 +const uint8_t +j__L_BranchBJPPopToWords[cJU_BITSPERSUBEXPB + 1] = +{ + 0, + 3, 5, 7, 11, 11, 15, 15, 23, + 23, 23, 23, 32, 32, 32, 32, 32, + 47, 47, 47, 47, 47, 47, 47, 64, + 64, 64, 64, 64, 64, 64, 64, 64 +}; + +// object uses 15 words +// cJL_LEAF1_MAXPOP1 = 13 +const uint8_t +j__L_Leaf1PopToWords[cJL_LEAF1_MAXPOP1 + 1] = +{ + 0, + 3, 3, 5, 5, 7, 7, 11, 11, + 11, 15, 15, 15, 15 +}; +const uint8_t +j__L_Leaf1Offset[cJL_LEAF1_MAXPOP1 + 1] = +{ + 0, + 1, 1, 1, 1, 1, 1, 2, 2, + 2, 2, 2, 2, 2 +}; + +// object uses 64 words +// cJL_LEAF2_MAXPOP1 = 51 +const uint8_t +j__L_Leaf2PopToWords[cJL_LEAF2_MAXPOP1 + 1] = +{ + 0, + 3, 3, 5, 5, 7, 11, 11, 11, + 15, 15, 15, 15, 23, 23, 23, 23, + 23, 23, 32, 32, 32, 32, 32, 32, + 32, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64 +}; +const uint8_t +j__L_Leaf2Offset[cJL_LEAF2_MAXPOP1 + 1] = +{ + 0, + 1, 1, 1, 1, 2, 3, 3, 3, + 3, 3, 3, 3, 5, 5, 5, 5, + 5, 5, 7, 7, 7, 7, 7, 7, + 7, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13 +}; + +// object uses 64 words +// cJL_LEAF3_MAXPOP1 = 46 +const uint8_t +j__L_Leaf3PopToWords[cJL_LEAF3_MAXPOP1 + 1] = +{ + 0, + 3, 3, 5, 7, 7, 11, 11, 11, + 15, 15, 23, 23, 23, 23, 23, 23, + 32, 32, 32, 32, 32, 32, 32, 47, + 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64 +}; +const uint8_t +j__L_Leaf3Offset[cJL_LEAF3_MAXPOP1 + 1] = +{ + 0, + 1, 1, 2, 2, 2, 3, 3, 3, + 4, 4, 6, 6, 6, 6, 6, 6, + 9, 9, 9, 9, 9, 9, 9, 13, + 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18 +}; + +// object uses 63 words +// cJL_LEAF4_MAXPOP1 = 42 +const uint8_t +j__L_Leaf4PopToWords[cJL_LEAF4_MAXPOP1 + 1] = +{ + 0, + 3, 3, 5, 7, 11, 11, 11, 15, + 15, 15, 23, 23, 23, 23, 23, 32, + 32, 32, 32, 32, 32, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 63, + 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63 +}; +const uint8_t +j__L_Leaf4Offset[cJL_LEAF4_MAXPOP1 + 1] = +{ + 0, + 1, 1, 2, 2, 4, 4, 4, 5, + 5, 5, 8, 8, 8, 8, 8, 11, + 11, 11, 11, 11, 11, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 21, + 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21 +}; + +// object uses 64 words +// cJL_LEAF5_MAXPOP1 = 39 +const uint8_t +j__L_Leaf5PopToWords[cJL_LEAF5_MAXPOP1 + 1] = +{ + 0, + 3, 5, 5, 7, 11, 11, 15, 15, + 15, 23, 23, 23, 23, 23, 32, 32, + 32, 32, 32, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64 +}; +const uint8_t +j__L_Leaf5Offset[cJL_LEAF5_MAXPOP1 + 1] = +{ + 0, + 2, 2, 2, 3, 4, 4, 6, 6, + 6, 9, 9, 9, 9, 9, 12, 12, + 12, 12, 12, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25 +}; + +// object uses 63 words +// cJL_LEAF6_MAXPOP1 = 36 +const uint8_t +j__L_Leaf6PopToWords[cJL_LEAF6_MAXPOP1 + 1] = +{ + 0, + 3, 5, 7, 7, 11, 11, 15, 15, + 23, 23, 23, 23, 23, 32, 32, 32, + 32, 32, 47, 47, 47, 47, 47, 47, + 47, 47, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63 +}; +const uint8_t +j__L_Leaf6Offset[cJL_LEAF6_MAXPOP1 + 1] = +{ + 0, + 1, 3, 3, 3, 5, 5, 6, 6, + 10, 10, 10, 10, 10, 14, 14, 14, + 14, 14, 20, 20, 20, 20, 20, 20, + 20, 20, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 27 +}; + +// object uses 64 words +// cJL_LEAF7_MAXPOP1 = 34 +const uint8_t +j__L_Leaf7PopToWords[cJL_LEAF7_MAXPOP1 + 1] = +{ + 0, + 3, 5, 7, 11, 11, 15, 15, 15, + 23, 23, 23, 23, 32, 32, 32, 32, + 32, 47, 47, 47, 47, 47, 47, 47, + 47, 64, 64, 64, 64, 64, 64, 64, + 64, 64 +}; +const uint8_t +j__L_Leaf7Offset[cJL_LEAF7_MAXPOP1 + 1] = +{ + 0, + 1, 3, 3, 5, 5, 7, 7, 7, + 11, 11, 11, 11, 15, 15, 15, 15, + 15, 22, 22, 22, 22, 22, 22, 22, + 22, 30, 30, 30, 30, 30, 30, 30, + 30, 30 +}; + +// object uses 63 words +// cJL_LEAFW_MAXPOP1 = 31 +const uint8_t +j__L_LeafWPopToWords[cJL_LEAFW_MAXPOP1 + 1] = +{ + 0, + 3, 5, 7, 11, 11, 15, 15, 23, + 23, 23, 23, 32, 32, 32, 32, 47, + 47, 47, 47, 47, 47, 47, 47, 63, + 63, 63, 63, 63, 63, 63, 63 +}; +const uint8_t +j__L_LeafWOffset[cJL_LEAFW_MAXPOP1 + 1] = +{ + 0, + 2, 3, 4, 6, 6, 8, 8, 12, + 12, 12, 12, 16, 16, 16, 16, 24, + 24, 24, 24, 24, 24, 24, 24, 32, + 32, 32, 32, 32, 32, 32, 32 +}; + +// object uses 64 words +// cJU_BITSPERSUBEXPL = 64 +const uint8_t +j__L_LeafVPopToWords[cJU_BITSPERSUBEXPL + 1] = +{ + 0, + 3, 3, 3, 5, 5, 7, 7, 11, + 11, 11, 11, 15, 15, 15, 15, 23, + 23, 23, 23, 23, 23, 23, 23, 32, + 32, 32, 32, 32, 32, 32, 32, 32, + 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 64, + 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64 +}; +#else // JU_32BIT +// object uses 64 words +// cJU_BITSPERSUBEXPB = 32 +const uint8_t +j__L_BranchBJPPopToWords[cJU_BITSPERSUBEXPB + 1] = +{ + 0, + 3, 5, 7, 11, 11, 15, 15, 23, + 23, 23, 23, 32, 32, 32, 32, 32, + 47, 47, 47, 47, 47, 47, 47, 64, + 64, 64, 64, 64, 64, 64, 64, 64 +}; + +// object uses 32 words +// cJL_LEAF1_MAXPOP1 = 25 +const uint8_t +j__L_Leaf1PopToWords[cJL_LEAF1_MAXPOP1 + 1] = +{ + 0, + 3, 3, 5, 5, 7, 11, 11, 11, + 15, 15, 15, 15, 23, 23, 23, 23, + 23, 23, 32, 32, 32, 32, 32, 32, + 32 +}; +const uint8_t +j__L_Leaf1Offset[cJL_LEAF1_MAXPOP1 + 1] = +{ + 0, + 1, 1, 1, 1, 2, 3, 3, 3, + 3, 3, 3, 3, 5, 5, 5, 5, + 5, 5, 7, 7, 7, 7, 7, 7, + 7 +}; + +// object uses 63 words +// cJL_LEAF2_MAXPOP1 = 42 +const uint8_t +j__L_Leaf2PopToWords[cJL_LEAF2_MAXPOP1 + 1] = +{ + 0, + 3, 3, 5, 7, 11, 11, 11, 15, + 15, 15, 23, 23, 23, 23, 23, 32, + 32, 32, 32, 32, 32, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 63, + 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63 +}; +const uint8_t +j__L_Leaf2Offset[cJL_LEAF2_MAXPOP1 + 1] = +{ + 0, + 1, 1, 2, 2, 4, 4, 4, 5, + 5, 5, 8, 8, 8, 8, 8, 11, + 11, 11, 11, 11, 11, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 21, + 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21 +}; + +// object uses 63 words +// cJL_LEAF3_MAXPOP1 = 36 +const uint8_t +j__L_Leaf3PopToWords[cJL_LEAF3_MAXPOP1 + 1] = +{ + 0, + 3, 5, 7, 7, 11, 11, 15, 15, + 23, 23, 23, 23, 23, 32, 32, 32, + 32, 32, 47, 47, 47, 47, 47, 47, + 47, 47, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63 +}; +const uint8_t +j__L_Leaf3Offset[cJL_LEAF3_MAXPOP1 + 1] = +{ + 0, + 1, 3, 3, 3, 5, 5, 6, 6, + 10, 10, 10, 10, 10, 14, 14, 14, + 14, 14, 20, 20, 20, 20, 20, 20, + 20, 20, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 27 +}; + +// object uses 63 words +// cJL_LEAFW_MAXPOP1 = 31 +const uint8_t +j__L_LeafWPopToWords[cJL_LEAFW_MAXPOP1 + 1] = +{ + 0, + 3, 5, 7, 11, 11, 15, 15, 23, + 23, 23, 23, 32, 32, 32, 32, 47, + 47, 47, 47, 47, 47, 47, 47, 63, + 63, 63, 63, 63, 63, 63, 63 +}; +const uint8_t +j__L_LeafWOffset[cJL_LEAFW_MAXPOP1 + 1] = +{ + 0, + 2, 3, 4, 6, 6, 8, 8, 12, + 12, 12, 12, 16, 16, 16, 16, 24, + 24, 24, 24, 24, 24, 24, 24, 32, + 32, 32, 32, 32, 32, 32, 32 +}; + +// object uses 32 words +// cJU_BITSPERSUBEXPL = 32 +const uint8_t +j__L_LeafVPopToWords[cJU_BITSPERSUBEXPL + 1] = +{ + 0, + 3, 3, 3, 5, 5, 7, 7, 11, + 11, 11, 11, 15, 15, 15, 15, 23, + 23, 23, 23, 23, 23, 23, 23, 32, + 32, 32, 32, 32, 32, 32, 32, 32 +}; +#endif // JU_64BIT diff --git a/libnetdata/libjudy/src/JudyL/JudyLTablesGen.c b/libnetdata/libjudy/src/JudyL/JudyLTablesGen.c deleted file mode 100644 index ce4b37153..000000000 --- a/libnetdata/libjudy/src/JudyL/JudyLTablesGen.c +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright (C) 2000 - 2002 Hewlett-Packard Company -// -// This program is free software; you can redistribute it and/or modify it -// under the term of the GNU Lesser General Public License as published by the -// Free Software Foundation; either version 2 of the License, or (at your -// option) any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License -// for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with this program; if not, write to the Free Software Foundation, -// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// _________________ - -// @(#) $Revision: 4.37.1-netdata $ $Source: JudyTables.c $ - -#ifndef JU_WIN -#include // unavailable on win_*. -#endif - -#include -#include - -#if (! (defined(JUDY1) || defined(JUDYL))) -#error: One of -DJUDY1 or -DJUDYL must be specified. -#endif - -#define TERMINATOR 999 // terminator for Alloc tables - -// define bytes per word -#ifdef JU_64BIT -#define BPW 8UL -#else -#define BPW 4UL -#endif - -#ifdef JUDY1 -#include "Judy1.h" -#else -#include "JudyL.h" -#endif - -FILE *fd; - -// Definitions come from header files Judy1.h and JudyL.h: - -int AllocSizes[] = ALLOCSIZES; - -#define ROUNDUP(BYTES,BPW,OFFSETW) \ - ((((BYTES) + (BPW) - 1) / (BPW)) + (OFFSETW)) - - -// **************************************************************************** -// G E N T A B L E -// -// Note: "const" is required for newer compilers. - -FUNCTION void GenTable( - const char * TableName, // name of table string - const char * TableSize, // dimentioned size string - int IndexBytes, // bytes per Index - int LeafSize, // number elements in object - int ValueBytes, // bytes per Value - int OffsetWords) // 1 for LEAFW -{ - int * PAllocSizes = AllocSizes; - int OWord; - int CurWord; - int IWord; - int ii; - int BytesOfIndex; - int BytesOfObject; - int Index; - int LastWords; - int Words [1000] = { 0 }; - int Offset[1000] = { 0 }; - int MaxWords; - - MaxWords = ROUNDUP((IndexBytes + ValueBytes) * LeafSize, BPW, OffsetWords); - Words[0] = 0; - Offset[0] = 0; - CurWord = TERMINATOR; - -// Walk through all number of Indexes in table: - - for (Index = 1; /* null */; ++Index) - { - -// Calculate byte required for next size: - - BytesOfIndex = IndexBytes * Index; - BytesOfObject = (IndexBytes + ValueBytes) * Index; - -// Round up and calculate words required for next size: - - OWord = ROUNDUP(BytesOfObject, BPW, OffsetWords); - IWord = ROUNDUP(BytesOfIndex, BPW, OffsetWords); - -// Root-level leaves of population of 1 and 2 do not have the 1 word offset: - -// Save minimum value of offset: - - Offset[Index] = IWord; - -// Round up to next available size of words: - - while (OWord > *PAllocSizes) PAllocSizes++; - - if (Index == LeafSize) - { - CurWord = Words[Index] = OWord; - break; - } -// end of available sizes ? - - if (*PAllocSizes == TERMINATOR) - { - fprintf(stderr, "BUG, in %sPopToWords, sizes not big enough for object\n", TableName); - exit(1); - } - -// Save words required and last word: - - if (*PAllocSizes < MaxWords) { CurWord = Words[Index] = *PAllocSizes; } - else { CurWord = Words[Index] = MaxWords; } - - } // for each index - - LastWords = TERMINATOR; - -// Round up to largest size in each group of malloc sizes: - - for (ii = LeafSize; ii > 0; ii--) - { - if (LastWords > (Words[ii] - ii)) LastWords = Offset[ii]; - else Offset[ii] = LastWords; - } - -// Print the PopToWords[] table: - - fprintf(fd,"\n//\tobject uses %d words\n", CurWord); - fprintf(fd,"//\t%s = %d\n", TableSize, LeafSize); - - fprintf(fd,"const uint8_t\n"); - fprintf(fd,"%sPopToWords[%s + 1] =\n", TableName, TableSize); - fprintf(fd,"{\n\t 0,"); - - for (ii = 1; ii <= LeafSize; ii++) - { - -// 8 columns per line, starting with 1: - - if ((ii % 8) == 1) fprintf(fd,"\n\t"); - - fprintf(fd,"%2d", Words[ii]); - -// If not last number place comma: - - if (ii != LeafSize) fprintf(fd,", "); - } - fprintf(fd,"\n};\n"); - -// Print the Offset table if needed: - - if (! ValueBytes) return; - - fprintf(fd,"const uint8_t\n"); - fprintf(fd,"%sOffset[%s + 1] =\n", TableName, TableSize); - fprintf(fd,"{\n"); - fprintf(fd,"\t 0,"); - - for (ii = 1; ii <= LeafSize; ii++) - { - if ((ii % 8) == 1) fprintf(fd,"\n\t"); - - fprintf(fd,"%2d", Offset[ii]); - - if (ii != LeafSize) fprintf(fd,", "); - } - fprintf(fd,"\n};\n"); - -} // GenTable() - - -// **************************************************************************** -// M A I N - -FUNCTION int main() -{ - int ii; - -#ifdef JUDY1 - char *fname = "Judy1Tables.c"; -#else - char *fname = "JudyLTables.c"; -#endif - - if ((fd = fopen(fname, "w")) == NULL){ - perror("FATAL ERROR: could not write to Judy[1L]Tables.c file\n"); - return (-1); - } - - - fprintf(fd,"// @(#) From generation tool: $Revision: 4.37.1-netdata $ $Source: JudyTables.c $\n"); - fprintf(fd,"//\n\n"); - - -// ================================ Judy1 ================================= -#ifdef JUDY1 - - fprintf(fd,"#include \"Judy1.h\"\n"); - - fprintf(fd,"// Leave the malloc() sizes readable in the binary (via " - "strings(1)):\n"); - fprintf(fd,"const char * Judy1MallocSizes = \"Judy1MallocSizes ="); - - for (ii = 0; AllocSizes[ii] != TERMINATOR; ii++) - fprintf(fd," %d,", AllocSizes[ii]); - -#ifndef JU_64BIT - fprintf(fd," Leaf1 = %d\";\n\n", cJ1_LEAF1_MAXPOP1); -#else - fprintf(fd,"\";\n\n"); // no Leaf1 in this case. -#endif - -// ================================ 32 bit ================================ -#ifndef JU_64BIT - - GenTable("j__1_BranchBJP","cJU_BITSPERSUBEXPB", 8, cJU_BITSPERSUBEXPB,0,0); - - GenTable("j__1_Leaf1", "cJ1_LEAF1_MAXPOP1", 1, cJ1_LEAF1_MAXPOP1, 0, 0); - GenTable("j__1_Leaf2", "cJ1_LEAF2_MAXPOP1", 2, cJ1_LEAF2_MAXPOP1, 0, 0); - GenTable("j__1_Leaf3", "cJ1_LEAF3_MAXPOP1", 3, cJ1_LEAF3_MAXPOP1, 0, 0); - GenTable("j__1_LeafW", "cJ1_LEAFW_MAXPOP1", 4, cJ1_LEAFW_MAXPOP1, 0, 1); - -#endif - -// ================================ 64 bit ================================ -#ifdef JU_64BIT - GenTable("j__1_BranchBJP","cJU_BITSPERSUBEXPB",16, cJU_BITSPERSUBEXPB,0,0); - - GenTable("j__1_Leaf2", "cJ1_LEAF2_MAXPOP1", 2, cJ1_LEAF2_MAXPOP1, 0, 0); - GenTable("j__1_Leaf3", "cJ1_LEAF3_MAXPOP1", 3, cJ1_LEAF3_MAXPOP1, 0, 0); - GenTable("j__1_Leaf4", "cJ1_LEAF4_MAXPOP1", 4, cJ1_LEAF4_MAXPOP1, 0, 0); - GenTable("j__1_Leaf5", "cJ1_LEAF5_MAXPOP1", 5, cJ1_LEAF5_MAXPOP1, 0, 0); - GenTable("j__1_Leaf6", "cJ1_LEAF6_MAXPOP1", 6, cJ1_LEAF6_MAXPOP1, 0, 0); - GenTable("j__1_Leaf7", "cJ1_LEAF7_MAXPOP1", 7, cJ1_LEAF7_MAXPOP1, 0, 0); - GenTable("j__1_LeafW", "cJ1_LEAFW_MAXPOP1", 8, cJ1_LEAFW_MAXPOP1, 0, 1); -#endif -#endif // JUDY1 - - -// ================================ JudyL ================================= -#ifdef JUDYL - - fprintf(fd,"#include \"JudyL.h\"\n"); - - fprintf(fd,"// Leave the malloc() sizes readable in the binary (via " - "strings(1)):\n"); - fprintf(fd,"const char * JudyLMallocSizes = \"JudyLMallocSizes ="); - - for (ii = 0; AllocSizes[ii] != TERMINATOR; ii++) - fprintf(fd," %d,", AllocSizes[ii]); - - fprintf(fd," Leaf1 = %ld\";\n\n", (Word_t)cJL_LEAF1_MAXPOP1); - -#ifndef JU_64BIT -// ================================ 32 bit ================================ - GenTable("j__L_BranchBJP","cJU_BITSPERSUBEXPB", 8, cJU_BITSPERSUBEXPB, 0,0); - - GenTable("j__L_Leaf1", "cJL_LEAF1_MAXPOP1", 1, cJL_LEAF1_MAXPOP1, BPW,0); - GenTable("j__L_Leaf2", "cJL_LEAF2_MAXPOP1", 2, cJL_LEAF2_MAXPOP1, BPW,0); - GenTable("j__L_Leaf3", "cJL_LEAF3_MAXPOP1", 3, cJL_LEAF3_MAXPOP1, BPW,0); - GenTable("j__L_LeafW", "cJL_LEAFW_MAXPOP1", 4, cJL_LEAFW_MAXPOP1, BPW,1); - GenTable("j__L_LeafV", "cJU_BITSPERSUBEXPL", 4, cJU_BITSPERSUBEXPL, 0,0); -#endif // 32 BIT - -#ifdef JU_64BIT -// ================================ 64 bit ================================ - GenTable("j__L_BranchBJP","cJU_BITSPERSUBEXPB",16, cJU_BITSPERSUBEXPB, 0,0); - - GenTable("j__L_Leaf1", "cJL_LEAF1_MAXPOP1", 1, cJL_LEAF1_MAXPOP1, BPW,0); - GenTable("j__L_Leaf2", "cJL_LEAF2_MAXPOP1", 2, cJL_LEAF2_MAXPOP1, BPW,0); - GenTable("j__L_Leaf3", "cJL_LEAF3_MAXPOP1", 3, cJL_LEAF3_MAXPOP1, BPW,0); - GenTable("j__L_Leaf4", "cJL_LEAF4_MAXPOP1", 4, cJL_LEAF4_MAXPOP1, BPW,0); - GenTable("j__L_Leaf5", "cJL_LEAF5_MAXPOP1", 5, cJL_LEAF5_MAXPOP1, BPW,0); - GenTable("j__L_Leaf6", "cJL_LEAF6_MAXPOP1", 6, cJL_LEAF6_MAXPOP1, BPW,0); - GenTable("j__L_Leaf7", "cJL_LEAF7_MAXPOP1", 7, cJL_LEAF7_MAXPOP1, BPW,0); - GenTable("j__L_LeafW", "cJL_LEAFW_MAXPOP1", 8, cJL_LEAFW_MAXPOP1, BPW,1); - GenTable("j__L_LeafV", "cJU_BITSPERSUBEXPL", 8, cJU_BITSPERSUBEXPL, 0,0); -#endif // 64 BIT - -#endif // JUDYL - fclose(fd); - - return(0); - -} // main() diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h index c24494930..062d8c6fa 100644 --- a/libnetdata/libnetdata.h +++ b/libnetdata/libnetdata.h @@ -176,9 +176,7 @@ extern "C" { #include #endif -#ifdef NETDATA_WITH_ZLIB #include -#endif #ifdef HAVE_CAPABILITY #include @@ -667,6 +665,8 @@ extern char *netdata_configured_host_prefix; #include "worker_utilization/worker_utilization.h" #include "parser/parser.h" #include "yaml.h" +#include "http/http_defs.h" +#include "gorilla/gorilla.h" // BEWARE: this exists in alarm-notify.sh #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud" diff --git a/libnetdata/parser/parser.c b/libnetdata/parser/parser.c index c3eebcd16..80c9a2639 100644 --- a/libnetdata/parser/parser.c +++ b/libnetdata/parser/parser.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later +#include +#include #include "parser.h" #include "collectors/plugins.d/pluginsd_parser.h" @@ -124,26 +126,77 @@ void parser_destroy(PARSER *parser) * */ -int parser_next(PARSER *parser, char *buffer, size_t buffer_size) -{ - char *tmp = fgets(buffer, (int)buffer_size, (FILE *)parser->fp_input); +typedef enum { + PARSER_FGETS_RESULT_OK, + PARSER_FGETS_RESULT_TIMEOUT, + PARSER_FGETS_RESULT_ERROR, + PARSER_FGETS_RESULT_EOF, +} PARSER_FGETS_RESULT; + +static inline PARSER_FGETS_RESULT parser_fgets(char *s, int size, FILE *stream) { + errno = 0; + + struct pollfd fds[1]; + int timeout_msecs = 2 * 60 * MSEC_PER_SEC; + + fds[0].fd = fileno(stream); + fds[0].events = POLLIN; + + int ret = poll(fds, 1, timeout_msecs); - if (unlikely(!tmp)) { - if (feof((FILE *)parser->fp_input)) - error("PARSER: read failed: end of file"); + if (ret > 0) { + /* There is data to read */ + if (fds[0].revents & POLLIN) { + char *tmp = fgets(s, size, stream); - else if (ferror((FILE *)parser->fp_input)) - error("PARSER: read failed: input error"); + if(unlikely(!tmp)) { + if (feof(stream)) { + error("PARSER: read failed: end of file."); + return PARSER_FGETS_RESULT_EOF; + } - else - error("PARSER: read failed: unknown error"); + else if (ferror(stream)) { + error("PARSER: read failed: input error."); + return PARSER_FGETS_RESULT_ERROR; + } - return 1; + error("PARSER: read failed: unknown error."); + return PARSER_FGETS_RESULT_ERROR; + } + + return PARSER_FGETS_RESULT_OK; + } + else if(fds[0].revents & POLLERR) { + error("PARSER: read failed: POLLERR."); + return PARSER_FGETS_RESULT_ERROR; + } + else if(fds[0].revents & POLLHUP) { + error("PARSER: read failed: POLLHUP."); + return PARSER_FGETS_RESULT_ERROR; + } + else if(fds[0].revents & POLLNVAL) { + error("PARSER: read failed: POLLNVAL."); + return PARSER_FGETS_RESULT_ERROR; + } + + error("PARSER: poll() returned positive number, but POLLIN|POLLERR|POLLHUP|POLLNVAL are not set."); + return PARSER_FGETS_RESULT_ERROR; + } + else if (ret == 0) { + error("PARSER: timeout while waiting for data."); + return PARSER_FGETS_RESULT_TIMEOUT; } - return 0; + error("PARSER: poll() failed with code %d.", ret); + return PARSER_FGETS_RESULT_ERROR; } +int parser_next(PARSER *parser, char *buffer, size_t buffer_size) { + if(likely(parser_fgets(buffer, (int)buffer_size, (FILE *)parser->fp_input) == PARSER_FGETS_RESULT_OK)) + return 0; + + return 1; +} /* * Takes an initialized parser object that has an unprocessed entry (by calling parser_next) @@ -202,7 +255,6 @@ inline int parser_action(PARSER *parser, char *input) else rc = PARSER_RC_ERROR; -#ifdef NETDATA_INTERNAL_CHECKS if(rc == PARSER_RC_ERROR) { BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL); for(size_t i = 0; i < num_words ;i++) { @@ -214,12 +266,11 @@ inline int parser_action(PARSER *parser, char *input) buffer_fast_strcat(wb, "\"", 1); } - internal_error(true, "PLUGINSD: parser_action('%s') failed on line %zu: { %s } (quotes added to show parsing)", + error("PLUGINSD: parser_action('%s') failed on line %zu: { %s } (quotes added to show parsing)", command, parser->line, buffer_tostring(wb)); buffer_free(wb); } -#endif return (rc == PARSER_RC_ERROR || rc == PARSER_RC_STOP); } diff --git a/libnetdata/parser/parser.h b/libnetdata/parser/parser.h index 9e0d3480d..c21cbaf7e 100644 --- a/libnetdata/parser/parser.h +++ b/libnetdata/parser/parser.h @@ -44,7 +44,7 @@ typedef struct parser { FILE *fp_input; // Input source e.g. stream FILE *fp_output; // Stream to send commands to plugin #ifdef ENABLE_HTTPS - struct netdata_ssl *ssl_output; + NETDATA_SSL *ssl_output; #endif void *user; // User defined structure to hold extra state between calls uint32_t flags; diff --git a/libnetdata/popen/popen.c b/libnetdata/popen/popen.c index 5ed74ae95..783c74a51 100644 --- a/libnetdata/popen/popen.c +++ b/libnetdata/popen/popen.c @@ -5,11 +5,13 @@ // ---------------------------------------------------------------------------- // popen with tracking -static pthread_mutex_t netdata_popen_tracking_mutex; -static bool netdata_popen_tracking_enabled = false; +static pthread_mutex_t netdata_popen_tracking_mutex = NETDATA_MUTEX_INITIALIZER; struct netdata_popen { pid_t pid; + bool reaped; + siginfo_t infop; + int waitid_ret; struct netdata_popen *next; struct netdata_popen *prev; }; @@ -18,29 +20,20 @@ static struct netdata_popen *netdata_popen_root = NULL; // myp_add_lock takes the lock if we're tracking. static void netdata_popen_tracking_lock(void) { - if(!netdata_popen_tracking_enabled) - return; - netdata_mutex_lock(&netdata_popen_tracking_mutex); } // myp_add_unlock release the lock if we're tracking. static void netdata_popen_tracking_unlock(void) { - if(!netdata_popen_tracking_enabled) - return; - netdata_mutex_unlock(&netdata_popen_tracking_mutex); } // myp_add_locked adds pid if we're tracking. // myp_add_lock must have been called previously. static void netdata_popen_tracking_add_pid_unsafe(pid_t pid) { - if(!netdata_popen_tracking_enabled) - return; - struct netdata_popen *mp; - mp = mallocz(sizeof(struct netdata_popen)); + mp = callocz(1, sizeof(struct netdata_popen)); mp->pid = pid; DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(netdata_popen_root, mp, prev, next); @@ -48,12 +41,9 @@ static void netdata_popen_tracking_add_pid_unsafe(pid_t pid) { // myp_del deletes pid if we're tracking. static void netdata_popen_tracking_del_pid(pid_t pid) { - if(!netdata_popen_tracking_enabled) - return; - struct netdata_popen *mp; - netdata_mutex_lock(&netdata_popen_tracking_mutex); + netdata_popen_tracking_lock(); DOUBLE_LINKED_LIST_FOREACH_FORWARD(netdata_popen_root, mp, prev, next) { if(unlikely(mp->pid == pid)) @@ -65,34 +55,15 @@ static void netdata_popen_tracking_del_pid(pid_t pid) { freez(mp); } else - error("Cannot find pid %d.", pid); - - netdata_mutex_unlock(&netdata_popen_tracking_mutex); -} + error("POPEN: Cannot find pid %d.", pid); -// netdata_popen_tracking_init() should be called by apps which act as init -// (pid 1) so that processes created by mypopen and mypopene -// are tracked. This enables the reaper to ignore processes -// which will be handled internally, by calling myp_reap, to -// avoid issues with already reaped processes during wait calls. -// -// Callers should call myp_free() to clean up resources. -void netdata_popen_tracking_init(void) { - info("process tracking enabled."); - netdata_popen_tracking_enabled = true; - - if (netdata_mutex_init(&netdata_popen_tracking_mutex) != 0) - fatal("netdata_popen_tracking_init() mutex init failed."); + netdata_popen_tracking_unlock(); } // myp_free cleans up any resources allocated for process // tracking. void netdata_popen_tracking_cleanup(void) { - if(!netdata_popen_tracking_enabled) - return; - - netdata_mutex_lock(&netdata_popen_tracking_mutex); - netdata_popen_tracking_enabled = false; + netdata_popen_tracking_lock(); while(netdata_popen_root) { struct netdata_popen *mp = netdata_popen_root; @@ -100,26 +71,45 @@ void netdata_popen_tracking_cleanup(void) { freez(mp); } - netdata_mutex_unlock(&netdata_popen_tracking_mutex); + netdata_popen_tracking_unlock(); } -// myp_reap returns 1 if pid should be reaped, 0 otherwise. -int netdata_popen_tracking_pid_shoud_be_reaped(pid_t pid) { - if(!netdata_popen_tracking_enabled) - return 0; +int netdata_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options) { + struct netdata_popen *mp = NULL; - netdata_mutex_lock(&netdata_popen_tracking_mutex); + if(idtype == P_PID && id != 0) { + // the caller is asking to waitid() for a specific child pid - int ret = 1; - struct netdata_popen *mp; - DOUBLE_LINKED_LIST_FOREACH_FORWARD(netdata_popen_root, mp, prev, next) { - if(unlikely(mp->pid == pid)) { - ret = 0; - break; + netdata_popen_tracking_lock(); + DOUBLE_LINKED_LIST_FOREACH_FORWARD(netdata_popen_root, mp, prev, next) { + if(unlikely(mp->pid == (pid_t)id)) + break; } + + if(!mp) + netdata_popen_tracking_unlock(); } - netdata_mutex_unlock(&netdata_popen_tracking_mutex); + int ret; + if(mp && mp->reaped) { + // we have already reaped this child + ret = mp->waitid_ret; + *infop = mp->infop; + } + else { + // we haven't reaped this child yet + ret = waitid(idtype, id, infop, options); + + if(mp && !mp->reaped) { + mp->reaped = true; + mp->infop = *infop; + mp->waitid_ret = ret; + } + } + + if(mp) + netdata_popen_tracking_unlock(); + return ret; } @@ -404,7 +394,7 @@ int netdata_pclose(FILE *fp_child_input, FILE *fp_child_output, pid_t pid) { errno = 0; - ret = waitid(P_PID, (id_t) pid, &info, WEXITED); + ret = netdata_waitid(P_PID, (id_t) pid, &info, WEXITED); netdata_popen_tracking_del_pid(pid); if (ret != -1) { @@ -415,8 +405,12 @@ int netdata_pclose(FILE *fp_child_input, FILE *fp_child_output, pid_t pid) { return(info.si_status); case CLD_KILLED: - if(info.si_status == 15) { - info("child pid %d killed by signal %d.", info.si_pid, info.si_status); + if(info.si_status == SIGTERM) { + info("child pid %d killed by SIGTERM", info.si_pid); + return(0); + } + else if(info.si_status == SIGPIPE) { + info("child pid %d killed by SIGPIPE.", info.si_pid); return(0); } else { @@ -450,7 +444,3 @@ int netdata_pclose(FILE *fp_child_input, FILE *fp_child_output, pid_t pid) { return 0; } - -int netdata_spawn_waitpid(pid_t pid) { - return netdata_pclose(NULL, NULL, pid); -} diff --git a/libnetdata/popen/popen.h b/libnetdata/popen/popen.h index c57a35a4e..4f86158bc 100644 --- a/libnetdata/popen/popen.h +++ b/libnetdata/popen/popen.h @@ -28,13 +28,6 @@ int netdata_popene_variadic_internal_dont_use_directly(volatile pid_t *pidptr, c int netdata_pclose(FILE *fp_child_input, FILE *fp_child_output, pid_t pid); int netdata_spawn(const char *command, volatile pid_t *pidptr); -int netdata_spawn_waitpid(pid_t pid); - -void netdata_popen_tracking_init(void); -void netdata_popen_tracking_cleanup(void); -int netdata_popen_tracking_pid_shoud_be_reaped(pid_t pid); - -void signals_unblock(void); -void signals_reset(void); +int netdata_waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options); #endif /* NETDATA_POPEN_H */ diff --git a/libnetdata/simple_pattern/README.md b/libnetdata/simple_pattern/README.md index e00006d37..5f56a3af7 100644 --- a/libnetdata/simple_pattern/README.md +++ b/libnetdata/simple_pattern/README.md @@ -16,7 +16,7 @@ to use, write and understand. So, Netdata supports **simple patterns**. Simple patterns are a space separated list of words, that can have `*` -as a wildcard. Each world may use any number of `*`. Simple patterns +as a wildcard. Each word may use any number of `*`. Simple patterns allow **negative** matches by prefixing a word with `!`. So, `pattern = !*bad* *` will match anything, except all those that diff --git a/libnetdata/socket/security.c b/libnetdata/socket/security.c index 7c5092150..d1181ad5f 100644 --- a/libnetdata/socket/security.c +++ b/libnetdata/socket/security.c @@ -3,13 +3,389 @@ #ifdef ENABLE_HTTPS SSL_CTX *netdata_ssl_exporting_ctx =NULL; -SSL_CTX *netdata_ssl_client_ctx =NULL; -SSL_CTX *netdata_ssl_srv_ctx =NULL; +SSL_CTX *netdata_ssl_streaming_sender_ctx =NULL; +SSL_CTX *netdata_ssl_web_server_ctx =NULL; const char *netdata_ssl_security_key =NULL; const char *netdata_ssl_security_cert =NULL; const char *tls_version=NULL; const char *tls_ciphers=NULL; -int netdata_ssl_validate_server = NETDATA_SSL_VALID_CERTIFICATE; +bool netdata_ssl_validate_certificate = true; +bool netdata_ssl_validate_certificate_sender = true; + +static SOCKET_PEERS netdata_ssl_peers(NETDATA_SSL *ssl) { + int sock_fd; + + if(unlikely(!ssl->conn)) + sock_fd = -1; + else + sock_fd = SSL_get_rfd(ssl->conn); + + return socket_peers(sock_fd); +} + +bool netdata_ssl_open(NETDATA_SSL *ssl, SSL_CTX *ctx, int fd) { + errno = 0; + ssl->ssl_errno = 0; + + if(ssl->conn) { + if(!ctx || SSL_get_SSL_CTX(ssl->conn) != ctx) { + SSL_free(ssl->conn); + ssl->conn = NULL; + } + else if (SSL_clear(ssl->conn) == 0) { + netdata_ssl_log_error_queue("SSL_clear", ssl); + SSL_free(ssl->conn); + ssl->conn = NULL; + } + } + + if(!ssl->conn) { + if(!ctx) { + internal_error(true, "SSL: not CTX given"); + ssl->state = NETDATA_SSL_STATE_FAILED; + return false; + } + + ssl->conn = SSL_new(ctx); + if (!ssl->conn) { + netdata_ssl_log_error_queue("SSL_new", ssl); + ssl->state = NETDATA_SSL_STATE_FAILED; + return false; + } + } + + if(SSL_set_fd(ssl->conn, fd) != 1) { + netdata_ssl_log_error_queue("SSL_set_fd", ssl); + ssl->state = NETDATA_SSL_STATE_FAILED; + return false; + } + + ssl->state = NETDATA_SSL_STATE_INIT; + + ERR_clear_error(); + + return true; +} + +void netdata_ssl_close(NETDATA_SSL *ssl) { + errno = 0; + ssl->ssl_errno = 0; + + if(ssl->conn) { + if(SSL_connection(ssl)) { + int ret = SSL_shutdown(ssl->conn); + if(ret == 0) + SSL_shutdown(ssl->conn); + } + + SSL_free(ssl->conn); + + ERR_clear_error(); + } + + *ssl = NETDATA_SSL_UNSET_CONNECTION; +} + +void netdata_ssl_log_error_queue(const char *call, NETDATA_SSL *ssl) { + error_limit_static_thread_var(erl, 1, 0); + unsigned long err; + while((err = ERR_get_error())) { + char *code; + + switch (err) { + case SSL_ERROR_NONE: + code = "SSL_ERROR_NONE"; + break; + + case SSL_ERROR_SSL: + code = "SSL_ERROR_SSL"; + ssl->state = NETDATA_SSL_STATE_FAILED; + break; + + case SSL_ERROR_WANT_READ: + code = "SSL_ERROR_WANT_READ"; + break; + + case SSL_ERROR_WANT_WRITE: + code = "SSL_ERROR_WANT_WRITE"; + break; + + case SSL_ERROR_WANT_X509_LOOKUP: + code = "SSL_ERROR_WANT_X509_LOOKUP"; + break; + + case SSL_ERROR_SYSCALL: + code = "SSL_ERROR_SYSCALL"; + ssl->state = NETDATA_SSL_STATE_FAILED; + break; + + case SSL_ERROR_ZERO_RETURN: + code = "SSL_ERROR_ZERO_RETURN"; + break; + + case SSL_ERROR_WANT_CONNECT: + code = "SSL_ERROR_WANT_CONNECT"; + break; + + case SSL_ERROR_WANT_ACCEPT: + code = "SSL_ERROR_WANT_ACCEPT"; + break; + +#ifdef SSL_ERROR_WANT_ASYNC + case SSL_ERROR_WANT_ASYNC: + code = "SSL_ERROR_WANT_ASYNC"; + break; +#endif + +#ifdef SSL_ERROR_WANT_ASYNC_JOB + case SSL_ERROR_WANT_ASYNC_JOB: + code = "SSL_ERROR_WANT_ASYNC_JOB"; + break; +#endif + +#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB + case SSL_ERROR_WANT_CLIENT_HELLO_CB: + code = "SSL_ERROR_WANT_CLIENT_HELLO_CB"; + break; +#endif + +#ifdef SSL_ERROR_WANT_RETRY_VERIFY + case SSL_ERROR_WANT_RETRY_VERIFY: + code = "SSL_ERROR_WANT_RETRY_VERIFY"; + break; +#endif + + default: + code = "SSL_ERROR_UNKNOWN"; + break; + } + + char str[1024 + 1]; + ERR_error_string_n(err, str, 1024); + str[1024] = '\0'; + SOCKET_PEERS peers = netdata_ssl_peers(ssl); + error_limit(&erl, "SSL: %s() on socket local [[%s]:%d] <-> remote [[%s]:%d], returned error %lu (%s): %s", + call, peers.local.ip, peers.local.port, peers.peer.ip, peers.peer.port, err, code, str); + } +} + +static inline bool is_handshake_complete(NETDATA_SSL *ssl, const char *op) { + error_limit_static_thread_var(erl, 1, 0); + + if(unlikely(!ssl->conn)) { + internal_error(true, "SSL: trying to %s on a NULL connection", op); + return false; + } + + switch(ssl->state) { + case NETDATA_SSL_STATE_NOT_SSL: { + SOCKET_PEERS peers = netdata_ssl_peers(ssl); + error_limit(&erl, "SSL: on socket local [[%s]:%d] <-> remote [[%s]:%d], attempt to %s on non-SSL connection", + peers.local.ip, peers.local.port, peers.peer.ip, peers.peer.port, op); + return false; + } + + case NETDATA_SSL_STATE_INIT: { + SOCKET_PEERS peers = netdata_ssl_peers(ssl); + error_limit(&erl, "SSL: on socket local [[%s]:%d] <-> remote [[%s]:%d], attempt to %s on an incomplete connection", + peers.local.ip, peers.local.port, peers.peer.ip, peers.peer.port, op); + return false; + } + + case NETDATA_SSL_STATE_FAILED: { + SOCKET_PEERS peers = netdata_ssl_peers(ssl); + error_limit(&erl, "SSL: on socket local [[%s]:%d] <-> remote [[%s]:%d], attempt to %s on a failed connection", + peers.local.ip, peers.local.port, peers.peer.ip, peers.peer.port, op); + return false; + } + + case NETDATA_SSL_STATE_COMPLETE: { + return true; + } + } + + return false; +} + +/* + * netdata_ssl_read() should return the same as read(): + * + * Positive value: The read() function succeeded and read some bytes. The exact number of bytes read is returned. + * + * Zero: For files and sockets, a return value of zero signifies end-of-file (EOF), meaning no more data is available + * for reading. For sockets, this usually means the other side has closed the connection. + * + * -1: An error occurred. The specific error can be found by examining the errno variable. + * EAGAIN or EWOULDBLOCK: The file descriptor is in non-blocking mode, and the read operation would block. + * (These are often the same value, but can be different on some systems.) + */ + +ssize_t netdata_ssl_read(NETDATA_SSL *ssl, void *buf, size_t num) { + errno = 0; + ssl->ssl_errno = 0; + + if(unlikely(!is_handshake_complete(ssl, "read"))) + return -1; + + int bytes = SSL_read(ssl->conn, buf, (int)num); + + if(unlikely(bytes <= 0)) { + int err = SSL_get_error(ssl->conn, bytes); + netdata_ssl_log_error_queue("SSL_read", ssl); + if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) { + ssl->ssl_errno = err; + errno = EWOULDBLOCK; + } + + bytes = -1; // according to read() or recv() + } + + return bytes; +} + +/* + * netdata_ssl_write() should return the same as write(): + * + * Positive value: The write() function succeeded and wrote some bytes. The exact number of bytes written is returned. + * + * Zero: It's technically possible for write() to return zero, indicating that zero bytes were written. However, for a + * socket, this generally does not happen unless the size of the data to be written is zero. + * + * -1: An error occurred. The specific error can be found by examining the errno variable. + * EAGAIN or EWOULDBLOCK: The file descriptor is in non-blocking mode, and the write operation would block. + * (These are often the same value, but can be different on some systems.) + */ + +ssize_t netdata_ssl_write(NETDATA_SSL *ssl, const void *buf, size_t num) { + errno = 0; + ssl->ssl_errno = 0; + + if(unlikely(!is_handshake_complete(ssl, "write"))) + return -1; + + int bytes = SSL_write(ssl->conn, (uint8_t *)buf, (int)num); + + if(unlikely(bytes <= 0)) { + int err = SSL_get_error(ssl->conn, bytes); + netdata_ssl_log_error_queue("SSL_write", ssl); + if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) { + ssl->ssl_errno = err; + errno = EWOULDBLOCK; + } + + bytes = -1; // according to write() or send() + } + + return bytes; +} + +static inline bool is_handshake_initialized(NETDATA_SSL *ssl, const char *op) { + error_limit_static_thread_var(erl, 1, 0); + + if(unlikely(!ssl->conn)) { + internal_error(true, "SSL: trying to %s on a NULL connection", op); + return false; + } + + switch(ssl->state) { + case NETDATA_SSL_STATE_NOT_SSL: { + SOCKET_PEERS peers = netdata_ssl_peers(ssl); + error_limit(&erl, "SSL: on socket local [[%s]:%d] <-> remote [[%s]:%d], attempt to %s on non-SSL connection", + peers.local.ip, peers.local.port, peers.peer.ip, peers.peer.port, op); + return false; + } + + case NETDATA_SSL_STATE_INIT: { + return true; + } + + case NETDATA_SSL_STATE_FAILED: { + SOCKET_PEERS peers = netdata_ssl_peers(ssl); + error_limit(&erl, "SSL: on socket local [[%s]:%d] <-> remote [[%s]:%d], attempt to %s on a failed connection", + peers.local.ip, peers.local.port, peers.peer.ip, peers.peer.port, op); + return false; + } + + case NETDATA_SSL_STATE_COMPLETE: { + SOCKET_PEERS peers = netdata_ssl_peers(ssl); + error_limit(&erl, "SSL: on socket local [[%s]:%d] <-> remote [[%s]:%d], attempt to %s on an complete connection", + peers.local.ip, peers.local.port, peers.peer.ip, peers.peer.port, op); + return false; + } + } + + return false; +} + +#define WANT_READ_WRITE_TIMEOUT_MS 10 + +static inline bool want_read_write_should_retry(NETDATA_SSL *ssl, int err) { + int ssl_errno = SSL_get_error(ssl->conn, err); + if(ssl_errno == SSL_ERROR_WANT_READ || ssl_errno == SSL_ERROR_WANT_WRITE) { + struct pollfd pfds[1] = { [0] = { + .fd = SSL_get_rfd(ssl->conn), + .events = (short)(((ssl_errno == SSL_ERROR_WANT_READ ) ? POLLIN : 0) | + ((ssl_errno == SSL_ERROR_WANT_WRITE) ? POLLOUT : 0)), + }}; + + if(poll(pfds, 1, WANT_READ_WRITE_TIMEOUT_MS) <= 0) + return false; // timeout (0) or error (<0) + + return true; // we have activity, so we should retry + } + + return false; // an unknown error +} + +bool netdata_ssl_connect(NETDATA_SSL *ssl) { + errno = 0; + ssl->ssl_errno = 0; + + if(unlikely(!is_handshake_initialized(ssl, "connect"))) + return false; + + SSL_set_connect_state(ssl->conn); + + int err; + while ((err = SSL_connect(ssl->conn)) != 1) { + if(!want_read_write_should_retry(ssl, err)) + break; + } + + if (err != 1) { + netdata_ssl_log_error_queue("SSL_connect", ssl); + ssl->state = NETDATA_SSL_STATE_FAILED; + return false; + } + + ssl->state = NETDATA_SSL_STATE_COMPLETE; + return true; +} + +bool netdata_ssl_accept(NETDATA_SSL *ssl) { + errno = 0; + ssl->ssl_errno = 0; + + if(unlikely(!is_handshake_initialized(ssl, "accept"))) + return false; + + SSL_set_accept_state(ssl->conn); + + int err; + while ((err = SSL_accept(ssl->conn)) != 1) { + if(!want_read_write_should_retry(ssl, err)) + break; + } + + if (err != 1) { + netdata_ssl_log_error_queue("SSL_accept", ssl); + ssl->state = NETDATA_SSL_STATE_FAILED; + return false; + } + + ssl->state = NETDATA_SSL_STATE_COMPLETE; + return true; +} /** * Info Callback @@ -20,7 +396,7 @@ int netdata_ssl_validate_server = NETDATA_SSL_VALID_CERTIFICATE; * @param where the variable with the flags set. * @param ret the return of the caller */ -static void security_info_callback(const SSL *ssl, int where, int ret __maybe_unused) { +static void netdata_ssl_info_callback(const SSL *ssl, int where, int ret __maybe_unused) { (void)ssl; if (where & SSL_CB_ALERT) { debug(D_WEB_CLIENT,"SSL INFO CALLBACK %s %s", SSL_alert_type_string(ret), SSL_alert_desc_string_long(ret)); @@ -32,8 +408,8 @@ static void security_info_callback(const SSL *ssl, int where, int ret __maybe_un * * Starts the openssl library for the Netdata. */ -void security_openssl_library() -{ +void netdata_ssl_initialize_openssl() { + #if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 # if (SSLEAY_VERSION_NUMBER >= OPENSSL_VERSION_097) OPENSSL_config(NULL); @@ -42,10 +418,13 @@ void security_openssl_library() SSL_load_error_strings(); SSL_library_init(); + #else + if (OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL) != 1) { error("SSL library cannot be initialized."); } + #endif } @@ -59,7 +438,7 @@ void security_openssl_library() * * @return it returns the version number. */ -int tls_select_version(const char *lversion) { +static int netdata_ssl_select_tls_version(const char *lversion) { if (!strcmp(lversion, "1") || !strcmp(lversion, "1.0")) return TLS1_VERSION; else if (!strcmp(lversion, "1.1")) @@ -79,36 +458,6 @@ int tls_select_version(const char *lversion) { } #endif -/** - * OpenSSL common options - * - * Clients and SERVER have common options, this function is responsible to set them in the context. - * - * @param ctx the initialized SSL context. - * @param side 0 means server, and 1 client. - */ -void security_openssl_common_options(SSL_CTX *ctx, int side) { -#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_110 - if (!side) { - int version = tls_select_version(tls_version) ; -#endif -#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 - SSL_CTX_set_options (ctx,SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_COMPRESSION); -#else - SSL_CTX_set_min_proto_version(ctx, TLS1_VERSION); - SSL_CTX_set_max_proto_version(ctx, version); - - if(tls_ciphers && strcmp(tls_ciphers, "none") != 0) { - if (!SSL_CTX_set_cipher_list(ctx, tls_ciphers)) { - error("SSL error. cannot set the cipher list"); - } - } - } -#endif - - SSL_CTX_set_mode(ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); -} - /** * Initialize Openssl Client * @@ -116,7 +465,7 @@ void security_openssl_common_options(SSL_CTX *ctx, int side) { * * @return It returns the context on success or NULL otherwise */ -SSL_CTX * security_initialize_openssl_client() { +SSL_CTX * netdata_ssl_create_client_ctx(unsigned long mode) { SSL_CTX *ctx; #if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 ctx = SSL_CTX_new(SSLv23_client_method()); @@ -138,6 +487,9 @@ SSL_CTX * security_initialize_openssl_client() { #endif } + if(mode) + SSL_CTX_set_mode(ctx, mode); + return ctx; } @@ -148,7 +500,7 @@ SSL_CTX * security_initialize_openssl_client() { * * @return It returns the context on success or NULL otherwise */ -static SSL_CTX * security_initialize_openssl_server() { +static SSL_CTX * netdata_ssl_create_server_ctx(unsigned long mode) { SSL_CTX *ctx; char lerror[512]; static int netdata_id_context = 1; @@ -171,7 +523,19 @@ static SSL_CTX * security_initialize_openssl_server() { SSL_CTX_use_certificate_chain_file(ctx, netdata_ssl_security_cert); #endif - security_openssl_common_options(ctx, 0); + +#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 + SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_COMPRESSION); +#else + SSL_CTX_set_min_proto_version(ctx, TLS1_VERSION); + SSL_CTX_set_max_proto_version(ctx, netdata_ssl_select_tls_version(tls_version)); + + if(tls_ciphers && strcmp(tls_ciphers, "none") != 0) { + if (!SSL_CTX_set_cipher_list(ctx, tls_ciphers)) { + error("SSL error. cannot set the cipher list"); + } + } +#endif SSL_CTX_use_PrivateKey_file(ctx, netdata_ssl_security_key,SSL_FILETYPE_PEM); @@ -183,13 +547,15 @@ static SSL_CTX * security_initialize_openssl_server() { } SSL_CTX_set_session_id_context(ctx,(void*)&netdata_id_context,(unsigned int)sizeof(netdata_id_context)); - SSL_CTX_set_info_callback(ctx,security_info_callback); + SSL_CTX_set_info_callback(ctx, netdata_ssl_info_callback); #if (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_095) SSL_CTX_set_verify_depth(ctx,1); #endif debug(D_WEB_CLIENT,"SSL GLOBAL CONTEXT STARTED\n"); + SSL_CTX_set_mode(ctx, mode); + return ctx; } @@ -203,39 +569,54 @@ static SSL_CTX * security_initialize_openssl_server() { * NETDATA_SSL_CONTEXT_STREAMING - Starts the streaming context. * NETDATA_SSL_CONTEXT_EXPORTING - Starts the OpenTSDB context */ -void security_start_ssl(int selector) { +void netdata_ssl_initialize_ctx(int selector) { static SPINLOCK sp = NETDATA_SPINLOCK_INITIALIZER; netdata_spinlock_lock(&sp); switch (selector) { - case NETDATA_SSL_CONTEXT_SERVER: { - if(!netdata_ssl_srv_ctx) { + case NETDATA_SSL_WEB_SERVER_CTX: { + if(!netdata_ssl_web_server_ctx) { struct stat statbuf; if (stat(netdata_ssl_security_key, &statbuf) || stat(netdata_ssl_security_cert, &statbuf)) info("To use encryption it is necessary to set \"ssl certificate\" and \"ssl key\" in [web] !\n"); else { - netdata_ssl_srv_ctx = security_initialize_openssl_server(); - SSL_CTX_set_mode(netdata_ssl_srv_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE); + netdata_ssl_web_server_ctx = netdata_ssl_create_server_ctx( + SSL_MODE_ENABLE_PARTIAL_WRITE | + SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | + // SSL_MODE_AUTO_RETRY | + 0); + + if(netdata_ssl_web_server_ctx && !netdata_ssl_validate_certificate) + SSL_CTX_set_verify(netdata_ssl_web_server_ctx, SSL_VERIFY_NONE, NULL); } } break; } - case NETDATA_SSL_CONTEXT_STREAMING: { - if(!netdata_ssl_client_ctx) { - netdata_ssl_client_ctx = security_initialize_openssl_client(); + case NETDATA_SSL_STREAMING_SENDER_CTX: { + if(!netdata_ssl_streaming_sender_ctx) { //This is necessary for the stream, because it is working sometimes with nonblock socket. //It returns the bitmask after to change, there is not any description of errors in the documentation - SSL_CTX_set_mode(netdata_ssl_client_ctx, - SSL_MODE_ENABLE_PARTIAL_WRITE | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | - SSL_MODE_AUTO_RETRY); + netdata_ssl_streaming_sender_ctx = netdata_ssl_create_client_ctx( + SSL_MODE_ENABLE_PARTIAL_WRITE | + SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | + // SSL_MODE_AUTO_RETRY | + 0 + ); + + if(netdata_ssl_streaming_sender_ctx && !netdata_ssl_validate_certificate_sender) + SSL_CTX_set_verify(netdata_ssl_streaming_sender_ctx, SSL_VERIFY_NONE, NULL); } break; } - case NETDATA_SSL_CONTEXT_EXPORTING: { - if(!netdata_ssl_exporting_ctx) - netdata_ssl_exporting_ctx = security_initialize_openssl_client(); + case NETDATA_SSL_EXPORTING_CTX: { + if(!netdata_ssl_exporting_ctx) { + netdata_ssl_exporting_ctx = netdata_ssl_create_client_ctx(0); + + if(netdata_ssl_exporting_ctx && !netdata_ssl_validate_certificate) + SSL_CTX_set_verify(netdata_ssl_exporting_ctx, SSL_VERIFY_NONE, NULL); + } break; } } @@ -248,18 +629,21 @@ void security_start_ssl(int selector) { * * Clean all the allocated contexts from netdata. */ -void security_clean_openssl() +void netdata_ssl_cleanup() { - if (netdata_ssl_srv_ctx) { - SSL_CTX_free(netdata_ssl_srv_ctx); + if (netdata_ssl_web_server_ctx) { + SSL_CTX_free(netdata_ssl_web_server_ctx); + netdata_ssl_web_server_ctx = NULL; } - if (netdata_ssl_client_ctx) { - SSL_CTX_free(netdata_ssl_client_ctx); + if (netdata_ssl_streaming_sender_ctx) { + SSL_CTX_free(netdata_ssl_streaming_sender_ctx); + netdata_ssl_streaming_sender_ctx = NULL; } if (netdata_ssl_exporting_ctx) { SSL_CTX_free(netdata_ssl_exporting_ctx); + netdata_ssl_exporting_ctx = NULL; } #if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 @@ -267,64 +651,6 @@ void security_clean_openssl() #endif } -/** - * Process accept - * - * Process the SSL handshake with the client case it is necessary. - * - * @param ssl is a pointer for the SSL structure - * @param msg is a copy of the first 8 bytes of the initial message received - * - * @return it returns 0 case it performs the handshake, 8 case it is clean connection - * and another integer power of 2 otherwise. - */ -int security_process_accept(SSL *ssl,int msg) { - int sock = SSL_get_fd(ssl); - int test; - if (msg > 0x17) - { - return NETDATA_SSL_NO_HANDSHAKE; - } - - ERR_clear_error(); - if ((test = SSL_accept(ssl)) <= 0) { - int sslerrno = SSL_get_error(ssl, test); - switch(sslerrno) { - case SSL_ERROR_WANT_READ: - { - error("SSL handshake did not finish and it wanna read on socket %d!", sock); - return NETDATA_SSL_WANT_READ; - } - case SSL_ERROR_WANT_WRITE: - { - error("SSL handshake did not finish and it wanna read on socket %d!", sock); - return NETDATA_SSL_WANT_WRITE; - } - case SSL_ERROR_NONE: - case SSL_ERROR_SSL: - case SSL_ERROR_SYSCALL: - default: - { - u_long err; - char buf[256]; - int counter = 0; - while ((err = ERR_get_error()) != 0) { - ERR_error_string_n(err, buf, sizeof(buf)); - error("%d SSL Handshake error (%s) on socket %d", counter++, ERR_error_string((long)SSL_get_error(ssl, test), NULL), sock); - } - return NETDATA_SSL_NO_HANDSHAKE; - } - } - } - - if (SSL_is_init_finished(ssl)) - { - debug(D_WEB_CLIENT_ACCESS,"SSL Handshake finished %s errno %d on socket fd %d", ERR_error_string((long)SSL_get_error(ssl, test), NULL), errno, sock); - } - - return NETDATA_SSL_HANDSHAKE_COMPLETE; -} - /** * Test Certificate * diff --git a/libnetdata/socket/security.h b/libnetdata/socket/security.h index ae7c595e3..c83b60ad1 100644 --- a/libnetdata/socket/security.h +++ b/libnetdata/socket/security.h @@ -1,20 +1,16 @@ #ifndef NETDATA_SECURITY_H # define NETDATA_SECURITY_H -# define NETDATA_SSL_HANDSHAKE_COMPLETE 0 //All the steps were successful -# define NETDATA_SSL_START 1 //Starting handshake, conn variable is NULL -# define NETDATA_SSL_WANT_READ 2 //The connection wanna read from socket -# define NETDATA_SSL_WANT_WRITE 4 //The connection wanna write on socket -# define NETDATA_SSL_NO_HANDSHAKE 8 //Continue without encrypt connection. -# define NETDATA_SSL_OPTIONAL 16 //Flag to define the HTTP request -# define NETDATA_SSL_FORCE 32 //We only accepts HTTPS request -# define NETDATA_SSL_INVALID_CERTIFICATE 64 //Accepts invalid certificate -# define NETDATA_SSL_VALID_CERTIFICATE 128 //Accepts invalid certificate -# define NETDATA_SSL_PROXY_HTTPS 256 //Proxy is using HTTPS - -#define NETDATA_SSL_CONTEXT_SERVER 0 -#define NETDATA_SSL_CONTEXT_STREAMING 1 -#define NETDATA_SSL_CONTEXT_EXPORTING 2 +typedef enum __attribute__((packed)) { + NETDATA_SSL_STATE_NOT_SSL = 1, // This connection is not SSL + NETDATA_SSL_STATE_INIT, // SSL handshake is initialized + NETDATA_SSL_STATE_FAILED, // SSL handshake failed + NETDATA_SSL_STATE_COMPLETE, // SSL handshake successful +} NETDATA_SSL_STATE; + +#define NETDATA_SSL_WEB_SERVER_CTX 0 +#define NETDATA_SSL_STREAMING_SENDER_CTX 1 +#define NETDATA_SSL_EXPORTING_CTX 2 # ifdef ENABLE_HTTPS @@ -37,27 +33,42 @@ #include #endif -struct netdata_ssl { - SSL *conn; //SSL connection - uint32_t flags; //The flags for SSL connection -}; +typedef struct netdata_ssl { + SSL *conn; // SSL connection + NETDATA_SSL_STATE state; // The state for SSL connection + unsigned long ssl_errno; // The SSL errno of the last SSL call +} NETDATA_SSL; + +#define NETDATA_SSL_UNSET_CONNECTION (NETDATA_SSL){ .conn = NULL, .state = NETDATA_SSL_STATE_NOT_SSL } + +#define SSL_connection(ssl) ((ssl)->conn && (ssl)->state != NETDATA_SSL_STATE_NOT_SSL) extern SSL_CTX *netdata_ssl_exporting_ctx; -extern SSL_CTX *netdata_ssl_client_ctx; -extern SSL_CTX *netdata_ssl_srv_ctx; +extern SSL_CTX *netdata_ssl_streaming_sender_ctx; +extern SSL_CTX *netdata_ssl_web_server_ctx; extern const char *netdata_ssl_security_key; extern const char *netdata_ssl_security_cert; extern const char *tls_version; extern const char *tls_ciphers; -extern int netdata_ssl_validate_server; +extern bool netdata_ssl_validate_certificate; +extern bool netdata_ssl_validate_certificate_sender; int ssl_security_location_for_context(SSL_CTX *ctx,char *file,char *path); -void security_openssl_library(); -void security_clean_openssl(); -void security_start_ssl(int selector); -int security_process_accept(SSL *ssl,int msg); +void netdata_ssl_initialize_openssl(); +void netdata_ssl_cleanup(); +void netdata_ssl_initialize_ctx(int selector); int security_test_certificate(SSL *ssl); -SSL_CTX * security_initialize_openssl_client(); +SSL_CTX * netdata_ssl_create_client_ctx(unsigned long mode); + +bool netdata_ssl_connect(NETDATA_SSL *ssl); +bool netdata_ssl_accept(NETDATA_SSL *ssl); + +bool netdata_ssl_open(NETDATA_SSL *ssl, SSL_CTX *ctx, int fd); +void netdata_ssl_close(NETDATA_SSL *ssl); +void netdata_ssl_log_error_queue(const char *call, NETDATA_SSL *ssl); + +ssize_t netdata_ssl_read(NETDATA_SSL *ssl, void *buf, size_t num); +ssize_t netdata_ssl_write(NETDATA_SSL *ssl, const void *buf, size_t num); # endif //ENABLE_HTTPS #endif //NETDATA_SECURITY_H diff --git a/libnetdata/socket/socket.c b/libnetdata/socket/socket.c index 7eb212b33..220db7601 100644 --- a/libnetdata/socket/socket.c +++ b/libnetdata/socket/socket.c @@ -10,6 +10,63 @@ #include "../libnetdata.h" + +SOCKET_PEERS socket_peers(int sock_fd) { + SOCKET_PEERS peers; + + if(sock_fd < 0) { + strncpyz(peers.peer.ip, "unknown", sizeof(peers.peer.ip) - 1); + peers.peer.port = 0; + + strncpyz(peers.local.ip, "unknown", sizeof(peers.local.ip) - 1); + peers.local.port = 0; + + return peers; + } + + struct sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + + // Get peer info + if (getpeername(sock_fd, (struct sockaddr *)&addr, &addr_len) == 0) { + if (addr.ss_family == AF_INET) { // IPv4 + struct sockaddr_in *s = (struct sockaddr_in *)&addr; + inet_ntop(AF_INET, &s->sin_addr, peers.peer.ip, sizeof(peers.peer.ip)); + peers.peer.port = ntohs(s->sin_port); + } + else { // IPv6 + struct sockaddr_in6 *s = (struct sockaddr_in6 *)&addr; + inet_ntop(AF_INET6, &s->sin6_addr, peers.peer.ip, sizeof(peers.peer.ip)); + peers.peer.port = ntohs(s->sin6_port); + } + } + else { + strncpyz(peers.peer.ip, "unknown", sizeof(peers.peer.ip) - 1); + peers.peer.port = 0; + } + + // Get local info + addr_len = sizeof(addr); + if (getsockname(sock_fd, (struct sockaddr *)&addr, &addr_len) == 0) { + if (addr.ss_family == AF_INET) { // IPv4 + struct sockaddr_in *s = (struct sockaddr_in *) &addr; + inet_ntop(AF_INET, &s->sin_addr, peers.local.ip, sizeof(peers.local.ip)); + peers.local.port = ntohs(s->sin_port); + } else { // IPv6 + struct sockaddr_in6 *s = (struct sockaddr_in6 *) &addr; + inet_ntop(AF_INET6, &s->sin6_addr, peers.local.ip, sizeof(peers.local.ip)); + peers.local.port = ntohs(s->sin6_port); + } + } + else { + strncpyz(peers.local.ip, "unknown", sizeof(peers.local.ip) - 1); + peers.local.port = 0; + } + + return peers; +} + + // -------------------------------------------------------------------------------------------------------------------- // various library calls @@ -967,49 +1024,11 @@ int connect_to_one_of_urls(const char *destination, int default_port, struct tim } -#ifdef ENABLE_HTTPS -ssize_t netdata_ssl_read(SSL *ssl, void *buf, size_t num) { - error_limit_static_thread_var(erl, 1, 0); - - int bytes, err; - - bytes = SSL_read(ssl, buf, (int)num); - err = SSL_get_error(ssl, bytes); - - if(unlikely(bytes <= 0)) { - if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) { - bytes = 0; - } else - error_limit(&erl, "SSL_write() returned %d bytes, SSL error %d", bytes, err); - } - - return bytes; -} - -ssize_t netdata_ssl_write(SSL *ssl, const void *buf, size_t num) { - error_limit_static_thread_var(erl, 1, 0); - - int bytes, err; - - bytes = SSL_write(ssl, (uint8_t *)buf, (int)num); - err = SSL_get_error(ssl, bytes); - - if(unlikely(bytes <= 0)) { - if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ) { - bytes = 0; - } else - error_limit(&erl, "SSL_write() returned %d bytes, SSL error %d", bytes, err); - } - - return bytes; -} -#endif - // -------------------------------------------------------------------------------------------------------------------- // helpers to send/receive data in one call, in blocking mode, with a timeout #ifdef ENABLE_HTTPS -ssize_t recv_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) { +ssize_t recv_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) { #else ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) { #endif @@ -1033,24 +1052,24 @@ ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) return -1; } - if(!retval) { + if(!retval) // timeout return 0; - } - if(fd.events & POLLIN) break; + if(fd.revents & POLLIN) + break; } #ifdef ENABLE_HTTPS - if (ssl->conn && ssl->flags == NETDATA_SSL_HANDSHAKE_COMPLETE) - return netdata_ssl_read(ssl->conn, buf, len); + if (SSL_connection(ssl)) + return netdata_ssl_read(ssl, buf, len); #endif return recv(sockfd, buf, len, flags); } #ifdef ENABLE_HTTPS -ssize_t send_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) { +ssize_t send_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout) { #else ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) { #endif @@ -1079,13 +1098,13 @@ ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) return 0; } - if(fd.events & POLLOUT) break; + if(fd.revents & POLLOUT) break; } #ifdef ENABLE_HTTPS if(ssl->conn) { - if (ssl->flags == NETDATA_SSL_HANDSHAKE_COMPLETE) { - return netdata_ssl_write(ssl->conn, buf, len); + if (SSL_connection(ssl)) { + return netdata_ssl_write(ssl, buf, len); } else { error("cannot write to SSL connection - connection is not ready."); diff --git a/libnetdata/socket/socket.h b/libnetdata/socket/socket.h index 110063014..0e29711e0 100644 --- a/libnetdata/socket/socket.h +++ b/libnetdata/socket/socket.h @@ -68,10 +68,8 @@ int connect_to_one_of_urls(const char *destination, int default_port, struct tim #ifdef ENABLE_HTTPS -ssize_t recv_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout); -ssize_t send_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout); -ssize_t netdata_ssl_read(SSL *ssl, void *buf, size_t num); -ssize_t netdata_ssl_write(SSL *ssl, const void *buf, size_t num); +ssize_t recv_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout); +ssize_t send_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout); #else ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout); ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout); @@ -219,4 +217,22 @@ void poll_events(LISTEN_SOCKETS *sockets , size_t max_tcp_sockets ); +#ifndef INET6_ADDRSTRLEN +#define INET6_ADDRSTRLEN 46 +#endif + +typedef struct socket_peers { + struct { + char ip[INET6_ADDRSTRLEN]; + int port; + } local; + + struct { + char ip[INET6_ADDRSTRLEN]; + int port; + } peer; +} SOCKET_PEERS; + +SOCKET_PEERS socket_peers(int sock_fd); + #endif //NETDATA_SOCKET_H -- cgit v1.2.3