diff options
Diffstat (limited to 'src/libnetdata')
208 files changed, 19029 insertions, 6840 deletions
diff --git a/src/libnetdata/README.md b/src/libnetdata/README.md index fd2c79730..c425c07a6 100644 --- a/src/libnetdata/README.md +++ b/src/libnetdata/README.md @@ -1,14 +1,3 @@ -<!-- -title: "libnetdata" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/README.md -sidebar_label: "libnetdata" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # libnetdata `libnetdata` is a collection of library code that is used by all Netdata `C` programs. - - diff --git a/src/libnetdata/adaptive_resortable_list/README.md b/src/libnetdata/adaptive_resortable_list/README.md index 9aa864c9e..e5c22c519 100644 --- a/src/libnetdata/adaptive_resortable_list/README.md +++ b/src/libnetdata/adaptive_resortable_list/README.md @@ -1,12 +1,3 @@ -<!-- -title: "Adaptive Re-sortable List (ARL)" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/adaptive_resortable_list/README.md -sidebar_label: "Adaptive Re-sortable List (ARL)" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Adaptive Re-sortable List (ARL) This library allows Netdata to read a series of `name - value` pairs diff --git a/src/libnetdata/aral/README.md b/src/libnetdata/aral/README.md index d999e820a..564f00f6d 100644 --- a/src/libnetdata/aral/README.md +++ b/src/libnetdata/aral/README.md @@ -1,12 +1,3 @@ -<!-- -title: "Array Allocator" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/aral/README.md -sidebar_label: "Array allocator" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Array Allocator Come on! Array allocators are embedded in libc! Why do we need such a thing in Netdata? diff --git a/src/libnetdata/aral/aral.c b/src/libnetdata/aral/aral.c index 64b63d8e0..2ec27b3df 100644 --- a/src/libnetdata/aral/aral.c +++ b/src/libnetdata/aral/aral.c @@ -72,7 +72,7 @@ struct aral { struct { bool enabled; const char *filename; - char **cache_dir; + const char **cache_dir; } mmap; } config; @@ -117,7 +117,7 @@ size_t aral_structures(ARAL *ar) { return aral_structures_from_stats(ar->stats); } -struct aral_statistics *aral_statistics(ARAL *ar) { +struct aral_statistics *aral_get_statistics(ARAL *ar) { return ar->stats; } @@ -709,7 +709,7 @@ size_t aral_element_size(ARAL *ar) { } ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_elements, size_t max_page_size, - struct aral_statistics *stats, const char *filename, char **cache_dir, bool mmap, bool lockless) { + struct aral_statistics *stats, const char *filename, const char **cache_dir, bool mmap, bool lockless) { ARAL *ar = callocz(1, sizeof(ARAL)); ar->config.options = (lockless) ? ARAL_LOCKLESS : 0; ar->config.requested_element_size = element_size; @@ -799,6 +799,7 @@ ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_ele aral_delete_leftover_files(ar->config.name, directory_name, file); } + errno_clear(); internal_error(true, "ARAL: '%s' " "element size %zu (requested %zu bytes), " @@ -1078,7 +1079,7 @@ int aral_stress_test(size_t threads, size_t elements, size_t seconds) { } int aral_unittest(size_t elements) { - char *cache_dir = "/tmp/"; + const char *cache_dir = "/tmp/"; struct aral_unittest_config auc = { .single_threaded = true, diff --git a/src/libnetdata/aral/aral.h b/src/libnetdata/aral/aral.h index 2e749bc4c..4cd21d17a 100644 --- a/src/libnetdata/aral/aral.h +++ b/src/libnetdata/aral/aral.h @@ -28,11 +28,11 @@ struct aral_statistics { }; ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_elements, size_t max_page_size, - struct aral_statistics *stats, const char *filename, char **cache_dir, bool mmap, bool lockless); + struct aral_statistics *stats, const char *filename, const char **cache_dir, bool mmap, bool lockless); size_t aral_element_size(ARAL *ar); size_t aral_overhead(ARAL *ar); size_t aral_structures(ARAL *ar); -struct aral_statistics *aral_statistics(ARAL *ar); +struct aral_statistics *aral_get_statistics(ARAL *ar); size_t aral_structures_from_stats(struct aral_statistics *stats); size_t aral_overhead_from_stats(struct aral_statistics *stats); diff --git a/src/libnetdata/avl/README.md b/src/libnetdata/avl/README.md index eb85f884e..0b4a39b43 100644 --- a/src/libnetdata/avl/README.md +++ b/src/libnetdata/avl/README.md @@ -1,12 +1,3 @@ -<!-- -title: "AVL" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/avl/README.md -sidebar_label: "AVL" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # AVL AVL is a library indexing objects in B-Trees. diff --git a/src/libnetdata/bitmap64.h b/src/libnetdata/bitmap/bitmap64.h index 425f3fd20..425f3fd20 100644 --- a/src/libnetdata/bitmap64.h +++ b/src/libnetdata/bitmap/bitmap64.h diff --git a/src/libnetdata/buffer/README.md b/src/libnetdata/buffer/README.md index a7850df72..460c07753 100644 --- a/src/libnetdata/buffer/README.md +++ b/src/libnetdata/buffer/README.md @@ -1,12 +1,3 @@ -<!-- -title: "BUFFER" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/buffer/README.md -sidebar_label: "BUFFER library" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # BUFFER `BUFFER` is a convenience library for working with strings in `C`. diff --git a/src/libnetdata/buffer/buffer.c b/src/libnetdata/buffer/buffer.c index 119216dd9..7194134b4 100644 --- a/src/libnetdata/buffer/buffer.c +++ b/src/libnetdata/buffer/buffer.c @@ -247,7 +247,7 @@ void buffer_free(BUFFER *b) { buffer_overflow_check(b); - netdata_log_debug(D_WEB_BUFFER, "Freeing web buffer of size %zu.", b->size); + netdata_log_debug(D_WEB_BUFFER, "Freeing web buffer of size %zu.", (size_t)b->size); if(b->statistics) __atomic_sub_fetch(b->statistics, b->size + sizeof(BUFFER) + sizeof(BUFFER_OVERFLOW_EOF) + 2, __ATOMIC_RELAXED); @@ -269,7 +269,7 @@ void buffer_increase(BUFFER *b, size_t free_size_required) { size_t optimal = (b->size > 5*1024*1024) ? b->size / 2 : b->size; if(optimal > increase) increase = optimal; - netdata_log_debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + increase); + netdata_log_debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", (size_t)b->size, (size_t)(b->size + increase)); b->buffer = reallocz(b->buffer, b->size + increase + sizeof(BUFFER_OVERFLOW_EOF) + 2); b->size += increase; diff --git a/src/libnetdata/buffer/buffer.h b/src/libnetdata/buffer/buffer.h index 92e14afb1..78ee49d54 100644 --- a/src/libnetdata/buffer/buffer.h +++ b/src/libnetdata/buffer/buffer.h @@ -3,6 +3,8 @@ #ifndef NETDATA_WEB_BUFFER_H #define NETDATA_WEB_BUFFER_H 1 +#include "../uuid/uuid.h" +#include "../http/content_type.h" #include "../string/utf8.h" #include "../libnetdata.h" @@ -39,14 +41,15 @@ typedef enum __attribute__ ((__packed__)) { } BUFFER_JSON_OPTIONS; typedef struct web_buffer { - size_t size; // allocation size of buffer, in bytes - size_t len; // current data length in buffer, in bytes - char *buffer; // the buffer itself + uint32_t size; // allocation size of buffer, in bytes + uint32_t len; // current data length in buffer, in bytes HTTP_CONTENT_TYPE content_type; // the content type of the data in the buffer BUFFER_OPTIONS options; // options related to the content + uint16_t response_code; time_t date; // the timestamp this content has been generated time_t expires; // the timestamp this content expires size_t *statistics; + char *buffer; // the buffer itself struct { char key_quote[BUFFER_QUOTE_MAX_SIZE + 1]; @@ -62,7 +65,7 @@ typedef struct web_buffer { #define buffer_cacheable(wb) do { (wb)->options |= WB_CONTENT_CACHEABLE; if((wb)->options & WB_CONTENT_NO_CACHEABLE) (wb)->options &= ~WB_CONTENT_NO_CACHEABLE; } while(0) #define buffer_no_cacheable(wb) do { (wb)->options |= WB_CONTENT_NO_CACHEABLE; if((wb)->options & WB_CONTENT_CACHEABLE) (wb)->options &= ~WB_CONTENT_CACHEABLE; (wb)->expires = 0; } while(0) -#define buffer_strlen(wb) ((wb)->len) +#define buffer_strlen(wb) (size_t)((wb)->len) #define BUFFER_OVERFLOW_EOF "EOF" @@ -152,13 +155,10 @@ static inline void _buffer_json_depth_pop(BUFFER *wb) { wb->json.depth--; } -static inline void buffer_fast_charcat(BUFFER *wb, const char c) { - +static inline void buffer_putc(BUFFER *wb, char c) { buffer_need_bytes(wb, 2); - *(&wb->buffer[wb->len]) = c; - wb->len += 1; + wb->buffer[wb->len++] = c; wb->buffer[wb->len] = '\0'; - buffer_overflow_check(wb); } @@ -181,13 +181,6 @@ static inline void buffer_fast_rawcat(BUFFER *wb, const char *txt, size_t len) { buffer_overflow_check(wb); } -static inline void buffer_putc(BUFFER *wb, char c) { - buffer_need_bytes(wb, 2); - wb->buffer[wb->len++] = c; - wb->buffer[wb->len] = '\0'; - buffer_overflow_check(wb); -} - static inline void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len) { if(unlikely(!txt || !*txt || !len)) return; @@ -423,6 +416,16 @@ static inline char *print_uint64_hex_reversed(char *dst, uint64_t value) { #endif } +static inline char *print_uint64_hex_reversed_full(char *dst, uint64_t value) { + char *d = dst; + for(size_t c = 0; c < sizeof(uint64_t) * 2; c++) { + *d++ = hex_digits[value & 0xf]; + value >>= 4; + } + + return d; +} + static inline char *print_uint64_base64_reversed(char *dst, uint64_t value) { char *d = dst; do *d++ = base64_digits[value & 63]; while ((value >>= 6)); @@ -498,47 +501,79 @@ static inline int print_netdata_double(char *dst, NETDATA_DOUBLE value) { return (int)(d - dst); } -static inline void buffer_print_uint64(BUFFER *wb, uint64_t value) { - buffer_need_bytes(wb, 50); - - char *s = &wb->buffer[wb->len]; +static inline size_t print_uint64(char *dst, uint64_t value) { + char *s = dst; char *d = print_uint64_reversed(s, value); char_array_reverse(s, d - 1); *d = '\0'; - wb->len += d - s; - - buffer_overflow_check(wb); + return d - s; } -static inline void buffer_print_int64(BUFFER *wb, int64_t value) { - buffer_need_bytes(wb, 50); +static inline size_t print_int64(char *dst, int64_t value) { + size_t len = 0; if(value < 0) { - buffer_fast_strcat(wb, "-", 1); + *dst++ = '-'; value = -value; + len++; } - buffer_print_uint64(wb, (uint64_t)value); + return print_uint64(dst, value) + len; +} +#define UINT64_MAX_LENGTH (24) // 21 should be enough +static inline void buffer_print_uint64(BUFFER *wb, uint64_t value) { + buffer_need_bytes(wb, UINT64_MAX_LENGTH); + wb->len += print_uint64(&wb->buffer[wb->len], value); buffer_overflow_check(wb); } -static inline void buffer_print_uint64_hex(BUFFER *wb, uint64_t value) { - buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1); +static inline void buffer_print_int64(BUFFER *wb, int64_t value) { + buffer_need_bytes(wb, UINT64_MAX_LENGTH); + wb->len += print_int64(&wb->buffer[wb->len], value); + buffer_overflow_check(wb); +} - buffer_fast_strcat(wb, HEX_PREFIX, sizeof(HEX_PREFIX) - 1); +#define UINT64_HEX_MAX_LENGTH ((sizeof(HEX_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1) +static inline size_t print_uint64_hex(char *dst, uint64_t value) { + char *d = dst; - char *s = &wb->buffer[wb->len]; - char *d = print_uint64_hex_reversed(s, value); - char_array_reverse(s, d - 1); - *d = '\0'; - wb->len += d - s; + const char *s = HEX_PREFIX; + while(*s) *d++ = *s++; + + char *e = print_uint64_hex_reversed(d, value); + char_array_reverse(d, e - 1); + *e = '\0'; + return e - dst; +} + +static inline size_t print_uint64_hex_full(char *dst, uint64_t value) { + char *d = dst; + + const char *s = HEX_PREFIX; + while(*s) *d++ = *s++; + + char *e = print_uint64_hex_reversed_full(d, value); + char_array_reverse(d, e - 1); + *e = '\0'; + return e - dst; +} + +static inline void buffer_print_uint64_hex(BUFFER *wb, uint64_t value) { + buffer_need_bytes(wb, UINT64_HEX_MAX_LENGTH); + wb->len += print_uint64_hex(&wb->buffer[wb->len], value); + buffer_overflow_check(wb); +} +static inline void buffer_print_uint64_hex_full(BUFFER *wb, uint64_t value) { + buffer_need_bytes(wb, UINT64_HEX_MAX_LENGTH); + wb->len += print_uint64_hex_full(&wb->buffer[wb->len], value); buffer_overflow_check(wb); } +#define UINT64_B64_MAX_LENGTH ((sizeof(IEEE754_UINT64_B64_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1) static inline void buffer_print_uint64_base64(BUFFER *wb, uint64_t value) { - buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1); + buffer_need_bytes(wb, UINT64_B64_MAX_LENGTH); buffer_fast_strcat(wb, IEEE754_UINT64_B64_PREFIX, sizeof(IEEE754_UINT64_B64_PREFIX) - 1); @@ -577,8 +612,9 @@ static inline void buffer_print_int64_base64(BUFFER *wb, int64_t value) { buffer_overflow_check(wb); } +#define DOUBLE_MAX_LENGTH (512) // 318 should be enough, including null static inline void buffer_print_netdata_double(BUFFER *wb, NETDATA_DOUBLE value) { - buffer_need_bytes(wb, 512 + 2); + buffer_need_bytes(wb, DOUBLE_MAX_LENGTH); if(isnan(value) || isinf(value)) { buffer_fast_strcat(wb, "null", 4); @@ -594,8 +630,9 @@ static inline void buffer_print_netdata_double(BUFFER *wb, NETDATA_DOUBLE value) buffer_overflow_check(wb); } +#define DOUBLE_HEX_MAX_LENGTH ((sizeof(IEEE754_DOUBLE_HEX_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1) static inline void buffer_print_netdata_double_hex(BUFFER *wb, NETDATA_DOUBLE value) { - buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1 + 1); + buffer_need_bytes(wb, DOUBLE_HEX_MAX_LENGTH); uint64_t *ptr = (uint64_t *) (&value); buffer_fast_strcat(wb, IEEE754_DOUBLE_HEX_PREFIX, sizeof(IEEE754_DOUBLE_HEX_PREFIX) - 1); @@ -609,8 +646,9 @@ static inline void buffer_print_netdata_double_hex(BUFFER *wb, NETDATA_DOUBLE va buffer_overflow_check(wb); } +#define DOUBLE_B64_MAX_LENGTH ((sizeof(IEEE754_DOUBLE_B64_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1) static inline void buffer_print_netdata_double_base64(BUFFER *wb, NETDATA_DOUBLE value) { - buffer_need_bytes(wb, sizeof(uint64_t) * 2 + 2 + 1 + 1); + buffer_need_bytes(wb, DOUBLE_B64_MAX_LENGTH); uint64_t *ptr = (uint64_t *) (&value); buffer_fast_strcat(wb, IEEE754_DOUBLE_B64_PREFIX, sizeof(IEEE754_DOUBLE_B64_PREFIX) - 1); @@ -775,7 +813,7 @@ static inline void buffer_json_member_add_quoted_string(BUFFER *wb, const char * wb->json.stack[wb->json.depth].count++; } -static inline void buffer_json_member_add_uuid(BUFFER *wb, const char *key, nd_uuid_t *value) { +static inline void buffer_json_member_add_uuid_ptr(BUFFER *wb, const char *key, nd_uuid_t *value) { buffer_print_json_comma_newline_spacing(wb); buffer_print_json_key(wb, key); buffer_fast_strcat(wb, ":", 1); @@ -791,6 +829,22 @@ static inline void buffer_json_member_add_uuid(BUFFER *wb, const char *key, nd_u wb->json.stack[wb->json.depth].count++; } +static inline void buffer_json_member_add_uuid(BUFFER *wb, const char *key, nd_uuid_t value) { + buffer_print_json_comma_newline_spacing(wb); + buffer_print_json_key(wb, key); + buffer_fast_strcat(wb, ":", 1); + + if(!uuid_is_null(value)) { + char uuid[GUID_LEN + 1]; + uuid_unparse_lower(value, uuid); + buffer_json_add_string_value(wb, uuid); + } + else + buffer_json_add_string_value(wb, NULL); + + wb->json.stack[wb->json.depth].count++; +} + static inline void buffer_json_member_add_boolean(BUFFER *wb, const char *key, bool value) { buffer_print_json_comma_newline_spacing(wb); buffer_print_json_key(wb, key); @@ -1066,6 +1120,7 @@ typedef enum __attribute__((packed)) { RRDF_FIELD_TRANSFORM_DURATION_S, // transform as duration in second to a human-readable duration RRDF_FIELD_TRANSFORM_DATETIME_MS, // UNIX epoch timestamp in ms RRDF_FIELD_TRANSFORM_DATETIME_USEC, // UNIX epoch timestamp in usec + RRDF_FIELD_TRANSFORM_XML, // format the field with an XML prettifier } RRDF_FIELD_TRANSFORM; static inline const char *rrdf_field_transform_to_string(RRDF_FIELD_TRANSFORM transform) { @@ -1085,6 +1140,9 @@ static inline const char *rrdf_field_transform_to_string(RRDF_FIELD_TRANSFORM tr case RRDF_FIELD_TRANSFORM_DATETIME_USEC: return "datetime_usec"; + + case RRDF_FIELD_TRANSFORM_XML: + return "xml"; } } diff --git a/src/libnetdata/buffered_reader/buffered_reader.h b/src/libnetdata/buffered_reader/buffered_reader.h index 1ec1d762b..505070b1c 100644 --- a/src/libnetdata/buffered_reader/buffered_reader.h +++ b/src/libnetdata/buffered_reader/buffered_reader.h @@ -55,9 +55,7 @@ static inline buffered_reader_ret_t buffered_reader_read(struct buffered_reader static inline buffered_reader_ret_t buffered_reader_read_timeout(struct buffered_reader *reader, int fd, int timeout_ms, bool log_error) { short int revents = 0; switch(wait_on_socket_or_cancel_with_timeout( -#ifdef ENABLE_HTTPS NULL, -#endif fd, timeout_ms, POLLIN, &revents)) { case 0: // data are waiting diff --git a/src/libnetdata/c_rhash/c_rhash.c b/src/libnetdata/c_rhash/c_rhash.c new file mode 100644 index 000000000..ec2c061a2 --- /dev/null +++ b/src/libnetdata/c_rhash/c_rhash.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" +#include "c_rhash_internal.h" + +c_rhash c_rhash_new(size_t bin_count) { + if (!bin_count) + bin_count = 1000; + + c_rhash hash = callocz(1, sizeof(struct c_rhash_s) + (bin_count * sizeof(struct bin_ll*)) ); + hash->bin_count = bin_count; + hash->bins = (c_rhash_bin *)((char*)hash + sizeof(struct c_rhash_s)); + + return hash; +} + +static size_t get_itemtype_len(uint8_t item_type, const void* item_data) { + switch (item_type) { + case ITEMTYPE_STRING: + return strlen(item_data) + 1; + case ITEMTYPE_UINT64: + return sizeof(uint64_t); + case ITEMTYPE_UINT8: + return 1; + case ITEMTYPE_OPAQUE_PTR: + return sizeof(void*); + default: + return 0; + } +} + +static int compare_bin_item(struct bin_item *item, uint8_t key_type, const void *key) { + if (item->key_type != key_type) + return 1; + + size_t key_value_len = get_itemtype_len(key_type, key); + + if(key_type == ITEMTYPE_STRING) { + size_t new_key_value_len = get_itemtype_len(item->key_type, item->key); + if (new_key_value_len != key_value_len) + return 1; + } + + if(memcmp(item->key, key, key_value_len) == 0) { + return 0; + } + + return 1; +} + +static int insert_into_bin(c_rhash_bin *bin, uint8_t key_type, const void *key, uint8_t value_type, const void *value) { + struct bin_item *prev = NULL; + while (*bin != NULL) { + if (!compare_bin_item(*bin, key_type, key)) { + freez((*bin)->value); + (*bin)->value_type = value_type; + (*bin)->value = mallocz(get_itemtype_len(value_type, value)); + memcpy((*bin)->value, value, get_itemtype_len(value_type, value)); + return 0; + } + prev = *bin; + bin = &(*bin)->next; + } + + if (*bin == NULL) + *bin = callocz(1, sizeof(struct bin_item)); + if (prev != NULL) + prev->next = *bin; + + (*bin)->key_type = key_type; + size_t len = get_itemtype_len(key_type, key); + (*bin)->key = mallocz(len); + memcpy((*bin)->key, key, len); + + (*bin)->value_type = value_type; + len = get_itemtype_len(value_type, value); + (*bin)->value = mallocz(len); + memcpy((*bin)->value, value, len); + return 0; +} + +static inline uint32_t get_bin_idx_str(c_rhash hash, const char *key) { + uint32_t nhash = simple_hash(key); + return nhash % hash->bin_count; +} + +static inline c_rhash_bin *get_binptr_by_str(c_rhash hash, const char *key) { + return &hash->bins[get_bin_idx_str(hash, key)]; +} + +int c_rhash_insert_str_ptr(c_rhash hash, const char *key, void *value) { + c_rhash_bin *bin = get_binptr_by_str(hash, key); + + return insert_into_bin(bin, ITEMTYPE_STRING, key, ITEMTYPE_OPAQUE_PTR, &value); +} + +int c_rhash_insert_str_uint8(c_rhash hash, const char *key, uint8_t value) { + c_rhash_bin *bin = get_binptr_by_str(hash, key); + + return insert_into_bin(bin, ITEMTYPE_STRING, key, ITEMTYPE_UINT8, &value); +} + +int c_rhash_insert_uint64_ptr(c_rhash hash, uint64_t key, void *value) { + c_rhash_bin *bin = &hash->bins[key % hash->bin_count]; + + return insert_into_bin(bin, ITEMTYPE_UINT64, &key, ITEMTYPE_OPAQUE_PTR, &value); +} + +int c_rhash_get_uint8_by_str(c_rhash hash, const char *key, uint8_t *ret_val) { + uint32_t nhash = get_bin_idx_str(hash, key); + + struct bin_item *bin = hash->bins[nhash]; + + while (bin) { + if (bin->key_type == ITEMTYPE_STRING) { + if (!strcmp(bin->key, key)) { + *ret_val = *(uint8_t*)bin->value; + return 0; + } + } + bin = bin->next; + } + return 1; +} + +int c_rhash_get_ptr_by_str(c_rhash hash, const char *key, void **ret_val) { + uint32_t nhash = get_bin_idx_str(hash, key); + + struct bin_item *bin = hash->bins[nhash]; + + while (bin) { + if (bin->key_type == ITEMTYPE_STRING) { + if (!strcmp(bin->key, key)) { + *ret_val = *((void**)bin->value); + return 0; + } + } + bin = bin->next; + } + *ret_val = NULL; + return 1; +} + +int c_rhash_get_ptr_by_uint64(c_rhash hash, uint64_t key, void **ret_val) { + uint32_t nhash = key % hash->bin_count; + + struct bin_item *bin = hash->bins[nhash]; + + while (bin) { + if (bin->key_type == ITEMTYPE_UINT64) { + if (*((uint64_t *)bin->key) == key) { + *ret_val = *((void**)bin->value); + return 0; + } + } + bin = bin->next; + } + *ret_val = NULL; + return 1; +} + +static void c_rhash_destroy_bin(c_rhash_bin bin) { + struct bin_item *next; + do { + next = bin->next; + freez(bin->key); + freez(bin->value); + freez(bin); + bin = next; + } while (bin != NULL); +} + +int c_rhash_iter_uint64_keys(c_rhash hash, c_rhash_iter_t *iter, uint64_t *key) { + while (iter->bin < hash->bin_count) { + if (iter->item != NULL) + iter->item = iter->item->next; + if (iter->item == NULL) { + if (iter->initialized) + iter->bin++; + else + iter->initialized = 1; + if (iter->bin < hash->bin_count) + iter->item = hash->bins[iter->bin]; + } + if (iter->item != NULL && iter->item->key_type == ITEMTYPE_UINT64) { + *key = *(uint64_t*)iter->item->key; + return 0; + } + } + return 1; +} + +int c_rhash_iter_str_keys(c_rhash hash, c_rhash_iter_t *iter, const char **key) { + while (iter->bin < hash->bin_count) { + if (iter->item != NULL) + iter->item = iter->item->next; + if (iter->item == NULL) { + if (iter->initialized) + iter->bin++; + else + iter->initialized = 1; + if (iter->bin < hash->bin_count) + iter->item = hash->bins[iter->bin]; + } + if (iter->item != NULL && iter->item->key_type == ITEMTYPE_STRING) { + *key = (const char*)iter->item->key; + return 0; + } + } + return 1; +} + +void c_rhash_destroy(c_rhash hash) { + for (size_t i = 0; i < hash->bin_count; i++) { + if (hash->bins[i] != NULL) + c_rhash_destroy_bin(hash->bins[i]); + } + freez(hash); +} diff --git a/src/libnetdata/c_rhash/c_rhash.h b/src/libnetdata/c_rhash/c_rhash.h new file mode 100644 index 000000000..990ef5432 --- /dev/null +++ b/src/libnetdata/c_rhash/c_rhash.h @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef C_RHASH_H +#define C_RHASH_H +#include "../libnetdata.h" + +#ifndef DEFAULT_BIN_COUNT + #define DEFAULT_BIN_COUNT 1000 +#endif + +#define ITEMTYPE_UNSET (0x0) +#define ITEMTYPE_STRING (0x1) +#define ITEMTYPE_UINT8 (0x2) +#define ITEMTYPE_UINT64 (0x3) +#define ITEMTYPE_OPAQUE_PTR (0x4) + +typedef struct c_rhash_s *c_rhash; + +c_rhash c_rhash_new(size_t bin_count); + +void c_rhash_destroy(c_rhash hash); + +// # Insert +// ## Insert where key is string +int c_rhash_insert_str_ptr(c_rhash hash, const char *key, void *value); +int c_rhash_insert_str_uint8(c_rhash hash, const char *key, uint8_t value); +// ## Insert where key is uint64 +int c_rhash_insert_uint64_ptr(c_rhash hash, uint64_t key, void *value); + +// # Get +// ## Get where key is string +int c_rhash_get_ptr_by_str(c_rhash hash, const char *key, void **ret_val); +int c_rhash_get_uint8_by_str(c_rhash hash, const char *key, uint8_t *ret_val); +// ## Get where key is uint64 +int c_rhash_get_ptr_by_uint64(c_rhash hash, uint64_t key, void **ret_val); + +typedef struct { + size_t bin; + struct bin_item *item; + int initialized; +} c_rhash_iter_t; + +#define C_RHASH_ITER_T_INITIALIZER { .bin = 0, .item = NULL, .initialized = 0 } + +#define c_rhash_iter_t_initialize(p_iter) memset(p_iter, 0, sizeof(c_rhash_iter_t)) + +/* + * goes trough whole hash map and returns every + * type uint64 key present/stored + * + * it is not necessary to finish iterating and iterator can be reinitialized + * there are no guarantees on the order in which the keys will come + * behavior here is implementation dependent and can change any time + * + * returns: + * 0 for every key and stores the key in *key + * 1 on error or when all keys of this type has been already iterated over + */ +int c_rhash_iter_uint64_keys(c_rhash hash, c_rhash_iter_t *iter, uint64_t *key); + +int c_rhash_iter_str_keys(c_rhash hash, c_rhash_iter_t *iter, const char **key); + +#endif diff --git a/src/libnetdata/c_rhash/c_rhash_internal.h b/src/libnetdata/c_rhash/c_rhash_internal.h new file mode 100644 index 000000000..c5800310c --- /dev/null +++ b/src/libnetdata/c_rhash/c_rhash_internal.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "c_rhash.h" + +struct bin_item { + uint8_t key_type:4; + void *key; + uint8_t value_type:4; + void *value; + + struct bin_item *next; +}; + +typedef struct bin_item *c_rhash_bin; + +struct c_rhash_s { + size_t bin_count; + c_rhash_bin *bins; +}; diff --git a/src/libnetdata/c_rhash/tests.c b/src/libnetdata/c_rhash/tests.c new file mode 100644 index 000000000..3caa7d003 --- /dev/null +++ b/src/libnetdata/c_rhash/tests.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <stdio.h> +#include <string.h> + +#include "c_rhash.h" + +// terminal color codes +#define KNRM "\x1B[0m" +#define KRED "\x1B[31m" +#define KGRN "\x1B[32m" +#define KYEL "\x1B[33m" +#define KBLU "\x1B[34m" +#define KMAG "\x1B[35m" +#define KCYN "\x1B[36m" +#define KWHT "\x1B[37m" + +#define KEY_1 "key1" +#define KEY_2 "keya" + +#define PRINT_ERR(str, ...) fprintf(stderr, "└─╼ ❌ " KRED str KNRM "\n" __VA_OPT__(,) __VA_ARGS__) + +#define ASSERT_RETVAL(fnc, comparator, expected_retval, ...) \ +{ int rval; \ +if(!((rval = fnc(__VA_ARGS__)) comparator expected_retval)) { \ + PRINT_ERR("Failed test. Value returned by \"%s\" in fnc:\"%s\",line:%d is not equal to expected value. Expected:%d, Got:%d", #fnc, __FUNCTION__, __LINE__, expected_retval, rval); \ + rc = 1; \ + goto test_cleanup; \ +} passed_subtest_count++;}; + +#define ASSERT_VAL_UINT8(returned, expected) \ +if(returned != expected) { \ + PRINT_ERR("Failed test. Value returned (%d) doesn't match expected (%d)! fnc:\"%s\",line:%d", returned, expected, __FUNCTION__, __LINE__); \ + rc = 1; \ + goto test_cleanup; \ +} passed_subtest_count++; + +#define ASSERT_VAL_PTR(returned, expected) \ +if((void*)returned != (void*)expected) { \ + PRINT_ERR("Failed test. Value returned(%p) doesn't match expected(%p)! fnc:\"%s\",line:%d", (void*)returned, (void*)expected, __FUNCTION__, __LINE__); \ + rc = 1; \ + goto test_cleanup; \ +} passed_subtest_count++; + +#define ALL_SUBTESTS_PASS() printf("└─╼ ✅" KGRN " Test \"%s\" DONE. All of %zu subtests PASS. (line:%d)\n" KNRM, __FUNCTION__, passed_subtest_count, __LINE__); + +#define TEST_START() size_t passed_subtest_count = 0; int rc = 0; printf("╒═ Starting test \"%s\"\n", __FUNCTION__); + +int test_str_uint8() { + c_rhash hash = c_rhash_new(100); + uint8_t val; + + TEST_START(); + // function should fail on empty hash + ASSERT_RETVAL(c_rhash_get_uint8_by_str, !=, 0, hash, KEY_1, &val); + + ASSERT_RETVAL(c_rhash_insert_str_uint8, ==, 0, hash, KEY_1, 5); + ASSERT_RETVAL(c_rhash_get_uint8_by_str, ==, 0, hash, KEY_1, &val); + ASSERT_VAL_UINT8(5, val); + + ASSERT_RETVAL(c_rhash_insert_str_uint8, ==, 0, hash, KEY_2, 8); + ASSERT_RETVAL(c_rhash_get_uint8_by_str, ==, 0, hash, KEY_1, &val); + ASSERT_VAL_UINT8(5, val); + ASSERT_RETVAL(c_rhash_get_uint8_by_str, ==, 0, hash, KEY_2, &val); + ASSERT_VAL_UINT8(8, val); + ASSERT_RETVAL(c_rhash_get_uint8_by_str, !=, 0, hash, "sndnskjdf", &val); + + // test update of key + ASSERT_RETVAL(c_rhash_insert_str_uint8, ==, 0, hash, KEY_1, 100); + ASSERT_RETVAL(c_rhash_get_uint8_by_str, ==, 0, hash, KEY_1, &val); + ASSERT_VAL_UINT8(100, val); + + ALL_SUBTESTS_PASS(); +test_cleanup: + c_rhash_destroy(hash); + return rc; +} + +int test_uint64_ptr() { + c_rhash hash = c_rhash_new(100); + void *val; + + TEST_START(); + + // function should fail on empty hash + ASSERT_RETVAL(c_rhash_get_ptr_by_uint64, !=, 0, hash, 0, &val); + + ASSERT_RETVAL(c_rhash_insert_uint64_ptr, ==, 0, hash, 0, &hash); + ASSERT_RETVAL(c_rhash_get_ptr_by_uint64, ==, 0, hash, 0, &val); + ASSERT_VAL_PTR(&hash, val); + + ASSERT_RETVAL(c_rhash_insert_uint64_ptr, ==, 0, hash, 1, &val); + ASSERT_RETVAL(c_rhash_get_ptr_by_uint64, ==, 0, hash, 0, &val); + ASSERT_VAL_PTR(&hash, val); + ASSERT_RETVAL(c_rhash_get_ptr_by_uint64, ==, 0, hash, 1, &val); + ASSERT_VAL_PTR(&val, val); + ASSERT_RETVAL(c_rhash_get_ptr_by_uint64, !=, 0, hash, 2, &val); + + ALL_SUBTESTS_PASS(); +test_cleanup: + c_rhash_destroy(hash); + return rc; +} + +#define UINT64_PTR_INC_ITERATION_COUNT 5000 +int test_uint64_ptr_incremental() { + c_rhash hash = c_rhash_new(100); + void *val; + + TEST_START(); + + char a = 0x20; + char *ptr = &a; + while(ptr < &a + UINT64_PTR_INC_ITERATION_COUNT) { + ASSERT_RETVAL(c_rhash_insert_uint64_ptr, ==, 0, hash, (ptr-&a), ptr); + ptr++; + } + + ptr = &a; + char *retptr; + for(int i = 0; i < UINT64_PTR_INC_ITERATION_COUNT; i++) { + ASSERT_RETVAL(c_rhash_get_ptr_by_uint64, ==, 0, hash, i, (void**)&retptr); + ASSERT_VAL_PTR(retptr, (&a+i)); + } + + ALL_SUBTESTS_PASS(); +test_cleanup: + c_rhash_destroy(hash); + return rc; +} + +struct test_string { + const char *str; + int counter; +}; + +struct test_string test_strings[] = { + { .str = "Cillum reprehenderit eiusmod elit nisi aliquip esse exercitation commodo Lorem voluptate esse.", .counter = 0 }, + { .str = "Ullamco eiusmod tempor occaecat ad.", .counter = 0 }, + { .str = "Esse aliquip tempor sint tempor ullamco duis aute incididunt ad.", .counter = 0 }, + { .str = "Cillum Lorem labore cupidatat commodo proident adipisicing.", .counter = 0 }, + { .str = "Quis ad cillum officia exercitation.", .counter = 0 }, + { .str = "Ipsum enim dolor ullamco amet sint nisi ut occaecat sint non.", .counter = 0 }, + { .str = "Id duis officia ipsum cupidatat velit fugiat.", .counter = 0 }, + { .str = "Aliqua non occaecat voluptate reprehenderit reprehenderit veniam minim exercitation ea aliquip enim aliqua deserunt qui.", .counter = 0 }, + { .str = "Ullamco elit tempor laboris reprehenderit quis deserunt duis quis tempor reprehenderit magna dolore reprehenderit exercitation.", .counter = 0 }, + { .str = "Culpa do dolor quis incididunt et labore in ex.", .counter = 0 }, + { .str = "Aliquip velit cupidatat qui incididunt ipsum nostrud eiusmod ut proident nisi magna fugiat excepteur.", .counter = 0 }, + { .str = "Aliqua qui dolore tempor id proident ullamco sunt magna.", .counter = 0 }, + { .str = "Labore eiusmod ut fugiat dolore reprehenderit mollit magna.", .counter = 0 }, + { .str = "Veniam aliquip dolor excepteur minim nulla esse cupidatat esse.", .counter = 0 }, + { .str = "Do quis dolor irure nostrud occaecat aute proident anim.", .counter = 0 }, + { .str = "Enim veniam non nulla ad quis sit amet.", .counter = 0 }, + { .str = "Cillum reprehenderit do enim esse do ullamco consectetur ea.", .counter = 0 }, + { .str = "Sit et duis sint anim qui ad anim labore exercitation sunt cupidatat.", .counter = 0 }, + { .str = "Dolor officia adipisicing sint pariatur in dolor occaecat officia reprehenderit magna.", .counter = 0 }, + { .str = "Aliquip dolore qui occaecat eiusmod sunt incididunt reprehenderit minim et.", .counter = 0 }, + { .str = "Aute fugiat laboris cillum tempor consequat tempor do non laboris culpa officia nisi.", .counter = 0 }, + { .str = "Et excepteur do aliquip fugiat nisi velit tempor officia enim quis elit incididunt.", .counter = 0 }, + { .str = "Eu officia adipisicing incididunt occaecat officia cupidatat enim sit sit officia.", .counter = 0 }, + { .str = "Do amet cillum duis pariatur commodo nulla cillum magna nulla Lorem veniam cupidatat.", .counter = 0 }, + { .str = "Dolor adipisicing voluptate laboris occaecat culpa aliquip ipsum ut consequat aliqua aliquip commodo sunt velit.", .counter = 0 }, + { .str = "Nulla proident ipsum quis nulla.", .counter = 0 }, + { .str = "Laborum adipisicing nulla do aute aliqua est quis sint culpa pariatur laborum voluptate qui.", .counter = 0 }, + { .str = "Proident eiusmod sunt et nulla elit pariatur dolore irure ex voluptate excepteur adipisicing consectetur.", .counter = 0 }, + { .str = "Consequat ex voluptate officia excepteur aute deserunt proident commodo et.", .counter = 0 }, + { .str = "Velit sit cupidatat dolor dolore.", .counter = 0 }, + { .str = "Sunt enim do non anim nostrud exercitation ullamco ex proident commodo.", .counter = 0 }, + { .str = "Id ex officia cillum ad.", .counter = 0 }, + { .str = "Laboris in sunt eiusmod veniam laboris nostrud.", .counter = 0 }, + { .str = "Ex magna occaecat ea ea incididunt aliquip.", .counter = 0 }, + { .str = "Sunt eiusmod ex nostrud eu pariatur sit cupidatat ea adipisicing cillum culpa esse consequat aliquip.", .counter = 0 }, + { .str = "Excepteur commodo qui incididunt enim culpa sunt non excepteur Lorem adipisicing.", .counter = 0 }, + { .str = "Quis officia est ullamco reprehenderit incididunt occaecat pariatur ex reprehenderit nisi.", .counter = 0 }, + { .str = "Culpa irure proident proident et eiusmod irure aliqua ipsum cupidatat minim sit.", .counter = 0 }, + { .str = "Qui cupidatat aliquip est velit magna veniam.", .counter = 0 }, + { .str = "Pariatur ad ad mollit nostrud non irure minim veniam anim aliquip quis eu.", .counter = 0 }, + { .str = "Nisi ex minim eu adipisicing tempor Lorem nisi do ad exercitation est non eu.", .counter = 0 }, + { .str = "Cupidatat do mollit ad commodo cupidatat ut.", .counter = 0 }, + { .str = "Est non excepteur eiusmod nostrud et eu.", .counter = 0 }, + { .str = "Cupidatat mollit nisi magna officia ut elit eiusmod.", .counter = 0 }, + { .str = "Est aliqua consectetur laboris ex consequat est ut dolor.", .counter = 0 }, + { .str = "Duis eu laboris laborum ut id Lorem nostrud qui ad velit proident fugiat minim ullamco.", .counter = 0 }, + { .str = "Pariatur esse excepteur anim amet excepteur irure sint quis esse ex cupidatat ut.", .counter = 0 }, + { .str = "Esse reprehenderit amet qui excepteur aliquip amet.", .counter = 0 }, + { .str = "Ullamco laboris elit labore adipisicing aute nulla qui laborum tempor officia ut dolor aute.", .counter = 0 }, + { .str = "Commodo sunt cillum velit minim laborum Lorem aliqua tempor ad id eu.", .counter = 0 }, + { .str = NULL, .counter = 0 } +}; + +uint32_t test_strings_contain_element(const char *str) { + struct test_string *str_desc = test_strings; + while(str_desc->str) { + if (!strcmp(str, str_desc->str)) + return str_desc - test_strings; + str_desc++; + } + return -1; +} + +#define TEST_INCREMENT_STR_KEYS_HASH_SIZE 20 +int test_increment_str_keys() { + c_rhash hash; + const char *key; + + TEST_START(); + + hash = c_rhash_new(TEST_INCREMENT_STR_KEYS_HASH_SIZE); // less than element count of test_strings + + c_rhash_iter_t iter = C_RHASH_ITER_T_INITIALIZER; + + // check iter on empty hash + ASSERT_RETVAL(c_rhash_iter_str_keys, !=, 0, hash, &iter, &key); + + int32_t element_count = 0; + while (test_strings[element_count].str) { + ASSERT_RETVAL(c_rhash_insert_str_ptr, ==, 0, hash, test_strings[element_count].str, NULL); + test_strings[element_count].counter++; // we want to test we got each key exactly once + element_count++; + } + + if (element_count <= TEST_INCREMENT_STR_KEYS_HASH_SIZE * 2) { + // verify we are actually test also iteration trough single bin (when 2 keys have same hash pointing them to same bin) + PRINT_ERR("For this test to properly test all the hash size needs to be much smaller than all test key count."); + rc = 1; + goto test_cleanup; + } + + // we insert another type of key as iterator should skip it + // in case is another type + ASSERT_RETVAL(c_rhash_insert_uint64_ptr, ==, 0, hash, 5, NULL); + + c_rhash_iter_t_initialize(&iter); + while(!c_rhash_iter_str_keys(hash, &iter, &key)) { + element_count--; + int i; + if ( (i = test_strings_contain_element(key)) < 0) { + PRINT_ERR("Key \"%s\" is not present in test_strings array! (Fnc: %s, Line: %d)", key, __FUNCTION__, __LINE__); + rc = 1; + goto test_cleanup; + } + passed_subtest_count++; + + test_strings[i].counter--; + } + ASSERT_VAL_UINT8(element_count, 0); // we added also same non string keys + + // check each key was present exactly once + struct test_string *str_desc = test_strings; + while (str_desc->str) { + ASSERT_VAL_UINT8(str_desc->counter, 0); + str_desc++; + } + + ALL_SUBTESTS_PASS(); +test_cleanup: + c_rhash_destroy(hash); + return rc; +} + +#define RUN_TEST(fnc) \ +if(fnc()) \ + return 1; + +int main(int argc, char *argv[]) { + RUN_TEST(test_str_uint8); + RUN_TEST(test_uint64_ptr); + RUN_TEST(test_uint64_ptr_incremental); + RUN_TEST(test_increment_str_keys); + // TODO hash with mixed key tests + // TODO iterator test + return 0; +} diff --git a/src/libnetdata/circular_buffer/README.md b/src/libnetdata/circular_buffer/README.md index b2d580cb9..db42e5b3b 100644 --- a/src/libnetdata/circular_buffer/README.md +++ b/src/libnetdata/circular_buffer/README.md @@ -1,12 +1,3 @@ -<!-- -title: "Circular Buffer" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/circular_buffer/README.md -sidebar_label: "Circular Buffer" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Circular Buffer `struct circular_buffer` is an adaptive circular buffer. It will start at an initial size diff --git a/src/libnetdata/clocks/clocks.c b/src/libnetdata/clocks/clocks.c index 5da450a2d..c65886198 100644 --- a/src/libnetdata/clocks/clocks.c +++ b/src/libnetdata/clocks/clocks.c @@ -78,7 +78,9 @@ static usec_t get_clock_resolution(clockid_t clock) { // perform any initializations required for clocks -void clocks_init(void) { +static __attribute__((constructor)) void clocks_init(void) { + os_get_system_HZ(); + // monotonic raw has to be tested before boottime test_clock_monotonic_raw(); @@ -87,6 +89,18 @@ void clocks_init(void) { clock_monotonic_resolution = get_clock_resolution(clock_monotonic_to_use); clock_realtime_resolution = get_clock_resolution(CLOCK_REALTIME); + +#if defined(OS_WINDOWS) + timeBeginPeriod(1); + clock_monotonic_resolution = 1 * USEC_PER_MS; + clock_realtime_resolution = 1 * USEC_PER_MS; +#endif +} + +static __attribute__((destructor)) void clocks_fin(void) { +#if defined(OS_WINDOWS) + timeEndPeriod(1); +#endif } inline time_t now_sec(clockid_t clk_id) { @@ -246,13 +260,18 @@ void sleep_to_absolute_time(usec_t usec) { } #endif -#define HEARTBEAT_ALIGNMENT_STATISTICS_SIZE 10 -netdata_mutex_t heartbeat_alignment_mutex = NETDATA_MUTEX_INITIALIZER; +#define HEARTBEAT_MIN_OFFSET_UT (150 * USEC_PER_MS) +#define HEARTBEAT_RANDOM_OFFSET_UT (350 * USEC_PER_MS) + +#define HEARTBEAT_ALIGNMENT_STATISTICS_SIZE 20 +static SPINLOCK heartbeat_alignment_spinlock = NETDATA_SPINLOCK_INITIALIZER; static size_t heartbeat_alignment_id = 0; struct heartbeat_thread_statistics { + pid_t tid; size_t sequence; usec_t dt; + usec_t randomness; }; static struct heartbeat_thread_statistics heartbeat_alignment_values[HEARTBEAT_ALIGNMENT_STATISTICS_SIZE] = { 0 }; @@ -290,19 +309,58 @@ void heartbeat_statistics(usec_t *min_ptr, usec_t *max_ptr, usec_t *average_ptr, memcpy(old, current, sizeof(struct heartbeat_thread_statistics) * HEARTBEAT_ALIGNMENT_STATISTICS_SIZE); } -inline void heartbeat_init(heartbeat_t *hb) { - hb->realtime = 0ULL; - hb->randomness = (usec_t)250 * USEC_PER_MS + ((usec_t)(now_realtime_usec() * clock_realtime_resolution) % (250 * USEC_PER_MS)); - hb->randomness -= (hb->randomness % clock_realtime_resolution); +static XXH64_hash_t heartbeat_hash(usec_t step, size_t statistics_id) { + struct { + usec_t step; + pid_t pid; + pid_t tid; + usec_t now_ut; + size_t statistics_id; + char tag[ND_THREAD_TAG_MAX + 1]; + } key = { + .step = step, + .pid = getpid(), + .tid = os_gettid(), + .now_ut = now_realtime_usec(), + .statistics_id = statistics_id, + }; + strncpyz(key.tag, nd_thread_tag(), sizeof(key.tag) - 1); + return XXH3_64bits(&key, sizeof(key)); +} + +static usec_t heartbeat_randomness(XXH64_hash_t hash) { + usec_t offset_ut = HEARTBEAT_MIN_OFFSET_UT + (hash % HEARTBEAT_RANDOM_OFFSET_UT); + + // Calculate the scheduler tick interval in microseconds + usec_t scheduler_step_ut = USEC_PER_SEC / (usec_t)system_hz; + if(scheduler_step_ut > 10 * USEC_PER_MS) + scheduler_step_ut = 10 * USEC_PER_MS; + + // if the offset is close to the scheduler tick, move it away from it + if(offset_ut % scheduler_step_ut < scheduler_step_ut / 4) + offset_ut += scheduler_step_ut / 4; + + return offset_ut; +} - netdata_mutex_lock(&heartbeat_alignment_mutex); +inline void heartbeat_init(heartbeat_t *hb, usec_t step) { + if(!step) step = USEC_PER_SEC; + + spinlock_lock(&heartbeat_alignment_spinlock); hb->statistics_id = heartbeat_alignment_id; heartbeat_alignment_id++; - netdata_mutex_unlock(&heartbeat_alignment_mutex); + spinlock_unlock(&heartbeat_alignment_spinlock); + + hb->step = step; + hb->realtime = 0ULL; + hb->hash = heartbeat_hash(hb->step, hb->statistics_id); + hb->randomness = heartbeat_randomness(hb->hash); if(hb->statistics_id < HEARTBEAT_ALIGNMENT_STATISTICS_SIZE) { heartbeat_alignment_values[hb->statistics_id].dt = 0; heartbeat_alignment_values[hb->statistics_id].sequence = 0; + heartbeat_alignment_values[hb->statistics_id].randomness = hb->randomness; + heartbeat_alignment_values[hb->statistics_id].tid = os_gettid(); } } @@ -310,17 +368,8 @@ inline void heartbeat_init(heartbeat_t *hb) { // it waits using the monotonic clock // it returns the dt using the realtime clock -usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) { - if(unlikely(hb->randomness > tick / 2)) { - // TODO: The heartbeat tick should be specified at the heartbeat_init() function - usec_t tmp = (now_realtime_usec() * clock_realtime_resolution) % (tick / 2); - - nd_log_limit_static_global_var(erl, 10, 0); - nd_log_limit(&erl, NDLS_DAEMON, NDLP_NOTICE, - "heartbeat randomness of %"PRIu64" is too big for a tick of %"PRIu64" - setting it to %"PRIu64"", - hb->randomness, tick, tmp); - hb->randomness = tmp; - } +usec_t heartbeat_next(heartbeat_t *hb) { + usec_t tick = hb->step; usec_t dt; usec_t now = now_realtime_usec(); @@ -331,10 +380,13 @@ usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) { next = next - (next % clock_realtime_resolution) + clock_realtime_resolution; // sleep_usec() has a loop to guarantee we will sleep for at least the requested time. - // According the specs, when we sleep for a relative time, clock adjustments should not affect the duration - // we sleep. + // According to the specs, when we sleep for a relative time, clock adjustments should + // not affect the duration we sleep. sleep_usec_with_now(next - now, now); + spinlock_lock(&heartbeat_alignment_spinlock); now = now_realtime_usec(); + spinlock_unlock(&heartbeat_alignment_spinlock); + dt = now - hb->realtime; if(hb->statistics_id < HEARTBEAT_ALIGNMENT_STATISTICS_SIZE) { @@ -368,22 +420,15 @@ usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) { return dt; } -#ifdef OS_WINDOWS - -#include "windows.h" - -void sleep_usec_with_now(usec_t usec, usec_t started_ut) -{ +#if defined(OS_WINDOWS) +void sleep_usec_with_now(usec_t usec, usec_t started_ut) { if (!started_ut) started_ut = now_realtime_usec(); usec_t end_ut = started_ut + usec; usec_t remaining_ut = usec; - timeBeginPeriod(1); - - while (remaining_ut >= 1000) - { + while (remaining_ut >= clock_realtime_resolution) { DWORD sleep_ms = (DWORD) (remaining_ut / USEC_PER_MS); Sleep(sleep_ms); @@ -393,8 +438,6 @@ void sleep_usec_with_now(usec_t usec, usec_t started_ut) remaining_ut = end_ut - now_ut; } - - timeEndPeriod(1); } #else void sleep_usec_with_now(usec_t usec, usec_t started_ut) { @@ -406,7 +449,7 @@ void sleep_usec_with_now(usec_t usec, usec_t started_ut) { }; // make sure errno is not EINTR - errno = 0; + errno_clear(); if(!started_ut) started_ut = now_realtime_usec(); @@ -419,7 +462,7 @@ void sleep_usec_with_now(usec_t usec, usec_t started_ut) { rem = (struct timespec){ 0, 0 }; // break an infinite loop - errno = 0; + errno_clear(); usec_t now_ut = now_realtime_usec(); if(now_ut >= end_ut) @@ -429,8 +472,8 @@ void sleep_usec_with_now(usec_t usec, usec_t started_ut) { usec_t check_ut = now_ut - started_ut; if(remaining_ut > check_ut) { req = (struct timespec){ - .tv_sec = (time_t) ( check_ut / USEC_PER_SEC), - .tv_nsec = (suseconds_t) ((check_ut % USEC_PER_SEC) * NSEC_PER_USEC) + .tv_sec = (time_t) ( check_ut / USEC_PER_SEC), + .tv_nsec = (suseconds_t) ((check_ut % USEC_PER_SEC) * NSEC_PER_USEC) }; } } @@ -452,7 +495,7 @@ static inline collected_number uptime_from_boottime(void) { } static procfile *read_proc_uptime_ff = NULL; -static inline collected_number read_proc_uptime(char *filename) { +static inline collected_number read_proc_uptime(const char *filename) { if(unlikely(!read_proc_uptime_ff)) { read_proc_uptime_ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT); if(unlikely(!read_proc_uptime_ff)) return 0; @@ -473,7 +516,7 @@ static inline collected_number read_proc_uptime(char *filename) { return (collected_number)(strtondd(procfile_lineword(read_proc_uptime_ff, 0, 0), NULL) * 1000.0); } -inline collected_number uptime_msec(char *filename){ +inline collected_number uptime_msec(const char *filename){ static int use_boottime = -1; if(unlikely(use_boottime == -1)) { diff --git a/src/libnetdata/clocks/clocks.h b/src/libnetdata/clocks/clocks.h index f989fd6b8..03860d66d 100644 --- a/src/libnetdata/clocks/clocks.h +++ b/src/libnetdata/clocks/clocks.h @@ -4,6 +4,7 @@ #define NETDATA_CLOCKS_H 1 #include "../libnetdata.h" +#include "libnetdata/os/random.h" #ifndef HAVE_CLOCK_GETTIME struct timespec { @@ -18,12 +19,19 @@ struct timespec { typedef uint64_t nsec_t; typedef uint64_t msec_t; typedef uint64_t usec_t; + +typedef int64_t snsec_t; typedef int64_t susec_t; +typedef int64_t smsec_t; + +typedef int64_t stime_t; typedef struct heartbeat { + usec_t step; usec_t realtime; usec_t randomness; size_t statistics_id; + XXH64_hash_t hash; } heartbeat_t; /* Linux value is as good as any other */ @@ -72,6 +80,8 @@ typedef struct heartbeat { #define MSEC_PER_SEC 1000ULL #endif +#define NS100_PER_MS 10000ULL + #define USEC_PER_MS 1000ULL #ifndef HAVE_CLOCK_GETTIME @@ -132,26 +142,24 @@ msec_t timeval_msec(struct timeval *tv); usec_t dt_usec(struct timeval *now, struct timeval *old); susec_t dt_usec_signed(struct timeval *now, struct timeval *old); -void heartbeat_init(heartbeat_t *hb); +void heartbeat_init(heartbeat_t *hb, usec_t step); /* Sleeps until next multiple of tick using monotonic clock. * Returns elapsed time in microseconds since previous heartbeat */ -usec_t heartbeat_next(heartbeat_t *hb, usec_t tick); +usec_t heartbeat_next(heartbeat_t *hb); void heartbeat_statistics(usec_t *min_ptr, usec_t *max_ptr, usec_t *average_ptr, size_t *count_ptr); void sleep_usec_with_now(usec_t usec, usec_t started_ut); #define sleep_usec(usec) sleep_usec_with_now(usec, 0) -void clocks_init(void); - // lower level functions - avoid using directly time_t now_sec(clockid_t clk_id); usec_t now_usec(clockid_t clk_id); int now_timeval(clockid_t clk_id, struct timeval *tv); -collected_number uptime_msec(char *filename); +collected_number uptime_msec(const char *filename); extern usec_t clock_monotonic_resolution; extern usec_t clock_realtime_resolution; diff --git a/src/libnetdata/common.h b/src/libnetdata/common.h new file mode 100644 index 000000000..aafe51176 --- /dev/null +++ b/src/libnetdata/common.h @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +/* + * This file should include everything from the operating system needed to compile Netdata, + * without any Netdata specific includes. + * + * It should be the baseline of includes (operating system and common libraries related). + */ + +#ifndef LIBNETDATA_COMMON_H +#define LIBNETDATA_COMMON_H + +# ifdef __cplusplus +extern "C" { +# endif + +#include "config.h" + +#if defined(NETDATA_DEV_MODE) && !defined(NETDATA_INTERNAL_CHECKS) +#define NETDATA_INTERNAL_CHECKS 1 +#endif + +#ifndef SIZEOF_VOID_P +#error SIZEOF_VOID_P is not defined +#endif + +#if SIZEOF_VOID_P == 4 +#define ENV32BIT 1 +#else +#define ENV64BIT 1 +#endif + +#ifdef HAVE_LIBDATACHANNEL +#define ENABLE_WEBRTC 1 +#endif + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) + +// -------------------------------------------------------------------------------------------------------------------- +// NETDATA_OS_TYPE + +#if defined(__FreeBSD__) +#include <pthread_np.h> +#define NETDATA_OS_TYPE "freebsd" +#elif defined(__APPLE__) +#define NETDATA_OS_TYPE "macos" +#elif defined(OS_WINDOWS) +#define NETDATA_OS_TYPE "windows" +#else +#define NETDATA_OS_TYPE "linux" +#endif /* __FreeBSD__, __APPLE__*/ + +// -------------------------------------------------------------------------------------------------------------------- +// memory allocators + +/* select the memory allocator, based on autoconf findings */ +#if defined(ENABLE_JEMALLOC) + +#if defined(HAVE_JEMALLOC_JEMALLOC_H) +#include <jemalloc/jemalloc.h> +#else // !defined(HAVE_JEMALLOC_JEMALLOC_H) +#include <malloc.h> +#endif // !defined(HAVE_JEMALLOC_JEMALLOC_H) + +#elif defined(ENABLE_TCMALLOC) + +#include <google/tcmalloc.h> + +#else /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */ + +#if !(defined(__FreeBSD__) || defined(__APPLE__)) +#include <malloc.h> +#endif /* __FreeBSD__ || __APPLE__ */ + +#endif /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */ + +// -------------------------------------------------------------------------------------------------------------------- + +#include <pthread.h> +#include <errno.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdarg.h> +#include <stddef.h> +#include <ctype.h> +#include <string.h> +#include <strings.h> +#include <libgen.h> +#include <dirent.h> +#include <fcntl.h> +#include <getopt.h> +#include <limits.h> +#include <locale.h> +#include <signal.h> +#include <sys/time.h> +#include <sys/types.h> +#include <time.h> +#include <unistd.h> +#include <uv.h> +#include <assert.h> + +#ifdef HAVE_ARPA_INET_H +#include <arpa/inet.h> +#endif + +#ifdef HAVE_NETINET_TCP_H +#include <netinet/tcp.h> +#endif + +#ifdef HAVE_SYS_IOCTL_H +#include <sys/ioctl.h> +#endif + +#ifdef HAVE_GRP_H +#include <grp.h> +#else +typedef uint32_t gid_t; +#endif + +#ifdef HAVE_PWD_H +#include <pwd.h> +#else +typedef uint32_t uid_t; +#endif + +#ifdef HAVE_NET_IF_H +#include <net/if.h> +#endif + +#ifdef HAVE_POLL_H +#include <poll.h> +#endif + +#ifdef HAVE_SYSLOG_H +#include <syslog.h> +#else +/* priorities */ +#define LOG_EMERG 0 /* system is unusable */ +#define LOG_ALERT 1 /* action must be taken immediately */ +#define LOG_CRIT 2 /* critical conditions */ +#define LOG_ERR 3 /* error conditions */ +#define LOG_WARNING 4 /* warning conditions */ +#define LOG_NOTICE 5 /* normal but significant condition */ +#define LOG_INFO 6 /* informational */ +#define LOG_DEBUG 7 /* debug-level messages */ + +/* facility codes */ +#define LOG_KERN (0<<3) /* kernel messages */ +#define LOG_USER (1<<3) /* random user-level messages */ +#define LOG_MAIL (2<<3) /* mail system */ +#define LOG_DAEMON (3<<3) /* system daemons */ +#define LOG_AUTH (4<<3) /* security/authorization messages */ +#define LOG_SYSLOG (5<<3) /* messages generated internally by syslogd */ +#define LOG_LPR (6<<3) /* line printer subsystem */ +#define LOG_NEWS (7<<3) /* network news subsystem */ +#define LOG_UUCP (8<<3) /* UUCP subsystem */ +#define LOG_CRON (9<<3) /* clock daemon */ +#define LOG_AUTHPRIV (10<<3) /* security/authorization messages (private) */ +#define LOG_FTP (11<<3) /* ftp daemon */ + +/* other codes through 15 reserved for system use */ +#define LOG_LOCAL0 (16<<3) /* reserved for local use */ +#define LOG_LOCAL1 (17<<3) /* reserved for local use */ +#define LOG_LOCAL2 (18<<3) /* reserved for local use */ +#define LOG_LOCAL3 (19<<3) /* reserved for local use */ +#define LOG_LOCAL4 (20<<3) /* reserved for local use */ +#define LOG_LOCAL5 (21<<3) /* reserved for local use */ +#define LOG_LOCAL6 (22<<3) /* reserved for local use */ +#define LOG_LOCAL7 (23<<3) /* reserved for local use */ +#endif + +#ifdef HAVE_SYS_MMAN_H +#include <sys/mman.h> +#endif + +#ifdef HAVE_SYS_RESOURCE_H +#include <sys/resource.h> +#endif + +#ifdef HAVE_SYS_SOCKET_H +#include <sys/socket.h> +#endif + +#ifdef HAVE_SYS_WAIT_H +#include <sys/wait.h> +#endif + +#ifdef HAVE_SYS_UN_H +#include <sys/un.h> +#endif + +#ifdef HAVE_SPAWN_H +#include <spawn.h> +#endif + +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif + +#ifdef HAVE_RESOLV_H +#include <resolv.h> +#endif + +#ifdef HAVE_NETDB_H +#include <netdb.h> +#endif + +#ifdef HAVE_SYS_PRCTL_H +#include <sys/prctl.h> +#endif + +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif + +#ifdef HAVE_SYS_VFS_H +#include <sys/vfs.h> +#endif + +#ifdef HAVE_SYS_STATFS_H +#include <sys/statfs.h> +#endif + +#ifdef HAVE_LINUX_MAGIC_H +#include <linux/magic.h> +#endif + +#ifdef HAVE_SYS_MOUNT_H +#include <sys/mount.h> +#endif + +#ifdef HAVE_SYS_STATVFS_H +#include <sys/statvfs.h> +#endif + +// #1408 +#ifdef MAJOR_IN_MKDEV +#include <sys/mkdev.h> +#endif +#ifdef MAJOR_IN_SYSMACROS +#include <sys/sysmacros.h> +#endif + +#include <math.h> +#include <float.h> + +#if defined(HAVE_INTTYPES_H) +#include <inttypes.h> +#elif defined(HAVE_STDINT_H) +#include <stdint.h> +#endif + +#include <zlib.h> + +#ifdef HAVE_SYS_CAPABILITY_H +#include <sys/capability.h> +#endif + +#define XXH_INLINE_ALL +#include "xxHash/xxhash.h" + +// -------------------------------------------------------------------------------------------------------------------- +// OpenSSL + +#define OPENSSL_VERSION_095 0x00905100L +#define OPENSSL_VERSION_097 0x0907000L +#define OPENSSL_VERSION_110 0x10100000L +#define OPENSSL_VERSION_111 0x10101000L +#define OPENSSL_VERSION_300 0x30000000L + +#include <openssl/ssl.h> +#include <openssl/rand.h> +#include <openssl/err.h> +#include <openssl/evp.h> +#include <openssl/pem.h> +#if (SSLEAY_VERSION_NUMBER >= OPENSSL_VERSION_097) && (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110) +#include <openssl/conf.h> +#endif + +#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_300 +#include <openssl/core_names.h> +#include <openssl/decoder.h> +#endif + +// -------------------------------------------------------------------------------------------------------------------- + +#ifndef O_CLOEXEC +#define O_CLOEXEC (0) +#endif + +// -------------------------------------------------------------------------------------------------------------------- +// FUNCTION ATTRIBUTES + +#define _cleanup_(x) __attribute__((__cleanup__(x))) + +#ifdef HAVE_FUNC_ATTRIBUTE_RETURNS_NONNULL +#define NEVERNULL __attribute__((returns_nonnull)) +#else +#define NEVERNULL +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_NOINLINE +#define NOINLINE __attribute__((noinline)) +#else +#define NOINLINE +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_MALLOC +#define MALLOCLIKE __attribute__((malloc)) +#else +#define MALLOCLIKE +#endif + +#if defined(HAVE_FUNC_ATTRIBUTE_FORMAT_GNU_PRINTF) +#define PRINTFLIKE(f, a) __attribute__ ((format(gnu_printf, f, a))) +#elif defined(HAVE_FUNC_ATTRIBUTE_FORMAT_PRINTF) +#define PRINTFLIKE(f, a) __attribute__ ((format(printf, f, a))) +#else +#define PRINTFLIKE(f, a) +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_NORETURN +#define NORETURN __attribute__ ((noreturn)) +#else +#define NORETURN +#endif + +#ifdef HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT +#define WARNUNUSED __attribute__ ((warn_unused_result)) +#else +#define WARNUNUSED +#endif + +#define UNUSED(x) (void)(x) + +#ifdef __GNUC__ +#define UNUSED_FUNCTION(x) __attribute__((unused)) UNUSED_##x +#else +#define UNUSED_FUNCTION(x) UNUSED_##x +#endif + +// -------------------------------------------------------------------------------------------------------------------- +// fix for alpine linux + +#if !defined(RUSAGE_THREAD) && defined(RUSAGE_CHILDREN) +#define RUSAGE_THREAD RUSAGE_CHILDREN +#endif + +// -------------------------------------------------------------------------------------------------------------------- +// HELPFUL MACROS + +#define ABS(x) (((x) < 0)? (-(x)) : (x)) +#define MIN(a,b) (((a)<(b))?(a):(b)) +#define MAX(a,b) (((a)>(b))?(a):(b)) +#define SWAP(a, b) do { \ + typeof(a) _tmp = b; \ + b = a; \ + a = _tmp; \ +} while(0) + +// -------------------------------------------------------------------------------------------------------------------- +// NETDATA CLOUD + +// BEWARE: this exists in alarm-notify.sh +#define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud" + +// -------------------------------------------------------------------------------------------------------------------- +// DBENGINE + +#define RRD_STORAGE_TIERS 5 + +// -------------------------------------------------------------------------------------------------------------------- +// PIPES + +#define PIPE_READ 0 +#define PIPE_WRITE 1 + +// -------------------------------------------------------------------------------------------------------------------- +// UUIDs + +#define GUID_LEN 36 + +// -------------------------------------------------------------------------------------------------------------------- +// Macro-only includes + +#include "linked_lists/linked_lists.h" + +// -------------------------------------------------------------------------------------------------------------------- + +// Taken from linux kernel +#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)])) + +// -------------------------------------------------------------------------------------------------------------------- + +#if defined(OS_WINDOWS) +#include <windows.h> +#include <wctype.h> +#include <wchar.h> +#include <tchar.h> +#include <guiddef.h> +#include <io.h> +#include <fcntl.h> +#include <process.h> +#include <tlhelp32.h> +#include <sys/cygwin.h> +#include <winevt.h> +#include <evntprov.h> +#include <wbemidl.h> +#include <sddl.h> +// #include <winternl.h> // conflicts on STRING, +#endif + +# ifdef __cplusplus +} +# endif + +#endif //LIBNETDATA_COMMON_H diff --git a/src/libnetdata/config/README.md b/src/libnetdata/config/README.md index 665a7196c..fb1473c8c 100644 --- a/src/libnetdata/config/README.md +++ b/src/libnetdata/config/README.md @@ -1,12 +1,3 @@ -<!-- -title: "Netdata ini config files" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/config/README.md -sidebar_label: "Netdata ini config files" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Netdata ini config files Configuration files `netdata.conf` and `stream.conf` are Netdata ini files. diff --git a/src/libnetdata/config/appconfig.c b/src/libnetdata/config/appconfig.c index 81946b594..f26417ac3 100644 --- a/src/libnetdata/config/appconfig.c +++ b/src/libnetdata/config/appconfig.c @@ -1,961 +1,82 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "../libnetdata.h" +#include "appconfig_internals.h" -/* - * @Input: - * Connector / instance to add to an internal structure - * @Return - * The current head of the linked list of connector_instance - * - */ - -_CONNECTOR_INSTANCE *add_connector_instance(struct section *connector, struct section *instance) -{ - static struct _connector_instance *global_connector_instance = NULL; - struct _connector_instance *local_ci, *local_ci_tmp; - - if (unlikely(!connector)) { - if (unlikely(!instance)) - return global_connector_instance; - - local_ci = global_connector_instance; - while (local_ci) { - local_ci_tmp = local_ci->next; - freez(local_ci); - local_ci = local_ci_tmp; - } - global_connector_instance = NULL; - return NULL; - } - - local_ci = callocz(1, sizeof(struct _connector_instance)); - local_ci->instance = instance; - local_ci->connector = connector; - strncpyz(local_ci->instance_name, instance->name, CONFIG_MAX_NAME); - strncpyz(local_ci->connector_name, connector->name, CONFIG_MAX_NAME); - local_ci->next = global_connector_instance; - global_connector_instance = local_ci; - - return global_connector_instance; -} - -int is_valid_connector(char *type, int check_reserved) -{ - int rc = 1; - - if (unlikely(!type)) - return 0; - - if (!check_reserved) { - if (unlikely(is_valid_connector(type,1))) { - return 0; - } - //if (unlikely(*type == ':') - // return 0; - char *separator = strrchr(type, ':'); - if (likely(separator)) { - *separator = '\0'; - rc = separator - type; - } else - return 0; - } -// else { -// if (unlikely(is_valid_connector(type,1))) { -// netdata_log_error("Section %s invalid -- reserved name", type); -// return 0; -// } -// } - - if (!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) { - return rc; - } else if (!strcmp(type, "graphite:http") || !strcmp(type, "graphite:https")) { - return rc; - } else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) { - return rc; - } else if (!strcmp(type, "json:http") || !strcmp(type, "json:https")) { - return rc; - } else if (!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) { - return rc; - } else if (!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) { - return rc; - } else if (!strcmp(type, "prometheus_remote_write")) { - return rc; - } else if (!strcmp(type, "prometheus_remote_write:http") || !strcmp(type, "prometheus_remote_write:https")) { - return rc; - } else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) { - return rc; - } else if (!strcmp(type, "pubsub") || !strcmp(type, "pubsub:plaintext")) { - return rc; - } else if (!strcmp(type, "mongodb") || !strcmp(type, "mongodb:plaintext")) { - return rc; - } - - return 0; -} - -// ---------------------------------------------------------------------------- -// locking - -inline void appconfig_wrlock(struct config *root) { - netdata_mutex_lock(&root->mutex); -} - -inline void appconfig_unlock(struct config *root) { - netdata_mutex_unlock(&root->mutex); -} - -inline void config_section_wrlock(struct section *co) { - netdata_mutex_lock(&co->mutex); -} - -inline void config_section_unlock(struct section *co) { - netdata_mutex_unlock(&co->mutex); -} - - -// ---------------------------------------------------------------------------- -// config name-value index - -static int appconfig_option_compare(void *a, void *b) { - if(((struct config_option *)a)->hash < ((struct config_option *)b)->hash) return -1; - else if(((struct config_option *)a)->hash > ((struct config_option *)b)->hash) return 1; - else return strcmp(((struct config_option *)a)->name, ((struct config_option *)b)->name); -} - -#define appconfig_option_index_add(co, cv) (struct config_option *)avl_insert_lock(&((co)->values_index), (avl_t *)(cv)) -#define appconfig_option_index_del(co, cv) (struct config_option *)avl_remove_lock(&((co)->values_index), (avl_t *)(cv)) - -static struct config_option *appconfig_option_index_find(struct section *co, const char *name, uint32_t hash) { - struct config_option tmp; - tmp.hash = (hash)?hash:simple_hash(name); - tmp.name = (char *)name; - - return (struct config_option *)avl_search_lock(&(co->values_index), (avl_t *) &tmp); -} - - -// ---------------------------------------------------------------------------- -// config sections index - -int appconfig_section_compare(void *a, void *b) { - if(((struct section *)a)->hash < ((struct section *)b)->hash) return -1; - else if(((struct section *)a)->hash > ((struct section *)b)->hash) return 1; - else return strcmp(((struct section *)a)->name, ((struct section *)b)->name); -} - -#define appconfig_index_add(root, cfg) (struct section *)avl_insert_lock(&(root)->index, (avl_t *)(cfg)) -#define appconfig_index_del(root, cfg) (struct section *)avl_remove_lock(&(root)->index, (avl_t *)(cfg)) - -static struct section *appconfig_index_find(struct config *root, const char *name, uint32_t hash) { - struct section tmp; - tmp.hash = (hash)?hash:simple_hash(name); - tmp.name = (char *)name; - - return (struct section *)avl_search_lock(&root->index, (avl_t *) &tmp); -} - - -// ---------------------------------------------------------------------------- -// config section methods - -static inline struct section *appconfig_section_find(struct config *root, const char *section) { - return appconfig_index_find(root, section, 0); -} - -static inline struct section *appconfig_section_create(struct config *root, const char *section) { - netdata_log_debug(D_CONFIG, "Creating section '%s'.", section); - - struct section *co = callocz(1, sizeof(struct section)); - co->name = strdupz(section); - co->hash = simple_hash(co->name); - netdata_mutex_init(&co->mutex); - - avl_init_lock(&co->values_index, appconfig_option_compare); - - if(unlikely(appconfig_index_add(root, co) != co)) - netdata_log_error("INTERNAL ERROR: indexing of section '%s', already exists.", co->name); +int appconfig_exists(struct config *root, const char *section, const char *name) { + struct config_section *sect = appconfig_section_find(root, section); + if(!sect) return 0; - appconfig_wrlock(root); - struct section *co2 = root->last_section; - if(co2) { - co2->next = co; - } else { - root->first_section = co; - } - root->last_section = co; - appconfig_unlock(root); + struct config_option *opt = appconfig_option_find(sect, name); + if(!opt) return 0; - return co; + return 1; } -void appconfig_section_destroy_non_loaded(struct config *root, const char *section) -{ - struct section *co; - struct config_option *cv, *cv_next; - - netdata_log_debug(D_CONFIG, "Destroying section '%s'.", section); - - co = appconfig_section_find(root, section); - if(!co) { - netdata_log_error("Could not destroy section '%s'. Not found.", section); +void appconfig_set_default_raw_value(struct config *root, const char *section, const char *name, const char *value) { + struct config_section *sect = appconfig_section_find(root, section); + if(!sect) { + appconfig_set_raw_value(root, section, name, value, CONFIG_VALUE_TYPE_UNKNOWN); return; } - config_section_wrlock(co); - for(cv = co->values; cv ; cv = cv->next) { - if (cv->flags & CONFIG_VALUE_LOADED) { - /* Do not destroy values that were loaded from the configuration files. */ - config_section_unlock(co); - return; - } - } - for(cv = co->values ; cv ; cv = cv_next) { - cv_next = cv->next; - if(unlikely(!appconfig_option_index_del(co, cv))) - netdata_log_error("Cannot remove config option '%s' from section '%s'.", cv->name, co->name); - freez(cv->value); - freez(cv->name); - freez(cv); - } - co->values = NULL; - config_section_unlock(co); - - if (unlikely(!appconfig_index_del(root, co))) { - netdata_log_error("Cannot remove section '%s' from config.", section); + struct config_option *opt = appconfig_option_find(sect, name); + if(!opt) { + appconfig_set_raw_value(root, section, name, value, CONFIG_VALUE_TYPE_UNKNOWN); return; } - - appconfig_wrlock(root); - - if (root->first_section == co) { - root->first_section = co->next; - - if (root->last_section == co) - root->last_section = root->first_section; - } else { - struct section *co_cur = root->first_section, *co_prev = NULL; - while(co_cur && co_cur != co) { - co_prev = co_cur; - co_cur = co_cur->next; - } - - if (co_cur) { - co_prev->next = co_cur->next; + opt->flags |= CONFIG_VALUE_USED; - if (root->last_section == co_cur) - root->last_section = co_prev; - } - } - - appconfig_unlock(root); - - avl_destroy_lock(&co->values_index); - freez(co->name); - pthread_mutex_destroy(&co->mutex); - freez(co); -} - -void appconfig_section_option_destroy_non_loaded(struct config *root, const char *section, const char *name) -{ - netdata_log_debug(D_CONFIG, "Destroying section option '%s -> %s'.", section, name); - - struct section *co; - co = appconfig_section_find(root, section); - if (!co) { - netdata_log_error("Could not destroy section option '%s -> %s'. The section not found.", section, name); - return; - } - - config_section_wrlock(co); - - struct config_option *cv; - - cv = appconfig_option_index_find(co, name, simple_hash(name)); - - if (cv && cv->flags & CONFIG_VALUE_LOADED) { - config_section_unlock(co); - return; - } - - if (unlikely(!(cv && appconfig_option_index_del(co, cv)))) { - config_section_unlock(co); - netdata_log_error("Could not destroy section option '%s -> %s'. The option not found.", section, name); + if(opt->flags & CONFIG_VALUE_LOADED) return; - } - - if (co->values == cv) { - co->values = co->values->next; - } else { - struct config_option *cv_cur = co->values, *cv_prev = NULL; - while (cv_cur && cv_cur != cv) { - cv_prev = cv_cur; - cv_cur = cv_cur->next; - } - if (cv_cur) { - cv_prev->next = cv_cur->next; - } - } - - freez(cv->value); - freez(cv->name); - freez(cv); - - config_section_unlock(co); - return; -} - -// ---------------------------------------------------------------------------- -// config name-value methods - -static inline struct config_option *appconfig_value_create(struct section *co, const char *name, const char *value) { - netdata_log_debug(D_CONFIG, "Creating config entry for name '%s', value '%s', in section '%s'.", name, value, co->name); - - struct config_option *cv = callocz(1, sizeof(struct config_option)); - cv->name = strdupz(name); - cv->hash = simple_hash(cv->name); - cv->value = strdupz(value); - - struct config_option *found = appconfig_option_index_add(co, cv); - if(found != cv) { - netdata_log_error("indexing of config '%s' in section '%s': already exists - using the existing one.", cv->name, co->name); - freez(cv->value); - freez(cv->name); - freez(cv); - return found; - } - - config_section_wrlock(co); - struct config_option *cv2 = co->values; - if(cv2) { - while (cv2->next) cv2 = cv2->next; - cv2->next = cv; - } - else co->values = cv; - config_section_unlock(co); - - return cv; -} - -int appconfig_exists(struct config *root, const char *section, const char *name) { - struct config_option *cv; - - netdata_log_debug(D_CONFIG, "request to get config in section '%s', name '%s'", section, name); - - struct section *co = appconfig_section_find(root, section); - if(!co) return 0; - - cv = appconfig_option_index_find(co, name, 0); - if(!cv) return 0; - - return 1; -} - -int appconfig_move(struct config *root, const char *section_old, const char *name_old, const char *section_new, const char *name_new) { - struct config_option *cv_old, *cv_new; - int ret = -1; - - netdata_log_debug(D_CONFIG, "request to rename config in section '%s', old name '%s', to section '%s', new name '%s'", section_old, name_old, section_new, name_new); - - struct section *co_old = appconfig_section_find(root, section_old); - if(!co_old) return ret; - - struct section *co_new = appconfig_section_find(root, section_new); - if(!co_new) co_new = appconfig_section_create(root, section_new); - - config_section_wrlock(co_old); - if(co_old != co_new) - config_section_wrlock(co_new); - - cv_old = appconfig_option_index_find(co_old, name_old, 0); - if(!cv_old) goto cleanup; - - cv_new = appconfig_option_index_find(co_new, name_new, 0); - if(cv_new) goto cleanup; - - if(unlikely(appconfig_option_index_del(co_old, cv_old) != cv_old)) - netdata_log_error("INTERNAL ERROR: deletion of config '%s' from section '%s', deleted the wrong config entry.", cv_old->name, co_old->name); - - if(co_old->values == cv_old) { - co_old->values = cv_old->next; - } - else { - struct config_option *t; - for(t = co_old->values; t && t->next != cv_old ;t = t->next) ; - if(!t || t->next != cv_old) - netdata_log_error("INTERNAL ERROR: cannot find variable '%s' in section '%s' of the config - but it should be there.", cv_old->name, co_old->name); - else - t->next = cv_old->next; - } - - freez(cv_old->name); - cv_old->name = strdupz(name_new); - cv_old->hash = simple_hash(cv_old->name); - - cv_new = cv_old; - cv_new->next = co_new->values; - co_new->values = cv_new; - - if(unlikely(appconfig_option_index_add(co_new, cv_old) != cv_old)) - netdata_log_error("INTERNAL ERROR: re-indexing of config '%s' in section '%s', already exists.", cv_old->name, co_new->name); - - ret = 0; - -cleanup: - if(co_old != co_new) - config_section_unlock(co_new); - config_section_unlock(co_old); - return ret; -} - -char *appconfig_get_by_section(struct section *co, const char *name, const char *default_value) -{ - struct config_option *cv; - // Only calls internal to this file check for a NULL result and they do not supply a NULL arg. - // External caller should treat NULL as an error case. - cv = appconfig_option_index_find(co, name, 0); - if (!cv) { - if (!default_value) return NULL; - cv = appconfig_value_create(co, name, default_value); - if (!cv) return NULL; - } - cv->flags |= CONFIG_VALUE_USED; + if(string_strcmp(opt->value, value) != 0) { + opt->flags |= CONFIG_VALUE_CHANGED; - if((cv->flags & CONFIG_VALUE_LOADED) || (cv->flags & CONFIG_VALUE_CHANGED)) { - // this is a loaded value from the config file - // if it is different than the default, mark it - if(!(cv->flags & CONFIG_VALUE_CHECKED)) { - if(default_value && strcmp(cv->value, default_value) != 0) cv->flags |= CONFIG_VALUE_CHANGED; - cv->flags |= CONFIG_VALUE_CHECKED; - } + string_freez(opt->value); + opt->value = string_strdupz(value); } - - return(cv->value); } +bool stream_conf_needs_dbengine(struct config *root) { + struct config_section *sect; + bool ret = false; -char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value) -{ - if (default_value == NULL) - netdata_log_debug(D_CONFIG, "request to get config in section '%s', name '%s' or fail", section, name); - else - netdata_log_debug(D_CONFIG, "request to get config in section '%s', name '%s', default_value '%s'", section, name, default_value); - - struct section *co = appconfig_section_find(root, section); - if (!co && !default_value) - return NULL; - if(!co) co = appconfig_section_create(root, section); - - return appconfig_get_by_section(co, name, default_value); -} + APPCONFIG_LOCK(root); + for(sect = root->sections; sect; sect = sect->next) { + if(string_strcmp(sect->name, "stream") == 0) + continue; // the first section is not relevant -long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value) -{ - char buffer[100], *s; - sprintf(buffer, "%lld", value); - - s = appconfig_get(root, section, name, buffer); - if(!s) return value; - - return strtoll(s, NULL, 0); -} - -NETDATA_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value) -{ - char buffer[100], *s; - sprintf(buffer, "%0.5" NETDATA_DOUBLE_MODIFIER, value); - - s = appconfig_get(root, section, name, buffer); - if(!s) return value; - - return str2ndd(s, NULL); -} - -inline int appconfig_test_boolean_value(char *s) { - if(!strcasecmp(s, "yes") || !strcasecmp(s, "true") || !strcasecmp(s, "on") - || !strcasecmp(s, "auto") || !strcasecmp(s, "on demand")) - return 1; - - return 0; -} - -int appconfig_get_boolean_by_section(struct section *co, const char *name, int value) { - char *s; - - s = appconfig_get_by_section(co, name, (!value)?"no":"yes"); - if(!s) return value; - - return appconfig_test_boolean_value(s); -} - -int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value) -{ - char *s; - if(value) s = "yes"; - else s = "no"; - - s = appconfig_get(root, section, name, s); - if(!s) return value; - - return appconfig_test_boolean_value(s); -} - -int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value) -{ - char *s; - - if(value == CONFIG_BOOLEAN_AUTO) - s = "auto"; - - else if(value == CONFIG_BOOLEAN_NO) - s = "no"; - - else - s = "yes"; - - s = appconfig_get(root, section, name, s); - if(!s) return value; - - if(!strcmp(s, "yes") || !strcmp(s, "true") || !strcmp(s, "on")) - return CONFIG_BOOLEAN_YES; - else if(!strcmp(s, "no") || !strcmp(s, "false") || !strcmp(s, "off")) - return CONFIG_BOOLEAN_NO; - else if(!strcmp(s, "auto") || !strcmp(s, "on demand")) - return CONFIG_BOOLEAN_AUTO; - - return value; -} - -const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value) -{ - struct config_option *cv; - - netdata_log_debug(D_CONFIG, "request to set default config in section '%s', name '%s', value '%s'", section, name, value); - - struct section *co = appconfig_section_find(root, section); - if(!co) return appconfig_set(root, section, name, value); - - cv = appconfig_option_index_find(co, name, 0); - if(!cv) return appconfig_set(root, section, name, value); - - cv->flags |= CONFIG_VALUE_USED; - - if(cv->flags & CONFIG_VALUE_LOADED) - return cv->value; - - if(strcmp(cv->value, value) != 0) { - cv->flags |= CONFIG_VALUE_CHANGED; - - freez(cv->value); - cv->value = strdupz(value); - } - - return cv->value; -} - -const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value) -{ - struct config_option *cv; - - netdata_log_debug(D_CONFIG, "request to set config in section '%s', name '%s', value '%s'", section, name, value); - - struct section *co = appconfig_section_find(root, section); - if(!co) co = appconfig_section_create(root, section); - - cv = appconfig_option_index_find(co, name, 0); - if(!cv) cv = appconfig_value_create(co, name, value); - cv->flags |= CONFIG_VALUE_USED; - - if(strcmp(cv->value, value) != 0) { - cv->flags |= CONFIG_VALUE_CHANGED; - - freez(cv->value); - cv->value = strdupz(value); - } - - return value; -} - -long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value) -{ - char buffer[100]; - sprintf(buffer, "%lld", value); - - appconfig_set(root, section, name, buffer); - - return value; -} - -NETDATA_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value) -{ - char buffer[100]; - sprintf(buffer, "%0.5" NETDATA_DOUBLE_MODIFIER, value); - - appconfig_set(root, section, name, buffer); - - return value; -} - -int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value) -{ - char *s; - if(value) s = "yes"; - else s = "no"; - - appconfig_set(root, section, name, s); - - return value; -} - -int appconfig_get_duration(struct config *root, const char *section, const char *name, const char *value) -{ - int result = 0; - const char *s; - - s = appconfig_get(root, section, name, value); - if(!s) goto fallback; - - if(!config_parse_duration(s, &result)) { - netdata_log_error("config option '[%s].%s = %s' is configured with an valid duration", section, name, s); - goto fallback; - } - - return result; - - fallback: - if(!config_parse_duration(value, &result)) - netdata_log_error("INTERNAL ERROR: default duration supplied for option '[%s].%s = %s' is not a valid duration", section, name, value); - - return result; -} - -// ---------------------------------------------------------------------------- -// config load/save - -int appconfig_load(struct config *root, char *filename, int overwrite_used, const char *section_name) -{ - int line = 0; - struct section *co = NULL; - int is_exporter_config = 0; - int _connectors = 0; // number of exporting connector sections we have - char working_instance[CONFIG_MAX_NAME + 1]; - char working_connector[CONFIG_MAX_NAME + 1]; - struct section *working_connector_section = NULL; - int global_exporting_section = 0; - - char buffer[CONFIG_FILE_LINE_MAX + 1], *s; - - if(!filename) filename = CONFIG_DIR "/" CONFIG_FILENAME; - - netdata_log_debug(D_CONFIG, "CONFIG: opening config file '%s'", filename); - - FILE *fp = fopen(filename, "r"); - if(!fp) { - // netdata_log_info("CONFIG: cannot open file '%s'. Using internal defaults.", filename); - return 0; - } - - uint32_t section_hash = 0; - if(section_name) { - section_hash = simple_hash(section_name); - } - is_exporter_config = (strstr(filename, EXPORTING_CONF) != NULL); - - while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) { - buffer[CONFIG_FILE_LINE_MAX] = '\0'; - line++; - - s = trim(buffer); - if(!s || *s == '#') { - netdata_log_debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', it is empty.", line, filename); - continue; - } - - int len = (int) strlen(s); - if(*s == '[' && s[len - 1] == ']') { - // new section - s[len - 1] = '\0'; - s++; - - if (is_exporter_config) { - global_exporting_section = - !(strcmp(s, CONFIG_SECTION_EXPORTING)) || !(strcmp(s, CONFIG_SECTION_PROMETHEUS)); - if (unlikely(!global_exporting_section)) { - int rc; - rc = is_valid_connector(s, 0); - if (likely(rc)) { - strncpyz(working_connector, s, CONFIG_MAX_NAME); - s = s + rc + 1; - if (unlikely(!(*s))) { - _connectors++; - sprintf(buffer, "instance_%d", _connectors); - s = buffer; - } - strncpyz(working_instance, s, CONFIG_MAX_NAME); - working_connector_section = NULL; - if (unlikely(appconfig_section_find(root, working_instance))) { - netdata_log_error("Instance (%s) already exists", working_instance); - co = NULL; - continue; - } - } else { - co = NULL; - netdata_log_error("Section (%s) does not specify a valid connector", s); - continue; - } - } - } - - co = appconfig_section_find(root, s); - if(!co) co = appconfig_section_create(root, s); - - if(co && section_name && overwrite_used && section_hash == co->hash && !strcmp(section_name, co->name)) { - config_section_wrlock(co); - struct config_option *cv2 = co->values; - while (cv2) { - struct config_option *save = cv2->next; - struct config_option *found = appconfig_option_index_del(co, cv2); - if(found != cv2) - netdata_log_error("INTERNAL ERROR: Cannot remove '%s' from section '%s', it was not inserted before.", - cv2->name, co->name); - - freez(cv2->name); - freez(cv2->value); - freez(cv2); - cv2 = save; - } - co->values = NULL; - config_section_unlock(co); - } - - continue; - } - - if(!co) { - // line outside a section - netdata_log_error("CONFIG: ignoring line %d ('%s') of file '%s', it is outside all sections.", line, s, filename); - continue; - } - - if(section_name && overwrite_used && section_hash != co->hash && strcmp(section_name, co->name)) { - continue; - } - - char *name = s; - char *value = strchr(s, '='); - if(!value) { - netdata_log_error("CONFIG: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename); - continue; - } - *value = '\0'; - value++; - - name = trim(name); - value = trim(value); - - if(!name || *name == '#') { - netdata_log_error("CONFIG: ignoring line %d of file '%s', name is empty.", line, filename); + struct config_option *opt = appconfig_get_raw_value_of_option_in_section(sect, "enabled", NULL, CONFIG_VALUE_TYPE_UNKNOWN, NULL); + if(!opt || !appconfig_test_boolean_value(string2str(opt->value))) continue; - } - - if(!value) value = ""; - - struct config_option *cv = appconfig_option_index_find(co, name, 0); - - if (!cv) { - cv = appconfig_value_create(co, name, value); - if (likely(is_exporter_config) && unlikely(!global_exporting_section)) { - if (unlikely(!working_connector_section)) { - working_connector_section = appconfig_section_find(root, working_connector); - if (!working_connector_section) - working_connector_section = appconfig_section_create(root, working_connector); - if (likely(working_connector_section)) { - add_connector_instance(working_connector_section, co); - } - } - } - } else { - if (((cv->flags & CONFIG_VALUE_USED) && overwrite_used) || !(cv->flags & CONFIG_VALUE_USED)) { - netdata_log_debug( - D_CONFIG, "CONFIG: line %d of file '%s', overwriting '%s/%s'.", line, filename, co->name, cv->name); - freez(cv->value); - cv->value = strdupz(value); - } else - netdata_log_debug( - D_CONFIG, - "CONFIG: ignoring line %d of file '%s', '%s/%s' is already present and used.", - line, - filename, - co->name, - cv->name); - } - cv->flags |= CONFIG_VALUE_LOADED; - } - fclose(fp); - - return 1; -} - -void appconfig_generate(struct config *root, BUFFER *wb, int only_changed) -{ - int i, pri; - struct section *co; - struct config_option *cv; - - { - int found_host_labels = 0; - for (co = root->first_section; co; co = co->next) - if(!strcmp(co->name, CONFIG_SECTION_HOST_LABEL)) - found_host_labels = 1; - - if(!found_host_labels) { - appconfig_section_create(root, CONFIG_SECTION_HOST_LABEL); - appconfig_get(root, CONFIG_SECTION_HOST_LABEL, "name", "value"); + opt = appconfig_get_raw_value_of_option_in_section(sect, "db", NULL, CONFIG_VALUE_TYPE_UNKNOWN, NULL); + if(opt && string_strcmp(opt->value, "dbengine") == 0) { + ret = true; + break; } } + APPCONFIG_UNLOCK(root); - buffer_strcat(wb, - "# netdata configuration\n" - "#\n" - "# You can download the latest version of this file, using:\n" - "#\n" - "# wget -O /etc/netdata/netdata.conf http://localhost:19999/netdata.conf\n" - "# or\n" - "# curl -o /etc/netdata/netdata.conf http://localhost:19999/netdata.conf\n" - "#\n" - "# You can uncomment and change any of the options below.\n" - "# The value shown in the commented settings, is the default value.\n" - "#\n" - "\n# global netdata configuration\n"); - - for(i = 0; i <= 17 ;i++) { - appconfig_wrlock(root); - for(co = root->first_section; co ; co = co->next) { - if(!strcmp(co->name, CONFIG_SECTION_GLOBAL)) pri = 0; - else if(!strcmp(co->name, CONFIG_SECTION_DB)) pri = 1; - else if(!strcmp(co->name, CONFIG_SECTION_DIRECTORIES)) pri = 2; - else if(!strcmp(co->name, CONFIG_SECTION_LOGS)) pri = 3; - else if(!strcmp(co->name, CONFIG_SECTION_ENV_VARS)) pri = 4; - else if(!strcmp(co->name, CONFIG_SECTION_HOST_LABEL)) pri = 5; - else if(!strcmp(co->name, CONFIG_SECTION_SQLITE)) pri = 6; - else if(!strcmp(co->name, CONFIG_SECTION_CLOUD)) pri = 7; - else if(!strcmp(co->name, CONFIG_SECTION_ML)) pri = 8; - else if(!strcmp(co->name, CONFIG_SECTION_HEALTH)) pri = 9; - else if(!strcmp(co->name, CONFIG_SECTION_WEB)) pri = 10; - else if(!strcmp(co->name, CONFIG_SECTION_WEBRTC)) pri = 11; - // by default, new sections will get pri = 12 (set at the end, below) - else if(!strcmp(co->name, CONFIG_SECTION_REGISTRY)) pri = 13; - else if(!strcmp(co->name, CONFIG_SECTION_GLOBAL_STATISTICS)) pri = 14; - else if(!strcmp(co->name, CONFIG_SECTION_PLUGINS)) pri = 15; - else if(!strcmp(co->name, CONFIG_SECTION_STATSD)) pri = 16; - else if(!strncmp(co->name, "plugin:", 7)) pri = 17; // << change the loop too if you change this - else pri = 12; // this is used for any new (currently unknown) sections - - if(i == pri) { - int loaded = 0; - int used = 0; - int changed = 0; - int count = 0; - - config_section_wrlock(co); - for(cv = co->values; cv ; cv = cv->next) { - used += (cv->flags & CONFIG_VALUE_USED)?1:0; - loaded += (cv->flags & CONFIG_VALUE_LOADED)?1:0; - changed += (cv->flags & CONFIG_VALUE_CHANGED)?1:0; - count++; - } - config_section_unlock(co); - - if(!count) continue; - if(only_changed && !changed && !loaded) continue; - - if(!used) { - buffer_sprintf(wb, "\n# section '%s' is not used.", co->name); - } - - buffer_sprintf(wb, "\n[%s]\n", co->name); - - config_section_wrlock(co); - for(cv = co->values; cv ; cv = cv->next) { - - if(used && !(cv->flags & CONFIG_VALUE_USED)) { - buffer_sprintf(wb, "\n\t# option '%s' is not used.\n", cv->name); - } - buffer_sprintf(wb, "\t%s%s = %s\n", ((!(cv->flags & CONFIG_VALUE_LOADED)) && (!(cv->flags & CONFIG_VALUE_CHANGED)) && (cv->flags & CONFIG_VALUE_USED))?"# ":"", cv->name, cv->value); - } - config_section_unlock(co); - } - } - appconfig_unlock(root); - } + return ret; } -/** - * Parse Duration - * - * Parse the string setting the result - * - * @param string the timestamp string - * @param result the output variable - * - * @return It returns 1 on success and 0 otherwise - */ -int config_parse_duration(const char* string, int* result) { - while(*string && isspace((uint8_t)*string)) string++; - - if(unlikely(!*string)) goto fallback; - - if(*string == 'n' && !strcmp(string, "never")) { - // this is a valid option - *result = 0; - return 1; - } +bool stream_conf_has_uuid_section(struct config *root) { + struct config_section *sect = NULL; + bool is_parent = false; - // make sure it is a number - if(!(isdigit((uint8_t)*string) || *string == '+' || *string == '-')) goto fallback; + APPCONFIG_LOCK(root); + for (sect = root->sections; sect; sect = sect->next) { + nd_uuid_t uuid; - char *e = NULL; - NETDATA_DOUBLE n = str2ndd(string, &e); - if(e && *e) { - switch (*e) { - case 'Y': - *result = (int) (n * 31536000); - break; - case 'M': - *result = (int) (n * 2592000); - break; - case 'w': - *result = (int) (n * 604800); - break; - case 'd': - *result = (int) (n * 86400); - break; - case 'h': - *result = (int) (n * 3600); - break; - case 'm': - *result = (int) (n * 60); - break; - case 's': - default: - *result = (int) (n); - break; + if (uuid_parse(string2str(sect->name), uuid) != -1 && + appconfig_get_boolean_by_section(sect, "enabled", 0)) { + is_parent = true; + break; } } - else - *result = (int)(n); - - return 1; - - fallback: - *result = 0; - return 0; -} + APPCONFIG_UNLOCK(root); -struct section *appconfig_get_section(struct config *root, const char *name) -{ - return appconfig_section_find(root, name); + return is_parent; } diff --git a/src/libnetdata/config/appconfig.h b/src/libnetdata/config/appconfig.h index 214a15edd..f1551b387 100644 --- a/src/libnetdata/config/appconfig.h +++ b/src/libnetdata/config/appconfig.h @@ -103,7 +103,6 @@ #define CONFIG_SECTION_GLOBAL_STATISTICS "global statistics" #define CONFIG_SECTION_DB "db" - // these are used to limit the configuration names and values lengths // they are not enforced by config.c functions (they will strdup() all strings, no matter of their length) #define CONFIG_MAX_NAME 1024 @@ -113,94 +112,43 @@ // Config definitions #define CONFIG_FILE_LINE_MAX ((CONFIG_MAX_NAME + CONFIG_MAX_VALUE + 1024) * 2) -#define CONFIG_VALUE_LOADED 0x01 // has been loaded from the config -#define CONFIG_VALUE_USED 0x02 // has been accessed from the program -#define CONFIG_VALUE_CHANGED 0x04 // has been changed from the loaded value or the internal default value -#define CONFIG_VALUE_CHECKED 0x08 // has been checked if the value is different from the default - -struct config_option { - avl_t avl_node; // the index entry of this entry - this has to be first! - - uint8_t flags; - uint32_t hash; // a simple hash to speed up searching - // we first compare hashes, and only if the hashes are equal we do string comparisons - - char *name; - char *value; - - struct config_option *next; // config->mutex protects just this -}; - -struct section { - avl_t avl_node; // the index entry of this section - this has to be first! - - uint32_t hash; // a simple hash to speed up searching - // we first compare hashes, and only if the hashes are equal we do string comparisons - - char *name; - - struct section *next; // global config_mutex protects just this - - struct config_option *values; - avl_tree_lock values_index; - - netdata_mutex_t mutex; // this locks only the writers, to ensure atomic updates - // readers are protected using the rwlock in avl_tree_lock -}; +struct config_section; struct config { - struct section *first_section; - struct section *last_section; // optimize inserting at the end - netdata_mutex_t mutex; + struct config_section *sections; + SPINLOCK spinlock; avl_tree_lock index; }; -#define CONFIG_BOOLEAN_INVALID 100 // an invalid value to check for validity (used as default initialization when needed) +#define APPCONFIG_INITIALIZER (struct config) { \ + .sections = NULL, \ + .spinlock = NETDATA_SPINLOCK_INITIALIZER, \ + .index = { \ + .avl_tree = { \ + .root = NULL, \ + .compar = appconfig_section_compare, \ + }, \ + .rwlock = AVL_LOCK_INITIALIZER, \ + }, \ + } -#define CONFIG_BOOLEAN_NO 0 // disabled -#define CONFIG_BOOLEAN_YES 1 // enabled +int appconfig_load(struct config *root, char *filename, int overwrite_used, const char *section_name); -#ifndef CONFIG_BOOLEAN_AUTO -#define CONFIG_BOOLEAN_AUTO 2 // enabled if it has useful info when enabled -#endif +typedef bool (*appconfig_foreach_value_cb_t)(void *data, const char *name, const char *value); +size_t appconfig_foreach_value_in_section(struct config *root, const char *section, appconfig_foreach_value_cb_t cb, void *data); -int appconfig_load(struct config *root, char *filename, int overwrite_used, const char *section_name); -void config_section_wrlock(struct section *co); -void config_section_unlock(struct section *co); - -char *appconfig_get_by_section(struct section *co, const char *name, const char *default_value); -char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value); -long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value); -NETDATA_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value); -int appconfig_get_boolean_by_section(struct section *co, const char *name, int value); -int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value); -int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value); -int appconfig_get_duration(struct config *root, const char *section, const char *name, const char *value); - -const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value); -const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value); -long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value); -NETDATA_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value); -int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value); +// sets a raw value, only if it is not loaded from the config +void appconfig_set_default_raw_value(struct config *root, const char *section, const char *name, const char *value); int appconfig_exists(struct config *root, const char *section, const char *name); int appconfig_move(struct config *root, const char *section_old, const char *name_old, const char *section_new, const char *name_new); +int appconfig_move_everywhere(struct config *root, const char *name_old, const char *name_new); -void appconfig_generate(struct config *root, BUFFER *wb, int only_changed); +void appconfig_generate(struct config *root, BUFFER *wb, int only_changed, bool netdata_conf); int appconfig_section_compare(void *a, void *b); -void appconfig_section_destroy_non_loaded(struct config *root, const char *section); -void appconfig_section_option_destroy_non_loaded(struct config *root, const char *section, const char *name); - -int config_parse_duration(const char* string, int* result); - -struct section *appconfig_get_section(struct config *root, const char *name); - -void appconfig_wrlock(struct config *root); -void appconfig_unlock(struct config *root); - -int appconfig_test_boolean_value(char *s); +bool appconfig_test_boolean_value(const char *s); struct connector_instance { char instance_name[CONFIG_MAX_NAME + 1]; @@ -208,13 +156,37 @@ struct connector_instance { }; typedef struct _connector_instance { - struct section *connector; // actual connector - struct section *instance; // This instance + struct config_section *connector; // actual connector + struct config_section *instance; // This instance char instance_name[CONFIG_MAX_NAME + 1]; char connector_name[CONFIG_MAX_NAME + 1]; struct _connector_instance *next; // Next instance } _CONNECTOR_INSTANCE; -_CONNECTOR_INSTANCE *add_connector_instance(struct section *connector, struct section *instance); +_CONNECTOR_INSTANCE *add_connector_instance(struct config_section *connector, struct config_section *instance); + +// ---------------------------------------------------------------------------- +// shortcuts for the default netdata configuration + +#define config_load(filename, overwrite_used, section) appconfig_load(&netdata_config, filename, overwrite_used, section) + +#define config_set_default_raw_value(section, name, value) appconfig_set_default_raw_value(&netdata_config, section, name, value) + +#define config_exists(section, name) appconfig_exists(&netdata_config, section, name) +#define config_move(section_old, name_old, section_new, name_new) appconfig_move(&netdata_config, section_old, name_old, section_new, name_new) + +#define netdata_conf_generate(buffer, only_changed) appconfig_generate(&netdata_config, buffer, only_changed, true) + +#define config_section_destroy(section) appconfig_section_destroy_non_loaded(&netdata_config, section) +#define config_section_option_destroy(section, name) appconfig_section_option_destroy_non_loaded(&netdata_config, section, name) + +bool stream_conf_needs_dbengine(struct config *root); +bool stream_conf_has_uuid_section(struct config *root); + +#include "appconfig_api_text.h" +#include "appconfig_api_numbers.h" +#include "appconfig_api_boolean.h" +#include "appconfig_api_sizes.h" +#include "appconfig_api_durations.h" -#endif /* NETDATA_CONFIG_H */
\ No newline at end of file +#endif // NETDATA_CONFIG_H diff --git a/src/libnetdata/config/appconfig_api_boolean.c b/src/libnetdata/config/appconfig_api_boolean.c new file mode 100644 index 000000000..abe515736 --- /dev/null +++ b/src/libnetdata/config/appconfig_api_boolean.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" +#include "appconfig_api_boolean.h" + +bool appconfig_test_boolean_value(const char *s) { + if(!strcasecmp(s, "yes") || !strcasecmp(s, "true") || !strcasecmp(s, "on") + || !strcasecmp(s, "auto") || !strcasecmp(s, "on demand")) + return true; + + return false; +} + +int appconfig_get_boolean_by_section(struct config_section *sect, const char *name, int value) { + struct config_option *opt = appconfig_get_raw_value_of_option_in_section( + sect, name, (!value) ? "no" : "yes", CONFIG_VALUE_TYPE_BOOLEAN, NULL); + if(!opt) return value; + + return appconfig_test_boolean_value(string2str(opt->value)); +} + +int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value) { + const char *s; + if(value) s = "yes"; + else s = "no"; + + struct config_option *opt = appconfig_get_raw_value(root, section, name, s, CONFIG_VALUE_TYPE_BOOLEAN, NULL); + if(!opt) return value; + s = string2str(opt->value); + + return appconfig_test_boolean_value(s); +} + +int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value) { + const char *s; + + if(value == CONFIG_BOOLEAN_AUTO) + s = "auto"; + + else if(value == CONFIG_BOOLEAN_NO) + s = "no"; + + else + s = "yes"; + + struct config_option *opt = appconfig_get_raw_value(root, section, name, s, CONFIG_VALUE_TYPE_BOOLEAN_ONDEMAND, NULL); + if(!opt) return value; + + s = string2str(opt->value); + if(!strcmp(s, "yes") || !strcmp(s, "true") || !strcmp(s, "on")) + return CONFIG_BOOLEAN_YES; + else if(!strcmp(s, "no") || !strcmp(s, "false") || !strcmp(s, "off")) + return CONFIG_BOOLEAN_NO; + else if(!strcmp(s, "auto") || !strcmp(s, "on demand")) + return CONFIG_BOOLEAN_AUTO; + + return value; +} + +int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value) { + const char *s; + if(value) s = "yes"; + else s = "no"; + + appconfig_set_raw_value(root, section, name, s, CONFIG_VALUE_TYPE_BOOLEAN); + + return value; +} diff --git a/src/libnetdata/config/appconfig_api_boolean.h b/src/libnetdata/config/appconfig_api_boolean.h new file mode 100644 index 000000000..2b05fce60 --- /dev/null +++ b/src/libnetdata/config/appconfig_api_boolean.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_APPCONFIG_API_BOOLEAN_H +#define NETDATA_APPCONFIG_API_BOOLEAN_H + +#define CONFIG_BOOLEAN_INVALID 100 // an invalid value to check for validity (used as default initialization when needed) + +#define CONFIG_BOOLEAN_NO 0 // disabled +#define CONFIG_BOOLEAN_YES 1 // enabled + +#ifndef CONFIG_BOOLEAN_AUTO +#define CONFIG_BOOLEAN_AUTO 2 // enabled if it has useful info when enabled +#endif + +int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value); +#define config_get_boolean(section, name, value) appconfig_get_boolean(&netdata_config, section, name, value) + +int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value); +#define config_get_boolean_ondemand(section, name, value) appconfig_get_boolean_ondemand(&netdata_config, section, name, value) + +int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value); +#define config_set_boolean(section, name, value) appconfig_set_boolean(&netdata_config, section, name, value) + +#endif //NETDATA_APPCONFIG_API_BOOLEAN_H diff --git a/src/libnetdata/config/appconfig_api_durations.c b/src/libnetdata/config/appconfig_api_durations.c new file mode 100644 index 000000000..88c462ac6 --- /dev/null +++ b/src/libnetdata/config/appconfig_api_durations.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" +#include "appconfig_api_durations.h" + + +static STRING *reformat_duration_seconds(STRING *value) { + int result = 0; + if(!duration_parse_seconds(string2str(value), &result)) + return value; + + char buf[128]; + if(duration_snprintf_time_t(buf, sizeof(buf), result) > 0 && string_strcmp(value, buf) != 0) { + string_freez(value); + return string_strdupz(buf); + } + + return value; +} + +time_t appconfig_get_duration_seconds(struct config *root, const char *section, const char *name, time_t default_value) { + char default_str[128]; + duration_snprintf_time_t(default_str, sizeof(default_str), default_value); + + struct config_option *opt = appconfig_get_raw_value( + root, section, name, default_str, CONFIG_VALUE_TYPE_DURATION_IN_SECS, reformat_duration_seconds); + if(!opt) + return default_value; + + const char *s = string2str(opt->value); + + int result = 0; + if(!duration_parse_seconds(s, &result)) { + appconfig_set_raw_value(root, section, name, default_str, CONFIG_VALUE_TYPE_DURATION_IN_SECS); + netdata_log_error("config option '[%s].%s = %s' is configured with an invalid duration", section, name, s); + return default_value; + } + + return ABS(result); +} + +time_t appconfig_set_duration_seconds(struct config *root, const char *section, const char *name, time_t value) { + char str[128]; + duration_snprintf_time_t(str, sizeof(str), value); + + appconfig_set_raw_value(root, section, name, str, CONFIG_VALUE_TYPE_DURATION_IN_SECS); + return value; +} + +static STRING *reformat_duration_ms(STRING *value) { + int64_t result = 0; + if(!duration_parse_msec_t(string2str(value), &result)) + return value; + + char buf[128]; + if(duration_snprintf_msec_t(buf, sizeof(buf), result) > 0 && string_strcmp(value, buf) != 0) { + string_freez(value); + return string_strdupz(buf); + } + + return value; +} + +msec_t appconfig_get_duration_ms(struct config *root, const char *section, const char *name, msec_t default_value) { + char default_str[128]; + duration_snprintf_msec_t(default_str, sizeof(default_str), default_value); + + struct config_option *opt = appconfig_get_raw_value( + root, section, name, default_str, CONFIG_VALUE_TYPE_DURATION_IN_MS, reformat_duration_ms); + if(!opt) + return default_value; + + const char *s = string2str(opt->value); + + smsec_t result = 0; + if(!duration_parse_msec_t(s, &result)) { + appconfig_set_raw_value(root, section, name, default_str, CONFIG_VALUE_TYPE_DURATION_IN_MS); + netdata_log_error("config option '[%s].%s = %s' is configured with an invalid duration", section, name, s); + return default_value; + } + + return ABS(result); +} + +msec_t appconfig_set_duration_ms(struct config *root, const char *section, const char *name, msec_t value) { + char str[128]; + duration_snprintf_msec_t(str, sizeof(str), (smsec_t)value); + + appconfig_set_raw_value(root, section, name, str, CONFIG_VALUE_TYPE_DURATION_IN_MS); + return value; +} + +static STRING *reformat_duration_days(STRING *value) { + int64_t result = 0; + if(!duration_parse_days(string2str(value), &result)) + return value; + + char buf[128]; + if(duration_snprintf_days(buf, sizeof(buf), result) > 0 && string_strcmp(value, buf) != 0) { + string_freez(value); + return string_strdupz(buf); + } + + return value; +} + +unsigned appconfig_get_duration_days(struct config *root, const char *section, const char *name, unsigned default_value) { + char default_str[128]; + duration_snprintf_days(default_str, sizeof(default_str), (int)default_value); + + struct config_option *opt = appconfig_get_raw_value( + root, section, name, default_str, CONFIG_VALUE_TYPE_DURATION_IN_DAYS, reformat_duration_days); + if(!opt) + return default_value; + + const char *s = string2str(opt->value); + + int64_t result = 0; + if(!duration_parse_days(s, &result)) { + appconfig_set_raw_value(root, section, name, default_str, CONFIG_VALUE_TYPE_DURATION_IN_DAYS); + netdata_log_error("config option '[%s].%s = %s' is configured with an invalid duration", section, name, s); + return default_value; + } + + return (unsigned)ABS(result); +} + +unsigned appconfig_set_duration_days(struct config *root, const char *section, const char *name, unsigned value) { + char str[128]; + duration_snprintf_days(str, sizeof(str), value); + appconfig_set_raw_value(root, section, name, str, CONFIG_VALUE_TYPE_DURATION_IN_DAYS); + return value; +} + diff --git a/src/libnetdata/config/appconfig_api_durations.h b/src/libnetdata/config/appconfig_api_durations.h new file mode 100644 index 000000000..26d6c6ba3 --- /dev/null +++ b/src/libnetdata/config/appconfig_api_durations.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_APPCONFIG_API_DURATIONS_H +#define NETDATA_APPCONFIG_API_DURATIONS_H + +msec_t appconfig_get_duration_ms(struct config *root, const char *section, const char *name, msec_t default_value); +msec_t appconfig_set_duration_ms(struct config *root, const char *section, const char *name, msec_t value); +#define config_get_duration_ms(section, name, value) appconfig_get_duration_ms(&netdata_config, section, name, value) +#define config_set_duration_ms(section, name, value) appconfig_set_duration_ms(&netdata_config, section, name, value) + +time_t appconfig_get_duration_seconds(struct config *root, const char *section, const char *name, time_t default_value); +time_t appconfig_set_duration_seconds(struct config *root, const char *section, const char *name, time_t value); +#define config_get_duration_seconds(section, name, value) appconfig_get_duration_seconds(&netdata_config, section, name, value) +#define config_set_duration_seconds(section, name, value) appconfig_set_duration_seconds(&netdata_config, section, name, value) + +unsigned appconfig_get_duration_days(struct config *root, const char *section, const char *name, unsigned default_value); +unsigned appconfig_set_duration_days(struct config *root, const char *section, const char *name, unsigned value); +#define config_get_duration_days(section, name, value) appconfig_get_duration_days(&netdata_config, section, name, value) +#define config_set_duration_days(section, name, value) appconfig_set_duration_days(&netdata_config, section, name, value) + +#endif //NETDATA_APPCONFIG_API_DURATIONS_H diff --git a/src/libnetdata/config/appconfig_api_numbers.c b/src/libnetdata/config/appconfig_api_numbers.c new file mode 100644 index 000000000..cc3776c18 --- /dev/null +++ b/src/libnetdata/config/appconfig_api_numbers.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" +#include "appconfig_api_numbers.h" + +long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value) { + char buffer[100]; + sprintf(buffer, "%lld", value); + + struct config_option *opt = appconfig_get_raw_value(root, section, name, buffer, CONFIG_VALUE_TYPE_INTEGER, NULL); + if(!opt) return value; + + const char *s = string2str(opt->value); + return strtoll(s, NULL, 0); +} + +NETDATA_DOUBLE appconfig_get_double(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value) { + char buffer[100]; + sprintf(buffer, "%0.5" NETDATA_DOUBLE_MODIFIER, value); + + struct config_option *opt = appconfig_get_raw_value(root, section, name, buffer, CONFIG_VALUE_TYPE_DOUBLE, NULL); + if(!opt) return value; + + const char *s = string2str(opt->value); + return str2ndd(s, NULL); +} + +long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value) { + char buffer[100]; + sprintf(buffer, "%lld", value); + + appconfig_set_raw_value(root, section, name, buffer, CONFIG_VALUE_TYPE_INTEGER); + return value; +} + +NETDATA_DOUBLE appconfig_set_double(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value) { + char buffer[100]; + sprintf(buffer, "%0.5" NETDATA_DOUBLE_MODIFIER, value); + + appconfig_set_raw_value(root, section, name, buffer, CONFIG_VALUE_TYPE_DOUBLE); + return value; +} + diff --git a/src/libnetdata/config/appconfig_api_numbers.h b/src/libnetdata/config/appconfig_api_numbers.h new file mode 100644 index 000000000..58d382e3d --- /dev/null +++ b/src/libnetdata/config/appconfig_api_numbers.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_APPCONFIG_API_NUMBERS_H +#define NETDATA_APPCONFIG_API_NUMBERS_H + +long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value); +long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value); +#define config_get_number(section, name, value) appconfig_get_number(&netdata_config, section, name, value) +#define config_set_number(section, name, value) appconfig_set_number(&netdata_config, section, name, value) + +NETDATA_DOUBLE appconfig_get_double(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value); +NETDATA_DOUBLE appconfig_set_double(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value); +#define config_get_double(section, name, value) appconfig_get_double(&netdata_config, section, name, value) +#define config_set_double(section, name, value) appconfig_set_float(&netdata_config, section, name, value) + +#endif //NETDATA_APPCONFIG_API_NUMBERS_H diff --git a/src/libnetdata/config/appconfig_api_sizes.c b/src/libnetdata/config/appconfig_api_sizes.c new file mode 100644 index 000000000..67b1dce9e --- /dev/null +++ b/src/libnetdata/config/appconfig_api_sizes.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" +#include "appconfig_api_sizes.h" + +static STRING *reformat_size_bytes(STRING *value) { + uint64_t result = 0; + if(!size_parse_bytes(string2str(value), &result)) + return value; + + char buf[128]; + if(size_snprintf_bytes(buf, sizeof(buf), result) > 0 && string_strcmp(value, buf) != 0) { + string_freez(value); + return string_strdupz(buf); + } + + return value; +} + +uint64_t appconfig_get_size_bytes(struct config *root, const char *section, const char *name, uint64_t default_value) { + char default_str[128]; + size_snprintf_bytes(default_str, sizeof(default_str), (int)default_value); + + struct config_option *opt = + appconfig_get_raw_value(root, section, name, default_str, CONFIG_VALUE_TYPE_SIZE_IN_BYTES, reformat_size_bytes); + if(!opt) + return default_value; + + const char *s = string2str(opt->value); + uint64_t result = 0; + if(!size_parse_bytes(s, &result)) { + appconfig_set_raw_value(root, section, name, default_str, CONFIG_VALUE_TYPE_SIZE_IN_BYTES); + netdata_log_error("config option '[%s].%s = %s' is configured with an invalid size", section, name, s); + return default_value; + } + + return result; +} + +uint64_t appconfig_set_size_bytes(struct config *root, const char *section, const char *name, uint64_t value) { + char str[128]; + size_snprintf_bytes(str, sizeof(str), value); + appconfig_set_raw_value(root, section, name, str, CONFIG_VALUE_TYPE_SIZE_IN_BYTES); + return value; +} + +static STRING *reformat_size_mb(STRING *value) { + uint64_t result = 0; + if(!size_parse_mb(string2str(value), &result)) + return value; + + char buf[128]; + if(size_snprintf_mb(buf, sizeof(buf), result) > 0 && string_strcmp(value, buf) != 0) { + string_freez(value); + return string_strdupz(buf); + } + + return value; +} + +uint64_t appconfig_get_size_mb(struct config *root, const char *section, const char *name, uint64_t default_value) { + char default_str[128]; + size_snprintf_mb(default_str, sizeof(default_str), (int)default_value); + + struct config_option *opt = + appconfig_get_raw_value(root, section, name, default_str, CONFIG_VALUE_TYPE_SIZE_IN_MB, reformat_size_mb); + if(!opt) + return default_value; + + const char *s = string2str(opt->value); + uint64_t result = 0; + if(!size_parse_mb(s, &result)) { + appconfig_set_raw_value(root, section, name, default_str, CONFIG_VALUE_TYPE_SIZE_IN_MB); + netdata_log_error("config option '[%s].%s = %s' is configured with an invalid size", section, name, s); + return default_value; + } + + return (unsigned)result; +} + +uint64_t appconfig_set_size_mb(struct config *root, const char *section, const char *name, uint64_t value) { + char str[128]; + size_snprintf_mb(str, sizeof(str), value); + appconfig_set_raw_value(root, section, name, str, CONFIG_VALUE_TYPE_SIZE_IN_MB); + return value; +} diff --git a/src/libnetdata/config/appconfig_api_sizes.h b/src/libnetdata/config/appconfig_api_sizes.h new file mode 100644 index 000000000..98ef209fe --- /dev/null +++ b/src/libnetdata/config/appconfig_api_sizes.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_APPCONFIG_API_SIZES_H +#define NETDATA_APPCONFIG_API_SIZES_H + +uint64_t appconfig_get_size_bytes(struct config *root, const char *section, const char *name, uint64_t default_value); +uint64_t appconfig_set_size_bytes(struct config *root, const char *section, const char *name, uint64_t value); +#define config_get_size_bytes(section, name, value) appconfig_get_size_bytes(&netdata_config, section, name, value) +#define config_set_size_bytes(section, name, value) appconfig_set_size_bytes(&netdata_config, section, name, value) + +uint64_t appconfig_get_size_mb(struct config *root, const char *section, const char *name, uint64_t default_value); +uint64_t appconfig_set_size_mb(struct config *root, const char *section, const char *name, uint64_t value); +#define config_get_size_mb(section, name, value) appconfig_get_size_mb(&netdata_config, section, name, value) +#define config_set_size_mb(section, name, value) appconfig_set_size_mb(&netdata_config, section, name, value) + +#endif //NETDATA_APPCONFIG_API_SIZES_H diff --git a/src/libnetdata/config/appconfig_api_text.c b/src/libnetdata/config/appconfig_api_text.c new file mode 100644 index 000000000..b314972f0 --- /dev/null +++ b/src/libnetdata/config/appconfig_api_text.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" +#include "appconfig_api_text.h" + +const char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value) { + struct config_option *opt = appconfig_get_raw_value(root, section, name, default_value, CONFIG_VALUE_TYPE_TEXT, NULL); + if(!opt) + return default_value; + + return string2str(opt->value); +} + +const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value) { + struct config_option *opt = appconfig_set_raw_value(root, section, name, value, CONFIG_VALUE_TYPE_TEXT); + return string2str(opt->value); +} diff --git a/src/libnetdata/config/appconfig_api_text.h b/src/libnetdata/config/appconfig_api_text.h new file mode 100644 index 000000000..7e1e85f7e --- /dev/null +++ b/src/libnetdata/config/appconfig_api_text.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_APPCONFIG_API_TEXT_H +#define NETDATA_APPCONFIG_API_TEXT_H + +const char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value); +const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value); +#define config_get(section, name, default_value) appconfig_get(&netdata_config, section, name, default_value) +#define config_set(section, name, default_value) appconfig_set(&netdata_config, section, name, default_value) + + +#endif //NETDATA_APPCONFIG_API_TEXT_H diff --git a/src/libnetdata/config/appconfig_cleanup.c b/src/libnetdata/config/appconfig_cleanup.c new file mode 100644 index 000000000..22f4ac3e9 --- /dev/null +++ b/src/libnetdata/config/appconfig_cleanup.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" + +void appconfig_section_destroy_non_loaded(struct config *root, const char *section) +{ + struct config_section *sect; + struct config_option *opt; + + netdata_log_debug(D_CONFIG, "Destroying section '%s'.", section); + + sect = appconfig_section_find(root, section); + if(!sect) { + netdata_log_error("Could not destroy section '%s'. Not found.", section); + return; + } + + SECTION_LOCK(sect); + + // find if there is any loaded option + for(opt = sect->values; opt; opt = opt->next) { + if (opt->flags & CONFIG_VALUE_LOADED) { + // do not destroy values that were loaded from the configuration files. + SECTION_UNLOCK(sect); + return; + } + } + + // no option is loaded, free them all + appconfig_section_remove_and_delete(root, sect, false, true); +} + +void appconfig_section_option_destroy_non_loaded(struct config *root, const char *section, const char *name) { + struct config_section *sect; + sect = appconfig_section_find(root, section); + if (!sect) { + netdata_log_error("Could not destroy section option '%s -> %s'. The section not found.", section, name); + return; + } + + SECTION_LOCK(sect); + + struct config_option *opt = appconfig_option_find(sect, name); + if (opt && opt->flags & CONFIG_VALUE_LOADED) { + SECTION_UNLOCK(sect); + return; + } + + if (unlikely(!(opt && appconfig_option_del(sect, opt)))) { + SECTION_UNLOCK(sect); + netdata_log_error("Could not destroy section option '%s -> %s'. The option not found.", section, name); + return; + } + + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(sect->values, opt, prev, next); + + appconfig_option_free(opt); + SECTION_UNLOCK(sect); +} + diff --git a/src/libnetdata/config/appconfig_conf_file.c b/src/libnetdata/config/appconfig_conf_file.c new file mode 100644 index 000000000..4ac8b376e --- /dev/null +++ b/src/libnetdata/config/appconfig_conf_file.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" + +ENUM_STR_MAP_DEFINE(CONFIG_VALUE_TYPES) = { + { .id = CONFIG_VALUE_TYPE_UNKNOWN, .name ="unknown", }, + { .id = CONFIG_VALUE_TYPE_TEXT, .name ="text", }, + { .id = CONFIG_VALUE_TYPE_HOSTNAME, .name ="hostname", }, + { .id = CONFIG_VALUE_TYPE_USERNAME, .name ="username", }, + { .id = CONFIG_VALUE_TYPE_FILENAME, .name ="filename", }, + { .id = CONFIG_VALUE_TYPE_PATH, .name ="path", }, + { .id = CONFIG_VALUE_TYPE_SIMPLE_PATTERN, .name ="simple pattern", }, + { .id = CONFIG_VALUE_TYPE_URL, .name ="URL", }, + { .id = CONFIG_VALUE_TYPE_ENUM, .name ="one of keywords", }, + { .id = CONFIG_VALUE_TYPE_BITMAP, .name ="any of keywords", }, + { .id = CONFIG_VALUE_TYPE_INTEGER, .name ="number (integer)", }, + { .id = CONFIG_VALUE_TYPE_DOUBLE, .name ="number (double)", }, + { .id = CONFIG_VALUE_TYPE_BOOLEAN, .name ="yes or no", }, + { .id = CONFIG_VALUE_TYPE_BOOLEAN_ONDEMAND, .name ="yes, no, or auto", }, + { .id = CONFIG_VALUE_TYPE_DURATION_IN_SECS, .name ="duration (seconds)", }, + { .id = CONFIG_VALUE_TYPE_DURATION_IN_MS, .name ="duration (ms)", }, + { .id = CONFIG_VALUE_TYPE_DURATION_IN_DAYS, .name ="duration (days)", }, + { .id = CONFIG_VALUE_TYPE_SIZE_IN_BYTES, .name ="size (bytes)", }, + { .id = CONFIG_VALUE_TYPE_SIZE_IN_MB, .name ="size (MiB)", }, +}; + +ENUM_STR_DEFINE_FUNCTIONS(CONFIG_VALUE_TYPES, CONFIG_VALUE_TYPE_UNKNOWN, "unknown"); + + +// ---------------------------------------------------------------------------- +// config load/save + +int appconfig_load(struct config *root, char *filename, int overwrite_used, const char *section_name) { + int line = 0; + struct config_section *sect = NULL; + int is_exporter_config = 0; + int _connectors = 0; // number of exporting connector sections we have + char working_instance[CONFIG_MAX_NAME + 1]; + char working_connector[CONFIG_MAX_NAME + 1]; + struct config_section *working_connector_section = NULL; + int global_exporting_section = 0; + + char buffer[CONFIG_FILE_LINE_MAX + 1], *s; + + if(!filename) filename = CONFIG_DIR "/" CONFIG_FILENAME; + + netdata_log_debug(D_CONFIG, "CONFIG: opening config file '%s'", filename); + + FILE *fp = fopen(filename, "r"); + if(!fp) { + if(errno != ENOENT) + netdata_log_info("CONFIG: cannot open file '%s'. Using internal defaults.", filename); + + return 0; + } + + CLEAN_STRING *section_string = string_strdupz(section_name); + is_exporter_config = (strstr(filename, EXPORTING_CONF) != NULL); + + while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) { + buffer[CONFIG_FILE_LINE_MAX] = '\0'; + line++; + + s = trim(buffer); + if(!s || *s == '#') { + netdata_log_debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', it is empty.", line, filename); + continue; + } + + int len = (int) strlen(s); + if(*s == '[' && s[len - 1] == ']') { + // new section + s[len - 1] = '\0'; + s++; + + if (is_exporter_config) { + global_exporting_section = !(strcmp(s, CONFIG_SECTION_EXPORTING)) || !(strcmp(s, CONFIG_SECTION_PROMETHEUS)); + + if (unlikely(!global_exporting_section)) { + int rc; + rc = is_valid_connector(s, 0); + if (likely(rc)) { + strncpyz(working_connector, s, CONFIG_MAX_NAME); + s = s + rc + 1; + if (unlikely(!(*s))) { + _connectors++; + sprintf(buffer, "instance_%d", _connectors); + s = buffer; + } + strncpyz(working_instance, s, CONFIG_MAX_NAME); + working_connector_section = NULL; + if (unlikely(appconfig_section_find(root, working_instance))) { + netdata_log_error("Instance (%s) already exists", working_instance); + sect = NULL; + continue; + } + } + else { + sect = NULL; + netdata_log_error("Section (%s) does not specify a valid connector", s); + continue; + } + } + } + + sect = appconfig_section_find(root, s); + if(!sect) + sect = appconfig_section_create(root, s); + + if(sect && section_string && overwrite_used && section_string == sect->name) { + SECTION_LOCK(sect); + + while(sect->values) + appconfig_option_remove_and_delete(sect, sect->values, true); + + SECTION_UNLOCK(sect); + } + + continue; + } + + if(!sect) { + // line outside a section + netdata_log_error("CONFIG: ignoring line %d ('%s') of file '%s', it is outside all sections.", line, s, filename); + continue; + } + + if(section_string && overwrite_used && section_string != sect->name) + continue; + + char *name = s; + char *value = strchr(s, '='); + if(!value) { + netdata_log_error("CONFIG: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename); + continue; + } + *value = '\0'; + value++; + + name = trim(name); + value = trim(value); + + if(!name || *name == '#') { + netdata_log_error("CONFIG: ignoring line %d of file '%s', name is empty.", line, filename); + continue; + } + + if(!value) value = ""; + + struct config_option *opt = appconfig_option_find(sect, name); + + if (!opt) { + opt = appconfig_option_create(sect, name, value); + if (likely(is_exporter_config) && unlikely(!global_exporting_section)) { + if (unlikely(!working_connector_section)) { + working_connector_section = appconfig_section_find(root, working_connector); + if (!working_connector_section) + working_connector_section = appconfig_section_create(root, working_connector); + if (likely(working_connector_section)) { + add_connector_instance(working_connector_section, sect); + } + } + } + } + else { + if (((opt->flags & CONFIG_VALUE_USED) && overwrite_used) || !(opt->flags & CONFIG_VALUE_USED)) { + string_freez(opt->value); + opt->value = string_strdupz(value); + } + } + opt->flags |= CONFIG_VALUE_LOADED; + } + + fclose(fp); + + return 1; +} + +void appconfig_generate(struct config *root, BUFFER *wb, int only_changed, bool netdata_conf) +{ + int i, pri; + struct config_section *sect; + struct config_option *opt; + + { + int found_host_labels = 0; + for (sect = root->sections; sect; sect = sect->next) + if(!string_strcmp(sect->name, CONFIG_SECTION_HOST_LABEL)) + found_host_labels = 1; + + if(netdata_conf && !found_host_labels) { + appconfig_section_create(root, CONFIG_SECTION_HOST_LABEL); + appconfig_get_raw_value(root, CONFIG_SECTION_HOST_LABEL, "name", "value", CONFIG_VALUE_TYPE_TEXT, NULL); + } + } + + if(netdata_conf) { + buffer_strcat(wb, + "# netdata configuration\n" + "#\n" + "# You can download the latest version of this file, using:\n" + "#\n" + "# wget -O /etc/netdata/netdata.conf http://localhost:19999/netdata.conf\n" + "# or\n" + "# curl -o /etc/netdata/netdata.conf http://localhost:19999/netdata.conf\n" + "#\n" + "# You can uncomment and change any of the options below.\n" + "# The value shown in the commented settings, is the default value.\n" + "#\n" + "\n# global netdata configuration\n"); + } + + for(i = 0; i <= 17 ;i++) { + APPCONFIG_LOCK(root); + for(sect = root->sections; sect; sect = sect->next) { + if(!string_strcmp(sect->name, CONFIG_SECTION_GLOBAL)) pri = 0; + else if(!string_strcmp(sect->name, CONFIG_SECTION_DB)) pri = 1; + else if(!string_strcmp(sect->name, CONFIG_SECTION_DIRECTORIES)) pri = 2; + else if(!string_strcmp(sect->name, CONFIG_SECTION_LOGS)) pri = 3; + else if(!string_strcmp(sect->name, CONFIG_SECTION_ENV_VARS)) pri = 4; + else if(!string_strcmp(sect->name, CONFIG_SECTION_HOST_LABEL)) pri = 5; + else if(!string_strcmp(sect->name, CONFIG_SECTION_SQLITE)) pri = 6; + else if(!string_strcmp(sect->name, CONFIG_SECTION_CLOUD)) pri = 7; + else if(!string_strcmp(sect->name, CONFIG_SECTION_ML)) pri = 8; + else if(!string_strcmp(sect->name, CONFIG_SECTION_HEALTH)) pri = 9; + else if(!string_strcmp(sect->name, CONFIG_SECTION_WEB)) pri = 10; + else if(!string_strcmp(sect->name, CONFIG_SECTION_WEBRTC)) pri = 11; + // by default, new sections will get pri = 12 (set at the end, below) + else if(!string_strcmp(sect->name, CONFIG_SECTION_REGISTRY)) pri = 13; + else if(!string_strcmp(sect->name, CONFIG_SECTION_GLOBAL_STATISTICS)) pri = 14; + else if(!string_strcmp(sect->name, CONFIG_SECTION_PLUGINS)) pri = 15; + else if(!string_strcmp(sect->name, CONFIG_SECTION_STATSD)) pri = 16; + else if(!string_strncmp(sect->name, "plugin:", 7)) pri = 17; // << change the loop too if you change this + else pri = 12; // this is used for any new (currently unknown) sections + + if(i == pri) { + int loaded = 0; + int used = 0; + int changed = 0; + int count = 0; + + SECTION_LOCK(sect); + for(opt = sect->values; opt; opt = opt->next) { + used += (opt->flags & CONFIG_VALUE_USED)?1:0; + loaded += (opt->flags & CONFIG_VALUE_LOADED)?1:0; + changed += (opt->flags & CONFIG_VALUE_CHANGED)?1:0; + count++; + } + SECTION_UNLOCK(sect); + + if(!count) continue; + if(only_changed && !changed && !loaded) continue; + + if(!used) + buffer_sprintf(wb, "\n# section '%s' is not used.", string2str(sect->name)); + + buffer_sprintf(wb, "\n[%s]\n", string2str(sect->name)); + + size_t options_added = 0; + bool last_had_comments = false; + SECTION_LOCK(sect); + for(opt = sect->values; opt; opt = opt->next) { + bool unused = used && !(opt->flags & CONFIG_VALUE_USED); + bool migrated = used && (opt->flags & CONFIG_VALUE_MIGRATED); + bool reformatted = used && (opt->flags & CONFIG_VALUE_REFORMATTED); + bool show_default = used && (opt->flags & (CONFIG_VALUE_LOADED|CONFIG_VALUE_CHANGED) && opt->value_default); + + if((unused || migrated || reformatted || show_default)) { + if(options_added) + buffer_strcat(wb, "\n"); + + buffer_sprintf(wb, "\t#| >>> [%s].%s <<<\n", + string2str(sect->name), string2str(opt->name)); + + last_had_comments = true; + } + else if(last_had_comments) { + buffer_strcat(wb, "\n"); + last_had_comments = false; + } + + if(unused) + buffer_sprintf(wb, "\t#| found in the config file, but is not used\n"); + + if(migrated && reformatted) + buffer_sprintf(wb, "\t#| migrated from: [%s].%s = %s\n", + string2str(opt->migrated.section), string2str(opt->migrated.name), + string2str(opt->value_original)); + else { + if (migrated) + buffer_sprintf(wb, "\t#| migrated from: [%s].%s\n", + string2str(opt->migrated.section), string2str(opt->migrated.name)); + + if (reformatted) + buffer_sprintf(wb, "\t#| reformatted from: %s\n", + string2str(opt->value_original)); + } + + if(show_default) + buffer_sprintf(wb, "\t#| datatype: %s, default value: %s\n", + CONFIG_VALUE_TYPES_2str(opt->type), + string2str(opt->value_default)); + + buffer_sprintf(wb, "\t%s%s = %s\n", + ( + !(opt->flags & CONFIG_VALUE_LOADED) && + !(opt->flags & CONFIG_VALUE_CHANGED) && + (opt->flags & CONFIG_VALUE_USED) + ) ? "# " : "", + string2str(opt->name), + string2str(opt->value)); + + options_added++; + } + SECTION_UNLOCK(sect); + } + } + APPCONFIG_UNLOCK(root); + } +} diff --git a/src/libnetdata/config/appconfig_exporters.c b/src/libnetdata/config/appconfig_exporters.c new file mode 100644 index 000000000..1fafb298c --- /dev/null +++ b/src/libnetdata/config/appconfig_exporters.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" + +/* + * @Input: + * Connector / instance to add to an internal structure + * @Return + * The current head of the linked list of connector_instance + * + */ + +_CONNECTOR_INSTANCE *add_connector_instance(struct config_section *connector, struct config_section *instance) +{ + static struct _connector_instance *global_connector_instance = NULL; + struct _connector_instance *local_ci, *local_ci_tmp; + + if (unlikely(!connector)) { + if (unlikely(!instance)) + return global_connector_instance; + + local_ci = global_connector_instance; + while (local_ci) { + local_ci_tmp = local_ci->next; + freez(local_ci); + local_ci = local_ci_tmp; + } + global_connector_instance = NULL; + return NULL; + } + + local_ci = callocz(1, sizeof(struct _connector_instance)); + local_ci->instance = instance; + local_ci->connector = connector; + strncpyz(local_ci->instance_name, string2str(instance->name), CONFIG_MAX_NAME); + strncpyz(local_ci->connector_name, string2str(connector->name), CONFIG_MAX_NAME); + local_ci->next = global_connector_instance; + global_connector_instance = local_ci; + + return global_connector_instance; +} + +int is_valid_connector(char *type, int check_reserved) { + int rc = 1; + + if (unlikely(!type)) + return 0; + + if (!check_reserved) { + if (unlikely(is_valid_connector(type,1))) { + return 0; + } + //if (unlikely(*type == ':') + // return 0; + char *separator = strrchr(type, ':'); + if (likely(separator)) { + *separator = '\0'; + rc = (int)(separator - type); + } else + return 0; + } + // else { + // if (unlikely(is_valid_connector(type,1))) { + // netdata_log_error("Section %s invalid -- reserved name", type); + // return 0; + // } + // } + + if (!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) { + return rc; + } else if (!strcmp(type, "graphite:http") || !strcmp(type, "graphite:https")) { + return rc; + } else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) { + return rc; + } else if (!strcmp(type, "json:http") || !strcmp(type, "json:https")) { + return rc; + } else if (!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) { + return rc; + } else if (!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) { + return rc; + } else if (!strcmp(type, "prometheus_remote_write")) { + return rc; + } else if (!strcmp(type, "prometheus_remote_write:http") || !strcmp(type, "prometheus_remote_write:https")) { + return rc; + } else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) { + return rc; + } else if (!strcmp(type, "pubsub") || !strcmp(type, "pubsub:plaintext")) { + return rc; + } else if (!strcmp(type, "mongodb") || !strcmp(type, "mongodb:plaintext")) { + return rc; + } + + return 0; +} + diff --git a/src/libnetdata/config/appconfig_internals.h b/src/libnetdata/config/appconfig_internals.h new file mode 100644 index 000000000..492e8ce5c --- /dev/null +++ b/src/libnetdata/config/appconfig_internals.h @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_APPCONFIG_INTERNALS_H +#define NETDATA_APPCONFIG_INTERNALS_H + +#include "appconfig.h" + +typedef enum __attribute__((packed)) { + CONFIG_VALUE_TYPE_UNKNOWN = 0, + CONFIG_VALUE_TYPE_TEXT, + CONFIG_VALUE_TYPE_HOSTNAME, + CONFIG_VALUE_TYPE_USERNAME, + CONFIG_VALUE_TYPE_FILENAME, + CONFIG_VALUE_TYPE_PATH, + CONFIG_VALUE_TYPE_SIMPLE_PATTERN, + CONFIG_VALUE_TYPE_URL, + CONFIG_VALUE_TYPE_ENUM, + CONFIG_VALUE_TYPE_BITMAP, + CONFIG_VALUE_TYPE_INTEGER, + CONFIG_VALUE_TYPE_DOUBLE, + CONFIG_VALUE_TYPE_BOOLEAN, + CONFIG_VALUE_TYPE_BOOLEAN_ONDEMAND, + CONFIG_VALUE_TYPE_DURATION_IN_SECS, + CONFIG_VALUE_TYPE_DURATION_IN_MS, + CONFIG_VALUE_TYPE_DURATION_IN_DAYS, + CONFIG_VALUE_TYPE_SIZE_IN_BYTES, + CONFIG_VALUE_TYPE_SIZE_IN_MB, +} CONFIG_VALUE_TYPES; + +typedef enum __attribute__((packed)) { + CONFIG_VALUE_LOADED = (1 << 0), // has been loaded from the config + CONFIG_VALUE_USED = (1 << 1), // has been accessed from the program + CONFIG_VALUE_CHANGED = (1 << 2), // has been changed from the loaded value or the internal default value + CONFIG_VALUE_CHECKED = (1 << 3), // has been checked if the value is different from the default + CONFIG_VALUE_MIGRATED = (1 << 4), // has been migrated from an old config + CONFIG_VALUE_REFORMATTED = (1 << 5), // has been reformatted with the official formatting +} CONFIG_VALUE_FLAGS; + +struct config_option { + avl_t avl_node; // the index entry of this entry - this has to be first! + + CONFIG_VALUE_TYPES type; + CONFIG_VALUE_FLAGS flags; + + STRING *name; + STRING *value; + + STRING *value_original; // the original value of this option (the first value it got, independently on how it got it) + STRING *value_default; // the internal default value of this option (the first value it got, from appconfig_get_XXX()) + + // when we move options around, this is where we keep the original + // section and name (of the first migration) + struct { + STRING *section; + STRING *name; + } migrated; + + struct config_option *prev, *next; // config->mutex protects just this +}; + +struct config_section { + avl_t avl_node; // the index entry of this section - this has to be first! + + STRING *name; + + struct config_option *values; + avl_tree_lock values_index; + + SPINLOCK spinlock; + struct config_section *prev, *next; // global config_mutex protects just this +}; + +// ---------------------------------------------------------------------------- +// locking + +#define APPCONFIG_LOCK(root) spinlock_lock(&((root)->spinlock)) +#define APPCONFIG_UNLOCK(root) spinlock_unlock(&((root)->spinlock)) +#define SECTION_LOCK(sect) spinlock_lock(&((sect)->spinlock)) +#define SECTION_UNLOCK(sect) spinlock_unlock(&((sect)->spinlock)); + +// config sections +void appconfig_section_free(struct config_section *sect); +void appconfig_section_remove_and_delete(struct config *root, struct config_section *sect, bool have_root_lock, bool have_sect_lock); +#define appconfig_section_add(root, cfg) (struct config_section *)avl_insert_lock(&(root)->index, (avl_t *)(cfg)) +#define appconfig_section_del(root, cfg) (struct config_section *)avl_remove_lock(&(root)->index, (avl_t *)(cfg)) +struct config_section *appconfig_section_find(struct config *root, const char *name); +struct config_section *appconfig_section_create(struct config *root, const char *section); + +// config options +void appconfig_option_cleanup(struct config_option *opt); +void appconfig_option_free(struct config_option *opt); +void appconfig_option_remove_and_delete(struct config_section *sect, struct config_option *opt, bool have_sect_lock); +void appconfig_option_remove_and_delete_all(struct config_section *sect, bool have_sect_lock); +int appconfig_option_compare(void *a, void *b); +#define appconfig_option_add(co, cv) (struct config_option *)avl_insert_lock(&((co)->values_index), (avl_t *)(cv)) +#define appconfig_option_del(co, cv) (struct config_option *)avl_remove_lock(&((co)->values_index), (avl_t *)(cv)) +struct config_option *appconfig_option_find(struct config_section *sect, const char *name); +struct config_option *appconfig_option_create(struct config_section *sect, const char *name, const char *value); + +// lookup +int appconfig_get_boolean_by_section(struct config_section *sect, const char *name, int value); + +typedef STRING *(*reformat_t)(STRING *value); +struct config_option *appconfig_get_raw_value_of_option_in_section(struct config_section *sect, const char *option, const char *default_value, CONFIG_VALUE_TYPES type, reformat_t cb); +struct config_option *appconfig_get_raw_value(struct config *root, const char *section, const char *option, const char *default_value, CONFIG_VALUE_TYPES type, reformat_t cb); + +void appconfig_set_raw_value_of_option(struct config_option *opt, const char *value, CONFIG_VALUE_TYPES type); +struct config_option *appconfig_set_raw_value_of_option_in_section(struct config_section *sect, const char *option, const char *value, CONFIG_VALUE_TYPES type); +struct config_option *appconfig_set_raw_value(struct config *root, const char *section, const char *option, const char *value, CONFIG_VALUE_TYPES type); + +// cleanup +void appconfig_section_destroy_non_loaded(struct config *root, const char *section); +void appconfig_section_option_destroy_non_loaded(struct config *root, const char *section, const char *name); + +// exporters +_CONNECTOR_INSTANCE *add_connector_instance(struct config_section *connector, struct config_section *instance); +int is_valid_connector(char *type, int check_reserved); + +#endif //NETDATA_APPCONFIG_INTERNALS_H diff --git a/src/libnetdata/config/appconfig_migrate.c b/src/libnetdata/config/appconfig_migrate.c new file mode 100644 index 000000000..0c21ec06c --- /dev/null +++ b/src/libnetdata/config/appconfig_migrate.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" + +int appconfig_move(struct config *root, const char *section_old, const char *name_old, const char *section_new, const char *name_new) { + struct config_option *opt_old, *opt_new; + int ret = -1; + + netdata_log_debug(D_CONFIG, "request to rename config in section '%s', old name '%s', to section '%s', new name '%s'", section_old, name_old, section_new, name_new); + + struct config_section *sect_old = appconfig_section_find(root, section_old); + if(!sect_old) return ret; + + struct config_section *sect_new = appconfig_section_find(root, section_new); + if(!sect_new) sect_new = appconfig_section_create(root, section_new); + + SECTION_LOCK(sect_old); + if(sect_old != sect_new) + SECTION_LOCK(sect_new); + + opt_old = appconfig_option_find(sect_old, name_old); + if(!opt_old) goto cleanup; + + opt_new = appconfig_option_find(sect_new, name_new); + if(opt_new) goto cleanup; + + if(unlikely(appconfig_option_del(sect_old, opt_old) != opt_old)) + netdata_log_error("INTERNAL ERROR: deletion of config '%s' from section '%s', deleted the wrong config entry.", + string2str(opt_old->name), string2str(sect_old->name)); + + // remember the old position of the item + struct config_option *opt_old_next = (sect_old == sect_new) ? opt_old->next : NULL; + + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(sect_old->values, opt_old, prev, next); + + nd_log(NDLS_DAEMON, NDLP_WARNING, + "CONFIG: option '[%s].%s' has been migrated to '[%s].%s'.", + section_old, name_old, + section_new, name_new); + + if(!opt_old->migrated.name) { + string_freez(opt_old->migrated.section); + opt_old->migrated.section = string_dup(sect_old->name); + opt_old->migrated.name = opt_old->name; + } + else + string_freez(opt_old->name); + + opt_old->name = string_strdupz(name_new); + opt_old->flags |= CONFIG_VALUE_MIGRATED; + + opt_new = opt_old; + + // put in the list, but try to keep the order + if(opt_old_next && sect_old == sect_new) + DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(sect_new->values, opt_old_next, opt_new, prev, next); + else { + // we don't have the old next item (probably a different section?) + // find the last MIGRATED one + struct config_option *t = sect_new->values ? sect_new->values->prev : NULL; + for (; t && t != sect_new->values ; t = t->prev) { + if (t->flags & CONFIG_VALUE_MIGRATED) + break; + } + if (t == sect_new->values) + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(sect_new->values, opt_new, prev, next); + else + DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(sect_new->values, t, opt_new, prev, next); + } + + if(unlikely(appconfig_option_add(sect_new, opt_old) != opt_old)) + netdata_log_error("INTERNAL ERROR: re-indexing of config '%s' in section '%s', already exists.", + string2str(opt_old->name), string2str(sect_new->name)); + + ret = 0; + +cleanup: + if(sect_old != sect_new) + SECTION_UNLOCK(sect_new); + SECTION_UNLOCK(sect_old); + return ret; +} + +int appconfig_move_everywhere(struct config *root, const char *name_old, const char *name_new) { + int ret = -1; + APPCONFIG_LOCK(root); + struct config_section *sect; + for(sect = root->sections; sect; sect = sect->next) { + if(appconfig_move(root, string2str(sect->name), name_old, string2str(sect->name), name_new) == 0) + ret = 0; + } + APPCONFIG_UNLOCK(root); + return ret; +} + diff --git a/src/libnetdata/config/appconfig_options.c b/src/libnetdata/config/appconfig_options.c new file mode 100644 index 000000000..f619d08a6 --- /dev/null +++ b/src/libnetdata/config/appconfig_options.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" + +// ---------------------------------------------------------------------------- +// config options index + +int appconfig_option_compare(void *a, void *b) { + if(((struct config_option *)a)->name < ((struct config_option *)b)->name) return -1; + else if(((struct config_option *)a)->name > ((struct config_option *)b)->name) return 1; + else return string_cmp(((struct config_option *)a)->name, ((struct config_option *)b)->name); +} + +struct config_option *appconfig_option_find(struct config_section *sect, const char *name) { + struct config_option opt_tmp = { + .name = string_strdupz(name), + }; + + struct config_option *rc = (struct config_option *)avl_search_lock(&(sect->values_index), (avl_t *) &opt_tmp); + + appconfig_option_cleanup(&opt_tmp); + return rc; +} + +// ---------------------------------------------------------------------------- +// config options methods + +void appconfig_option_cleanup(struct config_option *opt) { + string_freez(opt->value); + string_freez(opt->name); + string_freez(opt->migrated.section); + string_freez(opt->migrated.name); + string_freez(opt->value_original); + string_freez(opt->value_default); + + opt->value = NULL; + opt->name = NULL; + opt->migrated.section = NULL; + opt->migrated.name = NULL; + opt->value_original = NULL; + opt->value_default = NULL; +} + +void appconfig_option_free(struct config_option *opt) { + appconfig_option_cleanup(opt); + freez(opt); +} + +struct config_option *appconfig_option_create(struct config_section *sect, const char *name, const char *value) { + struct config_option *opt = callocz(1, sizeof(struct config_option)); + opt->name = string_strdupz(name); + opt->value = string_strdupz(value); + opt->value_original = string_dup(opt->value); + + struct config_option *opt_found = appconfig_option_add(sect, opt); + if(opt_found != opt) { + nd_log(NDLS_DAEMON, NDLP_INFO, + "CONFIG: config '%s' in section '%s': already exists - using the existing one.", + string2str(opt->name), string2str(sect->name)); + appconfig_option_free(opt); + return opt_found; + } + + SECTION_LOCK(sect); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(sect->values, opt, prev, next); + SECTION_UNLOCK(sect); + + return opt; +} + +void appconfig_option_remove_and_delete(struct config_section *sect, struct config_option *opt, bool have_sect_lock) { + struct config_option *opt_found = appconfig_option_del(sect, opt); + if(opt_found != opt) { + nd_log(NDLS_DAEMON, NDLP_ERR, + "INTERNAL ERROR: Cannot remove '%s' from section '%s', it was not inserted before.", + string2str(opt->name), string2str(sect->name)); + return; + } + + if(!have_sect_lock) + SECTION_LOCK(sect); + + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(sect->values, opt, prev, next); + + if(!have_sect_lock) + SECTION_UNLOCK(sect); + + appconfig_option_free(opt); +} + +void appconfig_option_remove_and_delete_all(struct config_section *sect, bool have_sect_lock) { + if(!have_sect_lock) + SECTION_LOCK(sect); + + while(sect->values) + appconfig_option_remove_and_delete(sect, sect->values, true); + + if(!have_sect_lock) + SECTION_UNLOCK(sect); +} + +void appconfig_get_raw_value_of_option(struct config_option *opt, const char *default_value, CONFIG_VALUE_TYPES type, reformat_t cb) { + opt->flags |= CONFIG_VALUE_USED; + + if(type != CONFIG_VALUE_TYPE_UNKNOWN) + opt->type = type; + + if((opt->flags & CONFIG_VALUE_LOADED) || (opt->flags & CONFIG_VALUE_CHANGED)) { + // this is a loaded value from the config file + // if it is different from the default, mark it + if(!(opt->flags & CONFIG_VALUE_CHECKED)) { + if(!(opt->flags & CONFIG_VALUE_REFORMATTED) && cb) { + STRING *value_old = opt->value; + opt->value = cb(opt->value); + if(opt->value != value_old) + opt->flags |= CONFIG_VALUE_REFORMATTED; + } + + if(default_value && string_strcmp(opt->value, default_value) != 0) + opt->flags |= CONFIG_VALUE_CHANGED; + + opt->flags |= CONFIG_VALUE_CHECKED; + } + } + + if(!opt->value_default) + opt->value_default = string_strdupz(default_value); +} + +struct config_option *appconfig_get_raw_value_of_option_in_section(struct config_section *sect, const char *option, const char *default_value, CONFIG_VALUE_TYPES type, reformat_t cb) { + // Only calls internal to this file check for a NULL result, and they do not supply a NULL arg. + // External caller should treat NULL as an error case. + struct config_option *opt = appconfig_option_find(sect, option); + if (!opt) { + if (!default_value) return NULL; + opt = appconfig_option_create(sect, option, default_value); + if (!opt) return NULL; + } + + appconfig_get_raw_value_of_option(opt, default_value, type, cb); + return opt; +} + +struct config_option *appconfig_get_raw_value(struct config *root, const char *section, const char *option, const char *default_value, CONFIG_VALUE_TYPES type, reformat_t cb) { + struct config_section *sect = appconfig_section_find(root, section); + if(!sect) { + if(!default_value) return NULL; + sect = appconfig_section_create(root, section); + } + + return appconfig_get_raw_value_of_option_in_section(sect, option, default_value, type, cb); +} + +void appconfig_set_raw_value_of_option(struct config_option *opt, const char *value, CONFIG_VALUE_TYPES type) { + opt->flags |= CONFIG_VALUE_USED; + + if(opt->type == CONFIG_VALUE_TYPE_UNKNOWN) + opt->type = type; + + if(string_strcmp(opt->value, value) != 0) { + opt->flags |= CONFIG_VALUE_CHANGED; + + string_freez(opt->value); + opt->value = string_strdupz(value); + } +} + +struct config_option *appconfig_set_raw_value_of_option_in_section(struct config_section *sect, const char *option, const char *value, CONFIG_VALUE_TYPES type) { + struct config_option *opt = appconfig_option_find(sect, option); + if(!opt) + opt = appconfig_option_create(sect, option, value); + + appconfig_set_raw_value_of_option(opt, value, type); + return opt; +} + +struct config_option *appconfig_set_raw_value(struct config *root, const char *section, const char *option, const char *value, CONFIG_VALUE_TYPES type) { + struct config_section *sect = appconfig_section_find(root, section); + if(!sect) + sect = appconfig_section_create(root, section); + + return appconfig_set_raw_value_of_option_in_section(sect, option, value, type); +} diff --git a/src/libnetdata/config/appconfig_sections.c b/src/libnetdata/config/appconfig_sections.c new file mode 100644 index 000000000..2180803a9 --- /dev/null +++ b/src/libnetdata/config/appconfig_sections.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" + +// ---------------------------------------------------------------------------- +// config sections index + +int appconfig_section_compare(void *a, void *b) { + if(((struct config_section *)a)->name < ((struct config_section *)b)->name) return -1; + else if(((struct config_section *)a)->name > ((struct config_section *)b)->name) return 1; + else return string_cmp(((struct config_section *)a)->name, ((struct config_section *)b)->name); +} + +struct config_section *appconfig_section_find(struct config *root, const char *name) { + struct config_section sect_tmp = { + .name = string_strdupz(name), + }; + + struct config_section *rc = (struct config_section *)avl_search_lock(&root->index, (avl_t *) §_tmp); + string_freez(sect_tmp.name); + return rc; +} + +// ---------------------------------------------------------------------------- +// config section methods + +void appconfig_section_free(struct config_section *sect) { + avl_destroy_lock(§->values_index); + string_freez(sect->name); + freez(sect); +} + +void appconfig_section_remove_and_delete(struct config *root, struct config_section *sect, bool have_root_lock, bool have_sect_lock) { + struct config_section *sect_found = appconfig_section_del(root, sect); + if(sect_found != sect) { + nd_log(NDLS_DAEMON, NDLP_ERR, + "INTERNAL ERROR: Cannot remove section '%s', it was not inserted before.", + string2str(sect->name)); + return; + } + + appconfig_option_remove_and_delete_all(sect, have_sect_lock); + + if(!have_root_lock) + APPCONFIG_LOCK(root); + + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(root->sections, sect, prev, next); + + if(!have_root_lock) + APPCONFIG_UNLOCK(root); + + // if the caller has the section lock, we will unlock it, to cleanup + if(have_sect_lock) + SECTION_UNLOCK(sect); + + appconfig_section_free(sect); +} + +struct config_section *appconfig_section_create(struct config *root, const char *section) { + struct config_section *sect = callocz(1, sizeof(struct config_section)); + sect->name = string_strdupz(section); + spinlock_init(§->spinlock); + + avl_init_lock(§->values_index, appconfig_option_compare); + + struct config_section *sect_found = appconfig_section_add(root, sect); + if(sect_found != sect) { + nd_log(NDLS_DAEMON, NDLP_ERR, + "CONFIG: section '%s', already exists, using existing.", + string2str(sect->name)); + appconfig_section_free(sect); + return sect_found; + } + + APPCONFIG_LOCK(root); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(root->sections, sect, prev, next); + APPCONFIG_UNLOCK(root); + + return sect; +} + + diff --git a/src/libnetdata/config/appconfig_traversal.c b/src/libnetdata/config/appconfig_traversal.c new file mode 100644 index 000000000..f26def2c2 --- /dev/null +++ b/src/libnetdata/config/appconfig_traversal.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "appconfig_internals.h" + +size_t appconfig_foreach_value_in_section(struct config *root, const char *section, appconfig_foreach_value_cb_t cb, void *data) { + size_t used = 0; + struct config_section *co = appconfig_section_find(root, section); + if(co) { + SECTION_LOCK(co); + struct config_option *cv; + for(cv = co->values; cv ; cv = cv->next) { + if(cb(data, string2str(cv->name), string2str(cv->value))) { + cv->flags |= CONFIG_VALUE_USED; + used++; + } + } + SECTION_UNLOCK(co); + } + + return used; +} diff --git a/src/libnetdata/config/dyncfg.c b/src/libnetdata/config/dyncfg.c index 244864c65..81b050f89 100644 --- a/src/libnetdata/config/dyncfg.c +++ b/src/libnetdata/config/dyncfg.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "../../libnetdata/libnetdata.h" +#include "../libnetdata.h" // ---------------------------------------------------------------------------- @@ -211,7 +211,7 @@ bool dyncfg_is_valid_id(const char *id) { return true; } -static inline bool is_forbidden_char(char c) { +static inline bool is_forbidden_filename_char(char c) { if(isspace((uint8_t)c) || !isprint((uint8_t)c)) return true; @@ -239,7 +239,7 @@ char *dyncfg_escape_id_for_filename(const char *id) { char *dest = escaped; while (*src) { - if (is_forbidden_char(*src)) { + if (is_forbidden_filename_char(*src)) { sprintf(dest, "%%%02X", (unsigned char)*src); dest += 3; } else { @@ -277,7 +277,7 @@ int dyncfg_node_find_and_call(DICTIONARY *dyncfg_nodes, const char *transaction, memcpy(buf, function, sizeof(buf)); char *words[MAX_FUNCTION_PARAMETERS]; // an array of pointers for the words in this line - size_t num_words = quoted_strings_splitter_pluginsd(buf, words, MAX_FUNCTION_PARAMETERS); + size_t num_words = quoted_strings_splitter_whitespace(buf, words, MAX_FUNCTION_PARAMETERS); const char *id = get_word(words, num_words, 1); const char *action = get_word(words, num_words, 2); diff --git a/src/libnetdata/datetime/README.md b/src/libnetdata/datetime/README.md index 303ba8bf2..e10848f20 100644 --- a/src/libnetdata/datetime/README.md +++ b/src/libnetdata/datetime/README.md @@ -1,11 +1,3 @@ -<!-- -title: "Datetime" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/datetime/README.md -sidebar_label: "Datetime" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Datetime Formatting dates and timestamps. diff --git a/src/libnetdata/dictionary/dictionary-hashtable.h b/src/libnetdata/dictionary/dictionary-hashtable.h index 865f0b360..14c81cfcc 100644 --- a/src/libnetdata/dictionary/dictionary-hashtable.h +++ b/src/libnetdata/dictionary/dictionary-hashtable.h @@ -8,96 +8,96 @@ // ---------------------------------------------------------------------------- // hashtable operations with simple hashtable -static inline bool compare_keys(void *key1, void *key2) { - const char *k1 = key1; - const char *k2 = key2; - return strcmp(k1, k2) == 0; -} - -static inline void *item_to_key(DICTIONARY_ITEM *item) { - return (void *)item_get_name(item); -} - -#define SIMPLE_HASHTABLE_VALUE_TYPE DICTIONARY_ITEM -#define SIMPLE_HASHTABLE_NAME _DICTIONARY -#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION item_to_key -#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION compare_keys -#include "..//simple_hashtable.h" - -static inline size_t hashtable_init_hashtable(DICTIONARY *dict) { - SIMPLE_HASHTABLE_DICTIONARY *ht = callocz(1, sizeof(*ht)); - simple_hashtable_init_DICTIONARY(ht, 4); - dict->index.JudyHSArray = ht; - return 0; -} - -static inline size_t hashtable_destroy_hashtable(DICTIONARY *dict) { - SIMPLE_HASHTABLE_DICTIONARY *ht = dict->index.JudyHSArray; - if(unlikely(!ht)) return 0; - - size_t mem = sizeof(*ht) + ht->size * sizeof(SIMPLE_HASHTABLE_SLOT_DICTIONARY); - simple_hashtable_destroy_DICTIONARY(ht); - freez(ht); - dict->index.JudyHSArray = NULL; - - return mem; -} - -static inline void *hashtable_insert_hashtable(DICTIONARY *dict, const char *name, size_t name_len) { - SIMPLE_HASHTABLE_DICTIONARY *ht = dict->index.JudyHSArray; - - char key[name_len+1]; - memcpy(key, name, name_len); - key[name_len] = '\0'; - - XXH64_hash_t hash = XXH3_64bits(name, name_len); - SIMPLE_HASHTABLE_SLOT_DICTIONARY *sl = simple_hashtable_get_slot_DICTIONARY(ht, hash, key, true); - sl->hash = hash; // we will need it in insert later - it is ok to overwrite - it is the same already - return sl; -} - -static inline DICTIONARY_ITEM *hashtable_insert_handle_to_item_hashtable(DICTIONARY *dict, void *handle) { - (void)dict; - SIMPLE_HASHTABLE_SLOT_DICTIONARY *sl = handle; - DICTIONARY_ITEM *item = SIMPLE_HASHTABLE_SLOT_DATA(sl); - return item; -} - -static inline void hashtable_set_item_hashtable(DICTIONARY *dict, void *handle, DICTIONARY_ITEM *item) { - SIMPLE_HASHTABLE_DICTIONARY *ht = dict->index.JudyHSArray; - SIMPLE_HASHTABLE_SLOT_DICTIONARY *sl = handle; - simple_hashtable_set_slot_DICTIONARY(ht, sl, sl->hash, item); -} - -static inline int hashtable_delete_hashtable(DICTIONARY *dict, const char *name, size_t name_len, DICTIONARY_ITEM *item_to_delete) { - (void)item_to_delete; - SIMPLE_HASHTABLE_DICTIONARY *ht = dict->index.JudyHSArray; - - char key[name_len+1]; - memcpy(key, name, name_len); - key[name_len] = '\0'; - - XXH64_hash_t hash = XXH3_64bits(name, name_len); - SIMPLE_HASHTABLE_SLOT_DICTIONARY *sl = simple_hashtable_get_slot_DICTIONARY(ht, hash, key, false); - DICTIONARY_ITEM *item = SIMPLE_HASHTABLE_SLOT_DATA(sl); - if(!item) return 0; // return not-found - - simple_hashtable_del_slot_DICTIONARY(ht, sl); - return 1; // return deleted -} - -static inline DICTIONARY_ITEM *hashtable_get_hashtable(DICTIONARY *dict, const char *name, size_t name_len) { - SIMPLE_HASHTABLE_DICTIONARY *ht = dict->index.JudyHSArray; - if(unlikely(!ht)) return NULL; - - char key[name_len+1]; - memcpy(key, name, name_len); - key[name_len] = '\0'; - - XXH64_hash_t hash = XXH3_64bits(name, name_len); - SIMPLE_HASHTABLE_SLOT_DICTIONARY *sl = simple_hashtable_get_slot_DICTIONARY(ht, hash, key, true); - return SIMPLE_HASHTABLE_SLOT_DATA(sl); -} +//static inline bool compare_keys(void *key1, void *key2) { +// const char *k1 = key1; +// const char *k2 = key2; +// return strcmp(k1, k2) == 0; +//} +// +//static inline void *item_to_key(DICTIONARY_ITEM *item) { +// return (void *)item_get_name(item); +//} +// +//#define SIMPLE_HASHTABLE_VALUE_TYPE DICTIONARY_ITEM +//#define SIMPLE_HASHTABLE_NAME _DICTIONARY +//#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION item_to_key +//#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION compare_keys +//#include "..//simple_hashtable.h" + +//static inline size_t hashtable_init_hashtable(DICTIONARY *dict) { +// SIMPLE_HASHTABLE_DICTIONARY *ht = callocz(1, sizeof(*ht)); +// simple_hashtable_init_DICTIONARY(ht, 4); +// dict->index.JudyHSArray = ht; +// return 0; +//} +// +//static inline size_t hashtable_destroy_hashtable(DICTIONARY *dict) { +// SIMPLE_HASHTABLE_DICTIONARY *ht = dict->index.JudyHSArray; +// if(unlikely(!ht)) return 0; +// +// size_t mem = sizeof(*ht) + ht->size * sizeof(SIMPLE_HASHTABLE_SLOT_DICTIONARY); +// simple_hashtable_destroy_DICTIONARY(ht); +// freez(ht); +// dict->index.JudyHSArray = NULL; +// +// return mem; +//} +// +//static inline void *hashtable_insert_hashtable(DICTIONARY *dict, const char *name, size_t name_len) { +// SIMPLE_HASHTABLE_DICTIONARY *ht = dict->index.JudyHSArray; +// +// char key[name_len+1]; +// memcpy(key, name, name_len); +// key[name_len] = '\0'; +// +// XXH64_hash_t hash = XXH3_64bits(name, name_len); +// SIMPLE_HASHTABLE_SLOT_DICTIONARY *sl = simple_hashtable_get_slot_DICTIONARY(ht, hash, key, true); +// sl->hash = hash; // we will need it in insert later - it is ok to overwrite - it is the same already +// return sl; +//} +// +//static inline DICTIONARY_ITEM *hashtable_insert_handle_to_item_hashtable(DICTIONARY *dict, void *handle) { +// (void)dict; +// SIMPLE_HASHTABLE_SLOT_DICTIONARY *sl = handle; +// DICTIONARY_ITEM *item = SIMPLE_HASHTABLE_SLOT_DATA(sl); +// return item; +//} +// +//static inline void hashtable_set_item_hashtable(DICTIONARY *dict, void *handle, DICTIONARY_ITEM *item) { +// SIMPLE_HASHTABLE_DICTIONARY *ht = dict->index.JudyHSArray; +// SIMPLE_HASHTABLE_SLOT_DICTIONARY *sl = handle; +// simple_hashtable_set_slot_DICTIONARY(ht, sl, sl->hash, item); +//} +// +//static inline int hashtable_delete_hashtable(DICTIONARY *dict, const char *name, size_t name_len, DICTIONARY_ITEM *item_to_delete) { +// (void)item_to_delete; +// SIMPLE_HASHTABLE_DICTIONARY *ht = dict->index.JudyHSArray; +// +// char key[name_len+1]; +// memcpy(key, name, name_len); +// key[name_len] = '\0'; +// +// XXH64_hash_t hash = XXH3_64bits(name, name_len); +// SIMPLE_HASHTABLE_SLOT_DICTIONARY *sl = simple_hashtable_get_slot_DICTIONARY(ht, hash, key, false); +// DICTIONARY_ITEM *item = SIMPLE_HASHTABLE_SLOT_DATA(sl); +// if(!item) return 0; // return not-found +// +// simple_hashtable_del_slot_DICTIONARY(ht, sl); +// return 1; // return deleted +//} +// +//static inline DICTIONARY_ITEM *hashtable_get_hashtable(DICTIONARY *dict, const char *name, size_t name_len) { +// SIMPLE_HASHTABLE_DICTIONARY *ht = dict->index.JudyHSArray; +// if(unlikely(!ht)) return NULL; +// +// char key[name_len+1]; +// memcpy(key, name, name_len); +// key[name_len] = '\0'; +// +// XXH64_hash_t hash = XXH3_64bits(name, name_len); +// SIMPLE_HASHTABLE_SLOT_DICTIONARY *sl = simple_hashtable_get_slot_DICTIONARY(ht, hash, key, true); +// return SIMPLE_HASHTABLE_SLOT_DATA(sl); +//} // ---------------------------------------------------------------------------- // hashtable operations with Judy @@ -201,40 +201,44 @@ static inline DICTIONARY_ITEM *hashtable_get_judy(DICTIONARY *dict, const char * // select the right hashtable static inline size_t hashtable_init_unsafe(DICTIONARY *dict) { - if(dict->options & DICT_OPTION_INDEX_JUDY) - return hashtable_init_judy(dict); - else - return hashtable_init_hashtable(dict); + return hashtable_init_judy(dict); +// if(dict->options & DICT_OPTION_INDEX_JUDY) +// return hashtable_init_judy(dict); +// else +// return hashtable_init_hashtable(dict); } static inline size_t hashtable_destroy_unsafe(DICTIONARY *dict) { pointer_destroy_index(dict); - if(dict->options & DICT_OPTION_INDEX_JUDY) - return hashtable_destroy_judy(dict); - else - return hashtable_destroy_hashtable(dict); +// if(dict->options & DICT_OPTION_INDEX_JUDY) + return hashtable_destroy_judy(dict); +// else +// return hashtable_destroy_hashtable(dict); } static inline void *hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { - if(dict->options & DICT_OPTION_INDEX_JUDY) - return hashtable_insert_judy(dict, name, name_len); - else - return hashtable_insert_hashtable(dict, name, name_len); + return hashtable_insert_judy(dict, name, name_len); +// if(dict->options & DICT_OPTION_INDEX_JUDY) +// return hashtable_insert_judy(dict, name, name_len); +// else +// return hashtable_insert_hashtable(dict, name, name_len); } static inline DICTIONARY_ITEM *hashtable_insert_handle_to_item_unsafe(DICTIONARY *dict, void *handle) { - if(dict->options & DICT_OPTION_INDEX_JUDY) - return hashtable_insert_handle_to_item_judy(dict, handle); - else - return hashtable_insert_handle_to_item_hashtable(dict, handle); + return hashtable_insert_handle_to_item_judy(dict, handle); +// if(dict->options & DICT_OPTION_INDEX_JUDY) +// return hashtable_insert_handle_to_item_judy(dict, handle); +// else +// return hashtable_insert_handle_to_item_hashtable(dict, handle); } static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, DICTIONARY_ITEM *item) { - if(dict->options & DICT_OPTION_INDEX_JUDY) - return hashtable_delete_judy(dict, name, name_len, item); - else - return hashtable_delete_hashtable(dict, name, name_len, item); + return hashtable_delete_judy(dict, name, name_len, item); +// if(dict->options & DICT_OPTION_INDEX_JUDY) +// return hashtable_delete_judy(dict, name, name_len, item); +// else +// return hashtable_delete_hashtable(dict, name, name_len, item); } static inline DICTIONARY_ITEM *hashtable_get_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { @@ -242,10 +246,11 @@ static inline DICTIONARY_ITEM *hashtable_get_unsafe(DICTIONARY *dict, const char DICTIONARY_ITEM *item; - if(dict->options & DICT_OPTION_INDEX_JUDY) - item = hashtable_get_judy(dict, name, name_len); - else - item = hashtable_get_hashtable(dict, name, name_len); + item = hashtable_get_judy(dict, name, name_len); +// if(dict->options & DICT_OPTION_INDEX_JUDY) +// item = hashtable_get_judy(dict, name, name_len); +// else +// item = hashtable_get_hashtable(dict, name, name_len); if(item) pointer_check(dict, item); @@ -254,10 +259,11 @@ static inline DICTIONARY_ITEM *hashtable_get_unsafe(DICTIONARY *dict, const char } static inline void hashtable_set_item_unsafe(DICTIONARY *dict, void *handle, DICTIONARY_ITEM *item) { - if(dict->options & DICT_OPTION_INDEX_JUDY) - hashtable_set_item_judy(dict, handle, item); - else - hashtable_set_item_hashtable(dict, handle, item); + hashtable_set_item_judy(dict, handle, item); +// if(dict->options & DICT_OPTION_INDEX_JUDY) +// hashtable_set_item_judy(dict, handle, item); +// else +// hashtable_set_item_hashtable(dict, handle, item); } #endif //NETDATA_DICTIONARY_HASHTABLE_H diff --git a/src/libnetdata/dictionary/dictionary.c b/src/libnetdata/dictionary/dictionary.c index 9d50ed62c..ebe67269a 100644 --- a/src/libnetdata/dictionary/dictionary.c +++ b/src/libnetdata/dictionary/dictionary.c @@ -318,10 +318,11 @@ static void dictionary_queue_for_destruction(DICTIONARY *dict) { } void cleanup_destroyed_dictionaries(void) { - if(!dictionaries_waiting_to_be_destroyed) - return; - netdata_mutex_lock(&dictionaries_waiting_to_be_destroyed_mutex); + if (!dictionaries_waiting_to_be_destroyed) { + netdata_mutex_unlock(&dictionaries_waiting_to_be_destroyed_mutex); + return; + } DICTIONARY *dict, *last = NULL, *next = NULL; for(dict = dictionaries_waiting_to_be_destroyed; dict ; dict = next) { @@ -497,8 +498,8 @@ static DICTIONARY *dictionary_create_internal(DICT_OPTIONS options, struct dicti else dict->value_aral = NULL; - if(!(dict->options & (DICT_OPTION_INDEX_JUDY|DICT_OPTION_INDEX_HASHTABLE))) - dict->options |= DICT_OPTION_INDEX_JUDY; +// if(!(dict->options & (DICT_OPTION_INDEX_JUDY|DICT_OPTION_INDEX_HASHTABLE))) + dict->options |= DICT_OPTION_INDEX_JUDY; size_t dict_size = 0; dict_size += sizeof(DICTIONARY); diff --git a/src/libnetdata/dictionary/dictionary.h b/src/libnetdata/dictionary/dictionary.h index 231fbfebd..3d041018d 100644 --- a/src/libnetdata/dictionary/dictionary.h +++ b/src/libnetdata/dictionary/dictionary.h @@ -59,7 +59,7 @@ typedef enum __attribute__((packed)) dictionary_options { DICT_OPTION_ADD_IN_FRONT = (1 << 4), // add dictionary items at the front of the linked list (default: at the end) DICT_OPTION_FIXED_SIZE = (1 << 5), // the items of the dictionary have a fixed size DICT_OPTION_INDEX_JUDY = (1 << 6), // the default, if no other indexing is set - DICT_OPTION_INDEX_HASHTABLE = (1 << 7), // use SIMPLE_HASHTABLE for indexing +// DICT_OPTION_INDEX_HASHTABLE = (1 << 7), // use SIMPLE_HASHTABLE for indexing } DICT_OPTIONS; struct dictionary_stats { @@ -299,7 +299,8 @@ typedef DICTFE_CONST struct dictionary_foreach { #define dfe_start_rw(dict, value, mode) \ do { \ - DICTFE value ## _dfe = {}; \ + /* automatically cleanup DFE, to allow using return from within the loop */ \ + DICTFE _cleanup_(dictionary_foreach_done) value ## _dfe = {}; \ (void)(value); /* needed to avoid warning when looping without using this */ \ for((value) = dictionary_foreach_start_rw(&value ## _dfe, (dict), (mode)); \ (value ## _dfe.item) || (value) ; \ @@ -308,7 +309,6 @@ typedef DICTFE_CONST struct dictionary_foreach { #define dfe_done(value) \ } \ - dictionary_foreach_done(&value ## _dfe); \ } while(0) #define dfe_unlock(value) dictionary_foreach_unlock(&value ## _dfe) diff --git a/src/libnetdata/ebpf/README.md b/src/libnetdata/ebpf/README.md index 8d9edb076..17fc13046 100644 --- a/src/libnetdata/ebpf/README.md +++ b/src/libnetdata/ebpf/README.md @@ -1,12 +1,3 @@ -<!-- -title: "eBPF" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/ebpf/README.md -sidebar_label: "eBPF" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # eBPF library Netdata's eBPF library supports the [eBPF collector](/src/collectors/ebpf.plugin/README.md). diff --git a/src/libnetdata/ebpf/ebpf.c b/src/libnetdata/ebpf/ebpf.c index 4e7c85943..27042a794 100644 --- a/src/libnetdata/ebpf/ebpf.c +++ b/src/libnetdata/ebpf/ebpf.c @@ -1014,7 +1014,7 @@ int ebpf_load_config(struct config *config, char *filename) } -static netdata_run_mode_t ebpf_select_mode(char *mode) +static netdata_run_mode_t ebpf_select_mode(const char *mode) { if (!strcasecmp(mode,EBPF_CFG_LOAD_MODE_RETURN )) return MODE_RETURN; @@ -1041,7 +1041,7 @@ static void ebpf_select_mode_string(char *output, size_t len, netdata_run_mode_t * * @return It returns the value to be used. */ -netdata_ebpf_load_mode_t epbf_convert_string_to_load_mode(char *str) +netdata_ebpf_load_mode_t epbf_convert_string_to_load_mode(const char *str) { if (!strcasecmp(str, EBPF_CFG_CORE_PROGRAM)) return EBPF_LOAD_CORE; @@ -1094,7 +1094,7 @@ static char *ebpf_convert_collect_pid_to_string(netdata_apps_level_t level) * * @return it returns the level associated to the string or default when it is a wrong value */ -netdata_apps_level_t ebpf_convert_string_to_apps_level(char *str) +netdata_apps_level_t ebpf_convert_string_to_apps_level(const char *str) { if (!strcasecmp(str, EBPF_CFG_PID_REAL_PARENT)) return NETDATA_APPS_LEVEL_REAL_PARENT; @@ -1114,7 +1114,7 @@ netdata_apps_level_t ebpf_convert_string_to_apps_level(char *str) * @param str value read from configuration file. * @param lmode load mode used by collector. */ -netdata_ebpf_program_loaded_t ebpf_convert_core_type(char *str, netdata_run_mode_t lmode) +netdata_ebpf_program_loaded_t ebpf_convert_core_type(const char *str, netdata_run_mode_t lmode) { if (!strcasecmp(str, EBPF_CFG_ATTACH_TRACEPOINT)) return EBPF_LOAD_TRACEPOINT; @@ -1174,7 +1174,7 @@ struct btf *ebpf_parse_btf_file(const char *filename) * @param path is the fullpath * @param filename is the file inside BTF path. */ -struct btf *ebpf_load_btf_file(char *path, char *filename) +struct btf *ebpf_load_btf_file(const char *path, const char *filename) { char fullpath[PATH_MAX + 1]; snprintfz(fullpath, PATH_MAX, "%s/%s", path, filename); @@ -1299,7 +1299,7 @@ void ebpf_update_module_using_config(ebpf_module_t *modules, netdata_ebpf_load_m { char default_value[EBPF_MAX_MODE_LENGTH + 1]; ebpf_select_mode_string(default_value, EBPF_MAX_MODE_LENGTH, modules->mode); - char *load_mode = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_LOAD_MODE, default_value); + const char *load_mode = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_LOAD_MODE, default_value); modules->mode = ebpf_select_mode(load_mode); modules->update_every = (int)appconfig_get_number(modules->cfg, EBPF_GLOBAL_SECTION, @@ -1318,17 +1318,17 @@ void ebpf_update_module_using_config(ebpf_module_t *modules, netdata_ebpf_load_m EBPF_CFG_LIFETIME, EBPF_DEFAULT_LIFETIME); char *value = ebpf_convert_load_mode_to_string(modules->load & NETDATA_EBPF_LOAD_METHODS); - char *type_format = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_TYPE_FORMAT, value); + const char *type_format = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_TYPE_FORMAT, value); netdata_ebpf_load_mode_t load = epbf_convert_string_to_load_mode(type_format); load = ebpf_select_load_mode(btf_file, load, kver, is_rh); modules->load = origin | load; - char *core_attach = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_CORE_ATTACH, EBPF_CFG_ATTACH_TRAMPOLINE); + const char *core_attach = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_CORE_ATTACH, EBPF_CFG_ATTACH_TRAMPOLINE); netdata_ebpf_program_loaded_t fill_lm = ebpf_convert_core_type(core_attach, modules->mode); ebpf_update_target_with_conf(modules, fill_lm); value = ebpf_convert_collect_pid_to_string(modules->apps_level); - char *collect_pid = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_COLLECT_PID, value); + const char *collect_pid = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_COLLECT_PID, value); modules->apps_level = ebpf_convert_string_to_apps_level(collect_pid); modules->maps_per_core = appconfig_get_boolean(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_MAPS_PER_CORE, diff --git a/src/libnetdata/ebpf/ebpf.h b/src/libnetdata/ebpf/ebpf.h index 1c612ad32..d65dc2205 100644 --- a/src/libnetdata/ebpf/ebpf.h +++ b/src/libnetdata/ebpf/ebpf.h @@ -470,13 +470,13 @@ int ebpf_disable_tracing_values(char *subsys, char *eventname); // BTF helpers #define NETDATA_EBPF_MAX_SYSCALL_LENGTH 255 -netdata_ebpf_load_mode_t epbf_convert_string_to_load_mode(char *str); -netdata_ebpf_program_loaded_t ebpf_convert_core_type(char *str, netdata_run_mode_t lmode); +netdata_ebpf_load_mode_t epbf_convert_string_to_load_mode(const char *str); +netdata_ebpf_program_loaded_t ebpf_convert_core_type(const char *str, netdata_run_mode_t lmode); void ebpf_select_host_prefix(char *output, size_t length, char *syscall, int kver); #ifdef LIBBPF_MAJOR_VERSION void ebpf_adjust_thread_load(ebpf_module_t *mod, struct btf *file); struct btf *ebpf_parse_btf_file(const char *filename); -struct btf *ebpf_load_btf_file(char *path, char *filename); +struct btf *ebpf_load_btf_file(const char *path, const char *filename); int ebpf_is_function_inside_btf(struct btf *file, char *function); void ebpf_update_map_type(struct bpf_map *map, ebpf_local_maps_t *w); void ebpf_define_map_type(ebpf_local_maps_t *maps, int maps_per_core, int kver); @@ -492,4 +492,13 @@ void ebpf_send_data_aral_chart(ARAL *memory, ebpf_module_t *em); int ebpf_can_plugin_load_code(int kver, char *plugin_name); int ebpf_adjust_memory_limit(); +#ifdef LIBBPF_MAJOR_VERSION +static inline int netdata_silent_libbpf_vfprintf(enum libbpf_print_level level __maybe_unused, + const char *format __maybe_unused, + va_list args __maybe_unused) +{ + return 0; +} +#endif + #endif /* NETDATA_EBPF_H */ diff --git a/src/libnetdata/eval/eval.c b/src/libnetdata/eval/eval.c index bacac9c17..7c587fa6d 100644 --- a/src/libnetdata/eval/eval.c +++ b/src/libnetdata/eval/eval.c @@ -387,29 +387,63 @@ static inline void skip_spaces(const char **string) { *string = s; } -// what character can appear just after an operator keyword -// like NOT AND OR ? -static inline int isoperatorterm_word(const char s) { - if(isspace(s) || s == '(' || s == '$' || s == '!' || s == '-' || s == '+' || isdigit(s) || !s) - return 1; - - return 0; +//static inline int old_isoperatorterm_word(const char s) { +// if (isspace(s) || s == '(' || s == '$' || s == '!' || s == '-' || s == '+' || isdigit(s) || !s) +// return 1; +// return 0; +//} +// +//static inline int old_isoperatorterm_symbol(const char s) { +// if (old_isoperatorterm_word(s) || isalpha(s)) +// return 1; +// return 0; +//} +// +//// return 1 if the character should never appear in a variable +//static inline int old_isvariableterm(const char s) { +// if (isalnum(s) || s == '.' || s == '_') +// return 0; +// return 1; +//} + +static inline bool is_operator_first_symbol_or_space(const char s) { + return ( + isspace((uint8_t)s) || !s || + s == '&' || s == '|' || s == '!' || s == '>' || s == '<' || + s == '=' || s == '+' || s == '-' || s == '*' || s == '/' || s == '?'); +} + +// what character can appear just after the operators: NOT, AND, OR +static inline bool is_valid_after_operator_word(const char s) { + bool rc = isspace((uint8_t)s) || s == '(' || s == '$' || s == '!' || s == '-' || s == '+' || isdigit((uint8_t)s) || !s; +// bool old = old_isoperatorterm_word(s); +// if(rc != old) { +// int x = 0; +// x++; +// } + return rc; } // what character can appear just after an operator symbol? -static inline int isoperatorterm_symbol(const char s) { - if(isoperatorterm_word(s) || isalpha(s)) - return 1; - - return 0; -} - -// return 1 if the character should never appear in a variable -static inline int isvariableterm(const char s) { - if(isalnum(s) || s == '.' || s == '_') - return 0; - - return 1; +static inline bool is_valid_after_operator_symbol(const char s) { + bool rc = is_valid_after_operator_word(s) || is_operator_first_symbol_or_space(s); +// bool old = old_isoperatorterm_symbol(s); +// if(rc != old) { +// int x = 0; +// x++; +// } + return rc; +} + +// return true if the character may appear in a variable name +static inline bool is_valid_variable_character(const char s) { + bool rc = !is_operator_first_symbol_or_space(s) && s != ')' && s != '}'; +// bool old = !old_isvariableterm(s); +// if(rc != old) { +// int x = 0; +// x++; +// } + return rc; } // ---------------------------------------------------------------------------- @@ -419,13 +453,14 @@ static inline int parse_and(const char **string) { const char *s = *string; // AND - if((s[0] == 'A' || s[0] == 'a') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'D' || s[2] == 'd') && isoperatorterm_word(s[3])) { + if((s[0] == 'A' || s[0] == 'a') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'D' || s[2] == 'd') && + is_valid_after_operator_word(s[3])) { *string = &s[4]; return 1; } // && - if(s[0] == '&' && s[1] == '&' && isoperatorterm_symbol(s[2])) { + if(s[0] == '&' && s[1] == '&' && is_valid_after_operator_symbol(s[2])) { *string = &s[2]; return 1; } @@ -437,13 +472,13 @@ static inline int parse_or(const char **string) { const char *s = *string; // OR - if((s[0] == 'O' || s[0] == 'o') && (s[1] == 'R' || s[1] == 'r') && isoperatorterm_word(s[2])) { + if((s[0] == 'O' || s[0] == 'o') && (s[1] == 'R' || s[1] == 'r') && is_valid_after_operator_word(s[2])) { *string = &s[3]; return 1; } // || - if(s[0] == '|' && s[1] == '|' && isoperatorterm_symbol(s[2])) { + if(s[0] == '|' && s[1] == '|' && is_valid_after_operator_symbol(s[2])) { *string = &s[2]; return 1; } @@ -455,7 +490,7 @@ static inline int parse_greater_than_or_equal(const char **string) { const char *s = *string; // >= - if(s[0] == '>' && s[1] == '=' && isoperatorterm_symbol(s[2])) { + if(s[0] == '>' && s[1] == '=' && is_valid_after_operator_symbol(s[2])) { *string = &s[2]; return 1; } @@ -467,7 +502,7 @@ static inline int parse_less_than_or_equal(const char **string) { const char *s = *string; // <= - if (s[0] == '<' && s[1] == '=' && isoperatorterm_symbol(s[2])) { + if (s[0] == '<' && s[1] == '=' && is_valid_after_operator_symbol(s[2])) { *string = &s[2]; return 1; } @@ -479,7 +514,7 @@ static inline int parse_greater(const char **string) { const char *s = *string; // > - if(s[0] == '>' && isoperatorterm_symbol(s[1])) { + if(s[0] == '>' && is_valid_after_operator_symbol(s[1])) { *string = &s[1]; return 1; } @@ -491,7 +526,7 @@ static inline int parse_less(const char **string) { const char *s = *string; // < - if(s[0] == '<' && isoperatorterm_symbol(s[1])) { + if(s[0] == '<' && is_valid_after_operator_symbol(s[1])) { *string = &s[1]; return 1; } @@ -503,13 +538,13 @@ static inline int parse_equal(const char **string) { const char *s = *string; // == - if(s[0] == '=' && s[1] == '=' && isoperatorterm_symbol(s[2])) { + if(s[0] == '=' && s[1] == '=' && is_valid_after_operator_symbol(s[2])) { *string = &s[2]; return 1; } // = - if(s[0] == '=' && isoperatorterm_symbol(s[1])) { + if(s[0] == '=' && is_valid_after_operator_symbol(s[1])) { *string = &s[1]; return 1; } @@ -521,13 +556,13 @@ static inline int parse_not_equal(const char **string) { const char *s = *string; // != - if(s[0] == '!' && s[1] == '=' && isoperatorterm_symbol(s[2])) { + if(s[0] == '!' && s[1] == '=' && is_valid_after_operator_symbol(s[2])) { *string = &s[2]; return 1; } // <> - if(s[0] == '<' && s[1] == '>' && isoperatorterm_symbol(s[2])) { + if(s[0] == '<' && s[1] == '>' && is_valid_after_operator_symbol(s[2])) { *string = &s[2]; } @@ -538,7 +573,8 @@ static inline int parse_not(const char **string) { const char *s = *string; // NOT - if((s[0] == 'N' || s[0] == 'n') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'T' || s[2] == 't') && isoperatorterm_word(s[3])) { + if((s[0] == 'N' || s[0] == 'n') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'T' || s[2] == 't') && + is_valid_after_operator_word(s[3])) { *string = &s[3]; return 1; } @@ -555,7 +591,7 @@ static inline int parse_multiply(const char **string) { const char *s = *string; // * - if(s[0] == '*' && isoperatorterm_symbol(s[1])) { + if(s[0] == '*' && is_valid_after_operator_symbol(s[1])) { *string = &s[1]; return 1; } @@ -567,7 +603,7 @@ static inline int parse_divide(const char **string) { const char *s = *string; // / - if(s[0] == '/' && isoperatorterm_symbol(s[1])) { + if(s[0] == '/' && is_valid_after_operator_symbol(s[1])) { *string = &s[1]; return 1; } @@ -579,7 +615,7 @@ static inline int parse_minus(const char **string) { const char *s = *string; // - - if(s[0] == '-' && isoperatorterm_symbol(s[1])) { + if(s[0] == '-' && is_valid_after_operator_symbol(s[1])) { *string = &s[1]; return 1; } @@ -591,7 +627,7 @@ static inline int parse_plus(const char **string) { const char *s = *string; // + - if(s[0] == '+' && isoperatorterm_symbol(s[1])) { + if(s[0] == '+' && is_valid_after_operator_symbol(s[1])) { *string = &s[1]; return 1; } @@ -646,7 +682,7 @@ static inline int parse_variable(const char **string, char *buffer, size_t len) else { // $variable_name - while (*s && !isvariableterm(*s) && i < len) + while (*s && is_valid_variable_character(*s) && i < len) buffer[i++] = *s++; } @@ -1219,7 +1255,7 @@ void expression_hardcode_variable(EVAL_EXPRESSION *expression, STRING *variable, } if (s) { - if (s == s1 && (isalnum((uint8_t)s[len]) || s[len] == '_')) { + if (s == s1 && is_valid_variable_character(s[len])) { // Move past the variable if it's part of a larger word. source_ptr = s + len; continue; diff --git a/src/libnetdata/facets/facets.c b/src/libnetdata/facets/facets.c index 3c746cbc3..230e03de5 100644 --- a/src/libnetdata/facets/facets.c +++ b/src/libnetdata/facets/facets.c @@ -102,25 +102,25 @@ static inline bool is_valid_string_hash(const char *s) { // hashtable for FACET_VALUE // cleanup hashtable defines -#include "../../libnetdata/simple_hashtable_undef.h" +#include "../simple_hashtable/simple_hashtable_undef.h" struct facet_value; // #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_value #define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_value #define SIMPLE_HASHTABLE_NAME _VALUE -#include "../simple_hashtable.h" +#include "../simple_hashtable/simple_hashtable.h" // ---------------------------------------------------------------------------- // hashtable for FACET_KEY // cleanup hashtable defines -#include "../../libnetdata/simple_hashtable_undef.h" +#include "../simple_hashtable/simple_hashtable_undef.h" struct facet_key; // #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_key #define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_key #define SIMPLE_HASHTABLE_NAME _KEY -#include "../simple_hashtable.h" +#include "../simple_hashtable/simple_hashtable.h" // ---------------------------------------------------------------------------- @@ -222,6 +222,7 @@ struct facets { SIMPLE_PATTERN *visible_keys; SIMPLE_PATTERN *excluded_keys; SIMPLE_PATTERN *included_keys; + bool all_keys_included_by_default; FACETS_OPTIONS options; @@ -255,6 +256,7 @@ struct facets { } keys_in_row; FACET_ROW *base; // double linked list of the selected facets rows + FACET_ROW_BIN_DATA bin_data; uint32_t items_to_return; uint32_t max_items_to_return; @@ -328,6 +330,10 @@ struct facets { struct { size_t searches; } fts; + + struct { + size_t bin_data_inflight; + }; } operations; struct { @@ -353,6 +359,27 @@ uint32_t facets_rows(FACETS *facets) { return facets->items_to_return; } +static const char *facets_key_id(FACET_KEY *k) { + if(k->facets->options & FACETS_OPTION_HASH_IDS) + return hash_to_static_string(k->hash); + else + return k->name ? k->name : hash_to_static_string(k->hash); +} + +static const char *facets_key_value_id(FACET_KEY *k, FACET_VALUE *v) { + if(k->facets->options & FACETS_OPTION_HASH_IDS) + return hash_to_static_string(v->hash); + else + return v->name ? v->name : hash_to_static_string(v->hash); +} + +void facets_use_hashes_for_ids(FACETS *facets, bool set) { + if(set) + facets->options |= FACETS_OPTION_HASH_IDS; + else + facets->options &= ~(FACETS_OPTION_HASH_IDS); +} + // ---------------------------------------------------------------------------- static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row); @@ -570,12 +597,12 @@ static inline void FACET_VALUE_ADD_CURRENT_VALUE_TO_INDEX(FACET_KEY *k) { k->facets->operations.values.indexed++; } -static inline void FACET_VALUE_ADD_OR_UPDATE_SELECTED(FACET_KEY *k, FACETS_HASH hash) { +static inline void FACET_VALUE_ADD_OR_UPDATE_SELECTED(FACET_KEY *k, const char *name, FACETS_HASH hash) { FACET_VALUE tv = { .hash = hash, .selected = true, - .name = NULL, - .name_len = 0, + .name = name, + .name_len = name ? strlen(name) : 0, }; FACET_VALUE_ADD_TO_INDEX(k, &tv); } @@ -643,6 +670,35 @@ bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, s return (v && v->selected) ? true : false; } +bool facets_foreach_selected_value_in_key(FACETS *facets, const char *key, size_t key_length, DICTIONARY *used_hashes_registry, facets_foreach_selected_value_in_key_t cb, void *data) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length); + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); + if(!k || k->default_selected_for_values) + return false; + + size_t selected = 0; + for(FACET_VALUE *v = k->values.ll; v ;v = v->next) { + if(!v->selected) continue; + + const char *value = v->name; + if(!value) { + if(used_hashes_registry) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(v->hash, hash_str); + value = dictionary_get(used_hashes_registry, hash_str); + } + + if(!value) + return false; + } + + if(!cb(facets, selected++, k->name, value, data)) + return false; + } + + return selected > 0; +} + void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) { FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length); FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); @@ -691,7 +747,7 @@ static inline FACET_KEY *FACETS_KEY_CREATE(FACETS *facets, FACETS_HASH hash, con k->current_value.b = buffer_create(sizeof(FACET_VALUE_UNSET), NULL); k->default_selected_for_values = true; - if(!(k->options & FACET_KEY_OPTION_REORDER)) + if(unlikely((k->options & (FACET_KEY_OPTION_REORDER | FACET_KEY_OPTION_REORDER_DONE)) == 0)) k->order = facets->order++; if((k->options & FACET_KEY_OPTION_FTS) || (facets->options & FACETS_OPTION_ALL_KEYS_FTS)) @@ -724,10 +780,11 @@ static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH has FACET_KEY *k = SIMPLE_HASHTABLE_SLOT_DATA(slot); facet_key_set_name(k, name, name_length); + k->options |= options; - if(unlikely(k->options & FACET_KEY_OPTION_REORDER)) { + if(unlikely((k->options & (FACET_KEY_OPTION_REORDER | FACET_KEY_OPTION_REORDER_DONE)) == FACET_KEY_OPTION_REORDER)) { k->order = facets->order++; - k->options &= ~FACET_KEY_OPTION_REORDER; + k->options |= FACET_KEY_OPTION_REORDER_DONE; } return k; @@ -1547,7 +1604,7 @@ static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) { } static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) { - bool included = true, excluded = false, never = false; + bool included = facets->all_keys_included_by_default, excluded = false, never = false; if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) { if(k->options & FACET_KEY_OPTION_FACET) { @@ -1591,9 +1648,39 @@ static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) { } // ---------------------------------------------------------------------------- +// bin_data management + +static inline void facets_row_bin_data_cleanup(FACETS *facets, FACET_ROW_BIN_DATA *bin_data) { + if(!bin_data->data) + return; + + bin_data->cleanup_cb(bin_data->data); + *bin_data = FACET_ROW_BIN_DATA_EMPTY; + + fatal_assert(facets->operations.bin_data_inflight > 0); + facets->operations.bin_data_inflight--; +} + +void facets_row_bin_data_set(FACETS *facets, void (*cleanup_cb)(void *data), void *data) { + // in case the caller tries to register bin_data multiple times + // for the same row. + facets_row_bin_data_cleanup(facets, &facets->bin_data); + + // set the new values + facets->bin_data.cleanup_cb = cleanup_cb; + facets->bin_data.data = data; + facets->operations.bin_data_inflight++; +} + +void *facets_row_bin_data_get(FACETS *facets __maybe_unused, FACET_ROW *row) { + return row->bin_data.data; +} + +// ---------------------------------------------------------------------------- FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) { FACETS *facets = callocz(1, sizeof(FACETS)); + facets->all_keys_included_by_default = true; facets->options = options; FACETS_KEYS_INDEX_CREATE(facets); @@ -1616,6 +1703,8 @@ FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const ch } void facets_destroy(FACETS *facets) { + if(!facets) return; + dictionary_destroy(facets->accepted_params); FACETS_KEYS_INDEX_DESTROY(facets); simple_pattern_free(facets->visible_keys); @@ -1629,6 +1718,13 @@ void facets_destroy(FACETS *facets) { facets_row_free(facets, r); } + // in case the caller did not call facets_row_finished() + // on the last row. + facets_row_bin_data_cleanup(facets, &facets->bin_data); + + // make sure we didn't lose any data + fatal_assert(facets->operations.bin_data_inflight == 0); + freez(facets->histogram.chart); freez(facets); } @@ -1691,6 +1787,40 @@ void facets_enable_slice_mode(FACETS *facets) { facets->options |= FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS | FACETS_OPTION_SORT_FACETS_ALPHABETICALLY; } +void facets_reset_and_disable_all_facets(FACETS *facets) { + facets->all_keys_included_by_default = false; + + simple_pattern_free(facets->included_keys); + facets->included_keys = NULL; + +// We need this, because the exclusions are good for controlling which key can become a facet. +// The excluded ones are not offered for facets at all. +// simple_pattern_free(facets->excluded_keys); +// facets->excluded_keys = NULL; + + simple_pattern_free(facets->visible_keys); + facets->visible_keys = NULL; + + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + k->options |= FACET_KEY_OPTION_NO_FACET; + k->options &= ~FACET_KEY_OPTION_FACET; + } + foreach_key_in_facets_done(k); +} + +inline FACET_KEY *facets_register_facet(FACETS *facets, const char *name, FACET_KEY_OPTIONS options) { + size_t name_length = strlen(name); + FACETS_HASH hash = FACETS_HASH_FUNCTION(name, name_length); + + FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, name, name_length, options); + k->options |= FACET_KEY_OPTION_FACET; + k->options &= ~FACET_KEY_OPTION_NO_FACET; + facet_key_late_init(facets, k); + + return k; +} + inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options) { if(!is_valid_string_hash(key_id)) return NULL; @@ -1708,16 +1838,25 @@ inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, F return k; } -void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *value_id, FACET_KEY_OPTIONS options) { +void facets_register_facet_filter_id(FACETS *facets, const char *key_id, const char *value_id, FACET_KEY_OPTIONS options) { FACET_KEY *k = facets_register_facet_id(facets, key_id, options); if(k) { if(is_valid_string_hash(value_id)) { k->default_selected_for_values = false; - FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, str_to_facets_hash(value_id)); + FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, NULL, str_to_facets_hash(value_id)); } } } +void facets_register_facet_filter(FACETS *facets, const char *key, const char *value, FACET_KEY_OPTIONS options) { + FACET_KEY *k = facets_register_facet(facets, key, options); + if(k) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(value, strlen(value)); + k->default_selected_for_values = false; + FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, value, hash); + } +} + void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity) { facets->current_row.severity = severity; } @@ -1834,6 +1973,10 @@ void facets_add_key_value(FACETS *facets, const char *key, const char *value) { } void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len) { + if(!key || !*key || !key_len || !value || !*value || !value_len) + // adding empty values, makes the rows unmatched + return; + FACET_KEY *k = facets_register_key_name_length(facets, key, key_len, 0); k->current_value.raw = value; k->current_value.raw_len = value_len; @@ -1879,7 +2022,9 @@ static void facet_row_key_value_delete_callback(const DICTIONARY_ITEM *item __ma // FACET_ROW management static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) { + facets_row_bin_data_cleanup(facets, &row->bin_data); dictionary_destroy(row->dict); + row->dict = NULL; freez(row); } @@ -1889,6 +2034,7 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into if(into) { row = into; facets->operations.rows.reused++; + facets_row_bin_data_cleanup(facets, &row->bin_data); } else { row = callocz(1, sizeof(FACET_ROW)); @@ -1899,6 +2045,11 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into facets->operations.rows.created++; } + // copy the bin_data to the row + // and forget about them in facets + row->bin_data = facets->bin_data; + facets->bin_data = FACET_ROW_BIN_DATA_EMPTY; + row->severity = facets->current_row.severity; row->usec = usec; @@ -2083,6 +2234,8 @@ static void facets_reset_keys_with_value_and_row(FACETS *facets) { facets->current_row.keys_matched_by_query_positive = 0; facets->current_row.keys_matched_by_query_negative = 0; facets->keys_in_row.used = 0; + + facets_row_bin_data_cleanup(facets, &facets->bin_data); } void facets_rows_begin(FACETS *facets) { @@ -2097,6 +2250,9 @@ void facets_rows_begin(FACETS *facets) { } bool facets_row_finished(FACETS *facets, usec_t usec) { +// char buf[RFC3339_MAX_LENGTH]; +// rfc3339_datetime_ut(buf, sizeof(buf), usec, 3, false); + facets->operations.rows.evaluated++; if(unlikely((facets->query && facets->keys_filtered_by_query && @@ -2207,11 +2363,11 @@ void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool w if(with_keys) { FACET_KEY *k; - foreach_key_in_facets(facets, k){ - if (!k->values.enabled) + foreach_key_in_facets(facets, k) { + if (!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN) continue; - buffer_json_add_array_item_string(wb, hash_to_static_string(k->hash)); + buffer_json_add_array_item_string(wb, facets_key_id(k)); } foreach_key_in_facets_done(k); } @@ -2391,8 +2547,8 @@ static uint32_t facets_sort_and_reorder_values(FACET_KEY *k) { return ret; } -void facets_table_config(BUFFER *wb) { - buffer_json_member_add_boolean(wb, "show_ids", false); // do not show the column ids to the user +void facets_table_config(FACETS *facets, BUFFER *wb) { + buffer_json_member_add_boolean(wb, "show_ids", (facets->options & FACETS_OPTION_HASH_IDS) ? false : true); buffer_json_member_add_boolean(wb, "has_history", true); // enable date-time picker with after-before buffer_json_member_add_object(wb, "pagination"); @@ -2408,8 +2564,9 @@ void facets_table_config(BUFFER *wb) { void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) { facets->report.used_hashes_registry = used_hashes_registry; + facets_table_config(facets, wb); + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) { - facets_table_config(wb); facets_accepted_parameters_to_json_array(facets, wb, true); } @@ -2434,19 +2591,21 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) CLEAN_BUFFER *tb = buffer_create(0, NULL); FACET_KEY *k; foreach_key_in_facets(facets, k) { - if(!k->values.enabled) + if(!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN) continue; - if(!facets_sort_and_reorder_values(k)) - // no values for this key - continue; + facets_sort_and_reorder_values(k); buffer_json_add_array_item_object(wb); // key { - buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash)); - buffer_json_member_add_string(wb, "name", facets_key_name_cached(k - , facets->report.used_hashes_registry - )); + buffer_json_member_add_string( + wb, "id", facets_key_id(k)); + + buffer_json_member_add_string( + wb, "name", + facets_key_name_cached(k, facets->report.used_hashes_registry)); + + // buffer_json_member_add_string(wb, "raw", k->name); if(!k->order) k->order = facets->order++; buffer_json_member_add_uint64(wb, "order", k->order); @@ -2463,10 +2622,11 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) buffer_json_add_array_item_object(wb); { - buffer_json_member_add_string(wb, "id", hash_to_static_string(v->hash)); + buffer_json_member_add_string(wb, "id", facets_key_value_id(k, v)); facets_key_value_transformed(facets, k, v, tb, FACETS_TRANSFORM_FACET); buffer_json_member_add_string(wb, "name", buffer_tostring(tb)); + // buffer_json_member_add_string(wb, "raw", v->name); buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter); buffer_json_member_add_uint64(wb, "order", v->order); } @@ -2517,38 +2677,40 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) FACET_KEY *k; foreach_key_in_facets(facets, k) { - RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP; - bool visible = k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY); + if(k->options & FACET_KEY_OPTION_HIDDEN) + continue; - if ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values.enabled)) - visible = true; + RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP; + RRDF_FIELD_VISUAL visual = (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE; + RRDF_FIELD_TRANSFORM transform = RRDF_FIELD_TRANSFORM_NONE; - if (!visible) - visible = simple_pattern_matches(facets->visible_keys, k->name); + if (k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY) || + ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE) && k->values.enabled) || + simple_pattern_matches(facets->visible_keys, k->name)) + options |= RRDF_FIELD_OPTS_VISIBLE; - if (visible) - options |= RRDF_FIELD_OPTS_VISIBLE; + if (k->options & FACET_KEY_OPTION_MAIN_TEXT) + options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP; - if (k->options & FACET_KEY_OPTION_MAIN_TEXT) - options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP; + if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER) + options |= RRDF_FIELD_OPTS_EXPANDED_FILTER; - if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER) - options |= RRDF_FIELD_OPTS_EXPANDED_FILTER; + if (k->options & FACET_KEY_OPTION_PRETTY_XML) + transform = RRDF_FIELD_TRANSFORM_XML; - const char *hash_str = hash_to_static_string(k->hash); + const char *key_id = facets_key_id(k); - buffer_rrdf_table_add_field( - wb, field_id++, - hash_str, k->name ? k->name : hash_str, - RRDF_FIELD_TYPE_STRING, - (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE, - RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, - RRDF_FIELD_SORT_FIXED, - NULL, - RRDF_FIELD_SUMMARY_COUNT, - (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET, - options, FACET_VALUE_UNSET); - } + buffer_rrdf_table_add_field( + wb, field_id++, + key_id, k->name ? k->name : key_id, + RRDF_FIELD_TYPE_STRING, + visual, transform, 0, NULL, NAN, + RRDF_FIELD_SORT_FIXED, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET, + options, FACET_VALUE_UNSET); + } foreach_key_in_facets_done(k); } buffer_json_object_close(wb); // columns @@ -2585,6 +2747,9 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) FACET_KEY *k; foreach_key_in_facets(facets, k) { + if(k->options & FACET_KEY_OPTION_HIDDEN) + continue; + FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name); if(unlikely(k->dynamic.cb)) { @@ -2627,14 +2792,14 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) { FACET_KEY *k; foreach_key_in_facets(facets, k) { - if (!k->values.enabled) + if (!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN) continue; if(unlikely(!first_histogram_hash)) first_histogram_hash = k->hash; buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash)); + buffer_json_member_add_string(wb, "id", facets_key_id(k)); buffer_json_member_add_string(wb, "name", k->name); buffer_json_member_add_uint64(wb, "order", k->order); buffer_json_object_close(wb); @@ -2662,7 +2827,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) } if(show_histogram) { - buffer_json_member_add_string(wb, "id", k ? hash_to_static_string(k->hash) : ""); + buffer_json_member_add_string(wb, "id", k ? facets_key_id(k) : ""); buffer_json_member_add_string(wb, "name", k ? k->name : ""); buffer_json_member_add_object(wb, "chart"); { diff --git a/src/libnetdata/facets/facets.h b/src/libnetdata/facets/facets.h index 8364d8612..1d2b89c2b 100644 --- a/src/libnetdata/facets/facets.h +++ b/src/libnetdata/facets/facets.h @@ -23,6 +23,7 @@ typedef enum __attribute__((packed)) { } FACETS_TRANSFORMATION_SCOPE; typedef enum __attribute__((packed)) { + FACET_KEY_OPTION_NONE = 0, FACET_KEY_OPTION_FACET = (1 << 0), // filterable values FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value FACET_KEY_OPTION_NEVER_FACET = (1 << 2), // never enable this field as facet @@ -32,8 +33,11 @@ typedef enum __attribute__((packed)) { FACET_KEY_OPTION_MAIN_TEXT = (1 << 6), // full width and wrap FACET_KEY_OPTION_RICH_TEXT = (1 << 7), FACET_KEY_OPTION_REORDER = (1 << 8), // give the key a new order id on first encounter - FACET_KEY_OPTION_TRANSFORM_VIEW = (1 << 9), // when registering the transformation, do it only at the view, not on all data - FACET_KEY_OPTION_EXPANDED_FILTER = (1 << 10), // the presentation should have this filter expanded by default + FACET_KEY_OPTION_REORDER_DONE = (1 << 9), // done re-ordering for this field + FACET_KEY_OPTION_TRANSFORM_VIEW = (1 << 10), // when registering the transformation, do it only at the view, not on all data + FACET_KEY_OPTION_EXPANDED_FILTER = (1 << 11), // the presentation should have this filter expanded by default + FACET_KEY_OPTION_PRETTY_XML = (1 << 12), // instruct the UI to parse this as an XML document + FACET_KEY_OPTION_HIDDEN = (1 << 13), // do not include this field in the response } FACET_KEY_OPTIONS; typedef enum __attribute__((packed)) { @@ -51,10 +55,18 @@ typedef struct facet_row_key_value { BUFFER *wb; } FACET_ROW_KEY_VALUE; +typedef struct facet_row_bin_data { + void (*cleanup_cb)(void *data); + void *data; +} FACET_ROW_BIN_DATA; + +#define FACET_ROW_BIN_DATA_EMPTY (FACET_ROW_BIN_DATA){.data = NULL, .cleanup_cb = NULL} + typedef struct facet_row { usec_t usec; DICTIONARY *dict; FACET_ROW_SEVERITY severity; + FACET_ROW_BIN_DATA bin_data; struct facet_row *prev, *next; } FACET_ROW; @@ -77,6 +89,7 @@ typedef enum __attribute__((packed)) { FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS = (1 << 5), // empty facet values will not be included in the report FACETS_OPTION_SORT_FACETS_ALPHABETICALLY = (1 << 6), FACETS_OPTION_SHOW_DELTAS = (1 << 7), + FACETS_OPTION_HASH_IDS = (1 << 8), // when set, the id of the facets, keys and values will be their hash } FACETS_OPTIONS; FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys); @@ -98,8 +111,14 @@ void facets_set_anchor(FACETS *facets, usec_t start_ut, usec_t stop_ut, FACETS_A void facets_enable_slice_mode(FACETS *facets); bool facets_row_candidate_to_keep(FACETS *facets, usec_t usec); +void facets_reset_and_disable_all_facets(FACETS *facets); + +FACET_KEY *facets_register_facet(FACETS *facets, const char *name, FACET_KEY_OPTIONS options); FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options); -void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *value_id, FACET_KEY_OPTIONS options); + +void facets_register_facet_filter(FACETS *facets, const char *key, const char *value, FACET_KEY_OPTIONS options); +void facets_register_facet_filter_id(FACETS *facets, const char *key_id, const char *value_id, FACET_KEY_OPTIONS options); + void facets_set_timeframe_and_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut); void facets_set_timeframe_and_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut); @@ -121,8 +140,16 @@ usec_t facets_row_oldest_ut(FACETS *facets); usec_t facets_row_newest_ut(FACETS *facets); uint32_t facets_rows(FACETS *facets); -void facets_table_config(BUFFER *wb); +void facets_table_config(FACETS *facets, BUFFER *wb); const char *facets_severity_to_string(FACET_ROW_SEVERITY severity); +typedef bool (*facets_foreach_selected_value_in_key_t)(FACETS *facets, size_t id, const char *key, const char *value, void *data); +bool facets_foreach_selected_value_in_key(FACETS *facets, const char *key, size_t key_length, DICTIONARY *used_hashes_registry, facets_foreach_selected_value_in_key_t cb, void *data); + +void facets_row_bin_data_set(FACETS *facets, void (*cleanup_cb)(void *data), void *data); +void *facets_row_bin_data_get(FACETS *facets __maybe_unused, FACET_ROW *row); + +void facets_use_hashes_for_ids(FACETS *facets, bool set); + #endif diff --git a/src/libnetdata/facets/logs_query_status.h b/src/libnetdata/facets/logs_query_status.h new file mode 100644 index 000000000..4fde24998 --- /dev/null +++ b/src/libnetdata/facets/logs_query_status.h @@ -0,0 +1,868 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_LOGS_QUERY_STATUS_H +#define NETDATA_LOGS_QUERY_STATUS_H + +#include "../libnetdata.h" + +#define LQS_PARAMETER_HELP "help" +#define LQS_PARAMETER_AFTER "after" +#define LQS_PARAMETER_BEFORE "before" +#define LQS_PARAMETER_ANCHOR "anchor" +#define LQS_PARAMETER_LAST "last" +#define LQS_PARAMETER_QUERY "query" +#define LQS_PARAMETER_FACETS "facets" +#define LQS_PARAMETER_HISTOGRAM "histogram" +#define LQS_PARAMETER_DIRECTION "direction" +#define LQS_PARAMETER_IF_MODIFIED_SINCE "if_modified_since" +#define LQS_PARAMETER_DATA_ONLY "data_only" +#define LQS_PARAMETER_SOURCE "__logs_sources" // this must never conflict with user fields +#define LQS_PARAMETER_INFO "info" +#define LQS_PARAMETER_SLICE "slice" +#define LQS_PARAMETER_DELTA "delta" +#define LQS_PARAMETER_TAIL "tail" +#define LQS_PARAMETER_SAMPLING "sampling" + +#define LQS_MAX_PARAMS 1000 +#define LQS_DEFAULT_QUERY_DURATION (1 * 3600) + +#undef LQS_SLICE_PARAMETER +#if LQS_DEFAULT_SLICE_MODE == 1 +#define LQS_SLICE_PARAMETER 1 +#endif + +typedef struct { + const char *transaction; + + FACET_KEY_OPTIONS default_facet; // the option to be used for internal fields. + // when the requests set facets, we disable all default facets, + // so that the UI has full control over them. + + bool fields_are_ids; // POST works with field names, GET works with field hashes (IDs) + bool info; // the request is an INFO request, do not execute a query. + + bool data_only; // return as fast as possible, with the requested amount of data, + // without scanning the entire duration. + + bool slice; // apply native backend filters to slice the events database. + bool delta; // return incremental data for the histogram (used with data_only) + bool tail; // return NOT MODIFIED if no more data are available after the anchor given. + + time_t after_s; // the starting timestamp of the query + time_t before_s; // the ending timestamp of the query + usec_t after_ut; // in microseconds + usec_t before_ut; // in microseconds + + usec_t anchor; // the anchor to seek to + FACETS_ANCHOR_DIRECTION direction; // the direction based on the anchor (or the query timeframe) + + usec_t if_modified_since; // the timestamp to check with tail == true + + size_t entries; // the number of log events to return in a single response + + const char *query; // full text search query string + const char *histogram; // the field to use for the histogram + + SIMPLE_PATTERN *sources; // custom log sources to query + LQS_SOURCE_TYPE source_type; // pre-defined log sources to query + + size_t filters; // the number of filters (facets selected) in the query + size_t sampling; // the number of log events to sample, when the query is too big + + time_t now_s; // the timestamp the query was received + time_t expires_s; // the timestamp the response expires +} LOGS_QUERY_REQUEST; + +#define LOGS_QUERY_REQUEST_DEFAULTS(function_transaction, default_slice, default_direction) \ + (LOGS_QUERY_REQUEST) { \ + .transaction = (function_transaction), \ + .default_facet = FACET_KEY_OPTION_FACET, \ + .info = false, \ + .data_only = false, \ + .slice = (default_slice), \ + .delta = false, \ + .tail = false, \ + .after_s = 0, \ + .before_s = 0, \ + .anchor = 0, \ + .if_modified_since = 0, \ + .entries = 0, \ + .direction = (default_direction), \ + .query = NULL, \ + .histogram = NULL, \ + .sources = NULL, \ + .source_type = LQS_SOURCE_TYPE_ALL, \ + .filters = 0, \ + .sampling = LQS_DEFAULT_ITEMS_SAMPLING, \ +} + +typedef struct { + FACETS *facets; + + LOGS_QUERY_REQUEST rq; + + bool *cancelled; // a pointer to the cancelling boolean + usec_t *stop_monotonic_ut; + + struct { + usec_t start_ut; + usec_t stop_ut; + usec_t delta_ut; + } anchor; + + struct { + usec_t start_ut; + usec_t stop_ut; + bool stop_when_full; + } query; + + usec_t last_modified; + + struct lqs_extension c; +} LOGS_QUERY_STATUS; + +struct logs_query_data { + const char *transaction; + FACETS *facets; + LOGS_QUERY_REQUEST *rq; + BUFFER *wb; +}; + +static inline FACETS_ANCHOR_DIRECTION lgs_get_direction(const char *value) { + return strcasecmp(value, "forward") == 0 ? FACETS_ANCHOR_DIRECTION_FORWARD : FACETS_ANCHOR_DIRECTION_BACKWARD; +} + +static inline void lqs_log_error(LOGS_QUERY_STATUS *lqs, const char *msg) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "LOGS QUERY ERROR: %s, on query " + "timeframe [%"PRIu64" - %"PRIu64"], " + "anchor [%"PRIu64" - %"PRIu64"], " + "if_modified_since %"PRIu64", " + "data_only:%s, delta:%s, tail:%s, direction:%s" + , msg + , lqs->rq.after_ut + , lqs->rq.before_ut + , lqs->anchor.start_ut + , lqs->anchor.stop_ut + , lqs->rq.if_modified_since + , lqs->rq.data_only ? "true" : "false" + , lqs->rq.delta ? "true" : "false" + , lqs->rq.tail ? "tail" : "false" + , lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward"); +} + +static inline void lqs_query_timeframe(LOGS_QUERY_STATUS *lqs, usec_t anchor_delta_ut) { + lqs->anchor.delta_ut = anchor_delta_ut; + + if(lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD) { + lqs->query.start_ut = (lqs->rq.data_only && lqs->anchor.start_ut) ? lqs->anchor.start_ut : lqs->rq.after_ut; + lqs->query.stop_ut = ((lqs->rq.data_only && lqs->anchor.stop_ut) ? lqs->anchor.stop_ut : lqs->rq.before_ut) + lqs->anchor.delta_ut; + } + else { + lqs->query.start_ut = ((lqs->rq.data_only && lqs->anchor.start_ut) ? lqs->anchor.start_ut : lqs->rq.before_ut) + lqs->anchor.delta_ut; + lqs->query.stop_ut = (lqs->rq.data_only && lqs->anchor.stop_ut) ? lqs->anchor.stop_ut : lqs->rq.after_ut; + } + + lqs->query.stop_when_full = (lqs->rq.data_only && !lqs->anchor.stop_ut); +} + +static inline void lqs_function_help(LOGS_QUERY_STATUS *lqs, BUFFER *wb) { + buffer_reset(wb); + wb->content_type = CT_TEXT_PLAIN; + wb->response_code = HTTP_RESP_OK; + + buffer_sprintf(wb, + "%s / %s\n" + "\n" + "%s\n" + "\n" + "The following parameters are supported:\n" + "\n" + , program_name + , LQS_FUNCTION_NAME + , LQS_FUNCTION_DESCRIPTION + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_HELP "\n" + " Shows this help message.\n" + "\n" + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_INFO "\n" + " Request initial configuration information about the plugin.\n" + " The key entity returned is the required_params array, which includes\n" + " all the available log sources.\n" + " When `" LQS_PARAMETER_INFO "` is requested, all other parameters are ignored.\n" + "\n" + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_DATA_ONLY ":true or " LQS_PARAMETER_DATA_ONLY ":false\n" + " Quickly respond with data requested, without generating a\n" + " `histogram`, `facets` counters and `items`.\n" + "\n" + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_DELTA ":true or " LQS_PARAMETER_DELTA ":false\n" + " When doing data only queries, include deltas for histogram, facets and items.\n" + "\n" + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_TAIL ":true or " LQS_PARAMETER_TAIL ":false\n" + " When doing data only queries, respond with the newest messages,\n" + " and up to the anchor, but calculate deltas (if requested) for\n" + " the duration [anchor - before].\n" + "\n" + ); + +#ifdef LQS_SLICE_PARAMETER + buffer_sprintf(wb, + " " LQS_PARAMETER_SLICE ":true or " LQS_PARAMETER_SLICE ":false\n" + " When it is turned on, the plugin is is slicing the logs database,\n" + " utilizing the underlying available indexes.\n" + " When it is off, all filtering is done by the plugin.\n" + " The default is: %s\n" + "\n" + , lqs->rq.slice ? "true" : "false" + ); +#endif + buffer_sprintf(wb, + " " LQS_PARAMETER_SOURCE ":SOURCE\n" + " Query only the specified log sources.\n" + " Do an `" LQS_PARAMETER_INFO "` query to find the sources.\n" + "\n" + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_BEFORE ":TIMESTAMP_IN_SECONDS\n" + " Absolute or relative (to now) timestamp in seconds, to start the query.\n" + " The query is always executed from the most recent to the oldest log entry.\n" + " If not given the default is: now.\n" + "\n" + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_AFTER ":TIMESTAMP_IN_SECONDS\n" + " Absolute or relative (to `before`) timestamp in seconds, to end the query.\n" + " If not given, the default is %d.\n" + "\n" + , -LQS_DEFAULT_QUERY_DURATION + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_LAST ":ITEMS\n" + " The number of items to return.\n" + " The default is %zu.\n" + "\n" + , lqs->rq.entries + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_SAMPLING ":ITEMS\n" + " The number of log entries to sample to estimate facets counters and histogram.\n" + " The default is %zu.\n" + "\n" + , lqs->rq.sampling + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_ANCHOR ":TIMESTAMP_IN_MICROSECONDS\n" + " Return items relative to this timestamp.\n" + " The exact items to be returned depend on the query `" LQS_PARAMETER_DIRECTION "`.\n" + "\n" + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_DIRECTION ":forward or " LQS_PARAMETER_DIRECTION ":backward\n" + " When set to `backward` (default) the items returned are the newest before the\n" + " `" LQS_PARAMETER_ANCHOR "`, (or `" LQS_PARAMETER_BEFORE "` if `" LQS_PARAMETER_ANCHOR "` is not set)\n" + " When set to `forward` the items returned are the oldest after the\n" + " `" LQS_PARAMETER_ANCHOR "`, (or `" LQS_PARAMETER_AFTER "` if `" LQS_PARAMETER_ANCHOR "` is not set)\n" + " The default is: %s\n" + "\n" + , lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward" + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_QUERY ":SIMPLE_PATTERN\n" + " Do a full text search to find the log entries matching the pattern given.\n" + " The plugin is searching for matches on all fields of the database.\n" + "\n" + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_IF_MODIFIED_SINCE ":TIMESTAMP_IN_MICROSECONDS\n" + " Each successful response, includes a `last_modified` field.\n" + " By providing the timestamp to the `" LQS_PARAMETER_IF_MODIFIED_SINCE "` parameter,\n" + " the plugin will return 200 with a successful response, or 304 if the source has not\n" + " been modified since that timestamp.\n" + "\n" + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_HISTOGRAM ":facet_id\n" + " Use the given `facet_id` for the histogram.\n" + " This parameter is ignored in `" LQS_PARAMETER_DATA_ONLY "` mode.\n" + "\n" + ); + + buffer_sprintf(wb, + " " LQS_PARAMETER_FACETS ":facet_id1,facet_id2,facet_id3,...\n" + " Add the given facets to the list of fields for which analysis is required.\n" + " The plugin will offer both a histogram and facet value counters for its values.\n" + " This parameter is ignored in `" LQS_PARAMETER_DATA_ONLY "` mode.\n" + "\n" + ); + + buffer_sprintf(wb, + " facet_id:value_id1,value_id2,value_id3,...\n" + " Apply filters to the query, based on the facet IDs returned.\n" + " Each `facet_id` can be given once, but multiple `facet_ids` can be given.\n" + "\n" + ); +} + +static inline bool lqs_request_parse_json_payload(json_object *jobj, const char *path, void *data, BUFFER *error) { + struct logs_query_data *qd = data; + LOGS_QUERY_REQUEST *rq = qd->rq; + BUFFER *wb = qd->wb; + FACETS *facets = qd->facets; + // const char *transaction = qd->transaction; + + buffer_flush(error); + + JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_INFO, rq->info, error, false); + JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_DELTA, rq->delta, error, false); + JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_TAIL, rq->tail, error, false); + JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_SLICE, rq->slice, error, false); + JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_DATA_ONLY, rq->data_only, error, false); + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_SAMPLING, rq->sampling, error, false); + JSONC_PARSE_INT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_AFTER, rq->after_s, error, false); + JSONC_PARSE_INT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_BEFORE, rq->before_s, error, false); + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_IF_MODIFIED_SINCE, rq->if_modified_since, error, false); + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_ANCHOR, rq->anchor, error, false); + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_LAST, rq->entries, error, false); + JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_DIRECTION, lgs_get_direction, rq->direction, error, false); + JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_QUERY, rq->query, error, false); + JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, LQS_PARAMETER_HISTOGRAM, rq->histogram, error, false); + + json_object *fcts; + if (json_object_object_get_ex(jobj, LQS_PARAMETER_FACETS, &fcts)) { + if (json_object_get_type(fcts) != json_type_array) { + buffer_sprintf(error, "member '%s' is not an array.", LQS_PARAMETER_FACETS); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: '%s' is not an array", LQS_PARAMETER_FACETS); + return false; + } + + rq->default_facet = FACET_KEY_OPTION_NONE; + facets_reset_and_disable_all_facets(facets); + + buffer_json_member_add_array(wb, LQS_PARAMETER_FACETS); + + size_t facets_len = json_object_array_length(fcts); + for (size_t i = 0; i < facets_len; i++) { + json_object *fct = json_object_array_get_idx(fcts, i); + + if (json_object_get_type(fct) != json_type_string) { + buffer_sprintf(error, "facets array item %zu is not a string", i); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: facets array item %zu is not a string", i); + return false; + } + + const char *value = json_object_get_string(fct); + facets_register_facet(facets, value, FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS|FACET_KEY_OPTION_REORDER); + buffer_json_add_array_item_string(wb, value); + } + + buffer_json_array_close(wb); // facets + } + + json_object *selections; + if (json_object_object_get_ex(jobj, "selections", &selections)) { + if (json_object_get_type(selections) != json_type_object) { + buffer_sprintf(error, "member 'selections' is not an object"); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: '%s' is not an object", "selections"); + return false; + } + + buffer_json_member_add_object(wb, "selections"); + + CLEAN_BUFFER *sources_list = buffer_create(0, NULL); + + json_object_object_foreach(selections, key, val) { + if(strcmp(key, "query") == 0) continue; + + if (json_object_get_type(val) != json_type_array) { + buffer_sprintf(error, "selection '%s' is not an array", key); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: selection '%s' is not an array", key); + return false; + } + + bool is_source = false; + if(strcmp(key, LQS_PARAMETER_SOURCE) == 0) { + // reset the sources, so that only what the user selects will be shown + is_source = true; + rq->source_type = LQS_SOURCE_TYPE_NONE; + } + + buffer_json_member_add_array(wb, key); + + size_t values_len = json_object_array_length(val); + for (size_t i = 0; i < values_len; i++) { + json_object *value_obj = json_object_array_get_idx(val, i); + + if (json_object_get_type(value_obj) != json_type_string) { + buffer_sprintf(error, "selection '%s' array item %zu is not a string", key, i); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: selection '%s' array item %zu is not a string", key, i); + return false; + } + + const char *value = json_object_get_string(value_obj); + + if(is_source) { + // processing sources + LQS_SOURCE_TYPE t = LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value); + if(t != LQS_SOURCE_TYPE_NONE) { + rq->source_type |= t; + value = NULL; + } + else { + // else, match the source, whatever it is + if(buffer_strlen(sources_list)) + buffer_putc(sources_list, '|'); + + buffer_strcat(sources_list, value); + } + } + else { + // Call facets_register_facet_id_filter for each value + facets_register_facet_filter( + facets, key, value, FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_REORDER); + + rq->filters++; + } + + buffer_json_add_array_item_string(wb, value); + } + + buffer_json_array_close(wb); // key + } + + if(buffer_strlen(sources_list)) { + simple_pattern_free(rq->sources); + rq->sources = simple_pattern_create(buffer_tostring(sources_list), "|", SIMPLE_PATTERN_EXACT, false); + } + + buffer_json_object_close(wb); // selections + } + + facets_use_hashes_for_ids(facets, false); + rq->fields_are_ids = false; + return true; +} + +static inline bool lqs_request_parse_POST(LOGS_QUERY_STATUS *lqs, BUFFER *wb, BUFFER *payload, const char *transaction) { + FACETS *facets = lqs->facets; + LOGS_QUERY_REQUEST *rq = &lqs->rq; + + buffer_json_member_add_object(wb, "_request"); + + struct logs_query_data qd = { + .transaction = transaction, + .facets = facets, + .rq = rq, + .wb = wb, + }; + + int code; + CLEAN_JSON_OBJECT *jobj = + json_parse_function_payload_or_error(wb, payload, &code, lqs_request_parse_json_payload, &qd); + wb->response_code = code; + + return (jobj && code == HTTP_RESP_OK); +} + +static inline bool lqs_request_parse_GET(LOGS_QUERY_STATUS *lqs, BUFFER *wb, char *function) { + FACETS *facets = lqs->facets; + LOGS_QUERY_REQUEST *rq = &lqs->rq; + + buffer_json_member_add_object(wb, "_request"); + + char func_copy[strlen(function) + 1]; + memcpy(func_copy, function, sizeof(func_copy)); + + char *words[LQS_MAX_PARAMS] = { NULL }; + size_t num_words = quoted_strings_splitter_whitespace(func_copy, words, LQS_MAX_PARAMS); + for(int i = 1; i < LQS_MAX_PARAMS;i++) { + char *keyword = get_word(words, num_words, i); + if(!keyword) break; + + if(strcmp(keyword, LQS_PARAMETER_HELP) == 0) { + lqs_function_help(lqs, wb); + return false; + } + else if(strcmp(keyword, LQS_PARAMETER_INFO) == 0) { + rq->info = true; + } + else if(strncmp(keyword, LQS_PARAMETER_DELTA ":", sizeof(LQS_PARAMETER_DELTA ":") - 1) == 0) { + char *v = &keyword[sizeof(LQS_PARAMETER_DELTA ":") - 1]; + + if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0) + rq->delta = false; + else + rq->delta = true; + } + else if(strncmp(keyword, LQS_PARAMETER_TAIL ":", sizeof(LQS_PARAMETER_TAIL ":") - 1) == 0) { + char *v = &keyword[sizeof(LQS_PARAMETER_TAIL ":") - 1]; + + if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0) + rq->tail = false; + else + rq->tail = true; + } + else if(strncmp(keyword, LQS_PARAMETER_SAMPLING ":", sizeof(LQS_PARAMETER_SAMPLING ":") - 1) == 0) { + rq->sampling = str2ul(&keyword[sizeof(LQS_PARAMETER_SAMPLING ":") - 1]); + } + else if(strncmp(keyword, LQS_PARAMETER_DATA_ONLY ":", sizeof(LQS_PARAMETER_DATA_ONLY ":") - 1) == 0) { + char *v = &keyword[sizeof(LQS_PARAMETER_DATA_ONLY ":") - 1]; + + if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0) + rq->data_only = false; + else + rq->data_only = true; + } + else if(strncmp(keyword, LQS_PARAMETER_SLICE ":", sizeof(LQS_PARAMETER_SLICE ":") - 1) == 0) { + char *v = &keyword[sizeof(LQS_PARAMETER_SLICE ":") - 1]; + + if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0) + rq->slice = false; + else + rq->slice = true; + } + else if(strncmp(keyword, LQS_PARAMETER_SOURCE ":", sizeof(LQS_PARAMETER_SOURCE ":") - 1) == 0) { + const char *value = &keyword[sizeof(LQS_PARAMETER_SOURCE ":") - 1]; + + buffer_json_member_add_array(wb, LQS_PARAMETER_SOURCE); + + CLEAN_BUFFER *sources_list = buffer_create(0, NULL); + + rq->source_type = LQS_SOURCE_TYPE_NONE; + while(value) { + char *sep = strchr(value, ','); + if(sep) + *sep++ = '\0'; + + buffer_json_add_array_item_string(wb, value); + + LQS_SOURCE_TYPE t = LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value); + if(t != LQS_SOURCE_TYPE_NONE) { + rq->source_type |= t; + value = NULL; + } + else { + // else, match the source, whatever it is + if(buffer_strlen(sources_list)) + buffer_putc(sources_list, '|'); + + buffer_strcat(sources_list, value); + } + + value = sep; + } + + if(buffer_strlen(sources_list)) { + simple_pattern_free(rq->sources); + rq->sources = simple_pattern_create(buffer_tostring(sources_list), "|", SIMPLE_PATTERN_EXACT, false); + } + + buffer_json_array_close(wb); // source + } + else if(strncmp(keyword, LQS_PARAMETER_AFTER ":", sizeof(LQS_PARAMETER_AFTER ":") - 1) == 0) { + rq->after_s = str2l(&keyword[sizeof(LQS_PARAMETER_AFTER ":") - 1]); + } + else if(strncmp(keyword, LQS_PARAMETER_BEFORE ":", sizeof(LQS_PARAMETER_BEFORE ":") - 1) == 0) { + rq->before_s = str2l(&keyword[sizeof(LQS_PARAMETER_BEFORE ":") - 1]); + } + else if(strncmp(keyword, LQS_PARAMETER_IF_MODIFIED_SINCE ":", sizeof(LQS_PARAMETER_IF_MODIFIED_SINCE ":") - 1) == 0) { + rq->if_modified_since = str2ull(&keyword[sizeof(LQS_PARAMETER_IF_MODIFIED_SINCE ":") - 1], NULL); + } + else if(strncmp(keyword, LQS_PARAMETER_ANCHOR ":", sizeof(LQS_PARAMETER_ANCHOR ":") - 1) == 0) { + rq->anchor = str2ull(&keyword[sizeof(LQS_PARAMETER_ANCHOR ":") - 1], NULL); + } + else if(strncmp(keyword, LQS_PARAMETER_DIRECTION ":", sizeof(LQS_PARAMETER_DIRECTION ":") - 1) == 0) { + rq->direction = lgs_get_direction(&keyword[sizeof(LQS_PARAMETER_DIRECTION ":") - 1]); + } + else if(strncmp(keyword, LQS_PARAMETER_LAST ":", sizeof(LQS_PARAMETER_LAST ":") - 1) == 0) { + rq->entries = str2ul(&keyword[sizeof(LQS_PARAMETER_LAST ":") - 1]); + } + else if(strncmp(keyword, LQS_PARAMETER_QUERY ":", sizeof(LQS_PARAMETER_QUERY ":") - 1) == 0) { + freez((void *)rq->query); + rq->query= strdupz(&keyword[sizeof(LQS_PARAMETER_QUERY ":") - 1]); + } + else if(strncmp(keyword, LQS_PARAMETER_HISTOGRAM ":", sizeof(LQS_PARAMETER_HISTOGRAM ":") - 1) == 0) { + freez((void *)rq->histogram); + rq->histogram = strdupz(&keyword[sizeof(LQS_PARAMETER_HISTOGRAM ":") - 1]); + } + else if(strncmp(keyword, LQS_PARAMETER_FACETS ":", sizeof(LQS_PARAMETER_FACETS ":") - 1) == 0) { + rq->default_facet = FACET_KEY_OPTION_NONE; + facets_reset_and_disable_all_facets(facets); + + char *value = &keyword[sizeof(LQS_PARAMETER_FACETS ":") - 1]; + if(*value) { + buffer_json_member_add_array(wb, LQS_PARAMETER_FACETS); + + while(value) { + char *sep = strchr(value, ','); + if(sep) + *sep++ = '\0'; + + facets_register_facet_id(facets, value, FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS|FACET_KEY_OPTION_REORDER); + buffer_json_add_array_item_string(wb, value); + + value = sep; + } + + buffer_json_array_close(wb); // facets + } + } + else { + char *value = strchr(keyword, ':'); + if(value) { + *value++ = '\0'; + + buffer_json_member_add_array(wb, keyword); + + while(value) { + char *sep = strchr(value, ','); + if(sep) + *sep++ = '\0'; + + facets_register_facet_filter_id( + facets, keyword, value, + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_REORDER); + + buffer_json_add_array_item_string(wb, value); + rq->filters++; + + value = sep; + } + + buffer_json_array_close(wb); // keyword + } + } + } + + facets_use_hashes_for_ids(facets, true); + rq->fields_are_ids = true; + return true; +} + +static inline void lqs_info_response(BUFFER *wb, FACETS *facets) { + // the buffer already has the request in it + // DO NOT FLUSH IT + + buffer_json_member_add_uint64(wb, "v", 3); + facets_accepted_parameters_to_json_array(facets, wb, false); + buffer_json_member_add_array(wb, "required_params"); + { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "id", LQS_PARAMETER_SOURCE); + buffer_json_member_add_string(wb, "name", LQS_PARAMETER_SOURCE_NAME); + buffer_json_member_add_string(wb, "help", "Select the logs source to query"); + buffer_json_member_add_string(wb, "type", "multiselect"); + buffer_json_member_add_array(wb, "options"); + { + LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb); + } + buffer_json_array_close(wb); // options array + } + buffer_json_object_close(wb); // required params object + } + buffer_json_array_close(wb); // required_params array + + facets_table_config(facets, wb); + + buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK); + buffer_json_member_add_string(wb, "type", "table"); + buffer_json_member_add_string(wb, "help", LQS_FUNCTION_DESCRIPTION); + buffer_json_finalize(wb); + + wb->content_type = CT_APPLICATION_JSON; + wb->response_code = HTTP_RESP_OK; +} + +static inline BUFFER *lqs_create_output_buffer(void) { + BUFFER *wb = buffer_create(0, NULL); + buffer_reset(wb); + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); + return wb; +} + +static inline FACETS *lqs_facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys, bool have_slice) { + FACETS *facets = facets_create(items_to_return, options, + visible_keys, facet_keys, non_facet_keys); + + facets_accepted_param(facets, LQS_PARAMETER_INFO); + facets_accepted_param(facets, LQS_PARAMETER_SOURCE); + facets_accepted_param(facets, LQS_PARAMETER_AFTER); + facets_accepted_param(facets, LQS_PARAMETER_BEFORE); + facets_accepted_param(facets, LQS_PARAMETER_ANCHOR); + facets_accepted_param(facets, LQS_PARAMETER_DIRECTION); + facets_accepted_param(facets, LQS_PARAMETER_LAST); + facets_accepted_param(facets, LQS_PARAMETER_QUERY); + facets_accepted_param(facets, LQS_PARAMETER_FACETS); + facets_accepted_param(facets, LQS_PARAMETER_HISTOGRAM); + facets_accepted_param(facets, LQS_PARAMETER_IF_MODIFIED_SINCE); + facets_accepted_param(facets, LQS_PARAMETER_DATA_ONLY); + facets_accepted_param(facets, LQS_PARAMETER_DELTA); + facets_accepted_param(facets, LQS_PARAMETER_TAIL); + facets_accepted_param(facets, LQS_PARAMETER_SAMPLING); + + if(have_slice) + facets_accepted_param(facets, LQS_PARAMETER_SLICE); + + return facets; +} + +static inline bool lqs_request_parse_and_validate(LOGS_QUERY_STATUS *lqs, BUFFER *wb, char *function, BUFFER *payload, bool have_slice, const char *default_histogram) { + LOGS_QUERY_REQUEST *rq = &lqs->rq; + FACETS *facets = lqs->facets; + + if( (payload && !lqs_request_parse_POST(lqs, wb, payload, rq->transaction)) || + (!payload && !lqs_request_parse_GET(lqs, wb, function)) ) + return false; + + // ---------------------------------------------------------------------------------------------------------------- + // validate parameters + + if(rq->query && !*rq->query) { + freez((void *)rq->query); + rq->query = NULL; + } + + if(rq->histogram && !*rq->histogram) { + freez((void *)rq->histogram); + rq->histogram = NULL; + } + + if(!rq->data_only) + rq->delta = false; + + if(!rq->data_only || !rq->if_modified_since) + rq->tail = false; + + rq->now_s = now_realtime_sec(); + rq->expires_s = rq->now_s + 1; + wb->expires = rq->expires_s; + + if(!rq->after_s && !rq->before_s) { + rq->before_s = rq->now_s; + rq->after_s = rq->before_s - LQS_DEFAULT_QUERY_DURATION; + } + else + rrdr_relative_window_to_absolute(&rq->after_s, &rq->before_s, rq->now_s); + + if(rq->after_s > rq->before_s) { + time_t tmp = rq->after_s; + rq->after_s = rq->before_s; + rq->before_s = tmp; + } + + if(rq->after_s == rq->before_s) + rq->after_s = rq->before_s - LQS_DEFAULT_QUERY_DURATION; + + rq->after_ut = rq->after_s * USEC_PER_SEC; + rq->before_ut = (rq->before_s * USEC_PER_SEC) + USEC_PER_SEC - 1; + + if(!rq->entries) + rq->entries = LQS_DEFAULT_ITEMS_PER_QUERY; + + // ---------------------------------------------------------------------------------------------------------------- + // validate the anchor + + lqs->last_modified = 0; + lqs->anchor.start_ut = lqs->rq.anchor; + lqs->anchor.stop_ut = 0; + + if(lqs->anchor.start_ut && lqs->rq.tail) { + // a tail request + // we need the top X entries from BEFORE + // but, we need to calculate the facets and the + // histogram up to the anchor + lqs->rq.direction = FACETS_ANCHOR_DIRECTION_BACKWARD; + lqs->anchor.start_ut = 0; + lqs->anchor.stop_ut = lqs->rq.anchor; + } + + if(lqs->rq.anchor && lqs->rq.anchor < lqs->rq.after_ut) { + lqs_log_error(lqs, "received anchor is too small for query timeframe, ignoring anchor"); + lqs->rq.anchor = 0; + lqs->anchor.start_ut = 0; + lqs->anchor.stop_ut = 0; + lqs->rq.direction = FACETS_ANCHOR_DIRECTION_BACKWARD; + } + else if(lqs->rq.anchor > lqs->rq.before_ut) { + lqs_log_error(lqs, "received anchor is too big for query timeframe, ignoring anchor"); + lqs->rq.anchor = 0; + lqs->anchor.start_ut = 0; + lqs->anchor.stop_ut = 0; + lqs->rq.direction = FACETS_ANCHOR_DIRECTION_BACKWARD; + } + + facets_set_anchor(facets, lqs->anchor.start_ut, lqs->anchor.stop_ut, lqs->rq.direction); + + facets_set_additional_options(facets, + ((lqs->rq.data_only) ? FACETS_OPTION_DATA_ONLY : 0) | + ((lqs->rq.delta) ? FACETS_OPTION_SHOW_DELTAS : 0)); + + facets_set_items(facets, lqs->rq.entries); + facets_set_query(facets, lqs->rq.query); + + if(lqs->rq.slice && have_slice) + facets_enable_slice_mode(facets); + else + lqs->rq.slice = false; + + if(lqs->rq.histogram) { + if(lqs->rq.fields_are_ids) + facets_set_timeframe_and_histogram_by_id(facets, lqs->rq.histogram, lqs->rq.after_ut, lqs->rq.before_ut); + else + facets_set_timeframe_and_histogram_by_name(facets, lqs->rq.histogram, lqs->rq.after_ut, lqs->rq.before_ut); + } + else if(default_histogram) + facets_set_timeframe_and_histogram_by_name(facets, default_histogram, lqs->rq.after_ut, lqs->rq.before_ut); + + // complete the request object + buffer_json_member_add_boolean(wb, LQS_PARAMETER_INFO, lqs->rq.info); + buffer_json_member_add_boolean(wb, LQS_PARAMETER_SLICE, lqs->rq.slice); + buffer_json_member_add_boolean(wb, LQS_PARAMETER_DATA_ONLY, lqs->rq.data_only); + buffer_json_member_add_boolean(wb, LQS_PARAMETER_DELTA, lqs->rq.delta); + buffer_json_member_add_boolean(wb, LQS_PARAMETER_TAIL, lqs->rq.tail); + buffer_json_member_add_uint64(wb, LQS_PARAMETER_SAMPLING, lqs->rq.sampling); + buffer_json_member_add_uint64(wb, "source_type", lqs->rq.source_type); + buffer_json_member_add_uint64(wb, LQS_PARAMETER_AFTER, lqs->rq.after_ut / USEC_PER_SEC); + buffer_json_member_add_uint64(wb, LQS_PARAMETER_BEFORE, lqs->rq.before_ut / USEC_PER_SEC); + buffer_json_member_add_uint64(wb, "if_modified_since", lqs->rq.if_modified_since); + buffer_json_member_add_uint64(wb, LQS_PARAMETER_ANCHOR, lqs->rq.anchor); + buffer_json_member_add_string(wb, LQS_PARAMETER_DIRECTION, lqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward"); + buffer_json_member_add_uint64(wb, LQS_PARAMETER_LAST, lqs->rq.entries); + buffer_json_member_add_string(wb, LQS_PARAMETER_QUERY, lqs->rq.query); + buffer_json_member_add_string(wb, LQS_PARAMETER_HISTOGRAM, lqs->rq.histogram); + buffer_json_object_close(wb); // request + + return true; +} + +static inline void lqs_cleanup(LOGS_QUERY_STATUS *lqs) { + freez((void *)lqs->rq.query); + freez((void *)lqs->rq.histogram); + simple_pattern_free(lqs->rq.sources); + facets_destroy(lqs->facets); +} + +#endif //NETDATA_LOGS_QUERY_STATUS_H diff --git a/src/libnetdata/functions_evloop/functions_evloop.c b/src/libnetdata/functions_evloop/functions_evloop.c index 5000d038f..fd0061844 100644 --- a/src/libnetdata/functions_evloop/functions_evloop.c +++ b/src/libnetdata/functions_evloop/functions_evloop.c @@ -137,6 +137,8 @@ static void worker_add_job(struct functions_evloop_globals *wg, const char *keyw function?function:"(unset)"); } else { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "WORKER JOB WITH PAYLOAD '%s'", payload ? buffer_tostring(payload) : "NONE"); + int timeout = str2i(timeout_s); const char *msg = "No function with this name found"; @@ -222,6 +224,8 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { char *s = (char *)buffer_tostring(buffer); if(strstr(&s[deferred.last_len], PLUGINSD_CALL_FUNCTION_PAYLOAD_END "\n") != NULL) { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION PAYLOAD END"); + if(deferred.last_len > 0) // remove the trailing newline from the buffer deferred.last_len--; @@ -249,11 +253,12 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { } char *words[MAX_FUNCTION_PARAMETERS] = { NULL }; - size_t num_words = quoted_strings_splitter_pluginsd((char *)buffer_tostring(buffer), words, MAX_FUNCTION_PARAMETERS); + size_t num_words = quoted_strings_splitter_whitespace((char *)buffer_tostring(buffer), words, MAX_FUNCTION_PARAMETERS); const char *keyword = get_word(words, num_words, 0); if(keyword && (strcmp(keyword, PLUGINSD_CALL_FUNCTION) == 0)) { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION CALL"); char *transaction = get_word(words, num_words, 1); char *timeout_s = get_word(words, num_words, 2); char *function = get_word(words, num_words, 3); @@ -262,6 +267,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { worker_add_job(wg, keyword, transaction, function, timeout_s, NULL, access, source); } else if(keyword && (strcmp(keyword, PLUGINSD_CALL_FUNCTION_PAYLOAD_BEGIN) == 0)) { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION PAYLOAD CALL"); char *transaction = get_word(words, num_words, 1); char *timeout_s = get_word(words, num_words, 2); char *function = get_word(words, num_words, 3); @@ -279,6 +285,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { deferred.enabled = true; } else if(keyword && strcmp(keyword, PLUGINSD_CALL_FUNCTION_CANCEL) == 0) { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION CANCEL"); char *transaction = get_word(words, num_words, 1); const DICTIONARY_ITEM *acquired = dictionary_get_and_acquire_item(wg->worker_queue, transaction); if(acquired) { @@ -292,6 +299,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received CANCEL for transaction '%s', but it not available here", transaction); } else if(keyword && strcmp(keyword, PLUGINSD_CALL_FUNCTION_PROGRESS) == 0) { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION PROGRESS"); char *transaction = get_word(words, num_words, 1); const DICTIONARY_ITEM *acquired = dictionary_get_and_acquire_item(wg->worker_queue, transaction); if(acquired) { @@ -305,7 +313,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received PROGRESS for transaction '%s', but it not available here", transaction); } else - nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received unknown command: %s", keyword?keyword:"(unset)"); + nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received unknown command: %s", keyword ? keyword : "(unset)"); buffer_flush(buffer); } diff --git a/src/libnetdata/functions_evloop/functions_evloop.h b/src/libnetdata/functions_evloop/functions_evloop.h index 5c575bd17..35defe355 100644 --- a/src/libnetdata/functions_evloop/functions_evloop.h +++ b/src/libnetdata/functions_evloop/functions_evloop.h @@ -71,6 +71,14 @@ #define PLUGINSD_KEYWORD_CONFIG_ACTION_STATUS "status" #define PLUGINSD_FUNCTION_CONFIG "config" +// claiming +#define PLUGINSD_KEYWORD_NODE_ID "NODE_ID" +#define PLUGINSD_KEYWORD_CLAIMED_ID "CLAIMED_ID" + +#define PLUGINSD_KEYWORD_JSON "JSON" +#define PLUGINSD_KEYWORD_JSON_END "JSON_PAYLOAD_END" +#define PLUGINSD_KEYWORD_STREAM_PATH "STREAM_PATH" + typedef void (*functions_evloop_worker_execute_t)(const char *transaction, char *function, usec_t *stop_monotonic_ut, bool *cancelled, BUFFER *payload, HTTP_ACCESS access, const char *source, void *data); @@ -125,9 +133,13 @@ static inline void pluginsd_function_json_error_to_stdout(const char *transactio fflush(stdout); } -static inline void pluginsd_function_result_to_stdout(const char *transaction, int code, const char *content_type, time_t expires, BUFFER *result) { - pluginsd_function_result_begin_to_stdout(transaction, code, content_type, expires); +static inline void pluginsd_function_result_to_stdout(const char *transaction, BUFFER *result) { + pluginsd_function_result_begin_to_stdout(transaction, result->response_code, + content_type_id2string(result->content_type), + result->expires); + fwrite(buffer_tostring(result), buffer_strlen(result), 1, stdout); + pluginsd_function_result_end_to_stdout(); fflush(stdout); } diff --git a/src/libnetdata/http/content_type.c b/src/libnetdata/http/content_type.c index 3e388a1da..e8f273912 100644 --- a/src/libnetdata/http/content_type.c +++ b/src/libnetdata/http/content_type.c @@ -10,14 +10,14 @@ static struct { const char *options; } content_types[] = { // primary - preferred during id-to-string conversions - { .format = "text/html", CT_TEXT_HTML, true }, + { .format = "application/json", CT_APPLICATION_JSON, true }, { .format = "text/plain", CT_TEXT_PLAIN, true }, + { .format = "text/html", CT_TEXT_HTML, true }, { .format = "text/css", CT_TEXT_CSS, true }, { .format = "text/yaml", CT_TEXT_YAML, true }, { .format = "application/yaml", CT_APPLICATION_YAML, true }, { .format = "text/xml", CT_TEXT_XML, true }, { .format = "text/xsl", CT_TEXT_XSL, true }, - { .format = "application/json", CT_APPLICATION_JSON, true }, { .format = "application/xml", CT_APPLICATION_XML, true }, { .format = "application/javascript", CT_APPLICATION_X_JAVASCRIPT, true }, { .format = "application/octet-stream", CT_APPLICATION_OCTET_STREAM, false }, @@ -42,16 +42,16 @@ static struct { // secondary - overlapping with primary - { .format = "text/plain", CT_PROMETHEUS, false, "version=0.0.4" }, - { .format = "prometheus", CT_PROMETHEUS }, - { .format = "text", CT_TEXT_PLAIN }, - { .format = "txt", CT_TEXT_PLAIN }, - { .format = "json", CT_APPLICATION_JSON }, - { .format = "html", CT_TEXT_HTML }, - { .format = "xml", CT_APPLICATION_XML }, + { .format = "text/plain", CT_PROMETHEUS, true, "version=0.0.4" }, + { .format = "prometheus", CT_PROMETHEUS, true }, + { .format = "text", CT_TEXT_PLAIN, true }, + { .format = "txt", CT_TEXT_PLAIN, true }, + { .format = "json", CT_APPLICATION_JSON, true }, + { .format = "html", CT_TEXT_HTML, true }, + { .format = "xml", CT_APPLICATION_XML, true }, // terminator - { .format = NULL, CT_TEXT_PLAIN }, + { .format = NULL, CT_TEXT_PLAIN, true }, }; HTTP_CONTENT_TYPE content_type_string2id(const char *format) { diff --git a/src/libnetdata/http/http_access.c b/src/libnetdata/http/http_access.c index 5be63bb19..398015cd3 100644 --- a/src/libnetdata/http/http_access.c +++ b/src/libnetdata/http/http_access.c @@ -3,24 +3,24 @@ #include "../libnetdata.h" static struct { - HTTP_USER_ROLE access; + HTTP_USER_ROLE role; const char *name; } user_roles[] = { - { .access = HTTP_USER_ROLE_NONE, .name = "none" }, - { .access = HTTP_USER_ROLE_ADMIN, .name = "admin" }, - { .access = HTTP_USER_ROLE_MANAGER, .name = "manager" }, - { .access = HTTP_USER_ROLE_TROUBLESHOOTER, .name = "troubleshooter" }, - { .access = HTTP_USER_ROLE_OBSERVER, .name = "observer" }, - { .access = HTTP_USER_ROLE_MEMBER, .name = "member" }, - { .access = HTTP_USER_ROLE_BILLING, .name = "billing" }, - { .access = HTTP_USER_ROLE_ANY, .name = "any" }, - - { .access = HTTP_USER_ROLE_MEMBER, .name = "members" }, - { .access = HTTP_USER_ROLE_ADMIN, .name = "admins" }, - { .access = HTTP_USER_ROLE_ANY, .name = "all" }, + { .role = HTTP_USER_ROLE_NONE, .name = "none" }, + { .role = HTTP_USER_ROLE_ADMIN, .name = "admin" }, + { .role = HTTP_USER_ROLE_MANAGER, .name = "manager" }, + { .role = HTTP_USER_ROLE_TROUBLESHOOTER, .name = "troubleshooter" }, + { .role = HTTP_USER_ROLE_OBSERVER, .name = "observer" }, + { .role = HTTP_USER_ROLE_MEMBER, .name = "member" }, + { .role = HTTP_USER_ROLE_BILLING, .name = "billing" }, + { .role = HTTP_USER_ROLE_ANY, .name = "any" }, + + { .role = HTTP_USER_ROLE_MEMBER, .name = "members" }, + { .role = HTTP_USER_ROLE_ADMIN, .name = "admins" }, + { .role = HTTP_USER_ROLE_ANY, .name = "all" }, // terminator - { .access = 0, .name = NULL }, + { .role = 0, .name = NULL }, }; HTTP_USER_ROLE http_user_role2id(const char *role) { @@ -29,7 +29,7 @@ HTTP_USER_ROLE http_user_role2id(const char *role) { for(size_t i = 0; user_roles[i].name ;i++) { if(strcmp(user_roles[i].name, role) == 0) - return user_roles[i].access; + return user_roles[i].role; } nd_log(NDLS_DAEMON, NDLP_WARNING, "HTTP user role '%s' is not valid", role); @@ -38,7 +38,7 @@ HTTP_USER_ROLE http_user_role2id(const char *role) { const char *http_id2user_role(HTTP_USER_ROLE role) { for(size_t i = 0; user_roles[i].name ;i++) { - if(role == user_roles[i].access) + if(role == user_roles[i].role) return user_roles[i].name; } diff --git a/src/libnetdata/http/http_access.h b/src/libnetdata/http/http_access.h index afc2e1dc7..00929f9b4 100644 --- a/src/libnetdata/http/http_access.h +++ b/src/libnetdata/http/http_access.h @@ -93,12 +93,16 @@ typedef enum __attribute__((packed)) { HTTP_ACL_WEBRTC = (1 << 6), // from WebRTC // HTTP_ACL_API takes the following additional ACLs, based on pattern matching of the client IP - HTTP_ACL_DASHBOARD = (1 << 10), - HTTP_ACL_REGISTRY = (1 << 11), - HTTP_ACL_BADGES = (1 << 12), - HTTP_ACL_MANAGEMENT = (1 << 13), - HTTP_ACL_STREAMING = (1 << 14), - HTTP_ACL_NETDATACONF = (1 << 15), + HTTP_ACL_METRICS = (1 << 10), + HTTP_ACL_FUNCTIONS = (1 << 11), + HTTP_ACL_NODES = (1 << 12), + HTTP_ACL_ALERTS = (1 << 13), + HTTP_ACL_DYNCFG = (1 << 14), + HTTP_ACL_REGISTRY = (1 << 15), + HTTP_ACL_BADGES = (1 << 16), + HTTP_ACL_MANAGEMENT = (1 << 17), + HTTP_ACL_STREAMING = (1 << 18), + HTTP_ACL_NETDATACONF = (1 << 19), // SSL related HTTP_ACL_SSL_OPTIONAL = (1 << 28), @@ -106,6 +110,14 @@ typedef enum __attribute__((packed)) { HTTP_ACL_SSL_DEFAULT = (1 << 30), } HTTP_ACL; +#define HTTP_ACL_DASHBOARD (HTTP_ACL)( \ + HTTP_ACL_METRICS \ + | HTTP_ACL_FUNCTIONS \ + | HTTP_ACL_ALERTS \ + | HTTP_ACL_NODES \ + | HTTP_ACL_DYNCFG \ + ) + #define HTTP_ACL_TRANSPORTS (HTTP_ACL)( \ HTTP_ACL_API \ | HTTP_ACL_API_UDP \ @@ -121,7 +133,11 @@ typedef enum __attribute__((packed)) { ) #define HTTP_ACL_ALL_FEATURES (HTTP_ACL)( \ - HTTP_ACL_DASHBOARD \ + HTTP_ACL_METRICS \ + | HTTP_ACL_FUNCTIONS \ + | HTTP_ACL_NODES \ + | HTTP_ACL_ALERTS \ + | HTTP_ACL_DYNCFG \ | HTTP_ACL_REGISTRY \ | HTTP_ACL_BADGES \ | HTTP_ACL_MANAGEMENT \ @@ -129,20 +145,24 @@ typedef enum __attribute__((packed)) { | HTTP_ACL_NETDATACONF \ ) +#define HTTP_ACL_ACLK_LICENSE_MANAGER (HTTP_ACL)( \ + HTTP_ACL_NODES \ +) + #ifdef NETDATA_DEV_MODE #define ACL_DEV_OPEN_ACCESS HTTP_ACL_NOCHECK #else #define ACL_DEV_OPEN_ACCESS 0 #endif -#define http_can_access_dashboard(w) ((w)->acl & HTTP_ACL_DASHBOARD) -#define http_can_access_registry(w) ((w)->acl & HTTP_ACL_REGISTRY) -#define http_can_access_badges(w) ((w)->acl & HTTP_ACL_BADGES) -#define http_can_access_mgmt(w) ((w)->acl & HTTP_ACL_MANAGEMENT) -#define http_can_access_stream(w) ((w)->acl & HTTP_ACL_STREAMING) -#define http_can_access_netdataconf(w) ((w)->acl & HTTP_ACL_NETDATACONF) -#define http_is_using_ssl_optional(w) ((w)->port_acl & HTTP_ACL_SSL_OPTIONAL) -#define http_is_using_ssl_force(w) ((w)->port_acl & HTTP_ACL_SSL_FORCE) -#define http_is_using_ssl_default(w) ((w)->port_acl & HTTP_ACL_SSL_DEFAULT) +#define http_can_access_dashboard(w) (((w)->acl & HTTP_ACL_DASHBOARD) == HTTP_ACL_DASHBOARD) +#define http_can_access_registry(w) (((w)->acl & HTTP_ACL_REGISTRY) == HTTP_ACL_REGISTRY) +#define http_can_access_badges(w) (((w)->acl & HTTP_ACL_BADGES) == HTTP_ACL_BADGES) +#define http_can_access_mgmt(w) (((w)->acl & HTTP_ACL_MANAGEMENT) == HTTP_ACL_MANAGEMENT) +#define http_can_access_stream(w) (((w)->acl & HTTP_ACL_STREAMING) == HTTP_ACL_STREAMING) +#define http_can_access_netdataconf(w) (((w)->acl & HTTP_ACL_NETDATACONF) == HTTP_ACL_NETDATACONF) +#define http_is_using_ssl_optional(w) (((w)->port_acl & HTTP_ACL_SSL_OPTIONAL) == HTTP_ACL_SSL_OPTIONAL) +#define http_is_using_ssl_force(w) (((w)->port_acl & HTTP_ACL_SSL_FORCE) == HTTP_ACL_SSL_FORCE) +#define http_is_using_ssl_default(w) (((w)->port_acl & HTTP_ACL_SSL_DEFAULT) == HTTP_ACL_SSL_DEFAULT) #endif //NETDATA_HTTP_ACCESS_H diff --git a/src/libnetdata/inlined.h b/src/libnetdata/inlined.h index 6b71590c9..50bc5e269 100644 --- a/src/libnetdata/inlined.h +++ b/src/libnetdata/inlined.h @@ -106,17 +106,6 @@ static inline uint64_t murmur64(uint64_t k) { return k; } -static inline size_t indexing_partition(Word_t ptr, Word_t modulo) __attribute__((const)); -static inline size_t indexing_partition(Word_t ptr, Word_t modulo) { -#ifdef ENV64BIT - uint64_t hash = murmur64(ptr); - return hash % modulo; -#else - uint32_t hash = murmur32(ptr); - return hash % modulo; -#endif -} - static inline unsigned int str2u(const char *s) { unsigned int n = 0; @@ -506,6 +495,43 @@ static inline int read_txt_file(const char *filename, char *buffer, size_t size) return 0; } +static inline bool read_txt_file_to_buffer(const char *filename, BUFFER *wb, size_t max_size) { + // Open the file + int fd = open(filename, O_RDONLY | O_CLOEXEC); + if (fd == -1) + return false; + + // Get the file size + struct stat st; + if (fstat(fd, &st) == -1) { + close(fd); + return false; + } + + size_t file_size = st.st_size; + + // Check if the file size exceeds the maximum allowed size + if (file_size > max_size) { + close(fd); + return false; // File size too large + } + + buffer_need_bytes(wb, file_size + 1); + + // Read the file contents into the buffer + ssize_t r = read(fd, &wb->buffer[wb->len], file_size); + if (r != (ssize_t)file_size) { + close(fd); + return false; // Read error + } + wb->len = r; + + // Close the file descriptor + close(fd); + + return true; // Success +} + static inline int read_proc_cmdline(const char *filename, char *buffer, size_t size) { if (unlikely(!size)) return 3; diff --git a/src/libnetdata/json/README.md b/src/libnetdata/json/README.md index 9ae5ff382..21cd42a79 100644 --- a/src/libnetdata/json/README.md +++ b/src/libnetdata/json/README.md @@ -1,12 +1,3 @@ -<!-- -title: "json" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/json/README.md -sidebar_label: "json" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # json `json` contains a parser for json strings, based on `jsmn` (<https://github.com/zserge/jsmn>), but case you have installed the JSON-C library, the installation script will prefer it, you can also force its use with `--enable-jsonc` in the compilation time. diff --git a/src/libnetdata/json/json-c-parser-inline.c b/src/libnetdata/json/json-c-parser-inline.c new file mode 100644 index 000000000..a17847a3e --- /dev/null +++ b/src/libnetdata/json/json-c-parser-inline.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +int rrd_call_function_error(BUFFER *wb, const char *msg, int code) { + buffer_reset(wb); + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); + buffer_json_member_add_int64(wb, "status", code); + buffer_json_member_add_string(wb, "error_message", msg); + buffer_json_finalize(wb); + wb->date = now_realtime_sec(); + wb->expires = wb->date + 1; + wb->response_code = code; + return code; +} + +struct json_object *json_parse_function_payload_or_error(BUFFER *output, BUFFER *payload, int *code, json_parse_function_payload_t cb, void *cb_data) { + if(!payload || !buffer_strlen(payload)) { + *code = rrd_call_function_error(output, "No payload given, but a payload is required for this feature.", HTTP_RESP_BAD_REQUEST); + return NULL; + } + + struct json_tokener *tokener = json_tokener_new(); + if (!tokener) { + *code = rrd_call_function_error(output, "Failed to initialize json parser.", HTTP_RESP_INTERNAL_SERVER_ERROR); + return NULL; + } + + struct json_object *jobj = json_tokener_parse_ex(tokener, buffer_tostring(payload), (int)buffer_strlen(payload)); + if (json_tokener_get_error(tokener) != json_tokener_success) { + const char *error_msg = json_tokener_error_desc(json_tokener_get_error(tokener)); + char tmp[strlen(error_msg) + 100]; + snprintf(tmp, sizeof(tmp), "JSON parser failed: %s", error_msg); + json_tokener_free(tokener); + *code = rrd_call_function_error(output, tmp, HTTP_RESP_INTERNAL_SERVER_ERROR); + return NULL; + } + json_tokener_free(tokener); + + CLEAN_BUFFER *error = buffer_create(0, NULL); + if(!cb(jobj, "", cb_data, error)) { + char tmp[buffer_strlen(error) + 100]; + snprintfz(tmp, sizeof(tmp), "JSON parser failed: %s", buffer_tostring(error)); + *code = rrd_call_function_error(output, tmp, HTTP_RESP_BAD_REQUEST); + json_object_put(jobj); + return NULL; + } + + *code = HTTP_RESP_OK; + + return jobj; +} diff --git a/src/libnetdata/json/json-c-parser-inline.h b/src/libnetdata/json/json-c-parser-inline.h index c1d60ca45..e51cb232e 100644 --- a/src/libnetdata/json/json-c-parser-inline.h +++ b/src/libnetdata/json/json-c-parser-inline.h @@ -25,6 +25,45 @@ } \ } while(0) +#define JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, member, dst, error, required) do { \ + json_object *_j; \ + if (json_object_object_get_ex(jobj, member, &_j) && json_object_is_type(_j, json_type_string)) { \ + freez((void *)dst); \ + dst = strdupz(json_object_get_string(_j)); \ + } \ + else if(required) { \ + buffer_sprintf(error, "missing or invalid type for '%s.%s' string", path, member); \ + return false; \ + } \ +} while(0) + +#define JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, member, dst, error, required) do { \ + json_object *_j; \ + if (json_object_object_get_ex(jobj, member, &_j)) { \ + if (json_object_is_type(_j, json_type_string)) { \ + if (uuid_parse(json_object_get_string(_j), dst) != 0) { \ + if(required) { \ + buffer_sprintf(error, "invalid UUID '%s.%s'", path, member); \ + return false; \ + } \ + else \ + uuid_clear(dst); \ + } \ + } \ + else if (json_object_is_type(_j, json_type_null)) { \ + uuid_clear(dst); \ + } \ + else if (required) { \ + buffer_sprintf(error, "expected UUID or null '%s.%s'", path, member); \ + return false; \ + } \ + } \ + else if (required) { \ + buffer_sprintf(error, "missing UUID '%s.%s'", path, member); \ + return false; \ + } \ +} while(0) + #define JSONC_PARSE_TXT2BUFFER_OR_ERROR_AND_RETURN(jobj, path, member, dst, error, required) do { \ json_object *_j; \ if (json_object_object_get_ex(jobj, member, &_j) && json_object_is_type(_j, json_type_string)) { \ @@ -111,7 +150,6 @@ } \ } while(0) - #define JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, member, converter, dst, error, required) do { \ json_object *_j; \ if (json_object_object_get_ex(jobj, member, &_j) && json_object_is_type(_j, json_type_string)) \ @@ -122,11 +160,11 @@ } \ } while(0) -#define JSONC_PARSE_INT_OR_ERROR_AND_RETURN(jobj, path, member, dst, error, required) do { \ +#define JSONC_PARSE_INT64_OR_ERROR_AND_RETURN(jobj, path, member, dst, error, required) do { \ json_object *_j; \ if (json_object_object_get_ex(jobj, member, &_j)) { \ if (_j != NULL && json_object_is_type(_j, json_type_int)) \ - dst = json_object_get_int(_j); \ + dst = json_object_get_int64(_j); \ else if (_j != NULL && json_object_is_type(_j, json_type_double)) \ dst = (typeof(dst))json_object_get_double(_j); \ else if (_j == NULL) \ @@ -136,7 +174,26 @@ return false; \ } \ } else if(required) { \ - buffer_sprintf(error, "missing or invalid type (expected int value or null) for '%s.%s'", path, member); \ + buffer_sprintf(error, "missing or invalid type (expected int value or null) for '%s.%s'", path, member);\ + return false; \ + } \ +} while(0) + +#define JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, member, dst, error, required) do { \ + json_object *_j; \ + if (json_object_object_get_ex(jobj, member, &_j)) { \ + if (_j != NULL && json_object_is_type(_j, json_type_int)) \ + dst = json_object_get_uint64(_j); \ + else if (_j != NULL && json_object_is_type(_j, json_type_double)) \ + dst = (typeof(dst))json_object_get_double(_j); \ + else if (_j == NULL) \ + dst = 0; \ + else { \ + buffer_sprintf(error, "not supported type (expected int) for '%s.%s'", path, member); \ + return false; \ + } \ + } else if(required) { \ + buffer_sprintf(error, "missing or invalid type (expected int value or null) for '%s.%s'", path, member);\ return false; \ } \ } while(0) @@ -174,4 +231,8 @@ } \ } while(0) +typedef bool (*json_parse_function_payload_t)(json_object *jobj, const char *path, void *data, BUFFER *error); +int rrd_call_function_error(BUFFER *wb, const char *msg, int code); +struct json_object *json_parse_function_payload_or_error(BUFFER *output, BUFFER *payload, int *code, json_parse_function_payload_t cb, void *cb_data); + #endif //NETDATA_JSON_C_PARSER_INLINE_H diff --git a/src/libnetdata/libjudy/judy-malloc.c b/src/libnetdata/libjudy/judy-malloc.c new file mode 100644 index 000000000..ec736393d --- /dev/null +++ b/src/libnetdata/libjudy/judy-malloc.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "judy-malloc.h" + +#define MAX_JUDY_SIZE_TO_ARAL 24 +static bool judy_sizes_config[MAX_JUDY_SIZE_TO_ARAL + 1] = { + [3] = true, + [4] = true, + [5] = true, + [6] = true, + [7] = true, + [8] = true, + [10] = true, + [11] = true, + [15] = true, + [23] = true, +}; +static ARAL *judy_sizes_aral[MAX_JUDY_SIZE_TO_ARAL + 1] = {}; + +struct aral_statistics judy_sizes_aral_statistics = {}; + +__attribute__((constructor)) void aral_judy_init(void) { + for(size_t Words = 0; Words <= MAX_JUDY_SIZE_TO_ARAL; Words++) + if(judy_sizes_config[Words]) { + char buf[30+1]; + snprintfz(buf, sizeof(buf) - 1, "judy-%zu", Words * sizeof(Word_t)); + judy_sizes_aral[Words] = aral_create( + buf, + Words * sizeof(Word_t), + 0, + 65536, + &judy_sizes_aral_statistics, + NULL, NULL, false, false); + } +} + +size_t judy_aral_overhead(void) { + return aral_overhead_from_stats(&judy_sizes_aral_statistics); +} + +size_t judy_aral_structures(void) { + return aral_structures_from_stats(&judy_sizes_aral_statistics); +} + +static ARAL *judy_size_aral(Word_t Words) { + if(Words <= MAX_JUDY_SIZE_TO_ARAL && judy_sizes_aral[Words]) + return judy_sizes_aral[Words]; + + return NULL; +} + +inline Word_t JudyMalloc(Word_t Words) { + Word_t Addr; + + ARAL *ar = judy_size_aral(Words); + if(ar) + Addr = (Word_t) aral_mallocz(ar); + else + Addr = (Word_t) mallocz(Words * sizeof(Word_t)); + + return(Addr); +} + +inline void JudyFree(void * PWord, Word_t Words) { + ARAL *ar = judy_size_aral(Words); + if(ar) + aral_freez(ar, PWord); + else + freez(PWord); +} + +Word_t JudyMallocVirtual(Word_t Words) { + return JudyMalloc(Words); +} + +void JudyFreeVirtual(void * PWord, Word_t Words) { + JudyFree(PWord, Words); +} diff --git a/src/libnetdata/libjudy/judy-malloc.h b/src/libnetdata/libjudy/judy-malloc.h new file mode 100644 index 000000000..65cba982b --- /dev/null +++ b/src/libnetdata/libjudy/judy-malloc.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_JUDY_MALLOC_H +#define NETDATA_JUDY_MALLOC_H + +#include "../libnetdata.h" + +size_t judy_aral_overhead(void); +size_t judy_aral_structures(void); + +#endif //NETDATA_JUDY_MALLOC_H diff --git a/src/libnetdata/libnetdata.c b/src/libnetdata/libnetdata.c index b36a139d2..e21bf119d 100644 --- a/src/libnetdata/libnetdata.c +++ b/src/libnetdata/libnetdata.c @@ -2,6 +2,10 @@ #include "libnetdata.h" +#define MALLOC_ALIGNMENT (sizeof(uintptr_t) * 2) +#define size_t_atomic_count(op, var, size) __atomic_## op ##_fetch(&(var), size, __ATOMIC_RELAXED) +#define size_t_atomic_bytes(op, var, size) __atomic_## op ##_fetch(&(var), ((size) % MALLOC_ALIGNMENT)?((size) + MALLOC_ALIGNMENT - ((size) % MALLOC_ALIGNMENT)):(size), __ATOMIC_RELAXED) + #if !defined(MADV_DONTFORK) #define MADV_DONTFORK 0 #endif @@ -13,88 +17,13 @@ struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 }; #if defined(MADV_MERGEABLE) -int enable_ksm = 1; +int enable_ksm = CONFIG_BOOLEAN_AUTO; #else int enable_ksm = 0; #endif volatile sig_atomic_t netdata_exit = 0; -#define MAX_JUDY_SIZE_TO_ARAL 24 -static bool judy_sizes_config[MAX_JUDY_SIZE_TO_ARAL + 1] = { - [3] = true, - [4] = true, - [5] = true, - [6] = true, - [7] = true, - [8] = true, - [10] = true, - [11] = true, - [15] = true, - [23] = true, -}; -static ARAL *judy_sizes_aral[MAX_JUDY_SIZE_TO_ARAL + 1] = {}; - -struct aral_statistics judy_sizes_aral_statistics = {}; - -void aral_judy_init(void) { - for(size_t Words = 0; Words <= MAX_JUDY_SIZE_TO_ARAL; Words++) - if(judy_sizes_config[Words]) { - char buf[30+1]; - snprintfz(buf, sizeof(buf) - 1, "judy-%zu", Words * sizeof(Word_t)); - judy_sizes_aral[Words] = aral_create( - buf, - Words * sizeof(Word_t), - 0, - 65536, - &judy_sizes_aral_statistics, - NULL, NULL, false, false); - } -} - -size_t judy_aral_overhead(void) { - return aral_overhead_from_stats(&judy_sizes_aral_statistics); -} - -size_t judy_aral_structures(void) { - return aral_structures_from_stats(&judy_sizes_aral_statistics); -} - -static ARAL *judy_size_aral(Word_t Words) { - if(Words <= MAX_JUDY_SIZE_TO_ARAL && judy_sizes_aral[Words]) - return judy_sizes_aral[Words]; - - return NULL; -} - -inline Word_t JudyMalloc(Word_t Words) { - Word_t Addr; - - ARAL *ar = judy_size_aral(Words); - if(ar) - Addr = (Word_t) aral_mallocz(ar); - else - Addr = (Word_t) mallocz(Words * sizeof(Word_t)); - - return(Addr); -} - -inline void JudyFree(void * PWord, Word_t Words) { - ARAL *ar = judy_size_aral(Words); - if(ar) - aral_freez(ar, PWord); - else - freez(PWord); -} - -Word_t JudyMallocVirtual(Word_t Words) { - return JudyMalloc(Words); -} - -void JudyFreeVirtual(void * PWord, Word_t Words) { - JudyFree(PWord, Words); -} - // ---------------------------------------------------------------------------- // memory allocation functions that handle failures @@ -553,536 +482,6 @@ void json_fix_string(char *s) { } } -unsigned char netdata_map_chart_names[256] = { - [0] = '\0', // - [1] = '_', // - [2] = '_', // - [3] = '_', // - [4] = '_', // - [5] = '_', // - [6] = '_', // - [7] = '_', // - [8] = '_', // - [9] = '_', // - [10] = '_', // - [11] = '_', // - [12] = '_', // - [13] = '_', // - [14] = '_', // - [15] = '_', // - [16] = '_', // - [17] = '_', // - [18] = '_', // - [19] = '_', // - [20] = '_', // - [21] = '_', // - [22] = '_', // - [23] = '_', // - [24] = '_', // - [25] = '_', // - [26] = '_', // - [27] = '_', // - [28] = '_', // - [29] = '_', // - [30] = '_', // - [31] = '_', // - [32] = '_', // - [33] = '_', // ! - [34] = '_', // " - [35] = '_', // # - [36] = '_', // $ - [37] = '_', // % - [38] = '_', // & - [39] = '_', // ' - [40] = '_', // ( - [41] = '_', // ) - [42] = '_', // * - [43] = '_', // + - [44] = '.', // , - [45] = '-', // - - [46] = '.', // . - [47] = '/', // / - [48] = '0', // 0 - [49] = '1', // 1 - [50] = '2', // 2 - [51] = '3', // 3 - [52] = '4', // 4 - [53] = '5', // 5 - [54] = '6', // 6 - [55] = '7', // 7 - [56] = '8', // 8 - [57] = '9', // 9 - [58] = '_', // : - [59] = '_', // ; - [60] = '_', // < - [61] = '_', // = - [62] = '_', // > - [63] = '_', // ? - [64] = '_', // @ - [65] = 'a', // A - [66] = 'b', // B - [67] = 'c', // C - [68] = 'd', // D - [69] = 'e', // E - [70] = 'f', // F - [71] = 'g', // G - [72] = 'h', // H - [73] = 'i', // I - [74] = 'j', // J - [75] = 'k', // K - [76] = 'l', // L - [77] = 'm', // M - [78] = 'n', // N - [79] = 'o', // O - [80] = 'p', // P - [81] = 'q', // Q - [82] = 'r', // R - [83] = 's', // S - [84] = 't', // T - [85] = 'u', // U - [86] = 'v', // V - [87] = 'w', // W - [88] = 'x', // X - [89] = 'y', // Y - [90] = 'z', // Z - [91] = '_', // [ - [92] = '/', // backslash - [93] = '_', // ] - [94] = '_', // ^ - [95] = '_', // _ - [96] = '_', // ` - [97] = 'a', // a - [98] = 'b', // b - [99] = 'c', // c - [100] = 'd', // d - [101] = 'e', // e - [102] = 'f', // f - [103] = 'g', // g - [104] = 'h', // h - [105] = 'i', // i - [106] = 'j', // j - [107] = 'k', // k - [108] = 'l', // l - [109] = 'm', // m - [110] = 'n', // n - [111] = 'o', // o - [112] = 'p', // p - [113] = 'q', // q - [114] = 'r', // r - [115] = 's', // s - [116] = 't', // t - [117] = 'u', // u - [118] = 'v', // v - [119] = 'w', // w - [120] = 'x', // x - [121] = 'y', // y - [122] = 'z', // z - [123] = '_', // { - [124] = '_', // | - [125] = '_', // } - [126] = '_', // ~ - [127] = '_', // - [128] = '_', // - [129] = '_', // - [130] = '_', // - [131] = '_', // - [132] = '_', // - [133] = '_', // - [134] = '_', // - [135] = '_', // - [136] = '_', // - [137] = '_', // - [138] = '_', // - [139] = '_', // - [140] = '_', // - [141] = '_', // - [142] = '_', // - [143] = '_', // - [144] = '_', // - [145] = '_', // - [146] = '_', // - [147] = '_', // - [148] = '_', // - [149] = '_', // - [150] = '_', // - [151] = '_', // - [152] = '_', // - [153] = '_', // - [154] = '_', // - [155] = '_', // - [156] = '_', // - [157] = '_', // - [158] = '_', // - [159] = '_', // - [160] = '_', // - [161] = '_', // - [162] = '_', // - [163] = '_', // - [164] = '_', // - [165] = '_', // - [166] = '_', // - [167] = '_', // - [168] = '_', // - [169] = '_', // - [170] = '_', // - [171] = '_', // - [172] = '_', // - [173] = '_', // - [174] = '_', // - [175] = '_', // - [176] = '_', // - [177] = '_', // - [178] = '_', // - [179] = '_', // - [180] = '_', // - [181] = '_', // - [182] = '_', // - [183] = '_', // - [184] = '_', // - [185] = '_', // - [186] = '_', // - [187] = '_', // - [188] = '_', // - [189] = '_', // - [190] = '_', // - [191] = '_', // - [192] = '_', // - [193] = '_', // - [194] = '_', // - [195] = '_', // - [196] = '_', // - [197] = '_', // - [198] = '_', // - [199] = '_', // - [200] = '_', // - [201] = '_', // - [202] = '_', // - [203] = '_', // - [204] = '_', // - [205] = '_', // - [206] = '_', // - [207] = '_', // - [208] = '_', // - [209] = '_', // - [210] = '_', // - [211] = '_', // - [212] = '_', // - [213] = '_', // - [214] = '_', // - [215] = '_', // - [216] = '_', // - [217] = '_', // - [218] = '_', // - [219] = '_', // - [220] = '_', // - [221] = '_', // - [222] = '_', // - [223] = '_', // - [224] = '_', // - [225] = '_', // - [226] = '_', // - [227] = '_', // - [228] = '_', // - [229] = '_', // - [230] = '_', // - [231] = '_', // - [232] = '_', // - [233] = '_', // - [234] = '_', // - [235] = '_', // - [236] = '_', // - [237] = '_', // - [238] = '_', // - [239] = '_', // - [240] = '_', // - [241] = '_', // - [242] = '_', // - [243] = '_', // - [244] = '_', // - [245] = '_', // - [246] = '_', // - [247] = '_', // - [248] = '_', // - [249] = '_', // - [250] = '_', // - [251] = '_', // - [252] = '_', // - [253] = '_', // - [254] = '_', // - [255] = '_' // -}; - -// make sure the supplied string -// is good for a netdata chart/dimension ID/NAME -void netdata_fix_chart_name(char *s) { - while ((*s = netdata_map_chart_names[(unsigned char) *s])) s++; -} - -unsigned char netdata_map_chart_ids[256] = { - [0] = '\0', // - [1] = '_', // - [2] = '_', // - [3] = '_', // - [4] = '_', // - [5] = '_', // - [6] = '_', // - [7] = '_', // - [8] = '_', // - [9] = '_', // - [10] = '_', // - [11] = '_', // - [12] = '_', // - [13] = '_', // - [14] = '_', // - [15] = '_', // - [16] = '_', // - [17] = '_', // - [18] = '_', // - [19] = '_', // - [20] = '_', // - [21] = '_', // - [22] = '_', // - [23] = '_', // - [24] = '_', // - [25] = '_', // - [26] = '_', // - [27] = '_', // - [28] = '_', // - [29] = '_', // - [30] = '_', // - [31] = '_', // - [32] = '_', // - [33] = '_', // ! - [34] = '_', // " - [35] = '_', // # - [36] = '_', // $ - [37] = '_', // % - [38] = '_', // & - [39] = '_', // ' - [40] = '_', // ( - [41] = '_', // ) - [42] = '_', // * - [43] = '_', // + - [44] = '.', // , - [45] = '-', // - - [46] = '.', // . - [47] = '_', // / - [48] = '0', // 0 - [49] = '1', // 1 - [50] = '2', // 2 - [51] = '3', // 3 - [52] = '4', // 4 - [53] = '5', // 5 - [54] = '6', // 6 - [55] = '7', // 7 - [56] = '8', // 8 - [57] = '9', // 9 - [58] = '_', // : - [59] = '_', // ; - [60] = '_', // < - [61] = '_', // = - [62] = '_', // > - [63] = '_', // ? - [64] = '_', // @ - [65] = 'a', // A - [66] = 'b', // B - [67] = 'c', // C - [68] = 'd', // D - [69] = 'e', // E - [70] = 'f', // F - [71] = 'g', // G - [72] = 'h', // H - [73] = 'i', // I - [74] = 'j', // J - [75] = 'k', // K - [76] = 'l', // L - [77] = 'm', // M - [78] = 'n', // N - [79] = 'o', // O - [80] = 'p', // P - [81] = 'q', // Q - [82] = 'r', // R - [83] = 's', // S - [84] = 't', // T - [85] = 'u', // U - [86] = 'v', // V - [87] = 'w', // W - [88] = 'x', // X - [89] = 'y', // Y - [90] = 'z', // Z - [91] = '_', // [ - [92] = '_', // backslash - [93] = '_', // ] - [94] = '_', // ^ - [95] = '_', // _ - [96] = '_', // ` - [97] = 'a', // a - [98] = 'b', // b - [99] = 'c', // c - [100] = 'd', // d - [101] = 'e', // e - [102] = 'f', // f - [103] = 'g', // g - [104] = 'h', // h - [105] = 'i', // i - [106] = 'j', // j - [107] = 'k', // k - [108] = 'l', // l - [109] = 'm', // m - [110] = 'n', // n - [111] = 'o', // o - [112] = 'p', // p - [113] = 'q', // q - [114] = 'r', // r - [115] = 's', // s - [116] = 't', // t - [117] = 'u', // u - [118] = 'v', // v - [119] = 'w', // w - [120] = 'x', // x - [121] = 'y', // y - [122] = 'z', // z - [123] = '_', // { - [124] = '_', // | - [125] = '_', // } - [126] = '_', // ~ - [127] = '_', // - [128] = '_', // - [129] = '_', // - [130] = '_', // - [131] = '_', // - [132] = '_', // - [133] = '_', // - [134] = '_', // - [135] = '_', // - [136] = '_', // - [137] = '_', // - [138] = '_', // - [139] = '_', // - [140] = '_', // - [141] = '_', // - [142] = '_', // - [143] = '_', // - [144] = '_', // - [145] = '_', // - [146] = '_', // - [147] = '_', // - [148] = '_', // - [149] = '_', // - [150] = '_', // - [151] = '_', // - [152] = '_', // - [153] = '_', // - [154] = '_', // - [155] = '_', // - [156] = '_', // - [157] = '_', // - [158] = '_', // - [159] = '_', // - [160] = '_', // - [161] = '_', // - [162] = '_', // - [163] = '_', // - [164] = '_', // - [165] = '_', // - [166] = '_', // - [167] = '_', // - [168] = '_', // - [169] = '_', // - [170] = '_', // - [171] = '_', // - [172] = '_', // - [173] = '_', // - [174] = '_', // - [175] = '_', // - [176] = '_', // - [177] = '_', // - [178] = '_', // - [179] = '_', // - [180] = '_', // - [181] = '_', // - [182] = '_', // - [183] = '_', // - [184] = '_', // - [185] = '_', // - [186] = '_', // - [187] = '_', // - [188] = '_', // - [189] = '_', // - [190] = '_', // - [191] = '_', // - [192] = '_', // - [193] = '_', // - [194] = '_', // - [195] = '_', // - [196] = '_', // - [197] = '_', // - [198] = '_', // - [199] = '_', // - [200] = '_', // - [201] = '_', // - [202] = '_', // - [203] = '_', // - [204] = '_', // - [205] = '_', // - [206] = '_', // - [207] = '_', // - [208] = '_', // - [209] = '_', // - [210] = '_', // - [211] = '_', // - [212] = '_', // - [213] = '_', // - [214] = '_', // - [215] = '_', // - [216] = '_', // - [217] = '_', // - [218] = '_', // - [219] = '_', // - [220] = '_', // - [221] = '_', // - [222] = '_', // - [223] = '_', // - [224] = '_', // - [225] = '_', // - [226] = '_', // - [227] = '_', // - [228] = '_', // - [229] = '_', // - [230] = '_', // - [231] = '_', // - [232] = '_', // - [233] = '_', // - [234] = '_', // - [235] = '_', // - [236] = '_', // - [237] = '_', // - [238] = '_', // - [239] = '_', // - [240] = '_', // - [241] = '_', // - [242] = '_', // - [243] = '_', // - [244] = '_', // - [245] = '_', // - [246] = '_', // - [247] = '_', // - [248] = '_', // - [249] = '_', // - [250] = '_', // - [251] = '_', // - [252] = '_', // - [253] = '_', // - [254] = '_', // - [255] = '_' // -}; - -// make sure the supplied string -// is good for a netdata chart/dimension ID/NAME -void netdata_fix_chart_id(char *s) { - while ((*s = netdata_map_chart_ids[(unsigned char) *s])) s++; -} - static int memory_file_open(const char *filename, size_t size) { // netdata_log_info("memory_file_open('%s', %zu", filename, size); @@ -1302,315 +701,6 @@ int snprintfz(char *dst, size_t n, const char *fmt, ...) { return ret; } -static int is_procfs(const char *path, char **reason) { -#if defined(__APPLE__) || defined(__FreeBSD__) - (void)path; - (void)reason; -#else - struct statfs stat; - - if (statfs(path, &stat) == -1) { - if (reason) - *reason = "failed to statfs()"; - return -1; - } - -#if defined PROC_SUPER_MAGIC - if (stat.f_type != PROC_SUPER_MAGIC) { - if (reason) - *reason = "type is not procfs"; - return -1; - } -#endif - -#endif - - return 0; -} - -static int is_sysfs(const char *path, char **reason) { -#if defined(__APPLE__) || defined(__FreeBSD__) - (void)path; - (void)reason; -#else - struct statfs stat; - - if (statfs(path, &stat) == -1) { - if (reason) - *reason = "failed to statfs()"; - return -1; - } - -#if defined SYSFS_MAGIC - if (stat.f_type != SYSFS_MAGIC) { - if (reason) - *reason = "type is not sysfs"; - return -1; - } -#endif - -#endif - - return 0; -} - -int verify_netdata_host_prefix(bool log_msg) { - if(!netdata_configured_host_prefix) - netdata_configured_host_prefix = ""; - - if(!*netdata_configured_host_prefix) - return 0; - - char buffer[FILENAME_MAX + 1]; - char *path = netdata_configured_host_prefix; - char *reason = "unknown reason"; - errno_clear(); - - struct stat sb; - if (stat(path, &sb) == -1) { - reason = "failed to stat()"; - goto failed; - } - - if((sb.st_mode & S_IFMT) != S_IFDIR) { - errno = EINVAL; - reason = "is not a directory"; - goto failed; - } - - path = buffer; - snprintfz(path, FILENAME_MAX, "%s/proc", netdata_configured_host_prefix); - if(is_procfs(path, &reason) == -1) - goto failed; - - snprintfz(path, FILENAME_MAX, "%s/sys", netdata_configured_host_prefix); - if(is_sysfs(path, &reason) == -1) - goto failed; - - if (netdata_configured_host_prefix && *netdata_configured_host_prefix) { - if (log_msg) - netdata_log_info("Using host prefix directory '%s'", netdata_configured_host_prefix); - } - - return 0; - -failed: - if (log_msg) - netdata_log_error("Ignoring host prefix '%s': path '%s' %s", netdata_configured_host_prefix, path, reason); - netdata_configured_host_prefix = ""; - return -1; -} - -char *strdupz_path_subpath(const char *path, const char *subpath) { - if(unlikely(!path || !*path)) path = "."; - if(unlikely(!subpath)) subpath = ""; - - // skip trailing slashes in path - size_t len = strlen(path); - while(len > 0 && path[len - 1] == '/') len--; - - // skip leading slashes in subpath - while(subpath[0] == '/') subpath++; - - // if the last character in path is / and (there is a subpath or path is now empty) - // keep the trailing slash in path and remove the additional slash - char *slash = "/"; - if(path[len] == '/' && (*subpath || len == 0)) { - slash = ""; - len++; - } - else if(!*subpath) { - // there is no subpath - // no need for trailing slash - slash = ""; - } - - char buffer[FILENAME_MAX + 1]; - snprintfz(buffer, FILENAME_MAX, "%.*s%s%s", (int)len, path, slash, subpath); - return strdupz(buffer); -} - -int path_is_dir(const char *path, const char *subpath) { - char *s = strdupz_path_subpath(path, subpath); - - size_t max_links = 100; - - int is_dir = 0; - struct stat statbuf; - while(max_links-- && stat(s, &statbuf) == 0) { - if((statbuf.st_mode & S_IFMT) == S_IFDIR) { - is_dir = 1; - break; - } - else if((statbuf.st_mode & S_IFMT) == S_IFLNK) { - char buffer[FILENAME_MAX + 1]; - ssize_t l = readlink(s, buffer, FILENAME_MAX); - if(l > 0) { - buffer[l] = '\0'; - freez(s); - s = strdupz(buffer); - continue; - } - else { - is_dir = 0; - break; - } - } - else { - is_dir = 0; - break; - } - } - - freez(s); - return is_dir; -} - -int path_is_file(const char *path, const char *subpath) { - char *s = strdupz_path_subpath(path, subpath); - - size_t max_links = 100; - - int is_file = 0; - struct stat statbuf; - while(max_links-- && stat(s, &statbuf) == 0) { - if((statbuf.st_mode & S_IFMT) == S_IFREG) { - is_file = 1; - break; - } - else if((statbuf.st_mode & S_IFMT) == S_IFLNK) { - char buffer[FILENAME_MAX + 1]; - ssize_t l = readlink(s, buffer, FILENAME_MAX); - if(l > 0) { - buffer[l] = '\0'; - freez(s); - s = strdupz(buffer); - continue; - } - else { - is_file = 0; - break; - } - } - else { - is_file = 0; - break; - } - } - - freez(s); - return is_file; -} - -void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *subpath, int (*callback)(const char *filename, void *data, bool stock_config), void *data, size_t depth) { - if(depth > 3) { - netdata_log_error("CONFIG: Max directory depth reached while reading user path '%s', stock path '%s', subpath '%s'", user_path, stock_path, subpath); - return; - } - - if(!stock_path) - stock_path = user_path; - - char *udir = strdupz_path_subpath(user_path, subpath); - char *sdir = strdupz_path_subpath(stock_path, subpath); - - netdata_log_debug(D_HEALTH, "CONFIG traversing user-config directory '%s', stock config directory '%s'", udir, sdir); - - DIR *dir = opendir(udir); - if (!dir) { - netdata_log_error("CONFIG cannot open user-config directory '%s'.", udir); - } - else { - struct dirent *de = NULL; - while((de = readdir(dir))) { - if(de->d_type == DT_DIR || de->d_type == DT_LNK) { - if( !de->d_name[0] || - (de->d_name[0] == '.' && de->d_name[1] == '\0') || - (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') - ) { - netdata_log_debug(D_HEALTH, "CONFIG ignoring user-config directory '%s/%s'", udir, de->d_name); - continue; - } - - if(path_is_dir(udir, de->d_name)) { - recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1); - continue; - } - } - - if(de->d_type == DT_UNKNOWN || de->d_type == DT_REG || de->d_type == DT_LNK) { - size_t len = strlen(de->d_name); - if(path_is_file(udir, de->d_name) && - len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { - char *filename = strdupz_path_subpath(udir, de->d_name); - netdata_log_debug(D_HEALTH, "CONFIG calling callback for user file '%s'", filename); - callback(filename, data, false); - freez(filename); - continue; - } - } - - netdata_log_debug(D_HEALTH, "CONFIG ignoring user-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); - } - - closedir(dir); - } - - netdata_log_debug(D_HEALTH, "CONFIG traversing stock config directory '%s', user config directory '%s'", sdir, udir); - - dir = opendir(sdir); - if (!dir) { - netdata_log_error("CONFIG cannot open stock config directory '%s'.", sdir); - } - else { - if (strcmp(udir, sdir)) { - struct dirent *de = NULL; - while((de = readdir(dir))) { - if(de->d_type == DT_DIR || de->d_type == DT_LNK) { - if( !de->d_name[0] || - (de->d_name[0] == '.' && de->d_name[1] == '\0') || - (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') - ) { - netdata_log_debug(D_HEALTH, "CONFIG ignoring stock config directory '%s/%s'", sdir, de->d_name); - continue; - } - - if(path_is_dir(sdir, de->d_name)) { - // we recurse in stock subdirectory, only when there is no corresponding - // user subdirectory - to avoid reading the files twice - - if(!path_is_dir(udir, de->d_name)) - recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1); - - continue; - } - } - - if(de->d_type == DT_UNKNOWN || de->d_type == DT_REG || de->d_type == DT_LNK) { - size_t len = strlen(de->d_name); - if(path_is_file(sdir, de->d_name) && !path_is_file(udir, de->d_name) && - len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { - char *filename = strdupz_path_subpath(sdir, de->d_name); - netdata_log_debug(D_HEALTH, "CONFIG calling callback for stock file '%s'", filename); - callback(filename, data, true); - freez(filename); - continue; - } - - } - - netdata_log_debug(D_HEALTH, "CONFIG ignoring stock-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); - } - } - closedir(dir); - } - - netdata_log_debug(D_HEALTH, "CONFIG done traversing user-config directory '%s', stock config directory '%s'", udir, sdir); - - freez(udir); - freez(sdir); -} - // Returns the number of bytes read from the file if file_size is not NULL. // The actual buffer has an extra byte set to zero (not included in the count). char *read_by_filename(const char *filename, long *file_size) @@ -1618,34 +708,37 @@ char *read_by_filename(const char *filename, long *file_size) FILE *f = fopen(filename, "r"); if (!f) return NULL; + if (fseek(f, 0, SEEK_END) < 0) { fclose(f); return NULL; } + long size = ftell(f); if (size <= 0 || fseek(f, 0, SEEK_END) < 0) { fclose(f); return NULL; } + char *contents = callocz(size + 1, 1); - if (!contents) { - fclose(f); - return NULL; - } if (fseek(f, 0, SEEK_SET) < 0) { fclose(f); freez(contents); return NULL; } + size_t res = fread(contents, 1, size, f); if ( res != (size_t)size) { freez(contents); fclose(f); return NULL; } + fclose(f); + if (file_size) *file_size = size; + return contents; } @@ -1685,7 +778,7 @@ BUFFER *run_command_and_get_output_to_buffer(const char *command, int max_line_l POPEN_INSTANCE *pi = spawn_popen_run(command); if(pi) { char buffer[max_line_length + 1]; - while (fgets(buffer, max_line_length, pi->child_stdout_fp)) { + while (fgets(buffer, max_line_length, spawn_popen_stdout(pi))) { buffer[max_line_length] = '\0'; buffer_strcat(wb, buffer); } @@ -1705,7 +798,7 @@ bool run_command_and_copy_output_to_stdout(const char *command, int max_line_len if(pi) { char buffer[max_line_length + 1]; - while (fgets(buffer, max_line_length, pi->child_stdout_fp)) + while (fgets(buffer, max_line_length, spawn_popen_stdout(pi))) fprintf(stdout, "%s", buffer); spawn_popen_kill(pi); @@ -1831,7 +924,6 @@ void timing_action(TIMING_ACTION action, TIMING_STEP step) { } } -#ifdef ENABLE_HTTPS int hash256_string(const unsigned char *string, size_t size, char *hash) { EVP_MD_CTX *ctx; ctx = EVP_MD_CTX_create(); @@ -1856,7 +948,6 @@ int hash256_string(const unsigned char *string, size_t size, char *hash) { EVP_MD_CTX_destroy(ctx); return 1; } -#endif bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t now) { @@ -1953,52 +1044,93 @@ bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_ return (absolute_period_requested != 1); } -int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size) { - static const unsigned char base64_table[256] = { - ['A'] = 0, ['B'] = 1, ['C'] = 2, ['D'] = 3, ['E'] = 4, ['F'] = 5, ['G'] = 6, ['H'] = 7, - ['I'] = 8, ['J'] = 9, ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15, - ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23, - ['Y'] = 24, ['Z'] = 25, ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31, - ['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39, - ['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47, - ['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51, ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55, - ['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63, - [0 ... '+' - 1] = 255, - ['+' + 1 ... '/' - 1] = 255, - ['9' + 1 ... 'A' - 1] = 255, - ['Z' + 1 ... 'a' - 1] = 255, - ['z' + 1 ... 255] = 255 - }; - size_t count = 0; - unsigned int tmp = 0; - int i, bit; - - if (decoded_size < 1) - return 0; // Buffer size must be at least 1 for null termination - - for (i = 0, bit = 0; encoded[i]; i++) { - unsigned char value = base64_table[(unsigned char)encoded[i]]; - if (value > 63) - return -1; // Invalid character in input - - tmp = tmp << 6 | value; - if (++bit == 4) { - if (count + 3 >= decoded_size) break; // Stop decoding if buffer is full - decoded[count++] = (tmp >> 16) & 0xFF; - decoded[count++] = (tmp >> 8) & 0xFF; - decoded[count++] = tmp & 0xFF; - tmp = 0; - bit = 0; - } - } +#if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 +static inline EVP_ENCODE_CTX *EVP_ENCODE_CTX_new(void) +{ + EVP_ENCODE_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); - if (bit > 0 && count + 1 < decoded_size) { - tmp <<= 6 * (4 - bit); - if (bit > 2 && count + 1 < decoded_size) decoded[count++] = (tmp >> 16) & 0xFF; - if (bit > 3 && count + 1 < decoded_size) decoded[count++] = (tmp >> 8) & 0xFF; + if (ctx != NULL) { + memset(ctx, 0, sizeof(*ctx)); } + return ctx; +} + +static void EVP_ENCODE_CTX_free(EVP_ENCODE_CTX *ctx) +{ + OPENSSL_free(ctx); +} +#endif - decoded[count] = '\0'; // Null terminate the output string - return count; +int netdata_base64_decode(unsigned char *out, const unsigned char *in, const int in_len) +{ + int outl; + unsigned char remaining_data[256]; + + EVP_ENCODE_CTX *ctx = EVP_ENCODE_CTX_new(); + EVP_DecodeInit(ctx); + EVP_DecodeUpdate(ctx, out, &outl, in, in_len); + int remainder = 0; + EVP_DecodeFinal(ctx, remaining_data, &remainder); + EVP_ENCODE_CTX_free(ctx); + if (remainder) + return -1; + + return outl; } + +int netdata_base64_encode(unsigned char *encoded, const unsigned char *input, size_t input_size) +{ + return EVP_EncodeBlock(encoded, input, input_size); +} + +// Keep internal implementation +// int netdata_base64_decode_internal(const char *encoded, char *decoded, size_t decoded_size) { +// static const unsigned char base64_table[256] = { +// ['A'] = 0, ['B'] = 1, ['C'] = 2, ['D'] = 3, ['E'] = 4, ['F'] = 5, ['G'] = 6, ['H'] = 7, +// ['I'] = 8, ['J'] = 9, ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15, +// ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23, +// ['Y'] = 24, ['Z'] = 25, ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31, +// ['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39, +// ['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47, +// ['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51, ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55, +// ['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63, +// [0 ... '+' - 1] = 255, +// ['+' + 1 ... '/' - 1] = 255, +// ['9' + 1 ... 'A' - 1] = 255, +// ['Z' + 1 ... 'a' - 1] = 255, +// ['z' + 1 ... 255] = 255 +// }; +// +// size_t count = 0; +// unsigned int tmp = 0; +// int i, bit; +// +// if (decoded_size < 1) +// return 0; // Buffer size must be at least 1 for null termination +// +// for (i = 0, bit = 0; encoded[i]; i++) { +// unsigned char value = base64_table[(unsigned char)encoded[i]]; +// if (value > 63) +// return -1; // Invalid character in input +// +// tmp = tmp << 6 | value; +// if (++bit == 4) { +// if (count + 3 >= decoded_size) break; // Stop decoding if buffer is full +// decoded[count++] = (tmp >> 16) & 0xFF; +// decoded[count++] = (tmp >> 8) & 0xFF; +// decoded[count++] = tmp & 0xFF; +// tmp = 0; +// bit = 0; +// } +// } +// +// if (bit > 0 && count + 1 < decoded_size) { +// tmp <<= 6 * (4 - bit); +// if (bit > 2 && count + 1 < decoded_size) decoded[count++] = (tmp >> 16) & 0xFF; +// if (bit > 3 && count + 1 < decoded_size) decoded[count++] = (tmp >> 8) & 0xFF; +// } +// +// decoded[count] = '\0'; // Null terminate the output string +// return count; +// } diff --git a/src/libnetdata/libnetdata.h b/src/libnetdata/libnetdata.h index b4bddb70a..acee0675f 100644 --- a/src/libnetdata/libnetdata.h +++ b/src/libnetdata/libnetdata.h @@ -7,333 +7,19 @@ extern "C" { # endif -#include "config.h" - -#ifdef ENABLE_OPENSSL -#define ENABLE_HTTPS 1 -#endif - -#ifdef HAVE_LIBDATACHANNEL -#define ENABLE_WEBRTC 1 -#endif - -#define STRINGIFY(x) #x -#define TOSTRING(x) STRINGIFY(x) +#include "common.h" #define JUDYHS_INDEX_SIZE_ESTIMATE(key_bytes) (((key_bytes) + sizeof(Word_t) - 1) / sizeof(Word_t) * 4) -#if defined(NETDATA_DEV_MODE) && !defined(NETDATA_INTERNAL_CHECKS) -#define NETDATA_INTERNAL_CHECKS 1 -#endif - -#ifndef SIZEOF_VOID_P -#error SIZEOF_VOID_P is not defined -#endif - -#if SIZEOF_VOID_P == 4 -#define ENV32BIT 1 -#else -#define ENV64BIT 1 -#endif - // NETDATA_TRACE_ALLOCATIONS does not work under musl libc, so don't enable it //#if defined(NETDATA_INTERNAL_CHECKS) && !defined(NETDATA_TRACE_ALLOCATIONS) //#define NETDATA_TRACE_ALLOCATIONS 1 //#endif -#define MALLOC_ALIGNMENT (sizeof(uintptr_t) * 2) -#define size_t_atomic_count(op, var, size) __atomic_## op ##_fetch(&(var), size, __ATOMIC_RELAXED) -#define size_t_atomic_bytes(op, var, size) __atomic_## op ##_fetch(&(var), ((size) % MALLOC_ALIGNMENT)?((size) + MALLOC_ALIGNMENT - ((size) % MALLOC_ALIGNMENT)):(size), __ATOMIC_RELAXED) - -// ---------------------------------------------------------------------------- -// system include files for all netdata C programs - -/* select the memory allocator, based on autoconf findings */ -#if defined(ENABLE_JEMALLOC) - -#if defined(HAVE_JEMALLOC_JEMALLOC_H) -#include <jemalloc/jemalloc.h> -#else // !defined(HAVE_JEMALLOC_JEMALLOC_H) -#include <malloc.h> -#endif // !defined(HAVE_JEMALLOC_JEMALLOC_H) - -#elif defined(ENABLE_TCMALLOC) - -#include <google/tcmalloc.h> - -#else /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */ - -#if !(defined(__FreeBSD__) || defined(__APPLE__)) -#include <malloc.h> -#endif /* __FreeBSD__ || __APPLE__ */ - -#endif /* !defined(ENABLE_JEMALLOC) && !defined(ENABLE_TCMALLOC) */ - -// ---------------------------------------------------------------------------- - -#if defined(__FreeBSD__) -#include <pthread_np.h> -#define NETDATA_OS_TYPE "freebsd" -#elif defined(__APPLE__) -#define NETDATA_OS_TYPE "macos" -#elif defined(OS_WINDOWS) -#define NETDATA_OS_TYPE "windows" -#else -#define NETDATA_OS_TYPE "linux" -#endif /* __FreeBSD__, __APPLE__*/ - -#include <pthread.h> -#include <errno.h> -#include <stdbool.h> -#include <stdio.h> -#include <stdlib.h> -#include <stdarg.h> -#include <stddef.h> -#include <ctype.h> -#include <string.h> -#include <strings.h> -#include <libgen.h> -#include <dirent.h> -#include <fcntl.h> -#include <getopt.h> -#include <limits.h> -#include <locale.h> -#include <signal.h> -#include <sys/time.h> -#include <sys/types.h> -#include <time.h> -#include <unistd.h> -#include <uv.h> -#include <assert.h> - -#ifdef HAVE_ARPA_INET_H -#include <arpa/inet.h> -#endif - -#ifdef HAVE_NETINET_TCP_H -#include <netinet/tcp.h> -#endif - -#ifdef HAVE_SYS_IOCTL_H -#include <sys/ioctl.h> -#endif - -#ifdef HAVE_GRP_H -#include <grp.h> -#else -typedef uint32_t gid_t; -#endif - -#ifdef HAVE_PWD_H -#include <pwd.h> -#else -typedef uint32_t uid_t; -#endif - -#ifdef HAVE_NET_IF_H -#include <net/if.h> -#endif - -#ifdef HAVE_POLL_H -#include <poll.h> -#endif - -#ifdef HAVE_SYSLOG_H -#include <syslog.h> -#else -/* priorities */ -#define LOG_EMERG 0 /* system is unusable */ -#define LOG_ALERT 1 /* action must be taken immediately */ -#define LOG_CRIT 2 /* critical conditions */ -#define LOG_ERR 3 /* error conditions */ -#define LOG_WARNING 4 /* warning conditions */ -#define LOG_NOTICE 5 /* normal but significant condition */ -#define LOG_INFO 6 /* informational */ -#define LOG_DEBUG 7 /* debug-level messages */ - -/* facility codes */ -#define LOG_KERN (0<<3) /* kernel messages */ -#define LOG_USER (1<<3) /* random user-level messages */ -#define LOG_MAIL (2<<3) /* mail system */ -#define LOG_DAEMON (3<<3) /* system daemons */ -#define LOG_AUTH (4<<3) /* security/authorization messages */ -#define LOG_SYSLOG (5<<3) /* messages generated internally by syslogd */ -#define LOG_LPR (6<<3) /* line printer subsystem */ -#define LOG_NEWS (7<<3) /* network news subsystem */ -#define LOG_UUCP (8<<3) /* UUCP subsystem */ -#define LOG_CRON (9<<3) /* clock daemon */ -#define LOG_AUTHPRIV (10<<3) /* security/authorization messages (private) */ -#define LOG_FTP (11<<3) /* ftp daemon */ - -/* other codes through 15 reserved for system use */ -#define LOG_LOCAL0 (16<<3) /* reserved for local use */ -#define LOG_LOCAL1 (17<<3) /* reserved for local use */ -#define LOG_LOCAL2 (18<<3) /* reserved for local use */ -#define LOG_LOCAL3 (19<<3) /* reserved for local use */ -#define LOG_LOCAL4 (20<<3) /* reserved for local use */ -#define LOG_LOCAL5 (21<<3) /* reserved for local use */ -#define LOG_LOCAL6 (22<<3) /* reserved for local use */ -#define LOG_LOCAL7 (23<<3) /* reserved for local use */ -#endif - -#ifdef HAVE_SYS_MMAN_H -#include <sys/mman.h> -#endif - -#ifdef HAVE_SYS_RESOURCE_H -#include <sys/resource.h> -#endif - -#ifdef HAVE_SYS_SOCKET_H -#include <sys/socket.h> -#endif - -#ifdef HAVE_SYS_WAIT_H -#include <sys/wait.h> -#endif - -#ifdef HAVE_SYS_UN_H -#include <sys/un.h> -#endif - -#ifdef HAVE_SPAWN_H -#include <spawn.h> -#endif - -#ifdef HAVE_NETINET_IN_H -#include <netinet/in.h> -#endif - -#ifdef HAVE_RESOLV_H -#include <resolv.h> -#endif - -#ifdef HAVE_NETDB_H -#include <netdb.h> -#endif - -#ifdef HAVE_SYS_PRCTL_H -#include <sys/prctl.h> -#endif - -#ifdef HAVE_SYS_STAT_H -#include <sys/stat.h> -#endif - -#ifdef HAVE_SYS_VFS_H -#include <sys/vfs.h> -#endif - -#ifdef HAVE_SYS_STATFS_H -#include <sys/statfs.h> -#endif - -#ifdef HAVE_LINUX_MAGIC_H -#include <linux/magic.h> -#endif - -#ifdef HAVE_SYS_MOUNT_H -#include <sys/mount.h> -#endif - -#ifdef HAVE_SYS_STATVFS_H -#include <sys/statvfs.h> -#endif - -// #1408 -#ifdef MAJOR_IN_MKDEV -#include <sys/mkdev.h> -#endif -#ifdef MAJOR_IN_SYSMACROS -#include <sys/sysmacros.h> -#endif - -#include <math.h> -#include <float.h> - -#if defined(HAVE_INTTYPES_H) -#include <inttypes.h> -#elif defined(HAVE_STDINT_H) -#include <stdint.h> -#endif - -#include <zlib.h> - -#ifdef HAVE_SYS_CAPABILITY_H -#include <sys/capability.h> -#endif - +#include "libjudy/judy-malloc.h" -#ifndef O_CLOEXEC -#define O_CLOEXEC (0) -#endif - -// ---------------------------------------------------------------------------- -// netdata common definitions - -#define _cleanup_(x) __attribute__((__cleanup__(x))) - -#ifdef HAVE_FUNC_ATTRIBUTE_RETURNS_NONNULL -#define NEVERNULL __attribute__((returns_nonnull)) -#else -#define NEVERNULL -#endif - -#ifdef HAVE_FUNC_ATTRIBUTE_NOINLINE -#define NOINLINE __attribute__((noinline)) -#else -#define NOINLINE -#endif - -#ifdef HAVE_FUNC_ATTRIBUTE_MALLOC -#define MALLOCLIKE __attribute__((malloc)) -#else -#define MALLOCLIKE -#endif - -#if defined(HAVE_FUNC_ATTRIBUTE_FORMAT_GNU_PRINTF) -#define PRINTFLIKE(f, a) __attribute__ ((format(gnu_printf, f, a))) -#elif defined(HAVE_FUNC_ATTRIBUTE_FORMAT_PRINTF) -#define PRINTFLIKE(f, a) __attribute__ ((format(printf, f, a))) -#else -#define PRINTFLIKE(f, a) -#endif - -#ifdef HAVE_FUNC_ATTRIBUTE_NORETURN -#define NORETURN __attribute__ ((noreturn)) -#else -#define NORETURN -#endif - -#ifdef HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT -#define WARNUNUSED __attribute__ ((warn_unused_result)) -#else -#define WARNUNUSED -#endif - -void aral_judy_init(void); -size_t judy_aral_overhead(void); -size_t judy_aral_structures(void); - -#define ABS(x) (((x) < 0)? (-(x)) : (x)) -#define MIN(a,b) (((a)<(b))?(a):(b)) -#define MAX(a,b) (((a)>(b))?(a):(b)) -#define SWAP(a, b) do { \ - typeof(a) _tmp = b; \ - b = a; \ - a = _tmp; \ -} while(0) - -#define GUID_LEN 36 - -#define PIPE_READ 0 -#define PIPE_WRITE 1 - -#include "linked-lists.h" #include "storage-point.h" - -void netdata_fix_chart_id(char *s); -void netdata_fix_chart_name(char *s); +#include "paths/paths.h" int madvise_sequential(void *mem, size_t len); int madvise_random(void *mem, size_t len); @@ -394,46 +80,15 @@ int verify_netdata_host_prefix(bool log_msg); extern volatile sig_atomic_t netdata_exit; -char *strdupz_path_subpath(const char *path, const char *subpath); -int path_is_dir(const char *path, const char *subpath); -int path_is_file(const char *path, const char *subpath); -void recursive_config_double_dir_load( - const char *user_path - , const char *stock_path - , const char *subpath - , int (*callback)(const char *filename, void *data, bool stock_config) - , void *data - , size_t depth -); char *read_by_filename(const char *filename, long *file_size); char *find_and_replace(const char *src, const char *find, const char *replace, const char *where); -/* fix for alpine linux */ -#ifndef RUSAGE_THREAD -#ifdef RUSAGE_CHILDREN -#define RUSAGE_THREAD RUSAGE_CHILDREN -#endif -#endif - #define BITS_IN_A_KILOBIT 1000 #define KILOBITS_IN_A_MEGABIT 1000 -/* misc. */ - -#define UNUSED(x) (void)(x) - -#ifdef __GNUC__ -#define UNUSED_FUNCTION(x) __attribute__((unused)) UNUSED_##x -#else -#define UNUSED_FUNCTION(x) UNUSED_##x -#endif - #define error_report(x, args...) do { errno_clear(); netdata_log_error(x, ##args); } while(0) -// Taken from linux kernel -#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)])) - -#include "bitmap64.h" +#include "bitmap/bitmap64.h" #define COMPRESSION_MAX_CHUNK 0x4000 #define COMPRESSION_MAX_OVERHEAD 128 @@ -449,47 +104,58 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data) NORETURN; #endif -extern char *netdata_configured_host_prefix; +extern const char *netdata_configured_host_prefix; -#include "os/os.h" +// safe includes before O/S specific functions +#include "template-enum.h" +#include "libjudy/src/Judy.h" +#include "july/july.h" -#define XXH_INLINE_ALL -#include "xxhash.h" +#include "string/string.h" +#include "buffer/buffer.h" #include "uuid/uuid.h" -#include "template-enum.h" -#include "http/http_access.h" #include "http/content_type.h" -#include "config/dyncfg.h" -#include "libjudy/src/Judy.h" -#include "july/july.h" +#include "http/http_access.h" + +#include "inlined.h" +#include "parsers/parsers.h" + #include "threads/threads.h" -#include "buffer/buffer.h" #include "locks/locks.h" -#include "circular_buffer/circular_buffer.h" +#include "completion/completion.h" +#include "clocks/clocks.h" +#include "simple_pattern/simple_pattern.h" +#include "libnetdata/log/nd_log.h" + +#include "socket/security.h" // must be before windows.h + +// this may include windows.h +#include "os/os.h" + +#include "socket/socket.h" #include "avl/avl.h" -#include "inlined.h" + #include "line_splitter/line_splitter.h" -#include "clocks/clocks.h" +#include "c_rhash/c_rhash.h" +#include "ringbuffer/ringbuffer.h" +#include "circular_buffer/circular_buffer.h" +#include "buffered_reader/buffered_reader.h" #include "datetime/iso8601.h" #include "datetime/rfc3339.h" #include "datetime/rfc7231.h" -#include "completion/completion.h" -#include "log/log.h" +#include "sanitizers/sanitizers.h" + +#include "config/dyncfg.h" +#include "config/appconfig.h" #include "spawn_server/spawn_server.h" #include "spawn_server/spawn_popen.h" -#include "simple_pattern/simple_pattern.h" -#ifdef ENABLE_HTTPS -# include "socket/security.h" -#endif -#include "socket/socket.h" -#include "config/appconfig.h" -#include "log/journal.h" -#include "buffered_reader/buffered_reader.h" #include "procfile/procfile.h" -#include "string/string.h" #include "dictionary/dictionary.h" #include "dictionary/thread-cache.h" + +#include "log/systemd-journal-helpers.h" + #if defined(HAVE_LIBBPF) && !defined(__cplusplus) #include "ebpf/ebpf.h" #endif @@ -510,11 +176,6 @@ extern char *netdata_configured_host_prefix; #include "functions_evloop/functions_evloop.h" #include "query_progress/progress.h" -// BEWARE: this exists in alarm-notify.sh -#define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud" - -#define RRD_STORAGE_TIERS 5 - static inline size_t struct_natural_alignment(size_t size) __attribute__((const)); #define STRUCT_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2) @@ -653,12 +314,14 @@ void timing_action(TIMING_ACTION action, TIMING_STEP step); int hash256_string(const unsigned char *string, size_t size, char *hash); extern bool unittest_running; -#define API_RELATIVE_TIME_MAX (3 * 365 * 86400) bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t now); bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_t *now_ptr, bool unittest); -int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size); +int netdata_base64_decode(unsigned char *out, const unsigned char *in, int in_len); +int netdata_base64_encode(unsigned char *encoded, const unsigned char *input, size_t input_size); + +// -------------------------------------------------------------------------------------------------------------------- static inline void freez_charp(char **p) { freez(*p); diff --git a/src/libnetdata/line_splitter/README.md b/src/libnetdata/line_splitter/README.md index b391a492c..1c8263421 100644 --- a/src/libnetdata/line_splitter/README.md +++ b/src/libnetdata/line_splitter/README.md @@ -1,12 +1,3 @@ -<!-- -title: "Log" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/log/README.md -sidebar_label: "Log" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Log The netdata log library supports debug, info, error and fatal error logging. diff --git a/src/libnetdata/line_splitter/line_splitter.c b/src/libnetdata/line_splitter/line_splitter.c index 6726d9096..aa56e9b4e 100644 --- a/src/libnetdata/line_splitter/line_splitter.c +++ b/src/libnetdata/line_splitter/line_splitter.c @@ -21,12 +21,29 @@ bool line_splitter_reconstruct_line(BUFFER *wb, void *ptr) { return added > 0; } -inline int pluginsd_isspace(char c) { +inline int isspace_whitespace(char c) { switch(c) { case ' ': case '\t': case '\r': case '\n': + case '\f': + case '\v': + return 1; + + default: + return 0; + } +} + +inline int isspace_pluginsd(char c) { + switch(c) { + case ' ': + case '\t': + case '\r': + case '\n': + case '\f': + case '\v': case '=': return 1; @@ -35,12 +52,14 @@ inline int pluginsd_isspace(char c) { } } -inline int config_isspace(char c) { +inline int isspace_config(char c) { switch (c) { case ' ': case '\t': case '\r': case '\n': + case '\f': + case '\v': case ',': return 1; @@ -49,20 +68,21 @@ inline int config_isspace(char c) { } } -inline int group_by_label_isspace(char c) { +inline int isspace_group_by_label(char c) { if(c == ',' || c == '|') return 1; return 0; } -inline int dyncfg_id_isspace(char c) { +inline int isspace_dyncfg_id(char c) { if(c == ':') return 1; return 0; } +bool isspace_map_whitespace[256] = {}; bool isspace_map_pluginsd[256] = {}; bool isspace_map_config[256] = {}; bool isspace_map_group_by_label[256] = {}; @@ -70,9 +90,10 @@ bool isspace_dyncfg_id_map[256] = {}; __attribute__((constructor)) void initialize_is_space_arrays(void) { for(int c = 0; c < 256 ; c++) { - isspace_map_pluginsd[c] = pluginsd_isspace((char) c); - isspace_map_config[c] = config_isspace((char) c); - isspace_map_group_by_label[c] = group_by_label_isspace((char) c); - isspace_dyncfg_id_map[c] = dyncfg_id_isspace((char)c); + isspace_map_whitespace[c] = isspace_whitespace((char) c); + isspace_map_pluginsd[c] = isspace_pluginsd((char) c); + isspace_map_config[c] = isspace_config((char) c); + isspace_map_group_by_label[c] = isspace_group_by_label((char) c); + isspace_dyncfg_id_map[c] = isspace_dyncfg_id((char) c); } } diff --git a/src/libnetdata/line_splitter/line_splitter.h b/src/libnetdata/line_splitter/line_splitter.h index 968930410..157d91786 100644 --- a/src/libnetdata/line_splitter/line_splitter.h +++ b/src/libnetdata/line_splitter/line_splitter.h @@ -19,11 +19,13 @@ static inline void line_splitter_reset(struct line_splitter *line) { line->num_words = 0; } -int pluginsd_isspace(char c); -int config_isspace(char c); -int group_by_label_isspace(char c); -int dyncfg_id_isspace(char c); +int isspace_pluginsd(char c); +int isspace_config(char c); +int isspace_group_by_label(char c); +int isspace_dyncfg_id(char c); +int isspace_whitespace(char c); +extern bool isspace_map_whitespace[256]; extern bool isspace_map_pluginsd[256]; extern bool isspace_map_config[256]; extern bool isspace_map_group_by_label[256]; @@ -55,6 +57,9 @@ static inline size_t quoted_strings_splitter(char *str, char **words, size_t max while (likely(*s)) { // if it is an escape if (unlikely(*s == '\\' && s[1])) { + // IMPORTANT: support for escaping is incomplete! + // The backslash character needs to be removed + // from the parsed string. s += 2; continue; } @@ -103,6 +108,9 @@ static inline size_t quoted_strings_splitter(char *str, char **words, size_t max return i; } +#define quoted_strings_splitter_whitespace(str, words, max_words) \ + quoted_strings_splitter(str, words, max_words, isspace_map_whitespace) + #define quoted_strings_splitter_query_group_by_label(str, words, max_words) \ quoted_strings_splitter(str, words, max_words, isspace_map_group_by_label) diff --git a/src/libnetdata/linked-lists.h b/src/libnetdata/linked_lists/linked_lists.h index 033d11226..033d11226 100644 --- a/src/libnetdata/linked-lists.h +++ b/src/libnetdata/linked_lists/linked_lists.h diff --git a/src/libnetdata/maps/local-sockets.h b/src/libnetdata/local-sockets/local-sockets.h index 6f2ffd81a..06ac08767 100644 --- a/src/libnetdata/maps/local-sockets.h +++ b/src/libnetdata/local-sockets/local-sockets.h @@ -5,7 +5,14 @@ #include "libnetdata/libnetdata.h" -#ifdef HAVE_LIBMNL +#ifndef _countof +#define _countof(x) (sizeof(x) / sizeof(*(x))) +#endif + +#define LOCAL_SOCKETS_USE_SETNS +#define USE_LIBMNL_AFTER_SETNS + +#if defined(HAVE_LIBMNL) #include <linux/rtnetlink.h> #include <linux/inet_diag.h> #include <linux/sock_diag.h> @@ -22,7 +29,7 @@ #define SIMPLE_HASHTABLE_VALUE_TYPE uint64_t #define SIMPLE_HASHTABLE_NAME _NET_NS -#include "libnetdata/simple_hashtable.h" +#include "libnetdata/simple_hashtable/simple_hashtable.h" // -------------------------------------------------------------------------------------------------------------------- // hashtable for keeping the sockets of PIDs @@ -31,7 +38,7 @@ struct pid_socket; #define SIMPLE_HASHTABLE_VALUE_TYPE struct pid_socket #define SIMPLE_HASHTABLE_NAME _PID_SOCKET -#include "libnetdata/simple_hashtable.h" +#include "libnetdata/simple_hashtable/simple_hashtable.h" // -------------------------------------------------------------------------------------------------------------------- // hashtable for keeping all the sockets @@ -40,7 +47,7 @@ struct pid_socket; struct local_socket; #define SIMPLE_HASHTABLE_VALUE_TYPE struct local_socket #define SIMPLE_HASHTABLE_NAME _LOCAL_SOCKET -#include "libnetdata/simple_hashtable.h" +#include "libnetdata/simple_hashtable/simple_hashtable.h" // -------------------------------------------------------------------------------------------------------------------- // hashtable for keeping all local IPs @@ -49,7 +56,7 @@ struct local_socket; union ipv46; #define SIMPLE_HASHTABLE_VALUE_TYPE union ipv46 #define SIMPLE_HASHTABLE_NAME _LOCAL_IP -#include "libnetdata/simple_hashtable.h" +#include "libnetdata/simple_hashtable/simple_hashtable.h" // -------------------------------------------------------------------------------------------------------------------- // hashtable for keeping all listening ports @@ -58,12 +65,12 @@ union ipv46; struct local_port; #define SIMPLE_HASHTABLE_VALUE_TYPE struct local_port #define SIMPLE_HASHTABLE_NAME _LISTENING_PORT -#include "libnetdata/simple_hashtable.h" +#include "libnetdata/simple_hashtable/simple_hashtable.h" // -------------------------------------------------------------------------------------------------------------------- struct local_socket_state; -typedef void (*local_sockets_cb_t)(struct local_socket_state *state, struct local_socket *n, void *data); +typedef void (*local_sockets_cb_t)(struct local_socket_state *state, const struct local_socket *n, void *data); struct local_sockets_config { bool listening; @@ -80,6 +87,9 @@ struct local_sockets_config { bool uid; bool namespaces; bool tcp_info; + bool no_mnl; + bool procfile; + bool report; size_t max_errors; size_t max_concurrent_namespaces; @@ -88,34 +98,68 @@ struct local_sockets_config { void *data; const char *host_prefix; +}; - // internal use +struct local_sockets_state { + uint32_t nl_seq; uint64_t net_ns_inode; + pid_t net_ns_pid; +}; + +struct timing_work { + usec_t start_ut; + usec_t end_ut; + const char *name; +}; + +struct local_sockets_ns_req { + struct local_sockets_config config; + struct local_sockets_state ns_state; }; typedef struct local_socket_state { struct local_sockets_config config; + struct local_sockets_state ns_state; struct { size_t mnl_sends; - size_t namespaces_found; size_t tcp_info_received; size_t pid_fds_processed; size_t pid_fds_opendir_failed; size_t pid_fds_readlink_failed; size_t pid_fds_parse_failed; size_t errors_encountered; + + size_t sockets_added; + + size_t namespaces_found; + size_t namespaces_absent; + size_t namespaces_invalid; +#if defined(LOCAL_SOCKETS_USE_SETNS) + size_t namespaces_forks_attempted; + size_t namespaces_forks_failed; + size_t namespaces_forks_unresponsive; + size_t namespaces_sockets_new; + size_t namespaces_sockets_existing; +#endif + + struct procfile_stats ff; } stats; + size_t timings_idx; + struct timing_work timings[30]; + +#if defined(LOCAL_SOCKETS_USE_SETNS) bool spawn_server_is_mine; SPAWN_SERVER *spawn_server; +#endif -#ifdef HAVE_LIBMNL - bool use_nl; - struct mnl_socket *nl; +#if defined(HAVE_LIBMNL) uint16_t tmp_protocol; #endif + procfile *ff; + ARAL *local_socket_aral; ARAL *pid_socket_aral; SPINLOCK spinlock; // for namespaces @@ -223,7 +267,9 @@ typedef struct local_socket { #endif } LOCAL_SOCKET; -static inline void local_sockets_spawn_server_callback(SPAWN_REQUEST *request); +#if defined(LOCAL_SOCKETS_USE_SETNS) +static inline int local_sockets_spawn_server_callback(SPAWN_REQUEST *request); +#endif // -------------------------------------------------------------------------------------------------------------------- @@ -254,7 +300,7 @@ static bool local_sockets_is_ipv4_mapped_ipv6_address(const struct in6_addr *add return memcmp(addr->s6_addr, ipv4_mapped_prefix, 12) == 0; } -static bool local_sockets_is_loopback_address(struct socket_endpoint *se) { +static bool local_sockets_is_loopback_address(const struct socket_endpoint *se) { if (se->family == AF_INET) { // For IPv4, loopback addresses are in the 127.0.0.0/8 range return (ntohl(se->ip.ipv4) >> 24) == 127; // Check if the first byte is 127 @@ -288,7 +334,7 @@ static inline bool local_sockets_is_ipv4_reserved_address(uint32_t ip) { ); } -static inline bool local_sockets_is_private_address(struct socket_endpoint *se) { +static inline bool local_sockets_is_private_address(const struct socket_endpoint *se) { if (se->family == AF_INET) { return local_sockets_is_ipv4_reserved_address(se->ip.ipv4); } @@ -322,7 +368,7 @@ static inline bool local_sockets_is_private_address(struct socket_endpoint *se) return false; } -static bool local_sockets_is_multicast_address(struct socket_endpoint *se) { +static bool local_sockets_is_multicast_address(const struct socket_endpoint *se) { if (se->family == AF_INET) { // For IPv4, check if the address is 0.0.0.0 uint32_t ip = htonl(se->ip.ipv4); @@ -337,7 +383,7 @@ static bool local_sockets_is_multicast_address(struct socket_endpoint *se) { return false; } -static bool local_sockets_is_zero_address(struct socket_endpoint *se) { +static bool local_sockets_is_zero_address(const struct socket_endpoint *se) { if (se->family == AF_INET) { // For IPv4, check if the address is 0.0.0.0 return se->ip.ipv4 == 0; @@ -350,7 +396,7 @@ static bool local_sockets_is_zero_address(struct socket_endpoint *se) { return false; } -static inline const char *local_sockets_address_space(struct socket_endpoint *se) { +static inline const char *local_sockets_address_space(const struct socket_endpoint *se) { if(local_sockets_is_zero_address(se)) return "zero"; else if(local_sockets_is_loopback_address(se)) @@ -363,14 +409,94 @@ static inline const char *local_sockets_address_space(struct socket_endpoint *se return "public"; } -// -------------------------------------------------------------------------------------------------------------------- +static inline void ipv6_address_to_txt(const struct in6_addr *in6_addr, char *dst) { + struct sockaddr_in6 sa = { 0 }; + + sa.sin6_family = AF_INET6; + sa.sin6_port = htons(0); + sa.sin6_addr = *in6_addr; + + // Convert to human-readable format + if (inet_ntop(AF_INET6, &(sa.sin6_addr), dst, INET6_ADDRSTRLEN) == NULL) + *dst = '\0'; +} + +static inline void ipv4_address_to_txt(uint32_t ip, char *dst) { + uint8_t octets[4]; + octets[0] = ip & 0xFF; + octets[1] = (ip >> 8) & 0xFF; + octets[2] = (ip >> 16) & 0xFF; + octets[3] = (ip >> 24) & 0xFF; + sprintf(dst, "%u.%u.%u.%u", octets[0], octets[1], octets[2], octets[3]); +} -static inline bool is_local_socket_ipv46(LOCAL_SOCKET *n) { +static inline bool is_local_socket_ipv46(const LOCAL_SOCKET *n) { return n->local.family == AF_INET6 && - n->direction == SOCKET_DIRECTION_LISTEN && - local_sockets_is_zero_address(&n->local) && - n->ipv6ony.checked && - n->ipv6ony.ipv46; + n->direction == SOCKET_DIRECTION_LISTEN && + local_sockets_is_zero_address(&n->local) && + n->ipv6ony.checked && + n->ipv6ony.ipv46; +} + +static inline const char *local_sockets_protocol_name(LOCAL_SOCKET *n) { + if(n->local.family == AF_INET) { + if(n->local.protocol == IPPROTO_TCP) + return "TCP"; + else if(n->local.protocol == IPPROTO_UDP) + return "UDP"; + else + return "UNKNOWN_IPV4"; + } + else if(is_local_socket_ipv46(n)) { + if (n->local.protocol == IPPROTO_TCP) + return "TCP46"; + else if(n->local.protocol == IPPROTO_UDP) + return "UDP46"; + else + return "UNKNOWN_IPV46"; + } + else if(n->local.family == AF_INET6) { + if (n->local.protocol == IPPROTO_TCP) + return "TCP6"; + else if(n->local.protocol == IPPROTO_UDP) + return "UDP6"; + else + return "UNKNOWN_IPV6"; + } + else + return "UNKNOWN"; +} + +static inline void local_listeners_print_socket(LS_STATE *ls __maybe_unused, const LOCAL_SOCKET *nn, void *data __maybe_unused) { + LOCAL_SOCKET *n = (LOCAL_SOCKET *)nn; + + char local_address[INET6_ADDRSTRLEN]; + char remote_address[INET6_ADDRSTRLEN]; + + if(n->local.family == AF_INET) { + ipv4_address_to_txt(n->local.ip.ipv4, local_address); + ipv4_address_to_txt(n->remote.ip.ipv4, remote_address); + } + else if(n->local.family == AF_INET6) { + ipv6_address_to_txt(&n->local.ip.ipv6, local_address); + ipv6_address_to_txt(&n->remote.ip.ipv6, remote_address); + } + + printf("%s, direction=%s%s%s%s%s pid=%d, state=0x%0x, ns=%"PRIu64", local=%s[:%u], remote=%s[:%u], uid=%u, inode=%"PRIu64", comm=%s\n", + local_sockets_protocol_name(n), + (n->direction & SOCKET_DIRECTION_LISTEN) ? "LISTEN," : "", + (n->direction & SOCKET_DIRECTION_INBOUND) ? "INBOUND," : "", + (n->direction & SOCKET_DIRECTION_OUTBOUND) ? "OUTBOUND," : "", + (n->direction & (SOCKET_DIRECTION_LOCAL_INBOUND|SOCKET_DIRECTION_LOCAL_OUTBOUND)) ? "LOCAL," : "", + (n->direction == 0) ? "NONE," : "", + n->pid, + (unsigned int)n->state, + n->net_ns_inode, + local_address, n->local.port, + remote_address, n->remote.port, + n->uid, + n->inode, + n->comm); } // -------------------------------------------------------------------------------------------------------------------- @@ -504,7 +630,9 @@ static inline bool local_sockets_find_all_sockets_in_proc(LS_STATE *ls, const ch if(!local_sockets_read_proc_inode_link(ls, filename, &inode, "socket")) continue; - SIMPLE_HASHTABLE_SLOT_PID_SOCKET *sl = simple_hashtable_get_slot_PID_SOCKET(&ls->pid_sockets_hashtable, inode, &inode, true); + // fprintf(stderr, "%d: PID %d is using socket inode %"PRIu64"\n", gettid_uncached(), pid, inode); + XXH64_hash_t inode_hash = XXH3_64bits(&inode, sizeof(inode)); + SIMPLE_HASHTABLE_SLOT_PID_SOCKET *sl = simple_hashtable_get_slot_PID_SOCKET(&ls->pid_sockets_hashtable, inode_hash, &inode, true); struct pid_socket *ps = SIMPLE_HASHTABLE_SLOT_DATA(sl); if(!ps || (ps->pid == 1 && pid != 1)) { if(uid == UID_UNSET && ls->config.uid) { @@ -545,7 +673,8 @@ static inline bool local_sockets_find_all_sockets_in_proc(LS_STATE *ls, const ch if(!net_ns_inode && ls->config.namespaces) { snprintfz(filename, sizeof(filename), "%s/%s/ns/net", proc_filename, proc_entry->d_name); if(local_sockets_read_proc_inode_link(ls, filename, &net_ns_inode, "net")) { - SIMPLE_HASHTABLE_SLOT_NET_NS *sl_ns = simple_hashtable_get_slot_NET_NS(&ls->ns_hashtable, net_ns_inode, (uint64_t *)net_ns_inode, true); + XXH64_hash_t net_ns_inode_hash = XXH3_64bits(&net_ns_inode, sizeof(net_ns_inode)); + SIMPLE_HASHTABLE_SLOT_NET_NS *sl_ns = simple_hashtable_get_slot_NET_NS(&ls->ns_hashtable, net_ns_inode_hash, (uint64_t *)net_ns_inode, true); simple_hashtable_set_slot_NET_NS(&ls->ns_hashtable, sl_ns, net_ns_inode, (uint64_t *)net_ns_inode); } } @@ -563,7 +692,8 @@ static inline bool local_sockets_find_all_sockets_in_proc(LS_STATE *ls, const ch freez(ps->cmdline); ps->cmdline = cmdline_trimmed ? strdupz(cmdline_trimmed) : NULL; - simple_hashtable_set_slot_PID_SOCKET(&ls->pid_sockets_hashtable, sl, inode, ps); + simple_hashtable_set_slot_PID_SOCKET(&ls->pid_sockets_hashtable, sl, inode_hash, ps); + // fprintf(stderr, "%d: PID %d indexed for using socket inode %"PRIu64"\n", gettid_uncached(), pid, inode); } } @@ -593,13 +723,16 @@ static inline void local_sockets_index_listening_port(LS_STATE *ls, LOCAL_SOCKET static inline bool local_sockets_add_socket(LS_STATE *ls, LOCAL_SOCKET *tmp) { if(!tmp->inode) return false; - SIMPLE_HASHTABLE_SLOT_LOCAL_SOCKET *sl = simple_hashtable_get_slot_LOCAL_SOCKET(&ls->sockets_hashtable, tmp->inode, &tmp->inode, true); + XXH64_hash_t inode_hash = XXH3_64bits(&tmp->inode, sizeof(tmp->inode)); + SIMPLE_HASHTABLE_SLOT_LOCAL_SOCKET *sl = simple_hashtable_get_slot_LOCAL_SOCKET(&ls->sockets_hashtable, inode_hash, &tmp->inode, true); LOCAL_SOCKET *n = SIMPLE_HASHTABLE_SLOT_DATA(sl); if(n) { local_sockets_log(ls, "inode %" PRIu64" already exists in hashtable - ignoring duplicate", tmp->inode); return false; } + ls->stats.sockets_added++; + n = aral_mallocz(ls->local_socket_aral); *n = *tmp; // copy all contents @@ -615,7 +748,7 @@ static inline bool local_sockets_add_socket(LS_STATE *ls, LOCAL_SOCKET *tmp) { // --- look up a pid for it ----------------------------------------------------------------------------------- - SIMPLE_HASHTABLE_SLOT_PID_SOCKET *sl_pid = simple_hashtable_get_slot_PID_SOCKET(&ls->pid_sockets_hashtable, n->inode, &n->inode, false); + SIMPLE_HASHTABLE_SLOT_PID_SOCKET *sl_pid = simple_hashtable_get_slot_PID_SOCKET(&ls->pid_sockets_hashtable, inode_hash, &n->inode, false); struct pid_socket *ps = SIMPLE_HASHTABLE_SLOT_DATA(sl_pid); if(ps) { n->net_ns_inode = ps->net_ns_inode; @@ -624,15 +757,19 @@ static inline bool local_sockets_add_socket(LS_STATE *ls, LOCAL_SOCKET *tmp) { if(ps->uid != UID_UNSET && n->uid == UID_UNSET) n->uid = ps->uid; - if(ps->cmdline) + if(ps->cmdline) { + if(n->cmdline) string_freez(n->cmdline); n->cmdline = string_strdupz(ps->cmdline); + } strncpyz(n->comm, ps->comm, sizeof(n->comm) - 1); } +// else +// fprintf(stderr, "%d: No PID found for inode %"PRIu64"\n", gettid_uncached(), n->inode); // --- index it ----------------------------------------------------------------------------------------------- - simple_hashtable_set_slot_LOCAL_SOCKET(&ls->sockets_hashtable, sl, n->inode, n); + simple_hashtable_set_slot_LOCAL_SOCKET(&ls->sockets_hashtable, sl, inode_hash, n); if(!local_sockets_is_zero_address(&n->local)) { // put all the local IPs into the local_ips hashtable @@ -668,31 +805,7 @@ static inline bool local_sockets_add_socket(LS_STATE *ls, LOCAL_SOCKET *tmp) { return true; } -#ifdef HAVE_LIBMNL - -static inline void local_sockets_libmnl_init(LS_STATE *ls) { - ls->nl = mnl_socket_open(NETLINK_INET_DIAG); - if (ls->nl == NULL) { - local_sockets_log(ls, "cannot open libmnl netlink socket"); - ls->use_nl = false; - } - else if (mnl_socket_bind(ls->nl, 0, MNL_SOCKET_AUTOPID) < 0) { - local_sockets_log(ls, "cannot bind libmnl netlink socket"); - mnl_socket_close(ls->nl); - ls->nl = NULL; - ls->use_nl = false; - } - else - ls->use_nl = true; -} - -static inline void local_sockets_libmnl_cleanup(LS_STATE *ls) { - if(ls->nl) { - mnl_socket_close(ls->nl); - ls->nl = NULL; - ls->use_nl = false; - } -} +#if defined(HAVE_LIBMNL) static inline int local_sockets_libmnl_cb_data(const struct nlmsghdr *nlh, void *data) { LS_STATE *ls = data; @@ -767,16 +880,30 @@ static inline int local_sockets_libmnl_cb_data(const struct nlmsghdr *nlh, void static inline bool local_sockets_libmnl_get_sockets(LS_STATE *ls, uint16_t family, uint16_t protocol) { ls->tmp_protocol = protocol; - char buf[MNL_SOCKET_BUFFER_SIZE]; - struct nlmsghdr *nlh; - struct inet_diag_req_v2 req; - unsigned int seq, portid = mnl_socket_get_portid(ls->nl); + struct mnl_socket *nl = mnl_socket_open(NETLINK_INET_DIAG); + if (nl == NULL) { + local_sockets_log(ls, "mnl_socket_open() failed"); + return false; + } - memset(&req, 0, sizeof(req)); - req.sdiag_family = family; - req.sdiag_protocol = protocol; - req.idiag_states = -1; - req.idiag_ext = 0; + if (mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID) < 0) { + local_sockets_log(ls, "mnl_socket_bind() failed"); + mnl_socket_close(nl); + return false; + } + + char buf[MNL_SOCKET_BUFFER_SIZE]; + struct nlmsghdr *nlh = mnl_nlmsg_put_header(buf); + nlh->nlmsg_type = SOCK_DIAG_BY_FAMILY; + nlh->nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP; + nlh->nlmsg_seq = ls->ns_state.nl_seq ? ls->ns_state.nl_seq++ : time(NULL); + + struct inet_diag_req_v2 req = { + .sdiag_family = family, + .sdiag_protocol = protocol, + .idiag_states = ~0, // Request all socket states + .idiag_ext = 0, + }; if(family == AF_INET6) req.idiag_ext |= 1 << (INET_DIAG_SKV6ONLY - 1); @@ -784,35 +911,106 @@ static inline bool local_sockets_libmnl_get_sockets(LS_STATE *ls, uint16_t famil if(protocol == IPPROTO_TCP && ls->config.tcp_info) req.idiag_ext |= 1 << (INET_DIAG_INFO - 1); - nlh = mnl_nlmsg_put_header(buf); - nlh->nlmsg_type = SOCK_DIAG_BY_FAMILY; - nlh->nlmsg_flags = NLM_F_ROOT | NLM_F_MATCH | NLM_F_REQUEST; - nlh->nlmsg_seq = seq = time(NULL); mnl_nlmsg_put_extra_header(nlh, sizeof(req)); memcpy(mnl_nlmsg_get_payload(nlh), &req, sizeof(req)); ls->stats.mnl_sends++; - if (mnl_socket_sendto(ls->nl, nlh, nlh->nlmsg_len) < 0) { - local_sockets_log(ls, "mnl_socket_send failed"); + if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) { + local_sockets_log(ls, "mnl_socket_sendto() failed"); + mnl_socket_close(nl); return false; } + bool rc = true; + size_t received = 0; ssize_t ret; - while ((ret = mnl_socket_recvfrom(ls->nl, buf, sizeof(buf))) > 0) { - ret = mnl_cb_run(buf, ret, seq, portid, local_sockets_libmnl_cb_data, ls); - if (ret <= MNL_CB_STOP) + while ((ret = mnl_socket_recvfrom(nl, buf, sizeof(buf))) > 0) { + ret = mnl_cb_run(buf, ret, 0, 0, local_sockets_libmnl_cb_data, ls); + if (ret == MNL_CB_ERROR) { + local_sockets_log(ls, "mnl_cb_run() failed"); + rc = false; + break; + } + else if (ret <= MNL_CB_STOP) break; + + received++; } + mnl_socket_close(nl); + if (ret == -1) { - local_sockets_log(ls, "mnl_socket_recvfrom"); + local_sockets_log(ls, "mnl_socket_recvfrom() failed"); + rc = false; + } + + return rc; +} +#endif // HAVE_LIBMNL + +static inline bool local_sockets_process_proc_line(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol, size_t line, char **words, size_t num_words) { + // char *sl_txt = get_word(words, num_words, 0); + char *local_ip_txt = get_word(words, num_words, 1); + char *local_port_txt = get_word(words, num_words, 2); + char *remote_ip_txt = get_word(words, num_words, 3); + char *remote_port_txt = get_word(words, num_words, 4); + char *state_txt = get_word(words, num_words, 5); + char *tx_queue_txt = get_word(words, num_words, 6); + char *rx_queue_txt = get_word(words, num_words, 7); + char *tr_txt = get_word(words, num_words, 8); + char *tm_when_txt = get_word(words, num_words, 9); + char *retrans_txt = get_word(words, num_words, 10); + char *uid_txt = get_word(words, num_words, 11); + // char *timeout_txt = get_word(words, num_words, 12); + char *inode_txt = get_word(words, num_words, 13); + + if(!local_ip_txt || !local_port_txt || !remote_ip_txt || !remote_port_txt || !state_txt || + !tx_queue_txt || !rx_queue_txt || !tr_txt || !tm_when_txt || !retrans_txt || !uid_txt || !inode_txt) { + local_sockets_log(ls, "cannot parse ipv4 line No %zu of filename '%s'", line, filename); return false; } + LOCAL_SOCKET n = { + .direction = SOCKET_DIRECTION_NONE, + .ipv6ony = { + .checked = false, + .ipv46 = false, + }, + .local = { + .family = family, + .protocol = protocol, + }, + .remote = { + .family = family, + .protocol = protocol, + }, + .uid = UID_UNSET, + }; + + n.local.port = str2uint32_hex(local_port_txt, NULL); + n.remote.port = str2uint32_hex(remote_port_txt, NULL); + n.state = str2uint32_hex(state_txt, NULL); + n.wqueue = str2uint32_hex(tx_queue_txt, NULL); + n.rqueue = str2uint32_hex(rx_queue_txt, NULL); + n.timer = str2uint32_hex(tr_txt, NULL); + n.expires = str2uint32_hex(tm_when_txt, NULL); + n.retransmits = str2uint32_hex(retrans_txt, NULL); + n.uid = str2uint32_t(uid_txt, NULL); + n.inode = str2uint64_t(inode_txt, NULL); + + if(family == AF_INET) { + n.local.ip.ipv4 = str2uint32_hex(local_ip_txt, NULL); + n.remote.ip.ipv4 = str2uint32_hex(remote_ip_txt, NULL); + } + else if(family == AF_INET6) { + ipv6_to_in6_addr(local_ip_txt, &n.local.ip.ipv6); + ipv6_to_in6_addr(remote_ip_txt, &n.remote.ip.ipv6); + } + + local_sockets_add_socket(ls, &n); return true; } -#endif // HAVE_LIBMNL -static inline bool local_sockets_read_proc_net_x(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) { +static inline bool local_sockets_read_proc_net_x_getline(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) { static bool is_space[256] = { [':'] = true, [' '] = true, @@ -846,67 +1044,9 @@ static inline bool local_sockets_read_proc_net_x(LS_STATE *ls, const char *filen continue; } - LOCAL_SOCKET n = { - .direction = SOCKET_DIRECTION_NONE, - .ipv6ony = { - .checked = false, - .ipv46 = false, - }, - .local = { - .family = family, - .protocol = protocol, - }, - .remote = { - .family = family, - .protocol = protocol, - }, - .uid = UID_UNSET, - }; - char *words[32]; size_t num_words = quoted_strings_splitter(line, words, 32, is_space); - // char *sl_txt = get_word(words, num_words, 0); - char *local_ip_txt = get_word(words, num_words, 1); - char *local_port_txt = get_word(words, num_words, 2); - char *remote_ip_txt = get_word(words, num_words, 3); - char *remote_port_txt = get_word(words, num_words, 4); - char *state_txt = get_word(words, num_words, 5); - char *tx_queue_txt = get_word(words, num_words, 6); - char *rx_queue_txt = get_word(words, num_words, 7); - char *tr_txt = get_word(words, num_words, 8); - char *tm_when_txt = get_word(words, num_words, 9); - char *retrans_txt = get_word(words, num_words, 10); - char *uid_txt = get_word(words, num_words, 11); - // char *timeout_txt = get_word(words, num_words, 12); - char *inode_txt = get_word(words, num_words, 13); - - if(!local_ip_txt || !local_port_txt || !remote_ip_txt || !remote_port_txt || !state_txt || - !tx_queue_txt || !rx_queue_txt || !tr_txt || !tm_when_txt || !retrans_txt || !uid_txt || !inode_txt) { - local_sockets_log(ls, "cannot parse ipv4 line No %zu of filename '%s'", counter, filename); - continue; - } - - n.local.port = str2uint32_hex(local_port_txt, NULL); - n.remote.port = str2uint32_hex(remote_port_txt, NULL); - n.state = str2uint32_hex(state_txt, NULL); - n.wqueue = str2uint32_hex(tx_queue_txt, NULL); - n.rqueue = str2uint32_hex(rx_queue_txt, NULL); - n.timer = str2uint32_hex(tr_txt, NULL); - n.expires = str2uint32_hex(tm_when_txt, NULL); - n.retransmits = str2uint32_hex(retrans_txt, NULL); - n.uid = str2uint32_t(uid_txt, NULL); - n.inode = str2uint64_t(inode_txt, NULL); - - if(family == AF_INET) { - n.local.ip.ipv4 = str2uint32_hex(local_ip_txt, NULL); - n.remote.ip.ipv4 = str2uint32_hex(remote_ip_txt, NULL); - } - else if(family == AF_INET6) { - ipv6_to_in6_addr(local_ip_txt, &n.local.ip.ipv6); - ipv6_to_in6_addr(remote_ip_txt, &n.remote.ip.ipv6); - } - - local_sockets_add_socket(ls, &n); + local_sockets_process_proc_line(ls, filename, family, protocol, counter, words, num_words); } fclose(fp); @@ -917,6 +1057,59 @@ static inline bool local_sockets_read_proc_net_x(LS_STATE *ls, const char *filen return true; } +#define INITIALLY_EXPECTED_PROC_NET_LINES 16384 +#define PROC_NET_BYTES_PER_LINE 155 // 105 for IPv4, 155 for IPv6 +#define PROC_NET_WORDS_PER_LINE 22 +#define INITIALLY_EXPECTED_PROC_NET_WORDS (INITIALLY_EXPECTED_PROC_NET_LINES * PROC_NET_WORDS_PER_LINE) +#define INITIALLY_EXPECTED_PROC_NET_BYTES (INITIALLY_EXPECTED_PROC_NET_LINES * PROC_NET_BYTES_PER_LINE) + +static inline bool local_sockets_read_proc_net_x_procfile(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) { + if(family != AF_INET && family != AF_INET6) + return false; + + procfile_set_adaptive_allocation(true, INITIALLY_EXPECTED_PROC_NET_BYTES, INITIALLY_EXPECTED_PROC_NET_LINES, INITIALLY_EXPECTED_PROC_NET_WORDS); + + bool copy_initial_ff_stats = ls->ff == NULL && ls->stats.ff.memory > 0; + ls->ff = procfile_reopen(ls->ff, filename, ls->ff ? NULL :" :", PROCFILE_FLAG_DEFAULT); + + // we just created ff, copy our old stats to it + if(ls->ff && copy_initial_ff_stats) ls->ff->stats = ls->stats.ff; + + ls->ff = procfile_readall(ls->ff); + if(!ls->ff) return false; + + // get the latest stats from ff; + ls->stats.ff = ls->ff->stats; + + for(size_t l = 1; l < procfile_lines(ls->ff) ;l++) { + size_t w = procfile_linewords(ls->ff, l); + if(!w) continue; + if(w < 14) { + local_sockets_log(ls, "too small line No %zu of filename '%s' (has %zu words)", l, filename, w); + continue; + } + + char *words[14] = { 0 }; + words[0] = procfile_lineword(ls->ff, l, 0); + words[1] = procfile_lineword(ls->ff, l, 1); + words[2] = procfile_lineword(ls->ff, l, 2); + words[3] = procfile_lineword(ls->ff, l, 3); + words[4] = procfile_lineword(ls->ff, l, 4); + words[5] = procfile_lineword(ls->ff, l, 5); + words[6] = procfile_lineword(ls->ff, l, 6); + words[7] = procfile_lineword(ls->ff, l, 7); + words[8] = procfile_lineword(ls->ff, l, 8); + words[9] = procfile_lineword(ls->ff, l, 9); + words[10] = procfile_lineword(ls->ff, l, 10); + words[11] = procfile_lineword(ls->ff, l, 11); + words[12] = procfile_lineword(ls->ff, l, 12); + words[13] = procfile_lineword(ls->ff, l, 13); + local_sockets_process_proc_line(ls, filename, family, protocol, l, words, _countof(words)); + } + + return true; +} + // -------------------------------------------------------------------------------------------------------------------- static inline void local_sockets_detect_directions(LS_STATE *ls) { @@ -1008,31 +1201,33 @@ static inline void local_sockets_init(LS_STATE *ls) { memset(&ls->stats, 0, sizeof(ls->stats)); -#ifdef HAVE_LIBMNL - ls->use_nl = false; - ls->nl = NULL; +#if defined(HAVE_LIBMNL) ls->tmp_protocol = 0; - local_sockets_libmnl_init(ls); #endif +#if defined(LOCAL_SOCKETS_USE_SETNS) if(ls->config.namespaces && ls->spawn_server == NULL) { ls->spawn_server = spawn_server_create(SPAWN_SERVER_OPTION_CALLBACK, NULL, local_sockets_spawn_server_callback, 0, NULL); ls->spawn_server_is_mine = true; } else ls->spawn_server_is_mine = false; +#endif } static inline void local_sockets_cleanup(LS_STATE *ls) { + if(ls->ff) { + ls->stats.ff = ls->ff->stats; + procfile_close(ls->ff); + ls->ff = NULL; + } +#if defined(LOCAL_SOCKETS_USE_SETNS) if(ls->spawn_server_is_mine) { spawn_server_destroy(ls->spawn_server); ls->spawn_server = NULL; ls->spawn_server_is_mine = false; } - -#ifdef HAVE_LIBMNL - local_sockets_libmnl_cleanup(ls); #endif // free the sockets hashtable data @@ -1070,28 +1265,93 @@ static inline void local_sockets_cleanup(LS_STATE *ls) { // -------------------------------------------------------------------------------------------------------------------- -static inline void local_sockets_do_family_protocol(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) { -#ifdef HAVE_LIBMNL - if(ls->nl && ls->use_nl) { - ls->use_nl = local_sockets_libmnl_get_sockets(ls, family, protocol); +static inline void local_sockets_track_time(LS_STATE *ls, const char *name) { + if(!ls->config.report || ls->timings_idx >= _countof(ls->timings)) + return; + + usec_t now_ut = now_monotonic_usec(); + + if(ls->timings_idx == 0 && !ls->timings[0].start_ut) { + ls->timings[0].start_ut = now_ut; + ls->timings[0].name = name; + } + else if(ls->timings_idx + 1 < _countof(ls->timings)) { + ls->timings[ls->timings_idx].end_ut = now_ut; + ls->timings_idx++; + ls->timings[ls->timings_idx].start_ut = now_ut; + ls->timings[ls->timings_idx].name = name; + } + else if(ls->timings_idx + 1 == _countof(ls->timings)) { + ls->timings[ls->timings_idx].end_ut = now_ut; + ls->timings_idx++; // out of bounds + } +} + +static void local_sockets_track_time_by_protocol(LS_STATE *ls, bool mnl, uint16_t family, uint16_t protocol) { + if(mnl) { + if(family == AF_INET) { + if(protocol == IPPROTO_TCP) + local_sockets_track_time(ls, "mnl_read_tcp4"); + else if(protocol == IPPROTO_UDP) + local_sockets_track_time(ls, "mnl_read_udp4"); + } + else if(family == AF_INET6) { + if(protocol == IPPROTO_TCP) + local_sockets_track_time(ls, "mnl_read_tcp6"); + else if(protocol == IPPROTO_UDP) + local_sockets_track_time(ls, "mnl_read_udp6"); + } + else + local_sockets_track_time(ls, "mnl_read_unknown"); + } + else { + if(family == AF_INET) { + if(protocol == IPPROTO_TCP) + local_sockets_track_time(ls, "proc_read_tcp4"); + else if(protocol == IPPROTO_UDP) + local_sockets_track_time(ls, "proc_read_udp4"); + } + else if(family == AF_INET6) { + if(protocol == IPPROTO_TCP) + local_sockets_track_time(ls, "proc_read_tcp6"); + else if(protocol == IPPROTO_UDP) + local_sockets_track_time(ls, "proc_read_udp6"); + } + else + local_sockets_track_time(ls, "proc_read_unknown"); + } +} - if(ls->use_nl) +static inline void local_sockets_do_family_protocol(LS_STATE *ls, const char *filename, uint16_t family, uint16_t protocol) { +#if defined(HAVE_LIBMNL) + if(!ls->config.no_mnl) { + local_sockets_track_time_by_protocol(ls, true, family, protocol); + if(local_sockets_libmnl_get_sockets(ls, family, protocol)) return; + + // else, do proc } #endif - local_sockets_read_proc_net_x(ls, filename, family, protocol); + local_sockets_track_time_by_protocol(ls, false, family, protocol); + + if(ls->config.procfile) + local_sockets_read_proc_net_x_procfile(ls, filename, family, protocol); + else + local_sockets_read_proc_net_x_getline(ls, filename, family, protocol); } static inline void local_sockets_read_all_system_sockets(LS_STATE *ls) { char path[FILENAME_MAX + 1]; if(ls->config.namespaces) { + local_sockets_track_time(ls, "read_namespaces"); snprintfz(path, sizeof(path), "%s/proc/self/ns/net", ls->config.host_prefix); local_sockets_read_proc_inode_link(ls, path, &ls->proc_self_net_ns_inode, "net"); } if(ls->config.cmdline || ls->config.comm || ls->config.pid || ls->config.namespaces) { + local_sockets_track_time(ls, "proc_read_pids"); snprintfz(path, sizeof(path), "%s/proc", ls->config.host_prefix); local_sockets_find_all_sockets_in_proc(ls, path); } @@ -1118,20 +1378,41 @@ static inline void local_sockets_read_all_system_sockets(LS_STATE *ls) { } // -------------------------------------------------------------------------------------------------------------------- +// switch namespaces to read namespace sockets + +#if defined(LOCAL_SOCKETS_USE_SETNS) struct local_sockets_child_work { int fd; uint64_t net_ns_inode; }; -static inline void local_sockets_send_to_parent(struct local_socket_state *ls __maybe_unused, struct local_socket *n, void *data) { +#define LOCAL_SOCKET_TERMINATOR (struct local_socket) { \ + .expires = UINT32_MAX, \ + .timer = UINT8_MAX, \ + .inode = UINT64_MAX, \ + .net_ns_inode = UINT64_MAX, \ +} + +static inline bool local_socket_is_terminator(const struct local_socket *n) { + static const struct local_socket t = LOCAL_SOCKET_TERMINATOR; + return (n->expires == t.expires && + n->timer == t.timer && + n->inode == t.inode && + n->net_ns_inode == t.net_ns_inode); +} + +static inline void local_sockets_send_to_parent(struct local_socket_state *ls, const struct local_socket *n, void *data) { struct local_sockets_child_work *cw = data; int fd = cw->fd; - if(n->net_ns_inode != cw->net_ns_inode) - return; - - // local_sockets_log(ls, "child is sending inode %"PRIu64" of namespace %"PRIu64, n->inode, n->net_ns_inode); + if(!local_socket_is_terminator(n)) { + ls->stats.errors_encountered = 0; +// local_sockets_log( +// ls, +// "child is sending inode %"PRIu64" of namespace %"PRIu64", from namespace %"PRIu64" for pid %d", +// n->inode, n->net_ns_inode, ls->proc_self_net_ns_inode, ls->ns_state.net_ns_pid); + } if(write(fd, n, sizeof(*n)) != sizeof(*n)) local_sockets_log(ls, "failed to write local socket to pipe"); @@ -1145,9 +1426,15 @@ static inline void local_sockets_send_to_parent(struct local_socket_state *ls __ local_sockets_log(ls, "failed to write cmdline to pipe"); } -static inline void local_sockets_spawn_server_callback(SPAWN_REQUEST *request) { +static inline int local_sockets_spawn_server_callback(SPAWN_REQUEST *request) { + static const struct local_socket terminator = LOCAL_SOCKET_TERMINATOR; + + struct local_sockets_ns_req *req = (struct local_sockets_ns_req *)request->data; + LS_STATE ls = { 0 }; - ls.config = *((struct local_sockets_config *)request->data); + ls.config = req->config; + ls.ns_state = req->ns_state; + ls.ns_state.nl_seq += gettid_uncached() * 10; // we don't need these inside namespaces ls.config.cmdline = false; @@ -1155,9 +1442,13 @@ static inline void local_sockets_spawn_server_callback(SPAWN_REQUEST *request) { ls.config.pid = false; ls.config.namespaces = false; +#if !defined(USE_LIBMNL_AFTER_SETNS) + ls.config.no_mnl = true; // disable mnl since this collects all sockets from the entire system +#endif + // initialize local sockets local_sockets_init(&ls); - + ls.proc_self_net_ns_inode = ls.ns_state.net_ns_inode; ls.config.host_prefix = ""; // we need the /proc of the container struct local_sockets_child_work cw = { @@ -1167,14 +1458,16 @@ static inline void local_sockets_spawn_server_callback(SPAWN_REQUEST *request) { ls.config.cb = local_sockets_send_to_parent; ls.config.data = &cw; - ls.proc_self_net_ns_inode = ls.config.net_ns_inode; // switch namespace using the custom fd passed via the spawn server if (setns(request->fds[3], CLONE_NEWNET) == -1) { local_sockets_log(&ls, "failed to switch network namespace at child process using fd %d", request->fds[3]); - exit(EXIT_FAILURE); + return EXIT_FAILURE; } + // close the custom fd + close(request->fds[3]); request->fds[3] = -1; + // read all sockets from /proc local_sockets_read_all_system_sockets(&ls); @@ -1182,12 +1475,11 @@ static inline void local_sockets_spawn_server_callback(SPAWN_REQUEST *request) { local_sockets_foreach_local_socket_call_cb(&ls); // send the terminating socket - struct local_socket zero = { - .net_ns_inode = ls.config.net_ns_inode, - }; - local_sockets_send_to_parent(&ls, &zero, &cw); + local_sockets_send_to_parent(&ls, &terminator, &cw); + + local_sockets_cleanup(&ls); - exit(EXIT_SUCCESS); + return EXIT_SUCCESS; } static inline bool local_sockets_get_namespace_sockets_with_pid(LS_STATE *ls, struct pid_socket *ps) { @@ -1198,6 +1490,8 @@ static inline bool local_sockets_get_namespace_sockets_with_pid(LS_STATE *ls, st int fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd == -1) { local_sockets_log(ls, "cannot open file '%s'", filename); + if(ls->config.report) + __atomic_add_fetch(&ls->stats.namespaces_absent, 1, __ATOMIC_RELAXED); return false; } @@ -1205,28 +1499,46 @@ static inline bool local_sockets_get_namespace_sockets_with_pid(LS_STATE *ls, st if (fstat(fd, &statbuf) == -1) { close(fd); local_sockets_log(ls, "failed to get file statistics for '%s'", filename); + if(ls->config.report) + __atomic_add_fetch(&ls->stats.namespaces_absent, 1, __ATOMIC_RELAXED); return false; } if (statbuf.st_ino != ps->net_ns_inode) { close(fd); local_sockets_log(ls, "pid %d is not in the wanted network namespace", ps->pid); + if(ls->config.report) + __atomic_add_fetch(&ls->stats.namespaces_invalid, 1, __ATOMIC_RELAXED); return false; } if(ls->spawn_server == NULL) { close(fd); local_sockets_log(ls, "spawn server is not available"); + if(ls->config.report) + __atomic_add_fetch(&ls->stats.namespaces_forks_failed, 1, __ATOMIC_RELAXED); return false; } - struct local_sockets_config config = ls->config; - config.net_ns_inode = ps->net_ns_inode; - SPAWN_INSTANCE *si = spawn_server_exec(ls->spawn_server, STDERR_FILENO, fd, NULL, &config, sizeof(config), SPAWN_INSTANCE_TYPE_CALLBACK); + struct local_sockets_ns_req req = { + .config = ls->config, + .ns_state = ls->ns_state, + }; + req.ns_state.net_ns_pid = ps->pid; + req.ns_state.net_ns_inode = ps->net_ns_inode; + + SPAWN_INSTANCE *si = spawn_server_exec(ls->spawn_server, STDERR_FILENO, fd, NULL, &req, sizeof(req), SPAWN_INSTANCE_TYPE_CALLBACK); close(fd); fd = -1; + if(ls->config.report) + __atomic_add_fetch(&ls->stats.namespaces_forks_attempted, 1, __ATOMIC_RELAXED); + if(si == NULL) { local_sockets_log(ls, "cannot create spawn instance"); + + if(ls->config.report) + __atomic_add_fetch(&ls->stats.namespaces_forks_failed, 1, __ATOMIC_RELAXED); + return false; } @@ -1251,36 +1563,35 @@ static inline bool local_sockets_get_namespace_sockets_with_pid(LS_STATE *ls, st received++; - struct local_socket zero = { - .net_ns_inode = ps->net_ns_inode, - }; - if(memcmp(&buf, &zero, sizeof(buf)) == 0) { - // the terminator + if(local_socket_is_terminator(&buf)) + // the child finished break; - } + + // overwrite the net_ns_inode we receive + buf.net_ns_inode = ps->net_ns_inode; spinlock_lock(&ls->spinlock); - SIMPLE_HASHTABLE_SLOT_LOCAL_SOCKET *sl = simple_hashtable_get_slot_LOCAL_SOCKET(&ls->sockets_hashtable, buf.inode, &buf, true); - LOCAL_SOCKET *n = SIMPLE_HASHTABLE_SLOT_DATA(sl); - if(n) { + if(!local_sockets_add_socket(ls, &buf)) { + // fprintf(stderr, "Failed to add duplicate namespace socket inode %"PRIu64"\n", buf.inode); string_freez(buf.cmdline); -// local_sockets_log(ls, -// "ns inode %" PRIu64" (comm: '%s', pid: %u, ns: %"PRIu64") already exists in hashtable (comm: '%s', pid: %u, ns: %"PRIu64") - ignoring duplicate", -// buf.inode, buf.comm, buf.pid, buf.net_ns_inode, n->comm, n->pid, n->net_ns_inode); + if(ls->config.report) + __atomic_add_fetch(&ls->stats.namespaces_sockets_existing, 1, __ATOMIC_RELAXED); } else { - n = aral_mallocz(ls->local_socket_aral); - memcpy(n, &buf, sizeof(*n)); - simple_hashtable_set_slot_LOCAL_SOCKET(&ls->sockets_hashtable, sl, n->inode, n); - - local_sockets_index_listening_port(ls, n); + // fprintf(stderr, "Added namespace socket inode %"PRIu64"\n", buf.inode); + if(ls->config.report) + __atomic_add_fetch(&ls->stats.namespaces_sockets_new, 1, __ATOMIC_RELAXED); } spinlock_unlock(&ls->spinlock); } spawn_server_exec_kill(ls->spawn_server, si); + + if(ls->config.report && received == 0) + __atomic_add_fetch(&ls->stats.namespaces_forks_unresponsive, 1, __ATOMIC_RELAXED); + return received > 0; } @@ -1289,7 +1600,7 @@ struct local_sockets_namespace_worker { uint64_t inode; }; -static inline void *local_sockets_get_namespace_sockets(void *arg) { +static inline void *local_sockets_get_namespace_sockets_worker(void *arg) { struct local_sockets_namespace_worker *data = arg; LS_STATE *ls = data->ls; const uint64_t inode = data->inode; @@ -1337,6 +1648,7 @@ static inline void local_sockets_namespaces(LS_STATE *ls) { const uint64_t inode = (uint64_t)SIMPLE_HASHTABLE_SLOT_DATA(sl); if(inode == ls->proc_self_net_ns_inode) + // skip our own namespace, we already have them continue; spinlock_unlock(&ls->spinlock); @@ -1356,8 +1668,10 @@ static inline void local_sockets_namespaces(LS_STATE *ls) { workers_data[last_thread].ls = ls; workers_data[last_thread].inode = inode; workers[last_thread] = nd_thread_create( - "local-sockets-worker", NETDATA_THREAD_OPTION_JOINABLE, - local_sockets_get_namespace_sockets, &workers_data[last_thread]); + "local-sockets-worker", + NETDATA_THREAD_OPTION_JOINABLE, + local_sockets_get_namespace_sockets_worker, + &workers_data[last_thread]); spinlock_lock(&ls->spinlock); } @@ -1371,49 +1685,137 @@ static inline void local_sockets_namespaces(LS_STATE *ls) { } } +#endif // LOCAL_SOCKETS_USE_SETNS + +// -------------------------------------------------------------------------------------------------------------------- +// read namespace sockets from the host's /proc + +#if !defined(LOCAL_SOCKETS_USE_SETNS) + +static inline bool local_sockets_namespaces_from_proc_with_pid(LS_STATE *ls, struct pid_socket *ps) { + char filename[1024]; + snprintfz(filename, sizeof(filename), "%s/proc/%d/ns/net", ls->config.host_prefix, ps->pid); + + // verify the pid is in the target namespace + int fd = open(filename, O_RDONLY | O_CLOEXEC); + if (fd == -1) { + local_sockets_log(ls, "cannot open file '%s'", filename); + if(ls->config.report) + __atomic_add_fetch(&ls->stats.namespaces_absent, 1, __ATOMIC_RELAXED); + return false; + } + + struct stat statbuf; + if (fstat(fd, &statbuf) == -1) { + close(fd); + local_sockets_log(ls, "failed to get file statistics for '%s'", filename); + if(ls->config.report) + __atomic_add_fetch(&ls->stats.namespaces_absent, 1, __ATOMIC_RELAXED); + return false; + } + + if (statbuf.st_ino != ps->net_ns_inode) { + close(fd); + local_sockets_log(ls, "pid %d is not in the wanted network namespace", ps->pid); + if(ls->config.report) + __atomic_add_fetch(&ls->stats.namespaces_invalid, 1, __ATOMIC_RELAXED); + return false; + } + + char path[FILENAME_MAX + 1]; + + if(ls->config.tcp4) { + snprintfz(path, sizeof(path), "%s/proc/%d/net/tcp", ls->config.host_prefix, ps->pid); + if(!local_sockets_read_proc_net_x(ls, path, AF_INET, IPPROTO_TCP)) + return false; + } + + if(ls->config.udp4) { + snprintfz(path, sizeof(path), "%s/proc/%d/net/udp", ls->config.host_prefix, ps->pid); + if(!local_sockets_read_proc_net_x(ls, path, AF_INET, IPPROTO_UDP)) + return false; + } + + if(ls->config.tcp6) { + snprintfz(path, sizeof(path), "%s/proc/%d/net/tcp6", ls->config.host_prefix, ps->pid); + if(!local_sockets_read_proc_net_x(ls, path, AF_INET6, IPPROTO_TCP)) + return false; + } + + if(ls->config.udp6) { + snprintfz(path, sizeof(path), "%s/proc/%d/net/udp6", ls->config.host_prefix, ps->pid); + if(!local_sockets_read_proc_net_x(ls, path, AF_INET6, IPPROTO_UDP)) + return false; + } + + return true; +} + +static inline void local_sockets_namespaces_from_proc(LS_STATE *ls) { + for(SIMPLE_HASHTABLE_SLOT_NET_NS *sl = simple_hashtable_first_read_only_NET_NS(&ls->ns_hashtable); + sl; + sl = simple_hashtable_next_read_only_NET_NS(&ls->ns_hashtable, sl)) { + const uint64_t inode = (uint64_t)SIMPLE_HASHTABLE_SLOT_DATA(sl); + + if (inode == ls->proc_self_net_ns_inode) + // skip our own namespace, we already have them + continue; + + ls->stats.namespaces_found++; + + for(SIMPLE_HASHTABLE_SLOT_PID_SOCKET *sl_pid = simple_hashtable_first_read_only_PID_SOCKET(&ls->pid_sockets_hashtable) ; + sl_pid ; + sl_pid = simple_hashtable_next_read_only_PID_SOCKET(&ls->pid_sockets_hashtable, sl_pid)) { + struct pid_socket *ps = SIMPLE_HASHTABLE_SLOT_DATA(sl_pid); + if(!ps || ps->net_ns_inode != inode) continue; + + // now we have a pid that has the same namespace inode + + if(local_sockets_namespaces_from_proc_with_pid(ls, ps)) + break; + } + } +} + +#endif + // -------------------------------------------------------------------------------------------------------------------- static inline void local_sockets_process(LS_STATE *ls) { + ls->timings_idx = 0; + local_sockets_track_time(ls, "init"); + // initialize our hashtables local_sockets_init(ls); + local_sockets_track_time(ls, "all_sockets"); + // read all sockets from /proc local_sockets_read_all_system_sockets(ls); // check all socket namespaces - if(ls->config.namespaces) + if(ls->config.namespaces) { + local_sockets_track_time(ls, "switch_namespaces"); +#if defined(LOCAL_SOCKETS_USE_SETNS) local_sockets_namespaces(ls); +#else + local_sockets_namespaces_from_proc(ls); +#endif + } // detect the directions of the sockets - if(ls->config.inbound || ls->config.outbound || ls->config.local) + if(ls->config.inbound || ls->config.outbound || ls->config.local) { + local_sockets_track_time(ls, "detect_direction"); local_sockets_detect_directions(ls); + } // call the callback for each socket + local_sockets_track_time(ls, "output"); local_sockets_foreach_local_socket_call_cb(ls); // free all memory + local_sockets_track_time(ls, "cleanup"); local_sockets_cleanup(ls); } -static inline void ipv6_address_to_txt(struct in6_addr *in6_addr, char *dst) { - struct sockaddr_in6 sa = { 0 }; - - sa.sin6_family = AF_INET6; - sa.sin6_port = htons(0); - sa.sin6_addr = *in6_addr; - - // Convert to human-readable format - if (inet_ntop(AF_INET6, &(sa.sin6_addr), dst, INET6_ADDRSTRLEN) == NULL) - *dst = '\0'; -} - -static inline void ipv4_address_to_txt(uint32_t ip, char *dst) { - uint8_t octets[4]; - octets[0] = ip & 0xFF; - octets[1] = (ip >> 8) & 0xFF; - octets[2] = (ip >> 16) & 0xFF; - octets[3] = (ip >> 24) & 0xFF; - sprintf(dst, "%u.%u.%u.%u", octets[0], octets[1], octets[2], octets[3]); -} - #endif //NETDATA_LOCAL_SOCKETS_H diff --git a/src/libnetdata/locks/README.md b/src/libnetdata/locks/README.md index 35d602f2a..25916b002 100644 --- a/src/libnetdata/locks/README.md +++ b/src/libnetdata/locks/README.md @@ -1,12 +1,3 @@ -<!-- -title: "Locks" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/locks/README.md -sidebar_label: "Locks" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Locks ## How to trace netdata locks @@ -58,13 +49,13 @@ If any call is expected to pause the caller (ie the caller is attempting a read ``` RW_LOCK ON LOCK 0x0x5651c9fcce20: 4190039 'HEALTH' (function health_execute_pending_updates() 661@health/health.c) WANTS a 'W' lock (while holding 1 rwlocks and 1 mutexes). There are 7 readers and 0 writers are holding the lock: - => 1: RW_LOCK: process 4190091 'WEB_SERVER[static14]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 709847 usec. - => 2: RW_LOCK: process 4190079 'WEB_SERVER[static6]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 709869 usec. - => 3: RW_LOCK: process 4190084 'WEB_SERVER[static10]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 709948 usec. - => 4: RW_LOCK: process 4190076 'WEB_SERVER[static3]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710190 usec. - => 5: RW_LOCK: process 4190092 'WEB_SERVER[static15]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710195 usec. - => 6: RW_LOCK: process 4190077 'WEB_SERVER[static4]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710208 usec. - => 7: RW_LOCK: process 4190044 'WEB_SERVER[static1]' (function web_client_api_request_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710221 usec. + => 1: RW_LOCK: process 4190091 'WEB_SERVER[static14]' (function api_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 709847 usec. + => 2: RW_LOCK: process 4190079 'WEB_SERVER[static6]' (function api_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 709869 usec. + => 3: RW_LOCK: process 4190084 'WEB_SERVER[static10]' (function api_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 709948 usec. + => 4: RW_LOCK: process 4190076 'WEB_SERVER[static3]' (function api_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710190 usec. + => 5: RW_LOCK: process 4190092 'WEB_SERVER[static15]' (function api_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710195 usec. + => 6: RW_LOCK: process 4190077 'WEB_SERVER[static4]' (function api_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710208 usec. + => 7: RW_LOCK: process 4190044 'WEB_SERVER[static1]' (function api_v1_data() 526@web/api/web_api_v1.c) is having 1 'R' lock for 710221 usec. ``` And each of the above is paired with a `GOT` log, like this: diff --git a/src/libnetdata/log/README.md b/src/libnetdata/log/README.md index ef9ca1ef3..c7a42f28b 100644 --- a/src/libnetdata/log/README.md +++ b/src/libnetdata/log/README.md @@ -1,12 +1,3 @@ -<!-- -title: "Log" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/log/README.md -sidebar_label: "Log" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Netdata Logging This document describes how Netdata generates its own logs, not how Netdata manages and queries logs databases. @@ -26,14 +17,15 @@ For each log source, Netdata supports the following output methods: - **off**, to disable this log source - **journal**, to send the logs to systemd-journal. +- **etw**, to send the logs to Event Tracing for Windows (ETW). +- **wel**, to send the logs to the Windows Event Log (WEL). - **syslog**, to send the logs to syslog. - **system**, to send the output to `stderr` or `stdout` depending on the log source. - **stdout**, to write the logs to Netdata's `stdout`. - **stderr**, to write the logs to Netdata's `stderr`. - **filename**, to send the logs to a file. -For `daemon` and `collector` the default is `journal` when systemd-journal is available. -To decide if systemd-journal is available, Netdata checks: +On Linux, when systemd-journal is available, the default is `journal` for `daemon` and `collector` and `filename` for the rest. To decide if systemd-journal is available, Netdata checks: 1. `stderr` is connected to systemd-journald 2. `/run/systemd/journal/socket` exists @@ -41,13 +33,16 @@ To decide if systemd-journal is available, Netdata checks: If any of the above is detected, Netdata will select `journal` for `daemon` and `collector` sources. -All other sources default to a file. +On Windows, the default is `etw` and if that is not available it falls back to `wel`. The availability of `etw` is decided at compile time. ## Log formats | Format | Description | |---------|--------------------------------------------------------------------------------------------------------| | journal | journald-specific log format. Automatically selected when logging to systemd-journal. | +| etw | Event Tracing for Windows specific format. Structured logging in Event Viewer. | +| wel | Windows Event Log specific format. Basic field-based logging in Event Viewer. | +| journal | journald-specific log format. Automatically selected when logging to systemd-journal. | | logfmt | logs data as a series of key/value pairs. The default when logging to any output other than `journal`. | | json | logs data in JSON format. | @@ -66,6 +61,9 @@ Each time Netdata logs, it assigns a priority to the log. It can be one of this | info | the default log level about information the user should know. | | debug | these are more verbose logs that can be ignored. | +For `etw` these are mapped to `Verbose`, `Informational`, `Warning`, `Error` and `Critical`. +For `wel` these are mapped to `Informational`, `Warning`, `Error`. + ## Logs Configuration In `netdata.conf`, there are the following settings: @@ -73,7 +71,7 @@ In `netdata.conf`, there are the following settings: ``` [logs] # logs to trigger flood protection = 1000 - # logs flood protection period = 60 + # logs flood protection period = 1m # facility = daemon # level = info # daemon = journal @@ -117,66 +115,69 @@ Sending a `SIGHUP` to Netdata, will instruct it to re-open all its log files. <details> <summary>All fields exposed by Netdata</summary> -| journal | logfmt | json | Description | -|:--------------------------------------:|:------------------------------:|:------------------------------:|:---------------------------------------------------------------------------------------------------------:| -| `_SOURCE_REALTIME_TIMESTAMP` | `time` | `time` | the timestamp of the event | -| `SYSLOG_IDENTIFIER` | `comm` | `comm` | the program logging the event | -| `ND_LOG_SOURCE` | `source` | `source` | one of the [log sources](#log-sources) | -| `PRIORITY`<br/>numeric | `level`<br/>text | `level`<br/>numeric | one of the [log levels](#log-levels) | -| `ERRNO` | `errno` | `errno` | the numeric value of `errno` | -| `INVOCATION_ID` | - | - | a unique UUID of the Netdata session, reset on every Netdata restart, inherited by systemd when available | -| `CODE_LINE` | - | - | the line number of of the source code logging this event | -| `CODE_FILE` | - | - | the filename of the source code logging this event | -| `CODE_FUNCTION` | - | - | the function name of the source code logging this event | -| `TID` | `tid` | `tid` | the thread id of the thread logging this event | -| `THREAD_TAG` | `thread` | `thread` | the name of the thread logging this event | -| `MESSAGE_ID` | `msg_id` | `msg_id` | see [message IDs](#message-ids) | -| `ND_MODULE` | `module` | `module` | the Netdata module logging this event | -| `ND_NIDL_NODE` | `node` | `node` | the hostname of the node the event is related to | -| `ND_NIDL_INSTANCE` | `instance` | `instance` | the instance of the node the event is related to | -| `ND_NIDL_CONTEXT` | `context` | `context` | the context the event is related to (this is usually the chart name, as shown on netdata dashboards | -| `ND_NIDL_DIMENSION` | `dimension` | `dimension` | the dimension the event is related to | -| `ND_SRC_TRANSPORT` | `src_transport` | `src_transport` | when the event happened during a request, this is the request transport | -| `ND_SRC_IP` | `src_ip` | `src_ip` | when the event happened during an inbound request, this is the IP the request came from | -| `ND_SRC_PORT` | `src_port` | `src_port` | when the event happened during an inbound request, this is the port the request came from | -| `ND_SRC_FORWARDED_HOST` | `src_forwarded_host` | `src_forwarded_host` | the contents of the HTTP header `X-Forwarded-Host` | -| `ND_SRC_FORWARDED_FOR` | `src_forwarded_for` | `src_forwarded_for` | the contents of the HTTP header `X-Forwarded-For` | -| `ND_SRC_CAPABILITIES` | `src_capabilities` | `src_capabilities` | when the request came from a child, this is the communication capabilities of the child | -| `ND_DST_TRANSPORT` | `dst_transport` | `dst_transport` | when the event happened during an outbound request, this is the outbound request transport | -| `ND_DST_IP` | `dst_ip` | `dst_ip` | when the event happened during an outbound request, this is the IP the request destination | -| `ND_DST_PORT` | `dst_port` | `dst_port` | when the event happened during an outbound request, this is the port the request destination | -| `ND_DST_CAPABILITIES` | `dst_capabilities` | `dst_capabilities` | when the request goes to a parent, this is the communication capabilities of the parent | -| `ND_REQUEST_METHOD` | `req_method` | `req_method` | when the event happened during an inbound request, this is the method the request was received | -| `ND_RESPONSE_CODE` | `code` | `code` | when responding to a request, this this the response code | -| `ND_CONNECTION_ID` | `conn` | `conn` | when there is a connection id for an inbound connection, this is the connection id | -| `ND_TRANSACTION_ID` | `transaction` | `transaction` | the transaction id (UUID) of all API requests | -| `ND_RESPONSE_SENT_BYTES` | `sent_bytes` | `sent_bytes` | the bytes we sent to API responses | -| `ND_RESPONSE_SIZE_BYTES` | `size_bytes` | `size_bytes` | the uncompressed bytes of the API responses | -| `ND_RESPONSE_PREP_TIME_USEC` | `prep_ut` | `prep_ut` | the time needed to prepare a response | -| `ND_RESPONSE_SENT_TIME_USEC` | `sent_ut` | `sent_ut` | the time needed to send a response | -| `ND_RESPONSE_TOTAL_TIME_USEC` | `total_ut` | `total_ut` | the total time needed to complete a response | -| `ND_ALERT_ID` | `alert_id` | `alert_id` | the alert id this event is related to | -| `ND_ALERT_EVENT_ID` | `alert_event_id` | `alert_event_id` | a sequential number of the alert transition (per host) | -| `ND_ALERT_UNIQUE_ID` | `alert_unique_id` | `alert_unique_id` | a sequential number of the alert transition (per alert) | -| `ND_ALERT_TRANSITION_ID` | `alert_transition_id` | `alert_transition_id` | the unique UUID of this alert transition | -| `ND_ALERT_CONFIG` | `alert_config` | `alert_config` | the alert configuration hash (UUID) | -| `ND_ALERT_NAME` | `alert` | `alert` | the alert name | -| `ND_ALERT_CLASS` | `alert_class` | `alert_class` | the alert classification | -| `ND_ALERT_COMPONENT` | `alert_component` | `alert_component` | the alert component | -| `ND_ALERT_TYPE` | `alert_type` | `alert_type` | the alert type | -| `ND_ALERT_EXEC` | `alert_exec` | `alert_exec` | the alert notification program | -| `ND_ALERT_RECIPIENT` | `alert_recipient` | `alert_recipient` | the alert recipient(s) | -| `ND_ALERT_VALUE` | `alert_value` | `alert_value` | the current alert value | -| `ND_ALERT_VALUE_OLD` | `alert_value_old` | `alert_value_old` | the previous alert value | -| `ND_ALERT_STATUS` | `alert_status` | `alert_status` | the current alert status | -| `ND_ALERT_STATUS_OLD` | `alert_value_old` | `alert_value_old` | the previous alert value | -| `ND_ALERT_UNITS` | `alert_units` | `alert_units` | the units of the alert | -| `ND_ALERT_SUMMARY` | `alert_summary` | `alert_summary` | the summary text of the alert | -| `ND_ALERT_INFO` | `alert_info` | `alert_info` | the info text of the alert | -| `ND_ALERT_DURATION` | `alert_duration` | `alert_duration` | the duration the alert was in its previous state | -| `ND_ALERT_NOTIFICATION_TIMESTAMP_USEC` | `alert_notification_timestamp` | `alert_notification_timestamp` | the timestamp the notification delivery is scheduled | -| `ND_REQUEST` | `request` | `request` | the full request during which the event happened | -| `MESSAGE` | `msg` | `msg` | the event message | +| `journal` | `logfmt` and `json` | `etw` | `wel` | Description | +|:--------------------------------------:|:------------------------------:|:-----------------------------:|:-----:|:----------------------------------------------------------------------------------------------------------| +| `_SOURCE_REALTIME_TIMESTAMP` | `time` | `Timestamp` | 1 | the timestamp of the event | +| `SYSLOG_IDENTIFIER` | `comm` | `Program` | 2 | the program logging the event | +| `ND_LOG_SOURCE` | `source` | `NetdataLogSource` | 3 | one of the [log sources](#log-sources) | +| `PRIORITY`<br/>numeric | `level`<br/>text | `Level`<br/>text | 4 | one of the [log levels](#log-levels) | +| `ERRNO` | `errno` | `UnixErrno` | 5 | the numeric value of `errno` | +| `INVOCATION_ID` | - | `InvocationID` | 7 | a unique UUID of the Netdata session, reset on every Netdata restart, inherited by systemd when available | +| `CODE_LINE` | - | `CodeLine` | 8 | the line number of of the source code logging this event | +| `CODE_FILE` | - | `CodeFile` | 9 | the filename of the source code logging this event | +| `CODE_FUNCTION` | - | `CodeFunction` | 10 | the function name of the source code logging this event | +| `TID` | `tid` | `ThreadID` | 11 | the thread id of the thread logging this event | +| `THREAD_TAG` | `thread` | `ThreadName` | 12 | the name of the thread logging this event | +| `MESSAGE_ID` | `msg_id` | `MessageID` | 13 | see [message IDs](#message-ids) | +| `ND_MODULE` | `module` | `Module` | 14 | the Netdata module logging this event | +| `ND_NIDL_NODE` | `node` | `Node` | 15 | the hostname of the node the event is related to | +| `ND_NIDL_INSTANCE` | `instance` | `Instance` | 16 | the instance of the node the event is related to | +| `ND_NIDL_CONTEXT` | `context` | `Context` | 17 | the context the event is related to (this is usually the chart name, as shown on netdata dashboards | +| `ND_NIDL_DIMENSION` | `dimension` | `Dimension` | 18 | the dimension the event is related to | +| `ND_SRC_TRANSPORT` | `src_transport` | `SourceTransport` | 19 | when the event happened during a request, this is the request transport | +| `ND_SRC_IP` | `src_ip` | `SourceIP` | 24 | when the event happened during an inbound request, this is the IP the request came from | +| `ND_SRC_PORT` | `src_port` | `SourcePort` | 25 | when the event happened during an inbound request, this is the port the request came from | +| `ND_SRC_FORWARDED_HOST` | `src_forwarded_host` | `SourceForwardedHost` | 26 | the contents of the HTTP header `X-Forwarded-Host` | +| `ND_SRC_FORWARDED_FOR` | `src_forwarded_for` | `SourceForwardedFor` | 27 | the contents of the HTTP header `X-Forwarded-For` | +| `ND_SRC_CAPABILITIES` | `src_capabilities` | `SourceCapabilities` | 28 | when the request came from a child, this is the communication capabilities of the child | +| `ND_DST_TRANSPORT` | `dst_transport` | `DestinationTransport` | 29 | when the event happened during an outbound request, this is the outbound request transport | +| `ND_DST_IP` | `dst_ip` | `DestinationIP` | 30 | when the event happened during an outbound request, this is the IP the request destination | +| `ND_DST_PORT` | `dst_port` | `DestinationPort` | 31 | when the event happened during an outbound request, this is the port the request destination | +| `ND_DST_CAPABILITIES` | `dst_capabilities` | `DestinationCapabilities` | 32 | when the request goes to a parent, this is the communication capabilities of the parent | +| `ND_REQUEST_METHOD` | `req_method` | `RequestMethod` | 33 | when the event happened during an inbound request, this is the method the request was received | +| `ND_RESPONSE_CODE` | `code` | `ResponseCode` | 34 | when responding to a request, this this the response code | +| `ND_CONNECTION_ID` | `conn` | `ConnectionID` | 35 | when there is a connection id for an inbound connection, this is the connection id | +| `ND_TRANSACTION_ID` | `transaction` | `TransactionID` | 36 | the transaction id (UUID) of all API requests | +| `ND_RESPONSE_SENT_BYTES` | `sent_bytes` | `ResponseSentBytes` | 37 | the bytes we sent to API responses | +| `ND_RESPONSE_SIZE_BYTES` | `size_bytes` | `ResponseSizeBytes` | 38 | the uncompressed bytes of the API responses | +| `ND_RESPONSE_PREP_TIME_USEC` | `prep_ut` | `ResponsePreparationTimeUsec` | 39 | the time needed to prepare a response | +| `ND_RESPONSE_SENT_TIME_USEC` | `sent_ut` | `ResponseSentTimeUsec` | 40 | the time needed to send a response | +| `ND_RESPONSE_TOTAL_TIME_USEC` | `total_ut` | `ResponseTotalTimeUsec` | 41 | the total time needed to complete a response | +| `ND_ALERT_ID` | `alert_id` | `AlertID` | 42 | the alert id this event is related to | +| `ND_ALERT_EVENT_ID` | `alert_event_id` | `AlertEventID` | 44 | a sequential number of the alert transition (per host) | +| `ND_ALERT_UNIQUE_ID` | `alert_unique_id` | `AlertUniqueID` | 43 | a sequential number of the alert transition (per alert) | +| `ND_ALERT_TRANSITION_ID` | `alert_transition_id` | `AlertTransitionID` | 45 | the unique UUID of this alert transition | +| `ND_ALERT_CONFIG` | `alert_config` | `AlertConfig` | 46 | the alert configuration hash (UUID) | +| `ND_ALERT_NAME` | `alert` | `AlertName` | 47 | the alert name | +| `ND_ALERT_CLASS` | `alert_class` | `AlertClass` | 48 | the alert classification | +| `ND_ALERT_COMPONENT` | `alert_component` | `AlertComponent` | 49 | the alert component | +| `ND_ALERT_TYPE` | `alert_type` | `AlertType` | 50 | the alert type | +| `ND_ALERT_EXEC` | `alert_exec` | `AlertExec` | 51 | the alert notification program | +| `ND_ALERT_RECIPIENT` | `alert_recipient` | `AlertRecipient` | 52 | the alert recipient(s) | +| `ND_ALERT_VALUE` | `alert_value` | `AlertValue` | 54 | the current alert value | +| `ND_ALERT_VALUE_OLD` | `alert_value_old` | `AlertOldValue` | 55 | the previous alert value | +| `ND_ALERT_STATUS` | `alert_status` | `AlertStatus` | 56 | the current alert status | +| `ND_ALERT_STATUS_OLD` | `alert_value_old` | `AlertOldStatus` | 57 | the previous alert status | +| `ND_ALERT_UNITS` | `alert_units` | `AlertUnits` | 59 | the units of the alert | +| `ND_ALERT_SUMMARY` | `alert_summary` | `AlertSummary` | 60 | the summary text of the alert | +| `ND_ALERT_INFO` | `alert_info` | `AlertInfo` | 61 | the info text of the alert | +| `ND_ALERT_DURATION` | `alert_duration` | `AlertDuration` | 53 | the duration the alert was in its previous state | +| `ND_ALERT_NOTIFICATION_TIMESTAMP_USEC` | `alert_notification_timestamp` | `AlertNotificationTimeUsec` | 62 | the timestamp the notification delivery is scheduled | +| `ND_REQUEST` | `request` | `Request` | 63 | the full request during which the event happened | +| `MESSAGE` | `msg` | `Message` | 64 | the event message | + +For `wel` (Windows Event Logs), all logs have an array of 64 fields strings, and their index number provides their meaning. +For `etw` (Event Tracing for Windows), Netdata logs in a structured way, and field names are available. </details> @@ -221,3 +222,117 @@ journalctl -u netdata --namespace=netdata # All netdata logs, the newest entries are displayed first journalctl -u netdata --namespace=netdata -r ``` + +## Using Event Tracing for Windows (ETW) + +ETW requires the publisher `Netdata` to be registered. Our Windows installer does this automatically. + +Registering the publisher is done via a manifest (`%SystemRoot%\System32\wevt_netdata_manifest.xml`) +and its messages resources DLL (`%SystemRoot%\System32\wevt_netdata.dll`). + +If needed, the publisher can be registered and unregistered manually using these commands: + +```bat +REM register the Netdata publisher +wevtutil im "%SystemRoot%\System32\wevt_netdata_manifest.xml" "/mf:%SystemRoot%\System32\wevt_netdata.dll" "/rf:%SystemRoot%\System32\wevt_netdata.dll" + +REM unregister the Netdata publisher +wevtutil um "%SystemRoot%\System32\wevt_netdata_manifest.xml" +``` + +The structure of the logs are as follows: + + - Publisher `Netdata` + - Channel `Netdata/Daemon`: general messages about the Netdata service + - Channel `Netdata/Collector`: general messages about Netdata external plugins + - Channel `Netdata/Health`: alert transitions and general messages generated by Netdata's health engine + - Channel `Netdata/Access`: all accesses to Netdata APIs + - Channel `Netdata/Aclk`: for cloud connectivity tracing (disabled by default) + +Retention can be configured per Channel via the Event Viewer. Netdata does not set a default, so the system default is used. + +> **IMPORTANT**<br/> +> Event Tracing for Windows (ETW) does not allow logging the percentage character `%`. +> The `%` followed by a number, is recursively used for fields expansion and ETW has not +> provided any way to escape the character for preventing further expansion.<br/> +> <br/> +> To work around this limitation, Netdata replaces all `%` which are followed by a number, with `℅` +> (the Unicode character `care of`). Visually, they look similar, but when copying IPv6 addresses +> or URLs from the logs, you have to be careful to manually replace `℅` with `%` before using them. + +## Using Windows Event Logs (WEL) + +WEL has a different logs structure and unfortunately WEL and ETW need to use different names if they are to be used +concurrently. + +For WEL, Netdata logs as follows: + + - Channel `NetdataWEL` (unfortunately `Netdata` cannot be used, it conflicts with the ETW Publisher name) + - Publisher `NetdataDaemon`: general messages about the Netdata service + - Publisher `NetdataCollector`: general messages about Netdata external plugins + - Publisher `NetdataHealth`: alert transitions and general messages generated by Netdata's health engine + - Publisher `NetdataAccess`: all accesses to Netdata APIs + - Publisher `NetdataAclk`: for cloud connectivity tracing (disabled by default) + +Publishers must have unique names system-wide, so we had to prefix them with `Netdata`. + +Retention can be configured per Publisher via the Event Viewer or the Registry. +Netdata sets by default 20MiB for all of them, except `NetdataAclk` (5MiB) and `NetdataAccess` (35MiB), +for a total of 100MiB. + +For WEL some registry entries are needed. Netdata automatically takes care of them when it starts. + +WEL does not have the problem ETW has with the percent character `%`, so Netdata logs it as-is. + +## Differences between ETW and WEL + +There are key differences between ETW and WEL. + +### Publishers and Providers +**Publishers** are collections of ETW Providers. A Publisher is implied by a manifest file, +each of which is considered a Publisher, and each manifest file can define multiple **Providers** in it. +Other than that there is no entity related to **Publishers** in the system. + +**Publishers** are not defined for WEL. + +**Providers** are the applications or modules logging. Provider names must be unique across the system, +for ETW and WEL together. + +To define a **Provider**: + +- ETW requires a **Publisher** manifest coupled with resources DLLs and must be registered + via `wevtutil` (handled by the Netdata Windows installer automatically). +- WEL requires some registry entries and a message resources DLL (handled by Netdata automatically on startup). + +The Provider appears as `Source` in the Event Viewer, for both WEL and ETW. + +### Channels +- **Channels** for WEL are collections of WEL Providers, (each WEL Provider is a single Stream of logs). +- **Channels** for ETW slice the logs of each Provider into multiple Streams. + +WEL Channels cannot have the same name as ETW Providers. This is why Netdata's ETW provider is +called `Netdata`, and WEL channel is called `NetdataWEL`. + +Despite the fact that ETW **Publishers** and WEL **Channels** are both collections of Providers, +they are not similar. In ETW a Publisher is a collection on the publisher's Providers, but in WEL +a Channel may include independent WEL Providers (e.g. the "Applications" Channel). Additionally, +WEL Channels cannot include ETW Providers. + +### Retention +Retention is always defined per Stream. + +- Retention in ETW is defined per ETW Channel (ETW Provider Stream). +- Retention in WEL is defined per WEL Provider (each WEL Provider is a single Stream). + +### Messages Formatting +- ETW supports recursive fields expansion, and therefore `%N` in fields is expanded recursively + (or replaced with an error message if expansion fails). Netdata replaces `%N` with `℅N` to stop + recursive expansion (since `%N` cannot be logged otherwise). +- WEL performs a single field expansion, and therefore the `%` character in fields is never expanded. + +### Usability + +- ETW names all the fields and allows multiple datatypes per field, enabling log consumers to know + what each field means and its datatype. +- WEL uses a simple string table for fields, and consumers need to map these string fields based on + their index. diff --git a/src/libnetdata/log/log.c b/src/libnetdata/log/log.c deleted file mode 100644 index a31127c42..000000000 --- a/src/libnetdata/log/log.c +++ /dev/null @@ -1,2545 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -// do not REMOVE this, it is used by systemd-journal includes to prevent saving the file, function, line of the -// source code that makes the calls, allowing our loggers to log the lines of source code that actually log -#define SD_JOURNAL_SUPPRESS_LOCATION - -#include "../libnetdata.h" - -#if defined(OS_WINDOWS) -#include <windows.h> -#endif - -#ifdef __FreeBSD__ -#include <sys/endian.h> -#endif - -#ifdef __APPLE__ -#include <machine/endian.h> -#endif - -#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE) -#include <execinfo.h> -#endif - -#ifdef HAVE_SYSTEMD -#include <systemd/sd-journal.h> -#endif - -const char *program_name = ""; - -uint64_t debug_flags = 0; - -#ifdef ENABLE_ACLK -int aclklog_enabled = 0; -#endif - -// ---------------------------------------------------------------------------- - -struct nd_log_source; -static bool nd_log_limit_reached(struct nd_log_source *source); - -// ---------------------------------------------------------------------------- - -void errno_clear(void) { - errno = 0; - -#if defined(OS_WINDOWS) - SetLastError(ERROR_SUCCESS); -#endif -} - -// ---------------------------------------------------------------------------- -// logging method - -typedef enum __attribute__((__packed__)) { - NDLM_DISABLED = 0, - NDLM_DEVNULL, - NDLM_DEFAULT, - NDLM_JOURNAL, - NDLM_SYSLOG, - NDLM_STDOUT, - NDLM_STDERR, - NDLM_FILE, -} ND_LOG_METHOD; - -static struct { - ND_LOG_METHOD method; - const char *name; -} nd_log_methods[] = { - { .method = NDLM_DISABLED, .name = "none" }, - { .method = NDLM_DEVNULL, .name = "/dev/null" }, - { .method = NDLM_DEFAULT, .name = "default" }, - { .method = NDLM_JOURNAL, .name = "journal" }, - { .method = NDLM_SYSLOG, .name = "syslog" }, - { .method = NDLM_STDOUT, .name = "stdout" }, - { .method = NDLM_STDERR, .name = "stderr" }, - { .method = NDLM_FILE, .name = "file" }, -}; - -static ND_LOG_METHOD nd_log_method2id(const char *method) { - if(!method || !*method) - return NDLM_DEFAULT; - - size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]); - for(size_t i = 0; i < entries ;i++) { - if(strcmp(nd_log_methods[i].name, method) == 0) - return nd_log_methods[i].method; - } - - return NDLM_FILE; -} - -static const char *nd_log_id2method(ND_LOG_METHOD method) { - size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]); - for(size_t i = 0; i < entries ;i++) { - if(method == nd_log_methods[i].method) - return nd_log_methods[i].name; - } - - return "unknown"; -} - -#define IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ndlo) ((ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || (ndlo) == NDLM_STDERR) - -const char *nd_log_method_for_external_plugins(const char *s) { - if(s && *s) { - ND_LOG_METHOD method = nd_log_method2id(s); - if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method)) - return nd_log_id2method(method); - } - - return nd_log_id2method(NDLM_STDERR); -} - -// ---------------------------------------------------------------------------- -// workaround strerror_r() - -#if defined(STRERROR_R_CHAR_P) -// GLIBC version of strerror_r -static const char *strerror_result(const char *a, const char *b) { (void)b; return a; } -#elif defined(HAVE_STRERROR_R) -// POSIX version of strerror_r -static const char *strerror_result(int a, const char *b) { (void)a; return b; } -#elif defined(HAVE_C__GENERIC) - -// what a trick! -// http://stackoverflow.com/questions/479207/function-overloading-in-c -static const char *strerror_result_int(int a, const char *b) { (void)a; return b; } -static const char *strerror_result_string(const char *a, const char *b) { (void)b; return a; } - -#define strerror_result(a, b) _Generic((a), \ - int: strerror_result_int, \ - char *: strerror_result_string \ - )(a, b) - -#else -#error "cannot detect the format of function strerror_r()" -#endif - -static const char *errno2str(int errnum, char *buf, size_t size) { - return strerror_result(strerror_r(errnum, buf, size), buf); -} - -// ---------------------------------------------------------------------------- -// facilities -// -// sys/syslog.h (Linux) -// sys/sys/syslog.h (FreeBSD) -// bsd/sys/syslog.h (darwin-xnu) - -static struct { - int facility; - const char *name; -} nd_log_facilities[] = { - { LOG_AUTH, "auth" }, - { LOG_AUTHPRIV, "authpriv" }, - { LOG_CRON, "cron" }, - { LOG_DAEMON, "daemon" }, - { LOG_FTP, "ftp" }, - { LOG_KERN, "kern" }, - { LOG_LPR, "lpr" }, - { LOG_MAIL, "mail" }, - { LOG_NEWS, "news" }, - { LOG_SYSLOG, "syslog" }, - { LOG_USER, "user" }, - { LOG_UUCP, "uucp" }, - { LOG_LOCAL0, "local0" }, - { LOG_LOCAL1, "local1" }, - { LOG_LOCAL2, "local2" }, - { LOG_LOCAL3, "local3" }, - { LOG_LOCAL4, "local4" }, - { LOG_LOCAL5, "local5" }, - { LOG_LOCAL6, "local6" }, - { LOG_LOCAL7, "local7" }, - -#ifdef __FreeBSD__ - { LOG_CONSOLE, "console" }, - { LOG_NTP, "ntp" }, - - // FreeBSD does not consider 'security' as deprecated. - { LOG_SECURITY, "security" }, -#else - // For all other O/S 'security' is mapped to 'auth'. - { LOG_AUTH, "security" }, -#endif - -#ifdef __APPLE__ - { LOG_INSTALL, "install" }, - { LOG_NETINFO, "netinfo" }, - { LOG_RAS, "ras" }, - { LOG_REMOTEAUTH, "remoteauth" }, - { LOG_LAUNCHD, "launchd" }, - -#endif -}; - -static int nd_log_facility2id(const char *facility) { - size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]); - for(size_t i = 0; i < entries ;i++) { - if(strcmp(nd_log_facilities[i].name, facility) == 0) - return nd_log_facilities[i].facility; - } - - return LOG_DAEMON; -} - -static const char *nd_log_id2facility(int facility) { - size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]); - for(size_t i = 0; i < entries ;i++) { - if(nd_log_facilities[i].facility == facility) - return nd_log_facilities[i].name; - } - - return "daemon"; -} - -// ---------------------------------------------------------------------------- -// priorities - -static struct { - ND_LOG_FIELD_PRIORITY priority; - const char *name; -} nd_log_priorities[] = { - { .priority = NDLP_EMERG, .name = "emergency" }, - { .priority = NDLP_EMERG, .name = "emerg" }, - { .priority = NDLP_ALERT, .name = "alert" }, - { .priority = NDLP_CRIT, .name = "critical" }, - { .priority = NDLP_CRIT, .name = "crit" }, - { .priority = NDLP_ERR, .name = "error" }, - { .priority = NDLP_ERR, .name = "err" }, - { .priority = NDLP_WARNING, .name = "warning" }, - { .priority = NDLP_WARNING, .name = "warn" }, - { .priority = NDLP_NOTICE, .name = "notice" }, - { .priority = NDLP_INFO, .name = NDLP_INFO_STR }, - { .priority = NDLP_DEBUG, .name = "debug" }, -}; - -int nd_log_priority2id(const char *priority) { - size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]); - for(size_t i = 0; i < entries ;i++) { - if(strcmp(nd_log_priorities[i].name, priority) == 0) - return nd_log_priorities[i].priority; - } - - return NDLP_INFO; -} - -const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority) { - size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]); - for(size_t i = 0; i < entries ;i++) { - if(priority == nd_log_priorities[i].priority) - return nd_log_priorities[i].name; - } - - return NDLP_INFO_STR; -} - -// ---------------------------------------------------------------------------- -// log sources - -const char *nd_log_sources[] = { - [NDLS_UNSET] = "UNSET", - [NDLS_ACCESS] = "access", - [NDLS_ACLK] = "aclk", - [NDLS_COLLECTORS] = "collector", - [NDLS_DAEMON] = "daemon", - [NDLS_HEALTH] = "health", - [NDLS_DEBUG] = "debug", -}; - -size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def) { - size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]); - for(size_t i = 0; i < entries ;i++) { - if(strcmp(nd_log_sources[i], source) == 0) - return i; - } - - return def; -} - - -static const char *nd_log_id2source(ND_LOG_SOURCES source) { - size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]); - if(source < entries) - return nd_log_sources[source]; - - return nd_log_sources[NDLS_COLLECTORS]; -} - -// ---------------------------------------------------------------------------- -// log output formats - -typedef enum __attribute__((__packed__)) { - NDLF_JOURNAL, - NDLF_LOGFMT, - NDLF_JSON, -} ND_LOG_FORMAT; - -static struct { - ND_LOG_FORMAT format; - const char *name; -} nd_log_formats[] = { - { .format = NDLF_JOURNAL, .name = "journal" }, - { .format = NDLF_LOGFMT, .name = "logfmt" }, - { .format = NDLF_JSON, .name = "json" }, -}; - -static ND_LOG_FORMAT nd_log_format2id(const char *format) { - if(!format || !*format) - return NDLF_LOGFMT; - - size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]); - for(size_t i = 0; i < entries ;i++) { - if(strcmp(nd_log_formats[i].name, format) == 0) - return nd_log_formats[i].format; - } - - return NDLF_LOGFMT; -} - -static const char *nd_log_id2format(ND_LOG_FORMAT format) { - size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]); - for(size_t i = 0; i < entries ;i++) { - if(format == nd_log_formats[i].format) - return nd_log_formats[i].name; - } - - return "logfmt"; -} - -// ---------------------------------------------------------------------------- -// format dates - -void log_date(char *buffer, size_t len, time_t now) { - if(unlikely(!buffer || !len)) - return; - - time_t t = now; - struct tm *tmp, tmbuf; - - tmp = localtime_r(&t, &tmbuf); - - if (unlikely(!tmp)) { - buffer[0] = '\0'; - return; - } - - if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0)) - buffer[0] = '\0'; - - buffer[len - 1] = '\0'; -} - -// ---------------------------------------------------------------------------- - -struct nd_log_limit { - usec_t started_monotonic_ut; - uint32_t counter; - uint32_t prevented; - - uint32_t throttle_period; - uint32_t logs_per_period; - uint32_t logs_per_period_backup; -}; - -#define ND_LOG_LIMITS_DEFAULT (struct nd_log_limit){ .logs_per_period = ND_LOG_DEFAULT_THROTTLE_LOGS, .logs_per_period_backup = ND_LOG_DEFAULT_THROTTLE_LOGS, .throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD, } -#define ND_LOG_LIMITS_UNLIMITED (struct nd_log_limit){ .logs_per_period = 0, .logs_per_period_backup = 0, .throttle_period = 0, } - -struct nd_log_source { - SPINLOCK spinlock; - ND_LOG_METHOD method; - ND_LOG_FORMAT format; - const char *filename; - int fd; - FILE *fp; - - ND_LOG_FIELD_PRIORITY min_priority; - const char *pending_msg; - struct nd_log_limit limits; -}; - -static struct { - nd_uuid_t invocation_id; - - ND_LOG_SOURCES overwrite_process_source; - - struct nd_log_source sources[_NDLS_MAX]; - - struct { - bool initialized; - } journal; - - struct { - bool initialized; - int fd; - char filename[FILENAME_MAX + 1]; - } journal_direct; - - struct { - bool initialized; - int facility; - } syslog; - - struct { - SPINLOCK spinlock; - bool initialized; - } std_output; - - struct { - SPINLOCK spinlock; - bool initialized; - } std_error; - -} nd_log = { - .overwrite_process_source = 0, - .journal = { - .initialized = false, - }, - .journal_direct = { - .initialized = false, - .fd = -1, - }, - .syslog = { - .initialized = false, - .facility = LOG_DAEMON, - }, - .std_output = { - .spinlock = NETDATA_SPINLOCK_INITIALIZER, - .initialized = false, - }, - .std_error = { - .spinlock = NETDATA_SPINLOCK_INITIALIZER, - .initialized = false, - }, - .sources = { - [NDLS_UNSET] = { - .spinlock = NETDATA_SPINLOCK_INITIALIZER, - .method = NDLM_DISABLED, - .format = NDLF_JOURNAL, - .filename = NULL, - .fd = -1, - .fp = NULL, - .min_priority = NDLP_EMERG, - .limits = ND_LOG_LIMITS_UNLIMITED, - }, - [NDLS_ACCESS] = { - .spinlock = NETDATA_SPINLOCK_INITIALIZER, - .method = NDLM_DEFAULT, - .format = NDLF_LOGFMT, - .filename = LOG_DIR "/access.log", - .fd = -1, - .fp = NULL, - .min_priority = NDLP_DEBUG, - .limits = ND_LOG_LIMITS_UNLIMITED, - }, - [NDLS_ACLK] = { - .spinlock = NETDATA_SPINLOCK_INITIALIZER, - .method = NDLM_FILE, - .format = NDLF_LOGFMT, - .filename = LOG_DIR "/aclk.log", - .fd = -1, - .fp = NULL, - .min_priority = NDLP_DEBUG, - .limits = ND_LOG_LIMITS_UNLIMITED, - }, - [NDLS_COLLECTORS] = { - .spinlock = NETDATA_SPINLOCK_INITIALIZER, - .method = NDLM_DEFAULT, - .format = NDLF_LOGFMT, - .filename = LOG_DIR "/collectors.log", - .fd = STDERR_FILENO, - .fp = NULL, - .min_priority = NDLP_INFO, - .limits = ND_LOG_LIMITS_DEFAULT, - }, - [NDLS_DEBUG] = { - .spinlock = NETDATA_SPINLOCK_INITIALIZER, - .method = NDLM_DISABLED, - .format = NDLF_LOGFMT, - .filename = LOG_DIR "/debug.log", - .fd = STDOUT_FILENO, - .fp = NULL, - .min_priority = NDLP_DEBUG, - .limits = ND_LOG_LIMITS_UNLIMITED, - }, - [NDLS_DAEMON] = { - .spinlock = NETDATA_SPINLOCK_INITIALIZER, - .method = NDLM_DEFAULT, - .filename = LOG_DIR "/daemon.log", - .format = NDLF_LOGFMT, - .fd = -1, - .fp = NULL, - .min_priority = NDLP_INFO, - .limits = ND_LOG_LIMITS_DEFAULT, - }, - [NDLS_HEALTH] = { - .spinlock = NETDATA_SPINLOCK_INITIALIZER, - .method = NDLM_DEFAULT, - .format = NDLF_LOGFMT, - .filename = LOG_DIR "/health.log", - .fd = -1, - .fp = NULL, - .min_priority = NDLP_DEBUG, - .limits = ND_LOG_LIMITS_UNLIMITED, - }, - }, -}; - -__attribute__((constructor)) void initialize_invocation_id(void) { - // check for a NETDATA_INVOCATION_ID - if(uuid_parse_flexi(getenv("NETDATA_INVOCATION_ID"), nd_log.invocation_id) != 0) { - // not found, check for systemd set INVOCATION_ID - if(uuid_parse_flexi(getenv("INVOCATION_ID"), nd_log.invocation_id) != 0) { - // not found, generate a new one - uuid_generate_random(nd_log.invocation_id); - } - } - - char uuid[UUID_COMPACT_STR_LEN]; - uuid_unparse_lower_compact(nd_log.invocation_id, uuid); - setenv("NETDATA_INVOCATION_ID", uuid, 1); -} - -int nd_log_health_fd(void) { - if(nd_log.sources[NDLS_HEALTH].method == NDLM_FILE && nd_log.sources[NDLS_HEALTH].fd != -1) - return nd_log.sources[NDLS_HEALTH].fd; - - return STDERR_FILENO; -} - -int nd_log_collectors_fd(void) { - if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_FILE && nd_log.sources[NDLS_COLLECTORS].fd != -1) - return nd_log.sources[NDLS_COLLECTORS].fd; - - return STDERR_FILENO; -} - -void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting) { - char buf[FILENAME_MAX + 100]; - if(setting && *setting) - strncpyz(buf, setting, sizeof(buf) - 1); - else - buf[0] = '\0'; - - struct nd_log_source *ls = &nd_log.sources[source]; - char *output = strrchr(buf, '@'); - - if(!output) - // all of it is the output - output = buf; - else { - // we found an '@', the next char is the output - *output = '\0'; - output++; - - // parse the other params - char *remaining = buf; - while(remaining) { - char *value = strsep_skip_consecutive_separators(&remaining, ","); - if (!value || !*value) continue; - - char *name = strsep_skip_consecutive_separators(&value, "="); - if (!name || !*name) continue; - - if(strcmp(name, "logfmt") == 0) - ls->format = NDLF_LOGFMT; - else if(strcmp(name, "json") == 0) - ls->format = NDLF_JSON; - else if(strcmp(name, "journal") == 0) - ls->format = NDLF_JOURNAL; - else if(strcmp(name, "level") == 0 && value && *value) - ls->min_priority = nd_log_priority2id(value); - else if(strcmp(name, "protection") == 0 && value && *value) { - if(strcmp(value, "off") == 0 || strcmp(value, "none") == 0) { - ls->limits = ND_LOG_LIMITS_UNLIMITED; - ls->limits.counter = 0; - ls->limits.prevented = 0; - } - else { - ls->limits = ND_LOG_LIMITS_DEFAULT; - - char *slash = strchr(value, '/'); - if(slash) { - *slash = '\0'; - slash++; - ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value); - ls->limits.throttle_period = str2u(slash); - } - else { - ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value); - ls->limits.throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD; - } - } - } - else - nd_log(NDLS_DAEMON, NDLP_ERR, "Error while parsing configuration of log source '%s'. " - "In config '%s', '%s' is not understood.", - nd_log_id2source(source), setting, name); - } - } - - if(!output || !*output || strcmp(output, "none") == 0 || strcmp(output, "off") == 0) { - ls->method = NDLM_DISABLED; - ls->filename = "/dev/null"; - } - else if(strcmp(output, "journal") == 0) { - ls->method = NDLM_JOURNAL; - ls->filename = NULL; - } - else if(strcmp(output, "syslog") == 0) { - ls->method = NDLM_SYSLOG; - ls->filename = NULL; - } - else if(strcmp(output, "/dev/null") == 0) { - ls->method = NDLM_DEVNULL; - ls->filename = "/dev/null"; - } - else if(strcmp(output, "system") == 0) { - if(ls->fd == STDERR_FILENO) { - ls->method = NDLM_STDERR; - ls->filename = NULL; - ls->fd = STDERR_FILENO; - } - else { - ls->method = NDLM_STDOUT; - ls->filename = NULL; - ls->fd = STDOUT_FILENO; - } - } - else if(strcmp(output, "stderr") == 0) { - ls->method = NDLM_STDERR; - ls->filename = NULL; - ls->fd = STDERR_FILENO; - } - else if(strcmp(output, "stdout") == 0) { - ls->method = NDLM_STDOUT; - ls->filename = NULL; - ls->fd = STDOUT_FILENO; - } - else { - ls->method = NDLM_FILE; - ls->filename = strdupz(output); - } - -#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE) - ls->min_priority = NDLP_DEBUG; -#endif - - if(source == NDLS_COLLECTORS) { - // set the method for the collector processes we will spawn - - ND_LOG_METHOD method; - ND_LOG_FORMAT format = ls->format; - ND_LOG_FIELD_PRIORITY priority = ls->min_priority; - - if(ls->method == NDLM_SYSLOG || ls->method == NDLM_JOURNAL) - method = ls->method; - else - method = NDLM_STDERR; - - setenv("NETDATA_LOG_METHOD", nd_log_id2method(method), 1); - setenv("NETDATA_LOG_FORMAT", nd_log_id2format(format), 1); - setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1); - } -} - -void nd_log_set_priority_level(const char *setting) { - if(!setting || !*setting) - setting = "info"; - - ND_LOG_FIELD_PRIORITY priority = nd_log_priority2id(setting); - -#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE) - priority = NDLP_DEBUG; -#endif - - for (size_t i = 0; i < _NDLS_MAX; i++) { - if (i != NDLS_DEBUG) - nd_log.sources[i].min_priority = priority; - } - - // the right one - setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1); -} - -void nd_log_set_facility(const char *facility) { - if(!facility || !*facility) - facility = "daemon"; - - nd_log.syslog.facility = nd_log_facility2id(facility); - setenv("NETDATA_SYSLOG_FACILITY", nd_log_id2facility(nd_log.syslog.facility), 1); -} - -void nd_log_set_flood_protection(size_t logs, time_t period) { - nd_log.sources[NDLS_DAEMON].limits.logs_per_period = - nd_log.sources[NDLS_DAEMON].limits.logs_per_period_backup; - nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period = - nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period_backup = logs; - - nd_log.sources[NDLS_DAEMON].limits.throttle_period = - nd_log.sources[NDLS_COLLECTORS].limits.throttle_period = period; - - char buf[100]; - snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )period); - setenv("NETDATA_ERRORS_THROTTLE_PERIOD", buf, 1); - snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )logs); - setenv("NETDATA_ERRORS_PER_PERIOD", buf, 1); -} - -static bool nd_log_journal_systemd_init(void) { -#ifdef HAVE_SYSTEMD - nd_log.journal.initialized = true; -#else - nd_log.journal.initialized = false; -#endif - - return nd_log.journal.initialized; -} - -static void nd_log_journal_direct_set_env(void) { - if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_JOURNAL) - setenv("NETDATA_SYSTEMD_JOURNAL_PATH", nd_log.journal_direct.filename, 1); -} - -static bool nd_log_journal_direct_init(const char *path) { - if(nd_log.journal_direct.initialized) { - nd_log_journal_direct_set_env(); - return true; - } - - int fd; - char filename[FILENAME_MAX + 1]; - if(!is_path_unix_socket(path)) { - - journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, "netdata"); - if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) { - - journal_construct_path(filename, sizeof(filename), netdata_configured_host_prefix, NULL); - if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) { - - journal_construct_path(filename, sizeof(filename), NULL, "netdata"); - if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) { - - journal_construct_path(filename, sizeof(filename), NULL, NULL); - if (!is_path_unix_socket(filename) || (fd = journal_direct_fd(filename)) == -1) - return false; - } - } - } - } - else { - snprintfz(filename, sizeof(filename), "%s", path); - fd = journal_direct_fd(filename); - } - - if(fd < 0) - return false; - - nd_log.journal_direct.fd = fd; - nd_log.journal_direct.initialized = true; - - strncpyz(nd_log.journal_direct.filename, filename, sizeof(nd_log.journal_direct.filename) - 1); - nd_log_journal_direct_set_env(); - - return true; -} - -static void nd_log_syslog_init() { - if(nd_log.syslog.initialized) - return; - - openlog(program_name, LOG_PID, nd_log.syslog.facility); - nd_log.syslog.initialized = true; -} - -void nd_log_initialize_for_external_plugins(const char *name) { - // if we don't run under Netdata, log to stderr, - // otherwise, use the logging method Netdata wants us to use. - setenv("NETDATA_LOG_METHOD", "stderr", 0); - setenv("NETDATA_LOG_FORMAT", "logfmt", 0); - - nd_log.overwrite_process_source = NDLS_COLLECTORS; - program_name = name; - - for(size_t i = 0; i < _NDLS_MAX ;i++) { - nd_log.sources[i].method = STDERR_FILENO; - nd_log.sources[i].fd = -1; - nd_log.sources[i].fp = NULL; - } - - nd_log_set_priority_level(getenv("NETDATA_LOG_LEVEL")); - nd_log_set_facility(getenv("NETDATA_SYSLOG_FACILITY")); - - time_t period = 1200; - size_t logs = 200; - const char *s = getenv("NETDATA_ERRORS_THROTTLE_PERIOD"); - if(s && *s >= '0' && *s <= '9') { - period = str2l(s); - if(period < 0) period = 0; - } - - s = getenv("NETDATA_ERRORS_PER_PERIOD"); - if(s && *s >= '0' && *s <= '9') - logs = str2u(s); - - nd_log_set_flood_protection(logs, period); - - if(!netdata_configured_host_prefix) { - s = getenv("NETDATA_HOST_PREFIX"); - if(s && *s) - netdata_configured_host_prefix = (char *)s; - } - - ND_LOG_METHOD method = nd_log_method2id(getenv("NETDATA_LOG_METHOD")); - ND_LOG_FORMAT format = nd_log_format2id(getenv("NETDATA_LOG_FORMAT")); - - if(!IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method)) { - if(is_stderr_connected_to_journal()) { - nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using journal."); - method = NDLM_JOURNAL; - } - else { - nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using stderr."); - method = NDLM_STDERR; - } - } - - switch(method) { - case NDLM_JOURNAL: - if(!nd_log_journal_direct_init(getenv("NETDATA_SYSTEMD_JOURNAL_PATH")) || - !nd_log_journal_direct_init(NULL) || !nd_log_journal_systemd_init()) { - nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize journal. Using stderr."); - method = NDLM_STDERR; - } - break; - - case NDLM_SYSLOG: - nd_log_syslog_init(); - break; - - default: - method = NDLM_STDERR; - break; - } - - for(size_t i = 0; i < _NDLS_MAX ;i++) { - nd_log.sources[i].method = method; - nd_log.sources[i].format = format; - nd_log.sources[i].fd = -1; - nd_log.sources[i].fp = NULL; - } - -// nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "FINAL_LOG_METHOD: %s", nd_log_id2method(method)); -} - -static bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd) { - if(new_fd == -1 || e->fd == -1 || - (e->fd == STDOUT_FILENO && nd_log.std_output.initialized) || - (e->fd == STDERR_FILENO && nd_log.std_error.initialized)) - return false; - - if(new_fd != e->fd) { - int t = dup2(new_fd, e->fd); - - bool ret = true; - if (t == -1) { - netdata_log_error("Cannot dup2() new fd %d to old fd %d for '%s'", new_fd, e->fd, e->filename); - ret = false; - } - else - close(new_fd); - - if(e->fd == STDOUT_FILENO) - nd_log.std_output.initialized = true; - else if(e->fd == STDERR_FILENO) - nd_log.std_error.initialized = true; - - return ret; - } - - return false; -} - -static void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source) { - if(e->method == NDLM_DEFAULT) - nd_log_set_user_settings(source, e->filename); - - if((e->method == NDLM_FILE && !e->filename) || - (e->method == NDLM_DEVNULL && e->fd == -1)) - e->method = NDLM_DISABLED; - - if(e->fp) - fflush(e->fp); - - switch(e->method) { - case NDLM_SYSLOG: - nd_log_syslog_init(); - break; - - case NDLM_JOURNAL: - nd_log_journal_direct_init(NULL); - nd_log_journal_systemd_init(); - break; - - case NDLM_STDOUT: - e->fp = stdout; - e->fd = STDOUT_FILENO; - break; - - case NDLM_DISABLED: - break; - - case NDLM_DEFAULT: - case NDLM_STDERR: - e->method = NDLM_STDERR; - e->fp = stderr; - e->fd = STDERR_FILENO; - break; - - case NDLM_DEVNULL: - case NDLM_FILE: { - int fd = open(e->filename, O_WRONLY | O_APPEND | O_CREAT, 0664); - if(fd == -1) { - if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO) { - e->fd = STDERR_FILENO; - e->method = NDLM_STDERR; - netdata_log_error("Cannot open log file '%s'. Falling back to stderr.", e->filename); - } - else - netdata_log_error("Cannot open log file '%s'. Leaving fd %d as-is.", e->filename, e->fd); - } - else { - if (!nd_log_replace_existing_fd(e, fd)) { - if(e->fd == STDOUT_FILENO || e->fd == STDERR_FILENO) { - if(e->fd == STDOUT_FILENO) - e->method = NDLM_STDOUT; - else if(e->fd == STDERR_FILENO) - e->method = NDLM_STDERR; - - // we have dup2() fd, so we can close the one we opened - if(fd != STDOUT_FILENO && fd != STDERR_FILENO) - close(fd); - } - else - e->fd = fd; - } - } - - // at this point we have e->fd set properly - - if(e->fd == STDOUT_FILENO) - e->fp = stdout; - else if(e->fd == STDERR_FILENO) - e->fp = stderr; - - if(!e->fp) { - e->fp = fdopen(e->fd, "a"); - if (!e->fp) { - netdata_log_error("Cannot fdopen() fd %d ('%s')", e->fd, e->filename); - - if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO) - close(e->fd); - - e->fp = stderr; - e->fd = STDERR_FILENO; - } - } - else { - if (setvbuf(e->fp, NULL, _IOLBF, 0) != 0) - netdata_log_error("Cannot set line buffering on fd %d ('%s')", e->fd, e->filename); - } - } - break; - } -} - -static void nd_log_stdin_init(int fd, const char *filename) { - int f = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0664); - if(f == -1) - return; - - if(f != fd) { - dup2(f, fd); - close(f); - } -} - -void nd_log_initialize(void) { - nd_log_stdin_init(STDIN_FILENO, "/dev/null"); - - for(size_t i = 0 ; i < _NDLS_MAX ; i++) - nd_log_open(&nd_log.sources[i], i); -} - -void nd_log_reopen_log_files(bool log) { - if(log) - netdata_log_info("Reopening all log files."); - - nd_log.std_output.initialized = false; - nd_log.std_error.initialized = false; - nd_log_initialize(); - - if(log) - netdata_log_info("Log files re-opened."); -} - -void nd_log_reopen_log_files_for_spawn_server(void) { - if(nd_log.syslog.initialized) { - closelog(); - nd_log.syslog.initialized = false; - nd_log_syslog_init(); - } - - if(nd_log.journal_direct.initialized) { - close(nd_log.journal_direct.fd); - nd_log.journal_direct.fd = -1; - nd_log.journal_direct.initialized = false; - nd_log_journal_direct_init(NULL); - } - - nd_log.sources[NDLS_UNSET].method = NDLM_DISABLED; - nd_log.sources[NDLS_ACCESS].method = NDLM_DISABLED; - nd_log.sources[NDLS_ACLK].method = NDLM_DISABLED; - nd_log.sources[NDLS_DEBUG].method = NDLM_DISABLED; - nd_log.sources[NDLS_HEALTH].method = NDLM_DISABLED; - nd_log_reopen_log_files(false); -} - -void chown_open_file(int fd, uid_t uid, gid_t gid) { - if(fd == -1) return; - - struct stat buf; - - if(fstat(fd, &buf) == -1) { - netdata_log_error("Cannot fstat() fd %d", fd); - return; - } - - if((buf.st_uid != uid || buf.st_gid != gid) && S_ISREG(buf.st_mode)) { - if(fchown(fd, uid, gid) == -1) - netdata_log_error("Cannot fchown() fd %d.", fd); - } -} - -void nd_log_chown_log_files(uid_t uid, gid_t gid) { - for(size_t i = 0 ; i < _NDLS_MAX ; i++) { - if(nd_log.sources[i].fd != -1 && nd_log.sources[i].fd != STDIN_FILENO) - chown_open_file(nd_log.sources[i].fd, uid, gid); - } -} - -// ---------------------------------------------------------------------------- -// annotators -struct log_field; -static void errno_annotator(BUFFER *wb, const char *key, struct log_field *lf); -static void priority_annotator(BUFFER *wb, const char *key, struct log_field *lf); -static void timestamp_usec_annotator(BUFFER *wb, const char *key, struct log_field *lf); - -#if defined(OS_WINDOWS) -static void winerror_annotator(BUFFER *wb, const char *key, struct log_field *lf); -#endif - -// ---------------------------------------------------------------------------- - -typedef void (*annotator_t)(BUFFER *wb, const char *key, struct log_field *lf); - -struct log_field { - const char *journal; - const char *logfmt; - annotator_t logfmt_annotator; - struct log_stack_entry entry; -}; - -#define THREAD_LOG_STACK_MAX 50 - -static __thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX]; -static __thread size_t thread_log_stack_next = 0; - -static __thread struct log_field thread_log_fields[_NDF_MAX] = { - // THE ORDER DEFINES THE ORDER FIELDS WILL APPEAR IN logfmt - - [NDF_STOP] = { // processing will not stop on this - so it is ok to be first - .journal = NULL, - .logfmt = NULL, - .logfmt_annotator = NULL, - }, - [NDF_TIMESTAMP_REALTIME_USEC] = { - .journal = NULL, - .logfmt = "time", - .logfmt_annotator = timestamp_usec_annotator, - }, - [NDF_SYSLOG_IDENTIFIER] = { - .journal = "SYSLOG_IDENTIFIER", // standard journald field - .logfmt = "comm", - }, - [NDF_LOG_SOURCE] = { - .journal = "ND_LOG_SOURCE", - .logfmt = "source", - }, - [NDF_PRIORITY] = { - .journal = "PRIORITY", // standard journald field - .logfmt = "level", - .logfmt_annotator = priority_annotator, - }, - [NDF_ERRNO] = { - .journal = "ERRNO", // standard journald field - .logfmt = "errno", - .logfmt_annotator = errno_annotator, - }, -#if defined(OS_WINDOWS) - [NDF_WINERROR] = { - .journal = "WINERROR", - .logfmt = "winerror", - .logfmt_annotator = winerror_annotator, - }, -#endif - [NDF_INVOCATION_ID] = { - .journal = "INVOCATION_ID", // standard journald field - .logfmt = NULL, - }, - [NDF_LINE] = { - .journal = "CODE_LINE", // standard journald field - .logfmt = NULL, - }, - [NDF_FILE] = { - .journal = "CODE_FILE", // standard journald field - .logfmt = NULL, - }, - [NDF_FUNC] = { - .journal = "CODE_FUNC", // standard journald field - .logfmt = NULL, - }, - [NDF_TID] = { - .journal = "TID", // standard journald field - .logfmt = "tid", - }, - [NDF_THREAD_TAG] = { - .journal = "THREAD_TAG", - .logfmt = "thread", - }, - [NDF_MESSAGE_ID] = { - .journal = "MESSAGE_ID", - .logfmt = "msg_id", - }, - [NDF_MODULE] = { - .journal = "ND_MODULE", - .logfmt = "module", - }, - [NDF_NIDL_NODE] = { - .journal = "ND_NIDL_NODE", - .logfmt = "node", - }, - [NDF_NIDL_INSTANCE] = { - .journal = "ND_NIDL_INSTANCE", - .logfmt = "instance", - }, - [NDF_NIDL_CONTEXT] = { - .journal = "ND_NIDL_CONTEXT", - .logfmt = "context", - }, - [NDF_NIDL_DIMENSION] = { - .journal = "ND_NIDL_DIMENSION", - .logfmt = "dimension", - }, - [NDF_SRC_TRANSPORT] = { - .journal = "ND_SRC_TRANSPORT", - .logfmt = "src_transport", - }, - [NDF_ACCOUNT_ID] = { - .journal = "ND_ACCOUNT_ID", - .logfmt = "account", - }, - [NDF_USER_NAME] = { - .journal = "ND_USER_NAME", - .logfmt = "user", - }, - [NDF_USER_ROLE] = { - .journal = "ND_USER_ROLE", - .logfmt = "role", - }, - [NDF_USER_ACCESS] = { - .journal = "ND_USER_PERMISSIONS", - .logfmt = "permissions", - }, - [NDF_SRC_IP] = { - .journal = "ND_SRC_IP", - .logfmt = "src_ip", - }, - [NDF_SRC_FORWARDED_HOST] = { - .journal = "ND_SRC_FORWARDED_HOST", - .logfmt = "src_forwarded_host", - }, - [NDF_SRC_FORWARDED_FOR] = { - .journal = "ND_SRC_FORWARDED_FOR", - .logfmt = "src_forwarded_for", - }, - [NDF_SRC_PORT] = { - .journal = "ND_SRC_PORT", - .logfmt = "src_port", - }, - [NDF_SRC_CAPABILITIES] = { - .journal = "ND_SRC_CAPABILITIES", - .logfmt = "src_capabilities", - }, - [NDF_DST_TRANSPORT] = { - .journal = "ND_DST_TRANSPORT", - .logfmt = "dst_transport", - }, - [NDF_DST_IP] = { - .journal = "ND_DST_IP", - .logfmt = "dst_ip", - }, - [NDF_DST_PORT] = { - .journal = "ND_DST_PORT", - .logfmt = "dst_port", - }, - [NDF_DST_CAPABILITIES] = { - .journal = "ND_DST_CAPABILITIES", - .logfmt = "dst_capabilities", - }, - [NDF_REQUEST_METHOD] = { - .journal = "ND_REQUEST_METHOD", - .logfmt = "req_method", - }, - [NDF_RESPONSE_CODE] = { - .journal = "ND_RESPONSE_CODE", - .logfmt = "code", - }, - [NDF_CONNECTION_ID] = { - .journal = "ND_CONNECTION_ID", - .logfmt = "conn", - }, - [NDF_TRANSACTION_ID] = { - .journal = "ND_TRANSACTION_ID", - .logfmt = "transaction", - }, - [NDF_RESPONSE_SENT_BYTES] = { - .journal = "ND_RESPONSE_SENT_BYTES", - .logfmt = "sent_bytes", - }, - [NDF_RESPONSE_SIZE_BYTES] = { - .journal = "ND_RESPONSE_SIZE_BYTES", - .logfmt = "size_bytes", - }, - [NDF_RESPONSE_PREPARATION_TIME_USEC] = { - .journal = "ND_RESPONSE_PREP_TIME_USEC", - .logfmt = "prep_ut", - }, - [NDF_RESPONSE_SENT_TIME_USEC] = { - .journal = "ND_RESPONSE_SENT_TIME_USEC", - .logfmt = "sent_ut", - }, - [NDF_RESPONSE_TOTAL_TIME_USEC] = { - .journal = "ND_RESPONSE_TOTAL_TIME_USEC", - .logfmt = "total_ut", - }, - [NDF_ALERT_ID] = { - .journal = "ND_ALERT_ID", - .logfmt = "alert_id", - }, - [NDF_ALERT_UNIQUE_ID] = { - .journal = "ND_ALERT_UNIQUE_ID", - .logfmt = "alert_unique_id", - }, - [NDF_ALERT_TRANSITION_ID] = { - .journal = "ND_ALERT_TRANSITION_ID", - .logfmt = "alert_transition_id", - }, - [NDF_ALERT_EVENT_ID] = { - .journal = "ND_ALERT_EVENT_ID", - .logfmt = "alert_event_id", - }, - [NDF_ALERT_CONFIG_HASH] = { - .journal = "ND_ALERT_CONFIG", - .logfmt = "alert_config", - }, - [NDF_ALERT_NAME] = { - .journal = "ND_ALERT_NAME", - .logfmt = "alert", - }, - [NDF_ALERT_CLASS] = { - .journal = "ND_ALERT_CLASS", - .logfmt = "alert_class", - }, - [NDF_ALERT_COMPONENT] = { - .journal = "ND_ALERT_COMPONENT", - .logfmt = "alert_component", - }, - [NDF_ALERT_TYPE] = { - .journal = "ND_ALERT_TYPE", - .logfmt = "alert_type", - }, - [NDF_ALERT_EXEC] = { - .journal = "ND_ALERT_EXEC", - .logfmt = "alert_exec", - }, - [NDF_ALERT_RECIPIENT] = { - .journal = "ND_ALERT_RECIPIENT", - .logfmt = "alert_recipient", - }, - [NDF_ALERT_VALUE] = { - .journal = "ND_ALERT_VALUE", - .logfmt = "alert_value", - }, - [NDF_ALERT_VALUE_OLD] = { - .journal = "ND_ALERT_VALUE_OLD", - .logfmt = "alert_value_old", - }, - [NDF_ALERT_STATUS] = { - .journal = "ND_ALERT_STATUS", - .logfmt = "alert_status", - }, - [NDF_ALERT_STATUS_OLD] = { - .journal = "ND_ALERT_STATUS_OLD", - .logfmt = "alert_value_old", - }, - [NDF_ALERT_UNITS] = { - .journal = "ND_ALERT_UNITS", - .logfmt = "alert_units", - }, - [NDF_ALERT_SUMMARY] = { - .journal = "ND_ALERT_SUMMARY", - .logfmt = "alert_summary", - }, - [NDF_ALERT_INFO] = { - .journal = "ND_ALERT_INFO", - .logfmt = "alert_info", - }, - [NDF_ALERT_DURATION] = { - .journal = "ND_ALERT_DURATION", - .logfmt = "alert_duration", - }, - [NDF_ALERT_NOTIFICATION_REALTIME_USEC] = { - .journal = "ND_ALERT_NOTIFICATION_TIMESTAMP_USEC", - .logfmt = "alert_notification_timestamp", - .logfmt_annotator = timestamp_usec_annotator, - }, - - // put new items here - // leave the request URL and the message last - - [NDF_REQUEST] = { - .journal = "ND_REQUEST", - .logfmt = "request", - }, - [NDF_MESSAGE] = { - .journal = "MESSAGE", - .logfmt = "msg", - }, -}; - -#define THREAD_FIELDS_MAX (sizeof(thread_log_fields) / sizeof(thread_log_fields[0])) - -ND_LOG_FIELD_ID nd_log_field_id_by_name(const char *field, size_t len) { - for(size_t i = 0; i < THREAD_FIELDS_MAX ;i++) { - if(thread_log_fields[i].journal && strlen(thread_log_fields[i].journal) == len && strncmp(field, thread_log_fields[i].journal, len) == 0) - return i; - } - - return NDF_STOP; -} - -void log_stack_pop(void *ptr) { - if(!ptr) return; - - struct log_stack_entry *lgs = *(struct log_stack_entry (*)[])ptr; - - if(unlikely(!thread_log_stack_next || lgs != thread_log_stack_base[thread_log_stack_next - 1])) { - fatal("You cannot pop in the middle of the stack, or an item not in the stack"); - return; - } - - thread_log_stack_next--; -} - -void log_stack_push(struct log_stack_entry *lgs) { - if(!lgs || thread_log_stack_next >= THREAD_LOG_STACK_MAX) return; - thread_log_stack_base[thread_log_stack_next++] = lgs; -} - -// ---------------------------------------------------------------------------- -// json formatter - -static void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max) { - - // --- FIELD_PARSER_VERSIONS --- - // - // IMPORTANT: - // THERE ARE 6 VERSIONS OF THIS CODE - // - // 1. journal (direct socket API), - // 2. journal (libsystemd API), - // 3. logfmt, - // 4. json, - // 5. convert to uint64 - // 6. convert to int64 - // - // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES - - buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); - CLEAN_BUFFER *tmp = NULL; - - for (size_t i = 0; i < fields_max; i++) { - if (!fields[i].entry.set || !fields[i].logfmt) - continue; - - const char *key = fields[i].logfmt; - - const char *s = NULL; - switch(fields[i].entry.type) { - case NDFT_TXT: - s = fields[i].entry.txt; - break; - case NDFT_STR: - s = string2str(fields[i].entry.str); - break; - case NDFT_BFR: - s = buffer_tostring(fields[i].entry.bfr); - break; - case NDFT_U64: - buffer_json_member_add_uint64(wb, key, fields[i].entry.u64); - break; - case NDFT_I64: - buffer_json_member_add_int64(wb, key, fields[i].entry.i64); - break; - case NDFT_DBL: - buffer_json_member_add_double(wb, key, fields[i].entry.dbl); - break; - case NDFT_UUID: - if(!uuid_is_null(*fields[i].entry.uuid)) { - char u[UUID_COMPACT_STR_LEN]; - uuid_unparse_lower_compact(*fields[i].entry.uuid, u); - buffer_json_member_add_string(wb, key, u); - } - break; - case NDFT_CALLBACK: { - if(!tmp) - tmp = buffer_create(1024, NULL); - else - buffer_flush(tmp); - if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) - s = buffer_tostring(tmp); - else - s = NULL; - } - break; - default: - s = "UNHANDLED"; - break; - } - - if(s && *s) - buffer_json_member_add_string(wb, key, s); - } - - buffer_json_finalize(wb); -} - -// ---------------------------------------------------------------------------- -// logfmt formatter - - -static int64_t log_field_to_int64(struct log_field *lf) { - - // --- FIELD_PARSER_VERSIONS --- - // - // IMPORTANT: - // THERE ARE 6 VERSIONS OF THIS CODE - // - // 1. journal (direct socket API), - // 2. journal (libsystemd API), - // 3. logfmt, - // 4. json, - // 5. convert to uint64 - // 6. convert to int64 - // - // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES - - CLEAN_BUFFER *tmp = NULL; - const char *s = NULL; - - switch(lf->entry.type) { - case NDFT_UUID: - case NDFT_UNSET: - return 0; - - case NDFT_TXT: - s = lf->entry.txt; - break; - - case NDFT_STR: - s = string2str(lf->entry.str); - break; - - case NDFT_BFR: - s = buffer_tostring(lf->entry.bfr); - break; - - case NDFT_CALLBACK: - tmp = buffer_create(0, NULL); - - if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data)) - s = buffer_tostring(tmp); - else - s = NULL; - break; - - case NDFT_U64: - return (int64_t)lf->entry.u64; - - case NDFT_I64: - return (int64_t)lf->entry.i64; - - case NDFT_DBL: - return (int64_t)lf->entry.dbl; - } - - if(s && *s) - return str2ll(s, NULL); - - return 0; -} - -static uint64_t log_field_to_uint64(struct log_field *lf) { - - // --- FIELD_PARSER_VERSIONS --- - // - // IMPORTANT: - // THERE ARE 6 VERSIONS OF THIS CODE - // - // 1. journal (direct socket API), - // 2. journal (libsystemd API), - // 3. logfmt, - // 4. json, - // 5. convert to uint64 - // 6. convert to int64 - // - // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES - - CLEAN_BUFFER *tmp = NULL; - const char *s = NULL; - - switch(lf->entry.type) { - case NDFT_UUID: - case NDFT_UNSET: - return 0; - - case NDFT_TXT: - s = lf->entry.txt; - break; - - case NDFT_STR: - s = string2str(lf->entry.str); - break; - - case NDFT_BFR: - s = buffer_tostring(lf->entry.bfr); - break; - - case NDFT_CALLBACK: - tmp = buffer_create(0, NULL); - - if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data)) - s = buffer_tostring(tmp); - else - s = NULL; - break; - - case NDFT_U64: - return lf->entry.u64; - - case NDFT_I64: - return lf->entry.i64; - - case NDFT_DBL: - return (uint64_t) lf->entry.dbl; - } - - if(s && *s) - return str2uint64_t(s, NULL); - - return 0; -} - -static void timestamp_usec_annotator(BUFFER *wb, const char *key, struct log_field *lf) { - usec_t ut = log_field_to_uint64(lf); - - if(!ut) - return; - - char datetime[RFC3339_MAX_LENGTH]; - rfc3339_datetime_ut(datetime, sizeof(datetime), ut, 3, false); - - if(buffer_strlen(wb)) - buffer_fast_strcat(wb, " ", 1); - - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=", 1); - buffer_json_strcat(wb, datetime); -} - -static void errno_annotator(BUFFER *wb, const char *key, struct log_field *lf) { - int64_t errnum = log_field_to_int64(lf); - - if(errnum == 0) - return; - - char buf[1024]; - const char *s = errno2str((int)errnum, buf, sizeof(buf)); - - if(buffer_strlen(wb)) - buffer_fast_strcat(wb, " ", 1); - - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=\"", 2); - buffer_print_int64(wb, errnum); - buffer_fast_strcat(wb, ", ", 2); - buffer_json_strcat(wb, s); - buffer_fast_strcat(wb, "\"", 1); -} - -#if defined(OS_WINDOWS) -static void winerror_annotator(BUFFER *wb, const char *key, struct log_field *lf) { - DWORD errnum = log_field_to_uint64(lf); - - if(errnum == 0) - return; - - char buf[1024]; - DWORD size = FormatMessageA( - FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - errnum, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - buf, - (DWORD)(sizeof(buf) - 1), - NULL - ); - if(size > 0) { - // remove \r\n at the end - while(size > 0 && (buf[size - 1] == '\r' || buf[size - 1] == '\n')) - buf[--size] = '\0'; - } - else - size = snprintf(buf, sizeof(buf) - 1, "unknown error code"); - - buf[size] = '\0'; - - if(buffer_strlen(wb)) - buffer_fast_strcat(wb, " ", 1); - - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=\"", 2); - buffer_print_int64(wb, errnum); - buffer_fast_strcat(wb, ", ", 2); - buffer_json_strcat(wb, buf); - buffer_fast_strcat(wb, "\"", 1); -} -#endif - -static void priority_annotator(BUFFER *wb, const char *key, struct log_field *lf) { - uint64_t pri = log_field_to_uint64(lf); - - if(buffer_strlen(wb)) - buffer_fast_strcat(wb, " ", 1); - - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=", 1); - buffer_strcat(wb, nd_log_id2priority(pri)); -} - -static bool needs_quotes_for_logfmt(const char *s) -{ - static bool safe_for_logfmt[256] = { - [' '] = true, ['!'] = true, ['"'] = false, ['#'] = true, ['$'] = true, ['%'] = true, ['&'] = true, - ['\''] = true, ['('] = true, [')'] = true, ['*'] = true, ['+'] = true, [','] = true, ['-'] = true, - ['.'] = true, ['/'] = true, ['0'] = true, ['1'] = true, ['2'] = true, ['3'] = true, ['4'] = true, - ['5'] = true, ['6'] = true, ['7'] = true, ['8'] = true, ['9'] = true, [':'] = true, [';'] = true, - ['<'] = true, ['='] = true, ['>'] = true, ['?'] = true, ['@'] = true, ['A'] = true, ['B'] = true, - ['C'] = true, ['D'] = true, ['E'] = true, ['F'] = true, ['G'] = true, ['H'] = true, ['I'] = true, - ['J'] = true, ['K'] = true, ['L'] = true, ['M'] = true, ['N'] = true, ['O'] = true, ['P'] = true, - ['Q'] = true, ['R'] = true, ['S'] = true, ['T'] = true, ['U'] = true, ['V'] = true, ['W'] = true, - ['X'] = true, ['Y'] = true, ['Z'] = true, ['['] = true, ['\\'] = false, [']'] = true, ['^'] = true, - ['_'] = true, ['`'] = true, ['a'] = true, ['b'] = true, ['c'] = true, ['d'] = true, ['e'] = true, - ['f'] = true, ['g'] = true, ['h'] = true, ['i'] = true, ['j'] = true, ['k'] = true, ['l'] = true, - ['m'] = true, ['n'] = true, ['o'] = true, ['p'] = true, ['q'] = true, ['r'] = true, ['s'] = true, - ['t'] = true, ['u'] = true, ['v'] = true, ['w'] = true, ['x'] = true, ['y'] = true, ['z'] = true, - ['{'] = true, ['|'] = true, ['}'] = true, ['~'] = true, [0x7f] = true, - }; - - if(!*s) - return true; - - while(*s) { - if(*s == '=' || isspace((uint8_t)*s) || !safe_for_logfmt[(uint8_t)*s]) - return true; - - s++; - } - - return false; -} - -static void string_to_logfmt(BUFFER *wb, const char *s) -{ - bool spaces = needs_quotes_for_logfmt(s); - - if(spaces) - buffer_fast_strcat(wb, "\"", 1); - - buffer_json_strcat(wb, s); - - if(spaces) - buffer_fast_strcat(wb, "\"", 1); -} - -static void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max) -{ - - // --- FIELD_PARSER_VERSIONS --- - // - // IMPORTANT: - // THERE ARE 6 VERSIONS OF THIS CODE - // - // 1. journal (direct socket API), - // 2. journal (libsystemd API), - // 3. logfmt, - // 4. json, - // 5. convert to uint64 - // 6. convert to int64 - // - // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES - - CLEAN_BUFFER *tmp = NULL; - - for (size_t i = 0; i < fields_max; i++) { - if (!fields[i].entry.set || !fields[i].logfmt) - continue; - - const char *key = fields[i].logfmt; - - if(fields[i].logfmt_annotator) - fields[i].logfmt_annotator(wb, key, &fields[i]); - else { - if(buffer_strlen(wb)) - buffer_fast_strcat(wb, " ", 1); - - switch(fields[i].entry.type) { - case NDFT_TXT: - if(*fields[i].entry.txt) { - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=", 1); - string_to_logfmt(wb, fields[i].entry.txt); - } - break; - case NDFT_STR: - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=", 1); - string_to_logfmt(wb, string2str(fields[i].entry.str)); - break; - case NDFT_BFR: - if(buffer_strlen(fields[i].entry.bfr)) { - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=", 1); - string_to_logfmt(wb, buffer_tostring(fields[i].entry.bfr)); - } - break; - case NDFT_U64: - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=", 1); - buffer_print_uint64(wb, fields[i].entry.u64); - break; - case NDFT_I64: - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=", 1); - buffer_print_int64(wb, fields[i].entry.i64); - break; - case NDFT_DBL: - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=", 1); - buffer_print_netdata_double(wb, fields[i].entry.dbl); - break; - case NDFT_UUID: - if(!uuid_is_null(*fields[i].entry.uuid)) { - char u[UUID_COMPACT_STR_LEN]; - uuid_unparse_lower_compact(*fields[i].entry.uuid, u); - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=", 1); - buffer_fast_strcat(wb, u, sizeof(u) - 1); - } - break; - case NDFT_CALLBACK: { - if(!tmp) - tmp = buffer_create(1024, NULL); - else - buffer_flush(tmp); - if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) { - buffer_strcat(wb, key); - buffer_fast_strcat(wb, "=", 1); - string_to_logfmt(wb, buffer_tostring(tmp)); - } - } - break; - default: - buffer_strcat(wb, "UNHANDLED"); - break; - } - } - } -} - -// ---------------------------------------------------------------------------- -// journal logger - -bool nd_log_journal_socket_available(void) { - if(netdata_configured_host_prefix && *netdata_configured_host_prefix) { - char filename[FILENAME_MAX + 1]; - - snprintfz(filename, sizeof(filename), "%s%s", - netdata_configured_host_prefix, "/run/systemd/journal/socket"); - - if(is_path_unix_socket(filename)) - return true; - } - - return is_path_unix_socket("/run/systemd/journal/socket"); -} - -static bool nd_logger_journal_libsystemd(struct log_field *fields __maybe_unused, size_t fields_max __maybe_unused) { -#ifdef HAVE_SYSTEMD - - // --- FIELD_PARSER_VERSIONS --- - // - // IMPORTANT: - // THERE ARE 6 VERSIONS OF THIS CODE - // - // 1. journal (direct socket API), - // 2. journal (libsystemd API), - // 3. logfmt, - // 4. json, - // 5. convert to uint64 - // 6. convert to int64 - // - // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES - - struct iovec iov[fields_max]; - int iov_count = 0; - - memset(iov, 0, sizeof(iov)); - - CLEAN_BUFFER *tmp = NULL; - - for (size_t i = 0; i < fields_max; i++) { - if (!fields[i].entry.set || !fields[i].journal) - continue; - - const char *key = fields[i].journal; - char *value = NULL; - int rc = 0; - switch (fields[i].entry.type) { - case NDFT_TXT: - if(*fields[i].entry.txt) - rc = asprintf(&value, "%s=%s", key, fields[i].entry.txt); - break; - case NDFT_STR: - rc = asprintf(&value, "%s=%s", key, string2str(fields[i].entry.str)); - break; - case NDFT_BFR: - if(buffer_strlen(fields[i].entry.bfr)) - rc = asprintf(&value, "%s=%s", key, buffer_tostring(fields[i].entry.bfr)); - break; - case NDFT_U64: - rc = asprintf(&value, "%s=%" PRIu64, key, fields[i].entry.u64); - break; - case NDFT_I64: - rc = asprintf(&value, "%s=%" PRId64, key, fields[i].entry.i64); - break; - case NDFT_DBL: - rc = asprintf(&value, "%s=%f", key, fields[i].entry.dbl); - break; - case NDFT_UUID: - if(!uuid_is_null(*fields[i].entry.uuid)) { - char u[UUID_COMPACT_STR_LEN]; - uuid_unparse_lower_compact(*fields[i].entry.uuid, u); - rc = asprintf(&value, "%s=%s", key, u); - } - break; - case NDFT_CALLBACK: { - if(!tmp) - tmp = buffer_create(1024, NULL); - else - buffer_flush(tmp); - if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) - rc = asprintf(&value, "%s=%s", key, buffer_tostring(tmp)); - } - break; - default: - rc = asprintf(&value, "%s=%s", key, "UNHANDLED"); - break; - } - - if (rc != -1 && value) { - iov[iov_count].iov_base = value; - iov[iov_count].iov_len = strlen(value); - iov_count++; - } - } - - int r = sd_journal_sendv(iov, iov_count); - - // Clean up allocated memory - for (int i = 0; i < iov_count; i++) { - if (iov[i].iov_base != NULL) { - free(iov[i].iov_base); - } - } - - return r == 0; -#else - return false; -#endif -} - -static bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max) { - if(!nd_log.journal_direct.initialized) - return false; - - // --- FIELD_PARSER_VERSIONS --- - // - // IMPORTANT: - // THERE ARE 6 VERSIONS OF THIS CODE - // - // 1. journal (direct socket API), - // 2. journal (libsystemd API), - // 3. logfmt, - // 4. json, - // 5. convert to uint64 - // 6. convert to int64 - // - // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES - - CLEAN_BUFFER *wb = buffer_create(4096, NULL); - CLEAN_BUFFER *tmp = NULL; - - for (size_t i = 0; i < fields_max; i++) { - if (!fields[i].entry.set || !fields[i].journal) - continue; - - const char *key = fields[i].journal; - - const char *s = NULL; - switch(fields[i].entry.type) { - case NDFT_TXT: - s = fields[i].entry.txt; - break; - case NDFT_STR: - s = string2str(fields[i].entry.str); - break; - case NDFT_BFR: - s = buffer_tostring(fields[i].entry.bfr); - break; - case NDFT_U64: - buffer_strcat(wb, key); - buffer_putc(wb, '='); - buffer_print_uint64(wb, fields[i].entry.u64); - buffer_putc(wb, '\n'); - break; - case NDFT_I64: - buffer_strcat(wb, key); - buffer_putc(wb, '='); - buffer_print_int64(wb, fields[i].entry.i64); - buffer_putc(wb, '\n'); - break; - case NDFT_DBL: - buffer_strcat(wb, key); - buffer_putc(wb, '='); - buffer_print_netdata_double(wb, fields[i].entry.dbl); - buffer_putc(wb, '\n'); - break; - case NDFT_UUID: - if(!uuid_is_null(*fields[i].entry.uuid)) { - char u[UUID_COMPACT_STR_LEN]; - uuid_unparse_lower_compact(*fields[i].entry.uuid, u); - buffer_strcat(wb, key); - buffer_putc(wb, '='); - buffer_fast_strcat(wb, u, sizeof(u) - 1); - buffer_putc(wb, '\n'); - } - break; - case NDFT_CALLBACK: { - if(!tmp) - tmp = buffer_create(1024, NULL); - else - buffer_flush(tmp); - if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) - s = buffer_tostring(tmp); - else - s = NULL; - } - break; - default: - s = "UNHANDLED"; - break; - } - - if(s && *s) { - buffer_strcat(wb, key); - if(!strchr(s, '\n')) { - buffer_putc(wb, '='); - buffer_strcat(wb, s); - buffer_putc(wb, '\n'); - } - else { - buffer_putc(wb, '\n'); - size_t size = strlen(s); - uint64_t le_size = htole64(size); - buffer_memcat(wb, &le_size, sizeof(le_size)); - buffer_memcat(wb, s, size); - buffer_putc(wb, '\n'); - } - } - } - - return journal_direct_send(nd_log.journal_direct.fd, buffer_tostring(wb), buffer_strlen(wb)); -} - -// ---------------------------------------------------------------------------- -// syslog logger - uses logfmt - -static bool nd_logger_syslog(int priority, ND_LOG_FORMAT format __maybe_unused, struct log_field *fields, size_t fields_max) { - CLEAN_BUFFER *wb = buffer_create(1024, NULL); - - nd_logger_logfmt(wb, fields, fields_max); - syslog(priority, "%s", buffer_tostring(wb)); - - return true; -} - -// ---------------------------------------------------------------------------- -// file logger - uses logfmt - -static bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max) { - BUFFER *wb = buffer_create(1024, NULL); - - if(format == NDLF_JSON) - nd_logger_json(wb, fields, fields_max); - else - nd_logger_logfmt(wb, fields, fields_max); - - int r = fprintf(fp, "%s\n", buffer_tostring(wb)); - fflush(fp); - - buffer_free(wb); - return r > 0; -} - -// ---------------------------------------------------------------------------- -// logger router - -static ND_LOG_METHOD nd_logger_select_output(ND_LOG_SOURCES source, FILE **fpp, SPINLOCK **spinlock) { - *spinlock = NULL; - ND_LOG_METHOD output = nd_log.sources[source].method; - - switch(output) { - case NDLM_JOURNAL: - if(unlikely(!nd_log.journal_direct.initialized && !nd_log.journal.initialized)) { - output = NDLM_FILE; - *fpp = stderr; - *spinlock = &nd_log.std_error.spinlock; - } - else { - *fpp = NULL; - *spinlock = NULL; - } - break; - - case NDLM_SYSLOG: - if(unlikely(!nd_log.syslog.initialized)) { - output = NDLM_FILE; - *spinlock = &nd_log.std_error.spinlock; - *fpp = stderr; - } - else { - *spinlock = NULL; - *fpp = NULL; - } - break; - - case NDLM_FILE: - if(!nd_log.sources[source].fp) { - *fpp = stderr; - *spinlock = &nd_log.std_error.spinlock; - } - else { - *fpp = nd_log.sources[source].fp; - *spinlock = &nd_log.sources[source].spinlock; - } - break; - - case NDLM_STDOUT: - output = NDLM_FILE; - *fpp = stdout; - *spinlock = &nd_log.std_output.spinlock; - break; - - default: - case NDLM_DEFAULT: - case NDLM_STDERR: - output = NDLM_FILE; - *fpp = stderr; - *spinlock = &nd_log.std_error.spinlock; - break; - - case NDLM_DISABLED: - case NDLM_DEVNULL: - output = NDLM_DISABLED; - *fpp = NULL; - *spinlock = NULL; - break; - } - - return output; -} - -// ---------------------------------------------------------------------------- -// high level logger - -static void nd_logger_log_fields(SPINLOCK *spinlock, FILE *fp, bool limit, ND_LOG_FIELD_PRIORITY priority, - ND_LOG_METHOD output, struct nd_log_source *source, - struct log_field *fields, size_t fields_max) { - if(spinlock) - spinlock_lock(spinlock); - - // check the limits - if(limit && nd_log_limit_reached(source)) - goto cleanup; - - if(output == NDLM_JOURNAL) { - if(!nd_logger_journal_direct(fields, fields_max) && !nd_logger_journal_libsystemd(fields, fields_max)) { - // we can't log to journal, let's log to stderr - if(spinlock) - spinlock_unlock(spinlock); - - output = NDLM_FILE; - spinlock = &nd_log.std_error.spinlock; - fp = stderr; - - if(spinlock) - spinlock_lock(spinlock); - } - } - - if(output == NDLM_SYSLOG) - nd_logger_syslog(priority, source->format, fields, fields_max); - - if(output == NDLM_FILE) - nd_logger_file(fp, source->format, fields, fields_max); - - -cleanup: - if(spinlock) - spinlock_unlock(spinlock); -} - -static void nd_logger_unset_all_thread_fields(void) { - size_t fields_max = THREAD_FIELDS_MAX; - for(size_t i = 0; i < fields_max ; i++) - thread_log_fields[i].entry.set = false; -} - -static void nd_logger_merge_log_stack_to_thread_fields(void) { - for(size_t c = 0; c < thread_log_stack_next ;c++) { - struct log_stack_entry *lgs = thread_log_stack_base[c]; - - for(size_t i = 0; lgs[i].id != NDF_STOP ; i++) { - if(lgs[i].id >= _NDF_MAX || !lgs[i].set) - continue; - - struct log_stack_entry *e = &lgs[i]; - ND_LOG_STACK_FIELD_TYPE type = lgs[i].type; - - // do not add empty / unset fields - if((type == NDFT_TXT && (!e->txt || !*e->txt)) || - (type == NDFT_BFR && (!e->bfr || !buffer_strlen(e->bfr))) || - (type == NDFT_STR && !e->str) || - (type == NDFT_UUID && (!e->uuid || uuid_is_null(*e->uuid))) || - (type == NDFT_CALLBACK && !e->cb.formatter) || - type == NDFT_UNSET) - continue; - - thread_log_fields[lgs[i].id].entry = *e; - } - } -} - -static void nd_logger(const char *file, const char *function, const unsigned long line, - ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, bool limit, - int saved_errno, size_t saved_winerror __maybe_unused, const char *fmt, va_list ap) { - - SPINLOCK *spinlock; - FILE *fp; - ND_LOG_METHOD output = nd_logger_select_output(source, &fp, &spinlock); - if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG) - return; - - // mark all fields as unset - nd_logger_unset_all_thread_fields(); - - // flatten the log stack into the fields - nd_logger_merge_log_stack_to_thread_fields(); - - // set the common fields that are automatically set by the logging subsystem - - if(likely(!thread_log_fields[NDF_INVOCATION_ID].entry.set)) - thread_log_fields[NDF_INVOCATION_ID].entry = ND_LOG_FIELD_UUID(NDF_INVOCATION_ID, &nd_log.invocation_id); - - if(likely(!thread_log_fields[NDF_LOG_SOURCE].entry.set)) - thread_log_fields[NDF_LOG_SOURCE].entry = ND_LOG_FIELD_TXT(NDF_LOG_SOURCE, nd_log_id2source(source)); - else { - ND_LOG_SOURCES src = source; - - if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_TXT) - src = nd_log_source2id(thread_log_fields[NDF_LOG_SOURCE].entry.txt, source); - else if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_U64) - src = thread_log_fields[NDF_LOG_SOURCE].entry.u64; - - if(src != source && src < _NDLS_MAX) { - source = src; - output = nd_logger_select_output(source, &fp, &spinlock); - if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG) - return; - } - } - - if(likely(!thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry.set)) - thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, program_name); - - if(likely(!thread_log_fields[NDF_LINE].entry.set)) { - thread_log_fields[NDF_LINE].entry = ND_LOG_FIELD_U64(NDF_LINE, line); - thread_log_fields[NDF_FILE].entry = ND_LOG_FIELD_TXT(NDF_FILE, file); - thread_log_fields[NDF_FUNC].entry = ND_LOG_FIELD_TXT(NDF_FUNC, function); - } - - if(likely(!thread_log_fields[NDF_PRIORITY].entry.set)) { - thread_log_fields[NDF_PRIORITY].entry = ND_LOG_FIELD_U64(NDF_PRIORITY, priority); - } - - if(likely(!thread_log_fields[NDF_TID].entry.set)) - thread_log_fields[NDF_TID].entry = ND_LOG_FIELD_U64(NDF_TID, gettid_cached()); - - if(likely(!thread_log_fields[NDF_THREAD_TAG].entry.set)) { - const char *thread_tag = nd_thread_tag(); - thread_log_fields[NDF_THREAD_TAG].entry = ND_LOG_FIELD_TXT(NDF_THREAD_TAG, thread_tag); - - // TODO: fix the ND_MODULE in logging by setting proper module name in threads -// if(!thread_log_fields[NDF_MODULE].entry.set) -// thread_log_fields[NDF_MODULE].entry = ND_LOG_FIELD_CB(NDF_MODULE, thread_tag_to_module, (void *)thread_tag); - } - - if(likely(!thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry.set)) - thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = ND_LOG_FIELD_U64(NDF_TIMESTAMP_REALTIME_USEC, now_realtime_usec()); - - if(saved_errno != 0 && !thread_log_fields[NDF_ERRNO].entry.set) - thread_log_fields[NDF_ERRNO].entry = ND_LOG_FIELD_I64(NDF_ERRNO, saved_errno); - -#if defined(OS_WINDOWS) - if(saved_winerror != 0 && !thread_log_fields[NDF_WINERROR].entry.set) - thread_log_fields[NDF_WINERROR].entry = ND_LOG_FIELD_U64(NDF_WINERROR, saved_winerror); -#endif - - CLEAN_BUFFER *wb = NULL; - if(fmt && !thread_log_fields[NDF_MESSAGE].entry.set) { - wb = buffer_create(1024, NULL); - buffer_vsprintf(wb, fmt, ap); - thread_log_fields[NDF_MESSAGE].entry = ND_LOG_FIELD_TXT(NDF_MESSAGE, buffer_tostring(wb)); - } - - nd_logger_log_fields(spinlock, fp, limit, priority, output, &nd_log.sources[source], - thread_log_fields, THREAD_FIELDS_MAX); - - if(nd_log.sources[source].pending_msg) { - // log a pending message - - nd_logger_unset_all_thread_fields(); - - thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = (struct log_stack_entry){ - .set = true, - .type = NDFT_U64, - .u64 = now_realtime_usec(), - }; - - thread_log_fields[NDF_LOG_SOURCE].entry = (struct log_stack_entry){ - .set = true, - .type = NDFT_TXT, - .txt = nd_log_id2source(source), - }; - - thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = (struct log_stack_entry){ - .set = true, - .type = NDFT_TXT, - .txt = program_name, - }; - - thread_log_fields[NDF_MESSAGE].entry = (struct log_stack_entry){ - .set = true, - .type = NDFT_TXT, - .txt = nd_log.sources[source].pending_msg, - }; - - nd_logger_log_fields(spinlock, fp, false, priority, output, - &nd_log.sources[source], - thread_log_fields, THREAD_FIELDS_MAX); - - freez((void *)nd_log.sources[source].pending_msg); - nd_log.sources[source].pending_msg = NULL; - } - - errno_clear(); -} - -static ND_LOG_SOURCES nd_log_validate_source(ND_LOG_SOURCES source) { - if(source >= _NDLS_MAX) - source = NDLS_DAEMON; - - if(nd_log.overwrite_process_source) - source = nd_log.overwrite_process_source; - - return source; -} - -// ---------------------------------------------------------------------------- -// public API for loggers - -void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... ) -{ - int saved_errno = errno; - - size_t saved_winerror = 0; -#if defined(OS_WINDOWS) - saved_winerror = GetLastError(); -#endif - - source = nd_log_validate_source(source); - - if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority) - return; - - va_list args; - va_start(args, fmt); - nd_logger(file, function, line, source, priority, - source == NDLS_DAEMON || source == NDLS_COLLECTORS, - saved_errno, saved_winerror, fmt, args); - va_end(args); -} - -void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) { - int saved_errno = errno; - - size_t saved_winerror = 0; -#if defined(OS_WINDOWS) - saved_winerror = GetLastError(); -#endif - - source = nd_log_validate_source(source); - - if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority) - return; - - if(erl->sleep_ut) - sleep_usec(erl->sleep_ut); - - spinlock_lock(&erl->spinlock); - - erl->count++; - time_t now = now_boottime_sec(); - if(now - erl->last_logged < erl->log_every) { - spinlock_unlock(&erl->spinlock); - return; - } - - spinlock_unlock(&erl->spinlock); - - va_list args; - va_start(args, fmt); - nd_logger(file, function, line, source, priority, - source == NDLS_DAEMON || source == NDLS_COLLECTORS, - saved_errno, saved_winerror, fmt, args); - va_end(args); - erl->last_logged = now; - erl->count = 0; -} - -void netdata_logger_fatal( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) { - int saved_errno = errno; - - size_t saved_winerror = 0; -#if defined(OS_WINDOWS) - saved_winerror = GetLastError(); -#endif - - ND_LOG_SOURCES source = NDLS_DAEMON; - source = nd_log_validate_source(source); - - va_list args; - va_start(args, fmt); - nd_logger(file, function, line, source, NDLP_ALERT, true, saved_errno, saved_winerror, fmt, args); - va_end(args); - - char date[LOG_DATE_LENGTH]; - log_date(date, LOG_DATE_LENGTH, now_realtime_sec()); - - char action_data[70+1]; - snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, saved_errno); - - const char *thread_tag = nd_thread_tag(); - const char *tag_to_send = thread_tag; - - // anonymize thread names - if(strncmp(thread_tag, THREAD_TAG_STREAM_RECEIVER, strlen(THREAD_TAG_STREAM_RECEIVER)) == 0) - tag_to_send = THREAD_TAG_STREAM_RECEIVER; - if(strncmp(thread_tag, THREAD_TAG_STREAM_SENDER, strlen(THREAD_TAG_STREAM_SENDER)) == 0) - tag_to_send = THREAD_TAG_STREAM_SENDER; - - char action_result[60+1]; - snprintfz(action_result, 60, "%s:%s", program_name, tag_to_send); - -#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE) - int fd = nd_log.sources[NDLS_DAEMON].fd; - if(fd == -1) - fd = STDERR_FILENO; - - int nptrs; - void *buffer[10000]; - - nptrs = backtrace(buffer, sizeof(buffer)); - if(nptrs) - backtrace_symbols_fd(buffer, nptrs, fd); -#endif - -#ifdef NETDATA_INTERNAL_CHECKS - abort(); -#endif - - netdata_cleanup_and_exit(1, "FATAL", action_result, action_data); -} - -// ---------------------------------------------------------------------------- -// log limits - -void nd_log_limits_reset(void) { - usec_t now_ut = now_monotonic_usec(); - - spinlock_lock(&nd_log.std_output.spinlock); - spinlock_lock(&nd_log.std_error.spinlock); - - for(size_t i = 0; i < _NDLS_MAX ;i++) { - spinlock_lock(&nd_log.sources[i].spinlock); - nd_log.sources[i].limits.prevented = 0; - nd_log.sources[i].limits.counter = 0; - nd_log.sources[i].limits.started_monotonic_ut = now_ut; - nd_log.sources[i].limits.logs_per_period = nd_log.sources[i].limits.logs_per_period_backup; - spinlock_unlock(&nd_log.sources[i].spinlock); - } - - spinlock_unlock(&nd_log.std_output.spinlock); - spinlock_unlock(&nd_log.std_error.spinlock); -} - -void nd_log_limits_unlimited(void) { - nd_log_limits_reset(); - for(size_t i = 0; i < _NDLS_MAX ;i++) { - nd_log.sources[i].limits.logs_per_period = 0; - } -} - -static bool nd_log_limit_reached(struct nd_log_source *source) { - if(source->limits.throttle_period == 0 || source->limits.logs_per_period == 0) - return false; - - usec_t now_ut = now_monotonic_usec(); - if(!source->limits.started_monotonic_ut) - source->limits.started_monotonic_ut = now_ut; - - source->limits.counter++; - - if(now_ut - source->limits.started_monotonic_ut > (usec_t)source->limits.throttle_period) { - if(source->limits.prevented) { - BUFFER *wb = buffer_create(1024, NULL); - buffer_sprintf(wb, - "LOG FLOOD PROTECTION: resuming logging " - "(prevented %"PRIu32" logs in the last %"PRIu32" seconds).", - source->limits.prevented, - source->limits.throttle_period); - - if(source->pending_msg) - freez((void *)source->pending_msg); - - source->pending_msg = strdupz(buffer_tostring(wb)); - - buffer_free(wb); - } - - // restart the period accounting - source->limits.started_monotonic_ut = now_ut; - source->limits.counter = 1; - source->limits.prevented = 0; - - // log this error - return false; - } - - if(source->limits.counter > source->limits.logs_per_period) { - if(!source->limits.prevented) { - BUFFER *wb = buffer_create(1024, NULL); - buffer_sprintf(wb, - "LOG FLOOD PROTECTION: too many logs (%"PRIu32" logs in %"PRId64" seconds, threshold is set to %"PRIu32" logs " - "in %"PRIu32" seconds). Preventing more logs from process '%s' for %"PRId64" seconds.", - source->limits.counter, - (int64_t)((now_ut - source->limits.started_monotonic_ut) / USEC_PER_SEC), - source->limits.logs_per_period, - source->limits.throttle_period, - program_name, - (int64_t)(((source->limits.started_monotonic_ut + (source->limits.throttle_period * USEC_PER_SEC) - now_ut)) / USEC_PER_SEC) - ); - - if(source->pending_msg) - freez((void *)source->pending_msg); - - source->pending_msg = strdupz(buffer_tostring(wb)); - - buffer_free(wb); - } - - source->limits.prevented++; - - // prevent logging this error -#ifdef NETDATA_INTERNAL_CHECKS - return false; -#else - return true; -#endif - } - - return false; -} diff --git a/src/libnetdata/log/nd_log-annotators.c b/src/libnetdata/log/nd_log-annotators.c new file mode 100644 index 000000000..92e9bf310 --- /dev/null +++ b/src/libnetdata/log/nd_log-annotators.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log-internals.h" + +const char *timestamp_usec_annotator(struct log_field *lf) { + usec_t ut = log_field_to_uint64(lf); + + if(!ut) + return NULL; + + static __thread char datetime[RFC3339_MAX_LENGTH]; + rfc3339_datetime_ut(datetime, sizeof(datetime), ut, 3, false); + return datetime; +} + +const char *errno_annotator(struct log_field *lf) { + int64_t errnum = log_field_to_int64(lf); + + if(errnum == 0) + return NULL; + + static __thread char buf[256]; + size_t len = print_uint64(buf, errnum); + buf[len++] = ','; + buf[len++] = ' '; + + char *msg_to = &buf[len]; + size_t msg_size = sizeof(buf) - len; + + const char *s = errno2str((int)errnum, msg_to, msg_size); + if(s != msg_to) + strncpyz(msg_to, s, msg_size - 1); + + return buf; +} + +#if defined(OS_WINDOWS) +const char *winerror_annotator(struct log_field *lf) { + DWORD errnum = log_field_to_uint64(lf); + + if (errnum == 0) + return NULL; + + static __thread char buf[256]; + size_t len = print_uint64(buf, errnum); + buf[len++] = ','; + buf[len++] = ' '; + + char *msg_to = &buf[len]; + size_t msg_size = sizeof(buf) - len; + + wchar_t wbuf[1024]; + DWORD size = FormatMessageW( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + errnum, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + wbuf, + (DWORD)(sizeof(wbuf) / sizeof(wchar_t) - 1), + NULL + ); + + if (size > 0) { + // Remove \r\n at the end + while (size > 0 && (wbuf[size - 1] == L'\r' || wbuf[size - 1] == L'\n')) + wbuf[--size] = L'\0'; + + // Convert wide string to UTF-8 + int utf8_size = WideCharToMultiByte(CP_UTF8, 0, wbuf, -1, msg_to, (int)msg_size, NULL, NULL); + if (utf8_size == 0) + snprintf(msg_to, msg_size - 1, "unknown error code"); + msg_to[msg_size - 1] = '\0'; + } + else + snprintf(msg_to, msg_size - 1, "unknown error code"); + + return buf; +} +#endif + +const char *priority_annotator(struct log_field *lf) { + uint64_t pri = log_field_to_uint64(lf); + return nd_log_id2priority(pri); +} diff --git a/src/libnetdata/log/nd_log-common.h b/src/libnetdata/log/nd_log-common.h new file mode 100644 index 000000000..d06bbbd16 --- /dev/null +++ b/src/libnetdata/log/nd_log-common.h @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_ND_LOG_COMMON_H +#define NETDATA_ND_LOG_COMMON_H + +#include <syslog.h> + +typedef enum __attribute__((__packed__)) { + NDLS_UNSET = 0, // internal use only + NDLS_ACCESS, // access.log + NDLS_ACLK, // aclk.log + NDLS_COLLECTORS, // collector.log + NDLS_DAEMON, // error.log + NDLS_HEALTH, // health.log + NDLS_DEBUG, // debug.log + + // terminator + _NDLS_MAX, +} ND_LOG_SOURCES; + +typedef enum __attribute__((__packed__)) { + NDLP_EMERG = LOG_EMERG, // from syslog.h + NDLP_ALERT = LOG_ALERT, // from syslog.h + NDLP_CRIT = LOG_CRIT, // from syslog.h + NDLP_ERR = LOG_ERR, // from syslog.h + NDLP_WARNING = LOG_WARNING, // from syslog.h + NDLP_NOTICE = LOG_NOTICE, // from syslog.h + NDLP_INFO = LOG_INFO, // from syslog.h + NDLP_DEBUG = LOG_DEBUG, // from syslog.h + + // terminator + _NDLP_MAX, +} ND_LOG_FIELD_PRIORITY; + +typedef enum __attribute__((__packed__)) { + // KEEP THESE IN THE SAME ORDER AS in thread_log_fields (log.c) + // so that it easy to audit for missing fields + + // NEVER RENUMBER THIS LIST + // The Windows Events Log has them at fixed positions + + NDF_STOP = 0, + NDF_TIMESTAMP_REALTIME_USEC = 1, // the timestamp of the log message - added automatically + NDF_SYSLOG_IDENTIFIER = 2, // the syslog identifier of the application - added automatically + NDF_LOG_SOURCE = 3, // DAEMON, COLLECTORS, HEALTH, MSGID_ACCESS, ACLK - set at the log call + NDF_PRIORITY = 4, // the syslog priority (severity) - set at the log call + NDF_ERRNO = 5, // the ERRNO at the time of the log call - added automatically + NDF_WINERROR = 6, // Windows GetLastError() + NDF_INVOCATION_ID = 7, // the INVOCATION_ID of Netdata - added automatically + NDF_LINE = 8, // the source code file line number - added automatically + NDF_FILE = 9, // the source code filename - added automatically + NDF_FUNC = 10, // the source code function - added automatically + NDF_TID = 11, // the thread ID of the thread logging - added automatically + NDF_THREAD_TAG = 12, // the thread tag of the thread logging - added automatically + NDF_MESSAGE_ID = 13, // for specific events + NDF_MODULE = 14, // for internal plugin module, all other get the NDF_THREAD_TAG + + NDF_NIDL_NODE = 15, // the node / rrdhost currently being worked + NDF_NIDL_INSTANCE = 16, // the instance / rrdset currently being worked + NDF_NIDL_CONTEXT = 17, // the context of the instance currently being worked + NDF_NIDL_DIMENSION = 18, // the dimension / rrddim currently being worked + + // web server, aclk and stream receiver + NDF_SRC_TRANSPORT = 19, // the transport we received the request, one of: http, https, pluginsd + + // Netdata Cloud Related + NDF_ACCOUNT_ID = 20, + NDF_USER_NAME = 21, + NDF_USER_ROLE = 22, + NDF_USER_ACCESS = 23, + + // web server and stream receiver + NDF_SRC_IP = 24, // the streaming / web server source IP + NDF_SRC_PORT = 25, // the streaming / web server source Port + NDF_SRC_FORWARDED_HOST = 26, + NDF_SRC_FORWARDED_FOR = 27, + NDF_SRC_CAPABILITIES = 28, // the stream receiver capabilities + + // stream sender (established links) + NDF_DST_TRANSPORT = 29, // the transport we send the request, one of: http, https + NDF_DST_IP = 30, // the destination streaming IP + NDF_DST_PORT = 31, // the destination streaming Port + NDF_DST_CAPABILITIES = 32, // the destination streaming capabilities + + // web server, aclk and stream receiver + NDF_REQUEST_METHOD = 33, // for http like requests, the http request method + NDF_RESPONSE_CODE = 34, // for http like requests, the http response code, otherwise a status string + + // web server (all), aclk (queries) + NDF_CONNECTION_ID = 35, // the web server connection ID + NDF_TRANSACTION_ID = 36, // the web server and API transaction ID + NDF_RESPONSE_SENT_BYTES = 37, // for http like requests, the response bytes + NDF_RESPONSE_SIZE_BYTES = 38, // for http like requests, the uncompressed response size + NDF_RESPONSE_PREPARATION_TIME_USEC = 39, // for http like requests, the preparation time + NDF_RESPONSE_SENT_TIME_USEC = 40, // for http like requests, the time to send the response back + NDF_RESPONSE_TOTAL_TIME_USEC = 41, // for http like requests, the total time to complete the response + + // health alerts + NDF_ALERT_ID = 42, + NDF_ALERT_UNIQUE_ID = 43, + NDF_ALERT_EVENT_ID = 44, + NDF_ALERT_TRANSITION_ID = 45, + NDF_ALERT_CONFIG_HASH = 46, + NDF_ALERT_NAME = 47, + NDF_ALERT_CLASS = 48, + NDF_ALERT_COMPONENT = 49, + NDF_ALERT_TYPE = 50, + NDF_ALERT_EXEC = 51, + NDF_ALERT_RECIPIENT = 52, + NDF_ALERT_DURATION = 53, + NDF_ALERT_VALUE = 54, + NDF_ALERT_VALUE_OLD = 55, + NDF_ALERT_STATUS = 56, + NDF_ALERT_STATUS_OLD = 57, + NDF_ALERT_SOURCE = 58, + NDF_ALERT_UNITS = 59, + NDF_ALERT_SUMMARY = 60, + NDF_ALERT_INFO = 61, + NDF_ALERT_NOTIFICATION_REALTIME_USEC = 62, + // NDF_ALERT_FLAGS, + + // put new items here + // leave the request URL and the message last + + NDF_REQUEST = 63, // the request we are currently working on + NDF_MESSAGE = 64, // the log message, if any + + // terminator + _NDF_MAX, +} ND_LOG_FIELD_ID; + +typedef enum __attribute__((__packed__)) { + NDFT_UNSET = 0, + NDFT_TXT, + NDFT_STR, + NDFT_BFR, + NDFT_U64, + NDFT_I64, + NDFT_DBL, + NDFT_UUID, + NDFT_CALLBACK, + + // terminator + _NDFT_MAX, +} ND_LOG_STACK_FIELD_TYPE; + +#endif //NETDATA_ND_LOG_COMMON_H diff --git a/src/libnetdata/log/nd_log-config.c b/src/libnetdata/log/nd_log-config.c new file mode 100644 index 000000000..c8e17402e --- /dev/null +++ b/src/libnetdata/log/nd_log-config.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log-internals.h" + +void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting) { + char buf[FILENAME_MAX + 100]; + if(setting && *setting) + strncpyz(buf, setting, sizeof(buf) - 1); + else + buf[0] = '\0'; + + struct nd_log_source *ls = &nd_log.sources[source]; + char *output = strrchr(buf, '@'); + + if(!output) + // all of it is the output + output = buf; + else { + // we found an '@', the next char is the output + *output = '\0'; + output++; + + // parse the other params + char *remaining = buf; + while(remaining) { + char *value = strsep_skip_consecutive_separators(&remaining, ","); + if (!value || !*value) continue; + + char *name = strsep_skip_consecutive_separators(&value, "="); + if (!name || !*name) continue; + + if(strcmp(name, "logfmt") == 0) + ls->format = NDLF_LOGFMT; + else if(strcmp(name, "json") == 0) + ls->format = NDLF_JSON; + else if(strcmp(name, "journal") == 0) + ls->format = NDLF_JOURNAL; +#if defined(OS_WINDOWS) +#if defined(HAVE_ETW) + else if(strcmp(name, ETW_NAME) == 0) + ls->format = NDLF_ETW; +#endif +#if defined(HAVE_WEL) + else if(strcmp(name, WEL_NAME) == 0) + ls->format = NDLF_WEL; +#endif +#endif + else if(strcmp(name, "level") == 0 && value && *value) + ls->min_priority = nd_log_priority2id(value); + else if(strcmp(name, "protection") == 0 && value && *value) { + if(strcmp(value, "off") == 0 || strcmp(value, "none") == 0) { + ls->limits = ND_LOG_LIMITS_UNLIMITED; + ls->limits.counter = 0; + ls->limits.prevented = 0; + } + else { + ls->limits = ND_LOG_LIMITS_DEFAULT; + + char *slash = strchr(value, '/'); + if(slash) { + *slash = '\0'; + slash++; + ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value); + + int period; + if(!duration_parse_seconds(slash, &period)) { + nd_log(NDLS_DAEMON, NDLP_ERR, "Error while parsing period '%s'", slash); + period = ND_LOG_DEFAULT_THROTTLE_PERIOD; + } + + ls->limits.throttle_period = period; + } + else { + ls->limits.logs_per_period = ls->limits.logs_per_period_backup = str2u(value); + ls->limits.throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD; + } + } + } + else + nd_log(NDLS_DAEMON, NDLP_ERR, + "Error while parsing configuration of log source '%s'. " + "In config '%s', '%s' is not understood.", + nd_log_id2source(source), setting, name); + } + } + + if(!output || !*output || strcmp(output, "none") == 0 || strcmp(output, "off") == 0) { + ls->method = NDLM_DISABLED; + ls->filename = "/dev/null"; + } + else if(strcmp(output, "journal") == 0) { + ls->method = NDLM_JOURNAL; + ls->filename = NULL; + } +#if defined(OS_WINDOWS) +#if defined(HAVE_ETW) + else if(strcmp(output, ETW_NAME) == 0) { + ls->method = NDLM_ETW; + ls->filename = NULL; + } +#endif +#if defined(HAVE_WEL) + else if(strcmp(output, WEL_NAME) == 0) { + ls->method = NDLM_WEL; + ls->filename = NULL; + } +#endif +#endif + else if(strcmp(output, "syslog") == 0) { + ls->method = NDLM_SYSLOG; + ls->filename = NULL; + } + else if(strcmp(output, "/dev/null") == 0) { + ls->method = NDLM_DEVNULL; + ls->filename = "/dev/null"; + } + else if(strcmp(output, "system") == 0) { + if(ls->fd == STDERR_FILENO) { + ls->method = NDLM_STDERR; + ls->filename = NULL; + ls->fd = STDERR_FILENO; + } + else { + ls->method = NDLM_STDOUT; + ls->filename = NULL; + ls->fd = STDOUT_FILENO; + } + } + else if(strcmp(output, "stderr") == 0) { + ls->method = NDLM_STDERR; + ls->filename = NULL; + ls->fd = STDERR_FILENO; + } + else if(strcmp(output, "stdout") == 0) { + ls->method = NDLM_STDOUT; + ls->filename = NULL; + ls->fd = STDOUT_FILENO; + } + else { + ls->method = NDLM_FILE; + ls->filename = strdupz(output); + } + +#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE) + ls->min_priority = NDLP_DEBUG; +#endif + + if(source == NDLS_COLLECTORS) { + // set the method for the collector processes we will spawn + + ND_LOG_METHOD method = NDLM_STDERR; + ND_LOG_FORMAT format = NDLF_LOGFMT; + ND_LOG_FIELD_PRIORITY priority = ls->min_priority; + + if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ls->method)) { + method = ls->method; + format = ls->format; + } + + nd_setenv("NETDATA_LOG_METHOD", nd_log_id2method(method), 1); + nd_setenv("NETDATA_LOG_FORMAT", nd_log_id2format(format), 1); + nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1); + } +} + +void nd_log_set_priority_level(const char *setting) { + if(!setting || !*setting) + setting = "info"; + + ND_LOG_FIELD_PRIORITY priority = nd_log_priority2id(setting); + +#if defined(NETDATA_INTERNAL_CHECKS) || defined(NETDATA_DEV_MODE) + priority = NDLP_DEBUG; +#endif + + for (size_t i = 0; i < _NDLS_MAX; i++) { + if (i != NDLS_DEBUG) + nd_log.sources[i].min_priority = priority; + } + + // the right one + nd_setenv("NETDATA_LOG_LEVEL", nd_log_id2priority(priority), 1); +} + +void nd_log_set_facility(const char *facility) { + if(!facility || !*facility) + facility = "daemon"; + + nd_log.syslog.facility = nd_log_facility2id(facility); + nd_setenv("NETDATA_SYSLOG_FACILITY", nd_log_id2facility(nd_log.syslog.facility), 1); +} + +void nd_log_set_flood_protection(size_t logs, time_t period) { + nd_log.sources[NDLS_DAEMON].limits.logs_per_period = + nd_log.sources[NDLS_DAEMON].limits.logs_per_period_backup; + nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period = + nd_log.sources[NDLS_COLLECTORS].limits.logs_per_period_backup = logs; + + nd_log.sources[NDLS_DAEMON].limits.throttle_period = + nd_log.sources[NDLS_COLLECTORS].limits.throttle_period = period; + + char buf[100]; + snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )period); + nd_setenv("NETDATA_ERRORS_THROTTLE_PERIOD", buf, 1); + snprintfz(buf, sizeof(buf), "%" PRIu64, (uint64_t )logs); + nd_setenv("NETDATA_ERRORS_PER_PERIOD", buf, 1); +} diff --git a/src/libnetdata/log/nd_log-field-formatters.c b/src/libnetdata/log/nd_log-field-formatters.c new file mode 100644 index 000000000..e1b3c0d08 --- /dev/null +++ b/src/libnetdata/log/nd_log-field-formatters.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log-internals.h" + +int64_t log_field_to_int64(struct log_field *lf) { + + // --- FIELD_PARSER_VERSIONS --- + // + // IMPORTANT: + // THERE ARE 6 VERSIONS OF THIS CODE + // + // 1. journal (direct socket API), + // 2. journal (libsystemd API), + // 3. logfmt, + // 4. json, + // 5. convert to uint64 + // 6. convert to int64 + // + // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES + + CLEAN_BUFFER *tmp = NULL; + const char *s = NULL; + + switch(lf->entry.type) { + default: + case NDFT_UUID: + case NDFT_UNSET: + return 0; + + case NDFT_TXT: + s = lf->entry.txt; + break; + + case NDFT_STR: + s = string2str(lf->entry.str); + break; + + case NDFT_BFR: + s = buffer_tostring(lf->entry.bfr); + break; + + case NDFT_CALLBACK: + tmp = buffer_create(0, NULL); + + if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data)) + s = buffer_tostring(tmp); + else + s = NULL; + break; + + case NDFT_U64: + return (int64_t)lf->entry.u64; + + case NDFT_I64: + return (int64_t)lf->entry.i64; + + case NDFT_DBL: + return (int64_t)lf->entry.dbl; + } + + if(s && *s) + return str2ll(s, NULL); + + return 0; +} + +uint64_t log_field_to_uint64(struct log_field *lf) { + + // --- FIELD_PARSER_VERSIONS --- + // + // IMPORTANT: + // THERE ARE 6 VERSIONS OF THIS CODE + // + // 1. journal (direct socket API), + // 2. journal (libsystemd API), + // 3. logfmt, + // 4. json, + // 5. convert to uint64 + // 6. convert to int64 + // + // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES + + CLEAN_BUFFER *tmp = NULL; + const char *s = NULL; + + switch(lf->entry.type) { + default: + case NDFT_UUID: + case NDFT_UNSET: + return 0; + + case NDFT_TXT: + s = lf->entry.txt; + break; + + case NDFT_STR: + s = string2str(lf->entry.str); + break; + + case NDFT_BFR: + s = buffer_tostring(lf->entry.bfr); + break; + + case NDFT_CALLBACK: + tmp = buffer_create(0, NULL); + + if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data)) + s = buffer_tostring(tmp); + else + s = NULL; + break; + + case NDFT_U64: + return lf->entry.u64; + + case NDFT_I64: + return lf->entry.i64; + + case NDFT_DBL: + return (uint64_t) lf->entry.dbl; + } + + if(s && *s) + return str2uint64_t(s, NULL); + + return 0; +} diff --git a/src/libnetdata/log/nd_log-format-json.c b/src/libnetdata/log/nd_log-format-json.c new file mode 100644 index 000000000..c25bf19c5 --- /dev/null +++ b/src/libnetdata/log/nd_log-format-json.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log-internals.h" + +void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max) { + + // --- FIELD_PARSER_VERSIONS --- + // + // IMPORTANT: + // THERE ARE 6 VERSIONS OF THIS CODE + // + // 1. journal (direct socket API), + // 2. journal (libsystemd API), + // 3. logfmt, + // 4. json, + // 5. convert to uint64 + // 6. convert to int64 + // + // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES + + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); + CLEAN_BUFFER *tmp = NULL; + + for (size_t i = 0; i < fields_max; i++) { + if (!fields[i].entry.set || !fields[i].logfmt) + continue; + + const char *key = fields[i].logfmt; + + const char *s = NULL; + switch(fields[i].entry.type) { + case NDFT_TXT: + s = fields[i].entry.txt; + break; + case NDFT_STR: + s = string2str(fields[i].entry.str); + break; + case NDFT_BFR: + s = buffer_tostring(fields[i].entry.bfr); + break; + case NDFT_U64: + buffer_json_member_add_uint64(wb, key, fields[i].entry.u64); + break; + case NDFT_I64: + buffer_json_member_add_int64(wb, key, fields[i].entry.i64); + break; + case NDFT_DBL: + buffer_json_member_add_double(wb, key, fields[i].entry.dbl); + break; + case NDFT_UUID: + if(!uuid_is_null(*fields[i].entry.uuid)) { + char u[UUID_COMPACT_STR_LEN]; + uuid_unparse_lower_compact(*fields[i].entry.uuid, u); + buffer_json_member_add_string(wb, key, u); + } + break; + case NDFT_CALLBACK: { + if(!tmp) + tmp = buffer_create(1024, NULL); + else + buffer_flush(tmp); + if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) + s = buffer_tostring(tmp); + else + s = NULL; + } + break; + default: + s = "UNHANDLED"; + break; + } + + if(s && *s) + buffer_json_member_add_string(wb, key, s); + } + + buffer_json_finalize(wb); +} diff --git a/src/libnetdata/log/nd_log-format-logfmt.c b/src/libnetdata/log/nd_log-format-logfmt.c new file mode 100644 index 000000000..d65211dfc --- /dev/null +++ b/src/libnetdata/log/nd_log-format-logfmt.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log-internals.h" + +static bool needs_quotes_for_logfmt(const char *s) +{ + static bool safe_for_logfmt[256] = { + [' '] = true, ['!'] = true, ['"'] = false, ['#'] = true, ['$'] = true, ['%'] = true, ['&'] = true, + ['\''] = true, ['('] = true, [')'] = true, ['*'] = true, ['+'] = true, [','] = true, ['-'] = true, + ['.'] = true, ['/'] = true, ['0'] = true, ['1'] = true, ['2'] = true, ['3'] = true, ['4'] = true, + ['5'] = true, ['6'] = true, ['7'] = true, ['8'] = true, ['9'] = true, [':'] = true, [';'] = true, + ['<'] = true, ['='] = true, ['>'] = true, ['?'] = true, ['@'] = true, ['A'] = true, ['B'] = true, + ['C'] = true, ['D'] = true, ['E'] = true, ['F'] = true, ['G'] = true, ['H'] = true, ['I'] = true, + ['J'] = true, ['K'] = true, ['L'] = true, ['M'] = true, ['N'] = true, ['O'] = true, ['P'] = true, + ['Q'] = true, ['R'] = true, ['S'] = true, ['T'] = true, ['U'] = true, ['V'] = true, ['W'] = true, + ['X'] = true, ['Y'] = true, ['Z'] = true, ['['] = true, ['\\'] = false, [']'] = true, ['^'] = true, + ['_'] = true, ['`'] = true, ['a'] = true, ['b'] = true, ['c'] = true, ['d'] = true, ['e'] = true, + ['f'] = true, ['g'] = true, ['h'] = true, ['i'] = true, ['j'] = true, ['k'] = true, ['l'] = true, + ['m'] = true, ['n'] = true, ['o'] = true, ['p'] = true, ['q'] = true, ['r'] = true, ['s'] = true, + ['t'] = true, ['u'] = true, ['v'] = true, ['w'] = true, ['x'] = true, ['y'] = true, ['z'] = true, + ['{'] = true, ['|'] = true, ['}'] = true, ['~'] = true, [0x7f] = true, + }; + + if(!*s) + return true; + + while(*s) { + if(*s == '=' || isspace((uint8_t)*s) || !safe_for_logfmt[(uint8_t)*s]) + return true; + + s++; + } + + return false; +} + +static void string_to_logfmt(BUFFER *wb, const char *s) +{ + bool spaces = needs_quotes_for_logfmt(s); + + if(spaces) + buffer_fast_strcat(wb, "\"", 1); + + buffer_json_strcat(wb, s); + + if(spaces) + buffer_fast_strcat(wb, "\"", 1); +} + +void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max) { + + // --- FIELD_PARSER_VERSIONS --- + // + // IMPORTANT: + // THERE ARE 6 VERSIONS OF THIS CODE + // + // 1. journal (direct socket API), + // 2. journal (libsystemd API), + // 3. logfmt, + // 4. json, + // 5. convert to uint64 + // 6. convert to int64 + // + // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES + + CLEAN_BUFFER *tmp = NULL; + + for (size_t i = 0; i < fields_max; i++) { + if (!fields[i].entry.set || !fields[i].logfmt) + continue; + + const char *key = fields[i].logfmt; + + if(fields[i].annotator) { + const char *s = fields[i].annotator(&fields[i]); + if(!s) continue; + + if(buffer_strlen(wb)) + buffer_fast_strcat(wb, " ", 1); + + buffer_strcat(wb, key); + buffer_fast_strcat(wb, "=", 1); + string_to_logfmt(wb, s); + } + else { + if(buffer_strlen(wb)) + buffer_fast_strcat(wb, " ", 1); + + switch(fields[i].entry.type) { + case NDFT_TXT: + if(*fields[i].entry.txt) { + buffer_strcat(wb, key); + buffer_fast_strcat(wb, "=", 1); + string_to_logfmt(wb, fields[i].entry.txt); + } + break; + case NDFT_STR: + buffer_strcat(wb, key); + buffer_fast_strcat(wb, "=", 1); + string_to_logfmt(wb, string2str(fields[i].entry.str)); + break; + case NDFT_BFR: + if(buffer_strlen(fields[i].entry.bfr)) { + buffer_strcat(wb, key); + buffer_fast_strcat(wb, "=", 1); + string_to_logfmt(wb, buffer_tostring(fields[i].entry.bfr)); + } + break; + case NDFT_U64: + buffer_strcat(wb, key); + buffer_fast_strcat(wb, "=", 1); + buffer_print_uint64(wb, fields[i].entry.u64); + break; + case NDFT_I64: + buffer_strcat(wb, key); + buffer_fast_strcat(wb, "=", 1); + buffer_print_int64(wb, fields[i].entry.i64); + break; + case NDFT_DBL: + buffer_strcat(wb, key); + buffer_fast_strcat(wb, "=", 1); + buffer_print_netdata_double(wb, fields[i].entry.dbl); + break; + case NDFT_UUID: + if(!uuid_is_null(*fields[i].entry.uuid)) { + char u[UUID_COMPACT_STR_LEN]; + uuid_unparse_lower_compact(*fields[i].entry.uuid, u); + buffer_strcat(wb, key); + buffer_fast_strcat(wb, "=", 1); + buffer_fast_strcat(wb, u, sizeof(u) - 1); + } + break; + case NDFT_CALLBACK: { + if(!tmp) + tmp = buffer_create(1024, NULL); + else + buffer_flush(tmp); + if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) { + buffer_strcat(wb, key); + buffer_fast_strcat(wb, "=", 1); + string_to_logfmt(wb, buffer_tostring(tmp)); + } + } + break; + default: + buffer_strcat(wb, "UNHANDLED"); + break; + } + } + } +} diff --git a/src/libnetdata/log/nd_log-init.c b/src/libnetdata/log/nd_log-init.c new file mode 100644 index 000000000..7f846b136 --- /dev/null +++ b/src/libnetdata/log/nd_log-init.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log-internals.h" + +// -------------------------------------------------------------------------------------------------------------------- + +__attribute__((constructor)) void initialize_invocation_id(void) { + // check for a NETDATA_INVOCATION_ID + if(uuid_parse_flexi(getenv("NETDATA_INVOCATION_ID"), nd_log.invocation_id) != 0) { + // not found, check for systemd set INVOCATION_ID + if(uuid_parse_flexi(getenv("INVOCATION_ID"), nd_log.invocation_id) != 0) { + // not found, generate a new one + uuid_generate_random(nd_log.invocation_id); + } + } + + char uuid[UUID_COMPACT_STR_LEN]; + uuid_unparse_lower_compact(nd_log.invocation_id, uuid); + nd_setenv("NETDATA_INVOCATION_ID", uuid, 1); +} + +// -------------------------------------------------------------------------------------------------------------------- + +void nd_log_initialize_for_external_plugins(const char *name) { + // if we don't run under Netdata, log to stderr, + // otherwise, use the logging method Netdata wants us to use. +#if defined(OS_WINDOWS) +#if defined(HAVE_ETW) + nd_setenv("NETDATA_LOG_METHOD", ETW_NAME, 0); + nd_setenv("NETDATA_LOG_FORMAT", ETW_NAME, 0); +#elif defined(HAVE_WEL) + nd_setenv("NETDATA_LOG_METHOD", WEL_NAME, 0); + nd_setenv("NETDATA_LOG_FORMAT", WEL_NAME, 0); +#else + nd_setenv("NETDATA_LOG_METHOD", "stderr", 0); + nd_setenv("NETDATA_LOG_FORMAT", "logfmt", 0); +#endif +#else + nd_setenv("NETDATA_LOG_METHOD", "stderr", 0); + nd_setenv("NETDATA_LOG_FORMAT", "logfmt", 0); +#endif + + nd_log.overwrite_process_source = NDLS_COLLECTORS; + program_name = name; + + for(size_t i = 0; i < _NDLS_MAX ;i++) { + nd_log.sources[i].method = NDLM_DEFAULT; + nd_log.sources[i].fd = -1; + nd_log.sources[i].fp = NULL; + } + + nd_log_set_priority_level(getenv("NETDATA_LOG_LEVEL")); + nd_log_set_facility(getenv("NETDATA_SYSLOG_FACILITY")); + + time_t period = 1200; + size_t logs = 200; + const char *s = getenv("NETDATA_ERRORS_THROTTLE_PERIOD"); + if(s && *s >= '0' && *s <= '9') { + period = str2l(s); + if(period < 0) period = 0; + } + + s = getenv("NETDATA_ERRORS_PER_PERIOD"); + if(s && *s >= '0' && *s <= '9') + logs = str2u(s); + + nd_log_set_flood_protection(logs, period); + + if(!netdata_configured_host_prefix) { + s = getenv("NETDATA_HOST_PREFIX"); + if(s && *s) + netdata_configured_host_prefix = (char *)s; + } + + ND_LOG_METHOD method = nd_log_method2id(getenv("NETDATA_LOG_METHOD")); + ND_LOG_FORMAT format = nd_log_format2id(getenv("NETDATA_LOG_FORMAT")); + + if(!IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method)) { + if(is_stderr_connected_to_journal()) { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using journal."); + method = NDLM_JOURNAL; + } + else { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "NETDATA_LOG_METHOD is not set. Using stderr."); + method = NDLM_STDERR; + } + } + + switch(method) { + case NDLM_JOURNAL: + if(!nd_log_journal_direct_init(getenv("NETDATA_SYSTEMD_JOURNAL_PATH")) || + !nd_log_journal_direct_init(NULL) || !nd_log_journal_systemd_init()) { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize journal. Using stderr."); + method = NDLM_STDERR; + } + break; + +#if defined(OS_WINDOWS) +#if defined(HAVE_ETW) + case NDLM_ETW: + if(!nd_log_init_etw()) { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize Events Tracing for Windows (ETW). Using stderr."); + method = NDLM_STDERR; + } + break; +#endif +#if defined(HAVE_WEL) + case NDLM_WEL: + if(!nd_log_init_wel()) { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize Windows Event Log (WEL). Using stderr."); + method = NDLM_STDERR; + } + break; +#endif +#endif + + case NDLM_SYSLOG: + nd_log_init_syslog(); + break; + + default: + method = NDLM_STDERR; + break; + } + + nd_log.sources[NDLS_COLLECTORS].method = method; + nd_log.sources[NDLS_COLLECTORS].format = format; + nd_log.sources[NDLS_COLLECTORS].fd = -1; + nd_log.sources[NDLS_COLLECTORS].fp = NULL; + + // nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "FINAL_LOG_METHOD: %s", nd_log_id2method(method)); +} + +// -------------------------------------------------------------------------------------------------------------------- + +void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source) { + if(e->method == NDLM_DEFAULT) + nd_log_set_user_settings(source, e->filename); + + if((e->method == NDLM_FILE && !e->filename) || + (e->method == NDLM_DEVNULL && e->fd == -1)) + e->method = NDLM_DISABLED; + + if(e->fp) + fflush(e->fp); + + switch(e->method) { + case NDLM_SYSLOG: + nd_log_init_syslog(); + break; + + case NDLM_JOURNAL: + nd_log_journal_direct_init(NULL); + nd_log_journal_systemd_init(); + break; + +#if defined(OS_WINDOWS) +#if defined(HAVE_ETW) + case NDLM_ETW: + nd_log_init_etw(); + break; +#endif +#if defined(HAVE_WEL) + case NDLM_WEL: + nd_log_init_wel(); + break; +#endif +#endif + + case NDLM_STDOUT: + e->fp = stdout; + e->fd = STDOUT_FILENO; + break; + + case NDLM_DISABLED: + break; + + case NDLM_DEFAULT: + case NDLM_STDERR: + e->method = NDLM_STDERR; + e->fp = stderr; + e->fd = STDERR_FILENO; + break; + + case NDLM_DEVNULL: + case NDLM_FILE: { + int fd = open(e->filename, O_WRONLY | O_APPEND | O_CREAT, 0664); + if(fd == -1) { + if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO) { + e->fd = STDERR_FILENO; + e->method = NDLM_STDERR; + netdata_log_error("Cannot open log file '%s'. Falling back to stderr.", e->filename); + } + else + netdata_log_error("Cannot open log file '%s'. Leaving fd %d as-is.", e->filename, e->fd); + } + else { + if (!nd_log_replace_existing_fd(e, fd)) { + if(e->fd == STDOUT_FILENO || e->fd == STDERR_FILENO) { + if(e->fd == STDOUT_FILENO) + e->method = NDLM_STDOUT; + else if(e->fd == STDERR_FILENO) + e->method = NDLM_STDERR; + + // we have dup2() fd, so we can close the one we opened + if(fd != STDOUT_FILENO && fd != STDERR_FILENO) + close(fd); + } + else + e->fd = fd; + } + } + + // at this point we have e->fd set properly + + if(e->fd == STDOUT_FILENO) + e->fp = stdout; + else if(e->fd == STDERR_FILENO) + e->fp = stderr; + + if(!e->fp) { + e->fp = fdopen(e->fd, "a"); + if (!e->fp) { + netdata_log_error("Cannot fdopen() fd %d ('%s')", e->fd, e->filename); + + if(e->fd != STDOUT_FILENO && e->fd != STDERR_FILENO) + close(e->fd); + + e->fp = stderr; + e->fd = STDERR_FILENO; + } + } + else { + if (setvbuf(e->fp, NULL, _IOLBF, 0) != 0) + netdata_log_error("Cannot set line buffering on fd %d ('%s')", e->fd, e->filename); + } + } + break; + } +} + +// -------------------------------------------------------------------------------------------------------------------- + +void nd_log_stdin_init(int fd, const char *filename) { + int f = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0664); + if(f == -1) + return; + + if(f != fd) { + dup2(f, fd); + close(f); + } +} + +void nd_log_initialize(void) { + nd_log_stdin_init(STDIN_FILENO, "/dev/null"); + + for(size_t i = 0 ; i < _NDLS_MAX ; i++) + nd_log_open(&nd_log.sources[i], i); +} + +void nd_log_reopen_log_files(bool log) { + if(log) + netdata_log_info("Reopening all log files."); + + nd_log_initialize(); + + if(log) + netdata_log_info("Log files re-opened."); +} + +int nd_log_systemd_journal_fd(void) { + return nd_log.journal.fd; +} + +void nd_log_reopen_log_files_for_spawn_server(const char *name) { + gettid_uncached(); + + if(nd_log.syslog.initialized) { + closelog(); + nd_log.syslog.initialized = false; + nd_log_init_syslog(); + } + + if(nd_log.journal_direct.initialized) { + close(nd_log.journal_direct.fd); + nd_log.journal_direct.fd = -1; + nd_log.journal_direct.initialized = false; + } + + for(size_t i = 0; i < _NDLS_MAX ;i++) { + spinlock_init(&nd_log.sources[i].spinlock); + nd_log.sources[i].method = NDLM_DEFAULT; + nd_log.sources[i].fd = -1; + nd_log.sources[i].fp = NULL; + nd_log.sources[i].pending_msg = NULL; +#if defined(OS_WINDOWS) + nd_log.sources[i].hEventLog = NULL; +#endif + } + + // initialize spinlocks + spinlock_init(&nd_log.std_output.spinlock); + spinlock_init(&nd_log.std_error.spinlock); + + nd_log.syslog.initialized = false; + nd_log.eventlog.initialized = false; + nd_log.std_output.initialized = false; + nd_log.std_error.initialized = false; + + nd_log_initialize_for_external_plugins(name); +} + diff --git a/src/libnetdata/log/nd_log-internals.c b/src/libnetdata/log/nd_log-internals.c new file mode 100644 index 000000000..97f521fad --- /dev/null +++ b/src/libnetdata/log/nd_log-internals.c @@ -0,0 +1,823 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log-internals.h" + +// -------------------------------------------------------------------------------------------------------------------- +// workaround strerror_r() + +#if defined(STRERROR_R_CHAR_P) +// GLIBC version of strerror_r +static const char *strerror_result(const char *a, const char *b) { (void)b; return a; } +#elif defined(HAVE_STRERROR_R) +// POSIX version of strerror_r +static const char *strerror_result(int a, const char *b) { (void)a; return b; } +#elif defined(HAVE_C__GENERIC) + +// what a trick! +// http://stackoverflow.com/questions/479207/function-overloading-in-c +static const char *strerror_result_int(int a, const char *b) { (void)a; return b; } +static const char *strerror_result_string(const char *a, const char *b) { (void)b; return a; } + +#define strerror_result(a, b) _Generic((a), \ + int: strerror_result_int, \ + char *: strerror_result_string \ + )(a, b) + +#else +#error "cannot detect the format of function strerror_r()" +#endif + +const char *errno2str(int errnum, char *buf, size_t size) { + return strerror_result(strerror_r(errnum, buf, size), buf); +} + +// -------------------------------------------------------------------------------------------------------------------- +// logging method + +static struct { + ND_LOG_METHOD method; + const char *name; +} nd_log_methods[] = { + { .method = NDLM_DISABLED, .name = "none" }, + { .method = NDLM_DEVNULL, .name = "/dev/null" }, + { .method = NDLM_DEFAULT, .name = "default" }, + { .method = NDLM_JOURNAL, .name = "journal" }, + { .method = NDLM_SYSLOG, .name = "syslog" }, + { .method = NDLM_STDOUT, .name = "stdout" }, + { .method = NDLM_STDERR, .name = "stderr" }, + { .method = NDLM_FILE, .name = "file" }, +#if defined(OS_WINDOWS) +#if defined(HAVE_ETW) + { .method = NDLM_ETW, .name = ETW_NAME }, +#endif +#if defined(HAVE_WEL) + { .method = NDLM_WEL, .name = WEL_NAME }, +#endif +#endif +}; + +ND_LOG_METHOD nd_log_method2id(const char *method) { + if(!method || !*method) + return NDLM_DEFAULT; + + size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]); + for(size_t i = 0; i < entries ;i++) { + if(strcmp(nd_log_methods[i].name, method) == 0) + return nd_log_methods[i].method; + } + + return NDLM_FILE; +} + +const char *nd_log_id2method(ND_LOG_METHOD method) { + size_t entries = sizeof(nd_log_methods) / sizeof(nd_log_methods[0]); + for(size_t i = 0; i < entries ;i++) { + if(method == nd_log_methods[i].method) + return nd_log_methods[i].name; + } + + return "unknown"; +} + +const char *nd_log_method_for_external_plugins(const char *s) { + if(s && *s) { + ND_LOG_METHOD method = nd_log_method2id(s); + if(IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(method)) + return nd_log_id2method(method); + } + + return nd_log_id2method(NDLM_STDERR); +} + +// -------------------------------------------------------------------------------------------------------------------- +// facilities +// +// sys/syslog.h (Linux) +// sys/sys/syslog.h (FreeBSD) +// bsd/sys/syslog.h (darwin-xnu) + +static struct { + int facility; + const char *name; +} nd_log_facilities[] = { + { LOG_AUTH, "auth" }, + { LOG_AUTHPRIV, "authpriv" }, + { LOG_CRON, "cron" }, + { LOG_DAEMON, "daemon" }, + { LOG_FTP, "ftp" }, + { LOG_KERN, "kern" }, + { LOG_LPR, "lpr" }, + { LOG_MAIL, "mail" }, + { LOG_NEWS, "news" }, + { LOG_SYSLOG, "syslog" }, + { LOG_USER, "user" }, + { LOG_UUCP, "uucp" }, + { LOG_LOCAL0, "local0" }, + { LOG_LOCAL1, "local1" }, + { LOG_LOCAL2, "local2" }, + { LOG_LOCAL3, "local3" }, + { LOG_LOCAL4, "local4" }, + { LOG_LOCAL5, "local5" }, + { LOG_LOCAL6, "local6" }, + { LOG_LOCAL7, "local7" }, + +#ifdef __FreeBSD__ + { LOG_CONSOLE, "console" }, + { LOG_NTP, "ntp" }, + + // FreeBSD does not consider 'security' as deprecated. + { LOG_SECURITY, "security" }, +#else + // For all other O/S 'security' is mapped to 'auth'. + { LOG_AUTH, "security" }, +#endif + +#ifdef __APPLE__ + { LOG_INSTALL, "install" }, + { LOG_NETINFO, "netinfo" }, + { LOG_RAS, "ras" }, + { LOG_REMOTEAUTH, "remoteauth" }, + { LOG_LAUNCHD, "launchd" }, + +#endif +}; + +int nd_log_facility2id(const char *facility) { + size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]); + for(size_t i = 0; i < entries ;i++) { + if(strcmp(nd_log_facilities[i].name, facility) == 0) + return nd_log_facilities[i].facility; + } + + return LOG_DAEMON; +} + +const char *nd_log_id2facility(int facility) { + size_t entries = sizeof(nd_log_facilities) / sizeof(nd_log_facilities[0]); + for(size_t i = 0; i < entries ;i++) { + if(nd_log_facilities[i].facility == facility) + return nd_log_facilities[i].name; + } + + return "daemon"; +} + +// -------------------------------------------------------------------------------------------------------------------- +// priorities + +static struct { + ND_LOG_FIELD_PRIORITY priority; + const char *name; +} nd_log_priorities[] = { + { .priority = NDLP_EMERG, .name = "emergency" }, + { .priority = NDLP_EMERG, .name = "emerg" }, + { .priority = NDLP_ALERT, .name = "alert" }, + { .priority = NDLP_CRIT, .name = "critical" }, + { .priority = NDLP_CRIT, .name = "crit" }, + { .priority = NDLP_ERR, .name = "error" }, + { .priority = NDLP_ERR, .name = "err" }, + { .priority = NDLP_WARNING, .name = "warning" }, + { .priority = NDLP_WARNING, .name = "warn" }, + { .priority = NDLP_NOTICE, .name = "notice" }, + { .priority = NDLP_INFO, .name = NDLP_INFO_STR }, + { .priority = NDLP_DEBUG, .name = "debug" }, +}; + +int nd_log_priority2id(const char *priority) { + size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]); + for(size_t i = 0; i < entries ;i++) { + if(strcmp(nd_log_priorities[i].name, priority) == 0) + return nd_log_priorities[i].priority; + } + + return NDLP_INFO; +} + +const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority) { + size_t entries = sizeof(nd_log_priorities) / sizeof(nd_log_priorities[0]); + for(size_t i = 0; i < entries ;i++) { + if(priority == nd_log_priorities[i].priority) + return nd_log_priorities[i].name; + } + + return NDLP_INFO_STR; +} + +// -------------------------------------------------------------------------------------------------------------------- +// log sources + +const char *nd_log_sources[] = { + [NDLS_UNSET] = "UNSET", + [NDLS_ACCESS] = "access", + [NDLS_ACLK] = "aclk", + [NDLS_COLLECTORS] = "collector", + [NDLS_DAEMON] = "daemon", + [NDLS_HEALTH] = "health", + [NDLS_DEBUG] = "debug", +}; + +size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def) { + size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]); + for(size_t i = 0; i < entries ;i++) { + if(strcmp(nd_log_sources[i], source) == 0) + return i; + } + + return def; +} + + +const char *nd_log_id2source(ND_LOG_SOURCES source) { + size_t entries = sizeof(nd_log_sources) / sizeof(nd_log_sources[0]); + if(source < entries) + return nd_log_sources[source]; + + return nd_log_sources[NDLS_COLLECTORS]; +} + +// -------------------------------------------------------------------------------------------------------------------- +// log output formats + +static struct { + ND_LOG_FORMAT format; + const char *name; +} nd_log_formats[] = { + { .format = NDLF_JOURNAL, .name = "journal" }, + { .format = NDLF_LOGFMT, .name = "logfmt" }, + { .format = NDLF_JSON, .name = "json" }, +#if defined(OS_WINDOWS) +#if defined(HAVE_ETW) + { .format = NDLF_ETW, .name = ETW_NAME }, +#endif +#if defined(HAVE_WEL) + { .format = NDLF_WEL, .name = WEL_NAME }, +#endif +#endif +}; + +ND_LOG_FORMAT nd_log_format2id(const char *format) { + if(!format || !*format) + return NDLF_LOGFMT; + + size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]); + for(size_t i = 0; i < entries ;i++) { + if(strcmp(nd_log_formats[i].name, format) == 0) + return nd_log_formats[i].format; + } + + return NDLF_LOGFMT; +} + +const char *nd_log_id2format(ND_LOG_FORMAT format) { + size_t entries = sizeof(nd_log_formats) / sizeof(nd_log_formats[0]); + for(size_t i = 0; i < entries ;i++) { + if(format == nd_log_formats[i].format) + return nd_log_formats[i].name; + } + + return "logfmt"; +} + +// -------------------------------------------------------------------------------------------------------------------- + +struct nd_log nd_log = { + .overwrite_process_source = 0, + .journal = { + .initialized = false, + .first_msg = false, + .fd = -1, + }, + .journal_direct = { + .initialized = false, + .fd = -1, + }, + .syslog = { + .initialized = false, + .facility = LOG_DAEMON, + }, +#if defined(OS_WINDOWS) + .eventlog = { + .initialized = false, + }, +#endif + .std_output = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .initialized = false, + }, + .std_error = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .initialized = false, + }, + .sources = { + [NDLS_UNSET] = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .method = NDLM_DISABLED, + .format = NDLF_JOURNAL, + .filename = NULL, + .fd = -1, + .fp = NULL, + .min_priority = NDLP_EMERG, + .limits = ND_LOG_LIMITS_UNLIMITED, + }, + [NDLS_ACCESS] = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .method = NDLM_DEFAULT, + .format = NDLF_LOGFMT, + .filename = LOG_DIR "/access.log", + .fd = -1, + .fp = NULL, + .min_priority = NDLP_DEBUG, + .limits = ND_LOG_LIMITS_UNLIMITED, + }, + [NDLS_ACLK] = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .method = NDLM_FILE, + .format = NDLF_LOGFMT, + .filename = LOG_DIR "/aclk.log", + .fd = -1, + .fp = NULL, + .min_priority = NDLP_DEBUG, + .limits = ND_LOG_LIMITS_UNLIMITED, + }, + [NDLS_COLLECTORS] = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .method = NDLM_DEFAULT, + .format = NDLF_LOGFMT, + .filename = LOG_DIR "/collector.log", + .fd = STDERR_FILENO, + .fp = NULL, + .min_priority = NDLP_INFO, + .limits = ND_LOG_LIMITS_DEFAULT, + }, + [NDLS_DEBUG] = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .method = NDLM_DISABLED, + .format = NDLF_LOGFMT, + .filename = LOG_DIR "/debug.log", + .fd = STDOUT_FILENO, + .fp = NULL, + .min_priority = NDLP_DEBUG, + .limits = ND_LOG_LIMITS_UNLIMITED, + }, + [NDLS_DAEMON] = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .method = NDLM_DEFAULT, + .filename = LOG_DIR "/daemon.log", + .format = NDLF_LOGFMT, + .fd = -1, + .fp = NULL, + .min_priority = NDLP_INFO, + .limits = ND_LOG_LIMITS_DEFAULT, + }, + [NDLS_HEALTH] = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .method = NDLM_DEFAULT, + .format = NDLF_LOGFMT, + .filename = LOG_DIR "/health.log", + .fd = -1, + .fp = NULL, + .min_priority = NDLP_DEBUG, + .limits = ND_LOG_LIMITS_UNLIMITED, + }, + }, +}; + +// -------------------------------------------------------------------------------------------------------------------- + +__thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX]; +__thread size_t thread_log_stack_next = 0; +__thread struct log_field thread_log_fields[_NDF_MAX] = { + // THE ORDER HERE IS IRRELEVANT (but keep them sorted by their number) + + [NDF_STOP] = { // processing will not stop on this - so it is ok to be first + .journal = NULL, + .logfmt = NULL, + .eventlog = NULL, + .annotator = NULL, + }, + [NDF_TIMESTAMP_REALTIME_USEC] = { + .journal = NULL, + .eventlog = "Timestamp", + .logfmt = "time", + .annotator = timestamp_usec_annotator, + }, + [NDF_SYSLOG_IDENTIFIER] = { + .journal = "SYSLOG_IDENTIFIER", // standard journald field + .eventlog = "Program", + .logfmt = "comm", + }, + [NDF_LOG_SOURCE] = { + .journal = "ND_LOG_SOURCE", + .eventlog = "NetdataLogSource", + .logfmt = "source", + }, + [NDF_PRIORITY] = { + .journal = "PRIORITY", // standard journald field + .eventlog = "Level", + .logfmt = "level", + .annotator = priority_annotator, + }, + [NDF_ERRNO] = { + .journal = "ERRNO", // standard journald field + .eventlog = "UnixErrno", + .logfmt = "errno", + .annotator = errno_annotator, + }, + [NDF_WINERROR] = { +#if defined(OS_WINDOWS) + .journal = "WINERROR", + .eventlog = "WindowsLastError", + .logfmt = "winerror", + .annotator = winerror_annotator, +#endif + }, + [NDF_INVOCATION_ID] = { + .journal = "INVOCATION_ID", // standard journald field + .eventlog = "InvocationID", + .logfmt = NULL, + }, + [NDF_LINE] = { + .journal = "CODE_LINE", // standard journald field + .eventlog = "CodeLine", + .logfmt = NULL, + }, + [NDF_FILE] = { + .journal = "CODE_FILE", // standard journald field + .eventlog = "CodeFile", + .logfmt = NULL, + }, + [NDF_FUNC] = { + .journal = "CODE_FUNC", // standard journald field + .eventlog = "CodeFunction", + .logfmt = NULL, + }, + [NDF_TID] = { + .journal = "TID", // standard journald field + .eventlog = "ThreadID", + .logfmt = "tid", + }, + [NDF_THREAD_TAG] = { + .journal = "THREAD_TAG", + .eventlog = "ThreadName", + .logfmt = "thread", + }, + [NDF_MESSAGE_ID] = { + .journal = "MESSAGE_ID", + .eventlog = "MessageID", + .logfmt = "msg_id", + }, + [NDF_MODULE] = { + .journal = "ND_MODULE", + .eventlog = "Module", + .logfmt = "module", + }, + [NDF_NIDL_NODE] = { + .journal = "ND_NIDL_NODE", + .eventlog = "Node", + .logfmt = "node", + }, + [NDF_NIDL_INSTANCE] = { + .journal = "ND_NIDL_INSTANCE", + .eventlog = "Instance", + .logfmt = "instance", + }, + [NDF_NIDL_CONTEXT] = { + .journal = "ND_NIDL_CONTEXT", + .eventlog = "Context", + .logfmt = "context", + }, + [NDF_NIDL_DIMENSION] = { + .journal = "ND_NIDL_DIMENSION", + .eventlog = "Dimension", + .logfmt = "dimension", + }, + [NDF_SRC_TRANSPORT] = { + .journal = "ND_SRC_TRANSPORT", + .eventlog = "SourceTransport", + .logfmt = "src_transport", + }, + [NDF_ACCOUNT_ID] = { + .journal = "ND_ACCOUNT_ID", + .eventlog = "AccountID", + .logfmt = "account", + }, + [NDF_USER_NAME] = { + .journal = "ND_USER_NAME", + .eventlog = "UserName", + .logfmt = "user", + }, + [NDF_USER_ROLE] = { + .journal = "ND_USER_ROLE", + .eventlog = "UserRole", + .logfmt = "role", + }, + [NDF_USER_ACCESS] = { + .journal = "ND_USER_PERMISSIONS", + .eventlog = "UserPermissions", + .logfmt = "permissions", + }, + [NDF_SRC_IP] = { + .journal = "ND_SRC_IP", + .eventlog = "SourceIP", + .logfmt = "src_ip", + }, + [NDF_SRC_FORWARDED_HOST] = { + .journal = "ND_SRC_FORWARDED_HOST", + .eventlog = "SourceForwardedHost", + .logfmt = "src_forwarded_host", + }, + [NDF_SRC_FORWARDED_FOR] = { + .journal = "ND_SRC_FORWARDED_FOR", + .eventlog = "SourceForwardedFor", + .logfmt = "src_forwarded_for", + }, + [NDF_SRC_PORT] = { + .journal = "ND_SRC_PORT", + .eventlog = "SourcePort", + .logfmt = "src_port", + }, + [NDF_SRC_CAPABILITIES] = { + .journal = "ND_SRC_CAPABILITIES", + .eventlog = "SourceCapabilities", + .logfmt = "src_capabilities", + }, + [NDF_DST_TRANSPORT] = { + .journal = "ND_DST_TRANSPORT", + .eventlog = "DestinationTransport", + .logfmt = "dst_transport", + }, + [NDF_DST_IP] = { + .journal = "ND_DST_IP", + .eventlog = "DestinationIP", + .logfmt = "dst_ip", + }, + [NDF_DST_PORT] = { + .journal = "ND_DST_PORT", + .eventlog = "DestinationPort", + .logfmt = "dst_port", + }, + [NDF_DST_CAPABILITIES] = { + .journal = "ND_DST_CAPABILITIES", + .eventlog = "DestinationCapabilities", + .logfmt = "dst_capabilities", + }, + [NDF_REQUEST_METHOD] = { + .journal = "ND_REQUEST_METHOD", + .eventlog = "RequestMethod", + .logfmt = "req_method", + }, + [NDF_RESPONSE_CODE] = { + .journal = "ND_RESPONSE_CODE", + .eventlog = "ResponseCode", + .logfmt = "code", + }, + [NDF_CONNECTION_ID] = { + .journal = "ND_CONNECTION_ID", + .eventlog = "ConnectionID", + .logfmt = "conn", + }, + [NDF_TRANSACTION_ID] = { + .journal = "ND_TRANSACTION_ID", + .eventlog = "TransactionID", + .logfmt = "transaction", + }, + [NDF_RESPONSE_SENT_BYTES] = { + .journal = "ND_RESPONSE_SENT_BYTES", + .eventlog = "ResponseSentBytes", + .logfmt = "sent_bytes", + }, + [NDF_RESPONSE_SIZE_BYTES] = { + .journal = "ND_RESPONSE_SIZE_BYTES", + .eventlog = "ResponseSizeBytes", + .logfmt = "size_bytes", + }, + [NDF_RESPONSE_PREPARATION_TIME_USEC] = { + .journal = "ND_RESPONSE_PREP_TIME_USEC", + .eventlog = "ResponsePreparationTimeUsec", + .logfmt = "prep_ut", + }, + [NDF_RESPONSE_SENT_TIME_USEC] = { + .journal = "ND_RESPONSE_SENT_TIME_USEC", + .eventlog = "ResponseSentTimeUsec", + .logfmt = "sent_ut", + }, + [NDF_RESPONSE_TOTAL_TIME_USEC] = { + .journal = "ND_RESPONSE_TOTAL_TIME_USEC", + .eventlog = "ResponseTotalTimeUsec", + .logfmt = "total_ut", + }, + [NDF_ALERT_ID] = { + .journal = "ND_ALERT_ID", + .eventlog = "AlertID", + .logfmt = "alert_id", + }, + [NDF_ALERT_UNIQUE_ID] = { + .journal = "ND_ALERT_UNIQUE_ID", + .eventlog = "AlertUniqueID", + .logfmt = "alert_unique_id", + }, + [NDF_ALERT_TRANSITION_ID] = { + .journal = "ND_ALERT_TRANSITION_ID", + .eventlog = "AlertTransitionID", + .logfmt = "alert_transition_id", + }, + [NDF_ALERT_EVENT_ID] = { + .journal = "ND_ALERT_EVENT_ID", + .eventlog = "AlertEventID", + .logfmt = "alert_event_id", + }, + [NDF_ALERT_CONFIG_HASH] = { + .journal = "ND_ALERT_CONFIG", + .eventlog = "AlertConfig", + .logfmt = "alert_config", + }, + [NDF_ALERT_NAME] = { + .journal = "ND_ALERT_NAME", + .eventlog = "AlertName", + .logfmt = "alert", + }, + [NDF_ALERT_CLASS] = { + .journal = "ND_ALERT_CLASS", + .eventlog = "AlertClass", + .logfmt = "alert_class", + }, + [NDF_ALERT_COMPONENT] = { + .journal = "ND_ALERT_COMPONENT", + .eventlog = "AlertComponent", + .logfmt = "alert_component", + }, + [NDF_ALERT_TYPE] = { + .journal = "ND_ALERT_TYPE", + .eventlog = "AlertType", + .logfmt = "alert_type", + }, + [NDF_ALERT_EXEC] = { + .journal = "ND_ALERT_EXEC", + .eventlog = "AlertExec", + .logfmt = "alert_exec", + }, + [NDF_ALERT_RECIPIENT] = { + .journal = "ND_ALERT_RECIPIENT", + .eventlog = "AlertRecipient", + .logfmt = "alert_recipient", + }, + [NDF_ALERT_VALUE] = { + .journal = "ND_ALERT_VALUE", + .eventlog = "AlertValue", + .logfmt = "alert_value", + }, + [NDF_ALERT_VALUE_OLD] = { + .journal = "ND_ALERT_VALUE_OLD", + .eventlog = "AlertOldValue", + .logfmt = "alert_value_old", + }, + [NDF_ALERT_STATUS] = { + .journal = "ND_ALERT_STATUS", + .eventlog = "AlertStatus", + .logfmt = "alert_status", + }, + [NDF_ALERT_STATUS_OLD] = { + .journal = "ND_ALERT_STATUS_OLD", + .eventlog = "AlertOldStatus", + .logfmt = "alert_value_old", + }, + [NDF_ALERT_UNITS] = { + .journal = "ND_ALERT_UNITS", + .eventlog = "AlertUnits", + .logfmt = "alert_units", + }, + [NDF_ALERT_SUMMARY] = { + .journal = "ND_ALERT_SUMMARY", + .eventlog = "AlertSummary", + .logfmt = "alert_summary", + }, + [NDF_ALERT_INFO] = { + .journal = "ND_ALERT_INFO", + .eventlog = "AlertInfo", + .logfmt = "alert_info", + }, + [NDF_ALERT_DURATION] = { + .journal = "ND_ALERT_DURATION", + .eventlog = "AlertDuration", + .logfmt = "alert_duration", + }, + [NDF_ALERT_NOTIFICATION_REALTIME_USEC] = { + .journal = "ND_ALERT_NOTIFICATION_TIMESTAMP_USEC", + .eventlog = "AlertNotificationTime", + .logfmt = "alert_notification_timestamp", + .annotator = timestamp_usec_annotator, + }, + + // put new items here + // leave the request URL and the message last + + [NDF_REQUEST] = { + .journal = "ND_REQUEST", + .eventlog = "Request", + .logfmt = "request", + }, + [NDF_MESSAGE] = { + .journal = "MESSAGE", + .eventlog = "Message", + .logfmt = "msg", + }, +}; + +// -------------------------------------------------------------------------------------------------------------------- + +void log_stack_pop(void *ptr) { + if(!ptr) return; + + struct log_stack_entry *lgs = *(struct log_stack_entry (*)[])ptr; + + if(unlikely(!thread_log_stack_next || lgs != thread_log_stack_base[thread_log_stack_next - 1])) { + fatal("You cannot pop in the middle of the stack, or an item not in the stack"); + return; + } + + thread_log_stack_next--; +} + +void log_stack_push(struct log_stack_entry *lgs) { + if(!lgs || thread_log_stack_next >= THREAD_LOG_STACK_MAX) return; + thread_log_stack_base[thread_log_stack_next++] = lgs; +} + +// -------------------------------------------------------------------------------------------------------------------- + +ND_LOG_FIELD_ID nd_log_field_id_by_journal_name(const char *field, size_t len) { + for(size_t i = 0; i < THREAD_FIELDS_MAX ;i++) { + if(thread_log_fields[i].journal && strlen(thread_log_fields[i].journal) == len && strncmp(field, thread_log_fields[i].journal, len) == 0) + return i; + } + + return NDF_STOP; +} + +// -------------------------------------------------------------------------------------------------------------------- + +int nd_log_health_fd(void) { + if(nd_log.sources[NDLS_HEALTH].method == NDLM_FILE && nd_log.sources[NDLS_HEALTH].fd != -1) + return nd_log.sources[NDLS_HEALTH].fd; + + return STDERR_FILENO; +} + +int nd_log_collectors_fd(void) { + if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_FILE && nd_log.sources[NDLS_COLLECTORS].fd != -1) + return nd_log.sources[NDLS_COLLECTORS].fd; + + return STDERR_FILENO; +} + +// -------------------------------------------------------------------------------------------------------------------- + +void log_date(char *buffer, size_t len, time_t now) { + if(unlikely(!buffer || !len)) + return; + + time_t t = now; + struct tm *tmp, tmbuf; + + tmp = localtime_r(&t, &tmbuf); + + if (unlikely(!tmp)) { + buffer[0] = '\0'; + return; + } + + if (unlikely(strftime(buffer, len, "%Y-%m-%d %H:%M:%S", tmp) == 0)) + buffer[0] = '\0'; + + buffer[len - 1] = '\0'; +} + +// -------------------------------------------------------------------------------------------------------------------- + +bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd) { + if(new_fd == -1 || e->fd == -1 || + (e->fd == STDOUT_FILENO && nd_log.std_output.initialized) || + (e->fd == STDERR_FILENO && nd_log.std_error.initialized)) + return false; + + if(new_fd != e->fd) { + int t = dup2(new_fd, e->fd); + + bool ret = true; + if (t == -1) { + netdata_log_error("Cannot dup2() new fd %d to old fd %d for '%s'", new_fd, e->fd, e->filename); + ret = false; + } + else + close(new_fd); + + if(e->fd == STDOUT_FILENO) + nd_log.std_output.initialized = true; + else if(e->fd == STDERR_FILENO) + nd_log.std_error.initialized = true; + + return ret; + } + + return false; +} diff --git a/src/libnetdata/log/nd_log-internals.h b/src/libnetdata/log/nd_log-internals.h new file mode 100644 index 000000000..7bebf5a4a --- /dev/null +++ b/src/libnetdata/log/nd_log-internals.h @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_ND_LOG_INTERNALS_H +#define NETDATA_ND_LOG_INTERNALS_H + +#include "../libnetdata.h" + +#ifdef __FreeBSD__ +#include <sys/endian.h> +#endif + +#ifdef __APPLE__ +#include <machine/endian.h> +#endif + +#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE) +#include <execinfo.h> +#endif + +#ifdef HAVE_SYSTEMD +#include <systemd/sd-journal.h> +#endif + +const char *errno2str(int errnum, char *buf, size_t size); + +// -------------------------------------------------------------------------------------------------------------------- +// ND_LOG_METHOD + +typedef enum __attribute__((__packed__)) { + NDLM_DISABLED = 0, + NDLM_DEVNULL, + NDLM_DEFAULT, + NDLM_JOURNAL, + NDLM_SYSLOG, + NDLM_STDOUT, + NDLM_STDERR, + NDLM_FILE, +#if defined(OS_WINDOWS) +#if defined(HAVE_ETW) + NDLM_ETW, +#endif +#if defined(HAVE_WEL) + NDLM_WEL, +#endif +#endif +} ND_LOG_METHOD; + +// all the log methods are finally mapped to these +#if defined(HAVE_ETW) +#define ETW_CONDITION(ndlo) ((ndlo) == NDLM_ETW) +#else +#define ETW_CONDITION(ndlo) (false) +#endif + +#if defined(HAVE_WEL) +#define WEL_CONDITION(ndlo) ((ndlo) == NDLM_WEL) +#else +#define WEL_CONDITION(ndlo) (false) +#endif + +#define IS_VALID_LOG_METHOD_FOR_EXTERNAL_PLUGINS(ndlo) ((ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || (ndlo) == NDLM_STDERR || ETW_CONDITION(ndlo) || WEL_CONDITION(ndlo)) +#define IS_FINAL_LOG_METHOD(ndlo) ((ndlo) == NDLM_FILE || (ndlo) == NDLM_JOURNAL || (ndlo) == NDLM_SYSLOG || ETW_CONDITION(ndlo) || WEL_CONDITION(ndlo)) + +ND_LOG_METHOD nd_log_method2id(const char *method); +const char *nd_log_id2method(ND_LOG_METHOD method); + +// -------------------------------------------------------------------------------------------------------------------- +// ND_LOG_FORMAT + +typedef enum __attribute__((__packed__)) { + NDLF_JOURNAL, + NDLF_LOGFMT, + NDLF_JSON, +#if defined(OS_WINDOWS) +#if defined(HAVE_ETW) + NDLF_ETW, // Event Tracing for Windows +#endif +#if defined(HAVE_WEL) + NDLF_WEL, // Windows Event Log +#endif +#endif +} ND_LOG_FORMAT; + +#define ETW_NAME "etw" +#define WEL_NAME "wel" + +const char *nd_log_id2format(ND_LOG_FORMAT format); +ND_LOG_FORMAT nd_log_format2id(const char *format); + +size_t nd_log_source2id(const char *source, ND_LOG_SOURCES def); +const char *nd_log_id2source(ND_LOG_SOURCES source); + +const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority); +int nd_log_priority2id(const char *priority); + +const char *nd_log_id2facility(int facility); +int nd_log_facility2id(const char *facility); + +#include "nd_log_limit.h" + +struct nd_log_source { + SPINLOCK spinlock; + ND_LOG_METHOD method; + ND_LOG_FORMAT format; + const char *filename; + int fd; + FILE *fp; + + ND_LOG_FIELD_PRIORITY min_priority; + const char *pending_msg; + struct nd_log_limit limits; + +#if defined(OS_WINDOWS) + ND_LOG_SOURCES source; + HANDLE hEventLog; + USHORT channelID; + UCHAR Opcode; + USHORT Task; + ULONGLONG Keyword; +#endif +}; + +struct nd_log { + nd_uuid_t invocation_id; + + ND_LOG_SOURCES overwrite_process_source; + + struct nd_log_source sources[_NDLS_MAX]; + + struct { + bool initialized; + bool first_msg; + int fd; // we don't control this, we just detect it to keep it open + } journal; + + struct { + bool initialized; + int fd; + char filename[FILENAME_MAX]; + } journal_direct; + + struct { + bool initialized; + int facility; + } syslog; + + struct { + bool etw; // when set use etw, otherwise wel + bool initialized; + } eventlog; + + struct { + SPINLOCK spinlock; + bool initialized; + } std_output; + + struct { + SPINLOCK spinlock; + bool initialized; + } std_error; + +}; + +// -------------------------------------------------------------------------------------------------------------------- + +struct log_field; +typedef const char *(*annotator_t)(struct log_field *lf); + +struct log_field { + const char *journal; + const char *logfmt; + const char *eventlog; + annotator_t annotator; + struct log_stack_entry entry; +}; + +#define THREAD_LOG_STACK_MAX 50 +#define THREAD_FIELDS_MAX (sizeof(thread_log_fields) / sizeof(thread_log_fields[0])) + +extern __thread struct log_stack_entry *thread_log_stack_base[THREAD_LOG_STACK_MAX]; +extern __thread size_t thread_log_stack_next; +extern __thread struct log_field thread_log_fields[_NDF_MAX]; + +// -------------------------------------------------------------------------------------------------------------------- + +extern struct nd_log nd_log; +bool nd_log_replace_existing_fd(struct nd_log_source *e, int new_fd); +void nd_log_open(struct nd_log_source *e, ND_LOG_SOURCES source); +void nd_log_stdin_init(int fd, const char *filename); + +// -------------------------------------------------------------------------------------------------------------------- +// annotators + +struct log_field; +const char *errno_annotator(struct log_field *lf); +const char *priority_annotator(struct log_field *lf); +const char *timestamp_usec_annotator(struct log_field *lf); + +#if defined(OS_WINDOWS) +const char *winerror_annotator(struct log_field *lf); +#endif + +// -------------------------------------------------------------------------------------------------------------------- +// field formatters + +uint64_t log_field_to_uint64(struct log_field *lf); +int64_t log_field_to_int64(struct log_field *lf); + +// -------------------------------------------------------------------------------------------------------------------- +// common text formatters + +void nd_logger_logfmt(BUFFER *wb, struct log_field *fields, size_t fields_max); +void nd_logger_json(BUFFER *wb, struct log_field *fields, size_t fields_max); + +// -------------------------------------------------------------------------------------------------------------------- +// output to syslog + +void nd_log_init_syslog(void); +void nd_log_reset_syslog(void); +bool nd_logger_syslog(int priority, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max); + +// -------------------------------------------------------------------------------------------------------------------- +// output to systemd-journal + +bool nd_log_journal_systemd_init(void); +bool nd_log_journal_direct_init(const char *path); +bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max); +bool nd_logger_journal_libsystemd(struct log_field *fields, size_t fields_max); + +// -------------------------------------------------------------------------------------------------------------------- +// output to file + +bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max); + +// -------------------------------------------------------------------------------------------------------------------- +// output to windows events log + +#if defined(OS_WINDOWS) +#if defined(HAVE_ETW) +bool nd_log_init_etw(void); +bool nd_logger_etw(struct nd_log_source *source, struct log_field *fields, size_t fields_max); +#endif +#if defined(HAVE_WEL) +bool nd_log_init_wel(void); +bool nd_logger_wel(struct nd_log_source *source, struct log_field *fields, size_t fields_max); +#endif +#endif + +#endif //NETDATA_ND_LOG_INTERNALS_H diff --git a/src/libnetdata/log/nd_log-to-file.c b/src/libnetdata/log/nd_log-to-file.c new file mode 100644 index 000000000..2de76536b --- /dev/null +++ b/src/libnetdata/log/nd_log-to-file.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log-internals.h" + +void chown_open_file(int fd, uid_t uid, gid_t gid) { + if(fd == -1) return; + + struct stat buf; + + if(fstat(fd, &buf) == -1) { + netdata_log_error("Cannot fstat() fd %d", fd); + return; + } + + if((buf.st_uid != uid || buf.st_gid != gid) && S_ISREG(buf.st_mode)) { + if(fchown(fd, uid, gid) == -1) + netdata_log_error("Cannot fchown() fd %d.", fd); + } +} + +void nd_log_chown_log_files(uid_t uid, gid_t gid) { + for(size_t i = 0 ; i < _NDLS_MAX ; i++) { + if(nd_log.sources[i].fd != -1 && nd_log.sources[i].fd != STDIN_FILENO) + chown_open_file(nd_log.sources[i].fd, uid, gid); + } +} + +bool nd_logger_file(FILE *fp, ND_LOG_FORMAT format, struct log_field *fields, size_t fields_max) { + BUFFER *wb = buffer_create(1024, NULL); + + if(format == NDLF_JSON) + nd_logger_json(wb, fields, fields_max); + else + nd_logger_logfmt(wb, fields, fields_max); + + int r = fprintf(fp, "%s\n", buffer_tostring(wb)); + fflush(fp); + + buffer_free(wb); + return r > 0; +} diff --git a/src/libnetdata/log/nd_log-to-syslog.c b/src/libnetdata/log/nd_log-to-syslog.c new file mode 100644 index 000000000..2903bf591 --- /dev/null +++ b/src/libnetdata/log/nd_log-to-syslog.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log-internals.h" + +void nd_log_init_syslog(void) { + if(nd_log.syslog.initialized) + return; + + openlog(program_name, LOG_PID, nd_log.syslog.facility); + nd_log.syslog.initialized = true; +} + +bool nd_logger_syslog(int priority, ND_LOG_FORMAT format __maybe_unused, struct log_field *fields, size_t fields_max) { + CLEAN_BUFFER *wb = buffer_create(1024, NULL); + + nd_logger_logfmt(wb, fields, fields_max); + syslog(priority, "%s", buffer_tostring(wb)); + + return true; +} diff --git a/src/libnetdata/log/nd_log-to-systemd-journal.c b/src/libnetdata/log/nd_log-to-systemd-journal.c new file mode 100644 index 000000000..922427777 --- /dev/null +++ b/src/libnetdata/log/nd_log-to-systemd-journal.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log-internals.h" + +bool nd_log_journal_systemd_init(void) { +#ifdef HAVE_SYSTEMD + nd_log.journal.initialized = true; +#else + nd_log.journal.initialized = false; +#endif + + return nd_log.journal.initialized; +} + +static int nd_log_journal_direct_fd_find_and_open(char *filename, size_t size) { + int fd; + + if(netdata_configured_host_prefix && *netdata_configured_host_prefix) { + journal_construct_path(filename, size, netdata_configured_host_prefix, "netdata"); + if (is_path_unix_socket(filename) && (fd = journal_direct_fd(filename)) != -1) + return fd; + + journal_construct_path(filename, size, netdata_configured_host_prefix, NULL); + if (is_path_unix_socket(filename) && (fd = journal_direct_fd(filename)) != -1) + return fd; + } + + journal_construct_path(filename, size, NULL, "netdata"); + if (is_path_unix_socket(filename) && (fd = journal_direct_fd(filename)) != -1) + return fd; + + journal_construct_path(filename, size, NULL, NULL); + if (is_path_unix_socket(filename) && (fd = journal_direct_fd(filename)) != -1) + return fd; + + return -1; +} + +bool nd_log_journal_socket_available(void) { + char filename[FILENAME_MAX]; + int fd = nd_log_journal_direct_fd_find_and_open(filename, sizeof(filename)); + if(fd == -1) return false; + close(fd); + return true; +} + +static void nd_log_journal_direct_set_env(void) { + if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_JOURNAL) + nd_setenv("NETDATA_SYSTEMD_JOURNAL_PATH", nd_log.journal_direct.filename, 1); +} + +bool nd_log_journal_direct_init(const char *path) { + if(nd_log.journal_direct.initialized) { + nd_log_journal_direct_set_env(); + return true; + } + + int fd; + char filename[FILENAME_MAX]; + if(!is_path_unix_socket(path)) + fd = nd_log_journal_direct_fd_find_and_open(filename, sizeof(filename)); + else { + snprintfz(filename, sizeof(filename), "%s", path); + fd = journal_direct_fd(filename); + } + + if(fd < 0) + return false; + + nd_log.journal_direct.fd = fd; + nd_log.journal_direct.initialized = true; + + strncpyz(nd_log.journal_direct.filename, filename, sizeof(nd_log.journal_direct.filename) - 1); + nd_log_journal_direct_set_env(); + + return true; +} + +bool nd_logger_journal_libsystemd(struct log_field *fields __maybe_unused, size_t fields_max __maybe_unused) { +#ifdef HAVE_SYSTEMD + + // --- FIELD_PARSER_VERSIONS --- + // + // IMPORTANT: + // THERE ARE 6 VERSIONS OF THIS CODE + // + // 1. journal (direct socket API), + // 2. journal (libsystemd API), + // 3. logfmt, + // 4. json, + // 5. convert to uint64 + // 6. convert to int64 + // + // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES + + struct iovec iov[fields_max]; + int iov_count = 0; + + memset(iov, 0, sizeof(iov)); + + CLEAN_BUFFER *tmp = NULL; + + for (size_t i = 0; i < fields_max; i++) { + if (!fields[i].entry.set || !fields[i].journal) + continue; + + const char *key = fields[i].journal; + char *value = NULL; + int rc = 0; + switch (fields[i].entry.type) { + case NDFT_TXT: + if(*fields[i].entry.txt) + rc = asprintf(&value, "%s=%s", key, fields[i].entry.txt); + break; + case NDFT_STR: + rc = asprintf(&value, "%s=%s", key, string2str(fields[i].entry.str)); + break; + case NDFT_BFR: + if(buffer_strlen(fields[i].entry.bfr)) + rc = asprintf(&value, "%s=%s", key, buffer_tostring(fields[i].entry.bfr)); + break; + case NDFT_U64: + rc = asprintf(&value, "%s=%" PRIu64, key, fields[i].entry.u64); + break; + case NDFT_I64: + rc = asprintf(&value, "%s=%" PRId64, key, fields[i].entry.i64); + break; + case NDFT_DBL: + rc = asprintf(&value, "%s=%f", key, fields[i].entry.dbl); + break; + case NDFT_UUID: + if(!uuid_is_null(*fields[i].entry.uuid)) { + char u[UUID_COMPACT_STR_LEN]; + uuid_unparse_lower_compact(*fields[i].entry.uuid, u); + rc = asprintf(&value, "%s=%s", key, u); + } + break; + case NDFT_CALLBACK: { + if(!tmp) + tmp = buffer_create(1024, NULL); + else + buffer_flush(tmp); + if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) + rc = asprintf(&value, "%s=%s", key, buffer_tostring(tmp)); + } + break; + default: + rc = asprintf(&value, "%s=%s", key, "UNHANDLED"); + break; + } + + if (rc != -1 && value) { + iov[iov_count].iov_base = value; + iov[iov_count].iov_len = strlen(value); + iov_count++; + } + } + + static bool sockets_before[1024]; + bool detect_systemd_socket = __atomic_load_n(&nd_log.journal.first_msg, __ATOMIC_RELAXED) == false; + if(detect_systemd_socket) { + for(int i = 3 ; (size_t)i < _countof(sockets_before); i++) + sockets_before[i] = fd_is_socket(i); + } + + int r = sd_journal_sendv(iov, iov_count); + + if(r == 0 && detect_systemd_socket) { + __atomic_store_n(&nd_log.journal.first_msg, true, __ATOMIC_RELAXED); + + // this is the first successful libsystemd log + // let's detect its fd number (we need it for the spawn server) + + for(int i = 3 ; (size_t)i < _countof(sockets_before); i++) { + if (!sockets_before[i] && fd_is_socket(i)) { + nd_log.journal.fd = i; + break; + } + } + } + + // Clean up allocated memory + for (int i = 0; i < iov_count; i++) { + if (iov[i].iov_base != NULL) { + free(iov[i].iov_base); + } + } + + return r == 0; +#else + return false; +#endif +} + +bool nd_logger_journal_direct(struct log_field *fields, size_t fields_max) { + if(!nd_log.journal_direct.initialized) + return false; + + // --- FIELD_PARSER_VERSIONS --- + // + // IMPORTANT: + // THERE ARE 6 VERSIONS OF THIS CODE + // + // 1. journal (direct socket API), + // 2. journal (libsystemd API), + // 3. logfmt, + // 4. json, + // 5. convert to uint64 + // 6. convert to int64 + // + // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES + + CLEAN_BUFFER *wb = buffer_create(4096, NULL); + CLEAN_BUFFER *tmp = NULL; + + for (size_t i = 0; i < fields_max; i++) { + if (!fields[i].entry.set || !fields[i].journal) + continue; + + const char *key = fields[i].journal; + + const char *s = NULL; + switch(fields[i].entry.type) { + case NDFT_TXT: + s = fields[i].entry.txt; + break; + case NDFT_STR: + s = string2str(fields[i].entry.str); + break; + case NDFT_BFR: + s = buffer_tostring(fields[i].entry.bfr); + break; + case NDFT_U64: + buffer_strcat(wb, key); + buffer_putc(wb, '='); + buffer_print_uint64(wb, fields[i].entry.u64); + buffer_putc(wb, '\n'); + break; + case NDFT_I64: + buffer_strcat(wb, key); + buffer_putc(wb, '='); + buffer_print_int64(wb, fields[i].entry.i64); + buffer_putc(wb, '\n'); + break; + case NDFT_DBL: + buffer_strcat(wb, key); + buffer_putc(wb, '='); + buffer_print_netdata_double(wb, fields[i].entry.dbl); + buffer_putc(wb, '\n'); + break; + case NDFT_UUID: + if(!uuid_is_null(*fields[i].entry.uuid)) { + char u[UUID_COMPACT_STR_LEN]; + uuid_unparse_lower_compact(*fields[i].entry.uuid, u); + buffer_strcat(wb, key); + buffer_putc(wb, '='); + buffer_fast_strcat(wb, u, sizeof(u) - 1); + buffer_putc(wb, '\n'); + } + break; + case NDFT_CALLBACK: { + if(!tmp) + tmp = buffer_create(1024, NULL); + else + buffer_flush(tmp); + if(fields[i].entry.cb.formatter(tmp, fields[i].entry.cb.formatter_data)) + s = buffer_tostring(tmp); + else + s = NULL; + } + break; + default: + s = "UNHANDLED"; + break; + } + + if(s && *s) { + buffer_strcat(wb, key); + if(!strchr(s, '\n')) { + buffer_putc(wb, '='); + buffer_strcat(wb, s); + buffer_putc(wb, '\n'); + } + else { + buffer_putc(wb, '\n'); + size_t size = strlen(s); + uint64_t le_size = htole64(size); + buffer_memcat(wb, &le_size, sizeof(le_size)); + buffer_memcat(wb, s, size); + buffer_putc(wb, '\n'); + } + } + } + + return journal_direct_send(nd_log.journal_direct.fd, buffer_tostring(wb), buffer_strlen(wb)); +} diff --git a/src/libnetdata/log/nd_log-to-windows-common.h b/src/libnetdata/log/nd_log-to-windows-common.h new file mode 100644 index 000000000..2b2833ed1 --- /dev/null +++ b/src/libnetdata/log/nd_log-to-windows-common.h @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_ND_LOG_TO_WINDOWS_COMMON_H +#define NETDATA_ND_LOG_TO_WINDOWS_COMMON_H + +// Helper macro to create wide string literals +#define WIDEN2(x) L ## x +#define WIDEN(x) WIDEN2(x) + +#define NETDATA_ETW_PROVIDER_GUID_STR "{96c5ca72-9bd8-4634-81e5-000014e7da7a}" +#define NETDATA_ETW_PROVIDER_GUID_STR_W WIDEN(NETDATA_ETW_PROVIDER_GUID) + +#define NETDATA_CHANNEL_NAME "Netdata" +#define NETDATA_CHANNEL_NAME_W WIDEN(NETDATA_CHANNEL_NAME) + +#define NETDATA_WEL_CHANNEL_NAME "NetdataWEL" +#define NETDATA_WEL_CHANNEL_NAME_W WIDEN(NETDATA_WEL_CHANNEL_NAME) + +#define NETDATA_ETW_CHANNEL_NAME "Netdata" +#define NETDATA_ETW_CHANNEL_NAME_W WIDEN(NETDATA_ETW_CHANNEL_NAME) + +#define NETDATA_ETW_PROVIDER_NAME "Netdata" +#define NETDATA_ETW_PROVIDER_NAME_W WIDEN(NETDATA_ETW_PROVIDER_NAME) + +#define NETDATA_WEL_PROVIDER_PREFIX "Netdata" +#define NETDATA_WEL_PROVIDER_PREFIX_W WIDEN(NETDATA_WEL_PROVIDER_PREFIX) + +#define NETDATA_WEL_PROVIDER_ACCESS NETDATA_WEL_PROVIDER_PREFIX "Access" +#define NETDATA_WEL_PROVIDER_ACCESS_W WIDEN(NETDATA_WEL_PROVIDER_ACCESS) + +#define NETDATA_WEL_PROVIDER_ACLK NETDATA_WEL_PROVIDER_PREFIX "Aclk" +#define NETDATA_WEL_PROVIDER_ACLK_W WIDEN(NETDATA_WEL_PROVIDER_ACLK) + +#define NETDATA_WEL_PROVIDER_COLLECTORS NETDATA_WEL_PROVIDER_PREFIX "Collectors" +#define NETDATA_WEL_PROVIDER_COLLECTORS_W WIDEN(NETDATA_WEL_PROVIDER_COLLECTORS) + +#define NETDATA_WEL_PROVIDER_DAEMON NETDATA_WEL_PROVIDER_PREFIX "Daemon" +#define NETDATA_WEL_PROVIDER_DAEMON_W WIDEN(NETDATA_WEL_PROVIDER_DAEMON) + +#define NETDATA_WEL_PROVIDER_HEALTH NETDATA_WEL_PROVIDER_PREFIX "Health" +#define NETDATA_WEL_PROVIDER_HEALTH_W WIDEN(NETDATA_WEL_PROVIDER_HEALTH) + + +#define NETDATA_ETW_SUBCHANNEL_ACCESS "Access" +#define NETDATA_ETW_SUBCHANNEL_ACCESS_W WIDEN(NETDATA_ETW_SUBCHANNEL_ACCESS) + +#define NETDATA_ETW_SUBCHANNEL_ACLK "Aclk" +#define NETDATA_ETW_SUBCHANNEL_ACLK_W WIDEN(NETDATA_ETW_SUBCHANNEL_ACLK) + +#define NETDATA_ETW_SUBCHANNEL_COLLECTORS "Collectors" +#define NETDATA_ETW_SUBCHANNEL_COLLECTORS_W WIDEN(NETDATA_ETW_SUBCHANNEL_COLLECTORS) + +#define NETDATA_ETW_SUBCHANNEL_DAEMON "Daemon" +#define NETDATA_ETW_SUBCHANNEL_DAEMON_W WIDEN(NETDATA_ETW_SUBCHANNEL_DAEMON) + +#define NETDATA_ETW_SUBCHANNEL_HEALTH "Health" +#define NETDATA_ETW_SUBCHANNEL_HEALTH_W WIDEN(NETDATA_ETW_SUBCHANNEL_HEALTH) + +// Define shift values +#define EVENT_ID_SEV_SHIFT 30 +#define EVENT_ID_C_SHIFT 29 +#define EVENT_ID_R_SHIFT 28 +#define EVENT_ID_FACILITY_SHIFT 16 +#define EVENT_ID_CODE_SHIFT 0 + +#define EVENT_ID_PRIORITY_SHIFT 0 // Shift 0 bits +#define EVENT_ID_SOURCE_SHIFT 4 // Shift 4 bits + +// Define masks +#define EVENT_ID_SEV_MASK 0xC0000000 // Bits 31-30 +#define EVENT_ID_C_MASK 0x20000000 // Bit 29 +#define EVENT_ID_R_MASK 0x10000000 // Bit 28 +#define EVENT_ID_FACILITY_MASK 0x0FFF0000 // Bits 27-16 +#define EVENT_ID_CODE_MASK 0x0000FFFF // Bits 15-0 + +#define EVENT_ID_PRIORITY_MASK 0x000F // Bits 0-3 +#define EVENT_ID_SOURCE_MASK 0x00F0 // Bits 4-7 + +typedef enum __attribute__((packed)) { + MSGID_MESSAGE_ONLY = 1, + MSGID_MESSAGE_ERRNO, + MSGID_REQUEST_ONLY, + MSGID_ALERT_TRANSITION, + MSGID_ACCESS, + MSGID_ACCESS_FORWARDER, + MSGID_ACCESS_USER, + MSGID_ACCESS_FORWARDER_USER, + MSGID_ACCESS_MESSAGE, + MSGID_ACCESS_MESSAGE_REQUEST, + MSGID_ACCESS_MESSAGE_USER, + + // terminator + _MSGID_MAX, +} MESSAGE_ID; + +static inline uint32_t get_event_type_from_priority(ND_LOG_FIELD_PRIORITY priority) { + switch (priority) { + case NDLP_EMERG: + case NDLP_ALERT: + case NDLP_CRIT: + case NDLP_ERR: + return EVENTLOG_ERROR_TYPE; + + case NDLP_WARNING: + return EVENTLOG_WARNING_TYPE; + + case NDLP_NOTICE: + case NDLP_INFO: + case NDLP_DEBUG: + default: + return EVENTLOG_INFORMATION_TYPE; + } +} + +static inline uint8_t get_severity_from_priority(ND_LOG_FIELD_PRIORITY priority) { + switch (priority) { + case NDLP_EMERG: + case NDLP_ALERT: + case NDLP_CRIT: + case NDLP_ERR: + return STATUS_SEVERITY_ERROR; + + case NDLP_WARNING: + return STATUS_SEVERITY_WARNING; + + case NDLP_NOTICE: + case NDLP_INFO: + case NDLP_DEBUG: + default: + return STATUS_SEVERITY_INFORMATIONAL; + } +} + +static inline uint8_t get_level_from_priority(ND_LOG_FIELD_PRIORITY priority) { + switch (priority) { + // return 0 = log an event regardless of any filtering applied + + case NDLP_EMERG: + case NDLP_ALERT: + case NDLP_CRIT: + return 1; + + case NDLP_ERR: + return 2; + + case NDLP_WARNING: + return 3; + + case NDLP_NOTICE: + case NDLP_INFO: + return 4; + + case NDLP_DEBUG: + default: + return 5; + } +} + +static inline const char *get_level_from_priority_str(ND_LOG_FIELD_PRIORITY priority) { + switch (priority) { + // return "win:LogAlways" to log an event regardless of any filtering applied + + case NDLP_EMERG: + case NDLP_ALERT: + case NDLP_CRIT: + return "win:Critical"; + + case NDLP_ERR: + return "win:Error"; + + case NDLP_WARNING: + return "win:Warning"; + + case NDLP_NOTICE: + case NDLP_INFO: + return "win:Informational"; + + case NDLP_DEBUG: + default: + return "win:Verbose"; + } +} + +static inline uint16_t construct_event_code(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, MESSAGE_ID messageID) { + return (source << 12 | priority << 8 | messageID << 0); +} + +#endif //NETDATA_ND_LOG_TO_WINDOWS_COMMON_H diff --git a/src/libnetdata/log/nd_log-to-windows-events.c b/src/libnetdata/log/nd_log-to-windows-events.c new file mode 100644 index 000000000..f32289daa --- /dev/null +++ b/src/libnetdata/log/nd_log-to-windows-events.c @@ -0,0 +1,554 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log-internals.h" + +#if defined(OS_WINDOWS) && (defined(HAVE_ETW) || defined(HAVE_WEL)) + +// -------------------------------------------------------------------------------------------------------------------- +// construct an event id + +// load message resources generated header +#include "wevt_netdata.h" + +// include the common definitions with the message resources and manifest generator +#include "nd_log-to-windows-common.h" + +#if defined(HAVE_ETW) +// we need the manifest, only in ETW mode + +// eliminate compiler warnings and load manifest generated header +#undef EXTERN_C +#define EXTERN_C +#undef __declspec +#define __declspec(x) +#include "wevt_netdata_manifest.h" + +static REGHANDLE regHandle; +#endif + +// Function to construct EventID +static DWORD complete_event_id(DWORD facility, DWORD severity, DWORD event_code) { + DWORD event_id = 0; + + // Set Severity + event_id |= ((DWORD)(severity) << EVENT_ID_SEV_SHIFT) & EVENT_ID_SEV_MASK; + + // Set Customer Code Flag (C) + event_id |= (0x0 << EVENT_ID_C_SHIFT) & EVENT_ID_C_MASK; + + // Set Reserved Bit (R) - typically 0 + event_id |= (0x0 << EVENT_ID_R_SHIFT) & EVENT_ID_R_MASK; + + // Set Facility + event_id |= ((DWORD)(facility) << EVENT_ID_FACILITY_SHIFT) & EVENT_ID_FACILITY_MASK; + + // Set Code + event_id |= ((DWORD)(event_code) << EVENT_ID_CODE_SHIFT) & EVENT_ID_CODE_MASK; + + return event_id; +} + +DWORD construct_event_id(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, MESSAGE_ID messageID) { + DWORD event_code = construct_event_code(source, priority, messageID); + return complete_event_id(FACILITY_NETDATA, get_severity_from_priority(priority), event_code); +} + +static bool check_event_id(ND_LOG_SOURCES source __maybe_unused, ND_LOG_FIELD_PRIORITY priority __maybe_unused, MESSAGE_ID messageID __maybe_unused, DWORD event_code __maybe_unused) { +#ifdef NETDATA_INTERNAL_CHECKS + DWORD generated = construct_event_id(source, priority, messageID); + if(generated != event_code) { + + // this is just used for a break point, to see the values in hex + char current[UINT64_HEX_MAX_LENGTH]; + print_uint64_hex(current, generated); + + char wanted[UINT64_HEX_MAX_LENGTH]; + print_uint64_hex(wanted, event_code); + + const char *got = current; + const char *good = wanted; + internal_fatal(true, "EventIDs mismatch, expected %s, got %s", good, got); + } +#endif + + return true; +} + +// -------------------------------------------------------------------------------------------------------------------- +// initialization + +// Define provider names per source (only when not using ETW) +static const wchar_t *wel_provider_per_source[_NDLS_MAX] = { + [NDLS_UNSET] = NULL, // not used, linked to NDLS_DAEMON + [NDLS_ACCESS] = NETDATA_WEL_PROVIDER_ACCESS_W, // + [NDLS_ACLK] = NETDATA_WEL_PROVIDER_ACLK_W, // + [NDLS_COLLECTORS] = NETDATA_WEL_PROVIDER_COLLECTORS_W,// + [NDLS_DAEMON] = NETDATA_WEL_PROVIDER_DAEMON_W, // + [NDLS_HEALTH] = NETDATA_WEL_PROVIDER_HEALTH_W, // + [NDLS_DEBUG] = NULL, // used, linked to NDLS_DAEMON +}; + +bool wel_replace_program_with_wevt_netdata_dll(wchar_t *str, size_t size) { + const wchar_t *replacement = L"\\wevt_netdata.dll"; + + // Find the last occurrence of '\\' to isolate the filename + wchar_t *lastBackslash = wcsrchr(str, L'\\'); + + if (lastBackslash != NULL) { + // Calculate new length after replacement + size_t newLen = (lastBackslash - str) + wcslen(replacement); + + // Ensure new length does not exceed buffer size + if (newLen >= size) + return false; // Not enough space in the buffer + + // Terminate the string at the last backslash + *lastBackslash = L'\0'; + + // Append the replacement filename + wcsncat(str, replacement, size - wcslen(str) - 1); + + // Check if the new file exists + if (GetFileAttributesW(str) != INVALID_FILE_ATTRIBUTES) + return true; // The file exists + else + return false; // The file does not exist + } + + return false; // No backslash found (likely invalid input) +} + +static bool wel_add_to_registry(const wchar_t *channel, const wchar_t *provider, DWORD defaultMaxSize) { + // Build the registry path: SYSTEM\CurrentControlSet\Services\EventLog\<LogName>\<SourceName> + wchar_t key[MAX_PATH]; + if(!provider) + swprintf(key, MAX_PATH, L"SYSTEM\\CurrentControlSet\\Services\\EventLog\\%ls", channel); + else + swprintf(key, MAX_PATH, L"SYSTEM\\CurrentControlSet\\Services\\EventLog\\%ls\\%ls", channel, provider); + + HKEY hRegKey; + DWORD disposition; + LONG result = RegCreateKeyExW(HKEY_LOCAL_MACHINE, key, + 0, NULL, REG_OPTION_NON_VOLATILE, KEY_SET_VALUE, NULL, &hRegKey, &disposition); + + if (result != ERROR_SUCCESS) + return false; // Could not create the registry key + + // Check if MaxSize is already set + DWORD maxSize = 0; + DWORD size = sizeof(maxSize); + if (RegQueryValueExW(hRegKey, L"MaxSize", NULL, NULL, (LPBYTE)&maxSize, &size) != ERROR_SUCCESS) { + // MaxSize is not set, set it to the default value + RegSetValueExW(hRegKey, L"MaxSize", 0, REG_DWORD, (const BYTE*)&defaultMaxSize, sizeof(defaultMaxSize)); + } + + wchar_t modulePath[MAX_PATH]; + if (GetModuleFileNameW(NULL, modulePath, MAX_PATH) == 0) { + RegCloseKey(hRegKey); + return false; + } + + if(wel_replace_program_with_wevt_netdata_dll(modulePath, _countof(modulePath))) { + RegSetValueExW(hRegKey, L"EventMessageFile", 0, REG_EXPAND_SZ, + (LPBYTE)modulePath, (wcslen(modulePath) + 1) * sizeof(wchar_t)); + + DWORD types_supported = EVENTLOG_SUCCESS | EVENTLOG_ERROR_TYPE | EVENTLOG_WARNING_TYPE | EVENTLOG_INFORMATION_TYPE; + RegSetValueExW(hRegKey, L"TypesSupported", 0, REG_DWORD, (LPBYTE)&types_supported, sizeof(DWORD)); + } + + RegCloseKey(hRegKey); + return true; +} + +#if defined(HAVE_ETW) +static void etw_set_source_meta(struct nd_log_source *source, USHORT channelID, const EVENT_DESCRIPTOR *ed) { + // It turns out that the keyword varies per only per channel! + // so, to log with the right keyword, Task, Opcode we copy the ids from the header + // the messages compiler (mc.exe) generated from the manifest. + + source->channelID = channelID; + source->Opcode = ed->Opcode; + source->Task = ed->Task; + source->Keyword = ed->Keyword; +} + +static bool etw_register_provider(void) { + // Register the ETW provider + if (EventRegister(&NETDATA_ETW_PROVIDER_GUID, NULL, NULL, ®Handle) != ERROR_SUCCESS) + return false; + + etw_set_source_meta(&nd_log.sources[NDLS_DAEMON], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY); + etw_set_source_meta(&nd_log.sources[NDLS_COLLECTORS], CHANNEL_COLLECTORS, &ED_COLLECTORS_INFO_MESSAGE_ONLY); + etw_set_source_meta(&nd_log.sources[NDLS_ACCESS], CHANNEL_ACCESS, &ED_ACCESS_INFO_MESSAGE_ONLY); + etw_set_source_meta(&nd_log.sources[NDLS_HEALTH], CHANNEL_HEALTH, &ED_HEALTH_INFO_MESSAGE_ONLY); + etw_set_source_meta(&nd_log.sources[NDLS_ACLK], CHANNEL_ACLK, &ED_ACLK_INFO_MESSAGE_ONLY); + etw_set_source_meta(&nd_log.sources[NDLS_UNSET], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY); + etw_set_source_meta(&nd_log.sources[NDLS_DEBUG], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY); + + return true; +} +#endif + +bool nd_log_init_windows(void) { + if(nd_log.eventlog.initialized) + return true; + + // validate we have the right keys + if( + !check_event_id(NDLS_COLLECTORS, NDLP_INFO, MSGID_MESSAGE_ONLY, MC_COLLECTORS_INFO_MESSAGE_ONLY) || + !check_event_id(NDLS_DAEMON, NDLP_ERR, MSGID_MESSAGE_ONLY, MC_DAEMON_ERR_MESSAGE_ONLY) || + !check_event_id(NDLS_ACCESS, NDLP_WARNING, MSGID_ACCESS_USER, MC_ACCESS_WARN_ACCESS_USER) || + !check_event_id(NDLS_HEALTH, NDLP_CRIT, MSGID_ALERT_TRANSITION, MC_HEALTH_CRIT_ALERT_TRANSITION) || + !check_event_id(NDLS_DEBUG, NDLP_ALERT, MSGID_ACCESS_FORWARDER_USER, MC_DEBUG_ALERT_ACCESS_FORWARDER_USER)) + return false; + +#if defined(HAVE_ETW) + if(nd_log.eventlog.etw && !etw_register_provider()) + return false; +#endif + +// if(!nd_log.eventlog.etw && !wel_add_to_registry(NETDATA_WEL_CHANNEL_NAME_W, NULL, 50 * 1024 * 1024)) +// return false; + + // Loop through each source and add it to the registry + for(size_t i = 0; i < _NDLS_MAX; i++) { + nd_log.sources[i].source = i; + + const wchar_t *sub_channel = wel_provider_per_source[i]; + + if(!sub_channel) + // we will map these to NDLS_DAEMON + continue; + + DWORD defaultMaxSize = 0; + switch (i) { + case NDLS_ACLK: + defaultMaxSize = 5 * 1024 * 1024; + break; + + case NDLS_HEALTH: + defaultMaxSize = 35 * 1024 * 1024; + break; + + default: + case NDLS_ACCESS: + case NDLS_COLLECTORS: + case NDLS_DAEMON: + defaultMaxSize = 20 * 1024 * 1024; + break; + } + + if(!nd_log.eventlog.etw) { + if(!wel_add_to_registry(NETDATA_WEL_CHANNEL_NAME_W, sub_channel, defaultMaxSize)) + return false; + + // when not using a manifest, each source is a provider + nd_log.sources[i].hEventLog = RegisterEventSourceW(NULL, sub_channel); + if (!nd_log.sources[i].hEventLog) + return false; + } + } + + if(!nd_log.eventlog.etw) { + // Map the unset ones to NDLS_DAEMON + for (size_t i = 0; i < _NDLS_MAX; i++) { + if (!nd_log.sources[i].hEventLog) + nd_log.sources[i].hEventLog = nd_log.sources[NDLS_DAEMON].hEventLog; + } + } + + nd_log.eventlog.initialized = true; + return true; +} + +bool nd_log_init_etw(void) { + nd_log.eventlog.etw = true; + return nd_log_init_windows(); +} + +bool nd_log_init_wel(void) { + nd_log.eventlog.etw = false; + return nd_log_init_windows(); +} + +// -------------------------------------------------------------------------------------------------------------------- +// we pass all our fields to the windows events logs +// numbered the same way we have them in memory. +// +// to avoid runtime memory allocations, we use a static allocations with ready to use buffers +// which are immediately available for logging. + +#define SMALL_WIDE_BUFFERS_SIZE 256 +#define MEDIUM_WIDE_BUFFERS_SIZE 2048 +#define BIG_WIDE_BUFFERS_SIZE 16384 +static wchar_t small_wide_buffers[_NDF_MAX][SMALL_WIDE_BUFFERS_SIZE]; +static wchar_t medium_wide_buffers[2][MEDIUM_WIDE_BUFFERS_SIZE]; +static wchar_t big_wide_buffers[2][BIG_WIDE_BUFFERS_SIZE]; + +static struct { + size_t size; + wchar_t *buf; +} fields_buffers[_NDF_MAX] = { 0 }; + +#if defined(HAVE_ETW) +static EVENT_DATA_DESCRIPTOR etw_eventData[_NDF_MAX - 1]; +#endif + +static LPCWSTR wel_messages[_NDF_MAX - 1]; + +__attribute__((constructor)) void wevents_initialize_buffers(void) { + for(size_t i = 0; i < _NDF_MAX ;i++) { + fields_buffers[i].buf = small_wide_buffers[i]; + fields_buffers[i].size = SMALL_WIDE_BUFFERS_SIZE; + } + + fields_buffers[NDF_NIDL_INSTANCE].buf = medium_wide_buffers[0]; + fields_buffers[NDF_NIDL_INSTANCE].size = MEDIUM_WIDE_BUFFERS_SIZE; + + fields_buffers[NDF_REQUEST].buf = big_wide_buffers[0]; + fields_buffers[NDF_REQUEST].size = BIG_WIDE_BUFFERS_SIZE; + fields_buffers[NDF_MESSAGE].buf = big_wide_buffers[1]; + fields_buffers[NDF_MESSAGE].size = BIG_WIDE_BUFFERS_SIZE; + + for(size_t i = 1; i < _NDF_MAX ;i++) + wel_messages[i - 1] = fields_buffers[i].buf; +} + +// -------------------------------------------------------------------------------------------------------------------- + +#define is_field_set(fields, fields_max, field) ((field) < (fields_max) && (fields)[field].entry.set) + +static const char *get_field_value_unsafe(struct log_field *fields, ND_LOG_FIELD_ID i, size_t fields_max, BUFFER **tmp) { + if(!is_field_set(fields, fields_max, i) || !fields[i].eventlog) + return ""; + + static char number_str[MAX(MAX(UINT64_MAX_LENGTH, DOUBLE_MAX_LENGTH), UUID_STR_LEN)]; + + const char *s = NULL; + if (fields[i].annotator) + s = fields[i].annotator(&fields[i]); + + else + switch (fields[i].entry.type) { + case NDFT_TXT: + s = fields[i].entry.txt; + break; + case NDFT_STR: + s = string2str(fields[i].entry.str); + break; + case NDFT_BFR: + s = buffer_tostring(fields[i].entry.bfr); + break; + case NDFT_U64: + print_uint64(number_str, fields[i].entry.u64); + s = number_str; + break; + case NDFT_I64: + print_int64(number_str, fields[i].entry.i64); + s = number_str; + break; + case NDFT_DBL: + print_netdata_double(number_str, fields[i].entry.dbl); + s = number_str; + break; + case NDFT_UUID: + if (!uuid_is_null(*fields[i].entry.uuid)) { + uuid_unparse_lower(*fields[i].entry.uuid, number_str); + s = number_str; + } + break; + case NDFT_CALLBACK: + if (!*tmp) + *tmp = buffer_create(1024, NULL); + else + buffer_flush(*tmp); + + if (fields[i].entry.cb.formatter(*tmp, fields[i].entry.cb.formatter_data)) + s = buffer_tostring(*tmp); + else + s = NULL; + break; + + default: + s = "UNHANDLED"; + break; + } + + if(!s || !*s) return ""; + return s; +} +static void etw_replace_percent_with_unicode(wchar_t *s, size_t size) { + size_t original_len = wcslen(s); + + // Traverse the string, replacing '%' with the Unicode fullwidth percent sign + for (size_t i = 0; i < original_len && i < size - 1; i++) { + if (s[i] == L'%' && iswdigit(s[i + 1])) { + // s[i] = 0xFF05; // Replace '%' with fullwidth percent sign '％' + // s[i] = 0x29BC; // ⦼ + s[i] = 0x2105; // ℅ + } + } + + // Ensure null termination if needed + s[size - 1] = L'\0'; +} + +static void wevt_generate_all_fields_unsafe(struct log_field *fields, size_t fields_max, BUFFER **tmp) { + for (size_t i = 0; i < fields_max; i++) { + fields_buffers[i].buf[0] = L'\0'; + + if (!fields[i].entry.set || !fields[i].eventlog) + continue; + + const char *s = get_field_value_unsafe(fields, i, fields_max, tmp); + if (s && *s) { + utf8_to_utf16(fields_buffers[i].buf, (int) fields_buffers[i].size, s, -1); + + if(nd_log.eventlog.etw) + // UNBELIEVABLE! they do recursive parameter expansion in ETW... + etw_replace_percent_with_unicode(fields_buffers[i].buf, fields_buffers[i].size); + } + } +} + +static bool has_user_role_permissions(struct log_field *fields, size_t fields_max, BUFFER **tmp) { + const char *t; + + t = get_field_value_unsafe(fields, NDF_USER_NAME, fields_max, tmp); + if (*t) return true; + + t = get_field_value_unsafe(fields, NDF_USER_ROLE, fields_max, tmp); + if (*t && strcmp(t, "none") != 0) return true; + + t = get_field_value_unsafe(fields, NDF_USER_ACCESS, fields_max, tmp); + if (*t && strcmp(t, "0x0") != 0) return true; + + return false; +} + +static bool nd_logger_windows(struct nd_log_source *source, struct log_field *fields, size_t fields_max) { + if (!nd_log.eventlog.initialized) + return false; + + ND_LOG_FIELD_PRIORITY priority = NDLP_INFO; + if (fields[NDF_PRIORITY].entry.set) + priority = (ND_LOG_FIELD_PRIORITY) fields[NDF_PRIORITY].entry.u64; + + DWORD wType = get_event_type_from_priority(priority); + (void) wType; + + CLEAN_BUFFER *tmp = NULL; + + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER; + spinlock_lock(&spinlock); + wevt_generate_all_fields_unsafe(fields, fields_max, &tmp); + + MESSAGE_ID messageID; + switch (source->source) { + default: + case NDLS_DEBUG: + case NDLS_DAEMON: + case NDLS_COLLECTORS: + messageID = MSGID_MESSAGE_ONLY; + break; + + case NDLS_HEALTH: + messageID = MSGID_ALERT_TRANSITION; + break; + + case NDLS_ACCESS: + if (is_field_set(fields, fields_max, NDF_MESSAGE)) { + messageID = MSGID_ACCESS_MESSAGE; + + if (has_user_role_permissions(fields, fields_max, &tmp)) + messageID = MSGID_ACCESS_MESSAGE_USER; + else if (*get_field_value_unsafe(fields, NDF_REQUEST, fields_max, &tmp)) + messageID = MSGID_ACCESS_MESSAGE_REQUEST; + } else if (is_field_set(fields, fields_max, NDF_RESPONSE_CODE)) { + messageID = MSGID_ACCESS; + + if (*get_field_value_unsafe(fields, NDF_SRC_FORWARDED_FOR, fields_max, &tmp)) + messageID = MSGID_ACCESS_FORWARDER; + + if (has_user_role_permissions(fields, fields_max, &tmp)) { + if (messageID == MSGID_ACCESS) + messageID = MSGID_ACCESS_USER; + else + messageID = MSGID_ACCESS_FORWARDER_USER; + } + } else + messageID = MSGID_REQUEST_ONLY; + break; + + case NDLS_ACLK: + messageID = MSGID_MESSAGE_ONLY; + break; + } + + if (messageID == MSGID_MESSAGE_ONLY && ( + *get_field_value_unsafe(fields, NDF_ERRNO, fields_max, &tmp) || + *get_field_value_unsafe(fields, NDF_WINERROR, fields_max, &tmp))) { + messageID = MSGID_MESSAGE_ERRNO; + } + + DWORD eventID = construct_event_id(source->source, priority, messageID); + + // wType + // + // without a manifest => this determines the Level of the event + // with a manifest => Level from the manifest is used (wType ignored) + // [however it is good to have, in case the manifest is not accessible somehow] + // + + // wCategory + // + // without a manifest => numeric Task values appear + // with a manifest => Task from the manifest is used (wCategory ignored) + + BOOL rc; +#if defined(HAVE_ETW) + if (nd_log.eventlog.etw) { + // metadata based logging - ETW + + for (size_t i = 1; i < _NDF_MAX; i++) + EventDataDescCreate(&etw_eventData[i - 1], fields_buffers[i].buf, + (wcslen(fields_buffers[i].buf) + 1) * sizeof(WCHAR)); + + EVENT_DESCRIPTOR EventDesc = { + .Id = eventID & EVENT_ID_CODE_MASK, // ETW needs the raw event id + .Version = 0, + .Channel = source->channelID, + .Level = get_level_from_priority(priority), + .Opcode = source->Opcode, + .Task = source->Task, + .Keyword = source->Keyword, + }; + + rc = ERROR_SUCCESS == EventWrite(regHandle, &EventDesc, _NDF_MAX - 1, etw_eventData); + + } + else +#endif + { + // eventID based logging - WEL + rc = ReportEventW(source->hEventLog, wType, 0, eventID, NULL, _NDF_MAX - 1, 0, wel_messages, NULL); + } + + spinlock_unlock(&spinlock); + + return rc == TRUE; +} + +#if defined(HAVE_ETW) +bool nd_logger_etw(struct nd_log_source *source, struct log_field *fields, size_t fields_max) { + return nd_logger_windows(source, fields, fields_max); +} +#endif + +#if defined(HAVE_WEL) +bool nd_logger_wel(struct nd_log_source *source, struct log_field *fields, size_t fields_max) { + return nd_logger_windows(source, fields, fields_max); +} +#endif + +#endif diff --git a/src/libnetdata/log/nd_log.c b/src/libnetdata/log/nd_log.c new file mode 100644 index 000000000..a605fe460 --- /dev/null +++ b/src/libnetdata/log/nd_log.c @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// do not REMOVE this, it is used by systemd-journal includes to prevent saving the file, function, line of the +// source code that makes the calls, allowing our loggers to log the lines of source code that actually log +#define SD_JOURNAL_SUPPRESS_LOCATION + +#include "../libnetdata.h" +#include "nd_log-internals.h" + +const char *program_name = ""; +uint64_t debug_flags = 0; +int aclklog_enabled = 0; + +// -------------------------------------------------------------------------------------------------------------------- + +void errno_clear(void) { + errno = 0; + +#if defined(OS_WINDOWS) + SetLastError(ERROR_SUCCESS); +#endif +} + +// -------------------------------------------------------------------------------------------------------------------- +// logger router + +static ND_LOG_METHOD nd_logger_select_output(ND_LOG_SOURCES source, FILE **fpp, SPINLOCK **spinlock) { + *spinlock = NULL; + ND_LOG_METHOD output = nd_log.sources[source].method; + + switch(output) { + case NDLM_JOURNAL: + if(unlikely(!nd_log.journal_direct.initialized && !nd_log.journal.initialized)) { + output = NDLM_FILE; + *fpp = stderr; + *spinlock = &nd_log.std_error.spinlock; + } + else { + *fpp = NULL; + *spinlock = NULL; + } + break; + +#if defined(OS_WINDOWS) && (defined(HAVE_ETW) || defined(HAVE_WEL)) +#if defined(HAVE_ETW) + case NDLM_ETW: +#endif +#if defined(HAVE_WEL) + case NDLM_WEL: +#endif + if(unlikely(!nd_log.eventlog.initialized)) { + output = NDLM_FILE; + *fpp = stderr; + *spinlock = &nd_log.std_error.spinlock; + } + else { + *fpp = NULL; + *spinlock = NULL; + } + break; +#endif + + case NDLM_SYSLOG: + if(unlikely(!nd_log.syslog.initialized)) { + output = NDLM_FILE; + *spinlock = &nd_log.std_error.spinlock; + *fpp = stderr; + } + else { + *spinlock = NULL; + *fpp = NULL; + } + break; + + case NDLM_FILE: + if(!nd_log.sources[source].fp) { + *fpp = stderr; + *spinlock = &nd_log.std_error.spinlock; + } + else { + *fpp = nd_log.sources[source].fp; + *spinlock = &nd_log.sources[source].spinlock; + } + break; + + case NDLM_STDOUT: + output = NDLM_FILE; + *fpp = stdout; + *spinlock = &nd_log.std_output.spinlock; + break; + + default: + case NDLM_DEFAULT: + case NDLM_STDERR: + output = NDLM_FILE; + *fpp = stderr; + *spinlock = &nd_log.std_error.spinlock; + break; + + case NDLM_DISABLED: + case NDLM_DEVNULL: + output = NDLM_DISABLED; + *fpp = NULL; + *spinlock = NULL; + break; + } + + return output; +} + +// -------------------------------------------------------------------------------------------------------------------- +// high level logger + +static void nd_logger_log_fields(SPINLOCK *spinlock, FILE *fp, bool limit, ND_LOG_FIELD_PRIORITY priority, + ND_LOG_METHOD output, struct nd_log_source *source, + struct log_field *fields, size_t fields_max) { + if(spinlock) + spinlock_lock(spinlock); + + // check the limits + if(limit && nd_log_limit_reached(source)) + goto cleanup; + + if(output == NDLM_JOURNAL) { + if(!nd_logger_journal_direct(fields, fields_max) && !nd_logger_journal_libsystemd(fields, fields_max)) { + // we can't log to journal, let's log to stderr + if(spinlock) + spinlock_unlock(spinlock); + + output = NDLM_FILE; + spinlock = &nd_log.std_error.spinlock; + fp = stderr; + + if(spinlock) + spinlock_lock(spinlock); + } + } + +#if defined(OS_WINDOWS) +#if defined(HAVE_ETW) + if(output == NDLM_ETW) { + if(!nd_logger_etw(source, fields, fields_max)) { + // we can't log to windows events, let's log to stderr + if(spinlock) + spinlock_unlock(spinlock); + + output = NDLM_FILE; + spinlock = &nd_log.std_error.spinlock; + fp = stderr; + + if(spinlock) + spinlock_lock(spinlock); + } + } +#endif +#if defined(HAVE_WEL) + if(output == NDLM_WEL) { + if(!nd_logger_wel(source, fields, fields_max)) { + // we can't log to windows events, let's log to stderr + if(spinlock) + spinlock_unlock(spinlock); + + output = NDLM_FILE; + spinlock = &nd_log.std_error.spinlock; + fp = stderr; + + if(spinlock) + spinlock_lock(spinlock); + } + } +#endif +#endif + + if(output == NDLM_SYSLOG) + nd_logger_syslog(priority, source->format, fields, fields_max); + + if(output == NDLM_FILE) + nd_logger_file(fp, source->format, fields, fields_max); + + +cleanup: + if(spinlock) + spinlock_unlock(spinlock); +} + +static void nd_logger_unset_all_thread_fields(void) { + size_t fields_max = THREAD_FIELDS_MAX; + for(size_t i = 0; i < fields_max ; i++) + thread_log_fields[i].entry.set = false; +} + +static void nd_logger_merge_log_stack_to_thread_fields(void) { + for(size_t c = 0; c < thread_log_stack_next ;c++) { + struct log_stack_entry *lgs = thread_log_stack_base[c]; + + for(size_t i = 0; lgs[i].id != NDF_STOP ; i++) { + if(lgs[i].id >= _NDF_MAX || !lgs[i].set) + continue; + + struct log_stack_entry *e = &lgs[i]; + ND_LOG_STACK_FIELD_TYPE type = lgs[i].type; + + // do not add empty / unset fields + if((type == NDFT_TXT && (!e->txt || !*e->txt)) || + (type == NDFT_BFR && (!e->bfr || !buffer_strlen(e->bfr))) || + (type == NDFT_STR && !e->str) || + (type == NDFT_UUID && (!e->uuid || uuid_is_null(*e->uuid))) || + (type == NDFT_CALLBACK && !e->cb.formatter) || + type == NDFT_UNSET) + continue; + + thread_log_fields[lgs[i].id].entry = *e; + } + } +} + +static void nd_logger(const char *file, const char *function, const unsigned long line, + ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, bool limit, + int saved_errno, size_t saved_winerror __maybe_unused, const char *fmt, va_list ap) { + + SPINLOCK *spinlock; + FILE *fp; + ND_LOG_METHOD output = nd_logger_select_output(source, &fp, &spinlock); + if(!IS_FINAL_LOG_METHOD(output)) + return; + + // mark all fields as unset + nd_logger_unset_all_thread_fields(); + + // flatten the log stack into the fields + nd_logger_merge_log_stack_to_thread_fields(); + + // set the common fields that are automatically set by the logging subsystem + + if(likely(!thread_log_fields[NDF_INVOCATION_ID].entry.set)) + thread_log_fields[NDF_INVOCATION_ID].entry = ND_LOG_FIELD_UUID(NDF_INVOCATION_ID, &nd_log.invocation_id); + + if(likely(!thread_log_fields[NDF_LOG_SOURCE].entry.set)) + thread_log_fields[NDF_LOG_SOURCE].entry = ND_LOG_FIELD_TXT(NDF_LOG_SOURCE, nd_log_id2source(source)); + else { + ND_LOG_SOURCES src = source; + + if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_TXT) + src = nd_log_source2id(thread_log_fields[NDF_LOG_SOURCE].entry.txt, source); + else if(thread_log_fields[NDF_LOG_SOURCE].entry.type == NDFT_U64) + src = thread_log_fields[NDF_LOG_SOURCE].entry.u64; + + if(src != source && src < _NDLS_MAX) { + source = src; + output = nd_logger_select_output(source, &fp, &spinlock); + if(output != NDLM_FILE && output != NDLM_JOURNAL && output != NDLM_SYSLOG) + return; + } + } + + if(likely(!thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry.set)) + thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, program_name); + + if(likely(!thread_log_fields[NDF_LINE].entry.set)) { + thread_log_fields[NDF_LINE].entry = ND_LOG_FIELD_U64(NDF_LINE, line); + thread_log_fields[NDF_FILE].entry = ND_LOG_FIELD_TXT(NDF_FILE, file); + thread_log_fields[NDF_FUNC].entry = ND_LOG_FIELD_TXT(NDF_FUNC, function); + } + + if(likely(!thread_log_fields[NDF_PRIORITY].entry.set)) { + thread_log_fields[NDF_PRIORITY].entry = ND_LOG_FIELD_U64(NDF_PRIORITY, priority); + } + + if(likely(!thread_log_fields[NDF_TID].entry.set)) + thread_log_fields[NDF_TID].entry = ND_LOG_FIELD_U64(NDF_TID, gettid_cached()); + + if(likely(!thread_log_fields[NDF_THREAD_TAG].entry.set)) { + const char *thread_tag = nd_thread_tag(); + thread_log_fields[NDF_THREAD_TAG].entry = ND_LOG_FIELD_TXT(NDF_THREAD_TAG, thread_tag); + + // TODO: fix the ND_MODULE in logging by setting proper module name in threads +// if(!thread_log_fields[NDF_MODULE].entry.set) +// thread_log_fields[NDF_MODULE].entry = ND_LOG_FIELD_CB(NDF_MODULE, thread_tag_to_module, (void *)thread_tag); + } + + if(likely(!thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry.set)) + thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = ND_LOG_FIELD_U64(NDF_TIMESTAMP_REALTIME_USEC, now_realtime_usec()); + + if(saved_errno != 0 && !thread_log_fields[NDF_ERRNO].entry.set) + thread_log_fields[NDF_ERRNO].entry = ND_LOG_FIELD_I64(NDF_ERRNO, saved_errno); + + if(saved_winerror != 0 && !thread_log_fields[NDF_WINERROR].entry.set) + thread_log_fields[NDF_WINERROR].entry = ND_LOG_FIELD_U64(NDF_WINERROR, saved_winerror); + + CLEAN_BUFFER *wb = NULL; + if(fmt && !thread_log_fields[NDF_MESSAGE].entry.set) { + wb = buffer_create(1024, NULL); + buffer_vsprintf(wb, fmt, ap); + thread_log_fields[NDF_MESSAGE].entry = ND_LOG_FIELD_TXT(NDF_MESSAGE, buffer_tostring(wb)); + } + + nd_logger_log_fields(spinlock, fp, limit, priority, output, &nd_log.sources[source], + thread_log_fields, THREAD_FIELDS_MAX); + + if(nd_log.sources[source].pending_msg) { + // log a pending message + + nd_logger_unset_all_thread_fields(); + + thread_log_fields[NDF_TIMESTAMP_REALTIME_USEC].entry = (struct log_stack_entry){ + .set = true, + .type = NDFT_U64, + .u64 = now_realtime_usec(), + }; + + thread_log_fields[NDF_LOG_SOURCE].entry = (struct log_stack_entry){ + .set = true, + .type = NDFT_TXT, + .txt = nd_log_id2source(source), + }; + + thread_log_fields[NDF_SYSLOG_IDENTIFIER].entry = (struct log_stack_entry){ + .set = true, + .type = NDFT_TXT, + .txt = program_name, + }; + + thread_log_fields[NDF_MESSAGE].entry = (struct log_stack_entry){ + .set = true, + .type = NDFT_TXT, + .txt = nd_log.sources[source].pending_msg, + }; + + nd_logger_log_fields(spinlock, fp, false, priority, output, + &nd_log.sources[source], + thread_log_fields, THREAD_FIELDS_MAX); + + freez((void *)nd_log.sources[source].pending_msg); + nd_log.sources[source].pending_msg = NULL; + } + + errno_clear(); +} + +static ND_LOG_SOURCES nd_log_validate_source(ND_LOG_SOURCES source) { + if(source >= _NDLS_MAX) + source = NDLS_DAEMON; + + if(nd_log.overwrite_process_source) + source = nd_log.overwrite_process_source; + + return source; +} + +// -------------------------------------------------------------------------------------------------------------------- +// public API for loggers + +void netdata_logger(ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file, const char *function, unsigned long line, const char *fmt, ... ) +{ + int saved_errno = errno; + + size_t saved_winerror = 0; +#if defined(OS_WINDOWS) + saved_winerror = GetLastError(); +#endif + + source = nd_log_validate_source(source); + + if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority) + return; + + va_list args; + va_start(args, fmt); + nd_logger(file, function, line, source, priority, + source == NDLS_DAEMON || source == NDLS_COLLECTORS, + saved_errno, saved_winerror, fmt, args); + va_end(args); +} + +void netdata_logger_with_limit(ERROR_LIMIT *erl, ND_LOG_SOURCES source, ND_LOG_FIELD_PRIORITY priority, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) { + int saved_errno = errno; + + size_t saved_winerror = 0; +#if defined(OS_WINDOWS) + saved_winerror = GetLastError(); +#endif + + source = nd_log_validate_source(source); + + if (source != NDLS_DEBUG && priority > nd_log.sources[source].min_priority) + return; + + if(erl->sleep_ut) + sleep_usec(erl->sleep_ut); + + spinlock_lock(&erl->spinlock); + + erl->count++; + time_t now = now_boottime_sec(); + if(now - erl->last_logged < erl->log_every) { + spinlock_unlock(&erl->spinlock); + return; + } + + spinlock_unlock(&erl->spinlock); + + va_list args; + va_start(args, fmt); + nd_logger(file, function, line, source, priority, + source == NDLS_DAEMON || source == NDLS_COLLECTORS, + saved_errno, saved_winerror, fmt, args); + va_end(args); + erl->last_logged = now; + erl->count = 0; +} + +void netdata_logger_fatal( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) { + int saved_errno = errno; + + size_t saved_winerror = 0; +#if defined(OS_WINDOWS) + saved_winerror = GetLastError(); +#endif + + ND_LOG_SOURCES source = NDLS_DAEMON; + source = nd_log_validate_source(source); + + va_list args; + va_start(args, fmt); + nd_logger(file, function, line, source, NDLP_ALERT, true, saved_errno, saved_winerror, fmt, args); + va_end(args); + + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH, now_realtime_sec()); + + char action_data[70+1]; + snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, saved_errno); + + const char *thread_tag = nd_thread_tag(); + const char *tag_to_send = thread_tag; + + // anonymize thread names + if(strncmp(thread_tag, THREAD_TAG_STREAM_RECEIVER, strlen(THREAD_TAG_STREAM_RECEIVER)) == 0) + tag_to_send = THREAD_TAG_STREAM_RECEIVER; + if(strncmp(thread_tag, THREAD_TAG_STREAM_SENDER, strlen(THREAD_TAG_STREAM_SENDER)) == 0) + tag_to_send = THREAD_TAG_STREAM_SENDER; + + char action_result[60+1]; + snprintfz(action_result, 60, "%s:%s", program_name, tag_to_send); + +#if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE) + int fd = nd_log.sources[NDLS_DAEMON].fd; + if(fd == -1) + fd = STDERR_FILENO; + + int nptrs; + void *buffer[10000]; + + nptrs = backtrace(buffer, sizeof(buffer)); + if(nptrs) + backtrace_symbols_fd(buffer, nptrs, fd); +#endif + +#ifdef NETDATA_INTERNAL_CHECKS + abort(); +#endif + + netdata_cleanup_and_exit(1, "FATAL", action_result, action_data); +} + diff --git a/src/libnetdata/log/log.h b/src/libnetdata/log/nd_log.h index 015c02eb6..1fefbe328 100644 --- a/src/libnetdata/log/log.h +++ b/src/libnetdata/log/nd_log.h @@ -1,150 +1,20 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#ifndef NETDATA_LOG_H -#define NETDATA_LOG_H 1 +#ifndef NETDATA_ND_LOG_H +#define NETDATA_ND_LOG_H 1 # ifdef __cplusplus extern "C" { # endif #include "../libnetdata.h" +#include "nd_log-common.h" #define ND_LOG_DEFAULT_THROTTLE_LOGS 1000 #define ND_LOG_DEFAULT_THROTTLE_PERIOD 60 -typedef enum __attribute__((__packed__)) { - NDLS_UNSET = 0, // internal use only - NDLS_ACCESS, // access.log - NDLS_ACLK, // aclk.log - NDLS_COLLECTORS, // collectors.log - NDLS_DAEMON, // error.log - NDLS_HEALTH, // health.log - NDLS_DEBUG, // debug.log - - // terminator - _NDLS_MAX, -} ND_LOG_SOURCES; - -typedef enum __attribute__((__packed__)) { - NDLP_EMERG = LOG_EMERG, - NDLP_ALERT = LOG_ALERT, - NDLP_CRIT = LOG_CRIT, - NDLP_ERR = LOG_ERR, - NDLP_WARNING = LOG_WARNING, - NDLP_NOTICE = LOG_NOTICE, - NDLP_INFO = LOG_INFO, - NDLP_DEBUG = LOG_DEBUG, -} ND_LOG_FIELD_PRIORITY; - -typedef enum __attribute__((__packed__)) { - // KEEP THESE IN THE SAME ORDER AS in thread_log_fields (log.c) - // so that it easy to audit for missing fields - - NDF_STOP = 0, - NDF_TIMESTAMP_REALTIME_USEC, // the timestamp of the log message - added automatically - NDF_SYSLOG_IDENTIFIER, // the syslog identifier of the application - added automatically - NDF_LOG_SOURCE, // DAEMON, COLLECTORS, HEALTH, ACCESS, ACLK - set at the log call - NDF_PRIORITY, // the syslog priority (severity) - set at the log call - NDF_ERRNO, // the ERRNO at the time of the log call - added automatically -#if defined(OS_WINDOWS) - NDF_WINERROR, // Windows GetLastError() -#endif - NDF_INVOCATION_ID, // the INVOCATION_ID of Netdata - added automatically - NDF_LINE, // the source code file line number - added automatically - NDF_FILE, // the source code filename - added automatically - NDF_FUNC, // the source code function - added automatically - NDF_TID, // the thread ID of the thread logging - added automatically - NDF_THREAD_TAG, // the thread tag of the thread logging - added automatically - NDF_MESSAGE_ID, // for specific events - NDF_MODULE, // for internal plugin module, all other get the NDF_THREAD_TAG - - NDF_NIDL_NODE, // the node / rrdhost currently being worked - NDF_NIDL_INSTANCE, // the instance / rrdset currently being worked - NDF_NIDL_CONTEXT, // the context of the instance currently being worked - NDF_NIDL_DIMENSION, // the dimension / rrddim currently being worked - - // web server, aclk and stream receiver - NDF_SRC_TRANSPORT, // the transport we received the request, one of: http, https, pluginsd - - // Netdata Cloud Related - NDF_ACCOUNT_ID, - NDF_USER_NAME, - NDF_USER_ROLE, - NDF_USER_ACCESS, - - // web server and stream receiver - NDF_SRC_IP, // the streaming / web server source IP - NDF_SRC_PORT, // the streaming / web server source Port - NDF_SRC_FORWARDED_HOST, - NDF_SRC_FORWARDED_FOR, - NDF_SRC_CAPABILITIES, // the stream receiver capabilities - - // stream sender (established links) - NDF_DST_TRANSPORT, // the transport we send the request, one of: http, https - NDF_DST_IP, // the destination streaming IP - NDF_DST_PORT, // the destination streaming Port - NDF_DST_CAPABILITIES, // the destination streaming capabilities - - // web server, aclk and stream receiver - NDF_REQUEST_METHOD, // for http like requests, the http request method - NDF_RESPONSE_CODE, // for http like requests, the http response code, otherwise a status string - - // web server (all), aclk (queries) - NDF_CONNECTION_ID, // the web server connection ID - NDF_TRANSACTION_ID, // the web server and API transaction ID - NDF_RESPONSE_SENT_BYTES, // for http like requests, the response bytes - NDF_RESPONSE_SIZE_BYTES, // for http like requests, the uncompressed response size - NDF_RESPONSE_PREPARATION_TIME_USEC, // for http like requests, the preparation time - NDF_RESPONSE_SENT_TIME_USEC, // for http like requests, the time to send the response back - NDF_RESPONSE_TOTAL_TIME_USEC, // for http like requests, the total time to complete the response - - // health alerts - NDF_ALERT_ID, - NDF_ALERT_UNIQUE_ID, - NDF_ALERT_EVENT_ID, - NDF_ALERT_TRANSITION_ID, - NDF_ALERT_CONFIG_HASH, - NDF_ALERT_NAME, - NDF_ALERT_CLASS, - NDF_ALERT_COMPONENT, - NDF_ALERT_TYPE, - NDF_ALERT_EXEC, - NDF_ALERT_RECIPIENT, - NDF_ALERT_DURATION, - NDF_ALERT_VALUE, - NDF_ALERT_VALUE_OLD, - NDF_ALERT_STATUS, - NDF_ALERT_STATUS_OLD, - NDF_ALERT_SOURCE, - NDF_ALERT_UNITS, - NDF_ALERT_SUMMARY, - NDF_ALERT_INFO, - NDF_ALERT_NOTIFICATION_REALTIME_USEC, - // NDF_ALERT_FLAGS, - - // put new items here - // leave the request URL and the message last - - NDF_REQUEST, // the request we are currently working on - NDF_MESSAGE, // the log message, if any - - // terminator - _NDF_MAX, -} ND_LOG_FIELD_ID; - -typedef enum __attribute__((__packed__)) { - NDFT_UNSET = 0, - NDFT_TXT, - NDFT_STR, - NDFT_BFR, - NDFT_U64, - NDFT_I64, - NDFT_DBL, - NDFT_UUID, - NDFT_CALLBACK, -} ND_LOG_STACK_FIELD_TYPE; - void errno_clear(void); +int nd_log_systemd_journal_fd(void); void nd_log_set_user_settings(ND_LOG_SOURCES source, const char *setting); void nd_log_set_facility(const char *facility); void nd_log_set_priority_level(const char *setting); @@ -154,9 +24,9 @@ void chown_open_file(int fd, uid_t uid, gid_t gid); void nd_log_chown_log_files(uid_t uid, gid_t gid); void nd_log_set_flood_protection(size_t logs, time_t period); void nd_log_initialize_for_external_plugins(const char *name); -void nd_log_reopen_log_files_for_spawn_server(void); +void nd_log_reopen_log_files_for_spawn_server(const char *name); bool nd_log_journal_socket_available(void); -ND_LOG_FIELD_ID nd_log_field_id_by_name(const char *field, size_t len); +ND_LOG_FIELD_ID nd_log_field_id_by_journal_name(const char *field, size_t len); int nd_log_priority2id(const char *priority); const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority); const char *nd_log_method_for_external_plugins(const char *s); @@ -238,12 +108,8 @@ void log_stack_push(struct log_stack_entry *lgs); #define D_SYSTEM 0x8000000000000000 extern uint64_t debug_flags; - extern const char *program_name; - -#ifdef ENABLE_ACLK extern int aclklog_enabled; -#endif #define LOG_DATE_LENGTH 26 void log_date(char *buffer, size_t len, time_t now); @@ -310,4 +176,4 @@ void netdata_logger_fatal( const char *file, const char *function, unsigned long } # endif -#endif /* NETDATA_LOG_H */ +#endif /* NETDATA_ND_LOG_H */ diff --git a/src/libnetdata/log/nd_log_limit.c b/src/libnetdata/log/nd_log_limit.c new file mode 100644 index 000000000..272138196 --- /dev/null +++ b/src/libnetdata/log/nd_log_limit.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "nd_log_limit.h" + +void nd_log_limits_reset(void) { + usec_t now_ut = now_monotonic_usec(); + + spinlock_lock(&nd_log.std_output.spinlock); + spinlock_lock(&nd_log.std_error.spinlock); + + for(size_t i = 0; i < _NDLS_MAX ;i++) { + spinlock_lock(&nd_log.sources[i].spinlock); + nd_log.sources[i].limits.prevented = 0; + nd_log.sources[i].limits.counter = 0; + nd_log.sources[i].limits.started_monotonic_ut = now_ut; + nd_log.sources[i].limits.logs_per_period = nd_log.sources[i].limits.logs_per_period_backup; + spinlock_unlock(&nd_log.sources[i].spinlock); + } + + spinlock_unlock(&nd_log.std_output.spinlock); + spinlock_unlock(&nd_log.std_error.spinlock); +} + +void nd_log_limits_unlimited(void) { + nd_log_limits_reset(); + for(size_t i = 0; i < _NDLS_MAX ;i++) { + nd_log.sources[i].limits.logs_per_period = 0; + } +} + +bool nd_log_limit_reached(struct nd_log_source *source) { + if(source->limits.throttle_period == 0 || source->limits.logs_per_period == 0) + return false; + + usec_t now_ut = now_monotonic_usec(); + if(!source->limits.started_monotonic_ut) + source->limits.started_monotonic_ut = now_ut; + + source->limits.counter++; + + if(now_ut - source->limits.started_monotonic_ut > (usec_t)source->limits.throttle_period) { + if(source->limits.prevented) { + BUFFER *wb = buffer_create(1024, NULL); + buffer_sprintf(wb, + "LOG FLOOD PROTECTION: resuming logging " + "(prevented %"PRIu32" logs in the last %"PRIu32" seconds).", + source->limits.prevented, + source->limits.throttle_period); + + if(source->pending_msg) + freez((void *)source->pending_msg); + + source->pending_msg = strdupz(buffer_tostring(wb)); + + buffer_free(wb); + } + + // restart the period accounting + source->limits.started_monotonic_ut = now_ut; + source->limits.counter = 1; + source->limits.prevented = 0; + + // log this error + return false; + } + + if(source->limits.counter > source->limits.logs_per_period) { + if(!source->limits.prevented) { + BUFFER *wb = buffer_create(1024, NULL); + buffer_sprintf(wb, + "LOG FLOOD PROTECTION: too many logs (%"PRIu32" logs in %"PRId64" seconds, threshold is set to %"PRIu32" logs " + "in %"PRIu32" seconds). Preventing more logs from process '%s' for %"PRId64" seconds.", + source->limits.counter, + (int64_t)((now_ut - source->limits.started_monotonic_ut) / USEC_PER_SEC), + source->limits.logs_per_period, + source->limits.throttle_period, + program_name, + (int64_t)(((source->limits.started_monotonic_ut + (source->limits.throttle_period * USEC_PER_SEC) - now_ut)) / USEC_PER_SEC) + ); + + if(source->pending_msg) + freez((void *)source->pending_msg); + + source->pending_msg = strdupz(buffer_tostring(wb)); + + buffer_free(wb); + } + + source->limits.prevented++; + + // prevent logging this error +#ifdef NETDATA_INTERNAL_CHECKS + return false; +#else + return true; +#endif + } + + return false; +} diff --git a/src/libnetdata/log/nd_log_limit.h b/src/libnetdata/log/nd_log_limit.h new file mode 100644 index 000000000..5486abde9 --- /dev/null +++ b/src/libnetdata/log/nd_log_limit.h @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_ND_LOG_LIMIT_H +#define NETDATA_ND_LOG_LIMIT_H + +#include "../libnetdata.h" + +struct nd_log_source; +bool nd_log_limit_reached(struct nd_log_source *source); + +struct nd_log_limit { + usec_t started_monotonic_ut; + uint32_t counter; + uint32_t prevented; + + uint32_t throttle_period; + uint32_t logs_per_period; + uint32_t logs_per_period_backup; +}; + +#define ND_LOG_LIMITS_DEFAULT (struct nd_log_limit){ .logs_per_period = ND_LOG_DEFAULT_THROTTLE_LOGS, .logs_per_period_backup = ND_LOG_DEFAULT_THROTTLE_LOGS, .throttle_period = ND_LOG_DEFAULT_THROTTLE_PERIOD, } +#define ND_LOG_LIMITS_UNLIMITED (struct nd_log_limit){ .logs_per_period = 0, .logs_per_period_backup = 0, .throttle_period = 0, } + +#include "nd_log-internals.h" + +#endif //NETDATA_ND_LOG_LIMIT_H diff --git a/src/libnetdata/log/nd_wevents_manifest.xml b/src/libnetdata/log/nd_wevents_manifest.xml new file mode 100644 index 000000000..9e326c1cb --- /dev/null +++ b/src/libnetdata/log/nd_wevents_manifest.xml @@ -0,0 +1,295 @@ +<?xml version="1.0" encoding="UTF-8"?> +<instrumentationManifest + xmlns="http://schemas.microsoft.com/win/2004/08/events" + xmlns:win="http://manifests.microsoft.com/win/2004/08/windows/events" + xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <instrumentation> + <events> + + <provider name="Netdata" + guid="{96c5ca72-9bd8-4634-81e5-000014e7da7a}" + symbol="ND_PROVIDER_NAME" + messageFileName="%SystemRoot%\System32\nd_wevents.dll" + resourceFileName="%SystemRoot%\System32\nd_wevents.dll" + parameterFileName="%SystemRoot%\System32\nd_wevents.dll" + message="$(string.ND_PROVIDER_NAME)"> + + <!-- Define the channels --> + <channels> + <channel name="Netdata/Daemon" + symbol="ND_CHANNEL_DAEMON" + type="Operational"/> + + <channel name="Netdata/Collectors" + symbol="ND_CHANNEL_COLLECTORS" + type="Operational"/> + + <channel name="Netdata/Access" + symbol="ND_CHANNEL_ACCESS" + type="Operational"/> + + <channel symbol="ND_CHANNEL_HEALTH" + name="Netdata/Alerts" + type="Operational"/> + + <channel name="Netdata/ACLK" + symbol="ND_CHANNEL_ACLK" + type="Operational"/> + </channels> + + <levels> + </levels> + + <opcodes> + </opcodes> + + <tasks> + <task name="Daemon" value="1" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Daemon)"/> + <task name="Collector" value="2" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Collector)"/> + <task name="Access" value="3" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Access)"/> + <task name="Health" value="4" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Health)"/> + <task name="Aclk" value="5" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Aclk)"/> + </tasks> + + <templates> + <template tid="NetdataLogTemplate"> + <!-- 0 (NDF_STOP) should not be here %1 is Timestamp, %64 is the Message --> + <data name="Timestamp" inType="win:UnicodeString"/> <!-- 1 (NDF_TIMESTAMP_REALTIME_USEC) --> + <data name="Program" inType="win:UnicodeString"/> <!-- 2 (NDF_SYSLOG_IDENTIFIER) --> + <data name="NetdataLogSource" inType="win:UnicodeString"/> <!-- 3 (NDF_LOG_SOURCE) --> + <data name="Level" inType="win:UnicodeString"/> <!-- 4 (NDF_PRIORITY) --> + <data name="UnixErrno" inType="win:UnicodeString"/> <!-- 5 (NDF_ERRNO) --> + <data name="WindowsLastError" inType="win:UnicodeString"/> <!-- 6 (NDF_WINERROR) --> + <data name="InvocationID" inType="win:UnicodeString"/> <!-- 7 (NDF_INVOCATION_ID) --> + <data name="CodeLine" inType="win:UInt32"/> <!-- 8 (NDF_LINE) --> + <data name="CodeFile" inType="win:UnicodeString"/> <!-- 9 (NDF_FILE) --> + <data name="CodeFunction" inType="win:UnicodeString"/> <!-- 10 (NDF_FUNC) --> + <data name="ThreadID" inType="win:UInt32"/> <!-- 11 (NDF_TID) --> + <data name="ThreadName" inType="win:UnicodeString"/> <!-- 12 (NDF_THREAD_TAG) --> + <data name="MessageID" inType="win:UnicodeString"/> <!-- 13 (NDF_MESSAGE_ID) --> + <data name="Module" inType="win:UnicodeString"/> <!-- 14 (NDF_MODULE) --> + <data name="Node" inType="win:UnicodeString"/> <!-- 15 (NDF_NIDL_NODE) --> + <data name="Instance" inType="win:UnicodeString"/> <!-- 16 (NDF_NIDL_INSTANCE) --> + <data name="Context" inType="win:UnicodeString"/> <!-- 17 (NDF_NIDL_CONTEXT) --> + <data name="Dimension" inType="win:UnicodeString"/> <!-- 18 (NDF_NIDL_DIMENSION) --> + <data name="SourceTransport" inType="win:UnicodeString"/> <!-- 19 (NDF_SRC_TRANSPORT) --> + <data name="AccountID" inType="win:UnicodeString"/> <!-- 20 (NDF_ACCOUNT_ID) --> + <data name="UserName" inType="win:UnicodeString"/> <!-- 21 (NDF_USER_NAME) --> + <data name="UserRole" inType="win:UnicodeString"/> <!-- 22 (NDF_USER_ROLE) --> + <data name="UserPermissions" inType="win:UnicodeString"/> <!-- 23 (NDF_USER_ACCESS) --> + <data name="SourceIP" inType="win:UnicodeString"/> <!-- 24 (NDF_SRC_IP) --> + <data name="SourceForwardedHost" inType="win:UnicodeString"/> <!-- 25 (NDF_SRC_PORT) --> + <data name="SourceForwardedFor" inType="win:UnicodeString"/> <!-- 26 (NDF_SRC_FORWARDED_HOST) --> + <data name="SourcePort" inType="win:UInt32"/> <!-- 27 (NDF_SRC_FORWARDED_FOR) --> + <data name="SourceCapabilities" inType="win:UnicodeString"/> <!-- 28 (NDF_SRC_CAPABILITIES) --> + <data name="DestinationTransport" inType="win:UnicodeString"/> <!-- 29 (NDF_DST_TRANSPORT) --> + <data name="DestinationIP" inType="win:UnicodeString"/> <!-- 30 (NDF_DST_IP) --> + <data name="DestinationPort" inType="win:UInt32"/> <!-- 31 (NDF_DST_PORT) --> + <data name="DestinationCapabilities" inType="win:UnicodeString"/> <!-- 32 (NDF_DST_CAPABILITIES) --> + <data name="RequestMethod" inType="win:UnicodeString"/> <!-- 33 (NDF_REQUEST_METHOD) --> + <data name="ResponseCode" inType="win:UInt32"/> <!-- 34 (NDF_RESPONSE_CODE) --> + <data name="ConnectionID" inType="win:UnicodeString"/> <!-- 35 (NDF_CONNECTION_ID) --> + <data name="TransactionID" inType="win:UnicodeString"/> <!-- 36 (NDF_TRANSACTION_ID) --> + <data name="ResponseSentBytes" inType="win:UInt64"/> <!-- 37 (NDF_RESPONSE_SENT_BYTES) --> + <data name="ResponseSizeBytes" inType="win:UInt64"/> <!-- 38 (NDF_RESPONSE_SIZE_BYTES) --> + <data name="ResponsePreparationTimeUsec" inType="win:UInt64"/> <!-- 39 (NDF_RESPONSE_PREPARATION_TIME_USEC) --> + <data name="ResponseSentTimeUsec" inType="win:UInt64"/> <!-- 40 (NDF_RESPONSE_SENT_TIME_USEC) --> + <data name="ResponseTotalTimeUsec" inType="win:UInt64"/> <!-- 41 (NDF_RESPONSE_TOTAL_TIME_USEC) --> + <data name="AlertID" inType="win:UnicodeString"/> <!-- 42 (NDF_ALERT_ID) --> + <data name="AlertUniqueID" inType="win:UnicodeString"/> <!-- 43 (NDF_ALERT_UNIQUE_ID) --> + <data name="AlertTransitionID" inType="win:UnicodeString"/> <!-- 44 (NDF_ALERT_TRANSITION_ID) --> + <data name="AlertEventID" inType="win:UnicodeString"/> <!-- 45 (NDF_ALERT_EVENT_ID) --> + <data name="AlertConfig" inType="win:UnicodeString"/> <!-- 46 (NDF_ALERT_CONFIG_HASH) --> + <data name="AlertName" inType="win:UnicodeString"/> <!-- 47 (NDF_ALERT_NAME) --> + <data name="AlertClass" inType="win:UnicodeString"/> <!-- 48 (NDF_ALERT_CLASS) --> + <data name="AlertComponent" inType="win:UnicodeString"/> <!-- 49 (NDF_ALERT_COMPONENT) --> + <data name="AlertType" inType="win:UnicodeString"/> <!-- 50 (NDF_ALERT_TYPE) --> + <data name="AlertExec" inType="win:UnicodeString"/> <!-- 51 (NDF_ALERT_EXEC) --> + <data name="AlertRecipient" inType="win:UnicodeString"/> <!-- 52 (NDF_ALERT_RECIPIENT) --> + <data name="AlertDuration" inType="win:UInt64"/> <!-- 53 (NDF_ALERT_DURATION) --> + <data name="AlertValue" inType="win:Double"/> <!-- 54 (NDF_ALERT_VALUE) --> + <data name="AlertOldValue" inType="win:Double"/> <!-- 55 (NDF_ALERT_VALUE_OLD) --> + <data name="AlertStatus" inType="win:UnicodeString"/> <!-- 56 (NDF_ALERT_STATUS) --> + <data name="AlertOldStatus" inType="win:UnicodeString"/> <!-- 57 (NDF_ALERT_STATUS_OLD) --> + <data name="Source" inType="win:UnicodeString"/> <!-- 58 (NDF_ALERT_SOURCE) --> + <data name="AlertUnits" inType="win:UnicodeString"/> <!-- 59 (NDF_ALERT_UNITS) --> + <data name="AlertSummary" inType="win:UnicodeString"/> <!-- 60 (NDF_ALERT_SUMMARY) --> + <data name="AlertInfo" inType="win:UnicodeString"/> <!-- 61 (NDF_ALERT_INFO) --> + <data name="AlertNotificationTime" inType="win:UInt64"/> <!-- 62 (NDF_ALERT_NOTIFICATION_REALTIME_USEC) --> + <data name="Request" inType="win:UnicodeString"/> <!-- 63 (NDF_REQUEST) --> + <data name="Message" inType="win:UnicodeString"/> <!-- 64 (NDF_MESSAGE) --> + </template> + </templates> + + <events> + <!-- Daemon Events --> + <event symbol="ND_EVENT_DAEMON_INFO" + value="0x1000" + message="$(string.ND_GENERIC_LOG_MESSAGE)" + channel="Netdata/Daemon" + level="win:Informational" + task="Daemon" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <event symbol="ND_EVENT_DAEMON_WARNING" + value="0x1001" + message="$(string.ND_GENERIC_LOG_MESSAGE)" + channel="Netdata/Daemon" + level="win:Warning" + task="Daemon" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <event symbol="ND_EVENT_DAEMON_ERROR" + value="0x1002" + message="$(string.ND_GENERIC_LOG_MESSAGE)" + channel="Netdata/Daemon" + level="win:Error" + task="Daemon" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <!-- Collector Events --> + <event symbol="ND_EVENT_COLLECTOR_INFO" + value="0x2000" + message="$(string.ND_GENERIC_LOG_MESSAGE)" + channel="Netdata/Collectors" + level="win:Informational" + task="Collector" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <event symbol="ND_EVENT_COLLECTOR_WARNING" + value="0x2001" + message="$(string.ND_GENERIC_LOG_MESSAGE)" + channel="Netdata/Collectors" + level="win:Warning" + task="Collector" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <event symbol="ND_EVENT_COLLECTOR_ERROR" + value="0x2002" + message="$(string.ND_GENERIC_LOG_MESSAGE)" + channel="Netdata/Collectors" + level="win:Error" + task="Collector" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <!-- Access Events --> + <event symbol="ND_EVENT_ACCESS_INFO" + value="0x3000" + message="$(string.ND_ACCESS_EVENT_MESSAGE)" + channel="Netdata/Access" + level="win:Informational" + task="Access" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <event symbol="ND_EVENT_ACCESS_WARNING" + value="0x3001" + message="$(string.ND_ACCESS_EVENT_MESSAGE)" + channel="Netdata/Access" + level="win:Warning" + task="Access" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <event symbol="ND_EVENT_ACCESS_ERROR" + value="0x3002" + message="$(string.ND_ACCESS_EVENT_MESSAGE)" + channel="Netdata/Access" + level="win:Error" + task="Access" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <!-- Health Events --> + <event symbol="ND_EVENT_HEALTH_INFO" + value="0x4000" + message="$(string.ND_HEALTH_EVENT_MESSAGE)" + channel="Netdata/Alerts" + level="win:Informational" + task="Health" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <event symbol="ND_EVENT_HEALTH_WARNING" + value="0x4001" + message="$(string.ND_HEALTH_EVENT_MESSAGE)" + channel="Netdata/Alerts" + level="win:Warning" + task="Health" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <event symbol="ND_EVENT_HEALTH_ERROR" + value="0x4002" + message="$(string.ND_HEALTH_EVENT_MESSAGE)" + channel="Netdata/Alerts" + level="win:Error" + task="Health" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <!-- ACLK Events --> + <event symbol="ND_EVENT_ACLK_INFO" + value="0x5000" + message="$(string.ND_GENERIC_LOG_MESSAGE)" + channel="Netdata/ACLK" + level="win:Informational" + task="Aclk" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <event symbol="ND_EVENT_ACLK_WARNING" + value="0x5001" + message="$(string.ND_GENERIC_LOG_MESSAGE)" + channel="Netdata/ACLK" + level="win:Warning" + task="Aclk" + opcode="win:Info" + template="NetdataLogTemplate"/> + + <event symbol="ND_EVENT_ACLK_ERROR" + value="0x5002" + message="$(string.ND_GENERIC_LOG_MESSAGE)" + channel="Netdata/ACLK" + level="win:Error" + task="Aclk" + opcode="win:Info" + template="NetdataLogTemplate"/> + + </events> + </provider> + </events> + </instrumentation> + + <localization> + <resources culture="en-US"> + <stringTable> + <string id="Task.Daemon" value="ND Daemon Log"/> + <string id="Task.Collector" value="ND Collector Log"/> + <string id="Task.Access" value="ND Access Log"/> + <string id="Task.Health" value="ND Health Log"/> + <string id="Task.Aclk" value="ND ACLK Log"/> + + <string id="ND_PROVIDER_NAME" value="Netdata"/> + <string id="ND_GENERIC_LOG_MESSAGE" value="%64"/> + <string id="ND_ACCESS_EVENT_MESSAGE" + value="Transaction %36, method: %33, path: %63 + + Source IP : %24, Forwarded-For: %27 + User : %21, role: %22, permissions: %23 + Timings (usec): prep %39, sent %40, total %41 + Response Size : sent %37, uncompressed %38 + Response Code : %34 +"/> + <string id="ND_HEALTH_EVENT_MESSAGE" + value="Alert '%47' of instance '%16' on node '%15', transitioned from %57 to %56"/> + </stringTable> + </resources> + </localization> +</instrumentationManifest> diff --git a/src/libnetdata/log/systemd-cat-native.c b/src/libnetdata/log/systemd-cat-native.c index 74d3728a3..2e4f55e97 100644 --- a/src/libnetdata/log/systemd-cat-native.c +++ b/src/libnetdata/log/systemd-cat-native.c @@ -11,7 +11,9 @@ #include <machine/endian.h> #endif -static inline void log_message_to_stderr(BUFFER *msg) { +bool verbose = false; + +static inline void log_message_to_stderr(BUFFER *msg, const char *scope) { CLEAN_BUFFER *tmp = buffer_create(0, NULL); for(size_t i = 0; i < msg->len ;i++) { @@ -24,13 +26,13 @@ static inline void log_message_to_stderr(BUFFER *msg) { } } - fprintf(stderr, "SENDING: %s\n", buffer_tostring(tmp)); + fprintf(stderr, "SENDING %s: %s\n", scope, buffer_tostring(tmp)); } static inline buffered_reader_ret_t get_next_line(struct buffered_reader *reader, BUFFER *line, int timeout_ms) { while(true) { if(unlikely(!buffered_reader_next_line(reader, line))) { - buffered_reader_ret_t ret = buffered_reader_read_timeout(reader, STDIN_FILENO, timeout_ms, false); + buffered_reader_ret_t ret = buffered_reader_read_timeout(reader, STDIN_FILENO, timeout_ms, verbose); if(unlikely(ret != BUFFERED_READER_READ_OK)) return ret; @@ -126,7 +128,7 @@ static inline void buffer_memcat_replacing_newlines(BUFFER *wb, const char *src, // ---------------------------------------------------------------------------- // log to a systemd-journal-remote -#ifdef HAVE_CURL +#ifdef HAVE_LIBCURL #include <curl/curl.h> #ifndef HOST_NAME_MAX @@ -203,8 +205,8 @@ static void journal_remote_complete_event(BUFFER *msg, usec_t *monotonic_ut) { buffer_sprintf(msg, "" - "__REALTIME_TIMESTAMP=%llu\n" - "__MONOTONIC_TIMESTAMP=%llu\n" + "__REALTIME_TIMESTAMP=%"PRIu64"\n" + "__MONOTONIC_TIMESTAMP=%"PRIu64"\n" "_MACHINE_ID=%s\n" "_BOOT_ID=%s\n" "_HOSTNAME=%s\n" @@ -226,7 +228,8 @@ static void journal_remote_complete_event(BUFFER *msg, usec_t *monotonic_ut) { static CURLcode journal_remote_send_buffer(CURL* curl, BUFFER *msg) { - // log_message_to_stderr(msg); + if(verbose) + log_message_to_stderr(msg, "REMOTE"); struct upload_data upload = {0}; @@ -260,8 +263,8 @@ static log_to_journal_remote_ret_t log_input_to_journal_remote(const char *url, global_boot_id[0] = '\0'; char buffer[1024]; - if(read_file(BOOT_ID_PATH, buffer, sizeof(buffer)) == 0) { - uuid_t uuid; + if(read_txt_file(BOOT_ID_PATH, buffer, sizeof(buffer)) == 0) { + nd_uuid_t uuid; if(uuid_parse_flexi(buffer, uuid) == 0) uuid_unparse_lower_compact(uuid, global_boot_id); else @@ -270,13 +273,13 @@ static log_to_journal_remote_ret_t log_input_to_journal_remote(const char *url, if(global_boot_id[0] == '\0') { fprintf(stderr, "WARNING: cannot read '%s'. Will generate a random _BOOT_ID.\n", BOOT_ID_PATH); - uuid_t uuid; + nd_uuid_t uuid; uuid_generate_random(uuid); uuid_unparse_lower_compact(uuid, global_boot_id); } - if(read_file(MACHINE_ID_PATH, buffer, sizeof(buffer)) == 0) { - uuid_t uuid; + if(read_txt_file(MACHINE_ID_PATH, buffer, sizeof(buffer)) == 0) { + nd_uuid_t uuid; if(uuid_parse_flexi(buffer, uuid) == 0) uuid_unparse_lower_compact(uuid, global_machine_id); else @@ -285,13 +288,13 @@ static log_to_journal_remote_ret_t log_input_to_journal_remote(const char *url, if(global_machine_id[0] == '\0') { fprintf(stderr, "WARNING: cannot read '%s'. Will generate a random _MACHINE_ID.\n", MACHINE_ID_PATH); - uuid_t uuid; + nd_uuid_t uuid; uuid_generate_random(uuid); uuid_unparse_lower_compact(uuid, global_boot_id); } if(global_stream_id[0] == '\0') { - uuid_t uuid; + nd_uuid_t uuid; uuid_generate_random(uuid); uuid_unparse_lower_compact(uuid, global_stream_id); } @@ -456,10 +459,11 @@ static int help(void) { "Usage:\n" "\n" " %s\n" + " [--verbose|-v]\n" " [--newline=STRING]\n" " [--log-as-netdata|-N]\n" " [--namespace=NAMESPACE] [--socket=PATH]\n" -#ifdef HAVE_CURL +#ifdef HAVE_LIBCURL " [--url=URL [--key=FILENAME] [--cert=FILENAME] [--trust=FILENAME|all]]\n" #endif "\n" @@ -488,7 +492,7 @@ static int help(void) { " the log destination. Only log fields defined by Netdata are accepted.\n" " If the environment variables expected by Netdata are not found, it\n" " falls back to stderr logging in logfmt format.\n" -#ifdef HAVE_CURL +#ifdef HAVE_LIBCURL "\n" " * Log to a systemd-journal-remote TCP socket, enabled with --url=URL\n" "\n" @@ -585,15 +589,16 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) { ND_LOG_STACK_PUSH(lgs); lgs_reset(lgs); + ND_LOG_SOURCES source = NDLS_HEALTH; + ND_LOG_FIELD_PRIORITY priority = NDLP_INFO; size_t fields_added = 0; size_t messages_logged = 0; - ND_LOG_FIELD_PRIORITY priority = NDLP_INFO; while(get_next_line(&reader, line, timeout_ms) == BUFFERED_READER_READ_OK) { if(!line->len) { // an empty line - we are done for this message - nd_log(NDLS_HEALTH, priority, + nd_log(source, priority, "added %zu fields", // if the user supplied a MESSAGE, this will be ignored fields_added); @@ -606,7 +611,7 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) { if(equal) { const char *field = line->buffer; size_t field_len = equal - line->buffer; - ND_LOG_FIELD_ID id = nd_log_field_id_by_name(field, field_len); + ND_LOG_FIELD_ID id = nd_log_field_id_by_journal_name(field, field_len); if(id != NDF_STOP) { const char *value = ++equal; @@ -625,7 +630,7 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) { struct log_stack_entry backup = lgs[NDF_MESSAGE]; lgs[NDF_MESSAGE] = ND_LOG_FIELD_TXT(NDF_MESSAGE, NULL); - nd_log(NDLS_COLLECTORS, NDLP_ERR, + nd_log(source, NDLP_ERR, "Field '%.*s' is not a Netdata field. Ignoring it.", (int)field_len, field); @@ -636,7 +641,7 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) { struct log_stack_entry backup = lgs[NDF_MESSAGE]; lgs[NDF_MESSAGE] = ND_LOG_FIELD_TXT(NDF_MESSAGE, NULL); - nd_log(NDLS_COLLECTORS, NDLP_ERR, + nd_log(source, NDLP_ERR, "Line does not contain an = sign; ignoring it: %s", line->buffer); @@ -648,7 +653,7 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) { } if(fields_added) { - nd_log(NDLS_HEALTH, priority, "added %zu fields", fields_added); + nd_log(source, priority, "added %zu fields", fields_added); messages_logged++; } @@ -659,7 +664,8 @@ static int log_input_as_netdata(const char *newline, int timeout_ms) { // log to a local systemd-journald static bool journal_local_send_buffer(int fd, BUFFER *msg) { - // log_message_to_stderr(msg); + if(verbose) + log_message_to_stderr(msg, "LOCAL"); bool ret = journal_direct_send(fd, msg->buffer, msg->len); if (!ret) @@ -720,19 +726,25 @@ static int log_input_to_journal(const char *socket, const char *namespace, const } cleanup: + if(verbose) { + if(failed_messages) + fprintf(stderr, "%zu messages failed to be logged\n", failed_messages); + if(!messages_logged) + fprintf(stderr, "No messages were logged!\n"); + } + return !failed_messages && messages_logged ? 0 : 1; } int main(int argc, char *argv[]) { - clocks_init(); nd_log_initialize_for_external_plugins(argv[0]); - int timeout_ms = -1; // wait forever + int timeout_ms = 0; // wait forever bool log_as_netdata = false; const char *newline = NULL; const char *namespace = NULL; const char *socket = getenv("NETDATA_SYSTEMD_JOURNAL_PATH"); -#ifdef HAVE_CURL +#ifdef HAVE_LIBCURL const char *url = NULL; const char *key = NULL; const char *cert = NULL; @@ -746,6 +758,9 @@ int main(int argc, char *argv[]) { if(strcmp(k, "--help") == 0 || strcmp(k, "-h") == 0) return help(); + else if(strcmp(k, "--verbose") == 0 || strcmp(k, "-v") == 0) + verbose = true; + else if(strcmp(k, "--log-as-netdata") == 0 || strcmp(k, "-N") == 0) log_as_netdata = true; @@ -758,7 +773,7 @@ int main(int argc, char *argv[]) { else if(strncmp(k, "--newline=", 10) == 0) newline = &k[10]; -#ifdef HAVE_CURL +#ifdef HAVE_LIBCURL else if (strncmp(k, "--url=", 6) == 0) url = &k[6]; @@ -780,7 +795,7 @@ int main(int argc, char *argv[]) { } } -#ifdef HAVE_CURL +#ifdef HAVE_LIBCURL if(log_as_netdata && url) { fprintf(stderr, "Cannot log to a systemd-journal-remote URL as Netdata. " "Please either give --url or --log-as-netdata, not both.\n"); @@ -804,7 +819,7 @@ int main(int argc, char *argv[]) { if(log_as_netdata) return log_input_as_netdata(newline, timeout_ms); -#ifdef HAVE_CURL +#ifdef HAVE_LIBCURL if(url) { if(url && namespace && *namespace) snprintfz(global_namespace, sizeof(global_namespace), "_NAMESPACE=%s\n", namespace); diff --git a/src/libnetdata/log/journal.c b/src/libnetdata/log/systemd-journal-helpers.c index 2182212f6..24553364b 100644 --- a/src/libnetdata/log/journal.c +++ b/src/libnetdata/log/systemd-journal-helpers.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "journal.h" +#include "systemd-journal-helpers.h" bool is_path_unix_socket(const char *path) { // Check if the path is valid diff --git a/src/libnetdata/log/journal.h b/src/libnetdata/log/systemd-journal-helpers.h index df8ece18b..a85f8e85a 100644 --- a/src/libnetdata/log/journal.h +++ b/src/libnetdata/log/systemd-journal-helpers.h @@ -2,8 +2,8 @@ #include "../libnetdata.h" -#ifndef NETDATA_LOG_JOURNAL_H -#define NETDATA_LOG_JOURNAL_H +#ifndef NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H +#define NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H #define JOURNAL_DIRECT_SOCKET "/run/systemd/journal/socket" @@ -15,4 +15,4 @@ bool journal_direct_send(int fd, const char *msg, size_t msg_len); bool is_path_unix_socket(const char *path); bool is_stderr_connected_to_journal(void); -#endif //NETDATA_LOG_JOURNAL_H +#endif // NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H diff --git a/src/libnetdata/log/wevt_netdata_compile.bat b/src/libnetdata/log/wevt_netdata_compile.bat new file mode 100644 index 000000000..279b6c31b --- /dev/null +++ b/src/libnetdata/log/wevt_netdata_compile.bat @@ -0,0 +1,121 @@ +@echo off
+setlocal enabledelayedexpansion
+
+echo PATH=%PATH%
+
+if "%~1"=="" (
+ echo Error: Missing .mc file path.
+ goto :usage
+)
+if "%~2"=="" (
+ echo Error: Missing destination directory.
+ goto :usage
+)
+
+REM Set variables
+set "SRC_DIR=%~1"
+set "BIN_DIR=%~2"
+set "MC_FILE=%BIN_DIR%\wevt_netdata.mc"
+set "MAN_FILE=%BIN_DIR%\wevt_netdata_manifest.xml"
+set "BASE_NAME=wevt_netdata"
+set "SDK_PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0\x64"
+set "VS_PATH=C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.39.33519\bin\Hostx64\x64"
+
+if not exist "%SRC_DIR%" (
+ echo Error: Source directory does not exist.
+ exit /b 1
+)
+
+if not exist "%BIN_DIR%" (
+ echo Error: Destination directory does not exist.
+ exit /b 1
+)
+
+if not exist "%MC_FILE%" (
+ echo Error: %MC_FILE% not found.
+ exit /b 1
+)
+
+if not exist "%MAN_FILE%" (
+ echo Error: %MAN_FILE% not found.
+ exit /b 1
+)
+
+REM Add SDK paths to PATH
+set "PATH=C:\Windows\System32;%SDK_PATH%;%VS_PATH%;%PATH%"
+
+REM Check if commands are available
+where mc >nul 2>nul
+if %errorlevel% neq 0 (
+ echo Error: mc.exe not found in PATH.
+ exit /b 1
+)
+where rc >nul 2>nul
+if %errorlevel% neq 0 (
+ echo Error: rc.exe not found in PATH.
+ exit /b 1
+)
+where link >nul 2>nul
+if %errorlevel% neq 0 (
+ echo Error: link.exe not found in PATH.
+ exit /b 1
+)
+where wevtutil >nul 2>nul
+if %errorlevel% neq 0 (
+ echo Error: wevtutil.exe not found in PATH.
+ exit /b 1
+)
+
+REM Change to the destination directory
+cd /d "%BIN_DIR%"
+
+echo.
+echo Running mc.exe...
+mc -v -b -U "%MC_FILE%" "%MAN_FILE%"
+if %errorlevel% neq 0 (
+ echo Error: mc.exe failed on messages.
+ exit /b 1
+)
+
+if not exist "%BASE_NAME%.rc" (
+ echo Error: %BASE_NAME%.rc not found.
+ exit /b 1
+)
+
+echo.
+echo Modifying %BASE_NAME%.rc to include the manifest...
+copy "%MAN_FILE%" %BASE_NAME%_manifest.man
+echo 1 2004 "%BASE_NAME%_manifest.man" >> %BASE_NAME%.rc
+
+echo.
+echo %BASE_NAME%.rc contents:
+type %BASE_NAME%.rc
+
+echo.
+echo Running rc.exe...
+rc /v /fo %BASE_NAME%.res %BASE_NAME%.rc
+if %errorlevel% neq 0 (
+ echo Error: rc.exe failed.
+ exit /b 1
+)
+
+if not exist "%BASE_NAME%.res" (
+ echo Error: %BASE_NAME%.res not found.
+ exit /b 1
+)
+
+echo.
+echo Running link.exe...
+link /dll /noentry /machine:x64 /out:%BASE_NAME%.dll %BASE_NAME%.res
+if %errorlevel% neq 0 (
+ echo Error: link.exe failed.
+ exit /b 1
+)
+
+echo.
+echo Process completed successfully.
+exit /b 0
+
+:usage
+echo Usage: %~nx0 [path_to_mc_file] [destination_directory]
+exit /b 1
diff --git a/src/libnetdata/log/wevt_netdata_compile.sh b/src/libnetdata/log/wevt_netdata_compile.sh new file mode 100644 index 000000000..eae510645 --- /dev/null +++ b/src/libnetdata/log/wevt_netdata_compile.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +mylocation=$(dirname "${0}") + +# Check if both parameters are provided +if [ $# -ne 2 ]; then + echo "Error: Incorrect number of parameters." + echo "Usage: $0 <source_directory> <destination_directory>" + exit 1 +fi + +# Get the parameters +src_dir="$1" +dest_dir="$2" + +# Get the directory of this script +SCRIPT_DIR="$(dirname "$0")" + +# Create a temporary batch file +temp_bat=$(mktemp --suffix=.bat) + +# Write the contents to the temporary batch file +# Use cygpath directly within the heredoc +cat << EOF > "$temp_bat" +@echo off +set "PATH=%SYSTEMROOT%;$("${mylocation}/../../../packaging/windows/find-sdk-path.sh" --sdk -w);$("${mylocation}/../../../packaging/windows/find-sdk-path.sh" --visualstudio -w)" +call "$(cygpath -w -a "$SCRIPT_DIR/wevt_netdata_compile.bat")" "$(cygpath -w -a "$src_dir")" "$(cygpath -w -a "$dest_dir")" +EOF + +# Execute the temporary batch file +echo +echo "Executing Windows Batch File..." +echo +cat "$temp_bat" +cmd.exe //c "$(cygpath -w -a "$temp_bat")" +exit_status=$? + +# Remove the temporary batch file +rm "$temp_bat" + +# Check the exit status +if [ $exit_status -eq 0 ]; then + echo "nd_wevents_compile.bat executed successfully." +else + echo "nd_wevents_compile.bat failed with exit status $exit_status." +fi + +exit $exit_status diff --git a/src/libnetdata/log/wevt_netdata_install.bat b/src/libnetdata/log/wevt_netdata_install.bat new file mode 100644 index 000000000..515607592 --- /dev/null +++ b/src/libnetdata/log/wevt_netdata_install.bat @@ -0,0 +1,52 @@ +@echo off
+setlocal enabledelayedexpansion
+
+set "MAN_SRC=%~dp0wevt_netdata_manifest.xml"
+set "DLL_SRC=%~dp0wevt_netdata.dll"
+set "DLL_DST=%SystemRoot%\System32\wevt_netdata.dll"
+
+where wevtutil >nul 2>nul
+if %errorlevel% neq 0 (
+ echo Error: wevtutil.exe not found in PATH.
+ exit /b 1
+)
+
+echo.
+echo Uninstalling previous manifest (if any)...
+wevtutil um "%MAN_SRC%"
+
+echo.
+echo Copying %DLL_SRC% to %DLL_DST%
+copy /y "%DLL_SRC%" "%DLL_DST%"
+if %errorlevel% neq 0 (
+ echo Error: Failed to copy %DLL_SRC% to %DLL_DST%
+ exit /b 1
+)
+
+echo.
+echo Granting access to %DLL_DST% for Windows Event Logging...
+icacls "%DLL_DST%" /grant "NT SERVICE\EventLog":R
+if %errorlevel% neq 0 (
+ echo Error: Failed to grant access to %DLL_DST%.
+ exit /b 1
+)
+
+echo.
+echo Importing the manifest...
+wevtutil im "%MAN_SRC%" /rf:"%DLL_DST%" /mf:"%DLL_DST%"
+if %errorlevel% neq 0 (
+ echo Error: Failed to import the manifest.
+ exit /b 1
+)
+
+echo.
+echo Verifying Netdata Publisher for Event Tracing for Windows (ETW)...
+wevtutil gp "Netdata"
+if %errorlevel% neq 0 (
+ echo Error: Failed to get publisher Netdata.
+ exit /b 1
+)
+
+echo.
+echo Netdata Event Tracing for Windows manifest installed successfully.
+exit /b 0
diff --git a/src/libnetdata/log/wevt_netdata_mc_generate.c b/src/libnetdata/log/wevt_netdata_mc_generate.c new file mode 100644 index 000000000..5ab2bdf17 --- /dev/null +++ b/src/libnetdata/log/wevt_netdata_mc_generate.c @@ -0,0 +1,518 @@ +#include <stdio.h> +#include <stdlib.h> +#include <inttypes.h> +#include <ctype.h> +#include <stdbool.h> +#include <string.h> + +// from winnt.h +#define EVENTLOG_SUCCESS 0x0000 +#define EVENTLOG_ERROR_TYPE 0x0001 +#define EVENTLOG_WARNING_TYPE 0x0002 +#define EVENTLOG_INFORMATION_TYPE 0x0004 +#define EVENTLOG_AUDIT_SUCCESS 0x0008 +#define EVENTLOG_AUDIT_FAILURE 0x0010 + +// the severities we define in .mc file +#define STATUS_SEVERITY_INFORMATIONAL 0x1 +#define STATUS_SEVERITY_WARNING 0x2 +#define STATUS_SEVERITY_ERROR 0x3 + +#define FACILITY_APPLICATION 0x0fff + +#include "nd_log-common.h" +#include "nd_log-to-windows-common.h" + +const char *get_msg_symbol(MESSAGE_ID msg) { + switch(msg) { + case MSGID_MESSAGE_ONLY: + return "MESSAGE_ONLY"; + + case MSGID_MESSAGE_ERRNO: + return "MESSAGE_ERRNO"; + + case MSGID_REQUEST_ONLY: + return "REQUEST_ONLY"; + + case MSGID_ACCESS_MESSAGE: + return "ACCESS_MESSAGE"; + + case MSGID_ACCESS_MESSAGE_REQUEST: + return "ACCESS_MESSAGE_REQUEST"; + + case MSGID_ACCESS_MESSAGE_USER: + return "ACCESS_MESSAGE_USER"; + + case MSGID_ACCESS: + return "ACCESS"; + + case MSGID_ACCESS_USER: + return "ACCESS_USER"; + + case MSGID_ACCESS_FORWARDER: + return "ACCESS_FORWARDER"; + + case MSGID_ACCESS_FORWARDER_USER: + return "ACCESS_FORWARDER_USER"; + + case MSGID_ALERT_TRANSITION: + return "ALERT_TRANSITION"; + + default: + fprintf(stderr, "\n\nInvalid message id %d!\n\n\n", msg); + exit(1); + } +} + +const char *get_msg_format(MESSAGE_ID msg) { + switch(msg) { + case MSGID_MESSAGE_ONLY: + return "%2(%12): %64\r\n"; + + case MSGID_MESSAGE_ERRNO: + return "%2(%12): %64%n\r\n" + "%n\r\n" + " Unix Errno : %5%n\r\n" + " Windows Error: %6%n\r\n" + ; + + case MSGID_REQUEST_ONLY: + return "%2(%12): %63\r\n"; + + case MSGID_ACCESS_MESSAGE: + return "%64\r\n"; + + case MSGID_ACCESS_MESSAGE_REQUEST: + return "%64%n\r\n" + "%n\r\n" + " Request: %63%n\r\n" + ; + + case MSGID_ACCESS_MESSAGE_USER: + return "%64%n\r\n" + "%n\r\n" + " User: %21, role: %22, permissions: %23%n\r\n" + ; + + case MSGID_ACCESS: + return "%33 %63%n\r\n" + "%n\r\n" + " Response Code : %34%n\r\n" + " Transaction ID: %36%n\r\n" + " Source IP : %24%n\r\n" + ; + + case MSGID_ACCESS_USER: + return "%33 %63%n\r\n" + "%n\r\n" + " Response Code : %34%n\r\n" + " Transaction ID: %36%n\r\n" + " Source IP : %24%n\r\n" + " User : %21, role: %22, permissions: %23%n\r\n" + ; + + case MSGID_ACCESS_FORWARDER: + return "%33 %63%n\r\n" + "%n\r\n" + " Response Code : %34%n\r\n" + " Transaction ID: %36%n\r\n" + " Source IP : %24, For %27%n\r\n" + ; + + case MSGID_ACCESS_FORWARDER_USER: + return "%33 %63%n\r\n" + "%n\r\n" + " Response Code : %34%n\r\n" + " Transaction ID: %36%n\r\n" + " Source IP : %24, For %27%n\r\n" + " User : %21, role: %22, permissions: %23%n\r\n" + ; + + case MSGID_ALERT_TRANSITION: + return "Alert '%47' of instance '%16' on node '%15' transitioned from %57 to %56\r\n"; + + default: + fprintf(stderr, "\n\nInvalid message id %d!\n\n\n", msg); + exit(1); + } +} + +int main(int argc, const char **argv) { + (void)argc; (void)argv; + + const char *header = NULL, *footer = NULL, *s_header = NULL, *s_footer = NULL; + + bool manifest = false; + if(argc == 2 && strcmp(argv[1], "--manifest") == 0) { + manifest = true; + + header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" + "<!--\r\n" + "\r\n" + " THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT\r\n" + "\r\n" + " This XML file can be verified by running mc.exe (the MS tool) with this manifest as param.\r\n" + "\r\n" + " \"c:\\Program Files (x86)\\Windows Kits\\10\\bin\\10.0.26100.0\\x64\\mc.exe\" wevt_netdata_manifest.xml wevt_netdata.mc\r\n" + "\r\n" + " -->\r\n" + "<instrumentationManifest\r\n" + " xmlns=\"http://schemas.microsoft.com/win/2004/08/events\"\r\n" + " xmlns:win=\"http://manifests.microsoft.com/win/2004/08/windows/events\"\r\n" + " xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">\r\n" + " <instrumentation>\r\n" + " <events>\r\n" + "\r\n" + " <provider name=\"" NETDATA_ETW_PROVIDER_NAME "\"\r\n" + " guid=\"" NETDATA_ETW_PROVIDER_GUID_STR "\"\r\n" + " symbol=\"NETDATA_ETW_PROVIDER_GUID\"\r\n" + " messageFileName=\"%SystemRoot%\\System32\\wevt_netdata.dll\"\r\n" + " resourceFileName=\"%SystemRoot%\\System32\\wevt_netdata.dll\"\r\n" + " message=\"$(string.ND_PROVIDER_NAME)\">\r\n" + "\r\n" + " <!-- Define the provider sub-channels -->\r\n" + " <channels>\r\n" + " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON "\"\r\n" + " symbol=\"CHANNEL_DAEMON\"\r\n" + " type=\"Operational\"\r\n" + " message=\"$(string.Channel.Daemon)\"\r\n" + " enabled=\"true\"\r\n" + " />\r\n" + "\r\n" + " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_COLLECTORS "\"\r\n" + " symbol=\"CHANNEL_COLLECTORS\"\r\n" + " type=\"Operational\"\r\n" + " message=\"$(string.Channel.Collectors)\"\r\n" + " enabled=\"true\"\r\n" + " />\r\n" + "\r\n" + " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACCESS "\"\r\n" + " symbol=\"CHANNEL_ACCESS\"\r\n" + " type=\"Operational\"\r\n" + " message=\"$(string.Channel.Access)\"\r\n" + " enabled=\"true\"\r\n" + " />\r\n" + "\r\n" + " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_HEALTH "\"\r\n" + " symbol=\"CHANNEL_HEALTH\"\r\n" + " type=\"Operational\"\r\n" + " message=\"$(string.Channel.Health)\"\r\n" + " enabled=\"true\"\r\n" + " />\r\n" + "\r\n" + " <channel name=\"" NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACLK "\"\r\n" + " symbol=\"CHANNEL_ACLK\"\r\n" + " type=\"Operational\"\r\n" + " message=\"$(string.Channel.Aclk)\"\r\n" + " enabled=\"true\"\r\n" + " />\r\n" + " </channels>\r\n" + "\r\n" + " <levels>\r\n" + " </levels>\r\n" + "\r\n" + " <opcodes>\r\n" + " </opcodes>\r\n" + "\r\n" + " <tasks>\r\n" + " </tasks>\r\n" + "\r\n" + " <templates>\r\n" + " <template tid=\"AllFieldsTemplate\">\r\n" + " <!-- 0 (NDF_STOP) should not be here %1 is Timestamp, %64 is the Message -->\r\n" + " <data name=\"Timestamp\" inType=\"win:UnicodeString\"/> <!-- 1 (NDF_TIMESTAMP_REALTIME_USEC) -->\r\n" + " <data name=\"Program\" inType=\"win:UnicodeString\"/> <!-- 2 (NDF_SYSLOG_IDENTIFIER) -->\r\n" + " <data name=\"NetdataLogSource\" inType=\"win:UnicodeString\"/> <!-- 3 (NDF_LOG_SOURCE) -->\r\n" + " <data name=\"Level\" inType=\"win:UnicodeString\"/> <!-- 4 (NDF_PRIORITY) -->\r\n" + " <data name=\"UnixErrno\" inType=\"win:UnicodeString\"/> <!-- 5 (NDF_ERRNO) -->\r\n" + " <data name=\"WindowsLastError\" inType=\"win:UnicodeString\"/> <!-- 6 (NDF_WINERROR) -->\r\n" + " <data name=\"InvocationID\" inType=\"win:UnicodeString\"/> <!-- 7 (NDF_INVOCATION_ID) -->\r\n" + " <data name=\"CodeLine\" inType=\"win:UnicodeString\"/> <!-- 8 (NDF_LINE) -->\r\n" + " <data name=\"CodeFile\" inType=\"win:UnicodeString\"/> <!-- 9 (NDF_FILE) -->\r\n" + " <data name=\"CodeFunction\" inType=\"win:UnicodeString\"/> <!-- 10 (NDF_FUNC) -->\r\n" + " <data name=\"ThreadID\" inType=\"win:UnicodeString\"/> <!-- 11 (NDF_TID) -->\r\n" + " <data name=\"ThreadName\" inType=\"win:UnicodeString\"/> <!-- 12 (NDF_THREAD_TAG) -->\r\n" + " <data name=\"MessageID\" inType=\"win:UnicodeString\"/> <!-- 13 (NDF_MESSAGE_ID) -->\r\n" + " <data name=\"Module\" inType=\"win:UnicodeString\"/> <!-- 14 (NDF_MODULE) -->\r\n" + " <data name=\"Node\" inType=\"win:UnicodeString\"/> <!-- 15 (NDF_NIDL_NODE) -->\r\n" + " <data name=\"Instance\" inType=\"win:UnicodeString\"/> <!-- 16 (NDF_NIDL_INSTANCE) -->\r\n" + " <data name=\"Context\" inType=\"win:UnicodeString\"/> <!-- 17 (NDF_NIDL_CONTEXT) -->\r\n" + " <data name=\"Dimension\" inType=\"win:UnicodeString\"/> <!-- 18 (NDF_NIDL_DIMENSION) -->\r\n" + " <data name=\"SourceTransport\" inType=\"win:UnicodeString\"/> <!-- 19 (NDF_SRC_TRANSPORT) -->\r\n" + " <data name=\"AccountID\" inType=\"win:UnicodeString\"/> <!-- 20 (NDF_ACCOUNT_ID) -->\r\n" + " <data name=\"UserName\" inType=\"win:UnicodeString\"/> <!-- 21 (NDF_USER_NAME) -->\r\n" + " <data name=\"UserRole\" inType=\"win:UnicodeString\"/> <!-- 22 (NDF_USER_ROLE) -->\r\n" + " <data name=\"UserPermissions\" inType=\"win:UnicodeString\"/> <!-- 23 (NDF_USER_ACCESS) -->\r\n" + " <data name=\"SourceIP\" inType=\"win:UnicodeString\"/> <!-- 24 (NDF_SRC_IP) -->\r\n" + " <data name=\"SourceForwardedHost\" inType=\"win:UnicodeString\"/> <!-- 25 (NDF_SRC_PORT) -->\r\n" + " <data name=\"SourceForwardedFor\" inType=\"win:UnicodeString\"/> <!-- 26 (NDF_SRC_FORWARDED_HOST) -->\r\n" + " <data name=\"SourcePort\" inType=\"win:UnicodeString\"/> <!-- 27 (NDF_SRC_FORWARDED_FOR) -->\r\n" + " <data name=\"SourceCapabilities\" inType=\"win:UnicodeString\"/> <!-- 28 (NDF_SRC_CAPABILITIES) -->\r\n" + " <data name=\"DestinationTransport\" inType=\"win:UnicodeString\"/> <!-- 29 (NDF_DST_TRANSPORT) -->\r\n" + " <data name=\"DestinationIP\" inType=\"win:UnicodeString\"/> <!-- 30 (NDF_DST_IP) -->\r\n" + " <data name=\"DestinationPort\" inType=\"win:UnicodeString\"/> <!-- 31 (NDF_DST_PORT) -->\r\n" + " <data name=\"DestinationCapabilities\" inType=\"win:UnicodeString\"/> <!-- 32 (NDF_DST_CAPABILITIES) -->\r\n" + " <data name=\"RequestMethod\" inType=\"win:UnicodeString\"/> <!-- 33 (NDF_REQUEST_METHOD) -->\r\n" + " <data name=\"ResponseCode\" inType=\"win:UnicodeString\"/> <!-- 34 (NDF_RESPONSE_CODE) -->\r\n" + " <data name=\"ConnectionID\" inType=\"win:UnicodeString\"/> <!-- 35 (NDF_CONNECTION_ID) -->\r\n" + " <data name=\"TransactionID\" inType=\"win:UnicodeString\"/> <!-- 36 (NDF_TRANSACTION_ID) -->\r\n" + " <data name=\"ResponseSentBytes\" inType=\"win:UnicodeString\"/> <!-- 37 (NDF_RESPONSE_SENT_BYTES) -->\r\n" + " <data name=\"ResponseSizeBytes\" inType=\"win:UnicodeString\"/> <!-- 38 (NDF_RESPONSE_SIZE_BYTES) -->\r\n" + " <data name=\"ResponsePreparationTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 39 (NDF_RESPONSE_PREPARATION_TIME_USEC) -->\r\n" + " <data name=\"ResponseSentTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 40 (NDF_RESPONSE_SENT_TIME_USEC) -->\r\n" + " <data name=\"ResponseTotalTimeUsec\" inType=\"win:UnicodeString\"/> <!-- 41 (NDF_RESPONSE_TOTAL_TIME_USEC) -->\r\n" + " <data name=\"AlertID\" inType=\"win:UnicodeString\"/> <!-- 42 (NDF_ALERT_ID) -->\r\n" + " <data name=\"AlertUniqueID\" inType=\"win:UnicodeString\"/> <!-- 43 (NDF_ALERT_UNIQUE_ID) -->\r\n" + " <data name=\"AlertTransitionID\" inType=\"win:UnicodeString\"/> <!-- 44 (NDF_ALERT_TRANSITION_ID) -->\r\n" + " <data name=\"AlertEventID\" inType=\"win:UnicodeString\"/> <!-- 45 (NDF_ALERT_EVENT_ID) -->\r\n" + " <data name=\"AlertConfig\" inType=\"win:UnicodeString\"/> <!-- 46 (NDF_ALERT_CONFIG_HASH) -->\r\n" + " <data name=\"AlertName\" inType=\"win:UnicodeString\"/> <!-- 47 (NDF_ALERT_NAME) -->\r\n" + " <data name=\"AlertClass\" inType=\"win:UnicodeString\"/> <!-- 48 (NDF_ALERT_CLASS) -->\r\n" + " <data name=\"AlertComponent\" inType=\"win:UnicodeString\"/> <!-- 49 (NDF_ALERT_COMPONENT) -->\r\n" + " <data name=\"AlertType\" inType=\"win:UnicodeString\"/> <!-- 50 (NDF_ALERT_TYPE) -->\r\n" + " <data name=\"AlertExec\" inType=\"win:UnicodeString\"/> <!-- 51 (NDF_ALERT_EXEC) -->\r\n" + " <data name=\"AlertRecipient\" inType=\"win:UnicodeString\"/> <!-- 52 (NDF_ALERT_RECIPIENT) -->\r\n" + " <data name=\"AlertDuration\" inType=\"win:UnicodeString\"/> <!-- 53 (NDF_ALERT_DURATION) -->\r\n" + " <data name=\"AlertValue\" inType=\"win:UnicodeString\"/> <!-- 54 (NDF_ALERT_VALUE) -->\r\n" + " <data name=\"AlertOldValue\" inType=\"win:UnicodeString\"/> <!-- 55 (NDF_ALERT_VALUE_OLD) -->\r\n" + " <data name=\"AlertStatus\" inType=\"win:UnicodeString\"/> <!-- 56 (NDF_ALERT_STATUS) -->\r\n" + " <data name=\"AlertOldStatus\" inType=\"win:UnicodeString\"/> <!-- 57 (NDF_ALERT_STATUS_OLD) -->\r\n" + " <data name=\"Source\" inType=\"win:UnicodeString\"/> <!-- 58 (NDF_ALERT_SOURCE) -->\r\n" + " <data name=\"AlertUnits\" inType=\"win:UnicodeString\"/> <!-- 59 (NDF_ALERT_UNITS) -->\r\n" + " <data name=\"AlertSummary\" inType=\"win:UnicodeString\"/> <!-- 60 (NDF_ALERT_SUMMARY) -->\r\n" + " <data name=\"AlertInfo\" inType=\"win:UnicodeString\"/> <!-- 61 (NDF_ALERT_INFO) -->\r\n" + " <data name=\"AlertNotificationTime\" inType=\"win:UnicodeString\"/> <!-- 62 (NDF_ALERT_NOTIFICATION_REALTIME_USEC) -->\r\n" + " <data name=\"Request\" inType=\"win:UnicodeString\"/> <!-- 63 (NDF_REQUEST) -->\r\n" + " <data name=\"Message\" inType=\"win:UnicodeString\"/> <!-- 64 (NDF_MESSAGE) -->\r\n" + " </template>\r\n" + " </templates>\r\n" + "\r\n" + " <events>\r\n" + ; + + footer = " </events>\r\n" + " </provider>\r\n" + " </events>\r\n" + " </instrumentation>\r\n" + ; + + s_header = " <localization>\r\n" + " <resources culture=\"en-US\">\r\n" + " <stringTable>\r\n" + " <string id=\"ND_PROVIDER_NAME\" value=\"" NETDATA_ETW_PROVIDER_NAME "\"/>\r\n" + "\r\n" + " <string id=\"Channel.Daemon\" value=\"Daemon\"/>\r\n" + " <string id=\"Channel.Collectors\" value=\"Collectors\"/>\r\n" + " <string id=\"Channel.Access\" value=\"Access\"/>\r\n" + " <string id=\"Channel.Health\" value=\"Health\"/>\r\n" + " <string id=\"Channel.Aclk\" value=\"Aclk\"/>\r\n" + "\r\n" + ; + + s_footer = " </stringTable>\r\n" + " </resources>\r\n" + " </localization>\r\n" + "</instrumentationManifest>\r\n" + ; + } + else { + header = ";// THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT\r\n" + "\r\n" + "MessageIdTypedef=DWORD\r\n" + "\r\n" + "SeverityNames=(\r\n" + " Informational=0x1:STATUS_SEVERITY_INFORMATIONAL\r\n" + " Warning=0x2:STATUS_SEVERITY_WARNING\r\n" + " Error=0x3:STATUS_SEVERITY_ERROR\r\n" + " )\r\n" + "\r\n" + "FacilityNames=(\r\n" + " " NETDATA_CHANNEL_NAME "=0x0FFF:FACILITY_NETDATA\r\n" + " )\r\n" + "\r\n" + "LanguageNames=(\r\n" + " English=0x409:MSG00409\r\n" + " )\r\n" + "\r\n" + ; + + footer = ""; + } + + bool done[UINT16_MAX] = { 0 }; + char symbol[1024]; + + printf("%s", header); + for(size_t src = 1; src < _NDLS_MAX ;src++) { + for(size_t pri = 0; pri < _NDLP_MAX ;pri++) { + uint8_t severity = get_severity_from_priority(pri); + + for(size_t msg = 1; msg < _MSGID_MAX ;msg++) { + + if(src >= 16) { + fprintf(stderr, "\n\nSource %zu is bigger than 4 bits!\n\n", src); + return 1; + } + + if(pri >= 16) { + fprintf(stderr, "\n\nPriority %zu is bigger than 4 bits!\n\n", pri); + return 1; + } + + if(msg >= 256) { + fprintf(stderr, "\n\nMessageID %zu is bigger than 8 bits!\n\n", msg); + return 1; + } + + uint16_t eventID = construct_event_code(src, pri, msg); + if((eventID & 0xFFFF) != eventID) { + fprintf(stderr, "\n\nEventID 0x%x is bigger than 16 bits!\n\n", eventID); + return 1; + } + + if(done[eventID]) continue; + done[eventID] = true; + + const char *level = get_level_from_priority_str(pri); + const char *pri_txt; + switch(pri) { + case NDLP_EMERG: + pri_txt = "EMERG"; + break; + + case NDLP_CRIT: + pri_txt = "CRIT"; + break; + + case NDLP_ALERT: + pri_txt = "ALERT"; + break; + + case NDLP_ERR: + pri_txt = "ERR"; + break; + + case NDLP_WARNING: + pri_txt = "WARN"; + break; + + case NDLP_INFO: + pri_txt = "INFO"; + break; + + case NDLP_NOTICE: + pri_txt = "NOTICE"; + break; + + case NDLP_DEBUG: + pri_txt = "DEBUG"; + break; + + default: + fprintf(stderr, "\n\nInvalid priority %zu!\n\n\n", pri); + return 1; + } + + const char *channel; + const char *src_txt; + switch(src) { + case NDLS_COLLECTORS: + src_txt = "COLLECTORS"; + channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_COLLECTORS; + break; + + case NDLS_ACCESS: + src_txt = "ACCESS"; + channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACCESS; + break; + + case NDLS_HEALTH: + src_txt = "HEALTH"; + channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_HEALTH; + break; + + case NDLS_DEBUG: + src_txt = "DEBUG"; + channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON; + break; + + case NDLS_DAEMON: + src_txt = "DAEMON"; + channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_DAEMON; + break; + + case NDLS_ACLK: + src_txt = "ACLK"; + channel = NETDATA_ETW_CHANNEL_NAME "/" NETDATA_ETW_SUBCHANNEL_ACLK; + break; + + default: + fprintf(stderr, "\n\nInvalid source %zu!\n\n\n", src); + return 1; + } + + const char *msg_txt = get_msg_symbol(msg); + const char *format = get_msg_format(msg); + + const char *severity_txt; + switch (severity) { + case STATUS_SEVERITY_INFORMATIONAL: + severity_txt = "Informational"; + break; + + case STATUS_SEVERITY_ERROR: + severity_txt = "Error"; + break; + + case STATUS_SEVERITY_WARNING: + severity_txt = "Warning"; + break; + + default: + fprintf(stderr, "\n\nInvalid severity id %u!\n\n\n", severity); + return 1; + } + + if(manifest) + snprintf(symbol, sizeof(symbol), "ED_%s_%s_%s", src_txt, pri_txt, msg_txt); + else + snprintf(symbol, sizeof(symbol), "MC_%s_%s_%s", src_txt, pri_txt, msg_txt); + + if(manifest) + printf(" <event symbol=\"%s\"\r\n" + " value=\"0x%x\"\r\n" + " message=\"$(string.msg.MAN_%s)\"\r\n" + " channel=\"%s\"\r\n" + " level=\"%s\"\r\n" + " task=\"win:None\"\r\n" + " opcode=\"win:Info\"\r\n" + " template=\"AllFieldsTemplate\"/>\r\n\r\n", + symbol, eventID, msg_txt, channel, level); + else + printf("MessageId=0x%x\r\n" + "Severity=%s\r\n" + "Facility=" NETDATA_CHANNEL_NAME "\r\n" + "SymbolicName=%s\r\n" + "Language=English\r\n" + "%s" + ".\r\n" + "\r\n", + eventID, severity_txt, symbol, format); + } + } + } + printf("%s", footer); + + if(s_header) { + printf("%s", s_header); + + for(size_t msg = 1; msg < _MSGID_MAX ;msg++) { + const char *msg_txt = get_msg_symbol(msg); + const char *format = get_msg_format(msg); + printf(" <string id=\"msg.MAN_%s\" value=\"%s\"/>\r\n", msg_txt, format); + } + + printf("%s", s_footer); + } +} + diff --git a/src/libnetdata/maps/system-groups.h b/src/libnetdata/maps/system-groups.h deleted file mode 100644 index fd042cd4e..000000000 --- a/src/libnetdata/maps/system-groups.h +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_SYSTEM_GROUPS_H -#define NETDATA_SYSTEM_GROUPS_H - -#include "libnetdata/libnetdata.h" - -// -------------------------------------------------------------------------------------------------------------------- -// hashtable for caching uid to username mappings -// key is the uid, value is username (STRING) - -#define SIMPLE_HASHTABLE_VALUE_TYPE STRING -#define SIMPLE_HASHTABLE_NAME _GROUPNAMES_CACHE -#include "libnetdata/simple_hashtable.h" - -typedef struct groupnames_cache { - SPINLOCK spinlock; - SIMPLE_HASHTABLE_GROUPNAMES_CACHE ht; -} GROUPNAMES_CACHE; - -static inline STRING *system_groupnames_cache_lookup_gid(GROUPNAMES_CACHE *gc, gid_t gid) { - spinlock_lock(&gc->spinlock); - - SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_get_slot_GROUPNAMES_CACHE(&gc->ht, gid, &gid, true); - STRING *g = SIMPLE_HASHTABLE_SLOT_DATA(sl); - if(!g) { - char tmp[1024 + 1]; - struct group grp, *result = NULL; - - if (getgrgid_r(gid, &grp, tmp, sizeof(tmp), &result) != 0 || !result || !grp.gr_name || !(*grp.gr_name)) { - char name[50]; - snprintfz(name, sizeof(name), "%u", gid); - g = string_strdupz(name); - } - else - g = string_strdupz(grp.gr_name); - - simple_hashtable_set_slot_GROUPNAMES_CACHE(&gc->ht, sl, gid, g); - } - - g = string_dup(g); - spinlock_unlock(&gc->spinlock); - return g; -} - -static inline GROUPNAMES_CACHE *system_groupnames_cache_init(void) { - GROUPNAMES_CACHE *gc = callocz(1, sizeof(*gc)); - spinlock_init(&gc->spinlock); - simple_hashtable_init_GROUPNAMES_CACHE(&gc->ht, 100); - return gc; -} - -static inline void system_groupnames_cache_destroy(GROUPNAMES_CACHE *gc) { - spinlock_lock(&gc->spinlock); - - for(SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_first_read_only_GROUPNAMES_CACHE(&gc->ht); - sl; - sl = simple_hashtable_next_read_only_GROUPNAMES_CACHE(&gc->ht, sl)) { - STRING *u = SIMPLE_HASHTABLE_SLOT_DATA(sl); - string_freez(u); - } - - simple_hashtable_destroy_GROUPNAMES_CACHE(&gc->ht); - freez(gc); -} - -#endif //NETDATA_SYSTEM_GROUPS_H diff --git a/src/libnetdata/maps/system-users.h b/src/libnetdata/maps/system-users.h deleted file mode 100644 index 5f7dfae1a..000000000 --- a/src/libnetdata/maps/system-users.h +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_SYSTEM_USERS_H -#define NETDATA_SYSTEM_USERS_H - -#include "libnetdata/libnetdata.h" - -// -------------------------------------------------------------------------------------------------------------------- -// hashtable for caching uid to username mappings -// key is the uid, value is username (STRING) - -#define SIMPLE_HASHTABLE_VALUE_TYPE STRING -#define SIMPLE_HASHTABLE_NAME _USERNAMES_CACHE -#include "libnetdata/simple_hashtable.h" - -typedef struct usernames_cache { - SPINLOCK spinlock; - SIMPLE_HASHTABLE_USERNAMES_CACHE ht; -} USERNAMES_CACHE; - -static inline STRING *system_usernames_cache_lookup_uid(USERNAMES_CACHE *uc, uid_t uid) { - spinlock_lock(&uc->spinlock); - - SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_get_slot_USERNAMES_CACHE(&uc->ht, uid, &uid, true); - STRING *u = SIMPLE_HASHTABLE_SLOT_DATA(sl); - if(!u) { - char tmp[1024 + 1]; - struct passwd pw, *result = NULL; - - if (getpwuid_r(uid, &pw, tmp, sizeof(tmp), &result) != 0 || !result || !pw.pw_name || !(*pw.pw_name)) { - char name[50]; - snprintfz(name, sizeof(name), "%u", uid); - u = string_strdupz(name); - } - else - u = string_strdupz(pw.pw_name); - - simple_hashtable_set_slot_USERNAMES_CACHE(&uc->ht, sl, uid, u); - } - - u = string_dup(u); - spinlock_unlock(&uc->spinlock); - return u; -} - -static inline USERNAMES_CACHE *system_usernames_cache_init(void) { - USERNAMES_CACHE *uc = callocz(1, sizeof(*uc)); - spinlock_init(&uc->spinlock); - simple_hashtable_init_USERNAMES_CACHE(&uc->ht, 100); - return uc; -} - -static inline void system_usernames_cache_destroy(USERNAMES_CACHE *uc) { - spinlock_lock(&uc->spinlock); - - for(SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_first_read_only_USERNAMES_CACHE(&uc->ht); - sl; - sl = simple_hashtable_next_read_only_USERNAMES_CACHE(&uc->ht, sl)) { - STRING *u = SIMPLE_HASHTABLE_SLOT_DATA(sl); - string_freez(u); - } - - simple_hashtable_destroy_USERNAMES_CACHE(&uc->ht); - freez(uc); -} - -#endif //NETDATA_SYSTEM_USERS_H diff --git a/src/libnetdata/onewayalloc/README.md b/src/libnetdata/onewayalloc/README.md index 082085db0..fbaad0aea 100644 --- a/src/libnetdata/onewayalloc/README.md +++ b/src/libnetdata/onewayalloc/README.md @@ -1,12 +1,3 @@ -<!-- -title: "One Way Allocator" -custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/libnetdata/onewayalloc/README.md" -sidebar_label: "One way allocator" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # One Way Allocator This is a very fast single-threaded-only memory allocator, that minimized system calls diff --git a/src/libnetdata/os/close_range.c b/src/libnetdata/os/close_range.c index 56d5c2527..2ee5837ee 100644 --- a/src/libnetdata/os/close_range.c +++ b/src/libnetdata/os/close_range.c @@ -7,6 +7,12 @@ static int fd_is_valid(int fd) { return fcntl(fd, F_GETFD) != -1 || errno != EBADF; } +static void setcloexec(int fd) { + int flags = fcntl(fd, F_GETFD); + if (flags != -1) + (void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC); +} + int os_get_fd_open_max(void) { static int fd_open_max = CLOSE_RANGE_FD_MAX; @@ -33,9 +39,9 @@ int os_get_fd_open_max(void) { return fd_open_max; } -void os_close_range(int first, int last) { +void os_close_range(int first, int last, int flags) { #if defined(HAVE_CLOSE_RANGE) - if(close_range(first, last, 0) == 0) return; + if(close_range(first, last, flags) == 0) return; #endif #if defined(OS_LINUX) @@ -44,8 +50,12 @@ void os_close_range(int first, int last) { struct dirent *entry; while ((entry = readdir(dir)) != NULL) { int fd = str2i(entry->d_name); - if (fd >= first && (last == CLOSE_RANGE_FD_MAX || fd <= last) && fd_is_valid(fd)) - (void)close(fd); + if (fd >= first && (last == CLOSE_RANGE_FD_MAX || fd <= last) && fd_is_valid(fd)) { + if(flags & CLOSE_RANGE_CLOEXEC) + setcloexec(fd); + else + (void)close(fd); + } } closedir(dir); return; @@ -57,7 +67,12 @@ void os_close_range(int first, int last) { last = os_get_fd_open_max(); for (int fd = first; fd <= last; fd++) { - if (fd_is_valid(fd)) (void)close(fd); + if (fd_is_valid(fd)) { + if(flags & CLOSE_RANGE_CLOEXEC) + setcloexec(fd); + else + (void)close(fd); + } } } @@ -67,9 +82,9 @@ static int compare_ints(const void *a, const void *b) { return (int_a > int_b) - (int_a < int_b); } -void os_close_all_non_std_open_fds_except(const int fds[], size_t fds_num) { +void os_close_all_non_std_open_fds_except(const int fds[], size_t fds_num, int flags) { if (fds_num == 0 || fds == NULL) { - os_close_range(STDERR_FILENO + 1, CLOSE_RANGE_FD_MAX); + os_close_range(STDERR_FILENO + 1, CLOSE_RANGE_FD_MAX, flags); return; } @@ -89,10 +104,10 @@ void os_close_all_non_std_open_fds_except(const int fds[], size_t fds_num) { // call os_close_range() as many times as needed for (; i < fds_num; i++) { if (fds_copy[i] > start) - os_close_range(start, fds_copy[i] - 1); + os_close_range(start, fds_copy[i] - 1, flags); start = fds_copy[i] + 1; } - os_close_range(start, CLOSE_RANGE_FD_MAX); + os_close_range(start, CLOSE_RANGE_FD_MAX, flags); } diff --git a/src/libnetdata/os/close_range.h b/src/libnetdata/os/close_range.h index e3cb93798..7914ac3f6 100644 --- a/src/libnetdata/os/close_range.h +++ b/src/libnetdata/os/close_range.h @@ -5,8 +5,16 @@ #define CLOSE_RANGE_FD_MAX (int)(~0U) +#ifndef CLOSE_RANGE_UNSHARE +#define CLOSE_RANGE_UNSHARE (1U << 1) +#endif + +#ifndef CLOSE_RANGE_CLOEXEC +#define CLOSE_RANGE_CLOEXEC (1U << 2) +#endif + int os_get_fd_open_max(void); -void os_close_range(int first, int last); -void os_close_all_non_std_open_fds_except(const int fds[], size_t fds_num); +void os_close_range(int first, int last, int flags); +void os_close_all_non_std_open_fds_except(const int fds[], size_t fds_num, int flags); #endif //CLOSE_RANGE_H diff --git a/src/libnetdata/os/get_system_cpus.c b/src/libnetdata/os/get_system_cpus.c index 5a76d8aa5..f8234d8bc 100644 --- a/src/libnetdata/os/get_system_cpus.c +++ b/src/libnetdata/os/get_system_cpus.c @@ -2,10 +2,6 @@ #include "../libnetdata.h" -#if defined(OS_WINDOWS) -#include <windows.h> -#endif - #define CPUS_FOR_COLLECTORS 0 #define CPUS_FOR_NETDATA 1 @@ -82,7 +78,14 @@ long os_get_system_cpus_cached(bool cache, bool for_netdata) { SYSTEM_INFO sysInfo; GetSystemInfo(&sysInfo); - return (long) sysInfo.dwNumberOfProcessors; + processors[index] = sysInfo.dwNumberOfProcessors; + + if(processors[index] < 1) { + processors[index] = 1; + netdata_log_error("Assuming system has %ld processors.", processors[index]); + } + + return processors[index]; #else diff --git a/src/libnetdata/os/gettid.c b/src/libnetdata/os/gettid.c index 273c428f8..d61819445 100644 --- a/src/libnetdata/os/gettid.c +++ b/src/libnetdata/os/gettid.c @@ -2,10 +2,6 @@ #include "../libnetdata.h" -#if defined(OS_WINDOWS) -#include <windows.h> -#endif - pid_t os_gettid(void) { #if defined(HAVE_GETTID) return gettid(); @@ -30,4 +26,9 @@ pid_t gettid_cached(void) { gettid_cached_tid = os_gettid(); return gettid_cached_tid; -}
\ No newline at end of file +} + +pid_t gettid_uncached(void) { + gettid_cached_tid = 0; + return gettid_cached(); +} diff --git a/src/libnetdata/os/gettid.h b/src/libnetdata/os/gettid.h index f04d9c365..6debfd928 100644 --- a/src/libnetdata/os/gettid.h +++ b/src/libnetdata/os/gettid.h @@ -7,5 +7,6 @@ pid_t os_gettid(void); pid_t gettid_cached(void); +pid_t gettid_uncached(void); #endif //NETDATA_GETTID_H diff --git a/src/libnetdata/os/os-windows-wrappers.c b/src/libnetdata/os/os-windows-wrappers.c index 64076eae2..a79ae41f2 100644 --- a/src/libnetdata/os/os-windows-wrappers.c +++ b/src/libnetdata/os/os-windows-wrappers.c @@ -3,8 +3,6 @@ #include "../libnetdata.h" #if defined(OS_WINDOWS) -#include <windows.h> - long netdata_registry_get_dword_from_open_key(unsigned int *out, void *lKey, char *name) { DWORD length = 260; @@ -58,4 +56,42 @@ bool netdata_registry_get_string(char *out, unsigned int length, void *hKey, cha return status; } +bool EnableWindowsPrivilege(const char *privilegeName) { + HANDLE hToken; + LUID luid; + TOKEN_PRIVILEGES tkp; + + // Open the process token with appropriate access rights + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken)) + return false; + + // Lookup the LUID for the specified privilege + if (!LookupPrivilegeValue(NULL, privilegeName, &luid)) { + CloseHandle(hToken); // Close the token handle before returning + return false; + } + + // Set up the TOKEN_PRIVILEGES structure + tkp.PrivilegeCount = 1; + tkp.Privileges[0].Luid = luid; + tkp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + + // Adjust the token's privileges + if (!AdjustTokenPrivileges(hToken, FALSE, &tkp, sizeof(tkp), NULL, NULL)) { + CloseHandle(hToken); // Close the token handle before returning + return false; + } + + // Check if AdjustTokenPrivileges succeeded + if (GetLastError() == ERROR_NOT_ALL_ASSIGNED) { + CloseHandle(hToken); // Close the token handle before returning + return false; + } + + // Close the handle to the token after success + CloseHandle(hToken); + + return true; +} + #endif diff --git a/src/libnetdata/os/os-windows-wrappers.h b/src/libnetdata/os/os-windows-wrappers.h index 5ae73043a..30e1fc50d 100644 --- a/src/libnetdata/os/os-windows-wrappers.h +++ b/src/libnetdata/os/os-windows-wrappers.h @@ -14,5 +14,7 @@ bool netdata_registry_get_dword(unsigned int *out, void *hKey, char *subKey, cha long netdata_registry_get_string_from_open_key(char *out, unsigned int length, void *lKey, char *name); bool netdata_registry_get_string(char *out, unsigned int length, void *hKey, char *subKey, char *name); +bool EnableWindowsPrivilege(const char *privilegeName); + #endif // OS_WINDOWS #endif //NETDATA_OS_WINDOWS_WRAPPERS_H diff --git a/src/libnetdata/os/os.c b/src/libnetdata/os/os.c index 1caa25f85..780801fa1 100644 --- a/src/libnetdata/os/os.c +++ b/src/libnetdata/os/os.c @@ -6,12 +6,13 @@ // system functions // to retrieve settings of the system -unsigned int system_hz; +unsigned int system_hz = 100; void os_get_system_HZ(void) { long ticks; if ((ticks = sysconf(_SC_CLK_TCK)) == -1) { netdata_log_error("Cannot get system clock ticks"); + ticks = 100; } system_hz = (unsigned int) ticks; diff --git a/src/libnetdata/os/os.h b/src/libnetdata/os/os.h index 15e74faa7..1846afb6d 100644 --- a/src/libnetdata/os/os.h +++ b/src/libnetdata/os/os.h @@ -7,6 +7,8 @@ #include <sys/syscall.h> #endif +#include "random.h" +#include "timestamps.h" #include "setproctitle.h" #include "close_range.h" #include "setresuid.h" @@ -16,12 +18,21 @@ #include "gettid.h" #include "get_pid_max.h" #include "get_system_cpus.h" -#include "tinysleep.h" +#include "sleep.h" #include "uuid_generate.h" #include "setenv.h" #include "os-freebsd-wrappers.h" #include "os-macos-wrappers.h" #include "os-windows-wrappers.h" +#include "system-maps/cached-uid-username.h" +#include "system-maps/cached-gid-groupname.h" +#include "system-maps/cache-host-users-and-groups.h" +#include "system-maps/cached-sid-username.h" +#include "windows-perflib/perflib.h" + +// this includes windows.h to the whole of netdata +// so various conflicts arise +// #include "windows-wmi/windows-wmi.h" // ===================================================================================================================== // common defs for Apple/FreeBSD/Linux diff --git a/src/libnetdata/os/random.c b/src/libnetdata/os/random.c new file mode 100644 index 000000000..125e1cdb5 --- /dev/null +++ b/src/libnetdata/os/random.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "libnetdata/libnetdata.h" + +#if !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_RAND_S) +static SPINLOCK random_lock = NETDATA_SPINLOCK_INITIALIZER; +static __attribute__((constructor)) void random_seed() { + // Use current time and process ID to create a high-entropy seed + struct timeval tv; + gettimeofday(&tv, NULL); + + uint32_t seed = (uint32_t)(tv.tv_sec ^ tv.tv_usec ^ getpid()); + + // Seed the random number generator + srandom(seed); +} + +static inline void random_bytes(void *buf, size_t bytes) { + spinlock_lock(&random_lock); + while (bytes > 0) { + if (bytes >= sizeof(uint32_t)) { + // Generate 4 bytes at a time + uint32_t temp = random(); + memcpy(buf, &temp, sizeof(uint32_t)); + buf = (uint8_t *)buf + sizeof(uint32_t); + bytes -= sizeof(uint32_t); + } else if (bytes >= sizeof(uint16_t)) { + // Generate 2 bytes at a time + uint16_t temp = random(); + memcpy(buf, &temp, sizeof(uint16_t)); + buf = (uint8_t *)buf + sizeof(uint16_t); + bytes -= sizeof(uint16_t); + } else { + // Generate remaining bytes + uint32_t temp = random(); + for (size_t i = 0; i < bytes; i++) { + ((uint8_t *)buf)[i] = temp & 0xFF; + temp >>= 8; + } + bytes = 0; + } + } + spinlock_unlock(&random_lock); +} + +#if defined(HAVE_GETRANDOM) +#include <sys/random.h> +static inline void getrandom_bytes(void *buf, size_t bytes) { + ssize_t result; + while (bytes > 0) { + result = getrandom(buf, bytes, 0); + if (result == -1) { + if (errno == EINTR) { + // Interrupted, retry + continue; + } else if (errno == EAGAIN) { + // Insufficient entropy; wait and retry + tinysleep(); + continue; + } else { + // fallback to RAND_bytes + random_bytes(buf, bytes); + return; + } + } + buf = (uint8_t *)buf + result; + bytes -= result; + } +} +#endif // HAVE_GETRANDOM +#endif // !HAVE_ARC4RANDOM_BUF && !HAVE_RAND_S + +#if defined(HAVE_RAND_S) +static inline void rand_s_bytes(void *buf, size_t bytes) { + while (bytes > 0) { + if (bytes >= sizeof(unsigned int)) { + unsigned int temp; + rand_s(&temp); + memcpy(buf, &temp, sizeof(unsigned int)); + buf = (uint8_t *)buf + sizeof(unsigned int); + bytes -= sizeof(unsigned int); + } else if (bytes >= sizeof(uint16_t)) { + // Generate 2 bytes at a time + unsigned int t; + rand_s(&t); + uint16_t temp = t; + memcpy(buf, &temp, sizeof(uint16_t)); + buf = (uint8_t *)buf + sizeof(uint16_t); + bytes -= sizeof(uint16_t); + } else { + // Generate remaining bytes + unsigned int temp; + rand_s(&temp); + for (size_t i = 0; i < sizeof(temp) && i < bytes; i++) { + ((uint8_t *)buf)[0] = temp & 0xFF; + temp >>= 8; + buf = (uint8_t *)buf + 1; + bytes--; + } + } + } +} +#endif + +inline void os_random_bytes(void *buf, size_t bytes) { +#if defined(HAVE_ARC4RANDOM_BUF) + arc4random_buf(buf, bytes); +#else + + if(RAND_bytes((unsigned char *)buf, bytes) == 1) + return; + +#if defined(HAVE_GETRANDOM) + getrandom_bytes(buf, bytes); +#elif defined(HAVE_RAND_S) + rand_s_bytes(buf, bytes); +#else + random_bytes(buf, bytes); +#endif +#endif +} + +// Generate an 8-bit random number +uint8_t os_random8(void) { + uint8_t value; + os_random_bytes(&value, sizeof(value)); + return value; +} + +// Generate a 16-bit random number +uint16_t os_random16(void) { + uint16_t value; + os_random_bytes(&value, sizeof(value)); + return value; +} + +// Generate a 32-bit random number +uint32_t os_random32(void) { + uint32_t value; + os_random_bytes(&value, sizeof(value)); + return value; +} + +// Generate a 64-bit random number +uint64_t os_random64(void) { + uint64_t value; + os_random_bytes(&value, sizeof(value)); + return value; +} + +/* + * Rejection Sampling + * To reduce bias, we can use rejection sampling without creating an infinite loop. + * This technique works by discarding values that would introduce bias, but limiting + * the number of retries to avoid infinite loops. +*/ + +// Calculate an upper limit so that the range evenly divides into max. +// Any values greater than this limit would introduce bias, so we discard them. +#define MAX_RETRIES 10 +#define os_random_rejection_sampling_X(type, type_max, func, max) \ + ({ \ + size_t retries = 0; \ + type value, upper_limit = type_max - (type_max % (max)); \ + while ((value = func()) >= upper_limit && retries++ < MAX_RETRIES); \ + value % (max); \ + }) + +uint64_t os_random(uint64_t max) { + if (max <= 1) return 0; + +#if defined(HAVE_ARC4RANDOM_UNIFORM) + if(max <= UINT32_MAX) + // this is not biased + return arc4random_uniform(max); +#endif + + if ((max & (max - 1)) == 0) { + // max is a power of 2 + // use bitmasking to directly generate an unbiased random number + + if (max <= UINT8_MAX) + return os_random8() & (max - 1); + else if (max <= UINT16_MAX) + return os_random16() & (max - 1); + else if (max <= UINT32_MAX) + return os_random32() & (max - 1); + else + return os_random64() & (max - 1); + } + + if (max <= UINT8_MAX) + return os_random_rejection_sampling_X(uint8_t, UINT8_MAX, os_random8, max); + else if (max <= UINT16_MAX) + return os_random_rejection_sampling_X(uint16_t, UINT16_MAX, os_random16, max); + else if (max <= UINT32_MAX) + return os_random_rejection_sampling_X(uint32_t, UINT32_MAX, os_random32, max); + else + return os_random_rejection_sampling_X(uint64_t, UINT64_MAX, os_random64, max); +} diff --git a/src/libnetdata/os/random.h b/src/libnetdata/os/random.h new file mode 100644 index 000000000..d09cee5ea --- /dev/null +++ b/src/libnetdata/os/random.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_RANDOM_H +#define NETDATA_RANDOM_H + +#include "libnetdata/common.h" + +// fill a buffer with random bytes +void os_random_bytes(void *buf, size_t bytes); + +// return a random number 0 to max - 1 +uint64_t os_random(uint64_t max); + +uint8_t os_random8(void); +uint16_t os_random16(void); +uint32_t os_random32(void); +uint64_t os_random64(void); + +#endif //NETDATA_RANDOM_H diff --git a/src/libnetdata/os/setenv.c b/src/libnetdata/os/setenv.c index 5aa4302b8..c0de1b4b6 100644 --- a/src/libnetdata/os/setenv.c +++ b/src/libnetdata/os/setenv.c @@ -1,13 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later
-#include "config.h"
+#include "libnetdata/libnetdata.h"
#ifndef HAVE_SETENV
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
int os_setenv(const char *name, const char *value, int overwrite) {
char *env_var;
int result;
@@ -28,3 +23,21 @@ int os_setenv(const char *name, const char *value, int overwrite) { }
#endif
+
+void nd_setenv(const char *name, const char *value, int overwrite) {
+#if defined(OS_WINDOWS)
+ if(overwrite)
+ SetEnvironmentVariable(name, value);
+ else {
+ char buf[1024];
+ if(GetEnvironmentVariable(name, buf, sizeof(buf)) == 0)
+ SetEnvironmentVariable(name, value);
+ }
+#endif
+
+#ifdef HAVE_SETENV
+ setenv(name, value, overwrite);
+#else
+ os_setenv(name, value, overwite);
+#endif
+}
diff --git a/src/libnetdata/os/setenv.h b/src/libnetdata/os/setenv.h index 3ed63714c..78e7224de 100644 --- a/src/libnetdata/os/setenv.h +++ b/src/libnetdata/os/setenv.h @@ -10,4 +10,6 @@ int os_setenv(const char *name, const char *value, int overwrite); #define setenv(name, value, overwrite) os_setenv(name, value, overwrite)
#endif
+void nd_setenv(const char *name, const char *value, int overwrite);
+
#endif //NETDATA_SETENV_H
diff --git a/src/libnetdata/os/sleep.c b/src/libnetdata/os/sleep.c new file mode 100644 index 000000000..131b47c44 --- /dev/null +++ b/src/libnetdata/os/sleep.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +#ifdef OS_WINDOWS +void tinysleep(void) { + Sleep(1); +} +#else +void tinysleep(void) { + static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; + nanosleep(&ns, NULL); +} +#endif + +#ifdef OS_WINDOWS +void microsleep(usec_t ut) { + size_t ms = ut / USEC_PER_MS + ((ut == 0 || (ut % USEC_PER_MS)) ? 1 : 0); + Sleep(ms); +} +#else +void microsleep(usec_t ut) { + time_t secs = (time_t)(ut / USEC_PER_SEC); + nsec_t nsec = (ut % USEC_PER_SEC) * NSEC_PER_USEC + ((ut == 0) ? 1 : 0); + + struct timespec remaining = { + .tv_sec = secs, + .tv_nsec = nsec, + }; + + errno_clear(); + while (nanosleep(&remaining, &remaining) == -1 && errno == EINTR && (remaining.tv_sec || remaining.tv_nsec)) { + // Loop continues if interrupted by a signal + } +} +#endif diff --git a/src/libnetdata/os/sleep.h b/src/libnetdata/os/sleep.h new file mode 100644 index 000000000..358238762 --- /dev/null +++ b/src/libnetdata/os/sleep.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_SLEEP_H
+#define NETDATA_SLEEP_H
+
+void tinysleep(void);
+void microsleep(usec_t ut);
+
+#endif //NETDATA_SLEEP_H
diff --git a/src/libnetdata/os/system-maps/cache-host-users-and-groups.c b/src/libnetdata/os/system-maps/cache-host-users-and-groups.c new file mode 100644 index 000000000..53825fd35 --- /dev/null +++ b/src/libnetdata/os/system-maps/cache-host-users-and-groups.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "libnetdata/libnetdata.h" + +static bool file_changed(const struct stat *statbuf __maybe_unused, struct timespec *last_modification_time __maybe_unused) { +#if defined(OS_MACOS) || defined(OS_WINDOWS) + return false; +#else + if(likely(statbuf->st_mtim.tv_sec == last_modification_time->tv_sec && + statbuf->st_mtim.tv_nsec == last_modification_time->tv_nsec)) return false; + + last_modification_time->tv_sec = statbuf->st_mtim.tv_sec; + last_modification_time->tv_nsec = statbuf->st_mtim.tv_nsec; + + return true; +#endif +} + +static size_t read_passwd_or_group(const char *filename, struct timespec *last_modification_time, void (*cb)(uint32_t gid, const char *name, uint32_t version), uint32_t version) { + struct stat statbuf; + if(unlikely(stat(filename, &statbuf) || !file_changed(&statbuf, last_modification_time))) + return 0; + + procfile *ff = procfile_open(filename, " :\t", PROCFILE_FLAG_DEFAULT); + if(unlikely(!ff)) return 0; + + ff = procfile_readall(ff); + if(unlikely(!ff)) return 0; + + size_t line, lines = procfile_lines(ff); + + size_t added = 0; + for(line = 0; line < lines ;line++) { + size_t words = procfile_linewords(ff, line); + if(unlikely(words < 3)) continue; + + char *name = procfile_lineword(ff, line, 0); + if(unlikely(!name || !*name)) continue; + + char *id_string = procfile_lineword(ff, line, 2); + if(unlikely(!id_string || !*id_string)) continue; + + uint32_t id = str2ull(id_string, NULL); + + cb(id, name, version); + added++; + } + + procfile_close(ff); + return added; +} + +void update_cached_host_users(void) { + if(!netdata_configured_host_prefix || !*netdata_configured_host_prefix) return; + + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER; + if(!spinlock_trylock(&spinlock)) return; + + char filename[FILENAME_MAX]; + static bool initialized = false; + + size_t added = 0; + + if(!initialized) { + initialized = true; + cached_usernames_init(); + } + + static uint32_t passwd_version = 0; + static struct timespec passwd_ts = { 0 }; + snprintfz(filename, FILENAME_MAX, "%s/etc/passwd", netdata_configured_host_prefix); + added = read_passwd_or_group(filename, &passwd_ts, cached_username_populate_by_uid, ++passwd_version); + if(added) cached_usernames_delete_old_versions(passwd_version); + + spinlock_unlock(&spinlock); +} + +void update_cached_host_groups(void) { + if(!netdata_configured_host_prefix || !*netdata_configured_host_prefix) return; + + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER; + if(!spinlock_trylock(&spinlock)) return; + + char filename[FILENAME_MAX]; + static bool initialized = false; + + size_t added = 0; + + if(!initialized) { + initialized = true; + cached_groupnames_init(); + } + + static uint32_t group_version = 0; + static struct timespec group_ts = { 0 }; + snprintfz(filename, FILENAME_MAX, "%s/etc/group", netdata_configured_host_prefix); + added = read_passwd_or_group(filename, &group_ts, cached_groupname_populate_by_gid, ++group_version); + if(added) cached_groupnames_delete_old_versions(group_version); + + spinlock_unlock(&spinlock); +} diff --git a/src/libnetdata/os/system-maps/cache-host-users-and-groups.h b/src/libnetdata/os/system-maps/cache-host-users-and-groups.h new file mode 100644 index 000000000..7a84bcadf --- /dev/null +++ b/src/libnetdata/os/system-maps/cache-host-users-and-groups.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_CACHE_HOST_USERS_AND_GROUPS_H +#define NETDATA_CACHE_HOST_USERS_AND_GROUPS_H + +void update_cached_host_users(void); +void update_cached_host_groups(void); + +#endif //NETDATA_CACHE_HOST_USERS_AND_GROUPS_H diff --git a/src/libnetdata/os/system-maps/cached-gid-groupname.c b/src/libnetdata/os/system-maps/cached-gid-groupname.c new file mode 100644 index 000000000..3fabe94a2 --- /dev/null +++ b/src/libnetdata/os/system-maps/cached-gid-groupname.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "cached-gid-groupname.h" + +// -------------------------------------------------------------------------------------------------------------------- +// hashtable for caching gid to groupname mappings +// key is the gid, value is groupname (STRING) + +#define SIMPLE_HASHTABLE_KEY_TYPE gid_t +#define SIMPLE_HASHTABLE_VALUE_TYPE CACHED_GROUPNAME +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION cached_groupname_to_gid_ptr +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION compar_gid_ptr +#define SIMPLE_HASHTABLE_NAME _GROUPNAMES_CACHE +#include "libnetdata/simple_hashtable/simple_hashtable.h" + +static struct { + bool initialized; + SPINLOCK spinlock; + SIMPLE_HASHTABLE_GROUPNAMES_CACHE ht; +} group_cache = { + .initialized = false, + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .ht = { 0 }, +}; + +static gid_t *cached_groupname_to_gid_ptr(CACHED_GROUPNAME *cu) { + return &cu->gid; +} + +static bool compar_gid_ptr(gid_t *a, gid_t *b) { + return *a == *b; +} + +void cached_groupname_populate_by_gid(gid_t gid, const char *groupname, uint32_t version) { + internal_fatal(!group_cache.initialized, "system-users cache needs to be initialized"); + if(!groupname || !*groupname) return; + + spinlock_lock(&group_cache.spinlock); + + XXH64_hash_t hash = XXH3_64bits(&gid, sizeof(gid)); + SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_get_slot_GROUPNAMES_CACHE(&group_cache.ht, hash, &gid, true); + CACHED_GROUPNAME *cg = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(!cg || (cg->version && version > cg->version)) { + internal_fatal(cg && cg->gid != gid, "invalid gid matched from cache"); + + if(cg) + string_freez(cg->groupname); + else + cg = callocz(1, sizeof(*cg)); + + cg->version = version; + cg->gid = gid; + cg->groupname = string_strdupz(groupname); + simple_hashtable_set_slot_GROUPNAMES_CACHE(&group_cache.ht, sl, hash, cg); + } + + spinlock_unlock(&group_cache.spinlock); +} + +CACHED_GROUPNAME cached_groupname_get_by_gid(gid_t gid) { + internal_fatal(!group_cache.initialized, "system-users cache needs to be initialized"); + + spinlock_lock(&group_cache.spinlock); + + XXH64_hash_t hash = XXH3_64bits(&gid, sizeof(gid)); + SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_get_slot_GROUPNAMES_CACHE(&group_cache.ht, hash, &gid, true); + CACHED_GROUPNAME *cg = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(!cg) { + cg = callocz(1, sizeof(*cg)); + + static char tmp[1024]; // we are inside a global spinlock - it is ok to be static + struct group gr, *result = NULL; + + if (getgrgid_r(gid, &gr, tmp, sizeof(tmp), &result) != 0 || !result || !gr.gr_name || !(*gr.gr_name)) { + char name[UINT64_MAX_LENGTH]; + print_uint64(name, gid); + cg->groupname = string_strdupz(name); + } + else + cg->groupname = string_strdupz(gr.gr_name); + + cg->gid = gid; + simple_hashtable_set_slot_GROUPNAMES_CACHE(&group_cache.ht, sl, hash, cg); + } + + internal_fatal(cg->gid != gid, "invalid gid matched from cache"); + + CACHED_GROUPNAME rc = { + .version = cg->version, + .gid = cg->gid, + .groupname = string_dup(cg->groupname), + }; + + spinlock_unlock(&group_cache.spinlock); + return rc; +} + +void cached_groupname_release(CACHED_GROUPNAME cg) { + string_freez(cg.groupname); +} + +void cached_groupnames_init(void) { + if(group_cache.initialized) return; + group_cache.initialized = true; + + spinlock_init(&group_cache.spinlock); + simple_hashtable_init_GROUPNAMES_CACHE(&group_cache.ht, 100); +} + +void cached_groupnames_destroy(void) { + if(!group_cache.initialized) return; + + spinlock_lock(&group_cache.spinlock); + + for(SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_first_read_only_GROUPNAMES_CACHE(&group_cache.ht); + sl; + sl = simple_hashtable_next_read_only_GROUPNAMES_CACHE(&group_cache.ht, sl)) { + CACHED_GROUPNAME *u = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(u) { + string_freez(u->groupname); + freez(u); + // simple_hashtable_del_slot_GROUPNAMES_CACHE(&uc.ht, sl); + } + } + + simple_hashtable_destroy_GROUPNAMES_CACHE(&group_cache.ht); + group_cache.initialized = false; + + spinlock_unlock(&group_cache.spinlock); +} + +void cached_groupnames_delete_old_versions(uint32_t version) { + if(!group_cache.initialized) return; + + spinlock_lock(&group_cache.spinlock); + + for(SIMPLE_HASHTABLE_SLOT_GROUPNAMES_CACHE *sl = simple_hashtable_first_read_only_GROUPNAMES_CACHE(&group_cache.ht); + sl; + sl = simple_hashtable_next_read_only_GROUPNAMES_CACHE(&group_cache.ht, sl)) { + CACHED_GROUPNAME *cg = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(cg && cg->version && cg->version < version) { + string_freez(cg->groupname); + freez(cg); + simple_hashtable_del_slot_GROUPNAMES_CACHE(&group_cache.ht, sl); + } + } + + spinlock_unlock(&group_cache.spinlock); +} diff --git a/src/libnetdata/os/system-maps/cached-gid-groupname.h b/src/libnetdata/os/system-maps/cached-gid-groupname.h new file mode 100644 index 000000000..81a62523e --- /dev/null +++ b/src/libnetdata/os/system-maps/cached-gid-groupname.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_CACHED_UID_GROUPNAME_H +#define NETDATA_CACHED_UID_GROUPNAME_H + +#include "libnetdata/libnetdata.h" + +struct netdata_string; + +typedef struct { + uint32_t version; + gid_t gid; + struct netdata_string *groupname; +} CACHED_GROUPNAME; + +void cached_groupname_populate_by_gid(gid_t gid, const char *groupname, uint32_t version); +CACHED_GROUPNAME cached_groupname_get_by_gid(gid_t gid); +void cached_groupname_release(CACHED_GROUPNAME cg); +void cached_groupnames_delete_old_versions(uint32_t version); + +void cached_groupnames_init(void); +void cached_groupnames_destroy(void); + +#endif //NETDATA_CACHED_UID_GROUPNAME_H diff --git a/src/libnetdata/os/system-maps/cached-sid-username.c b/src/libnetdata/os/system-maps/cached-sid-username.c new file mode 100644 index 000000000..a0f90c546 --- /dev/null +++ b/src/libnetdata/os/system-maps/cached-sid-username.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../../libnetdata.h" + +#if defined(OS_WINDOWS) +#include "cached-sid-username.h" + +typedef struct { + size_t len; + uint8_t sid[]; +} SID_KEY; + +typedef struct { + // IMPORTANT: + // This is malloc'd ! You have to manually set fields to zero. + + STRING *account; + STRING *domain; + STRING *full; + STRING *sid_str; + + // this needs to be last, because of its variable size + SID_KEY key; +} SID_VALUE; + +#define SIMPLE_HASHTABLE_NAME _SID +#define SIMPLE_HASHTABLE_VALUE_TYPE SID_VALUE +#define SIMPLE_HASHTABLE_KEY_TYPE SID_KEY +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION sid_value_to_key +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION sid_cache_compar +#define SIMPLE_HASHTABLE_SAMPLE_IMPLEMENTATION 1 +#include "libnetdata/simple_hashtable/simple_hashtable.h" + +static struct { + SPINLOCK spinlock; + struct simple_hashtable_SID hashtable; +} sid_globals = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .hashtable = { 0 }, +}; + +static inline SID_KEY *sid_value_to_key(SID_VALUE *s) { + return &s->key; +} + +static inline bool sid_cache_compar(SID_KEY *a, SID_KEY *b) { + return a->len == b->len && memcmp(&a->sid, &b->sid, a->len) == 0; +} + +void cached_sid_username_init(void) { + simple_hashtable_init_SID(&sid_globals.hashtable, 100); +} + +static char *account2utf8(const wchar_t *user) { + static __thread char buffer[256]; + if(utf16_to_utf8(buffer, sizeof(buffer), user, -1, NULL) == 0) + buffer[0] = '\0'; + return buffer; +} + +static char *domain2utf8(const wchar_t *domain) { + static __thread char buffer[256]; + if(utf16_to_utf8(buffer, sizeof(buffer), domain, -1, NULL) == 0) + buffer[0] = '\0'; + return buffer; +} + +static void lookup_user_in_system(SID_VALUE *sv) { + static __thread wchar_t account_unicode[256]; + static __thread wchar_t domain_unicode[256]; + static __thread char tmp[512 + 2]; + + DWORD account_name_size = sizeof(account_unicode) / sizeof(account_unicode[0]); + DWORD domain_name_size = sizeof(domain_unicode) / sizeof(domain_unicode[0]); + SID_NAME_USE sid_type; + + if (LookupAccountSidW(NULL, sv->key.sid, account_unicode, &account_name_size, domain_unicode, &domain_name_size, &sid_type)) { + const char *account = account2utf8(account_unicode); + const char *domain = domain2utf8(domain_unicode); + snprintfz(tmp, sizeof(tmp), "%s\\%s", domain, account); + sv->domain = string_strdupz(domain); + sv->account = string_strdupz(account); + sv->full = string_strdupz(tmp); + } + else { + sv->domain = NULL; + sv->account = NULL; + sv->full = NULL; + } + + wchar_t *sid_string = NULL; + if (ConvertSidToStringSidW(sv->key.sid, &sid_string)) + sv->sid_str = string_strdupz(account2utf8(sid_string)); + else + sv->sid_str = NULL; +} + +static SID_VALUE *lookup_or_convert_user_id_to_name_lookup(PSID sid) { + if(!sid || !IsValidSid(sid)) + return NULL; + + size_t size = GetLengthSid(sid); + + size_t tmp_size = sizeof(SID_VALUE) + size; + size_t tmp_key_size = sizeof(SID_KEY) + size; + uint8_t buf[tmp_size]; + SID_VALUE *tmp = (SID_VALUE *)&buf; + memcpy(&tmp->key.sid, sid, size); + tmp->key.len = size; + + spinlock_lock(&sid_globals.spinlock); + SID_VALUE *found = simple_hashtable_get_SID(&sid_globals.hashtable, &tmp->key, tmp_key_size); + spinlock_unlock(&sid_globals.spinlock); + if(found) return found; + + // allocate the SID_VALUE + found = mallocz(tmp_size); + memcpy(found, buf, tmp_size); + + lookup_user_in_system(found); + + // add it to the cache + spinlock_lock(&sid_globals.spinlock); + simple_hashtable_set_SID(&sid_globals.hashtable, &found->key, tmp_key_size, found); + spinlock_unlock(&sid_globals.spinlock); + + return found; +} + +bool cached_sid_to_account_domain_sidstr(PSID sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str) { + SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid); + + if(found) { + if (found->account) { + txt_utf8_resize(dst_account, string_strlen(found->account) + 1, false); + memcpy(dst_account->data, string2str(found->account), string_strlen(found->account) + 1); + dst_account->used = string_strlen(found->account) + 1; + } + else + txt_utf8_empty(dst_account); + + if (found->domain) { + txt_utf8_resize(dst_domain, string_strlen(found->domain) + 1, false); + memcpy(dst_domain->data, string2str(found->domain), string_strlen(found->domain) + 1); + dst_domain->used = string_strlen(found->domain) + 1; + } + else + txt_utf8_empty(dst_domain); + + if (found->sid_str) { + txt_utf8_resize(dst_sid_str, string_strlen(found->sid_str) + 1, false); + memcpy(dst_sid_str->data, string2str(found->sid_str), string_strlen(found->sid_str) + 1); + dst_sid_str->used = string_strlen(found->sid_str) + 1; + } + else + txt_utf8_empty(dst_sid_str); + + return true; + } + + txt_utf8_empty(dst_account); + txt_utf8_empty(dst_domain); + txt_utf8_empty(dst_sid_str); + return false; +} + +bool cached_sid_to_buffer_append(PSID sid, BUFFER *dst, const char *prefix) { + SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid); + size_t added = 0; + + if(found) { + if (found->full) { + if (prefix && *prefix) + buffer_strcat(dst, prefix); + + buffer_fast_strcat(dst, string2str(found->full), string_strlen(found->full)); + added++; + } + if (found->sid_str) { + if (prefix && *prefix) + buffer_strcat(dst, prefix); + + buffer_fast_strcat(dst, string2str(found->sid_str), string_strlen(found->sid_str)); + added++; + } + } + + return added > 0; +} + +STRING *cached_sid_fullname_or_sid_str(PSID sid) { + SID_VALUE *found = lookup_or_convert_user_id_to_name_lookup(sid); + if(found) { + if(found->full) return string_dup(found->full); + return string_dup(found->sid_str); + } + return NULL; +} + +#endif
\ No newline at end of file diff --git a/src/libnetdata/os/system-maps/cached-sid-username.h b/src/libnetdata/os/system-maps/cached-sid-username.h new file mode 100644 index 000000000..4077cad11 --- /dev/null +++ b/src/libnetdata/os/system-maps/cached-sid-username.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_CACHED_SID_USERNAME_H +#define NETDATA_CACHED_SID_USERNAME_H + +#include "../../libnetdata.h" + +#if defined(OS_WINDOWS) +#include "../../string/utf8.h" + +bool cached_sid_to_account_domain_sidstr(void *sid, TXT_UTF8 *dst_account, TXT_UTF8 *dst_domain, TXT_UTF8 *dst_sid_str); +bool cached_sid_to_buffer_append(void *sid, BUFFER *dst, const char *prefix); +void cached_sid_username_init(void); +STRING *cached_sid_fullname_or_sid_str(void *sid); +#endif + +#endif //NETDATA_CACHED_SID_USERNAME_H diff --git a/src/libnetdata/os/system-maps/cached-uid-username.c b/src/libnetdata/os/system-maps/cached-uid-username.c new file mode 100644 index 000000000..35d93f2f0 --- /dev/null +++ b/src/libnetdata/os/system-maps/cached-uid-username.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "cached-uid-username.h" + +// -------------------------------------------------------------------------------------------------------------------- +// hashtable for caching uid to username mappings +// key is the uid, value is username (STRING) + +#define SIMPLE_HASHTABLE_KEY_TYPE uid_t +#define SIMPLE_HASHTABLE_VALUE_TYPE CACHED_USERNAME +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION cached_username_to_uid_ptr +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION compar_uid_ptr +#define SIMPLE_HASHTABLE_NAME _USERNAMES_CACHE +#include "libnetdata/simple_hashtable/simple_hashtable.h" + +static struct { + bool initialized; + SPINLOCK spinlock; + SIMPLE_HASHTABLE_USERNAMES_CACHE ht; +} user_cache = { + .initialized = false, + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .ht = { 0 }, +}; + +static uid_t *cached_username_to_uid_ptr(CACHED_USERNAME *cu) { + return &cu->uid; +} + +static bool compar_uid_ptr(uid_t *a, uid_t *b) { + return *a == *b; +} + +void cached_username_populate_by_uid(uid_t uid, const char *username, uint32_t version) { + internal_fatal(!user_cache.initialized, "system-users cache needs to be initialized"); + if(!username || !*username) return; + + spinlock_lock(&user_cache.spinlock); + + XXH64_hash_t hash = XXH3_64bits(&uid, sizeof(uid)); + SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_get_slot_USERNAMES_CACHE(&user_cache.ht, hash, &uid, true); + CACHED_USERNAME *cu = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(!cu || (cu->version && version > cu->version)) { + internal_fatal(cu && cu->uid != uid, "invalid uid matched from cache"); + + if(cu) + string_freez(cu->username); + else + cu = callocz(1, sizeof(*cu)); + + cu->version = version; + cu->uid = uid; + cu->username = string_strdupz(username); + simple_hashtable_set_slot_USERNAMES_CACHE(&user_cache.ht, sl, hash, cu); + } + + spinlock_unlock(&user_cache.spinlock); +} + +CACHED_USERNAME cached_username_get_by_uid(uid_t uid) { + internal_fatal(!user_cache.initialized, "system-users cache needs to be initialized"); + + spinlock_lock(&user_cache.spinlock); + + XXH64_hash_t hash = XXH3_64bits(&uid, sizeof(uid)); + SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_get_slot_USERNAMES_CACHE(&user_cache.ht, hash, &uid, true); + CACHED_USERNAME *cu = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(!cu) { + cu = callocz(1, sizeof(*cu)); + + static char tmp[1024]; // we are inside a global spinlock - it is ok to be static + struct passwd pw, *result = NULL; + + if (getpwuid_r(uid, &pw, tmp, sizeof(tmp), &result) != 0 || !result || !pw.pw_name || !(*pw.pw_name)) { + char name[UINT64_MAX_LENGTH]; + print_uint64(name, uid); + cu->username = string_strdupz(name); + } + else + cu->username = string_strdupz(pw.pw_name); + + cu->uid = uid; + simple_hashtable_set_slot_USERNAMES_CACHE(&user_cache.ht, sl, hash, cu); + } + + internal_fatal(cu->uid != uid, "invalid uid matched from cache"); + + CACHED_USERNAME rc = { + .version = cu->version, + .uid = cu->uid, + .username = string_dup(cu->username), + }; + + spinlock_unlock(&user_cache.spinlock); + return rc; +} + +void cached_username_release(CACHED_USERNAME cu) { + string_freez(cu.username); +} + +void cached_usernames_init(void) { + if(user_cache.initialized) return; + user_cache.initialized = true; + + spinlock_init(&user_cache.spinlock); + simple_hashtable_init_USERNAMES_CACHE(&user_cache.ht, 100); +} + +void cached_usernames_destroy(void) { + if(!user_cache.initialized) return; + + spinlock_lock(&user_cache.spinlock); + + for(SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_first_read_only_USERNAMES_CACHE(&user_cache.ht); + sl; + sl = simple_hashtable_next_read_only_USERNAMES_CACHE(&user_cache.ht, sl)) { + CACHED_USERNAME *u = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(u) { + string_freez(u->username); + freez(u); + // simple_hashtable_del_slot_USERNAMES_CACHE(&uc.ht, sl); + } + } + + simple_hashtable_destroy_USERNAMES_CACHE(&user_cache.ht); + user_cache.initialized = false; + + spinlock_unlock(&user_cache.spinlock); +} + +void cached_usernames_delete_old_versions(uint32_t version) { + if(!user_cache.initialized) return; + + spinlock_lock(&user_cache.spinlock); + + for(SIMPLE_HASHTABLE_SLOT_USERNAMES_CACHE *sl = simple_hashtable_first_read_only_USERNAMES_CACHE(&user_cache.ht); + sl; + sl = simple_hashtable_next_read_only_USERNAMES_CACHE(&user_cache.ht, sl)) { + CACHED_USERNAME *cu = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(cu && cu->version && cu->version < version) { + string_freez(cu->username); + freez(cu); + simple_hashtable_del_slot_USERNAMES_CACHE(&user_cache.ht, sl); + } + } + + spinlock_unlock(&user_cache.spinlock); +} diff --git a/src/libnetdata/os/system-maps/cached-uid-username.h b/src/libnetdata/os/system-maps/cached-uid-username.h new file mode 100644 index 000000000..b7c52c7c4 --- /dev/null +++ b/src/libnetdata/os/system-maps/cached-uid-username.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_CACHED_UID_USERNAME_H +#define NETDATA_CACHED_UID_USERNAME_H + +#include "libnetdata/libnetdata.h" + +struct netdata_string; + +typedef struct { + uint32_t version; + uid_t uid; + struct netdata_string *username; +} CACHED_USERNAME; + +void cached_username_populate_by_uid(uid_t uid, const char *username, uint32_t version); +CACHED_USERNAME cached_username_get_by_uid(uid_t uid); +void cached_username_release(CACHED_USERNAME cu); + +void cached_usernames_init(void); +void cached_usernames_destroy(void); +void cached_usernames_delete_old_versions(uint32_t version); + +#endif //NETDATA_CACHED_UID_USERNAME_H diff --git a/src/libnetdata/maps/system-services.h b/src/libnetdata/os/system-maps/system-services.h index 123f4f10b..5d3592bbf 100644 --- a/src/libnetdata/maps/system-services.h +++ b/src/libnetdata/os/system-maps/system-services.h @@ -8,21 +8,17 @@ // -------------------------------------------------------------------------------------------------------------------- // hashtable for caching port and protocol to service name mappings -// key is the combination of protocol and port packed into a uint64_t, value is service name (STRING) +// key is the combination of protocol and port packed into an uint64_t, value is service name (STRING) #define SIMPLE_HASHTABLE_VALUE_TYPE STRING #define SIMPLE_HASHTABLE_NAME _SERVICENAMES_CACHE -#include "libnetdata/simple_hashtable.h" +#include "libnetdata/simple_hashtable/simple_hashtable.h" typedef struct servicenames_cache { SPINLOCK spinlock; SIMPLE_HASHTABLE_SERVICENAMES_CACHE ht; } SERVICENAMES_CACHE; -static inline uint64_t system_servicenames_key(uint16_t port, uint16_t ipproto) { - return ((uint64_t)ipproto << 16) | (uint64_t)port; -} - static inline const char *system_servicenames_ipproto2str(uint16_t ipproto) { return (ipproto == IPPROTO_TCP) ? "tcp" : "udp"; } @@ -38,10 +34,18 @@ static inline const char *static_portnames(uint16_t port, uint16_t ipproto) { } static inline STRING *system_servicenames_cache_lookup(SERVICENAMES_CACHE *sc, uint16_t port, uint16_t ipproto) { - uint64_t key = system_servicenames_key(port, ipproto); + struct { + uint16_t ipproto; + uint16_t port; + } key = { + .ipproto = ipproto, + .port = port, + }; + XXH64_hash_t hash = XXH3_64bits(&key, sizeof(key)); + spinlock_lock(&sc->spinlock); - SIMPLE_HASHTABLE_SLOT_SERVICENAMES_CACHE *sl = simple_hashtable_get_slot_SERVICENAMES_CACHE(&sc->ht, key, &key, true); + SIMPLE_HASHTABLE_SLOT_SERVICENAMES_CACHE *sl = simple_hashtable_get_slot_SERVICENAMES_CACHE(&sc->ht, hash, &key, true); STRING *s = SIMPLE_HASHTABLE_SLOT_DATA(sl); if (!s) { const char *st = static_portnames(port, ipproto); @@ -60,7 +64,7 @@ static inline STRING *system_servicenames_cache_lookup(SERVICENAMES_CACHE *sc, u s = string_strdupz(se->s_name); } - simple_hashtable_set_slot_SERVICENAMES_CACHE(&sc->ht, sl, key, s); + simple_hashtable_set_slot_SERVICENAMES_CACHE(&sc->ht, sl, hash, s); } s = string_dup(s); diff --git a/src/libnetdata/os/timestamps.c b/src/libnetdata/os/timestamps.c new file mode 100644 index 000000000..602899d34 --- /dev/null +++ b/src/libnetdata/os/timestamps.c @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "libnetdata/libnetdata.h" +#include "timestamps.h" diff --git a/src/libnetdata/os/timestamps.h b/src/libnetdata/os/timestamps.h new file mode 100644 index 000000000..3737a4f40 --- /dev/null +++ b/src/libnetdata/os/timestamps.h @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef LIBNETDATA_OS_TIMESTAMPS_H +#define LIBNETDATA_OS_TIMESTAMPS_H + +// Windows file time starts on January 1, 1601, Unix epoch starts on January 1, 1970 +// Difference in 100-nanosecond intervals between these two dates is 116444736000000000ULL + +// Convert Windows file time (in 100-nanosecond intervals) to Unix epoch in nanoseconds +#define os_windows_ulonglong_to_unix_epoch_ns(ft) (((uint64_t)(ft) - 116444736000000000ULL) * 100ULL) + +// Convert Unix epoch time (in nanoseconds) to Windows file time (in 100-nanosecond intervals) +#define os_unix_epoch_ns_to_windows_ulonglong(ns) (((uint64_t)(ns) / 100ULL) + 116444736000000000ULL) + +#if defined(OS_WINDOWS) +// Convert FILETIME to Unix epoch in nanoseconds +#define os_filetime_to_unix_epoch_ns(ft) \ + ((((uint64_t)(ft).dwHighDateTime << 32 | (ft).dwLowDateTime) - 116444736000000000ULL) * 100ULL) + +// Convert Unix epoch in nanoseconds to FILETIME (returns FILETIME) +#define os_unix_epoch_ns_to_filetime(ns) \ + ({ \ + uint64_t temp = ((uint64_t)(ns) / 100ULL) + 116444736000000000ULL; \ + FILETIME ft; \ + ft.dwLowDateTime = (uint32_t)(temp & 0xFFFFFFFF); \ + ft.dwHighDateTime = (uint32_t)(temp >> 32); \ + ft; \ + }) + +// Convert Unix epoch in microseconds to FILETIME (returns FILETIME) +#define os_unix_epoch_ut_to_filetime(ns) \ + ({ \ + uint64_t temp = ((uint64_t)(ns) * 10ULL) + 116444736000000000ULL; \ + FILETIME ft; \ + ft.dwLowDateTime = (uint32_t)(temp & 0xFFFFFFFF); \ + ft.dwHighDateTime = (uint32_t)(temp >> 32); \ + ft; \ + }) + +#endif //OS_WINDOWS + +#endif //LIBNETDATA_OS_TIMESTAMPS_H diff --git a/src/libnetdata/os/tinysleep.c b/src/libnetdata/os/tinysleep.c deleted file mode 100644 index f04cbdadc..000000000 --- a/src/libnetdata/os/tinysleep.c +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "../libnetdata.h" - -#ifdef OS_WINDOWS -#include <windows.h> - -void tinysleep(void) { - // Improve the system timer resolution to 1 ms - timeBeginPeriod(1); - - // Sleep for the desired duration - Sleep(1); - - // Reset the system timer resolution - timeEndPeriod(1); -} -#else -void tinysleep(void) { - static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; - nanosleep(&ns, NULL); -} -#endif diff --git a/src/libnetdata/os/tinysleep.h b/src/libnetdata/os/tinysleep.h deleted file mode 100644 index 480575a3a..000000000 --- a/src/libnetdata/os/tinysleep.h +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later
-
-#ifndef NETDATA_TINYSLEEP_H
-#define NETDATA_TINYSLEEP_H
-
-void tinysleep(void);
-
-#endif //NETDATA_TINYSLEEP_H
diff --git a/src/libnetdata/os/uuid_generate.c b/src/libnetdata/os/uuid_generate.c index 4a7a9b6bc..6019f8844 100644 --- a/src/libnetdata/os/uuid_generate.c +++ b/src/libnetdata/os/uuid_generate.c @@ -6,8 +6,6 @@ #undef uuid_generate_time #ifdef OS_WINDOWS -#include <windows.h> - void os_uuid_generate(void *out) { RPC_STATUS status = UuidCreate(out); while (status != RPC_S_OK && status != RPC_S_UUID_LOCAL_ONLY) { diff --git a/src/libnetdata/os/windows-perflib/perflib-dump.c b/src/libnetdata/os/windows-perflib/perflib-dump.c new file mode 100644 index 000000000..eaccb7827 --- /dev/null +++ b/src/libnetdata/os/windows-perflib/perflib-dump.c @@ -0,0 +1,531 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "perflib.h" + +#if defined(OS_WINDOWS) +static const char *getCounterType(DWORD CounterType) { + switch (CounterType) { + case PERF_COUNTER_COUNTER: + return "PERF_COUNTER_COUNTER"; + + case PERF_COUNTER_TIMER: + return "PERF_COUNTER_TIMER"; + + case PERF_COUNTER_QUEUELEN_TYPE: + return "PERF_COUNTER_QUEUELEN_TYPE"; + + case PERF_COUNTER_LARGE_QUEUELEN_TYPE: + return "PERF_COUNTER_LARGE_QUEUELEN_TYPE"; + + case PERF_COUNTER_100NS_QUEUELEN_TYPE: + return "PERF_COUNTER_100NS_QUEUELEN_TYPE"; + + case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: + return "PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE"; + + case PERF_COUNTER_BULK_COUNT: + return "PERF_COUNTER_BULK_COUNT"; + + case PERF_COUNTER_TEXT: + return "PERF_COUNTER_TEXT"; + + case PERF_COUNTER_RAWCOUNT: + return "PERF_COUNTER_RAWCOUNT"; + + case PERF_COUNTER_LARGE_RAWCOUNT: + return "PERF_COUNTER_LARGE_RAWCOUNT"; + + case PERF_COUNTER_RAWCOUNT_HEX: + return "PERF_COUNTER_RAWCOUNT_HEX"; + + case PERF_COUNTER_LARGE_RAWCOUNT_HEX: + return "PERF_COUNTER_LARGE_RAWCOUNT_HEX"; + + case PERF_SAMPLE_FRACTION: + return "PERF_SAMPLE_FRACTION"; + + case PERF_SAMPLE_COUNTER: + return "PERF_SAMPLE_COUNTER"; + + case PERF_COUNTER_NODATA: + return "PERF_COUNTER_NODATA"; + + case PERF_COUNTER_TIMER_INV: + return "PERF_COUNTER_TIMER_INV"; + + case PERF_SAMPLE_BASE: + return "PERF_SAMPLE_BASE"; + + case PERF_AVERAGE_TIMER: + return "PERF_AVERAGE_TIMER"; + + case PERF_AVERAGE_BASE: + return "PERF_AVERAGE_BASE"; + + case PERF_AVERAGE_BULK: + return "PERF_AVERAGE_BULK"; + + case PERF_OBJ_TIME_TIMER: + return "PERF_OBJ_TIME_TIMER"; + + case PERF_100NSEC_TIMER: + return "PERF_100NSEC_TIMER"; + + case PERF_100NSEC_TIMER_INV: + return "PERF_100NSEC_TIMER_INV"; + + case PERF_COUNTER_MULTI_TIMER: + return "PERF_COUNTER_MULTI_TIMER"; + + case PERF_COUNTER_MULTI_TIMER_INV: + return "PERF_COUNTER_MULTI_TIMER_INV"; + + case PERF_COUNTER_MULTI_BASE: + return "PERF_COUNTER_MULTI_BASE"; + + case PERF_100NSEC_MULTI_TIMER: + return "PERF_100NSEC_MULTI_TIMER"; + + case PERF_100NSEC_MULTI_TIMER_INV: + return "PERF_100NSEC_MULTI_TIMER_INV"; + + case PERF_RAW_FRACTION: + return "PERF_RAW_FRACTION"; + + case PERF_LARGE_RAW_FRACTION: + return "PERF_LARGE_RAW_FRACTION"; + + case PERF_RAW_BASE: + return "PERF_RAW_BASE"; + + case PERF_LARGE_RAW_BASE: + return "PERF_LARGE_RAW_BASE"; + + case PERF_ELAPSED_TIME: + return "PERF_ELAPSED_TIME"; + + case PERF_COUNTER_HISTOGRAM_TYPE: + return "PERF_COUNTER_HISTOGRAM_TYPE"; + + case PERF_COUNTER_DELTA: + return "PERF_COUNTER_DELTA"; + + case PERF_COUNTER_LARGE_DELTA: + return "PERF_COUNTER_LARGE_DELTA"; + + case PERF_PRECISION_SYSTEM_TIMER: + return "PERF_PRECISION_SYSTEM_TIMER"; + + case PERF_PRECISION_100NS_TIMER: + return "PERF_PRECISION_100NS_TIMER"; + + case PERF_PRECISION_OBJECT_TIMER: + return "PERF_PRECISION_OBJECT_TIMER"; + + default: + return "UNKNOWN_COUNTER_TYPE"; + } +} + +static const char *getCounterDescription(DWORD CounterType) { + switch (CounterType) { + case PERF_COUNTER_COUNTER: + return "32-bit Counter. Divide delta by delta time. Display suffix: \"/sec\""; + + case PERF_COUNTER_TIMER: + return "64-bit Timer. Divide delta by delta time. Display suffix: \"%\""; + + case PERF_COUNTER_QUEUELEN_TYPE: + case PERF_COUNTER_LARGE_QUEUELEN_TYPE: + return "Queue Length Space-Time Product. Divide delta by delta time. No Display Suffix"; + + case PERF_COUNTER_100NS_QUEUELEN_TYPE: + return "Queue Length Space-Time Product using 100 Ns timebase. Divide delta by delta time. No Display Suffix"; + + case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: + return "Queue Length Space-Time Product using Object specific timebase. Divide delta by delta time. No Display Suffix."; + + case PERF_COUNTER_BULK_COUNT: + return "64-bit Counter. Divide delta by delta time. Display Suffix: \"/sec\""; + + case PERF_COUNTER_TEXT: + return "Unicode text Display as text."; + + case PERF_COUNTER_RAWCOUNT: + case PERF_COUNTER_LARGE_RAWCOUNT: + return "A counter which should not be time averaged on display (such as an error counter on a serial line). Display as is. No Display Suffix."; + + case PERF_COUNTER_RAWCOUNT_HEX: + case PERF_COUNTER_LARGE_RAWCOUNT_HEX: + return "Special case for RAWCOUNT which should be displayed in hex. A counter which should not be time averaged on display (such as an error counter on a serial line). Display as is. No Display Suffix."; + + case PERF_SAMPLE_FRACTION: + return "A count which is either 1 or 0 on each sampling interrupt (% busy). Divide delta by delta base. Display Suffix: \"%\""; + + case PERF_SAMPLE_COUNTER: + return "A count which is sampled on each sampling interrupt (queue length). Divide delta by delta time. No Display Suffix."; + + case PERF_COUNTER_NODATA: + return "A label: no data is associated with this counter (it has 0 length). Do not display."; + + case PERF_COUNTER_TIMER_INV: + return "64-bit Timer inverse (e.g., idle is measured, but display busy %). Display 100 - delta divided by delta time. Display suffix: \"%\""; + + case PERF_SAMPLE_BASE: + return "The divisor for a sample, used with the previous counter to form a sampled %. You must check for >0 before dividing by this! This counter will directly follow the numerator counter. It should not be displayed to the user."; + + case PERF_AVERAGE_TIMER: + return "A timer which, when divided by an average base, produces a time in seconds which is the average time of some operation. This timer times total operations, and the base is the number of operations. Display Suffix: \"sec\""; + + case PERF_AVERAGE_BASE: + return "Used as the denominator in the computation of time or count averages. Must directly follow the numerator counter. Not displayed to the user."; + + case PERF_AVERAGE_BULK: + return "A bulk count which, when divided (typically) by the number of operations, gives (typically) the number of bytes per operation. No Display Suffix."; + + case PERF_OBJ_TIME_TIMER: + return "64-bit Timer in object specific units. Display delta divided by delta time as returned in the object type header structure. Display suffix: \"%\""; + + case PERF_100NSEC_TIMER: + return "64-bit Timer in 100 nsec units. Display delta divided by delta time. Display suffix: \"%\""; + + case PERF_100NSEC_TIMER_INV: + return "64-bit Timer inverse (e.g., idle is measured, but display busy %). Display 100 - delta divided by delta time. Display suffix: \"%\""; + + case PERF_COUNTER_MULTI_TIMER: + return "64-bit Timer. Divide delta by delta time. Display suffix: \"%\". Timer for multiple instances, so result can exceed 100%."; + + case PERF_COUNTER_MULTI_TIMER_INV: + return "64-bit Timer inverse (e.g., idle is measured, but display busy %). Display 100 * _MULTI_BASE - delta divided by delta time. Display suffix: \"%\" Timer for multiple instances, so result can exceed 100%. Followed by a counter of type _MULTI_BASE."; + + case PERF_COUNTER_MULTI_BASE: + return "Number of instances to which the preceding _MULTI_..._INV counter applies. Used as a factor to get the percentage."; + + case PERF_100NSEC_MULTI_TIMER: + return "64-bit Timer in 100 nsec units. Display delta divided by delta time. Display suffix: \"%\" Timer for multiple instances, so result can exceed 100%."; + + case PERF_100NSEC_MULTI_TIMER_INV: + return "64-bit Timer inverse (e.g., idle is measured, but display busy %). Display 100 * _MULTI_BASE - delta divided by delta time. Display suffix: \"%\" Timer for multiple instances, so result can exceed 100%. Followed by a counter of type _MULTI_BASE."; + + case PERF_LARGE_RAW_FRACTION: + case PERF_RAW_FRACTION: + return "Indicates the data is a fraction of the following counter which should not be time averaged on display (such as free space over total space.) Display as is. Display the quotient as \"%\""; + + case PERF_RAW_BASE: + case PERF_LARGE_RAW_BASE: + return "Indicates the data is a base for the preceding counter which should not be time averaged on display (such as free space over total space.)"; + + case PERF_ELAPSED_TIME: + return "The data collected in this counter is actually the start time of the item being measured. For display, this data is subtracted from the sample time to yield the elapsed time as the difference between the two. In the definition below, the PerfTime field of the Object contains the sample time as indicated by the PERF_OBJECT_TIMER bit and the difference is scaled by the PerfFreq of the Object to convert the time units into seconds."; + + case PERF_COUNTER_HISTOGRAM_TYPE: + return "Counter type can be used with the preceding types to define a range of values to be displayed in a histogram."; + + case PERF_COUNTER_DELTA: + case PERF_COUNTER_LARGE_DELTA: + return "This counter is used to display the difference from one sample to the next. The counter value is a constantly increasing number and the value displayed is the difference between the current value and the previous value. Negative numbers are not allowed which shouldn't be a problem as long as the counter value is increasing or unchanged."; + + case PERF_PRECISION_SYSTEM_TIMER: + return "The precision counters are timers that consist of two counter values:\r\n\t1) the count of elapsed time of the event being monitored\r\n\t2) the \"clock\" time in the same units\r\nthe precision timers are used where the standard system timers are not precise enough for accurate readings. It's assumed that the service providing the data is also providing a timestamp at the same time which will eliminate any error that may occur since some small and variable time elapses between the time the system timestamp is captured and when the data is collected from the performance DLL. Only in extreme cases has this been observed to be problematic.\r\nwhen using this type of timer, the definition of the PERF_PRECISION_TIMESTAMP counter must immediately follow the definition of the PERF_PRECISION_*_TIMER in the Object header\r\nThe timer used has the same frequency as the System Performance Timer"; + + case PERF_PRECISION_100NS_TIMER: + return "The precision counters are timers that consist of two counter values:\r\n\t1) the count of elapsed time of the event being monitored\r\n\t2) the \"clock\" time in the same units\r\nthe precision timers are used where the standard system timers are not precise enough for accurate readings. It's assumed that the service providing the data is also providing a timestamp at the same time which will eliminate any error that may occur since some small and variable time elapses between the time the system timestamp is captured and when the data is collected from the performance DLL. Only in extreme cases has this been observed to be problematic.\r\nwhen using this type of timer, the definition of the PERF_PRECISION_TIMESTAMP counter must immediately follow the definition of the PERF_PRECISION_*_TIMER in the Object header\r\nThe timer used has the same frequency as the 100 NanoSecond Timer"; + + case PERF_PRECISION_OBJECT_TIMER: + return "The precision counters are timers that consist of two counter values:\r\n\t1) the count of elapsed time of the event being monitored\r\n\t2) the \"clock\" time in the same units\r\nthe precision timers are used where the standard system timers are not precise enough for accurate readings. It's assumed that the service providing the data is also providing a timestamp at the same time which will eliminate any error that may occur since some small and variable time elapses between the time the system timestamp is captured and when the data is collected from the performance DLL. Only in extreme cases has this been observed to be problematic.\r\nwhen using this type of timer, the definition of the PERF_PRECISION_TIMESTAMP counter must immediately follow the definition of the PERF_PRECISION_*_TIMER in the Object header\r\nThe timer used is of the frequency specified in the Object header's. PerfFreq field (PerfTime is ignored)"; + + default: + return ""; + } +} + +static const char *getCounterAlgorithm(DWORD CounterType) { + switch (CounterType) + { + case PERF_COUNTER_COUNTER: + case PERF_SAMPLE_COUNTER: + case PERF_COUNTER_BULK_COUNT: + return "(data1 - data0) / ((time1 - time0) / frequency)"; + + case PERF_COUNTER_QUEUELEN_TYPE: + case PERF_COUNTER_100NS_QUEUELEN_TYPE: + case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: + case PERF_COUNTER_LARGE_QUEUELEN_TYPE: + case PERF_AVERAGE_BULK: // normally not displayed + return "(data1 - data0) / (time1 - time0)"; + + case PERF_OBJ_TIME_TIMER: + case PERF_COUNTER_TIMER: + case PERF_100NSEC_TIMER: + case PERF_PRECISION_SYSTEM_TIMER: + case PERF_PRECISION_100NS_TIMER: + case PERF_PRECISION_OBJECT_TIMER: + case PERF_SAMPLE_FRACTION: + return "100 * (data1 - data0) / (time1 - time0)"; + + case PERF_COUNTER_TIMER_INV: + return "100 * (1 - ((data1 - data0) / (time1 - time0)))"; + + case PERF_100NSEC_TIMER_INV: + return "100 * (1- (data1 - data0) / (time1 - time0))"; + + case PERF_COUNTER_MULTI_TIMER: + return "100 * ((data1 - data0) / ((time1 - time0) / frequency1)) / multi1"; + + case PERF_100NSEC_MULTI_TIMER: + return "100 * ((data1 - data0) / (time1 - time0)) / multi1"; + + case PERF_COUNTER_MULTI_TIMER_INV: + case PERF_100NSEC_MULTI_TIMER_INV: + return "100 * (multi1 - ((data1 - data0) / (time1 - time0)))"; + + case PERF_COUNTER_RAWCOUNT: + case PERF_COUNTER_LARGE_RAWCOUNT: + return "data0"; + + case PERF_COUNTER_RAWCOUNT_HEX: + case PERF_COUNTER_LARGE_RAWCOUNT_HEX: + return "hex(data0)"; + + case PERF_COUNTER_DELTA: + case PERF_COUNTER_LARGE_DELTA: + return "data1 - data0"; + + case PERF_RAW_FRACTION: + case PERF_LARGE_RAW_FRACTION: + return "100 * data0 / time0"; + + case PERF_AVERAGE_TIMER: + return "((data1 - data0) / frequency1) / (time1 - time0)"; + + case PERF_ELAPSED_TIME: + return "(time0 - data0) / frequency0"; + + case PERF_COUNTER_TEXT: + case PERF_SAMPLE_BASE: + case PERF_AVERAGE_BASE: + case PERF_COUNTER_MULTI_BASE: + case PERF_RAW_BASE: + case PERF_COUNTER_NODATA: + case PERF_PRECISION_TIMESTAMP: + default: + return ""; + } +} + +void dumpSystemTime(BUFFER *wb, SYSTEMTIME *st) { + buffer_json_member_add_uint64(wb, "Year", st->wYear); + buffer_json_member_add_uint64(wb, "Month", st->wMonth); + buffer_json_member_add_uint64(wb, "DayOfWeek", st->wDayOfWeek); + buffer_json_member_add_uint64(wb, "Day", st->wDay); + buffer_json_member_add_uint64(wb, "Hour", st->wHour); + buffer_json_member_add_uint64(wb, "Minute", st->wMinute); + buffer_json_member_add_uint64(wb, "Second", st->wSecond); + buffer_json_member_add_uint64(wb, "Milliseconds", st->wMilliseconds); +} + +bool dumpDataCb(PERF_DATA_BLOCK *pDataBlock, void *data) { + char name[4096]; + if(!getSystemName(pDataBlock, name, sizeof(name))) + strncpyz(name, "[failed]", sizeof(name) - 1); + + BUFFER *wb = data; + buffer_json_member_add_string(wb, "SystemName", name); + + // Number of types of objects being reported + // Type: DWORD + buffer_json_member_add_int64(wb, "NumObjectTypes", pDataBlock->NumObjectTypes); + + buffer_json_member_add_int64(wb, "LittleEndian", pDataBlock->LittleEndian); + + // Version and Revision of these data structures. + // Version starts at 1. + // Revision starts at 0 for each Version. + // Type: DWORD + buffer_json_member_add_int64(wb, "Version", pDataBlock->Version); + buffer_json_member_add_int64(wb, "Revision", pDataBlock->Revision); + + // Object Title Index of default object to display when data from this system is retrieved + // (-1 = none, but this is not expected to be used) + // Type: LONG + buffer_json_member_add_int64(wb, "DefaultObject", pDataBlock->DefaultObject); + + // Performance counter frequency at the system under measurement + // Type: LARGE_INTEGER + buffer_json_member_add_int64(wb, "PerfFreq", pDataBlock->PerfFreq.QuadPart); + + // Performance counter value at the system under measurement + // Type: LARGE_INTEGER + buffer_json_member_add_int64(wb, "PerfTime", pDataBlock->PerfTime.QuadPart); + + // Performance counter time in 100 nsec units at the system under measurement + // Type: LARGE_INTEGER + buffer_json_member_add_int64(wb, "PerfTime100nSec", pDataBlock->PerfTime100nSec.QuadPart); + + // Time at the system under measurement in UTC + // Type: SYSTEMTIME + buffer_json_member_add_object(wb, "SystemTime"); + dumpSystemTime(wb, &pDataBlock->SystemTime); + buffer_json_object_close(wb); + + if(pDataBlock->NumObjectTypes) + buffer_json_member_add_array(wb, "Objects"); + + return true; +} + +static const char *GetDetailLevel(DWORD num) { + switch (num) { + case 100: + return "Novice (100)"; + case 200: + return "Advanced (200)"; + case 300: + return "Expert (300)"; + case 400: + return "Wizard (400)"; + + default: + return "Unknown"; + } +} + +bool dumpObjectCb(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, void *data) { + (void)pDataBlock; + BUFFER *wb = data; + if(!pObjectType) { + buffer_json_array_close(wb); // instances or counters + buffer_json_object_close(wb); // objectType + return true; + } + + buffer_json_add_array_item_object(wb); // objectType + buffer_json_member_add_int64(wb, "NameId", pObjectType->ObjectNameTitleIndex); + buffer_json_member_add_string(wb, "Name", RegistryFindNameByID(pObjectType->ObjectNameTitleIndex)); + buffer_json_member_add_int64(wb, "HelpId", pObjectType->ObjectHelpTitleIndex); + buffer_json_member_add_string(wb, "Help", RegistryFindHelpByID(pObjectType->ObjectHelpTitleIndex)); + buffer_json_member_add_int64(wb, "NumInstances", pObjectType->NumInstances); + buffer_json_member_add_int64(wb, "NumCounters", pObjectType->NumCounters); + buffer_json_member_add_int64(wb, "PerfTime", pObjectType->PerfTime.QuadPart); + buffer_json_member_add_int64(wb, "PerfFreq", pObjectType->PerfFreq.QuadPart); + buffer_json_member_add_int64(wb, "CodePage", pObjectType->CodePage); + buffer_json_member_add_int64(wb, "DefaultCounter", pObjectType->DefaultCounter); + buffer_json_member_add_string(wb, "DetailLevel", GetDetailLevel(pObjectType->DetailLevel)); + + if(ObjectTypeHasInstances(pDataBlock, pObjectType)) + buffer_json_member_add_array(wb, "Instances"); + else + buffer_json_member_add_array(wb, "Counters"); + + return true; +} + +bool dumpInstanceCb(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, void *data) { + (void)pDataBlock; + BUFFER *wb = data; + if(!pInstance) { + buffer_json_array_close(wb); // counters + buffer_json_object_close(wb); // instance + return true; + } + + char name[4096]; + if(!getInstanceName(pDataBlock, pObjectType, pInstance, name, sizeof(name))) + strncpyz(name, "[failed]", sizeof(name) - 1); + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "Instance", name); + buffer_json_member_add_int64(wb, "UniqueID", pInstance->UniqueID); + buffer_json_member_add_array(wb, "Labels"); + { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "key", RegistryFindNameByID(pObjectType->ObjectNameTitleIndex)); + buffer_json_member_add_string(wb, "value", name); + } + buffer_json_object_close(wb); + + if(pInstance->ParentObjectTitleIndex) { + PERF_INSTANCE_DEFINITION *pi = pInstance; + while(pi->ParentObjectTitleIndex) { + PERF_OBJECT_TYPE *po = getObjectTypeByIndex(pDataBlock, pInstance->ParentObjectTitleIndex); + pi = getInstanceByPosition(pDataBlock, po, pi->ParentObjectInstance); + + if(!getInstanceName(pDataBlock, po, pi, name, sizeof(name))) + strncpyz(name, "[failed]", sizeof(name) - 1); + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "key", RegistryFindNameByID(po->ObjectNameTitleIndex)); + buffer_json_member_add_string(wb, "value", name); + } + buffer_json_object_close(wb); + } + } + } + buffer_json_array_close(wb); // rrdlabels + + buffer_json_member_add_array(wb, "Counters"); + return true; +} + +void dumpSample(BUFFER *wb, RAW_DATA *d) { + buffer_json_member_add_object(wb, "Value"); + buffer_json_member_add_uint64(wb, "data", d->Data); + buffer_json_member_add_int64(wb, "time", d->Time); + buffer_json_member_add_uint64(wb, "type", d->CounterType); + buffer_json_member_add_int64(wb, "multi", d->MultiCounterData); + buffer_json_member_add_int64(wb, "frequency", d->Frequency); + buffer_json_object_close(wb); +} + +bool dumpCounterCb(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_COUNTER_DEFINITION *pCounter, RAW_DATA *sample, void *data) { + (void)pDataBlock; + (void)pObjectType; + BUFFER *wb = data; + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "Counter", RegistryFindNameByID(pCounter->CounterNameTitleIndex)); + dumpSample(wb, sample); + buffer_json_member_add_string(wb, "Help", RegistryFindHelpByID(pCounter->CounterHelpTitleIndex)); + buffer_json_member_add_string(wb, "Type", getCounterType(pCounter->CounterType)); + buffer_json_member_add_string(wb, "Algorithm", getCounterAlgorithm(pCounter->CounterType)); + buffer_json_member_add_string(wb, "Description", getCounterDescription(pCounter->CounterType)); + buffer_json_object_close(wb); + return true; +} + +bool dumpInstanceCounterCb(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, PERF_COUNTER_DEFINITION *pCounter, RAW_DATA *sample, void *data) { + (void)pInstance; + return dumpCounterCb(pDataBlock, pObjectType, pCounter, sample, data); +} + + +int windows_perflib_dump(const char *key) { + if(key && !*key) + key = NULL; + + PerflibNamesRegistryInitialize(); + + DWORD id = 0; + if(key) { + id = RegistryFindIDByName(key); + if(id == PERFLIB_REGISTRY_NAME_NOT_FOUND) { + fprintf(stderr, "Cannot find key '%s' in Windows Performance Counters Registry.\n", key); + exit(1); + } + } + + CLEAN_BUFFER *wb = buffer_create(0, NULL); + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); + + perflibQueryAndTraverse(id, dumpDataCb, dumpObjectCb, dumpInstanceCb, dumpInstanceCounterCb, dumpCounterCb, wb); + + buffer_json_finalize(wb); + printf("\n%s\n", buffer_tostring(wb)); + + perflibFreePerformanceData(); + + return 0; +} + +#endif // OS_WINDOWS
\ No newline at end of file diff --git a/src/libnetdata/os/windows-perflib/perflib-names.c b/src/libnetdata/os/windows-perflib/perflib-names.c new file mode 100644 index 000000000..18ff2af65 --- /dev/null +++ b/src/libnetdata/os/windows-perflib/perflib-names.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "perflib.h" + +#if defined(OS_WINDOWS) +#define REGISTRY_KEY "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Perflib\\009" + +typedef struct perflib_registry { + DWORD id; + char *key; + char *help; +} perfLibRegistryEntry; + +static inline bool compare_perfLibRegistryEntry(const char *k1, const char *k2) { + return strcmp(k1, k2) == 0; +} + +static inline const char *value2key_perfLibRegistryEntry(perfLibRegistryEntry *entry) { + return entry->key; +} + +#define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION compare_perfLibRegistryEntry +#define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION value2key_perfLibRegistryEntry +#define SIMPLE_HASHTABLE_KEY_TYPE const char +#define SIMPLE_HASHTABLE_VALUE_TYPE perfLibRegistryEntry +#define SIMPLE_HASHTABLE_NAME _PERFLIB +#include "libnetdata/simple_hashtable/simple_hashtable.h" + +static struct { + SPINLOCK spinlock; + size_t size; + perfLibRegistryEntry **array; + struct simple_hashtable_PERFLIB hashtable; + FILETIME lastWriteTime; +} names_globals = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .size = 0, + .array = NULL, +}; + +DWORD RegistryFindIDByName(const char *name) { + DWORD rc = PERFLIB_REGISTRY_NAME_NOT_FOUND; + + spinlock_lock(&names_globals.spinlock); + XXH64_hash_t hash = XXH3_64bits((void *)name, strlen(name)); + SIMPLE_HASHTABLE_SLOT_PERFLIB *sl = simple_hashtable_get_slot_PERFLIB(&names_globals.hashtable, hash, name, false); + perfLibRegistryEntry *e = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(e) rc = e->id; + spinlock_unlock(&names_globals.spinlock); + + return rc; +} + +static inline void RegistryAddToHashTable_unsafe(perfLibRegistryEntry *entry) { + XXH64_hash_t hash = XXH3_64bits((void *)entry->key, strlen(entry->key)); + SIMPLE_HASHTABLE_SLOT_PERFLIB *sl = simple_hashtable_get_slot_PERFLIB(&names_globals.hashtable, hash, entry->key, true); + perfLibRegistryEntry *e = SIMPLE_HASHTABLE_SLOT_DATA(sl); + if(!e || e->id > entry->id) + simple_hashtable_set_slot_PERFLIB(&names_globals.hashtable, sl, hash, entry); +} + +static void RegistrySetData_unsafe(DWORD id, const char *key, const char *help) { + if(id >= names_globals.size) { + // increase the size of the array + + size_t old_size = names_globals.size; + + if(!names_globals.size) + names_globals.size = 20000; + else + names_globals.size *= 2; + + names_globals.array = reallocz(names_globals.array, names_globals.size * sizeof(perfLibRegistryEntry *)); + + memset(names_globals.array + old_size, 0, (names_globals.size - old_size) * sizeof(perfLibRegistryEntry *)); + } + + perfLibRegistryEntry *entry = names_globals.array[id]; + if(!entry) + entry = names_globals.array[id] = (perfLibRegistryEntry *)calloc(1, sizeof(perfLibRegistryEntry)); + + bool add_to_hash = false; + if(key && !entry->key) { + entry->key = strdup(key); + add_to_hash = true; + } + + if(help && !entry->help) + entry->help = strdup(help); + + entry->id = id; + + if(add_to_hash) + RegistryAddToHashTable_unsafe(entry); +} + +const char *RegistryFindNameByID(DWORD id) { + const char *s = ""; + spinlock_lock(&names_globals.spinlock); + + if(id < names_globals.size) { + perfLibRegistryEntry *titleEntry = names_globals.array[id]; + if(titleEntry && titleEntry->key) + s = titleEntry->key; + } + + spinlock_unlock(&names_globals.spinlock); + return s; +} + +const char *RegistryFindHelpByID(DWORD id) { + const char *s = ""; + spinlock_lock(&names_globals.spinlock); + + if(id < names_globals.size) { + perfLibRegistryEntry *titleEntry = names_globals.array[id]; + if(titleEntry && titleEntry->help) + s = titleEntry->help; + } + + spinlock_unlock(&names_globals.spinlock); + return s; +} + +// ---------------------------------------------------------- + +static inline void readRegistryKeys_unsafe(BOOL helps) { + TCHAR *pData = NULL; + + HKEY hKey; + DWORD dwType; + DWORD dwSize = 0; + LONG lStatus; + + LPCSTR valueName; + if(helps) + valueName = TEXT("help"); + else + valueName = TEXT("CounterDefinition"); + + // Open the key for the English counters + lStatus = RegOpenKeyEx(HKEY_LOCAL_MACHINE, TEXT(REGISTRY_KEY), 0, KEY_READ, &hKey); + if (lStatus != ERROR_SUCCESS) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Failed to open registry key HKEY_LOCAL_MACHINE, subkey '%s', error %ld\n", REGISTRY_KEY, (long)lStatus); + return; + } + + // Get the size of the 'Counters' data + lStatus = RegQueryValueEx(hKey, valueName, NULL, &dwType, NULL, &dwSize); + if (lStatus != ERROR_SUCCESS) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Failed to get registry key HKEY_LOCAL_MACHINE, subkey '%s', value '%s', size of data, error %ld\n", + REGISTRY_KEY, (const char *)valueName, (long)lStatus); + goto cleanup; + } + + // Allocate memory for the data + pData = mallocz(dwSize); + + // Read the 'Counters' data + lStatus = RegQueryValueEx(hKey, valueName, NULL, &dwType, (LPBYTE)pData, &dwSize); + if (lStatus != ERROR_SUCCESS || dwType != REG_MULTI_SZ) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Failed to get registry key HKEY_LOCAL_MACHINE, subkey '%s', value '%s', data, error %ld\n", + REGISTRY_KEY, (const char *)valueName, (long)lStatus); + goto cleanup; + } + + // Process the counter data + TCHAR *ptr = pData; + while (*ptr) { + TCHAR *sid = ptr; // First string is the ID + ptr += lstrlen(ptr) + 1; // Move to the next string + TCHAR *name = ptr; // Second string is the name + ptr += lstrlen(ptr) + 1; // Move to the next pair + + DWORD id = strtoul(sid, NULL, 10); + + if(helps) + RegistrySetData_unsafe(id, NULL, name); + else + RegistrySetData_unsafe(id, name, NULL); + } + +cleanup: + if(pData) freez(pData); + RegCloseKey(hKey); +} + +static BOOL RegistryKeyModification(FILETIME *lastWriteTime) { + HKEY hKey; + LONG lResult; + BOOL ret = FALSE; + + // Open the registry key + lResult = RegOpenKeyEx(HKEY_LOCAL_MACHINE, TEXT(REGISTRY_KEY), 0, KEY_READ, &hKey); + if (lResult != ERROR_SUCCESS) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Failed to open registry key HKEY_LOCAL_MACHINE, subkey '%s', error %ld\n", REGISTRY_KEY, (long)lResult); + return FALSE; + } + + // Get the last write time + lResult = RegQueryInfoKey(hKey, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, lastWriteTime); + if (lResult != ERROR_SUCCESS) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Failed to query registry key HKEY_LOCAL_MACHINE, subkey '%s', last write time, error %ld\n", REGISTRY_KEY, (long)lResult); + ret = FALSE; + } + else + ret = TRUE; + + RegCloseKey(hKey); + return ret; +} + +static inline void RegistryFetchAll_unsafe(void) { + readRegistryKeys_unsafe(FALSE); + readRegistryKeys_unsafe(TRUE); +} + +void PerflibNamesRegistryInitialize(void) { + spinlock_lock(&names_globals.spinlock); + simple_hashtable_init_PERFLIB(&names_globals.hashtable, 20000); + RegistryKeyModification(&names_globals.lastWriteTime); + RegistryFetchAll_unsafe(); + spinlock_unlock(&names_globals.spinlock); +} + +void PerflibNamesRegistryUpdate(void) { + FILETIME lastWriteTime = { 0 }; + RegistryKeyModification(&lastWriteTime); + + if(CompareFileTime(&lastWriteTime, &names_globals.lastWriteTime) > 0) { + spinlock_lock(&names_globals.spinlock); + if(CompareFileTime(&lastWriteTime, &names_globals.lastWriteTime) > 0) { + names_globals.lastWriteTime = lastWriteTime; + RegistryFetchAll_unsafe(); + } + spinlock_unlock(&names_globals.spinlock); + } +} + +#endif // OS_WINDOWS diff --git a/src/libnetdata/os/windows-perflib/perflib.c b/src/libnetdata/os/windows-perflib/perflib.c new file mode 100644 index 000000000..413a202fa --- /dev/null +++ b/src/libnetdata/os/windows-perflib/perflib.c @@ -0,0 +1,687 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "perflib.h" + +#if defined(OS_WINDOWS) +// -------------------------------------------------------------------------------- + +// Retrieve a buffer that contains the specified performance data. +// The pwszSource parameter determines the data that GetRegistryBuffer returns. +// +// Typically, when calling RegQueryValueEx, you can specify zero for the size of the buffer +// and the RegQueryValueEx will set your size variable to the required buffer size. However, +// if the source is "Global" or one or more object index values, you will need to increment +// the buffer size in a loop until RegQueryValueEx does not return ERROR_MORE_DATA. +static LPBYTE getPerformanceData(const char *pwszSource) { + static __thread DWORD size = 0; + static __thread LPBYTE buffer = NULL; + + if(pwszSource == (const char *)0x01) { + freez(buffer); + buffer = NULL; + size = 0; + return NULL; + } + + if(!size) { + size = 32 * 1024; + buffer = mallocz(size); + } + + LONG status = ERROR_SUCCESS; + while ((status = RegQueryValueEx(HKEY_PERFORMANCE_DATA, pwszSource, + NULL, NULL, buffer, &size)) == ERROR_MORE_DATA) { + size *= 2; + buffer = reallocz(buffer, size); + } + + if (status != ERROR_SUCCESS) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "RegQueryValueEx failed with 0x%x.\n", status); + return NULL; + } + + return buffer; +} + +void perflibFreePerformanceData(void) { + getPerformanceData((const char *)0x01); +} + +// -------------------------------------------------------------------------------------------------------------------- + +// Retrieve the raw counter value and any supporting data needed to calculate +// a displayable counter value. Use the counter type to determine the information +// needed to calculate the value. + +static BOOL getCounterData( + PERF_DATA_BLOCK *pDataBlock, + PERF_OBJECT_TYPE* pObject, + PERF_COUNTER_DEFINITION* pCounter, + PERF_COUNTER_BLOCK* pCounterDataBlock, + PRAW_DATA pRawData) +{ + PVOID pData = NULL; + UNALIGNED ULONGLONG* pullData = NULL; + PERF_COUNTER_DEFINITION* pBaseCounter = NULL; + BOOL fSuccess = TRUE; + + //Point to the raw counter data. + pData = (PVOID)((LPBYTE)pCounterDataBlock + pCounter->CounterOffset); + + //Now use the PERF_COUNTER_DEFINITION.CounterType value to figure out what + //other information you need to calculate a displayable value. + switch (pCounter->CounterType) { + + case PERF_COUNTER_COUNTER: + case PERF_COUNTER_QUEUELEN_TYPE: + case PERF_SAMPLE_COUNTER: + pRawData->Data = (ULONGLONG)(*(DWORD*)pData); + pRawData->Time = pDataBlock->PerfTime.QuadPart; + if (PERF_COUNTER_COUNTER == pCounter->CounterType || PERF_SAMPLE_COUNTER == pCounter->CounterType) + pRawData->Frequency = pDataBlock->PerfFreq.QuadPart; + break; + + case PERF_OBJ_TIME_TIMER: + pRawData->Data = (ULONGLONG)(*(DWORD*)pData); + pRawData->Time = pObject->PerfTime.QuadPart; + break; + + case PERF_COUNTER_100NS_QUEUELEN_TYPE: + pRawData->Data = *(UNALIGNED ULONGLONG *)pData; + pRawData->Time = pDataBlock->PerfTime100nSec.QuadPart; + break; + + case PERF_COUNTER_OBJ_TIME_QUEUELEN_TYPE: + pRawData->Data = *(UNALIGNED ULONGLONG *)pData; + pRawData->Time = pObject->PerfTime.QuadPart; + break; + + case PERF_COUNTER_TIMER: + case PERF_COUNTER_TIMER_INV: + case PERF_COUNTER_BULK_COUNT: + case PERF_COUNTER_LARGE_QUEUELEN_TYPE: + pullData = (UNALIGNED ULONGLONG *)pData; + pRawData->Data = *pullData; + pRawData->Time = pDataBlock->PerfTime.QuadPart; + if (pCounter->CounterType == PERF_COUNTER_BULK_COUNT) + pRawData->Frequency = pDataBlock->PerfFreq.QuadPart; + break; + + case PERF_COUNTER_MULTI_TIMER: + case PERF_COUNTER_MULTI_TIMER_INV: + pullData = (UNALIGNED ULONGLONG *)pData; + pRawData->Data = *pullData; + pRawData->Frequency = pDataBlock->PerfFreq.QuadPart; + pRawData->Time = pDataBlock->PerfTime.QuadPart; + + //These counter types have a second counter value that is adjacent to + //this counter value in the counter data block. The value is needed for + //the calculation. + if ((pCounter->CounterType & PERF_MULTI_COUNTER) == PERF_MULTI_COUNTER) { + ++pullData; + pRawData->MultiCounterData = *(DWORD*)pullData; + } + break; + + //These counters do not use any time reference. + case PERF_COUNTER_RAWCOUNT: + case PERF_COUNTER_RAWCOUNT_HEX: + case PERF_COUNTER_DELTA: + // some counters in these categories, have CounterSize = sizeof(ULONGLONG) + // but the official documentation always uses them as sizeof(DWORD) + pRawData->Data = (ULONGLONG)(*(DWORD*)pData); + pRawData->Time = 0; + break; + + case PERF_COUNTER_LARGE_RAWCOUNT: + case PERF_COUNTER_LARGE_RAWCOUNT_HEX: + case PERF_COUNTER_LARGE_DELTA: + pRawData->Data = *(UNALIGNED ULONGLONG*)pData; + pRawData->Time = 0; + break; + + //These counters use the 100ns time base in their calculation. + case PERF_100NSEC_TIMER: + case PERF_100NSEC_TIMER_INV: + case PERF_100NSEC_MULTI_TIMER: + case PERF_100NSEC_MULTI_TIMER_INV: + pullData = (UNALIGNED ULONGLONG*)pData; + pRawData->Data = *pullData; + pRawData->Time = pDataBlock->PerfTime100nSec.QuadPart; + + //These counter types have a second counter value that is adjacent to + //this counter value in the counter data block. The value is needed for + //the calculation. + if ((pCounter->CounterType & PERF_MULTI_COUNTER) == PERF_MULTI_COUNTER) { + ++pullData; + pRawData->MultiCounterData = *(DWORD*)pullData; + } + break; + + //These counters use two data points, this value and one from this counter's + //base counter. The base counter should be the next counter in the object's + //list of counters. + case PERF_SAMPLE_FRACTION: + case PERF_RAW_FRACTION: + pRawData->Data = (ULONGLONG)(*(DWORD*)pData); + pBaseCounter = pCounter + 1; //Get base counter + if ((pBaseCounter->CounterType & PERF_COUNTER_BASE) == PERF_COUNTER_BASE) { + pData = (PVOID)((LPBYTE)pCounterDataBlock + pBaseCounter->CounterOffset); + pRawData->Time = (LONGLONG)(*(DWORD*)pData); + } + else + fSuccess = FALSE; + break; + + case PERF_LARGE_RAW_FRACTION: + case PERF_PRECISION_SYSTEM_TIMER: + case PERF_PRECISION_100NS_TIMER: + case PERF_PRECISION_OBJECT_TIMER: + pRawData->Data = *(UNALIGNED ULONGLONG*)pData; + pBaseCounter = pCounter + 1; + if ((pBaseCounter->CounterType & PERF_COUNTER_BASE) == PERF_COUNTER_BASE) { + pData = (PVOID)((LPBYTE)pCounterDataBlock + pBaseCounter->CounterOffset); + pRawData->Time = *(LONGLONG*)pData; + } + else + fSuccess = FALSE; + break; + + case PERF_AVERAGE_TIMER: + case PERF_AVERAGE_BULK: + pRawData->Data = *(UNALIGNED ULONGLONG*)pData; + pBaseCounter = pCounter+1; + if ((pBaseCounter->CounterType & PERF_COUNTER_BASE) == PERF_COUNTER_BASE) { + pData = (PVOID)((LPBYTE)pCounterDataBlock + pBaseCounter->CounterOffset); + pRawData->Time = *(DWORD*)pData; + } + else + fSuccess = FALSE; + + if (pCounter->CounterType == PERF_AVERAGE_TIMER) + pRawData->Frequency = pDataBlock->PerfFreq.QuadPart; + break; + + //These are base counters and are used in calculations for other counters. + //This case should never be entered. + case PERF_SAMPLE_BASE: + case PERF_AVERAGE_BASE: + case PERF_COUNTER_MULTI_BASE: + case PERF_RAW_BASE: + case PERF_LARGE_RAW_BASE: + pRawData->Data = 0; + pRawData->Time = 0; + fSuccess = FALSE; + break; + + case PERF_ELAPSED_TIME: + pRawData->Data = *(UNALIGNED ULONGLONG*)pData; + pRawData->Time = pObject->PerfTime.QuadPart; + pRawData->Frequency = pObject->PerfFreq.QuadPart; + break; + + //These counters are currently not supported. + case PERF_COUNTER_TEXT: + case PERF_COUNTER_NODATA: + case PERF_COUNTER_HISTOGRAM_TYPE: + default: // unknown counter types + pRawData->Data = 0; + pRawData->Time = 0; + fSuccess = FALSE; + break; + } + + return fSuccess; +} + +// -------------------------------------------------------------------------------------------------------------------- + +static inline BOOL isValidPointer(PERF_DATA_BLOCK *pDataBlock __maybe_unused, void *ptr __maybe_unused) { +#ifdef NETDATA_INTERNAL_CHECKS + return (PBYTE)ptr >= (PBYTE)pDataBlock + pDataBlock->TotalByteLength ? FALSE : TRUE; +#else + return TRUE; +#endif +} + +static inline BOOL isValidStructure(PERF_DATA_BLOCK *pDataBlock __maybe_unused, void *ptr __maybe_unused, size_t length __maybe_unused) { +#ifdef NETDATA_INTERNAL_CHECKS + return (PBYTE)ptr + length > (PBYTE)pDataBlock + pDataBlock->TotalByteLength ? FALSE : TRUE; +#else + return TRUE; +#endif +} + +static inline PERF_DATA_BLOCK *getDataBlock(BYTE *pBuffer) { + PERF_DATA_BLOCK *pDataBlock = (PERF_DATA_BLOCK *)pBuffer; + + static WCHAR signature[] = { 'P', 'E', 'R', 'F' }; + + if(memcmp(pDataBlock->Signature, signature, sizeof(signature)) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid data block signature."); + return NULL; + } + + if(!isValidPointer(pDataBlock, (PBYTE)pDataBlock + pDataBlock->SystemNameOffset) || + !isValidStructure(pDataBlock, (PBYTE)pDataBlock + pDataBlock->SystemNameOffset, pDataBlock->SystemNameLength)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid system name array."); + return NULL; + } + + return pDataBlock; +} + +static inline PERF_OBJECT_TYPE *getObjectType(PERF_DATA_BLOCK* pDataBlock, PERF_OBJECT_TYPE *lastObjectType) { + PERF_OBJECT_TYPE* pObjectType = NULL; + + if(!lastObjectType) + pObjectType = (PERF_OBJECT_TYPE *)((PBYTE)pDataBlock + pDataBlock->HeaderLength); + else if (lastObjectType->TotalByteLength != 0) + pObjectType = (PERF_OBJECT_TYPE *)((PBYTE)lastObjectType + lastObjectType->TotalByteLength); + + if(pObjectType && (!isValidPointer(pDataBlock, pObjectType) || !isValidStructure(pDataBlock, pObjectType, pObjectType->TotalByteLength))) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid ObjectType!"); + pObjectType = NULL; + } + + return pObjectType; +} + +inline PERF_OBJECT_TYPE *getObjectTypeByIndex(PERF_DATA_BLOCK *pDataBlock, DWORD ObjectNameTitleIndex) { + PERF_OBJECT_TYPE *po = NULL; + for(DWORD o = 0; o < pDataBlock->NumObjectTypes ; o++) { + po = getObjectType(pDataBlock, po); + if(po->ObjectNameTitleIndex == ObjectNameTitleIndex) + return po; + } + + return NULL; +} + +static inline PERF_INSTANCE_DEFINITION *getInstance( + PERF_DATA_BLOCK *pDataBlock, + PERF_OBJECT_TYPE *pObjectType, + PERF_COUNTER_BLOCK *lastCounterBlock +) { + PERF_INSTANCE_DEFINITION *pInstance; + + if(!lastCounterBlock) + pInstance = (PERF_INSTANCE_DEFINITION *)((PBYTE)pObjectType + pObjectType->DefinitionLength); + else + pInstance = (PERF_INSTANCE_DEFINITION *)((PBYTE)lastCounterBlock + lastCounterBlock->ByteLength); + + if(pInstance && (!isValidPointer(pDataBlock, pInstance) || !isValidStructure(pDataBlock, pInstance, pInstance->ByteLength))) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid Instance Definition!"); + pInstance = NULL; + } + + return pInstance; +} + +static inline PERF_COUNTER_BLOCK *getObjectTypeCounterBlock( + PERF_DATA_BLOCK *pDataBlock, + PERF_OBJECT_TYPE *pObjectType +) { + PERF_COUNTER_BLOCK *pCounterBlock = (PERF_COUNTER_BLOCK *)((PBYTE)pObjectType + pObjectType->DefinitionLength); + + if(pCounterBlock && (!isValidPointer(pDataBlock, pCounterBlock) || !isValidStructure(pDataBlock, pCounterBlock, pCounterBlock->ByteLength))) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid ObjectType CounterBlock!"); + pCounterBlock = NULL; + } + + return pCounterBlock; +} + +static inline PERF_COUNTER_BLOCK *getInstanceCounterBlock( + PERF_DATA_BLOCK *pDataBlock, + PERF_OBJECT_TYPE *pObjectType, + PERF_INSTANCE_DEFINITION *pInstance +) { + (void)pObjectType; + PERF_COUNTER_BLOCK *pCounterBlock = (PERF_COUNTER_BLOCK *)((PBYTE)pInstance + pInstance->ByteLength); + + if(pCounterBlock && (!isValidPointer(pDataBlock, pCounterBlock) || !isValidStructure(pDataBlock, pCounterBlock, pCounterBlock->ByteLength))) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid Instance CounterBlock!"); + pCounterBlock = NULL; + } + + return pCounterBlock; +} + +inline PERF_INSTANCE_DEFINITION *getInstanceByPosition(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, DWORD instancePosition) { + PERF_INSTANCE_DEFINITION *pi = NULL; + PERF_COUNTER_BLOCK *pc = NULL; + for(DWORD i = 0; i <= instancePosition ;i++) { + pi = getInstance(pDataBlock, pObjectType, pc); + pc = getInstanceCounterBlock(pDataBlock, pObjectType, pi); + } + return pi; +} + +static inline PERF_COUNTER_DEFINITION *getCounterDefinition(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_COUNTER_DEFINITION *lastCounterDefinition) { + PERF_COUNTER_DEFINITION *pCounterDefinition = NULL; + + if(!lastCounterDefinition) + pCounterDefinition = (PERF_COUNTER_DEFINITION *)((PBYTE)pObjectType + pObjectType->HeaderLength); + else + pCounterDefinition = (PERF_COUNTER_DEFINITION *)((PBYTE)lastCounterDefinition + lastCounterDefinition->ByteLength); + + if(pCounterDefinition && (!isValidPointer(pDataBlock, pCounterDefinition) || !isValidStructure(pDataBlock, pCounterDefinition, pCounterDefinition->ByteLength))) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Invalid Counter Definition!"); + pCounterDefinition = NULL; + } + + return pCounterDefinition; +} + +// -------------------------------------------------------------------------------------------------------------------- + +static inline BOOL getEncodedStringToUTF8(char *dst, size_t dst_len, DWORD CodePage, char *start, DWORD length) { + static __thread wchar_t unicode[PERFLIB_MAX_NAME_LENGTH]; + + WCHAR *tempBuffer; // Temporary buffer for Unicode data + DWORD charsCopied = 0; + + if (CodePage == 0) { + // Input is already Unicode (UTF-16) + tempBuffer = (WCHAR *)start; + charsCopied = length / sizeof(WCHAR); // Convert byte length to number of WCHARs + } + else { + tempBuffer = unicode; + charsCopied = any_to_utf16(CodePage, unicode, _countof(unicode), start, (int)length, NULL); + if(!charsCopied) return FALSE; + } + + // Now convert from Unicode (UTF-16) to UTF-8 + int bytesCopied = WideCharToMultiByte(CP_UTF8, 0, tempBuffer, (int)charsCopied, dst, (int)dst_len, NULL, NULL); + if (bytesCopied == 0) { + dst[0] = '\0'; // Ensure the buffer is null-terminated even on failure + return FALSE; + } + + dst[bytesCopied - 1] = '\0'; // Ensure buffer is null-terminated + return TRUE; +} + +inline BOOL getInstanceName(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, + char *buffer, size_t bufferLen) { + (void)pDataBlock; + if (!pInstance || !buffer || !bufferLen) return FALSE; + + return getEncodedStringToUTF8(buffer, bufferLen, pObjectType->CodePage, + ((char *)pInstance + pInstance->NameOffset), pInstance->NameLength); +} + +inline BOOL getSystemName(PERF_DATA_BLOCK *pDataBlock, char *buffer, size_t bufferLen) { + return getEncodedStringToUTF8(buffer, bufferLen, 0, + ((char *)pDataBlock + pDataBlock->SystemNameOffset), pDataBlock->SystemNameLength); +} + +inline bool ObjectTypeHasInstances(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType) { + (void)pDataBlock; + return pObjectType->NumInstances != PERF_NO_INSTANCES && pObjectType->NumInstances > 0; +} + +PERF_OBJECT_TYPE *perflibFindObjectTypeByName(PERF_DATA_BLOCK *pDataBlock, const char *name) { + PERF_OBJECT_TYPE* pObjectType = NULL; + for(DWORD o = 0; o < pDataBlock->NumObjectTypes; o++) { + pObjectType = getObjectType(pDataBlock, pObjectType); + if(strcmp(name, RegistryFindNameByID(pObjectType->ObjectNameTitleIndex)) == 0) + return pObjectType; + } + + return NULL; +} + +PERF_INSTANCE_DEFINITION *perflibForEachInstance(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *lastInstance) { + if(!ObjectTypeHasInstances(pDataBlock, pObjectType)) + return NULL; + + return getInstance(pDataBlock, pObjectType, + lastInstance ? + getInstanceCounterBlock(pDataBlock, pObjectType, lastInstance) : + NULL ); +} + +bool perflibGetInstanceCounter(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, COUNTER_DATA *cd) { + DWORD id = cd->id; + const char *key = cd->key; + internal_fatal(key == NULL, "You have to set a key for this call."); + + if(unlikely(cd->failures >= PERFLIB_MAX_FAILURES_TO_FIND_METRIC)) { + // we don't want to lookup and compare strings all the time + // when a metric is not there, so we try to find it for + // XX times, and then we give up. + + if(cd->failures == PERFLIB_MAX_FAILURES_TO_FIND_METRIC) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Giving up on metric '%s' (tried to find it %u times).", + cd->key, cd->failures); + + cd->failures++; // increment it once, so that we will not log this again + } + + goto failed; + } + + PERF_COUNTER_DEFINITION *pCounterDefinition = NULL; + for(DWORD c = 0; c < pObjectType->NumCounters ;c++) { + pCounterDefinition = getCounterDefinition(pDataBlock, pObjectType, pCounterDefinition); + if(!pCounterDefinition) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read counter definition No %u (out of %u)", + c, pObjectType->NumCounters); + break; + } + + if(id) { + if(id != pCounterDefinition->CounterNameTitleIndex) + continue; + } + else { + const char *name = RegistryFindNameByID(pCounterDefinition->CounterNameTitleIndex); + if(strcmp(name, key) != 0) + continue; + + cd->id = pCounterDefinition->CounterNameTitleIndex; + } + + cd->current.CounterType = cd->OverwriteCounterType ? cd->OverwriteCounterType : pCounterDefinition->CounterType; + PERF_COUNTER_BLOCK *pCounterBlock = getInstanceCounterBlock(pDataBlock, pObjectType, pInstance); + + cd->previous = cd->current; + if(likely(getCounterData(pDataBlock, pObjectType, pCounterDefinition, pCounterBlock, &cd->current))) { + cd->updated = true; + cd->failures = 0; + return true; + } + } + + cd->failures++; + +failed: + cd->previous = cd->current; + cd->current = RAW_DATA_EMPTY; + cd->updated = false; + return false; +} + +bool perflibGetObjectCounter(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, COUNTER_DATA *cd) { + PERF_COUNTER_DEFINITION *pCounterDefinition = NULL; + for(DWORD c = 0; c < pObjectType->NumCounters ;c++) { + pCounterDefinition = getCounterDefinition(pDataBlock, pObjectType, pCounterDefinition); + if(!pCounterDefinition) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read counter definition No %u (out of %u)", + c, pObjectType->NumCounters); + break; + } + + if(cd->id) { + if(cd->id != pCounterDefinition->CounterNameTitleIndex) + continue; + } + else { + if(strcmp(RegistryFindNameByID(pCounterDefinition->CounterNameTitleIndex), cd->key) != 0) + continue; + + cd->id = pCounterDefinition->CounterNameTitleIndex; + } + + cd->current.CounterType = cd->OverwriteCounterType ? cd->OverwriteCounterType : pCounterDefinition->CounterType; + PERF_COUNTER_BLOCK *pCounterBlock = getObjectTypeCounterBlock(pDataBlock, pObjectType); + + cd->previous = cd->current; + cd->updated = getCounterData(pDataBlock, pObjectType, pCounterDefinition, pCounterBlock, &cd->current); + return cd->updated; + } + + cd->previous = cd->current; + cd->current = RAW_DATA_EMPTY; + cd->updated = false; + return false; +} + +PERF_DATA_BLOCK *perflibGetPerformanceData(DWORD id) { + char source[24]; + snprintfz(source, sizeof(source), "%u", id); + + LPBYTE pData = (LPBYTE)getPerformanceData((id > 0) ? source : NULL); + if (!pData) return NULL; + + PERF_DATA_BLOCK *pDataBlock = getDataBlock(pData); + if(!pDataBlock) return NULL; + + return pDataBlock; +} + +int perflibQueryAndTraverse(DWORD id, + perflib_data_cb dataCb, + perflib_object_cb objectCb, + perflib_instance_cb instanceCb, + perflib_instance_counter_cb instanceCounterCb, + perflib_counter_cb counterCb, + void *data) { + int counters = -1; + + PERF_DATA_BLOCK *pDataBlock = perflibGetPerformanceData(id); + if(!pDataBlock) goto cleanup; + + bool do_data = true; + if(dataCb) + do_data = dataCb(pDataBlock, data); + + PERF_OBJECT_TYPE* pObjectType = NULL; + for(DWORD o = 0; do_data && o < pDataBlock->NumObjectTypes; o++) { + pObjectType = getObjectType(pDataBlock, pObjectType); + if(!pObjectType) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read object type No %d (out of %d)", + o, pDataBlock->NumObjectTypes); + break; + } + + bool do_object = true; + if(objectCb) + do_object = objectCb(pDataBlock, pObjectType, data); + + if(!do_object) + continue; + + if(ObjectTypeHasInstances(pDataBlock, pObjectType)) { + PERF_INSTANCE_DEFINITION *pInstance = NULL; + PERF_COUNTER_BLOCK *pCounterBlock = NULL; + for(LONG i = 0; i < pObjectType->NumInstances ;i++) { + pInstance = getInstance(pDataBlock, pObjectType, pCounterBlock); + if(!pInstance) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read Instance No %d (out of %d)", + i, pObjectType->NumInstances); + break; + } + + pCounterBlock = getInstanceCounterBlock(pDataBlock, pObjectType, pInstance); + if(!pCounterBlock) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read CounterBlock of instance No %d (out of %d)", + i, pObjectType->NumInstances); + break; + } + + bool do_instance = true; + if(instanceCb) + do_instance = instanceCb(pDataBlock, pObjectType, pInstance, data); + + if(!do_instance) + continue; + + PERF_COUNTER_DEFINITION *pCounterDefinition = NULL; + for(DWORD c = 0; c < pObjectType->NumCounters ;c++) { + pCounterDefinition = getCounterDefinition(pDataBlock, pObjectType, pCounterDefinition); + if(!pCounterDefinition) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read counter definition No %u (out of %u)", + c, pObjectType->NumCounters); + break; + } + + RAW_DATA sample = { + .CounterType = pCounterDefinition->CounterType, + }; + if(getCounterData(pDataBlock, pObjectType, pCounterDefinition, pCounterBlock, &sample)) { + // DisplayCalculatedValue(&sample, &sample); + + if(instanceCounterCb) { + instanceCounterCb(pDataBlock, pObjectType, pInstance, pCounterDefinition, &sample, data); + counters++; + } + } + } + + if(instanceCb) + instanceCb(pDataBlock, pObjectType, NULL, data); + } + } + else { + PERF_COUNTER_BLOCK *pCounterBlock = getObjectTypeCounterBlock(pDataBlock, pObjectType); + PERF_COUNTER_DEFINITION *pCounterDefinition = NULL; + for(DWORD c = 0; c < pObjectType->NumCounters ;c++) { + pCounterDefinition = getCounterDefinition(pDataBlock, pObjectType, pCounterDefinition); + if(!pCounterDefinition) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "WINDOWS: PERFLIB: Cannot read counter definition No %u (out of %u)", + c, pObjectType->NumCounters); + break; + } + + RAW_DATA sample = { + .CounterType = pCounterDefinition->CounterType, + }; + if(getCounterData(pDataBlock, pObjectType, pCounterDefinition, pCounterBlock, &sample)) { + // DisplayCalculatedValue(&sample, &sample); + + if(counterCb) { + counterCb(pDataBlock, pObjectType, pCounterDefinition, &sample, data); + counters++; + } + } + } + } + + if(objectCb) + objectCb(pDataBlock, NULL, data); + } + +cleanup: + return counters; +} + +#endif // OS_WINDOWS
\ No newline at end of file diff --git a/src/libnetdata/os/windows-perflib/perflib.h b/src/libnetdata/os/windows-perflib/perflib.h new file mode 100644 index 000000000..650e5503b --- /dev/null +++ b/src/libnetdata/os/windows-perflib/perflib.h @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_PERFLIB_H +#define NETDATA_PERFLIB_H + +#include "libnetdata/libnetdata.h" + +#if defined(OS_WINDOWS) + +typedef uint32_t DWORD; +typedef long long LONGLONG; +typedef unsigned long long ULONGLONG; +typedef int BOOL; + +struct _PERF_DATA_BLOCK; +typedef struct _PERF_DATA_BLOCK PERF_DATA_BLOCK; +struct _PERF_OBJECT_TYPE; +typedef struct _PERF_OBJECT_TYPE PERF_OBJECT_TYPE; +struct _PERF_INSTANCE_DEFINITION; +typedef struct _PERF_INSTANCE_DEFINITION PERF_INSTANCE_DEFINITION; +struct _PERF_COUNTER_DEFINITION; +typedef struct _PERF_COUNTER_DEFINITION PERF_COUNTER_DEFINITION; + +const char *RegistryFindNameByID(DWORD id); +const char *RegistryFindHelpByID(DWORD id); +DWORD RegistryFindIDByName(const char *name); +#define PERFLIB_REGISTRY_NAME_NOT_FOUND (DWORD)-1 +#define PERFLIB_MAX_NAME_LENGTH 1024 + +PERF_DATA_BLOCK *perflibGetPerformanceData(DWORD id); +void perflibFreePerformanceData(void); +PERF_OBJECT_TYPE *perflibFindObjectTypeByName(PERF_DATA_BLOCK *pDataBlock, const char *name); +PERF_INSTANCE_DEFINITION *perflibForEachInstance(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *lastInstance); + +typedef struct _rawdata { + DWORD CounterType; + DWORD MultiCounterData; // Second raw counter value for multi-valued counters + ULONGLONG Data; // Raw counter data + LONGLONG Time; // Is a time value or a base value + LONGLONG Frequency; +} RAW_DATA, *PRAW_DATA; + +typedef struct _counterdata { + DWORD id; + bool updated; + uint8_t failures; // counts the number of failures to find this key + const char *key; + DWORD OverwriteCounterType; // if set, the counter type will be overwritten once read + RAW_DATA current; + RAW_DATA previous; +} COUNTER_DATA; + +#define PERFLIB_MAX_FAILURES_TO_FIND_METRIC 10 + +#define RAW_DATA_EMPTY (RAW_DATA){ 0 } + +bool perflibGetInstanceCounter(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, COUNTER_DATA *cd); +bool perflibGetObjectCounter(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, COUNTER_DATA *cd); + +typedef bool (*perflib_data_cb)(PERF_DATA_BLOCK *pDataBlock, void *data); +typedef bool (*perflib_object_cb)(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, void *data); +typedef bool (*perflib_instance_cb)(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, void *data); +typedef bool (*perflib_instance_counter_cb)(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, PERF_COUNTER_DEFINITION *pCounter, RAW_DATA *sample, void *data); +typedef bool (*perflib_counter_cb)(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_COUNTER_DEFINITION *pCounter, RAW_DATA *sample, void *data); + +int perflibQueryAndTraverse(DWORD id, + perflib_data_cb dataCb, + perflib_object_cb objectCb, + perflib_instance_cb instanceCb, + perflib_instance_counter_cb instanceCounterCb, + perflib_counter_cb counterCb, + void *data); + +bool ObjectTypeHasInstances(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType); + +BOOL getInstanceName(PERF_DATA_BLOCK *pDataBlock, PERF_OBJECT_TYPE *pObjectType, PERF_INSTANCE_DEFINITION *pInstance, + char *buffer, size_t bufferLen); + +BOOL getSystemName(PERF_DATA_BLOCK *pDataBlock, char *buffer, size_t bufferLen); + +PERF_OBJECT_TYPE *getObjectTypeByIndex(PERF_DATA_BLOCK *pDataBlock, DWORD ObjectNameTitleIndex); + +PERF_INSTANCE_DEFINITION *getInstanceByPosition( + PERF_DATA_BLOCK *pDataBlock, + PERF_OBJECT_TYPE *pObjectType, + DWORD instancePosition); + +void PerflibNamesRegistryInitialize(void); +void PerflibNamesRegistryUpdate(void); + +#endif // OS_WINDOWS +#endif //NETDATA_PERFLIB_H diff --git a/src/libnetdata/os/windows-wmi/windows-wmi-GetDiskDriveInfo.c b/src/libnetdata/os/windows-wmi/windows-wmi-GetDiskDriveInfo.c new file mode 100644 index 000000000..283c6f09e --- /dev/null +++ b/src/libnetdata/os/windows-wmi/windows-wmi-GetDiskDriveInfo.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "windows-wmi-GetDiskDriveInfo.h" + +#if defined(OS_WINDOWS) + +size_t GetDiskDriveInfo(DiskDriveInfoWMI *diskInfoArray, size_t array_size) { + if (InitializeWMI() != S_OK) return 0; + + HRESULT hr; + IEnumWbemClassObject* pEnumerator = NULL; + + // Execute the query, including new properties + BSTR query = SysAllocString(L"SELECT DeviceID, Model, Caption, Name, Partitions, Size, Status, Availability, Index, Manufacturer, InstallDate, MediaType, NeedsCleaning FROM WIN32_DiskDrive"); + BSTR wql = SysAllocString(L"WQL"); + hr = nd_wmi.pSvc->lpVtbl->ExecQuery( + nd_wmi.pSvc, + wql, + query, + WBEM_FLAG_FORWARD_ONLY | WBEM_FLAG_RETURN_IMMEDIATELY, + NULL, + &pEnumerator + ); + SysFreeString(query); + SysFreeString(wql); + + if (FAILED(hr)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "GetDiskDriveInfo() WMI query failed. Error code = 0x%X", hr); + return 0; + } + + // Iterate through the results + IWbemClassObject *pclsObj = NULL; + ULONG uReturn = 0; + size_t index = 0; + while (pEnumerator && index < array_size) { + hr = pEnumerator->lpVtbl->Next(pEnumerator, WBEM_INFINITE, 1, &pclsObj, &uReturn); + if (0 == uReturn) break; + + VARIANT vtProp; + + // Extract DeviceID + hr = pclsObj->lpVtbl->Get(pclsObj, L"DeviceID", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && vtProp.vt == VT_BSTR) { + wcstombs(diskInfoArray[index].DeviceID, vtProp.bstrVal, sizeof(diskInfoArray[index].DeviceID)); + } + VariantClear(&vtProp); + + // Extract Model + hr = pclsObj->lpVtbl->Get(pclsObj, L"Model", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && vtProp.vt == VT_BSTR) { + wcstombs(diskInfoArray[index].Model, vtProp.bstrVal, sizeof(diskInfoArray[index].Model)); + } + VariantClear(&vtProp); + + // Extract Caption + hr = pclsObj->lpVtbl->Get(pclsObj, L"Caption", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && vtProp.vt == VT_BSTR) { + wcstombs(diskInfoArray[index].Caption, vtProp.bstrVal, sizeof(diskInfoArray[index].Caption)); + } + VariantClear(&vtProp); + + // Extract Name + hr = pclsObj->lpVtbl->Get(pclsObj, L"Name", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && vtProp.vt == VT_BSTR) { + wcstombs(diskInfoArray[index].Name, vtProp.bstrVal, sizeof(diskInfoArray[index].Name)); + } + VariantClear(&vtProp); + + // Extract Partitions + hr = pclsObj->lpVtbl->Get(pclsObj, L"Partitions", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && (vtProp.vt == VT_I4 || vtProp.vt == VT_UI4)) { + diskInfoArray[index].Partitions = vtProp.intVal; + } + VariantClear(&vtProp); + + // Extract Size (convert BSTR to uint64) + hr = pclsObj->lpVtbl->Get(pclsObj, L"Size", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && vtProp.vt == VT_BSTR) { + char sizeStr[64]; + wcstombs(sizeStr, vtProp.bstrVal, sizeof(sizeStr)); + diskInfoArray[index].Size = strtoull(sizeStr, NULL, 10); + } + VariantClear(&vtProp); + + // Extract Status + hr = pclsObj->lpVtbl->Get(pclsObj, L"Status", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && vtProp.vt == VT_BSTR) { + wcstombs(diskInfoArray[index].Status, vtProp.bstrVal, sizeof(diskInfoArray[index].Status)); + } + VariantClear(&vtProp); + + // Extract Availability + hr = pclsObj->lpVtbl->Get(pclsObj, L"Availability", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && (vtProp.vt == VT_I4 || vtProp.vt == VT_UI4)) { + diskInfoArray[index].Availability = vtProp.intVal; + } + VariantClear(&vtProp); + + // Extract Index + hr = pclsObj->lpVtbl->Get(pclsObj, L"Index", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && (vtProp.vt == VT_I4 || vtProp.vt == VT_UI4)) { + diskInfoArray[index].Index = vtProp.intVal; + } + VariantClear(&vtProp); + + // Extract Manufacturer + hr = pclsObj->lpVtbl->Get(pclsObj, L"Manufacturer", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && vtProp.vt == VT_BSTR) { + wcstombs(diskInfoArray[index].Manufacturer, vtProp.bstrVal, sizeof(diskInfoArray[index].Manufacturer)); + } + VariantClear(&vtProp); + + // Extract InstallDate + hr = pclsObj->lpVtbl->Get(pclsObj, L"InstallDate", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && vtProp.vt == VT_BSTR) { + wcstombs(diskInfoArray[index].InstallDate, vtProp.bstrVal, sizeof(diskInfoArray[index].InstallDate)); + } + VariantClear(&vtProp); + + // Extract MediaType + hr = pclsObj->lpVtbl->Get(pclsObj, L"MediaType", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && vtProp.vt == VT_BSTR) { + wcstombs(diskInfoArray[index].MediaType, vtProp.bstrVal, sizeof(diskInfoArray[index].MediaType)); + } + VariantClear(&vtProp); + + // Extract NeedsCleaning + hr = pclsObj->lpVtbl->Get(pclsObj, L"NeedsCleaning", 0, &vtProp, 0, 0); + if (SUCCEEDED(hr) && (vtProp.vt == VT_BOOL)) { + diskInfoArray[index].NeedsCleaning = vtProp.boolVal; + } + VariantClear(&vtProp); + + pclsObj->lpVtbl->Release(pclsObj); + index++; + } + + pEnumerator->lpVtbl->Release(pEnumerator); + + return index; +} + +#endif
\ No newline at end of file diff --git a/src/libnetdata/os/windows-wmi/windows-wmi-GetDiskDriveInfo.h b/src/libnetdata/os/windows-wmi/windows-wmi-GetDiskDriveInfo.h new file mode 100644 index 000000000..cc9b46067 --- /dev/null +++ b/src/libnetdata/os/windows-wmi/windows-wmi-GetDiskDriveInfo.h @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_WINDOWS_WMI_GETDISKDRIVEINFO_H +#define NETDATA_WINDOWS_WMI_GETDISKDRIVEINFO_H + +#include "windows-wmi.h" + +#if defined(OS_WINDOWS) + +typedef struct { + char DeviceID[256]; + char Model[256]; + char Caption[256]; + char Name[256]; + int Partitions; + unsigned long long Size; + char Status[64]; + int Availability; + int Index; + char Manufacturer[256]; + char InstallDate[64]; + char MediaType[128]; + bool NeedsCleaning; +} DiskDriveInfoWMI; + +size_t GetDiskDriveInfo(DiskDriveInfoWMI *diskInfoArray, size_t array_size); + +#endif + +#endif //NETDATA_WINDOWS_WMI_GETDISKDRIVEINFO_H diff --git a/src/libnetdata/os/windows-wmi/windows-wmi.c b/src/libnetdata/os/windows-wmi/windows-wmi.c new file mode 100644 index 000000000..02d3faa7c --- /dev/null +++ b/src/libnetdata/os/windows-wmi/windows-wmi.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "windows-wmi.h" + +#if defined(OS_WINDOWS) + +__thread ND_WMI nd_wmi = { 0 }; + +HRESULT InitializeWMI(void) { + if(nd_wmi.pLoc && nd_wmi.pSvc) return S_OK; + CleanupWMI(); + + IWbemLocator **pLoc = &nd_wmi.pLoc; + IWbemServices **pSvc = &nd_wmi.pSvc; + + HRESULT hr; + + // Initialize COM + hr = CoInitializeEx(0, COINIT_MULTITHREADED); + if (FAILED(hr)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Failed to initialize COM library. Error code = 0x%X", hr); + CleanupWMI(); + return hr; + } + + // Set COM security levels + hr = CoInitializeSecurity( + NULL, + -1, + NULL, + NULL, + RPC_C_AUTHN_LEVEL_DEFAULT, + RPC_C_IMP_LEVEL_IMPERSONATE, + NULL, + EOAC_NONE, + NULL + ); + if (FAILED(hr)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Failed to initialize security. Error code = 0x%X", hr); + CleanupWMI(); + return hr; + } + + // Obtain the initial locator to WMI + hr = CoCreateInstance( + &CLSID_WbemLocator, 0, + CLSCTX_INPROC_SERVER, + &IID_IWbemLocator, (LPVOID *)pLoc + ); + if (FAILED(hr)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Failed to create IWbemLocator object. Error code = 0x%X", hr); + CleanupWMI(); + return hr; + } + + // Connect to WMI + BSTR namespacePath = SysAllocString(L"ROOT\\CIMV2"); + hr = (*pLoc)->lpVtbl->ConnectServer( + *pLoc, + namespacePath, + NULL, + NULL, + 0, + 0, + 0, + 0, + pSvc + ); + SysFreeString(namespacePath); + + if (FAILED(hr)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Could not connect to WMI server. Error code = 0x%X", hr); + CleanupWMI(); + return hr; + } + + // Set security levels on the proxy + hr = CoSetProxyBlanket( + (IUnknown *)*pSvc, + RPC_C_AUTHN_WINNT, + RPC_C_AUTHZ_NONE, + NULL, + RPC_C_AUTHN_LEVEL_CALL, + RPC_C_IMP_LEVEL_IMPERSONATE, + NULL, + EOAC_NONE + ); + if (FAILED(hr)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Could not set proxy blanket. Error code = 0x%X", hr); + CleanupWMI(); + return hr; + } + + return S_OK; +} + +void CleanupWMI(void) { + if(nd_wmi.pLoc) + nd_wmi.pLoc->lpVtbl->Release(nd_wmi.pLoc); + + if (nd_wmi.pSvc) + nd_wmi.pSvc->lpVtbl->Release(nd_wmi.pSvc); + + nd_wmi.pLoc = NULL; + nd_wmi.pSvc = NULL; + + CoUninitialize(); +} + +#endif
\ No newline at end of file diff --git a/src/libnetdata/os/windows-wmi/windows-wmi.h b/src/libnetdata/os/windows-wmi/windows-wmi.h new file mode 100644 index 000000000..69d7244aa --- /dev/null +++ b/src/libnetdata/os/windows-wmi/windows-wmi.h @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_WINDOWS_WMI_H +#define NETDATA_WINDOWS_WMI_H + +#include "../../libnetdata.h" + +#if defined(OS_WINDOWS) +typedef struct { + IWbemLocator *pLoc; + IWbemServices *pSvc; +} ND_WMI; + +extern __thread ND_WMI nd_wmi; + +HRESULT InitializeWMI(void); +void CleanupWMI(void); + +#include "windows-wmi-GetDiskDriveInfo.h" + +#endif + +#endif //NETDATA_WINDOWS_WMI_H diff --git a/src/libnetdata/parsers/duration.c b/src/libnetdata/parsers/duration.c new file mode 100644 index 000000000..16dc5170c --- /dev/null +++ b/src/libnetdata/parsers/duration.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "duration.h" + +#ifdef NSEC_PER_USEC +#undef NSEC_PER_USEC +#endif +#define NSEC_PER_USEC (1000ULL) + +#ifdef USEC_PER_MS +#undef USEC_PER_MS +#endif +#define USEC_PER_MS (1000ULL) + +#ifdef NSEC_PER_SEC +#undef NSEC_PER_SEC +#endif +#define NSEC_PER_SEC (1000000000ULL) + +#define NSEC_PER_MS (USEC_PER_MS * NSEC_PER_USEC) +#define NSEC_PER_MIN (NSEC_PER_SEC * 60ULL) +#define NSEC_PER_HOUR (NSEC_PER_MIN * 60ULL) +#define NSEC_PER_DAY (NSEC_PER_HOUR * 24ULL) +#define NSEC_PER_WEEK (NSEC_PER_DAY * 7ULL) +#define NSEC_PER_MONTH (NSEC_PER_DAY * 30ULL) +#define NSEC_PER_QUARTER (NSEC_PER_MONTH * 3ULL) + +// more accurate, but not an integer multiple of days, weeks, months +#define NSEC_PER_YEAR (NSEC_PER_DAY * 365ULL) + +// Define a structure to map time units to their multipliers +static const struct duration_unit { + const char *unit; + const bool formatter; // true when this unit should be used when formatting to string + const snsec_t multiplier; +} units[] = { + + // IMPORTANT: the order of this array is crucial! + // The array should be sorted from the smaller unit to the biggest unit. + + { .unit = "ns", .formatter = true, .multiplier = 1 }, // UCUM + { .unit = "us", .formatter = true, .multiplier = NSEC_PER_USEC }, // UCUM + { .unit = "ms", .formatter = true, .multiplier = NSEC_PER_MS }, // UCUM + { .unit = "s", .formatter = true, .multiplier = NSEC_PER_SEC }, // UCUM + { .unit = "m", .formatter = true, .multiplier = NSEC_PER_MIN }, // - + { .unit = "min", .formatter = false, .multiplier = NSEC_PER_MIN }, // UCUM + { .unit = "h", .formatter = true, .multiplier = NSEC_PER_HOUR }, // UCUM + { .unit = "d", .formatter = true, .multiplier = NSEC_PER_DAY }, // UCUM + { .unit = "w", .formatter = false, .multiplier = NSEC_PER_WEEK }, // - + { .unit = "wk", .formatter = false, .multiplier = NSEC_PER_WEEK }, // UCUM + { .unit = "mo", .formatter = true, .multiplier = NSEC_PER_MONTH }, // UCUM + { .unit = "M", .formatter = false, .multiplier = NSEC_PER_MONTH }, // compatibility + { .unit = "q", .formatter = false, .multiplier = NSEC_PER_QUARTER }, // - + { .unit = "y", .formatter = true, .multiplier = NSEC_PER_YEAR }, // - + { .unit = "Y", .formatter = false, .multiplier = NSEC_PER_YEAR }, // compatibility + { .unit = "a", .formatter = false, .multiplier = NSEC_PER_YEAR }, // UCUM +}; + +static inline const struct duration_unit *duration_find_unit(const char *unit) { + if(!unit || !*unit) + unit = "ns"; + + for (size_t i = 0; i < sizeof(units) / sizeof(units[0]); i++) { + const struct duration_unit *du = &units[i]; + if ((uint8_t)unit[0] == (uint8_t)du->unit[0] && strcmp(unit, du->unit) == 0) + return du; + } + + return NULL; +} + +inline int64_t duration_round_to_resolution(int64_t value, int64_t resolution) { + if(value > 0) + return (value + ((resolution - 1) / 2)) / resolution; + + if(value < 0) + return (value - ((resolution - 1) / 2)) / resolution; + + return 0; +} + +// ------------------------------------------------------------------------------------------------------------------- +// parse a duration string + +bool duration_parse(const char *duration, int64_t *result, const char *default_unit) { + if (!duration || !*duration) { + *result = 0; + return false; + } + + const struct duration_unit *du_def = duration_find_unit(default_unit); + if(!du_def) { + *result = 0; + return false; + } + + int64_t sign = 1; + const char *s = duration; + while (isspace((uint8_t)*s)) s++; + if(*s == '-') { + s++; + sign = -1; + } + + int64_t v = 0; + + while (*s) { + // Skip leading spaces + while (isspace((uint8_t)*s)) s++; + + // compatibility + if(*s == 'n' && strcmp(s, "never") == 0) { + *result = 0; + return true; + } + + if(*s == 'o' && strcmp(s, "off") == 0) { + *result = 0; + return true; + } + + // Parse the number + const char *number_start = s; + NETDATA_DOUBLE value = str2ndd(s, (char **)&s); + + // If no valid number found, return default + if (s == number_start) { + *result = 0; + return false; + } + + // Skip spaces between number and unit + while (isspace((uint8_t)*s)) s++; + + const char *unit_start = s; + while (isalpha((uint8_t)*s)) s++; + + char unit[4]; + size_t unit_len = s - unit_start; + const struct duration_unit *du; + if (unit_len == 0) + du = du_def; + else { + if (unit_len >= sizeof(unit)) unit_len = sizeof(unit) - 1; + strncpyz(unit, unit_start, unit_len); + du = duration_find_unit(unit); + if(!du) { + *result = 0; + return false; + } + } + + v += (int64_t)round(value * (NETDATA_DOUBLE)du->multiplier); + } + + v *= sign; + + if(du_def->multiplier == 1) + *result = v; + else + *result = duration_round_to_resolution(v, du_def->multiplier); + + return true; +} + +// -------------------------------------------------------------------------------------------------------------------- +// generate a string to represent a duration + +ssize_t duration_snprintf(char *dst, size_t dst_size, int64_t value, const char *unit, bool add_spaces) { + if (!dst || dst_size == 0) return -1; + if (dst_size == 1) { + dst[0] = '\0'; + return -2; + } + + if(value == 0) + return snprintfz(dst, dst_size, "off"); + + const char *sign = ""; + if(value < 0) { + sign = "-"; + value = -value; + } + + const struct duration_unit *du_min = duration_find_unit(unit); + size_t offset = 0; + + int64_t nsec = value * du_min->multiplier; + + // Iterate through units from largest to smallest + for (size_t i = sizeof(units) / sizeof(units[0]) - 1; i > 0 && nsec > 0; i--) { + const struct duration_unit *du = &units[i]; + if(!units[i].formatter && du != du_min) + continue; + + // IMPORTANT: + // The week (7 days) is not aligned to the quarter (~91 days) or the year (365.25 days). + // To make sure that the value returned can be parsed back without loss, + // we have to round the value per unit (inside this loop), not globally. + // Otherwise, we have to make sure that all larger units are integer multiples of the smaller ones. + + int64_t multiplier = units[i].multiplier; + int64_t rounded = (du == du_min) ? (duration_round_to_resolution(nsec, multiplier) * multiplier) : nsec; + + int64_t unit_count = rounded / multiplier; + if (unit_count > 0) { + const char *space = (add_spaces && offset) ? " " : ""; + int written = snprintfz(dst + offset, dst_size - offset, + "%s%s%" PRIi64 "%s", space, sign, unit_count, units[i].unit); + + if (written < 0) + return -3; + + sign = ""; + offset += written; + + if (offset >= dst_size) { + // buffer overflow + return (ssize_t)offset; + } + + if(unit_count * multiplier >= nsec) + break; + else + nsec -= unit_count * multiplier; + } + + if(du == du_min) + // we should not go to smaller units + break; + } + + if (offset == 0) + // nothing has been written + offset = snprintfz(dst, dst_size, "off"); + + return (ssize_t)offset; +} + +// -------------------------------------------------------------------------------------------------------------------- +// compatibility for parsing seconds in int. + +bool duration_parse_seconds(const char *str, int *result) { + int64_t v; + + if(duration_parse_time_t(str, &v)) { + *result = (int)v; + return true; + } + + return false; +} diff --git a/src/libnetdata/parsers/duration.h b/src/libnetdata/parsers/duration.h new file mode 100644 index 000000000..b95da5d2f --- /dev/null +++ b/src/libnetdata/parsers/duration.h @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef LIBNETDATA_PARSERS_DURATION_H +#define LIBNETDATA_PARSERS_DURATION_H + +#include "parsers.h" + +int64_t duration_round_to_resolution(int64_t value, int64_t resolution); + +// duration (string to number) +bool duration_parse(const char *duration, int64_t *result, const char *default_unit); +#define duration_parse_nsec_t(duration, ns_ptr) duration_parse(duration, ns_ptr, "ns") +#define duration_parse_usec_t(duration, us_ptr) duration_parse(duration, us_ptr, "us") +#define duration_parse_msec_t(duration, ms_ptr) duration_parse(duration, ms_ptr, "ms") +#define duration_parse_time_t(duration, secs_ptr) duration_parse(duration, secs_ptr, "s") +#define duration_parse_mins(duration, mins_ptr) duration_parse(duration, mins_ptr, "m") +#define duration_parse_hours(duration, hours_ptr) duration_parse(duration, hours_ptr, "h") +#define duration_parse_days(duration, days_ptr) duration_parse(duration, days_ptr, "d") + +// duration (number to string) +ssize_t duration_snprintf(char *dst, size_t dst_size, int64_t value, const char *unit, bool add_spaces); +#define duration_snprintf_nsec_t(dst, dst_size, ns) duration_snprintf(dst, dst_size, ns, "ns", false) +#define duration_snprintf_usec_t(dst, dst_size, us) duration_snprintf(dst, dst_size, us, "us", false) +#define duration_snprintf_msec_t(dst, dst_size, ms) duration_snprintf(dst, dst_size, ms, "ms", false) +#define duration_snprintf_time_t(dst, dst_size, secs) duration_snprintf(dst, dst_size, secs, "s", false) +#define duration_snprintf_mins(dst, dst_size, mins) duration_snprintf(dst, dst_size, mins, "m", false) +#define duration_snprintf_hours(dst, dst_size, hours) duration_snprintf(dst, dst_size, hours, "h", false) +#define duration_snprintf_days(dst, dst_size, days) duration_snprintf(dst, dst_size, days, "d", false) + +bool duration_parse_seconds(const char *str, int *result); + +#endif //LIBNETDATA_PARSERS_DURATION_H diff --git a/src/libnetdata/parsers/duration.html b/src/libnetdata/parsers/duration.html new file mode 100644 index 000000000..8f6f8a416 --- /dev/null +++ b/src/libnetdata/parsers/duration.html @@ -0,0 +1,205 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>Duration Converter</title> + <style> + table { + width: 50%; + border-collapse: collapse; + margin-top: 20px; + } + table, th, td { + border: 1px solid black; + } + th, td { + padding: 10px; + text-align: center; + } + .error { + color: red; + margin-top: 10px; + } + </style> +</head> +<body> +<h1>Duration Converter</h1> +<input type="text" id="durationInput" placeholder="Enter duration (e.g., 10d-12h)"> +<div id="errorMessage" class="error"></div> + +<table id="resultTable"> + <thead> + <tr> + <th>Unit</th> + <th>Value</th> + <th>Formatted</th> + <th>Check</th> + </tr> + </thead> + <tbody> + </tbody> +</table> + +<script> + const NSEC_PER_USEC = 1000; + const USEC_PER_MS = 1000; + const NSEC_PER_SEC = 1000000000; + const NSEC_PER_MS = USEC_PER_MS * NSEC_PER_USEC; + const NSEC_PER_MIN = NSEC_PER_SEC * 60; + const NSEC_PER_HOUR = NSEC_PER_MIN * 60; + const NSEC_PER_DAY = NSEC_PER_HOUR * 24; + const NSEC_PER_WEEK = NSEC_PER_DAY * 7; + const NSEC_PER_YEAR = NSEC_PER_DAY * 365; + const NSEC_PER_MONTH = NSEC_PER_DAY * 30; + const NSEC_PER_QUARTER = NSEC_PER_MONTH * 3; + + const units = [ + { unit: "ns", formatter: true, multiplier: 1 }, + { unit: "us", formatter: true, multiplier: NSEC_PER_USEC }, + { unit: "ms", formatter: true, multiplier: NSEC_PER_MS }, + { unit: "s", formatter: true, multiplier: NSEC_PER_SEC }, + { unit: "m", formatter: true, multiplier: NSEC_PER_MIN }, + { unit: "min", formatter: false, multiplier: NSEC_PER_MIN }, + { unit: "h", formatter: true, multiplier: NSEC_PER_HOUR }, + { unit: "d", formatter: true, multiplier: NSEC_PER_DAY }, + { unit: "w", formatter: false, multiplier: NSEC_PER_WEEK }, + { unit: "wk", formatter: false, multiplier: NSEC_PER_WEEK }, + { unit: "mo", formatter: true, multiplier: NSEC_PER_MONTH }, + { unit: "M", formatter: false, multiplier: NSEC_PER_MONTH }, + { unit: "q", formatter: false, multiplier: NSEC_PER_QUARTER }, + { unit: "y", formatter: true, multiplier: NSEC_PER_YEAR }, + { unit: "Y", formatter: false, multiplier: NSEC_PER_YEAR }, + { unit: "a", formatter: false, multiplier: NSEC_PER_YEAR } + ]; + + function durationFindUnit(unit) { + if (!unit) return units[0]; + return units.find(u => u.unit === unit) || null; + } + + function roundToResolution(value, resolution) { + if (value > 0) return Math.floor((value + (resolution - 1) / 2) / resolution); + if (value < 0) return Math.ceil((value - (resolution - 1) / 2) / resolution); + return 0; + } + + function parseDouble(str) { + str = str.trim(); + const match = str.match(/^[-+]?\d*\.?\d+/); + if (match) { + const number = parseFloat(match[0]); + const remainingStr = str.slice(match[0].length).trim(); + return { number, remainingStr }; + } + return { number: null, remainingStr: str }; + } + + function durationParse(duration, unit) { + if (!duration || !unit) return false; + + let s = duration.trim(); + let nsec = 0; + let isNegative = false; + + // Handle leading negative sign + if (s.startsWith("-")) { + isNegative = true; + s = s.slice(1).trim(); + } + + while (s.length > 0) { + s = s.trim(); + + if (s.startsWith("never") || s.startsWith("off")) + return 0; + + const { number, remainingStr } = parseDouble(s); + if (number === null) return false; + + s = remainingStr; + + const match = s.match(/^([a-zA-Z]*)/); + let currentUnit = unit; + if (match && match[0].length > 0) { + currentUnit = match[0]; + s = s.slice(match[0].length).trim(); + } + + const du = durationFindUnit(currentUnit); + if (!du) return false; + + nsec += number * du.multiplier; + } + + const unitMultiplier = durationFindUnit(unit).multiplier; + nsec = roundToResolution(nsec, unitMultiplier); + + return isNegative ? -nsec : nsec; + } + + function durationSnprintf(value, unit) { + if (value === 0) return "off"; + + const duMin = durationFindUnit(unit); + let nsec = Math.abs(value) * duMin.multiplier; + + const isNegative = value < 0; + let result = isNegative ? "-" : ""; + + for (let i = units.length - 1; i >= 0 && nsec !== 0; i--) { + const du = units[i]; + if (!du.formatter && du !== duMin) continue; + + const multiplier = du.multiplier; + const rounded = (du === duMin) ? roundToResolution(nsec, multiplier) * multiplier : nsec; + let unitCount = Math.floor(rounded / multiplier); + + if (unitCount !== 0) { + result += `${unitCount}${du.unit}`; + nsec -= unitCount * multiplier; + } + + if (du === duMin) break; + } + + return result || "off"; + } + + function updateTable() { + const duration = document.getElementById("durationInput").value; + const tableBody = document.getElementById("resultTable").querySelector("tbody"); + const errorMessage = document.getElementById("errorMessage"); + tableBody.innerHTML = ""; + errorMessage.textContent = ""; + + units.forEach(unit => { + let value = durationParse(duration, unit.unit); + let formatted; + let check; + if(value === false) { + value = "-"; + formatted = ""; + check = "parsing error"; + } + else { + formatted = durationSnprintf(value, unit.unit); + const parsedValue = durationParse(formatted, unit.unit); + check = (parsedValue === value) ? "ok" : `re-parsing error (${parsedValue})`; + } + + const row = `<tr> + <td>${unit.unit}</td> + <td>${value}</td> + <td>${formatted}</td> + <td>${check}</td> + </tr>`; + tableBody.innerHTML += row; + }); + } + + document.getElementById("durationInput").addEventListener("input", updateTable); +</script> +</body> +</html> diff --git a/src/libnetdata/parsers/durations.md b/src/libnetdata/parsers/durations.md new file mode 100644 index 000000000..e952faa1a --- /dev/null +++ b/src/libnetdata/parsers/durations.md @@ -0,0 +1,94 @@ +## Durations in Netdata + +Netdata provides a flexible and powerful way to specify durations for various configurations and operations, such as alerts, database retention, and other configuration options. Durations can be expressed in a variety of units, ranging from nanoseconds to years, allowing users to define time intervals in a human-readable format. + +### Supported Duration Units + +Netdata supports a wide range of duration units. The system follows the Unified Code for Units of Measure (UCUM) standard where applicable. Below is a table of all the supported units, their corresponding representations, and their compatibility: + +| Symbol | Description | Value | Compatibility | Formatter | +|:------:|:------------:|:--------:|:-------------:|:---------:| +| `ns` | Nanoseconds | `1ns` | UCUM | **Yes** | +| `us` | Microseconds | `1000ns` | UCUM | **Yes** | +| `ms` | Milliseconds | `1000us` | UCUM | **Yes** | +| `s` | Seconds | `1000ms` | UCUM | **Yes** | +| `m` | Minutes | `60s` | Natural | **Yes** | +| `min` | Minutes | `60s` | UCUM | No | +| `h` | Hours | `60m` | UCUM | **Yes** | +| `d` | Days | `24h` | UCUM | **Yes** | +| `w` | Weeks | `7d` | Natural | No | +| `wk` | Weeks | `7d` | UCUM | No | +| `mo` | Months | `30d` | UCUM | **Yes** | +| `M` | Months | `30d` | Backwards | No | +| `q` | Quarters | `3mo` | Natural | No | +| `y` | Years | `365d` | Natural | **Yes** | +| `Y` | Years | `365d` | Backwards | No | +| `a` | Years | `365d` | UCUM | No | + +- **UCUM**: The unit is specified in the Unified Code for Units of Measure (UCUM) standard. +- **Natural**: We feel that this is more natural for expressing durations with single letter units. +- **Backwards**: This unit has been used in the past in Netdata, and we support it for backwards compatibility. + +### Duration Expression Format + +Netdata allows users to express durations in both simple and complex formats. + +- **Simple Formats**: A duration can be specified using a number followed by a unit, such as `5m` (5 minutes), `2h` (2 hours), or `1d` (1 day). Fractional numbers are also supported, such as `1.5d`, `3.5mo` or `1.2y`. + +- **Complex Formats**: A duration can also be composed of multiple units added together. For example: + - `1y2mo3w4d` represents 1 year, 2 months, 3 weeks, and 4 days. + - `15d-12h` represents 15 days minus 12 hours (which equals 14 days and 12 hours). + +Each number given in durations can be either positive or negative. For example `1h15m` is 1 hour and 15 minutes, but `1h-15m` results to `45m`. + +The same unit can be given multiple times, so that `1d0.5d` is `1d12h` and `1d-0.5d` is `12h`. + +The order of units in the expressions is irrelevant, so that `1m2h3d` is the same to `3d2h1m`. + +The system will parse durations with spaces in them, but we suggest to write them down in compact form, without spaces. This is required, especially in alerts configuration, since spaces in durations will affect how parent expressions are tokenized. + +### Duration Rounding + +Netdata provides various functions to parse and round durations according to specific needs: + +- **Default Rounding to Seconds**: Most duration uses in Netdata are rounded to the nearest second. For example, a duration of `1.4s` would round to `1s`, while `1.5s` would round to `2s`. + +- **Rounding to Larger Units**: In some cases, such as database retention, durations are rounded to larger units like days. Even when rounding to a larger unit, durations can still be expressed in smaller units (e.g., `24h86400s` for `2d`). + +### Maximum and Minimum Duration Limits + +Netdata's duration expressions can handle durations ranging from the minimum possible value of `-INT64_MAX` to the maximum of `INT64_MAX` in nanoseconds. This range translates approximately to durations between -292 years to +292 years. + +### Inconsistencies in Duration Units + +While Netdata provides a flexible system for specifying durations, some inconsistencies arise due to the way different units are defined: + +- **1 Year (`y`) = 365 Days (`d`)**: In Netdata, a year is defined as 365 days. This is an approximation, since the average year is about 365.25 days. + +- **1 Month (`mo`) = 30 Days (`d`)**: Similarly, a month in Netdata is defined as 30 days, which is also an approximation. In reality, months vary in length (28 to 31 days). + +- **1 Quarter (`q`) = 3 Months (`mo`) = 90 Days (`d`)**: A quarter is defined as 3 months, or 90 days, which aligns with the approximation of each month being 30 days. + +These definitions can lead to some unexpected results when performing arithmetic with durations: + +**Example of Inconsistency**: + +`1y-1d` in Netdata calculates to `364d` but also as `12mo4d` because `1y = 365d` and `1mo = 30d`. This is inconsistent because `1y` is defined as `12mo5d` or `4q5d` (given the approximations above). + +### Negative Durations + +When the first letter of a duration expression is the minus character, Netdata parses the entire expression as positive and then it negates the result. for example: `-1m15s` is `-75s`, not `-45s`. To get `-45s` the expression should be `-1m-15s`. So the initial `-` is treated like `-(expression)`. + +The same rule is applied when generating duration expressions. + +### Example Duration Expressions + +Here are some examples of valid duration expressions: + +1. **`30s`**: 30 seconds. +2. **`5m`**: 5 minutes. +3. **`2h30m`**: 2 hours and 30 minutes. +4. **`1.5d`**: 1 day and 12 hours. +5. **`1w3d4h`**: 1 week, 3 days, and 4 hours. +6. **`1y2mo3d`**: 1 year, 2 months, and 3 days. +7. **`15d-12h`**: 14 days and 12 hours. diff --git a/src/libnetdata/parsers/entries.c b/src/libnetdata/parsers/entries.c new file mode 100644 index 000000000..d6ed31de1 --- /dev/null +++ b/src/libnetdata/parsers/entries.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "entries.h"
+
+// Define multipliers for base 10 (decimal) units
+#define ENTRIES_MULTIPLIER_BASE10 1000ULL
+#define ENTRIES_MULTIPLIER_K (ENTRIES_MULTIPLIER_BASE10)
+#define ENTRIES_MULTIPLIER_M (ENTRIES_MULTIPLIER_K * ENTRIES_MULTIPLIER_BASE10)
+#define ENTRIES_MULTIPLIER_G (ENTRIES_MULTIPLIER_M * ENTRIES_MULTIPLIER_BASE10)
+#define ENTRIES_MULTIPLIER_T (ENTRIES_MULTIPLIER_G * ENTRIES_MULTIPLIER_BASE10)
+#define ENTRIES_MULTIPLIER_P (ENTRIES_MULTIPLIER_T * ENTRIES_MULTIPLIER_BASE10)
+#define ENTRIES_MULTIPLIER_E (ENTRIES_MULTIPLIER_P * ENTRIES_MULTIPLIER_BASE10)
+#define ENTRIES_MULTIPLIER_Z (ENTRIES_MULTIPLIER_E * ENTRIES_MULTIPLIER_BASE10)
+#define ENTRIES_MULTIPLIER_Y (ENTRIES_MULTIPLIER_Z * ENTRIES_MULTIPLIER_BASE10)
+
+// Define a structure to map size units to their multipliers
+static const struct size_unit {
+ const char *unit;
+ const bool formatter; // true when this unit should be used when formatting to string
+ const uint64_t multiplier;
+} entries_units[] = {
+ // the order of this table is important: smaller to bigger units!
+
+ { .unit = "", .formatter = true, .multiplier = 1ULL },
+ { .unit = "k", .formatter = false, .multiplier = ENTRIES_MULTIPLIER_K },
+ { .unit = "K", .formatter = true, .multiplier = ENTRIES_MULTIPLIER_K },
+ { .unit = "M", .formatter = true, .multiplier = ENTRIES_MULTIPLIER_M },
+ { .unit = "G", .formatter = true, .multiplier = ENTRIES_MULTIPLIER_G },
+ { .unit = "T", .formatter = true, .multiplier = ENTRIES_MULTIPLIER_T },
+ { .unit = "P", .formatter = true, .multiplier = ENTRIES_MULTIPLIER_P },
+ { .unit = "E", .formatter = true, .multiplier = ENTRIES_MULTIPLIER_E },
+ { .unit = "Z", .formatter = true, .multiplier = ENTRIES_MULTIPLIER_Z },
+ { .unit = "Y", .formatter = true, .multiplier = ENTRIES_MULTIPLIER_Y },
+};
+
+static inline const struct size_unit *entries_find_unit(const char *unit) {
+ if (!unit || !*unit) unit = "";
+
+ for (size_t i = 0; i < sizeof(entries_units) / sizeof(entries_units[0]); i++) {
+ const struct size_unit *su = &entries_units[i];
+ if ((uint8_t)unit[0] == (uint8_t)su->unit[0] && strcmp(unit, su->unit) == 0)
+ return su;
+ }
+
+ return NULL;
+}
+
+static inline double entries_round_to_resolution_dbl2(uint64_t value, uint64_t resolution) {
+ double converted = (double)value / (double)resolution;
+ return round(converted * 100.0) / 100.0;
+}
+
+static inline uint64_t entries_round_to_resolution_int(uint64_t value, uint64_t resolution) {
+ return (value + (resolution / 2)) / resolution;
+}
+
+// -------------------------------------------------------------------------------------------------------------------
+// parse a size string
+
+bool entries_parse(const char *entries_str, uint64_t *result, const char *default_unit) {
+ if (!entries_str || !*entries_str) {
+ *result = 0;
+ return false;
+ }
+
+ const struct size_unit *su_def = entries_find_unit(default_unit);
+ if(!su_def) {
+ *result = 0;
+ return false;
+ }
+
+ const char *s = entries_str;
+
+ // Skip leading spaces
+ while (isspace((uint8_t)*s)) s++;
+
+ if(strcmp(s, "off") == 0) {
+ *result = 0;
+ return true;
+ }
+
+ // Parse the number
+ const char *number_start = s;
+ NETDATA_DOUBLE value = strtondd(s, (char **)&s);
+
+ // If no valid number found, return false
+ if (s == number_start || value < 0) {
+ *result = 0;
+ return false;
+ }
+
+ // Skip spaces between number and unit
+ while (isspace((uint8_t)*s)) s++;
+
+ const char *unit_start = s;
+ while (isalpha((uint8_t)*s)) s++;
+
+ char unit[4];
+ size_t unit_len = s - unit_start;
+ const struct size_unit *su;
+ if (unit_len == 0)
+ su = su_def;
+ else {
+ if (unit_len >= sizeof(unit)) unit_len = sizeof(unit) - 1;
+ strncpy(unit, unit_start, unit_len);
+ unit[unit_len] = '\0';
+ su = entries_find_unit(unit);
+ if (!su) {
+ *result = 0;
+ return false;
+ }
+ }
+
+ uint64_t bytes = (uint64_t)round(value * (NETDATA_DOUBLE)su->multiplier);
+ *result = entries_round_to_resolution_int(bytes, su_def->multiplier);
+
+ return true;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// generate a string to represent a size
+
+ssize_t entries_snprintf(char *dst, size_t dst_size, uint64_t value, const char *unit, bool accurate) {
+ if (!dst || dst_size == 0) return -1;
+ if (dst_size == 1) {
+ dst[0] = '\0';
+ return -2;
+ }
+
+ if (value == 0)
+ return snprintfz(dst, dst_size, "off");
+
+ const struct size_unit *su_def = entries_find_unit(unit);
+ if(!su_def) return -3;
+
+ // use the units multiplier to find the units
+ uint64_t bytes = value * su_def->multiplier;
+
+ // Find the best unit to represent the size with up to 2 fractional digits
+ const struct size_unit *su_best = su_def;
+ for (size_t i = 0; i < sizeof(entries_units) / sizeof(entries_units[0]); i++) {
+ const struct size_unit *su = &entries_units[i];
+ if (su->multiplier < su_def->multiplier || // the multiplier is too small
+ (!su->formatter && su != su_def) || // it is not to be used in formatting (except our unit)
+ (bytes < su->multiplier && su != su_def) ) // the converted value will be <1.0
+ continue;
+
+ double converted = entries_round_to_resolution_dbl2(bytes, su->multiplier);
+
+ uint64_t reversed_bytes = (uint64_t)(converted * (double)su->multiplier);
+
+ if(accurate) {
+ // no precision loss is required
+ if (reversed_bytes == bytes)
+ // no precision loss, this is good to use
+ su_best = su;
+ }
+ else {
+ if(converted > 1.0)
+ su_best = su;
+ }
+ }
+
+ double converted = entries_round_to_resolution_dbl2(bytes, su_best->multiplier);
+
+ // print it either with 0, 1 or 2 fractional digits
+ int written;
+ if(converted == (double)((uint64_t)converted))
+ written = snprintfz(dst, dst_size, "%.0f%s", converted, su_best->unit);
+ else if(converted * 10.0 == (double)((uint64_t)(converted * 10.0)))
+ written = snprintfz(dst, dst_size, "%.1f%s", converted, su_best->unit);
+ else
+ written = snprintfz(dst, dst_size, "%.2f%s", converted, su_best->unit);
+
+ if (written < 0)
+ return -4;
+
+ if ((size_t)written >= dst_size)
+ return (ssize_t)(dst_size - 1);
+
+ return written;
+}
+
diff --git a/src/libnetdata/parsers/entries.h b/src/libnetdata/parsers/entries.h new file mode 100644 index 000000000..a90b8f6f4 --- /dev/null +++ b/src/libnetdata/parsers/entries.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef LIBNETDATA_PARSERS_ENTRIES_H
+#define LIBNETDATA_PARSERS_ENTRIES_H
+
+#include "parsers.h"
+
+bool entries_parse(const char *entries_str, uint64_t *result, const char *default_unit);
+#define entries_parse_k(size_str, kb) size_parse(size_str, kb, "K")
+#define entries_parse_m(size_str, mb) size_parse(size_str, mb, "M")
+#define entries_parse_g(size_str, gb) size_parse(size_str, gb, "G")
+
+ssize_t entries_snprintf(char *dst, size_t dst_size, uint64_t value, const char *unit, bool accurate);
+#define entries_snprintf_n(dst, dst_size, value) size_snprintf(dst, dst_size, value, "", true)
+#define entries_snprintf_k(dst, dst_size, value) size_snprintf(dst, dst_size, value, "K", true)
+#define entries_snprintf_m(dst, dst_size, value) size_snprintf(dst, dst_size, value, "M", true)
+#define entries_snprintf_g(dst, dst_size, value) size_snprintf(dst, dst_size, value, "G", true)
+
+#endif //LIBNETDATA_PARSERS_ENTRIES_H
diff --git a/src/libnetdata/parsers/parsers.h b/src/libnetdata/parsers/parsers.h new file mode 100644 index 000000000..27b60b040 --- /dev/null +++ b/src/libnetdata/parsers/parsers.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_PARSERS_H +#define NETDATA_PARSERS_H + +#include "../libnetdata.h" +#include "size.h" +#include "entries.h" +#include "duration.h" +#include "timeframe.h" + +#endif //NETDATA_PARSERS_H diff --git a/src/libnetdata/parsers/size.c b/src/libnetdata/parsers/size.c new file mode 100644 index 000000000..d3a24c540 --- /dev/null +++ b/src/libnetdata/parsers/size.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "size.h" + +// Define multipliers for base 2 (binary) units +#define SIZE_MULTIPLIER_BASE2 1024ULL +#define SIZE_MULTIPLIER_KiB (SIZE_MULTIPLIER_BASE2) +#define SIZE_MULTIPLIER_MiB (SIZE_MULTIPLIER_KiB * SIZE_MULTIPLIER_BASE2) +#define SIZE_MULTIPLIER_GiB (SIZE_MULTIPLIER_MiB * SIZE_MULTIPLIER_BASE2) +#define SIZE_MULTIPLIER_TiB (SIZE_MULTIPLIER_GiB * SIZE_MULTIPLIER_BASE2) +#define SIZE_MULTIPLIER_PiB (SIZE_MULTIPLIER_TiB * SIZE_MULTIPLIER_BASE2) +//#define SIZE_MULTIPLIER_EiB (SIZE_MULTIPLIER_PiB * SIZE_MULTIPLIER_BASE2) +//#define SIZE_MULTIPLIER_ZiB (SIZE_MULTIPLIER_EiB * SIZE_MULTIPLIER_BASE2) +//#define SIZE_MULTIPLIER_YiB (SIZE_MULTIPLIER_ZiB * SIZE_MULTIPLIER_BASE2) + +// Define multipliers for base 10 (decimal) units +#define SIZE_MULTIPLIER_BASE10 1000ULL +#define SIZE_MULTIPLIER_K (SIZE_MULTIPLIER_BASE10) +#define SIZE_MULTIPLIER_M (SIZE_MULTIPLIER_K * SIZE_MULTIPLIER_BASE10) +#define SIZE_MULTIPLIER_G (SIZE_MULTIPLIER_M * SIZE_MULTIPLIER_BASE10) +#define SIZE_MULTIPLIER_T (SIZE_MULTIPLIER_G * SIZE_MULTIPLIER_BASE10) +#define SIZE_MULTIPLIER_P (SIZE_MULTIPLIER_T * SIZE_MULTIPLIER_BASE10) +//#define SIZE_MULTIPLIER_E (SIZE_MULTIPLIER_P * SIZE_MULTIPLIER_BASE10) +//#define SIZE_MULTIPLIER_Z (SIZE_MULTIPLIER_E * SIZE_MULTIPLIER_BASE10) +//#define SIZE_MULTIPLIER_Y (SIZE_MULTIPLIER_Z * SIZE_MULTIPLIER_BASE10) + +// Define a structure to map size units to their multipliers +static const struct size_unit { + const char *unit; + const uint8_t base; + const bool formatter; // true when this unit should be used when formatting to string + const uint64_t multiplier; +} size_units[] = { + // the order of this table is important: smaller to bigger units! + + { .unit = "B", .base = 2, .formatter = true, .multiplier = 1ULL }, + { .unit = "k", .base = 10, .formatter = false, .multiplier = SIZE_MULTIPLIER_K }, + { .unit = "K", .base = 10, .formatter = true, .multiplier = SIZE_MULTIPLIER_K }, + { .unit = "KB", .base = 10, .formatter = false, .multiplier = SIZE_MULTIPLIER_K }, + { .unit = "KiB", .base = 2, .formatter = true, .multiplier = SIZE_MULTIPLIER_KiB }, + { .unit = "M", .base = 10, .formatter = true, .multiplier = SIZE_MULTIPLIER_M }, + { .unit = "MB", .base = 10, .formatter = false, .multiplier = SIZE_MULTIPLIER_M }, + { .unit = "MiB", .base = 2, .formatter = true, .multiplier = SIZE_MULTIPLIER_MiB }, + { .unit = "G", .base = 10, .formatter = true, .multiplier = SIZE_MULTIPLIER_G }, + { .unit = "GB", .base = 10, .formatter = false, .multiplier = SIZE_MULTIPLIER_G }, + { .unit = "GiB", .base = 2, .formatter = true, .multiplier = SIZE_MULTIPLIER_GiB }, + { .unit = "T", .base = 10, .formatter = true, .multiplier = SIZE_MULTIPLIER_T }, + { .unit = "TB", .base = 10, .formatter = false, .multiplier = SIZE_MULTIPLIER_T }, + { .unit = "TiB", .base = 2, .formatter = true, .multiplier = SIZE_MULTIPLIER_TiB }, + { .unit = "P", .base = 10, .formatter = true, .multiplier = SIZE_MULTIPLIER_P }, + { .unit = "PB", .base = 10, .formatter = false, .multiplier = SIZE_MULTIPLIER_P }, + { .unit = "PiB", .base = 2, .formatter = true, .multiplier = SIZE_MULTIPLIER_PiB }, +// { .unit = "E", .base = 10, .formatter = true, .multiplier = SIZE_MULTIPLIER_E }, +// { .unit = "EB", .base = 10, .formatter = false, .multiplier = SIZE_MULTIPLIER_E }, +// { .unit = "EiB", .base = 2, .formatter = true, .multiplier = SIZE_MULTIPLIER_EiB }, +// { .unit = "Z", .base = 10, .formatter = true, .multiplier = SIZE_MULTIPLIER_Z }, +// { .unit = "ZB", .base = 10, .formatter = false, .multiplier = SIZE_MULTIPLIER_Z }, +// { .unit = "ZiB", .base = 2, .formatter = true, .multiplier = SIZE_MULTIPLIER_ZiB }, +// { .unit = "Y", .base = 10, .formatter = true, .multiplier = SIZE_MULTIPLIER_Y }, +// { .unit = "YB", .base = 10, .formatter = false, .multiplier = SIZE_MULTIPLIER_Y }, +// { .unit = "YiB", .base = 2, .formatter = true, .multiplier = SIZE_MULTIPLIER_YiB }, +}; + +static inline const struct size_unit *size_find_unit(const char *unit) { + if (!unit || !*unit) unit = "B"; + + for (size_t i = 0; i < sizeof(size_units) / sizeof(size_units[0]); i++) { + const struct size_unit *su = &size_units[i]; + if ((uint8_t)unit[0] == (uint8_t)su->unit[0] && strcmp(unit, su->unit) == 0) + return su; + } + + return NULL; +} + +static inline double size_round_to_resolution_dbl2(uint64_t value, uint64_t resolution) { + double converted = (double)value / (double)resolution; + return round(converted * 100.0) / 100.0; +} + +static inline uint64_t size_round_to_resolution_int(uint64_t value, uint64_t resolution) { + return (value + (resolution / 2)) / resolution; +} + +// ------------------------------------------------------------------------------------------------------------------- +// parse a size string + +bool size_parse(const char *size_str, uint64_t *result, const char *default_unit) { + if (!size_str || !*size_str) { + *result = 0; + return false; + } + + const struct size_unit *su_def = size_find_unit(default_unit); + if(!su_def) { + *result = 0; + return false; + } + + const char *s = size_str; + + // Skip leading spaces + while (isspace((uint8_t)*s)) s++; + + if(strcmp(s, "off") == 0) { + *result = 0; + return true; + } + + // Parse the number + const char *number_start = s; + NETDATA_DOUBLE value = strtondd(s, (char **)&s); + + // If no valid number found, return false + if (s == number_start || value < 0) { + *result = 0; + return false; + } + + // Skip spaces between number and unit + while (isspace((uint8_t)*s)) s++; + + const char *unit_start = s; + while (isalpha((uint8_t)*s)) s++; + + char unit[4]; + size_t unit_len = s - unit_start; + const struct size_unit *su; + if (unit_len == 0) + su = su_def; + else { + if (unit_len >= sizeof(unit)) unit_len = sizeof(unit) - 1; + strncpy(unit, unit_start, unit_len); + unit[unit_len] = '\0'; + su = size_find_unit(unit); + if (!su) { + *result = 0; + return false; + } + } + + uint64_t bytes = (uint64_t)round(value * (NETDATA_DOUBLE)su->multiplier); + *result = size_round_to_resolution_int(bytes, su_def->multiplier); + + return true; +} + +// -------------------------------------------------------------------------------------------------------------------- +// generate a string to represent a size + +ssize_t size_snprintf(char *dst, size_t dst_size, uint64_t value, const char *unit, bool accurate) { + if (!dst || dst_size == 0) return -1; + if (dst_size == 1) { + dst[0] = '\0'; + return -2; + } + + if (value == 0) + return snprintfz(dst, dst_size, "off"); + + const struct size_unit *su_def = size_find_unit(unit); + if(!su_def) return -3; + + // use the units multiplier to find the units + uint64_t bytes = value * su_def->multiplier; + + // Find the best unit to represent the size with up to 2 fractional digits + const struct size_unit *su_best = su_def; + for (size_t i = 0; i < sizeof(size_units) / sizeof(size_units[0]); i++) { + const struct size_unit *su = &size_units[i]; + if (su->base != su_def->base || // not the right base + su->multiplier < su_def->multiplier || // the multiplier is too small + (!su->formatter && su != su_def) || // it is not to be used in formatting (except our unit) + (bytes < su->multiplier && su != su_def) ) // the converted value will be <1.0 + continue; + + double converted = size_round_to_resolution_dbl2(bytes, su->multiplier); + + uint64_t reversed_bytes = (uint64_t)(converted * (double)su->multiplier); + + if(accurate) { + // no precision loss is required + if (reversed_bytes == bytes) + // no precision loss, this is good to use + su_best = su; + } + else { + if(converted > 1.0) + su_best = su; + } + } + + double converted = size_round_to_resolution_dbl2(bytes, su_best->multiplier); + + // print it either with 0, 1 or 2 fractional digits + int written; + if(converted == (double)((uint64_t)converted)) + written = snprintfz(dst, dst_size, "%.0f%s", converted, su_best->unit); + else if(converted * 10.0 == (double)((uint64_t)(converted * 10.0))) + written = snprintfz(dst, dst_size, "%.1f%s", converted, su_best->unit); + else + written = snprintfz(dst, dst_size, "%.2f%s", converted, su_best->unit); + + if (written < 0) + return -4; + + if ((size_t)written >= dst_size) + return (ssize_t)(dst_size - 1); + + return written; +} + diff --git a/src/libnetdata/parsers/size.h b/src/libnetdata/parsers/size.h new file mode 100644 index 000000000..6abfe7235 --- /dev/null +++ b/src/libnetdata/parsers/size.h @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef LIBNETDATA_PARSERS_SIZE_H +#define LIBNETDATA_PARSERS_SIZE_H + +#include "parsers.h" + +bool size_parse(const char *size_str, uint64_t *result, const char *default_unit); +#define size_parse_bytes(size_str, bytes) size_parse(size_str, bytes, "B") +#define size_parse_kb(size_str, kb) size_parse(size_str, kb, "KiB") +#define size_parse_mb(size_str, mb) size_parse(size_str, mb, "MiB") +#define size_parse_gb(size_str, gb) size_parse(size_str, gb, "GiB") + +ssize_t size_snprintf(char *dst, size_t dst_size, uint64_t value, const char *unit, bool accurate); +#define size_snprintf_bytes(dst, dst_size, value) size_snprintf(dst, dst_size, value, "B", true) +#define size_snprintf_kb(dst, dst_size, value) size_snprintf(dst, dst_size, value, "KiB", true) +#define size_snprintf_mb(dst, dst_size, value) size_snprintf(dst, dst_size, value, "MiB", true) +#define size_snprintf_gb(dst, dst_size, value) size_snprintf(dst, dst_size, value, "GiB", true) + +#endif //LIBNETDATA_PARSERS_SIZE_H diff --git a/src/libnetdata/parsers/sizes.md b/src/libnetdata/parsers/sizes.md new file mode 100644 index 000000000..ac9e09053 --- /dev/null +++ b/src/libnetdata/parsers/sizes.md @@ -0,0 +1,52 @@ +## Data Sizes in Netdata + +Netdata provides a flexible system for specifying and formatting data sizes, used in various configurations and operations such as disk space management, and memory usage. This system allows users to specify data sizes in a human-readable format using multiple units from bytes to terabytes, supporting both binary (base-2) and decimal (base-10) standards. All units are UCUM-based for consistency and clarity. + +### Supported Size Units + +The following table lists all supported units and their corresponding values: + +| Symbol | Description | Value | Base | Formatter | +|:------:|:-----------:|:---------:|:-------:|:---------:| +| `B` | Bytes | `1B` | - | **Yes** | +| `k` | Kilobytes | `1000B` | Base-10 | No | +| `K` | Kilobytes | `1000B` | Base-10 | No | +| `KB` | Kilobytes | `1000B` | Base-10 | No | +| `KiB` | Kibibytes | `1024B` | Base-2 | **Yes** | +| `M` | Megabytes | `1000K` | Base-10 | No | +| `MB` | Megabytes | `1000K` | Base-10 | No | +| `MiB` | Mebibytes | `1024KiB` | Base-2 | **Yes** | +| `G` | Gigabytes | `1000M` | Base-10 | No | +| `GB` | Gigabytes | `1000M` | Base-10 | No | +| `GiB` | Gibibytes | `1024MiB` | Base-2 | **Yes** | +| `T` | Terabytes | `1000G` | Base-10 | No | +| `TB` | Terabytes | `1000G` | Base-10 | No | +| `TiB` | Tebibytes | `1024GiB` | Base-2 | **Yes** | +| `P` | Petabytes | `1000T` | Base-10 | No | +| `PB` | Petabytes | `1000T` | Base-10 | No | +| `PiB` | Pebibytes | `1024TiB` | Base-2 | **Yes** | + +### Size Expression Format + +Netdata allows users to express sizes using a number followed by a unit, such as `500MiB` (500 Mebibytes), `1GB` (1 Gigabyte), or `256K` (256 Kilobytes). + +- **Case Sensitivity**: Note that the parsing of units is case-sensitive. + +### Size Formatting + +Netdata formats a numeric size value (in bytes) into a human-readable string with an appropriate unit. The formatter's goal is to select the largest unit that can represent the size exactly, using up to two fractional digits. If two fractional digits are not enough to precisely represent the byte count, the formatter will use a smaller unit until it can accurately express the size, eventually falling back to bytes (`B`) if necessary. + +When formatting, Netdata prefers Base-2 units (`KiB`, `MiB`, `GiB`, etc.). + +- **Examples of Size Formatting**: + - **10,485,760 bytes** is formatted as `10MiB` (10 Mebibytes). + - **1,024 bytes** is formatted as `1KiB` (1 Kibibyte). + - **1,500 bytes** remains formatted as `1500B` because it cannot be precisely represented in `KiB` or any larger unit using up to two fractional digits. + +### Example Size Expressions + +Here are some examples of valid size expressions: + +1. `1024B`: 1024 bytes. +2. `1KiB`: 1024 bytes. +3. `5MiB`: 5 mebibytes (5 * 1024 * 1024 bytes). diff --git a/src/libnetdata/parsers/timeframe.c b/src/libnetdata/parsers/timeframe.c new file mode 100644 index 000000000..33ea69750 --- /dev/null +++ b/src/libnetdata/parsers/timeframe.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "timeframe.h" + +// -------------------------------------------------------------------------------------------------------------------- +// timeframe +/* +TIMEFRAME timeframe_parse(const char *txt) { + if(!txt || !*txt) + return TIMEFRAME_INVALID; + +char buf[strlen(txt) + 1]; +memcpy(buf, txt, strlen(txt) + 1); +char *s = trim_all(buf); +if(!s) + return TIMEFRAME_INVALID; + +while(isspace(*s)) s++; + +if(strcasecmp(s, "this minute") == 0) { + return (TIMEFRAME) { + .after = API_RELATIVE_TIME_THIS_MINUTE, + .before = 0, + }; +} +if(strcasecmp(s, "this hour") == 0) { + return (TIMEFRAME) { + .after = API_RELATIVE_TIME_THIS_HOUR, + .before = 0, + }; +} +if(strcasecmp(s, "today") == 0) { + return (TIMEFRAME) { + .after = API_RELATIVE_TIME_TODAY, + .before = 0, + }; +} +if(strcasecmp(s, "this week") == 0) { + return (TIMEFRAME) { + .after = API_RELATIVE_TIME_THIS_WEEK, + .before = 0, + }; +} +if(strcasecmp(s, "this month") == 0) { + return (TIMEFRAME) { + .after = API_RELATIVE_TIME_THIS_MONTH, + .before = 0, + }; +} +if(strcasecmp(s, "this year") == 0) { + return (TIMEFRAME) { + .after = API_RELATIVE_TIME_THIS_YEAR, + .before = 0, + }; +} + +if(strcasecmp(s, "last minute") == 0) { + return (TIMEFRAME) { + .after = -60, + .before = API_RELATIVE_TIME_THIS_MINUTE, + }; +} +if(strcasecmp(s, "last hour") == 0) { + return (TIMEFRAME) { + .after = -3600, + .before = API_RELATIVE_TIME_THIS_HOUR, + }; +} +if(strcasecmp(s, "yesterday") == 0) { + return (TIMEFRAME) { + .after = -86400, + .before = API_RELATIVE_TIME_TODAY, + }; +} +if(strcasecmp(s, "this week") == 0) { + return (TIMEFRAME) { + .after = -86400 * 7, + .before = API_RELATIVE_TIME_THIS_WEEK, + }; +} +if(strcasecmp(s, "this month") == 0) { + return (TIMEFRAME) { + .after = API_RELATIVE_TIME_LAST_MONTH, + .before = API_RELATIVE_TIME_THIS_MONTH, + }; +} +if(strcasecmp(s, "this year") == 0) { + return (TIMEFRAME) { + .after = API_RELATIVE_TIME_LAST_YEAR, + .before = API_RELATIVE_TIME_THIS_YEAR, + }; +} + +const char *end; +double after = strtondd(s, (char **)&end); + +if(end == s) + return TIMEFRAME_INVALID; + +s = end; +while(isspace(*s)) s++; + +time_t multiplier = 1; +if(!isdigit(*s) && *s != '-') { + // after has units + bool found = false; + + for (size_t i = 0; i < sizeof(units) / sizeof(units[0]); i++) { + size_t len = strlen(units[i].unit); + + if (units[i].multiplier >= 1 * NSEC_PER_USEC && + strncmp(s, units[i].unit, len) == 0 && + (isspace(s[len]) || s[len] == '-')) { + multiplier = units[i].multiplier / NSEC_PER_SEC; + found = true; + s += len; + } + } + + if(!found) + return TIMEFRAME_INVALID; +} + +const char *dash = strchr(s, '-'); +if(!dash) return TIMEFRAME_INVALID; + +} +*/ diff --git a/src/libnetdata/parsers/timeframe.h b/src/libnetdata/parsers/timeframe.h new file mode 100644 index 000000000..a176dd30a --- /dev/null +++ b/src/libnetdata/parsers/timeframe.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_TIMEFRAME_H +#define NETDATA_TIMEFRAME_H + +#include "parsers.h" + +typedef struct { + time_t after; + time_t before; +} TIMEFRAME; + +#define API_RELATIVE_TIME_MAX (3 * 365 * 86400) + +#define API_RELATIVE_TIME_INVALID (-1000000000) + +#define API_RELATIVE_TIME_THIS_MINUTE (API_RELATIVE_TIME_INVALID - 1) // this minute at 00 seconds +#define API_RELATIVE_TIME_THIS_HOUR (API_RELATIVE_TIME_INVALID - 2) // this hour at 00 minutes, 00 seconds +#define API_RELATIVE_TIME_TODAY (API_RELATIVE_TIME_INVALID - 3) // today at 00:00:00 +#define API_RELATIVE_TIME_THIS_WEEK (API_RELATIVE_TIME_INVALID - 4) // this Monday, 00:00:00 +#define API_RELATIVE_TIME_THIS_MONTH (API_RELATIVE_TIME_INVALID - 5) // this month's 1st at 00:00:00 +#define API_RELATIVE_TIME_THIS_YEAR (API_RELATIVE_TIME_INVALID - 6) // this year's Jan 1st, at 00:00:00 +#define API_RELATIVE_TIME_LAST_MONTH (API_RELATIVE_TIME_INVALID - 7) // last month's 1st, at 00:00:00 +#define API_RELATIVE_TIME_LAST_YEAR (API_RELATIVE_TIME_INVALID - 8) // last year's Jan 1st, at 00:00:00 + +#define TIMEFRAME_INVALID (TIMEFRAME){ .after = API_RELATIVE_TIME_INVALID, .before = API_RELATIVE_TIME_INVALID } + +#endif //NETDATA_TIMEFRAME_H diff --git a/src/libnetdata/paths/paths.c b/src/libnetdata/paths/paths.c new file mode 100644 index 000000000..c68ee805f --- /dev/null +++ b/src/libnetdata/paths/paths.c @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "paths.h" + +static int is_procfs(const char *path, char **reason) { +#if defined(__APPLE__) || defined(__FreeBSD__) + (void)path; + (void)reason; +#else + struct statfs stat; + + if (statfs(path, &stat) == -1) { + if (reason) + *reason = "failed to statfs()"; + return -1; + } + +#if defined PROC_SUPER_MAGIC + if (stat.f_type != PROC_SUPER_MAGIC) { + if (reason) + *reason = "type is not procfs"; + return -1; + } +#endif + +#endif + + return 0; +} + +static int is_sysfs(const char *path, char **reason) { +#if defined(__APPLE__) || defined(__FreeBSD__) + (void)path; + (void)reason; +#else + struct statfs stat; + + if (statfs(path, &stat) == -1) { + if (reason) + *reason = "failed to statfs()"; + return -1; + } + +#if defined SYSFS_MAGIC + if (stat.f_type != SYSFS_MAGIC) { + if (reason) + *reason = "type is not sysfs"; + return -1; + } +#endif + +#endif + + return 0; +} + +int verify_netdata_host_prefix(bool log_msg) { + if(!netdata_configured_host_prefix) + netdata_configured_host_prefix = ""; + + if(!*netdata_configured_host_prefix) + return 0; + + char path[FILENAME_MAX]; + char *reason = "unknown reason"; + errno_clear(); + + strncpyz(path, netdata_configured_host_prefix, sizeof(path) - 1); + + struct stat sb; + if (stat(path, &sb) == -1) { + reason = "failed to stat()"; + goto failed; + } + + if((sb.st_mode & S_IFMT) != S_IFDIR) { + errno = EINVAL; + reason = "is not a directory"; + goto failed; + } + + snprintfz(path, sizeof(path), "%s/proc", netdata_configured_host_prefix); + if(is_procfs(path, &reason) == -1) + goto failed; + + snprintfz(path, sizeof(path), "%s/sys", netdata_configured_host_prefix); + if(is_sysfs(path, &reason) == -1) + goto failed; + + if (netdata_configured_host_prefix && *netdata_configured_host_prefix) { + if (log_msg) + netdata_log_info("Using host prefix directory '%s'", netdata_configured_host_prefix); + } + + return 0; + +failed: + if (log_msg) + netdata_log_error("Ignoring host prefix '%s': path '%s' %s", netdata_configured_host_prefix, path, reason); + + netdata_configured_host_prefix = ""; + return -1; +} + +size_t filename_from_path_entry(char out[FILENAME_MAX], const char *path, const char *entry, const char *extension) { + if(unlikely(!path || !*path)) path = "."; + if(unlikely(!entry)) entry = ""; + + // skip trailing slashes in path + size_t len = strlen(path); + while(len > 0 && path[len - 1] == '/') len--; + + // skip leading slashes in subpath + while(entry[0] == '/') entry++; + + // if the last character in path is / and (there is a subpath or path is now empty) + // keep the trailing slash in path and remove the additional slash + char *slash = "/"; + if(path[len] == '/' && (*entry || len == 0)) { + slash = ""; + len++; + } + else if(!*entry) { + // there is no entry + // no need for trailing slash + slash = ""; + } + + return snprintfz(out, FILENAME_MAX, "%.*s%s%s%s%s", (int)len, path, slash, entry, + extension && *extension ? "." : "", + extension && *extension ? extension : ""); +} + +char *filename_from_path_entry_strdupz(const char *path, const char *entry) { + char filename[FILENAME_MAX]; + filename_from_path_entry(filename, path, entry, NULL); + return strdupz(filename); +} + +bool filename_is_dir(const char *filename, bool create_it) { + CLEAN_CHAR_P *buffer = NULL; + + size_t max_links = 100; + + bool is_dir = false; + struct stat st; + while(max_links && stat(filename, &st) == 0) { + if ((st.st_mode & S_IFMT) == S_IFDIR) + is_dir = true; + else if ((st.st_mode & S_IFMT) == S_IFLNK) { + max_links--; + + if(!buffer) + buffer = mallocz(FILENAME_MAX); + + char link_dst[FILENAME_MAX]; + ssize_t l = readlink(filename, link_dst, FILENAME_MAX - 1); + if (l > 0) { + link_dst[l] = '\0'; + strncpyz(buffer, link_dst, FILENAME_MAX - 1); + filename = buffer; + continue; + } + } + + break; + } + + if(!is_dir && create_it && max_links == 100 && mkdir(filename, 0750) == 0) + is_dir = true; + + return is_dir; +} + +bool path_entry_is_dir(const char *path, const char *entry, bool create_it) { + char filename[FILENAME_MAX]; + filename_from_path_entry(filename, path, entry, NULL); + return filename_is_dir(filename, create_it); +} + +bool filename_is_file(const char *filename) { + CLEAN_CHAR_P *buffer = NULL; + + size_t max_links = 100; + + bool is_file = false; + struct stat st; + while(max_links && stat(filename, &st) == 0) { + if((st.st_mode & S_IFMT) == S_IFREG) + is_file = true; + else if((st.st_mode & S_IFMT) == S_IFLNK) { + max_links--; + + if(!buffer) + buffer = mallocz(FILENAME_MAX); + + char link_dst[FILENAME_MAX]; + ssize_t l = readlink(filename, link_dst, FILENAME_MAX - 1); + if(l > 0) { + link_dst[l] = '\0'; + strncpyz(buffer, link_dst, FILENAME_MAX - 1); + filename = buffer; + continue; + } + } + + break; + } + + return is_file; +} + +bool path_entry_is_file(const char *path, const char *entry) { + char filename[FILENAME_MAX]; + filename_from_path_entry(filename, path, entry, NULL); + return filename_is_file(filename); +} + +void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *entry, int (*callback)(const char *filename, void *data, bool stock_config), void *data, size_t depth) { + if(depth > 3) { + netdata_log_error("CONFIG: Max directory depth reached while reading user path '%s', stock path '%s', subpath '%s'", user_path, stock_path, + entry); + return; + } + + if(!stock_path) + stock_path = user_path; + + char *udir = filename_from_path_entry_strdupz(user_path, entry); + char *sdir = filename_from_path_entry_strdupz(stock_path, entry); + + netdata_log_debug(D_HEALTH, "CONFIG traversing user-config directory '%s', stock config directory '%s'", udir, sdir); + + DIR *dir = opendir(udir); + if (!dir) { + netdata_log_error("CONFIG cannot open user-config directory '%s'.", udir); + } + else { + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type == DT_DIR || de->d_type == DT_LNK) { + if( !de->d_name[0] || + (de->d_name[0] == '.' && de->d_name[1] == '\0') || + (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') + ) { + netdata_log_debug(D_HEALTH, "CONFIG ignoring user-config directory '%s/%s'", udir, de->d_name); + continue; + } + + if(path_entry_is_dir(udir, de->d_name, false)) { + recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1); + continue; + } + } + + if(de->d_type == DT_UNKNOWN || de->d_type == DT_REG || de->d_type == DT_LNK) { + size_t len = strlen(de->d_name); + if(path_entry_is_file(udir, de->d_name) && + len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { + char *filename = filename_from_path_entry_strdupz(udir, de->d_name); + netdata_log_debug(D_HEALTH, "CONFIG calling callback for user file '%s'", filename); + callback(filename, data, false); + freez(filename); + continue; + } + } + + netdata_log_debug(D_HEALTH, "CONFIG ignoring user-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); + } + + closedir(dir); + } + + netdata_log_debug(D_HEALTH, "CONFIG traversing stock config directory '%s', user config directory '%s'", sdir, udir); + + dir = opendir(sdir); + if (!dir) { + netdata_log_error("CONFIG cannot open stock config directory '%s'.", sdir); + } + else { + if (strcmp(udir, sdir)) { + struct dirent *de = NULL; + while((de = readdir(dir))) { + if(de->d_type == DT_DIR || de->d_type == DT_LNK) { + if( !de->d_name[0] || + (de->d_name[0] == '.' && de->d_name[1] == '\0') || + (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') + ) { + netdata_log_debug(D_HEALTH, "CONFIG ignoring stock config directory '%s/%s'", sdir, de->d_name); + continue; + } + + if(path_entry_is_dir(sdir, de->d_name, false)) { + // we recurse in stock subdirectory, only when there is no corresponding + // user subdirectory - to avoid reading the files twice + + if(!path_entry_is_dir(udir, de->d_name, false)) + recursive_config_double_dir_load(udir, sdir, de->d_name, callback, data, depth + 1); + + continue; + } + } + + if(de->d_type == DT_UNKNOWN || de->d_type == DT_REG || de->d_type == DT_LNK) { + size_t len = strlen(de->d_name); + if(path_entry_is_file(sdir, de->d_name) && !path_entry_is_file(udir, de->d_name) && + len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { + char *filename = filename_from_path_entry_strdupz(sdir, de->d_name); + netdata_log_debug(D_HEALTH, "CONFIG calling callback for stock file '%s'", filename); + callback(filename, data, true); + freez(filename); + continue; + } + + } + + netdata_log_debug(D_HEALTH, "CONFIG ignoring stock-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); + } + } + closedir(dir); + } + + netdata_log_debug(D_HEALTH, "CONFIG done traversing user-config directory '%s', stock config directory '%s'", udir, sdir); + + freez(udir); + freez(sdir); +} diff --git a/src/libnetdata/paths/paths.h b/src/libnetdata/paths/paths.h new file mode 100644 index 000000000..9c5a8a748 --- /dev/null +++ b/src/libnetdata/paths/paths.h @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_PATHS_H +#define NETDATA_PATHS_H + +#include "../libnetdata.h" + +size_t filename_from_path_entry(char out[FILENAME_MAX], const char *path, const char *entry, const char *extension); +char *filename_from_path_entry_strdupz(const char *path, const char *entry); + +bool filename_is_file(const char *filename); +bool filename_is_dir(const char *filename, bool create_it); + +bool path_entry_is_file(const char *path, const char *entry); +bool path_entry_is_dir(const char *path, const char *entry, bool create_it); + +void recursive_config_double_dir_load( + const char *user_path + , const char *stock_path + , const char *entry + , int (*callback)(const char *filename, void *data, bool stock_config) + , void *data + , size_t depth +); + +#endif //NETDATA_PATHS_H diff --git a/src/libnetdata/procfile/README.md b/src/libnetdata/procfile/README.md index 9e737a511..faa00c6a2 100644 --- a/src/libnetdata/procfile/README.md +++ b/src/libnetdata/procfile/README.md @@ -1,12 +1,3 @@ -<!-- -title: "PROCFILE" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/procfile/README.md -sidebar_label: "Procfile" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # PROCFILE procfile is a library for reading text data files (i.e `/proc` files) in the fastest possible way. diff --git a/src/libnetdata/procfile/procfile.c b/src/libnetdata/procfile/procfile.c index 2b7eeeb56..fb6b0f8c3 100644 --- a/src/libnetdata/procfile/procfile.c +++ b/src/libnetdata/procfile/procfile.c @@ -10,14 +10,23 @@ int procfile_open_flags = O_RDONLY | O_CLOEXEC; -int procfile_adaptive_initial_allocation = 0; - // if adaptive allocation is set, these store the // max values we have seen so far -size_t procfile_max_lines = PFLINES_INCREASE_STEP; -size_t procfile_max_words = PFWORDS_INCREASE_STEP; -size_t procfile_max_allocation = PROCFILE_INCREMENT_BUFFER; - +static bool procfile_adaptive_initial_allocation = false; +static size_t procfile_max_lines = PFLINES_INCREASE_STEP; +static size_t procfile_max_words = PFWORDS_INCREASE_STEP; +static size_t procfile_max_allocation = PROCFILE_INCREMENT_BUFFER; + +void procfile_set_adaptive_allocation(bool enable, size_t bytes, size_t lines, size_t words) { + procfile_adaptive_initial_allocation = enable; + + if(bytes > procfile_max_allocation) + procfile_max_allocation = bytes; + if(lines > procfile_max_lines) + procfile_max_lines = lines; + if(words > procfile_max_words) + procfile_max_words = words; +} // ---------------------------------------------------------------------------- @@ -59,6 +68,8 @@ static inline void procfile_words_add(procfile *ff, char *str) { ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + wanted) * sizeof(char *)); fw->size += wanted; + ff->stats.memory += wanted * sizeof(char *); + ff->stats.resizes++; } fw->words[fw->len++] = str; @@ -92,7 +103,7 @@ static inline void procfile_words_free(pfwords *fw) { // An array of lines NEVERNULL -static inline size_t *procfile_lines_add(procfile *ff) { +static inline uint32_t *procfile_lines_add(procfile *ff) { // netdata_log_debug(D_PROCFILE, PF_PREFIX ": adding line %d at word %d", fl->len, first_word); pflines *fl = ff->lines; @@ -104,6 +115,8 @@ static inline size_t *procfile_lines_add(procfile *ff) { ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + wanted) * sizeof(ffline)); fl->size += wanted; + ff->stats.memory += wanted * sizeof(ffline); + ff->stats.resizes++; } ffline *ffl = &fl->lines[fl->len++]; @@ -168,7 +181,7 @@ static void procfile_parser(procfile *ff) { char quote = 0; // the quote character - only when in quoted string size_t opened = 0; // counts the number of open parenthesis - size_t *line_words = procfile_lines_add(ff); + uint32_t *line_words = procfile_lines_add(ff); while(s < e) { PF_CHAR_TYPE ct = separators[(unsigned char)(*s)]; @@ -230,8 +243,12 @@ static void procfile_parser(procfile *ff) { } else if(likely(ct == PF_CHAR_IS_OPEN)) { if(s == t) { + if(!opened) + t = ++s; + else + ++s; + opened++; - t = ++s; } else if(opened) { opened++; @@ -275,6 +292,8 @@ static void procfile_parser(procfile *ff) { } procfile *procfile_readall(procfile *ff) { + if(!ff) return NULL; + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": Reading file '%s'.", ff->filename); ff->len = 0; // zero the used size @@ -291,9 +310,12 @@ procfile *procfile_readall(procfile *ff) { netdata_log_debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s' by %zu bytes.", procfile_filename(ff), wanted); ff = reallocz(ff, sizeof(procfile) + ff->size + wanted); ff->size += wanted; + ff->stats.memory += wanted; + ff->stats.resizes++; } - netdata_log_debug(D_PROCFILE, "Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s)); + // netdata_log_info("Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s)); + ff->stats.reads++; r = read(ff->fd, &ff->data[s], ff->size - s); if(unlikely(r == -1)) { if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) collector_error(PF_PREFIX ": Cannot read from file '%s' on fd %d", procfile_filename(ff), ff->fd); @@ -303,6 +325,9 @@ procfile *procfile_readall(procfile *ff) { return NULL; } + if((ssize_t)ff->stats.max_read_size < r) + ff->stats.max_read_size = r; + ff->len += r; } @@ -325,6 +350,17 @@ procfile *procfile_readall(procfile *ff) { if(unlikely(ff->words->len > procfile_max_words)) procfile_max_words = ff->words->len; } + if(ff->stats.max_source_bytes < ff->len) + ff->stats.max_source_bytes = ff->len; + + if(ff->stats.max_lines < ff->lines->len) + ff->stats.max_lines = ff->lines->len; + + if(ff->stats.max_words < ff->words->len) + ff->stats.max_words = ff->words->len; + + ff->stats.total_read_bytes += ff->len; + // netdata_log_debug(D_PROCFILE, "File '%s' updated.", ff->filename); return ff; } @@ -429,10 +465,18 @@ procfile *procfile_open(const char *filename, const char *separators, uint32_t f ff->size = size; ff->len = 0; ff->flags = flags; + ff->stats.opens = 1; + ff->stats.reads = ff->stats.resizes = 0; + ff->stats.max_lines = ff->stats.max_words = ff->stats.max_source_bytes = 0; + ff->stats.total_read_bytes = ff->stats.max_read_size = 0; ff->lines = procfile_lines_create(); ff->words = procfile_words_create(); + ff->stats.memory = sizeof(procfile) + size + + (sizeof(pflines) + ff->lines->size * sizeof(ffline)) + + (sizeof(pfwords) + ff->words->size * sizeof(char *)); + procfile_set_separators(ff, separators); netdata_log_debug(D_PROCFILE, "File '%s' opened.", filename); @@ -452,6 +496,7 @@ procfile *procfile_reopen(procfile *ff, const char *filename, const char *separa procfile_close(ff); return NULL; } + ff->stats.opens++; // netdata_log_info("PROCFILE: opened '%s' on fd %d", filename, ff->fd); @@ -479,7 +524,7 @@ void procfile_print(procfile *ff) { for(l = 0; likely(l < lines) ;l++) { size_t words = procfile_linewords(ff, l); - netdata_log_debug(D_PROCFILE, " line %zu starts at word %zu and has %zu words", l, ff->lines->lines[l].first, ff->lines->lines[l].words); + netdata_log_debug(D_PROCFILE, " line %zu starts at word %zu and has %zu words", l, (size_t)ff->lines->lines[l].first, (size_t)ff->lines->lines[l].words); size_t w; for(w = 0; likely(w < words) ;w++) { diff --git a/src/libnetdata/procfile/procfile.h b/src/libnetdata/procfile/procfile.h index 8db5b45f4..25b976988 100644 --- a/src/libnetdata/procfile/procfile.h +++ b/src/libnetdata/procfile/procfile.h @@ -19,9 +19,8 @@ typedef struct { // An array of lines typedef struct { - size_t words; // how many words this line has - size_t first; // the id of the first word of this line - // in the words array + uint32_t words; // how many words this line has + uint32_t first; // the id of the first word of this line in the words array } ffline; typedef struct { @@ -35,7 +34,7 @@ typedef struct { // The procfile #define PROCFILE_FLAG_DEFAULT 0x00000000 // To store inside `collector.log` -#define PROCFILE_FLAG_NO_ERROR_ON_FILE_IO 0x00000001 // Do not store nothing +#define PROCFILE_FLAG_NO_ERROR_ON_FILE_IO 0x00000001 // Do not log anything #define PROCFILE_FLAG_ERROR_ON_ERROR_LOG 0x00000002 // Store inside `error.log` typedef enum __attribute__ ((__packed__)) procfile_separator { @@ -47,7 +46,22 @@ typedef enum __attribute__ ((__packed__)) procfile_separator { PF_CHAR_IS_CLOSE } PF_CHAR_TYPE; +struct procfile_stats { + size_t opens; + size_t reads; + size_t resizes; + size_t memory; + size_t total_read_bytes; + size_t max_source_bytes; + size_t max_lines; + size_t max_words; + size_t max_read_size; +}; + + typedef struct procfile { + // this structure is malloc'd (you need to initialize it at procfile_open() + char *filename; // not populated until procfile_filename() is called uint32_t flags; int fd; // the file descriptor @@ -56,6 +70,7 @@ typedef struct procfile { pflines *lines; pfwords *words; PF_CHAR_TYPE separators[256]; + struct procfile_stats stats; char data[]; // allocated buffer to keep file contents } procfile; @@ -85,8 +100,8 @@ char *procfile_filename(procfile *ff); // set to the O_XXXX flags, to have procfile_open and procfile_reopen use them when opening proc files extern int procfile_open_flags; -// set this to 1, to have procfile adapt its initial buffer allocation to the max allocation used so far -extern int procfile_adaptive_initial_allocation; +// call this with true and the expected initial sizes to allow procfile learn the sizes needed +void procfile_set_adaptive_allocation(bool enable, size_t bytes, size_t lines, size_t words); // return the number of lines present #define procfile_lines(ff) ((ff)->lines->len) diff --git a/src/libnetdata/query_progress/progress.c b/src/libnetdata/query_progress/progress.c index 10e083e0c..157f20f11 100644 --- a/src/libnetdata/query_progress/progress.c +++ b/src/libnetdata/query_progress/progress.c @@ -8,7 +8,7 @@ // hashtable for HASHED_KEY // cleanup hashtable defines -#include "../simple_hashtable_undef.h" +#include "../simple_hashtable/simple_hashtable_undef.h" struct query; #define SIMPLE_HASHTABLE_VALUE_TYPE struct query @@ -16,7 +16,7 @@ struct query; #define SIMPLE_HASHTABLE_NAME _QUERY #define SIMPLE_HASHTABLE_VALUE2KEY_FUNCTION query_transaction #define SIMPLE_HASHTABLE_COMPARE_KEYS_FUNCTION query_compare_keys -#include "../simple_hashtable.h" +#include "../simple_hashtable/simple_hashtable.h" // ---------------------------------------------------------------------------- @@ -76,12 +76,7 @@ static struct progress { }; SIMPLE_HASHTABLE_HASH query_hash(nd_uuid_t *transaction) { - struct uuid_hi_lo_t { - uint64_t hi; - uint64_t lo; - } *parts = (struct uuid_hi_lo_t *)transaction; - - return parts->lo; + return XXH3_64bits(transaction, sizeof(*transaction)); } static void query_progress_init_unsafe(void) { diff --git a/src/libnetdata/required_dummies.h b/src/libnetdata/required_dummies.h index 3b23b87f7..cff4c563a 100644 --- a/src/libnetdata/required_dummies.h +++ b/src/libnetdata/required_dummies.h @@ -13,11 +13,6 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re exit(ret); } -// callbacks required by popen() -void signals_block(void){} -void signals_unblock(void){} -void signals_reset(void){} - void rrdset_thread_rda_free(void){} void sender_thread_buffer_free(void){} void query_target_free(void){} @@ -25,6 +20,6 @@ void service_exits(void){} void rrd_collector_finished(void){} // required by get_system_cpus() -char *netdata_configured_host_prefix = ""; +const char *netdata_configured_host_prefix = ""; #endif // NETDATA_LIB_DUMMIES_H diff --git a/src/libnetdata/ringbuffer/ringbuffer.c b/src/libnetdata/ringbuffer/ringbuffer.c new file mode 100644 index 000000000..b30b3c39a --- /dev/null +++ b/src/libnetdata/ringbuffer/ringbuffer.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" +#include "ringbuffer_internal.h" + +rbuf_t rbuf_create(size_t size) +{ + rbuf_t buffer = mallocz(sizeof(struct rbuf) + size); + memset(buffer, 0, sizeof(struct rbuf)); + + buffer->data = ((char*)buffer) + sizeof(struct rbuf); + + buffer->head = buffer->data; + buffer->tail = buffer->data; + buffer->size = size; + buffer->end = buffer->data + size; + + return buffer; +} + +void rbuf_free(rbuf_t buffer) +{ + freez(buffer); +} + +void rbuf_flush(rbuf_t buffer) +{ + buffer->head = buffer->data; + buffer->tail = buffer->data; + buffer->size_data = 0; +} + +char *rbuf_get_linear_insert_range(rbuf_t buffer, size_t *bytes) +{ + *bytes = 0; + if (buffer->head == buffer->tail && buffer->size_data) + return NULL; + + *bytes = ((buffer->head >= buffer->tail) ? buffer->end : buffer->tail) - buffer->head; + return buffer->head; +} + +char *rbuf_get_linear_read_range(rbuf_t buffer, size_t *bytes) +{ + *bytes = 0; + if(buffer->head == buffer->tail && !buffer->size_data) + return NULL; + + *bytes = ((buffer->tail >= buffer->head) ? buffer->end : buffer->head) - buffer->tail; + + return buffer->tail; +} + +int rbuf_bump_head(rbuf_t buffer, size_t bytes) +{ + size_t free_bytes = rbuf_bytes_free(buffer); + if (bytes > free_bytes) + return 0; + int i = buffer->head - buffer->data; + buffer->head = &buffer->data[(i + bytes) % buffer->size]; + buffer->size_data += bytes; + return 1; +} + +int rbuf_bump_tail_noopt(rbuf_t buffer, size_t bytes) +{ + if (bytes > buffer->size_data) + return 0; + int i = buffer->tail - buffer->data; + buffer->tail = &buffer->data[(i + bytes) % buffer->size]; + buffer->size_data -= bytes; + + return 1; +} + +int rbuf_bump_tail(rbuf_t buffer, size_t bytes) +{ + if(!rbuf_bump_tail_noopt(buffer, bytes)) + return 0; + + // if tail catched up with head + // start writing buffer from beggining + // this is not necessary (rbuf must work well without it) + // but helps to optimize big writes as rbuf_get_linear_insert_range + // will return bigger continuous region + if(buffer->tail == buffer->head) { + assert(buffer->size_data == 0); + rbuf_flush(buffer); + } + + return 1; +} + +size_t rbuf_get_capacity(rbuf_t buffer) +{ + return buffer->size; +} + +size_t rbuf_bytes_available(rbuf_t buffer) +{ + return buffer->size_data; +} + +size_t rbuf_bytes_free(rbuf_t buffer) +{ + return buffer->size - buffer->size_data; +} + +size_t rbuf_push(rbuf_t buffer, const char *data, size_t len) +{ + size_t to_cpy; + char *w_ptr = rbuf_get_linear_insert_range(buffer, &to_cpy); + if(!to_cpy) + return to_cpy; + + to_cpy = MIN(to_cpy, len); + memcpy(w_ptr, data, to_cpy); + rbuf_bump_head(buffer, to_cpy); + if(to_cpy < len) + to_cpy += rbuf_push(buffer, &data[to_cpy], len - to_cpy); + return to_cpy; +} + +size_t rbuf_pop(rbuf_t buffer, char *data, size_t len) +{ + size_t to_cpy; + const char *r_ptr = rbuf_get_linear_read_range(buffer, &to_cpy); + if(!to_cpy) + return to_cpy; + + to_cpy = MIN(to_cpy, len); + memcpy(data, r_ptr, to_cpy); + rbuf_bump_tail(buffer, to_cpy); + if(to_cpy < len) + to_cpy += rbuf_pop(buffer, &data[to_cpy], len - to_cpy); + return to_cpy; +} + +static inline void rbuf_ptr_inc(rbuf_t buffer, const char **ptr) +{ + (*ptr)++; + if(*ptr >= buffer->end) + *ptr = buffer->data; +} + +int rbuf_memcmp(rbuf_t buffer, const char *haystack, const char *needle, size_t needle_bytes) +{ + const char *end = needle + needle_bytes; + + // as head==tail can mean 2 things here + if (haystack == buffer->head && buffer->size_data) { + if (*haystack != *needle) + return (*haystack - *needle); + rbuf_ptr_inc(buffer, &haystack); + needle++; + } + + while (haystack != buffer->head && needle != end) { + if (*haystack != *needle) + return (*haystack - *needle); + rbuf_ptr_inc(buffer, &haystack); + needle++; + } + return 0; +} + +int rbuf_memcmp_n(rbuf_t buffer, const char *to_cmp, size_t to_cmp_bytes) +{ + return rbuf_memcmp(buffer, buffer->tail, to_cmp, to_cmp_bytes); +} + +char *rbuf_find_bytes(rbuf_t buffer, const char *needle, size_t needle_bytes, int *found_idx) +{ + const char *ptr = buffer->tail; + *found_idx = 0; + + if (!rbuf_bytes_available(buffer)) + return NULL; + + if (buffer->head == buffer->tail && buffer->size_data) { + if(!rbuf_memcmp(buffer, ptr, needle, needle_bytes)) + return (char *)ptr; + rbuf_ptr_inc(buffer, &ptr); + (*found_idx)++; + } + + while (ptr != buffer->head) + { + if(!rbuf_memcmp(buffer, ptr, needle, needle_bytes)) + return (char *)ptr; + rbuf_ptr_inc(buffer, &ptr); + (*found_idx)++; + } + return NULL; +} diff --git a/src/libnetdata/ringbuffer/ringbuffer.h b/src/libnetdata/ringbuffer/ringbuffer.h new file mode 100644 index 000000000..340112a8f --- /dev/null +++ b/src/libnetdata/ringbuffer/ringbuffer.h @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RINGBUFFER_H +#define RINGBUFFER_H +#include "../libnetdata.h" + +typedef struct rbuf *rbuf_t; + +rbuf_t rbuf_create(size_t size); +void rbuf_free(rbuf_t buffer); +void rbuf_flush(rbuf_t buffer); + +/* /param bytes how much bytes can be copied into pointer returned + * /return pointer where data can be copied to or NULL if buffer full + */ +char *rbuf_get_linear_insert_range(rbuf_t buffer, size_t *bytes); +char *rbuf_get_linear_read_range(rbuf_t buffer, size_t *bytes); + +int rbuf_bump_head(rbuf_t buffer, size_t bytes); +int rbuf_bump_tail(rbuf_t buffer, size_t bytes); + +/* @param buffer related buffer instance + * @returns total capacity of buffer in bytes (not free/used) + */ +size_t rbuf_get_capacity(rbuf_t buffer); + +/* @param buffer related buffer instance + * @returns count of bytes stored in the buffer + */ +size_t rbuf_bytes_available(rbuf_t buffer); + +/* @param buffer related buffer instance + * @returns count of bytes available/free in the buffer (how many more bytes you can store in this buffer) + */ +size_t rbuf_bytes_free(rbuf_t buffer); + +/* writes as many bytes from `data` into the `buffer` as possible + * but maximum `len` bytes + */ +size_t rbuf_push(rbuf_t buffer, const char *data, size_t len); +size_t rbuf_pop(rbuf_t buffer, char *data, size_t len); + +char *rbuf_find_bytes(rbuf_t buffer, const char *needle, size_t needle_bytes, int *found_idx); +int rbuf_memcmp_n(rbuf_t buffer, const char *to_cmp, size_t to_cmp_bytes); + +#endif diff --git a/src/libnetdata/ringbuffer/ringbuffer_internal.h b/src/libnetdata/ringbuffer/ringbuffer_internal.h new file mode 100644 index 000000000..0cc254aa8 --- /dev/null +++ b/src/libnetdata/ringbuffer/ringbuffer_internal.h @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RINGBUFFER_INTERNAL_H +#define RINGBUFFER_INTERNAL_H + +#include "ringbuffer.h" + +struct rbuf { + char *data; + + // points to next byte where we can write + char *head; + // points to oldest (next to be poped) readable byte + char *tail; + + // to avoid calculating data + size + // all the time + char *end; + + size_t size; + size_t size_data; +}; + +typedef struct rbuf *rbuf_t; + +#endif diff --git a/src/libnetdata/sanitizers/chart_id_and_name.c b/src/libnetdata/sanitizers/chart_id_and_name.c new file mode 100644 index 000000000..5af8aa686 --- /dev/null +++ b/src/libnetdata/sanitizers/chart_id_and_name.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +/* + * control characters become space, which are deduplicated. + * + * Character Name Sym To Why + * ---------------- --- --- ------------------------------------------------------------------------------------- + * space [ ] -> [_] + * exclamation mark [!] -> [_] (only when it is the first character) simple patterns negation + * double quotes ["] -> [_] needs escaping when parsing + * dollar [$] -> [_] health variables and security in alarm-notify.sh, cgroup-name.sh, etc. + * percent [%] -> [_] http GET encoded characters + * ampersand [&] -> [_] http GET fields separator + * single quote ['] -> [_] needs escaping when parsing + * asterisk [*] -> [_] simple pattern wildcard + * plus [+] -> [_] http GET space + * comma [,] -> [.] list separator (probably not used today) + * equal [=] -> [_] plugins.d protocol separator + * question mark [?] -> [_] http GET query string separator + * at [@] -> [_] hostname separator (on the UI) + * apostrophe [`] -> [_] bash expansion (security in alarm-notify.sh and other shell scripts) + * pipe [|] -> [_] list separator (simple patterns and http GET) + * backslash [\] -> [/] to avoid interfering with escaping logic + */ + +unsigned char chart_names_allowed_chars[256] = { + [0] = '\0', [1] = ' ', [2] = ' ', [3] = ' ', [4] = ' ', [5] = ' ', [6] = ' ', [7] = ' ', [8] = ' ', + + // control characters to be treated as spaces + ['\t'] = ' ', ['\n'] = ' ', ['\v'] = ' ', ['\f'] = ' ', ['\r'] = ' ', + + [14] = ' ', [15] = ' ', [16] = ' ', [17] = ' ', [18] = ' ', [19] = ' ', [20] = ' ', [21] = ' ', + [22] = ' ', [23] = ' ', [24] = ' ', [25] = ' ', [26] = ' ', [27] = ' ', [28] = ' ', [29] = ' ', + [30] = ' ', [31] = ' ', + + // symbols + [' '] = ' ', ['!'] = '!', ['"'] = '_', ['#'] = '#', ['$'] = '_', ['%'] = '_', ['&'] = '_', ['\''] = '_', + ['('] = '(', [')'] = ')', ['*'] = '_', ['+'] = '_', [','] = '.', ['-'] = '-', ['.'] = '.', ['/'] = '/', + + // numbers + ['0'] = '0', ['1'] = '1', ['2'] = '2', ['3'] = '3', ['4'] = '4', ['5'] = '5', ['6'] = '6', ['7'] = '7', + ['8'] = '8', ['9'] = '9', + + // symbols + [':'] = ':', [';'] = ';', ['<'] = '<', ['='] = '_', ['>'] = '>', ['?'] = '_', ['@'] = '_', + + // capitals + ['A'] = 'A', ['B'] = 'B', ['C'] = 'C', ['D'] = 'D', ['E'] = 'E', ['F'] = 'F', ['G'] = 'G', ['H'] = 'H', + ['I'] = 'I', ['J'] = 'J', ['K'] = 'K', ['L'] = 'L', ['M'] = 'M', ['N'] = 'N', ['O'] = 'O', ['P'] = 'P', + ['Q'] = 'Q', ['R'] = 'R', ['S'] = 'S', ['T'] = 'T', ['U'] = 'U', ['V'] = 'V', ['W'] = 'W', ['X'] = 'X', + ['Y'] = 'Y', ['Z'] = 'Z', + + // symbols + ['['] = '[', ['\\'] = '/', [']'] = ']', ['^'] = '_', ['_'] = '_', ['`'] = '_', + + // lower + ['a'] = 'a', ['b'] = 'b', ['c'] = 'c', ['d'] = 'd', ['e'] = 'e', ['f'] = 'f', ['g'] = 'g', ['h'] = 'h', + ['i'] = 'i', ['j'] = 'j', ['k'] = 'k', ['l'] = 'l', ['m'] = 'm', ['n'] = 'n', ['o'] = 'o', ['p'] = 'p', + ['q'] = 'q', ['r'] = 'r', ['s'] = 's', ['t'] = 't', ['u'] = 'u', ['v'] = 'v', ['w'] = 'w', ['x'] = 'x', + ['y'] = 'y', ['z'] = 'z', + + // symbols + ['{'] = '{', ['|'] = '_', ['}'] = '}', ['~'] = '~', + + // rest + [127] = ' ', [128] = ' ', [129] = ' ', [130] = ' ', [131] = ' ', [132] = ' ', [133] = ' ', [134] = ' ', + [135] = ' ', [136] = ' ', [137] = ' ', [138] = ' ', [139] = ' ', [140] = ' ', [141] = ' ', [142] = ' ', + [143] = ' ', [144] = ' ', [145] = ' ', [146] = ' ', [147] = ' ', [148] = ' ', [149] = ' ', [150] = ' ', + [151] = ' ', [152] = ' ', [153] = ' ', [154] = ' ', [155] = ' ', [156] = ' ', [157] = ' ', [158] = ' ', + [159] = ' ', [160] = ' ', [161] = ' ', [162] = ' ', [163] = ' ', [164] = ' ', [165] = ' ', [166] = ' ', + [167] = ' ', [168] = ' ', [169] = ' ', [170] = ' ', [171] = ' ', [172] = ' ', [173] = ' ', [174] = ' ', + [175] = ' ', [176] = ' ', [177] = ' ', [178] = ' ', [179] = ' ', [180] = ' ', [181] = ' ', [182] = ' ', + [183] = ' ', [184] = ' ', [185] = ' ', [186] = ' ', [187] = ' ', [188] = ' ', [189] = ' ', [190] = ' ', + [191] = ' ', [192] = ' ', [193] = ' ', [194] = ' ', [195] = ' ', [196] = ' ', [197] = ' ', [198] = ' ', + [199] = ' ', [200] = ' ', [201] = ' ', [202] = ' ', [203] = ' ', [204] = ' ', [205] = ' ', [206] = ' ', + [207] = ' ', [208] = ' ', [209] = ' ', [210] = ' ', [211] = ' ', [212] = ' ', [213] = ' ', [214] = ' ', + [215] = ' ', [216] = ' ', [217] = ' ', [218] = ' ', [219] = ' ', [220] = ' ', [221] = ' ', [222] = ' ', + [223] = ' ', [224] = ' ', [225] = ' ', [226] = ' ', [227] = ' ', [228] = ' ', [229] = ' ', [230] = ' ', + [231] = ' ', [232] = ' ', [233] = ' ', [234] = ' ', [235] = ' ', [236] = ' ', [237] = ' ', [238] = ' ', + [239] = ' ', [240] = ' ', [241] = ' ', [242] = ' ', [243] = ' ', [244] = ' ', [245] = ' ', [246] = ' ', + [247] = ' ', [248] = ' ', [249] = ' ', [250] = ' ', [251] = ' ', [252] = ' ', [253] = ' ', [254] = ' ', + [255] = ' ' +}; + +static inline void sanitize_chart_name(char *dst, const char *src, size_t dst_size) { + // text_sanitize deduplicates spaces + text_sanitize((unsigned char *)dst, (const unsigned char *)src, dst_size, + chart_names_allowed_chars, true, "", NULL); + + char *d = dst; + + // do not accept ! as the first character + if(*d == '!') *d = '_'; + + // convert remaining spaces to underscores + while(*d) { + if(*d == ' ') *d = '_'; + d++; + } +} + +// make sure the supplied string +// is good for a netdata chart/dimension ID/NAME +void netdata_fix_chart_name(char *s) { + sanitize_chart_name(s, s, strlen(s) + 1); +} + +void netdata_fix_chart_id(char *s) { + sanitize_chart_name(s, s, strlen(s) + 1); +// size_t len = strlen(s); +// char buf[len + 1]; +// +// text_sanitize((unsigned char *)buf, (const unsigned char *)s, sizeof(buf), +// chart_names_allowed_chars, true, "", NULL); +// +// if(memcmp(s, buf, sizeof(buf)) == 0) +// // they are the same +// return; +// +// // they differ +// XXH128_hash_t hash = XXH3_128bits(s, len); +// ND_UUID *uuid = (ND_UUID *)&hash; +// internal_fatal(sizeof(hash) != sizeof(ND_UUID), "XXH128 and ND_UUID do not have the same size"); +// buf[0] = 'x'; +// buf[1] = 'x'; +// buf[2] = 'h'; +// buf[3] = '_'; +// uuid_unparse_lower_compact(uuid->uuid, &buf[4]); +} + +char *rrdset_strncpyz_name(char *dst, const char *src, size_t dst_size_minus_1) { + // src starts with "type." + sanitize_chart_name(dst, src, dst_size_minus_1 + 1); + return dst; +} + +bool rrdvar_fix_name(char *variable) { + size_t len = strlen(variable); + char buf[len + 1]; + memcpy(buf, variable, sizeof(buf)); + sanitize_chart_name(variable, variable, len + 1); + return memcmp(buf, variable, sizeof(buf)) != 0; +} diff --git a/src/libnetdata/sanitizers/chart_id_and_name.h b/src/libnetdata/sanitizers/chart_id_and_name.h new file mode 100644 index 000000000..eda6e3f30 --- /dev/null +++ b/src/libnetdata/sanitizers/chart_id_and_name.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_CHART_ID_AND_NAME_H +#define NETDATA_CHART_ID_AND_NAME_H + +#include "../libnetdata.h" + +void netdata_fix_chart_id(char *s); +void netdata_fix_chart_name(char *s); +char *rrdset_strncpyz_name(char *dst, const char *src, size_t dst_size_minus_1); +bool rrdvar_fix_name(char *variable); + +extern unsigned char chart_names_allowed_chars[256]; +static inline bool is_netdata_api_valid_character(char c) { + if(IS_UTF8_BYTE(c)) return true; + unsigned char t = chart_names_allowed_chars[(unsigned char)c]; + // the translation converts space to space + // so we have to check explicitly + return t == (unsigned char)c && t != ' ' && t != '!'; +} + +#endif //NETDATA_CHART_ID_AND_NAME_H diff --git a/src/libnetdata/sanitizers/sanitizers-functions.c b/src/libnetdata/sanitizers/sanitizers-functions.c new file mode 100644 index 000000000..5e1d87c35 --- /dev/null +++ b/src/libnetdata/sanitizers/sanitizers-functions.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "sanitizers-functions.h" + +static unsigned char functions_allowed_chars[256] = { + [0] = '\0', [1] = ' ', [2] = ' ', [3] = ' ', [4] = ' ', [5] = ' ', [6] = ' ', [7] = ' ', [8] = ' ', + + // control characters to be treated as spaces + ['\t'] = ' ', ['\n'] = ' ', ['\v'] = ' ', ['\f'] = ' ', ['\r'] = ' ', + + [14] = ' ', [15] = ' ', [16] = ' ', [17] = ' ', [18] = ' ', [19] = ' ', [20] = ' ', [21] = ' ', + [22] = ' ', [23] = ' ', [24] = ' ', [25] = ' ', [26] = ' ', [27] = ' ', [28] = ' ', [29] = ' ', + [30] = ' ', [31] = ' ', + + // symbols + [' '] = ' ', ['!'] = '!', ['"'] = '\'', ['#'] = '#', ['$'] = '$', ['%'] = '%', ['&'] = '&', ['\''] = '\'', + ['('] = '(', [')'] = ')', ['*'] = '*', ['+'] = '+', [','] = ',', ['-'] = '-', ['.'] = '.', ['/'] = '/', + + // numbers + ['0'] = '0', ['1'] = '1', ['2'] = '2', ['3'] = '3', ['4'] = '4', ['5'] = '5', ['6'] = '6', ['7'] = '7', + ['8'] = '8', ['9'] = '9', + + // symbols + [':'] = ':', [';'] = ';', ['<'] = '<', ['='] = '=', ['>'] = '>', ['?'] = '?', ['@'] = '@', + + // capitals + ['A'] = 'A', ['B'] = 'B', ['C'] = 'C', ['D'] = 'D', ['E'] = 'E', ['F'] = 'F', ['G'] = 'G', ['H'] = 'H', + ['I'] = 'I', ['J'] = 'J', ['K'] = 'K', ['L'] = 'L', ['M'] = 'M', ['N'] = 'N', ['O'] = 'O', ['P'] = 'P', + ['Q'] = 'Q', ['R'] = 'R', ['S'] = 'S', ['T'] = 'T', ['U'] = 'U', ['V'] = 'V', ['W'] = 'W', ['X'] = 'X', + ['Y'] = 'Y', ['Z'] = 'Z', + + // symbols + ['['] = '[', ['\\'] = '\\', [']'] = ']', ['^'] = '^', ['_'] = '_', ['`'] = '`', + + // lower + ['a'] = 'a', ['b'] = 'b', ['c'] = 'c', ['d'] = 'd', ['e'] = 'e', ['f'] = 'f', ['g'] = 'g', ['h'] = 'h', + ['i'] = 'i', ['j'] = 'j', ['k'] = 'k', ['l'] = 'l', ['m'] = 'm', ['n'] = 'n', ['o'] = 'o', ['p'] = 'p', + ['q'] = 'q', ['r'] = 'r', ['s'] = 's', ['t'] = 't', ['u'] = 'u', ['v'] = 'v', ['w'] = 'w', ['x'] = 'x', + ['y'] = 'y', ['z'] = 'z', + + // symbols + ['{'] = '{', ['|'] = '|', ['}'] = '}', ['~'] = '~', + + // rest + [127] = ' ', [128] = ' ', [129] = ' ', [130] = ' ', [131] = ' ', [132] = ' ', [133] = ' ', [134] = ' ', + [135] = ' ', [136] = ' ', [137] = ' ', [138] = ' ', [139] = ' ', [140] = ' ', [141] = ' ', [142] = ' ', + [143] = ' ', [144] = ' ', [145] = ' ', [146] = ' ', [147] = ' ', [148] = ' ', [149] = ' ', [150] = ' ', + [151] = ' ', [152] = ' ', [153] = ' ', [154] = ' ', [155] = ' ', [156] = ' ', [157] = ' ', [158] = ' ', + [159] = ' ', [160] = ' ', [161] = ' ', [162] = ' ', [163] = ' ', [164] = ' ', [165] = ' ', [166] = ' ', + [167] = ' ', [168] = ' ', [169] = ' ', [170] = ' ', [171] = ' ', [172] = ' ', [173] = ' ', [174] = ' ', + [175] = ' ', [176] = ' ', [177] = ' ', [178] = ' ', [179] = ' ', [180] = ' ', [181] = ' ', [182] = ' ', + [183] = ' ', [184] = ' ', [185] = ' ', [186] = ' ', [187] = ' ', [188] = ' ', [189] = ' ', [190] = ' ', + [191] = ' ', [192] = ' ', [193] = ' ', [194] = ' ', [195] = ' ', [196] = ' ', [197] = ' ', [198] = ' ', + [199] = ' ', [200] = ' ', [201] = ' ', [202] = ' ', [203] = ' ', [204] = ' ', [205] = ' ', [206] = ' ', + [207] = ' ', [208] = ' ', [209] = ' ', [210] = ' ', [211] = ' ', [212] = ' ', [213] = ' ', [214] = ' ', + [215] = ' ', [216] = ' ', [217] = ' ', [218] = ' ', [219] = ' ', [220] = ' ', [221] = ' ', [222] = ' ', + [223] = ' ', [224] = ' ', [225] = ' ', [226] = ' ', [227] = ' ', [228] = ' ', [229] = ' ', [230] = ' ', + [231] = ' ', [232] = ' ', [233] = ' ', [234] = ' ', [235] = ' ', [236] = ' ', [237] = ' ', [238] = ' ', + [239] = ' ', [240] = ' ', [241] = ' ', [242] = ' ', [243] = ' ', [244] = ' ', [245] = ' ', [246] = ' ', + [247] = ' ', [248] = ' ', [249] = ' ', [250] = ' ', [251] = ' ', [252] = ' ', [253] = ' ', [254] = ' ', + [255] = ' ' +}; + +size_t rrd_functions_sanitize(char *dst, const char *src, size_t dst_len) { + return text_sanitize((unsigned char *)dst, (const unsigned char *)src, dst_len, + functions_allowed_chars, true, "", NULL); +} + diff --git a/src/libnetdata/sanitizers/sanitizers-functions.h b/src/libnetdata/sanitizers/sanitizers-functions.h new file mode 100644 index 000000000..f4c934040 --- /dev/null +++ b/src/libnetdata/sanitizers/sanitizers-functions.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SANITIZERS_FUNCTIONS_H +#define NETDATA_SANITIZERS_FUNCTIONS_H + +#include "../libnetdata.h" + +size_t rrd_functions_sanitize(char *dst, const char *src, size_t dst_len); + +#endif //NETDATA_SANITIZERS_FUNCTIONS_H diff --git a/src/libnetdata/sanitizers/sanitizers-labels.c b/src/libnetdata/sanitizers/sanitizers-labels.c new file mode 100644 index 000000000..714897a88 --- /dev/null +++ b/src/libnetdata/sanitizers/sanitizers-labels.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "sanitizers-labels.h" + +/* + * All labels follow these rules: + * + * Character Symbol Names Values + * UTF-8 characters UTF-8 -> _ yes + * Lower case letter [a-z] yes yes + * Upper case letter [A-Z] yes yes + * Digit [0-9] yes yes + * Underscore _ yes yes + * Minus - yes yes + * Plus + -> _ yes + * Colon : -> _ yes + * Semicolon ; -> _ -> : + * Equal = -> _ -> : + * Period . yes yes + * Comma , -> . -> . + * Slash / yes yes + * Backslash \ -> / -> / + * At @ -> _ yes + * Space -> _ yes + * Opening parenthesis ( -> _ yes + * Closing parenthesis ) -> _ yes + * anything else -> _ -> space +* + * The above rules should allow users to set in tags (indicative): + * + * 1. hostnames and domain names as-is + * 2. email addresses as-is + * 3. floating point numbers, converted to always use a dot as the decimal point + * + * Leading and trailing spaces and control characters are removed from both label + * names and values. + * + * Multiple spaces inside the label name or the value are removed (only 1 is retained). + * In names spaces are also converted to underscores. + * + * Names that are only underscores are rejected (they do not enter the dictionary). + * + * The above rules do not require any conversion to be included in JSON strings. + * + * Label names and values are truncated to LABELS_MAX_LENGTH (200) characters. + * + * When parsing, label key and value are separated by the first colon (:) found. + * So label:value1:value2 is parsed as key = "label", value = "value1:value2" + * + * This means a label key cannot contain a colon (:) - it is converted to + * underscore if it does. + * + */ + +static unsigned char prometheus_label_names_char_map[256]; +static unsigned char label_names_char_map[256]; +static unsigned char label_values_char_map[256] = { + [0] = '\0', [1] = ' ', [2] = ' ', [3] = ' ', [4] = ' ', [5] = ' ', [6] = ' ', [7] = ' ', [8] = ' ', + + // control characters to be treated as spaces + ['\t'] = ' ', ['\n'] = ' ', ['\v'] = ' ', ['\f'] = ' ', ['\r'] = ' ', + + [14] = ' ', [15] = ' ', [16] = ' ', [17] = ' ', [18] = ' ', [19] = ' ', [20] = ' ', [21] = ' ', + [22] = ' ', [23] = ' ', [24] = ' ', [25] = ' ', [26] = ' ', [27] = ' ', [28] = ' ', [29] = ' ', + [30] = ' ', [31] = ' ', + + // symbols + [' '] = ' ', ['!'] = '_', ['"'] = '_', ['#'] = '_', ['$'] = '_', ['%'] = '_', ['&'] = '_', ['\''] = '_', + ['('] = '(', [')'] = ')', ['*'] = '_', ['+'] = '+', [','] = '.', ['-'] = '-', ['.'] = '.', ['/'] = '/', + + // numbers + ['0'] = '0', ['1'] = '1', ['2'] = '2', ['3'] = '3', ['4'] = '4', ['5'] = '5', ['6'] = '6', ['7'] = '7', + ['8'] = '8', ['9'] = '9', + + // symbols + [':'] = ':', [';'] = ':', ['<'] = '_', ['='] = ':', ['>'] = '_', ['?'] = '_', ['@'] = '@', + + // capitals + ['A'] = 'A', ['B'] = 'B', ['C'] = 'C', ['D'] = 'D', ['E'] = 'E', ['F'] = 'F', ['G'] = 'G', ['H'] = 'H', + ['I'] = 'I', ['J'] = 'J', ['K'] = 'K', ['L'] = 'L', ['M'] = 'M', ['N'] = 'N', ['O'] = 'O', ['P'] = 'P', + ['Q'] = 'Q', ['R'] = 'R', ['S'] = 'S', ['T'] = 'T', ['U'] = 'U', ['V'] = 'V', ['W'] = 'W', ['X'] = 'X', + ['Y'] = 'Y', ['Z'] = 'Z', + + // symbols + ['['] = '[', ['\\'] = '/', [']'] = ']', ['^'] = '_', ['_'] = '_', ['`'] = '_', + + // lower + ['a'] = 'a', ['b'] = 'b', ['c'] = 'c', ['d'] = 'd', ['e'] = 'e', ['f'] = 'f', ['g'] = 'g', ['h'] = 'h', + ['i'] = 'i', ['j'] = 'j', ['k'] = 'k', ['l'] = 'l', ['m'] = 'm', ['n'] = 'n', ['o'] = 'o', ['p'] = 'p', + ['q'] = 'q', ['r'] = 'r', ['s'] = 's', ['t'] = 't', ['u'] = 'u', ['v'] = 'v', ['w'] = 'w', ['x'] = 'x', + ['y'] = 'y', ['z'] = 'z', + + // symbols + ['{'] = '_', ['|'] = '_', ['}'] = '_', ['~'] = '_', + + // rest + [127] = ' ', [128] = ' ', [129] = ' ', [130] = ' ', [131] = ' ', [132] = ' ', [133] = ' ', [134] = ' ', + [135] = ' ', [136] = ' ', [137] = ' ', [138] = ' ', [139] = ' ', [140] = ' ', [141] = ' ', [142] = ' ', + [143] = ' ', [144] = ' ', [145] = ' ', [146] = ' ', [147] = ' ', [148] = ' ', [149] = ' ', [150] = ' ', + [151] = ' ', [152] = ' ', [153] = ' ', [154] = ' ', [155] = ' ', [156] = ' ', [157] = ' ', [158] = ' ', + [159] = ' ', [160] = ' ', [161] = ' ', [162] = ' ', [163] = ' ', [164] = ' ', [165] = ' ', [166] = ' ', + [167] = ' ', [168] = ' ', [169] = ' ', [170] = ' ', [171] = ' ', [172] = ' ', [173] = ' ', [174] = ' ', + [175] = ' ', [176] = ' ', [177] = ' ', [178] = ' ', [179] = ' ', [180] = ' ', [181] = ' ', [182] = ' ', + [183] = ' ', [184] = ' ', [185] = ' ', [186] = ' ', [187] = ' ', [188] = ' ', [189] = ' ', [190] = ' ', + [191] = ' ', [192] = ' ', [193] = ' ', [194] = ' ', [195] = ' ', [196] = ' ', [197] = ' ', [198] = ' ', + [199] = ' ', [200] = ' ', [201] = ' ', [202] = ' ', [203] = ' ', [204] = ' ', [205] = ' ', [206] = ' ', + [207] = ' ', [208] = ' ', [209] = ' ', [210] = ' ', [211] = ' ', [212] = ' ', [213] = ' ', [214] = ' ', + [215] = ' ', [216] = ' ', [217] = ' ', [218] = ' ', [219] = ' ', [220] = ' ', [221] = ' ', [222] = ' ', + [223] = ' ', [224] = ' ', [225] = ' ', [226] = ' ', [227] = ' ', [228] = ' ', [229] = ' ', [230] = ' ', + [231] = ' ', [232] = ' ', [233] = ' ', [234] = ' ', [235] = ' ', [236] = ' ', [237] = ' ', [238] = ' ', + [239] = ' ', [240] = ' ', [241] = ' ', [242] = ' ', [243] = ' ', [244] = ' ', [245] = ' ', [246] = ' ', + [247] = ' ', [248] = ' ', [249] = ' ', [250] = ' ', [251] = ' ', [252] = ' ', [253] = ' ', [254] = ' ', + [255] = ' ' +}; + +__attribute__((constructor)) void initialize_labels_keys_char_map(void) { + // copy the values char map to the names char map + size_t i; + for(i = 0; i < 256 ;i++) + label_names_char_map[i] = label_values_char_map[i]; + + // apply overrides to the label names map + label_names_char_map['='] = '_'; + label_names_char_map[':'] = '_'; + label_names_char_map['+'] = '_'; + label_names_char_map[';'] = '_'; + label_names_char_map['@'] = '_'; + label_names_char_map['('] = '_'; + label_names_char_map[')'] = '_'; + label_names_char_map['\\'] = '/'; + + // prometheus label names + for(i = 0; i < 256 ;i++) prometheus_label_names_char_map[i] = '_'; + for(int s = 'A' ; s <= 'Z' ; s++) prometheus_label_names_char_map[s] = s; + for(int s = 'a' ; s <= 'z' ; s++) prometheus_label_names_char_map[s] = s; + for(int s = '0' ; s <= '9' ; s++) prometheus_label_names_char_map[s] = s; + prometheus_label_names_char_map[0] = '\0'; + prometheus_label_names_char_map[':'] = ':'; + prometheus_label_names_char_map['_'] = '_'; +} + +size_t rrdlabels_sanitize_name(char *dst, const char *src, size_t dst_size) { + size_t rc = text_sanitize((unsigned char *)dst, (const unsigned char *)src, dst_size, label_names_char_map, 0, "", NULL); + + for(size_t i = 0; i < rc ; i++) + if(dst[i] == ' ') dst[i] = '_'; + + return rc; +} + +size_t rrdlabels_sanitize_value(char *dst, const char *src, size_t dst_size) { + return text_sanitize((unsigned char *)dst, (const unsigned char *)src, dst_size, label_values_char_map, 1, "[none]", NULL); +} + +size_t prometheus_rrdlabels_sanitize_name(char *dst, const char *src, size_t dst_size) { + return text_sanitize((unsigned char *)dst, (const unsigned char *)src, dst_size, prometheus_label_names_char_map, 0, "", NULL); +} diff --git a/src/libnetdata/sanitizers/sanitizers-labels.h b/src/libnetdata/sanitizers/sanitizers-labels.h new file mode 100644 index 000000000..39fd6a67a --- /dev/null +++ b/src/libnetdata/sanitizers/sanitizers-labels.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SANITIZERS_LABELS_H +#define NETDATA_SANITIZERS_LABELS_H + +#include "../libnetdata.h" + +size_t rrdlabels_sanitize_name(char *dst, const char *src, size_t dst_size); +size_t rrdlabels_sanitize_value(char *dst, const char *src, size_t dst_size); + +size_t prometheus_rrdlabels_sanitize_name(char *dst, const char *src, size_t dst_size); + +#endif //NETDATA_SANITIZERS_LABELS_H diff --git a/src/libnetdata/sanitizers/sanitizers-pluginsd.c b/src/libnetdata/sanitizers/sanitizers-pluginsd.c new file mode 100644 index 000000000..2659cffee --- /dev/null +++ b/src/libnetdata/sanitizers/sanitizers-pluginsd.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "sanitizers-pluginsd.h" + +/* + * Undefined and control characters become underscores + * ! -> _ + * " -> _ + * ' -> _ + * ` -> _ + * \ -> / + * = -> _ + * | -> _ + */ + +static unsigned char external_plugins_map[256] = { + [0] = '\0', [1] = ' ', [2] = ' ', [3] = ' ', [4] = ' ', [5] = ' ', [6] = ' ', [7] = ' ', [8] = ' ', + + // control characters to be treated as spaces + ['\t'] = ' ', ['\n'] = ' ', ['\v'] = ' ', ['\f'] = ' ', ['\r'] = ' ', + + [14] = ' ', [15] = ' ', [16] = ' ', [17] = ' ', [18] = ' ', [19] = ' ', [20] = ' ', [21] = ' ', + [22] = ' ', [23] = ' ', [24] = ' ', [25] = ' ', [26] = ' ', [27] = ' ', [28] = ' ', [29] = ' ', + [30] = ' ', [31] = ' ', + + // symbols + [' '] = ' ', ['!'] = '_', ['"'] = '_', ['#'] = '#', ['$'] = '$', ['%'] = '%', ['&'] = '&', ['\''] = '_', + ['('] = '(', [')'] = ')', ['*'] = '*', ['+'] = '+', [','] = ',', ['-'] = '-', ['.'] = '.', ['/'] = '/', + + // numbers + ['0'] = '0', ['1'] = '1', ['2'] = '2', ['3'] = '3', ['4'] = '4', ['5'] = '5', ['6'] = '6', ['7'] = '7', + ['8'] = '8', ['9'] = '9', + + // symbols + [':'] = ':', [';'] = ';', ['<'] = '<', ['='] = '_', ['>'] = '>', ['?'] = '?', ['@'] = '@', + + // capitals + ['A'] = 'A', ['B'] = 'B', ['C'] = 'C', ['D'] = 'D', ['E'] = 'E', ['F'] = 'F', ['G'] = 'G', ['H'] = 'H', + ['I'] = 'I', ['J'] = 'J', ['K'] = 'K', ['L'] = 'L', ['M'] = 'M', ['N'] = 'N', ['O'] = 'O', ['P'] = 'P', + ['Q'] = 'Q', ['R'] = 'R', ['S'] = 'S', ['T'] = 'T', ['U'] = 'U', ['V'] = 'V', ['W'] = 'W', ['X'] = 'X', + ['Y'] = 'Y', ['Z'] = 'Z', + + // symbols + ['['] = '[', ['\\'] = '/', [']'] = ']', ['^'] = '^', ['_'] = '_', ['`'] = '_', + + // lower + ['a'] = 'a', ['b'] = 'b', ['c'] = 'c', ['d'] = 'd', ['e'] = 'e', ['f'] = 'f', ['g'] = 'g', ['h'] = 'h', + ['i'] = 'i', ['j'] = 'j', ['k'] = 'k', ['l'] = 'l', ['m'] = 'm', ['n'] = 'n', ['o'] = 'o', ['p'] = 'p', + ['q'] = 'q', ['r'] = 'r', ['s'] = 's', ['t'] = 't', ['u'] = 'u', ['v'] = 'v', ['w'] = 'w', ['x'] = 'x', + ['y'] = 'y', ['z'] = 'z', + + // symbols + ['{'] = '{', ['|'] = '_', ['}'] = '}', ['~'] = '~', + + // rest + [127] = ' ', [128] = ' ', [129] = ' ', [130] = ' ', [131] = ' ', [132] = ' ', [133] = ' ', [134] = ' ', + [135] = ' ', [136] = ' ', [137] = ' ', [138] = ' ', [139] = ' ', [140] = ' ', [141] = ' ', [142] = ' ', + [143] = ' ', [144] = ' ', [145] = ' ', [146] = ' ', [147] = ' ', [148] = ' ', [149] = ' ', [150] = ' ', + [151] = ' ', [152] = ' ', [153] = ' ', [154] = ' ', [155] = ' ', [156] = ' ', [157] = ' ', [158] = ' ', + [159] = ' ', [160] = ' ', [161] = ' ', [162] = ' ', [163] = ' ', [164] = ' ', [165] = ' ', [166] = ' ', + [167] = ' ', [168] = ' ', [169] = ' ', [170] = ' ', [171] = ' ', [172] = ' ', [173] = ' ', [174] = ' ', + [175] = ' ', [176] = ' ', [177] = ' ', [178] = ' ', [179] = ' ', [180] = ' ', [181] = ' ', [182] = ' ', + [183] = ' ', [184] = ' ', [185] = ' ', [186] = ' ', [187] = ' ', [188] = ' ', [189] = ' ', [190] = ' ', + [191] = ' ', [192] = ' ', [193] = ' ', [194] = ' ', [195] = ' ', [196] = ' ', [197] = ' ', [198] = ' ', + [199] = ' ', [200] = ' ', [201] = ' ', [202] = ' ', [203] = ' ', [204] = ' ', [205] = ' ', [206] = ' ', + [207] = ' ', [208] = ' ', [209] = ' ', [210] = ' ', [211] = ' ', [212] = ' ', [213] = ' ', [214] = ' ', + [215] = ' ', [216] = ' ', [217] = ' ', [218] = ' ', [219] = ' ', [220] = ' ', [221] = ' ', [222] = ' ', + [223] = ' ', [224] = ' ', [225] = ' ', [226] = ' ', [227] = ' ', [228] = ' ', [229] = ' ', [230] = ' ', + [231] = ' ', [232] = ' ', [233] = ' ', [234] = ' ', [235] = ' ', [236] = ' ', [237] = ' ', [238] = ' ', + [239] = ' ', [240] = ' ', [241] = ' ', [242] = ' ', [243] = ' ', [244] = ' ', [245] = ' ', [246] = ' ', + [247] = ' ', [248] = ' ', [249] = ' ', [250] = ' ', [251] = ' ', [252] = ' ', [253] = ' ', [254] = ' ', + [255] = ' ' +}; + +size_t external_plugins_sanitize(char *dst, const char *src, size_t dst_len) { + return text_sanitize((unsigned char *)dst, (const unsigned char *)src, dst_len, + external_plugins_map, true, "", NULL); +} + diff --git a/src/libnetdata/sanitizers/sanitizers-pluginsd.h b/src/libnetdata/sanitizers/sanitizers-pluginsd.h new file mode 100644 index 000000000..1779a1451 --- /dev/null +++ b/src/libnetdata/sanitizers/sanitizers-pluginsd.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SANITIZERS_PLUGINSD_H +#define NETDATA_SANITIZERS_PLUGINSD_H + +#include "../libnetdata.h" + +size_t external_plugins_sanitize(char *dst, const char *src, size_t dst_len); + +#endif //NETDATA_SANITIZERS_PLUGINSD_H diff --git a/src/libnetdata/sanitizers/sanitizers.h b/src/libnetdata/sanitizers/sanitizers.h new file mode 100644 index 000000000..d76b18f7d --- /dev/null +++ b/src/libnetdata/sanitizers/sanitizers.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SANITIZERS_H +#define NETDATA_SANITIZERS_H + +#include "utf8-sanitizer.h" +#include "sanitizers-labels.h" +#include "sanitizers-functions.h" +#include "sanitizers-pluginsd.h" +#include "chart_id_and_name.h" + +#endif //NETDATA_SANITIZERS_H diff --git a/src/libnetdata/sanitizers/utf8-sanitizer.c b/src/libnetdata/sanitizers/utf8-sanitizer.c new file mode 100644 index 000000000..e10d88f41 --- /dev/null +++ b/src/libnetdata/sanitizers/utf8-sanitizer.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, const unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length) { + if(unlikely(!dst || !dst_size)) return 0; + + // skip leading spaces and invalid characters + while(src && *src && !IS_UTF8_BYTE(*src) && (isspace(*src) || iscntrl(*src) || !isprint(*src))) + src++; + + if(unlikely(!src || !*src)) { + strncpyz((char *)dst, empty, dst_size); + dst[dst_size - 1] = '\0'; + size_t len = strlen((char *)dst); + if(multibyte_length) *multibyte_length = len; + return len; + } + + unsigned char *d = dst; + + // make room for the final string termination + unsigned char *end = &dst[dst_size - 1]; + + // copy while converting, but keep only one space + // we start wil last_is_space = 1 to skip leading spaces + int last_is_space = 1; + + size_t mblen = 0; + + while(*src && d < end) { + unsigned char c = *src; + + if(IS_UTF8_STARTBYTE(c) && IS_UTF8_BYTE(src[1]) && d + 2 <= end) { + // UTF-8 multi-byte encoded character + + // find how big this character is (2-4 bytes) + size_t utf_character_size = 2; + while(utf_character_size < 4 && + d + utf_character_size <= end && + IS_UTF8_BYTE(src[utf_character_size]) && + !IS_UTF8_STARTBYTE(src[utf_character_size])) + utf_character_size++; + + if(utf) { + while(utf_character_size) { + utf_character_size--; + *d++ = *src++; + } + } + else { + // UTF-8 characters are not allowed. + // Assume it is an underscore + // and skip all except the first byte + *d++ = '_'; + src += (utf_character_size - 1); + } + + last_is_space = 0; + mblen++; + continue; + } + + c = char_map[c]; + if(c == ' ') { + // a space character + + if(!last_is_space) { + // add one space + *d++ = c; + mblen++; + } + + last_is_space++; + } + else { + *d++ = c; + last_is_space = 0; + mblen++; + } + + src++; + } + + // remove trailing spaces + while(d > dst && !IS_UTF8_BYTE(*(d - 1)) && *(d - 1) == ' ') { + d--; + mblen--; + } + + // put a termination at the end of what we copied + *d = '\0'; + + // check if dst is all underscores and empty it if it is + if(*dst == '_') { + unsigned char *t = dst; + while (*t == '_') t++; + if (unlikely(*t == '\0')) { + *dst = '\0'; + mblen = 0; + } + } + + // check if it is empty + if(unlikely(*dst == '\0')) { + strncpyz((char *)dst, empty, dst_size); + dst[dst_size - 1] = '\0'; + mblen = strlen((char *)dst); + if(multibyte_length) *multibyte_length = mblen; + return mblen; + } + + if(multibyte_length) *multibyte_length = mblen; + + return d - dst; +} diff --git a/src/libnetdata/sanitizers/utf8-sanitizer.h b/src/libnetdata/sanitizers/utf8-sanitizer.h new file mode 100644 index 000000000..8b5f73a7f --- /dev/null +++ b/src/libnetdata/sanitizers/utf8-sanitizer.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_UTF8_SANITIZER_H +#define NETDATA_UTF8_SANITIZER_H + +#include "../libnetdata.h" + +size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, const unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length); + +#endif //NETDATA_UTF8_SANITIZER_H diff --git a/src/libnetdata/simple_hashtable.h b/src/libnetdata/simple_hashtable/simple_hashtable.h index 13cdcd10e..fe88d23f8 100644 --- a/src/libnetdata/simple_hashtable.h +++ b/src/libnetdata/simple_hashtable/simple_hashtable.h @@ -507,7 +507,7 @@ static inline SIMPLE_HASHTABLE_SLOT_NAMED *simple_hashtable_next_read_only_named #ifndef XXH_INLINE_ALL #define XXH_INLINE_ALL #endif -#include "xxhash.h" +#include "../xxHash/xxhash.h" #define simple_hashtable_set_named CONCAT(simple_hashtable_set, SIMPLE_HASHTABLE_NAME) #define simple_hashtable_get_named CONCAT(simple_hashtable_get, SIMPLE_HASHTABLE_NAME) @@ -520,13 +520,13 @@ static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_set_named(SIMPLE_HAS return SIMPLE_HASHTABLE_SLOT_DATA(sl); } -static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_get_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) { +static inline SIMPLE_HASHTABLE_VALUE_TYPE *simple_hashtable_get_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len) { XXH64_hash_t hash = XXH3_64bits((void *)key, key_len); SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, key, true); return SIMPLE_HASHTABLE_SLOT_DATA(sl); } -static inline bool simple_hashtable_del_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len, SIMPLE_HASHTABLE_VALUE_TYPE *data) { +static inline bool simple_hashtable_del_named(SIMPLE_HASHTABLE_NAMED *ht, SIMPLE_HASHTABLE_KEY_TYPE *key, size_t key_len) { XXH64_hash_t hash = XXH3_64bits((void *)key, key_len); SIMPLE_HASHTABLE_SLOT_NAMED *sl = simple_hashtable_get_slot_named(ht, hash, key, true); return simple_hashtable_del_slot_named(ht, sl); diff --git a/src/libnetdata/simple_hashtable_undef.h b/src/libnetdata/simple_hashtable/simple_hashtable_undef.h index 3fe5a708d..3fe5a708d 100644 --- a/src/libnetdata/simple_hashtable_undef.h +++ b/src/libnetdata/simple_hashtable/simple_hashtable_undef.h diff --git a/src/libnetdata/simple_pattern/README.md b/src/libnetdata/simple_pattern/README.md index cf8a0f640..93a6f2d8a 100644 --- a/src/libnetdata/simple_pattern/README.md +++ b/src/libnetdata/simple_pattern/README.md @@ -1,13 +1,3 @@ -<!-- -title: "Simple patterns" -description: "Netdata supports simple patterns, which are less cryptic versions of regular expressions. Use familiar notation for powerful results." -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/simple_pattern/README.md -sidebar_label: "Simple patterns" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Simple patterns Unix prefers regular expressions. But they are just too hard, too cryptic diff --git a/src/libnetdata/simple_pattern/simple_pattern.c b/src/libnetdata/simple_pattern/simple_pattern.c index 7a7f41b1c..d0feefb4d 100644 --- a/src/libnetdata/simple_pattern/simple_pattern.c +++ b/src/libnetdata/simple_pattern/simple_pattern.c @@ -78,18 +78,21 @@ SIMPLE_PATTERN *simple_pattern_create(const char *list, const char *separators, if(unlikely(!list || !*list)) return root; - char isseparator[256] = { - [' '] = 1 // space - , ['\t'] = 1 // tab - , ['\r'] = 1 // carriage return - , ['\n'] = 1 // new line - , ['\f'] = 1 // form feed - , ['\v'] = 1 // vertical tab + bool isseparator[256] = { + [' '] = true // space + , ['\t'] = true // tab + , ['\r'] = true // carriage return + , ['\n'] = true // new line + , ['\f'] = true // form feed + , ['\v'] = true // vertical tab }; - if (unlikely(separators && *separators)) { - memset(&isseparator[0], 0, sizeof(isseparator)); - while(*separators) isseparator[(unsigned char)*separators++] = 1; + if (unlikely(separators == SIMPLE_PATTERN_NO_SEPARATORS)) + memset(isseparator, false, sizeof(isseparator)); + + else if (unlikely(separators && *separators)) { + memset(isseparator, false, sizeof(isseparator)); + while(*separators) isseparator[(unsigned char)*separators++] = true; } char *buf = mallocz(strlen(list) + 1); diff --git a/src/libnetdata/simple_pattern/simple_pattern.h b/src/libnetdata/simple_pattern/simple_pattern.h index 1af0f87b9..2c105c54b 100644 --- a/src/libnetdata/simple_pattern/simple_pattern.h +++ b/src/libnetdata/simple_pattern/simple_pattern.h @@ -21,6 +21,8 @@ typedef enum __attribute__ ((__packed__)) { struct simple_pattern; typedef struct simple_pattern SIMPLE_PATTERN; +#define SIMPLE_PATTERN_NO_SEPARATORS (const char *)(0xFFFFFFFF) + // create a simple_pattern from the string given // default_mode is used in cases where EXACT matches, without an asterisk, // should be considered PREFIX matches. diff --git a/src/libnetdata/socket/security.c b/src/libnetdata/socket/security.c index 502998b79..33bf22d75 100644 --- a/src/libnetdata/socket/security.c +++ b/src/libnetdata/socket/security.c @@ -1,7 +1,5 @@ #include "../libnetdata.h" -#ifdef ENABLE_HTTPS - SSL_CTX *netdata_ssl_exporting_ctx =NULL; SSL_CTX *netdata_ssl_streaming_sender_ctx =NULL; SSL_CTX *netdata_ssl_web_server_ctx =NULL; @@ -732,7 +730,7 @@ int security_test_certificate(SSL *ssl) { * * @return It returns 0 on success and -1 otherwise. */ -int ssl_security_location_for_context(SSL_CTX *ctx, char *file, char *path) { +int ssl_security_location_for_context(SSL_CTX *ctx, const char *file, const char *path) { int load_custom = 1, load_default = 1; if (file || path) { if(!SSL_CTX_load_verify_locations(ctx, file, path)) { @@ -751,4 +749,3 @@ int ssl_security_location_for_context(SSL_CTX *ctx, char *file, char *path) { return 0; } -#endif diff --git a/src/libnetdata/socket/security.h b/src/libnetdata/socket/security.h index 283d81db8..7deb1d797 100644 --- a/src/libnetdata/socket/security.h +++ b/src/libnetdata/socket/security.h @@ -1,5 +1,5 @@ #ifndef NETDATA_SECURITY_H -# define NETDATA_SECURITY_H +#define NETDATA_SECURITY_H typedef enum __attribute__((packed)) { NETDATA_SSL_STATE_NOT_SSL = 1, // This connection is not SSL @@ -12,27 +12,6 @@ typedef enum __attribute__((packed)) { #define NETDATA_SSL_STREAMING_SENDER_CTX 1 #define NETDATA_SSL_EXPORTING_CTX 2 -# ifdef ENABLE_HTTPS - -#define OPENSSL_VERSION_095 0x00905100L -#define OPENSSL_VERSION_097 0x0907000L -#define OPENSSL_VERSION_110 0x10100000L -#define OPENSSL_VERSION_111 0x10101000L -#define OPENSSL_VERSION_300 0x30000000L - -# include <openssl/ssl.h> -# include <openssl/err.h> -# include <openssl/evp.h> -# include <openssl/pem.h> -# if (SSLEAY_VERSION_NUMBER >= OPENSSL_VERSION_097) && (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110) -# include <openssl/conf.h> -# endif - -#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_300 -#include <openssl/core_names.h> -#include <openssl/decoder.h> -#endif - typedef struct netdata_ssl { SSL *conn; // SSL connection NETDATA_SSL_STATE state; // The state for SSL connection @@ -52,7 +31,7 @@ extern const char *tls_version; extern const char *tls_ciphers; extern bool netdata_ssl_validate_certificate; extern bool netdata_ssl_validate_certificate_sender; -int ssl_security_location_for_context(SSL_CTX *ctx,char *file,char *path); +int ssl_security_location_for_context(SSL_CTX *ctx, const char *file, const char *path); void netdata_ssl_initialize_openssl(); void netdata_ssl_cleanup(); @@ -73,5 +52,4 @@ ssize_t netdata_ssl_write(NETDATA_SSL *ssl, const void *buf, size_t num); ssize_t netdata_ssl_pending(NETDATA_SSL *ssl); bool netdata_ssl_has_pending(NETDATA_SSL *ssl); -# endif //ENABLE_HTTPS #endif //NETDATA_SECURITY_H diff --git a/src/libnetdata/socket/socket.c b/src/libnetdata/socket/socket.c index f907fefeb..3b0d1f824 100644 --- a/src/libnetdata/socket/socket.c +++ b/src/libnetdata/socket/socket.c @@ -119,22 +119,17 @@ bool fd_is_socket(int fd) { return true; } -bool sock_has_output_error(int fd) { - if(fd < 0) { - //internal_error(true, "invalid socket %d", fd); - return false; - } +#ifdef POLLRDHUP +bool is_socket_closed(int fd) { + if(fd < 0) + return true; // if(!fd_is_socket(fd)) { // //internal_error(true, "fd %d is not a socket", fd); // return false; // } - short int errors = POLLERR | POLLHUP | POLLNVAL; - -#ifdef POLLRDHUP - errors |= POLLRDHUP; -#endif + short int errors = POLLERR | POLLHUP | POLLNVAL | POLLRDHUP; struct pollfd pfd = { .fd = fd, @@ -149,6 +144,31 @@ bool sock_has_output_error(int fd) { return ((pfd.revents & errors) || !(pfd.revents & POLLOUT)); } +#else +bool is_socket_closed(int fd) { + if(fd < 0) + return true; + + char buffer; + ssize_t result = recv(fd, &buffer, 1, MSG_PEEK | MSG_DONTWAIT); + if (result == 0) { + // Connection closed + return true; + } + else if (result < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + // No data available, but socket is still open + return false; + } else { + // An error occurred + return true; + } + } + + // Data is available, socket is open + return false; +} +#endif int sock_setnonblock(int fd) { int flags; @@ -515,7 +535,6 @@ HTTP_ACL socket_ssl_acl(char *acl) { //Due the format of the SSL command it is always the last command, //we finish it here to avoid problems with the ACLs *ssl = '\0'; -#ifdef ENABLE_HTTPS ssl++; if (!strncmp("SSL=",ssl,4)) { ssl += 4; @@ -526,7 +545,6 @@ HTTP_ACL socket_ssl_acl(char *acl) { return HTTP_ACL_SSL_FORCE; } } -#endif } return HTTP_ACL_NONE; @@ -558,7 +576,7 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, char buffer2[10 + 1]; snprintfz(buffer2, 10, "%d", default_port); - char *ip = buffer, *port = buffer2, *interface = "", *portconfig; + char *ip = buffer, *port = buffer2, *iface = "", *portconfig; int protocol = IPPROTO_TCP, socktype = SOCK_STREAM; const char *protocol_str = "tcp"; @@ -613,7 +631,7 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, if(*e == '%') { *e = '\0'; e++; - interface = e; + iface = e; while(*e && *e != ':' && *e != '=') e++; } @@ -650,13 +668,13 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, } uint32_t scope_id = 0; - if(*interface) { - scope_id = if_nametoindex(interface); + if(*iface) { + scope_id = if_nametoindex(iface); if(!scope_id) nd_log(NDLS_DAEMON, NDLP_ERR, "LISTENER: Cannot find a network interface named '%s'. " "Continuing with limiting the network interface", - interface); + iface); } if(!*ip || *ip == '*' || !strcmp(ip, "any") || !strcmp(ip, "all")) @@ -750,9 +768,9 @@ int listen_sockets_setup(LISTEN_SOCKETS *sockets) { } else sockets->default_port = (uint16_t)new_port; - char *s = appconfig_get(sockets->config, sockets->config_section, "bind to", sockets->default_bind_to); + const char *s = appconfig_get(sockets->config, sockets->config_section, "bind to", sockets->default_bind_to); while(*s) { - char *e = s; + const char *e = s; // skip separators, moving both s(tart) and e(nd) while(isspace((uint8_t)*e) || *e == ',') s = ++e; @@ -935,12 +953,10 @@ int connect_to_this_ip46( hostBfr, servBfr); // Convert 'struct timeval' to milliseconds for poll(): - int timeout_ms = timeout->tv_sec * 1000 + timeout->tv_usec / 1000; + int timeout_ms = timeout ? (timeout->tv_sec * 1000 + timeout->tv_usec / 1000) : 1000; switch(wait_on_socket_or_cancel_with_timeout( -#ifdef ENABLE_HTTPS - NULL, -#endif + NULL, fd, timeout_ms, POLLOUT, NULL)) { case 0: // proceed nd_log(NDLS_DAEMON, NDLP_DEBUG, @@ -1019,7 +1035,7 @@ int connect_to_this(const char *definition, int default_port, struct timeval *ti char default_service[10 + 1]; snprintfz(default_service, 10, "%d", default_port); - char *host = buffer, *service = default_service, *interface = ""; + char *host = buffer, *service = default_service, *iface = ""; int protocol = IPPROTO_TCP, socktype = SOCK_STREAM; uint32_t scope_id = 0; @@ -1058,7 +1074,7 @@ int connect_to_this(const char *definition, int default_port, struct timeval *ti if(*e == '%') { *e = '\0'; e++; - interface = e; + iface = e; while(*e && *e != ':') e++; } @@ -1076,12 +1092,12 @@ int connect_to_this(const char *definition, int default_port, struct timeval *ti return -1; } - if(*interface) { - scope_id = if_nametoindex(interface); + if(*iface) { + scope_id = if_nametoindex(iface); if(!scope_id) nd_log(NDLS_DAEMON, NDLP_ERR, "Cannot find a network interface named '%s'. Continuing with limiting the network interface", - interface); + iface); } if(!*service) @@ -1186,9 +1202,7 @@ int connect_to_one_of_urls(const char *destination, int default_port, struct tim // returns: -1 = thread cancelled, 0 = proceed to read/write, 1 = time exceeded, 2 = error on fd // timeout parameter can be zero to wait forever inline int wait_on_socket_or_cancel_with_timeout( -#ifdef ENABLE_HTTPS NETDATA_SSL *ssl, -#endif int fd, int timeout_ms, short int poll_events, short int *revents) { struct pollfd pfd = { .fd = fd, @@ -1204,10 +1218,8 @@ inline int wait_on_socket_or_cancel_with_timeout( return -1; } -#ifdef ENABLE_HTTPS if(poll_events == POLLIN && ssl && SSL_connection(ssl) && netdata_ssl_has_pending(ssl)) return 0; -#endif const int wait_ms = (timeout_ms >= ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS || forever) ? ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS : timeout_ms; @@ -1247,16 +1259,10 @@ inline int wait_on_socket_or_cancel_with_timeout( return 1; } -ssize_t recv_timeout( -#ifdef ENABLE_HTTPS - NETDATA_SSL *ssl, -#endif - int sockfd, void *buf, size_t len, int flags, int timeout) { +ssize_t recv_timeout(NETDATA_SSL *ssl, int sockfd, void *buf, size_t len, int flags, int timeout) { switch(wait_on_socket_or_cancel_with_timeout( -#ifdef ENABLE_HTTPS - ssl, -#endif + ssl, sockfd, timeout * 1000, POLLIN, NULL)) { case 0: // data are waiting break; @@ -1270,25 +1276,16 @@ ssize_t recv_timeout( return -1; } -#ifdef ENABLE_HTTPS - if (SSL_connection(ssl)) { + if (SSL_connection(ssl)) return netdata_ssl_read(ssl, buf, len); - } -#endif return recv(sockfd, buf, len, flags); } -ssize_t send_timeout( -#ifdef ENABLE_HTTPS - NETDATA_SSL *ssl, -#endif - int sockfd, void *buf, size_t len, int flags, int timeout) { +ssize_t send_timeout(NETDATA_SSL *ssl, int sockfd, void *buf, size_t len, int flags, int timeout) { switch(wait_on_socket_or_cancel_with_timeout( -#ifdef ENABLE_HTTPS - ssl, -#endif + ssl, sockfd, timeout * 1000, POLLOUT, NULL)) { case 0: // data are waiting break; @@ -1302,7 +1299,6 @@ ssize_t send_timeout( return -1; } -#ifdef ENABLE_HTTPS if(ssl->conn) { if (SSL_connection(ssl)) { return netdata_ssl_write(ssl, buf, len); @@ -1314,7 +1310,7 @@ ssize_t send_timeout( return -1; } } -#endif + return send(sockfd, buf, len, flags); } diff --git a/src/libnetdata/socket/socket.h b/src/libnetdata/socket/socket.h index 8147c9774..2c282c4c6 100644 --- a/src/libnetdata/socket/socket.h +++ b/src/libnetdata/socket/socket.h @@ -46,18 +46,12 @@ int connect_to_one_of(const char *destination, int default_port, struct timeval int connect_to_one_of_urls(const char *destination, int default_port, struct timeval *timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size); -#ifdef ENABLE_HTTPS ssize_t recv_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout); ssize_t send_timeout(NETDATA_SSL *ssl,int sockfd, void *buf, size_t len, int flags, int timeout); int wait_on_socket_or_cancel_with_timeout(NETDATA_SSL *ssl, int fd, int timeout_ms, short int poll_events, short int *revents); -#else -ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout); -ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout); -int wait_on_socket_or_cancel_with_timeout(int fd, int timeout_ms, short int poll_events, short int *revents); -#endif bool fd_is_socket(int fd); -bool sock_has_output_error(int fd); +bool is_socket_closed(int fd); int sock_setnonblock(int fd); int sock_delnonblock(int fd); @@ -200,7 +194,7 @@ void poll_events(LISTEN_SOCKETS *sockets #define INET6_ADDRSTRLEN 46 #endif -typedef struct socket_peers { +typedef struct { struct { char ip[INET6_ADDRSTRLEN]; int port; diff --git a/src/libnetdata/spawn_server/log-forwarder.c b/src/libnetdata/spawn_server/log-forwarder.c new file mode 100644 index 000000000..5c4db55ea --- /dev/null +++ b/src/libnetdata/spawn_server/log-forwarder.c @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../libnetdata.h"
+#include "log-forwarder.h"
+
+typedef struct LOG_FORWARDER_ENTRY {
+ int fd;
+ char *cmd;
+ pid_t pid;
+ BUFFER *wb;
+ size_t pfds_idx;
+ bool delete;
+
+ struct LOG_FORWARDER_ENTRY *prev;
+ struct LOG_FORWARDER_ENTRY *next;
+} LOG_FORWARDER_ENTRY;
+
+typedef struct LOG_FORWARDER {
+ LOG_FORWARDER_ENTRY *entries;
+ ND_THREAD *thread;
+ SPINLOCK spinlock;
+ int pipe_fds[2]; // Pipe for notifications
+ bool running;
+} LOG_FORWARDER;
+
+static void *log_forwarder_thread_func(void *arg);
+
+// --------------------------------------------------------------------------------------------------------------------
+// helper functions
+
+static inline LOG_FORWARDER_ENTRY *log_forwarder_find_entry_unsafe(LOG_FORWARDER *lf, int fd) {
+ for (LOG_FORWARDER_ENTRY *entry = lf->entries; entry; entry = entry->next) {
+ if (entry->fd == fd)
+ return entry;
+ }
+
+ return NULL;
+}
+
+static inline void log_forwarder_del_entry_unsafe(LOG_FORWARDER *lf, LOG_FORWARDER_ENTRY *entry) {
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(lf->entries, entry, prev, next);
+ buffer_free(entry->wb);
+ freez(entry->cmd);
+ close(entry->fd);
+ freez(entry);
+}
+
+static inline void log_forwarder_wake_up_worker(LOG_FORWARDER *lf) {
+ char ch = 0;
+ ssize_t bytes_written = write(lf->pipe_fds[PIPE_WRITE], &ch, 1);
+ if (bytes_written != 1)
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "Failed to write to notification pipe");
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// starting / stopping
+
+LOG_FORWARDER *log_forwarder_start(void) {
+ LOG_FORWARDER *lf = callocz(1, sizeof(LOG_FORWARDER));
+
+ spinlock_init(&lf->spinlock);
+ if (pipe(lf->pipe_fds) != 0) {
+ freez(lf);
+ return NULL;
+ }
+
+ // make sure read() will not block on this pipe
+ sock_setnonblock(lf->pipe_fds[PIPE_READ]);
+
+ lf->running = true;
+ lf->thread = nd_thread_create("log-fw", NETDATA_THREAD_OPTION_JOINABLE, log_forwarder_thread_func, lf);
+
+ return lf;
+}
+
+static inline void mark_all_entries_for_deletion_unsafe(LOG_FORWARDER *lf) {
+ for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ;entry = entry->next)
+ entry->delete = true;
+}
+
+void log_forwarder_stop(LOG_FORWARDER *lf) {
+ if(!lf || !lf->running) return;
+
+ // Signal the thread to stop
+ spinlock_lock(&lf->spinlock);
+ lf->running = false;
+
+ // mark them all for deletion
+ mark_all_entries_for_deletion_unsafe(lf);
+
+ // Send a byte to the pipe to wake up the thread
+ char ch = 0;
+ write(lf->pipe_fds[PIPE_WRITE], &ch, 1);
+ spinlock_unlock(&lf->spinlock);
+
+ // Wait for the thread to finish
+ close(lf->pipe_fds[PIPE_WRITE]); // force it to quit
+ nd_thread_join(lf->thread);
+ close(lf->pipe_fds[PIPE_READ]);
+
+ freez(lf);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// managing entries
+
+void log_forwarder_add_fd(LOG_FORWARDER *lf, int fd) {
+ if(!lf || !lf->running || fd < 0) return;
+
+ LOG_FORWARDER_ENTRY *entry = callocz(1, sizeof(LOG_FORWARDER_ENTRY));
+ entry->fd = fd;
+ entry->cmd = NULL;
+ entry->pid = 0;
+ entry->pfds_idx = 0;
+ entry->delete = false;
+ entry->wb = buffer_create(0, NULL);
+
+ spinlock_lock(&lf->spinlock);
+
+ // Append to the entries list
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(lf->entries, entry, prev, next);
+
+ // Send a byte to the pipe to wake up the thread
+ log_forwarder_wake_up_worker(lf);
+
+ spinlock_unlock(&lf->spinlock);
+}
+
+bool log_forwarder_del_and_close_fd(LOG_FORWARDER *lf, int fd) {
+ if(!lf || !lf->running || fd < 0) return false;
+
+ bool ret = false;
+
+ spinlock_lock(&lf->spinlock);
+
+ LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
+ if(entry) {
+ entry->delete = true;
+
+ // Send a byte to the pipe to wake up the thread
+ log_forwarder_wake_up_worker(lf);
+
+ ret = true;
+ }
+
+ spinlock_unlock(&lf->spinlock);
+
+ return ret;
+}
+
+void log_forwarder_annotate_fd_name(LOG_FORWARDER *lf, int fd, const char *cmd) {
+ if(!lf || !lf->running || fd < 0 || !cmd || !*cmd) return;
+
+ spinlock_lock(&lf->spinlock);
+
+ LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
+ if (entry) {
+ freez(entry->cmd);
+ entry->cmd = strdupz(cmd);
+ }
+
+ spinlock_unlock(&lf->spinlock);
+}
+
+void log_forwarder_annotate_fd_pid(LOG_FORWARDER *lf, int fd, pid_t pid) {
+ if(!lf || !lf->running || fd < 0) return;
+
+ spinlock_lock(&lf->spinlock);
+
+ LOG_FORWARDER_ENTRY *entry = log_forwarder_find_entry_unsafe(lf, fd);
+ if (entry)
+ entry->pid = pid;
+
+ spinlock_unlock(&lf->spinlock);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// log forwarder thread
+
+static inline void log_forwarder_log(LOG_FORWARDER *lf __maybe_unused, LOG_FORWARDER_ENTRY *entry, const char *msg) {
+ const char *s = msg;
+ while(*s && isspace((uint8_t)*s)) s++;
+ if(*s == '\0') return; // do not log empty lines
+
+ ND_LOG_STACK lgs[] = {
+ ND_LOG_FIELD_TXT(NDF_SYSLOG_IDENTIFIER, entry->cmd ? entry->cmd : "unknown"),
+ ND_LOG_FIELD_I64(NDF_TID, entry->pid),
+ ND_LOG_FIELD_END(),
+ };
+ ND_LOG_STACK_PUSH(lgs);
+
+ nd_log(NDLS_COLLECTORS, NDLP_WARNING, "STDERR: %s", msg);
+}
+
+// returns the number of entries active
+static inline size_t log_forwarder_remove_deleted_unsafe(LOG_FORWARDER *lf) {
+ size_t entries = 0;
+
+ LOG_FORWARDER_ENTRY *entry = lf->entries;
+ while(entry) {
+ LOG_FORWARDER_ENTRY *next = entry->next;
+
+ if(entry->delete) {
+ if (buffer_strlen(entry->wb))
+ // there is something not logged in it - log it
+ log_forwarder_log(lf, entry, buffer_tostring(entry->wb));
+
+ log_forwarder_del_entry_unsafe(lf, entry);
+ }
+ else
+ entries++;
+
+ entry = next;
+ }
+
+ return entries;
+}
+
+static void *log_forwarder_thread_func(void *arg) {
+ LOG_FORWARDER *lf = (LOG_FORWARDER *)arg;
+
+ while (1) {
+ spinlock_lock(&lf->spinlock);
+ if (!lf->running) {
+ mark_all_entries_for_deletion_unsafe(lf);
+ log_forwarder_remove_deleted_unsafe(lf);
+ spinlock_unlock(&lf->spinlock);
+ break;
+ }
+
+ // Count the number of fds
+ size_t nfds = 1 + log_forwarder_remove_deleted_unsafe(lf);
+
+ struct pollfd pfds[nfds];
+
+ // First, the notification pipe
+ pfds[0].fd = lf->pipe_fds[PIPE_READ];
+ pfds[0].events = POLLIN;
+
+ int idx = 1;
+ for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ; entry = entry->next, idx++) {
+ pfds[idx].fd = entry->fd;
+ pfds[idx].events = POLLIN;
+ entry->pfds_idx = idx;
+ }
+
+ spinlock_unlock(&lf->spinlock);
+
+ int timeout = 200; // 200ms
+ int ret = poll(pfds, nfds, timeout);
+
+ if (ret > 0) {
+ // Check the notification pipe
+ if (pfds[0].revents & POLLIN) {
+ // Read and discard the data
+ char buf[256];
+ ssize_t bytes_read = read(lf->pipe_fds[PIPE_READ], buf, sizeof(buf));
+ // Ignore the data; proceed regardless of the result
+ if (bytes_read == -1) {
+ if (errno != EAGAIN && errno != EWOULDBLOCK) {
+ // Handle read error if necessary
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "Failed to read from notification pipe");
+ return NULL;
+ }
+ }
+ }
+
+ // Now check the other fds
+ spinlock_lock(&lf->spinlock);
+
+ size_t to_remove = 0;
+
+ // read or mark them for deletion
+ for(LOG_FORWARDER_ENTRY *entry = lf->entries; entry ; entry = entry->next) {
+ if (entry->pfds_idx < 1 || entry->pfds_idx >= nfds || !(pfds[entry->pfds_idx].revents & POLLIN))
+ continue;
+
+ BUFFER *wb = entry->wb;
+ buffer_need_bytes(wb, 1024);
+
+ ssize_t bytes_read = read(entry->fd, &wb->buffer[wb->len], wb->size - wb->len - 1);
+ if(bytes_read > 0)
+ wb->len += bytes_read;
+ else if(bytes_read == 0 || (bytes_read == -1 && errno != EINTR && errno != EAGAIN)) {
+ // EOF or error
+ entry->delete = true;
+ to_remove++;
+ }
+
+ // log as many lines are they have been received
+ char *start = (char *)buffer_tostring(wb);
+ char *newline = strchr(start, '\n');
+ while(newline) {
+ *newline = '\0';
+ log_forwarder_log(lf, entry, start);
+
+ start = ++newline;
+ newline = strchr(newline, '\n');
+ }
+
+ if(start != wb->buffer) {
+ wb->len = strlen(start);
+ if (wb->len)
+ memmove(wb->buffer, start, wb->len);
+ }
+
+ entry->pfds_idx = 0;
+ }
+
+ spinlock_unlock(&lf->spinlock);
+ }
+ else if (ret == 0) {
+ // Timeout, nothing to do
+ continue;
+
+ }
+ else
+ nd_log(NDLS_COLLECTORS, NDLP_ERR, "Log forwarder: poll() error");
+ }
+
+ return NULL;
+}
diff --git a/src/libnetdata/spawn_server/log-forwarder.h b/src/libnetdata/spawn_server/log-forwarder.h new file mode 100644 index 000000000..344601c1f --- /dev/null +++ b/src/libnetdata/spawn_server/log-forwarder.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_LOG_FORWARDER_H
+#define NETDATA_LOG_FORWARDER_H
+
+#include "../libnetdata.h"
+
+typedef struct LOG_FORWARDER LOG_FORWARDER;
+
+LOG_FORWARDER *log_forwarder_start(void); // done once, at spawn_server_create()
+void log_forwarder_add_fd(LOG_FORWARDER *lf, int fd); // to add a new fd
+void log_forwarder_annotate_fd_name(LOG_FORWARDER *lf, int fd, const char *cmd); // set the syslog identifier
+void log_forwarder_annotate_fd_pid(LOG_FORWARDER *lf, int fd, pid_t pid); // set the pid of the child process
+bool log_forwarder_del_and_close_fd(LOG_FORWARDER *lf, int fd); // to remove an fd
+void log_forwarder_stop(LOG_FORWARDER *lf); // done once, at spawn_server_destroy()
+
+#endif //NETDATA_LOG_FORWARDER_H
diff --git a/src/libnetdata/spawn_server/spawn-tester.c b/src/libnetdata/spawn_server/spawn-tester.c new file mode 100644 index 000000000..fbd9431ac --- /dev/null +++ b/src/libnetdata/spawn_server/spawn-tester.c @@ -0,0 +1,493 @@ +#include "libnetdata/libnetdata.h" +#include "libnetdata/required_dummies.h" + +#define ENV_VAR_KEY "SPAWN_TESTER" +#define ENV_VAR_VALUE "1234567890" + +size_t warnings = 0; + +void child_check_environment(void) { + const char *s = getenv(ENV_VAR_KEY); + if(!s || !*s || strcmp(s, ENV_VAR_VALUE) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Wrong environment. Variable '%s' should have value '%s' but it has '%s'", + ENV_VAR_KEY, ENV_VAR_VALUE, s ? s : "(unset)"); + + exit(1); + } +} + +static bool is_valid_fd(int fd) { + errno_clear(); + return fcntl(fd, F_GETFD) != -1 || errno != EBADF; +} + +void child_check_fds(void) { + for(int fd = 0; fd < 3; fd++) { + if(!is_valid_fd(fd)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "fd No %d should be a valid file descriptor - but it isn't.", fd); + + exit(1); + } + } + + for(int fd = 3; fd < /* os_get_fd_open_max() */ 1024; fd++) { + if(is_valid_fd(fd)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "fd No %d is a valid file descriptor - it shouldn't.", fd); + + exit(1); + } + } + + errno_clear(); +} + +// -------------------------------------------------------------------------------------------------------------------- +// kill to stop + +int plugin_kill_to_stop() { + child_check_fds(); + child_check_environment(); + + char buffer[1024]; + while (fgets(buffer, sizeof(buffer), stdin) != NULL) { + fprintf(stderr, "+"); + printf("%s", buffer); + fflush(stdout); + } + + return 0; +} + +void test_int_fds_plugin_kill_to_stop(SPAWN_SERVER *server, const char *argv0) { + const char *params[] = { + argv0, + "plugin-kill-to-stop", + NULL, + }; + + SPAWN_INSTANCE *si = spawn_server_exec(server, STDERR_FILENO, 0, params, NULL, 0, SPAWN_INSTANCE_TYPE_EXEC); + if(!si) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot run myself as plugin (spawn)"); + exit(1); + } + + const char *msg = "Hello World!\n"; + ssize_t len = strlen(msg); + char buffer[len * 2]; + + for(size_t j = 0; j < 30 ;j++) { + fprintf(stderr, "-"); + memset(buffer, 0, sizeof(buffer)); + + ssize_t rc = write(spawn_server_instance_write_fd(si), msg, len); + + if (rc != len) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot write to plugin. Expected to write %zd bytes, wrote %zd bytes", + len, rc); + exit(1); + } + + rc = read(spawn_server_instance_read_fd(si), buffer, sizeof(buffer)); + if (rc != len) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot read from plugin. Expected to read %zd bytes, read %zd bytes", + len, rc); + exit(1); + } + + if (memcmp(msg, buffer, len) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Read corrupted data. Expected '%s', Read '%s'", + msg, buffer); + exit(1); + } + } + fprintf(stderr, "\n"); + + int code = spawn_server_exec_kill(server, si); + + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "child exited with code %d", + code); + + if(code != 15 && code != 0) { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "child should exit with code 0 or 15, but exited with code %d", code); + warnings++; + } +} + +void test_popen_plugin_kill_to_stop(const char *argv0) { + char cmd[FILENAME_MAX + 100]; + snprintfz(cmd, sizeof(cmd), "exec %s plugin-kill-to-stop", argv0); + POPEN_INSTANCE *pi = spawn_popen_run(cmd); + if(!pi) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot run myself as plugin (popen)"); + exit(1); + } + + const char *msg = "Hello World!\n"; + size_t len = strlen(msg); + char buffer[len * 2]; + + for(size_t j = 0; j < 30 ;j++) { + fprintf(stderr, "-"); + memset(buffer, 0, sizeof(buffer)); + + size_t rc = fwrite(msg, 1, len, spawn_popen_stdin(pi)); + if (rc != len) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot write to plugin. Expected to write %zu bytes, wrote %zu bytes", + len, rc); + exit(1); + } + fflush(spawn_popen_stdin(pi)); + + char *s = fgets(buffer, sizeof(buffer), spawn_popen_stdout(pi)); + if (!s || strlen(s) != len) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot read from plugin. Expected to read %zu bytes, read %zu bytes", + len, (size_t)(s ? strlen(s) : 0)); + exit(1); + } + if (memcmp(msg, buffer, len) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Read corrupted data. Expected '%s', Read '%s'", + msg, buffer); + exit(1); + } + } + fprintf(stderr, "\n"); + + int code = spawn_popen_kill(pi); + + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "child exited with code %d", + code); + + if(code != 0) { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "child should exit with code 0, but exited with code %d", code); + warnings++; + } +} + +// -------------------------------------------------------------------------------------------------------------------- +// close to stop + +int plugin_close_to_stop() { + child_check_fds(); + child_check_environment(); + + char buffer[1024]; + while (fgets(buffer, sizeof(buffer), stdin) != NULL) { + fprintf(stderr, "+"); + printf("%s", buffer); + fflush(stdout); + } + + nd_log(NDLS_COLLECTORS, NDLP_ERR, "child detected a closed pipe."); + exit(1); +} + +void test_int_fds_plugin_close_to_stop(SPAWN_SERVER *server, const char *argv0) { + const char *params[] = { + argv0, + "plugin-close-to-stop", + NULL, + }; + + SPAWN_INSTANCE *si = spawn_server_exec(server, STDERR_FILENO, 0, params, NULL, 0, SPAWN_INSTANCE_TYPE_EXEC); + if(!si) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot run myself as plugin (spawn)"); + exit(1); + } + + const char *msg = "Hello World!\n"; + ssize_t len = strlen(msg); + char buffer[len * 2]; + + for(size_t j = 0; j < 30 ;j++) { + fprintf(stderr, "-"); + memset(buffer, 0, sizeof(buffer)); + + ssize_t rc = write(spawn_server_instance_write_fd(si), msg, len); + if (rc != len) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot write to plugin. Expected to write %zd bytes, wrote %zd bytes", + len, rc); + exit(1); + } + + rc = read(spawn_server_instance_read_fd(si), buffer, sizeof(buffer)); + if (rc != len) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot read from plugin. Expected to read %zd bytes, read %zd bytes", + len, rc); + exit(1); + } + if (memcmp(msg, buffer, len) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Read corrupted data. Expected '%s', Read '%s'", + msg, buffer); + exit(1); + } + + break; + } + fprintf(stderr, "\n"); + + int code = spawn_server_exec_wait(server, si); + + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "child exited with code %d", + code); + + if(!WIFEXITED(code) || WEXITSTATUS(code) != 1) { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "child should exit with code 1, but exited with code %d", code); + warnings++; + } +} + +void test_popen_plugin_close_to_stop(const char *argv0) { + char cmd[FILENAME_MAX + 100]; + snprintfz(cmd, sizeof(cmd), "exec %s plugin-close-to-stop", argv0); + POPEN_INSTANCE *pi = spawn_popen_run(cmd); + if(!pi) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot run myself as plugin (popen)"); + exit(1); + } + + const char *msg = "Hello World!\n"; + size_t len = strlen(msg); + char buffer[len * 2]; + + for(size_t j = 0; j < 30 ;j++) { + fprintf(stderr, "-"); + memset(buffer, 0, sizeof(buffer)); + + size_t rc = fwrite(msg, 1, len, spawn_popen_stdin(pi)); + if (rc != len) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot write to plugin. Expected to write %zu bytes, wrote %zu bytes", + len, rc); + exit(1); + } + fflush(spawn_popen_stdin(pi)); + + char *s = fgets(buffer, sizeof(buffer), spawn_popen_stdout(pi)); + if (!s || strlen(s) != len) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot read from plugin. Expected to read %zu bytes, read %zu bytes", + len, (size_t)(s ? strlen(s) : 0)); + exit(1); + } + if (memcmp(msg, buffer, len) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Read corrupted data. Expected '%s', Read '%s'", + msg, buffer); + exit(1); + } + + break; + } + fprintf(stderr, "\n"); + + int code = spawn_popen_wait(pi); + + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "child exited with code %d", + code); + + if(code != 1) { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "child should exit with code 1, but exited with code %d", code); + warnings++; + } +} + +// -------------------------------------------------------------------------------------------------------------------- +// echo and exit + +#define ECHO_AND_EXIT_MSG "GOODBYE\n" + +int plugin_echo_and_exit() { + child_check_fds(); + child_check_environment(); + + printf(ECHO_AND_EXIT_MSG); + exit(0); +} + +void test_int_fds_plugin_echo_and_exit(SPAWN_SERVER *server, const char *argv0) { + const char *params[] = { + argv0, + "plugin-echo-and-exit", + NULL, + }; + + SPAWN_INSTANCE *si = spawn_server_exec(server, STDERR_FILENO, 0, params, NULL, 0, SPAWN_INSTANCE_TYPE_EXEC); + if(!si) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot run myself as plugin (spawn)"); + exit(1); + } + + char buffer[1024]; + size_t reads = 0; + + for(size_t j = 0; j < 30 ;j++) { + fprintf(stderr, "-"); + memset(buffer, 0, sizeof(buffer)); + + ssize_t rc = read(spawn_server_instance_read_fd(si), buffer, sizeof(buffer)); + if(rc <= 0) + break; + + reads++; + + if (rc != strlen(ECHO_AND_EXIT_MSG)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot read from plugin. Expected to read %zu bytes, read %zd bytes", + strlen(ECHO_AND_EXIT_MSG), rc); + exit(1); + } + if (memcmp(ECHO_AND_EXIT_MSG, buffer, strlen(ECHO_AND_EXIT_MSG)) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Read corrupted data. Expected '%s', Read '%s'", + ECHO_AND_EXIT_MSG, buffer); + exit(1); + } + } + fprintf(stderr, "\n"); + + if(reads != 1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot read from plugin. Expected to read %d times, but read %zu", + 1, reads); + exit(1); + } + + int code = spawn_server_exec_wait(server, si); + + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "child exited with code %d", + code); + + if(code != 0) { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "child should exit with code 0, but exited with code %d", code); + warnings++; + } +} + +void test_popen_plugin_echo_and_exit(const char *argv0) { + char cmd[FILENAME_MAX + 100]; + snprintfz(cmd, sizeof(cmd), "exec %s plugin-echo-and-exit", argv0); + POPEN_INSTANCE *pi = spawn_popen_run(cmd); + if(!pi) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot run myself as plugin (popen)"); + exit(1); + } + + char buffer[1024]; + size_t reads = 0; + for(size_t j = 0; j < 30 ;j++) { + fprintf(stderr, "-"); + memset(buffer, 0, sizeof(buffer)); + + char *s = fgets(buffer, sizeof(buffer), spawn_popen_stdout(pi)); + if(!s) break; + reads++; + if (strlen(s) != strlen(ECHO_AND_EXIT_MSG)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot read from plugin. Expected to read %zu bytes, read %zu bytes", + strlen(ECHO_AND_EXIT_MSG), (size_t)(s ? strlen(s) : 0)); + exit(1); + } + if (memcmp(ECHO_AND_EXIT_MSG, buffer, strlen(ECHO_AND_EXIT_MSG)) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Read corrupted data. Expected '%s', Read '%s'", + ECHO_AND_EXIT_MSG, buffer); + exit(1); + } + } + fprintf(stderr, "\n"); + + if(reads != 1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot read from plugin. Expected to read %d times, but read %zu", + 1, reads); + exit(1); + } + + int code = spawn_popen_wait(pi); + + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "child exited with code %d", + code); + + if(code != 0) { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "child should exit with code 0, but exited with code %d", code); + warnings++; + } +} + +// -------------------------------------------------------------------------------------------------------------------- + +int main(int argc, const char **argv) { + if(argc > 1 && strcmp(argv[1], "plugin-kill-to-stop") == 0) + return plugin_kill_to_stop(); + + if(argc > 1 && strcmp(argv[1], "plugin-echo-and-exit") == 0) + return plugin_echo_and_exit(); + + if(argc > 1 && strcmp(argv[1], "plugin-close-to-stop") == 0) + return plugin_close_to_stop(); + + if(argc <= 1 || strcmp(argv[1], "test") != 0) { + fprintf(stderr, "Run me with 'test' parameter!\n"); + exit(1); + } + + nd_setenv(ENV_VAR_KEY, ENV_VAR_VALUE, 1); + + fprintf(stderr, "\n\nTESTING fds\n\n"); + SPAWN_SERVER *server = spawn_server_create(SPAWN_SERVER_OPTION_EXEC, "test", NULL, argc, argv); + if(!server) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot create spawn server"); + exit(1); + } + for(size_t i = 0; i < 5; i++) { + fprintf(stderr, "\n\nTESTING fds No %zu (kill to stop)\n\n", i + 1); + test_int_fds_plugin_kill_to_stop(server, argv[0]); + } + for(size_t i = 0; i < 5; i++) { + fprintf(stderr, "\n\nTESTING fds No %zu (echo and exit)\n\n", i + 1); + test_int_fds_plugin_echo_and_exit(server, argv[0]); + } + for(size_t i = 0; i < 5; i++) { + fprintf(stderr, "\n\nTESTING fds No %zu (close to stop)\n\n", i + 1); + test_int_fds_plugin_close_to_stop(server, argv[0]); + } + spawn_server_destroy(server); + + fprintf(stderr, "\n\nTESTING popen\n\n"); + netdata_main_spawn_server_init("test", argc, argv); + for(size_t i = 0; i < 5; i++) { + fprintf(stderr, "\n\nTESTING popen No %zu (kill to stop)\n\n", i + 1); + test_popen_plugin_kill_to_stop(argv[0]); + } + for(size_t i = 0; i < 5; i++) { + fprintf(stderr, "\n\nTESTING popen No %zu (echo and exit)\n\n", i + 1); + test_popen_plugin_echo_and_exit(argv[0]); + } + for(size_t i = 0; i < 5; i++) { + fprintf(stderr, "\n\nTESTING popen No %zu (close to stop)\n\n", i + 1); + test_popen_plugin_close_to_stop(argv[0]); + } + netdata_main_spawn_server_cleanup(); + + fprintf(stderr, "\n\nTests passed! (%zu warnings)\n\n", warnings); + + exit(0); +} diff --git a/src/libnetdata/spawn_server/spawn_library.c b/src/libnetdata/spawn_server/spawn_library.c new file mode 100644 index 000000000..bdf64544c --- /dev/null +++ b/src/libnetdata/spawn_server/spawn_library.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "spawn_library.h" + +BUFFER *argv_to_cmdline_buffer(const char **argv) { + BUFFER *wb = buffer_create(0, NULL); + + for(size_t i = 0; argv[i] ;i++) { + const char *s = argv[i]; + size_t len = strlen(s); + buffer_need_bytes(wb, len * 2 + 1); + + bool needs_quotes = false; + for(const char *c = s; !needs_quotes && *c ; c++) { + switch(*c) { + case ' ': + case '\v': + case '\t': + case '\n': + case '"': + needs_quotes = true; + break; + + default: + break; + } + } + + if(needs_quotes && buffer_strlen(wb)) + buffer_strcat(wb, " \""); + else if(buffer_strlen(wb)) + buffer_putc(wb, ' '); + + for(const char *c = s; *c ; c++) { + switch(*c) { + case '"': + buffer_putc(wb, '\\'); + // fall through + + default: + buffer_putc(wb, *c); + break; + } + } + + if(needs_quotes) + buffer_strcat(wb, "\""); + } + + return wb; +} diff --git a/src/libnetdata/spawn_server/spawn_library.h b/src/libnetdata/spawn_server/spawn_library.h new file mode 100644 index 000000000..a9b9dc14d --- /dev/null +++ b/src/libnetdata/spawn_server/spawn_library.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SPAWN_LIBRARY_H +#define NETDATA_SPAWN_LIBRARY_H + +#include "../libnetdata.h" + +BUFFER *argv_to_cmdline_buffer(const char **argv); + +#endif //NETDATA_SPAWN_LIBRARY_H diff --git a/src/libnetdata/spawn_server/spawn_popen.c b/src/libnetdata/spawn_server/spawn_popen.c index f354b1f2a..b8ea0afe6 100644 --- a/src/libnetdata/spawn_server/spawn_popen.c +++ b/src/libnetdata/spawn_server/spawn_popen.c @@ -2,6 +2,12 @@ #include "spawn_popen.h" +struct popen_instance { + SPAWN_INSTANCE *si; + FILE *child_stdin_fp; + FILE *child_stdout_fp; +}; + SPAWN_SERVER *netdata_main_spawn_server = NULL; static SPINLOCK netdata_main_spawn_server_spinlock = NETDATA_SPINLOCK_INITIALIZER; @@ -27,6 +33,30 @@ void netdata_main_spawn_server_cleanup(void) { } } +FILE *spawn_popen_stdin(POPEN_INSTANCE *pi) { + if(!pi->child_stdin_fp) + pi->child_stdin_fp = fdopen(spawn_server_instance_write_fd(pi->si), "w"); + + if(!pi->child_stdin_fp) + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot open FILE on child's stdin on fd %d.", + spawn_server_instance_write_fd(pi->si)); + + return pi->child_stdin_fp; +} + +FILE *spawn_popen_stdout(POPEN_INSTANCE *pi) { + if(!pi->child_stdout_fp) + pi->child_stdout_fp = fdopen(spawn_server_instance_read_fd(pi->si), "r"); + + if(!pi->child_stdout_fp) + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "Cannot open FILE on child's stdout on fd %d.", + spawn_server_instance_read_fd(pi->si)); + + return pi->child_stdout_fp; +} + POPEN_INSTANCE *spawn_popen_run_argv(const char **argv) { netdata_main_spawn_server_init(NULL, 0, NULL); @@ -35,29 +65,9 @@ POPEN_INSTANCE *spawn_popen_run_argv(const char **argv) { if(si == NULL) return NULL; - POPEN_INSTANCE *pi = mallocz(sizeof(*pi)); + POPEN_INSTANCE *pi = callocz(1, sizeof(*pi)); pi->si = si; - pi->child_stdin_fp = fdopen(spawn_server_instance_write_fd(si), "w"); - pi->child_stdout_fp = fdopen(spawn_server_instance_read_fd(si), "r"); - - if(!pi->child_stdin_fp) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot open FILE on child's stdin on fd %d.", spawn_server_instance_write_fd(si)); - goto cleanup; - } - - if(!pi->child_stdout_fp) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot open FILE on child's stdout on fd %d.", spawn_server_instance_read_fd(si)); - goto cleanup; - } - return pi; - -cleanup: - if(pi->child_stdin_fp) { fclose(pi->child_stdin_fp); spawn_server_instance_write_fd(si); } - if(pi->child_stdout_fp) { fclose(pi->child_stdout_fp); spawn_server_instance_read_fd_unset(si); } - spawn_server_exec_kill(netdata_main_spawn_server, si); - freez(pi); - return NULL; } POPEN_INSTANCE *spawn_popen_run_variadic(const char *cmd, ...) { @@ -92,7 +102,33 @@ POPEN_INSTANCE *spawn_popen_run_variadic(const char *cmd, ...) { POPEN_INSTANCE *spawn_popen_run(const char *cmd) { if(!cmd || !*cmd) return NULL; - + +//#if defined(OS_WINDOWS) +// if(strncmp(cmd, "exec ", 5) == 0) { +// size_t len = strlen(cmd); +// char cmd_copy[strlen(cmd) + 1]; +// memcpy(cmd_copy, cmd, len + 1); +// char *words[100]; +// size_t num_words = quoted_strings_splitter(cmd_copy, words, 100, isspace_map_pluginsd); +// char *exec = get_word(words, num_words, 0); +// char *prog = get_word(words, num_words, 1); +// if (strcmp(exec, "exec") == 0 && +// prog && +// strendswith(prog, ".plugin") && +// !strendswith(prog, "charts.d.plugin") && +// !strendswith(prog, "ioping.plugin")) { +// const char *argv[num_words - 1 + 1]; // remove exec, add terminator +// +// size_t dst = 0; +// for (size_t i = 1; i < num_words; i++) +// argv[dst++] = get_word(words, num_words, i); +// +// argv[dst] = NULL; +// return spawn_popen_run_argv(argv); +// } +// } +//#endif + const char *argv[] = { "/bin/sh", "-c", @@ -121,11 +157,24 @@ static int spawn_popen_status_rc(int status) { return -1; } +static void spawn_popen_close_files(POPEN_INSTANCE *pi) { + if(pi->child_stdin_fp) { + fclose(pi->child_stdin_fp); + pi->child_stdin_fp = NULL; + spawn_server_instance_write_fd_unset(pi->si); + } + + if(pi->child_stdout_fp) { + fclose(pi->child_stdout_fp); + pi->child_stdout_fp = NULL; + spawn_server_instance_read_fd_unset(pi->si); + } +} + int spawn_popen_wait(POPEN_INSTANCE *pi) { if(!pi) return -1; - fclose(pi->child_stdin_fp); pi->child_stdin_fp = NULL; spawn_server_instance_write_fd_unset(pi->si); - fclose(pi->child_stdout_fp); pi->child_stdout_fp = NULL; spawn_server_instance_read_fd_unset(pi->si); + spawn_popen_close_files(pi); int status = spawn_server_exec_wait(netdata_main_spawn_server, pi->si); freez(pi); return spawn_popen_status_rc(status); @@ -134,9 +183,23 @@ int spawn_popen_wait(POPEN_INSTANCE *pi) { int spawn_popen_kill(POPEN_INSTANCE *pi) { if(!pi) return -1; - fclose(pi->child_stdin_fp); pi->child_stdin_fp = NULL; spawn_server_instance_write_fd_unset(pi->si); - fclose(pi->child_stdout_fp); pi->child_stdout_fp = NULL; spawn_server_instance_read_fd_unset(pi->si); + spawn_popen_close_files(pi); int status = spawn_server_exec_kill(netdata_main_spawn_server, pi->si); freez(pi); return spawn_popen_status_rc(status); } + +pid_t spawn_popen_pid(POPEN_INSTANCE *pi) { + if(!pi) return -1; + return spawn_server_instance_pid(pi->si); +} + +int spawn_popen_read_fd(POPEN_INSTANCE *pi) { + if(!pi) return -1; + return spawn_server_instance_read_fd(pi->si); +} + +int spawn_popen_write_fd(POPEN_INSTANCE *pi) { + if(!pi) return -1; + return spawn_server_instance_write_fd(pi->si); +} diff --git a/src/libnetdata/spawn_server/spawn_popen.h b/src/libnetdata/spawn_server/spawn_popen.h index 253d1f34b..5c00f32ff 100644 --- a/src/libnetdata/spawn_server/spawn_popen.h +++ b/src/libnetdata/spawn_server/spawn_popen.h @@ -9,11 +9,7 @@ extern SPAWN_SERVER *netdata_main_spawn_server; bool netdata_main_spawn_server_init(const char *name, int argc, const char **argv); void netdata_main_spawn_server_cleanup(void); -typedef struct { - SPAWN_INSTANCE *si; - FILE *child_stdin_fp; - FILE *child_stdout_fp; -} POPEN_INSTANCE; +typedef struct popen_instance POPEN_INSTANCE; POPEN_INSTANCE *spawn_popen_run(const char *cmd); POPEN_INSTANCE *spawn_popen_run_argv(const char **argv); @@ -21,4 +17,10 @@ POPEN_INSTANCE *spawn_popen_run_variadic(const char *cmd, ...); int spawn_popen_wait(POPEN_INSTANCE *pi); int spawn_popen_kill(POPEN_INSTANCE *pi); +pid_t spawn_popen_pid(POPEN_INSTANCE *pi); +int spawn_popen_read_fd(POPEN_INSTANCE *pi); +int spawn_popen_write_fd(POPEN_INSTANCE *pi); +FILE *spawn_popen_stdin(POPEN_INSTANCE *pi); +FILE *spawn_popen_stdout(POPEN_INSTANCE *pi); + #endif //SPAWN_POPEN_H diff --git a/src/libnetdata/spawn_server/spawn_server.h b/src/libnetdata/spawn_server/spawn_server.h index 5ba66ae38..e68a53ab4 100644 --- a/src/libnetdata/spawn_server/spawn_server.h +++ b/src/libnetdata/spawn_server/spawn_server.h @@ -7,16 +7,12 @@ typedef enum __attribute__((packed)) { SPAWN_INSTANCE_TYPE_EXEC = 0, -#if !defined(OS_WINDOWS) SPAWN_INSTANCE_TYPE_CALLBACK = 1 -#endif } SPAWN_INSTANCE_TYPE; typedef enum __attribute__((packed)) { SPAWN_SERVER_OPTION_EXEC = (1 << 0), -#if !defined(OS_WINDOWS) SPAWN_SERVER_OPTION_CALLBACK = (1 << 1), -#endif } SPAWN_SERVER_OPTIONS; // this is only used publicly for SPAWN_INSTANCE_TYPE_CALLBACK @@ -27,7 +23,7 @@ typedef struct spawn_request { pid_t pid; // the pid of the child int sock; // the socket for this request int fds[SPAWN_SERVER_TRANSFER_FDS]; // 0 = stdin, 1 = stdout, 2 = stderr, 3 = custom - const char **environment; // the environment of the parent process + const char **envp; // the environment of the parent process const char **argv; // the command line and its parameters const void *data; // the data structure for the callback size_t data_size; // the data structure size @@ -36,17 +32,18 @@ typedef struct spawn_request { struct spawn_request *prev, *next; // linking of active requests at the spawn server } SPAWN_REQUEST; -typedef void (*spawn_request_callback_t)(SPAWN_REQUEST *request); +typedef int (*spawn_request_callback_t)(SPAWN_REQUEST *request); -typedef struct spawm_instance SPAWN_INSTANCE; +typedef struct spawn_instance SPAWN_INSTANCE; typedef struct spawn_server SPAWN_SERVER; SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options, const char *name, spawn_request_callback_t child_callback, int argc, const char **argv); void spawn_server_destroy(SPAWN_SERVER *server); +pid_t spawn_server_pid(SPAWN_SERVER *server); SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custom_fd, const char **argv, const void *data, size_t data_size, SPAWN_INSTANCE_TYPE type); -int spawn_server_exec_kill(SPAWN_SERVER *server, SPAWN_INSTANCE *instance); -int spawn_server_exec_wait(SPAWN_SERVER *server, SPAWN_INSTANCE *instance); +int spawn_server_exec_kill(SPAWN_SERVER *server, SPAWN_INSTANCE *si); +int spawn_server_exec_wait(SPAWN_SERVER *server, SPAWN_INSTANCE *si); int spawn_server_instance_read_fd(SPAWN_INSTANCE *si); int spawn_server_instance_write_fd(SPAWN_INSTANCE *si); diff --git a/src/libnetdata/spawn_server/spawn_server_internals.h b/src/libnetdata/spawn_server/spawn_server_internals.h new file mode 100644 index 000000000..1031e3b1a --- /dev/null +++ b/src/libnetdata/spawn_server/spawn_server_internals.h @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SPAWN_SERVER_INTERNALS_H +#define NETDATA_SPAWN_SERVER_INTERNALS_H + +#include "../libnetdata.h" +#include "spawn_server.h" +#include "spawn_library.h" +#include "log-forwarder.h" + +#if defined(OS_WINDOWS) +#define SPAWN_SERVER_VERSION_WINDOWS 1 +// #define SPAWN_SERVER_VERSION_UV 1 +// #define SPAWN_SERVER_VERSION_POSIX_SPAWN 1 +#else +#define SPAWN_SERVER_VERSION_NOFORK 1 +// #define SPAWN_SERVER_VERSION_UV 1 +// #define SPAWN_SERVER_VERSION_POSIX_SPAWN 1 +#endif + +struct spawn_server { + size_t id; + size_t request_id; + const char *name; + +#if defined(SPAWN_SERVER_VERSION_UV) + uv_loop_t *loop; + uv_thread_t thread; + uv_async_t async; + bool stopping; + + SPINLOCK spinlock; + struct work_item *work_queue; +#endif + +#if defined(SPAWN_SERVER_VERSION_NOFORK) + SPAWN_SERVER_OPTIONS options; + + ND_UUID magic; // for authorizing requests, the client needs to know our random UUID + // it is ignored for PING requests + + int pipe[2]; + int sock; // the listening socket of the server + pid_t server_pid; + char *path; + spawn_request_callback_t cb; + + int argc; + const char **argv; +#endif + +#if defined(SPAWN_SERVER_VERSION_POSIX_SPAWN) +#endif + +#if defined(SPAWN_SERVER_VERSION_WINDOWS) + LOG_FORWARDER *log_forwarder; +#endif +}; + +struct spawn_instance { + size_t request_id; + int sock; + int write_fd; // the child's input pipe, writing side + int read_fd; // the child's output pipe, reading side + int stderr_fd; + pid_t child_pid; + +#if defined(SPAWN_SERVER_VERSION_UV) + uv_process_t process; + int exit_code; + uv_sem_t sem; +#endif + +#if defined(SPAWN_SERVER_VERSION_NOFORK) +#endif + +#if defined(SPAWN_SERVER_VERSION_POSIX_SPAWN) + const char *cmdline; + bool exited; + int waitpid_status; + struct spawn_instance *prev, *next; +#endif + +#if defined(SPAWN_SERVER_VERSION_WINDOWS) + HANDLE process_handle; + DWORD dwProcessId; +#endif +}; + +#endif //NETDATA_SPAWN_SERVER_INTERNALS_H diff --git a/src/libnetdata/spawn_server/spawn_server_libuv.c b/src/libnetdata/spawn_server/spawn_server_libuv.c new file mode 100644 index 000000000..e01c5407e --- /dev/null +++ b/src/libnetdata/spawn_server/spawn_server_libuv.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "spawn_server_internals.h" + +#if defined(SPAWN_SERVER_VERSION_UV) + +int spawn_server_instance_read_fd(SPAWN_INSTANCE *si) { return si->read_fd; } +int spawn_server_instance_write_fd(SPAWN_INSTANCE *si) { return si->write_fd; } +void spawn_server_instance_read_fd_unset(SPAWN_INSTANCE *si) { si->read_fd = -1; } +void spawn_server_instance_write_fd_unset(SPAWN_INSTANCE *si) { si->write_fd = -1; } +pid_t spawn_server_instance_pid(SPAWN_INSTANCE *si) { return uv_process_get_pid(&si->process); } + +typedef struct work_item { + int stderr_fd; + const char **argv; + uv_sem_t sem; + SPAWN_INSTANCE *instance; + struct work_item *prev; + struct work_item *next; +} work_item; + +int uv_errno_to_errno(int uv_err) { + switch (uv_err) { + case 0: return 0; + case UV_E2BIG: return E2BIG; + case UV_EACCES: return EACCES; + case UV_EADDRINUSE: return EADDRINUSE; + case UV_EADDRNOTAVAIL: return EADDRNOTAVAIL; + case UV_EAFNOSUPPORT: return EAFNOSUPPORT; + case UV_EAGAIN: return EAGAIN; + case UV_EAI_ADDRFAMILY: return EAI_ADDRFAMILY; + case UV_EAI_AGAIN: return EAI_AGAIN; + case UV_EAI_BADFLAGS: return EAI_BADFLAGS; +#if defined(EAI_CANCELED) + case UV_EAI_CANCELED: return EAI_CANCELED; +#endif + case UV_EAI_FAIL: return EAI_FAIL; + case UV_EAI_FAMILY: return EAI_FAMILY; + case UV_EAI_MEMORY: return EAI_MEMORY; + case UV_EAI_NODATA: return EAI_NODATA; + case UV_EAI_NONAME: return EAI_NONAME; + case UV_EAI_OVERFLOW: return EAI_OVERFLOW; + case UV_EAI_SERVICE: return EAI_SERVICE; + case UV_EAI_SOCKTYPE: return EAI_SOCKTYPE; + case UV_EALREADY: return EALREADY; + case UV_EBADF: return EBADF; + case UV_EBUSY: return EBUSY; + case UV_ECANCELED: return ECANCELED; + case UV_ECHARSET: return EILSEQ; // No direct mapping, using EILSEQ + case UV_ECONNABORTED: return ECONNABORTED; + case UV_ECONNREFUSED: return ECONNREFUSED; + case UV_ECONNRESET: return ECONNRESET; + case UV_EDESTADDRREQ: return EDESTADDRREQ; + case UV_EEXIST: return EEXIST; + case UV_EFAULT: return EFAULT; + case UV_EFBIG: return EFBIG; + case UV_EHOSTUNREACH: return EHOSTUNREACH; + case UV_EINTR: return EINTR; + case UV_EINVAL: return EINVAL; + case UV_EIO: return EIO; + case UV_EISCONN: return EISCONN; + case UV_EISDIR: return EISDIR; + case UV_ELOOP: return ELOOP; + case UV_EMFILE: return EMFILE; + case UV_EMSGSIZE: return EMSGSIZE; + case UV_ENAMETOOLONG: return ENAMETOOLONG; + case UV_ENETDOWN: return ENETDOWN; + case UV_ENETUNREACH: return ENETUNREACH; + case UV_ENFILE: return ENFILE; + case UV_ENOBUFS: return ENOBUFS; + case UV_ENODEV: return ENODEV; + case UV_ENOENT: return ENOENT; + case UV_ENOMEM: return ENOMEM; + case UV_ENONET: return ENONET; + case UV_ENOSPC: return ENOSPC; + case UV_ENOSYS: return ENOSYS; + case UV_ENOTCONN: return ENOTCONN; + case UV_ENOTDIR: return ENOTDIR; + case UV_ENOTEMPTY: return ENOTEMPTY; + case UV_ENOTSOCK: return ENOTSOCK; + case UV_ENOTSUP: return ENOTSUP; + case UV_ENOTTY: return ENOTTY; + case UV_ENXIO: return ENXIO; + case UV_EPERM: return EPERM; + case UV_EPIPE: return EPIPE; + case UV_EPROTO: return EPROTO; + case UV_EPROTONOSUPPORT: return EPROTONOSUPPORT; + case UV_EPROTOTYPE: return EPROTOTYPE; + case UV_ERANGE: return ERANGE; + case UV_EROFS: return EROFS; + case UV_ESHUTDOWN: return ESHUTDOWN; + case UV_ESPIPE: return ESPIPE; + case UV_ESRCH: return ESRCH; + case UV_ETIMEDOUT: return ETIMEDOUT; + case UV_ETXTBSY: return ETXTBSY; + case UV_EXDEV: return EXDEV; + default: return EINVAL; // Use EINVAL for unknown libuv errors + } +} + +static void posix_unmask_sigchld_on_thread(void) { + sigset_t sigset; + sigemptyset(&sigset); // Initialize the signal set to empty + sigaddset(&sigset, SIGCHLD); // Add SIGCHLD to the set + + if(pthread_sigmask(SIG_UNBLOCK, &sigset, NULL) != 0) + netdata_log_error("SPAWN SERVER: cannot unmask SIGCHLD"); +} + +static void server_thread(void *arg) { + SPAWN_SERVER *server = (SPAWN_SERVER *)arg; + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN SERVER: started"); + + // this thread needs to process SIGCHLD (by libuv) + // otherwise the on_exit() callback is never run + posix_unmask_sigchld_on_thread(); + + // run the event loop + uv_run(server->loop, UV_RUN_DEFAULT); + + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN SERVER: ended"); +} + +static void on_process_exit(uv_process_t *req, int64_t exit_status, int term_signal) { + SPAWN_INSTANCE *si = (SPAWN_INSTANCE *)req->data; + si->exit_code = (int)(term_signal ? term_signal : exit_status << 8); + uv_close((uv_handle_t *)req, NULL); // Properly close the process handle + + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN SERVER: process with pid %d exited with code %d and term_signal %d", + si->child_pid, (int)exit_status, term_signal); + + uv_sem_post(&si->sem); // Signal that the process has exited +} + +static SPAWN_INSTANCE *spawn_process_with_libuv(uv_loop_t *loop, int stderr_fd, const char **argv) { + SPAWN_INSTANCE *si = NULL; + bool si_sem_init = false; + + int stdin_pipe[2] = { -1, -1 }; + int stdout_pipe[2] = { -1, -1 }; + + if (pipe(stdin_pipe) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: stdin pipe() failed"); + goto cleanup; + } + + if (pipe(stdout_pipe) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: stdout pipe() failed"); + goto cleanup; + } + + si = callocz(1, sizeof(SPAWN_INSTANCE)); + si->exit_code = -1; + + if (uv_sem_init(&si->sem, 0)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: uv_sem_init() failed"); + goto cleanup; + } + si_sem_init = true; + + uv_stdio_container_t stdio[3] = { 0 }; + stdio[0].flags = UV_INHERIT_FD; + stdio[0].data.fd = stdin_pipe[PIPE_READ]; + stdio[1].flags = UV_INHERIT_FD; + stdio[1].data.fd = stdout_pipe[PIPE_WRITE]; + stdio[2].flags = UV_INHERIT_FD; + stdio[2].data.fd = stderr_fd; + + uv_process_options_t options = { 0 }; + options.stdio_count = 3; + options.stdio = stdio; + options.exit_cb = on_process_exit; + options.file = argv[0]; + options.args = (char **)argv; + options.env = (char **)environ; + + // uv_spawn() does not close all other open file descriptors + // we have to close them manually + int fds[3] = { stdio[0].data.fd, stdio[1].data.fd, stdio[2].data.fd }; + os_close_all_non_std_open_fds_except(fds, 3, CLOSE_RANGE_CLOEXEC); + + int rc = uv_spawn(loop, &si->process, &options); + if (rc) { + errno = uv_errno_to_errno(rc); + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN SERVER: uv_spawn() failed with error %s, %s", + uv_err_name(rc), uv_strerror(rc)); + goto cleanup; + } + + // Successfully spawned + + // get the pid of the process spawned + si->child_pid = uv_process_get_pid(&si->process); + + // on_process_exit() needs this to find the si + si->process.data = si; + + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN SERVER: process created with pid %d", si->child_pid); + + // close the child sides of the pipes + close(stdin_pipe[PIPE_READ]); + si->write_fd = stdin_pipe[PIPE_WRITE]; + si->read_fd = stdout_pipe[PIPE_READ]; + close(stdout_pipe[PIPE_WRITE]); + + return si; + +cleanup: + if(stdin_pipe[PIPE_READ] != -1) close(stdin_pipe[PIPE_READ]); + if(stdin_pipe[PIPE_WRITE] != -1) close(stdin_pipe[PIPE_WRITE]); + if(stdout_pipe[PIPE_READ] != -1) close(stdout_pipe[PIPE_READ]); + if(stdout_pipe[PIPE_WRITE] != -1) close(stdout_pipe[PIPE_WRITE]); + if(si) { + if(si_sem_init) + uv_sem_destroy(&si->sem); + + freez(si); + } + return NULL; +} + +static void async_callback(uv_async_t *handle) { + nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN SERVER: dequeue commands started"); + SPAWN_SERVER *server = (SPAWN_SERVER *)handle->data; + + // Check if the server is stopping + if (__atomic_load_n(&server->stopping, __ATOMIC_RELAXED)) { + nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN SERVER: stopping..."); + uv_stop(server->loop); + return; + } + + work_item *item; + spinlock_lock(&server->spinlock); + while (server->work_queue) { + item = server->work_queue; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(server->work_queue, item, prev, next); + spinlock_unlock(&server->spinlock); + + item->instance = spawn_process_with_libuv(server->loop, item->stderr_fd, item->argv); + uv_sem_post(&item->sem); + + spinlock_lock(&server->spinlock); + } + spinlock_unlock(&server->spinlock); + + nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN SERVER: dequeue commands done"); +} + + +SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options __maybe_unused, const char *name, spawn_request_callback_t cb __maybe_unused, int argc __maybe_unused, const char **argv __maybe_unused) { + SPAWN_SERVER* server = callocz(1, sizeof(SPAWN_SERVER)); + spinlock_init(&server->spinlock); + + if (name) + server->name = strdupz(name); + else + server->name = strdupz("unnamed"); + + server->loop = callocz(1, sizeof(uv_loop_t)); + if (uv_loop_init(server->loop)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: uv_loop_init() failed"); + freez(server->loop); + freez((void *)server->name); + freez(server); + return NULL; + } + + if (uv_async_init(server->loop, &server->async, async_callback)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: uv_async_init() failed"); + uv_loop_close(server->loop); + freez(server->loop); + freez((void *)server->name); + freez(server); + return NULL; + } + server->async.data = server; + + if (uv_thread_create(&server->thread, server_thread, server)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: uv_thread_create() failed"); + uv_close((uv_handle_t*)&server->async, NULL); + uv_loop_close(server->loop); + freez(server->loop); + freez((void *)server->name); + freez(server); + return NULL; + } + + return server; +} + +static void close_handle(uv_handle_t* handle, void* arg __maybe_unused) { + if (!uv_is_closing(handle)) { + uv_close(handle, NULL); + } +} + +void spawn_server_destroy(SPAWN_SERVER *server) { + if (!server) return; + + __atomic_store_n(&server->stopping, true, __ATOMIC_RELAXED); + + // Trigger the async callback to stop the event loop + uv_async_send(&server->async); + + // Wait for the server thread to finish + uv_thread_join(&server->thread); + + uv_stop(server->loop); + uv_close((uv_handle_t*)&server->async, NULL); + + // Walk through and close any remaining handles + uv_walk(server->loop, close_handle, NULL); + + uv_loop_close(server->loop); + freez(server->loop); + freez((void *)server->name); + freez(server); +} + +SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd __maybe_unused, int custom_fd __maybe_unused, const char **argv, const void *data __maybe_unused, size_t data_size __maybe_unused, SPAWN_INSTANCE_TYPE type) { + if (type != SPAWN_INSTANCE_TYPE_EXEC) + return NULL; + + work_item item = { 0 }; + item.stderr_fd = stderr_fd; + item.argv = argv; + + if (uv_sem_init(&item.sem, 0)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: uv_sem_init() failed"); + return NULL; + } + + spinlock_lock(&server->spinlock); + // item is in the stack, but the server will remove it before sending to us + // the semaphore, so it is safe to have the item in the stack. + work_item *item_ptr = &item; + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(server->work_queue, item_ptr, prev, next); + spinlock_unlock(&server->spinlock); + + uv_async_send(&server->async); + + nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN PARENT: queued command"); + + // Wait for the command to be executed + uv_sem_wait(&item.sem); + uv_sem_destroy(&item.sem); + + if (!item.instance) { + nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN PARENT: process failed to be started"); + return NULL; + } + + nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN PARENT: process started"); + + return item.instance; +} + +int spawn_server_exec_kill(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *si) { + if(!si) return -1; + + // close all pipe descriptors to force the child to exit + if(si->read_fd != -1) { close(si->read_fd); si->read_fd = -1; } + if(si->write_fd != -1) { close(si->write_fd); si->write_fd = -1; } + + if (uv_process_kill(&si->process, SIGTERM)) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: uv_process_kill() failed"); + return -1; + } + + return spawn_server_exec_wait(server, si); +} + +int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *si) { + if (!si) return -1; + + // close all pipe descriptors to force the child to exit + if(si->read_fd != -1) { close(si->read_fd); si->read_fd = -1; } + if(si->write_fd != -1) { close(si->write_fd); si->write_fd = -1; } + + // Wait for the process to exit + uv_sem_wait(&si->sem); + int exit_code = si->exit_code; + + uv_sem_destroy(&si->sem); + freez(si); + return exit_code; +} + +#endif diff --git a/src/libnetdata/spawn_server/spawn_server.c b/src/libnetdata/spawn_server/spawn_server_nofork.c index ef6755c32..9986740de 100644 --- a/src/libnetdata/spawn_server/spawn_server.c +++ b/src/libnetdata/spawn_server/spawn_server_nofork.c @@ -1,287 +1,20 @@ // SPDX-License-Identifier: GPL-3.0-or-later -#include "../libnetdata.h" +#include "spawn_server_internals.h" -#include "spawn_server.h" - -#if defined(OS_WINDOWS) -#include <windows.h> -#include <io.h> -#include <fcntl.h> -#include <process.h> -#include <sys/cygwin.h> -#endif - -struct spawn_server { - size_t id; - size_t request_id; - const char *name; -#if !defined(OS_WINDOWS) - SPAWN_SERVER_OPTIONS options; - - ND_UUID magic; // for authorizing requests, the client needs to know our random UUID - // it is ignored for PING requests - - int pipe[2]; - int sock; // the listening socket of the server - pid_t server_pid; - char *path; - spawn_request_callback_t cb; - - int argc; - const char **argv; -#endif -}; - -struct spawm_instance { - size_t request_id; - int sock; - int write_fd; - int read_fd; - pid_t child_pid; - -#if defined(OS_WINDOWS) - HANDLE process_handle; - HANDLE read_handle; - HANDLE write_handle; -#endif -}; +#if defined(SPAWN_SERVER_VERSION_NOFORK) +// the child's output pipe, reading side int spawn_server_instance_read_fd(SPAWN_INSTANCE *si) { return si->read_fd; } + +// the child's input pipe, writing side int spawn_server_instance_write_fd(SPAWN_INSTANCE *si) { return si->write_fd; } -pid_t spawn_server_instance_pid(SPAWN_INSTANCE *si) { return si->child_pid; } + void spawn_server_instance_read_fd_unset(SPAWN_INSTANCE *si) { si->read_fd = -1; } void spawn_server_instance_write_fd_unset(SPAWN_INSTANCE *si) { si->write_fd = -1; } +pid_t spawn_server_instance_pid(SPAWN_INSTANCE *si) { return si->child_pid; } -#if defined(OS_WINDOWS) - -SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options __maybe_unused, const char *name, spawn_request_callback_t cb __maybe_unused, int argc __maybe_unused, const char **argv __maybe_unused) { - SPAWN_SERVER* server = callocz(1, sizeof(SPAWN_SERVER)); - if(name) - server->name = strdupz(name); - else - server->name = strdupz("unnamed"); - return server; -} - -void spawn_server_destroy(SPAWN_SERVER *server) { - if (server) { - freez((void *)server->name); - freez(server); - } -} - -static BUFFER *argv_to_windows(const char **argv) { - BUFFER *wb = buffer_create(0, NULL); - - // argv[0] is the path - char b[strlen(argv[0]) * 2 + 1024]; - cygwin_conv_path(CCP_POSIX_TO_WIN_A | CCP_ABSOLUTE, argv[0], b, sizeof(b)); - - buffer_strcat(wb, "cmd.exe /C "); - - for(size_t i = 0; argv[i] ;i++) { - const char *s = (i == 0) ? b : argv[i]; - size_t len = strlen(s); - buffer_need_bytes(wb, len * 2 + 1); - - bool needs_quotes = false; - for(const char *c = s; !needs_quotes && *c ; c++) { - switch(*c) { - case ' ': - case '\v': - case '\t': - case '\n': - case '"': - needs_quotes = true; - break; - - default: - break; - } - } - - if(needs_quotes && buffer_strlen(wb)) - buffer_strcat(wb, " \""); - else - buffer_putc(wb, ' '); - - for(const char *c = s; *c ; c++) { - switch(*c) { - case '"': - buffer_putc(wb, '\\'); - // fall through - - default: - buffer_putc(wb, *c); - break; - } - } - - if(needs_quotes) - buffer_strcat(wb, "\""); - } - - return wb; -} - -SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custom_fd __maybe_unused, const char **argv, const void *data __maybe_unused, size_t data_size __maybe_unused, SPAWN_INSTANCE_TYPE type) { - static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER; - - if (type != SPAWN_INSTANCE_TYPE_EXEC) - return NULL; - - int pipe_stdin[2] = { -1, -1 }, pipe_stdout[2] = { -1, -1 }; - - errno_clear(); - - SPAWN_INSTANCE *instance = callocz(1, sizeof(*instance)); - instance->request_id = __atomic_add_fetch(&server->request_id, 1, __ATOMIC_RELAXED); - - CLEAN_BUFFER *wb = argv_to_windows(argv); - char *command = (char *)buffer_tostring(wb); - - if (pipe(pipe_stdin) == -1) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN PARENT: Cannot create stdin pipe() for request No %zu, command: %s", - instance->request_id, command); - goto cleanup; - } - - if (pipe(pipe_stdout) == -1) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN PARENT: Cannot create stdout pipe() for request No %zu, command: %s", - instance->request_id, command); - goto cleanup; - } - - // do not run multiple times this section - // to prevent handles leaking - spinlock_lock(&spinlock); - - // Convert POSIX file descriptors to Windows handles - HANDLE stdin_read_handle = (HANDLE)_get_osfhandle(pipe_stdin[0]); - HANDLE stdout_write_handle = (HANDLE)_get_osfhandle(pipe_stdout[1]); - HANDLE stderr_handle = (HANDLE)_get_osfhandle(stderr_fd); - - if (stdin_read_handle == INVALID_HANDLE_VALUE || stdout_write_handle == INVALID_HANDLE_VALUE || stderr_handle == INVALID_HANDLE_VALUE) { - spinlock_unlock(&spinlock); - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN PARENT: Invalid handle value(s) for request No %zu, command: %s", - instance->request_id, command); - goto cleanup; - } - - // Set handle inheritance - if (!SetHandleInformation(stdin_read_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) || - !SetHandleInformation(stdout_write_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) || - !SetHandleInformation(stderr_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) { - spinlock_unlock(&spinlock); - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN PARENT: Cannot set handle(s) inheritance for request No %zu, command: %s", - instance->request_id, command); - goto cleanup; - } - - // Set up the STARTUPINFO structure - STARTUPINFO si; - PROCESS_INFORMATION pi; - ZeroMemory(&si, sizeof(si)); - si.cb = sizeof(si); - si.dwFlags = STARTF_USESTDHANDLES; - si.hStdInput = stdin_read_handle; - si.hStdOutput = stdout_write_handle; - si.hStdError = stderr_handle; - - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN PARENT: Running request No %zu, command: %s", - instance->request_id, command); - - // Spawn the process - if (!CreateProcess(NULL, command, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi)) { - spinlock_unlock(&spinlock); - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN PARENT: cannot CreateProcess() for request No %zu, command: %s", - instance->request_id, command); - goto cleanup; - } - - CloseHandle(pi.hThread); - - // end of the critical section - spinlock_unlock(&spinlock); - - // Close unused pipe ends - close(pipe_stdin[0]); pipe_stdin[0] = -1; - close(pipe_stdout[1]); pipe_stdout[1] = -1; - - // Store process information in instance - instance->child_pid = cygwin_winpid_to_pid(pi.dwProcessId); - if(instance->child_pid == -1) instance->child_pid = pi.dwProcessId; - - instance->process_handle = pi.hProcess; - - // Convert handles to POSIX file descriptors - instance->write_fd = pipe_stdin[1]; - instance->read_fd = pipe_stdout[0]; - - errno_clear(); - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN PARENT: created process for request No %zu, pid %d, command: %s", - instance->request_id, (int)instance->child_pid, command); - - return instance; - -cleanup: - if (pipe_stdin[0] >= 0) close(pipe_stdin[0]); - if (pipe_stdin[1] >= 0) close(pipe_stdin[1]); - if (pipe_stdout[0] >= 0) close(pipe_stdout[0]); - if (pipe_stdout[1] >= 0) close(pipe_stdout[1]); - freez(instance); - return NULL; -} - -int spawn_server_exec_kill(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *instance) { - if(instance->read_fd != -1) { close(instance->read_fd); instance->read_fd = -1; } - if(instance->write_fd != -1) { close(instance->write_fd); instance->write_fd = -1; } - CloseHandle(instance->read_handle); instance->read_handle = NULL; - CloseHandle(instance->write_handle); instance->write_handle = NULL; - - TerminateProcess(instance->process_handle, 0); - - DWORD exit_code; - GetExitCodeProcess(instance->process_handle, &exit_code); - CloseHandle(instance->process_handle); - - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN PARENT: child of request No %zu, pid %d, killed and exited with code %d", - instance->request_id, (int)instance->child_pid, (int)exit_code); - - freez(instance); - return (int)exit_code; -} - -int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *instance) { - if(instance->read_fd != -1) { close(instance->read_fd); instance->read_fd = -1; } - if(instance->write_fd != -1) { close(instance->write_fd); instance->write_fd = -1; } - CloseHandle(instance->read_handle); instance->read_handle = NULL; - CloseHandle(instance->write_handle); instance->write_handle = NULL; - - WaitForSingleObject(instance->process_handle, INFINITE); - - DWORD exit_code = -1; - GetExitCodeProcess(instance->process_handle, &exit_code); - CloseHandle(instance->process_handle); - - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN PARENT: child of request No %zu, pid %d, waited and exited with code %d", - instance->request_id, (int)instance->child_pid, (int)exit_code); - - freez(instance); - return (int)exit_code; -} - -#else // !OS_WINDOWS +pid_t spawn_server_pid(SPAWN_SERVER *server) { return server->server_pid; } #ifdef __APPLE__ #include <crt_externs.h> @@ -313,7 +46,7 @@ static int connect_to_spawn_server(const char *path, bool log) { if (connect(sock, (struct sockaddr *)&server_addr, sizeof(server_addr)) == -1) { if(log) - nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Cannot connect() to spawn server."); + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Cannot connect() to spawn server on path '%s'.", path); close(sock); return -1; } @@ -322,80 +55,6 @@ static int connect_to_spawn_server(const char *path, bool log) { } // -------------------------------------------------------------------------------------------------------------------- -// the child created by the spawn server - -static void spawn_server_run_child(SPAWN_SERVER *server, SPAWN_REQUEST *rq) { - // close the server sockets; - close(server->sock); server->sock = -1; - if(server->pipe[0] != -1) { close(server->pipe[0]); server->pipe[0] = -1; } - if(server->pipe[1] != -1) { close(server->pipe[1]); server->pipe[1] = -1; } - - // set the process name - os_setproctitle("spawn-child", server->argc, server->argv); - - // get the fds from the request - int stdin_fd = rq->fds[0]; - int stdout_fd = rq->fds[1]; - int stderr_fd = rq->fds[2]; - int custom_fd = rq->fds[3]; (void)custom_fd; - - // change stdio fds to the ones in the request - if (dup2(stdin_fd, STDIN_FILENO) == -1) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN SERVER: cannot dup2(%d) stdin of request No %zu: %s", - stdin_fd, rq->request_id, rq->cmdline); - exit(1); - } - if (dup2(stdout_fd, STDOUT_FILENO) == -1) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN SERVER: cannot dup2(%d) stdin of request No %zu: %s", - stdout_fd, rq->request_id, rq->cmdline); - exit(1); - } - if (dup2(stderr_fd, STDERR_FILENO) == -1) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN SERVER: cannot dup2(%d) stderr of request No %zu: %s", - stderr_fd, rq->request_id, rq->cmdline); - exit(1); - } - - // close the excess fds - close(stdin_fd); stdin_fd = rq->fds[0] = STDIN_FILENO; - close(stdout_fd); stdout_fd = rq->fds[1] = STDOUT_FILENO; - close(stderr_fd); stderr_fd = rq->fds[2] = STDERR_FILENO; - - // overwrite the process environment - environ = (char **)rq->environment; - - // Perform different actions based on the type - switch (rq->type) { - - case SPAWN_INSTANCE_TYPE_EXEC: - // close all fds except the ones we need - os_close_all_non_std_open_fds_except(NULL, 0); - - // run the command - execvp(rq->argv[0], (char **)rq->argv); - - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN SERVER: Failed to execute command of request No %zu: %s", - rq->request_id, rq->cmdline); - - exit(1); - break; - - case SPAWN_INSTANCE_TYPE_CALLBACK: - server->cb(rq); - exit(0); - break; - - default: - nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: unknown request type %u", rq->type); - exit(1); - } -} - -// -------------------------------------------------------------------------------------------------------------------- // Encoding and decoding of spawn server request argv type of data // Function to encode argv or envp @@ -457,54 +116,6 @@ static const char** argv_decode(const char *buffer, size_t size) { return argv; } -static BUFFER *argv_to_cmdline_buffer(const char **argv) { - BUFFER *wb = buffer_create(0, NULL); - - for(size_t i = 0; argv[i] ;i++) { - const char *s = argv[i]; - size_t len = strlen(s); - buffer_need_bytes(wb, len * 2 + 1); - - bool needs_quotes = false; - for(const char *c = s; !needs_quotes && *c ; c++) { - switch(*c) { - case ' ': - case '\v': - case '\t': - case '\n': - case '"': - needs_quotes = true; - break; - - default: - break; - } - } - - if(needs_quotes && buffer_strlen(wb)) - buffer_strcat(wb, " \""); - else - buffer_putc(wb, ' '); - - for(const char *c = s; *c ; c++) { - switch(*c) { - case '"': - buffer_putc(wb, '\\'); - // fall through - - default: - buffer_putc(wb, *c); - break; - } - } - - if(needs_quotes) - buffer_strcat(wb, "\""); - } - - return wb; -} - // -------------------------------------------------------------------------------------------------------------------- // status reports @@ -602,66 +213,206 @@ static void request_free(SPAWN_REQUEST *rq) { if(rq->fds[3] != -1) close(rq->fds[3]); if(rq->sock != -1) close(rq->sock); freez((void *)rq->argv); - freez((void *)rq->environment); + freez((void *)rq->envp); freez((void *)rq->data); freez((void *)rq->cmdline); freez((void *)rq); } -static void spawn_server_execute_request(SPAWN_SERVER *server, SPAWN_REQUEST *rq) { - switch(rq->type) { - case SPAWN_INSTANCE_TYPE_EXEC: - // close custom_fd - it is not needed for exec mode - if(rq->fds[3] != -1) { close(rq->fds[3]); rq->fds[3] = -1; } +static bool spawn_external_command(SPAWN_SERVER *server __maybe_unused, SPAWN_REQUEST *rq) { + // Close custom_fd - it is not needed for exec mode + if(rq->fds[3] != -1) { close(rq->fds[3]); rq->fds[3] = -1; } - // create the cmdline for logs - if(rq->argv) { - CLEAN_BUFFER *wb = argv_to_cmdline_buffer(rq->argv); - rq->cmdline = strdupz(buffer_tostring(wb)); - } - break; + if(!rq->argv) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: there is no argv pointer to exec"); + return false; + } - case SPAWN_INSTANCE_TYPE_CALLBACK: - if(server->cb == NULL) { - errno = ENOSYS; - spawn_server_send_status_failure(rq); - request_free(rq); - return; - } - rq->cmdline = strdupz("callback() function"); - break; + if(rq->fds[0] == -1 || rq->fds[1] == -1 || rq->fds[2] == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: stdio fds are missing from the request"); + return false; + } - default: - errno = EINVAL; - spawn_server_send_status_failure(rq); - request_free(rq); - return; + CLEAN_BUFFER *wb = argv_to_cmdline_buffer(rq->argv); + rq->cmdline = strdupz(buffer_tostring(wb)); + + posix_spawn_file_actions_t file_actions; + if (posix_spawn_file_actions_init(&file_actions) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: posix_spawn_file_actions_init() failed: %s", rq->cmdline); + return false; + } + + posix_spawn_file_actions_adddup2(&file_actions, rq->fds[0], STDIN_FILENO); + posix_spawn_file_actions_adddup2(&file_actions, rq->fds[1], STDOUT_FILENO); + posix_spawn_file_actions_adddup2(&file_actions, rq->fds[2], STDERR_FILENO); + posix_spawn_file_actions_addclose(&file_actions, rq->fds[0]); + posix_spawn_file_actions_addclose(&file_actions, rq->fds[1]); + posix_spawn_file_actions_addclose(&file_actions, rq->fds[2]); + + posix_spawnattr_t attr; + if (posix_spawnattr_init(&attr) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: posix_spawnattr_init() failed: %s", rq->cmdline); + posix_spawn_file_actions_destroy(&file_actions); + return false; + } + + // Set the flags to reset the signal mask and signal actions + sigset_t empty_mask; + sigemptyset(&empty_mask); + if (posix_spawnattr_setsigmask(&attr, &empty_mask) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: posix_spawnattr_setsigmask() failed: %s", rq->cmdline); + posix_spawn_file_actions_destroy(&file_actions); + posix_spawnattr_destroy(&attr); + return false; + } + + short flags = POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF; + if (posix_spawnattr_setflags(&attr, flags) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: posix_spawnattr_setflags() failed: %s", rq->cmdline); + posix_spawn_file_actions_destroy(&file_actions); + posix_spawnattr_destroy(&attr); + return false; + } + + int fds_to_keep[] = { + rq->fds[0], + rq->fds[1], + rq->fds[2], + nd_log_systemd_journal_fd(), + }; + os_close_all_non_std_open_fds_except(fds_to_keep, _countof(fds_to_keep), CLOSE_RANGE_CLOEXEC); + + errno_clear(); + if (posix_spawn(&rq->pid, rq->argv[0], &file_actions, &attr, (char * const *)rq->argv, (char * const *)rq->envp) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: posix_spawn() failed: %s", rq->cmdline); + + posix_spawnattr_destroy(&attr); + posix_spawn_file_actions_destroy(&file_actions); + return false; + } + + // Destroy the posix_spawnattr_t and posix_spawn_file_actions_t structures + posix_spawnattr_destroy(&attr); + posix_spawn_file_actions_destroy(&file_actions); + + // Close the read end of the stdin pipe and the write end of the stdout pipe in the parent process + close(rq->fds[0]); rq->fds[0] = -1; + close(rq->fds[1]); rq->fds[1] = -1; + close(rq->fds[2]); rq->fds[2] = -1; + + nd_log(NDLS_COLLECTORS, NDLP_DEBUG, "SPAWN SERVER: process created with pid %d: %s", rq->pid, rq->cmdline); + return true; +} + +static bool spawn_server_run_callback(SPAWN_SERVER *server __maybe_unused, SPAWN_REQUEST *rq) { + rq->cmdline = strdupz("callback() function"); + + if(server->cb == NULL) { + errno = ENOSYS; + return false; } pid_t pid = fork(); if (pid < 0) { // fork failed - nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to fork() child."); - spawn_server_send_status_failure(rq); - request_free(rq); - return; + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to fork() child for callback."); + return false; } else if (pid == 0) { // the child - spawn_server_run_child(server, rq); - exit(63); + // close the server sockets; + close(server->sock); server->sock = -1; + if(server->pipe[0] != -1) { close(server->pipe[0]); server->pipe[0] = -1; } + if(server->pipe[1] != -1) { close(server->pipe[1]); server->pipe[1] = -1; } + + // set the process name + os_setproctitle("spawn-callback", server->argc, server->argv); + + // close all open file descriptors of the parent, but keep ours + int fds_to_keep[] = { + rq->fds[0], + rq->fds[1], + rq->fds[2], + rq->fds[3], + nd_log_systemd_journal_fd(), + }; + os_close_all_non_std_open_fds_except(fds_to_keep, _countof(fds_to_keep), 0); + nd_log_reopen_log_files_for_spawn_server("spawn-callback"); + + // get the fds from the request + int stdin_fd = rq->fds[0]; + int stdout_fd = rq->fds[1]; + int stderr_fd = rq->fds[2]; + int custom_fd = rq->fds[3]; (void)custom_fd; + + // change stdio fds to the ones in the request + if (dup2(stdin_fd, STDIN_FILENO) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN SERVER: cannot dup2(%d) stdin of request No %zu: %s", + stdin_fd, rq->request_id, rq->cmdline); + exit(EXIT_FAILURE); + } + if (dup2(stdout_fd, STDOUT_FILENO) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN SERVER: cannot dup2(%d) stdin of request No %zu: %s", + stdout_fd, rq->request_id, rq->cmdline); + exit(EXIT_FAILURE); + } + if (dup2(stderr_fd, STDERR_FILENO) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN SERVER: cannot dup2(%d) stderr of request No %zu: %s", + stderr_fd, rq->request_id, rq->cmdline); + exit(EXIT_FAILURE); + } + + // close the excess fds + close(stdin_fd); stdin_fd = rq->fds[0] = STDIN_FILENO; + close(stdout_fd); stdout_fd = rq->fds[1] = STDOUT_FILENO; + close(stderr_fd); stderr_fd = rq->fds[2] = STDERR_FILENO; + + // overwrite the process environment + environ = (char **)rq->envp; + + // run the callback and return its code + exit(server->cb(rq)); } // the parent rq->pid = pid; + return true; +} + +static void spawn_server_execute_request(SPAWN_SERVER *server, SPAWN_REQUEST *rq) { + bool done; + switch(rq->type) { + case SPAWN_INSTANCE_TYPE_EXEC: + done = spawn_external_command(server, rq); + break; + + case SPAWN_INSTANCE_TYPE_CALLBACK: + done = spawn_server_run_callback(server, rq); + break; + + default: + errno = EINVAL; + done = false; + break; + } + + if(!done) { + spawn_server_send_status_failure(rq); + request_free(rq); + return; + } + // let the parent know spawn_server_send_status_success(rq); // do not keep data we don't need at the parent - freez((void *)rq->environment); rq->environment = NULL; + freez((void *)rq->envp); rq->envp = NULL; freez((void *)rq->argv); rq->argv = NULL; freez((void *)rq->data); rq->data = NULL; rq->data_size = 0; @@ -747,7 +498,7 @@ static bool spawn_server_send_request(ND_UUID *magic, SPAWN_REQUEST *request) { bool ret = false; size_t env_size = 0; - void *encoded_env = argv_encode(request->environment, &env_size); + void *encoded_env = argv_encode(request->envp, &env_size); if (!encoded_env) goto cleanup; @@ -974,7 +725,7 @@ static void spawn_server_receive_request(int sock, SPAWN_SERVER *server) { [2] = stderr_fd, [3] = custom_fd, }, - .environment = argv_decode(envp_encoded, env_size), + .envp = argv_decode(envp_encoded, env_size), .argv = argv_decode(argv_encoded, argv_size), .data = data, .data_size = data_size, @@ -1023,6 +774,8 @@ static SPAWN_REQUEST *find_request_by_pid(pid_t pid) { static void spawn_server_process_sigchld(void) { // nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN SERVER: checking for exited children"); + spawn_server_sigchld = false; + int status; pid_t pid; @@ -1039,36 +792,36 @@ static void spawn_server_process_sigchld(void) { if(WIFEXITED(status)) { if(WEXITSTATUS(status)) - nd_log(NDLS_COLLECTORS, NDLP_INFO, + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "SPAWN SERVER: child with pid %d (request %zu) exited with exit code %d: %s", pid, request_id, WEXITSTATUS(status), rq ? rq->cmdline : "[request not found]"); send_report_remove_request = true; } else if(WIFSIGNALED(status)) { if(WCOREDUMP(status)) - nd_log(NDLS_COLLECTORS, NDLP_INFO, + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "SPAWN SERVER: child with pid %d (request %zu) coredump'd due to signal %d: %s", pid, request_id, WTERMSIG(status), rq ? rq->cmdline : "[request not found]"); else - nd_log(NDLS_COLLECTORS, NDLP_INFO, + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "SPAWN SERVER: child with pid %d (request %zu) killed by signal %d: %s", pid, request_id, WTERMSIG(status), rq ? rq->cmdline : "[request not found]"); send_report_remove_request = true; } else if(WIFSTOPPED(status)) { - nd_log(NDLS_COLLECTORS, NDLP_INFO, + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "SPAWN SERVER: child with pid %d (request %zu) stopped due to signal %d: %s", pid, request_id, WSTOPSIG(status), rq ? rq->cmdline : "[request not found]"); send_report_remove_request = false; } else if(WIFCONTINUED(status)) { - nd_log(NDLS_COLLECTORS, NDLP_INFO, + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "SPAWN SERVER: child with pid %d (request %zu) continued due to signal %d: %s", pid, request_id, SIGCONT, rq ? rq->cmdline : "[request not found]"); send_report_remove_request = false; } else { - nd_log(NDLS_COLLECTORS, NDLP_INFO, + nd_log(NDLS_COLLECTORS, NDLP_WARNING, "SPAWN SERVER: child with pid %d (request %zu) reports unhandled status: %s", pid, request_id, rq ? rq->cmdline : "[request not found]"); send_report_remove_request = false; @@ -1082,20 +835,21 @@ static void spawn_server_process_sigchld(void) { } } -static void signals_unblock(void) { +static void posix_unmask_sigchld_on_thread(void) { sigset_t sigset; - sigfillset(&sigset); + sigemptyset(&sigset); // Initialize the signal set to empty + sigaddset(&sigset, SIGCHLD); // Add SIGCHLD to the set - if(pthread_sigmask(SIG_UNBLOCK, &sigset, NULL) == -1) { - netdata_log_error("SPAWN SERVER: Could not unblock signals for threads"); - } + if(pthread_sigmask(SIG_UNBLOCK, &sigset, NULL) != 0) + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN SERVER: cannot unmask SIGCHLD"); } -static void spawn_server_event_loop(SPAWN_SERVER *server) { +static int spawn_server_event_loop(SPAWN_SERVER *server) { int pipe_fd = server->pipe[1]; close(server->pipe[0]); server->pipe[0] = -1; - signals_unblock(); + posix_unmask_sigchld_on_thread(); // Set up the signal handler for SIGCHLD and SIGTERM struct sigaction sa; @@ -1104,13 +858,13 @@ static void spawn_server_event_loop(SPAWN_SERVER *server) { sa.sa_flags = SA_RESTART | SA_NOCLDSTOP; if (sigaction(SIGCHLD, &sa, NULL) == -1) { nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: sigaction() failed for SIGCHLD"); - exit(1); + return 1; } sa.sa_handler = spawn_server_sigterm_handler; if (sigaction(SIGTERM, &sa, NULL) == -1) { nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: sigaction() failed for SIGTERM"); - exit(1); + return 1; } struct status_report sr = { @@ -1121,7 +875,7 @@ static void spawn_server_event_loop(SPAWN_SERVER *server) { }; if (write(pipe_fd, &sr, sizeof(sr)) != sizeof(sr)) { nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: failed to write initial status report."); - exit(1); + return 1; } struct pollfd fds[2]; @@ -1131,13 +885,12 @@ static void spawn_server_event_loop(SPAWN_SERVER *server) { fds[1].events = POLLHUP | POLLERR; while(!spawn_server_exit) { - int ret = poll(fds, 2, -1); - if (spawn_server_sigchld) { - spawn_server_sigchld = false; + int ret = poll(fds, 2, 500); + if (spawn_server_sigchld || ret == 0) { spawn_server_process_sigchld(); errno_clear(); - if(ret == -1) + if(ret == -1 || ret == 0) continue; } @@ -1148,7 +901,7 @@ static void spawn_server_event_loop(SPAWN_SERVER *server) { if (fds[1].revents & (POLLHUP|POLLERR)) { // Pipe has been closed (parent has exited) - nd_log(NDLS_COLLECTORS, NDLP_DEBUG, "SPAWN SERVER: Parent process has exited"); + nd_log(NDLS_COLLECTORS, NDLP_DEBUG, "SPAWN SERVER: Parent process closed socket (exited?)"); break; } @@ -1185,7 +938,7 @@ static void spawn_server_event_loop(SPAWN_SERVER *server) { // nd_log(NDLS_COLLECTORS, NDLP_INFO, "SPAWN SERVER: all %zu children finished", killed); } - exit(1); + return 0; } // -------------------------------------------------------------------------------------------------------------------- @@ -1242,22 +995,24 @@ static bool spawn_server_create_listening_socket(SPAWN_SERVER *server) { } static void replace_stdio_with_dev_null() { + // we cannot log in this function - the logger is not yet initialized after fork() + int dev_null_fd = open("/dev/null", O_RDWR); if (dev_null_fd == -1) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to open /dev/null: %s", strerror(errno)); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to open /dev/null: %s", strerror(errno)); return; } // Redirect stdin (fd 0) if (dup2(dev_null_fd, STDIN_FILENO) == -1) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdin to /dev/null: %s", strerror(errno)); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdin to /dev/null: %s", strerror(errno)); close(dev_null_fd); return; } // Redirect stdout (fd 1) if (dup2(dev_null_fd, STDOUT_FILENO) == -1) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdout to /dev/null: %s", strerror(errno)); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: Failed to redirect stdout to /dev/null: %s", strerror(errno)); close(dev_null_fd); return; } @@ -1329,16 +1084,19 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options, const char *name if (pid == 0) { // the child - the spawn server - { - char buf[15]; - snprintfz(buf, sizeof(buf), "spawn-%s", server->name); - os_setproctitle(buf, server->argc, server->argv); - } + char buf[16]; + snprintfz(buf, sizeof(buf), "spawn-%s", server->name); + os_setproctitle(buf, server->argc, server->argv); replace_stdio_with_dev_null(); - os_close_all_non_std_open_fds_except((int[]){ server->sock, server->pipe[1] }, 2); - nd_log_reopen_log_files_for_spawn_server(); - spawn_server_event_loop(server); + int fds_to_keep[] = { + server->sock, + server->pipe[1], + nd_log_systemd_journal_fd(), + }; + os_close_all_non_std_open_fds_except(fds_to_keep, _countof(fds_to_keep), 0); + nd_log_reopen_log_files_for_spawn_server(buf); + exit(spawn_server_event_loop(server)); } else if (pid > 0) { // the parent @@ -1362,6 +1120,8 @@ SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options, const char *name goto cleanup; } + nd_log(NDLS_COLLECTORS, NDLP_DEBUG, "SPAWN SERVER: server created on pid %d", server->server_pid); + return server; } @@ -1383,6 +1143,21 @@ void spawn_server_exec_destroy(SPAWN_INSTANCE *instance) { freez(instance); } +static void log_invalid_magic(SPAWN_INSTANCE *instance, struct status_report *sr) { + unsigned char buf[sizeof(*sr) + 1]; + memcpy(buf, sr, sizeof(*sr)); + buf[sizeof(buf) - 1] = '\0'; + + for(size_t i = 0; i < sizeof(buf) - 1; i++) { + if (iscntrl(buf[i]) || !isprint(buf[i])) + buf[i] = '_'; + } + + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: invalid final status report for child %d, request %zu (invalid magic %#x in response, reads like '%s')", + instance->child_pid, instance->request_id, sr->magic, buf); +} + int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *instance) { int rc = -1; @@ -1397,24 +1172,24 @@ int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE * "SPAWN PARENT: failed to read final status report for child %d, request %zu", instance->child_pid, instance->request_id); - else if(sr.magic != STATUS_REPORT_MAGIC) { - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN PARENT: invalid final status report for child %d, request %zu (invalid magic %#x in response)", - instance->child_pid, instance->request_id, sr.magic); - } - else switch(sr.status) { - case STATUS_REPORT_EXITED: - rc = sr.exited.waitpid_status; - break; + else if(sr.magic != STATUS_REPORT_MAGIC) + log_invalid_magic(instance, &sr); + else { + switch (sr.status) { + case STATUS_REPORT_EXITED: + rc = sr.exited.waitpid_status; + break; - case STATUS_REPORT_STARTED: - case STATUS_REPORT_FAILED: - default: - errno = 0; - nd_log(NDLS_COLLECTORS, NDLP_ERR, - "SPAWN PARENT: invalid status report to exec spawn request %zu for pid %d (status = %u)", - instance->request_id, instance->child_pid, sr.status); - break; + case STATUS_REPORT_STARTED: + case STATUS_REPORT_FAILED: + default: + errno = 0; + nd_log( + NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: invalid status report to exec spawn request %zu for pid %d (status = %u)", + instance->request_id, instance->child_pid, sr.status); + break; + } } instance->child_pid = 0; @@ -1458,7 +1233,7 @@ SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custo [2] = stderr_fd, [3] = custom_fd, }, - .environment = (const char **)environ, + .envp = (const char **)environ, .argv = argv, .data = data, .data_size = data_size, @@ -1530,4 +1305,4 @@ cleanup: return NULL; } -#endif // !OS_WINDOWS +#endif diff --git a/src/libnetdata/spawn_server/spawn_server_posix.c b/src/libnetdata/spawn_server/spawn_server_posix.c new file mode 100644 index 000000000..f96921bb9 --- /dev/null +++ b/src/libnetdata/spawn_server/spawn_server_posix.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "spawn_server_internals.h" + +#if defined(SPAWN_SERVER_VERSION_POSIX_SPAWN) + +#ifdef __APPLE__ +#include <crt_externs.h> +#define environ (*_NSGetEnviron()) +#else +extern char **environ; +#endif + +int spawn_server_instance_read_fd(SPAWN_INSTANCE *si) { return si->read_fd; } +int spawn_server_instance_write_fd(SPAWN_INSTANCE *si) { return si->write_fd; } +void spawn_server_instance_read_fd_unset(SPAWN_INSTANCE *si) { si->read_fd = -1; } +void spawn_server_instance_write_fd_unset(SPAWN_INSTANCE *si) { si->write_fd = -1; } +pid_t spawn_server_instance_pid(SPAWN_INSTANCE *si) { return si->child_pid; } + +static struct { + bool sigchld_initialized; + SPINLOCK spinlock; + SPAWN_INSTANCE *instances; +} spawn_globals = { + .spinlock = NETDATA_SPINLOCK_INITIALIZER, + .instances = NULL, +}; + +//static void sigchld_handler(int signum __maybe_unused) { +// pid_t pid; +// int status; +// +// while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { +// // Find the SPAWN_INSTANCE corresponding to this pid +// spinlock_lock(&spawn_globals.spinlock); +// for(SPAWN_INSTANCE *si = spawn_globals.instances; si ;si = si->next) { +// if (si->child_pid == pid) { +// __atomic_store_n(&si->waitpid_status, status, __ATOMIC_RELAXED); +// __atomic_store_n(&si->exited, true, __ATOMIC_RELAXED); +// break; +// } +// } +// spinlock_unlock(&spawn_globals.spinlock); +// } +//} + +SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options __maybe_unused, const char *name, spawn_request_callback_t cb __maybe_unused, int argc __maybe_unused, const char **argv __maybe_unused) { + SPAWN_SERVER* server = callocz(1, sizeof(SPAWN_SERVER)); + + if (name) + server->name = strdupz(name); + else + server->name = strdupz("unnamed"); + + if(!spawn_globals.sigchld_initialized) { + spawn_globals.sigchld_initialized = true; + +// struct sigaction sa; +// sa.sa_handler = sigchld_handler; +// sigemptyset(&sa.sa_mask); +// sa.sa_flags = SA_RESTART | SA_NOCLDSTOP; +// if (sigaction(SIGCHLD, &sa, NULL) == -1) { +// nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: Failed to set SIGCHLD handler"); +// freez((void *)server->name); +// freez(server); +// return NULL; +// } + } + + return server; +} + +void spawn_server_destroy(SPAWN_SERVER *server) { + if (!server) return; + freez((void *)server->name); + freez(server); +} + +SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd, int custom_fd __maybe_unused, const char **argv, const void *data __maybe_unused, size_t data_size __maybe_unused, SPAWN_INSTANCE_TYPE type) { + if (type != SPAWN_INSTANCE_TYPE_EXEC) + return NULL; + + CLEAN_BUFFER *cmdline_wb = argv_to_cmdline_buffer(argv); + const char *cmdline = buffer_tostring(cmdline_wb); + + SPAWN_INSTANCE *si = callocz(1, sizeof(SPAWN_INSTANCE)); + si->child_pid = -1; + si->request_id = __atomic_add_fetch(&server->request_id, 1, __ATOMIC_RELAXED); + + int stdin_pipe[2] = { -1, -1 }; + int stdout_pipe[2] = { -1, -1 }; + + if (pipe(stdin_pipe) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: stdin pipe() failed: %s", cmdline); + freez(si); + return NULL; + } + + if (pipe(stdout_pipe) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: stdout pipe() failed: %s", cmdline); + close(stdin_pipe[PIPE_READ]); + close(stdin_pipe[PIPE_WRITE]); + freez(si); + return NULL; + } + + posix_spawn_file_actions_t file_actions; + posix_spawnattr_t attr; + + if (posix_spawn_file_actions_init(&file_actions) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: posix_spawn_file_actions_init() failed: %s", cmdline); + close(stdin_pipe[PIPE_READ]); + close(stdin_pipe[PIPE_WRITE]); + close(stdout_pipe[PIPE_READ]); + close(stdout_pipe[PIPE_WRITE]); + freez(si); + return NULL; + } + + posix_spawn_file_actions_adddup2(&file_actions, stdin_pipe[PIPE_READ], STDIN_FILENO); + posix_spawn_file_actions_adddup2(&file_actions, stdout_pipe[PIPE_WRITE], STDOUT_FILENO); + posix_spawn_file_actions_addclose(&file_actions, stdin_pipe[PIPE_READ]); + posix_spawn_file_actions_addclose(&file_actions, stdin_pipe[PIPE_WRITE]); + posix_spawn_file_actions_addclose(&file_actions, stdout_pipe[PIPE_READ]); + posix_spawn_file_actions_addclose(&file_actions, stdout_pipe[PIPE_WRITE]); + if(stderr_fd != STDERR_FILENO) { + posix_spawn_file_actions_adddup2(&file_actions, stderr_fd, STDERR_FILENO); + posix_spawn_file_actions_addclose(&file_actions, stderr_fd); + } + + if (posix_spawnattr_init(&attr) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: posix_spawnattr_init() failed: %s", cmdline); + posix_spawn_file_actions_destroy(&file_actions); + close(stdin_pipe[PIPE_READ]); + close(stdin_pipe[PIPE_WRITE]); + close(stdout_pipe[PIPE_READ]); + close(stdout_pipe[PIPE_WRITE]); + freez(si); + return NULL; + } + + // Set the flags to reset the signal mask and signal actions + sigset_t empty_mask; + sigemptyset(&empty_mask); + if (posix_spawnattr_setsigmask(&attr, &empty_mask) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: posix_spawnattr_setsigmask() failed: %s", cmdline); + posix_spawn_file_actions_destroy(&file_actions); + posix_spawnattr_destroy(&attr); + return false; + } + + short flags = POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF; + if (posix_spawnattr_setflags(&attr, flags) != 0) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: posix_spawnattr_setflags() failed: %s", cmdline); + posix_spawn_file_actions_destroy(&file_actions); + posix_spawnattr_destroy(&attr); + return false; + } + + spinlock_lock(&spawn_globals.spinlock); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(spawn_globals.instances, si, prev, next); + spinlock_unlock(&spawn_globals.spinlock); + + // unfortunately, on CYGWIN/MSYS posix_spawn() is not thread safe + // so, we run it one by one. + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER; + spinlock_lock(&spinlock); + + int fds[3] = { stdin_pipe[PIPE_READ], stdout_pipe[PIPE_WRITE], stderr_fd }; + os_close_all_non_std_open_fds_except(fds, 3, CLOSE_RANGE_CLOEXEC); + + errno_clear(); + if (posix_spawn(&si->child_pid, argv[0], &file_actions, &attr, (char * const *)argv, environ) != 0) { + spinlock_unlock(&spinlock); + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN SERVER: posix_spawn() failed: %s", cmdline); + + spinlock_lock(&spawn_globals.spinlock); + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(spawn_globals.instances, si, prev, next); + spinlock_unlock(&spawn_globals.spinlock); + + posix_spawnattr_destroy(&attr); + posix_spawn_file_actions_destroy(&file_actions); + + close(stdin_pipe[PIPE_READ]); + close(stdin_pipe[PIPE_WRITE]); + close(stdout_pipe[PIPE_READ]); + close(stdout_pipe[PIPE_WRITE]); + freez(si); + return NULL; + } + spinlock_unlock(&spinlock); + + // Destroy the posix_spawnattr_t and posix_spawn_file_actions_t structures + posix_spawnattr_destroy(&attr); + posix_spawn_file_actions_destroy(&file_actions); + + // Close the read end of the stdin pipe and the write end of the stdout pipe in the parent process + close(stdin_pipe[PIPE_READ]); + close(stdout_pipe[PIPE_WRITE]); + + si->write_fd = stdin_pipe[PIPE_WRITE]; + si->read_fd = stdout_pipe[PIPE_READ]; + si->cmdline = strdupz(cmdline); + + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN SERVER: process created with pid %d: %s", + si->child_pid, cmdline); + return si; +} + +int spawn_server_exec_kill(SPAWN_SERVER *server, SPAWN_INSTANCE *si) { + if (!si) return -1; + + if (kill(si->child_pid, SIGTERM)) + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: kill() of pid %d failed: %s", + si->child_pid, si->cmdline); + + return spawn_server_exec_wait(server, si); +} + +static int spawn_server_waitpid(SPAWN_INSTANCE *si) { + int status; + pid_t pid; + + pid = waitpid(si->child_pid, &status, 0); + + if(pid != si->child_pid) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: failed to wait for pid %d: %s", + si->child_pid, si->cmdline); + + return -1; + } + + errno_clear(); + + if(WIFEXITED(status)) { + if(WEXITSTATUS(status)) + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN SERVER: child with pid %d (request %zu) exited with exit code %d: %s", + pid, si->request_id, WEXITSTATUS(status), si->cmdline); + } + else if(WIFSIGNALED(status)) { + if(WCOREDUMP(status)) + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN SERVER: child with pid %d (request %zu) coredump'd due to signal %d: %s", + pid, si->request_id, WTERMSIG(status), si->cmdline); + else + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN SERVER: child with pid %d (request %zu) killed by signal %d: %s", + pid, si->request_id, WTERMSIG(status), si->cmdline); + } + else if(WIFSTOPPED(status)) { + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN SERVER: child with pid %d (request %zu) stopped due to signal %d: %s", + pid, si->request_id, WSTOPSIG(status), si->cmdline); + } + else if(WIFCONTINUED(status)) { + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN SERVER: child with pid %d (request %zu) continued due to signal %d: %s", + pid, si->request_id, SIGCONT, si->cmdline); + } + else { + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN SERVER: child with pid %d (request %zu) reports unhandled status: %s", + pid, si->request_id, si->cmdline); + } + + return status; +} + +int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *si) { + if (!si) return -1; + + // Close all pipe descriptors to force the child to exit + if (si->read_fd != -1) close(si->read_fd); + if (si->write_fd != -1) close(si->write_fd); + + // Wait for the process to exit + int status = __atomic_load_n(&si->waitpid_status, __ATOMIC_RELAXED); + bool exited = __atomic_load_n(&si->exited, __ATOMIC_RELAXED); + if(!exited) + status = spawn_server_waitpid(si); + else + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN PARENT: child with pid %d exited with status %d (sighandler): %s", + si->child_pid, status, si->cmdline); + + spinlock_lock(&spawn_globals.spinlock); + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(spawn_globals.instances, si, prev, next); + spinlock_unlock(&spawn_globals.spinlock); + + freez((void *)si->cmdline); + freez(si); + return status; +} + +#endif diff --git a/src/libnetdata/spawn_server/spawn_server_windows.c b/src/libnetdata/spawn_server/spawn_server_windows.c new file mode 100644 index 000000000..f80925a24 --- /dev/null +++ b/src/libnetdata/spawn_server/spawn_server_windows.c @@ -0,0 +1,456 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "spawn_server_internals.h" + +#if defined(SPAWN_SERVER_VERSION_WINDOWS) + +int spawn_server_instance_read_fd(SPAWN_INSTANCE *si) { return si->read_fd; } +int spawn_server_instance_write_fd(SPAWN_INSTANCE *si) { return si->write_fd; } +void spawn_server_instance_read_fd_unset(SPAWN_INSTANCE *si) { si->read_fd = -1; } +void spawn_server_instance_write_fd_unset(SPAWN_INSTANCE *si) { si->write_fd = -1; } + +pid_t spawn_server_instance_pid(SPAWN_INSTANCE *si) { + if(si->child_pid != -1) + return si->child_pid; + + return (pid_t)si->dwProcessId; +} + +static void update_cygpath_env(void) { + static volatile bool done = false; + + if(done) return; + done = true; + + char win_path[MAX_PATH]; + + // Convert Cygwin root path to Windows path + cygwin_conv_path(CCP_POSIX_TO_WIN_A, "/", win_path, sizeof(win_path)); + + nd_setenv("NETDATA_CYGWIN_BASE_PATH", win_path, 1); + + nd_log(NDLS_COLLECTORS, NDLP_INFO, "Cygwin/MSYS2 base path set to '%s'", win_path); +} + +SPAWN_SERVER* spawn_server_create(SPAWN_SERVER_OPTIONS options __maybe_unused, const char *name, spawn_request_callback_t cb __maybe_unused, int argc __maybe_unused, const char **argv __maybe_unused) { + update_cygpath_env(); + + SPAWN_SERVER* server = callocz(1, sizeof(SPAWN_SERVER)); + if(name) + server->name = strdupz(name); + else + server->name = strdupz("unnamed"); + + server->log_forwarder = log_forwarder_start(); + + return server; +} + +void spawn_server_destroy(SPAWN_SERVER *server) { + if (server) { + if (server->log_forwarder) { + log_forwarder_stop(server->log_forwarder); + server->log_forwarder = NULL; + } + freez((void *)server->name); + freez(server); + } +} + +static BUFFER *argv_to_windows(const char **argv) { + BUFFER *wb = buffer_create(0, NULL); + + // argv[0] is the path + char b[strlen(argv[0]) * 2 + FILENAME_MAX]; + cygwin_conv_path(CCP_POSIX_TO_WIN_A | CCP_ABSOLUTE, argv[0], b, sizeof(b)); + + for(size_t i = 0; argv[i] ;i++) { + const char *s = (i == 0) ? b : argv[i]; + size_t len = strlen(s); + buffer_need_bytes(wb, len * 2 + 1); + + bool needs_quotes = false; + for(const char *c = s; !needs_quotes && *c ; c++) { + switch(*c) { + case ' ': + case '\v': + case '\t': + case '\n': + case '"': + needs_quotes = true; + break; + + default: + break; + } + } + + if(buffer_strlen(wb)) { + if (needs_quotes) + buffer_strcat(wb, " \""); + else + buffer_putc(wb, ' '); + } + else if (needs_quotes) + buffer_putc(wb, '"'); + + for(const char *c = s; *c ; c++) { + switch(*c) { + case '"': + buffer_putc(wb, '\\'); + // fall through + + default: + buffer_putc(wb, *c); + break; + } + } + + if(needs_quotes) + buffer_strcat(wb, "\""); + } + + return wb; +} + +int set_fd_blocking(int fd) { + int flags = fcntl(fd, F_GETFL, 0); + if (flags == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: fcntl(F_GETFL) failed"); + return -1; + } + + flags &= ~O_NONBLOCK; + if (fcntl(fd, F_SETFL, flags) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, "SPAWN PARENT: fcntl(F_SETFL) failed"); + return -1; + } + + return 0; +} + +//static void print_environment_block(char *env_block) { +// if (env_block == NULL) { +// fprintf(stderr, "Environment block is NULL\n"); +// return; +// } +// +// char *env = env_block; +// while (*env) { +// fprintf(stderr, "ENVIRONMENT: %s\n", env); +// // Move to the next string in the block +// env += strlen(env) + 1; +// } +//} + +SPAWN_INSTANCE* spawn_server_exec(SPAWN_SERVER *server, int stderr_fd __maybe_unused, int custom_fd __maybe_unused, const char **argv, const void *data __maybe_unused, size_t data_size __maybe_unused, SPAWN_INSTANCE_TYPE type) { + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER; + + if (type != SPAWN_INSTANCE_TYPE_EXEC) + return NULL; + + int pipe_stdin[2] = { -1, -1 }, pipe_stdout[2] = { -1, -1 }, pipe_stderr[2] = { -1, -1 }; + + errno_clear(); + + SPAWN_INSTANCE *instance = callocz(1, sizeof(*instance)); + instance->request_id = __atomic_add_fetch(&server->request_id, 1, __ATOMIC_RELAXED); + + CLEAN_BUFFER *wb = argv_to_windows(argv); + char *command = (char *)buffer_tostring(wb); + + if (pipe(pipe_stdin) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: Cannot create stdin pipe() for request No %zu, command: %s", + instance->request_id, command); + goto cleanup; + } + + if (pipe(pipe_stdout) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: Cannot create stdout pipe() for request No %zu, command: %s", + instance->request_id, command); + goto cleanup; + } + + if (pipe(pipe_stderr) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: Cannot create stderr pipe() for request No %zu, command: %s", + instance->request_id, command); + goto cleanup; + } + + // Ensure pipes are in blocking mode + if (set_fd_blocking(pipe_stdin[PIPE_READ]) == -1 || set_fd_blocking(pipe_stdin[PIPE_WRITE]) == -1 || + set_fd_blocking(pipe_stdout[PIPE_READ]) == -1 || set_fd_blocking(pipe_stdout[PIPE_WRITE]) == -1 || + set_fd_blocking(pipe_stderr[PIPE_READ]) == -1 || set_fd_blocking(pipe_stderr[PIPE_WRITE]) == -1) { + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: Failed to set blocking I/O on pipes for request No %zu, command: %s", + instance->request_id, command); + goto cleanup; + } + + // do not run multiple times this section + // to prevent handles leaking + spinlock_lock(&spinlock); + + // Convert POSIX file descriptors to Windows handles + HANDLE stdin_read_handle = (HANDLE)_get_osfhandle(pipe_stdin[PIPE_READ]); + HANDLE stdout_write_handle = (HANDLE)_get_osfhandle(pipe_stdout[PIPE_WRITE]); + HANDLE stderr_write_handle = (HANDLE)_get_osfhandle(pipe_stderr[PIPE_WRITE]); + + if (stdin_read_handle == INVALID_HANDLE_VALUE || stdout_write_handle == INVALID_HANDLE_VALUE || stderr_write_handle == INVALID_HANDLE_VALUE) { + spinlock_unlock(&spinlock); + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: Invalid handle value(s) for request No %zu, command: %s", + instance->request_id, command); + goto cleanup; + } + + // Set handle inheritance + if (!SetHandleInformation(stdin_read_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) || + !SetHandleInformation(stdout_write_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT) || + !SetHandleInformation(stderr_write_handle, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT)) { + spinlock_unlock(&spinlock); + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: Cannot set handle(s) inheritance for request No %zu, command: %s", + instance->request_id, command); + goto cleanup; + } + + // Set up the STARTUPINFO structure + STARTUPINFO si; + PROCESS_INFORMATION pi; + ZeroMemory(&si, sizeof(si)); + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdInput = stdin_read_handle; + si.hStdOutput = stdout_write_handle; + si.hStdError = stderr_write_handle; + + // Retrieve the current environment block + char* env_block = GetEnvironmentStrings(); +// print_environment_block(env_block); + + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN PARENT: Running request No %zu, command: '%s'", + instance->request_id, command); + + int fds_to_keep_open[] = { pipe_stdin[PIPE_READ], pipe_stdout[PIPE_WRITE], pipe_stderr[PIPE_WRITE] }; + os_close_all_non_std_open_fds_except(fds_to_keep_open, 3, CLOSE_RANGE_CLOEXEC); + + // Spawn the process + errno_clear(); + if (!CreateProcess(NULL, command, NULL, NULL, TRUE, 0, env_block, NULL, &si, &pi)) { + spinlock_unlock(&spinlock); + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: cannot CreateProcess() for request No %zu, command: %s", + instance->request_id, command); + goto cleanup; + } + + FreeEnvironmentStrings(env_block); + + // When we create a process with the CreateProcess function, it returns two handles: + // - one for the process (pi.hProcess) and + // - one for the primary thread of the new process (pi.hThread). + // Both of these handles need to be explicitly closed when they are no longer needed. + CloseHandle(pi.hThread); + + // end of the critical section + spinlock_unlock(&spinlock); + + // Close unused pipe ends + close(pipe_stdin[PIPE_READ]); pipe_stdin[PIPE_READ] = -1; + close(pipe_stdout[PIPE_WRITE]); pipe_stdout[PIPE_WRITE] = -1; + close(pipe_stderr[PIPE_WRITE]); pipe_stderr[PIPE_WRITE] = -1; + + // Store process information in instance + instance->dwProcessId = pi.dwProcessId; + instance->child_pid = cygwin_winpid_to_pid((pid_t)pi.dwProcessId); + instance->process_handle = pi.hProcess; + + // Convert handles to POSIX file descriptors + instance->write_fd = pipe_stdin[PIPE_WRITE]; + instance->read_fd = pipe_stdout[PIPE_READ]; + instance->stderr_fd = pipe_stderr[PIPE_READ]; + + // Add stderr_fd to the log forwarder + log_forwarder_add_fd(server->log_forwarder, instance->stderr_fd); + log_forwarder_annotate_fd_name(server->log_forwarder, instance->stderr_fd, command); + log_forwarder_annotate_fd_pid(server->log_forwarder, instance->stderr_fd, spawn_server_instance_pid(instance)); + + errno_clear(); + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN PARENT: created process for request No %zu, pid %d (winpid %d), command: %s", + instance->request_id, (int)instance->child_pid, (int)pi.dwProcessId, command); + + return instance; + + cleanup: + if (pipe_stdin[PIPE_READ] >= 0) close(pipe_stdin[PIPE_READ]); + if (pipe_stdin[PIPE_WRITE] >= 0) close(pipe_stdin[PIPE_WRITE]); + if (pipe_stdout[PIPE_READ] >= 0) close(pipe_stdout[PIPE_READ]); + if (pipe_stdout[PIPE_WRITE] >= 0) close(pipe_stdout[PIPE_WRITE]); + if (pipe_stderr[PIPE_READ] >= 0) close(pipe_stderr[PIPE_READ]); + if (pipe_stderr[PIPE_WRITE] >= 0) close(pipe_stderr[PIPE_WRITE]); + freez(instance); + return NULL; +} + +static char* GetErrorString(DWORD errorCode) { + DWORD lastError = GetLastError(); + + LPVOID lpMsgBuf; + DWORD bufLen = FormatMessage( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + errorCode, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPTSTR) &lpMsgBuf, + 0, NULL ); + + SetLastError(lastError); + + if (bufLen) { + char* errorString = (char*)LocalAlloc(LMEM_FIXED, bufLen + 1); + if (errorString) { + strcpy(errorString, (char*)lpMsgBuf); + } + LocalFree(lpMsgBuf); + return errorString; + } + + return NULL; +} + +static void TerminateChildProcesses(SPAWN_INSTANCE *si) { + HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if (hSnapshot == INVALID_HANDLE_VALUE) + return; + + PROCESSENTRY32 pe; + pe.dwSize = sizeof(PROCESSENTRY32); + + if (Process32First(hSnapshot, &pe)) { + do { + if (pe.th32ParentProcessID == si->dwProcessId) { + HANDLE hChildProcess = OpenProcess(PROCESS_TERMINATE, FALSE, pe.th32ProcessID); + if (hChildProcess) { + nd_log(NDLS_COLLECTORS, NDLP_WARNING, + "SPAWN PARENT: killing subprocess %u of request No %zu, pid %d (winpid %u)", + pe.th32ProcessID, si->request_id, (int)si->child_pid, si->dwProcessId); + + TerminateProcess(hChildProcess, STATUS_CONTROL_C_EXIT); + CloseHandle(hChildProcess); + } + } + } while (Process32Next(hSnapshot, &pe)); + } + + CloseHandle(hSnapshot); +} + +int map_status_code_to_signal(DWORD status_code) { + switch (status_code) { + case STATUS_ACCESS_VIOLATION: + return SIGSEGV; + case STATUS_ILLEGAL_INSTRUCTION: + return SIGILL; + case STATUS_FLOAT_DIVIDE_BY_ZERO: + case STATUS_INTEGER_DIVIDE_BY_ZERO: + case STATUS_ARRAY_BOUNDS_EXCEEDED: + case STATUS_FLOAT_OVERFLOW: + case STATUS_FLOAT_UNDERFLOW: + case STATUS_FLOAT_INVALID_OPERATION: + return SIGFPE; + case STATUS_BREAKPOINT: + case STATUS_SINGLE_STEP: + return SIGTRAP; + case STATUS_STACK_OVERFLOW: + case STATUS_INVALID_HANDLE: + case STATUS_INVALID_PARAMETER: + case STATUS_NO_MEMORY: + case STATUS_PRIVILEGED_INSTRUCTION: + case STATUS_DLL_NOT_FOUND: + case STATUS_DLL_INIT_FAILED: + case STATUS_ORDINAL_NOT_FOUND: + case STATUS_ENTRYPOINT_NOT_FOUND: + case STATUS_CONTROL_STACK_VIOLATION: + case STATUS_STACK_BUFFER_OVERRUN: + case STATUS_ASSERTION_FAILURE: + case STATUS_INVALID_CRUNTIME_PARAMETER: + case STATUS_HEAP_CORRUPTION: + return SIGABRT; + case STATUS_CONTROL_C_EXIT: + return SIGTERM; // we use this internally as such + case STATUS_FATAL_APP_EXIT: + return SIGTERM; + default: + return (status_code & 0xFF) << 8; + } +} + +int spawn_server_exec_kill(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *si) { + if(si->child_pid != -1 && kill(si->child_pid, SIGTERM) != 0) + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: child of request No %zu, pid %d (winpid %u), failed to be killed", + si->request_id, (int)si->child_pid, si->dwProcessId); + + // this gives some warnings at the spawn-tester, but it is generally better + // to have them, to avoid abnormal shutdown of the plugins + if(si->read_fd != -1) { close(si->read_fd); si->read_fd = -1; } + if(si->write_fd != -1) { close(si->write_fd); si->write_fd = -1; } + if(si->stderr_fd != -1) { + if(!log_forwarder_del_and_close_fd(server->log_forwarder, si->stderr_fd)) + close(si->stderr_fd); + + si->stderr_fd = -1; + } + + errno_clear(); + if(TerminateProcess(si->process_handle, STATUS_CONTROL_C_EXIT) == 0) + nd_log(NDLS_COLLECTORS, NDLP_ERR, + "SPAWN PARENT: child of request No %zu, pid %d (winpid %u), failed to be terminated", + si->request_id, (int)si->child_pid, si->dwProcessId); + + errno_clear(); + TerminateChildProcesses(si); + + return spawn_server_exec_wait(server, si); +} + +int spawn_server_exec_wait(SPAWN_SERVER *server __maybe_unused, SPAWN_INSTANCE *si) { + if(si->read_fd != -1) { close(si->read_fd); si->read_fd = -1; } + if(si->write_fd != -1) { close(si->write_fd); si->write_fd = -1; } + if(si->stderr_fd != -1) { + if(!log_forwarder_del_and_close_fd(server->log_forwarder, si->stderr_fd)) + close(si->stderr_fd); + + si->stderr_fd = -1; + } + + // wait for the process to end + WaitForSingleObject(si->process_handle, INFINITE); + + DWORD exit_code = -1; + GetExitCodeProcess(si->process_handle, &exit_code); + CloseHandle(si->process_handle); + + char *err = GetErrorString(exit_code); + + nd_log(NDLS_COLLECTORS, NDLP_INFO, + "SPAWN PARENT: child of request No %zu, pid %d (winpid %u), exited with code %u (0x%x): %s", + si->request_id, (int)si->child_pid, si->dwProcessId, + (unsigned)exit_code, (unsigned)exit_code, err ? err : "(no reason text)"); + + if(err) + LocalFree(err); + + freez(si); + return map_status_code_to_signal(exit_code); +} + +#endif diff --git a/src/libnetdata/statistical/README.md b/src/libnetdata/statistical/README.md index 1d1d2afd4..03a91dc45 100644 --- a/src/libnetdata/statistical/README.md +++ b/src/libnetdata/statistical/README.md @@ -1,12 +1,3 @@ -<!-- -title: "Statistical functions" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/statistical/README.md -sidebar_label: "Statistical functions" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Statistical functions A library for easy and fast calculations of statistical measurements like average, median etc. diff --git a/src/libnetdata/storage_number/README.md b/src/libnetdata/storage_number/README.md index f0096fb9b..360d44a0a 100644 --- a/src/libnetdata/storage_number/README.md +++ b/src/libnetdata/storage_number/README.md @@ -1,12 +1,3 @@ -<!-- -title: "Netdata storage number" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/storage_number/README.md -sidebar_label: "Storage number" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Netdata storage number Although `netdata` does all its calculations using `long double`, it stores all values using diff --git a/src/libnetdata/string/README.md b/src/libnetdata/string/README.md index 54c905946..c23160233 100644 --- a/src/libnetdata/string/README.md +++ b/src/libnetdata/string/README.md @@ -1,12 +1,3 @@ -<!-- -title: "String" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/string/README.md -sidebar_label: "String" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # STRING STRING provides a way to allocate and free text strings, while de-duplicating them. diff --git a/src/libnetdata/string/string.c b/src/libnetdata/string/string.c index 257a3cc4b..107c7eea5 100644 --- a/src/libnetdata/string/string.c +++ b/src/libnetdata/string/string.c @@ -347,16 +347,34 @@ void string_freez(STRING *string) { string_stats_atomic_increment(partition, releases); } -inline size_t string_strlen(STRING *string) { +inline size_t string_strlen(const STRING *string) { if(unlikely(!string)) return 0; return string->length - 1; } -inline const char *string2str(STRING *string) { +inline const char *string2str(const STRING *string) { if(unlikely(!string)) return ""; return string->str; } +bool string_ends_with_string(const STRING *whole, const STRING *end) { + if(whole == end) return true; + if(!whole || !end) return false; + if(end->length > whole->length) return false; + if(end->length == whole->length) return strcmp(string2str(whole), string2str(end)) == 0; + const char *we = string2str(whole); + we = &we[string_strlen(whole) - string_strlen(end)]; + return strncmp(we, end->str, string_strlen(end)) == 0; +} + +bool string_starts_with_string(const STRING *whole, const STRING *end) { + if(whole == end) return true; + if(!whole || !end) return false; + if(end->length > whole->length) return false; + if(end->length == whole->length) return strcmp(string2str(whole), string2str(end)) == 0; + return strncmp(string2str(whole), string2str(end), string_strlen(end)) == 0; +} + STRING *string_2way_merge(STRING *a, STRING *b) { static STRING *X = NULL; diff --git a/src/libnetdata/string/string.h b/src/libnetdata/string/string.h index c44696be2..e86ac6fb5 100644 --- a/src/libnetdata/string/string.h +++ b/src/libnetdata/string/string.h @@ -14,8 +14,10 @@ STRING *string_strndupz(const char *str, size_t len); STRING *string_dup(STRING *string); void string_freez(STRING *string); -size_t string_strlen(STRING *string); -const char *string2str(STRING *string) NEVERNULL; +size_t string_strlen(const STRING *string); +const char *string2str(const STRING *string) NEVERNULL; +bool string_ends_with_string(const STRING *whole, const STRING *end); +bool string_starts_with_string(const STRING *whole, const STRING *end); // keep common prefix/suffix and replace everything else with [x] STRING *string_2way_merge(STRING *a, STRING *b); @@ -30,10 +32,21 @@ static inline int string_strcmp(STRING *string, const char *s) { return strcmp(string2str(string), s); } +static inline int string_strncmp(STRING *string, const char *s, size_t n) { + return strncmp(string2str(string), s, n); +} + void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases); int string_unittest(size_t entries); void string_init(void); +static inline void cleanup_string_pp(STRING **stringpp) { + if(stringpp) + string_freez(*stringpp); +} + +#define CLEAN_STRING _cleanup_(cleanup_string_pp) STRING + #endif diff --git a/src/libnetdata/string/utf8.c b/src/libnetdata/string/utf8.c new file mode 100644 index 000000000..0b4f138a6 --- /dev/null +++ b/src/libnetdata/string/utf8.c @@ -0,0 +1,408 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "../libnetdata.h" + +#if defined(OS_WINDOWS) +/* + * Convert any CodePage to UTF16 + * Goals: + * 1. Destination is always NULL terminated + * 2. If the destination buffer is not enough, return as much as possible data (truncate) + * 3. Always return the number of wide characters written, including the null terminator + */ + +size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len, bool *truncated) { + if(!src || src_len == 0) { + // invalid input + if(truncated) + *truncated = true; + + if(dst && dst_size) + *dst = L'\0'; + return 0; + } + + if(!dst || !dst_size) { + // the caller wants to know the buffer to allocate for the conversion + + if(truncated) + *truncated = true; + + int required = MultiByteToWideChar(CodePage, 0, src, src_len, NULL, 0); + if(required <= 0) return 0; // error in the conversion + + // Add 1 for null terminator only if src_len is not -1 + // so that the caller can call us again to get the entire string (not truncated) + return (size_t)required + ((src_len != -1) ? 1 : 0); + } + + // do the conversion directly to the destination buffer + int rc = MultiByteToWideChar(CodePage, 0, src, src_len, dst, (int)dst_size); + if(rc <= 0) { + if(truncated) + *truncated = true; + + // conversion failed, let's see why... + DWORD status = GetLastError(); + if(status == ERROR_INSUFFICIENT_BUFFER) { + // it cannot fit entirely, let's allocate a new buffer to convert it + // and then truncate it to the destination buffer + + // clear errno and LastError to clear the error of the + // MultiByteToWideChar() that failed + errno_clear(); + + // get the required size + int required_size = MultiByteToWideChar(CodePage, 0, src, src_len, NULL, 0); + + // mallocz() never fails (exits the program on NULL) + wchar_t *tmp = mallocz(required_size * sizeof(wchar_t)); + + // convert it, now it should fit + rc = MultiByteToWideChar(CodePage, 0, src, src_len, tmp, required_size); + if (rc <= 0) { + // it failed! + *dst = L'\0'; + freez(tmp); + return 0; + } + + size_t len = rc; + + // copy as much as we can + memcpy(dst, tmp, MIN(len, (dst_size - 1)) * sizeof(wchar_t)); + + // null terminate it + dst[MIN(len, (dst_size - 1))] = L'\0'; + + // free the temporary buffer + freez(tmp); + + // return the actual bytes written + return MIN(len, dst_size); + } + + // empty the destination + *dst = L'\0'; + return 0; + } + + size_t len = rc; + + if(truncated) + *truncated = false; + + if(len >= dst_size) { + if(dst[dst_size - 1] != L'\0') { + if (truncated) + *truncated = true; + + // Truncate it to fit the null terminator + dst[dst_size - 1] = L'\0'; + } + return dst_size; + } + + if(dst[len - 1] != L'\0') { + // the result is not null terminated + // append the null + dst[len] = L'\0'; + return len + 1; + } + + // the result is already null terminated + return len; +} + +/* + * Convert UTF16 (wide-character string) to UTF8 + * Goals: + * 1. Destination is always NULL terminated + * 2. If the destination buffer is not enough, return as much as possible data (truncate) + * 3. Always return the number of bytes written, including the null terminator + */ + +size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len, bool *truncated) { + if (!src || src_len == 0) { + // invalid input + if(truncated) + *truncated = true; + + if(dst && dst_size) + *dst = '\0'; + + return 0; + } + + if (!dst || dst_size == 0) { + // The caller wants to know the buffer size required for the conversion + + if(truncated) + *truncated = true; + + int required = WideCharToMultiByte(CP_UTF8, 0, src, src_len, NULL, 0, NULL, NULL); + if (required <= 0) return 0; // error in the conversion + + // Add 1 for null terminator only if src_len is not -1 + return (size_t)required + ((src_len != -1) ? 1 : 0); + } + + // Perform the conversion directly into the destination buffer + int rc = WideCharToMultiByte(CP_UTF8, 0, src, src_len, dst, (int)dst_size, NULL, NULL); + if (rc <= 0) { + if(truncated) + *truncated = true; + + // Conversion failed, let's see why... + DWORD status = GetLastError(); + if (status == ERROR_INSUFFICIENT_BUFFER) { + // It cannot fit entirely, let's allocate a new buffer to convert it + // and then truncate it to the destination buffer + + // Clear errno and LastError to clear the error of the + // WideCharToMultiByte() that failed + errno_clear(); + + // Get the required size + int required_size = WideCharToMultiByte(CP_UTF8, 0, src, src_len, NULL, 0, NULL, NULL); + + // mallocz() never fails (exits the program on NULL) + char *tmp = mallocz(required_size * sizeof(char)); + + // Convert it, now it should fit + rc = WideCharToMultiByte(CP_UTF8, 0, src, src_len, tmp, required_size, NULL, NULL); + if (rc <= 0) { + // Conversion failed + *dst = '\0'; + freez(tmp); + return 0; + } + + size_t len = rc; + + // Copy as much as we can + memcpy(dst, tmp, MIN(len, (dst_size - 1)) * sizeof(char)); + + // Null-terminate it + dst[MIN(len, (dst_size - 1))] = '\0'; + + // Free the temporary buffer + freez(tmp); + + // Return the actual bytes written + return MIN(len, dst_size); + } + + // Empty the destination + *dst = '\0'; + return 0; + } + + size_t len = rc; + + if(truncated) + *truncated = false; + + if (len >= dst_size) { + if(dst[dst_size - 1] != '\0') { + if (truncated) + *truncated = true; + + // Truncate it to fit the null terminator + dst[dst_size - 1] = '\0'; + } + return dst_size; + } + + if (dst[len - 1] != '\0') { + // The result is not null-terminated + // Append the null terminator + dst[len] = '\0'; + return len + 1; + } + + // The result is already null-terminated + return len; +} + +// -------------------------------------------------------------------------------------------------------------------- + +size_t txt_compute_new_size(size_t old_size, size_t required_size) { + size_t size = (required_size % 2048 == 0) ? required_size : required_size + 2048; + size = (size / 2048) * 2048; + + if(size < old_size * 2) + size = old_size * 2; + + return size; +} + +// -------------------------------------------------------------------------------------------------------------------- +// TXT_UTF8 + +void txt_utf8_cleanup(TXT_UTF8 *dst) { + freez(dst->data); + dst->data = NULL; + dst->used = 0; +} + +void txt_utf8_resize(TXT_UTF8 *dst, size_t required_size, bool keep) { + if(required_size <= dst->size) + return; + + size_t new_size = txt_compute_new_size(dst->size, required_size); + + if(keep && dst->data) + dst->data = reallocz(dst->data, new_size); + else { + txt_utf8_cleanup(dst); + dst->data = mallocz(new_size); + dst->used = 0; + } + + dst->size = new_size; +} + +void txt_utf8_empty(TXT_UTF8 *dst) { + txt_utf8_resize(dst, 1, false); + dst->data[0] = '\0'; + dst->used = 1; +} + +void txt_utf8_set(TXT_UTF8 *dst, const char *txt, size_t txt_len) { + txt_utf8_resize(dst, txt_len + 1, false); + memcpy(dst->data, txt, txt_len); + dst->used = txt_len + 1; + dst->data[dst->used - 1] = '\0'; +} + +void txt_utf8_append(TXT_UTF8 *dst, const char *txt, size_t txt_len) { + if(dst->used <= 1) { + // the destination is empty + txt_utf8_set(dst, txt, txt_len); + } + else { + // there is something already in the buffer + txt_utf8_resize(dst, dst->used + txt_len, true); + memcpy(&dst->data[dst->used - 1], txt, txt_len); + dst->used += txt_len; // the null was already counted + dst->data[dst->used - 1] = '\0'; + } +} + +// -------------------------------------------------------------------------------------------------------------------- +// TXT_UTF16 + +void txt_utf16_cleanup(TXT_UTF16 *dst) { + freez(dst->data); +} + +void txt_utf16_resize(TXT_UTF16 *dst, size_t required_size, bool keep) { + if(required_size <= dst->size) + return; + + size_t new_size = txt_compute_new_size(dst->size, required_size); + + if (keep && dst->data) { + dst->data = reallocz(dst->data, new_size * sizeof(wchar_t)); + } else { + txt_utf16_cleanup(dst); + dst->data = mallocz(new_size * sizeof(wchar_t)); + dst->used = 0; + } + + dst->size = new_size; +} + +void txt_utf16_set(TXT_UTF16 *dst, const wchar_t *txt, size_t txt_len) { + txt_utf16_resize(dst, dst->used + txt_len + 1, true); + memcpy(dst->data, txt, txt_len * sizeof(wchar_t)); + dst->used = txt_len + 1; + dst->data[dst->used - 1] = '\0'; +} + +void txt_utf16_append(TXT_UTF16 *dst, const wchar_t *txt, size_t txt_len) { + if(dst->used <= 1) { + // the destination is empty + txt_utf16_set(dst, txt, txt_len); + } + else { + // there is something already in the buffer + txt_utf16_resize(dst, dst->used + txt_len, true); + memcpy(&dst->data[dst->used - 1], txt, txt_len * sizeof(wchar_t)); + dst->used += txt_len; // the null was already counted + dst->data[dst->used - 1] = '\0'; + } +} + +// -------------------------------------------------------------------------------------------------------------------- + +bool wchar_to_txt_utf8(TXT_UTF8 *dst, const wchar_t *src, int src_len) { + if(!src || !src_len) { + txt_utf8_empty(dst); + return false; + } + + if(!dst->data && !dst->size) { + size_t size = utf16_to_utf8(NULL, 0, src, src_len, NULL); + if(!size) { + txt_utf8_empty(dst); + return false; + } + + // we +1 here to avoid entering the next condition below + txt_utf8_resize(dst, size, false); + } + + bool truncated = false; + dst->used = utf16_to_utf8(dst->data, dst->size, src, src_len, &truncated); + if(truncated) { + // we need to resize + size_t needed = utf16_to_utf8(NULL, 0, src, src_len, NULL); // find the size needed + if(!needed) { + txt_utf8_empty(dst); + return false; + } + + txt_utf8_resize(dst, needed, false); + dst->used = utf16_to_utf8(dst->data, dst->size, src, src_len, NULL); + } + + // Make sure it is not zero padded at the end + while(dst->used >= 2 && dst->data[dst->used - 2] == 0) + dst->used--; + + internal_fatal(strlen(dst->data) + 1 != dst->used, + "Wrong UTF8 string length"); + + return true; +} + +bool txt_utf16_to_utf8(TXT_UTF8 *utf8, TXT_UTF16 *utf16) { + fatal_assert(utf8 && ((utf8->data && utf8->size) || (!utf8->data && !utf8->size))); + fatal_assert(utf16 && ((utf16->data && utf16->size) || (!utf16->data && !utf16->size))); + + // pass the entire utf16 size, including the null terminator + // so that the resulting utf8 message will be null terminated too. + return wchar_to_txt_utf8(utf8, utf16->data, (int)utf16->used - 1); +} + +char *utf16_to_utf8_strdupz(const wchar_t *src, size_t *dst_len) { + size_t size = utf16_to_utf8(NULL, 0, src, -1, NULL); + if (size) { + char *dst = mallocz(size); + + size = utf16_to_utf8(dst, size, src, -1, NULL); + if(dst_len) + *dst_len = size - 1; + + return dst; + } + + if(dst_len) + *dst_len = 0; + + return NULL; +} + +#endif diff --git a/src/libnetdata/string/utf8.h b/src/libnetdata/string/utf8.h index 3e6c8c288..f27ba5447 100644 --- a/src/libnetdata/string/utf8.h +++ b/src/libnetdata/string/utf8.h @@ -3,7 +3,81 @@ #ifndef NETDATA_STRING_UTF8_H #define NETDATA_STRING_UTF8_H 1 -#define IS_UTF8_BYTE(x) ((x) & 0x80) -#define IS_UTF8_STARTBYTE(x) (IS_UTF8_BYTE(x)&&((x) & 0x40)) +#include "../libnetdata.h" + +#define IS_UTF8_BYTE(x) ((uint8_t)(x) & (uint8_t)0x80) +#define IS_UTF8_STARTBYTE(x) (IS_UTF8_BYTE(x) && ((uint8_t)(x) & (uint8_t)0x40)) + +#ifndef _countof +#define _countof(x) (sizeof(x) / sizeof(*(x))) +#endif + +#if defined(OS_WINDOWS) + +// return an always null terminated wide string, truncate to given size if destination is not big enough, +// src_len can be -1 use all of it. +// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated). +size_t any_to_utf16(uint32_t CodePage, wchar_t *dst, size_t dst_size, const char *src, int src_len, bool *truncated); + +// always null terminated, truncated if it does not fit, src_len can be -1 to use all of it. +// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated). +#define utf8_to_utf16(utf16, utf16_count, src, src_len) any_to_utf16(CP_UTF8, utf16, utf16_count, src, src_len, NULL) + +// always null terminated, truncated if it does not fit, src_len can be -1 to use all of it. +// returns zero on errors, > 0 otherwise (including the null, even if src is not null terminated). +size_t utf16_to_utf8(char *dst, size_t dst_size, const wchar_t *src, int src_len, bool *truncated); + +// -------------------------------------------------------------------------------------------------------------------- +// TXT_UTF8 + +typedef enum __attribute__((packed)) { + TXT_SOURCE_UNKNOWN = 0, + TXT_SOURCE_PROVIDER, + TXT_SOURCE_FIELD_CACHE, + TXT_SOURCE_EVENT_LOG, + TXT_SOURCE_HARDCODED, + + // terminator + TXT_SOURCE_MAX, +} TXT_SOURCE; + +typedef struct { + char *data; + uint32_t size; // the allocated size of data buffer + uint32_t used; // the used size of the data buffer (including null terminators, if any) + TXT_SOURCE src; +} TXT_UTF8; + +void txt_utf8_append(TXT_UTF8 *dst, const char *txt, size_t txt_len); +void txt_utf8_set(TXT_UTF8 *dst, const char *txt, size_t txt_len); +void txt_utf8_empty(TXT_UTF8 *dst); +void txt_utf8_resize(TXT_UTF8 *dst, size_t required_size, bool keep); +void txt_utf8_cleanup(TXT_UTF8 *dst); + +// -------------------------------------------------------------------------------------------------------------------- +// TXT_UTF16 + +typedef struct { + wchar_t *data; + uint32_t size; // the allocated size of data buffer + uint32_t used; // the used size of the data buffer (including null terminators, if any) +} TXT_UTF16; + +void txt_utf16_cleanup(TXT_UTF16 *dst); +void txt_utf16_resize(TXT_UTF16 *dst, size_t required_size, bool keep); +void txt_utf16_set(TXT_UTF16 *dst, const wchar_t *txt, size_t txt_len); +void txt_utf16_append(TXT_UTF16 *dst, const wchar_t *txt, size_t txt_len); + +// -------------------------------------------------------------------------------------------------------------------- + +size_t txt_compute_new_size(size_t old_size, size_t required_size); + +bool txt_utf16_to_utf8(TXT_UTF8 *utf8, TXT_UTF16 *utf16); +bool wchar_to_txt_utf8(TXT_UTF8 *dst, const wchar_t *src, int src_len); +char *utf16_to_utf8_strdupz(const wchar_t *src, size_t *dst_len); + +// -------------------------------------------------------------------------------------------------------------------- + +#endif // OS_WINDOWS #endif /* NETDATA_STRING_UTF8_H */ diff --git a/src/libnetdata/template-enum.h b/src/libnetdata/template-enum.h index 393a6a945..2170ee86b 100644 --- a/src/libnetdata/template-enum.h +++ b/src/libnetdata/template-enum.h @@ -37,4 +37,47 @@ return def_str; \ } +// -------------------------------------------------------------------------------------------------------------------- + +#define BITMAP_STR_DEFINE_FUNCTIONS_EXTERN(type) \ + type type ## _2id_one(const char *str); \ + const char *type##_2str_one(type id); \ + const char *type##_2json(BUFFER *wb, const char *key, type id); + +#define BITMAP_STR_DEFINE_FUNCTIONS(type, def, def_str) \ + type type##_2id_one(const char *str) \ + { \ + if (!str || !*str) \ + return def; \ + \ + for (size_t i = 0; type ## _names[i].name; i++) { \ + if (strcmp(type ## _names[i].name, str) == 0) \ + return type ## _names[i].id; \ + } \ + \ + return def; \ + } \ + \ + const char *type##_2str_one(type id) \ + { \ + for (size_t i = 0; type ## _names[i].name; i++) { \ + if (id == type ## _names[i].id) \ + return type ## _names[i].name; \ + } \ + \ + return def_str; \ + } \ + \ + const char *type##_2json(BUFFER *wb, const char *key, type id) \ + { \ + buffer_json_member_add_array(wb, key); \ + for (size_t i = 0; type ## _names[i].name; i++) { \ + if ((id & type ## _names[i].id) == type ## _names[i].id) \ + buffer_json_add_array_item_string(wb, type ## _names[i].name); \ + } \ + buffer_json_array_close(wb); \ + \ + return def_str; \ + } + #endif //NETDATA_TEMPLATE_ENUM_H diff --git a/src/libnetdata/threads/README.md b/src/libnetdata/threads/README.md index 906f47952..adf38be74 100644 --- a/src/libnetdata/threads/README.md +++ b/src/libnetdata/threads/README.md @@ -1,12 +1,3 @@ -<!-- -title: Threads -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/threads/README.md -sidebar_label: "Threads" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # Threads Netdata uses a custom threads library diff --git a/src/libnetdata/url/README.md b/src/libnetdata/url/README.md index 01a2dddb6..35bdb19b9 100644 --- a/src/libnetdata/url/README.md +++ b/src/libnetdata/url/README.md @@ -1,12 +1,3 @@ -<!-- -title: "URL" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/url/README.md -sidebar_label: "URL" -learn_status: "Published" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # URL The URL library contains common functions useful for URLs, like conversion from/to hex, diff --git a/src/libnetdata/uuid/README.md b/src/libnetdata/uuid/README.md index a0da380a9..6bc1c7d27 100644 --- a/src/libnetdata/uuid/README.md +++ b/src/libnetdata/uuid/README.md @@ -1,11 +1,3 @@ -<!-- -title: "UUID" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/uuid/README.md -sidebar_label: "UUID" -learn_topic_type: "Tasks" -learn_rel_path: "Developers/libnetdata" ---> - # UUID Netdata uses libuuid for managing UUIDs. diff --git a/src/libnetdata/uuid/uuid.h b/src/libnetdata/uuid/uuid.h index cde457616..5fb1bce68 100644 --- a/src/libnetdata/uuid/uuid.h +++ b/src/libnetdata/uuid/uuid.h @@ -37,6 +37,8 @@ ND_UUID UUID_generate_from_hash(const void *payload, size_t payload_len); #define UUIDeq(a, b) ((a).parts.hig64 == (b).parts.hig64 && (a).parts.low64 == (b).parts.low64) +#define UUIDiszero(a) ((a).parts.hig64 == 0 && (a).parts.low64 == 0) + static inline ND_UUID uuid2UUID(const nd_uuid_t uu1) { // uu1 may not be aligned, so copy it to the output ND_UUID copy; diff --git a/src/libnetdata/worker_utilization/README.md b/src/libnetdata/worker_utilization/README.md index 1a354376c..17dd85e3e 100644 --- a/src/libnetdata/worker_utilization/README.md +++ b/src/libnetdata/worker_utilization/README.md @@ -1,12 +1,3 @@ -<!-- -title: "Worker Utilization" -custom_edit_url: https://github.com/netdata/netdata/edit/master/src/libnetdata/worker_utilization/README.md -sidebar_label: "Worker Utilization" -learn_status: "Published" -learn_topic_type: "References" -learn_rel_path: "Developers/libnetdata" ---> - # Worker Utilization This library is to be used when there are 1 or more worker threads accepting requests diff --git a/src/libnetdata/xxhash.h b/src/libnetdata/xxHash/xxhash.h index 5e2c0ed24..5e2c0ed24 100644 --- a/src/libnetdata/xxhash.h +++ b/src/libnetdata/xxHash/xxhash.h |