diff options
Diffstat (limited to 'libnetdata')
42 files changed, 1391 insertions, 1269 deletions
diff --git a/libnetdata/Makefile.am b/libnetdata/Makefile.am index 4bf779136..b81d620ba 100644 --- a/libnetdata/Makefile.am +++ b/libnetdata/Makefile.am @@ -20,7 +20,6 @@ SUBDIRS = \ locks \ log \ onewayalloc \ - parser \ popen \ procfile \ simple_pattern \ diff --git a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c index 6332fa174..b645927d4 100644 --- a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c +++ b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.c @@ -78,16 +78,16 @@ void arl_begin(ARL_BASE *base) { // do these checks after the ARL has been sorted if(unlikely(base->relinkings > (base->expected + base->allocated))) - info("ARL '%s' has %zu relinkings with %zu expected and %zu allocated entries. Is the source changing so fast?" + netdata_log_info("ARL '%s' has %zu relinkings with %zu expected and %zu allocated entries. Is the source changing so fast?" , base->name, base->relinkings, base->expected, base->allocated); if(unlikely(base->slow > base->fast)) - info("ARL '%s' has %zu fast searches and %zu slow searches. Is the source really changing so fast?" + netdata_log_info("ARL '%s' has %zu fast searches and %zu slow searches. Is the source really changing so fast?" , base->name, base->fast, base->slow); /* if(unlikely(base->iteration % 60 == 0)) { - info("ARL '%s' statistics: iteration %zu, expected %zu, wanted %zu, allocated %zu, fred %zu, relinkings %zu, found %zu, added %zu, fast %zu, slow %zu" + netdata_log_info("ARL '%s' statistics: iteration %zu, expected %zu, wanted %zu, allocated %zu, fred %zu, relinkings %zu, found %zu, added %zu, fast %zu, slow %zu" , base->name , base->iteration , base->expected @@ -242,7 +242,7 @@ int arl_find_or_create_and_relink(ARL_BASE *base, const char *s, const char *val #ifdef NETDATA_INTERNAL_CHECKS if(unlikely(base->iteration % 60 == 0 && e->flags & ARL_ENTRY_FLAG_FOUND)) - info("ARL '%s': entry '%s' is already found. Did you forget to call arl_begin()?", base->name, s); + netdata_log_info("ARL '%s': entry '%s' is already found. Did you forget to call arl_begin()?", base->name, s); #endif e->flags |= ARL_ENTRY_FLAG_FOUND; diff --git a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h index 294c52e81..bca0ff272 100644 --- a/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h +++ b/libnetdata/adaptive_resortable_list/adaptive_resortable_list.h @@ -94,7 +94,7 @@ static inline int arl_check(ARL_BASE *base, const char *keyword, const char *val #ifdef NETDATA_INTERNAL_CHECKS if(unlikely((base->fast + base->slow) % (base->expected + base->allocated) == 0 && (base->fast + base->slow) > (base->expected + base->allocated) * base->iteration)) - info("ARL '%s': Did you forget to call arl_begin()?", base->name); + netdata_log_info("ARL '%s': Did you forget to call arl_begin()?", base->name); #endif // it should be the first entry (pointed by base->next_keyword) diff --git a/libnetdata/aral/aral.c b/libnetdata/aral/aral.c index 60fe5e39a..16328db69 100644 --- a/libnetdata/aral/aral.c +++ b/libnetdata/aral/aral.c @@ -141,39 +141,39 @@ static size_t aral_align_alloc_size(ARAL *ar, uint64_t size) { static inline void aral_lock(ARAL *ar) { if(likely(!(ar->config.options & ARAL_LOCKLESS))) - netdata_spinlock_lock(&ar->aral_lock.spinlock); + spinlock_lock(&ar->aral_lock.spinlock); } static inline void aral_unlock(ARAL *ar) { if(likely(!(ar->config.options & ARAL_LOCKLESS))) - netdata_spinlock_unlock(&ar->aral_lock.spinlock); + spinlock_unlock(&ar->aral_lock.spinlock); } static inline void aral_page_free_lock(ARAL *ar, ARAL_PAGE *page) { if(likely(!(ar->config.options & ARAL_LOCKLESS))) - netdata_spinlock_lock(&page->free.spinlock); + spinlock_lock(&page->free.spinlock); } static inline void aral_page_free_unlock(ARAL *ar, ARAL_PAGE *page) { if(likely(!(ar->config.options & ARAL_LOCKLESS))) - netdata_spinlock_unlock(&page->free.spinlock); + spinlock_unlock(&page->free.spinlock); } static inline bool aral_adders_trylock(ARAL *ar) { if(likely(!(ar->config.options & ARAL_LOCKLESS))) - return netdata_spinlock_trylock(&ar->adders.spinlock); + return spinlock_trylock(&ar->adders.spinlock); return true; } static inline void aral_adders_lock(ARAL *ar) { if(likely(!(ar->config.options & ARAL_LOCKLESS))) - netdata_spinlock_lock(&ar->adders.spinlock); + spinlock_lock(&ar->adders.spinlock); } static inline void aral_adders_unlock(ARAL *ar) { if(likely(!(ar->config.options & ARAL_LOCKLESS))) - netdata_spinlock_unlock(&ar->adders.spinlock); + spinlock_unlock(&ar->adders.spinlock); } static void aral_delete_leftover_files(const char *name, const char *path, const char *required_prefix) { @@ -192,9 +192,9 @@ static void aral_delete_leftover_files(const char *name, const char *path, const continue; snprintfz(full_path, FILENAME_MAX, "%s/%s", path, de->d_name); - info("ARAL: '%s' removing left-over file '%s'", name, full_path); + netdata_log_info("ARAL: '%s' removing left-over file '%s'", name, full_path); if(unlikely(unlink(full_path) == -1)) - error("ARAL: '%s' cannot delete file '%s'", name, full_path); + netdata_log_error("ARAL: '%s' cannot delete file '%s'", name, full_path); } closedir(dir); @@ -273,7 +273,7 @@ size_t aral_next_allocation_size___adders_lock_needed(ARAL *ar) { static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) { ARAL_PAGE *page = callocz(1, sizeof(ARAL_PAGE)); - netdata_spinlock_init(&page->free.spinlock); + spinlock_init(&page->free.spinlock); page->size = size; page->max_elements = page->size / ar->config.element_size; page->aral_lock.free_elements = page->max_elements; @@ -324,7 +324,7 @@ void aral_del_page___no_lock_needed(ARAL *ar, ARAL_PAGE *page TRACE_ALLOCATIONS_ netdata_munmap(page->data, page->size); if (unlikely(unlink(page->filename) == 1)) - error("Cannot delete file '%s'", page->filename); + netdata_log_error("Cannot delete file '%s'", page->filename); freez((void *)page->filename); @@ -713,7 +713,7 @@ ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_ele ar->config.mmap.cache_dir = cache_dir; ar->config.mmap.enabled = mmap; strncpyz(ar->config.name, name, ARAL_MAX_NAME); - netdata_spinlock_init(&ar->aral_lock.spinlock); + spinlock_init(&ar->aral_lock.spinlock); if(stats) { ar->stats = stats; @@ -756,7 +756,7 @@ ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_ele ar->config.name, ar->config.requested_element_size, sizeof(uintptr_t), ARAL_NATURAL_ALIGNMENT, ar->config.element_size, ar->config.page_ptr_offset); - //info("ARAL: element size %zu, sizeof(uintptr_t) %zu, natural alignment %zu, final element size %zu, page_ptr_offset %zu", + //netdata_log_info("ARAL: element size %zu, sizeof(uintptr_t) %zu, natural alignment %zu, final element size %zu, page_ptr_offset %zu", // ar->element_size, sizeof(uintptr_t), ARAL_NATURAL_ALIGNMENT, ar->internal.element_size, ar->internal.page_ptr_offset); @@ -764,7 +764,7 @@ ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_ele ar->config.initial_page_elements = 2; if(ar->config.mmap.enabled && (!ar->config.mmap.cache_dir || !*ar->config.mmap.cache_dir)) { - error("ARAL: '%s' mmap cache directory is not configured properly, disabling mmap.", ar->config.name); + netdata_log_error("ARAL: '%s' mmap cache directory is not configured properly, disabling mmap.", ar->config.name); ar->config.mmap.enabled = false; internal_fatal(true, "ARAL: '%s' mmap cache directory is not configured properly", ar->config.name); } @@ -839,7 +839,7 @@ size_t aral_by_size_overhead(void) { } ARAL *aral_by_size_acquire(size_t size) { - netdata_spinlock_lock(&aral_by_size_globals.spinlock); + spinlock_lock(&aral_by_size_globals.spinlock); ARAL *ar = NULL; @@ -867,7 +867,7 @@ ARAL *aral_by_size_acquire(size_t size) { } } - netdata_spinlock_unlock(&aral_by_size_globals.spinlock); + spinlock_unlock(&aral_by_size_globals.spinlock); return ar; } @@ -876,7 +876,7 @@ void aral_by_size_release(ARAL *ar) { size_t size = aral_element_size(ar); if(size <= ARAL_BY_SIZE_MAX_SIZE) { - netdata_spinlock_lock(&aral_by_size_globals.spinlock); + spinlock_lock(&aral_by_size_globals.spinlock); internal_fatal(aral_by_size_globals.array[size].ar != ar, "ARAL BY SIZE: aral pointers do not match"); @@ -890,7 +890,7 @@ void aral_by_size_release(ARAL *ar) { // aral_by_size_globals.array[size].ar = NULL; // } - netdata_spinlock_unlock(&aral_by_size_globals.spinlock); + spinlock_unlock(&aral_by_size_globals.spinlock); } else aral_destroy(ar); @@ -1056,7 +1056,7 @@ int aral_stress_test(size_t threads, size_t elements, size_t seconds) { __atomic_add_fetch(&auc.errors, 1, __ATOMIC_RELAXED); } - info("ARAL: did %zu malloc, %zu free, " + netdata_log_info("ARAL: did %zu malloc, %zu free, " "using %zu threads, in %llu usecs", auc.ar->aral_lock.user_malloc_operations, auc.ar->aral_lock.user_free_operations, diff --git a/libnetdata/avl/avl.c b/libnetdata/avl/avl.c index 5a4c1a983..eef4c3116 100644 --- a/libnetdata/avl/avl.c +++ b/libnetdata/avl/avl.c @@ -316,34 +316,36 @@ int avl_traverse(avl_tree_type *tree, int (*callback)(void * /*entry*/, void * / // --------------------------- // locks -void avl_read_lock(avl_tree_lock *t) { -#ifndef AVL_WITHOUT_PTHREADS -#ifdef AVL_LOCK_WITH_MUTEX - netdata_mutex_lock(&t->mutex); -#else +static inline void avl_read_lock(avl_tree_lock *t) { +#if defined(AVL_LOCK_WITH_RWLOCK) netdata_rwlock_rdlock(&t->rwlock); +#else + rw_spinlock_read_lock(&t->rwlock); #endif -#endif /* AVL_WITHOUT_PTHREADS */ } -void avl_write_lock(avl_tree_lock *t) { -#ifndef AVL_WITHOUT_PTHREADS -#ifdef AVL_LOCK_WITH_MUTEX - netdata_mutex_lock(&t->mutex); -#else +static inline void avl_write_lock(avl_tree_lock *t) { +#if defined(AVL_LOCK_WITH_RWLOCK) netdata_rwlock_wrlock(&t->rwlock); +#else + rw_spinlock_write_lock(&t->rwlock); #endif -#endif /* AVL_WITHOUT_PTHREADS */ } -void avl_unlock(avl_tree_lock *t) { -#ifndef AVL_WITHOUT_PTHREADS -#ifdef AVL_LOCK_WITH_MUTEX - netdata_mutex_unlock(&t->mutex); +static inline void avl_read_unlock(avl_tree_lock *t) { +#if defined(AVL_LOCK_WITH_RWLOCK) + netdata_rwlock_unlock(&t->rwlock); #else + rw_spinlock_read_unlock(&t->rwlock); +#endif +} + +static inline void avl_write_unlock(avl_tree_lock *t) { +#if defined(AVL_LOCK_WITH_RWLOCK) netdata_rwlock_unlock(&t->rwlock); +#else + rw_spinlock_write_unlock(&t->rwlock); #endif -#endif /* AVL_WITHOUT_PTHREADS */ } // --------------------------- @@ -352,63 +354,46 @@ void avl_unlock(avl_tree_lock *t) { void avl_init_lock(avl_tree_lock *tree, int (*compar)(void * /*a*/, void * /*b*/)) { avl_init(&tree->avl_tree, compar); -#ifndef AVL_WITHOUT_PTHREADS - int lock; - -#ifdef AVL_LOCK_WITH_MUTEX - lock = netdata_mutex_init(&tree->mutex, NULL); +#if defined(AVL_LOCK_WITH_RWLOCK) + if(netdata_rwlock_init(&tree->rwlock) != 0) + fatal("Failed to initialize AVL rwlock"); #else - lock = netdata_rwlock_init(&tree->rwlock); + rw_spinlock_init(&tree->rwlock); #endif - - if(lock != 0) - fatal("Failed to initialize AVL mutex/rwlock, error: %d", lock); - -#endif /* AVL_WITHOUT_PTHREADS */ } -void avl_destroy_lock(avl_tree_lock *tree) { -#ifndef AVL_WITHOUT_PTHREADS - int lock; - -#ifdef AVL_LOCK_WITH_MUTEX - lock = netdata_mutex_destroy(&tree->mutex); -#else - lock = netdata_rwlock_destroy(&tree->rwlock); +void avl_destroy_lock(avl_tree_lock *tree __maybe_unused) { +#if defined(AVL_LOCK_WITH_RWLOCK) + if(netdata_rwlock_destroy(&tree->rwlock) != 0) + fatal("Failed to destroy AVL rwlock"); #endif - - if(lock != 0) - fatal("Failed to destroy AVL mutex/rwlock, error: %d", lock); - -#endif /* AVL_WITHOUT_PTHREADS */ } avl_t *avl_search_lock(avl_tree_lock *tree, avl_t *item) { avl_read_lock(tree); avl_t *ret = avl_search(&tree->avl_tree, item); - avl_unlock(tree); + avl_read_unlock(tree); return ret; } avl_t * avl_remove_lock(avl_tree_lock *tree, avl_t *item) { avl_write_lock(tree); avl_t *ret = avl_remove(&tree->avl_tree, item); - avl_unlock(tree); + avl_write_unlock(tree); return ret; } avl_t *avl_insert_lock(avl_tree_lock *tree, avl_t *item) { avl_write_lock(tree); avl_t * ret = avl_insert(&tree->avl_tree, item); - avl_unlock(tree); + avl_write_unlock(tree); return ret; } int avl_traverse_lock(avl_tree_lock *tree, int (*callback)(void * /*entry*/, void * /*data*/), void *data) { - int ret; avl_read_lock(tree); - ret = avl_traverse(&tree->avl_tree, callback, data); - avl_unlock(tree); + int ret = avl_traverse(&tree->avl_tree, callback, data); + avl_read_unlock(tree); return ret; } diff --git a/libnetdata/avl/avl.h b/libnetdata/avl/avl.h index eba967fd0..595d6ec6c 100644 --- a/libnetdata/avl/avl.h +++ b/libnetdata/avl/avl.h @@ -10,20 +10,11 @@ #define AVL_MAX_HEIGHT 92 #endif -#ifndef AVL_WITHOUT_PTHREADS -#include <pthread.h> - -// #define AVL_LOCK_WITH_MUTEX 1 - -#ifdef AVL_LOCK_WITH_MUTEX -#define AVL_LOCK_INITIALIZER NETDATA_MUTEX_INITIALIZER -#else /* AVL_LOCK_WITH_MUTEX */ +#if defined(AVL_LOCK_WITH_RWLOCK) #define AVL_LOCK_INITIALIZER NETDATA_RWLOCK_INITIALIZER -#endif /* AVL_LOCK_WITH_MUTEX */ - -#else /* AVL_WITHOUT_PTHREADS */ -#define AVL_LOCK_INITIALIZER -#endif /* AVL_WITHOUT_PTHREADS */ +#else +#define AVL_LOCK_INITIALIZER NETDATA_RW_SPINLOCK_INITIALIZER +#endif /* Data structures */ @@ -33,6 +24,11 @@ typedef struct avl_element { signed char avl_balance; /* Balance factor. */ } avl_t; +typedef struct __attribute__((packed)) avl_element_packed { + struct avl_element *avl_link[2]; /* Subtrees. */ + signed char avl_balance; /* Balance factor. */ +} avl_t_packed; + /* An AVL tree */ typedef struct avl_tree_type { avl_t *root; @@ -42,13 +38,11 @@ typedef struct avl_tree_type { typedef struct avl_tree_lock { avl_tree_type avl_tree; -#ifndef AVL_WITHOUT_PTHREADS -#ifdef AVL_LOCK_WITH_MUTEX - netdata_mutex_t mutex; -#else /* AVL_LOCK_WITH_MUTEX */ +#if defined(AVL_LOCK_WITH_RWLOCK) netdata_rwlock_t rwlock; -#endif /* AVL_LOCK_WITH_MUTEX */ -#endif /* AVL_WITHOUT_PTHREADS */ +#else + RW_SPINLOCK rwlock; +#endif } avl_tree_lock; /* Public methods */ diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c index 91bc4dd60..b43762863 100644 --- a/libnetdata/buffer/buffer.c +++ b/libnetdata/buffer/buffer.c @@ -107,7 +107,7 @@ void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args) do { need += space_remaining * 2; - debug(D_WEB_BUFFER, "web_buffer_sprintf(): increasing web_buffer at position %zu, size = %zu, by %zu bytes (wrote = %zu)\n", wb->len, wb->size, need, wrote); + netdata_log_debug(D_WEB_BUFFER, "web_buffer_sprintf(): increasing web_buffer at position %zu, size = %zu, by %zu bytes (wrote = %zu)\n", wb->len, wb->size, need, wrote); buffer_need_bytes(wb, need); space_remaining = wb->size - wb->len - 1; @@ -131,7 +131,7 @@ void buffer_sprintf(BUFFER *wb, const char *fmt, ...) do { need += space_remaining * 2; - debug(D_WEB_BUFFER, "web_buffer_sprintf(): increasing web_buffer at position %zu, size = %zu, by %zu bytes (wrote = %zu)\n", wb->len, wb->size, need, wrote); + netdata_log_debug(D_WEB_BUFFER, "web_buffer_sprintf(): increasing web_buffer at position %zu, size = %zu, by %zu bytes (wrote = %zu)\n", wb->len, wb->size, need, wrote); buffer_need_bytes(wb, need); space_remaining = wb->size - wb->len - 1; @@ -246,7 +246,7 @@ BUFFER *buffer_create(size_t size, size_t *statistics) { BUFFER *b; - debug(D_WEB_BUFFER, "Creating new web buffer of size %zu.", size); + netdata_log_debug(D_WEB_BUFFER, "Creating new web buffer of size %zu.", size); b = callocz(1, sizeof(BUFFER)); b->buffer = mallocz(size + sizeof(BUFFER_OVERFLOW_EOF) + 2); @@ -268,7 +268,7 @@ void buffer_free(BUFFER *b) { buffer_overflow_check(b); - debug(D_WEB_BUFFER, "Freeing web buffer of size %zu.", b->size); + netdata_log_debug(D_WEB_BUFFER, "Freeing web buffer of size %zu.", b->size); if(b->statistics) __atomic_sub_fetch(b->statistics, b->size + sizeof(BUFFER) + sizeof(BUFFER_OVERFLOW_EOF) + 2, __ATOMIC_RELAXED); @@ -290,7 +290,7 @@ void buffer_increase(BUFFER *b, size_t free_size_required) { size_t optimal = (b->size > 5*1024*1024) ? b->size / 2 : b->size; if(optimal > wanted) wanted = optimal; - debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + wanted); + netdata_log_debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + wanted); b->buffer = reallocz(b->buffer, b->size + wanted + sizeof(BUFFER_OVERFLOW_EOF) + 2); b->size += wanted; @@ -315,6 +315,8 @@ void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value if(add_anonymous_object) buffer_fast_strcat(wb, "{", 1); + + wb->content_type = CT_APPLICATION_JSON; } void buffer_json_finalize(BUFFER *wb) { @@ -365,8 +367,8 @@ static int buffer_expect(BUFFER *wb, const char *expected) { const char *generated = buffer_tostring(wb); if(strcmp(generated, expected) != 0) { - error("BUFFER: mismatch.\nGenerated:\n%s\nExpected:\n%s\n", - generated, expected); + netdata_log_error("BUFFER: mismatch.\nGenerated:\n%s\nExpected:\n%s\n", + generated, expected); return 1; } @@ -383,8 +385,8 @@ static int buffer_uint64_roundtrip(BUFFER *wb, NUMBER_ENCODING encoding, uint64_ uint64_t v = str2ull_encoded(buffer_tostring(wb)); if(v != value) { - error("BUFFER: string '%s' does resolves to %llu, expected %llu", - buffer_tostring(wb), (unsigned long long)v, (unsigned long long)value); + netdata_log_error("BUFFER: string '%s' does resolves to %llu, expected %llu", + buffer_tostring(wb), (unsigned long long)v, (unsigned long long)value); errors++; } buffer_flush(wb); @@ -401,8 +403,8 @@ static int buffer_int64_roundtrip(BUFFER *wb, NUMBER_ENCODING encoding, int64_t int64_t v = str2ll_encoded(buffer_tostring(wb)); if(v != value) { - error("BUFFER: string '%s' does resolves to %lld, expected %lld", - buffer_tostring(wb), (long long)v, (long long)value); + netdata_log_error("BUFFER: string '%s' does resolves to %lld, expected %lld", + buffer_tostring(wb), (long long)v, (long long)value); errors++; } buffer_flush(wb); @@ -419,8 +421,8 @@ static int buffer_double_roundtrip(BUFFER *wb, NUMBER_ENCODING encoding, NETDATA NETDATA_DOUBLE v = str2ndd_encoded(buffer_tostring(wb), NULL); if(v != value) { - error("BUFFER: string '%s' does resolves to %.12f, expected %.12f", - buffer_tostring(wb), v, value); + netdata_log_error("BUFFER: string '%s' does resolves to %.12f, expected %.12f", + buffer_tostring(wb), v, value); errors++; } buffer_flush(wb); @@ -503,7 +505,7 @@ int buffer_unittest(void) { return errors; } -#ifdef ENABLE_HTTPD +#ifdef ENABLE_H2O h2o_iovec_t buffer_to_h2o_iovec(BUFFER *wb) { h2o_iovec_t ret; ret.base = wb->buffer; diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h index 22686a5a1..d0078a521 100644 --- a/libnetdata/buffer/buffer.h +++ b/libnetdata/buffer/buffer.h @@ -6,7 +6,7 @@ #include "../string/utf8.h" #include "../libnetdata.h" -#ifdef ENABLE_HTTPD +#ifdef ENABLE_H2O #include "h2o/memory.h" #endif @@ -61,6 +61,11 @@ typedef enum __attribute__ ((__packed__)) { CT_IMAGE_ICNS, CT_IMAGE_BMP, CT_PROMETHEUS, + CT_AUDIO_MPEG, + CT_AUDIO_OGG, + CT_VIDEO_MP4, + CT_APPLICATION_PDF, + CT_APPLICATION_ZIP, } HTTP_CONTENT_TYPE; typedef struct web_buffer { @@ -133,7 +138,7 @@ void buffer_char_replace(BUFFER *wb, char from, char to); void buffer_print_sn_flags(BUFFER *wb, SN_FLAGS flags, bool send_anomaly_bit); -#ifdef ENABLE_HTTPD +#ifdef ENABLE_H2O h2o_iovec_t buffer_to_h2o_iovec(BUFFER *wb); #endif @@ -242,19 +247,16 @@ static inline void buffer_strncat(BUFFER *wb, const char *txt, size_t len) { if(unlikely(!txt || !*txt)) return; const char *t = txt; - while(*t) { - buffer_need_bytes(wb, len); - char *s = &wb->buffer[wb->len]; - char *d = s; - const char *e = &wb->buffer[wb->len + len]; + buffer_need_bytes(wb, len + 1); + char *s = &wb->buffer[wb->len]; + char *d = s; + const char *e = &wb->buffer[wb->len + len]; - while(*t && d < e) - *d++ = *t++; + while(*t && d < e) + *d++ = *t++; - wb->len += d - s; - } + wb->len += d - s; - buffer_need_bytes(wb, 1); wb->buffer[wb->len] = '\0'; buffer_overflow_check(wb); @@ -760,7 +762,7 @@ static inline void buffer_json_member_add_uuid(BUFFER *wb, const char *key, uuid buffer_print_json_key(wb, key); buffer_fast_strcat(wb, ":", 1); - if(value) { + if(value && !uuid_is_null(*value)) { char uuid[GUID_LEN + 1]; uuid_unparse_lower(*value, uuid); buffer_json_add_string_value(wb, uuid); @@ -921,4 +923,199 @@ static inline void buffer_json_array_close(BUFFER *wb) { _buffer_json_depth_pop(wb); } +typedef enum __attribute__((packed)) { + RRDF_FIELD_OPTS_NONE = 0, + RRDF_FIELD_OPTS_UNIQUE_KEY = (1 << 0), // the field is the unique key of the row + RRDF_FIELD_OPTS_VISIBLE = (1 << 1), // the field should be visible by default + RRDF_FIELD_OPTS_STICKY = (1 << 2), // the field should be sticky +} RRDF_FIELD_OPTIONS; + +typedef enum __attribute__((packed)) { + RRDF_FIELD_TYPE_INTEGER, + RRDF_FIELD_TYPE_STRING, + RRDF_FIELD_TYPE_DETAIL_STRING, + RRDF_FIELD_TYPE_BAR_WITH_INTEGER, + RRDF_FIELD_TYPE_DURATION, + RRDF_FIELD_TYPE_TIMESTAMP, + RRDF_FIELD_TYPE_ARRAY, +} RRDF_FIELD_TYPE; + +static inline const char *rrdf_field_type_to_string(RRDF_FIELD_TYPE type) { + switch(type) { + default: + case RRDF_FIELD_TYPE_INTEGER: + return "integer"; + + case RRDF_FIELD_TYPE_STRING: + return "string"; + + case RRDF_FIELD_TYPE_DETAIL_STRING: + return "detail-string"; + + case RRDF_FIELD_TYPE_BAR_WITH_INTEGER: + return "bar-with-integer"; + + case RRDF_FIELD_TYPE_DURATION: + return "duration"; + + case RRDF_FIELD_TYPE_TIMESTAMP: + return "timestamp"; + + case RRDF_FIELD_TYPE_ARRAY: + return "array"; + } +} + +typedef enum __attribute__((packed)) { + RRDF_FIELD_VISUAL_VALUE, // show the value, possibly applying a transformation + RRDF_FIELD_VISUAL_BAR, // show the value and a bar, respecting the max field to fill the bar at 100% + RRDF_FIELD_VISUAL_PILL, // array of values (transformation is respected) +} RRDF_FIELD_VISUAL; + +static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual) { + switch(visual) { + default: + case RRDF_FIELD_VISUAL_VALUE: + return "value"; + + case RRDF_FIELD_VISUAL_BAR: + return "bar"; + + case RRDF_FIELD_VISUAL_PILL: + return "pill"; + } +} + +typedef enum __attribute__((packed)) { + RRDF_FIELD_TRANSFORM_NONE, // show the value as-is + RRDF_FIELD_TRANSFORM_NUMBER, // show the value repsecting the decimal_points + RRDF_FIELD_TRANSFORM_DURATION, // transform as duration in second to a human readable duration + RRDF_FIELD_TRANSFORM_DATETIME, // UNIX epoch timestamp in ms +} RRDF_FIELD_TRANSFORM; + +static inline const char *rrdf_field_transform_to_string(RRDF_FIELD_TRANSFORM transform) { + switch(transform) { + default: + case RRDF_FIELD_TRANSFORM_NONE: + return "none"; + + case RRDF_FIELD_TRANSFORM_NUMBER: + return "number"; + + case RRDF_FIELD_TRANSFORM_DURATION: + return "duration"; + + case RRDF_FIELD_TRANSFORM_DATETIME: + return "datetime"; + } +} + +typedef enum __attribute__((packed)) { + RRDF_FIELD_SORT_ASCENDING = (1 << 0), + RRDF_FIELD_SORT_DESCENDING = (1 << 1), + + RRDF_FIELD_SORT_FIXED = (1 << 7), +} RRDF_FIELD_SORT; + +static inline const char *rrdf_field_sort_to_string(RRDF_FIELD_SORT sort) { + if(sort & RRDF_FIELD_SORT_DESCENDING) + return "descending"; + + else + return "ascending"; +} + +typedef enum __attribute__((packed)) { + RRDF_FIELD_SUMMARY_UNIQUECOUNT, // Finds the number of unique values of a group of rows + RRDF_FIELD_SUMMARY_SUM, // Sums the values of a group of rows + RRDF_FIELD_SUMMARY_MIN, // Finds the minimum value of a group of rows + RRDF_FIELD_SUMMARY_MAX, // Finds the maximum value of a group of rows + // RRDF_FIELD_SUMMARY_EXTENT, // Finds the minimum and maximum values of a group of rows + RRDF_FIELD_SUMMARY_MEAN, // Finds the mean/average value of a group of rows + RRDF_FIELD_SUMMARY_MEDIAN, // Finds the median value of a group of rows + // RRDF_FIELD_SUMMARY_UNIQUE, // Finds the unique values of a group of rows + RRDF_FIELD_SUMMARY_COUNT, // Calculates the number of rows in a group +} RRDF_FIELD_SUMMARY; + +static inline const char *rrdf_field_summary_to_string(RRDF_FIELD_SUMMARY summary) { + switch(summary) { + default: + case RRDF_FIELD_SUMMARY_COUNT: + return "count"; + + case RRDF_FIELD_SUMMARY_UNIQUECOUNT: + return "uniqueCount"; + + case RRDF_FIELD_SUMMARY_SUM: + return "sum"; + + case RRDF_FIELD_SUMMARY_MIN: + return "min"; + + case RRDF_FIELD_SUMMARY_MEAN: + return "mean"; + + case RRDF_FIELD_SUMMARY_MEDIAN: + return "median"; + + case RRDF_FIELD_SUMMARY_MAX: + return "max"; + } +} + +typedef enum __attribute__((packed)) { + RRDF_FIELD_FILTER_RANGE, + RRDF_FIELD_FILTER_MULTISELECT, +} RRDF_FIELD_FILTER; + +static inline const char *rrdf_field_filter_to_string(RRDF_FIELD_FILTER filter) { + switch(filter) { + default: + case RRDF_FIELD_FILTER_RANGE: + return "range"; + + case RRDF_FIELD_FILTER_MULTISELECT: + return "multiselect"; + } +} + +static inline void +buffer_rrdf_table_add_field(BUFFER *wb, size_t field_id, const char *key, const char *name, RRDF_FIELD_TYPE type, + RRDF_FIELD_VISUAL visual, RRDF_FIELD_TRANSFORM transform, size_t decimal_points, + const char *units, NETDATA_DOUBLE max, RRDF_FIELD_SORT sort, const char *pointer_to, + RRDF_FIELD_SUMMARY summary, RRDF_FIELD_FILTER filter, RRDF_FIELD_OPTIONS options, + const char *default_value) { + + buffer_json_member_add_object(wb, key); + { + buffer_json_member_add_uint64(wb, "index", field_id); + buffer_json_member_add_boolean(wb, "unique_key", options & RRDF_FIELD_OPTS_UNIQUE_KEY); + buffer_json_member_add_string(wb, "name", name); + buffer_json_member_add_boolean(wb, "visible", options & RRDF_FIELD_OPTS_VISIBLE); + buffer_json_member_add_string(wb, "type", rrdf_field_type_to_string(type)); + buffer_json_member_add_string_or_omit(wb, "units", units); + buffer_json_member_add_string(wb, "visualization", rrdf_field_visual_to_string(visual)); + + buffer_json_member_add_object(wb, "value_options"); + { + buffer_json_member_add_string_or_omit(wb, "units", units); + buffer_json_member_add_string(wb, "transform", rrdf_field_transform_to_string(transform)); + buffer_json_member_add_uint64(wb, "decimal_points", decimal_points); + buffer_json_member_add_string(wb, "default_value", default_value); + } + buffer_json_object_close(wb); + + if (!isnan((NETDATA_DOUBLE) (max))) + buffer_json_member_add_double(wb, "max", (NETDATA_DOUBLE) (max)); + + buffer_json_member_add_string_or_omit(wb, "pointer_to", pointer_to); + buffer_json_member_add_string(wb, "sort", rrdf_field_sort_to_string(sort)); + buffer_json_member_add_boolean(wb, "sortable", !(sort & RRDF_FIELD_SORT_FIXED)); + buffer_json_member_add_boolean(wb, "sticky", options & RRDF_FIELD_OPTS_STICKY); + buffer_json_member_add_string(wb, "summary", rrdf_field_summary_to_string(summary)); + buffer_json_member_add_string(wb, "filter", rrdf_field_filter_to_string(filter)); + } + buffer_json_object_close(wb); +} + #endif /* NETDATA_WEB_BUFFER_H */ diff --git a/libnetdata/clocks/clocks.c b/libnetdata/clocks/clocks.c index 19c66f0a5..806dc06a3 100644 --- a/libnetdata/clocks/clocks.c +++ b/libnetdata/clocks/clocks.c @@ -14,7 +14,7 @@ usec_t clock_realtime_resolution = 1000; inline int clock_gettime(clockid_t clk_id __maybe_unused, struct timespec *ts) { struct timeval tv; if(unlikely(gettimeofday(&tv, NULL) == -1)) { - error("gettimeofday() failed."); + netdata_log_error("gettimeofday() failed."); return -1; } ts->tv_sec = tv.tv_sec; @@ -79,7 +79,7 @@ void clocks_init(void) { inline time_t now_sec(clockid_t clk_id) { struct timespec ts; if(unlikely(clock_gettime(clk_id, &ts) == -1)) { - error("clock_gettime(%d, ×pec) failed.", clk_id); + netdata_log_error("clock_gettime(%d, ×pec) failed.", clk_id); return 0; } return ts.tv_sec; @@ -88,7 +88,7 @@ inline time_t now_sec(clockid_t clk_id) { inline usec_t now_usec(clockid_t clk_id) { struct timespec ts; if(unlikely(clock_gettime(clk_id, &ts) == -1)) { - error("clock_gettime(%d, ×pec) failed.", clk_id); + netdata_log_error("clock_gettime(%d, ×pec) failed.", clk_id); return 0; } return (usec_t)ts.tv_sec * USEC_PER_SEC + (ts.tv_nsec % NSEC_PER_SEC) / NSEC_PER_USEC; @@ -98,7 +98,7 @@ inline int now_timeval(clockid_t clk_id, struct timeval *tv) { struct timespec ts; if(unlikely(clock_gettime(clk_id, &ts) == -1)) { - error("clock_gettime(%d, ×pec) failed.", clk_id); + netdata_log_error("clock_gettime(%d, ×pec) failed.", clk_id); tv->tv_sec = 0; tv->tv_usec = 0; return -1; @@ -200,30 +200,27 @@ void sleep_to_absolute_time(usec_t usec) { if (ret == EINVAL) { if (!einval_printed) { einval_printed++; - error( - "Invalid time given to clock_nanosleep(): clockid = %d, tv_sec = %lld, tv_nsec = %ld", - clock, - (long long)req.tv_sec, - req.tv_nsec); + netdata_log_error("Invalid time given to clock_nanosleep(): clockid = %d, tv_sec = %lld, tv_nsec = %ld", + clock, + (long long)req.tv_sec, + req.tv_nsec); } } else if (ret == ENOTSUP) { if (!enotsup_printed) { enotsup_printed++; - error( - "Invalid clock id given to clock_nanosleep(): clockid = %d, tv_sec = %lld, tv_nsec = %ld", - clock, - (long long)req.tv_sec, - req.tv_nsec); + netdata_log_error("Invalid clock id given to clock_nanosleep(): clockid = %d, tv_sec = %lld, tv_nsec = %ld", + clock, + (long long)req.tv_sec, + req.tv_nsec); } } else { if (!eunknown_printed) { eunknown_printed++; - error( - "Unknown return value %d from clock_nanosleep(): clockid = %d, tv_sec = %lld, tv_nsec = %ld", - ret, - clock, - (long long)req.tv_sec, - req.tv_nsec); + netdata_log_error("Unknown return value %d from clock_nanosleep(): clockid = %d, tv_sec = %lld, tv_nsec = %ld", + ret, + clock, + (long long)req.tv_sec, + req.tv_nsec); } } sleep_usec(usec); @@ -384,7 +381,7 @@ void sleep_usec_with_now(usec_t usec, usec_t started_ut) { } } else { - error("Cannot nanosleep() for %llu microseconds.", usec); + netdata_log_error("Cannot nanosleep() for %llu microseconds.", usec); break; } } @@ -394,7 +391,7 @@ static inline collected_number uptime_from_boottime(void) { #ifdef CLOCK_BOOTTIME_IS_AVAILABLE return (collected_number)(now_boottime_usec() / USEC_PER_MS); #else - error("uptime cannot be read from CLOCK_BOOTTIME on this system."); + netdata_log_error("uptime cannot be read from CLOCK_BOOTTIME on this system."); return 0; #endif } @@ -410,11 +407,11 @@ static inline collected_number read_proc_uptime(char *filename) { if(unlikely(!read_proc_uptime_ff)) return 0; if(unlikely(procfile_lines(read_proc_uptime_ff) < 1)) { - error("/proc/uptime has no lines."); + netdata_log_error("/proc/uptime has no lines."); return 0; } if(unlikely(procfile_linewords(read_proc_uptime_ff, 0) < 1)) { - error("/proc/uptime has less than 1 word in it."); + netdata_log_error("/proc/uptime has less than 1 word in it."); return 0; } @@ -433,15 +430,15 @@ inline collected_number uptime_msec(char *filename){ if(delta <= 1000 && uptime_boottime != 0) { procfile_close(read_proc_uptime_ff); - info("Using now_boottime_usec() for uptime (dt is %lld ms)", delta); + netdata_log_info("Using now_boottime_usec() for uptime (dt is %lld ms)", delta); use_boottime = 1; } else if(uptime_proc != 0) { - info("Using /proc/uptime for uptime (dt is %lld ms)", delta); + netdata_log_info("Using /proc/uptime for uptime (dt is %lld ms)", delta); use_boottime = 0; } else { - error("Cannot find any way to read uptime on this system."); + netdata_log_error("Cannot find any way to read uptime on this system."); return 1; } } diff --git a/libnetdata/config/appconfig.c b/libnetdata/config/appconfig.c index d346da85f..fe4c1222d 100644 --- a/libnetdata/config/appconfig.c +++ b/libnetdata/config/appconfig.c @@ -62,7 +62,7 @@ int is_valid_connector(char *type, int check_reserved) } // else { // if (unlikely(is_valid_connector(type,1))) { -// error("Section %s invalid -- reserved name", type); +// netdata_log_error("Section %s invalid -- reserved name", type); // return 0; // } // } @@ -164,7 +164,7 @@ static inline struct section *appconfig_section_find(struct config *root, const } static inline struct section *appconfig_section_create(struct config *root, const char *section) { - debug(D_CONFIG, "Creating section '%s'.", section); + netdata_log_debug(D_CONFIG, "Creating section '%s'.", section); struct section *co = callocz(1, sizeof(struct section)); co->name = strdupz(section); @@ -174,7 +174,7 @@ static inline struct section *appconfig_section_create(struct config *root, cons avl_init_lock(&co->values_index, appconfig_option_compare); if(unlikely(appconfig_index_add(root, co) != co)) - error("INTERNAL ERROR: indexing of section '%s', already exists.", co->name); + netdata_log_error("INTERNAL ERROR: indexing of section '%s', already exists.", co->name); appconfig_wrlock(root); struct section *co2 = root->last_section; @@ -194,11 +194,11 @@ void appconfig_section_destroy_non_loaded(struct config *root, const char *secti struct section *co; struct config_option *cv, *cv_next; - debug(D_CONFIG, "Destroying section '%s'.", section); + netdata_log_debug(D_CONFIG, "Destroying section '%s'.", section); co = appconfig_section_find(root, section); if(!co) { - error("Could not destroy section '%s'. Not found.", section); + netdata_log_error("Could not destroy section '%s'. Not found.", section); return; } @@ -213,7 +213,7 @@ void appconfig_section_destroy_non_loaded(struct config *root, const char *secti for(cv = co->values ; cv ; cv = cv_next) { cv_next = cv->next; if(unlikely(!appconfig_option_index_del(co, cv))) - error("Cannot remove config option '%s' from section '%s'.", cv->name, co->name); + netdata_log_error("Cannot remove config option '%s' from section '%s'.", cv->name, co->name); freez(cv->value); freez(cv->name); freez(cv); @@ -222,7 +222,7 @@ void appconfig_section_destroy_non_loaded(struct config *root, const char *secti config_section_unlock(co); if (unlikely(!appconfig_index_del(root, co))) { - error("Cannot remove section '%s' from config.", section); + netdata_log_error("Cannot remove section '%s' from config.", section); return; } @@ -259,12 +259,12 @@ void appconfig_section_destroy_non_loaded(struct config *root, const char *secti void appconfig_section_option_destroy_non_loaded(struct config *root, const char *section, const char *name) { - debug(D_CONFIG, "Destroying section option '%s -> %s'.", section, name); + netdata_log_debug(D_CONFIG, "Destroying section option '%s -> %s'.", section, name); struct section *co; co = appconfig_section_find(root, section); if (!co) { - error("Could not destroy section option '%s -> %s'. The section not found.", section, name); + netdata_log_error("Could not destroy section option '%s -> %s'. The section not found.", section, name); return; } @@ -281,7 +281,7 @@ void appconfig_section_option_destroy_non_loaded(struct config *root, const char if (unlikely(!(cv && appconfig_option_index_del(co, cv)))) { config_section_unlock(co); - error("Could not destroy section option '%s -> %s'. The option not found.", section, name); + netdata_log_error("Could not destroy section option '%s -> %s'. The option not found.", section, name); return; } @@ -310,7 +310,7 @@ void appconfig_section_option_destroy_non_loaded(struct config *root, const char // config name-value methods static inline struct config_option *appconfig_value_create(struct section *co, const char *name, const char *value) { - debug(D_CONFIG, "Creating config entry for name '%s', value '%s', in section '%s'.", name, value, co->name); + netdata_log_debug(D_CONFIG, "Creating config entry for name '%s', value '%s', in section '%s'.", name, value, co->name); struct config_option *cv = callocz(1, sizeof(struct config_option)); cv->name = strdupz(name); @@ -319,7 +319,7 @@ static inline struct config_option *appconfig_value_create(struct section *co, c struct config_option *found = appconfig_option_index_add(co, cv); if(found != cv) { - error("indexing of config '%s' in section '%s': already exists - using the existing one.", cv->name, co->name); + netdata_log_error("indexing of config '%s' in section '%s': already exists - using the existing one.", cv->name, co->name); freez(cv->value); freez(cv->name); freez(cv); @@ -341,7 +341,7 @@ static inline struct config_option *appconfig_value_create(struct section *co, c int appconfig_exists(struct config *root, const char *section, const char *name) { struct config_option *cv; - debug(D_CONFIG, "request to get config in section '%s', name '%s'", section, name); + netdata_log_debug(D_CONFIG, "request to get config in section '%s', name '%s'", section, name); struct section *co = appconfig_section_find(root, section); if(!co) return 0; @@ -356,7 +356,7 @@ int appconfig_move(struct config *root, const char *section_old, const char *nam struct config_option *cv_old, *cv_new; int ret = -1; - debug(D_CONFIG, "request to rename config in section '%s', old name '%s', to section '%s', new name '%s'", section_old, name_old, section_new, name_new); + netdata_log_debug(D_CONFIG, "request to rename config in section '%s', old name '%s', to section '%s', new name '%s'", section_old, name_old, section_new, name_new); struct section *co_old = appconfig_section_find(root, section_old); if(!co_old) return ret; @@ -375,7 +375,7 @@ int appconfig_move(struct config *root, const char *section_old, const char *nam if(cv_new) goto cleanup; if(unlikely(appconfig_option_index_del(co_old, cv_old) != cv_old)) - error("INTERNAL ERROR: deletion of config '%s' from section '%s', deleted the wrong config entry.", cv_old->name, co_old->name); + netdata_log_error("INTERNAL ERROR: deletion of config '%s' from section '%s', deleted the wrong config entry.", cv_old->name, co_old->name); if(co_old->values == cv_old) { co_old->values = cv_old->next; @@ -384,7 +384,7 @@ int appconfig_move(struct config *root, const char *section_old, const char *nam struct config_option *t; for(t = co_old->values; t && t->next != cv_old ;t = t->next) ; if(!t || t->next != cv_old) - error("INTERNAL ERROR: cannot find variable '%s' in section '%s' of the config - but it should be there.", cv_old->name, co_old->name); + netdata_log_error("INTERNAL ERROR: cannot find variable '%s' in section '%s' of the config - but it should be there.", cv_old->name, co_old->name); else t->next = cv_old->next; } @@ -398,7 +398,7 @@ int appconfig_move(struct config *root, const char *section_old, const char *nam co_new->values = cv_new; if(unlikely(appconfig_option_index_add(co_new, cv_old) != cv_old)) - error("INTERNAL ERROR: re-indexing of config '%s' in section '%s', already exists.", cv_old->name, co_new->name); + netdata_log_error("INTERNAL ERROR: re-indexing of config '%s' in section '%s', already exists.", cv_old->name, co_new->name); ret = 0; @@ -439,9 +439,9 @@ char *appconfig_get_by_section(struct section *co, const char *name, const char char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value) { if (default_value == NULL) - debug(D_CONFIG, "request to get config in section '%s', name '%s' or fail", section, name); + netdata_log_debug(D_CONFIG, "request to get config in section '%s', name '%s' or fail", section, name); else - debug(D_CONFIG, "request to get config in section '%s', name '%s', default_value '%s'", section, name, default_value); + netdata_log_debug(D_CONFIG, "request to get config in section '%s', name '%s', default_value '%s'", section, name, default_value); struct section *co = appconfig_section_find(root, section); if (!co && !default_value) @@ -518,9 +518,9 @@ int appconfig_get_boolean_ondemand(struct config *root, const char *section, con s = appconfig_get(root, section, name, s); if(!s) return value; - if(!strcmp(s, "yes")) + if(!strcmp(s, "yes") || !strcmp(s, "true") || !strcmp(s, "on")) return CONFIG_BOOLEAN_YES; - else if(!strcmp(s, "no")) + else if(!strcmp(s, "no") || !strcmp(s, "false") || !strcmp(s, "off")) return CONFIG_BOOLEAN_NO; else if(!strcmp(s, "auto") || !strcmp(s, "on demand")) return CONFIG_BOOLEAN_AUTO; @@ -532,7 +532,7 @@ const char *appconfig_set_default(struct config *root, const char *section, cons { struct config_option *cv; - debug(D_CONFIG, "request to set default config in section '%s', name '%s', value '%s'", section, name, value); + netdata_log_debug(D_CONFIG, "request to set default config in section '%s', name '%s', value '%s'", section, name, value); struct section *co = appconfig_section_find(root, section); if(!co) return appconfig_set(root, section, name, value); @@ -559,7 +559,7 @@ const char *appconfig_set(struct config *root, const char *section, const char * { struct config_option *cv; - debug(D_CONFIG, "request to set config in section '%s', name '%s', value '%s'", section, name, value); + netdata_log_debug(D_CONFIG, "request to set config in section '%s', name '%s', value '%s'", section, name, value); struct section *co = appconfig_section_find(root, section); if(!co) co = appconfig_section_create(root, section); @@ -618,7 +618,7 @@ int appconfig_get_duration(struct config *root, const char *section, const char if(!s) goto fallback; if(!config_parse_duration(s, &result)) { - error("config option '[%s].%s = %s' is configured with an valid duration", section, name, s); + netdata_log_error("config option '[%s].%s = %s' is configured with an valid duration", section, name, s); goto fallback; } @@ -626,7 +626,7 @@ int appconfig_get_duration(struct config *root, const char *section, const char fallback: if(!config_parse_duration(value, &result)) - error("INTERNAL ERROR: default duration supplied for option '[%s].%s = %s' is not a valid duration", section, name, value); + netdata_log_error("INTERNAL ERROR: default duration supplied for option '[%s].%s = %s' is not a valid duration", section, name, value); return result; } @@ -649,11 +649,11 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used, cons if(!filename) filename = CONFIG_DIR "/" CONFIG_FILENAME; - debug(D_CONFIG, "CONFIG: opening config file '%s'", filename); + netdata_log_debug(D_CONFIG, "CONFIG: opening config file '%s'", filename); FILE *fp = fopen(filename, "r"); if(!fp) { - // info("CONFIG: cannot open file '%s'. Using internal defaults.", filename); + // netdata_log_info("CONFIG: cannot open file '%s'. Using internal defaults.", filename); return 0; } @@ -669,7 +669,7 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used, cons s = trim(buffer); if(!s || *s == '#') { - debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', it is empty.", line, filename); + netdata_log_debug(D_CONFIG, "CONFIG: ignoring line %d of file '%s', it is empty.", line, filename); continue; } @@ -696,13 +696,13 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used, cons strncpyz(working_instance, s, CONFIG_MAX_NAME); working_connector_section = NULL; if (unlikely(appconfig_section_find(root, working_instance))) { - error("Instance (%s) already exists", working_instance); + netdata_log_error("Instance (%s) already exists", working_instance); co = NULL; continue; } } else { co = NULL; - error("Section (%s) does not specify a valid connector", s); + netdata_log_error("Section (%s) does not specify a valid connector", s); continue; } } @@ -718,7 +718,7 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used, cons struct config_option *save = cv2->next; struct config_option *found = appconfig_option_index_del(co, cv2); if(found != cv2) - error("INTERNAL ERROR: Cannot remove '%s' from section '%s', it was not inserted before.", + netdata_log_error("INTERNAL ERROR: Cannot remove '%s' from section '%s', it was not inserted before.", cv2->name, co->name); freez(cv2->name); @@ -735,7 +735,7 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used, cons if(!co) { // line outside a section - error("CONFIG: ignoring line %d ('%s') of file '%s', it is outside all sections.", line, s, filename); + netdata_log_error("CONFIG: ignoring line %d ('%s') of file '%s', it is outside all sections.", line, s, filename); continue; } @@ -746,7 +746,7 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used, cons char *name = s; char *value = strchr(s, '='); if(!value) { - error("CONFIG: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename); + netdata_log_error("CONFIG: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename); continue; } *value = '\0'; @@ -756,7 +756,7 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used, cons value = trim(value); if(!name || *name == '#') { - error("CONFIG: ignoring line %d of file '%s', name is empty.", line, filename); + netdata_log_error("CONFIG: ignoring line %d of file '%s', name is empty.", line, filename); continue; } @@ -778,12 +778,12 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used, cons } } else { if (((cv->flags & CONFIG_VALUE_USED) && overwrite_used) || !(cv->flags & CONFIG_VALUE_USED)) { - debug( + netdata_log_debug( D_CONFIG, "CONFIG: line %d of file '%s', overwriting '%s/%s'.", line, filename, co->name, cv->name); freez(cv->value); cv->value = strdupz(value); } else - debug( + netdata_log_debug( D_CONFIG, "CONFIG: ignoring line %d of file '%s', '%s/%s' is already present and used.", line, diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c index 42e4a99f1..05da55344 100644 --- a/libnetdata/dictionary/dictionary.c +++ b/libnetdata/dictionary/dictionary.c @@ -147,14 +147,14 @@ struct dictionary { struct { // support for multiple indexing engines Pvoid_t JudyHSArray; // the hash table - netdata_rwlock_t rwlock; // protect the index + RW_SPINLOCK rw_spinlock; // protect the index } index; struct { DICTIONARY_ITEM *list; // the double linked list of all items in the dictionary - netdata_rwlock_t rwlock; // protect the linked-list + RW_SPINLOCK rw_spinlock; // protect the linked-list pid_t writer_pid; // the gettid() of the writer - size_t writer_depth; // nesting of write locks + uint32_t writer_depth; // nesting of write locks } items; struct dictionary_hooks *hooks; // pointer to external function callbacks to be called at certain points @@ -163,7 +163,7 @@ struct dictionary { DICTIONARY *master; // the master dictionary DICTIONARY *next; // linked list for delayed destruction (garbage collection of whole dictionaries) - size_t version; // the current version of the dictionary + uint32_t version; // the current version of the dictionary // it is incremented when: // - item added // - item removed @@ -171,9 +171,9 @@ struct dictionary { // - conflict callback returns true // - function dictionary_version_increment() is called - long int entries; // how many items are currently in the index (the linked list may have more) - long int referenced_items; // how many items of the dictionary are currently being used by 3rd parties - long int pending_deletion_items; // how many items of the dictionary have been deleted, but have not been removed yet + int32_t entries; // how many items are currently in the index (the linked list may have more) + int32_t referenced_items; // how many items of the dictionary are currently being used by 3rd parties + int32_t pending_deletion_items; // how many items of the dictionary have been deleted, but have not been removed yet #ifdef NETDATA_DICTIONARY_VALIDATE_POINTERS netdata_mutex_t global_pointer_registry_mutex; @@ -369,10 +369,6 @@ size_t dictionary_referenced_items(DICTIONARY *dict) { return referenced_items; } -long int dictionary_stats_for_registry(DICTIONARY *dict) { - if(unlikely(!dict)) return 0; - return (dict->stats->memory.index + dict->stats->memory.dict); -} void dictionary_version_increment(DICTIONARY *dict) { __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED); } @@ -632,19 +628,14 @@ static void dictionary_execute_delete_callback(DICTIONARY *dict, DICTIONARY_ITEM static inline size_t dictionary_locks_init(DICTIONARY *dict) { if(likely(!is_dictionary_single_threaded(dict))) { - netdata_rwlock_init(&dict->index.rwlock); - netdata_rwlock_init(&dict->items.rwlock); + rw_spinlock_init(&dict->index.rw_spinlock); + rw_spinlock_init(&dict->items.rw_spinlock); } return 0; } -static inline size_t dictionary_locks_destroy(DICTIONARY *dict) { - if(likely(!is_dictionary_single_threaded(dict))) { - netdata_rwlock_destroy(&dict->index.rwlock); - netdata_rwlock_destroy(&dict->items.rwlock); - } - +static inline size_t dictionary_locks_destroy(DICTIONARY *dict __maybe_unused) { return 0; } @@ -676,11 +667,11 @@ static inline void ll_recursive_lock(DICTIONARY *dict, char rw) { if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') { // read lock - netdata_rwlock_rdlock(&dict->items.rwlock); + rw_spinlock_read_lock(&dict->items.rw_spinlock); } else { // write lock - netdata_rwlock_wrlock(&dict->items.rwlock); + rw_spinlock_write_lock(&dict->items.rw_spinlock); ll_recursive_lock_set_thread_as_writer(dict); } } @@ -697,14 +688,14 @@ static inline void ll_recursive_unlock(DICTIONARY *dict, char rw) { if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') { // read unlock - netdata_rwlock_unlock(&dict->items.rwlock); + rw_spinlock_read_unlock(&dict->items.rw_spinlock); } else { // write unlock ll_recursive_unlock_unset_thread_writer(dict); - netdata_rwlock_unlock(&dict->items.rwlock); + rw_spinlock_write_unlock(&dict->items.rw_spinlock); } } @@ -719,27 +710,27 @@ static inline void dictionary_index_lock_rdlock(DICTIONARY *dict) { if(unlikely(is_dictionary_single_threaded(dict))) return; - netdata_rwlock_rdlock(&dict->index.rwlock); + rw_spinlock_read_lock(&dict->index.rw_spinlock); } static inline void dictionary_index_rdlock_unlock(DICTIONARY *dict) { if(unlikely(is_dictionary_single_threaded(dict))) return; - netdata_rwlock_unlock(&dict->index.rwlock); + rw_spinlock_read_unlock(&dict->index.rw_spinlock); } static inline void dictionary_index_lock_wrlock(DICTIONARY *dict) { if(unlikely(is_dictionary_single_threaded(dict))) return; - netdata_rwlock_wrlock(&dict->index.rwlock); + rw_spinlock_write_lock(&dict->index.rw_spinlock); } static inline void dictionary_index_wrlock_unlock(DICTIONARY *dict) { if(unlikely(is_dictionary_single_threaded(dict))) return; - netdata_rwlock_unlock(&dict->index.rwlock); + rw_spinlock_write_unlock(&dict->index.rw_spinlock); } // ---------------------------------------------------------------------------- @@ -957,7 +948,7 @@ static int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_ITEM *it if (having_index_lock) { // delete it from the hashtable if(hashtable_delete_unsafe(dict, item_get_name(item), item->key_len, item) == 0) - error("DICTIONARY: INTERNAL ERROR VIEW: tried to delete item with name '%s', name_len %u that is not in the index", item_get_name(item), (KEY_LEN_TYPE)(item->key_len - 1)); + netdata_log_error("DICTIONARY: INTERNAL ERROR VIEW: tried to delete item with name '%s', name_len %u that is not in the index", item_get_name(item), (KEY_LEN_TYPE)(item->key_len - 1)); else pointer_del(dict, item); @@ -1070,11 +1061,11 @@ static size_t hashtable_destroy_unsafe(DICTIONARY *dict) { JError_t J_Error; Word_t ret = JudyHSFreeArray(&dict->index.JudyHSArray, &J_Error); if(unlikely(ret == (Word_t) JERR)) { - error("DICTIONARY: Cannot destroy JudyHS, JU_ERRNO_* == %u, ID == %d", - JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); + netdata_log_error("DICTIONARY: Cannot destroy JudyHS, JU_ERRNO_* == %u, ID == %d", + JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); } - debug(D_DICTIONARY, "Dictionary: hash table freed %lu bytes", ret); + netdata_log_debug(D_DICTIONARY, "Dictionary: hash table freed %lu bytes", ret); dict->index.JudyHSArray = NULL; return (size_t)ret; @@ -1084,8 +1075,8 @@ static inline void **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, JError_t J_Error; Pvoid_t *Rc = JudyHSIns(&dict->index.JudyHSArray, (void *)name, name_len, &J_Error); if (unlikely(Rc == PJERR)) { - error("DICTIONARY: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d", - name, JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); + netdata_log_error("DICTIONARY: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d", + name, JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); } // if *Rc == 0, new item added to the array @@ -1105,8 +1096,9 @@ static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, si JError_t J_Error; int ret = JudyHSDel(&dict->index.JudyHSArray, (void *)name, name_len, &J_Error); if(unlikely(ret == JERR)) { - error("DICTIONARY: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d", name, - JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); + netdata_log_error("DICTIONARY: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d", + name, + JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); return 0; } @@ -1232,7 +1224,7 @@ void dictionary_static_items_aral_init(void) { static SPINLOCK spinlock; if(unlikely(!dict_items_aral || !dict_shared_items_aral)) { - netdata_spinlock_lock(&spinlock); + spinlock_lock(&spinlock); // we have to check again if(!dict_items_aral) @@ -1254,7 +1246,7 @@ void dictionary_static_items_aral_init(void) { aral_by_size_statistics(), NULL, NULL, false, false); - netdata_spinlock_unlock(&spinlock); + spinlock_unlock(&spinlock); } } @@ -1384,7 +1376,7 @@ static void dict_item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM * if(unlikely(is_view_dictionary(dict))) fatal("DICTIONARY: %s() should never be called on views.", __FUNCTION__ ); - debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", item_get_name(item)); + netdata_log_debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", item_get_name(item)); DICTIONARY_VALUE_RESETS_PLUS1(dict); @@ -1396,12 +1388,12 @@ static void dict_item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM * dictionary_execute_delete_callback(dict, item); if(likely(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE)) { - debug(D_DICTIONARY, "Dictionary: linking value to '%s'", item_get_name(item)); + netdata_log_debug(D_DICTIONARY, "Dictionary: linking value to '%s'", item_get_name(item)); item->shared->value = value; item->shared->value_len = value_len; } else { - debug(D_DICTIONARY, "Dictionary: cloning value to '%s'", item_get_name(item)); + netdata_log_debug(D_DICTIONARY, "Dictionary: cloning value to '%s'", item_get_name(item)); void *old_value = item->shared->value; void *new_value = NULL; @@ -1413,7 +1405,7 @@ static void dict_item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM * item->shared->value = new_value; item->shared->value_len = value_len; - debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", item_get_name(item)); + netdata_log_debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", item_get_name(item)); dict_item_value_freez(dict, old_value); } @@ -1421,7 +1413,7 @@ static void dict_item_reset_value_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM * } static size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item) { - debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", item_get_name(item)); + netdata_log_debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", item_get_name(item)); if(!item_flag_check(item, ITEM_FLAG_DELETED)) DICTIONARY_ENTRIES_MINUS1(dict); @@ -1436,7 +1428,7 @@ static size_t dict_item_free_with_hooks(DICTIONARY *dict, DICTIONARY_ITEM *item) dictionary_execute_delete_callback(dict, item); if(unlikely(!(dict->options & DICT_OPTION_VALUE_LINK_DONT_CLONE))) { - debug(D_DICTIONARY, "Dictionary freeing value of '%s'", item_get_name(item)); + netdata_log_debug(D_DICTIONARY, "Dictionary freeing value of '%s'", item_get_name(item)); dict_item_value_freez(dict, item->shared->value); item->shared->value = NULL; } @@ -1562,7 +1554,7 @@ static bool dict_item_del(DICTIONARY *dict, const char *name, ssize_t name_len) if(name_len == -1) name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too - debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name); + netdata_log_debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name); // Unfortunately, the JudyHSDel() does not return the value of the // item that was deleted, so we have to find it before we delete it, @@ -1578,7 +1570,9 @@ static bool dict_item_del(DICTIONARY *dict, const char *name, ssize_t name_len) } else { if(hashtable_delete_unsafe(dict, name, name_len, item) == 0) - error("DICTIONARY: INTERNAL ERROR: tried to delete item with name '%s', name_len %zd that is not in the index", name, name_len - 1); + netdata_log_error("DICTIONARY: INTERNAL ERROR: tried to delete item with name '%s', name_len %zd that is not in the index", + name, + name_len - 1); else pointer_del(dict, item); @@ -1611,7 +1605,7 @@ static DICTIONARY_ITEM *dict_item_add_or_reset_value_and_acquire(DICTIONARY *dic if(name_len == -1) name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too - debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name); + netdata_log_debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name); // DISCUSSION: // Is it better to gain a read-lock and do a hashtable_get_unsafe() @@ -1673,7 +1667,7 @@ static DICTIONARY_ITEM *dict_item_add_or_reset_value_and_acquire(DICTIONARY *dic // view dictionary // the item is already there and can be used if(item->shared != master_item->shared) - error("DICTIONARY: changing the master item on a view is not supported. The previous item will remain. To change the key of an item in a view, delete it and add it again."); + netdata_log_error("DICTIONARY: changing the master item on a view is not supported. The previous item will remain. To change the key of an item in a view, delete it and add it again."); } else { // master dictionary @@ -1730,7 +1724,7 @@ static DICTIONARY_ITEM *dict_item_find_and_acquire(DICTIONARY *dict, const char if(name_len == -1) name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too - debug(D_DICTIONARY, "GET dictionary entry with name '%s'.", name); + netdata_log_debug(D_DICTIONARY, "GET dictionary entry with name '%s'.", name); dictionary_index_lock_rdlock(dict); @@ -2096,7 +2090,7 @@ size_t dictionary_destroy(DICTIONARY *dict) { internal_error( true, - "DICTIONARY: delaying destruction of dictionary created from %s() %zu@%s, because it has %ld referenced items in it (%ld total).", + "DICTIONARY: delaying destruction of dictionary created from %s() %zu@%s, because it has %d referenced items in it (%d total).", dict->creation_function, dict->creation_line, dict->creation_file, @@ -2560,8 +2554,8 @@ void thread_cache_destroy(void) { JError_t J_Error; Word_t ret = JudyHSFreeArray(&thread_cache_judy_array, &J_Error); if(unlikely(ret == (Word_t) JERR)) { - error("THREAD_CACHE: Cannot destroy JudyHS, JU_ERRNO_* == %u, ID == %d", - JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); + netdata_log_error("THREAD_CACHE: Cannot destroy JudyHS, JU_ERRNO_* == %u, ID == %d", + JU_ERRNO(&J_Error), JU_ERRID(&J_Error)); } internal_error(true, "THREAD_CACHE: hash table freed %lu bytes", ret); @@ -2842,7 +2836,7 @@ static usec_t dictionary_unittest_run_and_measure_time(DICTIONARY *dict, char *m } } - fprintf(stderr, " %zu errors, %ld (found %ld) items in dictionary, %ld (found %ld) referenced, %ld (found %ld) deleted, %llu usec \n", + fprintf(stderr, " %zu errors, %d (found %ld) items in dictionary, %d (found %ld) referenced, %d (found %ld) deleted, %llu usec \n", errs, dict?dict->entries:0, found_ok, dict?dict->referenced_items:0, found_referenced, dict?dict->pending_deletion_items:0, found_deleted, dt); *errors += errs; return dt; @@ -2984,7 +2978,7 @@ static size_t unittest_check_dictionary(const char *label, DICTIONARY *dict, siz referenced++; } - fprintf(stderr, "DICT %-20s: dictionary active items reported %ld, counted %zu, expected %zu...\t\t\t", + fprintf(stderr, "DICT %-20s: dictionary active items reported %d, counted %zu, expected %zu...\t\t\t", label, dict->entries, active, active_items); if(active != active_items || active != (size_t)dict->entries) { fprintf(stderr, "FAILED\n"); @@ -3002,7 +2996,7 @@ static size_t unittest_check_dictionary(const char *label, DICTIONARY *dict, siz else fprintf(stderr, "OK\n"); - fprintf(stderr, "DICT %-20s: dictionary referenced items reported %ld, counted %zu, expected %zu...\t\t", + fprintf(stderr, "DICT %-20s: dictionary referenced items reported %d, counted %zu, expected %zu...\t\t", label, dict->referenced_items, referenced, referenced_items); if(referenced != referenced_items || dict->referenced_items != (long int)referenced) { fprintf(stderr, "FAILED\n"); @@ -3011,7 +3005,7 @@ static size_t unittest_check_dictionary(const char *label, DICTIONARY *dict, siz else fprintf(stderr, "OK\n"); - fprintf(stderr, "DICT %-20s: dictionary pending deletion items reported %ld, counted %zu, expected %zu...\t", + fprintf(stderr, "DICT %-20s: dictionary pending deletion items reported %d, counted %zu, expected %zu...\t", label, dict->pending_deletion_items, pending, pending_deletion); if(pending != pending_deletion || pending != (size_t)dict->pending_deletion_items) { fprintf(stderr, "FAILED\n"); @@ -3257,9 +3251,9 @@ static int dictionary_unittest_threads() { ", searches %zu" ", resets %zu" ", flushes %zu" - ", entries %ld" - ", referenced_items %ld" - ", pending deletions %ld" + ", entries %d" + ", referenced_items %d" + ", pending deletions %d" ", check spins %zu" ", insert spins %zu" ", delete spins %zu" @@ -3418,9 +3412,9 @@ static int dictionary_unittest_view_threads() { ", deletes %zu" ", searches %zu" ", resets %zu" - ", entries %ld" - ", referenced_items %ld" - ", pending deletions %ld" + ", entries %d" + ", referenced_items %d" + ", pending deletions %d" ", check spins %zu" ", insert spins %zu" ", delete spins %zu" @@ -3443,9 +3437,9 @@ static int dictionary_unittest_view_threads() { ", deletes %zu" ", searches %zu" ", resets %zu" - ", entries %ld" - ", referenced_items %ld" - ", pending deletions %ld" + ", entries %d" + ", referenced_items %d" + ", pending deletions %d" ", check spins %zu" ", insert spins %zu" ", delete spins %zu" diff --git a/libnetdata/dictionary/dictionary.h b/libnetdata/dictionary/dictionary.h index c13d784cb..eea14d3fa 100644 --- a/libnetdata/dictionary/dictionary.h +++ b/libnetdata/dictionary/dictionary.h @@ -46,7 +46,7 @@ typedef struct dictionary DICTIONARY; typedef struct dictionary_item DICTIONARY_ITEM; -typedef enum dictionary_options { +typedef enum __attribute__((packed)) dictionary_options { DICT_OPTION_NONE = 0, // the default is the opposite of all below DICT_OPTION_SINGLE_THREADED = (1 << 0), // don't use any locks (default: use locks) DICT_OPTION_VALUE_LINK_DONT_CLONE = (1 << 1), // don't copy the value, just point to the one provided (default: copy) @@ -310,7 +310,6 @@ void dictionary_foreach_unlock(DICTFE *dfe); size_t dictionary_version(DICTIONARY *dict); size_t dictionary_entries(DICTIONARY *dict); size_t dictionary_referenced_items(DICTIONARY *dict); -long int dictionary_stats_for_registry(DICTIONARY *dict); // for all cases that the caller does not provide a stats structure, this is where they are accumulated. extern struct dictionary_stats dictionary_stats_category_other; diff --git a/libnetdata/ebpf/ebpf.c b/libnetdata/ebpf/ebpf.c index b980d09ed..6793f403a 100644 --- a/libnetdata/ebpf/ebpf.c +++ b/libnetdata/ebpf/ebpf.c @@ -44,7 +44,6 @@ static int clean_kprobe_event(FILE *out, char *filename, char *father_pid, netda int clean_kprobe_events(FILE *out, int pid, netdata_ebpf_events_t *ptr) { - debug(D_EXIT, "Cleaning parent process events."); char filename[FILENAME_MAX + 1]; snprintf(filename, FILENAME_MAX, "%s%s", NETDATA_DEBUGFS, "kprobe_events"); @@ -183,7 +182,7 @@ static int kernel_is_rejected() if (read_file("/proc/version", version_string, VERSION_STRING_LEN)) { struct utsname uname_buf; if (!uname(&uname_buf)) { - info("Cannot check kernel version"); + netdata_log_info("Cannot check kernel version"); return 0; } version_string_len = @@ -230,7 +229,7 @@ static int kernel_is_rejected() while ((reject_string_len = getline(&reject_string, &buf_len, kernel_reject_list) - 1) > 0) { if (version_string_len >= reject_string_len) { if (!strncmp(version_string, reject_string, reject_string_len)) { - info("A buggy kernel is detected"); + netdata_log_info("A buggy kernel is detected"); fclose(kernel_reject_list); freez(reject_string); return 1; @@ -392,9 +391,10 @@ static void ebpf_mount_name(char *out, size_t len, char *path, uint32_t kver, co * Count the information from targets. * * @param report the output structure - * @param targets vector with information about the eBPF plugin. + * @param targets vector with information about the eBPF plugin. + * @param value factor used to update calculation */ -static void ebpf_stats_targets(ebpf_plugin_stats_t *report, netdata_ebpf_targets_t *targets) +static void ebpf_stats_targets(ebpf_plugin_stats_t *report, netdata_ebpf_targets_t *targets, int value) { if (!targets) { report->probes = report->tracepoints = report->trampolines = 0; @@ -405,19 +405,19 @@ static void ebpf_stats_targets(ebpf_plugin_stats_t *report, netdata_ebpf_targets while (targets[i].name) { switch (targets[i].mode) { case EBPF_LOAD_PROBE: { - report->probes++; + report->probes += value; break; } case EBPF_LOAD_RETPROBE: { - report->retprobes++; + report->retprobes += value; break; } case EBPF_LOAD_TRACEPOINT: { - report->tracepoints++; + report->tracepoints += value; break; } case EBPF_LOAD_TRAMPOLINE: { - report->trampolines++; + report->trampolines += value; break; } } @@ -438,27 +438,30 @@ static void ebpf_stats_targets(ebpf_plugin_stats_t *report, netdata_ebpf_targets */ void ebpf_update_stats(ebpf_plugin_stats_t *report, ebpf_module_t *em) { - report->threads++; + int value; // It is not necessary to report more information. - if (em->enabled != NETDATA_THREAD_EBPF_RUNNING) - return; + if (em->enabled > NETDATA_THREAD_EBPF_FUNCTION_RUNNING) + value = -1; + else + value = 1; - report->running++; + report->threads += value; + report->running += value; // In theory the `else if` is useless, because when this function is called, the module should not stay in // EBPF_LOAD_PLAY_DICE. We have this additional condition to detect errors from developers. if (em->load & EBPF_LOAD_LEGACY) - report->legacy++; + report->legacy += value; else if (em->load & EBPF_LOAD_CORE) - report->core++; + report->core += value; if (em->maps_per_core) - report->hash_percpu++; + report->hash_percpu += value; else - report->hash_unique++; + report->hash_unique += value; - ebpf_stats_targets(report, em->targets); + ebpf_stats_targets(report, em->targets, value); } /** @@ -477,7 +480,7 @@ void ebpf_update_kernel_memory(ebpf_plugin_stats_t *report, ebpf_local_maps_t *m snprintfz(filename, FILENAME_MAX, "/proc/self/fdinfo/%d", map->map_fd); procfile *ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT); if(unlikely(!ff)) { - error("Cannot open %s", filename); + netdata_log_error("Cannot open %s", filename); return; } @@ -496,7 +499,7 @@ void ebpf_update_kernel_memory(ebpf_plugin_stats_t *report, ebpf_local_maps_t *m report->memlock_kern += memsize; report->hash_tables += 1; #ifdef NETDATA_DEV_MODE - info("Hash table %u: %s (FD = %d) is consuming %lu bytes totalizing %lu bytes", + netdata_log_info("Hash table %u: %s (FD = %d) is consuming %lu bytes totalizing %lu bytes", report->hash_tables, map->name, map->map_fd, memsize, report->memlock_kern); #endif break; @@ -505,7 +508,7 @@ void ebpf_update_kernel_memory(ebpf_plugin_stats_t *report, ebpf_local_maps_t *m report->memlock_kern -= memsize; report->hash_tables -= 1; #ifdef NETDATA_DEV_MODE - info("Hash table %s (FD = %d) was removed releasing %lu bytes, now we have %u tables loaded totalizing %lu bytes.", + netdata_log_info("Hash table %s (FD = %d) was removed releasing %lu bytes, now we have %u tables loaded totalizing %lu bytes.", map->name, map->map_fd, memsize, report->hash_tables, report->memlock_kern); #endif break; @@ -529,8 +532,11 @@ void ebpf_update_kernel_memory(ebpf_plugin_stats_t *report, ebpf_local_maps_t *m * * @param report the output structure * @param map pointer to a map. Last map must fish with name = NULL + * @param action should plugin add or remove values from amount. */ -void ebpf_update_kernel_memory_with_vector(ebpf_plugin_stats_t *report, ebpf_local_maps_t *maps) +void ebpf_update_kernel_memory_with_vector(ebpf_plugin_stats_t *report, + ebpf_local_maps_t *maps, + ebpf_stats_action_t action) { if (!maps) return; @@ -542,7 +548,7 @@ void ebpf_update_kernel_memory_with_vector(ebpf_plugin_stats_t *report, ebpf_loc if (fd == ND_EBPF_MAP_FD_NOT_INITIALIZED) continue; - ebpf_update_kernel_memory(report, map, EBPF_ACTION_STAT_ADD); + ebpf_update_kernel_memory(report, map, action); } } @@ -570,7 +576,7 @@ void ebpf_update_map_size(struct bpf_map *map, ebpf_local_maps_t *lmap, ebpf_mod if (lmap->user_input && lmap->user_input != lmap->internal_input) { define_size = lmap->internal_input; #ifdef NETDATA_INTERNAL_CHECKS - info("Changing map %s from size %u to %u ", map_name, lmap->internal_input, lmap->user_input); + netdata_log_info("Changing map %s from size %u to %u ", map_name, lmap->internal_input, lmap->user_input); #endif } else if (((lmap->type & apps_type) == apps_type) && (!em->apps_charts) && (!em->cgroup_charts)) { lmap->user_input = ND_EBPF_DEFAULT_MIN_PID; @@ -613,7 +619,7 @@ void ebpf_update_map_size(struct bpf_map *map, ebpf_local_maps_t *lmap, ebpf_mod void ebpf_update_map_type(struct bpf_map *map, ebpf_local_maps_t *w) { if (bpf_map__set_type(map, w->map_type)) { - error("Cannot modify map type for %s", w->name); + netdata_log_error("Cannot modify map type for %s", w->name); } } @@ -794,7 +800,7 @@ void ebpf_update_controller(int fd, ebpf_module_t *em) for (key = NETDATA_CONTROLLER_APPS_ENABLED; key < end; key++) { int ret = bpf_map_update_elem(fd, &key, &values[key], 0); if (ret) - error("Add key(%u) for controller table failed.", key); + netdata_log_error("Add key(%u) for controller table failed.", key); } } @@ -867,7 +873,7 @@ struct bpf_link **ebpf_load_program(char *plugins_dir, ebpf_module_t *em, int kv ebpf_update_legacy_map(*obj, em); if (bpf_object__load(*obj)) { - error("ERROR: loading BPF object file failed %s\n", lpath); + netdata_log_error("ERROR: loading BPF object file failed %s\n", lpath); bpf_object__close(*obj); return NULL; } @@ -878,7 +884,7 @@ struct bpf_link **ebpf_load_program(char *plugins_dir, ebpf_module_t *em, int kv size_t count_programs = ebpf_count_programs(*obj); #ifdef NETDATA_INTERNAL_CHECKS - info("eBPF program %s loaded with success!", lpath); + netdata_log_info("eBPF program %s loaded with success!", lpath); #endif return ebpf_attach_programs(*obj, count_programs, em->names); @@ -891,7 +897,7 @@ char *ebpf_find_symbol(char *search) snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, NETDATA_KALLSYMS); procfile *ff = procfile_open(filename, " \t", PROCFILE_FLAG_DEFAULT); if(unlikely(!ff)) { - error("Cannot open %s%s", netdata_configured_host_prefix, NETDATA_KALLSYMS); + netdata_log_error("Cannot open %s%s", netdata_configured_host_prefix, NETDATA_KALLSYMS); return ret; } @@ -1107,7 +1113,7 @@ struct btf *ebpf_load_btf_file(char *path, char *filename) snprintfz(fullpath, PATH_MAX, "%s/%s", path, filename); struct btf *ret = ebpf_parse_btf_file(fullpath); if (!ret) - info("Your environment does not have BTF file %s/%s. The plugin will work with 'legacy' code.", + netdata_log_info("Your environment does not have BTF file %s/%s. The plugin will work with 'legacy' code.", path, filename); return ret; @@ -1239,6 +1245,9 @@ void ebpf_update_module_using_config(ebpf_module_t *modules, netdata_ebpf_load_m modules->pid_map_size = (uint32_t)appconfig_get_number(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_PID_SIZE, modules->pid_map_size); + modules->lifetime = (uint32_t) appconfig_get_number(modules->cfg, EBPF_GLOBAL_SECTION, + EBPF_CFG_LIFETIME, EBPF_DEFAULT_LIFETIME); + char *value = ebpf_convert_load_mode_to_string(modules->load & NETDATA_EBPF_LOAD_METHODS); char *type_format = appconfig_get(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_TYPE_FORMAT, value); netdata_ebpf_load_mode_t load = epbf_convert_string_to_load_mode(type_format); @@ -1259,7 +1268,7 @@ void ebpf_update_module_using_config(ebpf_module_t *modules, netdata_ebpf_load_m modules->maps_per_core = CONFIG_BOOLEAN_NO; #ifdef NETDATA_DEV_MODE - info("The thread %s was configured with: mode = %s; update every = %d; apps = %s; cgroup = %s; ebpf type format = %s; ebpf co-re tracing = %s; collect pid = %s; maps per core = %s", + netdata_log_info("The thread %s was configured with: mode = %s; update every = %d; apps = %s; cgroup = %s; ebpf type format = %s; ebpf co-re tracing = %s; collect pid = %s; maps per core = %s, lifetime=%u", modules->thread_name, load_mode, modules->update_every, @@ -1268,7 +1277,8 @@ void ebpf_update_module_using_config(ebpf_module_t *modules, netdata_ebpf_load_m type_format, core_attach, collect_pid, - (modules->maps_per_core)?"enabled":"disabled" + (modules->maps_per_core)?"enabled":"disabled", + modules->lifetime ); #endif } @@ -1295,7 +1305,7 @@ void ebpf_update_module(ebpf_module_t *em, struct btf *btf_file, int kver, int i if (!ebpf_load_config(em->cfg, filename)) { ebpf_mount_config_name(filename, FILENAME_MAX, ebpf_stock_config_dir, em->config_file); if (!ebpf_load_config(em->cfg, filename)) { - error("Cannot load the ebpf configuration file %s", em->config_file); + netdata_log_error("Cannot load the ebpf configuration file %s", em->config_file); return; } // If user defined data globally, we will have here EBPF_LOADED_FROM_USER, we need to consider this, to avoid @@ -1512,7 +1522,7 @@ int ebpf_is_tracepoint_enabled(char *subsys, char *eventname) static int ebpf_change_tracing_values(char *subsys, char *eventname, char *value) { if (strcmp("0", value) && strcmp("1", value)) { - error("Invalid value given to either enable or disable a tracepoint."); + netdata_log_error("Invalid value given to either enable or disable a tracepoint."); return -1; } diff --git a/libnetdata/ebpf/ebpf.h b/libnetdata/ebpf/ebpf.h index e82aaedd4..88dbca379 100644 --- a/libnetdata/ebpf/ebpf.h +++ b/libnetdata/ebpf/ebpf.h @@ -43,6 +43,7 @@ #define EBPF_CFG_MAPS_PER_CORE "maps per core" #define EBPF_CFG_UPDATE_EVERY "update every" +#define EBPF_CFG_LIFETIME "lifetime" #define EBPF_CFG_UPDATE_APPS_EVERY_DEFAULT 10 #define EBPF_CFG_PID_SIZE "pid table size" #define EBPF_CFG_APPLICATION "apps" @@ -270,15 +271,17 @@ typedef enum netdata_apps_integration_flags { #define NETDATA_EBPF_STAT_DIMENSION_ARAL "aral" enum ebpf_threads_status { - NETDATA_THREAD_EBPF_RUNNING, - NETDATA_THREAD_EBPF_STOPPING, - NETDATA_THREAD_EBPF_STOPPED, - NETDATA_THREAD_EBPF_NOT_RUNNING + NETDATA_THREAD_EBPF_RUNNING, // started by plugin + NETDATA_THREAD_EBPF_FUNCTION_RUNNING, // started by function + NETDATA_THREAD_EBPF_STOPPING, // stopping thread + NETDATA_THREAD_EBPF_STOPPED, // thread stopped + NETDATA_THREAD_EBPF_NOT_RUNNING // thread was never started }; typedef struct ebpf_module { const char *thread_name; const char *config_name; + const char *thread_description; enum ebpf_threads_status enabled; void *(*start_routine)(void *); int update_every; @@ -306,8 +309,16 @@ typedef struct ebpf_module { char memory_usage[NETDATA_EBPF_CHART_MEM_LENGTH]; char memory_allocations[NETDATA_EBPF_CHART_MEM_LENGTH]; int maps_per_core; + + // period to run + uint32_t running_time; // internal usage, this is used to reset a value when a new request happens. + uint32_t lifetime; } ebpf_module_t; +#define EBPF_DEFAULT_LIFETIME 300 +// This will be present until all functions are merged +#define EBPF_NON_FUNCTION_LIFE_TIME 86400 + int ebpf_get_kernel_version(); int get_redhat_release(); int has_condition_to_run(int version); @@ -336,10 +347,21 @@ void ebpf_update_map_size(struct bpf_map *map, ebpf_local_maps_t *lmap, ebpf_mod typedef struct netdata_ebpf_histogram { char *name; char *title; + char *ctx; int order; uint64_t histogram[NETDATA_EBPF_HIST_MAX_BINS]; } netdata_ebpf_histogram_t; +enum fs_btf_counters { + NETDATA_KEY_BTF_READ, + NETDATA_KEY_BTF_WRITE, + NETDATA_KEY_BTF_OPEN, + NETDATA_KEY_BTF_SYNC_ATTR, + NETDATA_KEY_BTF_OPEN2, + + NETDATA_FS_BTF_END +}; + typedef struct ebpf_filesystem_partitions { char *filesystem; char *optional_filesystem; @@ -359,6 +381,14 @@ typedef struct ebpf_filesystem_partitions { ebpf_addresses_t addresses; uint64_t kernels; ebpf_local_maps_t *fs_maps; + + // BPF structure +#ifdef LIBBPF_MAJOR_VERSION + struct filesystem_bpf *fs_obj; +#else + void *fs_obj; +#endif + const char *functions[NETDATA_FS_BTF_END]; } ebpf_filesystem_partitions_t; typedef struct ebpf_sync_syscalls { @@ -407,9 +437,11 @@ void ebpf_update_map_type(struct bpf_map *map, ebpf_local_maps_t *w); void ebpf_define_map_type(ebpf_local_maps_t *maps, int maps_per_core, int kver); #endif -void ebpf_update_kernel_memory_with_vector(ebpf_plugin_stats_t *report, ebpf_local_maps_t *maps); +void ebpf_update_kernel_memory_with_vector(ebpf_plugin_stats_t *report, ebpf_local_maps_t *maps, + ebpf_stats_action_t action); void ebpf_update_kernel_memory(ebpf_plugin_stats_t *report, ebpf_local_maps_t *map, ebpf_stats_action_t action); -void ebpf_statistic_create_aral_chart(char *name, ebpf_module_t *em); +int ebpf_statistic_create_aral_chart(char *name, ebpf_module_t *em); +void ebpf_statistic_obsolete_aral_chart(ebpf_module_t *em, int prio); void ebpf_send_data_aral_chart(ARAL *memory, ebpf_module_t *em); #endif /* NETDATA_EBPF_H */ diff --git a/libnetdata/eval/eval.c b/libnetdata/eval/eval.c index c7570bd2f..236be932b 100644 --- a/libnetdata/eval/eval.c +++ b/libnetdata/eval/eval.c @@ -1122,14 +1122,14 @@ EVAL_EXPRESSION *expression_parse(const char *string, const char **failed_at, in if(!op) { unsigned long pos = s - string + 1; - error("failed to parse expression '%s': %s at character %lu (i.e.: '%s').", string, expression_strerror(err), pos, s); + netdata_log_error("failed to parse expression '%s': %s at character %lu (i.e.: '%s').", string, expression_strerror(err), pos, s); return NULL; } BUFFER *out = buffer_create(1024, NULL); print_parsed_as_node(out, op, &err); if(err != EVAL_ERROR_OK) { - error("failed to re-generate expression '%s' with reason: %s", string, expression_strerror(err)); + netdata_log_error("failed to re-generate expression '%s' with reason: %s", string, expression_strerror(err)); eval_node_free(op); buffer_free(out); return NULL; diff --git a/libnetdata/eval/eval.h b/libnetdata/eval/eval.h index 1633ec505..05a26936b 100644 --- a/libnetdata/eval/eval.h +++ b/libnetdata/eval/eval.h @@ -12,9 +12,9 @@ typedef enum rrdcalc_status { RRDCALC_STATUS_UNDEFINED = -1, RRDCALC_STATUS_UNINITIALIZED = 0, RRDCALC_STATUS_CLEAR = 1, - RRDCALC_STATUS_RAISED = 2, - RRDCALC_STATUS_WARNING = 3, - RRDCALC_STATUS_CRITICAL = 4 + RRDCALC_STATUS_RAISED = 2, // DO NOT CHANGE THESE NUMBERS + RRDCALC_STATUS_WARNING = 3, // DO NOT CHANGE THESE NUMBERS + RRDCALC_STATUS_CRITICAL = 4, // DO NOT CHANGE THESE NUMBERS } RRDCALC_STATUS; typedef struct eval_variable { diff --git a/libnetdata/health/health.c b/libnetdata/health/health.c index d5403cefa..53ebecb42 100644 --- a/libnetdata/health/health.c +++ b/libnetdata/health/health.c @@ -11,7 +11,7 @@ SILENCERS *silencers; */ SILENCER *create_silencer(void) { SILENCER *t = callocz(1, sizeof(SILENCER)); - debug(D_HEALTH, "HEALTH command API: Created empty silencer"); + netdata_log_debug(D_HEALTH, "HEALTH command API: Created empty silencer"); return t; } @@ -27,7 +27,7 @@ void health_silencers_add(SILENCER *silencer) { // Add the created instance to the linked list in silencers silencer->next = silencers->silencers; silencers->silencers = silencer; - debug(D_HEALTH, "HEALTH command API: Added silencer %s:%s:%s:%s:%s", silencer->alarms, + netdata_log_debug(D_HEALTH, "HEALTH command API: Added silencer %s:%s:%s:%s:%s", silencer->alarms, silencer->charts, silencer->contexts, silencer->hosts, silencer->families ); } @@ -73,7 +73,7 @@ SILENCER *health_silencers_addparam(SILENCER *silencer, char *key, char *value) ) { silencer = create_silencer(); if(!silencer) { - error("Cannot add a new silencer to Netdata"); + netdata_log_error("Cannot add a new silencer to Netdata"); return NULL; } } @@ -116,7 +116,7 @@ int health_silencers_json_read_callback(JSON_ENTRY *e) e->callback_function = health_silencers_json_read_callback; if(strcmp(e->name,"")) { // init silencer - debug(D_HEALTH, "JSON: Got object with a name, initializing new silencer for %s",e->name); + netdata_log_debug(D_HEALTH, "JSON: Got object with a name, initializing new silencer for %s",e->name); #endif e->callback_data = create_silencer(); if(e->callback_data) { @@ -133,18 +133,18 @@ int health_silencers_json_read_callback(JSON_ENTRY *e) case JSON_STRING: if(!strcmp(e->name,"type")) { - debug(D_HEALTH, "JSON: Processing type=%s",e->data.string); + netdata_log_debug(D_HEALTH, "JSON: Processing type=%s",e->data.string); if (!strcmp(e->data.string,"SILENCE")) silencers->stype = STYPE_SILENCE_NOTIFICATIONS; else if (!strcmp(e->data.string,"DISABLE")) silencers->stype = STYPE_DISABLE_ALARMS; } else { - debug(D_HEALTH, "JSON: Adding %s=%s", e->name, e->data.string); + netdata_log_debug(D_HEALTH, "JSON: Adding %s=%s", e->name, e->data.string); if (e->callback_data) (void)health_silencers_addparam(e->callback_data, e->name, e->data.string); } break; case JSON_BOOLEAN: - debug(D_HEALTH, "JSON: Processing all_alarms"); + netdata_log_debug(D_HEALTH, "JSON: Processing all_alarms"); silencers->all_alarms=e->data.boolean?1:0; break; diff --git a/libnetdata/http/http_defs.h b/libnetdata/http/http_defs.h index 774ea0b71..906e3bdfb 100644 --- a/libnetdata/http/http_defs.h +++ b/libnetdata/http/http_defs.h @@ -10,6 +10,7 @@ #define HTTP_RESP_MOVED_PERM 301 #define HTTP_RESP_REDIR_TEMP 307 #define HTTP_RESP_REDIR_PERM 308 +#define HTTP_RESP_HTTPS_UPGRADE 399 // HTTP_CODES 4XX Client Errors #define HTTP_RESP_BAD_REQUEST 400 diff --git a/libnetdata/json/json.c b/libnetdata/json/json.c index 532b677ce..ec1452911 100644 --- a/libnetdata/json/json.c +++ b/libnetdata/json/json.c @@ -22,13 +22,13 @@ int json_tokens = JSON_TOKENS; #ifdef ENABLE_JSONC json_object *json_tokenise(char *js) { if(!js) { - error("JSON: json string is empty."); + netdata_log_error("JSON: json string is empty."); return NULL; } json_object *token = json_tokener_parse(js); if(!token) { - error("JSON: Invalid json string."); + netdata_log_error("JSON: Invalid json string."); return NULL; } @@ -39,7 +39,7 @@ jsmntok_t *json_tokenise(char *js, size_t len, size_t *count) { int n = json_tokens; if(!js || !len) { - error("JSON: json string is empty."); + netdata_log_error("JSON: json string is empty."); return NULL; } @@ -62,12 +62,12 @@ jsmntok_t *json_tokenise(char *js, size_t len, size_t *count) } if (ret == JSMN_ERROR_INVAL) { - error("JSON: Invalid json string."); + netdata_log_error("JSON: Invalid json string."); freez(tokens); return NULL; } else if (ret == JSMN_ERROR_PART) { - error("JSON: Truncated JSON string."); + netdata_log_error("JSON: Truncated JSON string."); freez(tokens); return NULL; } @@ -124,7 +124,7 @@ int json_callback_print(JSON_ENTRY *e) buffer_strcat(wb,"NULL"); break; } - info("JSON: %s", buffer_tostring(wb)); + netdata_log_info("JSON: %s", buffer_tostring(wb)); buffer_free(wb); return 0; } @@ -323,7 +323,7 @@ size_t json_walk_array(char *js, jsmntok_t *t, size_t nest, size_t start, JSON_E for(i = 0; i < size ; i++) { ne.pos = i; if (strlen(e->name) > JSON_NAME_LEN - 24 || strlen(e->fullname) > JSON_FULLNAME_LEN -24) { - info("JSON: JSON walk_array ignoring element with name:%s fullname:%s",e->name, e->fullname); + netdata_log_info("JSON: JSON walk_array ignoring element with name:%s fullname:%s",e->name, e->fullname); continue; } snprintfz(ne.name, JSON_NAME_LEN, "%s[%lu]", e->name, i); diff --git a/libnetdata/july/july.c b/libnetdata/july/july.c index 0ad5f13e5..56b8494b3 100644 --- a/libnetdata/july/july.c +++ b/libnetdata/july/july.c @@ -59,7 +59,7 @@ static struct { void julyl_cleanup1(void) { struct JulyL *item = NULL; - if(!netdata_spinlock_trylock(&julyl_globals.protected.spinlock)) + if(!spinlock_trylock(&julyl_globals.protected.spinlock)) return; if(julyl_globals.protected.available_items && julyl_globals.protected.available > 10) { @@ -68,7 +68,7 @@ void julyl_cleanup1(void) { julyl_globals.protected.available--; } - netdata_spinlock_unlock(&julyl_globals.protected.spinlock); + spinlock_unlock(&julyl_globals.protected.spinlock); if(item) { size_t bytes = item->bytes; @@ -81,7 +81,7 @@ void julyl_cleanup1(void) { struct JulyL *julyl_get(void) { struct JulyL *j; - netdata_spinlock_lock(&julyl_globals.protected.spinlock); + spinlock_lock(&julyl_globals.protected.spinlock); j = julyl_globals.protected.available_items; if(likely(j)) { @@ -89,7 +89,7 @@ struct JulyL *julyl_get(void) { julyl_globals.protected.available--; } - netdata_spinlock_unlock(&julyl_globals.protected.spinlock); + spinlock_unlock(&julyl_globals.protected.spinlock); if(unlikely(!j)) { size_t bytes = sizeof(struct JulyL) + JULYL_MIN_ENTRIES * sizeof(struct JulyL_item); @@ -113,10 +113,10 @@ static void julyl_release(struct JulyL *j) { __atomic_add_fetch(&julyl_globals.atomics.bytes_moved, j->bytes_moved, __ATOMIC_RELAXED); __atomic_add_fetch(&julyl_globals.atomics.reallocs, j->reallocs, __ATOMIC_RELAXED); - netdata_spinlock_lock(&julyl_globals.protected.spinlock); + spinlock_lock(&julyl_globals.protected.spinlock); DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(julyl_globals.protected.available_items, j, cache.prev, cache.next); julyl_globals.protected.available++; - netdata_spinlock_unlock(&julyl_globals.protected.spinlock); + spinlock_unlock(&julyl_globals.protected.spinlock); } size_t julyl_cache_size(void) { diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c index 19b861e39..272ba8f17 100644 --- a/libnetdata/libnetdata.c +++ b/libnetdata/libnetdata.c @@ -249,7 +249,7 @@ static avl_tree_lock malloc_trace_index = { .avl_tree = { .root = NULL, .compar = malloc_trace_compare}, - .rwlock = NETDATA_RWLOCK_INITIALIZER + .rwlock = AVL_LOCK_INITIALIZER }; int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data) { @@ -370,7 +370,7 @@ static struct malloc_header *malloc_get_header(void *ptr, const char *caller, co struct malloc_header *t = (struct malloc_header *)ret; if(t->signature.magic != 0x0BADCAFE) { - error("pointer %p is not our pointer (called %s() from %zu@%s, %s()).", ptr, caller, line, file, function); + netdata_log_error("pointer %p is not our pointer (called %s() from %zu@%s, %s()).", ptr, caller, line, file, function); return NULL; } @@ -1043,20 +1043,23 @@ void netdata_fix_chart_id(char *s) { } static int memory_file_open(const char *filename, size_t size) { - // info("memory_file_open('%s', %zu", filename, size); + // netdata_log_info("memory_file_open('%s', %zu", filename, size); int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME, 0664); if (fd != -1) { if (lseek(fd, size, SEEK_SET) == (off_t) size) { if (write(fd, "", 1) == 1) { if (ftruncate(fd, size)) - error("Cannot truncate file '%s' to size %zu. Will use the larger file.", filename, size); + netdata_log_error("Cannot truncate file '%s' to size %zu. Will use the larger file.", filename, size); } - else error("Cannot write to file '%s' at position %zu.", filename, size); + else + netdata_log_error("Cannot write to file '%s' at position %zu.", filename, size); } - else error("Cannot seek file '%s' to size %zu.", filename, size); + else + netdata_log_error("Cannot seek file '%s' to size %zu.", filename, size); } - else error("Cannot create/open file '%s'.", filename); + else + netdata_log_error("Cannot create/open file '%s'.", filename); return fd; } @@ -1065,7 +1068,8 @@ inline int madvise_sequential(void *mem, size_t len) { static int logger = 1; int ret = madvise(mem, len, MADV_SEQUENTIAL); - if (ret != 0 && logger-- > 0) error("madvise(MADV_SEQUENTIAL) failed."); + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_SEQUENTIAL) failed."); return ret; } @@ -1073,7 +1077,8 @@ inline int madvise_random(void *mem, size_t len) { static int logger = 1; int ret = madvise(mem, len, MADV_RANDOM); - if (ret != 0 && logger-- > 0) error("madvise(MADV_RANDOM) failed."); + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_RANDOM) failed."); return ret; } @@ -1081,7 +1086,8 @@ inline int madvise_dontfork(void *mem, size_t len) { static int logger = 1; int ret = madvise(mem, len, MADV_DONTFORK); - if (ret != 0 && logger-- > 0) error("madvise(MADV_DONTFORK) failed."); + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_DONTFORK) failed."); return ret; } @@ -1089,7 +1095,8 @@ inline int madvise_willneed(void *mem, size_t len) { static int logger = 1; int ret = madvise(mem, len, MADV_WILLNEED); - if (ret != 0 && logger-- > 0) error("madvise(MADV_WILLNEED) failed."); + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_WILLNEED) failed."); return ret; } @@ -1097,7 +1104,8 @@ inline int madvise_dontneed(void *mem, size_t len) { static int logger = 1; int ret = madvise(mem, len, MADV_DONTNEED); - if (ret != 0 && logger-- > 0) error("madvise(MADV_DONTNEED) failed."); + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_DONTNEED) failed."); return ret; } @@ -1106,7 +1114,8 @@ inline int madvise_dontdump(void *mem __maybe_unused, size_t len __maybe_unused) static int logger = 1; int ret = madvise(mem, len, MADV_DONTDUMP); - if (ret != 0 && logger-- > 0) error("madvise(MADV_DONTDUMP) failed."); + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_DONTDUMP) failed."); return ret; #else return 0; @@ -1118,7 +1127,8 @@ inline int madvise_mergeable(void *mem __maybe_unused, size_t len __maybe_unused static int logger = 1; int ret = madvise(mem, len, MADV_MERGEABLE); - if (ret != 0 && logger-- > 0) error("madvise(MADV_MERGEABLE) failed."); + if (ret != 0 && logger-- > 0) + netdata_log_error("madvise(MADV_MERGEABLE) failed."); return ret; #else return 0; @@ -1127,7 +1137,7 @@ inline int madvise_mergeable(void *mem __maybe_unused, size_t len __maybe_unused void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool read_only, int *open_fd) { - // info("netdata_mmap('%s', %zu", filename, size); + // netdata_log_info("netdata_mmap('%s', %zu", filename, size); // MAP_SHARED is used in memory mode map // MAP_PRIVATE is used in memory mode ram and save @@ -1177,9 +1187,9 @@ void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool r if(fd != -1 && fd_for_mmap == -1) { if (lseek(fd, 0, SEEK_SET) == 0) { if (read(fd, mem, size) != (ssize_t) size) - info("Cannot read from file '%s'", filename); + netdata_log_info("Cannot read from file '%s'", filename); } - else info("Cannot seek to beginning of file '%s'.", filename); + else netdata_log_info("Cannot seek to beginning of file '%s'.", filename); } // madvise_sequential(mem, size); @@ -1215,12 +1225,12 @@ int memory_file_save(const char *filename, void *mem, size_t size) { int fd = open(tmpfilename, O_RDWR | O_CREAT | O_NOATIME, 0664); if (fd < 0) { - error("Cannot create/open file '%s'.", filename); + netdata_log_error("Cannot create/open file '%s'.", filename); return -1; } if (write(fd, mem, size) != (ssize_t) size) { - error("Cannot write to file '%s' %ld bytes.", filename, (long) size); + netdata_log_error("Cannot write to file '%s' %ld bytes.", filename, (long) size); close(fd); return -1; } @@ -1228,7 +1238,7 @@ int memory_file_save(const char *filename, void *mem, size_t size) { close(fd); if (rename(tmpfilename, filename)) { - error("Cannot rename '%s' to '%s'", tmpfilename, filename); + netdata_log_error("Cannot rename '%s' to '%s'", tmpfilename, filename); return -1; } @@ -1298,7 +1308,7 @@ unsigned long end_tsc(void) { int recursively_delete_dir(const char *path, const char *reason) { DIR *dir = opendir(path); if(!dir) { - error("Cannot read %s directory to be deleted '%s'", reason?reason:"", path); + netdata_log_error("Cannot read %s directory to be deleted '%s'", reason?reason:"", path); return -1; } @@ -1321,16 +1331,16 @@ int recursively_delete_dir(const char *path, const char *reason) { continue; } - info("Deleting %s file '%s'", reason?reason:"", fullpath); + netdata_log_info("Deleting %s file '%s'", reason?reason:"", fullpath); if(unlikely(unlink(fullpath) == -1)) - error("Cannot delete %s file '%s'", reason?reason:"", fullpath); + netdata_log_error("Cannot delete %s file '%s'", reason?reason:"", fullpath); else ret++; } - info("Deleting empty directory '%s'", path); + netdata_log_info("Deleting empty directory '%s'", path); if(unlikely(rmdir(path) == -1)) - error("Cannot delete empty directory '%s'", path); + netdata_log_error("Cannot delete empty directory '%s'", path); else ret++; @@ -1399,12 +1409,12 @@ int verify_netdata_host_prefix() { goto failed; if(netdata_configured_host_prefix && *netdata_configured_host_prefix) - info("Using host prefix directory '%s'", netdata_configured_host_prefix); + netdata_log_info("Using host prefix directory '%s'", netdata_configured_host_prefix); return 0; failed: - error("Ignoring host prefix '%s': path '%s' %s", netdata_configured_host_prefix, path, reason); + netdata_log_error("Ignoring host prefix '%s': path '%s' %s", netdata_configured_host_prefix, path, reason); netdata_configured_host_prefix = ""; return -1; } @@ -1512,18 +1522,18 @@ int path_is_file(const char *path, const char *subpath) { void recursive_config_double_dir_load(const char *user_path, const char *stock_path, const char *subpath, int (*callback)(const char *filename, void *data), void *data, size_t depth) { if(depth > 3) { - error("CONFIG: Max directory depth reached while reading user path '%s', stock path '%s', subpath '%s'", user_path, stock_path, subpath); + netdata_log_error("CONFIG: Max directory depth reached while reading user path '%s', stock path '%s', subpath '%s'", user_path, stock_path, subpath); return; } char *udir = strdupz_path_subpath(user_path, subpath); char *sdir = strdupz_path_subpath(stock_path, subpath); - debug(D_HEALTH, "CONFIG traversing user-config directory '%s', stock config directory '%s'", udir, sdir); + netdata_log_debug(D_HEALTH, "CONFIG traversing user-config directory '%s', stock config directory '%s'", udir, sdir); DIR *dir = opendir(udir); if (!dir) { - error("CONFIG cannot open user-config directory '%s'.", udir); + netdata_log_error("CONFIG cannot open user-config directory '%s'.", udir); } else { struct dirent *de = NULL; @@ -1533,7 +1543,7 @@ void recursive_config_double_dir_load(const char *user_path, const char *stock_p (de->d_name[0] == '.' && de->d_name[1] == '\0') || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') ) { - debug(D_HEALTH, "CONFIG ignoring user-config directory '%s/%s'", udir, de->d_name); + netdata_log_debug(D_HEALTH, "CONFIG ignoring user-config directory '%s/%s'", udir, de->d_name); continue; } @@ -1548,24 +1558,24 @@ void recursive_config_double_dir_load(const char *user_path, const char *stock_p if(path_is_file(udir, de->d_name) && len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { char *filename = strdupz_path_subpath(udir, de->d_name); - debug(D_HEALTH, "CONFIG calling callback for user file '%s'", filename); + netdata_log_debug(D_HEALTH, "CONFIG calling callback for user file '%s'", filename); callback(filename, data); freez(filename); continue; } } - debug(D_HEALTH, "CONFIG ignoring user-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); + netdata_log_debug(D_HEALTH, "CONFIG ignoring user-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); } closedir(dir); } - debug(D_HEALTH, "CONFIG traversing stock config directory '%s', user config directory '%s'", sdir, udir); + netdata_log_debug(D_HEALTH, "CONFIG traversing stock config directory '%s', user config directory '%s'", sdir, udir); dir = opendir(sdir); if (!dir) { - error("CONFIG cannot open stock config directory '%s'.", sdir); + netdata_log_error("CONFIG cannot open stock config directory '%s'.", sdir); } else { if (strcmp(udir, sdir)) { @@ -1576,7 +1586,7 @@ void recursive_config_double_dir_load(const char *user_path, const char *stock_p (de->d_name[0] == '.' && de->d_name[1] == '\0') || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0') ) { - debug(D_HEALTH, "CONFIG ignoring stock config directory '%s/%s'", sdir, de->d_name); + netdata_log_debug(D_HEALTH, "CONFIG ignoring stock config directory '%s/%s'", sdir, de->d_name); continue; } @@ -1596,7 +1606,7 @@ void recursive_config_double_dir_load(const char *user_path, const char *stock_p if(path_is_file(sdir, de->d_name) && !path_is_file(udir, de->d_name) && len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) { char *filename = strdupz_path_subpath(sdir, de->d_name); - debug(D_HEALTH, "CONFIG calling callback for stock file '%s'", filename); + netdata_log_debug(D_HEALTH, "CONFIG calling callback for stock file '%s'", filename); callback(filename, data); freez(filename); continue; @@ -1604,13 +1614,13 @@ void recursive_config_double_dir_load(const char *user_path, const char *stock_p } - debug(D_HEALTH, "CONFIG ignoring stock-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); + netdata_log_debug(D_HEALTH, "CONFIG ignoring stock-config file '%s/%s' of type %d", udir, de->d_name, (int)de->d_type); } } closedir(dir); } - debug(D_HEALTH, "CONFIG done traversing user-config directory '%s', stock config directory '%s'", udir, sdir); + netdata_log_debug(D_HEALTH, "CONFIG done traversing user-config directory '%s', stock config directory '%s'", udir, sdir); freez(udir); freez(sdir); @@ -1684,7 +1694,7 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c return value; } -inline int pluginsd_space(char c) { +inline int pluginsd_isspace(char c) { switch(c) { case ' ': case '\t': @@ -1698,8 +1708,7 @@ inline int pluginsd_space(char c) { } } -inline int config_isspace(char c) -{ +inline int config_isspace(char c) { switch (c) { case ' ': case '\t': @@ -1713,100 +1722,23 @@ inline int config_isspace(char c) } } -// split a text into words, respecting quotes -inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words, int (*custom_isspace)(char)) -{ - char *s = str, quote = 0; - size_t i = 0; - - // skip all white space - while (unlikely(custom_isspace(*s))) - s++; - - if(unlikely(!*s)) { - words[i] = NULL; - return 0; - } - - // check for quote - if (unlikely(*s == '\'' || *s == '"')) { - quote = *s; // remember the quote - s++; // skip the quote - } - - // store the first word - words[i++] = s; - - // while we have something - while (likely(*s)) { - // if it is an escape - if (unlikely(*s == '\\' && s[1])) { - s += 2; - continue; - } - - // if it is a quote - else if (unlikely(*s == quote)) { - quote = 0; - *s = ' '; - continue; - } - - // if it is a space - else if (unlikely(quote == 0 && custom_isspace(*s))) { - // terminate the word - *s++ = '\0'; - - // skip all white space - while (likely(custom_isspace(*s))) - s++; - - // check for a quote - if (unlikely(*s == '\'' || *s == '"')) { - quote = *s; // remember the quote - s++; // skip the quote - } - - // if we reached the end, stop - if (unlikely(!*s)) - break; - - // store the next word - if (likely(i < max_words)) - words[i++] = s; - else - break; - } - - // anything else - else - s++; - } +inline int group_by_label_isspace(char c) { + if(c == ',' || c == '|') + return 1; - if (i < max_words) - words[i] = NULL; - - return i; -} - -inline size_t pluginsd_split_words(char *str, char **words, size_t max_words) -{ - return quoted_strings_splitter(str, words, max_words, pluginsd_space); + return 0; } -bool bitmap256_get_bit(BITMAP256 *ptr, uint8_t idx) { - if (unlikely(!ptr)) - return false; - return (ptr->data[idx / 64] & (1ULL << (idx % 64))); -} +bool isspace_map_pluginsd[256] = {}; +bool isspace_map_config[256] = {}; +bool isspace_map_group_by_label[256] = {}; -void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value) { - if (unlikely(!ptr)) - return; - if (likely(value)) - ptr->data[idx / 64] |= (1ULL << (idx % 64)); - else - ptr->data[idx / 64] &= ~(1ULL << (idx % 64)); +__attribute__((constructor)) void initialize_is_space_arrays(void) { + for(int c = 0; c < 256 ; c++) { + isspace_map_pluginsd[c] = pluginsd_isspace((char) c); + isspace_map_config[c] = config_isspace((char) c); + isspace_map_group_by_label[c] = group_by_label_isspace((char) c); + } } bool run_command_and_copy_output_to_stdout(const char *command, int max_line_length) { @@ -1819,7 +1751,7 @@ bool run_command_and_copy_output_to_stdout(const char *command, int max_line_len fprintf(stdout, "%s", buffer); } else { - error("Failed to execute command '%s'.", command); + netdata_log_error("Failed to execute command '%s'.", command); return false; } @@ -1837,7 +1769,7 @@ void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds){ if(!(excluded_fds & OPEN_FD_EXCLUDE_STDERR)) (void)close(STDERR_FILENO); #if defined(HAVE_CLOSE_RANGE) if(close_range(STDERR_FILENO + 1, ~0U, 0) == 0) return; - error("close_range() failed, will try to close fds one by one"); + netdata_log_error("close_range() failed, will try to close fds one by one"); #endif break; case OPEN_FD_ACTION_FD_CLOEXEC: @@ -1846,7 +1778,7 @@ void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds){ if(!(excluded_fds & OPEN_FD_EXCLUDE_STDERR)) (void)fcntl(STDERR_FILENO, F_SETFD, FD_CLOEXEC); #if defined(HAVE_CLOSE_RANGE) && defined(CLOSE_RANGE_CLOEXEC) // Linux >= 5.11, FreeBSD >= 13.1 if(close_range(STDERR_FILENO + 1, ~0U, CLOSE_RANGE_CLOEXEC) == 0) return; - error("close_range() failed, will try to mark fds for closing one by one"); + netdata_log_error("close_range() failed, will try to mark fds for closing one by one"); #endif break; default: @@ -1999,7 +1931,7 @@ void timing_action(TIMING_ACTION action, TIMING_STEP step) { ); } - info("TIMINGS REPORT:\n%sTIMINGS REPORT: total # %10zu, t %11.2f ms", + netdata_log_info("TIMINGS REPORT:\n%sTIMINGS REPORT: total # %10zu, t %11.2f ms", buffer_tostring(wb), total_reqs, (double)total_usec / USEC_PER_MS); memcpy(timings2, timings3, sizeof(timings2)); diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h index 062d8c6fa..8b8c7206e 100644 --- a/libnetdata/libnetdata.h +++ b/libnetdata/libnetdata.h @@ -11,6 +11,21 @@ extern "C" { #include <config.h> #endif +#ifdef ENABLE_LZ4 +#define ENABLE_RRDPUSH_COMPRESSION 1 +#endif + +#ifdef ENABLE_OPENSSL +#define ENABLE_HTTPS 1 +#endif + +#ifdef HAVE_LIBDATACHANNEL +#define ENABLE_WEBRTC 1 +#endif + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) + #define JUDYHS_INDEX_SIZE_ESTIMATE(key_bytes) (((key_bytes) + sizeof(Word_t) - 1) / sizeof(Word_t) * 4) #if defined(NETDATA_DEV_MODE) && !defined(NETDATA_INTERNAL_CHECKS) @@ -256,16 +271,17 @@ size_t judy_aral_structures(void); #define DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(head, item, prev, next) \ do { \ + \ + (item)->next = NULL; \ + \ if(likely(head)) { \ (item)->prev = (head)->prev; \ (head)->prev->next = (item); \ (head)->prev = (item); \ - (item)->next = NULL; \ } \ else { \ + (item)->prev = (item); \ (head) = (item); \ - (head)->prev = (head); \ - (head)->next = NULL; \ } \ \ } while (0) @@ -585,28 +601,182 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c #define UNUSED_FUNCTION(x) UNUSED_##x #endif -#define error_report(x, args...) do { errno = 0; error(x, ##args); } while(0) +#define error_report(x, args...) do { errno = 0; netdata_log_error(x, ##args); } while(0) // Taken from linux kernel #define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)])) +#ifdef ENV32BIT + +typedef struct bitmapX { + uint32_t bits; + uint32_t data[]; +} BITMAPX; + +typedef struct bitmap256 { + uint32_t bits; + uint32_t data[256 / 32]; +} BITMAP256; + +typedef struct bitmap1024 { + uint32_t bits; + uint32_t data[1024 / 32]; +} BITMAP1024; + +static inline BITMAPX *bitmapX_create(uint32_t bits) { + BITMAPX *bmp = (BITMAPX *)callocz(1, sizeof(BITMAPX) + sizeof(uint32_t) * ((bits + 31) / 32)); + uint32_t *p = (uint32_t *)&bmp->bits; + *p = bits; + return bmp; +} + +#define bitmapX_get_bit(ptr, idx) ((ptr)->data[(idx) >> 5] & (1U << ((idx) & 31))) +#define bitmapX_set_bit(ptr, idx, value) do { \ + register uint32_t _bitmask = 1U << ((idx) & 31); \ + if (value) \ + (ptr)->data[(idx) >> 5] |= _bitmask; \ + else \ + (ptr)->data[(idx) >> 5] &= ~_bitmask; \ +} while(0) + +#else // 64bit version of bitmaps + +typedef struct bitmapX { + uint32_t bits; + uint64_t data[]; +} BITMAPX; + typedef struct bitmap256 { - uint64_t data[4]; + uint32_t bits; + uint64_t data[256 / 64]; } BITMAP256; -bool bitmap256_get_bit(BITMAP256 *ptr, uint8_t idx); -void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value); +typedef struct bitmap1024 { + uint32_t bits; + uint64_t data[1024 / 64]; +} BITMAP1024; + +static inline BITMAPX *bitmapX_create(uint32_t bits) { + BITMAPX *bmp = (BITMAPX *)callocz(1, sizeof(BITMAPX) + sizeof(uint64_t) * ((bits + 63) / 64)); + bmp->bits = bits; + return bmp; +} + +#define bitmapX_get_bit(ptr, idx) ((ptr)->data[(idx) >> 6] & (1ULL << ((idx) & 63))) +#define bitmapX_set_bit(ptr, idx, value) do { \ + register uint64_t _bitmask = 1ULL << ((idx) & 63); \ + if (value) \ + (ptr)->data[(idx) >> 6] |= _bitmask; \ + else \ + (ptr)->data[(idx) >> 6] &= ~_bitmask; \ +} while(0) + +#endif // 64bit version of bitmaps + +#define BITMAPX_INITIALIZER(wanted_bits) { .bits = (wanted_bits), .data = {0} } +#define BITMAP256_INITIALIZER (BITMAP256)BITMAPX_INITIALIZER(256) +#define BITMAP1024_INITIALIZER (BITMAP1024)BITMAPX_INITIALIZER(1024) +#define bitmap256_get_bit(ptr, idx) bitmapX_get_bit((BITMAPX *)ptr, idx) +#define bitmap256_set_bit(ptr, idx, value) bitmapX_set_bit((BITMAPX *)ptr, idx, value) +#define bitmap1024_get_bit(ptr, idx) bitmapX_get_bit((BITMAPX *)ptr, idx) +#define bitmap1024_set_bit(ptr, idx, value) bitmapX_set_bit((BITMAPX *)ptr, idx, value) + #define COMPRESSION_MAX_MSG_SIZE 0x4000 #define PLUGINSD_LINE_MAX (COMPRESSION_MAX_MSG_SIZE - 1024) +int pluginsd_isspace(char c); int config_isspace(char c); -int pluginsd_space(char c); +int group_by_label_isspace(char c); + +extern bool isspace_map_pluginsd[256]; +extern bool isspace_map_config[256]; +extern bool isspace_map_group_by_label[256]; + +static inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words, bool *isspace_map) { + char *s = str, quote = 0; + size_t i = 0; + + // skip all white space + while (unlikely(isspace_map[(uint8_t)*s])) + s++; + + if(unlikely(!*s)) { + words[i] = NULL; + return 0; + } + + // check for quote + if (unlikely(*s == '\'' || *s == '"')) { + quote = *s; // remember the quote + s++; // skip the quote + } -size_t quoted_strings_splitter(char *str, char **words, size_t max_words, int (*custom_isspace)(char)); -size_t pluginsd_split_words(char *str, char **words, size_t max_words); + // store the first word + words[i++] = s; + + // while we have something + while (likely(*s)) { + // if it is an escape + if (unlikely(*s == '\\' && s[1])) { + s += 2; + continue; + } + + // if it is a quote + else if (unlikely(*s == quote)) { + quote = 0; + *s = ' '; + continue; + } + + // if it is a space + else if (unlikely(quote == 0 && isspace_map[(uint8_t)*s])) { + // terminate the word + *s++ = '\0'; + + // skip all white space + while (likely(isspace_map[(uint8_t)*s])) + s++; + + // check for a quote + if (unlikely(*s == '\'' || *s == '"')) { + quote = *s; // remember the quote + s++; // skip the quote + } + + // if we reached the end, stop + if (unlikely(!*s)) + break; + + // store the next word + if (likely(i < max_words)) + words[i++] = s; + else + break; + } + + // anything else + else + s++; + } + + if (likely(i < max_words)) + words[i] = NULL; + + return i; +} + +#define quoted_strings_splitter_query_group_by_label(str, words, max_words) \ + quoted_strings_splitter(str, words, max_words, isspace_map_group_by_label) + +#define quoted_strings_splitter_config(str, words, max_words) \ + quoted_strings_splitter(str, words, max_words, isspace_map_config) + +#define quoted_strings_splitter_pluginsd(str, words, max_words) \ + quoted_strings_splitter(str, words, max_words, isspace_map_pluginsd) static inline char *get_word(char **words, size_t num_words, size_t index) { - if (index >= num_words) + if (unlikely(index >= num_words)) return NULL; return words[index]; @@ -663,7 +833,6 @@ extern char *netdata_configured_host_prefix; #include "libnetdata/aral/aral.h" #include "onewayalloc/onewayalloc.h" #include "worker_utilization/worker_utilization.h" -#include "parser/parser.h" #include "yaml.h" #include "http/http_defs.h" #include "gorilla/gorilla.h" @@ -761,6 +930,32 @@ typedef enum { TIMING_STEP_END2_PROPAGATE, TIMING_STEP_END2_STORE, + TIMING_STEP_FREEIPMI_CTX_CREATE, + TIMING_STEP_FREEIPMI_DSR_CACHE_DIR, + TIMING_STEP_FREEIPMI_SENSOR_CONFIG_FILE, + TIMING_STEP_FREEIPMI_SENSOR_READINGS_BY_X, + TIMING_STEP_FREEIPMI_READ_record_id, + TIMING_STEP_FREEIPMI_READ_sensor_number, + TIMING_STEP_FREEIPMI_READ_sensor_type, + TIMING_STEP_FREEIPMI_READ_sensor_name, + TIMING_STEP_FREEIPMI_READ_sensor_state, + TIMING_STEP_FREEIPMI_READ_sensor_units, + TIMING_STEP_FREEIPMI_READ_sensor_bitmask_type, + TIMING_STEP_FREEIPMI_READ_sensor_bitmask, + TIMING_STEP_FREEIPMI_READ_sensor_bitmask_strings, + TIMING_STEP_FREEIPMI_READ_sensor_reading_type, + TIMING_STEP_FREEIPMI_READ_sensor_reading, + TIMING_STEP_FREEIPMI_READ_event_reading_type_code, + TIMING_STEP_FREEIPMI_READ_record_type, + TIMING_STEP_FREEIPMI_READ_record_type_class, + TIMING_STEP_FREEIPMI_READ_sel_state, + TIMING_STEP_FREEIPMI_READ_event_direction, + TIMING_STEP_FREEIPMI_READ_event_type_code, + TIMING_STEP_FREEIPMI_READ_event_offset_type, + TIMING_STEP_FREEIPMI_READ_event_offset, + TIMING_STEP_FREEIPMI_READ_event_offset_string, + TIMING_STEP_FREEIPMI_READ_manufacturer_id, + // terminator TIMING_STEP_MAX, } TIMING_STEP; diff --git a/libnetdata/locks/locks.c b/libnetdata/locks/locks.c index e73456d70..625dd052c 100644 --- a/libnetdata/locks/locks.c +++ b/libnetdata/locks/locks.c @@ -28,37 +28,52 @@ static __thread size_t netdata_locks_acquired_rwlocks = 0; static __thread size_t netdata_locks_acquired_mutexes = 0; inline void netdata_thread_disable_cancelability(void) { - int old; - int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old); - if(ret != 0) - error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret); - else { - if(!netdata_thread_nested_disables) - netdata_thread_first_cancelability = old; + if(!netdata_thread_nested_disables) { + int old; + int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old); + + if(ret != 0) + netdata_log_error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", + netdata_thread_tag(), ret); - netdata_thread_nested_disables++; + netdata_thread_first_cancelability = old; } + + netdata_thread_nested_disables++; } inline void netdata_thread_enable_cancelability(void) { - if(netdata_thread_nested_disables < 1) { - error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): invalid thread cancelability count %d on thread %s - results will be undefined - please report this!", - netdata_thread_nested_disables, netdata_thread_tag()); + if(unlikely(netdata_thread_nested_disables < 1)) { + internal_fatal(true, "THREAD_CANCELABILITY: trying to enable cancelability, but it was not not disabled"); + + netdata_log_error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): invalid thread cancelability count %d " + "on thread %s - results will be undefined - please report this!", + netdata_thread_nested_disables, netdata_thread_tag()); + + netdata_thread_nested_disables = 1; } - else if(netdata_thread_nested_disables == 1) { + + if(netdata_thread_nested_disables == 1) { int old = 1; int ret = pthread_setcancelstate(netdata_thread_first_cancelability, &old); if(ret != 0) - error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret); + netdata_log_error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", + netdata_thread_tag(), + ret); else { - if(old != PTHREAD_CANCEL_DISABLE) - error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): old thread cancelability on thread %s was changed, expected DISABLED (%d), found %s (%d) - please report this!", netdata_thread_tag(), PTHREAD_CANCEL_DISABLE, (old == PTHREAD_CANCEL_ENABLE)?"ENABLED":"UNKNOWN", old); + if(old != PTHREAD_CANCEL_DISABLE) { + internal_fatal(true, "THREAD_CANCELABILITY: invalid old state cancelability"); + + netdata_log_error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): old thread cancelability " + "on thread %s was changed, expected DISABLED (%d), found %s (%d) - please report this!", + netdata_thread_tag(), PTHREAD_CANCEL_DISABLE, + (old == PTHREAD_CANCEL_ENABLE) ? "ENABLED" : "UNKNOWN", + old); + } } - - netdata_thread_nested_disables = 0; } - else - netdata_thread_nested_disables--; + + netdata_thread_nested_disables--; } // ---------------------------------------------------------------------------- @@ -67,14 +82,14 @@ inline void netdata_thread_enable_cancelability(void) { int __netdata_mutex_init(netdata_mutex_t *mutex) { int ret = pthread_mutex_init(mutex, NULL); if(unlikely(ret != 0)) - error("MUTEX_LOCK: failed to initialize (code %d).", ret); + netdata_log_error("MUTEX_LOCK: failed to initialize (code %d).", ret); return ret; } int __netdata_mutex_destroy(netdata_mutex_t *mutex) { int ret = pthread_mutex_destroy(mutex); if(unlikely(ret != 0)) - error("MUTEX_LOCK: failed to destroy (code %d).", ret); + netdata_log_error("MUTEX_LOCK: failed to destroy (code %d).", ret); return ret; } @@ -84,7 +99,7 @@ int __netdata_mutex_lock(netdata_mutex_t *mutex) { int ret = pthread_mutex_lock(mutex); if(unlikely(ret != 0)) { netdata_thread_enable_cancelability(); - error("MUTEX_LOCK: failed to get lock (code %d)", ret); + netdata_log_error("MUTEX_LOCK: failed to get lock (code %d)", ret); } else netdata_locks_acquired_mutexes++; @@ -107,7 +122,7 @@ int __netdata_mutex_trylock(netdata_mutex_t *mutex) { int __netdata_mutex_unlock(netdata_mutex_t *mutex) { int ret = pthread_mutex_unlock(mutex); if(unlikely(ret != 0)) - error("MUTEX_LOCK: failed to unlock (code %d).", ret); + netdata_log_error("MUTEX_LOCK: failed to unlock (code %d).", ret); else { netdata_locks_acquired_mutexes--; netdata_thread_enable_cancelability(); @@ -120,29 +135,29 @@ int __netdata_mutex_unlock(netdata_mutex_t *mutex) { int netdata_mutex_init_debug(const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(%p) from %lu@%s, %s()", mutex, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(%p) from %lu@%s, %s()", mutex, line, file, function); int ret = __netdata_mutex_init(mutex); - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(%p) = %d, from %lu@%s, %s()", mutex, ret, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(%p) = %d, from %lu@%s, %s()", mutex, ret, line, file, function); return ret; } int netdata_mutex_destroy_debug(const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_destroy(%p) from %lu@%s, %s()", mutex, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_destroy(%p) from %lu@%s, %s()", mutex, line, file, function); int ret = __netdata_mutex_destroy(mutex); - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_destroy(%p) = %d, from %lu@%s, %s()", mutex, ret, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_destroy(%p) = %d, from %lu@%s, %s()", mutex, ret, line, file, function); return ret; } int netdata_mutex_lock_debug(const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(%p) from %lu@%s, %s()", mutex, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(%p) from %lu@%s, %s()", mutex, line, file, function); usec_t start_s = now_monotonic_high_precision_usec(); int ret = __netdata_mutex_lock(mutex); @@ -152,14 +167,14 @@ int netdata_mutex_lock_debug(const char *file __maybe_unused, const char *functi (void)start_s; (void)end_s; - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); return ret; } int netdata_mutex_trylock_debug(const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(%p) from %lu@%s, %s()", mutex, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(%p) from %lu@%s, %s()", mutex, line, file, function); usec_t start_s = now_monotonic_high_precision_usec(); int ret = __netdata_mutex_trylock(mutex); @@ -169,14 +184,14 @@ int netdata_mutex_trylock_debug(const char *file __maybe_unused, const char *fun (void)start_s; (void)end_s; - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); return ret; } int netdata_mutex_unlock_debug(const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(%p) from %lu@%s, %s()", mutex, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(%p) from %lu@%s, %s()", mutex, line, file, function); usec_t start_s = now_monotonic_high_precision_usec(); int ret = __netdata_mutex_unlock(mutex); @@ -186,7 +201,7 @@ int netdata_mutex_unlock_debug(const char *file __maybe_unused, const char *func (void)start_s; (void)end_s; - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); return ret; } @@ -199,14 +214,14 @@ int netdata_mutex_unlock_debug(const char *file __maybe_unused, const char *func int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_destroy(&rwlock->rwlock_t); if(unlikely(ret != 0)) - error("RW_LOCK: failed to destroy lock (code %d)", ret); + netdata_log_error("RW_LOCK: failed to destroy lock (code %d)", ret); return ret; } int __netdata_rwlock_init(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_init(&rwlock->rwlock_t, NULL); if(unlikely(ret != 0)) - error("RW_LOCK: failed to initialize lock (code %d)", ret); + netdata_log_error("RW_LOCK: failed to initialize lock (code %d)", ret); return ret; } @@ -216,7 +231,7 @@ int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_rdlock(&rwlock->rwlock_t); if(unlikely(ret != 0)) { netdata_thread_enable_cancelability(); - error("RW_LOCK: failed to obtain read lock (code %d)", ret); + netdata_log_error("RW_LOCK: failed to obtain read lock (code %d)", ret); } else netdata_locks_acquired_rwlocks++; @@ -229,7 +244,7 @@ int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_wrlock(&rwlock->rwlock_t); if(unlikely(ret != 0)) { - error("RW_LOCK: failed to obtain write lock (code %d)", ret); + netdata_log_error("RW_LOCK: failed to obtain write lock (code %d)", ret); netdata_thread_enable_cancelability(); } else @@ -241,7 +256,7 @@ int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) { int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_unlock(&rwlock->rwlock_t); if(unlikely(ret != 0)) - error("RW_LOCK: failed to release lock (code %d)", ret); + netdata_log_error("RW_LOCK: failed to release lock (code %d)", ret); else { netdata_thread_enable_cancelability(); netdata_locks_acquired_rwlocks--; @@ -278,11 +293,11 @@ int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) { // spinlock implementation // https://www.youtube.com/watch?v=rmGJc9PXpuE&t=41s -void netdata_spinlock_init(SPINLOCK *spinlock) { +void spinlock_init(SPINLOCK *spinlock) { memset(spinlock, 0, sizeof(SPINLOCK)); } -void netdata_spinlock_lock(SPINLOCK *spinlock) { +void spinlock_lock(SPINLOCK *spinlock) { static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; #ifdef NETDATA_INTERNAL_CHECKS @@ -314,7 +329,7 @@ void netdata_spinlock_lock(SPINLOCK *spinlock) { #endif } -void netdata_spinlock_unlock(SPINLOCK *spinlock) { +void spinlock_unlock(SPINLOCK *spinlock) { #ifdef NETDATA_INTERNAL_CHECKS spinlock->locker_pid = 0; #endif @@ -322,7 +337,7 @@ void netdata_spinlock_unlock(SPINLOCK *spinlock) { netdata_thread_enable_cancelability(); } -bool netdata_spinlock_trylock(SPINLOCK *spinlock) { +bool spinlock_trylock(SPINLOCK *spinlock) { netdata_thread_disable_cancelability(); if(!__atomic_load_n(&spinlock->locked, __ATOMIC_RELAXED) && @@ -331,9 +346,88 @@ bool netdata_spinlock_trylock(SPINLOCK *spinlock) { return true; // we didn't get the lock + netdata_thread_enable_cancelability(); + return false; +} + +// ---------------------------------------------------------------------------- +// rw_spinlock implementation + +void rw_spinlock_init(RW_SPINLOCK *rw_spinlock) { + rw_spinlock->readers = 0; + spinlock_init(&rw_spinlock->spinlock); +} + +void rw_spinlock_read_lock(RW_SPINLOCK *rw_spinlock) { + netdata_thread_disable_cancelability(); + + spinlock_lock(&rw_spinlock->spinlock); + __atomic_add_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED); + spinlock_unlock(&rw_spinlock->spinlock); +} + +void rw_spinlock_read_unlock(RW_SPINLOCK *rw_spinlock) { +#ifndef NETDATA_INTERNAL_CHECKS + __atomic_sub_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED); +#else + int32_t x = __atomic_sub_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED); + if(x < 0) + fatal("RW_SPINLOCK: readers is negative %d", x); +#endif + + netdata_thread_enable_cancelability(); +} + +void rw_spinlock_write_lock(RW_SPINLOCK *rw_spinlock) { + static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; + + size_t spins = 0; + while(1) { + spins++; + spinlock_lock(&rw_spinlock->spinlock); + + if(__atomic_load_n(&rw_spinlock->readers, __ATOMIC_RELAXED) == 0) + break; + + // Busy wait until all readers have released their locks. + spinlock_unlock(&rw_spinlock->spinlock); + nanosleep(&ns, NULL); + } + + (void)spins; +} + +void rw_spinlock_write_unlock(RW_SPINLOCK *rw_spinlock) { + spinlock_unlock(&rw_spinlock->spinlock); +} + +bool rw_spinlock_tryread_lock(RW_SPINLOCK *rw_spinlock) { + if(spinlock_trylock(&rw_spinlock->spinlock)) { + __atomic_add_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED); + spinlock_unlock(&rw_spinlock->spinlock); + netdata_thread_disable_cancelability(); + return true; + } + return false; } +bool rw_spinlock_trywrite_lock(RW_SPINLOCK *rw_spinlock) { + if(spinlock_trylock(&rw_spinlock->spinlock)) { + if (__atomic_load_n(&rw_spinlock->readers, __ATOMIC_RELAXED) == 0) { + // No readers, we've successfully acquired the write lock + return true; + } + else { + // There are readers, unlock the spinlock and return false + spinlock_unlock(&rw_spinlock->spinlock); + } + } + + return false; +} + + #ifdef NETDATA_TRACE_RWLOCKS // ---------------------------------------------------------------------------- diff --git a/libnetdata/locks/locks.h b/libnetdata/locks/locks.h index 89b110d5e..6b492ae47 100644 --- a/libnetdata/locks/locks.h +++ b/libnetdata/locks/locks.h @@ -20,10 +20,26 @@ typedef struct netdata_spinlock { #define NETDATA_SPINLOCK_INITIALIZER \ { .locked = false } -void netdata_spinlock_init(SPINLOCK *spinlock); -void netdata_spinlock_lock(SPINLOCK *spinlock); -void netdata_spinlock_unlock(SPINLOCK *spinlock); -bool netdata_spinlock_trylock(SPINLOCK *spinlock); +void spinlock_init(SPINLOCK *spinlock); +void spinlock_lock(SPINLOCK *spinlock); +void spinlock_unlock(SPINLOCK *spinlock); +bool spinlock_trylock(SPINLOCK *spinlock); + +typedef struct netdata_rw_spinlock { + int32_t readers; + SPINLOCK spinlock; +} RW_SPINLOCK; + +#define NETDATA_RW_SPINLOCK_INITIALIZER \ + { .readers = 0, .spinlock = NETDATA_SPINLOCK_INITIALIZER } + +void rw_spinlock_init(RW_SPINLOCK *rw_spinlock); +void rw_spinlock_read_lock(RW_SPINLOCK *rw_spinlock); +void rw_spinlock_read_unlock(RW_SPINLOCK *rw_spinlock); +void rw_spinlock_write_lock(RW_SPINLOCK *rw_spinlock); +void rw_spinlock_write_unlock(RW_SPINLOCK *rw_spinlock); +bool rw_spinlock_tryread_lock(RW_SPINLOCK *rw_spinlock); +bool rw_spinlock_trywrite_lock(RW_SPINLOCK *rw_spinlock); #ifdef NETDATA_TRACE_RWLOCKS diff --git a/libnetdata/log/log.c b/libnetdata/log/log.c index 6832d628d..e43a4f464 100644 --- a/libnetdata/log/log.c +++ b/libnetdata/log/log.c @@ -530,7 +530,7 @@ static FILE *open_log_file(int fd, FILE *fp, const char *filename, int *enabled_ else { f = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0664); if(f == -1) { - error("Cannot open file '%s'. Leaving %d to its default.", filename, fd); + netdata_log_error("Cannot open file '%s'. Leaving %d to its default.", filename, fd); if(fd_ptr) *fd_ptr = fd; return fp; } @@ -550,12 +550,12 @@ static FILE *open_log_file(int fd, FILE *fp, const char *filename, int *enabled_ // it automatically closes int t = dup2(f, fd); if (t == -1) { - error("Cannot dup2() new fd %d to old fd %d for '%s'", f, fd, filename); + netdata_log_error("Cannot dup2() new fd %d to old fd %d for '%s'", f, fd, filename); close(f); if(fd_ptr) *fd_ptr = fd; return fp; } - // info("dup2() new fd %d to old fd %d for '%s'", f, fd, filename); + // netdata_log_info("dup2() new fd %d to old fd %d for '%s'", f, fd, filename); close(f); } else fd = f; @@ -563,10 +563,10 @@ static FILE *open_log_file(int fd, FILE *fp, const char *filename, int *enabled_ if(!fp) { fp = fdopen(fd, "a"); if (!fp) - error("Cannot fdopen() fd %d ('%s')", fd, filename); + netdata_log_error("Cannot fdopen() fd %d ('%s')", fd, filename); else { if (setvbuf(fp, NULL, _IOLBF, 0) != 0) - error("Cannot set line buffering on fd %d ('%s')", fd, filename); + netdata_log_error("Cannot set line buffering on fd %d ('%s')", fd, filename); } } @@ -634,7 +634,7 @@ int error_log_limit(int reset) { static time_t start = 0; static unsigned long counter = 0, prevented = 0; - FILE *fp = stderror; + FILE *fp = stderror ? stderror : stderr; // fprintf(fp, "FLOOD: counter=%lu, allowed=%lu, backup=%lu, period=%llu\n", counter, error_log_errors_per_period, error_log_errors_per_period_backup, (unsigned long long)error_log_throttle_period); @@ -781,7 +781,7 @@ void debug_int( const char *file, const char *function, const unsigned long line void info_int( int is_collector, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) { va_list args; - FILE *fp = (is_collector) ? stderr : stderror; + FILE *fp = (is_collector || !stderror) ? stderr : stderror; log_lock(); @@ -841,7 +841,7 @@ static const char *strerror_result_string(const char *a, const char *b) { (void) #endif void error_limit_int(ERROR_LIMIT *erl, const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) { - FILE *fp = stderror; + FILE *fp = stderror ? stderror : stderr; if(erl->sleep_ut) sleep_usec(erl->sleep_ut); @@ -910,7 +910,7 @@ void error_limit_int(ERROR_LIMIT *erl, const char *prefix, const char *file __ma void error_int(int is_collector, const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) { // save a copy of errno - just in case this function generates a new error int __errno = errno; - FILE *fp = (is_collector) ? stderr : stderror; + FILE *fp = (is_collector || !stderror) ? stderr : stderror; va_list args; @@ -975,7 +975,7 @@ static void print_call_stack(void) { #endif void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) { - FILE *fp = stderror; + FILE *fp = stderror ? stderror : stderr; // save a copy of errno - just in case this function generates a new error int __errno = errno; @@ -1046,7 +1046,7 @@ void fatal_int( const char *file, const char *function, const unsigned long line // ---------------------------------------------------------------------------- // access log -void log_access( const char *fmt, ... ) { +void netdata_log_access( const char *fmt, ... ) { va_list args; if(access_log_syslog) { @@ -1078,7 +1078,7 @@ void log_access( const char *fmt, ... ) { // ---------------------------------------------------------------------------- // health log -void log_health( const char *fmt, ... ) { +void netdata_log_health( const char *fmt, ... ) { va_list args; if(health_log_syslog) { diff --git a/libnetdata/log/log.h b/libnetdata/log/log.h index 3d9f0927d..9ced07a9a 100644 --- a/libnetdata/log/log.h +++ b/libnetdata/log/log.h @@ -109,19 +109,19 @@ typedef struct error_with_limit { #define error_limit_static_thread_var(var, log_every_secs, sleep_usecs) static __thread ERROR_LIMIT var = { .last_logged = 0, .count = 0, .log_every = (log_every_secs), .sleep_ut = (sleep_usecs) } #ifdef NETDATA_INTERNAL_CHECKS -#define debug(type, args...) do { if(unlikely(debug_flags & type)) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0) +#define netdata_log_debug(type, args...) do { if(unlikely(debug_flags & type)) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0) #define internal_error(condition, args...) do { if(unlikely(condition)) error_int(0, "IERR", __FILE__, __FUNCTION__, __LINE__, ##args); } while(0) #define internal_fatal(condition, args...) do { if(unlikely(condition)) fatal_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0) #else -#define debug(type, args...) debug_dummy() +#define netdata_log_debug(type, args...) debug_dummy() #define internal_error(args...) debug_dummy() #define internal_fatal(args...) debug_dummy() #endif -#define info(args...) info_int(0, __FILE__, __FUNCTION__, __LINE__, ##args) +#define netdata_log_info(args...) info_int(0, __FILE__, __FUNCTION__, __LINE__, ##args) #define collector_info(args...) info_int(1, __FILE__, __FUNCTION__, __LINE__, ##args) #define infoerr(args...) error_int(0, "INFO", __FILE__, __FUNCTION__, __LINE__, ##args) -#define error(args...) error_int(0, "ERROR", __FILE__, __FUNCTION__, __LINE__, ##args) +#define netdata_log_error(args...) error_int(0, "ERROR", __FILE__, __FUNCTION__, __LINE__, ##args) #define collector_infoerr(args...) error_int(1, "INFO", __FILE__, __FUNCTION__, __LINE__, ##args) #define collector_error(args...) error_int(1, "ERROR", __FILE__, __FUNCTION__, __LINE__, ##args) #define error_limit(erl, args...) error_limit_int(erl, "ERROR", __FILE__, __FUNCTION__, __LINE__, ##args) @@ -134,8 +134,8 @@ void info_int( int is_collector, const char *file, const char *function, const u void error_int( int is_collector, const char *prefix, const char *file, const char *function, const unsigned long line, const char *fmt, ... ) PRINTFLIKE(6, 7); void error_limit_int(ERROR_LIMIT *erl, const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, unsigned long line __maybe_unused, const char *fmt, ... ) PRINTFLIKE(6, 7);; void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) NORETURN PRINTFLIKE(4, 5); -void log_access( const char *fmt, ... ) PRINTFLIKE(1, 2); -void log_health( const char *fmt, ... ) PRINTFLIKE(1, 2); +void netdata_log_access( const char *fmt, ... ) PRINTFLIKE(1, 2); +void netdata_log_health( const char *fmt, ... ) PRINTFLIKE(1, 2); #ifdef ENABLE_ACLK void log_aclk_message_bin( const char *data, const size_t data_len, int tx, const char *mqtt_topic, const char *message_name); diff --git a/libnetdata/onewayalloc/onewayalloc.c b/libnetdata/onewayalloc/onewayalloc.c index 489ce73d7..05c9f2a9d 100644 --- a/libnetdata/onewayalloc/onewayalloc.c +++ b/libnetdata/onewayalloc/onewayalloc.c @@ -176,7 +176,7 @@ void onewayalloc_freez(ONEWAYALLOC *owa __maybe_unused, const void *ptr __maybe_ // not found - it is not ours // let's free it with the system allocator - error("ONEWAYALLOC: request to free address 0x%p that is not allocated by this OWA", ptr); + netdata_log_error("ONEWAYALLOC: request to free address 0x%p that is not allocated by this OWA", ptr); #endif return; @@ -195,7 +195,7 @@ void onewayalloc_destroy(ONEWAYALLOC *owa) { OWA_PAGE *head = (OWA_PAGE *)owa; - //info("OWA: %zu allocations of %zu total bytes, in %zu pages of %zu total bytes", + //netdata_log_info("OWA: %zu allocations of %zu total bytes, in %zu pages of %zu total bytes", // head->stats_mallocs_made, head->stats_mallocs_size, // head->stats_pages, head->stats_pages_size); diff --git a/libnetdata/os.c b/libnetdata/os.c index 133c02248..e6475a453 100644 --- a/libnetdata/os.c +++ b/libnetdata/os.c @@ -36,7 +36,7 @@ long get_system_cpus_with_cache(bool cache, bool for_netdata) { processors[index] = 1; if(error) - error("Assuming system has %d processors.", processors[index]); + netdata_log_error("Assuming system has %d processors.", processors[index]); } return processors[index]; @@ -49,14 +49,14 @@ long get_system_cpus_with_cache(bool cache, bool for_netdata) { procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT); if(!ff) { processors[index] = 1; - error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]); + netdata_log_error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]); return processors[index]; } ff = procfile_readall(ff); if(!ff) { processors[index] = 1; - error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]); + netdata_log_error("Cannot open file '%s'. Assuming system has %ld processors.", filename, processors[index]); return processors[index]; } @@ -75,7 +75,7 @@ long get_system_cpus_with_cache(bool cache, bool for_netdata) { if(processors[index] < 1) processors[index] = 1; - debug(D_SYSTEM, "System has %ld processors.", processors[index]); + netdata_log_debug(D_SYSTEM, "System has %ld processors.", processors[index]); return processors[index]; #endif /* __APPLE__, __FreeBSD__ */ @@ -93,7 +93,7 @@ pid_t get_system_pid_max(void) { if (unlikely(GETSYSCTL_BY_NAME("kern.pid_max", tmp_pid_max))) { pid_max = 99999; - error("Assuming system's maximum pid is %d.", pid_max); + netdata_log_error("Assuming system's maximum pid is %d.", pid_max); } else { pid_max = tmp_pid_max; } @@ -110,12 +110,12 @@ pid_t get_system_pid_max(void) { unsigned long long max = 0; if(read_single_number_file(filename, &max) != 0) { - error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max); + netdata_log_error("Cannot open file '%s'. Assuming system supports %d pids.", filename, pid_max); return pid_max; } if(!max) { - error("Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max); + netdata_log_error("Cannot parse file '%s'. Assuming system supports %d pids.", filename, pid_max); return pid_max; } @@ -130,7 +130,7 @@ void get_system_HZ(void) { long ticks; if ((ticks = sysconf(_SC_CLK_TCK)) == -1) { - error("Cannot get system clock ticks"); + netdata_log_error("Cannot get system clock ticks"); } system_hz = (unsigned int) ticks; @@ -197,11 +197,11 @@ int getsysctl_by_name(const char *name, void *ptr, size_t len) { size_t nlen = len; if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) { - error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); return 1; } if (unlikely(nlen != len)) { - error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); return 1; } return 0; @@ -215,11 +215,11 @@ int getsysctl_simple(const char *name, int *mib, size_t miblen, void *ptr, size_ return 1; if (unlikely(sysctl(mib, miblen, ptr, &nlen, NULL, 0) == -1)) { - error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); return 1; } if (unlikely(nlen != len)) { - error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); return 1; } @@ -234,11 +234,11 @@ int getsysctl(const char *name, int *mib, size_t miblen, void *ptr, size_t *len) return 1; if (unlikely(sysctl(mib, miblen, ptr, len, NULL, 0) == -1)) { - error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); return 1; } if (unlikely(ptr != NULL && nlen != *len)) { - error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)*len, (unsigned long)nlen); + netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)*len, (unsigned long)nlen); return 1; } @@ -249,11 +249,11 @@ int getsysctl_mib(const char *name, int *mib, size_t len) { size_t nlen = len; if (unlikely(sysctlnametomib(name, mib, &nlen) == -1)) { - error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); + netdata_log_error("FREEBSD: sysctl(%s...) failed: %s", name, strerror(errno)); return 1; } if (unlikely(nlen != len)) { - error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + netdata_log_error("FREEBSD: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); return 1; } return 0; @@ -274,11 +274,11 @@ int getsysctl_by_name(const char *name, void *ptr, size_t len) { size_t nlen = len; if (unlikely(sysctlbyname(name, ptr, &nlen, NULL, 0) == -1)) { - error("MACOS: sysctl(%s...) failed: %s", name, strerror(errno)); + netdata_log_error("MACOS: sysctl(%s...) failed: %s", name, strerror(errno)); return 1; } if (unlikely(nlen != len)) { - error("MACOS: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); + netdata_log_error("MACOS: sysctl(%s...) expected %lu, got %lu", name, (unsigned long)len, (unsigned long)nlen); return 1; } return 0; diff --git a/libnetdata/parser/Makefile.am b/libnetdata/parser/Makefile.am deleted file mode 100644 index 02fe3a314..000000000 --- a/libnetdata/parser/Makefile.am +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in - -dist_noinst_DATA = \ - README.md \ - $(NULL) - diff --git a/libnetdata/parser/README.md b/libnetdata/parser/README.md deleted file mode 100644 index 136c23c69..000000000 --- a/libnetdata/parser/README.md +++ /dev/null @@ -1,28 +0,0 @@ -<!-- -title: "Parser" -custom_edit_url: https://github.com/netdata/netdata/blob/master/parser/README.md -sidebar_label: "Parser" -learn_status: "Published" -learn_topic_type: "References" -learn_rel_path: "Developers/Database" ---> - -# Parser - -## Introduction - -Generic parser that is used to register keywords and a corresponding function that will be executed when that -keyword is encountered in the command stream (either from plugins or via streaming) - -To use a parser do the following: - -1. Define a structure that will be used to share user state across calls (user defined `void *user`) -2. Initialize the parser using `parser_init` -3. Register keywords with their associated callback function using `parser_add_keyword` -4. Start a loop for as long there is input (or parser_action returns error) - 1. Fetch the next line using `parser_next` (if needed) - 2. Process the line using `parser_action` -5. Release the parser using `parser_destroy` -6. Release the user structure - -See examples in receiver.c / pluginsd_parser.c diff --git a/libnetdata/parser/parser.c b/libnetdata/parser/parser.c deleted file mode 100644 index 80c9a2639..000000000 --- a/libnetdata/parser/parser.c +++ /dev/null @@ -1,276 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later -#include <poll.h> -#include <stdio.h> - -#include "parser.h" -#include "collectors/plugins.d/pluginsd_parser.h" - -static inline int find_first_keyword(const char *src, char *dst, int dst_size, int (*custom_isspace)(char)) { - const char *s = src, *keyword_start; - - while (unlikely(custom_isspace(*s))) s++; - keyword_start = s; - - while (likely(*s && !custom_isspace(*s)) && dst_size > 1) { - *dst++ = *s++; - dst_size--; - } - *dst = '\0'; - return dst_size == 0 ? 0 : (int) (s - keyword_start); -} - -/* - * Initialize a parser - * user : as defined by the user, will be shared across calls - * input : main input stream (auto detect stream -- file, socket, pipe) - * buffer : This is the buffer to be used (if null a buffer of size will be allocated) - * size : buffer size either passed or will be allocated - * If the buffer is auto allocated, it will auto freed when the parser is destroyed - * - * - */ - -PARSER *parser_init(void *user, FILE *fp_input, FILE *fp_output, int fd, - PARSER_INPUT_TYPE flags, void *ssl __maybe_unused) -{ - PARSER *parser; - - parser = callocz(1, sizeof(*parser)); - parser->user = user; - parser->fd = fd; - parser->fp_input = fp_input; - parser->fp_output = fp_output; -#ifdef ENABLE_HTTPS - parser->ssl_output = ssl; -#endif - parser->flags = flags; - parser->worker_job_next_id = WORKER_PARSER_FIRST_JOB; - - return parser; -} - - -static inline PARSER_KEYWORD *parser_find_keyword(PARSER *parser, const char *command) { - uint32_t hash = parser_hash_function(command); - uint32_t slot = hash % PARSER_KEYWORDS_HASHTABLE_SIZE; - PARSER_KEYWORD *t = parser->keywords.hashtable[slot]; - - if(likely(t && strcmp(t->keyword, command) == 0)) - return t; - - return NULL; -} - -/* - * Add a keyword and the corresponding function that will be called - * Multiple functions may be added - * Input : keyword - * : callback function - * : flags - * Output: > 0 registered function number - * : 0 Error - */ - -void parser_add_keyword(PARSER *parser, char *keyword, keyword_function func) { - if(unlikely(!parser || !keyword || !*keyword || !func)) - fatal("PARSER: invalid parameters"); - - PARSER_KEYWORD *t = callocz(1, sizeof(*t)); - t->worker_job_id = parser->worker_job_next_id++; - t->keyword = strdupz(keyword); - t->func = func; - - uint32_t hash = parser_hash_function(keyword); - uint32_t slot = hash % PARSER_KEYWORDS_HASHTABLE_SIZE; - - if(unlikely(parser->keywords.hashtable[slot])) - fatal("PARSER: hashtable collision between keyword '%s' and '%s' on slot %u. " - "Change the hashtable size and / or the hashing function. " - "Run the unit test to find the optimal values.", - parser->keywords.hashtable[slot]->keyword, - t->keyword, - slot - ); - - parser->keywords.hashtable[slot] = t; - - worker_register_job_name(t->worker_job_id, t->keyword); -} - -/* - * Cleanup a previously allocated parser - */ - -void parser_destroy(PARSER *parser) -{ - if (unlikely(!parser)) - return; - - dictionary_destroy(parser->inflight.functions); - - // Remove keywords - for(size_t i = 0 ; i < PARSER_KEYWORDS_HASHTABLE_SIZE; i++) { - PARSER_KEYWORD *t = parser->keywords.hashtable[i]; - if (t) { - freez(t->keyword); - freez(t); - } - } - - freez(parser); -} - - -/* - * Fetch the next line to process - * - */ - -typedef enum { - PARSER_FGETS_RESULT_OK, - PARSER_FGETS_RESULT_TIMEOUT, - PARSER_FGETS_RESULT_ERROR, - PARSER_FGETS_RESULT_EOF, -} PARSER_FGETS_RESULT; - -static inline PARSER_FGETS_RESULT parser_fgets(char *s, int size, FILE *stream) { - errno = 0; - - struct pollfd fds[1]; - int timeout_msecs = 2 * 60 * MSEC_PER_SEC; - - fds[0].fd = fileno(stream); - fds[0].events = POLLIN; - - int ret = poll(fds, 1, timeout_msecs); - - if (ret > 0) { - /* There is data to read */ - if (fds[0].revents & POLLIN) { - char *tmp = fgets(s, size, stream); - - if(unlikely(!tmp)) { - if (feof(stream)) { - error("PARSER: read failed: end of file."); - return PARSER_FGETS_RESULT_EOF; - } - - else if (ferror(stream)) { - error("PARSER: read failed: input error."); - return PARSER_FGETS_RESULT_ERROR; - } - - error("PARSER: read failed: unknown error."); - return PARSER_FGETS_RESULT_ERROR; - } - - return PARSER_FGETS_RESULT_OK; - } - else if(fds[0].revents & POLLERR) { - error("PARSER: read failed: POLLERR."); - return PARSER_FGETS_RESULT_ERROR; - } - else if(fds[0].revents & POLLHUP) { - error("PARSER: read failed: POLLHUP."); - return PARSER_FGETS_RESULT_ERROR; - } - else if(fds[0].revents & POLLNVAL) { - error("PARSER: read failed: POLLNVAL."); - return PARSER_FGETS_RESULT_ERROR; - } - - error("PARSER: poll() returned positive number, but POLLIN|POLLERR|POLLHUP|POLLNVAL are not set."); - return PARSER_FGETS_RESULT_ERROR; - } - else if (ret == 0) { - error("PARSER: timeout while waiting for data."); - return PARSER_FGETS_RESULT_TIMEOUT; - } - - error("PARSER: poll() failed with code %d.", ret); - return PARSER_FGETS_RESULT_ERROR; -} - -int parser_next(PARSER *parser, char *buffer, size_t buffer_size) { - if(likely(parser_fgets(buffer, (int)buffer_size, (FILE *)parser->fp_input) == PARSER_FGETS_RESULT_OK)) - return 0; - - return 1; -} - -/* -* Takes an initialized parser object that has an unprocessed entry (by calling parser_next) -* and if it contains a valid keyword, it will execute all the callbacks -* -*/ - -inline int parser_action(PARSER *parser, char *input) -{ - parser->line++; - - if(unlikely(parser->flags & PARSER_DEFER_UNTIL_KEYWORD)) { - char command[PLUGINSD_LINE_MAX + 1]; - bool has_keyword = find_first_keyword(input, command, PLUGINSD_LINE_MAX, pluginsd_space); - - if(!has_keyword || strcmp(command, parser->defer.end_keyword) != 0) { - if(parser->defer.response) { - buffer_strcat(parser->defer.response, input); - if(buffer_strlen(parser->defer.response) > 10 * 1024 * 1024) { - // more than 10MB of data - // a bad plugin that did not send the end_keyword - internal_error(true, "PLUGINSD: deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response)); - return 1; - } - } - return 0; - } - else { - // call the action - parser->defer.action(parser, parser->defer.action_data); - - // empty everything - parser->defer.action = NULL; - parser->defer.action_data = NULL; - parser->defer.end_keyword = NULL; - parser->defer.response = NULL; - parser->flags &= ~PARSER_DEFER_UNTIL_KEYWORD; - } - return 0; - } - - char *words[PLUGINSD_MAX_WORDS]; - size_t num_words = pluginsd_split_words(input, words, PLUGINSD_MAX_WORDS); - const char *command = get_word(words, num_words, 0); - - if(unlikely(!command)) - return 0; - - PARSER_RC rc; - PARSER_KEYWORD *t = parser_find_keyword(parser, command); - if(likely(t)) { - worker_is_busy(t->worker_job_id); - rc = (*t->func)(words, num_words, parser->user); - worker_is_idle(); - } - else - rc = PARSER_RC_ERROR; - - if(rc == PARSER_RC_ERROR) { - BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL); - for(size_t i = 0; i < num_words ;i++) { - if(i) buffer_fast_strcat(wb, " ", 1); - - buffer_fast_strcat(wb, "\"", 1); - const char *s = get_word(words, num_words, i); - buffer_strcat(wb, s?s:""); - buffer_fast_strcat(wb, "\"", 1); - } - - error("PLUGINSD: parser_action('%s') failed on line %zu: { %s } (quotes added to show parsing)", - command, parser->line, buffer_tostring(wb)); - - buffer_free(wb); - } - - return (rc == PARSER_RC_ERROR || rc == PARSER_RC_STOP); -} diff --git a/libnetdata/parser/parser.h b/libnetdata/parser/parser.h deleted file mode 100644 index c21cbaf7e..000000000 --- a/libnetdata/parser/parser.h +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_INCREMENTAL_PARSER_H -#define NETDATA_INCREMENTAL_PARSER_H 1 - -#include "../libnetdata.h" - -#define WORKER_PARSER_FIRST_JOB 3 - -// this has to be in-sync with the same at receiver.c -#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION (WORKER_PARSER_FIRST_JOB - 3) - -#define PARSER_KEYWORDS_HASHTABLE_SIZE 73 // unittest finds this magic number -//#define parser_hash_function(s) djb2_hash32(s) -//#define parser_hash_function(s) fnv1_hash32(s) -//#define parser_hash_function(s) fnv1a_hash32(s) -//#define parser_hash_function(s) larson_hash32(s) -#define parser_hash_function(s) pluginsd_parser_hash32(s) - -// PARSER return codes -typedef enum __attribute__ ((__packed__)) parser_rc { - PARSER_RC_OK, // Callback was successful, go on - PARSER_RC_STOP, // Callback says STOP - PARSER_RC_ERROR // Callback failed (abort rest of callbacks) -} PARSER_RC; - -typedef enum __attribute__ ((__packed__)) parser_input_type { - PARSER_INPUT_SPLIT = (1 << 1), - PARSER_DEFER_UNTIL_KEYWORD = (1 << 2), -} PARSER_INPUT_TYPE; - -typedef PARSER_RC (*keyword_function)(char **words, size_t num_words, void *user_data); - -typedef struct parser_keyword { - size_t worker_job_id; - char *keyword; - keyword_function func; -} PARSER_KEYWORD; - -typedef struct parser { - size_t worker_job_next_id; - uint8_t version; // Parser version - int fd; // Socket - FILE *fp_input; // Input source e.g. stream - FILE *fp_output; // Stream to send commands to plugin -#ifdef ENABLE_HTTPS - NETDATA_SSL *ssl_output; -#endif - void *user; // User defined structure to hold extra state between calls - uint32_t flags; - size_t line; - - struct { - PARSER_KEYWORD *hashtable[PARSER_KEYWORDS_HASHTABLE_SIZE]; - } keywords; - - struct { - const char *end_keyword; - BUFFER *response; - void (*action)(struct parser *parser, void *action_data); - void *action_data; - } defer; - - struct { - DICTIONARY *functions; - usec_t smaller_timeout; - } inflight; -} PARSER; - -PARSER *parser_init(void *user, FILE *fp_input, FILE *fp_output, int fd, PARSER_INPUT_TYPE flags, void *ssl); -void parser_add_keyword(PARSER *working_parser, char *keyword, keyword_function func); -int parser_next(PARSER *working_parser, char *buffer, size_t buffer_size); -int parser_action(PARSER *working_parser, char *input); -void parser_destroy(PARSER *working_parser); - -PARSER_RC pluginsd_set(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_begin(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_end(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_chart(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_dimension(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_variable(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_flush(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_disable(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_label(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_overwrite(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_clabel_commit(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_clabel(char **words, size_t num_words, void *user); - -PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user); - -PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_end_v2(char **words, size_t num_words, void *user); -void pluginsd_cleanup_v2(void *user); - -#endif diff --git a/libnetdata/popen/popen.c b/libnetdata/popen/popen.c index 783c74a51..5f8bd2b4a 100644 --- a/libnetdata/popen/popen.c +++ b/libnetdata/popen/popen.c @@ -55,7 +55,7 @@ static void netdata_popen_tracking_del_pid(pid_t pid) { freez(mp); } else - error("POPEN: Cannot find pid %d.", pid); + netdata_log_error("POPEN: Cannot find pid %d.", pid); netdata_popen_tracking_unlock(); } @@ -140,7 +140,7 @@ static int popene_internal(volatile pid_t *pidptr, char **env, uint8_t flags, FI // create a string to be logged about the command we are running char command_to_be_logged[2048]; convert_argv_to_string(command_to_be_logged, sizeof(command_to_be_logged), spawn_argv); - // info("custom_popene() running command: %s", command_to_be_logged); + // netdata_log_info("custom_popene() running command: %s", command_to_be_logged); int ret = 0; // success by default int attr_rc = 1; // failure by default @@ -156,33 +156,33 @@ static int popene_internal(volatile pid_t *pidptr, char **env, uint8_t flags, FI unsigned int fds_to_exclude_from_closing = OPEN_FD_EXCLUDE_STDERR; if(posix_spawn_file_actions_init(&fa)) { - error("POPEN: posix_spawn_file_actions_init() failed."); + netdata_log_error("POPEN: posix_spawn_file_actions_init() failed."); ret = -1; goto set_return_values_and_return; } if(fpp_child_stdin) { if (pipe(pipefd_stdin) == -1) { - error("POPEN: stdin pipe() failed"); + netdata_log_error("POPEN: stdin pipe() failed"); ret = -1; goto cleanup_and_return; } if ((fp_child_stdin = fdopen(pipefd_stdin[PIPE_WRITE], "w")) == NULL) { - error("POPEN: fdopen() stdin failed"); + netdata_log_error("POPEN: fdopen() stdin failed"); ret = -1; goto cleanup_and_return; } if(posix_spawn_file_actions_adddup2(&fa, pipefd_stdin[PIPE_READ], STDIN_FILENO)) { - error("POPEN: posix_spawn_file_actions_adddup2() on stdin failed."); + netdata_log_error("POPEN: posix_spawn_file_actions_adddup2() on stdin failed."); ret = -1; goto cleanup_and_return; } } else { if (posix_spawn_file_actions_addopen(&fa, STDIN_FILENO, "/dev/null", O_RDONLY, 0)) { - error("POPEN: posix_spawn_file_actions_addopen() on stdin to /dev/null failed."); + netdata_log_error("POPEN: posix_spawn_file_actions_addopen() on stdin to /dev/null failed."); // this is not a fatal error fds_to_exclude_from_closing |= OPEN_FD_EXCLUDE_STDIN; } @@ -190,26 +190,26 @@ static int popene_internal(volatile pid_t *pidptr, char **env, uint8_t flags, FI if (fpp_child_stdout) { if (pipe(pipefd_stdout) == -1) { - error("POPEN: stdout pipe() failed"); + netdata_log_error("POPEN: stdout pipe() failed"); ret = -1; goto cleanup_and_return; } if ((fp_child_stdout = fdopen(pipefd_stdout[PIPE_READ], "r")) == NULL) { - error("POPEN: fdopen() stdout failed"); + netdata_log_error("POPEN: fdopen() stdout failed"); ret = -1; goto cleanup_and_return; } if(posix_spawn_file_actions_adddup2(&fa, pipefd_stdout[PIPE_WRITE], STDOUT_FILENO)) { - error("POPEN: posix_spawn_file_actions_adddup2() on stdout failed."); + netdata_log_error("POPEN: posix_spawn_file_actions_adddup2() on stdout failed."); ret = -1; goto cleanup_and_return; } } else { if (posix_spawn_file_actions_addopen(&fa, STDOUT_FILENO, "/dev/null", O_WRONLY, 0)) { - error("POPEN: posix_spawn_file_actions_addopen() on stdout to /dev/null failed."); + netdata_log_error("POPEN: posix_spawn_file_actions_addopen() on stdout to /dev/null failed."); // this is not a fatal error fds_to_exclude_from_closing |= OPEN_FD_EXCLUDE_STDOUT; } @@ -223,20 +223,20 @@ static int popene_internal(volatile pid_t *pidptr, char **env, uint8_t flags, FI attr_rc = posix_spawnattr_init(&attr); if(attr_rc) { // failed - error("POPEN: posix_spawnattr_init() failed."); + netdata_log_error("POPEN: posix_spawnattr_init() failed."); } else { // success // reset all signals in the child if (posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF)) - error("POPEN: posix_spawnattr_setflags() failed."); + netdata_log_error("POPEN: posix_spawnattr_setflags() failed."); sigset_t mask; sigemptyset(&mask); if (posix_spawnattr_setsigmask(&attr, &mask)) - error("POPEN: posix_spawnattr_setsigmask() failed."); + netdata_log_error("POPEN: posix_spawnattr_setsigmask() failed."); } // Take the lock while we fork to ensure we don't race with SIGCHLD @@ -251,7 +251,7 @@ static int popene_internal(volatile pid_t *pidptr, char **env, uint8_t flags, FI else { // failure netdata_popen_tracking_unlock(); - error("POPEN: failed to spawn command: \"%s\" from parent pid %d.", command_to_be_logged, getpid()); + netdata_log_error("POPEN: failed to spawn command: \"%s\" from parent pid %d.", command_to_be_logged, getpid()); ret = -1; goto cleanup_and_return; } @@ -263,11 +263,11 @@ cleanup_and_return: if(!attr_rc) { // posix_spawnattr_init() succeeded if (posix_spawnattr_destroy(&attr)) - error("POPEN: posix_spawnattr_destroy() failed"); + netdata_log_error("POPEN: posix_spawnattr_destroy() failed"); } if (posix_spawn_file_actions_destroy(&fa)) - error("POPEN: posix_spawn_file_actions_destroy() failed"); + netdata_log_error("POPEN: posix_spawn_file_actions_destroy() failed"); // the child end - close it if(pipefd_stdin[PIPE_READ] != -1) @@ -384,7 +384,7 @@ int netdata_pclose(FILE *fp_child_input, FILE *fp_child_output, pid_t pid) { int ret; siginfo_t info; - debug(D_EXIT, "Request to netdata_pclose() on pid %d", pid); + netdata_log_debug(D_EXIT, "Request to netdata_pclose() on pid %d", pid); if (fp_child_input) fclose(fp_child_input); @@ -401,46 +401,46 @@ int netdata_pclose(FILE *fp_child_input, FILE *fp_child_output, pid_t pid) { switch (info.si_code) { case CLD_EXITED: if(info.si_status) - error("child pid %d exited with code %d.", info.si_pid, info.si_status); + netdata_log_error("child pid %d exited with code %d.", info.si_pid, info.si_status); return(info.si_status); case CLD_KILLED: if(info.si_status == SIGTERM) { - info("child pid %d killed by SIGTERM", info.si_pid); + netdata_log_info("child pid %d killed by SIGTERM", info.si_pid); return(0); } else if(info.si_status == SIGPIPE) { - info("child pid %d killed by SIGPIPE.", info.si_pid); + netdata_log_info("child pid %d killed by SIGPIPE.", info.si_pid); return(0); } else { - error("child pid %d killed by signal %d.", info.si_pid, info.si_status); + netdata_log_error("child pid %d killed by signal %d.", info.si_pid, info.si_status); return(-1); } case CLD_DUMPED: - error("child pid %d core dumped by signal %d.", info.si_pid, info.si_status); + netdata_log_error("child pid %d core dumped by signal %d.", info.si_pid, info.si_status); return(-2); case CLD_STOPPED: - error("child pid %d stopped by signal %d.", info.si_pid, info.si_status); + netdata_log_error("child pid %d stopped by signal %d.", info.si_pid, info.si_status); return(0); case CLD_TRAPPED: - error("child pid %d trapped by signal %d.", info.si_pid, info.si_status); + netdata_log_error("child pid %d trapped by signal %d.", info.si_pid, info.si_status); return(-4); case CLD_CONTINUED: - error("child pid %d continued by signal %d.", info.si_pid, info.si_status); + netdata_log_error("child pid %d continued by signal %d.", info.si_pid, info.si_status); return(0); default: - error("child pid %d gave us a SIGCHLD with code %d and status %d.", info.si_pid, info.si_code, info.si_status); + netdata_log_error("child pid %d gave us a SIGCHLD with code %d and status %d.", info.si_pid, info.si_code, info.si_status); return(-5); } } else - error("Cannot waitid() for pid %d", pid); + netdata_log_error("Cannot waitid() for pid %d", pid); return 0; } diff --git a/libnetdata/procfile/procfile.c b/libnetdata/procfile/procfile.c index cdf0f9723..1a7e47a56 100644 --- a/libnetdata/procfile/procfile.c +++ b/libnetdata/procfile/procfile.c @@ -48,11 +48,11 @@ char *procfile_filename(procfile *ff) { // An array of words static inline void procfile_words_add(procfile *ff, char *str) { - // debug(D_PROCFILE, PF_PREFIX ": adding word No %d: '%s'", fw->len, str); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": adding word No %d: '%s'", fw->len, str); pfwords *fw = ff->words; if(unlikely(fw->len == fw->size)) { - // debug(D_PROCFILE, PF_PREFIX ": expanding words"); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": expanding words"); size_t minimum = PFWORDS_INCREASE_STEP; size_t optimal = fw->size / 2; size_t wanted = (optimal > minimum)?optimal:minimum; @@ -66,7 +66,7 @@ static inline void procfile_words_add(procfile *ff, char *str) { NEVERNULL static inline pfwords *procfile_words_create(void) { - // debug(D_PROCFILE, PF_PREFIX ": initializing words"); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": initializing words"); size_t size = (procfile_adaptive_initial_allocation) ? procfile_max_words : PFWORDS_INCREASE_STEP; @@ -77,12 +77,12 @@ static inline pfwords *procfile_words_create(void) { } static inline void procfile_words_reset(pfwords *fw) { - // debug(D_PROCFILE, PF_PREFIX ": resetting words"); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": resetting words"); fw->len = 0; } static inline void procfile_words_free(pfwords *fw) { - // debug(D_PROCFILE, PF_PREFIX ": freeing words"); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": freeing words"); freez(fw); } @@ -93,11 +93,11 @@ static inline void procfile_words_free(pfwords *fw) { NEVERNULL static inline size_t *procfile_lines_add(procfile *ff) { - // debug(D_PROCFILE, PF_PREFIX ": adding line %d at word %d", fl->len, first_word); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": adding line %d at word %d", fl->len, first_word); pflines *fl = ff->lines; if(unlikely(fl->len == fl->size)) { - // debug(D_PROCFILE, PF_PREFIX ": expanding lines"); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": expanding lines"); size_t minimum = PFLINES_INCREASE_STEP; size_t optimal = fl->size / 2; size_t wanted = (optimal > minimum)?optimal:minimum; @@ -115,7 +115,7 @@ static inline size_t *procfile_lines_add(procfile *ff) { NEVERNULL static inline pflines *procfile_lines_create(void) { - // debug(D_PROCFILE, PF_PREFIX ": initializing lines"); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": initializing lines"); size_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_words : PFLINES_INCREASE_STEP; @@ -126,13 +126,13 @@ static inline pflines *procfile_lines_create(void) { } static inline void procfile_lines_reset(pflines *fl) { - // debug(D_PROCFILE, PF_PREFIX ": resetting lines"); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": resetting lines"); fl->len = 0; } static inline void procfile_lines_free(pflines *fl) { - // debug(D_PROCFILE, PF_PREFIX ": freeing lines"); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": freeing lines"); freez(fl); } @@ -144,7 +144,7 @@ static inline void procfile_lines_free(pflines *fl) { void procfile_close(procfile *ff) { if(unlikely(!ff)) return; - debug(D_PROCFILE, PF_PREFIX ": Closing file '%s'", procfile_filename(ff)); + netdata_log_debug(D_PROCFILE, PF_PREFIX ": Closing file '%s'", procfile_filename(ff)); freez(ff->filename); procfile_lines_free(ff->lines); @@ -156,7 +156,7 @@ void procfile_close(procfile *ff) { NOINLINE static void procfile_parser(procfile *ff) { - // debug(D_PROCFILE, PF_PREFIX ": Parsing file '%s'", ff->filename); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": Parsing file '%s'", ff->filename); char *s = ff->data // our current position , *e = &ff->data[ff->len] // the terminating null @@ -206,7 +206,7 @@ static void procfile_parser(procfile *ff) { (*line_words)++; t = ++s; - // debug(D_PROCFILE, PF_PREFIX ": ended line %d with %d words", l, ff->lines->lines[l].words); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": ended line %d with %d words", l, ff->lines->lines[l].words); line_words = procfile_lines_add(ff); } @@ -275,7 +275,7 @@ static void procfile_parser(procfile *ff) { } procfile *procfile_readall(procfile *ff) { - // debug(D_PROCFILE, PF_PREFIX ": Reading file '%s'.", ff->filename); + // netdata_log_debug(D_PROCFILE, PF_PREFIX ": Reading file '%s'.", ff->filename); ff->len = 0; // zero the used size ssize_t r = 1; // read at least once @@ -288,16 +288,17 @@ procfile *procfile_readall(procfile *ff) { size_t optimal = ff->size / 2; size_t wanted = (optimal > minimum)?optimal:minimum; - debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s' by %zu bytes.", procfile_filename(ff), wanted); + netdata_log_debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s' by %zu bytes.", procfile_filename(ff), wanted); ff = reallocz(ff, sizeof(procfile) + ff->size + wanted); ff->size += wanted; } - debug(D_PROCFILE, "Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s)); + netdata_log_debug(D_PROCFILE, "Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s)); r = read(ff->fd, &ff->data[s], ff->size - s); if(unlikely(r == -1)) { if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) collector_error(PF_PREFIX ": Cannot read from file '%s' on fd %d", procfile_filename(ff), ff->fd); - else if(unlikely(ff->flags & PROCFILE_FLAG_ERROR_ON_ERROR_LOG)) error(PF_PREFIX ": Cannot read from file '%s' on fd %d", procfile_filename(ff), ff->fd); + else if(unlikely(ff->flags & PROCFILE_FLAG_ERROR_ON_ERROR_LOG)) + netdata_log_error(PF_PREFIX ": Cannot read from file '%s' on fd %d", procfile_filename(ff), ff->fd); procfile_close(ff); return NULL; } @@ -305,10 +306,11 @@ procfile *procfile_readall(procfile *ff) { ff->len += r; } - // debug(D_PROCFILE, "Rewinding file '%s'", ff->filename); + // netdata_log_debug(D_PROCFILE, "Rewinding file '%s'", ff->filename); if(unlikely(lseek(ff->fd, 0, SEEK_SET) == -1)) { if(unlikely(!(ff->flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) collector_error(PF_PREFIX ": Cannot rewind on file '%s'.", procfile_filename(ff)); - else if(unlikely(ff->flags & PROCFILE_FLAG_ERROR_ON_ERROR_LOG)) error(PF_PREFIX ": Cannot rewind on file '%s'.", procfile_filename(ff)); + else if(unlikely(ff->flags & PROCFILE_FLAG_ERROR_ON_ERROR_LOG)) + netdata_log_error(PF_PREFIX ": Cannot rewind on file '%s'.", procfile_filename(ff)); procfile_close(ff); return NULL; } @@ -323,7 +325,7 @@ procfile *procfile_readall(procfile *ff) { if(unlikely(ff->words->len > procfile_max_words)) procfile_max_words = ff->words->len; } - // debug(D_PROCFILE, "File '%s' updated.", ff->filename); + // netdata_log_debug(D_PROCFILE, "File '%s' updated.", ff->filename); return ff; } @@ -401,16 +403,17 @@ void procfile_set_open_close(procfile *ff, const char *open, const char *close) } procfile *procfile_open(const char *filename, const char *separators, uint32_t flags) { - debug(D_PROCFILE, PF_PREFIX ": Opening file '%s'", filename); + netdata_log_debug(D_PROCFILE, PF_PREFIX ": Opening file '%s'", filename); int fd = open(filename, procfile_open_flags, 0666); if(unlikely(fd == -1)) { if(unlikely(!(flags & PROCFILE_FLAG_NO_ERROR_ON_FILE_IO))) collector_error(PF_PREFIX ": Cannot open file '%s'", filename); - else if(unlikely(flags & PROCFILE_FLAG_ERROR_ON_ERROR_LOG)) error(PF_PREFIX ": Cannot open file '%s'", filename); + else if(unlikely(flags & PROCFILE_FLAG_ERROR_ON_ERROR_LOG)) + netdata_log_error(PF_PREFIX ": Cannot open file '%s'", filename); return NULL; } - // info("PROCFILE: opened '%s' on fd %d", filename, fd); + // netdata_log_info("PROCFILE: opened '%s' on fd %d", filename, fd); size_t size = (unlikely(procfile_adaptive_initial_allocation)) ? procfile_max_allocation : PROCFILE_INCREMENT_BUFFER; procfile *ff = mallocz(sizeof(procfile) + size); @@ -427,7 +430,7 @@ procfile *procfile_open(const char *filename, const char *separators, uint32_t f procfile_set_separators(ff, separators); - debug(D_PROCFILE, "File '%s' opened.", filename); + netdata_log_debug(D_PROCFILE, "File '%s' opened.", filename); return ff; } @@ -435,7 +438,7 @@ procfile *procfile_reopen(procfile *ff, const char *filename, const char *separa if(unlikely(!ff)) return procfile_open(filename, separators, flags); if(likely(ff->fd != -1)) { - // info("PROCFILE: closing fd %d", ff->fd); + // netdata_log_info("PROCFILE: closing fd %d", ff->fd); close(ff->fd); } @@ -445,7 +448,7 @@ procfile *procfile_reopen(procfile *ff, const char *filename, const char *separa return NULL; } - // info("PROCFILE: opened '%s' on fd %d", filename, ff->fd); + // netdata_log_info("PROCFILE: opened '%s' on fd %d", filename, ff->fd); //strncpyz(ff->filename, filename, FILENAME_MAX); freez(ff->filename); @@ -466,17 +469,17 @@ void procfile_print(procfile *ff) { char *s; (void)s; - debug(D_PROCFILE, "File '%s' with %zu lines and %zu words", procfile_filename(ff), ff->lines->len, ff->words->len); + netdata_log_debug(D_PROCFILE, "File '%s' with %zu lines and %zu words", procfile_filename(ff), ff->lines->len, ff->words->len); for(l = 0; likely(l < lines) ;l++) { size_t words = procfile_linewords(ff, l); - debug(D_PROCFILE, " line %zu starts at word %zu and has %zu words", l, ff->lines->lines[l].first, ff->lines->lines[l].words); + netdata_log_debug(D_PROCFILE, " line %zu starts at word %zu and has %zu words", l, ff->lines->lines[l].first, ff->lines->lines[l].words); size_t w; for(w = 0; likely(w < words) ;w++) { s = procfile_lineword(ff, l, w); - debug(D_PROCFILE, " [%zu.%zu] '%s'", l, w, s); + netdata_log_debug(D_PROCFILE, " [%zu.%zu] '%s'", l, w, s); } } } diff --git a/libnetdata/simple_pattern/simple_pattern.c b/libnetdata/simple_pattern/simple_pattern.c index a26ae4f92..70bde73a6 100644 --- a/libnetdata/simple_pattern/simple_pattern.c +++ b/libnetdata/simple_pattern/simple_pattern.c @@ -326,10 +326,10 @@ extern void simple_pattern_dump(uint64_t debug_type, SIMPLE_PATTERN *p) { struct simple_pattern *root = (struct simple_pattern *)p; if(root==NULL) { - debug(debug_type,"dump_pattern(NULL)"); + netdata_log_debug(debug_type,"dump_pattern(NULL)"); return; } - debug(debug_type,"dump_pattern(%p) child=%p next=%p mode=%u match=%s", root, root->child, root->next, root->mode, + netdata_log_debug(debug_type,"dump_pattern(%p) child=%p next=%p mode=%u match=%s", root, root->child, root->next, root->mode, root->match); if(root->child!=NULL) simple_pattern_dump(debug_type, (SIMPLE_PATTERN*)root->child); diff --git a/libnetdata/socket/security.c b/libnetdata/socket/security.c index abae71c66..c1bb76345 100644 --- a/libnetdata/socket/security.c +++ b/libnetdata/socket/security.c @@ -406,7 +406,7 @@ bool netdata_ssl_accept(NETDATA_SSL *ssl) { static void netdata_ssl_info_callback(const SSL *ssl, int where, int ret __maybe_unused) { (void)ssl; if (where & SSL_CB_ALERT) { - debug(D_WEB_CLIENT,"SSL INFO CALLBACK %s %s", SSL_alert_type_string(ret), SSL_alert_desc_string_long(ret)); + netdata_log_debug(D_WEB_CLIENT,"SSL INFO CALLBACK %s %s", SSL_alert_type_string(ret), SSL_alert_desc_string_long(ret)); } } @@ -429,7 +429,7 @@ void netdata_ssl_initialize_openssl() { #else if (OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL) != 1) { - error("SSL library cannot be initialized."); + netdata_log_error("SSL library cannot be initialized."); } #endif @@ -516,7 +516,7 @@ static SSL_CTX * netdata_ssl_create_server_ctx(unsigned long mode) { #if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110 ctx = SSL_CTX_new(SSLv23_server_method()); if (!ctx) { - error("Cannot create a new SSL context, netdata won't encrypt communication"); + netdata_log_error("Cannot create a new SSL context, netdata won't encrypt communication"); return NULL; } @@ -524,7 +524,7 @@ static SSL_CTX * netdata_ssl_create_server_ctx(unsigned long mode) { #else ctx = SSL_CTX_new(TLS_server_method()); if (!ctx) { - error("Cannot create a new SSL context, netdata won't encrypt communication"); + netdata_log_error("Cannot create a new SSL context, netdata won't encrypt communication"); return NULL; } @@ -539,7 +539,7 @@ static SSL_CTX * netdata_ssl_create_server_ctx(unsigned long mode) { if(tls_ciphers && strcmp(tls_ciphers, "none") != 0) { if (!SSL_CTX_set_cipher_list(ctx, tls_ciphers)) { - error("SSL error. cannot set the cipher list"); + netdata_log_error("SSL error. cannot set the cipher list"); } } #endif @@ -548,7 +548,7 @@ static SSL_CTX * netdata_ssl_create_server_ctx(unsigned long mode) { if (!SSL_CTX_check_private_key(ctx)) { ERR_error_string_n(ERR_get_error(),lerror,sizeof(lerror)); - error("SSL cannot check the private key: %s",lerror); + netdata_log_error("SSL cannot check the private key: %s",lerror); SSL_CTX_free(ctx); return NULL; } @@ -559,7 +559,7 @@ static SSL_CTX * netdata_ssl_create_server_ctx(unsigned long mode) { #if (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_095) SSL_CTX_set_verify_depth(ctx,1); #endif - debug(D_WEB_CLIENT,"SSL GLOBAL CONTEXT STARTED\n"); + netdata_log_debug(D_WEB_CLIENT,"SSL GLOBAL CONTEXT STARTED\n"); SSL_CTX_set_mode(ctx, mode); @@ -578,14 +578,14 @@ static SSL_CTX * netdata_ssl_create_server_ctx(unsigned long mode) { */ void netdata_ssl_initialize_ctx(int selector) { static SPINLOCK sp = NETDATA_SPINLOCK_INITIALIZER; - netdata_spinlock_lock(&sp); + spinlock_lock(&sp); switch (selector) { case NETDATA_SSL_WEB_SERVER_CTX: { if(!netdata_ssl_web_server_ctx) { struct stat statbuf; if (stat(netdata_ssl_security_key, &statbuf) || stat(netdata_ssl_security_cert, &statbuf)) - info("To use encryption it is necessary to set \"ssl certificate\" and \"ssl key\" in [web] !\n"); + netdata_log_info("To use encryption it is necessary to set \"ssl certificate\" and \"ssl key\" in [web] !\n"); else { netdata_ssl_web_server_ctx = netdata_ssl_create_server_ctx( SSL_MODE_ENABLE_PARTIAL_WRITE | @@ -628,7 +628,7 @@ void netdata_ssl_initialize_ctx(int selector) { } } - netdata_spinlock_unlock(&sp); + spinlock_unlock(&sp); } /** @@ -680,7 +680,7 @@ int security_test_certificate(SSL *ssl) { { char error[512]; ERR_error_string_n(ERR_get_error(), error, sizeof(error)); - error("SSL RFC4158 check: We have a invalid certificate, the tests result with %ld and message %s", status, error); + netdata_log_error("SSL RFC4158 check: We have a invalid certificate, the tests result with %ld and message %s", status, error); ret = -1; } else { ret = 0; @@ -705,13 +705,13 @@ int ssl_security_location_for_context(SSL_CTX *ctx, char *file, char *path) { int load_custom = 1, load_default = 1; if (file || path) { if(!SSL_CTX_load_verify_locations(ctx, file, path)) { - info("Netdata can not verify custom CAfile or CApath for parent's SSL certificate, so it will use the default OpenSSL configuration to validate certificates!"); + netdata_log_info("Netdata can not verify custom CAfile or CApath for parent's SSL certificate, so it will use the default OpenSSL configuration to validate certificates!"); load_custom = 0; } } if(!SSL_CTX_set_default_verify_paths(ctx)) { - info("Can not verify default OpenSSL configuration to validate certificates!"); + netdata_log_info("Can not verify default OpenSSL configuration to validate certificates!"); load_default = 0; } diff --git a/libnetdata/socket/socket.c b/libnetdata/socket/socket.c index 7f0b81fee..e7d0b4807 100644 --- a/libnetdata/socket/socket.c +++ b/libnetdata/socket/socket.c @@ -15,10 +15,10 @@ SOCKET_PEERS socket_peers(int sock_fd) { SOCKET_PEERS peers; if(sock_fd < 0) { - strncpyz(peers.peer.ip, "unknown", sizeof(peers.peer.ip) - 1); + strncpyz(peers.peer.ip, "not connected", sizeof(peers.peer.ip) - 1); peers.peer.port = 0; - strncpyz(peers.local.ip, "unknown", sizeof(peers.local.ip) - 1); + strncpyz(peers.local.ip, "not connected", sizeof(peers.local.ip) - 1); peers.local.port = 0; return peers; @@ -124,7 +124,7 @@ int sock_setnonblock(int fd) { int ret = fcntl(fd, F_SETFL, flags); if(ret < 0) - error("Failed to set O_NONBLOCK on socket %d", fd); + netdata_log_error("Failed to set O_NONBLOCK on socket %d", fd); return ret; } @@ -137,7 +137,7 @@ int sock_delnonblock(int fd) { int ret = fcntl(fd, F_SETFL, flags); if(ret < 0) - error("Failed to remove O_NONBLOCK on socket %d", fd); + netdata_log_error("Failed to remove O_NONBLOCK on socket %d", fd); return ret; } @@ -146,7 +146,7 @@ int sock_setreuse(int fd, int reuse) { int ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); if(ret == -1) - error("Failed to set SO_REUSEADDR on socket %d", fd); + netdata_log_error("Failed to set SO_REUSEADDR on socket %d", fd); return ret; } @@ -157,7 +157,7 @@ int sock_setreuse_port(int fd, int reuse) { #ifdef SO_REUSEPORT ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &reuse, sizeof(reuse)); if(ret == -1 && errno != ENOPROTOOPT) - error("failed to set SO_REUSEPORT on socket %d", fd); + netdata_log_error("failed to set SO_REUSEPORT on socket %d", fd); #else ret = -1; #endif @@ -171,7 +171,7 @@ int sock_enlarge_in(int fd) { ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &bs, sizeof(bs)); if(ret == -1) - error("Failed to set SO_RCVBUF on socket %d", fd); + netdata_log_error("Failed to set SO_RCVBUF on socket %d", fd); return ret; } @@ -181,7 +181,7 @@ int sock_enlarge_out(int fd) { ret = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &bs, sizeof(bs)); if(ret == -1) - error("Failed to set SO_SNDBUF on socket %d", fd); + netdata_log_error("Failed to set SO_SNDBUF on socket %d", fd); return ret; } @@ -216,11 +216,11 @@ char *strdup_client_description(int family, const char *protocol, const char *ip int create_listen_socket_unix(const char *path, int listen_backlog) { int sock; - debug(D_LISTENER, "LISTENER: UNIX creating new listening socket on path '%s'", path); + netdata_log_debug(D_LISTENER, "LISTENER: UNIX creating new listening socket on path '%s'", path); sock = socket(AF_UNIX, SOCK_STREAM, 0); if(sock < 0) { - error("LISTENER: UNIX socket() on path '%s' failed.", path); + netdata_log_error("LISTENER: UNIX socket() on path '%s' failed.", path); return -1; } @@ -234,37 +234,37 @@ int create_listen_socket_unix(const char *path, int listen_backlog) { errno = 0; if (unlink(path) == -1 && errno != ENOENT) - error("LISTENER: failed to remove existing (probably obsolete or left-over) file on UNIX socket path '%s'.", path); + netdata_log_error("LISTENER: failed to remove existing (probably obsolete or left-over) file on UNIX socket path '%s'.", path); if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { close(sock); - error("LISTENER: UNIX bind() on path '%s' failed.", path); + netdata_log_error("LISTENER: UNIX bind() on path '%s' failed.", path); return -1; } // we have to chmod this to 0777 so that the client will be able // to read from and write to this socket. if(chmod(path, 0777) == -1) - error("LISTENER: failed to chmod() socket file '%s'.", path); + netdata_log_error("LISTENER: failed to chmod() socket file '%s'.", path); if(listen(sock, listen_backlog) < 0) { close(sock); - error("LISTENER: UNIX listen() on path '%s' failed.", path); + netdata_log_error("LISTENER: UNIX listen() on path '%s' failed.", path); return -1; } - debug(D_LISTENER, "LISTENER: Listening on UNIX path '%s'", path); + netdata_log_debug(D_LISTENER, "LISTENER: Listening on UNIX path '%s'", path); return sock; } int create_listen_socket4(int socktype, const char *ip, uint16_t port, int listen_backlog) { int sock; - debug(D_LISTENER, "LISTENER: IPv4 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype); + netdata_log_debug(D_LISTENER, "LISTENER: IPv4 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype); sock = socket(AF_INET, socktype, 0); if(sock < 0) { - error("LISTENER: IPv4 socket() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + netdata_log_error("LISTENER: IPv4 socket() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); return -1; } @@ -280,24 +280,24 @@ int create_listen_socket4(int socktype, const char *ip, uint16_t port, int liste int ret = inet_pton(AF_INET, ip, (void *)&name.sin_addr.s_addr); if(ret != 1) { - error("LISTENER: Failed to convert IP '%s' to a valid IPv4 address.", ip); + netdata_log_error("LISTENER: Failed to convert IP '%s' to a valid IPv4 address.", ip); close(sock); return -1; } if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { close(sock); - error("LISTENER: IPv4 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + netdata_log_error("LISTENER: IPv4 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); return -1; } if(socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) { close(sock); - error("LISTENER: IPv4 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + netdata_log_error("LISTENER: IPv4 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); return -1; } - debug(D_LISTENER, "LISTENER: Listening on IPv4 ip '%s' port %d, socktype %d", ip, port, socktype); + netdata_log_debug(D_LISTENER, "LISTENER: Listening on IPv4 ip '%s' port %d, socktype %d", ip, port, socktype); return sock; } @@ -305,11 +305,11 @@ int create_listen_socket6(int socktype, uint32_t scope_id, const char *ip, int p int sock; int ipv6only = 1; - debug(D_LISTENER, "LISTENER: IPv6 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype); + netdata_log_debug(D_LISTENER, "LISTENER: IPv6 creating new listening socket on ip '%s' port %d, socktype %d", ip, port, socktype); sock = socket(AF_INET6, socktype, 0); if (sock < 0) { - error("LISTENER: IPv6 socket() on ip '%s' port %d, socktype %d, failed.", ip, port, socktype); + netdata_log_error("LISTENER: IPv6 socket() on ip '%s' port %d, socktype %d, failed.", ip, port, socktype); return -1; } @@ -320,7 +320,7 @@ int create_listen_socket6(int socktype, uint32_t scope_id, const char *ip, int p /* IPv6 only */ if(setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (void*)&ipv6only, sizeof(ipv6only)) != 0) - error("LISTENER: Cannot set IPV6_V6ONLY on ip '%s' port %d, socktype %d.", ip, port, socktype); + netdata_log_error("LISTENER: Cannot set IPV6_V6ONLY on ip '%s' port %d, socktype %d.", ip, port, socktype); struct sockaddr_in6 name; memset(&name, 0, sizeof(struct sockaddr_in6)); @@ -330,7 +330,7 @@ int create_listen_socket6(int socktype, uint32_t scope_id, const char *ip, int p int ret = inet_pton(AF_INET6, ip, (void *)&name.sin6_addr.s6_addr); if(ret != 1) { - error("LISTENER: Failed to convert IP '%s' to a valid IPv6 address.", ip); + netdata_log_error("LISTENER: Failed to convert IP '%s' to a valid IPv6 address.", ip); close(sock); return -1; } @@ -339,23 +339,23 @@ int create_listen_socket6(int socktype, uint32_t scope_id, const char *ip, int p if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) { close(sock); - error("LISTENER: IPv6 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + netdata_log_error("LISTENER: IPv6 bind() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); return -1; } if (socktype == SOCK_STREAM && listen(sock, listen_backlog) < 0) { close(sock); - error("LISTENER: IPv6 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); + netdata_log_error("LISTENER: IPv6 listen() on ip '%s' port %d, socktype %d failed.", ip, port, socktype); return -1; } - debug(D_LISTENER, "LISTENER: Listening on IPv6 ip '%s' port %d, socktype %d", ip, port, socktype); + netdata_log_debug(D_LISTENER, "LISTENER: Listening on IPv6 ip '%s' port %d, socktype %d", ip, port, socktype); return sock; } static inline int listen_sockets_add(LISTEN_SOCKETS *sockets, int fd, int family, int socktype, const char *protocol, const char *ip, uint16_t port, int acl_flags) { if(sockets->opened >= MAX_LISTEN_FDS) { - error("LISTENER: Too many listening sockets. Failed to add listening %s socket at ip '%s' port %d, protocol %s, socktype %d", protocol, ip, port, protocol, socktype); + netdata_log_error("LISTENER: Too many listening sockets. Failed to add listening %s socket at ip '%s' port %d, protocol %s, socktype %d", protocol, ip, port, protocol, socktype); close(fd); return -1; } @@ -485,7 +485,7 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, protocol_str = "unix"; int fd = create_listen_socket_unix(path, listen_backlog); if (fd == -1) { - error("LISTENER: Cannot create unix socket '%s'", path); + netdata_log_error("LISTENER: Cannot create unix socket '%s'", path); sockets->failed++; } else { acl_flags = WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_REGISTRY | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_NETDATACONF | WEB_CLIENT_ACL_STREAMING | WEB_CLIENT_ACL_SSL_DEFAULT; @@ -551,7 +551,7 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, if(*interface) { scope_id = if_nametoindex(interface); if(!scope_id) - error("LISTENER: Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface); + netdata_log_error("LISTENER: Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface); } if(!*ip || *ip == '*' || !strcmp(ip, "any") || !strcmp(ip, "all")) @@ -571,7 +571,7 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, int r = getaddrinfo(ip, port, &hints, &result); if (r != 0) { - error("LISTENER: getaddrinfo('%s', '%s'): %s\n", ip, port, gai_strerror(r)); + netdata_log_error("LISTENER: getaddrinfo('%s', '%s'): %s\n", ip, port, gai_strerror(r)); return -1; } @@ -588,7 +588,7 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, struct sockaddr_in *sin = (struct sockaddr_in *) rp->ai_addr; inet_ntop(AF_INET, &sin->sin_addr, rip, INET_ADDRSTRLEN); rport = ntohs(sin->sin_port); - // info("Attempting to listen on IPv4 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype); + // netdata_log_info("Attempting to listen on IPv4 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype); fd = create_listen_socket4(socktype, rip, rport, listen_backlog); break; } @@ -597,18 +597,18 @@ static inline int bind_to_this(LISTEN_SOCKETS *sockets, const char *definition, struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) rp->ai_addr; inet_ntop(AF_INET6, &sin6->sin6_addr, rip, INET6_ADDRSTRLEN); rport = ntohs(sin6->sin6_port); - // info("Attempting to listen on IPv6 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype); + // netdata_log_info("Attempting to listen on IPv6 '%s' ('%s'), port %d ('%s'), socktype %d", rip, ip, rport, port, socktype); fd = create_listen_socket6(socktype, scope_id, rip, rport, listen_backlog); break; } default: - debug(D_LISTENER, "LISTENER: Unknown socket family %d", family); + netdata_log_debug(D_LISTENER, "LISTENER: Unknown socket family %d", family); break; } if (fd == -1) { - error("LISTENER: Cannot bind to ip '%s', port %d", rip, rport); + netdata_log_error("LISTENER: Cannot bind to ip '%s', port %d", rip, rport); sockets->failed++; } else { @@ -630,12 +630,12 @@ int listen_sockets_setup(LISTEN_SOCKETS *sockets) { long long int old_port = sockets->default_port; long long int new_port = appconfig_get_number(sockets->config, sockets->config_section, "default port", sockets->default_port); if(new_port < 1 || new_port > 65535) { - error("LISTENER: Invalid listen port %lld given. Defaulting to %lld.", new_port, old_port); + netdata_log_error("LISTENER: Invalid listen port %lld given. Defaulting to %lld.", new_port, old_port); sockets->default_port = (uint16_t) appconfig_set_number(sockets->config, sockets->config_section, "default port", old_port); } else sockets->default_port = (uint16_t)new_port; - debug(D_OPTIONS, "LISTENER: Default listen port set to %d.", sockets->default_port); + netdata_log_debug(D_OPTIONS, "LISTENER: Default listen port set to %d.", sockets->default_port); char *s = appconfig_get(sockets->config, sockets->config_section, "bind to", sockets->default_bind_to); while(*s) { @@ -660,7 +660,7 @@ int listen_sockets_setup(LISTEN_SOCKETS *sockets) { if(sockets->failed) { size_t i; for(i = 0; i < sockets->opened ;i++) - info("LISTENER: Listen socket %s opened successfully.", sockets->fds_names[i]); + netdata_log_info("LISTENER: Listen socket %s opened successfully.", sockets->fds_names[i]); } return (int)sockets->opened; @@ -677,13 +677,13 @@ int listen_sockets_setup(LISTEN_SOCKETS *sockets) { static inline int connect_to_unix(const char *path, struct timeval *timeout) { int fd = socket(AF_UNIX, SOCK_STREAM, 0); if(fd == -1) { - error("Failed to create UNIX socket() for '%s'", path); + netdata_log_error("Failed to create UNIX socket() for '%s'", path); return -1; } if(timeout) { if(setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char *) timeout, sizeof(struct timeval)) < 0) - error("Failed to set timeout on UNIX socket '%s'", path); + netdata_log_error("Failed to set timeout on UNIX socket '%s'", path); } struct sockaddr_un addr; @@ -692,12 +692,12 @@ static inline int connect_to_unix(const char *path, struct timeval *timeout) { strncpy(addr.sun_path, path, sizeof(addr.sun_path)-1); if (connect(fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) { - error("Cannot connect to UNIX socket on path '%s'.", path); + netdata_log_error("Cannot connect to UNIX socket on path '%s'.", path); close(fd); return -1; } - debug(D_CONNECT_TO, "Connected to UNIX socket on path '%s'.", path); + netdata_log_debug(D_CONNECT_TO, "Connected to UNIX socket on path '%s'.", path); return fd; } @@ -723,7 +723,7 @@ int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t int ai_err = getaddrinfo(host, service, &hints, &ai_head); if (ai_err != 0) { - error("Cannot resolve host '%s', port '%s': %s", host, service, gai_strerror(ai_err)); + netdata_log_error("Cannot resolve host '%s', port '%s': %s", host, service, gai_strerror(ai_err)); return -1; } @@ -748,7 +748,7 @@ int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t sizeof(servBfr), NI_NUMERICHOST | NI_NUMERICSERV); - debug(D_CONNECT_TO, "Address info: host = '%s', service = '%s', ai_flags = 0x%02X, ai_family = %d (PF_INET = %d, PF_INET6 = %d), ai_socktype = %d (SOCK_STREAM = %d, SOCK_DGRAM = %d), ai_protocol = %d (IPPROTO_TCP = %d, IPPROTO_UDP = %d), ai_addrlen = %lu (sockaddr_in = %lu, sockaddr_in6 = %lu)", + netdata_log_debug(D_CONNECT_TO, "Address info: host = '%s', service = '%s', ai_flags = 0x%02X, ai_family = %d (PF_INET = %d, PF_INET6 = %d), ai_socktype = %d (SOCK_STREAM = %d, SOCK_DGRAM = %d), ai_protocol = %d (IPPROTO_TCP = %d, IPPROTO_UDP = %d), ai_addrlen = %lu (sockaddr_in = %lu, sockaddr_in6 = %lu)", hostBfr, servBfr, (unsigned int)ai->ai_flags, @@ -770,7 +770,7 @@ int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t struct sockaddr_in *pSadrIn = (struct sockaddr_in *)ai->ai_addr; (void)pSadrIn; - debug(D_CONNECT_TO, "ai_addr = sin_family: %d (AF_INET = %d, AF_INET6 = %d), sin_addr: '%s', sin_port: '%s'", + netdata_log_debug(D_CONNECT_TO, "ai_addr = sin_family: %d (AF_INET = %d, AF_INET6 = %d), sin_addr: '%s', sin_port: '%s'", pSadrIn->sin_family, AF_INET, AF_INET6, @@ -783,7 +783,7 @@ int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t struct sockaddr_in6 *pSadrIn6 = (struct sockaddr_in6 *) ai->ai_addr; (void)pSadrIn6; - debug(D_CONNECT_TO,"ai_addr = sin6_family: %d (AF_INET = %d, AF_INET6 = %d), sin6_addr: '%s', sin6_port: '%s', sin6_flowinfo: %u, sin6_scope_id: %u", + netdata_log_debug(D_CONNECT_TO,"ai_addr = sin6_family: %d (AF_INET = %d, AF_INET6 = %d), sin6_addr: '%s', sin6_port: '%s', sin6_flowinfo: %u, sin6_scope_id: %u", pSadrIn6->sin6_family, AF_INET, AF_INET6, @@ -795,7 +795,7 @@ int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t } default: { - debug(D_CONNECT_TO, "Unknown protocol family %d.", ai->ai_family); + netdata_log_debug(D_CONNECT_TO, "Unknown protocol family %d.", ai->ai_family); continue; } } @@ -804,42 +804,57 @@ int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t if(fd != -1) { if(timeout) { if(setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char *) timeout, sizeof(struct timeval)) < 0) - error("Failed to set timeout on the socket to ip '%s' port '%s'", hostBfr, servBfr); + netdata_log_error("Failed to set timeout on the socket to ip '%s' port '%s'", hostBfr, servBfr); } errno = 0; if(connect(fd, ai->ai_addr, ai->ai_addrlen) < 0) { if(errno == EALREADY || errno == EINPROGRESS) { - info("Waiting for connection to ip %s port %s to be established", hostBfr, servBfr); + netdata_log_info("Waiting for connection to ip %s port %s to be established", hostBfr, servBfr); + + // Convert 'struct timeval' to milliseconds for poll(): + int timeout_milliseconds = timeout->tv_sec * 1000 + timeout->tv_usec / 1000; - fd_set fds; - FD_ZERO(&fds); - FD_SET(0, &fds); - int rc = select (1, NULL, &fds, NULL, timeout); + struct pollfd fds[1]; + fds[0].fd = fd; + fds[0].events = POLLOUT; // We are looking for the ability to write to the socket - if(rc > 0 && FD_ISSET(fd, &fds)) { - info("connect() to ip %s port %s completed successfully", hostBfr, servBfr); + int ret = poll(fds, 1, timeout_milliseconds); + if (ret > 0) { + // poll() completed normally. We can check the revents to see what happened + if (fds[0].revents & POLLOUT) { + // connect() completed successfully, socket is writable. + netdata_log_info("connect() to ip %s port %s completed successfully", hostBfr, servBfr); + } + else { + // This means that the socket is in error. We will close it and set fd to -1 + netdata_log_error("Failed to connect to '%s', port '%s'.", hostBfr, servBfr); + close(fd); + fd = -1; + } } - else if(rc == -1) { - error("Failed to connect to '%s', port '%s'. select() returned %d", hostBfr, servBfr, rc); + else if (ret == 0) { + // poll() timed out, the connection is not established within the specified timeout. + netdata_log_error("Timed out while connecting to '%s', port '%s'.", hostBfr, servBfr); close(fd); fd = -1; } else { - error("Timed out while connecting to '%s', port '%s'. select() returned %d", hostBfr, servBfr, rc); + // poll() returned an error. + netdata_log_error("Failed to connect to '%s', port '%s'. poll() returned %d", hostBfr, servBfr, ret); close(fd); fd = -1; } } else { - error("Failed to connect to '%s', port '%s'", hostBfr, servBfr); + netdata_log_error("Failed to connect to '%s', port '%s'", hostBfr, servBfr); close(fd); fd = -1; } } if(fd != -1) - debug(D_CONNECT_TO, "Connected to '%s' on port '%s'.", hostBfr, servBfr); + netdata_log_debug(D_CONNECT_TO, "Connected to '%s' on port '%s'.", hostBfr, servBfr); } } @@ -915,17 +930,17 @@ int connect_to_this(const char *definition, int default_port, struct timeval *ti service = e; } - debug(D_CONNECT_TO, "Attempting connection to host = '%s', service = '%s', interface = '%s', protocol = %d (tcp = %d, udp = %d)", host, service, interface, protocol, IPPROTO_TCP, IPPROTO_UDP); + netdata_log_debug(D_CONNECT_TO, "Attempting connection to host = '%s', service = '%s', interface = '%s', protocol = %d (tcp = %d, udp = %d)", host, service, interface, protocol, IPPROTO_TCP, IPPROTO_UDP); if(!*host) { - error("Definition '%s' does not specify a host.", definition); + netdata_log_error("Definition '%s' does not specify a host.", definition); return -1; } if(*interface) { scope_id = if_nametoindex(interface); if(!scope_id) - error("Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface); + netdata_log_error("Cannot find a network interface named '%s'. Continuing with limiting the network interface", interface); } if(!*service) @@ -1110,7 +1125,7 @@ ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout) return netdata_ssl_write(ssl, buf, len); } else { - error("cannot write to SSL connection - connection is not ready."); + netdata_log_error("cannot write to SSL connection - connection is not ready."); return -1; } } @@ -1174,7 +1189,7 @@ int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) { int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list, const char *patname, int allow_dns) { - debug(D_LISTENER,"checking %s... (allow_dns=%d)", patname, allow_dns); + netdata_log_debug(D_LISTENER,"checking %s... (allow_dns=%d)", patname, allow_dns); if (!access_list) return 1; if (simple_pattern_matches(access_list, client_ip)) @@ -1189,7 +1204,7 @@ int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsi if (err != 0 || (err = getnameinfo((struct sockaddr *)&sadr, addrlen, client_host, (socklen_t)hostsize, NULL, 0, NI_NAMEREQD)) != 0) { - error("Incoming %s on '%s' does not match a numeric pattern, and host could not be resolved (err=%s)", + netdata_log_error("Incoming %s on '%s' does not match a numeric pattern, and host could not be resolved (err=%s)", patname, client_ip, gai_strerror(err)); if (hostsize >= 8) strcpy(client_host,"UNKNOWN"); @@ -1197,7 +1212,7 @@ int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsi } struct addrinfo *addr_infos = NULL; if (getaddrinfo(client_host, NULL, NULL, &addr_infos) !=0 ) { - error("LISTENER: cannot validate hostname '%s' from '%s' by resolving it", + netdata_log_error("LISTENER: cannot validate hostname '%s' from '%s' by resolving it", client_host, client_ip); if (hostsize >= 8) strcpy(client_host,"UNKNOWN"); @@ -1216,7 +1231,7 @@ int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsi inet_ntop(AF_INET6, &((struct sockaddr_in6*)(scan->ai_addr))->sin6_addr, address, INET6_ADDRSTRLEN); break; } - debug(D_LISTENER, "Incoming ip %s rev-resolved onto %s, validating against forward-resolution %s", + netdata_log_debug(D_LISTENER, "Incoming ip %s rev-resolved onto %s, validating against forward-resolution %s", client_ip, client_host, address); if (!strcmp(client_ip, address)) { validated = 1; @@ -1225,7 +1240,7 @@ int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsi scan = scan->ai_next; } if (!validated) { - error("LISTENER: Cannot validate '%s' as ip of '%s', not listed in DNS", client_ip, client_host); + netdata_log_error("LISTENER: Cannot validate '%s' as ip of '%s', not listed in DNS", client_ip, client_host); if (hostsize >= 8) strcpy(client_host,"UNKNOWN"); } @@ -1233,7 +1248,7 @@ int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsi freeaddrinfo(addr_infos); } if (!simple_pattern_matches(access_list, client_host)) { - debug(D_LISTENER, "Incoming connection on '%s' (%s) does not match allowed pattern for %s", + netdata_log_debug(D_LISTENER, "Incoming connection on '%s' (%s) does not match allowed pattern for %s", client_ip, client_host, patname); return 0; } @@ -1251,7 +1266,7 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien if (likely(nfd >= 0)) { if (getnameinfo((struct sockaddr *)&sadr, addrlen, client_ip, (socklen_t)ipsize, client_port, (socklen_t)portsize, NI_NUMERICHOST | NI_NUMERICSERV) != 0) { - error("LISTENER: cannot getnameinfo() on received client connection."); + netdata_log_error("LISTENER: cannot getnameinfo() on received client connection."); strncpyz(client_ip, "UNKNOWN", ipsize); strncpyz(client_port, "UNKNOWN", portsize); } @@ -1269,31 +1284,31 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien switch (((struct sockaddr *)&sadr)->sa_family) { case AF_UNIX: - debug(D_LISTENER, "New UNIX domain web client from %s on socket %d.", client_ip, fd); + netdata_log_debug(D_LISTENER, "New UNIX domain web client from %s on socket %d.", client_ip, fd); // set the port - certain versions of libc return garbage on unix sockets strncpyz(client_port, "UNIX", portsize); break; case AF_INET: - debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd); + netdata_log_debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd); break; case AF_INET6: if (strncmp(client_ip, "::ffff:", 7) == 0) { memmove(client_ip, &client_ip[7], strlen(&client_ip[7]) + 1); - debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd); + netdata_log_debug(D_LISTENER, "New IPv4 web client from %s port %s on socket %d.", client_ip, client_port, fd); } else - debug(D_LISTENER, "New IPv6 web client from %s port %s on socket %d.", client_ip, client_port, fd); + netdata_log_debug(D_LISTENER, "New IPv6 web client from %s port %s on socket %d.", client_ip, client_port, fd); break; default: - debug(D_LISTENER, "New UNKNOWN web client from %s port %s on socket %d.", client_ip, client_port, fd); + netdata_log_debug(D_LISTENER, "New UNKNOWN web client from %s port %s on socket %d.", client_ip, client_port, fd); break; } if (!connection_allowed(nfd, client_ip, client_host, hostsize, access_list, "connection", allow_dns)) { errno = 0; - error("Permission denied for client '%s', port '%s'", client_ip, client_port); + netdata_log_error("Permission denied for client '%s', port '%s'", client_ip, client_port); close(nfd); nfd = -1; errno = EPERM; @@ -1301,7 +1316,7 @@ int accept_socket(int fd, int flags, char *client_ip, size_t ipsize, char *clien } #ifdef HAVE_ACCEPT4 else if (errno == ENOSYS) - error("netdata has been compiled with the assumption that the system has the accept4() call, but it is not here. Recompile netdata like this: ./configure --disable-accept4 ..."); + netdata_log_error("netdata has been compiled with the assumption that the system has the accept4() call, but it is not here. Recompile netdata like this: ./configure --disable-accept4 ..."); #endif return nfd; @@ -1329,19 +1344,19 @@ inline POLLINFO *poll_add_fd(POLLJOB *p , int (*snd_callback)(POLLINFO * /*pi*/, short int * /*events*/) , void *data ) { - debug(D_POLLFD, "POLLFD: ADD: request to add fd %d, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", fd, p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + netdata_log_debug(D_POLLFD, "POLLFD: ADD: request to add fd %d, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", fd, p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); if(unlikely(fd < 0)) return NULL; //if(p->limit && p->used >= p->limit) { - // info("Max sockets limit reached (%zu sockets), dropping connection", p->used); + // netdata_log_info("Max sockets limit reached (%zu sockets), dropping connection", p->used); // close(fd); // return NULL; //} if(unlikely(!p->first_free)) { size_t new_slots = p->slots + POLL_FDS_INCREASE_STEP; - debug(D_POLLFD, "POLLFD: ADD: increasing size (current = %zu, new = %zu, used = %zu, min = %zu, max = %zu)", p->slots, new_slots, p->used, p->min, p->max); + netdata_log_debug(D_POLLFD, "POLLFD: ADD: increasing size (current = %zu, new = %zu, used = %zu, min = %zu, max = %zu)", p->slots, new_slots, p->used, p->min, p->max); p->fds = reallocz(p->fds, sizeof(struct pollfd) * new_slots); p->inf = reallocz(p->inf, sizeof(POLLINFO) * new_slots); @@ -1349,7 +1364,7 @@ inline POLLINFO *poll_add_fd(POLLJOB *p // reset all the newly added slots ssize_t i; for(i = new_slots - 1; i >= (ssize_t)p->slots ; i--) { - debug(D_POLLFD, "POLLFD: ADD: resetting new slot %zd", i); + netdata_log_debug(D_POLLFD, "POLLFD: ADD: resetting new slot %zd", i); p->fds[i].fd = -1; p->fds[i].events = 0; p->fds[i].revents = 0; @@ -1380,7 +1395,7 @@ inline POLLINFO *poll_add_fd(POLLJOB *p POLLINFO *pi = p->first_free; p->first_free = p->first_free->next; - debug(D_POLLFD, "POLLFD: ADD: selected slot %zu, next free is %zd", pi->slot, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + netdata_log_debug(D_POLLFD, "POLLFD: ADD: selected slot %zu, next free is %zd", pi->slot, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); struct pollfd *pf = &p->fds[pi->slot]; pf->fd = fd; @@ -1422,7 +1437,7 @@ inline POLLINFO *poll_add_fd(POLLJOB *p } netdata_thread_enable_cancelability(); - debug(D_POLLFD, "POLLFD: ADD: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + netdata_log_debug(D_POLLFD, "POLLFD: ADD: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); return pi; } @@ -1431,7 +1446,7 @@ inline void poll_close_fd(POLLINFO *pi) { POLLJOB *p = pi->p; struct pollfd *pf = &p->fds[pi->slot]; - debug(D_POLLFD, "POLLFD: DEL: request to clear slot %zu (fd %d), old next free was %zd", pi->slot, pf->fd, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + netdata_log_debug(D_POLLFD, "POLLFD: DEL: request to clear slot %zu (fd %d), old next free was %zd", pi->slot, pf->fd, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); if(unlikely(pf->fd == -1)) return; @@ -1442,7 +1457,7 @@ inline void poll_close_fd(POLLINFO *pi) { if(likely(!(pi->flags & POLLINFO_FLAG_DONT_CLOSE))) { if(close(pf->fd) == -1) - error("Failed to close() poll_events() socket %d", pf->fd); + netdata_log_error("Failed to close() poll_events() socket %d", pf->fd); } } @@ -1484,7 +1499,7 @@ inline void poll_close_fd(POLLINFO *pi) { } netdata_thread_enable_cancelability(); - debug(D_POLLFD, "POLLFD: DEL: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); + netdata_log_debug(D_POLLFD, "POLLFD: DEL: completed, slots = %zu, used = %zu, min = %zu, max = %zu, next free = %zd", p->slots, p->used, p->min, p->max, p->first_free?(ssize_t)p->first_free->slot:(ssize_t)-1); } void *poll_default_add_callback(POLLINFO *pi, short int *events, void *data) { @@ -1492,14 +1507,14 @@ void *poll_default_add_callback(POLLINFO *pi, short int *events, void *data) { (void)events; (void)data; - // error("POLLFD: internal error: poll_default_add_callback() called"); + // netdata_log_error("POLLFD: internal error: poll_default_add_callback() called"); return NULL; } void poll_default_del_callback(POLLINFO *pi) { if(pi->data) - error("POLLFD: internal error: del_callback_default() called with data pointer - possible memory leak"); + netdata_log_error("POLLFD: internal error: del_callback_default() called with data pointer - possible memory leak"); } int poll_default_rcv_callback(POLLINFO *pi, short int *events) { @@ -1513,12 +1528,12 @@ int poll_default_rcv_callback(POLLINFO *pi, short int *events) { if (rc < 0) { // read failed if (errno != EWOULDBLOCK && errno != EAGAIN) { - error("POLLFD: poll_default_rcv_callback(): recv() failed with %zd.", rc); + netdata_log_error("POLLFD: poll_default_rcv_callback(): recv() failed with %zd.", rc); return -1; } } else if (rc) { // data received - info("POLLFD: internal error: poll_default_rcv_callback() is discarding %zd bytes received on socket %d", rc, pi->fd); + netdata_log_info("POLLFD: internal error: poll_default_rcv_callback() is discarding %zd bytes received on socket %d", rc, pi->fd); } } while (rc != -1); @@ -1528,7 +1543,7 @@ int poll_default_rcv_callback(POLLINFO *pi, short int *events) { int poll_default_snd_callback(POLLINFO *pi, short int *events) { *events &= ~POLLOUT; - info("POLLFD: internal error: poll_default_snd_callback(): nothing to send on socket %d", pi->fd); + netdata_log_info("POLLFD: internal error: poll_default_snd_callback(): nothing to send on socket %d", pi->fd); return 0; } @@ -1550,7 +1565,7 @@ static void poll_events_cleanup(void *data) { } static int poll_process_error(POLLINFO *pi, struct pollfd *pf, short int revents) { - error("POLLFD: LISTENER: received %s %s %s on socket at slot %zu (fd %d) client '%s' port '%s' expecting %s %s %s, having %s %s %s" + netdata_log_error("POLLFD: LISTENER: received %s %s %s on socket at slot %zu (fd %d) client '%s' port '%s' expecting %s %s %s, having %s %s %s" , revents & POLLERR ? "POLLERR" : "" , revents & POLLHUP ? "POLLHUP" : "" , revents & POLLNVAL ? "POLLNVAL" : "" @@ -1571,7 +1586,7 @@ static inline int poll_process_send(POLLJOB *p, POLLINFO *pi, struct pollfd *pf, pi->last_sent_t = now; pi->send_count++; - debug(D_POLLFD, "POLLFD: LISTENER: sending data to socket on slot %zu (fd %d)", pi->slot, pf->fd); + netdata_log_debug(D_POLLFD, "POLLFD: LISTENER: sending data to socket on slot %zu (fd %d)", pi->slot, pf->fd); pf->events = 0; @@ -1592,7 +1607,7 @@ static inline int poll_process_tcp_read(POLLJOB *p, POLLINFO *pi, struct pollfd pi->last_received_t = now; pi->recv_count++; - debug(D_POLLFD, "POLLFD: LISTENER: reading data from TCP client slot %zu (fd %d)", pi->slot, pf->fd); + netdata_log_debug(D_POLLFD, "POLLFD: LISTENER: reading data from TCP client slot %zu (fd %d)", pi->slot, pf->fd); pf->events = 0; @@ -1613,7 +1628,7 @@ static inline int poll_process_udp_read(POLLINFO *pi, struct pollfd *pf, time_t pi->last_received_t = now; pi->recv_count++; - debug(D_POLLFD, "POLLFD: LISTENER: reading data from UDP slot %zu (fd %d)", pi->slot, pf->fd); + netdata_log_debug(D_POLLFD, "POLLFD: LISTENER: reading data from UDP slot %zu (fd %d)", pi->slot, pf->fd); // TODO: access_list is not applied to UDP // but checking the access list on every UDP packet will destroy @@ -1633,13 +1648,13 @@ static int poll_process_new_tcp_connection(POLLJOB *p, POLLINFO *pi, struct poll pi->last_received_t = now; pi->recv_count++; - debug(D_POLLFD, "POLLFD: LISTENER: accepting connections from slot %zu (fd %d)", pi->slot, pf->fd); + netdata_log_debug(D_POLLFD, "POLLFD: LISTENER: accepting connections from slot %zu (fd %d)", pi->slot, pf->fd); char client_ip[INET6_ADDRSTRLEN] = ""; char client_port[NI_MAXSERV] = ""; char client_host[NI_MAXHOST] = ""; - debug(D_POLLFD, "POLLFD: LISTENER: calling accept4() slot %zu (fd %d)", pi->slot, pf->fd); + netdata_log_debug(D_POLLFD, "POLLFD: LISTENER: calling accept4() slot %zu (fd %d)", pi->slot, pf->fd); int nfd = accept_socket( pf->fd,SOCK_NONBLOCK, @@ -1650,7 +1665,7 @@ static int poll_process_new_tcp_connection(POLLJOB *p, POLLINFO *pi, struct poll if (unlikely(nfd < 0)) { // accept failed - debug(D_POLLFD, "POLLFD: LISTENER: accept4() slot %zu (fd %d) failed.", pi->slot, pf->fd); + netdata_log_debug(D_POLLFD, "POLLFD: LISTENER: accept4() slot %zu (fd %d) failed.", pi->slot, pf->fd); if(unlikely(errno == EMFILE)) { error_limit_static_global_var(erl, 10, 1000); @@ -1658,7 +1673,7 @@ static int poll_process_new_tcp_connection(POLLJOB *p, POLLINFO *pi, struct poll p->used, p->limit); } else if(unlikely(errno != EWOULDBLOCK && errno != EAGAIN)) - error("POLLFD: LISTENER: accept() failed."); + netdata_log_error("POLLFD: LISTENER: accept() failed."); } else { @@ -1705,7 +1720,7 @@ void poll_events(LISTEN_SOCKETS *sockets , size_t max_tcp_sockets ) { if(!sockets || !sockets->opened) { - error("POLLFD: internal error: no listening sockets are opened"); + netdata_log_error("POLLFD: internal error: no listening sockets are opened"); return; } @@ -1758,7 +1773,7 @@ void poll_events(LISTEN_SOCKETS *sockets ); pi->data = data; - info("POLLFD: LISTENER: listening on '%s'", (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN"); + netdata_log_info("POLLFD: LISTENER: listening on '%s'", (sockets->fds_names[i])?sockets->fds_names[i]:"UNKNOWN"); } int listen_sockets_active = 1; @@ -1782,7 +1797,7 @@ void poll_events(LISTEN_SOCKETS *sockets now_usec = now_boottime_usec(); if(unlikely(timer_usec && now_usec >= next_timer_usec)) { - debug(D_POLLFD, "Calling timer callback after %zu usec", (size_t)(now_usec - last_timer_usec)); + netdata_log_debug(D_POLLFD, "Calling timer callback after %zu usec", (size_t)(now_usec - last_timer_usec)); last_timer_usec = now_usec; p.tmr_callback(p.timer_data); now_usec = now_boottime_usec(); @@ -1799,7 +1814,7 @@ void poll_events(LISTEN_SOCKETS *sockets // enable or disable the TCP listening sockets, based on the current number of sockets used and the limit set if((listen_sockets_active && (p.limit && p.used >= p.limit)) || (!listen_sockets_active && (!p.limit || p.used < p.limit))) { listen_sockets_active = !listen_sockets_active; - info("%s listening sockets (used TCP sockets %zu, max allowed for this worker %zu)", (listen_sockets_active)?"ENABLING":"DISABLING", p.used, p.limit); + netdata_log_info("%s listening sockets (used TCP sockets %zu, max allowed for this worker %zu)", (listen_sockets_active)?"ENABLING":"DISABLING", p.used, p.limit); for (i = 0; i <= p.max; i++) { if(p.inf[i].flags & POLLINFO_FLAG_SERVER_SOCKET && p.inf[i].socktype == SOCK_STREAM) { p.fds[i].events = (short int) ((listen_sockets_active) ? POLLIN : 0); @@ -1807,16 +1822,16 @@ void poll_events(LISTEN_SOCKETS *sockets } } - debug(D_POLLFD, "POLLFD: LISTENER: Waiting on %zu sockets for %zu ms...", p.max + 1, (size_t)timeout_ms); + netdata_log_debug(D_POLLFD, "POLLFD: LISTENER: Waiting on %zu sockets for %zu ms...", p.max + 1, (size_t)timeout_ms); retval = poll(p.fds, p.max + 1, timeout_ms); time_t now = now_boottime_sec(); if(unlikely(retval == -1)) { - error("POLLFD: LISTENER: poll() failed while waiting on %zu sockets.", p.max + 1); + netdata_log_error("POLLFD: LISTENER: poll() failed while waiting on %zu sockets.", p.max + 1); break; } else if(unlikely(!retval)) { - debug(D_POLLFD, "POLLFD: LISTENER: poll() timeout."); + netdata_log_debug(D_POLLFD, "POLLFD: LISTENER: poll() timeout."); } else { POLLINFO *pi; @@ -1870,7 +1885,7 @@ void poll_events(LISTEN_SOCKETS *sockets conns[conns_max++] = i; } else - error("POLLFD: LISTENER: server slot %zu (fd %d) connection from %s port %s using unhandled socket type %d." + netdata_log_error("POLLFD: LISTENER: server slot %zu (fd %d) connection from %s port %s using unhandled socket type %d." , i , pi->fd , pi->client_ip ? pi->client_ip : "<undefined-ip>" @@ -1879,7 +1894,7 @@ void poll_events(LISTEN_SOCKETS *sockets ); } else - error("POLLFD: LISTENER: client slot %zu (fd %d) data from %s port %s using flags %08X is neither client nor server." + netdata_log_error("POLLFD: LISTENER: client slot %zu (fd %d) data from %s port %s using flags %08X is neither client nor server." , i , pi->fd , pi->client_ip ? pi->client_ip : "<undefined-ip>" @@ -1888,7 +1903,7 @@ void poll_events(LISTEN_SOCKETS *sockets ); } else - error("POLLFD: LISTENER: socket slot %zu (fd %d) client %s port %s unhandled event id %d." + netdata_log_error("POLLFD: LISTENER: socket slot %zu (fd %d) client %s port %s unhandled event id %d." , i , pi->fd , pi->client_ip ? pi->client_ip : "<undefined-ip>" @@ -1947,7 +1962,7 @@ void poll_events(LISTEN_SOCKETS *sockets if(likely(pi->flags & POLLINFO_FLAG_CLIENT_SOCKET)) { if (unlikely(pi->send_count == 0 && p.complete_request_timeout > 0 && (now - pi->connected_t) >= p.complete_request_timeout)) { - info("POLLFD: LISTENER: client slot %zu (fd %d) from %s port %s has not sent a complete request in %zu seconds - closing it. " + netdata_log_info("POLLFD: LISTENER: client slot %zu (fd %d) from %s port %s has not sent a complete request in %zu seconds - closing it. " , i , pi->fd , pi->client_ip ? pi->client_ip : "<undefined-ip>" @@ -1957,7 +1972,7 @@ void poll_events(LISTEN_SOCKETS *sockets poll_close_fd(pi); } else if(unlikely(pi->recv_count && p.idle_timeout > 0 && now - ((pi->last_received_t > pi->last_sent_t) ? pi->last_received_t : pi->last_sent_t) >= p.idle_timeout )) { - info("POLLFD: LISTENER: client slot %zu (fd %d) from %s port %s is idle for more than %zu seconds - closing it. " + netdata_log_info("POLLFD: LISTENER: client slot %zu (fd %d) from %s port %s is idle for more than %zu seconds - closing it. " , i , pi->fd , pi->client_ip ? pi->client_ip : "<undefined-ip>" @@ -1972,5 +1987,5 @@ void poll_events(LISTEN_SOCKETS *sockets } netdata_thread_cleanup_pop(1); - debug(D_POLLFD, "POLLFD: LISTENER: cleanup completed"); + netdata_log_debug(D_POLLFD, "POLLFD: LISTENER: cleanup completed"); } diff --git a/libnetdata/socket/socket.h b/libnetdata/socket/socket.h index 0e29711e0..8331ecbbe 100644 --- a/libnetdata/socket/socket.h +++ b/libnetdata/socket/socket.h @@ -11,7 +11,7 @@ typedef enum web_client_acl { WEB_CLIENT_ACL_NONE = (0), - WEB_CLIENT_ACL_NOCHECK = (0), + WEB_CLIENT_ACL_NOCHECK = (0), // Don't check anything - this should work on all channels WEB_CLIENT_ACL_DASHBOARD = (1 << 0), WEB_CLIENT_ACL_REGISTRY = (1 << 1), WEB_CLIENT_ACL_BADGE = (1 << 2), @@ -23,9 +23,18 @@ typedef enum web_client_acl { WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 8), WEB_CLIENT_ACL_ACLK = (1 << 9), WEB_CLIENT_ACL_WEBRTC = (1 << 10), + WEB_CLIENT_ACL_BEARER_OPTIONAL = (1 << 11), // allow unprotected access if bearer is not enabled in netdata + WEB_CLIENT_ACL_BEARER_REQUIRED = (1 << 12), // allow access only if a valid bearer is used } WEB_CLIENT_ACL; -#define WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC) +#define WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_OPTIONAL) +#define WEB_CLIENT_ACL_ACLK_WEBRTC_DASHBOARD_WITH_BEARER (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC | WEB_CLIENT_ACL_BEARER_REQUIRED) + +#ifdef NETDATA_DEV_MODE +#define ACL_DEV_OPEN_ACCESS WEB_CLIENT_ACL_DASHBOARD +#else +#define ACL_DEV_OPEN_ACCESS 0 +#endif #define WEB_CLIENT_ACL_ALL 0xFFFF diff --git a/libnetdata/storage_number/storage_number.c b/libnetdata/storage_number/storage_number.c index ebae71d81..6468951bd 100644 --- a/libnetdata/storage_number/storage_number.c +++ b/libnetdata/storage_number/storage_number.c @@ -52,7 +52,7 @@ bool is_system_ieee754_double(void) { if(*ptr != tests[i].i && (tests[i].original == tests[i].d || (isnan(tests[i].original) && isnan(tests[i].d)))) { if(!logged) - info("IEEE754: test #%zu, value " NETDATA_DOUBLE_FORMAT_G " is represented in this system as %lX, but it was expected as %lX", + netdata_log_info("IEEE754: test #%zu, value " NETDATA_DOUBLE_FORMAT_G " is represented in this system as %lX, but it was expected as %lX", i+1, tests[i].original, *ptr, tests[i].i); errors++; } @@ -60,14 +60,14 @@ bool is_system_ieee754_double(void) { if(!errors && sizeof(NETDATA_DOUBLE) == sizeof(uint64_t)) { if(!logged) - info("IEEE754: system is using IEEE754 DOUBLE PRECISION values"); + netdata_log_info("IEEE754: system is using IEEE754 DOUBLE PRECISION values"); logged = true; return true; } else { if(!logged) - info("IEEE754: system is NOT compatible with IEEE754 DOUBLE PRECISION values"); + netdata_log_info("IEEE754: system is NOT compatible with IEEE754 DOUBLE PRECISION values"); logged = true; return false; @@ -121,7 +121,7 @@ storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) { if(n > (NETDATA_DOUBLE)0x00ffffff) { #ifdef NETDATA_INTERNAL_CHECKS - error("Number " NETDATA_DOUBLE_FORMAT " is too big.", value); + netdata_log_error("Number " NETDATA_DOUBLE_FORMAT " is too big.", value); #endif r += 0x00ffffff; return r; diff --git a/libnetdata/string/string.c b/libnetdata/string/string.c index 9385aa6e8..373d0c24c 100644 --- a/libnetdata/string/string.c +++ b/libnetdata/string/string.c @@ -8,6 +8,11 @@ typedef int32_t REFCOUNT; // ---------------------------------------------------------------------------- // STRING implementation - dedup all STRING +#define STRING_PARTITION_SHIFTS (0) +#define STRING_PARTITIONS (256 >> STRING_PARTITION_SHIFTS) +#define string_partition_str(str) ((uint8_t)((str)[0]) >> STRING_PARTITION_SHIFTS) +#define string_partition(string) (string_partition_str((string)->str)) + struct netdata_string { uint32_t length; // the string length including the terminating '\0' @@ -18,20 +23,22 @@ struct netdata_string { const char str[]; // the string itself, is appended to this structure }; -static struct string_hashtable { - Pvoid_t JudyHSArray; // the Judy array - hashtable - netdata_rwlock_t rwlock; // the R/W lock to protect the Judy array +static struct string_partition { + RW_SPINLOCK spinlock; // the R/W spinlock to protect the Judy array - long int entries; // the number of entries in the index - long int active_references; // the number of active references alive - long int memory; // the memory used, without the JudyHS index + Pvoid_t JudyHSArray; // the Judy array - hashtable - size_t inserts; // the number of successful inserts to the index - size_t deletes; // the number of successful deleted from the index size_t searches; // the number of successful searches in the index size_t duplications; // when a string is referenced size_t releases; // when a string is unreferenced + size_t inserts; // the number of successful inserts to the index + size_t deletes; // the number of successful deleted from the index + + long int entries; // the number of entries in the index + long int active_references; // the number of active references alive + long int memory; // the memory used, without the JudyHS index + #ifdef NETDATA_INTERNAL_CHECKS // internal statistics size_t found_deleted_on_search; @@ -41,50 +48,45 @@ static struct string_hashtable { size_t spins; #endif -} string_base = { - .JudyHSArray = NULL, - .rwlock = NETDATA_RWLOCK_INITIALIZER, -}; +} string_base[STRING_PARTITIONS] = { 0 }; #ifdef NETDATA_INTERNAL_CHECKS -#define string_internal_stats_add(var, val) __atomic_add_fetch(&string_base.var, val, __ATOMIC_RELAXED) +#define string_internal_stats_add(partition, var, val) __atomic_add_fetch(&string_base[partition].var, val, __ATOMIC_RELAXED) #else -#define string_internal_stats_add(var, val) do {;} while(0) +#define string_internal_stats_add(partition, var, val) do {;} while(0) #endif -#define string_stats_atomic_increment(var) __atomic_add_fetch(&string_base.var, 1, __ATOMIC_RELAXED) -#define string_stats_atomic_decrement(var) __atomic_sub_fetch(&string_base.var, 1, __ATOMIC_RELAXED) +#define string_stats_atomic_increment(partition, var) __atomic_add_fetch(&string_base[partition].var, 1, __ATOMIC_RELAXED) +#define string_stats_atomic_decrement(partition, var) __atomic_sub_fetch(&string_base[partition].var, 1, __ATOMIC_RELAXED) void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases) { - if(inserts) - *inserts = string_base.inserts; - - if(deletes) - *deletes = string_base.deletes; - - if(searches) - *searches = string_base.searches; - - if(entries) - *entries = (size_t)string_base.entries; - - if(references) - *references = (size_t)string_base.active_references; - - if(memory) - *memory = (size_t)string_base.memory; - - if(duplications) - *duplications = string_base.duplications; - - if(releases) - *releases = string_base.releases; + if (inserts) *inserts = 0; + if (deletes) *deletes = 0; + if (searches) *searches = 0; + if (entries) *entries = 0; + if (references) *references = 0; + if (memory) *memory = 0; + if (duplications) *duplications = 0; + if (releases) *releases = 0; + + for(size_t i = 0; i < STRING_PARTITIONS ;i++) { + if (inserts) *inserts += string_base[i].inserts; + if (deletes) *deletes += string_base[i].deletes; + if (searches) *searches += string_base[i].searches; + if (entries) *entries += (size_t) string_base[i].entries; + if (references) *references += (size_t) string_base[i].active_references; + if (memory) *memory += (size_t) string_base[i].memory; + if (duplications) *duplications += string_base[i].duplications; + if (releases) *releases += string_base[i].releases; + } } #define string_entry_acquire(se) __atomic_add_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST); #define string_entry_release(se) __atomic_sub_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST); static inline bool string_entry_check_and_acquire(STRING *se) { + uint8_t partition = string_partition(se); + REFCOUNT expected, desired, count = 0; expected = __atomic_load_n(&se->refcount, __ATOMIC_SEQ_CST); @@ -96,7 +98,7 @@ static inline bool string_entry_check_and_acquire(STRING *se) { // We cannot use this. // The reference counter reached value zero, // so another thread is deleting this. - string_internal_stats_add(spins, count - 1); + string_internal_stats_add(partition, spins, count - 1); return false; } @@ -104,11 +106,11 @@ static inline bool string_entry_check_and_acquire(STRING *se) { } while(!__atomic_compare_exchange_n(&se->refcount, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)); - string_internal_stats_add(spins, count - 1); + string_internal_stats_add(partition, spins, count - 1); // statistics // string_base.active_references is altered at the in string_strdupz() and string_freez() - string_stats_atomic_increment(duplications); + string_stats_atomic_increment(partition, duplications); return true; } @@ -123,9 +125,11 @@ STRING *string_dup(STRING *string) { string_entry_acquire(string); + uint8_t partition = string_partition(string); + // statistics - string_stats_atomic_increment(active_references); - string_stats_atomic_increment(duplications); + string_stats_atomic_increment(partition, active_references); + string_stats_atomic_increment(partition, duplications); return string; } @@ -134,26 +138,28 @@ STRING *string_dup(STRING *string) { static inline STRING *string_index_search(const char *str, size_t length) { STRING *string; + uint8_t partition = string_partition_str(str); + // Find the string in the index // With a read-lock so that multiple readers can use the index concurrently. - netdata_rwlock_rdlock(&string_base.rwlock); + rw_spinlock_read_lock(&string_base[partition].spinlock); Pvoid_t *Rc; - Rc = JudyHSGet(string_base.JudyHSArray, (void *)str, length); + Rc = JudyHSGet(string_base[partition].JudyHSArray, (void *)str, length - 1); if(likely(Rc)) { // found in the hash table string = *Rc; if(string_entry_check_and_acquire(string)) { // we can use this entry - string_internal_stats_add(found_available_on_search, 1); + string_internal_stats_add(partition, found_available_on_search, 1); } else { // this entry is about to be deleted by another thread // do not touch it, let it go... string = NULL; - string_internal_stats_add(found_deleted_on_search, 1); + string_internal_stats_add(partition, found_deleted_on_search, 1); } } else { @@ -161,8 +167,8 @@ static inline STRING *string_index_search(const char *str, size_t length) { string = NULL; } - string_stats_atomic_increment(searches); - netdata_rwlock_unlock(&string_base.rwlock); + string_stats_atomic_increment(partition, searches); + rw_spinlock_read_unlock(&string_base[partition].spinlock); return string; } @@ -175,12 +181,14 @@ static inline STRING *string_index_search(const char *str, size_t length) { static inline STRING *string_index_insert(const char *str, size_t length) { STRING *string; - netdata_rwlock_wrlock(&string_base.rwlock); + uint8_t partition = string_partition_str(str); + + rw_spinlock_write_lock(&string_base[partition].spinlock); STRING **ptr; { JError_t J_Error; - Pvoid_t *Rc = JudyHSIns(&string_base.JudyHSArray, (void *)str, length, &J_Error); + Pvoid_t *Rc = JudyHSIns(&string_base[partition].JudyHSArray, (void *)str, length - 1, &J_Error); if (unlikely(Rc == PJERR)) { fatal( "STRING: Cannot insert entry with name '%s' to JudyHS, JU_ERRNO_* == %u, ID == %d", @@ -199,9 +207,9 @@ static inline STRING *string_index_insert(const char *str, size_t length) { string->length = length; string->refcount = 1; *ptr = string; - string_base.inserts++; - string_base.entries++; - string_base.memory += (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(length)); + string_base[partition].inserts++; + string_base[partition].entries++; + string_base[partition].memory += (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(length)); } else { // the item is already in the index @@ -209,25 +217,27 @@ static inline STRING *string_index_insert(const char *str, size_t length) { if(string_entry_check_and_acquire(string)) { // we can use this entry - string_internal_stats_add(found_available_on_insert, 1); + string_internal_stats_add(partition, found_available_on_insert, 1); } else { // this entry is about to be deleted by another thread // do not touch it, let it go... string = NULL; - string_internal_stats_add(found_deleted_on_insert, 1); + string_internal_stats_add(partition, found_deleted_on_insert, 1); } - string_stats_atomic_increment(searches); + string_stats_atomic_increment(partition, searches); } - netdata_rwlock_unlock(&string_base.rwlock); + rw_spinlock_write_unlock(&string_base[partition].spinlock); return string; } // delete an entry from the index static inline void string_index_delete(STRING *string) { - netdata_rwlock_wrlock(&string_base.rwlock); + uint8_t partition = string_partition(string); + + rw_spinlock_write_lock(&string_base[partition].spinlock); #ifdef NETDATA_INTERNAL_CHECKS if(unlikely(__atomic_load_n(&string->refcount, __ATOMIC_SEQ_CST) != 0)) @@ -236,11 +246,11 @@ static inline void string_index_delete(STRING *string) { bool deleted = false; - if (likely(string_base.JudyHSArray)) { + if (likely(string_base[partition].JudyHSArray)) { JError_t J_Error; - int ret = JudyHSDel(&string_base.JudyHSArray, (void *)string->str, string->length, &J_Error); + int ret = JudyHSDel(&string_base[partition].JudyHSArray, (void *)string->str, string->length - 1, &J_Error); if (unlikely(ret == JERR)) { - error( + netdata_log_error( "STRING: Cannot delete entry with name '%s' from JudyHS, JU_ERRNO_* == %u, ID == %d", string->str, JU_ERRNO(&J_Error), @@ -250,21 +260,23 @@ static inline void string_index_delete(STRING *string) { } if (unlikely(!deleted)) - error("STRING: tried to delete '%s' that is not in the index. Ignoring it.", string->str); + netdata_log_error("STRING: tried to delete '%s' that is not in the index. Ignoring it.", string->str); else { size_t mem_size = sizeof(STRING) + string->length; - string_base.deletes++; - string_base.entries--; - string_base.memory -= (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(string->length)); + string_base[partition].deletes++; + string_base[partition].entries--; + string_base[partition].memory -= (long)(mem_size + JUDYHS_INDEX_SIZE_ESTIMATE(string->length)); freez(string); } - netdata_rwlock_unlock(&string_base.rwlock); + rw_spinlock_write_unlock(&string_base[partition].spinlock); } STRING *string_strdupz(const char *str) { if(unlikely(!str || !*str)) return NULL; + uint8_t partition = string_partition_str(str); + size_t length = strlen(str) + 1; STRING *string = string_index_search(str, length); @@ -277,7 +289,7 @@ STRING *string_strdupz(const char *str) { } // statistics - string_stats_atomic_increment(active_references); + string_stats_atomic_increment(partition, active_references); return string; } @@ -285,6 +297,7 @@ STRING *string_strdupz(const char *str) { void string_freez(STRING *string) { if(unlikely(!string)) return; + uint8_t partition = string_partition(string); REFCOUNT refcount = string_entry_release(string); #ifdef NETDATA_INTERNAL_CHECKS @@ -296,8 +309,8 @@ void string_freez(STRING *string) { string_index_delete(string); // statistics - string_stats_atomic_decrement(active_references); - string_stats_atomic_increment(releases); + string_stats_atomic_decrement(partition, active_references); + string_stats_atomic_increment(partition, releases); } inline size_t string_strlen(STRING *string) { @@ -405,6 +418,54 @@ static void string_unittest_free_char_pp(char **pp, size_t entries) { freez(pp); } +static long unittest_string_entries(void) { + long entries = 0; + for(size_t p = 0; p < STRING_PARTITIONS ;p++) + entries += string_base[p].entries; + + return entries; +} + +#ifdef NETDATA_INTERNAL_CHECKS + +static size_t unittest_string_found_deleted_on_search(void) { + size_t entries = 0; + for(size_t p = 0; p < STRING_PARTITIONS ;p++) + entries += string_base[p].found_deleted_on_search; + + return entries; +} +static size_t unittest_string_found_available_on_search(void) { + size_t entries = 0; + for(size_t p = 0; p < STRING_PARTITIONS ;p++) + entries += string_base[p].found_available_on_search; + + return entries; +} +static size_t unittest_string_found_deleted_on_insert(void) { + size_t entries = 0; + for(size_t p = 0; p < STRING_PARTITIONS ;p++) + entries += string_base[p].found_deleted_on_insert; + + return entries; +} +static size_t unittest_string_found_available_on_insert(void) { + size_t entries = 0; + for(size_t p = 0; p < STRING_PARTITIONS ;p++) + entries += string_base[p].found_available_on_insert; + + return entries; +} +static size_t unittest_string_spins(void) { + size_t entries = 0; + for(size_t p = 0; p < STRING_PARTITIONS ;p++) + entries += string_base[p].spins; + + return entries; +} + +#endif // NETDATA_INTERNAL_CHECKS + int string_unittest(size_t entries) { size_t errors = 0; @@ -413,7 +474,7 @@ int string_unittest(size_t entries) { // check string { - long int string_entries_starting = string_base.entries; + long entries_starting = unittest_string_entries(); fprintf(stderr, "\nChecking strings...\n"); @@ -496,9 +557,10 @@ int string_unittest(size_t entries) { freez(strings); - if(string_base.entries != string_entries_starting + 2) { + if(unittest_string_entries() != entries_starting + 2) { errors++; - fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n", string_entries_starting + 2, string_base.entries); + fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n", + entries_starting + 2, unittest_string_entries()); } else fprintf(stderr, "OK: strings dictionary has 2 items\n"); @@ -551,11 +613,11 @@ int string_unittest(size_t entries) { }; #ifdef NETDATA_INTERNAL_CHECKS - size_t ofound_deleted_on_search = string_base.found_deleted_on_search, - ofound_available_on_search = string_base.found_available_on_search, - ofound_deleted_on_insert = string_base.found_deleted_on_insert, - ofound_available_on_insert = string_base.found_available_on_insert, - ospins = string_base.spins; + size_t ofound_deleted_on_search = unittest_string_found_deleted_on_search(), + ofound_available_on_search = unittest_string_found_available_on_search(), + ofound_deleted_on_insert = unittest_string_found_deleted_on_insert(), + ofound_available_on_insert = unittest_string_found_available_on_insert(), + ospins = unittest_string_spins(); #endif size_t oinserts, odeletes, osearches, oentries, oreferences, omemory, oduplications, oreleases; @@ -592,11 +654,11 @@ int string_unittest(size_t entries) { inserts - oinserts, deletes - odeletes, searches - osearches, sentries - oentries, references - oreferences, memory - omemory, duplications - oduplications, releases - oreleases); #ifdef NETDATA_INTERNAL_CHECKS - size_t found_deleted_on_search = string_base.found_deleted_on_search, - found_available_on_search = string_base.found_available_on_search, - found_deleted_on_insert = string_base.found_deleted_on_insert, - found_available_on_insert = string_base.found_available_on_insert, - spins = string_base.spins; + size_t found_deleted_on_search = unittest_string_found_deleted_on_search(), + found_available_on_search = unittest_string_found_available_on_search(), + found_deleted_on_insert = unittest_string_found_deleted_on_insert(), + found_available_on_insert = unittest_string_found_available_on_insert(), + spins = unittest_string_spins(); fprintf(stderr, "on insert: %zu ok + %zu deleted\non search: %zu ok + %zu deleted\nspins: %zu\n", found_available_on_insert - ofound_available_on_insert, diff --git a/libnetdata/threads/threads.c b/libnetdata/threads/threads.c index a4591d5ac..adce0463c 100644 --- a/libnetdata/threads/threads.c +++ b/libnetdata/threads/threads.c @@ -133,7 +133,7 @@ size_t netdata_threads_init(void) { if(i != 0) fatal("pthread_attr_getstacksize() failed with code %d.", i); else - debug(D_OPTIONS, "initial pthread stack size is %zu bytes", stacksize); + netdata_log_debug(D_OPTIONS, "initial pthread stack size is %zu bytes", stacksize); return stacksize; } @@ -150,12 +150,12 @@ void netdata_threads_init_after_fork(size_t stacksize) { if(netdata_threads_attr && stacksize > (size_t)PTHREAD_STACK_MIN) { i = pthread_attr_setstacksize(netdata_threads_attr, stacksize); if(i != 0) - error("pthread_attr_setstacksize() to %zu bytes, failed with code %d.", stacksize, i); + netdata_log_error("pthread_attr_setstacksize() to %zu bytes, failed with code %d.", stacksize, i); else - info("Set threads stack size to %zu bytes", stacksize); + netdata_log_info("Set threads stack size to %zu bytes", stacksize); } else - error("Invalid pthread stacksize %zu", stacksize); + netdata_log_error("Invalid pthread stacksize %zu", stacksize); } // ---------------------------------------------------------------------------- @@ -169,11 +169,11 @@ void service_exits(void); static void thread_cleanup(void *ptr) { if(netdata_thread != ptr) { NETDATA_THREAD *info = (NETDATA_THREAD *)ptr; - error("THREADS: internal error - thread local variable does not match the one passed to this function. Expected thread '%s', passed thread '%s'", netdata_thread->tag, info->tag); + netdata_log_error("THREADS: internal error - thread local variable does not match the one passed to this function. Expected thread '%s', passed thread '%s'", netdata_thread->tag, info->tag); } if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP)) - info("thread with task id %d finished", gettid()); + netdata_log_info("thread with task id %d finished", gettid()); sender_thread_buffer_free(); rrdset_thread_rda_free(); @@ -205,9 +205,9 @@ static void thread_set_name_np(NETDATA_THREAD *nt) { #endif if (ret != 0) - error("cannot set pthread name of %d to %s. ErrCode: %d", gettid(), threadname, ret); + netdata_log_error("cannot set pthread name of %d to %s. ErrCode: %d", gettid(), threadname, ret); else - info("set name of thread %d to %s", gettid(), threadname); + netdata_log_info("set name of thread %d to %s", gettid(), threadname); } } @@ -230,7 +230,7 @@ void uv_thread_set_name_np(uv_thread_t ut, const char* name) { thread_name_get(true); if (ret) - info("cannot set libuv thread name to %s. Err: %d", threadname, ret); + netdata_log_info("cannot set libuv thread name to %s. Err: %d", threadname, ret); } void os_thread_get_current_name_np(char threadname[NETDATA_THREAD_NAME_MAX + 1]) @@ -247,13 +247,13 @@ static void *netdata_thread_init(void *ptr) { netdata_thread = (NETDATA_THREAD *)ptr; if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_STARTUP)) - info("thread created with task id %d", gettid()); + netdata_log_info("thread created with task id %d", gettid()); if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) - error("cannot set pthread cancel type to DEFERRED."); + netdata_log_error("cannot set pthread cancel type to DEFERRED."); if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) - error("cannot set pthread cancel state to ENABLE."); + netdata_log_error("cannot set pthread cancel state to ENABLE."); thread_set_name_np(ptr); @@ -275,13 +275,13 @@ int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THR int ret = pthread_create(thread, netdata_threads_attr, netdata_thread_init, info); if(ret != 0) - error("failed to create new thread for %s. pthread_create() failed with code %d", tag, ret); + netdata_log_error("failed to create new thread for %s. pthread_create() failed with code %d", tag, ret); else { if (!(options & NETDATA_THREAD_OPTION_JOINABLE)) { int ret2 = pthread_detach(*thread); if (ret2 != 0) - error("cannot request detach of newly created %s thread. pthread_detach() failed with code %d", tag, ret2); + netdata_log_error("cannot request detach of newly created %s thread. pthread_detach() failed with code %d", tag, ret2); } } @@ -298,9 +298,9 @@ int netdata_thread_cancel(netdata_thread_t thread) { int ret = pthread_cancel(thread); if(ret != 0) #ifdef NETDATA_INTERNAL_CHECKS - error("cannot cancel thread. pthread_cancel() failed with code %d at %d@%s, function %s()", ret, line, file, function); + netdata_log_error("cannot cancel thread. pthread_cancel() failed with code %d at %d@%s, function %s()", ret, line, file, function); #else - error("cannot cancel thread. pthread_cancel() failed with code %d.", ret); + netdata_log_error("cannot cancel thread. pthread_cancel() failed with code %d.", ret); #endif return ret; @@ -312,7 +312,7 @@ int netdata_thread_cancel(netdata_thread_t thread) { int netdata_thread_join(netdata_thread_t thread, void **retval) { int ret = pthread_join(thread, retval); if(ret != 0) - error("cannot join thread. pthread_join() failed with code %d.", ret); + netdata_log_error("cannot join thread. pthread_join() failed with code %d.", ret); return ret; } @@ -320,7 +320,7 @@ int netdata_thread_join(netdata_thread_t thread, void **retval) { int netdata_thread_detach(pthread_t thread) { int ret = pthread_detach(thread); if(ret != 0) - error("cannot detach thread. pthread_detach() failed with code %d.", ret); + netdata_log_error("cannot detach thread. pthread_detach() failed with code %d.", ret); return ret; } diff --git a/libnetdata/worker_utilization/worker_utilization.c b/libnetdata/worker_utilization/worker_utilization.c index d47d81c4e..ad45dbc7f 100644 --- a/libnetdata/worker_utilization/worker_utilization.c +++ b/libnetdata/worker_utilization/worker_utilization.c @@ -70,9 +70,9 @@ static inline usec_t worker_now_monotonic_usec(void) { } size_t workers_allocated_memory(void) { - netdata_spinlock_lock(&workers_globals.spinlock); + spinlock_lock(&workers_globals.spinlock); size_t memory = workers_globals.memory; - netdata_spinlock_unlock(&workers_globals.spinlock); + spinlock_unlock(&workers_globals.spinlock); return memory; } @@ -91,7 +91,7 @@ void worker_register(const char *name) { worker->last_action = WORKER_IDLE; size_t name_size = strlen(name) + 1; - netdata_spinlock_lock(&workers_globals.spinlock); + spinlock_lock(&workers_globals.spinlock); workers_globals.memory += sizeof(struct worker) + strlen(worker->tag) + 1 + strlen(worker->workname) + 1; @@ -100,25 +100,25 @@ void worker_register(const char *name) { struct workers_workname *workname = *PValue; if(!workname) { workname = mallocz(sizeof(struct workers_workname)); - netdata_spinlock_init(&workname->spinlock); + spinlock_init(&workname->spinlock); workname->base = NULL; *PValue = workname; workers_globals.memory += sizeof(struct workers_workname) + JUDYHS_INDEX_SIZE_ESTIMATE(name_size); } - netdata_spinlock_lock(&workname->spinlock); + spinlock_lock(&workname->spinlock); DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(workname->base, worker, prev, next); - netdata_spinlock_unlock(&workname->spinlock); + spinlock_unlock(&workname->spinlock); - netdata_spinlock_unlock(&workers_globals.spinlock); + spinlock_unlock(&workers_globals.spinlock); } void worker_register_job_custom_metric(size_t job_id, const char *name, const char *units, WORKER_METRIC_TYPE type) { if(unlikely(!worker)) return; if(unlikely(job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES)) { - error("WORKER_UTILIZATION: job_id %zu is too big. Max is %zu", job_id, (size_t)(WORKER_UTILIZATION_MAX_JOB_TYPES - 1)); + netdata_log_error("WORKER_UTILIZATION: job_id %zu is too big. Max is %zu", job_id, (size_t)(WORKER_UTILIZATION_MAX_JOB_TYPES - 1)); return; } @@ -127,7 +127,7 @@ void worker_register_job_custom_metric(size_t job_id, const char *name, const ch if(worker->per_job_type[job_id].name) { if(strcmp(string2str(worker->per_job_type[job_id].name), name) != 0 || worker->per_job_type[job_id].type != type || strcmp(string2str(worker->per_job_type[job_id].units), units) != 0) - error("WORKER_UTILIZATION: duplicate job registration: worker '%s' job id %zu is '%s', ignoring the later '%s'", worker->workname, job_id, string2str(worker->per_job_type[job_id].name), name); + netdata_log_error("WORKER_UTILIZATION: duplicate job registration: worker '%s' job id %zu is '%s', ignoring the later '%s'", worker->workname, job_id, string2str(worker->per_job_type[job_id].name), name); return; } @@ -144,13 +144,13 @@ void worker_unregister(void) { if(unlikely(!worker)) return; size_t workname_size = strlen(worker->workname) + 1; - netdata_spinlock_lock(&workers_globals.spinlock); + spinlock_lock(&workers_globals.spinlock); Pvoid_t *PValue = JudyHSGet(workers_globals.worknames_JudyHS, (void *)worker->workname, workname_size); if(PValue) { struct workers_workname *workname = *PValue; - netdata_spinlock_lock(&workname->spinlock); + spinlock_lock(&workname->spinlock); DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(workname->base, worker, prev, next); - netdata_spinlock_unlock(&workname->spinlock); + spinlock_unlock(&workname->spinlock); if(!workname->base) { JudyHSDel(&workers_globals.worknames_JudyHS, (void *) worker->workname, workname_size, PJE0); @@ -159,7 +159,7 @@ void worker_unregister(void) { } } workers_globals.memory -= sizeof(struct worker) + strlen(worker->tag) + 1 + strlen(worker->workname) + 1; - netdata_spinlock_unlock(&workers_globals.spinlock); + spinlock_unlock(&workers_globals.spinlock); for(int i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) { string_freez(worker->per_job_type[i].name); @@ -247,7 +247,7 @@ void workers_foreach(const char *name, void (*callback)( , NETDATA_DOUBLE *job_custom_values ) , void *data) { - netdata_spinlock_lock(&workers_globals.spinlock); + spinlock_lock(&workers_globals.spinlock); usec_t busy_time, delta; size_t i, jobs_started, jobs_running; @@ -256,12 +256,12 @@ void workers_foreach(const char *name, void (*callback)( Pvoid_t *PValue = JudyHSGet(workers_globals.worknames_JudyHS, (void *)name, workname_size); if(PValue) { workname = *PValue; - netdata_spinlock_lock(&workname->spinlock); + spinlock_lock(&workname->spinlock); } else workname = NULL; - netdata_spinlock_unlock(&workers_globals.spinlock); + spinlock_unlock(&workers_globals.spinlock); if(!workname) return; @@ -379,5 +379,5 @@ void workers_foreach(const char *name, void (*callback)( ); } - netdata_spinlock_unlock(&workname->spinlock); + spinlock_unlock(&workname->spinlock); } |