From cd4377fab21e0f500bef7f06543fa848a039c1e0 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 20 Jul 2023 06:50:01 +0200 Subject: Merging upstream version 1.41.0. Signed-off-by: Daniel Baumann --- libnetdata/locks/locks.c | 178 ++++++++++++++++++++++++++++++++++++----------- libnetdata/locks/locks.h | 24 +++++-- 2 files changed, 156 insertions(+), 46 deletions(-) (limited to 'libnetdata/locks') diff --git a/libnetdata/locks/locks.c b/libnetdata/locks/locks.c index e73456d70..625dd052c 100644 --- a/libnetdata/locks/locks.c +++ b/libnetdata/locks/locks.c @@ -28,37 +28,52 @@ static __thread size_t netdata_locks_acquired_rwlocks = 0; static __thread size_t netdata_locks_acquired_mutexes = 0; inline void netdata_thread_disable_cancelability(void) { - int old; - int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old); - if(ret != 0) - error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret); - else { - if(!netdata_thread_nested_disables) - netdata_thread_first_cancelability = old; + if(!netdata_thread_nested_disables) { + int old; + int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old); + + if(ret != 0) + netdata_log_error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", + netdata_thread_tag(), ret); - netdata_thread_nested_disables++; + netdata_thread_first_cancelability = old; } + + netdata_thread_nested_disables++; } inline void netdata_thread_enable_cancelability(void) { - if(netdata_thread_nested_disables < 1) { - error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): invalid thread cancelability count %d on thread %s - results will be undefined - please report this!", - netdata_thread_nested_disables, netdata_thread_tag()); + if(unlikely(netdata_thread_nested_disables < 1)) { + internal_fatal(true, "THREAD_CANCELABILITY: trying to enable cancelability, but it was not not disabled"); + + netdata_log_error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): invalid thread cancelability count %d " + "on thread %s - results will be undefined - please report this!", + netdata_thread_nested_disables, netdata_thread_tag()); + + netdata_thread_nested_disables = 1; } - else if(netdata_thread_nested_disables == 1) { + + if(netdata_thread_nested_disables == 1) { int old = 1; int ret = pthread_setcancelstate(netdata_thread_first_cancelability, &old); if(ret != 0) - error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret); + netdata_log_error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", + netdata_thread_tag(), + ret); else { - if(old != PTHREAD_CANCEL_DISABLE) - error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): old thread cancelability on thread %s was changed, expected DISABLED (%d), found %s (%d) - please report this!", netdata_thread_tag(), PTHREAD_CANCEL_DISABLE, (old == PTHREAD_CANCEL_ENABLE)?"ENABLED":"UNKNOWN", old); + if(old != PTHREAD_CANCEL_DISABLE) { + internal_fatal(true, "THREAD_CANCELABILITY: invalid old state cancelability"); + + netdata_log_error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): old thread cancelability " + "on thread %s was changed, expected DISABLED (%d), found %s (%d) - please report this!", + netdata_thread_tag(), PTHREAD_CANCEL_DISABLE, + (old == PTHREAD_CANCEL_ENABLE) ? "ENABLED" : "UNKNOWN", + old); + } } - - netdata_thread_nested_disables = 0; } - else - netdata_thread_nested_disables--; + + netdata_thread_nested_disables--; } // ---------------------------------------------------------------------------- @@ -67,14 +82,14 @@ inline void netdata_thread_enable_cancelability(void) { int __netdata_mutex_init(netdata_mutex_t *mutex) { int ret = pthread_mutex_init(mutex, NULL); if(unlikely(ret != 0)) - error("MUTEX_LOCK: failed to initialize (code %d).", ret); + netdata_log_error("MUTEX_LOCK: failed to initialize (code %d).", ret); return ret; } int __netdata_mutex_destroy(netdata_mutex_t *mutex) { int ret = pthread_mutex_destroy(mutex); if(unlikely(ret != 0)) - error("MUTEX_LOCK: failed to destroy (code %d).", ret); + netdata_log_error("MUTEX_LOCK: failed to destroy (code %d).", ret); return ret; } @@ -84,7 +99,7 @@ int __netdata_mutex_lock(netdata_mutex_t *mutex) { int ret = pthread_mutex_lock(mutex); if(unlikely(ret != 0)) { netdata_thread_enable_cancelability(); - error("MUTEX_LOCK: failed to get lock (code %d)", ret); + netdata_log_error("MUTEX_LOCK: failed to get lock (code %d)", ret); } else netdata_locks_acquired_mutexes++; @@ -107,7 +122,7 @@ int __netdata_mutex_trylock(netdata_mutex_t *mutex) { int __netdata_mutex_unlock(netdata_mutex_t *mutex) { int ret = pthread_mutex_unlock(mutex); if(unlikely(ret != 0)) - error("MUTEX_LOCK: failed to unlock (code %d).", ret); + netdata_log_error("MUTEX_LOCK: failed to unlock (code %d).", ret); else { netdata_locks_acquired_mutexes--; netdata_thread_enable_cancelability(); @@ -120,29 +135,29 @@ int __netdata_mutex_unlock(netdata_mutex_t *mutex) { int netdata_mutex_init_debug(const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(%p) from %lu@%s, %s()", mutex, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(%p) from %lu@%s, %s()", mutex, line, file, function); int ret = __netdata_mutex_init(mutex); - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(%p) = %d, from %lu@%s, %s()", mutex, ret, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(%p) = %d, from %lu@%s, %s()", mutex, ret, line, file, function); return ret; } int netdata_mutex_destroy_debug(const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_destroy(%p) from %lu@%s, %s()", mutex, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_destroy(%p) from %lu@%s, %s()", mutex, line, file, function); int ret = __netdata_mutex_destroy(mutex); - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_destroy(%p) = %d, from %lu@%s, %s()", mutex, ret, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_destroy(%p) = %d, from %lu@%s, %s()", mutex, ret, line, file, function); return ret; } int netdata_mutex_lock_debug(const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(%p) from %lu@%s, %s()", mutex, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(%p) from %lu@%s, %s()", mutex, line, file, function); usec_t start_s = now_monotonic_high_precision_usec(); int ret = __netdata_mutex_lock(mutex); @@ -152,14 +167,14 @@ int netdata_mutex_lock_debug(const char *file __maybe_unused, const char *functi (void)start_s; (void)end_s; - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); return ret; } int netdata_mutex_trylock_debug(const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(%p) from %lu@%s, %s()", mutex, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(%p) from %lu@%s, %s()", mutex, line, file, function); usec_t start_s = now_monotonic_high_precision_usec(); int ret = __netdata_mutex_trylock(mutex); @@ -169,14 +184,14 @@ int netdata_mutex_trylock_debug(const char *file __maybe_unused, const char *fun (void)start_s; (void)end_s; - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); return ret; } int netdata_mutex_unlock_debug(const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, netdata_mutex_t *mutex) { - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(%p) from %lu@%s, %s()", mutex, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(%p) from %lu@%s, %s()", mutex, line, file, function); usec_t start_s = now_monotonic_high_precision_usec(); int ret = __netdata_mutex_unlock(mutex); @@ -186,7 +201,7 @@ int netdata_mutex_unlock_debug(const char *file __maybe_unused, const char *func (void)start_s; (void)end_s; - debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); + netdata_log_debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, end_s - start_s, line, file, function); return ret; } @@ -199,14 +214,14 @@ int netdata_mutex_unlock_debug(const char *file __maybe_unused, const char *func int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_destroy(&rwlock->rwlock_t); if(unlikely(ret != 0)) - error("RW_LOCK: failed to destroy lock (code %d)", ret); + netdata_log_error("RW_LOCK: failed to destroy lock (code %d)", ret); return ret; } int __netdata_rwlock_init(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_init(&rwlock->rwlock_t, NULL); if(unlikely(ret != 0)) - error("RW_LOCK: failed to initialize lock (code %d)", ret); + netdata_log_error("RW_LOCK: failed to initialize lock (code %d)", ret); return ret; } @@ -216,7 +231,7 @@ int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_rdlock(&rwlock->rwlock_t); if(unlikely(ret != 0)) { netdata_thread_enable_cancelability(); - error("RW_LOCK: failed to obtain read lock (code %d)", ret); + netdata_log_error("RW_LOCK: failed to obtain read lock (code %d)", ret); } else netdata_locks_acquired_rwlocks++; @@ -229,7 +244,7 @@ int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_wrlock(&rwlock->rwlock_t); if(unlikely(ret != 0)) { - error("RW_LOCK: failed to obtain write lock (code %d)", ret); + netdata_log_error("RW_LOCK: failed to obtain write lock (code %d)", ret); netdata_thread_enable_cancelability(); } else @@ -241,7 +256,7 @@ int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) { int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock) { int ret = pthread_rwlock_unlock(&rwlock->rwlock_t); if(unlikely(ret != 0)) - error("RW_LOCK: failed to release lock (code %d)", ret); + netdata_log_error("RW_LOCK: failed to release lock (code %d)", ret); else { netdata_thread_enable_cancelability(); netdata_locks_acquired_rwlocks--; @@ -278,11 +293,11 @@ int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) { // spinlock implementation // https://www.youtube.com/watch?v=rmGJc9PXpuE&t=41s -void netdata_spinlock_init(SPINLOCK *spinlock) { +void spinlock_init(SPINLOCK *spinlock) { memset(spinlock, 0, sizeof(SPINLOCK)); } -void netdata_spinlock_lock(SPINLOCK *spinlock) { +void spinlock_lock(SPINLOCK *spinlock) { static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; #ifdef NETDATA_INTERNAL_CHECKS @@ -314,7 +329,7 @@ void netdata_spinlock_lock(SPINLOCK *spinlock) { #endif } -void netdata_spinlock_unlock(SPINLOCK *spinlock) { +void spinlock_unlock(SPINLOCK *spinlock) { #ifdef NETDATA_INTERNAL_CHECKS spinlock->locker_pid = 0; #endif @@ -322,7 +337,7 @@ void netdata_spinlock_unlock(SPINLOCK *spinlock) { netdata_thread_enable_cancelability(); } -bool netdata_spinlock_trylock(SPINLOCK *spinlock) { +bool spinlock_trylock(SPINLOCK *spinlock) { netdata_thread_disable_cancelability(); if(!__atomic_load_n(&spinlock->locked, __ATOMIC_RELAXED) && @@ -331,9 +346,88 @@ bool netdata_spinlock_trylock(SPINLOCK *spinlock) { return true; // we didn't get the lock + netdata_thread_enable_cancelability(); + return false; +} + +// ---------------------------------------------------------------------------- +// rw_spinlock implementation + +void rw_spinlock_init(RW_SPINLOCK *rw_spinlock) { + rw_spinlock->readers = 0; + spinlock_init(&rw_spinlock->spinlock); +} + +void rw_spinlock_read_lock(RW_SPINLOCK *rw_spinlock) { + netdata_thread_disable_cancelability(); + + spinlock_lock(&rw_spinlock->spinlock); + __atomic_add_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED); + spinlock_unlock(&rw_spinlock->spinlock); +} + +void rw_spinlock_read_unlock(RW_SPINLOCK *rw_spinlock) { +#ifndef NETDATA_INTERNAL_CHECKS + __atomic_sub_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED); +#else + int32_t x = __atomic_sub_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED); + if(x < 0) + fatal("RW_SPINLOCK: readers is negative %d", x); +#endif + + netdata_thread_enable_cancelability(); +} + +void rw_spinlock_write_lock(RW_SPINLOCK *rw_spinlock) { + static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 }; + + size_t spins = 0; + while(1) { + spins++; + spinlock_lock(&rw_spinlock->spinlock); + + if(__atomic_load_n(&rw_spinlock->readers, __ATOMIC_RELAXED) == 0) + break; + + // Busy wait until all readers have released their locks. + spinlock_unlock(&rw_spinlock->spinlock); + nanosleep(&ns, NULL); + } + + (void)spins; +} + +void rw_spinlock_write_unlock(RW_SPINLOCK *rw_spinlock) { + spinlock_unlock(&rw_spinlock->spinlock); +} + +bool rw_spinlock_tryread_lock(RW_SPINLOCK *rw_spinlock) { + if(spinlock_trylock(&rw_spinlock->spinlock)) { + __atomic_add_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED); + spinlock_unlock(&rw_spinlock->spinlock); + netdata_thread_disable_cancelability(); + return true; + } + return false; } +bool rw_spinlock_trywrite_lock(RW_SPINLOCK *rw_spinlock) { + if(spinlock_trylock(&rw_spinlock->spinlock)) { + if (__atomic_load_n(&rw_spinlock->readers, __ATOMIC_RELAXED) == 0) { + // No readers, we've successfully acquired the write lock + return true; + } + else { + // There are readers, unlock the spinlock and return false + spinlock_unlock(&rw_spinlock->spinlock); + } + } + + return false; +} + + #ifdef NETDATA_TRACE_RWLOCKS // ---------------------------------------------------------------------------- diff --git a/libnetdata/locks/locks.h b/libnetdata/locks/locks.h index 89b110d5e..6b492ae47 100644 --- a/libnetdata/locks/locks.h +++ b/libnetdata/locks/locks.h @@ -20,10 +20,26 @@ typedef struct netdata_spinlock { #define NETDATA_SPINLOCK_INITIALIZER \ { .locked = false } -void netdata_spinlock_init(SPINLOCK *spinlock); -void netdata_spinlock_lock(SPINLOCK *spinlock); -void netdata_spinlock_unlock(SPINLOCK *spinlock); -bool netdata_spinlock_trylock(SPINLOCK *spinlock); +void spinlock_init(SPINLOCK *spinlock); +void spinlock_lock(SPINLOCK *spinlock); +void spinlock_unlock(SPINLOCK *spinlock); +bool spinlock_trylock(SPINLOCK *spinlock); + +typedef struct netdata_rw_spinlock { + int32_t readers; + SPINLOCK spinlock; +} RW_SPINLOCK; + +#define NETDATA_RW_SPINLOCK_INITIALIZER \ + { .readers = 0, .spinlock = NETDATA_SPINLOCK_INITIALIZER } + +void rw_spinlock_init(RW_SPINLOCK *rw_spinlock); +void rw_spinlock_read_lock(RW_SPINLOCK *rw_spinlock); +void rw_spinlock_read_unlock(RW_SPINLOCK *rw_spinlock); +void rw_spinlock_write_lock(RW_SPINLOCK *rw_spinlock); +void rw_spinlock_write_unlock(RW_SPINLOCK *rw_spinlock); +bool rw_spinlock_tryread_lock(RW_SPINLOCK *rw_spinlock); +bool rw_spinlock_trywrite_lock(RW_SPINLOCK *rw_spinlock); #ifdef NETDATA_TRACE_RWLOCKS -- cgit v1.2.3