summaryrefslogtreecommitdiffstats
path: root/libnetdata/dictionary/dictionary.c
diff options
context:
space:
mode:
Diffstat (limited to 'libnetdata/dictionary/dictionary.c')
-rw-r--r--libnetdata/dictionary/dictionary.c282
1 files changed, 143 insertions, 139 deletions
diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c
index 061b671ab..42e4a99f1 100644
--- a/libnetdata/dictionary/dictionary.c
+++ b/libnetdata/dictionary/dictionary.c
@@ -10,9 +10,9 @@ typedef enum __attribute__ ((__packed__)) {
DICT_FLAG_DESTROYED = (1 << 0), // this dictionary has been destroyed
} DICT_FLAGS;
-#define dict_flag_check(dict, flag) (__atomic_load_n(&((dict)->flags), __ATOMIC_SEQ_CST) & (flag))
-#define dict_flag_set(dict, flag) __atomic_or_fetch(&((dict)->flags), flag, __ATOMIC_SEQ_CST)
-#define dict_flag_clear(dict, flag) __atomic_and_fetch(&((dict)->flags), ~(flag), __ATOMIC_SEQ_CST)
+#define dict_flag_check(dict, flag) (__atomic_load_n(&((dict)->flags), __ATOMIC_RELAXED) & (flag))
+#define dict_flag_set(dict, flag) __atomic_or_fetch(&((dict)->flags), flag, __ATOMIC_RELAXED)
+#define dict_flag_clear(dict, flag) __atomic_and_fetch(&((dict)->flags), ~(flag), __ATOMIC_RELAXED)
// flags macros
#define is_dictionary_destroyed(dict) dict_flag_check(dict, DICT_FLAG_DESTROYED)
@@ -37,13 +37,13 @@ typedef enum __attribute__ ((__packed__)) item_flags {
// IMPORTANT: This is 8-bit
} ITEM_FLAGS;
-#define item_flag_check(item, flag) (__atomic_load_n(&((item)->flags), __ATOMIC_SEQ_CST) & (flag))
-#define item_flag_set(item, flag) __atomic_or_fetch(&((item)->flags), flag, __ATOMIC_SEQ_CST)
-#define item_flag_clear(item, flag) __atomic_and_fetch(&((item)->flags), ~(flag), __ATOMIC_SEQ_CST)
+#define item_flag_check(item, flag) (__atomic_load_n(&((item)->flags), __ATOMIC_RELAXED) & (flag))
+#define item_flag_set(item, flag) __atomic_or_fetch(&((item)->flags), flag, __ATOMIC_RELAXED)
+#define item_flag_clear(item, flag) __atomic_and_fetch(&((item)->flags), ~(flag), __ATOMIC_RELAXED)
-#define item_shared_flag_check(item, flag) (__atomic_load_n(&((item)->shared->flags), __ATOMIC_SEQ_CST) & (flag))
-#define item_shared_flag_set(item, flag) __atomic_or_fetch(&((item)->shared->flags), flag, __ATOMIC_SEQ_CST)
-#define item_shared_flag_clear(item, flag) __atomic_and_fetch(&((item)->shared->flags), ~(flag), __ATOMIC_SEQ_CST)
+#define item_shared_flag_check(item, flag) (__atomic_load_n(&((item)->shared->flags), __ATOMIC_RELAXED) & (flag))
+#define item_shared_flag_set(item, flag) __atomic_or_fetch(&((item)->shared->flags), flag, __ATOMIC_RELAXED)
+#define item_shared_flag_clear(item, flag) __atomic_and_fetch(&((item)->shared->flags), ~(flag), __ATOMIC_RELAXED)
#define REFCOUNT_DELETING (-100)
@@ -175,7 +175,7 @@ struct dictionary {
long int referenced_items; // how many items of the dictionary are currently being used by 3rd parties
long int pending_deletion_items; // how many items of the dictionary have been deleted, but have not been removed yet
-#ifdef NETDATA_INTERNAL_CHECKS
+#ifdef NETDATA_DICTIONARY_VALIDATE_POINTERS
netdata_mutex_t global_pointer_registry_mutex;
Pvoid_t global_pointer_registry;
#endif
@@ -205,59 +205,47 @@ static inline int item_is_not_referenced_and_can_be_removed_advanced(DICTIONARY
// ----------------------------------------------------------------------------
// validate each pointer is indexed once - internal checks only
+#ifdef NETDATA_DICTIONARY_VALIDATE_POINTERS
static inline void pointer_index_init(DICTIONARY *dict __maybe_unused) {
-#ifdef NETDATA_INTERNAL_CHECKS
netdata_mutex_init(&dict->global_pointer_registry_mutex);
-#else
- ;
-#endif
}
static inline void pointer_destroy_index(DICTIONARY *dict __maybe_unused) {
-#ifdef NETDATA_INTERNAL_CHECKS
netdata_mutex_lock(&dict->global_pointer_registry_mutex);
JudyHSFreeArray(&dict->global_pointer_registry, PJE0);
netdata_mutex_unlock(&dict->global_pointer_registry_mutex);
-#else
- ;
-#endif
}
static inline void pointer_add(DICTIONARY *dict __maybe_unused, DICTIONARY_ITEM *item __maybe_unused) {
-#ifdef NETDATA_INTERNAL_CHECKS
netdata_mutex_lock(&dict->global_pointer_registry_mutex);
Pvoid_t *PValue = JudyHSIns(&dict->global_pointer_registry, &item, sizeof(void *), PJE0);
if(*PValue != NULL)
fatal("pointer already exists in registry");
*PValue = item;
netdata_mutex_unlock(&dict->global_pointer_registry_mutex);
-#else
- ;
-#endif
}
static inline void pointer_check(DICTIONARY *dict __maybe_unused, DICTIONARY_ITEM *item __maybe_unused) {
-#ifdef NETDATA_INTERNAL_CHECKS
netdata_mutex_lock(&dict->global_pointer_registry_mutex);
Pvoid_t *PValue = JudyHSGet(dict->global_pointer_registry, &item, sizeof(void *));
if(PValue == NULL)
fatal("pointer is not found in registry");
netdata_mutex_unlock(&dict->global_pointer_registry_mutex);
-#else
- ;
-#endif
}
static inline void pointer_del(DICTIONARY *dict __maybe_unused, DICTIONARY_ITEM *item __maybe_unused) {
-#ifdef NETDATA_INTERNAL_CHECKS
netdata_mutex_lock(&dict->global_pointer_registry_mutex);
int ret = JudyHSDel(&dict->global_pointer_registry, &item, sizeof(void *), PJE0);
if(!ret)
fatal("pointer to be deleted does not exist in registry");
netdata_mutex_unlock(&dict->global_pointer_registry_mutex);
-#else
- ;
-#endif
}
+#else // !NETDATA_DICTIONARY_VALIDATE_POINTERS
+#define pointer_index_init(dict) debug_dummy()
+#define pointer_destroy_index(dict) debug_dummy()
+#define pointer_add(dict, item) debug_dummy()
+#define pointer_check(dict, item) debug_dummy()
+#define pointer_del(dict, item) debug_dummy()
+#endif // !NETDATA_DICTIONARY_VALIDATE_POINTERS
// ----------------------------------------------------------------------------
// memory statistics
@@ -298,7 +286,7 @@ static inline void dictionary_hooks_allocate(DICTIONARY *dict) {
static inline size_t dictionary_hooks_free(DICTIONARY *dict) {
if(!dict->hooks) return 0;
- REFCOUNT links = __atomic_sub_fetch(&dict->hooks->links, 1, __ATOMIC_SEQ_CST);
+ REFCOUNT links = __atomic_sub_fetch(&dict->hooks->links, 1, __ATOMIC_ACQUIRE);
if(links == 0) {
freez(dict->hooks);
dict->hooks = NULL;
@@ -356,26 +344,25 @@ size_t dictionary_version(DICTIONARY *dict) {
if(unlikely(!dict)) return 0;
// this is required for views to return the right number
- garbage_collect_pending_deletes(dict);
+ // garbage_collect_pending_deletes(dict);
- return __atomic_load_n(&dict->version, __ATOMIC_SEQ_CST);
+ return __atomic_load_n(&dict->version, __ATOMIC_RELAXED);
}
size_t dictionary_entries(DICTIONARY *dict) {
if(unlikely(!dict)) return 0;
// this is required for views to return the right number
- garbage_collect_pending_deletes(dict);
+ // garbage_collect_pending_deletes(dict);
- long int entries = __atomic_load_n(&dict->entries, __ATOMIC_SEQ_CST);
- if(entries < 0)
- fatal("DICTIONARY: entries is negative: %ld", entries);
+ long int entries = __atomic_load_n(&dict->entries, __ATOMIC_RELAXED);
+ internal_fatal(entries < 0, "DICTIONARY: entries is negative: %ld", entries);
return entries;
}
size_t dictionary_referenced_items(DICTIONARY *dict) {
if(unlikely(!dict)) return 0;
- long int referenced_items = __atomic_load_n(&dict->referenced_items, __ATOMIC_SEQ_CST);
+ long int referenced_items = __atomic_load_n(&dict->referenced_items, __ATOMIC_RELAXED);
if(referenced_items < 0)
fatal("DICTIONARY: referenced items is negative: %ld", referenced_items);
@@ -387,7 +374,7 @@ long int dictionary_stats_for_registry(DICTIONARY *dict) {
return (dict->stats->memory.index + dict->stats->memory.dict);
}
void dictionary_version_increment(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->version, 1, __ATOMIC_SEQ_CST);
+ __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED);
}
// ----------------------------------------------------------------------------
@@ -409,9 +396,9 @@ static inline void DICTIONARY_ENTRIES_PLUS1(DICTIONARY *dict) {
}
else {
- __atomic_fetch_add(&dict->version, 1, __ATOMIC_SEQ_CST);
- __atomic_fetch_add(&dict->entries, 1, __ATOMIC_SEQ_CST);
- __atomic_fetch_add(&dict->referenced_items, 1, __ATOMIC_SEQ_CST);
+ __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED);
+ __atomic_fetch_add(&dict->entries, 1, __ATOMIC_RELAXED);
+ __atomic_fetch_add(&dict->referenced_items, 1, __ATOMIC_RELAXED);
}
}
static inline void DICTIONARY_ENTRIES_MINUS1(DICTIONARY *dict) {
@@ -425,17 +412,15 @@ static inline void DICTIONARY_ENTRIES_MINUS1(DICTIONARY *dict) {
entries = dict->entries++;
}
else {
- __atomic_fetch_add(&dict->version, 1, __ATOMIC_SEQ_CST);
- entries = __atomic_fetch_sub(&dict->entries, 1, __ATOMIC_SEQ_CST);
+ __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED);
+ entries = __atomic_fetch_sub(&dict->entries, 1, __ATOMIC_RELAXED);
}
-#ifdef NETDATA_INTERNAL_CHECKS
- if(unlikely(entries == 0))
- fatal("DICT: negative number of entries in dictionary created from %s() (%zu@%s)",
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
-#endif
+ internal_fatal(entries == 0,
+ "DICT: negative number of entries in dictionary created from %s() (%zu@%s)",
+ dict->creation_function,
+ dict->creation_line,
+ dict->creation_file);
}
static inline void DICTIONARY_VALUE_RESETS_PLUS1(DICTIONARY *dict) {
__atomic_fetch_add(&dict->stats->ops.resets, 1, __ATOMIC_RELAXED);
@@ -443,7 +428,7 @@ static inline void DICTIONARY_VALUE_RESETS_PLUS1(DICTIONARY *dict) {
if(unlikely(is_dictionary_single_threaded(dict)))
dict->version++;
else
- __atomic_fetch_add(&dict->version, 1, __ATOMIC_SEQ_CST);
+ __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED);
}
static inline void DICTIONARY_STATS_TRAVERSALS_PLUS1(DICTIONARY *dict) {
__atomic_fetch_add(&dict->stats->ops.traversals, 1, __ATOMIC_RELAXED);
@@ -464,16 +449,16 @@ static inline void DICTIONARY_STATS_SEARCH_IGNORES_PLUS1(DICTIONARY *dict) {
__atomic_fetch_add(&dict->stats->spin_locks.search_spins, 1, __ATOMIC_RELAXED);
}
static inline void DICTIONARY_STATS_CALLBACK_INSERTS_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->callbacks.inserts, 1, __ATOMIC_RELAXED);
+ __atomic_fetch_add(&dict->stats->callbacks.inserts, 1, __ATOMIC_RELEASE);
}
static inline void DICTIONARY_STATS_CALLBACK_CONFLICTS_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->callbacks.conflicts, 1, __ATOMIC_RELAXED);
+ __atomic_fetch_add(&dict->stats->callbacks.conflicts, 1, __ATOMIC_RELEASE);
}
static inline void DICTIONARY_STATS_CALLBACK_REACTS_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->callbacks.reacts, 1, __ATOMIC_RELAXED);
+ __atomic_fetch_add(&dict->stats->callbacks.reacts, 1, __ATOMIC_RELEASE);
}
static inline void DICTIONARY_STATS_CALLBACK_DELETES_PLUS1(DICTIONARY *dict) {
- __atomic_fetch_add(&dict->stats->callbacks.deletes, 1, __ATOMIC_RELAXED);
+ __atomic_fetch_add(&dict->stats->callbacks.deletes, 1, __ATOMIC_RELEASE);
}
static inline void DICTIONARY_STATS_GARBAGE_COLLECTIONS_PLUS1(DICTIONARY *dict) {
__atomic_fetch_add(&dict->stats->ops.garbage_collections, 1, __ATOMIC_RELAXED);
@@ -496,52 +481,48 @@ static inline void DICTIONARY_STATS_DICT_FLUSHES_PLUS1(DICTIONARY *dict) {
__atomic_fetch_add(&dict->stats->ops.flushes, 1, __ATOMIC_RELAXED);
}
-static inline long int DICTIONARY_REFERENCED_ITEMS_PLUS1(DICTIONARY *dict) {
+static inline void DICTIONARY_REFERENCED_ITEMS_PLUS1(DICTIONARY *dict) {
__atomic_fetch_add(&dict->stats->items.referenced, 1, __ATOMIC_RELAXED);
if(unlikely(is_dictionary_single_threaded(dict)))
- return ++dict->referenced_items;
+ ++dict->referenced_items;
else
- return __atomic_add_fetch(&dict->referenced_items, 1, __ATOMIC_SEQ_CST);
+ __atomic_add_fetch(&dict->referenced_items, 1, __ATOMIC_RELAXED);
}
-static inline long int DICTIONARY_REFERENCED_ITEMS_MINUS1(DICTIONARY *dict) {
+static inline void DICTIONARY_REFERENCED_ITEMS_MINUS1(DICTIONARY *dict) {
__atomic_fetch_sub(&dict->stats->items.referenced, 1, __ATOMIC_RELAXED);
- long int referenced_items;
+ long int referenced_items; (void)referenced_items;
if(unlikely(is_dictionary_single_threaded(dict)))
referenced_items = --dict->referenced_items;
else
referenced_items = __atomic_sub_fetch(&dict->referenced_items, 1, __ATOMIC_SEQ_CST);
-#ifdef NETDATA_INTERNAL_CHECKS
- if(unlikely(referenced_items < 0))
- fatal("DICT: negative number of referenced items (%ld) in dictionary created from %s() (%zu@%s)",
- referenced_items,
- dict->creation_function,
- dict->creation_line,
- dict->creation_file);
-#endif
-
- return referenced_items;
+ internal_fatal(referenced_items < 0,
+ "DICT: negative number of referenced items (%ld) in dictionary created from %s() (%zu@%s)",
+ referenced_items,
+ dict->creation_function,
+ dict->creation_line,
+ dict->creation_file);
}
-static inline long int DICTIONARY_PENDING_DELETES_PLUS1(DICTIONARY *dict) {
+static inline void DICTIONARY_PENDING_DELETES_PLUS1(DICTIONARY *dict) {
__atomic_fetch_add(&dict->stats->items.pending_deletion, 1, __ATOMIC_RELAXED);
if(unlikely(is_dictionary_single_threaded(dict)))
- return ++dict->pending_deletion_items;
+ ++dict->pending_deletion_items;
else
- return __atomic_add_fetch(&dict->pending_deletion_items, 1, __ATOMIC_SEQ_CST);
+ __atomic_add_fetch(&dict->pending_deletion_items, 1, __ATOMIC_RELEASE);
}
static inline long int DICTIONARY_PENDING_DELETES_MINUS1(DICTIONARY *dict) {
- __atomic_fetch_sub(&dict->stats->items.pending_deletion, 1, __ATOMIC_RELAXED);
+ __atomic_fetch_sub(&dict->stats->items.pending_deletion, 1, __ATOMIC_RELEASE);
if(unlikely(is_dictionary_single_threaded(dict)))
return --dict->pending_deletion_items;
else
- return __atomic_sub_fetch(&dict->pending_deletion_items, 1, __ATOMIC_SEQ_CST);
+ return __atomic_sub_fetch(&dict->pending_deletion_items, 1, __ATOMIC_ACQUIRE);
}
static inline long int DICTIONARY_PENDING_DELETES_GET(DICTIONARY *dict) {
@@ -555,11 +536,11 @@ static inline REFCOUNT DICTIONARY_ITEM_REFCOUNT_GET(DICTIONARY *dict, DICTIONARY
if(unlikely(dict && is_dictionary_single_threaded(dict))) // this is an exception, dict can be null
return item->refcount;
else
- return (REFCOUNT)__atomic_load_n(&item->refcount, __ATOMIC_SEQ_CST);
+ return (REFCOUNT)__atomic_load_n(&item->refcount, __ATOMIC_ACQUIRE);
}
static inline REFCOUNT DICTIONARY_ITEM_REFCOUNT_GET_SOLE(DICTIONARY_ITEM *item) {
- return (REFCOUNT)__atomic_load_n(&item->refcount, __ATOMIC_SEQ_CST);
+ return (REFCOUNT)__atomic_load_n(&item->refcount, __ATOMIC_ACQUIRE);
}
// ----------------------------------------------------------------------------
@@ -579,8 +560,8 @@ static void dictionary_execute_insert_callback(DICTIONARY *dict, DICTIONARY_ITEM
dict->creation_line,
dict->creation_file);
- DICTIONARY_STATS_CALLBACK_INSERTS_PLUS1(dict);
dict->hooks->ins_callback(item, item->shared->value, constructor_data?constructor_data:dict->hooks->ins_callback_data);
+ DICTIONARY_STATS_CALLBACK_INSERTS_PLUS1(dict);
}
static bool dictionary_execute_conflict_callback(DICTIONARY *dict, DICTIONARY_ITEM *item, void *new_value, void *constructor_data) {
@@ -597,10 +578,13 @@ static bool dictionary_execute_conflict_callback(DICTIONARY *dict, DICTIONARY_IT
dict->creation_line,
dict->creation_file);
- DICTIONARY_STATS_CALLBACK_CONFLICTS_PLUS1(dict);
- return dict->hooks->conflict_callback(
+ bool ret = dict->hooks->conflict_callback(
item, item->shared->value, new_value,
constructor_data ? constructor_data : dict->hooks->conflict_callback_data);
+
+ DICTIONARY_STATS_CALLBACK_CONFLICTS_PLUS1(dict);
+
+ return ret;
}
static void dictionary_execute_react_callback(DICTIONARY *dict, DICTIONARY_ITEM *item, void *constructor_data) {
@@ -617,9 +601,10 @@ static void dictionary_execute_react_callback(DICTIONARY *dict, DICTIONARY_ITEM
dict->creation_line,
dict->creation_file);
- DICTIONARY_STATS_CALLBACK_REACTS_PLUS1(dict);
dict->hooks->react_callback(item, item->shared->value,
constructor_data?constructor_data:dict->hooks->react_callback_data);
+
+ DICTIONARY_STATS_CALLBACK_REACTS_PLUS1(dict);
}
static void dictionary_execute_delete_callback(DICTIONARY *dict, DICTIONARY_ITEM *item) {
@@ -637,8 +622,9 @@ static void dictionary_execute_delete_callback(DICTIONARY *dict, DICTIONARY_ITEM
dict->creation_line,
dict->creation_file);
- DICTIONARY_STATS_CALLBACK_DELETES_PLUS1(dict);
dict->hooks->del_callback(item, item->shared->value, dict->hooks->del_callback_data);
+
+ DICTIONARY_STATS_CALLBACK_DELETES_PLUS1(dict);
}
// ----------------------------------------------------------------------------
@@ -648,8 +634,8 @@ static inline size_t dictionary_locks_init(DICTIONARY *dict) {
if(likely(!is_dictionary_single_threaded(dict))) {
netdata_rwlock_init(&dict->index.rwlock);
netdata_rwlock_init(&dict->items.rwlock);
- return 0;
}
+
return 0;
}
@@ -657,29 +643,29 @@ static inline size_t dictionary_locks_destroy(DICTIONARY *dict) {
if(likely(!is_dictionary_single_threaded(dict))) {
netdata_rwlock_destroy(&dict->index.rwlock);
netdata_rwlock_destroy(&dict->items.rwlock);
- return 0;
}
+
return 0;
}
static inline void ll_recursive_lock_set_thread_as_writer(DICTIONARY *dict) {
pid_t expected = 0, desired = gettid();
- if(!__atomic_compare_exchange_n(&dict->items.writer_pid, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
- fatal("DICTIONARY: Cannot set thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_SEQ_CST));
+ if(!__atomic_compare_exchange_n(&dict->items.writer_pid, &expected, desired, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED))
+ fatal("DICTIONARY: Cannot set thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_RELAXED));
}
static inline void ll_recursive_unlock_unset_thread_writer(DICTIONARY *dict) {
pid_t expected = gettid(), desired = 0;
- if(!__atomic_compare_exchange_n(&dict->items.writer_pid, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
- fatal("DICTIONARY: Cannot unset thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_SEQ_CST));
+ if(!__atomic_compare_exchange_n(&dict->items.writer_pid, &expected, desired, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED))
+ fatal("DICTIONARY: Cannot unset thread %d as exclusive writer, expected %d, desired %d, found %d.", gettid(), expected, desired, __atomic_load_n(&dict->items.writer_pid, __ATOMIC_RELAXED));
}
static inline bool ll_recursive_lock_is_thread_the_writer(DICTIONARY *dict) {
pid_t tid = gettid();
- return tid > 0 && tid == __atomic_load_n(&dict->items.writer_pid, __ATOMIC_SEQ_CST);
+ return tid > 0 && tid == __atomic_load_n(&dict->items.writer_pid, __ATOMIC_RELAXED);
}
-static void ll_recursive_lock(DICTIONARY *dict, char rw) {
+static inline void ll_recursive_lock(DICTIONARY *dict, char rw) {
if(unlikely(is_dictionary_single_threaded(dict)))
return;
@@ -699,7 +685,7 @@ static void ll_recursive_lock(DICTIONARY *dict, char rw) {
}
}
-static void ll_recursive_unlock(DICTIONARY *dict, char rw) {
+static inline void ll_recursive_unlock(DICTIONARY *dict, char rw) {
if(unlikely(is_dictionary_single_threaded(dict)))
return;
@@ -722,10 +708,10 @@ static void ll_recursive_unlock(DICTIONARY *dict, char rw) {
}
}
-void dictionary_write_lock(DICTIONARY *dict) {
+inline void dictionary_write_lock(DICTIONARY *dict) {
ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
}
-void dictionary_write_unlock(DICTIONARY *dict) {
+inline void dictionary_write_unlock(DICTIONARY *dict) {
ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
}
@@ -760,8 +746,8 @@ static inline void dictionary_index_wrlock_unlock(DICTIONARY *dict) {
// items garbage collector
static void garbage_collect_pending_deletes(DICTIONARY *dict) {
- usec_t last_master_deletion_us = dict->hooks?__atomic_load_n(&dict->hooks->last_master_deletion_us, __ATOMIC_SEQ_CST):0;
- usec_t last_gc_run_us = __atomic_load_n(&dict->last_gc_run_us, __ATOMIC_SEQ_CST);
+ usec_t last_master_deletion_us = dict->hooks?__atomic_load_n(&dict->hooks->last_master_deletion_us, __ATOMIC_RELAXED):0;
+ usec_t last_gc_run_us = __atomic_load_n(&dict->last_gc_run_us, __ATOMIC_RELAXED);
bool is_view = is_view_dictionary(dict);
@@ -773,7 +759,7 @@ static void garbage_collect_pending_deletes(DICTIONARY *dict) {
ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE);
- __atomic_store_n(&dict->last_gc_run_us, now_realtime_usec(), __ATOMIC_SEQ_CST);
+ __atomic_store_n(&dict->last_gc_run_us, now_realtime_usec(), __ATOMIC_RELAXED);
if(is_view)
dictionary_index_lock_wrlock(dict);
@@ -819,25 +805,27 @@ static void garbage_collect_pending_deletes(DICTIONARY *dict) {
(void)deleted;
(void)examined;
- internal_error(false, "DICTIONARY: garbage collected dictionary created by %s (%zu@%s), examined %zu items, deleted %zu items, still pending %zu items",
- dict->creation_function, dict->creation_line, dict->creation_file, examined, deleted, pending);
+ internal_error(false, "DICTIONARY: garbage collected dictionary created by %s (%zu@%s), "
+ "examined %zu items, deleted %zu items, still pending %zu items",
+ dict->creation_function, dict->creation_line, dict->creation_file,
+ examined, deleted, pending);
+}
+void dictionary_garbage_collect(DICTIONARY *dict) {
+ if(!dict) return;
+ garbage_collect_pending_deletes(dict);
}
// ----------------------------------------------------------------------------
// reference counters
-static inline size_t reference_counter_init(DICTIONARY *dict) {
- (void)dict;
-
+static inline size_t reference_counter_init(DICTIONARY *dict __maybe_unused) {
// allocate memory required for reference counters
// return number of bytes
return 0;
}
-static inline size_t reference_counter_free(DICTIONARY *dict) {
- (void)dict;
-
+static inline size_t reference_counter_free(DICTIONARY *dict __maybe_unused) {
// free memory required for reference counters
// return number of bytes
return 0;
@@ -846,13 +834,13 @@ static inline size_t reference_counter_free(DICTIONARY *dict) {
static void item_acquire(DICTIONARY *dict, DICTIONARY_ITEM *item) {
REFCOUNT refcount;
- if(unlikely(is_dictionary_single_threaded(dict))) {
+ if(unlikely(is_dictionary_single_threaded(dict)))
refcount = ++item->refcount;
- }
- else {
+
+ else
// increment the refcount
refcount = __atomic_add_fetch(&item->refcount, 1, __ATOMIC_SEQ_CST);
- }
+
if(refcount <= 0) {
internal_error(
@@ -900,7 +888,7 @@ static void item_release(DICTIONARY *dict, DICTIONARY_ITEM *item) {
is_deleted = item_flag_check(item, ITEM_FLAG_DELETED);
// decrement the refcount
- refcount = __atomic_sub_fetch(&item->refcount, 1, __ATOMIC_SEQ_CST);
+ refcount = __atomic_sub_fetch(&item->refcount, 1, __ATOMIC_RELEASE);
}
if(refcount < 0) {
@@ -956,14 +944,14 @@ static int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_ITEM *it
desired = refcount + 1;
- } while(!__atomic_compare_exchange_n(&item->refcount, &refcount, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST));
+ } while(!__atomic_compare_exchange_n(&item->refcount, &refcount, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED));
// if ret == ITEM_OK, we acquired the item
if(ret == RC_ITEM_OK) {
- if (is_view_dictionary(dict) &&
+ if (unlikely(is_view_dictionary(dict) &&
item_shared_flag_check(item, ITEM_FLAG_DELETED) &&
- !item_flag_check(item, ITEM_FLAG_DELETED)) {
+ !item_flag_check(item, ITEM_FLAG_DELETED))) {
// but, we can't use this item
if (having_index_lock) {
@@ -979,7 +967,7 @@ static int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_ITEM *it
dict_item_set_deleted(dict, item);
// decrement the refcount we incremented above
- if (__atomic_sub_fetch(&item->refcount, 1, __ATOMIC_SEQ_CST) == 0) {
+ if (__atomic_sub_fetch(&item->refcount, 1, __ATOMIC_RELEASE) == 0) {
// this is a deleted item, and we are the last one
DICTIONARY_PENDING_DELETES_PLUS1(dict);
}
@@ -988,7 +976,7 @@ static int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_ITEM *it
} else {
// this is traversal / walkthrough
// decrement the refcount we incremented above
- __atomic_sub_fetch(&item->refcount, 1, __ATOMIC_SEQ_CST);
+ __atomic_sub_fetch(&item->refcount, 1, __ATOMIC_RELEASE);
}
return RC_ITEM_MARKED_FOR_DELETION;
@@ -998,7 +986,6 @@ static int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_ITEM *it
DICTIONARY_REFERENCED_ITEMS_PLUS1(dict);
}
-
if(unlikely(spins > 1 && dict->stats))
DICTIONARY_STATS_CHECK_SPINS_PLUS(dict, spins - 1);
@@ -1037,7 +1024,7 @@ static inline int item_is_not_referenced_and_can_be_removed_advanced(DICTIONARY
ret = RC_ITEM_IS_CURRENTLY_BEING_CREATED;
break;
}
- } while(!__atomic_compare_exchange_n(&item->refcount, &refcount, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST));
+ } while(!__atomic_compare_exchange_n(&item->refcount, &refcount, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED));
#ifdef NETDATA_INTERNAL_CHECKS
if(ret == RC_ITEM_OK)
@@ -1055,8 +1042,8 @@ static inline int item_is_not_referenced_and_can_be_removed_advanced(DICTIONARY
static inline bool item_shared_release_and_check_if_it_can_be_freed(DICTIONARY *dict __maybe_unused, DICTIONARY_ITEM *item) {
// if we can set refcount to REFCOUNT_DELETING, we can delete this item
- REFCOUNT links = __atomic_sub_fetch(&item->shared->links, 1, __ATOMIC_SEQ_CST);
- if(links == 0 && __atomic_compare_exchange_n(&item->shared->links, &links, REFCOUNT_DELETING, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
+ REFCOUNT links = __atomic_sub_fetch(&item->shared->links, 1, __ATOMIC_RELEASE);
+ if(links == 0 && __atomic_compare_exchange_n(&item->shared->links, &links, REFCOUNT_DELETING, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
// we can delete it
return true;
@@ -1290,7 +1277,7 @@ static DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused, size_t
if(master_item) {
item->shared = master_item->shared;
- if(unlikely(__atomic_add_fetch(&item->shared->links, 1, __ATOMIC_SEQ_CST) <= 1))
+ if(unlikely(__atomic_add_fetch(&item->shared->links, 1, __ATOMIC_ACQUIRE) <= 1))
fatal("DICTIONARY: attempted to link to a shared item structure that had zero references");
}
else {
@@ -1478,7 +1465,7 @@ static void dict_item_shared_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item
item_shared_flag_set(item, ITEM_FLAG_DELETED);
if(dict->hooks)
- __atomic_store_n(&dict->hooks->last_master_deletion_us, now_realtime_usec(), __ATOMIC_SEQ_CST);
+ __atomic_store_n(&dict->hooks->last_master_deletion_us, now_realtime_usec(), __ATOMIC_RELAXED);
}
}
@@ -1486,7 +1473,7 @@ static void dict_item_shared_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item
static bool dict_item_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
ITEM_FLAGS expected, desired;
- expected = __atomic_load_n(&item->flags, __ATOMIC_SEQ_CST);
+ expected = __atomic_load_n(&item->flags, __ATOMIC_RELAXED);
do {
@@ -1495,7 +1482,7 @@ static bool dict_item_set_deleted(DICTIONARY *dict, DICTIONARY_ITEM *item) {
desired = expected | ITEM_FLAG_DELETED;
- } while(!__atomic_compare_exchange_n(&item->flags, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST));
+ } while(!__atomic_compare_exchange_n(&item->flags, &expected, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED));
DICTIONARY_ENTRIES_MINUS1(dict);
return true;
@@ -2063,11 +2050,11 @@ DICTIONARY *dictionary_create_view(DICTIONARY *master) {
dictionary_hooks_allocate(master);
- if(unlikely(__atomic_load_n(&master->hooks->links, __ATOMIC_SEQ_CST)) < 1)
+ if(unlikely(__atomic_load_n(&master->hooks->links, __ATOMIC_RELAXED)) < 1)
fatal("DICTIONARY: attempted to create a view that has %d links", master->hooks->links);
dict->hooks = master->hooks;
- __atomic_add_fetch(&master->hooks->links, 1, __ATOMIC_SEQ_CST);
+ __atomic_add_fetch(&master->hooks->links, 1, __ATOMIC_ACQUIRE);
#ifdef NETDATA_INTERNAL_CHECKS
dict->creation_function = function;
@@ -2167,6 +2154,8 @@ DICT_ITEM_CONST DICTIONARY_ITEM *dictionary_view_set_and_acquire_item_advanced(D
if(unlikely(is_master_dictionary(dict)))
fatal("DICTIONARY: this dictionary is a master, you cannot add items from other dictionaries.");
+ garbage_collect_pending_deletes(dict);
+
dictionary_acquired_item_dup(dict->master, master_item);
DICTIONARY_ITEM *item = dict_item_add_or_reset_value_and_acquire(dict, name, name_len, NULL, 0, NULL, master_item);
dictionary_acquired_item_release(dict->master, master_item);
@@ -2289,7 +2278,7 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {
dfe->counter = 0;
dfe->dict = dict;
dfe->rw = rw;
-
+ dfe->locked = true;
ll_recursive_lock(dict, dfe->rw);
DICTIONARY_STATS_TRAVERSALS_PLUS1(dict);
@@ -2312,8 +2301,10 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {
dfe->value = NULL;
}
- if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))
+ if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) {
ll_recursive_unlock(dfe->dict, dfe->rw);
+ dfe->locked = false;
+ }
return dfe->value;
}
@@ -2329,8 +2320,10 @@ void *dictionary_foreach_next(DICTFE *dfe) {
return NULL;
}
- if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))
+ if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT) || !dfe->locked) {
ll_recursive_lock(dfe->dict, dfe->rw);
+ dfe->locked = true;
+ }
// the item we just did
DICTIONARY_ITEM *item = dfe->item;
@@ -2360,12 +2353,21 @@ void *dictionary_foreach_next(DICTFE *dfe) {
dfe->value = NULL;
}
- if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT))
+ if(unlikely(dfe->rw == DICTIONARY_LOCK_REENTRANT)) {
ll_recursive_unlock(dfe->dict, dfe->rw);
+ dfe->locked = false;
+ }
return dfe->value;
}
+void dictionary_foreach_unlock(DICTFE *dfe) {
+ if(dfe->locked) {
+ ll_recursive_unlock(dfe->dict, dfe->rw);
+ dfe->locked = false;
+ }
+}
+
void dictionary_foreach_done(DICTFE *dfe) {
if(unlikely(!dfe || !dfe->dict)) return;
@@ -2383,8 +2385,10 @@ void dictionary_foreach_done(DICTFE *dfe) {
// item_release(dfe->dict, item);
}
- if(likely(dfe->rw != DICTIONARY_LOCK_REENTRANT))
+ if(likely(dfe->rw != DICTIONARY_LOCK_REENTRANT) && dfe->locked) {
ll_recursive_unlock(dfe->dict, dfe->rw);
+ dfe->locked = false;
+ }
dfe->dict = NULL;
dfe->item = NULL;
@@ -2472,7 +2476,7 @@ int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(
DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict);
ll_recursive_lock(dict, rw);
- size_t entries = __atomic_load_n(&dict->entries, __ATOMIC_SEQ_CST);
+ size_t entries = __atomic_load_n(&dict->entries, __ATOMIC_RELAXED);
DICTIONARY_ITEM **array = mallocz(sizeof(DICTIONARY_ITEM *) * entries);
size_t i;
@@ -3293,12 +3297,12 @@ static void *unittest_dict_master_thread(void *arg) {
DICTIONARY_ITEM *item = NULL;
int loops = 0;
- while(!__atomic_load_n(&tv->join, __ATOMIC_SEQ_CST)) {
+ while(!__atomic_load_n(&tv->join, __ATOMIC_RELAXED)) {
if(!item)
item = dictionary_set_and_acquire_item(tv->master, "ITEM1", "123", strlen("123") + 1);
- if(__atomic_load_n(&tv->item_master, __ATOMIC_SEQ_CST) != NULL) {
+ if(__atomic_load_n(&tv->item_master, __ATOMIC_RELAXED) != NULL) {
dictionary_acquired_item_release(tv->master, item);
dictionary_del(tv->master, "ITEM1");
item = NULL;
@@ -3307,7 +3311,7 @@ static void *unittest_dict_master_thread(void *arg) {
}
dictionary_acquired_item_dup(tv->master, item); // for the view thread
- __atomic_store_n(&tv->item_master, item, __ATOMIC_SEQ_CST);
+ __atomic_store_n(&tv->item_master, item, __ATOMIC_RELAXED);
dictionary_del(tv->master, "ITEM1");
@@ -3333,13 +3337,13 @@ static void *unittest_dict_view_thread(void *arg) {
DICTIONARY_ITEM *m_item = NULL;
- while(!__atomic_load_n(&tv->join, __ATOMIC_SEQ_CST)) {
- if(!(m_item = __atomic_load_n(&tv->item_master, __ATOMIC_SEQ_CST)))
+ while(!__atomic_load_n(&tv->join, __ATOMIC_RELAXED)) {
+ if(!(m_item = __atomic_load_n(&tv->item_master, __ATOMIC_RELAXED)))
continue;
DICTIONARY_ITEM *v_item = dictionary_view_set_and_acquire_item(tv->view, "ITEM2", m_item);
dictionary_acquired_item_release(tv->master, m_item);
- __atomic_store_n(&tv->item_master, NULL, __ATOMIC_SEQ_CST);
+ __atomic_store_n(&tv->item_master, NULL, __ATOMIC_RELAXED);
for(int i = 0; i < tv->dups ; i++) {
dictionary_acquired_item_dup(tv->view, v_item);
@@ -3351,7 +3355,7 @@ static void *unittest_dict_view_thread(void *arg) {
dictionary_del(tv->view, "ITEM2");
- while(!__atomic_load_n(&tv->join, __ATOMIC_SEQ_CST) && !(m_item = __atomic_load_n(&tv->item_master, __ATOMIC_SEQ_CST))) {
+ while(!__atomic_load_n(&tv->join, __ATOMIC_RELAXED) && !(m_item = __atomic_load_n(&tv->item_master, __ATOMIC_RELAXED))) {
dictionary_acquired_item_dup(tv->view, v_item);
dictionary_acquired_item_release(tv->view, v_item);
}
@@ -3522,7 +3526,7 @@ size_t dictionary_unittest_views(void) {
fprintf(stderr, "\nPASS 2: Deleting master item:\n");
dictionary_del(master, "KEY 1");
- dictionary_version(view);
+ garbage_collect_pending_deletes(view);
errors += unittest_check_dictionary("master", master, 0, 0, 1, 1, 0);
errors += unittest_check_dictionary("view", view, 0, 0, 1, 1, 0);
errors += unittest_check_item("master", master, item1_on_master, "KEY 1", item1_on_master->shared->value, 1, ITEM_FLAG_DELETED, false, false, true);