diff options
Diffstat (limited to 'libnetdata')
38 files changed, 11048 insertions, 760 deletions
diff --git a/libnetdata/Makefile.am b/libnetdata/Makefile.am index e85f4abe1..f01a1d34f 100644 --- a/libnetdata/Makefile.am +++ b/libnetdata/Makefile.am @@ -15,6 +15,7 @@ SUBDIRS = \ ebpf \ eval \ facets \ + functions_evloop \ json \ july \ health \ diff --git a/libnetdata/aral/aral.c b/libnetdata/aral/aral.c index 16328db69..7223ee359 100644 --- a/libnetdata/aral/aral.c +++ b/libnetdata/aral/aral.c @@ -1057,7 +1057,7 @@ int aral_stress_test(size_t threads, size_t elements, size_t seconds) { } netdata_log_info("ARAL: did %zu malloc, %zu free, " - "using %zu threads, in %llu usecs", + "using %zu threads, in %"PRIu64" usecs", auc.ar->aral_lock.user_malloc_operations, auc.ar->aral_lock.user_free_operations, threads, diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c index 2d09bb1ff..feee72fcf 100644 --- a/libnetdata/buffer/buffer.c +++ b/libnetdata/buffer/buffer.c @@ -20,16 +20,6 @@ void buffer_reset(BUFFER *wb) { buffer_overflow_check(wb); } -const char *buffer_tostring(BUFFER *wb) -{ - buffer_need_bytes(wb, 1); - wb->buffer[wb->len] = '\0'; - - buffer_overflow_check(wb); - - return(wb->buffer); -} - void buffer_char_replace(BUFFER *wb, char from, char to) { char *s = wb->buffer, *end = &wb->buffer[wb->len]; diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h index 0f1540287..26efe0070 100644 --- a/libnetdata/buffer/buffer.h +++ b/libnetdata/buffer/buffer.h @@ -97,7 +97,6 @@ typedef struct web_buffer { #define buffer_no_cacheable(wb) do { (wb)->options |= WB_CONTENT_NO_CACHEABLE; if((wb)->options & WB_CONTENT_CACHEABLE) (wb)->options &= ~WB_CONTENT_CACHEABLE; (wb)->expires = 0; } while(0) #define buffer_strlen(wb) ((wb)->len) -const char *buffer_tostring(BUFFER *wb); #define BUFFER_OVERFLOW_EOF "EOF" @@ -158,6 +157,16 @@ void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value void buffer_json_finalize(BUFFER *wb); +static const char *buffer_tostring(BUFFER *wb) +{ + buffer_need_bytes(wb, 1); + wb->buffer[wb->len] = '\0'; + + buffer_overflow_check(wb); + + return(wb->buffer); +} + static inline void _buffer_json_depth_push(BUFFER *wb, BUFFER_JSON_NODE_TYPE type) { #ifdef NETDATA_INTERNAL_CHECKS assert(wb->json.depth <= BUFFER_JSON_MAX_DEPTH && "BUFFER JSON: max nesting reached"); @@ -249,20 +258,25 @@ static inline void buffer_strcat(BUFFER *wb, const char *txt) { buffer_overflow_check(wb); } +static inline void buffer_contents_replace(BUFFER *wb, const char *txt, size_t len) { + wb->len = 0; + buffer_need_bytes(wb, len + 1); + + memcpy(wb->buffer, txt, len); + wb->len = len; + wb->buffer[wb->len] = '\0'; + + buffer_overflow_check(wb); +} + static inline void buffer_strncat(BUFFER *wb, const char *txt, size_t len) { if(unlikely(!txt || !*txt)) return; - const char *t = txt; buffer_need_bytes(wb, len + 1); - char *s = &wb->buffer[wb->len]; - char *d = s; - const char *e = &wb->buffer[wb->len + len]; - while(*t && d < e) - *d++ = *t++; - - wb->len += d - s; + memcpy(&wb->buffer[wb->len], txt, len); + wb->len += len; wb->buffer[wb->len] = '\0'; buffer_overflow_check(wb); @@ -944,10 +958,12 @@ typedef enum __attribute__((packed)) { RRDF_FIELD_OPTS_VISIBLE = (1 << 1), // the field should be visible by default RRDF_FIELD_OPTS_STICKY = (1 << 2), // the field should be sticky RRDF_FIELD_OPTS_FULL_WIDTH = (1 << 3), // the field should get full width - RRDF_FIELD_OPTS_WRAP = (1 << 4), // the field should get full width + RRDF_FIELD_OPTS_WRAP = (1 << 4), // the field should wrap + RRDR_FIELD_OPTS_DUMMY = (1 << 5), // not a presentable field } RRDF_FIELD_OPTIONS; typedef enum __attribute__((packed)) { + RRDF_FIELD_TYPE_NONE, RRDF_FIELD_TYPE_INTEGER, RRDF_FIELD_TYPE_STRING, RRDF_FIELD_TYPE_DETAIL_STRING, @@ -960,6 +976,9 @@ typedef enum __attribute__((packed)) { static inline const char *rrdf_field_type_to_string(RRDF_FIELD_TYPE type) { switch(type) { default: + case RRDF_FIELD_TYPE_NONE: + return "none"; + case RRDF_FIELD_TYPE_INTEGER: return "integer"; @@ -984,10 +1003,11 @@ static inline const char *rrdf_field_type_to_string(RRDF_FIELD_TYPE type) { } typedef enum __attribute__((packed)) { - RRDF_FIELD_VISUAL_VALUE, // show the value, possibly applying a transformation - RRDF_FIELD_VISUAL_BAR, // show the value and a bar, respecting the max field to fill the bar at 100% - RRDF_FIELD_VISUAL_PILL, // - RRDF_FIELD_VISUAL_MARKDOC, // + RRDF_FIELD_VISUAL_VALUE, // show the value, possibly applying a transformation + RRDF_FIELD_VISUAL_BAR, // show the value and a bar, respecting the max field to fill the bar at 100% + RRDF_FIELD_VISUAL_PILL, // + RRDF_FIELD_VISUAL_RICH, // + RRDR_FIELD_VISUAL_ROW_OPTIONS, // this is a dummy column that is used for row options } RRDF_FIELD_VISUAL; static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual) { @@ -1002,8 +1022,11 @@ static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual) case RRDF_FIELD_VISUAL_PILL: return "pill"; - case RRDF_FIELD_VISUAL_MARKDOC: - return "markdoc"; + case RRDF_FIELD_VISUAL_RICH: + return "richValue"; + + case RRDR_FIELD_VISUAL_ROW_OPTIONS: + return "rowOptions"; } } @@ -1150,6 +1173,9 @@ buffer_rrdf_table_add_field(BUFFER *wb, size_t field_id, const char *key, const buffer_json_member_add_boolean(wb, "full_width", options & RRDF_FIELD_OPTS_FULL_WIDTH); buffer_json_member_add_boolean(wb, "wrap", options & RRDF_FIELD_OPTS_WRAP); + + if(options & RRDR_FIELD_OPTS_DUMMY) + buffer_json_member_add_boolean(wb, "dummy", true); } buffer_json_object_close(wb); } diff --git a/libnetdata/clocks/clocks.c b/libnetdata/clocks/clocks.c index 806dc06a3..489e96855 100644 --- a/libnetdata/clocks/clocks.c +++ b/libnetdata/clocks/clocks.c @@ -299,7 +299,7 @@ usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) { usec_t tmp = (now_realtime_usec() * clock_realtime_resolution) % (tick / 2); error_limit_static_global_var(erl, 10, 0); - error_limit(&erl, "heartbeat randomness of %llu is too big for a tick of %llu - setting it to %llu", hb->randomness, tick, tmp); + error_limit(&erl, "heartbeat randomness of %"PRIu64" is too big for a tick of %"PRIu64" - setting it to %"PRIu64"", hb->randomness, tick, tmp); hb->randomness = tmp; } @@ -326,12 +326,12 @@ usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) { if(unlikely(now < next)) { errno = 0; error_limit_static_global_var(erl, 10, 0); - error_limit(&erl, "heartbeat clock: woke up %llu microseconds earlier than expected (can be due to the CLOCK_REALTIME set to the past).", next - now); + error_limit(&erl, "heartbeat clock: woke up %"PRIu64" microseconds earlier than expected (can be due to the CLOCK_REALTIME set to the past).", next - now); } else if(unlikely(now - next > tick / 2)) { errno = 0; error_limit_static_global_var(erl, 10, 0); - error_limit(&erl, "heartbeat clock: woke up %llu microseconds later than expected (can be due to system load or the CLOCK_REALTIME set to the future).", now - next); + error_limit(&erl, "heartbeat clock: woke up %"PRIu64" microseconds later than expected (can be due to system load or the CLOCK_REALTIME set to the future).", now - next); } if(unlikely(!hb->realtime)) { @@ -381,7 +381,7 @@ void sleep_usec_with_now(usec_t usec, usec_t started_ut) { } } else { - netdata_log_error("Cannot nanosleep() for %llu microseconds.", usec); + netdata_log_error("Cannot nanosleep() for %"PRIu64" microseconds.", usec); break; } } diff --git a/libnetdata/clocks/clocks.h b/libnetdata/clocks/clocks.h index b050b6254..5b88a4579 100644 --- a/libnetdata/clocks/clocks.h +++ b/libnetdata/clocks/clocks.h @@ -16,10 +16,10 @@ struct timespec { typedef int clockid_t; #endif -typedef unsigned long long nsec_t; -typedef unsigned long long msec_t; -typedef unsigned long long usec_t; -typedef long long susec_t; +typedef uint64_t nsec_t; +typedef uint64_t msec_t; +typedef uint64_t usec_t; +typedef int64_t susec_t; typedef struct heartbeat { usec_t realtime; diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c index 05da55344..a74a59583 100644 --- a/libnetdata/dictionary/dictionary.c +++ b/libnetdata/dictionary/dictionary.c @@ -250,6 +250,7 @@ static inline void pointer_del(DICTIONARY *dict __maybe_unused, DICTIONARY_ITEM // ---------------------------------------------------------------------------- // memory statistics +#ifdef DICT_WITH_STATS static inline void DICTIONARY_STATS_PLUS_MEMORY(DICTIONARY *dict, size_t key_size, size_t item_size, size_t value_size) { if(key_size) __atomic_fetch_add(&dict->stats->memory.index, (long)JUDYHS_INDEX_SIZE_ESTIMATE(key_size), __ATOMIC_RELAXED); @@ -260,6 +261,7 @@ static inline void DICTIONARY_STATS_PLUS_MEMORY(DICTIONARY *dict, size_t key_siz if(value_size) __atomic_fetch_add(&dict->stats->memory.values, (long)value_size, __ATOMIC_RELAXED); } + static inline void DICTIONARY_STATS_MINUS_MEMORY(DICTIONARY *dict, size_t key_size, size_t item_size, size_t value_size) { if(key_size) __atomic_fetch_sub(&dict->stats->memory.index, (long)JUDYHS_INDEX_SIZE_ESTIMATE(key_size), __ATOMIC_RELAXED); @@ -270,6 +272,10 @@ static inline void DICTIONARY_STATS_MINUS_MEMORY(DICTIONARY *dict, size_t key_si if(value_size) __atomic_fetch_sub(&dict->stats->memory.values, (long)value_size, __ATOMIC_RELAXED); } +#else +#define DICTIONARY_STATS_PLUS_MEMORY(dict, key_size, item_size, value_size) do {;} while(0) +#define DICTIONARY_STATS_MINUS_MEMORY(dict, key_size, item_size, value_size) do {;} while(0) +#endif // ---------------------------------------------------------------------------- // callbacks registration @@ -376,14 +382,21 @@ void dictionary_version_increment(DICTIONARY *dict) { // ---------------------------------------------------------------------------- // internal statistics API +#ifdef DICT_WITH_STATS static inline void DICTIONARY_STATS_SEARCHES_PLUS1(DICTIONARY *dict) { __atomic_fetch_add(&dict->stats->ops.searches, 1, __ATOMIC_RELAXED); } +#else +#define DICTIONARY_STATS_SEARCHES_PLUS1(dict) do {;} while(0) +#endif + static inline void DICTIONARY_ENTRIES_PLUS1(DICTIONARY *dict) { +#ifdef DICT_WITH_STATS // statistics __atomic_fetch_add(&dict->stats->items.entries, 1, __ATOMIC_RELAXED); __atomic_fetch_add(&dict->stats->items.referenced, 1, __ATOMIC_RELAXED); __atomic_fetch_add(&dict->stats->ops.inserts, 1, __ATOMIC_RELAXED); +#endif if(unlikely(is_dictionary_single_threaded(dict))) { dict->version++; @@ -397,10 +410,13 @@ static inline void DICTIONARY_ENTRIES_PLUS1(DICTIONARY *dict) { __atomic_fetch_add(&dict->referenced_items, 1, __ATOMIC_RELAXED); } } + static inline void DICTIONARY_ENTRIES_MINUS1(DICTIONARY *dict) { +#ifdef DICT_WITH_STATS // statistics __atomic_fetch_add(&dict->stats->ops.deletes, 1, __ATOMIC_RELAXED); __atomic_fetch_sub(&dict->stats->items.entries, 1, __ATOMIC_RELAXED); +#endif size_t entries; (void)entries; if(unlikely(is_dictionary_single_threaded(dict))) { @@ -418,14 +434,19 @@ static inline void DICTIONARY_ENTRIES_MINUS1(DICTIONARY *dict) { dict->creation_line, dict->creation_file); } + static inline void DICTIONARY_VALUE_RESETS_PLUS1(DICTIONARY *dict) { +#ifdef DICT_WITH_STATS __atomic_fetch_add(&dict->stats->ops.resets, 1, __ATOMIC_RELAXED); +#endif if(unlikely(is_dictionary_single_threaded(dict))) dict->version++; else __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED); } + +#ifdef DICT_WITH_STATS static inline void DICTIONARY_STATS_TRAVERSALS_PLUS1(DICTIONARY *dict) { __atomic_fetch_add(&dict->stats->ops.traversals, 1, __ATOMIC_RELAXED); } @@ -476,9 +497,29 @@ static inline void DICTIONARY_STATS_DICT_DESTROY_QUEUED_MINUS1(DICTIONARY *dict) static inline void DICTIONARY_STATS_DICT_FLUSHES_PLUS1(DICTIONARY *dict) { __atomic_fetch_add(&dict->stats->ops.flushes, 1, __ATOMIC_RELAXED); } +#else +#define DICTIONARY_STATS_TRAVERSALS_PLUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_CHECK_SPINS_PLUS(dict, count) do {;} while(0) +#define DICTIONARY_STATS_INSERT_SPINS_PLUS(dict, count) do {;} while(0) +#define DICTIONARY_STATS_DELETE_SPINS_PLUS(dict, count) do {;} while(0) +#define DICTIONARY_STATS_SEARCH_IGNORES_PLUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_CALLBACK_INSERTS_PLUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_CALLBACK_CONFLICTS_PLUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_CALLBACK_REACTS_PLUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_CALLBACK_DELETES_PLUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_GARBAGE_COLLECTIONS_PLUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_DICT_CREATIONS_PLUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_DICT_DESTRUCTIONS_PLUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_DICT_DESTROY_QUEUED_PLUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_DICT_DESTROY_QUEUED_MINUS1(dict) do {;} while(0) +#define DICTIONARY_STATS_DICT_FLUSHES_PLUS1(dict) do {;} while(0) +#endif static inline void DICTIONARY_REFERENCED_ITEMS_PLUS1(DICTIONARY *dict) { +#ifdef DICT_WITH_STATS __atomic_fetch_add(&dict->stats->items.referenced, 1, __ATOMIC_RELAXED); +#endif if(unlikely(is_dictionary_single_threaded(dict))) ++dict->referenced_items; @@ -487,7 +528,9 @@ static inline void DICTIONARY_REFERENCED_ITEMS_PLUS1(DICTIONARY *dict) { } static inline void DICTIONARY_REFERENCED_ITEMS_MINUS1(DICTIONARY *dict) { +#ifdef DICT_WITH_STATS __atomic_fetch_sub(&dict->stats->items.referenced, 1, __ATOMIC_RELAXED); +#endif long int referenced_items; (void)referenced_items; if(unlikely(is_dictionary_single_threaded(dict))) @@ -504,7 +547,9 @@ static inline void DICTIONARY_REFERENCED_ITEMS_MINUS1(DICTIONARY *dict) { } static inline void DICTIONARY_PENDING_DELETES_PLUS1(DICTIONARY *dict) { +#ifdef DICT_WITH_STATS __atomic_fetch_add(&dict->stats->items.pending_deletion, 1, __ATOMIC_RELAXED); +#endif if(unlikely(is_dictionary_single_threaded(dict))) ++dict->pending_deletion_items; @@ -513,7 +558,9 @@ static inline void DICTIONARY_PENDING_DELETES_PLUS1(DICTIONARY *dict) { } static inline long int DICTIONARY_PENDING_DELETES_MINUS1(DICTIONARY *dict) { +#ifdef DICT_WITH_STATS __atomic_fetch_sub(&dict->stats->items.pending_deletion, 1, __ATOMIC_RELEASE); +#endif if(unlikely(is_dictionary_single_threaded(dict))) return --dict->pending_deletion_items; @@ -977,7 +1024,7 @@ static int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_ITEM *it DICTIONARY_REFERENCED_ITEMS_PLUS1(dict); } - if(unlikely(spins > 1 && dict->stats)) + if(unlikely(spins > 1)) DICTIONARY_STATS_CHECK_SPINS_PLUS(dict, spins - 1); return ret; @@ -1022,7 +1069,7 @@ static inline int item_is_not_referenced_and_can_be_removed_advanced(DICTIONARY item->deleter_pid = gettid(); #endif - if(unlikely(spins > 1 && dict->stats)) + if(unlikely(spins > 1)) DICTIONARY_STATS_DELETE_SPINS_PLUS(dict, spins - 1); return ret; @@ -1535,22 +1582,6 @@ static inline void dict_item_release_and_check_if_it_is_deleted_and_can_be_remov } static bool dict_item_del(DICTIONARY *dict, const char *name, ssize_t name_len) { - if(unlikely(!name || !*name)) { - internal_error( - true, - "DICTIONARY: attempted to %s() without a name on a dictionary created from %s() %zu@%s.", - __FUNCTION__, - dict->creation_function, - dict->creation_line, - dict->creation_file); - return false; - } - - if(unlikely(is_dictionary_destroyed(dict))) { - internal_error(true, "DICTIONARY: attempted to dictionary_del() on a destroyed dictionary"); - return false; - } - if(name_len == -1) name_len = (ssize_t)strlen(name) + 1; // we need the terminating null too @@ -1695,7 +1726,7 @@ static DICTIONARY_ITEM *dict_item_add_or_reset_value_and_acquire(DICTIONARY *dic } while(!item); - if(unlikely(spins > 0 && dict->stats)) + if(unlikely(spins > 0)) DICTIONARY_STATS_INSERT_SPINS_PLUS(dict, spins); if(is_master_dictionary(dict) && added_or_updated) @@ -2064,11 +2095,15 @@ void dictionary_flush(DICTIONARY *dict) { if(unlikely(!dict)) return; - void *value; - dfe_start_write(dict, value) { - dictionary_del_advanced(dict, item_get_name(value_dfe.item), (ssize_t)item_get_name_len(value_dfe.item) + 1); + ll_recursive_lock(dict, DICTIONARY_LOCK_WRITE); + + DICTIONARY_ITEM *item, *next = NULL; + for(item = dict->items.list; item ;item = next) { + next = item->next; + dict_item_del(dict, item_get_name(item), (ssize_t) item_get_name_len(item) + 1); } - dfe_done(value); + + ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE); DICTIONARY_STATS_DICT_FLUSHES_PLUS1(dict); } @@ -2251,6 +2286,12 @@ bool dictionary_del_advanced(DICTIONARY *dict, const char *name, ssize_t name_le return false; api_internal_check(dict, NULL, false, true); + + if(unlikely(is_dictionary_destroyed(dict))) { + internal_error(true, "DICTIONARY: attempted to delete item on a destroyed dictionary"); + return false; + } + return dict_item_del(dict, name, name_len); } @@ -2260,6 +2301,8 @@ bool dictionary_del_advanced(DICTIONARY *dict, const char *name, ssize_t name_le void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) { if(unlikely(!dfe || !dict)) return NULL; + DICTIONARY_STATS_TRAVERSALS_PLUS1(dict); + if(unlikely(is_dictionary_destroyed(dict))) { internal_error(true, "DICTIONARY: attempted to dictionary_foreach_start_rw() on a destroyed dictionary"); dfe->counter = 0; @@ -2275,8 +2318,6 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) { dfe->locked = true; ll_recursive_lock(dict, dfe->rw); - DICTIONARY_STATS_TRAVERSALS_PLUS1(dict); - // get the first item from the list DICTIONARY_ITEM *item = dict->items.list; @@ -2836,7 +2877,7 @@ static usec_t dictionary_unittest_run_and_measure_time(DICTIONARY *dict, char *m } } - fprintf(stderr, " %zu errors, %d (found %ld) items in dictionary, %d (found %ld) referenced, %d (found %ld) deleted, %llu usec \n", + fprintf(stderr, " %zu errors, %d (found %ld) items in dictionary, %d (found %ld) referenced, %d (found %ld) deleted, %"PRIu64" usec \n", errs, dict?dict->entries:0, found_ok, dict?dict->referenced_items:0, found_referenced, dict?dict->pending_deletion_items:0, found_deleted, dt); *errors += errs; return dt; @@ -3129,6 +3170,9 @@ struct thread_unittest { int join; DICTIONARY *dict; int dups; + + netdata_thread_t thread; + struct dictionary_stats stats; }; static void *unittest_dict_thread(void *arg) { @@ -3140,46 +3184,59 @@ static void *unittest_dict_thread(void *arg) { DICT_ITEM_CONST DICTIONARY_ITEM *item = dictionary_set_and_acquire_item_advanced(tu->dict, "dict thread checking 1234567890", -1, NULL, 0, NULL); - + tu->stats.ops.inserts++; dictionary_get(tu->dict, dictionary_acquired_item_name(item)); + tu->stats.ops.searches++; void *t1; dfe_start_write(tu->dict, t1) { // this should delete the referenced item dictionary_del(tu->dict, t1_dfe.name); + tu->stats.ops.deletes++; void *t2; dfe_start_write(tu->dict, t2) { // this should add another dictionary_set(tu->dict, t2_dfe.name, NULL, 0); + tu->stats.ops.inserts++; dictionary_get(tu->dict, dictionary_acquired_item_name(item)); + tu->stats.ops.searches++; // and this should delete it again dictionary_del(tu->dict, t2_dfe.name); + tu->stats.ops.deletes++; } dfe_done(t2); + tu->stats.ops.traversals++; // this should fail to add it dictionary_set(tu->dict, t1_dfe.name, NULL, 0); + tu->stats.ops.inserts++; + dictionary_del(tu->dict, t1_dfe.name); + tu->stats.ops.deletes++; } dfe_done(t1); + tu->stats.ops.traversals++; for(int i = 0; i < tu->dups ; i++) { dictionary_acquired_item_dup(tu->dict, item); dictionary_get(tu->dict, dictionary_acquired_item_name(item)); + tu->stats.ops.searches++; } for(int i = 0; i < tu->dups ; i++) { dictionary_acquired_item_release(tu->dict, item); dictionary_del(tu->dict, dictionary_acquired_item_name(item)); + tu->stats.ops.deletes++; } dictionary_acquired_item_release(tu->dict, item); dictionary_del(tu->dict, "dict thread checking 1234567890"); + tu->stats.ops.deletes++; // test concurrent deletions and flushes { @@ -3189,16 +3246,19 @@ static void *unittest_dict_thread(void *arg) { for (int i = 0; i < 1000; i++) { snprintfz(buf, 256, "del/flush test %d", i); dictionary_set(tu->dict, buf, NULL, 0); + tu->stats.ops.inserts++; } for (int i = 0; i < 1000; i++) { snprintfz(buf, 256, "del/flush test %d", i); dictionary_del(tu->dict, buf); + tu->stats.ops.deletes++; } } else { for (int i = 0; i < 10; i++) { dictionary_flush(tu->dict); + tu->stats.ops.flushes++; } } } @@ -3208,47 +3268,75 @@ static void *unittest_dict_thread(void *arg) { } static int dictionary_unittest_threads() { - - struct thread_unittest tu = { - .join = 0, - .dict = NULL, - .dups = 1, - }; - - // threads testing of dictionary - tu.dict = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); time_t seconds_to_run = 5; int threads_to_create = 2; + + struct thread_unittest tu[threads_to_create]; + memset(tu, 0, sizeof(struct thread_unittest) * threads_to_create); + fprintf( - stderr, - "\nChecking dictionary concurrency with %d threads for %lld seconds...\n", - threads_to_create, - (long long)seconds_to_run); + stderr, + "\nChecking dictionary concurrency with %d threads for %lld seconds...\n", + threads_to_create, + (long long)seconds_to_run); + + // threads testing of dictionary + struct dictionary_stats stats = {}; + tu[0].join = 0; + tu[0].dups = 1; + tu[0].dict = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &stats, 0); - netdata_thread_t threads[threads_to_create]; - tu.join = 0; for (int i = 0; i < threads_to_create; i++) { + if(i) + tu[i] = tu[0]; + char buf[100 + 1]; snprintf(buf, 100, "dict%d", i); netdata_thread_create( - &threads[i], + &tu[i].thread, buf, NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, unittest_dict_thread, - &tu); + &tu[i]); } + sleep_usec(seconds_to_run * USEC_PER_SEC); - __atomic_store_n(&tu.join, 1, __ATOMIC_RELAXED); for (int i = 0; i < threads_to_create; i++) { + __atomic_store_n(&tu[i].join, 1, __ATOMIC_RELAXED); + void *retval; - netdata_thread_join(threads[i], &retval); + netdata_thread_join(tu[i].thread, &retval); + + if(i) { + tu[0].stats.ops.inserts += tu[i].stats.ops.inserts; + tu[0].stats.ops.deletes += tu[i].stats.ops.deletes; + tu[0].stats.ops.searches += tu[i].stats.ops.searches; + tu[0].stats.ops.flushes += tu[i].stats.ops.flushes; + tu[0].stats.ops.traversals += tu[i].stats.ops.traversals; + } } fprintf(stderr, - "inserts %zu" + "CALLS : inserts %zu" ", deletes %zu" ", searches %zu" + ", traversals %zu" + ", flushes %zu" + "\n", + tu[0].stats.ops.inserts, + tu[0].stats.ops.deletes, + tu[0].stats.ops.searches, + tu[0].stats.ops.traversals, + tu[0].stats.ops.flushes + ); + +#ifdef DICT_WITH_STATS + fprintf(stderr, + "ACTUAL: inserts %zu" + ", deletes %zu" + ", searches %zu" + ", traversals %zu" ", resets %zu" ", flushes %zu" ", entries %d" @@ -3259,22 +3347,23 @@ static int dictionary_unittest_threads() { ", delete spins %zu" ", search ignores %zu" "\n", - tu.dict->stats->ops.inserts, - tu.dict->stats->ops.deletes, - tu.dict->stats->ops.searches, - tu.dict->stats->ops.resets, - tu.dict->stats->ops.flushes, - tu.dict->entries, - tu.dict->referenced_items, - tu.dict->pending_deletion_items, - tu.dict->stats->spin_locks.use_spins, - tu.dict->stats->spin_locks.insert_spins, - tu.dict->stats->spin_locks.delete_spins, - tu.dict->stats->spin_locks.search_spins + stats.ops.inserts, + stats.ops.deletes, + stats.ops.searches, + stats.ops.traversals, + stats.ops.resets, + stats.ops.flushes, + tu[0].dict->entries, + tu[0].dict->referenced_items, + tu[0].dict->pending_deletion_items, + stats.spin_locks.use_spins, + stats.spin_locks.insert_spins, + stats.spin_locks.delete_spins, + stats.spin_locks.search_spins ); - dictionary_destroy(tu.dict); - tu.dict = NULL; +#endif + dictionary_destroy(tu[0].dict); return 0; } @@ -3407,6 +3496,7 @@ static int dictionary_unittest_view_threads() { netdata_thread_join(view_thread, &retval); netdata_thread_join(master_thread, &retval); +#ifdef DICT_WITH_STATS fprintf(stderr, "MASTER: inserts %zu" ", deletes %zu" @@ -3457,6 +3547,8 @@ static int dictionary_unittest_view_threads() { stats_view.spin_locks.delete_spins, stats_view.spin_locks.search_spins ); +#endif + dictionary_destroy(tv.master); dictionary_destroy(tv.view); diff --git a/libnetdata/dictionary/dictionary.h b/libnetdata/dictionary/dictionary.h index eea14d3fa..72efe1d03 100644 --- a/libnetdata/dictionary/dictionary.h +++ b/libnetdata/dictionary/dictionary.h @@ -35,6 +35,10 @@ * */ +#ifdef NETDATA_INTERNAL_CHECKS +#define DICT_WITH_STATS 1 +#endif + #ifdef DICTIONARY_INTERNALS #define DICTFE_CONST #define DICT_ITEM_CONST @@ -92,7 +96,7 @@ struct dictionary_stats { // memory struct { - long index; // bytes of keys indexed (indication of the index size) + long index; // bytes of keys indexed (indication of the index size) long values; // bytes of caller structures long dict; // bytes of the structures dictionary needs } memory; diff --git a/libnetdata/dyn_conf/README.md b/libnetdata/dyn_conf/README.md new file mode 100644 index 000000000..6c8127400 --- /dev/null +++ b/libnetdata/dyn_conf/README.md @@ -0,0 +1,167 @@ +# Netdata Dynamic Configuration + +Purpose of Netdata Dynamic Configuration is to allow configuration of select Netdata plugins and options through the +Netdata API and by extension by UI. + +## HTTP API documentation + +### Summary API + +For summary of all jobs and their statuses (for all children that stream to parent) use the following URL: + +| Method | Endpoint | Description | +|:-------:|-------------------------------|------------------------------------------------------------| +| **GET** | `api/v2/job_statuses` | list of Jobs | +| **GET** | `api/v2/job_statuses?grouped` | list of Jobs (hierarchical, grouped by host/plugin/module) | + +### Dyncfg API + +### Top level + +| Method | Endpoint | Description | +|:-------:|------------------|-----------------------------------------| +| **GET** | `/api/v2/config` | registered Plugins (sent DYNCFG_ENABLE) | + +### Plugin level + +| Method | Endpoint | Description | +|:-------:|-----------------------------------|------------------------------| +| **GET** | `/api/v2/config/[plugin]` | Plugin config | +| **PUT** | `/api/v2/config/[plugin]` | update Plugin config | +| **GET** | `/api/v2/config/[plugin]/modules` | Modules registered by Plugin | +| **GET** | `/api/v2/config/[plugin]/schema` | Plugin config schema | + +### Module level + +| Method | Endpoint | Description | +|:-------:|-----------------------------------------------|---------------------------| +| **GET** | `/api/v2/config/<plugin>/[module]` | Module config | +| **PUT** | `/api/v2/config/[plugin]/[module]` | update Module config | +| **GET** | `/api/v2/config/[plugin]/[module]/jobs` | Jobs registered by Module | +| **GET** | `/api/v2/config/[plugin]/[module]/job_schema` | Job config schema | +| **GET** | `/api/v2/config/[plugin]/[module]/schema` | Module config schema | + +### Job level - only for modules where `module_type == job_array` + +| Method | Endpoint | Description | +|:----------:|------------------------------------------|--------------------------------| +| **GET** | `/api/v2/config/[plugin]/[module]/[job]` | Job config | +| **PUT** | `/api/v2/config/[plugin]/[module]/[job]` | update Job config | +| **POST** | `/api/v2/config/[plugin]/[module]/[job]` | create Job | +| **DELETE** | `/api/v2/config/[plugin]/[module]/[job]` | delete Job (created by Dyncfg) | + +## AGENT<->PLUGIN interface documentation + +### 1. DYNCFG_ENABLE + +Plugin signifies to agent its ability to use new dynamic config and the name it wishes to use by sending + +``` +plugin->agent: +============= +DYNCFG_ENABLE [plugin_url_name] +``` + +This can be sent only once per lifetime of the plugin (at startup or later) sending it multiple times is considered a +protocol violation and plugin might get terminated. +After this command is sent the plugin has to be ready to accept all the new commands/keywords related to dynamic +configuration (this command lets agent know this plugin is dyncfg capable and wishes to use dyncfg functionality). + +After this command agent can call + +``` +agent->plugin: +============= +FUNCTION_PAYLOAD [UUID] 1 "set_plugin_config" +the new configuration +blah blah blah +FUNCTION_PAYLOAD_END + +plugin->agent: +============= +FUNCTION_RESULT_BEGIN [UUID] [(1/0)(accept/reject)] [text/plain] 5 +FUNCTION_RESULT_END +``` + +to set the new config which can be accepted/rejected by plugin by sending answer for this FUNCTION as it would with any +other regular function. + +The new `FUNCTION_PAYLOAD` command differs from regular `FUNCTION` command exclusively in its ability to send bigger +payloads (configuration file contents) to the plugin (not just parameters list). + +Agent can also call (after `DYNCFG_ENABLE`) + +``` +Agent->plugin: +============= +FUNCTION [UID] 1 "get_plugin_config" + +Plugin->agent: +============= +FUNCTION_RESULT_BEGIN [UID] 1 text/plain 5 +{ + "the currently used config from plugin" : "nice" +} +FUNCTION_RESULT_END +``` + +and + +``` +Agent->plugin: +============= +FUNCTION [UID] 1 "get_plugin_config_schema" + +Plugin->agent: +============= +FUNCTION_RESULT_BEGIN [UID] 1 text/plain 5 +{ + "the schema of plugin configuration" : "splendid" +} +FUNCTION_RESULT_END +``` + +Plugin can also register zero, one or more configurable modules using: + +``` +plugin->agent: +============= +DYNCFG_REGISTER_MODULE [module_url_name] (job_array|single) +``` + +modules can be added any time during plugins lifetime (you are not required to add them all at startup). + +### 2. DYNCFG_REGISTER_MODULE + +Module has to choose one of following types at registration: + +- `single` - module itself has configuration but does not accept any jobs *(this is useful mainly for internal netdata + configurable things like webserver etc.)* +- `job_array` - module itself **can** *(not must)* have configuration and it has an array of jobs which can be added, + modified and deleted. **this is what plugin developer needs in most cases** + +After module has been registered agent can call + +- `set_module_config [module]` FUNCTION_PAYLOAD +- `get_module_config [module]` FUNCTION +- `get_module_config_schema [module]` FUNCTION + +with same syntax as `set_plugin_config` and `get_plugin_config`. In case of `set` command the plugin has ability to +reject the new configuration pushed to it. + +In a case the module was registered as `job_array` type following commands can be used to manage jobs: + +### 3. Job interface for job_array modules + +- `get_job_config_schema [module]` - FUNCTION +- `get_job_config [module] [job]` - FUNCTION +- `set_job_config [module] [job]` - FUNCTION_PAYLOAD +- `delete_job_name [module] [job]` - FUNCTION + +### 4. Streaming + +When above commands are transferred trough streaming additionally `plugin_name` is prefixed as first parameter. This is +done to allow routing to appropriate plugin @child. + +As a plugin developer you don't need to concern yourself with this detail as that parameter is stripped when sent to the +plugin *(and added when sent trough streaming)* automagically. diff --git a/libnetdata/dyn_conf/dyn_conf.c b/libnetdata/dyn_conf/dyn_conf.c index 00289fdf5..ee4b4733a 100644 --- a/libnetdata/dyn_conf/dyn_conf.c +++ b/libnetdata/dyn_conf/dyn_conf.c @@ -3,7 +3,7 @@ #include "dyn_conf.h" #define DYN_CONF_PATH_MAX (4096) -#define DYN_CONF_DIR VARLIB_DIR "/etc" +#define DYN_CONF_DIR VARLIB_DIR "/dynconf" #define DYN_CONF_JOB_SCHEMA "job_schema" #define DYN_CONF_SCHEMA "schema" @@ -11,9 +11,20 @@ #define DYN_CONF_JOB_LIST "jobs" #define DYN_CONF_CFG_EXT ".cfg" -DICTIONARY *plugins_dict = NULL; +void job_flags_wallkthrough(dyncfg_job_flg_t flags, void (*cb)(const char *str, void *data), void *data) +{ + if (flags & JOB_FLG_PS_LOADED) + cb("JOB_FLG_PS_LOADED", data); + if (flags & JOB_FLG_PLUGIN_PUSHED) + cb("JOB_FLG_PLUGIN_PUSHED", data); + if (flags & JOB_FLG_STREAMING_PUSHED) + cb("JOB_FLG_STREAMING_PUSHED", data); + if (flags & JOB_FLG_USER_CREATED) + cb("JOB_FLG_USER_CREATED", data); +} struct deferred_cfg_send { + DICTIONARY *plugins_dict; char *plugin_name; char *module_name; char *job_name; @@ -33,7 +44,7 @@ static void deferred_config_free(struct deferred_cfg_send *dcs) freez(dcs); } -static void deferred_config_push_back(const char *plugin_name, const char *module_name, const char *job_name) +static void deferred_config_push_back(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name) { struct deferred_cfg_send *deferred = callocz(1, sizeof(struct deferred_cfg_send)); deferred->plugin_name = strdupz(plugin_name); @@ -42,6 +53,7 @@ static void deferred_config_push_back(const char *plugin_name, const char *modul if (job_name != NULL) deferred->job_name = strdupz(job_name); } + deferred->plugins_dict = plugins_dict; pthread_mutex_lock(&deferred_configs_lock); if (dyncfg_shutdown) { pthread_mutex_unlock(&deferred_configs_lock); @@ -95,7 +107,7 @@ static int _get_list_of_plugins_json_cb(const DICTIONARY_ITEM *item, void *entry return 0; } -json_object *get_list_of_plugins_json() +json_object *get_list_of_plugins_json(DICTIONARY *plugins_dict) { json_object *obj = json_object_new_array(); @@ -114,18 +126,7 @@ static int _get_list_of_modules_json_cb(const DICTIONARY_ITEM *item, void *entry json_object *json_item = json_object_new_string(module->name); json_object_object_add(json_module, "name", json_item); - const char *module_type; - switch (module->type) { - case MOD_TYPE_SINGLE: - module_type = "single"; - break; - case MOD_TYPE_ARRAY: - module_type = "job_array"; - break; - default: - module_type = "unknown"; - break; - } + const char *module_type = module_type2str(module->type); json_item = json_object_new_string(module_type); json_object_object_add(json_module, "type", json_item); @@ -163,17 +164,31 @@ const char *job_status2str(enum job_status status) } } -static int _get_list_of_jobs_json_cb(const DICTIONARY_ITEM *item, void *entry, void *data) +static void _job_flags2str_cb(const char *str, void *data) { - UNUSED(item); - json_object *obj = (json_object *)data; - struct job *job = (struct job *)entry; + json_object *json_item = json_object_new_string(str); + json_object_array_add((json_object *)data, json_item); +} +json_object *job2json(struct job *job) { json_object *json_job = json_object_new_object(); + json_object *json_item = json_object_new_string(job->name); json_object_object_add(json_job, "name", json_item); + + json_item = json_object_new_string(job_type2str(job->type)); + json_object_object_add(json_job, "type", json_item); + + netdata_mutex_lock(&job->lock); json_item = json_object_new_string(job_status2str(job->status)); + json_object_object_add(json_job, "status", json_item); + + json_item = json_object_new_int(job->state); json_object_object_add(json_job, "state", json_item); + + json_item = job->reason == NULL ? NULL : json_object_new_string(job->reason); + json_object_object_add(json_job, "reason", json_item); + int64_t last_state_update_s = job->last_state_update / USEC_PER_SEC; int64_t last_state_update_us = job->last_state_update % USEC_PER_SEC; @@ -183,6 +198,22 @@ static int _get_list_of_jobs_json_cb(const DICTIONARY_ITEM *item, void *entry, v json_item = json_object_new_int64(last_state_update_us); json_object_object_add(json_job, "last_state_update_us", json_item); + json_item = json_object_new_array(); + job_flags_wallkthrough(job->flags, _job_flags2str_cb, json_item); + json_object_object_add(json_job, "flags", json_item); + + netdata_mutex_unlock(&job->lock); + + return json_job; +} + +static int _get_list_of_jobs_json_cb(const DICTIONARY_ITEM *item, void *entry, void *data) +{ + UNUSED(item); + json_object *obj = (json_object *)data; + + json_object *json_job = job2json((struct job *)entry); + json_object_array_add(obj, json_job); return 0; @@ -206,24 +237,59 @@ struct job *get_job_by_name(struct module *module, const char *job_name) return dictionary_get(module->jobs, job_name); } -int remove_job(struct module *module, struct job *job) +void unlink_job(const char *plugin_name, const char *module_name, const char *job_name) { // as we are going to do unlink here we better make sure we have all to build proper path - if (unlikely(job->name == NULL || module == NULL || module->name == NULL || module->plugin == NULL || module->plugin->name == NULL)) - return 0; + if (unlikely(job_name == NULL || module_name == NULL || plugin_name == NULL)) + return; + BUFFER *buffer = buffer_create(DYN_CONF_PATH_MAX, NULL); + buffer_sprintf(buffer, DYN_CONF_DIR "/%s/%s/%s" DYN_CONF_CFG_EXT, plugin_name, module_name, job_name); + if (unlink(buffer_tostring(buffer))) + netdata_log_error("Cannot remove file %s", buffer_tostring(buffer)); - enum set_config_result rc = module->delete_job_cb(module->job_config_cb_usr_ctx, module->name, job->name); + buffer_free(buffer); +} + +void delete_job(struct configurable_plugin *plugin, const char *module_name, const char *job_name) +{ + struct module *module = get_module_by_name(plugin, module_name); + if (module == NULL) { + error_report("DYNCFG module \"%s\" not found", module_name); + return; + } + + struct job *job_item = get_job_by_name(module, job_name); + if (job_item == NULL) { + error_report("DYNCFG job \"%s\" not found", job_name); + return; + } + + dictionary_del(module->jobs, job_name); +} + +void delete_job_pname(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name) +{ + const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, plugin_name); + if (plugin_item == NULL) { + error_report("DYNCFG plugin \"%s\" not found", plugin_name); + return; + } + struct configurable_plugin *plugin = dictionary_acquired_item_value(plugin_item); + + delete_job(plugin, module_name, job_name); + + dictionary_acquired_item_release(plugins_dict, plugin_item); +} + +int remove_job(struct module *module, struct job *job) +{ + enum set_config_result rc = module->delete_job_cb(module->job_config_cb_usr_ctx, module->plugin->name, module->name, job->name); if (rc != SET_CONFIG_ACCEPTED) { error_report("DYNCFG module \"%s\" rejected delete job for \"%s\"", module->name, job->name); return 0; } - - BUFFER *buffer = buffer_create(DYN_CONF_PATH_MAX, NULL); - buffer_sprintf(buffer, DYN_CONF_DIR "/%s/%s/%s" DYN_CONF_CFG_EXT, module->plugin->name, module->name, job->name); - unlink(buffer_tostring(buffer)); - buffer_free(buffer); - return dictionary_del(module->jobs, job->name); + return 1; } struct module *get_module_by_name(struct configurable_plugin *plugin, const char *module_name) @@ -231,7 +297,7 @@ struct module *get_module_by_name(struct configurable_plugin *plugin, const char return dictionary_get(plugin->modules, module_name); } -inline struct configurable_plugin *get_plugin_by_name(const char *name) +inline struct configurable_plugin *get_plugin_by_name(DICTIONARY *plugins_dict, const char *name) { return dictionary_get(plugins_dict, name); } @@ -282,6 +348,59 @@ static int store_config(const char *module_name, const char *submodule_name, con return 0; } +#ifdef NETDATA_DEV_MODE +#define netdata_dev_fatal(...) fatal(__VA_ARGS__) +#else +#define netdata_dev_fatal(...) error_report(__VA_ARGS__) +#endif + +void dyn_conf_store_config(const char *function, const char *payload, struct configurable_plugin *plugin) { + dyncfg_config_t config = { + .data = (char*)payload, + .data_size = strlen(payload) + }; + + char *fnc = strdupz(function); + // split fnc to words + char *words[DYNCFG_MAX_WORDS]; + size_t words_c = quoted_strings_splitter(fnc, words, DYNCFG_MAX_WORDS, isspace_map_pluginsd); + + const char *fnc_name = get_word(words, words_c, 0); + if (fnc_name == NULL) { + error_report("Function name expected \"%s\"", function); + goto CLEANUP; + } + if (strncmp(fnc_name, FUNCTION_NAME_SET_PLUGIN_CONFIG, strlen(FUNCTION_NAME_SET_PLUGIN_CONFIG)) == 0) { + store_config(plugin->name, NULL, NULL, config); + goto CLEANUP; + } + + if (words_c < 2) { + error_report("Module name expected \"%s\"", function); + goto CLEANUP; + } + const char *module_name = get_word(words, words_c, 1); + if (strncmp(fnc_name, FUNCTION_NAME_SET_MODULE_CONFIG, strlen(FUNCTION_NAME_SET_MODULE_CONFIG)) == 0) { + store_config(plugin->name, module_name, NULL, config); + goto CLEANUP; + } + + if (words_c < 3) { + error_report("Job name expected \"%s\"", function); + goto CLEANUP; + } + const char *job_name = get_word(words, words_c, 2); + if (strncmp(fnc_name, FUNCTION_NAME_SET_JOB_CONFIG, strlen(FUNCTION_NAME_SET_JOB_CONFIG)) == 0) { + store_config(plugin->name, module_name, job_name, config); + goto CLEANUP; + } + + netdata_dev_fatal("Unknown function \"%s\"", function); + +CLEANUP: + freez(fnc); +} + dyncfg_config_t load_config(const char *plugin_name, const char *module_name, const char *job_id) { BUFFER *filename = buffer_create(DYN_CONF_PATH_MAX, NULL); @@ -310,16 +429,12 @@ dyncfg_config_t load_config(const char *plugin_name, const char *module_name, co char *set_plugin_config(struct configurable_plugin *plugin, dyncfg_config_t cfg) { - enum set_config_result rc = plugin->set_config_cb(plugin->cb_usr_ctx, &cfg); + enum set_config_result rc = plugin->set_config_cb(plugin->cb_usr_ctx, plugin->name, &cfg); if (rc != SET_CONFIG_ACCEPTED) { error_report("DYNCFG plugin \"%s\" rejected config", plugin->name); return "plugin rejected config"; } - if (store_config(plugin->name, NULL, NULL, cfg)) { - error_report("DYNCFG could not store config for module \"%s\"", plugin->name); - return "could not store config on disk"; - } return NULL; } @@ -327,62 +442,38 @@ static char *set_module_config(struct module *mod, dyncfg_config_t cfg) { struct configurable_plugin *plugin = mod->plugin; - enum set_config_result rc = mod->set_config_cb(mod->config_cb_usr_ctx, mod->name, &cfg); + enum set_config_result rc = mod->set_config_cb(mod->config_cb_usr_ctx, plugin->name, mod->name, &cfg); if (rc != SET_CONFIG_ACCEPTED) { error_report("DYNCFG module \"%s\" rejected config", plugin->name); return "module rejected config"; } - if (store_config(plugin->name, mod->name, NULL, cfg)) { - error_report("DYNCFG could not store config for module \"%s\"", mod->name); - return "could not store config on disk"; - } - return NULL; } -struct job *job_new() +struct job *job_new(const char *job_id) { struct job *job = callocz(1, sizeof(struct job)); job->state = JOB_STATUS_UNKNOWN; job->last_state_update = now_realtime_usec(); + job->name = strdupz(job_id); + netdata_mutex_init(&job->lock); return job; } -static int set_job_config(struct job *job, dyncfg_config_t cfg) +static inline void job_del(struct job *job) { - struct module *mod = job->module; - enum set_config_result rt = mod->set_job_config_cb(mod->job_config_cb_usr_ctx, mod->name, job->name, &cfg); - - if (rt != SET_CONFIG_ACCEPTED) { - error_report("DYNCFG module \"%s\" rejected config for job \"%s\"", mod->name, job->name); - return 1; - } - - if (store_config(mod->plugin->name, mod->name, job->name, cfg)) { - error_report("DYNCFG could not store config for module \"%s\"", mod->name); - return 1; - } - - return 0; + netdata_mutex_destroy(&job->lock); + freez(job->reason); + freez((void*)job->name); + freez(job); } -struct job *add_job(struct module *mod, const char *job_id, dyncfg_config_t cfg) +void job_del_cb(const DICTIONARY_ITEM *item, void *value, void *data) { - struct job *job = job_new(); - job->name = strdupz(job_id); - job->module = mod; - - if (set_job_config(job, cfg)) { - freez(job->name); - freez(job); - return NULL; - } - - dictionary_set(mod->jobs, job->name, job, sizeof(job)); - - return job; - + UNUSED(item); + UNUSED(data); + job_del((struct job *)value); } void module_del_cb(const DICTIONARY_ITEM *item, void *value, void *data) @@ -395,10 +486,9 @@ void module_del_cb(const DICTIONARY_ITEM *item, void *value, void *data) freez(mod); } - -const DICTIONARY_ITEM *register_plugin(struct configurable_plugin *plugin) +const DICTIONARY_ITEM *register_plugin(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, bool localhost) { - if (get_plugin_by_name(plugin->name) != NULL) { + if (get_plugin_by_name(plugins_dict, plugin->name) != NULL) { error_report("DYNCFG plugin \"%s\" already registered", plugin->name); return NULL; } @@ -413,7 +503,8 @@ const DICTIONARY_ITEM *register_plugin(struct configurable_plugin *plugin) plugin->modules = dictionary_create(DICT_OPTION_VALUE_LINK_DONT_CLONE); dictionary_register_delete_callback(plugin->modules, module_del_cb, NULL); - deferred_config_push_back(plugin->name, NULL, NULL); + if (localhost) + deferred_config_push_back(plugins_dict, plugin->name, NULL, NULL); dictionary_set(plugins_dict, plugin->name, plugin, sizeof(plugin)); @@ -421,24 +512,14 @@ const DICTIONARY_ITEM *register_plugin(struct configurable_plugin *plugin) return dictionary_get_and_acquire_item(plugins_dict, plugin->name); } -void unregister_plugin(const DICTIONARY_ITEM *plugin) +void unregister_plugin(DICTIONARY *plugins_dict, const DICTIONARY_ITEM *plugin) { struct configurable_plugin *plug = dictionary_acquired_item_value(plugin); dictionary_acquired_item_release(plugins_dict, plugin); dictionary_del(plugins_dict, plug->name); } -void job_del_cb(const DICTIONARY_ITEM *item, void *value, void *data) -{ - UNUSED(item); - UNUSED(data); - struct job *job = (struct job *)value; - freez(job->reason); - freez(job->name); - freez(job); -} - -int register_module(struct configurable_plugin *plugin, struct module *module) +int register_module(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, struct module *module, bool localhost) { if (get_module_by_name(plugin, module->name) != NULL) { error_report("DYNCFG module \"%s\" already registered", module->name); @@ -447,7 +528,8 @@ int register_module(struct configurable_plugin *plugin, struct module *module) pthread_mutex_init(&module->lock, NULL); - deferred_config_push_back(plugin->name, module->name, NULL); + if (localhost) + deferred_config_push_back(plugins_dict, plugin->name, module->name, NULL); module->plugin = plugin; @@ -455,34 +537,38 @@ int register_module(struct configurable_plugin *plugin, struct module *module) module->jobs = dictionary_create(DICT_OPTION_VALUE_LINK_DONT_CLONE); dictionary_register_delete_callback(module->jobs, job_del_cb, NULL); - // load all jobs from disk - BUFFER *path = buffer_create(DYN_CONF_PATH_MAX, NULL); - buffer_sprintf(path, "%s/%s/%s", DYN_CONF_DIR, plugin->name, module->name); - DIR *dir = opendir(buffer_tostring(path)); - if (dir != NULL) { - struct dirent *ent; - while ((ent = readdir(dir)) != NULL) { - if (ent->d_name[0] == '.') - continue; - if (ent->d_type != DT_REG) - continue; - size_t len = strnlen(ent->d_name, NAME_MAX); - if (len <= strlen(DYN_CONF_CFG_EXT)) - continue; - if (strcmp(ent->d_name + len - strlen(DYN_CONF_CFG_EXT), DYN_CONF_CFG_EXT) != 0) - continue; - ent->d_name[len - strlen(DYN_CONF_CFG_EXT)] = '\0'; - - struct job *job = job_new(); - job->name = strdupz(ent->d_name); - job->module = module; - dictionary_set(module->jobs, job->name, job, sizeof(job)); - - deferred_config_push_back(plugin->name, module->name, job->name); + if (localhost) { + // load all jobs from disk + BUFFER *path = buffer_create(DYN_CONF_PATH_MAX, NULL); + buffer_sprintf(path, "%s/%s/%s", DYN_CONF_DIR, plugin->name, module->name); + DIR *dir = opendir(buffer_tostring(path)); + if (dir != NULL) { + struct dirent *ent; + while ((ent = readdir(dir)) != NULL) { + if (ent->d_name[0] == '.') + continue; + if (ent->d_type != DT_REG) + continue; + size_t len = strnlen(ent->d_name, NAME_MAX); + if (len <= strlen(DYN_CONF_CFG_EXT)) + continue; + if (strcmp(ent->d_name + len - strlen(DYN_CONF_CFG_EXT), DYN_CONF_CFG_EXT) != 0) + continue; + ent->d_name[len - strlen(DYN_CONF_CFG_EXT)] = '\0'; + + struct job *job = job_new(ent->d_name); + job->module = module; + job->flags = JOB_FLG_PS_LOADED; + job->type = JOB_TYPE_USER; + + dictionary_set(module->jobs, job->name, job, sizeof(job)); + + deferred_config_push_back(plugins_dict, plugin->name, module->name, ent->d_name); + } + closedir(dir); } - closedir(dir); + buffer_free(path); } - buffer_free(path); } dictionary_set(plugin->modules, module->name, module, sizeof(module)); @@ -490,11 +576,49 @@ int register_module(struct configurable_plugin *plugin, struct module *module) return 0; } +int register_job(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_type job_type, dyncfg_job_flg_t flags, int ignore_existing) +{ + int rc = 1; + const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, plugin_name); + if (plugin_item == NULL) { + error_report("plugin \"%s\" not registered", plugin_name); + return rc; + } + struct configurable_plugin *plugin = dictionary_acquired_item_value(plugin_item); + struct module *mod = get_module_by_name(plugin, module_name); + if (mod == NULL) { + error_report("module \"%s\" not registered", module_name); + goto ERR_EXIT; + } + if (mod->type != MOD_TYPE_ARRAY) { + error_report("module \"%s\" is not an array", module_name); + goto ERR_EXIT; + } + if (get_job_by_name(mod, job_name) != NULL) { + if (!ignore_existing) + error_report("job \"%s\" already registered", job_name); + goto ERR_EXIT; + } + + struct job *job = job_new(job_name); + job->module = mod; + job->flags = flags; + job->type = job_type; + + dictionary_set(mod->jobs, job->name, job, sizeof(job)); + + rc = 0; +ERR_EXIT: + dictionary_acquired_item_release(plugins_dict, plugin_item); + return rc; +} + void freez_dyncfg(void *ptr) { freez(ptr); } -static void handle_dyncfg_root(struct uni_http_response *resp, int method) +#ifdef NETDATA_TEST_DYNCFG +static void handle_dyncfg_root(DICTIONARY *plugins_dict, struct uni_http_response *resp, int method) { if (method != HTTP_METHOD_GET) { resp->content = "method not allowed"; @@ -502,7 +626,7 @@ static void handle_dyncfg_root(struct uni_http_response *resp, int method) resp->status = HTTP_RESP_METHOD_NOT_ALLOWED; return; } - json_object *obj = get_list_of_plugins_json(); + json_object *obj = get_list_of_plugins_json(plugins_dict); json_object *wrapper = json_object_new_object(); json_object_object_add(wrapper, "configurable_plugins", obj); resp->content = strdupz(json_object_to_json_string_ext(wrapper, JSON_C_TO_STRING_PRETTY)); @@ -518,7 +642,7 @@ static void handle_plugin_root(struct uni_http_response *resp, int method, struc switch(method) { case HTTP_METHOD_GET: { - dyncfg_config_t cfg = plugin->get_config_cb(plugin->cb_usr_ctx); + dyncfg_config_t cfg = plugin->get_config_cb(plugin->cb_usr_ctx, plugin->name); resp->content = mallocz(cfg.data_size); memcpy(resp->content, cfg.data, cfg.data_size); resp->status = HTTP_RESP_OK; @@ -558,11 +682,12 @@ static void handle_plugin_root(struct uni_http_response *resp, int method, struc return; } } +#endif void handle_module_root(struct uni_http_response *resp, int method, struct configurable_plugin *plugin, const char *module, void *post_payload, size_t post_payload_size) { - if (strncmp(module, DYN_CONF_SCHEMA, strlen(DYN_CONF_SCHEMA)) == 0) { - dyncfg_config_t cfg = plugin->get_config_schema_cb(plugin->cb_usr_ctx); + if (strncmp(module, DYN_CONF_SCHEMA, sizeof(DYN_CONF_SCHEMA)) == 0) { + dyncfg_config_t cfg = plugin->get_config_schema_cb(plugin->cb_usr_ctx, plugin->name); resp->content = mallocz(cfg.data_size); memcpy(resp->content, cfg.data, cfg.data_size); resp->status = HTTP_RESP_OK; @@ -570,7 +695,7 @@ void handle_module_root(struct uni_http_response *resp, int method, struct confi resp->content_length = cfg.data_size; return; } - if (strncmp(module, DYN_CONF_MODULE_LIST, strlen(DYN_CONF_MODULE_LIST)) == 0) { + if (strncmp(module, DYN_CONF_MODULE_LIST, sizeof(DYN_CONF_MODULE_LIST)) == 0) { if (method != HTTP_METHOD_GET) { resp->content = "method not allowed (only GET)"; resp->content_length = strlen(resp->content); @@ -596,7 +721,7 @@ void handle_module_root(struct uni_http_response *resp, int method, struct confi return; } if (method == HTTP_METHOD_GET) { - dyncfg_config_t cfg = mod->get_config_cb(mod->config_cb_usr_ctx, mod->name); + dyncfg_config_t cfg = mod->get_config_cb(mod->config_cb_usr_ctx, plugin->name, mod->name); resp->content = mallocz(cfg.data_size); memcpy(resp->content, cfg.data, cfg.data_size); resp->status = HTTP_RESP_OK; @@ -651,8 +776,7 @@ static inline void _handle_job_root(struct uni_http_response *resp, int method, .data = post_payload, .data_size = post_payload_size }; - job = add_job(mod, job_id, cont); - if (job == NULL) { + if (mod->set_job_config_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name, job_id, &cont)) { resp->content = "failed to add job"; resp->content_length = strlen(resp->content); resp->status = HTTP_RESP_INTERNAL_SERVER_ERROR; @@ -672,7 +796,7 @@ static inline void _handle_job_root(struct uni_http_response *resp, int method, switch (method) { case HTTP_METHOD_GET: { - dyncfg_config_t cfg = mod->get_job_config_cb(mod->job_config_cb_usr_ctx, mod->name, job->name); + dyncfg_config_t cfg = mod->get_job_config_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name, job->name); resp->content = mallocz(cfg.data_size); memcpy(resp->content, cfg.data, cfg.data_size); resp->status = HTTP_RESP_OK; @@ -692,10 +816,11 @@ static inline void _handle_job_root(struct uni_http_response *resp, int method, .data = post_payload, .data_size = post_payload_size }; - if(set_job_config(job, cont)) { - resp->status = HTTP_RESP_BAD_REQUEST; + if (mod->set_job_config_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name, job->name, &cont) != SET_CONFIG_ACCEPTED) { + error_report("DYNCFG module \"%s\" rejected config for job \"%s\"", mod->name, job->name); resp->content = "failed to set job config"; resp->content_length = strlen(resp->content); + resp->status = HTTP_RESP_INTERNAL_SERVER_ERROR; return; } resp->status = HTTP_RESP_OK; @@ -726,8 +851,8 @@ static inline void _handle_job_root(struct uni_http_response *resp, int method, void handle_job_root(struct uni_http_response *resp, int method, struct module *mod, const char *job_id, void *post_payload, size_t post_payload_size) { - if (strncmp(job_id, DYN_CONF_SCHEMA, strlen(DYN_CONF_SCHEMA)) == 0) { - dyncfg_config_t cfg = mod->get_config_schema_cb(mod->config_cb_usr_ctx, mod->name); + if (strncmp(job_id, DYN_CONF_SCHEMA, sizeof(DYN_CONF_SCHEMA)) == 0) { + dyncfg_config_t cfg = mod->get_config_schema_cb(mod->config_cb_usr_ctx, mod->plugin->name, mod->name); resp->content = mallocz(cfg.data_size); memcpy(resp->content, cfg.data, cfg.data_size); resp->status = HTTP_RESP_OK; @@ -735,8 +860,8 @@ void handle_job_root(struct uni_http_response *resp, int method, struct module * resp->content_length = cfg.data_size; return; } - if (strncmp(job_id, DYN_CONF_JOB_SCHEMA, strlen(DYN_CONF_JOB_SCHEMA)) == 0) { - dyncfg_config_t cfg = mod->get_job_config_schema_cb(mod->job_config_cb_usr_ctx, mod->name); + if (strncmp(job_id, DYN_CONF_JOB_SCHEMA, sizeof(DYN_CONF_JOB_SCHEMA)) == 0) { + dyncfg_config_t cfg = mod->get_job_config_schema_cb(mod->job_config_cb_usr_ctx, mod->plugin->name, mod->name); resp->content = mallocz(cfg.data_size); memcpy(resp->content, cfg.data, cfg.data_size); resp->status = HTTP_RESP_OK; @@ -744,7 +869,7 @@ void handle_job_root(struct uni_http_response *resp, int method, struct module * resp->content_length = cfg.data_size; return; } - if (strncmp(job_id, DYN_CONF_JOB_LIST, strlen(DYN_CONF_JOB_LIST)) == 0) { + if (strncmp(job_id, DYN_CONF_JOB_LIST, sizeof(DYN_CONF_JOB_LIST)) == 0) { if (mod->type != MOD_TYPE_ARRAY) { resp->content = "module type is not job_array (can't get the list of jobs)"; resp->content_length = strlen(resp->content); @@ -776,7 +901,14 @@ void handle_job_root(struct uni_http_response *resp, int method, struct module * dictionary_acquired_item_release(mod->jobs, job_item); } -struct uni_http_response dyn_conf_process_http_request(int method, const char *plugin, const char *module, const char *job_id, void *post_payload, size_t post_payload_size) +struct uni_http_response dyn_conf_process_http_request( + DICTIONARY *plugins_dict __maybe_unused, + int method __maybe_unused, + const char *plugin __maybe_unused, + const char *module __maybe_unused, + const char *job_id __maybe_unused, + void *post_payload __maybe_unused, + size_t post_payload_size __maybe_unused) { struct uni_http_response resp = { .status = HTTP_RESP_INTERNAL_SERVER_ERROR, @@ -785,8 +917,14 @@ struct uni_http_response dyn_conf_process_http_request(int method, const char *p .content_free = NULL, .content_length = 0 }; +#ifndef NETDATA_TEST_DYNCFG + resp.content = "DYNCFG is disabled (as it is for now developer only feature). This will be enabled by default when ready for technical preview."; + resp.content_length = strlen(resp.content); + resp.status = HTTP_RESP_PRECOND_FAIL; + return resp; +#else if (plugin == NULL) { - handle_dyncfg_root(&resp, method); + handle_dyncfg_root(plugins_dict, &resp, method); return resp; } const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, plugin); @@ -814,9 +952,9 @@ struct uni_http_response dyn_conf_process_http_request(int method, const char *p goto EXIT_PLUGIN; } if (mod->type != MOD_TYPE_ARRAY) { - resp.content = "module is not array"; + resp.content = "400 - this module is not array type"; resp.content_length = strlen(resp.content); - resp.status = HTTP_RESP_NOT_FOUND; + resp.status = HTTP_RESP_BAD_REQUEST; goto EXIT_PLUGIN; } handle_job_root(&resp, method, mod, job_id, post_payload, post_payload_size); @@ -824,6 +962,7 @@ struct uni_http_response dyn_conf_process_http_request(int method, const char *p EXIT_PLUGIN: dictionary_acquired_item_release(plugins_dict, plugin_item); return resp; +#endif } void plugin_del_cb(const DICTIONARY_ITEM *item, void *value, void *data) @@ -836,41 +975,53 @@ void plugin_del_cb(const DICTIONARY_ITEM *item, void *value, void *data) freez(plugin); } -void report_job_status(struct configurable_plugin *plugin, const char *module_name, const char *job_name, enum job_status status, int status_code, char *reason) +// on failure - return NULL - all unlocked, nothing acquired +// on success - return pointer to job item - keep job and plugin acquired and locked!!! +// for caller convenience (to prevent another lock and races) +// caller is responsible to unlock the job and release it when not needed anymore +// this also avoids dependency creep +const DICTIONARY_ITEM *report_job_status_acq_lock(DICTIONARY *plugins_dict, const DICTIONARY_ITEM **plugin_acq_item, DICTIONARY **job_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_status status, int status_code, char *reason) { - const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(plugins_dict, plugin->name); - if (item == NULL) { - netdata_log_error("plugin %s not found", plugin->name); - return; + *plugin_acq_item = dictionary_get_and_acquire_item(plugins_dict, plugin_name); + if (*plugin_acq_item == NULL) { + netdata_log_error("plugin %s not found", plugin_name); + return NULL; } - struct configurable_plugin *plug = dictionary_acquired_item_value(item); + + struct configurable_plugin *plug = dictionary_acquired_item_value(*plugin_acq_item); struct module *mod = get_module_by_name(plug, module_name); if (mod == NULL) { netdata_log_error("module %s not found", module_name); - goto EXIT_PLUGIN; + dictionary_acquired_item_release(plugins_dict, *plugin_acq_item); + return NULL; } if (mod->type != MOD_TYPE_ARRAY) { netdata_log_error("module %s is not array", module_name); - goto EXIT_PLUGIN; + dictionary_acquired_item_release(plugins_dict, *plugin_acq_item); + return NULL; } + *job_dict = mod->jobs; const DICTIONARY_ITEM *job_item = dictionary_get_and_acquire_item(mod->jobs, job_name); if (job_item == NULL) { netdata_log_error("job %s not found", job_name); - goto EXIT_PLUGIN; + dictionary_acquired_item_release(plugins_dict, *plugin_acq_item); + return NULL; } struct job *job = dictionary_acquired_item_value(job_item); + + pthread_mutex_lock(&job->lock); job->status = status; job->state = status_code; if (job->reason != NULL) { freez(job->reason); } - job->reason = reason; + job->reason = reason != NULL ? strdupz(reason) : NULL; // reason is optional job->last_state_update = now_realtime_usec(); - dictionary_acquired_item_release(mod->jobs, job_item); + job->dirty = true; -EXIT_PLUGIN: - dictionary_acquired_item_release(plugins_dict, item); + // no unlock and acquired_item_release on success on purpose + return job_item; } int dyn_conf_init(void) @@ -882,9 +1033,6 @@ int dyn_conf_init(void) } } - plugins_dict = dictionary_create(DICT_OPTION_VALUE_LINK_DONT_CLONE); - dictionary_register_delete_callback(plugins_dict, plugin_del_cb, NULL); - return 0; } @@ -912,6 +1060,15 @@ void *dyncfg_main(void *ptr) while (!netdata_exit) { struct deferred_cfg_send *dcs = deferred_config_pop(ptr); + DICTIONARY *plugins_dict = dcs->plugins_dict; +#ifdef NETDATA_INTERNAL_CHECKS + if (plugins_dict == NULL) { + fatal("DYNCFG, plugins_dict is NULL"); + deferred_config_free(dcs); + continue; + } +#endif + const DICTIONARY_ITEM *plugin_item = dictionary_get_and_acquire_item(plugins_dict, dcs->plugin_name); if (plugin_item == NULL) { error_report("DYNCFG, plugin %s not found", dcs->plugin_name); @@ -922,21 +1079,21 @@ void *dyncfg_main(void *ptr) if (dcs->module_name == NULL) { dyncfg_config_t cfg = load_config(dcs->plugin_name, NULL, NULL); if (cfg.data != NULL) { - plugin->set_config_cb(plugin->cb_usr_ctx, &cfg); + plugin->set_config_cb(plugin->cb_usr_ctx, plugin->name, &cfg); freez(cfg.data); } } else if (dcs->job_name == NULL) { dyncfg_config_t cfg = load_config(dcs->plugin_name, dcs->module_name, NULL); if (cfg.data != NULL) { struct module *mod = get_module_by_name(plugin, dcs->module_name); - mod->set_config_cb(mod->config_cb_usr_ctx, mod->name, &cfg); + mod->set_config_cb(mod->config_cb_usr_ctx, plugin->name, mod->name, &cfg); freez(cfg.data); } } else { dyncfg_config_t cfg = load_config(dcs->plugin_name, dcs->module_name, dcs->job_name); if (cfg.data != NULL) { struct module *mod = get_module_by_name(plugin, dcs->module_name); - mod->set_job_config_cb(mod->job_config_cb_usr_ctx, mod->name, dcs->job_name, &cfg); + mod->set_job_config_cb(mod->job_config_cb_usr_ctx, plugin->name, mod->name, dcs->job_name, &cfg); freez(cfg.data); } } @@ -947,3 +1104,37 @@ void *dyncfg_main(void *ptr) netdata_thread_cleanup_pop(1); return NULL; } + +bool is_dyncfg_function(const char *function_name, uint8_t type) { + // TODO add hash to speed things up + if (type & (DYNCFG_FUNCTION_TYPE_GET | DYNCFG_FUNCTION_TYPE_REGULAR)) { + if (strncmp(function_name, FUNCTION_NAME_GET_PLUGIN_CONFIG, strlen(FUNCTION_NAME_GET_PLUGIN_CONFIG)) == 0) + return true; + if (strncmp(function_name, FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA, strlen(FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA)) == 0) + return true; + if (strncmp(function_name, FUNCTION_NAME_GET_MODULE_CONFIG, strlen(FUNCTION_NAME_GET_MODULE_CONFIG)) == 0) + return true; + if (strncmp(function_name, FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA, strlen(FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA)) == 0) + return true; + if (strncmp(function_name, FUNCTION_NAME_GET_JOB_CONFIG, strlen(FUNCTION_NAME_GET_JOB_CONFIG)) == 0) + return true; + if (strncmp(function_name, FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA, strlen(FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA)) == 0) + return true; + } + + if (type & (DYNCFG_FUNCTION_TYPE_SET | DYNCFG_FUNCTION_TYPE_PAYLOAD)) { + if (strncmp(function_name, FUNCTION_NAME_SET_PLUGIN_CONFIG, strlen(FUNCTION_NAME_SET_PLUGIN_CONFIG)) == 0) + return true; + if (strncmp(function_name, FUNCTION_NAME_SET_MODULE_CONFIG, strlen(FUNCTION_NAME_SET_MODULE_CONFIG)) == 0) + return true; + if (strncmp(function_name, FUNCTION_NAME_SET_JOB_CONFIG, strlen(FUNCTION_NAME_SET_JOB_CONFIG)) == 0) + return true; + } + + if (type & (DYNCFG_FUNCTION_TYPE_DELETE | DYNCFG_FUNCTION_TYPE_REGULAR)) { + if (strncmp(function_name, FUNCTION_NAME_DELETE_JOB, strlen(FUNCTION_NAME_DELETE_JOB)) == 0) + return true; + } + + return false; +} diff --git a/libnetdata/dyn_conf/dyn_conf.h b/libnetdata/dyn_conf/dyn_conf.h index f10ae6a12..f6a5fe49a 100644 --- a/libnetdata/dyn_conf/dyn_conf.h +++ b/libnetdata/dyn_conf/dyn_conf.h @@ -5,6 +5,21 @@ #include "../libnetdata.h" +#define FUNCTION_NAME_GET_PLUGIN_CONFIG "get_plugin_config" +#define FUNCTION_NAME_GET_PLUGIN_CONFIG_SCHEMA "get_plugin_config_schema" +#define FUNCTION_NAME_GET_MODULE_CONFIG "get_module_config" +#define FUNCTION_NAME_GET_MODULE_CONFIG_SCHEMA "get_module_config_schema" +#define FUNCTION_NAME_GET_JOB_CONFIG "get_job_config" +#define FUNCTION_NAME_GET_JOB_CONFIG_SCHEMA "get_job_config_schema" +#define FUNCTION_NAME_SET_PLUGIN_CONFIG "set_plugin_config" +#define FUNCTION_NAME_SET_MODULE_CONFIG "set_module_config" +#define FUNCTION_NAME_SET_JOB_CONFIG "set_job_config" +#define FUNCTION_NAME_DELETE_JOB "delete_job" + +#define DYNCFG_MAX_WORDS 5 + +#define DYNCFG_VFNC_RET_CFG_ACCEPTED (1) + enum module_type { MOD_TYPE_UNKNOWN = 0, MOD_TYPE_ARRAY, @@ -20,6 +35,18 @@ static inline enum module_type str2_module_type(const char *type_name) return MOD_TYPE_UNKNOWN; } +static inline const char *module_type2str(enum module_type type) +{ + switch (type) { + case MOD_TYPE_ARRAY: + return "job_array"; + case MOD_TYPE_SINGLE: + return "single"; + default: + return "unknown"; + } +} + struct dyncfg_config { void *data; size_t data_size; @@ -37,7 +64,7 @@ enum job_status { JOB_STATUS_ERROR }; -inline enum job_status str2job_state(const char *state_name) { +static inline enum job_status str2job_state(const char *state_name) { if (strcmp(state_name, "stopped") == 0) return JOB_STATUS_STOPPED; else if (strcmp(state_name, "running") == 0) @@ -47,24 +74,75 @@ inline enum job_status str2job_state(const char *state_name) { return JOB_STATUS_UNKNOWN; } +const char *job_status2str(enum job_status status); + enum set_config_result { SET_CONFIG_ACCEPTED = 0, SET_CONFIG_REJECTED, SET_CONFIG_DEFFER }; +typedef uint32_t dyncfg_job_flg_t; +enum job_flags { + JOB_FLG_PS_LOADED = 1 << 0, // PS abbr. Persistent Storage + JOB_FLG_PLUGIN_PUSHED = 1 << 1, // got it from plugin (e.g. autodiscovered job) + JOB_FLG_STREAMING_PUSHED = 1 << 2, // got it through streaming + JOB_FLG_USER_CREATED = 1 << 3, // user created this job during agent runtime +}; + +enum job_type { + JOB_TYPE_UNKNOWN = 0, + JOB_TYPE_STOCK = 1, + JOB_TYPE_USER = 2, + JOB_TYPE_AUTODISCOVERED = 3, +}; + +static inline const char* job_type2str(enum job_type type) +{ + switch (type) { + case JOB_TYPE_STOCK: + return "stock"; + case JOB_TYPE_USER: + return "user"; + case JOB_TYPE_AUTODISCOVERED: + return "autodiscovered"; + case JOB_TYPE_UNKNOWN: + default: + return "unknown"; + } +} + +static inline enum job_type str2job_type(const char *type_name) +{ + if (strcmp(type_name, "stock") == 0) + return JOB_TYPE_STOCK; + else if (strcmp(type_name, "user") == 0) + return JOB_TYPE_USER; + else if (strcmp(type_name, "autodiscovered") == 0) + return JOB_TYPE_AUTODISCOVERED; + error_report("Unknown job type: %s", type_name); + return JOB_TYPE_UNKNOWN; +} + struct job { - char *name; + const char *name; + enum job_type type; + struct module *module; + + pthread_mutex_t lock; + // lock protexts only fields below (which are modified during job existence) + // others are static during lifetime of job - //state reported by config + int dirty; // this relates to rrdpush, true if parent has different data than us + + // state reported by plugin + usec_t last_state_update; enum job_status status; // reported by plugin, enum as this has to be interpreted by UI int state; // code reported by plugin which can mean anything plugin wants char *reason; // reported by plugin, can be NULL (optional) - usec_t last_state_update; - - struct module *module; + dyncfg_job_flg_t flags; }; struct module @@ -76,18 +154,18 @@ struct module struct configurable_plugin *plugin; // module config - enum set_config_result (*set_config_cb)(void *usr_ctx, const char *module_name, dyncfg_config_t *cfg); - dyncfg_config_t (*get_config_cb)(void *usr_ctx, const char *name); - dyncfg_config_t (*get_config_schema_cb)(void *usr_ctx, const char *name); + enum set_config_result (*set_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, dyncfg_config_t *cfg); + dyncfg_config_t (*get_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name); + dyncfg_config_t (*get_config_schema_cb)(void *usr_ctx, const char *plugin_name, const char *module_name); void *config_cb_usr_ctx; DICTIONARY *jobs; // jobs config - dyncfg_config_t (*get_job_config_cb)(void *usr_ctx, const char *module_name, const char *job_name); - dyncfg_config_t (*get_job_config_schema_cb)(void *usr_ctx, const char *module_name); - enum set_config_result (*set_job_config_cb)(void *usr_ctx, const char *module_name, const char *job_name, dyncfg_config_t *cfg); - enum set_config_result (*delete_job_cb)(void *usr_ctx, const char *module_name, const char *job_name); + dyncfg_config_t (*get_job_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name); + dyncfg_config_t (*get_job_config_schema_cb)(void *usr_ctx, const char *plugin_name, const char *module_name); + enum set_config_result (*set_job_config_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name, dyncfg_config_t *cfg); + enum set_config_result (*delete_job_cb)(void *usr_ctx, const char *plugin_name, const char *module_name, const char *job_name); void *job_config_cb_usr_ctx; }; @@ -97,26 +175,34 @@ struct configurable_plugin { DICTIONARY *modules; const char *schema; - dyncfg_config_t (*get_config_cb)(void *usr_ctx); - dyncfg_config_t (*get_config_schema_cb)(void *usr_ctx); - enum set_config_result (*set_config_cb)(void *usr_ctx, dyncfg_config_t *cfg); + dyncfg_config_t (*get_config_cb)(void *usr_ctx, const char *plugin_name); + dyncfg_config_t (*get_config_schema_cb)(void *usr_ctx, const char *plugin_name); + enum set_config_result (*set_config_cb)(void *usr_ctx, const char *plugin_name, dyncfg_config_t *cfg); void *cb_usr_ctx; // context for all callbacks (split if needed in future) }; // API to be used by plugins -const DICTIONARY_ITEM *register_plugin(struct configurable_plugin *plugin); -void unregister_plugin(const DICTIONARY_ITEM *plugin); -int register_module(struct configurable_plugin *plugin, struct module *module); +const DICTIONARY_ITEM *register_plugin(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, bool localhost); +void unregister_plugin(DICTIONARY *plugins_dict, const DICTIONARY_ITEM *plugin); +int register_module(DICTIONARY *plugins_dict, struct configurable_plugin *plugin, struct module *module, bool localhost); +int register_job(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_type job_type, dyncfg_job_flg_t flags, int ignore_existing); + +const DICTIONARY_ITEM *report_job_status_acq_lock(DICTIONARY *plugins_dict, const DICTIONARY_ITEM **plugin_acq_item, DICTIONARY **job_dict, const char *plugin_name, const char *module_name, const char *job_name, enum job_status status, int status_code, char *reason); -void report_job_status(struct configurable_plugin *plugin, const char *module_name, const char *job_name, enum job_status status, int status_code, char *reason); +void dyn_conf_store_config(const char *function, const char *payload, struct configurable_plugin *plugin); +void unlink_job(const char *plugin_name, const char *module_name, const char *job_name); +void delete_job(struct configurable_plugin *plugin, const char *module_name, const char *job_name); +void delete_job_pname(DICTIONARY *plugins_dict, const char *plugin_name, const char *module_name, const char *job_name); // API to be used by the web server(s) -json_object *get_list_of_plugins_json(); -struct configurable_plugin *get_plugin_by_name(const char *name); +json_object *get_list_of_plugins_json(DICTIONARY *plugins_dict); +struct configurable_plugin *get_plugin_by_name(DICTIONARY *plugins_dict, const char *name); json_object *get_list_of_modules_json(struct configurable_plugin *plugin); struct module *get_module_by_name(struct configurable_plugin *plugin, const char *module_name); +json_object *job2json(struct job *job); + // helper struct to make interface between internal webserver and h2o same struct uni_http_response { int status; @@ -126,11 +212,26 @@ struct uni_http_response { void (*content_free)(void *); }; -struct uni_http_response dyn_conf_process_http_request(int method, const char *plugin, const char *module, const char *job_id, void *payload, size_t payload_size); +struct uni_http_response dyn_conf_process_http_request(DICTIONARY *plugins_dict, int method, const char *plugin, const char *module, const char *job_id, void *payload, size_t payload_size); // API to be used by main netdata process, initialization and destruction etc. int dyn_conf_init(void); void freez_dyncfg(void *ptr); + +#define dyncfg_dictionary_create() dictionary_create(DICT_OPTION_VALUE_LINK_DONT_CLONE) + +void plugin_del_cb(const DICTIONARY_ITEM *item, void *value, void *data); + void *dyncfg_main(void *in); +#define DYNCFG_FUNCTION_TYPE_REGULAR (1 << 0) +#define DYNCFG_FUNCTION_TYPE_PAYLOAD (1 << 1) +#define DYNCFG_FUNCTION_TYPE_GET (1 << 2) +#define DYNCFG_FUNCTION_TYPE_SET (1 << 3) +#define DYNCFG_FUNCTION_TYPE_DELETE (1 << 4) +#define DYNCFG_FUNCTION_TYPE_ALL \ + (DYNCFG_FUNCTION_TYPE_REGULAR | DYNCFG_FUNCTION_TYPE_PAYLOAD | DYNCFG_FUNCTION_TYPE_GET | DYNCFG_FUNCTION_TYPE_SET | DYNCFG_FUNCTION_TYPE_DELETE) + +bool is_dyncfg_function(const char *function_name, uint8_t type); + #endif //DYN_CONF_H diff --git a/libnetdata/dyn_conf/tests/sample_test_config.json b/libnetdata/dyn_conf/tests/sample_test_config.json new file mode 100644 index 000000000..a6595f124 --- /dev/null +++ b/libnetdata/dyn_conf/tests/sample_test_config.json @@ -0,0 +1,22 @@ +{ + "http_endpoints": { + "parent": { + "host": "127.0.0.1", + "mguid": null, + "port": 20001, + "ssl": false + }, + "child": { + "host": "127.0.0.1", + "mguid": "3bc2f7de-1445-11ee-9ed7-3c7c3f21784c", + "port": 19999, + "ssl": false + } + }, + "global": { + "test_plugin_name": "external_plugin", + "test_array_module_name": "module_of_the_future", + "test_single_module_name": "module_of_the_future_single_type", + "test_job_name": "fixed_job" + } +} diff --git a/libnetdata/dyn_conf/tests/sub_tests/test_parent_child.rb b/libnetdata/dyn_conf/tests/sub_tests/test_parent_child.rb new file mode 100644 index 000000000..820db77f8 --- /dev/null +++ b/libnetdata/dyn_conf/tests/sub_tests/test_parent_child.rb @@ -0,0 +1,192 @@ +class ParentChildTest + @@plugin_cfg = <<~HEREDOC +{ "test" : "true" } +HEREDOC + @@plugin_cfg2 = <<~HEREDOC +{ "asdfgh" : "asdfgh" } +HEREDOC + + @@job_cfg = <<~HEREDOC +{ "i am newly created job" : "true" } +HEREDOC + + def initialize + @parent = $config[:http_endpoints][:parent] + @child = $config[:http_endpoints][:child] + @plugin = $config[:global][:test_plugin_name] + @arry_mod = $config[:global][:test_array_module_name] + @single_mod = $config[:global][:test_single_module_name] + @test_job = $config[:global][:test_job_name] + end + def check_test_plugin_modules_list(host, child = nil) + rc = DynCfgHttpClient.get_plugin_module_list(host, @plugin, child) + assert_eq(rc.code, 200, "as HTTP code for get_module_list request on plugin \"#{@plugin}\"") + modules = nil + assert_nothing_raised do + modules = JSON.parse(rc.parsed_response, symbolize_names: true) + end + assert_has_key?(modules, :modules) + assert_eq(modules[:modules].count, 2, "as number of modules in plugin \"#{@plugin}\"") + modules[:modules].each do |m| + assert_has_key?(m, :name) + assert_has_key?(m, :type) + assert_is_one_of(m[:type], "job_array", "single") + end + assert_eq_str(modules[:modules][0][:name], @arry_mod, "name of first module in plugin \"#{@plugin}\"") + assert_eq_str(modules[:modules][1][:name], @single_mod, "name of second module in plugin \"#{@plugin}\"") + end + def run + TEST_SUITE("Parent/Child plugin config") + + TEST("parent/child/get_plugin_list", "Get child (hops:1) plugin list trough parent") + plugins = DynCfgHttpClient.get_plugin_list(@parent, @child) + assert_eq(plugins.code, 200, "as HTTP code for get_plugin_list request") + assert_nothing_raised do + plugins = JSON.parse(plugins.parsed_response, symbolize_names: true) + end + assert_has_key?(plugins, :configurable_plugins) + assert_array_include?(plugins[:configurable_plugins], @plugin) + PASS() + + TEST("parent/child/(set/get)plugin_config", "Set then get and compare child (hops:1) plugin config trough parent") + rc = DynCfgHttpClient.set_plugin_config(@parent, @plugin, @@plugin_cfg, @child) + assert_eq(rc.code, 200, "as HTTP code for set_plugin_config request") + + rc = DynCfgHttpClient.get_plugin_config(@parent, @plugin, @child) + assert_eq(rc.code, 200, "as HTTP code for get_plugin_config request") + assert_eq_str(rc.parsed_response.chomp!, @@plugin_cfg, "as plugin config") + + # We do this twice with different configs to ensure first config was not loaded from persistent storage (from previous tests) + rc = DynCfgHttpClient.set_plugin_config(@parent, @plugin, @@plugin_cfg2, @child) + assert_eq(rc.code, 200, "as HTTP code for set_plugin_config request 2") + + rc = DynCfgHttpClient.get_plugin_config(@parent, @plugin, @child) + assert_eq(rc.code, 200, "as HTTP code for get_plugin_config request 2") + assert_eq_str(rc.parsed_response.chomp!, @@plugin_cfg2, "set/get plugin config 2") + PASS() + + TEST("child/get_plugin_config", "Get child (hops:0) plugin config and compare with what we got trough parent (set_plugin_config from previous test)") + rc = DynCfgHttpClient.get_plugin_config(@child, @plugin, nil) + assert_eq(rc.code, 200, "as HTTP code for get_plugin_config request") + assert_eq_str(rc.parsed_response.chomp!, @@plugin_cfg2.chomp, "as plugin config") + PASS() + + TEST("parent/child/plugin_module_list", "Get child (hops:1) plugin module list trough parent and check its contents") + check_test_plugin_modules_list(@parent, @child) + PASS() + + TEST("child/plugin_module_list", "Get child (hops:0) plugin module list directly and check its contents") + check_test_plugin_modules_list(@child, nil) + PASS() + + TEST("parent/child/module/jobs", "Get list of jobs from child (hops:1) trough parent and check its contents, check job updates") + rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child) + assert_eq(rc.code, 200, "as HTTP code for get_jobs request") + jobs = nil + assert_nothing_raised do + jobs = JSON.parse(rc.parsed_response, symbolize_names: true) + end + assert_has_key?(jobs, :jobs) + new_job = jobs[:jobs].find {|i| i[:name] == @test_job} + assert_not_nil(new_job) + assert_has_key?(new_job, :status) + assert_not_eq_str(new_job[:status], "unknown", "job status is other than unknown") + assert_has_key?(new_job, :flags) + assert_array_include?(new_job[:flags], "JOB_FLG_STREAMING_PUSHED") + PASS() + + TEST("child/module/jobs", "Get list of jobs direct from child (hops:0) and check its contents, check job updates") + rc = DynCfgHttpClient.get_job_list(@child, @plugin, @arry_mod, nil) + assert_eq(rc.code, 200, "as HTTP code for get_jobs request") + jobs = nil + assert_nothing_raised do + jobs = JSON.parse(rc.parsed_response, symbolize_names: true) + end + assert_has_key?(jobs, :jobs) + new_job = jobs[:jobs].find {|i| i[:name] == @test_job} + assert_not_nil(new_job) + assert_has_key?(new_job, :status) + assert_not_eq_str(new_job[:status], "unknown", "job status is other than unknown") + assert_has_key?(new_job, :flags) + + assert_array_not_include?(new_job[:flags], "JOB_FLG_STREAMING_PUSHED") # this is plugin directly at child so it should not show this flag + PASS() + + TEST("parent/child/single_module/jobs", "Attempt getting list of jobs from child (hops:1) trough parent on single module. Check it fails properly") + rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @single_mod, @child) + assert_eq(rc.code, 400, "as HTTP code for get_jobs request") + assert_eq_str(rc.parsed_response, '400 - this module is not array type', "as HTTP code for get_jobs request on single module") + PASS() + + created_job = SecureRandom.uuid + TEST("parent/child/module/cr_del_job", "Create and delete job on child (hops:1) trough parent") + # create new job + rc = DynCfgHttpClient.create_job(@parent, @plugin, @arry_mod, created_job, @@job_cfg, @child) + assert_eq_http_code(rc, 200, "as HTTP code for create_job request") + # check this job is in job list @parent + rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child) + assert_eq(rc.code, 200, "as HTTP code for get_jobs request") + jobs = nil + assert_nothing_raised do + jobs = JSON.parse(rc.parsed_response, symbolize_names: true) + end + assert_has_key?(jobs, :jobs) + new_job = jobs[:jobs].find {|i| i[:name] == created_job} + assert_not_nil(new_job) + # check this job is in job list @child + rc = DynCfgHttpClient.get_job_list(@child, @plugin, @arry_mod, nil) + assert_eq(rc.code, 200, "as HTTP code for get_jobs request") + jobs = nil + assert_nothing_raised do + jobs = JSON.parse(rc.parsed_response, symbolize_names: true) + end + assert_has_key?(jobs, :jobs) + new_job = jobs[:jobs].find {|i| i[:name] == created_job} + assert_not_nil(new_job) + # check we can get job config back + rc = DynCfgHttpClient.get_job_config(@parent, @plugin, @arry_mod, created_job, @child) + assert_eq(rc.code, 200, "as HTTP code for get_job_config request") + assert_eq_str(rc.parsed_response.chomp!, @@job_cfg, "as job config") + # delete job + rc = DynCfgHttpClient.delete_job(@parent, @plugin, @arry_mod, created_job, @child) + assert_eq(rc.code, 200, "as HTTP code for delete_job request") + # Check it is not in parents job list anymore + rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child) + assert_eq(rc.code, 200, "as HTTP code for get_jobs request") + jobs = nil + assert_nothing_raised do + jobs = JSON.parse(rc.parsed_response, symbolize_names: true) + end + assert_has_key?(jobs, :jobs) + new_job = jobs[:jobs].find {|i| i[:name] == created_job} + assert_nil(new_job) + # Check it is not in childs job list anymore + rc = DynCfgHttpClient.get_job_list(@child, @plugin, @arry_mod, nil) + assert_eq(rc.code, 200, "as HTTP code for get_jobs request") + jobs = nil + assert_nothing_raised do + jobs = JSON.parse(rc.parsed_response, symbolize_names: true) + end + assert_has_key?(jobs, :jobs) + new_job = jobs[:jobs].find {|i| i[:name] == created_job} + assert_nil(new_job) + PASS() + + TEST("parent/child/module/del_undeletable_job", "Try delete job on child (child rejects), check failure case works (hops:1)") + # test if plugin rejects job deletion the job still remains in list as it should + rc = DynCfgHttpClient.delete_job(@parent, @plugin, @arry_mod, @test_job, @child) + assert_eq(rc.code, 500, "as HTTP code for delete_job request") + rc = DynCfgHttpClient.get_job_list(@parent, @plugin, @arry_mod, @child) + assert_eq(rc.code, 200, "as HTTP code for get_jobs request") + jobs = nil + assert_nothing_raised do + jobs = JSON.parse(rc.parsed_response, symbolize_names: true) + end + assert_has_key?(jobs, :jobs) + job = jobs[:jobs].find {|i| i[:name] == @test_job} + assert_not_nil(job) + PASS() + end +end + +ParentChildTest.new.run() diff --git a/libnetdata/dyn_conf/tests/test_dyncfg.rb b/libnetdata/dyn_conf/tests/test_dyncfg.rb new file mode 100755 index 000000000..1b4b3a068 --- /dev/null +++ b/libnetdata/dyn_conf/tests/test_dyncfg.rb @@ -0,0 +1,266 @@ +#!/usr/bin/env ruby + +require 'json' +require 'httparty' +require 'pastel' +require 'securerandom' + +ARGV.length == 1 or raise "Usage: #{$0} <config file>" +config_file = ARGV[0] + +File.exist?(config_file) or raise "File not found: #{config_file}" + +$config = JSON.parse(File.read(config_file), symbolize_names: true) + +$plugin_name = $config[:global][:test_plugin_name] +$pastel = Pastel.new + +class TestRunner + attr_reader :stats + def initialize + @stats = { + :suites => 0, + :tests => 0, + :assertions => 0 + } + @test = nil + end + def add_assertion() + @stats[:assertions] += 1 + end + def FAIL(msg, exception = nil, loc = nil) + puts $pastel.red.bold(" ✕ FAIL") + STDERR.print " " + if loc + STDERR.print $pastel.yellow("@#{loc.path}:#{loc.lineno}: ") + else + STDERR.print $pastel.yellow("@#{caller_locations(1, 1).first.path}:#{caller_locations(1, 1).first.lineno}: ") + end + STDERR.puts msg + STDERR.puts exception.full_message(:highlight => true) if exception + STDERR.puts $pastel.yellow(" Backtrace:") + caller.each do |line| + STDERR.puts " #{line}" + end + exit 1 + end + def PASS() + STDERR.puts $pastel.green.bold(" ✓ PASS") + @stats[:tests] += 1 + @test = nil + end + def TEST_SUITE(name) + puts $pastel.bold("• TEST SUITE: \"#{name}\"") + @stats[:suites] += 1 + end + def assert_no_test_running() + unless @test.nil? + STDERR.puts $pastel.red("\nFATAL: Test \"#{@test}\" did not call PASS() or FAIL()!") + exit 1 + end + end + def TEST(name, description = nil) + assert_no_test_running() + @test = name + col = 0 + txt = " ├─ T: #{name} " + col += txt.length + print $pastel.bold(txt) + + tab = 50 + rem = tab - (col % tab) + rem.times do putc ' ' end + col += rem + + if (description) + txt = " - #{description} " + col += txt.length + print txt + + tab = 180 + rem = tab - (col % tab) + rem.times do putc '.' end + end + end + def FINALIZE() + assert_no_test_running() + end +end + +$test_runner = TestRunner.new +def FAIL(msg, exception = nil, loc = nil) + $test_runner.FAIL(msg, exception, loc) +end +def PASS() + $test_runner.PASS() +end +def TEST_SUITE(name) + $test_runner.TEST_SUITE(name) +end +def TEST(name, description = nil) + $test_runner.TEST(name, description) +end + +def assert_eq(got, expected, msg = nil) + unless got == expected + FAIL("Expected #{expected}, got #{got} #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first) + end + $test_runner.add_assertion() +end +def assert_eq_http_code(got, expected, msg = nil) + unless got.code == expected + FAIL("Expected #{expected}, got #{got}. Server \"#{got.parsed_response}\" #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first) + end + $test_runner.add_assertion() +end +def assert_eq_str(got, expected, msg = nil) + unless got == expected + FAIL("Strings do not match #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first) + end + $test_runner.add_assertion() +end +def assert_not_eq_str(got, expected, msg = nil) + unless got != expected + FAIL("Strings shoud not match #{msg ? "(#{msg})" : ""}", nil, caller_locations(1, 1).first) + end + $test_runner.add_assertion() +end +def assert_nothing_raised() + begin + yield + rescue Exception => e + FAIL("Unexpected exception of type #{e.class} raised. Msg: \"#{e.message}\"", e, caller_locations(1, 1).first) + end + $test_runner.add_assertion() +end +def assert_has_key?(hash, key) + unless hash.has_key?(key) + FAIL("Expected key \"#{key}\" in hash", nil, caller_locations(1, 1).first) + end + $test_runner.add_assertion() +end +def assert_array_include?(array, value) + unless array.include?(value) + FAIL("Expected array to include \"#{value}\"", nil, caller_locations(1, 1).first) + end + $test_runner.add_assertion() +end +def assert_array_not_include?(array, value) + if array.include?(value) + FAIL("Expected array to not include \"#{value}\"", nil, caller_locations(1, 1).first) + end + $test_runner.add_assertion() +end +def assert_is_one_of(value, *values) + unless values.include?(value) + FAIL("Expected value to be one of #{values.join(", ")}", nil, caller_locations(1, 1).first) + end + $test_runner.add_assertion() +end +def assert_not_nil(value) + if value.nil? + FAIL("Expected value to not be nil", nil, caller_locations(1, 1).first) + end + $test_runner.add_assertion() +end +def assert_nil(value) + unless value.nil? + FAIL("Expected value to be nil", nil, caller_locations(1, 1).first) + end + $test_runner.add_assertion() +end + + +class DynCfgHttpClient + def self.protocol(cfg) + return cfg[:ssl] ? 'https://' : 'http://' + end + def self.url_base(host) + return "#{protocol(host)}#{host[:host]}:#{host[:port]}" + end + def self.get_url_cfg_base(host, child = nil) + url = url_base(host) + url += "/host/#{child[:mguid]}" if child + url += "/api/v2/config" + return url + end + def self.get_url_cfg_plugin(host, plugin, child = nil) + return get_url_cfg_base(host, child) + '/' + plugin + end + def self.get_url_cfg_module(host, plugin, mod, child = nil) + return get_url_cfg_plugin(host, plugin, child) + '/' + mod + end + def self.get_url_cfg_job(host, plugin, mod, job_id, child = nil) + return get_url_cfg_module(host, plugin, mod, child) + "/#{job_id}" + end + def self.get_plugin_list(host, child = nil) + begin + return HTTParty.get(get_url_cfg_base(host, child), verify: false, format: :plain) + rescue => e + FAIL(e.message, e) + end + end + def self.get_plugin_config(host, plugin, child = nil) + begin + return HTTParty.get(get_url_cfg_plugin(host, plugin, child), verify: false) + rescue => e + FAIL(e.message, e) + end + end + def self.set_plugin_config(host, plugin, cfg, child = nil) + begin + return HTTParty.put(get_url_cfg_plugin(host, plugin, child), verify: false, body: cfg) + rescue => e + FAIL(e.message, e) + end + end + def self.get_plugin_module_list(host, plugin, child = nil) + begin + return HTTParty.get(get_url_cfg_plugin(host, plugin, child) + "/modules", verify: false, format: :plain) + rescue => e + FAIL(e.message, e) + end + end + def self.get_job_list(host, plugin, mod, child = nil) + begin + return HTTParty.get(get_url_cfg_module(host, plugin, mod, child) + "/jobs", verify: false, format: :plain) + rescue => e + FAIL(e.message, e) + end + end + def self.create_job(host, plugin, mod, job_id, job_cfg, child = nil) + begin + return HTTParty.post(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false, body: job_cfg) + rescue => e + FAIL(e.message, e) + end + end + def self.delete_job(host, plugin, mod, job_id, child = nil) + begin + return HTTParty.delete(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false) + rescue => e + FAIL(e.message, e) + end + end + def self.get_job_config(host, plugin, mod, job_id, child = nil) + begin + return HTTParty.get(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false, format: :plain) + rescue => e + FAIL(e.message, e) + end + end + def self.set_job_config(host, plugin, mod, job_id, job_cfg, child = nil) + begin + return HTTParty.put(get_url_cfg_job(host, plugin, mod, job_id, child), verify: false, body: job_cfg) + rescue => e + FAIL(e.message, e) + end + end +end + +require_relative 'sub_tests/test_parent_child.rb' + +$test_runner.FINALIZE() +puts $pastel.green.bold("All tests passed!") +puts ("Total #{$test_runner.stats[:assertions]} assertions, #{$test_runner.stats[:tests]} tests in #{$test_runner.stats[:suites]} suites") +exit 0 diff --git a/libnetdata/dyn_conf/tests/test_plugin/test.plugin b/libnetdata/dyn_conf/tests/test_plugin/test.plugin new file mode 100755 index 000000000..b890ab314 --- /dev/null +++ b/libnetdata/dyn_conf/tests/test_plugin/test.plugin @@ -0,0 +1,250 @@ +#!/usr/bin/env ruby + +# bogus chart that we create just so there is at least one chart +CHART_TYPE = 'lines' +UPDATE_EVERY = 1 +PRIORITY = 100000 +CHART_NAME = 'number_of_processes' +DIMENSION_NAME = 'running' + +$plugin_name = "external_plugin" +$plugin_version = "0.0.1" +$plugin_config = <<-HEREDOC +test_plugin_config +hableba hableba hableba +HEREDOC + +$array_module_name = 'module_of_the_future' +$fixed_job_name = 'fixed_job' + +$modules = { + $array_module_name => { + :type => :job_array, + :jobs => { + $fixed_job_name => { + :type => :fixed, + :config => <<-HEREDOC +fixed_job_config +HEREDOC + }, + }, + :config => <<-HEREDOC +module_of_the_future_config +HEREDOC + }, + "module_of_the_future_single_type" => { + :type => :single, + :jobs => {}, + :config => <<-HEREDOC +module_of_the_future_single_type_config +HEREDOC + } +} + +def out(str) + $log.puts "2 NETDATA> #{str}" + $stdout.puts str + $stdout.flush + $log.flush +end + +def log(str) + $log.puts "LOG > #{str}" + $log.flush +end + +#TODO this is AI code, verify +def split_with_quotes(str) + result = [] + current_word = "" + in_quotes = false + escaped = false + + str.each_char do |char| + if char == '\\' && !escaped + escaped = true + next + end + + if char == '"' && !escaped + in_quotes = !in_quotes + current_word << char + elsif char == ' ' && !in_quotes + result << current_word unless current_word.empty? + current_word = "" + else + current_word << char + end + + escaped = false + end + + result << current_word unless current_word.empty? + + result +end + + +def print_startup_messages + out "DYNCFG_ENABLE #{$plugin_name}" + $modules.each do |name, module_config| + out "DYNCFG_REGISTER_MODULE #{name} #{module_config[:type]}" + end + out "CHART system.#{CHART_NAME} '' 'Number of running processes' 'processes' processes processes.#{CHART_NAME} #{CHART_TYPE} #{PRIORITY} #{UPDATE_EVERY}" + out "DIMENSION #{DIMENSION_NAME} '' absolute 1 1" + + $modules.each do |mod_name, mod| + next unless mod[:type] == :job_array + mod[:jobs].each do |job_name, job| + next unless job[:type] == :fixed + out "DYNCFG_REGISTER_JOB #{mod_name} #{job_name} stock 0" + out "REPORT_JOB_STATUS #{$array_module_name} #{$fixed_job_name} running 0" + end + end +end + +def function_result(txid, msg, result) + out "FUNCTION_RESULT_BEGIN #{txid} #{result} text/plain 5" + out msg + out "FUNCTION_RESULT_END" +end + +def process_payload_function(params) + log "payload function #{params[:fncname]}, #{params[:fncparams]}" + fnc_name, mod_name, job_name = params[:fncparams] + case fnc_name + when 'set_plugin_config' + $plugin_config = params[:payload] + function_result(params[:txid], "plugin config set", 1) + when 'set_module_config' + mod = $modules[mod_name] + return function_result(params[:txid], "no such module", 0) if mod.nil? + mod[:config] = params[:payload] + function_result(params[:txid], "module config set", 1) + when 'set_job_config' + mod = $modules[mod_name] + return function_result(params[:txid], "no such module", 0) if mod.nil? + job = mod[:jobs][job_name] + if job.nil? + job = Hash.new if job.nil? + job[:type] = :dynamic + mod[:jobs][job_name] = job + end + job[:config] = params[:payload] + function_result(params[:txid], "job config set", 1) + end +end + +def process_function(params) + log "normal function #{params[:fncname]}, #{params[:fncparams]}" + fnc_name, mod_name, job_name = params[:fncparams] + case fnc_name + when 'get_plugin_config' + function_result(params[:txid], $plugin_config, 1) + when 'get_module_config' + return function_result(params[:txid], "no such module", 0) unless $modules.has_key?(mod_name) + function_result(params[:txid], $modules[mod_name][:config], 1) + when 'get_job_config' + mod = $modules[mod_name] + return function_result(params[:txid], "no such module", 0) if mod.nil? + job = mod[:jobs][job_name] + return function_result(params[:txid], "no such job", 0) if job.nil? + function_result(params[:txid], job[:config], 1) + when 'delete_job' + mod = $modules[mod_name] + return function_result(params[:txid], "no such module", 0) if mod.nil? + job = mod[:jobs][job_name] + return function_result(params[:txid], "no such job", 0) if job.nil? + if job[:type] == :fixed + return function_result(params[:txid], "this job can't be deleted", 0) + else + mod[:jobs].delete(job_name) + function_result(params[:txid], "job deleted", 1) + end + end +end + +$inflight_incoming = nil +def process_input(input) + words = split_with_quotes(input) + + unless $inflight_incoming.nil? + if input == "FUNCTION_PAYLOAD_END" + log $inflight_incoming[:payload] + process_payload_function($inflight_incoming) + $inflight_incoming = nil + else + $inflight_incoming[:payload] << input + $inflight_incoming[:payload] << "\n" + end + return + end + + case words[0] + when "FUNCTION", "FUNCTION_PAYLOAD" + params = {} + params[:command] = words[0] + params[:txid] = words[1] + params[:timeout] = words[2].to_i + params[:fncname] = words[3] + params[:fncname] = params[:fncname][1..-2] if params[:fncname].start_with?('"') && params[:fncname].end_with?('"') + if params[:command] == "FUNCTION_PAYLOAD" + $inflight_incoming = Hash.new + params[:fncparams] = split_with_quotes(params[:fncname]) + params[:fncname] = params[:fncparams][0] + $inflight_incoming[:txid] = params[:txid] + $inflight_incoming[:fncname] = params[:fncname] + $inflight_incoming[:params] = params + $inflight_incoming[:fncparams] = params[:fncparams] + $inflight_incoming[:payload] = "" + else + params[:fncparams] = split_with_quotes(params[:fncname]) + params[:fncname] = params[:fncparams][0] + process_function(params) + end + end +end + +def read_and_output_metric + processes = `ps -e | wc -l`.to_i - 1 # -1 to exclude the header line + timestamp = Time.now.to_i + + puts "BEGIN system.#{CHART_NAME}" + puts "SET #{DIMENSION_NAME} = #{processes}" + puts "END" +end + +def the_main + $stderr.reopen("/tmp/test_plugin_err.log", "w") + $log = File.open("/tmp/test_plugin.log", "w") + $log.puts "Starting plugin" + print_startup_messages + $log.puts "init done" + $log.flush + + last_metric_time = Time.now + + loop do + time_since_last_metric = Time.now - last_metric_time + + # If it's been more than 1 second since we collected metrics, collect them now + if time_since_last_metric >= 1 + read_and_output_metric + last_metric_time = Time.now + end + + # Use select to wait for input, but only wait up to the time remaining until we need to collect metrics again + remaining_time = [1 - time_since_last_metric, 0].max + if select([$stdin], nil, nil, remaining_time) + input = $stdin.gets + next if input.class != String + input.chomp! + $log.puts "RAW INPUT< #{input}" + $log.flush + process_input(input) + end + end +end + + +the_main if __FILE__ == $PROGRAM_NAME diff --git a/libnetdata/ebpf/ebpf.c b/libnetdata/ebpf/ebpf.c index 6793f403a..1bd45ef25 100644 --- a/libnetdata/ebpf/ebpf.c +++ b/libnetdata/ebpf/ebpf.c @@ -792,13 +792,13 @@ void ebpf_update_controller(int fd, ebpf_module_t *em) { uint32_t values[NETDATA_CONTROLLER_END] = { (em->apps_charts & NETDATA_EBPF_APPS_FLAG_YES) | em->cgroup_charts, - em->apps_level + em->apps_level, 0, 0, 0, 0 }; uint32_t key; - uint32_t end = (em->apps_level != NETDATA_APPS_NOT_SET) ? NETDATA_CONTROLLER_END : NETDATA_CONTROLLER_APPS_LEVEL; + uint32_t end = NETDATA_CONTROLLER_PID_TABLE_ADD; for (key = NETDATA_CONTROLLER_APPS_ENABLED; key < end; key++) { - int ret = bpf_map_update_elem(fd, &key, &values[key], 0); + int ret = bpf_map_update_elem(fd, &key, &values[key], BPF_ANY); if (ret) netdata_log_error("Add key(%u) for controller table failed.", key); } @@ -855,7 +855,7 @@ struct bpf_link **ebpf_load_program(char *plugins_dir, ebpf_module_t *em, int kv uint32_t idx = ebpf_select_index(em->kernels, is_rhf, kver); - ebpf_mount_name(lpath, 4095, plugins_dir, idx, em->thread_name, em->mode, is_rhf); + ebpf_mount_name(lpath, 4095, plugins_dir, idx, em->info.thread_name, em->mode, is_rhf); // When this function is called ebpf.plugin is using legacy code, so we should reset the variable em->load &= ~ NETDATA_EBPF_LOAD_METHODS; @@ -1269,7 +1269,7 @@ void ebpf_update_module_using_config(ebpf_module_t *modules, netdata_ebpf_load_m #ifdef NETDATA_DEV_MODE netdata_log_info("The thread %s was configured with: mode = %s; update every = %d; apps = %s; cgroup = %s; ebpf type format = %s; ebpf co-re tracing = %s; collect pid = %s; maps per core = %s, lifetime=%u", - modules->thread_name, + modules->info.thread_name, load_mode, modules->update_every, (modules->apps_charts)?"enabled":"disabled", diff --git a/libnetdata/ebpf/ebpf.h b/libnetdata/ebpf/ebpf.h index 691a4c26e..6708f669a 100644 --- a/libnetdata/ebpf/ebpf.h +++ b/libnetdata/ebpf/ebpf.h @@ -301,11 +301,27 @@ enum ebpf_global_table_values { typedef uint64_t netdata_idx_t; typedef struct ebpf_module { - const char *thread_name; - const char *config_name; - const char *thread_description; + // Constants used with module + struct { + const char *thread_name; + const char *config_name; + const char *thread_description; + } info; + + // Helpers used with plugin + struct { + void *(*start_routine)(void *); // the thread function + void (*apps_routine)(struct ebpf_module *em, void *ptr); // the apps charts + void (*fnct_routine)(BUFFER *bf, struct ebpf_module *em); // the function used for exteernal requests + const char *fcnt_name; // name given to cloud + const char *fcnt_desc; // description given about function + const char *fcnt_thread_chart_name; + int order_thread_chart; + const char *fcnt_thread_lifetime_name; + int order_thread_lifetime; + } functions; + enum ebpf_threads_status enabled; - void *(*start_routine)(void *); int update_every; int global_charts; netdata_apps_integration_flags_t apps_charts; @@ -314,7 +330,6 @@ typedef struct ebpf_module { netdata_run_mode_t mode; uint32_t thread_id; int optional; - void (*apps_routine)(struct ebpf_module *em, void *ptr); ebpf_local_maps_t *maps; ebpf_specify_name_t *names; uint32_t pid_map_size; diff --git a/libnetdata/facets/facets.c b/libnetdata/facets/facets.c index b285baf03..52898feb3 100644 --- a/libnetdata/facets/facets.c +++ b/libnetdata/facets/facets.c @@ -1,77 +1,201 @@ +// SPDX-License-Identifier: GPL-3.0-or-later #include "facets.h" -#define HISTOGRAM_COLUMNS 60 +#define HISTOGRAM_COLUMNS 150 // the target number of points in a histogram +#define FACETS_KEYS_WITH_VALUES_MAX 200 // the max number of keys that can be facets +#define FACETS_KEYS_IN_ROW_MAX 500 // the max number of keys in a row -static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row); +#define FACETS_KEYS_HASHTABLE_ENTRIES 127 +#define FACETS_VALUES_HASHTABLE_ENTRIES 31 // ---------------------------------------------------------------------------- -time_t calculate_bar_width(time_t before, time_t after) { - // Array of valid durations in seconds - static time_t valid_durations[] = { - 1, - 15, - 30, - 1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes - 1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours - 1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days - 1 * (30*86400) // months - }; - static int array_size = sizeof(valid_durations) / sizeof(valid_durations[0]); +static const char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789"; +static const uint8_t id_encoding_characters_reverse[256] = { + ['A'] = 0, ['B'] = 1, ['C'] = 2, ['D'] = 3, + ['E'] = 4, ['F'] = 5, ['G'] = 6, ['H'] = 7, + ['I'] = 8, ['J'] = 9, ['K'] = 10, ['L'] = 11, + ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15, + ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, + ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23, + ['Y'] = 24, ['Z'] = 25, ['.'] = 26, ['a'] = 27, + ['b'] = 28, ['c'] = 29, ['d'] = 30, ['e'] = 31, + ['f'] = 32, ['g'] = 33, ['h'] = 34, ['i'] = 35, + ['j'] = 36, ['k'] = 37, ['l'] = 38, ['m'] = 39, + ['n'] = 40, ['o'] = 41, ['p'] = 42, ['q'] = 43, + ['r'] = 44, ['s'] = 45, ['t'] = 46, ['u'] = 47, + ['v'] = 48, ['w'] = 49, ['x'] = 50, ['y'] = 51, + ['z'] = 52, ['_'] = 53, ['0'] = 54, ['1'] = 55, + ['2'] = 56, ['3'] = 57, ['4'] = 58, ['5'] = 59, + ['6'] = 60, ['7'] = 61, ['8'] = 62, ['9'] = 63 +}; - time_t duration = before - after; - time_t bar_width = 1; +__attribute__((constructor)) void initialize_facets_id_encoding_characters_reverse(void) { - for (int i = array_size - 1; i >= 0; --i) { - if (duration / valid_durations[i] >= HISTOGRAM_COLUMNS) { - bar_width = valid_durations[i]; - break; +} + +#define FACET_STRING_HASH_SIZE 12 +#define FACETS_HASH XXH64_hash_t +#define FACETS_HASH_FUNCTION(src, len) XXH3_64bits(src, len) +#define FACETS_HASH_ZERO (FACETS_HASH)0 + +static inline void facets_hash_to_str(FACETS_HASH num, char *out) { + out[11] = '\0'; + out[10] = id_encoding_characters[num & 63]; num >>= 6; + out[9] = id_encoding_characters[num & 63]; num >>= 6; + out[8] = id_encoding_characters[num & 63]; num >>= 6; + out[7] = id_encoding_characters[num & 63]; num >>= 6; + out[6] = id_encoding_characters[num & 63]; num >>= 6; + out[5] = id_encoding_characters[num & 63]; num >>= 6; + out[4] = id_encoding_characters[num & 63]; num >>= 6; + out[3] = id_encoding_characters[num & 63]; num >>= 6; + out[2] = id_encoding_characters[num & 63]; num >>= 6; + out[1] = id_encoding_characters[num & 63]; num >>= 6; + out[0] = id_encoding_characters[num & 63]; +} + +static inline FACETS_HASH str_to_facets_hash(const char *str) { + FACETS_HASH num = 0; + int shifts = 6 * (FACET_STRING_HASH_SIZE - 2); + + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[0])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[1])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[2])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[3])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[4])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[5])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[6])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[7])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[8])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[9])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[10])])) << shifts; + + return num; +} + +static const char *hash_to_static_string(FACETS_HASH hash) { + static __thread char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(hash, hash_str); + return hash_str; +} + +static inline bool is_valid_string_hash(const char *s) { + if(strlen(s) != FACET_STRING_HASH_SIZE - 1) { + netdata_log_error("The user supplied key '%s' does not have the right length for a facets hash.", s); + return false; + } + + uint8_t *t = (uint8_t *)s; + while(*t) { + if(id_encoding_characters_reverse[*t] == 0 && *t != id_encoding_characters[0]) { + netdata_log_error("The user supplied key '%s' contains invalid characters for a facets hash.", s); + return false; } + + t++; } - return bar_width; + return true; } // ---------------------------------------------------------------------------- -static inline void uint32_to_char(uint32_t num, char *out) { - static char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789"; +typedef uint64_t SIMPLE_HASHTABLE_HASH; +#define SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS 32 - int i; - for(i = 5; i >= 0; --i) { - out[i] = id_encoding_characters[num & 63]; - num >>= 6; - } - out[6] = '\0'; +typedef struct simple_hashtable_slot { + SIMPLE_HASHTABLE_HASH hash; + void *data; +} SIMPLE_HASHTABLE_SLOT; + +typedef struct simple_hashtable { + size_t size; + SIMPLE_HASHTABLE_SLOT *hashtable; +} SIMPLE_HASHTABLE; + +static void simple_hashtable_init(SIMPLE_HASHTABLE *ht, size_t size) { + ht->size = size; + ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable)); } -inline void facets_string_hash(const char *src, char *out) { - uint32_t hash1 = fnv1a_hash32(src); - uint32_t hash2 = djb2_hash32(src); - uint32_t hash3 = larson_hash32(src); +static void simple_hashtable_free(SIMPLE_HASHTABLE *ht) { + freez(ht->hashtable); + ht->hashtable = NULL; + ht->size = 0; +} + +static inline SIMPLE_HASHTABLE_SLOT *simple_hashtable_get_slot(SIMPLE_HASHTABLE *ht, SIMPLE_HASHTABLE_HASH hash) { + // IMPORTANT: + // If the hashtable supported deletions, we would need to have a special slot.data value + // to mark deleted values and assume they are occupied during lookup, but empty during insert. + // But for our case, we don't need it, since we never delete items from the hashtable. + + size_t slot = hash % ht->size; + if(!ht->hashtable[slot].data || ht->hashtable[slot].hash == hash) + return &ht->hashtable[slot]; + + slot = ((hash >> SIMPLE_HASHTABLE_HASH_SECOND_HASH_SHIFTS) + 1) % ht->size; + // Linear probing until we find it + while (ht->hashtable[slot].data && ht->hashtable[slot].hash != hash) + slot = (slot + 1) % ht->size; // Wrap around if necessary + + return &ht->hashtable[slot]; +} - uint32_to_char(hash1, out); - uint32_to_char(hash2, &out[6]); - uint32_to_char(hash3, &out[12]); +static void simple_hashtable_resize_double(SIMPLE_HASHTABLE *ht) { + SIMPLE_HASHTABLE_SLOT *old = ht->hashtable; + size_t old_size = ht->size; - out[18] = '\0'; + ht->size = (ht->size * 2) + 1; + ht->hashtable = callocz(ht->size, sizeof(*ht->hashtable)); + for(size_t i = 0 ; i < old_size ; i++) { + if(!old[i].data) + continue; + + SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(ht, old[i].hash); + *slot = old[i]; + } + + freez(old); } + // ---------------------------------------------------------------------------- typedef struct facet_value { + FACETS_HASH hash; const char *name; + uint32_t name_len; bool selected; + bool empty; uint32_t rows_matching_facet_value; uint32_t final_facet_value_counter; + uint32_t order; + + uint32_t *histogram; + uint32_t min, max, sum; + + struct facet_value *prev, *next; } FACET_VALUE; +typedef enum { + FACET_KEY_VALUE_NONE = 0, + FACET_KEY_VALUE_UPDATED = (1 << 0), + FACET_KEY_VALUE_EMPTY = (1 << 1), + FACET_KEY_VALUE_COPIED = (1 << 2), +} FACET_KEY_VALUE_FLAGS; + +#define facet_key_value_updated(k) ((k)->current_value.flags & FACET_KEY_VALUE_UPDATED) +#define facet_key_value_empty(k) ((k)->current_value.flags & FACET_KEY_VALUE_EMPTY) +#define facet_key_value_copied(k) ((k)->current_value.flags & FACET_KEY_VALUE_COPIED) + struct facet_key { - const char *name; + FACETS *facets; - DICTIONARY *values; + FACETS_HASH hash; + const char *name; FACET_KEY_OPTIONS options; @@ -80,14 +204,27 @@ struct facet_key { // members about the current row uint32_t key_found_in_row; uint32_t key_values_selected_in_row; + uint32_t order; struct { - char hash[FACET_STRING_HASH_SIZE]; - bool updated; + bool enabled; + uint32_t used; + FACET_VALUE *ll; + SIMPLE_HASHTABLE ht; + } values; + + struct { + FACETS_HASH hash; + FACET_KEY_VALUE_FLAGS flags; + const char *raw; + uint32_t raw_len; BUFFER *b; + FACET_VALUE *v; } current_value; - uint32_t order; + struct { + FACET_VALUE *v; + } empty_value; struct { facet_dynamic_row_t cb; @@ -95,6 +232,7 @@ struct facet_key { } dynamic; struct { + bool view_only; facets_key_transformer_t cb; void *data; } transform; @@ -108,16 +246,36 @@ struct facets { SIMPLE_PATTERN *included_keys; FACETS_OPTIONS options; - usec_t anchor; + + struct { + usec_t start_ut; + usec_t stop_ut; + FACETS_ANCHOR_DIRECTION direction; + } anchor; SIMPLE_PATTERN *query; // the full text search pattern size_t keys_filtered_by_query; // the number of fields we do full text search (constant) - size_t keys_matched_by_query; // the number of fields matched the full text search (per row) DICTIONARY *accepted_params; - FACET_KEY *keys_ll; - DICTIONARY *keys; + struct { + size_t count; + FACET_KEY *ll; + SIMPLE_HASHTABLE ht; + } keys; + + struct { + // this is like a stack, of the keys that are used as facets + size_t used; + FACET_KEY *array[FACETS_KEYS_WITH_VALUES_MAX]; + } keys_with_values; + + struct { + // this is like a stack, of the keys that need to clean up between each row + size_t used; + FACET_KEY *array[FACETS_KEYS_IN_ROW_MAX]; + } keys_in_row; + FACET_ROW *base; // double linked list of the selected facets rows uint32_t items_to_return; @@ -125,10 +283,33 @@ struct facets { uint32_t order; struct { - FACET_ROW *last_added; + FACET_ROW_SEVERITY severity; + size_t keys_matched_by_query; // the number of fields matched the full text search (per row) + } current_row; + + struct { + usec_t after_ut; + usec_t before_ut; + } timeframe; - size_t evaluated; - size_t matched; + struct { + FACET_KEY *key; + FACETS_HASH hash; + char *chart; + bool enabled; + uint32_t slots; + usec_t slot_width_ut; + usec_t after_ut; + usec_t before_ut; + } histogram; + + struct { + facet_row_severity_t cb; + void *data; + } severity; + + struct { + FACET_ROW *last_added; size_t first; size_t forwards; @@ -138,117 +319,338 @@ struct facets { size_t prepends; size_t appends; size_t shifts; + + struct { + size_t evaluated; + size_t matched; + size_t created; + size_t reused; + } rows; + + struct { + size_t registered; + size_t unique; + size_t hashtable_increases; + } keys; + + struct { + size_t registered; + size_t transformed; + size_t dynamic; + size_t empty; + size_t indexed; + size_t inserts; + size_t conflicts; + size_t hashtable_increases; + } values; + + struct { + size_t searches; + } fts; } operations; }; -// ---------------------------------------------------------------------------- +usec_t facets_row_oldest_ut(FACETS *facets) { + if(facets->base) + return facets->base->prev->usec; -static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) { - if(!k->key_found_in_row) - v->rows_matching_facet_value++; + return 0; +} - k->key_found_in_row++; +usec_t facets_row_newest_ut(FACETS *facets) { + if(facets->base) + return facets->base->usec; - if(v->selected) - k->key_values_selected_in_row++; + return 0; } -static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) { - bool included = true, excluded = false; +uint32_t facets_rows(FACETS *facets) { + return facets->items_to_return; +} - if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET)) { - if(k->options & FACET_KEY_OPTION_FACET) { - included = true; - excluded = false; - } - else if(k->options & FACET_KEY_OPTION_NO_FACET) { - included = false; - excluded = true; - } +// ---------------------------------------------------------------------------- + +static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row); +static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v); +static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k); + +// ---------------------------------------------------------------------------- +// The FACET_VALUE index within each FACET_KEY + +#define foreach_value_in_key(k, v) \ + for((v) = (k)->values.ll; (v) ;(v) = (v)->next) + +#define foreach_value_in_key_done(v) do { ; } while(0) + +static inline void FACETS_VALUES_INDEX_CREATE(FACET_KEY *k) { + k->values.ll = NULL; + k->values.used = 0; + simple_hashtable_init(&k->values.ht, FACETS_VALUES_HASHTABLE_ENTRIES); +} + +static inline void FACETS_VALUES_INDEX_DESTROY(FACET_KEY *k) { + FACET_VALUE *v = k->values.ll; + while(v) { + FACET_VALUE *next = v->next; + freez(v->histogram); + freez((void *)v->name); + freez(v); + v = next; } - else { - if (facets->included_keys) { - if (!simple_pattern_matches(facets->included_keys, k->name)) - included = false; - } + k->values.ll = NULL; + k->values.used = 0; + k->values.enabled = false; - if (facets->excluded_keys) { - if (simple_pattern_matches(facets->excluded_keys, k->name)) - excluded = true; - } + simple_hashtable_free(&k->values.ht); +} + +static inline const char *facets_key_get_value(FACET_KEY *k) { + return facet_key_value_copied(k) ? buffer_tostring(k->current_value.b) : k->current_value.raw; +} + +static inline uint32_t facets_key_get_value_length(FACET_KEY *k) { + return facet_key_value_copied(k) ? buffer_strlen(k->current_value.b) : k->current_value.raw_len; +} + +static inline void facets_key_value_copy_to_buffer(FACET_KEY *k) { + if(!facet_key_value_copied(k)) { + buffer_contents_replace(k->current_value.b, k->current_value.raw, k->current_value.raw_len); + k->current_value.flags |= FACET_KEY_VALUE_COPIED; } +} - if(included && !excluded) { - k->options |= FACET_KEY_OPTION_FACET; - k->options &= ~FACET_KEY_OPTION_NO_FACET; - return true; +static const char *facets_value_dup(const char *s, uint32_t len) { + char *d = mallocz(len + 1); + + if(len) + memcpy(d, s, len); + + d[len] = '\0'; + + return d; +} + +static inline void FACET_VALUE_ADD_CONFLICT(FACET_KEY *k, FACET_VALUE *v, const FACET_VALUE * const nv) { + if(!v->name && !v->name_len && nv->name && nv->name_len) { + // an actual value, not a filter + v->name = facets_value_dup(nv->name, nv->name_len); + v->name_len = nv->name_len; } - k->options |= FACET_KEY_OPTION_NO_FACET; - k->options &= ~FACET_KEY_OPTION_FACET; - return false; + if(v->name && v->name_len) + facet_value_is_used(k, v); + + internal_fatal(v->name && nv->name && v->name_len == nv->name_len && memcmp(v->name, nv->name, v->name_len) != 0, + "value hash conflict: '%s' and '%s' have the same hash '%s'", + v->name, nv->name, hash_to_static_string(v->hash)); + + k->facets->operations.values.conflicts++; } -// ---------------------------------------------------------------------------- -// FACET_VALUE dictionary hooks +static inline FACET_VALUE *FACET_VALUE_GET_FROM_INDEX(FACET_KEY *k, FACETS_HASH hash) { + SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&k->values.ht, hash); + return slot->data; +} -static void facet_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { - FACET_VALUE *v = value; - FACET_KEY *k = data; +static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VALUE * const tv) { + SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&k->values.ht, tv->hash); + + if(slot->data) { + // already exists + + FACET_VALUE *v = slot->data; + FACET_VALUE_ADD_CONFLICT(k, v, tv); + return v; + } + + // we have to add it + + FACET_VALUE *v = mallocz(sizeof(*v)); + slot->hash = tv->hash; + slot->data = v; + + memcpy(v, tv, sizeof(*v)); + + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(k->values.ll, v, prev, next); + k->values.used++; if(!v->selected) v->selected = k->default_selected_for_values; - if(v->name) { + if(v->name && v->name_len) { // an actual value, not a filter - v->name = strdupz(v->name); + v->name = facets_value_dup(v->name, v->name_len); facet_value_is_used(k, v); } + else { + v->name = NULL; + v->name_len = 0; + } + + k->facets->operations.values.inserts++; + + if(unlikely(k->values.used > k->values.ht.size / 2)) { + simple_hashtable_resize_double(&k->values.ht); + k->facets->operations.values.hashtable_increases++; + } + + return v; } -static bool facet_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) { - FACET_VALUE *v = old_value; - FACET_VALUE *nv = new_value; - FACET_KEY *k = data; +static inline void FACET_VALUE_ADD_EMPTY_VALUE_TO_INDEX(FACET_KEY *k) { + static const FACET_VALUE tv = { + .hash = FACETS_HASH_ZERO, + .name = FACET_VALUE_UNSET, + .name_len = sizeof(FACET_VALUE_UNSET) - 1, + }; + + k->current_value.hash = FACETS_HASH_ZERO; - if(!v->name && nv->name) - // an actual value, not a filter - v->name = strdupz(nv->name); + if(k->empty_value.v) { + FACET_VALUE_ADD_CONFLICT(k, k->empty_value.v, &tv); + k->current_value.v = k->empty_value.v; + } + else { + FACET_VALUE *v = FACET_VALUE_ADD_TO_INDEX(k, &tv); + v->empty = true; + k->empty_value.v = v; + k->current_value.v = v; + } +} - if(v->name) - facet_value_is_used(k, v); +static inline void FACET_VALUE_ADD_CURRENT_VALUE_TO_INDEX(FACET_KEY *k) { + static __thread FACET_VALUE tv = { 0 }; - internal_fatal(v->name && strcmp(v->name, nv->name) != 0, "hash conflict: '%s' and '%s' have the same hash '%s'", v->name, nv->name, - dictionary_acquired_item_name(item)); + internal_fatal(!facet_key_value_updated(k), "trying to add a non-updated value to the index"); - return false; + tv.name = facets_key_get_value(k); + tv.name_len = facets_key_get_value_length(k); + tv.hash = FACETS_HASH_FUNCTION(tv.name, tv.name_len); + + k->current_value.v = FACET_VALUE_ADD_TO_INDEX(k, &tv); + k->facets->operations.values.indexed++; } -static void facet_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { - FACET_VALUE *v = value; - freez((char *)v->name); +static inline void FACET_VALUE_ADD_OR_UPDATE_SELECTED(FACET_KEY *k, FACETS_HASH hash) { + FACET_VALUE tv = { + .hash = hash, + .selected = true, + .name = NULL, + .name_len = 0, + }; + FACET_VALUE_ADD_TO_INDEX(k, &tv); } // ---------------------------------------------------------------------------- -// FACET_KEY dictionary hooks +// The FACET_KEY index within each FACET + +#define foreach_key_in_facets(facets, k) \ + for((k) = (facets)->keys.ll; (k) ;(k) = (k)->next) + +#define foreach_key_in_facets_done(k) do { ; } while(0) static inline void facet_key_late_init(FACETS *facets, FACET_KEY *k) { - if(k->values) + if(k->values.enabled) return; if(facets_key_is_facet(facets, k)) { - k->values = dictionary_create_advanced( - DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, - NULL, sizeof(FACET_VALUE)); - dictionary_register_insert_callback(k->values, facet_value_insert_callback, k); - dictionary_register_conflict_callback(k->values, facet_value_conflict_callback, k); - dictionary_register_delete_callback(k->values, facet_value_delete_callback, k); + FACETS_VALUES_INDEX_CREATE(k); + k->values.enabled = true; + if(facets->keys_with_values.used < FACETS_KEYS_WITH_VALUES_MAX) + facets->keys_with_values.array[facets->keys_with_values.used++] = k; } } -static void facet_key_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { - FACET_KEY *k = value; - FACETS *facets = data; +static inline void FACETS_KEYS_INDEX_CREATE(FACETS *facets) { + facets->keys.ll = NULL; + facets->keys.count = 0; + facets->keys_with_values.used = 0; + + simple_hashtable_init(&facets->keys.ht, FACETS_KEYS_HASHTABLE_ENTRIES); +} + +static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) { + FACET_KEY *k = facets->keys.ll; + while(k) { + FACET_KEY *next = k->next; + + FACETS_VALUES_INDEX_DESTROY(k); + buffer_free(k->current_value.b); + freez((void *)k->name); + freez(k); + + k = next; + } + facets->keys.ll = NULL; + facets->keys.count = 0; + facets->keys_with_values.used = 0; + + simple_hashtable_free(&facets->keys.ht); +} + +static inline FACET_KEY *FACETS_KEY_GET_FROM_INDEX(FACETS *facets, FACETS_HASH hash) { + SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&facets->keys.ht, hash); + return slot->data; +} + +bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length); + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); + if(!k || k->default_selected_for_values) + return false; + + hash = FACETS_HASH_FUNCTION(value, value_length); + FACET_VALUE *v = FACET_VALUE_GET_FROM_INDEX(k, hash); + return (v && v->selected) ? true : false; +} + +void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length); + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); + if(!k) return; + + hash = FACETS_HASH_FUNCTION(value, value_length); + FACET_VALUE *v = FACET_VALUE_GET_FROM_INDEX(k, hash); + if(v && v->name && v->name_len) return; + + FACET_VALUE tv = { + .hash = hash, + .name = value, + .name_len = value_length, + }; + FACET_VALUE_ADD_TO_INDEX(k, &tv); +} + +static void facet_key_set_name(FACET_KEY *k, const char *name, size_t name_length) { + internal_fatal(k->name && name && (strncmp(k->name, name, name_length) != 0 || k->name[name_length] != '\0'), + "key hash conflict: '%s' and '%s' have the same hash", + k->name, name); + + if(likely(k->name || !name || !name_length)) + return; + + // an actual value, not a filter + + char buf[name_length + 1]; + memcpy(buf, name, name_length); + buf[name_length] = '\0'; + + internal_fatal(strchr(buf, '='), "found = in key"); + + k->name = strdupz(buf); + facet_key_late_init(k->facets, k); +} + +static inline FACET_KEY *FACETS_KEY_CREATE(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) { + facets->operations.keys.unique++; + + FACET_KEY *k = callocz(1, sizeof(*k)); + + k->hash = hash; + k->facets = facets; + k->options = options; + k->current_value.b = buffer_create(sizeof(FACET_VALUE_UNSET), NULL); + k->default_selected_for_values = true; if(!(k->options & FACET_KEY_OPTION_REORDER)) k->order = facets->order++; @@ -256,56 +658,735 @@ static void facet_key_insert_callback(const DICTIONARY_ITEM *item __maybe_unused if((k->options & FACET_KEY_OPTION_FTS) || (facets->options & FACETS_OPTION_ALL_KEYS_FTS)) facets->keys_filtered_by_query++; - if(k->name) { - // an actual value, not a filter - k->name = strdupz(k->name); - facet_key_late_init(facets, k); - } + facet_key_set_name(k, name, name_length); - k->current_value.b = buffer_create(0, NULL); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->keys.ll, k, prev, next); + facets->keys.count++; - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->keys_ll, k, prev, next); + return k; } -static bool facet_key_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) { - FACET_KEY *k = old_value; - FACET_KEY *nk = new_value; - FACETS *facets = data; +static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) { + facets->operations.keys.registered++; - if(!k->name && nk->name) { - // an actual value, not a filter - k->name = strdupz(nk->name); - facet_key_late_init(facets, k); + SIMPLE_HASHTABLE_SLOT *slot = simple_hashtable_get_slot(&facets->keys.ht, hash); + + if(unlikely(!slot->data)) { + // we have to add it + FACET_KEY *k = FACETS_KEY_CREATE(facets, hash, name, name_length, options); + + slot->hash = hash; + slot->data = k; + + if(facets->keys.count > facets->keys.ht.size / 2) { + simple_hashtable_resize_double(&facets->keys.ht); + facets->operations.keys.hashtable_increases++; + } + + return k; } - if(k->options & FACET_KEY_OPTION_REORDER) { + // already in the index + + FACET_KEY *k = slot->data; + + facet_key_set_name(k, name, name_length); + + if(unlikely(k->options & FACET_KEY_OPTION_REORDER)) { k->order = facets->order++; k->options &= ~FACET_KEY_OPTION_REORDER; } - return false; + return k; +} + +bool facets_key_name_is_filter(FACETS *facets, const char *key) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, strlen(key)); + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); + return (!k || k->default_selected_for_values) ? false : true; +} + +bool facets_key_name_is_facet(FACETS *facets, const char *key) { + size_t key_len = strlen(key); + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_len); + FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, key, key_len, 0); + return (k && (k->options & FACET_KEY_OPTION_FACET)); +} + +// ---------------------------------------------------------------------------- + +static usec_t calculate_histogram_bar_width(usec_t after_ut, usec_t before_ut) { + // Array of valid durations in seconds + static time_t valid_durations_s[] = { + 1, 2, 5, 10, 15, 30, // seconds + 1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes + 1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours + 1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days + 1 * (30*86400) // months + }; + static int array_size = sizeof(valid_durations_s) / sizeof(valid_durations_s[0]); + + usec_t duration_ut = before_ut - after_ut; + usec_t bar_width_ut = 1 * USEC_PER_SEC; + + for (int i = array_size - 1; i >= 0; --i) { + if (duration_ut / (valid_durations_s[i] * USEC_PER_SEC) >= HISTOGRAM_COLUMNS) { + bar_width_ut = valid_durations_s[i] * USEC_PER_SEC; + break; + } + } + + return bar_width_ut; +} + +static inline usec_t facets_histogram_slot_baseline_ut(FACETS *facets, usec_t ut) { + usec_t delta_ut = ut % facets->histogram.slot_width_ut; + return ut - delta_ut; +} + +void facets_set_timeframe_and_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut) { + if(after_ut > before_ut) { + usec_t t = after_ut; + after_ut = before_ut; + before_ut = t; + } + + facets->histogram.enabled = true; + + if(key_id && *key_id && strlen(key_id) == FACET_STRING_HASH_SIZE - 1) { + facets->histogram.chart = strdupz(key_id); + facets->histogram.hash = str_to_facets_hash(facets->histogram.chart); + } + else { + freez(facets->histogram.chart); + facets->histogram.chart = NULL; + facets->histogram.hash = FACETS_HASH_ZERO; + } + + facets->timeframe.after_ut = after_ut; + facets->timeframe.before_ut = before_ut; + + facets->histogram.slot_width_ut = calculate_histogram_bar_width(after_ut, before_ut); + facets->histogram.after_ut = facets_histogram_slot_baseline_ut(facets, after_ut); + facets->histogram.before_ut = facets_histogram_slot_baseline_ut(facets, before_ut) + facets->histogram.slot_width_ut; + facets->histogram.slots = (facets->histogram.before_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut + 1; + + internal_fatal(after_ut < facets->histogram.after_ut, "histogram after_ut is not less or equal to wanted after_ut"); + internal_fatal(before_ut > facets->histogram.before_ut, "histogram before_ut is not more or equal to wanted before_ut"); + + if(facets->histogram.slots > 1000) { + facets->histogram.slots = 1000 + 1; + facets->histogram.slot_width_ut = (facets->histogram.before_ut - facets->histogram.after_ut) / 1000; + } +} + +void facets_set_timeframe_and_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut) { + char hash_str[FACET_STRING_HASH_SIZE]; + FACETS_HASH hash = FACETS_HASH_FUNCTION(key_name, strlen(key_name)); + facets_hash_to_str(hash, hash_str); + facets_set_timeframe_and_histogram_by_id(facets, hash_str, after_ut, before_ut); +} + +static inline void facets_histogram_update_value(FACETS *facets, usec_t usec) { + if(!facets->histogram.enabled || + !facets->histogram.key || + !facets->histogram.key->values.enabled || + !facet_key_value_updated(facets->histogram.key) || + usec < facets->histogram.after_ut || + usec > facets->histogram.before_ut) + return; + + FACET_VALUE *v = facets->histogram.key->current_value.v; + + if(unlikely(!v->histogram)) + v->histogram = callocz(facets->histogram.slots, sizeof(*v->histogram)); + + usec_t base_ut = facets_histogram_slot_baseline_ut(facets, usec); + + if(unlikely(base_ut < facets->histogram.after_ut)) + base_ut = facets->histogram.after_ut; + + if(unlikely(base_ut > facets->histogram.before_ut)) + base_ut = facets->histogram.before_ut; + + uint32_t slot = (base_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut; + + if(unlikely(slot >= facets->histogram.slots)) + slot = facets->histogram.slots - 1; + + v->histogram[slot]++; +} + +static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, const char *first_key) { + BUFFER *tb = NULL; + + buffer_json_member_add_array(wb, key); + { + if(first_key) + buffer_json_add_array_item_string(wb, first_key); + + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + if(!v->empty && k->transform.cb && k->transform.view_only) { + if(!tb) + tb = buffer_create(0, NULL); + + buffer_contents_replace(tb, v->name, v->name_len); + k->transform.cb(facets, tb, FACETS_TRANSFORM_HISTOGRAM, k->transform.data); + buffer_json_add_array_item_string(wb, buffer_tostring(tb)); + } + else + buffer_json_add_array_item_string(wb, v->name); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key + + buffer_free(tb); +} + +static inline void facets_histogram_value_units(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_string(wb, "events"); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_min(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_uint64(wb, v->min); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_max(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_uint64(wb, v->max); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_avg(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_double(wb, (double) v->sum / (double) facets->histogram.slots); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_arp(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_uint64(wb, 0); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_con(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, uint32_t sum) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_double(wb, (double) v->sum * 100.0 / (double) sum); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb) { + size_t dimensions = 0; + uint32_t min = UINT32_MAX, max = 0, sum = 0, count = 0; + + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + dimensions++; + + v->min = UINT32_MAX; + v->max = 0; + v->sum = 0; + + for(uint32_t i = 0; i < facets->histogram.slots ;i++) { + uint32_t n = v->histogram[i]; + + if(n < min) + min = n; + + if(n > max) + max = n; + + sum += n; + count++; + + if(n < v->min) + v->min = n; + + if(n > v->max) + v->max = n; + + v->sum += n; + } + } + foreach_value_in_key_done(v); + } + + buffer_json_member_add_object(wb, "summary"); + { + buffer_json_member_add_array(wb, "nodes"); + { + buffer_json_add_array_item_object(wb); // node + { + buffer_json_member_add_string(wb, "mg", "default"); + buffer_json_member_add_string(wb, "nm", "facets.histogram"); + buffer_json_member_add_uint64(wb, "ni", 0); + buffer_json_member_add_object(wb, "st"); + { + buffer_json_member_add_uint64(wb, "ai", 0); + buffer_json_member_add_uint64(wb, "code", 200); + buffer_json_member_add_string(wb, "msg", ""); + } + buffer_json_object_close(wb); // st + + if(dimensions) { + buffer_json_member_add_object(wb, "is"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // is + + buffer_json_member_add_object(wb, "ds"); + { + buffer_json_member_add_uint64(wb, "sl", dimensions); + buffer_json_member_add_uint64(wb, "qr", dimensions); + } + buffer_json_object_close(wb); // ds + } + + if(count) { + buffer_json_member_add_object(wb, "sts"); + { + buffer_json_member_add_uint64(wb, "min", min); + buffer_json_member_add_uint64(wb, "max", max); + buffer_json_member_add_double(wb, "avg", (double) sum / (double) count); + buffer_json_member_add_double(wb, "con", 100.0); + } + buffer_json_object_close(wb); // sts + } + } + buffer_json_object_close(wb); // node + } + buffer_json_array_close(wb); // nodes + + buffer_json_member_add_array(wb, "contexts"); + { + buffer_json_add_array_item_object(wb); // context + { + buffer_json_member_add_string(wb, "id", "facets.histogram"); + + if(dimensions) { + buffer_json_member_add_object(wb, "is"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // is + + buffer_json_member_add_object(wb, "ds"); + { + buffer_json_member_add_uint64(wb, "sl", dimensions); + buffer_json_member_add_uint64(wb, "qr", dimensions); + } + buffer_json_object_close(wb); // ds + } + + if(count) { + buffer_json_member_add_object(wb, "sts"); + { + buffer_json_member_add_uint64(wb, "min", min); + buffer_json_member_add_uint64(wb, "max", max); + buffer_json_member_add_double(wb, "avg", (double) sum / (double) count); + buffer_json_member_add_double(wb, "con", 100.0); + } + buffer_json_object_close(wb); // sts + } + } + buffer_json_object_close(wb); // context + } + buffer_json_array_close(wb); // contexts + + buffer_json_member_add_array(wb, "instances"); + { + buffer_json_add_array_item_object(wb); // instance + { + buffer_json_member_add_string(wb, "id", "facets.histogram"); + buffer_json_member_add_uint64(wb, "ni", 0); + + if(dimensions) { + buffer_json_member_add_object(wb, "ds"); + { + buffer_json_member_add_uint64(wb, "sl", dimensions); + buffer_json_member_add_uint64(wb, "qr", dimensions); + } + buffer_json_object_close(wb); // ds + } + + if(count) { + buffer_json_member_add_object(wb, "sts"); + { + buffer_json_member_add_uint64(wb, "min", min); + buffer_json_member_add_uint64(wb, "max", max); + buffer_json_member_add_double(wb, "avg", (double) sum / (double) count); + buffer_json_member_add_double(wb, "con", 100.0); + } + buffer_json_object_close(wb); // sts + } + } + buffer_json_object_close(wb); // instance + } + buffer_json_array_close(wb); // instances + + buffer_json_member_add_array(wb, "dimensions"); + if(dimensions && k && k->values.enabled) { + size_t pri = 0; + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if(unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_object(wb); // dimension + { + buffer_json_member_add_string(wb, "id", v->name); + buffer_json_member_add_object(wb, "ds"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // ds + buffer_json_member_add_object(wb, "sts"); + { + buffer_json_member_add_uint64(wb, "min", v->min); + buffer_json_member_add_uint64(wb, "max", v->max); + buffer_json_member_add_double(wb, "avg", (double)v->sum / (double)facets->histogram.slots); + buffer_json_member_add_double(wb, "con", (double)v->sum * 100.0 / (double)sum); + } + buffer_json_object_close(wb); // sts + buffer_json_member_add_uint64(wb, "pri", pri++); + } + buffer_json_object_close(wb); // dimension + } + foreach_value_in_key_done(v); + } + buffer_json_array_close(wb); // dimensions + + buffer_json_member_add_array(wb, "labels"); + buffer_json_array_close(wb); // labels + + buffer_json_member_add_array(wb, "alerts"); + buffer_json_array_close(wb); // alerts + } + buffer_json_object_close(wb); // summary + + buffer_json_member_add_object(wb, "totals"); + { + buffer_json_member_add_object(wb, "nodes"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // nodes + + if(dimensions) { + buffer_json_member_add_object(wb, "contexts"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // contexts + buffer_json_member_add_object(wb, "instances"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // instances + + buffer_json_member_add_object(wb, "dimensions"); + { + buffer_json_member_add_uint64(wb, "sl", dimensions); + buffer_json_member_add_uint64(wb, "qr", dimensions); + } + buffer_json_object_close(wb); // dimension + } + } + buffer_json_object_close(wb); // totals + + buffer_json_member_add_object(wb, "result"); + { + facets_histogram_value_names(wb, facets, k, "labels", "time"); + + buffer_json_member_add_object(wb, "point"); + { + buffer_json_member_add_uint64(wb, "value", 0); + buffer_json_member_add_uint64(wb, "arp", 1); + buffer_json_member_add_uint64(wb, "pa", 2); + } + buffer_json_object_close(wb); // point + + buffer_json_member_add_array(wb, "data"); + if(k && k->values.enabled) { + usec_t t = facets->histogram.after_ut; + for(uint32_t i = 0; i < facets->histogram.slots ;i++) { + buffer_json_add_array_item_array(wb); // row + { + buffer_json_add_array_item_time_ms(wb, t / USEC_PER_SEC); + + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_array(wb); // point + + buffer_json_add_array_item_uint64(wb, v->histogram[i]); + buffer_json_add_array_item_uint64(wb, 0); // arp - anomaly rate + buffer_json_add_array_item_uint64(wb, 0); // pa - point annotation + + buffer_json_array_close(wb); // point + } + foreach_value_in_key_done(v); + } + buffer_json_array_close(wb); // row + + t += facets->histogram.slot_width_ut; + } + } + buffer_json_array_close(wb); //data + } + buffer_json_object_close(wb); // result + + buffer_json_member_add_object(wb, "db"); + { + buffer_json_member_add_uint64(wb, "tiers", 1); + buffer_json_member_add_uint64(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC); +// we should add these only when we know the retention of the db +// buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC); +// buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC); + buffer_json_member_add_string(wb, "units", "events"); + buffer_json_member_add_object(wb, "dimensions"); + { + facets_histogram_value_names(wb, facets, k, "ids", NULL); + facets_histogram_value_units(wb, facets, k, "units"); + + buffer_json_member_add_object(wb, "sts"); + { + facets_histogram_value_min(wb, facets, k, "min"); + facets_histogram_value_max(wb, facets, k, "max"); + facets_histogram_value_avg(wb, facets, k, "avg"); + facets_histogram_value_arp(wb, facets, k, "arp"); + facets_histogram_value_con(wb, facets, k, "con", sum); + } + buffer_json_object_close(wb); // sts + } + buffer_json_object_close(wb); // dimensions + + buffer_json_member_add_array(wb, "per_tier"); + { + buffer_json_add_array_item_object(wb); // tier0 + { + buffer_json_member_add_uint64(wb, "tier", 0); + buffer_json_member_add_uint64(wb, "queries", 1); + buffer_json_member_add_uint64(wb, "points", count); + buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC); +// we should add these only when we know the retention of the db +// buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC); +// buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC); + } + buffer_json_object_close(wb); // tier0 + } + buffer_json_array_close(wb); // per_tier + } + buffer_json_object_close(wb); // db + + buffer_json_member_add_object(wb, "view"); + { + char title[1024 + 1] = "Events Distribution"; + FACET_KEY *kt = FACETS_KEY_GET_FROM_INDEX(facets, facets->histogram.hash); + if(kt && kt->name) + snprintfz(title, 1024, "Events Distribution by %s", kt->name); + + buffer_json_member_add_string(wb, "title", title); + buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC); + buffer_json_member_add_time_t(wb, "after", facets->histogram.after_ut / USEC_PER_SEC); + buffer_json_member_add_time_t(wb, "before", facets->histogram.before_ut / USEC_PER_SEC); + buffer_json_member_add_string(wb, "units", "events"); + buffer_json_member_add_string(wb, "chart_type", "stackedBar"); + buffer_json_member_add_object(wb, "dimensions"); + { + buffer_json_member_add_array(wb, "grouped_by"); + { + buffer_json_add_array_item_string(wb, "dimension"); + } + buffer_json_array_close(wb); // grouped_by + + facets_histogram_value_names(wb, facets, k, "ids", NULL); + facets_histogram_value_names(wb, facets, k, "names", NULL); + facets_histogram_value_units(wb, facets, k, "units"); + + buffer_json_member_add_object(wb, "sts"); + { + facets_histogram_value_min(wb, facets, k, "min"); + facets_histogram_value_max(wb, facets, k, "max"); + facets_histogram_value_avg(wb, facets, k, "avg"); + facets_histogram_value_arp(wb, facets, k, "arp"); + facets_histogram_value_con(wb, facets, k, "con", sum); + } + buffer_json_object_close(wb); // sts + } + buffer_json_object_close(wb); // dimensions + + buffer_json_member_add_uint64(wb, "min", min); + buffer_json_member_add_uint64(wb, "max", max); + } + buffer_json_object_close(wb); // view + + buffer_json_member_add_array(wb, "agents"); + { + buffer_json_add_array_item_object(wb); // agent + { + buffer_json_member_add_string(wb, "mg", "default"); + buffer_json_member_add_string(wb, "nm", "facets.histogram"); + buffer_json_member_add_time_t(wb, "now", now_realtime_sec()); + buffer_json_member_add_uint64(wb, "ai", 0); + } + buffer_json_object_close(wb); // agent + } + buffer_json_array_close(wb); // agents } -static void facet_key_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { - FACET_KEY *k = value; - FACETS *facets = data; +// ---------------------------------------------------------------------------- + +static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) { + if(!k->key_found_in_row) + v->rows_matching_facet_value++; + + k->key_found_in_row++; + + if(v->selected) + k->key_values_selected_in_row++; +} + +static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) { + bool included = true, excluded = false, never = false; + + if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) { + if(k->options & FACET_KEY_OPTION_FACET) { + included = true; + excluded = false; + never = false; + } + else if(k->options & (FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) { + included = false; + excluded = true; + never = true; + } + } + else { + if (facets->included_keys) { + if (!simple_pattern_matches(facets->included_keys, k->name)) + included = false; + } + + if (facets->excluded_keys) { + if (simple_pattern_matches(facets->excluded_keys, k->name)) { + excluded = true; + never = true; + } + } + } - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->keys_ll, k, prev, next); + if(included && !excluded) { + k->options |= FACET_KEY_OPTION_FACET; + k->options &= ~FACET_KEY_OPTION_NO_FACET; + return true; + } - dictionary_destroy(k->values); - buffer_free(k->current_value.b); - freez((char *)k->name); + k->options |= FACET_KEY_OPTION_NO_FACET; + k->options &= ~FACET_KEY_OPTION_FACET; + + if(never) + k->options |= FACET_KEY_OPTION_NEVER_FACET; + + return false; } // ---------------------------------------------------------------------------- -FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) { +FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) { FACETS *facets = callocz(1, sizeof(FACETS)); facets->options = options; - facets->keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_KEY)); - dictionary_register_insert_callback(facets->keys, facet_key_insert_callback, facets); - dictionary_register_conflict_callback(facets->keys, facet_key_conflict_callback, facets); - dictionary_register_delete_callback(facets->keys, facet_key_delete_callback, facets); + FACETS_KEYS_INDEX_CREATE(facets); if(facet_keys && *facet_keys) facets->included_keys = simple_pattern_create(facet_keys, "|", SIMPLE_PATTERN_EXACT, true); @@ -316,8 +1397,10 @@ FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS op if(visible_keys && *visible_keys) facets->visible_keys = simple_pattern_create(visible_keys, "|", SIMPLE_PATTERN_EXACT, true); - facets->max_items_to_return = items_to_return; - facets->anchor = anchor; + facets->max_items_to_return = items_to_return > 1 ? items_to_return : 2; + facets->anchor.start_ut = 0; + facets->anchor.stop_ut = 0; + facets->anchor.direction = FACETS_ANCHOR_DIRECTION_BACKWARD; facets->order = 1; return facets; @@ -325,7 +1408,7 @@ FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS op void facets_destroy(FACETS *facets) { dictionary_destroy(facets->accepted_params); - dictionary_destroy(facets->keys); + FACETS_KEYS_INDEX_DESTROY(facets); simple_pattern_free(facets->visible_keys); simple_pattern_free(facets->included_keys); simple_pattern_free(facets->excluded_keys); @@ -337,6 +1420,7 @@ void facets_destroy(FACETS *facets) { facets_row_free(facets, r); } + freez(facets->histogram.chart); freez(facets); } @@ -347,26 +1431,24 @@ void facets_accepted_param(FACETS *facets, const char *param) { dictionary_set(facets->accepted_params, param, NULL, 0); } -inline FACET_KEY *facets_register_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options) { - FACET_KEY tk = { - .name = key, - .options = options, - .default_selected_for_values = true, - }; - char hash[FACET_STRING_HASH_SIZE]; - facets_string_hash(tk.name, hash); - return dictionary_set(facets->keys, hash, &tk, sizeof(tk)); +static inline FACET_KEY *facets_register_key_name_length(FACETS *facets, const char *key, size_t key_length, FACET_KEY_OPTIONS options) { + return FACETS_KEY_ADD_TO_INDEX(facets, FACETS_HASH_FUNCTION(key, key_length), key, key_length, options); } -inline FACET_KEY *facets_register_key_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data) { - FACET_KEY *k = facets_register_key(facets, key, options); +inline FACET_KEY *facets_register_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options) { + return facets_register_key_name_length(facets, key, strlen(key), options); +} + +inline FACET_KEY *facets_register_key_name_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data) { + FACET_KEY *k = facets_register_key_name(facets, key, options); k->transform.cb = cb; k->transform.data = data; + k->transform.view_only = (options & FACET_KEY_OPTION_TRANSFORM_VIEW) ? true : false; return k; } -inline FACET_KEY *facets_register_dynamic_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data) { - FACET_KEY *k = facets_register_key(facets, key, options); +inline FACET_KEY *facets_register_dynamic_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data) { + FACET_KEY *k = facets_register_key_name(facets, key, options); k->dynamic.cb = cb; k->dynamic.data = data; return k; @@ -376,64 +1458,124 @@ void facets_set_query(FACETS *facets, const char *query) { if(!query) return; - facets->query = simple_pattern_create(query, " \t", SIMPLE_PATTERN_SUBSTRING, false); + facets->query = simple_pattern_create(query, "|", SIMPLE_PATTERN_SUBSTRING, false); } void facets_set_items(FACETS *facets, uint32_t items) { - facets->max_items_to_return = items; + facets->max_items_to_return = items > 1 ? items : 2; +} + +void facets_set_anchor(FACETS *facets, usec_t start_ut, usec_t stop_ut, FACETS_ANCHOR_DIRECTION direction) { + facets->anchor.start_ut = start_ut; + facets->anchor.stop_ut = stop_ut; + facets->anchor.direction = direction; + + if((facets->anchor.direction == FACETS_ANCHOR_DIRECTION_BACKWARD && facets->anchor.start_ut && facets->anchor.start_ut < facets->anchor.stop_ut) || + (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD && facets->anchor.stop_ut && facets->anchor.stop_ut < facets->anchor.start_ut)) { + internal_error(true, "start and stop anchors are flipped"); + facets->anchor.start_ut = stop_ut; + facets->anchor.stop_ut = start_ut; + } } -void facets_set_anchor(FACETS *facets, usec_t anchor) { - facets->anchor = anchor; +void facets_enable_slice_mode(FACETS *facets) { + facets->options |= FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS | FACETS_OPTION_SORT_FACETS_ALPHABETICALLY; } -void facets_register_facet_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options) { - FACET_KEY tk = { - .options = options, - }; - FACET_KEY *k = dictionary_set(facets->keys, key_id, &tk, sizeof(tk)); +inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options) { + if(!is_valid_string_hash(key_id)) + return NULL; - k->default_selected_for_values = false; + FACETS_HASH hash = str_to_facets_hash(key_id); + + internal_error(strcmp(hash_to_static_string(hash), key_id) != 0, + "Regenerating the user supplied key, does not produce the same hash string"); + + FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, NULL, 0, options); k->options |= FACET_KEY_OPTION_FACET; k->options &= ~FACET_KEY_OPTION_NO_FACET; facet_key_late_init(facets, k); - FACET_VALUE tv = { - .selected = true, - }; - dictionary_set(k->values, value_ids, &tv, sizeof(tv)); + return k; +} + +void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *value_id, FACET_KEY_OPTIONS options) { + FACET_KEY *k = facets_register_facet_id(facets, key_id, options); + if(k) { + if(is_valid_string_hash(value_id)) { + k->default_selected_for_values = false; + FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, str_to_facets_hash(value_id)); + } + } +} + +void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity) { + facets->current_row.severity = severity; +} + +void facets_register_row_severity(FACETS *facets, facet_row_severity_t cb, void *data) { + facets->severity.cb = cb; + facets->severity.data = data; +} + +void facets_set_additional_options(FACETS *facets, FACETS_OPTIONS options) { + facets->options |= options; } // ---------------------------------------------------------------------------- -static inline void facets_check_value(FACETS *facets __maybe_unused, FACET_KEY *k) { - if(!k->current_value.updated) - buffer_flush(k->current_value.b); +static inline void facets_key_set_empty_value(FACETS *facets, FACET_KEY *k) { + if(likely(!facet_key_value_updated(k) && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX)) + facets->keys_in_row.array[facets->keys_in_row.used++] = k; + + k->current_value.flags |= FACET_KEY_VALUE_UPDATED | FACET_KEY_VALUE_EMPTY; - if(k->transform.cb) - k->transform.cb(facets, k->current_value.b, k->transform.data); + facets->operations.values.registered++; + facets->operations.values.empty++; - if(!k->current_value.updated) { - buffer_strcat(k->current_value.b, FACET_VALUE_UNSET); - k->current_value.updated = true; + // no need to copy the UNSET value + // empty values are exported as empty + k->current_value.raw = NULL; + k->current_value.raw_len = 0; + k->current_value.b->len = 0; + k->current_value.flags &= ~FACET_KEY_VALUE_COPIED; + + if(unlikely(k->values.enabled)) + FACET_VALUE_ADD_EMPTY_VALUE_TO_INDEX(k); + else { + k->key_found_in_row++; + k->key_values_selected_in_row++; + } +} + +static inline void facets_key_check_value(FACETS *facets, FACET_KEY *k) { + if(likely(!facet_key_value_updated(k) && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX)) + facets->keys_in_row.array[facets->keys_in_row.used++] = k; + + k->current_value.flags |= FACET_KEY_VALUE_UPDATED; + k->current_value.flags &= ~FACET_KEY_VALUE_EMPTY; + + facets->operations.values.registered++; + + if(k->transform.cb && !k->transform.view_only) { + facets->operations.values.transformed++; + facets_key_value_copy_to_buffer(k); + k->transform.cb(facets, k->current_value.b, FACETS_TRANSFORM_VALUE, k->transform.data); } // bool found = false; // if(strstr(buffer_tostring(k->current_value), "fprintd") != NULL) // found = true; - if(facets->query && ((k->options & FACET_KEY_OPTION_FTS) || facets->options & FACETS_OPTION_ALL_KEYS_FTS)) { + if(facets->query && !facet_key_value_empty(k) && ((k->options & FACET_KEY_OPTION_FTS) || facets->options & FACETS_OPTION_ALL_KEYS_FTS)) { + facets->operations.fts.searches++; + facets_key_value_copy_to_buffer(k); if(simple_pattern_matches(facets->query, buffer_tostring(k->current_value.b))) - facets->keys_matched_by_query++; + facets->current_row.keys_matched_by_query++; } - if(k->values) { - FACET_VALUE tk = { - .name = buffer_tostring(k->current_value.b), - }; - facets_string_hash(tk.name, k->current_value.hash); - dictionary_set(k->values, k->current_value.hash, &tk, sizeof(tk)); - } + if(k->values.enabled) + FACET_VALUE_ADD_CURRENT_VALUE_TO_INDEX(k); else { k->key_found_in_row++; k->key_values_selected_in_row++; @@ -441,21 +1583,19 @@ static inline void facets_check_value(FACETS *facets __maybe_unused, FACET_KEY * } void facets_add_key_value(FACETS *facets, const char *key, const char *value) { - FACET_KEY *k = facets_register_key(facets, key, 0); - buffer_flush(k->current_value.b); - buffer_strcat(k->current_value.b, value); - k->current_value.updated = true; + FACET_KEY *k = facets_register_key_name(facets, key, 0); + k->current_value.raw = value; + k->current_value.raw_len = strlen(value); - facets_check_value(facets, k); + facets_key_check_value(facets, k); } -void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len) { - FACET_KEY *k = facets_register_key(facets, key, 0); - buffer_flush(k->current_value.b); - buffer_strncat(k->current_value.b, value, value_len); - k->current_value.updated = true; +void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len) { + FACET_KEY *k = facets_register_key_name_length(facets, key, key_len, 0); + k->current_value.raw = value; + k->current_value.raw_len = value_len; - facets_check_value(facets, k); + facets_key_check_value(facets, k); } // ---------------------------------------------------------------------------- @@ -466,7 +1606,8 @@ static void facet_row_key_value_insert_callback(const DICTIONARY_ITEM *item __ma FACET_ROW *row = data; (void)row; rkv->wb = buffer_create(0, NULL); - buffer_strcat(rkv->wb, rkv->tmp && *rkv->tmp ? rkv->tmp : FACET_VALUE_UNSET); + if(!rkv->empty) + buffer_contents_replace(rkv->wb, rkv->tmp, rkv->tmp_len); } static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) { @@ -474,8 +1615,12 @@ static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __ FACET_ROW_KEY_VALUE *n_rkv = new_value; FACET_ROW *row = data; (void)row; - buffer_flush(rkv->wb); - buffer_strcat(rkv->wb, n_rkv->tmp && *n_rkv->tmp ? n_rkv->tmp : FACET_VALUE_UNSET); + rkv->empty = n_rkv->empty; + + if(!rkv->empty) + buffer_contents_replace(rkv->wb, n_rkv->tmp, n_rkv->tmp_len); + else + buffer_flush(rkv->wb); return false; } @@ -498,52 +1643,52 @@ static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) { static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into) { FACET_ROW *row; - if(into) + if(into) { row = into; + facets->operations.rows.reused++; + } else { row = callocz(1, sizeof(FACET_ROW)); row->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_ROW_KEY_VALUE)); dictionary_register_insert_callback(row->dict, facet_row_key_value_insert_callback, row); dictionary_register_conflict_callback(row->dict, facet_row_key_value_conflict_callback, row); dictionary_register_delete_callback(row->dict, facet_row_key_value_delete_callback, row); + facets->operations.rows.created++; } + row->severity = facets->current_row.severity; row->usec = usec; FACET_KEY *k; - dfe_start_read(facets->keys, k) { + foreach_key_in_facets(facets, k) { FACET_ROW_KEY_VALUE t = { - .tmp = (k->current_value.updated && buffer_strlen(k->current_value.b)) ? - buffer_tostring(k->current_value.b) : FACET_VALUE_UNSET, + .tmp = NULL, + .tmp_len = 0, .wb = NULL, + .empty = true, }; + + if(facet_key_value_updated(k) && !facet_key_value_empty(k)) { + t.tmp = facets_key_get_value(k); + t.tmp_len = facets_key_get_value_length(k); + t.empty = false; + } + dictionary_set(row->dict, k->name, &t, sizeof(t)); } - dfe_done(k); + foreach_key_in_facets_done(k); return row; } // ---------------------------------------------------------------------------- -static void facets_row_keep(FACETS *facets, usec_t usec) { - facets->operations.matched++; - - if(usec < facets->anchor) { - facets->operations.skips_before++; - return; - } - - if(unlikely(!facets->base)) { - facets->operations.last_added = facets_row_create(facets, usec, NULL); - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next); - facets->items_to_return++; - facets->operations.first++; - return; - } +static inline FACET_ROW *facets_row_keep_seek_to_position(FACETS *facets, usec_t usec) { + if(usec < facets->base->prev->usec) + return facets->base->prev; - if(likely(usec > facets->base->prev->usec)) - facets->operations.last_added = facets->base->prev; + if(usec > facets->base->usec) + return facets->base; FACET_ROW *last = facets->operations.last_added; while(last->prev != facets->base->prev && usec > last->prev->usec) { @@ -556,137 +1701,248 @@ static void facets_row_keep(FACETS *facets, usec_t usec) { facets->operations.forwards++; } - if(facets->items_to_return >= facets->max_items_to_return) { - if(last == facets->base->prev && usec < last->usec) { - facets->operations.skips_after++; - return; + return last; +} + +static void facets_row_keep_first_entry(FACETS *facets, usec_t usec) { + facets->operations.last_added = facets_row_create(facets, usec, NULL); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next); + facets->items_to_return++; + facets->operations.first++; +} + +static inline bool facets_is_entry_within_anchor(FACETS *facets, usec_t usec) { + if(facets->anchor.start_ut || facets->anchor.stop_ut) { + // we have an anchor key + // we don't want to keep rows on the other side of the direction + + switch (facets->anchor.direction) { + default: + case FACETS_ANCHOR_DIRECTION_BACKWARD: + // we need to keep only the smaller timestamps + if (facets->anchor.start_ut && usec >= facets->anchor.start_ut) { + facets->operations.skips_before++; + return false; + } + if (facets->anchor.stop_ut && usec <= facets->anchor.stop_ut) { + facets->operations.skips_after++; + return false; + } + break; + + case FACETS_ANCHOR_DIRECTION_FORWARD: + // we need to keep only the bigger timestamps + if (facets->anchor.start_ut && usec <= facets->anchor.start_ut) { + facets->operations.skips_after++; + return false; + } + if (facets->anchor.stop_ut && usec >= facets->anchor.stop_ut) { + facets->operations.skips_before++; + return false; + } + break; } } - facets->items_to_return++; + return true; +} + +static void facets_row_keep(FACETS *facets, usec_t usec) { + facets->operations.rows.matched++; + + if(unlikely(!facets->base)) { + // the first row to keep + facets_row_keep_first_entry(facets, usec); + return; + } + + FACET_ROW *closest = facets_row_keep_seek_to_position(facets, usec); + FACET_ROW *to_replace = NULL; + + if(likely(facets->items_to_return >= facets->max_items_to_return)) { + // we have enough items to return already + + switch(facets->anchor.direction) { + default: + case FACETS_ANCHOR_DIRECTION_BACKWARD: + if(closest == facets->base->prev && usec < closest->usec) { + // this is to the end of the list, belonging to the next page + facets->operations.skips_after++; + return; + } + + // it seems we need to remove an item - the last one + to_replace = facets->base->prev; + if(closest == to_replace) + closest = to_replace->prev; + + break; + + case FACETS_ANCHOR_DIRECTION_FORWARD: + if(closest == facets->base && usec > closest->usec) { + // this is to the beginning of the list, belonging to the next page + facets->operations.skips_before++; + return; + } + + // it seems we need to remove an item - the first one + to_replace = facets->base; + if(closest == to_replace) + closest = to_replace->next; - if(usec > last->usec) { - if(facets->items_to_return > facets->max_items_to_return) { - facets->items_to_return--; - facets->operations.shifts++; - facets->operations.last_added = facets->base->prev; - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next); - facets->operations.last_added = facets_row_create(facets, usec, facets->operations.last_added); + break; } - DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next); - facets->operations.prepends++; + + facets->operations.shifts++; + facets->items_to_return--; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, to_replace, prev, next); } - else { - facets->operations.last_added = facets_row_create(facets, usec, NULL); - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next); + + internal_fatal(!closest, "FACETS: closest cannot be NULL"); + internal_fatal(closest == to_replace, "FACETS: closest cannot be the same as to_replace"); + + facets->operations.last_added = facets_row_create(facets, usec, to_replace); + + if(usec < closest->usec) { + DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next); facets->operations.appends++; } + else { + DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next); + facets->operations.prepends++; + } - while(facets->items_to_return > facets->max_items_to_return) { - // we have to remove something + facets->items_to_return++; +} - FACET_ROW *tmp = facets->base->prev; - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, tmp, prev, next); - facets->items_to_return--; +static inline void facets_reset_key(FACET_KEY *k) { + k->key_found_in_row = 0; + k->key_values_selected_in_row = 0; + k->current_value.flags = FACET_KEY_VALUE_NONE; + k->current_value.hash = FACETS_HASH_ZERO; +} - if(unlikely(facets->operations.last_added == tmp)) - facets->operations.last_added = facets->base->prev; +static void facets_reset_keys_with_value_and_row(FACETS *facets) { + size_t entries = facets->keys_in_row.used; - facets_row_free(facets, tmp); - facets->operations.shifts++; + for(size_t p = 0; p < entries ;p++) { + FACET_KEY *k = facets->keys_in_row.array[p]; + facets_reset_key(k); } + + facets->current_row.severity = FACET_ROW_SEVERITY_NORMAL; + facets->current_row.keys_matched_by_query = 0; + facets->keys_in_row.used = 0; } void facets_rows_begin(FACETS *facets) { FACET_KEY *k; - // dfe_start_read(facets->keys, k) { - for(k = facets->keys_ll ; k ; k = k->next) { - k->key_found_in_row = 0; - k->key_values_selected_in_row = 0; - k->current_value.updated = false; - k->current_value.hash[0] = '\0'; + foreach_key_in_facets(facets, k) { + facets_reset_key(k); } - // dfe_done(k); + foreach_key_in_facets_done(k); - facets->keys_matched_by_query = 0; + facets->keys_in_row.used = 0; + facets_reset_keys_with_value_and_row(facets); } -void facets_row_finished(FACETS *facets, usec_t usec) { - if(facets->query && facets->keys_filtered_by_query && !facets->keys_matched_by_query) - goto cleanup; +bool facets_row_finished(FACETS *facets, usec_t usec) { + facets->operations.rows.evaluated++; + + if(unlikely((facets->query && facets->keys_filtered_by_query && !facets->current_row.keys_matched_by_query) || + (facets->timeframe.before_ut && usec > facets->timeframe.before_ut) || + (facets->timeframe.after_ut && usec < facets->timeframe.after_ut))) { + // this row is not useful + // 1. not matched by full text search, or + // 2. not in our timeframe + facets_reset_keys_with_value_and_row(facets); + return false; + } - facets->operations.evaluated++; + bool within_anchor = facets_is_entry_within_anchor(facets, usec); + if(unlikely(!within_anchor && (facets->options & FACETS_OPTION_DATA_ONLY))) { + facets_reset_keys_with_value_and_row(facets); + return false; + } - uint32_t total_keys = 0; - uint32_t selected_by = 0; + size_t entries = facets->keys_with_values.used; + size_t total_keys = 0; + size_t selected_keys = 0; - FACET_KEY *k; - // dfe_start_read(facets->keys, k) { - for(k = facets->keys_ll ; k ; k = k->next) { - if(!k->key_found_in_row) { + for(size_t p = 0; p < entries ;p++) { + FACET_KEY *k = facets->keys_with_values.array[p]; + + if(!facet_key_value_updated(k)) // put the FACET_VALUE_UNSET value into it - facets_check_value(facets, k); - } + facets_key_set_empty_value(facets, k); - internal_fatal(!k->key_found_in_row, "all keys should be found in the row at this point"); - internal_fatal(k->key_found_in_row != 1, "all keys should be matched exactly once at this point"); - internal_fatal(k->key_values_selected_in_row > 1, "key values are selected in row more than once"); + total_keys++; - k->key_found_in_row = 1; + if(k->key_values_selected_in_row) + selected_keys++; - total_keys += k->key_found_in_row; - selected_by += (k->key_values_selected_in_row) ? 1 : 0; + if(unlikely(!facets->histogram.key && facets->histogram.hash == k->hash)) + facets->histogram.key = k; } - // dfe_done(k); - if(selected_by >= total_keys - 1) { - uint32_t found = 0; + if(selected_keys >= total_keys - 1) { + size_t found = 0; + (void) found; + + for(size_t p = 0; p < entries; p++) { + FACET_KEY *k = facets->keys_with_values.array[p]; - // dfe_start_read(facets->keys, k){ - for(k = facets->keys_ll ; k ; k = k->next) { - uint32_t counted_by = selected_by; + size_t counted_by = selected_keys; - if (counted_by != total_keys && !k->key_values_selected_in_row) + if(counted_by != total_keys && !k->key_values_selected_in_row) counted_by++; if(counted_by == total_keys) { - if(k->values) { - if(!k->current_value.hash[0]) - facets_string_hash(buffer_tostring(k->current_value.b), k->current_value.hash); - - FACET_VALUE *v = dictionary_get(k->values, k->current_value.hash); - v->final_facet_value_counter++; - } - + k->current_value.v->final_facet_value_counter++; found++; } } - // dfe_done(k); internal_fatal(!found, "We should find at least one facet to count this row"); - (void)found; } - if(selected_by == total_keys) - facets_row_keep(facets, usec); + if(selected_keys == total_keys) { + // we need to keep this row + facets_histogram_update_value(facets, usec); + + if(within_anchor) + facets_row_keep(facets, usec); + } + + facets_reset_keys_with_value_and_row(facets); -cleanup: - facets_rows_begin(facets); + return selected_keys == total_keys; } // ---------------------------------------------------------------------------- // output -void facets_report(FACETS *facets, BUFFER *wb) { - buffer_json_member_add_boolean(wb, "show_ids", false); - buffer_json_member_add_boolean(wb, "has_history", true); +static const char *facets_severity_to_string(FACET_ROW_SEVERITY severity) { + switch(severity) { + default: + case FACET_ROW_SEVERITY_NORMAL: + return "normal"; - buffer_json_member_add_object(wb, "pagination"); - buffer_json_member_add_boolean(wb, "enabled", true); - buffer_json_member_add_string(wb, "key", "anchor"); - buffer_json_member_add_string(wb, "column", "timestamp"); - buffer_json_object_close(wb); + case FACET_ROW_SEVERITY_DEBUG: + return "debug"; + + case FACET_ROW_SEVERITY_NOTICE: + return "notice"; + + case FACET_ROW_SEVERITY_WARNING: + return "warning"; + + case FACET_ROW_SEVERITY_CRITICAL: + return "critical"; + } +} +void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool with_keys) { buffer_json_member_add_array(wb, "accepted_params"); { if(facets->accepted_params) { @@ -697,54 +1953,338 @@ void facets_report(FACETS *facets, BUFFER *wb) { dfe_done(t); } - FACET_KEY *k; - dfe_start_read(facets->keys, k) { - if(!k->values) - continue; + if(with_keys) { + FACET_KEY *k; + foreach_key_in_facets(facets, k){ + if (!k->values.enabled) + continue; - buffer_json_add_array_item_string(wb, k_dfe.name); + buffer_json_add_array_item_string(wb, hash_to_static_string(k->hash)); + } + foreach_key_in_facets_done(k); } - dfe_done(k); } buffer_json_array_close(wb); // accepted_params +} + +static int facets_keys_reorder_compar(const void *a, const void *b) { + const FACET_KEY *ak = *((const FACET_KEY **)a); + const FACET_KEY *bk = *((const FACET_KEY **)b); + + const char *an = ak->name; + const char *bn = bk->name; + + if(!an) an = "0"; + if(!bn) bn = "0"; + + while(*an && ispunct(*an)) an++; + while(*bn && ispunct(*bn)) bn++; + + return strcasecmp(an, bn); +} + +void facets_sort_and_reorder_keys(FACETS *facets) { + size_t entries = facets->keys_with_values.used; + if(!entries) + return; + + FACET_KEY *keys[entries]; + memcpy(keys, facets->keys_with_values.array, sizeof(FACET_KEY *) * entries); + + qsort(keys, entries, sizeof(FACET_KEY *), facets_keys_reorder_compar); + + for(size_t i = 0; i < entries ;i++) + keys[i]->order = i + 1; +} + +static int facets_key_values_reorder_by_name_compar(const void *a, const void *b) { + const FACET_VALUE *av = *((const FACET_VALUE **)a); + const FACET_VALUE *bv = *((const FACET_VALUE **)b); + + const char *an = (av->name && av->name_len) ? av->name : "0"; + const char *bn = (bv->name && bv->name_len) ? bv->name : "0"; + + while(*an && ispunct(*an)) an++; + while(*bn && ispunct(*bn)) bn++; + + int ret = strcasecmp(an, bn); + return ret; +} + +static int facets_key_values_reorder_by_count_compar(const void *a, const void *b) { + const FACET_VALUE *av = *((const FACET_VALUE **)a); + const FACET_VALUE *bv = *((const FACET_VALUE **)b); + + if(av->final_facet_value_counter < bv->final_facet_value_counter) + return 1; + + if(av->final_facet_value_counter > bv->final_facet_value_counter) + return -1; + + return facets_key_values_reorder_by_name_compar(a, b); +} + +static int facets_key_values_reorder_by_name_numeric_compar(const void *a, const void *b) { + const FACET_VALUE *av = *((const FACET_VALUE **)a); + const FACET_VALUE *bv = *((const FACET_VALUE **)b); - buffer_json_member_add_array(wb, "facets"); + const char *an = (av->name && av->name_len) ? av->name : "0"; + const char *bn = (bv->name && bv->name_len) ? bv->name : "0"; + + if(strcmp(an, FACET_VALUE_UNSET) == 0) an = "0"; + if(strcmp(bn, FACET_VALUE_UNSET) == 0) bn = "0"; + + int64_t ad = str2ll(an, NULL); + int64_t bd = str2ll(bn, NULL); + + if(ad < bd) + return -1; + + if(ad > bd) + return 1; + + return facets_key_values_reorder_by_name_compar(a, b); +} + +static uint32_t facets_sort_and_reorder_values_internal(FACET_KEY *k) { + bool all_values_numeric = true; + size_t entries = k->values.used; + FACET_VALUE *values[entries], *v; + uint32_t used = 0; + foreach_value_in_key(k, v) { + if((k->facets->options & FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS) && v->empty) + continue; + + if(all_values_numeric && !v->empty && v->name && v->name_len) { + const char *s = v->name; + while(isdigit(*s)) s++; + if(*s != '\0') + all_values_numeric = false; + } + + values[used++] = v; + + if(used >= entries) + break; + } + foreach_value_in_key_done(v); + + if(!used) + return 0; + + if(k->facets->options & FACETS_OPTION_SORT_FACETS_ALPHABETICALLY) { + if(all_values_numeric) + qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_name_numeric_compar); + else + qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_name_compar); + } + else + qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_count_compar); + + for(size_t i = 0; i < used; i++) + values[i]->order = i + 1; + + return used; +} + +static uint32_t facets_sort_and_reorder_values(FACET_KEY *k) { + if(!k->values.enabled || !k->values.ll || !k->values.used) + return 0; + + if(!k->transform.cb || !(k->facets->options & FACETS_OPTION_SORT_FACETS_ALPHABETICALLY)) + return facets_sort_and_reorder_values_internal(k); + + // we have a transformation and has to be sorted alphabetically + + BUFFER *tb = buffer_create(0, NULL); + uint32_t ret = 0; + + size_t entries = k->values.used; + struct { + const char *name; + uint32_t name_len; + } values[entries]; + FACET_VALUE *v; + uint32_t used = 0; + + foreach_value_in_key(k, v) { + if(used >= entries) + break; + + values[used].name = v->name; + values[used].name_len = v->name_len; + used++; + + buffer_contents_replace(tb, v->name, v->name_len); + k->transform.cb(k->facets, tb, FACETS_TRANSFORM_FACET_SORT, k->transform.data); + v->name = strdupz(buffer_tostring(tb)); + v->name_len = buffer_strlen(tb); + } + foreach_value_in_key_done(v); + + ret = facets_sort_and_reorder_values_internal(k); + + used = 0; + foreach_value_in_key(k, v) { + if(used >= entries) + break; + + freez((void *)v->name); + v->name = values[used].name; + v->name_len = values[used].name_len; + used++; + } + foreach_value_in_key_done(v); + + buffer_free(tb); + return ret; +} + +void facets_table_config(BUFFER *wb) { + buffer_json_member_add_boolean(wb, "show_ids", false); // do not show the column ids to the user + buffer_json_member_add_boolean(wb, "has_history", true); // enable date-time picker with after-before + + buffer_json_member_add_object(wb, "pagination"); { - FACET_KEY *k; - dfe_start_read(facets->keys, k) { - if(!k->values) - continue; + buffer_json_member_add_boolean(wb, "enabled", true); + buffer_json_member_add_string(wb, "key", "anchor"); + buffer_json_member_add_string(wb, "column", "timestamp"); + buffer_json_member_add_string(wb, "units", "timestamp_usec"); + } + buffer_json_object_close(wb); // pagination +} - buffer_json_add_array_item_object(wb); // key - { - buffer_json_member_add_string(wb, "id", k_dfe.name); - buffer_json_member_add_string(wb, "name", k->name); +static const char *facets_json_key_name_string(FACET_KEY *k, DICTIONARY *used_hashes_registry) { + if(k->name) { + if(used_hashes_registry && !k->default_selected_for_values) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(k->hash, hash_str); + dictionary_set(used_hashes_registry, hash_str, (void *)k->name, strlen(k->name) + 1); + } - if(!k->order) - k->order = facets->order++; + return k->name; + } - buffer_json_member_add_uint64(wb, "order", k->order); - buffer_json_member_add_array(wb, "options"); + // key has no name + const char *name = "[UNAVAILABLE_FIELD]"; + + if(used_hashes_registry) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(k->hash, hash_str); + const char *s = dictionary_get(used_hashes_registry, hash_str); + if(s) name = s; + } + + return name; +} + +static const char *facets_json_key_value_string(FACET_KEY *k, FACET_VALUE *v, DICTIONARY *used_hashes_registry) { + if(v->name && v->name_len) { + if(used_hashes_registry && !k->default_selected_for_values && v->selected) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(v->hash, hash_str); + dictionary_set(used_hashes_registry, hash_str, (void *)v->name, v->name_len + 1); + } + + return v->name; + } + + // key has no name + const char *name = "[unavailable field]"; + + if(used_hashes_registry) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(v->hash, hash_str); + const char *s = dictionary_get(used_hashes_registry, hash_str); + if(s) name = s; + } + + return name; +} + +void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) { + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) { + facets_table_config(wb); + facets_accepted_parameters_to_json_array(facets, wb, true); + } + + // ------------------------------------------------------------------------ + // facets + + if(!(facets->options & FACETS_OPTION_DONT_SEND_FACETS)) { + bool show_facets = false; + + if(facets->options & FACETS_OPTION_DATA_ONLY) { + if(facets->options & FACETS_OPTION_SHOW_DELTAS) { + buffer_json_member_add_array(wb, "facets_delta"); + show_facets = true; + } + } + else { + buffer_json_member_add_array(wb, "facets"); + show_facets = true; + } + + if(show_facets) { + BUFFER *tb = NULL; + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + if(!k->values.enabled) + continue; + + if(!facets_sort_and_reorder_values(k)) + // no values for this key + continue; + + buffer_json_add_array_item_object(wb); // key { - FACET_VALUE *v; - dfe_start_read(k->values, v) { - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "id", v_dfe.name); - buffer_json_member_add_string(wb, "name", v->name); - buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter); + buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash)); + buffer_json_member_add_string(wb, "name", facets_json_key_name_string(k, used_hashes_registry)); + + if(!k->order) k->order = facets->order++; + buffer_json_member_add_uint64(wb, "order", k->order); + + buffer_json_member_add_array(wb, "options"); + { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if((facets->options & FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS) && v->empty) + continue; + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "id", hash_to_static_string(v->hash)); + + if(!v->empty && k->transform.cb && k->transform.view_only) { + if(!tb) + tb = buffer_create(0, NULL); + + buffer_contents_replace(tb, v->name, v->name_len); + k->transform.cb(facets, tb, FACETS_TRANSFORM_FACET, k->transform.data); + buffer_json_member_add_string(wb, "name", buffer_tostring(tb)); + } + else + buffer_json_member_add_string(wb, "name", facets_json_key_value_string(k, v, used_hashes_registry)); + + buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter); + buffer_json_member_add_uint64(wb, "order", v->order); + } + buffer_json_object_close(wb); } - buffer_json_object_close(wb); + foreach_value_in_key_done(v); } - dfe_done(v); + buffer_json_array_close(wb); // options } - buffer_json_array_close(wb); // options + buffer_json_object_close(wb); // key } - buffer_json_object_close(wb); // key + foreach_key_in_facets_done(k); + buffer_free(tb); + buffer_json_array_close(wb); // facets } - dfe_done(k); } - buffer_json_array_close(wb); // facets + + // ------------------------------------------------------------------------ + // columns buffer_json_member_add_object(wb, "columns"); { @@ -755,56 +2295,94 @@ void facets_report(FACETS *facets, BUFFER *wb) { RRDF_FIELD_TYPE_TIMESTAMP, RRDF_FIELD_VISUAL_VALUE, RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN, - RRDF_FIELD_SORT_DESCENDING, + RRDF_FIELD_SORT_DESCENDING|RRDF_FIELD_SORT_FIXED, NULL, RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_RANGE, - RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY, + RRDF_FIELD_OPTS_WRAP | RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY, NULL); - FACET_KEY *k; - dfe_start_read(facets->keys, k) { - RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_NONE; - bool visible = k->options & (FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_STICKY); - - if((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values)) - visible = true; - - if(!visible) - visible = simple_pattern_matches(facets->visible_keys, k->name); - - if(visible) - options |= RRDF_FIELD_OPTS_VISIBLE; - - if(k->options & FACET_KEY_OPTION_MAIN_TEXT) - options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP; + buffer_rrdf_table_add_field( + wb, field_id++, + "rowOptions", "rowOptions", + RRDF_FIELD_TYPE_NONE, + RRDR_FIELD_VISUAL_ROW_OPTIONS, + RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, + RRDF_FIELD_SORT_FIXED, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + RRDF_FIELD_FILTER_NONE, + RRDR_FIELD_OPTS_DUMMY, + NULL); - buffer_rrdf_table_add_field( - wb, field_id++, - k_dfe.name, k->name ? k->name : k_dfe.name, - RRDF_FIELD_TYPE_STRING, - RRDF_FIELD_VISUAL_VALUE, - RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, - RRDF_FIELD_SORT_ASCENDING, - NULL, - RRDF_FIELD_SUMMARY_COUNT, - k->values ? RRDF_FIELD_FILTER_FACET : RRDF_FIELD_FILTER_NONE, - options, - FACET_VALUE_UNSET); - } - dfe_done(k); + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP; + bool visible = k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY); + + if ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values.enabled)) + visible = true; + + if (!visible) + visible = simple_pattern_matches(facets->visible_keys, k->name); + + if (visible) + options |= RRDF_FIELD_OPTS_VISIBLE; + + if (k->options & FACET_KEY_OPTION_MAIN_TEXT) + options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP; + + const char *hash_str = hash_to_static_string(k->hash); + + buffer_rrdf_table_add_field( + wb, field_id++, + hash_str, k->name ? k->name : hash_str, + RRDF_FIELD_TYPE_STRING, + (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE, + RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, + RRDF_FIELD_SORT_FIXED, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE + : RRDF_FIELD_FILTER_FACET, + options, FACET_VALUE_UNSET); + } + foreach_key_in_facets_done(k); } buffer_json_object_close(wb); // columns + // ------------------------------------------------------------------------ + // rows data + buffer_json_member_add_array(wb, "data"); { + usec_t last_usec = 0; (void)last_usec; + for(FACET_ROW *row = facets->base ; row ;row = row->next) { + + internal_fatal( + facets->anchor.start_ut && ( + (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_BACKWARD && row->usec >= facets->anchor.start_ut) || + (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD && row->usec <= facets->anchor.start_ut) + ), "Wrong data returned related to %s start anchor!", facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward"); + + internal_fatal(last_usec && row->usec > last_usec, "Wrong order of data returned!"); + + last_usec = row->usec; + buffer_json_add_array_item_array(wb); // each row buffer_json_add_array_item_uint64(wb, row->usec); + buffer_json_add_array_item_object(wb); + { + if(facets->severity.cb) + row->severity = facets->severity.cb(facets, row, facets->severity.data); + + buffer_json_member_add_string(wb, "severity", facets_severity_to_string(row->severity)); + } + buffer_json_object_close(wb); FACET_KEY *k; - dfe_start_read(facets->keys, k) - { + foreach_key_in_facets(facets, k) { FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name); if(unlikely(k->dynamic.cb)) { @@ -812,32 +2390,117 @@ void facets_report(FACETS *facets, BUFFER *wb) { rkv = dictionary_set(row->dict, k->name, NULL, sizeof(*rkv)); k->dynamic.cb(facets, wb, rkv, row, k->dynamic.data); + facets->operations.values.dynamic++; + } + else { + if(!rkv || rkv->empty) { + buffer_json_add_array_item_string(wb, NULL); + } + else if(unlikely(k->transform.cb && k->transform.view_only)) { + k->transform.cb(facets, rkv->wb, FACETS_TRANSFORM_DATA, k->transform.data); + buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb)); + } + else + buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb)); } - else - buffer_json_add_array_item_string(wb, rkv ? buffer_tostring(rkv->wb) : FACET_VALUE_UNSET); } - dfe_done(k); + foreach_key_in_facets_done(k); buffer_json_array_close(wb); // each row } } buffer_json_array_close(wb); // data - buffer_json_member_add_string(wb, "default_sort_column", "timestamp"); - buffer_json_member_add_array(wb, "default_charts"); - buffer_json_array_close(wb); + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) { + buffer_json_member_add_string(wb, "default_sort_column", "timestamp"); + buffer_json_member_add_array(wb, "default_charts"); + buffer_json_array_close(wb); + } - buffer_json_member_add_object(wb, "items"); - { - buffer_json_member_add_uint64(wb, "evaluated", facets->operations.evaluated); - buffer_json_member_add_uint64(wb, "matched", facets->operations.matched); + // ------------------------------------------------------------------------ + // histogram + + if(facets->histogram.enabled && !(facets->options & FACETS_OPTION_DONT_SEND_HISTOGRAM)) { + FACETS_HASH first_histogram_hash = 0; + buffer_json_member_add_array(wb, "available_histograms"); + { + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + if (!k->values.enabled) + continue; + + if(unlikely(!first_histogram_hash)) + first_histogram_hash = k->hash; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash)); + buffer_json_member_add_string(wb, "name", k->name); + buffer_json_member_add_uint64(wb, "order", k->order); + buffer_json_object_close(wb); + } + foreach_key_in_facets_done(k); + } + buffer_json_array_close(wb); + + { + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, facets->histogram.hash); + if(!k || !k->values.enabled) + k = FACETS_KEY_GET_FROM_INDEX(facets, first_histogram_hash); + + bool show_histogram = false; + + if(facets->options & FACETS_OPTION_DATA_ONLY) { + if(facets->options & FACETS_OPTION_SHOW_DELTAS) { + buffer_json_member_add_object(wb, "histogram_delta"); + show_histogram = true; + } + } + else { + buffer_json_member_add_object(wb, "histogram"); + show_histogram = true; + } + + if(show_histogram) { + buffer_json_member_add_string(wb, "id", k ? hash_to_static_string(k->hash) : ""); + buffer_json_member_add_string(wb, "name", k ? k->name : ""); + buffer_json_member_add_object(wb, "chart"); + { + facets_histogram_generate(facets, k, wb); + } + buffer_json_object_close(wb); // chart + buffer_json_object_close(wb); // histogram + } + } + } + + // ------------------------------------------------------------------------ + // items + + bool show_items = false; + if(facets->options & FACETS_OPTION_DATA_ONLY) { + if(facets->options & FACETS_OPTION_SHOW_DELTAS) { + buffer_json_member_add_object(wb, "items_delta"); + show_items = true; + } + } + else { + buffer_json_member_add_object(wb, "items"); + show_items = true; + } + + if(show_items) { + buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated); + buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched); buffer_json_member_add_uint64(wb, "returned", facets->items_to_return); buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return); buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before); buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts); + buffer_json_object_close(wb); // items } - buffer_json_object_close(wb); // items - buffer_json_member_add_object(wb, "stats"); + // ------------------------------------------------------------------------ + // stats + + buffer_json_member_add_object(wb, "_stats"); { buffer_json_member_add_uint64(wb, "first", facets->operations.first); buffer_json_member_add_uint64(wb, "forwards", facets->operations.forwards); @@ -847,7 +2510,38 @@ void facets_report(FACETS *facets, BUFFER *wb) { buffer_json_member_add_uint64(wb, "prepends", facets->operations.prepends); buffer_json_member_add_uint64(wb, "appends", facets->operations.appends); buffer_json_member_add_uint64(wb, "shifts", facets->operations.shifts); + buffer_json_member_add_object(wb, "rows"); + { + buffer_json_member_add_uint64(wb, "created", facets->operations.rows.created); + buffer_json_member_add_uint64(wb, "reused", facets->operations.rows.reused); + buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated); + buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched); + } + buffer_json_object_close(wb); // rows + buffer_json_member_add_object(wb, "keys"); + { + buffer_json_member_add_uint64(wb, "registered", facets->operations.keys.registered); + buffer_json_member_add_uint64(wb, "unique", facets->operations.keys.unique); + buffer_json_member_add_uint64(wb, "hashtable_increases", facets->operations.keys.hashtable_increases); + } + buffer_json_object_close(wb); // keys + buffer_json_member_add_object(wb, "values"); + { + buffer_json_member_add_uint64(wb, "registered", facets->operations.values.registered); + buffer_json_member_add_uint64(wb, "transformed", facets->operations.values.transformed); + buffer_json_member_add_uint64(wb, "dynamic", facets->operations.values.dynamic); + buffer_json_member_add_uint64(wb, "empty", facets->operations.values.empty); + buffer_json_member_add_uint64(wb, "indexed", facets->operations.values.indexed); + buffer_json_member_add_uint64(wb, "inserts", facets->operations.values.inserts); + buffer_json_member_add_uint64(wb, "conflicts", facets->operations.values.conflicts); + buffer_json_member_add_uint64(wb, "hashtable_increases", facets->operations.values.hashtable_increases); + } + buffer_json_object_close(wb); // values + buffer_json_member_add_object(wb, "fts"); + { + buffer_json_member_add_uint64(wb, "searches", facets->operations.fts.searches); + } + buffer_json_object_close(wb); // fts } buffer_json_object_close(wb); // items - } diff --git a/libnetdata/facets/facets.h b/libnetdata/facets/facets.h index 796d15a04..759725617 100644 --- a/libnetdata/facets/facets.h +++ b/libnetdata/facets/facets.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + #ifndef FACETS_H #define FACETS_H 1 @@ -6,59 +8,111 @@ #define FACET_VALUE_UNSET "-" typedef enum __attribute__((packed)) { - FACET_KEY_OPTION_FACET = (1 << 0), // filterable values - FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value - FACET_KEY_OPTION_STICKY = (1 << 2), // should be sticky in the table - FACET_KEY_OPTION_VISIBLE = (1 << 3), // should be in the default table - FACET_KEY_OPTION_FTS = (1 << 4), // the key is filterable by full text search (FTS) - FACET_KEY_OPTION_MAIN_TEXT = (1 << 5), // full width and wrap - FACET_KEY_OPTION_REORDER = (1 << 6), // give the key a new order id on first encounter + FACETS_ANCHOR_DIRECTION_FORWARD, + FACETS_ANCHOR_DIRECTION_BACKWARD, +} FACETS_ANCHOR_DIRECTION; + +typedef enum __attribute__((packed)) { + FACETS_TRANSFORM_VALUE, + FACETS_TRANSFORM_HISTOGRAM, + FACETS_TRANSFORM_FACET, + FACETS_TRANSFORM_DATA, + FACETS_TRANSFORM_FACET_SORT, +} FACETS_TRANSFORMATION_SCOPE; + +typedef enum __attribute__((packed)) { + FACET_KEY_OPTION_FACET = (1 << 0), // filterable values + FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value + FACET_KEY_OPTION_NEVER_FACET = (1 << 2), // never enable this field as facet + FACET_KEY_OPTION_STICKY = (1 << 3), // should be sticky in the table + FACET_KEY_OPTION_VISIBLE = (1 << 4), // should be in the default table + FACET_KEY_OPTION_FTS = (1 << 5), // the key is filterable by full text search (FTS) + FACET_KEY_OPTION_MAIN_TEXT = (1 << 6), // full width and wrap + FACET_KEY_OPTION_RICH_TEXT = (1 << 7), + FACET_KEY_OPTION_REORDER = (1 << 8), // give the key a new order id on first encounter + FACET_KEY_OPTION_TRANSFORM_VIEW = (1 << 9), // when registering the transformation, do it only at the view, not on all data } FACET_KEY_OPTIONS; +typedef enum __attribute__((packed)) { + FACET_ROW_SEVERITY_DEBUG, // lowest - not important + FACET_ROW_SEVERITY_NORMAL, // the default + FACET_ROW_SEVERITY_NOTICE, // bold + FACET_ROW_SEVERITY_WARNING, // yellow + bold + FACET_ROW_SEVERITY_CRITICAL, // red + bold +} FACET_ROW_SEVERITY; + typedef struct facet_row_key_value { const char *tmp; + uint32_t tmp_len; + bool empty; BUFFER *wb; } FACET_ROW_KEY_VALUE; typedef struct facet_row { usec_t usec; DICTIONARY *dict; + FACET_ROW_SEVERITY severity; struct facet_row *prev, *next; } FACET_ROW; typedef struct facets FACETS; typedef struct facet_key FACET_KEY; -#define FACET_STRING_HASH_SIZE 19 -void facets_string_hash(const char *src, char *out); - -typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, void *data); +typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data); typedef void (*facet_dynamic_row_t)(FACETS *facets, BUFFER *json_array, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data); -FACET_KEY *facets_register_dynamic_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data); -FACET_KEY *facets_register_key_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data); +typedef FACET_ROW_SEVERITY (*facet_row_severity_t)(FACETS *facets, FACET_ROW *row, void *data); +FACET_KEY *facets_register_dynamic_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data); +FACET_KEY *facets_register_key_name_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data); +void facets_register_row_severity(FACETS *facets, facet_row_severity_t cb, void *data); typedef enum __attribute__((packed)) { - FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets, should be visible by default in the table - FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search + FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets should be visible by default in the table + FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search + FACETS_OPTION_DONT_SEND_FACETS = (1 << 2), // "facets" object will not be included in the report + FACETS_OPTION_DONT_SEND_HISTOGRAM = (1 << 3), // "histogram" object will not be included in the report + FACETS_OPTION_DATA_ONLY = (1 << 4), + FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS = (1 << 5), // empty facet values will not be included in the report + FACETS_OPTION_SORT_FACETS_ALPHABETICALLY = (1 << 6), + FACETS_OPTION_SHOW_DELTAS = (1 << 7), } FACETS_OPTIONS; -FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys); +FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys); void facets_destroy(FACETS *facets); void facets_accepted_param(FACETS *facets, const char *param); void facets_rows_begin(FACETS *facets); -void facets_row_finished(FACETS *facets, usec_t usec); +bool facets_row_finished(FACETS *facets, usec_t usec); -FACET_KEY *facets_register_key(FACETS *facets, const char *param, FACET_KEY_OPTIONS options); +FACET_KEY *facets_register_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options); void facets_set_query(FACETS *facets, const char *query); void facets_set_items(FACETS *facets, uint32_t items); -void facets_set_anchor(FACETS *facets, usec_t anchor); -void facets_register_facet_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options); +void facets_set_anchor(FACETS *facets, usec_t start_ut, usec_t stop_ut, FACETS_ANCHOR_DIRECTION direction); +void facets_enable_slice_mode(FACETS *facets); + +FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options); +void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *value_id, FACET_KEY_OPTIONS options); +void facets_set_timeframe_and_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut); +void facets_set_timeframe_and_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut); void facets_add_key_value(FACETS *facets, const char *key, const char *value); -void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len); +void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len); + +void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry); +void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool with_keys); +void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity); +void facets_set_additional_options(FACETS *facets, FACETS_OPTIONS options); + +bool facets_key_name_is_filter(FACETS *facets, const char *key); +bool facets_key_name_is_facet(FACETS *facets, const char *key); +bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length); +void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length); + +void facets_sort_and_reorder_keys(FACETS *facets); +usec_t facets_row_oldest_ut(FACETS *facets); +usec_t facets_row_newest_ut(FACETS *facets); +uint32_t facets_rows(FACETS *facets); -void facets_report(FACETS *facets, BUFFER *wb); +void facets_table_config(BUFFER *wb); #endif diff --git a/libnetdata/functions_evloop/Makefile.am b/libnetdata/functions_evloop/Makefile.am new file mode 100644 index 000000000..161784b8f --- /dev/null +++ b/libnetdata/functions_evloop/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/functions_evloop/README.md b/libnetdata/functions_evloop/README.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/libnetdata/functions_evloop/README.md diff --git a/libnetdata/functions_evloop/functions_evloop.c b/libnetdata/functions_evloop/functions_evloop.c new file mode 100644 index 000000000..3fcd70aa1 --- /dev/null +++ b/libnetdata/functions_evloop/functions_evloop.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "functions_evloop.h" + +#define MAX_FUNCTION_PARAMETERS 1024 + +struct functions_evloop_worker_job { + bool used; + bool running; + bool cancelled; + char *cmd; + const char *transaction; + time_t timeout; + functions_evloop_worker_execute_t cb; +}; + +struct rrd_functions_expectation { + const char *function; + size_t function_length; + functions_evloop_worker_execute_t cb; + time_t default_timeout; + struct rrd_functions_expectation *prev, *next; +}; + +struct functions_evloop_globals { + const char *tag; + + DICTIONARY *worker_queue; + pthread_mutex_t worker_mutex; + pthread_cond_t worker_cond_var; + size_t workers; + + netdata_mutex_t *stdout_mutex; + bool *plugin_should_exit; + + netdata_thread_t reader_thread; + netdata_thread_t *worker_threads; + + struct rrd_functions_expectation *expectations; +}; + +static void *rrd_functions_worker_globals_worker_main(void *arg) { + struct functions_evloop_globals *wg = arg; + + bool last_acquired = true; + while (true) { + pthread_mutex_lock(&wg->worker_mutex); + + if(dictionary_entries(wg->worker_queue) == 0 || !last_acquired) + pthread_cond_wait(&wg->worker_cond_var, &wg->worker_mutex); + + const DICTIONARY_ITEM *acquired = NULL; + struct functions_evloop_worker_job *j; + dfe_start_write(wg->worker_queue, j) { + if(j->running || j->cancelled) + continue; + + acquired = dictionary_acquired_item_dup(wg->worker_queue, j_dfe.item); + j->running = true; + break; + } + dfe_done(j); + + pthread_mutex_unlock(&wg->worker_mutex); + + if(acquired) { + last_acquired = true; + j = dictionary_acquired_item_value(acquired); + j->cb(j->transaction, j->cmd, j->timeout, &j->cancelled); + dictionary_del(wg->worker_queue, j->transaction); + dictionary_acquired_item_release(wg->worker_queue, acquired); + dictionary_garbage_collect(wg->worker_queue); + } + else + last_acquired = false; + } + return NULL; +} + +static void *rrd_functions_worker_globals_reader_main(void *arg) { + struct functions_evloop_globals *wg = arg; + + char buffer[PLUGINSD_LINE_MAX + 1]; + + char *s = NULL; + while(!(*wg->plugin_should_exit) && (s = fgets(buffer, PLUGINSD_LINE_MAX, stdin))) { + + char *words[MAX_FUNCTION_PARAMETERS] = { NULL }; + size_t num_words = quoted_strings_splitter_pluginsd(buffer, words, MAX_FUNCTION_PARAMETERS); + + const char *keyword = get_word(words, num_words, 0); + + if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION) == 0) { + char *transaction = get_word(words, num_words, 1); + char *timeout_s = get_word(words, num_words, 2); + char *function = get_word(words, num_words, 3); + + if(!transaction || !*transaction || !timeout_s || !*timeout_s || !function || !*function) { + netdata_log_error("Received incomplete %s (transaction = '%s', timeout = '%s', function = '%s'). Ignoring it.", + keyword, + transaction?transaction:"(unset)", + timeout_s?timeout_s:"(unset)", + function?function:"(unset)"); + } + else { + int timeout = str2i(timeout_s); + + bool found = false; + struct rrd_functions_expectation *we; + for(we = wg->expectations; we ;we = we->next) { + if(strncmp(function, we->function, we->function_length) == 0) { + struct functions_evloop_worker_job t = { + .cmd = strdupz(function), + .transaction = strdupz(transaction), + .running = false, + .cancelled = false, + .timeout = timeout > 0 ? timeout : we->default_timeout, + .used = false, + .cb = we->cb, + }; + struct functions_evloop_worker_job *j = dictionary_set(wg->worker_queue, transaction, &t, sizeof(t)); + if(j->used) { + netdata_log_error("Received duplicate function transaction '%s'", transaction); + freez((void *)t.cmd); + freez((void *)t.transaction); + } + else { + found = true; + j->used = true; + pthread_cond_signal(&wg->worker_cond_var); + } + } + } + + if(!found) { + netdata_mutex_lock(wg->stdout_mutex); + pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND, + "No function with this name found."); + netdata_mutex_unlock(wg->stdout_mutex); + } + } + } + else if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION_CANCEL) == 0) { + char *transaction = get_word(words, num_words, 1); + const DICTIONARY_ITEM *acquired = dictionary_get_and_acquire_item(wg->worker_queue, transaction); + if(acquired) { + struct functions_evloop_worker_job *j = dictionary_acquired_item_value(acquired); + __atomic_store_n(&j->cancelled, true, __ATOMIC_RELAXED); + dictionary_acquired_item_release(wg->worker_queue, acquired); + dictionary_del(wg->worker_queue, transaction); + dictionary_garbage_collect(wg->worker_queue); + } + else + netdata_log_error("Received CANCEL for transaction '%s', but it not available here", transaction); + } + else + netdata_log_error("Received unknown command: %s", keyword?keyword:"(unset)"); + } + + if(!s || feof(stdin) || ferror(stdin)) { + *wg->plugin_should_exit = true; + netdata_log_error("Received error on stdin."); + } + + exit(1); +} + +void worker_queue_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + struct functions_evloop_worker_job *j = value; + freez((void *)j->cmd); + freez((void *)j->transaction); +} + +struct functions_evloop_globals *functions_evloop_init(size_t worker_threads, const char *tag, netdata_mutex_t *stdout_mutex, bool *plugin_should_exit) { + struct functions_evloop_globals *wg = callocz(1, sizeof(struct functions_evloop_globals)); + + wg->worker_queue = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + dictionary_register_delete_callback(wg->worker_queue, worker_queue_delete_cb, NULL); + + pthread_mutex_init(&wg->worker_mutex, NULL); + pthread_cond_init(&wg->worker_cond_var, NULL); + + wg->plugin_should_exit = plugin_should_exit; + wg->stdout_mutex = stdout_mutex; + wg->workers = worker_threads; + wg->worker_threads = callocz(wg->workers, sizeof(netdata_thread_t )); + wg->tag = tag; + + char tag_buffer[NETDATA_THREAD_TAG_MAX + 1]; + snprintfz(tag_buffer, NETDATA_THREAD_TAG_MAX, "%s_READER", wg->tag); + netdata_thread_create(&wg->reader_thread, tag_buffer, NETDATA_THREAD_OPTION_DONT_LOG, + rrd_functions_worker_globals_reader_main, wg); + + for(size_t i = 0; i < wg->workers ; i++) { + snprintfz(tag_buffer, NETDATA_THREAD_TAG_MAX, "%s_WORK[%zu]", wg->tag, i+1); + netdata_thread_create(&wg->worker_threads[i], tag_buffer, NETDATA_THREAD_OPTION_DONT_LOG, + rrd_functions_worker_globals_worker_main, wg); + } + + return wg; +} + +void functions_evloop_add_function(struct functions_evloop_globals *wg, const char *function, functions_evloop_worker_execute_t cb, time_t default_timeout) { + struct rrd_functions_expectation *we = callocz(1, sizeof(*we)); + we->function = function; + we->function_length = strlen(we->function); + we->cb = cb; + we->default_timeout = default_timeout; + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(wg->expectations, we, prev, next); +} diff --git a/libnetdata/functions_evloop/functions_evloop.h b/libnetdata/functions_evloop/functions_evloop.h new file mode 100644 index 000000000..ee0f72cb5 --- /dev/null +++ b/libnetdata/functions_evloop/functions_evloop.h @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_FUNCTIONS_EVLOOP_H +#define NETDATA_FUNCTIONS_EVLOOP_H + +#include "../libnetdata.h" + +#define PLUGINSD_KEYWORD_CHART "CHART" +#define PLUGINSD_KEYWORD_CHART_DEFINITION_END "CHART_DEFINITION_END" +#define PLUGINSD_KEYWORD_DIMENSION "DIMENSION" +#define PLUGINSD_KEYWORD_BEGIN "BEGIN" +#define PLUGINSD_KEYWORD_SET "SET" +#define PLUGINSD_KEYWORD_END "END" +#define PLUGINSD_KEYWORD_FLUSH "FLUSH" +#define PLUGINSD_KEYWORD_DISABLE "DISABLE" +#define PLUGINSD_KEYWORD_VARIABLE "VARIABLE" +#define PLUGINSD_KEYWORD_LABEL "LABEL" +#define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE" +#define PLUGINSD_KEYWORD_CLABEL "CLABEL" +#define PLUGINSD_KEYWORD_CLABEL_COMMIT "CLABEL_COMMIT" +#define PLUGINSD_KEYWORD_FUNCTION "FUNCTION" +#define PLUGINSD_KEYWORD_FUNCTION_CANCEL "FUNCTION_CANCEL" +#define PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN "FUNCTION_RESULT_BEGIN" +#define PLUGINSD_KEYWORD_FUNCTION_RESULT_END "FUNCTION_RESULT_END" + +#define PLUGINSD_KEYWORD_REPLAY_CHART "REPLAY_CHART" +#define PLUGINSD_KEYWORD_REPLAY_BEGIN "RBEGIN" +#define PLUGINSD_KEYWORD_REPLAY_SET "RSET" +#define PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE "RDSTATE" +#define PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE "RSSTATE" +#define PLUGINSD_KEYWORD_REPLAY_END "REND" + +#define PLUGINSD_KEYWORD_BEGIN_V2 "BEGIN2" +#define PLUGINSD_KEYWORD_SET_V2 "SET2" +#define PLUGINSD_KEYWORD_END_V2 "END2" + +#define PLUGINSD_KEYWORD_HOST_DEFINE "HOST_DEFINE" +#define PLUGINSD_KEYWORD_HOST_DEFINE_END "HOST_DEFINE_END" +#define PLUGINSD_KEYWORD_HOST_LABEL "HOST_LABEL" +#define PLUGINSD_KEYWORD_HOST "HOST" + +#define PLUGINSD_KEYWORD_DYNCFG_ENABLE "DYNCFG_ENABLE" +#define PLUGINSD_KEYWORD_DYNCFG_REGISTER_MODULE "DYNCFG_REGISTER_MODULE" + +#define PLUGINSD_KEYWORD_REPORT_JOB_STATUS "REPORT_JOB_STATUS" + +#define PLUGINSD_KEYWORD_EXIT "EXIT" + +#define PLUGINS_FUNCTIONS_TIMEOUT_DEFAULT 10 // seconds + +typedef void (*functions_evloop_worker_execute_t)(const char *transaction, char *function, int timeout, bool *cancelled); +struct functions_evloop_worker_job; +struct functions_evloop_globals *functions_evloop_init(size_t worker_threads, const char *tag, netdata_mutex_t *stdout_mutex, bool *plugin_should_exit); +void functions_evloop_add_function(struct functions_evloop_globals *wg, const char *function, functions_evloop_worker_execute_t cb, time_t default_timeout); + + +#define pluginsd_function_result_begin_to_buffer(wb, transaction, code, content_type, expires) \ + buffer_sprintf(wb \ + , PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " \"%s\" %d \"%s\" %ld\n" \ + , (transaction) ? (transaction) : "" \ + , (int)(code) \ + , (content_type) ? (content_type) : "" \ + , (long int)(expires) \ + ) + +#define pluginsd_function_result_end_to_buffer(wb) \ + buffer_strcat(wb, "\n" PLUGINSD_KEYWORD_FUNCTION_RESULT_END "\n") + +#define pluginsd_function_result_begin_to_stdout(transaction, code, content_type, expires) \ + fprintf(stdout \ + , PLUGINSD_KEYWORD_FUNCTION_RESULT_BEGIN " \"%s\" %d \"%s\" %ld\n" \ + , (transaction) ? (transaction) : "" \ + , (int)(code) \ + , (content_type) ? (content_type) : "" \ + , (long int)(expires) \ + ) + +#define pluginsd_function_result_end_to_stdout() \ + fprintf(stdout, "\n" PLUGINSD_KEYWORD_FUNCTION_RESULT_END "\n") + +static inline void pluginsd_function_json_error_to_stdout(const char *transaction, int code, const char *msg) { + char buffer[PLUGINSD_LINE_MAX + 1]; + json_escape_string(buffer, msg, PLUGINSD_LINE_MAX); + + pluginsd_function_result_begin_to_stdout(transaction, code, "application/json", now_realtime_sec()); + fprintf(stdout, "{\"status\":%d,\"error_message\":\"%s\"}", code, buffer); + pluginsd_function_result_end_to_stdout(); + fflush(stdout); +} + +static inline void pluginsd_function_result_to_stdout(const char *transaction, int code, const char *content_type, time_t expires, BUFFER *result) { + pluginsd_function_result_begin_to_stdout(transaction, code, content_type, expires); + fwrite(buffer_tostring(result), buffer_strlen(result), 1, stdout); + pluginsd_function_result_end_to_stdout(); + fflush(stdout); +} + +#endif //NETDATA_FUNCTIONS_EVLOOP_H diff --git a/libnetdata/health/health.c b/libnetdata/health/health.c index 53ebecb42..f2dc46e3d 100644 --- a/libnetdata/health/health.c +++ b/libnetdata/health/health.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + #include "health.h" SILENCERS *silencers; @@ -27,8 +29,8 @@ void health_silencers_add(SILENCER *silencer) { // Add the created instance to the linked list in silencers silencer->next = silencers->silencers; silencers->silencers = silencer; - netdata_log_debug(D_HEALTH, "HEALTH command API: Added silencer %s:%s:%s:%s:%s", silencer->alarms, - silencer->charts, silencer->contexts, silencer->hosts, silencer->families + netdata_log_debug(D_HEALTH, "HEALTH command API: Added silencer %s:%s:%s:%s", silencer->alarms, + silencer->charts, silencer->contexts, silencer->hosts ); } @@ -49,8 +51,7 @@ SILENCER *health_silencers_addparam(SILENCER *silencer, char *key, char *value) hash_template = 0, hash_chart = 0, hash_context = 0, - hash_host = 0, - hash_families = 0; + hash_host = 0; if (unlikely(!hash_alarm)) { hash_alarm = simple_uhash(HEALTH_ALARM_KEY); @@ -58,7 +59,6 @@ SILENCER *health_silencers_addparam(SILENCER *silencer, char *key, char *value) hash_chart = simple_uhash(HEALTH_CHART_KEY); hash_context = simple_uhash(HEALTH_CONTEXT_KEY); hash_host = simple_uhash(HEALTH_HOST_KEY); - hash_families = simple_uhash(HEALTH_FAMILIES_KEY); } uint32_t hash = simple_uhash(key); @@ -68,8 +68,7 @@ SILENCER *health_silencers_addparam(SILENCER *silencer, char *key, char *value) (hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) || (hash == hash_chart && !strcasecmp(key, HEALTH_CHART_KEY)) || (hash == hash_context && !strcasecmp(key, HEALTH_CONTEXT_KEY)) || - (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) || - (hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) + (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) ) { silencer = create_silencer(); if(!silencer) { @@ -91,9 +90,6 @@ SILENCER *health_silencers_addparam(SILENCER *silencer, char *key, char *value) } else if (hash == hash_host && !strcasecmp(key, HEALTH_HOST_KEY)) { silencer->hosts = strdupz(value); silencer->hosts_pattern = simple_pattern_create(silencer->hosts, NULL, SIMPLE_PATTERN_EXACT, true); - } else if (hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) { - silencer->families = strdupz(value); - silencer->families_pattern = simple_pattern_create(silencer->families, NULL, SIMPLE_PATTERN_EXACT, true); } return silencer; @@ -170,4 +166,4 @@ int health_initialize_global_silencers() { silencers->silencers=NULL; return 0; -}
\ No newline at end of file +} diff --git a/libnetdata/health/health.h b/libnetdata/health/health.h index 6b8f9b384..31173fe61 100644 --- a/libnetdata/health/health.h +++ b/libnetdata/health/health.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + #ifndef NETDATA_HEALTH_LIB # define NETDATA_HEALTH_LIB 1 @@ -9,7 +11,6 @@ #define HEALTH_CHART_KEY "chart" #define HEALTH_HOST_KEY "hosts" #define HEALTH_OS_KEY "os" -#define HEALTH_FAMILIES_KEY "families" #define HEALTH_LOOKUP_KEY "lookup" #define HEALTH_CALC_KEY "calc" @@ -26,9 +27,6 @@ typedef struct silencer { char *charts; SIMPLE_PATTERN *charts_pattern; - char *families; - SIMPLE_PATTERN *families_pattern; - struct silencer *next; } SILENCER; diff --git a/libnetdata/http/http_defs.h b/libnetdata/http/http_defs.h index ffd1c344b..71bd442ce 100644 --- a/libnetdata/http/http_defs.h +++ b/libnetdata/http/http_defs.h @@ -8,6 +8,7 @@ // HTTP_CODES 3XX Redirections #define HTTP_RESP_MOVED_PERM 301 +#define HTTP_RESP_NOT_MODIFIED 304 #define HTTP_RESP_REDIR_TEMP 307 #define HTTP_RESP_REDIR_PERM 308 #define HTTP_RESP_HTTPS_UPGRADE 399 @@ -22,12 +23,12 @@ #define HTTP_RESP_CONFLICT 409 #define HTTP_RESP_PRECOND_FAIL 412 #define HTTP_RESP_CONTENT_TOO_LONG 413 +#define HTTP_RESP_CLIENT_CLOSED_REQUEST 499 // nginx's enxtension to the standard // HTTP_CODES 5XX Server Errors #define HTTP_RESP_INTERNAL_SERVER_ERROR 500 #define HTTP_RESP_INTERNAL_SERVER_ERROR_STR "Internal Server Error" -#define HTTP_RESP_BACKEND_FETCH_FAILED 503 // 503 is right -#define HTTP_RESP_SERVICE_UNAVAILABLE 503 // 503 is right +#define HTTP_RESP_SERVICE_UNAVAILABLE 503 #define HTTP_RESP_GATEWAY_TIMEOUT 504 #define HTTP_RESP_BACKEND_RESPONSE_INVALID 591 diff --git a/libnetdata/inlined.h b/libnetdata/inlined.h index eb55f0fe7..9c07d97b6 100644 --- a/libnetdata/inlined.h +++ b/libnetdata/inlined.h @@ -394,10 +394,10 @@ static inline NETDATA_DOUBLE str2ndd_encoded(const char *src, char **endptr) { return str2ndd(src, endptr) * sign; } -static inline char *strncpyz(char *dst, const char *src, size_t n) { +static inline char *strncpyz(char *dst, const char *src, size_t dst_size_minus_1) { char *p = dst; - while (*src && n--) + while (*src && dst_size_minus_1--) *dst++ = *src++; *dst = '\0'; diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c index 26582ffe2..159e89950 100644 --- a/libnetdata/libnetdata.c +++ b/libnetdata/libnetdata.c @@ -1942,6 +1942,7 @@ void timing_action(TIMING_ACTION action, TIMING_STEP step) { } } +#ifdef ENABLE_HTTPS int hash256_string(const unsigned char *string, size_t size, char *hash) { EVP_MD_CTX *ctx; ctx = EVP_MD_CTX_create(); @@ -1966,19 +1967,15 @@ int hash256_string(const unsigned char *string, size_t size, char *hash) { EVP_MD_CTX_destroy(ctx); return 1; } +#endif -// Returns 1 if an absolute period was requested or 0 if it was a relative period -bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running) { - time_t now = now_realtime_sec() - 1; - if(now_ptr) - *now_ptr = now; +bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t now) { + if(!now) now = now_realtime_sec(); int absolute_period_requested = -1; - long long after_requested, before_requested; - - before_requested = *before; - after_requested = *after; + time_t before_requested = *before; + time_t after_requested = *after; // allow relative for before (smaller than API_RELATIVE_TIME_MAX) if(ABS(before_requested) <= API_RELATIVE_TIME_MAX) { @@ -2023,11 +2020,29 @@ bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now // shift the query back to be in the present time // (this may also happen because of the rules above) if(before_requested > now) { - long long delta = before_requested - now; + time_t delta = before_requested - now; before_requested -= delta; after_requested -= delta; } + *before = before_requested; + *after = after_requested; + + return (absolute_period_requested != 1); +} + +// Returns 1 if an absolute period was requested or 0 if it was a relative period +bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running) { + time_t now = now_realtime_sec() - 1; + + if(now_ptr) + *now_ptr = now; + + time_t before_requested = *before; + time_t after_requested = *after; + + int absolute_period_requested = rrdr_relative_window_to_absolute(&after_requested, &before_requested, now); + time_t absolute_minimum_time = now - (10 * 365 * 86400); time_t absolute_maximum_time = now + (1 * 365 * 86400); diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h index b8dedf515..c337b3b5d 100644 --- a/libnetdata/libnetdata.h +++ b/libnetdata/libnetdata.h @@ -798,6 +798,10 @@ void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds); void netdata_cleanup_and_exit(int ret) NORETURN; void send_statistics(const char *action, const char *action_result, const char *action_data); extern char *netdata_configured_host_prefix; + +#define XXH_INLINE_ALL +#include "xxhash.h" + #include "libjudy/src/Judy.h" #include "july/july.h" #include "os.h" @@ -838,6 +842,7 @@ extern char *netdata_configured_host_prefix; #include "gorilla/gorilla.h" #include "facets/facets.h" #include "dyn_conf/dyn_conf.h" +#include "functions_evloop/functions_evloop.h" // BEWARE: this exists in alarm-notify.sh #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud" @@ -984,7 +989,8 @@ int hash256_string(const unsigned char *string, size_t size, char *hash); extern bool unittest_running; #define API_RELATIVE_TIME_MAX (3 * 365 * 86400) -bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running); +bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t now); +bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_t *now_ptr, bool unittest_running); int netdata_base64_decode(const char *encoded, char *decoded, size_t decoded_size); diff --git a/libnetdata/log/README.md b/libnetdata/log/README.md index 3684abd68..f811bb4b3 100644 --- a/libnetdata/log/README.md +++ b/libnetdata/log/README.md @@ -12,4 +12,3 @@ learn_rel_path: "Developers/libnetdata" The netdata log library supports debug, info, error and fatal error logging. By default we have an access log, an error log and a collectors log. - diff --git a/libnetdata/log/log.c b/libnetdata/log/log.c index e43a4f464..02bb776c5 100644 --- a/libnetdata/log/log.c +++ b/libnetdata/log/log.c @@ -12,11 +12,11 @@ int web_server_is_multithreaded = 1; const char *program_name = ""; uint64_t debug_flags = 0; -int access_log_syslog = 1; -int error_log_syslog = 1; -int collector_log_syslog = 1; -int output_log_syslog = 1; // debug log -int health_log_syslog = 1; +int access_log_syslog = 0; +int error_log_syslog = 0; +int collector_log_syslog = 0; +int output_log_syslog = 0; // debug log +int health_log_syslog = 0; int stdaccess_fd = -1; FILE *stdaccess = NULL; @@ -34,6 +34,8 @@ const char *facility_log = NULL; const char *stdhealth_filename = NULL; const char *stdcollector_filename = NULL; +netdata_log_level_t global_log_severity_level = NETDATA_LOG_LEVEL_INFO; + #ifdef ENABLE_ACLK const char *aclklog_filename = NULL; int aclklog_fd = -1; @@ -780,6 +782,11 @@ void debug_int( const char *file, const char *function, const unsigned long line void info_int( int is_collector, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) { +#if !defined(NETDATA_INTERNAL_CHECKS) && !defined(NETDATA_DEV_MODE) + if (NETDATA_LOG_LEVEL_INFO > global_log_severity_level) + return; +#endif + va_list args; FILE *fp = (is_collector || !stderror) ? stderr : stderror; @@ -890,7 +897,7 @@ void error_limit_int(ERROR_LIMIT *erl, const char *prefix, const char *file __ma erl->count, (unsigned long long)(erl->last_logged ? now - erl->last_logged : 0)); if(erl->sleep_ut) - fprintf(fp, " (sleeping for %llu microseconds every time this happens)", erl->sleep_ut); + fprintf(fp, " (sleeping for %"PRIu64" microseconds every time this happens)", erl->sleep_ut); if(__errno) { char buf[1024]; @@ -908,6 +915,11 @@ void error_limit_int(ERROR_LIMIT *erl, const char *prefix, const char *file __ma } void error_int(int is_collector, const char *prefix, const char *file __maybe_unused, const char *function __maybe_unused, const unsigned long line __maybe_unused, const char *fmt, ... ) { +#if !defined(NETDATA_INTERNAL_CHECKS) && !defined(NETDATA_DEV_MODE) + if (NETDATA_LOG_LEVEL_ERROR > global_log_severity_level) + return; +#endif + // save a copy of errno - just in case this function generates a new error int __errno = errno; FILE *fp = (is_collector || !stderror) ? stderr : stderror; @@ -1125,3 +1137,37 @@ void log_aclk_message_bin( const char *data, const size_t data_len, int tx, cons } } #endif + +void log_set_global_severity_level(netdata_log_level_t value) +{ + global_log_severity_level = value; +} + +netdata_log_level_t log_severity_string_to_severity_level(char *level) +{ + if (!strcmp(level, NETDATA_LOG_LEVEL_INFO_STR)) + return NETDATA_LOG_LEVEL_INFO; + if (!strcmp(level, NETDATA_LOG_LEVEL_ERROR_STR) || !strcmp(level, NETDATA_LOG_LEVEL_ERROR_SHORT_STR)) + return NETDATA_LOG_LEVEL_ERROR; + + return NETDATA_LOG_LEVEL_INFO; +} + +char *log_severity_level_to_severity_string(netdata_log_level_t level) +{ + switch (level) { + case NETDATA_LOG_LEVEL_ERROR: + return NETDATA_LOG_LEVEL_ERROR_STR; + case NETDATA_LOG_LEVEL_INFO: + default: + return NETDATA_LOG_LEVEL_INFO_STR; + } +} + +void log_set_global_severity_for_external_plugins() { + char *s = getenv("NETDATA_LOG_SEVERITY_LEVEL"); + if (!s) + return; + netdata_log_level_t level = log_severity_string_to_severity_level(s); + log_set_global_severity_level(level); +} diff --git a/libnetdata/log/log.h b/libnetdata/log/log.h index 9ced07a9a..cf0865cf9 100644 --- a/libnetdata/log/log.h +++ b/libnetdata/log/log.h @@ -43,9 +43,6 @@ extern "C" { #define D_ANALYTICS 0x0000000080000000 #define D_RRDENGINE 0x0000000100000000 #define D_ACLK 0x0000000200000000 -#define D_METADATALOG 0x0000000400000000 -#define D_ACLK_SYNC 0x0000000800000000 -#define D_META_SYNC 0x0000001000000000 #define D_REPLICATION 0x0000002000000000 #define D_SYSTEM 0x8000000000000000 @@ -105,6 +102,23 @@ typedef struct error_with_limit { usec_t sleep_ut; } ERROR_LIMIT; +typedef enum netdata_log_level { + NETDATA_LOG_LEVEL_ERROR, + NETDATA_LOG_LEVEL_INFO, + + NETDATA_LOG_LEVEL_END +} netdata_log_level_t; + +#define NETDATA_LOG_LEVEL_INFO_STR "info" +#define NETDATA_LOG_LEVEL_ERROR_STR "error" +#define NETDATA_LOG_LEVEL_ERROR_SHORT_STR "err" + +extern netdata_log_level_t global_log_severity_level; +netdata_log_level_t log_severity_string_to_severity_level(char *level); +char *log_severity_level_to_severity_string(netdata_log_level_t level); +void log_set_global_severity_level(netdata_log_level_t value); +void log_set_global_severity_for_external_plugins(); + #define error_limit_static_global_var(var, log_every_secs, sleep_usecs) static ERROR_LIMIT var = { .last_logged = 0, .count = 0, .log_every = (log_every_secs), .sleep_ut = (sleep_usecs) } #define error_limit_static_thread_var(var, log_every_secs, sleep_usecs) static __thread ERROR_LIMIT var = { .last_logged = 0, .count = 0, .log_every = (log_every_secs), .sleep_ut = (sleep_usecs) } diff --git a/libnetdata/required_dummies.h b/libnetdata/required_dummies.h index 5a0d4e050..1ffe1e9e5 100644 --- a/libnetdata/required_dummies.h +++ b/libnetdata/required_dummies.h @@ -37,6 +37,7 @@ void rrdset_thread_rda_free(void){}; void sender_thread_buffer_free(void){}; void query_target_free(void){}; void service_exits(void){}; +void rrd_collector_finished(void){}; // required by get_system_cpus() char *netdata_configured_host_prefix = ""; diff --git a/libnetdata/socket/socket.c b/libnetdata/socket/socket.c index e7d0b4807..67dc4c71c 100644 --- a/libnetdata/socket/socket.c +++ b/libnetdata/socket/socket.c @@ -10,6 +10,40 @@ #include "../libnetdata.h" +bool ip_to_hostname(const char *ip, char *dst, size_t dst_len) { + if(!dst || !dst_len) + return false; + + struct sockaddr_in sa; + struct sockaddr_in6 sa6; + struct sockaddr *sa_ptr; + int sa_len; + + // Try to convert the IP address to sockaddr_in (IPv4) + if (inet_pton(AF_INET, ip, &(sa.sin_addr)) == 1) { + sa.sin_family = AF_INET; + sa_ptr = (struct sockaddr *)&sa; + sa_len = sizeof(sa); + } + // Try to convert the IP address to sockaddr_in6 (IPv6) + else if (inet_pton(AF_INET6, ip, &(sa6.sin6_addr)) == 1) { + sa6.sin6_family = AF_INET6; + sa_ptr = (struct sockaddr *)&sa6; + sa_len = sizeof(sa6); + } + + else { + dst[0] = '\0'; + return false; + } + + // Perform the reverse lookup + int res = getnameinfo(sa_ptr, sa_len, dst, dst_len, NULL, 0, NI_NAMEREQD); + if(res != 0) + return false; + + return true; +} SOCKET_PEERS socket_peers(int sock_fd) { SOCKET_PEERS peers; @@ -810,7 +844,7 @@ int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t errno = 0; if(connect(fd, ai->ai_addr, ai->ai_addrlen) < 0) { if(errno == EALREADY || errno == EINPROGRESS) { - netdata_log_info("Waiting for connection to ip %s port %s to be established", hostBfr, servBfr); + internal_error(true, "Waiting for connection to ip %s port %s to be established", hostBfr, servBfr); // Convert 'struct timeval' to milliseconds for poll(): int timeout_milliseconds = timeout->tv_sec * 1000 + timeout->tv_usec / 1000; @@ -835,6 +869,7 @@ int connect_to_this_ip46(int protocol, int socktype, const char *host, uint32_t } else if (ret == 0) { // poll() timed out, the connection is not established within the specified timeout. + errno = 0; netdata_log_error("Timed out while connecting to '%s', port '%s'.", hostBfr, servBfr); close(fd); fd = -1; diff --git a/libnetdata/socket/socket.h b/libnetdata/socket/socket.h index c4bd47360..e4ca08d47 100644 --- a/libnetdata/socket/socket.h +++ b/libnetdata/socket/socket.h @@ -243,5 +243,6 @@ typedef struct socket_peers { } SOCKET_PEERS; SOCKET_PEERS socket_peers(int sock_fd); +bool ip_to_hostname(const char *ip, char *dst, size_t dst_len); #endif //NETDATA_SOCKET_H diff --git a/libnetdata/string/string.c b/libnetdata/string/string.c index 373d0c24c..54b8f171f 100644 --- a/libnetdata/string/string.c +++ b/libnetdata/string/string.c @@ -28,19 +28,22 @@ static struct string_partition { Pvoid_t JudyHSArray; // the Judy array - hashtable - size_t searches; // the number of successful searches in the index - size_t duplications; // when a string is referenced - size_t releases; // when a string is unreferenced - size_t inserts; // the number of successful inserts to the index size_t deletes; // the number of successful deleted from the index long int entries; // the number of entries in the index - long int active_references; // the number of active references alive long int memory; // the memory used, without the JudyHS index #ifdef NETDATA_INTERNAL_CHECKS // internal statistics + + struct { + size_t searches; // the number of successful searches in the index + size_t releases; // when a string is unreferenced + size_t duplications; // when a string is referenced + long int active_references; // the number of active references alive + } atomic; + size_t found_deleted_on_search; size_t found_available_on_search; size_t found_deleted_on_insert; @@ -51,14 +54,15 @@ static struct string_partition { } string_base[STRING_PARTITIONS] = { 0 }; #ifdef NETDATA_INTERNAL_CHECKS +#define string_stats_atomic_increment(partition, var) __atomic_add_fetch(&string_base[partition].atomic.var, 1, __ATOMIC_RELAXED) +#define string_stats_atomic_decrement(partition, var) __atomic_sub_fetch(&string_base[partition].atomic.var, 1, __ATOMIC_RELAXED) #define string_internal_stats_add(partition, var, val) __atomic_add_fetch(&string_base[partition].var, val, __ATOMIC_RELAXED) #else +#define string_stats_atomic_increment(partition, var) do {;} while(0) +#define string_stats_atomic_decrement(partition, var) do {;} while(0) #define string_internal_stats_add(partition, var, val) do {;} while(0) #endif -#define string_stats_atomic_increment(partition, var) __atomic_add_fetch(&string_base[partition].var, 1, __ATOMIC_RELAXED) -#define string_stats_atomic_decrement(partition, var) __atomic_sub_fetch(&string_base[partition].var, 1, __ATOMIC_RELAXED) - void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_t *entries, size_t *references, size_t *memory, size_t *duplications, size_t *releases) { if (inserts) *inserts = 0; if (deletes) *deletes = 0; @@ -72,12 +76,15 @@ void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_ for(size_t i = 0; i < STRING_PARTITIONS ;i++) { if (inserts) *inserts += string_base[i].inserts; if (deletes) *deletes += string_base[i].deletes; - if (searches) *searches += string_base[i].searches; if (entries) *entries += (size_t) string_base[i].entries; - if (references) *references += (size_t) string_base[i].active_references; if (memory) *memory += (size_t) string_base[i].memory; - if (duplications) *duplications += string_base[i].duplications; - if (releases) *releases += string_base[i].releases; + +#ifdef NETDATA_INTERNAL_CHECKS + if (searches) *searches += string_base[i].atomic.searches; + if (references) *references += (size_t) string_base[i].atomic.active_references; + if (duplications) *duplications += string_base[i].atomic.duplications; + if (releases) *releases += string_base[i].atomic.releases; +#endif } } @@ -85,7 +92,9 @@ void string_statistics(size_t *inserts, size_t *deletes, size_t *searches, size_ #define string_entry_release(se) __atomic_sub_fetch(&((se)->refcount), 1, __ATOMIC_SEQ_CST); static inline bool string_entry_check_and_acquire(STRING *se) { +#ifdef NETDATA_INTERNAL_CHECKS uint8_t partition = string_partition(se); +#endif REFCOUNT expected, desired, count = 0; @@ -125,7 +134,9 @@ STRING *string_dup(STRING *string) { string_entry_acquire(string); +#ifdef NETDATA_INTERNAL_CHECKS uint8_t partition = string_partition(string); +#endif // statistics string_stats_atomic_increment(partition, active_references); @@ -275,7 +286,9 @@ static inline void string_index_delete(STRING *string) { STRING *string_strdupz(const char *str) { if(unlikely(!str || !*str)) return NULL; +#ifdef NETDATA_INTERNAL_CHECKS uint8_t partition = string_partition_str(str); +#endif size_t length = strlen(str) + 1; STRING *string = string_index_search(str, length); @@ -297,7 +310,9 @@ STRING *string_strdupz(const char *str) { void string_freez(STRING *string) { if(unlikely(!string)) return; +#ifdef NETDATA_INTERNAL_CHECKS uint8_t partition = string_partition(string); +#endif REFCOUNT refcount = string_entry_release(string); #ifdef NETDATA_INTERNAL_CHECKS @@ -518,42 +533,42 @@ int string_unittest(size_t entries) { strings[i] = string_strdupz(names[i]); } end_ut = now_realtime_usec(); - fprintf(stderr, "Created %zu strings in %llu usecs\n", entries, end_ut - start_ut); + fprintf(stderr, "Created %zu strings in %"PRIu64" usecs\n", entries, end_ut - start_ut); start_ut = now_realtime_usec(); for(size_t i = 0; i < entries ;i++) { strings[i] = string_dup(strings[i]); } end_ut = now_realtime_usec(); - fprintf(stderr, "Cloned %zu strings in %llu usecs\n", entries, end_ut - start_ut); + fprintf(stderr, "Cloned %zu strings in %"PRIu64" usecs\n", entries, end_ut - start_ut); start_ut = now_realtime_usec(); for(size_t i = 0; i < entries ;i++) { strings[i] = string_strdupz(string2str(strings[i])); } end_ut = now_realtime_usec(); - fprintf(stderr, "Found %zu existing strings in %llu usecs\n", entries, end_ut - start_ut); + fprintf(stderr, "Found %zu existing strings in %"PRIu64" usecs\n", entries, end_ut - start_ut); start_ut = now_realtime_usec(); for(size_t i = 0; i < entries ;i++) { string_freez(strings[i]); } end_ut = now_realtime_usec(); - fprintf(stderr, "Released %zu referenced strings in %llu usecs\n", entries, end_ut - start_ut); + fprintf(stderr, "Released %zu referenced strings in %"PRIu64" usecs\n", entries, end_ut - start_ut); start_ut = now_realtime_usec(); for(size_t i = 0; i < entries ;i++) { string_freez(strings[i]); } end_ut = now_realtime_usec(); - fprintf(stderr, "Released (again) %zu referenced strings in %llu usecs\n", entries, end_ut - start_ut); + fprintf(stderr, "Released (again) %zu referenced strings in %"PRIu64" usecs\n", entries, end_ut - start_ut); start_ut = now_realtime_usec(); for(size_t i = 0; i < entries ;i++) { string_freez(strings[i]); } end_ut = now_realtime_usec(); - fprintf(stderr, "Freed %zu strings in %llu usecs\n", entries, end_ut - start_ut); + fprintf(stderr, "Freed %zu strings in %"PRIu64" usecs\n", entries, end_ut - start_ut); freez(strings); diff --git a/libnetdata/threads/threads.c b/libnetdata/threads/threads.c index ae3c7106d..cc5690600 100644 --- a/libnetdata/threads/threads.c +++ b/libnetdata/threads/threads.c @@ -9,8 +9,8 @@ static pthread_attr_t *netdata_threads_attr = NULL; typedef struct { void *arg; - pthread_t *thread; char tag[NETDATA_THREAD_NAME_MAX + 1]; + SPINLOCK detach_lock; void *(*start_routine) (void *); NETDATA_THREAD_OPTIONS options; } NETDATA_THREAD; @@ -178,16 +178,19 @@ void rrdset_thread_rda_free(void); void sender_thread_buffer_free(void); void query_target_free(void); void service_exits(void); +void rrd_collector_finished(void); static void thread_cleanup(void *ptr) { if(netdata_thread != ptr) { NETDATA_THREAD *info = (NETDATA_THREAD *)ptr; netdata_log_error("THREADS: internal error - thread local variable does not match the one passed to this function. Expected thread '%s', passed thread '%s'", netdata_thread->tag, info->tag); } + spinlock_lock(&netdata_thread->detach_lock); if(!(netdata_thread->options & NETDATA_THREAD_OPTION_DONT_LOG_CLEANUP)) netdata_log_info("thread with task id %d finished", gettid()); + rrd_collector_finished(); sender_thread_buffer_free(); rrdset_thread_rda_free(); query_target_free(); @@ -197,6 +200,7 @@ static void thread_cleanup(void *ptr) { netdata_thread->tag[0] = '\0'; + spinlock_unlock(&netdata_thread->detach_lock); freez(netdata_thread); netdata_thread = NULL; } @@ -232,12 +236,12 @@ void uv_thread_set_name_np(uv_thread_t ut, const char* name) { strncpyz(threadname, name, NETDATA_THREAD_NAME_MAX); #if defined(__FreeBSD__) - pthread_set_name_np(ut, threadname); + pthread_set_name_np(ut ? ut : pthread_self(), threadname); #elif defined(__APPLE__) // Apple can only set its own name UNUSED(ut); #else - ret = pthread_setname_np(ut, threadname); + ret = pthread_setname_np(ut ? ut : pthread_self(), threadname); #endif thread_name_get(true); @@ -279,13 +283,15 @@ static void *netdata_thread_init(void *ptr) { } int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THREAD_OPTIONS options, void *(*start_routine) (void *), void *arg) { - NETDATA_THREAD *info = mallocz(sizeof(NETDATA_THREAD)); + NETDATA_THREAD *info = callocz(1, sizeof(NETDATA_THREAD)); info->arg = arg; - info->thread = thread; info->start_routine = start_routine; info->options = options; strncpyz(info->tag, tag, NETDATA_THREAD_NAME_MAX); + spinlock_init(&info->detach_lock); + spinlock_lock(&info->detach_lock); + int ret = pthread_create(thread, netdata_threads_attr, netdata_thread_init, info); if(ret != 0) netdata_log_error("failed to create new thread for %s. pthread_create() failed with code %d", tag, ret); @@ -298,6 +304,7 @@ int netdata_thread_create(netdata_thread_t *thread, const char *tag, NETDATA_THR } } + spinlock_unlock(&info->detach_lock); return ret; } diff --git a/libnetdata/xxhash.h b/libnetdata/xxhash.h new file mode 100644 index 000000000..5e2c0ed24 --- /dev/null +++ b/libnetdata/xxhash.h @@ -0,0 +1,6773 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2023 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +/*! + * @mainpage xxHash + * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include <string.h> + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include <stdio.h> + * #include <assert.h> + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * @file xxhash.h + * xxHash prototypes and implementation + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ + +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ + +/*! + * @brief Exposes the implementation and marks all functions as `inline`. + * + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +/*! @brief Marks a global symbol. */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Compiler specifics +***************************************/ + +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((const)) +# define XXH_PUREF __attribute__((pure)) +# define XXH_MALLOCF __attribute__((malloc)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 2 +/*! @brief Version number, encoded as two digits each */ +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return @ref XXH_VERSION_NUMBER of the invoked library. + */ +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include <stddef.h> /* size_t */ +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + typedef uint32_t XXH32_hash_t; + +#else +# include <limits.h> +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +#endif + +/*! + * @} + * + * @defgroup XXH32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * See @ref single_shot_example "Single Shot Example" for an example. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +#ifndef XXH_NO_STREAM +/*! + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * @see streaming_example at the top of @ref xxhash.h for an example. + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/* + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + */ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +/*! @cond Doxygen ignores this part */ +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ +#define XXH_C23_VN 201711L +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((noescape)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ + + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include <stdint.h> */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + typedef uint64_t XXH64_hash_t; +#else +# include <limits.h> +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup XXH64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * This function usually runs faster on 64-bit systems, but slower on 32-bit + * systems (see benchmark). + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! + * @brief Allocates an @ref XXH64_state_t. + * + * Must be freed with XXH64_freeState(). + * @return An allocated XXH64_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); + +/*! + * @brief Frees an @ref XXH64_state_t. + * + * Must be allocated with XXH64_createState(). + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH64_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash64 value from that state. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; + +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); + +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 + +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generate exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/*! + * @brief 64-bit unseeded variant of XXH3. + * + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of 0, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH32(), XXH64(), XXH3_128bits(): equivalent for the other xxHash algorithms + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see + * XXH3_64bits_reset(), XXH3_64bits_update(), XXH3_64bits_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief 64-bit seeded variant of XXH3 + * + * This variant generates a custom secret on the fly based on default secret + * altered using the `seed` value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * @param input The data to hash + * @param length The length + * @param seed The 64-bit seed to alter the state. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/*! + * @brief 64-bit variant of XXH3 with a custom "secret". + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing "XXH3_generateSecret()" instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * This function resets `statePtr` and generate a secret with default parameters. Call it before @ref XXH3_64bits_update(). + * Digest will be equivalent to `XXH3_64bits()`. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * This function resets `statePtr` and generate a secret from `seed`. Call it before @ref XXH3_64bits_update(). + * Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the state. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); + +/*! + * XXH3_64bits_reset_withSecret(): + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +/*! + * @brief Unseeded 128-bit variant of XXH3 + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of 0, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH32(), XXH64(), XXH3_64bits(): equivalent for the other xxHash algorithms + * @see + * XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see + * XXH3_128bits_reset(), XXH3_128bits_update(), XXH3_128bits_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Seeded 128-bit variant of XXH3. @see XXH3_64bits_withSeed(). */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! @brief Custom secret 128-bit variant of XXH3. @see XXH3_64bits_withSecret(). */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * This function resets `statePtr` and generate a secret with default parameters. Call it before @ref XXH3_128bits_update(). + * Digest will be equivalent to `XXH3_128bits()`. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * This function resets `statePtr` and generate a secret from `seed`. Call it before @ref XXH3_128bits_update(). + * Digest will be equivalent to `XXH3_128bits_withSeed()`. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the state. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! @brief Custom secret 128-bit variant of XXH3. @see XXH_64bits_reset_withSecret(). */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * XXH128_isEqual(): + * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * @brief Compares two @ref XXH128_hash_t + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * @return: >0 if *h128_1 > *h128_2 + * =0 if *h128_1 == *h128_2 + * <0 if *h128_1 < *h128_2 + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; + + +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); + +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include <stdalign.h> +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) + + +/*! + * simple alias to pre-selected XXH3_128bits variant + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/*! + * XXH3_generateSecret(): + * + * Derive a high-entropy secret from any user-defined content, named customSeed. + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * + * Example code: + * @code{.c} + * #include <stdio.h> + * #include <stdlib.h> + * #include <string.h> + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); + +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include <string> + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes + * @param seed The seed to seed the state. + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); + +/*! + * These variants generate hash values using either + * @p seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes) + * or @p secret for "large" keys (>= XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); +/*! @copydoc XXH3_64bits_withSecretandSeed() */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#ifndef XXH_NO_STREAM +/*! @copydoc XXH3_64bits_withSecretandSeed() */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +/*! @copydoc XXH3_64bits_withSecretandSeed() */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#endif /* !XXH_NO_STREAM */ + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. + */ +# define XXH_SIZE_OPT 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ + +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } + +#else + +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include <stdlib.h> + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#endif /* XXH_NO_STDLIB */ + +#include <string.h> + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include <limits.h> /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include <assert.h> /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) +# endif +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + + + +/* + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include <stddef.h> + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include <utility> + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `<version>` header. + * We don't use that as including `<utility>` in `extern "C"` blocks + * doesn't work on GCC12 + */ + +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() + +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) + +#else +# define XXH_UNREACHABLE() +#endif + +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) +#else +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup XXH32_impl XXH32 implementation + * @ingroup impl + * + * Details on the XXH32 implementation. + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param hash The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + * @see XXH64_finalize(). + */ +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! + * @ingroup XXH32_family + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * + * The canonical representation uses big endian convention, the same convention + * as human-readable numbers (large digits first). + * + * This way, hash values can be written into a file or buffer, remaining + * comparable across different systems. + * + * The following functions allow transformation of hash values to and from their + * canonical format. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) +{ + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +static XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(hash); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (input<limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + XXH_PRIME64_5; + } + + h64 += (xxh_u64) len; + + return XXH64_finalize(h64, input, len, align); +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif +} + +/******* Hash Streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) +/* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +# define XXH_HAS_INCLUDE(x) __has_include(x) +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include <arm_sve.h> +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include <arm_neon.h> +# undef inline +# elif defined(__AVX2__) +# include <immintrin.h> +# elif defined(__SSE2__) +# include <emmintrin.h> +# endif +#endif + +#if defined(_MSC_VER) +# include <intrin.h> +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * internal macro XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +# define XXH_SVE 6 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((may_alias)) +#else +# define XXH_ALIASING /* nothing */ +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + +#if XXH_VECTOR == XXH_NEON + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; + +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ +{ + return *(xxh_aliasing_uint64x2_t const *)ptr; +} +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) +{ + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); +} +#endif + +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; +} +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); +} +#endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. + * + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel + +# if defined(__s390x__) +# include <s390intrin.h> +# else +# include <altivec.h> +# endif + +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if XXH_SIZE_OPT >= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); + } +} + +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_<name>() and it calls + * XXH3_accumulate_512_<name>(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ +} + + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. + */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; + + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); +#endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); +#endif + + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; +#else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } + } +} +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + xacc[i] = acc_vec; + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } +} + +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } + } +} + +#endif + +/* scalar variants - universal */ + +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; +} +#else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 +#endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__GNUC__) && defined(__aarch64__) + /* + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_accumulate XXH3_accumulate_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_accumulate XXH3_accumulate_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_accumulate XXH3_accumulate_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_accumulate XXH3_accumulate_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#endif + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) +{ + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * Must be freed with XXH3_freeState(). + * @return An allocated XXH3_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * Must be allocated with XXH3_createState(). + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; +} + +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + + } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; + + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ +#ifndef XXH_NO_STREAM +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_64bits_update(state, input, len); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ + +#include <string.h> /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n<nbSeg16; n++) { + XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n); + XXH3_combine16((char*)secretBuffer + n*16, h128); + } + /* last segment */ + XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler)); + } + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed) +{ + XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + XXH3_initCustomSecret(secret, seed); + XXH_ASSERT(secretBuffer != NULL); + memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE); +} + + + +/* Pop our optimization override from above */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC pop_options +#endif + +#endif /* XXH_NO_LONG_LONG */ + +#endif /* XXH_NO_XXH3 */ + +/*! + * @} + */ +#endif /* XXH_IMPLEMENTATION */ + + +#if defined (__cplusplus) +} /* extern "C" */ +#endif |