diff options
Diffstat (limited to '')
| -rw-r--r-- | libnetdata/facets/facets.c | 2787 |
1 files changed, 2787 insertions, 0 deletions
diff --git a/libnetdata/facets/facets.c b/libnetdata/facets/facets.c new file mode 100644 index 00000000..4a5f5442 --- /dev/null +++ b/libnetdata/facets/facets.c @@ -0,0 +1,2787 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +#include "facets.h" + +#define FACETS_HISTOGRAM_COLUMNS 150 // the target number of points in a histogram +#define FACETS_KEYS_WITH_VALUES_MAX 200 // the max number of keys that can be facets +#define FACETS_KEYS_IN_ROW_MAX 500 // the max number of keys in a row + +#define FACETS_KEYS_HASHTABLE_ENTRIES 15 +#define FACETS_VALUES_HASHTABLE_ENTRIES 15 + +static inline void facets_reset_key(FACET_KEY *k); + +// ---------------------------------------------------------------------------- + +static const char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789"; +static const uint8_t id_encoding_characters_reverse[256] = { + ['A'] = 0, ['B'] = 1, ['C'] = 2, ['D'] = 3, + ['E'] = 4, ['F'] = 5, ['G'] = 6, ['H'] = 7, + ['I'] = 8, ['J'] = 9, ['K'] = 10, ['L'] = 11, + ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15, + ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, + ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23, + ['Y'] = 24, ['Z'] = 25, ['.'] = 26, ['a'] = 27, + ['b'] = 28, ['c'] = 29, ['d'] = 30, ['e'] = 31, + ['f'] = 32, ['g'] = 33, ['h'] = 34, ['i'] = 35, + ['j'] = 36, ['k'] = 37, ['l'] = 38, ['m'] = 39, + ['n'] = 40, ['o'] = 41, ['p'] = 42, ['q'] = 43, + ['r'] = 44, ['s'] = 45, ['t'] = 46, ['u'] = 47, + ['v'] = 48, ['w'] = 49, ['x'] = 50, ['y'] = 51, + ['z'] = 52, ['_'] = 53, ['0'] = 54, ['1'] = 55, + ['2'] = 56, ['3'] = 57, ['4'] = 58, ['5'] = 59, + ['6'] = 60, ['7'] = 61, ['8'] = 62, ['9'] = 63 +}; + +#define FACET_STRING_HASH_SIZE 12 +#define FACETS_HASH XXH64_hash_t +#define FACETS_HASH_FUNCTION(src, len) XXH3_64bits(src, len) +#define FACETS_HASH_ZERO (FACETS_HASH)0 +#define FACETS_HASH_UNSAMPLED (FACETS_HASH)(UINT64_MAX - 1) +#define FACETS_HASH_ESTIMATED (FACETS_HASH)UINT64_MAX + +static inline void facets_hash_to_str(FACETS_HASH num, char *out) { + out[11] = '\0'; + out[10] = id_encoding_characters[num & 63]; num >>= 6; + out[9] = id_encoding_characters[num & 63]; num >>= 6; + out[8] = id_encoding_characters[num & 63]; num >>= 6; + out[7] = id_encoding_characters[num & 63]; num >>= 6; + out[6] = id_encoding_characters[num & 63]; num >>= 6; + out[5] = id_encoding_characters[num & 63]; num >>= 6; + out[4] = id_encoding_characters[num & 63]; num >>= 6; + out[3] = id_encoding_characters[num & 63]; num >>= 6; + out[2] = id_encoding_characters[num & 63]; num >>= 6; + out[1] = id_encoding_characters[num & 63]; num >>= 6; + out[0] = id_encoding_characters[num & 63]; +} + +static inline FACETS_HASH str_to_facets_hash(const char *str) { + FACETS_HASH num = 0; + int shifts = 6 * (FACET_STRING_HASH_SIZE - 2); + + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[0])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[1])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[2])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[3])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[4])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[5])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[6])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[7])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[8])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[9])])) << shifts; shifts -= 6; + num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[10])])) << shifts; + + return num; +} + +static const char *hash_to_static_string(FACETS_HASH hash) { + static __thread char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(hash, hash_str); + return hash_str; +} + +static inline bool is_valid_string_hash(const char *s) { + if(strlen(s) != FACET_STRING_HASH_SIZE - 1) { + netdata_log_error("The user supplied key '%s' does not have the right length for a facets hash.", s); + return false; + } + + uint8_t *t = (uint8_t *)s; + while(*t) { + if(id_encoding_characters_reverse[*t] == 0 && *t != id_encoding_characters[0]) { + netdata_log_error("The user supplied key '%s' contains invalid characters for a facets hash.", s); + return false; + } + + t++; + } + + return true; +} + +// ---------------------------------------------------------------------------- +// hashtable for FACET_VALUE + +// cleanup hashtable defines +#undef SIMPLE_HASHTABLE_SORT_FUNCTION +#undef SIMPLE_HASHTABLE_VALUE_TYPE +#undef SIMPLE_HASHTABLE_NAME +#undef NETDATA_SIMPLE_HASHTABLE_H + +struct facet_value; +// #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_value +#define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_value +#define SIMPLE_HASHTABLE_NAME _VALUE +#include "../simple_hashtable.h" + +// ---------------------------------------------------------------------------- +// hashtable for FACET_KEY + +// cleanup hashtable defines +#undef SIMPLE_HASHTABLE_SORT_FUNCTION +#undef SIMPLE_HASHTABLE_VALUE_TYPE +#undef SIMPLE_HASHTABLE_NAME +#undef NETDATA_SIMPLE_HASHTABLE_H + +struct facet_key; +// #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_key +#define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_key +#define SIMPLE_HASHTABLE_NAME _KEY +#include "../simple_hashtable.h" + +// ---------------------------------------------------------------------------- + +typedef struct facet_value { + FACETS_HASH hash; + const char *name; + const char *color; + uint32_t name_len; + + bool selected; + bool empty; + bool unsampled; + bool estimated; + + uint32_t rows_matching_facet_value; + uint32_t final_facet_value_counter; + uint32_t order; + + uint32_t *histogram; + uint32_t min, max, sum; + + struct facet_value *prev, *next; +} FACET_VALUE; + +typedef enum { + FACET_KEY_VALUE_NONE = 0, + FACET_KEY_VALUE_UPDATED = (1 << 0), + FACET_KEY_VALUE_EMPTY = (1 << 1), + FACET_KEY_VALUE_UNSAMPLED = (1 << 2), + FACET_KEY_VALUE_ESTIMATED = (1 << 3), + FACET_KEY_VALUE_COPIED = (1 << 4), +} FACET_KEY_VALUE_FLAGS; + +#define facet_key_value_updated(k) ((k)->current_value.flags & FACET_KEY_VALUE_UPDATED) +#define facet_key_value_empty(k) ((k)->current_value.flags & FACET_KEY_VALUE_EMPTY) +#define facet_key_value_unsampled(k) ((k)->current_value.flags & FACET_KEY_VALUE_UNSAMPLED) +#define facet_key_value_estimated(k) ((k)->current_value.flags & FACET_KEY_VALUE_ESTIMATED) +#define facet_key_value_empty_or_unsampled_or_estimated(k) ((k)->current_value.flags & (FACET_KEY_VALUE_EMPTY|FACET_KEY_VALUE_UNSAMPLED|FACET_KEY_VALUE_ESTIMATED)) +#define facet_key_value_copied(k) ((k)->current_value.flags & FACET_KEY_VALUE_COPIED) + +struct facet_key { + FACETS *facets; + + FACETS_HASH hash; + const char *name; + + FACET_KEY_OPTIONS options; + + bool default_selected_for_values; // the default "selected" for all values in the dictionary + + // members about the current row + uint32_t key_found_in_row; + uint32_t key_values_selected_in_row; + uint32_t order; + + struct { + bool enabled; + uint32_t used; + FACET_VALUE *ll; + SIMPLE_HASHTABLE_VALUE ht; + } values; + + struct { + FACETS_HASH hash; + FACET_KEY_VALUE_FLAGS flags; + const char *raw; + uint32_t raw_len; + BUFFER *b; + FACET_VALUE *v; + } current_value; + + struct { + FACET_VALUE *v; + } empty_value; + + struct { + FACET_VALUE *v; + } unsampled_value; + + struct { + FACET_VALUE *v; + } estimated_value; + + struct { + facet_dynamic_row_t cb; + void *data; + } dynamic; + + struct { + bool view_only; + facets_key_transformer_t cb; + void *data; + } transform; + + struct facet_key *prev, *next; +}; + +struct facets { + SIMPLE_PATTERN *visible_keys; + SIMPLE_PATTERN *excluded_keys; + SIMPLE_PATTERN *included_keys; + + FACETS_OPTIONS options; + + struct { + usec_t start_ut; + usec_t stop_ut; + FACETS_ANCHOR_DIRECTION direction; + } anchor; + + SIMPLE_PATTERN *query; // the full text search pattern + size_t keys_filtered_by_query; // the number of fields we do full text search (constant) + + DICTIONARY *accepted_params; + + struct { + size_t count; + FACET_KEY *ll; + SIMPLE_HASHTABLE_KEY ht; + } keys; + + struct { + // this is like a stack, of the keys that are used as facets + size_t used; + FACET_KEY *array[FACETS_KEYS_WITH_VALUES_MAX]; + } keys_with_values; + + struct { + // this is like a stack, of the keys that need to clean up between each row + size_t used; + FACET_KEY *array[FACETS_KEYS_IN_ROW_MAX]; + } keys_in_row; + + FACET_ROW *base; // double linked list of the selected facets rows + + uint32_t items_to_return; + uint32_t max_items_to_return; + uint32_t order; + + struct { + FACET_ROW_SEVERITY severity; + size_t keys_matched_by_query_positive; // the number of fields matched the full text search (per row) + size_t keys_matched_by_query_negative; // the number of fields matched the full text search (per row) + } current_row; + + struct { + usec_t after_ut; + usec_t before_ut; + } timeframe; + + struct { + FACET_KEY *key; + FACETS_HASH hash; + char *chart; + bool enabled; + uint32_t slots; + usec_t slot_width_ut; + usec_t after_ut; + usec_t before_ut; + } histogram; + + struct { + facet_row_severity_t cb; + void *data; + } severity; + + struct { + FACET_ROW *last_added; + + size_t first; + size_t forwards; + size_t backwards; + size_t skips_before; + size_t skips_after; + size_t prepends; + size_t appends; + size_t shifts; + + struct { + size_t evaluated; + size_t matched; + size_t unsampled; + size_t estimated; + size_t created; + size_t reused; + } rows; + + struct { + size_t registered; + size_t unique; + } keys; + + struct { + size_t registered; + size_t transformed; + size_t dynamic; + size_t empty; + size_t unsampled; + size_t estimated; + size_t indexed; + size_t inserts; + size_t conflicts; + } values; + + struct { + size_t searches; + } fts; + } operations; + + struct { + DICTIONARY *used_hashes_registry; + } report; +}; + +usec_t facets_row_oldest_ut(FACETS *facets) { + if(facets->base) + return facets->base->prev->usec; + + return 0; +} + +usec_t facets_row_newest_ut(FACETS *facets) { + if(facets->base) + return facets->base->usec; + + return 0; +} + +uint32_t facets_rows(FACETS *facets) { + return facets->items_to_return; +} + +// ---------------------------------------------------------------------------- + +static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row); +static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v); +static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k); + +// ---------------------------------------------------------------------------- +// The FACET_VALUE index within each FACET_KEY + +#define foreach_value_in_key(k, v) \ + for((v) = (k)->values.ll; (v) ;(v) = (v)->next) + +#define foreach_value_in_key_done(v) do { ; } while(0) + +static inline void FACETS_VALUES_INDEX_CREATE(FACET_KEY *k) { + k->values.ll = NULL; + k->values.used = 0; + simple_hashtable_init_VALUE(&k->values.ht, FACETS_VALUES_HASHTABLE_ENTRIES); +} + +static inline void FACETS_VALUES_INDEX_DESTROY(FACET_KEY *k) { + FACET_VALUE *v = k->values.ll; + while(v) { + FACET_VALUE *next = v->next; + freez(v->histogram); + freez((void *)v->name); + freez(v); + v = next; + } + k->values.ll = NULL; + k->values.used = 0; + k->values.enabled = false; + + simple_hashtable_destroy_VALUE(&k->values.ht); +} + +static inline const char *facets_key_get_value(FACET_KEY *k) { + return facet_key_value_copied(k) ? buffer_tostring(k->current_value.b) : k->current_value.raw; +} + +static inline uint32_t facets_key_get_value_length(FACET_KEY *k) { + return facet_key_value_copied(k) ? buffer_strlen(k->current_value.b) : k->current_value.raw_len; +} + +static inline void facets_key_value_copy_to_buffer(FACET_KEY *k) { + if(!facet_key_value_copied(k)) { + buffer_contents_replace(k->current_value.b, k->current_value.raw, k->current_value.raw_len); + k->current_value.flags |= FACET_KEY_VALUE_COPIED; + } +} + +static const char *facets_value_dup(const char *s, uint32_t len) { + char *d = mallocz(len + 1); + + if(len) + memcpy(d, s, len); + + d[len] = '\0'; + + return d; +} + +static inline void FACET_VALUE_ADD_CONFLICT(FACET_KEY *k, FACET_VALUE *v, const FACET_VALUE * const nv) { + if(!v->name && !v->name_len && nv->name && nv->name_len) { + // an actual value, not a filter + v->name = facets_value_dup(nv->name, nv->name_len); + v->name_len = nv->name_len; + } + + if(v->name && v->name_len) + facet_value_is_used(k, v); + + internal_fatal(v->name && nv->name && v->name_len == nv->name_len && memcmp(v->name, nv->name, v->name_len) != 0, + "value hash conflict: '%s' and '%s' have the same hash '%s'", + v->name, nv->name, hash_to_static_string(v->hash)); + + k->facets->operations.values.conflicts++; +} + +static inline FACET_VALUE *FACET_VALUE_GET_FROM_INDEX(FACET_KEY *k, FACETS_HASH hash) { + SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, hash, true); + return SIMPLE_HASHTABLE_SLOT_DATA(slot); +} + +static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VALUE * const tv) { + SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, tv->hash, true); + + if(SIMPLE_HASHTABLE_SLOT_DATA(slot)) { + // already exists + + FACET_VALUE *v = SIMPLE_HASHTABLE_SLOT_DATA(slot); + FACET_VALUE_ADD_CONFLICT(k, v, tv); + return v; + } + + // we have to add it + + FACET_VALUE *v = mallocz(sizeof(*v)); + simple_hashtable_set_slot_VALUE(&k->values.ht, slot, tv->hash, v); + + memcpy(v, tv, sizeof(*v)); + + if(v->estimated || v->unsampled) { + if(k->values.ll && k->values.ll->estimated) { + FACET_VALUE *estimated = k->values.ll; + DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(k->values.ll, estimated, v, prev, next); + } + else + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(k->values.ll, v, prev, next); + } + else + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(k->values.ll, v, prev, next); + + k->values.used++; + + if(!v->selected) + v->selected = k->default_selected_for_values; + + if(v->name && v->name_len) { + // an actual value, not a filter + v->name = facets_value_dup(v->name, v->name_len); + facet_value_is_used(k, v); + } + else { + v->name = NULL; + v->name_len = 0; + } + + k->facets->operations.values.inserts++; + + return v; +} + +static inline void FACET_VALUE_ADD_UNSAMPLED_VALUE_TO_INDEX(FACET_KEY *k) { + static const FACET_VALUE tv = { + .hash = FACETS_HASH_UNSAMPLED, + .name = FACET_VALUE_UNSAMPLED, + .name_len = sizeof(FACET_VALUE_UNSAMPLED) - 1, + .unsampled = true, + .color = "offline", + }; + + k->current_value.hash = FACETS_HASH_UNSAMPLED; + + if(k->unsampled_value.v) { + FACET_VALUE_ADD_CONFLICT(k, k->unsampled_value.v, &tv); + k->current_value.v = k->unsampled_value.v; + } + else { + FACET_VALUE *v = FACET_VALUE_ADD_TO_INDEX(k, &tv); + v->unsampled = true; + k->unsampled_value.v = v; + k->current_value.v = v; + } +} + +static inline void FACET_VALUE_ADD_ESTIMATED_VALUE_TO_INDEX(FACET_KEY *k) { + static const FACET_VALUE tv = { + .hash = FACETS_HASH_ESTIMATED, + .name = FACET_VALUE_ESTIMATED, + .name_len = sizeof(FACET_VALUE_ESTIMATED) - 1, + .estimated = true, + .color = "generic", + }; + + k->current_value.hash = FACETS_HASH_ESTIMATED; + + if(k->estimated_value.v) { + FACET_VALUE_ADD_CONFLICT(k, k->estimated_value.v, &tv); + k->current_value.v = k->estimated_value.v; + } + else { + FACET_VALUE *v = FACET_VALUE_ADD_TO_INDEX(k, &tv); + v->estimated = true; + k->estimated_value.v = v; + k->current_value.v = v; + } +} + +static inline void FACET_VALUE_ADD_EMPTY_VALUE_TO_INDEX(FACET_KEY *k) { + static const FACET_VALUE tv = { + .hash = FACETS_HASH_ZERO, + .name = FACET_VALUE_UNSET, + .name_len = sizeof(FACET_VALUE_UNSET) - 1, + .empty = true, + }; + + k->current_value.hash = FACETS_HASH_ZERO; + + if(k->empty_value.v) { + FACET_VALUE_ADD_CONFLICT(k, k->empty_value.v, &tv); + k->current_value.v = k->empty_value.v; + } + else { + FACET_VALUE *v = FACET_VALUE_ADD_TO_INDEX(k, &tv); + v->empty = true; + k->empty_value.v = v; + k->current_value.v = v; + } +} + +static inline void FACET_VALUE_ADD_CURRENT_VALUE_TO_INDEX(FACET_KEY *k) { + static __thread FACET_VALUE tv = { 0 }; + + internal_fatal(!facet_key_value_updated(k), "trying to add a non-updated value to the index"); + + tv.name = facets_key_get_value(k); + tv.name_len = facets_key_get_value_length(k); + tv.hash = FACETS_HASH_FUNCTION(tv.name, tv.name_len); + tv.empty = false; + tv.estimated = false; + tv.unsampled = false; + + k->current_value.v = FACET_VALUE_ADD_TO_INDEX(k, &tv); + k->facets->operations.values.indexed++; +} + +static inline void FACET_VALUE_ADD_OR_UPDATE_SELECTED(FACET_KEY *k, FACETS_HASH hash) { + FACET_VALUE tv = { + .hash = hash, + .selected = true, + .name = NULL, + .name_len = 0, + }; + FACET_VALUE_ADD_TO_INDEX(k, &tv); +} + +// ---------------------------------------------------------------------------- +// The FACET_KEY index within each FACET + +#define foreach_key_in_facets(facets, k) \ + for((k) = (facets)->keys.ll; (k) ;(k) = (k)->next) + +#define foreach_key_in_facets_done(k) do { ; } while(0) + +static inline void facet_key_late_init(FACETS *facets, FACET_KEY *k) { + if(k->values.enabled) + return; + + if(facets_key_is_facet(facets, k)) { + FACETS_VALUES_INDEX_CREATE(k); + k->values.enabled = true; + if(facets->keys_with_values.used < FACETS_KEYS_WITH_VALUES_MAX) + facets->keys_with_values.array[facets->keys_with_values.used++] = k; + } +} + +static inline void FACETS_KEYS_INDEX_CREATE(FACETS *facets) { + facets->keys.ll = NULL; + facets->keys.count = 0; + facets->keys_with_values.used = 0; + + simple_hashtable_init_KEY(&facets->keys.ht, FACETS_KEYS_HASHTABLE_ENTRIES); +} + +static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) { + FACET_KEY *k = facets->keys.ll; + while(k) { + FACET_KEY *next = k->next; + + FACETS_VALUES_INDEX_DESTROY(k); + buffer_free(k->current_value.b); + freez((void *)k->name); + freez(k); + + k = next; + } + facets->keys.ll = NULL; + facets->keys.count = 0; + facets->keys_with_values.used = 0; + + simple_hashtable_destroy_KEY(&facets->keys.ht); +} + +static inline FACET_KEY *FACETS_KEY_GET_FROM_INDEX(FACETS *facets, FACETS_HASH hash) { + SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true); + return SIMPLE_HASHTABLE_SLOT_DATA(slot); +} + +bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length); + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); + if(!k || k->default_selected_for_values) + return false; + + hash = FACETS_HASH_FUNCTION(value, value_length); + FACET_VALUE *v = FACET_VALUE_GET_FROM_INDEX(k, hash); + return (v && v->selected) ? true : false; +} + +void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length); + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); + if(!k) return; + + hash = FACETS_HASH_FUNCTION(value, value_length); + FACET_VALUE *v = FACET_VALUE_GET_FROM_INDEX(k, hash); + if(v && v->name && v->name_len) return; + + FACET_VALUE tv = { + .hash = hash, + .name = value, + .name_len = value_length, + }; + FACET_VALUE_ADD_TO_INDEX(k, &tv); +} + +static void facet_key_set_name(FACET_KEY *k, const char *name, size_t name_length) { + internal_fatal(k->name && name && (strncmp(k->name, name, name_length) != 0 || k->name[name_length] != '\0'), + "key hash conflict: '%s' and '%s' have the same hash", + k->name, name); + + if(likely(k->name || !name || !name_length)) + return; + + // an actual value, not a filter + + char buf[name_length + 1]; + memcpy(buf, name, name_length); + buf[name_length] = '\0'; + + internal_fatal(strchr(buf, '='), "found = in key"); + + k->name = strdupz(buf); + facet_key_late_init(k->facets, k); +} + +static inline FACET_KEY *FACETS_KEY_CREATE(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) { + facets->operations.keys.unique++; + + FACET_KEY *k = callocz(1, sizeof(*k)); + + k->hash = hash; + k->facets = facets; + k->options = options; + k->current_value.b = buffer_create(sizeof(FACET_VALUE_UNSET), NULL); + k->default_selected_for_values = true; + + if(!(k->options & FACET_KEY_OPTION_REORDER)) + k->order = facets->order++; + + if((k->options & FACET_KEY_OPTION_FTS) || (facets->options & FACETS_OPTION_ALL_KEYS_FTS)) + facets->keys_filtered_by_query++; + + facet_key_set_name(k, name, name_length); + + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->keys.ll, k, prev, next); + facets->keys.count++; + + return k; +} + +static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) { + facets->operations.keys.registered++; + + SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true); + + if(unlikely(!SIMPLE_HASHTABLE_SLOT_DATA(slot))) { + // we have to add it + FACET_KEY *k = FACETS_KEY_CREATE(facets, hash, name, name_length, options); + + simple_hashtable_set_slot_KEY(&facets->keys.ht, slot, hash, k); + + return k; + } + + // already in the index + + FACET_KEY *k = SIMPLE_HASHTABLE_SLOT_DATA(slot); + + facet_key_set_name(k, name, name_length); + + if(unlikely(k->options & FACET_KEY_OPTION_REORDER)) { + k->order = facets->order++; + k->options &= ~FACET_KEY_OPTION_REORDER; + } + + return k; +} + +bool facets_key_name_is_filter(FACETS *facets, const char *key) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, strlen(key)); + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); + return (!k || k->default_selected_for_values) ? false : true; +} + +bool facets_key_name_is_facet(FACETS *facets, const char *key) { + size_t key_len = strlen(key); + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_len); + FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, key, key_len, 0); + return (k && (k->options & FACET_KEY_OPTION_FACET)); +} + +// ---------------------------------------------------------------------------- + +size_t facets_histogram_slots(FACETS *facets) { + return facets->histogram.slots; +} + +static usec_t calculate_histogram_bar_width(usec_t after_ut, usec_t before_ut) { + // Array of valid durations in seconds + static time_t valid_durations_s[] = { + 1, 2, 5, 10, 15, 30, // seconds + 1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes + 1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours + 1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days + 1 * (30*86400) // months + }; + static int array_size = sizeof(valid_durations_s) / sizeof(valid_durations_s[0]); + + usec_t duration_ut = before_ut - after_ut; + usec_t bar_width_ut = 1 * USEC_PER_SEC; + + for (int i = array_size - 1; i >= 0; --i) { + if (duration_ut / (valid_durations_s[i] * USEC_PER_SEC) >= FACETS_HISTOGRAM_COLUMNS) { + bar_width_ut = valid_durations_s[i] * USEC_PER_SEC; + break; + } + } + + return bar_width_ut; +} + +static inline usec_t facets_histogram_slot_baseline_ut(FACETS *facets, usec_t ut) { + usec_t delta_ut = ut % facets->histogram.slot_width_ut; + return ut - delta_ut; +} + +void facets_set_timeframe_and_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut) { + if(after_ut > before_ut) { + usec_t t = after_ut; + after_ut = before_ut; + before_ut = t; + } + + facets->histogram.enabled = true; + + if(key_id && *key_id && strlen(key_id) == FACET_STRING_HASH_SIZE - 1) { + facets->histogram.chart = strdupz(key_id); + facets->histogram.hash = str_to_facets_hash(facets->histogram.chart); + } + else { + freez(facets->histogram.chart); + facets->histogram.chart = NULL; + facets->histogram.hash = FACETS_HASH_ZERO; + } + + facets->timeframe.after_ut = after_ut; + facets->timeframe.before_ut = before_ut; + + facets->histogram.slot_width_ut = calculate_histogram_bar_width(after_ut, before_ut); + facets->histogram.after_ut = facets_histogram_slot_baseline_ut(facets, after_ut); + facets->histogram.before_ut = facets_histogram_slot_baseline_ut(facets, before_ut) + facets->histogram.slot_width_ut; + facets->histogram.slots = (facets->histogram.before_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut + 1; + + internal_fatal(after_ut < facets->histogram.after_ut, "histogram after_ut is not less or equal to wanted after_ut"); + internal_fatal(before_ut > facets->histogram.before_ut, "histogram before_ut is not more or equal to wanted before_ut"); + + if(facets->histogram.slots > 1000) { + facets->histogram.slots = 1000 + 1; + facets->histogram.slot_width_ut = (facets->histogram.before_ut - facets->histogram.after_ut) / 1000; + } +} + +void facets_set_timeframe_and_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut) { + char hash_str[FACET_STRING_HASH_SIZE]; + FACETS_HASH hash = FACETS_HASH_FUNCTION(key_name, strlen(key_name)); + facets_hash_to_str(hash, hash_str); + facets_set_timeframe_and_histogram_by_id(facets, hash_str, after_ut, before_ut); +} + +static inline uint32_t facets_histogram_slot_at_time_ut(FACETS *facets, usec_t usec, FACET_VALUE *v) { + if(unlikely(!v->histogram)) + v->histogram = callocz(facets->histogram.slots, sizeof(*v->histogram)); + + usec_t base_ut = facets_histogram_slot_baseline_ut(facets, usec); + + if(unlikely(base_ut < facets->histogram.after_ut)) + base_ut = facets->histogram.after_ut; + + if(unlikely(base_ut > facets->histogram.before_ut)) + base_ut = facets->histogram.before_ut; + + uint32_t slot = (base_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut; + + if(unlikely(slot >= facets->histogram.slots)) + slot = facets->histogram.slots - 1; + + return slot; +} + +static inline void facets_histogram_update_value_slot(FACETS *facets, usec_t usec, FACET_VALUE *v) { + uint32_t slot = facets_histogram_slot_at_time_ut(facets, usec, v); + v->histogram[slot]++; +} + +static inline void facets_histogram_update_value(FACETS *facets, usec_t usec) { + if(!facets->histogram.enabled || + !facets->histogram.key || + !facets->histogram.key->values.enabled || + !facet_key_value_updated(facets->histogram.key) || + usec < facets->histogram.after_ut || + usec > facets->histogram.before_ut) + return; + + FACET_VALUE *v = facets->histogram.key->current_value.v; + facets_histogram_update_value_slot(facets, usec, v); +} + +static usec_t overlap_duration_ut(usec_t start1, usec_t end1, usec_t start2, usec_t end2) { + usec_t overlap_start = MAX(start1, start2); + usec_t overlap_end = MIN(end1, end2); + + if (overlap_start < overlap_end) + return overlap_end - overlap_start; + else + return 0; // No overlap +} + +void facets_update_estimations(FACETS *facets, usec_t from_ut, usec_t to_ut, size_t entries) { + if(unlikely(!facets->histogram.enabled)) + return; + + if(unlikely(!overlap_duration_ut(facets->histogram.after_ut, facets->histogram.before_ut, from_ut, to_ut))) + return; + + facets->operations.rows.evaluated += entries; + facets->operations.rows.matched += entries; + facets->operations.rows.estimated += entries; + + if (!facets->histogram.enabled || + !facets->histogram.key || + !facets->histogram.key->values.enabled) + return; + + if (from_ut < facets->histogram.after_ut) + from_ut = facets->histogram.after_ut; + + if (to_ut > facets->histogram.before_ut) + to_ut = facets->histogram.before_ut; + + if (!facets->histogram.key->estimated_value.v) + FACET_VALUE_ADD_ESTIMATED_VALUE_TO_INDEX(facets->histogram.key); + + FACET_VALUE *v = facets->histogram.key->estimated_value.v; + + size_t slots = 0; + size_t total_ut = to_ut - from_ut; + ssize_t remaining_entries = (ssize_t)entries; + size_t slot = facets_histogram_slot_at_time_ut(facets, from_ut, v); + for(; slot < facets->histogram.slots ;slot++) { + usec_t slot_start_ut = facets->histogram.after_ut + slot * facets->histogram.slot_width_ut; + usec_t slot_end_ut = slot_start_ut + facets->histogram.slot_width_ut; + + if(slot_start_ut > to_ut) + break; + + usec_t overlap_ut = overlap_duration_ut(from_ut, to_ut, slot_start_ut, slot_end_ut); + + size_t slot_entries = (overlap_ut * entries) / total_ut; + v->histogram[slot] += slot_entries; + remaining_entries -= (ssize_t)slot_entries; + slots++; + } + + // Check if all entries are assigned + // This should always be true if the distribution is correct + internal_fatal(remaining_entries < 0 || remaining_entries >= (ssize_t)(slots), + "distribution of estimations is not accurate - there are %zd remaining entries", + remaining_entries); +} + +void facets_row_finished_unsampled(FACETS *facets, usec_t usec) { + facets->operations.rows.evaluated++; + facets->operations.rows.matched++; + facets->operations.rows.unsampled++; + + if(!facets->histogram.enabled || + !facets->histogram.key || + !facets->histogram.key->values.enabled || + usec < facets->histogram.after_ut || + usec > facets->histogram.before_ut) + return; + + if(!facets->histogram.key->unsampled_value.v) + FACET_VALUE_ADD_UNSAMPLED_VALUE_TO_INDEX(facets->histogram.key); + + FACET_VALUE *v = facets->histogram.key->unsampled_value.v; + facets_histogram_update_value_slot(facets, usec, v); + + facets_reset_key(facets->histogram.key); +} + +static const char *facets_key_name_cached(FACET_KEY *k, DICTIONARY *used_hashes_registry) { + if(k->name) { + if(used_hashes_registry && !k->default_selected_for_values) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(k->hash, hash_str); + dictionary_set(used_hashes_registry, hash_str, (void *)k->name, strlen(k->name) + 1); + } + + return k->name; + } + + // key has no name + const char *name = "[UNAVAILABLE_FIELD]"; + + if(used_hashes_registry) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(k->hash, hash_str); + const char *s = dictionary_get(used_hashes_registry, hash_str); + if(s) name = s; + } + + return name; +} + +static const char *facets_key_value_cached(FACET_KEY *k, FACET_VALUE *v, DICTIONARY *used_hashes_registry) { + if(v->empty || v->estimated || v->unsampled) + return v->name; + + if(v->name && v->name_len) { + if(used_hashes_registry && !k->default_selected_for_values && v->selected) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(v->hash, hash_str); + dictionary_set(used_hashes_registry, hash_str, (void *)v->name, v->name_len + 1); + } + + return v->name; + } + + // key has no name + const char *name = "[unavailable field]"; + + if(used_hashes_registry) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(v->hash, hash_str); + const char *s = dictionary_get(used_hashes_registry, hash_str); + if(s) name = s; + } + + return name; +} + +static inline void facets_key_value_transformed(FACETS *facets, FACET_KEY *k, FACET_VALUE *v, BUFFER *dst, FACETS_TRANSFORMATION_SCOPE scope) { + buffer_flush(dst); + + if(v->empty || v->unsampled || v->estimated) + buffer_strcat(dst, v->name); + else if(k->transform.cb && k->transform.view_only) { + buffer_contents_replace(dst, v->name, v->name_len); + k->transform.cb(facets, dst, scope, k->transform.data); + } + else + buffer_strcat(dst, facets_key_value_cached(k, v, facets->report.used_hashes_registry)); +} + +static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, const char *first_key) { + CLEAN_BUFFER *tb = buffer_create(0, NULL); + + buffer_json_member_add_array(wb, key); + { + if(first_key) + buffer_json_add_array_item_string(wb, first_key); + + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + facets_key_value_transformed(facets, k, v, tb, FACETS_TRANSFORM_HISTOGRAM); + buffer_json_add_array_item_string(wb, buffer_tostring(tb)); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_colors(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_string(wb, v->color); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_units(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_string(wb, "events"); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_min(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_uint64(wb, v->min); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_max(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_uint64(wb, v->max); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_avg(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_double(wb, (double) v->sum / (double) facets->histogram.slots); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_arp(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_uint64(wb, 0); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static inline void facets_histogram_value_con(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, uint32_t sum) { + buffer_json_member_add_array(wb, key); + { + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_double(wb, (double) v->sum * 100.0 / (double) sum); + } + foreach_value_in_key_done(v); + } + } + buffer_json_array_close(wb); // key +} + +static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb) { + CLEAN_BUFFER *tmp = buffer_create(0, NULL); + + size_t dimensions = 0; + uint32_t min = UINT32_MAX, max = 0, sum = 0, count = 0; + + if(k && k->values.enabled) { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + dimensions++; + + v->min = UINT32_MAX; + v->max = 0; + v->sum = 0; + + for(uint32_t i = 0; i < facets->histogram.slots ;i++) { + uint32_t n = v->histogram[i]; + + if(n < min) + min = n; + + if(n > max) + max = n; + + sum += n; + count++; + + if(n < v->min) + v->min = n; + + if(n > v->max) + v->max = n; + + v->sum += n; + } + } + foreach_value_in_key_done(v); + } + + buffer_json_member_add_object(wb, "summary"); + { + // summary.nodes + buffer_json_member_add_array(wb, "nodes"); + { + buffer_json_add_array_item_object(wb); // node + { + buffer_json_member_add_string(wb, "mg", "default"); + buffer_json_member_add_string(wb, "nm", "facets.histogram"); + buffer_json_member_add_uint64(wb, "ni", 0); + buffer_json_member_add_object(wb, "st"); + { + buffer_json_member_add_uint64(wb, "ai", 0); + buffer_json_member_add_uint64(wb, "code", 200); + buffer_json_member_add_string(wb, "msg", ""); + } + buffer_json_object_close(wb); // st + + if(dimensions) { + buffer_json_member_add_object(wb, "is"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // is + + buffer_json_member_add_object(wb, "ds"); + { + buffer_json_member_add_uint64(wb, "sl", dimensions); + buffer_json_member_add_uint64(wb, "qr", dimensions); + } + buffer_json_object_close(wb); // ds + } + + if(count) { + buffer_json_member_add_object(wb, "sts"); + { + buffer_json_member_add_uint64(wb, "min", min); + buffer_json_member_add_uint64(wb, "max", max); + buffer_json_member_add_double(wb, "avg", (double) sum / (double) count); + buffer_json_member_add_double(wb, "con", 100.0); + } + buffer_json_object_close(wb); // sts + } + } + buffer_json_object_close(wb); // node + } + buffer_json_array_close(wb); // nodes + + // summary.contexts + buffer_json_member_add_array(wb, "contexts"); + { + buffer_json_add_array_item_object(wb); // context + { + buffer_json_member_add_string(wb, "id", "facets.histogram"); + + if(dimensions) { + buffer_json_member_add_object(wb, "is"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // is + + buffer_json_member_add_object(wb, "ds"); + { + buffer_json_member_add_uint64(wb, "sl", dimensions); + buffer_json_member_add_uint64(wb, "qr", dimensions); + } + buffer_json_object_close(wb); // ds + } + + if(count) { + buffer_json_member_add_object(wb, "sts"); + { + buffer_json_member_add_uint64(wb, "min", min); + buffer_json_member_add_uint64(wb, "max", max); + buffer_json_member_add_double(wb, "avg", (double) sum / (double) count); + buffer_json_member_add_double(wb, "con", 100.0); + } + buffer_json_object_close(wb); // sts + } + } + buffer_json_object_close(wb); // context + } + buffer_json_array_close(wb); // contexts + + // summary.instances + buffer_json_member_add_array(wb, "instances"); + { + buffer_json_add_array_item_object(wb); // instance + { + buffer_json_member_add_string(wb, "id", "facets.histogram"); + buffer_json_member_add_uint64(wb, "ni", 0); + + if(dimensions) { + buffer_json_member_add_object(wb, "ds"); + { + buffer_json_member_add_uint64(wb, "sl", dimensions); + buffer_json_member_add_uint64(wb, "qr", dimensions); + } + buffer_json_object_close(wb); // ds + } + + if(count) { + buffer_json_member_add_object(wb, "sts"); + { + buffer_json_member_add_uint64(wb, "min", min); + buffer_json_member_add_uint64(wb, "max", max); + buffer_json_member_add_double(wb, "avg", (double) sum / (double) count); + buffer_json_member_add_double(wb, "con", 100.0); + } + buffer_json_object_close(wb); // sts + } + } + buffer_json_object_close(wb); // instance + } + buffer_json_array_close(wb); // instances + + // summary.dimensions + buffer_json_member_add_array(wb, "dimensions"); + if(dimensions && k && k->values.enabled) { + size_t pri = 0; + FACET_VALUE *v; + + foreach_value_in_key(k, v) { + if(unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_object(wb); // dimension + { + facets_key_value_transformed(facets, k, v, tmp, FACETS_TRANSFORM_HISTOGRAM); + buffer_json_member_add_string(wb, "id", buffer_tostring(tmp)); + buffer_json_member_add_object(wb, "ds"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // ds + buffer_json_member_add_object(wb, "sts"); + { + buffer_json_member_add_uint64(wb, "min", v->min); + buffer_json_member_add_uint64(wb, "max", v->max); + buffer_json_member_add_double(wb, "avg", (double)v->sum / (double)facets->histogram.slots); + buffer_json_member_add_double(wb, "con", (double)v->sum * 100.0 / (double)sum); + } + buffer_json_object_close(wb); // sts + buffer_json_member_add_uint64(wb, "pri", pri++); + } + buffer_json_object_close(wb); // dimension + } + foreach_value_in_key_done(v); + } + buffer_json_array_close(wb); // dimensions + + buffer_json_member_add_array(wb, "labels"); + buffer_json_array_close(wb); // labels + + buffer_json_member_add_array(wb, "alerts"); + buffer_json_array_close(wb); // alerts + } + buffer_json_object_close(wb); // summary + + buffer_json_member_add_object(wb, "totals"); + { + buffer_json_member_add_object(wb, "nodes"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // nodes + + if(dimensions) { + buffer_json_member_add_object(wb, "contexts"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // contexts + buffer_json_member_add_object(wb, "instances"); + { + buffer_json_member_add_uint64(wb, "sl", 1); + buffer_json_member_add_uint64(wb, "qr", 1); + } + buffer_json_object_close(wb); // instances + + buffer_json_member_add_object(wb, "dimensions"); + { + buffer_json_member_add_uint64(wb, "sl", dimensions); + buffer_json_member_add_uint64(wb, "qr", dimensions); + } + buffer_json_object_close(wb); // dimension + } + } + buffer_json_object_close(wb); // totals + + buffer_json_member_add_object(wb, "result"); + { + facets_histogram_value_names(wb, facets, k, "labels", "time"); + + buffer_json_member_add_object(wb, "point"); + { + buffer_json_member_add_uint64(wb, "value", 0); + buffer_json_member_add_uint64(wb, "arp", 1); + buffer_json_member_add_uint64(wb, "pa", 2); + } + buffer_json_object_close(wb); // point + + buffer_json_member_add_array(wb, "data"); + if(k && k->values.enabled) { + usec_t t = facets->histogram.after_ut; + for(uint32_t i = 0; i < facets->histogram.slots ;i++) { + buffer_json_add_array_item_array(wb); // row + { + buffer_json_add_array_item_time_ms(wb, t / USEC_PER_SEC); + + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if (unlikely(!v->histogram)) + continue; + + buffer_json_add_array_item_array(wb); // point + + buffer_json_add_array_item_uint64(wb, v->histogram[i]); + buffer_json_add_array_item_uint64(wb, 0); // arp - anomaly rate + buffer_json_add_array_item_uint64(wb, 0); // pa - point annotation + + buffer_json_array_close(wb); // point + } + foreach_value_in_key_done(v); + } + buffer_json_array_close(wb); // row + + t += facets->histogram.slot_width_ut; + } + } + buffer_json_array_close(wb); //data + } + buffer_json_object_close(wb); // result + + buffer_json_member_add_object(wb, "db"); + { + buffer_json_member_add_uint64(wb, "tiers", 1); + buffer_json_member_add_uint64(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC); +// we should add these only when we know the retention of the db +// buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC); +// buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC); + buffer_json_member_add_string(wb, "units", "events"); + buffer_json_member_add_object(wb, "dimensions"); + { + facets_histogram_value_names(wb, facets, k, "ids", NULL); + facets_histogram_value_units(wb, facets, k, "units"); + + buffer_json_member_add_object(wb, "sts"); + { + facets_histogram_value_min(wb, facets, k, "min"); + facets_histogram_value_max(wb, facets, k, "max"); + facets_histogram_value_avg(wb, facets, k, "avg"); + facets_histogram_value_arp(wb, facets, k, "arp"); + facets_histogram_value_con(wb, facets, k, "con", sum); + } + buffer_json_object_close(wb); // sts + } + buffer_json_object_close(wb); // dimensions + + buffer_json_member_add_array(wb, "per_tier"); + { + buffer_json_add_array_item_object(wb); // tier0 + { + buffer_json_member_add_uint64(wb, "tier", 0); + buffer_json_member_add_uint64(wb, "queries", 1); + buffer_json_member_add_uint64(wb, "points", count); + buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC); +// we should add these only when we know the retention of the db +// buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC); +// buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC); + } + buffer_json_object_close(wb); // tier0 + } + buffer_json_array_close(wb); // per_tier + } + buffer_json_object_close(wb); // db + + buffer_json_member_add_object(wb, "view"); + { + char title[1024 + 1] = "Events Distribution"; + FACET_KEY *kt = FACETS_KEY_GET_FROM_INDEX(facets, facets->histogram.hash); + if(kt && kt->name) + snprintfz(title, sizeof(title) - 1, "Events Distribution by %s", kt->name); + + buffer_json_member_add_string(wb, "title", title); + buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC); + buffer_json_member_add_time_t(wb, "after", facets->histogram.after_ut / USEC_PER_SEC); + buffer_json_member_add_time_t(wb, "before", facets->histogram.before_ut / USEC_PER_SEC); + buffer_json_member_add_string(wb, "units", "events"); + buffer_json_member_add_string(wb, "chart_type", "stackedBar"); + buffer_json_member_add_object(wb, "dimensions"); + { + buffer_json_member_add_array(wb, "grouped_by"); + { + buffer_json_add_array_item_string(wb, "dimension"); + } + buffer_json_array_close(wb); // grouped_by + + facets_histogram_value_names(wb, facets, k, "ids", NULL); + facets_histogram_value_names(wb, facets, k, "names", NULL); + facets_histogram_value_colors(wb, facets, k, "colors"); + facets_histogram_value_units(wb, facets, k, "units"); + + buffer_json_member_add_object(wb, "sts"); + { + facets_histogram_value_min(wb, facets, k, "min"); + facets_histogram_value_max(wb, facets, k, "max"); + facets_histogram_value_avg(wb, facets, k, "avg"); + facets_histogram_value_arp(wb, facets, k, "arp"); + facets_histogram_value_con(wb, facets, k, "con", sum); + } + buffer_json_object_close(wb); // sts + } + buffer_json_object_close(wb); // dimensions + + buffer_json_member_add_uint64(wb, "min", min); + buffer_json_member_add_uint64(wb, "max", max); + } + buffer_json_object_close(wb); // view + + buffer_json_member_add_array(wb, "agents"); + { + buffer_json_add_array_item_object(wb); // agent + { + buffer_json_member_add_string(wb, "mg", "default"); + buffer_json_member_add_string(wb, "nm", "facets.histogram"); + buffer_json_member_add_time_t(wb, "now", now_realtime_sec()); + buffer_json_member_add_uint64(wb, "ai", 0); + } + buffer_json_object_close(wb); // agent + } + buffer_json_array_close(wb); // agents +} + +// ---------------------------------------------------------------------------- + +static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) { + if(!k->key_found_in_row) + v->rows_matching_facet_value++; + + k->key_found_in_row++; + + if(v->selected) + k->key_values_selected_in_row++; +} + +static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) { + bool included = true, excluded = false, never = false; + + if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) { + if(k->options & FACET_KEY_OPTION_FACET) { + included = true; + excluded = false; + never = false; + } + else if(k->options & (FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) { + included = false; + excluded = true; + never = true; + } + } + else { + if (facets->included_keys) { + if (!simple_pattern_matches(facets->included_keys, k->name)) + included = false; + } + + if (facets->excluded_keys) { + if (simple_pattern_matches(facets->excluded_keys, k->name)) { + excluded = true; + never = true; + } + } + } + + if(included && !excluded) { + k->options |= FACET_KEY_OPTION_FACET; + k->options &= ~FACET_KEY_OPTION_NO_FACET; + return true; + } + + k->options |= FACET_KEY_OPTION_NO_FACET; + k->options &= ~FACET_KEY_OPTION_FACET; + + if(never) + k->options |= FACET_KEY_OPTION_NEVER_FACET; + + return false; +} + +// ---------------------------------------------------------------------------- + +FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) { + FACETS *facets = callocz(1, sizeof(FACETS)); + facets->options = options; + FACETS_KEYS_INDEX_CREATE(facets); + + if(facet_keys && *facet_keys) + facets->included_keys = simple_pattern_create(facet_keys, "|", SIMPLE_PATTERN_EXACT, true); + + if(non_facet_keys && *non_facet_keys) + facets->excluded_keys = simple_pattern_create(non_facet_keys, "|", SIMPLE_PATTERN_EXACT, true); + + if(visible_keys && *visible_keys) + facets->visible_keys = simple_pattern_create(visible_keys, "|", SIMPLE_PATTERN_EXACT, true); + + facets->max_items_to_return = items_to_return > 1 ? items_to_return : 2; + facets->anchor.start_ut = 0; + facets->anchor.stop_ut = 0; + facets->anchor.direction = FACETS_ANCHOR_DIRECTION_BACKWARD; + facets->order = 1; + + return facets; +} + +void facets_destroy(FACETS *facets) { + dictionary_destroy(facets->accepted_params); + FACETS_KEYS_INDEX_DESTROY(facets); + simple_pattern_free(facets->visible_keys); + simple_pattern_free(facets->included_keys); + simple_pattern_free(facets->excluded_keys); + + while(facets->base) { + FACET_ROW *r = facets->base; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, r, prev, next); + + facets_row_free(facets, r); + } + + freez(facets->histogram.chart); + freez(facets); +} + +void facets_accepted_param(FACETS *facets, const char *param) { + if(!facets->accepted_params) + facets->accepted_params = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_set(facets->accepted_params, param, NULL, 0); +} + +static inline FACET_KEY *facets_register_key_name_length(FACETS *facets, const char *key, size_t key_length, FACET_KEY_OPTIONS options) { + return FACETS_KEY_ADD_TO_INDEX(facets, FACETS_HASH_FUNCTION(key, key_length), key, key_length, options); +} + +inline FACET_KEY *facets_register_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options) { + return facets_register_key_name_length(facets, key, strlen(key), options); +} + +inline FACET_KEY *facets_register_key_name_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data) { + FACET_KEY *k = facets_register_key_name(facets, key, options); + k->transform.cb = cb; + k->transform.data = data; + k->transform.view_only = (options & FACET_KEY_OPTION_TRANSFORM_VIEW) ? true : false; + return k; +} + +inline FACET_KEY *facets_register_dynamic_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data) { + FACET_KEY *k = facets_register_key_name(facets, key, options); + k->dynamic.cb = cb; + k->dynamic.data = data; + return k; +} + +void facets_set_query(FACETS *facets, const char *query) { + if(!query) + return; + + facets->query = simple_pattern_create(query, "|", SIMPLE_PATTERN_SUBSTRING, false); +} + +void facets_set_items(FACETS *facets, uint32_t items) { + facets->max_items_to_return = items > 1 ? items : 2; +} + +void facets_set_anchor(FACETS *facets, usec_t start_ut, usec_t stop_ut, FACETS_ANCHOR_DIRECTION direction) { + facets->anchor.start_ut = start_ut; + facets->anchor.stop_ut = stop_ut; + facets->anchor.direction = direction; + + if((facets->anchor.direction == FACETS_ANCHOR_DIRECTION_BACKWARD && facets->anchor.start_ut && facets->anchor.start_ut < facets->anchor.stop_ut) || + (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD && facets->anchor.stop_ut && facets->anchor.stop_ut < facets->anchor.start_ut)) { + internal_error(true, "start and stop anchors are flipped"); + facets->anchor.start_ut = stop_ut; + facets->anchor.stop_ut = start_ut; + } +} + +void facets_enable_slice_mode(FACETS *facets) { + facets->options |= FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS | FACETS_OPTION_SORT_FACETS_ALPHABETICALLY; +} + +inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options) { + if(!is_valid_string_hash(key_id)) + return NULL; + + FACETS_HASH hash = str_to_facets_hash(key_id); + + internal_error(strcmp(hash_to_static_string(hash), key_id) != 0, + "Regenerating the user supplied key, does not produce the same hash string"); + + FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, NULL, 0, options); + k->options |= FACET_KEY_OPTION_FACET; + k->options &= ~FACET_KEY_OPTION_NO_FACET; + facet_key_late_init(facets, k); + + return k; +} + +void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *value_id, FACET_KEY_OPTIONS options) { + FACET_KEY *k = facets_register_facet_id(facets, key_id, options); + if(k) { + if(is_valid_string_hash(value_id)) { + k->default_selected_for_values = false; + FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, str_to_facets_hash(value_id)); + } + } +} + +void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity) { + facets->current_row.severity = severity; +} + +void facets_register_row_severity(FACETS *facets, facet_row_severity_t cb, void *data) { + facets->severity.cb = cb; + facets->severity.data = data; +} + +void facets_set_additional_options(FACETS *facets, FACETS_OPTIONS options) { + facets->options |= options; +} + +// ---------------------------------------------------------------------------- + +static inline void facets_key_set_unsampled_value(FACETS *facets, FACET_KEY *k) { + if(likely(!facet_key_value_updated(k) && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX)) + facets->keys_in_row.array[facets->keys_in_row.used++] = k; + + k->current_value.flags |= FACET_KEY_VALUE_UPDATED | FACET_KEY_VALUE_UNSAMPLED; + + facets->operations.values.registered++; + facets->operations.values.unsampled++; + + // no need to copy the UNSET value + // empty values are exported as empty + k->current_value.raw = NULL; + k->current_value.raw_len = 0; + k->current_value.b->len = 0; + k->current_value.flags &= ~FACET_KEY_VALUE_COPIED; + + if(unlikely(k->values.enabled)) + FACET_VALUE_ADD_UNSAMPLED_VALUE_TO_INDEX(k); + else { + k->key_found_in_row++; + k->key_values_selected_in_row++; + } +} + +static inline void facets_key_set_empty_value(FACETS *facets, FACET_KEY *k) { + if(likely(!facet_key_value_updated(k) && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX)) + facets->keys_in_row.array[facets->keys_in_row.used++] = k; + + k->current_value.flags |= FACET_KEY_VALUE_UPDATED | FACET_KEY_VALUE_EMPTY; + + facets->operations.values.registered++; + facets->operations.values.empty++; + + // no need to copy the UNSET value + // empty values are exported as empty + k->current_value.raw = NULL; + k->current_value.raw_len = 0; + k->current_value.b->len = 0; + k->current_value.flags &= ~FACET_KEY_VALUE_COPIED; + + if(unlikely(k->values.enabled)) + FACET_VALUE_ADD_EMPTY_VALUE_TO_INDEX(k); + else { + k->key_found_in_row++; + k->key_values_selected_in_row++; + } +} + +static inline void facets_key_check_value(FACETS *facets, FACET_KEY *k) { + if(likely(!facet_key_value_updated(k) && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX)) + facets->keys_in_row.array[facets->keys_in_row.used++] = k; + + k->current_value.flags |= FACET_KEY_VALUE_UPDATED; + k->current_value.flags &= ~(FACET_KEY_VALUE_EMPTY|FACET_KEY_VALUE_UNSAMPLED|FACET_KEY_VALUE_ESTIMATED); + + facets->operations.values.registered++; + + if(k->transform.cb && !k->transform.view_only) { + facets->operations.values.transformed++; + facets_key_value_copy_to_buffer(k); + k->transform.cb(facets, k->current_value.b, FACETS_TRANSFORM_VALUE, k->transform.data); + } + +// bool found = false; +// if(strstr(buffer_tostring(k->current_value), "fprintd") != NULL) +// found = true; + + if(facets->query && !facet_key_value_empty_or_unsampled_or_estimated(k) && ((k->options & FACET_KEY_OPTION_FTS) || facets->options & FACETS_OPTION_ALL_KEYS_FTS)) { + facets->operations.fts.searches++; + facets_key_value_copy_to_buffer(k); + switch(simple_pattern_matches_extract(facets->query, buffer_tostring(k->current_value.b), NULL, 0)) { + case SP_MATCHED_POSITIVE: + facets->current_row.keys_matched_by_query_positive++; + break; + + case SP_MATCHED_NEGATIVE: + facets->current_row.keys_matched_by_query_negative++; + break; + + case SP_NOT_MATCHED: + break; + } + } + + if(k->values.enabled) + FACET_VALUE_ADD_CURRENT_VALUE_TO_INDEX(k); + else { + k->key_found_in_row++; + k->key_values_selected_in_row++; + } +} + +void facets_add_key_value(FACETS *facets, const char *key, const char *value) { + FACET_KEY *k = facets_register_key_name(facets, key, 0); + k->current_value.raw = value; + k->current_value.raw_len = strlen(value); + + facets_key_check_value(facets, k); +} + +void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len) { + FACET_KEY *k = facets_register_key_name_length(facets, key, key_len, 0); + k->current_value.raw = value; + k->current_value.raw_len = value_len; + + facets_key_check_value(facets, k); +} + +// ---------------------------------------------------------------------------- +// FACET_ROW dictionary hooks + +static void facet_row_key_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + FACET_ROW_KEY_VALUE *rkv = value; + FACET_ROW *row = data; (void)row; + + rkv->wb = buffer_create(0, NULL); + if(!rkv->empty) + buffer_contents_replace(rkv->wb, rkv->tmp, rkv->tmp_len); +} + +static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) { + FACET_ROW_KEY_VALUE *rkv = old_value; + FACET_ROW_KEY_VALUE *n_rkv = new_value; + FACET_ROW *row = data; (void)row; + + rkv->empty = n_rkv->empty; + + if(!rkv->empty) + buffer_contents_replace(rkv->wb, n_rkv->tmp, n_rkv->tmp_len); + else + buffer_flush(rkv->wb); + + return false; +} + +static void facet_row_key_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + FACET_ROW_KEY_VALUE *rkv = value; + FACET_ROW *row = data; (void)row; + + buffer_free(rkv->wb); +} + +// ---------------------------------------------------------------------------- +// FACET_ROW management + +static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) { + dictionary_destroy(row->dict); + freez(row); +} + +static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into) { + FACET_ROW *row; + + if(into) { + row = into; + facets->operations.rows.reused++; + } + else { + row = callocz(1, sizeof(FACET_ROW)); + row->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_ROW_KEY_VALUE)); + dictionary_register_insert_callback(row->dict, facet_row_key_value_insert_callback, row); + dictionary_register_conflict_callback(row->dict, facet_row_key_value_conflict_callback, row); + dictionary_register_delete_callback(row->dict, facet_row_key_value_delete_callback, row); + facets->operations.rows.created++; + } + + row->severity = facets->current_row.severity; + row->usec = usec; + + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + FACET_ROW_KEY_VALUE t = { + .tmp = NULL, + .tmp_len = 0, + .wb = NULL, + .empty = true, + }; + + if(facet_key_value_updated(k) && !facet_key_value_empty_or_unsampled_or_estimated(k)) { + t.tmp = facets_key_get_value(k); + t.tmp_len = facets_key_get_value_length(k); + t.empty = false; + } + + dictionary_set(row->dict, k->name, &t, sizeof(t)); + } + foreach_key_in_facets_done(k); + + return row; +} + +// ---------------------------------------------------------------------------- + +static inline FACET_ROW *facets_row_keep_seek_to_position(FACETS *facets, usec_t usec) { + if(usec < facets->base->prev->usec) + return facets->base->prev; + + if(usec > facets->base->usec) + return facets->base; + + FACET_ROW *last = facets->operations.last_added; + while(last->prev != facets->base->prev && usec > last->prev->usec) { + last = last->prev; + facets->operations.backwards++; + } + + while(last->next && usec < last->next->usec) { + last = last->next; + facets->operations.forwards++; + } + + return last; +} + +static void facets_row_keep_first_entry(FACETS *facets, usec_t usec) { + facets->operations.last_added = facets_row_create(facets, usec, NULL); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next); + facets->items_to_return++; + facets->operations.first++; +} + +static inline bool facets_is_entry_within_anchor(FACETS *facets, usec_t usec) { + if(facets->anchor.start_ut || facets->anchor.stop_ut) { + // we have an anchor key + // we don't want to keep rows on the other side of the direction + + switch (facets->anchor.direction) { + default: + case FACETS_ANCHOR_DIRECTION_BACKWARD: + // we need to keep only the smaller timestamps + if (facets->anchor.start_ut && usec >= facets->anchor.start_ut) { + facets->operations.skips_before++; + return false; + } + if (facets->anchor.stop_ut && usec <= facets->anchor.stop_ut) { + facets->operations.skips_after++; + return false; + } + break; + + case FACETS_ANCHOR_DIRECTION_FORWARD: + // we need to keep only the bigger timestamps + if (facets->anchor.start_ut && usec <= facets->anchor.start_ut) { + facets->operations.skips_after++; + return false; + } + if (facets->anchor.stop_ut && usec >= facets->anchor.stop_ut) { + facets->operations.skips_before++; + return false; + } + break; + } + } + + return true; +} + +bool facets_row_candidate_to_keep(FACETS *facets, usec_t usec) { + return !facets->base || + (usec >= facets->base->prev->usec && usec <= facets->base->usec && facets_is_entry_within_anchor(facets, usec)) || + facets->items_to_return < facets->max_items_to_return; +} + +static void facets_row_keep(FACETS *facets, usec_t usec) { + facets->operations.rows.matched++; + + if(unlikely(!facets->base)) { + // the first row to keep + facets_row_keep_first_entry(facets, usec); + return; + } + + FACET_ROW *closest = facets_row_keep_seek_to_position(facets, usec); + FACET_ROW *to_replace = NULL; + + if(likely(facets->items_to_return >= facets->max_items_to_return)) { + // we have enough items to return already + + switch(facets->anchor.direction) { + default: + case FACETS_ANCHOR_DIRECTION_BACKWARD: + if(closest == facets->base->prev && usec < closest->usec) { + // this is to the end of the list, belonging to the next page + facets->operations.skips_after++; + return; + } + + // it seems we need to remove an item - the last one + to_replace = facets->base->prev; + if(closest == to_replace) + closest = to_replace->prev; + + break; + + case FACETS_ANCHOR_DIRECTION_FORWARD: + if(closest == facets->base && usec > closest->usec) { + // this is to the beginning of the list, belonging to the next page + facets->operations.skips_before++; + return; + } + + // it seems we need to remove an item - the first one + to_replace = facets->base; + if(closest == to_replace) + closest = to_replace->next; + + break; + } + + facets->operations.shifts++; + facets->items_to_return--; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, to_replace, prev, next); + } + + internal_fatal(!closest, "FACETS: closest cannot be NULL"); + internal_fatal(closest == to_replace, "FACETS: closest cannot be the same as to_replace"); + + facets->operations.last_added = facets_row_create(facets, usec, to_replace); + + if(usec < closest->usec) { + DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next); + facets->operations.appends++; + } + else { + DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next); + facets->operations.prepends++; + } + + facets->items_to_return++; +} + +static inline void facets_reset_key(FACET_KEY *k) { + k->key_found_in_row = 0; + k->key_values_selected_in_row = 0; + k->current_value.flags = FACET_KEY_VALUE_NONE; + k->current_value.hash = FACETS_HASH_ZERO; +} + +static void facets_reset_keys_with_value_and_row(FACETS *facets) { + size_t entries = facets->keys_in_row.used; + + for(size_t p = 0; p < entries ;p++) { + FACET_KEY *k = facets->keys_in_row.array[p]; + facets_reset_key(k); + } + + facets->current_row.severity = FACET_ROW_SEVERITY_NORMAL; + facets->current_row.keys_matched_by_query_positive = 0; + facets->current_row.keys_matched_by_query_negative = 0; + facets->keys_in_row.used = 0; +} + +void facets_rows_begin(FACETS *facets) { + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + facets_reset_key(k); + } + foreach_key_in_facets_done(k); + + facets->keys_in_row.used = 0; + facets_reset_keys_with_value_and_row(facets); +} + +bool facets_row_finished(FACETS *facets, usec_t usec) { + facets->operations.rows.evaluated++; + + if(unlikely((facets->query && facets->keys_filtered_by_query && + (!facets->current_row.keys_matched_by_query_positive || facets->current_row.keys_matched_by_query_negative)) || + (facets->timeframe.before_ut && usec > facets->timeframe.before_ut) || + (facets->timeframe.after_ut && usec < facets->timeframe.after_ut))) { + // this row is not useful + // 1. not matched by full text search, or + // 2. not in our timeframe + facets_reset_keys_with_value_and_row(facets); + return false; + } + + bool within_anchor = facets_is_entry_within_anchor(facets, usec); + if(unlikely(!within_anchor && (facets->options & FACETS_OPTION_DATA_ONLY))) { + facets_reset_keys_with_value_and_row(facets); + return false; + } + + size_t entries = facets->keys_with_values.used; + size_t total_keys = 0; + size_t selected_keys = 0; + + for(size_t p = 0; p < entries ;p++) { + FACET_KEY *k = facets->keys_with_values.array[p]; + + if(!facet_key_value_updated(k)) { + // put the FACET_VALUE_UNSET value into it + facets_key_set_empty_value(facets, k); + } + + total_keys++; + + if(k->key_values_selected_in_row) + selected_keys++; + + if(unlikely(!facets->histogram.key && facets->histogram.hash == k->hash)) + facets->histogram.key = k; + } + + if(selected_keys >= total_keys - 1) { + size_t found = 0; + (void) found; + + for(size_t p = 0; p < entries; p++) { + FACET_KEY *k = facets->keys_with_values.array[p]; + + size_t counted_by = selected_keys; + + if(counted_by != total_keys && !k->key_values_selected_in_row) + counted_by++; + + if(counted_by == total_keys) { + k->current_value.v->final_facet_value_counter++; + found++; + } + } + + internal_fatal(!found, "We should find at least one facet to count this row"); + } + + if(selected_keys == total_keys) { + // we need to keep this row + facets_histogram_update_value(facets, usec); + + if(within_anchor) + facets_row_keep(facets, usec); + } + + facets_reset_keys_with_value_and_row(facets); + + return selected_keys == total_keys; +} + +// ---------------------------------------------------------------------------- +// output + +const char *facets_severity_to_string(FACET_ROW_SEVERITY severity) { + switch(severity) { + default: + case FACET_ROW_SEVERITY_NORMAL: + return "normal"; + + case FACET_ROW_SEVERITY_DEBUG: + return "debug"; + + case FACET_ROW_SEVERITY_NOTICE: + return "notice"; + + case FACET_ROW_SEVERITY_WARNING: + return "warning"; + + case FACET_ROW_SEVERITY_CRITICAL: + return "critical"; + } +} + +void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool with_keys) { + buffer_json_member_add_array(wb, "accepted_params"); + { + if(facets->accepted_params) { + void *t; + dfe_start_read(facets->accepted_params, t) { + buffer_json_add_array_item_string(wb, t_dfe.name); + } + dfe_done(t); + } + + if(with_keys) { + FACET_KEY *k; + foreach_key_in_facets(facets, k){ + if (!k->values.enabled) + continue; + + buffer_json_add_array_item_string(wb, hash_to_static_string(k->hash)); + } + foreach_key_in_facets_done(k); + } + } + buffer_json_array_close(wb); // accepted_params +} + +static int facets_keys_reorder_compar(const void *a, const void *b) { + const FACET_KEY *ak = *((const FACET_KEY **)a); + const FACET_KEY *bk = *((const FACET_KEY **)b); + + const char *an = ak->name; + const char *bn = bk->name; + + if(!an) an = "0"; + if(!bn) bn = "0"; + + while(*an && ispunct(*an)) an++; + while(*bn && ispunct(*bn)) bn++; + + return strcasecmp(an, bn); +} + +void facets_sort_and_reorder_keys(FACETS *facets) { + size_t entries = facets->keys_with_values.used; + if(!entries) + return; + + FACET_KEY *keys[entries]; + memcpy(keys, facets->keys_with_values.array, sizeof(FACET_KEY *) * entries); + + qsort(keys, entries, sizeof(FACET_KEY *), facets_keys_reorder_compar); + + for(size_t i = 0; i < entries ;i++) + keys[i]->order = i + 1; +} + +static int facets_key_values_reorder_by_name_compar(const void *a, const void *b) { + const FACET_VALUE *av = *((const FACET_VALUE **)a); + const FACET_VALUE *bv = *((const FACET_VALUE **)b); + + const char *an = (av->name && av->name_len) ? av->name : "0"; + const char *bn = (bv->name && bv->name_len) ? bv->name : "0"; + + while(*an && ispunct(*an)) an++; + while(*bn && ispunct(*bn)) bn++; + + int ret = strcasecmp(an, bn); + return ret; +} + +static int facets_key_values_reorder_by_count_compar(const void *a, const void *b) { + const FACET_VALUE *av = *((const FACET_VALUE **)a); + const FACET_VALUE *bv = *((const FACET_VALUE **)b); + + if(av->final_facet_value_counter < bv->final_facet_value_counter) + return 1; + + if(av->final_facet_value_counter > bv->final_facet_value_counter) + return -1; + + return facets_key_values_reorder_by_name_compar(a, b); +} + +static int facets_key_values_reorder_by_name_numeric_compar(const void *a, const void *b) { + const FACET_VALUE *av = *((const FACET_VALUE **)a); + const FACET_VALUE *bv = *((const FACET_VALUE **)b); + + const char *an = (av->name && av->name_len) ? av->name : "0"; + const char *bn = (bv->name && bv->name_len) ? bv->name : "0"; + + if(strcmp(an, FACET_VALUE_UNSET) == 0) an = "0"; + if(strcmp(bn, FACET_VALUE_UNSET) == 0) bn = "0"; + + int64_t ad = str2ll(an, NULL); + int64_t bd = str2ll(bn, NULL); + + if(ad < bd) + return -1; + + if(ad > bd) + return 1; + + return facets_key_values_reorder_by_name_compar(a, b); +} + +static uint32_t facets_sort_and_reorder_values_internal(FACET_KEY *k) { + bool all_values_numeric = true; + size_t entries = k->values.used; + FACET_VALUE *values[entries], *v; + uint32_t used = 0; + foreach_value_in_key(k, v) { + if((k->facets->options & FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS) && v->empty) + continue; + + if(all_values_numeric && !v->empty && v->name && v->name_len) { + const char *s = v->name; + while(isdigit(*s)) s++; + if(*s != '\0') + all_values_numeric = false; + } + + values[used++] = v; + + if(used >= entries) + break; + } + foreach_value_in_key_done(v); + + if(!used) + return 0; + + if(k->facets->options & FACETS_OPTION_SORT_FACETS_ALPHABETICALLY) { + if(all_values_numeric) + qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_name_numeric_compar); + else + qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_name_compar); + } + else + qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_count_compar); + + for(size_t i = 0; i < used; i++) + values[i]->order = i + 1; + + return used; +} + +static uint32_t facets_sort_and_reorder_values(FACET_KEY *k) { + if(!k->values.enabled || !k->values.ll || !k->values.used) + return 0; + + if(!k->transform.cb || !k->transform.view_only || !(k->facets->options & FACETS_OPTION_SORT_FACETS_ALPHABETICALLY)) + return facets_sort_and_reorder_values_internal(k); + + // we have a transformation and has to be sorted alphabetically + + BUFFER *tb = buffer_create(0, NULL); + uint32_t ret = 0; + + size_t entries = k->values.used; + struct { + const char *name; + uint32_t name_len; + } values[entries]; + FACET_VALUE *v; + uint32_t used = 0; + + foreach_value_in_key(k, v) { + if(used >= entries) + break; + + values[used].name = v->name; + values[used].name_len = v->name_len; + used++; + + facets_key_value_transformed(k->facets, k, v, tb, FACETS_TRANSFORM_FACET_SORT); + v->name = strdupz(buffer_tostring(tb)); + v->name_len = buffer_strlen(tb); + } + foreach_value_in_key_done(v); + + ret = facets_sort_and_reorder_values_internal(k); + + used = 0; + foreach_value_in_key(k, v) { + if(used >= entries) + break; + + freez((void *)v->name); + v->name = values[used].name; + v->name_len = values[used].name_len; + used++; + } + foreach_value_in_key_done(v); + + buffer_free(tb); + return ret; +} + +void facets_table_config(BUFFER *wb) { + buffer_json_member_add_boolean(wb, "show_ids", false); // do not show the column ids to the user + buffer_json_member_add_boolean(wb, "has_history", true); // enable date-time picker with after-before + + buffer_json_member_add_object(wb, "pagination"); + { + buffer_json_member_add_boolean(wb, "enabled", true); + buffer_json_member_add_string(wb, "key", "anchor"); + buffer_json_member_add_string(wb, "column", "timestamp"); + buffer_json_member_add_string(wb, "units", "timestamp_usec"); + } + buffer_json_object_close(wb); // pagination +} + +void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) { + facets->report.used_hashes_registry = used_hashes_registry; + + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) { + facets_table_config(wb); + facets_accepted_parameters_to_json_array(facets, wb, true); + } + + // ------------------------------------------------------------------------ + // facets + + if(!(facets->options & FACETS_OPTION_DONT_SEND_FACETS)) { + bool show_facets = false; + + if(facets->options & FACETS_OPTION_DATA_ONLY) { + if(facets->options & FACETS_OPTION_SHOW_DELTAS) { + buffer_json_member_add_array(wb, "facets_delta"); + show_facets = true; + } + } + else { + buffer_json_member_add_array(wb, "facets"); + show_facets = true; + } + + if(show_facets) { + CLEAN_BUFFER *tb = buffer_create(0, NULL); + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + if(!k->values.enabled) + continue; + + if(!facets_sort_and_reorder_values(k)) + // no values for this key + continue; + + buffer_json_add_array_item_object(wb); // key + { + buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash)); + buffer_json_member_add_string(wb, "name", facets_key_name_cached(k + , facets->report.used_hashes_registry + )); + + if(!k->order) k->order = facets->order++; + buffer_json_member_add_uint64(wb, "order", k->order); + + buffer_json_member_add_array(wb, "options"); + { + FACET_VALUE *v; + foreach_value_in_key(k, v) { + if((facets->options & FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS) && v->empty) + continue; + + if(v->unsampled || v->estimated) + continue; + + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "id", hash_to_static_string(v->hash)); + + facets_key_value_transformed(facets, k, v, tb, FACETS_TRANSFORM_FACET); + buffer_json_member_add_string(wb, "name", buffer_tostring(tb)); + buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter); + buffer_json_member_add_uint64(wb, "order", v->order); + } + buffer_json_object_close(wb); + } + foreach_value_in_key_done(v); + } + buffer_json_array_close(wb); // options + } + buffer_json_object_close(wb); // key + } + foreach_key_in_facets_done(k); + buffer_json_array_close(wb); // facets + } + } + + // ------------------------------------------------------------------------ + // columns + + buffer_json_member_add_object(wb, "columns"); + { + size_t field_id = 0; + buffer_rrdf_table_add_field( + wb, field_id++, + "timestamp", "Timestamp", + RRDF_FIELD_TYPE_TIMESTAMP, + RRDF_FIELD_VISUAL_VALUE, + RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN, + RRDF_FIELD_SORT_DESCENDING|RRDF_FIELD_SORT_FIXED, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + RRDF_FIELD_FILTER_RANGE, + RRDF_FIELD_OPTS_WRAP | RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY, + NULL); + + buffer_rrdf_table_add_field( + wb, field_id++, + "rowOptions", "rowOptions", + RRDF_FIELD_TYPE_NONE, + RRDR_FIELD_VISUAL_ROW_OPTIONS, + RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, + RRDF_FIELD_SORT_FIXED, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + RRDF_FIELD_FILTER_NONE, + RRDF_FIELD_OPTS_DUMMY, + NULL); + + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP; + bool visible = k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY); + + if ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values.enabled)) + visible = true; + + if (!visible) + visible = simple_pattern_matches(facets->visible_keys, k->name); + + if (visible) + options |= RRDF_FIELD_OPTS_VISIBLE; + + if (k->options & FACET_KEY_OPTION_MAIN_TEXT) + options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP; + + if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER) + options |= RRDF_FIELD_OPTS_EXPANDED_FILTER; + + const char *hash_str = hash_to_static_string(k->hash); + + buffer_rrdf_table_add_field( + wb, field_id++, + hash_str, k->name ? k->name : hash_str, + RRDF_FIELD_TYPE_STRING, + (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE, + RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, + RRDF_FIELD_SORT_FIXED, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET, + options, FACET_VALUE_UNSET); + } + foreach_key_in_facets_done(k); + } + buffer_json_object_close(wb); // columns + + // ------------------------------------------------------------------------ + // rows data + + buffer_json_member_add_array(wb, "data"); + { + usec_t last_usec = 0; (void)last_usec; + + for(FACET_ROW *row = facets->base ; row ;row = row->next) { + + internal_fatal( + facets->anchor.start_ut && ( + (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_BACKWARD && row->usec >= facets->anchor.start_ut) || + (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD && row->usec <= facets->anchor.start_ut) + ), "Wrong data returned related to %s start anchor!", facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward"); + + internal_fatal(last_usec && row->usec > last_usec, "Wrong order of data returned!"); + + last_usec = row->usec; + + buffer_json_add_array_item_array(wb); // each row + buffer_json_add_array_item_uint64(wb, row->usec); + buffer_json_add_array_item_object(wb); + { + if(facets->severity.cb) + row->severity = facets->severity.cb(facets, row, facets->severity.data); + + buffer_json_member_add_string(wb, "severity", facets_severity_to_string(row->severity)); + } + buffer_json_object_close(wb); + + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name); + + if(unlikely(k->dynamic.cb)) { + if(unlikely(!rkv)) + rkv = dictionary_set(row->dict, k->name, NULL, sizeof(*rkv)); + + k->dynamic.cb(facets, wb, rkv, row, k->dynamic.data); + facets->operations.values.dynamic++; + } + else { + if(!rkv || rkv->empty) { + buffer_json_add_array_item_string(wb, NULL); + } + else if(unlikely(k->transform.cb && k->transform.view_only)) { + k->transform.cb(facets, rkv->wb, FACETS_TRANSFORM_DATA, k->transform.data); + buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb)); + } + else + buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb)); + } + } + foreach_key_in_facets_done(k); + buffer_json_array_close(wb); // each row + } + } + buffer_json_array_close(wb); // data + + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) { + buffer_json_member_add_string(wb, "default_sort_column", "timestamp"); + buffer_json_member_add_array(wb, "default_charts"); + buffer_json_array_close(wb); + } + + // ------------------------------------------------------------------------ + // histogram + + if(facets->histogram.enabled && !(facets->options & FACETS_OPTION_DONT_SEND_HISTOGRAM)) { + FACETS_HASH first_histogram_hash = 0; + buffer_json_member_add_array(wb, "available_histograms"); + { + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + if (!k->values.enabled) + continue; + + if(unlikely(!first_histogram_hash)) + first_histogram_hash = k->hash; + + buffer_json_add_array_item_object(wb); + buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash)); + buffer_json_member_add_string(wb, "name", k->name); + buffer_json_member_add_uint64(wb, "order", k->order); + buffer_json_object_close(wb); + } + foreach_key_in_facets_done(k); + } + buffer_json_array_close(wb); + + { + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, facets->histogram.hash); + if(!k || !k->values.enabled) + k = FACETS_KEY_GET_FROM_INDEX(facets, first_histogram_hash); + + bool show_histogram = false; + + if(facets->options & FACETS_OPTION_DATA_ONLY) { + if(facets->options & FACETS_OPTION_SHOW_DELTAS) { + buffer_json_member_add_object(wb, "histogram_delta"); + show_histogram = true; + } + } + else { + buffer_json_member_add_object(wb, "histogram"); + show_histogram = true; + } + + if(show_histogram) { + buffer_json_member_add_string(wb, "id", k ? hash_to_static_string(k->hash) : ""); + buffer_json_member_add_string(wb, "name", k ? k->name : ""); + buffer_json_member_add_object(wb, "chart"); + { + facets_histogram_generate(facets, k, wb); + } + buffer_json_object_close(wb); // chart + buffer_json_object_close(wb); // histogram + } + } + } + + // ------------------------------------------------------------------------ + // items + + bool show_items = false; + if(facets->options & FACETS_OPTION_DATA_ONLY) { + if(facets->options & FACETS_OPTION_SHOW_DELTAS) { + buffer_json_member_add_object(wb, "items_delta"); + show_items = true; + } + } + else { + buffer_json_member_add_object(wb, "items"); + show_items = true; + } + + if(show_items) { + buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated); + buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched); + buffer_json_member_add_uint64(wb, "unsampled", facets->operations.rows.unsampled); + buffer_json_member_add_uint64(wb, "estimated", facets->operations.rows.estimated); + buffer_json_member_add_uint64(wb, "returned", facets->items_to_return); + buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return); + buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before); + buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts); + buffer_json_object_close(wb); // items + } + + // ------------------------------------------------------------------------ + // stats + + buffer_json_member_add_object(wb, "_stats"); + { + buffer_json_member_add_uint64(wb, "first", facets->operations.first); + buffer_json_member_add_uint64(wb, "forwards", facets->operations.forwards); + buffer_json_member_add_uint64(wb, "backwards", facets->operations.backwards); + buffer_json_member_add_uint64(wb, "skips_before", facets->operations.skips_before); + buffer_json_member_add_uint64(wb, "skips_after", facets->operations.skips_after); + buffer_json_member_add_uint64(wb, "prepends", facets->operations.prepends); + buffer_json_member_add_uint64(wb, "appends", facets->operations.appends); + buffer_json_member_add_uint64(wb, "shifts", facets->operations.shifts); + buffer_json_member_add_object(wb, "rows"); + { + buffer_json_member_add_uint64(wb, "created", facets->operations.rows.created); + buffer_json_member_add_uint64(wb, "reused", facets->operations.rows.reused); + buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated); + buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched); + } + buffer_json_object_close(wb); // rows + buffer_json_member_add_object(wb, "keys"); + { + size_t resizes = 0, searches = 0, collisions = 0, used = 0, size = 0, count = 0; + count++; + used += facets->keys.ht.used; + size += facets->keys.ht.size; + resizes += facets->keys.ht.resizes; + searches += facets->keys.ht.searches; + collisions += facets->keys.ht.collisions; + + buffer_json_member_add_uint64(wb, "registered", facets->operations.keys.registered); + buffer_json_member_add_uint64(wb, "unique", facets->operations.keys.unique); + buffer_json_member_add_uint64(wb, "hashtables", count); + buffer_json_member_add_uint64(wb, "hashtable_used", used); + buffer_json_member_add_uint64(wb, "hashtable_size", size); + buffer_json_member_add_uint64(wb, "hashtable_searches", searches); + buffer_json_member_add_uint64(wb, "hashtable_collisions", collisions); + buffer_json_member_add_uint64(wb, "hashtable_resizes", resizes); + } + buffer_json_object_close(wb); // keys + buffer_json_member_add_object(wb, "values"); + { + size_t resizes = 0, searches = 0, collisions = 0, used = 0, size = 0, count = 0; + for(FACET_KEY *k = facets->keys.ll; k ; k = k->next) { + count++; + used += k->values.ht.used; + size += k->values.ht.size; + resizes += k->values.ht.resizes; + searches += k->values.ht.searches; + collisions += k->values.ht.collisions; + } + + buffer_json_member_add_uint64(wb, "registered", facets->operations.values.registered); + buffer_json_member_add_uint64(wb, "transformed", facets->operations.values.transformed); + buffer_json_member_add_uint64(wb, "dynamic", facets->operations.values.dynamic); + buffer_json_member_add_uint64(wb, "empty", facets->operations.values.empty); + buffer_json_member_add_uint64(wb, "unsampled", facets->operations.values.unsampled); + buffer_json_member_add_uint64(wb, "estimated", facets->operations.values.estimated); + buffer_json_member_add_uint64(wb, "indexed", facets->operations.values.indexed); + buffer_json_member_add_uint64(wb, "inserts", facets->operations.values.inserts); + buffer_json_member_add_uint64(wb, "conflicts", facets->operations.values.conflicts); + buffer_json_member_add_uint64(wb, "hashtables", count); + buffer_json_member_add_uint64(wb, "hashtable_used", used); + buffer_json_member_add_uint64(wb, "hashtable_size", size); + buffer_json_member_add_uint64(wb, "hashtable_searches", searches); + buffer_json_member_add_uint64(wb, "hashtable_collisions", collisions); + buffer_json_member_add_uint64(wb, "hashtable_resizes", resizes); + } + buffer_json_object_close(wb); // values + buffer_json_member_add_object(wb, "fts"); + { + buffer_json_member_add_uint64(wb, "searches", facets->operations.fts.searches); + } + buffer_json_object_close(wb); // fts + } + buffer_json_object_close(wb); // items +} |