summaryrefslogtreecommitdiffstats
path: root/libnetdata/facets
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 02:57:58 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 02:57:58 +0000
commitbe1c7e50e1e8809ea56f2c9d472eccd8ffd73a97 (patch)
tree9754ff1ca740f6346cf8483ec915d4054bc5da2d /libnetdata/facets
parentInitial commit. (diff)
downloadnetdata-upstream.tar.xz
netdata-upstream.zip
Adding upstream version 1.44.3.upstream/1.44.3upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--libnetdata/facets/Makefile.am8
-rw-r--r--libnetdata/facets/README.md0
-rw-r--r--libnetdata/facets/facets.c2787
-rw-r--r--libnetdata/facets/facets.h128
4 files changed, 2923 insertions, 0 deletions
diff --git a/libnetdata/facets/Makefile.am b/libnetdata/facets/Makefile.am
new file mode 100644
index 00000000..161784b8
--- /dev/null
+++ b/libnetdata/facets/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/libnetdata/facets/README.md b/libnetdata/facets/README.md
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/libnetdata/facets/README.md
diff --git a/libnetdata/facets/facets.c b/libnetdata/facets/facets.c
new file mode 100644
index 00000000..4a5f5442
--- /dev/null
+++ b/libnetdata/facets/facets.c
@@ -0,0 +1,2787 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#include "facets.h"
+
+#define FACETS_HISTOGRAM_COLUMNS 150 // the target number of points in a histogram
+#define FACETS_KEYS_WITH_VALUES_MAX 200 // the max number of keys that can be facets
+#define FACETS_KEYS_IN_ROW_MAX 500 // the max number of keys in a row
+
+#define FACETS_KEYS_HASHTABLE_ENTRIES 15
+#define FACETS_VALUES_HASHTABLE_ENTRIES 15
+
+static inline void facets_reset_key(FACET_KEY *k);
+
+// ----------------------------------------------------------------------------
+
+static const char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789";
+static const uint8_t id_encoding_characters_reverse[256] = {
+ ['A'] = 0, ['B'] = 1, ['C'] = 2, ['D'] = 3,
+ ['E'] = 4, ['F'] = 5, ['G'] = 6, ['H'] = 7,
+ ['I'] = 8, ['J'] = 9, ['K'] = 10, ['L'] = 11,
+ ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15,
+ ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19,
+ ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23,
+ ['Y'] = 24, ['Z'] = 25, ['.'] = 26, ['a'] = 27,
+ ['b'] = 28, ['c'] = 29, ['d'] = 30, ['e'] = 31,
+ ['f'] = 32, ['g'] = 33, ['h'] = 34, ['i'] = 35,
+ ['j'] = 36, ['k'] = 37, ['l'] = 38, ['m'] = 39,
+ ['n'] = 40, ['o'] = 41, ['p'] = 42, ['q'] = 43,
+ ['r'] = 44, ['s'] = 45, ['t'] = 46, ['u'] = 47,
+ ['v'] = 48, ['w'] = 49, ['x'] = 50, ['y'] = 51,
+ ['z'] = 52, ['_'] = 53, ['0'] = 54, ['1'] = 55,
+ ['2'] = 56, ['3'] = 57, ['4'] = 58, ['5'] = 59,
+ ['6'] = 60, ['7'] = 61, ['8'] = 62, ['9'] = 63
+};
+
+#define FACET_STRING_HASH_SIZE 12
+#define FACETS_HASH XXH64_hash_t
+#define FACETS_HASH_FUNCTION(src, len) XXH3_64bits(src, len)
+#define FACETS_HASH_ZERO (FACETS_HASH)0
+#define FACETS_HASH_UNSAMPLED (FACETS_HASH)(UINT64_MAX - 1)
+#define FACETS_HASH_ESTIMATED (FACETS_HASH)UINT64_MAX
+
+static inline void facets_hash_to_str(FACETS_HASH num, char *out) {
+ out[11] = '\0';
+ out[10] = id_encoding_characters[num & 63]; num >>= 6;
+ out[9] = id_encoding_characters[num & 63]; num >>= 6;
+ out[8] = id_encoding_characters[num & 63]; num >>= 6;
+ out[7] = id_encoding_characters[num & 63]; num >>= 6;
+ out[6] = id_encoding_characters[num & 63]; num >>= 6;
+ out[5] = id_encoding_characters[num & 63]; num >>= 6;
+ out[4] = id_encoding_characters[num & 63]; num >>= 6;
+ out[3] = id_encoding_characters[num & 63]; num >>= 6;
+ out[2] = id_encoding_characters[num & 63]; num >>= 6;
+ out[1] = id_encoding_characters[num & 63]; num >>= 6;
+ out[0] = id_encoding_characters[num & 63];
+}
+
+static inline FACETS_HASH str_to_facets_hash(const char *str) {
+ FACETS_HASH num = 0;
+ int shifts = 6 * (FACET_STRING_HASH_SIZE - 2);
+
+ num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[0])])) << shifts; shifts -= 6;
+ num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[1])])) << shifts; shifts -= 6;
+ num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[2])])) << shifts; shifts -= 6;
+ num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[3])])) << shifts; shifts -= 6;
+ num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[4])])) << shifts; shifts -= 6;
+ num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[5])])) << shifts; shifts -= 6;
+ num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[6])])) << shifts; shifts -= 6;
+ num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[7])])) << shifts; shifts -= 6;
+ num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[8])])) << shifts; shifts -= 6;
+ num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[9])])) << shifts; shifts -= 6;
+ num |= ((FACETS_HASH)(id_encoding_characters_reverse[(uint8_t)(str[10])])) << shifts;
+
+ return num;
+}
+
+static const char *hash_to_static_string(FACETS_HASH hash) {
+ static __thread char hash_str[FACET_STRING_HASH_SIZE];
+ facets_hash_to_str(hash, hash_str);
+ return hash_str;
+}
+
+static inline bool is_valid_string_hash(const char *s) {
+ if(strlen(s) != FACET_STRING_HASH_SIZE - 1) {
+ netdata_log_error("The user supplied key '%s' does not have the right length for a facets hash.", s);
+ return false;
+ }
+
+ uint8_t *t = (uint8_t *)s;
+ while(*t) {
+ if(id_encoding_characters_reverse[*t] == 0 && *t != id_encoding_characters[0]) {
+ netdata_log_error("The user supplied key '%s' contains invalid characters for a facets hash.", s);
+ return false;
+ }
+
+ t++;
+ }
+
+ return true;
+}
+
+// ----------------------------------------------------------------------------
+// hashtable for FACET_VALUE
+
+// cleanup hashtable defines
+#undef SIMPLE_HASHTABLE_SORT_FUNCTION
+#undef SIMPLE_HASHTABLE_VALUE_TYPE
+#undef SIMPLE_HASHTABLE_NAME
+#undef NETDATA_SIMPLE_HASHTABLE_H
+
+struct facet_value;
+// #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_value
+#define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_value
+#define SIMPLE_HASHTABLE_NAME _VALUE
+#include "../simple_hashtable.h"
+
+// ----------------------------------------------------------------------------
+// hashtable for FACET_KEY
+
+// cleanup hashtable defines
+#undef SIMPLE_HASHTABLE_SORT_FUNCTION
+#undef SIMPLE_HASHTABLE_VALUE_TYPE
+#undef SIMPLE_HASHTABLE_NAME
+#undef NETDATA_SIMPLE_HASHTABLE_H
+
+struct facet_key;
+// #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_key
+#define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_key
+#define SIMPLE_HASHTABLE_NAME _KEY
+#include "../simple_hashtable.h"
+
+// ----------------------------------------------------------------------------
+
+typedef struct facet_value {
+ FACETS_HASH hash;
+ const char *name;
+ const char *color;
+ uint32_t name_len;
+
+ bool selected;
+ bool empty;
+ bool unsampled;
+ bool estimated;
+
+ uint32_t rows_matching_facet_value;
+ uint32_t final_facet_value_counter;
+ uint32_t order;
+
+ uint32_t *histogram;
+ uint32_t min, max, sum;
+
+ struct facet_value *prev, *next;
+} FACET_VALUE;
+
+typedef enum {
+ FACET_KEY_VALUE_NONE = 0,
+ FACET_KEY_VALUE_UPDATED = (1 << 0),
+ FACET_KEY_VALUE_EMPTY = (1 << 1),
+ FACET_KEY_VALUE_UNSAMPLED = (1 << 2),
+ FACET_KEY_VALUE_ESTIMATED = (1 << 3),
+ FACET_KEY_VALUE_COPIED = (1 << 4),
+} FACET_KEY_VALUE_FLAGS;
+
+#define facet_key_value_updated(k) ((k)->current_value.flags & FACET_KEY_VALUE_UPDATED)
+#define facet_key_value_empty(k) ((k)->current_value.flags & FACET_KEY_VALUE_EMPTY)
+#define facet_key_value_unsampled(k) ((k)->current_value.flags & FACET_KEY_VALUE_UNSAMPLED)
+#define facet_key_value_estimated(k) ((k)->current_value.flags & FACET_KEY_VALUE_ESTIMATED)
+#define facet_key_value_empty_or_unsampled_or_estimated(k) ((k)->current_value.flags & (FACET_KEY_VALUE_EMPTY|FACET_KEY_VALUE_UNSAMPLED|FACET_KEY_VALUE_ESTIMATED))
+#define facet_key_value_copied(k) ((k)->current_value.flags & FACET_KEY_VALUE_COPIED)
+
+struct facet_key {
+ FACETS *facets;
+
+ FACETS_HASH hash;
+ const char *name;
+
+ FACET_KEY_OPTIONS options;
+
+ bool default_selected_for_values; // the default "selected" for all values in the dictionary
+
+ // members about the current row
+ uint32_t key_found_in_row;
+ uint32_t key_values_selected_in_row;
+ uint32_t order;
+
+ struct {
+ bool enabled;
+ uint32_t used;
+ FACET_VALUE *ll;
+ SIMPLE_HASHTABLE_VALUE ht;
+ } values;
+
+ struct {
+ FACETS_HASH hash;
+ FACET_KEY_VALUE_FLAGS flags;
+ const char *raw;
+ uint32_t raw_len;
+ BUFFER *b;
+ FACET_VALUE *v;
+ } current_value;
+
+ struct {
+ FACET_VALUE *v;
+ } empty_value;
+
+ struct {
+ FACET_VALUE *v;
+ } unsampled_value;
+
+ struct {
+ FACET_VALUE *v;
+ } estimated_value;
+
+ struct {
+ facet_dynamic_row_t cb;
+ void *data;
+ } dynamic;
+
+ struct {
+ bool view_only;
+ facets_key_transformer_t cb;
+ void *data;
+ } transform;
+
+ struct facet_key *prev, *next;
+};
+
+struct facets {
+ SIMPLE_PATTERN *visible_keys;
+ SIMPLE_PATTERN *excluded_keys;
+ SIMPLE_PATTERN *included_keys;
+
+ FACETS_OPTIONS options;
+
+ struct {
+ usec_t start_ut;
+ usec_t stop_ut;
+ FACETS_ANCHOR_DIRECTION direction;
+ } anchor;
+
+ SIMPLE_PATTERN *query; // the full text search pattern
+ size_t keys_filtered_by_query; // the number of fields we do full text search (constant)
+
+ DICTIONARY *accepted_params;
+
+ struct {
+ size_t count;
+ FACET_KEY *ll;
+ SIMPLE_HASHTABLE_KEY ht;
+ } keys;
+
+ struct {
+ // this is like a stack, of the keys that are used as facets
+ size_t used;
+ FACET_KEY *array[FACETS_KEYS_WITH_VALUES_MAX];
+ } keys_with_values;
+
+ struct {
+ // this is like a stack, of the keys that need to clean up between each row
+ size_t used;
+ FACET_KEY *array[FACETS_KEYS_IN_ROW_MAX];
+ } keys_in_row;
+
+ FACET_ROW *base; // double linked list of the selected facets rows
+
+ uint32_t items_to_return;
+ uint32_t max_items_to_return;
+ uint32_t order;
+
+ struct {
+ FACET_ROW_SEVERITY severity;
+ size_t keys_matched_by_query_positive; // the number of fields matched the full text search (per row)
+ size_t keys_matched_by_query_negative; // the number of fields matched the full text search (per row)
+ } current_row;
+
+ struct {
+ usec_t after_ut;
+ usec_t before_ut;
+ } timeframe;
+
+ struct {
+ FACET_KEY *key;
+ FACETS_HASH hash;
+ char *chart;
+ bool enabled;
+ uint32_t slots;
+ usec_t slot_width_ut;
+ usec_t after_ut;
+ usec_t before_ut;
+ } histogram;
+
+ struct {
+ facet_row_severity_t cb;
+ void *data;
+ } severity;
+
+ struct {
+ FACET_ROW *last_added;
+
+ size_t first;
+ size_t forwards;
+ size_t backwards;
+ size_t skips_before;
+ size_t skips_after;
+ size_t prepends;
+ size_t appends;
+ size_t shifts;
+
+ struct {
+ size_t evaluated;
+ size_t matched;
+ size_t unsampled;
+ size_t estimated;
+ size_t created;
+ size_t reused;
+ } rows;
+
+ struct {
+ size_t registered;
+ size_t unique;
+ } keys;
+
+ struct {
+ size_t registered;
+ size_t transformed;
+ size_t dynamic;
+ size_t empty;
+ size_t unsampled;
+ size_t estimated;
+ size_t indexed;
+ size_t inserts;
+ size_t conflicts;
+ } values;
+
+ struct {
+ size_t searches;
+ } fts;
+ } operations;
+
+ struct {
+ DICTIONARY *used_hashes_registry;
+ } report;
+};
+
+usec_t facets_row_oldest_ut(FACETS *facets) {
+ if(facets->base)
+ return facets->base->prev->usec;
+
+ return 0;
+}
+
+usec_t facets_row_newest_ut(FACETS *facets) {
+ if(facets->base)
+ return facets->base->usec;
+
+ return 0;
+}
+
+uint32_t facets_rows(FACETS *facets) {
+ return facets->items_to_return;
+}
+
+// ----------------------------------------------------------------------------
+
+static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row);
+static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v);
+static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k);
+
+// ----------------------------------------------------------------------------
+// The FACET_VALUE index within each FACET_KEY
+
+#define foreach_value_in_key(k, v) \
+ for((v) = (k)->values.ll; (v) ;(v) = (v)->next)
+
+#define foreach_value_in_key_done(v) do { ; } while(0)
+
+static inline void FACETS_VALUES_INDEX_CREATE(FACET_KEY *k) {
+ k->values.ll = NULL;
+ k->values.used = 0;
+ simple_hashtable_init_VALUE(&k->values.ht, FACETS_VALUES_HASHTABLE_ENTRIES);
+}
+
+static inline void FACETS_VALUES_INDEX_DESTROY(FACET_KEY *k) {
+ FACET_VALUE *v = k->values.ll;
+ while(v) {
+ FACET_VALUE *next = v->next;
+ freez(v->histogram);
+ freez((void *)v->name);
+ freez(v);
+ v = next;
+ }
+ k->values.ll = NULL;
+ k->values.used = 0;
+ k->values.enabled = false;
+
+ simple_hashtable_destroy_VALUE(&k->values.ht);
+}
+
+static inline const char *facets_key_get_value(FACET_KEY *k) {
+ return facet_key_value_copied(k) ? buffer_tostring(k->current_value.b) : k->current_value.raw;
+}
+
+static inline uint32_t facets_key_get_value_length(FACET_KEY *k) {
+ return facet_key_value_copied(k) ? buffer_strlen(k->current_value.b) : k->current_value.raw_len;
+}
+
+static inline void facets_key_value_copy_to_buffer(FACET_KEY *k) {
+ if(!facet_key_value_copied(k)) {
+ buffer_contents_replace(k->current_value.b, k->current_value.raw, k->current_value.raw_len);
+ k->current_value.flags |= FACET_KEY_VALUE_COPIED;
+ }
+}
+
+static const char *facets_value_dup(const char *s, uint32_t len) {
+ char *d = mallocz(len + 1);
+
+ if(len)
+ memcpy(d, s, len);
+
+ d[len] = '\0';
+
+ return d;
+}
+
+static inline void FACET_VALUE_ADD_CONFLICT(FACET_KEY *k, FACET_VALUE *v, const FACET_VALUE * const nv) {
+ if(!v->name && !v->name_len && nv->name && nv->name_len) {
+ // an actual value, not a filter
+ v->name = facets_value_dup(nv->name, nv->name_len);
+ v->name_len = nv->name_len;
+ }
+
+ if(v->name && v->name_len)
+ facet_value_is_used(k, v);
+
+ internal_fatal(v->name && nv->name && v->name_len == nv->name_len && memcmp(v->name, nv->name, v->name_len) != 0,
+ "value hash conflict: '%s' and '%s' have the same hash '%s'",
+ v->name, nv->name, hash_to_static_string(v->hash));
+
+ k->facets->operations.values.conflicts++;
+}
+
+static inline FACET_VALUE *FACET_VALUE_GET_FROM_INDEX(FACET_KEY *k, FACETS_HASH hash) {
+ SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, hash, true);
+ return SIMPLE_HASHTABLE_SLOT_DATA(slot);
+}
+
+static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VALUE * const tv) {
+ SIMPLE_HASHTABLE_SLOT_VALUE *slot = simple_hashtable_get_slot_VALUE(&k->values.ht, tv->hash, true);
+
+ if(SIMPLE_HASHTABLE_SLOT_DATA(slot)) {
+ // already exists
+
+ FACET_VALUE *v = SIMPLE_HASHTABLE_SLOT_DATA(slot);
+ FACET_VALUE_ADD_CONFLICT(k, v, tv);
+ return v;
+ }
+
+ // we have to add it
+
+ FACET_VALUE *v = mallocz(sizeof(*v));
+ simple_hashtable_set_slot_VALUE(&k->values.ht, slot, tv->hash, v);
+
+ memcpy(v, tv, sizeof(*v));
+
+ if(v->estimated || v->unsampled) {
+ if(k->values.ll && k->values.ll->estimated) {
+ FACET_VALUE *estimated = k->values.ll;
+ DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(k->values.ll, estimated, v, prev, next);
+ }
+ else
+ DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(k->values.ll, v, prev, next);
+ }
+ else
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(k->values.ll, v, prev, next);
+
+ k->values.used++;
+
+ if(!v->selected)
+ v->selected = k->default_selected_for_values;
+
+ if(v->name && v->name_len) {
+ // an actual value, not a filter
+ v->name = facets_value_dup(v->name, v->name_len);
+ facet_value_is_used(k, v);
+ }
+ else {
+ v->name = NULL;
+ v->name_len = 0;
+ }
+
+ k->facets->operations.values.inserts++;
+
+ return v;
+}
+
+static inline void FACET_VALUE_ADD_UNSAMPLED_VALUE_TO_INDEX(FACET_KEY *k) {
+ static const FACET_VALUE tv = {
+ .hash = FACETS_HASH_UNSAMPLED,
+ .name = FACET_VALUE_UNSAMPLED,
+ .name_len = sizeof(FACET_VALUE_UNSAMPLED) - 1,
+ .unsampled = true,
+ .color = "offline",
+ };
+
+ k->current_value.hash = FACETS_HASH_UNSAMPLED;
+
+ if(k->unsampled_value.v) {
+ FACET_VALUE_ADD_CONFLICT(k, k->unsampled_value.v, &tv);
+ k->current_value.v = k->unsampled_value.v;
+ }
+ else {
+ FACET_VALUE *v = FACET_VALUE_ADD_TO_INDEX(k, &tv);
+ v->unsampled = true;
+ k->unsampled_value.v = v;
+ k->current_value.v = v;
+ }
+}
+
+static inline void FACET_VALUE_ADD_ESTIMATED_VALUE_TO_INDEX(FACET_KEY *k) {
+ static const FACET_VALUE tv = {
+ .hash = FACETS_HASH_ESTIMATED,
+ .name = FACET_VALUE_ESTIMATED,
+ .name_len = sizeof(FACET_VALUE_ESTIMATED) - 1,
+ .estimated = true,
+ .color = "generic",
+ };
+
+ k->current_value.hash = FACETS_HASH_ESTIMATED;
+
+ if(k->estimated_value.v) {
+ FACET_VALUE_ADD_CONFLICT(k, k->estimated_value.v, &tv);
+ k->current_value.v = k->estimated_value.v;
+ }
+ else {
+ FACET_VALUE *v = FACET_VALUE_ADD_TO_INDEX(k, &tv);
+ v->estimated = true;
+ k->estimated_value.v = v;
+ k->current_value.v = v;
+ }
+}
+
+static inline void FACET_VALUE_ADD_EMPTY_VALUE_TO_INDEX(FACET_KEY *k) {
+ static const FACET_VALUE tv = {
+ .hash = FACETS_HASH_ZERO,
+ .name = FACET_VALUE_UNSET,
+ .name_len = sizeof(FACET_VALUE_UNSET) - 1,
+ .empty = true,
+ };
+
+ k->current_value.hash = FACETS_HASH_ZERO;
+
+ if(k->empty_value.v) {
+ FACET_VALUE_ADD_CONFLICT(k, k->empty_value.v, &tv);
+ k->current_value.v = k->empty_value.v;
+ }
+ else {
+ FACET_VALUE *v = FACET_VALUE_ADD_TO_INDEX(k, &tv);
+ v->empty = true;
+ k->empty_value.v = v;
+ k->current_value.v = v;
+ }
+}
+
+static inline void FACET_VALUE_ADD_CURRENT_VALUE_TO_INDEX(FACET_KEY *k) {
+ static __thread FACET_VALUE tv = { 0 };
+
+ internal_fatal(!facet_key_value_updated(k), "trying to add a non-updated value to the index");
+
+ tv.name = facets_key_get_value(k);
+ tv.name_len = facets_key_get_value_length(k);
+ tv.hash = FACETS_HASH_FUNCTION(tv.name, tv.name_len);
+ tv.empty = false;
+ tv.estimated = false;
+ tv.unsampled = false;
+
+ k->current_value.v = FACET_VALUE_ADD_TO_INDEX(k, &tv);
+ k->facets->operations.values.indexed++;
+}
+
+static inline void FACET_VALUE_ADD_OR_UPDATE_SELECTED(FACET_KEY *k, FACETS_HASH hash) {
+ FACET_VALUE tv = {
+ .hash = hash,
+ .selected = true,
+ .name = NULL,
+ .name_len = 0,
+ };
+ FACET_VALUE_ADD_TO_INDEX(k, &tv);
+}
+
+// ----------------------------------------------------------------------------
+// The FACET_KEY index within each FACET
+
+#define foreach_key_in_facets(facets, k) \
+ for((k) = (facets)->keys.ll; (k) ;(k) = (k)->next)
+
+#define foreach_key_in_facets_done(k) do { ; } while(0)
+
+static inline void facet_key_late_init(FACETS *facets, FACET_KEY *k) {
+ if(k->values.enabled)
+ return;
+
+ if(facets_key_is_facet(facets, k)) {
+ FACETS_VALUES_INDEX_CREATE(k);
+ k->values.enabled = true;
+ if(facets->keys_with_values.used < FACETS_KEYS_WITH_VALUES_MAX)
+ facets->keys_with_values.array[facets->keys_with_values.used++] = k;
+ }
+}
+
+static inline void FACETS_KEYS_INDEX_CREATE(FACETS *facets) {
+ facets->keys.ll = NULL;
+ facets->keys.count = 0;
+ facets->keys_with_values.used = 0;
+
+ simple_hashtable_init_KEY(&facets->keys.ht, FACETS_KEYS_HASHTABLE_ENTRIES);
+}
+
+static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) {
+ FACET_KEY *k = facets->keys.ll;
+ while(k) {
+ FACET_KEY *next = k->next;
+
+ FACETS_VALUES_INDEX_DESTROY(k);
+ buffer_free(k->current_value.b);
+ freez((void *)k->name);
+ freez(k);
+
+ k = next;
+ }
+ facets->keys.ll = NULL;
+ facets->keys.count = 0;
+ facets->keys_with_values.used = 0;
+
+ simple_hashtable_destroy_KEY(&facets->keys.ht);
+}
+
+static inline FACET_KEY *FACETS_KEY_GET_FROM_INDEX(FACETS *facets, FACETS_HASH hash) {
+ SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true);
+ return SIMPLE_HASHTABLE_SLOT_DATA(slot);
+}
+
+bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) {
+ FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length);
+ FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash);
+ if(!k || k->default_selected_for_values)
+ return false;
+
+ hash = FACETS_HASH_FUNCTION(value, value_length);
+ FACET_VALUE *v = FACET_VALUE_GET_FROM_INDEX(k, hash);
+ return (v && v->selected) ? true : false;
+}
+
+void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) {
+ FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length);
+ FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash);
+ if(!k) return;
+
+ hash = FACETS_HASH_FUNCTION(value, value_length);
+ FACET_VALUE *v = FACET_VALUE_GET_FROM_INDEX(k, hash);
+ if(v && v->name && v->name_len) return;
+
+ FACET_VALUE tv = {
+ .hash = hash,
+ .name = value,
+ .name_len = value_length,
+ };
+ FACET_VALUE_ADD_TO_INDEX(k, &tv);
+}
+
+static void facet_key_set_name(FACET_KEY *k, const char *name, size_t name_length) {
+ internal_fatal(k->name && name && (strncmp(k->name, name, name_length) != 0 || k->name[name_length] != '\0'),
+ "key hash conflict: '%s' and '%s' have the same hash",
+ k->name, name);
+
+ if(likely(k->name || !name || !name_length))
+ return;
+
+ // an actual value, not a filter
+
+ char buf[name_length + 1];
+ memcpy(buf, name, name_length);
+ buf[name_length] = '\0';
+
+ internal_fatal(strchr(buf, '='), "found = in key");
+
+ k->name = strdupz(buf);
+ facet_key_late_init(k->facets, k);
+}
+
+static inline FACET_KEY *FACETS_KEY_CREATE(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) {
+ facets->operations.keys.unique++;
+
+ FACET_KEY *k = callocz(1, sizeof(*k));
+
+ k->hash = hash;
+ k->facets = facets;
+ k->options = options;
+ k->current_value.b = buffer_create(sizeof(FACET_VALUE_UNSET), NULL);
+ k->default_selected_for_values = true;
+
+ if(!(k->options & FACET_KEY_OPTION_REORDER))
+ k->order = facets->order++;
+
+ if((k->options & FACET_KEY_OPTION_FTS) || (facets->options & FACETS_OPTION_ALL_KEYS_FTS))
+ facets->keys_filtered_by_query++;
+
+ facet_key_set_name(k, name, name_length);
+
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->keys.ll, k, prev, next);
+ facets->keys.count++;
+
+ return k;
+}
+
+static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) {
+ facets->operations.keys.registered++;
+
+ SIMPLE_HASHTABLE_SLOT_KEY *slot = simple_hashtable_get_slot_KEY(&facets->keys.ht, hash, true);
+
+ if(unlikely(!SIMPLE_HASHTABLE_SLOT_DATA(slot))) {
+ // we have to add it
+ FACET_KEY *k = FACETS_KEY_CREATE(facets, hash, name, name_length, options);
+
+ simple_hashtable_set_slot_KEY(&facets->keys.ht, slot, hash, k);
+
+ return k;
+ }
+
+ // already in the index
+
+ FACET_KEY *k = SIMPLE_HASHTABLE_SLOT_DATA(slot);
+
+ facet_key_set_name(k, name, name_length);
+
+ if(unlikely(k->options & FACET_KEY_OPTION_REORDER)) {
+ k->order = facets->order++;
+ k->options &= ~FACET_KEY_OPTION_REORDER;
+ }
+
+ return k;
+}
+
+bool facets_key_name_is_filter(FACETS *facets, const char *key) {
+ FACETS_HASH hash = FACETS_HASH_FUNCTION(key, strlen(key));
+ FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash);
+ return (!k || k->default_selected_for_values) ? false : true;
+}
+
+bool facets_key_name_is_facet(FACETS *facets, const char *key) {
+ size_t key_len = strlen(key);
+ FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_len);
+ FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, key, key_len, 0);
+ return (k && (k->options & FACET_KEY_OPTION_FACET));
+}
+
+// ----------------------------------------------------------------------------
+
+size_t facets_histogram_slots(FACETS *facets) {
+ return facets->histogram.slots;
+}
+
+static usec_t calculate_histogram_bar_width(usec_t after_ut, usec_t before_ut) {
+ // Array of valid durations in seconds
+ static time_t valid_durations_s[] = {
+ 1, 2, 5, 10, 15, 30, // seconds
+ 1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes
+ 1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours
+ 1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days
+ 1 * (30*86400) // months
+ };
+ static int array_size = sizeof(valid_durations_s) / sizeof(valid_durations_s[0]);
+
+ usec_t duration_ut = before_ut - after_ut;
+ usec_t bar_width_ut = 1 * USEC_PER_SEC;
+
+ for (int i = array_size - 1; i >= 0; --i) {
+ if (duration_ut / (valid_durations_s[i] * USEC_PER_SEC) >= FACETS_HISTOGRAM_COLUMNS) {
+ bar_width_ut = valid_durations_s[i] * USEC_PER_SEC;
+ break;
+ }
+ }
+
+ return bar_width_ut;
+}
+
+static inline usec_t facets_histogram_slot_baseline_ut(FACETS *facets, usec_t ut) {
+ usec_t delta_ut = ut % facets->histogram.slot_width_ut;
+ return ut - delta_ut;
+}
+
+void facets_set_timeframe_and_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut) {
+ if(after_ut > before_ut) {
+ usec_t t = after_ut;
+ after_ut = before_ut;
+ before_ut = t;
+ }
+
+ facets->histogram.enabled = true;
+
+ if(key_id && *key_id && strlen(key_id) == FACET_STRING_HASH_SIZE - 1) {
+ facets->histogram.chart = strdupz(key_id);
+ facets->histogram.hash = str_to_facets_hash(facets->histogram.chart);
+ }
+ else {
+ freez(facets->histogram.chart);
+ facets->histogram.chart = NULL;
+ facets->histogram.hash = FACETS_HASH_ZERO;
+ }
+
+ facets->timeframe.after_ut = after_ut;
+ facets->timeframe.before_ut = before_ut;
+
+ facets->histogram.slot_width_ut = calculate_histogram_bar_width(after_ut, before_ut);
+ facets->histogram.after_ut = facets_histogram_slot_baseline_ut(facets, after_ut);
+ facets->histogram.before_ut = facets_histogram_slot_baseline_ut(facets, before_ut) + facets->histogram.slot_width_ut;
+ facets->histogram.slots = (facets->histogram.before_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut + 1;
+
+ internal_fatal(after_ut < facets->histogram.after_ut, "histogram after_ut is not less or equal to wanted after_ut");
+ internal_fatal(before_ut > facets->histogram.before_ut, "histogram before_ut is not more or equal to wanted before_ut");
+
+ if(facets->histogram.slots > 1000) {
+ facets->histogram.slots = 1000 + 1;
+ facets->histogram.slot_width_ut = (facets->histogram.before_ut - facets->histogram.after_ut) / 1000;
+ }
+}
+
+void facets_set_timeframe_and_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut) {
+ char hash_str[FACET_STRING_HASH_SIZE];
+ FACETS_HASH hash = FACETS_HASH_FUNCTION(key_name, strlen(key_name));
+ facets_hash_to_str(hash, hash_str);
+ facets_set_timeframe_and_histogram_by_id(facets, hash_str, after_ut, before_ut);
+}
+
+static inline uint32_t facets_histogram_slot_at_time_ut(FACETS *facets, usec_t usec, FACET_VALUE *v) {
+ if(unlikely(!v->histogram))
+ v->histogram = callocz(facets->histogram.slots, sizeof(*v->histogram));
+
+ usec_t base_ut = facets_histogram_slot_baseline_ut(facets, usec);
+
+ if(unlikely(base_ut < facets->histogram.after_ut))
+ base_ut = facets->histogram.after_ut;
+
+ if(unlikely(base_ut > facets->histogram.before_ut))
+ base_ut = facets->histogram.before_ut;
+
+ uint32_t slot = (base_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut;
+
+ if(unlikely(slot >= facets->histogram.slots))
+ slot = facets->histogram.slots - 1;
+
+ return slot;
+}
+
+static inline void facets_histogram_update_value_slot(FACETS *facets, usec_t usec, FACET_VALUE *v) {
+ uint32_t slot = facets_histogram_slot_at_time_ut(facets, usec, v);
+ v->histogram[slot]++;
+}
+
+static inline void facets_histogram_update_value(FACETS *facets, usec_t usec) {
+ if(!facets->histogram.enabled ||
+ !facets->histogram.key ||
+ !facets->histogram.key->values.enabled ||
+ !facet_key_value_updated(facets->histogram.key) ||
+ usec < facets->histogram.after_ut ||
+ usec > facets->histogram.before_ut)
+ return;
+
+ FACET_VALUE *v = facets->histogram.key->current_value.v;
+ facets_histogram_update_value_slot(facets, usec, v);
+}
+
+static usec_t overlap_duration_ut(usec_t start1, usec_t end1, usec_t start2, usec_t end2) {
+ usec_t overlap_start = MAX(start1, start2);
+ usec_t overlap_end = MIN(end1, end2);
+
+ if (overlap_start < overlap_end)
+ return overlap_end - overlap_start;
+ else
+ return 0; // No overlap
+}
+
+void facets_update_estimations(FACETS *facets, usec_t from_ut, usec_t to_ut, size_t entries) {
+ if(unlikely(!facets->histogram.enabled))
+ return;
+
+ if(unlikely(!overlap_duration_ut(facets->histogram.after_ut, facets->histogram.before_ut, from_ut, to_ut)))
+ return;
+
+ facets->operations.rows.evaluated += entries;
+ facets->operations.rows.matched += entries;
+ facets->operations.rows.estimated += entries;
+
+ if (!facets->histogram.enabled ||
+ !facets->histogram.key ||
+ !facets->histogram.key->values.enabled)
+ return;
+
+ if (from_ut < facets->histogram.after_ut)
+ from_ut = facets->histogram.after_ut;
+
+ if (to_ut > facets->histogram.before_ut)
+ to_ut = facets->histogram.before_ut;
+
+ if (!facets->histogram.key->estimated_value.v)
+ FACET_VALUE_ADD_ESTIMATED_VALUE_TO_INDEX(facets->histogram.key);
+
+ FACET_VALUE *v = facets->histogram.key->estimated_value.v;
+
+ size_t slots = 0;
+ size_t total_ut = to_ut - from_ut;
+ ssize_t remaining_entries = (ssize_t)entries;
+ size_t slot = facets_histogram_slot_at_time_ut(facets, from_ut, v);
+ for(; slot < facets->histogram.slots ;slot++) {
+ usec_t slot_start_ut = facets->histogram.after_ut + slot * facets->histogram.slot_width_ut;
+ usec_t slot_end_ut = slot_start_ut + facets->histogram.slot_width_ut;
+
+ if(slot_start_ut > to_ut)
+ break;
+
+ usec_t overlap_ut = overlap_duration_ut(from_ut, to_ut, slot_start_ut, slot_end_ut);
+
+ size_t slot_entries = (overlap_ut * entries) / total_ut;
+ v->histogram[slot] += slot_entries;
+ remaining_entries -= (ssize_t)slot_entries;
+ slots++;
+ }
+
+ // Check if all entries are assigned
+ // This should always be true if the distribution is correct
+ internal_fatal(remaining_entries < 0 || remaining_entries >= (ssize_t)(slots),
+ "distribution of estimations is not accurate - there are %zd remaining entries",
+ remaining_entries);
+}
+
+void facets_row_finished_unsampled(FACETS *facets, usec_t usec) {
+ facets->operations.rows.evaluated++;
+ facets->operations.rows.matched++;
+ facets->operations.rows.unsampled++;
+
+ if(!facets->histogram.enabled ||
+ !facets->histogram.key ||
+ !facets->histogram.key->values.enabled ||
+ usec < facets->histogram.after_ut ||
+ usec > facets->histogram.before_ut)
+ return;
+
+ if(!facets->histogram.key->unsampled_value.v)
+ FACET_VALUE_ADD_UNSAMPLED_VALUE_TO_INDEX(facets->histogram.key);
+
+ FACET_VALUE *v = facets->histogram.key->unsampled_value.v;
+ facets_histogram_update_value_slot(facets, usec, v);
+
+ facets_reset_key(facets->histogram.key);
+}
+
+static const char *facets_key_name_cached(FACET_KEY *k, DICTIONARY *used_hashes_registry) {
+ if(k->name) {
+ if(used_hashes_registry && !k->default_selected_for_values) {
+ char hash_str[FACET_STRING_HASH_SIZE];
+ facets_hash_to_str(k->hash, hash_str);
+ dictionary_set(used_hashes_registry, hash_str, (void *)k->name, strlen(k->name) + 1);
+ }
+
+ return k->name;
+ }
+
+ // key has no name
+ const char *name = "[UNAVAILABLE_FIELD]";
+
+ if(used_hashes_registry) {
+ char hash_str[FACET_STRING_HASH_SIZE];
+ facets_hash_to_str(k->hash, hash_str);
+ const char *s = dictionary_get(used_hashes_registry, hash_str);
+ if(s) name = s;
+ }
+
+ return name;
+}
+
+static const char *facets_key_value_cached(FACET_KEY *k, FACET_VALUE *v, DICTIONARY *used_hashes_registry) {
+ if(v->empty || v->estimated || v->unsampled)
+ return v->name;
+
+ if(v->name && v->name_len) {
+ if(used_hashes_registry && !k->default_selected_for_values && v->selected) {
+ char hash_str[FACET_STRING_HASH_SIZE];
+ facets_hash_to_str(v->hash, hash_str);
+ dictionary_set(used_hashes_registry, hash_str, (void *)v->name, v->name_len + 1);
+ }
+
+ return v->name;
+ }
+
+ // key has no name
+ const char *name = "[unavailable field]";
+
+ if(used_hashes_registry) {
+ char hash_str[FACET_STRING_HASH_SIZE];
+ facets_hash_to_str(v->hash, hash_str);
+ const char *s = dictionary_get(used_hashes_registry, hash_str);
+ if(s) name = s;
+ }
+
+ return name;
+}
+
+static inline void facets_key_value_transformed(FACETS *facets, FACET_KEY *k, FACET_VALUE *v, BUFFER *dst, FACETS_TRANSFORMATION_SCOPE scope) {
+ buffer_flush(dst);
+
+ if(v->empty || v->unsampled || v->estimated)
+ buffer_strcat(dst, v->name);
+ else if(k->transform.cb && k->transform.view_only) {
+ buffer_contents_replace(dst, v->name, v->name_len);
+ k->transform.cb(facets, dst, scope, k->transform.data);
+ }
+ else
+ buffer_strcat(dst, facets_key_value_cached(k, v, facets->report.used_hashes_registry));
+}
+
+static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, const char *first_key) {
+ CLEAN_BUFFER *tb = buffer_create(0, NULL);
+
+ buffer_json_member_add_array(wb, key);
+ {
+ if(first_key)
+ buffer_json_add_array_item_string(wb, first_key);
+
+ if(k && k->values.enabled) {
+ FACET_VALUE *v;
+ foreach_value_in_key(k, v) {
+ if (unlikely(!v->histogram))
+ continue;
+
+ facets_key_value_transformed(facets, k, v, tb, FACETS_TRANSFORM_HISTOGRAM);
+ buffer_json_add_array_item_string(wb, buffer_tostring(tb));
+ }
+ foreach_value_in_key_done(v);
+ }
+ }
+ buffer_json_array_close(wb); // key
+}
+
+static inline void facets_histogram_value_colors(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
+ buffer_json_member_add_array(wb, key);
+ {
+ if(k && k->values.enabled) {
+ FACET_VALUE *v;
+ foreach_value_in_key(k, v) {
+ if (unlikely(!v->histogram))
+ continue;
+
+ buffer_json_add_array_item_string(wb, v->color);
+ }
+ foreach_value_in_key_done(v);
+ }
+ }
+ buffer_json_array_close(wb); // key
+}
+
+static inline void facets_histogram_value_units(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
+ buffer_json_member_add_array(wb, key);
+ {
+ if(k && k->values.enabled) {
+ FACET_VALUE *v;
+ foreach_value_in_key(k, v) {
+ if (unlikely(!v->histogram))
+ continue;
+
+ buffer_json_add_array_item_string(wb, "events");
+ }
+ foreach_value_in_key_done(v);
+ }
+ }
+ buffer_json_array_close(wb); // key
+}
+
+static inline void facets_histogram_value_min(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
+ buffer_json_member_add_array(wb, key);
+ {
+ if(k && k->values.enabled) {
+ FACET_VALUE *v;
+ foreach_value_in_key(k, v) {
+ if (unlikely(!v->histogram))
+ continue;
+
+ buffer_json_add_array_item_uint64(wb, v->min);
+ }
+ foreach_value_in_key_done(v);
+ }
+ }
+ buffer_json_array_close(wb); // key
+}
+
+static inline void facets_histogram_value_max(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
+ buffer_json_member_add_array(wb, key);
+ {
+ if(k && k->values.enabled) {
+ FACET_VALUE *v;
+ foreach_value_in_key(k, v) {
+ if (unlikely(!v->histogram))
+ continue;
+
+ buffer_json_add_array_item_uint64(wb, v->max);
+ }
+ foreach_value_in_key_done(v);
+ }
+ }
+ buffer_json_array_close(wb); // key
+}
+
+static inline void facets_histogram_value_avg(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
+ buffer_json_member_add_array(wb, key);
+ {
+ if(k && k->values.enabled) {
+ FACET_VALUE *v;
+ foreach_value_in_key(k, v) {
+ if (unlikely(!v->histogram))
+ continue;
+
+ buffer_json_add_array_item_double(wb, (double) v->sum / (double) facets->histogram.slots);
+ }
+ foreach_value_in_key_done(v);
+ }
+ }
+ buffer_json_array_close(wb); // key
+}
+
+static inline void facets_histogram_value_arp(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
+ buffer_json_member_add_array(wb, key);
+ {
+ if(k && k->values.enabled) {
+ FACET_VALUE *v;
+ foreach_value_in_key(k, v) {
+ if (unlikely(!v->histogram))
+ continue;
+
+ buffer_json_add_array_item_uint64(wb, 0);
+ }
+ foreach_value_in_key_done(v);
+ }
+ }
+ buffer_json_array_close(wb); // key
+}
+
+static inline void facets_histogram_value_con(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, uint32_t sum) {
+ buffer_json_member_add_array(wb, key);
+ {
+ if(k && k->values.enabled) {
+ FACET_VALUE *v;
+ foreach_value_in_key(k, v) {
+ if (unlikely(!v->histogram))
+ continue;
+
+ buffer_json_add_array_item_double(wb, (double) v->sum * 100.0 / (double) sum);
+ }
+ foreach_value_in_key_done(v);
+ }
+ }
+ buffer_json_array_close(wb); // key
+}
+
+static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb) {
+ CLEAN_BUFFER *tmp = buffer_create(0, NULL);
+
+ size_t dimensions = 0;
+ uint32_t min = UINT32_MAX, max = 0, sum = 0, count = 0;
+
+ if(k && k->values.enabled) {
+ FACET_VALUE *v;
+ foreach_value_in_key(k, v) {
+ if (unlikely(!v->histogram))
+ continue;
+
+ dimensions++;
+
+ v->min = UINT32_MAX;
+ v->max = 0;
+ v->sum = 0;
+
+ for(uint32_t i = 0; i < facets->histogram.slots ;i++) {
+ uint32_t n = v->histogram[i];
+
+ if(n < min)
+ min = n;
+
+ if(n > max)
+ max = n;
+
+ sum += n;
+ count++;
+
+ if(n < v->min)
+ v->min = n;
+
+ if(n > v->max)
+ v->max = n;
+
+ v->sum += n;
+ }
+ }
+ foreach_value_in_key_done(v);
+ }
+
+ buffer_json_member_add_object(wb, "summary");
+ {
+ // summary.nodes
+ buffer_json_member_add_array(wb, "nodes");
+ {
+ buffer_json_add_array_item_object(wb); // node
+ {
+ buffer_json_member_add_string(wb, "mg", "default");
+ buffer_json_member_add_string(wb, "nm", "facets.histogram");
+ buffer_json_member_add_uint64(wb, "ni", 0);
+ buffer_json_member_add_object(wb, "st");
+ {
+ buffer_json_member_add_uint64(wb, "ai", 0);
+ buffer_json_member_add_uint64(wb, "code", 200);
+ buffer_json_member_add_string(wb, "msg", "");
+ }
+ buffer_json_object_close(wb); // st
+
+ if(dimensions) {
+ buffer_json_member_add_object(wb, "is");
+ {
+ buffer_json_member_add_uint64(wb, "sl", 1);
+ buffer_json_member_add_uint64(wb, "qr", 1);
+ }
+ buffer_json_object_close(wb); // is
+
+ buffer_json_member_add_object(wb, "ds");
+ {
+ buffer_json_member_add_uint64(wb, "sl", dimensions);
+ buffer_json_member_add_uint64(wb, "qr", dimensions);
+ }
+ buffer_json_object_close(wb); // ds
+ }
+
+ if(count) {
+ buffer_json_member_add_object(wb, "sts");
+ {
+ buffer_json_member_add_uint64(wb, "min", min);
+ buffer_json_member_add_uint64(wb, "max", max);
+ buffer_json_member_add_double(wb, "avg", (double) sum / (double) count);
+ buffer_json_member_add_double(wb, "con", 100.0);
+ }
+ buffer_json_object_close(wb); // sts
+ }
+ }
+ buffer_json_object_close(wb); // node
+ }
+ buffer_json_array_close(wb); // nodes
+
+ // summary.contexts
+ buffer_json_member_add_array(wb, "contexts");
+ {
+ buffer_json_add_array_item_object(wb); // context
+ {
+ buffer_json_member_add_string(wb, "id", "facets.histogram");
+
+ if(dimensions) {
+ buffer_json_member_add_object(wb, "is");
+ {
+ buffer_json_member_add_uint64(wb, "sl", 1);
+ buffer_json_member_add_uint64(wb, "qr", 1);
+ }
+ buffer_json_object_close(wb); // is
+
+ buffer_json_member_add_object(wb, "ds");
+ {
+ buffer_json_member_add_uint64(wb, "sl", dimensions);
+ buffer_json_member_add_uint64(wb, "qr", dimensions);
+ }
+ buffer_json_object_close(wb); // ds
+ }
+
+ if(count) {
+ buffer_json_member_add_object(wb, "sts");
+ {
+ buffer_json_member_add_uint64(wb, "min", min);
+ buffer_json_member_add_uint64(wb, "max", max);
+ buffer_json_member_add_double(wb, "avg", (double) sum / (double) count);
+ buffer_json_member_add_double(wb, "con", 100.0);
+ }
+ buffer_json_object_close(wb); // sts
+ }
+ }
+ buffer_json_object_close(wb); // context
+ }
+ buffer_json_array_close(wb); // contexts
+
+ // summary.instances
+ buffer_json_member_add_array(wb, "instances");
+ {
+ buffer_json_add_array_item_object(wb); // instance
+ {
+ buffer_json_member_add_string(wb, "id", "facets.histogram");
+ buffer_json_member_add_uint64(wb, "ni", 0);
+
+ if(dimensions) {
+ buffer_json_member_add_object(wb, "ds");
+ {
+ buffer_json_member_add_uint64(wb, "sl", dimensions);
+ buffer_json_member_add_uint64(wb, "qr", dimensions);
+ }
+ buffer_json_object_close(wb); // ds
+ }
+
+ if(count) {
+ buffer_json_member_add_object(wb, "sts");
+ {
+ buffer_json_member_add_uint64(wb, "min", min);
+ buffer_json_member_add_uint64(wb, "max", max);
+ buffer_json_member_add_double(wb, "avg", (double) sum / (double) count);
+ buffer_json_member_add_double(wb, "con", 100.0);
+ }
+ buffer_json_object_close(wb); // sts
+ }
+ }
+ buffer_json_object_close(wb); // instance
+ }
+ buffer_json_array_close(wb); // instances
+
+ // summary.dimensions
+ buffer_json_member_add_array(wb, "dimensions");
+ if(dimensions && k && k->values.enabled) {
+ size_t pri = 0;
+ FACET_VALUE *v;
+
+ foreach_value_in_key(k, v) {
+ if(unlikely(!v->histogram))
+ continue;
+
+ buffer_json_add_array_item_object(wb); // dimension
+ {
+ facets_key_value_transformed(facets, k, v, tmp, FACETS_TRANSFORM_HISTOGRAM);
+ buffer_json_member_add_string(wb, "id", buffer_tostring(tmp));
+ buffer_json_member_add_object(wb, "ds");
+ {
+ buffer_json_member_add_uint64(wb, "sl", 1);
+ buffer_json_member_add_uint64(wb, "qr", 1);
+ }
+ buffer_json_object_close(wb); // ds
+ buffer_json_member_add_object(wb, "sts");
+ {
+ buffer_json_member_add_uint64(wb, "min", v->min);
+ buffer_json_member_add_uint64(wb, "max", v->max);
+ buffer_json_member_add_double(wb, "avg", (double)v->sum / (double)facets->histogram.slots);
+ buffer_json_member_add_double(wb, "con", (double)v->sum * 100.0 / (double)sum);
+ }
+ buffer_json_object_close(wb); // sts
+ buffer_json_member_add_uint64(wb, "pri", pri++);
+ }
+ buffer_json_object_close(wb); // dimension
+ }
+ foreach_value_in_key_done(v);
+ }
+ buffer_json_array_close(wb); // dimensions
+
+ buffer_json_member_add_array(wb, "labels");
+ buffer_json_array_close(wb); // labels
+
+ buffer_json_member_add_array(wb, "alerts");
+ buffer_json_array_close(wb); // alerts
+ }
+ buffer_json_object_close(wb); // summary
+
+ buffer_json_member_add_object(wb, "totals");
+ {
+ buffer_json_member_add_object(wb, "nodes");
+ {
+ buffer_json_member_add_uint64(wb, "sl", 1);
+ buffer_json_member_add_uint64(wb, "qr", 1);
+ }
+ buffer_json_object_close(wb); // nodes
+
+ if(dimensions) {
+ buffer_json_member_add_object(wb, "contexts");
+ {
+ buffer_json_member_add_uint64(wb, "sl", 1);
+ buffer_json_member_add_uint64(wb, "qr", 1);
+ }
+ buffer_json_object_close(wb); // contexts
+ buffer_json_member_add_object(wb, "instances");
+ {
+ buffer_json_member_add_uint64(wb, "sl", 1);
+ buffer_json_member_add_uint64(wb, "qr", 1);
+ }
+ buffer_json_object_close(wb); // instances
+
+ buffer_json_member_add_object(wb, "dimensions");
+ {
+ buffer_json_member_add_uint64(wb, "sl", dimensions);
+ buffer_json_member_add_uint64(wb, "qr", dimensions);
+ }
+ buffer_json_object_close(wb); // dimension
+ }
+ }
+ buffer_json_object_close(wb); // totals
+
+ buffer_json_member_add_object(wb, "result");
+ {
+ facets_histogram_value_names(wb, facets, k, "labels", "time");
+
+ buffer_json_member_add_object(wb, "point");
+ {
+ buffer_json_member_add_uint64(wb, "value", 0);
+ buffer_json_member_add_uint64(wb, "arp", 1);
+ buffer_json_member_add_uint64(wb, "pa", 2);
+ }
+ buffer_json_object_close(wb); // point
+
+ buffer_json_member_add_array(wb, "data");
+ if(k && k->values.enabled) {
+ usec_t t = facets->histogram.after_ut;
+ for(uint32_t i = 0; i < facets->histogram.slots ;i++) {
+ buffer_json_add_array_item_array(wb); // row
+ {
+ buffer_json_add_array_item_time_ms(wb, t / USEC_PER_SEC);
+
+ FACET_VALUE *v;
+ foreach_value_in_key(k, v) {
+ if (unlikely(!v->histogram))
+ continue;
+
+ buffer_json_add_array_item_array(wb); // point
+
+ buffer_json_add_array_item_uint64(wb, v->histogram[i]);
+ buffer_json_add_array_item_uint64(wb, 0); // arp - anomaly rate
+ buffer_json_add_array_item_uint64(wb, 0); // pa - point annotation
+
+ buffer_json_array_close(wb); // point
+ }
+ foreach_value_in_key_done(v);
+ }
+ buffer_json_array_close(wb); // row
+
+ t += facets->histogram.slot_width_ut;
+ }
+ }
+ buffer_json_array_close(wb); //data
+ }
+ buffer_json_object_close(wb); // result
+
+ buffer_json_member_add_object(wb, "db");
+ {
+ buffer_json_member_add_uint64(wb, "tiers", 1);
+ buffer_json_member_add_uint64(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);
+// we should add these only when we know the retention of the db
+// buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC);
+// buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC);
+ buffer_json_member_add_string(wb, "units", "events");
+ buffer_json_member_add_object(wb, "dimensions");
+ {
+ facets_histogram_value_names(wb, facets, k, "ids", NULL);
+ facets_histogram_value_units(wb, facets, k, "units");
+
+ buffer_json_member_add_object(wb, "sts");
+ {
+ facets_histogram_value_min(wb, facets, k, "min");
+ facets_histogram_value_max(wb, facets, k, "max");
+ facets_histogram_value_avg(wb, facets, k, "avg");
+ facets_histogram_value_arp(wb, facets, k, "arp");
+ facets_histogram_value_con(wb, facets, k, "con", sum);
+ }
+ buffer_json_object_close(wb); // sts
+ }
+ buffer_json_object_close(wb); // dimensions
+
+ buffer_json_member_add_array(wb, "per_tier");
+ {
+ buffer_json_add_array_item_object(wb); // tier0
+ {
+ buffer_json_member_add_uint64(wb, "tier", 0);
+ buffer_json_member_add_uint64(wb, "queries", 1);
+ buffer_json_member_add_uint64(wb, "points", count);
+ buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);
+// we should add these only when we know the retention of the db
+// buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC);
+// buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC);
+ }
+ buffer_json_object_close(wb); // tier0
+ }
+ buffer_json_array_close(wb); // per_tier
+ }
+ buffer_json_object_close(wb); // db
+
+ buffer_json_member_add_object(wb, "view");
+ {
+ char title[1024 + 1] = "Events Distribution";
+ FACET_KEY *kt = FACETS_KEY_GET_FROM_INDEX(facets, facets->histogram.hash);
+ if(kt && kt->name)
+ snprintfz(title, sizeof(title) - 1, "Events Distribution by %s", kt->name);
+
+ buffer_json_member_add_string(wb, "title", title);
+ buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);
+ buffer_json_member_add_time_t(wb, "after", facets->histogram.after_ut / USEC_PER_SEC);
+ buffer_json_member_add_time_t(wb, "before", facets->histogram.before_ut / USEC_PER_SEC);
+ buffer_json_member_add_string(wb, "units", "events");
+ buffer_json_member_add_string(wb, "chart_type", "stackedBar");
+ buffer_json_member_add_object(wb, "dimensions");
+ {
+ buffer_json_member_add_array(wb, "grouped_by");
+ {
+ buffer_json_add_array_item_string(wb, "dimension");
+ }
+ buffer_json_array_close(wb); // grouped_by
+
+ facets_histogram_value_names(wb, facets, k, "ids", NULL);
+ facets_histogram_value_names(wb, facets, k, "names", NULL);
+ facets_histogram_value_colors(wb, facets, k, "colors");
+ facets_histogram_value_units(wb, facets, k, "units");
+
+ buffer_json_member_add_object(wb, "sts");
+ {
+ facets_histogram_value_min(wb, facets, k, "min");
+ facets_histogram_value_max(wb, facets, k, "max");
+ facets_histogram_value_avg(wb, facets, k, "avg");
+ facets_histogram_value_arp(wb, facets, k, "arp");
+ facets_histogram_value_con(wb, facets, k, "con", sum);
+ }
+ buffer_json_object_close(wb); // sts
+ }
+ buffer_json_object_close(wb); // dimensions
+
+ buffer_json_member_add_uint64(wb, "min", min);
+ buffer_json_member_add_uint64(wb, "max", max);
+ }
+ buffer_json_object_close(wb); // view
+
+ buffer_json_member_add_array(wb, "agents");
+ {
+ buffer_json_add_array_item_object(wb); // agent
+ {
+ buffer_json_member_add_string(wb, "mg", "default");
+ buffer_json_member_add_string(wb, "nm", "facets.histogram");
+ buffer_json_member_add_time_t(wb, "now", now_realtime_sec());
+ buffer_json_member_add_uint64(wb, "ai", 0);
+ }
+ buffer_json_object_close(wb); // agent
+ }
+ buffer_json_array_close(wb); // agents
+}
+
+// ----------------------------------------------------------------------------
+
+static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) {
+ if(!k->key_found_in_row)
+ v->rows_matching_facet_value++;
+
+ k->key_found_in_row++;
+
+ if(v->selected)
+ k->key_values_selected_in_row++;
+}
+
+static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) {
+ bool included = true, excluded = false, never = false;
+
+ if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) {
+ if(k->options & FACET_KEY_OPTION_FACET) {
+ included = true;
+ excluded = false;
+ never = false;
+ }
+ else if(k->options & (FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) {
+ included = false;
+ excluded = true;
+ never = true;
+ }
+ }
+ else {
+ if (facets->included_keys) {
+ if (!simple_pattern_matches(facets->included_keys, k->name))
+ included = false;
+ }
+
+ if (facets->excluded_keys) {
+ if (simple_pattern_matches(facets->excluded_keys, k->name)) {
+ excluded = true;
+ never = true;
+ }
+ }
+ }
+
+ if(included && !excluded) {
+ k->options |= FACET_KEY_OPTION_FACET;
+ k->options &= ~FACET_KEY_OPTION_NO_FACET;
+ return true;
+ }
+
+ k->options |= FACET_KEY_OPTION_NO_FACET;
+ k->options &= ~FACET_KEY_OPTION_FACET;
+
+ if(never)
+ k->options |= FACET_KEY_OPTION_NEVER_FACET;
+
+ return false;
+}
+
+// ----------------------------------------------------------------------------
+
+FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) {
+ FACETS *facets = callocz(1, sizeof(FACETS));
+ facets->options = options;
+ FACETS_KEYS_INDEX_CREATE(facets);
+
+ if(facet_keys && *facet_keys)
+ facets->included_keys = simple_pattern_create(facet_keys, "|", SIMPLE_PATTERN_EXACT, true);
+
+ if(non_facet_keys && *non_facet_keys)
+ facets->excluded_keys = simple_pattern_create(non_facet_keys, "|", SIMPLE_PATTERN_EXACT, true);
+
+ if(visible_keys && *visible_keys)
+ facets->visible_keys = simple_pattern_create(visible_keys, "|", SIMPLE_PATTERN_EXACT, true);
+
+ facets->max_items_to_return = items_to_return > 1 ? items_to_return : 2;
+ facets->anchor.start_ut = 0;
+ facets->anchor.stop_ut = 0;
+ facets->anchor.direction = FACETS_ANCHOR_DIRECTION_BACKWARD;
+ facets->order = 1;
+
+ return facets;
+}
+
+void facets_destroy(FACETS *facets) {
+ dictionary_destroy(facets->accepted_params);
+ FACETS_KEYS_INDEX_DESTROY(facets);
+ simple_pattern_free(facets->visible_keys);
+ simple_pattern_free(facets->included_keys);
+ simple_pattern_free(facets->excluded_keys);
+
+ while(facets->base) {
+ FACET_ROW *r = facets->base;
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, r, prev, next);
+
+ facets_row_free(facets, r);
+ }
+
+ freez(facets->histogram.chart);
+ freez(facets);
+}
+
+void facets_accepted_param(FACETS *facets, const char *param) {
+ if(!facets->accepted_params)
+ facets->accepted_params = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE);
+
+ dictionary_set(facets->accepted_params, param, NULL, 0);
+}
+
+static inline FACET_KEY *facets_register_key_name_length(FACETS *facets, const char *key, size_t key_length, FACET_KEY_OPTIONS options) {
+ return FACETS_KEY_ADD_TO_INDEX(facets, FACETS_HASH_FUNCTION(key, key_length), key, key_length, options);
+}
+
+inline FACET_KEY *facets_register_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options) {
+ return facets_register_key_name_length(facets, key, strlen(key), options);
+}
+
+inline FACET_KEY *facets_register_key_name_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data) {
+ FACET_KEY *k = facets_register_key_name(facets, key, options);
+ k->transform.cb = cb;
+ k->transform.data = data;
+ k->transform.view_only = (options & FACET_KEY_OPTION_TRANSFORM_VIEW) ? true : false;
+ return k;
+}
+
+inline FACET_KEY *facets_register_dynamic_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data) {
+ FACET_KEY *k = facets_register_key_name(facets, key, options);
+ k->dynamic.cb = cb;
+ k->dynamic.data = data;
+ return k;
+}
+
+void facets_set_query(FACETS *facets, const char *query) {
+ if(!query)
+ return;
+
+ facets->query = simple_pattern_create(query, "|", SIMPLE_PATTERN_SUBSTRING, false);
+}
+
+void facets_set_items(FACETS *facets, uint32_t items) {
+ facets->max_items_to_return = items > 1 ? items : 2;
+}
+
+void facets_set_anchor(FACETS *facets, usec_t start_ut, usec_t stop_ut, FACETS_ANCHOR_DIRECTION direction) {
+ facets->anchor.start_ut = start_ut;
+ facets->anchor.stop_ut = stop_ut;
+ facets->anchor.direction = direction;
+
+ if((facets->anchor.direction == FACETS_ANCHOR_DIRECTION_BACKWARD && facets->anchor.start_ut && facets->anchor.start_ut < facets->anchor.stop_ut) ||
+ (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD && facets->anchor.stop_ut && facets->anchor.stop_ut < facets->anchor.start_ut)) {
+ internal_error(true, "start and stop anchors are flipped");
+ facets->anchor.start_ut = stop_ut;
+ facets->anchor.stop_ut = start_ut;
+ }
+}
+
+void facets_enable_slice_mode(FACETS *facets) {
+ facets->options |= FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS | FACETS_OPTION_SORT_FACETS_ALPHABETICALLY;
+}
+
+inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options) {
+ if(!is_valid_string_hash(key_id))
+ return NULL;
+
+ FACETS_HASH hash = str_to_facets_hash(key_id);
+
+ internal_error(strcmp(hash_to_static_string(hash), key_id) != 0,
+ "Regenerating the user supplied key, does not produce the same hash string");
+
+ FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, NULL, 0, options);
+ k->options |= FACET_KEY_OPTION_FACET;
+ k->options &= ~FACET_KEY_OPTION_NO_FACET;
+ facet_key_late_init(facets, k);
+
+ return k;
+}
+
+void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *value_id, FACET_KEY_OPTIONS options) {
+ FACET_KEY *k = facets_register_facet_id(facets, key_id, options);
+ if(k) {
+ if(is_valid_string_hash(value_id)) {
+ k->default_selected_for_values = false;
+ FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, str_to_facets_hash(value_id));
+ }
+ }
+}
+
+void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity) {
+ facets->current_row.severity = severity;
+}
+
+void facets_register_row_severity(FACETS *facets, facet_row_severity_t cb, void *data) {
+ facets->severity.cb = cb;
+ facets->severity.data = data;
+}
+
+void facets_set_additional_options(FACETS *facets, FACETS_OPTIONS options) {
+ facets->options |= options;
+}
+
+// ----------------------------------------------------------------------------
+
+static inline void facets_key_set_unsampled_value(FACETS *facets, FACET_KEY *k) {
+ if(likely(!facet_key_value_updated(k) && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX))
+ facets->keys_in_row.array[facets->keys_in_row.used++] = k;
+
+ k->current_value.flags |= FACET_KEY_VALUE_UPDATED | FACET_KEY_VALUE_UNSAMPLED;
+
+ facets->operations.values.registered++;
+ facets->operations.values.unsampled++;
+
+ // no need to copy the UNSET value
+ // empty values are exported as empty
+ k->current_value.raw = NULL;
+ k->current_value.raw_len = 0;
+ k->current_value.b->len = 0;
+ k->current_value.flags &= ~FACET_KEY_VALUE_COPIED;
+
+ if(unlikely(k->values.enabled))
+ FACET_VALUE_ADD_UNSAMPLED_VALUE_TO_INDEX(k);
+ else {
+ k->key_found_in_row++;
+ k->key_values_selected_in_row++;
+ }
+}
+
+static inline void facets_key_set_empty_value(FACETS *facets, FACET_KEY *k) {
+ if(likely(!facet_key_value_updated(k) && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX))
+ facets->keys_in_row.array[facets->keys_in_row.used++] = k;
+
+ k->current_value.flags |= FACET_KEY_VALUE_UPDATED | FACET_KEY_VALUE_EMPTY;
+
+ facets->operations.values.registered++;
+ facets->operations.values.empty++;
+
+ // no need to copy the UNSET value
+ // empty values are exported as empty
+ k->current_value.raw = NULL;
+ k->current_value.raw_len = 0;
+ k->current_value.b->len = 0;
+ k->current_value.flags &= ~FACET_KEY_VALUE_COPIED;
+
+ if(unlikely(k->values.enabled))
+ FACET_VALUE_ADD_EMPTY_VALUE_TO_INDEX(k);
+ else {
+ k->key_found_in_row++;
+ k->key_values_selected_in_row++;
+ }
+}
+
+static inline void facets_key_check_value(FACETS *facets, FACET_KEY *k) {
+ if(likely(!facet_key_value_updated(k) && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX))
+ facets->keys_in_row.array[facets->keys_in_row.used++] = k;
+
+ k->current_value.flags |= FACET_KEY_VALUE_UPDATED;
+ k->current_value.flags &= ~(FACET_KEY_VALUE_EMPTY|FACET_KEY_VALUE_UNSAMPLED|FACET_KEY_VALUE_ESTIMATED);
+
+ facets->operations.values.registered++;
+
+ if(k->transform.cb && !k->transform.view_only) {
+ facets->operations.values.transformed++;
+ facets_key_value_copy_to_buffer(k);
+ k->transform.cb(facets, k->current_value.b, FACETS_TRANSFORM_VALUE, k->transform.data);
+ }
+
+// bool found = false;
+// if(strstr(buffer_tostring(k->current_value), "fprintd") != NULL)
+// found = true;
+
+ if(facets->query && !facet_key_value_empty_or_unsampled_or_estimated(k) && ((k->options & FACET_KEY_OPTION_FTS) || facets->options & FACETS_OPTION_ALL_KEYS_FTS)) {
+ facets->operations.fts.searches++;
+ facets_key_value_copy_to_buffer(k);
+ switch(simple_pattern_matches_extract(facets->query, buffer_tostring(k->current_value.b), NULL, 0)) {
+ case SP_MATCHED_POSITIVE:
+ facets->current_row.keys_matched_by_query_positive++;
+ break;
+
+ case SP_MATCHED_NEGATIVE:
+ facets->current_row.keys_matched_by_query_negative++;
+ break;
+
+ case SP_NOT_MATCHED:
+ break;
+ }
+ }
+
+ if(k->values.enabled)
+ FACET_VALUE_ADD_CURRENT_VALUE_TO_INDEX(k);
+ else {
+ k->key_found_in_row++;
+ k->key_values_selected_in_row++;
+ }
+}
+
+void facets_add_key_value(FACETS *facets, const char *key, const char *value) {
+ FACET_KEY *k = facets_register_key_name(facets, key, 0);
+ k->current_value.raw = value;
+ k->current_value.raw_len = strlen(value);
+
+ facets_key_check_value(facets, k);
+}
+
+void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len) {
+ FACET_KEY *k = facets_register_key_name_length(facets, key, key_len, 0);
+ k->current_value.raw = value;
+ k->current_value.raw_len = value_len;
+
+ facets_key_check_value(facets, k);
+}
+
+// ----------------------------------------------------------------------------
+// FACET_ROW dictionary hooks
+
+static void facet_row_key_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
+ FACET_ROW_KEY_VALUE *rkv = value;
+ FACET_ROW *row = data; (void)row;
+
+ rkv->wb = buffer_create(0, NULL);
+ if(!rkv->empty)
+ buffer_contents_replace(rkv->wb, rkv->tmp, rkv->tmp_len);
+}
+
+static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) {
+ FACET_ROW_KEY_VALUE *rkv = old_value;
+ FACET_ROW_KEY_VALUE *n_rkv = new_value;
+ FACET_ROW *row = data; (void)row;
+
+ rkv->empty = n_rkv->empty;
+
+ if(!rkv->empty)
+ buffer_contents_replace(rkv->wb, n_rkv->tmp, n_rkv->tmp_len);
+ else
+ buffer_flush(rkv->wb);
+
+ return false;
+}
+
+static void facet_row_key_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
+ FACET_ROW_KEY_VALUE *rkv = value;
+ FACET_ROW *row = data; (void)row;
+
+ buffer_free(rkv->wb);
+}
+
+// ----------------------------------------------------------------------------
+// FACET_ROW management
+
+static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) {
+ dictionary_destroy(row->dict);
+ freez(row);
+}
+
+static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into) {
+ FACET_ROW *row;
+
+ if(into) {
+ row = into;
+ facets->operations.rows.reused++;
+ }
+ else {
+ row = callocz(1, sizeof(FACET_ROW));
+ row->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_ROW_KEY_VALUE));
+ dictionary_register_insert_callback(row->dict, facet_row_key_value_insert_callback, row);
+ dictionary_register_conflict_callback(row->dict, facet_row_key_value_conflict_callback, row);
+ dictionary_register_delete_callback(row->dict, facet_row_key_value_delete_callback, row);
+ facets->operations.rows.created++;
+ }
+
+ row->severity = facets->current_row.severity;
+ row->usec = usec;
+
+ FACET_KEY *k;
+ foreach_key_in_facets(facets, k) {
+ FACET_ROW_KEY_VALUE t = {
+ .tmp = NULL,
+ .tmp_len = 0,
+ .wb = NULL,
+ .empty = true,
+ };
+
+ if(facet_key_value_updated(k) && !facet_key_value_empty_or_unsampled_or_estimated(k)) {
+ t.tmp = facets_key_get_value(k);
+ t.tmp_len = facets_key_get_value_length(k);
+ t.empty = false;
+ }
+
+ dictionary_set(row->dict, k->name, &t, sizeof(t));
+ }
+ foreach_key_in_facets_done(k);
+
+ return row;
+}
+
+// ----------------------------------------------------------------------------
+
+static inline FACET_ROW *facets_row_keep_seek_to_position(FACETS *facets, usec_t usec) {
+ if(usec < facets->base->prev->usec)
+ return facets->base->prev;
+
+ if(usec > facets->base->usec)
+ return facets->base;
+
+ FACET_ROW *last = facets->operations.last_added;
+ while(last->prev != facets->base->prev && usec > last->prev->usec) {
+ last = last->prev;
+ facets->operations.backwards++;
+ }
+
+ while(last->next && usec < last->next->usec) {
+ last = last->next;
+ facets->operations.forwards++;
+ }
+
+ return last;
+}
+
+static void facets_row_keep_first_entry(FACETS *facets, usec_t usec) {
+ facets->operations.last_added = facets_row_create(facets, usec, NULL);
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);
+ facets->items_to_return++;
+ facets->operations.first++;
+}
+
+static inline bool facets_is_entry_within_anchor(FACETS *facets, usec_t usec) {
+ if(facets->anchor.start_ut || facets->anchor.stop_ut) {
+ // we have an anchor key
+ // we don't want to keep rows on the other side of the direction
+
+ switch (facets->anchor.direction) {
+ default:
+ case FACETS_ANCHOR_DIRECTION_BACKWARD:
+ // we need to keep only the smaller timestamps
+ if (facets->anchor.start_ut && usec >= facets->anchor.start_ut) {
+ facets->operations.skips_before++;
+ return false;
+ }
+ if (facets->anchor.stop_ut && usec <= facets->anchor.stop_ut) {
+ facets->operations.skips_after++;
+ return false;
+ }
+ break;
+
+ case FACETS_ANCHOR_DIRECTION_FORWARD:
+ // we need to keep only the bigger timestamps
+ if (facets->anchor.start_ut && usec <= facets->anchor.start_ut) {
+ facets->operations.skips_after++;
+ return false;
+ }
+ if (facets->anchor.stop_ut && usec >= facets->anchor.stop_ut) {
+ facets->operations.skips_before++;
+ return false;
+ }
+ break;
+ }
+ }
+
+ return true;
+}
+
+bool facets_row_candidate_to_keep(FACETS *facets, usec_t usec) {
+ return !facets->base ||
+ (usec >= facets->base->prev->usec && usec <= facets->base->usec && facets_is_entry_within_anchor(facets, usec)) ||
+ facets->items_to_return < facets->max_items_to_return;
+}
+
+static void facets_row_keep(FACETS *facets, usec_t usec) {
+ facets->operations.rows.matched++;
+
+ if(unlikely(!facets->base)) {
+ // the first row to keep
+ facets_row_keep_first_entry(facets, usec);
+ return;
+ }
+
+ FACET_ROW *closest = facets_row_keep_seek_to_position(facets, usec);
+ FACET_ROW *to_replace = NULL;
+
+ if(likely(facets->items_to_return >= facets->max_items_to_return)) {
+ // we have enough items to return already
+
+ switch(facets->anchor.direction) {
+ default:
+ case FACETS_ANCHOR_DIRECTION_BACKWARD:
+ if(closest == facets->base->prev && usec < closest->usec) {
+ // this is to the end of the list, belonging to the next page
+ facets->operations.skips_after++;
+ return;
+ }
+
+ // it seems we need to remove an item - the last one
+ to_replace = facets->base->prev;
+ if(closest == to_replace)
+ closest = to_replace->prev;
+
+ break;
+
+ case FACETS_ANCHOR_DIRECTION_FORWARD:
+ if(closest == facets->base && usec > closest->usec) {
+ // this is to the beginning of the list, belonging to the next page
+ facets->operations.skips_before++;
+ return;
+ }
+
+ // it seems we need to remove an item - the first one
+ to_replace = facets->base;
+ if(closest == to_replace)
+ closest = to_replace->next;
+
+ break;
+ }
+
+ facets->operations.shifts++;
+ facets->items_to_return--;
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, to_replace, prev, next);
+ }
+
+ internal_fatal(!closest, "FACETS: closest cannot be NULL");
+ internal_fatal(closest == to_replace, "FACETS: closest cannot be the same as to_replace");
+
+ facets->operations.last_added = facets_row_create(facets, usec, to_replace);
+
+ if(usec < closest->usec) {
+ DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next);
+ facets->operations.appends++;
+ }
+ else {
+ DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next);
+ facets->operations.prepends++;
+ }
+
+ facets->items_to_return++;
+}
+
+static inline void facets_reset_key(FACET_KEY *k) {
+ k->key_found_in_row = 0;
+ k->key_values_selected_in_row = 0;
+ k->current_value.flags = FACET_KEY_VALUE_NONE;
+ k->current_value.hash = FACETS_HASH_ZERO;
+}
+
+static void facets_reset_keys_with_value_and_row(FACETS *facets) {
+ size_t entries = facets->keys_in_row.used;
+
+ for(size_t p = 0; p < entries ;p++) {
+ FACET_KEY *k = facets->keys_in_row.array[p];
+ facets_reset_key(k);
+ }
+
+ facets->current_row.severity = FACET_ROW_SEVERITY_NORMAL;
+ facets->current_row.keys_matched_by_query_positive = 0;
+ facets->current_row.keys_matched_by_query_negative = 0;
+ facets->keys_in_row.used = 0;
+}
+
+void facets_rows_begin(FACETS *facets) {
+ FACET_KEY *k;
+ foreach_key_in_facets(facets, k) {
+ facets_reset_key(k);
+ }
+ foreach_key_in_facets_done(k);
+
+ facets->keys_in_row.used = 0;
+ facets_reset_keys_with_value_and_row(facets);
+}
+
+bool facets_row_finished(FACETS *facets, usec_t usec) {
+ facets->operations.rows.evaluated++;
+
+ if(unlikely((facets->query && facets->keys_filtered_by_query &&
+ (!facets->current_row.keys_matched_by_query_positive || facets->current_row.keys_matched_by_query_negative)) ||
+ (facets->timeframe.before_ut && usec > facets->timeframe.before_ut) ||
+ (facets->timeframe.after_ut && usec < facets->timeframe.after_ut))) {
+ // this row is not useful
+ // 1. not matched by full text search, or
+ // 2. not in our timeframe
+ facets_reset_keys_with_value_and_row(facets);
+ return false;
+ }
+
+ bool within_anchor = facets_is_entry_within_anchor(facets, usec);
+ if(unlikely(!within_anchor && (facets->options & FACETS_OPTION_DATA_ONLY))) {
+ facets_reset_keys_with_value_and_row(facets);
+ return false;
+ }
+
+ size_t entries = facets->keys_with_values.used;
+ size_t total_keys = 0;
+ size_t selected_keys = 0;
+
+ for(size_t p = 0; p < entries ;p++) {
+ FACET_KEY *k = facets->keys_with_values.array[p];
+
+ if(!facet_key_value_updated(k)) {
+ // put the FACET_VALUE_UNSET value into it
+ facets_key_set_empty_value(facets, k);
+ }
+
+ total_keys++;
+
+ if(k->key_values_selected_in_row)
+ selected_keys++;
+
+ if(unlikely(!facets->histogram.key && facets->histogram.hash == k->hash))
+ facets->histogram.key = k;
+ }
+
+ if(selected_keys >= total_keys - 1) {
+ size_t found = 0;
+ (void) found;
+
+ for(size_t p = 0; p < entries; p++) {
+ FACET_KEY *k = facets->keys_with_values.array[p];
+
+ size_t counted_by = selected_keys;
+
+ if(counted_by != total_keys && !k->key_values_selected_in_row)
+ counted_by++;
+
+ if(counted_by == total_keys) {
+ k->current_value.v->final_facet_value_counter++;
+ found++;
+ }
+ }
+
+ internal_fatal(!found, "We should find at least one facet to count this row");
+ }
+
+ if(selected_keys == total_keys) {
+ // we need to keep this row
+ facets_histogram_update_value(facets, usec);
+
+ if(within_anchor)
+ facets_row_keep(facets, usec);
+ }
+
+ facets_reset_keys_with_value_and_row(facets);
+
+ return selected_keys == total_keys;
+}
+
+// ----------------------------------------------------------------------------
+// output
+
+const char *facets_severity_to_string(FACET_ROW_SEVERITY severity) {
+ switch(severity) {
+ default:
+ case FACET_ROW_SEVERITY_NORMAL:
+ return "normal";
+
+ case FACET_ROW_SEVERITY_DEBUG:
+ return "debug";
+
+ case FACET_ROW_SEVERITY_NOTICE:
+ return "notice";
+
+ case FACET_ROW_SEVERITY_WARNING:
+ return "warning";
+
+ case FACET_ROW_SEVERITY_CRITICAL:
+ return "critical";
+ }
+}
+
+void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool with_keys) {
+ buffer_json_member_add_array(wb, "accepted_params");
+ {
+ if(facets->accepted_params) {
+ void *t;
+ dfe_start_read(facets->accepted_params, t) {
+ buffer_json_add_array_item_string(wb, t_dfe.name);
+ }
+ dfe_done(t);
+ }
+
+ if(with_keys) {
+ FACET_KEY *k;
+ foreach_key_in_facets(facets, k){
+ if (!k->values.enabled)
+ continue;
+
+ buffer_json_add_array_item_string(wb, hash_to_static_string(k->hash));
+ }
+ foreach_key_in_facets_done(k);
+ }
+ }
+ buffer_json_array_close(wb); // accepted_params
+}
+
+static int facets_keys_reorder_compar(const void *a, const void *b) {
+ const FACET_KEY *ak = *((const FACET_KEY **)a);
+ const FACET_KEY *bk = *((const FACET_KEY **)b);
+
+ const char *an = ak->name;
+ const char *bn = bk->name;
+
+ if(!an) an = "0";
+ if(!bn) bn = "0";
+
+ while(*an && ispunct(*an)) an++;
+ while(*bn && ispunct(*bn)) bn++;
+
+ return strcasecmp(an, bn);
+}
+
+void facets_sort_and_reorder_keys(FACETS *facets) {
+ size_t entries = facets->keys_with_values.used;
+ if(!entries)
+ return;
+
+ FACET_KEY *keys[entries];
+ memcpy(keys, facets->keys_with_values.array, sizeof(FACET_KEY *) * entries);
+
+ qsort(keys, entries, sizeof(FACET_KEY *), facets_keys_reorder_compar);
+
+ for(size_t i = 0; i < entries ;i++)
+ keys[i]->order = i + 1;
+}
+
+static int facets_key_values_reorder_by_name_compar(const void *a, const void *b) {
+ const FACET_VALUE *av = *((const FACET_VALUE **)a);
+ const FACET_VALUE *bv = *((const FACET_VALUE **)b);
+
+ const char *an = (av->name && av->name_len) ? av->name : "0";
+ const char *bn = (bv->name && bv->name_len) ? bv->name : "0";
+
+ while(*an && ispunct(*an)) an++;
+ while(*bn && ispunct(*bn)) bn++;
+
+ int ret = strcasecmp(an, bn);
+ return ret;
+}
+
+static int facets_key_values_reorder_by_count_compar(const void *a, const void *b) {
+ const FACET_VALUE *av = *((const FACET_VALUE **)a);
+ const FACET_VALUE *bv = *((const FACET_VALUE **)b);
+
+ if(av->final_facet_value_counter < bv->final_facet_value_counter)
+ return 1;
+
+ if(av->final_facet_value_counter > bv->final_facet_value_counter)
+ return -1;
+
+ return facets_key_values_reorder_by_name_compar(a, b);
+}
+
+static int facets_key_values_reorder_by_name_numeric_compar(const void *a, const void *b) {
+ const FACET_VALUE *av = *((const FACET_VALUE **)a);
+ const FACET_VALUE *bv = *((const FACET_VALUE **)b);
+
+ const char *an = (av->name && av->name_len) ? av->name : "0";
+ const char *bn = (bv->name && bv->name_len) ? bv->name : "0";
+
+ if(strcmp(an, FACET_VALUE_UNSET) == 0) an = "0";
+ if(strcmp(bn, FACET_VALUE_UNSET) == 0) bn = "0";
+
+ int64_t ad = str2ll(an, NULL);
+ int64_t bd = str2ll(bn, NULL);
+
+ if(ad < bd)
+ return -1;
+
+ if(ad > bd)
+ return 1;
+
+ return facets_key_values_reorder_by_name_compar(a, b);
+}
+
+static uint32_t facets_sort_and_reorder_values_internal(FACET_KEY *k) {
+ bool all_values_numeric = true;
+ size_t entries = k->values.used;
+ FACET_VALUE *values[entries], *v;
+ uint32_t used = 0;
+ foreach_value_in_key(k, v) {
+ if((k->facets->options & FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS) && v->empty)
+ continue;
+
+ if(all_values_numeric && !v->empty && v->name && v->name_len) {
+ const char *s = v->name;
+ while(isdigit(*s)) s++;
+ if(*s != '\0')
+ all_values_numeric = false;
+ }
+
+ values[used++] = v;
+
+ if(used >= entries)
+ break;
+ }
+ foreach_value_in_key_done(v);
+
+ if(!used)
+ return 0;
+
+ if(k->facets->options & FACETS_OPTION_SORT_FACETS_ALPHABETICALLY) {
+ if(all_values_numeric)
+ qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_name_numeric_compar);
+ else
+ qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_name_compar);
+ }
+ else
+ qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_count_compar);
+
+ for(size_t i = 0; i < used; i++)
+ values[i]->order = i + 1;
+
+ return used;
+}
+
+static uint32_t facets_sort_and_reorder_values(FACET_KEY *k) {
+ if(!k->values.enabled || !k->values.ll || !k->values.used)
+ return 0;
+
+ if(!k->transform.cb || !k->transform.view_only || !(k->facets->options & FACETS_OPTION_SORT_FACETS_ALPHABETICALLY))
+ return facets_sort_and_reorder_values_internal(k);
+
+ // we have a transformation and has to be sorted alphabetically
+
+ BUFFER *tb = buffer_create(0, NULL);
+ uint32_t ret = 0;
+
+ size_t entries = k->values.used;
+ struct {
+ const char *name;
+ uint32_t name_len;
+ } values[entries];
+ FACET_VALUE *v;
+ uint32_t used = 0;
+
+ foreach_value_in_key(k, v) {
+ if(used >= entries)
+ break;
+
+ values[used].name = v->name;
+ values[used].name_len = v->name_len;
+ used++;
+
+ facets_key_value_transformed(k->facets, k, v, tb, FACETS_TRANSFORM_FACET_SORT);
+ v->name = strdupz(buffer_tostring(tb));
+ v->name_len = buffer_strlen(tb);
+ }
+ foreach_value_in_key_done(v);
+
+ ret = facets_sort_and_reorder_values_internal(k);
+
+ used = 0;
+ foreach_value_in_key(k, v) {
+ if(used >= entries)
+ break;
+
+ freez((void *)v->name);
+ v->name = values[used].name;
+ v->name_len = values[used].name_len;
+ used++;
+ }
+ foreach_value_in_key_done(v);
+
+ buffer_free(tb);
+ return ret;
+}
+
+void facets_table_config(BUFFER *wb) {
+ buffer_json_member_add_boolean(wb, "show_ids", false); // do not show the column ids to the user
+ buffer_json_member_add_boolean(wb, "has_history", true); // enable date-time picker with after-before
+
+ buffer_json_member_add_object(wb, "pagination");
+ {
+ buffer_json_member_add_boolean(wb, "enabled", true);
+ buffer_json_member_add_string(wb, "key", "anchor");
+ buffer_json_member_add_string(wb, "column", "timestamp");
+ buffer_json_member_add_string(wb, "units", "timestamp_usec");
+ }
+ buffer_json_object_close(wb); // pagination
+}
+
+void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) {
+ facets->report.used_hashes_registry = used_hashes_registry;
+
+ if(!(facets->options & FACETS_OPTION_DATA_ONLY)) {
+ facets_table_config(wb);
+ facets_accepted_parameters_to_json_array(facets, wb, true);
+ }
+
+ // ------------------------------------------------------------------------
+ // facets
+
+ if(!(facets->options & FACETS_OPTION_DONT_SEND_FACETS)) {
+ bool show_facets = false;
+
+ if(facets->options & FACETS_OPTION_DATA_ONLY) {
+ if(facets->options & FACETS_OPTION_SHOW_DELTAS) {
+ buffer_json_member_add_array(wb, "facets_delta");
+ show_facets = true;
+ }
+ }
+ else {
+ buffer_json_member_add_array(wb, "facets");
+ show_facets = true;
+ }
+
+ if(show_facets) {
+ CLEAN_BUFFER *tb = buffer_create(0, NULL);
+ FACET_KEY *k;
+ foreach_key_in_facets(facets, k) {
+ if(!k->values.enabled)
+ continue;
+
+ if(!facets_sort_and_reorder_values(k))
+ // no values for this key
+ continue;
+
+ buffer_json_add_array_item_object(wb); // key
+ {
+ buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash));
+ buffer_json_member_add_string(wb, "name", facets_key_name_cached(k
+ , facets->report.used_hashes_registry
+ ));
+
+ if(!k->order) k->order = facets->order++;
+ buffer_json_member_add_uint64(wb, "order", k->order);
+
+ buffer_json_member_add_array(wb, "options");
+ {
+ FACET_VALUE *v;
+ foreach_value_in_key(k, v) {
+ if((facets->options & FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS) && v->empty)
+ continue;
+
+ if(v->unsampled || v->estimated)
+ continue;
+
+ buffer_json_add_array_item_object(wb);
+ {
+ buffer_json_member_add_string(wb, "id", hash_to_static_string(v->hash));
+
+ facets_key_value_transformed(facets, k, v, tb, FACETS_TRANSFORM_FACET);
+ buffer_json_member_add_string(wb, "name", buffer_tostring(tb));
+ buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter);
+ buffer_json_member_add_uint64(wb, "order", v->order);
+ }
+ buffer_json_object_close(wb);
+ }
+ foreach_value_in_key_done(v);
+ }
+ buffer_json_array_close(wb); // options
+ }
+ buffer_json_object_close(wb); // key
+ }
+ foreach_key_in_facets_done(k);
+ buffer_json_array_close(wb); // facets
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // columns
+
+ buffer_json_member_add_object(wb, "columns");
+ {
+ size_t field_id = 0;
+ buffer_rrdf_table_add_field(
+ wb, field_id++,
+ "timestamp", "Timestamp",
+ RRDF_FIELD_TYPE_TIMESTAMP,
+ RRDF_FIELD_VISUAL_VALUE,
+ RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN,
+ RRDF_FIELD_SORT_DESCENDING|RRDF_FIELD_SORT_FIXED,
+ NULL,
+ RRDF_FIELD_SUMMARY_COUNT,
+ RRDF_FIELD_FILTER_RANGE,
+ RRDF_FIELD_OPTS_WRAP | RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY,
+ NULL);
+
+ buffer_rrdf_table_add_field(
+ wb, field_id++,
+ "rowOptions", "rowOptions",
+ RRDF_FIELD_TYPE_NONE,
+ RRDR_FIELD_VISUAL_ROW_OPTIONS,
+ RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN,
+ RRDF_FIELD_SORT_FIXED,
+ NULL,
+ RRDF_FIELD_SUMMARY_COUNT,
+ RRDF_FIELD_FILTER_NONE,
+ RRDF_FIELD_OPTS_DUMMY,
+ NULL);
+
+ FACET_KEY *k;
+ foreach_key_in_facets(facets, k) {
+ RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP;
+ bool visible = k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY);
+
+ if ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values.enabled))
+ visible = true;
+
+ if (!visible)
+ visible = simple_pattern_matches(facets->visible_keys, k->name);
+
+ if (visible)
+ options |= RRDF_FIELD_OPTS_VISIBLE;
+
+ if (k->options & FACET_KEY_OPTION_MAIN_TEXT)
+ options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;
+
+ if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER)
+ options |= RRDF_FIELD_OPTS_EXPANDED_FILTER;
+
+ const char *hash_str = hash_to_static_string(k->hash);
+
+ buffer_rrdf_table_add_field(
+ wb, field_id++,
+ hash_str, k->name ? k->name : hash_str,
+ RRDF_FIELD_TYPE_STRING,
+ (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE,
+ RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN,
+ RRDF_FIELD_SORT_FIXED,
+ NULL,
+ RRDF_FIELD_SUMMARY_COUNT,
+ (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET,
+ options, FACET_VALUE_UNSET);
+ }
+ foreach_key_in_facets_done(k);
+ }
+ buffer_json_object_close(wb); // columns
+
+ // ------------------------------------------------------------------------
+ // rows data
+
+ buffer_json_member_add_array(wb, "data");
+ {
+ usec_t last_usec = 0; (void)last_usec;
+
+ for(FACET_ROW *row = facets->base ; row ;row = row->next) {
+
+ internal_fatal(
+ facets->anchor.start_ut && (
+ (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_BACKWARD && row->usec >= facets->anchor.start_ut) ||
+ (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD && row->usec <= facets->anchor.start_ut)
+ ), "Wrong data returned related to %s start anchor!", facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward");
+
+ internal_fatal(last_usec && row->usec > last_usec, "Wrong order of data returned!");
+
+ last_usec = row->usec;
+
+ buffer_json_add_array_item_array(wb); // each row
+ buffer_json_add_array_item_uint64(wb, row->usec);
+ buffer_json_add_array_item_object(wb);
+ {
+ if(facets->severity.cb)
+ row->severity = facets->severity.cb(facets, row, facets->severity.data);
+
+ buffer_json_member_add_string(wb, "severity", facets_severity_to_string(row->severity));
+ }
+ buffer_json_object_close(wb);
+
+ FACET_KEY *k;
+ foreach_key_in_facets(facets, k) {
+ FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name);
+
+ if(unlikely(k->dynamic.cb)) {
+ if(unlikely(!rkv))
+ rkv = dictionary_set(row->dict, k->name, NULL, sizeof(*rkv));
+
+ k->dynamic.cb(facets, wb, rkv, row, k->dynamic.data);
+ facets->operations.values.dynamic++;
+ }
+ else {
+ if(!rkv || rkv->empty) {
+ buffer_json_add_array_item_string(wb, NULL);
+ }
+ else if(unlikely(k->transform.cb && k->transform.view_only)) {
+ k->transform.cb(facets, rkv->wb, FACETS_TRANSFORM_DATA, k->transform.data);
+ buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb));
+ }
+ else
+ buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb));
+ }
+ }
+ foreach_key_in_facets_done(k);
+ buffer_json_array_close(wb); // each row
+ }
+ }
+ buffer_json_array_close(wb); // data
+
+ if(!(facets->options & FACETS_OPTION_DATA_ONLY)) {
+ buffer_json_member_add_string(wb, "default_sort_column", "timestamp");
+ buffer_json_member_add_array(wb, "default_charts");
+ buffer_json_array_close(wb);
+ }
+
+ // ------------------------------------------------------------------------
+ // histogram
+
+ if(facets->histogram.enabled && !(facets->options & FACETS_OPTION_DONT_SEND_HISTOGRAM)) {
+ FACETS_HASH first_histogram_hash = 0;
+ buffer_json_member_add_array(wb, "available_histograms");
+ {
+ FACET_KEY *k;
+ foreach_key_in_facets(facets, k) {
+ if (!k->values.enabled)
+ continue;
+
+ if(unlikely(!first_histogram_hash))
+ first_histogram_hash = k->hash;
+
+ buffer_json_add_array_item_object(wb);
+ buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash));
+ buffer_json_member_add_string(wb, "name", k->name);
+ buffer_json_member_add_uint64(wb, "order", k->order);
+ buffer_json_object_close(wb);
+ }
+ foreach_key_in_facets_done(k);
+ }
+ buffer_json_array_close(wb);
+
+ {
+ FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, facets->histogram.hash);
+ if(!k || !k->values.enabled)
+ k = FACETS_KEY_GET_FROM_INDEX(facets, first_histogram_hash);
+
+ bool show_histogram = false;
+
+ if(facets->options & FACETS_OPTION_DATA_ONLY) {
+ if(facets->options & FACETS_OPTION_SHOW_DELTAS) {
+ buffer_json_member_add_object(wb, "histogram_delta");
+ show_histogram = true;
+ }
+ }
+ else {
+ buffer_json_member_add_object(wb, "histogram");
+ show_histogram = true;
+ }
+
+ if(show_histogram) {
+ buffer_json_member_add_string(wb, "id", k ? hash_to_static_string(k->hash) : "");
+ buffer_json_member_add_string(wb, "name", k ? k->name : "");
+ buffer_json_member_add_object(wb, "chart");
+ {
+ facets_histogram_generate(facets, k, wb);
+ }
+ buffer_json_object_close(wb); // chart
+ buffer_json_object_close(wb); // histogram
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // items
+
+ bool show_items = false;
+ if(facets->options & FACETS_OPTION_DATA_ONLY) {
+ if(facets->options & FACETS_OPTION_SHOW_DELTAS) {
+ buffer_json_member_add_object(wb, "items_delta");
+ show_items = true;
+ }
+ }
+ else {
+ buffer_json_member_add_object(wb, "items");
+ show_items = true;
+ }
+
+ if(show_items) {
+ buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated);
+ buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched);
+ buffer_json_member_add_uint64(wb, "unsampled", facets->operations.rows.unsampled);
+ buffer_json_member_add_uint64(wb, "estimated", facets->operations.rows.estimated);
+ buffer_json_member_add_uint64(wb, "returned", facets->items_to_return);
+ buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return);
+ buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before);
+ buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts);
+ buffer_json_object_close(wb); // items
+ }
+
+ // ------------------------------------------------------------------------
+ // stats
+
+ buffer_json_member_add_object(wb, "_stats");
+ {
+ buffer_json_member_add_uint64(wb, "first", facets->operations.first);
+ buffer_json_member_add_uint64(wb, "forwards", facets->operations.forwards);
+ buffer_json_member_add_uint64(wb, "backwards", facets->operations.backwards);
+ buffer_json_member_add_uint64(wb, "skips_before", facets->operations.skips_before);
+ buffer_json_member_add_uint64(wb, "skips_after", facets->operations.skips_after);
+ buffer_json_member_add_uint64(wb, "prepends", facets->operations.prepends);
+ buffer_json_member_add_uint64(wb, "appends", facets->operations.appends);
+ buffer_json_member_add_uint64(wb, "shifts", facets->operations.shifts);
+ buffer_json_member_add_object(wb, "rows");
+ {
+ buffer_json_member_add_uint64(wb, "created", facets->operations.rows.created);
+ buffer_json_member_add_uint64(wb, "reused", facets->operations.rows.reused);
+ buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated);
+ buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched);
+ }
+ buffer_json_object_close(wb); // rows
+ buffer_json_member_add_object(wb, "keys");
+ {
+ size_t resizes = 0, searches = 0, collisions = 0, used = 0, size = 0, count = 0;
+ count++;
+ used += facets->keys.ht.used;
+ size += facets->keys.ht.size;
+ resizes += facets->keys.ht.resizes;
+ searches += facets->keys.ht.searches;
+ collisions += facets->keys.ht.collisions;
+
+ buffer_json_member_add_uint64(wb, "registered", facets->operations.keys.registered);
+ buffer_json_member_add_uint64(wb, "unique", facets->operations.keys.unique);
+ buffer_json_member_add_uint64(wb, "hashtables", count);
+ buffer_json_member_add_uint64(wb, "hashtable_used", used);
+ buffer_json_member_add_uint64(wb, "hashtable_size", size);
+ buffer_json_member_add_uint64(wb, "hashtable_searches", searches);
+ buffer_json_member_add_uint64(wb, "hashtable_collisions", collisions);
+ buffer_json_member_add_uint64(wb, "hashtable_resizes", resizes);
+ }
+ buffer_json_object_close(wb); // keys
+ buffer_json_member_add_object(wb, "values");
+ {
+ size_t resizes = 0, searches = 0, collisions = 0, used = 0, size = 0, count = 0;
+ for(FACET_KEY *k = facets->keys.ll; k ; k = k->next) {
+ count++;
+ used += k->values.ht.used;
+ size += k->values.ht.size;
+ resizes += k->values.ht.resizes;
+ searches += k->values.ht.searches;
+ collisions += k->values.ht.collisions;
+ }
+
+ buffer_json_member_add_uint64(wb, "registered", facets->operations.values.registered);
+ buffer_json_member_add_uint64(wb, "transformed", facets->operations.values.transformed);
+ buffer_json_member_add_uint64(wb, "dynamic", facets->operations.values.dynamic);
+ buffer_json_member_add_uint64(wb, "empty", facets->operations.values.empty);
+ buffer_json_member_add_uint64(wb, "unsampled", facets->operations.values.unsampled);
+ buffer_json_member_add_uint64(wb, "estimated", facets->operations.values.estimated);
+ buffer_json_member_add_uint64(wb, "indexed", facets->operations.values.indexed);
+ buffer_json_member_add_uint64(wb, "inserts", facets->operations.values.inserts);
+ buffer_json_member_add_uint64(wb, "conflicts", facets->operations.values.conflicts);
+ buffer_json_member_add_uint64(wb, "hashtables", count);
+ buffer_json_member_add_uint64(wb, "hashtable_used", used);
+ buffer_json_member_add_uint64(wb, "hashtable_size", size);
+ buffer_json_member_add_uint64(wb, "hashtable_searches", searches);
+ buffer_json_member_add_uint64(wb, "hashtable_collisions", collisions);
+ buffer_json_member_add_uint64(wb, "hashtable_resizes", resizes);
+ }
+ buffer_json_object_close(wb); // values
+ buffer_json_member_add_object(wb, "fts");
+ {
+ buffer_json_member_add_uint64(wb, "searches", facets->operations.fts.searches);
+ }
+ buffer_json_object_close(wb); // fts
+ }
+ buffer_json_object_close(wb); // items
+}
diff --git a/libnetdata/facets/facets.h b/libnetdata/facets/facets.h
new file mode 100644
index 00000000..8364d861
--- /dev/null
+++ b/libnetdata/facets/facets.h
@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef FACETS_H
+#define FACETS_H 1
+
+#include "../libnetdata.h"
+
+#define FACET_VALUE_UNSET "-"
+#define FACET_VALUE_UNSAMPLED "[unsampled]"
+#define FACET_VALUE_ESTIMATED "[estimated]"
+
+typedef enum __attribute__((packed)) {
+ FACETS_ANCHOR_DIRECTION_FORWARD,
+ FACETS_ANCHOR_DIRECTION_BACKWARD,
+} FACETS_ANCHOR_DIRECTION;
+
+typedef enum __attribute__((packed)) {
+ FACETS_TRANSFORM_VALUE,
+ FACETS_TRANSFORM_HISTOGRAM,
+ FACETS_TRANSFORM_FACET,
+ FACETS_TRANSFORM_DATA,
+ FACETS_TRANSFORM_FACET_SORT,
+} FACETS_TRANSFORMATION_SCOPE;
+
+typedef enum __attribute__((packed)) {
+ FACET_KEY_OPTION_FACET = (1 << 0), // filterable values
+ FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value
+ FACET_KEY_OPTION_NEVER_FACET = (1 << 2), // never enable this field as facet
+ FACET_KEY_OPTION_STICKY = (1 << 3), // should be sticky in the table
+ FACET_KEY_OPTION_VISIBLE = (1 << 4), // should be in the default table
+ FACET_KEY_OPTION_FTS = (1 << 5), // the key is filterable by full text search (FTS)
+ FACET_KEY_OPTION_MAIN_TEXT = (1 << 6), // full width and wrap
+ FACET_KEY_OPTION_RICH_TEXT = (1 << 7),
+ FACET_KEY_OPTION_REORDER = (1 << 8), // give the key a new order id on first encounter
+ FACET_KEY_OPTION_TRANSFORM_VIEW = (1 << 9), // when registering the transformation, do it only at the view, not on all data
+ FACET_KEY_OPTION_EXPANDED_FILTER = (1 << 10), // the presentation should have this filter expanded by default
+} FACET_KEY_OPTIONS;
+
+typedef enum __attribute__((packed)) {
+ FACET_ROW_SEVERITY_DEBUG, // lowest - not important
+ FACET_ROW_SEVERITY_NORMAL, // the default
+ FACET_ROW_SEVERITY_NOTICE, // bold
+ FACET_ROW_SEVERITY_WARNING, // yellow + bold
+ FACET_ROW_SEVERITY_CRITICAL, // red + bold
+} FACET_ROW_SEVERITY;
+
+typedef struct facet_row_key_value {
+ const char *tmp;
+ uint32_t tmp_len;
+ bool empty;
+ BUFFER *wb;
+} FACET_ROW_KEY_VALUE;
+
+typedef struct facet_row {
+ usec_t usec;
+ DICTIONARY *dict;
+ FACET_ROW_SEVERITY severity;
+ struct facet_row *prev, *next;
+} FACET_ROW;
+
+typedef struct facets FACETS;
+typedef struct facet_key FACET_KEY;
+
+typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
+typedef void (*facet_dynamic_row_t)(FACETS *facets, BUFFER *json_array, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data);
+typedef FACET_ROW_SEVERITY (*facet_row_severity_t)(FACETS *facets, FACET_ROW *row, void *data);
+FACET_KEY *facets_register_dynamic_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data);
+FACET_KEY *facets_register_key_name_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data);
+void facets_register_row_severity(FACETS *facets, facet_row_severity_t cb, void *data);
+
+typedef enum __attribute__((packed)) {
+ FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets should be visible by default in the table
+ FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search
+ FACETS_OPTION_DONT_SEND_FACETS = (1 << 2), // "facets" object will not be included in the report
+ FACETS_OPTION_DONT_SEND_HISTOGRAM = (1 << 3), // "histogram" object will not be included in the report
+ FACETS_OPTION_DATA_ONLY = (1 << 4),
+ FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS = (1 << 5), // empty facet values will not be included in the report
+ FACETS_OPTION_SORT_FACETS_ALPHABETICALLY = (1 << 6),
+ FACETS_OPTION_SHOW_DELTAS = (1 << 7),
+} FACETS_OPTIONS;
+
+FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys);
+void facets_destroy(FACETS *facets);
+
+void facets_accepted_param(FACETS *facets, const char *param);
+
+void facets_rows_begin(FACETS *facets);
+bool facets_row_finished(FACETS *facets, usec_t usec);
+
+void facets_row_finished_unsampled(FACETS *facets, usec_t usec);
+void facets_update_estimations(FACETS *facets, usec_t from_ut, usec_t to_ut, size_t entries);
+size_t facets_histogram_slots(FACETS *facets);
+
+FACET_KEY *facets_register_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options);
+void facets_set_query(FACETS *facets, const char *query);
+void facets_set_items(FACETS *facets, uint32_t items);
+void facets_set_anchor(FACETS *facets, usec_t start_ut, usec_t stop_ut, FACETS_ANCHOR_DIRECTION direction);
+void facets_enable_slice_mode(FACETS *facets);
+bool facets_row_candidate_to_keep(FACETS *facets, usec_t usec);
+
+FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options);
+void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *value_id, FACET_KEY_OPTIONS options);
+void facets_set_timeframe_and_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut);
+void facets_set_timeframe_and_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut);
+
+void facets_add_key_value(FACETS *facets, const char *key, const char *value);
+void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len);
+
+void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry);
+void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool with_keys);
+void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity);
+void facets_set_additional_options(FACETS *facets, FACETS_OPTIONS options);
+
+bool facets_key_name_is_filter(FACETS *facets, const char *key);
+bool facets_key_name_is_facet(FACETS *facets, const char *key);
+bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length);
+void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length);
+
+void facets_sort_and_reorder_keys(FACETS *facets);
+usec_t facets_row_oldest_ut(FACETS *facets);
+usec_t facets_row_newest_ut(FACETS *facets);
+uint32_t facets_rows(FACETS *facets);
+
+void facets_table_config(BUFFER *wb);
+
+const char *facets_severity_to_string(FACET_ROW_SEVERITY severity);
+
+#endif
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-08 20:23:30 +0000