diff options
Diffstat (limited to 'libnetdata/facets')
| -rw-r--r-- | libnetdata/facets/Makefile.am | 8 | ||||
| -rw-r--r-- | libnetdata/facets/README.md | 0 | ||||
| -rw-r--r-- | libnetdata/facets/facets.c | 851 | ||||
| -rw-r--r-- | libnetdata/facets/facets.h | 62 |
4 files changed, 921 insertions, 0 deletions
diff --git a/libnetdata/facets/Makefile.am b/libnetdata/facets/Makefile.am new file mode 100644 index 000000000..161784b8f --- /dev/null +++ b/libnetdata/facets/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/facets/README.md b/libnetdata/facets/README.md new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/libnetdata/facets/README.md diff --git a/libnetdata/facets/facets.c b/libnetdata/facets/facets.c new file mode 100644 index 000000000..8762b43b9 --- /dev/null +++ b/libnetdata/facets/facets.c @@ -0,0 +1,851 @@ +#include "facets.h" + +#define FACET_VALUE_UNSET "-" +#define HISTOGRAM_COLUMNS 60 + +static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row); + +// ---------------------------------------------------------------------------- + +time_t calculate_bar_width(time_t before, time_t after) { + // Array of valid durations in seconds + static time_t valid_durations[] = { + 1, + 15, + 30, + 1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes + 1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours + 1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days + 1 * (30*86400) // months + }; + static int array_size = sizeof(valid_durations) / sizeof(valid_durations[0]); + + time_t duration = before - after; + time_t bar_width = 1; + + for (int i = array_size - 1; i >= 0; --i) { + if (duration / valid_durations[i] >= HISTOGRAM_COLUMNS) { + bar_width = valid_durations[i]; + break; + } + } + + return bar_width; +} + +// ---------------------------------------------------------------------------- + +static inline void uint32_to_char(uint32_t num, char *out) { + static char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789"; + + int i; + for(i = 5; i >= 0; --i) { + out[i] = id_encoding_characters[num & 63]; + num >>= 6; + } + out[6] = '\0'; +} + +inline void facets_string_hash(const char *src, char *out) { + uint32_t hash1 = fnv1a_hash32(src); + uint32_t hash2 = djb2_hash32(src); + uint32_t hash3 = larson_hash32(src); + + uint32_to_char(hash1, out); + uint32_to_char(hash2, &out[6]); + uint32_to_char(hash3, &out[12]); + + out[18] = '\0'; +} + +// ---------------------------------------------------------------------------- + +typedef struct facet_value { + const char *name; + + bool selected; + + uint32_t rows_matching_facet_value; + uint32_t final_facet_value_counter; +} FACET_VALUE; + +struct facet_key { + const char *name; + + DICTIONARY *values; + + FACET_KEY_OPTIONS options; + + bool default_selected_for_values; // the default "selected" for all values in the dictionary + + // members about the current row + uint32_t key_found_in_row; + uint32_t key_values_selected_in_row; + + struct { + char hash[FACET_STRING_HASH_SIZE]; + bool updated; + BUFFER *b; + } current_value; + + uint32_t order; + + struct { + facet_dynamic_row_t cb; + void *data; + } dynamic; + + struct { + facets_key_transformer_t cb; + void *data; + } transform; + + struct facet_key *prev, *next; +}; + +struct facets { + SIMPLE_PATTERN *visible_keys; + SIMPLE_PATTERN *excluded_keys; + SIMPLE_PATTERN *included_keys; + + FACETS_OPTIONS options; + usec_t anchor; + + SIMPLE_PATTERN *query; // the full text search pattern + size_t keys_filtered_by_query; // the number of fields we do full text search (constant) + size_t keys_matched_by_query; // the number of fields matched the full text search (per row) + + DICTIONARY *accepted_params; + + FACET_KEY *keys_ll; + DICTIONARY *keys; + FACET_ROW *base; // double linked list of the selected facets rows + + uint32_t items_to_return; + uint32_t max_items_to_return; + uint32_t order; + + struct { + FACET_ROW *last_added; + + size_t evaluated; + size_t matched; + + size_t first; + size_t forwards; + size_t backwards; + size_t skips_before; + size_t skips_after; + size_t prepends; + size_t appends; + size_t shifts; + } operations; +}; + +// ---------------------------------------------------------------------------- + +static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) { + if(!k->key_found_in_row) + v->rows_matching_facet_value++; + + k->key_found_in_row++; + + if(v->selected) + k->key_values_selected_in_row++; +} + +static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) { + bool included = true, excluded = false; + + if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET)) { + if(k->options & FACET_KEY_OPTION_FACET) { + included = true; + excluded = false; + } + else if(k->options & FACET_KEY_OPTION_NO_FACET) { + included = false; + excluded = true; + } + } + else { + if (facets->included_keys) { + if (!simple_pattern_matches(facets->included_keys, k->name)) + included = false; + } + + if (facets->excluded_keys) { + if (simple_pattern_matches(facets->excluded_keys, k->name)) + excluded = true; + } + } + + if(included && !excluded) { + k->options |= FACET_KEY_OPTION_FACET; + k->options &= ~FACET_KEY_OPTION_NO_FACET; + return true; + } + + k->options |= FACET_KEY_OPTION_NO_FACET; + k->options &= ~FACET_KEY_OPTION_FACET; + return false; +} + +// ---------------------------------------------------------------------------- +// FACET_VALUE dictionary hooks + +static void facet_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + FACET_VALUE *v = value; + FACET_KEY *k = data; + + if(v->name) { + // an actual value, not a filter + v->name = strdupz(v->name); + facet_value_is_used(k, v); + } + + if(!v->selected) + v->selected = k->default_selected_for_values; +} + +static bool facet_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) { + FACET_VALUE *v = old_value; + FACET_VALUE *nv = new_value; + FACET_KEY *k = data; + + if(!v->name && nv->name) + // an actual value, not a filter + v->name = strdupz(nv->name); + + if(v->name) + facet_value_is_used(k, v); + + return false; +} + +static void facet_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + FACET_VALUE *v = value; + freez((char *)v->name); +} + +// ---------------------------------------------------------------------------- +// FACET_KEY dictionary hooks + +static inline void facet_key_late_init(FACETS *facets, FACET_KEY *k) { + if(k->values) + return; + + if(facets_key_is_facet(facets, k)) { + k->values = dictionary_create_advanced( + DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, sizeof(FACET_VALUE)); + dictionary_register_insert_callback(k->values, facet_value_insert_callback, k); + dictionary_register_conflict_callback(k->values, facet_value_conflict_callback, k); + dictionary_register_delete_callback(k->values, facet_value_delete_callback, k); + } +} + +static void facet_key_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + FACET_KEY *k = value; + FACETS *facets = data; + + if(!(k->options & FACET_KEY_OPTION_REORDER)) + k->order = facets->order++; + + if((k->options & FACET_KEY_OPTION_FTS) || (facets->options & FACETS_OPTION_ALL_KEYS_FTS)) + facets->keys_filtered_by_query++; + + if(k->name) { + // an actual value, not a filter + k->name = strdupz(k->name); + facet_key_late_init(facets, k); + } + + k->current_value.b = buffer_create(0, NULL); + + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->keys_ll, k, prev, next); +} + +static bool facet_key_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) { + FACET_KEY *k = old_value; + FACET_KEY *nk = new_value; + FACETS *facets = data; + + if(!k->name && nk->name) { + // an actual value, not a filter + k->name = strdupz(nk->name); + facet_key_late_init(facets, k); + } + + if(k->options & FACET_KEY_OPTION_REORDER) { + k->order = facets->order++; + k->options &= ~FACET_KEY_OPTION_REORDER; + } + + return false; +} + +static void facet_key_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) { + FACET_KEY *k = value; + FACETS *facets = data; + + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->keys_ll, k, prev, next); + + dictionary_destroy(k->values); + buffer_free(k->current_value.b); + freez((char *)k->name); +} + +// ---------------------------------------------------------------------------- + +FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) { + FACETS *facets = callocz(1, sizeof(FACETS)); + facets->options = options; + facets->keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_KEY)); + dictionary_register_insert_callback(facets->keys, facet_key_insert_callback, facets); + dictionary_register_conflict_callback(facets->keys, facet_key_conflict_callback, facets); + dictionary_register_delete_callback(facets->keys, facet_key_delete_callback, facets); + + if(facet_keys && *facet_keys) + facets->included_keys = simple_pattern_create(facet_keys, "|", SIMPLE_PATTERN_EXACT, true); + + if(non_facet_keys && *non_facet_keys) + facets->excluded_keys = simple_pattern_create(non_facet_keys, "|", SIMPLE_PATTERN_EXACT, true); + + if(visible_keys && *visible_keys) + facets->visible_keys = simple_pattern_create(visible_keys, "|", SIMPLE_PATTERN_EXACT, true); + + facets->max_items_to_return = items_to_return; + facets->anchor = anchor; + facets->order = 1; + + return facets; +} + +void facets_destroy(FACETS *facets) { + dictionary_destroy(facets->accepted_params); + dictionary_destroy(facets->keys); + simple_pattern_free(facets->visible_keys); + simple_pattern_free(facets->included_keys); + simple_pattern_free(facets->excluded_keys); + + while(facets->base) { + FACET_ROW *r = facets->base; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, r, prev, next); + + facets_row_free(facets, r); + } + + freez(facets); +} + +void facets_accepted_param(FACETS *facets, const char *param) { + if(!facets->accepted_params) + facets->accepted_params = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE); + + dictionary_set(facets->accepted_params, param, NULL, 0); +} + +inline FACET_KEY *facets_register_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options) { + FACET_KEY tk = { + .name = key, + .options = options, + .default_selected_for_values = true, + }; + char hash[FACET_STRING_HASH_SIZE]; + facets_string_hash(tk.name, hash); + return dictionary_set(facets->keys, hash, &tk, sizeof(tk)); +} + +inline FACET_KEY *facets_register_key_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data) { + FACET_KEY *k = facets_register_key(facets, key, options); + k->transform.cb = cb; + k->transform.data = data; + return k; +} + +inline FACET_KEY *facets_register_dynamic_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data) { + FACET_KEY *k = facets_register_key(facets, key, options); + k->dynamic.cb = cb; + k->dynamic.data = data; + return k; +} + +void facets_set_query(FACETS *facets, const char *query) { + if(!query) + return; + + facets->query = simple_pattern_create(query, " \t", SIMPLE_PATTERN_SUBSTRING, false); +} + +void facets_set_items(FACETS *facets, uint32_t items) { + facets->max_items_to_return = items; +} + +void facets_set_anchor(FACETS *facets, usec_t anchor) { + facets->anchor = anchor; +} + +void facets_register_facet_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options) { + FACET_KEY tk = { + .options = options, + }; + FACET_KEY *k = dictionary_set(facets->keys, key_id, &tk, sizeof(tk)); + + k->default_selected_for_values = false; + k->options |= FACET_KEY_OPTION_FACET; + k->options &= ~FACET_KEY_OPTION_NO_FACET; + facet_key_late_init(facets, k); + + FACET_VALUE tv = { + .selected = true, + }; + dictionary_set(k->values, value_ids, &tv, sizeof(tv)); +} + +// ---------------------------------------------------------------------------- + +static inline void facets_check_value(FACETS *facets __maybe_unused, FACET_KEY *k) { + if(!k->current_value.updated) + buffer_flush(k->current_value.b); + + if(k->transform.cb) + k->transform.cb(facets, k->current_value.b, k->transform.data); + + if(!k->current_value.updated) { + buffer_strcat(k->current_value.b, FACET_VALUE_UNSET); + k->current_value.updated = true; + } + +// bool found = false; +// if(strstr(buffer_tostring(k->current_value), "fprintd") != NULL) +// found = true; + + if(facets->query && ((k->options & FACET_KEY_OPTION_FTS) || facets->options & FACETS_OPTION_ALL_KEYS_FTS)) { + if(simple_pattern_matches(facets->query, buffer_tostring(k->current_value.b))) + facets->keys_matched_by_query++; + } + + if(k->values) { + FACET_VALUE tk = { + .name = buffer_tostring(k->current_value.b), + }; + facets_string_hash(tk.name, k->current_value.hash); + dictionary_set(k->values, k->current_value.hash, &tk, sizeof(tk)); + } + else { + k->key_found_in_row++; + k->key_values_selected_in_row++; + } +} + +void facets_add_key_value(FACETS *facets, const char *key, const char *value) { + FACET_KEY *k = facets_register_key(facets, key, 0); + buffer_flush(k->current_value.b); + buffer_strcat(k->current_value.b, value); + k->current_value.updated = true; + + facets_check_value(facets, k); +} + +void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len) { + FACET_KEY *k = facets_register_key(facets, key, 0); + buffer_flush(k->current_value.b); + buffer_strncat(k->current_value.b, value, value_len); + k->current_value.updated = true; + + facets_check_value(facets, k); +} + +// ---------------------------------------------------------------------------- +// FACET_ROW dictionary hooks + +static void facet_row_key_value_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + FACET_ROW_KEY_VALUE *rkv = value; + FACET_ROW *row = data; (void)row; + + rkv->wb = buffer_create(0, NULL); + buffer_strcat(rkv->wb, rkv->tmp && *rkv->tmp ? rkv->tmp : FACET_VALUE_UNSET); +} + +static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) { + FACET_ROW_KEY_VALUE *rkv = old_value; + FACET_ROW_KEY_VALUE *n_rkv = new_value; + FACET_ROW *row = data; (void)row; + + buffer_flush(rkv->wb); + buffer_strcat(rkv->wb, n_rkv->tmp && *n_rkv->tmp ? n_rkv->tmp : FACET_VALUE_UNSET); + + return false; +} + +static void facet_row_key_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) { + FACET_ROW_KEY_VALUE *rkv = value; + FACET_ROW *row = data; (void)row; + + buffer_free(rkv->wb); +} + +// ---------------------------------------------------------------------------- +// FACET_ROW management + +static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) { + dictionary_destroy(row->dict); + freez(row); +} + +static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into) { + FACET_ROW *row; + + if(into) + row = into; + else { + row = callocz(1, sizeof(FACET_ROW)); + row->dict = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_ROW_KEY_VALUE)); + dictionary_register_insert_callback(row->dict, facet_row_key_value_insert_callback, row); + dictionary_register_conflict_callback(row->dict, facet_row_key_value_conflict_callback, row); + dictionary_register_delete_callback(row->dict, facet_row_key_value_delete_callback, row); + } + + row->usec = usec; + + FACET_KEY *k; + dfe_start_read(facets->keys, k) { + FACET_ROW_KEY_VALUE t = { + .tmp = (k->current_value.updated && buffer_strlen(k->current_value.b)) ? + buffer_tostring(k->current_value.b) : FACET_VALUE_UNSET, + .wb = NULL, + }; + dictionary_set(row->dict, k->name, &t, sizeof(t)); + } + dfe_done(k); + + return row; +} + +// ---------------------------------------------------------------------------- + +static void facets_row_keep(FACETS *facets, usec_t usec) { + facets->operations.matched++; + + if(usec < facets->anchor) { + facets->operations.skips_before++; + return; + } + + if(unlikely(!facets->base)) { + facets->operations.last_added = facets_row_create(facets, usec, NULL); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next); + facets->items_to_return++; + facets->operations.first++; + return; + } + + if(likely(usec > facets->base->prev->usec)) + facets->operations.last_added = facets->base->prev; + + FACET_ROW *last = facets->operations.last_added; + while(last->prev != facets->base->prev && usec > last->prev->usec) { + last = last->prev; + facets->operations.backwards++; + } + + while(last->next && usec < last->next->usec) { + last = last->next; + facets->operations.forwards++; + } + + if(facets->items_to_return >= facets->max_items_to_return) { + if(last == facets->base->prev && usec < last->usec) { + facets->operations.skips_after++; + return; + } + } + + facets->items_to_return++; + + if(usec > last->usec) { + if(facets->items_to_return > facets->max_items_to_return) { + facets->items_to_return--; + facets->operations.shifts++; + facets->operations.last_added = facets->base->prev; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next); + facets->operations.last_added = facets_row_create(facets, usec, facets->operations.last_added); + } + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next); + facets->operations.prepends++; + } + else { + facets->operations.last_added = facets_row_create(facets, usec, NULL); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next); + facets->operations.appends++; + } + + while(facets->items_to_return > facets->max_items_to_return) { + // we have to remove something + + FACET_ROW *tmp = facets->base->prev; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, tmp, prev, next); + facets->items_to_return--; + + if(unlikely(facets->operations.last_added == tmp)) + facets->operations.last_added = facets->base->prev; + + facets_row_free(facets, tmp); + facets->operations.shifts++; + } +} + +void facets_rows_begin(FACETS *facets) { + FACET_KEY *k; + // dfe_start_read(facets->keys, k) { + for(k = facets->keys_ll ; k ; k = k->next) { + k->key_found_in_row = 0; + k->key_values_selected_in_row = 0; + k->current_value.updated = false; + k->current_value.hash[0] = '\0'; + } + // dfe_done(k); + + facets->keys_matched_by_query = 0; +} + +void facets_row_finished(FACETS *facets, usec_t usec) { + if(facets->query && facets->keys_filtered_by_query && !facets->keys_matched_by_query) + goto cleanup; + + facets->operations.evaluated++; + + uint32_t total_keys = 0; + uint32_t selected_by = 0; + + FACET_KEY *k; + // dfe_start_read(facets->keys, k) { + for(k = facets->keys_ll ; k ; k = k->next) { + if(!k->key_found_in_row) { + // put the FACET_VALUE_UNSET value into it + facets_check_value(facets, k); + } + + internal_fatal(!k->key_found_in_row, "all keys should be found in the row at this point"); + internal_fatal(k->key_found_in_row != 1, "all keys should be matched exactly once at this point"); + internal_fatal(k->key_values_selected_in_row > 1, "key values are selected in row more than once"); + + k->key_found_in_row = 1; + + total_keys += k->key_found_in_row; + selected_by += (k->key_values_selected_in_row) ? 1 : 0; + } + // dfe_done(k); + + if(selected_by >= total_keys - 1) { + uint32_t found = 0; + + // dfe_start_read(facets->keys, k){ + for(k = facets->keys_ll ; k ; k = k->next) { + uint32_t counted_by = selected_by; + + if (counted_by != total_keys && !k->key_values_selected_in_row) + counted_by++; + + if(counted_by == total_keys) { + if(k->values) { + if(!k->current_value.hash[0]) + facets_string_hash(buffer_tostring(k->current_value.b), k->current_value.hash); + + FACET_VALUE *v = dictionary_get(k->values, k->current_value.hash); + v->final_facet_value_counter++; + } + + found++; + } + } + // dfe_done(k); + + internal_fatal(!found, "We should find at least one facet to count this row"); + (void)found; + } + + if(selected_by == total_keys) + facets_row_keep(facets, usec); + +cleanup: + facets_rows_begin(facets); +} + +// ---------------------------------------------------------------------------- +// output + +void facets_report(FACETS *facets, BUFFER *wb) { + buffer_json_member_add_boolean(wb, "show_ids", false); + buffer_json_member_add_boolean(wb, "has_history", true); + + buffer_json_member_add_object(wb, "pagination"); + buffer_json_member_add_boolean(wb, "enabled", true); + buffer_json_member_add_string(wb, "key", "anchor"); + buffer_json_member_add_string(wb, "column", "timestamp"); + buffer_json_object_close(wb); + + buffer_json_member_add_array(wb, "accepted_params"); + { + if(facets->accepted_params) { + void *t; + dfe_start_read(facets->accepted_params, t) { + buffer_json_add_array_item_string(wb, t_dfe.name); + } + dfe_done(t); + } + + FACET_KEY *k; + dfe_start_read(facets->keys, k) { + if(!k->values) + continue; + + buffer_json_add_array_item_string(wb, k_dfe.name); + } + dfe_done(k); + } + buffer_json_array_close(wb); // accepted_params + + buffer_json_member_add_array(wb, "facets"); + { + FACET_KEY *k; + dfe_start_read(facets->keys, k) { + if(!k->values) + continue; + + buffer_json_add_array_item_object(wb); // key + { + buffer_json_member_add_string(wb, "id", k_dfe.name); + buffer_json_member_add_string(wb, "name", k->name); + + if(!k->order) + k->order = facets->order++; + + buffer_json_member_add_uint64(wb, "order", k->order); + buffer_json_member_add_array(wb, "options"); + { + FACET_VALUE *v; + dfe_start_read(k->values, v) { + buffer_json_add_array_item_object(wb); + { + buffer_json_member_add_string(wb, "id", v_dfe.name); + buffer_json_member_add_string(wb, "name", v->name); + buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter); + } + buffer_json_object_close(wb); + } + dfe_done(v); + } + buffer_json_array_close(wb); // options + } + buffer_json_object_close(wb); // key + } + dfe_done(k); + } + buffer_json_array_close(wb); // facets + + buffer_json_member_add_object(wb, "columns"); + { + size_t field_id = 0; + buffer_rrdf_table_add_field( + wb, field_id++, + "timestamp", "Timestamp", + RRDF_FIELD_TYPE_TIMESTAMP, + RRDF_FIELD_VISUAL_VALUE, + RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN, + RRDF_FIELD_SORT_DESCENDING, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + RRDF_FIELD_FILTER_RANGE, + RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY, + NULL); + + FACET_KEY *k; + dfe_start_read(facets->keys, k) { + RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_NONE; + bool visible = k->options & (FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_STICKY); + + if((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values)) + visible = true; + + if(!visible) + visible = simple_pattern_matches(facets->visible_keys, k->name); + + if(visible) + options |= RRDF_FIELD_OPTS_VISIBLE; + + if(k->options & FACET_KEY_OPTION_MAIN_TEXT) + options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP; + + buffer_rrdf_table_add_field( + wb, field_id++, + k_dfe.name, k->name ? k->name : k_dfe.name, + RRDF_FIELD_TYPE_STRING, + RRDF_FIELD_VISUAL_VALUE, + RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, + RRDF_FIELD_SORT_ASCENDING, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + k->values ? RRDF_FIELD_FILTER_FACET : RRDF_FIELD_FILTER_NONE, + options, + FACET_VALUE_UNSET); + } + dfe_done(k); + } + buffer_json_object_close(wb); // columns + + buffer_json_member_add_array(wb, "data"); + { + for(FACET_ROW *row = facets->base ; row ;row = row->next) { + buffer_json_add_array_item_array(wb); // each row + buffer_json_add_array_item_uint64(wb, row->usec); + + FACET_KEY *k; + dfe_start_read(facets->keys, k) + { + FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name); + + if(unlikely(k->dynamic.cb)) { + if(unlikely(!rkv)) + rkv = dictionary_set(row->dict, k->name, NULL, sizeof(*rkv)); + + k->dynamic.cb(facets, wb, rkv, row, k->dynamic.data); + } + else + buffer_json_add_array_item_string(wb, rkv ? buffer_tostring(rkv->wb) : FACET_VALUE_UNSET); + } + dfe_done(k); + buffer_json_array_close(wb); // each row + } + } + buffer_json_array_close(wb); // data + + buffer_json_member_add_string(wb, "default_sort_column", "timestamp"); + buffer_json_member_add_array(wb, "default_charts"); + buffer_json_array_close(wb); + + buffer_json_member_add_object(wb, "items"); + { + buffer_json_member_add_uint64(wb, "evaluated", facets->operations.evaluated); + buffer_json_member_add_uint64(wb, "matched", facets->operations.matched); + buffer_json_member_add_uint64(wb, "returned", facets->items_to_return); + buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return); + buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before); + buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts); + } + buffer_json_object_close(wb); // items + + buffer_json_member_add_object(wb, "stats"); + { + buffer_json_member_add_uint64(wb, "first", facets->operations.first); + buffer_json_member_add_uint64(wb, "forwards", facets->operations.forwards); + buffer_json_member_add_uint64(wb, "backwards", facets->operations.backwards); + buffer_json_member_add_uint64(wb, "skips_before", facets->operations.skips_before); + buffer_json_member_add_uint64(wb, "skips_after", facets->operations.skips_after); + buffer_json_member_add_uint64(wb, "prepends", facets->operations.prepends); + buffer_json_member_add_uint64(wb, "appends", facets->operations.appends); + buffer_json_member_add_uint64(wb, "shifts", facets->operations.shifts); + } + buffer_json_object_close(wb); // items + +} diff --git a/libnetdata/facets/facets.h b/libnetdata/facets/facets.h new file mode 100644 index 000000000..c0f7f80c8 --- /dev/null +++ b/libnetdata/facets/facets.h @@ -0,0 +1,62 @@ +#ifndef FACETS_H +#define FACETS_H 1 + +#include "../libnetdata.h" + +typedef enum __attribute__((packed)) { + FACET_KEY_OPTION_FACET = (1 << 0), // filterable values + FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value + FACET_KEY_OPTION_STICKY = (1 << 2), // should be sticky in the table + FACET_KEY_OPTION_VISIBLE = (1 << 3), // should be in the default table + FACET_KEY_OPTION_FTS = (1 << 4), // the key is filterable by full text search (FTS) + FACET_KEY_OPTION_MAIN_TEXT = (1 << 5), // full width and wrap + FACET_KEY_OPTION_REORDER = (1 << 6), // give the key a new order id on first encounter +} FACET_KEY_OPTIONS; + +typedef struct facet_row_key_value { + const char *tmp; + BUFFER *wb; +} FACET_ROW_KEY_VALUE; + +typedef struct facet_row { + usec_t usec; + DICTIONARY *dict; + struct facet_row *prev, *next; +} FACET_ROW; + +typedef struct facets FACETS; +typedef struct facet_key FACET_KEY; + +#define FACET_STRING_HASH_SIZE 19 +void facets_string_hash(const char *src, char *out); + +typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, void *data); +typedef void (*facet_dynamic_row_t)(FACETS *facets, BUFFER *json_array, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data); +FACET_KEY *facets_register_dynamic_key(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data); +FACET_KEY *facets_register_key_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data); + +typedef enum __attribute__((packed)) { + FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets, should be visible by default in the table + FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search +} FACETS_OPTIONS; + +FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys); +void facets_destroy(FACETS *facets); + +void facets_accepted_param(FACETS *facets, const char *param); + +void facets_rows_begin(FACETS *facets); +void facets_row_finished(FACETS *facets, usec_t usec); + +FACET_KEY *facets_register_key(FACETS *facets, const char *param, FACET_KEY_OPTIONS options); +void facets_set_query(FACETS *facets, const char *query); +void facets_set_items(FACETS *facets, uint32_t items); +void facets_set_anchor(FACETS *facets, usec_t anchor); +void facets_register_facet_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options); + +void facets_add_key_value(FACETS *facets, const char *key, const char *value); +void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len); + +void facets_report(FACETS *facets, BUFFER *wb); + +#endif |