diff options
Diffstat (limited to 'src/libnetdata/facets/facets.c')
| -rw-r--r-- | src/libnetdata/facets/facets.c | 277 |
1 files changed, 221 insertions, 56 deletions
diff --git a/src/libnetdata/facets/facets.c b/src/libnetdata/facets/facets.c index 3c746cbc3..230e03de5 100644 --- a/src/libnetdata/facets/facets.c +++ b/src/libnetdata/facets/facets.c @@ -102,25 +102,25 @@ static inline bool is_valid_string_hash(const char *s) { // hashtable for FACET_VALUE // cleanup hashtable defines -#include "../../libnetdata/simple_hashtable_undef.h" +#include "../simple_hashtable/simple_hashtable_undef.h" struct facet_value; // #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_value #define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_value #define SIMPLE_HASHTABLE_NAME _VALUE -#include "../simple_hashtable.h" +#include "../simple_hashtable/simple_hashtable.h" // ---------------------------------------------------------------------------- // hashtable for FACET_KEY // cleanup hashtable defines -#include "../../libnetdata/simple_hashtable_undef.h" +#include "../simple_hashtable/simple_hashtable_undef.h" struct facet_key; // #define SIMPLE_HASHTABLE_SORT_FUNCTION compare_facet_key #define SIMPLE_HASHTABLE_VALUE_TYPE struct facet_key #define SIMPLE_HASHTABLE_NAME _KEY -#include "../simple_hashtable.h" +#include "../simple_hashtable/simple_hashtable.h" // ---------------------------------------------------------------------------- @@ -222,6 +222,7 @@ struct facets { SIMPLE_PATTERN *visible_keys; SIMPLE_PATTERN *excluded_keys; SIMPLE_PATTERN *included_keys; + bool all_keys_included_by_default; FACETS_OPTIONS options; @@ -255,6 +256,7 @@ struct facets { } keys_in_row; FACET_ROW *base; // double linked list of the selected facets rows + FACET_ROW_BIN_DATA bin_data; uint32_t items_to_return; uint32_t max_items_to_return; @@ -328,6 +330,10 @@ struct facets { struct { size_t searches; } fts; + + struct { + size_t bin_data_inflight; + }; } operations; struct { @@ -353,6 +359,27 @@ uint32_t facets_rows(FACETS *facets) { return facets->items_to_return; } +static const char *facets_key_id(FACET_KEY *k) { + if(k->facets->options & FACETS_OPTION_HASH_IDS) + return hash_to_static_string(k->hash); + else + return k->name ? k->name : hash_to_static_string(k->hash); +} + +static const char *facets_key_value_id(FACET_KEY *k, FACET_VALUE *v) { + if(k->facets->options & FACETS_OPTION_HASH_IDS) + return hash_to_static_string(v->hash); + else + return v->name ? v->name : hash_to_static_string(v->hash); +} + +void facets_use_hashes_for_ids(FACETS *facets, bool set) { + if(set) + facets->options |= FACETS_OPTION_HASH_IDS; + else + facets->options &= ~(FACETS_OPTION_HASH_IDS); +} + // ---------------------------------------------------------------------------- static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row); @@ -570,12 +597,12 @@ static inline void FACET_VALUE_ADD_CURRENT_VALUE_TO_INDEX(FACET_KEY *k) { k->facets->operations.values.indexed++; } -static inline void FACET_VALUE_ADD_OR_UPDATE_SELECTED(FACET_KEY *k, FACETS_HASH hash) { +static inline void FACET_VALUE_ADD_OR_UPDATE_SELECTED(FACET_KEY *k, const char *name, FACETS_HASH hash) { FACET_VALUE tv = { .hash = hash, .selected = true, - .name = NULL, - .name_len = 0, + .name = name, + .name_len = name ? strlen(name) : 0, }; FACET_VALUE_ADD_TO_INDEX(k, &tv); } @@ -643,6 +670,35 @@ bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, s return (v && v->selected) ? true : false; } +bool facets_foreach_selected_value_in_key(FACETS *facets, const char *key, size_t key_length, DICTIONARY *used_hashes_registry, facets_foreach_selected_value_in_key_t cb, void *data) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length); + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); + if(!k || k->default_selected_for_values) + return false; + + size_t selected = 0; + for(FACET_VALUE *v = k->values.ll; v ;v = v->next) { + if(!v->selected) continue; + + const char *value = v->name; + if(!value) { + if(used_hashes_registry) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(v->hash, hash_str); + value = dictionary_get(used_hashes_registry, hash_str); + } + + if(!value) + return false; + } + + if(!cb(facets, selected++, k->name, value, data)) + return false; + } + + return selected > 0; +} + void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) { FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length); FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); @@ -691,7 +747,7 @@ static inline FACET_KEY *FACETS_KEY_CREATE(FACETS *facets, FACETS_HASH hash, con k->current_value.b = buffer_create(sizeof(FACET_VALUE_UNSET), NULL); k->default_selected_for_values = true; - if(!(k->options & FACET_KEY_OPTION_REORDER)) + if(unlikely((k->options & (FACET_KEY_OPTION_REORDER | FACET_KEY_OPTION_REORDER_DONE)) == 0)) k->order = facets->order++; if((k->options & FACET_KEY_OPTION_FTS) || (facets->options & FACETS_OPTION_ALL_KEYS_FTS)) @@ -724,10 +780,11 @@ static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH has FACET_KEY *k = SIMPLE_HASHTABLE_SLOT_DATA(slot); facet_key_set_name(k, name, name_length); + k->options |= options; - if(unlikely(k->options & FACET_KEY_OPTION_REORDER)) { + if(unlikely((k->options & (FACET_KEY_OPTION_REORDER | FACET_KEY_OPTION_REORDER_DONE)) == FACET_KEY_OPTION_REORDER)) { k->order = facets->order++; - k->options &= ~FACET_KEY_OPTION_REORDER; + k->options |= FACET_KEY_OPTION_REORDER_DONE; } return k; @@ -1547,7 +1604,7 @@ static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) { } static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) { - bool included = true, excluded = false, never = false; + bool included = facets->all_keys_included_by_default, excluded = false, never = false; if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) { if(k->options & FACET_KEY_OPTION_FACET) { @@ -1591,9 +1648,39 @@ static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) { } // ---------------------------------------------------------------------------- +// bin_data management + +static inline void facets_row_bin_data_cleanup(FACETS *facets, FACET_ROW_BIN_DATA *bin_data) { + if(!bin_data->data) + return; + + bin_data->cleanup_cb(bin_data->data); + *bin_data = FACET_ROW_BIN_DATA_EMPTY; + + fatal_assert(facets->operations.bin_data_inflight > 0); + facets->operations.bin_data_inflight--; +} + +void facets_row_bin_data_set(FACETS *facets, void (*cleanup_cb)(void *data), void *data) { + // in case the caller tries to register bin_data multiple times + // for the same row. + facets_row_bin_data_cleanup(facets, &facets->bin_data); + + // set the new values + facets->bin_data.cleanup_cb = cleanup_cb; + facets->bin_data.data = data; + facets->operations.bin_data_inflight++; +} + +void *facets_row_bin_data_get(FACETS *facets __maybe_unused, FACET_ROW *row) { + return row->bin_data.data; +} + +// ---------------------------------------------------------------------------- FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) { FACETS *facets = callocz(1, sizeof(FACETS)); + facets->all_keys_included_by_default = true; facets->options = options; FACETS_KEYS_INDEX_CREATE(facets); @@ -1616,6 +1703,8 @@ FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const ch } void facets_destroy(FACETS *facets) { + if(!facets) return; + dictionary_destroy(facets->accepted_params); FACETS_KEYS_INDEX_DESTROY(facets); simple_pattern_free(facets->visible_keys); @@ -1629,6 +1718,13 @@ void facets_destroy(FACETS *facets) { facets_row_free(facets, r); } + // in case the caller did not call facets_row_finished() + // on the last row. + facets_row_bin_data_cleanup(facets, &facets->bin_data); + + // make sure we didn't lose any data + fatal_assert(facets->operations.bin_data_inflight == 0); + freez(facets->histogram.chart); freez(facets); } @@ -1691,6 +1787,40 @@ void facets_enable_slice_mode(FACETS *facets) { facets->options |= FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS | FACETS_OPTION_SORT_FACETS_ALPHABETICALLY; } +void facets_reset_and_disable_all_facets(FACETS *facets) { + facets->all_keys_included_by_default = false; + + simple_pattern_free(facets->included_keys); + facets->included_keys = NULL; + +// We need this, because the exclusions are good for controlling which key can become a facet. +// The excluded ones are not offered for facets at all. +// simple_pattern_free(facets->excluded_keys); +// facets->excluded_keys = NULL; + + simple_pattern_free(facets->visible_keys); + facets->visible_keys = NULL; + + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + k->options |= FACET_KEY_OPTION_NO_FACET; + k->options &= ~FACET_KEY_OPTION_FACET; + } + foreach_key_in_facets_done(k); +} + +inline FACET_KEY *facets_register_facet(FACETS *facets, const char *name, FACET_KEY_OPTIONS options) { + size_t name_length = strlen(name); + FACETS_HASH hash = FACETS_HASH_FUNCTION(name, name_length); + + FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, name, name_length, options); + k->options |= FACET_KEY_OPTION_FACET; + k->options &= ~FACET_KEY_OPTION_NO_FACET; + facet_key_late_init(facets, k); + + return k; +} + inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options) { if(!is_valid_string_hash(key_id)) return NULL; @@ -1708,16 +1838,25 @@ inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, F return k; } -void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *value_id, FACET_KEY_OPTIONS options) { +void facets_register_facet_filter_id(FACETS *facets, const char *key_id, const char *value_id, FACET_KEY_OPTIONS options) { FACET_KEY *k = facets_register_facet_id(facets, key_id, options); if(k) { if(is_valid_string_hash(value_id)) { k->default_selected_for_values = false; - FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, str_to_facets_hash(value_id)); + FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, NULL, str_to_facets_hash(value_id)); } } } +void facets_register_facet_filter(FACETS *facets, const char *key, const char *value, FACET_KEY_OPTIONS options) { + FACET_KEY *k = facets_register_facet(facets, key, options); + if(k) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(value, strlen(value)); + k->default_selected_for_values = false; + FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, value, hash); + } +} + void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity) { facets->current_row.severity = severity; } @@ -1834,6 +1973,10 @@ void facets_add_key_value(FACETS *facets, const char *key, const char *value) { } void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len) { + if(!key || !*key || !key_len || !value || !*value || !value_len) + // adding empty values, makes the rows unmatched + return; + FACET_KEY *k = facets_register_key_name_length(facets, key, key_len, 0); k->current_value.raw = value; k->current_value.raw_len = value_len; @@ -1879,7 +2022,9 @@ static void facet_row_key_value_delete_callback(const DICTIONARY_ITEM *item __ma // FACET_ROW management static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row) { + facets_row_bin_data_cleanup(facets, &row->bin_data); dictionary_destroy(row->dict); + row->dict = NULL; freez(row); } @@ -1889,6 +2034,7 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into if(into) { row = into; facets->operations.rows.reused++; + facets_row_bin_data_cleanup(facets, &row->bin_data); } else { row = callocz(1, sizeof(FACET_ROW)); @@ -1899,6 +2045,11 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into facets->operations.rows.created++; } + // copy the bin_data to the row + // and forget about them in facets + row->bin_data = facets->bin_data; + facets->bin_data = FACET_ROW_BIN_DATA_EMPTY; + row->severity = facets->current_row.severity; row->usec = usec; @@ -2083,6 +2234,8 @@ static void facets_reset_keys_with_value_and_row(FACETS *facets) { facets->current_row.keys_matched_by_query_positive = 0; facets->current_row.keys_matched_by_query_negative = 0; facets->keys_in_row.used = 0; + + facets_row_bin_data_cleanup(facets, &facets->bin_data); } void facets_rows_begin(FACETS *facets) { @@ -2097,6 +2250,9 @@ void facets_rows_begin(FACETS *facets) { } bool facets_row_finished(FACETS *facets, usec_t usec) { +// char buf[RFC3339_MAX_LENGTH]; +// rfc3339_datetime_ut(buf, sizeof(buf), usec, 3, false); + facets->operations.rows.evaluated++; if(unlikely((facets->query && facets->keys_filtered_by_query && @@ -2207,11 +2363,11 @@ void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool w if(with_keys) { FACET_KEY *k; - foreach_key_in_facets(facets, k){ - if (!k->values.enabled) + foreach_key_in_facets(facets, k) { + if (!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN) continue; - buffer_json_add_array_item_string(wb, hash_to_static_string(k->hash)); + buffer_json_add_array_item_string(wb, facets_key_id(k)); } foreach_key_in_facets_done(k); } @@ -2391,8 +2547,8 @@ static uint32_t facets_sort_and_reorder_values(FACET_KEY *k) { return ret; } -void facets_table_config(BUFFER *wb) { - buffer_json_member_add_boolean(wb, "show_ids", false); // do not show the column ids to the user +void facets_table_config(FACETS *facets, BUFFER *wb) { + buffer_json_member_add_boolean(wb, "show_ids", (facets->options & FACETS_OPTION_HASH_IDS) ? false : true); buffer_json_member_add_boolean(wb, "has_history", true); // enable date-time picker with after-before buffer_json_member_add_object(wb, "pagination"); @@ -2408,8 +2564,9 @@ void facets_table_config(BUFFER *wb) { void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) { facets->report.used_hashes_registry = used_hashes_registry; + facets_table_config(facets, wb); + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) { - facets_table_config(wb); facets_accepted_parameters_to_json_array(facets, wb, true); } @@ -2434,19 +2591,21 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) CLEAN_BUFFER *tb = buffer_create(0, NULL); FACET_KEY *k; foreach_key_in_facets(facets, k) { - if(!k->values.enabled) + if(!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN) continue; - if(!facets_sort_and_reorder_values(k)) - // no values for this key - continue; + facets_sort_and_reorder_values(k); buffer_json_add_array_item_object(wb); // key { - buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash)); - buffer_json_member_add_string(wb, "name", facets_key_name_cached(k - , facets->report.used_hashes_registry - )); + buffer_json_member_add_string( + wb, "id", facets_key_id(k)); + + buffer_json_member_add_string( + wb, "name", + facets_key_name_cached(k, facets->report.used_hashes_registry)); + + // buffer_json_member_add_string(wb, "raw", k->name); if(!k->order) k->order = facets->order++; buffer_json_member_add_uint64(wb, "order", k->order); @@ -2463,10 +2622,11 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) buffer_json_add_array_item_object(wb); { - buffer_json_member_add_string(wb, "id", hash_to_static_string(v->hash)); + buffer_json_member_add_string(wb, "id", facets_key_value_id(k, v)); facets_key_value_transformed(facets, k, v, tb, FACETS_TRANSFORM_FACET); buffer_json_member_add_string(wb, "name", buffer_tostring(tb)); + // buffer_json_member_add_string(wb, "raw", v->name); buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter); buffer_json_member_add_uint64(wb, "order", v->order); } @@ -2517,38 +2677,40 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) FACET_KEY *k; foreach_key_in_facets(facets, k) { - RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP; - bool visible = k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY); + if(k->options & FACET_KEY_OPTION_HIDDEN) + continue; - if ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values.enabled)) - visible = true; + RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_WRAP; + RRDF_FIELD_VISUAL visual = (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE; + RRDF_FIELD_TRANSFORM transform = RRDF_FIELD_TRANSFORM_NONE; - if (!visible) - visible = simple_pattern_matches(facets->visible_keys, k->name); + if (k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY) || + ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE) && k->values.enabled) || + simple_pattern_matches(facets->visible_keys, k->name)) + options |= RRDF_FIELD_OPTS_VISIBLE; - if (visible) - options |= RRDF_FIELD_OPTS_VISIBLE; + if (k->options & FACET_KEY_OPTION_MAIN_TEXT) + options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP; - if (k->options & FACET_KEY_OPTION_MAIN_TEXT) - options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP; + if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER) + options |= RRDF_FIELD_OPTS_EXPANDED_FILTER; - if (k->options & FACET_KEY_OPTION_EXPANDED_FILTER) - options |= RRDF_FIELD_OPTS_EXPANDED_FILTER; + if (k->options & FACET_KEY_OPTION_PRETTY_XML) + transform = RRDF_FIELD_TRANSFORM_XML; - const char *hash_str = hash_to_static_string(k->hash); + const char *key_id = facets_key_id(k); - buffer_rrdf_table_add_field( - wb, field_id++, - hash_str, k->name ? k->name : hash_str, - RRDF_FIELD_TYPE_STRING, - (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE, - RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, - RRDF_FIELD_SORT_FIXED, - NULL, - RRDF_FIELD_SUMMARY_COUNT, - (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET, - options, FACET_VALUE_UNSET); - } + buffer_rrdf_table_add_field( + wb, field_id++, + key_id, k->name ? k->name : key_id, + RRDF_FIELD_TYPE_STRING, + visual, transform, 0, NULL, NAN, + RRDF_FIELD_SORT_FIXED, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET, + options, FACET_VALUE_UNSET); + } foreach_key_in_facets_done(k); } buffer_json_object_close(wb); // columns @@ -2585,6 +2747,9 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) FACET_KEY *k; foreach_key_in_facets(facets, k) { + if(k->options & FACET_KEY_OPTION_HIDDEN) + continue; + FACET_ROW_KEY_VALUE *rkv = dictionary_get(row->dict, k->name); if(unlikely(k->dynamic.cb)) { @@ -2627,14 +2792,14 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) { FACET_KEY *k; foreach_key_in_facets(facets, k) { - if (!k->values.enabled) + if (!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN) continue; if(unlikely(!first_histogram_hash)) first_histogram_hash = k->hash; buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash)); + buffer_json_member_add_string(wb, "id", facets_key_id(k)); buffer_json_member_add_string(wb, "name", k->name); buffer_json_member_add_uint64(wb, "order", k->order); buffer_json_object_close(wb); @@ -2662,7 +2827,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) } if(show_histogram) { - buffer_json_member_add_string(wb, "id", k ? hash_to_static_string(k->hash) : ""); + buffer_json_member_add_string(wb, "id", k ? facets_key_id(k) : ""); buffer_json_member_add_string(wb, "name", k ? k->name : ""); buffer_json_member_add_object(wb, "chart"); { |