diff options
Diffstat (limited to 'database/contexts/rrdcontext.h')
| -rw-r--r-- | database/contexts/rrdcontext.h | 553 |
1 files changed, 553 insertions, 0 deletions
diff --git a/database/contexts/rrdcontext.h b/database/contexts/rrdcontext.h new file mode 100644 index 00000000..5328483d --- /dev/null +++ b/database/contexts/rrdcontext.h @@ -0,0 +1,553 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_RRDCONTEXT_H +#define NETDATA_RRDCONTEXT_H 1 + +// ---------------------------------------------------------------------------- +// RRDMETRIC + +typedef struct rrdmetric_acquired RRDMETRIC_ACQUIRED; + +// ---------------------------------------------------------------------------- +// RRDINSTANCE + +typedef struct rrdinstance_acquired RRDINSTANCE_ACQUIRED; + +// ---------------------------------------------------------------------------- +// RRDCONTEXT + +typedef struct rrdcontext_acquired RRDCONTEXT_ACQUIRED; + +// ---------------------------------------------------------------------------- + +#include "../rrd.h" + +bool rrdinstance_acquired_id_and_name_are_same(RRDINSTANCE_ACQUIRED *ria); +const char *rrdmetric_acquired_id(RRDMETRIC_ACQUIRED *rma); +const char *rrdmetric_acquired_name(RRDMETRIC_ACQUIRED *rma); +bool rrdmetric_acquired_has_name(RRDMETRIC_ACQUIRED *rma); + +STRING *rrdmetric_acquired_id_dup(RRDMETRIC_ACQUIRED *rma); +STRING *rrdmetric_acquired_name_dup(RRDMETRIC_ACQUIRED *rma); + +NETDATA_DOUBLE rrdmetric_acquired_last_stored_value(RRDMETRIC_ACQUIRED *rma); +time_t rrdmetric_acquired_first_entry(RRDMETRIC_ACQUIRED *rma); +time_t rrdmetric_acquired_last_entry(RRDMETRIC_ACQUIRED *rma); +bool rrdmetric_acquired_belongs_to_instance(RRDMETRIC_ACQUIRED *rma, RRDINSTANCE_ACQUIRED *ria); + +const char *rrdinstance_acquired_id(RRDINSTANCE_ACQUIRED *ria); +const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria); +bool rrdinstance_acquired_has_name(RRDINSTANCE_ACQUIRED *ria); +const char *rrdinstance_acquired_units(RRDINSTANCE_ACQUIRED *ria); +STRING *rrdinstance_acquired_units_dup(RRDINSTANCE_ACQUIRED *ria); +DICTIONARY *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria); +DICTIONARY *rrdinstance_acquired_functions(RRDINSTANCE_ACQUIRED *ria); +RRDHOST *rrdinstance_acquired_rrdhost(RRDINSTANCE_ACQUIRED *ria); +RRDSET *rrdinstance_acquired_rrdset(RRDINSTANCE_ACQUIRED *ria); + +bool rrdinstance_acquired_belongs_to_context(RRDINSTANCE_ACQUIRED *ria, RRDCONTEXT_ACQUIRED *rca); +time_t rrdinstance_acquired_update_every(RRDINSTANCE_ACQUIRED *ria); + +const char *rrdcontext_acquired_units(RRDCONTEXT_ACQUIRED *rca); +const char *rrdcontext_acquired_title(RRDCONTEXT_ACQUIRED *rca); +RRDSET_TYPE rrdcontext_acquired_chart_type(RRDCONTEXT_ACQUIRED *rca); + +// ---------------------------------------------------------------------------- +// public API for rrdhost + +void rrdhost_load_rrdcontext_data(RRDHOST *host); +void rrdhost_create_rrdcontexts(RRDHOST *host); +void rrdhost_destroy_rrdcontexts(RRDHOST *host); + +void rrdcontext_host_child_connected(RRDHOST *host); +void rrdcontext_host_child_disconnected(RRDHOST *host); + +int rrdcontext_foreach_instance_with_rrdset_in_context(RRDHOST *host, const char *context, int (*callback)(RRDSET *st, void *data), void *data); + +typedef enum { + RRDCONTEXT_OPTION_NONE = 0, + RRDCONTEXT_OPTION_SHOW_METRICS = (1 << 0), + RRDCONTEXT_OPTION_SHOW_INSTANCES = (1 << 1), + RRDCONTEXT_OPTION_SHOW_LABELS = (1 << 2), + RRDCONTEXT_OPTION_SHOW_QUEUED = (1 << 3), + RRDCONTEXT_OPTION_SHOW_FLAGS = (1 << 4), + RRDCONTEXT_OPTION_SHOW_DELETED = (1 << 5), + RRDCONTEXT_OPTION_DEEPSCAN = (1 << 6), + RRDCONTEXT_OPTION_SHOW_UUIDS = (1 << 7), + RRDCONTEXT_OPTION_SHOW_HIDDEN = (1 << 8), + RRDCONTEXT_OPTION_SKIP_ID = (1 << 31), // internal use +} RRDCONTEXT_TO_JSON_OPTIONS; + +#define RRDCONTEXT_OPTIONS_ALL (RRDCONTEXT_OPTION_SHOW_METRICS|RRDCONTEXT_OPTION_SHOW_INSTANCES|RRDCONTEXT_OPTION_SHOW_LABELS|RRDCONTEXT_OPTION_SHOW_QUEUED|RRDCONTEXT_OPTION_SHOW_FLAGS|RRDCONTEXT_OPTION_SHOW_DELETED|RRDCONTEXT_OPTION_SHOW_UUIDS|RRDCONTEXT_OPTION_SHOW_HIDDEN) + +int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, const char *context, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions); +int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions); + +// ---------------------------------------------------------------------------- +// public API for rrdcontexts + +const char *rrdcontext_acquired_id(RRDCONTEXT_ACQUIRED *rca); +bool rrdcontext_acquired_belongs_to_host(RRDCONTEXT_ACQUIRED *rca, RRDHOST *host); + +// ---------------------------------------------------------------------------- +// public API for rrddims + +void rrdcontext_updated_rrddim(RRDDIM *rd); +void rrdcontext_removed_rrddim(RRDDIM *rd); +void rrdcontext_updated_rrddim_algorithm(RRDDIM *rd); +void rrdcontext_updated_rrddim_multiplier(RRDDIM *rd); +void rrdcontext_updated_rrddim_divisor(RRDDIM *rd); +void rrdcontext_updated_rrddim_flags(RRDDIM *rd); +void rrdcontext_collected_rrddim(RRDDIM *rd); +int rrdcontext_find_dimension_uuid(RRDSET *st, const char *id, uuid_t *store_uuid); + +// ---------------------------------------------------------------------------- +// public API for rrdsets + +void rrdcontext_updated_rrdset(RRDSET *st); +void rrdcontext_removed_rrdset(RRDSET *st); +void rrdcontext_updated_rrdset_name(RRDSET *st); +void rrdcontext_updated_rrdset_flags(RRDSET *st); +void rrdcontext_updated_retention_rrdset(RRDSET *st); +void rrdcontext_collected_rrdset(RRDSET *st); +int rrdcontext_find_chart_uuid(RRDSET *st, uuid_t *store_uuid); + +// ---------------------------------------------------------------------------- +// public API for ACLK + +void rrdcontext_hub_checkpoint_command(void *cmd); +void rrdcontext_hub_stop_streaming_command(void *cmd); + + +// ---------------------------------------------------------------------------- +// public API for threads + +void rrdcontext_db_rotation(void); +void *rrdcontext_main(void *); + +// ---------------------------------------------------------------------------- +// public API for queries + +typedef enum __attribute__ ((__packed__)) { + QUERY_STATUS_NONE = 0, + QUERY_STATUS_QUERIED = (1 << 0), + QUERY_STATUS_DIMENSION_HIDDEN = (1 << 1), + QUERY_STATUS_EXCLUDED = (1 << 2), + QUERY_STATUS_FAILED = (1 << 3), +} QUERY_STATUS; + +typedef struct query_plan_entry { + size_t tier; + time_t after; + time_t before; +} QUERY_PLAN_ENTRY; + +#define QUERY_PLANS_MAX (RRD_STORAGE_TIERS) + +typedef struct query_metrics_counts { // counts the number of metrics related to an object + size_t selected; // selected to be queried + size_t excluded; // not selected to be queried + size_t queried; // successfully queried + size_t failed; // failed to be queried +} QUERY_METRICS_COUNTS; + +typedef struct query_instances_counts { // counts the number of instances related to an object + size_t selected; // selected to be queried + size_t excluded; // not selected to be queried + size_t queried; // successfully queried + size_t failed; // failed to be queried +} QUERY_INSTANCES_COUNTS; + +typedef struct query_alerts_counts { // counts the number of alerts related to an object + size_t clear; // number of alerts in clear state + size_t warning; // number of alerts in warning state + size_t critical; // number of alerts in critical state + size_t other; // number of alerts in any other state +} QUERY_ALERTS_COUNTS; + +typedef struct query_node { + uint32_t slot; + RRDHOST *rrdhost; + char node_id[UUID_STR_LEN]; + usec_t duration_ut; + + STORAGE_POINT query_points; + QUERY_INSTANCES_COUNTS instances; + QUERY_METRICS_COUNTS metrics; + QUERY_ALERTS_COUNTS alerts; +} QUERY_NODE; + +typedef struct query_context { + uint32_t slot; + RRDCONTEXT_ACQUIRED *rca; + + STORAGE_POINT query_points; + QUERY_INSTANCES_COUNTS instances; + QUERY_METRICS_COUNTS metrics; + QUERY_ALERTS_COUNTS alerts; +} QUERY_CONTEXT; + +typedef struct query_instance { + uint32_t slot; + uint32_t query_host_id; + RRDINSTANCE_ACQUIRED *ria; + STRING *id_fqdn; // never access this directly - it is created on demand via query_instance_id_fqdn() + STRING *name_fqdn; // never access this directly - it is created on demand via query_instance_name_fqdn() + + STORAGE_POINT query_points; + QUERY_METRICS_COUNTS metrics; + QUERY_ALERTS_COUNTS alerts; +} QUERY_INSTANCE; + +typedef struct query_dimension { + uint32_t slot; + uint32_t priority; + RRDMETRIC_ACQUIRED *rma; + QUERY_STATUS status; +} QUERY_DIMENSION; + +typedef struct query_metric { + RRDR_DIMENSION_FLAGS status; + + struct query_metric_tier { + STORAGE_METRIC_HANDLE *db_metric_handle; + time_t db_first_time_s; // the oldest timestamp available for this tier + time_t db_last_time_s; // the latest timestamp available for this tier + time_t db_update_every_s; // latest update every for this tier + long weight; + } tiers[RRD_STORAGE_TIERS]; + + struct { + size_t used; + QUERY_PLAN_ENTRY array[QUERY_PLANS_MAX]; + } plan; + + struct { + uint32_t query_node_id; + uint32_t query_context_id; + uint32_t query_instance_id; + uint32_t query_dimension_id; + } link; + + STORAGE_POINT query_points; + + struct { + uint32_t slot; + uint32_t first_slot; + STRING *id; + STRING *name; + STRING *units; + } grouped_as; + + usec_t duration_ut; +} QUERY_METRIC; + +#define MAX_QUERY_TARGET_ID_LENGTH 255 +#define MAX_QUERY_GROUP_BY_PASSES 2 + +typedef bool (*qt_interrupt_callback_t)(void *data); + +struct group_by_pass { + RRDR_GROUP_BY group_by; + char *group_by_label; + RRDR_GROUP_BY_FUNCTION aggregation; +}; + +typedef struct query_target_request { + size_t version; + + const char *scope_nodes; + const char *scope_contexts; + + // selecting / filtering metrics to be queried + RRDHOST *host; // the host to be queried (can be NULL, hosts will be used) + RRDCONTEXT_ACQUIRED *rca; // the context to be queried (can be NULL) + RRDINSTANCE_ACQUIRED *ria; // the instance to be queried (can be NULL) + RRDMETRIC_ACQUIRED *rma; // the metric to be queried (can be NULL) + RRDSET *st; // the chart to be queried (NULL, for context queries) + const char *nodes; // hosts simple pattern + const char *contexts; // contexts simple pattern (context queries) + const char *instances; // charts simple pattern (for context queries) + const char *dimensions; // dimensions simple pattern + const char *chart_label_key; // select only the chart having this label key + const char *labels; // select only the charts having this combo of label key:value + const char *alerts; // select only the charts having this combo of alert name:status + + time_t after; // the requested timeframe + time_t before; // the requested timeframe + size_t points; // the requested number of points to be returned + + uint32_t format; // DATASOURCE_FORMAT + RRDR_OPTIONS options; + time_t timeout_ms; // the timeout of the query in milliseconds + + size_t tier; + QUERY_SOURCE query_source; + STORAGE_PRIORITY priority; + + // resampling metric values across time + time_t resampling_time; + + // grouping metric values across time + RRDR_TIME_GROUPING time_group_method; + const char *time_group_options; + + // group by across multiple time-series + struct group_by_pass group_by[MAX_QUERY_GROUP_BY_PASSES]; + + usec_t received_ut; + + qt_interrupt_callback_t interrupt_callback; + void *interrupt_callback_data; +} QUERY_TARGET_REQUEST; + +#define GROUP_BY_MAX_LABEL_KEYS 10 + +struct query_tier_statistics { + size_t queries; + size_t points; + time_t update_every; + struct { + time_t first_time_s; + time_t last_time_s; + } retention; +}; + +struct query_versions { + uint64_t contexts_hard_hash; + uint64_t contexts_soft_hash; + uint64_t alerts_hard_hash; + uint64_t alerts_soft_hash; +}; + +struct query_timings { + usec_t received_ut; + usec_t preprocessed_ut; + usec_t executed_ut; + usec_t finished_ut; +}; + +#define query_view_update_every(qt) ((qt)->window.group * (qt)->window.query_granularity) + +typedef struct query_target { + char id[MAX_QUERY_TARGET_ID_LENGTH + 1]; // query identifier (for logging) + QUERY_TARGET_REQUEST request; + + struct { + time_t now; // the current timestamp, the absolute max for any query timestamp + bool relative; // true when the request made with relative timestamps, true if it was absolute + bool aligned; + time_t after; // the absolute timestamp this query is about + time_t before; // the absolute timestamp this query is about + time_t query_granularity; + size_t points; // the number of points the query will return (maybe different from the request) + size_t group; + RRDR_TIME_GROUPING time_group_method; + const char *time_group_options; + size_t resampling_group; + NETDATA_DOUBLE resampling_divisor; + RRDR_OPTIONS options; + size_t tier; + } window; + + struct { + size_t queries[RRD_STORAGE_TIERS]; + time_t first_time_s; // the combined first_time_t of all metrics in the query, across all tiers + time_t last_time_s; // the combined last_time_T of all metrics in the query, across all tiers + time_t minimum_latest_update_every_s; // the min update every of the metrics in the query + struct query_tier_statistics tiers[RRD_STORAGE_TIERS]; + } db; + + struct { + QUERY_METRIC *array; // the metrics to be queried (all of them should be queried, no exceptions) + uint32_t used; // how many items of the array are used + uint32_t size; // the size of the array + SIMPLE_PATTERN *pattern; + } query; + + struct { + QUERY_DIMENSION *array; + uint32_t used; // how many items of the array are used + uint32_t size; // the size of the array + } dimensions; + + struct { + QUERY_INSTANCE *array; + uint32_t used; // how many items of the array are used + uint32_t size; // the size of the array + SIMPLE_PATTERN *pattern; + SIMPLE_PATTERN *labels_pattern; + SIMPLE_PATTERN *alerts_pattern; + SIMPLE_PATTERN *chart_label_key_pattern; + } instances; + + struct { + QUERY_CONTEXT *array; + uint32_t used; // how many items of the array are used + uint32_t size; // the size of the array + SIMPLE_PATTERN *pattern; + SIMPLE_PATTERN *scope_pattern; + } contexts; + + struct { + QUERY_NODE *array; + uint32_t used; // how many items of the array are used + uint32_t size; // the size of the array + SIMPLE_PATTERN *pattern; + SIMPLE_PATTERN *scope_pattern; + } nodes; + + struct { + size_t used; + char *label_keys[GROUP_BY_MAX_LABEL_KEYS * MAX_QUERY_GROUP_BY_PASSES]; + } group_by[MAX_QUERY_GROUP_BY_PASSES]; + + STORAGE_POINT query_points; + struct query_versions versions; + struct query_timings timings; + + struct { + SPINLOCK spinlock; + bool used; // when true, this query is currently being used + size_t queries; // how many query we have done so far with this QUERY_TARGET - not related to database queries + struct query_target *prev; + struct query_target *next; + } internal; +} QUERY_TARGET; + +static inline NEVERNULL QUERY_NODE *query_node(QUERY_TARGET *qt, size_t id) { + internal_fatal(id >= qt->nodes.used, "QUERY: invalid query host id"); + return &qt->nodes.array[id]; +} + +static inline NEVERNULL QUERY_CONTEXT *query_context(QUERY_TARGET *qt, size_t query_context_id) { + internal_fatal(query_context_id >= qt->contexts.used, "QUERY: invalid query context id"); + return &qt->contexts.array[query_context_id]; +} + +static inline NEVERNULL QUERY_INSTANCE *query_instance(QUERY_TARGET *qt, size_t query_instance_id) { + internal_fatal(query_instance_id >= qt->instances.used, "QUERY: invalid query instance id"); + return &qt->instances.array[query_instance_id]; +} + +static inline NEVERNULL QUERY_DIMENSION *query_dimension(QUERY_TARGET *qt, size_t query_dimension_id) { + internal_fatal(query_dimension_id >= qt->dimensions.used, "QUERY: invalid query dimension id"); + return &qt->dimensions.array[query_dimension_id]; +} + +static inline NEVERNULL QUERY_METRIC *query_metric(QUERY_TARGET *qt, size_t id) { + internal_fatal(id >= qt->query.used, "QUERY: invalid query metric id"); + return &qt->query.array[id]; +} + +static inline const char *query_metric_id(QUERY_TARGET *qt, QUERY_METRIC *qm) { + QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id); + return rrdmetric_acquired_id(qd->rma); +} + +static inline const char *query_metric_name(QUERY_TARGET *qt, QUERY_METRIC *qm) { + QUERY_DIMENSION *qd = query_dimension(qt, qm->link.query_dimension_id); + return rrdmetric_acquired_name(qd->rma); +} + +struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier); + +STRING *query_instance_id_fqdn(QUERY_INSTANCE *qi, size_t version); +STRING *query_instance_name_fqdn(QUERY_INSTANCE *qi, size_t version); + +void query_target_free(void); +void query_target_release(QUERY_TARGET *qt); + +QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr); + +struct api_v2_contexts_request { + char *scope_nodes; + char *scope_contexts; + char *nodes; + char *contexts; + char *q; + + time_t timeout_ms; + + qt_interrupt_callback_t interrupt_callback; + void *interrupt_callback_data; +}; + +typedef enum __attribute__ ((__packed__)) { + CONTEXTS_V2_DEBUG = (1 << 0), + CONTEXTS_V2_SEARCH = (1 << 1), + CONTEXTS_V2_NODES = (1 << 2), + CONTEXTS_V2_NODES_DETAILED = (1 << 3), + CONTEXTS_V2_CONTEXTS = (1 << 4), +} CONTEXTS_V2_OPTIONS; + +int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_OPTIONS options); + +RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o); +void buffer_json_agents_array_v2(BUFFER *wb, struct query_timings *timings, time_t now_s); +void buffer_json_node_add_v2(BUFFER *wb, RRDHOST *host, size_t ni, usec_t duration_ut); +void buffer_json_query_timings(BUFFER *wb, const char *key, struct query_timings *timings); +void buffer_json_cloud_timings(BUFFER *wb, const char *key, struct query_timings *timings); + +// ---------------------------------------------------------------------------- +// scope + +typedef ssize_t (*foreach_host_cb_t)(void *data, RRDHOST *host, bool queryable); +ssize_t query_scope_foreach_host(SIMPLE_PATTERN *scope_hosts_sp, SIMPLE_PATTERN *hosts_sp, + foreach_host_cb_t cb, void *data, + struct query_versions *versions, + char *host_node_id_str); + +typedef ssize_t (*foreach_context_cb_t)(void *data, RRDCONTEXT_ACQUIRED *rca, bool queryable_context); +ssize_t query_scope_foreach_context(RRDHOST *host, const char *scope_contexts, SIMPLE_PATTERN *scope_contexts_sp, SIMPLE_PATTERN *contexts_sp, foreach_context_cb_t cb, bool queryable_host, void *data); + +// ---------------------------------------------------------------------------- +// public API for weights + +typedef ssize_t (*weights_add_metric_t)(void *data, RRDHOST *host, RRDCONTEXT_ACQUIRED *rca, RRDINSTANCE_ACQUIRED *ria, RRDMETRIC_ACQUIRED *rma); +ssize_t weights_foreach_rrdmetric_in_context(RRDCONTEXT_ACQUIRED *rca, + SIMPLE_PATTERN *instances_sp, + SIMPLE_PATTERN *chart_label_key_sp, + SIMPLE_PATTERN *labels_sp, + SIMPLE_PATTERN *alerts_sp, + SIMPLE_PATTERN *dimensions_sp, + bool match_ids, bool match_names, + size_t version, + weights_add_metric_t cb, + void *data); + +bool rrdcontext_retention_match(RRDCONTEXT_ACQUIRED *rca, time_t after, time_t before); + +#define query_matches_retention(after, before, first_entry_s, last_entry_s, update_every_s) \ + (((first_entry_s) - ((update_every_s) * 2) <= (before)) && \ + ((last_entry_s) + ((update_every_s) * 2) >= (after))) + +#define query_target_aggregatable(qt) ((qt)->window.options & RRDR_OPTION_RETURN_RAW) + +static inline bool query_target_has_percentage_of_instance(QUERY_TARGET *qt) { + for(size_t g = 0; g < MAX_QUERY_GROUP_BY_PASSES ;g++) + if(qt->request.group_by[g].group_by & RRDR_GROUP_BY_PERCENTAGE_OF_INSTANCE) + return true; + + return false; +} + +static inline bool query_target_needs_all_dimensions(QUERY_TARGET *qt) { + if(qt->request.options & RRDR_OPTION_PERCENTAGE) + return true; + + return query_target_has_percentage_of_instance(qt); +} + +static inline bool query_target_has_percentage_units(QUERY_TARGET *qt) { + if(qt->window.time_group_method == RRDR_GROUPING_CV) + return true; + + if((qt->request.options & RRDR_OPTION_PERCENTAGE) && !(qt->window.options & RRDR_OPTION_RETURN_RAW)) + return true; + + return query_target_has_percentage_of_instance(qt); +} + +#endif // NETDATA_RRDCONTEXT_H + |