diff options
Diffstat (limited to 'src/database/rrd.h')
| -rw-r--r-- | src/database/rrd.h | 142 |
1 files changed, 79 insertions, 63 deletions
diff --git a/src/database/rrd.h b/src/database/rrd.h index bd31e21e1..c914b783d 100644 --- a/src/database/rrd.h +++ b/src/database/rrd.h @@ -98,14 +98,55 @@ struct ml_metrics_statistics { size_t silenced; }; + +// use this for configuration flags, not for state control +// flags are set/unset in a manner that is not thread safe +// and may lead to missing information. +typedef enum __attribute__ ((__packed__)) rrdset_flags { + RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart + RRDSET_FLAG_OBSOLETE = (1 << 3), // this is marked by the collector/module as obsolete + RRDSET_FLAG_EXPORTING_SEND = (1 << 4), // if set, this chart should be sent to Prometheus web API and external databases + RRDSET_FLAG_EXPORTING_IGNORE = (1 << 5), // if set, this chart should not be sent to Prometheus web API and external databases + + RRDSET_FLAG_UPSTREAM_SEND = (1 << 6), // if set, this chart should be sent upstream (streaming) + RRDSET_FLAG_UPSTREAM_IGNORE = (1 << 7), // if set, this chart should not be sent upstream (streaming) + + RRDSET_FLAG_STORE_FIRST = (1 << 8), // if set, do not eliminate the first collection during interpolation + RRDSET_FLAG_HETEROGENEOUS = (1 << 9), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers) + RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 10), // if set, the chart should be checked to determine if the dimensions are homogeneous + RRDSET_FLAG_HIDDEN = (1 << 11), // if set, do not show this chart on the dashboard, but use it for exporting + RRDSET_FLAG_SYNC_CLOCK = (1 << 12), // if set, microseconds on next data collection will be ignored (the chart will be synced to now) + RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 13), // this is marked by the collector/module when a chart has obsolete dimensions + + RRDSET_FLAG_METADATA_UPDATE = (1 << 14), // Mark that metadata needs to be stored + RRDSET_FLAG_ANOMALY_DETECTION = (1 << 15), // flag to identify anomaly detection charts. + RRDSET_FLAG_INDEXED_ID = (1 << 16), // the rrdset is indexed by its id + RRDSET_FLAG_INDEXED_NAME = (1 << 17), // the rrdset is indexed by its name + + RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 18), + + RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS = (1 << 19), // the sending side has replication in progress + RRDSET_FLAG_SENDER_REPLICATION_FINISHED = (1 << 20), // the sending side has completed replication + RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS = (1 << 21), // the receiving side has replication in progress + RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED = (1 << 22), // the receiving side has completed replication + + RRDSET_FLAG_UPSTREAM_SEND_VARIABLES = (1 << 23), // a custom variable has been updated and needs to be exposed to parent + + RRDSET_FLAG_COLLECTION_FINISHED = (1 << 24), // when set, data collection is not available for this chart + + RRDSET_FLAG_HAS_RRDCALC_LINKED = (1 << 25), // this chart has at least one rrdcal linked +} RRDSET_FLAGS; + #include "daemon/common.h" #include "web/api/queries/query.h" #include "web/api/queries/rrdr.h" #include "health/rrdvar.h" #include "health/rrdcalc.h" #include "rrdlabels.h" +#include "streaming/stream-capabilities.h" +#include "streaming/stream-path.h" #include "streaming/rrdpush.h" -#include "aclk/aclk_rrdhost_state.h" +//#include "aclk/aclk_rrdhost_state.h" #include "sqlite/sqlite_health.h" typedef struct storage_query_handle STORAGE_QUERY_HANDLE; @@ -268,7 +309,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s // RRD DIMENSION - this is a metric struct rrddim { - nd_uuid_t metric_uuid; // global UUID for this metric (unique_across hosts) + nd_uuid_t metric_uuid; // global UUID for this metric (unique_across hosts) // ------------------------------------------------------------------------ // dimension definition @@ -282,15 +323,16 @@ struct rrddim { int32_t multiplier; // the multiplier of the collected values int32_t divisor; // the divider of the collected values + int32_t dimension_id; // Dimension id // ------------------------------------------------------------------------ // operational state members struct rrdset *rrdset; - rrd_ml_dimension_t *ml_dimension; // machine learning data about this dimension + rrd_ml_dimension_t *ml_dimension; // machine learning data about this dimension struct { - RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension + RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension bool collected; } rrdcontexts; @@ -661,47 +703,6 @@ STORAGE_ENGINE* storage_engine_find(const char* name); // ---------------------------------------------------------------------------- // RRDSET - this is a chart -// use this for configuration flags, not for state control -// flags are set/unset in a manner that is not thread safe -// and may lead to missing information. - -typedef enum __attribute__ ((__packed__)) rrdset_flags { - RRDSET_FLAG_DETAIL = (1 << 1), // if set, the data set should be considered as a detail of another - // (the master data set should be the one that has the same family and is not detail) - RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart - RRDSET_FLAG_OBSOLETE = (1 << 3), // this is marked by the collector/module as obsolete - RRDSET_FLAG_EXPORTING_SEND = (1 << 4), // if set, this chart should be sent to Prometheus web API and external databases - RRDSET_FLAG_EXPORTING_IGNORE = (1 << 5), // if set, this chart should not be sent to Prometheus web API and external databases - - RRDSET_FLAG_UPSTREAM_SEND = (1 << 6), // if set, this chart should be sent upstream (streaming) - RRDSET_FLAG_UPSTREAM_IGNORE = (1 << 7), // if set, this chart should not be sent upstream (streaming) - - RRDSET_FLAG_STORE_FIRST = (1 << 8), // if set, do not eliminate the first collection during interpolation - RRDSET_FLAG_HETEROGENEOUS = (1 << 9), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers) - RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 10), // if set, the chart should be checked to determine if the dimensions are homogeneous - RRDSET_FLAG_HIDDEN = (1 << 11), // if set, do not show this chart on the dashboard, but use it for exporting - RRDSET_FLAG_SYNC_CLOCK = (1 << 12), // if set, microseconds on next data collection will be ignored (the chart will be synced to now) - RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 13), // this is marked by the collector/module when a chart has obsolete dimensions - - RRDSET_FLAG_METADATA_UPDATE = (1 << 14), // Mark that metadata needs to be stored - RRDSET_FLAG_ANOMALY_DETECTION = (1 << 15), // flag to identify anomaly detection charts. - RRDSET_FLAG_INDEXED_ID = (1 << 16), // the rrdset is indexed by its id - RRDSET_FLAG_INDEXED_NAME = (1 << 17), // the rrdset is indexed by its name - - RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 18), - - RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS = (1 << 19), // the sending side has replication in progress - RRDSET_FLAG_SENDER_REPLICATION_FINISHED = (1 << 20), // the sending side has completed replication - RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS = (1 << 21), // the receiving side has replication in progress - RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED = (1 << 22), // the receiving side has completed replication - - RRDSET_FLAG_UPSTREAM_SEND_VARIABLES = (1 << 23), // a custom variable has been updated and needs to be exposed to parent - - RRDSET_FLAG_COLLECTION_FINISHED = (1 << 24), // when set, data collection is not available for this chart - - RRDSET_FLAG_HAS_RRDCALC_LINKED = (1 << 25), // this chart has at least one rrdcal linked -} RRDSET_FLAGS; - #define rrdset_flag_get(st) __atomic_load_n(&((st)->flags), __ATOMIC_ACQUIRE) #define rrdset_flag_check(st, flag) (__atomic_load_n(&((st)->flags), __ATOMIC_ACQUIRE) & (flag)) #define rrdset_flag_set(st, flag) __atomic_or_fetch(&((st)->flags), flag, __ATOMIC_RELEASE) @@ -1094,7 +1095,7 @@ typedef struct alarm_log { uint32_t next_alarm_id; unsigned int count; unsigned int max; - uint32_t health_log_history; // the health log history in seconds to be kept in db + uint32_t health_log_retention_s; // the health log retention in seconds to be kept in db ALARM_ENTRY *alarms; RW_SPINLOCK spinlock; } ALARM_LOG; @@ -1123,6 +1124,7 @@ struct rrdhost_system_info { char *host_os_detection; char *host_cores; char *host_cpu_freq; + char *host_cpu_model; char *host_ram_total; char *host_disk_space; char *container_os_name; @@ -1173,6 +1175,14 @@ struct rrdhost { int32_t rrd_update_every; // the update frequency of the host int32_t rrd_history_entries; // the number of history entries for the host's charts + struct { + uint32_t dimension_count; // Dimension count for this host + uint32_t currently_collected; // Currectly collected metrics cache + time_t cache_timestamp; + Pvoid_t JudyL; // Store metrics collected -- link to rrddim + SPINLOCK spinlock; + } accounting; + RRD_MEMORY_MODE rrd_memory_mode; // the configured memory more for the charts of this host // the actual per tier is at .db[tier].mode @@ -1188,7 +1198,7 @@ struct rrdhost { struct rrdhost_system_info *system_info; // information collected from the host environment // ------------------------------------------------------------------------ - // streaming of data to remote hosts - rrdpush sender + // streaming of data to remote hosts - rrdpush struct { struct { @@ -1204,6 +1214,10 @@ struct rrdhost { uint32_t last_used; // the last slot we used for a chart (increments only) } pluginsd_chart_slots; + + char *destination; // where to send metrics to + char *api_key; // the api key at the receiving netdata + SIMPLE_PATTERN *charts_matching; // pattern to match the charts to be sent } send; struct { @@ -1213,13 +1227,12 @@ struct rrdhost { RRDSET **array; } pluginsd_chart_slots; } receive; + + RRDHOST_STREAM_PATH path; } rrdpush; - char *rrdpush_send_destination; // where to send metrics to - char *rrdpush_send_api_key; // the api key at the receiving netdata struct rrdpush_destinations *destinations; // a linked list of possible destinations struct rrdpush_destinations *destination; // the current destination from the above list - SIMPLE_PATTERN *rrdpush_send_charts_matching; // pattern to match the charts to be sent int32_t rrdpush_last_receiver_exit_reason; time_t rrdpush_seconds_to_replicate; // max time we want to replicate from the child @@ -1247,7 +1260,7 @@ struct rrdhost { int connected_children_count; // number of senders currently streaming struct receiver_state *receiver; - netdata_mutex_t receiver_lock; + SPINLOCK receiver_lock; int trigger_chart_obsoletion_check; // set when child connects, will instruct parent to // trigger a check for obsoleted charts since previous connect @@ -1306,11 +1319,13 @@ struct rrdhost { time_t last_time_s; } retention; - nd_uuid_t host_uuid; // Global GUID for this host - nd_uuid_t *node_id; // Cloud node_id + ND_UUID host_id; // Global GUID for this host + ND_UUID node_id; // Cloud node_id - netdata_mutex_t aclk_state_lock; - aclk_rrdhost_state aclk_state; + struct { + ND_UUID claim_id_of_origin; + ND_UUID claim_id_of_parent; + } aclk; struct rrdhost *next; struct rrdhost *prev; @@ -1325,9 +1340,6 @@ extern RRDHOST *localhost; #define rrdhost_program_name(host) string2str((host)->program_name) #define rrdhost_program_version(host) string2str((host)->program_version) -#define rrdhost_aclk_state_lock(host) netdata_mutex_lock(&((host)->aclk_state_lock)) -#define rrdhost_aclk_state_unlock(host) netdata_mutex_unlock(&((host)->aclk_state_lock)) - #define rrdhost_receiver_replicating_charts(host) (__atomic_load_n(&((host)->rrdpush_receiver_replicating_charts), __ATOMIC_RELAXED)) #define rrdhost_receiver_replicating_charts_plus_one(host) (__atomic_add_fetch(&((host)->rrdpush_receiver_replicating_charts), 1, __ATOMIC_RELAXED)) #define rrdhost_receiver_replicating_charts_minus_one(host) (__atomic_sub_fetch(&((host)->rrdpush_receiver_replicating_charts), 1, __ATOMIC_RELAXED)) @@ -1380,7 +1392,7 @@ void rrddim_index_destroy(RRDSET *st); extern time_t rrdhost_free_orphan_time_s; extern time_t rrdhost_free_ephemeral_time_s; -int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest); +int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool unittest); RRDHOST *rrdhost_find_by_hostname(const char *hostname); RRDHOST *rrdhost_find_by_guid(const char *guid); @@ -1401,9 +1413,9 @@ RRDHOST *rrdhost_find_or_create( RRD_MEMORY_MODE mode, unsigned int health_enabled, unsigned int rrdpush_enabled, - char *rrdpush_destination, - char *rrdpush_api_key, - char *rrdpush_send_charts_matching, + const char *rrdpush_destination, + const char *rrdpush_api_key, + const char *rrdpush_send_charts_matching, bool rrdpush_enable_replication, time_t rrdpush_seconds_to_replicate, time_t rrdpush_replication_step, @@ -1564,7 +1576,6 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, // ---------------------------------------------------------------------------- // Miscellaneous functions -char *rrdset_strncpyz_name(char *to, const char *from, size_t length); void reload_host_labels(void); void rrdhost_set_is_parent_label(void); @@ -1626,6 +1637,11 @@ static inline double rrddim_get_last_stored_value(RRDDIM *rd_dim, double *max_va return value; } +static inline uint32_t get_uint32_id() +{ + return now_realtime_sec() & UINT32_MAX; +} + // // RRD DB engine declarations |