summaryrefslogtreecommitdiffstats
path: root/database/rrd.h
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2023-07-20 04:50:01 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2023-07-20 04:50:01 +0000
commitcd4377fab21e0f500bef7f06543fa848a039c1e0 (patch)
treeba00a55e430c052d6bed0b61c0f8bbe8ebedd313 /database/rrd.h
parentReleasing debian version 1.40.1-1. (diff)
downloadnetdata-cd4377fab21e0f500bef7f06543fa848a039c1e0.tar.xz
netdata-cd4377fab21e0f500bef7f06543fa848a039c1e0.zip
Merging upstream version 1.41.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'database/rrd.h')
-rw-r--r--database/rrd.h226
1 files changed, 157 insertions, 69 deletions
diff --git a/database/rrd.h b/database/rrd.h
index 3f125c5a7..95da17c82 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -7,6 +7,8 @@
extern "C" {
#endif
+#include "libnetdata/libnetdata.h"
+
// non-existing structs instead of voids
// to enable type checking at compile time
typedef struct storage_instance STORAGE_INSTANCE;
@@ -95,6 +97,14 @@ extern RRD_MEMORY_MODE default_rrd_memory_mode;
const char *rrd_memory_mode_name(RRD_MEMORY_MODE id);
RRD_MEMORY_MODE rrd_memory_mode_id(const char *name);
+struct ml_metrics_statistics {
+ size_t anomalous;
+ size_t normal;
+ size_t trained;
+ size_t pending;
+ size_t silenced;
+};
+
#include "daemon/common.h"
#include "web/api/queries/query.h"
#include "web/api/queries/rrdr.h"
@@ -225,13 +235,15 @@ typedef enum __attribute__ ((__packed__)) rrddim_options {
RRDDIM_OPTION_HIDDEN = (1 << 0), // this dimension will not be offered to callers
RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers
RRDDIM_OPTION_BACKFILLED_HIGH_TIERS = (1 << 2), // when set, we have backfilled higher tiers
+ RRDDIM_OPTION_UPDATED = (1 << 3), // single-threaded collector updated flag
+ RRDDIM_OPTION_EXPOSED = (1 << 4), // single-threaded collector exposed flag
// this is 8-bit
} RRDDIM_OPTIONS;
-#define rrddim_option_check(rd, option) ((rd)->options & (option))
-#define rrddim_option_set(rd, option) (rd)->options |= (option)
-#define rrddim_option_clear(rd, option) (rd)->options &= ~(option)
+#define rrddim_option_check(rd, option) ((rd)->collector.options & (option))
+#define rrddim_option_set(rd, option) (rd)->collector.options |= (option)
+#define rrddim_option_clear(rd, option) (rd)->collector.options &= ~(option)
// flags are runtime changing status flags (atomics are required to alter/access them)
typedef enum __attribute__ ((__packed__)) rrddim_flags {
@@ -301,7 +313,7 @@ void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src);
void reload_host_labels(void);
void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels);
void rrdset_save_rrdlabels_to_sql(RRDSET *st);
-void rrdhost_set_is_parent_label(int count);
+void rrdhost_set_is_parent_label(void);
int rrdlabels_unittest(void);
// unfortunately this break when defined in exporting_engine.h
@@ -340,70 +352,76 @@ struct rrddim {
STRING *name; // the name of this dimension (as presented to user)
RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values
- RRDDIM_OPTIONS options; // permanent configuration options
RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension
RRDDIM_FLAGS flags; // run time changing status flags
- bool updated; // 1 when the dimension has been updated since the last processing
- bool exposed; // 1 when set what have sent this dimension to the central netdata
-
- collected_number multiplier; // the multiplier of the collected values
- collected_number divisor; // the divider of the collected values
-
- int update_every; // every how many seconds is this updated
- // TODO - remove update_every from rrddim
- // it is always the same in rrdset
+ int32_t multiplier; // the multiplier of the collected values
+ int32_t divisor; // the divider of the collected values
// ------------------------------------------------------------------------
// operational state members
+ struct rrdset *rrdset;
rrd_ml_dimension_t *ml_dimension; // machine learning data about this dimension
+ RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension
- // ------------------------------------------------------------------------
- // linking to siblings and parents
+#ifdef NETDATA_LOG_COLLECTION_ERRORS
+ usec_t rrddim_store_metric_last_ut; // the timestamp we last called rrddim_store_metric()
+ size_t rrddim_store_metric_count; // the rrddim_store_metric() counter
+ const char *rrddim_store_metric_last_caller; // the name of the function that last called rrddim_store_metric()
+#endif
- struct rrdset *rrdset;
+ // ------------------------------------------------------------------------
+ // db mode RAM, SAVE, MAP, ALLOC, NONE specifics
+ // TODO - they should be managed by storage engine
+ // (RRDDIM_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)
- RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension
+ struct {
+ size_t memsize; // the memory allocated for this dimension (without RRDDIM)
+ void *rd_on_file; // pointer to the header written on disk
+ storage_number *data; // the array of values
+ } db;
// ------------------------------------------------------------------------
// data collection members
- struct rrddim_tier tiers[RRD_STORAGE_TIERS]; // our tiers of databases
+ struct {
+ RRDDIM_OPTIONS options; // permanent configuration options
- struct timeval last_collected_time; // when was this dimension last updated
- // this is actual date time we updated the last_collected_value
- // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET
+ uint32_t counter; // the number of times we added values to this rrddim
- size_t collections_counter; // the number of times we added values to this rrddim
- collected_number collected_value_max; // the absolute maximum of the collected value
+ collected_number collected_value; // the current value, as collected - resets to 0 after being used
+ collected_number collected_value_max; // the absolute maximum of the collected value
+ collected_number last_collected_value; // the last value that was collected, after being processed
- NETDATA_DOUBLE calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used
- NETDATA_DOUBLE last_calculated_value; // the last calculated value processed
- NETDATA_DOUBLE last_stored_value; // the last value as stored in the database (after interpolation)
+ struct timeval last_collected_time; // when was this dimension last updated
+ // this is actual date time we updated the last_collected_value
+ // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET
- collected_number collected_value; // the current value, as collected - resets to 0 after being used
- collected_number last_collected_value; // the last value that was collected, after being processed
+ NETDATA_DOUBLE calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used
+ NETDATA_DOUBLE last_calculated_value; // the last calculated value processed
-#ifdef NETDATA_LOG_COLLECTION_ERRORS
- usec_t rrddim_store_metric_last_ut; // the timestamp we last called rrddim_store_metric()
- size_t rrddim_store_metric_count; // the rrddim_store_metric() counter
- const char *rrddim_store_metric_last_caller; // the name of the function that last called rrddim_store_metric()
-#endif
+ NETDATA_DOUBLE last_stored_value; // the last value as stored in the database (after interpolation)
+ } collector;
// ------------------------------------------------------------------------
- // db mode RAM, SAVE, MAP, ALLOC, NONE specifics
- // TODO - they should be managed by storage engine
- // (RRDDIM_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)
- size_t memsize; // the memory allocated for this dimension (without RRDDIM)
- void *rd_on_file; // pointer to the header written on disk
- storage_number *db; // the array of values
+ struct rrddim_tier tiers[]; // our tiers of databases
};
+size_t rrddim_size(void);
+
#define rrddim_id(rd) string2str((rd)->id)
#define rrddim_name(rd) string2str((rd) ->name)
+#define rrddim_check_updated(rd) ((rd)->collector.options & RRDDIM_OPTION_UPDATED)
+#define rrddim_set_updated(rd) (rd)->collector.options |= RRDDIM_OPTION_UPDATED
+#define rrddim_clear_updated(rd) (rd)->collector.options &= ~RRDDIM_OPTION_UPDATED
+
+#define rrddim_check_exposed(rd) ((rd)->collector.options & RRDDIM_OPTION_EXPOSED)
+#define rrddim_set_exposed(rd) (rd)->collector.options |= RRDDIM_OPTION_EXPOSED
+#define rrddim_clear_exposed(rd) (rd)->collector.options &= ~RRDDIM_OPTION_EXPOSED
+
// returns the RRDDIM cache filename, or NULL if it does not exist
const char *rrddim_cache_filename(RRDDIM *rd);
@@ -487,6 +505,48 @@ static inline void storage_engine_store_metric(
count, anomaly_count, flags);
}
+size_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance);
+static inline size_t storage_engine_disk_space_max(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
+#ifdef ENABLE_DBENGINE
+ if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
+ return rrdeng_disk_space_max(db_instance);
+#endif
+
+ return 0;
+}
+
+size_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance);
+static inline size_t storage_engine_disk_space_used(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
+#ifdef ENABLE_DBENGINE
+ if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
+ return rrdeng_disk_space_used(db_instance);
+#endif
+
+ // TODO - calculate the total host disk space for memory mode save and map
+ return 0;
+}
+
+time_t rrdeng_global_first_time_s(STORAGE_INSTANCE *db_instance);
+static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
+#ifdef ENABLE_DBENGINE
+ if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
+ return rrdeng_global_first_time_s(db_instance);
+#endif
+
+ return now_realtime_sec() - (time_t)(default_rrd_history_entries * default_rrd_update_every);
+}
+
+size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *db_instance);
+static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
+#ifdef ENABLE_DBENGINE
+ if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
+ return rrdeng_currently_collected_metrics(db_instance);
+#endif
+
+ // TODO - calculate the total host disk space for memory mode save and map
+ return 0;
+}
+
void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle);
static inline void storage_engine_store_flush(STORAGE_COLLECT_HANDLE *collection_handle) {
@@ -739,18 +799,15 @@ struct rrdset {
STRING *plugin_name; // the name of the plugin that generated this
STRING *module_name; // the name of the plugin module that generated this
- RRDSET_TYPE chart_type; // line, area, stacked
-
- long priority; // the sorting priority of this chart
-
- int update_every; // data collection frequency
+ int32_t priority; // the sorting priority of this chart
+ int32_t update_every; // data collection frequency
DICTIONARY *rrdlabels; // chart labels
DICTIONARY *rrdsetvar_root_index; // chart variables
DICTIONARY *rrddimvar_root_index; // dimension variables
// we use this dictionary to manage their allocation
- rrd_ml_chart_t *ml_chart;
+ RRDSET_TYPE chart_type; // line, area, stacked
// ------------------------------------------------------------------------
// operational state members
@@ -760,6 +817,8 @@ struct rrdset {
DICTIONARY *rrddim_root_index; // dimensions index
+ rrd_ml_chart_t *ml_chart;
+
STORAGE_METRICS_GROUP *storage_metrics_groups[RRD_STORAGE_TIERS];
// ------------------------------------------------------------------------
@@ -775,11 +834,10 @@ struct rrdset {
SPINLOCK data_collection_lock;
- size_t counter; // the number of times we added values to this database
- size_t counter_done; // the number of times rrdset_done() has been called
+ uint32_t counter; // the number of times we added values to this database
+ uint32_t counter_done; // the number of times rrdset_done() has been called
time_t last_accessed_time_s; // the last time this RRDSET has been accessed
-
usec_t usec_since_last_update; // the time in microseconds since the last collection of data
struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function)
@@ -799,18 +857,15 @@ struct rrdset {
// TODO - they should be managed by storage engine
// (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)
- char *cache_dir; // the directory to store dimensions
- void *st_on_file; // compatibility with V019 RRDSET files
-
- // ------------------------------------------------------------------------
- // db mode RAM, SAVE, MAP, ALLOC, NONE specifics
- // TODO - they should be managed by storage engine
- // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)
+ struct {
+ char *cache_dir; // the directory to store dimensions
+ void *st_on_file; // compatibility with V019 RRDSET files
- long entries; // total number of entries in the data set
+ int32_t entries; // total number of entries in the data set
- long current_entry; // the entry that is currently being updated
- // it goes around in a round-robin fashion
+ int32_t current_entry; // the entry that is currently being updated
+ // it goes around in a round-robin fashion
+ } db;
// ------------------------------------------------------------------------
// exporting to 3rd party time-series members
@@ -831,14 +886,14 @@ struct rrdset {
const RRDFAMILY_ACQUIRED *rrdfamily; // pointer to RRDFAMILY dictionary item, this chart belongs to
struct {
- netdata_rwlock_t rwlock; // protection for RRDCALC *base
+ RW_SPINLOCK spinlock; // protection for RRDCALC *base
RRDCALC *base; // double linked list of RRDCALC related to this RRDSET
} alerts;
struct {
- size_t pos;
- size_t size;
- size_t used;
+ uint32_t pos;
+ uint32_t size;
+ uint32_t used;
RRDDIM_ACQUIRED **rda;
} pluginsd;
@@ -924,6 +979,7 @@ typedef enum __attribute__ ((__packed__)) rrdhost_flags {
// ACLK
RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 21), // when set, we should send ACLK stream context updates
RRDHOST_FLAG_ACLK_STREAM_ALERTS = (1 << 22), // set when the receiver part is disconnected
+
// Metadata
RRDHOST_FLAG_METADATA_UPDATE = (1 << 23), // metadata needs to be stored in the database
RRDHOST_FLAG_METADATA_LABELS = (1 << 24), // metadata needs to be stored in the database
@@ -959,6 +1015,8 @@ typedef enum __attribute__ ((__packed__)) {
RRDHOST_OPTION_DELETE_ORPHAN_HOST = (1 << 4), // delete the entire host when orphan
RRDHOST_OPTION_REPLICATION = (1 << 5), // when set, we support replication for this host
+
+ RRDHOST_OPTION_VIRTUAL_HOST = (1 << 6), // when set, this host is a virtual one
} RRDHOST_OPTIONS;
#define rrdhost_option_check(host, flag) ((host)->options & (flag))
@@ -976,6 +1034,7 @@ struct alarm_entry {
uint32_t unique_id;
uint32_t alarm_id;
uint32_t alarm_event_id;
+ usec_t global_id;
uuid_t config_hash_id;
uuid_t transition_id;
@@ -1046,8 +1105,9 @@ typedef struct alarm_log {
uint32_t next_alarm_id;
unsigned int count;
unsigned int max;
+ uint32_t health_log_history; // the health log history in seconds to be kept in db
ALARM_ENTRY *alarms;
- netdata_rwlock_t alarm_log_rwlock;
+ RW_SPINLOCK spinlock;
} ALARM_LOG;
typedef struct health {
@@ -1124,8 +1184,8 @@ struct rrdhost {
RRDHOST_FLAGS flags; // runtime flags about this RRDHOST (atomics on this)
RRDHOST_FLAGS *exporting_flags; // array of flags for exporting connector instances
- int rrd_update_every; // the update frequency of the host
- long rrd_history_entries; // the number of history entries for the host's charts
+ int32_t rrd_update_every; // the update frequency of the host
+ int32_t rrd_history_entries; // the number of history entries for the host's charts
RRD_MEMORY_MODE rrd_memory_mode; // the configured memory more for the charts of this host
// the actual per tier is at .db[tier].mode
@@ -1150,7 +1210,7 @@ struct rrdhost {
struct rrdpush_destinations *destination; // the current destination from the above list
SIMPLE_PATTERN *rrdpush_send_charts_matching; // pattern to match the charts to be sent
- const char *rrdpush_last_receiver_exit_reason;
+ int32_t rrdpush_last_receiver_exit_reason;
time_t rrdpush_seconds_to_replicate; // max time we want to replicate from the child
time_t rrdpush_replication_step; // seconds per replication step
size_t rrdpush_receiver_replicating_charts; // the number of charts currently being replicated from a child
@@ -1163,6 +1223,9 @@ struct rrdhost {
size_t rrdpush_sender_replicating_charts; // the number of charts currently being replicated to a parent
void *aclk_sync_host_config;
+ uint32_t rrdpush_receiver_connection_counter; // the number of times this receiver has connected
+ uint32_t rrdpush_sender_connection_counter; // the number of times this sender has connected
+
// ------------------------------------------------------------------------
// streaming of data from remote hosts - rrdpush receiver
@@ -1225,8 +1288,16 @@ struct rrdhost {
DICTIONARY *contexts;
DICTIONARY *hub_queue;
DICTIONARY *pp_queue;
+ uint32_t metrics;
+ uint32_t instances;
} rrdctx;
+ struct {
+ SPINLOCK spinlock;
+ time_t first_time_s;
+ time_t last_time_s;
+ } retention;
+
uuid_t host_uuid; // Global GUID for this host
uuid_t *node_id; // Cloud node_id
@@ -1260,6 +1331,9 @@ extern RRDHOST *localhost;
#define rrdhost_sender_replicating_charts_minus_one(host) (__atomic_sub_fetch(&((host)->rrdpush_sender_replicating_charts), 1, __ATOMIC_RELAXED))
#define rrdhost_sender_replicating_charts_zero(host) (__atomic_store_n(&((host)->rrdpush_sender_replicating_charts), 0, __ATOMIC_RELAXED))
+#define rrdhost_is_online(host) ((host) == localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST) || !rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN | RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED))
+bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now);
+
extern DICTIONARY *rrdhost_root_index;
size_t rrdhost_hosts_available(void);
@@ -1457,8 +1531,8 @@ RRDDIM *rrddim_add_custom(RRDSET *st
int rrddim_reset_name(RRDSET *st, RRDDIM *rd, const char *name);
int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm);
-int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multiplier);
-int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor);
+int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, int32_t multiplier);
+int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, int32_t divisor);
RRDDIM *rrddim_find(RRDSET *st, const char *id);
RRDDIM_ACQUIRED *rrddim_find_and_acquire(RRDSET *st, const char *id);
@@ -1522,6 +1596,20 @@ void set_host_properties(
size_t get_tier_grouping(size_t tier);
void store_metric_collection_completed(void);
+static inline void rrdhost_retention(RRDHOST *host, time_t now, bool online, time_t *from, time_t *to) {
+ time_t first_time_s = 0, last_time_s = 0;
+ spinlock_lock(&host->retention.spinlock);
+ first_time_s = host->retention.first_time_s;
+ last_time_s = host->retention.last_time_s;
+ spinlock_unlock(&host->retention.spinlock);
+
+ if(from)
+ *from = first_time_s;
+
+ if(to)
+ *to = online ? now : last_time_s;
+}
+
// ----------------------------------------------------------------------------
// RRD DB engine declarations