summaryrefslogtreecommitdiffstats
path: root/database/rrd.h
diff options
context:
space:
mode:
Diffstat (limited to 'database/rrd.h')
-rw-r--r--database/rrd.h546
1 files changed, 269 insertions, 277 deletions
diff --git a/database/rrd.h b/database/rrd.h
index dc32b2a2d..605ff50bc 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -3,6 +3,15 @@
#ifndef NETDATA_RRD_H
#define NETDATA_RRD_H 1
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// non-existing structs instead of voids
+// to enable type checking at compile time
+typedef struct storage_instance STORAGE_INSTANCE;
+typedef struct storage_metric_handle STORAGE_METRIC_HANDLE;
+
// forward typedefs
typedef struct rrdhost RRDHOST;
typedef struct rrddim RRDDIM;
@@ -19,10 +28,10 @@ typedef void *ml_host_t;
typedef void *ml_dimension_t;
// forward declarations
-struct rrddim_volatile;
+struct rrddim_tier;
struct rrdset_volatile;
struct context_param;
-struct label;
+
#ifdef ENABLE_DBENGINE
struct rrdeng_page_descr;
struct rrdengine_instance;
@@ -31,6 +40,7 @@ struct pg_cache_page_index;
#include "daemon/common.h"
#include "web/api/queries/query.h"
+#include "web/api/queries/rrdr.h"
#include "rrdvar.h"
#include "rrdsetvar.h"
#include "rrddimvar.h"
@@ -39,6 +49,18 @@ struct pg_cache_page_index;
#include "streaming/rrdpush.h"
#include "aclk/aclk_rrdhost_state.h"
#include "sqlite/sqlite_health.h"
+#include "rrdcontext.h"
+
+extern int storage_tiers;
+extern int storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
+
+typedef enum {
+ RRD_BACKFILL_NONE,
+ RRD_BACKFILL_FULL,
+ RRD_BACKFILL_NEW
+} RRD_BACKFILL;
+
+extern RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS];
enum {
CONTEXT_FLAGS_ARCHIVE = 0x01,
@@ -71,9 +93,6 @@ extern time_t rrdset_free_obsolete_time;
#define RRD_ID_LENGTH_MAX 200
-#define RRDSET_MAGIC "NETDATA RRD SET FILE V019"
-#define RRDDIMENSION_MAGIC "NETDATA RRD DIMENSION FILE V019"
-
typedef long long total_number;
#define TOTAL_NUMBER_FORMAT "%lld"
@@ -177,66 +196,48 @@ typedef enum rrddim_flags {
#define rrddim_flag_set(rd, flag) __atomic_or_fetch(&((rd)->flags), (flag), __ATOMIC_SEQ_CST)
#define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~(flag), __ATOMIC_SEQ_CST)
-typedef enum label_source {
- LABEL_SOURCE_AUTO = 0,
- LABEL_SOURCE_NETDATA_CONF = 1,
- LABEL_SOURCE_DOCKER = 2,
- LABEL_SOURCE_ENVIRONMENT = 3,
- LABEL_SOURCE_KUBERNETES = 4
-} LABEL_SOURCE;
-
-#define LABEL_FLAG_UPDATE_STREAM 1
-#define LABEL_FLAG_STOP_STREAM 2
-
-struct label {
- char *key, *value;
- uint32_t key_hash;
- LABEL_SOURCE label_source;
- struct label *next;
-};
+typedef enum rrdlabel_source {
+ RRDLABEL_SRC_AUTO = (1 << 0), // set when Netdata found the label by some automation
+ RRDLABEL_SRC_CONFIG = (1 << 1), // set when the user configured the label
+ RRDLABEL_SRC_K8S = (1 << 2), // set when this label is found from k8s (RRDLABEL_SRC_AUTO should also be set)
+ RRDLABEL_SRC_ACLK = (1 << 3), // set when this label is found from ACLK (RRDLABEL_SRC_AUTO should also be set)
-struct label_index {
- struct label *head; // Label list
- netdata_rwlock_t labels_rwlock; // lock for the label list
- uint32_t labels_flag; // Flags for labels
-};
+ // more sources can be added here
+
+ RRDLABEL_FLAG_PERMANENT = (1 << 29), // set when this label should never be removed (can be overwritten though)
+ RRDLABEL_FLAG_OLD = (1 << 30), // marks for rrdlabels internal use - they are not exposed outside rrdlabels
+ RRDLABEL_FLAG_NEW = (1 << 31) // marks for rrdlabels internal use - they are not exposed outside rrdlabels
+} RRDLABEL_SRC;
+
+#define RRDLABEL_FLAG_INTERNAL (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT)
+
+extern DICTIONARY *rrdlabels_create(void);
+extern void rrdlabels_destroy(DICTIONARY *labels_dict);
+extern void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls);
+extern void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls);
+extern void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null);
+
+extern void rrdlabels_unmark_all(DICTIONARY *labels);
+extern void rrdlabels_remove_all_unmarked(DICTIONARY *labels);
+
+extern int rrdlabels_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
+extern int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
+
+extern void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb);
+extern bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt);
+extern bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal);
+extern int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them, bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data, void (*name_sanitizer)(char *dst, const char *src, size_t dst_size), void (*value_sanitizer)(char *dst, const char *src, size_t dst_size));
+
+extern void rrdlabels_migrate_to_these(DICTIONARY *dst, DICTIONARY *src);
+extern void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src);
-typedef enum strip_quotes {
- DO_NOT_STRIP_QUOTES,
- STRIP_QUOTES
-} STRIP_QUOTES_OPTION;
-
-typedef enum skip_escaped_characters {
- DO_NOT_SKIP_ESCAPED_CHARACTERS,
- SKIP_ESCAPED_CHARACTERS
-} SKIP_ESCAPED_CHARACTERS_OPTION;
-
-char *translate_label_source(LABEL_SOURCE l);
-struct label *create_label(char *key, char *value, LABEL_SOURCE label_source);
-extern struct label *add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source);
-extern void update_label_list(struct label **labels, struct label *new_labels);
-extern void replace_label_list(struct label_index *labels, struct label *new_labels);
-extern int is_valid_label_value(char *value);
-extern int is_valid_label_key(char *key);
-extern void free_label_list(struct label *labels);
-extern struct label *label_list_lookup_key(struct label *head, char *key, uint32_t key_hash);
-extern struct label *label_list_lookup_keylist(struct label *head, char *keylist);
-extern int label_list_contains_keylist(struct label *head, char *keylist);
-extern int label_list_contains_key(struct label *head, char *key, uint32_t key_hash);
-extern int label_list_contains(struct label *head, struct label *check);
-extern struct label *merge_label_lists(struct label *lo_pri, struct label *hi_pri);
-extern void strip_last_symbol(
- char *str,
- char symbol,
- SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters);
-extern char *strip_double_quotes(char *str, SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters);
void reload_host_labels(void);
-extern void rrdset_add_label_to_new_list(RRDSET *st, char *key, char *value, LABEL_SOURCE source);
-extern void rrdset_finalize_labels(RRDSET *st);
-extern void rrdset_update_labels(RRDSET *st, struct label *labels);
-extern int rrdset_contains_label_keylist(RRDSET *st, char *key);
-extern int rrdset_matches_label_keys(RRDSET *st, char *key, char *words[], uint32_t *hash_key_list, int *word_count, int size);
-extern struct label *rrdset_lookup_label_key(RRDSET *st, char *key, uint32_t key_hash);
+extern void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels);
+
+extern int rrdlabels_unittest(void);
+
+// unfortunately this break when defined in exporting_engine.h
+extern bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
// ----------------------------------------------------------------------------
// RRD DIMENSION - this is a metric
@@ -247,6 +248,8 @@ struct rrddim {
avl_t avl; // the binary index - this has to be first member!
+ uuid_t metric_uuid; // global UUID for this metric (unique_across hosts)
+
// ------------------------------------------------------------------------
// the dimension definition
@@ -255,55 +258,54 @@ struct rrddim {
// this is a pointer to the config structure
// since the config always has a higher priority
// (the user overwrites the name of the charts)
- // DO NOT FREE THIS - IT IS ALLOCATED IN CONFIG
+ uint32_t hash; // a simple hash of the id, to speed up searching / indexing
+ // instead of strcmp() every item in the binary index
+ // we first compare the hashes
+ uint32_t hash_name; // a simple hash of the name
+
RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values
RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension
+ RRDDIM_FLAGS flags; // configuration flags for the dimension
+
+ bool updated; // 1 when the dimension has been updated since the last processing
+ bool exposed; // 1 when set what have sent this dimension to the central netdata
collected_number multiplier; // the multiplier of the collected values
collected_number divisor; // the divider of the collected values
- uint32_t flags; // configuration flags for the dimension
-
// ------------------------------------------------------------------------
// members for temporary data we need for calculations
- uint32_t hash; // a simple hash of the id, to speed up searching / indexing
- // instead of strcmp() every item in the binary index
- // we first compare the hashes
-
- uint32_t hash_name; // a simple hash of the name
-
- char *cache_filename; // the filename we load/save from/to this set
-
- size_t collections_counter; // the number of times we added values to this rrdim
- struct rrddim_volatile *state; // volatile state that is not persistently stored
- size_t unused[8];
-
- collected_number collected_value_max; // the absolute maximum of the collected value
-
- unsigned int updated:1; // 1 when the dimension has been updated since the last processing
- unsigned int exposed:1; // 1 when set what have sent this dimension to the central netdata
-
struct timeval last_collected_time; // when was this dimension last updated
// this is actual date time we updated the last_collected_value
// THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET
- calculated_number calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used
- calculated_number last_calculated_value; // the last calculated value processed
+#ifdef ENABLE_ACLK
+ int aclk_live_status;
+#endif
+ ml_dimension_t ml_dimension;
+
+ struct rrddim_tier *tiers[RRD_STORAGE_TIERS]; // our tiers of databases
+
+ size_t collections_counter; // the number of times we added values to this rrddim
+ collected_number collected_value_max; // the absolute maximum of the collected value
- calculated_number last_stored_value; // the last value as stored in the database (after interpolation)
+ NETDATA_DOUBLE calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used
+ NETDATA_DOUBLE last_calculated_value; // the last calculated value processed
+ NETDATA_DOUBLE last_stored_value; // the last value as stored in the database (after interpolation)
collected_number collected_value; // the current value, as collected - resets to 0 after being used
collected_number last_collected_value; // the last value that was collected, after being processed
// the *_volume members are used to calculate the accuracy of the rounding done by the
// storage number - they are printed to debug.log when debug is enabled for a set.
- calculated_number collected_volume; // the sum of all collected values so far
- calculated_number stored_volume; // the sum of all stored values so far
+ NETDATA_DOUBLE collected_volume; // the sum of all collected values so far
+ NETDATA_DOUBLE stored_volume; // the sum of all stored values so far
struct rrddim *next; // linking of dimensions within the same data set
struct rrdset *rrdset;
+ RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension
// ------------------------------------------------------------------------
// members for checking the data when loading from disk
@@ -314,18 +316,33 @@ struct rrddim {
int update_every; // every how many seconds is this updated
- size_t memsize; // the memory allocated for this dimension
-
- char magic[sizeof(RRDDIMENSION_MAGIC) + 1]; // a string to be saved, used to identify our data file
+ size_t memsize; // the memory allocated for this dimension (without RRDDIM)
struct rrddimvar *variables;
// ------------------------------------------------------------------------
// the values stored in this dimension, using our floating point numbers
- storage_number values[]; // the array of values - THIS HAS TO BE THE LAST MEMBER
+ void *rd_on_file; // pointer to the header written on disk
+ storage_number *db; // the array of values
};
+// returns the RRDDIM cache filename, or NULL if it does not exist
+extern const char *rrddim_cache_filename(RRDDIM *rd);
+
+// updated the header with the latest RRDDIM value, for memory mode MAP and SAVE
+extern void rrddim_memory_file_update(RRDDIM *rd);
+
+// free the memory file structures for memory mode MAP and SAVE
+extern void rrddim_memory_file_free(RRDDIM *rd);
+
+extern bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MODE memory_mode);
+
+// return the v019 header size of RRDDIM files
+extern size_t rrddim_memory_file_header_size(void);
+
+extern void rrddim_memory_file_save(RRDDIM *rd);
+
// ----------------------------------------------------------------------------
// engine-specific iterator state for dimension data collection
typedef struct storage_collect_handle STORAGE_COLLECT_HANDLE;
@@ -340,30 +357,71 @@ struct rrddim_query_handle {
RRDDIM *rd;
time_t start_time;
time_t end_time;
+ TIER_QUERY_FETCH tier_query_fetch_type;
STORAGE_QUERY_HANDLE* handle;
};
+typedef struct storage_point {
+ NETDATA_DOUBLE min; // when count > 1, this is the minimum among them
+ NETDATA_DOUBLE max; // when count > 1, this is the maximum among them
+ NETDATA_DOUBLE sum; // the point sum - divided by count gives the average
+
+ // end_time - start_time = point duration
+ time_t start_time; // the time the point starts
+ time_t end_time; // the time the point ends
+
+ unsigned count; // the number of original points aggregated
+ unsigned anomaly_count; // the number of original points found anomalous
+
+ SN_FLAGS flags; // flags stored with the point
+} STORAGE_POINT;
+
+#define storage_point_unset(x) do { \
+ (x).min = (x).max = (x).sum = NAN; \
+ (x).count = 0; \
+ (x).anomaly_count = 0; \
+ (x).flags = SN_FLAG_NONE; \
+ (x).start_time = 0; \
+ (x).end_time = 0; \
+ } while(0)
+
+#define storage_point_empty(x, start_t, end_t) do { \
+ (x).min = (x).max = (x).sum = NAN; \
+ (x).count = 1; \
+ (x).anomaly_count = 0; \
+ (x).flags = SN_FLAG_NONE; \
+ (x).start_time = start_t; \
+ (x).end_time = end_t; \
+ } while(0)
+
+#define storage_point_is_unset(x) (!(x).count)
+#define storage_point_is_empty(x) (!netdata_double_isnumber((x).sum))
+
// ------------------------------------------------------------------------
// function pointers that handle data collection
struct rrddim_collect_ops {
// an initialization function to run before starting collection
- void (*init)(RRDDIM *rd);
+ STORAGE_COLLECT_HANDLE *(*init)(STORAGE_METRIC_HANDLE *db_metric_handle);
// run this to store each metric into the database
- void (*store_metric)(RRDDIM *rd, usec_t point_in_time, storage_number number);
+ void (*store_metric)(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time, NETDATA_DOUBLE number, NETDATA_DOUBLE min_value,
+ NETDATA_DOUBLE max_value, uint16_t count, uint16_t anomaly_count, SN_FLAGS flags);
+
+ // run this to flush / reset the current data collection sequence
+ void (*flush)(STORAGE_COLLECT_HANDLE *collection_handle);
// an finalization function to run after collection is over
// returns 1 if it's safe to delete the dimension
- int (*finalize)(RRDDIM *rd);
+ int (*finalize)(STORAGE_COLLECT_HANDLE *collection_handle);
};
// function pointers that handle database queries
struct rrddim_query_ops {
// run this before starting a series of next_metric() database queries
- void (*init)(RRDDIM *rd, struct rrddim_query_handle *handle, time_t start_time, time_t end_time);
+ void (*init)(STORAGE_METRIC_HANDLE *db_metric_handle, struct rrddim_query_handle *handle, time_t start_time, time_t end_time, TIER_QUERY_FETCH tier_query_fetch_type);
// run this to load each metric number from the database
- storage_number (*next_metric)(struct rrddim_query_handle *handle, time_t *current_time);
+ STORAGE_POINT (*next_metric)(struct rrddim_query_handle *handle);
// run this to test if the series of next_metric() database queries is finished
int (*is_finished)(struct rrddim_query_handle *handle);
@@ -372,29 +430,31 @@ struct rrddim_query_ops {
void (*finalize)(struct rrddim_query_handle *handle);
// get the timestamp of the last entry of this metric
- time_t (*latest_time)(RRDDIM *rd);
+ time_t (*latest_time)(STORAGE_METRIC_HANDLE *db_metric_handle);
// get the timestamp of the first entry of this metric
- time_t (*oldest_time)(RRDDIM *rd);
+ time_t (*oldest_time)(STORAGE_METRIC_HANDLE *db_metric_handle);
};
+
// ----------------------------------------------------------------------------
-// volatile state per RRD dimension
-struct rrddim_volatile {
-#ifdef ENABLE_DBENGINE
- uuid_t *rrdeng_uuid; // database engine metric UUID
- struct pg_cache_page_index *page_index;
-#endif
-#ifdef ENABLE_ACLK
- int aclk_live_status;
-#endif
- uuid_t metric_uuid; // global UUID for this metric (unique_across hosts)
- STORAGE_COLLECT_HANDLE* handle;
+// Storage tier data for every dimension
+
+struct rrddim_tier {
+ int tier_grouping;
+ RRD_MEMORY_MODE mode; // the memory mode of this tier
+ RRD_BACKFILL backfill; // backfilling configuration
+ STORAGE_METRIC_HANDLE *db_metric_handle; // the metric handle inside the database
+ STORAGE_COLLECT_HANDLE *db_collection_handle; // the data collection handle
+ STORAGE_POINT virtual_point;
+ time_t next_point_time;
+ usec_t last_collected_ut;
struct rrddim_collect_ops collect_ops;
struct rrddim_query_ops query_ops;
- ml_dimension_t ml_dimension;
};
+extern void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, int tier, time_t now);
+
// ----------------------------------------------------------------------------
// volatile state per chart
struct rrdset_volatile {
@@ -402,8 +462,7 @@ struct rrdset_volatile {
char *old_units;
char *old_context;
uuid_t hash_id;
- struct label *new_labels;
- struct label_index labels;
+ DICTIONARY *chart_labels;
bool is_ar_chart;
};
@@ -469,8 +528,6 @@ struct rrdset {
// since the config always has a higher priority
// (the user overwrites the name of the charts)
- void *unused_ptr; // Unused field (previously it held the config section of the chart)
-
char *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options)
char *family; // grouping sets under the same family
char *title; // title shown to user
@@ -479,6 +536,9 @@ struct rrdset {
char *context; // the template of this data set
uint32_t hash_context; // the hash of the chart's context
+ RRDINSTANCE_ACQUIRED *rrdinstance; // the rrdinstance of this chart
+ RRDCONTEXT_ACQUIRED *rrdcontext; // the rrdcontext this chart belongs to
+
RRDSET_TYPE chart_type; // line, area, stacked
int update_every; // every how many seconds is this updated?
@@ -503,7 +563,6 @@ struct rrdset {
RRD_MEMORY_MODE rrd_memory_mode; // if set to 1, this is memory mapped
char *cache_dir; // the directory to store dimensions
- char cache_filename[FILENAME_MAX+1]; // the filename to store this set
netdata_rwlock_t rrdset_rwlock; // protects dimensions linked list
@@ -521,7 +580,6 @@ struct rrdset {
uuid_t *chart_uuid; // Store the global GUID for this chart
// this object.
struct rrdset_volatile *state; // volatile state that is not persistently stored
- size_t unused[3];
size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine
@@ -546,8 +604,8 @@ struct rrdset {
// ------------------------------------------------------------------------
// local variables
- calculated_number green; // green threshold for this chart
- calculated_number red; // red threshold for this chart
+ NETDATA_DOUBLE green; // green threshold for this chart
+ NETDATA_DOUBLE red; // red threshold for this chart
avl_tree_lock rrdvar_root_index; // RRDVAR index for this chart
RRDSETVAR *variables; // RRDSETVAR linked list for this chart (one RRDSETVAR, many RRDVARs)
@@ -557,15 +615,13 @@ struct rrdset {
// members for checking the data when loading from disk
unsigned long memsize; // how much mem we have allocated for this (without dimensions)
-
- char magic[sizeof(RRDSET_MAGIC) + 1]; // our magic
+ void *st_on_file; // compatibility with V019 RRDSET files
// ------------------------------------------------------------------------
// the dimensions
avl_tree_lock dimensions_index; // the root of the dimensions index
RRDDIM *dimensions; // the actual data for every dimension
-
};
#define rrdset_rdlock(st) netdata_rwlock_rdlock(&((st)->rrdset_rwlock))
@@ -583,6 +639,12 @@ struct rrdset {
for((st) = (host)->rrdset_root, rrdhost_check_wrlock(host); st ; (st) = (st)->next)
+extern void rrdset_memory_file_save(RRDSET *st);
+extern void rrdset_memory_file_free(RRDSET *st);
+extern void rrdset_memory_file_update(RRDSET *st);
+extern const char *rrdset_cache_filename(RRDSET *st);
+extern bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY_MODE memory_mode);
+
// ----------------------------------------------------------------------------
// RRDHOST flags
// use this for configuration flags, not for state control
@@ -590,14 +652,17 @@ struct rrdset {
// and may lead to missing information.
typedef enum rrdhost_flags {
- RRDHOST_FLAG_ORPHAN = 1 << 0, // this host is orphan (not receiving data)
- RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = 1 << 1, // delete files of obsolete charts
- RRDHOST_FLAG_DELETE_ORPHAN_HOST = 1 << 2, // delete the entire host when orphan
- RRDHOST_FLAG_EXPORTING_SEND = 1 << 3, // send it to external databases
- RRDHOST_FLAG_EXPORTING_DONT_SEND = 1 << 4, // don't send it to external databases
- RRDHOST_FLAG_ARCHIVED = 1 << 5, // The host is archived, no collected charts yet
- RRDHOST_FLAG_MULTIHOST = 1 << 6, // Host belongs to localhost/megadb
- RRDHOST_FLAG_PENDING_FOREACH_ALARMS = 1 << 7, // contains dims with uninitialized foreach alarms
+ RRDHOST_FLAG_ORPHAN = (1 << 0), // this host is orphan (not receiving data)
+ RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = (1 << 1), // delete files of obsolete charts
+ RRDHOST_FLAG_DELETE_ORPHAN_HOST = (1 << 2), // delete the entire host when orphan
+ RRDHOST_FLAG_EXPORTING_SEND = (1 << 3), // send it to external databases
+ RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 4), // don't send it to external databases
+ RRDHOST_FLAG_ARCHIVED = (1 << 5), // The host is archived, no collected charts yet
+ RRDHOST_FLAG_MULTIHOST = (1 << 6), // Host belongs to localhost/megadb
+ RRDHOST_FLAG_PENDING_FOREACH_ALARMS = (1 << 7), // contains dims with uninitialized foreach alarms
+ RRDHOST_FLAG_STREAM_LABELS_UPDATE = (1 << 8),
+ RRDHOST_FLAG_STREAM_LABELS_STOP = (1 << 9),
+ RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 10), // when set, we should send ACLK stream context updates
} RRDHOST_FLAGS;
#define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag))
@@ -629,6 +694,7 @@ struct alarm_entry {
char *chart;
uint32_t hash_chart;
+ char *chart_context;
char *family;
@@ -646,8 +712,8 @@ struct alarm_entry {
char *units;
char *info;
- calculated_number old_value;
- calculated_number new_value;
+ NETDATA_DOUBLE old_value;
+ NETDATA_DOUBLE new_value;
char *old_value_string;
char *new_value_string;
@@ -740,10 +806,6 @@ struct rrdhost {
const char *tags; // tags for this host
const char *timezone; // the timezone of the host
-#ifdef ENABLE_ACLK
- long deleted_charts_count;
-#endif
-
const char *abbrev_timezone; // the abbriviated timezone of the host
int32_t utc_offset; // the offset in seconds from utc
@@ -778,7 +840,7 @@ struct rrdhost {
netdata_thread_t rrdpush_sender_thread; // the sender thread
void *dbsync_worker;
- volatile unsigned int rrdpush_sender_connected; // 1 when the sender is ready to push metrics
+ bool rrdpush_sender_connected; // 1 when the sender is ready to push metrics
int rrdpush_sender_socket; // the fd of the socket to the remote host, or -1
volatile unsigned int rrdpush_sender_error_shown; // 1 when we have logged a communication error
@@ -859,7 +921,7 @@ struct rrdhost {
// ------------------------------------------------------------------------
// Support for host-level labels
- struct label_index labels;
+ DICTIONARY *host_labels;
// ------------------------------------------------------------------------
// indexes
@@ -870,9 +932,11 @@ struct rrdhost {
avl_tree_lock rrdfamily_root_index; // the host's chart families index
avl_tree_lock rrdvar_root_index; // the host's chart variables index
-#ifdef ENABLE_DBENGINE
- struct rrdengine_instance *rrdeng_ctx; // DB engine instance for this host
-#endif
+ STORAGE_INSTANCE *storage_instance[RRD_STORAGE_TIERS]; // the database instances of the storage tiers
+
+ RRDCONTEXTS *rrdctx_queue;
+ RRDCONTEXTS *rrdctx;
+
uuid_t host_uuid; // Global GUID for this host
uuid_t *node_id; // Cloud node_id
@@ -916,6 +980,10 @@ extern netdata_rwlock_t rrd_rwlock;
// ----------------------------------------------------------------------------
+extern bool is_storage_engine_shared(STORAGE_INSTANCE *engine);
+
+// ----------------------------------------------------------------------------
+
extern size_t rrd_hosts_available;
extern time_t rrdhost_free_orphan_time;
@@ -944,6 +1012,7 @@ extern RRDHOST *rrdhost_find_or_create(
, char *rrdpush_api_key
, char *rrdpush_send_charts_matching
, struct rrdhost_system_info *system_info
+ , bool is_archived
);
extern void rrdhost_update(RRDHOST *host
@@ -1027,7 +1096,7 @@ extern void rrdhost_cleanup_all(void);
extern void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected_host);
extern void rrdhost_system_info_free(struct rrdhost_system_info *system_info);
-extern void rrdhost_free(RRDHOST *host);
+extern void rrdhost_free(RRDHOST *host, bool force);
extern void rrdhost_save_charts(RRDHOST *host);
extern void rrdhost_delete_charts(RRDHOST *host);
extern void rrd_cleanup_obsolete_charts();
@@ -1083,28 +1152,49 @@ extern void rrdset_isnot_obsolete(RRDSET *st);
#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
#define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
-// get the total duration in seconds of the round robin database
-#define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every ))
-
// get the timestamp of the last entry in the round robin database
-static inline time_t rrdset_last_entry_t_nolock(RRDSET *st)
-{
- if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
- RRDDIM *rd;
- time_t last_entry_t = 0;
+static inline time_t rrddim_last_entry_t(RRDDIM *rd) {
+ time_t latest = rd->tiers[0]->query_ops.latest_time(rd->tiers[0]->db_metric_handle);
- rrddim_foreach_read(rd, st) {
- last_entry_t = MAX(last_entry_t, rd->state->query_ops.latest_time(rd));
- }
+ for(int tier = 1; tier < storage_tiers ;tier++) {
+ if(unlikely(!rd->tiers[tier])) continue;
- return last_entry_t;
- } else {
- return (time_t)st->last_updated.tv_sec;
+ time_t t = rd->tiers[tier]->query_ops.latest_time(rd->tiers[tier]->db_metric_handle);
+ if(t > latest)
+ latest = t;
}
+
+ return latest;
}
-static inline time_t rrdset_last_entry_t(RRDSET *st)
-{
+static inline time_t rrddim_first_entry_t(RRDDIM *rd) {
+ time_t oldest = 0;
+
+ for(int tier = 0; tier < storage_tiers ;tier++) {
+ if(unlikely(!rd->tiers[tier])) continue;
+
+ time_t t = rd->tiers[tier]->query_ops.oldest_time(rd->tiers[tier]->db_metric_handle);
+ if(t != 0 && (oldest == 0 || t < oldest))
+ oldest = t;
+ }
+
+ return oldest;
+}
+
+// get the timestamp of the last entry in the round robin database
+static inline time_t rrdset_last_entry_t_nolock(RRDSET *st) {
+ RRDDIM *rd;
+ time_t last_entry_t = 0;
+
+ rrddim_foreach_read(rd, st) {
+ time_t t = rrddim_last_entry_t(rd);
+ if(t > last_entry_t) last_entry_t = t;
+ }
+
+ return last_entry_t;
+}
+
+static inline time_t rrdset_last_entry_t(RRDSET *st) {
time_t last_entry_t;
netdata_rwlock_rdlock(&st->rrdset_rwlock);
@@ -1115,24 +1205,18 @@ static inline time_t rrdset_last_entry_t(RRDSET *st)
}
// get the timestamp of first entry in the round robin database
-static inline time_t rrdset_first_entry_t_nolock(RRDSET *st)
-{
- if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
- RRDDIM *rd;
- time_t first_entry_t = LONG_MAX;
-
- rrddim_foreach_read(rd, st) {
- first_entry_t =
- MIN(first_entry_t,
- rd->state->query_ops.oldest_time(rd) > st->update_every ?
- rd->state->query_ops.oldest_time(rd) - st->update_every : 0);
- }
-
- if (unlikely(LONG_MAX == first_entry_t)) return 0;
- return first_entry_t;
- } else {
- return (time_t)(rrdset_last_entry_t_nolock(st) - rrdset_duration(st));
+static inline time_t rrdset_first_entry_t_nolock(RRDSET *st) {
+ RRDDIM *rd;
+ time_t first_entry_t = LONG_MAX;
+
+ rrddim_foreach_read(rd, st) {
+ time_t t = rrddim_first_entry_t(rd);
+ if(t < first_entry_t)
+ first_entry_t = t;
}
+
+ if (unlikely(LONG_MAX == first_entry_t)) return 0;
+ return first_entry_t;
}
static inline time_t rrdset_first_entry_t(RRDSET *st)
@@ -1146,107 +1230,8 @@ static inline time_t rrdset_first_entry_t(RRDSET *st)
return first_entry_t;
}
-// get the timestamp of the last entry in the round robin database
-static inline time_t rrddim_last_entry_t(RRDDIM *rd) {
- if (rd->rrdset->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
- return rd->state->query_ops.latest_time(rd);
- return (time_t)rd->rrdset->last_updated.tv_sec;
-}
-
-static inline time_t rrddim_first_entry_t(RRDDIM *rd) {
- if (rd->rrdset->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
- return rd->state->query_ops.oldest_time(rd);
- return (time_t)(rd->rrdset->last_updated.tv_sec - rrdset_duration(rd->rrdset));
-}
-
time_t rrdhost_last_entry_t(RRDHOST *h);
-// get the last slot updated in the round robin database
-#define rrdset_last_slot(st) ((size_t)(((st)->current_entry == 0) ? (st)->entries - 1 : (st)->current_entry - 1))
-
-// get the first / oldest slot updated in the round robin database
-// #define rrdset_first_slot(st) ((size_t)( (((st)->counter >= ((unsigned long)(st)->entries)) ? (unsigned long)( ((unsigned long)(st)->current_entry > 0) ? ((unsigned long)(st)->current_entry) : ((unsigned long)(st)->entries) ) - 1 : 0) ))
-
-// return the slot that has the oldest value
-
-static inline size_t rrdset_first_slot(RRDSET *st) {
- if(st->counter >= (size_t)st->entries) {
- // the database has been rotated at least once
- // the oldest entry is the one that will be next
- // overwritten by data collection
- return (size_t)st->current_entry;
- }
-
- // we do not have rotated the db yet
- // so 0 is the first entry
- return 0;
-}
-
-// get the slot of the round robin database, for the given timestamp (t)
-// it always returns a valid slot, although may not be for the time requested if the time is outside the round robin database
-// only valid when not using dbengine
-static inline size_t rrdset_time2slot(RRDSET *st, time_t t) {
- size_t ret = 0;
- time_t last_entry_t = rrdset_last_entry_t_nolock(st);
- time_t first_entry_t = rrdset_first_entry_t_nolock(st);
-
- if(t >= last_entry_t) {
- // the requested time is after the last entry we have
- ret = rrdset_last_slot(st);
- }
- else {
- if(t <= first_entry_t) {
- // the requested time is before the first entry we have
- ret = rrdset_first_slot(st);
- }
- else {
- if(rrdset_last_slot(st) >= ((last_entry_t - t) / (size_t)(st->update_every)))
- ret = rrdset_last_slot(st) - ((last_entry_t - t) / (size_t)(st->update_every));
- else
- ret = rrdset_last_slot(st) - ((last_entry_t - t) / (size_t)(st->update_every)) + (unsigned long)st->entries;
- }
- }
-
- if(unlikely(ret >= (size_t)st->entries)) {
- error("INTERNAL ERROR: rrdset_time2slot() on %s returns values outside entries", st->name);
- ret = (size_t)(st->entries - 1);
- }
-
- return ret;
-}
-
-// get the timestamp of a specific slot in the round robin database
-// only valid when not using dbengine
-static inline time_t rrdset_slot2time(RRDSET *st, size_t slot) {
- time_t ret;
- time_t last_entry_t = rrdset_last_entry_t_nolock(st);
- time_t first_entry_t = rrdset_first_entry_t_nolock(st);
-
- if(slot >= (size_t)st->entries) {
- error("INTERNAL ERROR: caller of rrdset_slot2time() gives invalid slot %zu", slot);
- slot = (size_t)st->entries - 1;
- }
-
- if(slot > rrdset_last_slot(st)) {
- ret = last_entry_t - (size_t)st->update_every * (rrdset_last_slot(st) - slot + (size_t)st->entries);
- }
- else {
- ret = last_entry_t - (size_t)st->update_every;
- }
-
- if(unlikely(ret < first_entry_t)) {
- error("INTERNAL ERROR: rrdset_slot2time() on %s returns time too far in the past", st->name);
- ret = first_entry_t;
- }
-
- if(unlikely(ret > last_entry_t)) {
- error("INTERNAL ERROR: rrdset_slot2time() on %s returns time into the future", st->name);
- ret = last_entry_t;
- }
-
- return ret;
-}
-
// ----------------------------------------------------------------------------
// RRD DIMENSION functions
@@ -1281,7 +1266,7 @@ extern void rrddim_isnot_obsolete(RRDSET *st, RRDDIM *rd);
extern collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number value);
extern collected_number rrddim_set(RRDSET *st, const char *id, collected_number value);
-#if defined(ENABLE_ACLK) && defined(ENABLE_NEW_CLOUD_PROTOCOL)
+#ifdef ENABLE_ACLK
extern time_t calc_dimension_liveness(RRDDIM *rd, time_t now);
#endif
extern long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries);
@@ -1319,8 +1304,7 @@ extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st);
extern void rrdset_free(RRDSET *st);
extern void rrdset_reset(RRDSET *st);
extern void rrdset_save(RRDSET *st);
-#define rrdset_delete(st) rrdset_delete_custom(st, 0)
-extern void rrdset_delete_custom(RRDSET *st, int db_rotated);
+extern void rrdset_delete_files(RRDSET *st);
extern void rrdset_delete_obsolete_dimensions(RRDSET *st);
extern RRDHOST *rrdhost_create(
@@ -1328,7 +1312,7 @@ extern RRDHOST *rrdhost_create(
const char *abbrev_timezone, int32_t utc_offset,const char *tags, const char *program_name, const char *program_version,
int update_every, long entries, RRD_MEMORY_MODE memory_mode, unsigned int health_enabled, unsigned int rrdpush_enabled,
char *rrdpush_destination, char *rrdpush_api_key, char *rrdpush_send_charts_matching, struct rrdhost_system_info *system_info,
- int is_localhost); //TODO: Remove , int is_archived);
+ int is_localhost, bool is_archived);
#endif /* NETDATA_RRD_INTERNALS */
@@ -1337,6 +1321,8 @@ extern void set_host_properties(
const char *guid, const char *os, const char *tags, const char *tzone, const char *abbrev_tzone, int32_t utc_offset,
const char *program_name, const char *program_version);
+extern int get_tier_grouping(int tier);
+
// ----------------------------------------------------------------------------
// RRD DB engine declarations
@@ -1344,9 +1330,15 @@ extern void set_host_properties(
#include "database/engine/rrdengineapi.h"
#endif
#include "sqlite/sqlite_functions.h"
+#include "sqlite/sqlite_context.h"
#include "sqlite/sqlite_aclk.h"
#include "sqlite/sqlite_aclk_chart.h"
#include "sqlite/sqlite_aclk_alert.h"
#include "sqlite/sqlite_aclk_node.h"
#include "sqlite/sqlite_health.h"
+
+#ifdef __cplusplus
+}
+#endif
+
#endif /* NETDATA_RRD_H */