diff options
Diffstat (limited to 'src/rrd.h')
| -rw-r--r-- | src/rrd.h | 358 |
1 files changed, 215 insertions, 143 deletions
@@ -1,10 +1,3 @@ -#include <sys/time.h> -#include <pthread.h> -#include <stdint.h> - -#include "avl.h" -#include "storage_number.h" - #ifndef NETDATA_RRD_H #define NETDATA_RRD_H 1 @@ -20,10 +13,10 @@ extern int rrd_default_history_entries; // set to zero to disable this feature extern int rrd_delete_unupdated_dimensions; -#define RRD_ID_LENGTH_MAX 1024 +#define RRD_ID_LENGTH_MAX 400 -#define RRDSET_MAGIC "NETDATA RRD SET FILE V017" -#define RRDDIMENSION_MAGIC "NETDATA RRD DIMENSION FILE V017" +#define RRDSET_MAGIC "NETDATA RRD SET FILE V018" +#define RRDDIMENSION_MAGIC "NETDATA RRD DIMENSION FILE V018" typedef long long total_number; #define TOTAL_NUMBER_FORMAT "%lld" @@ -35,8 +28,8 @@ typedef long long total_number; #define RRDSET_TYPE_AREA_NAME "area" #define RRDSET_TYPE_STACKED_NAME "stacked" -#define RRDSET_TYPE_LINE 0 -#define RRDSET_TYPE_AREA 1 +#define RRDSET_TYPE_LINE 0 +#define RRDSET_TYPE_AREA 1 #define RRDSET_TYPE_STACKED 2 int rrdset_type_id(const char *name); @@ -63,15 +56,15 @@ extern int rrd_memory_mode_id(const char *name); // ---------------------------------------------------------------------------- // algorithms types -#define RRDDIM_ABSOLUTE_NAME "absolute" -#define RRDDIM_INCREMENTAL_NAME "incremental" -#define RRDDIM_PCENT_OVER_DIFF_TOTAL_NAME "percentage-of-incremental-row" -#define RRDDIM_PCENT_OVER_ROW_TOTAL_NAME "percentage-of-absolute-row" +#define RRDDIM_ABSOLUTE_NAME "absolute" +#define RRDDIM_INCREMENTAL_NAME "incremental" +#define RRDDIM_PCENT_OVER_DIFF_TOTAL_NAME "percentage-of-incremental-row" +#define RRDDIM_PCENT_OVER_ROW_TOTAL_NAME "percentage-of-absolute-row" -#define RRDDIM_ABSOLUTE 0 -#define RRDDIM_INCREMENTAL 1 -#define RRDDIM_PCENT_OVER_DIFF_TOTAL 2 -#define RRDDIM_PCENT_OVER_ROW_TOTAL 3 +#define RRDDIM_ABSOLUTE 0 +#define RRDDIM_INCREMENTAL 1 +#define RRDDIM_PCENT_OVER_DIFF_TOTAL 2 +#define RRDDIM_PCENT_OVER_ROW_TOTAL 3 extern int rrddim_algorithm_id(const char *name); extern const char *rrddim_algorithm_name(int chart_type); @@ -83,80 +76,103 @@ extern const char *rrddim_algorithm_name(int chart_type); #define RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS 0x00000002 // do not offer RESET or OVERFLOW info to callers // ---------------------------------------------------------------------------- +// RRD CONTEXT + +struct rrdfamily { + avl avl; + + const char *family; + uint32_t hash_family; + + size_t use_count; + + avl_tree_lock variables_root_index; +}; +typedef struct rrdfamily RRDFAMILY; + +// ---------------------------------------------------------------------------- // RRD DIMENSION struct rrddim { - // ------------------------------------------------------------------------ - // binary indexing structures + // ------------------------------------------------------------------------ + // binary indexing structures + + avl avl; // the binary index - this has to be first member! + + // ------------------------------------------------------------------------ + // the dimension definition + + char id[RRD_ID_LENGTH_MAX + 1]; // the id of this dimension (for internal identification) - avl avl; // the binary index - this has to be first member! + const char *name; // the name of this dimension (as presented to user) + // this is a pointer to the config structure + // since the config always has a higher priority + // (the user overwrites the name of the charts) - // ------------------------------------------------------------------------ - // the dimension definition + int algorithm; // the algorithm that is applied to add new collected values + long multiplier; // the multiplier of the collected values + long divisor; // the divider of the collected values - char id[RRD_ID_LENGTH_MAX + 1]; // the id of this dimension (for internal identification) + int mapped; // if set to non zero, this dimension is mapped to a file - const char *name; // the name of this dimension (as presented to user) - // this is a pointer to the config structure - // since the config always has a higher priority - // (the user overwrites the name of the charts) + // ------------------------------------------------------------------------ + // members for temporary data we need for calculations - int algorithm; // the algorithm that is applied to add new collected values - long multiplier; // the multiplier of the collected values - long divisor; // the divider of the collected values + uint32_t hash; // a simple hash of the id, to speed up searching / indexing + // instead of strcmp() every item in the binary index + // we first compare the hashes - int mapped; // if set to non zero, this dimension is mapped to a file + // FIXME + // we need the hash_name too! - // ------------------------------------------------------------------------ - // members for temporary data we need for calculations + uint32_t flags; - uint32_t hash; // a simple hash of the id, to speed up searching / indexing - // instead of strcmp() every item in the binary index - // we first compare the hashes + char cache_filename[FILENAME_MAX+1]; // the filename we load/save from/to this set - uint32_t flags; + unsigned long counter; // the number of times we added values to this rrdim - char cache_filename[FILENAME_MAX+1]; // the filename we load/save from/to this set + int updated; // set to 0 after each calculation, to 1 after each collected value + // we use this to detect that a dimension is not updated - unsigned long counter; // the number of times we added values to this rrdim + struct timeval last_collected_time; // when was this dimension last updated + // this is actual date time we updated the last_collected_value + // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET - int updated; // set to 0 after each calculation, to 1 after each collected value - // we use this to detect that a dimension is not updated + calculated_number calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used + calculated_number last_calculated_value; // the last calculated value processed - struct timeval last_collected_time; // when was this dimension last updated - // this is actual date time we updated the last_collected_value - // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET + calculated_number last_stored_value; // the last value as stored in the database (after interpolation) - calculated_number calculated_value; // the current calculated value, after applying the algorithm - calculated_number last_calculated_value; // the last calculated value + collected_number collected_value; // the current value, as collected - resets to 0 after being used + collected_number last_collected_value; // the last value that was collected, after being processed - collected_number collected_value; // the current value, as collected - collected_number last_collected_value; // the last value that was collected + // the *_volume members are used to calculate the accuracy of the rounding done by the + // storage number - they are printed to debug.log when debug is enabled for a set. + calculated_number collected_volume; // the sum of all collected values so far + calculated_number stored_volume; // the sum of all stored values so far - // the *_volume members are used to calculate the accuracy of the rounding done by the - // storage number - they are printed to debug.log when debug is enabled for a set. - calculated_number collected_volume; // the sum of all collected values so far - calculated_number stored_volume; // the sum of all stored values so far + struct rrddim *next; // linking of dimensions within the same data set + struct rrdset *rrdset; - struct rrddim *next; // linking of dimensions within the same data set + // ------------------------------------------------------------------------ + // members for checking the data when loading from disk - // ------------------------------------------------------------------------ - // members for checking the data when loading from disk + long entries; // how many entries this dimension has in ram + // this is the same to the entries of the data set + // we set it here, to check the data when we load it from disk. - long entries; // how many entries this dimension has in ram - // this is the same to the entries of the data set - // we set it here, to check the data when we load it from disk. + int update_every; // every how many seconds is this updated - int update_every; // every how many seconds is this updated + unsigned long memsize; // the memory allocated for this dimension - unsigned long memsize; // the memory allocated for this dimension + char magic[sizeof(RRDDIMENSION_MAGIC) + 1]; // a string to be saved, used to identify our data file - char magic[sizeof(RRDDIMENSION_MAGIC) + 1]; // a string to be saved, used to identify our data file + struct rrddimvar *variables; - // ------------------------------------------------------------------------ - // the values stored in this dimension, using our floating point numbers + // ------------------------------------------------------------------------ + // the values stored in this dimension, using our floating point numbers - storage_number values[]; // the array of values - THIS HAS TO BE THE LAST MEMBER + storage_number values[]; // the array of values - THIS HAS TO BE THE LAST MEMBER }; typedef struct rrddim RRDDIM; @@ -165,99 +181,155 @@ typedef struct rrddim RRDDIM; // RRDSET struct rrdset { - // ------------------------------------------------------------------------ - // binary indexing structures + // ------------------------------------------------------------------------ + // binary indexing structures + + avl avl; // the index, with key the id - this has to be first! + avl avlname; // the index, with key the name + + // ------------------------------------------------------------------------ + // the set configuration + + char id[RRD_ID_LENGTH_MAX + 1]; // id of the data set - avl avl; // the index, with key the id - this has to be first! - avl avlname; // the index, with key the name + const char *name; // the name of this dimension (as presented to user) + // this is a pointer to the config structure + // since the config always has a higher priority + // (the user overwrites the name of the charts) - // ------------------------------------------------------------------------ - // the set configuration + char *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options) + char *family; // grouping sets under the same family + char *title; // title shown to user + char *units; // units of measurement - char id[RRD_ID_LENGTH_MAX + 1]; // id of the data set + char *context; // the template of this data set + uint32_t hash_context; - const char *name; // the name of this dimension (as presented to user) - // this is a pointer to the config structure - // since the config always has a higher priority - // (the user overwrites the name of the charts) + int chart_type; - char *type; // the type of graph RRD_TYPE_* (a category, for determining graphing options) - char *family; // grouping sets under the same family - char *context; // the template of this data set - char *title; // title shown to user - char *units; // units of measurement + int update_every; // every how many seconds is this updated? - int chart_type; + long entries; // total number of entries in the data set - int update_every; // every how many seconds is this updated? + long current_entry; // the entry that is currently being updated + // it goes around in a round-robin fashion - long entries; // total number of entries in the data set + int enabled; - long current_entry; // the entry that is currently being updated - // it goes around in a round-robin fashion + int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored + // netdata will interpolate values for gaps lower than this - int enabled; + long priority; - int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored - // netdata will interpolate values for gaps lower than this + int isdetail; // if set, the data set should be considered as a detail of another + // (the master data set should be the one that has the same family and is not detail) - long priority; + // ------------------------------------------------------------------------ + // members for temporary data we need for calculations - int isdetail; // if set, the data set should be considered as a detail of another - // (the master data set should be the one that has the same family and is not detail) + int mapped; // if set to 1, this is memory mapped - // ------------------------------------------------------------------------ - // members for temporary data we need for calculations + int debug; - int mapped; // if set to 1, this is memory mapped + char *cache_dir; // the directory to store dimensions + char cache_filename[FILENAME_MAX+1]; // the filename to store this set - int debug; + pthread_rwlock_t rwlock; - char *cache_dir; // the directory to store dimensions - char cache_filename[FILENAME_MAX+1]; // the filename to store this set + unsigned long counter; // the number of times we added values to this rrd + unsigned long counter_done; // the number of times we added values to this rrd - pthread_rwlock_t rwlock; + uint32_t hash; // a simple hash on the id, to speed up searching + // we first compare hashes, and only if the hashes are equal we do string comparisons - unsigned long counter; // the number of times we added values to this rrd - unsigned long counter_done; // the number of times we added values to this rrd + uint32_t hash_name; // a simple hash on the name - uint32_t hash; // a simple hash on the id, to speed up searching - // we first compare hashes, and only if the hashes are equal we do string comparisons + unsigned long long usec_since_last_update; // the time in microseconds since the last collection of data - uint32_t hash_name; // a simple hash on the name + struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function) + struct timeval last_collected_time; // when did this data set last collected values - unsigned long long usec_since_last_update; // the time in microseconds since the last collection of data + total_number collected_total; // used internally to calculate percentages + total_number last_collected_total; // used internally to calculate percentages - struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function) - struct timeval last_collected_time; // when did this data set last collected values + RRDFAMILY *rrdfamily; + struct rrdhost *rrdhost; - total_number collected_total; // used internally to calculate percentages - total_number last_collected_total; // used internally to calculate percentages + struct rrdset *next; // linking of rrdsets - struct rrdset *next; // linking of rrdsets + // ------------------------------------------------------------------------ + // local variables - // ------------------------------------------------------------------------ - // members for checking the data when loading from disk + calculated_number green; + calculated_number red; - unsigned long memsize; // how much mem we have allocated for this (without dimensions) + avl_tree_lock variables_root_index; + RRDSETVAR *variables; + RRDCALC *alarms; - char magic[sizeof(RRDSET_MAGIC) + 1]; // our magic + // ------------------------------------------------------------------------ + // members for checking the data when loading from disk - // ------------------------------------------------------------------------ - // the dimensions + unsigned long memsize; // how much mem we have allocated for this (without dimensions) + + char magic[sizeof(RRDSET_MAGIC) + 1]; // our magic + + // ------------------------------------------------------------------------ + // the dimensions + + avl_tree_lock dimensions_index; // the root of the dimensions index + RRDDIM *dimensions; // the actual data for every dimension - avl_tree_lock dimensions_index; // the root of the dimensions index - RRDDIM *dimensions; // the actual data for every dimension }; typedef struct rrdset RRDSET; -extern RRDSET *rrdset_root; -extern pthread_rwlock_t rrdset_root_rwlock; +// ---------------------------------------------------------------------------- +// RRD HOST + +struct rrdhost { + avl avl; + + char *hostname; + + RRDSET *rrdset_root; + pthread_rwlock_t rrdset_root_rwlock; + + avl_tree_lock rrdset_root_index; + avl_tree_lock rrdset_root_index_name; + + avl_tree_lock rrdfamily_root_index; + avl_tree_lock variables_root_index; + + // all RRDCALCs are primarily allocated and linked here + // RRDCALCs may be linked to charts at any point + // (charts may or may not exist when these are loaded) + RRDCALC *alarms; + ALARM_LOG health_log; + + RRDCALCTEMPLATE *templates; +}; +typedef struct rrdhost RRDHOST; +extern RRDHOST localhost; + +#ifdef NETDATA_INTERNAL_CHECKS +#define rrdhost_check_wrlock(host) rrdhost_check_wrlock_int(host, __FILE__, __FUNCTION__, __LINE__) +#define rrdhost_check_rdlock(host) rrdhost_check_rdlock_int(host, __FILE__, __FUNCTION__, __LINE__) +#else +#define rrdhost_check_rdlock(host) (void)0 +#define rrdhost_check_wrlock(host) (void)0 +#endif + +extern void rrdhost_check_wrlock_int(RRDHOST *host, const char *file, const char *function, const unsigned long line); +extern void rrdhost_check_rdlock_int(RRDHOST *host, const char *file, const char *function, const unsigned long line); + +extern void rrdhost_rwlock(RRDHOST *host); +extern void rrdhost_rdlock(RRDHOST *host); +extern void rrdhost_unlock(RRDHOST *host); // ---------------------------------------------------------------------------- // RRD SET functions -extern char *rrdset_strncpy_name(char *to, const char *from, int length); +extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length); extern void rrdset_set_name(RRDSET *st, const char *name); extern char *rrdset_cache_dir(const char *id); @@ -265,15 +337,15 @@ extern char *rrdset_cache_dir(const char *id); extern void rrdset_reset(RRDSET *st); extern RRDSET *rrdset_create(const char *type - , const char *id - , const char *name - , const char *family - , const char *context - , const char *title - , const char *units - , long priority - , int update_every - , int chart_type); + , const char *id + , const char *name + , const char *family + , const char *context + , const char *title + , const char *units + , long priority + , int update_every + , int chart_type); extern void rrdset_free_all(void); extern void rrdset_save_all(void); @@ -306,20 +378,20 @@ extern unsigned long long rrdset_done(RRDSET *st); // get the slot of the round robin database, for the given timestamp (t) // it always returns a valid slot, although may not be for the time requested if the time is outside the round robin database #define rrdset_time2slot(st, t) ( \ - ( (time_t)(t) >= rrdset_last_entry_t(st)) ? ( rrdset_last_slot(st) ) : \ - ( ((time_t)(t) <= rrdset_first_entry_t(st)) ? rrdset_first_slot(st) : \ - ( (rrdset_last_slot(st) >= (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) ) ? \ - (rrdset_last_slot(st) - (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) ) : \ - (rrdset_last_slot(st) - (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) + (unsigned long)(st)->entries ) \ - ))) + ( (time_t)(t) >= rrdset_last_entry_t(st)) ? ( rrdset_last_slot(st) ) : \ + ( ((time_t)(t) <= rrdset_first_entry_t(st)) ? rrdset_first_slot(st) : \ + ( (rrdset_last_slot(st) >= (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) ) ? \ + (rrdset_last_slot(st) - (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) ) : \ + (rrdset_last_slot(st) - (unsigned long)((rrdset_last_entry_t(st) - (time_t)(t)) / (unsigned long)((st)->update_every)) + (unsigned long)(st)->entries ) \ + ))) // get the timestamp of a specific slot in the round robin database #define rrdset_slot2time(st, slot) ( rrdset_last_entry_t(st) - \ - ((unsigned long)(st)->update_every * ( \ - ( (unsigned long)(slot) > rrdset_last_slot(st)) ? \ - ( (rrdset_last_slot(st) - (unsigned long)(slot) + (unsigned long)(st)->entries) ) : \ - ( (rrdset_last_slot(st) - (unsigned long)(slot)) )) \ - )) + ((unsigned long)(st)->update_every * ( \ + ( (unsigned long)(slot) > rrdset_last_slot(st)) ? \ + ( (rrdset_last_slot(st) - (unsigned long)(slot) + (unsigned long)(st)->entries) ) : \ + ( (rrdset_last_slot(st) - (unsigned long)(slot)) )) \ + )) // ---------------------------------------------------------------------------- // RRD DIMENSION functions |