summaryrefslogtreecommitdiffstats
path: root/ml/ml-private.h
diff options
context:
space:
mode:
Diffstat (limited to 'ml/ml-private.h')
-rw-r--r--ml/ml-private.h336
1 files changed, 329 insertions, 7 deletions
diff --git a/ml/ml-private.h b/ml/ml-private.h
index e479f2351..327cc59a2 100644
--- a/ml/ml-private.h
+++ b/ml/ml-private.h
@@ -1,13 +1,335 @@
// SPDX-License-Identifier: GPL-3.0-or-later
-#ifndef ML_PRIVATE_H
-#define ML_PRIVATE_H
+#ifndef NETDATA_ML_PRIVATE_H
+#define NETDATA_ML_PRIVATE_H
-#include "KMeans.h"
+#include "dlib/matrix.h"
#include "ml/ml.h"
-#include <map>
-#include <mutex>
-#include <sstream>
+#include <vector>
+#include <queue>
-#endif /* ML_PRIVATE_H */
+typedef double calculated_number_t;
+typedef dlib::matrix<calculated_number_t, 6, 1> DSample;
+
+/*
+ * Features
+ */
+
+typedef struct {
+ size_t diff_n;
+ size_t smooth_n;
+ size_t lag_n;
+
+ calculated_number_t *dst;
+ size_t dst_n;
+
+ calculated_number_t *src;
+ size_t src_n;
+
+ std::vector<DSample> &preprocessed_features;
+} ml_features_t;
+
+/*
+ * KMeans
+ */
+
+typedef struct {
+ std::vector<DSample> cluster_centers;
+
+ calculated_number_t min_dist;
+ calculated_number_t max_dist;
+
+ uint32_t after;
+ uint32_t before;
+} ml_kmeans_t;
+
+typedef struct machine_learning_stats_t {
+ size_t num_machine_learning_status_enabled;
+ size_t num_machine_learning_status_disabled_sp;
+
+ size_t num_metric_type_constant;
+ size_t num_metric_type_variable;
+
+ size_t num_training_status_untrained;
+ size_t num_training_status_pending_without_model;
+ size_t num_training_status_trained;
+ size_t num_training_status_pending_with_model;
+
+ size_t num_anomalous_dimensions;
+ size_t num_normal_dimensions;
+} ml_machine_learning_stats_t;
+
+typedef struct training_stats_t {
+ size_t queue_size;
+ size_t num_popped_items;
+
+ usec_t allotted_ut;
+ usec_t consumed_ut;
+ usec_t remaining_ut;
+
+ size_t training_result_ok;
+ size_t training_result_invalid_query_time_range;
+ size_t training_result_not_enough_collected_values;
+ size_t training_result_null_acquired_dimension;
+ size_t training_result_chart_under_replication;
+} ml_training_stats_t;
+
+enum ml_metric_type {
+ // The dimension has constant values, no need to train
+ METRIC_TYPE_CONSTANT,
+
+ // The dimension's values fluctuate, we need to generate a model
+ METRIC_TYPE_VARIABLE,
+};
+
+enum ml_machine_learning_status {
+ // Enable training/prediction
+ MACHINE_LEARNING_STATUS_ENABLED,
+
+ // Disable because configuration pattern matches the chart's id
+ MACHINE_LEARNING_STATUS_DISABLED_DUE_TO_EXCLUDED_CHART,
+};
+
+enum ml_training_status {
+ // We don't have a model for this dimension
+ TRAINING_STATUS_UNTRAINED,
+
+ // Request for training sent, but we don't have any models yet
+ TRAINING_STATUS_PENDING_WITHOUT_MODEL,
+
+ // Request to update existing models sent
+ TRAINING_STATUS_PENDING_WITH_MODEL,
+
+ // Have a valid, up-to-date model
+ TRAINING_STATUS_TRAINED,
+};
+
+enum ml_training_result {
+ // We managed to create a KMeans model
+ TRAINING_RESULT_OK,
+
+ // Could not query DB with a correct time range
+ TRAINING_RESULT_INVALID_QUERY_TIME_RANGE,
+
+ // Did not gather enough data from DB to run KMeans
+ TRAINING_RESULT_NOT_ENOUGH_COLLECTED_VALUES,
+
+ // Acquired a null dimension
+ TRAINING_RESULT_NULL_ACQUIRED_DIMENSION,
+
+ // Chart is under replication
+ TRAINING_RESULT_CHART_UNDER_REPLICATION,
+};
+
+typedef struct {
+ // Chart/dimension we want to train
+ char machine_guid[GUID_LEN + 1];
+ STRING *chart_id;
+ STRING *dimension_id;
+
+ // Creation time of request
+ time_t request_time;
+
+ // First/last entry of this dimension in DB
+ // at the point the request was made
+ time_t first_entry_on_request;
+ time_t last_entry_on_request;
+} ml_training_request_t;
+
+typedef struct {
+ // Time when the request for this response was made
+ time_t request_time;
+
+ // First/last entry of the dimension in DB when generating the request
+ time_t first_entry_on_request;
+ time_t last_entry_on_request;
+
+ // First/last entry of the dimension in DB when generating the response
+ time_t first_entry_on_response;
+ time_t last_entry_on_response;
+
+ // After/Before timestamps of our DB query
+ time_t query_after_t;
+ time_t query_before_t;
+
+ // Actual after/before returned by the DB query ops
+ time_t db_after_t;
+ time_t db_before_t;
+
+ // Number of doubles returned by the DB query
+ size_t collected_values;
+
+ // Number of values we return to the caller
+ size_t total_values;
+
+ // Result of training response
+ enum ml_training_result result;
+} ml_training_response_t;
+
+/*
+ * Queue
+*/
+
+typedef struct {
+ std::queue<ml_training_request_t> internal;
+ netdata_mutex_t mutex;
+ pthread_cond_t cond_var;
+ std::atomic<bool> exit;
+} ml_queue_t;
+
+typedef struct {
+ RRDDIM *rd;
+
+ enum ml_metric_type mt;
+ enum ml_training_status ts;
+ enum ml_machine_learning_status mls;
+
+ ml_training_response_t tr;
+ time_t last_training_time;
+
+ std::vector<calculated_number_t> cns;
+
+ std::vector<ml_kmeans_t> km_contexts;
+ netdata_mutex_t mutex;
+ ml_kmeans_t kmeans;
+ std::vector<DSample> feature;
+} ml_dimension_t;
+
+typedef struct {
+ RRDSET *rs;
+ ml_machine_learning_stats_t mls;
+
+ netdata_mutex_t mutex;
+} ml_chart_t;
+
+void ml_chart_update_dimension(ml_chart_t *chart, ml_dimension_t *dim, bool is_anomalous);
+
+typedef struct {
+ RRDHOST *rh;
+
+ ml_machine_learning_stats_t mls;
+
+ calculated_number_t host_anomaly_rate;
+
+ netdata_mutex_t mutex;
+
+ ml_queue_t *training_queue;
+
+ /*
+ * bookkeeping for anomaly detection charts
+ */
+
+ RRDSET *machine_learning_status_rs;
+ RRDDIM *machine_learning_status_enabled_rd;
+ RRDDIM *machine_learning_status_disabled_sp_rd;
+
+ RRDSET *metric_type_rs;
+ RRDDIM *metric_type_constant_rd;
+ RRDDIM *metric_type_variable_rd;
+
+ RRDSET *training_status_rs;
+ RRDDIM *training_status_untrained_rd;
+ RRDDIM *training_status_pending_without_model_rd;
+ RRDDIM *training_status_trained_rd;
+ RRDDIM *training_status_pending_with_model_rd;
+
+ RRDSET *dimensions_rs;
+ RRDDIM *dimensions_anomalous_rd;
+ RRDDIM *dimensions_normal_rd;
+
+ RRDSET *anomaly_rate_rs;
+ RRDDIM *anomaly_rate_rd;
+
+ RRDSET *detector_events_rs;
+ RRDDIM *detector_events_above_threshold_rd;
+ RRDDIM *detector_events_new_anomaly_event_rd;
+} ml_host_t;
+
+typedef struct {
+ uuid_t metric_uuid;
+ ml_kmeans_t kmeans;
+} ml_model_info_t;
+
+typedef struct {
+ size_t id;
+ netdata_thread_t nd_thread;
+ netdata_mutex_t nd_mutex;
+
+ ml_queue_t *training_queue;
+ ml_training_stats_t training_stats;
+
+ calculated_number_t *training_cns;
+ calculated_number_t *scratch_training_cns;
+ std::vector<DSample> training_samples;
+
+ std::vector<ml_model_info_t> pending_model_info;
+
+ RRDSET *queue_stats_rs;
+ RRDDIM *queue_stats_queue_size_rd;
+ RRDDIM *queue_stats_popped_items_rd;
+
+ RRDSET *training_time_stats_rs;
+ RRDDIM *training_time_stats_allotted_rd;
+ RRDDIM *training_time_stats_consumed_rd;
+ RRDDIM *training_time_stats_remaining_rd;
+
+ RRDSET *training_results_rs;
+ RRDDIM *training_results_ok_rd;
+ RRDDIM *training_results_invalid_query_time_range_rd;
+ RRDDIM *training_results_not_enough_collected_values_rd;
+ RRDDIM *training_results_null_acquired_dimension_rd;
+ RRDDIM *training_results_chart_under_replication_rd;
+} ml_training_thread_t;
+
+typedef struct {
+ bool enable_anomaly_detection;
+
+ unsigned max_train_samples;
+ unsigned min_train_samples;
+ unsigned train_every;
+
+ unsigned num_models_to_use;
+
+ unsigned db_engine_anomaly_rate_every;
+
+ unsigned diff_n;
+ unsigned smooth_n;
+ unsigned lag_n;
+
+ double random_sampling_ratio;
+ unsigned max_kmeans_iters;
+
+ double dimension_anomaly_score_threshold;
+
+ double host_anomaly_rate_threshold;
+ RRDR_TIME_GROUPING anomaly_detection_grouping_method;
+ time_t anomaly_detection_query_duration;
+
+ bool stream_anomaly_detection_charts;
+
+ std::string hosts_to_skip;
+ SIMPLE_PATTERN *sp_host_to_skip;
+
+ std::string charts_to_skip;
+ SIMPLE_PATTERN *sp_charts_to_skip;
+
+ std::vector<uint32_t> random_nums;
+
+ netdata_thread_t detection_thread;
+ std::atomic<bool> detection_stop;
+
+ size_t num_training_threads;
+ size_t flush_models_batch_size;
+
+ std::vector<ml_training_thread_t> training_threads;
+ std::atomic<bool> training_stop;
+
+ bool enable_statistics_charts;
+} ml_config_t;
+
+void ml_config_load(ml_config_t *cfg);
+
+extern ml_config_t Cfg;
+
+#endif /* NETDATA_ML_PRIVATE_H */