1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
// SPDX-License-Identifier: GPL-3.0-or-later
#include "Config.h"
#include "ml-private.h"
using namespace ml;
/*
* Global configuration instance to be shared between training and
* prediction threads.
*/
Config ml::Cfg;
template <typename T>
static T clamp(const T& Value, const T& Min, const T& Max) {
return std::max(Min, std::min(Value, Max));
}
/*
* Initialize global configuration variable.
*/
void Config::readMLConfig(void) {
const char *ConfigSectionML = CONFIG_SECTION_ML;
bool EnableAnomalyDetection = config_get_boolean(ConfigSectionML, "enabled", true);
/*
* Read values
*/
unsigned MaxTrainSamples = config_get_number(ConfigSectionML, "maximum num samples to train", 4 * 3600);
unsigned MinTrainSamples = config_get_number(ConfigSectionML, "minimum num samples to train", 1 * 900);
unsigned TrainEvery = config_get_number(ConfigSectionML, "train every", 1 * 3600);
unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1);
unsigned DiffN = config_get_number(ConfigSectionML, "num samples to diff", 1);
unsigned SmoothN = config_get_number(ConfigSectionML, "num samples to smooth", 3);
unsigned LagN = config_get_number(ConfigSectionML, "num samples to lag", 5);
double RandomSamplingRatio = config_get_float(ConfigSectionML, "random sampling ratio", 1.0 / LagN);
unsigned MaxKMeansIters = config_get_number(ConfigSectionML, "maximum number of k-means iterations", 1000);
double DimensionAnomalyScoreThreshold = config_get_float(ConfigSectionML, "dimension anomaly score threshold", 0.99);
double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 1.0);
std::string AnomalyDetectionGroupingMethod = config_get(ConfigSectionML, "anomaly detection grouping method", "average");
time_t AnomalyDetectionQueryDuration = config_get_number(ConfigSectionML, "anomaly detection grouping duration", 5 * 60);
/*
* Clamp
*/
MaxTrainSamples = clamp<unsigned>(MaxTrainSamples, 1 * 3600, 24 * 3600);
MinTrainSamples = clamp<unsigned>(MinTrainSamples, 1 * 900, 6 * 3600);
TrainEvery = clamp<unsigned>(TrainEvery, 1 * 3600, 6 * 3600);
NumModelsToUse = clamp<unsigned>(NumModelsToUse, 1, 7 * 24);
DiffN = clamp(DiffN, 0u, 1u);
SmoothN = clamp(SmoothN, 0u, 5u);
LagN = clamp(LagN, 1u, 5u);
RandomSamplingRatio = clamp(RandomSamplingRatio, 0.2, 1.0);
MaxKMeansIters = clamp(MaxKMeansIters, 500u, 1000u);
DimensionAnomalyScoreThreshold = clamp(DimensionAnomalyScoreThreshold, 0.01, 5.00);
HostAnomalyRateThreshold = clamp(HostAnomalyRateThreshold, 0.1, 10.0);
AnomalyDetectionQueryDuration = clamp<time_t>(AnomalyDetectionQueryDuration, 60, 15 * 60);
/*
* Validate
*/
if (MinTrainSamples >= MaxTrainSamples) {
error("invalid min/max train samples found (%u >= %u)", MinTrainSamples, MaxTrainSamples);
MinTrainSamples = 1 * 3600;
MaxTrainSamples = 4 * 3600;
}
/*
* Assign to config instance
*/
Cfg.EnableAnomalyDetection = EnableAnomalyDetection;
Cfg.MaxTrainSamples = MaxTrainSamples;
Cfg.MinTrainSamples = MinTrainSamples;
Cfg.TrainEvery = TrainEvery;
Cfg.NumModelsToUse = NumModelsToUse;
Cfg.DiffN = DiffN;
Cfg.SmoothN = SmoothN;
Cfg.LagN = LagN;
Cfg.RandomSamplingRatio = RandomSamplingRatio;
Cfg.MaxKMeansIters = MaxKMeansIters;
Cfg.DimensionAnomalyScoreThreshold = DimensionAnomalyScoreThreshold;
Cfg.HostAnomalyRateThreshold = HostAnomalyRateThreshold;
Cfg.AnomalyDetectionGroupingMethod = web_client_api_request_v1_data_group(AnomalyDetectionGroupingMethod.c_str(), RRDR_GROUPING_AVERAGE);
Cfg.AnomalyDetectionQueryDuration = AnomalyDetectionQueryDuration;
Cfg.HostsToSkip = config_get(ConfigSectionML, "hosts to skip from training", "!*");
Cfg.SP_HostsToSkip = simple_pattern_create(Cfg.HostsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
// Always exclude anomaly_detection charts from training.
Cfg.ChartsToSkip = "anomaly_detection.* ";
Cfg.ChartsToSkip += config_get(ConfigSectionML, "charts to skip from training", "netdata.*");
Cfg.SP_ChartsToSkip = simple_pattern_create(Cfg.ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
Cfg.StreamADCharts = config_get_boolean(ConfigSectionML, "stream anomaly detection charts", true);
}
|
