summaryrefslogtreecommitdiffstats
path: root/ml/Config.cc
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2022-11-30 18:47:05 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2022-11-30 18:47:05 +0000
commit97e01009d69b8fbebfebf68f51e3d126d0ed43fc (patch)
tree02e8b836c3a9d89806f3e67d4a5fe9f52dbb0061 /ml/Config.cc
parentReleasing debian version 1.36.1-1. (diff)
downloadnetdata-97e01009d69b8fbebfebf68f51e3d126d0ed43fc.tar.xz
netdata-97e01009d69b8fbebfebf68f51e3d126d0ed43fc.zip
Merging upstream version 1.37.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'ml/Config.cc')
-rw-r--r--ml/Config.cc52
1 files changed, 14 insertions, 38 deletions
diff --git a/ml/Config.cc b/ml/Config.cc
index 65b05a34d..eedd8c29f 100644
--- a/ml/Config.cc
+++ b/ml/Config.cc
@@ -31,8 +31,7 @@ void Config::readMLConfig(void) {
unsigned MaxTrainSamples = config_get_number(ConfigSectionML, "maximum num samples to train", 4 * 3600);
unsigned MinTrainSamples = config_get_number(ConfigSectionML, "minimum num samples to train", 1 * 900);
unsigned TrainEvery = config_get_number(ConfigSectionML, "train every", 1 * 3600);
-
- unsigned DBEngineAnomalyRateEvery = config_get_number(ConfigSectionML, "dbengine anomaly rate every", 30);
+ unsigned NumModelsToUse = config_get_number(ConfigSectionML, "number of models per dimension", 1 * 24);
unsigned DiffN = config_get_number(ConfigSectionML, "num samples to diff", 1);
unsigned SmoothN = config_get_number(ConfigSectionML, "num samples to smooth", 3);
@@ -42,27 +41,19 @@ void Config::readMLConfig(void) {
unsigned MaxKMeansIters = config_get_number(ConfigSectionML, "maximum number of k-means iterations", 1000);
double DimensionAnomalyScoreThreshold = config_get_float(ConfigSectionML, "dimension anomaly score threshold", 0.99);
- double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 0.01);
-
- double ADMinWindowSize = config_get_float(ConfigSectionML, "minimum window size", 30);
- double ADMaxWindowSize = config_get_float(ConfigSectionML, "maximum window size", 600);
- double ADIdleWindowSize = config_get_float(ConfigSectionML, "idle window size", 30);
- double ADWindowRateThreshold = config_get_float(ConfigSectionML, "window minimum anomaly rate", 0.25);
- double ADDimensionRateThreshold = config_get_float(ConfigSectionML, "anomaly event min dimension rate threshold", 0.05);
- std::stringstream SS;
- SS << netdata_configured_cache_dir << "/anomaly-detection.db";
- Cfg.AnomalyDBPath = SS.str();
+ double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 1.0);
+ std::string AnomalyDetectionGroupingMethod = config_get(ConfigSectionML, "anomaly detection grouping method", "average");
+ time_t AnomalyDetectionQueryDuration = config_get_number(ConfigSectionML, "anomaly detection grouping duration", 5 * 60);
/*
* Clamp
*/
- MaxTrainSamples = clamp(MaxTrainSamples, 1 * 3600u, 24 * 3600u);
- MinTrainSamples = clamp(MinTrainSamples, 1 * 900u, 6 * 3600u);
- TrainEvery = clamp(TrainEvery, 1 * 3600u, 6 * 3600u);
-
- DBEngineAnomalyRateEvery = clamp(DBEngineAnomalyRateEvery, 1 * 30u, 15 * 60u);
+ MaxTrainSamples = clamp<unsigned>(MaxTrainSamples, 1 * 3600, 24 * 3600);
+ MinTrainSamples = clamp<unsigned>(MinTrainSamples, 1 * 900, 6 * 3600);
+ TrainEvery = clamp<unsigned>(TrainEvery, 1 * 3600, 6 * 3600);
+ NumModelsToUse = clamp<unsigned>(TrainEvery, 1, 7 * 24);
DiffN = clamp(DiffN, 0u, 1u);
SmoothN = clamp(SmoothN, 0u, 5u);
@@ -72,13 +63,9 @@ void Config::readMLConfig(void) {
MaxKMeansIters = clamp(MaxKMeansIters, 500u, 1000u);
DimensionAnomalyScoreThreshold = clamp(DimensionAnomalyScoreThreshold, 0.01, 5.00);
- HostAnomalyRateThreshold = clamp(HostAnomalyRateThreshold, 0.01, 1.0);
- ADMinWindowSize = clamp(ADMinWindowSize, 30.0, 300.0);
- ADMaxWindowSize = clamp(ADMaxWindowSize, 60.0, 900.0);
- ADIdleWindowSize = clamp(ADIdleWindowSize, 30.0, 900.0);
- ADWindowRateThreshold = clamp(ADWindowRateThreshold, 0.01, 0.99);
- ADDimensionRateThreshold = clamp(ADDimensionRateThreshold, 0.01, 0.99);
+ HostAnomalyRateThreshold = clamp(HostAnomalyRateThreshold, 0.1, 10.0);
+ AnomalyDetectionQueryDuration = clamp<time_t>(AnomalyDetectionQueryDuration, 60, 15 * 60);
/*
* Validate
@@ -91,13 +78,6 @@ void Config::readMLConfig(void) {
MaxTrainSamples = 4 * 3600;
}
- if (ADMinWindowSize >= ADMaxWindowSize) {
- error("invalid min/max anomaly window size found (%lf >= %lf)", ADMinWindowSize, ADMaxWindowSize);
-
- ADMinWindowSize = 30.0;
- ADMaxWindowSize = 600.0;
- }
-
/*
* Assign to config instance
*/
@@ -107,8 +87,7 @@ void Config::readMLConfig(void) {
Cfg.MaxTrainSamples = MaxTrainSamples;
Cfg.MinTrainSamples = MinTrainSamples;
Cfg.TrainEvery = TrainEvery;
-
- Cfg.DBEngineAnomalyRateEvery = DBEngineAnomalyRateEvery;
+ Cfg.NumModelsToUse = NumModelsToUse;
Cfg.DiffN = DiffN;
Cfg.SmoothN = SmoothN;
@@ -118,13 +97,10 @@ void Config::readMLConfig(void) {
Cfg.MaxKMeansIters = MaxKMeansIters;
Cfg.DimensionAnomalyScoreThreshold = DimensionAnomalyScoreThreshold;
- Cfg.HostAnomalyRateThreshold = HostAnomalyRateThreshold;
- Cfg.ADMinWindowSize = ADMinWindowSize;
- Cfg.ADMaxWindowSize = ADMaxWindowSize;
- Cfg.ADIdleWindowSize = ADIdleWindowSize;
- Cfg.ADWindowRateThreshold = ADWindowRateThreshold;
- Cfg.ADDimensionRateThreshold = ADDimensionRateThreshold;
+ Cfg.HostAnomalyRateThreshold = HostAnomalyRateThreshold;
+ Cfg.AnomalyDetectionGroupingMethod = web_client_api_request_v1_data_group(AnomalyDetectionGroupingMethod.c_str(), RRDR_GROUPING_AVERAGE);
+ Cfg.AnomalyDetectionQueryDuration = AnomalyDetectionQueryDuration;
Cfg.HostsToSkip = config_get(ConfigSectionML, "hosts to skip from training", "!*");
Cfg.SP_HostsToSkip = simple_pattern_create(Cfg.HostsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);