From 7877a98bd9c00db5e81dd2f8c734cba2bab20be7 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 12 Aug 2022 09:26:17 +0200 Subject: Merging upstream version 1.36.0. Signed-off-by: Daniel Baumann --- ml/Config.cc | 6 +++--- ml/Dimension.cc | 55 +++---------------------------------------------------- ml/Dimension.h | 6 +++--- ml/Query.h | 15 +++++++-------- ml/README.md | 53 +++++++++++++++++++++++------------------------------ ml/ml.cc | 14 +++++++------- 6 files changed, 46 insertions(+), 103 deletions(-) (limited to 'ml') diff --git a/ml/Config.cc b/ml/Config.cc index 37d0bb29e..65b05a34d 100644 --- a/ml/Config.cc +++ b/ml/Config.cc @@ -22,14 +22,14 @@ static T clamp(const T& Value, const T& Min, const T& Max) { void Config::readMLConfig(void) { const char *ConfigSectionML = CONFIG_SECTION_ML; - bool EnableAnomalyDetection = config_get_boolean(ConfigSectionML, "enabled", false); + bool EnableAnomalyDetection = config_get_boolean(ConfigSectionML, "enabled", true); /* * Read values */ unsigned MaxTrainSamples = config_get_number(ConfigSectionML, "maximum num samples to train", 4 * 3600); - unsigned MinTrainSamples = config_get_number(ConfigSectionML, "minimum num samples to train", 1 * 3600); + unsigned MinTrainSamples = config_get_number(ConfigSectionML, "minimum num samples to train", 1 * 900); unsigned TrainEvery = config_get_number(ConfigSectionML, "train every", 1 * 3600); unsigned DBEngineAnomalyRateEvery = config_get_number(ConfigSectionML, "dbengine anomaly rate every", 30); @@ -42,7 +42,7 @@ void Config::readMLConfig(void) { unsigned MaxKMeansIters = config_get_number(ConfigSectionML, "maximum number of k-means iterations", 1000); double DimensionAnomalyScoreThreshold = config_get_float(ConfigSectionML, "dimension anomaly score threshold", 0.99); - double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 0.02); + double HostAnomalyRateThreshold = config_get_float(ConfigSectionML, "host anomaly rate threshold", 0.01); double ADMinWindowSize = config_get_float(ConfigSectionML, "minimum window size", 30); double ADMaxWindowSize = config_get_float(ConfigSectionML, "maximum window size", 600); diff --git a/ml/Dimension.cc b/ml/Dimension.cc index 3146e45a6..0fe07530c 100644 --- a/ml/Dimension.cc +++ b/ml/Dimension.cc @@ -6,55 +6,6 @@ using namespace ml; -/* - * Copy of the unpack_storage_number which allows us to convert - * a storage_number to double. - */ -static CalculatedNumber unpack_storage_number_dbl(storage_number value) { - if(!value) - return 0; - - int sign = 0, exp = 0; - int factor = 10; - - // bit 32 = 0:positive, 1:negative - if(unlikely(value & (1 << 31))) - sign = 1; - - // bit 31 = 0:divide, 1:multiply - if(unlikely(value & (1 << 30))) - exp = 1; - - // bit 27 SN_EXISTS_100 - if(unlikely(value & (1 << 26))) - factor = 100; - - // bit 26 SN_EXISTS_RESET - // bit 25 SN_ANOMALY_BIT - - // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total) - int mul = (value & ((1<<29)|(1<<28)|(1<<27))) >> 27; - - // bit 24 to bit 1 = the value, so remove all other bits - value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24)); - - CalculatedNumber CN = value; - - if(exp) { - for(; mul; mul--) - CN *= factor; - } - else { - for( ; mul ; mul--) - CN /= 10; - } - - if(sign) - CN = -CN; - - return CN; -} - std::pair TrainableDimension::getCalculatedNumbers() { size_t MinN = Cfg.MinTrainSamples; @@ -89,10 +40,10 @@ TrainableDimension::getCalculatedNumbers() { break; auto P = Q.nextMetric(); - storage_number SN = P.second; + CalculatedNumber Value = P.second; - if (does_storage_number_exist(SN)) { - CNs[Idx] = unpack_storage_number_dbl(SN); + if (netdata_double_isnumber(Value)) { + CNs[Idx] = Value; LastValue = CNs[Idx]; CollectedValues++; } else diff --git a/ml/Dimension.h b/ml/Dimension.h index e4f8bd1c7..4fbc09b98 100644 --- a/ml/Dimension.h +++ b/ml/Dimension.h @@ -12,13 +12,13 @@ namespace ml { class RrdDimension { public: - RrdDimension(RRDDIM *RD) : RD(RD), Ops(&RD->state->query_ops) { } + RrdDimension(RRDDIM *RD) : RD(RD), Ops(&RD->tiers[0]->query_ops) { } RRDDIM *getRD() const { return RD; } - time_t latestTime() { return Ops->latest_time(RD); } + time_t latestTime() { return Ops->latest_time(RD->tiers[0]->db_metric_handle); } - time_t oldestTime() { return Ops->oldest_time(RD); } + time_t oldestTime() { return Ops->oldest_time(RD->tiers[0]->db_metric_handle); } unsigned updateEvery() const { return RD->update_every; } diff --git a/ml/Query.h b/ml/Query.h index 8b84bb73e..24c5fa384 100644 --- a/ml/Query.h +++ b/ml/Query.h @@ -8,29 +8,28 @@ namespace ml { class Query { public: Query(RRDDIM *RD) : RD(RD) { - Ops = &RD->state->query_ops; + Ops = &RD->tiers[0]->query_ops; } time_t latestTime() { - return Ops->latest_time(RD); + return Ops->latest_time(RD->tiers[0]->db_metric_handle); } time_t oldestTime() { - return Ops->oldest_time(RD); + return Ops->oldest_time(RD->tiers[0]->db_metric_handle); } void init(time_t AfterT, time_t BeforeT) { - Ops->init(RD, &Handle, AfterT, BeforeT); + Ops->init(RD->tiers[0]->db_metric_handle, &Handle, AfterT, BeforeT, TIER_QUERY_FETCH_SUM); } bool isFinished() { return Ops->is_finished(&Handle); } - std::pair nextMetric() { - time_t CurrT; - storage_number SN = Ops->next_metric(&Handle, &CurrT); - return { CurrT, SN }; + std::pair nextMetric() { + STORAGE_POINT sp = Ops->next_metric(&Handle); + return { sp.start_time, sp.sum / sp.count }; } ~Query() { diff --git a/ml/README.md b/ml/README.md index cb8384a66..2578993e2 100644 --- a/ml/README.md +++ b/ml/README.md @@ -1,5 +1,5 @@