summaryrefslogtreecommitdiffstats
path: root/web/api/queries/des
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-06 01:22:31 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-06 01:22:31 +0000
commit8d4f58e49b9dc7d3545651023a36729de773ad86 (patch)
tree7bc7be4a8e9e298daa1349348400aa2a653866f2 /web/api/queries/des
parentInitial commit. (diff)
downloadnetdata-8d4f58e49b9dc7d3545651023a36729de773ad86.tar.xz
netdata-8d4f58e49b9dc7d3545651023a36729de773ad86.zip
Adding upstream version 1.12.0.upstream/1.12.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--web/api/queries/des/Makefile.am8
-rw-r--r--web/api/queries/des/README.md68
-rw-r--r--web/api/queries/des/des.c139
-rw-r--r--web/api/queries/des/des.h17
4 files changed, 232 insertions, 0 deletions
diff --git a/web/api/queries/des/Makefile.am b/web/api/queries/des/Makefile.am
new file mode 100644
index 0000000..19554be
--- /dev/null
+++ b/web/api/queries/des/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/web/api/queries/des/README.md b/web/api/queries/des/README.md
new file mode 100644
index 0000000..486221c
--- /dev/null
+++ b/web/api/queries/des/README.md
@@ -0,0 +1,68 @@
+# double exponential smoothing
+
+Exponential smoothing is one of many window functions commonly applied to smooth data in signal
+processing, acting as low-pass filters to remove high frequency noise.
+
+Simple exponential smoothing does not do well when there is a trend in the data.
+In such situations, several methods were devised under the name "double exponential smoothing"
+or "second-order exponential smoothing.", which is the recursive application of an exponential
+filter twice, thus being termed "double exponential smoothing".
+
+In simple terms, this is like an average value, but more recent values are given more weight
+and the trend of the values influences significantly the result.
+
+> **IMPORTANT**
+>
+> It is common for `des` to provide "average" values that far beyond the minimum or the maximum
+> values found in the time-series.
+> `des` estimates these values because of it takes into account the trend.
+
+This module implements the "Holt-Winters double exponential smoothing".
+
+Netdata automatically adjusts the weight (`alpha`) and the trend (`beta`) based on the number
+of values processed, using the formula:
+
+```
+window = max(number of values, 15)
+alpha = 2 / (window + 1)
+beta = 2 / (window + 1)
+```
+
+You can change the fixed value `15` by setting in `netdata.conf`:
+
+```
+[web]
+ des max window = 15
+```
+
+## how to use
+
+Use it in alarms like this:
+
+```
+ alarm: my_alarm
+ on: my_chart
+lookup: des -1m unaligned of my_dimension
+ warn: $this > 1000
+```
+
+`des` does not change the units. For example, if the chart units is `requests/sec`, the result
+will be again expressed in the same units.
+
+It can also be used in APIs and badges as `&group=des` in the URL.
+
+## Examples
+
+Examining last 1 minute `successful` web server responses:
+
+- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=min&after=-60&label=min)
+- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=average&after=-60&label=average&value_color=yellow)
+- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=ses&after=-60&label=single+exponential+smoothing&value_color=yellow)
+- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=des&after=-60&label=double+exponential+smoothing&value_color=orange)
+- ![](https://registry.my-netdata.io/api/v1/badge.svg?chart=web_log_nginx.response_statuses&options=unaligned&dimensions=success&group=max&after=-60&label=max)
+
+## References
+
+- [https://en.wikipedia.org/wiki/Exponential_smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing).
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fweb%2Fapi%2Fqueries%2Fdes%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)]()
diff --git a/web/api/queries/des/des.c b/web/api/queries/des/des.c
new file mode 100644
index 0000000..93d7247
--- /dev/null
+++ b/web/api/queries/des/des.c
@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <web/api/queries/rrdr.h>
+#include "des.h"
+
+
+// ----------------------------------------------------------------------------
+// single exponential smoothing
+
+struct grouping_des {
+ calculated_number alpha;
+ calculated_number alpha_other;
+ calculated_number beta;
+ calculated_number beta_other;
+
+ calculated_number level;
+ calculated_number trend;
+
+ size_t count;
+};
+
+static size_t max_window_size = 15;
+
+void grouping_init_des(void) {
+ long long ret = config_get_number(CONFIG_SECTION_WEB, "des max window", (long long)max_window_size);
+ if(ret <= 1) {
+ config_set_number(CONFIG_SECTION_WEB, "des max window", (long long)max_window_size);
+ }
+ else {
+ max_window_size = (size_t) ret;
+ }
+}
+
+static inline calculated_number window(RRDR *r, struct grouping_des *g) {
+ (void)g;
+
+ calculated_number points;
+ if(r->group == 1) {
+ // provide a running DES
+ points = r->internal.points_wanted;
+ }
+ else {
+ // provide a SES with flush points
+ points = r->group;
+ }
+
+ // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
+ // A commonly used value for alpha is 2 / (N + 1)
+ return (points > max_window_size) ? max_window_size : points;
+}
+
+static inline void set_alpha(RRDR *r, struct grouping_des *g) {
+ // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
+ // A commonly used value for alpha is 2 / (N + 1)
+
+ g->alpha = 2.0 / (window(r, g) + 1.0);
+ g->alpha_other = 1.0 - g->alpha;
+
+ //info("alpha for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->alpha);
+}
+
+static inline void set_beta(RRDR *r, struct grouping_des *g) {
+ // https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
+ // A commonly used value for alpha is 2 / (N + 1)
+
+ g->beta = 2.0 / (window(r, g) + 1.0);
+ g->beta_other = 1.0 - g->beta;
+
+ //info("beta for chart '%s' is " CALCULATED_NUMBER_FORMAT, r->st->name, g->beta);
+}
+
+void *grouping_create_des(RRDR *r) {
+ struct grouping_des *g = (struct grouping_des *)malloc(sizeof(struct grouping_des));
+ set_alpha(r, g);
+ set_beta(r, g);
+ g->level = 0.0;
+ g->trend = 0.0;
+ g->count = 0;
+ return g;
+}
+
+// resets when switches dimensions
+// so, clear everything to restart
+void grouping_reset_des(RRDR *r) {
+ struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
+ g->level = 0.0;
+ g->trend = 0.0;
+ g->count = 0;
+
+ // fprintf(stderr, "\nDES: ");
+
+}
+
+void grouping_free_des(RRDR *r) {
+ freez(r->internal.grouping_data);
+ r->internal.grouping_data = NULL;
+}
+
+void grouping_add_des(RRDR *r, calculated_number value) {
+ struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
+
+ if(isnormal(value)) {
+ if(likely(g->count > 0)) {
+ // we have at least a number so far
+
+ if(unlikely(g->count == 1)) {
+ // the second value we got
+ g->trend = value - g->trend;
+ g->level = value;
+ }
+
+ // for the values, except the first
+ calculated_number last_level = g->level;
+ g->level = (g->alpha * value) + (g->alpha_other * (g->level + g->trend));
+ g->trend = (g->beta * (g->level - last_level)) + (g->beta_other * g->trend);
+ }
+ else {
+ // the first value we got
+ g->level = g->trend = value;
+ }
+
+ g->count++;
+ }
+
+ //fprintf(stderr, "value: " CALCULATED_NUMBER_FORMAT ", level: " CALCULATED_NUMBER_FORMAT ", trend: " CALCULATED_NUMBER_FORMAT "\n", value, g->level, g->trend);
+}
+
+calculated_number grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr) {
+ struct grouping_des *g = (struct grouping_des *)r->internal.grouping_data;
+
+ if(unlikely(!g->count || !isnormal(g->level))) {
+ *rrdr_value_options_ptr |= RRDR_VALUE_EMPTY;
+ return 0.0;
+ }
+
+ //fprintf(stderr, " RESULT for %zu values = " CALCULATED_NUMBER_FORMAT " \n", g->count, g->level);
+
+ return g->level;
+}
diff --git a/web/api/queries/des/des.h b/web/api/queries/des/des.h
new file mode 100644
index 0000000..360513e
--- /dev/null
+++ b/web/api/queries/des/des.h
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_API_QUERIES_DES_H
+#define NETDATA_API_QUERIES_DES_H
+
+#include "../query.h"
+#include "../rrdr.h"
+
+extern void grouping_init_des(void);
+
+extern void *grouping_create_des(RRDR *r);
+extern void grouping_reset_des(RRDR *r);
+extern void grouping_free_des(RRDR *r);
+extern void grouping_add_des(RRDR *r, calculated_number value);
+extern calculated_number grouping_flush_des(RRDR *r, RRDR_VALUE_FLAGS *rrdr_value_options_ptr);
+
+#endif //NETDATA_API_QUERIES_DES_H