summaryrefslogtreecommitdiffstats
path: root/collectors/profile.plugin
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2023-07-20 04:50:01 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2023-07-20 04:50:01 +0000
commitcd4377fab21e0f500bef7f06543fa848a039c1e0 (patch)
treeba00a55e430c052d6bed0b61c0f8bbe8ebedd313 /collectors/profile.plugin
parentReleasing debian version 1.40.1-1. (diff)
downloadnetdata-cd4377fab21e0f500bef7f06543fa848a039c1e0.tar.xz
netdata-cd4377fab21e0f500bef7f06543fa848a039c1e0.zip
Merging upstream version 1.41.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'collectors/profile.plugin')
-rw-r--r--collectors/profile.plugin/Makefile.am8
-rw-r--r--collectors/profile.plugin/README.md34
-rw-r--r--collectors/profile.plugin/plugin_profile.cc228
3 files changed, 270 insertions, 0 deletions
diff --git a/collectors/profile.plugin/Makefile.am b/collectors/profile.plugin/Makefile.am
new file mode 100644
index 000000000..161784b8f
--- /dev/null
+++ b/collectors/profile.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+ README.md \
+ $(NULL)
diff --git a/collectors/profile.plugin/README.md b/collectors/profile.plugin/README.md
new file mode 100644
index 000000000..1f200fc31
--- /dev/null
+++ b/collectors/profile.plugin/README.md
@@ -0,0 +1,34 @@
+# profile.plugin
+
+This plugin allows someone to backfill an agent with random data.
+
+A user can specify:
+
+ - The number charts they want,
+ - the number of dimensions per chart,
+ - the desire update every collection frequency,
+ - the number of seconds to backfill.
+ - the number of collection threads.
+
+## Configuration
+
+Edit the `netdata.conf` configuration file using [`edit-config`](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md#use-edit-config-to-edit-configuration-files) from the [Netdata config directory](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md#the-netdata-config-directory), which is typically at `/etc/netdata`.
+
+Scroll down to the `[plugin:profile]` section to find the available options:
+
+```
+[plugin:profile]
+ update every = 5
+ number of charts = 200
+ number of dimensions per chart = 5
+ seconds to backfill = 86400
+ number of threads = 16
+```
+
+The `number of threads` option will create the specified number of collection
+threads. The rest of the options apply to each thread individually, eg. the
+above configuration will create 3200 charts, 16000 dimensions in total, which will be
+backfilled for the duration of 1 day.
+
+Note that all but the 1st chart created in each thread will be marked as hidden
+in order to ease the load on the dashboard's UI.
diff --git a/collectors/profile.plugin/plugin_profile.cc b/collectors/profile.plugin/plugin_profile.cc
new file mode 100644
index 000000000..5f7b22d25
--- /dev/null
+++ b/collectors/profile.plugin/plugin_profile.cc
@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "daemon/common.h"
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <random>
+#include <thread>
+#include <vector>
+
+#define PLUGIN_PROFILE_NAME "profile.plugin"
+
+#define CONFIG_SECTION_PROFILE "plugin:profile"
+
+class Generator {
+public:
+ Generator(size_t N) : Offset(0) {
+ std::random_device RandDev;
+ std::mt19937 Gen(RandDev());
+ std::uniform_int_distribution<int> D(-16, 16);
+
+ V.reserve(N);
+ for (size_t Idx = 0; Idx != N; Idx++)
+ V.push_back(D(Gen));
+ }
+
+ double getRandValue() {
+ return V[Offset++ % V.size()];
+ }
+
+private:
+ size_t Offset;
+ std::vector<double> V;
+};
+
+class Profiler {
+public:
+ Profiler(size_t ID, size_t NumCharts, size_t NumDimsPerChart, time_t SecondsToBackfill, int UpdateEvery) :
+ ID(ID),
+ NumCharts(NumCharts),
+ NumDimsPerChart(NumDimsPerChart),
+ SecondsToBackfill(SecondsToBackfill),
+ UpdateEvery(UpdateEvery),
+ Gen(1024 * 1024)
+ {}
+
+ void create() {
+ char ChartId[1024];
+ char DimId[1024];
+
+ Charts.reserve(NumCharts);
+ for (size_t I = 0; I != NumCharts; I++) {
+ size_t CID = ID + Charts.size() + 1;
+
+ snprintfz(ChartId, 1024 - 1, "chart_%zu", CID);
+
+ RRDSET *RS = rrdset_create_localhost(
+ "profile", // type
+ ChartId, // id
+ nullptr, // name,
+ "profile_family", // family
+ "profile_context", // context
+ "profile_title", // title
+ "profile_units", // units
+ "profile_plugin", // plugin
+ "profile_module", // module
+ 12345678 + CID, // priority
+ UpdateEvery, // update_every
+ RRDSET_TYPE_LINE // chart_type
+ );
+ if (I != 0)
+ rrdset_flag_set(RS, RRDSET_FLAG_HIDDEN);
+ Charts.push_back(RS);
+
+ Dimensions.reserve(NumDimsPerChart);
+ for (size_t J = 0; J != NumDimsPerChart; J++) {
+ snprintfz(DimId, 1024 - 1, "dim_%zu", J);
+
+ RRDDIM *RD = rrddim_add(
+ RS, // st
+ DimId, // id
+ nullptr, // name
+ 1, // multiplier
+ 1, // divisor
+ RRD_ALGORITHM_ABSOLUTE // algorithm
+ );
+
+ Dimensions.push_back(RD);
+ }
+ }
+ }
+
+ void update(const struct timeval &Now) {
+ for (RRDSET *RS: Charts) {
+ for (RRDDIM *RD : Dimensions) {
+ rrddim_timed_set_by_pointer(RS, RD, Now, Gen.getRandValue());
+ }
+
+ rrdset_timed_done(RS, Now, RS->counter_done != 0);
+ }
+ }
+
+ void run() {
+ #define WORKER_JOB_CREATE_CHARTS 0
+ #define WORKER_JOB_UPDATE_CHARTS 1
+ #define WORKER_JOB_METRIC_DURATION_TO_BACKFILL 2
+ #define WORKER_JOB_METRIC_POINTS_BACKFILLED 3
+
+ worker_register("PROFILER");
+ worker_register_job_name(WORKER_JOB_CREATE_CHARTS, "create charts");
+ worker_register_job_name(WORKER_JOB_UPDATE_CHARTS, "update charts");
+ worker_register_job_custom_metric(WORKER_JOB_METRIC_DURATION_TO_BACKFILL, "duration to backfill", "seconds", WORKER_METRIC_ABSOLUTE);
+ worker_register_job_custom_metric(WORKER_JOB_METRIC_POINTS_BACKFILLED, "points backfilled", "points", WORKER_METRIC_ABSOLUTE);
+
+ heartbeat_t HB;
+ heartbeat_init(&HB);
+
+ worker_is_busy(WORKER_JOB_CREATE_CHARTS);
+ create();
+
+ struct timeval CollectionTV;
+ now_realtime_timeval(&CollectionTV);
+
+ if (SecondsToBackfill) {
+ CollectionTV.tv_sec -= SecondsToBackfill;
+ CollectionTV.tv_sec -= (CollectionTV.tv_sec % UpdateEvery);
+
+ CollectionTV.tv_usec = 0;
+ }
+
+ size_t BackfilledPoints = 0;
+ struct timeval NowTV, PrevTV;
+ now_realtime_timeval(&NowTV);
+ PrevTV = NowTV;
+
+ while (service_running(SERVICE_COLLECTORS)) {
+ worker_is_busy(WORKER_JOB_UPDATE_CHARTS);
+
+ update(CollectionTV);
+ CollectionTV.tv_sec += UpdateEvery;
+
+ now_realtime_timeval(&NowTV);
+
+ ++BackfilledPoints;
+ if (NowTV.tv_sec > PrevTV.tv_sec) {
+ PrevTV = NowTV;
+ worker_set_metric(WORKER_JOB_METRIC_POINTS_BACKFILLED, BackfilledPoints * NumCharts * NumDimsPerChart);
+ BackfilledPoints = 0;
+ }
+
+ size_t RemainingSeconds = (CollectionTV.tv_sec >= NowTV.tv_sec) ? 0 : (NowTV.tv_sec - CollectionTV.tv_sec);
+ worker_set_metric(WORKER_JOB_METRIC_DURATION_TO_BACKFILL, RemainingSeconds);
+
+ if (CollectionTV.tv_sec >= NowTV.tv_sec) {
+ worker_is_idle();
+ heartbeat_next(&HB, UpdateEvery * USEC_PER_SEC);
+ }
+ }
+ }
+
+private:
+ size_t ID;
+ size_t NumCharts;
+ size_t NumDimsPerChart;
+ size_t SecondsToBackfill;
+ int UpdateEvery;
+
+ Generator Gen;
+ std::vector<RRDSET *> Charts;
+ std::vector<RRDDIM *> Dimensions;
+};
+
+static void *subprofile_main(void* Arg) {
+ Profiler *P = reinterpret_cast<Profiler *>(Arg);
+ P->run();
+ return nullptr;
+}
+
+static void profile_main_cleanup(void *ptr) {
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *) ptr;
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+
+ netdata_log_info("cleaning up...");
+
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+}
+
+extern "C" void *profile_main(void *ptr) {
+ netdata_thread_cleanup_push(profile_main_cleanup, ptr);
+
+ int UpdateEvery = (int) config_get_number(CONFIG_SECTION_PROFILE, "update every", 1);
+ if (UpdateEvery < localhost->rrd_update_every)
+ UpdateEvery = localhost->rrd_update_every;
+
+ // pick low-default values, in case this plugin is ever enabled accidentaly.
+ size_t NumThreads = config_get_number(CONFIG_SECTION_PROFILE, "number of threads", 2);
+ size_t NumCharts = config_get_number(CONFIG_SECTION_PROFILE, "number of charts", 2);
+ size_t NumDimsPerChart = config_get_number(CONFIG_SECTION_PROFILE, "number of dimensions per chart", 2);
+ size_t SecondsToBackfill = config_get_number(CONFIG_SECTION_PROFILE, "seconds to backfill", 10 * 60);
+
+ std::vector<Profiler> Profilers;
+
+ for (size_t Idx = 0; Idx != NumThreads; Idx++) {
+ Profiler P(1e8 + Idx * 1e6, NumCharts, NumDimsPerChart, SecondsToBackfill, UpdateEvery);
+ Profilers.push_back(P);
+ }
+
+ std::vector<netdata_thread_t> Threads(NumThreads);
+
+ for (size_t Idx = 0; Idx != NumThreads; Idx++) {
+ char Tag[NETDATA_THREAD_TAG_MAX + 1];
+
+ snprintfz(Tag, NETDATA_THREAD_TAG_MAX, "PROFILER[%zu]", Idx);
+ netdata_thread_create(&Threads[Idx], Tag, NETDATA_THREAD_OPTION_JOINABLE, subprofile_main, static_cast<void *>(&Profilers[Idx]));
+ }
+
+ for (size_t Idx = 0; Idx != NumThreads; Idx++)
+ netdata_thread_join(Threads[Idx], nullptr);
+
+ netdata_thread_cleanup_pop(1);
+ return NULL;
+}