From cd4377fab21e0f500bef7f06543fa848a039c1e0 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 20 Jul 2023 06:50:01 +0200 Subject: Merging upstream version 1.41.0. Signed-off-by: Daniel Baumann --- collectors/profile.plugin/Makefile.am | 8 + collectors/profile.plugin/README.md | 34 +++++ collectors/profile.plugin/plugin_profile.cc | 228 ++++++++++++++++++++++++++++ 3 files changed, 270 insertions(+) create mode 100644 collectors/profile.plugin/Makefile.am create mode 100644 collectors/profile.plugin/README.md create mode 100644 collectors/profile.plugin/plugin_profile.cc (limited to 'collectors/profile.plugin') diff --git a/collectors/profile.plugin/Makefile.am b/collectors/profile.plugin/Makefile.am new file mode 100644 index 000000000..161784b8f --- /dev/null +++ b/collectors/profile.plugin/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/collectors/profile.plugin/README.md b/collectors/profile.plugin/README.md new file mode 100644 index 000000000..1f200fc31 --- /dev/null +++ b/collectors/profile.plugin/README.md @@ -0,0 +1,34 @@ +# profile.plugin + +This plugin allows someone to backfill an agent with random data. + +A user can specify: + + - The number charts they want, + - the number of dimensions per chart, + - the desire update every collection frequency, + - the number of seconds to backfill. + - the number of collection threads. + +## Configuration + +Edit the `netdata.conf` configuration file using [`edit-config`](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md#use-edit-config-to-edit-configuration-files) from the [Netdata config directory](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md#the-netdata-config-directory), which is typically at `/etc/netdata`. + +Scroll down to the `[plugin:profile]` section to find the available options: + +``` +[plugin:profile] + update every = 5 + number of charts = 200 + number of dimensions per chart = 5 + seconds to backfill = 86400 + number of threads = 16 +``` + +The `number of threads` option will create the specified number of collection +threads. The rest of the options apply to each thread individually, eg. the +above configuration will create 3200 charts, 16000 dimensions in total, which will be +backfilled for the duration of 1 day. + +Note that all but the 1st chart created in each thread will be marked as hidden +in order to ease the load on the dashboard's UI. diff --git a/collectors/profile.plugin/plugin_profile.cc b/collectors/profile.plugin/plugin_profile.cc new file mode 100644 index 000000000..5f7b22d25 --- /dev/null +++ b/collectors/profile.plugin/plugin_profile.cc @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifdef __cplusplus +extern "C" { +#endif + +#include "daemon/common.h" + +#ifdef __cplusplus +} +#endif + +#include +#include +#include + +#define PLUGIN_PROFILE_NAME "profile.plugin" + +#define CONFIG_SECTION_PROFILE "plugin:profile" + +class Generator { +public: + Generator(size_t N) : Offset(0) { + std::random_device RandDev; + std::mt19937 Gen(RandDev()); + std::uniform_int_distribution D(-16, 16); + + V.reserve(N); + for (size_t Idx = 0; Idx != N; Idx++) + V.push_back(D(Gen)); + } + + double getRandValue() { + return V[Offset++ % V.size()]; + } + +private: + size_t Offset; + std::vector V; +}; + +class Profiler { +public: + Profiler(size_t ID, size_t NumCharts, size_t NumDimsPerChart, time_t SecondsToBackfill, int UpdateEvery) : + ID(ID), + NumCharts(NumCharts), + NumDimsPerChart(NumDimsPerChart), + SecondsToBackfill(SecondsToBackfill), + UpdateEvery(UpdateEvery), + Gen(1024 * 1024) + {} + + void create() { + char ChartId[1024]; + char DimId[1024]; + + Charts.reserve(NumCharts); + for (size_t I = 0; I != NumCharts; I++) { + size_t CID = ID + Charts.size() + 1; + + snprintfz(ChartId, 1024 - 1, "chart_%zu", CID); + + RRDSET *RS = rrdset_create_localhost( + "profile", // type + ChartId, // id + nullptr, // name, + "profile_family", // family + "profile_context", // context + "profile_title", // title + "profile_units", // units + "profile_plugin", // plugin + "profile_module", // module + 12345678 + CID, // priority + UpdateEvery, // update_every + RRDSET_TYPE_LINE // chart_type + ); + if (I != 0) + rrdset_flag_set(RS, RRDSET_FLAG_HIDDEN); + Charts.push_back(RS); + + Dimensions.reserve(NumDimsPerChart); + for (size_t J = 0; J != NumDimsPerChart; J++) { + snprintfz(DimId, 1024 - 1, "dim_%zu", J); + + RRDDIM *RD = rrddim_add( + RS, // st + DimId, // id + nullptr, // name + 1, // multiplier + 1, // divisor + RRD_ALGORITHM_ABSOLUTE // algorithm + ); + + Dimensions.push_back(RD); + } + } + } + + void update(const struct timeval &Now) { + for (RRDSET *RS: Charts) { + for (RRDDIM *RD : Dimensions) { + rrddim_timed_set_by_pointer(RS, RD, Now, Gen.getRandValue()); + } + + rrdset_timed_done(RS, Now, RS->counter_done != 0); + } + } + + void run() { + #define WORKER_JOB_CREATE_CHARTS 0 + #define WORKER_JOB_UPDATE_CHARTS 1 + #define WORKER_JOB_METRIC_DURATION_TO_BACKFILL 2 + #define WORKER_JOB_METRIC_POINTS_BACKFILLED 3 + + worker_register("PROFILER"); + worker_register_job_name(WORKER_JOB_CREATE_CHARTS, "create charts"); + worker_register_job_name(WORKER_JOB_UPDATE_CHARTS, "update charts"); + worker_register_job_custom_metric(WORKER_JOB_METRIC_DURATION_TO_BACKFILL, "duration to backfill", "seconds", WORKER_METRIC_ABSOLUTE); + worker_register_job_custom_metric(WORKER_JOB_METRIC_POINTS_BACKFILLED, "points backfilled", "points", WORKER_METRIC_ABSOLUTE); + + heartbeat_t HB; + heartbeat_init(&HB); + + worker_is_busy(WORKER_JOB_CREATE_CHARTS); + create(); + + struct timeval CollectionTV; + now_realtime_timeval(&CollectionTV); + + if (SecondsToBackfill) { + CollectionTV.tv_sec -= SecondsToBackfill; + CollectionTV.tv_sec -= (CollectionTV.tv_sec % UpdateEvery); + + CollectionTV.tv_usec = 0; + } + + size_t BackfilledPoints = 0; + struct timeval NowTV, PrevTV; + now_realtime_timeval(&NowTV); + PrevTV = NowTV; + + while (service_running(SERVICE_COLLECTORS)) { + worker_is_busy(WORKER_JOB_UPDATE_CHARTS); + + update(CollectionTV); + CollectionTV.tv_sec += UpdateEvery; + + now_realtime_timeval(&NowTV); + + ++BackfilledPoints; + if (NowTV.tv_sec > PrevTV.tv_sec) { + PrevTV = NowTV; + worker_set_metric(WORKER_JOB_METRIC_POINTS_BACKFILLED, BackfilledPoints * NumCharts * NumDimsPerChart); + BackfilledPoints = 0; + } + + size_t RemainingSeconds = (CollectionTV.tv_sec >= NowTV.tv_sec) ? 0 : (NowTV.tv_sec - CollectionTV.tv_sec); + worker_set_metric(WORKER_JOB_METRIC_DURATION_TO_BACKFILL, RemainingSeconds); + + if (CollectionTV.tv_sec >= NowTV.tv_sec) { + worker_is_idle(); + heartbeat_next(&HB, UpdateEvery * USEC_PER_SEC); + } + } + } + +private: + size_t ID; + size_t NumCharts; + size_t NumDimsPerChart; + size_t SecondsToBackfill; + int UpdateEvery; + + Generator Gen; + std::vector Charts; + std::vector Dimensions; +}; + +static void *subprofile_main(void* Arg) { + Profiler *P = reinterpret_cast(Arg); + P->run(); + return nullptr; +} + +static void profile_main_cleanup(void *ptr) { + struct netdata_static_thread *static_thread = (struct netdata_static_thread *) ptr; + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; + + netdata_log_info("cleaning up..."); + + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; +} + +extern "C" void *profile_main(void *ptr) { + netdata_thread_cleanup_push(profile_main_cleanup, ptr); + + int UpdateEvery = (int) config_get_number(CONFIG_SECTION_PROFILE, "update every", 1); + if (UpdateEvery < localhost->rrd_update_every) + UpdateEvery = localhost->rrd_update_every; + + // pick low-default values, in case this plugin is ever enabled accidentaly. + size_t NumThreads = config_get_number(CONFIG_SECTION_PROFILE, "number of threads", 2); + size_t NumCharts = config_get_number(CONFIG_SECTION_PROFILE, "number of charts", 2); + size_t NumDimsPerChart = config_get_number(CONFIG_SECTION_PROFILE, "number of dimensions per chart", 2); + size_t SecondsToBackfill = config_get_number(CONFIG_SECTION_PROFILE, "seconds to backfill", 10 * 60); + + std::vector Profilers; + + for (size_t Idx = 0; Idx != NumThreads; Idx++) { + Profiler P(1e8 + Idx * 1e6, NumCharts, NumDimsPerChart, SecondsToBackfill, UpdateEvery); + Profilers.push_back(P); + } + + std::vector Threads(NumThreads); + + for (size_t Idx = 0; Idx != NumThreads; Idx++) { + char Tag[NETDATA_THREAD_TAG_MAX + 1]; + + snprintfz(Tag, NETDATA_THREAD_TAG_MAX, "PROFILER[%zu]", Idx); + netdata_thread_create(&Threads[Idx], Tag, NETDATA_THREAD_OPTION_JOINABLE, subprofile_main, static_cast(&Profilers[Idx])); + } + + for (size_t Idx = 0; Idx != NumThreads; Idx++) + netdata_thread_join(Threads[Idx], nullptr); + + netdata_thread_cleanup_pop(1); + return NULL; +} -- cgit v1.2.3