diff options
Diffstat (limited to 'collectors')
69 files changed, 2418 insertions, 400 deletions
diff --git a/collectors/Makefile.am b/collectors/Makefile.am index 24e4c3f09..2aec3dd3e 100644 --- a/collectors/Makefile.am +++ b/collectors/Makefile.am @@ -8,6 +8,7 @@ SUBDIRS = \ cgroups.plugin \ charts.d.plugin \ cups.plugin \ + debugfs.plugin \ diskspace.plugin \ timex.plugin \ ioping.plugin \ diff --git a/collectors/all.h b/collectors/all.h index a0ce5d7fc..653729bbc 100644 --- a/collectors/all.h +++ b/collectors/all.h @@ -24,8 +24,17 @@ #define NETDATA_CHART_PRIO_SYSTEM_PGPGIO 151 #define NETDATA_CHART_PRIO_SYSTEM_RAM 200 #define NETDATA_CHART_PRIO_SYSTEM_SWAP 201 +#define NETDATA_CHART_PRIO_SYSTEM_SWAP_CALLS 202 #define NETDATA_CHART_PRIO_SYSTEM_SWAPIO 250 #define NETDATA_CHART_PRIO_SYSTEM_ZSWAPIO 300 +#define NETDATA_CHART_PRIO_SYSTEM_ZSWAP_COMPRESS_RATIO 301 +#define NETDATA_CHART_PRIO_SYSTEM_ZSWAP_POOL_TOT_SIZE 302 +#define NETDATA_CHART_PRIO_SYSTEM_ZSWAP_STORED_PAGE 303 +#define NETDATA_CHART_PRIO_SYSTEM_ZSWAP_REJECTS 304 +#define NETDATA_CHART_PRIO_SYSTEM_ZSWAP_POOL_LIM_HIT 305 +#define NETDATA_CHART_PRIO_SYSTEM_ZSWAP_WRT_BACK_PAGES 306 +#define NETDATA_CHART_PRIO_SYSTEM_ZSWAP_SAME_FILL_PAGE 307 +#define NETDATA_CHART_PRIO_SYSTEM_ZSWAP_DUPP_ENTRY 308 #define NETDATA_CHART_PRIO_SYSTEM_NET 500 #define NETDATA_CHART_PRIO_SYSTEM_IPV4 500 // freebsd only #define NETDATA_CHART_PRIO_SYSTEM_IP 501 @@ -103,6 +112,7 @@ #define NETDATA_CHART_PRIO_MEM_ZRAM_SAVINGS 1601 #define NETDATA_CHART_PRIO_MEM_ZRAM_RATIO 1602 #define NETDATA_CHART_PRIO_MEM_ZRAM_EFFICIENCY 1603 +#define NETDATA_CHART_PRIO_MEM_FRAGMENTATION 1700 // Disks diff --git a/collectors/apps.plugin/apps_groups.conf b/collectors/apps.plugin/apps_groups.conf index f35454fde..659bd0f03 100644 --- a/collectors/apps.plugin/apps_groups.conf +++ b/collectors/apps.plugin/apps_groups.conf @@ -89,6 +89,7 @@ ioping: ioping go.d.plugin: *go.d.plugin* slabinfo.plugin: slabinfo.plugin ebpf.plugin: *ebpf.plugin* +debugfs.plugin: *debugfs.plugin* # agent-service-discovery agent_sd: agent_sd diff --git a/collectors/charts.d.plugin/README.md b/collectors/charts.d.plugin/README.md index 3e4edf562..97c2446fa 100644 --- a/collectors/charts.d.plugin/README.md +++ b/collectors/charts.d.plugin/README.md @@ -17,6 +17,8 @@ memory, collecting data with as little overheads as possible `charts.d.plugin` looks for scripts in `/usr/lib/netdata/charts.d`. The scripts should have the filename suffix: `.chart.sh`. +By default, `charts.d.plugin` is not included as part of the install when using [our official native DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md). You can install it by installing the `netdata-plugin-chartsd` package. + ## Configuration `charts.d.plugin` itself can be [configured](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md#use-edit-config-to-edit-configuration-files) using the configuration file `/etc/netdata/charts.d.conf`. This file is also a BASH script. diff --git a/collectors/charts.d.plugin/ap/README.md b/collectors/charts.d.plugin/ap/README.md index bc7460a28..339ad1375 100644 --- a/collectors/charts.d.plugin/ap/README.md +++ b/collectors/charts.d.plugin/ap/README.md @@ -85,6 +85,8 @@ Station 40:b8:37:5a:ed:5e (on wlan0) ## Configuration +If using [our official native DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md), make sure `netdata-plugin-chartsd` is installed. + Edit the `charts.d/ap.conf` configuration file using `edit-config` from the Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md), which is typically at `/etc/netdata`. diff --git a/collectors/charts.d.plugin/apcupsd/README.md b/collectors/charts.d.plugin/apcupsd/README.md index 6934d59c0..00e9697dc 100644 --- a/collectors/charts.d.plugin/apcupsd/README.md +++ b/collectors/charts.d.plugin/apcupsd/README.md @@ -13,6 +13,8 @@ Monitors different APC UPS models and retrieves status information using `apcacc ## Configuration +If using [our official native DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md), make sure `netdata-plugin-chartsd` is installed. + Edit the `charts.d/apcupsd.conf` configuration file using `edit-config` from the Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md), which is typically at `/etc/netdata`. diff --git a/collectors/charts.d.plugin/libreswan/README.md b/collectors/charts.d.plugin/libreswan/README.md index a20eb86c0..b6eeb0180 100644 --- a/collectors/charts.d.plugin/libreswan/README.md +++ b/collectors/charts.d.plugin/libreswan/README.md @@ -24,6 +24,8 @@ The following charts are created, **per tunnel**: ## Configuration +If using [our official native DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md), make sure `netdata-plugin-chartsd` is installed. + Edit the `charts.d/libreswan.conf` configuration file using `edit-config` from the Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md), which is typically at `/etc/netdata`. diff --git a/collectors/charts.d.plugin/nut/README.md b/collectors/charts.d.plugin/nut/README.md index 448825445..4608ce3e1 100644 --- a/collectors/charts.d.plugin/nut/README.md +++ b/collectors/charts.d.plugin/nut/README.md @@ -53,6 +53,8 @@ The following charts will be created: ## Configuration +If using [our official native DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md), make sure `netdata-plugin-chartsd` is installed. + Edit the `charts.d/nut.conf` configuration file using `edit-config` from the Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md), which is typically at `/etc/netdata`. diff --git a/collectors/charts.d.plugin/opensips/README.md b/collectors/charts.d.plugin/opensips/README.md index c278b53a0..1d7322140 100644 --- a/collectors/charts.d.plugin/opensips/README.md +++ b/collectors/charts.d.plugin/opensips/README.md @@ -11,6 +11,8 @@ learn_rel_path: "Integrations/Monitor/Networking" ## Configuration +If using [our official native DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md), make sure `netdata-plugin-chartsd` is installed. + Edit the `charts.d/opensips.conf` configuration file using `edit-config` from the Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md), which is typically at `/etc/netdata`. diff --git a/collectors/charts.d.plugin/sensors/README.md b/collectors/charts.d.plugin/sensors/README.md index 2601a2b65..0dbe96225 100644 --- a/collectors/charts.d.plugin/sensors/README.md +++ b/collectors/charts.d.plugin/sensors/README.md @@ -21,13 +21,15 @@ One chart for every sensor chip found and each of the above will be created. ## Enable the collector +If using [our official native DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md), make sure `netdata-plugin-chartsd` is installed. + The `sensors` collector is disabled by default. -To enable the collector, you need to edit the configuration file of `charts.d/sensors.conf`. You can do so by using the `edit config` script. +To enable the collector, you need to edit the configuration file of `charts.d/sensors.conf`. You can do so by using the `edit config` script. > ### Info > -> To edit configuration files in a safe way, we provide the [`edit config` script](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md#use-edit-config-to-edit-configuration-files) located in your [Netdata config directory](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md#the-netdata-config-directory) (typically is `/etc/netdata`) that creates the proper file and opens it in an editor automatically. +> To edit configuration files in a safe way, we provide the [`edit config` script](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md#use-edit-config-to-edit-configuration-files) located in your [Netdata config directory](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md#the-netdata-config-directory) (typically is `/etc/netdata`) that creates the proper file and opens it in an editor automatically. > It is recommended to use this way for configuring Netdata. > > Please also note that after most configuration changes you will need to [restart the Agent](https://github.com/netdata/netdata/blob/master/docs/configure/start-stop-restart.md) for the changes to take effect. diff --git a/collectors/debugfs.plugin/Makefile.am b/collectors/debugfs.plugin/Makefile.am new file mode 100644 index 000000000..02fe3a314 --- /dev/null +++ b/collectors/debugfs.plugin/Makefile.am @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) + diff --git a/collectors/debugfs.plugin/README.md b/collectors/debugfs.plugin/README.md new file mode 100644 index 000000000..a2dc9c0f6 --- /dev/null +++ b/collectors/debugfs.plugin/README.md @@ -0,0 +1,65 @@ +# OS provided metrics (debugfs.plugin) + +`debugfs.plugin` gathers metrics from the `/sys/kernel/debug` folder on Linux +systems. [Debugfs](https://docs.kernel.org/filesystems/debugfs.html) exists as an easy way for kernel developers to +make information available to user space. + +This plugin +is [external](https://github.com/netdata/netdata/tree/master/collectors#collector-architecture-and-terminology), +the netdata daemon spawns it as a long-running independent process. + +In detail, it collects metrics from: + +- `/sys/kernel/debug/extfrag` (Memory fragmentation index for each order and zone). +- `/sys/kernel/debug/zswap` ([Zswap](https://www.kernel.org/doc/Documentation/vm/zswap.txt) performance statistics). + +## Prerequisites + +### Permissions + +> No user action required. + +The debugfs root directory is accessible only to the root user by default. Netdata +uses [Linux Capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) to give the plugin access +to debugfs. `CAP_DAC_READ_SEARCH` is added automatically during installation. This capability allows bypassing file read +permission checks and directory read and execute permission checks. If file capabilities are not usable, then the plugin is instead installed with the SUID bit set in permissions so that it runs as root. + +## Metrics + +| Metric | Scope | Dimensions | Units | Labels | +|-------------------------------------|:---------:|:---------------------------------------------------------------------------------------:|:------------:|:---------:| +| mem.fragmentation_index_dma | numa node | order0, order1, order2, order3, order4, order5, order6, order7, order8, order9, order10 | index | numa_node | +| mem.fragmentation_index_dma32 | numa node | order0, order1, order2, order3, order4, order5, order6, order7, order8, order9, order10 | index | numa_node | +| mem.fragmentation_index_normal | numa node | order0, order1, order2, order3, order4, order5, order6, order7, order8, order9, order10 | index | numa_node | +| system.zswap_pool_compression_ratio | | compression_ratio | ratio | | +| system.zswap_pool_compressed_size | | compressed_size | bytes | | +| system.zswap_pool_raw_size | | uncompressed_size | bytes | | +| system.zswap_rejections | | compress_poor, kmemcache_fail, alloc_fail, reclaim_fail | rejections/s | | +| system.zswap_pool_limit_hit | | limit | events/s | | +| system.zswap_written_back_raw_bytes | | written_back | bytes/s | | +| system.zswap_same_filled_raw_size | | same_filled | bytes | | +| system.zswap_duplicate_entry | | entries | entries/s | | + +## Troubleshooting + +To troubleshoot issues with the collector, run the `debugfs.plugin` in the terminal. The output +should give you clues as to why the collector isn't working. + +- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on + your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`. + + ```bash + cd /usr/libexec/netdata/plugins.d/ + ``` + +- Switch to the `netdata` user. + + ```bash + sudo -u netdata -s + ``` + +- Run the `debugfs.plugin` to debug the collector: + + ```bash + ./debugfs.plugin + ``` diff --git a/collectors/debugfs.plugin/debugfs_extfrag.c b/collectors/debugfs.plugin/debugfs_extfrag.c new file mode 100644 index 000000000..75da4deca --- /dev/null +++ b/collectors/debugfs.plugin/debugfs_extfrag.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "debugfs_plugin.h" + +#define NETDATA_ORDER_FRAGMENTATION 11 + +static char *orders[NETDATA_ORDER_FRAGMENTATION] = { "order0", "order1", "order2", "order3", "order4", + "order5", "order6", "order7", "order8", "order9", + "order10" +}; + +static struct netdata_extrafrag { + char *node_zone; + uint32_t hash; + + char *id; + + collected_number orders[NETDATA_ORDER_FRAGMENTATION]; + + struct netdata_extrafrag *next; +} *netdata_extrafrags_root = NULL; + +static struct netdata_extrafrag *find_or_create_extrafrag(const char *name) +{ + struct netdata_extrafrag *extrafrag; + uint32_t hash = simple_hash(name); + + // search it, from beginning to the end + for (extrafrag = netdata_extrafrags_root ; extrafrag ; extrafrag = extrafrag->next) { + if (unlikely(hash == extrafrag->hash && !strcmp(name, extrafrag->node_zone))) { + return extrafrag; + } + } + + extrafrag = callocz(1, sizeof(struct netdata_extrafrag)); + extrafrag->node_zone = strdupz(name); + extrafrag->hash = hash; + + if (netdata_extrafrags_root) { + struct netdata_extrafrag *last_node; + for (last_node = netdata_extrafrags_root; last_node->next ; last_node = last_node->next); + + last_node->next = extrafrag; + } else + netdata_extrafrags_root = extrafrag; + + + return extrafrag; +} + +static void extfrag_send_chart(char *chart_id, collected_number *values) +{ + int i; + fprintf(stdout, "BEGIN mem.fragmentation_index_%s\n", chart_id); + for (i = 0; i < NETDATA_ORDER_FRAGMENTATION; i++) { + fprintf(stdout, "SET %s = %lld\n", orders[i], values[i]); + } + fprintf(stdout, "END\n"); + fflush(stdout); +} + +int do_debugfs_extfrag(int update_every, const char *name) { + static procfile *ff = NULL; + static int chart_order = NETDATA_CHART_PRIO_MEM_FRAGMENTATION; + + if (unlikely(!ff)) { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, + FILENAME_MAX, + "%s%s", + netdata_configured_host_prefix, + "/sys/kernel/debug/extfrag/extfrag_index"); + + ff = procfile_open(filename, " \t,", PROCFILE_FLAG_DEFAULT); + if (unlikely(!ff)) return 1; + } + + ff = procfile_readall(ff); + if (unlikely(!ff)) return 1; + + size_t l, i, j, lines = procfile_lines(ff); + for (l = 0; l < lines; l++) { + char chart_id[64]; + char zone_lowercase[32]; + if (unlikely(procfile_linewords(ff, l) < 15)) continue; + char *zone = procfile_lineword(ff, l, 3); + strncpyz(zone_lowercase, zone, 31); + debugfs2lower(zone_lowercase); + + char *id = procfile_lineword(ff, l, 1); + snprintfz(chart_id, 63, "node_%s_%s", id, zone_lowercase); + debugfs2lower(chart_id); + + struct netdata_extrafrag *extrafrag = find_or_create_extrafrag(chart_id); + collected_number *line_orders = extrafrag->orders; + for (i = 4, j = 0 ; i < 15; i++, j++) { + NETDATA_DOUBLE value = str2ndd(procfile_lineword(ff, l, i), NULL); + line_orders[j] = (collected_number) (value * 1000.0); + } + + if (unlikely(!extrafrag->id)) { + extrafrag->id = extrafrag->node_zone; + fprintf( + stdout, + "CHART mem.fragmentation_index_%s '' 'Memory fragmentation index for each order' 'index' 'fragmentation' 'mem.fragmentation_index_%s' 'line' %d %d '' 'debugfs.plugin' '%s'\n", + extrafrag->node_zone, + zone_lowercase, + chart_order++, // FIXME: the same zones must have the same order + update_every, + name); + for (i = 0; i < NETDATA_ORDER_FRAGMENTATION; i++) { + fprintf(stdout, "DIMENSION '%s' '%s' absolute 1 1000 ''\n", orders[i], orders[i]); + } + fprintf(stdout, + "CLABEL 'numa_node' 'node%s' 1\n" + "CLABEL_COMMIT\n", + id); + } + extfrag_send_chart(chart_id, line_orders); + } + + return 0; +} diff --git a/collectors/debugfs.plugin/debugfs_plugin.c b/collectors/debugfs.plugin/debugfs_plugin.c new file mode 100644 index 000000000..9713be320 --- /dev/null +++ b/collectors/debugfs.plugin/debugfs_plugin.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "debugfs_plugin.h" +#include "libnetdata/required_dummies.h" + +static char *user_config_dir = CONFIG_DIR; +static char *stock_config_dir = LIBCONFIG_DIR; + +static int update_every = 1; + +static struct debugfs_module { + const char *name; + + int enabled; + + int (*func)(int update_every, const char *name); +} debugfs_modules[] = { + // Memory Fragmentation + { .name = "/sys/kernel/debug/extfrag", .enabled = CONFIG_BOOLEAN_YES, + .func = do_debugfs_extfrag}, + { .name = "/sys/kernel/debug/zswap", .enabled = CONFIG_BOOLEAN_YES, + .func = do_debugfs_zswap}, + + // The terminator + { .name = NULL, .enabled = CONFIG_BOOLEAN_NO, .func = NULL} +}; + +#ifdef HAVE_CAPABILITY +static int debugfs_check_capabilities() +{ + cap_t caps = cap_get_proc(); + if (!caps) { + error("Cannot get current capabilities."); + return 0; + } + + int ret = 1; + cap_flag_value_t cfv = CAP_CLEAR; + if (cap_get_flag(caps, CAP_DAC_READ_SEARCH, CAP_EFFECTIVE, &cfv) == -1) { + error("Cannot find if CAP_DAC_READ_SEARCH is effective."); + ret = 0; + } else { + if (cfv != CAP_SET) { + error("debugfs.plugin should run with CAP_DAC_READ_SEARCH."); + ret = 0; + } + } + cap_free(caps); + + return ret; +} +#else +static int debugfs_check_capabilities() +{ + return 0; +} +#endif + +// TODO: This is a function used by 3 different collector, we should do it global (next PR) +static int debugfs_am_i_running_as_root() +{ + uid_t uid = getuid(), euid = geteuid(); + + if (uid == 0 || euid == 0) { + return 1; + } + + return 0; +} + +void debugfs2lower(char *name) +{ + while (*name) { + *name = tolower(*name); + name++; + } +} + +// Consiidering our goal to redce binaries, I preferred to copy function, instead to force link with unecessary libs +const char *debugfs_rrdset_type_name(RRDSET_TYPE chart_type) { + switch(chart_type) { + case RRDSET_TYPE_LINE: + default: + return RRDSET_TYPE_LINE_NAME; + + case RRDSET_TYPE_AREA: + return RRDSET_TYPE_AREA_NAME; + + case RRDSET_TYPE_STACKED: + return RRDSET_TYPE_STACKED_NAME; + } +} + +const char *debugfs_rrd_algorithm_name(RRD_ALGORITHM algorithm) { + switch(algorithm) { + case RRD_ALGORITHM_ABSOLUTE: + default: + return RRD_ALGORITHM_ABSOLUTE_NAME; + + case RRD_ALGORITHM_INCREMENTAL: + return RRD_ALGORITHM_INCREMENTAL_NAME; + + case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL: + return RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME; + + case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL: + return RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME; + } +} + +int debugfs_check_sys_permission() { + int ret = 0; + + char filename[FILENAME_MAX + 1]; + + snprintfz(filename, FILENAME_MAX, "%s/sys/kernel/debug/extfrag/extfrag_index", netdata_configured_host_prefix); + + procfile *ff = procfile_open(filename, NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO); + if(!ff) goto dcsp_cleanup; + + ff = procfile_readall(ff); + if(!ff) goto dcsp_cleanup; + + ret = 1; + +dcsp_cleanup: + if (!ret) + perror("Cannot open /sys/kernel/debug/extfrag/extfrag_index file"); + procfile_close(ff); + return ret; +} + +static void debugfs_parse_args(int argc, char **argv) +{ + int i, freq = 0; + for(i = 1; i < argc; i++) { + if(!freq) { + int n = (int)str2l(argv[i]); + if(n > 0) { + freq = n; + continue; + } + } + + if(strcmp("test-permissions", argv[i]) == 0 || strcmp("-t", argv[i]) == 0) { + if(!debugfs_check_sys_permission()) { + exit(2); + } + printf("OK\n"); + exit(0); + } + } + + if(freq > 0) update_every = freq; +} + +int main(int argc, char **argv) +{ + // debug_flags = D_PROCFILE; + stderror = stderr; + + // set the name for logging + program_name = "debugfs.plugin"; + + // disable syslog for debugfs.plugin + error_log_syslog = 0; + + netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX"); + if (verify_netdata_host_prefix() == -1) + exit(1); + + user_config_dir = getenv("NETDATA_USER_CONFIG_DIR"); + if (user_config_dir == NULL) { + user_config_dir = CONFIG_DIR; + } + + stock_config_dir = getenv("NETDATA_STOCK_CONFIG_DIR"); + if (stock_config_dir == NULL) { + // info("NETDATA_CONFIG_DIR is not passed from netdata"); + stock_config_dir = LIBCONFIG_DIR; + } + + // FIXME: should first check if /sys/kernel/debug is mounted + + // FIXME: remove debugfs_check_sys_permission() after https://github.com/netdata/netdata/issues/15048 is fixed + if (!debugfs_check_capabilities() && !debugfs_am_i_running_as_root() && !debugfs_check_sys_permission()) { + uid_t uid = getuid(), euid = geteuid(); +#ifdef HAVE_CAPABILITY + error( + "debugfs.plugin should either run as root (now running with uid %u, euid %u) or have special capabilities. " + "Without these, debugfs.plugin cannot access /sys/kernel/debug. " + "To enable capabilities run: sudo setcap cap_dac_read_search,cap_sys_ptrace+ep %s; " + "To enable setuid to root run: sudo chown root:netdata %s; sudo chmod 4750 %s; ", + uid, + euid, + argv[0], + argv[0], + argv[0]); +#else + error( + "debugfs.plugin should either run as root (now running with uid %u, euid %u) or have special capabilities. " + "Without these, debugfs.plugin cannot access /sys/kernel/debug." + "Your system does not support capabilities. " + "To enable setuid to root run: sudo chown root:netdata %s; sudo chmod 4750 %s; ", + uid, + euid, + argv[0], + argv[0]); +#endif + exit(1); + } + + // if (!debugfs_check_sys_permission()) { + // exit(2); + // } + + debugfs_parse_args(argc, argv); + + size_t iteration; + usec_t step = update_every * USEC_PER_SEC; + heartbeat_t hb; + heartbeat_init(&hb); + + for (iteration = 0; iteration < 86400; iteration++) { + heartbeat_next(&hb, step); + int enabled = 0; + + for (int i = 0; debugfs_modules[i].name; i++) { + struct debugfs_module *pm = &debugfs_modules[i]; + if (unlikely(!pm->enabled)) + continue; + + pm->enabled = !pm->func(update_every, pm->name); + if (likely(pm->enabled)) + enabled++; + } + if (!enabled) { + info("all modules are disabled, exiting..."); + return 1; + } + } + + fprintf(stdout, "EXIT\n"); + fflush(stdout); + return 0; +} diff --git a/collectors/debugfs.plugin/debugfs_plugin.h b/collectors/debugfs.plugin/debugfs_plugin.h new file mode 100644 index 000000000..c53187d6e --- /dev/null +++ b/collectors/debugfs.plugin/debugfs_plugin.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_DEBUGFS_PLUGIN_H +#define NETDATA_DEBUGFS_PLUGIN_H 1 + +#include "libnetdata/libnetdata.h" +#include "collectors/all.h" +#include "database/rrd.h" + +int do_debugfs_extfrag(int update_every, const char *name); +int do_debugfs_zswap(int update_every, const char *name); +void debugfs2lower(char *name); +const char *debugfs_rrdset_type_name(RRDSET_TYPE chart_type); +const char *debugfs_rrd_algorithm_name(RRD_ALGORITHM algorithm); + +#endif // NETDATA_DEBUGFS_PLUGIN_H diff --git a/collectors/debugfs.plugin/debugfs_zswap.c b/collectors/debugfs.plugin/debugfs_zswap.c new file mode 100644 index 000000000..a2991b9f1 --- /dev/null +++ b/collectors/debugfs.plugin/debugfs_zswap.c @@ -0,0 +1,437 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "debugfs_plugin.h" + +static long system_page_size = 4096; + +static collected_number pages_to_bytes(collected_number value) +{ + return value * system_page_size; +} + +struct netdata_zswap_metric { + const char *filename; + + const char *chart_id; + const char *title; + const char *units; + RRDSET_TYPE charttype; + int prio; + const char *dimension; + RRD_ALGORITHM algorithm; + int divisor; + + int enabled; + int chart_created; + + collected_number value; + collected_number (*convertv)(collected_number v); +}; + +static struct netdata_zswap_metric zswap_calculated_metrics[] = { + {.filename = "", + .chart_id = "pool_compression_ratio", + .dimension = "compression_ratio", + .units = "ratio", + .title = "Zswap compression ratio", + .algorithm = RRD_ALGORITHM_ABSOLUTE, + .charttype = RRDSET_TYPE_LINE, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_COMPRESS_RATIO, + .divisor = 100, + .convertv = NULL, + .value = -1}, +}; + +enum netdata_zswap_calculated { + NETDATA_ZSWAP_COMPRESSION_RATIO_CHART, +}; + +enum netdata_zwap_independent { + NETDATA_ZSWAP_POOL_TOTAL_SIZE, + NETDATA_ZSWAP_STORED_PAGES, + NETDATA_ZSWAP_POOL_LIMIT_HIT, + NETDATA_ZSWAP_WRITTEN_BACK_PAGES, + NETDATA_ZSWAP_SAME_FILLED_PAGES, + NETDATA_ZSWAP_DUPLICATE_ENTRY, + + // Terminator + NETDATA_ZSWAP_SITE_END +}; + +static struct netdata_zswap_metric zswap_independent_metrics[] = { + // https://elixir.bootlin.com/linux/latest/source/mm/zswap.c + {.filename = "/sys/kernel/debug/zswap/pool_total_size", + .chart_id = "pool_compressed_size", + .dimension = "compressed_size", + .units = "bytes", + .title = "Zswap compressed bytes currently stored", + .algorithm = RRD_ALGORITHM_ABSOLUTE, + .charttype = RRDSET_TYPE_AREA, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_POOL_TOT_SIZE, + .divisor = 1, + .convertv = NULL, + .value = -1}, + {.filename = "/sys/kernel/debug/zswap/stored_pages", + .chart_id = "pool_raw_size", + .dimension = "uncompressed_size", + .units = "bytes", + .title = "Zswap uncompressed bytes currently stored", + .algorithm = RRD_ALGORITHM_ABSOLUTE, + .charttype = RRDSET_TYPE_AREA, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_STORED_PAGE, + .divisor = 1, + .convertv = pages_to_bytes, + .value = -1}, + {.filename = "/sys/kernel/debug/zswap/pool_limit_hit", + .chart_id = "pool_limit_hit", + .dimension = "limit", + .units = "events/s", + .title = "Zswap pool limit was reached", + .algorithm = RRD_ALGORITHM_INCREMENTAL, + .charttype = RRDSET_TYPE_LINE, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_POOL_LIM_HIT, + .divisor = 1, + .convertv = NULL, + .value = -1}, + {.filename = "/sys/kernel/debug/zswap/written_back_pages", + .chart_id = "written_back_raw_bytes", + .dimension = "written_back", + .units = "bytes/s", + .title = "Zswap uncomressed bytes written back when pool limit was reached", + .algorithm = RRD_ALGORITHM_INCREMENTAL, + .charttype = RRDSET_TYPE_AREA, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_WRT_BACK_PAGES, + .divisor = 1, + .convertv = pages_to_bytes, + .value = -1}, + {.filename = "/sys/kernel/debug/zswap/same_filled_pages", + .chart_id = "same_filled_raw_size", + .dimension = "same_filled", + .units = "bytes", + .title = "Zswap same-value filled uncompressed bytes currently stored", + .algorithm = RRD_ALGORITHM_ABSOLUTE, + .charttype = RRDSET_TYPE_AREA, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_SAME_FILL_PAGE, + .divisor = 1, + .convertv = pages_to_bytes, + .value = -1}, + {.filename = "/sys/kernel/debug/zswap/duplicate_entry", + .chart_id = "duplicate_entry", + .dimension = "duplicate", + .units = "entries/s", + .title = "Zswap duplicate store was encountered", + .algorithm = RRD_ALGORITHM_INCREMENTAL, + .charttype = RRDSET_TYPE_LINE, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_DUPP_ENTRY, + .divisor = 1, + .convertv = NULL, + .value = -1}, + + // The terminator + {.filename = NULL, + .chart_id = NULL, + .dimension = NULL, + .units = NULL, + .title = NULL, + .algorithm = RRD_ALGORITHM_ABSOLUTE, + .charttype = RRDSET_TYPE_LINE, + .enabled = CONFIG_BOOLEAN_NO, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = -1, + .value = -1}}; + +enum netdata_zswap_rejected { + NETDATA_ZSWAP_REJECTED_CHART, + NETDATA_ZSWAP_REJECTED_COMPRESS_POOR, + NETDATA_ZSWAP_REJECTED_KMEM_FAIL, + NETDATA_ZSWAP_REJECTED_RALLOC_FAIL, + NETDATA_ZSWAP_REJECTED_RRECLAIM_FAIL, + + // Terminator + NETDATA_ZSWAP_REJECTED_END +}; + +static struct netdata_zswap_metric zswap_rejected_metrics[] = { + {.filename = "/sys/kernel/debug/zswap/", + .chart_id = "rejections", + .dimension = NULL, + .units = "rejections/s", + .title = "Zswap rejections", + .algorithm = RRD_ALGORITHM_INCREMENTAL, + .charttype = RRDSET_TYPE_STACKED, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_REJECTS, + .divisor = 1, + .convertv = NULL, + .value = -1}, + {.filename = "/sys/kernel/debug/zswap/reject_compress_poor", + .chart_id = "reject_compress_poor", + .dimension = "compress_poor", + .units = NULL, + .title = NULL, + .algorithm = RRD_ALGORITHM_INCREMENTAL, + .charttype = RRDSET_TYPE_STACKED, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_REJECTS, + .divisor = 1, + .convertv = NULL, + .value = -1}, + {.filename = "/sys/kernel/debug/zswap/reject_kmemcache_fail", + .chart_id = "reject_kmemcache_fail", + .dimension = "kmemcache_fail", + .units = NULL, + .title = NULL, + .algorithm = RRD_ALGORITHM_INCREMENTAL, + .charttype = RRDSET_TYPE_STACKED, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_REJECTS, + .divisor = 1, + .convertv = NULL, + .value = -1}, + {.filename = "/sys/kernel/debug/zswap/reject_alloc_fail", + .chart_id = "reject_alloc_fail", + .dimension = "alloc_fail", + .units = NULL, + .title = NULL, + .algorithm = RRD_ALGORITHM_INCREMENTAL, + .charttype = RRDSET_TYPE_STACKED, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_REJECTS, + .divisor = 1, + .convertv = NULL, + .value = -1}, + {.filename = "/sys/kernel/debug/zswap/reject_reclaim_fail", + .chart_id = "reject_reclaim_fail", + .dimension = "reclaim_fail", + .units = NULL, + .title = NULL, + .algorithm = RRD_ALGORITHM_INCREMENTAL, + .charttype = RRDSET_TYPE_STACKED, + .enabled = CONFIG_BOOLEAN_YES, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = NETDATA_CHART_PRIO_SYSTEM_ZSWAP_REJECTS, + .divisor = 1, + .convertv = NULL, + .value = -1}, + + // The terminator + {.filename = NULL, + .chart_id = NULL, + .dimension = NULL, + .units = NULL, + .title = NULL, + .algorithm = RRD_ALGORITHM_ABSOLUTE, + .charttype = RRDSET_TYPE_STACKED, + .enabled = CONFIG_BOOLEAN_NO, + .chart_created = CONFIG_BOOLEAN_NO, + .prio = -1, + .value = -1}}; + +int zswap_collect_data(struct netdata_zswap_metric *metric) +{ + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, metric->filename); + + if (read_single_number_file(filename, (unsigned long long *)&metric->value)) { + error("Cannot read file %s", filename); + return 1; + } + + if (metric->convertv) + metric->value = metric->convertv(metric->value); + + return 0; +} + +static void +zswap_send_chart(struct netdata_zswap_metric *metric, int update_every, const char *name, const char *option) +{ + fprintf( + stdout, + "CHART system.zswap_%s '' '%s' '%s' 'zswap' '' '%s' %d %d '%s' 'debugfs.plugin' '%s'\n", + metric->chart_id, + metric->title, + metric->units, + debugfs_rrdset_type_name(metric->charttype), + metric->prio, + update_every, + (!option) ? "" : option, + name); +} + +static void zswap_send_dimension(struct netdata_zswap_metric *metric) +{ + int div = metric->divisor > 0 ? metric->divisor : 1; + fprintf( + stdout, + "DIMENSION '%s' '%s' %s 1 %d ''\n", + metric->dimension, + metric->dimension, + debugfs_rrd_algorithm_name(metric->algorithm), + div); +} + +static void zswap_send_begin(struct netdata_zswap_metric *metric) +{ + fprintf(stdout, "BEGIN system.zswap_%s\n", metric->chart_id); +} + +static void zswap_send_set(struct netdata_zswap_metric *metric) +{ + fprintf(stdout, "SET %s = %lld\n", metric->dimension, metric->value); +} + +static void zswap_send_end_and_flush() +{ + fprintf(stdout, "END\n"); + fflush(stdout); +} + +static void zswap_independent_chart(struct netdata_zswap_metric *metric, int update_every, const char *name) +{ + if (unlikely(!metric->chart_created)) { + metric->chart_created = CONFIG_BOOLEAN_YES; + + zswap_send_chart(metric, update_every, name, NULL); + zswap_send_dimension(metric); + } + + zswap_send_begin(metric); + zswap_send_set(metric); + zswap_send_end_and_flush(); +} + +void zswap_reject_chart(int update_every, const char *name) +{ + struct netdata_zswap_metric *metric = &zswap_rejected_metrics[NETDATA_ZSWAP_REJECTED_CHART]; + + if (unlikely(!metric->chart_created)) { + metric->chart_created = CONFIG_BOOLEAN_YES; + + zswap_send_chart(metric, update_every, name, NULL); + for (int i = NETDATA_ZSWAP_REJECTED_COMPRESS_POOR; zswap_rejected_metrics[i].filename; i++) { + metric = &zswap_rejected_metrics[i]; + if (likely(metric->enabled)) + zswap_send_dimension(metric); + } + } + + metric = &zswap_rejected_metrics[NETDATA_ZSWAP_REJECTED_CHART]; + zswap_send_begin(metric); + for (int i = NETDATA_ZSWAP_REJECTED_COMPRESS_POOR; zswap_rejected_metrics[i].filename; i++) { + metric = &zswap_rejected_metrics[i]; + if (likely(metric->enabled)) + zswap_send_set(metric); + } + zswap_send_end_and_flush(); +} + +static void zswap_obsolete_charts(int update_every, const char *name) +{ + struct netdata_zswap_metric *metric = NULL; + + for (int i = 0; zswap_independent_metrics[i].filename; i++) { + metric = &zswap_independent_metrics[i]; + if (likely(metric->chart_created)) + zswap_send_chart(metric, update_every, name, "obsolete"); + } + + metric = &zswap_rejected_metrics[NETDATA_ZSWAP_REJECTED_CHART]; + if (likely(metric->chart_created)) + zswap_send_chart(metric, update_every, name, "obsolete"); + + metric = &zswap_calculated_metrics[NETDATA_ZSWAP_COMPRESSION_RATIO_CHART]; + if (likely(metric->chart_created)) + zswap_send_chart(metric, update_every, name, "obsolete"); +} + +#define ZSWAP_STATE_SIZE 1 // Y or N +static int debugfs_is_zswap_enabled() +{ + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "/sys/module/zswap/parameters/enabled"); // host prefix is not needed here + char state[ZSWAP_STATE_SIZE + 1]; + + int ret = read_file(filename, state, ZSWAP_STATE_SIZE); + + if (unlikely(!ret && !strcmp(state, "Y"))) { + return 0; + } + return 1; +} + +int do_debugfs_zswap(int update_every, const char *name) +{ + static int check_if_enabled = 1; + + if (likely(check_if_enabled && debugfs_is_zswap_enabled())) { + info("Zswap is disabled"); + return 1; + } + + check_if_enabled = 0; + + system_page_size = sysconf(_SC_PAGESIZE); + struct netdata_zswap_metric *metric = NULL; + int enabled = 0; + + for (int i = 0; zswap_independent_metrics[i].filename; i++) { + metric = &zswap_independent_metrics[i]; + if (unlikely(!metric->enabled)) + continue; + if (unlikely(!(metric->enabled = !zswap_collect_data(metric)))) + continue; + zswap_independent_chart(metric, update_every, name); + enabled++; + } + + struct netdata_zswap_metric *metric_size = &zswap_independent_metrics[NETDATA_ZSWAP_POOL_TOTAL_SIZE]; + struct netdata_zswap_metric *metric_raw_size = &zswap_independent_metrics[NETDATA_ZSWAP_STORED_PAGES]; + if (metric_size->enabled && metric_raw_size->enabled) { + metric = &zswap_calculated_metrics[NETDATA_ZSWAP_COMPRESSION_RATIO_CHART]; + metric->value = 0; + if (metric_size->value > 0) + metric->value = + (collected_number)((NETDATA_DOUBLE)metric_raw_size->value / (NETDATA_DOUBLE)metric_size->value * 100); + zswap_independent_chart(metric, update_every, name); + } + + int enabled_rejected = 0; + for (int i = NETDATA_ZSWAP_REJECTED_COMPRESS_POOR; zswap_rejected_metrics[i].filename; i++) { + metric = &zswap_rejected_metrics[i]; + if (unlikely(!metric->enabled)) + continue; + if (unlikely(!(metric->enabled = !zswap_collect_data(metric)))) + continue; + enabled++; + enabled_rejected++; + } + + if (likely(enabled_rejected > 0)) + zswap_reject_chart(update_every, name); + + if (unlikely(!enabled)) { + zswap_obsolete_charts(update_every, name); + return 1; + } + + return 0; +} diff --git a/collectors/debugfs.plugin/metrics.csv b/collectors/debugfs.plugin/metrics.csv new file mode 100644 index 000000000..a21383941 --- /dev/null +++ b/collectors/debugfs.plugin/metrics.csv @@ -0,0 +1,12 @@ +metric,scode,dimensions,unit,description,chart_type,labels,plugin,module +mem.fragmentation_index_dma,node,"order0, order1, order2, order3, order4, order5, order6, order7, order8, order9, order10",index,Memory fragmentation index for each order,line,numa_node,debugfs.plugin,/sys/kernel/debug/extfrag +mem.fragmentation_index_dma32,node,"order0, order1, order2, order3, order4, order5, order6, order7, order8, order9, order10",index,Memory fragmentation index for each order,line,numa_node,debugfs.plugin,/sys/kernel/debug/extfrag +mem.fragmentation_index_normal,node,"order0, order1, order2, order3, order4, order5, order6, order7, order8, order9, order10",index,Memory fragmentation index for each order,line,numa_node,debugfs.plugin,/sys/kernel/debug/extfrag +system.zswap_pool_compression_ratio,,compression_ratio,ratio,Zswap compression ratio,line,,debugfs.plugin,/sys/kernel/debug/zswap +system.zswap_pool_compressed_size,,compressed_size,bytes,Zswap compressed bytes currently stored,area,,debugfs.plugin,/sys/kernel/debug/zswap +system.zswap_pool_raw_size,,uncompressed_size,bytes,Zswap uncompressed bytes currently stored,area,,debugfs.plugin,/sys/kernel/debug/zswap +system.zswap_rejections,,"compress_poor, kmemcache_fail, alloc_fail, reclaim_fail",rejections/s,Zswap rejections,stacked,,debugfs.plugin,/sys/kernel/debug/zswap +system.zswap_pool_limit_hit,,limit,events/s,Zswap pool limit was reached,line,,debugfs.plugin,/sys/kernel/debug/zswap +system.zswap_written_back_raw_bytes,,written_back,bytes/s,Zswap uncomressed bytes written back when pool limit was reached,area,,debugfs.plugin,/sys/kernel/debug/zswap +system.zswap_same_filled_raw_size,,same_filled,bytes,Zswap same-value filled uncompressed bytes currently stored,area,,debugfs.plugin,/sys/kernel/debug/zswap +system.zswap_duplicate_entry,,duplicate,entries/s,Zswap duplicate store was encountered,line,,debugfs.plugin,/sys/kernel/debug/zswap diff --git a/collectors/diskspace.plugin/README.md b/collectors/diskspace.plugin/README.md index b70bbf008..5ca1090fd 100644 --- a/collectors/diskspace.plugin/README.md +++ b/collectors/diskspace.plugin/README.md @@ -13,6 +13,8 @@ Simple patterns can be used to exclude mounts from showed statistics based on pa By default, Netdata will enable monitoring metrics only when they are not zero. If they are constantly zero they are ignored. Metrics that will start having values, after Netdata is started, will be detected and charts will be automatically added to the dashboard (a refresh of the dashboard is needed for them to appear though). +Netdata will try to detect mounts that are duplicates (i.e. from the same device), or binds, and will not display charts for them, as the device is usually already monitored. + To configure this plugin, you need to edit the configuration file `netdata.conf`. You can do so by using the `edit config` script. > ### Info diff --git a/collectors/ebpf.plugin/README.md b/collectors/ebpf.plugin/README.md index 75f44a6e5..94bbc184d 100644 --- a/collectors/ebpf.plugin/README.md +++ b/collectors/ebpf.plugin/README.md @@ -99,8 +99,6 @@ accepts the following values: - `return`: In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates new charts for the return of these functions, such as errors. Monitoring function returns can help in debugging software, such as failing to close file descriptors or creating zombie processes. -- `update every`: Number of seconds used for eBPF to send data for Netdata. -- `pid table size`: Defines the maximum number of PIDs stored inside the application hash table. #### Integration with `apps.plugin` @@ -115,11 +113,6 @@ If you want to enable `apps.plugin` integration, change the "apps" setting to "y apps = yes ``` -When the integration is enabled, eBPF collector allocates memory for each process running. The total allocated memory -has direct relationship with the kernel version. When the eBPF plugin is running on kernels newer than `4.15`, it uses -per-cpu maps to speed up the update of hash tables. This also implies storing data for the same PID for each processor -it runs. - #### Integration with `cgroups.plugin` The eBPF collector also creates charts for each cgroup through an integration with the @@ -138,6 +131,13 @@ If you do not need to monitor specific metrics for your `cgroups`, you can enabl `ebpf.d.conf`, and then disable the plugin for a specific `thread` by following the steps in the [Configuration](#configuring-ebpfplugin) section. +#### Maps per Core + +When netdata is running on kernels newer than `4.6` users are allowed to modify how the `ebpf.plugin` creates maps (hash or +array). When `maps per core` is defined as `yes`, plugin will create a map per core on host, on the other hand, +when the value is set as `no` only one hash table will be created, this option will use less memory, but it also can +increase overhead for processes. + #### Collect PID When one of the previous integrations is enabled, `ebpf.plugin` will use Process Identifier (`PID`) to identify the @@ -157,6 +157,16 @@ The threads that have integration with other collectors have an internal clean u will only enable these threads integrated with other collectors when the kernel is compiled with `CONFIG_DEBUG_INFO_BTF`, unless you enable them manually. +#### Collection period + +The plugin uses the option `update every` to define the number of seconds used for eBPF to send data for Netdata. The default value +is 5 seconds. + +#### PID table size + +The option `pid table size` defines the maximum number of PIDs stored inside the application hash table. The default value +is defined according [kernel](https://elixir.bootlin.com/linux/v6.0.19/source/include/linux/threads.h#L28) source code. + #### Integration Dashboard Elements When an integration is enabled, your dashboard will also show the following cgroups and apps charts using low-level @@ -880,14 +890,24 @@ These are tracepoints related to [OOM](https://en.wikipedia.org/wiki/Out_of_memo eBPF monitoring is complex and produces a large volume of metrics. We've discovered scenarios where the eBPF plugin significantly increases kernel memory usage by several hundred MB. -If your node is experiencing high memory usage and there is no obvious culprit to be found in the `apps.mem` chart, -consider testing for high kernel memory usage by [disabling eBPF monitoring](#configuring-ebpfplugin). Next, -[restart Netdata](https://github.com/netdata/netdata/blob/master/docs/configure/start-stop-restart.md) with `sudo systemctl restart netdata` to see if system memory -usage (see the `system.ram` chart) has dropped significantly. +When the integration with apps or cgroup is enabled, the eBPF collector allocates memory for each process running. If your +node is experiencing high memory usage and there is no obvious culprit to be found in the `apps.mem` chart, consider: + +- Modify [maps per core](#maps-per-core) to use only one map. +- Disable [integration with apps](#integration-with-appsplugin). +- Disable [integration with cgroup](#integration-with-cgroupsplugin). -Beginning with `v1.31`, kernel memory usage is configurable via the [`pid table size` setting](#ebpf-load-mode) +If with these changes you still suspect eBPF using too much memory, and there is no obvious culprit to be found +in the `apps.mem` chart, consider testing for high kernel memory usage by [disabling eBPF monitoring](#configuring-ebpfplugin). +Next, [restart Netdata](https://github.com/netdata/netdata/blob/master/docs/configure/start-stop-restart.md) with +`sudo systemctl restart netdata` to see if system memory usage (see the `system.ram` chart) has dropped significantly. + +Beginning with `v1.31`, kernel memory usage is configurable via the [`pid table size` setting](#pid-table-size) in `ebpf.conf`. +The total memory usage is a well known [issue](https://lore.kernel.org/all/167821082315.1693.6957546778534183486.git-patchwork-notify@kernel.org/) +for eBPF, this is not a bug present in plugin. + ### SELinux When [SELinux](https://www.redhat.com/en/topics/linux/what-is-selinux) is enabled, it may prevent `ebpf.plugin` from diff --git a/collectors/ebpf.plugin/ebpf.c b/collectors/ebpf.plugin/ebpf.c index c0764c600..45303574f 100644 --- a/collectors/ebpf.plugin/ebpf.c +++ b/collectors/ebpf.plugin/ebpf.c @@ -6,6 +6,7 @@ #include "ebpf.h" #include "ebpf_socket.h" +#include "ebpf_unittest.h" #include "libnetdata/required_dummies.h" /***************************************************************** @@ -54,7 +55,8 @@ ebpf_module_t ebpf_modules[] = { .config_file = NETDATA_PROCESS_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_10 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "socket", .config_name = "socket", .enabled = 0, .start_routine = ebpf_socket_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, @@ -62,7 +64,8 @@ ebpf_module_t ebpf_modules[] = { .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &socket_config, .config_file = NETDATA_NETWORK_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = socket_targets, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = socket_targets, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "cachestat", .config_name = "cachestat", .enabled = 0, .start_routine = ebpf_cachestat_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, @@ -71,7 +74,8 @@ ebpf_module_t ebpf_modules[] = { .config_file = NETDATA_CACHESTAT_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18| NETDATA_V5_4 | NETDATA_V5_14 | NETDATA_V5_15 | NETDATA_V5_16, - .load = EBPF_LOAD_LEGACY, .targets = cachestat_targets, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = cachestat_targets, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "sync", .config_name = "sync", .enabled = 0, .start_routine = ebpf_sync_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, @@ -79,7 +83,8 @@ ebpf_module_t ebpf_modules[] = { .config_file = NETDATA_SYNC_CONFIG_FILE, // All syscalls have the same kernels .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = sync_targets, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = sync_targets, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "dc", .config_name = "dc", .enabled = 0, .start_routine = ebpf_dcstat_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, @@ -87,7 +92,8 @@ ebpf_module_t ebpf_modules[] = { .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &dcstat_config, .config_file = NETDATA_DIRECTORY_DCSTAT_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = dc_targets, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = dc_targets, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "swap", .config_name = "swap", .enabled = 0, .start_routine = ebpf_swap_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, @@ -95,7 +101,8 @@ ebpf_module_t ebpf_modules[] = { .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &swap_config, .config_file = NETDATA_DIRECTORY_SWAP_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = swap_targets, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = swap_targets, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "vfs", .config_name = "vfs", .enabled = 0, .start_routine = ebpf_vfs_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, @@ -103,28 +110,32 @@ ebpf_module_t ebpf_modules[] = { .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &vfs_config, .config_file = NETDATA_DIRECTORY_VFS_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = vfs_targets, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = vfs_targets, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "filesystem", .config_name = "filesystem", .enabled = 0, .start_routine = ebpf_filesystem_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &fs_config, .config_file = NETDATA_FILESYSTEM_CONFIG_FILE, //We are setting kernels as zero, because we load eBPF programs according the kernel running. - .kernels = 0, .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL }, + .kernels = 0, .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES }, { .thread_name = "disk", .config_name = "disk", .enabled = 0, .start_routine = ebpf_disk_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &disk_config, .config_file = NETDATA_DISK_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "mount", .config_name = "mount", .enabled = 0, .start_routine = ebpf_mount_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &mount_config, .config_file = NETDATA_MOUNT_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = mount_targets, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = mount_targets, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "fd", .config_name = "fd", .enabled = 0, .start_routine = ebpf_fd_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, @@ -133,21 +144,24 @@ ebpf_module_t ebpf_modules[] = { .config_file = NETDATA_FD_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_11 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = fd_targets, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = fd_targets, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "hardirq", .config_name = "hardirq", .enabled = 0, .start_routine = ebpf_hardirq_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &hardirq_config, .config_file = NETDATA_HARDIRQ_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "softirq", .config_name = "softirq", .enabled = 0, .start_routine = ebpf_softirq_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &softirq_config, .config_file = NETDATA_SOFTIRQ_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "oomkill", .config_name = "oomkill", .enabled = 0, .start_routine = ebpf_oomkill_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, @@ -155,7 +169,8 @@ ebpf_module_t ebpf_modules[] = { .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &oomkill_config, .config_file = NETDATA_OOMKILL_CONFIG_FILE, .kernels = NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "shm", .config_name = "shm", .enabled = 0, .start_routine = ebpf_shm_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, @@ -163,19 +178,21 @@ ebpf_module_t ebpf_modules[] = { .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &shm_config, .config_file = NETDATA_DIRECTORY_SHM_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = shm_targets, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = shm_targets, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = "mdflush", .config_name = "mdflush", .enabled = 0, .start_routine = ebpf_mdflush_thread, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &mdflush_config, .config_file = NETDATA_DIRECTORY_MDFLUSH_CONFIG_FILE, .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14, - .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, + .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, { .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 0, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = 0, .names = NULL, .cfg = NULL, .config_name = NULL, .kernels = 0, .load = EBPF_LOAD_LEGACY, - .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL}, + .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL, .maps_per_core = CONFIG_BOOLEAN_YES}, }; struct netdata_static_thread ebpf_threads[] = { @@ -360,7 +377,8 @@ ebpf_filesystem_partitions_t localfs[] = .flags = NETDATA_FILESYSTEM_FLAG_NO_PARTITION, .enabled = CONFIG_BOOLEAN_YES, .addresses = {.function = NULL, .addr = 0}, - .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4}, + .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4, + .fs_maps = NULL}, {.filesystem = "xfs", .optional_filesystem = NULL, .family = "xfs", @@ -369,7 +387,8 @@ ebpf_filesystem_partitions_t localfs[] = .flags = NETDATA_FILESYSTEM_FLAG_NO_PARTITION, .enabled = CONFIG_BOOLEAN_YES, .addresses = {.function = NULL, .addr = 0}, - .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4}, + .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4, + .fs_maps = NULL}, {.filesystem = "nfs", .optional_filesystem = "nfs4", .family = "nfs", @@ -378,7 +397,8 @@ ebpf_filesystem_partitions_t localfs[] = .flags = NETDATA_FILESYSTEM_ATTR_CHARTS, .enabled = CONFIG_BOOLEAN_YES, .addresses = {.function = NULL, .addr = 0}, - .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4}, + .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4, + .fs_maps = NULL}, {.filesystem = "zfs", .optional_filesystem = NULL, .family = "zfs", @@ -387,7 +407,8 @@ ebpf_filesystem_partitions_t localfs[] = .flags = NETDATA_FILESYSTEM_FLAG_NO_PARTITION, .enabled = CONFIG_BOOLEAN_YES, .addresses = {.function = NULL, .addr = 0}, - .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4}, + .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4, + .fs_maps = NULL}, {.filesystem = "btrfs", .optional_filesystem = NULL, .family = "btrfs", @@ -396,7 +417,8 @@ ebpf_filesystem_partitions_t localfs[] = .flags = NETDATA_FILESYSTEM_FILL_ADDRESS_TABLE, .enabled = CONFIG_BOOLEAN_YES, .addresses = {.function = "btrfs_file_operations", .addr = 0}, - .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_10}, + .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_10, + .fs_maps = NULL}, {.filesystem = NULL, .optional_filesystem = NULL, .family = NULL, @@ -405,43 +427,50 @@ ebpf_filesystem_partitions_t localfs[] = .flags = NETDATA_FILESYSTEM_FLAG_NO_PARTITION, .enabled = CONFIG_BOOLEAN_YES, .addresses = {.function = NULL, .addr = 0}, - .kernels = 0}}; + .kernels = 0, .fs_maps = NULL}}; ebpf_sync_syscalls_t local_syscalls[] = { {.syscall = NETDATA_SYSCALLS_SYNC, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL, #ifdef LIBBPF_MAJOR_VERSION - .sync_obj = NULL + .sync_obj = NULL, #endif + .sync_maps = NULL }, {.syscall = NETDATA_SYSCALLS_SYNCFS, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL, #ifdef LIBBPF_MAJOR_VERSION - .sync_obj = NULL + .sync_obj = NULL, #endif + .sync_maps = NULL }, {.syscall = NETDATA_SYSCALLS_MSYNC, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL, #ifdef LIBBPF_MAJOR_VERSION - .sync_obj = NULL + .sync_obj = NULL, #endif + .sync_maps = NULL }, {.syscall = NETDATA_SYSCALLS_FSYNC, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL, #ifdef LIBBPF_MAJOR_VERSION - .sync_obj = NULL + .sync_obj = NULL, #endif + .sync_maps = NULL }, {.syscall = NETDATA_SYSCALLS_FDATASYNC, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL, #ifdef LIBBPF_MAJOR_VERSION - .sync_obj = NULL + .sync_obj = NULL, #endif + .sync_maps = NULL }, {.syscall = NETDATA_SYSCALLS_SYNC_FILE_RANGE, .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL, #ifdef LIBBPF_MAJOR_VERSION - .sync_obj = NULL + .sync_obj = NULL, #endif + .sync_maps = NULL }, {.syscall = NULL, .enabled = CONFIG_BOOLEAN_NO, .objects = NULL, .probe_links = NULL, #ifdef LIBBPF_MAJOR_VERSION - .sync_obj = NULL + .sync_obj = NULL, #endif + .sync_maps = NULL } }; @@ -550,7 +579,7 @@ static void ebpf_exit() * @param objects objects loaded from eBPF programs * @param probe_links links from loader */ -static void ebpf_unload_legacy_code(struct bpf_object *objects, struct bpf_link **probe_links) +void ebpf_unload_legacy_code(struct bpf_object *objects, struct bpf_link **probe_links) { if (!probe_links || !objects) return; @@ -1738,6 +1767,21 @@ static inline void epbf_update_load_mode(char *str, netdata_ebpf_load_mode_t ori } /** + * Update Map per core + * + * Define the map type used with some hash tables. + */ +static void ebpf_update_map_per_core() +{ + int i; + int value = appconfig_get_boolean(&collector_config, EBPF_GLOBAL_SECTION, + EBPF_CFG_MAPS_PER_CORE, CONFIG_BOOLEAN_YES); + for (i = 0; ebpf_modules[i].thread_name; i++) { + ebpf_modules[i].maps_per_core = value; + } +} + +/** * Read collector values * * @param disable_apps variable to store information related to apps. @@ -1790,6 +1834,8 @@ static void read_collector_values(int *disable_apps, int *disable_cgroups, enabled = appconfig_get_boolean(&collector_config, EBPF_GLOBAL_SECTION, EBPF_CFG_CGROUP, CONFIG_BOOLEAN_NO); *disable_cgroups = (enabled == CONFIG_BOOLEAN_NO)?CONFIG_BOOLEAN_YES:CONFIG_BOOLEAN_NO; + ebpf_update_map_per_core(); + // Read ebpf programs section enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, ebpf_modules[EBPF_MODULE_PROCESS_IDX].config_name, CONFIG_BOOLEAN_YES); @@ -2015,6 +2061,48 @@ static inline void ebpf_load_thread_config() } /** + * Check Conditions + * + * This function checks kernel that plugin is running and permissions. + * + * @return It returns 0 on success and -1 otherwise + */ +int ebpf_check_conditions() +{ + if (!has_condition_to_run(running_on_kernel)) { + error("The current collector cannot run on this kernel."); + return -1; + } + + if (!am_i_running_as_root()) { + error( + "ebpf.plugin should either run as root (now running with uid %u, euid %u) or have special capabilities..", + (unsigned int)getuid(), (unsigned int)geteuid()); + return -1; + } + + return 0; +} + +/** + * Adjust memory + * + * Adjust memory values to load eBPF programs. + * + * @return It returns 0 on success and -1 otherwise + */ +int ebpf_adjust_memory_limit() +{ + struct rlimit r = { RLIM_INFINITY, RLIM_INFINITY }; + if (setrlimit(RLIMIT_MEMLOCK, &r)) { + error("Setrlimit(RLIMIT_MEMLOCK)"); + return -1; + } + + return 0; +} + +/** * Parse arguments given from user. * * @param argc the number of arguments @@ -2052,6 +2140,7 @@ static void ebpf_parse_args(int argc, char **argv) {"return", no_argument, 0, 0 }, {"legacy", no_argument, 0, 0 }, {"core", no_argument, 0, 0 }, + {"unittest", no_argument, 0, 0 }, {0, 0, 0, 0} }; @@ -2242,6 +2331,33 @@ static void ebpf_parse_args(int argc, char **argv) #endif break; } + case EBPF_OPTION_UNITTEST: { + // if we cannot run until the end, we will cancel the unittest + int exit_code = ECANCELED; + if (ebpf_check_conditions()) + goto unittest; + + if (ebpf_adjust_memory_limit()) + goto unittest; + + // Load binary in entry mode + ebpf_ut_initialize_structure(MODE_ENTRY); + if (ebpf_ut_load_real_binary()) + goto unittest; + + ebpf_ut_cleanup_memory(); + + // Do not load a binary in entry mode + ebpf_ut_initialize_structure(MODE_ENTRY); + if (ebpf_ut_load_fake_binary()) + goto unittest; + + ebpf_ut_cleanup_memory(); + + exit_code = 0; +unittest: + exit(exit_code); + } default: { break; } @@ -2460,17 +2576,8 @@ int main(int argc, char **argv) ebpf_parse_args(argc, argv); ebpf_manage_pid(getpid()); - if (!has_condition_to_run(running_on_kernel)) { - error("The current collector cannot run on this kernel."); + if (ebpf_check_conditions()) return 2; - } - - if (!am_i_running_as_root()) { - error( - "ebpf.plugin should either run as root (now running with uid %u, euid %u) or have special capabilities..", - (unsigned int)getuid(), (unsigned int)geteuid()); - return 3; - } // set name program_name = "ebpf.plugin"; @@ -2482,11 +2589,8 @@ int main(int argc, char **argv) error_log_errors_per_period = 100; error_log_throttle_period = 3600; - struct rlimit r = { RLIM_INFINITY, RLIM_INFINITY }; - if (setrlimit(RLIMIT_MEMLOCK, &r)) { - error("Setrlimit(RLIMIT_MEMLOCK)"); - return 4; - } + if (ebpf_adjust_memory_limit()) + return 3; signal(SIGINT, ebpf_stop_threads); signal(SIGQUIT, ebpf_stop_threads); @@ -2540,6 +2644,7 @@ int main(int argc, char **argv) heartbeat_init(&hb); int update_apps_every = (int) EBPF_CFG_UPDATE_APPS_EVERY_DEFAULT; int update_apps_list = update_apps_every - 1; + int process_maps_per_core = ebpf_modules[EBPF_MODULE_PROCESS_IDX].maps_per_core; //Plugin will be killed when it receives a signal while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, step); @@ -2550,7 +2655,7 @@ int main(int argc, char **argv) if (++update_apps_list == update_apps_every) { update_apps_list = 0; cleanup_exited_pids(); - collect_data_for_all_processes(process_pid_fd); + collect_data_for_all_processes(process_pid_fd, process_maps_per_core); pthread_mutex_lock(&lock); ebpf_create_apps_charts(apps_groups_root_target); @@ -2565,3 +2670,4 @@ int main(int argc, char **argv) return 0; } + diff --git a/collectors/ebpf.plugin/ebpf.d.conf b/collectors/ebpf.plugin/ebpf.d.conf index 6a5ec5c39..8807f9a3a 100644 --- a/collectors/ebpf.plugin/ebpf.d.conf +++ b/collectors/ebpf.plugin/ebpf.d.conf @@ -15,6 +15,10 @@ # # The `pid table size` defines the maximum number of PIDs stored in the application hash tables. # +# The `btf path` specifies where to find the BTF files. +# +# The `maps per core` defines if hash tables will be per core or not. This option is ignored on kernels older than 4.15. +# [global] ebpf load mode = entry apps = no @@ -22,6 +26,7 @@ update every = 5 pid table size = 32768 btf path = /sys/kernel/btf/ + maps per core = yes # # eBPF Programs @@ -63,7 +68,7 @@ shm = yes socket = no softirq = yes - sync = yes + sync = no swap = yes vfs = no network connections = no diff --git a/collectors/ebpf.plugin/ebpf.d/cachestat.conf b/collectors/ebpf.plugin/ebpf.d/cachestat.conf index 52466be51..82f870c98 100644 --- a/collectors/ebpf.plugin/ebpf.d/cachestat.conf +++ b/collectors/ebpf.plugin/ebpf.d/cachestat.conf @@ -24,6 +24,8 @@ # `parent` : Only stores parent PID. # `all` : Stores all PIDs used by software. This is the most expensive option. # +# The `maps per core` defines if hash tables will be per core or not. This option is ignored on kernels older than 4.6. +# # Uncomment lines to define specific options for thread. [global] # ebpf load mode = entry @@ -34,3 +36,4 @@ ebpf type format = auto ebpf co-re tracing = trampoline collect pid = real parent +# maps per core = yes diff --git a/collectors/ebpf.plugin/ebpf.d/dcstat.conf b/collectors/ebpf.plugin/ebpf.d/dcstat.conf index 8aed8f783..f741b62a8 100644 --- a/collectors/ebpf.plugin/ebpf.d/dcstat.conf +++ b/collectors/ebpf.plugin/ebpf.d/dcstat.conf @@ -22,6 +22,8 @@ # `parent` : Only stores parent PID. # `all` : Stores all PIDs used by software. This is the most expensive option. # +# The `maps per core` defines if hash tables will be per core or not. This option is ignored on kernels older than 4.6. +# # Uncomment lines to define specific options for thread. [global] # ebpf load mode = entry @@ -32,3 +34,4 @@ ebpf type format = auto ebpf co-re tracing = trampoline collect pid = real parent +# maps per core = yes diff --git a/collectors/ebpf.plugin/ebpf.d/fd.conf b/collectors/ebpf.plugin/ebpf.d/fd.conf index 8333520fc..30a5fcfd9 100644 --- a/collectors/ebpf.plugin/ebpf.d/fd.conf +++ b/collectors/ebpf.plugin/ebpf.d/fd.conf @@ -10,6 +10,8 @@ # # The `pid table size` defines the maximum number of PIDs stored inside the hash table. # +# The `maps per core` defines if hash tables will be per core or not. This option is ignored on kernels older than 4.6. +# # Uncomment lines to define specific options for thread. [global] # ebpf load mode = entry @@ -19,3 +21,4 @@ # pid table size = 32768 ebpf type format = auto ebpf co-re tracing = trampoline +# maps per core = yes diff --git a/collectors/ebpf.plugin/ebpf.d/network.conf b/collectors/ebpf.plugin/ebpf.d/network.conf index d939d8e1f..75644a772 100644 --- a/collectors/ebpf.plugin/ebpf.d/network.conf +++ b/collectors/ebpf.plugin/ebpf.d/network.conf @@ -24,6 +24,9 @@ # `tracepoint`: When available, the eBPF collector will use kernel tracepoint to monitor syscall. # `probe` : This is the same as legacy code. # +# The `maps per core` defines if hash tables will be per core or not. This option is ignored on kernels older than 4.6. +# +# Uncomment lines to define specific options for thread. [global] # ebpf load mode = entry # apps = yes @@ -35,6 +38,7 @@ udp connection table size = 4096 ebpf type format = auto ebpf co-re tracing = trampoline + maps per core = no # # Network Connection diff --git a/collectors/ebpf.plugin/ebpf.d/process.conf b/collectors/ebpf.plugin/ebpf.d/process.conf index 1da5f84d3..f5e8804cd 100644 --- a/collectors/ebpf.plugin/ebpf.d/process.conf +++ b/collectors/ebpf.plugin/ebpf.d/process.conf @@ -15,11 +15,14 @@ # `parent` : Only stores parent PID. # `all` : Stores all PIDs used by software. This is the most expensive option. # +# The `maps per core` defines if hash tables will be per core or not. This option is ignored on kernels older than 4.6. +# # Uncomment lines to define specific options for thread. -#[global] +[global] # ebpf load mode = entry # apps = yes # cgroups = no # update every = 10 # pid table size = 32768 -# collect pid = real parent + collect pid = real parent +# maps per core = yes diff --git a/collectors/ebpf.plugin/ebpf.d/shm.conf b/collectors/ebpf.plugin/ebpf.d/shm.conf index 23ab96da4..f8ec1a18f 100644 --- a/collectors/ebpf.plugin/ebpf.d/shm.conf +++ b/collectors/ebpf.plugin/ebpf.d/shm.conf @@ -18,6 +18,8 @@ # `tracepoint`: When available, the eBPF collector will use kernel tracepoint to monitor syscall. # `probe` : This is the same as legacy code. # +# The `maps per core` defines if hash tables will be per core or not. This option is ignored on kernels older than 4.6. +# # Uncomment lines to define specific options for thread. [global] # ebpf load mode = entry @@ -27,6 +29,7 @@ # pid table size = 32768 ebpf type format = auto ebpf co-re tracing = trampoline +# maps per core = yes # List of monitored syscalls [syscalls] diff --git a/collectors/ebpf.plugin/ebpf.d/swap.conf b/collectors/ebpf.plugin/ebpf.d/swap.conf index 3986ae4f8..5bad04424 100644 --- a/collectors/ebpf.plugin/ebpf.d/swap.conf +++ b/collectors/ebpf.plugin/ebpf.d/swap.conf @@ -17,6 +17,8 @@ # `trampoline`: This is the default mode used by the eBPF collector, due the small overhead added to host. # `probe` : This is the same as legacy code. # +# The `maps per core` defines if hash tables will be per core or not. This option is ignored on kernels older than 4.6. +# # Uncomment lines to define specific options for thread. [global] # ebpf load mode = entry @@ -26,3 +28,4 @@ # pid table size = 32768 ebpf type format = auto ebpf co-re tracing = trampoline +# maps per core = yes diff --git a/collectors/ebpf.plugin/ebpf.d/sync.conf b/collectors/ebpf.plugin/ebpf.d/sync.conf index ebec5d38e..fefbd4ee6 100644 --- a/collectors/ebpf.plugin/ebpf.d/sync.conf +++ b/collectors/ebpf.plugin/ebpf.d/sync.conf @@ -17,7 +17,10 @@ # `trampoline`: This is the default mode used by the eBPF collector, due the small overhead added to host. # `tracepoint`: When available, the eBPF collector will use kernel tracepoint to monitor syscall. # `probe` : This is the same as legacy code. +# +# The `maps per core` defines if hash tables will be per core or not. This option is ignored on kernels older than 4.6. # +# Uncomment lines to define specific options for thread. [global] # ebpf load mode = entry # apps = yes @@ -25,6 +28,7 @@ # update every = 10 ebpf type format = auto ebpf co-re tracing = trampoline +# maps per core = yes # List of monitored syscalls [syscalls] diff --git a/collectors/ebpf.plugin/ebpf.d/vfs.conf b/collectors/ebpf.plugin/ebpf.d/vfs.conf index fa5d5b4e9..b4e5daac0 100644 --- a/collectors/ebpf.plugin/ebpf.d/vfs.conf +++ b/collectors/ebpf.plugin/ebpf.d/vfs.conf @@ -8,6 +8,18 @@ # If you want to disable the integration with `apps.plugin` or `cgroups.plugin` along with the above charts, change # the setting `apps` and `cgroups` to 'no'. # +# The `ebpf type format` option accepts the following values : +# `auto` : The eBPF collector will investigate hardware and select between the two next options. +# `legacy`: The eBPF collector will load the legacy code. Note: This has a bigger overload. +# `co-re` : The eBPF collector will use latest tracing method. Note: This is not available on all platforms. +# +# The `ebpf co-re tracing` option accepts the following values: +# `trampoline`: This is the default mode used by the eBPF collector, due the small overhead added to host. +# `tracepoint`: When available, the eBPF collector will use kernel tracepoint to monitor syscall. +# `probe` : This is the same as legacy code. +# +# The `maps per core` defines if hash tables will be per core or not. This option is ignored on kernels older than 4.6. +# # Uncomment lines to define specific options for thread. [global] # ebpf load mode = entry @@ -17,3 +29,4 @@ # pid table size = 32768 ebpf type format = auto ebpf co-re tracing = trampoline +# maps per core = yes diff --git a/collectors/ebpf.plugin/ebpf.h b/collectors/ebpf.plugin/ebpf.h index 5b48adc62..ae24c302c 100644 --- a/collectors/ebpf.plugin/ebpf.h +++ b/collectors/ebpf.plugin/ebpf.h @@ -119,7 +119,8 @@ enum ebpf_main_index { EBPF_OPTION_GLOBAL_CHART, EBPF_OPTION_RETURN_MODE, EBPF_OPTION_LEGACY, - EBPF_OPTION_CORE + EBPF_OPTION_CORE, + EBPF_OPTION_UNITTEST }; typedef struct ebpf_tracepoint { @@ -159,6 +160,7 @@ typedef struct ebpf_tracepoint { #define NETDATA_EBPF_LOAD_METHOD "ebpf_load_methods" #define NETDATA_EBPF_KERNEL_MEMORY "ebpf_kernel_memory" #define NETDATA_EBPF_HASH_TABLES_LOADED "ebpf_hash_tables_count" +#define NETDATA_EBPF_HASH_TABLES_PER_CORE "ebpf_hash_tables_per_core" // Log file #define NETDATA_DEVELOPER_LOG_FILE "developer.log" @@ -307,6 +309,8 @@ void ebpf_write_chart_obsolete(char *type, char *id, char *title, char *units, c void write_histogram_chart(char *family, char *name, const netdata_idx_t *hist, char **dimensions, uint32_t end); void ebpf_update_disabled_plugin_stats(ebpf_module_t *em); ARAL *ebpf_allocate_pid_aral(char *name, size_t size); +void ebpf_unload_legacy_code(struct bpf_object *objects, struct bpf_link **probe_links); + extern ebpf_filesystem_partitions_t localfs[]; extern ebpf_sync_syscalls_t local_syscalls[]; extern int ebpf_exit_plugin; diff --git a/collectors/ebpf.plugin/ebpf_apps.c b/collectors/ebpf.plugin/ebpf_apps.c index d6db4c676..3826f8efc 100644 --- a/collectors/ebpf.plugin/ebpf_apps.c +++ b/collectors/ebpf.plugin/ebpf_apps.c @@ -1415,14 +1415,37 @@ static inline void aggregate_pid_on_target(struct ebpf_target *w, struct ebpf_pi } /** + * Process Accumulator + * + * Sum all values read from kernel and store in the first address. + * + * @param out the vector with read values. + * @param maps_per_core do I need to read all cores? + */ +void ebpf_process_apps_accumulator(ebpf_process_stat_t *out, int maps_per_core) +{ + int i, end = (maps_per_core) ? ebpf_nprocs : 1; + ebpf_process_stat_t *total = &out[0]; + for (i = 1; i < end; i++) { + ebpf_process_stat_t *w = &out[i]; + total->exit_call += w->exit_call; + total->task_err += w->task_err; + total->create_thread += w->create_thread; + total->create_process += w->create_process; + total->release_call += w->release_call; + } +} + +/** * Collect data for all process * * Read data from hash table and store it in appropriate vectors. * It also creates the link between targets and PIDs. * * @param tbl_pid_stats_fd The mapped file descriptor for the hash table. + * @param maps_per_core do I have hash maps per core? */ -void collect_data_for_all_processes(int tbl_pid_stats_fd) +void collect_data_for_all_processes(int tbl_pid_stats_fd, int maps_per_core) { if (unlikely(!ebpf_all_pids)) return; @@ -1448,6 +1471,10 @@ void collect_data_for_all_processes(int tbl_pid_stats_fd) uint32_t key; pids = ebpf_root_of_pids; // global list of all processes running // while (bpf_map_get_next_key(tbl_pid_stats_fd, &key, &next_key) == 0) { + size_t length = sizeof(ebpf_process_stat_t); + if (maps_per_core) + length *= ebpf_nprocs; + while (pids) { key = pids->pid; ebpf_process_stat_t *w = global_process_stats[key]; @@ -1456,7 +1483,7 @@ void collect_data_for_all_processes(int tbl_pid_stats_fd) global_process_stats[key] = w; } - if (bpf_map_lookup_elem(tbl_pid_stats_fd, &key, w)) { + if (bpf_map_lookup_elem(tbl_pid_stats_fd, &key, process_stat_vector)) { // Clean Process structures ebpf_process_stat_release(w); global_process_stats[key] = NULL; @@ -1467,6 +1494,12 @@ void collect_data_for_all_processes(int tbl_pid_stats_fd) continue; } + ebpf_process_apps_accumulator(process_stat_vector, maps_per_core); + + memcpy(w, process_stat_vector, sizeof(ebpf_process_stat_t)); + + memset(process_stat_vector, 0, length); + pids = pids->next; } diff --git a/collectors/ebpf.plugin/ebpf_apps.h b/collectors/ebpf.plugin/ebpf_apps.h index d33442af5..ad2e338d4 100644 --- a/collectors/ebpf.plugin/ebpf_apps.h +++ b/collectors/ebpf.plugin/ebpf_apps.h @@ -213,7 +213,8 @@ size_t read_processes_statistic_using_pid_on_target(ebpf_process_stat_t **ep, size_t read_bandwidth_statistic_using_pid_on_target(ebpf_bandwidth_t **ep, int fd, struct ebpf_pid_on_target *pids); -void collect_data_for_all_processes(int tbl_pid_stats_fd); +void collect_data_for_all_processes(int tbl_pid_stats_fd, int maps_per_core); +void ebpf_process_apps_accumulator(ebpf_process_stat_t *out, int maps_per_core); extern ebpf_process_stat_t **global_process_stats; extern netdata_publish_cachestat_t **cachestat_pid; @@ -235,6 +236,7 @@ extern void ebpf_aral_init(void); extern ebpf_process_stat_t *ebpf_process_stat_get(void); extern void ebpf_process_stat_release(ebpf_process_stat_t *stat); +extern ebpf_process_stat_t *process_stat_vector; extern ARAL *ebpf_aral_socket_pid; void ebpf_socket_aral_init(); diff --git a/collectors/ebpf.plugin/ebpf_cachestat.c b/collectors/ebpf.plugin/ebpf_cachestat.c index b2b006dd3..5bbbe1f43 100644 --- a/collectors/ebpf.plugin/ebpf_cachestat.c +++ b/collectors/ebpf.plugin/ebpf_cachestat.c @@ -14,19 +14,34 @@ static netdata_idx_t cachestat_hash_values[NETDATA_CACHESTAT_END]; static netdata_idx_t *cachestat_values = NULL; ebpf_local_maps_t cachestat_maps[] = {{.name = "cstat_global", .internal_input = NETDATA_CACHESTAT_END, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "cstat_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE, - .user_input = 0, - .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "cstat_ctrl", .internal_input = NETDATA_CONTROLLER_END, - .user_input = 0, - .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = NULL, .internal_input = 0, .user_input = 0, - .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}}; + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = "cstat_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE, + .user_input = 0, + .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, + {.name = "cstat_ctrl", .internal_input = NETDATA_CONTROLLER_END, + .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION +#endif + }}; struct config cachestat_config = { .first_section = NULL, .last_section = NULL, @@ -233,10 +248,14 @@ static int ebpf_cachestat_attach_probe(struct cachestat_bpf *obj) * @param obj is the main structure for bpf objects. * @param em structure with configuration */ -static void ebpf_cachestat_adjust_map_size(struct cachestat_bpf *obj, ebpf_module_t *em) +static void ebpf_cachestat_adjust_map(struct cachestat_bpf *obj, ebpf_module_t *em) { ebpf_update_map_size(obj->maps.cstat_pid, &cachestat_maps[NETDATA_CACHESTAT_PID_STATS], em, bpf_map__name(obj->maps.cstat_pid)); + + ebpf_update_map_type(obj->maps.cstat_global, &cachestat_maps[NETDATA_CACHESTAT_GLOBAL_STATS]); + ebpf_update_map_type(obj->maps.cstat_pid, &cachestat_maps[NETDATA_CACHESTAT_PID_STATS]); + ebpf_update_map_type(obj->maps.cstat_ctrl, &cachestat_maps[NETDATA_CACHESTAT_CTRL]); } /** @@ -291,7 +310,7 @@ static inline int ebpf_cachestat_load_and_attach(struct cachestat_bpf *obj, ebpf ebpf_cachestat_disable_specific_probe(obj); } - ebpf_cachestat_adjust_map_size(obj, em); + ebpf_cachestat_adjust_map(obj, em); if (!em->apps_charts && !em->cgroup_charts) ebpf_cachestat_disable_release_task(obj); @@ -445,10 +464,11 @@ static void calculate_stats(netdata_publish_cachestat_t *publish) { * Sum all values read from kernel and store in the first address. * * @param out the vector with read values. + * @param maps_per_core do I need to read all cores? */ -static void cachestat_apps_accumulator(netdata_cachestat_pid_t *out) +static void cachestat_apps_accumulator(netdata_cachestat_pid_t *out, int maps_per_core) { - int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1; + int i, end = (maps_per_core) ? ebpf_nprocs : 1; netdata_cachestat_pid_t *total = &out[0]; for (i = 1; i < end; i++) { netdata_cachestat_pid_t *w = &out[i]; @@ -504,14 +524,19 @@ static void cachestat_fill_pid(uint32_t current_pid, netdata_cachestat_pid_t *pu * Read APPS table * * Read the apps table and store data inside the structure. + * + * @param maps_per_core do I need to read all cores? */ -static void read_apps_table() +static void ebpf_read_cachestat_apps_table(int maps_per_core) { netdata_cachestat_pid_t *cv = cachestat_vector; uint32_t key; struct ebpf_pid_stat *pids = ebpf_root_of_pids; int fd = cachestat_maps[NETDATA_CACHESTAT_PID_STATS].map_fd; - size_t length = sizeof(netdata_cachestat_pid_t)*ebpf_nprocs; + size_t length = sizeof(netdata_cachestat_pid_t); + if (maps_per_core) + length *= ebpf_nprocs; + while (pids) { key = pids->pid; @@ -520,7 +545,7 @@ static void read_apps_table() continue; } - cachestat_apps_accumulator(cv); + cachestat_apps_accumulator(cv, maps_per_core); cachestat_fill_pid(key, cv); @@ -535,12 +560,16 @@ static void read_apps_table() * Update cgroup * * Update cgroup data based in + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_update_cachestat_cgroup() +static void ebpf_update_cachestat_cgroup(int maps_per_core) { netdata_cachestat_pid_t *cv = cachestat_vector; int fd = cachestat_maps[NETDATA_CACHESTAT_PID_STATS].map_fd; - size_t length = sizeof(netdata_cachestat_pid_t) * ebpf_nprocs; + size_t length = sizeof(netdata_cachestat_pid_t); + if (maps_per_core) + length *= ebpf_nprocs; ebpf_cgroup_target_t *ect; pthread_mutex_lock(&mutex_cgroup_shm); @@ -559,7 +588,7 @@ static void ebpf_update_cachestat_cgroup() continue; } - cachestat_apps_accumulator(cv); + cachestat_apps_accumulator(cv, maps_per_core); memcpy(out, cv, sizeof(netdata_cachestat_pid_t)); } @@ -627,8 +656,10 @@ void ebpf_cachestat_create_apps_charts(struct ebpf_module *em, void *ptr) * Read global counter * * Read the table with number of calls for all functions + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_cachestat_read_global_table() +static void ebpf_cachestat_read_global_table(int maps_per_core) { uint32_t idx; netdata_idx_t *val = cachestat_hash_values; @@ -638,7 +669,7 @@ static void ebpf_cachestat_read_global_table() for (idx = NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU; idx < NETDATA_CACHESTAT_END; idx++) { if (!bpf_map_lookup_elem(fd, &idx, stored)) { int i; - int end = ebpf_nprocs; + int end = (maps_per_core) ? ebpf_nprocs: 1; netdata_idx_t total = 0; for (i = 0; i < end; i++) total += stored[i]; @@ -1053,6 +1084,7 @@ static void cachestat_collector(ebpf_module_t *em) memset(&publish, 0, sizeof(publish)); int cgroups = em->cgroup_charts; int update_every = em->update_every; + int maps_per_core = em->maps_per_core; heartbeat_t hb; heartbeat_init(&hb); int counter = update_every - 1; @@ -1065,13 +1097,13 @@ static void cachestat_collector(ebpf_module_t *em) counter = 0; netdata_apps_integration_flags_t apps = em->apps_charts; - ebpf_cachestat_read_global_table(); + ebpf_cachestat_read_global_table(maps_per_core); pthread_mutex_lock(&collect_data_mutex); if (apps) - read_apps_table(); + ebpf_read_cachestat_apps_table(maps_per_core); if (cgroups) - ebpf_update_cachestat_cgroup(); + ebpf_update_cachestat_cgroup(maps_per_core); pthread_mutex_lock(&lock); @@ -1216,6 +1248,10 @@ static int ebpf_cachestat_set_internal_value() */ static int ebpf_cachestat_load_bpf(ebpf_module_t *em) { +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(cachestat_maps, em->maps_per_core, running_on_kernel); +#endif + int ret = 0; ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU].mode); if (em->load & EBPF_LOAD_LEGACY) { diff --git a/collectors/ebpf.plugin/ebpf_dcstat.c b/collectors/ebpf.plugin/ebpf_dcstat.c index 5f1400601..5a07e4619 100644 --- a/collectors/ebpf.plugin/ebpf_dcstat.c +++ b/collectors/ebpf.plugin/ebpf_dcstat.c @@ -19,19 +19,35 @@ struct config dcstat_config = { .first_section = NULL, .rwlock = AVL_LOCK_INITIALIZER } }; ebpf_local_maps_t dcstat_maps[] = {{.name = "dcstat_global", .internal_input = NETDATA_DIRECTORY_CACHE_END, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "dcstat_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE, - .user_input = 0, - .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "dcstat_ctrl", .internal_input = NETDATA_CONTROLLER_END, - .user_input = 0, - .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = NULL, .internal_input = 0, .user_input = 0, - .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}}; + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = "dcstat_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE, + .user_input = 0, + .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, + {.name = "dcstat_ctrl", .internal_input = NETDATA_CONTROLLER_END, + .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; static ebpf_specify_name_t dc_optional_name[] = { {.program_name = "netdata_lookup_fast", .function_to_attach = "lookup_fast", @@ -138,10 +154,14 @@ static int ebpf_dc_attach_probes(struct dc_bpf *obj) * @param obj is the main structure for bpf objects. * @param em structure with configuration */ -static void ebpf_dc_adjust_map_size(struct dc_bpf *obj, ebpf_module_t *em) +static void ebpf_dc_adjust_map(struct dc_bpf *obj, ebpf_module_t *em) { ebpf_update_map_size(obj->maps.dcstat_pid, &dcstat_maps[NETDATA_DCSTAT_PID_STATS], em, bpf_map__name(obj->maps.dcstat_pid)); + + ebpf_update_map_type(obj->maps.dcstat_global, &dcstat_maps[NETDATA_DCSTAT_GLOBAL_STATS]); + ebpf_update_map_type(obj->maps.dcstat_pid, &dcstat_maps[NETDATA_DCSTAT_PID_STATS]); + ebpf_update_map_type(obj->maps.dcstat_ctrl, &dcstat_maps[NETDATA_DCSTAT_CTRL]); } /** @@ -215,7 +235,7 @@ static inline int ebpf_dc_load_and_attach(struct dc_bpf *obj, ebpf_module_t *em) ebpf_dc_disable_trampoline(obj); } - ebpf_dc_adjust_map_size(obj, em); + ebpf_dc_adjust_map(obj, em); if (!em->apps_charts && !em->cgroup_charts) ebpf_dc_disable_release_task(obj); @@ -382,10 +402,11 @@ void ebpf_dcstat_create_apps_charts(struct ebpf_module *em, void *ptr) * Sum all values read from kernel and store in the first address. * * @param out the vector with read values. + * @param maps_per_core do I need to read all cores? */ -static void dcstat_apps_accumulator(netdata_dcstat_pid_t *out) +static void dcstat_apps_accumulator(netdata_dcstat_pid_t *out, int maps_per_core) { - int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1; + int i, end = (maps_per_core) ? ebpf_nprocs : 1; netdata_dcstat_pid_t *total = &out[0]; for (i = 1; i < end; i++) { netdata_dcstat_pid_t *w = &out[i]; @@ -428,17 +449,22 @@ static void dcstat_fill_pid(uint32_t current_pid, netdata_dcstat_pid_t *publish) } /** - * Read APPS table + * Read Directory Cache APPS table * * Read the apps table and store data inside the structure. + * + * @param maps_per_core do I need to read all cores? */ -static void read_apps_table() +static void read_dc_apps_table(int maps_per_core) { netdata_dcstat_pid_t *cv = dcstat_vector; uint32_t key; struct ebpf_pid_stat *pids = ebpf_root_of_pids; int fd = dcstat_maps[NETDATA_DCSTAT_PID_STATS].map_fd; - size_t length = sizeof(netdata_dcstat_pid_t)*ebpf_nprocs; + size_t length = sizeof(netdata_dcstat_pid_t); + if (maps_per_core) + length *= ebpf_nprocs; + while (pids) { key = pids->pid; @@ -447,7 +473,7 @@ static void read_apps_table() continue; } - dcstat_apps_accumulator(cv); + dcstat_apps_accumulator(cv, maps_per_core); dcstat_fill_pid(key, cv); @@ -461,9 +487,11 @@ static void read_apps_table() /** * Update cgroup * - * Update cgroup data based in + * Update cgroup data based in collected PID. + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_update_dc_cgroup() +static void ebpf_update_dc_cgroup(int maps_per_core) { netdata_dcstat_pid_t *cv = dcstat_vector; int fd = dcstat_maps[NETDATA_DCSTAT_PID_STATS].map_fd; @@ -486,7 +514,7 @@ static void ebpf_update_dc_cgroup() continue; } - dcstat_apps_accumulator(cv); + dcstat_apps_accumulator(cv, maps_per_core); memcpy(out, cv, sizeof(netdata_dcstat_pid_t)); } @@ -499,8 +527,10 @@ static void ebpf_update_dc_cgroup() * Read global table * * Read the table with number of calls for all functions + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_dc_read_global_table() +static void ebpf_dc_read_global_table(int maps_per_core) { uint32_t idx; netdata_idx_t *val = dcstat_hash_values; @@ -510,7 +540,7 @@ static void ebpf_dc_read_global_table() for (idx = NETDATA_KEY_DC_REFERENCE; idx < NETDATA_DIRECTORY_CACHE_END; idx++) { if (!bpf_map_lookup_elem(fd, &idx, stored)) { int i; - int end = ebpf_nprocs; + int end = (maps_per_core) ? ebpf_nprocs: 1; netdata_idx_t total = 0; for (i = 0; i < end; i++) total += stored[i]; @@ -974,6 +1004,7 @@ static void dcstat_collector(ebpf_module_t *em) heartbeat_t hb; heartbeat_init(&hb); int counter = update_every - 1; + int maps_per_core = em->maps_per_core; while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, USEC_PER_SEC); @@ -982,13 +1013,13 @@ static void dcstat_collector(ebpf_module_t *em) counter = 0; netdata_apps_integration_flags_t apps = em->apps_charts; - ebpf_dc_read_global_table(); + ebpf_dc_read_global_table(maps_per_core); pthread_mutex_lock(&collect_data_mutex); if (apps) - read_apps_table(); + read_dc_apps_table(maps_per_core); if (cgroups) - ebpf_update_dc_cgroup(); + ebpf_update_dc_cgroup(maps_per_core); pthread_mutex_lock(&lock); @@ -1084,6 +1115,10 @@ static void ebpf_dcstat_allocate_global_vectors(int apps) */ static int ebpf_dcstat_load_bpf(ebpf_module_t *em) { +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(dcstat_maps, em->maps_per_core, running_on_kernel); +#endif + int ret = 0; ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_DC_TARGET_LOOKUP_FAST].mode); if (em->load & EBPF_LOAD_LEGACY) { diff --git a/collectors/ebpf.plugin/ebpf_disk.c b/collectors/ebpf.plugin/ebpf_disk.c index e1a579441..71c972777 100644 --- a/collectors/ebpf.plugin/ebpf_disk.c +++ b/collectors/ebpf.plugin/ebpf_disk.c @@ -14,10 +14,25 @@ struct config disk_config = { .first_section = NULL, static ebpf_local_maps_t disk_maps[] = {{.name = "tbl_disk_iocall", .internal_input = NETDATA_DISK_HISTOGRAM_LENGTH, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, + {.name = "tmp_disk_tp_stat", .internal_input = 8192, .user_input = 8192, + .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, {.name = NULL, .internal_input = 0, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}}; + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; static avl_tree_lock disk_tree; netdata_ebpf_disks_t *disk_list = NULL; @@ -503,11 +518,12 @@ static void ebpf_fill_plot_disks(netdata_ebpf_disks_t *ptr) /** * Read hard disk table * - * @param table file descriptor for table - * * Read the table with number of calls for all functions + * + * @param table file descriptor for table + * @param maps_per_core do I need to read all cores? */ -static void read_hard_disk_tables(int table) +static void read_hard_disk_tables(int table, int maps_per_core) { netdata_idx_t *values = disk_hash_values; block_key_t key = {}; @@ -548,7 +564,7 @@ static void read_hard_disk_tables(int table) uint64_t total = 0; int i; - int end = (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs; + int end = (maps_per_core) ? 1 : ebpf_nprocs; for (i = 0; i < end; i++) { total += values[i]; } @@ -690,6 +706,7 @@ static void disk_collector(ebpf_module_t *em) heartbeat_t hb; heartbeat_init(&hb); int counter = update_every - 1; + int maps_per_core = em->maps_per_core; while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, USEC_PER_SEC); @@ -697,7 +714,7 @@ static void disk_collector(ebpf_module_t *em) continue; counter = 0; - read_hard_disk_tables(disk_maps[NETDATA_DISK_READ].map_fd); + read_hard_disk_tables(disk_maps[NETDATA_DISK_READ].map_fd, maps_per_core); pthread_mutex_lock(&lock); ebpf_remove_pointer_from_plot_disk(em); ebpf_latency_send_hd_data(update_every); @@ -774,6 +791,9 @@ void *ebpf_disk_thread(void *ptr) goto enddisk; } +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(disk_maps, em->maps_per_core, running_on_kernel); +#endif em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects); if (!em->probe_links) { goto enddisk; diff --git a/collectors/ebpf.plugin/ebpf_disk.h b/collectors/ebpf.plugin/ebpf_disk.h index c606d6594..69c705875 100644 --- a/collectors/ebpf.plugin/ebpf_disk.h +++ b/collectors/ebpf.plugin/ebpf_disk.h @@ -55,7 +55,8 @@ typedef struct netdata_ebpf_disks { } netdata_ebpf_disks_t; enum ebpf_disk_tables { - NETDATA_DISK_READ + NETDATA_DISK_READ, + NETDATA_DISK_TMP }; typedef struct block_key { diff --git a/collectors/ebpf.plugin/ebpf_fd.c b/collectors/ebpf.plugin/ebpf_fd.c index 96da91b0a..6d3868952 100644 --- a/collectors/ebpf.plugin/ebpf_fd.c +++ b/collectors/ebpf.plugin/ebpf_fd.c @@ -15,17 +15,33 @@ static netdata_publish_syscall_t fd_publish_aggregated[NETDATA_FD_SYSCALL_END]; static ebpf_local_maps_t fd_maps[] = {{.name = "tbl_fd_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE, .user_input = 0, .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, {.name = "tbl_fd_global", .internal_input = NETDATA_KEY_END_VECTOR, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, {.name = "fd_ctrl", .internal_input = NETDATA_CONTROLLER_END, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, {.name = NULL, .internal_input = 0, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}}; + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; struct config fd_config = { .first_section = NULL, .last_section = NULL, .mutex = NETDATA_MUTEX_INITIALIZER, @@ -271,10 +287,14 @@ static void ebpf_fd_set_hash_tables(struct fd_bpf *obj) * @param obj is the main structure for bpf objects. * @param em structure with configuration */ -static void ebpf_fd_adjust_map_size(struct fd_bpf *obj, ebpf_module_t *em) +static void ebpf_fd_adjust_map(struct fd_bpf *obj, ebpf_module_t *em) { ebpf_update_map_size(obj->maps.tbl_fd_pid, &fd_maps[NETDATA_FD_PID_STATS], em, bpf_map__name(obj->maps.tbl_fd_pid)); + + ebpf_update_map_type(obj->maps.tbl_fd_global, &fd_maps[NETDATA_FD_GLOBAL_STATS]); + ebpf_update_map_type(obj->maps.tbl_fd_pid, &fd_maps[NETDATA_FD_PID_STATS]); + ebpf_update_map_type(obj->maps.fd_ctrl, &fd_maps[NETDATA_FD_CONTROLLER]); } /** @@ -322,7 +342,7 @@ static inline int ebpf_fd_load_and_attach(struct fd_bpf *obj, ebpf_module_t *em) ebpf_disable_specific_probes(obj); } - ebpf_fd_adjust_map_size(obj, em); + ebpf_fd_adjust_map(obj, em); if (!em->apps_charts && !em->cgroup_charts) ebpf_fd_disable_release_task(obj); @@ -415,8 +435,10 @@ static void ebpf_fd_send_data(ebpf_module_t *em) * Read global counter * * Read the table with number of calls for all functions + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_fd_read_global_table() +static void ebpf_fd_read_global_table(int maps_per_core) { uint32_t idx; netdata_idx_t *val = fd_hash_values; @@ -426,7 +448,7 @@ static void ebpf_fd_read_global_table() for (idx = NETDATA_KEY_CALLS_DO_SYS_OPEN; idx < NETDATA_FD_COUNTER; idx++) { if (!bpf_map_lookup_elem(fd, &idx, stored)) { int i; - int end = ebpf_nprocs; + int end = (maps_per_core) ? ebpf_nprocs: 1; netdata_idx_t total = 0; for (i = 0; i < end; i++) total += stored[i]; @@ -442,10 +464,11 @@ static void ebpf_fd_read_global_table() * Sum all values read from kernel and store in the first address. * * @param out the vector with read values. + * @param maps_per_core do I need to read all cores? */ -static void fd_apps_accumulator(netdata_fd_stat_t *out) +static void fd_apps_accumulator(netdata_fd_stat_t *out, int maps_per_core) { - int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1; + int i, end = (maps_per_core) ? ebpf_nprocs : 1; netdata_fd_stat_t *total = &out[0]; for (i = 1; i < end; i++) { netdata_fd_stat_t *w = &out[i]; @@ -479,14 +502,19 @@ static void fd_fill_pid(uint32_t current_pid, netdata_fd_stat_t *publish) * Read APPS table * * Read the apps table and store data inside the structure. + * + * @param maps_per_core do I need to read all cores? */ -static void read_apps_table() +static void read_fd_apps_table(int maps_per_core) { netdata_fd_stat_t *fv = fd_vector; uint32_t key; struct ebpf_pid_stat *pids = ebpf_root_of_pids; int fd = fd_maps[NETDATA_FD_PID_STATS].map_fd; - size_t length = sizeof(netdata_fd_stat_t) * ebpf_nprocs; + size_t length = sizeof(netdata_fd_stat_t); + if (maps_per_core) + length *= ebpf_nprocs; + while (pids) { key = pids->pid; @@ -495,7 +523,7 @@ static void read_apps_table() continue; } - fd_apps_accumulator(fv); + fd_apps_accumulator(fv, maps_per_core); fd_fill_pid(key, fv); @@ -509,9 +537,11 @@ static void read_apps_table() /** * Update cgroup * - * Update cgroup data based in + * Update cgroup data collected per PID. + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_update_fd_cgroup() +static void ebpf_update_fd_cgroup(int maps_per_core) { ebpf_cgroup_target_t *ect ; netdata_fd_stat_t *fv = fd_vector; @@ -531,7 +561,7 @@ static void ebpf_update_fd_cgroup() } else { memset(fv, 0, length); if (!bpf_map_lookup_elem(fd, &pid, fv)) { - fd_apps_accumulator(fv); + fd_apps_accumulator(fv, maps_per_core); memcpy(out, fv, sizeof(netdata_fd_stat_t)); } @@ -915,6 +945,7 @@ static void fd_collector(ebpf_module_t *em) heartbeat_init(&hb); int update_every = em->update_every; int counter = update_every - 1; + int maps_per_core = em->maps_per_core; while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, USEC_PER_SEC); @@ -923,21 +954,21 @@ static void fd_collector(ebpf_module_t *em) counter = 0; netdata_apps_integration_flags_t apps = em->apps_charts; - ebpf_fd_read_global_table(); + ebpf_fd_read_global_table(maps_per_core); pthread_mutex_lock(&collect_data_mutex); if (apps) - read_apps_table(); + read_fd_apps_table(maps_per_core); + + if (cgroups) + ebpf_update_fd_cgroup(maps_per_core); + + pthread_mutex_lock(&lock); #ifdef NETDATA_DEV_MODE if (ebpf_aral_fd_pid) ebpf_send_data_aral_chart(ebpf_aral_fd_pid, em); #endif - if (cgroups) - ebpf_update_fd_cgroup(); - - pthread_mutex_lock(&lock); - ebpf_fd_send_data(em); if (apps & NETDATA_EBPF_APPS_FLAG_CHART_CREATED) @@ -1082,6 +1113,10 @@ static void ebpf_fd_allocate_global_vectors(int apps) */ static int ebpf_fd_load_bpf(ebpf_module_t *em) { +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(fd_maps, em->maps_per_core, running_on_kernel); +#endif + int ret = 0; ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_FD_SYSCALL_OPEN].mode); if (em->load & EBPF_LOAD_LEGACY) { diff --git a/collectors/ebpf.plugin/ebpf_filesystem.c b/collectors/ebpf.plugin/ebpf_filesystem.c index f8b28195c..63f592eb9 100644 --- a/collectors/ebpf.plugin/ebpf_filesystem.c +++ b/collectors/ebpf.plugin/ebpf_filesystem.c @@ -8,27 +8,122 @@ struct config fs_config = { .first_section = NULL, .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare }, .rwlock = AVL_LOCK_INITIALIZER } }; -static ebpf_local_maps_t fs_maps[] = {{.name = "tbl_ext4", .internal_input = NETDATA_KEY_CALLS_SYNC, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "tbl_xfs", .internal_input = NETDATA_KEY_CALLS_SYNC, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "tbl_nfs", .internal_input = NETDATA_KEY_CALLS_SYNC, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "tbl_zfs", .internal_input = NETDATA_KEY_CALLS_SYNC, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "tbl_btrfs", .internal_input = NETDATA_KEY_CALLS_SYNC, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "tbl_ext_addr", .internal_input = 1, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = NULL, .internal_input = 0, .user_input = 0, - .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}}; +ebpf_local_maps_t ext4_maps[] = {{.name = "tbl_ext4", .internal_input = NETDATA_KEY_CALLS_SYNC, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = "tmp_ext4", .internal_input = 4192, .user_input = 4192, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }}; + +ebpf_local_maps_t xfs_maps[] = {{.name = "tbl_xfs", .internal_input = NETDATA_KEY_CALLS_SYNC, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = "tmp_xfs", .internal_input = 4192, .user_input = 4192, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }}; + +ebpf_local_maps_t nfs_maps[] = {{.name = "tbl_nfs", .internal_input = NETDATA_KEY_CALLS_SYNC, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = "tmp_nfs", .internal_input = 4192, .user_input = 4192, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }}; + +ebpf_local_maps_t zfs_maps[] = {{.name = "tbl_zfs", .internal_input = NETDATA_KEY_CALLS_SYNC, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = "tmp_zfs", .internal_input = 4192, .user_input = 4192, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }}; + +ebpf_local_maps_t btrfs_maps[] = {{.name = "tbl_btrfs", .internal_input = NETDATA_KEY_CALLS_SYNC, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = "tbl_ext_addr", .internal_input = 1, .user_input = 1, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, + {.name = "tmp_btrfs", .internal_input = 4192, .user_input = 4192, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }}; static netdata_syscall_stat_t filesystem_aggregated_data[NETDATA_EBPF_HIST_MAX_BINS]; static netdata_publish_syscall_t filesystem_publish_aggregated[NETDATA_EBPF_HIST_MAX_BINS]; @@ -176,26 +271,32 @@ int ebpf_filesystem_initialize_ebpf_data(ebpf_module_t *em) if (!efp->probe_links && efp->flags & NETDATA_FILESYSTEM_LOAD_EBPF_PROGRAM) { em->thread_name = efp->filesystem; em->kernels = efp->kernels; + em->maps = efp->fs_maps; +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(em->maps, em->maps_per_core, running_on_kernel); +#endif efp->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &efp->objects); if (!efp->probe_links) { em->thread_name = saved_name; em->kernels = kernels; + em->maps = NULL; return -1; } efp->flags |= NETDATA_FILESYSTEM_FLAG_HAS_PARTITION; pthread_mutex_lock(&lock); - ebpf_update_kernel_memory(&plugin_statistics, &fs_maps[i], EBPF_ACTION_STAT_ADD); + ebpf_update_kernel_memory(&plugin_statistics, efp->fs_maps, EBPF_ACTION_STAT_ADD); pthread_mutex_unlock(&lock); // Nedeed for filesystems like btrfs if ((efp->flags & NETDATA_FILESYSTEM_FILL_ADDRESS_TABLE) && (efp->addresses.function)) { - ebpf_load_addresses(&efp->addresses, fs_maps[i + 1].map_fd); + ebpf_load_addresses(&efp->addresses, efp->fs_maps[NETDATA_ADDR_FS_TABLE].map_fd); } } efp->flags &= ~NETDATA_FILESYSTEM_LOAD_EBPF_PROGRAM; } em->thread_name = saved_name; em->kernels = kernels; + em->maps = NULL; if (!dimensions) { dimensions = ebpf_fill_histogram_dimension(NETDATA_EBPF_HIST_MAX_BINS); @@ -394,11 +495,13 @@ static inline netdata_ebpf_histogram_t *select_hist(ebpf_filesystem_partitions_t /** * Read hard disk table * - * @param table index for the hash table + * @param efp structure with filesystem monitored + * @param fd file descriptor to get data. + * @param maps_per_core do I need to read all cores? * * Read the table with number of calls for all functions */ -static void read_filesystem_table(ebpf_filesystem_partitions_t *efp, int fd) +static void read_filesystem_table(ebpf_filesystem_partitions_t *efp, int fd, int maps_per_core) { netdata_idx_t *values = filesystem_hash_values; uint32_t key; @@ -416,7 +519,7 @@ static void read_filesystem_table(ebpf_filesystem_partitions_t *efp, int fd) uint64_t total = 0; int i; - int end = ebpf_nprocs; + int end = (maps_per_core) ? ebpf_nprocs : 1; for (i = 0; i < end; i++) { total += values[i]; } @@ -430,17 +533,17 @@ static void read_filesystem_table(ebpf_filesystem_partitions_t *efp, int fd) /** * Read hard disk table * - * @param table index for the hash table - * * Read the table with number of calls for all functions + * + * @param maps_per_core do I need to read all cores? */ -static void read_filesystem_tables() +static void read_filesystem_tables(int maps_per_core) { int i; for (i = 0; localfs[i].filesystem; i++) { ebpf_filesystem_partitions_t *efp = &localfs[i]; if (efp->flags & NETDATA_FILESYSTEM_FLAG_HAS_PARTITION) { - read_filesystem_table(efp, fs_maps[i].map_fd); + read_filesystem_table(efp, efp->fs_maps[NETDATA_MAIN_FS_TABLE].map_fd, maps_per_core); } } } @@ -464,7 +567,7 @@ void ebpf_filesystem_read_hash(ebpf_module_t *em) if (em->optional) return; - read_filesystem_tables(); + read_filesystem_tables(em->maps_per_core); } /** @@ -546,6 +649,21 @@ static void ebpf_update_filesystem() } /** + * Set maps + * + * When thread is initialized the variable fs_maps is set as null, + * this function fills the variable before to use. + */ +static void ebpf_set_maps() +{ + localfs[NETDATA_FS_LOCALFS_EXT4].fs_maps = ext4_maps; + localfs[NETDATA_FS_LOCALFS_XFS].fs_maps = xfs_maps; + localfs[NETDATA_FS_LOCALFS_NFS].fs_maps = nfs_maps; + localfs[NETDATA_FS_LOCALFS_ZFS].fs_maps = zfs_maps; + localfs[NETDATA_FS_LOCALFS_BTRFS].fs_maps = btrfs_maps; +} + +/** * Filesystem thread * * Thread used to generate socket charts. @@ -559,7 +677,7 @@ void *ebpf_filesystem_thread(void *ptr) netdata_thread_cleanup_push(ebpf_filesystem_exit, ptr); ebpf_module_t *em = (ebpf_module_t *)ptr; - em->maps = fs_maps; + ebpf_set_maps(); ebpf_update_filesystem(); // Initialize optional as zero, to identify when there are not partitions to monitor diff --git a/collectors/ebpf.plugin/ebpf_filesystem.h b/collectors/ebpf.plugin/ebpf_filesystem.h index cf19b253e..b1126badb 100644 --- a/collectors/ebpf.plugin/ebpf_filesystem.h +++ b/collectors/ebpf.plugin/ebpf_filesystem.h @@ -42,6 +42,16 @@ enum netdata_filesystem_table { NETDATA_ADDR_FS_TABLE }; +enum netdata_filesystem_localfs_idx { + NETDATA_FS_LOCALFS_EXT4, + NETDATA_FS_LOCALFS_XFS, + NETDATA_FS_LOCALFS_NFS, + NETDATA_FS_LOCALFS_ZFS, + NETDATA_FS_LOCALFS_BTRFS, + + NETDATA_FS_LOCALFS_END, +}; + void *ebpf_filesystem_thread(void *ptr); extern struct config fs_config; diff --git a/collectors/ebpf.plugin/ebpf_hardirq.c b/collectors/ebpf.plugin/ebpf_hardirq.c index b4d49dc00..113648ec9 100644 --- a/collectors/ebpf.plugin/ebpf_hardirq.c +++ b/collectors/ebpf.plugin/ebpf_hardirq.c @@ -17,14 +17,20 @@ static ebpf_local_maps_t hardirq_maps[] = { .internal_input = NETDATA_HARDIRQ_MAX_IRQS, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif }, { .name = "tbl_hardirq_static", .internal_input = HARDIRQ_EBPF_STATIC_END, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif }, /* end */ { @@ -32,7 +38,10 @@ static ebpf_local_maps_t hardirq_maps[] = { .internal_input = 0, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif } }; @@ -555,6 +564,9 @@ void *ebpf_hardirq_thread(void *ptr) goto endhardirq; } +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(em->maps, em->maps_per_core, running_on_kernel); +#endif em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects); if (!em->probe_links) { goto endhardirq; diff --git a/collectors/ebpf.plugin/ebpf_mdflush.c b/collectors/ebpf.plugin/ebpf_mdflush.c index fc794e5e5..321bd97ee 100644 --- a/collectors/ebpf.plugin/ebpf_mdflush.c +++ b/collectors/ebpf.plugin/ebpf_mdflush.c @@ -16,7 +16,10 @@ static ebpf_local_maps_t mdflush_maps[] = { .internal_input = 1024, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif }, /* end */ { @@ -87,7 +90,14 @@ static int mdflush_val_cmp(void *a, void *b) } } -static void mdflush_read_count_map() +/** + * Read count map + * + * Read the hash table and store data to allocated vectors. + * + * @param maps_per_core do I need to read all cores? + */ +static void mdflush_read_count_map(int maps_per_core) { int mapfd = mdflush_maps[MDFLUSH_MAP_COUNT].map_fd; mdflush_ebpf_key_t curr_key = (uint32_t)-1; @@ -137,7 +147,7 @@ static void mdflush_read_count_map() // we must add up count value for this record across all CPUs. uint64_t total_cnt = 0; int i; - int end = (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs; + int end = (!maps_per_core) ? 1 : ebpf_nprocs; for (i = 0; i < end; i++) { total_cnt += mdflush_ebpf_vals[i]; } @@ -215,6 +225,7 @@ static void mdflush_collector(ebpf_module_t *em) heartbeat_t hb; heartbeat_init(&hb); int counter = update_every - 1; + int maps_per_core = em->maps_per_core; while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, USEC_PER_SEC); @@ -222,7 +233,8 @@ static void mdflush_collector(ebpf_module_t *em) continue; counter = 0; - mdflush_read_count_map(); + mdflush_read_count_map(maps_per_core); + pthread_mutex_lock(&lock); // write dims now for all hitherto discovered devices. write_begin_chart("mdstat", "mdstat_flush"); avl_traverse_lock(&mdflush_pub, mdflush_write_dims, NULL); @@ -251,6 +263,9 @@ void *ebpf_mdflush_thread(void *ptr) goto endmdflush; } +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(em->maps, em->maps_per_core, running_on_kernel); +#endif em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects); if (!em->probe_links) { goto endmdflush; diff --git a/collectors/ebpf.plugin/ebpf_mount.c b/collectors/ebpf.plugin/ebpf_mount.c index a2a4c5530..e0951f8c4 100644 --- a/collectors/ebpf.plugin/ebpf_mount.c +++ b/collectors/ebpf.plugin/ebpf_mount.c @@ -5,10 +5,18 @@ static ebpf_local_maps_t mount_maps[] = {{.name = "tbl_mount", .internal_input = NETDATA_MOUNT_END, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, {.name = NULL, .internal_input = 0, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}}; + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; static char *mount_dimension_name[NETDATA_EBPF_MOUNT_SYSCALL] = { "mount", "umount" }; static netdata_syscall_stat_t mount_aggregated_data[NETDATA_EBPF_MOUNT_SYSCALL]; @@ -192,6 +200,8 @@ static inline int ebpf_mount_load_and_attach(struct mount_bpf *obj, ebpf_module_ ebpf_mount_disable_trampoline(obj); } + ebpf_update_map_type(obj->maps.tbl_mount, &mount_maps[NETDATA_KEY_MOUNT_TABLE]); + int ret = mount_bpf__load(obj); if (!ret) { if (test != EBPF_LOAD_PROBE && test != EBPF_LOAD_RETPROBE ) @@ -249,8 +259,10 @@ static void ebpf_mount_exit(void *ptr) * Read global table * * Read the table with number of calls for all functions + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_mount_read_global_table() +static void ebpf_mount_read_global_table(int maps_per_core) { static netdata_idx_t *mount_values = NULL; if (!mount_values) @@ -259,17 +271,22 @@ static void ebpf_mount_read_global_table() uint32_t idx; netdata_idx_t *val = mount_hash_values; netdata_idx_t *stored = mount_values; + size_t length = sizeof(netdata_idx_t); + if (maps_per_core) + length *= ebpf_nprocs; + int fd = mount_maps[NETDATA_KEY_MOUNT_TABLE].map_fd; for (idx = NETDATA_KEY_MOUNT_CALL; idx < NETDATA_MOUNT_END; idx++) { if (!bpf_map_lookup_elem(fd, &idx, stored)) { int i; - int end = ebpf_nprocs; + int end = (maps_per_core) ? ebpf_nprocs : 1; netdata_idx_t total = 0; for (i = 0; i < end; i++) total += stored[i]; val[idx] = total; + memset(stored, 0, length); } } } @@ -304,13 +321,14 @@ static void mount_collector(ebpf_module_t *em) heartbeat_init(&hb); int update_every = em->update_every; int counter = update_every - 1; + int maps_per_core = em->maps_per_core; while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, USEC_PER_SEC); if (ebpf_exit_plugin || ++counter != update_every) continue; counter = 0; - ebpf_mount_read_global_table(); + ebpf_mount_read_global_table(maps_per_core); pthread_mutex_lock(&lock); ebpf_mount_send_data(); @@ -372,6 +390,10 @@ static void ebpf_create_mount_charts(int update_every) */ static int ebpf_mount_load_bpf(ebpf_module_t *em) { +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(em->maps, em->maps_per_core, running_on_kernel); +#endif + int ret = 0; if (em->load & EBPF_LOAD_LEGACY) { em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects); diff --git a/collectors/ebpf.plugin/ebpf_oomkill.c b/collectors/ebpf.plugin/ebpf_oomkill.c index 856c922ec..094875292 100644 --- a/collectors/ebpf.plugin/ebpf_oomkill.c +++ b/collectors/ebpf.plugin/ebpf_oomkill.c @@ -16,7 +16,10 @@ static ebpf_local_maps_t oomkill_maps[] = { .internal_input = NETDATA_OOMKILL_MAX_ENTRIES, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif }, /* end */ { @@ -24,7 +27,10 @@ static ebpf_local_maps_t oomkill_maps[] = { .internal_input = 0, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif } }; @@ -285,6 +291,8 @@ static void ebpf_update_oomkill_cgroup(int32_t *keys, uint32_t total) /** * Main loop for this collector. + * + * @param em the thread main structure. */ static void oomkill_collector(ebpf_module_t *em) { @@ -384,6 +392,9 @@ void *ebpf_oomkill_thread(void *ptr) goto endoomkill; } +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(em->maps, em->maps_per_core, running_on_kernel); +#endif em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects); if (!em->probe_links) { goto endoomkill; diff --git a/collectors/ebpf.plugin/ebpf_process.c b/collectors/ebpf.plugin/ebpf_process.c index 66af47857..17a9809d3 100644 --- a/collectors/ebpf.plugin/ebpf_process.c +++ b/collectors/ebpf.plugin/ebpf_process.c @@ -18,17 +18,33 @@ static char *status[] = { "process", "zombie" }; static ebpf_local_maps_t process_maps[] = {{.name = "tbl_pid_stats", .internal_input = ND_EBPF_DEFAULT_PID_SIZE, .user_input = 0, .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, {.name = "tbl_total_stats", .internal_input = NETDATA_KEY_END_VECTOR, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, {.name = "process_ctrl", .internal_input = NETDATA_CONTROLLER_END, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, {.name = NULL, .internal_input = 0, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}}; + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; char *tracepoint_sched_type = { "sched" } ; char *tracepoint_sched_process_exit = { "sched_process_exit" }; @@ -39,6 +55,7 @@ static int was_sched_process_exec_enabled = 0; static int was_sched_process_fork_enabled = 0; static netdata_idx_t *process_hash_values = NULL; +ebpf_process_stat_t *process_stat_vector = NULL; static netdata_syscall_stat_t process_aggregated_data[NETDATA_KEY_PUBLISH_PROCESS_END]; static netdata_publish_syscall_t process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_END]; @@ -55,6 +72,7 @@ static char *threads_stat[NETDATA_EBPF_THREAD_STAT_END] = {"total", "running"}; static char *load_event_stat[NETDATA_EBPF_LOAD_STAT_END] = {"legacy", "co-re"}; static char *memlock_stat = {"memory_locked"}; static char *hash_table_stat = {"hash_table"}; +static char *hash_table_core[NETDATA_EBPF_LOAD_STAT_END] = {"per_core", "unique"}; /***************************************************************** * @@ -251,8 +269,10 @@ void ebpf_process_send_apps_data(struct ebpf_target *root, ebpf_module_t *em) /** * Read the hash table and store data to allocated vectors. + * + * @param maps_per_core do I need to read all cores? */ -static void read_hash_global_tables() +static void ebpf_read_process_hash_global_tables(int maps_per_core) { uint64_t idx; netdata_idx_t res[NETDATA_KEY_END_VECTOR]; @@ -263,7 +283,7 @@ static void read_hash_global_tables() if (!bpf_map_lookup_elem(fd, &idx, val)) { uint64_t total = 0; int i; - int end = ebpf_nprocs; + int end = (maps_per_core) ? ebpf_nprocs : 1; for (i = 0; i < end; i++) total += val[i]; @@ -285,13 +305,18 @@ static void read_hash_global_tables() /** * Update cgroup * - * Update cgroup data based in + * Update cgroup data based in PID running. + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_update_process_cgroup() +static void ebpf_update_process_cgroup(int maps_per_core) { ebpf_cgroup_target_t *ect ; int pid_fd = process_maps[NETDATA_PROCESS_PID_TABLE].map_fd; + size_t length = sizeof(ebpf_process_stat_t); + if (maps_per_core) + length *= ebpf_nprocs; pthread_mutex_lock(&mutex_cgroup_shm); for (ect = ebpf_cgroup_pids; ect; ect = ect->next) { struct pid_on_target2 *pids; @@ -303,9 +328,15 @@ static void ebpf_update_process_cgroup() memcpy(out, in, sizeof(ebpf_process_stat_t)); } else { - if (bpf_map_lookup_elem(pid_fd, &pid, out)) { + if (bpf_map_lookup_elem(pid_fd, &pid, process_stat_vector)) { memset(out, 0, sizeof(ebpf_process_stat_t)); } + + ebpf_process_apps_accumulator(process_stat_vector, maps_per_core); + + memcpy(out, process_stat_vector, sizeof(ebpf_process_stat_t)); + + memset(process_stat_vector, 0, length); } } } @@ -507,6 +538,35 @@ static inline void ebpf_create_statistic_hash_tables(ebpf_module_t *em) } /** + * Create chart for percpu stats + * + * Write to standard output current values for threads. + * + * @param em a pointer to the structure with the default values. + */ +static inline void ebpf_create_statistic_hash_per_core(ebpf_module_t *em) +{ + ebpf_write_chart_cmd(NETDATA_MONITORING_FAMILY, + NETDATA_EBPF_HASH_TABLES_PER_CORE, + "How threads are loading hash/array tables.", + "threads", + NETDATA_EBPF_FAMILY, + NETDATA_EBPF_CHART_TYPE_LINE, + NULL, + 140004, + em->update_every, + NETDATA_EBPF_MODULE_NAME_PROCESS); + + ebpf_write_global_dimension(hash_table_core[NETDATA_EBPF_THREAD_PER_CORE], + hash_table_core[NETDATA_EBPF_THREAD_PER_CORE], + ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX]); + + ebpf_write_global_dimension(hash_table_core[NETDATA_EBPF_THREAD_UNIQUE], + hash_table_core[NETDATA_EBPF_THREAD_UNIQUE], + ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX]); +} + +/** * Update Internal Metric variable * * By default eBPF.plugin sends internal metrics for netdata, but user can @@ -541,6 +601,8 @@ static void ebpf_create_statistic_charts(ebpf_module_t *em) ebpf_create_statistic_kernel_memory(em); ebpf_create_statistic_hash_tables(em); + + ebpf_create_statistic_hash_per_core(em); } /** @@ -647,6 +709,7 @@ static void ebpf_process_exit(void *ptr) ebpf_module_t *em = (ebpf_module_t *)ptr; freez(process_hash_values); + freez(process_stat_vector); ebpf_process_disable_tracepoints(); @@ -1010,6 +1073,11 @@ void ebpf_send_statistic_data() write_begin_chart(NETDATA_MONITORING_FAMILY, NETDATA_EBPF_HASH_TABLES_LOADED); write_chart_dimension(hash_table_stat, (long long)plugin_statistics.hash_tables); write_end_chart(); + + write_begin_chart(NETDATA_MONITORING_FAMILY, NETDATA_EBPF_HASH_TABLES_PER_CORE); + write_chart_dimension(hash_table_core[NETDATA_EBPF_THREAD_PER_CORE], (long long)plugin_statistics.hash_percpu); + write_chart_dimension(hash_table_core[NETDATA_EBPF_THREAD_UNIQUE], (long long)plugin_statistics.hash_unique); + write_end_chart(); } /** @@ -1032,6 +1100,7 @@ static void process_collector(ebpf_module_t *em) int update_every = em->update_every; int counter = update_every - 1; + int maps_per_core = em->maps_per_core; while (!ebpf_exit_plugin) { usec_t dt = heartbeat_next(&hb, USEC_PER_SEC); (void)dt; @@ -1041,14 +1110,14 @@ static void process_collector(ebpf_module_t *em) if (++counter == update_every) { counter = 0; - read_hash_global_tables(); + ebpf_read_process_hash_global_tables(maps_per_core); netdata_apps_integration_flags_t apps_enabled = em->apps_charts; pthread_mutex_lock(&collect_data_mutex); if (ebpf_all_pids_count > 0) { if (cgroups && shm_ebpf_cgroup.header) { - ebpf_update_process_cgroup(); + ebpf_update_process_cgroup(maps_per_core); } } @@ -1099,6 +1168,7 @@ static void ebpf_process_allocate_global_vectors(size_t length) memset(process_aggregated_data, 0, length * sizeof(netdata_syscall_stat_t)); memset(process_publish_aggregated, 0, length * sizeof(netdata_publish_syscall_t)); process_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t)); + process_stat_vector = callocz(ebpf_nprocs, sizeof(ebpf_process_stat_t)); global_process_stats = callocz((size_t)pid_max, sizeof(ebpf_process_stat_t *)); } @@ -1195,8 +1265,7 @@ void *ebpf_process_thread(void *ptr) set_local_pointers(); em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects); if (!em->probe_links) { - pthread_mutex_unlock(&lock); - goto endprocess; + em->enabled = em->global_charts = em->apps_charts = em->cgroup_charts = NETDATA_THREAD_EBPF_STOPPING; } int algorithms[NETDATA_KEY_PUBLISH_PROCESS_END] = { @@ -1225,7 +1294,6 @@ void *ebpf_process_thread(void *ptr) process_collector(em); -endprocess: pthread_mutex_lock(&ebpf_exit_cleanup); if (em->enabled == NETDATA_THREAD_EBPF_RUNNING) ebpf_update_disabled_plugin_stats(em); diff --git a/collectors/ebpf.plugin/ebpf_process.h b/collectors/ebpf.plugin/ebpf_process.h index 5f119aea1..bccdc0eb5 100644 --- a/collectors/ebpf.plugin/ebpf_process.h +++ b/collectors/ebpf.plugin/ebpf_process.h @@ -56,6 +56,13 @@ enum netdata_ebpf_load_mode_stats{ NETDATA_EBPF_LOAD_STAT_END }; +enum netdata_ebpf_thread_per_core{ + NETDATA_EBPF_THREAD_PER_CORE, + NETDATA_EBPF_THREAD_UNIQUE, + + NETDATA_EBPF_PER_CORE_END +}; + // Index from kernel typedef enum ebpf_process_index { NETDATA_KEY_CALLS_DO_EXIT, diff --git a/collectors/ebpf.plugin/ebpf_shm.c b/collectors/ebpf.plugin/ebpf_shm.c index f81c01964..093d65b60 100644 --- a/collectors/ebpf.plugin/ebpf_shm.c +++ b/collectors/ebpf.plugin/ebpf_shm.c @@ -21,15 +21,27 @@ struct config shm_config = { .first_section = NULL, static ebpf_local_maps_t shm_maps[] = {{.name = "tbl_pid_shm", .internal_input = ND_EBPF_DEFAULT_PID_SIZE, .user_input = 0, .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, {.name = "shm_ctrl", .internal_input = NETDATA_CONTROLLER_END, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, {.name = "tbl_shm", .internal_input = NETDATA_SHM_END, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, {.name = NULL, .internal_input = 0, .user_input = 0}}; netdata_ebpf_targets_t shm_targets[] = { {.name = "shmget", .mode = EBPF_LOAD_TRAMPOLINE}, @@ -215,10 +227,14 @@ static void ebpf_shm_disable_release_task(struct shm_bpf *obj) * @param obj is the main structure for bpf objects. * @param em structure with configuration */ -static void ebpf_shm_adjust_map_size(struct shm_bpf *obj, ebpf_module_t *em) +static void ebpf_shm_adjust_map(struct shm_bpf *obj, ebpf_module_t *em) { ebpf_update_map_size(obj->maps.tbl_pid_shm, &shm_maps[NETDATA_PID_SHM_TABLE], em, bpf_map__name(obj->maps.tbl_pid_shm)); + + ebpf_update_map_type(obj->maps.tbl_shm, &shm_maps[NETDATA_SHM_GLOBAL_TABLE]); + ebpf_update_map_type(obj->maps.tbl_pid_shm, &shm_maps[NETDATA_PID_SHM_TABLE]); + ebpf_update_map_type(obj->maps.shm_ctrl, &shm_maps[NETDATA_SHM_CONTROLLER]); } /** @@ -250,7 +266,7 @@ static inline int ebpf_shm_load_and_attach(struct shm_bpf *obj, ebpf_module_t *e ebpf_disable_trampoline(obj); } - ebpf_shm_adjust_map_size(obj, em); + ebpf_shm_adjust_map(obj, em); if (!em->apps_charts && !em->cgroup_charts) ebpf_shm_disable_release_task(obj); @@ -312,10 +328,11 @@ static void ebpf_shm_exit(void *ptr) * Sum all values read from kernel and store in the first address. * * @param out the vector with read values. + * @param maps_per_core do I need to read all cores? */ -static void shm_apps_accumulator(netdata_publish_shm_t *out) +static void shm_apps_accumulator(netdata_publish_shm_t *out, int maps_per_core) { - int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1; + int i, end = (maps_per_core) ? ebpf_nprocs : 1; netdata_publish_shm_t *total = &out[0]; for (i = 1; i < end; i++) { netdata_publish_shm_t *w = &out[i]; @@ -349,12 +366,17 @@ static void shm_fill_pid(uint32_t current_pid, netdata_publish_shm_t *publish) * Update cgroup * * Update cgroup data based in + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_update_shm_cgroup() +static void ebpf_update_shm_cgroup(int maps_per_core) { netdata_publish_shm_t *cv = shm_vector; int fd = shm_maps[NETDATA_PID_SHM_TABLE].map_fd; - size_t length = sizeof(netdata_publish_shm_t) * ebpf_nprocs; + size_t length = sizeof(netdata_publish_shm_t); + if (maps_per_core) + length *= ebpf_nprocs; + ebpf_cgroup_target_t *ect; memset(cv, 0, length); @@ -371,7 +393,7 @@ static void ebpf_update_shm_cgroup() memcpy(out, in, sizeof(netdata_publish_shm_t)); } else { if (!bpf_map_lookup_elem(fd, &pid, cv)) { - shm_apps_accumulator(cv); + shm_apps_accumulator(cv, maps_per_core); memcpy(out, cv, sizeof(netdata_publish_shm_t)); @@ -389,14 +411,19 @@ static void ebpf_update_shm_cgroup() * Read APPS table * * Read the apps table and store data inside the structure. + * + * @param maps_per_core do I need to read all cores? */ -static void read_apps_table() +static void read_shm_apps_table(int maps_per_core) { netdata_publish_shm_t *cv = shm_vector; uint32_t key; struct ebpf_pid_stat *pids = ebpf_root_of_pids; int fd = shm_maps[NETDATA_PID_SHM_TABLE].map_fd; - size_t length = sizeof(netdata_publish_shm_t)*ebpf_nprocs; + size_t length = sizeof(netdata_publish_shm_t); + if (maps_per_core) + length *= ebpf_nprocs; + while (pids) { key = pids->pid; @@ -405,7 +432,7 @@ static void read_apps_table() continue; } - shm_apps_accumulator(cv); + shm_apps_accumulator(cv, maps_per_core); shm_fill_pid(key, cv); @@ -446,23 +473,29 @@ static void shm_send_global() * Read global counter * * Read the table with number of calls for all functions + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_shm_read_global_table() +static void ebpf_shm_read_global_table(int maps_per_core) { netdata_idx_t *stored = shm_values; netdata_idx_t *val = shm_hash_values; int fd = shm_maps[NETDATA_SHM_GLOBAL_TABLE].map_fd; + size_t length = sizeof(netdata_idx_t); + if (maps_per_core) + length *= ebpf_nprocs; uint32_t i, end = NETDATA_SHM_END; for (i = NETDATA_KEY_SHMGET_CALL; i < end; i++) { if (!bpf_map_lookup_elem(fd, &i, stored)) { int j; - int last = ebpf_nprocs; + int last = (maps_per_core) ? ebpf_nprocs : 1; netdata_idx_t total = 0; for (j = 0; j < last; j++) total += stored[j]; val[i] = total; + memset(stored, 0 , length); } } } @@ -831,6 +864,7 @@ static void shm_collector(ebpf_module_t *em) heartbeat_t hb; heartbeat_init(&hb); int counter = update_every - 1; + int maps_per_core = em->maps_per_core; while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, USEC_PER_SEC); if (ebpf_exit_plugin || ++counter != update_every) @@ -838,14 +872,14 @@ static void shm_collector(ebpf_module_t *em) counter = 0; netdata_apps_integration_flags_t apps = em->apps_charts; - ebpf_shm_read_global_table(); + ebpf_shm_read_global_table(maps_per_core); pthread_mutex_lock(&collect_data_mutex); if (apps) { - read_apps_table(); + read_shm_apps_table(maps_per_core); } if (cgroups) { - ebpf_update_shm_cgroup(); + ebpf_update_shm_cgroup(maps_per_core); } pthread_mutex_lock(&lock); @@ -984,6 +1018,10 @@ static void ebpf_create_shm_charts(int update_every) */ static int ebpf_shm_load_bpf(ebpf_module_t *em) { +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(em->maps, em->maps_per_core, running_on_kernel); +#endif + int ret = 0; ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_KEY_SHMGET_CALL].mode); diff --git a/collectors/ebpf.plugin/ebpf_socket.c b/collectors/ebpf.plugin/ebpf_socket.c index aebc9ca12..b45dec7d9 100644 --- a/collectors/ebpf.plugin/ebpf_socket.c +++ b/collectors/ebpf.plugin/ebpf_socket.c @@ -27,35 +27,67 @@ static ebpf_local_maps_t socket_maps[] = {{.name = "tbl_bandwidth", .internal_input = NETDATA_COMPILED_CONNECTIONS_ALLOWED, .user_input = NETDATA_MAXIMUM_CONNECTIONS_ALLOWED, .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, {.name = "tbl_global_sock", .internal_input = NETDATA_SOCKET_COUNTER, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, {.name = "tbl_lports", .internal_input = NETDATA_SOCKET_COUNTER, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, {.name = "tbl_conn_ipv4", .internal_input = NETDATA_COMPILED_CONNECTIONS_ALLOWED, .user_input = NETDATA_MAXIMUM_CONNECTIONS_ALLOWED, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, {.name = "tbl_conn_ipv6", .internal_input = NETDATA_COMPILED_CONNECTIONS_ALLOWED, .user_input = NETDATA_MAXIMUM_CONNECTIONS_ALLOWED, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, {.name = "tbl_nv_udp", .internal_input = NETDATA_COMPILED_UDP_CONNECTIONS_ALLOWED, .user_input = NETDATA_MAXIMUM_UDP_CONNECTIONS_ALLOWED, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, {.name = "socket_ctrl", .internal_input = NETDATA_CONTROLLER_END, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = NULL, .internal_input = 0, .user_input = 0}}; + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; static netdata_idx_t *socket_hash_values = NULL; static netdata_syscall_stat_t socket_aggregated_data[NETDATA_MAX_SOCKET_VECTOR]; @@ -362,7 +394,7 @@ static void ebpf_socket_set_hash_tables(struct socket_bpf *obj) * @param obj is the main structure for bpf objects. * @param em structure with configuration */ -static void ebpf_socket_adjust_map_size(struct socket_bpf *obj, ebpf_module_t *em) +static void ebpf_socket_adjust_map(struct socket_bpf *obj, ebpf_module_t *em) { ebpf_update_map_size(obj->maps.tbl_bandwidth, &socket_maps[NETDATA_SOCKET_TABLE_BANDWIDTH], em, bpf_map__name(obj->maps.tbl_bandwidth)); @@ -375,6 +407,15 @@ static void ebpf_socket_adjust_map_size(struct socket_bpf *obj, ebpf_module_t *e ebpf_update_map_size(obj->maps.tbl_nv_udp, &socket_maps[NETDATA_SOCKET_TABLE_UDP], em, bpf_map__name(obj->maps.tbl_nv_udp)); + + + ebpf_update_map_type(obj->maps.tbl_bandwidth, &socket_maps[NETDATA_SOCKET_TABLE_BANDWIDTH]); + ebpf_update_map_type(obj->maps.tbl_conn_ipv4, &socket_maps[NETDATA_SOCKET_TABLE_IPV4]); + ebpf_update_map_type(obj->maps.tbl_conn_ipv6, &socket_maps[NETDATA_SOCKET_TABLE_IPV6]); + ebpf_update_map_type(obj->maps.tbl_nv_udp, &socket_maps[NETDATA_SOCKET_TABLE_UDP]); + ebpf_update_map_type(obj->maps.socket_ctrl, &socket_maps[NETDATA_SOCKET_TABLE_CTRL]); + ebpf_update_map_type(obj->maps.tbl_global_sock, &socket_maps[NETDATA_SOCKET_GLOBAL]); + ebpf_update_map_type(obj->maps.tbl_lports, &socket_maps[NETDATA_SOCKET_LPORTS]); } /** @@ -403,14 +444,14 @@ static inline int ebpf_socket_load_and_attach(struct socket_bpf *obj, ebpf_modul ebpf_socket_disable_specific_probe(obj, em->mode); } + ebpf_socket_adjust_map(obj, em); + int ret = socket_bpf__load(obj); if (ret) { fprintf(stderr, "failed to load BPF object: %d\n", ret); return ret; } - ebpf_socket_adjust_map_size(obj, em); - if (test == EBPF_LOAD_TRAMPOLINE) { ret = socket_bpf__attach(obj); } else { @@ -1988,17 +2029,23 @@ static void hash_accumulator(netdata_socket_t *values, netdata_socket_idx_t *key * * @param fd the hash table with data. * @param family the family associated to the hash table + * @param maps_per_core do I need to read all cores? * * @return it returns 0 on success and -1 otherwise. */ -static void ebpf_read_socket_hash_table(int fd, int family) +static void ebpf_read_socket_hash_table(int fd, int family, int maps_per_core) { netdata_socket_idx_t key = {}; netdata_socket_idx_t next_key = {}; netdata_socket_t *values = socket_values; - size_t length = ebpf_nprocs*sizeof(netdata_socket_t); - int test, end = (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs; + size_t length = sizeof(netdata_socket_t); + int test, end; + if (maps_per_core) { + length *= ebpf_nprocs; + end = ebpf_nprocs; + } else + end = 1; while (bpf_map_get_next_key(fd, &key, &next_key) == 0) { // We need to reset the values when we are working on kernel 4.15 or newer, because kernel does not create @@ -2122,11 +2169,13 @@ static void read_listen_table() void *ebpf_socket_read_hash(void *ptr) { netdata_thread_cleanup_push(ebpf_socket_cleanup, ptr); + ebpf_module_t *em = (ebpf_module_t *)ptr; heartbeat_t hb; heartbeat_init(&hb); int fd_ipv4 = socket_maps[NETDATA_SOCKET_TABLE_IPV4].map_fd; int fd_ipv6 = socket_maps[NETDATA_SOCKET_TABLE_IPV6].map_fd; + int maps_per_core = em->maps_per_core; // This thread is cancelled from another thread for (;;) { (void)heartbeat_next(&hb, USEC_PER_SEC); @@ -2134,8 +2183,8 @@ void *ebpf_socket_read_hash(void *ptr) break; pthread_mutex_lock(&nv_mutex); - ebpf_read_socket_hash_table(fd_ipv4, AF_INET); - ebpf_read_socket_hash_table(fd_ipv6, AF_INET6); + ebpf_read_socket_hash_table(fd_ipv4, AF_INET, maps_per_core); + ebpf_read_socket_hash_table(fd_ipv6, AF_INET6, maps_per_core); pthread_mutex_unlock(&nv_mutex); } @@ -2145,23 +2194,30 @@ void *ebpf_socket_read_hash(void *ptr) /** * Read the hash table and store data to allocated vectors. + * + * @param maps_per_core do I need to read all cores? */ -static void read_hash_global_tables() +static void read_hash_global_tables(int maps_per_core) { uint64_t idx; netdata_idx_t res[NETDATA_SOCKET_COUNTER]; netdata_idx_t *val = socket_hash_values; + size_t length = sizeof(netdata_idx_t); + if (maps_per_core) + length *= ebpf_nprocs; + int fd = socket_maps[NETDATA_SOCKET_GLOBAL].map_fd; for (idx = 0; idx < NETDATA_SOCKET_COUNTER; idx++) { if (!bpf_map_lookup_elem(fd, &idx, val)) { uint64_t total = 0; int i; - int end = ebpf_nprocs; + int end = (maps_per_core) ? ebpf_nprocs : 1; for (i = 0; i < end; i++) total += val[i]; res[idx] = total; + memset(socket_hash_values, 0, length); } else { res[idx] = 0; } @@ -2220,9 +2276,9 @@ void ebpf_socket_fill_publish_apps(uint32_t current_pid, ebpf_bandwidth_t *eb) * * @param out the vector with the values to sum */ -void ebpf_socket_bandwidth_accumulator(ebpf_bandwidth_t *out) +void ebpf_socket_bandwidth_accumulator(ebpf_bandwidth_t *out, int maps_per_core) { - int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1; + int i, end = (maps_per_core) ? ebpf_nprocs : 1; ebpf_bandwidth_t *total = &out[0]; for (i = 1; i < end; i++) { ebpf_bandwidth_t *move = &out[i]; @@ -2241,13 +2297,18 @@ void ebpf_socket_bandwidth_accumulator(ebpf_bandwidth_t *out) /** * Update the apps data reading information from the hash table + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_socket_update_apps_data() +static void ebpf_socket_update_apps_data(int maps_per_core) { int fd = socket_maps[NETDATA_SOCKET_TABLE_BANDWIDTH].map_fd; ebpf_bandwidth_t *eb = bandwidth_vector; uint32_t key; struct ebpf_pid_stat *pids = ebpf_root_of_pids; + size_t length = sizeof(ebpf_bandwidth_t); + if (maps_per_core) + length *= ebpf_nprocs; while (pids) { key = pids->pid; @@ -2256,10 +2317,12 @@ static void ebpf_socket_update_apps_data() continue; } - ebpf_socket_bandwidth_accumulator(eb); + ebpf_socket_bandwidth_accumulator(eb, maps_per_core); ebpf_socket_fill_publish_apps(key, eb); + memset(eb, 0, length); + pids = pids->next; } } @@ -2267,15 +2330,21 @@ static void ebpf_socket_update_apps_data() /** * Update cgroup * - * Update cgroup data based in + * Update cgroup data based in PIDs. + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_update_socket_cgroup() +static void ebpf_update_socket_cgroup(int maps_per_core) { ebpf_cgroup_target_t *ect ; ebpf_bandwidth_t *eb = bandwidth_vector; int fd = socket_maps[NETDATA_SOCKET_TABLE_BANDWIDTH].map_fd; + size_t length = sizeof(ebpf_bandwidth_t); + if (maps_per_core) + length *= ebpf_nprocs; + pthread_mutex_lock(&mutex_cgroup_shm); for (ect = ebpf_cgroup_pids; ect; ect = ect->next) { struct pid_on_target2 *pids; @@ -2298,7 +2367,7 @@ static void ebpf_update_socket_cgroup() publish->call_tcp_v6_connection = in->call_tcp_v6_connection; } else { if (!bpf_map_lookup_elem(fd, &pid, eb)) { - ebpf_socket_bandwidth_accumulator(eb); + ebpf_socket_bandwidth_accumulator(eb, maps_per_core); memcpy(out, eb, sizeof(ebpf_bandwidth_t)); @@ -2312,6 +2381,8 @@ static void ebpf_update_socket_cgroup() publish->call_close = out->close; publish->call_tcp_v4_connection = out->tcp_v4_connection; publish->call_tcp_v6_connection = out->tcp_v6_connection; + + memset(eb, 0, length); } } } @@ -2845,6 +2916,7 @@ static void socket_collector(ebpf_module_t *em) int socket_global_enabled = em->global_charts; int update_every = em->update_every; + int maps_per_core = em->maps_per_core; int counter = update_every - 1; while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, USEC_PER_SEC); @@ -2855,15 +2927,15 @@ static void socket_collector(ebpf_module_t *em) netdata_apps_integration_flags_t socket_apps_enabled = em->apps_charts; if (socket_global_enabled) { read_listen_table(); - read_hash_global_tables(); + read_hash_global_tables(maps_per_core); } pthread_mutex_lock(&collect_data_mutex); if (socket_apps_enabled) - ebpf_socket_update_apps_data(); + ebpf_socket_update_apps_data(maps_per_core); if (cgroups) - ebpf_update_socket_cgroup(); + ebpf_update_socket_cgroup(maps_per_core); if (network_connection) calculate_nv_plot(); @@ -3855,6 +3927,10 @@ void parse_table_size_options(struct config *cfg) */ static int ebpf_socket_load_bpf(ebpf_module_t *em) { +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(em->maps, em->maps_per_core, running_on_kernel); +#endif + int ret = 0; if (em->load & EBPF_LOAD_LEGACY) { diff --git a/collectors/ebpf.plugin/ebpf_softirq.c b/collectors/ebpf.plugin/ebpf_softirq.c index 33abbdf5e..01e2d0a52 100644 --- a/collectors/ebpf.plugin/ebpf_softirq.c +++ b/collectors/ebpf.plugin/ebpf_softirq.c @@ -16,7 +16,10 @@ static ebpf_local_maps_t softirq_maps[] = { .internal_input = NETDATA_SOFTIRQ_MAX_IRQS, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif }, /* end */ { @@ -24,7 +27,10 @@ static ebpf_local_maps_t softirq_maps[] = { .internal_input = 0, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif } }; @@ -94,10 +100,21 @@ static void softirq_cleanup(void *ptr) * MAIN LOOP *****************************************************************/ -static void softirq_read_latency_map() +/** + * Read Latency Map + * + * Read data from kernel ring to plot for users. + * + * @param maps_per_core do I need to read all cores? + */ +static void softirq_read_latency_map(int maps_per_core) { int fd = softirq_maps[SOFTIRQ_MAP_LATENCY].map_fd; int i; + size_t length = sizeof(softirq_ebpf_val_t); + if (maps_per_core) + length *= ebpf_nprocs; + for (i = 0; i < NETDATA_SOFTIRQ_MAX_IRQS; i++) { int test = bpf_map_lookup_elem(fd, &i, softirq_ebpf_vals); if (unlikely(test < 0)) { @@ -106,12 +123,13 @@ static void softirq_read_latency_map() uint64_t total_latency = 0; int cpu_i; - int end = ebpf_nprocs; + int end = (maps_per_core) ? ebpf_nprocs : 1; for (cpu_i = 0; cpu_i < end; cpu_i++) { total_latency += softirq_ebpf_vals[cpu_i].latency/1000; } softirq_vals[i].latency = total_latency; + memset(softirq_ebpf_vals, 0, length); } } @@ -172,6 +190,7 @@ static void softirq_collector(ebpf_module_t *em) heartbeat_init(&hb); int update_every = em->update_every; int counter = update_every - 1; + int maps_per_core = em->maps_per_core; //This will be cancelled by its parent while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, USEC_PER_SEC); @@ -179,7 +198,7 @@ static void softirq_collector(ebpf_module_t *em) continue; counter = 0; - softirq_read_latency_map(); + softirq_read_latency_map(maps_per_core); pthread_mutex_lock(&lock); // write dims now for all hitherto discovered IRQs. @@ -212,6 +231,9 @@ void *ebpf_softirq_thread(void *ptr) goto endsoftirq; } +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(em->maps, em->maps_per_core, running_on_kernel); +#endif em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects); if (!em->probe_links) { goto endsoftirq; diff --git a/collectors/ebpf.plugin/ebpf_swap.c b/collectors/ebpf.plugin/ebpf_swap.c index 2352470a4..c9129a3fa 100644 --- a/collectors/ebpf.plugin/ebpf_swap.c +++ b/collectors/ebpf.plugin/ebpf_swap.c @@ -21,16 +21,32 @@ struct config swap_config = { .first_section = NULL, static ebpf_local_maps_t swap_maps[] = {{.name = "tbl_pid_swap", .internal_input = ND_EBPF_DEFAULT_PID_SIZE, .user_input = 0, .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, {.name = "swap_ctrl", .internal_input = NETDATA_CONTROLLER_END, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, {.name = "tbl_swap", .internal_input = NETDATA_SWAP_END, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = NULL, .internal_input = 0, .user_input = 0}}; + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; netdata_ebpf_targets_t swap_targets[] = { {.name = "swap_readpage", .mode = EBPF_LOAD_TRAMPOLINE}, {.name = "swap_writepage", .mode = EBPF_LOAD_TRAMPOLINE}, @@ -133,17 +149,21 @@ static void ebpf_swap_set_hash_tables(struct swap_bpf *obj) } /** - * Adjust Map Size + * Adjust Map * * Resize maps according input from users. * * @param obj is the main structure for bpf objects. * @param em structure with configuration */ -static void ebpf_swap_adjust_map_size(struct swap_bpf *obj, ebpf_module_t *em) +static void ebpf_swap_adjust_map(struct swap_bpf *obj, ebpf_module_t *em) { ebpf_update_map_size(obj->maps.tbl_pid_swap, &swap_maps[NETDATA_PID_SWAP_TABLE], em, bpf_map__name(obj->maps.tbl_pid_swap)); + + ebpf_update_map_type(obj->maps.tbl_pid_swap, &swap_maps[NETDATA_PID_SWAP_TABLE]); + ebpf_update_map_type(obj->maps.tbl_swap, &swap_maps[NETDATA_SWAP_GLOBAL_TABLE]); + ebpf_update_map_type(obj->maps.swap_ctrl, &swap_maps[NETDATA_SWAP_CONTROLLER]); } /** @@ -182,7 +202,7 @@ static inline int ebpf_swap_load_and_attach(struct swap_bpf *obj, ebpf_module_t ebpf_swap_disable_trampoline(obj); } - ebpf_swap_adjust_map_size(obj, em); + ebpf_swap_adjust_map(obj, em); if (!em->apps_charts && !em->cgroup_charts) ebpf_swap_disable_release_task(obj); @@ -251,10 +271,11 @@ static void ebpf_swap_exit(void *ptr) * Sum all values read from kernel and store in the first address. * * @param out the vector with read values. + * @param maps_per_core do I need to read all cores? */ -static void swap_apps_accumulator(netdata_publish_swap_t *out) +static void swap_apps_accumulator(netdata_publish_swap_t *out, int maps_per_core) { - int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1; + int i, end = (maps_per_core) ? ebpf_nprocs : 1; netdata_publish_swap_t *total = &out[0]; for (i = 1; i < end; i++) { netdata_publish_swap_t *w = &out[i]; @@ -286,13 +307,17 @@ static void swap_fill_pid(uint32_t current_pid, netdata_publish_swap_t *publish) * Update cgroup * * Update cgroup data based in + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_update_swap_cgroup() +static void ebpf_update_swap_cgroup(int maps_per_core) { ebpf_cgroup_target_t *ect ; netdata_publish_swap_t *cv = swap_vector; int fd = swap_maps[NETDATA_PID_SWAP_TABLE].map_fd; - size_t length = sizeof(netdata_publish_swap_t)*ebpf_nprocs; + size_t length = sizeof(netdata_publish_swap_t); + if (maps_per_core) + length *= ebpf_nprocs; pthread_mutex_lock(&mutex_cgroup_shm); for (ect = ebpf_cgroup_pids; ect; ect = ect->next) { struct pid_on_target2 *pids; @@ -306,9 +331,12 @@ static void ebpf_update_swap_cgroup() } else { memset(cv, 0, length); if (!bpf_map_lookup_elem(fd, &pid, cv)) { - swap_apps_accumulator(cv); + swap_apps_accumulator(cv, maps_per_core); memcpy(out, cv, sizeof(netdata_publish_swap_t)); + + // We are cleaning to avoid passing data read from one process to other. + memset(cv, 0, length); } } } @@ -320,14 +348,18 @@ static void ebpf_update_swap_cgroup() * Read APPS table * * Read the apps table and store data inside the structure. + * + * @param maps_per_core do I need to read all cores? */ -static void read_apps_table() +static void read_swap_apps_table(int maps_per_core) { netdata_publish_swap_t *cv = swap_vector; uint32_t key; struct ebpf_pid_stat *pids = ebpf_root_of_pids; int fd = swap_maps[NETDATA_PID_SWAP_TABLE].map_fd; - size_t length = sizeof(netdata_publish_swap_t)*ebpf_nprocs; + size_t length = sizeof(netdata_publish_swap_t); + if (maps_per_core) + length *= ebpf_nprocs; while (pids) { key = pids->pid; @@ -336,7 +368,7 @@ static void read_apps_table() continue; } - swap_apps_accumulator(cv); + swap_apps_accumulator(cv, maps_per_core); swap_fill_pid(key, cv); @@ -365,8 +397,10 @@ static void swap_send_global() * Read global counter * * Read the table with number of calls to all functions + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_swap_read_global_table() +static void ebpf_swap_read_global_table(int maps_per_core) { netdata_idx_t *stored = swap_values; netdata_idx_t *val = swap_hash_values; @@ -376,7 +410,7 @@ static void ebpf_swap_read_global_table() for (i = NETDATA_KEY_SWAP_READPAGE_CALL; i < end; i++) { if (!bpf_map_lookup_elem(fd, &i, stored)) { int j; - int last = ebpf_nprocs; + int last = (maps_per_core) ? ebpf_nprocs : 1; netdata_idx_t total = 0; for (j = 0; j < last; j++) total += stored[j]; @@ -646,6 +680,7 @@ static void swap_collector(ebpf_module_t *em) heartbeat_t hb; heartbeat_init(&hb); int counter = update_every - 1; + int maps_per_core = em->maps_per_core; while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, USEC_PER_SEC); if (ebpf_exit_plugin || ++counter != update_every) @@ -653,13 +688,13 @@ static void swap_collector(ebpf_module_t *em) counter = 0; netdata_apps_integration_flags_t apps = em->apps_charts; - ebpf_swap_read_global_table(); + ebpf_swap_read_global_table(maps_per_core); pthread_mutex_lock(&collect_data_mutex); if (apps) - read_apps_table(); + read_swap_apps_table(maps_per_core); if (cgroup) - ebpf_update_swap_cgroup(); + ebpf_update_swap_cgroup(maps_per_core); pthread_mutex_lock(&lock); @@ -752,7 +787,7 @@ static void ebpf_create_swap_charts(int update_every) EBPF_COMMON_DIMENSION_CALL, NETDATA_SYSTEM_SWAP_SUBMENU, NULL, NETDATA_EBPF_CHART_TYPE_LINE, - 202, + NETDATA_CHART_PRIO_SYSTEM_SWAP_CALLS, ebpf_create_global_dimension, swap_publish_aggregated, NETDATA_SWAP_END, update_every, NETDATA_EBPF_MODULE_NAME_SWAP); @@ -767,6 +802,10 @@ static void ebpf_create_swap_charts(int update_every) */ static int ebpf_swap_load_bpf(ebpf_module_t *em) { +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(em->maps, em->maps_per_core, running_on_kernel); +#endif + int ret = 0; ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_KEY_SWAP_READPAGE_CALL].mode); if (em->load & EBPF_LOAD_LEGACY) { diff --git a/collectors/ebpf.plugin/ebpf_sync.c b/collectors/ebpf.plugin/ebpf_sync.c index f838b65af..66e9c742c 100644 --- a/collectors/ebpf.plugin/ebpf_sync.c +++ b/collectors/ebpf.plugin/ebpf_sync.c @@ -10,27 +10,95 @@ static netdata_publish_syscall_t sync_counter_publish_aggregated[NETDATA_SYNC_ID static netdata_idx_t sync_hash_values[NETDATA_SYNC_IDX_END]; -static ebpf_local_maps_t sync_maps[] = {{.name = "tbl_sync", .internal_input = NETDATA_SYNC_END, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "tbl_syncfs", .internal_input = NETDATA_SYNC_END, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "tbl_msync", .internal_input = NETDATA_SYNC_END, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "tbl_fsync", .internal_input = NETDATA_SYNC_END, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "tbl_fdatasync", .internal_input = NETDATA_SYNC_END, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = "tbl_syncfr", .internal_input = NETDATA_SYNC_END, - .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = NULL, .internal_input = 0, .user_input = 0, - .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}}; +ebpf_local_maps_t sync_maps[] = {{.name = "tbl_sync", .internal_input = NETDATA_SYNC_END, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; + +ebpf_local_maps_t syncfs_maps[] = {{.name = "tbl_syncfs", .internal_input = NETDATA_SYNC_END, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; + +ebpf_local_maps_t msync_maps[] = {{.name = "tbl_msync", .internal_input = NETDATA_SYNC_END, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; + +ebpf_local_maps_t fsync_maps[] = {{.name = "tbl_fsync", .internal_input = NETDATA_SYNC_END, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; + +ebpf_local_maps_t fdatasync_maps[] = {{.name = "tbl_fdatasync", .internal_input = NETDATA_SYNC_END, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; + +ebpf_local_maps_t sync_file_range_maps[] = {{.name = "tbl_syncfr", .internal_input = NETDATA_SYNC_END, + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, + .type = NETDATA_EBPF_MAP_CONTROLLER, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; struct config sync_config = { .first_section = NULL, .last_section = NULL, @@ -111,12 +179,12 @@ void ebpf_sync_disable_tracepoints(struct sync_bpf *obj, sync_syscalls_index_t i * * Set the values for maps according the value given by kernel. * - * @param obj is the main structure for bpf objects. - * @param idx the index for the main structure + * @param map the map loaded. + * @param obj the main structure for bpf objects. */ -static void ebpf_sync_set_hash_tables(struct sync_bpf *obj, sync_syscalls_index_t idx) +static void ebpf_sync_set_hash_tables(ebpf_local_maps_t *map, struct sync_bpf *obj) { - sync_maps[idx].map_fd = bpf_map__fd(obj->maps.tbl_sync); + map->map_fd = bpf_map__fd(obj->maps.tbl_sync); } /** @@ -154,6 +222,8 @@ static inline int ebpf_sync_load_and_attach(struct sync_bpf *obj, ebpf_module_t ebpf_sync_disable_tracepoints(obj, idx); } + ebpf_update_map_type(obj->maps.tbl_sync, &em->maps[NETDATA_SYNC_GLOBAL_TABLE]); + int ret = sync_bpf__load(obj); if (!ret) { if (test != EBPF_LOAD_PROBE && test != EBPF_LOAD_RETPROBE) { @@ -165,7 +235,7 @@ static inline int ebpf_sync_load_and_attach(struct sync_bpf *obj, ebpf_module_t } if (!ret) - ebpf_sync_set_hash_tables(obj, idx); + ebpf_sync_set_hash_tables(&em->maps[NETDATA_SYNC_GLOBAL_TABLE], obj); } return ret; @@ -264,11 +334,21 @@ static int ebpf_sync_load_legacy(ebpf_sync_syscalls_t *w, ebpf_module_t *em) */ static int ebpf_sync_initialize_syscall(ebpf_module_t *em) { +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(sync_maps, em->maps_per_core, running_on_kernel); + ebpf_define_map_type(syncfs_maps, em->maps_per_core, running_on_kernel); + ebpf_define_map_type(msync_maps, em->maps_per_core, running_on_kernel); + ebpf_define_map_type(fsync_maps, em->maps_per_core, running_on_kernel); + ebpf_define_map_type(fdatasync_maps, em->maps_per_core, running_on_kernel); + ebpf_define_map_type(sync_file_range_maps, em->maps_per_core, running_on_kernel); +#endif + int i; const char *saved_name = em->thread_name; int errors = 0; for (i = 0; local_syscalls[i].syscall; i++) { ebpf_sync_syscalls_t *w = &local_syscalls[i]; + w->sync_maps = local_syscalls[i].sync_maps; if (w->enabled) { if (em->load & EBPF_LOAD_LEGACY) { if (ebpf_sync_load_legacy(w, em)) @@ -317,17 +397,25 @@ static int ebpf_sync_initialize_syscall(ebpf_module_t *em) * Read global table * * Read the table with number of calls for all functions + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_sync_read_global_table() +static void ebpf_sync_read_global_table(int maps_per_core) { - netdata_idx_t stored; + netdata_idx_t stored[ebpf_nprocs]; uint32_t idx = NETDATA_SYNC_CALL; int i; for (i = 0; local_syscalls[i].syscall; i++) { - if (local_syscalls[i].enabled) { - int fd = sync_maps[i].map_fd; + ebpf_sync_syscalls_t *w = &local_syscalls[i]; + if (w->enabled) { + int fd = w->sync_maps[NETDATA_SYNC_GLOBAL_TABLE].map_fd; if (!bpf_map_lookup_elem(fd, &idx, &stored)) { - sync_hash_values[i] = stored; + int j, end = (maps_per_core) ? ebpf_nprocs : 1; + netdata_idx_t total = 0; + for (j = 0; j < end ;j++ ) + total += stored[j]; + + sync_hash_values[i] = total; } } } @@ -352,7 +440,7 @@ static void ebpf_send_sync_chart(char *id, while (move && idx <= end) { if (local_syscalls[idx].enabled) - write_chart_dimension(move->name, sync_hash_values[idx]); + write_chart_dimension(move->name, (long long)sync_hash_values[idx]); move = move->next; idx++; @@ -396,13 +484,14 @@ static void sync_collector(ebpf_module_t *em) heartbeat_init(&hb); int update_every = em->update_every; int counter = update_every - 1; + int maps_per_core = em->maps_per_core; while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, USEC_PER_SEC); if (ebpf_exit_plugin || ++counter != update_every) continue; counter = 0; - ebpf_sync_read_global_table(); + ebpf_sync_read_global_table(maps_per_core); pthread_mutex_lock(&lock); sync_send_data(); @@ -498,6 +587,22 @@ static void ebpf_sync_parse_syscalls() } /** + * Set sync maps + * + * When thread is initialized the variable sync_maps is set as null, + * this function fills the variable before to use. + */ +static void ebpf_set_sync_maps() +{ + local_syscalls[NETDATA_SYNC_SYNC_IDX].sync_maps = sync_maps; + local_syscalls[NETDATA_SYNC_SYNCFS_IDX].sync_maps = syncfs_maps; + local_syscalls[NETDATA_SYNC_MSYNC_IDX].sync_maps = msync_maps; + local_syscalls[NETDATA_SYNC_FSYNC_IDX].sync_maps = fsync_maps; + local_syscalls[NETDATA_SYNC_FDATASYNC_IDX].sync_maps = fdatasync_maps; + local_syscalls[NETDATA_SYNC_SYNC_FILE_RANGE_IDX].sync_maps = sync_file_range_maps; +} + +/** * Sync thread * * Thread used to make sync thread @@ -513,6 +618,7 @@ void *ebpf_sync_thread(void *ptr) ebpf_module_t *em = (ebpf_module_t *)ptr; em->maps = sync_maps; + ebpf_set_sync_maps(); ebpf_sync_parse_syscalls(); #ifdef LIBBPF_MAJOR_VERSION diff --git a/collectors/ebpf.plugin/ebpf_unittest.c b/collectors/ebpf.plugin/ebpf_unittest.c new file mode 100644 index 000000000..3e1443ad3 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf_unittest.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "ebpf_unittest.h" + +ebpf_module_t test_em; + +/** + * Initialize structure + * + * Initialize structure used to run unittests + */ +void ebpf_ut_initialize_structure(netdata_run_mode_t mode) +{ + memset(&test_em, 0, sizeof(ebpf_module_t)); + test_em.thread_name = strdupz("process"); + test_em.config_name = test_em.thread_name; + test_em.kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_10 | + NETDATA_V5_14; + test_em.pid_map_size = ND_EBPF_DEFAULT_PID_SIZE; + test_em.apps_level = NETDATA_APPS_LEVEL_REAL_PARENT; + test_em.mode = mode; +} + +/** + * Clean UP Memory + * + * Clean up allocated data during unit test; + */ +void ebpf_ut_cleanup_memory() +{ + freez((void *)test_em.thread_name); +} + +/** + * Load Binary + * + * Test load of legacy eBPF programs. + * + * @return It returns 0 on success and -1 otherwise. + */ +static int ebpf_ut_load_binary() +{ + test_em.probe_links = ebpf_load_program(ebpf_plugin_dir, &test_em, running_on_kernel, isrh, &test_em.objects); + if (!test_em.probe_links) + return -1; + + ebpf_unload_legacy_code(test_em.objects, test_em.probe_links); + + return 0; +} + +/** + * Load Real Binary + * + * Load an existent binary inside plugin directory. + * + * @return It returns 0 on success and -1 otherwise. + */ +int ebpf_ut_load_real_binary() +{ + return ebpf_ut_load_binary(); +} +/** + * Load fake Binary + * + * Try to load a binary not generated by netdata. + * + * @return It returns 0 on success and -1 otherwise. The success for this function means we could work properly with + * expected fails. + */ +int ebpf_ut_load_fake_binary() +{ + const char *original = test_em.thread_name; + + test_em.thread_name = strdupz("I_am_not_here"); + int ret = ebpf_ut_load_binary(); + + ebpf_ut_cleanup_memory(); + + test_em.thread_name = original; + + return !ret; +} diff --git a/collectors/ebpf.plugin/ebpf_unittest.h b/collectors/ebpf.plugin/ebpf_unittest.h new file mode 100644 index 000000000..429cbe628 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf_unittest.h @@ -0,0 +1,10 @@ +#ifndef NETDATA_EBPF_PLUGIN_UNITTEST_H_ +# define NETDATA_EBPF_PLUGIN_UNITTEST_H_ 1 + +#include "ebpf.h" + +void ebpf_ut_initialize_structure(netdata_run_mode_t mode); +int ebpf_ut_load_real_binary(); +int ebpf_ut_load_fake_binary(); +void ebpf_ut_cleanup_memory(); +#endif diff --git a/collectors/ebpf.plugin/ebpf_vfs.c b/collectors/ebpf.plugin/ebpf_vfs.c index e2d87fd52..bfc7ee8f7 100644 --- a/collectors/ebpf.plugin/ebpf_vfs.c +++ b/collectors/ebpf.plugin/ebpf_vfs.c @@ -17,15 +17,31 @@ netdata_publish_vfs_t *vfs_vector = NULL; static ebpf_local_maps_t vfs_maps[] = {{.name = "tbl_vfs_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE, .user_input = 0, .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_HASH +#endif + }, {.name = "tbl_vfs_stats", .internal_input = NETDATA_VFS_COUNTER, .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, {.name = "vfs_ctrl", .internal_input = NETDATA_CONTROLLER_END, .user_input = 0, .type = NETDATA_EBPF_MAP_CONTROLLER, - .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}, - {.name = NULL, .internal_input = 0, .user_input = 0}}; + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }, + {.name = NULL, .internal_input = 0, .user_input = 0, +#ifdef LIBBPF_MAJOR_VERSION + .map_type = BPF_MAP_TYPE_PERCPU_ARRAY +#endif + }}; struct config vfs_config = { .first_section = NULL, .last_section = NULL, @@ -293,17 +309,21 @@ static int ebpf_vfs_attach_probe(struct vfs_bpf *obj) } /** - * Adjust Map Size + * Adjust Size * * Resize maps according input from users. * * @param obj is the main structure for bpf objects. * @param em structure with configuration */ -static void ebpf_vfs_adjust_map_size(struct vfs_bpf *obj, ebpf_module_t *em) +static void ebpf_vfs_adjust_map(struct vfs_bpf *obj, ebpf_module_t *em) { ebpf_update_map_size(obj->maps.tbl_vfs_pid, &vfs_maps[NETDATA_VFS_PID], em, bpf_map__name(obj->maps.tbl_vfs_pid)); + + ebpf_update_map_type(obj->maps.tbl_vfs_pid, &vfs_maps[NETDATA_VFS_PID]); + ebpf_update_map_type(obj->maps.tbl_vfs_stats, &vfs_maps[NETDATA_VFS_ALL]); + ebpf_update_map_type(obj->maps.vfs_ctrl, &vfs_maps[NETDATA_VFS_CTRL]); } /** @@ -356,7 +376,7 @@ static inline int ebpf_vfs_load_and_attach(struct vfs_bpf *obj, ebpf_module_t *e ebpf_vfs_disable_trampoline(obj); } - ebpf_vfs_adjust_map_size(obj, em); + ebpf_vfs_adjust_map(obj, em); if (!em->apps_charts && !em->cgroup_charts) ebpf_vfs_disable_release_task(obj); @@ -475,23 +495,30 @@ static void ebpf_vfs_send_data(ebpf_module_t *em) /** * Read the hash table and store data to allocated vectors. + * + * @param maps_per_core do I need to read all cores? */ -static void ebpf_vfs_read_global_table() +static void ebpf_vfs_read_global_table(int maps_per_core) { uint64_t idx; netdata_idx_t res[NETDATA_VFS_COUNTER]; netdata_idx_t *val = vfs_hash_values; + size_t length = sizeof(netdata_idx_t); + if (maps_per_core) + length *= ebpf_nprocs; + int fd = vfs_maps[NETDATA_VFS_ALL].map_fd; for (idx = 0; idx < NETDATA_VFS_COUNTER; idx++) { uint64_t total = 0; if (!bpf_map_lookup_elem(fd, &idx, val)) { int i; - int end = ebpf_nprocs; + int end = (maps_per_core) ? ebpf_nprocs : 1; for (i = 0; i < end; i++) total += val[i]; } res[idx] = total; + memset(val, 0, length); } vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_UNLINK].ncall = res[NETDATA_KEY_CALLS_VFS_UNLINK]; @@ -723,9 +750,9 @@ void ebpf_vfs_send_apps_data(ebpf_module_t *em, struct ebpf_target *root) * * @param out the vector with read values. */ -static void vfs_apps_accumulator(netdata_publish_vfs_t *out) +static void vfs_apps_accumulator(netdata_publish_vfs_t *out, int maps_per_core) { - int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1; + int i, end = (maps_per_core) ? ebpf_nprocs : 1; netdata_publish_vfs_t *total = &out[0]; for (i = 1; i < end; i++) { netdata_publish_vfs_t *w = &out[i]; @@ -771,12 +798,15 @@ static void vfs_fill_pid(uint32_t current_pid, netdata_publish_vfs_t *publish) /** * Read the hash table and store data to allocated vectors. */ -static void ebpf_vfs_read_apps() +static void ebpf_vfs_read_apps(int maps_per_core) { struct ebpf_pid_stat *pids = ebpf_root_of_pids; netdata_publish_vfs_t *vv = vfs_vector; int fd = vfs_maps[NETDATA_VFS_PID].map_fd; - size_t length = sizeof(netdata_publish_vfs_t) * ebpf_nprocs; + size_t length = sizeof(netdata_publish_vfs_t); + if (maps_per_core) + length *= ebpf_nprocs; + while (pids) { uint32_t key = pids->pid; @@ -785,7 +815,7 @@ static void ebpf_vfs_read_apps() continue; } - vfs_apps_accumulator(vv); + vfs_apps_accumulator(vv, maps_per_core); vfs_fill_pid(key, vv); @@ -799,14 +829,18 @@ static void ebpf_vfs_read_apps() /** * Update cgroup * - * Update cgroup data based in + * Update cgroup data based in PID. + * + * @param maps_per_core do I need to read all cores? */ -static void read_update_vfs_cgroup() +static void read_update_vfs_cgroup(int maps_per_core) { ebpf_cgroup_target_t *ect ; netdata_publish_vfs_t *vv = vfs_vector; int fd = vfs_maps[NETDATA_VFS_PID].map_fd; - size_t length = sizeof(netdata_publish_vfs_t) * ebpf_nprocs; + size_t length = sizeof(netdata_publish_vfs_t); + if (maps_per_core) + length *= ebpf_nprocs; pthread_mutex_lock(&mutex_cgroup_shm); for (ect = ebpf_cgroup_pids; ect; ect = ect->next) { @@ -821,7 +855,7 @@ static void read_update_vfs_cgroup() } else { memset(vv, 0, length); if (!bpf_map_lookup_elem(fd, &pid, vv)) { - vfs_apps_accumulator(vv); + vfs_apps_accumulator(vv, maps_per_core); memcpy(out, vv, sizeof(netdata_publish_vfs_t)); } @@ -1458,6 +1492,7 @@ static void vfs_collector(ebpf_module_t *em) heartbeat_init(&hb); int update_every = em->update_every; int counter = update_every - 1; + int maps_per_core = em->maps_per_core; while (!ebpf_exit_plugin) { (void)heartbeat_next(&hb, USEC_PER_SEC); if (ebpf_exit_plugin || ++counter != update_every) @@ -1465,21 +1500,21 @@ static void vfs_collector(ebpf_module_t *em) counter = 0; netdata_apps_integration_flags_t apps = em->apps_charts; - ebpf_vfs_read_global_table(); + ebpf_vfs_read_global_table(maps_per_core); pthread_mutex_lock(&collect_data_mutex); if (apps) - ebpf_vfs_read_apps(); + ebpf_vfs_read_apps(maps_per_core); + + if (cgroups) + read_update_vfs_cgroup(maps_per_core); + + pthread_mutex_lock(&lock); #ifdef NETDATA_DEV_MODE if (ebpf_aral_vfs_pid) ebpf_send_data_aral_chart(ebpf_aral_vfs_pid, em); #endif - if (cgroups) - read_update_vfs_cgroup(); - - pthread_mutex_lock(&lock); - ebpf_vfs_send_data(em); fflush(stdout); @@ -1843,6 +1878,10 @@ static void ebpf_vfs_allocate_global_vectors(int apps) */ static int ebpf_vfs_load_bpf(ebpf_module_t *em) { +#ifdef LIBBPF_MAJOR_VERSION + ebpf_define_map_type(em->maps, em->maps_per_core, running_on_kernel); +#endif + int ret = 0; ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_EBPF_VFS_WRITE].mode); if (em->load & EBPF_LOAD_LEGACY) { diff --git a/collectors/freeipmi.plugin/freeipmi_plugin.c b/collectors/freeipmi.plugin/freeipmi_plugin.c index bcc5139f3..a2251891a 100644 --- a/collectors/freeipmi.plugin/freeipmi_plugin.c +++ b/collectors/freeipmi.plugin/freeipmi_plugin.c @@ -1833,6 +1833,21 @@ int main (int argc, char **argv) { for(iteration = 0; 1 ; iteration++) { usec_t dt = heartbeat_next(&hb, step); + if (iteration) { + if (iteration == 1) { + fprintf( + stdout, + "CHART netdata.freeipmi_availability_status '' 'Plugin availability status' 'status' plugins netdata.plugin_availability_status line 146000 %d\n" + "DIMENSION available '' absolute 1 1\n", + netdata_update_every); + } + fprintf( + stdout, + "BEGIN netdata.freeipmi_availability_status\n" + "SET available = 1\n" + "END\n"); + } + if(debug && iteration) fprintf(stderr, "freeipmi.plugin: iteration %zu, dt %llu usec, sensors collected %zu, sensors sent to netdata %zu \n" , iteration @@ -1852,6 +1867,11 @@ int main (int argc, char **argv) { fflush(stdout); // restart check (14400 seconds) - if(now_monotonic_sec() - started_t > 14400) exit(0); + if (now_monotonic_sec() - started_t > 14400) { + fprintf(stdout, "EXIT\n"); + fflush(stdout); + exit(0); + } } } + diff --git a/collectors/nfacct.plugin/README.md b/collectors/nfacct.plugin/README.md index e8502236f..ae6597a40 100644 --- a/collectors/nfacct.plugin/README.md +++ b/collectors/nfacct.plugin/README.md @@ -13,6 +13,11 @@ learn_rel_path: "Integrations/Monitor/Networking" ## Prerequisites +If you are using [our official native DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md), install the +`netdata-plugin-nfacct` package using your system package manager. + +If you built Netdata locally: + 1. install `libmnl-dev` and `libnetfilter-acct-dev` using the package manager of your system. 2. re-install Netdata from source. The installer will detect that the required libraries are now available and will also build `netdata.plugin`. diff --git a/collectors/perf.plugin/README.md b/collectors/perf.plugin/README.md index e519be9c4..a8bd4b0e5 100644 --- a/collectors/perf.plugin/README.md +++ b/collectors/perf.plugin/README.md @@ -14,6 +14,9 @@ the `perf_event_open()` system call. ## Important Notes +If you are using [our official native DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md), you will need to install +the `netdata-plugin-perf` package using your system package manager. + Accessing hardware PMUs requires root permissions, so the plugin is setuid to root. Keep in mind that the number of PMUs in a system is usually quite limited and every hardware monitoring diff --git a/collectors/plugins.d/plugins_d.c b/collectors/plugins.d/plugins_d.c index dc13cd2ee..da5226a5c 100644 --- a/collectors/plugins.d/plugins_d.c +++ b/collectors/plugins.d/plugins_d.c @@ -72,7 +72,7 @@ static void pluginsd_worker_thread_cleanup(void *arg) info("PLUGINSD: 'host:%s', waiting for data collection child process pid %d to exit...", rrdhost_hostname(cd->host), pid); - waitid(P_PID, (id_t)pid, &info, WEXITED); + netdata_waitid(P_PID, (id_t)pid, &info, WEXITED); } } } diff --git a/collectors/plugins.d/plugins_d.h b/collectors/plugins.d/plugins_d.h index 68ed4940f..fe43a19f5 100644 --- a/collectors/plugins.d/plugins_d.h +++ b/collectors/plugins.d/plugins_d.h @@ -87,6 +87,7 @@ struct plugind { extern struct plugind *pluginsd_root; size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugin_input, FILE *fp_plugin_output, int trust_durations); +void pluginsd_process_thread_cleanup(void *ptr); size_t pluginsd_initialize_plugin_directories(); diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c index 28fc0bd49..097e5ea60 100644 --- a/collectors/plugins.d/pluginsd_parser.c +++ b/collectors/plugins.d/pluginsd_parser.c @@ -11,10 +11,10 @@ static int send_to_plugin(const char *txt, void *data) { return 0; #ifdef ENABLE_HTTPS - struct netdata_ssl *ssl = parser->ssl_output; + NETDATA_SSL *ssl = parser->ssl_output; if(ssl) { - if(ssl->conn && ssl->flags == NETDATA_SSL_HANDSHAKE_COMPLETE) - return (int)netdata_ssl_write(ssl->conn, (void *)txt, strlen(txt)); + if(SSL_connection(ssl)) + return (int)netdata_ssl_write(ssl, (void *)txt, strlen(txt)); error("PLUGINSD: cannot send command (SSL)"); return -1; @@ -108,11 +108,12 @@ void pluginsd_rrdset_cleanup(RRDSET *st) { st->pluginsd.pos = 0; } -static inline void pluginsd_set_chart_from_parent(void *user, RRDSET *st, const char *keyword) { +static inline void pluginsd_unlock_previous_chart(void *user, const char *keyword, bool stale) { PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user; if(unlikely(pluginsd_unlock_rrdset_data_collection(user))) { - error("PLUGINSD: 'host:%s/chart:%s/' stale data collection lock found during %s; it has been unlocked", + if(stale) + error("PLUGINSD: 'host:%s/chart:%s/' stale data collection lock found during %s; it has been unlocked", rrdhost_hostname(u->st->rrdhost), rrdset_id(u->st), keyword); } @@ -120,9 +121,16 @@ static inline void pluginsd_set_chart_from_parent(void *user, RRDSET *st, const ml_chart_update_end(u->st); u->v2.ml_locked = false; - error("PLUGINSD: 'host:%s/chart:%s/' stale ML lock found during %s, it has been unlocked", + if(stale) + error("PLUGINSD: 'host:%s/chart:%s/' stale ML lock found during %s, it has been unlocked", rrdhost_hostname(u->st->rrdhost), rrdset_id(u->st), keyword); } +} + +static inline void pluginsd_set_chart_from_parent(void *user, RRDSET *st, const char *keyword) { + PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user; + + pluginsd_unlock_previous_chart(user, keyword, true); if(st) { size_t dims = dictionary_entries(st->rrddim_root_index); @@ -1459,9 +1467,11 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user) time_t started = st->rrdhost->receiver->replication_first_time_t; time_t current = ((PARSER_USER_OBJECT *) user)->replay.end_time; - if(started && current > started) + if(started && current > started) { + host->rrdpush_receiver_replication_percent = (NETDATA_DOUBLE) (current - started) * 100.0 / (NETDATA_DOUBLE) (now - started); worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, - (NETDATA_DOUBLE)(current - started) * 100.0 / (NETDATA_DOUBLE)(now - started)); + host->rrdpush_receiver_replication_percent); + } } ((PARSER_USER_OBJECT *) user)->replay.start_time = 0; @@ -1501,7 +1511,8 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user) pluginsd_set_chart_from_parent(user, NULL, PLUGINSD_KEYWORD_REPLAY_END); - worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, 100.0); + host->rrdpush_receiver_replication_percent = 100.0; + worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, host->rrdpush_receiver_replication_percent); return PARSER_RC_OK; } @@ -1783,12 +1794,7 @@ PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_ // ------------------------------------------------------------------------ // unblock data collection - ml_chart_update_end(st); - u->v2.ml_locked = false; - - timing_step(TIMING_STEP_END2_ML); - - pluginsd_unlock_rrdset_data_collection(user); + pluginsd_unlock_previous_chart(user, PLUGINSD_KEYWORD_END_V2, false); rrdcontext_collected_rrdset(st); store_metric_collection_completed(); @@ -1823,13 +1829,14 @@ PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_ return PARSER_RC_OK; } -static void pluginsd_process_thread_cleanup(void *ptr) { +void pluginsd_process_thread_cleanup(void *ptr) { PARSER *parser = (PARSER *)ptr; pluginsd_cleanup_v2(parser->user); pluginsd_host_define_cleanup(parser->user); rrd_collector_finished(); + parser_destroy(parser); } diff --git a/collectors/proc.plugin/proc_self_mountinfo.c b/collectors/proc.plugin/proc_self_mountinfo.c index 0483749c3..194791603 100644 --- a/collectors/proc.plugin/proc_self_mountinfo.c +++ b/collectors/proc.plugin/proc_self_mountinfo.c @@ -360,6 +360,18 @@ struct mountinfo *mountinfo_read(int do_statvfs) { else { mi->st_dev = 0; } + + //try to detect devices with same minor and major modes. Within these, + //the larger mount point is considered a bind. + struct mountinfo *mt; + for(mt = root; mt; mt = mt->next) { + if(unlikely(mt->major == mi->major && mt->minor == mi->minor && !(mi->flags & MOUNTINFO_IS_BIND))) { + if(strlen(mi->root) < strlen(mt->root)) + mt->flags |= MOUNTINFO_IS_BIND; + else + mi->flags |= MOUNTINFO_IS_BIND; + } + } } else { mi->filesystem = NULL; diff --git a/collectors/python.d.plugin/oracledb/README.md b/collectors/python.d.plugin/oracledb/README.md index 722c77b75..315816de0 100644 --- a/collectors/python.d.plugin/oracledb/README.md +++ b/collectors/python.d.plugin/oracledb/README.md @@ -13,8 +13,7 @@ Monitors the performance and health metrics of the Oracle database. ## Requirements -- `cx_Oracle` package. -- Oracle Client (using `cx_Oracle` requires Oracle Client libraries to be installed). +- `oracledb` package. It produces following charts: @@ -53,18 +52,13 @@ It produces following charts: To use the Oracle module do the following: -1. Install `cx_Oracle` package ([link](https://cx-oracle.readthedocs.io/en/latest/user_guide/installation.html)). +1. Install `oracledb` package ([link](https://python-oracledb.readthedocs.io/en/latest/user_guide/installation.html)). -2. Install Oracle Client libraries - ([link](https://cx-oracle.readthedocs.io/en/latest/user_guide/installation.html#install-oracle-client)). - -3. Create a read-only `netdata` user with proper access to your Oracle Database Server. +2. Create a read-only `netdata` user with proper access to your Oracle Database Server. Connect to your Oracle database with an administrative user and execute: -``` -ALTER SESSION SET "_ORACLE_SCRIPT"=true; - +```SQL CREATE USER netdata IDENTIFIED BY <PASSWORD>; GRANT CONNECT TO netdata; @@ -88,6 +82,7 @@ local: server: 'localhost:1521' service: 'XE' + remote: user: 'netdata' password: 'secret' diff --git a/collectors/python.d.plugin/oracledb/oracledb.chart.py b/collectors/python.d.plugin/oracledb/oracledb.chart.py index 28ef8db10..455cf270e 100644 --- a/collectors/python.d.plugin/oracledb/oracledb.chart.py +++ b/collectors/python.d.plugin/oracledb/oracledb.chart.py @@ -8,11 +8,18 @@ from copy import deepcopy from bases.FrameworkServices.SimpleService import SimpleService try: - import cx_Oracle + import oracledb as cx_Oracle - HAS_ORACLE = True + HAS_ORACLE_NEW = True + HAS_ORACLE_OLD = False except ImportError: - HAS_ORACLE = False + HAS_ORACLE_NEW = False + try: + import cx_Oracle + + HAS_ORACLE_OLD = True + except ImportError: + HAS_ORACLE_OLD = False ORDER = [ 'session_count', @@ -187,7 +194,7 @@ CHARTS = { }, } -CX_CONNECT_STRING = "{0}/{1}@//{2}/{3}" +CX_CONNECT_STRING_OLD = "{0}/{1}@//{2}/{3}" QUERY_SYSTEM = ''' SELECT @@ -322,6 +329,7 @@ class Service(SimpleService): self.password = configuration.get('password') self.server = configuration.get('server') self.service = configuration.get('service') + self.protocol = configuration.get('protocol', 'tcps') self.alive = False self.conn = None self.active_tablespaces = set() @@ -330,18 +338,25 @@ class Service(SimpleService): if self.conn: self.conn.close() self.conn = None - - try: - self.conn = cx_Oracle.connect( - CX_CONNECT_STRING.format( - self.user, - self.password, - self.server, - self.service, - )) - except cx_Oracle.DatabaseError as error: - self.error(error) - return False + if HAS_ORACLE_NEW: + try: + self.conn = cx_Oracle.connect( + f'{self.user}/{self.password}@{self.protocol}://{self.server}/{self.service}') + except cx_Oracle.DatabaseError as error: + self.error(error) + return False + else: + try: + self.conn = cx_Oracle.connect( + CX_CONNECT_STRING_OLD.format( + self.user, + self.password, + self.server, + self.service, + )) + except cx_Oracle.DatabaseError as error: + self.error(error) + return False self.alive = True return True @@ -350,15 +365,15 @@ class Service(SimpleService): return self.connect() def check(self): - if not HAS_ORACLE: - self.error("'cx_Oracle' package is needed to use oracledb module") + if not HAS_ORACLE_NEW and not HAS_ORACLE_OLD: + self.error("'oracledb' package is needed to use oracledb module") return False if not all([ self.user, self.password, self.server, - self.service, + self.service ]): self.error("one of these parameters is not specified: user, password, server, service") return False @@ -812,7 +827,7 @@ class Service(SimpleService): 'absolute', 1, 1000, - ]) + ]) self.charts['allocated_usage'].add_dimension( [ '{0}_allocated_used'.format(name), @@ -820,7 +835,7 @@ class Service(SimpleService): 'absolute', 1, 1000, - ]) + ]) self.charts['allocated_usage_in_percent'].add_dimension( [ '{0}_allocated_used_in_percent'.format(name), diff --git a/collectors/python.d.plugin/oracledb/oracledb.conf b/collectors/python.d.plugin/oracledb/oracledb.conf index 625717299..027215dad 100644 --- a/collectors/python.d.plugin/oracledb/oracledb.conf +++ b/collectors/python.d.plugin/oracledb/oracledb.conf @@ -63,9 +63,11 @@ # # user: username # the username for the user account. Required. # password: password # the password for the user account. Required. -# server: localhost:1521 # the IP address or hostname of the Oracle Database Server. Required. +# server: localhost:1521 # the IP address or hostname (and port) of the Oracle Database Server. Required. # service: XE # the Oracle Database service name. Required. To view the services available on your server, -# run this query: `SELECT value FROM v$parameter WHERE name='service_names'`. +# run this query: `select SERVICE_NAME from gv$session where sid in (select sid from V$MYSTAT)`. +# protocol: tcp/tcps # one of the strings "tcp" or "tcps" indicating whether to use unencrypted network traffic +# or encrypted network traffic # # ---------------------------------------------------------------------- # AUTO-DETECTION JOBS @@ -76,9 +78,11 @@ # password: 'secret' # server: 'localhost:1521' # service: 'XE' +# protocol: 'tcps' #remote: # user: 'netdata' # password: 'secret' # server: '10.0.0.1:1521' -# service: 'XE'
\ No newline at end of file +# service: 'XE' +# protocol: 'tcps' diff --git a/collectors/python.d.plugin/smartd_log/smartd_log.conf b/collectors/python.d.plugin/smartd_log/smartd_log.conf index 6c01d953b..3e81317f1 100644 --- a/collectors/python.d.plugin/smartd_log/smartd_log.conf +++ b/collectors/python.d.plugin/smartd_log/smartd_log.conf @@ -63,6 +63,7 @@ # # log_path: '/path/to/smartd_logs' # path to smartd log files. Default is /var/log/smartd # exclude_disks: 'PATTERN1 PATTERN2' # space separated patterns. If the pattern is in the drive name, the module will not collect data for it. +# age: 30 # time in minutes since the last dump to file. If smartd has not dumped data within this time the job exits. # # ---------------------------------------------------------------------- diff --git a/collectors/python.d.plugin/tor/tor.chart.py b/collectors/python.d.plugin/tor/tor.chart.py index 8dc021a63..f7bc2d79b 100644 --- a/collectors/python.d.plugin/tor/tor.chart.py +++ b/collectors/python.d.plugin/tor/tor.chart.py @@ -17,6 +17,7 @@ except ImportError: STEM_AVAILABLE = False DEF_PORT = 'default' +DEF_ADDR = '127.0.0.1' ORDER = [ 'traffic', @@ -41,6 +42,7 @@ class Service(SimpleService): self.order = ORDER self.definitions = CHARTS self.port = self.configuration.get('control_port', DEF_PORT) + self.addr = self.configuration.get('control_addr', DEF_ADDR) self.password = self.configuration.get('password') self.use_socket = isinstance(self.port, str) and self.port != DEF_PORT and not self.port.isdigit() self.conn = None @@ -78,7 +80,7 @@ class Service(SimpleService): def connect_via_port(self): try: - self.conn = stem.control.Controller.from_port(port=self.port) + self.conn = stem.control.Controller.from_port(address=self.addr, port=self.port) except (stem.SocketError, ValueError) as error: self.error(error) diff --git a/collectors/python.d.plugin/tor/tor.conf b/collectors/python.d.plugin/tor/tor.conf index bf09b21fe..c7c98dc0b 100644 --- a/collectors/python.d.plugin/tor/tor.conf +++ b/collectors/python.d.plugin/tor/tor.conf @@ -61,6 +61,7 @@ # # Additionally to the above, tor plugin also supports the following: # +# control_addr: 'address' # tor control IP address (defaults to '127.0.0.1') # control_port: 'port' # tor control port # password: 'password' # tor control password # @@ -71,6 +72,7 @@ # local_tcp: # name: 'local' # control_port: 9051 +# control_addr: 127.0.0.1 # password: <password> # # local_socket: diff --git a/collectors/slabinfo.plugin/README.md b/collectors/slabinfo.plugin/README.md index e0abaff80..abcbe1e3f 100644 --- a/collectors/slabinfo.plugin/README.md +++ b/collectors/slabinfo.plugin/README.md @@ -18,6 +18,9 @@ Each internal structure (process, file descriptor, inode...) is stored within a The plugin is disabled by default because it collects and displays a huge amount of metrics. To enable it set `slabinfo = yes` in the `plugins` section of the `netdata.conf` configuration file. +If you are using [our official native DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md), you will additionally need to install the `netdata-plugin-slabinfo` +package using your system package manager. + There is currently no configuration needed for the plugin itself. As `/proc/slabinfo` is only readable by root, this plugin is setuid root. diff --git a/collectors/tc.plugin/plugin_tc.c b/collectors/tc.plugin/plugin_tc.c index b7e493b69..b833fd3c2 100644 --- a/collectors/tc.plugin/plugin_tc.c +++ b/collectors/tc.plugin/plugin_tc.c @@ -864,7 +864,7 @@ static void tc_main_cleanup(void *ptr) { siginfo_t info; collector_info("TC: waiting for tc plugin child process pid %d to exit...", tc_child_pid); - waitid(P_PID, (id_t) tc_child_pid, &info, WEXITED); + netdata_waitid(P_PID, (id_t) tc_child_pid, &info, WEXITED); } tc_child_pid = 0; |