diff options
Diffstat (limited to 'collectors/ebpf.plugin')
21 files changed, 2606 insertions, 1066 deletions
diff --git a/collectors/ebpf.plugin/Makefile.am b/collectors/ebpf.plugin/Makefile.am index 1327d47a6..4fb2056fd 100644 --- a/collectors/ebpf.plugin/Makefile.am +++ b/collectors/ebpf.plugin/Makefile.am @@ -10,6 +10,12 @@ CLEANFILES = \ include $(top_srcdir)/build/subst.inc SUFFIXES = .in +userebpfconfigdir=$(configdir)/ebpf.d + +# Explicitly install directories to avoid permission issues due to umask +install-exec-local: + $(INSTALL) -d $(DESTDIR)$(userebpfconfigdir) + dist_plugins_SCRIPTS = \ reset_netdata_trace.sh \ $(NULL) @@ -19,7 +25,15 @@ dist_noinst_DATA = \ README.md \ $(NULL) +ebpfconfigdir=$(libconfigdir)/ebpf.d dist_libconfig_DATA = \ - ebpf.conf \ - ebpf_kernel_reject_list.txt \ + ebpf.d.conf \ + $(NULL) + +dist_ebpfconfig_DATA = \ + ebpf.d/ebpf_kernel_reject_list.txt \ + ebpf.d/cachestat.conf \ + ebpf.d/network.conf \ + ebpf.d/process.conf \ + ebpf.d/sync.conf \ $(NULL) diff --git a/collectors/ebpf.plugin/README.md b/collectors/ebpf.plugin/README.md index 5ea3b4951..405eab875 100644 --- a/collectors/ebpf.plugin/README.md +++ b/collectors/ebpf.plugin/README.md @@ -148,6 +148,7 @@ accepts the following values: - `return`: In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates new charts for the return of these functions, such as errors. Monitoring function returns can help in debugging software, such as failing to close file descriptors or creating zombie processes. +- `update every`: Number of seconds used for eBPF to send data for Netdata. #### Integration with `apps.plugin` @@ -186,16 +187,45 @@ If you want to _disable_ the integration with `apps.plugin` along with the above apps = yes ``` -### `[ebpf programs]` +#### `[ebpf programs]` The eBPF collector enables and runs the following eBPF programs by default: +- `cachestat`: Netdata's eBPF data collector creates charts about the memory page cache. When the integration with + [`apps.plugin`](/collectors/apps.plugin/README.md) is enabled, this collector creates charts for the whole host _and_ + for each application. - `process`: This eBPF program creates charts that show information about process creation, VFS IO, and files removed. When in `return` mode, it also creates charts showing errors when these operations are executed. - `network viewer`: This eBPF program creates charts with information about `TCP` and `UDP` functions, including the bandwidth consumed by each. +- `sync`: Montitor calls for syscalls sync(2), fsync(2), fdatasync(2), syncfs(2), msync(2), and sync_file_range(2). -### `[network connections]` +## Thread configuration + +You can configure each thread of the eBPF data collector by editing either the `cachestat.conf`, `process.conf`, +or `network.conf` files. Use [`edit-config`](/docs/configure/nodes.md) from your Netdata config directory: + +```bash +cd /etc/netdata/ # Replace with your Netdata configuration directory, if not /etc/netdata/ +./edit-config ebpf.d/process.conf +``` + +### Configuration files + +The following configuration files are available: + +- `cachestat.conf`: Configuration for the `cachestat` thread. +- `process.conf`: Configuration for the `process` thread. +- `network.conf`: Configuration for the `network viewer` thread. This config file overwrites the global options and + also lets you specify which network the eBPF collector monitors. +- `sync.conf`: Configuration for the `sync` thread. + +### Network configuration + +The network configuration has specific options to configure which network(s) the eBPF collector monitors. These options +are divided in the following sections: + +#### `[network connections]` You can configure the information shown on `outbound` and `inbound` charts with the settings in this section. @@ -232,7 +262,7 @@ The dimensions for the traffic charts are created using the destination IPs of t changed setting `resolve hostname ips = yes` and restarting Netdata, after this Netdata will create dimensions using the `hostnames` every time that is possible to resolve IPs to their hostnames. -### `[service name]` +#### `[service name]` Netdata uses the list of services in `/etc/services` to plot network connection charts. If this file does not contain the name for a particular service you use in your infrastructure, you will need to add it to the `[service name]` section. @@ -245,6 +275,21 @@ service in network connection charts, and thus see the name of the service inste 19999 = Netdata ``` +### Sync configuration + +The sync configuration has specific options to disable monitoring for syscalls, as default option all syscalls are +monitored. + +```conf +[syscalls] + sync = yes + msync = yes + fsync = yes + fdatasync = yes + syncfs = yes + sync_file_range = yes +``` + ## Troubleshooting If the eBPF collector does not work, you can troubleshoot it by running the `ebpf.plugin` command and investigating its diff --git a/collectors/ebpf.plugin/ebpf.c b/collectors/ebpf.plugin/ebpf.c index 26bcfcf17..26dacfd3e 100644 --- a/collectors/ebpf.plugin/ebpf.c +++ b/collectors/ebpf.plugin/ebpf.c @@ -52,8 +52,6 @@ void netdata_cleanup_and_exit(int ret) *****************************************************************/ char *ebpf_plugin_dir = PLUGINS_DIR; -char *ebpf_user_config_dir = CONFIG_DIR; -char *ebpf_stock_config_dir = LIBCONFIG_DIR; static char *ebpf_configured_log_dir = LOG_DIR; char *ebpf_algorithms[] = {"absolute", "incremental"}; @@ -79,13 +77,19 @@ pthread_cond_t collect_data_cond_var; ebpf_module_t ebpf_modules[] = { { .thread_name = "process", .config_name = "process", .enabled = 0, .start_routine = ebpf_process_thread, .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY, - .optional = 0 }, + .optional = 0, .apps_routine = ebpf_process_create_apps_charts }, { .thread_name = "socket", .config_name = "socket", .enabled = 0, .start_routine = ebpf_socket_thread, .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY, - .optional = 0 }, + .optional = 0, .apps_routine = ebpf_socket_create_apps_charts }, + { .thread_name = "cachestat", .config_name = "cachestat", .enabled = 0, .start_routine = ebpf_cachestat_thread, + .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY, + .optional = 0, .apps_routine = ebpf_cachestat_create_apps_charts }, + { .thread_name = "sync", .config_name = "sync", .enabled = 0, .start_routine = ebpf_sync_thread, + .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY, + .optional = 0, .apps_routine = NULL }, { .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_time = 1, .global_charts = 0, .apps_charts = 1, .mode = MODE_ENTRY, - .optional = 0 }, + .optional = 0, .apps_routine = NULL }, }; // Link with apps.plugin @@ -101,58 +105,6 @@ ebpf_network_viewer_options_t network_viewer_opt; *****************************************************************/ /** - * Cleanup publish syscall - * - * @param nps list of structures to clean - */ -void ebpf_cleanup_publish_syscall(netdata_publish_syscall_t *nps) -{ - while (nps) { - freez(nps->algorithm); - nps = nps->next; - } -} - -/** - * Clean port Structure - * - * Clean the allocated list. - * - * @param clean the list that will be cleaned - */ -void clean_port_structure(ebpf_network_viewer_port_list_t **clean) -{ - ebpf_network_viewer_port_list_t *move = *clean; - while (move) { - ebpf_network_viewer_port_list_t *next = move->next; - freez(move->value); - freez(move); - - move = next; - } - *clean = NULL; -} - -/** - * Clean IP structure - * - * Clean the allocated list. - * - * @param clean the list that will be cleaned - */ -static void clean_ip_structure(ebpf_network_viewer_ip_list_t **clean) -{ - ebpf_network_viewer_ip_list_t *move = *clean; - while (move) { - ebpf_network_viewer_ip_list_t *next = move->next; - freez(move); - - move = next; - } - *clean = NULL; -} - -/** * Clean Loaded Events * * This function cleans the events previous loaded on Linux. @@ -319,6 +271,25 @@ void write_err_chart(char *name, char *family, netdata_publish_syscall_t *move, } /** + * Write charts + * + * Write the current information to publish the charts. + * + * @param family chart family + * @param chart chart id + * @param dim dimension name + * @param v1 value. + */ +void ebpf_one_dimension_write_charts(char *family, char *chart, char *dim, long long v1) +{ + write_begin_chart(family, chart); + + write_chart_dimension(dim, v1); + + write_end_chart(); +} + +/** * Call the necessary functions to create a chart. * * @param chart the chart name @@ -343,23 +314,26 @@ void write_io_chart(char *chart, char *family, char *dwrite, long long vwrite, c /** * Write chart cmd on standard output * - * @param type the chart type - * @param id the chart id - * @param title the chart title - * @param units the units label - * @param family the group name used to attach the chart on dashaboard - * @param charttype the chart type - * @param order the chart order + * @param type chart type + * @param id chart id + * @param title chart title + * @param units units label + * @param family group name used to attach the chart on dashaboard + * @param charttype chart type + * @param context chart context + * @param order chart order */ -void ebpf_write_chart_cmd(char *type, char *id, char *title, char *units, char *family, char *charttype, int order) +void ebpf_write_chart_cmd(char *type, char *id, char *title, char *units, char *family, + char *charttype, char *context, int order) { - printf("CHART %s.%s '' '%s' '%s' '%s' '' %s %d %d\n", + printf("CHART %s.%s '' '%s' '%s' '%s' '%s' '%s' %d %d\n", type, id, title, units, - family, - charttype, + (family)?family:"", + (context)?context:"", + (charttype)?charttype:"", order, update_every); } @@ -398,26 +372,31 @@ void ebpf_create_global_dimension(void *ptr, int end) /** * Call write_chart_cmd to create the charts * - * @param type the chart type - * @param id the chart id - * @param units the axis label - * @param family the group name used to attach the chart on dashaboard - * @param order the order number of the specified chart - * @param ncd a pointer to a function called to create dimensions - * @param move a pointer for a structure that has the dimensions - * @param end number of dimensions for the chart created + * @param type chart type + * @param id chart id + * @param title chart title + * @param units axis label + * @param family group name used to attach the chart on dashaboard + * @param context chart context + * @param charttype chart type + * @param order order number of the specified chart + * @param ncd a pointer to a function called to create dimensions + * @param move a pointer for a structure that has the dimensions + * @param end number of dimensions for the chart created */ void ebpf_create_chart(char *type, char *id, char *title, char *units, char *family, + char *context, + char *charttype, int order, void (*ncd)(void *, int), void *move, int end) { - ebpf_write_chart_cmd(type, id, title, units, family, "line", order); + ebpf_write_chart_cmd(type, id, title, units, family, charttype, context, order); ncd(move, end); } @@ -429,15 +408,16 @@ void ebpf_create_chart(char *type, * @param title the value displayed on vertical axis. * @param units the value displayed on vertical axis. * @param family Submenu that the chart will be attached on dashboard. + * @param charttype chart type * @param order the chart order * @param algorithm the algorithm used by dimension * @param root structure used to create the dimensions. */ -void ebpf_create_charts_on_apps(char *id, char *title, char *units, char *family, int order, +void ebpf_create_charts_on_apps(char *id, char *title, char *units, char *family, char *charttype, int order, char *algorithm, struct target *root) { struct target *w; - ebpf_write_chart_cmd(NETDATA_APPS_FAMILY, id, title, units, family, "stacked", order); + ebpf_write_chart_cmd(NETDATA_APPS_FAMILY, id, title, units, family, charttype, NULL, order); for (w = root; w; w = w->next) { if (unlikely(w->exposed)) @@ -580,22 +560,26 @@ void ebpf_print_help() "\n" " Available command line options:\n" "\n" - " SECONDS set the data collection frequency.\n" + " SECONDS Set the data collection frequency.\n" "\n" - " --help or -h show this help.\n" + " --help or -h Show this help.\n" "\n" - " --version or -v show software version.\n" + " --version or -v Show software version.\n" "\n" - " --global or -g disable charts per application.\n" + " --global or -g Disable charts per application.\n" "\n" - " --all or -a Enable all chart groups (global and apps), unless -g is also given.\n" + " --all or -a Enable all chart groups (global and apps), unless -g is also given.\n" "\n" - " --net or -n Enable network viewer charts.\n" + " --cachestat or -c Enable charts related to process run time.\n" "\n" - " --process or -p Enable charts related to process run time.\n" + " --net or -n Enable network viewer charts.\n" "\n" - " --return or -r Run the collector in return mode.\n" + " --process or -p Enable charts related to process run time.\n" + "\n" + " --return or -r Run the collector in return mode.\n" "\n", + " --sync or -s Enable chart related to sync run time.\n" + "\n" VERSION, (year >= 116) ? year + 1900 : 2020); } @@ -607,87 +591,6 @@ void ebpf_print_help() *****************************************************************/ /** - * Is ip inside the range - * - * Check if the ip is inside a IP range - * - * @param rfirst the first ip address of the range - * @param rlast the last ip address of the range - * @param cmpfirst the first ip to compare - * @param cmplast the last ip to compare - * @param family the IP family - * - * @return It returns 1 if the IP is inside the range and 0 otherwise - */ -static int is_ip_inside_range(union netdata_ip_t *rfirst, union netdata_ip_t *rlast, - union netdata_ip_t *cmpfirst, union netdata_ip_t *cmplast, int family) -{ - if (family == AF_INET) { - if (ntohl(rfirst->addr32[0]) <= ntohl(cmpfirst->addr32[0]) && - ntohl(rlast->addr32[0]) >= ntohl(cmplast->addr32[0])) - return 1; - } else { - if (memcmp(rfirst->addr8, cmpfirst->addr8, sizeof(union netdata_ip_t)) <= 0 && - memcmp(rlast->addr8, cmplast->addr8, sizeof(union netdata_ip_t)) >= 0) { - return 1; - } - - } - return 0; -} - - -/** - * Fill IP list - * - * @param out a pointer to the link list. - * @param in the structure that will be linked. - */ -static inline void fill_ip_list(ebpf_network_viewer_ip_list_t **out, ebpf_network_viewer_ip_list_t *in, char *table) -{ -#ifndef NETDATA_INTERNAL_CHECKS - UNUSED(table); -#endif - if (likely(*out)) { - ebpf_network_viewer_ip_list_t *move = *out, *store = *out; - while (move) { - if (in->ver == move->ver && is_ip_inside_range(&move->first, &move->last, &in->first, &in->last, in->ver)) { - info("The range/value (%s) is inside the range/value (%s) already inserted, it will be ignored.", - in->value, move->value); - freez(in->value); - freez(in); - return; - } - store = move; - move = move->next; - } - - store->next = in; - } else { - *out = in; - } - -#ifdef NETDATA_INTERNAL_CHECKS - char first[512], last[512]; - if (in->ver == AF_INET) { - if (inet_ntop(AF_INET, in->first.addr8, first, INET_ADDRSTRLEN) && - inet_ntop(AF_INET, in->last.addr8, last, INET_ADDRSTRLEN)) - info("Adding values %s - %s to %s IP list \"%s\" used on network viewer", - first, last, - (*out == network_viewer_opt.included_ips)?"included":"excluded", - table); - } else { - if (inet_ntop(AF_INET6, in->first.addr8, first, INET6_ADDRSTRLEN) && - inet_ntop(AF_INET6, in->last.addr8, last, INET6_ADDRSTRLEN)) - info("Adding values %s - %s to %s IP list \"%s\" used on network viewer", - first, last, - (*out == network_viewer_opt.included_ips)?"included":"excluded", - table); - } -#endif -} - -/** * Read Local Ports * * Parse /proc/net/{tcp,udp} and get the ports Linux is listening. @@ -838,789 +741,26 @@ void fill_ebpf_data(ebpf_data_t *ef) */ static inline void how_to_load(char *ptr) { - if (!strcasecmp(ptr, "return")) + if (!strcasecmp(ptr, EBPF_CFG_LOAD_MODE_RETURN)) ebpf_set_thread_mode(MODE_RETURN); - else if (!strcasecmp(ptr, "entry")) + else if (!strcasecmp(ptr, EBPF_CFG_LOAD_MODE_DEFAULT)) ebpf_set_thread_mode(MODE_ENTRY); else error("the option %s for \"ebpf load mode\" is not a valid option.", ptr); } /** - * Fill Port list - * - * @param out a pointer to the link list. - * @param in the structure that will be linked. - */ -static inline void fill_port_list(ebpf_network_viewer_port_list_t **out, ebpf_network_viewer_port_list_t *in) -{ - if (likely(*out)) { - ebpf_network_viewer_port_list_t *move = *out, *store = *out; - uint16_t first = ntohs(in->first); - uint16_t last = ntohs(in->last); - while (move) { - uint16_t cmp_first = ntohs(move->first); - uint16_t cmp_last = ntohs(move->last); - if (cmp_first <= first && first <= cmp_last && - cmp_first <= last && last <= cmp_last ) { - info("The range/value (%u, %u) is inside the range/value (%u, %u) already inserted, it will be ignored.", - first, last, cmp_first, cmp_last); - freez(in->value); - freez(in); - return; - } else if (first <= cmp_first && cmp_first <= last && - first <= cmp_last && cmp_last <= last) { - info("The range (%u, %u) is bigger than previous range (%u, %u) already inserted, the previous will be ignored.", - first, last, cmp_first, cmp_last); - freez(move->value); - move->value = in->value; - move->first = in->first; - move->last = in->last; - freez(in); - return; - } - - store = move; - move = move->next; - } - - store->next = in; - } else { - *out = in; - } - -#ifdef NETDATA_INTERNAL_CHECKS - info("Adding values %s( %u, %u) to %s port list used on network viewer", - in->value, ntohs(in->first), ntohs(in->last), - (*out == network_viewer_opt.included_port)?"included":"excluded"); -#endif -} - -/** - * Fill port list - * - * Fill an allocated port list with the range given - * - * @param out a pointer to store the link list - * @param range the informed range for the user. - */ -static void parse_port_list(void **out, char *range) -{ - int first, last; - ebpf_network_viewer_port_list_t **list = (ebpf_network_viewer_port_list_t **)out; - - char *copied = strdupz(range); - if (*range == '*' && *(range+1) == '\0') { - first = 1; - last = 65535; - - clean_port_structure(list); - goto fillenvpl; - } - - char *end = range; - //Move while I cannot find a separator - while (*end && *end != ':' && *end != '-') end++; - - //It has a range - if (likely(*end)) { - *end++ = '\0'; - if (*end == '!') { - info("The exclusion cannot be in the second part of the range, the range %s will be ignored.", copied); - freez(copied); - return; - } - last = str2i((const char *)end); - } else { - last = 0; - } - - first = str2i((const char *)range); - if (first < NETDATA_MINIMUM_PORT_VALUE || first > NETDATA_MAXIMUM_PORT_VALUE) { - info("The first port %d of the range \"%s\" is invalid and it will be ignored!", first, copied); - freez(copied); - return; - } - - if (!last) - last = first; - - if (last < NETDATA_MINIMUM_PORT_VALUE || last > NETDATA_MAXIMUM_PORT_VALUE) { - info("The second port %d of the range \"%s\" is invalid and the whole range will be ignored!", last, copied); - freez(copied); - return; - } - - if (first > last) { - info("The specified order %s is wrong, the smallest value is always the first, it will be ignored!", copied); - freez(copied); - return; - } - - ebpf_network_viewer_port_list_t *w; -fillenvpl: - w = callocz(1, sizeof(ebpf_network_viewer_port_list_t)); - w->value = copied; - w->hash = simple_hash(copied); - w->first = (uint16_t)htons((uint16_t)first); - w->last = (uint16_t)htons((uint16_t)last); - w->cmp_first = (uint16_t)first; - w->cmp_last = (uint16_t)last; - - fill_port_list(list, w); -} - -/** - * Parse Service List - * - * @param out a pointer to store the link list - * @param service the service used to create the structure that will be linked. - */ -static void parse_service_list(void **out, char *service) -{ - ebpf_network_viewer_port_list_t **list = (ebpf_network_viewer_port_list_t **)out; - struct servent *serv = getservbyname((const char *)service, "tcp"); - if (!serv) - serv = getservbyname((const char *)service, "udp"); - - if (!serv) { - info("Cannot resolv the service '%s' with protocols TCP and UDP, it will be ignored", service); - return; - } - - ebpf_network_viewer_port_list_t *w = callocz(1, sizeof(ebpf_network_viewer_port_list_t)); - w->value = strdupz(service); - w->hash = simple_hash(service); - - w->first = w->last = (uint16_t)serv->s_port; - - fill_port_list(list, w); -} - -/** - * Netmask - * - * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h) - * - * @param prefix create the netmask based in the CIDR value. - * - * @return - */ -static inline in_addr_t netmask(int prefix) { - - if (prefix == 0) - return (~((in_addr_t) - 1)); - else - return (in_addr_t)(~((1 << (32 - prefix)) - 1)); - -} - -/** - * Broadcast - * - * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h) - * - * @param addr is the ip address - * @param prefix is the CIDR value. - * - * @return It returns the last address of the range - */ -static inline in_addr_t broadcast(in_addr_t addr, int prefix) -{ - return (addr | ~netmask(prefix)); -} - -/** - * Network - * - * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h) - * - * @param addr is the ip address - * @param prefix is the CIDR value. - * - * @return It returns the first address of the range. - */ -static inline in_addr_t ipv4_network(in_addr_t addr, int prefix) -{ - return (addr & netmask(prefix)); -} - -/** - * IP to network long - * - * @param dst the vector to store the result - * @param ip the source ip given by our users. - * @param domain the ip domain (IPV4 or IPV6) - * @param source the original string - * - * @return it returns 0 on success and -1 otherwise. - */ -static inline int ip2nl(uint8_t *dst, char *ip, int domain, char *source) -{ - if (inet_pton(domain, ip, dst) <= 0) { - error("The address specified (%s) is invalid ", source); - return -1; - } - - return 0; -} - -/** - * Get IPV6 Last Address + * Update interval * - * @param out the address to store the last address. - * @param in the address used to do the math. - * @param prefix number of bits used to calculate the address + * Update default interval with value from user */ -static void get_ipv6_last_addr(union netdata_ip_t *out, union netdata_ip_t *in, uint64_t prefix) +static void ebpf_update_interval() { - uint64_t mask,tmp; - uint64_t ret[2]; - memcpy(ret, in->addr32, sizeof(union netdata_ip_t)); - - if (prefix == 128) { - memcpy(out->addr32, in->addr32, sizeof(union netdata_ip_t)); - return; - } else if (!prefix) { - ret[0] = ret[1] = 0xFFFFFFFFFFFFFFFF; - memcpy(out->addr32, ret, sizeof(union netdata_ip_t)); - return; - } else if (prefix <= 64) { - ret[1] = 0xFFFFFFFFFFFFFFFFULL; - - tmp = be64toh(ret[0]); - if (prefix > 0) { - mask = 0xFFFFFFFFFFFFFFFFULL << (64 - prefix); - tmp |= ~mask; - } - ret[0] = htobe64(tmp); - } else { - mask = 0xFFFFFFFFFFFFFFFFULL << (128 - prefix); - tmp = be64toh(ret[1]); - tmp |= ~mask; - ret[1] = htobe64(tmp); - } - - memcpy(out->addr32, ret, sizeof(union netdata_ip_t)); -} - -/** - * Calculate ipv6 first address - * - * @param out the address to store the first address. - * @param in the address used to do the math. - * @param prefix number of bits used to calculate the address - */ -static void get_ipv6_first_addr(union netdata_ip_t *out, union netdata_ip_t *in, uint64_t prefix) -{ - uint64_t mask,tmp; - uint64_t ret[2]; - - memcpy(ret, in->addr32, sizeof(union netdata_ip_t)); - - if (prefix == 128) { - memcpy(out->addr32, in->addr32, sizeof(union netdata_ip_t)); - return; - } else if (!prefix) { - ret[0] = ret[1] = 0; - memcpy(out->addr32, ret, sizeof(union netdata_ip_t)); - return; - } else if (prefix <= 64) { - ret[1] = 0ULL; - - tmp = be64toh(ret[0]); - if (prefix > 0) { - mask = 0xFFFFFFFFFFFFFFFFULL << (64 - prefix); - tmp &= mask; - } - ret[0] = htobe64(tmp); - } else { - mask = 0xFFFFFFFFFFFFFFFFULL << (128 - prefix); - tmp = be64toh(ret[1]); - tmp &= mask; - ret[1] = htobe64(tmp); - } - - memcpy(out->addr32, ret, sizeof(union netdata_ip_t)); -} - -/** - * Parse IP List - * - * Parse IP list and link it. - * - * @param out a pointer to store the link list - * @param ip the value given as parameter - */ -static void parse_ip_list(void **out, char *ip) -{ - ebpf_network_viewer_ip_list_t **list = (ebpf_network_viewer_ip_list_t **)out; - - char *ipdup = strdupz(ip); - union netdata_ip_t first = { }; - union netdata_ip_t last = { }; - char *is_ipv6; - if (*ip == '*' && *(ip+1) == '\0') { - memset(first.addr8, 0, sizeof(first.addr8)); - memset(last.addr8, 0xFF, sizeof(last.addr8)); - - is_ipv6 = ip; - - clean_ip_structure(list); - goto storethisip; - } - - char *end = ip; - // Move while I cannot find a separator - while (*end && *end != '/' && *end != '-') end++; - - // We will use only the classic IPV6 for while, but we could consider the base 85 in a near future - // https://tools.ietf.org/html/rfc1924 - is_ipv6 = strchr(ip, ':'); - - int select; - if (*end && !is_ipv6) { // IPV4 range - select = (*end == '/') ? 0 : 1; - *end++ = '\0'; - if (*end == '!') { - info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup); - goto cleanipdup; - } - - if (!select) { // CIDR - select = ip2nl(first.addr8, ip, AF_INET, ipdup); - if (select) - goto cleanipdup; - - select = (int) str2i(end); - if (select < NETDATA_MINIMUM_IPV4_CIDR || select > NETDATA_MAXIMUM_IPV4_CIDR) { - info("The specified CIDR %s is not valid, the IP %s will be ignored.", end, ip); - goto cleanipdup; - } - - last.addr32[0] = htonl(broadcast(ntohl(first.addr32[0]), select)); - // This was added to remove - // https://app.codacy.com/manual/netdata/netdata/pullRequest?prid=5810941&bid=19021977 - UNUSED(last.addr32[0]); - - uint32_t ipv4_test = htonl(ipv4_network(ntohl(first.addr32[0]), select)); - if (first.addr32[0] != ipv4_test) { - first.addr32[0] = ipv4_test; - struct in_addr ipv4_convert; - ipv4_convert.s_addr = ipv4_test; - char ipv4_msg[INET_ADDRSTRLEN]; - if(inet_ntop(AF_INET, &ipv4_convert, ipv4_msg, INET_ADDRSTRLEN)) - info("The network value of CIDR %s was updated for %s .", ipdup, ipv4_msg); - } - } else { // Range - select = ip2nl(first.addr8, ip, AF_INET, ipdup); - if (select) - goto cleanipdup; - - select = ip2nl(last.addr8, end, AF_INET, ipdup); - if (select) - goto cleanipdup; - } - - if (htonl(first.addr32[0]) > htonl(last.addr32[0])) { - info("The specified range %s is invalid, the second address is smallest than the first, it will be ignored.", - ipdup); - goto cleanipdup; - } - } else if (is_ipv6) { // IPV6 - if (!*end) { // Unique - select = ip2nl(first.addr8, ip, AF_INET6, ipdup); - if (select) - goto cleanipdup; - - memcpy(last.addr8, first.addr8, sizeof(first.addr8)); - } else if (*end == '-') { - *end++ = 0x00; - if (*end == '!') { - info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup); - goto cleanipdup; - } - - select = ip2nl(first.addr8, ip, AF_INET6, ipdup); - if (select) - goto cleanipdup; - - select = ip2nl(last.addr8, end, AF_INET6, ipdup); - if (select) - goto cleanipdup; - } else { // CIDR - *end++ = 0x00; - if (*end == '!') { - info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup); - goto cleanipdup; - } - - select = str2i(end); - if (select < 0 || select > 128) { - info("The CIDR %s is not valid, the address %s will be ignored.", end, ip); - goto cleanipdup; - } - - uint64_t prefix = (uint64_t)select; - select = ip2nl(first.addr8, ip, AF_INET6, ipdup); - if (select) - goto cleanipdup; - - get_ipv6_last_addr(&last, &first, prefix); - - union netdata_ip_t ipv6_test; - get_ipv6_first_addr(&ipv6_test, &first, prefix); - - if (memcmp(first.addr8, ipv6_test.addr8, sizeof(union netdata_ip_t)) != 0) { - memcpy(first.addr8, ipv6_test.addr8, sizeof(union netdata_ip_t)); - - struct in6_addr ipv6_convert; - memcpy(ipv6_convert.s6_addr, ipv6_test.addr8, sizeof(union netdata_ip_t)); - - char ipv6_msg[INET6_ADDRSTRLEN]; - if(inet_ntop(AF_INET6, &ipv6_convert, ipv6_msg, INET6_ADDRSTRLEN)) - info("The network value of CIDR %s was updated for %s .", ipdup, ipv6_msg); - } - } - - if ((be64toh(*(uint64_t *)&first.addr32[2]) > be64toh(*(uint64_t *)&last.addr32[2]) && - !memcmp(first.addr32, last.addr32, 2*sizeof(uint32_t))) || - (be64toh(*(uint64_t *)&first.addr32) > be64toh(*(uint64_t *)&last.addr32)) ) { - info("The specified range %s is invalid, the second address is smallest than the first, it will be ignored.", - ipdup); - goto cleanipdup; - } - } else { // Unique ip - select = ip2nl(first.addr8, ip, AF_INET, ipdup); - if (select) - goto cleanipdup; - - memcpy(last.addr8, first.addr8, sizeof(first.addr8)); - } - - ebpf_network_viewer_ip_list_t *store; - -storethisip: - store = callocz(1, sizeof(ebpf_network_viewer_ip_list_t)); - store->value = ipdup; - store->hash = simple_hash(ipdup); - store->ver = (uint8_t)(!is_ipv6)?AF_INET:AF_INET6; - memcpy(store->first.addr8, first.addr8, sizeof(first.addr8)); - memcpy(store->last.addr8, last.addr8, sizeof(last.addr8)); - - fill_ip_list(list, store, "socket"); - return; - -cleanipdup: - freez(ipdup); -} - -/** - * Parse IP Range - * - * Parse the IP ranges given and create Network Viewer IP Structure - * - * @param ptr is a pointer with the text to parse. - */ -static void parse_ips(char *ptr) -{ - // No value - if (unlikely(!ptr)) - return; - - while (likely(ptr)) { - // Move forward until next valid character - while (isspace(*ptr)) ptr++; - - // No valid value found - if (unlikely(!*ptr)) - return; - - // Find space that ends the list - char *end = strchr(ptr, ' '); - if (end) { - *end++ = '\0'; - } - - int neg = 0; - if (*ptr == '!') { - neg++; - ptr++; - } - - if (isascii(*ptr)) { // Parse port - parse_ip_list((!neg)?(void **)&network_viewer_opt.included_ips:(void **)&network_viewer_opt.excluded_ips, - ptr); - } - - ptr = end; - } -} - - -/** - * Parse Port Range - * - * Parse the port ranges given and create Network Viewer Port Structure - * - * @param ptr is a pointer with the text to parse. - */ -static void parse_ports(char *ptr) -{ - // No value - if (unlikely(!ptr)) - return; - - while (likely(ptr)) { - // Move forward until next valid character - while (isspace(*ptr)) ptr++; - - // No valid value found - if (unlikely(!*ptr)) - return; - - // Find space that ends the list - char *end = strchr(ptr, ' '); - if (end) { - *end++ = '\0'; - } - - int neg = 0; - if (*ptr == '!') { - neg++; - ptr++; - } - - if (isdigit(*ptr)) { // Parse port - parse_port_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port, - ptr); - } else if (isalpha(*ptr)) { // Parse service - parse_service_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port, - ptr); - } else if (*ptr == '*') { // All - parse_port_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port, - ptr); - } - - ptr = end; - } -} - -/** - * Link hostname - * - * @param out is the output link list - * @param in the hostname to add to list. - */ -static void link_hostname(ebpf_network_viewer_hostname_list_t **out, ebpf_network_viewer_hostname_list_t *in) -{ - if (likely(*out)) { - ebpf_network_viewer_hostname_list_t *move = *out; - for (; move->next ; move = move->next ) { - if (move->hash == in->hash && !strcmp(move->value, in->value)) { - info("The hostname %s was already inserted, it will be ignored.", in->value); - freez(in->value); - simple_pattern_free(in->value_pattern); - freez(in); - return; - } - } - - move->next = in; - } else { - *out = in; - } -#ifdef NETDATA_INTERNAL_CHECKS - info("Adding value %s to %s hostname list used on network viewer", - in->value, - (*out == network_viewer_opt.included_hostnames)?"included":"excluded"); -#endif -} - -/** - * Link Hostnames - * - * Parse the list of hostnames to create the link list. - * This is not associated with the IP, because simple patterns like *example* cannot be resolved to IP. - * - * @param out is the output link list - * @param parse is a pointer with the text to parser. - */ -static void link_hostnames(char *parse) -{ - // No value - if (unlikely(!parse)) - return; - - while (likely(parse)) { - // Find the first valid value - while (isspace(*parse)) parse++; - - // No valid value found - if (unlikely(!*parse)) - return; - - // Find space that ends the list - char *end = strchr(parse, ' '); - if (end) { - *end++ = '\0'; - } - - int neg = 0; - if (*parse == '!') { - neg++; - parse++; - } - - ebpf_network_viewer_hostname_list_t *hostname = callocz(1 , sizeof(ebpf_network_viewer_hostname_list_t)); - hostname->value = strdupz(parse); - hostname->hash = simple_hash(parse); - hostname->value_pattern = simple_pattern_create(parse, NULL, SIMPLE_PATTERN_EXACT); - - link_hostname((!neg)?&network_viewer_opt.included_hostnames:&network_viewer_opt.excluded_hostnames, - hostname); - - parse = end; - } -} - -/** - * Read max dimension. - * - * Netdata plot two dimensions per connection, so it is necessary to adjust the values. - */ -static void read_max_dimension() -{ - int maxdim ; - maxdim = (int) appconfig_get_number(&collector_config, - EBPF_NETWORK_VIEWER_SECTION, - "maximum dimensions", - NETDATA_NV_CAP_VALUE); - if (maxdim < 0) { - error("'maximum dimensions = %d' must be a positive number, Netdata will change for default value %ld.", - maxdim, NETDATA_NV_CAP_VALUE); - maxdim = NETDATA_NV_CAP_VALUE; - } - - maxdim /= 2; - if (!maxdim) { - info("The number of dimensions is too small (%u), we are setting it to minimum 2", network_viewer_opt.max_dim); - network_viewer_opt.max_dim = 1; - } - - network_viewer_opt.max_dim = (uint32_t)maxdim; -} - -/** - * Parse network viewer section - */ -static void parse_network_viewer_section() -{ - read_max_dimension(); - - network_viewer_opt.hostname_resolution_enabled = appconfig_get_boolean(&collector_config, - EBPF_NETWORK_VIEWER_SECTION, - "resolve hostnames", - CONFIG_BOOLEAN_NO); - - network_viewer_opt.service_resolution_enabled = appconfig_get_boolean(&collector_config, - EBPF_NETWORK_VIEWER_SECTION, - "resolve service names", - CONFIG_BOOLEAN_NO); - - char *value = appconfig_get(&collector_config, EBPF_NETWORK_VIEWER_SECTION, - "ports", NULL); - parse_ports(value); - - if (network_viewer_opt.hostname_resolution_enabled) { - value = appconfig_get(&collector_config, EBPF_NETWORK_VIEWER_SECTION, "hostnames", NULL); - link_hostnames(value); - } else { - info("Name resolution is disabled, collector will not parser \"hostnames\" list."); - } - - value = appconfig_get(&collector_config, EBPF_NETWORK_VIEWER_SECTION, - "ips", "!127.0.0.1/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 fc00::/7 !::1/128"); - parse_ips(value); -} - -/** - * Link dimension name - * - * Link user specified names inside a link list. - * - * @param port the port number associated to the dimension name. - * @param hash the calculated hash for the dimension name. - * @param name the dimension name. - */ -static void link_dimension_name(char *port, uint32_t hash, char *value) -{ - int test = str2i(port); - if (test < NETDATA_MINIMUM_PORT_VALUE || test > NETDATA_MAXIMUM_PORT_VALUE){ - error("The dimension given (%s = %s) has an invalid value and it will be ignored.", port, value); - return; - } - - ebpf_network_viewer_dim_name_t *w; - w = callocz(1, sizeof(ebpf_network_viewer_dim_name_t)); - - w->name = strdupz(value); - w->hash = hash; - - w->port = (uint16_t) htons(test); - - ebpf_network_viewer_dim_name_t *names = network_viewer_opt.names; - if (unlikely(!names)) { - network_viewer_opt.names = w; - } else { - for (; names->next; names = names->next) { - if (names->port == w->port) { - info("Dupplicated definition for a service, the name %s will be ignored. ", names->name); - freez(names->name); - names->name = w->name; - names->hash = w->hash; - freez(w); - return; - } - } - names->next = w; - } - -#ifdef NETDATA_INTERNAL_CHECKS - info("Adding values %s( %u) to dimension name list used on network viewer", w->name, htons(w->port)); -#endif -} - -/** - * Parse service Name section. - * - * This function gets the values that will be used to overwrite dimensions. - */ -static void parse_service_name_section() -{ - struct section *co = appconfig_get_section(&collector_config, EBPF_SERVICE_NAME_SECTION); - if (co) { - struct config_option *cv; - for (cv = co->values; cv ; cv = cv->next) { - link_dimension_name(cv->name, cv->hash, cv->value); - } - } - - // Always associated the default port to Netdata - ebpf_network_viewer_dim_name_t *names = network_viewer_opt.names; - if (names) { - uint16_t default_port = htons(19999); - while (names) { - if (names->port == default_port) - return; - - names = names->next; - } + int i; + int value = (int) appconfig_get_number(&collector_config, EBPF_GLOBAL_SECTION, EBPF_CFG_UPDATE_EVERY, 1); + for (i = 0; ebpf_modules[i].thread_name; i++) { + ebpf_modules[i].update_time = value; } - - char *port_string = getenv("NETDATA_LISTEN_PORT"); - if (port_string) - link_dimension_name(port_string, simple_hash(port_string), "Netdata"); } /** @@ -1633,18 +773,22 @@ static void read_collector_values(int *disable_apps) // Read global section char *value; if (appconfig_exists(&collector_config, EBPF_GLOBAL_SECTION, "load")) // Backward compatibility - value = appconfig_get(&collector_config, EBPF_GLOBAL_SECTION, "load", "entry"); + value = appconfig_get(&collector_config, EBPF_GLOBAL_SECTION, "load", + EBPF_CFG_LOAD_MODE_DEFAULT); else - value = appconfig_get(&collector_config, EBPF_GLOBAL_SECTION, "ebpf load mode", "entry"); + value = appconfig_get(&collector_config, EBPF_GLOBAL_SECTION, EBPF_CFG_LOAD_MODE, + EBPF_CFG_LOAD_MODE_DEFAULT); how_to_load(value); + ebpf_update_interval(); + // This is kept to keep compatibility uint32_t enabled = appconfig_get_boolean(&collector_config, EBPF_GLOBAL_SECTION, "disable apps", CONFIG_BOOLEAN_NO); if (!enabled) { // Apps is a positive sentence, so we need to invert the values to disable apps. - enabled = appconfig_get_boolean(&collector_config, EBPF_GLOBAL_SECTION, "apps", + enabled = appconfig_get_boolean(&collector_config, EBPF_GLOBAL_SECTION, EBPF_CFG_APPLICATION, CONFIG_BOOLEAN_YES); enabled = (enabled == CONFIG_BOOLEAN_NO)?CONFIG_BOOLEAN_YES:CONFIG_BOOLEAN_NO; } @@ -1652,7 +796,7 @@ static void read_collector_values(int *disable_apps) // Read ebpf programs section enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, - ebpf_modules[0].config_name, CONFIG_BOOLEAN_YES); + ebpf_modules[EBPF_MODULE_PROCESS_IDX].config_name, CONFIG_BOOLEAN_YES); int started = 0; if (enabled) { ebpf_enable_chart(EBPF_MODULE_PROCESS_IDX, *disable_apps); @@ -1663,14 +807,16 @@ static void read_collector_values(int *disable_apps) enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "network viewer", CONFIG_BOOLEAN_NO); if (!enabled) - enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, ebpf_modules[1].config_name, + enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, + ebpf_modules[EBPF_MODULE_SOCKET_IDX].config_name, CONFIG_BOOLEAN_NO); if (enabled) { ebpf_enable_chart(EBPF_MODULE_SOCKET_IDX, *disable_apps); // Read network viewer section if network viewer is enabled - parse_network_viewer_section(); - parse_service_name_section(); + // This is kept here to keep backward compatibility + parse_network_viewer_section(&collector_config); + parse_service_name_section(&collector_config); started++; } @@ -1680,13 +826,30 @@ static void read_collector_values(int *disable_apps) if (!enabled) enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "network connections", CONFIG_BOOLEAN_NO); - ebpf_modules[1].optional = enabled; + ebpf_modules[EBPF_MODULE_SOCKET_IDX].optional = enabled; + + enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "cachestat", + CONFIG_BOOLEAN_NO); + + if (enabled) { + ebpf_enable_chart(EBPF_MODULE_CACHESTAT_IDX, *disable_apps); + started++; + } + + enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "sync", + CONFIG_BOOLEAN_YES); + + if (enabled) { + ebpf_enable_chart(EBPF_MODULE_SYNC_IDX, *disable_apps); + started++; + } if (!started){ ebpf_enable_all_charts(*disable_apps); // Read network viewer section - parse_network_viewer_section(); - parse_service_name_section(); + // This is kept here to keep backward compatibility + parse_network_viewer_section(&collector_config); + parse_service_name_section(&collector_config); } } @@ -1702,10 +865,13 @@ static int load_collector_config(char *path, int *disable_apps) { char lpath[4096]; - snprintf(lpath, 4095, "%s/%s", path, "ebpf.conf"); - - if (!appconfig_load(&collector_config, lpath, 0, NULL)) - return -1; + snprintf(lpath, 4095, "%s/%s", path, NETDATA_EBPF_CONFIG_FILE); + if (!appconfig_load(&collector_config, lpath, 0, NULL)) { + snprintf(lpath, 4095, "%s/%s", path, NETDATA_EBPF_OLD_CONFIG_FILE); + if (!appconfig_load(&collector_config, lpath, 0, NULL)) { + return -1; + } + } read_collector_values(disable_apps); @@ -1756,13 +922,15 @@ static void parse_args(int argc, char **argv) int freq = 0; int option_index = 0; static struct option long_options[] = { - {"help", no_argument, 0, 'h' }, - {"version", no_argument, 0, 'v' }, - {"global", no_argument, 0, 'g' }, - {"all", no_argument, 0, 'a' }, - {"net", no_argument, 0, 'n' }, - {"process", no_argument, 0, 'p' }, - {"return", no_argument, 0, 'r' }, + {"help", no_argument, 0, 'h' }, + {"version", no_argument, 0, 'v' }, + {"global", no_argument, 0, 'g' }, + {"all", no_argument, 0, 'a' }, + {"cachestat", no_argument, 0, 'c' }, + {"net", no_argument, 0, 'n' }, + {"process", no_argument, 0, 'p' }, + {"return", no_argument, 0, 'r' }, + {"sync", no_argument, 0, 's' }, {0, 0, 0, 0} }; @@ -1777,7 +945,7 @@ static void parse_args(int argc, char **argv) } while (1) { - int c = getopt_long(argc, argv, "hvganpr", long_options, &option_index); + int c = getopt_long(argc, argv, "hvgcanprs", long_options, &option_index); if (c == -1) break; @@ -1806,6 +974,15 @@ static void parse_args(int argc, char **argv) #endif break; } + case 'c': { + enabled = 1; + ebpf_enable_chart(EBPF_MODULE_CACHESTAT_IDX, disable_apps); +#ifdef NETDATA_INTERNAL_CHECKS + info( + "EBPF enabling \"CACHESTAT\" charts, because it was started with the option \"--cachestat\" or \"-c\"."); +#endif + break; + } case 'n': { enabled = 1; ebpf_enable_chart(EBPF_MODULE_SOCKET_IDX, disable_apps); @@ -1830,6 +1007,14 @@ static void parse_args(int argc, char **argv) #endif break; } + case 's': { + enabled = 1; + ebpf_enable_chart(EBPF_MODULE_SYNC_IDX, disable_apps); +#ifdef NETDATA_INTERNAL_CHECKS + info("EBPF enabling \"sync\" chart, because it was started with the option \"--sync\" or \"-s\"."); +#endif + break; + } default: { break; } @@ -1948,9 +1133,16 @@ int main(int argc, char **argv) read_local_ports("/proc/net/udp6", IPPROTO_UDP); struct netdata_static_thread ebpf_threads[] = { - {"EBPF PROCESS", NULL, NULL, 1, NULL, NULL, ebpf_modules[0].start_routine}, - {"EBPF SOCKET" , NULL, NULL, 1, NULL, NULL, ebpf_modules[1].start_routine}, - {NULL , NULL, NULL, 0, NULL, NULL, NULL} + {"EBPF PROCESS", NULL, NULL, 1, + NULL, NULL, ebpf_modules[EBPF_MODULE_PROCESS_IDX].start_routine}, + {"EBPF SOCKET" , NULL, NULL, 1, + NULL, NULL, ebpf_modules[EBPF_MODULE_SOCKET_IDX].start_routine}, + {"EBPF CACHESTAT" , NULL, NULL, 1, + NULL, NULL, ebpf_modules[EBPF_MODULE_CACHESTAT_IDX].start_routine}, + {"EBPF SYNC" , NULL, NULL, 1, + NULL, NULL, ebpf_modules[EBPF_MODULE_SYNC_IDX].start_routine}, + {NULL , NULL, NULL, 0, + NULL, NULL, NULL} }; //clean_loaded_events(); diff --git a/collectors/ebpf.plugin/ebpf.conf b/collectors/ebpf.plugin/ebpf.d.conf index 3a5b77395..7191d7416 100644 --- a/collectors/ebpf.plugin/ebpf.conf +++ b/collectors/ebpf.plugin/ebpf.d.conf @@ -10,36 +10,27 @@ # If you want to disable the integration with `apps.plugin` along with the above charts, change the setting `apps` to # 'no'. # +# The `update every` option defines the number of seconds used to read data from kernel and send to netdata [global] ebpf load mode = entry apps = yes + update every = 1 # # eBPF Programs # # The eBPF collector enables and runs the following eBPF programs by default: # +# `cachestat`: Make charts for kernel functions related to page cache. # `process` : This eBPF program creates charts that show information about process creation, VFS IO, and # files removed. # `socket` : This eBPF program creates charts with information about `TCP` and `UDP` functions, including the # bandwidth consumed by each. +# `sync` : Montitor calls for syscall sync(2). [ebpf programs] + cachestat = no process = yes socket = yes + sync = yes network connections = no -# -# Network Connection -# -# This is a feature with status WIP(Work in Progress) -# -[network connections] - maximum dimensions = 50 - resolve hostnames = no - resolve service names = no - ports = * - ips = !127.0.0.1/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 fc00::/7 !::1/128 - hostnames = * - -[service name] - 19999 = Netdata diff --git a/collectors/ebpf.plugin/ebpf.d/cachestat.conf b/collectors/ebpf.plugin/ebpf.d/cachestat.conf new file mode 100644 index 000000000..78277cf56 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf.d/cachestat.conf @@ -0,0 +1,14 @@ +# The `ebpf load mode` option accepts the following values : +# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors. +# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates +# new charts for the return of these functions, such as errors. +# +# The eBPF collector also creates charts for each running application through an integration with the `apps plugin`. +# If you want to disable the integration with `apps.plugin` along with the above charts, change the setting `apps` to +# 'no'. +# +# +[global] + ebpf load mode = entry + apps = yes + update every = 2 diff --git a/collectors/ebpf.plugin/ebpf.d/ebpf_kernel_reject_list.txt b/collectors/ebpf.plugin/ebpf.d/ebpf_kernel_reject_list.txt new file mode 100644 index 000000000..539bf357f --- /dev/null +++ b/collectors/ebpf.plugin/ebpf.d/ebpf_kernel_reject_list.txt @@ -0,0 +1 @@ +Ubuntu 4.18.0 diff --git a/collectors/ebpf.plugin/ebpf.d/network.conf b/collectors/ebpf.plugin/ebpf.d/network.conf new file mode 100644 index 000000000..b033bc39c --- /dev/null +++ b/collectors/ebpf.plugin/ebpf.d/network.conf @@ -0,0 +1,30 @@ +# The `ebpf load mode` option accepts the following values : +# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors. +# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates +# new charts for the return of these functions, such as errors. +# +# The eBPF collector also creates charts for each running application through an integration with the `apps plugin`. +# If you want to disable the integration with `apps.plugin` along with the above charts, change the setting `apps` to +# 'no'. +# +# +[global] + ebpf load mode = entry + apps = yes + update every = 1 + +# +# Network Connection +# +# This is a feature with status WIP(Work in Progress) +# +[network connections] + maximum dimensions = 50 + resolve hostnames = no + resolve service names = no + ports = * + ips = !127.0.0.1/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 fc00::/7 !::1/128 + hostnames = * + +[service name] + 19999 = Netdata diff --git a/collectors/ebpf.plugin/ebpf.d/process.conf b/collectors/ebpf.plugin/ebpf.d/process.conf new file mode 100644 index 000000000..7806dc844 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf.d/process.conf @@ -0,0 +1,14 @@ +# The `ebpf load mode` option accepts the following values : +# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors. +# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates +# new charts for the return of these functions, such as errors. +# +# The eBPF collector also creates charts for each running application through an integration with the `apps plugin`. +# If you want to disable the integration with `apps.plugin` along with the above charts, change the setting `apps` to +# 'no'. +# +# +[global] + ebpf load mode = entry + apps = yes + update every = 1 diff --git a/collectors/ebpf.plugin/ebpf.d/sync.conf b/collectors/ebpf.plugin/ebpf.d/sync.conf new file mode 100644 index 000000000..de28f3394 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf.d/sync.conf @@ -0,0 +1,23 @@ +# The `ebpf load mode` option accepts the following values : +# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors. +# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates +# new charts for the return of these functions, such as errors. +# +# The eBPF collector also creates charts for each running application through an integration with the `apps plugin`. +# If you want to disable the integration with `apps.plugin` along with the above charts, change the setting `apps` to +# 'no'. +# +# +[global] + ebpf load mode = entry + apps = yes + update every = 2 + +# List of monitored syscalls +[syscalls] + sync = yes + msync = yes + fsync = yes + fdatasync = yes + syncfs = yes + sync_file_range = yes diff --git a/collectors/ebpf.plugin/ebpf.h b/collectors/ebpf.plugin/ebpf.h index 35013c2b2..6796dcdad 100644 --- a/collectors/ebpf.plugin/ebpf.h +++ b/collectors/ebpf.plugin/ebpf.h @@ -31,6 +31,9 @@ #include "ebpf_apps.h" +#define NETDATA_EBPF_OLD_CONFIG_FILE "ebpf.conf" +#define NETDATA_EBPF_CONFIG_FILE "ebpf.d.conf" + typedef struct netdata_syscall_stat { unsigned long bytes; // total number of bytes uint64_t call; // total number of calls @@ -70,8 +73,12 @@ typedef struct netdata_error_report { } netdata_error_report_t; extern ebpf_module_t ebpf_modules[]; -#define EBPF_MODULE_PROCESS_IDX 0 -#define EBPF_MODULE_SOCKET_IDX 1 +enum ebpf_module_indexes { + EBPF_MODULE_PROCESS_IDX, + EBPF_MODULE_SOCKET_IDX, + EBPF_MODULE_CACHESTAT_IDX, + EBPF_MODULE_SYNC_IDX +}; // Copied from musl header #ifndef offsetof @@ -84,6 +91,9 @@ extern ebpf_module_t ebpf_modules[]; // Chart defintions #define NETDATA_EBPF_FAMILY "ebpf" +#define NETDATA_EBPF_CHART_TYPE_LINE "line" +#define NETDATA_EBPF_CHART_TYPE_STACKED "stacked" +#define NETDATA_EBPF_MEMORY_GROUP "mem" // Log file #define NETDATA_DEVELOPER_LOG_FILE "developer.log" @@ -133,6 +143,7 @@ extern void ebpf_write_chart_cmd(char *type, char *units, char *family, char *charttype, + char *context, int order); extern void ebpf_write_global_dimension(char *name, char *id, char *algorithm); @@ -144,6 +155,8 @@ extern void ebpf_create_chart(char *type, char *title, char *units, char *family, + char *context, + char *charttype, int order, void (*ncd)(void *, int), void *move, @@ -166,6 +179,7 @@ extern void ebpf_create_charts_on_apps(char *name, char *title, char *units, char *family, + char *charttype, int order, char *algorithm, struct target *root); @@ -174,11 +188,9 @@ extern void write_end_chart(); extern void ebpf_cleanup_publish_syscall(netdata_publish_syscall_t *nps); -#define EBPF_GLOBAL_SECTION "global" #define EBPF_PROGRAMS_SECTION "ebpf programs" -#define EBPF_NETWORK_VIEWER_SECTION "network connections" -#define EBPF_SERVICE_NAME_SECTION "service name" +#define EBPF_COMMON_DIMENSION_PERCENTAGE "%" #define EBPF_COMMON_DIMENSION_CALL "calls/s" #define EBPF_COMMON_DIMENSION_BITS "kilobits/s" #define EBPF_COMMON_DIMENSION_BYTES "bytes/s" @@ -186,22 +198,24 @@ extern void ebpf_cleanup_publish_syscall(netdata_publish_syscall_t *nps); #define EBPF_COMMON_DIMENSION_PACKETS "packets" // Common variables -extern char *ebpf_user_config_dir; -extern char *ebpf_stock_config_dir; extern int debug_enabled; extern struct pid_stat *root_of_pids; extern char *ebpf_algorithms[]; - -// Socket functions and variables -// Common functions -extern void ebpf_socket_create_apps_charts(ebpf_module_t *em, struct target *root); -extern collected_number get_value_from_structure(char *basis, size_t offset); +extern struct config collector_config; extern struct pid_stat *root_of_pids; extern ebpf_process_stat_t *global_process_stat; extern size_t all_pids_count; extern int update_every; extern uint32_t finalized_threads; +// Socket functions and variables +// Common functions +extern void ebpf_process_create_apps_charts(struct ebpf_module *em, void *ptr); +extern void ebpf_socket_create_apps_charts(struct ebpf_module *em, void *ptr); +extern void ebpf_cachestat_create_apps_charts(struct ebpf_module *em, void *root); +extern void ebpf_one_dimension_write_charts(char *family, char *chart, char *dim, long long v1); +extern collected_number get_value_from_structure(char *basis, size_t offset); + #define EBPF_MAX_SYNCHRONIZATION_TIME 300 #endif /* NETDATA_COLLECTOR_EBPF_H */ diff --git a/collectors/ebpf.plugin/ebpf_apps.c b/collectors/ebpf.plugin/ebpf_apps.c index 844ce23b8..1be7b9260 100644 --- a/collectors/ebpf.plugin/ebpf_apps.c +++ b/collectors/ebpf.plugin/ebpf_apps.c @@ -910,6 +910,26 @@ static inline void del_pid_entry(pid_t pid) } /** + * Cleanup variable from other threads + * + * @param pid current pid. + */ +void cleanup_variables_from_other_threads(uint32_t pid) +{ + // Clean socket structures + if (socket_bandwidth_curr) { + freez(socket_bandwidth_curr[pid]); + socket_bandwidth_curr[pid] = NULL; + } + + // Clean cachestat strcture + if (cachestat_pid) { + freez(cachestat_pid[pid]); + cachestat_pid[pid] = NULL; + } +} + +/** * Remove PIDs when they are not running more. */ void cleanup_exited_pids() @@ -932,11 +952,7 @@ void cleanup_exited_pids() freez(current_apps_data[r]); current_apps_data[r] = NULL; - // Clean socket structures - if (socket_bandwidth_curr) { - freez(socket_bandwidth_curr[r]); - socket_bandwidth_curr[r] = NULL; - } + cleanup_variables_from_other_threads(r); } else { if (unlikely(p->keep)) p->keeploops++; @@ -1054,11 +1070,7 @@ void collect_data_for_all_processes(int tbl_pid_stats_fd) freez(current_apps_data[key]); current_apps_data[key] = NULL; - // Clean socket structures - if (socket_bandwidth_curr) { - freez(socket_bandwidth_curr[key]); - socket_bandwidth_curr[key] = NULL; - } + cleanup_variables_from_other_threads(key); pids = pids->next; continue; diff --git a/collectors/ebpf.plugin/ebpf_apps.h b/collectors/ebpf.plugin/ebpf_apps.h index f8cb7ac72..eb54754c6 100644 --- a/collectors/ebpf.plugin/ebpf_apps.h +++ b/collectors/ebpf.plugin/ebpf_apps.h @@ -11,12 +11,15 @@ #include "libnetdata/ebpf/ebpf.h" #define NETDATA_APPS_FAMILY "apps" -#define NETDATA_APPS_FILE_GROUP "ebpf file" -#define NETDATA_APPS_VFS_GROUP "ebpf vfs" -#define NETDATA_APPS_PROCESS_GROUP "ebpf process" -#define NETDATA_APPS_NET_GROUP "ebpf net" +#define NETDATA_APPS_FILE_GROUP "file (eBPF)" +#define NETDATA_APPS_VFS_GROUP "vfs (eBPF)" +#define NETDATA_APPS_PROCESS_GROUP "process (eBPF)" +#define NETDATA_APPS_NET_GROUP "net (eBPF)" +#define NETDATA_APPS_CACHESTAT_GROUP "page cache (eBPF)" #include "ebpf_process.h" +#include "ebpf_cachestat.h" +#include "ebpf_sync.h" #define MAX_COMPARE_NAME 100 #define MAX_NAME 100 @@ -105,6 +108,9 @@ struct target { uid_t uid; gid_t gid; + // Page cache statistic per process + netdata_publish_cachestat_t cachestat; + /* These variables are not necessary for eBPF collector kernel_uint_t minflt; kernel_uint_t cminflt; @@ -426,5 +432,6 @@ extern void collect_data_for_all_processes(int tbl_pid_stats_fd); extern ebpf_process_stat_t **global_process_stats; extern ebpf_process_publish_apps_t **current_apps_data; +extern netdata_publish_cachestat_t **cachestat_pid; #endif /* NETDATA_EBPF_APPS_H */ diff --git a/collectors/ebpf.plugin/ebpf_cachestat.c b/collectors/ebpf.plugin/ebpf_cachestat.c new file mode 100644 index 000000000..6516d4da2 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf_cachestat.c @@ -0,0 +1,655 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "ebpf.h" +#include "ebpf_cachestat.h" + +static ebpf_data_t cachestat_data; +netdata_publish_cachestat_t **cachestat_pid; + +static struct bpf_link **probe_links = NULL; +static struct bpf_object *objects = NULL; + +static char *cachestat_counter_dimension_name[NETDATA_CACHESTAT_END] = { "ratio", "dirty", "hit", + "miss" }; +static netdata_syscall_stat_t cachestat_counter_aggregated_data[NETDATA_CACHESTAT_END]; +static netdata_publish_syscall_t cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_END]; + +netdata_cachestat_pid_t *cachestat_vector = NULL; + +static netdata_idx_t *cachestat_hash_values = NULL; + +static int read_thread_closed = 1; + +struct netdata_static_thread cachestat_threads = {"CACHESTAT KERNEL", + NULL, NULL, 1, NULL, + NULL, NULL}; + +static int *map_fd = NULL; + +struct config cachestat_config = { .first_section = NULL, + .last_section = NULL, + .mutex = NETDATA_MUTEX_INITIALIZER, + .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare }, + .rwlock = AVL_LOCK_INITIALIZER } }; + +/***************************************************************** + * + * FUNCTIONS TO CLOSE THE THREAD + * + *****************************************************************/ + +/** + * Clean PID structures + * + * Clean the allocated structures. + */ +static void clean_pid_structures() { + struct pid_stat *pids = root_of_pids; + while (pids) { + freez(cachestat_pid[pids->pid]); + + pids = pids->next; + } +} + +/** + * Clean up the main thread. + * + * @param ptr thread data. + */ +static void ebpf_cachestat_cleanup(void *ptr) +{ + ebpf_module_t *em = (ebpf_module_t *)ptr; + if (!em->enabled) + return; + + heartbeat_t hb; + heartbeat_init(&hb); + uint32_t tick = 2*USEC_PER_MS; + while (!read_thread_closed) { + usec_t dt = heartbeat_next(&hb, tick); + UNUSED(dt); + } + + clean_pid_structures(); + freez(cachestat_pid); + + ebpf_cleanup_publish_syscall(cachestat_counter_publish_aggregated); + + freez(cachestat_vector); + freez(cachestat_hash_values); + + struct bpf_program *prog; + size_t i = 0 ; + bpf_object__for_each_program(prog, objects) { + bpf_link__destroy(probe_links[i]); + i++; + } + bpf_object__close(objects); +} + +/***************************************************************** + * + * COMMON FUNCTIONS + * + *****************************************************************/ + +/** + * Update publish + * + * Update publish values before to write dimension. + * + * @param out strcuture that will receive data. + * @param mpa calls for mark_page_accessed during the last second. + * @param mbd calls for mark_buffer_dirty during the last second. + * @param apcl calls for add_to_page_cache_lru during the last second. + * @param apd calls for account_page_dirtied during the last second. + */ +void cachestat_update_publish(netdata_publish_cachestat_t *out, uint64_t mpa, uint64_t mbd, + uint64_t apcl, uint64_t apd) +{ + // Adapted algorithm from https://github.com/iovisor/bcc/blob/master/tools/cachestat.py#L126-L138 + calculated_number total = (calculated_number) (((long long)mpa) - ((long long)mbd)); + if (total < 0) + total = 0; + + calculated_number misses = (calculated_number) ( ((long long) apcl) - ((long long) apd) ); + if (misses < 0) + misses = 0; + + // If hits are < 0, then its possible misses are overestimate due to possibly page cache read ahead adding + // more pages than needed. In this case just assume misses as total and reset hits. + calculated_number hits = total - misses; + if (hits < 0 ) { + misses = total; + hits = 0; + } + + calculated_number ratio = (total > 0) ? hits/total : 0; + + out->ratio = (long long )(ratio*100); + out->hit = (long long)hits; + out->miss = (long long)misses; +} + +/** + * Save previous values + * + * Save values used this time. + * + * @param publish + */ +static void save_previous_values(netdata_publish_cachestat_t *publish) { + publish->prev.mark_page_accessed = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED]; + publish->prev.account_page_dirtied = cachestat_hash_values[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED]; + publish->prev.add_to_page_cache_lru = cachestat_hash_values[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU]; + publish->prev.mark_buffer_dirty = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY]; +} + +/** + * Calculate statistics + * + * @param publish the structure where we will store the data. + */ +static void calculate_stats(netdata_publish_cachestat_t *publish) { + if (!publish->prev.mark_page_accessed) { + save_previous_values(publish); + return; + } + + uint64_t mpa = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED] - publish->prev.mark_page_accessed; + uint64_t mbd = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY] - publish->prev.mark_buffer_dirty; + uint64_t apcl = cachestat_hash_values[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU] - publish->prev.add_to_page_cache_lru; + uint64_t apd = cachestat_hash_values[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED] - publish->prev.account_page_dirtied; + + save_previous_values(publish); + + // We are changing the original algorithm to have a smooth ratio. + cachestat_update_publish(publish, mpa, mbd, apcl, apd); +} + + +/***************************************************************** + * + * APPS + * + *****************************************************************/ + +/** + * Apps Accumulator + * + * Sum all values read from kernel and store in the first address. + * + * @param out the vector with read values. + */ +static void cachestat_apps_accumulator(netdata_cachestat_pid_t *out) +{ + int i, end = (running_on_kernel >= NETDATA_KERNEL_V4_15) ? ebpf_nprocs : 1; + netdata_cachestat_pid_t *total = &out[0]; + for (i = 1; i < end; i++) { + netdata_cachestat_pid_t *w = &out[i]; + total->account_page_dirtied += w->account_page_dirtied; + total->add_to_page_cache_lru += w->add_to_page_cache_lru; + total->mark_buffer_dirty += w->mark_buffer_dirty; + total->mark_page_accessed += w->mark_page_accessed; + } +} + +/** + * Save Pid values + * + * Save the current values inside the structure + * + * @param out vector used to plot charts + * @param publish vector with values read from hash tables. + */ +static inline void cachestat_save_pid_values(netdata_publish_cachestat_t *out, netdata_cachestat_pid_t *publish) +{ + if (!out->current.mark_page_accessed) { + memcpy(&out->current, &publish[0], sizeof(netdata_cachestat_pid_t)); + return; + } + + memcpy(&out->prev, &out->current, sizeof(netdata_cachestat_pid_t)); + memcpy(&out->current, &publish[0], sizeof(netdata_cachestat_pid_t)); +} + +/** + * Fill PID + * + * Fill PID structures + * + * @param current_pid pid that we are collecting data + * @param out values read from hash tables; + */ +static void cachestat_fill_pid(uint32_t current_pid, netdata_cachestat_pid_t *publish) +{ + netdata_publish_cachestat_t *curr = cachestat_pid[current_pid]; + if (!curr) { + curr = callocz(1, sizeof(netdata_publish_cachestat_t)); + cachestat_pid[current_pid] = curr; + + cachestat_save_pid_values(curr, publish); + return; + } + + cachestat_save_pid_values(curr, publish); +} + +/** + * Read APPS table + * + * Read the apps table and store data inside the structure. + */ +static void read_apps_table() +{ + netdata_cachestat_pid_t *cv = cachestat_vector; + uint32_t key; + struct pid_stat *pids = root_of_pids; + int fd = map_fd[NETDATA_CACHESTAT_PID_STATS]; + size_t length = sizeof(netdata_cachestat_pid_t)*ebpf_nprocs; + while (pids) { + key = pids->pid; + + if (bpf_map_lookup_elem(fd, &key, cv)) { + pids = pids->next; + continue; + } + + cachestat_apps_accumulator(cv); + + cachestat_fill_pid(key, cv); + + // We are cleaning to avoid passing data read from one process to other. + memset(cv, 0, length); + + pids = pids->next; + } +} + +/** + * Create apps charts + * + * Call ebpf_create_chart to create the charts on apps submenu. + * + * @param em a pointer to the structure with the default values. + */ +void ebpf_cachestat_create_apps_charts(struct ebpf_module *em, void *ptr) +{ + UNUSED(em); + struct target *root = ptr; + ebpf_create_charts_on_apps(NETDATA_CACHESTAT_HIT_RATIO_CHART, + "The ratio is calculated dividing the Hit pages per total cache accesses without counting dirties.", + EBPF_COMMON_DIMENSION_PERCENTAGE, + NETDATA_APPS_CACHESTAT_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, + 20090, + ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX], + root); + + ebpf_create_charts_on_apps(NETDATA_CACHESTAT_DIRTY_CHART, + "Number of pages marked as dirty. When a page is called dirty, this means that the data stored inside the page needs to be written to devices.", + EBPF_CACHESTAT_DIMENSION_PAGE, + NETDATA_APPS_CACHESTAT_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, + 20091, + ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], + root); + + ebpf_create_charts_on_apps(NETDATA_CACHESTAT_HIT_CHART, + "Number of cache access without counting dirty pages and page additions.", + EBPF_CACHESTAT_DIMENSION_HITS, + NETDATA_APPS_CACHESTAT_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, + 20092, + ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX], + root); + + ebpf_create_charts_on_apps(NETDATA_CACHESTAT_MISSES_CHART, + "Page caches added without counting dirty pages", + EBPF_CACHESTAT_DIMENSION_MISSES, + NETDATA_APPS_CACHESTAT_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, + 20093, + ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX], + root); +} + +/***************************************************************** + * + * MAIN LOOP + * + *****************************************************************/ + +/** + * Read global counter + * + * Read the table with number of calls for all functions + */ +static void read_global_table() +{ + uint32_t idx; + netdata_idx_t *val = cachestat_hash_values; + netdata_idx_t stored; + int fd = map_fd[NETDATA_CACHESTAT_GLOBAL_STATS]; + + for (idx = NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU; idx < NETDATA_CACHESTAT_END; idx++) { + if (!bpf_map_lookup_elem(fd, &idx, &stored)) { + val[idx] = stored; + } + } +} + +/** + * Socket read hash + * + * This is the thread callback. + * This thread is necessary, because we cannot freeze the whole plugin to read the data on very busy socket. + * + * @param ptr It is a NULL value for this thread. + * + * @return It always returns NULL. + */ +void *ebpf_cachestat_read_hash(void *ptr) +{ + read_thread_closed = 0; + + heartbeat_t hb; + heartbeat_init(&hb); + + ebpf_module_t *em = (ebpf_module_t *)ptr; + + usec_t step = NETDATA_LATENCY_CACHESTAT_SLEEP_MS * em->update_time; + int apps = em->apps_charts; + while (!close_ebpf_plugin) { + usec_t dt = heartbeat_next(&hb, step); + (void)dt; + + read_global_table(); + + if (apps) + read_apps_table(); + } + read_thread_closed = 1; + + return NULL; +} + +/** + * Send global + * + * Send global charts to Netdata + */ +static void cachestat_send_global(netdata_publish_cachestat_t *publish) +{ + calculate_stats(publish); + + netdata_publish_syscall_t *ptr = cachestat_counter_publish_aggregated; + // The algorithm sets this value to zero sometimes, we are not written them to have a smooth chart + if (publish->ratio) { + ebpf_one_dimension_write_charts( + NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_RATIO_CHART, ptr[NETDATA_CACHESTAT_IDX_RATIO].dimension, + publish->ratio); + } + + ebpf_one_dimension_write_charts( + NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_DIRTY_CHART, ptr[NETDATA_CACHESTAT_IDX_DIRTY].dimension, + cachestat_hash_values[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY]); + + ebpf_one_dimension_write_charts( + NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_CHART, ptr[NETDATA_CACHESTAT_IDX_HIT].dimension, publish->hit); + + ebpf_one_dimension_write_charts( + NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_MISSES_CHART, ptr[NETDATA_CACHESTAT_IDX_MISS].dimension, + publish->miss); +} + +/** + * Cachestat sum PIDs + * + * Sum values for all PIDs associated to a group + * + * @param publish output structure. + * @param root structure with listed IPs + */ +void ebpf_cachestat_sum_pids(netdata_publish_cachestat_t *publish, struct pid_on_target *root) +{ + memcpy(&publish->prev, &publish->current,sizeof(publish->current)); + memset(&publish->current, 0, sizeof(publish->current)); + + netdata_cachestat_pid_t *dst = &publish->current; + while (root) { + int32_t pid = root->pid; + netdata_publish_cachestat_t *w = cachestat_pid[pid]; + if (w) { + netdata_cachestat_pid_t *src = &w->current; + dst->account_page_dirtied += src->account_page_dirtied; + dst->add_to_page_cache_lru += src->add_to_page_cache_lru; + dst->mark_buffer_dirty += src->mark_buffer_dirty; + dst->mark_page_accessed += src->mark_page_accessed; + } + + root = root->next; + } +} + +/** + * Send data to Netdata calling auxiliar functions. + * + * @param root the target list. +*/ +void ebpf_cache_send_apps_data(struct target *root) +{ + struct target *w; + collected_number value; + + write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_HIT_RATIO_CHART); + for (w = root; w; w = w->next) { + if (unlikely(w->exposed && w->processes)) { + ebpf_cachestat_sum_pids(&w->cachestat, w->root_pid); + netdata_cachestat_pid_t *current = &w->cachestat.current; + netdata_cachestat_pid_t *prev = &w->cachestat.prev; + + uint64_t mpa = current->mark_page_accessed - prev->mark_page_accessed; + uint64_t mbd = current->mark_buffer_dirty - prev->mark_buffer_dirty; + w->cachestat.dirty = current->mark_buffer_dirty; + uint64_t apcl = current->add_to_page_cache_lru - prev->add_to_page_cache_lru; + uint64_t apd = current->account_page_dirtied - prev->account_page_dirtied; + + cachestat_update_publish(&w->cachestat, mpa, mbd, apcl, apd); + value = (collected_number) w->cachestat.ratio; + // Here we are using different approach to have a chart more smooth + write_chart_dimension(w->name, value); + } + } + write_end_chart(); + + write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_DIRTY_CHART); + for (w = root; w; w = w->next) { + if (unlikely(w->exposed && w->processes)) { + value = (collected_number) w->cachestat.dirty; + write_chart_dimension(w->name, value); + } + } + write_end_chart(); + + write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_HIT_CHART); + for (w = root; w; w = w->next) { + if (unlikely(w->exposed && w->processes)) { + value = (collected_number) w->cachestat.hit; + write_chart_dimension(w->name, value); + } + } + write_end_chart(); + + write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_MISSES_CHART); + for (w = root; w; w = w->next) { + if (unlikely(w->exposed && w->processes)) { + value = (collected_number) w->cachestat.miss; + write_chart_dimension(w->name, value); + } + } + write_end_chart(); +} + +/** +* Main loop for this collector. +*/ +static void cachestat_collector(ebpf_module_t *em) +{ + cachestat_threads.thread = mallocz(sizeof(netdata_thread_t)); + cachestat_threads.start_routine = ebpf_cachestat_read_hash; + + map_fd = cachestat_data.map_fd; + + netdata_thread_create(cachestat_threads.thread, cachestat_threads.name, NETDATA_THREAD_OPTION_JOINABLE, + ebpf_cachestat_read_hash, em); + + netdata_publish_cachestat_t publish; + memset(&publish, 0, sizeof(publish)); + int apps = em->apps_charts; + while (!close_ebpf_plugin) { + pthread_mutex_lock(&collect_data_mutex); + pthread_cond_wait(&collect_data_cond_var, &collect_data_mutex); + + pthread_mutex_lock(&lock); + + cachestat_send_global(&publish); + + if (apps) + ebpf_cache_send_apps_data(apps_groups_root_target); + + pthread_mutex_unlock(&lock); + pthread_mutex_unlock(&collect_data_mutex); + } +} + +/***************************************************************** + * + * INITIALIZE THREAD + * + *****************************************************************/ + +/** + * Create global charts + * + * Call ebpf_create_chart to create the charts for the collector. + */ +static void ebpf_create_memory_charts() +{ + ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_RATIO_CHART, + "Hit is calculating using total cache added without dirties per total added because of red misses.", + EBPF_CACHESTAT_DIMENSION_HITS, NETDATA_CACHESTAT_SUBMENU, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, + 21100, + ebpf_create_global_dimension, + cachestat_counter_publish_aggregated, 1); + + ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_DIRTY_CHART, + "Number of dirty pages added to the page cache.", + EBPF_CACHESTAT_DIMENSION_PAGE, NETDATA_CACHESTAT_SUBMENU, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, + 21101, + ebpf_create_global_dimension, + &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_DIRTY], 1); + + ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_CHART, + "Hits are function calls that Netdata counts.", + EBPF_CACHESTAT_DIMENSION_HITS, NETDATA_CACHESTAT_SUBMENU, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, + 21102, + ebpf_create_global_dimension, + &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_HIT], 1); + + ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_MISSES_CHART, + "Misses are function calls that Netdata counts.", + EBPF_CACHESTAT_DIMENSION_MISSES, NETDATA_CACHESTAT_SUBMENU, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, + 21103, + ebpf_create_global_dimension, + &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_MISS], 1); + + fflush(stdout); +} + +/** + * Allocate vectors used with this thread. + * + * We are not testing the return, because callocz does this and shutdown the software + * case it was not possible to allocate. + * + * @param length is the length for the vectors used inside the collector. + */ +static void ebpf_cachestat_allocate_global_vectors(size_t length) +{ + cachestat_pid = callocz((size_t)pid_max, sizeof(netdata_publish_cachestat_t *)); + cachestat_vector = callocz((size_t)ebpf_nprocs, sizeof(netdata_cachestat_pid_t)); + + cachestat_hash_values = callocz(length, sizeof(netdata_idx_t)); + + memset(cachestat_counter_aggregated_data, 0, length * sizeof(netdata_syscall_stat_t)); + memset(cachestat_counter_publish_aggregated, 0, length * sizeof(netdata_publish_syscall_t)); +} + +/***************************************************************** + * + * MAIN THREAD + * + *****************************************************************/ + +/** + * Cachestat thread + * + * Thread used to make cachestat thread + * + * @param ptr a pointer to `struct ebpf_module` + * + * @return It always return NULL + */ +void *ebpf_cachestat_thread(void *ptr) +{ + netdata_thread_cleanup_push(ebpf_cachestat_cleanup, ptr); + + ebpf_module_t *em = (ebpf_module_t *)ptr; + fill_ebpf_data(&cachestat_data); + + ebpf_update_module(em, &cachestat_config, NETDATA_CACHESTAT_CONFIG_FILE); + + if (!em->enabled) + goto endcachestat; + + pthread_mutex_lock(&lock); + ebpf_cachestat_allocate_global_vectors(NETDATA_CACHESTAT_END); + if (ebpf_update_kernel(&cachestat_data)) { + pthread_mutex_unlock(&lock); + goto endcachestat; + } + + probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &objects, cachestat_data.map_fd); + if (!probe_links) { + pthread_mutex_unlock(&lock); + goto endcachestat; + } + + int algorithms[NETDATA_CACHESTAT_END] = { + NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX + }; + + ebpf_global_labels(cachestat_counter_aggregated_data, cachestat_counter_publish_aggregated, + cachestat_counter_dimension_name, cachestat_counter_dimension_name, + algorithms, NETDATA_CACHESTAT_END); + + ebpf_create_memory_charts(); + + pthread_mutex_unlock(&lock); + + cachestat_collector(em); + +endcachestat: + netdata_thread_cleanup_pop(1); + return NULL; +} diff --git a/collectors/ebpf.plugin/ebpf_cachestat.h b/collectors/ebpf.plugin/ebpf_cachestat.h new file mode 100644 index 000000000..daf678975 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf_cachestat.h @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_EBPF_CACHESTAT_H +#define NETDATA_EBPF_CACHESTAT_H 1 + +// charts +#define NETDATA_CACHESTAT_HIT_RATIO_CHART "cachestat_ratio" +#define NETDATA_CACHESTAT_DIRTY_CHART "cachestat_dirties" +#define NETDATA_CACHESTAT_HIT_CHART "cachestat_hits" +#define NETDATA_CACHESTAT_MISSES_CHART "cachestat_misses" + +#define NETDATA_CACHESTAT_SUBMENU "page cache (eBPF)" + +#define EBPF_CACHESTAT_DIMENSION_PAGE "pages/s" +#define EBPF_CACHESTAT_DIMENSION_HITS "hits/s" +#define EBPF_CACHESTAT_DIMENSION_MISSES "misses/s" + +#define NETDATA_LATENCY_CACHESTAT_SLEEP_MS 600000ULL + +// configuration file +#define NETDATA_CACHESTAT_CONFIG_FILE "cachestat.conf" + +// variables +enum cachestat_counters { + NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU, + NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED, + NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED, + NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY, + + NETDATA_CACHESTAT_END +}; + +enum cachestat_indexes { + NETDATA_CACHESTAT_IDX_RATIO, + NETDATA_CACHESTAT_IDX_DIRTY, + NETDATA_CACHESTAT_IDX_HIT, + NETDATA_CACHESTAT_IDX_MISS +}; + +enum cachesta_tables { + NETDATA_CACHESTAT_GLOBAL_STATS, + NETDATA_CACHESTAT_PID_STATS +}; + +typedef struct netdata_publish_cachestat_pid { + uint64_t add_to_page_cache_lru; + uint64_t mark_page_accessed; + uint64_t account_page_dirtied; + uint64_t mark_buffer_dirty; +} netdata_cachestat_pid_t; + +typedef struct netdata_publish_cachestat { + long long ratio; + long long dirty; + long long hit; + long long miss; + + netdata_cachestat_pid_t current; + netdata_cachestat_pid_t prev; +} netdata_publish_cachestat_t; + +extern void *ebpf_cachestat_thread(void *ptr); + +#endif // NETDATA_EBPF_CACHESTAT_H diff --git a/collectors/ebpf.plugin/ebpf_kernel_reject_list.txt b/collectors/ebpf.plugin/ebpf_kernel_reject_list.txt deleted file mode 100644 index d56b216a9..000000000 --- a/collectors/ebpf.plugin/ebpf_kernel_reject_list.txt +++ /dev/null @@ -1 +0,0 @@ -Ubuntu 4.18.0-13. diff --git a/collectors/ebpf.plugin/ebpf_process.c b/collectors/ebpf.plugin/ebpf_process.c index 27e39d1a5..5fa930b2d 100644 --- a/collectors/ebpf.plugin/ebpf_process.c +++ b/collectors/ebpf.plugin/ebpf_process.c @@ -19,8 +19,8 @@ static char *process_id_names[NETDATA_KEY_PUBLISH_PROCESS_END] = { "do_sys_open" static char *status[] = { "process", "zombie" }; static netdata_idx_t *process_hash_values = NULL; -static netdata_syscall_stat_t *process_aggregated_data = NULL; -static netdata_publish_syscall_t *process_publish_aggregated = NULL; +static netdata_syscall_stat_t process_aggregated_data[NETDATA_KEY_PUBLISH_PROCESS_END]; +static netdata_publish_syscall_t process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_END]; static ebpf_data_t process_data; @@ -33,6 +33,12 @@ static int *map_fd = NULL; static struct bpf_object *objects = NULL; static struct bpf_link **probe_links = NULL; +struct config process_config = { .first_section = NULL, + .last_section = NULL, + .mutex = NETDATA_MUTEX_INITIALIZER, + .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare }, + .rwlock = AVL_LOCK_INITIALIZER } }; + /***************************************************************** * * PROCESS DATA AND SEND TO NETDATA @@ -520,6 +526,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "Open and close calls", EBPF_COMMON_DIMENSION_CALL, NETDATA_FILE_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21000, ebpf_create_global_dimension, process_publish_aggregated, @@ -531,6 +539,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "Open fails", EBPF_COMMON_DIMENSION_CALL, NETDATA_FILE_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21001, ebpf_create_global_dimension, process_publish_aggregated, @@ -542,6 +552,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "Remove files", EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21002, ebpf_create_global_dimension, &process_publish_aggregated[NETDATA_DEL_START], @@ -552,6 +564,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "Calls to IO", EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21003, ebpf_create_global_dimension, &process_publish_aggregated[NETDATA_IN_START_BYTE], @@ -569,6 +583,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "Fails to write or read", EBPF_COMMON_DIMENSION_CALL, NETDATA_VFS_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21005, ebpf_create_global_dimension, &process_publish_aggregated[2], @@ -580,6 +596,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "Start process", EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21006, ebpf_create_global_dimension, &process_publish_aggregated[NETDATA_PROCESS_START], @@ -590,6 +608,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "Exit process", EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21007, ebpf_create_global_dimension, &process_publish_aggregated[NETDATA_EXIT_START], @@ -608,6 +628,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "Fails to create process", EBPF_COMMON_DIMENSION_CALL, NETDATA_PROCESS_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21009, ebpf_create_global_dimension, &process_publish_aggregated[NETDATA_PROCESS_START], @@ -621,14 +643,16 @@ static void ebpf_create_global_charts(ebpf_module_t *em) * Call ebpf_create_chart to create the charts on apps submenu. * * @param em a pointer to the structure with the default values. - * @param root a pointer for the targets. + * @param ptr a pointer for the targets. */ -static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *root) +void ebpf_process_create_apps_charts(struct ebpf_module *em, void *ptr) { + struct target *root = ptr; ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_FILE_OPEN, "Number of open files", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20061, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -638,6 +662,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro "Fails to open files", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20062, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -647,6 +672,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro "Files closed", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20063, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -656,6 +682,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro "Fails to close files", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_FILE_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20064, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -665,6 +692,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro "Files deleted", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_VFS_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20065, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -673,6 +701,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro "Write to disk", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_VFS_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20066, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], apps_groups_root_target); @@ -682,6 +711,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro "Fails to write", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_VFS_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20067, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -691,6 +721,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro "Read from disk", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_VFS_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20068, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -700,6 +731,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro "Fails to read", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_VFS_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20069, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -708,6 +740,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_WRITE_BYTES, "Bytes written on disk", EBPF_COMMON_DIMENSION_BYTES, NETDATA_APPS_VFS_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20070, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -715,6 +748,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro ebpf_create_charts_on_apps(NETDATA_SYSCALL_APPS_VFS_READ_BYTES, "Bytes read from disk", EBPF_COMMON_DIMENSION_BYTES, NETDATA_APPS_VFS_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20071, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -723,6 +757,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro "Process started", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_PROCESS_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20072, ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX], root); @@ -731,6 +766,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro "Threads started", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_PROCESS_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20073, ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX], root); @@ -739,6 +775,7 @@ static void ebpf_process_create_apps_charts(ebpf_module_t *em, struct target *ro "Tasks closed", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_PROCESS_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20074, ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX], root); @@ -786,11 +823,12 @@ static void ebpf_create_apps_charts(ebpf_module_t *em, struct target *root) if (!newly_added) return; - if (ebpf_modules[EBPF_MODULE_PROCESS_IDX].apps_charts) - ebpf_process_create_apps_charts(em, root); - - if (ebpf_modules[EBPF_MODULE_SOCKET_IDX].apps_charts) - ebpf_socket_create_apps_charts(NULL, root); + int counter; + for (counter = 0; ebpf_modules[counter].thread_name; counter++) { + ebpf_module_t *current = &ebpf_modules[counter]; + if (current->enabled && current->apps_charts && current->apps_routine) + current->apps_routine(em, root); + } } /***************************************************************** @@ -904,9 +942,7 @@ static void ebpf_process_cleanup(void *ptr) UNUSED(dt); } - freez(process_aggregated_data); ebpf_cleanup_publish_syscall(process_publish_aggregated); - freez(process_publish_aggregated); freez(process_hash_values); clean_global_memory(); @@ -940,8 +976,8 @@ static void ebpf_process_cleanup(void *ptr) */ static void ebpf_process_allocate_global_vectors(size_t length) { - process_aggregated_data = callocz(length, sizeof(netdata_syscall_stat_t)); - process_publish_aggregated = callocz(length, sizeof(netdata_publish_syscall_t)); + memset(process_aggregated_data, 0, length * sizeof(netdata_syscall_stat_t)); + memset(process_publish_aggregated, 0, length * sizeof(netdata_publish_syscall_t)); process_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t)); global_process_stats = callocz((size_t)pid_max, sizeof(ebpf_process_stat_t *)); @@ -1018,13 +1054,15 @@ void *ebpf_process_thread(void *ptr) fill_ebpf_data(&process_data); pthread_mutex_lock(&lock); - ebpf_process_allocate_global_vectors(NETDATA_MAX_MONITOR_VECTOR); + ebpf_process_allocate_global_vectors(NETDATA_KEY_PUBLISH_PROCESS_END); if (ebpf_update_kernel(&process_data)) { pthread_mutex_unlock(&lock); goto endprocess; } + ebpf_update_module(em, &process_config, NETDATA_PROCESS_CONFIG_FILE); + set_local_pointers(); probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &objects, process_data.map_fd); if (!probe_links) { @@ -1040,7 +1078,7 @@ void *ebpf_process_thread(void *ptr) ebpf_global_labels( process_aggregated_data, process_publish_aggregated, process_dimension_names, process_id_names, - algorithms, NETDATA_MAX_MONITOR_VECTOR); + algorithms, NETDATA_KEY_PUBLISH_PROCESS_END); if (process_enabled) { ebpf_create_global_charts(em); diff --git a/collectors/ebpf.plugin/ebpf_process.h b/collectors/ebpf.plugin/ebpf_process.h index aa6ed66d6..a731227e1 100644 --- a/collectors/ebpf.plugin/ebpf_process.h +++ b/collectors/ebpf.plugin/ebpf_process.h @@ -10,7 +10,6 @@ // Internal constants #define NETDATA_GLOBAL_VECTOR 24 -#define NETDATA_MAX_MONITOR_VECTOR 9 #define NETDATA_VFS_ERRORS 3 // Map index @@ -52,6 +51,9 @@ #define NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS_ERROR "vfs_write_error" #define NETDATA_SYSCALL_APPS_VFS_READ_CALLS_ERROR "vfs_read_error" +// Process configuration name +#define NETDATA_PROCESS_CONFIG_FILE "process.conf" + // Index from kernel typedef enum ebpf_process_index { NETDATA_KEY_CALLS_DO_SYS_OPEN, diff --git a/collectors/ebpf.plugin/ebpf_socket.c b/collectors/ebpf.plugin/ebpf_socket.c index 7fbc24421..a142d43b3 100644 --- a/collectors/ebpf.plugin/ebpf_socket.c +++ b/collectors/ebpf.plugin/ebpf_socket.c @@ -17,8 +17,8 @@ static char *socket_id_names[NETDATA_MAX_SOCKET_VECTOR] = { "tcp_sendmsg", "tcp_ "udp_sendmsg", "udp_recvmsg", "tcp_retransmit_skb" }; static netdata_idx_t *socket_hash_values = NULL; -static netdata_syscall_stat_t *socket_aggregated_data = NULL; -static netdata_publish_syscall_t *socket_publish_aggregated = NULL; +static netdata_syscall_stat_t socket_aggregated_data[NETDATA_MAX_SOCKET_VECTOR]; +static netdata_publish_syscall_t socket_publish_aggregated[NETDATA_MAX_SOCKET_VECTOR]; static ebpf_data_t socket_data; @@ -40,6 +40,12 @@ static int *map_fd = NULL; static struct bpf_object *objects = NULL; static struct bpf_link **probe_links = NULL; +struct config socket_config = { .first_section = NULL, + .last_section = NULL, + .mutex = NETDATA_MUTEX_INITIALIZER, + .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare }, + .rwlock = AVL_LOCK_INITIALIZER } }; + /***************************************************************** * * PROCESS DATA AND SEND TO NETDATA @@ -279,17 +285,20 @@ static void ebpf_socket_send_data(ebpf_module_t *em) NETDATA_TCP_FUNCTION_ERROR, NETDATA_EBPF_FAMILY, socket_publish_aggregated, 2); } write_count_chart( - NETDATA_TCP_RETRANSMIT, NETDATA_EBPF_FAMILY, &socket_publish_aggregated[NETDATA_RETRANSMIT_START], 1); + NETDATA_TCP_RETRANSMIT, NETDATA_EBPF_FAMILY, &socket_publish_aggregated[NETDATA_IDX_TCP_RETRANSMIT], + 1); write_count_chart( - NETDATA_UDP_FUNCTION_COUNT, NETDATA_EBPF_FAMILY, &socket_publish_aggregated[NETDATA_UDP_START], 2); + NETDATA_UDP_FUNCTION_COUNT, NETDATA_EBPF_FAMILY, &socket_publish_aggregated[NETDATA_IDX_UDP_RECVBUF], + 2); write_io_chart( NETDATA_UDP_FUNCTION_BITS, NETDATA_EBPF_FAMILY, socket_id_names[3],(long long)common_udp.write*8/100, socket_id_names[4], (long long)common_udp.read*8/1000); if (em->mode < MODE_ENTRY) { write_err_chart( - NETDATA_UDP_FUNCTION_ERROR, NETDATA_EBPF_FAMILY, &socket_publish_aggregated[NETDATA_UDP_START], 2); + NETDATA_UDP_FUNCTION_ERROR, NETDATA_EBPF_FAMILY, &socket_publish_aggregated[NETDATA_UDP_START], + 2); } } @@ -427,6 +436,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "Calls to internal functions", EBPF_COMMON_DIMENSION_CALL, NETDATA_SOCKET_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21070, ebpf_create_global_dimension, socket_publish_aggregated, @@ -435,6 +446,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em) ebpf_create_chart(NETDATA_EBPF_FAMILY, NETDATA_TCP_FUNCTION_BITS, "TCP bandwidth", EBPF_COMMON_DIMENSION_BITS, NETDATA_SOCKET_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21071, ebpf_create_global_dimension, socket_publish_aggregated, @@ -446,6 +459,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "TCP errors", EBPF_COMMON_DIMENSION_CALL, NETDATA_SOCKET_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21072, ebpf_create_global_dimension, socket_publish_aggregated, @@ -457,9 +472,11 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "Packages retransmitted", EBPF_COMMON_DIMENSION_CALL, NETDATA_SOCKET_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21073, ebpf_create_global_dimension, - &socket_publish_aggregated[NETDATA_RETRANSMIT_START], + &socket_publish_aggregated[NETDATA_IDX_TCP_RETRANSMIT], 1); ebpf_create_chart(NETDATA_EBPF_FAMILY, @@ -467,17 +484,21 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "UDP calls", EBPF_COMMON_DIMENSION_CALL, NETDATA_SOCKET_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21074, ebpf_create_global_dimension, - &socket_publish_aggregated[NETDATA_UDP_START], + &socket_publish_aggregated[NETDATA_IDX_UDP_RECVBUF], 2); ebpf_create_chart(NETDATA_EBPF_FAMILY, NETDATA_UDP_FUNCTION_BITS, "UDP bandwidth", EBPF_COMMON_DIMENSION_BITS, NETDATA_SOCKET_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21075, ebpf_create_global_dimension, - &socket_publish_aggregated[NETDATA_UDP_START], + &socket_publish_aggregated[NETDATA_IDX_UDP_RECVBUF], 2); if (em->mode < MODE_ENTRY) { @@ -486,9 +507,11 @@ static void ebpf_create_global_charts(ebpf_module_t *em) "UDP errors", EBPF_COMMON_DIMENSION_CALL, NETDATA_SOCKET_GROUP, + NULL, + NETDATA_EBPF_CHART_TYPE_LINE, 21076, ebpf_create_global_dimension, - &socket_publish_aggregated[NETDATA_UDP_START], + &socket_publish_aggregated[NETDATA_IDX_UDP_RECVBUF], 2); } } @@ -498,14 +521,17 @@ static void ebpf_create_global_charts(ebpf_module_t *em) * * Call ebpf_create_chart to create the charts on apps submenu. * - * @param em a pointer to the structure with the default values. + * @param em a pointer to the structure with the default values. + * @param ptr a pointer for targets */ -void ebpf_socket_create_apps_charts(ebpf_module_t *em, struct target *root) +void ebpf_socket_create_apps_charts(struct ebpf_module *em, void *ptr) { UNUSED(em); + struct target *root = ptr;; ebpf_create_charts_on_apps(NETDATA_NET_APPS_BANDWIDTH_SENT, "Bytes sent", EBPF_COMMON_DIMENSION_BITS, NETDATA_APPS_NET_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20080, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -513,6 +539,7 @@ void ebpf_socket_create_apps_charts(ebpf_module_t *em, struct target *root) ebpf_create_charts_on_apps(NETDATA_NET_APPS_BANDWIDTH_RECV, "bytes received", EBPF_COMMON_DIMENSION_BITS, NETDATA_APPS_NET_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20081, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -521,6 +548,7 @@ void ebpf_socket_create_apps_charts(ebpf_module_t *em, struct target *root) "Calls for tcp_sendmsg", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20082, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -529,6 +557,7 @@ void ebpf_socket_create_apps_charts(ebpf_module_t *em, struct target *root) "Calls for tcp_cleanup_rbuf", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20083, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -537,6 +566,7 @@ void ebpf_socket_create_apps_charts(ebpf_module_t *em, struct target *root) "Calls for tcp_retransmit", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20084, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -545,6 +575,7 @@ void ebpf_socket_create_apps_charts(ebpf_module_t *em, struct target *root) "Calls for udp_sendmsg", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20085, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -553,6 +584,7 @@ void ebpf_socket_create_apps_charts(ebpf_module_t *em, struct target *root) "Calls for udp_recvmsg", EBPF_COMMON_DIMENSION_CALL, NETDATA_APPS_NET_GROUP, + NETDATA_EBPF_CHART_TYPE_STACKED, 20086, ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX], root); @@ -580,7 +612,8 @@ static void ebpf_socket_create_nv_chart(char *id, char *title, char *units, title, units, family, - "stacked", + NETDATA_EBPF_CHART_TYPE_STACKED, + NULL, order); uint32_t i; @@ -616,7 +649,8 @@ static void ebpf_socket_create_nv_retransmit(char *id, char *title, char *units, title, units, family, - "stacked", + NETDATA_EBPF_CHART_TYPE_STACKED, + NULL, order); uint32_t i; @@ -1137,7 +1171,7 @@ static void store_socket_inside_avl(netdata_vector_plot_t *out, netdata_socket_t memcpy(&test.index, lindex, sizeof(netdata_socket_idx_t)); test.flags = flags; - ret = (netdata_socket_plot_t *) avl_search_lock(&out->tree, (avl *)&test); + ret = (netdata_socket_plot_t *) avl_search_lock(&out->tree, (avl_t *)&test); if (ret) { if (lvalues->ct > ret->plot.last_time) { update_socket_data(&ret->sock, lvalues); @@ -1175,7 +1209,7 @@ static void store_socket_inside_avl(netdata_vector_plot_t *out, netdata_socket_t w->flags = flags; netdata_socket_plot_t *check ; - check = (netdata_socket_plot_t *) avl_insert_lock(&out->tree, (avl *)w); + check = (netdata_socket_plot_t *) avl_insert_lock(&out->tree, (avl_t *)w); if (check != w) error("Internal error, cannot insert the AVL tree."); @@ -1427,7 +1461,7 @@ void *ebpf_socket_read_hash(void *ptr) read_thread_closed = 0; heartbeat_t hb; heartbeat_init(&hb); - usec_t step = NETDATA_SOCKET_READ_SLEEP_MS; + usec_t step = NETDATA_SOCKET_READ_SLEEP_MS * em->update_time; int fd_ipv4 = map_fd[NETDATA_SOCKET_IPV4_HASH_TABLE]; int fd_ipv6 = map_fd[NETDATA_SOCKET_IPV6_HASH_TABLE]; int network_connection = em->optional; @@ -1471,22 +1505,22 @@ static void read_hash_global_tables() } } - socket_aggregated_data[0].call = res[NETDATA_KEY_CALLS_TCP_SENDMSG]; - socket_aggregated_data[1].call = res[NETDATA_KEY_CALLS_TCP_CLEANUP_RBUF]; - socket_aggregated_data[2].call = res[NETDATA_KEY_CALLS_TCP_CLOSE]; - socket_aggregated_data[3].call = res[NETDATA_KEY_CALLS_UDP_RECVMSG]; - socket_aggregated_data[4].call = res[NETDATA_KEY_CALLS_UDP_SENDMSG]; - socket_aggregated_data[5].call = res[NETDATA_KEY_TCP_RETRANSMIT]; - - socket_aggregated_data[0].ecall = res[NETDATA_KEY_ERROR_TCP_SENDMSG]; - socket_aggregated_data[1].ecall = res[NETDATA_KEY_ERROR_TCP_CLEANUP_RBUF]; - socket_aggregated_data[3].ecall = res[NETDATA_KEY_ERROR_UDP_RECVMSG]; - socket_aggregated_data[4].ecall = res[NETDATA_KEY_ERROR_UDP_SENDMSG]; - - socket_aggregated_data[0].bytes = res[NETDATA_KEY_BYTES_TCP_SENDMSG]; - socket_aggregated_data[1].bytes = res[NETDATA_KEY_BYTES_TCP_CLEANUP_RBUF]; - socket_aggregated_data[3].bytes = res[NETDATA_KEY_BYTES_UDP_RECVMSG]; - socket_aggregated_data[4].bytes = res[NETDATA_KEY_BYTES_UDP_SENDMSG]; + socket_aggregated_data[NETDATA_IDX_TCP_SENDMSG].call = res[NETDATA_KEY_CALLS_TCP_SENDMSG]; + socket_aggregated_data[NETDATA_IDX_TCP_CLEANUP_RBUF].call = res[NETDATA_KEY_CALLS_TCP_CLEANUP_RBUF]; + socket_aggregated_data[NETDATA_IDX_TCP_CLOSE].call = res[NETDATA_KEY_CALLS_TCP_CLOSE]; + socket_aggregated_data[NETDATA_IDX_UDP_RECVBUF].call = res[NETDATA_KEY_CALLS_UDP_RECVMSG]; + socket_aggregated_data[NETDATA_IDX_UDP_SENDMSG].call = res[NETDATA_KEY_CALLS_UDP_SENDMSG]; + socket_aggregated_data[NETDATA_IDX_TCP_RETRANSMIT].call = res[NETDATA_KEY_TCP_RETRANSMIT]; + + socket_aggregated_data[NETDATA_IDX_TCP_SENDMSG].ecall = res[NETDATA_KEY_ERROR_TCP_SENDMSG]; + socket_aggregated_data[NETDATA_IDX_TCP_CLEANUP_RBUF].ecall = res[NETDATA_KEY_ERROR_TCP_CLEANUP_RBUF]; + socket_aggregated_data[NETDATA_IDX_UDP_RECVBUF].ecall = res[NETDATA_KEY_ERROR_UDP_RECVMSG]; + socket_aggregated_data[NETDATA_IDX_UDP_SENDMSG].ecall = res[NETDATA_KEY_ERROR_UDP_SENDMSG]; + + socket_aggregated_data[NETDATA_IDX_TCP_SENDMSG].bytes = res[NETDATA_KEY_BYTES_TCP_SENDMSG]; + socket_aggregated_data[NETDATA_IDX_TCP_CLEANUP_RBUF].bytes = res[NETDATA_KEY_BYTES_TCP_CLEANUP_RBUF]; + socket_aggregated_data[NETDATA_IDX_UDP_RECVBUF].bytes = res[NETDATA_KEY_BYTES_UDP_RECVMSG]; + socket_aggregated_data[NETDATA_IDX_UDP_SENDMSG].bytes = res[NETDATA_KEY_BYTES_UDP_SENDMSG]; } /** @@ -1745,6 +1779,59 @@ void clean_thread_structures() { } /** + * Cleanup publish syscall + * + * @param nps list of structures to clean + */ +void ebpf_cleanup_publish_syscall(netdata_publish_syscall_t *nps) +{ + while (nps) { + freez(nps->algorithm); + nps = nps->next; + } +} + +/** + * Clean port Structure + * + * Clean the allocated list. + * + * @param clean the list that will be cleaned + */ +void clean_port_structure(ebpf_network_viewer_port_list_t **clean) +{ + ebpf_network_viewer_port_list_t *move = *clean; + while (move) { + ebpf_network_viewer_port_list_t *next = move->next; + freez(move->value); + freez(move); + + move = next; + } + *clean = NULL; +} + +/** + * Clean IP structure + * + * Clean the allocated list. + * + * @param clean the list that will be cleaned + */ +static void clean_ip_structure(ebpf_network_viewer_ip_list_t **clean) +{ + ebpf_network_viewer_ip_list_t *move = *clean; + while (move) { + ebpf_network_viewer_ip_list_t *next = move->next; + freez(move->value); + freez(move); + + move = next; + } + *clean = NULL; +} + +/** * Clean up the main thread. * * @param ptr thread data. @@ -1763,9 +1850,7 @@ static void ebpf_socket_cleanup(void *ptr) UNUSED(dt); } - freez(socket_aggregated_data); ebpf_cleanup_publish_syscall(socket_publish_aggregated); - freez(socket_publish_aggregated); freez(socket_hash_values); clean_thread_structures(); @@ -1817,8 +1902,8 @@ static void ebpf_socket_cleanup(void *ptr) */ static void ebpf_socket_allocate_global_vectors(size_t length) { - socket_aggregated_data = callocz(length, sizeof(netdata_syscall_stat_t)); - socket_publish_aggregated = callocz(length, sizeof(netdata_publish_syscall_t)); + memset(socket_aggregated_data, 0 ,length * sizeof(netdata_syscall_stat_t)); + memset(socket_publish_aggregated, 0 ,length * sizeof(netdata_publish_syscall_t)); socket_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t)); socket_bandwidth_curr = callocz((size_t)pid_max, sizeof(ebpf_socket_publish_apps_t *)); @@ -1857,6 +1942,874 @@ static void initialize_inbound_outbound() *****************************************************************/ /** + * Fill Port list + * + * @param out a pointer to the link list. + * @param in the structure that will be linked. + */ +static inline void fill_port_list(ebpf_network_viewer_port_list_t **out, ebpf_network_viewer_port_list_t *in) +{ + if (likely(*out)) { + ebpf_network_viewer_port_list_t *move = *out, *store = *out; + uint16_t first = ntohs(in->first); + uint16_t last = ntohs(in->last); + while (move) { + uint16_t cmp_first = ntohs(move->first); + uint16_t cmp_last = ntohs(move->last); + if (cmp_first <= first && first <= cmp_last && + cmp_first <= last && last <= cmp_last ) { + info("The range/value (%u, %u) is inside the range/value (%u, %u) already inserted, it will be ignored.", + first, last, cmp_first, cmp_last); + freez(in->value); + freez(in); + return; + } else if (first <= cmp_first && cmp_first <= last && + first <= cmp_last && cmp_last <= last) { + info("The range (%u, %u) is bigger than previous range (%u, %u) already inserted, the previous will be ignored.", + first, last, cmp_first, cmp_last); + freez(move->value); + move->value = in->value; + move->first = in->first; + move->last = in->last; + freez(in); + return; + } + + store = move; + move = move->next; + } + + store->next = in; + } else { + *out = in; + } + +#ifdef NETDATA_INTERNAL_CHECKS + info("Adding values %s( %u, %u) to %s port list used on network viewer", + in->value, ntohs(in->first), ntohs(in->last), + (*out == network_viewer_opt.included_port)?"included":"excluded"); +#endif +} + +/** + * Parse Service List + * + * @param out a pointer to store the link list + * @param service the service used to create the structure that will be linked. + */ +static void parse_service_list(void **out, char *service) +{ + ebpf_network_viewer_port_list_t **list = (ebpf_network_viewer_port_list_t **)out; + struct servent *serv = getservbyname((const char *)service, "tcp"); + if (!serv) + serv = getservbyname((const char *)service, "udp"); + + if (!serv) { + info("Cannot resolv the service '%s' with protocols TCP and UDP, it will be ignored", service); + return; + } + + ebpf_network_viewer_port_list_t *w = callocz(1, sizeof(ebpf_network_viewer_port_list_t)); + w->value = strdupz(service); + w->hash = simple_hash(service); + + w->first = w->last = (uint16_t)serv->s_port; + + fill_port_list(list, w); +} + +/** + * Netmask + * + * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h) + * + * @param prefix create the netmask based in the CIDR value. + * + * @return + */ +static inline in_addr_t netmask(int prefix) { + + if (prefix == 0) + return (~((in_addr_t) - 1)); + else + return (in_addr_t)(~((1 << (32 - prefix)) - 1)); + +} + +/** + * Broadcast + * + * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h) + * + * @param addr is the ip address + * @param prefix is the CIDR value. + * + * @return It returns the last address of the range + */ +static inline in_addr_t broadcast(in_addr_t addr, int prefix) +{ + return (addr | ~netmask(prefix)); +} + +/** + * Network + * + * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h) + * + * @param addr is the ip address + * @param prefix is the CIDR value. + * + * @return It returns the first address of the range. + */ +static inline in_addr_t ipv4_network(in_addr_t addr, int prefix) +{ + return (addr & netmask(prefix)); +} + +/** + * IP to network long + * + * @param dst the vector to store the result + * @param ip the source ip given by our users. + * @param domain the ip domain (IPV4 or IPV6) + * @param source the original string + * + * @return it returns 0 on success and -1 otherwise. + */ +static inline int ip2nl(uint8_t *dst, char *ip, int domain, char *source) +{ + if (inet_pton(domain, ip, dst) <= 0) { + error("The address specified (%s) is invalid ", source); + return -1; + } + + return 0; +} + +/** + * Get IPV6 Last Address + * + * @param out the address to store the last address. + * @param in the address used to do the math. + * @param prefix number of bits used to calculate the address + */ +static void get_ipv6_last_addr(union netdata_ip_t *out, union netdata_ip_t *in, uint64_t prefix) +{ + uint64_t mask,tmp; + uint64_t ret[2]; + memcpy(ret, in->addr32, sizeof(union netdata_ip_t)); + + if (prefix == 128) { + memcpy(out->addr32, in->addr32, sizeof(union netdata_ip_t)); + return; + } else if (!prefix) { + ret[0] = ret[1] = 0xFFFFFFFFFFFFFFFF; + memcpy(out->addr32, ret, sizeof(union netdata_ip_t)); + return; + } else if (prefix <= 64) { + ret[1] = 0xFFFFFFFFFFFFFFFFULL; + + tmp = be64toh(ret[0]); + if (prefix > 0) { + mask = 0xFFFFFFFFFFFFFFFFULL << (64 - prefix); + tmp |= ~mask; + } + ret[0] = htobe64(tmp); + } else { + mask = 0xFFFFFFFFFFFFFFFFULL << (128 - prefix); + tmp = be64toh(ret[1]); + tmp |= ~mask; + ret[1] = htobe64(tmp); + } + + memcpy(out->addr32, ret, sizeof(union netdata_ip_t)); +} + +/** + * Calculate ipv6 first address + * + * @param out the address to store the first address. + * @param in the address used to do the math. + * @param prefix number of bits used to calculate the address + */ +static void get_ipv6_first_addr(union netdata_ip_t *out, union netdata_ip_t *in, uint64_t prefix) +{ + uint64_t mask,tmp; + uint64_t ret[2]; + + memcpy(ret, in->addr32, sizeof(union netdata_ip_t)); + + if (prefix == 128) { + memcpy(out->addr32, in->addr32, sizeof(union netdata_ip_t)); + return; + } else if (!prefix) { + ret[0] = ret[1] = 0; + memcpy(out->addr32, ret, sizeof(union netdata_ip_t)); + return; + } else if (prefix <= 64) { + ret[1] = 0ULL; + + tmp = be64toh(ret[0]); + if (prefix > 0) { + mask = 0xFFFFFFFFFFFFFFFFULL << (64 - prefix); + tmp &= mask; + } + ret[0] = htobe64(tmp); + } else { + mask = 0xFFFFFFFFFFFFFFFFULL << (128 - prefix); + tmp = be64toh(ret[1]); + tmp &= mask; + ret[1] = htobe64(tmp); + } + + memcpy(out->addr32, ret, sizeof(union netdata_ip_t)); +} + +/** + * Is ip inside the range + * + * Check if the ip is inside a IP range + * + * @param rfirst the first ip address of the range + * @param rlast the last ip address of the range + * @param cmpfirst the first ip to compare + * @param cmplast the last ip to compare + * @param family the IP family + * + * @return It returns 1 if the IP is inside the range and 0 otherwise + */ +static int is_ip_inside_range(union netdata_ip_t *rfirst, union netdata_ip_t *rlast, + union netdata_ip_t *cmpfirst, union netdata_ip_t *cmplast, int family) +{ + if (family == AF_INET) { + if (ntohl(rfirst->addr32[0]) <= ntohl(cmpfirst->addr32[0]) && + ntohl(rlast->addr32[0]) >= ntohl(cmplast->addr32[0])) + return 1; + } else { + if (memcmp(rfirst->addr8, cmpfirst->addr8, sizeof(union netdata_ip_t)) <= 0 && + memcmp(rlast->addr8, cmplast->addr8, sizeof(union netdata_ip_t)) >= 0) { + return 1; + } + + } + return 0; +} + +/** + * Fill IP list + * + * @param out a pointer to the link list. + * @param in the structure that will be linked. + */ +void fill_ip_list(ebpf_network_viewer_ip_list_t **out, ebpf_network_viewer_ip_list_t *in, char *table) +{ +#ifndef NETDATA_INTERNAL_CHECKS + UNUSED(table); +#endif + if (likely(*out)) { + ebpf_network_viewer_ip_list_t *move = *out, *store = *out; + while (move) { + if (in->ver == move->ver && is_ip_inside_range(&move->first, &move->last, &in->first, &in->last, in->ver)) { + info("The range/value (%s) is inside the range/value (%s) already inserted, it will be ignored.", + in->value, move->value); + freez(in->value); + freez(in); + return; + } + store = move; + move = move->next; + } + + store->next = in; + } else { + *out = in; + } + +#ifdef NETDATA_INTERNAL_CHECKS + char first[512], last[512]; + if (in->ver == AF_INET) { + if (inet_ntop(AF_INET, in->first.addr8, first, INET_ADDRSTRLEN) && + inet_ntop(AF_INET, in->last.addr8, last, INET_ADDRSTRLEN)) + info("Adding values %s - %s to %s IP list \"%s\" used on network viewer", + first, last, + (*out == network_viewer_opt.included_ips)?"included":"excluded", + table); + } else { + if (inet_ntop(AF_INET6, in->first.addr8, first, INET6_ADDRSTRLEN) && + inet_ntop(AF_INET6, in->last.addr8, last, INET6_ADDRSTRLEN)) + info("Adding values %s - %s to %s IP list \"%s\" used on network viewer", + first, last, + (*out == network_viewer_opt.included_ips)?"included":"excluded", + table); + } +#endif +} + +/** + * Parse IP List + * + * Parse IP list and link it. + * + * @param out a pointer to store the link list + * @param ip the value given as parameter + */ +static void parse_ip_list(void **out, char *ip) +{ + ebpf_network_viewer_ip_list_t **list = (ebpf_network_viewer_ip_list_t **)out; + + char *ipdup = strdupz(ip); + union netdata_ip_t first = { }; + union netdata_ip_t last = { }; + char *is_ipv6; + if (*ip == '*' && *(ip+1) == '\0') { + memset(first.addr8, 0, sizeof(first.addr8)); + memset(last.addr8, 0xFF, sizeof(last.addr8)); + + is_ipv6 = ip; + + clean_ip_structure(list); + goto storethisip; + } + + char *end = ip; + // Move while I cannot find a separator + while (*end && *end != '/' && *end != '-') end++; + + // We will use only the classic IPV6 for while, but we could consider the base 85 in a near future + // https://tools.ietf.org/html/rfc1924 + is_ipv6 = strchr(ip, ':'); + + int select; + if (*end && !is_ipv6) { // IPV4 range + select = (*end == '/') ? 0 : 1; + *end++ = '\0'; + if (*end == '!') { + info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup); + goto cleanipdup; + } + + if (!select) { // CIDR + select = ip2nl(first.addr8, ip, AF_INET, ipdup); + if (select) + goto cleanipdup; + + select = (int) str2i(end); + if (select < NETDATA_MINIMUM_IPV4_CIDR || select > NETDATA_MAXIMUM_IPV4_CIDR) { + info("The specified CIDR %s is not valid, the IP %s will be ignored.", end, ip); + goto cleanipdup; + } + + last.addr32[0] = htonl(broadcast(ntohl(first.addr32[0]), select)); + // This was added to remove + // https://app.codacy.com/manual/netdata/netdata/pullRequest?prid=5810941&bid=19021977 + UNUSED(last.addr32[0]); + + uint32_t ipv4_test = htonl(ipv4_network(ntohl(first.addr32[0]), select)); + if (first.addr32[0] != ipv4_test) { + first.addr32[0] = ipv4_test; + struct in_addr ipv4_convert; + ipv4_convert.s_addr = ipv4_test; + char ipv4_msg[INET_ADDRSTRLEN]; + if(inet_ntop(AF_INET, &ipv4_convert, ipv4_msg, INET_ADDRSTRLEN)) + info("The network value of CIDR %s was updated for %s .", ipdup, ipv4_msg); + } + } else { // Range + select = ip2nl(first.addr8, ip, AF_INET, ipdup); + if (select) + goto cleanipdup; + + select = ip2nl(last.addr8, end, AF_INET, ipdup); + if (select) + goto cleanipdup; + } + + if (htonl(first.addr32[0]) > htonl(last.addr32[0])) { + info("The specified range %s is invalid, the second address is smallest than the first, it will be ignored.", + ipdup); + goto cleanipdup; + } + } else if (is_ipv6) { // IPV6 + if (!*end) { // Unique + select = ip2nl(first.addr8, ip, AF_INET6, ipdup); + if (select) + goto cleanipdup; + + memcpy(last.addr8, first.addr8, sizeof(first.addr8)); + } else if (*end == '-') { + *end++ = 0x00; + if (*end == '!') { + info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup); + goto cleanipdup; + } + + select = ip2nl(first.addr8, ip, AF_INET6, ipdup); + if (select) + goto cleanipdup; + + select = ip2nl(last.addr8, end, AF_INET6, ipdup); + if (select) + goto cleanipdup; + } else { // CIDR + *end++ = 0x00; + if (*end == '!') { + info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup); + goto cleanipdup; + } + + select = str2i(end); + if (select < 0 || select > 128) { + info("The CIDR %s is not valid, the address %s will be ignored.", end, ip); + goto cleanipdup; + } + + uint64_t prefix = (uint64_t)select; + select = ip2nl(first.addr8, ip, AF_INET6, ipdup); + if (select) + goto cleanipdup; + + get_ipv6_last_addr(&last, &first, prefix); + + union netdata_ip_t ipv6_test; + get_ipv6_first_addr(&ipv6_test, &first, prefix); + + if (memcmp(first.addr8, ipv6_test.addr8, sizeof(union netdata_ip_t)) != 0) { + memcpy(first.addr8, ipv6_test.addr8, sizeof(union netdata_ip_t)); + + struct in6_addr ipv6_convert; + memcpy(ipv6_convert.s6_addr, ipv6_test.addr8, sizeof(union netdata_ip_t)); + + char ipv6_msg[INET6_ADDRSTRLEN]; + if(inet_ntop(AF_INET6, &ipv6_convert, ipv6_msg, INET6_ADDRSTRLEN)) + info("The network value of CIDR %s was updated for %s .", ipdup, ipv6_msg); + } + } + + if ((be64toh(*(uint64_t *)&first.addr32[2]) > be64toh(*(uint64_t *)&last.addr32[2]) && + !memcmp(first.addr32, last.addr32, 2*sizeof(uint32_t))) || + (be64toh(*(uint64_t *)&first.addr32) > be64toh(*(uint64_t *)&last.addr32)) ) { + info("The specified range %s is invalid, the second address is smallest than the first, it will be ignored.", + ipdup); + goto cleanipdup; + } + } else { // Unique ip + select = ip2nl(first.addr8, ip, AF_INET, ipdup); + if (select) + goto cleanipdup; + + memcpy(last.addr8, first.addr8, sizeof(first.addr8)); + } + + ebpf_network_viewer_ip_list_t *store; + + storethisip: + store = callocz(1, sizeof(ebpf_network_viewer_ip_list_t)); + store->value = ipdup; + store->hash = simple_hash(ipdup); + store->ver = (uint8_t)(!is_ipv6)?AF_INET:AF_INET6; + memcpy(store->first.addr8, first.addr8, sizeof(first.addr8)); + memcpy(store->last.addr8, last.addr8, sizeof(last.addr8)); + + fill_ip_list(list, store, "socket"); + return; + +cleanipdup: + freez(ipdup); +} + +/** + * Parse IP Range + * + * Parse the IP ranges given and create Network Viewer IP Structure + * + * @param ptr is a pointer with the text to parse. + */ +static void parse_ips(char *ptr) +{ + // No value + if (unlikely(!ptr)) + return; + + while (likely(ptr)) { + // Move forward until next valid character + while (isspace(*ptr)) ptr++; + + // No valid value found + if (unlikely(!*ptr)) + return; + + // Find space that ends the list + char *end = strchr(ptr, ' '); + if (end) { + *end++ = '\0'; + } + + int neg = 0; + if (*ptr == '!') { + neg++; + ptr++; + } + + if (isascii(*ptr)) { // Parse port + parse_ip_list((!neg)?(void **)&network_viewer_opt.included_ips:(void **)&network_viewer_opt.excluded_ips, + ptr); + } + + ptr = end; + } +} + + + +/** + * Parse port list + * + * Parse an allocated port list with the range given + * + * @param out a pointer to store the link list + * @param range the informed range for the user. + */ +static void parse_port_list(void **out, char *range) +{ + int first, last; + ebpf_network_viewer_port_list_t **list = (ebpf_network_viewer_port_list_t **)out; + + char *copied = strdupz(range); + if (*range == '*' && *(range+1) == '\0') { + first = 1; + last = 65535; + + clean_port_structure(list); + goto fillenvpl; + } + + char *end = range; + //Move while I cannot find a separator + while (*end && *end != ':' && *end != '-') end++; + + //It has a range + if (likely(*end)) { + *end++ = '\0'; + if (*end == '!') { + info("The exclusion cannot be in the second part of the range, the range %s will be ignored.", copied); + freez(copied); + return; + } + last = str2i((const char *)end); + } else { + last = 0; + } + + first = str2i((const char *)range); + if (first < NETDATA_MINIMUM_PORT_VALUE || first > NETDATA_MAXIMUM_PORT_VALUE) { + info("The first port %d of the range \"%s\" is invalid and it will be ignored!", first, copied); + freez(copied); + return; + } + + if (!last) + last = first; + + if (last < NETDATA_MINIMUM_PORT_VALUE || last > NETDATA_MAXIMUM_PORT_VALUE) { + info("The second port %d of the range \"%s\" is invalid and the whole range will be ignored!", last, copied); + freez(copied); + return; + } + + if (first > last) { + info("The specified order %s is wrong, the smallest value is always the first, it will be ignored!", copied); + freez(copied); + return; + } + + ebpf_network_viewer_port_list_t *w; +fillenvpl: + w = callocz(1, sizeof(ebpf_network_viewer_port_list_t)); + w->value = copied; + w->hash = simple_hash(copied); + w->first = (uint16_t)htons((uint16_t)first); + w->last = (uint16_t)htons((uint16_t)last); + w->cmp_first = (uint16_t)first; + w->cmp_last = (uint16_t)last; + + fill_port_list(list, w); +} + +/** + * Read max dimension. + * + * Netdata plot two dimensions per connection, so it is necessary to adjust the values. + * + * @param cfg the configuration structure + */ +static void read_max_dimension(struct config *cfg) +{ + int maxdim ; + maxdim = (int) appconfig_get_number(cfg, + EBPF_NETWORK_VIEWER_SECTION, + EBPF_MAXIMUM_DIMENSIONS, + NETDATA_NV_CAP_VALUE); + if (maxdim < 0) { + error("'maximum dimensions = %d' must be a positive number, Netdata will change for default value %ld.", + maxdim, NETDATA_NV_CAP_VALUE); + maxdim = NETDATA_NV_CAP_VALUE; + } + + maxdim /= 2; + if (!maxdim) { + info("The number of dimensions is too small (%u), we are setting it to minimum 2", network_viewer_opt.max_dim); + network_viewer_opt.max_dim = 1; + return; + } + + network_viewer_opt.max_dim = (uint32_t)maxdim; +} + +/** + * Parse Port Range + * + * Parse the port ranges given and create Network Viewer Port Structure + * + * @param ptr is a pointer with the text to parse. + */ +static void parse_ports(char *ptr) +{ + // No value + if (unlikely(!ptr)) + return; + + while (likely(ptr)) { + // Move forward until next valid character + while (isspace(*ptr)) ptr++; + + // No valid value found + if (unlikely(!*ptr)) + return; + + // Find space that ends the list + char *end = strchr(ptr, ' '); + if (end) { + *end++ = '\0'; + } + + int neg = 0; + if (*ptr == '!') { + neg++; + ptr++; + } + + if (isdigit(*ptr)) { // Parse port + parse_port_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port, + ptr); + } else if (isalpha(*ptr)) { // Parse service + parse_service_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port, + ptr); + } else if (*ptr == '*') { // All + parse_port_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port, + ptr); + } + + ptr = end; + } +} + +/** + * Link hostname + * + * @param out is the output link list + * @param in the hostname to add to list. + */ +static void link_hostname(ebpf_network_viewer_hostname_list_t **out, ebpf_network_viewer_hostname_list_t *in) +{ + if (likely(*out)) { + ebpf_network_viewer_hostname_list_t *move = *out; + for (; move->next ; move = move->next ) { + if (move->hash == in->hash && !strcmp(move->value, in->value)) { + info("The hostname %s was already inserted, it will be ignored.", in->value); + freez(in->value); + simple_pattern_free(in->value_pattern); + freez(in); + return; + } + } + + move->next = in; + } else { + *out = in; + } +#ifdef NETDATA_INTERNAL_CHECKS + info("Adding value %s to %s hostname list used on network viewer", + in->value, + (*out == network_viewer_opt.included_hostnames)?"included":"excluded"); +#endif +} + +/** + * Link Hostnames + * + * Parse the list of hostnames to create the link list. + * This is not associated with the IP, because simple patterns like *example* cannot be resolved to IP. + * + * @param out is the output link list + * @param parse is a pointer with the text to parser. + */ +static void link_hostnames(char *parse) +{ + // No value + if (unlikely(!parse)) + return; + + while (likely(parse)) { + // Find the first valid value + while (isspace(*parse)) parse++; + + // No valid value found + if (unlikely(!*parse)) + return; + + // Find space that ends the list + char *end = strchr(parse, ' '); + if (end) { + *end++ = '\0'; + } + + int neg = 0; + if (*parse == '!') { + neg++; + parse++; + } + + ebpf_network_viewer_hostname_list_t *hostname = callocz(1 , sizeof(ebpf_network_viewer_hostname_list_t)); + hostname->value = strdupz(parse); + hostname->hash = simple_hash(parse); + hostname->value_pattern = simple_pattern_create(parse, NULL, SIMPLE_PATTERN_EXACT); + + link_hostname((!neg)?&network_viewer_opt.included_hostnames:&network_viewer_opt.excluded_hostnames, + hostname); + + parse = end; + } +} + +/** + * Parse network viewer section + * + * @param cfg the configuration structure + */ +void parse_network_viewer_section(struct config *cfg) +{ + read_max_dimension(cfg); + + network_viewer_opt.hostname_resolution_enabled = appconfig_get_boolean(cfg, + EBPF_NETWORK_VIEWER_SECTION, + EBPF_CONFIG_RESOLVE_HOSTNAME, + CONFIG_BOOLEAN_NO); + + network_viewer_opt.service_resolution_enabled = appconfig_get_boolean(cfg, + EBPF_NETWORK_VIEWER_SECTION, + EBPF_CONFIG_RESOLVE_SERVICE, + CONFIG_BOOLEAN_NO); + + char *value = appconfig_get(cfg, EBPF_NETWORK_VIEWER_SECTION, EBPF_CONFIG_PORTS, NULL); + parse_ports(value); + + if (network_viewer_opt.hostname_resolution_enabled) { + value = appconfig_get(cfg, EBPF_NETWORK_VIEWER_SECTION, EBPF_CONFIG_HOSTNAMES, NULL); + link_hostnames(value); + } else { + info("Name resolution is disabled, collector will not parser \"hostnames\" list."); + } + + value = appconfig_get(cfg, EBPF_NETWORK_VIEWER_SECTION, + "ips", "!127.0.0.1/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 fc00::/7 !::1/128"); + parse_ips(value); +} + +/** + * Link dimension name + * + * Link user specified names inside a link list. + * + * @param port the port number associated to the dimension name. + * @param hash the calculated hash for the dimension name. + * @param name the dimension name. + */ +static void link_dimension_name(char *port, uint32_t hash, char *value) +{ + int test = str2i(port); + if (test < NETDATA_MINIMUM_PORT_VALUE || test > NETDATA_MAXIMUM_PORT_VALUE){ + error("The dimension given (%s = %s) has an invalid value and it will be ignored.", port, value); + return; + } + + ebpf_network_viewer_dim_name_t *w; + w = callocz(1, sizeof(ebpf_network_viewer_dim_name_t)); + + w->name = strdupz(value); + w->hash = hash; + + w->port = (uint16_t) htons(test); + + ebpf_network_viewer_dim_name_t *names = network_viewer_opt.names; + if (unlikely(!names)) { + network_viewer_opt.names = w; + } else { + for (; names->next; names = names->next) { + if (names->port == w->port) { + info("Dupplicated definition for a service, the name %s will be ignored. ", names->name); + freez(names->name); + names->name = w->name; + names->hash = w->hash; + freez(w); + return; + } + } + names->next = w; + } + +#ifdef NETDATA_INTERNAL_CHECKS + info("Adding values %s( %u) to dimension name list used on network viewer", w->name, htons(w->port)); +#endif +} + +/** + * Parse service Name section. + * + * This function gets the values that will be used to overwrite dimensions. + * + * @param cfg the configuration structure + */ +void parse_service_name_section(struct config *cfg) +{ + struct section *co = appconfig_get_section(cfg, EBPF_SERVICE_NAME_SECTION); + if (co) { + struct config_option *cv; + for (cv = co->values; cv ; cv = cv->next) { + link_dimension_name(cv->name, cv->hash, cv->value); + } + } + + // Always associated the default port to Netdata + ebpf_network_viewer_dim_name_t *names = network_viewer_opt.names; + if (names) { + uint16_t default_port = htons(19999); + while (names) { + if (names->port == default_port) + return; + + names = names->next; + } + } + + char *port_string = getenv("NETDATA_LISTEN_PORT"); + if (port_string) { + // if variable has an invalid value, we assume netdata is using 19999 + int default_port = str2i(port_string); + if (default_port > 0 && default_port < 65536) + link_dimension_name(port_string, simple_hash(port_string), "Netdata"); + } +} + +/** * Socket thread * * Thread used to generate socket charts. @@ -1875,6 +2828,10 @@ void *ebpf_socket_thread(void *ptr) ebpf_module_t *em = (ebpf_module_t *)ptr; fill_ebpf_data(&socket_data); + ebpf_update_module(em, &socket_config, NETDATA_NETWORK_CONFIG_FILE); + parse_network_viewer_section(&socket_config); + parse_service_name_section(&socket_config); + if (!em->enabled) goto endsocket; diff --git a/collectors/ebpf.plugin/ebpf_socket.h b/collectors/ebpf.plugin/ebpf_socket.h index 1316c003a..81001bab6 100644 --- a/collectors/ebpf.plugin/ebpf_socket.h +++ b/collectors/ebpf.plugin/ebpf_socket.h @@ -5,7 +5,6 @@ #include "libnetdata/avl/avl.h" // Vector indexes -#define NETDATA_MAX_SOCKET_VECTOR 6 #define NETDATA_UDP_START 3 #define NETDATA_RETRANSMIT_START 5 @@ -17,6 +16,28 @@ #define NETDATA_SOCKET_READ_SLEEP_MS 800000ULL +// config file +#define NETDATA_NETWORK_CONFIG_FILE "network.conf" +#define EBPF_NETWORK_VIEWER_SECTION "network connections" +#define EBPF_SERVICE_NAME_SECTION "service name" +#define EBPF_CONFIG_RESOLVE_HOSTNAME "resolve hostnames" +#define EBPF_CONFIG_RESOLVE_SERVICE "resolve service names" +#define EBPF_CONFIG_PORTS "ports" +#define EBPF_CONFIG_HOSTNAMES "hostnames" +#define EBPF_MAXIMUM_DIMENSIONS "maximum dimensions" + +enum ebpf_socket_publish_index { + NETDATA_IDX_TCP_SENDMSG, + NETDATA_IDX_TCP_CLEANUP_RBUF, + NETDATA_IDX_TCP_CLOSE, + NETDATA_IDX_UDP_RECVBUF, + NETDATA_IDX_UDP_SENDMSG, + NETDATA_IDX_TCP_RETRANSMIT, + + // Keep this as last and don't skip numbers as it is used as element counter + NETDATA_MAX_SOCKET_VECTOR +}; + typedef enum ebpf_socket_idx { NETDATA_KEY_CALLS_TCP_SENDMSG, NETDATA_KEY_ERROR_TCP_SENDMSG, @@ -38,6 +59,7 @@ typedef enum ebpf_socket_idx { NETDATA_KEY_TCP_RETRANSMIT, + // Keep this as last and don't skip numbers as it is used as element counter NETDATA_SOCKET_COUNTER } ebpf_socket_index_t; @@ -232,7 +254,7 @@ typedef struct netdata_socket_idx { */ typedef struct netdata_socket_plot { // Search - avl avl; + avl_t avl; netdata_socket_idx_t index; // Current data @@ -269,6 +291,9 @@ typedef struct netdata_vector_plot { extern void clean_port_structure(ebpf_network_viewer_port_list_t **clean); extern ebpf_network_viewer_port_list_t *listen_ports; extern void update_listen_table(uint16_t value, uint8_t proto); +extern void parse_network_viewer_section(struct config *cfg); +extern void fill_ip_list(ebpf_network_viewer_ip_list_t **out, ebpf_network_viewer_ip_list_t *in, char *table); +extern void parse_service_name_section(struct config *cfg); extern ebpf_socket_publish_apps_t **socket_bandwidth_curr; diff --git a/collectors/ebpf.plugin/ebpf_sync.c b/collectors/ebpf.plugin/ebpf_sync.c new file mode 100644 index 000000000..f0db1cc4a --- /dev/null +++ b/collectors/ebpf.plugin/ebpf_sync.c @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "ebpf.h" +#include "ebpf_sync.h" + +static ebpf_data_t sync_data; + +static char *sync_counter_dimension_name[NETDATA_SYNC_IDX_END] = { "sync", "syncfs", "msync", "fsync", "fdatasync", + "sync_file_range" }; +static netdata_syscall_stat_t sync_counter_aggregated_data[NETDATA_SYNC_IDX_END]; +static netdata_publish_syscall_t sync_counter_publish_aggregated[NETDATA_SYNC_IDX_END]; + +static int read_thread_closed = 1; + +static netdata_idx_t sync_hash_values[NETDATA_SYNC_IDX_END]; + +struct netdata_static_thread sync_threads = {"SYNC KERNEL", NULL, NULL, 1, + NULL, NULL, NULL}; + +struct config sync_config = { .first_section = NULL, + .last_section = NULL, + .mutex = NETDATA_MUTEX_INITIALIZER, + .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare }, + .rwlock = AVL_LOCK_INITIALIZER } }; + +ebpf_sync_syscalls_t local_syscalls[] = { + {.syscall = "sync", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL}, + {.syscall = "syncfs", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL}, + {.syscall = "msync", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL}, + {.syscall = "fsync", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL}, + {.syscall = "fdatasync", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL}, + {.syscall = "sync_file_range", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL}, + {.syscall = NULL, .enabled = CONFIG_BOOLEAN_NO, .objects = NULL, .probe_links = NULL} +}; + +/***************************************************************** + * + * INITIALIZE THREAD + * + *****************************************************************/ + +/* + * Initialize Syscalls + * + * Load the eBPF programs to monitor syscalls + * + * @return 0 on success and -1 otherwise. + */ +static int ebpf_sync_initialize_syscall(ebpf_module_t *em) +{ + int i; + const char *saved_name = em->thread_name; + for (i = 0; local_syscalls[i].syscall; i++) { + ebpf_sync_syscalls_t *w = &local_syscalls[i]; + if (!w->probe_links && w->enabled) { + fill_ebpf_data(&w->kernel_info); + if (ebpf_update_kernel(&w->kernel_info)) { + em->thread_name = saved_name; + error("Cannot update the kernel for eBPF module %s", w->syscall); + return -1; + } + + em->thread_name = w->syscall; + w->probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &w->objects, w->kernel_info.map_fd); + if (!w->probe_links) { + em->thread_name = saved_name; + return -1; + } + } + } + em->thread_name = saved_name; + + memset(sync_counter_aggregated_data, 0 , NETDATA_SYNC_IDX_END * sizeof(netdata_syscall_stat_t)); + memset(sync_counter_publish_aggregated, 0 , NETDATA_SYNC_IDX_END * sizeof(netdata_publish_syscall_t)); + memset(sync_hash_values, 0 , NETDATA_SYNC_IDX_END * sizeof(netdata_idx_t)); + + return 0; +} + +/***************************************************************** + * + * DATA THREAD + * + *****************************************************************/ + +/** + * Read global table + * + * Read the table with number of calls for all functions + */ +static void read_global_table() +{ + netdata_idx_t stored; + uint32_t idx = NETDATA_SYNC_CALL; + int i; + for (i = 0; local_syscalls[i].syscall; i++) { + if (local_syscalls[i].enabled) { + int fd = local_syscalls[i].kernel_info.map_fd[NETDATA_SYNC_GLOBLAL_TABLE]; + if (!bpf_map_lookup_elem(fd, &idx, &stored)) { + sync_hash_values[i] = stored; + } + } + } +} + +/** + * Sync read hash + * + * This is the thread callback. + * + * @param ptr It is a NULL value for this thread. + * + * @return It always returns NULL. + */ +void *ebpf_sync_read_hash(void *ptr) +{ + ebpf_module_t *em = (ebpf_module_t *)ptr; + read_thread_closed = 0; + + heartbeat_t hb; + heartbeat_init(&hb); + usec_t step = NETDATA_EBPF_SYNC_SLEEP_MS * em->update_time; + + while (!close_ebpf_plugin) { + usec_t dt = heartbeat_next(&hb, step); + (void)dt; + + read_global_table(); + } + read_thread_closed = 1; + + return NULL; +} + +/** + * Create Sync charts + * + * Create charts and dimensions according user input. + * + * @param id chart id + * @param idx the first index with data. + * @param end the last index with data. + */ +static void ebpf_send_sync_chart(char *id, + int idx, + int end) +{ + write_begin_chart(NETDATA_EBPF_MEMORY_GROUP, id); + + netdata_publish_syscall_t *move = &sync_counter_publish_aggregated[idx]; + + while (move && idx <= end) { + if (local_syscalls[idx].enabled) + write_chart_dimension(move->name, sync_hash_values[idx]); + + move = move->next; + idx++; + } + + write_end_chart(); +} + +/** + * Send data + * + * Send global charts to Netdata + */ +static void sync_send_data() +{ + if (local_syscalls[NETDATA_SYNC_FSYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_FDATASYNC_IDX].enabled) { + ebpf_send_sync_chart(NETDATA_EBPF_FILE_SYNC_CHART, NETDATA_SYNC_FSYNC_IDX, NETDATA_SYNC_FDATASYNC_IDX); + } + + if (local_syscalls[NETDATA_SYNC_MSYNC_IDX].enabled) + ebpf_one_dimension_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_EBPF_MSYNC_CHART, + sync_counter_publish_aggregated[NETDATA_SYNC_MSYNC_IDX].dimension, + sync_hash_values[NETDATA_SYNC_MSYNC_IDX]); + + if (local_syscalls[NETDATA_SYNC_SYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_SYNCFS_IDX].enabled) { + ebpf_send_sync_chart(NETDATA_EBPF_SYNC_CHART, NETDATA_SYNC_SYNC_IDX, NETDATA_SYNC_SYNCFS_IDX); + } + + if (local_syscalls[NETDATA_SYNC_SYNC_FILE_RANGE_IDX].enabled) + ebpf_one_dimension_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_EBPF_FILE_SEGMENT_CHART, + sync_counter_publish_aggregated[NETDATA_SYNC_SYNC_FILE_RANGE_IDX].dimension, + sync_hash_values[NETDATA_SYNC_SYNC_FILE_RANGE_IDX]); +} + +/** +* Main loop for this collector. +*/ +static void sync_collector(ebpf_module_t *em) +{ + sync_threads.thread = mallocz(sizeof(netdata_thread_t)); + sync_threads.start_routine = ebpf_sync_read_hash; + + netdata_thread_create(sync_threads.thread, sync_threads.name, NETDATA_THREAD_OPTION_JOINABLE, + ebpf_sync_read_hash, em); + + while (!close_ebpf_plugin) { + pthread_mutex_lock(&collect_data_mutex); + pthread_cond_wait(&collect_data_cond_var, &collect_data_mutex); + + pthread_mutex_lock(&lock); + + sync_send_data(); + + pthread_mutex_unlock(&lock); + pthread_mutex_unlock(&collect_data_mutex); + } +} + + +/***************************************************************** + * + * CLEANUP THREAD + * + *****************************************************************/ + +/** + * Cleanup Objects + * + * Cleanup loaded objects when thread was initialized. + */ +void ebpf_sync_cleanup_objects() +{ + int i; + for (i = 0; local_syscalls[i].syscall; i++) { + ebpf_sync_syscalls_t *w = &local_syscalls[i]; + if (w->probe_links) { + freez(w->kernel_info.map_fd); + + struct bpf_program *prog; + size_t j = 0 ; + bpf_object__for_each_program(prog, w->objects) { + bpf_link__destroy(w->probe_links[j]); + j++; + } + bpf_object__close(w->objects); + } + } +} + +/** + * Clean up the main thread. + * + * @param ptr thread data. + */ +static void ebpf_sync_cleanup(void *ptr) +{ + ebpf_module_t *em = (ebpf_module_t *)ptr; + if (!em->enabled) + return; + + heartbeat_t hb; + heartbeat_init(&hb); + uint32_t tick = 2*USEC_PER_MS; + while (!read_thread_closed) { + usec_t dt = heartbeat_next(&hb, tick); + UNUSED(dt); + } + + ebpf_sync_cleanup_objects(); + freez(sync_threads.thread); +} + +/***************************************************************** + * + * MAIN THREAD + * + *****************************************************************/ + +/** + * Create Sync charts + * + * Create charts and dimensions according user input. + * + * @param id chart id + * @param title chart title + * @param order order number of the specified chart + * @param idx the first index with data. + * @param end the last index with data. + */ +static void ebpf_create_sync_chart(char *id, + char *title, + int order, + int idx, + int end) +{ + ebpf_write_chart_cmd(NETDATA_EBPF_MEMORY_GROUP, id, title, EBPF_COMMON_DIMENSION_CALL, + NETDATA_EBPF_SYNC_SUBMENU, NETDATA_EBPF_CHART_TYPE_LINE, NULL, order); + + netdata_publish_syscall_t *move = &sync_counter_publish_aggregated[idx]; + + while (move && idx <= end) { + if (local_syscalls[idx].enabled) + ebpf_write_global_dimension(move->name, move->dimension, move->algorithm); + + move = move->next; + idx++; + } +} + +/** + * Create global charts + * + * Call ebpf_create_chart to create the charts for the collector. + */ +static void ebpf_create_sync_charts() +{ + if (local_syscalls[NETDATA_SYNC_FSYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_FDATASYNC_IDX].enabled) + ebpf_create_sync_chart(NETDATA_EBPF_FILE_SYNC_CHART, + "Monitor calls for <code>fsync(2)</code> and <code>fdatasync(2)</code>.", 21300, + NETDATA_SYNC_FSYNC_IDX, NETDATA_SYNC_FDATASYNC_IDX); + + if (local_syscalls[NETDATA_SYNC_MSYNC_IDX].enabled) + ebpf_create_sync_chart(NETDATA_EBPF_MSYNC_CHART, + "Monitor calls for <code>msync(2)</code>.", 21301, + NETDATA_SYNC_MSYNC_IDX, NETDATA_SYNC_MSYNC_IDX); + + if (local_syscalls[NETDATA_SYNC_SYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_SYNCFS_IDX].enabled) + ebpf_create_sync_chart(NETDATA_EBPF_SYNC_CHART, + "Monitor calls for <code>sync(2)</code> and <code>syncfs(2)</code>.", 21302, + NETDATA_SYNC_SYNC_IDX, NETDATA_SYNC_SYNCFS_IDX); + + if (local_syscalls[NETDATA_SYNC_SYNC_FILE_RANGE_IDX].enabled) + ebpf_create_sync_chart(NETDATA_EBPF_FILE_SEGMENT_CHART, + "Monitor calls for <code>sync_file_range(2)</code>.", 21303, + NETDATA_SYNC_SYNC_FILE_RANGE_IDX, NETDATA_SYNC_SYNC_FILE_RANGE_IDX); +} + +/** + * Parse Syscalls + * + * Parse syscall options available inside ebpf.d/sync.conf + */ +static void ebpf_sync_parse_syscalls() +{ + int i; + for (i = 0; local_syscalls[i].syscall; i++) { + local_syscalls[i].enabled = appconfig_get_boolean(&sync_config, NETDATA_SYNC_CONFIG_NAME, + local_syscalls[i].syscall, CONFIG_BOOLEAN_YES); + } +} + +/** + * Sync thread + * + * Thread used to make sync thread + * + * @param ptr a pointer to `struct ebpf_module` + * + * @return It always return NULL + */ +void *ebpf_sync_thread(void *ptr) +{ + netdata_thread_cleanup_push(ebpf_sync_cleanup, ptr); + + ebpf_module_t *em = (ebpf_module_t *)ptr; + fill_ebpf_data(&sync_data); + + ebpf_update_module(em, &sync_config, NETDATA_SYNC_CONFIG_FILE); + ebpf_sync_parse_syscalls(); + + if (!em->enabled) + goto endsync; + + if (ebpf_sync_initialize_syscall(em)) { + pthread_mutex_unlock(&lock); + goto endsync; + } + + int algorithms[NETDATA_SYNC_IDX_END] = { NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX, + NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX, + NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX }; + ebpf_global_labels(sync_counter_aggregated_data, sync_counter_publish_aggregated, + sync_counter_dimension_name, sync_counter_dimension_name, + algorithms, NETDATA_SYNC_IDX_END); + + pthread_mutex_lock(&lock); + ebpf_create_sync_charts(); + pthread_mutex_unlock(&lock); + + sync_collector(em); + +endsync: + netdata_thread_cleanup_pop(1); + return NULL; +} diff --git a/collectors/ebpf.plugin/ebpf_sync.h b/collectors/ebpf.plugin/ebpf_sync.h new file mode 100644 index 000000000..458318218 --- /dev/null +++ b/collectors/ebpf.plugin/ebpf_sync.h @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_EBPF_SYNC_H +#define NETDATA_EBPF_SYNC_H 1 + +// charts +#define NETDATA_EBPF_SYNC_CHART "sync" +#define NETDATA_EBPF_MSYNC_CHART "memory_map" +#define NETDATA_EBPF_FILE_SYNC_CHART "file_sync" +#define NETDATA_EBPF_FILE_SEGMENT_CHART "file_segment" +#define NETDATA_EBPF_SYNC_SUBMENU "synchronization (eBPF)" + +#define NETDATA_EBPF_SYNC_SLEEP_MS 800000ULL + +// configuration file +#define NETDATA_SYNC_CONFIG_FILE "sync.conf" +#define NETDATA_SYNC_CONFIG_NAME "syscalls" + +enum sync_syscalls_index { + NETDATA_SYNC_SYNC_IDX, + NETDATA_SYNC_SYNCFS_IDX, + NETDATA_SYNC_MSYNC_IDX, + NETDATA_SYNC_FSYNC_IDX, + NETDATA_SYNC_FDATASYNC_IDX, + NETDATA_SYNC_SYNC_FILE_RANGE_IDX, + + NETDATA_SYNC_IDX_END +}; + +typedef struct ebpf_sync_syscalls { + char *syscall; + int enabled; + uint32_t flags; + + struct bpf_object *objects; + struct bpf_link **probe_links; + + ebpf_data_t kernel_info; +} ebpf_sync_syscalls_t; + +enum netdata_sync_charts { + NETDATA_SYNC_CALL, + + // Keep this as last and don't skip numbers as it is used as element counter + NETDATA_SYNC_END +}; + +enum netdata_sync_table { + NETDATA_SYNC_GLOBLAL_TABLE +}; + +extern void *ebpf_sync_thread(void *ptr); + +#endif /* NETDATA_EBPF_SYNC_H */ |