summaryrefslogtreecommitdiffstats
path: root/collectors/proc.plugin/sys_class_infiniband.c
diff options
context:
space:
mode:
Diffstat (limited to 'collectors/proc.plugin/sys_class_infiniband.c')
-rw-r--r--collectors/proc.plugin/sys_class_infiniband.c704
1 files changed, 704 insertions, 0 deletions
diff --git a/collectors/proc.plugin/sys_class_infiniband.c b/collectors/proc.plugin/sys_class_infiniband.c
new file mode 100644
index 0000000..46f40f2
--- /dev/null
+++ b/collectors/proc.plugin/sys_class_infiniband.c
@@ -0,0 +1,704 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Heavily inspired from proc_net_dev.c
+#include "plugin_proc.h"
+
+#define PLUGIN_PROC_MODULE_INFINIBAND_NAME "/sys/class/infiniband"
+#define CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND \
+ "plugin:" PLUGIN_PROC_CONFIG_NAME ":" PLUGIN_PROC_MODULE_INFINIBAND_NAME
+
+// ib_device::name[IB_DEVICE_NAME_MAX(64)] + "-" + ib_device::phys_port_cnt[u8 = 3 chars]
+#define IBNAME_MAX 68
+
+// ----------------------------------------------------------------------------
+// infiniband & omnipath standard counters
+
+// I use macro as there's no single file acting as summary, but a lot of different files, so can't use helpers like
+// procfile(). Also, omnipath generates other counters, that are not provided by infiniband
+#define FOREACH_COUNTER(GEN, ...) \
+ FOREACH_COUNTER_BYTES(GEN, __VA_ARGS__) \
+ FOREACH_COUNTER_PACKETS(GEN, __VA_ARGS__) \
+ FOREACH_COUNTER_ERRORS(GEN, __VA_ARGS__)
+
+#define FOREACH_COUNTER_BYTES(GEN, ...) \
+ GEN(port_rcv_data, bytes, "Received", 1, __VA_ARGS__) \
+ GEN(port_xmit_data, bytes, "Sent", -1, __VA_ARGS__)
+
+#define FOREACH_COUNTER_PACKETS(GEN, ...) \
+ GEN(port_rcv_packets, packets, "Received", 1, __VA_ARGS__) \
+ GEN(port_xmit_packets, packets, "Sent", -1, __VA_ARGS__) \
+ GEN(multicast_rcv_packets, packets, "Mcast rcvd", 1, __VA_ARGS__) \
+ GEN(multicast_xmit_packets, packets, "Mcast sent", -1, __VA_ARGS__) \
+ GEN(unicast_rcv_packets, packets, "Ucast rcvd", 1, __VA_ARGS__) \
+ GEN(unicast_xmit_packets, packets, "Ucast sent", -1, __VA_ARGS__)
+
+#define FOREACH_COUNTER_ERRORS(GEN, ...) \
+ GEN(port_rcv_errors, errors, "Pkts malformated", 1, __VA_ARGS__) \
+ GEN(port_rcv_constraint_errors, errors, "Pkts rcvd discarded ", 1, __VA_ARGS__) \
+ GEN(port_xmit_discards, errors, "Pkts sent discarded", 1, __VA_ARGS__) \
+ GEN(port_xmit_wait, errors, "Tick Wait to send", 1, __VA_ARGS__) \
+ GEN(VL15_dropped, errors, "Pkts missed ressource", 1, __VA_ARGS__) \
+ GEN(excessive_buffer_overrun_errors, errors, "Buffer overrun", 1, __VA_ARGS__) \
+ GEN(link_downed, errors, "Link Downed", 1, __VA_ARGS__) \
+ GEN(link_error_recovery, errors, "Link recovered", 1, __VA_ARGS__) \
+ GEN(local_link_integrity_errors, errors, "Link integrity err", 1, __VA_ARGS__) \
+ GEN(symbol_error, errors, "Link minor errors", 1, __VA_ARGS__) \
+ GEN(port_rcv_remote_physical_errors, errors, "Pkts rcvd with EBP", 1, __VA_ARGS__) \
+ GEN(port_rcv_switch_relay_errors, errors, "Pkts rcvd discarded by switch", 1, __VA_ARGS__) \
+ GEN(port_xmit_constraint_errors, errors, "Pkts sent discarded by switch", 1, __VA_ARGS__)
+
+//
+// Hardware Counters
+//
+
+// IMPORTANT: These are vendor-specific fields.
+// If you want to add a new vendor, search this for for 'VENDORS:' keyword and
+// add your definition as 'VENDOR-<key>:' where <key> if the string part that
+// is shown in /sys/class/infiniband/<key>X_Y
+// EG: for Mellanox, shown as mlx0_1, it's 'mlx'
+// for Intel, shown as hfi1_1, it's 'hfi'
+
+// VENDORS: List of implemented hardware vendors
+#define FOREACH_HWCOUNTER_NAME(GEN, ...) GEN(mlx, __VA_ARGS__)
+
+// VENDOR-MLX: HW Counters for Mellanox ConnectX Devices
+#define FOREACH_HWCOUNTER_MLX(GEN, ...) \
+ FOREACH_HWCOUNTER_MLX_PACKETS(GEN, __VA_ARGS__) \
+ FOREACH_HWCOUNTER_MLX_ERRORS(GEN, __VA_ARGS__)
+
+#define FOREACH_HWCOUNTER_MLX_PACKETS(GEN, ...) \
+ GEN(np_cnp_sent, hwpackets, "RoCEv2 Congestion sent", 1, __VA_ARGS__) \
+ GEN(np_ecn_marked_roce_packets, hwpackets, "RoCEv2 Congestion rcvd", -1, __VA_ARGS__) \
+ GEN(rp_cnp_handled, hwpackets, "IB Congestion handled", 1, __VA_ARGS__) \
+ GEN(rx_atomic_requests, hwpackets, "ATOMIC req. rcvd", 1, __VA_ARGS__) \
+ GEN(rx_dct_connect, hwpackets, "Connection req. rcvd", 1, __VA_ARGS__) \
+ GEN(rx_read_requests, hwpackets, "Read req. rcvd", 1, __VA_ARGS__) \
+ GEN(rx_write_requests, hwpackets, "Write req. rcvd", 1, __VA_ARGS__) \
+ GEN(roce_adp_retrans, hwpackets, "RoCE retrans adaptive", 1, __VA_ARGS__) \
+ GEN(roce_adp_retrans_to, hwpackets, "RoCE retrans timeout", 1, __VA_ARGS__) \
+ GEN(roce_slow_restart, hwpackets, "RoCE slow restart", 1, __VA_ARGS__) \
+ GEN(roce_slow_restart_cnps, hwpackets, "RoCE slow restart congestion", 1, __VA_ARGS__) \
+ GEN(roce_slow_restart_trans, hwpackets, "RoCE slow restart count", 1, __VA_ARGS__)
+
+#define FOREACH_HWCOUNTER_MLX_ERRORS(GEN, ...) \
+ GEN(duplicate_request, hwerrors, "Duplicated packets", -1, __VA_ARGS__) \
+ GEN(implied_nak_seq_err, hwerrors, "Pkt Seq Num gap", 1, __VA_ARGS__) \
+ GEN(local_ack_timeout_err, hwerrors, "Ack timer expired", 1, __VA_ARGS__) \
+ GEN(out_of_buffer, hwerrors, "Drop missing buffer", 1, __VA_ARGS__) \
+ GEN(out_of_sequence, hwerrors, "Drop out of sequence", 1, __VA_ARGS__) \
+ GEN(packet_seq_err, hwerrors, "NAK sequence rcvd", 1, __VA_ARGS__) \
+ GEN(req_cqe_error, hwerrors, "CQE err Req", 1, __VA_ARGS__) \
+ GEN(resp_cqe_error, hwerrors, "CQE err Resp", 1, __VA_ARGS__) \
+ GEN(req_cqe_flush_error, hwerrors, "CQE Flushed err Req", 1, __VA_ARGS__) \
+ GEN(resp_cqe_flush_error, hwerrors, "CQE Flushed err Resp", 1, __VA_ARGS__) \
+ GEN(req_remote_access_errors, hwerrors, "Remote access err Req", 1, __VA_ARGS__) \
+ GEN(resp_remote_access_errors, hwerrors, "Remote access err Resp", 1, __VA_ARGS__) \
+ GEN(req_remote_invalid_request, hwerrors, "Remote invalid req", 1, __VA_ARGS__) \
+ GEN(resp_local_length_error, hwerrors, "Local length err Resp", 1, __VA_ARGS__) \
+ GEN(rnr_nak_retry_err, hwerrors, "RNR NAK Packets", 1, __VA_ARGS__) \
+ GEN(rp_cnp_ignored, hwerrors, "CNP Pkts ignored", 1, __VA_ARGS__) \
+ GEN(rx_icrc_encapsulated, hwerrors, "RoCE ICRC Errors", 1, __VA_ARGS__)
+
+// Common definitions used more than once
+#define GEN_RRD_DIM_ADD(NAME, GRP, DESC, DIR, PORT) \
+ GEN_RRD_DIM_ADD_CUSTOM(NAME, GRP, DESC, DIR, PORT, 1, 1, RRD_ALGORITHM_INCREMENTAL)
+
+#define GEN_RRD_DIM_ADD_CUSTOM(NAME, GRP, DESC, DIR, PORT, MULT, DIV, ALGO) \
+ PORT->rd_##NAME = rrddim_add(PORT->st_##GRP, DESC, NULL, DIR * MULT, DIV, ALGO);
+
+#define GEN_RRD_DIM_ADD_HW(NAME, GRP, DESC, DIR, PORT, HW) \
+ HW->rd_##NAME = rrddim_add(PORT->st_##GRP, DESC, NULL, DIR, 1, RRD_ALGORITHM_INCREMENTAL);
+
+#define GEN_RRD_DIM_SETP(NAME, GRP, DESC, DIR, PORT) \
+ rrddim_set_by_pointer(PORT->st_##GRP, PORT->rd_##NAME, (collected_number)PORT->NAME);
+
+#define GEN_RRD_DIM_SETP_HW(NAME, GRP, DESC, DIR, PORT, HW) \
+ rrddim_set_by_pointer(PORT->st_##GRP, HW->rd_##NAME, (collected_number)HW->NAME);
+
+// https://community.mellanox.com/s/article/understanding-mlx5-linux-counters-and-status-parameters
+// https://community.mellanox.com/s/article/infiniband-port-counters
+static struct ibport {
+ char *name;
+ char *counters_path;
+ char *hwcounters_path;
+ int len;
+
+ // flags
+ int configured;
+ int enabled;
+ int discovered;
+
+ int do_bytes;
+ int do_packets;
+ int do_errors;
+ int do_hwpackets;
+ int do_hwerrors;
+
+ const char *chart_type_bytes;
+ const char *chart_type_packets;
+ const char *chart_type_errors;
+ const char *chart_type_hwpackets;
+ const char *chart_type_hwerrors;
+
+ const char *chart_id_bytes;
+ const char *chart_id_packets;
+ const char *chart_id_errors;
+ const char *chart_id_hwpackets;
+ const char *chart_id_hwerrors;
+
+ const char *chart_family;
+
+ unsigned long priority;
+
+ // Port details using drivers/infiniband/core/sysfs.c :: rate_show()
+ RRDDIM *rd_speed;
+ uint64_t speed;
+ uint64_t width;
+
+// Stats from /$device/ports/$portid/counters
+// as drivers/infiniband/hw/qib/qib_verbs.h
+// All uint64 except vl15_dropped, local_link_integrity_errors, excessive_buffer_overrun_errors uint32
+// Will generate 2 elements for each counter:
+// - uint64_t to store the value
+// - char* to store the filename path
+// - RRDDIM* to store the RRD Dimension
+#define GEN_DEF_COUNTER(NAME, ...) \
+ uint64_t NAME; \
+ char *file_##NAME; \
+ RRDDIM *rd_##NAME;
+ FOREACH_COUNTER(GEN_DEF_COUNTER)
+
+// Vendor specific hwcounters from /$device/ports/$portid/hw_counters
+// We will generate one struct pointer per vendor to avoid future casting
+#define GEN_DEF_HWCOUNTER_PTR(VENDOR, ...) struct ibporthw_##VENDOR *hwcounters_##VENDOR;
+ FOREACH_HWCOUNTER_NAME(GEN_DEF_HWCOUNTER_PTR)
+
+ // Function pointer to the "infiniband_hwcounters_parse_<vendor>" function
+ void (*hwcounters_parse)(struct ibport *);
+ void (*hwcounters_dorrd)(struct ibport *);
+
+ // charts and dim
+ RRDSET *st_bytes;
+ RRDSET *st_packets;
+ RRDSET *st_errors;
+ RRDSET *st_hwpackets;
+ RRDSET *st_hwerrors;
+
+ RRDSETVAR *stv_speed;
+
+ usec_t speed_last_collected_usec;
+
+ struct ibport *next;
+} *ibport_root = NULL, *ibport_last_used = NULL;
+
+// VENDORS: reading / calculation functions
+#define GEN_DEF_HWCOUNTER(NAME, ...) \
+ uint64_t NAME; \
+ char *file_##NAME; \
+ RRDDIM *rd_##NAME;
+
+#define GEN_DO_HWCOUNTER_READ(NAME, GRP, DESC, DIR, PORT, HW, ...) \
+ if (HW->file_##NAME) { \
+ if (read_single_number_file(HW->file_##NAME, (unsigned long long *)&HW->NAME)) { \
+ error("cannot read iface '%s' hwcounter '" #HW "'", PORT->name); \
+ HW->file_##NAME = NULL; \
+ } \
+ }
+
+// VENDOR-MLX: Mellanox
+struct ibporthw_mlx {
+ FOREACH_HWCOUNTER_MLX(GEN_DEF_HWCOUNTER)
+};
+void infiniband_hwcounters_parse_mlx(struct ibport *port)
+{
+ if (port->do_hwerrors != CONFIG_BOOLEAN_NO)
+ FOREACH_HWCOUNTER_MLX_ERRORS(GEN_DO_HWCOUNTER_READ, port, port->hwcounters_mlx)
+ if (port->do_hwpackets != CONFIG_BOOLEAN_NO)
+ FOREACH_HWCOUNTER_MLX_PACKETS(GEN_DO_HWCOUNTER_READ, port, port->hwcounters_mlx)
+}
+void infiniband_hwcounters_dorrd_mlx(struct ibport *port)
+{
+ if (port->do_hwerrors != CONFIG_BOOLEAN_NO) {
+ FOREACH_HWCOUNTER_MLX_ERRORS(GEN_RRD_DIM_SETP_HW, port, port->hwcounters_mlx)
+ rrdset_done(port->st_hwerrors);
+ }
+ if (port->do_hwpackets != CONFIG_BOOLEAN_NO) {
+ FOREACH_HWCOUNTER_MLX_PACKETS(GEN_RRD_DIM_SETP_HW, port, port->hwcounters_mlx)
+ rrdset_done(port->st_hwpackets);
+ }
+}
+
+// ----------------------------------------------------------------------------
+
+static struct ibport *get_ibport(const char *dev, const char *port)
+{
+ struct ibport *p;
+
+ char name[IBNAME_MAX + 1];
+ snprintfz(name, IBNAME_MAX, "%s-%s", dev, port);
+
+ // search it, resuming from the last position in sequence
+ for (p = ibport_last_used; p; p = p->next) {
+ if (unlikely(!strcmp(name, p->name))) {
+ ibport_last_used = p->next;
+ return p;
+ }
+ }
+
+ // new round, from the beginning to the last position used this time
+ for (p = ibport_root; p != ibport_last_used; p = p->next) {
+ if (unlikely(!strcmp(name, p->name))) {
+ ibport_last_used = p->next;
+ return p;
+ }
+ }
+
+ // create a new one
+ p = callocz(1, sizeof(struct ibport));
+ p->name = strdupz(name);
+ p->len = strlen(p->name);
+
+ p->chart_type_bytes = strdupz("infiniband_cnt_bytes");
+ p->chart_type_packets = strdupz("infiniband_cnt_packets");
+ p->chart_type_errors = strdupz("infiniband_cnt_errors");
+ p->chart_type_hwpackets = strdupz("infiniband_hwc_packets");
+ p->chart_type_hwerrors = strdupz("infiniband_hwc_errors");
+
+ char buffer[RRD_ID_LENGTH_MAX + 1];
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cntbytes_%s", p->name);
+ p->chart_id_bytes = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cntpackets_%s", p->name);
+ p->chart_id_packets = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_cnterrors_%s", p->name);
+ p->chart_id_errors = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_hwcntpackets_%s", p->name);
+ p->chart_id_hwpackets = strdupz(buffer);
+
+ snprintfz(buffer, RRD_ID_LENGTH_MAX, "ib_hwcnterrors_%s", p->name);
+ p->chart_id_hwerrors = strdupz(buffer);
+
+ p->chart_family = strdupz(p->name);
+ p->priority = NETDATA_CHART_PRIO_INFINIBAND;
+
+ // Link current ibport to last one in the list
+ if (ibport_root) {
+ struct ibport *t;
+ for (t = ibport_root; t->next; t = t->next)
+ ;
+ t->next = p;
+ } else
+ ibport_root = p;
+
+ return p;
+}
+
+int do_sys_class_infiniband(int update_every, usec_t dt)
+{
+ (void)dt;
+ static SIMPLE_PATTERN *disabled_list = NULL;
+ static int initialized = 0;
+ static int enable_new_ports = -1, enable_only_active = CONFIG_BOOLEAN_YES;
+ static int do_bytes = -1, do_packets = -1, do_errors = -1, do_hwpackets = -1, do_hwerrors = -1;
+ static char *sys_class_infiniband_dirname = NULL;
+
+ static long long int dt_to_refresh_ports = 0, last_refresh_ports_usec = 0;
+
+ if (unlikely(enable_new_ports == -1)) {
+ char dirname[FILENAME_MAX + 1];
+
+ snprintfz(dirname, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/sys/class/infiniband");
+ sys_class_infiniband_dirname =
+ config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "dirname to monitor", dirname);
+
+ do_bytes = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "bandwidth counters", CONFIG_BOOLEAN_YES);
+ do_packets = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "packets counters", CONFIG_BOOLEAN_YES);
+ do_errors = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "errors counters", CONFIG_BOOLEAN_YES);
+ do_hwpackets = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "hardware packets counters", CONFIG_BOOLEAN_AUTO);
+ do_hwerrors = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "hardware errors counters", CONFIG_BOOLEAN_AUTO);
+
+ enable_only_active = config_get_boolean_ondemand(
+ CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "monitor only active ports", CONFIG_BOOLEAN_AUTO);
+ disabled_list = simple_pattern_create(
+ config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "disable by default interfaces matching", ""), NULL,
+ SIMPLE_PATTERN_EXACT);
+
+ dt_to_refresh_ports =
+ config_get_number(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "refresh ports state every seconds", 30) *
+ USEC_PER_SEC;
+ if (dt_to_refresh_ports < 0)
+ dt_to_refresh_ports = 0;
+ }
+
+ // init listing of /sys/class/infiniband/ (or rediscovery)
+ if (unlikely(!initialized) || unlikely(last_refresh_ports_usec >= dt_to_refresh_ports)) {
+ // If folder does not exists, return 1 to disable
+ DIR *devices_dir = opendir(sys_class_infiniband_dirname);
+ if (unlikely(!devices_dir))
+ return 1;
+
+ // Work on all device available
+ struct dirent *dev_dent;
+ while ((dev_dent = readdir(devices_dir))) {
+ // Skip special folders
+ if (!strcmp(dev_dent->d_name, "..") || !strcmp(dev_dent->d_name, "."))
+ continue;
+
+ // /sys/class/infiniband/<dev>/ports
+ char ports_dirname[FILENAME_MAX + 1];
+ snprintfz(ports_dirname, FILENAME_MAX, "%s/%s/%s", sys_class_infiniband_dirname, dev_dent->d_name, "ports");
+
+ DIR *ports_dir = opendir(ports_dirname);
+ if (unlikely(!ports_dir))
+ continue;
+
+ struct dirent *port_dent;
+ while ((port_dent = readdir(ports_dir))) {
+ // Skip special folders
+ if (!strcmp(port_dent->d_name, "..") || !strcmp(port_dent->d_name, "."))
+ continue;
+
+ char buffer[FILENAME_MAX + 1];
+
+ // Check if counters are availablea (mandatory)
+ // /sys/class/infiniband/<device>/ports/<port>/counters
+ char counters_dirname[FILENAME_MAX + 1];
+ snprintfz(counters_dirname, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "counters");
+ DIR *counters_dir = opendir(counters_dirname);
+ // Standard counters are mandatory
+ if (!counters_dir)
+ continue;
+ closedir(counters_dir);
+
+ // Hardware Counters are optionnal, used later
+ char hwcounters_dirname[FILENAME_MAX + 1];
+ snprintfz(
+ hwcounters_dirname, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "hw_counters");
+
+ // Get new or existing ibport
+ struct ibport *p = get_ibport(dev_dent->d_name, port_dent->d_name);
+ if (!p)
+ continue;
+
+ // Prepare configuration
+ if (!p->configured) {
+ p->configured = 1;
+
+ // Enable by default, will be filtered out later
+ p->enabled = 1;
+
+ p->counters_path = strdupz(counters_dirname);
+ p->hwcounters_path = strdupz(hwcounters_dirname);
+
+ snprintfz(buffer, FILENAME_MAX, "plugin:proc:/sys/class/infiniband:%s", p->name);
+
+ // Standard counters
+ p->do_bytes = config_get_boolean_ondemand(buffer, "bytes", do_bytes);
+ p->do_packets = config_get_boolean_ondemand(buffer, "packets", do_packets);
+ p->do_errors = config_get_boolean_ondemand(buffer, "errors", do_errors);
+
+// Gen filename allocation and concatenation
+#define GEN_DO_COUNTER_NAME(NAME, GRP, DESC, DIR, PORT, ...) \
+ PORT->file_##NAME = callocz(1, strlen(PORT->counters_path) + sizeof(#NAME) + 3); \
+ strcat(PORT->file_##NAME, PORT->counters_path); \
+ strcat(PORT->file_##NAME, "/" #NAME);
+ FOREACH_COUNTER(GEN_DO_COUNTER_NAME, p)
+
+ // Check HW Counters vendor dependent
+ DIR *hwcounters_dir = opendir(hwcounters_dirname);
+ if (hwcounters_dir) {
+ // By default set standard
+ p->do_hwpackets = config_get_boolean_ondemand(buffer, "hwpackets", do_hwpackets);
+ p->do_hwerrors = config_get_boolean_ondemand(buffer, "hwerrors", do_hwerrors);
+
+// VENDORS: Set your own
+
+// Allocate the chars to the filenames
+#define GEN_DO_HWCOUNTER_NAME(NAME, GRP, DESC, DIR, PORT, HW, ...) \
+ HW->file_##NAME = callocz(1, strlen(PORT->hwcounters_path) + sizeof(#NAME) + 3); \
+ strcat(HW->file_##NAME, PORT->hwcounters_path); \
+ strcat(HW->file_##NAME, "/" #NAME);
+
+ // VENDOR-MLX: Mellanox
+ if (strncmp(dev_dent->d_name, "mlx", 3) == 0) {
+ // Allocate the vendor specific struct
+ p->hwcounters_mlx = callocz(1, sizeof(struct ibporthw_mlx));
+
+ FOREACH_HWCOUNTER_MLX(GEN_DO_HWCOUNTER_NAME, p, p->hwcounters_mlx)
+
+ // Set the function pointer for hwcounter parsing
+ p->hwcounters_parse = &infiniband_hwcounters_parse_mlx;
+ p->hwcounters_dorrd = &infiniband_hwcounters_dorrd_mlx;
+ }
+
+ // VENDOR: Unknown
+ else {
+ p->do_hwpackets = CONFIG_BOOLEAN_NO;
+ p->do_hwerrors = CONFIG_BOOLEAN_NO;
+ }
+ closedir(hwcounters_dir);
+ }
+ }
+
+ // Check port state to keep activation
+ if (enable_only_active) {
+ snprintfz(buffer, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "state");
+ unsigned long long active;
+ // File is "1: DOWN" or "4: ACTIVE", but str2ull will stop on first non-decimal char
+ read_single_number_file(buffer, &active);
+
+ // Want "IB_PORT_ACTIVE" == "4", as defined by drivers/infiniband/core/sysfs.c::state_show()
+ if (active == 4)
+ p->enabled = 1;
+ else
+ p->enabled = 0;
+ }
+
+ if (p->enabled)
+ p->enabled = !simple_pattern_matches(disabled_list, p->name);
+
+ // Check / Update the link speed & width frm "rate" file
+ // Sample output: "100 Gb/sec (4X EDR)"
+ snprintfz(buffer, FILENAME_MAX, "%s/%s/%s", ports_dirname, port_dent->d_name, "rate");
+ char buffer_rate[65];
+ if (read_file(buffer, buffer_rate, 64)) {
+ error("Unable to read '%s'", buffer);
+ p->width = 1;
+ } else {
+ char *buffer_width = strstr(buffer_rate, "(");
+ buffer_width++;
+ // str2ull will stop on first non-decimal value
+ p->speed = str2ull(buffer_rate);
+ p->width = str2ull(buffer_width);
+ }
+
+ if (!p->discovered)
+ info(
+ "Infiniband card %s port %s at speed %lu width %lu", dev_dent->d_name, port_dent->d_name,
+ p->speed, p->width);
+
+ p->discovered = 1;
+ }
+ closedir(ports_dir);
+ }
+ closedir(devices_dir);
+
+ initialized = 1;
+ last_refresh_ports_usec = 0;
+ }
+ last_refresh_ports_usec += dt;
+
+ // Update all ports values
+ struct ibport *port;
+ for (port = ibport_root; port; port = port->next) {
+ if (!port->enabled)
+ continue;
+ //
+ // Read values from system to struct
+ //
+
+// counter from file and place it in ibport struct
+#define GEN_DO_COUNTER_READ(NAME, GRP, DESC, DIR, PORT, ...) \
+ if (PORT->file_##NAME) { \
+ if (read_single_number_file(PORT->file_##NAME, (unsigned long long *)&PORT->NAME)) { \
+ error("cannot read iface '%s' counter '" #NAME "'", PORT->name); \
+ PORT->file_##NAME = NULL; \
+ } \
+ }
+
+ // Update charts
+ if (port->do_bytes != CONFIG_BOOLEAN_NO) {
+ // Read values from sysfs
+ FOREACH_COUNTER_BYTES(GEN_DO_COUNTER_READ, port)
+
+ // First creation of RRD Set (charts)
+ if (unlikely(!port->st_bytes)) {
+ port->st_bytes = rrdset_create_localhost(
+ "Infiniband",
+ port->chart_id_bytes,
+ NULL,
+ port->chart_family,
+ "ib.bytes",
+ "Bandwidth usage",
+ "kilobits/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_INFINIBAND_NAME,
+ port->priority + 1,
+ update_every,
+ RRDSET_TYPE_AREA);
+ // Create Dimensions
+ rrdset_flag_set(port->st_bytes, RRDSET_FLAG_DETAIL);
+ // On this chart, we want to have a KB/s so the dashboard will autoscale it
+ // The reported values are also per-lane, so we must multiply it by the width
+ FOREACH_COUNTER_BYTES(GEN_RRD_DIM_ADD_CUSTOM, port, 8 * port->width, 1024, RRD_ALGORITHM_INCREMENTAL)
+
+ port->stv_speed = rrdsetvar_custom_chart_variable_create(port->st_bytes, "link_speed");
+ } else
+ rrdset_next(port->st_bytes);
+
+ // Link read values to dimensions
+ FOREACH_COUNTER_BYTES(GEN_RRD_DIM_SETP, port)
+
+ // For link speed set only variable
+ rrdsetvar_custom_chart_variable_set(port->stv_speed, port->speed);
+
+ rrdset_done(port->st_bytes);
+ }
+
+ if (port->do_packets != CONFIG_BOOLEAN_NO) {
+ // Read values from sysfs
+ FOREACH_COUNTER_PACKETS(GEN_DO_COUNTER_READ, port)
+
+ // First creation of RRD Set (charts)
+ if (unlikely(!port->st_packets)) {
+ port->st_packets = rrdset_create_localhost(
+ "Infiniband",
+ port->chart_id_packets,
+ NULL,
+ port->chart_family,
+ "ib.packets",
+ "Packets Statistics",
+ "packets/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_INFINIBAND_NAME,
+ port->priority + 2,
+ update_every,
+ RRDSET_TYPE_AREA);
+ // Create Dimensions
+ rrdset_flag_set(port->st_packets, RRDSET_FLAG_DETAIL);
+ FOREACH_COUNTER_PACKETS(GEN_RRD_DIM_ADD, port)
+ } else
+ rrdset_next(port->st_packets);
+
+ // Link read values to dimensions
+ FOREACH_COUNTER_PACKETS(GEN_RRD_DIM_SETP, port)
+ rrdset_done(port->st_packets);
+ }
+
+ if (port->do_errors != CONFIG_BOOLEAN_NO) {
+ // Read values from sysfs
+ FOREACH_COUNTER_ERRORS(GEN_DO_COUNTER_READ, port)
+
+ // First creation of RRD Set (charts)
+ if (unlikely(!port->st_errors)) {
+ port->st_errors = rrdset_create_localhost(
+ "Infiniband",
+ port->chart_id_errors,
+ NULL,
+ port->chart_family,
+ "ib.errors",
+ "Error Counters",
+ "errors/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_INFINIBAND_NAME,
+ port->priority + 3,
+ update_every,
+ RRDSET_TYPE_LINE);
+ // Create Dimensions
+ rrdset_flag_set(port->st_errors, RRDSET_FLAG_DETAIL);
+ FOREACH_COUNTER_ERRORS(GEN_RRD_DIM_ADD, port)
+ } else
+ rrdset_next(port->st_errors);
+
+ // Link read values to dimensions
+ FOREACH_COUNTER_ERRORS(GEN_RRD_DIM_SETP, port)
+ rrdset_done(port->st_errors);
+ }
+
+ //
+ // HW Counters
+ //
+
+ // Call the function for parsing and setting hwcounters
+ if (port->hwcounters_parse && port->hwcounters_dorrd) {
+ // Read all values (done by vendor-specific function)
+ (*port->hwcounters_parse)(port);
+
+ if (port->do_hwerrors != CONFIG_BOOLEAN_NO) {
+ // First creation of RRD Set (charts)
+ if (unlikely(!port->st_hwerrors)) {
+ port->st_hwerrors = rrdset_create_localhost(
+ "Infiniband",
+ port->chart_id_hwerrors,
+ NULL,
+ port->chart_family,
+ "ib.hwerrors",
+ "Hardware Errors",
+ "errors/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_INFINIBAND_NAME,
+ port->priority + 4,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_flag_set(port->st_hwerrors, RRDSET_FLAG_DETAIL);
+
+ // VENDORS: Set your selection
+
+ // VENDOR: Mellanox
+ if (strncmp(port->name, "mlx", 3) == 0) {
+ FOREACH_HWCOUNTER_MLX_ERRORS(GEN_RRD_DIM_ADD_HW, port, port->hwcounters_mlx)
+ }
+
+ // Unknown vendor, should not happen
+ else {
+ error(
+ "Unmanaged vendor for '%s', do_hwerrors should have been set to no. Please report this bug",
+ port->name);
+ port->do_hwerrors = CONFIG_BOOLEAN_NO;
+ }
+ } else
+ rrdset_next(port->st_hwerrors);
+ }
+
+ if (port->do_hwpackets != CONFIG_BOOLEAN_NO) {
+ // First creation of RRD Set (charts)
+ if (unlikely(!port->st_hwpackets)) {
+ port->st_hwpackets = rrdset_create_localhost(
+ "Infiniband",
+ port->chart_id_hwpackets,
+ NULL,
+ port->chart_family,
+ "ib.hwpackets",
+ "Hardware Packets Statistics",
+ "packets/s",
+ PLUGIN_PROC_NAME,
+ PLUGIN_PROC_MODULE_INFINIBAND_NAME,
+ port->priority + 5,
+ update_every,
+ RRDSET_TYPE_LINE);
+
+ rrdset_flag_set(port->st_hwpackets, RRDSET_FLAG_DETAIL);
+
+ // VENDORS: Set your selection
+
+ // VENDOR: Mellanox
+ if (strncmp(port->name, "mlx", 3) == 0) {
+ FOREACH_HWCOUNTER_MLX_PACKETS(GEN_RRD_DIM_ADD_HW, port, port->hwcounters_mlx)
+ }
+
+ // Unknown vendor, should not happen
+ else {
+ error(
+ "Unmanaged vendor for '%s', do_hwpackets should have been set to no. Please report this bug",
+ port->name);
+ port->do_hwpackets = CONFIG_BOOLEAN_NO;
+ }
+ } else
+ rrdset_next(port->st_hwpackets);
+ }
+
+ // Update values to rrd (done by vendor-specific function)
+ (*port->hwcounters_dorrd)(port);
+ }
+ }
+
+ return 0;
+}