diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox')
77 files changed, 2396 insertions, 616 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx4/cmd.c b/drivers/net/ethernet/mellanox/mlx4/cmd.c index f5b1f8c78..7f2081345 100644 --- a/drivers/net/ethernet/mellanox/mlx4/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx4/cmd.c @@ -2199,8 +2199,9 @@ reset_slave: if (cmd != MLX4_COMM_CMD_RESET) { mlx4_warn(dev, "Turn on internal error to force reset, slave=%d, cmd=0x%x\n", slave, cmd); - /* Turn on internal error letting slave reset itself immeditaly, - * otherwise it might take till timeout on command is passed + /* Turn on internal error letting slave reset itself + * immediately, otherwise it might take till timeout on + * command is passed */ reply |= ((u32)COMM_CHAN_EVENT_INTERNAL_ERR); } @@ -2954,7 +2955,7 @@ static bool mlx4_valid_vf_state_change(struct mlx4_dev *dev, int port, dummy_admin.default_vlan = vlan; /* VF wants to move to other VST state which is valid with current - * rate limit. Either differnt default vlan in VST or other + * rate limit. Either different default vlan in VST or other * supported QoS priority. Otherwise we don't allow this change when * the TX rate is still configured. */ diff --git a/drivers/net/ethernet/mellanox/mlx4/cq.c b/drivers/net/ethernet/mellanox/mlx4/cq.c index 4d4f9cf9f..e130e7259 100644 --- a/drivers/net/ethernet/mellanox/mlx4/cq.c +++ b/drivers/net/ethernet/mellanox/mlx4/cq.c @@ -115,7 +115,7 @@ void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn) return; } - /* Acessing the CQ outside of rcu_read_lock is safe, because + /* Accessing the CQ outside of rcu_read_lock is safe, because * the CQ is freed only after interrupt handling is completed. */ ++cq->arm_sn; @@ -137,7 +137,7 @@ void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type) return; } - /* Acessing the CQ outside of rcu_read_lock is safe, because + /* Accessing the CQ outside of rcu_read_lock is safe, because * the CQ is freed only after interrupt handling is completed. */ cq->event(cq, event_type); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_clock.c b/drivers/net/ethernet/mellanox/mlx4/en_clock.c index 9e3b76182..cd754cd76 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_clock.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_clock.c @@ -96,8 +96,8 @@ void mlx4_en_remove_timestamp(struct mlx4_en_dev *mdev) #define MLX4_EN_WRAP_AROUND_SEC 10UL /* By scheduling the overflow check every 5 seconds, we have a reasonably - * good chance we wont miss a wrap around. - * TOTO: Use a timer instead of a work queue to increase the guarantee. + * good chance we won't miss a wrap around. + * TODO: Use a timer instead of a work queue to increase the guarantee. */ #define MLX4_EN_OVERFLOW_PERIOD (MLX4_EN_WRAP_AROUND_SEC * HZ / 2) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index 33bbcced8..5d3fde63b 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -42,6 +42,7 @@ #include <net/ip.h> #include <net/vxlan.h> #include <net/devlink.h> +#include <net/rps.h> #include <linux/mlx4/driver.h> #include <linux/mlx4/device.h> @@ -1072,7 +1073,8 @@ static void mlx4_en_do_multicast(struct mlx4_en_priv *priv, 1, MLX4_MCAST_CONFIG); /* Update multicast list - we cache all addresses so they won't - * change while HW is updated holding the command semaphor */ + * change while HW is updated holding the command semaphore + */ netif_addr_lock_bh(dev); mlx4_en_cache_mclist(dev); netif_addr_unlock_bh(dev); @@ -1817,7 +1819,7 @@ int mlx4_en_start_port(struct net_device *dev) mlx4_en_set_rss_steer_rules(priv)) mlx4_warn(mdev, "Failed setting steering rules\n"); - /* Attach rx QP to bradcast address */ + /* Attach rx QP to broadcast address */ eth_broadcast_addr(&mc_list[10]); mc_list[5] = priv->port; /* needed for B0 steering support */ if (mlx4_multicast_attach(mdev->dev, priv->rss_map.indir_qp, mc_list, diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index a09b6e053..eac49657b 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -762,7 +762,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud /* Drop packet on bad receive or bad checksum */ if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_ERROR)) { - en_err(priv, "CQE completed in error - vendor syndrom:%d syndrom:%d\n", + en_err(priv, "CQE completed in error - vendor syndrome:%d syndrome:%d\n", ((struct mlx4_err_cqe *)cqe)->vendor_err_syndrome, ((struct mlx4_err_cqe *)cqe)->syndrome); goto next; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index 65cb63f6c..1ddb11cb2 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -992,7 +992,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) tx_info->ts_requested = 1; } - /* Prepare ctrl segement apart opcode+ownership, which depends on + /* Prepare ctrl segment apart opcode+ownership, which depends on * whether LSO is used */ tx_desc->ctrl.srcrb_flags = priv->ctrl_flags; if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { diff --git a/drivers/net/ethernet/mellanox/mlx4/eq.c b/drivers/net/ethernet/mellanox/mlx4/eq.c index 6598b10a9..9572a45f6 100644 --- a/drivers/net/ethernet/mellanox/mlx4/eq.c +++ b/drivers/net/ethernet/mellanox/mlx4/eq.c @@ -210,7 +210,7 @@ static void slave_event(struct mlx4_dev *dev, u8 slave, struct mlx4_eqe *eqe) memcpy(s_eqe, eqe, sizeof(struct mlx4_eqe) - 1); s_eqe->slave_id = slave; - /* ensure all information is written before setting the ownersip bit */ + /* ensure all information is written before setting the ownership bit */ dma_wmb(); s_eqe->owner = !!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE) ? 0x0 : 0x80; ++slave_eq->prod; diff --git a/drivers/net/ethernet/mellanox/mlx4/fw_qos.h b/drivers/net/ethernet/mellanox/mlx4/fw_qos.h index 954b86faa..40ca29bb9 100644 --- a/drivers/net/ethernet/mellanox/mlx4/fw_qos.h +++ b/drivers/net/ethernet/mellanox/mlx4/fw_qos.h @@ -44,7 +44,7 @@ /* Default supported priorities for VPP allocation */ #define MLX4_DEFAULT_QOS_PRIO (0) -/* Derived from FW feature definition, 0 is the default vport fo all QPs */ +/* Derived from FW feature definition, 0 is the default vport for all QPs */ #define MLX4_VPP_DEFAULT_VPORT (0) struct mlx4_vport_qos_param { @@ -98,7 +98,7 @@ int mlx4_SET_PORT_SCHEDULER(struct mlx4_dev *dev, u8 port, u8 *tc_tx_bw, int mlx4_ALLOCATE_VPP_get(struct mlx4_dev *dev, u8 port, u16 *available_vpp, u8 *vpp_p_up); /** - * mlx4_ALLOCATE_VPP_set - Distribution of VPPs among differnt priorities. + * mlx4_ALLOCATE_VPP_set - Distribution of VPPs among different priorities. * The total number of VPPs assigned to all for a port must not exceed * the value reported by available_vpp in mlx4_ALLOCATE_VPP_get. * VPP allocation is allowed only after the port type has been set, @@ -113,7 +113,7 @@ int mlx4_ALLOCATE_VPP_get(struct mlx4_dev *dev, u8 port, int mlx4_ALLOCATE_VPP_set(struct mlx4_dev *dev, u8 port, u8 *vpp_p_up); /** - * mlx4_SET_VPORT_QOS_get - Query QoS proporties of a Vport. + * mlx4_SET_VPORT_QOS_get - Query QoS properties of a Vport. * Each priority allowed for the Vport is assigned with a share of the BW, * and a BW limitation. This commands query the current QoS values. * @@ -128,7 +128,7 @@ int mlx4_SET_VPORT_QOS_get(struct mlx4_dev *dev, u8 port, u8 vport, struct mlx4_vport_qos_param *out_param); /** - * mlx4_SET_VPORT_QOS_set - Set QoS proporties of a Vport. + * mlx4_SET_VPORT_QOS_set - Set QoS properties of a Vport. * QoS parameters can be modified at any time, but must be initialized * before any QP is associated with the VPort. * diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index 258122683..7b02ff611 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -129,7 +129,7 @@ static const struct mlx4_profile default_profile = { .num_cq = 1 << 16, .num_mcg = 1 << 13, .num_mpt = 1 << 19, - .num_mtt = 1 << 20, /* It is really num mtt segements */ + .num_mtt = 1 << 20, /* It is really num mtt segments */ }; static const struct mlx4_profile low_mem_profile = { @@ -1508,7 +1508,7 @@ static int mlx4_port_map_set(struct mlx4_dev *dev, struct mlx4_port_map *v2p) priv->v2p.port1 = port1; priv->v2p.port2 = port2; } else { - mlx4_err(dev, "Failed to change port mape: %d\n", err); + mlx4_err(dev, "Failed to change port map: %d\n", err); } } diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_stats.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_stats.h index e9cd4bb6f..d3d9ec042 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_stats.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_stats.h @@ -112,7 +112,7 @@ struct mlx4_en_stat_out_flow_control_mbox { __be64 tx_pause_duration; /* Number of transmitter transitions from XOFF state to XON state */ __be64 tx_pause_transition; - /* Reserverd */ + /* Reserved */ __be64 reserved[2]; }; diff --git a/drivers/net/ethernet/mellanox/mlx4/port.c b/drivers/net/ethernet/mellanox/mlx4/port.c index 256a06b3c..4e43f4a7d 100644 --- a/drivers/net/ethernet/mellanox/mlx4/port.c +++ b/drivers/net/ethernet/mellanox/mlx4/port.c @@ -2118,7 +2118,7 @@ static void mlx4_qsfp_eeprom_params_set(u8 *i2c_addr, u8 *page_num, u16 *offset) * @data: output buffer to put the requested data into. * * Reads cable module eeprom data, puts the outcome data into - * data pointer paramer. + * data pointer parameter. * Returns num of read bytes on success or a negative error * code. */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index c44870b17..76dc5a9b9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -29,7 +29,7 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en/rqt.o en/tir.o en/rss.o en/rx_res.o \ en/reporter_tx.o en/reporter_rx.o en/params.o en/xsk/pool.o \ en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o \ en/qos.o en/htb.o en/trap.o en/fs_tt_redirect.o en/selq.o \ - lib/crypto.o + lib/crypto.o lib/sd.o # # Netdev extra diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c index cf0477f53..47e7c2639 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c @@ -210,7 +210,7 @@ static bool is_dpll_supported(struct mlx5_core_dev *dev) return false; if (!MLX5_CAP_MCAM_REG2(dev, synce_registers)) { - mlx5_core_warn(dev, "Missing SyncE capability\n"); + mlx5_core_dbg(dev, "Missing SyncE capability\n"); return false; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c index d74a5aaf4..904e08de8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c @@ -41,6 +41,7 @@ struct mlx5_dpll_synce_status { enum mlx5_msees_oper_status oper_status; bool ho_acq; bool oper_freq_measure; + enum mlx5_msees_failure_reason failure_reason; s32 frequency_diff; }; @@ -60,6 +61,7 @@ mlx5_dpll_synce_status_get(struct mlx5_core_dev *mdev, synce_status->oper_status = MLX5_GET(msees_reg, out, oper_status); synce_status->ho_acq = MLX5_GET(msees_reg, out, ho_acq); synce_status->oper_freq_measure = MLX5_GET(msees_reg, out, oper_freq_measure); + synce_status->failure_reason = MLX5_GET(msees_reg, out, failure_reason); synce_status->frequency_diff = MLX5_GET(msees_reg, out, frequency_diff); return 0; } @@ -99,6 +101,26 @@ mlx5_dpll_lock_status_get(struct mlx5_dpll_synce_status *synce_status) } } +static enum dpll_lock_status_error +mlx5_dpll_lock_status_error_get(struct mlx5_dpll_synce_status *synce_status) +{ + switch (synce_status->oper_status) { + case MLX5_MSEES_OPER_STATUS_FAIL_HOLDOVER: + fallthrough; + case MLX5_MSEES_OPER_STATUS_FAIL_FREE_RUNNING: + switch (synce_status->failure_reason) { + case MLX5_MSEES_FAILURE_REASON_PORT_DOWN: + return DPLL_LOCK_STATUS_ERROR_MEDIA_DOWN; + case MLX5_MSEES_FAILURE_REASON_TOO_HIGH_FREQUENCY_DIFF: + return DPLL_LOCK_STATUS_ERROR_FRACTIONAL_FREQUENCY_OFFSET_TOO_HIGH; + default: + return DPLL_LOCK_STATUS_ERROR_UNDEFINED; + } + default: + return DPLL_LOCK_STATUS_ERROR_NONE; + } +} + static enum dpll_pin_state mlx5_dpll_pin_state_get(struct mlx5_dpll_synce_status *synce_status) { @@ -118,10 +140,11 @@ mlx5_dpll_pin_ffo_get(struct mlx5_dpll_synce_status *synce_status, return 0; } -static int mlx5_dpll_device_lock_status_get(const struct dpll_device *dpll, - void *priv, - enum dpll_lock_status *status, - struct netlink_ext_ack *extack) +static int +mlx5_dpll_device_lock_status_get(const struct dpll_device *dpll, void *priv, + enum dpll_lock_status *status, + enum dpll_lock_status_error *status_error, + struct netlink_ext_ack *extack) { struct mlx5_dpll_synce_status synce_status; struct mlx5_dpll *mdpll = priv; @@ -131,6 +154,7 @@ static int mlx5_dpll_device_lock_status_get(const struct dpll_device *dpll, if (err) return err; *status = mlx5_dpll_lock_status_get(&synce_status); + *status_error = mlx5_dpll_lock_status_error_get(&synce_status); return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 55c6ace0a..84db05fb9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -60,6 +60,7 @@ #include "lib/clock.h" #include "en/rx_res.h" #include "en/selq.h" +#include "lib/sd.h" extern const struct net_device_ops mlx5e_netdev_ops; struct page_pool; @@ -791,6 +792,8 @@ struct mlx5e_channel { struct hwtstamp_config *tstamp; DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES); int ix; + int vec_ix; + int sd_ix; int cpu; /* Sync between icosq recovery and XSK enable/disable. */ struct mutex icosq_recovery_lock; @@ -914,7 +917,7 @@ struct mlx5e_priv { bool tx_ptp_opened; bool rx_ptp_opened; struct hwtstamp_config tstamp; - u16 q_counter; + u16 q_counter[MLX5_SD_MAX_GROUP_SZ]; u16 drop_rq_q_counter; struct notifier_block events_nb; struct notifier_block blocking_events_nb; @@ -1029,12 +1032,12 @@ struct mlx5e_xsk_param; struct mlx5e_rq_param; int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, - struct mlx5e_xsk_param *xsk, int node, + struct mlx5e_xsk_param *xsk, int node, u16 q_counter, struct mlx5e_rq *rq); #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */ int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time); void mlx5e_close_rq(struct mlx5e_rq *rq); -int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param); +int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter); void mlx5e_destroy_rq(struct mlx5e_rq *rq); struct mlx5e_sq_param; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c index 48581ea3a..874a10166 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c @@ -23,20 +23,26 @@ bool mlx5e_channels_is_xsk(struct mlx5e_channels *chs, unsigned int ix) return test_bit(MLX5E_CHANNEL_STATE_XSK, c->state); } -void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn) +void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn, + u32 *vhca_id) { struct mlx5e_channel *c = mlx5e_channels_get(chs, ix); *rqn = c->rq.rqn; + if (vhca_id) + *vhca_id = MLX5_CAP_GEN(c->mdev, vhca_id); } -void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn) +void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn, + u32 *vhca_id) { struct mlx5e_channel *c = mlx5e_channels_get(chs, ix); WARN_ON_ONCE(!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)); *rqn = c->xskrq.rqn; + if (vhca_id) + *vhca_id = MLX5_CAP_GEN(c->mdev, vhca_id); } bool mlx5e_channels_get_ptp_rqn(struct mlx5e_channels *chs, u32 *rqn) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h index 637ca90da..6715aa938 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h @@ -10,8 +10,10 @@ struct mlx5e_channels; unsigned int mlx5e_channels_get_num(struct mlx5e_channels *chs); bool mlx5e_channels_is_xsk(struct mlx5e_channels *chs, unsigned int ix); -void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn); -void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn); +void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn, + u32 *vhca_id); +void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn, + u32 *vhca_id); bool mlx5e_channels_get_ptp_rqn(struct mlx5e_channels *chs, u32 *rqn); #endif /* __MLX5_EN_CHANNELS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c index 40c8df111..e2d8d2754 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c @@ -20,10 +20,8 @@ #define NUM_REQ_PPCNT_COUNTER_S1 MLX5_CMD_SET_MONITOR_NUM_PPCNT_COUNTER_SET1 #define NUM_REQ_Q_COUNTERS_S1 MLX5_CMD_SET_MONITOR_NUM_Q_COUNTERS_SET1 -int mlx5e_monitor_counter_supported(struct mlx5e_priv *priv) +static int mlx5e_monitor_counter_cap(struct mlx5_core_dev *mdev) { - struct mlx5_core_dev *mdev = priv->mdev; - if (!MLX5_CAP_GEN(mdev, max_num_of_monitor_counters)) return false; if (MLX5_CAP_PCAM_REG(mdev, ppcnt) && @@ -36,24 +34,38 @@ int mlx5e_monitor_counter_supported(struct mlx5e_priv *priv) return true; } -static void mlx5e_monitor_counter_arm(struct mlx5e_priv *priv) +int mlx5e_monitor_counter_supported(struct mlx5e_priv *priv) +{ + struct mlx5_core_dev *pos; + int i; + + mlx5_sd_for_each_dev(i, priv->mdev, pos) + if (!mlx5e_monitor_counter_cap(pos)) + return false; + return true; +} + +static void mlx5e_monitor_counter_arm(struct mlx5_core_dev *mdev) { u32 in[MLX5_ST_SZ_DW(arm_monitor_counter_in)] = {}; MLX5_SET(arm_monitor_counter_in, in, opcode, MLX5_CMD_OP_ARM_MONITOR_COUNTER); - mlx5_cmd_exec_in(priv->mdev, arm_monitor_counter, in); + mlx5_cmd_exec_in(mdev, arm_monitor_counter, in); } static void mlx5e_monitor_counters_work(struct work_struct *work) { struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, monitor_counters_work); + struct mlx5_core_dev *pos; + int i; mutex_lock(&priv->state_lock); mlx5e_stats_update_ndo_stats(priv); mutex_unlock(&priv->state_lock); - mlx5e_monitor_counter_arm(priv); + mlx5_sd_for_each_dev(i, priv->mdev, pos) + mlx5e_monitor_counter_arm(pos); } static int mlx5e_monitor_event_handler(struct notifier_block *nb, @@ -97,15 +109,13 @@ static int fill_monitor_counter_q_counter_set1(int cnt, int q_counter, u32 *in) } /* check if mlx5e_monitor_counter_supported before calling this function*/ -static void mlx5e_set_monitor_counter(struct mlx5e_priv *priv) +static void mlx5e_set_monitor_counter(struct mlx5_core_dev *mdev, int q_counter) { - struct mlx5_core_dev *mdev = priv->mdev; int max_num_of_counters = MLX5_CAP_GEN(mdev, max_num_of_monitor_counters); int num_q_counters = MLX5_CAP_GEN(mdev, num_q_monitor_counters); int num_ppcnt_counters = !MLX5_CAP_PCAM_REG(mdev, ppcnt) ? 0 : MLX5_CAP_GEN(mdev, num_ppcnt_monitor_counters); u32 in[MLX5_ST_SZ_DW(set_monitor_counter_in)] = {}; - int q_counter = priv->q_counter; int cnt = 0; if (num_ppcnt_counters >= NUM_REQ_PPCNT_COUNTER_S1 && @@ -127,13 +137,17 @@ static void mlx5e_set_monitor_counter(struct mlx5e_priv *priv) /* check if mlx5e_monitor_counter_supported before calling this function*/ void mlx5e_monitor_counter_init(struct mlx5e_priv *priv) { + struct mlx5_core_dev *pos; + int i; + INIT_WORK(&priv->monitor_counters_work, mlx5e_monitor_counters_work); MLX5_NB_INIT(&priv->monitor_counters_nb, mlx5e_monitor_event_handler, MONITOR_COUNTER); - mlx5_eq_notifier_register(priv->mdev, &priv->monitor_counters_nb); - - mlx5e_set_monitor_counter(priv); - mlx5e_monitor_counter_arm(priv); + mlx5_sd_for_each_dev(i, priv->mdev, pos) { + mlx5_eq_notifier_register(pos, &priv->monitor_counters_nb); + mlx5e_set_monitor_counter(pos, priv->q_counter[i]); + mlx5e_monitor_counter_arm(pos); + } queue_work(priv->wq, &priv->update_stats_work); } @@ -141,11 +155,15 @@ void mlx5e_monitor_counter_init(struct mlx5e_priv *priv) void mlx5e_monitor_counter_cleanup(struct mlx5e_priv *priv) { u32 in[MLX5_ST_SZ_DW(set_monitor_counter_in)] = {}; + struct mlx5_core_dev *pos; + int i; MLX5_SET(set_monitor_counter_in, in, opcode, MLX5_CMD_OP_SET_MONITOR_COUNTER); - mlx5_cmd_exec_in(priv->mdev, set_monitor_counter, in); - mlx5_eq_notifier_unregister(priv->mdev, &priv->monitor_counters_nb); + mlx5_sd_for_each_dev(i, priv->mdev, pos) { + mlx5_cmd_exec_in(pos, set_monitor_counter, in); + mlx5_eq_notifier_unregister(pos, &priv->monitor_counters_nb); + } cancel_work_sync(&priv->monitor_counters_work); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 5d213a988..a3f31d9d5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -240,11 +240,14 @@ static u32 mlx5e_rx_get_linear_sz_xsk(struct mlx5e_params *params, return xsk->headroom + hw_mtu; } -static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool xsk) +static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool no_head_tail_room) { - /* SKBs built on XDP_PASS on XSK RQs don't have headroom. */ - u16 headroom = xsk ? 0 : mlx5e_get_linear_rq_headroom(params, NULL); u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); + u16 headroom; + + if (no_head_tail_room) + return SKB_DATA_ALIGN(hw_mtu); + headroom = mlx5e_get_linear_rq_headroom(params, NULL); return MLX5_SKB_FRAG_SZ(headroom + hw_mtu); } @@ -254,6 +257,7 @@ static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk, bool mpwqe) { + bool no_head_tail_room; u32 sz; /* XSK frames are mapped as individual pages, because frames may come in @@ -262,7 +266,13 @@ static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev, if (xsk) return mpwqe ? 1 << mlx5e_mpwrq_page_shift(mdev, xsk) : PAGE_SIZE; - sz = roundup_pow_of_two(mlx5e_rx_get_linear_sz_skb(params, false)); + no_head_tail_room = params->xdp_prog && mpwqe && !mlx5e_rx_is_linear_skb(mdev, params, xsk); + + /* When no_head_tail_room is set, headroom and tailroom are excluded from skb calculations. + * no_head_tail_room should be set in the case of XDP with Striding RQ + * when SKB is not linear. This is because another page is allocated for the linear part. + */ + sz = roundup_pow_of_two(mlx5e_rx_get_linear_sz_skb(params, no_head_tail_room)); /* XDP in mlx5e doesn't support multiple packets per page. * Do not assume sz <= PAGE_SIZE if params->xdp_prog is set. @@ -289,7 +299,11 @@ bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) return false; - /* Both XSK and non-XSK cases allocate an SKB on XDP_PASS. Packet data + /* Call mlx5e_rx_get_linear_sz_skb with the no_head_tail_room parameter set + * to exclude headroom and tailroom from calculations. + * no_head_tail_room is true when SKB is built on XDP_PASS on XSK RQs + * since packet data buffers don't have headroom and tailroom resreved for the SKB. + * Both XSK and non-XSK cases allocate an SKB on XDP_PASS. Packet data * must fit into a CPU page. */ if (mlx5e_rx_get_linear_sz_skb(params, xsk) > PAGE_SIZE) @@ -674,7 +688,7 @@ void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e .napi = &c->napi, .ch_stats = c->stats, .node = cpu_to_node(c->cpu), - .ix = c->ix, + .ix = c->vec_ix, }; } @@ -945,7 +959,6 @@ static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *param int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, - u16 q_counter, struct mlx5e_rq_param *param) { void *rqc = param->rqc; @@ -1007,7 +1020,6 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, MLX5_SET(wq, wq, log_wq_stride, mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs)); MLX5_SET(wq, wq, pd, mdev->mlx5e_res.hw_objs.pdn); - MLX5_SET(rqc, rqc, counter_set_id, q_counter); MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable); MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); @@ -1018,7 +1030,6 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, } void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev, - u16 q_counter, struct mlx5e_rq_param *param) { void *rqc = param->rqc; @@ -1027,7 +1038,6 @@ void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev, MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); MLX5_SET(wq, wq, log_wq_stride, mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1)); - MLX5_SET(rqc, rqc, counter_set_id, q_counter); param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); } @@ -1292,13 +1302,12 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - u16 q_counter, struct mlx5e_channel_param *cparam) { u8 icosq_log_wq_sz, async_icosq_log_wq_sz; int err; - err = mlx5e_build_rq_param(mdev, params, NULL, q_counter, &cparam->rq); + err = mlx5e_build_rq_param(mdev, params, NULL, &cparam->rq); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index 6800949da..9a781f18b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -130,10 +130,8 @@ void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, - u16 q_counter, struct mlx5e_rq_param *param); void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev, - u16 q_counter, struct mlx5e_rq_param *param); void mlx5e_build_sq_param_common(struct mlx5_core_dev *mdev, struct mlx5e_sq_param *param); @@ -149,7 +147,6 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, struct mlx5e_sq_param *param); int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - u16 q_counter, struct mlx5e_channel_param *cparam); u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index ca05b3252..d0af7271d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -646,7 +646,6 @@ static void mlx5e_ptp_build_sq_param(struct mlx5_core_dev *mdev, static void mlx5e_ptp_build_rq_param(struct mlx5_core_dev *mdev, struct net_device *netdev, - u16 q_counter, struct mlx5e_ptp_params *ptp_params) { struct mlx5e_rq_param *rq_params = &ptp_params->rq_param; @@ -655,7 +654,7 @@ static void mlx5e_ptp_build_rq_param(struct mlx5_core_dev *mdev, params->rq_wq_type = MLX5_WQ_TYPE_CYCLIC; mlx5e_init_rq_type_params(mdev, params); params->sw_mtu = netdev->max_mtu; - mlx5e_build_rq_param(mdev, params, NULL, q_counter, rq_params); + mlx5e_build_rq_param(mdev, params, NULL, rq_params); } static void mlx5e_ptp_build_params(struct mlx5e_ptp *c, @@ -681,7 +680,7 @@ static void mlx5e_ptp_build_params(struct mlx5e_ptp *c, /* RQ */ if (test_bit(MLX5E_PTP_STATE_RX, c->state)) { params->vlan_strip_disable = orig->vlan_strip_disable; - mlx5e_ptp_build_rq_param(c->mdev, c->netdev, c->priv->q_counter, cparams); + mlx5e_ptp_build_rq_param(c->mdev, c->netdev, cparams); } } @@ -714,13 +713,16 @@ static int mlx5e_ptp_open_rq(struct mlx5e_ptp *c, struct mlx5e_params *params, struct mlx5e_rq_param *rq_param) { int node = dev_to_node(c->mdev->device); - int err; + int err, sd_ix; + u16 q_counter; err = mlx5e_init_ptp_rq(c, params, &c->rq); if (err) return err; - return mlx5e_open_rq(params, rq_param, NULL, node, &c->rq); + sd_ix = mlx5_sd_ch_ix_get_dev_ix(c->mdev, MLX5E_PTP_CHANNEL_IX); + q_counter = c->priv->q_counter[sd_ix]; + return mlx5e_open_rq(params, rq_param, NULL, node, q_counter, &c->rq); } static int mlx5e_ptp_open_queues(struct mlx5e_ptp *c, @@ -935,6 +937,7 @@ void mlx5e_ptp_activate_channel(struct mlx5e_ptp *c) if (test_bit(MLX5E_PTP_STATE_RX, c->state)) { mlx5e_ptp_rx_set_fs(c->priv); mlx5e_activate_rq(&c->rq); + netif_queue_set_napi(c->netdev, c->rq.ix, NETDEV_QUEUE_TYPE_RX, &c->napi); } mlx5e_trigger_napi_sched(&c->napi); } @@ -943,8 +946,10 @@ void mlx5e_ptp_deactivate_channel(struct mlx5e_ptp *c) { int tc; - if (test_bit(MLX5E_PTP_STATE_RX, c->state)) + if (test_bit(MLX5E_PTP_STATE_RX, c->state)) { + netif_queue_set_napi(c->netdev, c->rq.ix, NETDEV_QUEUE_TYPE_RX, NULL); mlx5e_deactivate_rq(&c->rq); + } if (test_bit(MLX5E_PTP_STATE_TX, c->state)) { for (tc = 0; tc < c->num_tc; tc++) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c index 922bc5b7c..6743806b8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c @@ -123,8 +123,8 @@ int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, memset(¶m_sq, 0, sizeof(param_sq)); memset(¶m_cq, 0, sizeof(param_cq)); - mlx5e_build_sq_param(priv->mdev, params, ¶m_sq); - mlx5e_build_tx_cq_param(priv->mdev, params, ¶m_cq); + mlx5e_build_sq_param(c->mdev, params, ¶m_sq); + mlx5e_build_tx_cq_param(c->mdev, params, ¶m_cq); err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, ¶m_cq, &ccp, &sq->cq); if (err) goto err_free_sq; @@ -177,7 +177,7 @@ int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id) */ smp_wmb(); - qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node_qid); + qos_dbg(sq->mdev, "Activate QoS SQ qid %u\n", node_qid); mlx5e_activate_txqsq(sq); return 0; @@ -191,7 +191,7 @@ void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) if (!sq) /* Handle the case when the SQ failed to open. */ return; - qos_dbg(priv->mdev, "Deactivate QoS SQ qid %u\n", qid); + qos_dbg(sq->mdev, "Deactivate QoS SQ qid %u\n", qid); mlx5e_deactivate_txqsq(sq); priv->txq2sq[mlx5e_qid_from_qos(&priv->channels, qid)] = NULL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c index 4358798d6..25d751eba 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c @@ -294,8 +294,8 @@ static void mlx5e_rx_reporter_diagnose_generic_rq(struct mlx5e_rq *rq, params = &priv->channels.params; rq_sz = mlx5e_rqwq_get_size(rq); - real_time = mlx5_is_real_time_rq(priv->mdev); - rq_stride = BIT(mlx5e_mpwqe_get_log_stride_size(priv->mdev, params, NULL)); + real_time = mlx5_is_real_time_rq(rq->mdev); + rq_stride = BIT(mlx5e_mpwqe_get_log_stride_size(rq->mdev, params, NULL)); mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RQ"); devlink_fmsg_u8_pair_put(fmsg, "type", params->rq_wq_type); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 6b44ddce1..22918b2ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -108,7 +108,10 @@ static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) mlx5e_reset_txqsq_cc_pc(sq); sq->stats->recover++; clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); + rtnl_lock(); mlx5e_activate_txqsq(sq); + rtnl_unlock(); + if (sq->channel) mlx5e_trigger_napi_icosq(sq->channel); else @@ -179,12 +182,16 @@ static int mlx5e_tx_reporter_ptpsq_unhealthy_recover(void *ctx) carrier_ok = netif_carrier_ok(netdev); netif_carrier_off(netdev); + rtnl_lock(); mlx5e_deactivate_priv_channels(priv); + rtnl_unlock(); mlx5e_ptp_close(chs->ptp); err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp); + rtnl_lock(); mlx5e_activate_priv_channels(priv); + rtnl_unlock(); /* return carrier back if needed */ if (carrier_ok) @@ -219,7 +226,6 @@ mlx5e_tx_reporter_build_diagnose_output_sq_common(struct devlink_fmsg *fmsg, struct mlx5e_txqsq *sq, int tc) { bool stopped = netif_xmit_stopped(sq->txq); - struct mlx5e_priv *priv = sq->priv; u8 state; int err; @@ -227,7 +233,7 @@ mlx5e_tx_reporter_build_diagnose_output_sq_common(struct devlink_fmsg *fmsg, devlink_fmsg_u32_pair_put(fmsg, "txq ix", sq->txq_ix); devlink_fmsg_u32_pair_put(fmsg, "sqn", sq->sqn); - err = mlx5_core_query_sq_state(priv->mdev, sq->sqn, &state); + err = mlx5_core_query_sq_state(sq->mdev, sq->sqn, &state); if (!err) devlink_fmsg_u8_pair_put(fmsg, "HW state", state); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c index 7b8ff7a71..8d9a3b5ec 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c @@ -4,6 +4,33 @@ #include "rqt.h" #include <linux/mlx5/transobj.h> +static bool verify_num_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, + unsigned int size) +{ + unsigned int max_num_vhca_id = MLX5_CAP_GEN_2(mdev, max_rqt_vhca_id); + int i; + + /* Verify that all vhca_ids are in range [0, max_num_vhca_ids - 1] */ + for (i = 0; i < size; i++) + if (vhca_ids[i] >= max_num_vhca_id) + return false; + return true; +} + +static bool rqt_verify_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, + unsigned int size) +{ + if (!vhca_ids) + return true; + + if (!MLX5_CAP_GEN(mdev, cross_vhca_rqt)) + return false; + if (!verify_num_vhca_ids(mdev, vhca_ids, size)) + return false; + + return true; +} + void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir, unsigned int num_channels) { @@ -13,19 +40,38 @@ void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir, indir->table[i] = i % num_channels; } +static void fill_rqn_list(void *rqtc, u32 *rqns, u32 *vhca_ids, unsigned int size) +{ + unsigned int i; + + if (vhca_ids) { + MLX5_SET(rqtc, rqtc, rq_vhca_id_format, 1); + for (i = 0; i < size; i++) { + MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_num, rqns[i]); + MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_vhca_id, vhca_ids[i]); + } + } else { + for (i = 0; i < size; i++) + MLX5_SET(rqtc, rqtc, rq_num[i], rqns[i]); + } +} static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, - u16 max_size, u32 *init_rqns, u16 init_size) + u16 max_size, u32 *init_rqns, u32 *init_vhca_ids, u16 init_size) { + int entry_sz; void *rqtc; int inlen; int err; u32 *in; - int i; + + if (!rqt_verify_vhca_ids(mdev, init_vhca_ids, init_size)) + return -EOPNOTSUPP; rqt->mdev = mdev; rqt->size = max_size; - inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * init_size; + entry_sz = init_vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num); + inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + entry_sz * init_size; in = kvzalloc(inlen, GFP_KERNEL); if (!in) return -ENOMEM; @@ -33,10 +79,9 @@ static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); MLX5_SET(rqtc, rqtc, rqt_max_size, rqt->size); - MLX5_SET(rqtc, rqtc, rqt_actual_size, init_size); - for (i = 0; i < init_size; i++) - MLX5_SET(rqtc, rqtc, rq_num[i], init_rqns[i]); + + fill_rqn_list(rqtc, init_rqns, init_vhca_ids, init_size); err = mlx5_core_create_rqt(rqt->mdev, in, inlen, &rqt->rqtn); @@ -49,7 +94,7 @@ int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, { u16 max_size = indir_enabled ? indir_table_size : 1; - return mlx5e_rqt_init(rqt, mdev, max_size, &init_rqn, 1); + return mlx5e_rqt_init(rqt, mdev, max_size, &init_rqn, NULL, 1); } static int mlx5e_bits_invert(unsigned long a, int size) @@ -63,7 +108,8 @@ static int mlx5e_bits_invert(unsigned long a, int size) return inv; } -static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, unsigned int num_rqns, +static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, u32 *rss_vhca_ids, u32 *vhca_ids, + unsigned int num_rqns, u8 hfunc, struct mlx5e_rss_params_indir *indir) { unsigned int i; @@ -82,30 +128,42 @@ static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, unsigned int num_rqns */ return -EINVAL; rss_rqns[i] = rqns[ix]; + if (vhca_ids) + rss_vhca_ids[i] = vhca_ids[ix]; } return 0; } int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, - u32 *rqns, unsigned int num_rqns, + u32 *rqns, u32 *vhca_ids, unsigned int num_rqns, u8 hfunc, struct mlx5e_rss_params_indir *indir) { - u32 *rss_rqns; + u32 *rss_rqns, *rss_vhca_ids = NULL; int err; rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL); if (!rss_rqns) return -ENOMEM; - err = mlx5e_calc_indir_rqns(rss_rqns, rqns, num_rqns, hfunc, indir); + if (vhca_ids) { + rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids), + GFP_KERNEL); + if (!rss_vhca_ids) { + kvfree(rss_rqns); + return -ENOMEM; + } + } + + err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir); if (err) goto out; - err = mlx5e_rqt_init(rqt, mdev, indir->max_table_size, rss_rqns, + err = mlx5e_rqt_init(rqt, mdev, indir->max_table_size, rss_rqns, rss_vhca_ids, indir->actual_table_size); out: + kvfree(rss_vhca_ids); kvfree(rss_rqns); return err; } @@ -121,20 +179,32 @@ u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels) return min_t(u32, rqt_size, max_cap_rqt_size); } +#define MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH 256 + +unsigned int mlx5e_rqt_max_num_channels_allowed_for_xor8(void) +{ + return MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH / MLX5E_UNIFORM_SPREAD_RQT_FACTOR; +} + void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt) { mlx5_core_destroy_rqt(rqt->mdev, rqt->rqtn); } -static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int size) +static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, + unsigned int size) { - unsigned int i; + int entry_sz; void *rqtc; int inlen; u32 *in; int err; - inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * size; + if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, size)) + return -EINVAL; + + entry_sz = vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num); + inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + entry_sz * size; in = kvzalloc(inlen, GFP_KERNEL); if (!in) return -ENOMEM; @@ -143,8 +213,8 @@ static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int siz MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1); MLX5_SET(rqtc, rqtc, rqt_actual_size, size); - for (i = 0; i < size; i++) - MLX5_SET(rqtc, rqtc, rq_num[i], rqns[i]); + + fill_rqn_list(rqtc, rqns, vhca_ids, size); err = mlx5_core_modify_rqt(rqt->mdev, rqt->rqtn, in, inlen); @@ -152,17 +222,21 @@ static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int siz return err; } -int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn) +int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id) { - return mlx5e_rqt_redirect(rqt, &rqn, 1); + return mlx5e_rqt_redirect(rqt, &rqn, vhca_id, 1); } -int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int num_rqns, +int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, + unsigned int num_rqns, u8 hfunc, struct mlx5e_rss_params_indir *indir) { - u32 *rss_rqns; + u32 *rss_rqns, *rss_vhca_ids = NULL; int err; + if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, num_rqns)) + return -EINVAL; + if (WARN_ON(rqt->size != indir->max_table_size)) return -EINVAL; @@ -170,13 +244,23 @@ int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int num_ if (!rss_rqns) return -ENOMEM; - err = mlx5e_calc_indir_rqns(rss_rqns, rqns, num_rqns, hfunc, indir); + if (vhca_ids) { + rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids), + GFP_KERNEL); + if (!rss_vhca_ids) { + kvfree(rss_rqns); + return -ENOMEM; + } + } + + err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir); if (err) goto out; - err = mlx5e_rqt_redirect(rqt, rss_rqns, indir->actual_table_size); + err = mlx5e_rqt_redirect(rqt, rss_rqns, rss_vhca_ids, indir->actual_table_size); out: + kvfree(rss_vhca_ids); kvfree(rss_rqns); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h index 77fba3ebd..2f9e04a84 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h @@ -20,7 +20,7 @@ void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir, unsigned int num_channels); struct mlx5e_rqt { - struct mlx5_core_dev *mdev; + struct mlx5_core_dev *mdev; /* primary */ u32 rqtn; u16 size; }; @@ -28,7 +28,7 @@ struct mlx5e_rqt { int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, bool indir_enabled, u32 init_rqn, u32 indir_table_size); int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, - u32 *rqns, unsigned int num_rqns, + u32 *rqns, u32 *vhca_ids, unsigned int num_rqns, u8 hfunc, struct mlx5e_rss_params_indir *indir); void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt); @@ -38,8 +38,10 @@ static inline u32 mlx5e_rqt_get_rqtn(struct mlx5e_rqt *rqt) } u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels); -int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn); -int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int num_rqns, +unsigned int mlx5e_rqt_max_num_channels_allowed_for_xor8(void); +int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id); +int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, + unsigned int num_rqns, u8 hfunc, struct mlx5e_rss_params_indir *indir); #endif /* __MLX5_EN_RQT_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c index c1545a2e8..5f742f896 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c @@ -74,7 +74,7 @@ struct mlx5e_rss { struct mlx5e_tir *tir[MLX5E_NUM_INDIR_TIRS]; struct mlx5e_tir *inner_tir[MLX5E_NUM_INDIR_TIRS]; struct mlx5e_rqt rqt; - struct mlx5_core_dev *mdev; + struct mlx5_core_dev *mdev; /* primary */ u32 drop_rqn; bool inner_ft_support; bool enabled; @@ -473,21 +473,22 @@ int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, return 0; } -static int mlx5e_rss_apply(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns) +static int mlx5e_rss_apply(struct mlx5e_rss *rss, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns) { int err; - err = mlx5e_rqt_redirect_indir(&rss->rqt, rqns, num_rqns, rss->hash.hfunc, &rss->indir); + err = mlx5e_rqt_redirect_indir(&rss->rqt, rqns, vhca_ids, num_rqns, rss->hash.hfunc, + &rss->indir); if (err) mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to channels: err = %d\n", mlx5e_rqt_get_rqtn(&rss->rqt), err); return err; } -void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns) +void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns) { rss->enabled = true; - mlx5e_rss_apply(rss, rqns, num_rqns); + mlx5e_rss_apply(rss, rqns, vhca_ids, num_rqns); } void mlx5e_rss_disable(struct mlx5e_rss *rss) @@ -495,7 +496,7 @@ void mlx5e_rss_disable(struct mlx5e_rss *rss) int err; rss->enabled = false; - err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->drop_rqn); + err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->drop_rqn, NULL); if (err) mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to drop RQ %#x: err = %d\n", mlx5e_rqt_get_rqtn(&rss->rqt), rss->drop_rqn, err); @@ -568,7 +569,7 @@ int mlx5e_rss_get_rxfh(struct mlx5e_rss *rss, u32 *indir, u8 *key, u8 *hfunc) int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir, const u8 *key, const u8 *hfunc, - u32 *rqns, unsigned int num_rqns) + u32 *rqns, u32 *vhca_ids, unsigned int num_rqns) { bool changed_indir = false; bool changed_hash = false; @@ -608,7 +609,7 @@ int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir, } if (changed_indir && rss->enabled) { - err = mlx5e_rss_apply(rss, rqns, num_rqns); + err = mlx5e_rss_apply(rss, rqns, vhca_ids, num_rqns); if (err) { mlx5e_rss_copy(rss, old_rss); goto out; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h index d1d0bc350..d0df98963 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h @@ -39,7 +39,7 @@ int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, const struct mlx5e_packet_merge_param *init_pkt_merge_param, bool inner, u32 *tirn); -void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns); +void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns); void mlx5e_rss_disable(struct mlx5e_rss *rss); int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss, @@ -47,7 +47,7 @@ int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss, int mlx5e_rss_get_rxfh(struct mlx5e_rss *rss, u32 *indir, u8 *key, u8 *hfunc); int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir, const u8 *key, const u8 *hfunc, - u32 *rqns, unsigned int num_rqns); + u32 *rqns, u32 *vhca_ids, unsigned int num_rqns); struct mlx5e_rss_params_hash mlx5e_rss_get_hash(struct mlx5e_rss *rss); u8 mlx5e_rss_get_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt); int mlx5e_rss_set_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c index b23e224e3..a86eade9a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c @@ -8,7 +8,7 @@ #define MLX5E_MAX_NUM_RSS 16 struct mlx5e_rx_res { - struct mlx5_core_dev *mdev; + struct mlx5_core_dev *mdev; /* primary */ enum mlx5e_rx_res_features features; unsigned int max_nch; u32 drop_rqn; @@ -19,6 +19,7 @@ struct mlx5e_rx_res { struct mlx5e_rss *rss[MLX5E_MAX_NUM_RSS]; bool rss_active; u32 *rss_rqns; + u32 *rss_vhca_ids; unsigned int rss_nch; struct { @@ -34,6 +35,13 @@ struct mlx5e_rx_res { /* API for rx_res_rss_* */ +static u32 *get_vhca_ids(struct mlx5e_rx_res *res, int offset) +{ + bool multi_vhca = res->features & MLX5E_RX_RES_FEATURE_MULTI_VHCA; + + return multi_vhca ? res->rss_vhca_ids + offset : NULL; +} + void mlx5e_rx_res_rss_update_num_channels(struct mlx5e_rx_res *res, u32 nch) { int i; @@ -85,8 +93,11 @@ int mlx5e_rx_res_rss_init(struct mlx5e_rx_res *res, u32 *rss_idx, unsigned int i return PTR_ERR(rss); mlx5e_rss_set_indir_uniform(rss, init_nch); - if (res->rss_active) - mlx5e_rss_enable(rss, res->rss_rqns, res->rss_nch); + if (res->rss_active) { + u32 *vhca_ids = get_vhca_ids(res, 0); + + mlx5e_rss_enable(rss, res->rss_rqns, vhca_ids, res->rss_nch); + } res->rss[i] = rss; *rss_idx = i; @@ -153,10 +164,12 @@ static void mlx5e_rx_res_rss_enable(struct mlx5e_rx_res *res) for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) { struct mlx5e_rss *rss = res->rss[i]; + u32 *vhca_ids; if (!rss) continue; - mlx5e_rss_enable(rss, res->rss_rqns, res->rss_nch); + vhca_ids = get_vhca_ids(res, 0); + mlx5e_rss_enable(rss, res->rss_rqns, vhca_ids, res->rss_nch); } } @@ -200,6 +213,7 @@ int mlx5e_rx_res_rss_get_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, int mlx5e_rx_res_rss_set_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, const u32 *indir, const u8 *key, const u8 *hfunc) { + u32 *vhca_ids = get_vhca_ids(res, 0); struct mlx5e_rss *rss; if (rss_idx >= MLX5E_MAX_NUM_RSS) @@ -209,7 +223,8 @@ int mlx5e_rx_res_rss_set_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, if (!rss) return -ENOENT; - return mlx5e_rss_set_rxfh(rss, indir, key, hfunc, res->rss_rqns, res->rss_nch); + return mlx5e_rss_set_rxfh(rss, indir, key, hfunc, res->rss_rqns, vhca_ids, + res->rss_nch); } int mlx5e_rx_res_rss_get_hash_fields(struct mlx5e_rx_res *res, u32 rss_idx, @@ -280,11 +295,13 @@ struct mlx5e_rss *mlx5e_rx_res_rss_get(struct mlx5e_rx_res *res, u32 rss_idx) static void mlx5e_rx_res_free(struct mlx5e_rx_res *res) { + kvfree(res->rss_vhca_ids); kvfree(res->rss_rqns); kvfree(res); } -static struct mlx5e_rx_res *mlx5e_rx_res_alloc(struct mlx5_core_dev *mdev, unsigned int max_nch) +static struct mlx5e_rx_res *mlx5e_rx_res_alloc(struct mlx5_core_dev *mdev, unsigned int max_nch, + bool multi_vhca) { struct mlx5e_rx_res *rx_res; @@ -298,6 +315,15 @@ static struct mlx5e_rx_res *mlx5e_rx_res_alloc(struct mlx5_core_dev *mdev, unsig return NULL; } + if (multi_vhca) { + rx_res->rss_vhca_ids = kvcalloc(max_nch, sizeof(*rx_res->rss_vhca_ids), GFP_KERNEL); + if (!rx_res->rss_vhca_ids) { + kvfree(rx_res->rss_rqns); + kvfree(rx_res); + return NULL; + } + } + return rx_res; } @@ -424,10 +450,11 @@ mlx5e_rx_res_create(struct mlx5_core_dev *mdev, enum mlx5e_rx_res_features featu const struct mlx5e_packet_merge_param *init_pkt_merge_param, unsigned int init_nch) { + bool multi_vhca = features & MLX5E_RX_RES_FEATURE_MULTI_VHCA; struct mlx5e_rx_res *res; int err; - res = mlx5e_rx_res_alloc(mdev, max_nch); + res = mlx5e_rx_res_alloc(mdev, max_nch, multi_vhca); if (!res) return ERR_PTR(-ENOMEM); @@ -504,10 +531,11 @@ static void mlx5e_rx_res_channel_activate_direct(struct mlx5e_rx_res *res, struct mlx5e_channels *chs, unsigned int ix) { + u32 *vhca_id = get_vhca_ids(res, ix); u32 rqn = res->rss_rqns[ix]; int err; - err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn); + err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn, vhca_id); if (err) mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (channel %u): err = %d\n", mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), @@ -519,7 +547,7 @@ static void mlx5e_rx_res_channel_deactivate_direct(struct mlx5e_rx_res *res, { int err; - err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn); + err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn, NULL); if (err) mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n", mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), @@ -534,10 +562,12 @@ void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_chann nch = mlx5e_channels_get_num(chs); for (ix = 0; ix < chs->num; ix++) { + u32 *vhca_id = get_vhca_ids(res, ix); + if (mlx5e_channels_is_xsk(chs, ix)) - mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]); + mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix], vhca_id); else - mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]); + mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix], vhca_id); } res->rss_nch = chs->num; @@ -554,7 +584,7 @@ void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_chann if (!mlx5e_channels_get_ptp_rqn(chs, &rqn)) rqn = res->drop_rqn; - err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, rqn); + err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, rqn, NULL); if (err) mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (PTP): err = %d\n", mlx5e_rqt_get_rqtn(&res->ptp.rqt), @@ -573,7 +603,7 @@ void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res) mlx5e_rx_res_channel_deactivate_direct(res, ix); if (res->features & MLX5E_RX_RES_FEATURE_PTP) { - err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, res->drop_rqn); + err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, res->drop_rqn, NULL); if (err) mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (PTP): err = %d\n", mlx5e_rqt_get_rqtn(&res->ptp.rqt), @@ -584,10 +614,12 @@ void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res) void mlx5e_rx_res_xsk_update(struct mlx5e_rx_res *res, struct mlx5e_channels *chs, unsigned int ix, bool xsk) { + u32 *vhca_id = get_vhca_ids(res, ix); + if (xsk) - mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]); + mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix], vhca_id); else - mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]); + mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix], vhca_id); mlx5e_rx_res_rss_enable(res); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h index 82aaba8a8..7b1a9f0f1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h @@ -18,6 +18,7 @@ struct mlx5e_rss_params_hash; enum mlx5e_rx_res_features { MLX5E_RX_RES_FEATURE_INNER_FT = BIT(0), MLX5E_RX_RES_FEATURE_PTP = BIT(1), + MLX5E_RX_RES_FEATURE_MULTI_VHCA = BIT(2), }; /* Setup */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c index ac458a8d1..53ca16cb9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c @@ -63,10 +63,12 @@ static int mlx5e_open_trap_rq(struct mlx5e_priv *priv, struct mlx5e_trap *t) struct mlx5e_create_cq_param ccp = {}; struct dim_cq_moder trap_moder = {}; struct mlx5e_rq *rq = &t->rq; + u16 q_counter; int node; int err; node = dev_to_node(mdev->device); + q_counter = priv->q_counter[0]; ccp.netdev = priv->netdev; ccp.wq = priv->wq; @@ -79,7 +81,7 @@ static int mlx5e_open_trap_rq(struct mlx5e_priv *priv, struct mlx5e_trap *t) return err; mlx5e_init_trap_rq(t, &t->params, rq); - err = mlx5e_open_rq(&t->params, rq_param, NULL, node, rq); + err = mlx5e_open_rq(&t->params, rq_param, NULL, node, q_counter, rq); if (err) goto err_destroy_cq; @@ -116,15 +118,14 @@ static int mlx5e_create_trap_direct_rq_tir(struct mlx5_core_dev *mdev, struct ml } static void mlx5e_build_trap_params(struct mlx5_core_dev *mdev, - int max_mtu, u16 q_counter, - struct mlx5e_trap *t) + int max_mtu, struct mlx5e_trap *t) { struct mlx5e_params *params = &t->params; params->rq_wq_type = MLX5_WQ_TYPE_CYCLIC; mlx5e_init_rq_type_params(mdev, params); params->sw_mtu = max_mtu; - mlx5e_build_rq_param(mdev, params, NULL, q_counter, &t->rq_param); + mlx5e_build_rq_param(mdev, params, NULL, &t->rq_param); } static struct mlx5e_trap *mlx5e_open_trap(struct mlx5e_priv *priv) @@ -138,7 +139,7 @@ static struct mlx5e_trap *mlx5e_open_trap(struct mlx5e_priv *priv) if (!t) return ERR_PTR(-ENOMEM); - mlx5e_build_trap_params(priv->mdev, netdev->max_mtu, priv->q_counter, t); + mlx5e_build_trap_params(priv->mdev, netdev->max_mtu, t); t->priv = priv; t->mdev = priv->mdev; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c index ebada0c5a..db776e515 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c @@ -6,10 +6,10 @@ #include "setup.h" #include "en/params.h" -static int mlx5e_xsk_map_pool(struct mlx5e_priv *priv, +static int mlx5e_xsk_map_pool(struct mlx5_core_dev *mdev, struct xsk_buff_pool *pool) { - struct device *dev = mlx5_core_dma_dev(priv->mdev); + struct device *dev = mlx5_core_dma_dev(mdev); return xsk_pool_dma_map(pool, dev, DMA_ATTR_SKIP_CPU_SYNC); } @@ -89,7 +89,7 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, if (unlikely(!mlx5e_xsk_is_pool_sane(pool))) return -EINVAL; - err = mlx5e_xsk_map_pool(priv, pool); + err = mlx5e_xsk_map_pool(mlx5_sd_ch_ix_get_dev(priv->mdev, ix), pool); if (unlikely(err)) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c index c969b8e70..9240cfe25 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c @@ -51,10 +51,9 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params, static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, - u16 q_counter, struct mlx5e_channel_param *cparam) { - mlx5e_build_rq_param(mdev, params, xsk, q_counter, &cparam->rq); + mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq); mlx5e_build_xdpsq_param(mdev, params, xsk, &cparam->xdp_sq); } @@ -95,6 +94,7 @@ static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *param struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool, struct mlx5e_xsk_param *xsk) { + u16 q_counter = c->priv->q_counter[c->sd_ix]; struct mlx5e_rq *xskrq = &c->xskrq; int err; @@ -102,7 +102,7 @@ static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *param if (err) return err; - err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), xskrq); + err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), q_counter, xskrq); if (err) return err; @@ -127,7 +127,7 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, if (!cparam) return -ENOMEM; - mlx5e_build_xsk_cparam(priv->mdev, params, xsk, priv->q_counter, cparam); + mlx5e_build_xsk_cparam(priv->mdev, params, xsk, cparam); err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, &c->xskrq.cq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h index caa34b9c1..33e32584b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h @@ -102,8 +102,14 @@ static inline void mlx5e_udp_gso_handle_tx_skb(struct sk_buff *skb) { int payload_len = skb_shinfo(skb)->gso_size + sizeof(struct udphdr); + struct udphdr *udphdr; - udp_hdr(skb)->len = htons(payload_len); + if (skb->encapsulation) + udphdr = (struct udphdr *)skb_inner_transport_header(skb); + else + udphdr = udp_hdr(skb); + + udphdr->len = htons(payload_len); } struct mlx5e_accel_tx_state { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 05612d9c6..c54fd01ea 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -984,21 +984,41 @@ static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) queue_work(sa_entry->ipsec->wq, &work->work); } -static void mlx5e_xfrm_update_curlft(struct xfrm_state *x) +static void mlx5e_xfrm_update_stats(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; + struct net *net = dev_net(x->xso.dev); u64 packets, bytes, lastuse; lockdep_assert(lockdep_is_held(&x->lock) || - lockdep_is_held(&dev_net(x->xso.real_dev)->xfrm.xfrm_cfg_mutex)); + lockdep_is_held(&dev_net(x->xso.real_dev)->xfrm.xfrm_cfg_mutex) || + lockdep_is_held(&dev_net(x->xso.real_dev)->xfrm.xfrm_state_lock)); if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ) return; + if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { + mlx5_fc_query_cached(ipsec_rule->auth.fc, &bytes, &packets, &lastuse); + x->stats.integrity_failed += packets; + XFRM_ADD_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR, packets); + + mlx5_fc_query_cached(ipsec_rule->trailer.fc, &bytes, &packets, &lastuse); + XFRM_ADD_STATS(net, LINUX_MIB_XFRMINHDRERROR, packets); + } + + if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET) + return; + mlx5_fc_query_cached(ipsec_rule->fc, &bytes, &packets, &lastuse); x->curlft.packets += packets; x->curlft.bytes += bytes; + + if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { + mlx5_fc_query_cached(ipsec_rule->replay.fc, &bytes, &packets, &lastuse); + x->stats.replay += packets; + XFRM_ADD_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR, packets); + } } static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev, @@ -1156,7 +1176,7 @@ static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { .xdo_dev_offload_ok = mlx5e_ipsec_offload_ok, .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, - .xdo_dev_state_update_curlft = mlx5e_xfrm_update_curlft, + .xdo_dev_state_update_stats = mlx5e_xfrm_update_stats, .xdo_dev_policy_add = mlx5e_xfrm_add_policy, .xdo_dev_policy_delete = mlx5e_xfrm_del_policy, .xdo_dev_policy_free = mlx5e_xfrm_free_policy, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index adaea3493..7d943e93c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -137,7 +137,6 @@ struct mlx5e_ipsec_hw_stats { struct mlx5e_ipsec_sw_stats { atomic64_t ipsec_rx_drop_sp_alloc; atomic64_t ipsec_rx_drop_sadb_miss; - atomic64_t ipsec_rx_drop_syndrome; atomic64_t ipsec_tx_drop_bundle; atomic64_t ipsec_tx_drop_no_state; atomic64_t ipsec_tx_drop_not_ip; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 41a2543a5..e51b03d4c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -750,8 +750,7 @@ err_fs: err_fs_ft: if (rx->allow_tunnel_mode) mlx5_eswitch_unblock_encap(mdev); - mlx5_del_flow_rules(rx->status.rule); - mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); + mlx5_ipsec_rx_status_destroy(ipsec, rx); err_add: mlx5_destroy_flow_table(rx->ft.status); err_fs_ft_status: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index 51a144246..727fa7c18 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c @@ -304,12 +304,6 @@ drop: return false; } -enum { - MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED, - MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_AUTH_FAILED, - MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_BAD_TRAILER, -}; - void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, u32 ipsec_meta_data) @@ -343,20 +337,7 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev, xo = xfrm_offload(skb); xo->flags = CRYPTO_DONE; - - switch (MLX5_IPSEC_METADATA_SYNDROM(ipsec_meta_data)) { - case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED: - xo->status = CRYPTO_SUCCESS; - break; - case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_AUTH_FAILED: - xo->status = CRYPTO_TUNNEL_ESP_AUTH_FAILED; - break; - case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_BAD_TRAILER: - xo->status = CRYPTO_INVALID_PACKET_SYNTAX; - break; - default: - atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_syndrome); - } + xo->status = CRYPTO_SUCCESS; } int mlx5_esw_ipsec_rx_make_metadata(struct mlx5e_priv *priv, u32 id, u32 *metadata) @@ -374,8 +355,6 @@ int mlx5_esw_ipsec_rx_make_metadata(struct mlx5e_priv *priv, u32 id, u32 *metada return err; } - *metadata = MLX5_IPSEC_METADATA_CREATE(ipsec_obj_id, - MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED); - + *metadata = ipsec_obj_id; return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h index 2ed99772f..359050f0b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h @@ -43,7 +43,6 @@ #define MLX5_IPSEC_METADATA_MARKER(metadata) (((metadata) >> 31) & 0x1) #define MLX5_IPSEC_METADATA_SYNDROM(metadata) (((metadata) >> 24) & GENMASK(5, 0)) #define MLX5_IPSEC_METADATA_HANDLE(metadata) ((metadata) & GENMASK(23, 0)) -#define MLX5_IPSEC_METADATA_CREATE(id, syndrome) ((id) | ((syndrome) << 24)) struct mlx5e_accel_tx_ipsec_state { struct xfrm_offload *xo; @@ -98,18 +97,11 @@ mlx5e_ipsec_feature_check(struct sk_buff *skb, netdev_features_t features) if (!x || !x->xso.offload_handle) goto out_disable; - if (xo->inner_ipproto) { - /* Cannot support tunnel packet over IPsec tunnel mode - * because we cannot offload three IP header csum - */ - if (x->props.mode == XFRM_MODE_TUNNEL) - goto out_disable; - - /* Only support UDP or TCP L4 checksum */ - if (xo->inner_ipproto != IPPROTO_UDP && - xo->inner_ipproto != IPPROTO_TCP) - goto out_disable; - } + /* Only support UDP or TCP L4 checksum */ + if (xo->inner_ipproto && + xo->inner_ipproto != IPPROTO_UDP && + xo->inner_ipproto != IPPROTO_TCP) + goto out_disable; return features; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c index e0e36a097..dd36b04e3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_stats.c @@ -51,7 +51,6 @@ static const struct counter_desc mlx5e_ipsec_hw_stats_desc[] = { static const struct counter_desc mlx5e_ipsec_sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_sw_stats, ipsec_rx_drop_sp_alloc) }, { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_sw_stats, ipsec_rx_drop_sadb_miss) }, - { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_sw_stats, ipsec_rx_drop_syndrome) }, { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_sw_stats, ipsec_tx_drop_bundle) }, { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_sw_stats, ipsec_tx_drop_no_state) }, { MLX5E_DECLARE_STAT(struct mlx5e_ipsec_sw_stats, ipsec_tx_drop_not_ip) }, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index 984fa04bd..e3e57c849 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -96,7 +96,7 @@ bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev) { u8 max_sq_wqebbs = mlx5e_get_max_sq_wqebbs(mdev); - if (is_kdump_kernel() || !MLX5_CAP_GEN(mdev, tls_rx)) + if (is_kdump_kernel() || !MLX5_CAP_GEN(mdev, tls_rx) || mlx5_get_sd(mdev)) return false; /* Check the possibility to post the required ICOSQ WQEs. */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index f11075e67..adc6d8ea0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -11,6 +11,7 @@ #ifdef CONFIG_MLX5_EN_TLS #include "lib/crypto.h" +#include "lib/mlx5.h" struct mlx5_crypto_dek *mlx5_ktls_create_key(struct mlx5_crypto_dek_pool *dek_pool, struct tls_crypto_info *crypto_info); @@ -61,7 +62,8 @@ void mlx5e_ktls_rx_resync_destroy_resp_list(struct mlx5e_ktls_resync_resp *resp_ static inline bool mlx5e_is_ktls_tx(struct mlx5_core_dev *mdev) { - return !is_kdump_kernel() && MLX5_CAP_GEN(mdev, tls_tx); + return !is_kdump_kernel() && MLX5_CAP_GEN(mdev, tls_tx) && + !mlx5_get_sd(mdev); } bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c index 9b597cb24..65ccb33ed 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -267,7 +267,7 @@ resync_post_get_progress_params(struct mlx5e_icosq *sq, goto err_out; } - pdev = mlx5_core_dma_dev(sq->channel->priv->mdev); + pdev = mlx5_core_dma_dev(sq->channel->mdev); buf->dma_addr = dma_map_single(pdev, &buf->progress, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); if (unlikely(dma_mapping_error(pdev, buf->dma_addr))) { @@ -425,14 +425,12 @@ void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi, { struct mlx5e_ktls_rx_resync_buf *buf = wi->tls_get_params.buf; struct mlx5e_ktls_offload_context_rx *priv_rx; - struct mlx5e_ktls_rx_resync_ctx *resync; u8 tracker_state, auth_state, *ctx; struct device *dev; u32 hw_seq; priv_rx = buf->priv_rx; - resync = &priv_rx->resync; - dev = mlx5_core_dma_dev(resync->priv->mdev); + dev = mlx5_core_dma_dev(sq->channel->mdev); if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) goto out; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c index 415fec776..93cf23278 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c @@ -34,6 +34,7 @@ #include <linux/mlx5/fs.h> #include <linux/ip.h> #include <linux/ipv6.h> +#include <net/rps.h> #include "en.h" #define ARFS_HASH_SHIFT BITS_PER_BYTE diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 93461b0c5..67a29826b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -451,6 +451,17 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, mutex_lock(&priv->state_lock); + if (mlx5e_rx_res_get_current_hash(priv->rx_res).hfunc == ETH_RSS_HASH_XOR) { + unsigned int xor8_max_channels = mlx5e_rqt_max_num_channels_allowed_for_xor8(); + + if (count > xor8_max_channels) { + err = -EINVAL; + netdev_err(priv->netdev, "%s: Requested number of channels (%d) exceeds the maximum allowed by the XOR8 RSS hfunc (%d)\n", + __func__, count, xor8_max_channels); + goto out; + } + } + /* If RXFH is configured, changing the channels number is allowed only if * it does not require resizing the RSS table. This is because the previous * configuration may no longer be compatible with the new RSS table. @@ -578,12 +589,12 @@ static int mlx5e_get_coalesce(struct net_device *netdev, static void mlx5e_set_priv_channels_tx_coalesce(struct mlx5e_priv *priv, struct ethtool_coalesce *coal) { - struct mlx5_core_dev *mdev = priv->mdev; int tc; int i; for (i = 0; i < priv->channels.num; ++i) { struct mlx5e_channel *c = priv->channels.c[i]; + struct mlx5_core_dev *mdev = c->mdev; for (tc = 0; tc < c->num_tc; tc++) { mlx5_core_modify_cq_moderation(mdev, @@ -597,11 +608,11 @@ mlx5e_set_priv_channels_tx_coalesce(struct mlx5e_priv *priv, struct ethtool_coal static void mlx5e_set_priv_channels_rx_coalesce(struct mlx5e_priv *priv, struct ethtool_coalesce *coal) { - struct mlx5_core_dev *mdev = priv->mdev; int i; for (i = 0; i < priv->channels.num; ++i) { struct mlx5e_channel *c = priv->channels.c[i]; + struct mlx5_core_dev *mdev = c->mdev; mlx5_core_modify_cq_moderation(mdev, &c->rq.cq.mcq, coal->rx_coalesce_usecs, @@ -1298,17 +1309,30 @@ int mlx5e_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh, struct mlx5e_priv *priv = netdev_priv(dev); u32 *rss_context = &rxfh->rss_context; u8 hfunc = rxfh->hfunc; + unsigned int count; int err; mutex_lock(&priv->state_lock); + + count = priv->channels.params.num_channels; + + if (hfunc == ETH_RSS_HASH_XOR) { + unsigned int xor8_max_channels = mlx5e_rqt_max_num_channels_allowed_for_xor8(); + + if (count > xor8_max_channels) { + err = -EINVAL; + netdev_err(priv->netdev, "%s: Cannot set RSS hash function to XOR, current number of channels (%d) exceeds the maximum allowed for XOR8 RSS hfunc (%d)\n", + __func__, count, xor8_max_channels); + goto unlock; + } + } + if (*rss_context && rxfh->rss_delete) { err = mlx5e_rx_res_rss_destroy(priv->rx_res, *rss_context); goto unlock; } if (*rss_context == ETH_RXFH_CONTEXT_ALLOC) { - unsigned int count = priv->channels.params.num_channels; - err = mlx5e_rx_res_rss_init(priv->rx_res, rss_context, count); if (err) goto unlock; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 952f1f981..981a3e058 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -70,6 +70,7 @@ #include "qos.h" #include "en/trap.h" #include "lib/devcom.h" +#include "lib/sd.h" bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, enum mlx5e_mpwrq_umr_mode umr_mode) @@ -208,8 +209,8 @@ static int mlx5e_devcom_init_mpv(struct mlx5e_priv *priv, u64 *data) *data, mlx5e_devcom_event_mpv, priv); - if (IS_ERR_OR_NULL(priv->devcom)) - return -EOPNOTSUPP; + if (IS_ERR(priv->devcom)) + return PTR_ERR(priv->devcom); if (mlx5_core_is_mp_master(priv->mdev)) { mlx5_devcom_send_event(priv->devcom, MPV_DEVCOM_MASTER_UP, @@ -1024,7 +1025,7 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq) mlx5_wq_destroy(&rq->wq_ctrl); } -int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param) +int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter) { struct mlx5_core_dev *mdev = rq->mdev; u8 ts_format; @@ -1051,6 +1052,7 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param) MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); MLX5_SET(rqc, rqc, ts_format, ts_format); + MLX5_SET(rqc, rqc, counter_set_id, q_counter); MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); @@ -1274,7 +1276,7 @@ void mlx5e_free_rx_descs(struct mlx5e_rq *rq) } int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, - struct mlx5e_xsk_param *xsk, int node, + struct mlx5e_xsk_param *xsk, int node, u16 q_counter, struct mlx5e_rq *rq) { struct mlx5_core_dev *mdev = rq->mdev; @@ -1287,7 +1289,7 @@ int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, if (err) return err; - err = mlx5e_create_rq(rq, param); + err = mlx5e_create_rq(rq, param, q_counter); if (err) goto err_free_rq; @@ -1806,6 +1808,7 @@ void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq) set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); netdev_tx_reset_queue(sq->txq); netif_tx_start_queue(sq->txq); + netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, sq->cq.napi); } void mlx5e_tx_disable_queue(struct netdev_queue *txq) @@ -1819,6 +1822,7 @@ void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq) { struct mlx5_wq_cyc *wq = &sq->wq; + netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, NULL); clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */ @@ -2333,13 +2337,14 @@ static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, struct mlx5e_rq_param *rq_params) { + u16 q_counter = c->priv->q_counter[c->sd_ix]; int err; err = mlx5e_init_rxq_rq(c, params, rq_params->xdp_frag_size, &c->rq); if (err) return err; - return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), &c->rq); + return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), q_counter, &c->rq); } static int mlx5e_open_queues(struct mlx5e_channel *c, @@ -2526,14 +2531,20 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, struct xsk_buff_pool *xsk_pool, struct mlx5e_channel **cp) { - int cpu = mlx5_comp_vector_get_cpu(priv->mdev, ix); struct net_device *netdev = priv->netdev; + struct mlx5_core_dev *mdev; struct mlx5e_xsk_param xsk; struct mlx5e_channel *c; unsigned int irq; + int vec_ix; + int cpu; int err; - err = mlx5_comp_irqn_get(priv->mdev, ix, &irq); + mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); + vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); + cpu = mlx5_comp_vector_get_cpu(mdev, vec_ix); + + err = mlx5_comp_irqn_get(mdev, vec_ix, &irq); if (err) return err; @@ -2546,20 +2557,23 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, return -ENOMEM; c->priv = priv; - c->mdev = priv->mdev; + c->mdev = mdev; c->tstamp = &priv->tstamp; c->ix = ix; + c->vec_ix = vec_ix; + c->sd_ix = mlx5_sd_ch_ix_get_dev_ix(mdev, ix); c->cpu = cpu; - c->pdev = mlx5_core_dma_dev(priv->mdev); + c->pdev = mlx5_core_dma_dev(mdev); c->netdev = priv->netdev; - c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey); + c->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); c->num_tc = mlx5e_get_dcb_num_tc(params); c->xdp = !!params->xdp_prog; c->stats = &priv->channel_stats[ix]->ch; c->aff_mask = irq_get_effective_affinity_mask(irq); - c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix); + c->lag_port = mlx5e_enumerate_lag_port(mdev, ix); netif_napi_add(netdev, &c->napi, mlx5e_napi_poll); + netif_napi_set_irq(&c->napi, irq); err = mlx5e_open_queues(c, params, cparam); if (unlikely(err)) @@ -2602,12 +2616,16 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c) mlx5e_activate_xsk(c); else mlx5e_activate_rq(&c->rq); + + netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, &c->napi); } static void mlx5e_deactivate_channel(struct mlx5e_channel *c) { int tc; + netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, NULL); + if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) mlx5e_deactivate_xsk(c); else @@ -2647,7 +2665,7 @@ int mlx5e_open_channels(struct mlx5e_priv *priv, if (!chs->c || !cparam) goto err_free; - err = mlx5e_build_channel_param(priv->mdev, &chs->params, priv->q_counter, cparam); + err = mlx5e_build_channel_param(priv->mdev, &chs->params, cparam); if (err) goto err_free; @@ -2935,15 +2953,18 @@ static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_netdev_queues); static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv, struct mlx5e_params *params) { - struct mlx5_core_dev *mdev = priv->mdev; - int num_comp_vectors, ix, irq; - - num_comp_vectors = mlx5_comp_vectors_max(mdev); + int ix; for (ix = 0; ix < params->num_channels; ix++) { + int num_comp_vectors, irq, vec_ix; + struct mlx5_core_dev *mdev; + + mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); + num_comp_vectors = mlx5_comp_vectors_max(mdev); cpumask_clear(priv->scratchpad.cpumask); + vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); - for (irq = ix; irq < num_comp_vectors; irq += params->num_channels) { + for (irq = vec_ix; irq < num_comp_vectors; irq += params->num_channels) { int cpu = mlx5_comp_vector_get_cpu(mdev, irq); cpumask_set_cpu(cpu, priv->scratchpad.cpumask); @@ -3335,7 +3356,7 @@ int mlx5e_open_drop_rq(struct mlx5e_priv *priv, struct mlx5e_cq *cq = &drop_rq->cq; int err; - mlx5e_build_drop_rq_param(mdev, priv->drop_rq_q_counter, &rq_param); + mlx5e_build_drop_rq_param(mdev, &rq_param); err = mlx5e_alloc_drop_cq(priv, cq, &cq_param); if (err) @@ -3349,7 +3370,7 @@ int mlx5e_open_drop_rq(struct mlx5e_priv *priv, if (err) goto err_destroy_cq; - err = mlx5e_create_rq(drop_rq, &rq_param); + err = mlx5e_create_rq(drop_rq, &rq_param, priv->drop_rq_q_counter); if (err) goto err_free_rq; @@ -3769,7 +3790,7 @@ mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) mlx5e_fold_sw_stats64(priv, stats); } - stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer; + stats->rx_missed_errors = priv->stats.qcnt.rx_out_of_buffer; stats->rx_length_errors = PPORT_802_3_GET(pstats, a_in_range_length_errors) + @@ -4717,7 +4738,7 @@ static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, /* Verify if UDP port is being offloaded by HW */ if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port)) - return features; + return vxlan_features_check(skb, features); #if IS_ENABLED(CONFIG_GENEVE) /* Support Geneve offload for default UDP port */ @@ -4743,7 +4764,6 @@ netdev_features_t mlx5e_features_check(struct sk_buff *skb, struct mlx5e_priv *priv = netdev_priv(netdev); features = vlan_features_check(skb, features); - features = vxlan_features_check(skb, features); /* Validate if the tunneled packet is being offloaded by HW */ if (skb->encapsulation && @@ -5264,13 +5284,17 @@ void mlx5e_create_q_counters(struct mlx5e_priv *priv) u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; struct mlx5_core_dev *mdev = priv->mdev; - int err; + struct mlx5_core_dev *pos; + int err, i; MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); - err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); - if (!err) - priv->q_counter = - MLX5_GET(alloc_q_counter_out, out, counter_set_id); + + mlx5_sd_for_each_dev(i, mdev, pos) { + err = mlx5_cmd_exec_inout(pos, alloc_q_counter, in, out); + if (!err) + priv->q_counter[i] = + MLX5_GET(alloc_q_counter_out, out, counter_set_id); + } err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); if (!err) @@ -5281,13 +5305,17 @@ void mlx5e_create_q_counters(struct mlx5e_priv *priv) void mlx5e_destroy_q_counters(struct mlx5e_priv *priv) { u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; + struct mlx5_core_dev *pos; + int i; MLX5_SET(dealloc_q_counter_in, in, opcode, MLX5_CMD_OP_DEALLOC_Q_COUNTER); - if (priv->q_counter) { - MLX5_SET(dealloc_q_counter_in, in, counter_set_id, - priv->q_counter); - mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in); + mlx5_sd_for_each_dev(i, priv->mdev, pos) { + if (priv->q_counter[i]) { + MLX5_SET(dealloc_q_counter_in, in, counter_set_id, + priv->q_counter[i]); + mlx5_cmd_exec_in(pos, dealloc_q_counter, in); + } } if (priv->drop_rq_q_counter) { @@ -5371,6 +5399,8 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) features = MLX5E_RX_RES_FEATURE_PTP; if (mlx5_tunnel_inner_ft_supported(mdev)) features |= MLX5E_RX_RES_FEATURE_INNER_FT; + if (mlx5_get_sd(priv->mdev)) + features |= MLX5E_RX_RES_FEATURE_MULTI_VHCA; priv->rx_res = mlx5e_rx_res_create(priv->mdev, features, priv->max_nch, priv->drop_rq.rqn, &priv->channels.params.packet_merge, @@ -5978,52 +6008,91 @@ void mlx5e_destroy_netdev(struct mlx5e_priv *priv) free_netdev(netdev); } -static int mlx5e_resume(struct auxiliary_device *adev) +static int _mlx5e_resume(struct auxiliary_device *adev) { struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); struct mlx5e_priv *priv = mlx5e_dev->priv; struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = edev->mdev; - int err; + struct mlx5_core_dev *pos, *to; + int err, i; if (netif_device_present(netdev)) return 0; - err = mlx5e_create_mdev_resources(mdev, true); - if (err) - return err; + mlx5_sd_for_each_dev(i, mdev, pos) { + err = mlx5e_create_mdev_resources(pos, true); + if (err) + goto err_destroy_mdev_res; + } err = mlx5e_attach_netdev(priv); - if (err) { - mlx5e_destroy_mdev_resources(mdev); + if (err) + goto err_destroy_mdev_res; + + return 0; + +err_destroy_mdev_res: + to = pos; + mlx5_sd_for_each_dev_to(i, mdev, to, pos) + mlx5e_destroy_mdev_resources(pos); + return err; +} + +static int mlx5e_resume(struct auxiliary_device *adev) +{ + struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); + struct mlx5_core_dev *mdev = edev->mdev; + struct auxiliary_device *actual_adev; + int err; + + err = mlx5_sd_init(mdev); + if (err) return err; - } + actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); + if (actual_adev) + return _mlx5e_resume(actual_adev); return 0; } -static int _mlx5e_suspend(struct auxiliary_device *adev) +static int _mlx5e_suspend(struct auxiliary_device *adev, bool pre_netdev_reg) { struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); struct mlx5e_priv *priv = mlx5e_dev->priv; struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5_core_dev *pos; + int i; - if (!netif_device_present(netdev)) { + if (!pre_netdev_reg && !netif_device_present(netdev)) { if (test_bit(MLX5E_STATE_DESTROYING, &priv->state)) - mlx5e_destroy_mdev_resources(mdev); + mlx5_sd_for_each_dev(i, mdev, pos) + mlx5e_destroy_mdev_resources(pos); return -ENODEV; } mlx5e_detach_netdev(priv); - mlx5e_destroy_mdev_resources(mdev); + mlx5_sd_for_each_dev(i, mdev, pos) + mlx5e_destroy_mdev_resources(pos); + return 0; } static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state) { - return _mlx5e_suspend(adev); + struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); + struct mlx5_core_dev *mdev = edev->mdev; + struct auxiliary_device *actual_adev; + int err = 0; + + actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); + if (actual_adev) + err = _mlx5e_suspend(actual_adev, false); + + mlx5_sd_cleanup(mdev); + return err; } static int _mlx5e_probe(struct auxiliary_device *adev) @@ -6069,9 +6138,9 @@ static int _mlx5e_probe(struct auxiliary_device *adev) goto err_destroy_netdev; } - err = mlx5e_resume(adev); + err = _mlx5e_resume(adev); if (err) { - mlx5_core_err(mdev, "mlx5e_resume failed, %d\n", err); + mlx5_core_err(mdev, "_mlx5e_resume failed, %d\n", err); goto err_profile_cleanup; } @@ -6087,7 +6156,7 @@ static int _mlx5e_probe(struct auxiliary_device *adev) return 0; err_resume: - _mlx5e_suspend(adev); + _mlx5e_suspend(adev, true); err_profile_cleanup: profile->cleanup(priv); err_destroy_netdev: @@ -6102,24 +6171,51 @@ err_devlink_unregister: static int mlx5e_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { - return _mlx5e_probe(adev); + struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); + struct mlx5_core_dev *mdev = edev->mdev; + struct auxiliary_device *actual_adev; + int err; + + err = mlx5_sd_init(mdev); + if (err) + return err; + + actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); + if (actual_adev) + return _mlx5e_probe(actual_adev); + return 0; } -static void mlx5e_remove(struct auxiliary_device *adev) +static void _mlx5e_remove(struct auxiliary_device *adev) { + struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); struct mlx5e_priv *priv = mlx5e_dev->priv; + struct mlx5_core_dev *mdev = edev->mdev; - mlx5_core_uplink_netdev_set(priv->mdev, NULL); + mlx5_core_uplink_netdev_set(mdev, NULL); mlx5e_dcbnl_delete_app(priv); unregister_netdev(priv->netdev); - _mlx5e_suspend(adev); + _mlx5e_suspend(adev, false); priv->profile->cleanup(priv); mlx5e_destroy_netdev(priv); mlx5e_devlink_port_unregister(mlx5e_dev); mlx5e_destroy_devlink(mlx5e_dev); } +static void mlx5e_remove(struct auxiliary_device *adev) +{ + struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); + struct mlx5_core_dev *mdev = edev->mdev; + struct auxiliary_device *actual_adev; + + actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); + if (actual_adev) + _mlx5e_remove(actual_adev); + + mlx5_sd_cleanup(mdev); +} + static const struct auxiliary_device_id mlx5e_id_table[] = { { .name = MLX5_ADEV_NAME ".eth", }, {}, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 4b96ad657..f3d0898bd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -561,11 +561,23 @@ static const struct counter_desc drop_rq_stats_desc[] = { #define NUM_Q_COUNTERS ARRAY_SIZE(q_stats_desc) #define NUM_DROP_RQ_COUNTERS ARRAY_SIZE(drop_rq_stats_desc) +static bool q_counter_any(struct mlx5e_priv *priv) +{ + struct mlx5_core_dev *pos; + int i; + + mlx5_sd_for_each_dev(i, priv->mdev, pos) + if (priv->q_counter[i++]) + return true; + + return false; +} + static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(qcnt) { int num_stats = 0; - if (priv->q_counter) + if (q_counter_any(priv)) num_stats += NUM_Q_COUNTERS; if (priv->drop_rq_q_counter) @@ -578,7 +590,7 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(qcnt) { int i; - for (i = 0; i < NUM_Q_COUNTERS && priv->q_counter; i++) + for (i = 0; i < NUM_Q_COUNTERS && q_counter_any(priv); i++) strcpy(data + (idx++) * ETH_GSTRING_LEN, q_stats_desc[i].format); @@ -593,7 +605,7 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(qcnt) { int i; - for (i = 0; i < NUM_Q_COUNTERS && priv->q_counter; i++) + for (i = 0; i < NUM_Q_COUNTERS && q_counter_any(priv); i++) data[idx++] = MLX5E_READ_CTR32_CPU(&priv->stats.qcnt, q_stats_desc, i); for (i = 0; i < NUM_DROP_RQ_COUNTERS && priv->drop_rq_q_counter; i++) @@ -607,18 +619,23 @@ static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(qcnt) struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt; u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {}; u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {}; - int ret; + struct mlx5_core_dev *pos; + u32 rx_out_of_buffer = 0; + int ret, i; MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); - if (priv->q_counter) { - MLX5_SET(query_q_counter_in, in, counter_set_id, - priv->q_counter); - ret = mlx5_cmd_exec_inout(priv->mdev, query_q_counter, in, out); - if (!ret) - qcnt->rx_out_of_buffer = MLX5_GET(query_q_counter_out, - out, out_of_buffer); + mlx5_sd_for_each_dev(i, priv->mdev, pos) { + if (priv->q_counter[i]) { + MLX5_SET(query_q_counter_in, in, counter_set_id, + priv->q_counter[i]); + ret = mlx5_cmd_exec_inout(pos, query_q_counter, in, out); + if (!ret) + rx_out_of_buffer += MLX5_GET(query_q_counter_out, + out, out_of_buffer); + } } + qcnt->rx_out_of_buffer = rx_out_of_buffer; if (priv->drop_rq_q_counter) { MLX5_SET(query_q_counter_in, in, counter_set_id, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 9fb2c057b..31ed26cac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -766,7 +766,7 @@ static int mlx5e_hairpin_create_indirect_rqt(struct mlx5e_hairpin *hp) return err; mlx5e_rss_params_indir_init_uniform(&indir, hp->num_channels); - err = mlx5e_rqt_init_indir(&hp->indir_rqt, mdev, hp->pair->rqn, hp->num_channels, + err = mlx5e_rqt_init_indir(&hp->indir_rqt, mdev, hp->pair->rqn, NULL, hp->num_channels, mlx5e_rx_res_get_current_hash(priv->rx_res).hfunc, &indir); @@ -1169,7 +1169,7 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, MLX5_CAP_GEN(priv->mdev, log_min_hairpin_wq_data_sz), MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz)); - params.q_counter = priv->q_counter; + params.q_counter = priv->q_counter[0]; err = devl_param_driverinit_value_get( devlink, MLX5_DEVLINK_PARAM_ID_HAIRPIN_NUM_QUEUES, &val); if (err) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index e21a3b412..0964b16ca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -153,7 +153,11 @@ mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb, int *hopbyhop) *hopbyhop = 0; if (skb->encapsulation) { - ihs = skb_inner_tcp_all_headers(skb); + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) + ihs = skb_inner_transport_offset(skb) + + sizeof(struct udphdr); + else + ihs = skb_inner_tcp_all_headers(skb); stats->tso_inner_packets++; stats->tso_inner_bytes += skb->len - ihs; } else { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 58529d1a9..e8caf12f4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -43,6 +43,7 @@ #include "rdma.h" #include "en.h" #include "fs_core.h" +#include "lib/mlx5.h" #include "lib/devcom.h" #include "lib/eq.h" #include "lib/fs_chains.h" @@ -3067,7 +3068,7 @@ void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key) key, mlx5_esw_offloads_devcom_event, esw); - if (IS_ERR_OR_NULL(esw->devcom)) + if (IS_ERR(esw->devcom)) return; mlx5_devcom_send_event(esw->devcom, @@ -3719,6 +3720,12 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (esw_mode_from_devlink(mode, &mlx5_mode)) return -EINVAL; + if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && mlx5_get_sd(esw->dev)) { + NL_SET_ERR_MSG_MOD(extack, + "Can't change E-Switch mode to switchdev when multi-PF netdev (Socket Direct) is configured."); + return -EPERM; + } + mlx5_lag_disable_change(esw->dev); err = mlx5_esw_try_lock(esw); if (err < 0) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index 58f4c0d0f..6c7f2471f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -366,18 +366,22 @@ int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev) return -EIO; } - mlx5_set_nic_state(dev, MLX5_NIC_IFC_DISABLED); + mlx5_set_nic_state(dev, MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED); /* Loop until device state turns to disable */ end = jiffies + msecs_to_jiffies(delay_ms); do { - if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) + if (mlx5_get_nic_state(dev) == MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED) break; + if (pci_channel_offline(dev->pdev)) { + mlx5_core_err(dev, "PCI channel offline, stop waiting for NIC IFC\n"); + return -EACCES; + } cond_resched(); } while (!time_after(jiffies, end)); - if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) { + if (mlx5_get_nic_state(dev) != MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED) { dev_err(&dev->pdev->dev, "NIC IFC still %d after %lums.\n", mlx5_get_nic_state(dev), delay_ms); return -EIO; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index b5c709bba..a6329ca2d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -116,9 +116,9 @@ u32 mlx5_health_check_fatal_sensors(struct mlx5_core_dev *dev) return MLX5_SENSOR_PCI_COMM_ERR; if (pci_channel_offline(dev->pdev)) return MLX5_SENSOR_PCI_ERR; - if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) + if (mlx5_get_nic_state(dev) == MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED) return MLX5_SENSOR_NIC_DISABLED; - if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_SW_RESET) + if (mlx5_get_nic_state(dev) == MLX5_INITIAL_SEG_NIC_INTERFACE_SW_RESET) return MLX5_SENSOR_NIC_SW_RESET; if (sensor_fw_synd_rfr(dev)) return MLX5_SENSOR_FW_SYND_RFR; @@ -185,7 +185,7 @@ static bool reset_fw_if_needed(struct mlx5_core_dev *dev) /* Write the NIC interface field to initiate the reset, the command * interface address also resides here, don't overwrite it. */ - mlx5_set_nic_state(dev, MLX5_NIC_IFC_SW_RESET); + mlx5_set_nic_state(dev, MLX5_INITIAL_SEG_NIC_INTERFACE_SW_RESET); return true; } @@ -246,13 +246,17 @@ recover_from_sw_reset: /* Recover from SW reset */ end = jiffies + msecs_to_jiffies(delay_ms); do { - if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) + if (mlx5_get_nic_state(dev) == MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED) break; + if (pci_channel_offline(dev->pdev)) { + mlx5_core_err(dev, "PCI channel offline, stop waiting for NIC IFC\n"); + goto unlock; + } msleep(20); } while (!time_after(jiffies, end)); - if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) { + if (mlx5_get_nic_state(dev) != MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED) { dev_err(&dev->pdev->dev, "NIC IFC still %d after %lums.\n", mlx5_get_nic_state(dev), delay_ms); } @@ -272,26 +276,26 @@ static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) u8 nic_interface = mlx5_get_nic_state(dev); switch (nic_interface) { - case MLX5_NIC_IFC_FULL: + case MLX5_INITIAL_SEG_NIC_INTERFACE_FULL_DRIVER: mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n"); break; - case MLX5_NIC_IFC_DISABLED: + case MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED: mlx5_core_warn(dev, "starting teardown\n"); break; - case MLX5_NIC_IFC_NO_DRAM_NIC: + case MLX5_INITIAL_SEG_NIC_INTERFACE_NO_DRAM_NIC: mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n"); break; - case MLX5_NIC_IFC_SW_RESET: + case MLX5_INITIAL_SEG_NIC_INTERFACE_SW_RESET: /* The IFC mode field is 3 bits, so it will read 0x7 in 2 cases: * 1. PCI has been disabled (ie. PCI-AER, PF driver unloaded * and this is a VF), this is not recoverable by SW reset. * Logging of this is handled elsewhere. * 2. FW reset has been issued by another function, driver can * be reloaded to recover after the mode switches to - * MLX5_NIC_IFC_DISABLED. + * MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED. */ if (dev->priv.health.fatal_error != MLX5_SENSOR_PCI_COMM_ERR) mlx5_core_warn(dev, "NIC SW reset in progress\n"); @@ -317,6 +321,10 @@ int mlx5_health_wait_pci_up(struct mlx5_core_dev *dev) mlx5_core_warn(dev, "device is being removed, stop waiting for PCI\n"); return -ENODEV; } + if (pci_channel_offline(dev->pdev)) { + mlx5_core_err(dev, "PCI channel offline, stop waiting for PCI\n"); + return -EACCES; + } msleep(100); } return 0; @@ -555,12 +563,17 @@ static void mlx5_fw_reporter_err_work(struct work_struct *work) &fw_reporter_ctx); } -static const struct devlink_health_reporter_ops mlx5_fw_reporter_ops = { +static const struct devlink_health_reporter_ops mlx5_fw_reporter_pf_ops = { .name = "fw", .diagnose = mlx5_fw_reporter_diagnose, .dump = mlx5_fw_reporter_dump, }; +static const struct devlink_health_reporter_ops mlx5_fw_reporter_ops = { + .name = "fw", + .diagnose = mlx5_fw_reporter_diagnose, +}; + static int mlx5_fw_fatal_reporter_recover(struct devlink_health_reporter *reporter, void *priv_ctx, @@ -646,12 +659,17 @@ static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work) } } -static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = { +static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_pf_ops = { .name = "fw_fatal", .recover = mlx5_fw_fatal_reporter_recover, .dump = mlx5_fw_fatal_reporter_dump, }; +static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = { + .name = "fw_fatal", + .recover = mlx5_fw_fatal_reporter_recover, +}; + #define MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD 180000 #define MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD 60000 #define MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD 30000 @@ -659,10 +677,14 @@ static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = { void mlx5_fw_reporters_create(struct mlx5_core_dev *dev) { + const struct devlink_health_reporter_ops *fw_fatal_ops; struct mlx5_core_health *health = &dev->priv.health; + const struct devlink_health_reporter_ops *fw_ops; struct devlink *devlink = priv_to_devlink(dev); u64 grace_period; + fw_fatal_ops = &mlx5_fw_fatal_reporter_pf_ops; + fw_ops = &mlx5_fw_reporter_pf_ops; if (mlx5_core_is_ecpf(dev)) { grace_period = MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD; } else if (mlx5_core_is_pf(dev)) { @@ -670,18 +692,19 @@ void mlx5_fw_reporters_create(struct mlx5_core_dev *dev) } else { /* VF or SF */ grace_period = MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD; + fw_fatal_ops = &mlx5_fw_fatal_reporter_ops; + fw_ops = &mlx5_fw_reporter_ops; } health->fw_reporter = - devl_health_reporter_create(devlink, &mlx5_fw_reporter_ops, - 0, dev); + devl_health_reporter_create(devlink, fw_ops, 0, dev); if (IS_ERR(health->fw_reporter)) mlx5_core_warn(dev, "Failed to create fw reporter, err = %ld\n", PTR_ERR(health->fw_reporter)); health->fw_fatal_reporter = devl_health_reporter_create(devlink, - &mlx5_fw_fatal_reporter_ops, + fw_fatal_ops, grace_period, dev); if (IS_ERR(health->fw_fatal_reporter)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 37598d116..58a452d20 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -720,6 +720,7 @@ bool mlx5_lag_check_prereq(struct mlx5_lag *ldev) struct mlx5_core_dev *dev; u8 mode; #endif + bool roce_support; int i; for (i = 0; i < ldev->ports; i++) @@ -746,6 +747,11 @@ bool mlx5_lag_check_prereq(struct mlx5_lag *ldev) if (mlx5_sriov_is_enabled(ldev->pf[i].dev)) return false; #endif + roce_support = mlx5_get_roce_state(ldev->pf[MLX5_LAG_P1].dev); + for (i = 1; i < ldev->ports; i++) + if (mlx5_get_roce_state(ldev->pf[i].dev) != roce_support) + return false; + return true; } @@ -913,8 +919,10 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) } else if (roce_lag) { dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; mlx5_rescan_drivers_locked(dev0); - for (i = 1; i < ldev->ports; i++) - mlx5_nic_vport_enable_roce(ldev->pf[i].dev); + for (i = 1; i < ldev->ports; i++) { + if (mlx5_get_roce_state(ldev->pf[i].dev)) + mlx5_nic_vport_enable_roce(ldev->pf[i].dev); + } } else if (shared_fdb) { int i; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c index 101b3bb90..e12bc4cd8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c @@ -88,9 +88,13 @@ static int mlx5_lag_create_port_sel_table(struct mlx5_lag *ldev, &dest, 1); if (IS_ERR(lag_definer->rules[idx])) { err = PTR_ERR(lag_definer->rules[idx]); - while (i--) - while (j--) + do { + while (j--) { + idx = i * ldev->buckets + j; mlx5_del_flow_rules(lag_definer->rules[idx]); + } + j = ldev->buckets; + } while (i--); goto destroy_fg; } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c index e7d59cfa8..7b0766c89 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c @@ -220,7 +220,7 @@ mlx5_devcom_register_component(struct mlx5_devcom_dev *devc, struct mlx5_devcom_comp *comp; if (IS_ERR_OR_NULL(devc)) - return NULL; + return ERR_PTR(-EINVAL); mutex_lock(&comp_list_lock); comp = devcom_component_get(devc, id, key, handler); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h index ec32b686f..d58032dd0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h @@ -10,6 +10,7 @@ enum mlx5_devcom_component { MLX5_DEVCOM_ESW_OFFLOADS, MLX5_DEVCOM_MPV, MLX5_DEVCOM_HCA_PORTS, + MLX5_DEVCOM_SD_GROUP, MLX5_DEVCOM_NUM_COMPONENTS, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h index 2b5826a78..37d5f4455 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h @@ -54,4 +54,16 @@ static inline struct net_device *mlx5_uplink_netdev_get(struct mlx5_core_dev *md { return mdev->mlx5e_res.uplink_netdev; } + +struct mlx5_sd; + +static inline struct mlx5_sd *mlx5_get_sd(struct mlx5_core_dev *dev) +{ + return dev->sd; +} + +static inline void mlx5_set_sd(struct mlx5_core_dev *dev, struct mlx5_sd *sd) +{ + dev->sd = sd; +} #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/pci_vsc.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/pci_vsc.c index 6b774e0c2..d0b595ba6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/pci_vsc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/pci_vsc.c @@ -74,6 +74,10 @@ int mlx5_vsc_gw_lock(struct mlx5_core_dev *dev) ret = -EBUSY; goto pci_unlock; } + if (pci_channel_offline(dev->pdev)) { + ret = -EACCES; + goto pci_unlock; + } /* Check if semaphore is already locked */ ret = vsc_read(dev, VSC_SEMAPHORE_OFFSET, &lock_val); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c new file mode 100644 index 000000000..f6deb5a3f --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include "lib/sd.h" +#include "mlx5_core.h" +#include "lib/mlx5.h" +#include "fs_cmd.h" +#include <linux/mlx5/vport.h> +#include <linux/debugfs.h> + +#define sd_info(__dev, format, ...) \ + dev_info((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__) +#define sd_warn(__dev, format, ...) \ + dev_warn((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__) + +struct mlx5_sd { + u32 group_id; + u8 host_buses; + struct mlx5_devcom_comp_dev *devcom; + struct dentry *dfs; + bool primary; + union { + struct { /* primary */ + struct mlx5_core_dev *secondaries[MLX5_SD_MAX_GROUP_SZ - 1]; + struct mlx5_flow_table *tx_ft; + }; + struct { /* secondary */ + struct mlx5_core_dev *primary_dev; + u32 alias_obj_id; + }; + }; +}; + +static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev) +{ + struct mlx5_sd *sd = mlx5_get_sd(dev); + + if (!sd) + return 1; + + return sd->host_buses; +} + +static struct mlx5_core_dev *mlx5_sd_get_primary(struct mlx5_core_dev *dev) +{ + struct mlx5_sd *sd = mlx5_get_sd(dev); + + if (!sd) + return dev; + + return sd->primary ? dev : sd->primary_dev; +} + +struct mlx5_core_dev * +mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx) +{ + struct mlx5_sd *sd; + + if (idx == 0) + return primary; + + if (idx >= mlx5_sd_get_host_buses(primary)) + return NULL; + + sd = mlx5_get_sd(primary); + return sd->secondaries[idx - 1]; +} + +int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix) +{ + return ch_ix % mlx5_sd_get_host_buses(dev); +} + +int mlx5_sd_ch_ix_get_vec_ix(struct mlx5_core_dev *dev, int ch_ix) +{ + return ch_ix / mlx5_sd_get_host_buses(dev); +} + +struct mlx5_core_dev *mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev *primary, int ch_ix) +{ + int mdev_idx = mlx5_sd_ch_ix_get_dev_ix(primary, ch_ix); + + return mlx5_sd_primary_get_peer(primary, mdev_idx); +} + +static bool ft_create_alias_supported(struct mlx5_core_dev *dev) +{ + u64 obj_allowed = MLX5_CAP_GEN_2_64(dev, allowed_object_for_other_vhca_access); + u32 obj_supp = MLX5_CAP_GEN_2(dev, cross_vhca_object_to_object_supported); + + if (!(obj_supp & + MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_ROOT_TO_REMOTE_FLOW_TABLE)) + return false; + + if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE)) + return false; + + return true; +} + +static bool mlx5_sd_is_supported(struct mlx5_core_dev *dev, u8 host_buses) +{ + /* Honor the SW implementation limit */ + if (host_buses > MLX5_SD_MAX_GROUP_SZ) + return false; + + /* Disconnect secondaries from the network */ + if (!MLX5_CAP_GEN(dev, eswitch_manager)) + return false; + if (!MLX5_CAP_GEN(dev, silent_mode)) + return false; + + /* RX steering from primary to secondaries */ + if (!MLX5_CAP_GEN(dev, cross_vhca_rqt)) + return false; + if (host_buses > MLX5_CAP_GEN_2(dev, max_rqt_vhca_id)) + return false; + + /* TX steering from secondaries to primary */ + if (!ft_create_alias_supported(dev)) + return false; + if (!MLX5_CAP_FLOWTABLE_NIC_TX(dev, reset_root_to_default)) + return false; + + return true; +} + +static int mlx5_query_sd(struct mlx5_core_dev *dev, bool *sdm, + u8 *host_buses, u8 *sd_group) +{ + u32 out[MLX5_ST_SZ_DW(mpir_reg)]; + int err; + + err = mlx5_query_mpir_reg(dev, out); + if (err) + return err; + + err = mlx5_query_nic_vport_sd_group(dev, sd_group); + if (err) + return err; + + *sdm = MLX5_GET(mpir_reg, out, sdm); + *host_buses = MLX5_GET(mpir_reg, out, host_buses); + + return 0; +} + +static u32 mlx5_sd_group_id(struct mlx5_core_dev *dev, u8 sd_group) +{ + return (u32)((MLX5_CAP_GEN(dev, native_port_num) << 8) | sd_group); +} + +static int sd_init(struct mlx5_core_dev *dev) +{ + u8 host_buses, sd_group; + struct mlx5_sd *sd; + u32 group_id; + bool sdm; + int err; + + /* Feature is currently implemented for PFs only */ + if (!mlx5_core_is_pf(dev)) + return 0; + + /* Block on embedded CPU PFs */ + if (mlx5_core_is_ecpf(dev)) + return 0; + + if (!MLX5_CAP_MCAM_REG(dev, mpir)) + return 0; + + err = mlx5_query_sd(dev, &sdm, &host_buses, &sd_group); + if (err) + return err; + + if (!sdm) + return 0; + + if (!sd_group) + return 0; + + group_id = mlx5_sd_group_id(dev, sd_group); + + if (!mlx5_sd_is_supported(dev, host_buses)) { + sd_warn(dev, "can't support requested netdev combining for group id 0x%x), skipping\n", + group_id); + return 0; + } + + sd = kzalloc(sizeof(*sd), GFP_KERNEL); + if (!sd) + return -ENOMEM; + + sd->host_buses = host_buses; + sd->group_id = group_id; + + mlx5_set_sd(dev, sd); + + return 0; +} + +static void sd_cleanup(struct mlx5_core_dev *dev) +{ + struct mlx5_sd *sd = mlx5_get_sd(dev); + + mlx5_set_sd(dev, NULL); + kfree(sd); +} + +static int sd_register(struct mlx5_core_dev *dev) +{ + struct mlx5_devcom_comp_dev *devcom, *pos; + struct mlx5_core_dev *peer, *primary; + struct mlx5_sd *sd, *primary_sd; + int err, i; + + sd = mlx5_get_sd(dev); + devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP, + sd->group_id, NULL, dev); + if (IS_ERR(devcom)) + return PTR_ERR(devcom); + + sd->devcom = devcom; + + if (mlx5_devcom_comp_get_size(devcom) != sd->host_buses) + return 0; + + mlx5_devcom_comp_lock(devcom); + mlx5_devcom_comp_set_ready(devcom, true); + mlx5_devcom_comp_unlock(devcom); + + if (!mlx5_devcom_for_each_peer_begin(devcom)) { + err = -ENODEV; + goto err_devcom_unreg; + } + + primary = dev; + mlx5_devcom_for_each_peer_entry(devcom, peer, pos) + if (peer->pdev->bus->number < primary->pdev->bus->number) + primary = peer; + + primary_sd = mlx5_get_sd(primary); + primary_sd->primary = true; + i = 0; + /* loop the secondaries */ + mlx5_devcom_for_each_peer_entry(primary_sd->devcom, peer, pos) { + struct mlx5_sd *peer_sd = mlx5_get_sd(peer); + + primary_sd->secondaries[i++] = peer; + peer_sd->primary = false; + peer_sd->primary_dev = primary; + } + + mlx5_devcom_for_each_peer_end(devcom); + return 0; + +err_devcom_unreg: + mlx5_devcom_comp_lock(sd->devcom); + mlx5_devcom_comp_set_ready(sd->devcom, false); + mlx5_devcom_comp_unlock(sd->devcom); + mlx5_devcom_unregister_component(sd->devcom); + return err; +} + +static void sd_unregister(struct mlx5_core_dev *dev) +{ + struct mlx5_sd *sd = mlx5_get_sd(dev); + + mlx5_devcom_comp_lock(sd->devcom); + mlx5_devcom_comp_set_ready(sd->devcom, false); + mlx5_devcom_comp_unlock(sd->devcom); + mlx5_devcom_unregister_component(sd->devcom); +} + +static int sd_cmd_set_primary(struct mlx5_core_dev *primary, u8 *alias_key) +{ + struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {}; + struct mlx5_sd *sd = mlx5_get_sd(primary); + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_namespace *nic_ns; + struct mlx5_flow_table *ft; + int err; + + nic_ns = mlx5_get_flow_namespace(primary, MLX5_FLOW_NAMESPACE_EGRESS); + if (!nic_ns) + return -EOPNOTSUPP; + + ft = mlx5_create_flow_table(nic_ns, &ft_attr); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + return err; + } + sd->tx_ft = ft; + memcpy(allow_attr.access_key, alias_key, ACCESS_KEY_LEN); + allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS; + allow_attr.obj_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id; + + err = mlx5_cmd_allow_other_vhca_access(primary, &allow_attr); + if (err) { + mlx5_core_err(primary, "Failed to allow other vhca access err=%d\n", + err); + mlx5_destroy_flow_table(ft); + return err; + } + + return 0; +} + +static void sd_cmd_unset_primary(struct mlx5_core_dev *primary) +{ + struct mlx5_sd *sd = mlx5_get_sd(primary); + + mlx5_destroy_flow_table(sd->tx_ft); +} + +static int sd_secondary_create_alias_ft(struct mlx5_core_dev *secondary, + struct mlx5_core_dev *primary, + struct mlx5_flow_table *ft, + u32 *obj_id, u8 *alias_key) +{ + u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id; + u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(primary, vhca_id); + struct mlx5_cmd_alias_obj_create_attr alias_attr = {}; + int ret; + + memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN); + alias_attr.obj_id = aliased_object_id; + alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS; + alias_attr.vhca_id = vhca_id_to_be_accessed; + ret = mlx5_cmd_alias_obj_create(secondary, &alias_attr, obj_id); + if (ret) { + mlx5_core_err(secondary, "Failed to create alias object err=%d\n", + ret); + return ret; + } + + return 0; +} + +static void sd_secondary_destroy_alias_ft(struct mlx5_core_dev *secondary) +{ + struct mlx5_sd *sd = mlx5_get_sd(secondary); + + mlx5_cmd_alias_obj_destroy(secondary, sd->alias_obj_id, + MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS); +} + +static int sd_cmd_set_secondary(struct mlx5_core_dev *secondary, + struct mlx5_core_dev *primary, + u8 *alias_key) +{ + struct mlx5_sd *primary_sd = mlx5_get_sd(primary); + struct mlx5_sd *sd = mlx5_get_sd(secondary); + int err; + + err = mlx5_fs_cmd_set_l2table_entry_silent(secondary, 1); + if (err) + return err; + + err = sd_secondary_create_alias_ft(secondary, primary, primary_sd->tx_ft, + &sd->alias_obj_id, alias_key); + if (err) + goto err_unset_silent; + + err = mlx5_fs_cmd_set_tx_flow_table_root(secondary, sd->alias_obj_id, false); + if (err) + goto err_destroy_alias_ft; + + return 0; + +err_destroy_alias_ft: + sd_secondary_destroy_alias_ft(secondary); +err_unset_silent: + mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0); + return err; +} + +static void sd_cmd_unset_secondary(struct mlx5_core_dev *secondary) +{ + mlx5_fs_cmd_set_tx_flow_table_root(secondary, 0, true); + sd_secondary_destroy_alias_ft(secondary); + mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0); +} + +static void sd_print_group(struct mlx5_core_dev *primary) +{ + struct mlx5_sd *sd = mlx5_get_sd(primary); + struct mlx5_core_dev *pos; + int i; + + sd_info(primary, "group id %#x, primary %s, vhca %#x\n", + sd->group_id, pci_name(primary->pdev), + MLX5_CAP_GEN(primary, vhca_id)); + mlx5_sd_for_each_secondary(i, primary, pos) + sd_info(primary, "group id %#x, secondary_%d %s, vhca %#x\n", + sd->group_id, i - 1, pci_name(pos->pdev), + MLX5_CAP_GEN(pos, vhca_id)); +} + +static ssize_t dev_read(struct file *filp, char __user *buf, size_t count, + loff_t *pos) +{ + struct mlx5_core_dev *dev; + char tbuf[32]; + int ret; + + dev = filp->private_data; + ret = snprintf(tbuf, sizeof(tbuf), "%s vhca %#x\n", pci_name(dev->pdev), + MLX5_CAP_GEN(dev, vhca_id)); + + return simple_read_from_buffer(buf, count, pos, tbuf, ret); +} + +static const struct file_operations dev_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = dev_read, +}; + +int mlx5_sd_init(struct mlx5_core_dev *dev) +{ + struct mlx5_core_dev *primary, *pos, *to; + struct mlx5_sd *sd = mlx5_get_sd(dev); + u8 alias_key[ACCESS_KEY_LEN]; + int err, i; + + err = sd_init(dev); + if (err) + return err; + + sd = mlx5_get_sd(dev); + if (!sd) + return 0; + + err = sd_register(dev); + if (err) + goto err_sd_cleanup; + + if (!mlx5_devcom_comp_is_ready(sd->devcom)) + return 0; + + primary = mlx5_sd_get_primary(dev); + + for (i = 0; i < ACCESS_KEY_LEN; i++) + alias_key[i] = get_random_u8(); + + err = sd_cmd_set_primary(primary, alias_key); + if (err) + goto err_sd_unregister; + + sd->dfs = debugfs_create_dir("multi-pf", mlx5_debugfs_get_dev_root(primary)); + debugfs_create_x32("group_id", 0400, sd->dfs, &sd->group_id); + debugfs_create_file("primary", 0400, sd->dfs, primary, &dev_fops); + + mlx5_sd_for_each_secondary(i, primary, pos) { + char name[32]; + + err = sd_cmd_set_secondary(pos, primary, alias_key); + if (err) + goto err_unset_secondaries; + + snprintf(name, sizeof(name), "secondary_%d", i - 1); + debugfs_create_file(name, 0400, sd->dfs, pos, &dev_fops); + + } + + sd_info(primary, "group id %#x, size %d, combined\n", + sd->group_id, mlx5_devcom_comp_get_size(sd->devcom)); + sd_print_group(primary); + + return 0; + +err_unset_secondaries: + to = pos; + mlx5_sd_for_each_secondary_to(i, primary, to, pos) + sd_cmd_unset_secondary(pos); + sd_cmd_unset_primary(primary); + debugfs_remove_recursive(sd->dfs); +err_sd_unregister: + sd_unregister(dev); +err_sd_cleanup: + sd_cleanup(dev); + return err; +} + +void mlx5_sd_cleanup(struct mlx5_core_dev *dev) +{ + struct mlx5_sd *sd = mlx5_get_sd(dev); + struct mlx5_core_dev *primary, *pos; + int i; + + if (!sd) + return; + + if (!mlx5_devcom_comp_is_ready(sd->devcom)) + goto out; + + primary = mlx5_sd_get_primary(dev); + mlx5_sd_for_each_secondary(i, primary, pos) + sd_cmd_unset_secondary(pos); + sd_cmd_unset_primary(primary); + debugfs_remove_recursive(sd->dfs); + + sd_info(primary, "group id %#x, uncombined\n", sd->group_id); +out: + sd_unregister(dev); + sd_cleanup(dev); +} + +struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev, + struct auxiliary_device *adev, + int idx) +{ + struct mlx5_sd *sd = mlx5_get_sd(dev); + struct mlx5_core_dev *primary; + + if (!sd) + return adev; + + if (!mlx5_devcom_comp_is_ready(sd->devcom)) + return NULL; + + primary = mlx5_sd_get_primary(dev); + if (dev == primary) + return adev; + + return &primary->priv.adev[idx]->adev; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h new file mode 100644 index 000000000..137efaf9a --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_LIB_SD_H__ +#define __MLX5_LIB_SD_H__ + +#define MLX5_SD_MAX_GROUP_SZ 2 + +struct mlx5_sd; + +struct mlx5_core_dev *mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx); +int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix); +int mlx5_sd_ch_ix_get_vec_ix(struct mlx5_core_dev *dev, int ch_ix); +struct mlx5_core_dev *mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev *primary, int ch_ix); +struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev, + struct auxiliary_device *adev, + int idx); + +int mlx5_sd_init(struct mlx5_core_dev *dev); +void mlx5_sd_cleanup(struct mlx5_core_dev *dev); + +#define mlx5_sd_for_each_dev_from_to(i, primary, ix_from, to, pos) \ + for (i = ix_from; \ + (pos = mlx5_sd_primary_get_peer(primary, i)) && pos != (to); i++) + +#define mlx5_sd_for_each_dev(i, primary, pos) \ + mlx5_sd_for_each_dev_from_to(i, primary, 0, NULL, pos) + +#define mlx5_sd_for_each_dev_to(i, primary, to, pos) \ + mlx5_sd_for_each_dev_from_to(i, primary, 0, to, pos) + +#define mlx5_sd_for_each_secondary(i, primary, pos) \ + mlx5_sd_for_each_dev_from_to(i, primary, 1, NULL, pos) + +#define mlx5_sd_for_each_secondary_to(i, primary, to, pos) \ + mlx5_sd_for_each_dev_from_to(i, primary, 1, to, pos) + +#endif /* __MLX5_LIB_SD_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 0288e19e3..459a836a5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -187,31 +187,36 @@ static struct mlx5_profile profile[] = { }; static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili, - u32 warn_time_mili) + u32 warn_time_mili, const char *init_state) { unsigned long warn = jiffies + msecs_to_jiffies(warn_time_mili); unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili); u32 fw_initializing; - int err = 0; do { fw_initializing = ioread32be(&dev->iseg->initializing); if (!(fw_initializing >> 31)) break; - if (time_after(jiffies, end) || - test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) { - err = -EBUSY; - break; + if (time_after(jiffies, end)) { + mlx5_core_err(dev, "Firmware over %u MS in %s state, aborting\n", + max_wait_mili, init_state); + return -ETIMEDOUT; + } + if (test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) { + mlx5_core_warn(dev, "device is being removed, stop waiting for FW %s\n", + init_state); + return -ENODEV; } if (warn_time_mili && time_after(jiffies, warn)) { - mlx5_core_warn(dev, "Waiting for FW initialization, timeout abort in %ds (0x%x)\n", - jiffies_to_msecs(end - warn) / 1000, fw_initializing); + mlx5_core_warn(dev, "Waiting for FW %s, timeout abort in %ds (0x%x)\n", + init_state, jiffies_to_msecs(end - warn) / 1000, + fw_initializing); warn = jiffies + msecs_to_jiffies(warn_time_mili); } msleep(mlx5_tout_ms(dev, FW_PRE_INIT_WAIT)); } while (true); - return err; + return 0; } static void mlx5_set_driver_version(struct mlx5_core_dev *dev) @@ -951,7 +956,7 @@ static void mlx5_register_hca_devcom_comp(struct mlx5_core_dev *dev) mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_HCA_PORTS, mlx5_query_nic_system_image_guid(dev), NULL, dev); - if (IS_ERR_OR_NULL(dev->priv.hca_devcom_comp)) + if (IS_ERR(dev->priv.hca_devcom_comp)) mlx5_core_err(dev, "Failed to register devcom HCA component\n"); } @@ -1151,12 +1156,10 @@ static int mlx5_function_enable(struct mlx5_core_dev *dev, bool boot, u64 timeou /* wait for firmware to accept initialization segments configurations */ err = wait_fw_init(dev, timeout, - mlx5_tout_ms(dev, FW_PRE_INIT_WARN_MESSAGE_INTERVAL)); - if (err) { - mlx5_core_err(dev, "Firmware over %llu MS in pre-initializing state, aborting\n", - timeout); + mlx5_tout_ms(dev, FW_PRE_INIT_WARN_MESSAGE_INTERVAL), + "pre-initializing"); + if (err) return err; - } err = mlx5_cmd_enable(dev); if (err) { @@ -1166,12 +1169,9 @@ static int mlx5_function_enable(struct mlx5_core_dev *dev, bool boot, u64 timeou mlx5_tout_query_iseg(dev); - err = wait_fw_init(dev, mlx5_tout_ms(dev, FW_INIT), 0); - if (err) { - mlx5_core_err(dev, "Firmware over %llu MS in initializing state, aborting\n", - mlx5_tout_ms(dev, FW_INIT)); + err = wait_fw_init(dev, mlx5_tout_ms(dev, FW_INIT), 0, "initializing"); + if (err) goto err_cmd_cleanup; - } dev->caps.embedded_cpu = mlx5_read_embedded_cpu(dev); mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_UP); @@ -1298,6 +1298,9 @@ static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot) if (!err) mlx5_function_disable(dev, boot); + else + mlx5_stop_health_poll(dev, boot); + return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index a79b79593..58732f449 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -312,13 +312,6 @@ static inline int mlx5_rescan_drivers(struct mlx5_core_dev *dev) return ret; } -enum { - MLX5_NIC_IFC_FULL = 0, - MLX5_NIC_IFC_DISABLED = 1, - MLX5_NIC_IFC_NO_DRAM_NIC = 2, - MLX5_NIC_IFC_SW_RESET = 7 -}; - u8 mlx5_get_nic_state(struct mlx5_core_dev *dev); void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c index c93492b67..99219ea52 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c @@ -74,7 +74,8 @@ static void mlx5_sf_dev_release(struct device *device) kfree(sf_dev); } -static void mlx5_sf_dev_remove(struct mlx5_core_dev *dev, struct mlx5_sf_dev *sf_dev) +static void mlx5_sf_dev_remove_aux(struct mlx5_core_dev *dev, + struct mlx5_sf_dev *sf_dev) { int id; @@ -138,7 +139,7 @@ static void mlx5_sf_dev_add(struct mlx5_core_dev *dev, u16 sf_index, u16 fn_id, return; xa_err: - mlx5_sf_dev_remove(dev, sf_dev); + mlx5_sf_dev_remove_aux(dev, sf_dev); add_err: mlx5_core_err(dev, "SF DEV: fail device add for index=%d sfnum=%d err=%d\n", sf_index, sfnum, err); @@ -149,7 +150,7 @@ static void mlx5_sf_dev_del(struct mlx5_core_dev *dev, struct mlx5_sf_dev *sf_de struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table; xa_erase(&table->devices, sf_index); - mlx5_sf_dev_remove(dev, sf_dev); + mlx5_sf_dev_remove_aux(dev, sf_dev); } static int @@ -367,7 +368,7 @@ static void mlx5_sf_dev_destroy_all(struct mlx5_sf_dev_table *table) xa_for_each(&table->devices, index, sf_dev) { xa_erase(&table->devices, index); - mlx5_sf_dev_remove(table->dev, sf_dev); + mlx5_sf_dev_remove_aux(table->dev, sf_dev); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c index 7e36e1062..030a5776c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c @@ -54,6 +54,107 @@ enum dr_dump_rec_type { DR_DUMP_REC_TYPE_ACTION_MATCH_RANGE = 3425, }; +static struct mlx5dr_dbg_dump_buff * +mlx5dr_dbg_dump_data_init_new_buff(struct mlx5dr_dbg_dump_data *dump_data) +{ + struct mlx5dr_dbg_dump_buff *new_buff; + + new_buff = kzalloc(sizeof(*new_buff), GFP_KERNEL); + if (!new_buff) + return NULL; + + new_buff->buff = kvzalloc(MLX5DR_DEBUG_DUMP_BUFF_SIZE, GFP_KERNEL); + if (!new_buff->buff) { + kfree(new_buff); + return NULL; + } + + INIT_LIST_HEAD(&new_buff->node); + list_add_tail(&new_buff->node, &dump_data->buff_list); + + return new_buff; +} + +static struct mlx5dr_dbg_dump_data * +mlx5dr_dbg_create_dump_data(void) +{ + struct mlx5dr_dbg_dump_data *dump_data; + + dump_data = kzalloc(sizeof(*dump_data), GFP_KERNEL); + if (!dump_data) + return NULL; + + INIT_LIST_HEAD(&dump_data->buff_list); + + if (!mlx5dr_dbg_dump_data_init_new_buff(dump_data)) { + kfree(dump_data); + return NULL; + } + + return dump_data; +} + +static void +mlx5dr_dbg_destroy_dump_data(struct mlx5dr_dbg_dump_data *dump_data) +{ + struct mlx5dr_dbg_dump_buff *dump_buff, *tmp_buff; + + if (!dump_data) + return; + + list_for_each_entry_safe(dump_buff, tmp_buff, &dump_data->buff_list, node) { + kvfree(dump_buff->buff); + list_del(&dump_buff->node); + kfree(dump_buff); + } + + kfree(dump_data); +} + +static int +mlx5dr_dbg_dump_data_print(struct seq_file *file, char *str, u32 size) +{ + struct mlx5dr_domain *dmn = file->private; + struct mlx5dr_dbg_dump_data *dump_data; + struct mlx5dr_dbg_dump_buff *buff; + u32 buff_capacity, write_size; + int remain_size, ret; + + if (size >= MLX5DR_DEBUG_DUMP_BUFF_SIZE) + return -EINVAL; + + dump_data = dmn->dump_info.dump_data; + buff = list_last_entry(&dump_data->buff_list, + struct mlx5dr_dbg_dump_buff, node); + + buff_capacity = (MLX5DR_DEBUG_DUMP_BUFF_SIZE - 1) - buff->index; + remain_size = buff_capacity - size; + write_size = (remain_size > 0) ? size : buff_capacity; + + if (likely(write_size)) { + ret = snprintf(buff->buff + buff->index, write_size + 1, "%s", str); + if (ret < 0) + return ret; + + buff->index += write_size; + } + + if (remain_size < 0) { + remain_size *= -1; + buff = mlx5dr_dbg_dump_data_init_new_buff(dump_data); + if (!buff) + return -ENOMEM; + + ret = snprintf(buff->buff, remain_size + 1, "%s", str + write_size); + if (ret < 0) + return ret; + + buff->index += remain_size; + } + + return 0; +} + void mlx5dr_dbg_tbl_add(struct mlx5dr_table *tbl) { mutex_lock(&tbl->dmn->dump_info.dbg_mutex); @@ -104,41 +205,72 @@ dr_dump_hex_print(char hex[DR_HEX_SIZE], char *src, u32 size) } static int -dr_dump_rule_action_mem(struct seq_file *file, const u64 rule_id, +dr_dump_rule_action_mem(struct seq_file *file, char *buff, const u64 rule_id, struct mlx5dr_rule_action_member *action_mem) { struct mlx5dr_action *action = action_mem->action; const u64 action_id = DR_DBG_PTR_TO_ID(action); u64 hit_tbl_ptr, miss_tbl_ptr; u32 hit_tbl_id, miss_tbl_id; + int ret; switch (action->action_type) { case DR_ACTION_TYP_DROP: - seq_printf(file, "%d,0x%llx,0x%llx\n", - DR_DUMP_REC_TYPE_ACTION_DROP, action_id, rule_id); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx\n", + DR_DUMP_REC_TYPE_ACTION_DROP, action_id, + rule_id); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_FT: if (action->dest_tbl->is_fw_tbl) - seq_printf(file, "%d,0x%llx,0x%llx,0x%x,0x%x\n", - DR_DUMP_REC_TYPE_ACTION_FT, action_id, - rule_id, action->dest_tbl->fw_tbl.id, - -1); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_FT, action_id, + rule_id, action->dest_tbl->fw_tbl.id, + -1); else - seq_printf(file, "%d,0x%llx,0x%llx,0x%x,0x%llx\n", - DR_DUMP_REC_TYPE_ACTION_FT, action_id, - rule_id, action->dest_tbl->tbl->table_id, - DR_DBG_PTR_TO_ID(action->dest_tbl->tbl)); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x,0x%llx\n", + DR_DUMP_REC_TYPE_ACTION_FT, action_id, + rule_id, action->dest_tbl->tbl->table_id, + DR_DBG_PTR_TO_ID(action->dest_tbl->tbl)); + + if (ret < 0) + return ret; + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_CTR: - seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n", - DR_DUMP_REC_TYPE_ACTION_CTR, action_id, rule_id, - action->ctr->ctr_id + action->ctr->offset); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_CTR, action_id, rule_id, + action->ctr->ctr_id + action->ctr->offset); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_TAG: - seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n", - DR_DUMP_REC_TYPE_ACTION_TAG, action_id, rule_id, - action->flow_tag->flow_tag); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_TAG, action_id, rule_id, + action->flow_tag->flow_tag); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_MODIFY_HDR: { @@ -150,83 +282,171 @@ dr_dump_rule_action_mem(struct seq_file *file, const u64 rule_id, ptrn_arg = !action->rewrite->single_action_opt && ptrn && arg; - seq_printf(file, "%d,0x%llx,0x%llx,0x%x,%d,0x%x,0x%x,0x%x", - DR_DUMP_REC_TYPE_ACTION_MODIFY_HDR, action_id, - rule_id, action->rewrite->index, - action->rewrite->single_action_opt, - ptrn_arg ? action->rewrite->num_of_actions : 0, - ptrn_arg ? ptrn->index : 0, - ptrn_arg ? mlx5dr_arg_get_obj_id(arg) : 0); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x,%d,0x%x,0x%x,0x%x", + DR_DUMP_REC_TYPE_ACTION_MODIFY_HDR, action_id, + rule_id, action->rewrite->index, + action->rewrite->single_action_opt, + ptrn_arg ? action->rewrite->num_of_actions : 0, + ptrn_arg ? ptrn->index : 0, + ptrn_arg ? mlx5dr_arg_get_obj_id(arg) : 0); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; if (ptrn_arg) { for (i = 0; i < action->rewrite->num_of_actions; i++) { - seq_printf(file, ",0x%016llx", - be64_to_cpu(((__be64 *)rewrite_data)[i])); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + ",0x%016llx", + be64_to_cpu(((__be64 *)rewrite_data)[i])); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; } } - seq_puts(file, "\n"); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, "\n"); + if (ret < 0) + return ret; + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; } case DR_ACTION_TYP_VPORT: - seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n", - DR_DUMP_REC_TYPE_ACTION_VPORT, action_id, rule_id, - action->vport->caps->num); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_VPORT, action_id, rule_id, + action->vport->caps->num); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_TNL_L2_TO_L2: - seq_printf(file, "%d,0x%llx,0x%llx\n", - DR_DUMP_REC_TYPE_ACTION_DECAP_L2, action_id, - rule_id); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx\n", + DR_DUMP_REC_TYPE_ACTION_DECAP_L2, action_id, + rule_id); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_TNL_L3_TO_L2: - seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n", - DR_DUMP_REC_TYPE_ACTION_DECAP_L3, action_id, - rule_id, - (action->rewrite->ptrn && action->rewrite->arg) ? - mlx5dr_arg_get_obj_id(action->rewrite->arg) : - action->rewrite->index); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_DECAP_L3, action_id, + rule_id, + (action->rewrite->ptrn && action->rewrite->arg) ? + mlx5dr_arg_get_obj_id(action->rewrite->arg) : + action->rewrite->index); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_L2_TO_TNL_L2: - seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n", - DR_DUMP_REC_TYPE_ACTION_ENCAP_L2, action_id, - rule_id, action->reformat->id); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_ENCAP_L2, action_id, + rule_id, action->reformat->id); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_L2_TO_TNL_L3: - seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n", - DR_DUMP_REC_TYPE_ACTION_ENCAP_L3, action_id, - rule_id, action->reformat->id); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_ENCAP_L3, action_id, + rule_id, action->reformat->id); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_POP_VLAN: - seq_printf(file, "%d,0x%llx,0x%llx\n", - DR_DUMP_REC_TYPE_ACTION_POP_VLAN, action_id, - rule_id); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx\n", + DR_DUMP_REC_TYPE_ACTION_POP_VLAN, action_id, + rule_id); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_PUSH_VLAN: - seq_printf(file, "%d,0x%llx,0x%llx,0x%x\n", - DR_DUMP_REC_TYPE_ACTION_PUSH_VLAN, action_id, - rule_id, action->push_vlan->vlan_hdr); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_PUSH_VLAN, action_id, + rule_id, action->push_vlan->vlan_hdr); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_INSERT_HDR: - seq_printf(file, "%d,0x%llx,0x%llx,0x%x,0x%x,0x%x\n", - DR_DUMP_REC_TYPE_ACTION_INSERT_HDR, action_id, - rule_id, action->reformat->id, - action->reformat->param_0, - action->reformat->param_1); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x,0x%x,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_INSERT_HDR, action_id, + rule_id, action->reformat->id, + action->reformat->param_0, + action->reformat->param_1); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_REMOVE_HDR: - seq_printf(file, "%d,0x%llx,0x%llx,0x%x,0x%x,0x%x\n", - DR_DUMP_REC_TYPE_ACTION_REMOVE_HDR, action_id, - rule_id, action->reformat->id, - action->reformat->param_0, - action->reformat->param_1); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x,0x%x,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_REMOVE_HDR, action_id, + rule_id, action->reformat->id, + action->reformat->param_0, + action->reformat->param_1); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_SAMPLER: - seq_printf(file, - "%d,0x%llx,0x%llx,0x%x,0x%x,0x%x,0x%llx,0x%llx\n", - DR_DUMP_REC_TYPE_ACTION_SAMPLER, action_id, rule_id, - 0, 0, action->sampler->sampler_id, - action->sampler->rx_icm_addr, - action->sampler->tx_icm_addr); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x,0x%x,0x%x,0x%llx,0x%llx\n", + DR_DUMP_REC_TYPE_ACTION_SAMPLER, action_id, + rule_id, 0, 0, action->sampler->sampler_id, + action->sampler->rx_icm_addr, + action->sampler->tx_icm_addr); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; case DR_ACTION_TYP_RANGE: if (action->range->hit_tbl_action->dest_tbl->is_fw_tbl) { @@ -247,10 +467,17 @@ dr_dump_rule_action_mem(struct seq_file *file, const u64 rule_id, DR_DBG_PTR_TO_ID(action->range->miss_tbl_action->dest_tbl->tbl); } - seq_printf(file, "%d,0x%llx,0x%llx,0x%x,0x%llx,0x%x,0x%llx,0x%x\n", - DR_DUMP_REC_TYPE_ACTION_MATCH_RANGE, action_id, rule_id, - hit_tbl_id, hit_tbl_ptr, miss_tbl_id, miss_tbl_ptr, - action->range->definer_id); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x,0x%llx,0x%x,0x%llx,0x%x\n", + DR_DUMP_REC_TYPE_ACTION_MATCH_RANGE, action_id, + rule_id, hit_tbl_id, hit_tbl_ptr, miss_tbl_id, + miss_tbl_ptr, action->range->definer_id); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; break; default: return 0; @@ -260,11 +487,12 @@ dr_dump_rule_action_mem(struct seq_file *file, const u64 rule_id, } static int -dr_dump_rule_mem(struct seq_file *file, struct mlx5dr_ste *ste, +dr_dump_rule_mem(struct seq_file *file, char *buff, struct mlx5dr_ste *ste, bool is_rx, const u64 rule_id, u8 format_ver) { char hw_ste_dump[DR_HEX_SIZE]; u32 mem_rec_type; + int ret; if (format_ver == MLX5_STEERING_FORMAT_CONNECTX_5) { mem_rec_type = is_rx ? DR_DUMP_REC_TYPE_RULE_RX_ENTRY_V0 : @@ -277,15 +505,23 @@ dr_dump_rule_mem(struct seq_file *file, struct mlx5dr_ste *ste, dr_dump_hex_print(hw_ste_dump, (char *)mlx5dr_ste_get_hw_ste(ste), DR_STE_SIZE_REDUCED); - seq_printf(file, "%d,0x%llx,0x%llx,%s\n", mem_rec_type, - dr_dump_icm_to_idx(mlx5dr_ste_get_icm_addr(ste)), rule_id, - hw_ste_dump); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,%s\n", mem_rec_type, + dr_dump_icm_to_idx(mlx5dr_ste_get_icm_addr(ste)), + rule_id, hw_ste_dump); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; return 0; } static int -dr_dump_rule_rx_tx(struct seq_file *file, struct mlx5dr_rule_rx_tx *rule_rx_tx, +dr_dump_rule_rx_tx(struct seq_file *file, char *buff, + struct mlx5dr_rule_rx_tx *rule_rx_tx, bool is_rx, const u64 rule_id, u8 format_ver) { struct mlx5dr_ste *ste_arr[DR_RULE_MAX_STES + DR_ACTION_MAX_STES]; @@ -296,7 +532,7 @@ dr_dump_rule_rx_tx(struct seq_file *file, struct mlx5dr_rule_rx_tx *rule_rx_tx, return 0; while (i--) { - ret = dr_dump_rule_mem(file, ste_arr[i], is_rx, rule_id, + ret = dr_dump_rule_mem(file, buff, ste_arr[i], is_rx, rule_id, format_ver); if (ret < 0) return ret; @@ -305,10 +541,12 @@ dr_dump_rule_rx_tx(struct seq_file *file, struct mlx5dr_rule_rx_tx *rule_rx_tx, return 0; } -static int dr_dump_rule(struct seq_file *file, struct mlx5dr_rule *rule) +static noinline_for_stack int +dr_dump_rule(struct seq_file *file, struct mlx5dr_rule *rule) { struct mlx5dr_rule_action_member *action_mem; const u64 rule_id = DR_DBG_PTR_TO_ID(rule); + char buff[MLX5DR_DEBUG_DUMP_BUFF_LENGTH]; struct mlx5dr_rule_rx_tx *rx = &rule->rx; struct mlx5dr_rule_rx_tx *tx = &rule->tx; u8 format_ver; @@ -316,23 +554,30 @@ static int dr_dump_rule(struct seq_file *file, struct mlx5dr_rule *rule) format_ver = rule->matcher->tbl->dmn->info.caps.sw_format_ver; - seq_printf(file, "%d,0x%llx,0x%llx\n", DR_DUMP_REC_TYPE_RULE, rule_id, - DR_DBG_PTR_TO_ID(rule->matcher)); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx\n", DR_DUMP_REC_TYPE_RULE, + rule_id, DR_DBG_PTR_TO_ID(rule->matcher)); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; if (rx->nic_matcher) { - ret = dr_dump_rule_rx_tx(file, rx, true, rule_id, format_ver); + ret = dr_dump_rule_rx_tx(file, buff, rx, true, rule_id, format_ver); if (ret < 0) return ret; } if (tx->nic_matcher) { - ret = dr_dump_rule_rx_tx(file, tx, false, rule_id, format_ver); + ret = dr_dump_rule_rx_tx(file, buff, tx, false, rule_id, format_ver); if (ret < 0) return ret; } list_for_each_entry(action_mem, &rule->rule_actions_list, list) { - ret = dr_dump_rule_action_mem(file, rule_id, action_mem); + ret = dr_dump_rule_action_mem(file, buff, rule_id, action_mem); if (ret < 0) return ret; } @@ -341,65 +586,123 @@ static int dr_dump_rule(struct seq_file *file, struct mlx5dr_rule *rule) } static int -dr_dump_matcher_mask(struct seq_file *file, struct mlx5dr_match_param *mask, +dr_dump_matcher_mask(struct seq_file *file, char *buff, + struct mlx5dr_match_param *mask, u8 criteria, const u64 matcher_id) { char dump[DR_HEX_SIZE]; + int ret; - seq_printf(file, "%d,0x%llx,", DR_DUMP_REC_TYPE_MATCHER_MASK, - matcher_id); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, "%d,0x%llx,", + DR_DUMP_REC_TYPE_MATCHER_MASK, matcher_id); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; if (criteria & DR_MATCHER_CRITERIA_OUTER) { dr_dump_hex_print(dump, (char *)&mask->outer, sizeof(mask->outer)); - seq_printf(file, "%s,", dump); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%s,", dump); } else { - seq_puts(file, ","); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, ","); } + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; + if (criteria & DR_MATCHER_CRITERIA_INNER) { dr_dump_hex_print(dump, (char *)&mask->inner, sizeof(mask->inner)); - seq_printf(file, "%s,", dump); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%s,", dump); } else { - seq_puts(file, ","); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, ","); } + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; + if (criteria & DR_MATCHER_CRITERIA_MISC) { dr_dump_hex_print(dump, (char *)&mask->misc, sizeof(mask->misc)); - seq_printf(file, "%s,", dump); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%s,", dump); } else { - seq_puts(file, ","); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, ","); } + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; + if (criteria & DR_MATCHER_CRITERIA_MISC2) { dr_dump_hex_print(dump, (char *)&mask->misc2, sizeof(mask->misc2)); - seq_printf(file, "%s,", dump); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%s,", dump); } else { - seq_puts(file, ","); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, ","); } + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; + if (criteria & DR_MATCHER_CRITERIA_MISC3) { dr_dump_hex_print(dump, (char *)&mask->misc3, sizeof(mask->misc3)); - seq_printf(file, "%s\n", dump); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%s\n", dump); } else { - seq_puts(file, ",\n"); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, ",\n"); } + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; + return 0; } static int -dr_dump_matcher_builder(struct seq_file *file, struct mlx5dr_ste_build *builder, +dr_dump_matcher_builder(struct seq_file *file, char *buff, + struct mlx5dr_ste_build *builder, u32 index, bool is_rx, const u64 matcher_id) { - seq_printf(file, "%d,0x%llx,%d,%d,0x%x\n", - DR_DUMP_REC_TYPE_MATCHER_BUILDER, matcher_id, index, is_rx, - builder->lu_type); + int ret; + + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,%d,%d,0x%x\n", + DR_DUMP_REC_TYPE_MATCHER_BUILDER, matcher_id, index, + is_rx, builder->lu_type); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; return 0; } static int -dr_dump_matcher_rx_tx(struct seq_file *file, bool is_rx, +dr_dump_matcher_rx_tx(struct seq_file *file, char *buff, bool is_rx, struct mlx5dr_matcher_rx_tx *matcher_rx_tx, const u64 matcher_id) { @@ -412,14 +715,22 @@ dr_dump_matcher_rx_tx(struct seq_file *file, bool is_rx, s_icm_addr = mlx5dr_icm_pool_get_chunk_icm_addr(matcher_rx_tx->s_htbl->chunk); e_icm_addr = mlx5dr_icm_pool_get_chunk_icm_addr(matcher_rx_tx->e_anchor->chunk); - seq_printf(file, "%d,0x%llx,0x%llx,%d,0x%llx,0x%llx\n", - rec_type, DR_DBG_PTR_TO_ID(matcher_rx_tx), - matcher_id, matcher_rx_tx->num_of_builders, - dr_dump_icm_to_idx(s_icm_addr), - dr_dump_icm_to_idx(e_icm_addr)); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,%d,0x%llx,0x%llx\n", + rec_type, DR_DBG_PTR_TO_ID(matcher_rx_tx), + matcher_id, matcher_rx_tx->num_of_builders, + dr_dump_icm_to_idx(s_icm_addr), + dr_dump_icm_to_idx(e_icm_addr)); + + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; for (i = 0; i < matcher_rx_tx->num_of_builders; i++) { - ret = dr_dump_matcher_builder(file, + ret = dr_dump_matcher_builder(file, buff, &matcher_rx_tx->ste_builder[i], i, is_rx, matcher_id); if (ret < 0) @@ -429,32 +740,41 @@ dr_dump_matcher_rx_tx(struct seq_file *file, bool is_rx, return 0; } -static int +static noinline_for_stack int dr_dump_matcher(struct seq_file *file, struct mlx5dr_matcher *matcher) { struct mlx5dr_matcher_rx_tx *rx = &matcher->rx; struct mlx5dr_matcher_rx_tx *tx = &matcher->tx; + char buff[MLX5DR_DEBUG_DUMP_BUFF_LENGTH]; u64 matcher_id; int ret; matcher_id = DR_DBG_PTR_TO_ID(matcher); - seq_printf(file, "%d,0x%llx,0x%llx,%d\n", DR_DUMP_REC_TYPE_MATCHER, - matcher_id, DR_DBG_PTR_TO_ID(matcher->tbl), matcher->prio); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,%d\n", DR_DUMP_REC_TYPE_MATCHER, + matcher_id, DR_DBG_PTR_TO_ID(matcher->tbl), + matcher->prio); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; - ret = dr_dump_matcher_mask(file, &matcher->mask, + ret = dr_dump_matcher_mask(file, buff, &matcher->mask, matcher->match_criteria, matcher_id); if (ret < 0) return ret; if (rx->nic_tbl) { - ret = dr_dump_matcher_rx_tx(file, true, rx, matcher_id); + ret = dr_dump_matcher_rx_tx(file, buff, true, rx, matcher_id); if (ret < 0) return ret; } if (tx->nic_tbl) { - ret = dr_dump_matcher_rx_tx(file, false, tx, matcher_id); + ret = dr_dump_matcher_rx_tx(file, buff, false, tx, matcher_id); if (ret < 0) return ret; } @@ -482,42 +802,59 @@ dr_dump_matcher_all(struct seq_file *file, struct mlx5dr_matcher *matcher) } static int -dr_dump_table_rx_tx(struct seq_file *file, bool is_rx, +dr_dump_table_rx_tx(struct seq_file *file, char *buff, bool is_rx, struct mlx5dr_table_rx_tx *table_rx_tx, const u64 table_id) { enum dr_dump_rec_type rec_type; u64 s_icm_addr; + int ret; rec_type = is_rx ? DR_DUMP_REC_TYPE_TABLE_RX : DR_DUMP_REC_TYPE_TABLE_TX; s_icm_addr = mlx5dr_icm_pool_get_chunk_icm_addr(table_rx_tx->s_anchor->chunk); - seq_printf(file, "%d,0x%llx,0x%llx\n", rec_type, table_id, - dr_dump_icm_to_idx(s_icm_addr)); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx\n", rec_type, table_id, + dr_dump_icm_to_idx(s_icm_addr)); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; return 0; } -static int dr_dump_table(struct seq_file *file, struct mlx5dr_table *table) +static noinline_for_stack int +dr_dump_table(struct seq_file *file, struct mlx5dr_table *table) { struct mlx5dr_table_rx_tx *rx = &table->rx; struct mlx5dr_table_rx_tx *tx = &table->tx; + char buff[MLX5DR_DEBUG_DUMP_BUFF_LENGTH]; int ret; - seq_printf(file, "%d,0x%llx,0x%llx,%d,%d\n", DR_DUMP_REC_TYPE_TABLE, - DR_DBG_PTR_TO_ID(table), DR_DBG_PTR_TO_ID(table->dmn), - table->table_type, table->level); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,%d,%d\n", DR_DUMP_REC_TYPE_TABLE, + DR_DBG_PTR_TO_ID(table), DR_DBG_PTR_TO_ID(table->dmn), + table->table_type, table->level); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; if (rx->nic_dmn) { - ret = dr_dump_table_rx_tx(file, true, rx, + ret = dr_dump_table_rx_tx(file, buff, true, rx, DR_DBG_PTR_TO_ID(table)); if (ret < 0) return ret; } if (tx->nic_dmn) { - ret = dr_dump_table_rx_tx(file, false, tx, + ret = dr_dump_table_rx_tx(file, buff, false, tx, DR_DBG_PTR_TO_ID(table)); if (ret < 0) return ret; @@ -543,82 +880,123 @@ static int dr_dump_table_all(struct seq_file *file, struct mlx5dr_table *tbl) } static int -dr_dump_send_ring(struct seq_file *file, struct mlx5dr_send_ring *ring, +dr_dump_send_ring(struct seq_file *file, char *buff, + struct mlx5dr_send_ring *ring, const u64 domain_id) { - seq_printf(file, "%d,0x%llx,0x%llx,0x%x,0x%x\n", - DR_DUMP_REC_TYPE_DOMAIN_SEND_RING, DR_DBG_PTR_TO_ID(ring), - domain_id, ring->cq->mcq.cqn, ring->qp->qpn); + int ret; + + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%llx,0x%x,0x%x\n", + DR_DUMP_REC_TYPE_DOMAIN_SEND_RING, + DR_DBG_PTR_TO_ID(ring), domain_id, + ring->cq->mcq.cqn, ring->qp->qpn); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; + return 0; } static int dr_dump_domain_info_flex_parser(struct seq_file *file, + char *buff, const char *flex_parser_name, const u8 flex_parser_value, const u64 domain_id) { - seq_printf(file, "%d,0x%llx,%s,0x%x\n", - DR_DUMP_REC_TYPE_DOMAIN_INFO_FLEX_PARSER, domain_id, - flex_parser_name, flex_parser_value); + int ret; + + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,%s,0x%x\n", + DR_DUMP_REC_TYPE_DOMAIN_INFO_FLEX_PARSER, domain_id, + flex_parser_name, flex_parser_value); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; + return 0; } static int -dr_dump_domain_info_caps(struct seq_file *file, struct mlx5dr_cmd_caps *caps, +dr_dump_domain_info_caps(struct seq_file *file, char *buff, + struct mlx5dr_cmd_caps *caps, const u64 domain_id) { struct mlx5dr_cmd_vport_cap *vport_caps; unsigned long i, vports_num; + int ret; xa_for_each(&caps->vports.vports_caps_xa, vports_num, vport_caps) ; /* count the number of vports in xarray */ - seq_printf(file, "%d,0x%llx,0x%x,0x%llx,0x%llx,0x%x,%lu,%d\n", - DR_DUMP_REC_TYPE_DOMAIN_INFO_CAPS, domain_id, caps->gvmi, - caps->nic_rx_drop_address, caps->nic_tx_drop_address, - caps->flex_protocols, vports_num, caps->eswitch_manager); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,0x%x,0x%llx,0x%llx,0x%x,%lu,%d\n", + DR_DUMP_REC_TYPE_DOMAIN_INFO_CAPS, domain_id, caps->gvmi, + caps->nic_rx_drop_address, caps->nic_tx_drop_address, + caps->flex_protocols, vports_num, caps->eswitch_manager); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; xa_for_each(&caps->vports.vports_caps_xa, i, vport_caps) { vport_caps = xa_load(&caps->vports.vports_caps_xa, i); - seq_printf(file, "%d,0x%llx,%lu,0x%x,0x%llx,0x%llx\n", - DR_DUMP_REC_TYPE_DOMAIN_INFO_VPORT, domain_id, i, - vport_caps->vport_gvmi, vport_caps->icm_address_rx, - vport_caps->icm_address_tx); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,%lu,0x%x,0x%llx,0x%llx\n", + DR_DUMP_REC_TYPE_DOMAIN_INFO_VPORT, + domain_id, i, vport_caps->vport_gvmi, + vport_caps->icm_address_rx, + vport_caps->icm_address_tx); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; } return 0; } static int -dr_dump_domain_info(struct seq_file *file, struct mlx5dr_domain_info *info, +dr_dump_domain_info(struct seq_file *file, char *buff, + struct mlx5dr_domain_info *info, const u64 domain_id) { int ret; - ret = dr_dump_domain_info_caps(file, &info->caps, domain_id); + ret = dr_dump_domain_info_caps(file, buff, &info->caps, domain_id); if (ret < 0) return ret; - ret = dr_dump_domain_info_flex_parser(file, "icmp_dw0", + ret = dr_dump_domain_info_flex_parser(file, buff, "icmp_dw0", info->caps.flex_parser_id_icmp_dw0, domain_id); if (ret < 0) return ret; - ret = dr_dump_domain_info_flex_parser(file, "icmp_dw1", + ret = dr_dump_domain_info_flex_parser(file, buff, "icmp_dw1", info->caps.flex_parser_id_icmp_dw1, domain_id); if (ret < 0) return ret; - ret = dr_dump_domain_info_flex_parser(file, "icmpv6_dw0", + ret = dr_dump_domain_info_flex_parser(file, buff, "icmpv6_dw0", info->caps.flex_parser_id_icmpv6_dw0, domain_id); if (ret < 0) return ret; - ret = dr_dump_domain_info_flex_parser(file, "icmpv6_dw1", + ret = dr_dump_domain_info_flex_parser(file, buff, "icmpv6_dw1", info->caps.flex_parser_id_icmpv6_dw1, domain_id); if (ret < 0) @@ -627,31 +1005,39 @@ dr_dump_domain_info(struct seq_file *file, struct mlx5dr_domain_info *info, return 0; } -static int +static noinline_for_stack int dr_dump_domain(struct seq_file *file, struct mlx5dr_domain *dmn) { + char buff[MLX5DR_DEBUG_DUMP_BUFF_LENGTH]; u64 domain_id = DR_DBG_PTR_TO_ID(dmn); int ret; - seq_printf(file, "%d,0x%llx,%d,0%x,%d,%u.%u.%u,%s,%d,%u,%u,%u\n", - DR_DUMP_REC_TYPE_DOMAIN, - domain_id, dmn->type, dmn->info.caps.gvmi, - dmn->info.supp_sw_steering, - /* package version */ - LINUX_VERSION_MAJOR, LINUX_VERSION_PATCHLEVEL, - LINUX_VERSION_SUBLEVEL, - pci_name(dmn->mdev->pdev), - 0, /* domain flags */ - dmn->num_buddies[DR_ICM_TYPE_STE], - dmn->num_buddies[DR_ICM_TYPE_MODIFY_ACTION], - dmn->num_buddies[DR_ICM_TYPE_MODIFY_HDR_PTRN]); - - ret = dr_dump_domain_info(file, &dmn->info, domain_id); + ret = snprintf(buff, MLX5DR_DEBUG_DUMP_BUFF_LENGTH, + "%d,0x%llx,%d,0%x,%d,%u.%u.%u,%s,%d,%u,%u,%u\n", + DR_DUMP_REC_TYPE_DOMAIN, + domain_id, dmn->type, dmn->info.caps.gvmi, + dmn->info.supp_sw_steering, + /* package version */ + LINUX_VERSION_MAJOR, LINUX_VERSION_PATCHLEVEL, + LINUX_VERSION_SUBLEVEL, + pci_name(dmn->mdev->pdev), + 0, /* domain flags */ + dmn->num_buddies[DR_ICM_TYPE_STE], + dmn->num_buddies[DR_ICM_TYPE_MODIFY_ACTION], + dmn->num_buddies[DR_ICM_TYPE_MODIFY_HDR_PTRN]); + if (ret < 0) + return ret; + + ret = mlx5dr_dbg_dump_data_print(file, buff, ret); + if (ret) + return ret; + + ret = dr_dump_domain_info(file, buff, &dmn->info, domain_id); if (ret < 0) return ret; if (dmn->info.supp_sw_steering) { - ret = dr_dump_send_ring(file, dmn->send_ring, domain_id); + ret = dr_dump_send_ring(file, buff, dmn->send_ring, domain_id); if (ret < 0) return ret; } @@ -683,11 +1069,91 @@ unlock_mutex: return ret; } -static int dr_dump_show(struct seq_file *file, void *priv) +static void * +dr_dump_start(struct seq_file *file, loff_t *pos) { - return dr_dump_domain_all(file, file->private); + struct mlx5dr_domain *dmn = file->private; + struct mlx5dr_dbg_dump_data *dump_data; + + if (atomic_read(&dmn->dump_info.state) != MLX5DR_DEBUG_DUMP_STATE_FREE) { + mlx5_core_warn(dmn->mdev, "Dump already in progress\n"); + return ERR_PTR(-EBUSY); + } + + atomic_set(&dmn->dump_info.state, MLX5DR_DEBUG_DUMP_STATE_IN_PROGRESS); + dump_data = dmn->dump_info.dump_data; + + if (dump_data) { + return seq_list_start(&dump_data->buff_list, *pos); + } else if (*pos == 0) { + dump_data = mlx5dr_dbg_create_dump_data(); + if (!dump_data) + goto exit; + + dmn->dump_info.dump_data = dump_data; + if (dr_dump_domain_all(file, dmn)) { + mlx5dr_dbg_destroy_dump_data(dump_data); + dmn->dump_info.dump_data = NULL; + goto exit; + } + + return seq_list_start(&dump_data->buff_list, *pos); + } + +exit: + atomic_set(&dmn->dump_info.state, MLX5DR_DEBUG_DUMP_STATE_FREE); + return NULL; } -DEFINE_SHOW_ATTRIBUTE(dr_dump); + +static void * +dr_dump_next(struct seq_file *file, void *v, loff_t *pos) +{ + struct mlx5dr_domain *dmn = file->private; + struct mlx5dr_dbg_dump_data *dump_data; + + dump_data = dmn->dump_info.dump_data; + + return seq_list_next(v, &dump_data->buff_list, pos); +} + +static void +dr_dump_stop(struct seq_file *file, void *v) +{ + struct mlx5dr_domain *dmn = file->private; + struct mlx5dr_dbg_dump_data *dump_data; + + if (v && IS_ERR(v)) + return; + + if (!v) { + dump_data = dmn->dump_info.dump_data; + if (dump_data) { + mlx5dr_dbg_destroy_dump_data(dump_data); + dmn->dump_info.dump_data = NULL; + } + } + + atomic_set(&dmn->dump_info.state, MLX5DR_DEBUG_DUMP_STATE_FREE); +} + +static int +dr_dump_show(struct seq_file *file, void *v) +{ + struct mlx5dr_dbg_dump_buff *entry; + + entry = list_entry(v, struct mlx5dr_dbg_dump_buff, node); + seq_printf(file, "%s", entry->buff); + + return 0; +} + +static const struct seq_operations dr_dump_sops = { + .start = dr_dump_start, + .next = dr_dump_next, + .stop = dr_dump_stop, + .show = dr_dump_show, +}; +DEFINE_SEQ_ATTRIBUTE(dr_dump); void mlx5dr_dbg_init_dump(struct mlx5dr_domain *dmn) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.h index def6cf853..57c6b363b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.h @@ -1,10 +1,30 @@ /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ +#define MLX5DR_DEBUG_DUMP_BUFF_SIZE (64 * 1024 * 1024) +#define MLX5DR_DEBUG_DUMP_BUFF_LENGTH 512 + +enum { + MLX5DR_DEBUG_DUMP_STATE_FREE, + MLX5DR_DEBUG_DUMP_STATE_IN_PROGRESS, +}; + +struct mlx5dr_dbg_dump_buff { + char *buff; + u32 index; + struct list_head node; +}; + +struct mlx5dr_dbg_dump_data { + struct list_head buff_list; +}; + struct mlx5dr_dbg_dump_info { struct mutex dbg_mutex; /* protect dbg lists */ struct dentry *steering_debugfs; struct dentry *fdb_debugfs; + struct mlx5dr_dbg_dump_data *dump_data; + atomic_t state; }; void mlx5dr_dbg_init_dump(struct mlx5dr_domain *dmn); diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c index 253d7ad9b..8b63968bb 100644 --- a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c @@ -124,6 +124,41 @@ static void mlxbf_gige_get_pauseparam(struct net_device *netdev, pause->tx_pause = 1; } +static bool mlxbf_gige_llu_counters_enabled(struct mlxbf_gige *priv) +{ + u32 data; + + if (priv->hw_version == MLXBF_GIGE_VERSION_BF2) { + data = readl(priv->llu_base + MLXBF_GIGE_BF2_LLU_GENERAL_CONFIG); + if (data & MLXBF_GIGE_BF2_LLU_COUNTERS_EN) + return true; + } else { + data = readl(priv->llu_base + MLXBF_GIGE_BF3_LLU_GENERAL_CONFIG); + if (data & MLXBF_GIGE_BF3_LLU_COUNTERS_EN) + return true; + } + + return false; +} + +static void mlxbf_gige_get_pause_stats(struct net_device *netdev, + struct ethtool_pause_stats *pause_stats) +{ + struct mlxbf_gige *priv = netdev_priv(netdev); + u64 data_lo, data_hi; + + /* Read LLU counters to provide stats only if counters are enabled */ + if (mlxbf_gige_llu_counters_enabled(priv)) { + data_lo = readl(priv->llu_base + MLXBF_GIGE_TX_PAUSE_CNT_LO); + data_hi = readl(priv->llu_base + MLXBF_GIGE_TX_PAUSE_CNT_HI); + pause_stats->tx_pause_frames = (data_hi << 32) | data_lo; + + data_lo = readl(priv->llu_base + MLXBF_GIGE_RX_PAUSE_CNT_LO); + data_hi = readl(priv->llu_base + MLXBF_GIGE_RX_PAUSE_CNT_HI); + pause_stats->rx_pause_frames = (data_hi << 32) | data_lo; + } +} + const struct ethtool_ops mlxbf_gige_ethtool_ops = { .get_link = ethtool_op_get_link, .get_ringparam = mlxbf_gige_get_ringparam, @@ -134,6 +169,7 @@ const struct ethtool_ops mlxbf_gige_ethtool_ops = { .get_ethtool_stats = mlxbf_gige_get_ethtool_stats, .nway_reset = phy_ethtool_nway_reset, .get_pauseparam = mlxbf_gige_get_pauseparam, + .get_pause_stats = mlxbf_gige_get_pause_stats, .get_link_ksettings = phy_ethtool_get_link_ksettings, .set_link_ksettings = phy_ethtool_set_link_ksettings, }; diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h index cd0973229..98a8681c2 100644 --- a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h @@ -99,4 +99,34 @@ #define MLXBF_GIGE_100M_IPG_SIZE 119 #define MLXBF_GIGE_10M_IPG_SIZE 1199 +/* Offsets into OOB LLU block for pause frame counters */ +#define MLXBF_GIGE_BF2_TX_PAUSE_CNT_HI 0x33d8 +#define MLXBF_GIGE_BF2_TX_PAUSE_CNT_LO 0x33dc +#define MLXBF_GIGE_BF2_RX_PAUSE_CNT_HI 0x3210 +#define MLXBF_GIGE_BF2_RX_PAUSE_CNT_LO 0x3214 + +#define MLXBF_GIGE_BF3_TX_PAUSE_CNT_HI 0x3a88 +#define MLXBF_GIGE_BF3_TX_PAUSE_CNT_LO 0x3a8c +#define MLXBF_GIGE_BF3_RX_PAUSE_CNT_HI 0x38c0 +#define MLXBF_GIGE_BF3_RX_PAUSE_CNT_LO 0x38c4 + +#define MLXBF_GIGE_TX_PAUSE_CNT_HI ((priv->hw_version == MLXBF_GIGE_VERSION_BF2) ? \ + MLXBF_GIGE_BF2_TX_PAUSE_CNT_HI : \ + MLXBF_GIGE_BF3_TX_PAUSE_CNT_HI) +#define MLXBF_GIGE_TX_PAUSE_CNT_LO ((priv->hw_version == MLXBF_GIGE_VERSION_BF2) ? \ + MLXBF_GIGE_BF2_TX_PAUSE_CNT_LO : \ + MLXBF_GIGE_BF3_TX_PAUSE_CNT_LO) +#define MLXBF_GIGE_RX_PAUSE_CNT_HI ((priv->hw_version == MLXBF_GIGE_VERSION_BF2) ? \ + MLXBF_GIGE_BF2_RX_PAUSE_CNT_HI : \ + MLXBF_GIGE_BF3_RX_PAUSE_CNT_HI) +#define MLXBF_GIGE_RX_PAUSE_CNT_LO ((priv->hw_version == MLXBF_GIGE_VERSION_BF2) ? \ + MLXBF_GIGE_BF2_RX_PAUSE_CNT_LO : \ + MLXBF_GIGE_BF3_RX_PAUSE_CNT_LO) + +#define MLXBF_GIGE_BF2_LLU_GENERAL_CONFIG 0x2110 +#define MLXBF_GIGE_BF3_LLU_GENERAL_CONFIG 0x2030 + +#define MLXBF_GIGE_BF2_LLU_COUNTERS_EN BIT(0) +#define MLXBF_GIGE_BF3_LLU_COUNTERS_EN BIT(4) + #endif /* !defined(__MLXBF_GIGE_REGS_H__) */ diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c index f1b48d661..5c511e1a8 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c @@ -44,16 +44,19 @@ static const struct thermal_trip default_thermal_trips[] = { .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_ASIC_TEMP_NORM, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* In range - 40-100% PWM */ .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_ASIC_TEMP_HIGH, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* Warning */ .type = THERMAL_TRIP_HOT, .temperature = MLXSW_THERMAL_ASIC_TEMP_HOT, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, }; @@ -62,16 +65,19 @@ static const struct thermal_trip default_thermal_module_trips[] = { .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_MODULE_TEMP_NORM, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* In range - 40-100% PWM */ .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_MODULE_TEMP_HIGH, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* Warning */ .type = THERMAL_TRIP_HOT, .temperature = MLXSW_THERMAL_MODULE_TEMP_HOT, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, }; @@ -92,9 +98,6 @@ static const struct mlxsw_cooling_states default_cooling_states[] = { #define MLXSW_THERMAL_NUM_TRIPS ARRAY_SIZE(default_thermal_trips) -/* Make sure all trips are writable */ -#define MLXSW_THERMAL_TRIP_MASK (BIT(MLXSW_THERMAL_NUM_TRIPS) - 1) - struct mlxsw_thermal; struct mlxsw_thermal_module { @@ -420,7 +423,6 @@ mlxsw_thermal_module_tz_init(struct mlxsw_thermal_module *module_tz) module_tz->tzdev = thermal_zone_device_register_with_trips(tz_name, module_tz->trips, MLXSW_THERMAL_NUM_TRIPS, - MLXSW_THERMAL_TRIP_MASK, module_tz, &mlxsw_thermal_module_ops, &mlxsw_thermal_params, @@ -548,7 +550,6 @@ mlxsw_thermal_gearbox_tz_init(struct mlxsw_thermal_module *gearbox_tz) gearbox_tz->tzdev = thermal_zone_device_register_with_trips(tz_name, gearbox_tz->trips, MLXSW_THERMAL_NUM_TRIPS, - MLXSW_THERMAL_TRIP_MASK, gearbox_tz, &mlxsw_thermal_gearbox_ops, &mlxsw_thermal_params, 0, @@ -773,7 +774,6 @@ int mlxsw_thermal_init(struct mlxsw_core *core, thermal->tzdev = thermal_zone_device_register_with_trips("mlxsw", thermal->trips, MLXSW_THERMAL_NUM_TRIPS, - MLXSW_THERMAL_TRIP_MASK, thermal, &mlxsw_thermal_ops, &mlxsw_thermal_params, 0, diff --git a/drivers/net/ethernet/mellanox/mlxsw/minimal.c b/drivers/net/ethernet/mellanox/mlxsw/minimal.c index 6b98c3287..f0ceb196a 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/minimal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/minimal.c @@ -708,7 +708,6 @@ static const struct i2c_device_id mlxsw_m_i2c_id[] = { static struct i2c_driver mlxsw_m_i2c_driver = { .driver.name = "mlxsw_minimal", - .class = I2C_CLASS_HWMON, .id_table = mlxsw_m_i2c_id, }; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 5d3413636..bb642e9bb 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -176,13 +176,15 @@ MLXSW_ITEM32(tx, hdr, fid, 0x08, 16, 16); MLXSW_ITEM32(tx, hdr, type, 0x0C, 0, 4); int mlxsw_sp_flow_counter_get(struct mlxsw_sp *mlxsw_sp, - unsigned int counter_index, u64 *packets, - u64 *bytes) + unsigned int counter_index, bool clear, + u64 *packets, u64 *bytes) { + enum mlxsw_reg_mgpc_opcode op = clear ? MLXSW_REG_MGPC_OPCODE_CLEAR : + MLXSW_REG_MGPC_OPCODE_NOP; char mgpc_pl[MLXSW_REG_MGPC_LEN]; int err; - mlxsw_reg_mgpc_pack(mgpc_pl, counter_index, MLXSW_REG_MGPC_OPCODE_NOP, + mlxsw_reg_mgpc_pack(mgpc_pl, counter_index, op, MLXSW_REG_FLOW_COUNTER_SET_TYPE_PACKETS_BYTES); err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(mgpc), mgpc_pl); if (err) @@ -2695,23 +2697,18 @@ static void mlxsw_sp_traps_fini(struct mlxsw_sp *mlxsw_sp) static int mlxsw_sp_lag_pgt_init(struct mlxsw_sp *mlxsw_sp) { char sgcr_pl[MLXSW_REG_SGCR_LEN]; - u16 max_lag; int err; if (mlxsw_core_lag_mode(mlxsw_sp->core) != MLXSW_CMD_MBOX_CONFIG_PROFILE_LAG_MODE_SW) return 0; - err = mlxsw_core_max_lag(mlxsw_sp->core, &max_lag); - if (err) - return err; - /* In DDD mode, which we by default use, each LAG entry is 8 PGT * entries. The LAG table address needs to be 8-aligned, but that ought * to be the case, since the LAG table is allocated first. */ err = mlxsw_sp_pgt_mid_alloc_range(mlxsw_sp, &mlxsw_sp->lag_pgt_base, - max_lag * 8); + mlxsw_sp->max_lag * 8); if (err) return err; if (WARN_ON_ONCE(mlxsw_sp->lag_pgt_base % 8)) { @@ -2728,33 +2725,31 @@ static int mlxsw_sp_lag_pgt_init(struct mlxsw_sp *mlxsw_sp) err_mid_alloc_range: mlxsw_sp_pgt_mid_free_range(mlxsw_sp, mlxsw_sp->lag_pgt_base, - max_lag * 8); + mlxsw_sp->max_lag * 8); return err; } static void mlxsw_sp_lag_pgt_fini(struct mlxsw_sp *mlxsw_sp) { - u16 max_lag; - int err; - if (mlxsw_core_lag_mode(mlxsw_sp->core) != MLXSW_CMD_MBOX_CONFIG_PROFILE_LAG_MODE_SW) return; - err = mlxsw_core_max_lag(mlxsw_sp->core, &max_lag); - if (err) - return; - mlxsw_sp_pgt_mid_free_range(mlxsw_sp, mlxsw_sp->lag_pgt_base, - max_lag * 8); + mlxsw_sp->max_lag * 8); } #define MLXSW_SP_LAG_SEED_INIT 0xcafecafe +struct mlxsw_sp_lag { + struct net_device *dev; + refcount_t ref_count; + u16 lag_id; +}; + static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp) { char slcr_pl[MLXSW_REG_SLCR_LEN]; - u16 max_lag; u32 seed; int err; @@ -2773,7 +2768,7 @@ static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp) if (err) return err; - err = mlxsw_core_max_lag(mlxsw_sp->core, &max_lag); + err = mlxsw_core_max_lag(mlxsw_sp->core, &mlxsw_sp->max_lag); if (err) return err; @@ -2784,7 +2779,7 @@ static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp) if (err) return err; - mlxsw_sp->lags = kcalloc(max_lag, sizeof(struct mlxsw_sp_upper), + mlxsw_sp->lags = kcalloc(mlxsw_sp->max_lag, sizeof(struct mlxsw_sp_lag), GFP_KERNEL); if (!mlxsw_sp->lags) { err = -ENOMEM; @@ -4269,19 +4264,48 @@ mlxsw_sp_port_lag_uppers_cleanup(struct mlxsw_sp_port *mlxsw_sp_port, } } -static int mlxsw_sp_lag_create(struct mlxsw_sp *mlxsw_sp, u16 lag_id) +static struct mlxsw_sp_lag * +mlxsw_sp_lag_create(struct mlxsw_sp *mlxsw_sp, struct net_device *lag_dev, + struct netlink_ext_ack *extack) { char sldr_pl[MLXSW_REG_SLDR_LEN]; + struct mlxsw_sp_lag *lag; + u16 lag_id; + int i, err; + + for (i = 0; i < mlxsw_sp->max_lag; i++) { + if (!mlxsw_sp->lags[i].dev) + break; + } + + if (i == mlxsw_sp->max_lag) { + NL_SET_ERR_MSG_MOD(extack, + "Exceeded number of supported LAG devices"); + return ERR_PTR(-EBUSY); + } + lag_id = i; mlxsw_reg_sldr_lag_create_pack(sldr_pl, lag_id); - return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sldr), sldr_pl); + err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sldr), sldr_pl); + if (err) + return ERR_PTR(err); + + lag = &mlxsw_sp->lags[lag_id]; + lag->lag_id = lag_id; + lag->dev = lag_dev; + refcount_set(&lag->ref_count, 1); + + return lag; } -static int mlxsw_sp_lag_destroy(struct mlxsw_sp *mlxsw_sp, u16 lag_id) +static int +mlxsw_sp_lag_destroy(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_lag *lag) { char sldr_pl[MLXSW_REG_SLDR_LEN]; - mlxsw_reg_sldr_lag_destroy_pack(sldr_pl, lag_id); + lag->dev = NULL; + + mlxsw_reg_sldr_lag_destroy_pack(sldr_pl, lag->lag_id); return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sldr), sldr_pl); } @@ -4329,34 +4353,44 @@ static int mlxsw_sp_lag_col_port_disable(struct mlxsw_sp_port *mlxsw_sp_port, return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(slcor), slcor_pl); } -static int mlxsw_sp_lag_index_get(struct mlxsw_sp *mlxsw_sp, - struct net_device *lag_dev, - u16 *p_lag_id) +static struct mlxsw_sp_lag * +mlxsw_sp_lag_find(struct mlxsw_sp *mlxsw_sp, struct net_device *lag_dev) { - struct mlxsw_sp_upper *lag; - int free_lag_id = -1; - u16 max_lag; - int err, i; + int i; - err = mlxsw_core_max_lag(mlxsw_sp->core, &max_lag); - if (err) - return err; + for (i = 0; i < mlxsw_sp->max_lag; i++) { + if (!mlxsw_sp->lags[i].dev) + continue; - for (i = 0; i < max_lag; i++) { - lag = mlxsw_sp_lag_get(mlxsw_sp, i); - if (lag->ref_count) { - if (lag->dev == lag_dev) { - *p_lag_id = i; - return 0; - } - } else if (free_lag_id < 0) { - free_lag_id = i; - } + if (mlxsw_sp->lags[i].dev == lag_dev) + return &mlxsw_sp->lags[i]; } - if (free_lag_id < 0) - return -EBUSY; - *p_lag_id = free_lag_id; - return 0; + + return NULL; +} + +static struct mlxsw_sp_lag * +mlxsw_sp_lag_get(struct mlxsw_sp *mlxsw_sp, struct net_device *lag_dev, + struct netlink_ext_ack *extack) +{ + struct mlxsw_sp_lag *lag; + + lag = mlxsw_sp_lag_find(mlxsw_sp, lag_dev); + if (lag) { + refcount_inc(&lag->ref_count); + return lag; + } + + return mlxsw_sp_lag_create(mlxsw_sp, lag_dev, extack); +} + +static void +mlxsw_sp_lag_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_lag *lag) +{ + if (!refcount_dec_and_test(&lag->ref_count)) + return; + + mlxsw_sp_lag_destroy(mlxsw_sp, lag); } static bool @@ -4365,12 +4399,6 @@ mlxsw_sp_master_lag_check(struct mlxsw_sp *mlxsw_sp, struct netdev_lag_upper_info *lag_upper_info, struct netlink_ext_ack *extack) { - u16 lag_id; - - if (mlxsw_sp_lag_index_get(mlxsw_sp, lag_dev, &lag_id) != 0) { - NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported LAG devices"); - return false; - } if (lag_upper_info->tx_type != NETDEV_LAG_TX_TYPE_HASH) { NL_SET_ERR_MSG_MOD(extack, "LAG device using unsupported Tx type"); return false; @@ -4482,22 +4510,16 @@ static int mlxsw_sp_port_lag_join(struct mlxsw_sp_port *mlxsw_sp_port, struct netlink_ext_ack *extack) { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - struct mlxsw_sp_upper *lag; + struct mlxsw_sp_lag *lag; u16 lag_id; u8 port_index; int err; - err = mlxsw_sp_lag_index_get(mlxsw_sp, lag_dev, &lag_id); - if (err) - return err; - lag = mlxsw_sp_lag_get(mlxsw_sp, lag_id); - if (!lag->ref_count) { - err = mlxsw_sp_lag_create(mlxsw_sp, lag_id); - if (err) - return err; - lag->dev = lag_dev; - } + lag = mlxsw_sp_lag_get(mlxsw_sp, lag_dev, extack); + if (IS_ERR(lag)) + return PTR_ERR(lag); + lag_id = lag->lag_id; err = mlxsw_sp_port_lag_index_get(mlxsw_sp, lag_id, &port_index); if (err) return err; @@ -4515,7 +4537,6 @@ static int mlxsw_sp_port_lag_join(struct mlxsw_sp_port *mlxsw_sp_port, mlxsw_sp_port->local_port); mlxsw_sp_port->lag_id = lag_id; mlxsw_sp_port->lagged = 1; - lag->ref_count++; err = mlxsw_sp_fid_port_join_lag(mlxsw_sp_port); if (err) @@ -4542,7 +4563,6 @@ err_replay: err_router_join: mlxsw_sp_fid_port_leave_lag(mlxsw_sp_port); err_fid_port_join_lag: - lag->ref_count--; mlxsw_sp_port->lagged = 0; mlxsw_core_lag_mapping_clear(mlxsw_sp->core, lag_id, mlxsw_sp_port->local_port); @@ -4550,8 +4570,7 @@ err_fid_port_join_lag: err_col_port_add: mlxsw_sp_lag_uppers_bridge_leave(mlxsw_sp_port, lag_dev); err_lag_uppers_bridge_join: - if (!lag->ref_count) - mlxsw_sp_lag_destroy(mlxsw_sp, lag_id); + mlxsw_sp_lag_put(mlxsw_sp, lag); return err; } @@ -4560,12 +4579,11 @@ static void mlxsw_sp_port_lag_leave(struct mlxsw_sp_port *mlxsw_sp_port, { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; u16 lag_id = mlxsw_sp_port->lag_id; - struct mlxsw_sp_upper *lag; + struct mlxsw_sp_lag *lag; if (!mlxsw_sp_port->lagged) return; - lag = mlxsw_sp_lag_get(mlxsw_sp, lag_id); - WARN_ON(lag->ref_count == 0); + lag = &mlxsw_sp->lags[lag_id]; mlxsw_sp_lag_col_port_remove(mlxsw_sp_port, lag_id); @@ -4579,13 +4597,11 @@ static void mlxsw_sp_port_lag_leave(struct mlxsw_sp_port *mlxsw_sp_port, mlxsw_sp_fid_port_leave_lag(mlxsw_sp_port); - if (lag->ref_count == 1) - mlxsw_sp_lag_destroy(mlxsw_sp, lag_id); + mlxsw_sp_lag_put(mlxsw_sp, lag); mlxsw_core_lag_mapping_clear(mlxsw_sp->core, lag_id, mlxsw_sp_port->local_port); mlxsw_sp_port->lagged = 0; - lag->ref_count--; /* Make sure untagged frames are allowed to ingress */ mlxsw_sp_port_pvid_set(mlxsw_sp_port, MLXSW_SP_DEFAULT_VID, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h index a0c9775fa..3beb5d084 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h @@ -78,11 +78,6 @@ struct mlxsw_sp_span_entry; enum mlxsw_sp_l3proto; union mlxsw_sp_l3addr; -struct mlxsw_sp_upper { - struct net_device *dev; - unsigned int ref_count; -}; - enum mlxsw_sp_rif_type { MLXSW_SP_RIF_TYPE_SUBPORT, MLXSW_SP_RIF_TYPE_VLAN, @@ -136,6 +131,7 @@ struct mlxsw_sp_span_ops; struct mlxsw_sp_qdisc_state; struct mlxsw_sp_mall_entry; struct mlxsw_sp_pgt; +struct mlxsw_sp_lag; struct mlxsw_sp_port_mapping { u8 module; @@ -164,7 +160,8 @@ struct mlxsw_sp { const struct mlxsw_bus_info *bus_info; unsigned char base_mac[ETH_ALEN]; const unsigned char *mac_mask; - struct mlxsw_sp_upper *lags; + struct mlxsw_sp_lag *lags; + u16 max_lag; struct mlxsw_sp_port_mapping *port_mapping; struct mlxsw_sp_port_mapping_events port_mapping_events; struct rhashtable sample_trigger_ht; @@ -257,12 +254,6 @@ struct mlxsw_sp_fid_core_ops { void (*fini)(struct mlxsw_sp *mlxsw_sp); }; -static inline struct mlxsw_sp_upper * -mlxsw_sp_lag_get(struct mlxsw_sp *mlxsw_sp, u16 lag_id) -{ - return &mlxsw_sp->lags[lag_id]; -} - struct mlxsw_sp_port_pcpu_stats { u64 rx_packets; u64 rx_bytes; @@ -715,8 +706,8 @@ int mlxsw_sp_port_kill_vid(struct net_device *dev, int mlxsw_sp_port_vlan_set(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid_begin, u16 vid_end, bool is_member, bool untagged); int mlxsw_sp_flow_counter_get(struct mlxsw_sp *mlxsw_sp, - unsigned int counter_index, u64 *packets, - u64 *bytes); + unsigned int counter_index, bool clear, + u64 *packets, u64 *bytes); int mlxsw_sp_flow_counter_alloc(struct mlxsw_sp *mlxsw_sp, unsigned int *p_counter_index); void mlxsw_sp_flow_counter_free(struct mlxsw_sp *mlxsw_sp, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c index b01b000bc..3e70cee4d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c @@ -1024,7 +1024,7 @@ int mlxsw_sp_acl_rule_get_stats(struct mlxsw_sp *mlxsw_sp, rulei = mlxsw_sp_acl_rule_rulei(rule); if (rulei->counter_valid) { err = mlxsw_sp_flow_counter_get(mlxsw_sp, rulei->counter_index, - ¤t_packets, + false, ¤t_packets, ¤t_bytes); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c index c8a356acc..ca80af064 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c @@ -1181,9 +1181,11 @@ static int mlxsw_sp_dpipe_table_adj_counters_update(void *priv, bool enable) char ratr_pl[MLXSW_REG_RATR_LEN]; struct mlxsw_sp *mlxsw_sp = priv; struct mlxsw_sp_nexthop *nh; + unsigned int n_done = 0; u32 adj_hash_index = 0; u32 adj_index = 0; u32 adj_size = 0; + int err; mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router) { if (!mlxsw_sp_nexthop_is_forward(nh) || @@ -1192,15 +1194,27 @@ static int mlxsw_sp_dpipe_table_adj_counters_update(void *priv, bool enable) mlxsw_sp_nexthop_indexes(nh, &adj_index, &adj_size, &adj_hash_index); - if (enable) - mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); - else - mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); + if (enable) { + err = mlxsw_sp_nexthop_counter_enable(mlxsw_sp, nh); + if (err) + goto err_counter_enable; + } else { + mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); + } mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + adj_hash_index, nh, true, ratr_pl); + n_done++; } return 0; + +err_counter_enable: + mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router) { + if (!n_done--) + break; + mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); + } + return err; } static u64 diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr_tcam.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr_tcam.c index 221aa6a47..01d81ae36 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr_tcam.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr_tcam.c @@ -361,7 +361,7 @@ static int mlxsw_sp_mr_tcam_route_stats(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_mr_tcam_route *route = route_priv; return mlxsw_sp_flow_counter_get(mlxsw_sp, route->counter_index, - packets, bytes); + false, packets, bytes); } static int diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index 87617df69..40ba314fb 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -19,6 +19,7 @@ #include <linux/net_namespace.h> #include <linux/mutex.h> #include <linux/genalloc.h> +#include <linux/xarray.h> #include <net/netevent.h> #include <net/neighbour.h> #include <net/arp.h> @@ -2251,7 +2252,7 @@ int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, return -EINVAL; return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, - p_counter, NULL); + false, p_counter, NULL); } static struct mlxsw_sp_neigh_entry * @@ -3049,6 +3050,8 @@ struct mlxsw_sp_nexthop_key { struct fib_nh *fib_nh; }; +struct mlxsw_sp_nexthop_counter; + struct mlxsw_sp_nexthop { struct list_head neigh_list_node; /* member of neigh entry list */ struct list_head crif_list_node; @@ -3080,8 +3083,8 @@ struct mlxsw_sp_nexthop { struct mlxsw_sp_neigh_entry *neigh_entry; struct mlxsw_sp_ipip_entry *ipip_entry; }; - unsigned int counter_index; - bool counter_valid; + struct mlxsw_sp_nexthop_counter *counter; + u32 id; /* NH ID for members of a NH object group. */ }; static struct net_device * @@ -3106,8 +3109,10 @@ struct mlxsw_sp_nexthop_group_info { int sum_norm_weight; u8 adj_index_valid:1, gateway:1, /* routes using the group use a gateway */ - is_resilient:1; + is_resilient:1, + hw_stats:1; struct list_head list; /* member in nh_res_grp_list */ + struct xarray nexthop_counters; struct mlxsw_sp_nexthop nexthops[] __counted_by(count); }; @@ -3151,39 +3156,148 @@ struct mlxsw_sp_nexthop_group { bool can_destroy; }; -void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, +struct mlxsw_sp_nexthop_counter { + unsigned int counter_index; + refcount_t ref_count; +}; + +static struct mlxsw_sp_nexthop_counter * +mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp) +{ + struct mlxsw_sp_nexthop_counter *nhct; + int err; + + nhct = kzalloc(sizeof(*nhct), GFP_KERNEL); + if (!nhct) + return ERR_PTR(-ENOMEM); + + err = mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nhct->counter_index); + if (err) + goto err_counter_alloc; + + refcount_set(&nhct->ref_count, 1); + return nhct; + +err_counter_alloc: + kfree(nhct); + return ERR_PTR(err); +} + +static void +mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_nexthop_counter *nhct) +{ + mlxsw_sp_flow_counter_free(mlxsw_sp, nhct->counter_index); + kfree(nhct); +} + +static struct mlxsw_sp_nexthop_counter * +mlxsw_sp_nexthop_sh_counter_get(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_nexthop *nh) +{ + struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; + struct mlxsw_sp_nexthop_counter *nhct; + void *ptr; + int err; + + nhct = xa_load(&nh_grp->nhgi->nexthop_counters, nh->id); + if (nhct) { + refcount_inc(&nhct->ref_count); + return nhct; + } + + nhct = mlxsw_sp_nexthop_counter_alloc(mlxsw_sp); + if (IS_ERR(nhct)) + return nhct; + + ptr = xa_store(&nh_grp->nhgi->nexthop_counters, nh->id, nhct, + GFP_KERNEL); + if (IS_ERR(ptr)) { + err = PTR_ERR(ptr); + goto err_store; + } + + return nhct; + +err_store: + mlxsw_sp_nexthop_counter_free(mlxsw_sp, nhct); + return ERR_PTR(err); +} + +static void mlxsw_sp_nexthop_sh_counter_put(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_nexthop *nh) +{ + struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; + struct mlxsw_sp_nexthop_counter *nhct; + + nhct = xa_load(&nh_grp->nhgi->nexthop_counters, nh->id); + if (WARN_ON(!nhct)) + return; + + if (!refcount_dec_and_test(&nhct->ref_count)) + return; + + xa_erase(&nh_grp->nhgi->nexthop_counters, nh->id); + mlxsw_sp_nexthop_counter_free(mlxsw_sp, nhct); +} + +int mlxsw_sp_nexthop_counter_enable(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_nexthop *nh) { + const char *table_adj = MLXSW_SP_DPIPE_TABLE_NAME_ADJ; + struct mlxsw_sp_nexthop_counter *nhct; struct devlink *devlink; + bool dpipe_stats; + + if (nh->counter) + return 0; devlink = priv_to_devlink(mlxsw_sp->core); - if (!devlink_dpipe_table_counter_enabled(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) - return; + dpipe_stats = devlink_dpipe_table_counter_enabled(devlink, table_adj); + if (!(nh->nhgi->hw_stats || dpipe_stats)) + return 0; - if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) - return; + if (nh->id) + nhct = mlxsw_sp_nexthop_sh_counter_get(mlxsw_sp, nh); + else + nhct = mlxsw_sp_nexthop_counter_alloc(mlxsw_sp); + if (IS_ERR(nhct)) + return PTR_ERR(nhct); - nh->counter_valid = true; + nh->counter = nhct; + return 0; } -void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_nexthop *nh) +void mlxsw_sp_nexthop_counter_disable(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_nexthop *nh) { - if (!nh->counter_valid) + if (!nh->counter) return; - mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); - nh->counter_valid = false; + + if (nh->id) + mlxsw_sp_nexthop_sh_counter_put(mlxsw_sp, nh); + else + mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh->counter); + nh->counter = NULL; +} + +static int mlxsw_sp_nexthop_counter_update(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_nexthop *nh) +{ + if (nh->nhgi->hw_stats) + return mlxsw_sp_nexthop_counter_enable(mlxsw_sp, nh); + mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); + return 0; } int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_nexthop *nh, u64 *p_counter) { - if (!nh->counter_valid) + if (!nh->counter) return -EINVAL; - return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, - p_counter, NULL); + return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter->counter_index, + true, p_counter, NULL); } struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, @@ -3656,8 +3770,9 @@ static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, WARN_ON_ONCE(1); return -EINVAL; } - if (nh->counter_valid) - mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); + if (nh->counter) + mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter->counter_index, + true); else mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); @@ -3744,6 +3859,7 @@ mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, nh = &nhgi->nexthops[i]; if (!nh->should_offload) { + mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); nh->offloaded = 0; continue; } @@ -3751,6 +3867,10 @@ mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, if (nh->update || reallocate) { int err = 0; + err = mlxsw_sp_nexthop_counter_update(mlxsw_sp, nh); + if (err) + return err; + err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, true, ratr_pl); if (err) @@ -4507,7 +4627,10 @@ static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, if (err) return err; - mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); + err = mlxsw_sp_nexthop_counter_enable(mlxsw_sp, nh); + if (err) + goto err_counter_enable; + list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); if (!dev) @@ -4531,7 +4654,8 @@ static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, err_nexthop_neigh_init: list_del(&nh->router_list_node); - mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); + mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); +err_counter_enable: mlxsw_sp_nexthop_remove(mlxsw_sp, nh); return err; } @@ -4541,7 +4665,7 @@ static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, { mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); list_del(&nh->router_list_node); - mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); + mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); mlxsw_sp_nexthop_remove(mlxsw_sp, nh); } @@ -5006,9 +5130,9 @@ mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, break; } - mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); nh->ifindex = dev->ifindex; + nh->id = nh_obj->id; err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); if (err) @@ -5030,7 +5154,6 @@ mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, err_type_init: list_del(&nh->router_list_node); - mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); return err; } @@ -5041,7 +5164,7 @@ static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); list_del(&nh->router_list_node); - mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); + mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); nh->should_offload = 0; } @@ -5053,6 +5176,7 @@ mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_nexthop_group_info *nhgi; struct mlxsw_sp_nexthop *nh; bool is_resilient = false; + bool hw_stats = false; unsigned int nhs; int err, i; @@ -5062,9 +5186,11 @@ mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, break; case NH_NOTIFIER_INFO_TYPE_GRP: nhs = info->nh_grp->num_nh; + hw_stats = info->nh_grp->hw_stats; break; case NH_NOTIFIER_INFO_TYPE_RES_TABLE: nhs = info->nh_res_table->num_nh_buckets; + hw_stats = info->nh_res_table->hw_stats; is_resilient = true; break; default: @@ -5079,6 +5205,10 @@ mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); nhgi->is_resilient = is_resilient; nhgi->count = nhs; + nhgi->hw_stats = hw_stats; + + xa_init_flags(&nhgi->nexthop_counters, XA_FLAGS_ALLOC1); + for (i = 0; i < nhgi->count; i++) { struct nh_notifier_single_info *nh_obj; int weight; @@ -5161,6 +5291,8 @@ mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, } mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); WARN_ON_ONCE(nhgi->adj_index_valid); + WARN_ON(!xa_empty(&nhgi->nexthop_counters)); + xa_destroy(&nhgi->nexthop_counters); kfree(nhgi); } @@ -5300,6 +5432,43 @@ err_out: return err; } +static int mlxsw_sp_nexthop_obj_res_group_pre(struct mlxsw_sp *mlxsw_sp, + struct nh_notifier_info *info) +{ + struct nh_notifier_grp_info *grp_info = info->nh_grp; + struct mlxsw_sp_nexthop_group_info *nhgi; + struct mlxsw_sp_nexthop_group *nh_grp; + int err; + int i; + + nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); + if (!nh_grp) + return 0; + nhgi = nh_grp->nhgi; + + if (nhgi->hw_stats == grp_info->hw_stats) + return 0; + + nhgi->hw_stats = grp_info->hw_stats; + + for (i = 0; i < nhgi->count; i++) { + struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; + + if (nh->offloaded) + nh->update = 1; + } + + err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); + if (err) + goto err_group_refresh; + + return 0; + +err_group_refresh: + nhgi->hw_stats = !grp_info->hw_stats; + return err; +} + static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, struct nh_notifier_info *info) { @@ -5476,6 +5645,79 @@ err_nexthop_obj_init: return err; } +static void +mlxsw_sp_nexthop_obj_mp_hw_stats_get(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_nexthop_group_info *nhgi, + struct nh_notifier_grp_hw_stats_info *info) +{ + int nhi; + + for (nhi = 0; nhi < info->num_nh; nhi++) { + struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[nhi]; + u64 packets; + int err; + + err = mlxsw_sp_nexthop_counter_get(mlxsw_sp, nh, &packets); + if (err) + continue; + + nh_grp_hw_stats_report_delta(info, nhi, packets); + } +} + +static void +mlxsw_sp_nexthop_obj_res_hw_stats_get(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_nexthop_group_info *nhgi, + struct nh_notifier_grp_hw_stats_info *info) +{ + int nhi = -1; + int bucket; + + for (bucket = 0; bucket < nhgi->count; bucket++) { + struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[bucket]; + u64 packets; + int err; + + if (nhi == -1 || info->stats[nhi].id != nh->id) { + for (nhi = 0; nhi < info->num_nh; nhi++) + if (info->stats[nhi].id == nh->id) + break; + if (WARN_ON_ONCE(nhi == info->num_nh)) { + nhi = -1; + continue; + } + } + + err = mlxsw_sp_nexthop_counter_get(mlxsw_sp, nh, &packets); + if (err) + continue; + + nh_grp_hw_stats_report_delta(info, nhi, packets); + } +} + +static void mlxsw_sp_nexthop_obj_hw_stats_get(struct mlxsw_sp *mlxsw_sp, + struct nh_notifier_info *info) +{ + struct mlxsw_sp_nexthop_group_info *nhgi; + struct mlxsw_sp_nexthop_group *nh_grp; + + if (info->type != NH_NOTIFIER_INFO_TYPE_GRP_HW_STATS) + return; + + nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); + if (!nh_grp) + return; + nhgi = nh_grp->nhgi; + + if (nhgi->is_resilient) + mlxsw_sp_nexthop_obj_res_hw_stats_get(mlxsw_sp, nhgi, + info->nh_grp_hw_stats); + else + mlxsw_sp_nexthop_obj_mp_hw_stats_get(mlxsw_sp, nhgi, + info->nh_grp_hw_stats); +} + static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, unsigned long event, void *ptr) { @@ -5491,6 +5733,10 @@ static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, mutex_lock(&router->lock); switch (event) { + case NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE: + err = mlxsw_sp_nexthop_obj_res_group_pre(router->mlxsw_sp, + info); + break; case NEXTHOP_EVENT_REPLACE: err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); break; @@ -5501,6 +5747,9 @@ static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp, info); break; + case NEXTHOP_EVENT_HW_STATS_REPORT_DELTA: + mlxsw_sp_nexthop_obj_hw_stats_get(router->mlxsw_sp, info); + break; default: break; } @@ -6734,7 +6983,10 @@ static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, #if IS_ENABLED(CONFIG_IPV6) nh->neigh_tbl = &nd_tbl; #endif - mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); + + err = mlxsw_sp_nexthop_counter_enable(mlxsw_sp, nh); + if (err) + return err; list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); @@ -6750,7 +7002,7 @@ static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, err_nexthop_type_init: list_del(&nh->router_list_node); - mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); + mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); return err; } @@ -6759,7 +7011,7 @@ static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, { mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); list_del(&nh->router_list_node); - mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); + mlxsw_sp_nexthop_counter_disable(mlxsw_sp, nh); } static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h index ed3b628ca..0432c7cc6 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h @@ -156,10 +156,10 @@ int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, struct mlxsw_sp_nexthop *nh, bool force, char *ratr_pl); -void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, +int mlxsw_sp_nexthop_counter_enable(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_nexthop *nh); -void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_nexthop *nh); +void mlxsw_sp_nexthop_counter_disable(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_nexthop *nh); static inline bool mlxsw_sp_l3addr_eq(const union mlxsw_sp_l3addr *addr1, const union mlxsw_sp_l3addr *addr2) diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c index af50ff9e5..ce49c9514 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c @@ -539,7 +539,7 @@ mlxsw_sp_span_gretap6_route(const struct net_device *to_dev, if (!dst || dst->error) goto out; - rt6 = container_of(dst, struct rt6_info, dst); + rt6 = dst_rt6_info(dst); dev = dst->dev; *saddrp = fl6.saddr; |