/* SPDX-License-Identifier: BSD-3-Clause * Copyright(C) 2019 Marvell International Ltd. */ #include #include "otx2_ethdev.h" #include "otx2_tm.h" /* Use last LVL_CNT nodes as default nodes */ #define NIX_DEFAULT_NODE_ID_START (RTE_TM_NODE_ID_NULL - NIX_TXSCH_LVL_CNT) enum otx2_tm_node_level { OTX2_TM_LVL_ROOT = 0, OTX2_TM_LVL_SCH1, OTX2_TM_LVL_SCH2, OTX2_TM_LVL_SCH3, OTX2_TM_LVL_SCH4, OTX2_TM_LVL_QUEUE, OTX2_TM_LVL_MAX, }; static inline uint64_t shaper2regval(struct shaper_params *shaper) { return (shaper->burst_exponent << 37) | (shaper->burst_mantissa << 29) | (shaper->div_exp << 13) | (shaper->exponent << 9) | (shaper->mantissa << 1); } int otx2_nix_get_link(struct otx2_eth_dev *dev) { int link = 13 /* SDP */; uint16_t lmac_chan; uint16_t map; lmac_chan = dev->tx_chan_base; /* CGX lmac link */ if (lmac_chan >= 0x800) { map = lmac_chan & 0x7FF; link = 4 * ((map >> 8) & 0xF) + ((map >> 4) & 0xF); } else if (lmac_chan < 0x700) { /* LBK channel */ link = 12; } return link; } static uint8_t nix_get_relchan(struct otx2_eth_dev *dev) { return dev->tx_chan_base & 0xff; } static bool nix_tm_have_tl1_access(struct otx2_eth_dev *dev) { bool is_lbk = otx2_dev_is_lbk(dev); return otx2_dev_is_pf(dev) && !otx2_dev_is_Ax(dev) && !is_lbk; } static bool nix_tm_is_leaf(struct otx2_eth_dev *dev, int lvl) { if (nix_tm_have_tl1_access(dev)) return (lvl == OTX2_TM_LVL_QUEUE); return (lvl == OTX2_TM_LVL_SCH4); } static int find_prio_anchor(struct otx2_eth_dev *dev, uint32_t node_id) { struct otx2_nix_tm_node *child_node; TAILQ_FOREACH(child_node, &dev->node_list, node) { if (!child_node->parent) continue; if (!(child_node->parent->id == node_id)) continue; if (child_node->priority == child_node->parent->rr_prio) continue; return child_node->hw_id - child_node->priority; } return 0; } static struct otx2_nix_tm_shaper_profile * nix_tm_shaper_profile_search(struct otx2_eth_dev *dev, uint32_t shaper_id) { struct otx2_nix_tm_shaper_profile *tm_shaper_profile; TAILQ_FOREACH(tm_shaper_profile, &dev->shaper_profile_list, shaper) { if (tm_shaper_profile->shaper_profile_id == shaper_id) return tm_shaper_profile; } return NULL; } static inline uint64_t shaper_rate_to_nix(uint64_t value, uint64_t *exponent_p, uint64_t *mantissa_p, uint64_t *div_exp_p) { uint64_t div_exp, exponent, mantissa; /* Boundary checks */ if (value < MIN_SHAPER_RATE || value > MAX_SHAPER_RATE) return 0; if (value <= SHAPER_RATE(0, 0, 0)) { /* Calculate rate div_exp and mantissa using * the following formula: * * value = (2E6 * (256 + mantissa) * / ((1 << div_exp) * 256)) */ div_exp = 0; exponent = 0; mantissa = MAX_RATE_MANTISSA; while (value < (NIX_SHAPER_RATE_CONST / (1 << div_exp))) div_exp += 1; while (value < ((NIX_SHAPER_RATE_CONST * (256 + mantissa)) / ((1 << div_exp) * 256))) mantissa -= 1; } else { /* Calculate rate exponent and mantissa using * the following formula: * * value = (2E6 * ((256 + mantissa) << exponent)) / 256 * */ div_exp = 0; exponent = MAX_RATE_EXPONENT; mantissa = MAX_RATE_MANTISSA; while (value < (NIX_SHAPER_RATE_CONST * (1 << exponent))) exponent -= 1; while (value < ((NIX_SHAPER_RATE_CONST * ((256 + mantissa) << exponent)) / 256)) mantissa -= 1; } if (div_exp > MAX_RATE_DIV_EXP || exponent > MAX_RATE_EXPONENT || mantissa > MAX_RATE_MANTISSA) return 0; if (div_exp_p) *div_exp_p = div_exp; if (exponent_p) *exponent_p = exponent; if (mantissa_p) *mantissa_p = mantissa; /* Calculate real rate value */ return SHAPER_RATE(exponent, mantissa, div_exp); } static inline uint64_t shaper_burst_to_nix(uint64_t value, uint64_t *exponent_p, uint64_t *mantissa_p) { uint64_t exponent, mantissa; if (value < MIN_SHAPER_BURST || value > MAX_SHAPER_BURST) return 0; /* Calculate burst exponent and mantissa using * the following formula: * * value = (((256 + mantissa) << (exponent + 1) / 256) * */ exponent = MAX_BURST_EXPONENT; mantissa = MAX_BURST_MANTISSA; while (value < (1ull << (exponent + 1))) exponent -= 1; while (value < ((256 + mantissa) << (exponent + 1)) / 256) mantissa -= 1; if (exponent > MAX_BURST_EXPONENT || mantissa > MAX_BURST_MANTISSA) return 0; if (exponent_p) *exponent_p = exponent; if (mantissa_p) *mantissa_p = mantissa; return SHAPER_BURST(exponent, mantissa); } static void shaper_config_to_nix(struct otx2_nix_tm_shaper_profile *profile, struct shaper_params *cir, struct shaper_params *pir) { struct rte_tm_shaper_params *param = &profile->params; if (!profile) return; /* Calculate CIR exponent and mantissa */ if (param->committed.rate) cir->rate = shaper_rate_to_nix(param->committed.rate, &cir->exponent, &cir->mantissa, &cir->div_exp); /* Calculate PIR exponent and mantissa */ if (param->peak.rate) pir->rate = shaper_rate_to_nix(param->peak.rate, &pir->exponent, &pir->mantissa, &pir->div_exp); /* Calculate CIR burst exponent and mantissa */ if (param->committed.size) cir->burst = shaper_burst_to_nix(param->committed.size, &cir->burst_exponent, &cir->burst_mantissa); /* Calculate PIR burst exponent and mantissa */ if (param->peak.size) pir->burst = shaper_burst_to_nix(param->peak.size, &pir->burst_exponent, &pir->burst_mantissa); } static void shaper_default_red_algo(struct otx2_eth_dev *dev, struct otx2_nix_tm_node *tm_node, struct otx2_nix_tm_shaper_profile *profile) { struct shaper_params cir, pir; /* C0 doesn't support STALL when both PIR & CIR are enabled */ if (profile && otx2_dev_is_96xx_Cx(dev)) { memset(&cir, 0, sizeof(cir)); memset(&pir, 0, sizeof(pir)); shaper_config_to_nix(profile, &cir, &pir); if (pir.rate && cir.rate) { tm_node->red_algo = NIX_REDALG_DISCARD; tm_node->flags |= NIX_TM_NODE_RED_DISCARD; return; } } tm_node->red_algo = NIX_REDALG_STD; tm_node->flags &= ~NIX_TM_NODE_RED_DISCARD; } static int populate_tm_tl1_default(struct otx2_eth_dev *dev, uint32_t schq) { struct otx2_mbox *mbox = dev->mbox; struct nix_txschq_config *req; /* * Default config for TL1. * For VF this is always ignored. */ req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox); req->lvl = NIX_TXSCH_LVL_TL1; /* Set DWRR quantum */ req->reg[0] = NIX_AF_TL1X_SCHEDULE(schq); req->regval[0] = TXSCH_TL1_DFLT_RR_QTM; req->num_regs++; req->reg[1] = NIX_AF_TL1X_TOPOLOGY(schq); req->regval[1] = (TXSCH_TL1_DFLT_RR_PRIO << 1); req->num_regs++; req->reg[2] = NIX_AF_TL1X_CIR(schq); req->regval[2] = 0; req->num_regs++; return otx2_mbox_process(mbox); } static uint8_t prepare_tm_sched_reg(struct otx2_eth_dev *dev, struct otx2_nix_tm_node *tm_node, volatile uint64_t *reg, volatile uint64_t *regval) { uint64_t strict_prio = tm_node->priority; uint32_t hw_lvl = tm_node->hw_lvl; uint32_t schq = tm_node->hw_id; uint64_t rr_quantum; uint8_t k = 0; rr_quantum = NIX_TM_WEIGHT_TO_RR_QUANTUM(tm_node->weight); /* For children to root, strict prio is default if either * device root is TL2 or TL1 Static Priority is disabled. */ if (hw_lvl == NIX_TXSCH_LVL_TL2 && (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 || dev->tm_flags & NIX_TM_TL1_NO_SP)) strict_prio = TXSCH_TL1_DFLT_RR_PRIO; otx2_tm_dbg("Schedule config node %s(%u) lvl %u id %u, " "prio 0x%" PRIx64 ", rr_quantum 0x%" PRIx64 " (%p)", nix_hwlvl2str(tm_node->hw_lvl), schq, tm_node->lvl, tm_node->id, strict_prio, rr_quantum, tm_node); switch (hw_lvl) { case NIX_TXSCH_LVL_SMQ: reg[k] = NIX_AF_MDQX_SCHEDULE(schq); regval[k] = (strict_prio << 24) | rr_quantum; k++; break; case NIX_TXSCH_LVL_TL4: reg[k] = NIX_AF_TL4X_SCHEDULE(schq); regval[k] = (strict_prio << 24) | rr_quantum; k++; break; case NIX_TXSCH_LVL_TL3: reg[k] = NIX_AF_TL3X_SCHEDULE(schq); regval[k] = (strict_prio << 24) | rr_quantum; k++; break; case NIX_TXSCH_LVL_TL2: reg[k] = NIX_AF_TL2X_SCHEDULE(schq); regval[k] = (strict_prio << 24) | rr_quantum; k++; break; case NIX_TXSCH_LVL_TL1: reg[k] = NIX_AF_TL1X_SCHEDULE(schq); regval[k] = rr_quantum; k++; break; } return k; } static uint8_t prepare_tm_shaper_reg(struct otx2_nix_tm_node *tm_node, struct otx2_nix_tm_shaper_profile *profile, volatile uint64_t *reg, volatile uint64_t *regval) { struct shaper_params cir, pir; uint32_t schq = tm_node->hw_id; uint8_t k = 0; memset(&cir, 0, sizeof(cir)); memset(&pir, 0, sizeof(pir)); shaper_config_to_nix(profile, &cir, &pir); otx2_tm_dbg("Shaper config node %s(%u) lvl %u id %u, " "pir %" PRIu64 "(%" PRIu64 "B)," " cir %" PRIu64 "(%" PRIu64 "B) (%p)", nix_hwlvl2str(tm_node->hw_lvl), schq, tm_node->lvl, tm_node->id, pir.rate, pir.burst, cir.rate, cir.burst, tm_node); switch (tm_node->hw_lvl) { case NIX_TXSCH_LVL_SMQ: /* Configure PIR, CIR */ reg[k] = NIX_AF_MDQX_PIR(schq); regval[k] = (pir.rate && pir.burst) ? (shaper2regval(&pir) | 1) : 0; k++; reg[k] = NIX_AF_MDQX_CIR(schq); regval[k] = (cir.rate && cir.burst) ? (shaper2regval(&cir) | 1) : 0; k++; /* Configure RED ALG */ reg[k] = NIX_AF_MDQX_SHAPE(schq); regval[k] = ((uint64_t)tm_node->red_algo << 9); k++; break; case NIX_TXSCH_LVL_TL4: /* Configure PIR, CIR */ reg[k] = NIX_AF_TL4X_PIR(schq); regval[k] = (pir.rate && pir.burst) ? (shaper2regval(&pir) | 1) : 0; k++; reg[k] = NIX_AF_TL4X_CIR(schq); regval[k] = (cir.rate && cir.burst) ? (shaper2regval(&cir) | 1) : 0; k++; /* Configure RED algo */ reg[k] = NIX_AF_TL4X_SHAPE(schq); regval[k] = ((uint64_t)tm_node->red_algo << 9); k++; break; case NIX_TXSCH_LVL_TL3: /* Configure PIR, CIR */ reg[k] = NIX_AF_TL3X_PIR(schq); regval[k] = (pir.rate && pir.burst) ? (shaper2regval(&pir) | 1) : 0; k++; reg[k] = NIX_AF_TL3X_CIR(schq); regval[k] = (cir.rate && cir.burst) ? (shaper2regval(&cir) | 1) : 0; k++; /* Configure RED algo */ reg[k] = NIX_AF_TL3X_SHAPE(schq); regval[k] = ((uint64_t)tm_node->red_algo << 9); k++; break; case NIX_TXSCH_LVL_TL2: /* Configure PIR, CIR */ reg[k] = NIX_AF_TL2X_PIR(schq); regval[k] = (pir.rate && pir.burst) ? (shaper2regval(&pir) | 1) : 0; k++; reg[k] = NIX_AF_TL2X_CIR(schq); regval[k] = (cir.rate && cir.burst) ? (shaper2regval(&cir) | 1) : 0; k++; /* Configure RED algo */ reg[k] = NIX_AF_TL2X_SHAPE(schq); regval[k] = ((uint64_t)tm_node->red_algo << 9); k++; break; case NIX_TXSCH_LVL_TL1: /* Configure CIR */ reg[k] = NIX_AF_TL1X_CIR(schq); regval[k] = (cir.rate && cir.burst) ? (shaper2regval(&cir) | 1) : 0; k++; break; } return k; } static uint8_t prepare_tm_sw_xoff(struct otx2_nix_tm_node *tm_node, bool enable, volatile uint64_t *reg, volatile uint64_t *regval) { uint32_t hw_lvl = tm_node->hw_lvl; uint32_t schq = tm_node->hw_id; uint8_t k = 0; otx2_tm_dbg("sw xoff config node %s(%u) lvl %u id %u, enable %u (%p)", nix_hwlvl2str(hw_lvl), schq, tm_node->lvl, tm_node->id, enable, tm_node); regval[k] = enable; switch (hw_lvl) { case NIX_TXSCH_LVL_MDQ: reg[k] = NIX_AF_MDQX_SW_XOFF(schq); k++; break; case NIX_TXSCH_LVL_TL4: reg[k] = NIX_AF_TL4X_SW_XOFF(schq); k++; break; case NIX_TXSCH_LVL_TL3: reg[k] = NIX_AF_TL3X_SW_XOFF(schq); k++; break; case NIX_TXSCH_LVL_TL2: reg[k] = NIX_AF_TL2X_SW_XOFF(schq); k++; break; case NIX_TXSCH_LVL_TL1: reg[k] = NIX_AF_TL1X_SW_XOFF(schq); k++; break; default: break; } return k; } static int populate_tm_reg(struct otx2_eth_dev *dev, struct otx2_nix_tm_node *tm_node) { struct otx2_nix_tm_shaper_profile *profile; uint64_t regval_mask[MAX_REGS_PER_MBOX_MSG]; uint64_t regval[MAX_REGS_PER_MBOX_MSG]; uint64_t reg[MAX_REGS_PER_MBOX_MSG]; struct otx2_mbox *mbox = dev->mbox; uint64_t parent = 0, child = 0; uint32_t hw_lvl, rr_prio, schq; struct nix_txschq_config *req; int rc = -EFAULT; uint8_t k = 0; memset(regval_mask, 0, sizeof(regval_mask)); profile = nix_tm_shaper_profile_search(dev, tm_node->params.shaper_profile_id); rr_prio = tm_node->rr_prio; hw_lvl = tm_node->hw_lvl; schq = tm_node->hw_id; /* Root node will not have a parent node */ if (hw_lvl == dev->otx2_tm_root_lvl) parent = tm_node->parent_hw_id; else parent = tm_node->parent->hw_id; /* Do we need this trigger to configure TL1 */ if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 && hw_lvl == dev->otx2_tm_root_lvl) { rc = populate_tm_tl1_default(dev, parent); if (rc) goto error; } if (hw_lvl != NIX_TXSCH_LVL_SMQ) child = find_prio_anchor(dev, tm_node->id); /* Override default rr_prio when TL1 * Static Priority is disabled */ if (hw_lvl == NIX_TXSCH_LVL_TL1 && dev->tm_flags & NIX_TM_TL1_NO_SP) { rr_prio = TXSCH_TL1_DFLT_RR_PRIO; child = 0; } otx2_tm_dbg("Topology config node %s(%u)->%s(%"PRIu64") lvl %u, id %u" " prio_anchor %"PRIu64" rr_prio %u (%p)", nix_hwlvl2str(hw_lvl), schq, nix_hwlvl2str(hw_lvl + 1), parent, tm_node->lvl, tm_node->id, child, rr_prio, tm_node); /* Prepare Topology and Link config */ switch (hw_lvl) { case NIX_TXSCH_LVL_SMQ: /* Set xoff which will be cleared later and minimum length * which will be used for zero padding if packet length is * smaller */ reg[k] = NIX_AF_SMQX_CFG(schq); regval[k] = BIT_ULL(50) | NIX_MIN_HW_FRS; regval_mask[k] = ~(BIT_ULL(50) | 0x7f); k++; /* Parent and schedule conf */ reg[k] = NIX_AF_MDQX_PARENT(schq); regval[k] = parent << 16; k++; break; case NIX_TXSCH_LVL_TL4: /* Parent and schedule conf */ reg[k] = NIX_AF_TL4X_PARENT(schq); regval[k] = parent << 16; k++; reg[k] = NIX_AF_TL4X_TOPOLOGY(schq); regval[k] = (child << 32) | (rr_prio << 1); k++; /* Configure TL4 to send to SDP channel instead of CGX/LBK */ if (otx2_dev_is_sdp(dev)) { reg[k] = NIX_AF_TL4X_SDP_LINK_CFG(schq); regval[k] = BIT_ULL(12); k++; } break; case NIX_TXSCH_LVL_TL3: /* Parent and schedule conf */ reg[k] = NIX_AF_TL3X_PARENT(schq); regval[k] = parent << 16; k++; reg[k] = NIX_AF_TL3X_TOPOLOGY(schq); regval[k] = (child << 32) | (rr_prio << 1); k++; /* Link configuration */ if (!otx2_dev_is_sdp(dev) && dev->link_cfg_lvl == NIX_TXSCH_LVL_TL3) { reg[k] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, otx2_nix_get_link(dev)); regval[k] = BIT_ULL(12) | nix_get_relchan(dev); k++; } break; case NIX_TXSCH_LVL_TL2: /* Parent and schedule conf */ reg[k] = NIX_AF_TL2X_PARENT(schq); regval[k] = parent << 16; k++; reg[k] = NIX_AF_TL2X_TOPOLOGY(schq); regval[k] = (child << 32) | (rr_prio << 1); k++; /* Link configuration */ if (!otx2_dev_is_sdp(dev) && dev->link_cfg_lvl == NIX_TXSCH_LVL_TL2) { reg[k] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, otx2_nix_get_link(dev)); regval[k] = BIT_ULL(12) | nix_get_relchan(dev); k++; } break; case NIX_TXSCH_LVL_TL1: reg[k] = NIX_AF_TL1X_TOPOLOGY(schq); regval[k] = (child << 32) | (rr_prio << 1 /*RR_PRIO*/); k++; break; } /* Prepare schedule config */ k += prepare_tm_sched_reg(dev, tm_node, ®[k], ®val[k]); /* Prepare shaping config */ k += prepare_tm_shaper_reg(tm_node, profile, ®[k], ®val[k]); if (!k) return 0; /* Copy and send config mbox */ req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox); req->lvl = hw_lvl; req->num_regs = k; otx2_mbox_memcpy(req->reg, reg, sizeof(uint64_t) * k); otx2_mbox_memcpy(req->regval, regval, sizeof(uint64_t) * k); otx2_mbox_memcpy(req->regval_mask, regval_mask, sizeof(uint64_t) * k); rc = otx2_mbox_process(mbox); if (rc) goto error; return 0; error: otx2_err("Txschq cfg request failed for node %p, rc=%d", tm_node, rc); return rc; } static int nix_tm_txsch_reg_config(struct otx2_eth_dev *dev) { struct otx2_nix_tm_node *tm_node; uint32_t hw_lvl; int rc = 0; for (hw_lvl = 0; hw_lvl <= dev->otx2_tm_root_lvl; hw_lvl++) { TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (tm_node->hw_lvl == hw_lvl && tm_node->hw_lvl != NIX_TXSCH_LVL_CNT) { rc = populate_tm_reg(dev, tm_node); if (rc) goto exit; } } } exit: return rc; } static struct otx2_nix_tm_node * nix_tm_node_search(struct otx2_eth_dev *dev, uint32_t node_id, bool user) { struct otx2_nix_tm_node *tm_node; TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (tm_node->id == node_id && (user == !!(tm_node->flags & NIX_TM_NODE_USER))) return tm_node; } return NULL; } static uint32_t check_rr(struct otx2_eth_dev *dev, uint32_t priority, uint32_t parent_id) { struct otx2_nix_tm_node *tm_node; uint32_t rr_num = 0; TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (!tm_node->parent) continue; if (!(tm_node->parent->id == parent_id)) continue; if (tm_node->priority == priority) rr_num++; } return rr_num; } static int nix_tm_update_parent_info(struct otx2_eth_dev *dev) { struct otx2_nix_tm_node *tm_node_child; struct otx2_nix_tm_node *tm_node; struct otx2_nix_tm_node *parent; uint32_t rr_num = 0; uint32_t priority; TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (!tm_node->parent) continue; /* Count group of children of same priority i.e are RR */ parent = tm_node->parent; priority = tm_node->priority; rr_num = check_rr(dev, priority, parent->id); /* Assuming that multiple RR groups are * not configured based on capability. */ if (rr_num > 1) { parent->rr_prio = priority; parent->rr_num = rr_num; } /* Find out static priority children that are not in RR */ TAILQ_FOREACH(tm_node_child, &dev->node_list, node) { if (!tm_node_child->parent) continue; if (parent->id != tm_node_child->parent->id) continue; if (parent->max_prio == UINT32_MAX && tm_node_child->priority != parent->rr_prio) parent->max_prio = 0; if (parent->max_prio < tm_node_child->priority && parent->rr_prio != tm_node_child->priority) parent->max_prio = tm_node_child->priority; } } return 0; } static int nix_tm_node_add_to_list(struct otx2_eth_dev *dev, uint32_t node_id, uint32_t parent_node_id, uint32_t priority, uint32_t weight, uint16_t hw_lvl, uint16_t lvl, bool user, struct rte_tm_node_params *params) { struct otx2_nix_tm_shaper_profile *profile; struct otx2_nix_tm_node *tm_node, *parent_node; uint32_t profile_id; profile_id = params->shaper_profile_id; profile = nix_tm_shaper_profile_search(dev, profile_id); parent_node = nix_tm_node_search(dev, parent_node_id, user); tm_node = rte_zmalloc("otx2_nix_tm_node", sizeof(struct otx2_nix_tm_node), 0); if (!tm_node) return -ENOMEM; tm_node->lvl = lvl; tm_node->hw_lvl = hw_lvl; /* Maintain minimum weight */ if (!weight) weight = 1; tm_node->id = node_id; tm_node->priority = priority; tm_node->weight = weight; tm_node->rr_prio = 0xf; tm_node->max_prio = UINT32_MAX; tm_node->hw_id = UINT32_MAX; tm_node->flags = 0; if (user) tm_node->flags = NIX_TM_NODE_USER; rte_memcpy(&tm_node->params, params, sizeof(struct rte_tm_node_params)); if (profile) profile->reference_count++; tm_node->parent = parent_node; tm_node->parent_hw_id = UINT32_MAX; shaper_default_red_algo(dev, tm_node, profile); TAILQ_INSERT_TAIL(&dev->node_list, tm_node, node); return 0; } static int nix_tm_clear_shaper_profiles(struct otx2_eth_dev *dev) { struct otx2_nix_tm_shaper_profile *shaper_profile; while ((shaper_profile = TAILQ_FIRST(&dev->shaper_profile_list))) { if (shaper_profile->reference_count) otx2_tm_dbg("Shaper profile %u has non zero references", shaper_profile->shaper_profile_id); TAILQ_REMOVE(&dev->shaper_profile_list, shaper_profile, shaper); rte_free(shaper_profile); } return 0; } static int nix_clear_path_xoff(struct otx2_eth_dev *dev, struct otx2_nix_tm_node *tm_node) { struct nix_txschq_config *req; struct otx2_nix_tm_node *p; int rc; /* Manipulating SW_XOFF not supported on Ax */ if (otx2_dev_is_Ax(dev)) return 0; /* Enable nodes in path for flush to succeed */ if (!nix_tm_is_leaf(dev, tm_node->lvl)) p = tm_node; else p = tm_node->parent; while (p) { if (!(p->flags & NIX_TM_NODE_ENABLED) && (p->flags & NIX_TM_NODE_HWRES)) { req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox); req->lvl = p->hw_lvl; req->num_regs = prepare_tm_sw_xoff(p, false, req->reg, req->regval); rc = otx2_mbox_process(dev->mbox); if (rc) return rc; p->flags |= NIX_TM_NODE_ENABLED; } p = p->parent; } return 0; } static int nix_smq_xoff(struct otx2_eth_dev *dev, struct otx2_nix_tm_node *tm_node, bool enable) { struct otx2_mbox *mbox = dev->mbox; struct nix_txschq_config *req; uint16_t smq; int rc; smq = tm_node->hw_id; otx2_tm_dbg("Setting SMQ %u XOFF/FLUSH to %s", smq, enable ? "enable" : "disable"); rc = nix_clear_path_xoff(dev, tm_node); if (rc) return rc; req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox); req->lvl = NIX_TXSCH_LVL_SMQ; req->num_regs = 1; req->reg[0] = NIX_AF_SMQX_CFG(smq); req->regval[0] = enable ? (BIT_ULL(50) | BIT_ULL(49)) : 0; req->regval_mask[0] = enable ? ~(BIT_ULL(50) | BIT_ULL(49)) : ~BIT_ULL(50); return otx2_mbox_process(mbox); } int otx2_nix_sq_sqb_aura_fc(void *__txq, bool enable) { struct otx2_eth_txq *txq = __txq; struct npa_aq_enq_req *req; struct npa_aq_enq_rsp *rsp; struct otx2_npa_lf *lf; struct otx2_mbox *mbox; uint64_t aura_handle; int rc; otx2_tm_dbg("Setting SQ %u SQB aura FC to %s", txq->sq, enable ? "enable" : "disable"); lf = otx2_npa_lf_obj_get(); if (!lf) return -EFAULT; mbox = lf->mbox; /* Set/clear sqb aura fc_ena */ aura_handle = txq->sqb_pool->pool_id; req = otx2_mbox_alloc_msg_npa_aq_enq(mbox); req->aura_id = npa_lf_aura_handle_to_aura(aura_handle); req->ctype = NPA_AQ_CTYPE_AURA; req->op = NPA_AQ_INSTOP_WRITE; /* Below is not needed for aura writes but AF driver needs it */ /* AF will translate to associated poolctx */ req->aura.pool_addr = req->aura_id; req->aura.fc_ena = enable; req->aura_mask.fc_ena = 1; rc = otx2_mbox_process(mbox); if (rc) return rc; /* Read back npa aura ctx */ req = otx2_mbox_alloc_msg_npa_aq_enq(mbox); req->aura_id = npa_lf_aura_handle_to_aura(aura_handle); req->ctype = NPA_AQ_CTYPE_AURA; req->op = NPA_AQ_INSTOP_READ; rc = otx2_mbox_process_msg(mbox, (void *)&rsp); if (rc) return rc; /* Init when enabled as there might be no triggers */ if (enable) *(volatile uint64_t *)txq->fc_mem = rsp->aura.count; else *(volatile uint64_t *)txq->fc_mem = txq->nb_sqb_bufs; /* Sync write barrier */ rte_wmb(); return 0; } static int nix_txq_flush_sq_spin(struct otx2_eth_txq *txq) { uint16_t sqb_cnt, head_off, tail_off; struct otx2_eth_dev *dev = txq->dev; uint64_t wdata, val, prev; uint16_t sq = txq->sq; int64_t *regaddr; uint64_t timeout;/* 10's of usec */ /* Wait for enough time based on shaper min rate */ timeout = (txq->qconf.nb_desc * NIX_MAX_HW_FRS * 8 * 1E5); timeout = timeout / dev->tm_rate_min; if (!timeout) timeout = 10000; wdata = ((uint64_t)sq << 32); regaddr = (int64_t *)(dev->base + NIX_LF_SQ_OP_STATUS); val = otx2_atomic64_add_nosync(wdata, regaddr); /* Spin multiple iterations as "txq->fc_cache_pkts" can still * have space to send pkts even though fc_mem is disabled */ while (true) { prev = val; rte_delay_us(10); val = otx2_atomic64_add_nosync(wdata, regaddr); /* Continue on error */ if (val & BIT_ULL(63)) continue; if (prev != val) continue; sqb_cnt = val & 0xFFFF; head_off = (val >> 20) & 0x3F; tail_off = (val >> 28) & 0x3F; /* SQ reached quiescent state */ if (sqb_cnt <= 1 && head_off == tail_off && (*txq->fc_mem == txq->nb_sqb_bufs)) { break; } /* Timeout */ if (!timeout) goto exit; timeout--; } return 0; exit: otx2_nix_tm_dump(dev); return -EFAULT; } /* Flush and disable tx queue and its parent SMQ */ int otx2_nix_sq_flush_pre(void *_txq, bool dev_started) { struct otx2_nix_tm_node *tm_node, *sibling; struct otx2_eth_txq *txq; struct otx2_eth_dev *dev; uint16_t sq; bool user; int rc; txq = _txq; dev = txq->dev; sq = txq->sq; user = !!(dev->tm_flags & NIX_TM_COMMITTED); /* Find the node for this SQ */ tm_node = nix_tm_node_search(dev, sq, user); if (!tm_node || !(tm_node->flags & NIX_TM_NODE_ENABLED)) { otx2_err("Invalid node/state for sq %u", sq); return -EFAULT; } /* Enable CGX RXTX to drain pkts */ if (!dev_started) { /* Though it enables both RX MCAM Entries and CGX Link * we assume all the rx queues are stopped way back. */ otx2_mbox_alloc_msg_nix_lf_start_rx(dev->mbox); rc = otx2_mbox_process(dev->mbox); if (rc) { otx2_err("cgx start failed, rc=%d", rc); return rc; } } /* Disable smq xoff for case it was enabled earlier */ rc = nix_smq_xoff(dev, tm_node->parent, false); if (rc) { otx2_err("Failed to enable smq %u, rc=%d", tm_node->parent->hw_id, rc); return rc; } /* As per HRM, to disable an SQ, all other SQ's * that feed to same SMQ must be paused before SMQ flush. */ TAILQ_FOREACH(sibling, &dev->node_list, node) { if (sibling->parent != tm_node->parent) continue; if (!(sibling->flags & NIX_TM_NODE_ENABLED)) continue; sq = sibling->id; txq = dev->eth_dev->data->tx_queues[sq]; if (!txq) continue; rc = otx2_nix_sq_sqb_aura_fc(txq, false); if (rc) { otx2_err("Failed to disable sqb aura fc, rc=%d", rc); goto cleanup; } /* Wait for sq entries to be flushed */ rc = nix_txq_flush_sq_spin(txq); if (rc) { otx2_err("Failed to drain sq %u, rc=%d\n", txq->sq, rc); return rc; } } tm_node->flags &= ~NIX_TM_NODE_ENABLED; /* Disable and flush */ rc = nix_smq_xoff(dev, tm_node->parent, true); if (rc) { otx2_err("Failed to disable smq %u, rc=%d", tm_node->parent->hw_id, rc); goto cleanup; } cleanup: /* Restore cgx state */ if (!dev_started) { otx2_mbox_alloc_msg_nix_lf_stop_rx(dev->mbox); rc |= otx2_mbox_process(dev->mbox); } return rc; } int otx2_nix_sq_flush_post(void *_txq) { struct otx2_nix_tm_node *tm_node, *sibling; struct otx2_eth_txq *txq = _txq; struct otx2_eth_txq *s_txq; struct otx2_eth_dev *dev; bool once = false; uint16_t sq, s_sq; bool user; int rc; dev = txq->dev; sq = txq->sq; user = !!(dev->tm_flags & NIX_TM_COMMITTED); /* Find the node for this SQ */ tm_node = nix_tm_node_search(dev, sq, user); if (!tm_node) { otx2_err("Invalid node for sq %u", sq); return -EFAULT; } /* Enable all the siblings back */ TAILQ_FOREACH(sibling, &dev->node_list, node) { if (sibling->parent != tm_node->parent) continue; if (sibling->id == sq) continue; if (!(sibling->flags & NIX_TM_NODE_ENABLED)) continue; s_sq = sibling->id; s_txq = dev->eth_dev->data->tx_queues[s_sq]; if (!s_txq) continue; if (!once) { /* Enable back if any SQ is still present */ rc = nix_smq_xoff(dev, tm_node->parent, false); if (rc) { otx2_err("Failed to enable smq %u, rc=%d", tm_node->parent->hw_id, rc); return rc; } once = true; } rc = otx2_nix_sq_sqb_aura_fc(s_txq, true); if (rc) { otx2_err("Failed to enable sqb aura fc, rc=%d", rc); return rc; } } return 0; } static int nix_sq_sched_data(struct otx2_eth_dev *dev, struct otx2_nix_tm_node *tm_node, bool rr_quantum_only) { struct rte_eth_dev *eth_dev = dev->eth_dev; struct otx2_mbox *mbox = dev->mbox; uint16_t sq = tm_node->id, smq; struct nix_aq_enq_req *req; uint64_t rr_quantum; int rc; smq = tm_node->parent->hw_id; rr_quantum = NIX_TM_WEIGHT_TO_RR_QUANTUM(tm_node->weight); if (rr_quantum_only) otx2_tm_dbg("Update sq(%u) rr_quantum 0x%"PRIx64, sq, rr_quantum); else otx2_tm_dbg("Enabling sq(%u)->smq(%u), rr_quantum 0x%"PRIx64, sq, smq, rr_quantum); if (sq > eth_dev->data->nb_tx_queues) return -EFAULT; req = otx2_mbox_alloc_msg_nix_aq_enq(mbox); req->qidx = sq; req->ctype = NIX_AQ_CTYPE_SQ; req->op = NIX_AQ_INSTOP_WRITE; /* smq update only when needed */ if (!rr_quantum_only) { req->sq.smq = smq; req->sq_mask.smq = ~req->sq_mask.smq; } req->sq.smq_rr_quantum = rr_quantum; req->sq_mask.smq_rr_quantum = ~req->sq_mask.smq_rr_quantum; rc = otx2_mbox_process(mbox); if (rc) otx2_err("Failed to set smq, rc=%d", rc); return rc; } int otx2_nix_sq_enable(void *_txq) { struct otx2_eth_txq *txq = _txq; int rc; /* Enable sqb_aura fc */ rc = otx2_nix_sq_sqb_aura_fc(txq, true); if (rc) { otx2_err("Failed to enable sqb aura fc, rc=%d", rc); return rc; } return 0; } static int nix_tm_free_resources(struct otx2_eth_dev *dev, uint32_t flags_mask, uint32_t flags, bool hw_only) { struct otx2_nix_tm_shaper_profile *profile; struct otx2_nix_tm_node *tm_node, *next_node; struct otx2_mbox *mbox = dev->mbox; struct nix_txsch_free_req *req; uint32_t profile_id; int rc = 0; next_node = TAILQ_FIRST(&dev->node_list); while (next_node) { tm_node = next_node; next_node = TAILQ_NEXT(tm_node, node); /* Check for only requested nodes */ if ((tm_node->flags & flags_mask) != flags) continue; if (!nix_tm_is_leaf(dev, tm_node->lvl) && tm_node->hw_lvl != NIX_TXSCH_LVL_TL1 && tm_node->flags & NIX_TM_NODE_HWRES) { /* Free specific HW resource */ otx2_tm_dbg("Free hwres %s(%u) lvl %u id %u (%p)", nix_hwlvl2str(tm_node->hw_lvl), tm_node->hw_id, tm_node->lvl, tm_node->id, tm_node); rc = nix_clear_path_xoff(dev, tm_node); if (rc) return rc; req = otx2_mbox_alloc_msg_nix_txsch_free(mbox); req->flags = 0; req->schq_lvl = tm_node->hw_lvl; req->schq = tm_node->hw_id; rc = otx2_mbox_process(mbox); if (rc) return rc; tm_node->flags &= ~NIX_TM_NODE_HWRES; } /* Leave software elements if needed */ if (hw_only) continue; otx2_tm_dbg("Free node lvl %u id %u (%p)", tm_node->lvl, tm_node->id, tm_node); profile_id = tm_node->params.shaper_profile_id; profile = nix_tm_shaper_profile_search(dev, profile_id); if (profile) profile->reference_count--; TAILQ_REMOVE(&dev->node_list, tm_node, node); rte_free(tm_node); } if (!flags_mask) { /* Free all hw resources */ req = otx2_mbox_alloc_msg_nix_txsch_free(mbox); req->flags = TXSCHQ_FREE_ALL; return otx2_mbox_process(mbox); } return rc; } static uint8_t nix_tm_copy_rsp_to_dev(struct otx2_eth_dev *dev, struct nix_txsch_alloc_rsp *rsp) { uint16_t schq; uint8_t lvl; for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { for (schq = 0; schq < MAX_TXSCHQ_PER_FUNC; schq++) { dev->txschq_list[lvl][schq] = rsp->schq_list[lvl][schq]; dev->txschq_contig_list[lvl][schq] = rsp->schq_contig_list[lvl][schq]; } dev->txschq[lvl] = rsp->schq[lvl]; dev->txschq_contig[lvl] = rsp->schq_contig[lvl]; } return 0; } static int nix_tm_assign_id_to_node(struct otx2_eth_dev *dev, struct otx2_nix_tm_node *child, struct otx2_nix_tm_node *parent) { uint32_t hw_id, schq_con_index, prio_offset; uint32_t l_id, schq_index; otx2_tm_dbg("Assign hw id for child node %s lvl %u id %u (%p)", nix_hwlvl2str(child->hw_lvl), child->lvl, child->id, child); child->flags |= NIX_TM_NODE_HWRES; /* Process root nodes */ if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 && child->hw_lvl == dev->otx2_tm_root_lvl && !parent) { int idx = 0; uint32_t tschq_con_index; l_id = child->hw_lvl; tschq_con_index = dev->txschq_contig_index[l_id]; hw_id = dev->txschq_contig_list[l_id][tschq_con_index]; child->hw_id = hw_id; dev->txschq_contig_index[l_id]++; /* Update TL1 hw_id for its parent for config purpose */ idx = dev->txschq_index[NIX_TXSCH_LVL_TL1]++; hw_id = dev->txschq_list[NIX_TXSCH_LVL_TL1][idx]; child->parent_hw_id = hw_id; return 0; } if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL1 && child->hw_lvl == dev->otx2_tm_root_lvl && !parent) { uint32_t tschq_con_index; l_id = child->hw_lvl; tschq_con_index = dev->txschq_index[l_id]; hw_id = dev->txschq_list[l_id][tschq_con_index]; child->hw_id = hw_id; dev->txschq_index[l_id]++; return 0; } /* Process children with parents */ l_id = child->hw_lvl; schq_index = dev->txschq_index[l_id]; schq_con_index = dev->txschq_contig_index[l_id]; if (child->priority == parent->rr_prio) { hw_id = dev->txschq_list[l_id][schq_index]; child->hw_id = hw_id; child->parent_hw_id = parent->hw_id; dev->txschq_index[l_id]++; } else { prio_offset = schq_con_index + child->priority; hw_id = dev->txschq_contig_list[l_id][prio_offset]; child->hw_id = hw_id; } return 0; } static int nix_tm_assign_hw_id(struct otx2_eth_dev *dev) { struct otx2_nix_tm_node *parent, *child; uint32_t child_hw_lvl, con_index_inc, i; for (i = NIX_TXSCH_LVL_TL1; i > 0; i--) { TAILQ_FOREACH(parent, &dev->node_list, node) { child_hw_lvl = parent->hw_lvl - 1; if (parent->hw_lvl != i) continue; TAILQ_FOREACH(child, &dev->node_list, node) { if (!child->parent) continue; if (child->parent->id != parent->id) continue; nix_tm_assign_id_to_node(dev, child, parent); } con_index_inc = parent->max_prio + 1; dev->txschq_contig_index[child_hw_lvl] += con_index_inc; /* * Explicitly assign id to parent node if it * doesn't have a parent */ if (parent->hw_lvl == dev->otx2_tm_root_lvl) nix_tm_assign_id_to_node(dev, parent, NULL); } } return 0; } static uint8_t nix_tm_count_req_schq(struct otx2_eth_dev *dev, struct nix_txsch_alloc_req *req, uint8_t lvl) { struct otx2_nix_tm_node *tm_node; uint8_t contig_count; TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (lvl == tm_node->hw_lvl) { req->schq[lvl - 1] += tm_node->rr_num; if (tm_node->max_prio != UINT32_MAX) { contig_count = tm_node->max_prio + 1; req->schq_contig[lvl - 1] += contig_count; } } if (lvl == dev->otx2_tm_root_lvl && dev->otx2_tm_root_lvl && lvl == NIX_TXSCH_LVL_TL2 && tm_node->hw_lvl == dev->otx2_tm_root_lvl) { req->schq_contig[dev->otx2_tm_root_lvl]++; } } req->schq[NIX_TXSCH_LVL_TL1] = 1; req->schq_contig[NIX_TXSCH_LVL_TL1] = 0; return 0; } static int nix_tm_prepare_txschq_req(struct otx2_eth_dev *dev, struct nix_txsch_alloc_req *req) { uint8_t i; for (i = NIX_TXSCH_LVL_TL1; i > 0; i--) nix_tm_count_req_schq(dev, req, i); for (i = 0; i < NIX_TXSCH_LVL_CNT; i++) { dev->txschq_index[i] = 0; dev->txschq_contig_index[i] = 0; } return 0; } static int nix_tm_send_txsch_alloc_msg(struct otx2_eth_dev *dev) { struct otx2_mbox *mbox = dev->mbox; struct nix_txsch_alloc_req *req; struct nix_txsch_alloc_rsp *rsp; int rc; req = otx2_mbox_alloc_msg_nix_txsch_alloc(mbox); rc = nix_tm_prepare_txschq_req(dev, req); if (rc) return rc; rc = otx2_mbox_process_msg(mbox, (void *)&rsp); if (rc) return rc; nix_tm_copy_rsp_to_dev(dev, rsp); dev->link_cfg_lvl = rsp->link_cfg_lvl; nix_tm_assign_hw_id(dev); return 0; } static int nix_tm_alloc_resources(struct rte_eth_dev *eth_dev, bool xmit_enable) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_nix_tm_node *tm_node; struct otx2_eth_txq *txq; uint16_t sq; int rc; nix_tm_update_parent_info(dev); rc = nix_tm_send_txsch_alloc_msg(dev); if (rc) { otx2_err("TM failed to alloc tm resources=%d", rc); return rc; } rc = nix_tm_txsch_reg_config(dev); if (rc) { otx2_err("TM failed to configure sched registers=%d", rc); return rc; } /* Trigger MTU recalculate as SMQ needs MTU conf */ if (eth_dev->data->dev_started && eth_dev->data->nb_rx_queues) { rc = otx2_nix_recalc_mtu(eth_dev); if (rc) { otx2_err("TM MTU update failed, rc=%d", rc); return rc; } } /* Mark all non-leaf's as enabled */ TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (!nix_tm_is_leaf(dev, tm_node->lvl)) tm_node->flags |= NIX_TM_NODE_ENABLED; } if (!xmit_enable) return 0; /* Update SQ Sched Data while SQ is idle */ TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (!nix_tm_is_leaf(dev, tm_node->lvl)) continue; rc = nix_sq_sched_data(dev, tm_node, false); if (rc) { otx2_err("SQ %u sched update failed, rc=%d", tm_node->id, rc); return rc; } } /* Finally XON all SMQ's */ TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (tm_node->hw_lvl != NIX_TXSCH_LVL_SMQ) continue; rc = nix_smq_xoff(dev, tm_node, false); if (rc) { otx2_err("Failed to enable smq %u, rc=%d", tm_node->hw_id, rc); return rc; } } /* Enable xmit as all the topology is ready */ TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (!nix_tm_is_leaf(dev, tm_node->lvl)) continue; sq = tm_node->id; txq = eth_dev->data->tx_queues[sq]; rc = otx2_nix_sq_enable(txq); if (rc) { otx2_err("TM sw xon failed on SQ %u, rc=%d", tm_node->id, rc); return rc; } tm_node->flags |= NIX_TM_NODE_ENABLED; } return 0; } static int send_tm_reqval(struct otx2_mbox *mbox, struct nix_txschq_config *req, struct rte_tm_error *error) { int rc; if (!req->num_regs || req->num_regs > MAX_REGS_PER_MBOX_MSG) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "invalid config"; return -EIO; } rc = otx2_mbox_process(mbox); if (rc) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "unexpected fatal error"; } return rc; } static uint16_t nix_tm_lvl2nix(struct otx2_eth_dev *dev, uint32_t lvl) { if (nix_tm_have_tl1_access(dev)) { switch (lvl) { case OTX2_TM_LVL_ROOT: return NIX_TXSCH_LVL_TL1; case OTX2_TM_LVL_SCH1: return NIX_TXSCH_LVL_TL2; case OTX2_TM_LVL_SCH2: return NIX_TXSCH_LVL_TL3; case OTX2_TM_LVL_SCH3: return NIX_TXSCH_LVL_TL4; case OTX2_TM_LVL_SCH4: return NIX_TXSCH_LVL_SMQ; default: return NIX_TXSCH_LVL_CNT; } } else { switch (lvl) { case OTX2_TM_LVL_ROOT: return NIX_TXSCH_LVL_TL2; case OTX2_TM_LVL_SCH1: return NIX_TXSCH_LVL_TL3; case OTX2_TM_LVL_SCH2: return NIX_TXSCH_LVL_TL4; case OTX2_TM_LVL_SCH3: return NIX_TXSCH_LVL_SMQ; default: return NIX_TXSCH_LVL_CNT; } } } static uint16_t nix_max_prio(struct otx2_eth_dev *dev, uint16_t hw_lvl) { if (hw_lvl >= NIX_TXSCH_LVL_CNT) return 0; /* MDQ doesn't support SP */ if (hw_lvl == NIX_TXSCH_LVL_MDQ) return 0; /* PF's TL1 with VF's enabled doesn't support SP */ if (hw_lvl == NIX_TXSCH_LVL_TL1 && (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 || (dev->tm_flags & NIX_TM_TL1_NO_SP))) return 0; return TXSCH_TLX_SP_PRIO_MAX - 1; } static int validate_prio(struct otx2_eth_dev *dev, uint32_t lvl, uint32_t parent_id, uint32_t priority, struct rte_tm_error *error) { uint8_t priorities[TXSCH_TLX_SP_PRIO_MAX]; struct otx2_nix_tm_node *tm_node; uint32_t rr_num = 0; int i; /* Validate priority against max */ if (priority > nix_max_prio(dev, nix_tm_lvl2nix(dev, lvl - 1))) { error->type = RTE_TM_ERROR_TYPE_CAPABILITIES; error->message = "unsupported priority value"; return -EINVAL; } if (parent_id == RTE_TM_NODE_ID_NULL) return 0; memset(priorities, 0, TXSCH_TLX_SP_PRIO_MAX); priorities[priority] = 1; TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (!tm_node->parent) continue; if (!(tm_node->flags & NIX_TM_NODE_USER)) continue; if (tm_node->parent->id != parent_id) continue; priorities[tm_node->priority]++; } for (i = 0; i < TXSCH_TLX_SP_PRIO_MAX; i++) if (priorities[i] > 1) rr_num++; /* At max, one rr groups per parent */ if (rr_num > 1) { error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY; error->message = "multiple DWRR node priority"; return -EINVAL; } /* Check for previous priority to avoid holes in priorities */ if (priority && !priorities[priority - 1]) { error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY; error->message = "priority not in order"; return -EINVAL; } return 0; } static int read_tm_reg(struct otx2_mbox *mbox, uint64_t reg, uint64_t *regval, uint32_t hw_lvl) { volatile struct nix_txschq_config *req; struct nix_txschq_config *rsp; int rc; req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox); req->read = 1; req->lvl = hw_lvl; req->reg[0] = reg; req->num_regs = 1; rc = otx2_mbox_process_msg(mbox, (void **)&rsp); if (rc) return rc; *regval = rsp->regval[0]; return 0; } /* Search for min rate in topology */ static void nix_tm_shaper_profile_update_min(struct otx2_eth_dev *dev) { struct otx2_nix_tm_shaper_profile *profile; uint64_t rate_min = 1E9; /* 1 Gbps */ TAILQ_FOREACH(profile, &dev->shaper_profile_list, shaper) { if (profile->params.peak.rate && profile->params.peak.rate < rate_min) rate_min = profile->params.peak.rate; if (profile->params.committed.rate && profile->params.committed.rate < rate_min) rate_min = profile->params.committed.rate; } dev->tm_rate_min = rate_min; } static int nix_xmit_disable(struct rte_eth_dev *eth_dev) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); uint16_t sq_cnt = eth_dev->data->nb_tx_queues; uint16_t sqb_cnt, head_off, tail_off; struct otx2_nix_tm_node *tm_node; struct otx2_eth_txq *txq; uint64_t wdata, val; int i, rc; otx2_tm_dbg("Disabling xmit on %s", eth_dev->data->name); /* Enable CGX RXTX to drain pkts */ if (!eth_dev->data->dev_started) { otx2_mbox_alloc_msg_nix_lf_start_rx(dev->mbox); rc = otx2_mbox_process(dev->mbox); if (rc) return rc; } /* XON all SMQ's */ TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (tm_node->hw_lvl != NIX_TXSCH_LVL_SMQ) continue; if (!(tm_node->flags & NIX_TM_NODE_HWRES)) continue; rc = nix_smq_xoff(dev, tm_node, false); if (rc) { otx2_err("Failed to enable smq %u, rc=%d", tm_node->hw_id, rc); goto cleanup; } } /* Flush all tx queues */ for (i = 0; i < sq_cnt; i++) { txq = eth_dev->data->tx_queues[i]; rc = otx2_nix_sq_sqb_aura_fc(txq, false); if (rc) { otx2_err("Failed to disable sqb aura fc, rc=%d", rc); goto cleanup; } /* Wait for sq entries to be flushed */ rc = nix_txq_flush_sq_spin(txq); if (rc) { otx2_err("Failed to drain sq, rc=%d\n", rc); goto cleanup; } } /* XOFF & Flush all SMQ's. HRM mandates * all SQ's empty before SMQ flush is issued. */ TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (tm_node->hw_lvl != NIX_TXSCH_LVL_SMQ) continue; if (!(tm_node->flags & NIX_TM_NODE_HWRES)) continue; rc = nix_smq_xoff(dev, tm_node, true); if (rc) { otx2_err("Failed to enable smq %u, rc=%d", tm_node->hw_id, rc); goto cleanup; } } /* Verify sanity of all tx queues */ for (i = 0; i < sq_cnt; i++) { txq = eth_dev->data->tx_queues[i]; wdata = ((uint64_t)txq->sq << 32); val = otx2_atomic64_add_nosync(wdata, (int64_t *)(dev->base + NIX_LF_SQ_OP_STATUS)); sqb_cnt = val & 0xFFFF; head_off = (val >> 20) & 0x3F; tail_off = (val >> 28) & 0x3F; if (sqb_cnt > 1 || head_off != tail_off || (*txq->fc_mem != txq->nb_sqb_bufs)) otx2_err("Failed to gracefully flush sq %u", txq->sq); } cleanup: /* restore cgx state */ if (!eth_dev->data->dev_started) { otx2_mbox_alloc_msg_nix_lf_stop_rx(dev->mbox); rc |= otx2_mbox_process(dev->mbox); } return rc; } static int otx2_nix_tm_node_type_get(struct rte_eth_dev *eth_dev, uint32_t node_id, int *is_leaf, struct rte_tm_error *error) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_nix_tm_node *tm_node; if (is_leaf == NULL) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; return -EINVAL; } tm_node = nix_tm_node_search(dev, node_id, true); if (node_id == RTE_TM_NODE_ID_NULL || !tm_node) { error->type = RTE_TM_ERROR_TYPE_NODE_ID; return -EINVAL; } if (nix_tm_is_leaf(dev, tm_node->lvl)) *is_leaf = true; else *is_leaf = false; return 0; } static int otx2_nix_tm_capa_get(struct rte_eth_dev *eth_dev, struct rte_tm_capabilities *cap, struct rte_tm_error *error) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_mbox *mbox = dev->mbox; int rc, max_nr_nodes = 0, i; struct free_rsrcs_rsp *rsp; memset(cap, 0, sizeof(*cap)); otx2_mbox_alloc_msg_free_rsrc_cnt(mbox); rc = otx2_mbox_process_msg(mbox, (void *)&rsp); if (rc) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "unexpected fatal error"; return rc; } for (i = 0; i < NIX_TXSCH_LVL_TL1; i++) max_nr_nodes += rsp->schq[i]; cap->n_nodes_max = max_nr_nodes + dev->tm_leaf_cnt; /* TL1 level is reserved for PF */ cap->n_levels_max = nix_tm_have_tl1_access(dev) ? OTX2_TM_LVL_MAX : OTX2_TM_LVL_MAX - 1; cap->non_leaf_nodes_identical = 1; cap->leaf_nodes_identical = 1; /* Shaper Capabilities */ cap->shaper_private_n_max = max_nr_nodes; cap->shaper_n_max = max_nr_nodes; cap->shaper_private_dual_rate_n_max = max_nr_nodes; cap->shaper_private_rate_min = MIN_SHAPER_RATE / 8; cap->shaper_private_rate_max = MAX_SHAPER_RATE / 8; cap->shaper_pkt_length_adjust_min = 0; cap->shaper_pkt_length_adjust_max = 0; /* Schedule Capabilities */ cap->sched_n_children_max = rsp->schq[NIX_TXSCH_LVL_MDQ]; cap->sched_sp_n_priorities_max = TXSCH_TLX_SP_PRIO_MAX; cap->sched_wfq_n_children_per_group_max = cap->sched_n_children_max; cap->sched_wfq_n_groups_max = 1; cap->sched_wfq_weight_max = MAX_SCHED_WEIGHT; cap->dynamic_update_mask = RTE_TM_UPDATE_NODE_PARENT_KEEP_LEVEL | RTE_TM_UPDATE_NODE_SUSPEND_RESUME; cap->stats_mask = RTE_TM_STATS_N_PKTS | RTE_TM_STATS_N_BYTES | RTE_TM_STATS_N_PKTS_RED_DROPPED | RTE_TM_STATS_N_BYTES_RED_DROPPED; for (i = 0; i < RTE_COLORS; i++) { cap->mark_vlan_dei_supported[i] = false; cap->mark_ip_ecn_tcp_supported[i] = false; cap->mark_ip_dscp_supported[i] = false; } return 0; } static int otx2_nix_tm_level_capa_get(struct rte_eth_dev *eth_dev, uint32_t lvl, struct rte_tm_level_capabilities *cap, struct rte_tm_error *error) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_mbox *mbox = dev->mbox; struct free_rsrcs_rsp *rsp; uint16_t hw_lvl; int rc; memset(cap, 0, sizeof(*cap)); otx2_mbox_alloc_msg_free_rsrc_cnt(mbox); rc = otx2_mbox_process_msg(mbox, (void *)&rsp); if (rc) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "unexpected fatal error"; return rc; } hw_lvl = nix_tm_lvl2nix(dev, lvl); if (nix_tm_is_leaf(dev, lvl)) { /* Leaf */ cap->n_nodes_max = dev->tm_leaf_cnt; cap->n_nodes_leaf_max = dev->tm_leaf_cnt; cap->leaf_nodes_identical = 1; cap->leaf.stats_mask = RTE_TM_STATS_N_PKTS | RTE_TM_STATS_N_BYTES; } else if (lvl == OTX2_TM_LVL_ROOT) { /* Root node, aka TL2(vf)/TL1(pf) */ cap->n_nodes_max = 1; cap->n_nodes_nonleaf_max = 1; cap->non_leaf_nodes_identical = 1; cap->nonleaf.shaper_private_supported = true; cap->nonleaf.shaper_private_dual_rate_supported = nix_tm_have_tl1_access(dev) ? false : true; cap->nonleaf.shaper_private_rate_min = MIN_SHAPER_RATE / 8; cap->nonleaf.shaper_private_rate_max = MAX_SHAPER_RATE / 8; cap->nonleaf.sched_n_children_max = rsp->schq[hw_lvl - 1]; cap->nonleaf.sched_sp_n_priorities_max = nix_max_prio(dev, hw_lvl) + 1; cap->nonleaf.sched_wfq_n_groups_max = 1; cap->nonleaf.sched_wfq_weight_max = MAX_SCHED_WEIGHT; if (nix_tm_have_tl1_access(dev)) cap->nonleaf.stats_mask = RTE_TM_STATS_N_PKTS_RED_DROPPED | RTE_TM_STATS_N_BYTES_RED_DROPPED; } else if ((lvl < OTX2_TM_LVL_MAX) && (hw_lvl < NIX_TXSCH_LVL_CNT)) { /* TL2, TL3, TL4, MDQ */ cap->n_nodes_max = rsp->schq[hw_lvl]; cap->n_nodes_nonleaf_max = cap->n_nodes_max; cap->non_leaf_nodes_identical = 1; cap->nonleaf.shaper_private_supported = true; cap->nonleaf.shaper_private_dual_rate_supported = true; cap->nonleaf.shaper_private_rate_min = MIN_SHAPER_RATE / 8; cap->nonleaf.shaper_private_rate_max = MAX_SHAPER_RATE / 8; /* MDQ doesn't support Strict Priority */ if (hw_lvl == NIX_TXSCH_LVL_MDQ) cap->nonleaf.sched_n_children_max = dev->tm_leaf_cnt; else cap->nonleaf.sched_n_children_max = rsp->schq[hw_lvl - 1]; cap->nonleaf.sched_sp_n_priorities_max = nix_max_prio(dev, hw_lvl) + 1; cap->nonleaf.sched_wfq_n_groups_max = 1; cap->nonleaf.sched_wfq_weight_max = MAX_SCHED_WEIGHT; } else { /* unsupported level */ error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; return rc; } return 0; } static int otx2_nix_tm_node_capa_get(struct rte_eth_dev *eth_dev, uint32_t node_id, struct rte_tm_node_capabilities *cap, struct rte_tm_error *error) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_mbox *mbox = dev->mbox; struct otx2_nix_tm_node *tm_node; struct free_rsrcs_rsp *rsp; int rc, hw_lvl, lvl; memset(cap, 0, sizeof(*cap)); tm_node = nix_tm_node_search(dev, node_id, true); if (!tm_node) { error->type = RTE_TM_ERROR_TYPE_NODE_ID; error->message = "no such node"; return -EINVAL; } hw_lvl = tm_node->hw_lvl; lvl = tm_node->lvl; /* Leaf node */ if (nix_tm_is_leaf(dev, lvl)) { cap->stats_mask = RTE_TM_STATS_N_PKTS | RTE_TM_STATS_N_BYTES; return 0; } otx2_mbox_alloc_msg_free_rsrc_cnt(mbox); rc = otx2_mbox_process_msg(mbox, (void *)&rsp); if (rc) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "unexpected fatal error"; return rc; } /* Non Leaf Shaper */ cap->shaper_private_supported = true; cap->shaper_private_dual_rate_supported = (hw_lvl == NIX_TXSCH_LVL_TL1) ? false : true; cap->shaper_private_rate_min = MIN_SHAPER_RATE / 8; cap->shaper_private_rate_max = MAX_SHAPER_RATE / 8; /* Non Leaf Scheduler */ if (hw_lvl == NIX_TXSCH_LVL_MDQ) cap->nonleaf.sched_n_children_max = dev->tm_leaf_cnt; else cap->nonleaf.sched_n_children_max = rsp->schq[hw_lvl - 1]; cap->nonleaf.sched_sp_n_priorities_max = nix_max_prio(dev, hw_lvl) + 1; cap->nonleaf.sched_wfq_n_children_per_group_max = cap->nonleaf.sched_n_children_max; cap->nonleaf.sched_wfq_n_groups_max = 1; cap->nonleaf.sched_wfq_weight_max = MAX_SCHED_WEIGHT; if (hw_lvl == NIX_TXSCH_LVL_TL1) cap->stats_mask = RTE_TM_STATS_N_PKTS_RED_DROPPED | RTE_TM_STATS_N_BYTES_RED_DROPPED; return 0; } static int otx2_nix_tm_shaper_profile_add(struct rte_eth_dev *eth_dev, uint32_t profile_id, struct rte_tm_shaper_params *params, struct rte_tm_error *error) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_nix_tm_shaper_profile *profile; profile = nix_tm_shaper_profile_search(dev, profile_id); if (profile) { error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; error->message = "shaper profile ID exist"; return -EINVAL; } /* Committed rate and burst size can be enabled/disabled */ if (params->committed.size || params->committed.rate) { if (params->committed.size < MIN_SHAPER_BURST || params->committed.size > MAX_SHAPER_BURST) { error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE; return -EINVAL; } else if (!shaper_rate_to_nix(params->committed.rate * 8, NULL, NULL, NULL)) { error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE; error->message = "shaper committed rate invalid"; return -EINVAL; } } /* Peak rate and burst size can be enabled/disabled */ if (params->peak.size || params->peak.rate) { if (params->peak.size < MIN_SHAPER_BURST || params->peak.size > MAX_SHAPER_BURST) { error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE; return -EINVAL; } else if (!shaper_rate_to_nix(params->peak.rate * 8, NULL, NULL, NULL)) { error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE; error->message = "shaper peak rate invalid"; return -EINVAL; } } profile = rte_zmalloc("otx2_nix_tm_shaper_profile", sizeof(struct otx2_nix_tm_shaper_profile), 0); if (!profile) return -ENOMEM; profile->shaper_profile_id = profile_id; rte_memcpy(&profile->params, params, sizeof(struct rte_tm_shaper_params)); TAILQ_INSERT_TAIL(&dev->shaper_profile_list, profile, shaper); otx2_tm_dbg("Added TM shaper profile %u, " " pir %" PRIu64 " , pbs %" PRIu64 ", cir %" PRIu64 ", cbs %" PRIu64 " , adj %u", profile_id, params->peak.rate * 8, params->peak.size, params->committed.rate * 8, params->committed.size, params->pkt_length_adjust); /* Translate rate as bits per second */ profile->params.peak.rate = profile->params.peak.rate * 8; profile->params.committed.rate = profile->params.committed.rate * 8; /* Always use PIR for single rate shaping */ if (!params->peak.rate && params->committed.rate) { profile->params.peak = profile->params.committed; memset(&profile->params.committed, 0, sizeof(profile->params.committed)); } /* update min rate */ nix_tm_shaper_profile_update_min(dev); return 0; } static int otx2_nix_tm_shaper_profile_delete(struct rte_eth_dev *eth_dev, uint32_t profile_id, struct rte_tm_error *error) { struct otx2_nix_tm_shaper_profile *profile; struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); profile = nix_tm_shaper_profile_search(dev, profile_id); if (!profile) { error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; error->message = "shaper profile ID not exist"; return -EINVAL; } if (profile->reference_count) { error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE; error->message = "shaper profile in use"; return -EINVAL; } otx2_tm_dbg("Removing TM shaper profile %u", profile_id); TAILQ_REMOVE(&dev->shaper_profile_list, profile, shaper); rte_free(profile); /* update min rate */ nix_tm_shaper_profile_update_min(dev); return 0; } static int otx2_nix_tm_node_add(struct rte_eth_dev *eth_dev, uint32_t node_id, uint32_t parent_node_id, uint32_t priority, uint32_t weight, uint32_t lvl, struct rte_tm_node_params *params, struct rte_tm_error *error) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_nix_tm_node *parent_node; int rc, clear_on_fail = 0; uint32_t exp_next_lvl; uint16_t hw_lvl; /* we don't support dynamic updates */ if (dev->tm_flags & NIX_TM_COMMITTED) { error->type = RTE_TM_ERROR_TYPE_CAPABILITIES; error->message = "dynamic update not supported"; return -EIO; } /* Leaf nodes have to be same priority */ if (nix_tm_is_leaf(dev, lvl) && priority != 0) { error->type = RTE_TM_ERROR_TYPE_CAPABILITIES; error->message = "queue shapers must be priority 0"; return -EIO; } parent_node = nix_tm_node_search(dev, parent_node_id, true); /* find the right level */ if (lvl == RTE_TM_NODE_LEVEL_ID_ANY) { if (parent_node_id == RTE_TM_NODE_ID_NULL) { lvl = OTX2_TM_LVL_ROOT; } else if (parent_node) { lvl = parent_node->lvl + 1; } else { /* Neigher proper parent nor proper level id given */ error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; error->message = "invalid parent node id"; return -ERANGE; } } /* Translate rte_tm level id's to nix hw level id's */ hw_lvl = nix_tm_lvl2nix(dev, lvl); if (hw_lvl == NIX_TXSCH_LVL_CNT && !nix_tm_is_leaf(dev, lvl)) { error->type = RTE_TM_ERROR_TYPE_LEVEL_ID; error->message = "invalid level id"; return -ERANGE; } if (node_id < dev->tm_leaf_cnt) exp_next_lvl = NIX_TXSCH_LVL_SMQ; else exp_next_lvl = hw_lvl + 1; /* Check if there is no parent node yet */ if (hw_lvl != dev->otx2_tm_root_lvl && (!parent_node || parent_node->hw_lvl != exp_next_lvl)) { error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; error->message = "invalid parent node id"; return -EINVAL; } /* Check if a node already exists */ if (nix_tm_node_search(dev, node_id, true)) { error->type = RTE_TM_ERROR_TYPE_NODE_ID; error->message = "node already exists"; return -EINVAL; } /* Check if shaper profile exists for non leaf node */ if (!nix_tm_is_leaf(dev, lvl) && params->shaper_profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE && !nix_tm_shaper_profile_search(dev, params->shaper_profile_id)) { error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; error->message = "invalid shaper profile"; return -EINVAL; } /* Check if there is second DWRR already in siblings or holes in prio */ if (validate_prio(dev, lvl, parent_node_id, priority, error)) return -EINVAL; if (weight > MAX_SCHED_WEIGHT) { error->type = RTE_TM_ERROR_TYPE_NODE_WEIGHT; error->message = "max weight exceeded"; return -EINVAL; } rc = nix_tm_node_add_to_list(dev, node_id, parent_node_id, priority, weight, hw_lvl, lvl, true, params); if (rc) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; /* cleanup user added nodes */ if (clear_on_fail) nix_tm_free_resources(dev, NIX_TM_NODE_USER, NIX_TM_NODE_USER, false); error->message = "failed to add node"; return rc; } error->type = RTE_TM_ERROR_TYPE_NONE; return 0; } static int otx2_nix_tm_node_delete(struct rte_eth_dev *eth_dev, uint32_t node_id, struct rte_tm_error *error) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_nix_tm_node *tm_node, *child_node; struct otx2_nix_tm_shaper_profile *profile; uint32_t profile_id; /* we don't support dynamic updates yet */ if (dev->tm_flags & NIX_TM_COMMITTED) { error->type = RTE_TM_ERROR_TYPE_CAPABILITIES; error->message = "hierarchy exists"; return -EIO; } if (node_id == RTE_TM_NODE_ID_NULL) { error->type = RTE_TM_ERROR_TYPE_NODE_ID; error->message = "invalid node id"; return -EINVAL; } tm_node = nix_tm_node_search(dev, node_id, true); if (!tm_node) { error->type = RTE_TM_ERROR_TYPE_NODE_ID; error->message = "no such node"; return -EINVAL; } /* Check for any existing children */ TAILQ_FOREACH(child_node, &dev->node_list, node) { if (child_node->parent == tm_node) { error->type = RTE_TM_ERROR_TYPE_NODE_ID; error->message = "children exist"; return -EINVAL; } } /* Remove shaper profile reference */ profile_id = tm_node->params.shaper_profile_id; profile = nix_tm_shaper_profile_search(dev, profile_id); profile->reference_count--; TAILQ_REMOVE(&dev->node_list, tm_node, node); rte_free(tm_node); return 0; } static int nix_tm_node_suspend_resume(struct rte_eth_dev *eth_dev, uint32_t node_id, struct rte_tm_error *error, bool suspend) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_mbox *mbox = dev->mbox; struct otx2_nix_tm_node *tm_node; struct nix_txschq_config *req; uint16_t flags; int rc; tm_node = nix_tm_node_search(dev, node_id, true); if (!tm_node) { error->type = RTE_TM_ERROR_TYPE_NODE_ID; error->message = "no such node"; return -EINVAL; } if (!(dev->tm_flags & NIX_TM_COMMITTED)) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "hierarchy doesn't exist"; return -EINVAL; } flags = tm_node->flags; flags = suspend ? (flags & ~NIX_TM_NODE_ENABLED) : (flags | NIX_TM_NODE_ENABLED); if (tm_node->flags == flags) return 0; /* send mbox for state change */ req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox); req->lvl = tm_node->hw_lvl; req->num_regs = prepare_tm_sw_xoff(tm_node, suspend, req->reg, req->regval); rc = send_tm_reqval(mbox, req, error); if (!rc) tm_node->flags = flags; return rc; } static int otx2_nix_tm_node_suspend(struct rte_eth_dev *eth_dev, uint32_t node_id, struct rte_tm_error *error) { return nix_tm_node_suspend_resume(eth_dev, node_id, error, true); } static int otx2_nix_tm_node_resume(struct rte_eth_dev *eth_dev, uint32_t node_id, struct rte_tm_error *error) { return nix_tm_node_suspend_resume(eth_dev, node_id, error, false); } static int otx2_nix_tm_hierarchy_commit(struct rte_eth_dev *eth_dev, int clear_on_fail, struct rte_tm_error *error) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_nix_tm_node *tm_node; uint32_t leaf_cnt = 0; int rc; if (dev->tm_flags & NIX_TM_COMMITTED) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "hierarchy exists"; return -EINVAL; } /* Check if we have all the leaf nodes */ TAILQ_FOREACH(tm_node, &dev->node_list, node) { if (tm_node->flags & NIX_TM_NODE_USER && tm_node->id < dev->tm_leaf_cnt) leaf_cnt++; } if (leaf_cnt != dev->tm_leaf_cnt) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "incomplete hierarchy"; return -EINVAL; } /* * Disable xmit will be enabled when * new topology is available. */ rc = nix_xmit_disable(eth_dev); if (rc) { otx2_err("failed to disable TX, rc=%d", rc); return -EIO; } /* Delete default/ratelimit tree */ if (dev->tm_flags & (NIX_TM_DEFAULT_TREE | NIX_TM_RATE_LIMIT_TREE)) { rc = nix_tm_free_resources(dev, NIX_TM_NODE_USER, 0, false); if (rc) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "failed to free default resources"; return rc; } dev->tm_flags &= ~(NIX_TM_DEFAULT_TREE | NIX_TM_RATE_LIMIT_TREE); } /* Free up user alloc'ed resources */ rc = nix_tm_free_resources(dev, NIX_TM_NODE_USER, NIX_TM_NODE_USER, true); if (rc) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "failed to free user resources"; return rc; } rc = nix_tm_alloc_resources(eth_dev, true); if (rc) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "alloc resources failed"; /* TODO should we restore default config ? */ if (clear_on_fail) nix_tm_free_resources(dev, 0, 0, false); return rc; } error->type = RTE_TM_ERROR_TYPE_NONE; dev->tm_flags |= NIX_TM_COMMITTED; return 0; } static int otx2_nix_tm_node_shaper_update(struct rte_eth_dev *eth_dev, uint32_t node_id, uint32_t profile_id, struct rte_tm_error *error) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_nix_tm_shaper_profile *profile = NULL; struct otx2_mbox *mbox = dev->mbox; struct otx2_nix_tm_node *tm_node; struct nix_txschq_config *req; uint8_t k; int rc; tm_node = nix_tm_node_search(dev, node_id, true); if (!tm_node || nix_tm_is_leaf(dev, tm_node->lvl)) { error->type = RTE_TM_ERROR_TYPE_NODE_ID; error->message = "invalid node"; return -EINVAL; } if (profile_id == tm_node->params.shaper_profile_id) return 0; if (profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE) { profile = nix_tm_shaper_profile_search(dev, profile_id); if (!profile) { error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID; error->message = "shaper profile ID not exist"; return -EINVAL; } } tm_node->params.shaper_profile_id = profile_id; /* Nothing to do if not yet committed */ if (!(dev->tm_flags & NIX_TM_COMMITTED)) return 0; tm_node->flags &= ~NIX_TM_NODE_ENABLED; /* Flush the specific node with SW_XOFF */ req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox); req->lvl = tm_node->hw_lvl; k = prepare_tm_sw_xoff(tm_node, true, req->reg, req->regval); req->num_regs = k; rc = send_tm_reqval(mbox, req, error); if (rc) return rc; shaper_default_red_algo(dev, tm_node, profile); /* Update the PIR/CIR and clear SW XOFF */ req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox); req->lvl = tm_node->hw_lvl; k = prepare_tm_shaper_reg(tm_node, profile, req->reg, req->regval); k += prepare_tm_sw_xoff(tm_node, false, &req->reg[k], &req->regval[k]); req->num_regs = k; rc = send_tm_reqval(mbox, req, error); if (!rc) tm_node->flags |= NIX_TM_NODE_ENABLED; return rc; } static int otx2_nix_tm_node_parent_update(struct rte_eth_dev *eth_dev, uint32_t node_id, uint32_t new_parent_id, uint32_t priority, uint32_t weight, struct rte_tm_error *error) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_nix_tm_node *tm_node, *sibling; struct otx2_nix_tm_node *new_parent; struct nix_txschq_config *req; uint8_t k; int rc; if (!(dev->tm_flags & NIX_TM_COMMITTED)) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "hierarchy doesn't exist"; return -EINVAL; } tm_node = nix_tm_node_search(dev, node_id, true); if (!tm_node) { error->type = RTE_TM_ERROR_TYPE_NODE_ID; error->message = "no such node"; return -EINVAL; } /* Parent id valid only for non root nodes */ if (tm_node->hw_lvl != dev->otx2_tm_root_lvl) { new_parent = nix_tm_node_search(dev, new_parent_id, true); if (!new_parent) { error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; error->message = "no such parent node"; return -EINVAL; } /* Current support is only for dynamic weight update */ if (tm_node->parent != new_parent || tm_node->priority != priority) { error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID; error->message = "only weight update supported"; return -EINVAL; } } /* Skip if no change */ if (tm_node->weight == weight) return 0; tm_node->weight = weight; /* For leaf nodes, SQ CTX needs update */ if (nix_tm_is_leaf(dev, tm_node->lvl)) { /* Update SQ quantum data on the fly */ rc = nix_sq_sched_data(dev, tm_node, true); if (rc) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "sq sched data update failed"; return rc; } } else { /* XOFF Parent node */ req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox); req->lvl = tm_node->parent->hw_lvl; req->num_regs = prepare_tm_sw_xoff(tm_node->parent, true, req->reg, req->regval); rc = send_tm_reqval(dev->mbox, req, error); if (rc) return rc; /* XOFF this node and all other siblings */ req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox); req->lvl = tm_node->hw_lvl; k = 0; TAILQ_FOREACH(sibling, &dev->node_list, node) { if (sibling->parent != tm_node->parent) continue; k += prepare_tm_sw_xoff(sibling, true, &req->reg[k], &req->regval[k]); } req->num_regs = k; rc = send_tm_reqval(dev->mbox, req, error); if (rc) return rc; /* Update new weight for current node */ req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox); req->lvl = tm_node->hw_lvl; req->num_regs = prepare_tm_sched_reg(dev, tm_node, req->reg, req->regval); rc = send_tm_reqval(dev->mbox, req, error); if (rc) return rc; /* XON this node and all other siblings */ req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox); req->lvl = tm_node->hw_lvl; k = 0; TAILQ_FOREACH(sibling, &dev->node_list, node) { if (sibling->parent != tm_node->parent) continue; k += prepare_tm_sw_xoff(sibling, false, &req->reg[k], &req->regval[k]); } req->num_regs = k; rc = send_tm_reqval(dev->mbox, req, error); if (rc) return rc; /* XON Parent node */ req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox); req->lvl = tm_node->parent->hw_lvl; req->num_regs = prepare_tm_sw_xoff(tm_node->parent, false, req->reg, req->regval); rc = send_tm_reqval(dev->mbox, req, error); if (rc) return rc; } return 0; } static int otx2_nix_tm_node_stats_read(struct rte_eth_dev *eth_dev, uint32_t node_id, struct rte_tm_node_stats *stats, uint64_t *stats_mask, int clear, struct rte_tm_error *error) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_nix_tm_node *tm_node; uint64_t reg, val; int64_t *addr; int rc = 0; tm_node = nix_tm_node_search(dev, node_id, true); if (!tm_node) { error->type = RTE_TM_ERROR_TYPE_NODE_ID; error->message = "no such node"; return -EINVAL; } /* Stats support only for leaf node or TL1 root */ if (nix_tm_is_leaf(dev, tm_node->lvl)) { reg = (((uint64_t)tm_node->id) << 32); /* Packets */ addr = (int64_t *)(dev->base + NIX_LF_SQ_OP_PKTS); val = otx2_atomic64_add_nosync(reg, addr); if (val & OP_ERR) val = 0; stats->n_pkts = val - tm_node->last_pkts; /* Bytes */ addr = (int64_t *)(dev->base + NIX_LF_SQ_OP_OCTS); val = otx2_atomic64_add_nosync(reg, addr); if (val & OP_ERR) val = 0; stats->n_bytes = val - tm_node->last_bytes; if (clear) { tm_node->last_pkts = stats->n_pkts; tm_node->last_bytes = stats->n_bytes; } *stats_mask = RTE_TM_STATS_N_PKTS | RTE_TM_STATS_N_BYTES; } else if (tm_node->hw_lvl == NIX_TXSCH_LVL_TL1) { error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED; error->message = "stats read error"; /* RED Drop packets */ reg = NIX_AF_TL1X_DROPPED_PACKETS(tm_node->hw_id); rc = read_tm_reg(dev->mbox, reg, &val, NIX_TXSCH_LVL_TL1); if (rc) goto exit; stats->leaf.n_pkts_dropped[RTE_COLOR_RED] = val - tm_node->last_pkts; /* RED Drop bytes */ reg = NIX_AF_TL1X_DROPPED_BYTES(tm_node->hw_id); rc = read_tm_reg(dev->mbox, reg, &val, NIX_TXSCH_LVL_TL1); if (rc) goto exit; stats->leaf.n_bytes_dropped[RTE_COLOR_RED] = val - tm_node->last_bytes; /* Clear stats */ if (clear) { tm_node->last_pkts = stats->leaf.n_pkts_dropped[RTE_COLOR_RED]; tm_node->last_bytes = stats->leaf.n_bytes_dropped[RTE_COLOR_RED]; } *stats_mask = RTE_TM_STATS_N_PKTS_RED_DROPPED | RTE_TM_STATS_N_BYTES_RED_DROPPED; } else { error->type = RTE_TM_ERROR_TYPE_NODE_ID; error->message = "unsupported node"; rc = -EINVAL; } exit: return rc; } const struct rte_tm_ops otx2_tm_ops = { .node_type_get = otx2_nix_tm_node_type_get, .capabilities_get = otx2_nix_tm_capa_get, .level_capabilities_get = otx2_nix_tm_level_capa_get, .node_capabilities_get = otx2_nix_tm_node_capa_get, .shaper_profile_add = otx2_nix_tm_shaper_profile_add, .shaper_profile_delete = otx2_nix_tm_shaper_profile_delete, .node_add = otx2_nix_tm_node_add, .node_delete = otx2_nix_tm_node_delete, .node_suspend = otx2_nix_tm_node_suspend, .node_resume = otx2_nix_tm_node_resume, .hierarchy_commit = otx2_nix_tm_hierarchy_commit, .node_shaper_update = otx2_nix_tm_node_shaper_update, .node_parent_update = otx2_nix_tm_node_parent_update, .node_stats_read = otx2_nix_tm_node_stats_read, }; static int nix_tm_prepare_default_tree(struct rte_eth_dev *eth_dev) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); uint32_t def = eth_dev->data->nb_tx_queues; struct rte_tm_node_params params; uint32_t leaf_parent, i; int rc = 0, leaf_level; /* Default params */ memset(¶ms, 0, sizeof(params)); params.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE; if (nix_tm_have_tl1_access(dev)) { dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL1; rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL1, OTX2_TM_LVL_ROOT, false, ¶ms); if (rc) goto exit; rc = nix_tm_node_add_to_list(dev, def + 1, def, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL2, OTX2_TM_LVL_SCH1, false, ¶ms); if (rc) goto exit; rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL3, OTX2_TM_LVL_SCH2, false, ¶ms); if (rc) goto exit; rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL4, OTX2_TM_LVL_SCH3, false, ¶ms); if (rc) goto exit; rc = nix_tm_node_add_to_list(dev, def + 4, def + 3, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_SMQ, OTX2_TM_LVL_SCH4, false, ¶ms); if (rc) goto exit; leaf_parent = def + 4; leaf_level = OTX2_TM_LVL_QUEUE; } else { dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL2; rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL2, OTX2_TM_LVL_ROOT, false, ¶ms); if (rc) goto exit; rc = nix_tm_node_add_to_list(dev, def + 1, def, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL3, OTX2_TM_LVL_SCH1, false, ¶ms); if (rc) goto exit; rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL4, OTX2_TM_LVL_SCH2, false, ¶ms); if (rc) goto exit; rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_SMQ, OTX2_TM_LVL_SCH3, false, ¶ms); if (rc) goto exit; leaf_parent = def + 3; leaf_level = OTX2_TM_LVL_SCH4; } /* Add leaf nodes */ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { rc = nix_tm_node_add_to_list(dev, i, leaf_parent, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_CNT, leaf_level, false, ¶ms); if (rc) break; } exit: return rc; } void otx2_nix_tm_conf_init(struct rte_eth_dev *eth_dev) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); TAILQ_INIT(&dev->node_list); TAILQ_INIT(&dev->shaper_profile_list); dev->tm_rate_min = 1E9; /* 1Gbps */ } int otx2_nix_tm_init_default(struct rte_eth_dev *eth_dev) { struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); uint16_t sq_cnt = eth_dev->data->nb_tx_queues; int rc; /* Free up all resources already held */ rc = nix_tm_free_resources(dev, 0, 0, false); if (rc) { otx2_err("Failed to freeup existing resources,rc=%d", rc); return rc; } /* Clear shaper profiles */ nix_tm_clear_shaper_profiles(dev); dev->tm_flags = NIX_TM_DEFAULT_TREE; /* Disable TL1 Static Priority when VF's are enabled * as otherwise VF's TL2 reallocation will be needed * runtime to support a specific topology of PF. */ if (pci_dev->max_vfs) dev->tm_flags |= NIX_TM_TL1_NO_SP; rc = nix_tm_prepare_default_tree(eth_dev); if (rc != 0) return rc; rc = nix_tm_alloc_resources(eth_dev, false); if (rc != 0) return rc; dev->tm_leaf_cnt = sq_cnt; return 0; } static int nix_tm_prepare_rate_limited_tree(struct rte_eth_dev *eth_dev) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); uint32_t def = eth_dev->data->nb_tx_queues; struct rte_tm_node_params params; uint32_t leaf_parent, i, rc = 0; memset(¶ms, 0, sizeof(params)); if (nix_tm_have_tl1_access(dev)) { dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL1; rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL1, OTX2_TM_LVL_ROOT, false, ¶ms); if (rc) goto error; rc = nix_tm_node_add_to_list(dev, def + 1, def, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL2, OTX2_TM_LVL_SCH1, false, ¶ms); if (rc) goto error; rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL3, OTX2_TM_LVL_SCH2, false, ¶ms); if (rc) goto error; rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL4, OTX2_TM_LVL_SCH3, false, ¶ms); if (rc) goto error; leaf_parent = def + 3; /* Add per queue SMQ nodes */ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { rc = nix_tm_node_add_to_list(dev, leaf_parent + 1 + i, leaf_parent, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_SMQ, OTX2_TM_LVL_SCH4, false, ¶ms); if (rc) goto error; } /* Add leaf nodes */ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { rc = nix_tm_node_add_to_list(dev, i, leaf_parent + 1 + i, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_CNT, OTX2_TM_LVL_QUEUE, false, ¶ms); if (rc) goto error; } return 0; } dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL2; rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL2, OTX2_TM_LVL_ROOT, false, ¶ms); if (rc) goto error; rc = nix_tm_node_add_to_list(dev, def + 1, def, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL3, OTX2_TM_LVL_SCH1, false, ¶ms); if (rc) goto error; rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL4, OTX2_TM_LVL_SCH2, false, ¶ms); if (rc) goto error; leaf_parent = def + 2; /* Add per queue SMQ nodes */ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { rc = nix_tm_node_add_to_list(dev, leaf_parent + 1 + i, leaf_parent, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_SMQ, OTX2_TM_LVL_SCH3, false, ¶ms); if (rc) goto error; } /* Add leaf nodes */ for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { rc = nix_tm_node_add_to_list(dev, i, leaf_parent + 1 + i, 0, DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_CNT, OTX2_TM_LVL_SCH4, false, ¶ms); if (rc) break; } error: return rc; } static int otx2_nix_tm_rate_limit_mdq(struct rte_eth_dev *eth_dev, struct otx2_nix_tm_node *tm_node, uint64_t tx_rate) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); struct otx2_nix_tm_shaper_profile profile; struct otx2_mbox *mbox = dev->mbox; volatile uint64_t *reg, *regval; struct nix_txschq_config *req; uint16_t flags; uint8_t k = 0; int rc; flags = tm_node->flags; req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox); req->lvl = NIX_TXSCH_LVL_MDQ; reg = req->reg; regval = req->regval; if (tx_rate == 0) { k += prepare_tm_sw_xoff(tm_node, true, ®[k], ®val[k]); flags &= ~NIX_TM_NODE_ENABLED; goto exit; } if (!(flags & NIX_TM_NODE_ENABLED)) { k += prepare_tm_sw_xoff(tm_node, false, ®[k], ®val[k]); flags |= NIX_TM_NODE_ENABLED; } /* Use only PIR for rate limit */ memset(&profile, 0, sizeof(profile)); profile.params.peak.rate = tx_rate; /* Minimum burst of ~4us Bytes of Tx */ profile.params.peak.size = RTE_MAX(NIX_MAX_HW_FRS, (4ull * tx_rate) / (1E6 * 8)); if (!dev->tm_rate_min || dev->tm_rate_min > tx_rate) dev->tm_rate_min = tx_rate; k += prepare_tm_shaper_reg(tm_node, &profile, ®[k], ®val[k]); exit: req->num_regs = k; rc = otx2_mbox_process(mbox); if (rc) return rc; tm_node->flags = flags; return 0; } int otx2_nix_tm_set_queue_rate_limit(struct rte_eth_dev *eth_dev, uint16_t queue_idx, uint16_t tx_rate_mbps) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); uint64_t tx_rate = tx_rate_mbps * (uint64_t)1E6; struct otx2_nix_tm_node *tm_node; int rc; /* Check for supported revisions */ if (otx2_dev_is_95xx_Ax(dev) || otx2_dev_is_96xx_Ax(dev)) return -EINVAL; if (queue_idx >= eth_dev->data->nb_tx_queues) return -EINVAL; if (!(dev->tm_flags & NIX_TM_DEFAULT_TREE) && !(dev->tm_flags & NIX_TM_RATE_LIMIT_TREE)) goto error; if ((dev->tm_flags & NIX_TM_DEFAULT_TREE) && eth_dev->data->nb_tx_queues > 1) { /* For TM topology change ethdev needs to be stopped */ if (eth_dev->data->dev_started) return -EBUSY; /* * Disable xmit will be enabled when * new topology is available. */ rc = nix_xmit_disable(eth_dev); if (rc) { otx2_err("failed to disable TX, rc=%d", rc); return -EIO; } rc = nix_tm_free_resources(dev, 0, 0, false); if (rc < 0) { otx2_tm_dbg("failed to free default resources, rc %d", rc); return -EIO; } rc = nix_tm_prepare_rate_limited_tree(eth_dev); if (rc < 0) { otx2_tm_dbg("failed to prepare tm tree, rc=%d", rc); return rc; } rc = nix_tm_alloc_resources(eth_dev, true); if (rc != 0) { otx2_tm_dbg("failed to allocate tm tree, rc=%d", rc); return rc; } dev->tm_flags &= ~NIX_TM_DEFAULT_TREE; dev->tm_flags |= NIX_TM_RATE_LIMIT_TREE; } tm_node = nix_tm_node_search(dev, queue_idx, false); /* check if we found a valid leaf node */ if (!tm_node || !nix_tm_is_leaf(dev, tm_node->lvl) || !tm_node->parent || tm_node->parent->hw_id == UINT32_MAX) return -EIO; return otx2_nix_tm_rate_limit_mdq(eth_dev, tm_node->parent, tx_rate); error: otx2_tm_dbg("Unsupported TM tree 0x%0x", dev->tm_flags); return -EINVAL; } int otx2_nix_tm_ops_get(struct rte_eth_dev *eth_dev, void *arg) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); if (!arg) return -EINVAL; /* Check for supported revisions */ if (otx2_dev_is_95xx_Ax(dev) || otx2_dev_is_96xx_Ax(dev)) return -EINVAL; *(const void **)arg = &otx2_tm_ops; return 0; } int otx2_nix_tm_fini(struct rte_eth_dev *eth_dev) { struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev); int rc; /* Xmit is assumed to be disabled */ /* Free up resources already held */ rc = nix_tm_free_resources(dev, 0, 0, false); if (rc) { otx2_err("Failed to freeup existing resources,rc=%d", rc); return rc; } /* Clear shaper profiles */ nix_tm_clear_shaper_profiles(dev); dev->tm_flags = 0; return 0; } int otx2_nix_tm_get_leaf_data(struct otx2_eth_dev *dev, uint16_t sq, uint32_t *rr_quantum, uint16_t *smq) { struct otx2_nix_tm_node *tm_node; int rc; /* 0..sq_cnt-1 are leaf nodes */ if (sq >= dev->tm_leaf_cnt) return -EINVAL; /* Search for internal node first */ tm_node = nix_tm_node_search(dev, sq, false); if (!tm_node) tm_node = nix_tm_node_search(dev, sq, true); /* Check if we found a valid leaf node */ if (!tm_node || !nix_tm_is_leaf(dev, tm_node->lvl) || !tm_node->parent || tm_node->parent->hw_id == UINT32_MAX) { return -EIO; } /* Get SMQ Id of leaf node's parent */ *smq = tm_node->parent->hw_id; *rr_quantum = NIX_TM_WEIGHT_TO_RR_QUANTUM(tm_node->weight); rc = nix_smq_xoff(dev, tm_node->parent, false); if (rc) return rc; tm_node->flags |= NIX_TM_NODE_ENABLED; return 0; }