From 76cb841cb886eef6b3bee341a2266c76578724ad Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 6 May 2024 03:02:30 +0200 Subject: Adding upstream version 4.19.249. Signed-off-by: Daniel Baumann --- drivers/infiniband/hw/bnxt_re/Kconfig | 10 + drivers/infiniband/hw/bnxt_re/Makefile | 7 + drivers/infiniband/hw/bnxt_re/bnxt_re.h | 191 ++ drivers/infiniband/hw/bnxt_re/hw_counters.c | 239 ++ drivers/infiniband/hw/bnxt_re/hw_counters.h | 101 + drivers/infiniband/hw/bnxt_re/ib_verbs.c | 3792 +++++++++++++++++++++++++++ drivers/infiniband/hw/bnxt_re/ib_verbs.h | 225 ++ drivers/infiniband/hw/bnxt_re/main.c | 1664 ++++++++++++ drivers/infiniband/hw/bnxt_re/qplib_fp.c | 2845 ++++++++++++++++++++ drivers/infiniband/hw/bnxt_re/qplib_fp.h | 529 ++++ drivers/infiniband/hw/bnxt_re/qplib_rcfw.c | 775 ++++++ drivers/infiniband/hw/bnxt_re/qplib_rcfw.h | 222 ++ drivers/infiniband/hw/bnxt_re/qplib_res.c | 831 ++++++ drivers/infiniband/hw/bnxt_re/qplib_res.h | 229 ++ drivers/infiniband/hw/bnxt_re/qplib_sp.c | 850 ++++++ drivers/infiniband/hw/bnxt_re/qplib_sp.h | 253 ++ drivers/infiniband/hw/bnxt_re/roce_hsi.h | 2969 +++++++++++++++++++++ 17 files changed, 15732 insertions(+) create mode 100644 drivers/infiniband/hw/bnxt_re/Kconfig create mode 100644 drivers/infiniband/hw/bnxt_re/Makefile create mode 100644 drivers/infiniband/hw/bnxt_re/bnxt_re.h create mode 100644 drivers/infiniband/hw/bnxt_re/hw_counters.c create mode 100644 drivers/infiniband/hw/bnxt_re/hw_counters.h create mode 100644 drivers/infiniband/hw/bnxt_re/ib_verbs.c create mode 100644 drivers/infiniband/hw/bnxt_re/ib_verbs.h create mode 100644 drivers/infiniband/hw/bnxt_re/main.c create mode 100644 drivers/infiniband/hw/bnxt_re/qplib_fp.c create mode 100644 drivers/infiniband/hw/bnxt_re/qplib_fp.h create mode 100644 drivers/infiniband/hw/bnxt_re/qplib_rcfw.c create mode 100644 drivers/infiniband/hw/bnxt_re/qplib_rcfw.h create mode 100644 drivers/infiniband/hw/bnxt_re/qplib_res.c create mode 100644 drivers/infiniband/hw/bnxt_re/qplib_res.h create mode 100644 drivers/infiniband/hw/bnxt_re/qplib_sp.c create mode 100644 drivers/infiniband/hw/bnxt_re/qplib_sp.h create mode 100644 drivers/infiniband/hw/bnxt_re/roce_hsi.h (limited to 'drivers/infiniband/hw/bnxt_re') diff --git a/drivers/infiniband/hw/bnxt_re/Kconfig b/drivers/infiniband/hw/bnxt_re/Kconfig new file mode 100644 index 000000000..18f5ed082 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/Kconfig @@ -0,0 +1,10 @@ +config INFINIBAND_BNXT_RE + tristate "Broadcom Netxtreme HCA support" + depends on ETHERNET && NETDEVICES && PCI && INET && DCB + depends on MAY_USE_DEVLINK + select NET_VENDOR_BROADCOM + select BNXT + ---help--- + This driver supports Broadcom NetXtreme-E 10/25/40/50 gigabit + RoCE HCAs. To compile this driver as a module, choose M here: + the module will be called bnxt_re. diff --git a/drivers/infiniband/hw/bnxt_re/Makefile b/drivers/infiniband/hw/bnxt_re/Makefile new file mode 100644 index 000000000..6e3bc25cc --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 + +ccflags-y := -Idrivers/net/ethernet/broadcom/bnxt +obj-$(CONFIG_INFINIBAND_BNXT_RE) += bnxt_re.o +bnxt_re-y := main.o ib_verbs.o \ + qplib_res.o qplib_rcfw.o \ + qplib_sp.o qplib_fp.o hw_counters.o diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h new file mode 100644 index 000000000..802942ade --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h @@ -0,0 +1,191 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Slow Path Operators (header) + * + */ + +#ifndef __BNXT_RE_H__ +#define __BNXT_RE_H__ +#define ROCE_DRV_MODULE_NAME "bnxt_re" +#define ROCE_DRV_MODULE_VERSION "1.0.0" + +#define BNXT_RE_DESC "Broadcom NetXtreme-C/E RoCE Driver" +#define BNXT_RE_PAGE_SHIFT_4K (12) +#define BNXT_RE_PAGE_SHIFT_8K (13) +#define BNXT_RE_PAGE_SHIFT_64K (16) +#define BNXT_RE_PAGE_SHIFT_2M (21) +#define BNXT_RE_PAGE_SHIFT_8M (23) +#define BNXT_RE_PAGE_SHIFT_1G (30) + +#define BNXT_RE_PAGE_SIZE_4K BIT(BNXT_RE_PAGE_SHIFT_4K) +#define BNXT_RE_PAGE_SIZE_8K BIT(BNXT_RE_PAGE_SHIFT_8K) +#define BNXT_RE_PAGE_SIZE_64K BIT(BNXT_RE_PAGE_SHIFT_64K) +#define BNXT_RE_PAGE_SIZE_2M BIT(BNXT_RE_PAGE_SHIFT_2M) +#define BNXT_RE_PAGE_SIZE_8M BIT(BNXT_RE_PAGE_SHIFT_8M) +#define BNXT_RE_PAGE_SIZE_1G BIT(BNXT_RE_PAGE_SHIFT_1G) + +#define BNXT_RE_MAX_MR_SIZE_LOW BIT_ULL(BNXT_RE_PAGE_SHIFT_1G) +#define BNXT_RE_MAX_MR_SIZE_HIGH BIT_ULL(39) +#define BNXT_RE_MAX_MR_SIZE BNXT_RE_MAX_MR_SIZE_HIGH + +#define BNXT_RE_MAX_QPC_COUNT (64 * 1024) +#define BNXT_RE_MAX_MRW_COUNT (64 * 1024) +#define BNXT_RE_MAX_SRQC_COUNT (64 * 1024) +#define BNXT_RE_MAX_CQ_COUNT (64 * 1024) +#define BNXT_RE_MAX_MRW_COUNT_64K (64 * 1024) +#define BNXT_RE_MAX_MRW_COUNT_256K (256 * 1024) + +/* Number of MRs to reserve for PF, leaving remainder for VFs */ +#define BNXT_RE_RESVD_MR_FOR_PF (32 * 1024) +#define BNXT_RE_MAX_GID_PER_VF 128 + +/* + * Percentage of resources of each type reserved for PF. + * Remaining resources are divided equally among VFs. + * [0, 100] + */ +#define BNXT_RE_PCT_RSVD_FOR_PF 50 + +#define BNXT_RE_UD_QP_HW_STALL 0x400000 + +#define BNXT_RE_RQ_WQE_THRESHOLD 32 + +/* + * Setting the default ack delay value to 16, which means + * the default timeout is approx. 260ms(4 usec * 2 ^(timeout)) + */ + +#define BNXT_RE_DEFAULT_ACK_DELAY 16 + +struct bnxt_re_work { + struct work_struct work; + unsigned long event; + struct bnxt_re_dev *rdev; + struct net_device *vlan_dev; +}; + +struct bnxt_re_sqp_entries { + struct bnxt_qplib_sge sge; + u64 wrid; + /* For storing the actual qp1 cqe */ + struct bnxt_qplib_cqe cqe; + struct bnxt_re_qp *qp1_qp; +}; + +#define BNXT_RE_MIN_MSIX 2 +#define BNXT_RE_MAX_MSIX 9 +#define BNXT_RE_AEQ_IDX 0 +#define BNXT_RE_NQ_IDX 1 + +struct bnxt_re_dev { + struct ib_device ibdev; + struct list_head list; + unsigned long flags; +#define BNXT_RE_FLAG_NETDEV_REGISTERED 0 +#define BNXT_RE_FLAG_IBDEV_REGISTERED 1 +#define BNXT_RE_FLAG_GOT_MSIX 2 +#define BNXT_RE_FLAG_HAVE_L2_REF 3 +#define BNXT_RE_FLAG_RCFW_CHANNEL_EN 4 +#define BNXT_RE_FLAG_QOS_WORK_REG 5 +#define BNXT_RE_FLAG_RESOURCES_ALLOCATED 7 +#define BNXT_RE_FLAG_RESOURCES_INITIALIZED 8 +#define BNXT_RE_FLAG_ISSUE_ROCE_STATS 29 + struct net_device *netdev; + unsigned int version, major, minor; + struct bnxt_en_dev *en_dev; + struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX]; + int num_msix; + + int id; + + struct delayed_work worker; + u8 cur_prio_map; + u8 active_speed; + u8 active_width; + + /* FP Notification Queue (CQ & SRQ) */ + struct tasklet_struct nq_task; + + /* RCFW Channel */ + struct bnxt_qplib_rcfw rcfw; + + /* NQ */ + struct bnxt_qplib_nq nq[BNXT_RE_MAX_MSIX]; + + /* Device Resources */ + struct bnxt_qplib_dev_attr dev_attr; + struct bnxt_qplib_ctx qplib_ctx; + struct bnxt_qplib_res qplib_res; + struct bnxt_qplib_dpi dpi_privileged; + + atomic_t qp_count; + struct mutex qp_lock; /* protect qp list */ + struct list_head qp_list; + + atomic_t cq_count; + atomic_t srq_count; + atomic_t mr_count; + atomic_t mw_count; + atomic_t sched_count; + /* Max of 2 lossless traffic class supported per port */ + u16 cosq[2]; + + /* QP for for handling QP1 packets */ + u32 sqp_id; + struct bnxt_re_qp *qp1_sqp; + struct bnxt_re_ah *sqp_ah; + struct bnxt_re_sqp_entries sqp_tbl[1024]; + atomic_t nq_alloc_cnt; + u32 is_virtfn; + u32 num_vfs; + struct bnxt_qplib_roce_stats stats; +}; + +#define to_bnxt_re_dev(ptr, member) \ + container_of((ptr), struct bnxt_re_dev, member) + +#define BNXT_RE_ROCE_V1_PACKET 0 +#define BNXT_RE_ROCEV2_IPV4_PACKET 2 +#define BNXT_RE_ROCEV2_IPV6_PACKET 3 + +static inline struct device *rdev_to_dev(struct bnxt_re_dev *rdev) +{ + if (rdev) + return &rdev->ibdev.dev; + return NULL; +} + +#endif diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.c b/drivers/infiniband/hw/bnxt_re/hw_counters.c new file mode 100644 index 000000000..77416bc61 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/hw_counters.c @@ -0,0 +1,239 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Statistics + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "bnxt_ulp.h" +#include "roce_hsi.h" +#include "qplib_res.h" +#include "qplib_sp.h" +#include "qplib_fp.h" +#include "qplib_rcfw.h" +#include "bnxt_re.h" +#include "hw_counters.h" + +static const char * const bnxt_re_stat_name[] = { + [BNXT_RE_ACTIVE_QP] = "active_qps", + [BNXT_RE_ACTIVE_SRQ] = "active_srqs", + [BNXT_RE_ACTIVE_CQ] = "active_cqs", + [BNXT_RE_ACTIVE_MR] = "active_mrs", + [BNXT_RE_ACTIVE_MW] = "active_mws", + [BNXT_RE_RX_PKTS] = "rx_pkts", + [BNXT_RE_RX_BYTES] = "rx_bytes", + [BNXT_RE_TX_PKTS] = "tx_pkts", + [BNXT_RE_TX_BYTES] = "tx_bytes", + [BNXT_RE_RECOVERABLE_ERRORS] = "recoverable_errors", + [BNXT_RE_TO_RETRANSMITS] = "to_retransmits", + [BNXT_RE_SEQ_ERR_NAKS_RCVD] = "seq_err_naks_rcvd", + [BNXT_RE_MAX_RETRY_EXCEEDED] = "max_retry_exceeded", + [BNXT_RE_RNR_NAKS_RCVD] = "rnr_naks_rcvd", + [BNXT_RE_MISSING_RESP] = "missin_resp", + [BNXT_RE_UNRECOVERABLE_ERR] = "unrecoverable_err", + [BNXT_RE_BAD_RESP_ERR] = "bad_resp_err", + [BNXT_RE_LOCAL_QP_OP_ERR] = "local_qp_op_err", + [BNXT_RE_LOCAL_PROTECTION_ERR] = "local_protection_err", + [BNXT_RE_MEM_MGMT_OP_ERR] = "mem_mgmt_op_err", + [BNXT_RE_REMOTE_INVALID_REQ_ERR] = "remote_invalid_req_err", + [BNXT_RE_REMOTE_ACCESS_ERR] = "remote_access_err", + [BNXT_RE_REMOTE_OP_ERR] = "remote_op_err", + [BNXT_RE_DUP_REQ] = "dup_req", + [BNXT_RE_RES_EXCEED_MAX] = "res_exceed_max", + [BNXT_RE_RES_LENGTH_MISMATCH] = "res_length_mismatch", + [BNXT_RE_RES_EXCEEDS_WQE] = "res_exceeds_wqe", + [BNXT_RE_RES_OPCODE_ERR] = "res_opcode_err", + [BNXT_RE_RES_RX_INVALID_RKEY] = "res_rx_invalid_rkey", + [BNXT_RE_RES_RX_DOMAIN_ERR] = "res_rx_domain_err", + [BNXT_RE_RES_RX_NO_PERM] = "res_rx_no_perm", + [BNXT_RE_RES_RX_RANGE_ERR] = "res_rx_range_err", + [BNXT_RE_RES_TX_INVALID_RKEY] = "res_tx_invalid_rkey", + [BNXT_RE_RES_TX_DOMAIN_ERR] = "res_tx_domain_err", + [BNXT_RE_RES_TX_NO_PERM] = "res_tx_no_perm", + [BNXT_RE_RES_TX_RANGE_ERR] = "res_tx_range_err", + [BNXT_RE_RES_IRRQ_OFLOW] = "res_irrq_oflow", + [BNXT_RE_RES_UNSUP_OPCODE] = "res_unsup_opcode", + [BNXT_RE_RES_UNALIGNED_ATOMIC] = "res_unaligned_atomic", + [BNXT_RE_RES_REM_INV_ERR] = "res_rem_inv_err", + [BNXT_RE_RES_MEM_ERROR] = "res_mem_err", + [BNXT_RE_RES_SRQ_ERR] = "res_srq_err", + [BNXT_RE_RES_CMP_ERR] = "res_cmp_err", + [BNXT_RE_RES_INVALID_DUP_RKEY] = "res_invalid_dup_rkey", + [BNXT_RE_RES_WQE_FORMAT_ERR] = "res_wqe_format_err", + [BNXT_RE_RES_CQ_LOAD_ERR] = "res_cq_load_err", + [BNXT_RE_RES_SRQ_LOAD_ERR] = "res_srq_load_err", + [BNXT_RE_RES_TX_PCI_ERR] = "res_tx_pci_err", + [BNXT_RE_RES_RX_PCI_ERR] = "res_rx_pci_err" +}; + +int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev, + struct rdma_hw_stats *stats, + u8 port, int index) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct ctx_hw_stats *bnxt_re_stats = rdev->qplib_ctx.stats.dma; + int rc = 0; + + if (!port || !stats) + return -EINVAL; + + stats->value[BNXT_RE_ACTIVE_QP] = atomic_read(&rdev->qp_count); + stats->value[BNXT_RE_ACTIVE_SRQ] = atomic_read(&rdev->srq_count); + stats->value[BNXT_RE_ACTIVE_CQ] = atomic_read(&rdev->cq_count); + stats->value[BNXT_RE_ACTIVE_MR] = atomic_read(&rdev->mr_count); + stats->value[BNXT_RE_ACTIVE_MW] = atomic_read(&rdev->mw_count); + if (bnxt_re_stats) { + stats->value[BNXT_RE_RECOVERABLE_ERRORS] = + le64_to_cpu(bnxt_re_stats->tx_bcast_pkts); + stats->value[BNXT_RE_RX_PKTS] = + le64_to_cpu(bnxt_re_stats->rx_ucast_pkts); + stats->value[BNXT_RE_RX_BYTES] = + le64_to_cpu(bnxt_re_stats->rx_ucast_bytes); + stats->value[BNXT_RE_TX_PKTS] = + le64_to_cpu(bnxt_re_stats->tx_ucast_pkts); + stats->value[BNXT_RE_TX_BYTES] = + le64_to_cpu(bnxt_re_stats->tx_ucast_bytes); + } + if (test_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags)) { + rc = bnxt_qplib_get_roce_stats(&rdev->rcfw, &rdev->stats); + if (rc) + clear_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, + &rdev->flags); + stats->value[BNXT_RE_TO_RETRANSMITS] = + rdev->stats.to_retransmits; + stats->value[BNXT_RE_SEQ_ERR_NAKS_RCVD] = + rdev->stats.seq_err_naks_rcvd; + stats->value[BNXT_RE_MAX_RETRY_EXCEEDED] = + rdev->stats.max_retry_exceeded; + stats->value[BNXT_RE_RNR_NAKS_RCVD] = + rdev->stats.rnr_naks_rcvd; + stats->value[BNXT_RE_MISSING_RESP] = + rdev->stats.missing_resp; + stats->value[BNXT_RE_UNRECOVERABLE_ERR] = + rdev->stats.unrecoverable_err; + stats->value[BNXT_RE_BAD_RESP_ERR] = + rdev->stats.bad_resp_err; + stats->value[BNXT_RE_LOCAL_QP_OP_ERR] = + rdev->stats.local_qp_op_err; + stats->value[BNXT_RE_LOCAL_PROTECTION_ERR] = + rdev->stats.local_protection_err; + stats->value[BNXT_RE_MEM_MGMT_OP_ERR] = + rdev->stats.mem_mgmt_op_err; + stats->value[BNXT_RE_REMOTE_INVALID_REQ_ERR] = + rdev->stats.remote_invalid_req_err; + stats->value[BNXT_RE_REMOTE_ACCESS_ERR] = + rdev->stats.remote_access_err; + stats->value[BNXT_RE_REMOTE_OP_ERR] = + rdev->stats.remote_op_err; + stats->value[BNXT_RE_DUP_REQ] = + rdev->stats.dup_req; + stats->value[BNXT_RE_RES_EXCEED_MAX] = + rdev->stats.res_exceed_max; + stats->value[BNXT_RE_RES_LENGTH_MISMATCH] = + rdev->stats.res_length_mismatch; + stats->value[BNXT_RE_RES_EXCEEDS_WQE] = + rdev->stats.res_exceeds_wqe; + stats->value[BNXT_RE_RES_OPCODE_ERR] = + rdev->stats.res_opcode_err; + stats->value[BNXT_RE_RES_RX_INVALID_RKEY] = + rdev->stats.res_rx_invalid_rkey; + stats->value[BNXT_RE_RES_RX_DOMAIN_ERR] = + rdev->stats.res_rx_domain_err; + stats->value[BNXT_RE_RES_RX_NO_PERM] = + rdev->stats.res_rx_no_perm; + stats->value[BNXT_RE_RES_RX_RANGE_ERR] = + rdev->stats.res_rx_range_err; + stats->value[BNXT_RE_RES_TX_INVALID_RKEY] = + rdev->stats.res_tx_invalid_rkey; + stats->value[BNXT_RE_RES_TX_DOMAIN_ERR] = + rdev->stats.res_tx_domain_err; + stats->value[BNXT_RE_RES_TX_NO_PERM] = + rdev->stats.res_tx_no_perm; + stats->value[BNXT_RE_RES_TX_RANGE_ERR] = + rdev->stats.res_tx_range_err; + stats->value[BNXT_RE_RES_IRRQ_OFLOW] = + rdev->stats.res_irrq_oflow; + stats->value[BNXT_RE_RES_UNSUP_OPCODE] = + rdev->stats.res_unsup_opcode; + stats->value[BNXT_RE_RES_UNALIGNED_ATOMIC] = + rdev->stats.res_unaligned_atomic; + stats->value[BNXT_RE_RES_REM_INV_ERR] = + rdev->stats.res_rem_inv_err; + stats->value[BNXT_RE_RES_MEM_ERROR] = + rdev->stats.res_mem_error; + stats->value[BNXT_RE_RES_SRQ_ERR] = + rdev->stats.res_srq_err; + stats->value[BNXT_RE_RES_CMP_ERR] = + rdev->stats.res_cmp_err; + stats->value[BNXT_RE_RES_INVALID_DUP_RKEY] = + rdev->stats.res_invalid_dup_rkey; + stats->value[BNXT_RE_RES_WQE_FORMAT_ERR] = + rdev->stats.res_wqe_format_err; + stats->value[BNXT_RE_RES_CQ_LOAD_ERR] = + rdev->stats.res_cq_load_err; + stats->value[BNXT_RE_RES_SRQ_LOAD_ERR] = + rdev->stats.res_srq_load_err; + stats->value[BNXT_RE_RES_TX_PCI_ERR] = + rdev->stats.res_tx_pci_err; + stats->value[BNXT_RE_RES_RX_PCI_ERR] = + rdev->stats.res_rx_pci_err; + } + + return ARRAY_SIZE(bnxt_re_stat_name); +} + +struct rdma_hw_stats *bnxt_re_ib_alloc_hw_stats(struct ib_device *ibdev, + u8 port_num) +{ + BUILD_BUG_ON(ARRAY_SIZE(bnxt_re_stat_name) != BNXT_RE_NUM_COUNTERS); + /* We support only per port stats */ + if (!port_num) + return NULL; + + return rdma_alloc_hw_stats_struct(bnxt_re_stat_name, + ARRAY_SIZE(bnxt_re_stat_name), + RDMA_HW_STATS_DEFAULT_LIFESPAN); +} diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.h b/drivers/infiniband/hw/bnxt_re/hw_counters.h new file mode 100644 index 000000000..a01a92271 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/hw_counters.h @@ -0,0 +1,101 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Statistics (header) + * + */ + +#ifndef __BNXT_RE_HW_STATS_H__ +#define __BNXT_RE_HW_STATS_H__ + +enum bnxt_re_hw_stats { + BNXT_RE_ACTIVE_QP, + BNXT_RE_ACTIVE_SRQ, + BNXT_RE_ACTIVE_CQ, + BNXT_RE_ACTIVE_MR, + BNXT_RE_ACTIVE_MW, + BNXT_RE_RX_PKTS, + BNXT_RE_RX_BYTES, + BNXT_RE_TX_PKTS, + BNXT_RE_TX_BYTES, + BNXT_RE_RECOVERABLE_ERRORS, + BNXT_RE_TO_RETRANSMITS, + BNXT_RE_SEQ_ERR_NAKS_RCVD, + BNXT_RE_MAX_RETRY_EXCEEDED, + BNXT_RE_RNR_NAKS_RCVD, + BNXT_RE_MISSING_RESP, + BNXT_RE_UNRECOVERABLE_ERR, + BNXT_RE_BAD_RESP_ERR, + BNXT_RE_LOCAL_QP_OP_ERR, + BNXT_RE_LOCAL_PROTECTION_ERR, + BNXT_RE_MEM_MGMT_OP_ERR, + BNXT_RE_REMOTE_INVALID_REQ_ERR, + BNXT_RE_REMOTE_ACCESS_ERR, + BNXT_RE_REMOTE_OP_ERR, + BNXT_RE_DUP_REQ, + BNXT_RE_RES_EXCEED_MAX, + BNXT_RE_RES_LENGTH_MISMATCH, + BNXT_RE_RES_EXCEEDS_WQE, + BNXT_RE_RES_OPCODE_ERR, + BNXT_RE_RES_RX_INVALID_RKEY, + BNXT_RE_RES_RX_DOMAIN_ERR, + BNXT_RE_RES_RX_NO_PERM, + BNXT_RE_RES_RX_RANGE_ERR, + BNXT_RE_RES_TX_INVALID_RKEY, + BNXT_RE_RES_TX_DOMAIN_ERR, + BNXT_RE_RES_TX_NO_PERM, + BNXT_RE_RES_TX_RANGE_ERR, + BNXT_RE_RES_IRRQ_OFLOW, + BNXT_RE_RES_UNSUP_OPCODE, + BNXT_RE_RES_UNALIGNED_ATOMIC, + BNXT_RE_RES_REM_INV_ERR, + BNXT_RE_RES_MEM_ERROR, + BNXT_RE_RES_SRQ_ERR, + BNXT_RE_RES_CMP_ERR, + BNXT_RE_RES_INVALID_DUP_RKEY, + BNXT_RE_RES_WQE_FORMAT_ERR, + BNXT_RE_RES_CQ_LOAD_ERR, + BNXT_RE_RES_SRQ_LOAD_ERR, + BNXT_RE_RES_TX_PCI_ERR, + BNXT_RE_RES_RX_PCI_ERR, + BNXT_RE_NUM_COUNTERS +}; + +struct rdma_hw_stats *bnxt_re_ib_alloc_hw_stats(struct ib_device *ibdev, + u8 port_num); +int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev, + struct rdma_hw_stats *stats, + u8 port, int index); +#endif /* __BNXT_RE_HW_STATS_H__ */ diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c new file mode 100644 index 000000000..f8c9caa8a --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -0,0 +1,3792 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: IB Verbs interpreter + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "bnxt_ulp.h" + +#include "roce_hsi.h" +#include "qplib_res.h" +#include "qplib_sp.h" +#include "qplib_fp.h" +#include "qplib_rcfw.h" + +#include "bnxt_re.h" +#include "ib_verbs.h" +#include + +static int __from_ib_access_flags(int iflags) +{ + int qflags = 0; + + if (iflags & IB_ACCESS_LOCAL_WRITE) + qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; + if (iflags & IB_ACCESS_REMOTE_READ) + qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; + if (iflags & IB_ACCESS_REMOTE_WRITE) + qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; + if (iflags & IB_ACCESS_REMOTE_ATOMIC) + qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; + if (iflags & IB_ACCESS_MW_BIND) + qflags |= BNXT_QPLIB_ACCESS_MW_BIND; + if (iflags & IB_ZERO_BASED) + qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; + if (iflags & IB_ACCESS_ON_DEMAND) + qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; + return qflags; +}; + +static enum ib_access_flags __to_ib_access_flags(int qflags) +{ + enum ib_access_flags iflags = 0; + + if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) + iflags |= IB_ACCESS_LOCAL_WRITE; + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) + iflags |= IB_ACCESS_REMOTE_WRITE; + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) + iflags |= IB_ACCESS_REMOTE_READ; + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) + iflags |= IB_ACCESS_REMOTE_ATOMIC; + if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) + iflags |= IB_ACCESS_MW_BIND; + if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) + iflags |= IB_ZERO_BASED; + if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) + iflags |= IB_ACCESS_ON_DEMAND; + return iflags; +}; + +static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list, + struct bnxt_qplib_sge *sg_list, int num) +{ + int i, total = 0; + + for (i = 0; i < num; i++) { + sg_list[i].addr = ib_sg_list[i].addr; + sg_list[i].lkey = ib_sg_list[i].lkey; + sg_list[i].size = ib_sg_list[i].length; + total += sg_list[i].size; + } + return total; +} + +/* Device */ +struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct net_device *netdev = NULL; + + rcu_read_lock(); + if (rdev) + netdev = rdev->netdev; + if (netdev) + dev_hold(netdev); + + rcu_read_unlock(); + return netdev; +} + +int bnxt_re_query_device(struct ib_device *ibdev, + struct ib_device_attr *ib_attr, + struct ib_udata *udata) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; + + memset(ib_attr, 0, sizeof(*ib_attr)); + memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, + min(sizeof(dev_attr->fw_ver), + sizeof(ib_attr->fw_ver))); + bnxt_qplib_get_guid(rdev->netdev->dev_addr, + (u8 *)&ib_attr->sys_image_guid); + ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; + ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M; + + ib_attr->vendor_id = rdev->en_dev->pdev->vendor; + ib_attr->vendor_part_id = rdev->en_dev->pdev->device; + ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; + ib_attr->max_qp = dev_attr->max_qp; + ib_attr->max_qp_wr = dev_attr->max_qp_wqes; + ib_attr->device_cap_flags = + IB_DEVICE_CURR_QP_STATE_MOD + | IB_DEVICE_RC_RNR_NAK_GEN + | IB_DEVICE_SHUTDOWN_PORT + | IB_DEVICE_SYS_IMAGE_GUID + | IB_DEVICE_LOCAL_DMA_LKEY + | IB_DEVICE_RESIZE_MAX_WR + | IB_DEVICE_PORT_ACTIVE_EVENT + | IB_DEVICE_N_NOTIFY_CQ + | IB_DEVICE_MEM_WINDOW + | IB_DEVICE_MEM_WINDOW_TYPE_2B + | IB_DEVICE_MEM_MGT_EXTENSIONS; + ib_attr->max_send_sge = dev_attr->max_qp_sges; + ib_attr->max_recv_sge = dev_attr->max_qp_sges; + ib_attr->max_sge_rd = dev_attr->max_qp_sges; + ib_attr->max_cq = dev_attr->max_cq; + ib_attr->max_cqe = dev_attr->max_cq_wqes; + ib_attr->max_mr = dev_attr->max_mr; + ib_attr->max_pd = dev_attr->max_pd; + ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; + ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; + ib_attr->atomic_cap = IB_ATOMIC_NONE; + ib_attr->masked_atomic_cap = IB_ATOMIC_NONE; + + ib_attr->max_ee_rd_atom = 0; + ib_attr->max_res_rd_atom = 0; + ib_attr->max_ee_init_rd_atom = 0; + ib_attr->max_ee = 0; + ib_attr->max_rdd = 0; + ib_attr->max_mw = dev_attr->max_mw; + ib_attr->max_raw_ipv6_qp = 0; + ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; + ib_attr->max_mcast_grp = 0; + ib_attr->max_mcast_qp_attach = 0; + ib_attr->max_total_mcast_qp_attach = 0; + ib_attr->max_ah = dev_attr->max_ah; + + ib_attr->max_fmr = 0; + ib_attr->max_map_per_fmr = 0; + + ib_attr->max_srq = dev_attr->max_srq; + ib_attr->max_srq_wr = dev_attr->max_srq_wqes; + ib_attr->max_srq_sge = dev_attr->max_srq_sges; + + ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; + + ib_attr->max_pkeys = 1; + ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; + return 0; +} + +int bnxt_re_modify_device(struct ib_device *ibdev, + int device_modify_mask, + struct ib_device_modify *device_modify) +{ + switch (device_modify_mask) { + case IB_DEVICE_MODIFY_SYS_IMAGE_GUID: + /* Modify the GUID requires the modification of the GID table */ + /* GUID should be made as READ-ONLY */ + break; + case IB_DEVICE_MODIFY_NODE_DESC: + /* Node Desc should be made as READ-ONLY */ + break; + default: + break; + } + return 0; +} + +/* Port */ +int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, + struct ib_port_attr *port_attr) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; + + memset(port_attr, 0, sizeof(*port_attr)); + + if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) { + port_attr->state = IB_PORT_ACTIVE; + port_attr->phys_state = 5; + } else { + port_attr->state = IB_PORT_DOWN; + port_attr->phys_state = 3; + } + port_attr->max_mtu = IB_MTU_4096; + port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu); + port_attr->gid_tbl_len = dev_attr->max_sgid; + port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | + IB_PORT_DEVICE_MGMT_SUP | + IB_PORT_VENDOR_CLASS_SUP; + port_attr->ip_gids = true; + + port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; + port_attr->bad_pkey_cntr = 0; + port_attr->qkey_viol_cntr = 0; + port_attr->pkey_tbl_len = dev_attr->max_pkey; + port_attr->lid = 0; + port_attr->sm_lid = 0; + port_attr->lmc = 0; + port_attr->max_vl_num = 4; + port_attr->sm_sl = 0; + port_attr->subnet_timeout = 0; + port_attr->init_type_reply = 0; + port_attr->active_speed = rdev->active_speed; + port_attr->active_width = rdev->active_width; + + return 0; +} + +int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable) +{ + struct ib_port_attr port_attr; + + if (bnxt_re_query_port(ibdev, port_num, &port_attr)) + return -EINVAL; + + immutable->pkey_tbl_len = port_attr.pkey_tbl_len; + immutable->gid_tbl_len = port_attr.gid_tbl_len; + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; + immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP; + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + return 0; +} + +void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + + snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d", + rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1], + rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]); +} + +int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, + u16 index, u16 *pkey) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + + /* Ignore port_num */ + + memset(pkey, 0, sizeof(*pkey)); + return bnxt_qplib_get_pkey(&rdev->qplib_res, + &rdev->qplib_res.pkey_tbl, index, pkey); +} + +int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, + int index, union ib_gid *gid) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + int rc = 0; + + /* Ignore port_num */ + memset(gid, 0, sizeof(*gid)); + rc = bnxt_qplib_get_sgid(&rdev->qplib_res, + &rdev->qplib_res.sgid_tbl, index, + (struct bnxt_qplib_gid *)gid); + return rc; +} + +int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context) +{ + int rc = 0; + struct bnxt_re_gid_ctx *ctx, **ctx_tbl; + struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); + struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; + struct bnxt_qplib_gid *gid_to_del; + + /* Delete the entry from the hardware */ + ctx = *context; + if (!ctx) + return -EINVAL; + + if (sgid_tbl && sgid_tbl->active) { + if (ctx->idx >= sgid_tbl->max) + return -EINVAL; + gid_to_del = &sgid_tbl->tbl[ctx->idx]; + /* DEL_GID is called in WQ context(netdevice_event_work_handler) + * or via the ib_unregister_device path. In the former case QP1 + * may not be destroyed yet, in which case just return as FW + * needs that entry to be present and will fail it's deletion. + * We could get invoked again after QP1 is destroyed OR get an + * ADD_GID call with a different GID value for the same index + * where we issue MODIFY_GID cmd to update the GID entry -- TBD + */ + if (ctx->idx == 0 && + rdma_link_local_addr((struct in6_addr *)gid_to_del) && + ctx->refcnt == 1 && rdev->qp1_sqp) { + dev_dbg(rdev_to_dev(rdev), + "Trying to delete GID0 while QP1 is alive\n"); + return -EFAULT; + } + ctx->refcnt--; + if (!ctx->refcnt) { + rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, true); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to remove GID: %#x", rc); + } else { + ctx_tbl = sgid_tbl->ctx; + ctx_tbl[ctx->idx] = NULL; + kfree(ctx); + } + } + } else { + return -EINVAL; + } + return rc; +} + +int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context) +{ + int rc; + u32 tbl_idx = 0; + u16 vlan_id = 0xFFFF; + struct bnxt_re_gid_ctx *ctx, **ctx_tbl; + struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); + struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; + + if ((attr->ndev) && is_vlan_dev(attr->ndev)) + vlan_id = vlan_dev_vlan_id(attr->ndev); + + rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid, + rdev->qplib_res.netdev->dev_addr, + vlan_id, true, &tbl_idx); + if (rc == -EALREADY) { + ctx_tbl = sgid_tbl->ctx; + ctx_tbl[tbl_idx]->refcnt++; + *context = ctx_tbl[tbl_idx]; + return 0; + } + + if (rc < 0) { + dev_err(rdev_to_dev(rdev), "Failed to add GID: %#x", rc); + return rc; + } + + ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + ctx_tbl = sgid_tbl->ctx; + ctx->idx = tbl_idx; + ctx->refcnt = 1; + ctx_tbl[tbl_idx] = ctx; + *context = ctx; + + return rc; +} + +enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, + u8 port_num) +{ + return IB_LINK_LAYER_ETHERNET; +} + +#define BNXT_RE_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE) + +static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd) +{ + struct bnxt_re_fence_data *fence = &pd->fence; + struct ib_mr *ib_mr = &fence->mr->ib_mr; + struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; + + memset(wqe, 0, sizeof(*wqe)); + wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; + wqe->wr_id = BNXT_QPLIB_FENCE_WRID; + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + wqe->bind.zero_based = false; + wqe->bind.parent_l_key = ib_mr->lkey; + wqe->bind.va = (u64)(unsigned long)fence->va; + wqe->bind.length = fence->size; + wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); + wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; + + /* Save the initial rkey in fence structure for now; + * wqe->bind.r_key will be set at (re)bind time. + */ + fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); +} + +static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp) +{ + struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, + qplib_qp); + struct ib_pd *ib_pd = qp->ib_qp.pd; + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); + struct bnxt_re_fence_data *fence = &pd->fence; + struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; + struct bnxt_qplib_swqe wqe; + int rc; + + memcpy(&wqe, fence_wqe, sizeof(wqe)); + wqe.bind.r_key = fence->bind_rkey; + fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); + + dev_dbg(rdev_to_dev(qp->rdev), + "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", + wqe.bind.r_key, qp->qplib_qp.id, pd); + rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); + if (rc) { + dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n"); + return rc; + } + bnxt_qplib_post_send_db(&qp->qplib_qp); + + return rc; +} + +static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd) +{ + struct bnxt_re_fence_data *fence = &pd->fence; + struct bnxt_re_dev *rdev = pd->rdev; + struct device *dev = &rdev->en_dev->pdev->dev; + struct bnxt_re_mr *mr = fence->mr; + + if (fence->mw) { + bnxt_re_dealloc_mw(fence->mw); + fence->mw = NULL; + } + if (mr) { + if (mr->ib_mr.rkey) + bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, + true); + if (mr->ib_mr.lkey) + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); + kfree(mr); + fence->mr = NULL; + } + if (fence->dma_addr) { + dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES, + DMA_BIDIRECTIONAL); + fence->dma_addr = 0; + } +} + +static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd) +{ + int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND; + struct bnxt_re_fence_data *fence = &pd->fence; + struct bnxt_re_dev *rdev = pd->rdev; + struct device *dev = &rdev->en_dev->pdev->dev; + struct bnxt_re_mr *mr = NULL; + dma_addr_t dma_addr = 0; + struct ib_mw *mw; + u64 pbl_tbl; + int rc; + + dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES, + DMA_BIDIRECTIONAL); + rc = dma_mapping_error(dev, dma_addr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n"); + rc = -EIO; + fence->dma_addr = 0; + goto fail; + } + fence->dma_addr = dma_addr; + + /* Allocate a MR */ + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + rc = -ENOMEM; + goto fail; + } + fence->mr = mr; + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n"); + goto fail; + } + + /* Register MR */ + mr->ib_mr.lkey = mr->qplib_mr.lkey; + mr->qplib_mr.va = (u64)(unsigned long)fence->va; + mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES; + pbl_tbl = dma_addr; + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl_tbl, + BNXT_RE_FENCE_PBL_SIZE, false, PAGE_SIZE); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n"); + goto fail; + } + mr->ib_mr.rkey = mr->qplib_mr.rkey; + + /* Create a fence MW only for kernel consumers */ + mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL); + if (IS_ERR(mw)) { + dev_err(rdev_to_dev(rdev), + "Failed to create fence-MW for PD: %p\n", pd); + rc = PTR_ERR(mw); + goto fail; + } + fence->mw = mw; + + bnxt_re_create_fence_wqe(pd); + return 0; + +fail: + bnxt_re_destroy_fence_mr(pd); + return rc; +} + +/* Protection Domains */ +int bnxt_re_dealloc_pd(struct ib_pd *ib_pd) +{ + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); + struct bnxt_re_dev *rdev = pd->rdev; + int rc; + + bnxt_re_destroy_fence_mr(pd); + + if (pd->qplib_pd.id) { + rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res, + &rdev->qplib_res.pd_tbl, + &pd->qplib_pd); + if (rc) + dev_err(rdev_to_dev(rdev), "Failed to deallocate HW PD"); + } + + kfree(pd); + return 0; +} + +struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev, + struct ib_ucontext *ucontext, + struct ib_udata *udata) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_re_ucontext *ucntx = container_of(ucontext, + struct bnxt_re_ucontext, + ib_uctx); + struct bnxt_re_pd *pd; + int rc; + + pd = kzalloc(sizeof(*pd), GFP_KERNEL); + if (!pd) + return ERR_PTR(-ENOMEM); + + pd->rdev = rdev; + if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) { + dev_err(rdev_to_dev(rdev), "Failed to allocate HW PD"); + rc = -ENOMEM; + goto fail; + } + + if (udata) { + struct bnxt_re_pd_resp resp; + + if (!ucntx->dpi.dbr) { + /* Allocate DPI in alloc_pd to avoid failing of + * ibv_devinfo and family of application when DPIs + * are depleted. + */ + if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, + &ucntx->dpi, ucntx)) { + rc = -ENOMEM; + goto dbfail; + } + } + + resp.pdid = pd->qplib_pd.id; + /* Still allow mapping this DBR to the new user PD. */ + resp.dpi = ucntx->dpi.dpi; + resp.dbr = (u64)ucntx->dpi.umdbr; + + rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to copy user response\n"); + goto dbfail; + } + } + + if (!udata) + if (bnxt_re_create_fence_mr(pd)) + dev_warn(rdev_to_dev(rdev), + "Failed to create Fence-MR\n"); + return &pd->ib_pd; +dbfail: + (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, + &pd->qplib_pd); +fail: + kfree(pd); + return ERR_PTR(rc); +} + +/* Address Handles */ +int bnxt_re_destroy_ah(struct ib_ah *ib_ah) +{ + struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); + struct bnxt_re_dev *rdev = ah->rdev; + int rc; + + rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to destroy HW AH"); + return rc; + } + kfree(ah); + return 0; +} + +struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd, + struct rdma_ah_attr *ah_attr, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_ah *ah; + const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); + int rc; + u8 nw_type; + + if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) { + dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set"); + return ERR_PTR(-EINVAL); + } + ah = kzalloc(sizeof(*ah), GFP_ATOMIC); + if (!ah) + return ERR_PTR(-ENOMEM); + + ah->rdev = rdev; + ah->qplib_ah.pd = &pd->qplib_pd; + + /* Supply the configuration for the HW */ + memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw, + sizeof(union ib_gid)); + /* + * If RoCE V2 is enabled, stack will have two entries for + * each GID entry. Avoiding this duplicte entry in HW. Dividing + * the GID index by 2 for RoCE V2 + */ + ah->qplib_ah.sgid_index = grh->sgid_index / 2; + ah->qplib_ah.host_sgid_index = grh->sgid_index; + ah->qplib_ah.traffic_class = grh->traffic_class; + ah->qplib_ah.flow_label = grh->flow_label; + ah->qplib_ah.hop_limit = grh->hop_limit; + ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr); + if (ib_pd->uobject && + !rdma_is_multicast_addr((struct in6_addr *) + grh->dgid.raw) && + !rdma_link_local_addr((struct in6_addr *) + grh->dgid.raw)) { + const struct ib_gid_attr *sgid_attr; + + sgid_attr = grh->sgid_attr; + /* Get network header type for this GID */ + nw_type = rdma_gid_attr_network_type(sgid_attr); + switch (nw_type) { + case RDMA_NETWORK_IPV4: + ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; + break; + case RDMA_NETWORK_IPV6: + ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; + break; + default: + ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V1; + break; + } + } + + memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN); + rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH"); + goto fail; + } + + /* Write AVID to shared page. */ + if (ib_pd->uobject) { + struct ib_ucontext *ib_uctx = ib_pd->uobject->context; + struct bnxt_re_ucontext *uctx; + unsigned long flag; + u32 *wrptr; + + uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx); + spin_lock_irqsave(&uctx->sh_lock, flag); + wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT); + *wrptr = ah->qplib_ah.id; + wmb(); /* make sure cache is updated. */ + spin_unlock_irqrestore(&uctx->sh_lock, flag); + } + + return &ah->ib_ah; + +fail: + kfree(ah); + return ERR_PTR(rc); +} + +int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) +{ + return 0; +} + +int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) +{ + struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); + + ah_attr->type = ib_ah->type; + rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl); + memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN); + rdma_ah_set_grh(ah_attr, NULL, 0, + ah->qplib_ah.host_sgid_index, + 0, ah->qplib_ah.traffic_class); + rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data); + rdma_ah_set_port_num(ah_attr, 1); + rdma_ah_set_static_rate(ah_attr, 0); + return 0; +} + +unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) + __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock) +{ + unsigned long flags; + + spin_lock_irqsave(&qp->scq->cq_lock, flags); + if (qp->rcq != qp->scq) + spin_lock(&qp->rcq->cq_lock); + else + __acquire(&qp->rcq->cq_lock); + + return flags; +} + +void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, + unsigned long flags) + __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock) +{ + if (qp->rcq != qp->scq) + spin_unlock(&qp->rcq->cq_lock); + else + __release(&qp->rcq->cq_lock); + spin_unlock_irqrestore(&qp->scq->cq_lock, flags); +} + +/* Queue Pairs */ +int bnxt_re_destroy_qp(struct ib_qp *ib_qp) +{ + struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_re_dev *rdev = qp->rdev; + int rc; + unsigned int flags; + + bnxt_qplib_flush_cqn_wq(&qp->qplib_qp); + rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP"); + return rc; + } + + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_clean_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); + + if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) { + rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, + &rdev->sqp_ah->qplib_ah); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to destroy HW AH for shadow QP"); + return rc; + } + + bnxt_qplib_clean_qp(&qp->qplib_qp); + rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, + &rdev->qp1_sqp->qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to destroy Shadow QP"); + return rc; + } + bnxt_qplib_free_qp_res(&rdev->qplib_res, + &rdev->qp1_sqp->qplib_qp); + mutex_lock(&rdev->qp_lock); + list_del(&rdev->qp1_sqp->list); + atomic_dec(&rdev->qp_count); + mutex_unlock(&rdev->qp_lock); + + kfree(rdev->sqp_ah); + kfree(rdev->qp1_sqp); + rdev->qp1_sqp = NULL; + rdev->sqp_ah = NULL; + } + + if (!IS_ERR_OR_NULL(qp->rumem)) + ib_umem_release(qp->rumem); + if (!IS_ERR_OR_NULL(qp->sumem)) + ib_umem_release(qp->sumem); + + mutex_lock(&rdev->qp_lock); + list_del(&qp->list); + atomic_dec(&rdev->qp_count); + mutex_unlock(&rdev->qp_lock); + kfree(qp); + return 0; +} + +static u8 __from_ib_qp_type(enum ib_qp_type type) +{ + switch (type) { + case IB_QPT_GSI: + return CMDQ_CREATE_QP1_TYPE_GSI; + case IB_QPT_RC: + return CMDQ_CREATE_QP_TYPE_RC; + case IB_QPT_UD: + return CMDQ_CREATE_QP_TYPE_UD; + default: + return IB_QPT_MAX; + } +} + +static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, + struct bnxt_re_qp *qp, struct ib_udata *udata) +{ + struct bnxt_re_qp_req ureq; + struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; + struct ib_umem *umem; + int bytes = 0; + struct ib_ucontext *context = pd->ib_pd.uobject->context; + struct bnxt_re_ucontext *cntx = container_of(context, + struct bnxt_re_ucontext, + ib_uctx); + if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) + return -EFAULT; + + bytes = (qplib_qp->sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); + /* Consider mapping PSN search memory only for RC QPs. */ + if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) + bytes += (qplib_qp->sq.max_wqe * sizeof(struct sq_psn_search)); + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get(context, ureq.qpsva, bytes, + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(umem)) + return PTR_ERR(umem); + + qp->sumem = umem; + qplib_qp->sq.sglist = umem->sg_head.sgl; + qplib_qp->sq.nmap = umem->nmap; + qplib_qp->qp_handle = ureq.qp_handle; + + if (!qp->qplib_qp.srq) { + bytes = (qplib_qp->rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get(context, ureq.qprva, bytes, + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(umem)) + goto rqfail; + qp->rumem = umem; + qplib_qp->rq.sglist = umem->sg_head.sgl; + qplib_qp->rq.nmap = umem->nmap; + } + + qplib_qp->dpi = &cntx->dpi; + return 0; +rqfail: + ib_umem_release(qp->sumem); + qp->sumem = NULL; + qplib_qp->sq.sglist = NULL; + qplib_qp->sq.nmap = 0; + + return PTR_ERR(umem); +} + +static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah + (struct bnxt_re_pd *pd, + struct bnxt_qplib_res *qp1_res, + struct bnxt_qplib_qp *qp1_qp) +{ + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_ah *ah; + union ib_gid sgid; + int rc; + + ah = kzalloc(sizeof(*ah), GFP_KERNEL); + if (!ah) + return NULL; + + ah->rdev = rdev; + ah->qplib_ah.pd = &pd->qplib_pd; + + rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); + if (rc) + goto fail; + + /* supply the dgid data same as sgid */ + memcpy(ah->qplib_ah.dgid.data, &sgid.raw, + sizeof(union ib_gid)); + ah->qplib_ah.sgid_index = 0; + + ah->qplib_ah.traffic_class = 0; + ah->qplib_ah.flow_label = 0; + ah->qplib_ah.hop_limit = 1; + ah->qplib_ah.sl = 0; + /* Have DMAC same as SMAC */ + ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr); + + rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to allocate HW AH for Shadow QP"); + goto fail; + } + + return ah; + +fail: + kfree(ah); + return NULL; +} + +static struct bnxt_re_qp *bnxt_re_create_shadow_qp + (struct bnxt_re_pd *pd, + struct bnxt_qplib_res *qp1_res, + struct bnxt_qplib_qp *qp1_qp) +{ + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_qp *qp; + int rc; + + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) + return NULL; + + qp->rdev = rdev; + + /* Initialize the shadow QP structure from the QP1 values */ + ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); + + qp->qplib_qp.pd = &pd->qplib_pd; + qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); + qp->qplib_qp.type = IB_QPT_UD; + + qp->qplib_qp.max_inline_data = 0; + qp->qplib_qp.sig_type = true; + + /* Shadow QP SQ depth should be same as QP1 RQ depth */ + qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; + qp->qplib_qp.sq.max_sge = 2; + /* Q full delta can be 1 since it is internal QP */ + qp->qplib_qp.sq.q_full_delta = 1; + + qp->qplib_qp.scq = qp1_qp->scq; + qp->qplib_qp.rcq = qp1_qp->rcq; + + qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; + qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; + /* Q full delta can be 1 since it is internal QP */ + qp->qplib_qp.rq.q_full_delta = 1; + + qp->qplib_qp.mtu = qp1_qp->mtu; + + qp->qplib_qp.sq_hdr_buf_size = 0; + qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; + qp->qplib_qp.dpi = &rdev->dpi_privileged; + + rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); + if (rc) + goto fail; + + rdev->sqp_id = qp->qplib_qp.id; + + spin_lock_init(&qp->sq_lock); + INIT_LIST_HEAD(&qp->list); + mutex_lock(&rdev->qp_lock); + list_add_tail(&qp->list, &rdev->qp_list); + atomic_inc(&rdev->qp_count); + mutex_unlock(&rdev->qp_lock); + return qp; +fail: + kfree(qp); + return NULL; +} + +struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd, + struct ib_qp_init_attr *qp_init_attr, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; + struct bnxt_re_qp *qp; + struct bnxt_re_cq *cq; + struct bnxt_re_srq *srq; + int rc, entries; + + if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || + (qp_init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || + (qp_init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || + (qp_init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || + (qp_init_attr->cap.max_inline_data > dev_attr->max_inline_data)) + return ERR_PTR(-EINVAL); + + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) + return ERR_PTR(-ENOMEM); + + qp->rdev = rdev; + ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); + qp->qplib_qp.pd = &pd->qplib_pd; + qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); + qp->qplib_qp.type = __from_ib_qp_type(qp_init_attr->qp_type); + if (qp->qplib_qp.type == IB_QPT_MAX) { + dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported", + qp->qplib_qp.type); + rc = -EINVAL; + goto fail; + } + qp->qplib_qp.max_inline_data = qp_init_attr->cap.max_inline_data; + qp->qplib_qp.sig_type = ((qp_init_attr->sq_sig_type == + IB_SIGNAL_ALL_WR) ? true : false); + + qp->qplib_qp.sq.max_sge = qp_init_attr->cap.max_send_sge; + if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) + qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; + + if (qp_init_attr->send_cq) { + cq = container_of(qp_init_attr->send_cq, struct bnxt_re_cq, + ib_cq); + if (!cq) { + dev_err(rdev_to_dev(rdev), "Send CQ not found"); + rc = -EINVAL; + goto fail; + } + qp->qplib_qp.scq = &cq->qplib_cq; + qp->scq = cq; + } + + if (qp_init_attr->recv_cq) { + cq = container_of(qp_init_attr->recv_cq, struct bnxt_re_cq, + ib_cq); + if (!cq) { + dev_err(rdev_to_dev(rdev), "Receive CQ not found"); + rc = -EINVAL; + goto fail; + } + qp->qplib_qp.rcq = &cq->qplib_cq; + qp->rcq = cq; + } + + if (qp_init_attr->srq) { + srq = container_of(qp_init_attr->srq, struct bnxt_re_srq, + ib_srq); + if (!srq) { + dev_err(rdev_to_dev(rdev), "SRQ not found"); + rc = -EINVAL; + goto fail; + } + qp->qplib_qp.srq = &srq->qplib_srq; + qp->qplib_qp.rq.max_wqe = 0; + } else { + /* Allocate 1 more than what's provided so posting max doesn't + * mean empty + */ + entries = roundup_pow_of_two(qp_init_attr->cap.max_recv_wr + 1); + qp->qplib_qp.rq.max_wqe = min_t(u32, entries, + dev_attr->max_qp_wqes + 1); + + qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - + qp_init_attr->cap.max_recv_wr; + + qp->qplib_qp.rq.max_sge = qp_init_attr->cap.max_recv_sge; + if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) + qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; + } + + qp->qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); + + if (qp_init_attr->qp_type == IB_QPT_GSI) { + /* Allocate 1 more than what's provided */ + entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1); + qp->qplib_qp.sq.max_wqe = min_t(u32, entries, + dev_attr->max_qp_wqes + 1); + qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - + qp_init_attr->cap.max_send_wr; + qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; + if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges) + qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges; + qp->qplib_qp.sq.max_sge++; + if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges) + qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges; + + qp->qplib_qp.rq_hdr_buf_size = + BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; + + qp->qplib_qp.sq_hdr_buf_size = + BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2; + qp->qplib_qp.dpi = &rdev->dpi_privileged; + rc = bnxt_qplib_create_qp1(&rdev->qplib_res, &qp->qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to create HW QP1"); + goto fail; + } + /* Create a shadow QP to handle the QP1 traffic */ + rdev->qp1_sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, + &qp->qplib_qp); + if (!rdev->qp1_sqp) { + rc = -EINVAL; + dev_err(rdev_to_dev(rdev), + "Failed to create Shadow QP for QP1"); + goto qp_destroy; + } + rdev->sqp_ah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, + &qp->qplib_qp); + if (!rdev->sqp_ah) { + bnxt_qplib_destroy_qp(&rdev->qplib_res, + &rdev->qp1_sqp->qplib_qp); + rc = -EINVAL; + dev_err(rdev_to_dev(rdev), + "Failed to create AH entry for ShadowQP"); + goto qp_destroy; + } + + } else { + /* Allocate 128 + 1 more than what's provided */ + entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + + BNXT_QPLIB_RESERVED_QP_WRS + 1); + qp->qplib_qp.sq.max_wqe = min_t(u32, entries, + dev_attr->max_qp_wqes + + BNXT_QPLIB_RESERVED_QP_WRS + 1); + qp->qplib_qp.sq.q_full_delta = BNXT_QPLIB_RESERVED_QP_WRS + 1; + + /* + * Reserving one slot for Phantom WQE. Application can + * post one extra entry in this case. But allowing this to avoid + * unexpected Queue full condition + */ + + qp->qplib_qp.sq.q_full_delta -= 1; + + qp->qplib_qp.max_rd_atomic = dev_attr->max_qp_rd_atom; + qp->qplib_qp.max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; + if (udata) { + rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); + if (rc) + goto fail; + } else { + qp->qplib_qp.dpi = &rdev->dpi_privileged; + } + + rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to create HW QP"); + goto free_umem; + } + } + + qp->ib_qp.qp_num = qp->qplib_qp.id; + spin_lock_init(&qp->sq_lock); + spin_lock_init(&qp->rq_lock); + + if (udata) { + struct bnxt_re_qp_resp resp; + + resp.qpid = qp->ib_qp.qp_num; + resp.rsvd = 0; + rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to copy QP udata"); + goto qp_destroy; + } + } + INIT_LIST_HEAD(&qp->list); + mutex_lock(&rdev->qp_lock); + list_add_tail(&qp->list, &rdev->qp_list); + atomic_inc(&rdev->qp_count); + mutex_unlock(&rdev->qp_lock); + + return &qp->ib_qp; +qp_destroy: + bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); +free_umem: + if (udata) { + if (qp->rumem) + ib_umem_release(qp->rumem); + if (qp->sumem) + ib_umem_release(qp->sumem); + } +fail: + kfree(qp); + return ERR_PTR(rc); +} + +static u8 __from_ib_qp_state(enum ib_qp_state state) +{ + switch (state) { + case IB_QPS_RESET: + return CMDQ_MODIFY_QP_NEW_STATE_RESET; + case IB_QPS_INIT: + return CMDQ_MODIFY_QP_NEW_STATE_INIT; + case IB_QPS_RTR: + return CMDQ_MODIFY_QP_NEW_STATE_RTR; + case IB_QPS_RTS: + return CMDQ_MODIFY_QP_NEW_STATE_RTS; + case IB_QPS_SQD: + return CMDQ_MODIFY_QP_NEW_STATE_SQD; + case IB_QPS_SQE: + return CMDQ_MODIFY_QP_NEW_STATE_SQE; + case IB_QPS_ERR: + default: + return CMDQ_MODIFY_QP_NEW_STATE_ERR; + } +} + +static enum ib_qp_state __to_ib_qp_state(u8 state) +{ + switch (state) { + case CMDQ_MODIFY_QP_NEW_STATE_RESET: + return IB_QPS_RESET; + case CMDQ_MODIFY_QP_NEW_STATE_INIT: + return IB_QPS_INIT; + case CMDQ_MODIFY_QP_NEW_STATE_RTR: + return IB_QPS_RTR; + case CMDQ_MODIFY_QP_NEW_STATE_RTS: + return IB_QPS_RTS; + case CMDQ_MODIFY_QP_NEW_STATE_SQD: + return IB_QPS_SQD; + case CMDQ_MODIFY_QP_NEW_STATE_SQE: + return IB_QPS_SQE; + case CMDQ_MODIFY_QP_NEW_STATE_ERR: + default: + return IB_QPS_ERR; + } +} + +static u32 __from_ib_mtu(enum ib_mtu mtu) +{ + switch (mtu) { + case IB_MTU_256: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; + case IB_MTU_512: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; + case IB_MTU_1024: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; + case IB_MTU_2048: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; + case IB_MTU_4096: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; + default: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; + } +} + +static enum ib_mtu __to_ib_mtu(u32 mtu) +{ + switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { + case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: + return IB_MTU_256; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: + return IB_MTU_512; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: + return IB_MTU_1024; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: + return IB_MTU_2048; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: + return IB_MTU_4096; + default: + return IB_MTU_2048; + } +} + +/* Shared Receive Queues */ +int bnxt_re_destroy_srq(struct ib_srq *ib_srq) +{ + struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, + ib_srq); + struct bnxt_re_dev *rdev = srq->rdev; + struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; + struct bnxt_qplib_nq *nq = NULL; + int rc; + + if (qplib_srq->cq) + nq = qplib_srq->cq->nq; + rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Destroy HW SRQ failed!"); + return rc; + } + + if (srq->umem) + ib_umem_release(srq->umem); + kfree(srq); + atomic_dec(&rdev->srq_count); + if (nq) + nq->budget--; + return 0; +} + +static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, + struct bnxt_re_pd *pd, + struct bnxt_re_srq *srq, + struct ib_udata *udata) +{ + struct bnxt_re_srq_req ureq; + struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; + struct ib_umem *umem; + int bytes = 0; + struct ib_ucontext *context = pd->ib_pd.uobject->context; + struct bnxt_re_ucontext *cntx = container_of(context, + struct bnxt_re_ucontext, + ib_uctx); + if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) + return -EFAULT; + + bytes = (qplib_srq->max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get(context, ureq.srqva, bytes, + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(umem)) + return PTR_ERR(umem); + + srq->umem = umem; + qplib_srq->nmap = umem->nmap; + qplib_srq->sglist = umem->sg_head.sgl; + qplib_srq->srq_handle = ureq.srq_handle; + qplib_srq->dpi = &cntx->dpi; + + return 0; +} + +struct ib_srq *bnxt_re_create_srq(struct ib_pd *ib_pd, + struct ib_srq_init_attr *srq_init_attr, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; + struct bnxt_re_srq *srq; + struct bnxt_qplib_nq *nq = NULL; + int rc, entries; + + if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { + dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded"); + rc = -EINVAL; + goto exit; + } + + if (srq_init_attr->srq_type != IB_SRQT_BASIC) { + rc = -EOPNOTSUPP; + goto exit; + } + + srq = kzalloc(sizeof(*srq), GFP_KERNEL); + if (!srq) { + rc = -ENOMEM; + goto exit; + } + srq->rdev = rdev; + srq->qplib_srq.pd = &pd->qplib_pd; + srq->qplib_srq.dpi = &rdev->dpi_privileged; + /* Allocate 1 more than what's provided so posting max doesn't + * mean empty + */ + entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1); + if (entries > dev_attr->max_srq_wqes + 1) + entries = dev_attr->max_srq_wqes + 1; + + srq->qplib_srq.max_wqe = entries; + srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; + srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; + srq->srq_limit = srq_init_attr->attr.srq_limit; + srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id; + nq = &rdev->nq[0]; + + if (udata) { + rc = bnxt_re_init_user_srq(rdev, pd, srq, udata); + if (rc) + goto fail; + } + + rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!"); + goto fail; + } + + if (udata) { + struct bnxt_re_srq_resp resp; + + resp.srqid = srq->qplib_srq.id; + rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + if (rc) { + dev_err(rdev_to_dev(rdev), "SRQ copy to udata failed!"); + bnxt_qplib_destroy_srq(&rdev->qplib_res, + &srq->qplib_srq); + goto fail; + } + } + if (nq) + nq->budget++; + atomic_inc(&rdev->srq_count); + + return &srq->ib_srq; + +fail: + if (srq->umem) + ib_umem_release(srq->umem); + kfree(srq); +exit: + return ERR_PTR(rc); +} + +int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, + enum ib_srq_attr_mask srq_attr_mask, + struct ib_udata *udata) +{ + struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, + ib_srq); + struct bnxt_re_dev *rdev = srq->rdev; + int rc; + + switch (srq_attr_mask) { + case IB_SRQ_MAX_WR: + /* SRQ resize is not supported */ + break; + case IB_SRQ_LIMIT: + /* Change the SRQ threshold */ + if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) + return -EINVAL; + + srq->qplib_srq.threshold = srq_attr->srq_limit; + rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!"); + return rc; + } + /* On success, update the shadow */ + srq->srq_limit = srq_attr->srq_limit; + /* No need to Build and send response back to udata */ + break; + default: + dev_err(rdev_to_dev(rdev), + "Unsupported srq_attr_mask 0x%x", srq_attr_mask); + return -EINVAL; + } + return 0; +} + +int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) +{ + struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, + ib_srq); + struct bnxt_re_srq tsrq; + struct bnxt_re_dev *rdev = srq->rdev; + int rc; + + /* Get live SRQ attr */ + tsrq.qplib_srq.id = srq->qplib_srq.id; + rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Query HW SRQ failed!"); + return rc; + } + srq_attr->max_wr = srq->qplib_srq.max_wqe; + srq_attr->max_sge = srq->qplib_srq.max_sge; + srq_attr->srq_limit = tsrq.qplib_srq.threshold; + + return 0; +} + +int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, + ib_srq); + struct bnxt_qplib_swqe wqe; + unsigned long flags; + int rc = 0; + + spin_lock_irqsave(&srq->lock, flags); + while (wr) { + /* Transcribe each ib_recv_wr to qplib_swqe */ + wqe.num_sge = wr->num_sge; + bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); + wqe.wr_id = wr->wr_id; + wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; + + rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); + if (rc) { + *bad_wr = wr; + break; + } + wr = wr->next; + } + spin_unlock_irqrestore(&srq->lock, flags); + + return rc; +} +static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp1_qp, + int qp_attr_mask) +{ + struct bnxt_re_qp *qp = rdev->qp1_sqp; + int rc = 0; + + if (qp_attr_mask & IB_QP_STATE) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; + qp->qplib_qp.state = qp1_qp->qplib_qp.state; + } + if (qp_attr_mask & IB_QP_PKEY_INDEX) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; + qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; + } + + if (qp_attr_mask & IB_QP_QKEY) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; + /* Using a Random QKEY */ + qp->qplib_qp.qkey = 0x81818181; + } + if (qp_attr_mask & IB_QP_SQ_PSN) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; + qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; + } + + rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) + dev_err(rdev_to_dev(rdev), + "Failed to modify Shadow QP for QP1"); + return rc; +} + +int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_udata *udata) +{ + struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_re_dev *rdev = qp->rdev; + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; + enum ib_qp_state curr_qp_state, new_qp_state; + int rc, entries; + unsigned int flags; + u8 nw_type; + + qp->qplib_qp.modify_flags = 0; + if (qp_attr_mask & IB_QP_STATE) { + curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); + new_qp_state = qp_attr->qp_state; + if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state, + ib_qp->qp_type, qp_attr_mask, + IB_LINK_LAYER_ETHERNET)) { + dev_err(rdev_to_dev(rdev), + "Invalid attribute mask: %#x specified ", + qp_attr_mask); + dev_err(rdev_to_dev(rdev), + "for qpn: %#x type: %#x", + ib_qp->qp_num, ib_qp->qp_type); + dev_err(rdev_to_dev(rdev), + "curr_qp_state=0x%x, new_qp_state=0x%x\n", + curr_qp_state, new_qp_state); + return -EINVAL; + } + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; + qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); + + if (!qp->sumem && + qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { + dev_dbg(rdev_to_dev(rdev), + "Move QP = %p to flush list\n", + qp); + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_add_flush_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + } + if (!qp->sumem && + qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { + dev_dbg(rdev_to_dev(rdev), + "Move QP = %p out of flush list\n", + qp); + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_clean_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + } + } + if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; + qp->qplib_qp.en_sqd_async_notify = true; + } + if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; + qp->qplib_qp.access = + __from_ib_access_flags(qp_attr->qp_access_flags); + /* LOCAL_WRITE access must be set to allow RC receive */ + qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; + } + if (qp_attr_mask & IB_QP_PKEY_INDEX) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; + qp->qplib_qp.pkey_index = qp_attr->pkey_index; + } + if (qp_attr_mask & IB_QP_QKEY) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; + qp->qplib_qp.qkey = qp_attr->qkey; + } + if (qp_attr_mask & IB_QP_AV) { + const struct ib_global_route *grh = + rdma_ah_read_grh(&qp_attr->ah_attr); + const struct ib_gid_attr *sgid_attr; + + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | + CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | + CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | + CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | + CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; + memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw, + sizeof(qp->qplib_qp.ah.dgid.data)); + qp->qplib_qp.ah.flow_label = grh->flow_label; + /* If RoCE V2 is enabled, stack will have two entries for + * each GID entry. Avoiding this duplicte entry in HW. Dividing + * the GID index by 2 for RoCE V2 + */ + qp->qplib_qp.ah.sgid_index = grh->sgid_index / 2; + qp->qplib_qp.ah.host_sgid_index = grh->sgid_index; + qp->qplib_qp.ah.hop_limit = grh->hop_limit; + qp->qplib_qp.ah.traffic_class = grh->traffic_class; + qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr); + ether_addr_copy(qp->qplib_qp.ah.dmac, + qp_attr->ah_attr.roce.dmac); + + sgid_attr = qp_attr->ah_attr.grh.sgid_attr; + memcpy(qp->qplib_qp.smac, sgid_attr->ndev->dev_addr, + ETH_ALEN); + nw_type = rdma_gid_attr_network_type(sgid_attr); + switch (nw_type) { + case RDMA_NETWORK_IPV4: + qp->qplib_qp.nw_type = + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; + break; + case RDMA_NETWORK_IPV6: + qp->qplib_qp.nw_type = + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; + break; + default: + qp->qplib_qp.nw_type = + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; + break; + } + } + + if (qp_attr_mask & IB_QP_PATH_MTU) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; + qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu); + qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu); + } else if (qp_attr->qp_state == IB_QPS_RTR) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; + qp->qplib_qp.path_mtu = + __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu)); + qp->qplib_qp.mtu = + ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); + } + + if (qp_attr_mask & IB_QP_TIMEOUT) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; + qp->qplib_qp.timeout = qp_attr->timeout; + } + if (qp_attr_mask & IB_QP_RETRY_CNT) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; + qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; + } + if (qp_attr_mask & IB_QP_RNR_RETRY) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; + qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; + } + if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; + qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; + } + if (qp_attr_mask & IB_QP_RQ_PSN) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; + qp->qplib_qp.rq.psn = qp_attr->rq_psn; + } + if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; + /* Cap the max_rd_atomic to device max */ + qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, + dev_attr->max_qp_rd_atom); + } + if (qp_attr_mask & IB_QP_SQ_PSN) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; + qp->qplib_qp.sq.psn = qp_attr->sq_psn; + } + if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { + if (qp_attr->max_dest_rd_atomic > + dev_attr->max_qp_init_rd_atom) { + dev_err(rdev_to_dev(rdev), + "max_dest_rd_atomic requested%d is > dev_max%d", + qp_attr->max_dest_rd_atomic, + dev_attr->max_qp_init_rd_atom); + return -EINVAL; + } + + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; + qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; + } + if (qp_attr_mask & IB_QP_CAP) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | + CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; + if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || + (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || + (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || + (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || + (qp_attr->cap.max_inline_data >= + dev_attr->max_inline_data)) { + dev_err(rdev_to_dev(rdev), + "Create QP failed - max exceeded"); + return -EINVAL; + } + entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); + qp->qplib_qp.sq.max_wqe = min_t(u32, entries, + dev_attr->max_qp_wqes + 1); + qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - + qp_attr->cap.max_send_wr; + /* + * Reserving one slot for Phantom WQE. Some application can + * post one extra entry in this case. Allowing this to avoid + * unexpected Queue full condition + */ + qp->qplib_qp.sq.q_full_delta -= 1; + qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; + if (qp->qplib_qp.rq.max_wqe) { + entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); + qp->qplib_qp.rq.max_wqe = + min_t(u32, entries, dev_attr->max_qp_wqes + 1); + qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - + qp_attr->cap.max_recv_wr; + qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; + } else { + /* SRQ was used prior, just ignore the RQ caps */ + } + } + if (qp_attr_mask & IB_QP_DEST_QPN) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; + qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; + } + rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to modify HW QP"); + return rc; + } + if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) + rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); + return rc; +} + +int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) +{ + struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_re_dev *rdev = qp->rdev; + struct bnxt_qplib_qp *qplib_qp; + int rc; + + qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL); + if (!qplib_qp) + return -ENOMEM; + + qplib_qp->id = qp->qplib_qp.id; + qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; + + rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to query HW QP"); + goto out; + } + qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); + qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state); + qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; + qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); + qp_attr->pkey_index = qplib_qp->pkey_index; + qp_attr->qkey = qplib_qp->qkey; + qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; + rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label, + qplib_qp->ah.host_sgid_index, + qplib_qp->ah.hop_limit, + qplib_qp->ah.traffic_class); + rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data); + rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl); + ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac); + qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); + qp_attr->timeout = qplib_qp->timeout; + qp_attr->retry_cnt = qplib_qp->retry_cnt; + qp_attr->rnr_retry = qplib_qp->rnr_retry; + qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; + qp_attr->rq_psn = qplib_qp->rq.psn; + qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; + qp_attr->sq_psn = qplib_qp->sq.psn; + qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; + qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : + IB_SIGNAL_REQ_WR; + qp_attr->dest_qp_num = qplib_qp->dest_qpn; + + qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; + qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; + qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; + qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; + qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; + qp_init_attr->cap = qp_attr->cap; + +out: + kfree(qplib_qp); + return rc; +} + +/* Routine for sending QP1 packets for RoCE V1 an V2 + */ +static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, + const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe, + int payload_size) +{ + struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, + ib_ah); + struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; + const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr; + struct bnxt_qplib_sge sge; + u8 nw_type; + u16 ether_type; + union ib_gid dgid; + bool is_eth = false; + bool is_vlan = false; + bool is_grh = false; + bool is_udp = false; + u8 ip_version = 0; + u16 vlan_id = 0xFFFF; + void *buf; + int i, rc = 0; + + memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); + + if (is_vlan_dev(sgid_attr->ndev)) + vlan_id = vlan_dev_vlan_id(sgid_attr->ndev); + /* Get network header type for this GID */ + nw_type = rdma_gid_attr_network_type(sgid_attr); + switch (nw_type) { + case RDMA_NETWORK_IPV4: + nw_type = BNXT_RE_ROCEV2_IPV4_PACKET; + break; + case RDMA_NETWORK_IPV6: + nw_type = BNXT_RE_ROCEV2_IPV6_PACKET; + break; + default: + nw_type = BNXT_RE_ROCE_V1_PACKET; + break; + } + memcpy(&dgid.raw, &qplib_ah->dgid, 16); + is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; + if (is_udp) { + if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) { + ip_version = 4; + ether_type = ETH_P_IP; + } else { + ip_version = 6; + ether_type = ETH_P_IPV6; + } + is_grh = false; + } else { + ether_type = ETH_P_IBOE; + is_grh = true; + } + + is_eth = true; + is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false; + + ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh, + ip_version, is_udp, 0, &qp->qp1_hdr); + + /* ETH */ + ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac); + ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac); + + /* For vlan, check the sgid for vlan existence */ + + if (!is_vlan) { + qp->qp1_hdr.eth.type = cpu_to_be16(ether_type); + } else { + qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type); + qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id); + } + + if (is_grh || (ip_version == 6)) { + memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw, + sizeof(sgid_attr->gid)); + memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, + sizeof(sgid_attr->gid)); + qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit; + } + + if (ip_version == 4) { + qp->qp1_hdr.ip4.tos = 0; + qp->qp1_hdr.ip4.id = 0; + qp->qp1_hdr.ip4.frag_off = htons(IP_DF); + qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit; + + memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4); + memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4); + qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr); + } + + if (is_udp) { + qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT); + qp->qp1_hdr.udp.sport = htons(0x8CD1); + qp->qp1_hdr.udp.csum = 0; + } + + /* BTH */ + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; + qp->qp1_hdr.immediate_present = 1; + } else { + qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; + } + if (wr->send_flags & IB_SEND_SOLICITED) + qp->qp1_hdr.bth.solicited_event = 1; + /* pad_count */ + qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; + + /* P_key for QP1 is for all members */ + qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); + qp->qp1_hdr.bth.destination_qpn = IB_QP1; + qp->qp1_hdr.bth.ack_req = 0; + qp->send_psn++; + qp->send_psn &= BTH_PSN_MASK; + qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); + /* DETH */ + /* Use the priviledged Q_Key for QP1 */ + qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); + qp->qp1_hdr.deth.source_qpn = IB_QP1; + + /* Pack the QP1 to the transmit buffer */ + buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); + if (buf) { + ib_ud_header_pack(&qp->qp1_hdr, buf); + for (i = wqe->num_sge; i; i--) { + wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; + wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; + wqe->sg_list[i].size = wqe->sg_list[i - 1].size; + } + + /* + * Max Header buf size for IPV6 RoCE V2 is 86, + * which is same as the QP1 SQ header buffer. + * Header buf size for IPV4 RoCE V2 can be 66. + * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20). + * Subtract 20 bytes from QP1 SQ header buf size + */ + if (is_udp && ip_version == 4) + sge.size -= 20; + /* + * Max Header buf size for RoCE V1 is 78. + * ETH(14) + VLAN(4) + GRH(40) + BTH(20). + * Subtract 8 bytes from QP1 SQ header buf size + */ + if (!is_udp) + sge.size -= 8; + + /* Subtract 4 bytes for non vlan packets */ + if (!is_vlan) + sge.size -= 4; + + wqe->sg_list[0].addr = sge.addr; + wqe->sg_list[0].lkey = sge.lkey; + wqe->sg_list[0].size = sge.size; + wqe->num_sge++; + + } else { + dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!"); + rc = -ENOMEM; + } + return rc; +} + +/* For the MAD layer, it only provides the recv SGE the size of + * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, + * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire + * receive packet (334 bytes) with no VLAN and then copy the GRH + * and the MAD datagram out to the provided SGE. + */ +static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, + const struct ib_recv_wr *wr, + struct bnxt_qplib_swqe *wqe, + int payload_size) +{ + struct bnxt_qplib_sge ref, sge; + u32 rq_prod_index; + struct bnxt_re_sqp_entries *sqp_entry; + + rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); + + if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) + return -ENOMEM; + + /* Create 1 SGE to receive the entire + * ethernet packet + */ + /* Save the reference from ULP */ + ref.addr = wqe->sg_list[0].addr; + ref.lkey = wqe->sg_list[0].lkey; + ref.size = wqe->sg_list[0].size; + + sqp_entry = &qp->rdev->sqp_tbl[rq_prod_index]; + + /* SGE 1 */ + wqe->sg_list[0].addr = sge.addr; + wqe->sg_list[0].lkey = sge.lkey; + wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; + sge.size -= wqe->sg_list[0].size; + + sqp_entry->sge.addr = ref.addr; + sqp_entry->sge.lkey = ref.lkey; + sqp_entry->sge.size = ref.size; + /* Store the wrid for reporting completion */ + sqp_entry->wrid = wqe->wr_id; + /* change the wqe->wrid to table index */ + wqe->wr_id = rq_prod_index; + return 0; +} + +static int is_ud_qp(struct bnxt_re_qp *qp) +{ + return qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD; +} + +static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, + const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_ah *ah = NULL; + + if (is_ud_qp(qp)) { + ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah); + wqe->send.q_key = ud_wr(wr)->remote_qkey; + wqe->send.dst_qp = ud_wr(wr)->remote_qpn; + wqe->send.avid = ah->qplib_ah.id; + } + switch (wr->opcode) { + case IB_WR_SEND: + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; + break; + case IB_WR_SEND_WITH_IMM: + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; + wqe->send.imm_data = wr->ex.imm_data; + break; + case IB_WR_SEND_WITH_INV: + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; + wqe->send.inv_key = wr->ex.invalidate_rkey; + break; + default: + return -EINVAL; + } + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_FENCE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + if (wr->send_flags & IB_SEND_INLINE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; + + return 0; +} + +static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + switch (wr->opcode) { + case IB_WR_RDMA_WRITE: + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; + break; + case IB_WR_RDMA_WRITE_WITH_IMM: + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; + wqe->rdma.imm_data = wr->ex.imm_data; + break; + case IB_WR_RDMA_READ: + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; + wqe->rdma.inv_key = wr->ex.invalidate_rkey; + break; + default: + return -EINVAL; + } + wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; + wqe->rdma.r_key = rdma_wr(wr)->rkey; + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_FENCE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + if (wr->send_flags & IB_SEND_INLINE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; + + return 0; +} + +static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + switch (wr->opcode) { + case IB_WR_ATOMIC_CMP_AND_SWP: + wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; + wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; + wqe->atomic.swap_data = atomic_wr(wr)->swap; + break; + case IB_WR_ATOMIC_FETCH_AND_ADD: + wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; + wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; + break; + default: + return -EINVAL; + } + wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; + wqe->atomic.r_key = atomic_wr(wr)->rkey; + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_FENCE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + return 0; +} + +static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; + wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; + + /* Need unconditional fence for local invalidate + * opcode to work as expected. + */ + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + + return 0; +} + +static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr); + struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; + int access = wr->access; + + wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; + wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; + wqe->frmr.page_list = mr->pages; + wqe->frmr.page_list_len = mr->npages; + wqe->frmr.levels = qplib_frpl->hwq.level + 1; + wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; + + /* Need unconditional fence for reg_mr + * opcode to function as expected. + */ + + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + + if (wr->wr.send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + + if (access & IB_ACCESS_LOCAL_WRITE) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; + if (access & IB_ACCESS_REMOTE_READ) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; + if (access & IB_ACCESS_REMOTE_WRITE) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; + if (access & IB_ACCESS_REMOTE_ATOMIC) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; + if (access & IB_ACCESS_MW_BIND) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; + + wqe->frmr.l_key = wr->key; + wqe->frmr.length = wr->mr->length; + wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1; + wqe->frmr.va = wr->mr->iova; + return 0; +} + +static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, + const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + /* Copy the inline data to the data field */ + u8 *in_data; + u32 i, sge_len; + void *sge_addr; + + in_data = wqe->inline_data; + for (i = 0; i < wr->num_sge; i++) { + sge_addr = (void *)(unsigned long) + wr->sg_list[i].addr; + sge_len = wr->sg_list[i].length; + + if ((sge_len + wqe->inline_len) > + BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { + dev_err(rdev_to_dev(rdev), + "Inline data size requested > supported value"); + return -EINVAL; + } + sge_len = wr->sg_list[i].length; + + memcpy(in_data, sge_addr, sge_len); + in_data += wr->sg_list[i].length; + wqe->inline_len += wr->sg_list[i].length; + } + return wqe->inline_len; +} + +static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev, + const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + int payload_sz = 0; + + if (wr->send_flags & IB_SEND_INLINE) + payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); + else + payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, + wqe->num_sge); + + return payload_sz; +} + +static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) +{ + if ((qp->ib_qp.qp_type == IB_QPT_UD || + qp->ib_qp.qp_type == IB_QPT_GSI || + qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && + qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { + int qp_attr_mask; + struct ib_qp_attr qp_attr; + + qp_attr_mask = IB_QP_STATE; + qp_attr.qp_state = IB_QPS_RTS; + bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); + qp->qplib_qp.wqe_cnt = 0; + } +} + +static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp, + const struct ib_send_wr *wr) +{ + struct bnxt_qplib_swqe wqe; + int rc = 0, payload_sz = 0; + unsigned long flags; + + spin_lock_irqsave(&qp->sq_lock, flags); + memset(&wqe, 0, sizeof(wqe)); + while (wr) { + /* House keeping */ + memset(&wqe, 0, sizeof(wqe)); + + /* Common */ + wqe.num_sge = wr->num_sge; + if (wr->num_sge > qp->qplib_qp.sq.max_sge) { + dev_err(rdev_to_dev(rdev), + "Limit exceeded for Send SGEs"); + rc = -EINVAL; + goto bad; + } + + payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); + if (payload_sz < 0) { + rc = -EINVAL; + goto bad; + } + wqe.wr_id = wr->wr_id; + + wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; + + rc = bnxt_re_build_send_wqe(qp, wr, &wqe); + if (!rc) + rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); +bad: + if (rc) { + dev_err(rdev_to_dev(rdev), + "Post send failed opcode = %#x rc = %d", + wr->opcode, rc); + break; + } + wr = wr->next; + } + bnxt_qplib_post_send_db(&qp->qplib_qp); + bnxt_ud_qp_hw_stall_workaround(qp); + spin_unlock_irqrestore(&qp->sq_lock, flags); + return rc; +} + +int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_qplib_swqe wqe; + int rc = 0, payload_sz = 0; + unsigned long flags; + + spin_lock_irqsave(&qp->sq_lock, flags); + while (wr) { + /* House keeping */ + memset(&wqe, 0, sizeof(wqe)); + + /* Common */ + wqe.num_sge = wr->num_sge; + if (wr->num_sge > qp->qplib_qp.sq.max_sge) { + dev_err(rdev_to_dev(qp->rdev), + "Limit exceeded for Send SGEs"); + rc = -EINVAL; + goto bad; + } + + payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); + if (payload_sz < 0) { + rc = -EINVAL; + goto bad; + } + wqe.wr_id = wr->wr_id; + + switch (wr->opcode) { + case IB_WR_SEND: + case IB_WR_SEND_WITH_IMM: + if (ib_qp->qp_type == IB_QPT_GSI) { + rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe, + payload_sz); + if (rc) + goto bad; + wqe.rawqp1.lflags |= + SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; + } + switch (wr->send_flags) { + case IB_SEND_IP_CSUM: + wqe.rawqp1.lflags |= + SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; + break; + default: + break; + } + /* fall through */ + case IB_WR_SEND_WITH_INV: + rc = bnxt_re_build_send_wqe(qp, wr, &wqe); + break; + case IB_WR_RDMA_WRITE: + case IB_WR_RDMA_WRITE_WITH_IMM: + case IB_WR_RDMA_READ: + rc = bnxt_re_build_rdma_wqe(wr, &wqe); + break; + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + rc = bnxt_re_build_atomic_wqe(wr, &wqe); + break; + case IB_WR_RDMA_READ_WITH_INV: + dev_err(rdev_to_dev(qp->rdev), + "RDMA Read with Invalidate is not supported"); + rc = -EINVAL; + goto bad; + case IB_WR_LOCAL_INV: + rc = bnxt_re_build_inv_wqe(wr, &wqe); + break; + case IB_WR_REG_MR: + rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); + break; + default: + /* Unsupported WRs */ + dev_err(rdev_to_dev(qp->rdev), + "WR (%#x) is not supported", wr->opcode); + rc = -EINVAL; + goto bad; + } + if (!rc) + rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); +bad: + if (rc) { + dev_err(rdev_to_dev(qp->rdev), + "post_send failed op:%#x qps = %#x rc = %d\n", + wr->opcode, qp->qplib_qp.state, rc); + *bad_wr = wr; + break; + } + wr = wr->next; + } + bnxt_qplib_post_send_db(&qp->qplib_qp); + bnxt_ud_qp_hw_stall_workaround(qp); + spin_unlock_irqrestore(&qp->sq_lock, flags); + + return rc; +} + +static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp, + const struct ib_recv_wr *wr) +{ + struct bnxt_qplib_swqe wqe; + int rc = 0; + + memset(&wqe, 0, sizeof(wqe)); + while (wr) { + /* House keeping */ + memset(&wqe, 0, sizeof(wqe)); + + /* Common */ + wqe.num_sge = wr->num_sge; + if (wr->num_sge > qp->qplib_qp.rq.max_sge) { + dev_err(rdev_to_dev(rdev), + "Limit exceeded for Receive SGEs"); + rc = -EINVAL; + break; + } + bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); + wqe.wr_id = wr->wr_id; + wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; + + rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); + if (rc) + break; + + wr = wr->next; + } + if (!rc) + bnxt_qplib_post_recv_db(&qp->qplib_qp); + return rc; +} + +int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_qplib_swqe wqe; + int rc = 0, payload_sz = 0; + unsigned long flags; + u32 count = 0; + + spin_lock_irqsave(&qp->rq_lock, flags); + while (wr) { + /* House keeping */ + memset(&wqe, 0, sizeof(wqe)); + + /* Common */ + wqe.num_sge = wr->num_sge; + if (wr->num_sge > qp->qplib_qp.rq.max_sge) { + dev_err(rdev_to_dev(qp->rdev), + "Limit exceeded for Receive SGEs"); + rc = -EINVAL; + *bad_wr = wr; + break; + } + + payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, + wr->num_sge); + wqe.wr_id = wr->wr_id; + wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; + + if (ib_qp->qp_type == IB_QPT_GSI) + rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe, + payload_sz); + if (!rc) + rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); + if (rc) { + *bad_wr = wr; + break; + } + + /* Ring DB if the RQEs posted reaches a threshold value */ + if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) { + bnxt_qplib_post_recv_db(&qp->qplib_qp); + count = 0; + } + + wr = wr->next; + } + + if (count) + bnxt_qplib_post_recv_db(&qp->qplib_qp); + + spin_unlock_irqrestore(&qp->rq_lock, flags); + + return rc; +} + +/* Completion Queues */ +int bnxt_re_destroy_cq(struct ib_cq *ib_cq) +{ + int rc; + struct bnxt_re_cq *cq; + struct bnxt_qplib_nq *nq; + struct bnxt_re_dev *rdev; + + cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); + rdev = cq->rdev; + nq = cq->qplib_cq.nq; + + rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ"); + return rc; + } + if (!IS_ERR_OR_NULL(cq->umem)) + ib_umem_release(cq->umem); + + atomic_dec(&rdev->cq_count); + nq->budget--; + kfree(cq->cql); + kfree(cq); + + return 0; +} + +struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_ucontext *context, + struct ib_udata *udata) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; + struct bnxt_re_cq *cq = NULL; + int rc, entries; + int cqe = attr->cqe; + struct bnxt_qplib_nq *nq = NULL; + unsigned int nq_alloc_cnt; + + /* Validate CQ fields */ + if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { + dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded"); + return ERR_PTR(-EINVAL); + } + cq = kzalloc(sizeof(*cq), GFP_KERNEL); + if (!cq) + return ERR_PTR(-ENOMEM); + + cq->rdev = rdev; + cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq); + + entries = roundup_pow_of_two(cqe + 1); + if (entries > dev_attr->max_cq_wqes + 1) + entries = dev_attr->max_cq_wqes + 1; + + if (context) { + struct bnxt_re_cq_req req; + struct bnxt_re_ucontext *uctx = container_of + (context, + struct bnxt_re_ucontext, + ib_uctx); + if (ib_copy_from_udata(&req, udata, sizeof(req))) { + rc = -EFAULT; + goto fail; + } + + cq->umem = ib_umem_get(context, req.cq_va, + entries * sizeof(struct cq_base), + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(cq->umem)) { + rc = PTR_ERR(cq->umem); + goto fail; + } + cq->qplib_cq.sghead = cq->umem->sg_head.sgl; + cq->qplib_cq.nmap = cq->umem->nmap; + cq->qplib_cq.dpi = &uctx->dpi; + } else { + cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL); + cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), + GFP_KERNEL); + if (!cq->cql) { + rc = -ENOMEM; + goto fail; + } + + cq->qplib_cq.dpi = &rdev->dpi_privileged; + cq->qplib_cq.sghead = NULL; + cq->qplib_cq.nmap = 0; + } + /* + * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a + * used for getting the NQ index. + */ + nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt); + nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)]; + cq->qplib_cq.max_wqe = entries; + cq->qplib_cq.cnq_hw_ring_id = nq->ring_id; + cq->qplib_cq.nq = nq; + + rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to create HW CQ"); + goto fail; + } + + cq->ib_cq.cqe = entries; + cq->cq_period = cq->qplib_cq.period; + nq->budget++; + + atomic_inc(&rdev->cq_count); + spin_lock_init(&cq->cq_lock); + + if (context) { + struct bnxt_re_cq_resp resp; + + resp.cqid = cq->qplib_cq.id; + resp.tail = cq->qplib_cq.hwq.cons; + resp.phase = cq->qplib_cq.period; + resp.rsvd = 0; + rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to copy CQ udata"); + bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + goto c2fail; + } + } + + return &cq->ib_cq; + +c2fail: + if (context) + ib_umem_release(cq->umem); +fail: + kfree(cq->cql); + kfree(cq); + return ERR_PTR(rc); +} + +static u8 __req_to_ib_wc_status(u8 qstatus) +{ + switch (qstatus) { + case CQ_REQ_STATUS_OK: + return IB_WC_SUCCESS; + case CQ_REQ_STATUS_BAD_RESPONSE_ERR: + return IB_WC_BAD_RESP_ERR; + case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: + return IB_WC_LOC_LEN_ERR; + case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: + return IB_WC_LOC_QP_OP_ERR; + case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: + return IB_WC_LOC_PROT_ERR; + case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: + return IB_WC_GENERAL_ERR; + case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: + return IB_WC_REM_INV_REQ_ERR; + case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: + return IB_WC_REM_ACCESS_ERR; + case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: + return IB_WC_REM_OP_ERR; + case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: + return IB_WC_RNR_RETRY_EXC_ERR; + case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: + return IB_WC_RETRY_EXC_ERR; + case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: + return IB_WC_WR_FLUSH_ERR; + default: + return IB_WC_GENERAL_ERR; + } + return 0; +} + +static u8 __rawqp1_to_ib_wc_status(u8 qstatus) +{ + switch (qstatus) { + case CQ_RES_RAWETH_QP1_STATUS_OK: + return IB_WC_SUCCESS; + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: + return IB_WC_LOC_ACCESS_ERR; + case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: + return IB_WC_LOC_LEN_ERR; + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: + return IB_WC_LOC_PROT_ERR; + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: + return IB_WC_LOC_QP_OP_ERR; + case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: + return IB_WC_GENERAL_ERR; + case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: + return IB_WC_WR_FLUSH_ERR; + case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: + return IB_WC_WR_FLUSH_ERR; + default: + return IB_WC_GENERAL_ERR; + } +} + +static u8 __rc_to_ib_wc_status(u8 qstatus) +{ + switch (qstatus) { + case CQ_RES_RC_STATUS_OK: + return IB_WC_SUCCESS; + case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: + return IB_WC_LOC_ACCESS_ERR; + case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: + return IB_WC_LOC_LEN_ERR; + case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: + return IB_WC_LOC_PROT_ERR; + case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: + return IB_WC_LOC_QP_OP_ERR; + case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: + return IB_WC_GENERAL_ERR; + case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: + return IB_WC_REM_INV_REQ_ERR; + case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: + return IB_WC_WR_FLUSH_ERR; + case CQ_RES_RC_STATUS_HW_FLUSH_ERR: + return IB_WC_WR_FLUSH_ERR; + default: + return IB_WC_GENERAL_ERR; + } +} + +static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) +{ + switch (cqe->type) { + case BNXT_QPLIB_SWQE_TYPE_SEND: + wc->opcode = IB_WC_SEND; + break; + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: + wc->opcode = IB_WC_SEND; + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: + wc->opcode = IB_WC_SEND; + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + break; + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: + wc->opcode = IB_WC_RDMA_WRITE; + break; + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RDMA_WRITE; + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: + wc->opcode = IB_WC_RDMA_READ; + break; + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: + wc->opcode = IB_WC_COMP_SWAP; + break; + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: + wc->opcode = IB_WC_FETCH_ADD; + break; + case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: + wc->opcode = IB_WC_LOCAL_INV; + break; + case BNXT_QPLIB_SWQE_TYPE_REG_MR: + wc->opcode = IB_WC_REG_MR; + break; + default: + wc->opcode = IB_WC_SEND; + break; + } + + wc->status = __req_to_ib_wc_status(cqe->status); +} + +static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, + u16 raweth_qp1_flags2) +{ + bool is_ipv6 = false, is_ipv4 = false; + + /* raweth_qp1_flags Bit 9-6 indicates itype */ + if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) + != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) + return -1; + + if (raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && + raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { + /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ + (raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? + (is_ipv6 = true) : (is_ipv4 = true); + return ((is_ipv6) ? + BNXT_RE_ROCEV2_IPV6_PACKET : + BNXT_RE_ROCEV2_IPV4_PACKET); + } else { + return BNXT_RE_ROCE_V1_PACKET; + } +} + +static int bnxt_re_to_ib_nw_type(int nw_type) +{ + u8 nw_hdr_type = 0xFF; + + switch (nw_type) { + case BNXT_RE_ROCE_V1_PACKET: + nw_hdr_type = RDMA_NETWORK_ROCE_V1; + break; + case BNXT_RE_ROCEV2_IPV4_PACKET: + nw_hdr_type = RDMA_NETWORK_IPV4; + break; + case BNXT_RE_ROCEV2_IPV6_PACKET: + nw_hdr_type = RDMA_NETWORK_IPV6; + break; + } + return nw_hdr_type; +} + +static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, + void *rq_hdr_buf) +{ + u8 *tmp_buf = NULL; + struct ethhdr *eth_hdr; + u16 eth_type; + bool rc = false; + + tmp_buf = (u8 *)rq_hdr_buf; + /* + * If dest mac is not same as I/F mac, this could be a + * loopback address or multicast address, check whether + * it is a loopback packet + */ + if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) { + tmp_buf += 4; + /* Check the ether type */ + eth_hdr = (struct ethhdr *)tmp_buf; + eth_type = ntohs(eth_hdr->h_proto); + switch (eth_type) { + case ETH_P_IBOE: + rc = true; + break; + case ETH_P_IP: + case ETH_P_IPV6: { + u32 len; + struct udphdr *udp_hdr; + + len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) : + sizeof(struct ipv6hdr)); + tmp_buf += sizeof(struct ethhdr) + len; + udp_hdr = (struct udphdr *)tmp_buf; + if (ntohs(udp_hdr->dest) == + ROCE_V2_UDP_DPORT) + rc = true; + break; + } + default: + break; + } + } + + return rc; +} + +static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp, + struct bnxt_qplib_cqe *cqe) +{ + struct bnxt_re_dev *rdev = qp1_qp->rdev; + struct bnxt_re_sqp_entries *sqp_entry = NULL; + struct bnxt_re_qp *qp = rdev->qp1_sqp; + struct ib_send_wr *swr; + struct ib_ud_wr udwr; + struct ib_recv_wr rwr; + int pkt_type = 0; + u32 tbl_idx; + void *rq_hdr_buf; + dma_addr_t rq_hdr_buf_map; + dma_addr_t shrq_hdr_buf_map; + u32 offset = 0; + u32 skip_bytes = 0; + struct ib_sge s_sge[2]; + struct ib_sge r_sge[2]; + int rc; + + memset(&udwr, 0, sizeof(udwr)); + memset(&rwr, 0, sizeof(rwr)); + memset(&s_sge, 0, sizeof(s_sge)); + memset(&r_sge, 0, sizeof(r_sge)); + + swr = &udwr.wr; + tbl_idx = cqe->wr_id; + + rq_hdr_buf = qp1_qp->qplib_qp.rq_hdr_buf + + (tbl_idx * qp1_qp->qplib_qp.rq_hdr_buf_size); + rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp1_qp->qplib_qp, + tbl_idx); + + /* Shadow QP header buffer */ + shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp->qplib_qp, + tbl_idx); + sqp_entry = &rdev->sqp_tbl[tbl_idx]; + + /* Store this cqe */ + memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); + sqp_entry->qp1_qp = qp1_qp; + + /* Find packet type from the cqe */ + + pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, + cqe->raweth_qp1_flags2); + if (pkt_type < 0) { + dev_err(rdev_to_dev(rdev), "Invalid packet\n"); + return -EINVAL; + } + + /* Adjust the offset for the user buffer and post in the rq */ + + if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) + offset = 20; + + /* + * QP1 loopback packet has 4 bytes of internal header before + * ether header. Skip these four bytes. + */ + if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) + skip_bytes = 4; + + /* First send SGE . Skip the ether header*/ + s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE + + skip_bytes; + s_sge[0].lkey = 0xFFFFFFFF; + s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : + BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; + + /* Second Send SGE */ + s_sge[1].addr = s_sge[0].addr + s_sge[0].length + + BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; + if (pkt_type != BNXT_RE_ROCE_V1_PACKET) + s_sge[1].addr += 8; + s_sge[1].lkey = 0xFFFFFFFF; + s_sge[1].length = 256; + + /* First recv SGE */ + + r_sge[0].addr = shrq_hdr_buf_map; + r_sge[0].lkey = 0xFFFFFFFF; + r_sge[0].length = 40; + + r_sge[1].addr = sqp_entry->sge.addr + offset; + r_sge[1].lkey = sqp_entry->sge.lkey; + r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; + + /* Create receive work request */ + rwr.num_sge = 2; + rwr.sg_list = r_sge; + rwr.wr_id = tbl_idx; + rwr.next = NULL; + + rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &rwr); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to post Rx buffers to shadow QP"); + return -ENOMEM; + } + + swr->num_sge = 2; + swr->sg_list = s_sge; + swr->wr_id = tbl_idx; + swr->opcode = IB_WR_SEND; + swr->next = NULL; + + udwr.ah = &rdev->sqp_ah->ib_ah; + udwr.remote_qpn = rdev->qp1_sqp->qplib_qp.id; + udwr.remote_qkey = rdev->qp1_sqp->qplib_qp.qkey; + + /* post data received in the send queue */ + rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr); + + return 0; +} + +static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + wc->opcode = IB_WC_RECV; + wc->status = __rawqp1_to_ib_wc_status(cqe->status); + wc->wc_flags |= IB_WC_GRH; +} + +static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, + u16 vlan_id) +{ + /* + * Check if the vlan is configured in the host. If not configured, it + * can be a transparent VLAN. So dont report the vlan id. + */ + if (!__vlan_find_dev_deep_rcu(rdev->netdev, + htons(ETH_P_8021Q), vlan_id)) + return false; + return true; +} + +static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe, + u16 *vid, u8 *sl) +{ + bool ret = false; + u32 metadata; + u16 tpid; + + metadata = orig_cqe->raweth_qp1_metadata; + if (orig_cqe->raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) { + tpid = ((metadata & + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >> + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); + if (tpid == ETH_P_8021Q) { + *vid = metadata & + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK; + *sl = (metadata & + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >> + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT; + ret = true; + } + } + + return ret; +} + +static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + wc->opcode = IB_WC_RECV; + wc->status = __rc_to_ib_wc_status(cqe->status); + + if (cqe->flags & CQ_RES_RC_FLAGS_IMM) + wc->wc_flags |= IB_WC_WITH_IMM; + if (cqe->flags & CQ_RES_RC_FLAGS_INV) + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == + (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; +} + +static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp, + struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + struct bnxt_re_dev *rdev = qp->rdev; + struct bnxt_re_qp *qp1_qp = NULL; + struct bnxt_qplib_cqe *orig_cqe = NULL; + struct bnxt_re_sqp_entries *sqp_entry = NULL; + int nw_type; + u32 tbl_idx; + u16 vlan_id; + u8 sl; + + tbl_idx = cqe->wr_id; + + sqp_entry = &rdev->sqp_tbl[tbl_idx]; + qp1_qp = sqp_entry->qp1_qp; + orig_cqe = &sqp_entry->cqe; + + wc->wr_id = sqp_entry->wrid; + wc->byte_len = orig_cqe->length; + wc->qp = &qp1_qp->ib_qp; + + wc->ex.imm_data = orig_cqe->immdata; + wc->src_qp = orig_cqe->src_qp; + memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); + if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) { + if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { + wc->vlan_id = vlan_id; + wc->sl = sl; + wc->wc_flags |= IB_WC_WITH_VLAN; + } + } + wc->port_num = 1; + wc->vendor_err = orig_cqe->status; + + wc->opcode = IB_WC_RECV; + wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); + wc->wc_flags |= IB_WC_GRH; + + nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, + orig_cqe->raweth_qp1_flags2); + if (nw_type >= 0) { + wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); + wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; + } +} + +static void bnxt_re_process_res_ud_wc(struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + wc->opcode = IB_WC_RECV; + wc->status = __rc_to_ib_wc_status(cqe->status); + + if (cqe->flags & CQ_RES_RC_FLAGS_IMM) + wc->wc_flags |= IB_WC_WITH_IMM; + if (cqe->flags & CQ_RES_RC_FLAGS_INV) + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == + (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; +} + +static int send_phantom_wqe(struct bnxt_re_qp *qp) +{ + struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; + unsigned long flags; + int rc = 0; + + spin_lock_irqsave(&qp->sq_lock, flags); + + rc = bnxt_re_bind_fence_mw(lib_qp); + if (!rc) { + lib_qp->sq.phantom_wqe_cnt++; + dev_dbg(&lib_qp->sq.hwq.pdev->dev, + "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", + lib_qp->id, lib_qp->sq.hwq.prod, + HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), + lib_qp->sq.phantom_wqe_cnt); + } + + spin_unlock_irqrestore(&qp->sq_lock, flags); + return rc; +} + +int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) +{ + struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); + struct bnxt_re_qp *qp; + struct bnxt_qplib_cqe *cqe; + int i, ncqe, budget; + struct bnxt_qplib_q *sq; + struct bnxt_qplib_qp *lib_qp; + u32 tbl_idx; + struct bnxt_re_sqp_entries *sqp_entry = NULL; + unsigned long flags; + + spin_lock_irqsave(&cq->cq_lock, flags); + budget = min_t(u32, num_entries, cq->max_cql); + num_entries = budget; + if (!cq->cql) { + dev_err(rdev_to_dev(cq->rdev), "POLL CQ : no CQL to use"); + goto exit; + } + cqe = &cq->cql[0]; + while (budget) { + lib_qp = NULL; + ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); + if (lib_qp) { + sq = &lib_qp->sq; + if (sq->send_phantom) { + qp = container_of(lib_qp, + struct bnxt_re_qp, qplib_qp); + if (send_phantom_wqe(qp) == -ENOMEM) + dev_err(rdev_to_dev(cq->rdev), + "Phantom failed! Scheduled to send again\n"); + else + sq->send_phantom = false; + } + } + if (ncqe < budget) + ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, + cqe + ncqe, + budget - ncqe); + + if (!ncqe) + break; + + for (i = 0; i < ncqe; i++, cqe++) { + /* Transcribe each qplib_wqe back to ib_wc */ + memset(wc, 0, sizeof(*wc)); + + wc->wr_id = cqe->wr_id; + wc->byte_len = cqe->length; + qp = container_of + ((struct bnxt_qplib_qp *) + (unsigned long)(cqe->qp_handle), + struct bnxt_re_qp, qplib_qp); + if (!qp) { + dev_err(rdev_to_dev(cq->rdev), + "POLL CQ : bad QP handle"); + continue; + } + wc->qp = &qp->ib_qp; + wc->ex.imm_data = cqe->immdata; + wc->src_qp = cqe->src_qp; + memcpy(wc->smac, cqe->smac, ETH_ALEN); + wc->port_num = 1; + wc->vendor_err = cqe->status; + + switch (cqe->opcode) { + case CQ_BASE_CQE_TYPE_REQ: + if (qp->qplib_qp.id == + qp->rdev->qp1_sqp->qplib_qp.id) { + /* Handle this completion with + * the stored completion + */ + memset(wc, 0, sizeof(*wc)); + continue; + } + bnxt_re_process_req_wc(wc, cqe); + break; + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: + if (!cqe->status) { + int rc = 0; + + rc = bnxt_re_process_raw_qp_pkt_rx + (qp, cqe); + if (!rc) { + memset(wc, 0, sizeof(*wc)); + continue; + } + cqe->status = -1; + } + /* Errors need not be looped back. + * But change the wr_id to the one + * stored in the table + */ + tbl_idx = cqe->wr_id; + sqp_entry = &cq->rdev->sqp_tbl[tbl_idx]; + wc->wr_id = sqp_entry->wrid; + bnxt_re_process_res_rawqp1_wc(wc, cqe); + break; + case CQ_BASE_CQE_TYPE_RES_RC: + bnxt_re_process_res_rc_wc(wc, cqe); + break; + case CQ_BASE_CQE_TYPE_RES_UD: + if (qp->qplib_qp.id == + qp->rdev->qp1_sqp->qplib_qp.id) { + /* Handle this completion with + * the stored completion + */ + if (cqe->status) { + continue; + } else { + bnxt_re_process_res_shadow_qp_wc + (qp, wc, cqe); + break; + } + } + bnxt_re_process_res_ud_wc(wc, cqe); + break; + default: + dev_err(rdev_to_dev(cq->rdev), + "POLL CQ : type 0x%x not handled", + cqe->opcode); + continue; + } + wc++; + budget--; + } + } +exit: + spin_unlock_irqrestore(&cq->cq_lock, flags); + return num_entries - budget; +} + +int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, + enum ib_cq_notify_flags ib_cqn_flags) +{ + struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); + int type = 0, rc = 0; + unsigned long flags; + + spin_lock_irqsave(&cq->cq_lock, flags); + /* Trigger on the very next completion */ + if (ib_cqn_flags & IB_CQ_NEXT_COMP) + type = DBR_DBR_TYPE_CQ_ARMALL; + /* Trigger on the next solicited completion */ + else if (ib_cqn_flags & IB_CQ_SOLICITED) + type = DBR_DBR_TYPE_CQ_ARMSE; + + /* Poll to see if there are missed events */ + if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) && + !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) { + rc = 1; + goto exit; + } + bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); + +exit: + spin_unlock_irqrestore(&cq->cq_lock, flags); + return rc; +} + +/* Memory Regions */ +struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) +{ + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_mr *mr; + u64 pbl = 0; + int rc; + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); + + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; + + /* Allocate and register 0 as the address */ + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) + goto fail; + + mr->qplib_mr.hwq.level = PBL_LVL_MAX; + mr->qplib_mr.total_size = -1; /* Infinte length */ + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false, + PAGE_SIZE); + if (rc) + goto fail_mr; + + mr->ib_mr.lkey = mr->qplib_mr.lkey; + if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | + IB_ACCESS_REMOTE_ATOMIC)) + mr->ib_mr.rkey = mr->ib_mr.lkey; + atomic_inc(&rdev->mr_count); + + return &mr->ib_mr; + +fail_mr: + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); +fail: + kfree(mr); + return ERR_PTR(rc); +} + +int bnxt_re_dereg_mr(struct ib_mr *ib_mr) +{ + struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); + struct bnxt_re_dev *rdev = mr->rdev; + int rc; + + rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Dereg MR failed: %#x\n", rc); + return rc; + } + + if (mr->pages) { + rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, + &mr->qplib_frpl); + kfree(mr->pages); + mr->npages = 0; + mr->pages = NULL; + } + if (!IS_ERR_OR_NULL(mr->ib_umem)) + ib_umem_release(mr->ib_umem); + + kfree(mr); + atomic_dec(&rdev->mr_count); + return rc; +} + +static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) +{ + struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); + + if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) + return -ENOMEM; + + mr->pages[mr->npages++] = addr; + return 0; +} + +int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, + unsigned int *sg_offset) +{ + struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); + + mr->npages = 0; + return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page); +} + +struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, + u32 max_num_sg) +{ + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_mr *mr = NULL; + int rc; + + if (type != IB_MR_TYPE_MEM_REG) { + dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported", type); + return ERR_PTR(-EINVAL); + } + if (max_num_sg > MAX_PBL_LVL_1_PGS) + return ERR_PTR(-EINVAL); + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); + + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; + + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) + goto bail; + + mr->ib_mr.lkey = mr->qplib_mr.lkey; + mr->ib_mr.rkey = mr->ib_mr.lkey; + + mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); + if (!mr->pages) { + rc = -ENOMEM; + goto fail; + } + rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, + &mr->qplib_frpl, max_num_sg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to allocate HW FR page list"); + goto fail_mr; + } + + atomic_inc(&rdev->mr_count); + return &mr->ib_mr; + +fail_mr: + kfree(mr->pages); +fail: + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); +bail: + kfree(mr); + return ERR_PTR(rc); +} + +struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_mw *mw; + int rc; + + mw = kzalloc(sizeof(*mw), GFP_KERNEL); + if (!mw) + return ERR_PTR(-ENOMEM); + mw->rdev = rdev; + mw->qplib_mw.pd = &pd->qplib_pd; + + mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? + CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 : + CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B); + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); + if (rc) { + dev_err(rdev_to_dev(rdev), "Allocate MW failed!"); + goto fail; + } + mw->ib_mw.rkey = mw->qplib_mw.rkey; + + atomic_inc(&rdev->mw_count); + return &mw->ib_mw; + +fail: + kfree(mw); + return ERR_PTR(rc); +} + +int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) +{ + struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw); + struct bnxt_re_dev *rdev = mw->rdev; + int rc; + + rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); + if (rc) { + dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc); + return rc; + } + + kfree(mw); + atomic_dec(&rdev->mw_count); + return rc; +} + +static int bnxt_re_page_size_ok(int page_shift) +{ + switch (page_shift) { + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G: + return 1; + default: + return 0; + } +} + +static int fill_umem_pbl_tbl(struct ib_umem *umem, u64 *pbl_tbl_orig, + int page_shift) +{ + u64 *pbl_tbl = pbl_tbl_orig; + u64 paddr; + u64 page_mask = (1ULL << page_shift) - 1; + int i, pages; + struct scatterlist *sg; + int entry; + + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { + pages = sg_dma_len(sg) >> PAGE_SHIFT; + for (i = 0; i < pages; i++) { + paddr = sg_dma_address(sg) + (i << PAGE_SHIFT); + if (pbl_tbl == pbl_tbl_orig) + *pbl_tbl++ = paddr & ~page_mask; + else if ((paddr & page_mask) == 0) + *pbl_tbl++ = paddr; + } + } + return pbl_tbl - pbl_tbl_orig; +} + +/* uverbs */ +struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_mr *mr; + struct ib_umem *umem; + u64 *pbl_tbl = NULL; + int umem_pgs, page_shift, rc; + + if (length > BNXT_RE_MAX_MR_SIZE) { + dev_err(rdev_to_dev(rdev), "MR Size: %lld > Max supported:%lld\n", + length, BNXT_RE_MAX_MR_SIZE); + return ERR_PTR(-ENOMEM); + } + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); + + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; + + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to allocate MR"); + goto free_mr; + } + /* The fixed portion of the rkey is the same as the lkey */ + mr->ib_mr.rkey = mr->qplib_mr.rkey; + + umem = ib_umem_get(ib_pd->uobject->context, start, length, + mr_access_flags, 0); + if (IS_ERR(umem)) { + dev_err(rdev_to_dev(rdev), "Failed to get umem"); + rc = -EFAULT; + goto free_mrw; + } + mr->ib_umem = umem; + + mr->qplib_mr.va = virt_addr; + umem_pgs = ib_umem_page_count(umem); + if (!umem_pgs) { + dev_err(rdev_to_dev(rdev), "umem is invalid!"); + rc = -EINVAL; + goto free_umem; + } + mr->qplib_mr.total_size = length; + + pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL); + if (!pbl_tbl) { + rc = -ENOMEM; + goto free_umem; + } + + page_shift = umem->page_shift; + + if (!bnxt_re_page_size_ok(page_shift)) { + dev_err(rdev_to_dev(rdev), "umem page size unsupported!"); + rc = -EFAULT; + goto fail; + } + + if (!umem->hugetlb && length > BNXT_RE_MAX_MR_SIZE_LOW) { + dev_err(rdev_to_dev(rdev), "Requested MR Sz:%llu Max sup:%llu", + length, (u64)BNXT_RE_MAX_MR_SIZE_LOW); + rc = -EINVAL; + goto fail; + } + if (umem->hugetlb && length > BNXT_RE_PAGE_SIZE_2M) { + page_shift = BNXT_RE_PAGE_SHIFT_2M; + dev_warn(rdev_to_dev(rdev), "umem hugetlb set page_size %x", + 1 << page_shift); + } + + /* Map umem buf ptrs to the PBL */ + umem_pgs = fill_umem_pbl_tbl(umem, pbl_tbl, page_shift); + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl, + umem_pgs, false, 1 << page_shift); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to register user MR"); + goto fail; + } + + kfree(pbl_tbl); + + mr->ib_mr.lkey = mr->qplib_mr.lkey; + mr->ib_mr.rkey = mr->qplib_mr.lkey; + atomic_inc(&rdev->mr_count); + + return &mr->ib_mr; +fail: + kfree(pbl_tbl); +free_umem: + ib_umem_release(umem); +free_mrw: + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); +free_mr: + kfree(mr); + return ERR_PTR(rc); +} + +struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev, + struct ib_udata *udata) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_re_uctx_resp resp; + struct bnxt_re_ucontext *uctx; + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; + int rc; + + dev_dbg(rdev_to_dev(rdev), "ABI version requested %d", + ibdev->uverbs_abi_ver); + + if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) { + dev_dbg(rdev_to_dev(rdev), " is different from the device %d ", + BNXT_RE_ABI_VERSION); + return ERR_PTR(-EPERM); + } + + uctx = kzalloc(sizeof(*uctx), GFP_KERNEL); + if (!uctx) + return ERR_PTR(-ENOMEM); + + uctx->rdev = rdev; + + uctx->shpg = (void *)__get_free_page(GFP_KERNEL); + if (!uctx->shpg) { + rc = -ENOMEM; + goto fail; + } + spin_lock_init(&uctx->sh_lock); + + resp.dev_id = rdev->en_dev->pdev->devfn; /*Temp, Use idr_alloc instead*/ + resp.max_qp = rdev->qplib_ctx.qpc_count; + resp.pg_size = PAGE_SIZE; + resp.cqe_sz = sizeof(struct cq_base); + resp.max_cqd = dev_attr->max_cq_wqes; + resp.rsvd = 0; + + rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to copy user context"); + rc = -EFAULT; + goto cfail; + } + + return &uctx->ib_uctx; +cfail: + free_page((unsigned long)uctx->shpg); + uctx->shpg = NULL; +fail: + kfree(uctx); + return ERR_PTR(rc); +} + +int bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) +{ + struct bnxt_re_ucontext *uctx = container_of(ib_uctx, + struct bnxt_re_ucontext, + ib_uctx); + + struct bnxt_re_dev *rdev = uctx->rdev; + int rc = 0; + + if (uctx->shpg) + free_page((unsigned long)uctx->shpg); + + if (uctx->dpi.dbr) { + /* Free DPI only if this is the first PD allocated by the + * application and mark the context dpi as NULL + */ + rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, + &rdev->qplib_res.dpi_tbl, + &uctx->dpi); + if (rc) + dev_err(rdev_to_dev(rdev), "Deallocate HW DPI failed!"); + /* Don't fail, continue*/ + uctx->dpi.dbr = NULL; + } + + kfree(uctx); + return 0; +} + +/* Helper function to mmap the virtual memory from user app */ +int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) +{ + struct bnxt_re_ucontext *uctx = container_of(ib_uctx, + struct bnxt_re_ucontext, + ib_uctx); + struct bnxt_re_dev *rdev = uctx->rdev; + u64 pfn; + + if (vma->vm_end - vma->vm_start != PAGE_SIZE) + return -EINVAL; + + if (vma->vm_pgoff) { + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + PAGE_SIZE, vma->vm_page_prot)) { + dev_err(rdev_to_dev(rdev), "Failed to map DPI"); + return -EAGAIN; + } + } else { + pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT; + if (remap_pfn_range(vma, vma->vm_start, + pfn, PAGE_SIZE, vma->vm_page_prot)) { + dev_err(rdev_to_dev(rdev), + "Failed to map shared page"); + return -EAGAIN; + } + } + + return 0; +} diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h new file mode 100644 index 000000000..aa33e7b82 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -0,0 +1,225 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: IB Verbs interpreter (header) + */ + +#ifndef __BNXT_RE_IB_VERBS_H__ +#define __BNXT_RE_IB_VERBS_H__ + +struct bnxt_re_gid_ctx { + u32 idx; + u32 refcnt; +}; + +#define BNXT_RE_FENCE_BYTES 64 +struct bnxt_re_fence_data { + u32 size; + u8 va[BNXT_RE_FENCE_BYTES]; + dma_addr_t dma_addr; + struct bnxt_re_mr *mr; + struct ib_mw *mw; + struct bnxt_qplib_swqe bind_wqe; + u32 bind_rkey; +}; + +struct bnxt_re_pd { + struct bnxt_re_dev *rdev; + struct ib_pd ib_pd; + struct bnxt_qplib_pd qplib_pd; + struct bnxt_re_fence_data fence; +}; + +struct bnxt_re_ah { + struct bnxt_re_dev *rdev; + struct ib_ah ib_ah; + struct bnxt_qplib_ah qplib_ah; +}; + +struct bnxt_re_srq { + struct bnxt_re_dev *rdev; + u32 srq_limit; + struct ib_srq ib_srq; + struct bnxt_qplib_srq qplib_srq; + struct ib_umem *umem; + spinlock_t lock; /* protect srq */ +}; + +struct bnxt_re_qp { + struct list_head list; + struct bnxt_re_dev *rdev; + struct ib_qp ib_qp; + spinlock_t sq_lock; /* protect sq */ + spinlock_t rq_lock; /* protect rq */ + struct bnxt_qplib_qp qplib_qp; + struct ib_umem *sumem; + struct ib_umem *rumem; + /* QP1 */ + u32 send_psn; + struct ib_ud_header qp1_hdr; + struct bnxt_re_cq *scq; + struct bnxt_re_cq *rcq; +}; + +struct bnxt_re_cq { + struct bnxt_re_dev *rdev; + spinlock_t cq_lock; /* protect cq */ + u16 cq_count; + u16 cq_period; + struct ib_cq ib_cq; + struct bnxt_qplib_cq qplib_cq; + struct bnxt_qplib_cqe *cql; +#define MAX_CQL_PER_POLL 1024 + u32 max_cql; + struct ib_umem *umem; +}; + +struct bnxt_re_mr { + struct bnxt_re_dev *rdev; + struct ib_mr ib_mr; + struct ib_umem *ib_umem; + struct bnxt_qplib_mrw qplib_mr; + u32 npages; + u64 *pages; + struct bnxt_qplib_frpl qplib_frpl; +}; + +struct bnxt_re_frpl { + struct bnxt_re_dev *rdev; + struct bnxt_qplib_frpl qplib_frpl; + u64 *page_list; +}; + +struct bnxt_re_fmr { + struct bnxt_re_dev *rdev; + struct ib_fmr ib_fmr; + struct bnxt_qplib_mrw qplib_fmr; +}; + +struct bnxt_re_mw { + struct bnxt_re_dev *rdev; + struct ib_mw ib_mw; + struct bnxt_qplib_mrw qplib_mw; +}; + +struct bnxt_re_ucontext { + struct bnxt_re_dev *rdev; + struct ib_ucontext ib_uctx; + struct bnxt_qplib_dpi dpi; + void *shpg; + spinlock_t sh_lock; /* protect shpg */ +}; + +struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num); + +int bnxt_re_query_device(struct ib_device *ibdev, + struct ib_device_attr *ib_attr, + struct ib_udata *udata); +int bnxt_re_modify_device(struct ib_device *ibdev, + int device_modify_mask, + struct ib_device_modify *device_modify); +int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, + struct ib_port_attr *port_attr); +int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable); +void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str); +int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, + u16 index, u16 *pkey); +int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context); +int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context); +int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, + int index, union ib_gid *gid); +enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, + u8 port_num); +struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev, + struct ib_ucontext *context, + struct ib_udata *udata); +int bnxt_re_dealloc_pd(struct ib_pd *pd); +struct ib_ah *bnxt_re_create_ah(struct ib_pd *pd, + struct rdma_ah_attr *ah_attr, + struct ib_udata *udata); +int bnxt_re_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr); +int bnxt_re_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr); +int bnxt_re_destroy_ah(struct ib_ah *ah); +struct ib_srq *bnxt_re_create_srq(struct ib_pd *pd, + struct ib_srq_init_attr *srq_init_attr, + struct ib_udata *udata); +int bnxt_re_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr, + enum ib_srq_attr_mask srq_attr_mask, + struct ib_udata *udata); +int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); +int bnxt_re_destroy_srq(struct ib_srq *srq); +int bnxt_re_post_srq_recv(struct ib_srq *srq, const struct ib_recv_wr *recv_wr, + const struct ib_recv_wr **bad_recv_wr); +struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd, + struct ib_qp_init_attr *qp_init_attr, + struct ib_udata *udata); +int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_udata *udata); +int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); +int bnxt_re_destroy_qp(struct ib_qp *qp); +int bnxt_re_post_send(struct ib_qp *qp, const struct ib_send_wr *send_wr, + const struct ib_send_wr **bad_send_wr); +int bnxt_re_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, + const struct ib_recv_wr **bad_recv_wr); +struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_ucontext *context, + struct ib_udata *udata); +int bnxt_re_destroy_cq(struct ib_cq *cq); +int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc); +int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); +struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags); + +int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, + unsigned int *sg_offset); +struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type mr_type, + u32 max_num_sg); +int bnxt_re_dereg_mr(struct ib_mr *mr); +struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, + struct ib_udata *udata); +int bnxt_re_dealloc_mw(struct ib_mw *mw); +struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, + struct ib_udata *udata); +struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev, + struct ib_udata *udata); +int bnxt_re_dealloc_ucontext(struct ib_ucontext *context); +int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma); + +unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp); +void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags); +#endif /* __BNXT_RE_IB_VERBS_H__ */ diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c new file mode 100644 index 000000000..f1b666c80 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -0,0 +1,1664 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Main component of the bnxt_re driver + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "bnxt_ulp.h" +#include "roce_hsi.h" +#include "qplib_res.h" +#include "qplib_sp.h" +#include "qplib_fp.h" +#include "qplib_rcfw.h" +#include "bnxt_re.h" +#include "ib_verbs.h" +#include +#include "bnxt.h" +#include "hw_counters.h" + +static char version[] = + BNXT_RE_DESC " v" ROCE_DRV_MODULE_VERSION "\n"; + +MODULE_AUTHOR("Eddie Wai "); +MODULE_DESCRIPTION(BNXT_RE_DESC " Driver"); +MODULE_LICENSE("Dual BSD/GPL"); + +/* globals */ +static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list); +/* Mutex to protect the list of bnxt_re devices added */ +static DEFINE_MUTEX(bnxt_re_dev_lock); +static struct workqueue_struct *bnxt_re_wq; +static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev); + +/* SR-IOV helper functions */ + +static void bnxt_re_get_sriov_func_type(struct bnxt_re_dev *rdev) +{ + struct bnxt *bp; + + bp = netdev_priv(rdev->en_dev->net); + if (BNXT_VF(bp)) + rdev->is_virtfn = 1; +} + +/* Set the maximum number of each resource that the driver actually wants + * to allocate. This may be up to the maximum number the firmware has + * reserved for the function. The driver may choose to allocate fewer + * resources than the firmware maximum. + */ +static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) +{ + u32 vf_qps = 0, vf_srqs = 0, vf_cqs = 0, vf_mrws = 0, vf_gids = 0; + u32 i; + u32 vf_pct; + u32 num_vfs; + struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; + + rdev->qplib_ctx.qpc_count = min_t(u32, BNXT_RE_MAX_QPC_COUNT, + dev_attr->max_qp); + + rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT_256K; + /* Use max_mr from fw since max_mrw does not get set */ + rdev->qplib_ctx.mrw_count = min_t(u32, rdev->qplib_ctx.mrw_count, + dev_attr->max_mr); + rdev->qplib_ctx.srqc_count = min_t(u32, BNXT_RE_MAX_SRQC_COUNT, + dev_attr->max_srq); + rdev->qplib_ctx.cq_count = min_t(u32, BNXT_RE_MAX_CQ_COUNT, + dev_attr->max_cq); + + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) + rdev->qplib_ctx.tqm_count[i] = + rdev->dev_attr.tqm_alloc_reqs[i]; + + if (rdev->num_vfs) { + /* + * Reserve a set of resources for the PF. Divide the remaining + * resources among the VFs + */ + vf_pct = 100 - BNXT_RE_PCT_RSVD_FOR_PF; + num_vfs = 100 * rdev->num_vfs; + vf_qps = (rdev->qplib_ctx.qpc_count * vf_pct) / num_vfs; + vf_srqs = (rdev->qplib_ctx.srqc_count * vf_pct) / num_vfs; + vf_cqs = (rdev->qplib_ctx.cq_count * vf_pct) / num_vfs; + /* + * The driver allows many more MRs than other resources. If the + * firmware does also, then reserve a fixed amount for the PF + * and divide the rest among VFs. VFs may use many MRs for NFS + * mounts, ISER, NVME applications, etc. If the firmware + * severely restricts the number of MRs, then let PF have + * half and divide the rest among VFs, as for the other + * resource types. + */ + if (rdev->qplib_ctx.mrw_count < BNXT_RE_MAX_MRW_COUNT_64K) + vf_mrws = rdev->qplib_ctx.mrw_count * vf_pct / num_vfs; + else + vf_mrws = (rdev->qplib_ctx.mrw_count - + BNXT_RE_RESVD_MR_FOR_PF) / rdev->num_vfs; + vf_gids = BNXT_RE_MAX_GID_PER_VF; + } + rdev->qplib_ctx.vf_res.max_mrw_per_vf = vf_mrws; + rdev->qplib_ctx.vf_res.max_gid_per_vf = vf_gids; + rdev->qplib_ctx.vf_res.max_qp_per_vf = vf_qps; + rdev->qplib_ctx.vf_res.max_srq_per_vf = vf_srqs; + rdev->qplib_ctx.vf_res.max_cq_per_vf = vf_cqs; +} + +/* for handling bnxt_en callbacks later */ +static void bnxt_re_stop(void *p) +{ +} + +static void bnxt_re_start(void *p) +{ +} + +static void bnxt_re_sriov_config(void *p, int num_vfs) +{ + struct bnxt_re_dev *rdev = p; + + if (!rdev) + return; + + rdev->num_vfs = num_vfs; + bnxt_re_set_resource_limits(rdev); + bnxt_qplib_set_func_resources(&rdev->qplib_res, &rdev->rcfw, + &rdev->qplib_ctx); +} + +static void bnxt_re_shutdown(void *p) +{ + struct bnxt_re_dev *rdev = p; + + if (!rdev) + return; + + bnxt_re_ib_unreg(rdev); +} + +static void bnxt_re_stop_irq(void *handle) +{ + struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle; + struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw; + struct bnxt_qplib_nq *nq; + int indx; + + for (indx = BNXT_RE_NQ_IDX; indx < rdev->num_msix; indx++) { + nq = &rdev->nq[indx - 1]; + bnxt_qplib_nq_stop_irq(nq, false); + } + + bnxt_qplib_rcfw_stop_irq(rcfw, false); +} + +static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent) +{ + struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle; + struct bnxt_msix_entry *msix_ent = rdev->msix_entries; + struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw; + struct bnxt_qplib_nq *nq; + int indx, rc; + + if (!ent) { + /* Not setting the f/w timeout bit in rcfw. + * During the driver unload the first command + * to f/w will timeout and that will set the + * timeout bit. + */ + dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n"); + return; + } + + /* Vectors may change after restart, so update with new vectors + * in device sctructure. + */ + for (indx = 0; indx < rdev->num_msix; indx++) + rdev->msix_entries[indx].vector = ent[indx].vector; + + bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector, + false); + for (indx = BNXT_RE_NQ_IDX ; indx < rdev->num_msix; indx++) { + nq = &rdev->nq[indx - 1]; + rc = bnxt_qplib_nq_start_irq(nq, indx - 1, + msix_ent[indx].vector, false); + if (rc) + dev_warn(rdev_to_dev(rdev), + "Failed to reinit NQ index %d\n", indx - 1); + } +} + +static struct bnxt_ulp_ops bnxt_re_ulp_ops = { + .ulp_async_notifier = NULL, + .ulp_stop = bnxt_re_stop, + .ulp_start = bnxt_re_start, + .ulp_sriov_config = bnxt_re_sriov_config, + .ulp_shutdown = bnxt_re_shutdown, + .ulp_irq_stop = bnxt_re_stop_irq, + .ulp_irq_restart = bnxt_re_start_irq +}; + +/* RoCE -> Net driver */ + +/* Driver registration routines used to let the networking driver (bnxt_en) + * to know that the RoCE driver is now installed + */ +static int bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev; + int rc; + + if (!rdev) + return -EINVAL; + + en_dev = rdev->en_dev; + + rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, + BNXT_ROCE_ULP); + return rc; +} + +static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev; + int rc = 0; + + if (!rdev) + return -EINVAL; + + en_dev = rdev->en_dev; + + rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP, + &bnxt_re_ulp_ops, rdev); + return rc; +} + +static int bnxt_re_free_msix(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev; + int rc; + + if (!rdev) + return -EINVAL; + + en_dev = rdev->en_dev; + + + rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); + + return rc; +} + +static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) +{ + int rc = 0, num_msix_want = BNXT_RE_MAX_MSIX, num_msix_got; + struct bnxt_en_dev *en_dev; + + if (!rdev) + return -EINVAL; + + en_dev = rdev->en_dev; + + num_msix_want = min_t(u32, BNXT_RE_MAX_MSIX, num_online_cpus()); + + num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, + rdev->msix_entries, + num_msix_want); + if (num_msix_got < BNXT_RE_MIN_MSIX) { + rc = -EINVAL; + goto done; + } + if (num_msix_got != num_msix_want) { + dev_warn(rdev_to_dev(rdev), + "Requested %d MSI-X vectors, got %d\n", + num_msix_want, num_msix_got); + } + rdev->num_msix = num_msix_got; +done: + return rc; +} + +static void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, struct input *hdr, + u16 opcd, u16 crid, u16 trid) +{ + hdr->req_type = cpu_to_le16(opcd); + hdr->cmpl_ring = cpu_to_le16(crid); + hdr->target_id = cpu_to_le16(trid); +} + +static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg, + int msg_len, void *resp, int resp_max_len, + int timeout) +{ + fw_msg->msg = msg; + fw_msg->msg_len = msg_len; + fw_msg->resp = resp; + fw_msg->resp_max_len = resp_max_len; + fw_msg->timeout = timeout; +} + +static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_ring_free_input req = {0}; + struct hwrm_ring_free_output resp; + struct bnxt_fw_msg fw_msg; + int rc = -EINVAL; + + if (!en_dev) + return rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); + req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; + req.ring_id = cpu_to_le16(fw_ring_id); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) + dev_err(rdev_to_dev(rdev), + "Failed to free HW ring:%d :%#x", req.ring_id, rc); + return rc; +} + +static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, dma_addr_t *dma_arr, + int pages, int type, u32 ring_mask, + u32 map_index, u16 *fw_ring_id) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_ring_alloc_input req = {0}; + struct hwrm_ring_alloc_output resp; + struct bnxt_fw_msg fw_msg; + int rc = -EINVAL; + + if (!en_dev) + return rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); + req.enables = 0; + req.page_tbl_addr = cpu_to_le64(dma_arr[0]); + if (pages > 1) { + /* Page size is in log2 units */ + req.page_size = BNXT_PAGE_SHIFT; + req.page_tbl_depth = 1; + } + req.fbo = 0; + /* Association of ring index with doorbell index and MSIX number */ + req.logical_id = cpu_to_le16(map_index); + req.length = cpu_to_le32(ring_mask + 1); + req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; + req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (!rc) + *fw_ring_id = le16_to_cpu(resp.ring_id); + + return rc; +} + +static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, + u32 fw_stats_ctx_id) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_stat_ctx_free_input req = {0}; + struct bnxt_fw_msg fw_msg; + int rc = -EINVAL; + + if (!en_dev) + return rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, -1); + req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&req, + sizeof(req), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) + dev_err(rdev_to_dev(rdev), + "Failed to free HW stats context %#x", rc); + + return rc; +} + +static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, + dma_addr_t dma_map, + u32 *fw_stats_ctx_id) +{ + struct hwrm_stat_ctx_alloc_output resp = {0}; + struct hwrm_stat_ctx_alloc_input req = {0}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + int rc = -EINVAL; + + *fw_stats_ctx_id = INVALID_STATS_CTX_ID; + + if (!en_dev) + return rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_ALLOC, -1, -1); + req.update_period_ms = cpu_to_le32(1000); + req.stats_dma_addr = cpu_to_le64(dma_map); + req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE; + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (!rc) + *fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id); + + return rc; +} + +/* Device */ + +static bool is_bnxt_re_dev(struct net_device *netdev) +{ + struct ethtool_drvinfo drvinfo; + + if (netdev->ethtool_ops && netdev->ethtool_ops->get_drvinfo) { + memset(&drvinfo, 0, sizeof(drvinfo)); + netdev->ethtool_ops->get_drvinfo(netdev, &drvinfo); + + if (strcmp(drvinfo.driver, "bnxt_en")) + return false; + return true; + } + return false; +} + +static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev) +{ + struct bnxt_re_dev *rdev; + + rcu_read_lock(); + list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { + if (rdev->netdev == netdev) { + rcu_read_unlock(); + return rdev; + } + } + rcu_read_unlock(); + return NULL; +} + +static void bnxt_re_dev_unprobe(struct net_device *netdev, + struct bnxt_en_dev *en_dev) +{ + dev_put(netdev); + module_put(en_dev->pdev->driver->driver.owner); +} + +static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev) +{ + struct bnxt *bp = netdev_priv(netdev); + struct bnxt_en_dev *en_dev; + struct pci_dev *pdev; + + /* Call bnxt_en's RoCE probe via indirect API */ + if (!bp->ulp_probe) + return ERR_PTR(-EINVAL); + + en_dev = bp->ulp_probe(netdev); + if (IS_ERR(en_dev)) + return en_dev; + + pdev = en_dev->pdev; + if (!pdev) + return ERR_PTR(-EINVAL); + + if (!(en_dev->flags & BNXT_EN_FLAG_ROCE_CAP)) { + dev_info(&pdev->dev, + "%s: probe error: RoCE is not supported on this device", + ROCE_DRV_MODULE_NAME); + return ERR_PTR(-ENODEV); + } + + /* Bump net device reference count */ + if (!try_module_get(pdev->driver->driver.owner)) + return ERR_PTR(-ENODEV); + + dev_hold(netdev); + + return en_dev; +} + +static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev) +{ + ib_unregister_device(&rdev->ibdev); +} + +static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) +{ + struct ib_device *ibdev = &rdev->ibdev; + + /* ib device init */ + ibdev->owner = THIS_MODULE; + ibdev->node_type = RDMA_NODE_IB_CA; + strlcpy(ibdev->name, "bnxt_re%d", IB_DEVICE_NAME_MAX); + strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", + strlen(BNXT_RE_DESC) + 5); + ibdev->phys_port_cnt = 1; + + bnxt_qplib_get_guid(rdev->netdev->dev_addr, (u8 *)&ibdev->node_guid); + + ibdev->num_comp_vectors = 1; + ibdev->dev.parent = &rdev->en_dev->pdev->dev; + ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; + + /* User space */ + ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; + ibdev->uverbs_cmd_mask = + (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | + (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | + (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | + (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_REG_MR) | + (1ull << IB_USER_VERBS_CMD_REREG_MR) | + (1ull << IB_USER_VERBS_CMD_DEREG_MR) | + (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | + (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | + (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_QP) | + (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | + (1ull << IB_USER_VERBS_CMD_QUERY_QP) | + (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | + (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | + (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | + (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_AH) | + (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | + (1ull << IB_USER_VERBS_CMD_QUERY_AH) | + (1ull << IB_USER_VERBS_CMD_DESTROY_AH); + /* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */ + + /* Kernel verbs */ + ibdev->query_device = bnxt_re_query_device; + ibdev->modify_device = bnxt_re_modify_device; + + ibdev->query_port = bnxt_re_query_port; + ibdev->get_port_immutable = bnxt_re_get_port_immutable; + ibdev->get_dev_fw_str = bnxt_re_query_fw_str; + ibdev->query_pkey = bnxt_re_query_pkey; + ibdev->get_netdev = bnxt_re_get_netdev; + ibdev->add_gid = bnxt_re_add_gid; + ibdev->del_gid = bnxt_re_del_gid; + ibdev->get_link_layer = bnxt_re_get_link_layer; + + ibdev->alloc_pd = bnxt_re_alloc_pd; + ibdev->dealloc_pd = bnxt_re_dealloc_pd; + + ibdev->create_ah = bnxt_re_create_ah; + ibdev->modify_ah = bnxt_re_modify_ah; + ibdev->query_ah = bnxt_re_query_ah; + ibdev->destroy_ah = bnxt_re_destroy_ah; + + ibdev->create_srq = bnxt_re_create_srq; + ibdev->modify_srq = bnxt_re_modify_srq; + ibdev->query_srq = bnxt_re_query_srq; + ibdev->destroy_srq = bnxt_re_destroy_srq; + ibdev->post_srq_recv = bnxt_re_post_srq_recv; + + ibdev->create_qp = bnxt_re_create_qp; + ibdev->modify_qp = bnxt_re_modify_qp; + ibdev->query_qp = bnxt_re_query_qp; + ibdev->destroy_qp = bnxt_re_destroy_qp; + + ibdev->post_send = bnxt_re_post_send; + ibdev->post_recv = bnxt_re_post_recv; + + ibdev->create_cq = bnxt_re_create_cq; + ibdev->destroy_cq = bnxt_re_destroy_cq; + ibdev->poll_cq = bnxt_re_poll_cq; + ibdev->req_notify_cq = bnxt_re_req_notify_cq; + + ibdev->get_dma_mr = bnxt_re_get_dma_mr; + ibdev->dereg_mr = bnxt_re_dereg_mr; + ibdev->alloc_mr = bnxt_re_alloc_mr; + ibdev->map_mr_sg = bnxt_re_map_mr_sg; + + ibdev->reg_user_mr = bnxt_re_reg_user_mr; + ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; + ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; + ibdev->mmap = bnxt_re_mmap; + ibdev->get_hw_stats = bnxt_re_ib_get_hw_stats; + ibdev->alloc_hw_stats = bnxt_re_ib_alloc_hw_stats; + + ibdev->driver_id = RDMA_DRIVER_BNXT_RE; + return ib_register_device(ibdev, NULL); +} + +static ssize_t show_rev(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); + + return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); +} + +static ssize_t show_hca(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); + + return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); +} + +static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); +static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); + +static struct device_attribute *bnxt_re_attributes[] = { + &dev_attr_hw_rev, + &dev_attr_hca_type +}; + +static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev) +{ + dev_put(rdev->netdev); + rdev->netdev = NULL; + + mutex_lock(&bnxt_re_dev_lock); + list_del_rcu(&rdev->list); + mutex_unlock(&bnxt_re_dev_lock); + + synchronize_rcu(); + + ib_dealloc_device(&rdev->ibdev); + /* rdev is gone */ +} + +static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev, + struct bnxt_en_dev *en_dev) +{ + struct bnxt_re_dev *rdev; + + /* Allocate bnxt_re_dev instance here */ + rdev = (struct bnxt_re_dev *)ib_alloc_device(sizeof(*rdev)); + if (!rdev) { + dev_err(NULL, "%s: bnxt_re_dev allocation failure!", + ROCE_DRV_MODULE_NAME); + return NULL; + } + /* Default values */ + rdev->netdev = netdev; + dev_hold(rdev->netdev); + rdev->en_dev = en_dev; + rdev->id = rdev->en_dev->pdev->devfn; + INIT_LIST_HEAD(&rdev->qp_list); + mutex_init(&rdev->qp_lock); + atomic_set(&rdev->qp_count, 0); + atomic_set(&rdev->cq_count, 0); + atomic_set(&rdev->srq_count, 0); + atomic_set(&rdev->mr_count, 0); + atomic_set(&rdev->mw_count, 0); + rdev->cosq[0] = 0xFFFF; + rdev->cosq[1] = 0xFFFF; + + mutex_lock(&bnxt_re_dev_lock); + list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); + mutex_unlock(&bnxt_re_dev_lock); + return rdev; +} + +static int bnxt_re_handle_unaffi_async_event(struct creq_func_event + *unaffi_async) +{ + switch (unaffi_async->event) { + case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: + break; + default: + return -EINVAL; + } + return 0; +} + +static int bnxt_re_handle_qp_async_event(struct creq_qp_event *qp_event, + struct bnxt_re_qp *qp) +{ + struct ib_event event; + unsigned int flags; + + if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR && + rdma_is_kernel_res(&qp->ib_qp.res)) { + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_add_flush_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + } + + memset(&event, 0, sizeof(event)); + if (qp->qplib_qp.srq) { + event.device = &qp->rdev->ibdev; + event.element.qp = &qp->ib_qp; + event.event = IB_EVENT_QP_LAST_WQE_REACHED; + } + + if (event.device && qp->ib_qp.event_handler) + qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); + + return 0; +} + +static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async, + void *obj) +{ + int rc = 0; + u8 event; + + if (!obj) + return rc; /* QP was already dead, still return success */ + + event = affi_async->event; + if (event == CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION) { + struct bnxt_qplib_qp *lib_qp = obj; + struct bnxt_re_qp *qp = container_of(lib_qp, struct bnxt_re_qp, + qplib_qp); + rc = bnxt_re_handle_qp_async_event(affi_async, qp); + } + return rc; +} + +static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, + void *aeqe, void *obj) +{ + struct creq_qp_event *affi_async; + struct creq_func_event *unaffi_async; + u8 type; + int rc; + + type = ((struct creq_base *)aeqe)->type; + if (type == CREQ_BASE_TYPE_FUNC_EVENT) { + unaffi_async = aeqe; + rc = bnxt_re_handle_unaffi_async_event(unaffi_async); + } else { + affi_async = aeqe; + rc = bnxt_re_handle_affi_async_event(affi_async, obj); + } + + return rc; +} + +static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_srq *handle, u8 event) +{ + struct bnxt_re_srq *srq = container_of(handle, struct bnxt_re_srq, + qplib_srq); + struct ib_event ib_event; + int rc = 0; + + if (!srq) { + dev_err(NULL, "%s: SRQ is NULL, SRQN not handled", + ROCE_DRV_MODULE_NAME); + rc = -EINVAL; + goto done; + } + ib_event.device = &srq->rdev->ibdev; + ib_event.element.srq = &srq->ib_srq; + if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT) + ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED; + else + ib_event.event = IB_EVENT_SRQ_ERR; + + if (srq->ib_srq.event_handler) { + /* Lock event_handler? */ + (*srq->ib_srq.event_handler)(&ib_event, + srq->ib_srq.srq_context); + } +done: + return rc; +} + +static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_cq *handle) +{ + struct bnxt_re_cq *cq = container_of(handle, struct bnxt_re_cq, + qplib_cq); + + if (!cq) { + dev_err(NULL, "%s: CQ is NULL, CQN not handled", + ROCE_DRV_MODULE_NAME); + return -EINVAL; + } + if (cq->ib_cq.comp_handler) { + /* Lock comp_handler? */ + (*cq->ib_cq.comp_handler)(&cq->ib_cq, cq->ib_cq.cq_context); + } + + return 0; +} + +static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev) +{ + int i; + + for (i = 1; i < rdev->num_msix; i++) + bnxt_qplib_disable_nq(&rdev->nq[i - 1]); + + if (rdev->qplib_res.rcfw) + bnxt_qplib_cleanup_res(&rdev->qplib_res); +} + +static int bnxt_re_init_res(struct bnxt_re_dev *rdev) +{ + int rc = 0, i; + int num_vec_enabled = 0; + + bnxt_qplib_init_res(&rdev->qplib_res); + + for (i = 1; i < rdev->num_msix ; i++) { + rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq[i - 1], + i - 1, rdev->msix_entries[i].vector, + rdev->msix_entries[i].db_offset, + &bnxt_re_cqn_handler, + &bnxt_re_srqn_handler); + + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to enable NQ with rc = 0x%x", rc); + goto fail; + } + num_vec_enabled++; + } + return 0; +fail: + for (i = num_vec_enabled; i >= 0; i--) + bnxt_qplib_disable_nq(&rdev->nq[i]); + + return rc; +} + +static void bnxt_re_free_nq_res(struct bnxt_re_dev *rdev) +{ + int i; + + for (i = 0; i < rdev->num_msix - 1; i++) { + bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id); + bnxt_qplib_free_nq(&rdev->nq[i]); + } +} + +static void bnxt_re_free_res(struct bnxt_re_dev *rdev) +{ + bnxt_re_free_nq_res(rdev); + + if (rdev->qplib_res.dpi_tbl.max) { + bnxt_qplib_dealloc_dpi(&rdev->qplib_res, + &rdev->qplib_res.dpi_tbl, + &rdev->dpi_privileged); + } + if (rdev->qplib_res.rcfw) { + bnxt_qplib_free_res(&rdev->qplib_res); + rdev->qplib_res.rcfw = NULL; + } +} + +static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev) +{ + int rc = 0, i; + int num_vec_created = 0; + + /* Configure and allocate resources for qplib */ + rdev->qplib_res.rcfw = &rdev->rcfw; + rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr, + rdev->is_virtfn); + if (rc) + goto fail; + + rc = bnxt_qplib_alloc_res(&rdev->qplib_res, rdev->en_dev->pdev, + rdev->netdev, &rdev->dev_attr); + if (rc) + goto fail; + + rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, + &rdev->dpi_privileged, + rdev); + if (rc) + goto dealloc_res; + + for (i = 0; i < rdev->num_msix - 1; i++) { + rdev->nq[i].hwq.max_elements = BNXT_RE_MAX_CQ_COUNT + + BNXT_RE_MAX_SRQC_COUNT + 2; + rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq[i]); + if (rc) { + dev_err(rdev_to_dev(rdev), "Alloc Failed NQ%d rc:%#x", + i, rc); + goto free_nq; + } + rc = bnxt_re_net_ring_alloc + (rdev, rdev->nq[i].hwq.pbl[PBL_LVL_0].pg_map_arr, + rdev->nq[i].hwq.pbl[rdev->nq[i].hwq.level].pg_count, + HWRM_RING_ALLOC_CMPL, + BNXT_QPLIB_NQE_MAX_CNT - 1, + rdev->msix_entries[i + 1].ring_idx, + &rdev->nq[i].ring_id); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to allocate NQ fw id with rc = 0x%x", + rc); + bnxt_qplib_free_nq(&rdev->nq[i]); + goto free_nq; + } + num_vec_created++; + } + return 0; +free_nq: + for (i = num_vec_created; i >= 0; i--) { + bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id); + bnxt_qplib_free_nq(&rdev->nq[i]); + } + bnxt_qplib_dealloc_dpi(&rdev->qplib_res, + &rdev->qplib_res.dpi_tbl, + &rdev->dpi_privileged); +dealloc_res: + bnxt_qplib_free_res(&rdev->qplib_res); + +fail: + rdev->qplib_res.rcfw = NULL; + return rc; +} + +static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, + u8 port_num, enum ib_event_type event) +{ + struct ib_event ib_event; + + ib_event.device = ibdev; + if (qp) { + ib_event.element.qp = qp; + ib_event.event = event; + if (qp->event_handler) + qp->event_handler(&ib_event, qp->qp_context); + + } else { + ib_event.element.port_num = port_num; + ib_event.event = event; + ib_dispatch_event(&ib_event); + } +} + +#define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN 0x02 +static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir, + u64 *cid_map) +{ + struct hwrm_queue_pri2cos_qcfg_input req = {0}; + struct bnxt *bp = netdev_priv(rdev->netdev); + struct hwrm_queue_pri2cos_qcfg_output resp; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + u32 flags = 0; + u8 *qcfgmap, *tmp_map; + int rc = 0, i; + + if (!cid_map) + return -EINVAL; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_QUEUE_PRI2COS_QCFG, -1, -1); + flags |= (dir & 0x01); + flags |= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN; + req.flags = cpu_to_le32(flags); + req.port_id = bp->pf.port_id; + + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) + return rc; + + if (resp.queue_cfg_info) { + dev_warn(rdev_to_dev(rdev), + "Asymmetric cos queue configuration detected"); + dev_warn(rdev_to_dev(rdev), + " on device, QoS may not be fully functional\n"); + } + qcfgmap = &resp.pri0_cos_queue_id; + tmp_map = (u8 *)cid_map; + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) + tmp_map[i] = qcfgmap[i]; + + return rc; +} + +static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp) +{ + return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->qp1_sqp); +} + +static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev) +{ + int mask = IB_QP_STATE; + struct ib_qp_attr qp_attr; + struct bnxt_re_qp *qp; + + qp_attr.qp_state = IB_QPS_ERR; + mutex_lock(&rdev->qp_lock); + list_for_each_entry(qp, &rdev->qp_list, list) { + /* Modify the state of all QPs except QP1/Shadow QP */ + if (!bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { + if (qp->qplib_qp.state != + CMDQ_MODIFY_QP_NEW_STATE_RESET && + qp->qplib_qp.state != + CMDQ_MODIFY_QP_NEW_STATE_ERR) { + bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, + 1, IB_EVENT_QP_FATAL); + bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask, + NULL); + } + } + } + mutex_unlock(&rdev->qp_lock); +} + +static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; + struct bnxt_qplib_gid gid; + u16 gid_idx, index; + int rc = 0; + + if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) + return 0; + + if (!sgid_tbl) { + dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated"); + return -EINVAL; + } + + for (index = 0; index < sgid_tbl->active; index++) { + gid_idx = sgid_tbl->hw_id[index]; + + if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero))) + continue; + /* need to modify the VLAN enable setting of non VLAN GID only + * as setting is done for VLAN GID while adding GID + */ + if (sgid_tbl->vlan[index]) + continue; + + memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid)); + + rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx, + rdev->qplib_res.netdev->dev_addr); + } + + return rc; +} + +static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev) +{ + u32 prio_map = 0, tmp_map = 0; + struct net_device *netdev; + struct dcb_app app; + + netdev = rdev->netdev; + + memset(&app, 0, sizeof(app)); + app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE; + app.protocol = ETH_P_IBOE; + tmp_map = dcb_ieee_getapp_mask(netdev, &app); + prio_map = tmp_map; + + app.selector = IEEE_8021QAZ_APP_SEL_DGRAM; + app.protocol = ROCE_V2_UDP_DPORT; + tmp_map = dcb_ieee_getapp_mask(netdev, &app); + prio_map |= tmp_map; + + return prio_map; +} + +static void bnxt_re_parse_cid_map(u8 prio_map, u8 *cid_map, u16 *cosq) +{ + u16 prio; + u8 id; + + for (prio = 0, id = 0; prio < 8; prio++) { + if (prio_map & (1 << prio)) { + cosq[id] = cid_map[prio]; + id++; + if (id == 2) /* Max 2 tcs supported */ + break; + } + } +} + +static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev) +{ + u8 prio_map = 0; + u64 cid_map; + int rc; + + /* Get priority for roce */ + prio_map = bnxt_re_get_priority_mask(rdev); + + if (prio_map == rdev->cur_prio_map) + return 0; + rdev->cur_prio_map = prio_map; + /* Get cosq id for this priority */ + rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &cid_map); + if (rc) { + dev_warn(rdev_to_dev(rdev), "no cos for p_mask %x\n", prio_map); + return rc; + } + /* Parse CoS IDs for app priority */ + bnxt_re_parse_cid_map(prio_map, (u8 *)&cid_map, rdev->cosq); + + /* Config BONO. */ + rc = bnxt_qplib_map_tc2cos(&rdev->qplib_res, rdev->cosq); + if (rc) { + dev_warn(rdev_to_dev(rdev), "no tc for cos{%x, %x}\n", + rdev->cosq[0], rdev->cosq[1]); + return rc; + } + + /* Actual priorities are not programmed as they are already + * done by L2 driver; just enable or disable priority vlan tagging + */ + if ((prio_map == 0 && rdev->qplib_res.prio) || + (prio_map != 0 && !rdev->qplib_res.prio)) { + rdev->qplib_res.prio = prio_map ? true : false; + + bnxt_re_update_gid(rdev); + } + + return 0; +} + +static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev) +{ + int i, rc; + + if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { + for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) + device_remove_file(&rdev->ibdev.dev, + bnxt_re_attributes[i]); + /* Cleanup ib dev */ + bnxt_re_unregister_ib(rdev); + } + if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags)) + cancel_delayed_work(&rdev->worker); + + if (test_and_clear_bit(BNXT_RE_FLAG_RESOURCES_INITIALIZED, + &rdev->flags)) + bnxt_re_cleanup_res(rdev); + if (test_and_clear_bit(BNXT_RE_FLAG_RESOURCES_ALLOCATED, &rdev->flags)) + bnxt_re_free_res(rdev); + + if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) { + rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); + if (rc) + dev_warn(rdev_to_dev(rdev), + "Failed to deinitialize RCFW: %#x", rc); + bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id); + bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); + bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); + bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id); + bnxt_qplib_free_rcfw_channel(&rdev->rcfw); + } + if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) { + rc = bnxt_re_free_msix(rdev); + if (rc) + dev_warn(rdev_to_dev(rdev), + "Failed to free MSI-X vectors: %#x", rc); + } + if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { + rc = bnxt_re_unregister_netdev(rdev); + if (rc) + dev_warn(rdev_to_dev(rdev), + "Failed to unregister with netdev: %#x", rc); + } +} + +/* worker thread for polling periodic events. Now used for QoS programming*/ +static void bnxt_re_worker(struct work_struct *work) +{ + struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, + worker.work); + + bnxt_re_setup_qos(rdev); + schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); +} + +static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev) +{ + int i, j, rc; + + bool locked; + + /* Acquire rtnl lock through out this function */ + rtnl_lock(); + locked = true; + + /* Registered a new RoCE device instance to netdev */ + rc = bnxt_re_register_netdev(rdev); + if (rc) { + rtnl_unlock(); + pr_err("Failed to register with netedev: %#x\n", rc); + return -EINVAL; + } + set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); + + /* Check whether VF or PF */ + bnxt_re_get_sriov_func_type(rdev); + + rc = bnxt_re_request_msix(rdev); + if (rc) { + pr_err("Failed to get MSI-X vectors: %#x\n", rc); + rc = -EINVAL; + goto fail; + } + set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); + + /* Establish RCFW Communication Channel to initialize the context + * memory for the function and all child VFs + */ + rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw, + BNXT_RE_MAX_QPC_COUNT); + if (rc) { + pr_err("Failed to allocate RCFW Channel: %#x\n", rc); + goto fail; + } + rc = bnxt_re_net_ring_alloc + (rdev, rdev->rcfw.creq.pbl[PBL_LVL_0].pg_map_arr, + rdev->rcfw.creq.pbl[rdev->rcfw.creq.level].pg_count, + HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_CREQE_MAX_CNT - 1, + rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx, + &rdev->rcfw.creq_ring_id); + if (rc) { + pr_err("Failed to allocate CREQ: %#x\n", rc); + goto free_rcfw; + } + rc = bnxt_qplib_enable_rcfw_channel + (rdev->en_dev->pdev, &rdev->rcfw, + rdev->msix_entries[BNXT_RE_AEQ_IDX].vector, + rdev->msix_entries[BNXT_RE_AEQ_IDX].db_offset, + rdev->is_virtfn, &bnxt_re_aeq_handler); + if (rc) { + pr_err("Failed to enable RCFW channel: %#x\n", rc); + goto free_ring; + } + + rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr, + rdev->is_virtfn); + if (rc) + goto disable_rcfw; + if (!rdev->is_virtfn) + bnxt_re_set_resource_limits(rdev); + + rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0); + if (rc) { + pr_err("Failed to allocate QPLIB context: %#x\n", rc); + goto disable_rcfw; + } + rc = bnxt_re_net_stats_ctx_alloc(rdev, + rdev->qplib_ctx.stats.dma_map, + &rdev->qplib_ctx.stats.fw_id); + if (rc) { + pr_err("Failed to allocate stats context: %#x\n", rc); + goto free_ctx; + } + + rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx, + rdev->is_virtfn); + if (rc) { + pr_err("Failed to initialize RCFW: %#x\n", rc); + goto free_sctx; + } + set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); + + /* Resources based on the 'new' device caps */ + rc = bnxt_re_alloc_res(rdev); + if (rc) { + pr_err("Failed to allocate resources: %#x\n", rc); + goto fail; + } + set_bit(BNXT_RE_FLAG_RESOURCES_ALLOCATED, &rdev->flags); + rc = bnxt_re_init_res(rdev); + if (rc) { + pr_err("Failed to initialize resources: %#x\n", rc); + goto fail; + } + + set_bit(BNXT_RE_FLAG_RESOURCES_INITIALIZED, &rdev->flags); + + if (!rdev->is_virtfn) { + rc = bnxt_re_setup_qos(rdev); + if (rc) + pr_info("RoCE priority not yet configured\n"); + + INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); + set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags); + schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); + } + + rtnl_unlock(); + locked = false; + + /* Register ib dev */ + rc = bnxt_re_register_ib(rdev); + if (rc) { + pr_err("Failed to register with IB: %#x\n", rc); + goto fail; + } + set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); + dev_info(rdev_to_dev(rdev), "Device registered successfully"); + for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { + rc = device_create_file(&rdev->ibdev.dev, + bnxt_re_attributes[i]); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to create IB sysfs: %#x", rc); + /* Must clean up all created device files */ + for (j = 0; j < i; j++) + device_remove_file(&rdev->ibdev.dev, + bnxt_re_attributes[j]); + bnxt_re_unregister_ib(rdev); + goto fail; + } + } + ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed, + &rdev->active_width); + set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); + + return 0; +free_sctx: + bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id); +free_ctx: + bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); +disable_rcfw: + bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); +free_ring: + bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id); +free_rcfw: + bnxt_qplib_free_rcfw_channel(&rdev->rcfw); +fail: + if (!locked) + rtnl_lock(); + bnxt_re_ib_unreg(rdev); + rtnl_unlock(); + + return rc; +} + +static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct net_device *netdev = rdev->netdev; + + bnxt_re_dev_remove(rdev); + + if (netdev) + bnxt_re_dev_unprobe(netdev, en_dev); +} + +static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev) +{ + struct bnxt_en_dev *en_dev; + int rc = 0; + + if (!is_bnxt_re_dev(netdev)) + return -ENODEV; + + en_dev = bnxt_re_dev_probe(netdev); + if (IS_ERR(en_dev)) { + if (en_dev != ERR_PTR(-ENODEV)) + pr_err("%s: Failed to probe\n", ROCE_DRV_MODULE_NAME); + rc = PTR_ERR(en_dev); + goto exit; + } + *rdev = bnxt_re_dev_add(netdev, en_dev); + if (!*rdev) { + rc = -ENOMEM; + bnxt_re_dev_unprobe(netdev, en_dev); + goto exit; + } +exit: + return rc; +} + +static void bnxt_re_remove_one(struct bnxt_re_dev *rdev) +{ + pci_dev_put(rdev->en_dev->pdev); +} + +/* Handle all deferred netevents tasks */ +static void bnxt_re_task(struct work_struct *work) +{ + struct bnxt_re_work *re_work; + struct bnxt_re_dev *rdev; + int rc = 0; + + re_work = container_of(work, struct bnxt_re_work, work); + rdev = re_work->rdev; + + if (re_work->event != NETDEV_REGISTER && + !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) + return; + + switch (re_work->event) { + case NETDEV_REGISTER: + rc = bnxt_re_ib_reg(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to register with IB: %#x", rc); + bnxt_re_remove_one(rdev); + bnxt_re_dev_unreg(rdev); + goto exit; + } + break; + case NETDEV_UP: + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, + IB_EVENT_PORT_ACTIVE); + break; + case NETDEV_DOWN: + bnxt_re_dev_stop(rdev); + break; + case NETDEV_CHANGE: + if (!netif_carrier_ok(rdev->netdev)) + bnxt_re_dev_stop(rdev); + else if (netif_carrier_ok(rdev->netdev)) + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, + IB_EVENT_PORT_ACTIVE); + ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed, + &rdev->active_width); + break; + default: + break; + } + smp_mb__before_atomic(); + atomic_dec(&rdev->sched_count); +exit: + kfree(re_work); +} + +static void bnxt_re_init_one(struct bnxt_re_dev *rdev) +{ + pci_dev_get(rdev->en_dev->pdev); +} + +/* + * "Notifier chain callback can be invoked for the same chain from + * different CPUs at the same time". + * + * For cases when the netdev is already present, our call to the + * register_netdevice_notifier() will actually get the rtnl_lock() + * before sending NETDEV_REGISTER and (if up) NETDEV_UP + * events. + * + * But for cases when the netdev is not already present, the notifier + * chain is subjected to be invoked from different CPUs simultaneously. + * + * This is protected by the netdev_mutex. + */ +static int bnxt_re_netdev_event(struct notifier_block *notifier, + unsigned long event, void *ptr) +{ + struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr); + struct bnxt_re_work *re_work; + struct bnxt_re_dev *rdev; + int rc = 0; + bool sch_work = false; + + real_dev = rdma_vlan_dev_real_dev(netdev); + if (!real_dev) + real_dev = netdev; + + rdev = bnxt_re_from_netdev(real_dev); + if (!rdev && event != NETDEV_REGISTER) + goto exit; + if (real_dev != netdev) + goto exit; + + switch (event) { + case NETDEV_REGISTER: + if (rdev) + break; + rc = bnxt_re_dev_reg(&rdev, real_dev); + if (rc == -ENODEV) + break; + if (rc) { + pr_err("Failed to register with the device %s: %#x\n", + real_dev->name, rc); + break; + } + bnxt_re_init_one(rdev); + sch_work = true; + break; + + case NETDEV_UNREGISTER: + /* netdev notifier will call NETDEV_UNREGISTER again later since + * we are still holding the reference to the netdev + */ + if (atomic_read(&rdev->sched_count) > 0) + goto exit; + bnxt_re_ib_unreg(rdev); + bnxt_re_remove_one(rdev); + bnxt_re_dev_unreg(rdev); + break; + + default: + sch_work = true; + break; + } + if (sch_work) { + /* Allocate for the deferred task */ + re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC); + if (re_work) { + re_work->rdev = rdev; + re_work->event = event; + re_work->vlan_dev = (real_dev == netdev ? + NULL : netdev); + INIT_WORK(&re_work->work, bnxt_re_task); + atomic_inc(&rdev->sched_count); + queue_work(bnxt_re_wq, &re_work->work); + } + } + +exit: + return NOTIFY_DONE; +} + +static struct notifier_block bnxt_re_netdev_notifier = { + .notifier_call = bnxt_re_netdev_event +}; + +static int __init bnxt_re_mod_init(void) +{ + int rc = 0; + + pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version); + + bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); + if (!bnxt_re_wq) + return -ENOMEM; + + INIT_LIST_HEAD(&bnxt_re_dev_list); + + rc = register_netdevice_notifier(&bnxt_re_netdev_notifier); + if (rc) { + pr_err("%s: Cannot register to netdevice_notifier", + ROCE_DRV_MODULE_NAME); + goto err_netdev; + } + return 0; + +err_netdev: + destroy_workqueue(bnxt_re_wq); + + return rc; +} + +static void __exit bnxt_re_mod_exit(void) +{ + struct bnxt_re_dev *rdev, *next; + LIST_HEAD(to_be_deleted); + + mutex_lock(&bnxt_re_dev_lock); + /* Free all adapter allocated resources */ + if (!list_empty(&bnxt_re_dev_list)) + list_splice_init(&bnxt_re_dev_list, &to_be_deleted); + mutex_unlock(&bnxt_re_dev_lock); + /* + * Cleanup the devices in reverse order so that the VF device + * cleanup is done before PF cleanup + */ + list_for_each_entry_safe_reverse(rdev, next, &to_be_deleted, list) { + dev_info(rdev_to_dev(rdev), "Unregistering Device"); + /* + * Flush out any scheduled tasks before destroying the + * resources + */ + flush_workqueue(bnxt_re_wq); + bnxt_re_dev_stop(rdev); + /* Acquire the rtnl_lock as the L2 resources are freed here */ + rtnl_lock(); + bnxt_re_ib_unreg(rdev); + rtnl_unlock(); + bnxt_re_remove_one(rdev); + bnxt_re_dev_unreg(rdev); + } + unregister_netdevice_notifier(&bnxt_re_netdev_notifier); + if (bnxt_re_wq) + destroy_workqueue(bnxt_re_wq); +} + +module_init(bnxt_re_mod_init); +module_exit(bnxt_re_mod_exit); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c new file mode 100644 index 000000000..d52ae7259 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -0,0 +1,2845 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Fast Path Operators + */ + +#include +#include +#include +#include +#include +#include + +#include "roce_hsi.h" + +#include "qplib_res.h" +#include "qplib_rcfw.h" +#include "qplib_sp.h" +#include "qplib_fp.h" + +static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq); +static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp); +static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type); + +static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp) +{ + qp->sq.condition = false; + qp->sq.send_phantom = false; + qp->sq.single = false; +} + +/* Flush list */ +static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_cq *scq, *rcq; + + scq = qp->scq; + rcq = qp->rcq; + + if (!qp->sq.flushed) { + dev_dbg(&scq->hwq.pdev->dev, + "QPLIB: FP: Adding to SQ Flush list = %p", + qp); + bnxt_qplib_cancel_phantom_processing(qp); + list_add_tail(&qp->sq_flush, &scq->sqf_head); + qp->sq.flushed = true; + } + if (!qp->srq) { + if (!qp->rq.flushed) { + dev_dbg(&rcq->hwq.pdev->dev, + "QPLIB: FP: Adding to RQ Flush list = %p", + qp); + list_add_tail(&qp->rq_flush, &rcq->rqf_head); + qp->rq.flushed = true; + } + } +} + +static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp, + unsigned long *flags) + __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock) +{ + spin_lock_irqsave(&qp->scq->flush_lock, *flags); + if (qp->scq == qp->rcq) + __acquire(&qp->rcq->flush_lock); + else + spin_lock(&qp->rcq->flush_lock); +} + +static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp, + unsigned long *flags) + __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock) +{ + if (qp->scq == qp->rcq) + __release(&qp->rcq->flush_lock); + else + spin_unlock(&qp->rcq->flush_lock); + spin_unlock_irqrestore(&qp->scq->flush_lock, *flags); +} + +void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) +{ + unsigned long flags; + + bnxt_qplib_acquire_cq_flush_locks(qp, &flags); + __bnxt_qplib_add_flush_qp(qp); + bnxt_qplib_release_cq_flush_locks(qp, &flags); +} + +static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp) +{ + if (qp->sq.flushed) { + qp->sq.flushed = false; + list_del(&qp->sq_flush); + } + if (!qp->srq) { + if (qp->rq.flushed) { + qp->rq.flushed = false; + list_del(&qp->rq_flush); + } + } +} + +void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp) +{ + unsigned long flags; + + bnxt_qplib_acquire_cq_flush_locks(qp, &flags); + __clean_cq(qp->scq, (u64)(unsigned long)qp); + qp->sq.hwq.prod = 0; + qp->sq.hwq.cons = 0; + __clean_cq(qp->rcq, (u64)(unsigned long)qp); + qp->rq.hwq.prod = 0; + qp->rq.hwq.cons = 0; + + __bnxt_qplib_del_flush_qp(qp); + bnxt_qplib_release_cq_flush_locks(qp, &flags); +} + +static void bnxt_qpn_cqn_sched_task(struct work_struct *work) +{ + struct bnxt_qplib_nq_work *nq_work = + container_of(work, struct bnxt_qplib_nq_work, work); + + struct bnxt_qplib_cq *cq = nq_work->cq; + struct bnxt_qplib_nq *nq = nq_work->nq; + + if (cq && nq) { + spin_lock_bh(&cq->compl_lock); + if (atomic_read(&cq->arm_state) && nq->cqn_handler) { + dev_dbg(&nq->pdev->dev, + "%s:Trigger cq = %p event nq = %p\n", + __func__, cq, nq); + nq->cqn_handler(nq, cq); + } + spin_unlock_bh(&cq->compl_lock); + } + kfree(nq_work); +} + +static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_q *rq = &qp->rq; + struct bnxt_qplib_q *sq = &qp->sq; + + if (qp->rq_hdr_buf) + dma_free_coherent(&res->pdev->dev, + rq->hwq.max_elements * qp->rq_hdr_buf_size, + qp->rq_hdr_buf, qp->rq_hdr_buf_map); + if (qp->sq_hdr_buf) + dma_free_coherent(&res->pdev->dev, + sq->hwq.max_elements * qp->sq_hdr_buf_size, + qp->sq_hdr_buf, qp->sq_hdr_buf_map); + qp->rq_hdr_buf = NULL; + qp->sq_hdr_buf = NULL; + qp->rq_hdr_buf_map = 0; + qp->sq_hdr_buf_map = 0; + qp->sq_hdr_buf_size = 0; + qp->rq_hdr_buf_size = 0; +} + +static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_q *rq = &qp->rq; + struct bnxt_qplib_q *sq = &qp->sq; + int rc = 0; + + if (qp->sq_hdr_buf_size && sq->hwq.max_elements) { + qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, + sq->hwq.max_elements * + qp->sq_hdr_buf_size, + &qp->sq_hdr_buf_map, GFP_KERNEL); + if (!qp->sq_hdr_buf) { + rc = -ENOMEM; + dev_err(&res->pdev->dev, + "QPLIB: Failed to create sq_hdr_buf"); + goto fail; + } + } + + if (qp->rq_hdr_buf_size && rq->hwq.max_elements) { + qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, + rq->hwq.max_elements * + qp->rq_hdr_buf_size, + &qp->rq_hdr_buf_map, + GFP_KERNEL); + if (!qp->rq_hdr_buf) { + rc = -ENOMEM; + dev_err(&res->pdev->dev, + "QPLIB: Failed to create rq_hdr_buf"); + goto fail; + } + } + return 0; + +fail: + bnxt_qplib_free_qp_hdr_buf(res, qp); + return rc; +} + +static void bnxt_qplib_service_nq(unsigned long data) +{ + struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; + struct bnxt_qplib_hwq *hwq = &nq->hwq; + struct nq_base *nqe, **nq_ptr; + struct bnxt_qplib_cq *cq; + int num_cqne_processed = 0; + int num_srqne_processed = 0; + u32 sw_cons, raw_cons; + u16 type; + int budget = nq->budget; + uintptr_t q_handle; + + /* Service the NQ until empty */ + raw_cons = hwq->cons; + while (budget--) { + sw_cons = HWQ_CMP(raw_cons, hwq); + nq_ptr = (struct nq_base **)hwq->pbl_ptr; + nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]; + if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements)) + break; + + /* + * The valid test of the entry must be done first before + * reading any further. + */ + dma_rmb(); + + type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; + switch (type) { + case NQ_BASE_TYPE_CQ_NOTIFICATION: + { + struct nq_cn *nqcne = (struct nq_cn *)nqe; + + q_handle = le32_to_cpu(nqcne->cq_handle_low); + q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) + << 32; + cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle; + bnxt_qplib_arm_cq_enable(cq); + spin_lock_bh(&cq->compl_lock); + atomic_set(&cq->arm_state, 0); + if (!nq->cqn_handler(nq, (cq))) + num_cqne_processed++; + else + dev_warn(&nq->pdev->dev, + "QPLIB: cqn - type 0x%x not handled", + type); + spin_unlock_bh(&cq->compl_lock); + break; + } + case NQ_BASE_TYPE_SRQ_EVENT: + { + struct nq_srq_event *nqsrqe = + (struct nq_srq_event *)nqe; + + q_handle = le32_to_cpu(nqsrqe->srq_handle_low); + q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) + << 32; + bnxt_qplib_arm_srq((struct bnxt_qplib_srq *)q_handle, + DBR_DBR_TYPE_SRQ_ARMENA); + if (!nq->srqn_handler(nq, + (struct bnxt_qplib_srq *)q_handle, + nqsrqe->event)) + num_srqne_processed++; + else + dev_warn(&nq->pdev->dev, + "QPLIB: SRQ event 0x%x not handled", + nqsrqe->event); + break; + } + case NQ_BASE_TYPE_DBQ_EVENT: + break; + default: + dev_warn(&nq->pdev->dev, + "QPLIB: nqe with type = 0x%x not handled", + type); + break; + } + raw_cons++; + } + if (hwq->cons != raw_cons) { + hwq->cons = raw_cons; + NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements); + } +} + +static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) +{ + struct bnxt_qplib_nq *nq = dev_instance; + struct bnxt_qplib_hwq *hwq = &nq->hwq; + struct nq_base **nq_ptr; + u32 sw_cons; + + /* Prefetch the NQ element */ + sw_cons = HWQ_CMP(hwq->cons, hwq); + nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr; + prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]); + + /* Fan out to CPU affinitized kthreads? */ + tasklet_schedule(&nq->worker); + + return IRQ_HANDLED; +} + +void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill) +{ + tasklet_disable(&nq->worker); + /* Mask h/w interrupt */ + NQ_DB(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements); + /* Sync with last running IRQ handler */ + synchronize_irq(nq->vector); + if (kill) + tasklet_kill(&nq->worker); + if (nq->requested) { + irq_set_affinity_hint(nq->vector, NULL); + free_irq(nq->vector, nq); + nq->requested = false; + } +} + +void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) +{ + if (nq->cqn_wq) { + destroy_workqueue(nq->cqn_wq); + nq->cqn_wq = NULL; + } + + /* Make sure the HW is stopped! */ + if (nq->requested) + bnxt_qplib_nq_stop_irq(nq, true); + + if (nq->bar_reg_iomem) + iounmap(nq->bar_reg_iomem); + nq->bar_reg_iomem = NULL; + + nq->cqn_handler = NULL; + nq->srqn_handler = NULL; + nq->vector = 0; +} + +int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, + int msix_vector, bool need_init) +{ + int rc; + + if (nq->requested) + return -EFAULT; + + nq->vector = msix_vector; + if (need_init) + tasklet_init(&nq->worker, bnxt_qplib_service_nq, + (unsigned long)nq); + else + tasklet_enable(&nq->worker); + + snprintf(nq->name, sizeof(nq->name), "bnxt_qplib_nq-%d", nq_indx); + rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, nq->name, nq); + if (rc) + return rc; + + cpumask_clear(&nq->mask); + cpumask_set_cpu(nq_indx, &nq->mask); + rc = irq_set_affinity_hint(nq->vector, &nq->mask); + if (rc) { + dev_warn(&nq->pdev->dev, + "QPLIB: set affinity failed; vector: %d nq_idx: %d\n", + nq->vector, nq_indx); + } + nq->requested = true; + NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements); + + return rc; +} + +int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, + int nq_idx, int msix_vector, int bar_reg_offset, + int (*cqn_handler)(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_cq *), + int (*srqn_handler)(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_srq *, + u8 event)) +{ + resource_size_t nq_base; + int rc = -1; + + if (cqn_handler) + nq->cqn_handler = cqn_handler; + + if (srqn_handler) + nq->srqn_handler = srqn_handler; + + /* Have a task to schedule CQ notifiers in post send case */ + nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq"); + if (!nq->cqn_wq) + return -ENOMEM; + + nq->bar_reg = NQ_CONS_PCI_BAR_REGION; + nq->bar_reg_off = bar_reg_offset; + nq_base = pci_resource_start(pdev, nq->bar_reg); + if (!nq_base) { + rc = -ENOMEM; + goto fail; + } + nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4); + if (!nq->bar_reg_iomem) { + rc = -ENOMEM; + goto fail; + } + + rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true); + if (rc) { + dev_err(&nq->pdev->dev, + "QPLIB: Failed to request irq for nq-idx %d", nq_idx); + goto fail; + } + + return 0; +fail: + bnxt_qplib_disable_nq(nq); + return rc; +} + +void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq) +{ + if (nq->hwq.max_elements) { + bnxt_qplib_free_hwq(nq->pdev, &nq->hwq); + nq->hwq.max_elements = 0; + } +} + +int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq) +{ + nq->pdev = pdev; + if (!nq->hwq.max_elements || + nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) + nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; + + if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0, + &nq->hwq.max_elements, + BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0, + PAGE_SIZE, HWQ_TYPE_L2_CMPL)) + return -ENOMEM; + + nq->budget = 8; + return 0; +} + +/* SRQ */ +static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type) +{ + struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; + struct dbr_dbr db_msg = { 0 }; + void __iomem *db; + u32 sw_prod = 0; + + /* Ring DB */ + sw_prod = (arm_type == DBR_DBR_TYPE_SRQ_ARM) ? srq->threshold : + HWQ_CMP(srq_hwq->prod, srq_hwq); + db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & + DBR_DBR_INDEX_MASK); + db_msg.type_xid = cpu_to_le32(((srq->id << DBR_DBR_XID_SFT) & + DBR_DBR_XID_MASK) | arm_type); + db = (arm_type == DBR_DBR_TYPE_SRQ_ARMENA) ? + srq->dbr_base : srq->dpi->dbr; + wmb(); /* barrier before db ring */ + __iowrite64_copy(db, &db_msg, sizeof(db_msg) / sizeof(u64)); +} + +int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_destroy_srq req; + struct creq_destroy_srq_resp resp; + u16 cmd_flags = 0; + int rc; + + RCFW_CMD_PREP(req, DESTROY_SRQ, cmd_flags); + + /* Configure the request */ + req.srq_cid = cpu_to_le32(srq->id); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) + return rc; + + bnxt_qplib_free_hwq(res->pdev, &srq->hwq); + kfree(srq->swq); + return 0; +} + +int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_create_srq req; + struct creq_create_srq_resp resp; + struct bnxt_qplib_pbl *pbl; + u16 cmd_flags = 0; + int rc, idx; + + srq->hwq.max_elements = srq->max_wqe; + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &srq->hwq, srq->sglist, + srq->nmap, &srq->hwq.max_elements, + BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, + PAGE_SIZE, HWQ_TYPE_QUEUE); + if (rc) + goto exit; + + srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq), + GFP_KERNEL); + if (!srq->swq) { + rc = -ENOMEM; + goto fail; + } + + RCFW_CMD_PREP(req, CREATE_SRQ, cmd_flags); + + /* Configure the request */ + req.dpi = cpu_to_le32(srq->dpi->dpi); + req.srq_handle = cpu_to_le64((uintptr_t)srq); + + req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements); + pbl = &srq->hwq.pbl[PBL_LVL_0]; + req.pg_size_lvl = cpu_to_le16((((u16)srq->hwq.level & + CMDQ_CREATE_SRQ_LVL_MASK) << + CMDQ_CREATE_SRQ_LVL_SFT) | + (pbl->pg_size == ROCE_PG_SIZE_4K ? + CMDQ_CREATE_SRQ_PG_SIZE_PG_4K : + pbl->pg_size == ROCE_PG_SIZE_8K ? + CMDQ_CREATE_SRQ_PG_SIZE_PG_8K : + pbl->pg_size == ROCE_PG_SIZE_64K ? + CMDQ_CREATE_SRQ_PG_SIZE_PG_64K : + pbl->pg_size == ROCE_PG_SIZE_2M ? + CMDQ_CREATE_SRQ_PG_SIZE_PG_2M : + pbl->pg_size == ROCE_PG_SIZE_8M ? + CMDQ_CREATE_SRQ_PG_SIZE_PG_8M : + pbl->pg_size == ROCE_PG_SIZE_1G ? + CMDQ_CREATE_SRQ_PG_SIZE_PG_1G : + CMDQ_CREATE_SRQ_PG_SIZE_PG_4K)); + req.pbl = cpu_to_le64(pbl->pg_map_arr[0]); + req.pd_id = cpu_to_le32(srq->pd->id); + req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) + goto fail; + + spin_lock_init(&srq->lock); + srq->start_idx = 0; + srq->last_idx = srq->hwq.max_elements - 1; + for (idx = 0; idx < srq->hwq.max_elements; idx++) + srq->swq[idx].next_idx = idx + 1; + srq->swq[srq->last_idx].next_idx = -1; + + srq->id = le32_to_cpu(resp.xid); + srq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem; + if (srq->threshold) + bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARMENA); + srq->arm_req = false; + + return 0; +fail: + bnxt_qplib_free_hwq(res->pdev, &srq->hwq); + kfree(srq->swq); +exit: + return rc; +} + +int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; + u32 sw_prod, sw_cons, count = 0; + + sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq); + sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq); + + count = sw_prod > sw_cons ? sw_prod - sw_cons : + srq_hwq->max_elements - sw_cons + sw_prod; + if (count > srq->threshold) { + srq->arm_req = false; + bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM); + } else { + /* Deferred arming */ + srq->arm_req = true; + } + + return 0; +} + +int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_query_srq req; + struct creq_query_srq_resp resp; + struct bnxt_qplib_rcfw_sbuf *sbuf; + struct creq_query_srq_resp_sb *sb; + u16 cmd_flags = 0; + int rc = 0; + + RCFW_CMD_PREP(req, QUERY_SRQ, cmd_flags); + + /* Configure the request */ + sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); + if (!sbuf) + return -ENOMEM; + req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; + req.srq_cid = cpu_to_le32(srq->id); + sb = sbuf->sb; + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, + (void *)sbuf, 0); + srq->threshold = le16_to_cpu(sb->srq_limit); + bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); + + return rc; +} + +int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; + struct rq_wqe *srqe, **srqe_ptr; + struct sq_sge *hw_sge; + u32 sw_prod, sw_cons, count = 0; + int i, rc = 0, next; + + spin_lock(&srq_hwq->lock); + if (srq->start_idx == srq->last_idx) { + dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!", + srq->id); + rc = -EINVAL; + spin_unlock(&srq_hwq->lock); + goto done; + } + next = srq->start_idx; + srq->start_idx = srq->swq[next].next_idx; + spin_unlock(&srq_hwq->lock); + + sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq); + srqe_ptr = (struct rq_wqe **)srq_hwq->pbl_ptr; + srqe = &srqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)]; + memset(srqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); + /* Calculate wqe_size16 and data_len */ + for (i = 0, hw_sge = (struct sq_sge *)srqe->data; + i < wqe->num_sge; i++, hw_sge++) { + hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); + hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); + hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); + } + srqe->wqe_type = wqe->type; + srqe->flags = wqe->flags; + srqe->wqe_size = wqe->num_sge + + ((offsetof(typeof(*srqe), data) + 15) >> 4); + srqe->wr_id[0] = cpu_to_le32((u32)next); + srq->swq[next].wr_id = wqe->wr_id; + + srq_hwq->prod++; + + spin_lock(&srq_hwq->lock); + sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq); + /* retaining srq_hwq->cons for this logic + * actually the lock is only required to + * read srq_hwq->cons. + */ + sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq); + count = sw_prod > sw_cons ? sw_prod - sw_cons : + srq_hwq->max_elements - sw_cons + sw_prod; + spin_unlock(&srq_hwq->lock); + /* Ring DB */ + bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ); + if (srq->arm_req == true && count > srq->threshold) { + srq->arm_req = false; + bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM); + } +done: + return rc; +} + +/* QP */ +int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_create_qp1 req; + struct creq_create_qp1_resp resp; + struct bnxt_qplib_pbl *pbl; + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_q *rq = &qp->rq; + int rc; + u16 cmd_flags = 0; + u32 qp_flags = 0; + + RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags); + + /* General */ + req.type = qp->type; + req.dpi = cpu_to_le32(qp->dpi->dpi); + req.qp_handle = cpu_to_le64(qp->qp_handle); + + /* SQ */ + sq->hwq.max_elements = sq->max_wqe; + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0, + &sq->hwq.max_elements, + BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0, + PAGE_SIZE, HWQ_TYPE_QUEUE); + if (rc) + goto exit; + + sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); + if (!sq->swq) { + rc = -ENOMEM; + goto fail_sq; + } + pbl = &sq->hwq.pbl[PBL_LVL_0]; + req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); + req.sq_pg_size_sq_lvl = + ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) + << CMDQ_CREATE_QP1_SQ_LVL_SFT) | + (pbl->pg_size == ROCE_PG_SIZE_4K ? + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K : + pbl->pg_size == ROCE_PG_SIZE_8K ? + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K : + pbl->pg_size == ROCE_PG_SIZE_64K ? + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K : + pbl->pg_size == ROCE_PG_SIZE_2M ? + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M : + pbl->pg_size == ROCE_PG_SIZE_8M ? + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M : + pbl->pg_size == ROCE_PG_SIZE_1G ? + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G : + CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K); + + if (qp->scq) + req.scq_cid = cpu_to_le32(qp->scq->id); + + qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; + + /* RQ */ + if (rq->max_wqe) { + rq->hwq.max_elements = qp->rq.max_wqe; + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0, + &rq->hwq.max_elements, + BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, + PAGE_SIZE, HWQ_TYPE_QUEUE); + if (rc) + goto fail_sq; + + rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), + GFP_KERNEL); + if (!rq->swq) { + rc = -ENOMEM; + goto fail_rq; + } + pbl = &rq->hwq.pbl[PBL_LVL_0]; + req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); + req.rq_pg_size_rq_lvl = + ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) << + CMDQ_CREATE_QP1_RQ_LVL_SFT) | + (pbl->pg_size == ROCE_PG_SIZE_4K ? + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K : + pbl->pg_size == ROCE_PG_SIZE_8K ? + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K : + pbl->pg_size == ROCE_PG_SIZE_64K ? + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K : + pbl->pg_size == ROCE_PG_SIZE_2M ? + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M : + pbl->pg_size == ROCE_PG_SIZE_8M ? + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M : + pbl->pg_size == ROCE_PG_SIZE_1G ? + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G : + CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K); + if (qp->rcq) + req.rcq_cid = cpu_to_le32(qp->rcq->id); + } + + /* Header buffer - allow hdr_buf pass in */ + rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); + if (rc) { + rc = -ENOMEM; + goto fail; + } + req.qp_flags = cpu_to_le32(qp_flags); + req.sq_size = cpu_to_le32(sq->hwq.max_elements); + req.rq_size = cpu_to_le32(rq->hwq.max_elements); + + req.sq_fwo_sq_sge = + cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << + CMDQ_CREATE_QP1_SQ_SGE_SFT); + req.rq_fwo_rq_sge = + cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) << + CMDQ_CREATE_QP1_RQ_SGE_SFT); + + req.pd_id = cpu_to_le32(qp->pd->id); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) + goto fail; + + qp->id = le32_to_cpu(resp.xid); + qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; + rcfw->qp_tbl[qp->id].qp_id = qp->id; + rcfw->qp_tbl[qp->id].qp_handle = (void *)qp; + + return 0; + +fail: + bnxt_qplib_free_qp_hdr_buf(res, qp); +fail_rq: + bnxt_qplib_free_hwq(res->pdev, &rq->hwq); + kfree(rq->swq); +fail_sq: + bnxt_qplib_free_hwq(res->pdev, &sq->hwq); + kfree(sq->swq); +exit: + return rc; +} + +int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; + struct cmdq_create_qp req; + struct creq_create_qp_resp resp; + struct bnxt_qplib_pbl *pbl; + struct sq_psn_search **psn_search_ptr; + unsigned long int psn_search, poff = 0; + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_q *rq = &qp->rq; + struct bnxt_qplib_hwq *xrrq; + int i, rc, req_size, psn_sz; + u16 cmd_flags = 0, max_ssge; + u32 sw_prod, qp_flags = 0; + + RCFW_CMD_PREP(req, CREATE_QP, cmd_flags); + + /* General */ + req.type = qp->type; + req.dpi = cpu_to_le32(qp->dpi->dpi); + req.qp_handle = cpu_to_le64(qp->qp_handle); + + /* SQ */ + psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ? + sizeof(struct sq_psn_search) : 0; + sq->hwq.max_elements = sq->max_wqe; + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist, + sq->nmap, &sq->hwq.max_elements, + BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, + psn_sz, + PAGE_SIZE, HWQ_TYPE_QUEUE); + if (rc) + goto exit; + + sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL); + if (!sq->swq) { + rc = -ENOMEM; + goto fail_sq; + } + hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; + if (psn_sz) { + psn_search_ptr = (struct sq_psn_search **) + &hw_sq_send_ptr[get_sqe_pg + (sq->hwq.max_elements)]; + psn_search = (unsigned long int) + &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)] + [get_sqe_idx(sq->hwq.max_elements)]; + if (psn_search & ~PAGE_MASK) { + /* If the psn_search does not start on a page boundary, + * then calculate the offset + */ + poff = (psn_search & ~PAGE_MASK) / + BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE; + } + for (i = 0; i < sq->hwq.max_elements; i++) + sq->swq[i].psn_search = + &psn_search_ptr[get_psne_pg(i + poff)] + [get_psne_idx(i + poff)]; + } + pbl = &sq->hwq.pbl[PBL_LVL_0]; + req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); + req.sq_pg_size_sq_lvl = + ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) + << CMDQ_CREATE_QP_SQ_LVL_SFT) | + (pbl->pg_size == ROCE_PG_SIZE_4K ? + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K : + pbl->pg_size == ROCE_PG_SIZE_8K ? + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K : + pbl->pg_size == ROCE_PG_SIZE_64K ? + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K : + pbl->pg_size == ROCE_PG_SIZE_2M ? + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M : + pbl->pg_size == ROCE_PG_SIZE_8M ? + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M : + pbl->pg_size == ROCE_PG_SIZE_1G ? + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G : + CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K); + + /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */ + hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; + for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) { + hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] + [get_sqe_idx(sw_prod)]; + hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID; + } + + if (qp->scq) + req.scq_cid = cpu_to_le32(qp->scq->id); + + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; + if (qp->sig_type) + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; + + /* RQ */ + if (rq->max_wqe) { + rq->hwq.max_elements = rq->max_wqe; + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist, + rq->nmap, &rq->hwq.max_elements, + BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0, + PAGE_SIZE, HWQ_TYPE_QUEUE); + if (rc) + goto fail_sq; + + rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq), + GFP_KERNEL); + if (!rq->swq) { + rc = -ENOMEM; + goto fail_rq; + } + pbl = &rq->hwq.pbl[PBL_LVL_0]; + req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); + req.rq_pg_size_rq_lvl = + ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) << + CMDQ_CREATE_QP_RQ_LVL_SFT) | + (pbl->pg_size == ROCE_PG_SIZE_4K ? + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K : + pbl->pg_size == ROCE_PG_SIZE_8K ? + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K : + pbl->pg_size == ROCE_PG_SIZE_64K ? + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K : + pbl->pg_size == ROCE_PG_SIZE_2M ? + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M : + pbl->pg_size == ROCE_PG_SIZE_8M ? + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M : + pbl->pg_size == ROCE_PG_SIZE_1G ? + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G : + CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K); + } else { + /* SRQ */ + if (qp->srq) { + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED; + req.srq_cid = cpu_to_le32(qp->srq->id); + } + } + + if (qp->rcq) + req.rcq_cid = cpu_to_le32(qp->rcq->id); + req.qp_flags = cpu_to_le32(qp_flags); + req.sq_size = cpu_to_le32(sq->hwq.max_elements); + req.rq_size = cpu_to_le32(rq->hwq.max_elements); + qp->sq_hdr_buf = NULL; + qp->rq_hdr_buf = NULL; + + rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp); + if (rc) + goto fail_rq; + + /* CTRL-22434: Irrespective of the requested SGE count on the SQ + * always create the QP with max send sges possible if the requested + * inline size is greater than 0. + */ + max_ssge = qp->max_inline_data ? 6 : sq->max_sge; + req.sq_fwo_sq_sge = cpu_to_le16( + ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK) + << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); + req.rq_fwo_rq_sge = cpu_to_le16( + ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK) + << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); + /* ORRQ and IRRQ */ + if (psn_sz) { + xrrq = &qp->orrq; + xrrq->max_elements = + ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); + req_size = xrrq->max_elements * + BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; + req_size &= ~(PAGE_SIZE - 1); + rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, + &xrrq->max_elements, + BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE, + 0, req_size, HWQ_TYPE_CTX); + if (rc) + goto fail_buf_free; + pbl = &xrrq->pbl[PBL_LVL_0]; + req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); + + xrrq = &qp->irrq; + xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( + qp->max_dest_rd_atomic); + req_size = xrrq->max_elements * + BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; + req_size &= ~(PAGE_SIZE - 1); + + rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0, + &xrrq->max_elements, + BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE, + 0, req_size, HWQ_TYPE_CTX); + if (rc) + goto fail_orrq; + + pbl = &xrrq->pbl[PBL_LVL_0]; + req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); + } + req.pd_id = cpu_to_le32(qp->pd->id); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) + goto fail; + + qp->id = le32_to_cpu(resp.xid); + qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; + INIT_LIST_HEAD(&qp->sq_flush); + INIT_LIST_HEAD(&qp->rq_flush); + rcfw->qp_tbl[qp->id].qp_id = qp->id; + rcfw->qp_tbl[qp->id].qp_handle = (void *)qp; + + return 0; + +fail: + if (qp->irrq.max_elements) + bnxt_qplib_free_hwq(res->pdev, &qp->irrq); +fail_orrq: + if (qp->orrq.max_elements) + bnxt_qplib_free_hwq(res->pdev, &qp->orrq); +fail_buf_free: + bnxt_qplib_free_qp_hdr_buf(res, qp); +fail_rq: + bnxt_qplib_free_hwq(res->pdev, &rq->hwq); + kfree(rq->swq); +fail_sq: + bnxt_qplib_free_hwq(res->pdev, &sq->hwq); + kfree(sq->swq); +exit: + return rc; +} + +static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp) +{ + switch (qp->state) { + case CMDQ_MODIFY_QP_NEW_STATE_RTR: + /* INIT->RTR, configure the path_mtu to the default + * 2048 if not being requested + */ + if (!(qp->modify_flags & + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) { + qp->modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; + qp->path_mtu = + CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; + } + qp->modify_flags &= + ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; + /* Bono FW require the max_dest_rd_atomic to be >= 1 */ + if (qp->max_dest_rd_atomic < 1) + qp->max_dest_rd_atomic = 1; + qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; + /* Bono FW 20.6.5 requires SGID_INDEX configuration */ + if (!(qp->modify_flags & + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) { + qp->modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX; + qp->ah.sgid_index = 0; + } + break; + default: + break; + } +} + +static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp) +{ + switch (qp->state) { + case CMDQ_MODIFY_QP_NEW_STATE_RTS: + /* Bono FW requires the max_rd_atomic to be >= 1 */ + if (qp->max_rd_atomic < 1) + qp->max_rd_atomic = 1; + /* Bono FW does not allow PKEY_INDEX, + * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT, + * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN, + * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID + * modification + */ + qp->modify_flags &= + ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY | + CMDQ_MODIFY_QP_MODIFY_MASK_DGID | + CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | + CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | + CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN | + CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER | + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID); + break; + default: + break; + } +} + +static void __filter_modify_flags(struct bnxt_qplib_qp *qp) +{ + switch (qp->cur_qp_state) { + case CMDQ_MODIFY_QP_NEW_STATE_RESET: + break; + case CMDQ_MODIFY_QP_NEW_STATE_INIT: + __modify_flags_from_init_state(qp); + break; + case CMDQ_MODIFY_QP_NEW_STATE_RTR: + __modify_flags_from_rtr_state(qp); + break; + case CMDQ_MODIFY_QP_NEW_STATE_RTS: + break; + case CMDQ_MODIFY_QP_NEW_STATE_SQD: + break; + case CMDQ_MODIFY_QP_NEW_STATE_SQE: + break; + case CMDQ_MODIFY_QP_NEW_STATE_ERR: + break; + default: + break; + } +} + +int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_modify_qp req; + struct creq_modify_qp_resp resp; + u16 cmd_flags = 0, pkey; + u32 temp32[4]; + u32 bmask; + int rc; + + RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags); + + /* Filter out the qp_attr_mask based on the state->new transition */ + __filter_modify_flags(qp); + bmask = qp->modify_flags; + req.modify_mask = cpu_to_le32(qp->modify_flags); + req.qp_cid = cpu_to_le32(qp->id); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) { + req.network_type_en_sqd_async_notify_new_state = + (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | + (qp->en_sqd_async_notify ? + CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0); + } + req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) + req.access = qp->access; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) { + if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl, + qp->pkey_index, &pkey)) + req.pkey = cpu_to_le16(pkey); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) + req.qkey = cpu_to_le32(qp->qkey); + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) { + memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); + req.dgid[0] = cpu_to_le32(temp32[0]); + req.dgid[1] = cpu_to_le32(temp32[1]); + req.dgid[2] = cpu_to_le32(temp32[2]); + req.dgid[3] = cpu_to_le32(temp32[3]); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) + req.flow_label = cpu_to_le32(qp->ah.flow_label); + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) + req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id + [qp->ah.sgid_index]); + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) + req.hop_limit = qp->ah.hop_limit; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) + req.traffic_class = qp->ah.traffic_class; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) + memcpy(req.dest_mac, qp->ah.dmac, 6); + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) + req.path_mtu = qp->path_mtu; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) + req.timeout = qp->timeout; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) + req.retry_cnt = qp->retry_cnt; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) + req.rnr_retry = qp->rnr_retry; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) + req.min_rnr_timer = qp->min_rnr_timer; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) + req.rq_psn = cpu_to_le32(qp->rq.psn); + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) + req.sq_psn = cpu_to_le32(qp->sq.psn); + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) + req.max_rd_atomic = + ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) + req.max_dest_rd_atomic = + IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); + + req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); + req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); + req.sq_sge = cpu_to_le16(qp->sq.max_sge); + req.rq_sge = cpu_to_le16(qp->rq.max_sge); + req.max_inline_data = cpu_to_le32(qp->max_inline_data); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID) + req.dest_qp_id = cpu_to_le32(qp->dest_qpn); + + req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) + return rc; + qp->cur_qp_state = qp->state; + return 0; +} + +int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_query_qp req; + struct creq_query_qp_resp resp; + struct bnxt_qplib_rcfw_sbuf *sbuf; + struct creq_query_qp_resp_sb *sb; + u16 cmd_flags = 0; + u32 temp32[4]; + int i, rc = 0; + + RCFW_CMD_PREP(req, QUERY_QP, cmd_flags); + + sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); + if (!sbuf) + return -ENOMEM; + sb = sbuf->sb; + + req.qp_cid = cpu_to_le32(qp->id); + req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, + (void *)sbuf, 0); + if (rc) + goto bail; + /* Extract the context from the side buffer */ + qp->state = sb->en_sqd_async_notify_state & + CREQ_QUERY_QP_RESP_SB_STATE_MASK; + qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & + CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? + true : false; + qp->access = sb->access; + qp->pkey_index = le16_to_cpu(sb->pkey); + qp->qkey = le32_to_cpu(sb->qkey); + + temp32[0] = le32_to_cpu(sb->dgid[0]); + temp32[1] = le32_to_cpu(sb->dgid[1]); + temp32[2] = le32_to_cpu(sb->dgid[2]); + temp32[3] = le32_to_cpu(sb->dgid[3]); + memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); + + qp->ah.flow_label = le32_to_cpu(sb->flow_label); + + qp->ah.sgid_index = 0; + for (i = 0; i < res->sgid_tbl.max; i++) { + if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { + qp->ah.sgid_index = i; + break; + } + } + if (i == res->sgid_tbl.max) + dev_warn(&res->pdev->dev, "QPLIB: SGID not found??"); + + qp->ah.hop_limit = sb->hop_limit; + qp->ah.traffic_class = sb->traffic_class; + memcpy(qp->ah.dmac, sb->dest_mac, 6); + qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & + CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >> + CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT; + qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & + CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >> + CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT; + qp->timeout = sb->timeout; + qp->retry_cnt = sb->retry_cnt; + qp->rnr_retry = sb->rnr_retry; + qp->min_rnr_timer = sb->min_rnr_timer; + qp->rq.psn = le32_to_cpu(sb->rq_psn); + qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); + qp->sq.psn = le32_to_cpu(sb->sq_psn); + qp->max_dest_rd_atomic = + IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); + qp->sq.max_wqe = qp->sq.hwq.max_elements; + qp->rq.max_wqe = qp->rq.hwq.max_elements; + qp->sq.max_sge = le16_to_cpu(sb->sq_sge); + qp->rq.max_sge = le16_to_cpu(sb->rq_sge); + qp->max_inline_data = le32_to_cpu(sb->max_inline_data); + qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); + memcpy(qp->smac, sb->src_mac, 6); + qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); +bail: + bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); + return rc; +} + +static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) +{ + struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; + struct cq_base *hw_cqe, **hw_cqe_ptr; + int i; + + for (i = 0; i < cq_hwq->max_elements; i++) { + hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr; + hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)]; + if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements)) + continue; + /* + * The valid test of the entry must be done first before + * reading any further. + */ + dma_rmb(); + switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { + case CQ_BASE_CQE_TYPE_REQ: + case CQ_BASE_CQE_TYPE_TERMINAL: + { + struct cq_req *cqe = (struct cq_req *)hw_cqe; + + if (qp == le64_to_cpu(cqe->qp_handle)) + cqe->qp_handle = 0; + break; + } + case CQ_BASE_CQE_TYPE_RES_RC: + case CQ_BASE_CQE_TYPE_RES_UD: + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: + { + struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe; + + if (qp == le64_to_cpu(cqe->qp_handle)) + cqe->qp_handle = 0; + break; + } + default: + break; + } + } +} + +int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_destroy_qp req; + struct creq_destroy_qp_resp resp; + u16 cmd_flags = 0; + int rc; + + rcfw->qp_tbl[qp->id].qp_id = BNXT_QPLIB_QP_ID_INVALID; + rcfw->qp_tbl[qp->id].qp_handle = NULL; + + RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags); + + req.qp_cid = cpu_to_le32(qp->id); + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) { + rcfw->qp_tbl[qp->id].qp_id = qp->id; + rcfw->qp_tbl[qp->id].qp_handle = qp; + return rc; + } + + return 0; +} + +void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp) +{ + bnxt_qplib_free_qp_hdr_buf(res, qp); + bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq); + kfree(qp->sq.swq); + + bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq); + kfree(qp->rq.swq); + + if (qp->irrq.max_elements) + bnxt_qplib_free_hwq(res->pdev, &qp->irrq); + if (qp->orrq.max_elements) + bnxt_qplib_free_hwq(res->pdev, &qp->orrq); + +} + +void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge) +{ + struct bnxt_qplib_q *sq = &qp->sq; + u32 sw_prod; + + memset(sge, 0, sizeof(*sge)); + + if (qp->sq_hdr_buf) { + sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); + sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map + + sw_prod * qp->sq_hdr_buf_size); + sge->lkey = 0xFFFFFFFF; + sge->size = qp->sq_hdr_buf_size; + return qp->sq_hdr_buf + sw_prod * sge->size; + } + return NULL; +} + +u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_q *rq = &qp->rq; + + return HWQ_CMP(rq->hwq.prod, &rq->hwq); +} + +dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) +{ + return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size); +} + +void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge) +{ + struct bnxt_qplib_q *rq = &qp->rq; + u32 sw_prod; + + memset(sge, 0, sizeof(*sge)); + + if (qp->rq_hdr_buf) { + sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); + sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map + + sw_prod * qp->rq_hdr_buf_size); + sge->lkey = 0xFFFFFFFF; + sge->size = qp->rq_hdr_buf_size; + return qp->rq_hdr_buf + sw_prod * sge->size; + } + return NULL; +} + +void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_q *sq = &qp->sq; + struct dbr_dbr db_msg = { 0 }; + u32 sw_prod; + + sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); + + db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & + DBR_DBR_INDEX_MASK); + db_msg.type_xid = + cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | + DBR_DBR_TYPE_SQ); + /* Flush all the WQE writes to HW */ + wmb(); + __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); +} + +int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_swq *swq; + struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; + struct sq_sge *hw_sge; + struct bnxt_qplib_nq_work *nq_work = NULL; + bool sch_handler = false; + u32 sw_prod; + u8 wqe_size16; + int i, rc = 0, data_len = 0, pkt_num = 0; + __le32 temp32; + + if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) { + if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { + sch_handler = true; + dev_dbg(&sq->hwq.pdev->dev, + "%s Error QP. Scheduling for poll_cq\n", + __func__); + goto queue_err; + } + } + + if (bnxt_qplib_queue_full(sq)) { + dev_err(&sq->hwq.pdev->dev, + "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x", + sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements, + sq->q_full_delta); + rc = -ENOMEM; + goto done; + } + sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); + swq = &sq->swq[sw_prod]; + swq->wr_id = wqe->wr_id; + swq->type = wqe->type; + swq->flags = wqe->flags; + if (qp->sig_type) + swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; + swq->start_psn = sq->psn & BTH_PSN_MASK; + + hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; + hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] + [get_sqe_idx(sw_prod)]; + + memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE); + + if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { + /* Copy the inline data */ + if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { + dev_warn(&sq->hwq.pdev->dev, + "QPLIB: Inline data length > 96 detected"); + data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH; + } else { + data_len = wqe->inline_len; + } + memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len); + wqe_size16 = (data_len + 15) >> 4; + } else { + for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data; + i < wqe->num_sge; i++, hw_sge++) { + hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); + hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); + hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); + data_len += wqe->sg_list[i].size; + } + /* Each SGE entry = 1 WQE size16 */ + wqe_size16 = wqe->num_sge; + /* HW requires wqe size has room for atleast one SGE even if + * none was supplied by ULP + */ + if (!wqe->num_sge) + wqe_size16++; + } + + /* Specifics */ + switch (wqe->type) { + case BNXT_QPLIB_SWQE_TYPE_SEND: + if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { + /* Assemble info for Raw Ethertype QPs */ + struct sq_send_raweth_qp1 *sqe = + (struct sq_send_raweth_qp1 *)hw_sq_send_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->wqe_size = wqe_size16 + + ((offsetof(typeof(*sqe), data) + 15) >> 4); + sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); + sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); + sqe->length = cpu_to_le32(data_len); + sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & + SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << + SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); + + break; + } + /* fall thru */ + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: + { + struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->wqe_size = wqe_size16 + + ((offsetof(typeof(*sqe), data) + 15) >> 4); + sqe->inv_key_or_imm_data = cpu_to_le32( + wqe->send.inv_key); + if (qp->type == CMDQ_CREATE_QP_TYPE_UD) { + sqe->q_key = cpu_to_le32(wqe->send.q_key); + sqe->dst_qp = cpu_to_le32( + wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); + sqe->length = cpu_to_le32(data_len); + sqe->avid = cpu_to_le32(wqe->send.avid & + SQ_SEND_AVID_MASK); + sq->psn = (sq->psn + 1) & BTH_PSN_MASK; + } else { + sqe->length = cpu_to_le32(data_len); + sqe->dst_qp = 0; + sqe->avid = 0; + if (qp->mtu) + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; + if (!pkt_num) + pkt_num = 1; + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; + } + break; + } + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: + case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: + { + struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->wqe_size = wqe_size16 + + ((offsetof(typeof(*sqe), data) + 15) >> 4); + sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); + sqe->length = cpu_to_le32((u32)data_len); + sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); + sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); + if (qp->mtu) + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; + if (!pkt_num) + pkt_num = 1; + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; + break; + } + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: + { + struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); + sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); + sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); + sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); + if (qp->mtu) + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; + if (!pkt_num) + pkt_num = 1; + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; + break; + } + case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: + { + struct sq_localinvalidate *sqe = + (struct sq_localinvalidate *)hw_sq_send_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); + + break; + } + case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: + { + struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->access_cntl = wqe->frmr.access_cntl | + SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; + sqe->zero_based_page_size_log = + (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << + SQ_FR_PMR_PAGE_SIZE_LOG_SFT | + (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0); + sqe->l_key = cpu_to_le32(wqe->frmr.l_key); + temp32 = cpu_to_le32(wqe->frmr.length); + memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); + sqe->numlevels_pbl_page_size_log = + ((wqe->frmr.pbl_pg_sz_log << + SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) & + SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) | + ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & + SQ_FR_PMR_NUMLEVELS_MASK); + + for (i = 0; i < wqe->frmr.page_list_len; i++) + wqe->frmr.pbl_ptr[i] = cpu_to_le64( + wqe->frmr.page_list[i] | + PTU_PTE_VALID); + sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); + sqe->va = cpu_to_le64(wqe->frmr.va); + + break; + } + case BNXT_QPLIB_SWQE_TYPE_BIND_MW: + { + struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->access_cntl = wqe->bind.access_cntl; + sqe->mw_type_zero_based = wqe->bind.mw_type | + (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0); + sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); + sqe->l_key = cpu_to_le32(wqe->bind.r_key); + sqe->va = cpu_to_le64(wqe->bind.va); + temp32 = cpu_to_le32(wqe->bind.length); + memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length)); + break; + } + default: + /* Bad wqe, return error */ + rc = -EINVAL; + goto done; + } + swq->next_psn = sq->psn & BTH_PSN_MASK; + if (swq->psn_search) { + swq->psn_search->opcode_start_psn = cpu_to_le32( + ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & + SQ_PSN_SEARCH_START_PSN_MASK) | + ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & + SQ_PSN_SEARCH_OPCODE_MASK)); + swq->psn_search->flags_next_psn = cpu_to_le32( + ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & + SQ_PSN_SEARCH_NEXT_PSN_MASK)); + } +queue_err: + if (sch_handler) { + /* Store the ULP info in the software structures */ + sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); + swq = &sq->swq[sw_prod]; + swq->wr_id = wqe->wr_id; + swq->type = wqe->type; + swq->flags = wqe->flags; + if (qp->sig_type) + swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; + swq->start_psn = sq->psn & BTH_PSN_MASK; + } + sq->hwq.prod++; + qp->wqe_cnt++; + +done: + if (sch_handler) { + nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); + if (nq_work) { + nq_work->cq = qp->scq; + nq_work->nq = qp->scq->nq; + INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); + queue_work(qp->scq->nq->cqn_wq, &nq_work->work); + } else { + dev_err(&sq->hwq.pdev->dev, + "QPLIB: FP: Failed to allocate SQ nq_work!"); + rc = -ENOMEM; + } + } + return rc; +} + +void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_q *rq = &qp->rq; + struct dbr_dbr db_msg = { 0 }; + u32 sw_prod; + + sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); + db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & + DBR_DBR_INDEX_MASK); + db_msg.type_xid = + cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | + DBR_DBR_TYPE_RQ); + + /* Flush the writes to HW Rx WQE before the ringing Rx DB */ + wmb(); + __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); +} + +int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_qplib_q *rq = &qp->rq; + struct rq_wqe *rqe, **rqe_ptr; + struct sq_sge *hw_sge; + struct bnxt_qplib_nq_work *nq_work = NULL; + bool sch_handler = false; + u32 sw_prod; + int i, rc = 0; + + if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { + sch_handler = true; + dev_dbg(&rq->hwq.pdev->dev, + "%s Error QP. Scheduling for poll_cq\n", + __func__); + goto queue_err; + } + if (bnxt_qplib_queue_full(rq)) { + dev_err(&rq->hwq.pdev->dev, + "QPLIB: FP: QP (0x%x) RQ is full!", qp->id); + rc = -EINVAL; + goto done; + } + sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); + rq->swq[sw_prod].wr_id = wqe->wr_id; + + rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr; + rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)]; + + memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE); + + /* Calculate wqe_size16 and data_len */ + for (i = 0, hw_sge = (struct sq_sge *)rqe->data; + i < wqe->num_sge; i++, hw_sge++) { + hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); + hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); + hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); + } + rqe->wqe_type = wqe->type; + rqe->flags = wqe->flags; + rqe->wqe_size = wqe->num_sge + + ((offsetof(typeof(*rqe), data) + 15) >> 4); + /* HW requires wqe size has room for atleast one SGE even if none + * was supplied by ULP + */ + if (!wqe->num_sge) + rqe->wqe_size++; + + /* Supply the rqe->wr_id index to the wr_id_tbl for now */ + rqe->wr_id[0] = cpu_to_le32(sw_prod); + +queue_err: + if (sch_handler) { + /* Store the ULP info in the software structures */ + sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); + rq->swq[sw_prod].wr_id = wqe->wr_id; + } + + rq->hwq.prod++; + if (sch_handler) { + nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); + if (nq_work) { + nq_work->cq = qp->rcq; + nq_work->nq = qp->rcq->nq; + INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); + queue_work(qp->rcq->nq->cqn_wq, &nq_work->work); + } else { + dev_err(&rq->hwq.pdev->dev, + "QPLIB: FP: Failed to allocate RQ nq_work!"); + rc = -ENOMEM; + } + } +done: + return rc; +} + +/* CQ */ + +/* Spinlock must be held */ +static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq) +{ + struct dbr_dbr db_msg = { 0 }; + + db_msg.type_xid = + cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | + DBR_DBR_TYPE_CQ_ARMENA); + /* Flush memory writes before enabling the CQ */ + wmb(); + __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64)); +} + +static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type) +{ + struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; + struct dbr_dbr db_msg = { 0 }; + u32 sw_cons; + + /* Ring DB */ + sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq); + db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) & + DBR_DBR_INDEX_MASK); + db_msg.type_xid = + cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | + arm_type); + /* flush memory writes before arming the CQ */ + wmb(); + __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); +} + +int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_create_cq req; + struct creq_create_cq_resp resp; + struct bnxt_qplib_pbl *pbl; + u16 cmd_flags = 0; + int rc; + + cq->hwq.max_elements = cq->max_wqe; + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead, + cq->nmap, &cq->hwq.max_elements, + BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0, + PAGE_SIZE, HWQ_TYPE_QUEUE); + if (rc) + goto exit; + + RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags); + + if (!cq->dpi) { + dev_err(&rcfw->pdev->dev, + "QPLIB: FP: CREATE_CQ failed due to NULL DPI"); + return -EINVAL; + } + req.dpi = cpu_to_le32(cq->dpi->dpi); + req.cq_handle = cpu_to_le64(cq->cq_handle); + + req.cq_size = cpu_to_le32(cq->hwq.max_elements); + pbl = &cq->hwq.pbl[PBL_LVL_0]; + req.pg_size_lvl = cpu_to_le32( + ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) << + CMDQ_CREATE_CQ_LVL_SFT) | + (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K : + pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K : + pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K : + pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M : + pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M : + pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G : + CMDQ_CREATE_CQ_PG_SIZE_PG_4K)); + + req.pbl = cpu_to_le64(pbl->pg_map_arr[0]); + + req.cq_fco_cnq_id = cpu_to_le32( + (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << + CMDQ_CREATE_CQ_CNQ_ID_SFT); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) + goto fail; + + cq->id = le32_to_cpu(resp.xid); + cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem; + cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; + init_waitqueue_head(&cq->waitq); + INIT_LIST_HEAD(&cq->sqf_head); + INIT_LIST_HEAD(&cq->rqf_head); + spin_lock_init(&cq->compl_lock); + spin_lock_init(&cq->flush_lock); + + bnxt_qplib_arm_cq_enable(cq); + return 0; + +fail: + bnxt_qplib_free_hwq(res->pdev, &cq->hwq); +exit: + return rc; +} + +int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_destroy_cq req; + struct creq_destroy_cq_resp resp; + u16 cmd_flags = 0; + int rc; + + RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags); + + req.cq_cid = cpu_to_le32(cq->id); + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) + return rc; + bnxt_qplib_free_hwq(res->pdev, &cq->hwq); + return 0; +} + +static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, + struct bnxt_qplib_cqe **pcqe, int *budget) +{ + u32 sw_prod, sw_cons; + struct bnxt_qplib_cqe *cqe; + int rc = 0; + + /* Now complete all outstanding SQEs with FLUSHED_ERR */ + sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); + cqe = *pcqe; + while (*budget) { + sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); + if (sw_cons == sw_prod) { + break; + } + /* Skip the FENCE WQE completions */ + if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) { + bnxt_qplib_cancel_phantom_processing(qp); + goto skip_compl; + } + memset(cqe, 0, sizeof(*cqe)); + cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; + cqe->qp_handle = (u64)(unsigned long)qp; + cqe->wr_id = sq->swq[sw_cons].wr_id; + cqe->src_qp = qp->id; + cqe->type = sq->swq[sw_cons].type; + cqe++; + (*budget)--; +skip_compl: + sq->hwq.cons++; + } + *pcqe = cqe; + if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod) + /* Out of budget */ + rc = -EAGAIN; + + return rc; +} + +static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, + struct bnxt_qplib_cqe **pcqe, int *budget) +{ + struct bnxt_qplib_cqe *cqe; + u32 sw_prod, sw_cons; + int rc = 0; + int opcode = 0; + + switch (qp->type) { + case CMDQ_CREATE_QP1_TYPE_GSI: + opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1; + break; + case CMDQ_CREATE_QP_TYPE_RC: + opcode = CQ_BASE_CQE_TYPE_RES_RC; + break; + case CMDQ_CREATE_QP_TYPE_UD: + opcode = CQ_BASE_CQE_TYPE_RES_UD; + break; + } + + /* Flush the rest of the RQ */ + sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); + cqe = *pcqe; + while (*budget) { + sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq); + if (sw_cons == sw_prod) + break; + memset(cqe, 0, sizeof(*cqe)); + cqe->status = + CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; + cqe->opcode = opcode; + cqe->qp_handle = (unsigned long)qp; + cqe->wr_id = rq->swq[sw_cons].wr_id; + cqe++; + (*budget)--; + rq->hwq.cons++; + } + *pcqe = cqe; + if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod) + /* Out of budget */ + rc = -EAGAIN; + + return rc; +} + +void bnxt_qplib_mark_qp_error(void *qp_handle) +{ + struct bnxt_qplib_qp *qp = qp_handle; + + if (!qp) + return; + + /* Must block new posting of SQ and RQ */ + qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; + bnxt_qplib_cancel_phantom_processing(qp); +} + +/* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive) + * CQE is track from sw_cq_cons to max_element but valid only if VALID=1 + */ +static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, + u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons) +{ + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_swq *swq; + u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx; + struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr; + struct cq_req *peek_req_hwcqe; + struct bnxt_qplib_qp *peek_qp; + struct bnxt_qplib_q *peek_sq; + int i, rc = 0; + + /* Normal mode */ + /* Check for the psn_search marking before completing */ + swq = &sq->swq[sw_sq_cons]; + if (swq->psn_search && + le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { + /* Unmark */ + swq->psn_search->flags_next_psn = cpu_to_le32 + (le32_to_cpu(swq->psn_search->flags_next_psn) + & ~0x80000000); + dev_dbg(&cq->hwq.pdev->dev, + "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n", + cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); + sq->condition = true; + sq->send_phantom = true; + + /* TODO: Only ARM if the previous SQE is ARMALL */ + bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL); + + rc = -EAGAIN; + goto out; + } + if (sq->condition) { + /* Peek at the completions */ + peek_raw_cq_cons = cq->hwq.cons; + peek_sw_cq_cons = cq_cons; + i = cq->hwq.max_elements; + while (i--) { + peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq); + peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; + peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)] + [CQE_IDX(peek_sw_cq_cons)]; + /* If the next hwcqe is VALID */ + if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons, + cq->hwq.max_elements)) { + /* + * The valid test of the entry must be done first before + * reading any further. + */ + dma_rmb(); + /* If the next hwcqe is a REQ */ + if ((peek_hwcqe->cqe_type_toggle & + CQ_BASE_CQE_TYPE_MASK) == + CQ_BASE_CQE_TYPE_REQ) { + peek_req_hwcqe = (struct cq_req *) + peek_hwcqe; + peek_qp = (struct bnxt_qplib_qp *) + ((unsigned long) + le64_to_cpu + (peek_req_hwcqe->qp_handle)); + peek_sq = &peek_qp->sq; + peek_sq_cons_idx = HWQ_CMP(le16_to_cpu( + peek_req_hwcqe->sq_cons_idx) - 1 + , &sq->hwq); + /* If the hwcqe's sq's wr_id matches */ + if (peek_sq == sq && + sq->swq[peek_sq_cons_idx].wr_id == + BNXT_QPLIB_FENCE_WRID) { + /* + * Unbreak only if the phantom + * comes back + */ + dev_dbg(&cq->hwq.pdev->dev, + "FP:Got Phantom CQE"); + sq->condition = false; + sq->single = true; + rc = 0; + goto out; + } + } + /* Valid but not the phantom, so keep looping */ + } else { + /* Not valid yet, just exit and wait */ + rc = -EINVAL; + goto out; + } + peek_sw_cq_cons++; + peek_raw_cq_cons++; + } + dev_err(&cq->hwq.pdev->dev, + "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x", + cq_cons, qp->id, sw_sq_cons, cqe_sq_cons); + rc = -EINVAL; + } +out: + return rc; +} + +static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, + struct cq_req *hwcqe, + struct bnxt_qplib_cqe **pcqe, int *budget, + u32 cq_cons, struct bnxt_qplib_qp **lib_qp) +{ + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *sq; + struct bnxt_qplib_cqe *cqe; + u32 sw_sq_cons, cqe_sq_cons; + struct bnxt_qplib_swq *swq; + int rc = 0; + + qp = (struct bnxt_qplib_qp *)((unsigned long) + le64_to_cpu(hwcqe->qp_handle)); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: Process Req qp is NULL"); + return -EINVAL; + } + sq = &qp->sq; + + cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq); + if (cqe_sq_cons > sq->hwq.max_elements) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process req reported "); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", + cqe_sq_cons, sq->hwq.max_elements); + return -EINVAL; + } + + if (qp->sq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + /* Require to walk the sq's swq to fabricate CQEs for all previously + * signaled SWQEs due to CQE aggregation from the current sq cons + * to the cqe_sq_cons + */ + cqe = *pcqe; + while (*budget) { + sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); + if (sw_sq_cons == cqe_sq_cons) + /* Done */ + break; + + swq = &sq->swq[sw_sq_cons]; + memset(cqe, 0, sizeof(*cqe)); + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; + cqe->qp_handle = (u64)(unsigned long)qp; + cqe->src_qp = qp->id; + cqe->wr_id = swq->wr_id; + if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID) + goto skip; + cqe->type = swq->type; + + /* For the last CQE, check for status. For errors, regardless + * of the request being signaled or not, it must complete with + * the hwcqe error status + */ + if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons && + hwcqe->status != CQ_REQ_STATUS_OK) { + cqe->status = hwcqe->status; + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed Req "); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id[%d] = 0x%llx with status 0x%x", + sw_sq_cons, cqe->wr_id, cqe->status); + cqe++; + (*budget)--; + bnxt_qplib_mark_qp_error(qp); + /* Add qp to flush list of the CQ */ + bnxt_qplib_add_flush_qp(qp); + } else { + /* Before we complete, do WA 9060 */ + if (do_wa9060(qp, cq, cq_cons, sw_sq_cons, + cqe_sq_cons)) { + *lib_qp = qp; + goto out; + } + if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) { + cqe->status = CQ_REQ_STATUS_OK; + cqe++; + (*budget)--; + } + } +skip: + sq->hwq.cons++; + if (sq->single) + break; + } +out: + *pcqe = cqe; + if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) { + /* Out of budget */ + rc = -EAGAIN; + goto done; + } + /* + * Back to normal completion mode only after it has completed all of + * the WC for this CQE + */ + sq->single = false; +done: + return rc; +} + +static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag) +{ + spin_lock(&srq->hwq.lock); + srq->swq[srq->last_idx].next_idx = (int)tag; + srq->last_idx = (int)tag; + srq->swq[srq->last_idx].next_idx = -1; + srq->hwq.cons++; /* Support for SRQE counter */ + spin_unlock(&srq->hwq.lock); +} + +static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, + struct cq_res_rc *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *rq; + struct bnxt_qplib_srq *srq; + struct bnxt_qplib_cqe *cqe; + u32 wr_id_idx; + int rc = 0; + + qp = (struct bnxt_qplib_qp *)((unsigned long) + le64_to_cpu(hwcqe->qp_handle)); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL"); + return -EINVAL; + } + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + + cqe = *pcqe; + cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; + cqe->length = le32_to_cpu(hwcqe->length); + cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); + cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); + cqe->flags = le16_to_cpu(hwcqe->flags); + cqe->status = hwcqe->status; + cqe->qp_handle = (u64)(unsigned long)qp; + + wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & + CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK; + if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) { + srq = qp->srq; + if (!srq) + return -EINVAL; + if (wr_id_idx >= srq->hwq.max_elements) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process RC "); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x", + wr_id_idx, srq->hwq.max_elements); + return -EINVAL; + } + cqe->wr_id = srq->swq[wr_id_idx].wr_id; + bnxt_qplib_release_srqe(srq, wr_id_idx); + cqe++; + (*budget)--; + *pcqe = cqe; + } else { + rq = &qp->rq; + if (wr_id_idx >= rq->hwq.max_elements) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process RC "); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x", + wr_id_idx, rq->hwq.max_elements); + return -EINVAL; + } + cqe->wr_id = rq->swq[wr_id_idx].wr_id; + cqe++; + (*budget)--; + rq->hwq.cons++; + *pcqe = cqe; + + if (hwcqe->status != CQ_RES_RC_STATUS_OK) { + qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; + /* Add qp to flush list of the CQ */ + bnxt_qplib_add_flush_qp(qp); + } + } + +done: + return rc; +} + +static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, + struct cq_res_ud *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *rq; + struct bnxt_qplib_srq *srq; + struct bnxt_qplib_cqe *cqe; + u32 wr_id_idx; + int rc = 0; + + qp = (struct bnxt_qplib_qp *)((unsigned long) + le64_to_cpu(hwcqe->qp_handle)); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL"); + return -EINVAL; + } + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + cqe = *pcqe; + cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; + cqe->length = le32_to_cpu(hwcqe->length); + cqe->invrkey = le32_to_cpu(hwcqe->imm_data); + cqe->flags = le16_to_cpu(hwcqe->flags); + cqe->status = hwcqe->status; + cqe->qp_handle = (u64)(unsigned long)qp; + memcpy(cqe->smac, hwcqe->src_mac, 6); + wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) + & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK; + cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | + ((le32_to_cpu( + hwcqe->src_qp_high_srq_or_rq_wr_id) & + CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8); + + if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) { + srq = qp->srq; + if (!srq) + return -EINVAL; + + if (wr_id_idx >= srq->hwq.max_elements) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process UD "); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x", + wr_id_idx, srq->hwq.max_elements); + return -EINVAL; + } + cqe->wr_id = srq->swq[wr_id_idx].wr_id; + bnxt_qplib_release_srqe(srq, wr_id_idx); + cqe++; + (*budget)--; + *pcqe = cqe; + } else { + rq = &qp->rq; + if (wr_id_idx >= rq->hwq.max_elements) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process UD "); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x", + wr_id_idx, rq->hwq.max_elements); + return -EINVAL; + } + + cqe->wr_id = rq->swq[wr_id_idx].wr_id; + cqe++; + (*budget)--; + rq->hwq.cons++; + *pcqe = cqe; + + if (hwcqe->status != CQ_RES_RC_STATUS_OK) { + qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; + /* Add qp to flush list of the CQ */ + bnxt_qplib_add_flush_qp(qp); + } + } +done: + return rc; +} + +bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq) +{ + struct cq_base *hw_cqe, **hw_cqe_ptr; + u32 sw_cons, raw_cons; + bool rc = true; + + raw_cons = cq->hwq.cons; + sw_cons = HWQ_CMP(raw_cons, &cq->hwq); + hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; + hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)]; + + /* Check for Valid bit. If the CQE is valid, return false */ + rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements); + return rc; +} + +static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, + struct cq_res_raweth_qp1 *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *rq; + struct bnxt_qplib_srq *srq; + struct bnxt_qplib_cqe *cqe; + u32 wr_id_idx; + int rc = 0; + + qp = (struct bnxt_qplib_qp *)((unsigned long) + le64_to_cpu(hwcqe->qp_handle)); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: process_cq Raw/QP1 qp is NULL"); + return -EINVAL; + } + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + cqe = *pcqe; + cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; + cqe->flags = le16_to_cpu(hwcqe->flags); + cqe->qp_handle = (u64)(unsigned long)qp; + + wr_id_idx = + le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) + & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK; + cqe->src_qp = qp->id; + if (qp->id == 1 && !cqe->length) { + /* Add workaround for the length misdetection */ + cqe->length = 296; + } else { + cqe->length = le16_to_cpu(hwcqe->length); + } + cqe->pkey_index = qp->pkey_index; + memcpy(cqe->smac, qp->smac, 6); + + cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); + cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); + cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata); + + if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) { + srq = qp->srq; + if (!srq) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: SRQ used but not defined??"); + return -EINVAL; + } + if (wr_id_idx >= srq->hwq.max_elements) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Raw/QP1 "); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x", + wr_id_idx, srq->hwq.max_elements); + return -EINVAL; + } + cqe->wr_id = srq->swq[wr_id_idx].wr_id; + bnxt_qplib_release_srqe(srq, wr_id_idx); + cqe++; + (*budget)--; + *pcqe = cqe; + } else { + rq = &qp->rq; + if (wr_id_idx >= rq->hwq.max_elements) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id "); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: ix 0x%x exceeded RQ max 0x%x", + wr_id_idx, rq->hwq.max_elements); + return -EINVAL; + } + cqe->wr_id = rq->swq[wr_id_idx].wr_id; + cqe++; + (*budget)--; + rq->hwq.cons++; + *pcqe = cqe; + + if (hwcqe->status != CQ_RES_RC_STATUS_OK) { + qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; + /* Add qp to flush list of the CQ */ + bnxt_qplib_add_flush_qp(qp); + } + } + +done: + return rc; +} + +static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, + struct cq_terminal *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *sq, *rq; + struct bnxt_qplib_cqe *cqe; + u32 sw_cons = 0, cqe_cons; + int rc = 0; + + /* Check the Status */ + if (hwcqe->status != CQ_TERMINAL_STATUS_OK) + dev_warn(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Terminal Error status = 0x%x", + hwcqe->status); + + qp = (struct bnxt_qplib_qp *)((unsigned long) + le64_to_cpu(hwcqe->qp_handle)); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process terminal qp is NULL"); + return -EINVAL; + } + + /* Must block new posting of SQ and RQ */ + qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; + + sq = &qp->sq; + rq = &qp->rq; + + cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); + if (cqe_cons == 0xFFFF) + goto do_rq; + + if (cqe_cons > sq->hwq.max_elements) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process terminal reported "); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x", + cqe_cons, sq->hwq.max_elements); + goto do_rq; + } + + if (qp->sq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto sq_done; + } + + /* Terminal CQE can also include aggregated successful CQEs prior. + * So we must complete all CQEs from the current sq's cons to the + * cq_cons with status OK + */ + cqe = *pcqe; + while (*budget) { + sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq); + if (sw_cons == cqe_cons) + break; + if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { + memset(cqe, 0, sizeof(*cqe)); + cqe->status = CQ_REQ_STATUS_OK; + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; + cqe->qp_handle = (u64)(unsigned long)qp; + cqe->src_qp = qp->id; + cqe->wr_id = sq->swq[sw_cons].wr_id; + cqe->type = sq->swq[sw_cons].type; + cqe++; + (*budget)--; + } + sq->hwq.cons++; + } + *pcqe = cqe; + if (!(*budget) && sw_cons != cqe_cons) { + /* Out of budget */ + rc = -EAGAIN; + goto sq_done; + } +sq_done: + if (rc) + return rc; +do_rq: + cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); + if (cqe_cons == 0xFFFF) { + goto done; + } else if (cqe_cons > rq->hwq.max_elements) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed terminal "); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x", + cqe_cons, rq->hwq.max_elements); + goto done; + } + + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + rc = 0; + goto done; + } + + /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR + * from the current rq->cons to the rq->prod regardless what the + * rq->cons the terminal CQE indicates + */ + + /* Add qp to flush list of the CQ */ + bnxt_qplib_add_flush_qp(qp); +done: + return rc; +} + +static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq, + struct cq_cutoff *hwcqe) +{ + /* Check the Status */ + if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Cutoff Error status = 0x%x", + hwcqe->status); + return -EINVAL; + } + clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); + wake_up_interruptible(&cq->waitq); + + return 0; +} + +int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, + struct bnxt_qplib_cqe *cqe, + int num_cqes) +{ + struct bnxt_qplib_qp *qp = NULL; + u32 budget = num_cqes; + unsigned long flags; + + spin_lock_irqsave(&cq->flush_lock, flags); + list_for_each_entry(qp, &cq->sqf_head, sq_flush) { + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: Flushing SQ QP= %p", + qp); + __flush_sq(&qp->sq, qp, &cqe, &budget); + } + + list_for_each_entry(qp, &cq->rqf_head, rq_flush) { + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: Flushing RQ QP= %p", + qp); + __flush_rq(&qp->rq, qp, &cqe, &budget); + } + spin_unlock_irqrestore(&cq->flush_lock, flags); + + return num_cqes - budget; +} + +int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, + int num_cqes, struct bnxt_qplib_qp **lib_qp) +{ + struct cq_base *hw_cqe, **hw_cqe_ptr; + u32 sw_cons, raw_cons; + int budget, rc = 0; + + raw_cons = cq->hwq.cons; + budget = num_cqes; + + while (budget) { + sw_cons = HWQ_CMP(raw_cons, &cq->hwq); + hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr; + hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)]; + + /* Check for Valid bit */ + if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements)) + break; + + /* + * The valid test of the entry must be done first before + * reading any further. + */ + dma_rmb(); + /* From the device's respective CQE format to qplib_wc*/ + switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { + case CQ_BASE_CQE_TYPE_REQ: + rc = bnxt_qplib_cq_process_req(cq, + (struct cq_req *)hw_cqe, + &cqe, &budget, + sw_cons, lib_qp); + break; + case CQ_BASE_CQE_TYPE_RES_RC: + rc = bnxt_qplib_cq_process_res_rc(cq, + (struct cq_res_rc *) + hw_cqe, &cqe, + &budget); + break; + case CQ_BASE_CQE_TYPE_RES_UD: + rc = bnxt_qplib_cq_process_res_ud + (cq, (struct cq_res_ud *)hw_cqe, &cqe, + &budget); + break; + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: + rc = bnxt_qplib_cq_process_res_raweth_qp1 + (cq, (struct cq_res_raweth_qp1 *) + hw_cqe, &cqe, &budget); + break; + case CQ_BASE_CQE_TYPE_TERMINAL: + rc = bnxt_qplib_cq_process_terminal + (cq, (struct cq_terminal *)hw_cqe, + &cqe, &budget); + break; + case CQ_BASE_CQE_TYPE_CUT_OFF: + bnxt_qplib_cq_process_cutoff + (cq, (struct cq_cutoff *)hw_cqe); + /* Done processing this CQ */ + goto exit; + default: + dev_err(&cq->hwq.pdev->dev, + "QPLIB: process_cq unknown type 0x%lx", + hw_cqe->cqe_type_toggle & + CQ_BASE_CQE_TYPE_MASK); + rc = -EINVAL; + break; + } + if (rc < 0) { + if (rc == -EAGAIN) + break; + /* Error while processing the CQE, just skip to the + * next one + */ + dev_err(&cq->hwq.pdev->dev, + "QPLIB: process_cqe error rc = 0x%x", rc); + } + raw_cons++; + } + if (cq->hwq.cons != raw_cons) { + cq->hwq.cons = raw_cons; + bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ); + } +exit: + return num_cqes - budget; +} + +void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type) +{ + if (arm_type) + bnxt_qplib_arm_cq(cq, arm_type); + /* Using cq->arm_state variable to track whether to issue cq handler */ + atomic_set(&cq->arm_state, 1); +} + +void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp) +{ + flush_workqueue(qp->scq->nq->cqn_wq); + if (qp->scq != qp->rcq) + flush_workqueue(qp->rcq->nq->cqn_wq); +} diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h new file mode 100644 index 000000000..72352ca80 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h @@ -0,0 +1,529 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Fast Path Operators (header) + */ + +#ifndef __BNXT_QPLIB_FP_H__ +#define __BNXT_QPLIB_FP_H__ + +struct bnxt_qplib_srq { + struct bnxt_qplib_pd *pd; + struct bnxt_qplib_dpi *dpi; + void __iomem *dbr_base; + u64 srq_handle; + u32 id; + u32 max_wqe; + u32 max_sge; + u32 threshold; + bool arm_req; + struct bnxt_qplib_cq *cq; + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_swq *swq; + struct scatterlist *sglist; + int start_idx; + int last_idx; + u32 nmap; + u16 eventq_hw_ring_id; + spinlock_t lock; /* protect SRQE link list */ +}; + +struct bnxt_qplib_sge { + u64 addr; + u32 lkey; + u32 size; +}; + +#define BNXT_QPLIB_MAX_SQE_ENTRY_SIZE sizeof(struct sq_send) + +#define SQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_SQE_ENTRY_SIZE) +#define SQE_MAX_IDX_PER_PG (SQE_CNT_PER_PG - 1) + +static inline u32 get_sqe_pg(u32 val) +{ + return ((val & ~SQE_MAX_IDX_PER_PG) / SQE_CNT_PER_PG); +} + +static inline u32 get_sqe_idx(u32 val) +{ + return (val & SQE_MAX_IDX_PER_PG); +} + +#define BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE sizeof(struct sq_psn_search) + +#define PSNE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE) +#define PSNE_MAX_IDX_PER_PG (PSNE_CNT_PER_PG - 1) + +static inline u32 get_psne_pg(u32 val) +{ + return ((val & ~PSNE_MAX_IDX_PER_PG) / PSNE_CNT_PER_PG); +} + +static inline u32 get_psne_idx(u32 val) +{ + return (val & PSNE_MAX_IDX_PER_PG); +} + +#define BNXT_QPLIB_QP_MAX_SGL 6 + +struct bnxt_qplib_swq { + u64 wr_id; + int next_idx; + u8 type; + u8 flags; + u32 start_psn; + u32 next_psn; + struct sq_psn_search *psn_search; +}; + +struct bnxt_qplib_swqe { + /* General */ +#define BNXT_QPLIB_FENCE_WRID 0x46454E43 /* "FENC" */ + u64 wr_id; + u8 reqs_type; + u8 type; +#define BNXT_QPLIB_SWQE_TYPE_SEND 0 +#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1 +#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2 +#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4 +#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5 +#define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6 +#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8 +#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11 +#define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12 +#define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13 +#define BNXT_QPLIB_SWQE_TYPE_REG_MR 13 +#define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14 +#define BNXT_QPLIB_SWQE_TYPE_RECV 128 +#define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129 + u8 flags; +#define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP BIT(0) +#define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE BIT(1) +#define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE BIT(2) +#define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT BIT(3) +#define BNXT_QPLIB_SWQE_FLAGS_INLINE BIT(4) + struct bnxt_qplib_sge sg_list[BNXT_QPLIB_QP_MAX_SGL]; + int num_sge; + /* Max inline data is 96 bytes */ + u32 inline_len; +#define BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH 96 + u8 inline_data[BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH]; + + union { + /* Send, with imm, inval key */ + struct { + union { + __be32 imm_data; + u32 inv_key; + }; + u32 q_key; + u32 dst_qp; + u16 avid; + } send; + + /* Send Raw Ethernet and QP1 */ + struct { + u16 lflags; + u16 cfa_action; + u32 cfa_meta; + } rawqp1; + + /* RDMA write, with imm, read */ + struct { + union { + __be32 imm_data; + u32 inv_key; + }; + u64 remote_va; + u32 r_key; + } rdma; + + /* Atomic cmp/swap, fetch/add */ + struct { + u64 remote_va; + u32 r_key; + u64 swap_data; + u64 cmp_data; + } atomic; + + /* Local Invalidate */ + struct { + u32 inv_l_key; + } local_inv; + + /* FR-PMR */ + struct { + u8 access_cntl; + u8 pg_sz_log; + bool zero_based; + u32 l_key; + u32 length; + u8 pbl_pg_sz_log; +#define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18 + u8 levels; +#define PAGE_SHIFT_4K 12 + __le64 *pbl_ptr; + dma_addr_t pbl_dma_ptr; + u64 *page_list; + u16 page_list_len; + u64 va; + } frmr; + + /* Bind */ + struct { + u8 access_cntl; +#define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE BIT(0) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ BIT(1) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE BIT(2) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC BIT(3) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND BIT(4) + bool zero_based; + u8 mw_type; + u32 parent_l_key; + u32 r_key; + u64 va; + u32 length; + } bind; + }; +}; + +#define BNXT_QPLIB_MAX_RQE_ENTRY_SIZE sizeof(struct rq_wqe) + +#define RQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_RQE_ENTRY_SIZE) +#define RQE_MAX_IDX_PER_PG (RQE_CNT_PER_PG - 1) +#define RQE_PG(x) (((x) & ~RQE_MAX_IDX_PER_PG) / RQE_CNT_PER_PG) +#define RQE_IDX(x) ((x) & RQE_MAX_IDX_PER_PG) + +struct bnxt_qplib_q { + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_swq *swq; + struct scatterlist *sglist; + u32 nmap; + u32 max_wqe; + u16 q_full_delta; + u16 max_sge; + u32 psn; + bool condition; + bool single; + bool send_phantom; + u32 phantom_wqe_cnt; + u32 phantom_cqe_cnt; + u32 next_cq_cons; + bool flushed; +}; + +struct bnxt_qplib_qp { + struct bnxt_qplib_pd *pd; + struct bnxt_qplib_dpi *dpi; + u64 qp_handle; +#define BNXT_QPLIB_QP_ID_INVALID 0xFFFFFFFF + u32 id; + u8 type; + u8 sig_type; + u32 modify_flags; + u8 state; + u8 cur_qp_state; + u32 max_inline_data; + u32 mtu; + u8 path_mtu; + bool en_sqd_async_notify; + u16 pkey_index; + u32 qkey; + u32 dest_qp_id; + u8 access; + u8 timeout; + u8 retry_cnt; + u8 rnr_retry; + u64 wqe_cnt; + u32 min_rnr_timer; + u32 max_rd_atomic; + u32 max_dest_rd_atomic; + u32 dest_qpn; + u8 smac[6]; + u16 vlan_id; + u8 nw_type; + struct bnxt_qplib_ah ah; + +#define BTH_PSN_MASK ((1 << 24) - 1) + /* SQ */ + struct bnxt_qplib_q sq; + /* RQ */ + struct bnxt_qplib_q rq; + /* SRQ */ + struct bnxt_qplib_srq *srq; + /* CQ */ + struct bnxt_qplib_cq *scq; + struct bnxt_qplib_cq *rcq; + /* IRRQ and ORRQ */ + struct bnxt_qplib_hwq irrq; + struct bnxt_qplib_hwq orrq; + /* Header buffer for QP1 */ + int sq_hdr_buf_size; + int rq_hdr_buf_size; +/* + * Buffer space for ETH(14), IP or GRH(40), UDP header(8) + * and ib_bth + ib_deth (20). + * Max required is 82 when RoCE V2 is enabled + */ +#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86 + /* Ethernet header = 14 */ + /* ib_grh = 40 (provided by MAD) */ + /* ib_bth + ib_deth = 20 */ + /* MAD = 256 (provided by MAD) */ + /* iCRC = 4 */ +#define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14 +#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512 +#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20 +#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40 +#define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20 + void *sq_hdr_buf; + dma_addr_t sq_hdr_buf_map; + void *rq_hdr_buf; + dma_addr_t rq_hdr_buf_map; + struct list_head sq_flush; + struct list_head rq_flush; +}; + +#define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base) + +#define CQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_CQE_ENTRY_SIZE) +#define CQE_MAX_IDX_PER_PG (CQE_CNT_PER_PG - 1) +#define CQE_PG(x) (((x) & ~CQE_MAX_IDX_PER_PG) / CQE_CNT_PER_PG) +#define CQE_IDX(x) ((x) & CQE_MAX_IDX_PER_PG) + +#define ROCE_CQE_CMP_V 0 +#define CQE_CMP_VALID(hdr, raw_cons, cp_bit) \ + (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \ + !((raw_cons) & (cp_bit))) + +static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *qplib_q) +{ + return HWQ_CMP((qplib_q->hwq.prod + qplib_q->q_full_delta), + &qplib_q->hwq) == HWQ_CMP(qplib_q->hwq.cons, + &qplib_q->hwq); +} + +struct bnxt_qplib_cqe { + u8 status; + u8 type; + u8 opcode; + u32 length; + u64 wr_id; + union { + __be32 immdata; + u32 invrkey; + }; + u64 qp_handle; + u64 mr_handle; + u16 flags; + u8 smac[6]; + u32 src_qp; + u16 raweth_qp1_flags; + u16 raweth_qp1_errors; + u16 raweth_qp1_cfa_code; + u32 raweth_qp1_flags2; + u32 raweth_qp1_metadata; + u8 raweth_qp1_payload_offset; + u16 pkey_index; +}; + +#define BNXT_QPLIB_QUEUE_START_PERIOD 0x01 +struct bnxt_qplib_cq { + struct bnxt_qplib_dpi *dpi; + void __iomem *dbr_base; + u32 max_wqe; + u32 id; + u16 count; + u16 period; + struct bnxt_qplib_hwq hwq; + u32 cnq_hw_ring_id; + struct bnxt_qplib_nq *nq; + bool resize_in_progress; + struct scatterlist *sghead; + u32 nmap; + u64 cq_handle; + +#define CQ_RESIZE_WAIT_TIME_MS 500 + unsigned long flags; +#define CQ_FLAGS_RESIZE_IN_PROG 1 + wait_queue_head_t waitq; + struct list_head sqf_head, rqf_head; + atomic_t arm_state; + spinlock_t compl_lock; /* synch CQ handlers */ +/* Locking Notes: + * QP can move to error state from modify_qp, async error event or error + * CQE as part of poll_cq. When QP is moved to error state, it gets added + * to two flush lists, one each for SQ and RQ. + * Each flush list is protected by qplib_cq->flush_lock. Both scq and rcq + * flush_locks should be acquired when QP is moved to error. The control path + * operations(modify_qp and async error events) are synchronized with poll_cq + * using upper level CQ locks (bnxt_re_cq->cq_lock) of both SCQ and RCQ. + * The qplib_cq->flush_lock is required to synchronize two instances of poll_cq + * of the same QP while manipulating the flush list. + */ + spinlock_t flush_lock; /* QP flush management */ +}; + +#define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq) +#define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq) +#define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * (x) + 2) +#define IRRQ_SLOTS_TO_IRD_LIMIT(s) (((s) >> 1) - 1) +#define ORD_LIMIT_TO_ORRQ_SLOTS(x) ((x) + 1) +#define ORRQ_SLOTS_TO_ORD_LIMIT(s) ((s) - 1) + +#define BNXT_QPLIB_MAX_NQE_ENTRY_SIZE sizeof(struct nq_base) + +#define NQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_NQE_ENTRY_SIZE) +#define NQE_MAX_IDX_PER_PG (NQE_CNT_PER_PG - 1) +#define NQE_PG(x) (((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG) +#define NQE_IDX(x) ((x) & NQE_MAX_IDX_PER_PG) + +#define NQE_CMP_VALID(hdr, raw_cons, cp_bit) \ + (!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) == \ + !((raw_cons) & (cp_bit))) + +#define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024) + +#define NQ_CONS_PCI_BAR_REGION 2 +#define NQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT) +#define NQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID +#define NQ_DB_IRQ_DIS CMPL_DOORBELL_MASK +#define NQ_DB_CP_FLAGS_REARM (NQ_DB_KEY_CP | \ + NQ_DB_IDX_VALID) +#define NQ_DB_CP_FLAGS (NQ_DB_KEY_CP | \ + NQ_DB_IDX_VALID | \ + NQ_DB_IRQ_DIS) +#define NQ_DB_REARM(db, raw_cons, cp_bit) \ + writel(NQ_DB_CP_FLAGS_REARM | ((raw_cons) & ((cp_bit) - 1)), db) +#define NQ_DB(db, raw_cons, cp_bit) \ + writel(NQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db) + +struct bnxt_qplib_nq { + struct pci_dev *pdev; + + int vector; + cpumask_t mask; + int budget; + bool requested; + struct tasklet_struct worker; + struct bnxt_qplib_hwq hwq; + + u16 bar_reg; + u16 bar_reg_off; + u16 ring_id; + void __iomem *bar_reg_iomem; + + int (*cqn_handler)(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_cq *cq); + int (*srqn_handler)(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_srq *srq, + u8 event); + struct workqueue_struct *cqn_wq; + char name[32]; +}; + +struct bnxt_qplib_nq_work { + struct work_struct work; + struct bnxt_qplib_nq *nq; + struct bnxt_qplib_cq *cq; +}; + +void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill); +void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq); +int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, + int msix_vector, bool need_init); +int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, + int nq_idx, int msix_vector, int bar_reg_offset, + int (*cqn_handler)(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_cq *cq), + int (*srqn_handler)(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_srq *srq, + u8 event)); +int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, + struct bnxt_qplib_swqe *wqe); +int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp); +void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp); +void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge); +void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge); +u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp); +dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, + u32 index); +void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp); +int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe); +void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp); +int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe); +int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); +int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); +int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, + int num, struct bnxt_qplib_qp **qp); +bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq); +void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type); +void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq); +int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq); +void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp); +void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp, + unsigned long *flags); +void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp, + unsigned long *flags); +int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, + struct bnxt_qplib_cqe *cqe, + int num_cqes); +void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp); +#endif /* __BNXT_QPLIB_FP_H__ */ diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c new file mode 100644 index 000000000..8b3b5fdc1 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c @@ -0,0 +1,775 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: RDMA Controller HW interface + */ +#include +#include +#include +#include +#include + +#include "roce_hsi.h" +#include "qplib_res.h" +#include "qplib_rcfw.h" +#include "qplib_sp.h" +#include "qplib_fp.h" + +static void bnxt_qplib_service_creq(unsigned long data); + +/* Hardware communication channel */ +static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) +{ + u16 cbit; + int rc; + + cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; + rc = wait_event_timeout(rcfw->waitq, + !test_bit(cbit, rcfw->cmdq_bitmap), + msecs_to_jiffies(RCFW_CMD_WAIT_TIME_MS)); + return rc ? 0 : -ETIMEDOUT; +}; + +static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) +{ + u32 count = RCFW_BLOCKED_CMD_WAIT_COUNT; + u16 cbit; + + cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; + if (!test_bit(cbit, rcfw->cmdq_bitmap)) + goto done; + do { + mdelay(1); /* 1m sec */ + bnxt_qplib_service_creq((unsigned long)rcfw); + } while (test_bit(cbit, rcfw->cmdq_bitmap) && --count); +done: + return count ? 0 : -ETIMEDOUT; +}; + +static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req, + struct creq_base *resp, void *sb, u8 is_block) +{ + struct bnxt_qplib_cmdqe *cmdqe, **cmdq_ptr; + struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq; + struct bnxt_qplib_crsq *crsqe; + u32 sw_prod, cmdq_prod; + unsigned long flags; + u32 size, opcode; + u16 cookie, cbit; + u8 *preq; + + opcode = req->opcode; + if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) && + (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC && + opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW && + opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) { + dev_err(&rcfw->pdev->dev, + "QPLIB: RCFW not initialized, reject opcode 0x%x", + opcode); + return -EINVAL; + } + + if (test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) && + opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) { + dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!"); + return -EINVAL; + } + + if (test_bit(FIRMWARE_TIMED_OUT, &rcfw->flags)) + return -ETIMEDOUT; + + /* Cmdq are in 16-byte units, each request can consume 1 or more + * cmdqe + */ + spin_lock_irqsave(&cmdq->lock, flags); + if (req->cmd_size >= HWQ_FREE_SLOTS(cmdq)) { + dev_err(&rcfw->pdev->dev, "QPLIB: RCFW: CMDQ is full!"); + spin_unlock_irqrestore(&cmdq->lock, flags); + return -EAGAIN; + } + + + cookie = rcfw->seq_num & RCFW_MAX_COOKIE_VALUE; + cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; + if (is_block) + cookie |= RCFW_CMD_IS_BLOCKING; + + set_bit(cbit, rcfw->cmdq_bitmap); + req->cookie = cpu_to_le16(cookie); + crsqe = &rcfw->crsqe_tbl[cbit]; + if (crsqe->resp) { + spin_unlock_irqrestore(&cmdq->lock, flags); + return -EBUSY; + } + memset(resp, 0, sizeof(*resp)); + crsqe->resp = (struct creq_qp_event *)resp; + crsqe->resp->cookie = req->cookie; + crsqe->req_size = req->cmd_size; + if (req->resp_size && sb) { + struct bnxt_qplib_rcfw_sbuf *sbuf = sb; + + req->resp_addr = cpu_to_le64(sbuf->dma_addr); + req->resp_size = (sbuf->size + BNXT_QPLIB_CMDQE_UNITS - 1) / + BNXT_QPLIB_CMDQE_UNITS; + } + + cmdq_ptr = (struct bnxt_qplib_cmdqe **)cmdq->pbl_ptr; + preq = (u8 *)req; + size = req->cmd_size * BNXT_QPLIB_CMDQE_UNITS; + do { + /* Locate the next cmdq slot */ + sw_prod = HWQ_CMP(cmdq->prod, cmdq); + cmdqe = &cmdq_ptr[get_cmdq_pg(sw_prod)][get_cmdq_idx(sw_prod)]; + if (!cmdqe) { + dev_err(&rcfw->pdev->dev, + "QPLIB: RCFW request failed with no cmdqe!"); + goto done; + } + /* Copy a segment of the req cmd to the cmdq */ + memset(cmdqe, 0, sizeof(*cmdqe)); + memcpy(cmdqe, preq, min_t(u32, size, sizeof(*cmdqe))); + preq += min_t(u32, size, sizeof(*cmdqe)); + size -= min_t(u32, size, sizeof(*cmdqe)); + cmdq->prod++; + rcfw->seq_num++; + } while (size > 0); + + rcfw->seq_num++; + + cmdq_prod = cmdq->prod; + if (test_bit(FIRMWARE_FIRST_FLAG, &rcfw->flags)) { + /* The very first doorbell write + * is required to set this flag + * which prompts the FW to reset + * its internal pointers + */ + cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG); + clear_bit(FIRMWARE_FIRST_FLAG, &rcfw->flags); + } + + /* ring CMDQ DB */ + wmb(); + writel(cmdq_prod, rcfw->cmdq_bar_reg_iomem + + rcfw->cmdq_bar_reg_prod_off); + writel(RCFW_CMDQ_TRIG_VAL, rcfw->cmdq_bar_reg_iomem + + rcfw->cmdq_bar_reg_trig_off); +done: + spin_unlock_irqrestore(&cmdq->lock, flags); + /* Return the CREQ response pointer */ + return 0; +} + +int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, + struct cmdq_base *req, + struct creq_base *resp, + void *sb, u8 is_block) +{ + struct creq_qp_event *evnt = (struct creq_qp_event *)resp; + u16 cookie; + u8 opcode, retry_cnt = 0xFF; + int rc = 0; + + do { + opcode = req->opcode; + rc = __send_message(rcfw, req, resp, sb, is_block); + cookie = le16_to_cpu(req->cookie) & RCFW_MAX_COOKIE_VALUE; + if (!rc) + break; + + if (!retry_cnt || (rc != -EAGAIN && rc != -EBUSY)) { + /* send failed */ + dev_err(&rcfw->pdev->dev, "QPLIB: cmdq[%#x]=%#x send failed", + cookie, opcode); + return rc; + } + is_block ? mdelay(1) : usleep_range(500, 1000); + + } while (retry_cnt--); + + if (is_block) + rc = __block_for_resp(rcfw, cookie); + else + rc = __wait_for_resp(rcfw, cookie); + if (rc) { + /* timed out */ + dev_err(&rcfw->pdev->dev, "QPLIB: cmdq[%#x]=%#x timedout (%d)msec", + cookie, opcode, RCFW_CMD_WAIT_TIME_MS); + set_bit(FIRMWARE_TIMED_OUT, &rcfw->flags); + return rc; + } + + if (evnt->status) { + /* failed with status */ + dev_err(&rcfw->pdev->dev, "QPLIB: cmdq[%#x]=%#x status %#x", + cookie, opcode, evnt->status); + rc = -EFAULT; + } + + return rc; +} +/* Completions */ +static int bnxt_qplib_process_func_event(struct bnxt_qplib_rcfw *rcfw, + struct creq_func_event *func_event) +{ + switch (func_event->event) { + case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: + /* SRQ ctx error, call srq_handler?? + * But there's no SRQ handle! + */ + break; + case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: + break; + case CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST: + break; + case CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED: + break; + default: + return -EINVAL; + } + return 0; +} + +static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw, + struct creq_qp_event *qp_event) +{ + struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq; + struct creq_qp_error_notification *err_event; + struct bnxt_qplib_crsq *crsqe; + unsigned long flags; + struct bnxt_qplib_qp *qp; + u16 cbit, blocked = 0; + u16 cookie; + __le16 mcookie; + u32 qp_id; + + switch (qp_event->event) { + case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: + err_event = (struct creq_qp_error_notification *)qp_event; + qp_id = le32_to_cpu(err_event->xid); + qp = rcfw->qp_tbl[qp_id].qp_handle; + dev_dbg(&rcfw->pdev->dev, + "QPLIB: Received QP error notification"); + dev_dbg(&rcfw->pdev->dev, + "QPLIB: qpid 0x%x, req_err=0x%x, resp_err=0x%x\n", + qp_id, err_event->req_err_state_reason, + err_event->res_err_state_reason); + if (!qp) + break; + bnxt_qplib_mark_qp_error(qp); + rcfw->aeq_handler(rcfw, qp_event, qp); + break; + default: + /* + * Command Response + * cmdq->lock needs to be acquired to synchronie + * the command send and completion reaping. This function + * is always called with creq->lock held. Using + * the nested variant of spin_lock. + * + */ + + spin_lock_irqsave_nested(&cmdq->lock, flags, + SINGLE_DEPTH_NESTING); + cookie = le16_to_cpu(qp_event->cookie); + mcookie = qp_event->cookie; + blocked = cookie & RCFW_CMD_IS_BLOCKING; + cookie &= RCFW_MAX_COOKIE_VALUE; + cbit = cookie % RCFW_MAX_OUTSTANDING_CMD; + crsqe = &rcfw->crsqe_tbl[cbit]; + if (crsqe->resp && + crsqe->resp->cookie == mcookie) { + memcpy(crsqe->resp, qp_event, sizeof(*qp_event)); + crsqe->resp = NULL; + } else { + dev_err(&rcfw->pdev->dev, + "QPLIB: CMD %s resp->cookie = %#x, evnt->cookie = %#x", + crsqe->resp ? "mismatch" : "collision", + crsqe->resp ? crsqe->resp->cookie : 0, mcookie); + } + if (!test_and_clear_bit(cbit, rcfw->cmdq_bitmap)) + dev_warn(&rcfw->pdev->dev, + "QPLIB: CMD bit %d was not requested", cbit); + cmdq->cons += crsqe->req_size; + crsqe->req_size = 0; + + if (!blocked) + wake_up(&rcfw->waitq); + spin_unlock_irqrestore(&cmdq->lock, flags); + } + return 0; +} + +/* SP - CREQ Completion handlers */ +static void bnxt_qplib_service_creq(unsigned long data) +{ + struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data; + struct bnxt_qplib_hwq *creq = &rcfw->creq; + struct creq_base *creqe, **creq_ptr; + u32 sw_cons, raw_cons; + unsigned long flags; + u32 type, budget = CREQ_ENTRY_POLL_BUDGET; + + /* Service the CREQ until budget is over */ + spin_lock_irqsave(&creq->lock, flags); + raw_cons = creq->cons; + while (budget > 0) { + sw_cons = HWQ_CMP(raw_cons, creq); + creq_ptr = (struct creq_base **)creq->pbl_ptr; + creqe = &creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]; + if (!CREQ_CMP_VALID(creqe, raw_cons, creq->max_elements)) + break; + /* The valid test of the entry must be done first before + * reading any further. + */ + dma_rmb(); + + type = creqe->type & CREQ_BASE_TYPE_MASK; + switch (type) { + case CREQ_BASE_TYPE_QP_EVENT: + bnxt_qplib_process_qp_event + (rcfw, (struct creq_qp_event *)creqe); + rcfw->creq_qp_event_processed++; + break; + case CREQ_BASE_TYPE_FUNC_EVENT: + if (!bnxt_qplib_process_func_event + (rcfw, (struct creq_func_event *)creqe)) + rcfw->creq_func_event_processed++; + else + dev_warn + (&rcfw->pdev->dev, "QPLIB:aeqe:%#x Not handled", + type); + break; + default: + dev_warn(&rcfw->pdev->dev, "QPLIB: creqe with "); + dev_warn(&rcfw->pdev->dev, + "QPLIB: op_event = 0x%x not handled", type); + break; + } + raw_cons++; + budget--; + } + + if (creq->cons != raw_cons) { + creq->cons = raw_cons; + CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, raw_cons, + creq->max_elements); + } + spin_unlock_irqrestore(&creq->lock, flags); +} + +static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance) +{ + struct bnxt_qplib_rcfw *rcfw = dev_instance; + struct bnxt_qplib_hwq *creq = &rcfw->creq; + struct creq_base **creq_ptr; + u32 sw_cons; + + /* Prefetch the CREQ element */ + sw_cons = HWQ_CMP(creq->cons, creq); + creq_ptr = (struct creq_base **)rcfw->creq.pbl_ptr; + prefetch(&creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]); + + tasklet_schedule(&rcfw->worker); + + return IRQ_HANDLED; +} + +/* RCFW */ +int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw) +{ + struct cmdq_deinitialize_fw req; + struct creq_deinitialize_fw_resp resp; + u16 cmd_flags = 0; + int rc; + + RCFW_CMD_PREP(req, DEINITIALIZE_FW, cmd_flags); + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, + NULL, 0); + if (rc) + return rc; + + clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags); + return 0; +} + +static int __get_pbl_pg_idx(struct bnxt_qplib_pbl *pbl) +{ + return (pbl->pg_size == ROCE_PG_SIZE_4K ? + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K : + pbl->pg_size == ROCE_PG_SIZE_8K ? + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K : + pbl->pg_size == ROCE_PG_SIZE_64K ? + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K : + pbl->pg_size == ROCE_PG_SIZE_2M ? + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M : + pbl->pg_size == ROCE_PG_SIZE_8M ? + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M : + pbl->pg_size == ROCE_PG_SIZE_1G ? + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G : + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K); +} + +int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_ctx *ctx, int is_virtfn) +{ + struct cmdq_initialize_fw req; + struct creq_initialize_fw_resp resp; + u16 cmd_flags = 0, level; + int rc; + + RCFW_CMD_PREP(req, INITIALIZE_FW, cmd_flags); + /* Supply (log-base-2-of-host-page-size - base-page-shift) + * to bono to adjust the doorbell page sizes. + */ + req.log2_dbr_pg_size = cpu_to_le16(PAGE_SHIFT - + RCFW_DBR_BASE_PAGE_SHIFT); + /* + * VFs need not setup the HW context area, PF + * shall setup this area for VF. Skipping the + * HW programming + */ + if (is_virtfn) + goto skip_ctx_setup; + + level = ctx->qpc_tbl.level; + req.qpc_pg_size_qpc_lvl = (level << CMDQ_INITIALIZE_FW_QPC_LVL_SFT) | + __get_pbl_pg_idx(&ctx->qpc_tbl.pbl[level]); + level = ctx->mrw_tbl.level; + req.mrw_pg_size_mrw_lvl = (level << CMDQ_INITIALIZE_FW_MRW_LVL_SFT) | + __get_pbl_pg_idx(&ctx->mrw_tbl.pbl[level]); + level = ctx->srqc_tbl.level; + req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) | + __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]); + level = ctx->cq_tbl.level; + req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) | + __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]); + level = ctx->srqc_tbl.level; + req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) | + __get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]); + level = ctx->cq_tbl.level; + req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) | + __get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]); + level = ctx->tim_tbl.level; + req.tim_pg_size_tim_lvl = (level << CMDQ_INITIALIZE_FW_TIM_LVL_SFT) | + __get_pbl_pg_idx(&ctx->tim_tbl.pbl[level]); + level = ctx->tqm_pde_level; + req.tqm_pg_size_tqm_lvl = (level << CMDQ_INITIALIZE_FW_TQM_LVL_SFT) | + __get_pbl_pg_idx(&ctx->tqm_pde.pbl[level]); + + req.qpc_page_dir = + cpu_to_le64(ctx->qpc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); + req.mrw_page_dir = + cpu_to_le64(ctx->mrw_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); + req.srq_page_dir = + cpu_to_le64(ctx->srqc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); + req.cq_page_dir = + cpu_to_le64(ctx->cq_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); + req.tim_page_dir = + cpu_to_le64(ctx->tim_tbl.pbl[PBL_LVL_0].pg_map_arr[0]); + req.tqm_page_dir = + cpu_to_le64(ctx->tqm_pde.pbl[PBL_LVL_0].pg_map_arr[0]); + + req.number_of_qp = cpu_to_le32(ctx->qpc_tbl.max_elements); + req.number_of_mrw = cpu_to_le32(ctx->mrw_tbl.max_elements); + req.number_of_srq = cpu_to_le32(ctx->srqc_tbl.max_elements); + req.number_of_cq = cpu_to_le32(ctx->cq_tbl.max_elements); + + req.max_qp_per_vf = cpu_to_le32(ctx->vf_res.max_qp_per_vf); + req.max_mrw_per_vf = cpu_to_le32(ctx->vf_res.max_mrw_per_vf); + req.max_srq_per_vf = cpu_to_le32(ctx->vf_res.max_srq_per_vf); + req.max_cq_per_vf = cpu_to_le32(ctx->vf_res.max_cq_per_vf); + req.max_gid_per_vf = cpu_to_le32(ctx->vf_res.max_gid_per_vf); + +skip_ctx_setup: + req.stat_ctx_id = cpu_to_le32(ctx->stats.fw_id); + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, + NULL, 0); + if (rc) + return rc; + set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags); + return 0; +} + +void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) +{ + kfree(rcfw->qp_tbl); + kfree(rcfw->crsqe_tbl); + bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->cmdq); + bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->creq); + rcfw->pdev = NULL; +} + +int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev, + struct bnxt_qplib_rcfw *rcfw, + int qp_tbl_sz) +{ + rcfw->pdev = pdev; + rcfw->creq.max_elements = BNXT_QPLIB_CREQE_MAX_CNT; + if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->creq, NULL, 0, + &rcfw->creq.max_elements, + BNXT_QPLIB_CREQE_UNITS, 0, PAGE_SIZE, + HWQ_TYPE_L2_CMPL)) { + dev_err(&rcfw->pdev->dev, + "QPLIB: HW channel CREQ allocation failed"); + goto fail; + } + rcfw->cmdq.max_elements = BNXT_QPLIB_CMDQE_MAX_CNT; + if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->cmdq, NULL, 0, + &rcfw->cmdq.max_elements, + BNXT_QPLIB_CMDQE_UNITS, 0, PAGE_SIZE, + HWQ_TYPE_CTX)) { + dev_err(&rcfw->pdev->dev, + "QPLIB: HW channel CMDQ allocation failed"); + goto fail; + } + + rcfw->crsqe_tbl = kcalloc(rcfw->cmdq.max_elements, + sizeof(*rcfw->crsqe_tbl), GFP_KERNEL); + if (!rcfw->crsqe_tbl) + goto fail; + + rcfw->qp_tbl_size = qp_tbl_sz; + rcfw->qp_tbl = kcalloc(qp_tbl_sz, sizeof(struct bnxt_qplib_qp_node), + GFP_KERNEL); + if (!rcfw->qp_tbl) + goto fail; + + return 0; + +fail: + bnxt_qplib_free_rcfw_channel(rcfw); + return -ENOMEM; +} + +void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill) +{ + tasklet_disable(&rcfw->worker); + /* Mask h/w interrupts */ + CREQ_DB(rcfw->creq_bar_reg_iomem, rcfw->creq.cons, + rcfw->creq.max_elements); + /* Sync with last running IRQ-handler */ + synchronize_irq(rcfw->vector); + if (kill) + tasklet_kill(&rcfw->worker); + + if (rcfw->requested) { + free_irq(rcfw->vector, rcfw); + rcfw->requested = false; + } +} + +void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) +{ + unsigned long indx; + + bnxt_qplib_rcfw_stop_irq(rcfw, true); + + iounmap(rcfw->cmdq_bar_reg_iomem); + iounmap(rcfw->creq_bar_reg_iomem); + + indx = find_first_bit(rcfw->cmdq_bitmap, rcfw->bmap_size); + if (indx != rcfw->bmap_size) + dev_err(&rcfw->pdev->dev, + "QPLIB: disabling RCFW with pending cmd-bit %lx", indx); + kfree(rcfw->cmdq_bitmap); + rcfw->bmap_size = 0; + + rcfw->cmdq_bar_reg_iomem = NULL; + rcfw->creq_bar_reg_iomem = NULL; + rcfw->aeq_handler = NULL; + rcfw->vector = 0; +} + +int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector, + bool need_init) +{ + int rc; + + if (rcfw->requested) + return -EFAULT; + + rcfw->vector = msix_vector; + if (need_init) + tasklet_init(&rcfw->worker, + bnxt_qplib_service_creq, (unsigned long)rcfw); + else + tasklet_enable(&rcfw->worker); + rc = request_irq(rcfw->vector, bnxt_qplib_creq_irq, 0, + "bnxt_qplib_creq", rcfw); + if (rc) + return rc; + rcfw->requested = true; + CREQ_DB_REARM(rcfw->creq_bar_reg_iomem, rcfw->creq.cons, + rcfw->creq.max_elements); + + return 0; +} + +int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev, + struct bnxt_qplib_rcfw *rcfw, + int msix_vector, + int cp_bar_reg_off, int virt_fn, + int (*aeq_handler)(struct bnxt_qplib_rcfw *, + void *, void *)) +{ + resource_size_t res_base; + struct cmdq_init init; + u16 bmap_size; + int rc; + + /* General */ + rcfw->seq_num = 0; + set_bit(FIRMWARE_FIRST_FLAG, &rcfw->flags); + bmap_size = BITS_TO_LONGS(RCFW_MAX_OUTSTANDING_CMD * + sizeof(unsigned long)); + rcfw->cmdq_bitmap = kzalloc(bmap_size, GFP_KERNEL); + if (!rcfw->cmdq_bitmap) + return -ENOMEM; + rcfw->bmap_size = bmap_size; + + /* CMDQ */ + rcfw->cmdq_bar_reg = RCFW_COMM_PCI_BAR_REGION; + res_base = pci_resource_start(pdev, rcfw->cmdq_bar_reg); + if (!res_base) + return -ENOMEM; + + rcfw->cmdq_bar_reg_iomem = ioremap_nocache(res_base + + RCFW_COMM_BASE_OFFSET, + RCFW_COMM_SIZE); + if (!rcfw->cmdq_bar_reg_iomem) { + dev_err(&rcfw->pdev->dev, + "QPLIB: CMDQ BAR region %d mapping failed", + rcfw->cmdq_bar_reg); + return -ENOMEM; + } + + rcfw->cmdq_bar_reg_prod_off = virt_fn ? RCFW_VF_COMM_PROD_OFFSET : + RCFW_PF_COMM_PROD_OFFSET; + + rcfw->cmdq_bar_reg_trig_off = RCFW_COMM_TRIG_OFFSET; + + /* CREQ */ + rcfw->creq_bar_reg = RCFW_COMM_CONS_PCI_BAR_REGION; + res_base = pci_resource_start(pdev, rcfw->creq_bar_reg); + if (!res_base) + dev_err(&rcfw->pdev->dev, + "QPLIB: CREQ BAR region %d resc start is 0!", + rcfw->creq_bar_reg); + rcfw->creq_bar_reg_iomem = ioremap_nocache(res_base + cp_bar_reg_off, + 4); + if (!rcfw->creq_bar_reg_iomem) { + dev_err(&rcfw->pdev->dev, + "QPLIB: CREQ BAR region %d mapping failed", + rcfw->creq_bar_reg); + iounmap(rcfw->cmdq_bar_reg_iomem); + rcfw->cmdq_bar_reg_iomem = NULL; + return -ENOMEM; + } + rcfw->creq_qp_event_processed = 0; + rcfw->creq_func_event_processed = 0; + + if (aeq_handler) + rcfw->aeq_handler = aeq_handler; + init_waitqueue_head(&rcfw->waitq); + + rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true); + if (rc) { + dev_err(&rcfw->pdev->dev, + "QPLIB: Failed to request IRQ for CREQ rc = 0x%x", rc); + bnxt_qplib_disable_rcfw_channel(rcfw); + return rc; + } + + init.cmdq_pbl = cpu_to_le64(rcfw->cmdq.pbl[PBL_LVL_0].pg_map_arr[0]); + init.cmdq_size_cmdq_lvl = cpu_to_le16( + ((BNXT_QPLIB_CMDQE_MAX_CNT << CMDQ_INIT_CMDQ_SIZE_SFT) & + CMDQ_INIT_CMDQ_SIZE_MASK) | + ((rcfw->cmdq.level << CMDQ_INIT_CMDQ_LVL_SFT) & + CMDQ_INIT_CMDQ_LVL_MASK)); + init.creq_ring_id = cpu_to_le16(rcfw->creq_ring_id); + + /* Write to the Bono mailbox register */ + __iowrite32_copy(rcfw->cmdq_bar_reg_iomem, &init, sizeof(init) / 4); + return 0; +} + +struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf( + struct bnxt_qplib_rcfw *rcfw, + u32 size) +{ + struct bnxt_qplib_rcfw_sbuf *sbuf; + + sbuf = kzalloc(sizeof(*sbuf), GFP_ATOMIC); + if (!sbuf) + return NULL; + + sbuf->size = size; + sbuf->sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf->size, + &sbuf->dma_addr, GFP_ATOMIC); + if (!sbuf->sb) + goto bail; + + return sbuf; +bail: + kfree(sbuf); + return NULL; +} + +void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_rcfw_sbuf *sbuf) +{ + if (sbuf->sb) + dma_free_coherent(&rcfw->pdev->dev, sbuf->size, + sbuf->sb, sbuf->dma_addr); + kfree(sbuf); +} diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h new file mode 100644 index 000000000..46416dfe8 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h @@ -0,0 +1,222 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: RDMA Controller HW interface (header) + */ + +#ifndef __BNXT_QPLIB_RCFW_H__ +#define __BNXT_QPLIB_RCFW_H__ + +#define RCFW_CMDQ_TRIG_VAL 1 +#define RCFW_COMM_PCI_BAR_REGION 0 +#define RCFW_COMM_CONS_PCI_BAR_REGION 2 +#define RCFW_COMM_BASE_OFFSET 0x600 +#define RCFW_PF_COMM_PROD_OFFSET 0xc +#define RCFW_VF_COMM_PROD_OFFSET 0xc +#define RCFW_COMM_TRIG_OFFSET 0x100 +#define RCFW_COMM_SIZE 0x104 + +#define RCFW_DBR_PCI_BAR_REGION 2 +#define RCFW_DBR_BASE_PAGE_SHIFT 12 + +#define RCFW_CMD_PREP(req, CMD, cmd_flags) \ + do { \ + memset(&(req), 0, sizeof((req))); \ + (req).opcode = CMDQ_BASE_OPCODE_##CMD; \ + (req).cmd_size = (sizeof((req)) + \ + BNXT_QPLIB_CMDQE_UNITS - 1) / \ + BNXT_QPLIB_CMDQE_UNITS; \ + (req).flags = cpu_to_le16(cmd_flags); \ + } while (0) + +#define RCFW_CMD_WAIT_TIME_MS 20000 /* 20 Seconds timeout */ + +/* CMDQ elements */ +#define BNXT_QPLIB_CMDQE_MAX_CNT 256 +#define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe) +#define BNXT_QPLIB_CMDQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CMDQE_UNITS) + +#define MAX_CMDQ_IDX (BNXT_QPLIB_CMDQE_MAX_CNT - 1) +#define MAX_CMDQ_IDX_PER_PG (BNXT_QPLIB_CMDQE_CNT_PER_PG - 1) + +#define RCFW_MAX_OUTSTANDING_CMD BNXT_QPLIB_CMDQE_MAX_CNT +#define RCFW_MAX_COOKIE_VALUE 0x7FFF +#define RCFW_CMD_IS_BLOCKING 0x8000 +#define RCFW_BLOCKED_CMD_WAIT_COUNT 0x4E20 + +/* Cmdq contains a fix number of a 16-Byte slots */ +struct bnxt_qplib_cmdqe { + u8 data[16]; +}; + +static inline u32 get_cmdq_pg(u32 val) +{ + return (val & ~MAX_CMDQ_IDX_PER_PG) / BNXT_QPLIB_CMDQE_CNT_PER_PG; +} + +static inline u32 get_cmdq_idx(u32 val) +{ + return val & MAX_CMDQ_IDX_PER_PG; +} + +/* Crsq buf is 1024-Byte */ +struct bnxt_qplib_crsbe { + u8 data[1024]; +}; + +/* CREQ */ +/* Allocate 1 per QP for async error notification for now */ +#define BNXT_QPLIB_CREQE_MAX_CNT (64 * 1024) +#define BNXT_QPLIB_CREQE_UNITS 16 /* 16-Bytes per prod unit */ +#define BNXT_QPLIB_CREQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_CREQE_UNITS) + +#define MAX_CREQ_IDX (BNXT_QPLIB_CREQE_MAX_CNT - 1) +#define MAX_CREQ_IDX_PER_PG (BNXT_QPLIB_CREQE_CNT_PER_PG - 1) + +static inline u32 get_creq_pg(u32 val) +{ + return (val & ~MAX_CREQ_IDX_PER_PG) / BNXT_QPLIB_CREQE_CNT_PER_PG; +} + +static inline u32 get_creq_idx(u32 val) +{ + return val & MAX_CREQ_IDX_PER_PG; +} + +#define BNXT_QPLIB_CREQE_PER_PG (PAGE_SIZE / sizeof(struct creq_base)) + +#define CREQ_CMP_VALID(hdr, raw_cons, cp_bit) \ + (!!((hdr)->v & CREQ_BASE_V) == \ + !((raw_cons) & (cp_bit))) + +#define CREQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT) +#define CREQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID +#define CREQ_DB_IRQ_DIS CMPL_DOORBELL_MASK +#define CREQ_DB_CP_FLAGS_REARM (CREQ_DB_KEY_CP | \ + CREQ_DB_IDX_VALID) +#define CREQ_DB_CP_FLAGS (CREQ_DB_KEY_CP | \ + CREQ_DB_IDX_VALID | \ + CREQ_DB_IRQ_DIS) +#define CREQ_DB_REARM(db, raw_cons, cp_bit) \ + writel(CREQ_DB_CP_FLAGS_REARM | ((raw_cons) & ((cp_bit) - 1)), db) +#define CREQ_DB(db, raw_cons, cp_bit) \ + writel(CREQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db) + +#define CREQ_ENTRY_POLL_BUDGET 0x100 + +/* HWQ */ + +struct bnxt_qplib_crsq { + struct creq_qp_event *resp; + u32 req_size; +}; + +struct bnxt_qplib_rcfw_sbuf { + void *sb; + dma_addr_t dma_addr; + u32 size; +}; + +struct bnxt_qplib_qp_node { + u32 qp_id; /* QP id */ + void *qp_handle; /* ptr to qplib_qp */ +}; + +/* RCFW Communication Channels */ +struct bnxt_qplib_rcfw { + struct pci_dev *pdev; + int vector; + struct tasklet_struct worker; + bool requested; + unsigned long *cmdq_bitmap; + u32 bmap_size; + unsigned long flags; +#define FIRMWARE_INITIALIZED_FLAG 0 +#define FIRMWARE_FIRST_FLAG 31 +#define FIRMWARE_TIMED_OUT 3 + wait_queue_head_t waitq; + int (*aeq_handler)(struct bnxt_qplib_rcfw *, + void *, void *); + u32 seq_num; + + /* Bar region info */ + void __iomem *cmdq_bar_reg_iomem; + u16 cmdq_bar_reg; + u16 cmdq_bar_reg_prod_off; + u16 cmdq_bar_reg_trig_off; + u16 creq_ring_id; + u16 creq_bar_reg; + void __iomem *creq_bar_reg_iomem; + + /* Cmd-Resp and Async Event notification queue */ + struct bnxt_qplib_hwq creq; + u64 creq_qp_event_processed; + u64 creq_func_event_processed; + + /* Actual Cmd and Resp Queues */ + struct bnxt_qplib_hwq cmdq; + struct bnxt_qplib_crsq *crsqe_tbl; + int qp_tbl_size; + struct bnxt_qplib_qp_node *qp_tbl; +}; + +void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw); +int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev, + struct bnxt_qplib_rcfw *rcfw, int qp_tbl_sz); +void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill); +void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw); +int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector, + bool need_init); +int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev, + struct bnxt_qplib_rcfw *rcfw, + int msix_vector, + int cp_bar_reg_off, int virt_fn, + int (*aeq_handler)(struct bnxt_qplib_rcfw *, + void *aeqe, void *obj)); + +struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf( + struct bnxt_qplib_rcfw *rcfw, + u32 size); +void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_rcfw_sbuf *sbuf); +int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, + struct cmdq_base *req, struct creq_base *resp, + void *sbuf, u8 is_block); + +int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw); +int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_ctx *ctx, int is_virtfn); +void bnxt_qplib_mark_qp_error(void *qp_handle); +#endif /* __BNXT_QPLIB_RCFW_H__ */ diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c new file mode 100644 index 000000000..655952a6c --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -0,0 +1,831 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: QPLib resource manager + */ + +#include +#include +#include +#include +#include +#include +#include "roce_hsi.h" +#include "qplib_res.h" +#include "qplib_sp.h" +#include "qplib_rcfw.h" + +static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev, + struct bnxt_qplib_stats *stats); +static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev, + struct bnxt_qplib_stats *stats); + +/* PBL */ +static void __free_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl, + bool is_umem) +{ + int i; + + if (!is_umem) { + for (i = 0; i < pbl->pg_count; i++) { + if (pbl->pg_arr[i]) + dma_free_coherent(&pdev->dev, pbl->pg_size, + (void *)((unsigned long) + pbl->pg_arr[i] & + PAGE_MASK), + pbl->pg_map_arr[i]); + else + dev_warn(&pdev->dev, + "QPLIB: PBL free pg_arr[%d] empty?!", + i); + pbl->pg_arr[i] = NULL; + } + } + kfree(pbl->pg_arr); + pbl->pg_arr = NULL; + kfree(pbl->pg_map_arr); + pbl->pg_map_arr = NULL; + pbl->pg_count = 0; + pbl->pg_size = 0; +} + +static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl, + struct scatterlist *sghead, u32 pages, u32 pg_size) +{ + struct scatterlist *sg; + bool is_umem = false; + int i; + + /* page ptr arrays */ + pbl->pg_arr = kcalloc(pages, sizeof(void *), GFP_KERNEL); + if (!pbl->pg_arr) + return -ENOMEM; + + pbl->pg_map_arr = kcalloc(pages, sizeof(dma_addr_t), GFP_KERNEL); + if (!pbl->pg_map_arr) { + kfree(pbl->pg_arr); + pbl->pg_arr = NULL; + return -ENOMEM; + } + pbl->pg_count = 0; + pbl->pg_size = pg_size; + + if (!sghead) { + for (i = 0; i < pages; i++) { + pbl->pg_arr[i] = dma_zalloc_coherent(&pdev->dev, + pbl->pg_size, + &pbl->pg_map_arr[i], + GFP_KERNEL); + if (!pbl->pg_arr[i]) + goto fail; + pbl->pg_count++; + } + } else { + i = 0; + is_umem = true; + for_each_sg(sghead, sg, pages, i) { + pbl->pg_map_arr[i] = sg_dma_address(sg); + pbl->pg_arr[i] = sg_virt(sg); + if (!pbl->pg_arr[i]) + goto fail; + + pbl->pg_count++; + } + } + + return 0; + +fail: + __free_pbl(pdev, pbl, is_umem); + return -ENOMEM; +} + +/* HWQ */ +void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq) +{ + int i; + + if (!hwq->max_elements) + return; + if (hwq->level >= PBL_LVL_MAX) + return; + + for (i = 0; i < hwq->level + 1; i++) { + if (i == hwq->level) + __free_pbl(pdev, &hwq->pbl[i], hwq->is_user); + else + __free_pbl(pdev, &hwq->pbl[i], false); + } + + hwq->level = PBL_LVL_MAX; + hwq->max_elements = 0; + hwq->element_size = 0; + hwq->prod = 0; + hwq->cons = 0; + hwq->cp_bit = 0; +} + +/* All HWQs are power of 2 in size */ +int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq, + struct scatterlist *sghead, int nmap, + u32 *elements, u32 element_size, u32 aux, + u32 pg_size, enum bnxt_qplib_hwq_type hwq_type) +{ + u32 pages, slots, size, aux_pages = 0, aux_size = 0; + dma_addr_t *src_phys_ptr, **dst_virt_ptr; + int i, rc; + + hwq->level = PBL_LVL_MAX; + + slots = roundup_pow_of_two(*elements); + if (aux) { + aux_size = roundup_pow_of_two(aux); + aux_pages = (slots * aux_size) / pg_size; + if ((slots * aux_size) % pg_size) + aux_pages++; + } + size = roundup_pow_of_two(element_size); + + if (!sghead) { + hwq->is_user = false; + pages = (slots * size) / pg_size + aux_pages; + if ((slots * size) % pg_size) + pages++; + if (!pages) + return -EINVAL; + } else { + hwq->is_user = true; + pages = nmap; + } + + /* Alloc the 1st memory block; can be a PDL/PTL/PBL */ + if (sghead && (pages == MAX_PBL_LVL_0_PGS)) + rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], sghead, + pages, pg_size); + else + rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], NULL, 1, pg_size); + if (rc) + goto fail; + + hwq->level = PBL_LVL_0; + + if (pages > MAX_PBL_LVL_0_PGS) { + if (pages > MAX_PBL_LVL_1_PGS) { + /* 2 levels of indirection */ + rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], NULL, + MAX_PBL_LVL_1_PGS_FOR_LVL_2, pg_size); + if (rc) + goto fail; + /* Fill in lvl0 PBL */ + dst_virt_ptr = + (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; + src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; + for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = + src_phys_ptr[i] | PTU_PDE_VALID; + hwq->level = PBL_LVL_1; + + rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_2], sghead, + pages, pg_size); + if (rc) + goto fail; + + /* Fill in lvl1 PBL */ + dst_virt_ptr = + (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr; + src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr; + for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) { + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = + src_phys_ptr[i] | PTU_PTE_VALID; + } + if (hwq_type == HWQ_TYPE_QUEUE) { + /* Find the last pg of the size */ + i = hwq->pbl[PBL_LVL_2].pg_count; + dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= + PTU_PTE_LAST; + if (i > 1) + dst_virt_ptr[PTR_PG(i - 2)] + [PTR_IDX(i - 2)] |= + PTU_PTE_NEXT_TO_LAST; + } + hwq->level = PBL_LVL_2; + } else { + u32 flag = hwq_type == HWQ_TYPE_L2_CMPL ? 0 : + PTU_PTE_VALID; + + /* 1 level of indirection */ + rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], sghead, + pages, pg_size); + if (rc) + goto fail; + /* Fill in lvl0 PBL */ + dst_virt_ptr = + (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; + src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; + for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) { + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = + src_phys_ptr[i] | flag; + } + if (hwq_type == HWQ_TYPE_QUEUE) { + /* Find the last pg of the size */ + i = hwq->pbl[PBL_LVL_1].pg_count; + dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= + PTU_PTE_LAST; + if (i > 1) + dst_virt_ptr[PTR_PG(i - 2)] + [PTR_IDX(i - 2)] |= + PTU_PTE_NEXT_TO_LAST; + } + hwq->level = PBL_LVL_1; + } + } + hwq->pdev = pdev; + spin_lock_init(&hwq->lock); + hwq->prod = 0; + hwq->cons = 0; + *elements = hwq->max_elements = slots; + hwq->element_size = size; + + /* For direct access to the elements */ + hwq->pbl_ptr = hwq->pbl[hwq->level].pg_arr; + hwq->pbl_dma_ptr = hwq->pbl[hwq->level].pg_map_arr; + + return 0; + +fail: + bnxt_qplib_free_hwq(pdev, hwq); + return -ENOMEM; +} + +/* Context Tables */ +void bnxt_qplib_free_ctx(struct pci_dev *pdev, + struct bnxt_qplib_ctx *ctx) +{ + int i; + + bnxt_qplib_free_hwq(pdev, &ctx->qpc_tbl); + bnxt_qplib_free_hwq(pdev, &ctx->mrw_tbl); + bnxt_qplib_free_hwq(pdev, &ctx->srqc_tbl); + bnxt_qplib_free_hwq(pdev, &ctx->cq_tbl); + bnxt_qplib_free_hwq(pdev, &ctx->tim_tbl); + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) + bnxt_qplib_free_hwq(pdev, &ctx->tqm_tbl[i]); + bnxt_qplib_free_hwq(pdev, &ctx->tqm_pde); + bnxt_qplib_free_stats_ctx(pdev, &ctx->stats); +} + +/* + * Routine: bnxt_qplib_alloc_ctx + * Description: + * Context tables are memories which are used by the chip fw. + * The 6 tables defined are: + * QPC ctx - holds QP states + * MRW ctx - holds memory region and window + * SRQ ctx - holds shared RQ states + * CQ ctx - holds completion queue states + * TQM ctx - holds Tx Queue Manager context + * TIM ctx - holds timer context + * Depending on the size of the tbl requested, either a 1 Page Buffer List + * or a 1-to-2-stage indirection Page Directory List + 1 PBL is used + * instead. + * Table might be employed as follows: + * For 0 < ctx size <= 1 PAGE, 0 level of ind is used + * For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used + * For 512 < ctx size <= MAX, 2 levels of ind is used + * Returns: + * 0 if success, else -ERRORS + */ +int bnxt_qplib_alloc_ctx(struct pci_dev *pdev, + struct bnxt_qplib_ctx *ctx, + bool virt_fn) +{ + int i, j, k, rc = 0; + int fnz_idx = -1; + __le64 **pbl_ptr; + + if (virt_fn) + goto stats_alloc; + + /* QPC Tables */ + ctx->qpc_tbl.max_elements = ctx->qpc_count; + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->qpc_tbl, NULL, 0, + &ctx->qpc_tbl.max_elements, + BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE, 0, + PAGE_SIZE, HWQ_TYPE_CTX); + if (rc) + goto fail; + + /* MRW Tables */ + ctx->mrw_tbl.max_elements = ctx->mrw_count; + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->mrw_tbl, NULL, 0, + &ctx->mrw_tbl.max_elements, + BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE, 0, + PAGE_SIZE, HWQ_TYPE_CTX); + if (rc) + goto fail; + + /* SRQ Tables */ + ctx->srqc_tbl.max_elements = ctx->srqc_count; + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->srqc_tbl, NULL, 0, + &ctx->srqc_tbl.max_elements, + BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE, 0, + PAGE_SIZE, HWQ_TYPE_CTX); + if (rc) + goto fail; + + /* CQ Tables */ + ctx->cq_tbl.max_elements = ctx->cq_count; + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->cq_tbl, NULL, 0, + &ctx->cq_tbl.max_elements, + BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE, 0, + PAGE_SIZE, HWQ_TYPE_CTX); + if (rc) + goto fail; + + /* TQM Buffer */ + ctx->tqm_pde.max_elements = 512; + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_pde, NULL, 0, + &ctx->tqm_pde.max_elements, sizeof(u64), + 0, PAGE_SIZE, HWQ_TYPE_CTX); + if (rc) + goto fail; + + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) { + if (!ctx->tqm_count[i]) + continue; + ctx->tqm_tbl[i].max_elements = ctx->qpc_count * + ctx->tqm_count[i]; + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_tbl[i], NULL, 0, + &ctx->tqm_tbl[i].max_elements, 1, + 0, PAGE_SIZE, HWQ_TYPE_CTX); + if (rc) + goto fail; + } + pbl_ptr = (__le64 **)ctx->tqm_pde.pbl_ptr; + for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ; + i++, j += MAX_TQM_ALLOC_BLK_SIZE) { + if (!ctx->tqm_tbl[i].max_elements) + continue; + if (fnz_idx == -1) + fnz_idx = i; + switch (ctx->tqm_tbl[i].level) { + case PBL_LVL_2: + for (k = 0; k < ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_count; + k++) + pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)] = + cpu_to_le64( + ctx->tqm_tbl[i].pbl[PBL_LVL_1].pg_map_arr[k] + | PTU_PTE_VALID); + break; + case PBL_LVL_1: + case PBL_LVL_0: + default: + pbl_ptr[PTR_PG(j)][PTR_IDX(j)] = cpu_to_le64( + ctx->tqm_tbl[i].pbl[PBL_LVL_0].pg_map_arr[0] | + PTU_PTE_VALID); + break; + } + } + if (fnz_idx == -1) + fnz_idx = 0; + ctx->tqm_pde_level = ctx->tqm_tbl[fnz_idx].level == PBL_LVL_2 ? + PBL_LVL_2 : ctx->tqm_tbl[fnz_idx].level + 1; + + /* TIM Buffer */ + ctx->tim_tbl.max_elements = ctx->qpc_count * 16; + rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tim_tbl, NULL, 0, + &ctx->tim_tbl.max_elements, 1, + 0, PAGE_SIZE, HWQ_TYPE_CTX); + if (rc) + goto fail; + +stats_alloc: + /* Stats */ + rc = bnxt_qplib_alloc_stats_ctx(pdev, &ctx->stats); + if (rc) + goto fail; + + return 0; + +fail: + bnxt_qplib_free_ctx(pdev, ctx); + return rc; +} + +/* GUID */ +void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid) +{ + u8 mac[ETH_ALEN]; + + /* MAC-48 to EUI-64 mapping */ + memcpy(mac, dev_addr, ETH_ALEN); + guid[0] = mac[0] ^ 2; + guid[1] = mac[1]; + guid[2] = mac[2]; + guid[3] = 0xff; + guid[4] = 0xfe; + guid[5] = mac[3]; + guid[6] = mac[4]; + guid[7] = mac[5]; +} + +static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_sgid_tbl *sgid_tbl) +{ + kfree(sgid_tbl->tbl); + kfree(sgid_tbl->hw_id); + kfree(sgid_tbl->ctx); + kfree(sgid_tbl->vlan); + sgid_tbl->tbl = NULL; + sgid_tbl->hw_id = NULL; + sgid_tbl->ctx = NULL; + sgid_tbl->vlan = NULL; + sgid_tbl->max = 0; + sgid_tbl->active = 0; +} + +static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_sgid_tbl *sgid_tbl, + u16 max) +{ + sgid_tbl->tbl = kcalloc(max, sizeof(struct bnxt_qplib_gid), GFP_KERNEL); + if (!sgid_tbl->tbl) + return -ENOMEM; + + sgid_tbl->hw_id = kcalloc(max, sizeof(u16), GFP_KERNEL); + if (!sgid_tbl->hw_id) + goto out_free1; + + sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL); + if (!sgid_tbl->ctx) + goto out_free2; + + sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL); + if (!sgid_tbl->vlan) + goto out_free3; + + sgid_tbl->max = max; + return 0; +out_free3: + kfree(sgid_tbl->ctx); + sgid_tbl->ctx = NULL; +out_free2: + kfree(sgid_tbl->hw_id); + sgid_tbl->hw_id = NULL; +out_free1: + kfree(sgid_tbl->tbl); + sgid_tbl->tbl = NULL; + return -ENOMEM; +}; + +static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_sgid_tbl *sgid_tbl) +{ + int i; + + for (i = 0; i < sgid_tbl->max; i++) { + if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero))) + bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i], true); + } + memset(sgid_tbl->tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl->max); + memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); + memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max); + sgid_tbl->active = 0; +} + +static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct net_device *netdev) +{ + memset(sgid_tbl->tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl->max); + memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); +} + +static void bnxt_qplib_free_pkey_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_pkey_tbl *pkey_tbl) +{ + if (!pkey_tbl->tbl) + dev_dbg(&res->pdev->dev, "QPLIB: PKEY tbl not present"); + else + kfree(pkey_tbl->tbl); + + pkey_tbl->tbl = NULL; + pkey_tbl->max = 0; + pkey_tbl->active = 0; +} + +static int bnxt_qplib_alloc_pkey_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_pkey_tbl *pkey_tbl, + u16 max) +{ + pkey_tbl->tbl = kcalloc(max, sizeof(u16), GFP_KERNEL); + if (!pkey_tbl->tbl) + return -ENOMEM; + + pkey_tbl->max = max; + return 0; +}; + +/* PDs */ +int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd) +{ + u32 bit_num; + + bit_num = find_first_bit(pdt->tbl, pdt->max); + if (bit_num == pdt->max) + return -ENOMEM; + + /* Found unused PD */ + clear_bit(bit_num, pdt->tbl); + pd->id = bit_num; + return 0; +} + +int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, + struct bnxt_qplib_pd_tbl *pdt, + struct bnxt_qplib_pd *pd) +{ + if (test_and_set_bit(pd->id, pdt->tbl)) { + dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d", + pd->id); + return -EINVAL; + } + pd->id = 0; + return 0; +} + +static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt) +{ + kfree(pdt->tbl); + pdt->tbl = NULL; + pdt->max = 0; +} + +static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_pd_tbl *pdt, + u32 max) +{ + u32 bytes; + + bytes = max >> 3; + if (!bytes) + bytes = 1; + pdt->tbl = kmalloc(bytes, GFP_KERNEL); + if (!pdt->tbl) + return -ENOMEM; + + pdt->max = max; + memset((u8 *)pdt->tbl, 0xFF, bytes); + + return 0; +} + +/* DPIs */ +int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit, + struct bnxt_qplib_dpi *dpi, + void *app) +{ + u32 bit_num; + + bit_num = find_first_bit(dpit->tbl, dpit->max); + if (bit_num == dpit->max) + return -ENOMEM; + + /* Found unused DPI */ + clear_bit(bit_num, dpit->tbl); + dpit->app_tbl[bit_num] = app; + + dpi->dpi = bit_num; + dpi->dbr = dpit->dbr_bar_reg_iomem + (bit_num * PAGE_SIZE); + dpi->umdbr = dpit->unmapped_dbr + (bit_num * PAGE_SIZE); + + return 0; +} + +int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi_tbl *dpit, + struct bnxt_qplib_dpi *dpi) +{ + if (dpi->dpi >= dpit->max) { + dev_warn(&res->pdev->dev, "Invalid DPI? dpi = %d", dpi->dpi); + return -EINVAL; + } + if (test_and_set_bit(dpi->dpi, dpit->tbl)) { + dev_warn(&res->pdev->dev, "Freeing an unused DPI? dpi = %d", + dpi->dpi); + return -EINVAL; + } + if (dpit->app_tbl) + dpit->app_tbl[dpi->dpi] = NULL; + memset(dpi, 0, sizeof(*dpi)); + + return 0; +} + +static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi_tbl *dpit) +{ + kfree(dpit->tbl); + kfree(dpit->app_tbl); + if (dpit->dbr_bar_reg_iomem) + pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); + memset(dpit, 0, sizeof(*dpit)); +} + +static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi_tbl *dpit, + u32 dbr_offset) +{ + u32 dbr_bar_reg = RCFW_DBR_PCI_BAR_REGION; + resource_size_t bar_reg_base; + u32 dbr_len, bytes; + + if (dpit->dbr_bar_reg_iomem) { + dev_err(&res->pdev->dev, + "QPLIB: DBR BAR region %d already mapped", dbr_bar_reg); + return -EALREADY; + } + + bar_reg_base = pci_resource_start(res->pdev, dbr_bar_reg); + if (!bar_reg_base) { + dev_err(&res->pdev->dev, + "QPLIB: BAR region %d resc start failed", dbr_bar_reg); + return -ENOMEM; + } + + dbr_len = pci_resource_len(res->pdev, dbr_bar_reg) - dbr_offset; + if (!dbr_len || ((dbr_len & (PAGE_SIZE - 1)) != 0)) { + dev_err(&res->pdev->dev, "QPLIB: Invalid DBR length %d", + dbr_len); + return -ENOMEM; + } + + dpit->dbr_bar_reg_iomem = ioremap_nocache(bar_reg_base + dbr_offset, + dbr_len); + if (!dpit->dbr_bar_reg_iomem) { + dev_err(&res->pdev->dev, + "QPLIB: FP: DBR BAR region %d mapping failed", + dbr_bar_reg); + return -ENOMEM; + } + + dpit->unmapped_dbr = bar_reg_base + dbr_offset; + dpit->max = dbr_len / PAGE_SIZE; + + dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL); + if (!dpit->app_tbl) + goto unmap_io; + + bytes = dpit->max >> 3; + if (!bytes) + bytes = 1; + + dpit->tbl = kmalloc(bytes, GFP_KERNEL); + if (!dpit->tbl) { + kfree(dpit->app_tbl); + dpit->app_tbl = NULL; + goto unmap_io; + } + + memset((u8 *)dpit->tbl, 0xFF, bytes); + + return 0; + +unmap_io: + pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem); + dpit->dbr_bar_reg_iomem = NULL; + return -ENOMEM; +} + +/* PKEYs */ +static void bnxt_qplib_cleanup_pkey_tbl(struct bnxt_qplib_pkey_tbl *pkey_tbl) +{ + memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max); + pkey_tbl->active = 0; +} + +static void bnxt_qplib_init_pkey_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_pkey_tbl *pkey_tbl) +{ + u16 pkey = 0xFFFF; + + memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max); + + /* pkey default = 0xFFFF */ + bnxt_qplib_add_pkey(res, pkey_tbl, &pkey, false); +} + +/* Stats */ +static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev, + struct bnxt_qplib_stats *stats) +{ + if (stats->dma) { + dma_free_coherent(&pdev->dev, stats->size, + stats->dma, stats->dma_map); + } + memset(stats, 0, sizeof(*stats)); + stats->fw_id = -1; +} + +static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev, + struct bnxt_qplib_stats *stats) +{ + memset(stats, 0, sizeof(*stats)); + stats->fw_id = -1; + stats->size = sizeof(struct ctx_hw_stats); + stats->dma = dma_alloc_coherent(&pdev->dev, stats->size, + &stats->dma_map, GFP_KERNEL); + if (!stats->dma) { + dev_err(&pdev->dev, "QPLIB: Stats DMA allocation failed"); + return -ENOMEM; + } + return 0; +} + +void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res) +{ + bnxt_qplib_cleanup_pkey_tbl(&res->pkey_tbl); + bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl); +} + +int bnxt_qplib_init_res(struct bnxt_qplib_res *res) +{ + bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev); + bnxt_qplib_init_pkey_tbl(res, &res->pkey_tbl); + + return 0; +} + +void bnxt_qplib_free_res(struct bnxt_qplib_res *res) +{ + bnxt_qplib_free_pkey_tbl(res, &res->pkey_tbl); + bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl); + bnxt_qplib_free_pd_tbl(&res->pd_tbl); + bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl); + + res->netdev = NULL; + res->pdev = NULL; +} + +int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev, + struct net_device *netdev, + struct bnxt_qplib_dev_attr *dev_attr) +{ + int rc = 0; + + res->pdev = pdev; + res->netdev = netdev; + + rc = bnxt_qplib_alloc_sgid_tbl(res, &res->sgid_tbl, dev_attr->max_sgid); + if (rc) + goto fail; + + rc = bnxt_qplib_alloc_pkey_tbl(res, &res->pkey_tbl, dev_attr->max_pkey); + if (rc) + goto fail; + + rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd); + if (rc) + goto fail; + + rc = bnxt_qplib_alloc_dpi_tbl(res, &res->dpi_tbl, dev_attr->l2_db_size); + if (rc) + goto fail; + + return 0; +fail: + bnxt_qplib_free_res(res); + return rc; +} diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h new file mode 100644 index 000000000..2e5c052da --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h @@ -0,0 +1,229 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: QPLib resource manager (header) + */ + +#ifndef __BNXT_QPLIB_RES_H__ +#define __BNXT_QPLIB_RES_H__ + +extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero; + +#define PTR_CNT_PER_PG (PAGE_SIZE / sizeof(void *)) +#define PTR_MAX_IDX_PER_PG (PTR_CNT_PER_PG - 1) +#define PTR_PG(x) (((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG) +#define PTR_IDX(x) ((x) & PTR_MAX_IDX_PER_PG) + +#define HWQ_CMP(idx, hwq) ((idx) & ((hwq)->max_elements - 1)) + +#define HWQ_FREE_SLOTS(hwq) (hwq->max_elements - \ + ((HWQ_CMP(hwq->prod, hwq)\ + - HWQ_CMP(hwq->cons, hwq))\ + & (hwq->max_elements - 1))) +enum bnxt_qplib_hwq_type { + HWQ_TYPE_CTX, + HWQ_TYPE_QUEUE, + HWQ_TYPE_L2_CMPL +}; + +#define MAX_PBL_LVL_0_PGS 1 +#define MAX_PBL_LVL_1_PGS 512 +#define MAX_PBL_LVL_1_PGS_SHIFT 9 +#define MAX_PBL_LVL_1_PGS_FOR_LVL_2 256 +#define MAX_PBL_LVL_2_PGS (256 * 512) + +enum bnxt_qplib_pbl_lvl { + PBL_LVL_0, + PBL_LVL_1, + PBL_LVL_2, + PBL_LVL_MAX +}; + +#define ROCE_PG_SIZE_4K (4 * 1024) +#define ROCE_PG_SIZE_8K (8 * 1024) +#define ROCE_PG_SIZE_64K (64 * 1024) +#define ROCE_PG_SIZE_2M (2 * 1024 * 1024) +#define ROCE_PG_SIZE_8M (8 * 1024 * 1024) +#define ROCE_PG_SIZE_1G (1024 * 1024 * 1024) + +struct bnxt_qplib_pbl { + u32 pg_count; + u32 pg_size; + void **pg_arr; + dma_addr_t *pg_map_arr; +}; + +struct bnxt_qplib_hwq { + struct pci_dev *pdev; + /* lock to protect qplib_hwq */ + spinlock_t lock; + struct bnxt_qplib_pbl pbl[PBL_LVL_MAX]; + enum bnxt_qplib_pbl_lvl level; /* 0, 1, or 2 */ + /* ptr for easy access to the PBL entries */ + void **pbl_ptr; + /* ptr for easy access to the dma_addr */ + dma_addr_t *pbl_dma_ptr; + u32 max_elements; + u16 element_size; /* Size of each entry */ + + u32 prod; /* raw */ + u32 cons; /* raw */ + u8 cp_bit; + u8 is_user; +}; + +/* Tables */ +struct bnxt_qplib_pd_tbl { + unsigned long *tbl; + u32 max; +}; + +struct bnxt_qplib_sgid_tbl { + struct bnxt_qplib_gid *tbl; + u16 *hw_id; + u16 max; + u16 active; + void *ctx; + u8 *vlan; +}; + +struct bnxt_qplib_pkey_tbl { + u16 *tbl; + u16 max; + u16 active; +}; + +struct bnxt_qplib_dpi { + u32 dpi; + void __iomem *dbr; + u64 umdbr; +}; + +struct bnxt_qplib_dpi_tbl { + void **app_tbl; + unsigned long *tbl; + u16 max; + void __iomem *dbr_bar_reg_iomem; + u64 unmapped_dbr; +}; + +struct bnxt_qplib_stats { + dma_addr_t dma_map; + void *dma; + u32 size; + u32 fw_id; +}; + +struct bnxt_qplib_vf_res { + u32 max_qp_per_vf; + u32 max_mrw_per_vf; + u32 max_srq_per_vf; + u32 max_cq_per_vf; + u32 max_gid_per_vf; +}; + +#define BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE 448 +#define BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE 64 +#define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE 64 +#define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE 128 + +struct bnxt_qplib_ctx { + u32 qpc_count; + struct bnxt_qplib_hwq qpc_tbl; + u32 mrw_count; + struct bnxt_qplib_hwq mrw_tbl; + u32 srqc_count; + struct bnxt_qplib_hwq srqc_tbl; + u32 cq_count; + struct bnxt_qplib_hwq cq_tbl; + struct bnxt_qplib_hwq tim_tbl; +#define MAX_TQM_ALLOC_REQ 48 +#define MAX_TQM_ALLOC_BLK_SIZE 8 + u8 tqm_count[MAX_TQM_ALLOC_REQ]; + struct bnxt_qplib_hwq tqm_pde; + u32 tqm_pde_level; + struct bnxt_qplib_hwq tqm_tbl[MAX_TQM_ALLOC_REQ]; + struct bnxt_qplib_stats stats; + struct bnxt_qplib_vf_res vf_res; +}; + +struct bnxt_qplib_res { + struct pci_dev *pdev; + struct net_device *netdev; + + struct bnxt_qplib_rcfw *rcfw; + + struct bnxt_qplib_pd_tbl pd_tbl; + struct bnxt_qplib_sgid_tbl sgid_tbl; + struct bnxt_qplib_pkey_tbl pkey_tbl; + struct bnxt_qplib_dpi_tbl dpi_tbl; + bool prio; +}; + +#define to_bnxt_qplib(ptr, type, member) \ + container_of(ptr, type, member) + +struct bnxt_qplib_pd; +struct bnxt_qplib_dev_attr; + +void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq); +int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq, + struct scatterlist *sl, int nmap, u32 *elements, + u32 elements_per_page, u32 aux, u32 pg_size, + enum bnxt_qplib_hwq_type hwq_type); +void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid); +int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pd_tbl, + struct bnxt_qplib_pd *pd); +int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, + struct bnxt_qplib_pd_tbl *pd_tbl, + struct bnxt_qplib_pd *pd); +int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit, + struct bnxt_qplib_dpi *dpi, + void *app); +int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi_tbl *dpi_tbl, + struct bnxt_qplib_dpi *dpi); +void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res); +int bnxt_qplib_init_res(struct bnxt_qplib_res *res); +void bnxt_qplib_free_res(struct bnxt_qplib_res *res); +int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev, + struct net_device *netdev, + struct bnxt_qplib_dev_attr *dev_attr); +void bnxt_qplib_free_ctx(struct pci_dev *pdev, + struct bnxt_qplib_ctx *ctx); +int bnxt_qplib_alloc_ctx(struct pci_dev *pdev, + struct bnxt_qplib_ctx *ctx, + bool virt_fn); +#endif /* __BNXT_QPLIB_RES_H__ */ diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c new file mode 100644 index 000000000..336144876 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c @@ -0,0 +1,850 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Slow Path Operators + */ + +#include +#include +#include +#include + +#include "roce_hsi.h" + +#include "qplib_res.h" +#include "qplib_rcfw.h" +#include "qplib_sp.h" + +const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 } }; + +/* Device */ + +static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw, + char *fw_ver) +{ + struct cmdq_query_version req; + struct creq_query_version_resp resp; + u16 cmd_flags = 0; + int rc = 0; + + RCFW_CMD_PREP(req, QUERY_VERSION, cmd_flags); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) + return; + fw_ver[0] = resp.fw_maj; + fw_ver[1] = resp.fw_minor; + fw_ver[2] = resp.fw_bld; + fw_ver[3] = resp.fw_rsvd; +} + +int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_dev_attr *attr, bool vf) +{ + struct cmdq_query_func req; + struct creq_query_func_resp resp; + struct bnxt_qplib_rcfw_sbuf *sbuf; + struct creq_query_func_resp_sb *sb; + u16 cmd_flags = 0; + u32 temp; + u8 *tqm_alloc; + int i, rc = 0; + + RCFW_CMD_PREP(req, QUERY_FUNC, cmd_flags); + + sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); + if (!sbuf) { + dev_err(&rcfw->pdev->dev, + "QPLIB: SP: QUERY_FUNC alloc side buffer failed"); + return -ENOMEM; + } + + sb = sbuf->sb; + req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, + (void *)sbuf, 0); + if (rc) + goto bail; + + /* Extract the context from the side buffer */ + attr->max_qp = le32_to_cpu(sb->max_qp); + /* max_qp value reported by FW for PF doesn't include the QP1 for PF */ + if (!vf) + attr->max_qp += 1; + attr->max_qp_rd_atom = + sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? + BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom; + attr->max_qp_init_rd_atom = + sb->max_qp_init_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? + BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_init_rd_atom; + attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr); + /* + * 128 WQEs needs to be reserved for the HW (8916). Prevent + * reporting the max number + */ + attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS; + attr->max_qp_sges = sb->max_sge; + attr->max_cq = le32_to_cpu(sb->max_cq); + attr->max_cq_wqes = le32_to_cpu(sb->max_cqe); + attr->max_cq_sges = attr->max_qp_sges; + attr->max_mr = le32_to_cpu(sb->max_mr); + attr->max_mw = le32_to_cpu(sb->max_mw); + + attr->max_mr_size = le64_to_cpu(sb->max_mr_size); + attr->max_pd = 64 * 1024; + attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp); + attr->max_ah = le32_to_cpu(sb->max_ah); + + attr->max_fmr = le32_to_cpu(sb->max_fmr); + attr->max_map_per_fmr = sb->max_map_per_fmr; + + attr->max_srq = le16_to_cpu(sb->max_srq); + attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1; + attr->max_srq_sges = sb->max_srq_sge; + /* Bono only reports 1 PKEY for now, but it can support > 1 */ + attr->max_pkey = le32_to_cpu(sb->max_pkeys); + + attr->max_inline_data = le32_to_cpu(sb->max_inline_data); + attr->l2_db_size = (sb->l2_db_space_size + 1) * + (0x01 << RCFW_DBR_BASE_PAGE_SHIFT); + attr->max_sgid = BNXT_QPLIB_NUM_GIDS_SUPPORTED; + + bnxt_qplib_query_version(rcfw, attr->fw_ver); + + for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) { + temp = le32_to_cpu(sb->tqm_alloc_reqs[i]); + tqm_alloc = (u8 *)&temp; + attr->tqm_alloc_reqs[i * 4] = *tqm_alloc; + attr->tqm_alloc_reqs[i * 4 + 1] = *(++tqm_alloc); + attr->tqm_alloc_reqs[i * 4 + 2] = *(++tqm_alloc); + attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc); + } + + attr->is_atomic = false; +bail: + bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); + return rc; +} + +int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res, + struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_ctx *ctx) +{ + struct cmdq_set_func_resources req; + struct creq_set_func_resources_resp resp; + u16 cmd_flags = 0; + int rc = 0; + + RCFW_CMD_PREP(req, SET_FUNC_RESOURCES, cmd_flags); + + req.number_of_qp = cpu_to_le32(ctx->qpc_count); + req.number_of_mrw = cpu_to_le32(ctx->mrw_count); + req.number_of_srq = cpu_to_le32(ctx->srqc_count); + req.number_of_cq = cpu_to_le32(ctx->cq_count); + + req.max_qp_per_vf = cpu_to_le32(ctx->vf_res.max_qp_per_vf); + req.max_mrw_per_vf = cpu_to_le32(ctx->vf_res.max_mrw_per_vf); + req.max_srq_per_vf = cpu_to_le32(ctx->vf_res.max_srq_per_vf); + req.max_cq_per_vf = cpu_to_le32(ctx->vf_res.max_cq_per_vf); + req.max_gid_per_vf = cpu_to_le32(ctx->vf_res.max_gid_per_vf); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, + NULL, 0); + if (rc) { + dev_err(&res->pdev->dev, + "QPLIB: Failed to set function resources"); + } + return rc; +} + +/* SGID */ +int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, + struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, + struct bnxt_qplib_gid *gid) +{ + if (index >= sgid_tbl->max) { + dev_err(&res->pdev->dev, + "QPLIB: Index %d exceeded SGID table max (%d)", + index, sgid_tbl->max); + return -EINVAL; + } + memcpy(gid, &sgid_tbl->tbl[index], sizeof(*gid)); + return 0; +} + +int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, bool update) +{ + struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, + struct bnxt_qplib_res, + sgid_tbl); + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + int index; + + if (!sgid_tbl) { + dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated"); + return -EINVAL; + } + /* Do we need a sgid_lock here? */ + if (!sgid_tbl->active) { + dev_err(&res->pdev->dev, + "QPLIB: SGID table has no active entries"); + return -ENOMEM; + } + for (index = 0; index < sgid_tbl->max; index++) { + if (!memcmp(&sgid_tbl->tbl[index], gid, sizeof(*gid))) + break; + } + if (index == sgid_tbl->max) { + dev_warn(&res->pdev->dev, "GID not found in the SGID table"); + return 0; + } + /* Remove GID from the SGID table */ + if (update) { + struct cmdq_delete_gid req; + struct creq_delete_gid_resp resp; + u16 cmd_flags = 0; + int rc; + + RCFW_CMD_PREP(req, DELETE_GID, cmd_flags); + if (sgid_tbl->hw_id[index] == 0xFFFF) { + dev_err(&res->pdev->dev, + "QPLIB: GID entry contains an invalid HW id"); + return -EINVAL; + } + req.gid_index = cpu_to_le16(sgid_tbl->hw_id[index]); + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) + return rc; + } + memcpy(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero)); + sgid_tbl->vlan[index] = 0; + sgid_tbl->active--; + dev_dbg(&res->pdev->dev, + "QPLIB: SGID deleted hw_id[0x%x] = 0x%x active = 0x%x", + index, sgid_tbl->hw_id[index], sgid_tbl->active); + sgid_tbl->hw_id[index] = (u16)-1; + + /* unlock */ + return 0; +} + +int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, u8 *smac, u16 vlan_id, + bool update, u32 *index) +{ + struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, + struct bnxt_qplib_res, + sgid_tbl); + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + int i, free_idx; + + if (!sgid_tbl) { + dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated"); + return -EINVAL; + } + /* Do we need a sgid_lock here? */ + if (sgid_tbl->active == sgid_tbl->max) { + dev_err(&res->pdev->dev, "QPLIB: SGID table is full"); + return -ENOMEM; + } + free_idx = sgid_tbl->max; + for (i = 0; i < sgid_tbl->max; i++) { + if (!memcmp(&sgid_tbl->tbl[i], gid, sizeof(*gid))) { + dev_dbg(&res->pdev->dev, + "QPLIB: SGID entry already exist in entry %d!", + i); + *index = i; + return -EALREADY; + } else if (!memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero)) && + free_idx == sgid_tbl->max) { + free_idx = i; + } + } + if (free_idx == sgid_tbl->max) { + dev_err(&res->pdev->dev, + "QPLIB: SGID table is FULL but count is not MAX??"); + return -ENOMEM; + } + if (update) { + struct cmdq_add_gid req; + struct creq_add_gid_resp resp; + u16 cmd_flags = 0; + int rc; + + RCFW_CMD_PREP(req, ADD_GID, cmd_flags); + + req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]); + req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]); + req.gid[2] = cpu_to_be32(((u32 *)gid->data)[1]); + req.gid[3] = cpu_to_be32(((u32 *)gid->data)[0]); + /* + * driver should ensure that all RoCE traffic is always VLAN + * tagged if RoCE traffic is running on non-zero VLAN ID or + * RoCE traffic is running on non-zero Priority. + */ + if ((vlan_id != 0xFFFF) || res->prio) { + if (vlan_id != 0xFFFF) + req.vlan = cpu_to_le16 + (vlan_id & CMDQ_ADD_GID_VLAN_VLAN_ID_MASK); + req.vlan |= cpu_to_le16 + (CMDQ_ADD_GID_VLAN_TPID_TPID_8100 | + CMDQ_ADD_GID_VLAN_VLAN_EN); + } + + /* MAC in network format */ + req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]); + req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]); + req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) + return rc; + sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp.xid); + } + /* Add GID to the sgid_tbl */ + memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid)); + sgid_tbl->active++; + if (vlan_id != 0xFFFF) + sgid_tbl->vlan[free_idx] = 1; + + dev_dbg(&res->pdev->dev, + "QPLIB: SGID added hw_id[0x%x] = 0x%x active = 0x%x", + free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active); + + *index = free_idx; + /* unlock */ + return 0; +} + +int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, u16 gid_idx, + u8 *smac) +{ + struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, + struct bnxt_qplib_res, + sgid_tbl); + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_modify_gid_resp resp; + struct cmdq_modify_gid req; + int rc; + u16 cmd_flags = 0; + + RCFW_CMD_PREP(req, MODIFY_GID, cmd_flags); + + req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]); + req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]); + req.gid[2] = cpu_to_be32(((u32 *)gid->data)[1]); + req.gid[3] = cpu_to_be32(((u32 *)gid->data)[0]); + if (res->prio) { + req.vlan |= cpu_to_le16 + (CMDQ_ADD_GID_VLAN_TPID_TPID_8100 | + CMDQ_ADD_GID_VLAN_VLAN_EN); + } + + /* MAC in network format */ + req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]); + req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]); + req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]); + + req.gid_index = cpu_to_le16(gid_idx); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + return rc; +} + +/* pkeys */ +int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res, + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index, + u16 *pkey) +{ + if (index == 0xFFFF) { + *pkey = 0xFFFF; + return 0; + } + if (index >= pkey_tbl->max) { + dev_err(&res->pdev->dev, + "QPLIB: Index %d exceeded PKEY table max (%d)", + index, pkey_tbl->max); + return -EINVAL; + } + memcpy(pkey, &pkey_tbl->tbl[index], sizeof(*pkey)); + return 0; +} + +int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res, + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, + bool update) +{ + int i, rc = 0; + + if (!pkey_tbl) { + dev_err(&res->pdev->dev, "QPLIB: PKEY table not allocated"); + return -EINVAL; + } + + /* Do we need a pkey_lock here? */ + if (!pkey_tbl->active) { + dev_err(&res->pdev->dev, + "QPLIB: PKEY table has no active entries"); + return -ENOMEM; + } + for (i = 0; i < pkey_tbl->max; i++) { + if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey))) + break; + } + if (i == pkey_tbl->max) { + dev_err(&res->pdev->dev, + "QPLIB: PKEY 0x%04x not found in the pkey table", + *pkey); + return -ENOMEM; + } + memset(&pkey_tbl->tbl[i], 0, sizeof(*pkey)); + pkey_tbl->active--; + + /* unlock */ + return rc; +} + +int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res, + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, + bool update) +{ + int i, free_idx, rc = 0; + + if (!pkey_tbl) { + dev_err(&res->pdev->dev, "QPLIB: PKEY table not allocated"); + return -EINVAL; + } + + /* Do we need a pkey_lock here? */ + if (pkey_tbl->active == pkey_tbl->max) { + dev_err(&res->pdev->dev, "QPLIB: PKEY table is full"); + return -ENOMEM; + } + free_idx = pkey_tbl->max; + for (i = 0; i < pkey_tbl->max; i++) { + if (!memcmp(&pkey_tbl->tbl[i], pkey, sizeof(*pkey))) + return -EALREADY; + else if (!pkey_tbl->tbl[i] && free_idx == pkey_tbl->max) + free_idx = i; + } + if (free_idx == pkey_tbl->max) { + dev_err(&res->pdev->dev, + "QPLIB: PKEY table is FULL but count is not MAX??"); + return -ENOMEM; + } + /* Add PKEY to the pkey_tbl */ + memcpy(&pkey_tbl->tbl[free_idx], pkey, sizeof(*pkey)); + pkey_tbl->active++; + + /* unlock */ + return rc; +} + +/* AH */ +int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_create_ah req; + struct creq_create_ah_resp resp; + u16 cmd_flags = 0; + u32 temp32[4]; + u16 temp16[3]; + int rc; + + RCFW_CMD_PREP(req, CREATE_AH, cmd_flags); + + memcpy(temp32, ah->dgid.data, sizeof(struct bnxt_qplib_gid)); + req.dgid[0] = cpu_to_le32(temp32[0]); + req.dgid[1] = cpu_to_le32(temp32[1]); + req.dgid[2] = cpu_to_le32(temp32[2]); + req.dgid[3] = cpu_to_le32(temp32[3]); + + req.type = ah->nw_type; + req.hop_limit = ah->hop_limit; + req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[ah->sgid_index]); + req.dest_vlan_id_flow_label = cpu_to_le32((ah->flow_label & + CMDQ_CREATE_AH_FLOW_LABEL_MASK) | + CMDQ_CREATE_AH_DEST_VLAN_ID_MASK); + req.pd_id = cpu_to_le32(ah->pd->id); + req.traffic_class = ah->traffic_class; + + /* MAC in network format */ + memcpy(temp16, ah->dmac, 6); + req.dest_mac[0] = cpu_to_le16(temp16[0]); + req.dest_mac[1] = cpu_to_le16(temp16[1]); + req.dest_mac[2] = cpu_to_le16(temp16[2]); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, + NULL, 1); + if (rc) + return rc; + + ah->id = le32_to_cpu(resp.xid); + return 0; +} + +int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_destroy_ah req; + struct creq_destroy_ah_resp resp; + u16 cmd_flags = 0; + int rc; + + /* Clean up the AH table in the device */ + RCFW_CMD_PREP(req, DESTROY_AH, cmd_flags); + + req.ah_cid = cpu_to_le32(ah->id); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, + NULL, 1); + if (rc) + return rc; + return 0; +} + +/* MRW */ +int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_deallocate_key req; + struct creq_deallocate_key_resp resp; + u16 cmd_flags = 0; + int rc; + + if (mrw->lkey == 0xFFFFFFFF) { + dev_info(&res->pdev->dev, + "QPLIB: SP: Free a reserved lkey MRW"); + return 0; + } + + RCFW_CMD_PREP(req, DEALLOCATE_KEY, cmd_flags); + + req.mrw_flags = mrw->type; + + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) + req.key = cpu_to_le32(mrw->rkey); + else + req.key = cpu_to_le32(mrw->lkey); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, + NULL, 0); + if (rc) + return rc; + + /* Free the qplib's MRW memory */ + if (mrw->hwq.max_elements) + bnxt_qplib_free_hwq(res->pdev, &mrw->hwq); + + return 0; +} + +int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_allocate_mrw req; + struct creq_allocate_mrw_resp resp; + u16 cmd_flags = 0; + unsigned long tmp; + int rc; + + RCFW_CMD_PREP(req, ALLOCATE_MRW, cmd_flags); + + req.pd_id = cpu_to_le32(mrw->pd->id); + req.mrw_flags = mrw->type; + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR && + mrw->flags & BNXT_QPLIB_FR_PMR) || + mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A || + mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B) + req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY; + tmp = (unsigned long)mrw; + req.mrw_handle = cpu_to_le64(tmp); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, 0); + if (rc) + return rc; + + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) + mrw->rkey = le32_to_cpu(resp.xid); + else + mrw->lkey = le32_to_cpu(resp.xid); + return 0; +} + +int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, + bool block) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_deregister_mr req; + struct creq_deregister_mr_resp resp; + u16 cmd_flags = 0; + int rc; + + RCFW_CMD_PREP(req, DEREGISTER_MR, cmd_flags); + + req.lkey = cpu_to_le32(mrw->lkey); + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, block); + if (rc) + return rc; + + /* Free the qplib's MR memory */ + if (mrw->hwq.max_elements) { + mrw->va = 0; + mrw->total_size = 0; + bnxt_qplib_free_hwq(res->pdev, &mrw->hwq); + } + + return 0; +} + +int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, + u64 *pbl_tbl, int num_pbls, bool block, u32 buf_pg_size) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_register_mr req; + struct creq_register_mr_resp resp; + u16 cmd_flags = 0, level; + int pg_ptrs, pages, i, rc; + dma_addr_t **pbl_ptr; + u32 pg_size; + + if (num_pbls) { + /* Allocate memory for the non-leaf pages to store buf ptrs. + * Non-leaf pages always uses system PAGE_SIZE + */ + pg_ptrs = roundup_pow_of_two(num_pbls); + pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT; + if (!pages) + pages++; + + if (pages > MAX_PBL_LVL_1_PGS) { + dev_err(&res->pdev->dev, "QPLIB: SP: Reg MR pages "); + dev_err(&res->pdev->dev, + "requested (0x%x) exceeded max (0x%x)", + pages, MAX_PBL_LVL_1_PGS); + return -ENOMEM; + } + /* Free the hwq if it already exist, must be a rereg */ + if (mr->hwq.max_elements) + bnxt_qplib_free_hwq(res->pdev, &mr->hwq); + + mr->hwq.max_elements = pages; + /* Use system PAGE_SIZE */ + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &mr->hwq, NULL, 0, + &mr->hwq.max_elements, + PAGE_SIZE, 0, PAGE_SIZE, + HWQ_TYPE_CTX); + if (rc) { + dev_err(&res->pdev->dev, + "SP: Reg MR memory allocation failed"); + return -ENOMEM; + } + /* Write to the hwq */ + pbl_ptr = (dma_addr_t **)mr->hwq.pbl_ptr; + for (i = 0; i < num_pbls; i++) + pbl_ptr[PTR_PG(i)][PTR_IDX(i)] = + (pbl_tbl[i] & PAGE_MASK) | PTU_PTE_VALID; + } + + RCFW_CMD_PREP(req, REGISTER_MR, cmd_flags); + + /* Configure the request */ + if (mr->hwq.level == PBL_LVL_MAX) { + /* No PBL provided, just use system PAGE_SIZE */ + level = 0; + req.pbl = 0; + pg_size = PAGE_SIZE; + } else { + level = mr->hwq.level + 1; + req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]); + } + pg_size = buf_pg_size ? buf_pg_size : PAGE_SIZE; + req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) | + ((ilog2(pg_size) << + CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) & + CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK); + req.log2_pbl_pg_size = cpu_to_le16(((ilog2(PAGE_SIZE) << + CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT) & + CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK)); + req.access = (mr->flags & 0xFFFF); + req.va = cpu_to_le64(mr->va); + req.key = cpu_to_le32(mr->lkey); + req.mr_size = cpu_to_le64(mr->total_size); + + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, + (void *)&resp, NULL, block); + if (rc) + goto fail; + + return 0; + +fail: + if (mr->hwq.max_elements) + bnxt_qplib_free_hwq(res->pdev, &mr->hwq); + return rc; +} + +int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl, + int max_pg_ptrs) +{ + int pg_ptrs, pages, rc; + + /* Re-calculate the max to fit the HWQ allocation model */ + pg_ptrs = roundup_pow_of_two(max_pg_ptrs); + pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT; + if (!pages) + pages++; + + if (pages > MAX_PBL_LVL_1_PGS) + return -ENOMEM; + + frpl->hwq.max_elements = pages; + rc = bnxt_qplib_alloc_init_hwq(res->pdev, &frpl->hwq, NULL, 0, + &frpl->hwq.max_elements, PAGE_SIZE, 0, + PAGE_SIZE, HWQ_TYPE_CTX); + if (!rc) + frpl->max_pg_ptrs = pg_ptrs; + + return rc; +} + +int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl) +{ + bnxt_qplib_free_hwq(res->pdev, &frpl->hwq); + return 0; +} + +int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_map_tc_to_cos req; + struct creq_map_tc_to_cos_resp resp; + u16 cmd_flags = 0; + + RCFW_CMD_PREP(req, MAP_TC_TO_COS, cmd_flags); + req.cos0 = cpu_to_le16(cids[0]); + req.cos1 = cpu_to_le16(cids[1]); + + return bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, + NULL, 0); +} + +int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_roce_stats *stats) +{ + struct cmdq_query_roce_stats req; + struct creq_query_roce_stats_resp resp; + struct bnxt_qplib_rcfw_sbuf *sbuf; + struct creq_query_roce_stats_resp_sb *sb; + u16 cmd_flags = 0; + int rc = 0; + + RCFW_CMD_PREP(req, QUERY_ROCE_STATS, cmd_flags); + + sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb)); + if (!sbuf) { + dev_err(&rcfw->pdev->dev, + "QPLIB: SP: QUERY_ROCE_STATS alloc side buffer failed"); + return -ENOMEM; + } + + sb = sbuf->sb; + req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS; + rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, + (void *)sbuf, 0); + if (rc) + goto bail; + /* Extract the context from the side buffer */ + stats->to_retransmits = le64_to_cpu(sb->to_retransmits); + stats->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd); + stats->max_retry_exceeded = le64_to_cpu(sb->max_retry_exceeded); + stats->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd); + stats->missing_resp = le64_to_cpu(sb->missing_resp); + stats->unrecoverable_err = le64_to_cpu(sb->unrecoverable_err); + stats->bad_resp_err = le64_to_cpu(sb->bad_resp_err); + stats->local_qp_op_err = le64_to_cpu(sb->local_qp_op_err); + stats->local_protection_err = le64_to_cpu(sb->local_protection_err); + stats->mem_mgmt_op_err = le64_to_cpu(sb->mem_mgmt_op_err); + stats->remote_invalid_req_err = le64_to_cpu(sb->remote_invalid_req_err); + stats->remote_access_err = le64_to_cpu(sb->remote_access_err); + stats->remote_op_err = le64_to_cpu(sb->remote_op_err); + stats->dup_req = le64_to_cpu(sb->dup_req); + stats->res_exceed_max = le64_to_cpu(sb->res_exceed_max); + stats->res_length_mismatch = le64_to_cpu(sb->res_length_mismatch); + stats->res_exceeds_wqe = le64_to_cpu(sb->res_exceeds_wqe); + stats->res_opcode_err = le64_to_cpu(sb->res_opcode_err); + stats->res_rx_invalid_rkey = le64_to_cpu(sb->res_rx_invalid_rkey); + stats->res_rx_domain_err = le64_to_cpu(sb->res_rx_domain_err); + stats->res_rx_no_perm = le64_to_cpu(sb->res_rx_no_perm); + stats->res_rx_range_err = le64_to_cpu(sb->res_rx_range_err); + stats->res_tx_invalid_rkey = le64_to_cpu(sb->res_tx_invalid_rkey); + stats->res_tx_domain_err = le64_to_cpu(sb->res_tx_domain_err); + stats->res_tx_no_perm = le64_to_cpu(sb->res_tx_no_perm); + stats->res_tx_range_err = le64_to_cpu(sb->res_tx_range_err); + stats->res_irrq_oflow = le64_to_cpu(sb->res_irrq_oflow); + stats->res_unsup_opcode = le64_to_cpu(sb->res_unsup_opcode); + stats->res_unaligned_atomic = le64_to_cpu(sb->res_unaligned_atomic); + stats->res_rem_inv_err = le64_to_cpu(sb->res_rem_inv_err); + stats->res_mem_error = le64_to_cpu(sb->res_mem_error); + stats->res_srq_err = le64_to_cpu(sb->res_srq_err); + stats->res_cmp_err = le64_to_cpu(sb->res_cmp_err); + stats->res_invalid_dup_rkey = le64_to_cpu(sb->res_invalid_dup_rkey); + stats->res_wqe_format_err = le64_to_cpu(sb->res_wqe_format_err); + stats->res_cq_load_err = le64_to_cpu(sb->res_cq_load_err); + stats->res_srq_load_err = le64_to_cpu(sb->res_srq_load_err); + stats->res_tx_pci_err = le64_to_cpu(sb->res_tx_pci_err); + stats->res_rx_pci_err = le64_to_cpu(sb->res_rx_pci_err); +bail: + bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf); + return rc; +} diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h new file mode 100644 index 000000000..b6e9e0ef7 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h @@ -0,0 +1,253 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Slow Path Operators (header) + * + */ + +#ifndef __BNXT_QPLIB_SP_H__ +#define __BNXT_QPLIB_SP_H__ + +#define BNXT_QPLIB_RESERVED_QP_WRS 128 + +#define PCI_EXP_DEVCTL2_ATOMIC_REQ 0x0040 + +struct bnxt_qplib_dev_attr { +#define FW_VER_ARR_LEN 4 + u8 fw_ver[FW_VER_ARR_LEN]; +#define BNXT_QPLIB_NUM_GIDS_SUPPORTED 256 + u16 max_sgid; + u16 max_mrw; + u32 max_qp; +#define BNXT_QPLIB_MAX_OUT_RD_ATOM 126 + u32 max_qp_rd_atom; + u32 max_qp_init_rd_atom; + u32 max_qp_wqes; + u32 max_qp_sges; + u32 max_cq; + u32 max_cq_wqes; + u32 max_cq_sges; + u32 max_mr; + u64 max_mr_size; + u32 max_pd; + u32 max_mw; + u32 max_raw_ethy_qp; + u32 max_ah; + u32 max_fmr; + u32 max_map_per_fmr; + u32 max_srq; + u32 max_srq_wqes; + u32 max_srq_sges; + u32 max_pkey; + u32 max_inline_data; + u32 l2_db_size; + u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; + bool is_atomic; +}; + +struct bnxt_qplib_pd { + u32 id; +}; + +struct bnxt_qplib_gid { + u8 data[16]; +}; + +struct bnxt_qplib_ah { + struct bnxt_qplib_gid dgid; + struct bnxt_qplib_pd *pd; + u32 id; + u8 sgid_index; + /* For Query AH if the hw table and SW table are differnt */ + u8 host_sgid_index; + u8 traffic_class; + u32 flow_label; + u8 hop_limit; + u8 sl; + u8 dmac[6]; + u16 vlan_id; + u8 nw_type; +}; + +struct bnxt_qplib_mrw { + struct bnxt_qplib_pd *pd; + int type; + u32 flags; +#define BNXT_QPLIB_FR_PMR 0x80000000 + u32 lkey; + u32 rkey; +#define BNXT_QPLIB_RSVD_LKEY 0xFFFFFFFF + u64 va; + u64 total_size; + u32 npages; + u64 mr_handle; + struct bnxt_qplib_hwq hwq; +}; + +struct bnxt_qplib_frpl { + int max_pg_ptrs; + struct bnxt_qplib_hwq hwq; +}; + +#define BNXT_QPLIB_ACCESS_LOCAL_WRITE BIT(0) +#define BNXT_QPLIB_ACCESS_REMOTE_READ BIT(1) +#define BNXT_QPLIB_ACCESS_REMOTE_WRITE BIT(2) +#define BNXT_QPLIB_ACCESS_REMOTE_ATOMIC BIT(3) +#define BNXT_QPLIB_ACCESS_MW_BIND BIT(4) +#define BNXT_QPLIB_ACCESS_ZERO_BASED BIT(5) +#define BNXT_QPLIB_ACCESS_ON_DEMAND BIT(6) + +struct bnxt_qplib_roce_stats { + u64 to_retransmits; + u64 seq_err_naks_rcvd; + /* seq_err_naks_rcvd is 64 b */ + u64 max_retry_exceeded; + /* max_retry_exceeded is 64 b */ + u64 rnr_naks_rcvd; + /* rnr_naks_rcvd is 64 b */ + u64 missing_resp; + u64 unrecoverable_err; + /* unrecoverable_err is 64 b */ + u64 bad_resp_err; + /* bad_resp_err is 64 b */ + u64 local_qp_op_err; + /* local_qp_op_err is 64 b */ + u64 local_protection_err; + /* local_protection_err is 64 b */ + u64 mem_mgmt_op_err; + /* mem_mgmt_op_err is 64 b */ + u64 remote_invalid_req_err; + /* remote_invalid_req_err is 64 b */ + u64 remote_access_err; + /* remote_access_err is 64 b */ + u64 remote_op_err; + /* remote_op_err is 64 b */ + u64 dup_req; + /* dup_req is 64 b */ + u64 res_exceed_max; + /* res_exceed_max is 64 b */ + u64 res_length_mismatch; + /* res_length_mismatch is 64 b */ + u64 res_exceeds_wqe; + /* res_exceeds_wqe is 64 b */ + u64 res_opcode_err; + /* res_opcode_err is 64 b */ + u64 res_rx_invalid_rkey; + /* res_rx_invalid_rkey is 64 b */ + u64 res_rx_domain_err; + /* res_rx_domain_err is 64 b */ + u64 res_rx_no_perm; + /* res_rx_no_perm is 64 b */ + u64 res_rx_range_err; + /* res_rx_range_err is 64 b */ + u64 res_tx_invalid_rkey; + /* res_tx_invalid_rkey is 64 b */ + u64 res_tx_domain_err; + /* res_tx_domain_err is 64 b */ + u64 res_tx_no_perm; + /* res_tx_no_perm is 64 b */ + u64 res_tx_range_err; + /* res_tx_range_err is 64 b */ + u64 res_irrq_oflow; + /* res_irrq_oflow is 64 b */ + u64 res_unsup_opcode; + /* res_unsup_opcode is 64 b */ + u64 res_unaligned_atomic; + /* res_unaligned_atomic is 64 b */ + u64 res_rem_inv_err; + /* res_rem_inv_err is 64 b */ + u64 res_mem_error; + /* res_mem_error is 64 b */ + u64 res_srq_err; + /* res_srq_err is 64 b */ + u64 res_cmp_err; + /* res_cmp_err is 64 b */ + u64 res_invalid_dup_rkey; + /* res_invalid_dup_rkey is 64 b */ + u64 res_wqe_format_err; + /* res_wqe_format_err is 64 b */ + u64 res_cq_load_err; + /* res_cq_load_err is 64 b */ + u64 res_srq_load_err; + /* res_srq_load_err is 64 b */ + u64 res_tx_pci_err; + /* res_tx_pci_err is 64 b */ + u64 res_rx_pci_err; + /* res_rx_pci_err is 64 b */ +}; + +int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, + struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, + struct bnxt_qplib_gid *gid); +int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, bool update); +int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, u8 *mac, u16 vlan_id, + bool update, u32 *index); +int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, u16 gid_idx, u8 *smac); +int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res, + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index, + u16 *pkey); +int bnxt_qplib_del_pkey(struct bnxt_qplib_res *res, + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, + bool update); +int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res, + struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey, + bool update); +int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_dev_attr *attr, bool vf); +int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res, + struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_ctx *ctx); +int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah); +int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah); +int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, + struct bnxt_qplib_mrw *mrw); +int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, + bool block); +int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, + u64 *pbl_tbl, int num_pbls, bool block, u32 buf_pg_size); +int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr); +int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res, + struct bnxt_qplib_mrw *mr, int max); +int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl, int max); +int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl); +int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids); +int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_roce_stats *stats); +#endif /* __BNXT_QPLIB_SP_H__*/ diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h new file mode 100644 index 000000000..3e5a4f760 --- /dev/null +++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h @@ -0,0 +1,2969 @@ +/* + * Broadcom NetXtreme-E RoCE driver. + * + * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: RoCE HSI File - Autogenerated + */ + +#ifndef __BNXT_RE_HSI_H__ +#define __BNXT_RE_HSI_H__ + +/* include bnxt_hsi.h from bnxt_en driver */ +#include "bnxt_hsi.h" + +/* CMP Door Bell Format (4 bytes) */ +struct cmpl_doorbell { + __le32 key_mask_valid_idx; + #define CMPL_DOORBELL_IDX_MASK 0xffffffUL + #define CMPL_DOORBELL_IDX_SFT 0 + #define CMPL_DOORBELL_RESERVED_MASK 0x3000000UL + #define CMPL_DOORBELL_RESERVED_SFT 24 + #define CMPL_DOORBELL_IDX_VALID 0x4000000UL + #define CMPL_DOORBELL_MASK 0x8000000UL + #define CMPL_DOORBELL_KEY_MASK 0xf0000000UL + #define CMPL_DOORBELL_KEY_SFT 28 + #define CMPL_DOORBELL_KEY_CMPL (0x2UL << 28) +}; + +/* Status Door Bell Format (4 bytes) */ +struct status_doorbell { + __le32 key_idx; + #define STATUS_DOORBELL_IDX_MASK 0xffffffUL + #define STATUS_DOORBELL_IDX_SFT 0 + #define STATUS_DOORBELL_RESERVED_MASK 0xf000000UL + #define STATUS_DOORBELL_RESERVED_SFT 24 + #define STATUS_DOORBELL_KEY_MASK 0xf0000000UL + #define STATUS_DOORBELL_KEY_SFT 28 + #define STATUS_DOORBELL_KEY_STAT (0x3UL << 28) +}; + +/* RoCE Host Structures */ + +/* Doorbell Structures */ +/* 64b Doorbell Format (8 bytes) */ +struct dbr_dbr { + __le32 index; + #define DBR_DBR_INDEX_MASK 0xfffffUL + #define DBR_DBR_INDEX_SFT 0 + #define DBR_DBR_RESERVED12_MASK 0xfff00000UL + #define DBR_DBR_RESERVED12_SFT 20 + __le32 type_xid; + #define DBR_DBR_XID_MASK 0xfffffUL + #define DBR_DBR_XID_SFT 0 + #define DBR_DBR_RESERVED8_MASK 0xff00000UL + #define DBR_DBR_RESERVED8_SFT 20 + #define DBR_DBR_TYPE_MASK 0xf0000000UL + #define DBR_DBR_TYPE_SFT 28 + #define DBR_DBR_TYPE_SQ (0x0UL << 28) + #define DBR_DBR_TYPE_RQ (0x1UL << 28) + #define DBR_DBR_TYPE_SRQ (0x2UL << 28) + #define DBR_DBR_TYPE_SRQ_ARM (0x3UL << 28) + #define DBR_DBR_TYPE_CQ (0x4UL << 28) + #define DBR_DBR_TYPE_CQ_ARMSE (0x5UL << 28) + #define DBR_DBR_TYPE_CQ_ARMALL (0x6UL << 28) + #define DBR_DBR_TYPE_CQ_ARMENA (0x7UL << 28) + #define DBR_DBR_TYPE_SRQ_ARMENA (0x8UL << 28) + #define DBR_DBR_TYPE_CQ_CUTOFF_ACK (0x9UL << 28) + #define DBR_DBR_TYPE_NULL (0xfUL << 28) +}; + +/* 32b Doorbell Format (4 bytes) */ +struct dbr_dbr32 { + __le32 type_abs_incr_xid; + #define DBR_DBR32_XID_MASK 0xfffffUL + #define DBR_DBR32_XID_SFT 0 + #define DBR_DBR32_RESERVED4_MASK 0xf00000UL + #define DBR_DBR32_RESERVED4_SFT 20 + #define DBR_DBR32_INCR_MASK 0xf000000UL + #define DBR_DBR32_INCR_SFT 24 + #define DBR_DBR32_ABS 0x10000000UL + #define DBR_DBR32_TYPE_MASK 0xe0000000UL + #define DBR_DBR32_TYPE_SFT 29 + #define DBR_DBR32_TYPE_SQ (0x0UL << 29) +}; + +/* SQ WQE Structures */ +/* Base SQ WQE (8 bytes) */ +struct sq_base { + u8 wqe_type; + #define SQ_BASE_WQE_TYPE_SEND 0x0UL + #define SQ_BASE_WQE_TYPE_SEND_W_IMMEAD 0x1UL + #define SQ_BASE_WQE_TYPE_SEND_W_INVALID 0x2UL + #define SQ_BASE_WQE_TYPE_WRITE_WQE 0x4UL + #define SQ_BASE_WQE_TYPE_WRITE_W_IMMEAD 0x5UL + #define SQ_BASE_WQE_TYPE_READ_WQE 0x6UL + #define SQ_BASE_WQE_TYPE_ATOMIC_CS 0x8UL + #define SQ_BASE_WQE_TYPE_ATOMIC_FA 0xbUL + #define SQ_BASE_WQE_TYPE_LOCAL_INVALID 0xcUL + #define SQ_BASE_WQE_TYPE_FR_PMR 0xdUL + #define SQ_BASE_WQE_TYPE_BIND 0xeUL + u8 unused_0[7]; +}; + +/* WQE SGE (16 bytes) */ +struct sq_sge { + __le64 va_or_pa; + __le32 l_key; + __le32 size; +}; + +/* PSN Search Structure (8 bytes) */ +struct sq_psn_search { + __le32 opcode_start_psn; + #define SQ_PSN_SEARCH_START_PSN_MASK 0xffffffUL + #define SQ_PSN_SEARCH_START_PSN_SFT 0 + #define SQ_PSN_SEARCH_OPCODE_MASK 0xff000000UL + #define SQ_PSN_SEARCH_OPCODE_SFT 24 + __le32 flags_next_psn; + #define SQ_PSN_SEARCH_NEXT_PSN_MASK 0xffffffUL + #define SQ_PSN_SEARCH_NEXT_PSN_SFT 0 + #define SQ_PSN_SEARCH_FLAGS_MASK 0xff000000UL + #define SQ_PSN_SEARCH_FLAGS_SFT 24 +}; + +/* Send SQ WQE (40 bytes) */ +struct sq_send { + u8 wqe_type; + #define SQ_SEND_WQE_TYPE_SEND 0x0UL + #define SQ_SEND_WQE_TYPE_SEND_W_IMMEAD 0x1UL + #define SQ_SEND_WQE_TYPE_SEND_W_INVALID 0x2UL + u8 flags; + #define SQ_SEND_FLAGS_SIGNAL_COMP 0x1UL + #define SQ_SEND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL + #define SQ_SEND_FLAGS_UC_FENCE 0x4UL + #define SQ_SEND_FLAGS_SE 0x8UL + #define SQ_SEND_FLAGS_INLINE 0x10UL + u8 wqe_size; + u8 reserved8_1; + __le32 inv_key_or_imm_data; + __le32 length; + __le32 q_key; + __le32 dst_qp; + #define SQ_SEND_DST_QP_MASK 0xffffffUL + #define SQ_SEND_DST_QP_SFT 0 + #define SQ_SEND_RESERVED8_2_MASK 0xff000000UL + #define SQ_SEND_RESERVED8_2_SFT 24 + __le32 avid; + #define SQ_SEND_AVID_MASK 0xfffffUL + #define SQ_SEND_AVID_SFT 0 + #define SQ_SEND_RESERVED_AVID_MASK 0xfff00000UL + #define SQ_SEND_RESERVED_AVID_SFT 20 + __le64 reserved64; + __le32 data[24]; +}; + +/* Send Raw Ethernet and QP1 SQ WQE (40 bytes) */ +struct sq_send_raweth_qp1 { + u8 wqe_type; + #define SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND 0x0UL + u8 flags; + #define SQ_SEND_RAWETH_QP1_FLAGS_SIGNAL_COMP 0x1UL + #define SQ_SEND_RAWETH_QP1_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL + #define SQ_SEND_RAWETH_QP1_FLAGS_UC_FENCE 0x4UL + #define SQ_SEND_RAWETH_QP1_FLAGS_SE 0x8UL + #define SQ_SEND_RAWETH_QP1_FLAGS_INLINE 0x10UL + u8 wqe_size; + u8 reserved8; + __le16 lflags; + #define SQ_SEND_RAWETH_QP1_LFLAGS_TCP_UDP_CHKSUM 0x1UL + #define SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM 0x2UL + #define SQ_SEND_RAWETH_QP1_LFLAGS_NOCRC 0x4UL + #define SQ_SEND_RAWETH_QP1_LFLAGS_STAMP 0x8UL + #define SQ_SEND_RAWETH_QP1_LFLAGS_T_IP_CHKSUM 0x10UL + #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_1 0x20UL + #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_2 0x40UL + #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_3 0x80UL + #define SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC 0x100UL + #define SQ_SEND_RAWETH_QP1_LFLAGS_FCOE_CRC 0x200UL + __le16 cfa_action; + __le32 length; + __le32 reserved32_1; + __le32 cfa_meta; + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK 0xfffUL + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT 0 + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_DE 0x1000UL + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_MASK 0xe000UL + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_SFT 13 + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_MASK 0x70000UL + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_SFT 16 + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID88A8 (0x0UL << 16) + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID8100 (0x1UL << 16) + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9100 (0x2UL << 16) + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9200 (0x3UL << 16) + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9300 (0x4UL << 16) + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG (0x5UL << 16) + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_LAST \ + SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_MASK 0xff80000UL + #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_SFT 19 + #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_MASK 0xf0000000UL + #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_SFT 28 + #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_NONE (0x0UL << 28) + #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG (0x1UL << 28) + #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_LAST \ + SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG + __le32 reserved32_2; + __le64 reserved64; + __le32 data[24]; +}; + +/* RDMA SQ WQE (40 bytes) */ +struct sq_rdma { + u8 wqe_type; + #define SQ_RDMA_WQE_TYPE_WRITE_WQE 0x4UL + #define SQ_RDMA_WQE_TYPE_WRITE_W_IMMEAD 0x5UL + #define SQ_RDMA_WQE_TYPE_READ_WQE 0x6UL + u8 flags; + #define SQ_RDMA_FLAGS_SIGNAL_COMP 0x1UL + #define SQ_RDMA_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL + #define SQ_RDMA_FLAGS_UC_FENCE 0x4UL + #define SQ_RDMA_FLAGS_SE 0x8UL + #define SQ_RDMA_FLAGS_INLINE 0x10UL + u8 wqe_size; + u8 reserved8; + __le32 imm_data; + __le32 length; + __le32 reserved32_1; + __le64 remote_va; + __le32 remote_key; + __le32 reserved32_2; + __le32 data[24]; +}; + +/* Atomic SQ WQE (40 bytes) */ +struct sq_atomic { + u8 wqe_type; + #define SQ_ATOMIC_WQE_TYPE_ATOMIC_CS 0x8UL + #define SQ_ATOMIC_WQE_TYPE_ATOMIC_FA 0xbUL + u8 flags; + #define SQ_ATOMIC_FLAGS_SIGNAL_COMP 0x1UL + #define SQ_ATOMIC_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL + #define SQ_ATOMIC_FLAGS_UC_FENCE 0x4UL + #define SQ_ATOMIC_FLAGS_SE 0x8UL + #define SQ_ATOMIC_FLAGS_INLINE 0x10UL + __le16 reserved16; + __le32 remote_key; + __le64 remote_va; + __le64 swap_data; + __le64 cmp_data; + __le32 data[24]; +}; + +/* Local Invalidate SQ WQE (40 bytes) */ +struct sq_localinvalidate { + u8 wqe_type; + #define SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID 0xcUL + u8 flags; + #define SQ_LOCALINVALIDATE_FLAGS_SIGNAL_COMP 0x1UL + #define SQ_LOCALINVALIDATE_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL + #define SQ_LOCALINVALIDATE_FLAGS_UC_FENCE 0x4UL + #define SQ_LOCALINVALIDATE_FLAGS_SE 0x8UL + #define SQ_LOCALINVALIDATE_FLAGS_INLINE 0x10UL + __le16 reserved16; + __le32 inv_l_key; + __le64 reserved64; + __le32 reserved128[4]; + __le32 data[24]; +}; + +/* FR-PMR SQ WQE (40 bytes) */ +struct sq_fr_pmr { + u8 wqe_type; + #define SQ_FR_PMR_WQE_TYPE_FR_PMR 0xdUL + u8 flags; + #define SQ_FR_PMR_FLAGS_SIGNAL_COMP 0x1UL + #define SQ_FR_PMR_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL + #define SQ_FR_PMR_FLAGS_UC_FENCE 0x4UL + #define SQ_FR_PMR_FLAGS_SE 0x8UL + #define SQ_FR_PMR_FLAGS_INLINE 0x10UL + u8 access_cntl; + #define SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE 0x1UL + #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ 0x2UL + #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE 0x4UL + #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL + #define SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND 0x10UL + u8 zero_based_page_size_log; + #define SQ_FR_PMR_PAGE_SIZE_LOG_MASK 0x1fUL + #define SQ_FR_PMR_PAGE_SIZE_LOG_SFT 0 + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4K 0x0UL + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8K 0x1UL + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64K 0x4UL + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256K 0x6UL + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1M 0x8UL + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2M 0x9UL + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4M 0xaUL + #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1G 0x12UL + #define SQ_FR_PMR_ZERO_BASED 0x20UL + #define SQ_FR_PMR_RESERVED2_MASK 0xc0UL + #define SQ_FR_PMR_RESERVED2_SFT 6 + __le32 l_key; + u8 length[5]; + u8 reserved8_1; + u8 reserved8_2; + u8 numlevels_pbl_page_size_log; + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK 0x1fUL + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT 0 + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4K 0x0UL + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8K 0x1UL + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64K 0x4UL + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256K 0x6UL + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1M 0x8UL + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2M 0x9UL + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4M 0xaUL + #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1G 0x12UL + #define SQ_FR_PMR_RESERVED1 0x20UL + #define SQ_FR_PMR_NUMLEVELS_MASK 0xc0UL + #define SQ_FR_PMR_NUMLEVELS_SFT 6 + #define SQ_FR_PMR_NUMLEVELS_PHYSICAL (0x0UL << 6) + #define SQ_FR_PMR_NUMLEVELS_LAYER1 (0x1UL << 6) + #define SQ_FR_PMR_NUMLEVELS_LAYER2 (0x2UL << 6) + __le64 pblptr; + __le64 va; + __le32 data[24]; +}; + +/* Bind SQ WQE (40 bytes) */ +struct sq_bind { + u8 wqe_type; + #define SQ_BIND_WQE_TYPE_BIND 0xeUL + u8 flags; + #define SQ_BIND_FLAGS_SIGNAL_COMP 0x1UL + #define SQ_BIND_FLAGS_RD_OR_ATOMIC_FENCE 0x2UL + #define SQ_BIND_FLAGS_UC_FENCE 0x4UL + #define SQ_BIND_FLAGS_SE 0x8UL + #define SQ_BIND_FLAGS_INLINE 0x10UL + u8 access_cntl; + #define SQ_BIND_ACCESS_CNTL_LOCAL_WRITE 0x1UL + #define SQ_BIND_ACCESS_CNTL_REMOTE_READ 0x2UL + #define SQ_BIND_ACCESS_CNTL_REMOTE_WRITE 0x4UL + #define SQ_BIND_ACCESS_CNTL_REMOTE_ATOMIC 0x8UL + #define SQ_BIND_ACCESS_CNTL_WINDOW_BIND 0x10UL + u8 reserved8_1; + u8 mw_type_zero_based; + #define SQ_BIND_ZERO_BASED 0x1UL + #define SQ_BIND_MW_TYPE 0x2UL + #define SQ_BIND_MW_TYPE_TYPE1 (0x0UL << 1) + #define SQ_BIND_MW_TYPE_TYPE2 (0x1UL << 1) + #define SQ_BIND_RESERVED6_MASK 0xfcUL + #define SQ_BIND_RESERVED6_SFT 2 + u8 reserved8_2; + __le16 reserved16; + __le32 parent_l_key; + __le32 l_key; + __le64 va; + u8 length[5]; + u8 data_reserved24[99]; + #define SQ_BIND_RESERVED24_MASK 0xffffff00UL + #define SQ_BIND_RESERVED24_SFT 8 + #define SQ_BIND_DATA_MASK 0xffffffffUL + #define SQ_BIND_DATA_SFT 0 +}; + +/* RQ/SRQ WQE Structures */ +/* RQ/SRQ WQE (40 bytes) */ +struct rq_wqe { + u8 wqe_type; + #define RQ_WQE_WQE_TYPE_RCV 0x80UL + u8 flags; + u8 wqe_size; + u8 reserved8; + __le32 reserved32; + __le32 wr_id[2]; + #define RQ_WQE_WR_ID_MASK 0xfffffUL + #define RQ_WQE_WR_ID_SFT 0 + #define RQ_WQE_RESERVED44_MASK 0xfff00000UL + #define RQ_WQE_RESERVED44_SFT 20 + __le32 reserved128[4]; + __le32 data[24]; +}; + +/* CQ CQE Structures */ +/* Base CQE (32 bytes) */ +struct cq_base { + __le64 reserved64_1; + __le64 reserved64_2; + __le64 reserved64_3; + u8 cqe_type_toggle; + #define CQ_BASE_TOGGLE 0x1UL + #define CQ_BASE_CQE_TYPE_MASK 0x1eUL + #define CQ_BASE_CQE_TYPE_SFT 1 + #define CQ_BASE_CQE_TYPE_REQ (0x0UL << 1) + #define CQ_BASE_CQE_TYPE_RES_RC (0x1UL << 1) + #define CQ_BASE_CQE_TYPE_RES_UD (0x2UL << 1) + #define CQ_BASE_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1) + #define CQ_BASE_CQE_TYPE_TERMINAL (0xeUL << 1) + #define CQ_BASE_CQE_TYPE_CUT_OFF (0xfUL << 1) + #define CQ_BASE_RESERVED3_MASK 0xe0UL + #define CQ_BASE_RESERVED3_SFT 5 + u8 status; + __le16 reserved16; + __le32 reserved32; +}; + +/* Requester CQ CQE (32 bytes) */ +struct cq_req { + __le64 qp_handle; + __le16 sq_cons_idx; + __le16 reserved16_1; + __le32 reserved32_2; + __le64 reserved64; + u8 cqe_type_toggle; + #define CQ_REQ_TOGGLE 0x1UL + #define CQ_REQ_CQE_TYPE_MASK 0x1eUL + #define CQ_REQ_CQE_TYPE_SFT 1 + #define CQ_REQ_CQE_TYPE_REQ (0x0UL << 1) + #define CQ_REQ_RESERVED3_MASK 0xe0UL + #define CQ_REQ_RESERVED3_SFT 5 + u8 status; + #define CQ_REQ_STATUS_OK 0x0UL + #define CQ_REQ_STATUS_BAD_RESPONSE_ERR 0x1UL + #define CQ_REQ_STATUS_LOCAL_LENGTH_ERR 0x2UL + #define CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR 0x3UL + #define CQ_REQ_STATUS_LOCAL_PROTECTION_ERR 0x4UL + #define CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL + #define CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL + #define CQ_REQ_STATUS_REMOTE_ACCESS_ERR 0x7UL + #define CQ_REQ_STATUS_REMOTE_OPERATION_ERR 0x8UL + #define CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR 0x9UL + #define CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR 0xaUL + #define CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR 0xbUL + __le16 reserved16_2; + __le32 reserved32_1; +}; + +/* Responder RC CQE (32 bytes) */ +struct cq_res_rc { + __le32 length; + __le32 imm_data_or_inv_r_key; + __le64 qp_handle; + __le64 mr_handle; + u8 cqe_type_toggle; + #define CQ_RES_RC_TOGGLE 0x1UL + #define CQ_RES_RC_CQE_TYPE_MASK 0x1eUL + #define CQ_RES_RC_CQE_TYPE_SFT 1 + #define CQ_RES_RC_CQE_TYPE_RES_RC (0x1UL << 1) + #define CQ_RES_RC_RESERVED3_MASK 0xe0UL + #define CQ_RES_RC_RESERVED3_SFT 5 + u8 status; + #define CQ_RES_RC_STATUS_OK 0x0UL + #define CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR 0x1UL + #define CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR 0x2UL + #define CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR 0x3UL + #define CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL + #define CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL + #define CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR 0x6UL + #define CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL + #define CQ_RES_RC_STATUS_HW_FLUSH_ERR 0x8UL + __le16 flags; + #define CQ_RES_RC_FLAGS_SRQ 0x1UL + #define CQ_RES_RC_FLAGS_SRQ_RQ (0x0UL << 0) + #define CQ_RES_RC_FLAGS_SRQ_SRQ (0x1UL << 0) + #define CQ_RES_RC_FLAGS_SRQ_LAST CQ_RES_RC_FLAGS_SRQ_SRQ + #define CQ_RES_RC_FLAGS_IMM 0x2UL + #define CQ_RES_RC_FLAGS_INV 0x4UL + #define CQ_RES_RC_FLAGS_RDMA 0x8UL + #define CQ_RES_RC_FLAGS_RDMA_SEND (0x0UL << 3) + #define CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE (0x1UL << 3) + #define CQ_RES_RC_FLAGS_RDMA_LAST CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE + __le32 srq_or_rq_wr_id; + #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL + #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_SFT 0 + #define CQ_RES_RC_RESERVED12_MASK 0xfff00000UL + #define CQ_RES_RC_RESERVED12_SFT 20 +}; + +/* Responder UD CQE (32 bytes) */ +struct cq_res_ud { + __le32 length; + #define CQ_RES_UD_LENGTH_MASK 0x3fffUL + #define CQ_RES_UD_LENGTH_SFT 0 + #define CQ_RES_UD_RESERVED18_MASK 0xffffc000UL + #define CQ_RES_UD_RESERVED18_SFT 14 + __le32 imm_data; + __le64 qp_handle; + __le16 src_mac[3]; + __le16 src_qp_low; + u8 cqe_type_toggle; + #define CQ_RES_UD_TOGGLE 0x1UL + #define CQ_RES_UD_CQE_TYPE_MASK 0x1eUL + #define CQ_RES_UD_CQE_TYPE_SFT 1 + #define CQ_RES_UD_CQE_TYPE_RES_UD (0x2UL << 1) + #define CQ_RES_UD_RESERVED3_MASK 0xe0UL + #define CQ_RES_UD_RESERVED3_SFT 5 + u8 status; + #define CQ_RES_UD_STATUS_OK 0x0UL + #define CQ_RES_UD_STATUS_LOCAL_ACCESS_ERROR 0x1UL + #define CQ_RES_UD_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL + #define CQ_RES_UD_STATUS_LOCAL_PROTECTION_ERR 0x3UL + #define CQ_RES_UD_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL + #define CQ_RES_UD_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL + #define CQ_RES_UD_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL + #define CQ_RES_UD_STATUS_HW_FLUSH_ERR 0x8UL + __le16 flags; + #define CQ_RES_UD_FLAGS_SRQ 0x1UL + #define CQ_RES_UD_FLAGS_SRQ_RQ (0x0UL << 0) + #define CQ_RES_UD_FLAGS_SRQ_SRQ (0x1UL << 0) + #define CQ_RES_UD_FLAGS_SRQ_LAST CQ_RES_UD_FLAGS_SRQ_SRQ + #define CQ_RES_UD_FLAGS_IMM 0x2UL + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK 0xcUL + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT 2 + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V1 (0x0UL << 2) + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV4 (0x2UL << 2) + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 (0x3UL << 2) + #define CQ_RES_UD_FLAGS_ROCE_IP_VER_LAST \ + CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 + __le32 src_qp_high_srq_or_rq_wr_id; + #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL + #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_SFT 0 + #define CQ_RES_UD_RESERVED4_MASK 0xf00000UL + #define CQ_RES_UD_RESERVED4_SFT 20 + #define CQ_RES_UD_SRC_QP_HIGH_MASK 0xff000000UL + #define CQ_RES_UD_SRC_QP_HIGH_SFT 24 +}; + +/* Responder RawEth and QP1 CQE (32 bytes) */ +struct cq_res_raweth_qp1 { + __le16 length; + #define CQ_RES_RAWETH_QP1_LENGTH_MASK 0x3fffUL + #define CQ_RES_RAWETH_QP1_LENGTH_SFT 0 + #define CQ_RES_RAWETH_QP1_RESERVED2_MASK 0xc000UL + #define CQ_RES_RAWETH_QP1_RESERVED2_SFT 14 + __le16 raweth_qp1_flags; + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ERROR 0x1UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_MASK 0x3eUL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_SFT 1 + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_MASK 0x3c0UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_SFT 6 + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN (0x0UL << 6) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_IP (0x1UL << 6) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_TCP (0x2UL << 6) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_UDP (0x3UL << 6) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_FCOE (0x4UL << 6) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE (0x5UL << 6) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ICMP (0x7UL << 6) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP \ + (0x8UL << 6) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP \ + (0x9UL << 6) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_LAST \ + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_MASK 0x3ffUL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_SFT 0 + #define CQ_RES_RAWETH_QP1_RESERVED6_MASK 0xfc00UL + #define CQ_RES_RAWETH_QP1_RESERVED6_SFT 10 + __le16 raweth_qp1_errors; + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_MASK 0xfUL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_SFT 0 + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_IP_CS_ERROR 0x10UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_L4_CS_ERROR 0x20UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_IP_CS_ERROR 0x40UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_L4_CS_ERROR 0x80UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_CRC_ERROR 0x100UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK 0xe00UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT 9 + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR \ + (0x0UL << 9) + #define \ + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION \ + (0x1UL << 9) + #define \ + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN \ + (0x2UL << 9) + #define \ + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR \ + (0x3UL << 9) + #define \ + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR \ + (0x4UL << 9) + #define \ + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR \ + (0x5UL << 9) + #define \ + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL \ + (0x6UL << 9) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST \ + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_MASK 0xf000UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_SFT 12 + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR \ + (0x0UL << 12) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION \ + (0x1UL << 12) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN \ + (0x2UL << 12) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL \ + (0x3UL << 12) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR \ + (0x4UL << 12) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR \ + (0x5UL << 12) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN \ + (0x6UL << 12) + #define \ + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL\ + (0x7UL << 12) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN \ + (0x8UL << 12) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_LAST \ + CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN + __le16 raweth_qp1_cfa_code; + __le64 qp_handle; + __le32 raweth_qp1_flags2; + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC 0x1UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC 0x2UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_IP_CS_CALC 0x4UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_L4_CS_CALC 0x8UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_MASK 0xf0UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_SFT 4 + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_NONE \ + (0x0UL << 4) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN \ + (0x1UL << 4) + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_LAST\ + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE 0x100UL + __le32 raweth_qp1_metadata; + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK 0xfffUL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_SFT 0 + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_DE 0x1000UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK 0xe000UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT 13 + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK 0xffff0000UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT 16 + u8 cqe_type_toggle; + #define CQ_RES_RAWETH_QP1_TOGGLE 0x1UL + #define CQ_RES_RAWETH_QP1_CQE_TYPE_MASK 0x1eUL + #define CQ_RES_RAWETH_QP1_CQE_TYPE_SFT 1 + #define CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1 (0x3UL << 1) + #define CQ_RES_RAWETH_QP1_RESERVED3_MASK 0xe0UL + #define CQ_RES_RAWETH_QP1_RESERVED3_SFT 5 + u8 status; + #define CQ_RES_RAWETH_QP1_STATUS_OK 0x0UL + #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR 0x1UL + #define CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR 0x2UL + #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR 0x3UL + #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR 0x4UL + #define CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR 0x5UL + #define CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR 0x7UL + #define CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR 0x8UL + __le16 flags; + #define CQ_RES_RAWETH_QP1_FLAGS_SRQ 0x1UL + #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_RQ 0x0UL + #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ 0x1UL + #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_LAST \ + CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ + __le32 raweth_qp1_payload_offset_srq_or_rq_wr_id; + #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK 0xfffffUL + #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_SFT 0 + #define CQ_RES_RAWETH_QP1_RESERVED4_MASK 0xf00000UL + #define CQ_RES_RAWETH_QP1_RESERVED4_SFT 20 + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_MASK 0xff000000UL + #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_SFT 24 +}; + +/* Terminal CQE (32 bytes) */ +struct cq_terminal { + __le64 qp_handle; + __le16 sq_cons_idx; + __le16 rq_cons_idx; + __le32 reserved32_1; + __le64 reserved64_3; + u8 cqe_type_toggle; + #define CQ_TERMINAL_TOGGLE 0x1UL + #define CQ_TERMINAL_CQE_TYPE_MASK 0x1eUL + #define CQ_TERMINAL_CQE_TYPE_SFT 1 + #define CQ_TERMINAL_CQE_TYPE_TERMINAL (0xeUL << 1) + #define CQ_TERMINAL_RESERVED3_MASK 0xe0UL + #define CQ_TERMINAL_RESERVED3_SFT 5 + u8 status; + #define CQ_TERMINAL_STATUS_OK 0x0UL + __le16 reserved16; + __le32 reserved32_2; +}; + +/* Cutoff CQE (32 bytes) */ +struct cq_cutoff { + __le64 reserved64_1; + __le64 reserved64_2; + __le64 reserved64_3; + u8 cqe_type_toggle; + #define CQ_CUTOFF_TOGGLE 0x1UL + #define CQ_CUTOFF_CQE_TYPE_MASK 0x1eUL + #define CQ_CUTOFF_CQE_TYPE_SFT 1 + #define CQ_CUTOFF_CQE_TYPE_CUT_OFF (0xfUL << 1) + #define CQ_CUTOFF_RESERVED3_MASK 0xe0UL + #define CQ_CUTOFF_RESERVED3_SFT 5 + u8 status; + #define CQ_CUTOFF_STATUS_OK 0x0UL + __le16 reserved16; + __le32 reserved32; +}; + +/* Notification Queue (NQ) Structures */ +/* Base NQ Record (16 bytes) */ +struct nq_base { + __le16 info10_type; + #define NQ_BASE_TYPE_MASK 0x3fUL + #define NQ_BASE_TYPE_SFT 0 + #define NQ_BASE_TYPE_CQ_NOTIFICATION 0x30UL + #define NQ_BASE_TYPE_SRQ_EVENT 0x32UL + #define NQ_BASE_TYPE_DBQ_EVENT 0x34UL + #define NQ_BASE_TYPE_QP_EVENT 0x38UL + #define NQ_BASE_TYPE_FUNC_EVENT 0x3aUL + #define NQ_BASE_INFO10_MASK 0xffc0UL + #define NQ_BASE_INFO10_SFT 6 + __le16 info16; + __le32 info32; + __le32 info63_v[2]; + #define NQ_BASE_V 0x1UL + #define NQ_BASE_INFO63_MASK 0xfffffffeUL + #define NQ_BASE_INFO63_SFT 1 +}; + +/* Completion Queue Notification (16 bytes) */ +struct nq_cn { + __le16 type; + #define NQ_CN_TYPE_MASK 0x3fUL + #define NQ_CN_TYPE_SFT 0 + #define NQ_CN_TYPE_CQ_NOTIFICATION 0x30UL + #define NQ_CN_RESERVED9_MASK 0xffc0UL + #define NQ_CN_RESERVED9_SFT 6 + __le16 reserved16; + __le32 cq_handle_low; + __le32 v; + #define NQ_CN_V 0x1UL + #define NQ_CN_RESERVED31_MASK 0xfffffffeUL + #define NQ_CN_RESERVED31_SFT 1 + __le32 cq_handle_high; +}; + +/* SRQ Event Notification (16 bytes) */ +struct nq_srq_event { + u8 type; + #define NQ_SRQ_EVENT_TYPE_MASK 0x3fUL + #define NQ_SRQ_EVENT_TYPE_SFT 0 + #define NQ_SRQ_EVENT_TYPE_SRQ_EVENT 0x32UL + #define NQ_SRQ_EVENT_RESERVED1_MASK 0xc0UL + #define NQ_SRQ_EVENT_RESERVED1_SFT 6 + u8 event; + #define NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT 0x1UL + __le16 reserved16; + __le32 srq_handle_low; + __le32 v; + #define NQ_SRQ_EVENT_V 0x1UL + #define NQ_SRQ_EVENT_RESERVED31_MASK 0xfffffffeUL + #define NQ_SRQ_EVENT_RESERVED31_SFT 1 + __le32 srq_handle_high; +}; + +/* DBQ Async Event Notification (16 bytes) */ +struct nq_dbq_event { + u8 type; + #define NQ_DBQ_EVENT_TYPE_MASK 0x3fUL + #define NQ_DBQ_EVENT_TYPE_SFT 0 + #define NQ_DBQ_EVENT_TYPE_DBQ_EVENT 0x34UL + #define NQ_DBQ_EVENT_RESERVED1_MASK 0xc0UL + #define NQ_DBQ_EVENT_RESERVED1_SFT 6 + u8 event; + #define NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT 0x1UL + __le16 db_pfid; + #define NQ_DBQ_EVENT_DB_PFID_MASK 0xfUL + #define NQ_DBQ_EVENT_DB_PFID_SFT 0 + #define NQ_DBQ_EVENT_RESERVED12_MASK 0xfff0UL + #define NQ_DBQ_EVENT_RESERVED12_SFT 4 + __le32 db_dpi; + #define NQ_DBQ_EVENT_DB_DPI_MASK 0xfffffUL + #define NQ_DBQ_EVENT_DB_DPI_SFT 0 + #define NQ_DBQ_EVENT_RESERVED12_2_MASK 0xfff00000UL + #define NQ_DBQ_EVENT_RESERVED12_2_SFT 20 + __le32 v; + #define NQ_DBQ_EVENT_V 0x1UL + #define NQ_DBQ_EVENT_RESERVED32_MASK 0xfffffffeUL + #define NQ_DBQ_EVENT_RESERVED32_SFT 1 + __le32 db_type_db_xid; + #define NQ_DBQ_EVENT_DB_XID_MASK 0xfffffUL + #define NQ_DBQ_EVENT_DB_XID_SFT 0 + #define NQ_DBQ_EVENT_RESERVED8_MASK 0xff00000UL + #define NQ_DBQ_EVENT_RESERVED8_SFT 20 + #define NQ_DBQ_EVENT_DB_TYPE_MASK 0xf0000000UL + #define NQ_DBQ_EVENT_DB_TYPE_SFT 28 +}; + +/* Read Request/Response Queue Structures */ +/* Input Read Request Queue (IRRQ) Message (32 bytes) */ +struct xrrq_irrq { + __le16 credits_type; + #define XRRQ_IRRQ_TYPE 0x1UL + #define XRRQ_IRRQ_TYPE_READ_REQ 0x0UL + #define XRRQ_IRRQ_TYPE_ATOMIC_REQ 0x1UL + #define XRRQ_IRRQ_RESERVED10_MASK 0x7feUL + #define XRRQ_IRRQ_RESERVED10_SFT 1 + #define XRRQ_IRRQ_CREDITS_MASK 0xf800UL + #define XRRQ_IRRQ_CREDITS_SFT 11 + __le16 reserved16; + __le32 reserved32; + __le32 psn; + #define XRRQ_IRRQ_PSN_MASK 0xffffffUL + #define XRRQ_IRRQ_PSN_SFT 0 + #define XRRQ_IRRQ_RESERVED8_1_MASK 0xff000000UL + #define XRRQ_IRRQ_RESERVED8_1_SFT 24 + __le32 msn; + #define XRRQ_IRRQ_MSN_MASK 0xffffffUL + #define XRRQ_IRRQ_MSN_SFT 0 + #define XRRQ_IRRQ_RESERVED8_2_MASK 0xff000000UL + #define XRRQ_IRRQ_RESERVED8_2_SFT 24 + __le64 va_or_atomic_result; + __le32 rdma_r_key; + __le32 length; +}; + +/* Output Read Request Queue (ORRQ) Message (32 bytes) */ +struct xrrq_orrq { + __le16 num_sges_type; + #define XRRQ_ORRQ_TYPE 0x1UL + #define XRRQ_ORRQ_TYPE_READ_REQ 0x0UL + #define XRRQ_ORRQ_TYPE_ATOMIC_REQ 0x1UL + #define XRRQ_ORRQ_RESERVED10_MASK 0x7feUL + #define XRRQ_ORRQ_RESERVED10_SFT 1 + #define XRRQ_ORRQ_NUM_SGES_MASK 0xf800UL + #define XRRQ_ORRQ_NUM_SGES_SFT 11 + __le16 reserved16; + __le32 length; + __le32 psn; + #define XRRQ_ORRQ_PSN_MASK 0xffffffUL + #define XRRQ_ORRQ_PSN_SFT 0 + #define XRRQ_ORRQ_RESERVED8_1_MASK 0xff000000UL + #define XRRQ_ORRQ_RESERVED8_1_SFT 24 + __le32 end_psn; + #define XRRQ_ORRQ_END_PSN_MASK 0xffffffUL + #define XRRQ_ORRQ_END_PSN_SFT 0 + #define XRRQ_ORRQ_RESERVED8_2_MASK 0xff000000UL + #define XRRQ_ORRQ_RESERVED8_2_SFT 24 + __le64 first_sge_phy_or_sing_sge_va; + __le32 single_sge_l_key; + __le32 single_sge_size; +}; + +/* Page Buffer List Memory Structures (PBL) */ +/* Page Table Entry (PTE) (8 bytes) */ +struct ptu_pte { + __le32 page_next_to_last_last_valid[2]; + #define PTU_PTE_VALID 0x1UL + #define PTU_PTE_LAST 0x2UL + #define PTU_PTE_NEXT_TO_LAST 0x4UL + #define PTU_PTE_PAGE_MASK 0xfffff000UL + #define PTU_PTE_PAGE_SFT 12 +}; + +/* Page Directory Entry (PDE) (8 bytes) */ +struct ptu_pde { + __le32 page_valid[2]; + #define PTU_PDE_VALID 0x1UL + #define PTU_PDE_PAGE_MASK 0xfffff000UL + #define PTU_PDE_PAGE_SFT 12 +}; + +/* RoCE Fastpath Host Structures */ +/* Command Queue (CMDQ) Interface */ +/* Init CMDQ (16 bytes) */ +struct cmdq_init { + __le64 cmdq_pbl; + __le16 cmdq_size_cmdq_lvl; + #define CMDQ_INIT_CMDQ_LVL_MASK 0x3UL + #define CMDQ_INIT_CMDQ_LVL_SFT 0 + #define CMDQ_INIT_CMDQ_SIZE_MASK 0xfffcUL + #define CMDQ_INIT_CMDQ_SIZE_SFT 2 + __le16 creq_ring_id; + __le32 prod_idx; +}; + +/* Update CMDQ producer index (16 bytes) */ +struct cmdq_update { + __le64 reserved64; + __le32 reserved32; + __le32 prod_idx; +}; + +/* CMDQ common header structure (16 bytes) */ +struct cmdq_base { + u8 opcode; + #define CMDQ_BASE_OPCODE_CREATE_QP 0x1UL + #define CMDQ_BASE_OPCODE_DESTROY_QP 0x2UL + #define CMDQ_BASE_OPCODE_MODIFY_QP 0x3UL + #define CMDQ_BASE_OPCODE_QUERY_QP 0x4UL + #define CMDQ_BASE_OPCODE_CREATE_SRQ 0x5UL + #define CMDQ_BASE_OPCODE_DESTROY_SRQ 0x6UL + #define CMDQ_BASE_OPCODE_QUERY_SRQ 0x8UL + #define CMDQ_BASE_OPCODE_CREATE_CQ 0x9UL + #define CMDQ_BASE_OPCODE_DESTROY_CQ 0xaUL + #define CMDQ_BASE_OPCODE_RESIZE_CQ 0xcUL + #define CMDQ_BASE_OPCODE_ALLOCATE_MRW 0xdUL + #define CMDQ_BASE_OPCODE_DEALLOCATE_KEY 0xeUL + #define CMDQ_BASE_OPCODE_REGISTER_MR 0xfUL + #define CMDQ_BASE_OPCODE_DEREGISTER_MR 0x10UL + #define CMDQ_BASE_OPCODE_ADD_GID 0x11UL + #define CMDQ_BASE_OPCODE_DELETE_GID 0x12UL + #define CMDQ_BASE_OPCODE_MODIFY_GID 0x17UL + #define CMDQ_BASE_OPCODE_QUERY_GID 0x18UL + #define CMDQ_BASE_OPCODE_CREATE_QP1 0x13UL + #define CMDQ_BASE_OPCODE_DESTROY_QP1 0x14UL + #define CMDQ_BASE_OPCODE_CREATE_AH 0x15UL + #define CMDQ_BASE_OPCODE_DESTROY_AH 0x16UL + #define CMDQ_BASE_OPCODE_INITIALIZE_FW 0x80UL + #define CMDQ_BASE_OPCODE_DEINITIALIZE_FW 0x81UL + #define CMDQ_BASE_OPCODE_STOP_FUNC 0x82UL + #define CMDQ_BASE_OPCODE_QUERY_FUNC 0x83UL + #define CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES 0x84UL + #define CMDQ_BASE_OPCODE_READ_CONTEXT 0x85UL + #define CMDQ_BASE_OPCODE_VF_BACKCHANNEL_REQUEST 0x86UL + #define CMDQ_BASE_OPCODE_READ_VF_MEMORY 0x87UL + #define CMDQ_BASE_OPCODE_COMPLETE_VF_REQUEST 0x88UL + #define CMDQ_BASE_OPCODE_EXTEND_CONTEXT_ARRRAY 0x89UL + #define CMDQ_BASE_OPCODE_MAP_TC_TO_COS 0x8aUL + #define CMDQ_BASE_OPCODE_QUERY_VERSION 0x8bUL + #define CMDQ_BASE_OPCODE_MODIFY_CC 0x8cUL + #define CMDQ_BASE_OPCODE_QUERY_CC 0x8dUL + #define CMDQ_BASE_OPCODE_QUERY_ROCE_STATS 0x8eUL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; +}; + +/* Create QP command (96 bytes) */ +struct cmdq_create_qp { + u8 opcode; + #define CMDQ_CREATE_QP_OPCODE_CREATE_QP 0x1UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le64 qp_handle; + __le32 qp_flags; + #define CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED 0x1UL + #define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION 0x2UL + #define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL + #define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED 0x8UL + u8 type; + #define CMDQ_CREATE_QP_TYPE_RC 0x2UL + #define CMDQ_CREATE_QP_TYPE_UD 0x4UL + #define CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE 0x6UL + u8 sq_pg_size_sq_lvl; + #define CMDQ_CREATE_QP_SQ_LVL_MASK 0xfUL + #define CMDQ_CREATE_QP_SQ_LVL_SFT 0 + #define CMDQ_CREATE_QP_SQ_LVL_LVL_0 0x0UL + #define CMDQ_CREATE_QP_SQ_LVL_LVL_1 0x1UL + #define CMDQ_CREATE_QP_SQ_LVL_LVL_2 0x2UL + #define CMDQ_CREATE_QP_SQ_PG_SIZE_MASK 0xf0UL + #define CMDQ_CREATE_QP_SQ_PG_SIZE_SFT 4 + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K (0x0UL << 4) + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K (0x1UL << 4) + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K (0x2UL << 4) + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M (0x3UL << 4) + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M (0x4UL << 4) + #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G (0x5UL << 4) + u8 rq_pg_size_rq_lvl; + #define CMDQ_CREATE_QP_RQ_LVL_MASK 0xfUL + #define CMDQ_CREATE_QP_RQ_LVL_SFT 0 + #define CMDQ_CREATE_QP_RQ_LVL_LVL_0 0x0UL + #define CMDQ_CREATE_QP_RQ_LVL_LVL_1 0x1UL + #define CMDQ_CREATE_QP_RQ_LVL_LVL_2 0x2UL + #define CMDQ_CREATE_QP_RQ_PG_SIZE_MASK 0xf0UL + #define CMDQ_CREATE_QP_RQ_PG_SIZE_SFT 4 + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K (0x0UL << 4) + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K (0x1UL << 4) + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K (0x2UL << 4) + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M (0x3UL << 4) + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M (0x4UL << 4) + #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G (0x5UL << 4) + u8 unused_0; + __le32 dpi; + __le32 sq_size; + __le32 rq_size; + __le16 sq_fwo_sq_sge; + #define CMDQ_CREATE_QP_SQ_SGE_MASK 0xfUL + #define CMDQ_CREATE_QP_SQ_SGE_SFT 0 + #define CMDQ_CREATE_QP_SQ_FWO_MASK 0xfff0UL + #define CMDQ_CREATE_QP_SQ_FWO_SFT 4 + __le16 rq_fwo_rq_sge; + #define CMDQ_CREATE_QP_RQ_SGE_MASK 0xfUL + #define CMDQ_CREATE_QP_RQ_SGE_SFT 0 + #define CMDQ_CREATE_QP_RQ_FWO_MASK 0xfff0UL + #define CMDQ_CREATE_QP_RQ_FWO_SFT 4 + __le32 scq_cid; + __le32 rcq_cid; + __le32 srq_cid; + __le32 pd_id; + __le64 sq_pbl; + __le64 rq_pbl; + __le64 irrq_addr; + __le64 orrq_addr; +}; + +/* Destroy QP command (24 bytes) */ +struct cmdq_destroy_qp { + u8 opcode; + #define CMDQ_DESTROY_QP_OPCODE_DESTROY_QP 0x2UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 qp_cid; + __le32 unused_0; +}; + +/* Modify QP command (112 bytes) */ +struct cmdq_modify_qp { + u8 opcode; + #define CMDQ_MODIFY_QP_OPCODE_MODIFY_QP 0x3UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 modify_mask; + #define CMDQ_MODIFY_QP_MODIFY_MASK_STATE 0x1UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY 0x2UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS 0x4UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_PKEY 0x8UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_QKEY 0x10UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_DGID 0x20UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL 0x40UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX 0x80UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT 0x100UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS 0x200UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC 0x400UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU 0x1000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT 0x2000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT 0x4000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY 0x8000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN 0x10000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC 0x20000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER 0x40000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN 0x80000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC 0x100000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE 0x200000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE 0x400000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE 0x800000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE 0x1000000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA 0x2000000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID 0x4000000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC 0x8000000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID 0x10000000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC 0x20000000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN 0x40000000UL + #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP 0x80000000UL + __le32 qp_cid; + u8 network_type_en_sqd_async_notify_new_state; + #define CMDQ_MODIFY_QP_NEW_STATE_MASK 0xfUL + #define CMDQ_MODIFY_QP_NEW_STATE_SFT 0 + #define CMDQ_MODIFY_QP_NEW_STATE_RESET 0x0UL + #define CMDQ_MODIFY_QP_NEW_STATE_INIT 0x1UL + #define CMDQ_MODIFY_QP_NEW_STATE_RTR 0x2UL + #define CMDQ_MODIFY_QP_NEW_STATE_RTS 0x3UL + #define CMDQ_MODIFY_QP_NEW_STATE_SQD 0x4UL + #define CMDQ_MODIFY_QP_NEW_STATE_SQE 0x5UL + #define CMDQ_MODIFY_QP_NEW_STATE_ERR 0x6UL + #define CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY 0x10UL + #define CMDQ_MODIFY_QP_NETWORK_TYPE_MASK 0xc0UL + #define CMDQ_MODIFY_QP_NETWORK_TYPE_SFT 6 + #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1 (0x0UL << 6) + #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 (0x2UL << 6) + #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6 (0x3UL << 6) + u8 access; + #define CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE 0x1UL + #define CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE 0x2UL + #define CMDQ_MODIFY_QP_ACCESS_REMOTE_READ 0x4UL + #define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC 0x8UL + __le16 pkey; + __le32 qkey; + __le32 dgid[4]; + __le32 flow_label; + __le16 sgid_index; + u8 hop_limit; + u8 traffic_class; + __le16 dest_mac[3]; + u8 tos_dscp_tos_ecn; + #define CMDQ_MODIFY_QP_TOS_ECN_MASK 0x3UL + #define CMDQ_MODIFY_QP_TOS_ECN_SFT 0 + #define CMDQ_MODIFY_QP_TOS_DSCP_MASK 0xfcUL + #define CMDQ_MODIFY_QP_TOS_DSCP_SFT 2 + u8 path_mtu; + #define CMDQ_MODIFY_QP_PATH_MTU_MASK 0xf0UL + #define CMDQ_MODIFY_QP_PATH_MTU_SFT 4 + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_256 (0x0UL << 4) + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_512 (0x1UL << 4) + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_1024 (0x2UL << 4) + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_2048 (0x3UL << 4) + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_4096 (0x4UL << 4) + #define CMDQ_MODIFY_QP_PATH_MTU_MTU_8192 (0x5UL << 4) + u8 timeout; + u8 retry_cnt; + u8 rnr_retry; + u8 min_rnr_timer; + __le32 rq_psn; + __le32 sq_psn; + u8 max_rd_atomic; + u8 max_dest_rd_atomic; + __le16 enable_cc; + #define CMDQ_MODIFY_QP_ENABLE_CC 0x1UL + __le32 sq_size; + __le32 rq_size; + __le16 sq_sge; + __le16 rq_sge; + __le32 max_inline_data; + __le32 dest_qp_id; + __le32 unused_3; + __le16 src_mac[3]; + __le16 vlan_pcp_vlan_dei_vlan_id; + #define CMDQ_MODIFY_QP_VLAN_ID_MASK 0xfffUL + #define CMDQ_MODIFY_QP_VLAN_ID_SFT 0 + #define CMDQ_MODIFY_QP_VLAN_DEI 0x1000UL + #define CMDQ_MODIFY_QP_VLAN_PCP_MASK 0xe000UL + #define CMDQ_MODIFY_QP_VLAN_PCP_SFT 13 +}; + +/* Query QP command (24 bytes) */ +struct cmdq_query_qp { + u8 opcode; + #define CMDQ_QUERY_QP_OPCODE_QUERY_QP 0x4UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 qp_cid; + __le32 unused_0; +}; + +/* Create SRQ command (48 bytes) */ +struct cmdq_create_srq { + u8 opcode; + #define CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ 0x5UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le64 srq_handle; + __le16 pg_size_lvl; + #define CMDQ_CREATE_SRQ_LVL_MASK 0x3UL + #define CMDQ_CREATE_SRQ_LVL_SFT 0 + #define CMDQ_CREATE_SRQ_LVL_LVL_0 0x0UL + #define CMDQ_CREATE_SRQ_LVL_LVL_1 0x1UL + #define CMDQ_CREATE_SRQ_LVL_LVL_2 0x2UL + #define CMDQ_CREATE_SRQ_PG_SIZE_MASK 0x1cUL + #define CMDQ_CREATE_SRQ_PG_SIZE_SFT 2 + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_4K (0x0UL << 2) + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8K (0x1UL << 2) + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_64K (0x2UL << 2) + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_2M (0x3UL << 2) + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8M (0x4UL << 2) + #define CMDQ_CREATE_SRQ_PG_SIZE_PG_1G (0x5UL << 2) + __le16 eventq_id; + #define CMDQ_CREATE_SRQ_EVENTQ_ID_MASK 0xfffUL + #define CMDQ_CREATE_SRQ_EVENTQ_ID_SFT 0 + __le16 srq_size; + __le16 srq_fwo; + __le32 dpi; + __le32 pd_id; + __le64 pbl; +}; + +/* Destroy SRQ command (24 bytes) */ +struct cmdq_destroy_srq { + u8 opcode; + #define CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ 0x6UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 srq_cid; + __le32 unused_0; +}; + +/* Query SRQ command (24 bytes) */ +struct cmdq_query_srq { + u8 opcode; + #define CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ 0x8UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 srq_cid; + __le32 unused_0; +}; + +/* Create CQ command (48 bytes) */ +struct cmdq_create_cq { + u8 opcode; + #define CMDQ_CREATE_CQ_OPCODE_CREATE_CQ 0x9UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le64 cq_handle; + __le32 pg_size_lvl; + #define CMDQ_CREATE_CQ_LVL_MASK 0x3UL + #define CMDQ_CREATE_CQ_LVL_SFT 0 + #define CMDQ_CREATE_CQ_LVL_LVL_0 0x0UL + #define CMDQ_CREATE_CQ_LVL_LVL_1 0x1UL + #define CMDQ_CREATE_CQ_LVL_LVL_2 0x2UL + #define CMDQ_CREATE_CQ_PG_SIZE_MASK 0x1cUL + #define CMDQ_CREATE_CQ_PG_SIZE_SFT 2 + #define CMDQ_CREATE_CQ_PG_SIZE_PG_4K (0x0UL << 2) + #define CMDQ_CREATE_CQ_PG_SIZE_PG_8K (0x1UL << 2) + #define CMDQ_CREATE_CQ_PG_SIZE_PG_64K (0x2UL << 2) + #define CMDQ_CREATE_CQ_PG_SIZE_PG_2M (0x3UL << 2) + #define CMDQ_CREATE_CQ_PG_SIZE_PG_8M (0x4UL << 2) + #define CMDQ_CREATE_CQ_PG_SIZE_PG_1G (0x5UL << 2) + __le32 cq_fco_cnq_id; + #define CMDQ_CREATE_CQ_CNQ_ID_MASK 0xfffUL + #define CMDQ_CREATE_CQ_CNQ_ID_SFT 0 + #define CMDQ_CREATE_CQ_CQ_FCO_MASK 0xfffff000UL + #define CMDQ_CREATE_CQ_CQ_FCO_SFT 12 + __le32 dpi; + __le32 cq_size; + __le64 pbl; +}; + +/* Destroy CQ command (24 bytes) */ +struct cmdq_destroy_cq { + u8 opcode; + #define CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ 0xaUL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 cq_cid; + __le32 unused_0; +}; + +/* Resize CQ command (40 bytes) */ +struct cmdq_resize_cq { + u8 opcode; + #define CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ 0xcUL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 cq_cid; + __le32 new_cq_size_pg_size_lvl; + #define CMDQ_RESIZE_CQ_LVL_MASK 0x3UL + #define CMDQ_RESIZE_CQ_LVL_SFT 0 + #define CMDQ_RESIZE_CQ_LVL_LVL_0 0x0UL + #define CMDQ_RESIZE_CQ_LVL_LVL_1 0x1UL + #define CMDQ_RESIZE_CQ_LVL_LVL_2 0x2UL + #define CMDQ_RESIZE_CQ_PG_SIZE_MASK 0x1cUL + #define CMDQ_RESIZE_CQ_PG_SIZE_SFT 2 + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_4K (0x0UL << 2) + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8K (0x1UL << 2) + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_64K (0x2UL << 2) + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_2M (0x3UL << 2) + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8M (0x4UL << 2) + #define CMDQ_RESIZE_CQ_PG_SIZE_PG_1G (0x5UL << 2) + #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK 0x1fffe0UL + #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT 5 + __le64 new_pbl; + __le32 new_cq_fco; + __le32 unused_2; +}; + +/* Allocate MRW command (32 bytes) */ +struct cmdq_allocate_mrw { + u8 opcode; + #define CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW 0xdUL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le64 mrw_handle; + u8 mrw_flags; + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MASK 0xfUL + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_SFT 0 + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR 0x0UL + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR 0x1UL + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 0x2UL + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A 0x3UL + #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B 0x4UL + u8 access; + #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_MASK 0x1fUL + #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_SFT 0 + #define CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY 0x20UL + __le16 unused_1; + __le32 pd_id; +}; + +/* De-allocate key command (24 bytes) */ +struct cmdq_deallocate_key { + u8 opcode; + #define CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY 0xeUL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + u8 mrw_flags; + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MASK 0xfUL + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_SFT 0 + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MR 0x0UL + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_PMR 0x1UL + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE1 0x2UL + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2A 0x3UL + #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B 0x4UL + u8 unused_1[3]; + __le32 key; +}; + +/* Register MR command (48 bytes) */ +struct cmdq_register_mr { + u8 opcode; + #define CMDQ_REGISTER_MR_OPCODE_REGISTER_MR 0xfUL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + u8 log2_pg_size_lvl; + #define CMDQ_REGISTER_MR_LVL_MASK 0x3UL + #define CMDQ_REGISTER_MR_LVL_SFT 0 + #define CMDQ_REGISTER_MR_LVL_LVL_0 0x0UL + #define CMDQ_REGISTER_MR_LVL_LVL_1 0x1UL + #define CMDQ_REGISTER_MR_LVL_LVL_2 0x2UL + #define CMDQ_REGISTER_MR_LVL_LAST CMDQ_REGISTER_MR_LVL_LVL_2 + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK 0x7cUL + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT 2 + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4K (0xcUL << 2) + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_8K (0xdUL << 2) + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_64K (0x10UL << 2) + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_256K (0x12UL << 2) + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1M (0x14UL << 2) + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_2M (0x15UL << 2) + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4M (0x16UL << 2) + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G (0x1eUL << 2) + #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_LAST \ + CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G + #define CMDQ_REGISTER_MR_UNUSED1 0x80UL + u8 access; + #define CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE 0x1UL + #define CMDQ_REGISTER_MR_ACCESS_REMOTE_READ 0x2UL + #define CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE 0x4UL + #define CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC 0x8UL + #define CMDQ_REGISTER_MR_ACCESS_MW_BIND 0x10UL + #define CMDQ_REGISTER_MR_ACCESS_ZERO_BASED 0x20UL + __le16 log2_pbl_pg_size; + #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK 0x1fUL + #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT 0 + #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K 0xcUL + #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K 0xdUL + #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K 0x10UL + #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K 0x12UL + #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M 0x14UL + #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M 0x15UL + #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M 0x16UL + #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G 0x1eUL + #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_LAST \ + CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G + #define CMDQ_REGISTER_MR_UNUSED11_MASK 0xffe0UL + #define CMDQ_REGISTER_MR_UNUSED11_SFT 5 + __le32 key; + __le64 pbl; + __le64 va; + __le64 mr_size; +}; + +/* Deregister MR command (24 bytes) */ +struct cmdq_deregister_mr { + u8 opcode; + #define CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR 0x10UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 lkey; + __le32 unused_0; +}; + +/* Add GID command (48 bytes) */ +struct cmdq_add_gid { + u8 opcode; + #define CMDQ_ADD_GID_OPCODE_ADD_GID 0x11UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __be32 gid[4]; + __be16 src_mac[3]; + __le16 vlan; + #define CMDQ_ADD_GID_VLAN_VLAN_ID_MASK 0xfffUL + #define CMDQ_ADD_GID_VLAN_VLAN_ID_SFT 0 + #define CMDQ_ADD_GID_VLAN_TPID_MASK 0x7000UL + #define CMDQ_ADD_GID_VLAN_TPID_SFT 12 + #define CMDQ_ADD_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12) + #define CMDQ_ADD_GID_VLAN_TPID_TPID_8100 (0x1UL << 12) + #define CMDQ_ADD_GID_VLAN_TPID_TPID_9100 (0x2UL << 12) + #define CMDQ_ADD_GID_VLAN_TPID_TPID_9200 (0x3UL << 12) + #define CMDQ_ADD_GID_VLAN_TPID_TPID_9300 (0x4UL << 12) + #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12) + #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12) + #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12) + #define CMDQ_ADD_GID_VLAN_TPID_LAST CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 + #define CMDQ_ADD_GID_VLAN_VLAN_EN 0x8000UL + __le16 ipid; + __le16 stats_ctx; + #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL + #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_SFT 0 + #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL + __le32 unused_0; +}; + +/* Delete GID command (24 bytes) */ +struct cmdq_delete_gid { + u8 opcode; + #define CMDQ_DELETE_GID_OPCODE_DELETE_GID 0x12UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le16 gid_index; + __le16 unused_0; + __le32 unused_1; +}; + +/* Modify GID command (48 bytes) */ +struct cmdq_modify_gid { + u8 opcode; + #define CMDQ_MODIFY_GID_OPCODE_MODIFY_GID 0x17UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __be32 gid[4]; + __be16 src_mac[3]; + __le16 vlan; + #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK 0xfffUL + #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT 0 + #define CMDQ_MODIFY_GID_VLAN_TPID_MASK 0x7000UL + #define CMDQ_MODIFY_GID_VLAN_TPID_SFT 12 + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_88A8 (0x0UL << 12) + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_8100 (0x1UL << 12) + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9100 (0x2UL << 12) + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9200 (0x3UL << 12) + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9300 (0x4UL << 12) + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG1 (0x5UL << 12) + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG2 (0x6UL << 12) + #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 (0x7UL << 12) + #define CMDQ_MODIFY_GID_VLAN_TPID_LAST \ + CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 + #define CMDQ_MODIFY_GID_VLAN_VLAN_EN 0x8000UL + __le16 ipid; + __le16 gid_index; + __le16 stats_ctx; + #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_MASK 0x7fffUL + #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_SFT 0 + #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_VALID 0x8000UL + __le16 unused_0; +}; + +/* Query GID command (24 bytes) */ +struct cmdq_query_gid { + u8 opcode; + #define CMDQ_QUERY_GID_OPCODE_QUERY_GID 0x18UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le16 gid_index; + __le16 unused_0; + __le32 unused_1; +}; + +/* Create QP1 command (80 bytes) */ +struct cmdq_create_qp1 { + u8 opcode; + #define CMDQ_CREATE_QP1_OPCODE_CREATE_QP1 0x13UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le64 qp_handle; + __le32 qp_flags; + #define CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED 0x1UL + #define CMDQ_CREATE_QP1_QP_FLAGS_FORCE_COMPLETION 0x2UL + #define CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE 0x4UL + u8 type; + #define CMDQ_CREATE_QP1_TYPE_GSI 0x1UL + u8 sq_pg_size_sq_lvl; + #define CMDQ_CREATE_QP1_SQ_LVL_MASK 0xfUL + #define CMDQ_CREATE_QP1_SQ_LVL_SFT 0 + #define CMDQ_CREATE_QP1_SQ_LVL_LVL_0 0x0UL + #define CMDQ_CREATE_QP1_SQ_LVL_LVL_1 0x1UL + #define CMDQ_CREATE_QP1_SQ_LVL_LVL_2 0x2UL + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_MASK 0xf0UL + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT 4 + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K (0x0UL << 4) + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K (0x1UL << 4) + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K (0x2UL << 4) + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M (0x3UL << 4) + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M (0x4UL << 4) + #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G (0x5UL << 4) + u8 rq_pg_size_rq_lvl; + #define CMDQ_CREATE_QP1_RQ_LVL_MASK 0xfUL + #define CMDQ_CREATE_QP1_RQ_LVL_SFT 0 + #define CMDQ_CREATE_QP1_RQ_LVL_LVL_0 0x0UL + #define CMDQ_CREATE_QP1_RQ_LVL_LVL_1 0x1UL + #define CMDQ_CREATE_QP1_RQ_LVL_LVL_2 0x2UL + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_MASK 0xf0UL + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT 4 + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K (0x0UL << 4) + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K (0x1UL << 4) + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K (0x2UL << 4) + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M (0x3UL << 4) + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M (0x4UL << 4) + #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G (0x5UL << 4) + u8 unused_0; + __le32 dpi; + __le32 sq_size; + __le32 rq_size; + __le16 sq_fwo_sq_sge; + #define CMDQ_CREATE_QP1_SQ_SGE_MASK 0xfUL + #define CMDQ_CREATE_QP1_SQ_SGE_SFT 0 + #define CMDQ_CREATE_QP1_SQ_FWO_MASK 0xfff0UL + #define CMDQ_CREATE_QP1_SQ_FWO_SFT 4 + __le16 rq_fwo_rq_sge; + #define CMDQ_CREATE_QP1_RQ_SGE_MASK 0xfUL + #define CMDQ_CREATE_QP1_RQ_SGE_SFT 0 + #define CMDQ_CREATE_QP1_RQ_FWO_MASK 0xfff0UL + #define CMDQ_CREATE_QP1_RQ_FWO_SFT 4 + __le32 scq_cid; + __le32 rcq_cid; + __le32 srq_cid; + __le32 pd_id; + __le64 sq_pbl; + __le64 rq_pbl; +}; + +/* Destroy QP1 command (24 bytes) */ +struct cmdq_destroy_qp1 { + u8 opcode; + #define CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1 0x14UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 qp1_cid; + __le32 unused_0; +}; + +/* Create AH command (64 bytes) */ +struct cmdq_create_ah { + u8 opcode; + #define CMDQ_CREATE_AH_OPCODE_CREATE_AH 0x15UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le64 ah_handle; + __le32 dgid[4]; + u8 type; + #define CMDQ_CREATE_AH_TYPE_V1 0x0UL + #define CMDQ_CREATE_AH_TYPE_V2IPV4 0x2UL + #define CMDQ_CREATE_AH_TYPE_V2IPV6 0x3UL + u8 hop_limit; + __le16 sgid_index; + __le32 dest_vlan_id_flow_label; + #define CMDQ_CREATE_AH_FLOW_LABEL_MASK 0xfffffUL + #define CMDQ_CREATE_AH_FLOW_LABEL_SFT 0 + #define CMDQ_CREATE_AH_DEST_VLAN_ID_MASK 0xfff00000UL + #define CMDQ_CREATE_AH_DEST_VLAN_ID_SFT 20 + __le32 pd_id; + __le32 unused_0; + __le16 dest_mac[3]; + u8 traffic_class; + u8 unused_1; +}; + +/* Destroy AH command (24 bytes) */ +struct cmdq_destroy_ah { + u8 opcode; + #define CMDQ_DESTROY_AH_OPCODE_DESTROY_AH 0x16UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 ah_cid; + __le32 unused_0; +}; + +/* Initialize Firmware command (112 bytes) */ +struct cmdq_initialize_fw { + u8 opcode; + #define CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW 0x80UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + u8 qpc_pg_size_qpc_lvl; + #define CMDQ_INITIALIZE_FW_QPC_LVL_MASK 0xfUL + #define CMDQ_INITIALIZE_FW_QPC_LVL_SFT 0 + #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_0 0x0UL + #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_1 0x1UL + #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2 0x2UL + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_MASK 0xf0UL + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT 4 + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K (0x0UL << 4) + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K (0x1UL << 4) + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K (0x2UL << 4) + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M (0x3UL << 4) + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M (0x4UL << 4) + #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G (0x5UL << 4) + u8 mrw_pg_size_mrw_lvl; + #define CMDQ_INITIALIZE_FW_MRW_LVL_MASK 0xfUL + #define CMDQ_INITIALIZE_FW_MRW_LVL_SFT 0 + #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_0 0x0UL + #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_1 0x1UL + #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2 0x2UL + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_MASK 0xf0UL + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT 4 + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_4K (0x0UL << 4) + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8K (0x1UL << 4) + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_64K (0x2UL << 4) + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_2M (0x3UL << 4) + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8M (0x4UL << 4) + #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G (0x5UL << 4) + u8 srq_pg_size_srq_lvl; + #define CMDQ_INITIALIZE_FW_SRQ_LVL_MASK 0xfUL + #define CMDQ_INITIALIZE_FW_SRQ_LVL_SFT 0 + #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_0 0x0UL + #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_1 0x1UL + #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2 0x2UL + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_MASK 0xf0UL + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT 4 + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_4K (0x0UL << 4) + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8K (0x1UL << 4) + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_64K (0x2UL << 4) + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_2M (0x3UL << 4) + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8M (0x4UL << 4) + #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G (0x5UL << 4) + u8 cq_pg_size_cq_lvl; + #define CMDQ_INITIALIZE_FW_CQ_LVL_MASK 0xfUL + #define CMDQ_INITIALIZE_FW_CQ_LVL_SFT 0 + #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_0 0x0UL + #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_1 0x1UL + #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2 0x2UL + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_MASK 0xf0UL + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT 4 + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_4K (0x0UL << 4) + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8K (0x1UL << 4) + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_64K (0x2UL << 4) + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_2M (0x3UL << 4) + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8M (0x4UL << 4) + #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G (0x5UL << 4) + u8 tqm_pg_size_tqm_lvl; + #define CMDQ_INITIALIZE_FW_TQM_LVL_MASK 0xfUL + #define CMDQ_INITIALIZE_FW_TQM_LVL_SFT 0 + #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_0 0x0UL + #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_1 0x1UL + #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2 0x2UL + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_MASK 0xf0UL + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT 4 + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_4K (0x0UL << 4) + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8K (0x1UL << 4) + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_64K (0x2UL << 4) + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_2M (0x3UL << 4) + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8M (0x4UL << 4) + #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G (0x5UL << 4) + u8 tim_pg_size_tim_lvl; + #define CMDQ_INITIALIZE_FW_TIM_LVL_MASK 0xfUL + #define CMDQ_INITIALIZE_FW_TIM_LVL_SFT 0 + #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_0 0x0UL + #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_1 0x1UL + #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2 0x2UL + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_MASK 0xf0UL + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT 4 + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_4K (0x0UL << 4) + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8K (0x1UL << 4) + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_64K (0x2UL << 4) + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M (0x3UL << 4) + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M (0x4UL << 4) + #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G (0x5UL << 4) + /* This value is (log-base-2-of-DBR-page-size - 12). + * 0 for 4KB. HW supported values are enumerated below. + */ + __le16 log2_dbr_pg_size; + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_MASK 0xfUL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_SFT 0 + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4K 0x0UL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8K 0x1UL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16K 0x2UL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32K 0x3UL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64K 0x4UL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128K 0x5UL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_256K 0x6UL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_512K 0x7UL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_1M 0x8UL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_2M 0x9UL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4M 0xaUL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8M 0xbUL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16M 0xcUL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32M 0xdUL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64M 0xeUL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M 0xfUL + #define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_LAST \ + CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M + __le64 qpc_page_dir; + __le64 mrw_page_dir; + __le64 srq_page_dir; + __le64 cq_page_dir; + __le64 tqm_page_dir; + __le64 tim_page_dir; + __le32 number_of_qp; + __le32 number_of_mrw; + __le32 number_of_srq; + __le32 number_of_cq; + __le32 max_qp_per_vf; + __le32 max_mrw_per_vf; + __le32 max_srq_per_vf; + __le32 max_cq_per_vf; + __le32 max_gid_per_vf; + __le32 stat_ctx_id; +}; + +/* De-initialize Firmware command (16 bytes) */ +struct cmdq_deinitialize_fw { + u8 opcode; + #define CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW 0x81UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; +}; + +/* Stop function command (16 bytes) */ +struct cmdq_stop_func { + u8 opcode; + #define CMDQ_STOP_FUNC_OPCODE_STOP_FUNC 0x82UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; +}; + +/* Query function command (16 bytes) */ +struct cmdq_query_func { + u8 opcode; + #define CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC 0x83UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; +}; + +/* Set function resources command (16 bytes) */ +struct cmdq_set_func_resources { + u8 opcode; + #define CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES 0x84UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 number_of_qp; + __le32 number_of_mrw; + __le32 number_of_srq; + __le32 number_of_cq; + __le32 max_qp_per_vf; + __le32 max_mrw_per_vf; + __le32 max_srq_per_vf; + __le32 max_cq_per_vf; + __le32 max_gid_per_vf; + __le32 stat_ctx_id; +}; + +/* Read hardware resource context command (24 bytes) */ +struct cmdq_read_context { + u8 opcode; + #define CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT 0x85UL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le32 type_xid; + #define CMDQ_READ_CONTEXT_XID_MASK 0xffffffUL + #define CMDQ_READ_CONTEXT_XID_SFT 0 + #define CMDQ_READ_CONTEXT_TYPE_MASK 0xff000000UL + #define CMDQ_READ_CONTEXT_TYPE_SFT 24 + #define CMDQ_READ_CONTEXT_TYPE_QPC (0x0UL << 24) + #define CMDQ_READ_CONTEXT_TYPE_CQ (0x1UL << 24) + #define CMDQ_READ_CONTEXT_TYPE_MRW (0x2UL << 24) + #define CMDQ_READ_CONTEXT_TYPE_SRQ (0x3UL << 24) + __le32 unused_0; +}; + +/* Map TC to COS. Can only be issued from a PF (24 bytes) */ +struct cmdq_map_tc_to_cos { + u8 opcode; + #define CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS 0x8aUL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; + __le16 cos0; + #define CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE 0xffffUL + __le16 cos1; + #define CMDQ_MAP_TC_TO_COS_COS1_DISABLE 0x8000UL + #define CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE 0xffffUL + __le32 unused_0; +}; + +/* Query version command (16 bytes) */ +struct cmdq_query_version { + u8 opcode; + #define CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION 0x8bUL + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; +}; + +/* Command-Response Event Queue (CREQ) Structures */ +/* Base CREQ Record (16 bytes) */ +struct creq_base { + u8 type; + #define CREQ_BASE_TYPE_MASK 0x3fUL + #define CREQ_BASE_TYPE_SFT 0 + #define CREQ_BASE_TYPE_QP_EVENT 0x38UL + #define CREQ_BASE_TYPE_FUNC_EVENT 0x3aUL + #define CREQ_BASE_RESERVED2_MASK 0xc0UL + #define CREQ_BASE_RESERVED2_SFT 6 + u8 reserved56[7]; + u8 v; + #define CREQ_BASE_V 0x1UL + #define CREQ_BASE_RESERVED7_MASK 0xfeUL + #define CREQ_BASE_RESERVED7_SFT 1 + u8 event; + __le16 reserved48[3]; +}; + +/* RoCE Function Async Event Notification (16 bytes) */ +struct creq_func_event { + u8 type; + #define CREQ_FUNC_EVENT_TYPE_MASK 0x3fUL + #define CREQ_FUNC_EVENT_TYPE_SFT 0 + #define CREQ_FUNC_EVENT_TYPE_FUNC_EVENT 0x3aUL + #define CREQ_FUNC_EVENT_RESERVED2_MASK 0xc0UL + #define CREQ_FUNC_EVENT_RESERVED2_SFT 6 + u8 reserved56[7]; + u8 v; + #define CREQ_FUNC_EVENT_V 0x1UL + #define CREQ_FUNC_EVENT_RESERVED7_MASK 0xfeUL + #define CREQ_FUNC_EVENT_RESERVED7_SFT 1 + u8 event; + #define CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR 0x1UL + #define CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR 0x2UL + #define CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR 0x3UL + #define CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR 0x4UL + #define CREQ_FUNC_EVENT_EVENT_CQ_ERROR 0x5UL + #define CREQ_FUNC_EVENT_EVENT_TQM_ERROR 0x6UL + #define CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR 0x7UL + #define CREQ_FUNC_EVENT_EVENT_CFCS_ERROR 0x8UL + #define CREQ_FUNC_EVENT_EVENT_CFCC_ERROR 0x9UL + #define CREQ_FUNC_EVENT_EVENT_CFCM_ERROR 0xaUL + #define CREQ_FUNC_EVENT_EVENT_TIM_ERROR 0xbUL + #define CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST 0x80UL + #define CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED 0x81UL + __le16 reserved48[3]; +}; + +/* RoCE Slowpath Command Completion (16 bytes) */ +struct creq_qp_event { + u8 type; + #define CREQ_QP_EVENT_TYPE_MASK 0x3fUL + #define CREQ_QP_EVENT_TYPE_SFT 0 + #define CREQ_QP_EVENT_TYPE_QP_EVENT 0x38UL + #define CREQ_QP_EVENT_RESERVED2_MASK 0xc0UL + #define CREQ_QP_EVENT_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 reserved32; + u8 v; + #define CREQ_QP_EVENT_V 0x1UL + #define CREQ_QP_EVENT_RESERVED7_MASK 0xfeUL + #define CREQ_QP_EVENT_RESERVED7_SFT 1 + u8 event; + #define CREQ_QP_EVENT_EVENT_CREATE_QP 0x1UL + #define CREQ_QP_EVENT_EVENT_DESTROY_QP 0x2UL + #define CREQ_QP_EVENT_EVENT_MODIFY_QP 0x3UL + #define CREQ_QP_EVENT_EVENT_QUERY_QP 0x4UL + #define CREQ_QP_EVENT_EVENT_CREATE_SRQ 0x5UL + #define CREQ_QP_EVENT_EVENT_DESTROY_SRQ 0x6UL + #define CREQ_QP_EVENT_EVENT_QUERY_SRQ 0x8UL + #define CREQ_QP_EVENT_EVENT_CREATE_CQ 0x9UL + #define CREQ_QP_EVENT_EVENT_DESTROY_CQ 0xaUL + #define CREQ_QP_EVENT_EVENT_RESIZE_CQ 0xcUL + #define CREQ_QP_EVENT_EVENT_ALLOCATE_MRW 0xdUL + #define CREQ_QP_EVENT_EVENT_DEALLOCATE_KEY 0xeUL + #define CREQ_QP_EVENT_EVENT_REGISTER_MR 0xfUL + #define CREQ_QP_EVENT_EVENT_DEREGISTER_MR 0x10UL + #define CREQ_QP_EVENT_EVENT_ADD_GID 0x11UL + #define CREQ_QP_EVENT_EVENT_DELETE_GID 0x12UL + #define CREQ_QP_EVENT_EVENT_MODIFY_GID 0x17UL + #define CREQ_QP_EVENT_EVENT_QUERY_GID 0x18UL + #define CREQ_QP_EVENT_EVENT_CREATE_QP1 0x13UL + #define CREQ_QP_EVENT_EVENT_DESTROY_QP1 0x14UL + #define CREQ_QP_EVENT_EVENT_CREATE_AH 0x15UL + #define CREQ_QP_EVENT_EVENT_DESTROY_AH 0x16UL + #define CREQ_QP_EVENT_EVENT_INITIALIZE_FW 0x80UL + #define CREQ_QP_EVENT_EVENT_DEINITIALIZE_FW 0x81UL + #define CREQ_QP_EVENT_EVENT_STOP_FUNC 0x82UL + #define CREQ_QP_EVENT_EVENT_QUERY_FUNC 0x83UL + #define CREQ_QP_EVENT_EVENT_SET_FUNC_RESOURCES 0x84UL + #define CREQ_QP_EVENT_EVENT_MAP_TC_TO_COS 0x8aUL + #define CREQ_QP_EVENT_EVENT_QUERY_VERSION 0x8bUL + #define CREQ_QP_EVENT_EVENT_MODIFY_CC 0x8cUL + #define CREQ_QP_EVENT_EVENT_QUERY_CC 0x8dUL + #define CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION 0xc0UL + __le16 reserved48[3]; +}; + +/* Create QP command response (16 bytes) */ +struct creq_create_qp_resp { + u8 type; + #define CREQ_CREATE_QP_RESP_TYPE_MASK 0x3fUL + #define CREQ_CREATE_QP_RESP_TYPE_SFT 0 + #define CREQ_CREATE_QP_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_CREATE_QP_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_CREATE_QP_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_CREATE_QP_RESP_V 0x1UL + #define CREQ_CREATE_QP_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_CREATE_QP_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_CREATE_QP_RESP_EVENT_CREATE_QP 0x1UL + __le16 reserved48[3]; +}; + +/* Destroy QP command response (16 bytes) */ +struct creq_destroy_qp_resp { + u8 type; + #define CREQ_DESTROY_QP_RESP_TYPE_MASK 0x3fUL + #define CREQ_DESTROY_QP_RESP_TYPE_SFT 0 + #define CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_DESTROY_QP_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_DESTROY_QP_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_DESTROY_QP_RESP_V 0x1UL + #define CREQ_DESTROY_QP_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_DESTROY_QP_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP 0x2UL + __le16 reserved48[3]; +}; + +/* Modify QP command response (16 bytes) */ +struct creq_modify_qp_resp { + u8 type; + #define CREQ_MODIFY_QP_RESP_TYPE_MASK 0x3fUL + #define CREQ_MODIFY_QP_RESP_TYPE_SFT 0 + #define CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_MODIFY_QP_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_MODIFY_QP_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_MODIFY_QP_RESP_V 0x1UL + #define CREQ_MODIFY_QP_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_MODIFY_QP_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP 0x3UL + __le16 reserved48[3]; +}; + +/* cmdq_query_roce_stats (size:128b/16B) */ +struct cmdq_query_roce_stats { + u8 opcode; + #define CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS 0x8eUL + #define CMDQ_QUERY_ROCE_STATS_OPCODE_LAST \ + CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS + u8 cmd_size; + __le16 flags; + __le16 cookie; + u8 resp_size; + u8 reserved8; + __le64 resp_addr; +}; + +/* Query QP command response (16 bytes) */ +struct creq_query_qp_resp { + u8 type; + #define CREQ_QUERY_QP_RESP_TYPE_MASK 0x3fUL + #define CREQ_QUERY_QP_RESP_TYPE_SFT 0 + #define CREQ_QUERY_QP_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_QUERY_QP_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_QUERY_QP_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 size; + u8 v; + #define CREQ_QUERY_QP_RESP_V 0x1UL + #define CREQ_QUERY_QP_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_QUERY_QP_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_QUERY_QP_RESP_EVENT_QUERY_QP 0x4UL + __le16 reserved48[3]; +}; + +/* Query QP command response side buffer structure (104 bytes) */ +struct creq_query_qp_resp_sb { + u8 opcode; + #define CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP 0x4UL + u8 status; + __le16 cookie; + __le16 flags; + u8 resp_size; + u8 reserved8; + __le32 xid; + u8 en_sqd_async_notify_state; + #define CREQ_QUERY_QP_RESP_SB_STATE_MASK 0xfUL + #define CREQ_QUERY_QP_RESP_SB_STATE_SFT 0 + #define CREQ_QUERY_QP_RESP_SB_STATE_RESET 0x0UL + #define CREQ_QUERY_QP_RESP_SB_STATE_INIT 0x1UL + #define CREQ_QUERY_QP_RESP_SB_STATE_RTR 0x2UL + #define CREQ_QUERY_QP_RESP_SB_STATE_RTS 0x3UL + #define CREQ_QUERY_QP_RESP_SB_STATE_SQD 0x4UL + #define CREQ_QUERY_QP_RESP_SB_STATE_SQE 0x5UL + #define CREQ_QUERY_QP_RESP_SB_STATE_ERR 0x6UL + #define CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY 0x10UL + u8 access; + #define CREQ_QUERY_QP_RESP_SB_ACCESS_LOCAL_WRITE 0x1UL + #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_WRITE 0x2UL + #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_READ 0x4UL + #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC 0x8UL + __le16 pkey; + __le32 qkey; + __le32 reserved32; + __le32 dgid[4]; + __le32 flow_label; + __le16 sgid_index; + u8 hop_limit; + u8 traffic_class; + __le16 dest_mac[3]; + __le16 path_mtu_dest_vlan_id; + #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_MASK 0xfffUL + #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_SFT 0 + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK 0xf000UL + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT 12 + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_256 (0x0UL << 12) + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_512 (0x1UL << 12) + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_1024 (0x2UL << 12) + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_2048 (0x3UL << 12) + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_4096 (0x4UL << 12) + #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192 (0x5UL << 12) + u8 timeout; + u8 retry_cnt; + u8 rnr_retry; + u8 min_rnr_timer; + __le32 rq_psn; + __le32 sq_psn; + u8 max_rd_atomic; + u8 max_dest_rd_atomic; + u8 tos_dscp_tos_ecn; + #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_MASK 0x3UL + #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_SFT 0 + #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_MASK 0xfcUL + #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_SFT 2 + u8 enable_cc; + #define CREQ_QUERY_QP_RESP_SB_ENABLE_CC 0x1UL + #define CREQ_QUERY_QP_RESP_SB_RESERVED7_MASK 0xfeUL + #define CREQ_QUERY_QP_RESP_SB_RESERVED7_SFT 1 + __le32 sq_size; + __le32 rq_size; + __le16 sq_sge; + __le16 rq_sge; + __le32 max_inline_data; + __le32 dest_qp_id; + __le32 unused_1; + __le16 src_mac[3]; + __le16 vlan_pcp_vlan_dei_vlan_id; + #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK 0xfffUL + #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT 0 + #define CREQ_QUERY_QP_RESP_SB_VLAN_DEI 0x1000UL + #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_MASK 0xe000UL + #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_SFT 13 +}; + +/* Create SRQ command response (16 bytes) */ +struct creq_create_srq_resp { + u8 type; + #define CREQ_CREATE_SRQ_RESP_TYPE_MASK 0x3fUL + #define CREQ_CREATE_SRQ_RESP_TYPE_SFT 0 + #define CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_CREATE_SRQ_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_CREATE_SRQ_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_CREATE_SRQ_RESP_V 0x1UL + #define CREQ_CREATE_SRQ_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_CREATE_SRQ_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ 0x5UL + __le16 reserved48[3]; +}; + +/* Destroy SRQ command response (16 bytes) */ +struct creq_destroy_srq_resp { + u8 type; + #define CREQ_DESTROY_SRQ_RESP_TYPE_MASK 0x3fUL + #define CREQ_DESTROY_SRQ_RESP_TYPE_SFT 0 + #define CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_DESTROY_SRQ_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_DESTROY_SRQ_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_DESTROY_SRQ_RESP_V 0x1UL + #define CREQ_DESTROY_SRQ_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_DESTROY_SRQ_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ 0x6UL + __le16 enable_for_arm[3]; + #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_MASK 0x30000UL + #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_SFT 16 + #define CREQ_DESTROY_SRQ_RESP_RESERVED46_MASK 0xfffc0000UL + #define CREQ_DESTROY_SRQ_RESP_RESERVED46_SFT 18 +}; + +/* Query SRQ command response (16 bytes) */ +struct creq_query_srq_resp { + u8 type; + #define CREQ_QUERY_SRQ_RESP_TYPE_MASK 0x3fUL + #define CREQ_QUERY_SRQ_RESP_TYPE_SFT 0 + #define CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_QUERY_SRQ_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_QUERY_SRQ_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 size; + u8 v; + #define CREQ_QUERY_SRQ_RESP_V 0x1UL + #define CREQ_QUERY_SRQ_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_QUERY_SRQ_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ 0x8UL + __le16 reserved48[3]; +}; + +/* Query SRQ command response side buffer structure (24 bytes) */ +struct creq_query_srq_resp_sb { + u8 opcode; + #define CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ 0x8UL + u8 status; + __le16 cookie; + __le16 flags; + u8 resp_size; + u8 reserved8; + __le32 xid; + __le16 srq_limit; + __le16 reserved16; + __le32 data[4]; +}; + +/* Create CQ command Response (16 bytes) */ +struct creq_create_cq_resp { + u8 type; + #define CREQ_CREATE_CQ_RESP_TYPE_MASK 0x3fUL + #define CREQ_CREATE_CQ_RESP_TYPE_SFT 0 + #define CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_CREATE_CQ_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_CREATE_CQ_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_CREATE_CQ_RESP_V 0x1UL + #define CREQ_CREATE_CQ_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_CREATE_CQ_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ 0x9UL + __le16 reserved48[3]; +}; + +/* Destroy CQ command response (16 bytes) */ +struct creq_destroy_cq_resp { + u8 type; + #define CREQ_DESTROY_CQ_RESP_TYPE_MASK 0x3fUL + #define CREQ_DESTROY_CQ_RESP_TYPE_SFT 0 + #define CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_DESTROY_CQ_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_DESTROY_CQ_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_DESTROY_CQ_RESP_V 0x1UL + #define CREQ_DESTROY_CQ_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_DESTROY_CQ_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ 0xaUL + __le16 cq_arm_lvl; + #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_MASK 0x3UL + #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_SFT 0 + #define CREQ_DESTROY_CQ_RESP_RESERVED14_MASK 0xfffcUL + #define CREQ_DESTROY_CQ_RESP_RESERVED14_SFT 2 + __le16 total_cnq_events; + __le16 reserved16; +}; + +/* Resize CQ command response (16 bytes) */ +struct creq_resize_cq_resp { + u8 type; + #define CREQ_RESIZE_CQ_RESP_TYPE_MASK 0x3fUL + #define CREQ_RESIZE_CQ_RESP_TYPE_SFT 0 + #define CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_RESIZE_CQ_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_RESIZE_CQ_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_RESIZE_CQ_RESP_V 0x1UL + #define CREQ_RESIZE_CQ_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_RESIZE_CQ_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ 0xcUL + __le16 reserved48[3]; +}; + +/* Allocate MRW command response (16 bytes) */ +struct creq_allocate_mrw_resp { + u8 type; + #define CREQ_ALLOCATE_MRW_RESP_TYPE_MASK 0x3fUL + #define CREQ_ALLOCATE_MRW_RESP_TYPE_SFT 0 + #define CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_ALLOCATE_MRW_RESP_V 0x1UL + #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW 0xdUL + __le16 reserved48[3]; +}; + +/* De-allocate key command response (16 bytes) */ +struct creq_deallocate_key_resp { + u8 type; + #define CREQ_DEALLOCATE_KEY_RESP_TYPE_MASK 0x3fUL + #define CREQ_DEALLOCATE_KEY_RESP_TYPE_SFT 0 + #define CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_DEALLOCATE_KEY_RESP_V 0x1UL + #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY 0xeUL + __le16 reserved16; + __le32 bound_window_info; +}; + +/* Register MR command response (16 bytes) */ +struct creq_register_mr_resp { + u8 type; + #define CREQ_REGISTER_MR_RESP_TYPE_MASK 0x3fUL + #define CREQ_REGISTER_MR_RESP_TYPE_SFT 0 + #define CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_REGISTER_MR_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_REGISTER_MR_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_REGISTER_MR_RESP_V 0x1UL + #define CREQ_REGISTER_MR_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_REGISTER_MR_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR 0xfUL + __le16 reserved48[3]; +}; + +/* Deregister MR command response (16 bytes) */ +struct creq_deregister_mr_resp { + u8 type; + #define CREQ_DEREGISTER_MR_RESP_TYPE_MASK 0x3fUL + #define CREQ_DEREGISTER_MR_RESP_TYPE_SFT 0 + #define CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_DEREGISTER_MR_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_DEREGISTER_MR_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_DEREGISTER_MR_RESP_V 0x1UL + #define CREQ_DEREGISTER_MR_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_DEREGISTER_MR_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR 0x10UL + __le16 reserved16; + __le32 bound_windows; +}; + +/* Add GID command response (16 bytes) */ +struct creq_add_gid_resp { + u8 type; + #define CREQ_ADD_GID_RESP_TYPE_MASK 0x3fUL + #define CREQ_ADD_GID_RESP_TYPE_SFT 0 + #define CREQ_ADD_GID_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_ADD_GID_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_ADD_GID_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_ADD_GID_RESP_V 0x1UL + #define CREQ_ADD_GID_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_ADD_GID_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_ADD_GID_RESP_EVENT_ADD_GID 0x11UL + __le16 reserved48[3]; +}; + +/* Delete GID command response (16 bytes) */ +struct creq_delete_gid_resp { + u8 type; + #define CREQ_DELETE_GID_RESP_TYPE_MASK 0x3fUL + #define CREQ_DELETE_GID_RESP_TYPE_SFT 0 + #define CREQ_DELETE_GID_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_DELETE_GID_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_DELETE_GID_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_DELETE_GID_RESP_V 0x1UL + #define CREQ_DELETE_GID_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_DELETE_GID_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_DELETE_GID_RESP_EVENT_DELETE_GID 0x12UL + __le16 reserved48[3]; +}; + +/* Modify GID command response (16 bytes) */ +struct creq_modify_gid_resp { + u8 type; + #define CREQ_MODIFY_GID_RESP_TYPE_MASK 0x3fUL + #define CREQ_MODIFY_GID_RESP_TYPE_SFT 0 + #define CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_MODIFY_GID_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_MODIFY_GID_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_MODIFY_GID_RESP_V 0x1UL + #define CREQ_MODIFY_GID_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_MODIFY_GID_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_MODIFY_GID_RESP_EVENT_ADD_GID 0x11UL + __le16 reserved48[3]; +}; + +/* Query GID command response (16 bytes) */ +struct creq_query_gid_resp { + u8 type; + #define CREQ_QUERY_GID_RESP_TYPE_MASK 0x3fUL + #define CREQ_QUERY_GID_RESP_TYPE_SFT 0 + #define CREQ_QUERY_GID_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_QUERY_GID_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_QUERY_GID_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 size; + u8 v; + #define CREQ_QUERY_GID_RESP_V 0x1UL + #define CREQ_QUERY_GID_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_QUERY_GID_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_QUERY_GID_RESP_EVENT_QUERY_GID 0x18UL + __le16 reserved48[3]; +}; + +/* Query GID command response side buffer structure (40 bytes) */ +struct creq_query_gid_resp_sb { + u8 opcode; + #define CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID 0x18UL + u8 status; + __le16 cookie; + __le16 flags; + u8 resp_size; + u8 reserved8; + __le32 gid[4]; + __le16 src_mac[3]; + __le16 vlan; + #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_MASK 0xfffUL + #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_SFT 0 + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_MASK 0x7000UL + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_SFT 12 + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_88A8 (0x0UL << 12) + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_8100 (0x1UL << 12) + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9100 (0x2UL << 12) + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9200 (0x3UL << 12) + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9300 (0x4UL << 12) + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG1 (0x5UL << 12) + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG2 (0x6UL << 12) + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 (0x7UL << 12) + #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_LAST \ + CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 + #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN 0x8000UL + __le16 ipid; + __le16 gid_index; + __le32 unused_0; +}; + +/* Create QP1 command response (16 bytes) */ +struct creq_create_qp1_resp { + u8 type; + #define CREQ_CREATE_QP1_RESP_TYPE_MASK 0x3fUL + #define CREQ_CREATE_QP1_RESP_TYPE_SFT 0 + #define CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_CREATE_QP1_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_CREATE_QP1_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_CREATE_QP1_RESP_V 0x1UL + #define CREQ_CREATE_QP1_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_CREATE_QP1_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1 0x13UL + __le16 reserved48[3]; +}; + +/* Destroy QP1 command response (16 bytes) */ +struct creq_destroy_qp1_resp { + u8 type; + #define CREQ_DESTROY_QP1_RESP_TYPE_MASK 0x3fUL + #define CREQ_DESTROY_QP1_RESP_TYPE_SFT 0 + #define CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_DESTROY_QP1_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_DESTROY_QP1_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_DESTROY_QP1_RESP_V 0x1UL + #define CREQ_DESTROY_QP1_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_DESTROY_QP1_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1 0x14UL + __le16 reserved48[3]; +}; + +/* Create AH command response (16 bytes) */ +struct creq_create_ah_resp { + u8 type; + #define CREQ_CREATE_AH_RESP_TYPE_MASK 0x3fUL + #define CREQ_CREATE_AH_RESP_TYPE_SFT 0 + #define CREQ_CREATE_AH_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_CREATE_AH_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_CREATE_AH_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_CREATE_AH_RESP_V 0x1UL + #define CREQ_CREATE_AH_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_CREATE_AH_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_CREATE_AH_RESP_EVENT_CREATE_AH 0x15UL + __le16 reserved48[3]; +}; + +/* Destroy AH command response (16 bytes) */ +struct creq_destroy_ah_resp { + u8 type; + #define CREQ_DESTROY_AH_RESP_TYPE_MASK 0x3fUL + #define CREQ_DESTROY_AH_RESP_TYPE_SFT 0 + #define CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_DESTROY_AH_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_DESTROY_AH_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 xid; + u8 v; + #define CREQ_DESTROY_AH_RESP_V 0x1UL + #define CREQ_DESTROY_AH_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_DESTROY_AH_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH 0x16UL + __le16 reserved48[3]; +}; + +/* Initialize Firmware command response (16 bytes) */ +struct creq_initialize_fw_resp { + u8 type; + #define CREQ_INITIALIZE_FW_RESP_TYPE_MASK 0x3fUL + #define CREQ_INITIALIZE_FW_RESP_TYPE_SFT 0 + #define CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_INITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_INITIALIZE_FW_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 reserved32; + u8 v; + #define CREQ_INITIALIZE_FW_RESP_V 0x1UL + #define CREQ_INITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_INITIALIZE_FW_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW 0x80UL + __le16 reserved48[3]; +}; + +/* De-initialize Firmware command response (16 bytes) */ +struct creq_deinitialize_fw_resp { + u8 type; + #define CREQ_DEINITIALIZE_FW_RESP_TYPE_MASK 0x3fUL + #define CREQ_DEINITIALIZE_FW_RESP_TYPE_SFT 0 + #define CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 reserved32; + u8 v; + #define CREQ_DEINITIALIZE_FW_RESP_V 0x1UL + #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW 0x81UL + __le16 reserved48[3]; +}; + +/* Stop function command response (16 bytes) */ +struct creq_stop_func_resp { + u8 type; + #define CREQ_STOP_FUNC_RESP_TYPE_MASK 0x3fUL + #define CREQ_STOP_FUNC_RESP_TYPE_SFT 0 + #define CREQ_STOP_FUNC_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_STOP_FUNC_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_STOP_FUNC_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 reserved32; + u8 v; + #define CREQ_STOP_FUNC_RESP_V 0x1UL + #define CREQ_STOP_FUNC_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_STOP_FUNC_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_STOP_FUNC_RESP_EVENT_STOP_FUNC 0x82UL + __le16 reserved48[3]; +}; + +/* Query function command response (16 bytes) */ +struct creq_query_func_resp { + u8 type; + #define CREQ_QUERY_FUNC_RESP_TYPE_MASK 0x3fUL + #define CREQ_QUERY_FUNC_RESP_TYPE_SFT 0 + #define CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_QUERY_FUNC_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_QUERY_FUNC_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 size; + u8 v; + #define CREQ_QUERY_FUNC_RESP_V 0x1UL + #define CREQ_QUERY_FUNC_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_QUERY_FUNC_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC 0x83UL + __le16 reserved48[3]; +}; + +/* Query function command response side buffer structure (88 bytes) */ +struct creq_query_func_resp_sb { + u8 opcode; + #define CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC 0x83UL + u8 status; + __le16 cookie; + __le16 flags; + u8 resp_size; + u8 reserved8; + __le64 max_mr_size; + __le32 max_qp; + __le16 max_qp_wr; + __le16 dev_cap_flags; + #define CREQ_QUERY_FUNC_RESP_SB_DEV_CAP_FLAGS_RESIZE_QP 0x1UL + __le32 max_cq; + __le32 max_cqe; + __le32 max_pd; + u8 max_sge; + u8 max_srq_sge; + u8 max_qp_rd_atom; + u8 max_qp_init_rd_atom; + __le32 max_mr; + __le32 max_mw; + __le32 max_raw_eth_qp; + __le32 max_ah; + __le32 max_fmr; + __le32 max_srq_wr; + __le32 max_pkeys; + __le32 max_inline_data; + u8 max_map_per_fmr; + u8 l2_db_space_size; + __le16 max_srq; + __le32 max_gid; + __le32 tqm_alloc_reqs[12]; +}; + +/* Set resources command response (16 bytes) */ +struct creq_set_func_resources_resp { + u8 type; + #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_MASK 0x3fUL + #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_SFT 0 + #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 reserved32; + u8 v; + #define CREQ_SET_FUNC_RESOURCES_RESP_V 0x1UL + #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES 0x84UL + __le16 reserved48[3]; +}; + +/* Map TC to COS response (16 bytes) */ +struct creq_map_tc_to_cos_resp { + u8 type; + #define CREQ_MAP_TC_TO_COS_RESP_TYPE_MASK 0x3fUL + #define CREQ_MAP_TC_TO_COS_RESP_TYPE_SFT 0 + #define CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 reserved32; + u8 v; + #define CREQ_MAP_TC_TO_COS_RESP_V 0x1UL + #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS 0x8aUL + __le16 reserved48[3]; +}; + +/* Query version response (16 bytes) */ +struct creq_query_version_resp { + u8 type; + #define CREQ_QUERY_VERSION_RESP_TYPE_MASK 0x3fUL + #define CREQ_QUERY_VERSION_RESP_TYPE_SFT 0 + #define CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_QUERY_VERSION_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_QUERY_VERSION_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + u8 fw_maj; + u8 fw_minor; + u8 fw_bld; + u8 fw_rsvd; + u8 v; + #define CREQ_QUERY_VERSION_RESP_V 0x1UL + #define CREQ_QUERY_VERSION_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_QUERY_VERSION_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION 0x8bUL + __le16 reserved16; + u8 intf_maj; + u8 intf_minor; + u8 intf_bld; + u8 intf_rsvd; +}; + +/* Modify congestion control command response (16 bytes) */ +struct creq_modify_cc_resp { + u8 type; + #define CREQ_MODIFY_CC_RESP_TYPE_MASK 0x3fUL + #define CREQ_MODIFY_CC_RESP_TYPE_SFT 0 + #define CREQ_MODIFY_CC_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_MODIFY_CC_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_MODIFY_CC_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 reserved32; + u8 v; + #define CREQ_MODIFY_CC_RESP_V 0x1UL + #define CREQ_MODIFY_CC_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_MODIFY_CC_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_MODIFY_CC_RESP_EVENT_MODIFY_CC 0x8cUL + __le16 reserved48[3]; +}; + +/* Query congestion control command response (16 bytes) */ +struct creq_query_cc_resp { + u8 type; + #define CREQ_QUERY_CC_RESP_TYPE_MASK 0x3fUL + #define CREQ_QUERY_CC_RESP_TYPE_SFT 0 + #define CREQ_QUERY_CC_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_QUERY_CC_RESP_RESERVED2_MASK 0xc0UL + #define CREQ_QUERY_CC_RESP_RESERVED2_SFT 6 + u8 status; + __le16 cookie; + __le32 size; + u8 v; + #define CREQ_QUERY_CC_RESP_V 0x1UL + #define CREQ_QUERY_CC_RESP_RESERVED7_MASK 0xfeUL + #define CREQ_QUERY_CC_RESP_RESERVED7_SFT 1 + u8 event; + #define CREQ_QUERY_CC_RESP_EVENT_QUERY_CC 0x8dUL + __le16 reserved48[3]; +}; + +/* Query congestion control command response side buffer structure (32 bytes) */ +struct creq_query_cc_resp_sb { + u8 opcode; + #define CREQ_QUERY_CC_RESP_SB_OPCODE_QUERY_CC 0x8dUL + u8 status; + __le16 cookie; + __le16 flags; + u8 resp_size; + u8 reserved8; + u8 enable_cc; + #define CREQ_QUERY_CC_RESP_SB_ENABLE_CC 0x1UL + u8 g; + #define CREQ_QUERY_CC_RESP_SB_G_MASK 0x7UL + #define CREQ_QUERY_CC_RESP_SB_G_SFT 0 + u8 num_phases_per_state; + __le16 init_cr; + u8 unused_2; + __le16 unused_3; + u8 unused_4; + __le16 init_tr; + u8 tos_dscp_tos_ecn; + #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_MASK 0x3UL + #define CREQ_QUERY_CC_RESP_SB_TOS_ECN_SFT 0 + #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_MASK 0xfcUL + #define CREQ_QUERY_CC_RESP_SB_TOS_DSCP_SFT 2 + __le64 reserved64; + __le64 reserved64_1; +}; + +/* creq_query_roce_stats_resp (size:128b/16B) */ +struct creq_query_roce_stats_resp { + u8 type; + #define CREQ_QUERY_ROCE_STATS_RESP_TYPE_MASK 0x3fUL + #define CREQ_QUERY_ROCE_STATS_RESP_TYPE_SFT 0 + #define CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT 0x38UL + #define CREQ_QUERY_ROCE_STATS_RESP_TYPE_LAST \ + CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT + u8 status; + __le16 cookie; + __le32 size; + u8 v; + #define CREQ_QUERY_ROCE_STATS_RESP_V 0x1UL + u8 event; + #define CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS 0x8eUL + #define CREQ_QUERY_ROCE_STATS_RESP_EVENT_LAST \ + CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS + u8 reserved48[6]; +}; + +/* creq_query_roce_stats_resp_sb (size:2624b/328B) */ +struct creq_query_roce_stats_resp_sb { + u8 opcode; + #define CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS 0x8eUL + #define CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_LAST \ + CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS + u8 status; + __le16 cookie; + __le16 flags; + u8 resp_size; + u8 rsvd; + __le32 num_counters; + __le32 rsvd1; + __le64 to_retransmits; + __le64 seq_err_naks_rcvd; + __le64 max_retry_exceeded; + __le64 rnr_naks_rcvd; + __le64 missing_resp; + __le64 unrecoverable_err; + __le64 bad_resp_err; + __le64 local_qp_op_err; + __le64 local_protection_err; + __le64 mem_mgmt_op_err; + __le64 remote_invalid_req_err; + __le64 remote_access_err; + __le64 remote_op_err; + __le64 dup_req; + __le64 res_exceed_max; + __le64 res_length_mismatch; + __le64 res_exceeds_wqe; + __le64 res_opcode_err; + __le64 res_rx_invalid_rkey; + __le64 res_rx_domain_err; + __le64 res_rx_no_perm; + __le64 res_rx_range_err; + __le64 res_tx_invalid_rkey; + __le64 res_tx_domain_err; + __le64 res_tx_no_perm; + __le64 res_tx_range_err; + __le64 res_irrq_oflow; + __le64 res_unsup_opcode; + __le64 res_unaligned_atomic; + __le64 res_rem_inv_err; + __le64 res_mem_error; + __le64 res_srq_err; + __le64 res_cmp_err; + __le64 res_invalid_dup_rkey; + __le64 res_wqe_format_err; + __le64 res_cq_load_err; + __le64 res_srq_load_err; + __le64 res_tx_pci_err; + __le64 res_rx_pci_err; +}; + +/* QP error notification event (16 bytes) */ +struct creq_qp_error_notification { + u8 type; + #define CREQ_QP_ERROR_NOTIFICATION_TYPE_MASK 0x3fUL + #define CREQ_QP_ERROR_NOTIFICATION_TYPE_SFT 0 + #define CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT 0x38UL + #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_MASK 0xc0UL + #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_SFT 6 + u8 status; + u8 req_slow_path_state; + u8 req_err_state_reason; + __le32 xid; + u8 v; + #define CREQ_QP_ERROR_NOTIFICATION_V 0x1UL + #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_MASK 0xfeUL + #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_SFT 1 + u8 event; + #define CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION 0xc0UL + u8 res_slow_path_state; + u8 res_err_state_reason; + __le16 sq_cons_idx; + __le16 rq_cons_idx; +}; + +/* RoCE Slowpath HSI Specification 1.6.0 */ +#define ROCE_SP_HSI_VERSION_MAJOR 1 +#define ROCE_SP_HSI_VERSION_MINOR 6 +#define ROCE_SP_HSI_VERSION_UPDATE 0 + +#define ROCE_SP_HSI_VERSION_STR "1.6.0" +/* + * Following is the signature for ROCE_SP_HSI message field that indicates not + * applicable (All F's). Need to cast it the size of the field if needed. + */ +#define ROCE_SP_HSI_NA_SIGNATURE ((__le32)(-1)) +#endif /* __BNXT_RE_HSI_H__ */ -- cgit v1.2.3