diff options
Diffstat (limited to '')
23 files changed, 23093 insertions, 0 deletions
diff --git a/drivers/infiniband/hw/hns/Kconfig b/drivers/infiniband/hw/hns/Kconfig new file mode 100644 index 000000000..18d10ebf9 --- /dev/null +++ b/drivers/infiniband/hw/hns/Kconfig @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: GPL-2.0-only +config INFINIBAND_HNS + tristate "HNS RoCE Driver" + depends on NET_VENDOR_HISILICON + depends on ARM64 || (COMPILE_TEST && 64BIT) + depends on (HNS_DSAF && HNS_ENET) || HNS3 + help + This is a RoCE/RDMA driver for the Hisilicon RoCE engine. The engine + is used in Hisilicon Hip06 and more further ICT SoC based on + platform device. + + To compile HIP06 or HIP08 driver as module, choose M here. + +config INFINIBAND_HNS_HIP06 + bool "Hisilicon Hip06 Family RoCE support" + depends on INFINIBAND_HNS && HNS && HNS_DSAF && HNS_ENET + depends on INFINIBAND_HNS=m || (HNS_DSAF=y && HNS_ENET=y) + help + RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip06 and + Hip07 SoC. These RoCE engines are platform devices. + + To compile this driver, choose Y here: if INFINIBAND_HNS is m, this + module will be called hns-roce-hw-v1 + +config INFINIBAND_HNS_HIP08 + bool "Hisilicon Hip08 Family RoCE support" + depends on INFINIBAND_HNS && PCI && HNS3 + depends on INFINIBAND_HNS=m || HNS3=y + help + RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip08 SoC. + The RoCE engine is a PCI device. + + To compile this driver, choose Y here: if INFINIBAND_HNS is m, this + module will be called hns-roce-hw-v2. diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile new file mode 100644 index 000000000..e105945b9 --- /dev/null +++ b/drivers/infiniband/hw/hns/Makefile @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Hisilicon RoCE drivers. +# + +ccflags-y := -I $(srctree)/drivers/net/ethernet/hisilicon/hns3 + +hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ + hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \ + hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o + +ifdef CONFIG_INFINIBAND_HNS_HIP06 +hns-roce-hw-v1-objs := hns_roce_hw_v1.o $(hns-roce-objs) +obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v1.o +endif + +ifdef CONFIG_INFINIBAND_HNS_HIP08 +hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs) +obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o +endif diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c new file mode 100644 index 000000000..174b19e39 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/platform_device.h> +#include <linux/pci.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include "hns_roce_device.h" + +static inline u16 get_ah_udp_sport(const struct rdma_ah_attr *ah_attr) +{ + u32 fl = ah_attr->grh.flow_label; + u16 sport; + + if (!fl) + sport = get_random_u32() % + (IB_ROCE_UDP_ENCAP_VALID_PORT_MAX + 1 - + IB_ROCE_UDP_ENCAP_VALID_PORT_MIN) + + IB_ROCE_UDP_ENCAP_VALID_PORT_MIN; + else + sport = rdma_flow_label_to_udp_sport(fl); + + return sport; +} + +int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, + struct ib_udata *udata) +{ + struct rdma_ah_attr *ah_attr = init_attr->ah_attr; + const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); + struct hns_roce_dev *hr_dev = to_hr_dev(ibah->device); + struct hns_roce_ah *ah = to_hr_ah(ibah); + int ret = 0; + + ah->av.port = rdma_ah_get_port_num(ah_attr); + ah->av.gid_index = grh->sgid_index; + + if (rdma_ah_get_static_rate(ah_attr)) + ah->av.stat_rate = IB_RATE_10_GBPS; + + ah->av.hop_limit = grh->hop_limit; + ah->av.flowlabel = grh->flow_label; + ah->av.udp_sport = get_ah_udp_sport(ah_attr); + ah->av.sl = rdma_ah_get_sl(ah_attr); + ah->av.tclass = get_tclass(grh); + + memcpy(ah->av.dgid, grh->dgid.raw, HNS_ROCE_GID_SIZE); + memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN); + + /* HIP08 needs to record vlan info in Address Vector */ + if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08) { + ret = rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr, + &ah->av.vlan_id, NULL); + if (ret) + return ret; + + ah->av.vlan_en = ah->av.vlan_id < VLAN_N_VID; + } + + return ret; +} + +int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) +{ + struct hns_roce_ah *ah = to_hr_ah(ibah); + + memset(ah_attr, 0, sizeof(*ah_attr)); + + rdma_ah_set_sl(ah_attr, ah->av.sl); + rdma_ah_set_port_num(ah_attr, ah->av.port); + rdma_ah_set_static_rate(ah_attr, ah->av.stat_rate); + rdma_ah_set_grh(ah_attr, NULL, ah->av.flowlabel, + ah->av.gid_index, ah->av.hop_limit, ah->av.tclass); + rdma_ah_set_dgid_raw(ah_attr, ah->av.dgid); + + return 0; +} diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c new file mode 100644 index 000000000..5b2baf89d --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -0,0 +1,294 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/platform_device.h> +#include <linux/vmalloc.h> +#include "hns_roce_device.h" +#include <rdma/ib_umem.h> + +int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj) +{ + int ret = 0; + + spin_lock(&bitmap->lock); + *obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); + if (*obj >= bitmap->max) { + bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) + & bitmap->mask; + *obj = find_first_zero_bit(bitmap->table, bitmap->max); + } + + if (*obj < bitmap->max) { + set_bit(*obj, bitmap->table); + bitmap->last = (*obj + 1); + if (bitmap->last == bitmap->max) + bitmap->last = 0; + *obj |= bitmap->top; + } else { + ret = -EINVAL; + } + + spin_unlock(&bitmap->lock); + + return ret; +} + +void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj, + int rr) +{ + hns_roce_bitmap_free_range(bitmap, obj, 1, rr); +} + +int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt, + int align, unsigned long *obj) +{ + int ret = 0; + int i; + + if (likely(cnt == 1 && align == 1)) + return hns_roce_bitmap_alloc(bitmap, obj); + + spin_lock(&bitmap->lock); + + *obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, + bitmap->last, cnt, align - 1); + if (*obj >= bitmap->max) { + bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) + & bitmap->mask; + *obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, 0, + cnt, align - 1); + } + + if (*obj < bitmap->max) { + for (i = 0; i < cnt; i++) + set_bit(*obj + i, bitmap->table); + + if (*obj == bitmap->last) { + bitmap->last = (*obj + cnt); + if (bitmap->last >= bitmap->max) + bitmap->last = 0; + } + *obj |= bitmap->top; + } else { + ret = -EINVAL; + } + + spin_unlock(&bitmap->lock); + + return ret; +} + +void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap, + unsigned long obj, int cnt, + int rr) +{ + int i; + + obj &= bitmap->max + bitmap->reserved_top - 1; + + spin_lock(&bitmap->lock); + for (i = 0; i < cnt; i++) + clear_bit(obj + i, bitmap->table); + + if (!rr) + bitmap->last = min(bitmap->last, obj); + bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) + & bitmap->mask; + spin_unlock(&bitmap->lock); +} + +int hns_roce_bitmap_init(struct hns_roce_bitmap *bitmap, u32 num, u32 mask, + u32 reserved_bot, u32 reserved_top) +{ + u32 i; + + if (num != roundup_pow_of_two(num)) + return -EINVAL; + + bitmap->last = 0; + bitmap->top = 0; + bitmap->max = num - reserved_top; + bitmap->mask = mask; + bitmap->reserved_top = reserved_top; + spin_lock_init(&bitmap->lock); + bitmap->table = kcalloc(BITS_TO_LONGS(bitmap->max), sizeof(long), + GFP_KERNEL); + if (!bitmap->table) + return -ENOMEM; + + for (i = 0; i < reserved_bot; ++i) + set_bit(i, bitmap->table); + + return 0; +} + +void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap) +{ + kfree(bitmap->table); +} + +void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf) +{ + struct device *dev = hr_dev->dev; + u32 size = buf->size; + int i; + + if (size == 0) + return; + + buf->size = 0; + + if (hns_roce_buf_is_direct(buf)) { + dma_free_coherent(dev, size, buf->direct.buf, buf->direct.map); + } else { + for (i = 0; i < buf->npages; ++i) + if (buf->page_list[i].buf) + dma_free_coherent(dev, 1 << buf->page_shift, + buf->page_list[i].buf, + buf->page_list[i].map); + kfree(buf->page_list); + buf->page_list = NULL; + } +} + +int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, + struct hns_roce_buf *buf, u32 page_shift) +{ + struct hns_roce_buf_list *buf_list; + struct device *dev = hr_dev->dev; + u32 page_size; + int i; + + /* The minimum shift of the page accessed by hw is HNS_HW_PAGE_SHIFT */ + buf->page_shift = max_t(int, HNS_HW_PAGE_SHIFT, page_shift); + + page_size = 1 << buf->page_shift; + buf->npages = DIV_ROUND_UP(size, page_size); + + /* required size is not bigger than one trunk size */ + if (size <= max_direct) { + buf->page_list = NULL; + buf->direct.buf = dma_alloc_coherent(dev, size, + &buf->direct.map, + GFP_KERNEL); + if (!buf->direct.buf) + return -ENOMEM; + } else { + buf_list = kcalloc(buf->npages, sizeof(*buf_list), GFP_KERNEL); + if (!buf_list) + return -ENOMEM; + + for (i = 0; i < buf->npages; i++) { + buf_list[i].buf = dma_alloc_coherent(dev, page_size, + &buf_list[i].map, + GFP_KERNEL); + if (!buf_list[i].buf) + break; + } + + if (i != buf->npages && i > 0) { + while (i-- > 0) + dma_free_coherent(dev, page_size, + buf_list[i].buf, + buf_list[i].map); + kfree(buf_list); + return -ENOMEM; + } + buf->page_list = buf_list; + } + buf->size = size; + + return 0; +} + +int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct hns_roce_buf *buf) +{ + int i, end; + int total; + + end = start + buf_cnt; + if (end > buf->npages) { + dev_err(hr_dev->dev, + "failed to check kmem bufs, end %d + %d total %u!\n", + start, buf_cnt, buf->npages); + return -EINVAL; + } + + total = 0; + for (i = start; i < end; i++) + bufs[total++] = hns_roce_buf_page(buf, i); + + return total; +} + +int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct ib_umem *umem, + unsigned int page_shift) +{ + struct ib_block_iter biter; + int total = 0; + int idx = 0; + u64 addr; + + if (page_shift < HNS_HW_PAGE_SHIFT) { + dev_err(hr_dev->dev, "failed to check umem page shift %u!\n", + page_shift); + return -EINVAL; + } + + /* convert system page cnt to hw page cnt */ + rdma_umem_for_each_dma_block(umem, &biter, 1 << page_shift) { + addr = rdma_block_iter_dma_address(&biter); + if (idx >= start) { + bufs[total++] = addr; + if (total >= buf_cnt) + goto done; + } + idx++; + } + +done: + return total; +} + +void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev) +{ + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) + hns_roce_cleanup_srq_table(hr_dev); + hns_roce_cleanup_qp_table(hr_dev); + hns_roce_cleanup_cq_table(hr_dev); + hns_roce_cleanup_mr_table(hr_dev); + hns_roce_cleanup_pd_table(hr_dev); + hns_roce_cleanup_uar_table(hr_dev); +} diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c new file mode 100644 index 000000000..c493d7644 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c @@ -0,0 +1,291 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/dmapool.h> +#include <linux/platform_device.h> +#include "hns_roce_common.h" +#include "hns_roce_device.h" +#include "hns_roce_cmd.h" + +#define CMD_POLL_TOKEN 0xffff +#define CMD_MAX_NUM 32 +#define CMD_TOKEN_MASK 0x1f + +static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, u32 in_modifier, + u8 op_modifier, u16 op, u16 token, + int event) +{ + struct hns_roce_cmdq *cmd = &hr_dev->cmd; + int ret; + + mutex_lock(&cmd->hcr_mutex); + ret = hr_dev->hw->post_mbox(hr_dev, in_param, out_param, in_modifier, + op_modifier, op, token, event); + mutex_unlock(&cmd->hcr_mutex); + + return ret; +} + +/* this should be called with "poll_sem" */ +static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, unsigned long in_modifier, + u8 op_modifier, u16 op, + unsigned long timeout) +{ + struct device *dev = hr_dev->dev; + int ret; + + ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param, + in_modifier, op_modifier, op, + CMD_POLL_TOKEN, 0); + if (ret) { + dev_err(dev, "[cmd_poll]hns_roce_cmd_mbox_post_hw failed\n"); + return ret; + } + + return hr_dev->hw->chk_mbox(hr_dev, timeout); +} + +static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, unsigned long in_modifier, + u8 op_modifier, u16 op, unsigned long timeout) +{ + int ret; + + down(&hr_dev->cmd.poll_sem); + ret = __hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param, in_modifier, + op_modifier, op, timeout); + up(&hr_dev->cmd.poll_sem); + + return ret; +} + +void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status, + u64 out_param) +{ + struct hns_roce_cmd_context *context = + &hr_dev->cmd.context[token % hr_dev->cmd.max_cmds]; + + if (token != context->token) + return; + + context->result = (status == HNS_ROCE_CMD_SUCCESS) ? 0 : (-EIO); + context->out_param = out_param; + complete(&context->done); +} + +/* this should be called with "use_events" */ +static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, unsigned long in_modifier, + u8 op_modifier, u16 op, + unsigned long timeout) +{ + struct hns_roce_cmdq *cmd = &hr_dev->cmd; + struct hns_roce_cmd_context *context; + struct device *dev = hr_dev->dev; + int ret; + + spin_lock(&cmd->context_lock); + WARN_ON(cmd->free_head < 0); + context = &cmd->context[cmd->free_head]; + context->token += cmd->token_mask + 1; + cmd->free_head = context->next; + spin_unlock(&cmd->context_lock); + + init_completion(&context->done); + + ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param, + in_modifier, op_modifier, op, + context->token, 1); + if (ret) + goto out; + + /* + * It is timeout when wait_for_completion_timeout return 0 + * The return value is the time limit set in advance + * how many seconds showing + */ + if (!wait_for_completion_timeout(&context->done, + msecs_to_jiffies(timeout))) { + dev_err(dev, "[cmd]wait_for_completion_timeout timeout\n"); + ret = -EBUSY; + goto out; + } + + ret = context->result; + if (ret) { + dev_err(dev, "[cmd]event mod cmd process error!err=%d\n", ret); + goto out; + } + +out: + spin_lock(&cmd->context_lock); + context->next = cmd->free_head; + cmd->free_head = context - cmd->context; + spin_unlock(&cmd->context_lock); + + return ret; +} + +static int hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, unsigned long in_modifier, + u8 op_modifier, u16 op, unsigned long timeout) +{ + int ret; + + down(&hr_dev->cmd.event_sem); + ret = __hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param, in_modifier, + op_modifier, op, timeout); + up(&hr_dev->cmd.event_sem); + + return ret; +} + +int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, + unsigned long in_modifier, u8 op_modifier, u16 op, + unsigned long timeout) +{ + int ret; + + if (hr_dev->hw->rst_prc_mbox) { + ret = hr_dev->hw->rst_prc_mbox(hr_dev); + if (ret == CMD_RST_PRC_SUCCESS) + return 0; + else if (ret == CMD_RST_PRC_EBUSY) + return -EBUSY; + } + + if (hr_dev->cmd.use_events) + ret = hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param, + in_modifier, op_modifier, op, + timeout); + else + ret = hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param, + in_modifier, op_modifier, op, + timeout); + + if (ret == CMD_RST_PRC_EBUSY) + return -EBUSY; + + if (ret && (hr_dev->hw->rst_prc_mbox && + hr_dev->hw->rst_prc_mbox(hr_dev) == CMD_RST_PRC_SUCCESS)) + return 0; + + return ret; +} + +int hns_roce_cmd_init(struct hns_roce_dev *hr_dev) +{ + struct device *dev = hr_dev->dev; + + mutex_init(&hr_dev->cmd.hcr_mutex); + sema_init(&hr_dev->cmd.poll_sem, 1); + hr_dev->cmd.use_events = 0; + hr_dev->cmd.max_cmds = CMD_MAX_NUM; + hr_dev->cmd.pool = dma_pool_create("hns_roce_cmd", dev, + HNS_ROCE_MAILBOX_SIZE, + HNS_ROCE_MAILBOX_SIZE, 0); + if (!hr_dev->cmd.pool) + return -ENOMEM; + + return 0; +} + +void hns_roce_cmd_cleanup(struct hns_roce_dev *hr_dev) +{ + dma_pool_destroy(hr_dev->cmd.pool); +} + +int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmdq *hr_cmd = &hr_dev->cmd; + int i; + + hr_cmd->context = + kcalloc(hr_cmd->max_cmds, sizeof(*hr_cmd->context), GFP_KERNEL); + if (!hr_cmd->context) + return -ENOMEM; + + for (i = 0; i < hr_cmd->max_cmds; ++i) { + hr_cmd->context[i].token = i; + hr_cmd->context[i].next = i + 1; + } + + hr_cmd->context[hr_cmd->max_cmds - 1].next = -1; + hr_cmd->free_head = 0; + + sema_init(&hr_cmd->event_sem, hr_cmd->max_cmds); + spin_lock_init(&hr_cmd->context_lock); + + hr_cmd->token_mask = CMD_TOKEN_MASK; + hr_cmd->use_events = 1; + + return 0; +} + +void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmdq *hr_cmd = &hr_dev->cmd; + + kfree(hr_cmd->context); + hr_cmd->use_events = 0; +} + +struct hns_roce_cmd_mailbox * +hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmd_mailbox *mailbox; + + mailbox = kmalloc(sizeof(*mailbox), GFP_KERNEL); + if (!mailbox) + return ERR_PTR(-ENOMEM); + + mailbox->buf = + dma_pool_alloc(hr_dev->cmd.pool, GFP_KERNEL, &mailbox->dma); + if (!mailbox->buf) { + kfree(mailbox); + return ERR_PTR(-ENOMEM); + } + + return mailbox; +} + +void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev, + struct hns_roce_cmd_mailbox *mailbox) +{ + if (!mailbox) + return; + + dma_pool_free(hr_dev->cmd.pool, mailbox->buf, mailbox->dma); + kfree(mailbox); +} diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h new file mode 100644 index 000000000..8e63b827f --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _HNS_ROCE_CMD_H +#define _HNS_ROCE_CMD_H + +#define HNS_ROCE_MAILBOX_SIZE 4096 +#define HNS_ROCE_CMD_TIMEOUT_MSECS 10000 + +enum { + /* QPC BT commands */ + HNS_ROCE_CMD_WRITE_QPC_BT0 = 0x0, + HNS_ROCE_CMD_WRITE_QPC_BT1 = 0x1, + HNS_ROCE_CMD_WRITE_QPC_BT2 = 0x2, + HNS_ROCE_CMD_READ_QPC_BT0 = 0x4, + HNS_ROCE_CMD_READ_QPC_BT1 = 0x5, + HNS_ROCE_CMD_READ_QPC_BT2 = 0x6, + HNS_ROCE_CMD_DESTROY_QPC_BT0 = 0x8, + HNS_ROCE_CMD_DESTROY_QPC_BT1 = 0x9, + HNS_ROCE_CMD_DESTROY_QPC_BT2 = 0xa, + + /* QPC operation */ + HNS_ROCE_CMD_MODIFY_QPC = 0x41, + HNS_ROCE_CMD_QUERY_QPC = 0x42, + + HNS_ROCE_CMD_MODIFY_CQC = 0x52, + HNS_ROCE_CMD_QUERY_CQC = 0x53, + /* CQC BT commands */ + HNS_ROCE_CMD_WRITE_CQC_BT0 = 0x10, + HNS_ROCE_CMD_WRITE_CQC_BT1 = 0x11, + HNS_ROCE_CMD_WRITE_CQC_BT2 = 0x12, + HNS_ROCE_CMD_READ_CQC_BT0 = 0x14, + HNS_ROCE_CMD_READ_CQC_BT1 = 0x15, + HNS_ROCE_CMD_READ_CQC_BT2 = 0x1b, + HNS_ROCE_CMD_DESTROY_CQC_BT0 = 0x18, + HNS_ROCE_CMD_DESTROY_CQC_BT1 = 0x19, + HNS_ROCE_CMD_DESTROY_CQC_BT2 = 0x1a, + + /* MPT BT commands */ + HNS_ROCE_CMD_WRITE_MPT_BT0 = 0x20, + HNS_ROCE_CMD_WRITE_MPT_BT1 = 0x21, + HNS_ROCE_CMD_WRITE_MPT_BT2 = 0x22, + HNS_ROCE_CMD_READ_MPT_BT0 = 0x24, + HNS_ROCE_CMD_READ_MPT_BT1 = 0x25, + HNS_ROCE_CMD_READ_MPT_BT2 = 0x26, + HNS_ROCE_CMD_DESTROY_MPT_BT0 = 0x28, + HNS_ROCE_CMD_DESTROY_MPT_BT1 = 0x29, + HNS_ROCE_CMD_DESTROY_MPT_BT2 = 0x2a, + + /* CQC TIMER commands */ + HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0 = 0x23, + HNS_ROCE_CMD_READ_CQC_TIMER_BT0 = 0x27, + + /* MPT commands */ + HNS_ROCE_CMD_QUERY_MPT = 0x62, + + /* SRQC BT commands */ + HNS_ROCE_CMD_WRITE_SRQC_BT0 = 0x30, + HNS_ROCE_CMD_WRITE_SRQC_BT1 = 0x31, + HNS_ROCE_CMD_WRITE_SRQC_BT2 = 0x32, + HNS_ROCE_CMD_READ_SRQC_BT0 = 0x34, + HNS_ROCE_CMD_READ_SRQC_BT1 = 0x35, + HNS_ROCE_CMD_READ_SRQC_BT2 = 0x36, + HNS_ROCE_CMD_DESTROY_SRQC_BT0 = 0x38, + HNS_ROCE_CMD_DESTROY_SRQC_BT1 = 0x39, + HNS_ROCE_CMD_DESTROY_SRQC_BT2 = 0x3a, + + /* QPC TIMER commands */ + HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0 = 0x33, + HNS_ROCE_CMD_READ_QPC_TIMER_BT0 = 0x37, + + /* EQC commands */ + HNS_ROCE_CMD_CREATE_AEQC = 0x80, + HNS_ROCE_CMD_MODIFY_AEQC = 0x81, + HNS_ROCE_CMD_QUERY_AEQC = 0x82, + HNS_ROCE_CMD_DESTROY_AEQC = 0x83, + HNS_ROCE_CMD_CREATE_CEQC = 0x90, + HNS_ROCE_CMD_MODIFY_CEQC = 0x91, + HNS_ROCE_CMD_QUERY_CEQC = 0x92, + HNS_ROCE_CMD_DESTROY_CEQC = 0x93, + + /* SCC CTX BT commands */ + HNS_ROCE_CMD_READ_SCCC_BT0 = 0xa4, + HNS_ROCE_CMD_WRITE_SCCC_BT0 = 0xa5, +}; + +enum { + /* TPT commands */ + HNS_ROCE_CMD_CREATE_MPT = 0xd, + HNS_ROCE_CMD_DESTROY_MPT = 0xf, + + /* CQ commands */ + HNS_ROCE_CMD_CREATE_CQC = 0x16, + HNS_ROCE_CMD_DESTROY_CQC = 0x17, + + /* QP/EE commands */ + HNS_ROCE_CMD_RST2INIT_QP = 0x19, + HNS_ROCE_CMD_INIT2RTR_QP = 0x1a, + HNS_ROCE_CMD_RTR2RTS_QP = 0x1b, + HNS_ROCE_CMD_RTS2RTS_QP = 0x1c, + HNS_ROCE_CMD_2ERR_QP = 0x1e, + HNS_ROCE_CMD_RTS2SQD_QP = 0x1f, + HNS_ROCE_CMD_SQD2RTS_QP = 0x20, + HNS_ROCE_CMD_2RST_QP = 0x21, + HNS_ROCE_CMD_QUERY_QP = 0x22, + HNS_ROCE_CMD_SQD2SQD_QP = 0x38, + HNS_ROCE_CMD_CREATE_SRQ = 0x70, + HNS_ROCE_CMD_MODIFY_SRQC = 0x72, + HNS_ROCE_CMD_QUERY_SRQC = 0x73, + HNS_ROCE_CMD_DESTROY_SRQ = 0x74, +}; + +int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, + unsigned long in_modifier, u8 op_modifier, u16 op, + unsigned long timeout); + +struct hns_roce_cmd_mailbox * +hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev); +void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev, + struct hns_roce_cmd_mailbox *mailbox); + +#endif /* _HNS_ROCE_CMD_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h new file mode 100644 index 000000000..f5669ff8c --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_common.h @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _HNS_ROCE_COMMON_H +#define _HNS_ROCE_COMMON_H + +#define roce_write(dev, reg, val) writel((val), (dev)->reg_base + (reg)) +#define roce_read(dev, reg) readl((dev)->reg_base + (reg)) +#define roce_raw_write(value, addr) \ + __raw_writel((__force u32)cpu_to_le32(value), (addr)) + +#define roce_get_field(origin, mask, shift) \ + (((le32_to_cpu(origin)) & (mask)) >> (shift)) + +#define roce_get_bit(origin, shift) \ + roce_get_field((origin), (1ul << (shift)), (shift)) + +#define roce_set_field(origin, mask, shift, val) \ + do { \ + (origin) &= ~cpu_to_le32(mask); \ + (origin) |= cpu_to_le32(((u32)(val) << (shift)) & (mask)); \ + } while (0) + +#define roce_set_bit(origin, shift, val) \ + roce_set_field((origin), (1ul << (shift)), (shift), (val)) + +#define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3 +#define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4 + +#define ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S 5 + +#define ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S 6 + +#define ROCEE_GLB_CFG_ROCEE_PORT_ST_S 10 +#define ROCEE_GLB_CFG_ROCEE_PORT_ST_M \ + (((1UL << 6) - 1) << ROCEE_GLB_CFG_ROCEE_PORT_ST_S) + +#define ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S 16 + +#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S 0 +#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M \ + (((1UL << 24) - 1) << ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S) + +#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S 24 +#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M \ + (((1UL << 4) - 1) << ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S) + +#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S 0 +#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M \ + (((1UL << 24) - 1) << ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S) + +#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S 24 +#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M \ + (((1UL << 4) - 1) << ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S) + +#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S 0 +#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M \ + (((1UL << 16) - 1) << ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S) + +#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S 16 +#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M \ + (((1UL << 16) - 1) << ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S) + +#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S 0 +#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M \ + (((1UL << 16) - 1) << ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S) + +#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S 16 +#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M \ + (((1UL << 16) - 1) << ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S) + +#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_S 0 +#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_M \ + (((1UL << 8) - 1) << ROCEE_RAQ_WL_ROCEE_RAQ_WL_S) + +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S 0 +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M \ + (((1UL << 15) - 1) << \ + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S) + +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S 16 +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M \ + (((1UL << 4) - 1) << \ + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S) + +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S 20 + +#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE 21 + +#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S 0 +#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M \ + (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S) + +#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S 5 +#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M \ + (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S) + +#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S 0 +#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M \ + (((1UL << 5) - 1) << ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S) + +#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S 5 +#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M \ + (((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S) + +#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S 0 +#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M \ + (((1UL << 5) - 1) << ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S) + +#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S 8 +#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M \ + (((1UL << 5) - 1) << ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S) + +#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S 0 +#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M \ + (((1UL << 19) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S) + +#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_S 19 + +#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S 20 +#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M \ + (((1UL << 2) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S) + +#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S 22 +#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M \ + (((1UL << 5) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S) + +#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S 31 + +#define ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S 0 +#define ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M \ + (((1UL << 3) - 1) << ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S) + +#define ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S 0 +#define ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M \ + (((1UL << 15) - 1) << ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S) + +#define ROCEE_MB6_ROCEE_MB_CMD_S 0 +#define ROCEE_MB6_ROCEE_MB_CMD_M \ + (((1UL << 8) - 1) << ROCEE_MB6_ROCEE_MB_CMD_S) + +#define ROCEE_MB6_ROCEE_MB_CMD_MDF_S 8 +#define ROCEE_MB6_ROCEE_MB_CMD_MDF_M \ + (((1UL << 4) - 1) << ROCEE_MB6_ROCEE_MB_CMD_MDF_S) + +#define ROCEE_MB6_ROCEE_MB_EVENT_S 14 + +#define ROCEE_MB6_ROCEE_MB_HW_RUN_S 15 + +#define ROCEE_MB6_ROCEE_MB_TOKEN_S 16 +#define ROCEE_MB6_ROCEE_MB_TOKEN_M \ + (((1UL << 16) - 1) << ROCEE_MB6_ROCEE_MB_TOKEN_S) + +#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S 0 +#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M \ + (((1UL << 24) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S) + +#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S 24 +#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M \ + (((1UL << 4) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S) + +#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S 28 +#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M \ + (((1UL << 3) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S) + +#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S 31 + +#define ROCEE_SMAC_H_ROCEE_SMAC_H_S 0 +#define ROCEE_SMAC_H_ROCEE_SMAC_H_M \ + (((1UL << 16) - 1) << ROCEE_SMAC_H_ROCEE_SMAC_H_S) + +#define ROCEE_SMAC_H_ROCEE_PORT_MTU_S 16 +#define ROCEE_SMAC_H_ROCEE_PORT_MTU_M \ + (((1UL << 4) - 1) << ROCEE_SMAC_H_ROCEE_PORT_MTU_S) + +#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S 0 +#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M \ + (((1UL << 2) - 1) << ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S) + +#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S 8 +#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M \ + (((1UL << 4) - 1) << ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S) + +#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S 17 + +#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S 0 +#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M \ + (((1UL << 5) - 1) << ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S) + +#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S 16 +#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M \ + (((1UL << 16) - 1) << ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S) + +#define ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S 0 +#define ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M \ + (((1UL << 16) - 1) << ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S) + +#define ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S 16 +#define ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S 1 +#define ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S 0 + +#define ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S 0 +#define ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S 1 + +#define ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S 0 + +#define ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S 0 +#define ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M \ + (((1UL << 28) - 1) << ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S) + +#define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S 0 +#define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M \ + (((1UL << 28) - 1) << ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) + +#define ROCEE_SDB_INV_CNT_SDB_INV_CNT_S 0 +#define ROCEE_SDB_INV_CNT_SDB_INV_CNT_M \ + (((1UL << 16) - 1) << ROCEE_SDB_INV_CNT_SDB_INV_CNT_S) + +#define ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S 0 +#define ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M \ + (((1UL << 16) - 1) << ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S) + +#define ROCEE_SDB_CNT_CMP_BITS 16 + +#define ROCEE_TSP_BP_ST_QH_FIFO_ENTRY_S 20 + +#define ROCEE_CNT_CLR_CE_CNT_CLR_CE_S 0 + +/*************ROCEE_REG DEFINITION****************/ +#define ROCEE_VENDOR_ID_REG 0x0 +#define ROCEE_VENDOR_PART_ID_REG 0x4 + +#define ROCEE_SYS_IMAGE_GUID_L_REG 0xC +#define ROCEE_SYS_IMAGE_GUID_H_REG 0x10 + +#define ROCEE_PORT_GID_L_0_REG 0x50 +#define ROCEE_PORT_GID_ML_0_REG 0x54 +#define ROCEE_PORT_GID_MH_0_REG 0x58 +#define ROCEE_PORT_GID_H_0_REG 0x5C + +#define ROCEE_BT_CMD_H_REG 0x204 + +#define ROCEE_SMAC_L_0_REG 0x240 +#define ROCEE_SMAC_H_0_REG 0x244 + +#define ROCEE_QP1C_CFG3_0_REG 0x27C + +#define ROCEE_CAEP_AEQE_CONS_IDX_REG 0x3AC +#define ROCEE_CAEP_CEQC_CONS_IDX_0_REG 0x3BC + +#define ROCEE_ECC_UCERR_ALM1_REG 0xB38 +#define ROCEE_ECC_UCERR_ALM2_REG 0xB3C +#define ROCEE_ECC_CERR_ALM1_REG 0xB44 +#define ROCEE_ECC_CERR_ALM2_REG 0xB48 + +#define ROCEE_ACK_DELAY_REG 0x14 +#define ROCEE_GLB_CFG_REG 0x18 + +#define ROCEE_DMAE_USER_CFG1_REG 0x40 +#define ROCEE_DMAE_USER_CFG2_REG 0x44 + +#define ROCEE_DB_SQ_WL_REG 0x154 +#define ROCEE_DB_OTHERS_WL_REG 0x158 +#define ROCEE_RAQ_WL_REG 0x15C +#define ROCEE_WRMS_POL_TIME_INTERVAL_REG 0x160 +#define ROCEE_EXT_DB_SQ_REG 0x164 +#define ROCEE_EXT_DB_SQ_H_REG 0x168 +#define ROCEE_EXT_DB_OTH_REG 0x16C + +#define ROCEE_EXT_DB_OTH_H_REG 0x170 +#define ROCEE_EXT_DB_SQ_WL_EMPTY_REG 0x174 +#define ROCEE_EXT_DB_SQ_WL_REG 0x178 +#define ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG 0x17C +#define ROCEE_EXT_DB_OTHERS_WL_REG 0x180 +#define ROCEE_EXT_RAQ_REG 0x184 +#define ROCEE_EXT_RAQ_H_REG 0x188 + +#define ROCEE_CAEP_CE_INTERVAL_CFG_REG 0x190 +#define ROCEE_CAEP_CE_BURST_NUM_CFG_REG 0x194 +#define ROCEE_BT_CMD_L_REG 0x200 + +#define ROCEE_MB1_REG 0x210 +#define ROCEE_MB6_REG 0x224 +#define ROCEE_DB_SQ_L_0_REG 0x230 +#define ROCEE_DB_OTHERS_L_0_REG 0x238 +#define ROCEE_QP1C_CFG0_0_REG 0x270 + +#define ROCEE_CAEP_AEQC_AEQE_SHIFT_REG 0x3A0 +#define ROCEE_CAEP_CEQC_SHIFT_0_REG 0x3B0 +#define ROCEE_CAEP_CE_IRQ_MASK_0_REG 0x3C0 +#define ROCEE_CAEP_CEQ_ALM_OVF_0_REG 0x3C4 +#define ROCEE_CAEP_AE_MASK_REG 0x6C8 +#define ROCEE_CAEP_AE_ST_REG 0x6CC + +#define ROCEE_CAEP_CQE_WCMD_EMPTY 0x850 +#define ROCEE_SCAEP_WR_CQE_CNT 0x8D0 +#define ROCEE_ECC_UCERR_ALM0_REG 0xB34 +#define ROCEE_ECC_CERR_ALM0_REG 0xB40 + +/* V2 ROCEE REG */ +#define ROCEE_TX_CMQ_BASEADDR_L_REG 0x07000 +#define ROCEE_TX_CMQ_BASEADDR_H_REG 0x07004 +#define ROCEE_TX_CMQ_DEPTH_REG 0x07008 +#define ROCEE_TX_CMQ_TAIL_REG 0x07010 +#define ROCEE_TX_CMQ_HEAD_REG 0x07014 + +#define ROCEE_RX_CMQ_BASEADDR_L_REG 0x07018 +#define ROCEE_RX_CMQ_BASEADDR_H_REG 0x0701c +#define ROCEE_RX_CMQ_DEPTH_REG 0x07020 +#define ROCEE_RX_CMQ_TAIL_REG 0x07024 +#define ROCEE_RX_CMQ_HEAD_REG 0x07028 + +#define ROCEE_VF_EQ_DB_CFG0_REG 0x238 +#define ROCEE_VF_EQ_DB_CFG1_REG 0x23C + +#define ROCEE_VF_ABN_INT_CFG_REG 0x13000 +#define ROCEE_VF_ABN_INT_ST_REG 0x13004 +#define ROCEE_VF_ABN_INT_EN_REG 0x13008 +#define ROCEE_VF_EVENT_INT_EN_REG 0x1300c + +#endif /* _HNS_ROCE_COMMON_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c new file mode 100644 index 000000000..9200e6477 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -0,0 +1,417 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/platform_device.h> +#include <rdma/ib_umem.h> +#include <rdma/uverbs_ioctl.h> +#include "hns_roce_device.h" +#include "hns_roce_cmd.h" +#include "hns_roce_hem.h" +#include <rdma/hns-abi.h> +#include "hns_roce_common.h" + +static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_cmd_mailbox *mailbox; + struct hns_roce_cq_table *cq_table; + u64 mtts[MTT_MIN_COUNT] = { 0 }; + dma_addr_t dma_handle; + int ret; + + ret = hns_roce_mtr_find(hr_dev, &hr_cq->mtr, 0, mtts, ARRAY_SIZE(mtts), + &dma_handle); + if (!ret) { + ibdev_err(ibdev, "failed to find CQ mtr, ret = %d.\n", ret); + return -EINVAL; + } + + cq_table = &hr_dev->cq_table; + ret = hns_roce_bitmap_alloc(&cq_table->bitmap, &hr_cq->cqn); + if (ret) { + ibdev_err(ibdev, "failed to alloc CQ bitmap, ret = %d.\n", ret); + return ret; + } + + /* Get CQC memory HEM(Hardware Entry Memory) table */ + ret = hns_roce_table_get(hr_dev, &cq_table->table, hr_cq->cqn); + if (ret) { + ibdev_err(ibdev, "failed to get CQ(0x%lx) context, ret = %d.\n", + hr_cq->cqn, ret); + goto err_out; + } + + ret = xa_err(xa_store(&cq_table->array, hr_cq->cqn, hr_cq, GFP_KERNEL)); + if (ret) { + ibdev_err(ibdev, "failed to xa_store CQ, ret = %d.\n", ret); + goto err_put; + } + + /* Allocate mailbox memory */ + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) { + ret = PTR_ERR(mailbox); + goto err_xa; + } + + hr_dev->hw->write_cqc(hr_dev, hr_cq, mailbox->buf, mtts, dma_handle); + + /* Send mailbox to hw */ + ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 0, + HNS_ROCE_CMD_CREATE_CQC, HNS_ROCE_CMD_TIMEOUT_MSECS); + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + if (ret) { + ibdev_err(ibdev, + "failed to send create cmd for CQ(0x%lx), ret = %d.\n", + hr_cq->cqn, ret); + goto err_xa; + } + + hr_cq->cons_index = 0; + hr_cq->arm_sn = 1; + + atomic_set(&hr_cq->refcount, 1); + init_completion(&hr_cq->free); + + return 0; + +err_xa: + xa_erase(&cq_table->array, hr_cq->cqn); + +err_put: + hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); + +err_out: + hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR); + return ret; +} + +static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) +{ + struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; + struct device *dev = hr_dev->dev; + int ret; + + ret = hns_roce_cmd_mbox(hr_dev, 0, 0, hr_cq->cqn, 1, + HNS_ROCE_CMD_DESTROY_CQC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + if (ret) + dev_err(dev, "DESTROY_CQ failed (%d) for CQN %06lx\n", ret, + hr_cq->cqn); + + xa_erase(&cq_table->array, hr_cq->cqn); + + /* Waiting interrupt process procedure carried out */ + synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq); + + /* wait for all interrupt processed */ + if (atomic_dec_and_test(&hr_cq->refcount)) + complete(&hr_cq->free); + wait_for_completion(&hr_cq->free); + + hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); + hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR); +} + +static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, + struct ib_udata *udata, unsigned long addr) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_buf_attr buf_attr = {}; + int ret; + + buf_attr.page_shift = hr_dev->caps.cqe_buf_pg_sz + HNS_HW_PAGE_SHIFT; + buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size; + buf_attr.region[0].hopnum = hr_dev->caps.cqe_hop_num; + buf_attr.region_count = 1; + buf_attr.fixed_page = true; + + ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr, + hr_dev->caps.cqe_ba_pg_sz + HNS_HW_PAGE_SHIFT, + udata, addr); + if (ret) + ibdev_err(ibdev, "failed to alloc CQ mtr, ret = %d.\n", ret); + + return ret; +} + +static void free_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) +{ + hns_roce_mtr_destroy(hr_dev, &hr_cq->mtr); +} + +static int alloc_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, + struct ib_udata *udata, unsigned long addr, + struct hns_roce_ib_create_cq_resp *resp) +{ + bool has_db = hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB; + struct hns_roce_ucontext *uctx; + int err; + + if (udata) { + if (has_db && + udata->outlen >= offsetofend(typeof(*resp), cap_flags)) { + uctx = rdma_udata_to_drv_context(udata, + struct hns_roce_ucontext, ibucontext); + err = hns_roce_db_map_user(uctx, udata, addr, + &hr_cq->db); + if (err) + return err; + hr_cq->flags |= HNS_ROCE_CQ_FLAG_RECORD_DB; + resp->cap_flags |= HNS_ROCE_CQ_FLAG_RECORD_DB; + } + } else { + if (has_db) { + err = hns_roce_alloc_db(hr_dev, &hr_cq->db, 1); + if (err) + return err; + hr_cq->set_ci_db = hr_cq->db.db_record; + *hr_cq->set_ci_db = 0; + hr_cq->flags |= HNS_ROCE_CQ_FLAG_RECORD_DB; + } + hr_cq->cq_db_l = hr_dev->reg_base + hr_dev->odb_offset + + DB_REG_OFFSET * hr_dev->priv_uar.index; + } + + return 0; +} + +static void free_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, + struct ib_udata *udata) +{ + struct hns_roce_ucontext *uctx; + + if (!(hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)) + return; + + hr_cq->flags &= ~HNS_ROCE_CQ_FLAG_RECORD_DB; + if (udata) { + uctx = rdma_udata_to_drv_context(udata, + struct hns_roce_ucontext, + ibucontext); + hns_roce_db_unmap_user(uctx, &hr_cq->db); + } else { + hns_roce_free_db(hr_dev, &hr_cq->db); + } +} + +static void set_cqe_size(struct hns_roce_cq *hr_cq, struct ib_udata *udata, + struct hns_roce_ib_create_cq *ucmd) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); + + if (udata) { + if (udata->inlen >= offsetofend(typeof(*ucmd), cqe_size)) + hr_cq->cqe_size = ucmd->cqe_size; + else + hr_cq->cqe_size = HNS_ROCE_V2_CQE_SIZE; + } else { + hr_cq->cqe_size = hr_dev->caps.cqe_sz; + } +} + +int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); + struct hns_roce_ib_create_cq_resp resp = {}; + struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_ib_create_cq ucmd = {}; + int vector = attr->comp_vector; + u32 cq_entries = attr->cqe; + int ret; + + if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) { + ibdev_err(ibdev, "failed to check CQ count %u, max = %u.\n", + cq_entries, hr_dev->caps.max_cqes); + return -EINVAL; + } + + if (vector >= hr_dev->caps.num_comp_vectors) { + ibdev_err(ibdev, "failed to check CQ vector = %d, max = %d.\n", + vector, hr_dev->caps.num_comp_vectors); + return -EINVAL; + } + + cq_entries = max(cq_entries, hr_dev->caps.min_cqes); + cq_entries = roundup_pow_of_two(cq_entries); + hr_cq->ib_cq.cqe = cq_entries - 1; /* used as cqe index */ + hr_cq->cq_depth = cq_entries; + hr_cq->vector = vector; + spin_lock_init(&hr_cq->lock); + INIT_LIST_HEAD(&hr_cq->sq_list); + INIT_LIST_HEAD(&hr_cq->rq_list); + + if (udata) { + ret = ib_copy_from_udata(&ucmd, udata, + min(udata->inlen, sizeof(ucmd))); + if (ret) { + ibdev_err(ibdev, "failed to copy CQ udata, ret = %d.\n", + ret); + return ret; + } + } + + set_cqe_size(hr_cq, udata, &ucmd); + + ret = alloc_cq_buf(hr_dev, hr_cq, udata, ucmd.buf_addr); + if (ret) { + ibdev_err(ibdev, "failed to alloc CQ buf, ret = %d.\n", ret); + return ret; + } + + ret = alloc_cq_db(hr_dev, hr_cq, udata, ucmd.db_addr, &resp); + if (ret) { + ibdev_err(ibdev, "failed to alloc CQ db, ret = %d.\n", ret); + goto err_cq_buf; + } + + ret = alloc_cqc(hr_dev, hr_cq); + if (ret) { + ibdev_err(ibdev, + "failed to alloc CQ context, ret = %d.\n", ret); + goto err_cq_db; + } + + /* + * For the QP created by kernel space, tptr value should be initialized + * to zero; For the QP created by user space, it will cause synchronous + * problems if tptr is set to zero here, so we initialize it in user + * space. + */ + if (!udata && hr_cq->tptr_addr) + *hr_cq->tptr_addr = 0; + + if (udata) { + resp.cqn = hr_cq->cqn; + ret = ib_copy_to_udata(udata, &resp, + min(udata->outlen, sizeof(resp))); + if (ret) + goto err_cqc; + } + + return 0; + +err_cqc: + free_cqc(hr_dev, hr_cq); +err_cq_db: + free_cq_db(hr_dev, hr_cq, udata); +err_cq_buf: + free_cq_buf(hr_dev, hr_cq); + return ret; +} + +int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); + struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); + + if (hr_dev->hw->destroy_cq) + hr_dev->hw->destroy_cq(ib_cq, udata); + + free_cq_buf(hr_dev, hr_cq); + free_cq_db(hr_dev, hr_cq, udata); + free_cqc(hr_dev, hr_cq); + return 0; +} + +void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn) +{ + struct hns_roce_cq *hr_cq; + struct ib_cq *ibcq; + + hr_cq = xa_load(&hr_dev->cq_table.array, + cqn & (hr_dev->caps.num_cqs - 1)); + if (!hr_cq) { + dev_warn(hr_dev->dev, "Completion event for bogus CQ 0x%06x\n", + cqn); + return; + } + + ++hr_cq->arm_sn; + ibcq = &hr_cq->ib_cq; + if (ibcq->comp_handler) + ibcq->comp_handler(ibcq, ibcq->cq_context); +} + +void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) +{ + struct device *dev = hr_dev->dev; + struct hns_roce_cq *hr_cq; + struct ib_event event; + struct ib_cq *ibcq; + + hr_cq = xa_load(&hr_dev->cq_table.array, + cqn & (hr_dev->caps.num_cqs - 1)); + if (!hr_cq) { + dev_warn(dev, "Async event for bogus CQ 0x%06x\n", cqn); + return; + } + + if (event_type != HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID && + event_type != HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR && + event_type != HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW) { + dev_err(dev, "Unexpected event type 0x%x on CQ 0x%06x\n", + event_type, cqn); + return; + } + + atomic_inc(&hr_cq->refcount); + + ibcq = &hr_cq->ib_cq; + if (ibcq->event_handler) { + event.device = ibcq->device; + event.element.cq = ibcq; + event.event = IB_EVENT_CQ_ERR; + ibcq->event_handler(&event, ibcq->cq_context); + } + + if (atomic_dec_and_test(&hr_cq->refcount)) + complete(&hr_cq->free); +} + +int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; + + xa_init(&cq_table->array); + + return hns_roce_bitmap_init(&cq_table->bitmap, hr_dev->caps.num_cqs, + hr_dev->caps.num_cqs - 1, + hr_dev->caps.reserved_cqs, 0); +} + +void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev) +{ + hns_roce_bitmap_cleanup(&hr_dev->cq_table.bitmap); +} diff --git a/drivers/infiniband/hw/hns/hns_roce_db.c b/drivers/infiniband/hw/hns/hns_roce_db.c new file mode 100644 index 000000000..bff6abdcc --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_db.c @@ -0,0 +1,182 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ +/* + * Copyright (c) 2017 Hisilicon Limited. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + */ + +#include <linux/platform_device.h> +#include <rdma/ib_umem.h> +#include "hns_roce_device.h" + +int hns_roce_db_map_user(struct hns_roce_ucontext *context, + struct ib_udata *udata, unsigned long virt, + struct hns_roce_db *db) +{ + unsigned long page_addr = virt & PAGE_MASK; + struct hns_roce_user_db_page *page; + unsigned int offset; + int ret = 0; + + mutex_lock(&context->page_mutex); + + list_for_each_entry(page, &context->page_list, list) + if (page->user_virt == page_addr) + goto found; + + page = kmalloc(sizeof(*page), GFP_KERNEL); + if (!page) { + ret = -ENOMEM; + goto out; + } + + refcount_set(&page->refcount, 1); + page->user_virt = page_addr; + page->umem = ib_umem_get(context->ibucontext.device, page_addr, + PAGE_SIZE, 0); + if (IS_ERR(page->umem)) { + ret = PTR_ERR(page->umem); + kfree(page); + goto out; + } + + list_add(&page->list, &context->page_list); + +found: + offset = virt - page_addr; + db->dma = sg_dma_address(page->umem->sg_head.sgl) + offset; + db->virt_addr = sg_virt(page->umem->sg_head.sgl) + offset; + db->u.user_page = page; + refcount_inc(&page->refcount); + +out: + mutex_unlock(&context->page_mutex); + + return ret; +} + +void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, + struct hns_roce_db *db) +{ + mutex_lock(&context->page_mutex); + + refcount_dec(&db->u.user_page->refcount); + if (refcount_dec_if_one(&db->u.user_page->refcount)) { + list_del(&db->u.user_page->list); + ib_umem_release(db->u.user_page->umem); + kfree(db->u.user_page); + } + + mutex_unlock(&context->page_mutex); +} + +static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir( + struct device *dma_device) +{ + struct hns_roce_db_pgdir *pgdir; + + pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); + if (!pgdir) + return NULL; + + bitmap_fill(pgdir->order1, + HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT); + pgdir->bits[0] = pgdir->order0; + pgdir->bits[1] = pgdir->order1; + pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE, + &pgdir->db_dma, GFP_KERNEL); + if (!pgdir->page) { + kfree(pgdir); + return NULL; + } + + return pgdir; +} + +static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir, + struct hns_roce_db *db, int order) +{ + int o; + int i; + + for (o = order; o <= 1; ++o) { + i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o); + if (i < HNS_ROCE_DB_PER_PAGE >> o) + goto found; + } + + return -ENOMEM; + +found: + clear_bit(i, pgdir->bits[o]); + + i <<= o; + + if (o > order) + set_bit(i ^ 1, pgdir->bits[order]); + + db->u.pgdir = pgdir; + db->index = i; + db->db_record = pgdir->page + db->index; + db->dma = pgdir->db_dma + db->index * HNS_ROCE_DB_UNIT_SIZE; + db->order = order; + + return 0; +} + +int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db, + int order) +{ + struct hns_roce_db_pgdir *pgdir; + int ret = 0; + + mutex_lock(&hr_dev->pgdir_mutex); + + list_for_each_entry(pgdir, &hr_dev->pgdir_list, list) + if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order)) + goto out; + + pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev); + if (!pgdir) { + ret = -ENOMEM; + goto out; + } + + list_add(&pgdir->list, &hr_dev->pgdir_list); + + /* This should never fail -- we just allocated an empty page: */ + WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order)); + +out: + mutex_unlock(&hr_dev->pgdir_mutex); + + return ret; +} + +void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) +{ + int o; + int i; + + mutex_lock(&hr_dev->pgdir_mutex); + + o = db->order; + i = db->index; + + if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { + clear_bit(i ^ 1, db->u.pgdir->order0); + ++o; + } + + i >>= o; + set_bit(i, db->u.pgdir->bits[o]); + + if (bitmap_full(db->u.pgdir->order1, + HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) { + dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page, + db->u.pgdir->db_dma); + list_del(&db->u.pgdir->list); + kfree(db->u.pgdir); + } + + mutex_unlock(&hr_dev->pgdir_mutex); +} diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h new file mode 100644 index 000000000..09b5e4935 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -0,0 +1,1287 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _HNS_ROCE_DEVICE_H +#define _HNS_ROCE_DEVICE_H + +#include <rdma/ib_verbs.h> + +#define DRV_NAME "hns_roce" + +#define PCI_REVISION_ID_HIP08 0x21 +#define PCI_REVISION_ID_HIP09 0x30 + +#define HNS_ROCE_HW_VER1 ('h' << 24 | 'i' << 16 | '0' << 8 | '6') + +#define HNS_ROCE_MAX_MSG_LEN 0x80000000 + +#define HNS_ROCE_IB_MIN_SQ_STRIDE 6 + +#define BA_BYTE_LEN 8 + +/* Hardware specification only for v1 engine */ +#define HNS_ROCE_MIN_CQE_NUM 0x40 +#define HNS_ROCE_MIN_WQE_NUM 0x20 + +/* Hardware specification only for v1 engine */ +#define HNS_ROCE_MAX_INNER_MTPT_NUM 0x7 +#define HNS_ROCE_MAX_MTPT_PBL_NUM 0x100000 + +#define HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS 20 +#define HNS_ROCE_MAX_FREE_CQ_WAIT_CNT \ + (5000 / HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS) +#define HNS_ROCE_CQE_WCMD_EMPTY_BIT 0x2 +#define HNS_ROCE_MIN_CQE_CNT 16 + +#define HNS_ROCE_MAX_IRQ_NUM 128 + +#define HNS_ROCE_SGE_IN_WQE 2 +#define HNS_ROCE_SGE_SHIFT 4 + +#define EQ_ENABLE 1 +#define EQ_DISABLE 0 + +#define HNS_ROCE_CEQ 0 +#define HNS_ROCE_AEQ 1 + +#define HNS_ROCE_CEQE_SIZE 0x4 +#define HNS_ROCE_AEQE_SIZE 0x10 + +#define HNS_ROCE_V3_EQE_SIZE 0x40 + +#define HNS_ROCE_V2_CQE_SIZE 32 +#define HNS_ROCE_V3_CQE_SIZE 64 + +#define HNS_ROCE_V2_QPC_SZ 256 +#define HNS_ROCE_V3_QPC_SZ 512 + +#define HNS_ROCE_MAX_PORTS 6 +#define HNS_ROCE_GID_SIZE 16 +#define HNS_ROCE_SGE_SIZE 16 + +#define HNS_ROCE_HOP_NUM_0 0xff + +#define BITMAP_NO_RR 0 +#define BITMAP_RR 1 + +#define MR_TYPE_MR 0x00 +#define MR_TYPE_FRMR 0x01 +#define MR_TYPE_DMA 0x03 + +#define HNS_ROCE_FRMR_MAX_PA 512 + +#define PKEY_ID 0xffff +#define GUID_LEN 8 +#define NODE_DESC_SIZE 64 +#define DB_REG_OFFSET 0x1000 + +/* Configure to HW for PAGE_SIZE larger than 4KB */ +#define PG_SHIFT_OFFSET (PAGE_SHIFT - 12) + +#define PAGES_SHIFT_8 8 +#define PAGES_SHIFT_16 16 +#define PAGES_SHIFT_24 24 +#define PAGES_SHIFT_32 32 + +#define HNS_ROCE_IDX_QUE_ENTRY_SZ 4 +#define SRQ_DB_REG 0x230 + +/* The chip implementation of the consumer index is calculated + * according to twice the actual EQ depth + */ +#define EQ_DEPTH_COEFF 2 + +enum { + SERV_TYPE_RC, + SERV_TYPE_UC, + SERV_TYPE_RD, + SERV_TYPE_UD, +}; + +enum { + HNS_ROCE_QP_CAP_RQ_RECORD_DB = BIT(0), + HNS_ROCE_QP_CAP_SQ_RECORD_DB = BIT(1), +}; + +enum hns_roce_cq_flags { + HNS_ROCE_CQ_FLAG_RECORD_DB = BIT(0), +}; + +enum hns_roce_qp_state { + HNS_ROCE_QP_STATE_RST, + HNS_ROCE_QP_STATE_INIT, + HNS_ROCE_QP_STATE_RTR, + HNS_ROCE_QP_STATE_RTS, + HNS_ROCE_QP_STATE_SQD, + HNS_ROCE_QP_STATE_ERR, + HNS_ROCE_QP_NUM_STATE, +}; + +enum hns_roce_event { + HNS_ROCE_EVENT_TYPE_PATH_MIG = 0x01, + HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED = 0x02, + HNS_ROCE_EVENT_TYPE_COMM_EST = 0x03, + HNS_ROCE_EVENT_TYPE_SQ_DRAINED = 0x04, + HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, + HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR = 0x06, + HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR = 0x07, + HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH = 0x08, + HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH = 0x09, + HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR = 0x0a, + HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR = 0x0b, + HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW = 0x0c, + HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID = 0x0d, + HNS_ROCE_EVENT_TYPE_PORT_CHANGE = 0x0f, + /* 0x10 and 0x11 is unused in currently application case */ + HNS_ROCE_EVENT_TYPE_DB_OVERFLOW = 0x12, + HNS_ROCE_EVENT_TYPE_MB = 0x13, + HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW = 0x14, + HNS_ROCE_EVENT_TYPE_FLR = 0x15, +}; + +/* Local Work Queue Catastrophic Error,SUBTYPE 0x5 */ +enum { + HNS_ROCE_LWQCE_QPC_ERROR = 1, + HNS_ROCE_LWQCE_MTU_ERROR = 2, + HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR = 3, + HNS_ROCE_LWQCE_WQE_ADDR_ERROR = 4, + HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR = 5, + HNS_ROCE_LWQCE_SL_ERROR = 6, + HNS_ROCE_LWQCE_PORT_ERROR = 7, +}; + +/* Local Access Violation Work Queue Error,SUBTYPE 0x7 */ +enum { + HNS_ROCE_LAVWQE_R_KEY_VIOLATION = 1, + HNS_ROCE_LAVWQE_LENGTH_ERROR = 2, + HNS_ROCE_LAVWQE_VA_ERROR = 3, + HNS_ROCE_LAVWQE_PD_ERROR = 4, + HNS_ROCE_LAVWQE_RW_ACC_ERROR = 5, + HNS_ROCE_LAVWQE_KEY_STATE_ERROR = 6, + HNS_ROCE_LAVWQE_MR_OPERATION_ERROR = 7, +}; + +/* DOORBELL overflow subtype */ +enum { + HNS_ROCE_DB_SUBTYPE_SDB_OVF = 1, + HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF = 2, + HNS_ROCE_DB_SUBTYPE_ODB_OVF = 3, + HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF = 4, + HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP = 5, + HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP = 6, +}; + +enum { + /* RQ&SRQ related operations */ + HNS_ROCE_OPCODE_SEND_DATA_RECEIVE = 0x06, + HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE = 0x07, +}; + +#define HNS_ROCE_CAP_FLAGS_EX_SHIFT 12 + +enum { + HNS_ROCE_CAP_FLAG_REREG_MR = BIT(0), + HNS_ROCE_CAP_FLAG_ROCE_V1_V2 = BIT(1), + HNS_ROCE_CAP_FLAG_RQ_INLINE = BIT(2), + HNS_ROCE_CAP_FLAG_RECORD_DB = BIT(3), + HNS_ROCE_CAP_FLAG_SQ_RECORD_DB = BIT(4), + HNS_ROCE_CAP_FLAG_SRQ = BIT(5), + HNS_ROCE_CAP_FLAG_MW = BIT(7), + HNS_ROCE_CAP_FLAG_FRMR = BIT(8), + HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9), + HNS_ROCE_CAP_FLAG_ATOMIC = BIT(10), +}; + +#define HNS_ROCE_DB_TYPE_COUNT 2 +#define HNS_ROCE_DB_UNIT_SIZE 4 + +enum { + HNS_ROCE_DB_PER_PAGE = PAGE_SIZE / 4 +}; + +enum hns_roce_reset_stage { + HNS_ROCE_STATE_NON_RST, + HNS_ROCE_STATE_RST_BEF_DOWN, + HNS_ROCE_STATE_RST_DOWN, + HNS_ROCE_STATE_RST_UNINIT, + HNS_ROCE_STATE_RST_INIT, + HNS_ROCE_STATE_RST_INITED, +}; + +enum hns_roce_instance_state { + HNS_ROCE_STATE_NON_INIT, + HNS_ROCE_STATE_INIT, + HNS_ROCE_STATE_INITED, + HNS_ROCE_STATE_UNINIT, +}; + +enum { + HNS_ROCE_RST_DIRECT_RETURN = 0, +}; + +enum { + CMD_RST_PRC_OTHERS, + CMD_RST_PRC_SUCCESS, + CMD_RST_PRC_EBUSY, +}; + +#define HNS_ROCE_CMD_SUCCESS 1 + +#define HNS_ROCE_PORT_DOWN 0 +#define HNS_ROCE_PORT_UP 1 + +/* The minimum page size is 4K for hardware */ +#define HNS_HW_PAGE_SHIFT 12 +#define HNS_HW_PAGE_SIZE (1 << HNS_HW_PAGE_SHIFT) + +/* The minimum page count for hardware access page directly. */ +#define HNS_HW_DIRECT_PAGE_COUNT 2 + +struct hns_roce_uar { + u64 pfn; + unsigned long index; + unsigned long logic_idx; +}; + +struct hns_roce_ucontext { + struct ib_ucontext ibucontext; + struct hns_roce_uar uar; + struct list_head page_list; + struct mutex page_mutex; +}; + +struct hns_roce_pd { + struct ib_pd ibpd; + unsigned long pdn; +}; + +struct hns_roce_bitmap { + /* Bitmap Traversal last a bit which is 1 */ + unsigned long last; + unsigned long top; + unsigned long max; + unsigned long reserved_top; + unsigned long mask; + spinlock_t lock; + unsigned long *table; +}; + +/* For Hardware Entry Memory */ +struct hns_roce_hem_table { + /* HEM type: 0 = qpc, 1 = mtt, 2 = cqc, 3 = srq, 4 = other */ + u32 type; + /* HEM array elment num */ + unsigned long num_hem; + /* HEM entry record obj total num */ + unsigned long num_obj; + /* Single obj size */ + unsigned long obj_size; + unsigned long table_chunk_size; + int lowmem; + struct mutex mutex; + struct hns_roce_hem **hem; + u64 **bt_l1; + dma_addr_t *bt_l1_dma_addr; + u64 **bt_l0; + dma_addr_t *bt_l0_dma_addr; +}; + +struct hns_roce_buf_region { + int offset; /* page offset */ + u32 count; /* page count */ + int hopnum; /* addressing hop num */ +}; + +#define HNS_ROCE_MAX_BT_REGION 3 +#define HNS_ROCE_MAX_BT_LEVEL 3 +struct hns_roce_hem_list { + struct list_head root_bt; + /* link all bt dma mem by hop config */ + struct list_head mid_bt[HNS_ROCE_MAX_BT_REGION][HNS_ROCE_MAX_BT_LEVEL]; + struct list_head btm_bt; /* link all bottom bt in @mid_bt */ + dma_addr_t root_ba; /* pointer to the root ba table */ +}; + +struct hns_roce_buf_attr { + struct { + size_t size; /* region size */ + int hopnum; /* multi-hop addressing hop num */ + } region[HNS_ROCE_MAX_BT_REGION]; + int region_count; /* valid region count */ + unsigned int page_shift; /* buffer page shift */ + bool fixed_page; /* decide page shift is fixed-size or maximum size */ + int user_access; /* umem access flag */ + bool mtt_only; /* only alloc buffer-required MTT memory */ +}; + +struct hns_roce_hem_cfg { + dma_addr_t root_ba; /* root BA table's address */ + bool is_direct; /* addressing without BA table */ + unsigned int ba_pg_shift; /* BA table page shift */ + unsigned int buf_pg_shift; /* buffer page shift */ + unsigned int buf_pg_count; /* buffer page count */ + struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION]; + int region_count; +}; + +/* memory translate region */ +struct hns_roce_mtr { + struct hns_roce_hem_list hem_list; /* multi-hop addressing resource */ + struct ib_umem *umem; /* user space buffer */ + struct hns_roce_buf *kmem; /* kernel space buffer */ + struct hns_roce_hem_cfg hem_cfg; /* config for hardware addressing */ +}; + +struct hns_roce_mw { + struct ib_mw ibmw; + u32 pdn; + u32 rkey; + int enabled; /* MW's active status */ + u32 pbl_hop_num; + u32 pbl_ba_pg_sz; + u32 pbl_buf_pg_sz; +}; + +/* Only support 4K page size for mr register */ +#define MR_SIZE_4K 0 + +struct hns_roce_mr { + struct ib_mr ibmr; + u64 iova; /* MR's virtual orignal addr */ + u64 size; /* Address range of MR */ + u32 key; /* Key of MR */ + u32 pd; /* PD num of MR */ + u32 access; /* Access permission of MR */ + int enabled; /* MR's active status */ + int type; /* MR's register type */ + u32 pbl_hop_num; /* multi-hop number */ + struct hns_roce_mtr pbl_mtr; + u32 npages; + dma_addr_t *page_list; +}; + +struct hns_roce_mr_table { + struct hns_roce_bitmap mtpt_bitmap; + struct hns_roce_hem_table mtpt_table; +}; + +struct hns_roce_wq { + u64 *wrid; /* Work request ID */ + spinlock_t lock; + u32 wqe_cnt; /* WQE num */ + int max_gs; + int offset; + int wqe_shift; /* WQE size */ + u32 head; + u32 tail; + void __iomem *db_reg_l; +}; + +struct hns_roce_sge { + unsigned int sge_cnt; /* SGE num */ + int offset; + int sge_shift; /* SGE size */ +}; + +struct hns_roce_buf_list { + void *buf; + dma_addr_t map; +}; + +struct hns_roce_buf { + struct hns_roce_buf_list direct; + struct hns_roce_buf_list *page_list; + u32 npages; + u32 size; + unsigned int page_shift; +}; + +struct hns_roce_db_pgdir { + struct list_head list; + DECLARE_BITMAP(order0, HNS_ROCE_DB_PER_PAGE); + DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT); + unsigned long *bits[HNS_ROCE_DB_TYPE_COUNT]; + u32 *page; + dma_addr_t db_dma; +}; + +struct hns_roce_user_db_page { + struct list_head list; + struct ib_umem *umem; + unsigned long user_virt; + refcount_t refcount; +}; + +struct hns_roce_db { + u32 *db_record; + union { + struct hns_roce_db_pgdir *pgdir; + struct hns_roce_user_db_page *user_page; + } u; + dma_addr_t dma; + void *virt_addr; + int index; + int order; +}; + +struct hns_roce_cq { + struct ib_cq ib_cq; + struct hns_roce_mtr mtr; + struct hns_roce_db db; + u32 flags; + spinlock_t lock; + u32 cq_depth; + u32 cons_index; + u32 *set_ci_db; + void __iomem *cq_db_l; + u16 *tptr_addr; + int arm_sn; + int cqe_size; + unsigned long cqn; + u32 vector; + atomic_t refcount; + struct completion free; + struct list_head sq_list; /* all qps on this send cq */ + struct list_head rq_list; /* all qps on this recv cq */ + int is_armed; /* cq is armed */ + struct list_head node; /* all armed cqs are on a list */ +}; + +struct hns_roce_idx_que { + struct hns_roce_mtr mtr; + int entry_shift; + unsigned long *bitmap; +}; + +struct hns_roce_srq { + struct ib_srq ibsrq; + unsigned long srqn; + u32 wqe_cnt; + int max_gs; + int wqe_shift; + void __iomem *db_reg_l; + + atomic_t refcount; + struct completion free; + + struct hns_roce_mtr buf_mtr; + + u64 *wrid; + struct hns_roce_idx_que idx_que; + spinlock_t lock; + int head; + int tail; + struct mutex mutex; + void (*event)(struct hns_roce_srq *srq, enum hns_roce_event event); +}; + +struct hns_roce_uar_table { + struct hns_roce_bitmap bitmap; +}; + +struct hns_roce_qp_table { + struct hns_roce_bitmap bitmap; + struct hns_roce_hem_table qp_table; + struct hns_roce_hem_table irrl_table; + struct hns_roce_hem_table trrl_table; + struct hns_roce_hem_table sccc_table; + struct mutex scc_mutex; +}; + +struct hns_roce_cq_table { + struct hns_roce_bitmap bitmap; + struct xarray array; + struct hns_roce_hem_table table; +}; + +struct hns_roce_srq_table { + struct hns_roce_bitmap bitmap; + struct xarray xa; + struct hns_roce_hem_table table; +}; + +struct hns_roce_raq_table { + struct hns_roce_buf_list *e_raq_buf; +}; + +struct hns_roce_av { + u8 port; + u8 gid_index; + u8 stat_rate; + u8 hop_limit; + u32 flowlabel; + u16 udp_sport; + u8 sl; + u8 tclass; + u8 dgid[HNS_ROCE_GID_SIZE]; + u8 mac[ETH_ALEN]; + u16 vlan_id; + u8 vlan_en; +}; + +struct hns_roce_ah { + struct ib_ah ibah; + struct hns_roce_av av; +}; + +struct hns_roce_cmd_context { + struct completion done; + int result; + int next; + u64 out_param; + u16 token; +}; + +struct hns_roce_cmdq { + struct dma_pool *pool; + struct mutex hcr_mutex; + struct semaphore poll_sem; + /* + * Event mode: cmd register mutex protection, + * ensure to not exceed max_cmds and user use limit region + */ + struct semaphore event_sem; + int max_cmds; + spinlock_t context_lock; + int free_head; + struct hns_roce_cmd_context *context; + /* + * Result of get integer part + * which max_comds compute according a power of 2 + */ + u16 token_mask; + /* + * Process whether use event mode, init default non-zero + * After the event queue of cmd event ready, + * can switch into event mode + * close device, switch into poll mode(non event mode) + */ + u8 use_events; +}; + +struct hns_roce_cmd_mailbox { + void *buf; + dma_addr_t dma; +}; + +struct hns_roce_dev; + +struct hns_roce_rinl_sge { + void *addr; + u32 len; +}; + +struct hns_roce_rinl_wqe { + struct hns_roce_rinl_sge *sg_list; + u32 sge_cnt; +}; + +struct hns_roce_rinl_buf { + struct hns_roce_rinl_wqe *wqe_list; + u32 wqe_cnt; +}; + +enum { + HNS_ROCE_FLUSH_FLAG = 0, +}; + +struct hns_roce_work { + struct hns_roce_dev *hr_dev; + struct work_struct work; + u32 qpn; + u32 cqn; + int event_type; + int sub_type; +}; + +struct hns_roce_qp { + struct ib_qp ibqp; + struct hns_roce_wq rq; + struct hns_roce_db rdb; + struct hns_roce_db sdb; + unsigned long en_flags; + u32 doorbell_qpn; + enum ib_sig_type sq_signal_bits; + struct hns_roce_wq sq; + + struct hns_roce_mtr mtr; + + u32 buff_size; + struct mutex mutex; + u8 port; + u8 phy_port; + u8 sl; + u8 resp_depth; + u8 state; + u32 access_flags; + u32 atomic_rd_en; + u32 pkey_index; + u32 qkey; + void (*event)(struct hns_roce_qp *qp, + enum hns_roce_event event_type); + unsigned long qpn; + + atomic_t refcount; + struct completion free; + + struct hns_roce_sge sge; + u32 next_sge; + enum ib_mtu path_mtu; + u32 max_inline_data; + + /* 0: flush needed, 1: unneeded */ + unsigned long flush_flag; + struct hns_roce_work flush_work; + struct hns_roce_rinl_buf rq_inl_buf; + struct list_head node; /* all qps are on a list */ + struct list_head rq_node; /* all recv qps are on a list */ + struct list_head sq_node; /* all send qps are on a list */ +}; + +struct hns_roce_ib_iboe { + spinlock_t lock; + struct net_device *netdevs[HNS_ROCE_MAX_PORTS]; + struct notifier_block nb; + u8 phy_port[HNS_ROCE_MAX_PORTS]; +}; + +enum { + HNS_ROCE_EQ_STAT_INVALID = 0, + HNS_ROCE_EQ_STAT_VALID = 2, +}; + +struct hns_roce_ceqe { + __le32 comp; + __le32 rsv[15]; +}; + +struct hns_roce_aeqe { + __le32 asyn; + union { + struct { + __le32 qp; + u32 rsv0; + u32 rsv1; + } qp_event; + + struct { + __le32 srq; + u32 rsv0; + u32 rsv1; + } srq_event; + + struct { + __le32 cq; + u32 rsv0; + u32 rsv1; + } cq_event; + + struct { + __le32 ceqe; + u32 rsv0; + u32 rsv1; + } ce_event; + + struct { + __le64 out_param; + __le16 token; + u8 status; + u8 rsv0; + } __packed cmd; + } event; + __le32 rsv[12]; +}; + +struct hns_roce_eq { + struct hns_roce_dev *hr_dev; + void __iomem *doorbell; + + int type_flag; /* Aeq:1 ceq:0 */ + int eqn; + u32 entries; + int log_entries; + int eqe_size; + int irq; + int log_page_size; + int cons_index; + struct hns_roce_buf_list *buf_list; + int over_ignore; + int coalesce; + int arm_st; + int hop_num; + struct hns_roce_mtr mtr; + u16 eq_max_cnt; + int eq_period; + int shift; + int event_type; + int sub_type; +}; + +struct hns_roce_eq_table { + struct hns_roce_eq *eq; + void __iomem **eqc_base; /* only for hw v1 */ +}; + +struct hns_roce_caps { + u64 fw_ver; + u8 num_ports; + int gid_table_len[HNS_ROCE_MAX_PORTS]; + int pkey_table_len[HNS_ROCE_MAX_PORTS]; + int local_ca_ack_delay; + int num_uars; + u32 phy_num_uars; + u32 max_sq_sg; + u32 max_sq_inline; + u32 max_rq_sg; + u32 max_extend_sg; + int num_qps; + int reserved_qps; + int num_qpc_timer; + int num_cqc_timer; + int num_srqs; + u32 max_wqes; + u32 max_srq_wrs; + u32 max_srq_sges; + u32 max_sq_desc_sz; + u32 max_rq_desc_sz; + u32 max_srq_desc_sz; + int max_qp_init_rdma; + int max_qp_dest_rdma; + int num_cqs; + u32 max_cqes; + u32 min_cqes; + u32 min_wqes; + int reserved_cqs; + int reserved_srqs; + int num_aeq_vectors; + int num_comp_vectors; + int num_other_vectors; + int num_mtpts; + u32 num_mtt_segs; + u32 num_cqe_segs; + u32 num_srqwqe_segs; + u32 num_idx_segs; + int reserved_mrws; + int reserved_uars; + int num_pds; + int reserved_pds; + u32 mtt_entry_sz; + u32 cqe_sz; + u32 page_size_cap; + u32 reserved_lkey; + int mtpt_entry_sz; + int qpc_sz; + int irrl_entry_sz; + int trrl_entry_sz; + int cqc_entry_sz; + int sccc_sz; + int qpc_timer_entry_sz; + int cqc_timer_entry_sz; + int srqc_entry_sz; + int idx_entry_sz; + u32 pbl_ba_pg_sz; + u32 pbl_buf_pg_sz; + u32 pbl_hop_num; + int aeqe_depth; + int ceqe_depth; + u32 aeqe_size; + u32 ceqe_size; + enum ib_mtu max_mtu; + u32 qpc_bt_num; + u32 qpc_timer_bt_num; + u32 srqc_bt_num; + u32 cqc_bt_num; + u32 cqc_timer_bt_num; + u32 mpt_bt_num; + u32 sccc_bt_num; + u32 qpc_ba_pg_sz; + u32 qpc_buf_pg_sz; + u32 qpc_hop_num; + u32 srqc_ba_pg_sz; + u32 srqc_buf_pg_sz; + u32 srqc_hop_num; + u32 cqc_ba_pg_sz; + u32 cqc_buf_pg_sz; + u32 cqc_hop_num; + u32 mpt_ba_pg_sz; + u32 mpt_buf_pg_sz; + u32 mpt_hop_num; + u32 mtt_ba_pg_sz; + u32 mtt_buf_pg_sz; + u32 mtt_hop_num; + u32 wqe_sq_hop_num; + u32 wqe_sge_hop_num; + u32 wqe_rq_hop_num; + u32 sccc_ba_pg_sz; + u32 sccc_buf_pg_sz; + u32 sccc_hop_num; + u32 qpc_timer_ba_pg_sz; + u32 qpc_timer_buf_pg_sz; + u32 qpc_timer_hop_num; + u32 cqc_timer_ba_pg_sz; + u32 cqc_timer_buf_pg_sz; + u32 cqc_timer_hop_num; + u32 cqe_ba_pg_sz; /* page_size = 4K*(2^cqe_ba_pg_sz) */ + u32 cqe_buf_pg_sz; + u32 cqe_hop_num; + u32 srqwqe_ba_pg_sz; + u32 srqwqe_buf_pg_sz; + u32 srqwqe_hop_num; + u32 idx_ba_pg_sz; + u32 idx_buf_pg_sz; + u32 idx_hop_num; + u32 eqe_ba_pg_sz; + u32 eqe_buf_pg_sz; + u32 eqe_hop_num; + u32 sl_num; + u32 tsq_buf_pg_sz; + u32 tpq_buf_pg_sz; + u32 chunk_sz; /* chunk size in non multihop mode */ + u64 flags; + u16 default_ceq_max_cnt; + u16 default_ceq_period; + u16 default_aeq_max_cnt; + u16 default_aeq_period; + u16 default_aeq_arm_st; + u16 default_ceq_arm_st; +}; + +struct hns_roce_dfx_hw { + int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn, + int *buffer); +}; + +enum hns_roce_device_state { + HNS_ROCE_DEVICE_STATE_INITED, + HNS_ROCE_DEVICE_STATE_RST_DOWN, + HNS_ROCE_DEVICE_STATE_UNINIT, +}; + +struct hns_roce_hw { + int (*reset)(struct hns_roce_dev *hr_dev, bool enable); + int (*cmq_init)(struct hns_roce_dev *hr_dev); + void (*cmq_exit)(struct hns_roce_dev *hr_dev); + int (*hw_profile)(struct hns_roce_dev *hr_dev); + int (*hw_init)(struct hns_roce_dev *hr_dev); + void (*hw_exit)(struct hns_roce_dev *hr_dev); + int (*post_mbox)(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, u32 in_modifier, u8 op_modifier, u16 op, + u16 token, int event); + int (*chk_mbox)(struct hns_roce_dev *hr_dev, unsigned long timeout); + int (*rst_prc_mbox)(struct hns_roce_dev *hr_dev); + int (*set_gid)(struct hns_roce_dev *hr_dev, u8 port, int gid_index, + const union ib_gid *gid, const struct ib_gid_attr *attr); + int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr); + void (*set_mtu)(struct hns_roce_dev *hr_dev, u8 phy_port, + enum ib_mtu mtu); + int (*write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf, + struct hns_roce_mr *mr, unsigned long mtpt_idx); + int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev, + struct hns_roce_mr *mr, int flags, u32 pdn, + int mr_access_flags, u64 iova, u64 size, + void *mb_buf); + int (*frmr_write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf, + struct hns_roce_mr *mr); + int (*mw_write_mtpt)(void *mb_buf, struct hns_roce_mw *mw); + void (*write_cqc)(struct hns_roce_dev *hr_dev, + struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts, + dma_addr_t dma_handle); + int (*set_hem)(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, int obj, int step_idx); + int (*clear_hem)(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, int obj, + int step_idx); + int (*query_qp)(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); + int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr, + int attr_mask, enum ib_qp_state cur_state, + enum ib_qp_state new_state); + int (*destroy_qp)(struct ib_qp *ibqp, struct ib_udata *udata); + int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp); + int (*post_send)(struct ib_qp *ibqp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr); + int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, + const struct ib_recv_wr **bad_recv_wr); + int (*req_notify_cq)(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); + int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); + int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, + struct ib_udata *udata); + int (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata); + int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period); + int (*init_eq)(struct hns_roce_dev *hr_dev); + void (*cleanup_eq)(struct hns_roce_dev *hr_dev); + void (*write_srqc)(struct hns_roce_dev *hr_dev, + struct hns_roce_srq *srq, u32 pdn, u16 xrcd, u32 cqn, + void *mb_buf, u64 *mtts_wqe, u64 *mtts_idx, + dma_addr_t dma_handle_wqe, + dma_addr_t dma_handle_idx); + int (*modify_srq)(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr, + enum ib_srq_attr_mask srq_attr_mask, + struct ib_udata *udata); + int (*query_srq)(struct ib_srq *ibsrq, struct ib_srq_attr *attr); + int (*post_srq_recv)(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); + const struct ib_device_ops *hns_roce_dev_ops; + const struct ib_device_ops *hns_roce_dev_srq_ops; +}; + +struct hns_roce_dev { + struct ib_device ib_dev; + struct platform_device *pdev; + struct pci_dev *pci_dev; + struct device *dev; + struct hns_roce_uar priv_uar; + const char *irq_names[HNS_ROCE_MAX_IRQ_NUM]; + spinlock_t sm_lock; + spinlock_t bt_cmd_lock; + bool active; + bool is_reset; + bool dis_db; + unsigned long reset_cnt; + struct hns_roce_ib_iboe iboe; + enum hns_roce_device_state state; + struct list_head qp_list; /* list of all qps on this dev */ + spinlock_t qp_list_lock; /* protect qp_list */ + + struct list_head pgdir_list; + struct mutex pgdir_mutex; + int irq[HNS_ROCE_MAX_IRQ_NUM]; + u8 __iomem *reg_base; + struct hns_roce_caps caps; + struct xarray qp_table_xa; + + unsigned char dev_addr[HNS_ROCE_MAX_PORTS][ETH_ALEN]; + u64 sys_image_guid; + u32 vendor_id; + u32 vendor_part_id; + u32 hw_rev; + void __iomem *priv_addr; + + struct hns_roce_cmdq cmd; + struct hns_roce_bitmap pd_bitmap; + struct hns_roce_uar_table uar_table; + struct hns_roce_mr_table mr_table; + struct hns_roce_cq_table cq_table; + struct hns_roce_srq_table srq_table; + struct hns_roce_qp_table qp_table; + struct hns_roce_eq_table eq_table; + struct hns_roce_hem_table qpc_timer_table; + struct hns_roce_hem_table cqc_timer_table; + + int cmd_mod; + int loop_idc; + u32 sdb_offset; + u32 odb_offset; + dma_addr_t tptr_dma_addr; /* only for hw v1 */ + u32 tptr_size; /* only for hw v1 */ + const struct hns_roce_hw *hw; + void *priv; + struct workqueue_struct *irq_workq; + const struct hns_roce_dfx_hw *dfx; +}; + +static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev) +{ + return container_of(ib_dev, struct hns_roce_dev, ib_dev); +} + +static inline struct hns_roce_ucontext + *to_hr_ucontext(struct ib_ucontext *ibucontext) +{ + return container_of(ibucontext, struct hns_roce_ucontext, ibucontext); +} + +static inline struct hns_roce_pd *to_hr_pd(struct ib_pd *ibpd) +{ + return container_of(ibpd, struct hns_roce_pd, ibpd); +} + +static inline struct hns_roce_ah *to_hr_ah(struct ib_ah *ibah) +{ + return container_of(ibah, struct hns_roce_ah, ibah); +} + +static inline struct hns_roce_mr *to_hr_mr(struct ib_mr *ibmr) +{ + return container_of(ibmr, struct hns_roce_mr, ibmr); +} + +static inline struct hns_roce_mw *to_hr_mw(struct ib_mw *ibmw) +{ + return container_of(ibmw, struct hns_roce_mw, ibmw); +} + +static inline struct hns_roce_qp *to_hr_qp(struct ib_qp *ibqp) +{ + return container_of(ibqp, struct hns_roce_qp, ibqp); +} + +static inline struct hns_roce_cq *to_hr_cq(struct ib_cq *ib_cq) +{ + return container_of(ib_cq, struct hns_roce_cq, ib_cq); +} + +static inline struct hns_roce_srq *to_hr_srq(struct ib_srq *ibsrq) +{ + return container_of(ibsrq, struct hns_roce_srq, ibsrq); +} + +static inline void hns_roce_write64_k(__le32 val[2], void __iomem *dest) +{ + __raw_writeq(*(u64 *) val, dest); +} + +static inline struct hns_roce_qp + *__hns_roce_qp_lookup(struct hns_roce_dev *hr_dev, u32 qpn) +{ + return xa_load(&hr_dev->qp_table_xa, qpn & (hr_dev->caps.num_qps - 1)); +} + +static inline bool hns_roce_buf_is_direct(struct hns_roce_buf *buf) +{ + if (buf->page_list) + return false; + + return true; +} + +static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, int offset) +{ + if (hns_roce_buf_is_direct(buf)) + return (char *)(buf->direct.buf) + (offset & (buf->size - 1)); + + return (char *)(buf->page_list[offset >> buf->page_shift].buf) + + (offset & ((1 << buf->page_shift) - 1)); +} + +static inline dma_addr_t hns_roce_buf_page(struct hns_roce_buf *buf, int idx) +{ + if (hns_roce_buf_is_direct(buf)) + return buf->direct.map + ((dma_addr_t)idx << buf->page_shift); + else + return buf->page_list[idx].map; +} + +#define hr_hw_page_align(x) ALIGN(x, 1 << HNS_HW_PAGE_SHIFT) + +static inline u64 to_hr_hw_page_addr(u64 addr) +{ + return addr >> HNS_HW_PAGE_SHIFT; +} + +static inline u32 to_hr_hw_page_shift(u32 page_shift) +{ + return page_shift - HNS_HW_PAGE_SHIFT; +} + +static inline u32 to_hr_hem_hopnum(u32 hopnum, u32 count) +{ + if (count > 0) + return hopnum == HNS_ROCE_HOP_NUM_0 ? 0 : hopnum; + + return 0; +} + +static inline u32 to_hr_hem_entries_size(u32 count, u32 buf_shift) +{ + return hr_hw_page_align(count << buf_shift); +} + +static inline u32 to_hr_hem_entries_count(u32 count, u32 buf_shift) +{ + return hr_hw_page_align(count << buf_shift) >> buf_shift; +} + +static inline u32 to_hr_hem_entries_shift(u32 count, u32 buf_shift) +{ + if (!count) + return 0; + + return ilog2(to_hr_hem_entries_count(count, buf_shift)); +} + +#define DSCP_SHIFT 2 + +static inline u8 get_tclass(const struct ib_global_route *grh) +{ + return grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP ? + grh->traffic_class >> DSCP_SHIFT : grh->traffic_class; +} + +int hns_roce_init_uar_table(struct hns_roce_dev *dev); +int hns_roce_uar_alloc(struct hns_roce_dev *dev, struct hns_roce_uar *uar); +void hns_roce_uar_free(struct hns_roce_dev *dev, struct hns_roce_uar *uar); +void hns_roce_cleanup_uar_table(struct hns_roce_dev *dev); + +int hns_roce_cmd_init(struct hns_roce_dev *hr_dev); +void hns_roce_cmd_cleanup(struct hns_roce_dev *hr_dev); +void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status, + u64 out_param); +int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev); +void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev); + +/* hns roce hw need current block and next block addr from mtt */ +#define MTT_MIN_COUNT 2 +int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr); +int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + struct hns_roce_buf_attr *buf_attr, + unsigned int page_shift, struct ib_udata *udata, + unsigned long user_addr); +void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, + struct hns_roce_mtr *mtr); +int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + dma_addr_t *pages, int page_cnt); + +int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev); +int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev); +int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev); +int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev); +int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev); + +void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev); +void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev); +void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev); +void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev); +void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev); +void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev); + +int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj); +void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj, + int rr); +int hns_roce_bitmap_init(struct hns_roce_bitmap *bitmap, u32 num, u32 mask, + u32 reserved_bot, u32 resetrved_top); +void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap); +void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev); +int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt, + int align, unsigned long *obj); +void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap, + unsigned long obj, int cnt, + int rr); + +int hns_roce_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr, + struct ib_udata *udata); +int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); +static inline int hns_roce_destroy_ah(struct ib_ah *ah, u32 flags) +{ + return 0; +} + +int hns_roce_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); +int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); + +struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc); +struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, + u64 virt_addr, int access_flags, + struct ib_udata *udata); +int hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, struct ib_pd *pd, + struct ib_udata *udata); +struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, + u32 max_num_sg); +int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, + unsigned int *sg_offset); +int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); +int hns_roce_hw_destroy_mpt(struct hns_roce_dev *hr_dev, + struct hns_roce_cmd_mailbox *mailbox, + unsigned long mpt_index); +unsigned long key_to_hw_index(u32 key); + +int hns_roce_alloc_mw(struct ib_mw *mw, struct ib_udata *udata); +int hns_roce_dealloc_mw(struct ib_mw *ibmw); + +void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf); +int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, + struct hns_roce_buf *buf, u32 page_shift); + +int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct hns_roce_buf *buf); +int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct ib_umem *umem, + unsigned int page_shift); + +int hns_roce_create_srq(struct ib_srq *srq, + struct ib_srq_init_attr *srq_init_attr, + struct ib_udata *udata); +int hns_roce_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr, + enum ib_srq_attr_mask srq_attr_mask, + struct ib_udata *udata); +int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); + +struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata); +int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata); +void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp); +void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, int n); +void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, int n); +void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, int n); +bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, + struct ib_cq *ib_cq); +enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state); +void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, + struct hns_roce_cq *recv_cq); +void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, + struct hns_roce_cq *recv_cq); +void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp); +void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_udata *udata); +__be32 send_ieth(const struct ib_send_wr *wr); +int to_hr_qp_type(int qp_type); + +int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); + +int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); +int hns_roce_db_map_user(struct hns_roce_ucontext *context, + struct ib_udata *udata, unsigned long virt, + struct hns_roce_db *db); +void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, + struct hns_roce_db *db); +int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db, + int order); +void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db); + +void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn); +void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type); +void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type); +void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type); +int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index); +void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev); +int hns_roce_init(struct hns_roce_dev *hr_dev); +void hns_roce_exit(struct hns_roce_dev *hr_dev); + +int hns_roce_fill_res_cq_entry(struct sk_buff *msg, + struct ib_cq *ib_cq); +#endif /* _HNS_ROCE_DEVICE_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c new file mode 100644 index 000000000..854b41c14 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -0,0 +1,1423 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/platform_device.h> +#include "hns_roce_device.h" +#include "hns_roce_hem.h" +#include "hns_roce_common.h" + +#define HEM_INDEX_BUF BIT(0) +#define HEM_INDEX_L0 BIT(1) +#define HEM_INDEX_L1 BIT(2) +struct hns_roce_hem_index { + u64 buf; + u64 l0; + u64 l1; + u32 inited; /* indicate which index is available */ +}; + +bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) +{ + int hop_num = 0; + + switch (type) { + case HEM_TYPE_QPC: + hop_num = hr_dev->caps.qpc_hop_num; + break; + case HEM_TYPE_MTPT: + hop_num = hr_dev->caps.mpt_hop_num; + break; + case HEM_TYPE_CQC: + hop_num = hr_dev->caps.cqc_hop_num; + break; + case HEM_TYPE_SRQC: + hop_num = hr_dev->caps.srqc_hop_num; + break; + case HEM_TYPE_SCCC: + hop_num = hr_dev->caps.sccc_hop_num; + break; + case HEM_TYPE_QPC_TIMER: + hop_num = hr_dev->caps.qpc_timer_hop_num; + break; + case HEM_TYPE_CQC_TIMER: + hop_num = hr_dev->caps.cqc_timer_hop_num; + break; + default: + return false; + } + + return hop_num ? true : false; +} + +static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx, + u32 bt_chunk_num, u64 hem_max_num) +{ + u64 start_idx = round_down(hem_idx, bt_chunk_num); + u64 check_max_num = start_idx + bt_chunk_num; + u64 i; + + for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++) + if (i != hem_idx && hem[i]) + return false; + + return true; +} + +static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num) +{ + u64 start_idx = round_down(ba_idx, bt_chunk_num); + int i; + + for (i = 0; i < bt_chunk_num; i++) + if (i != ba_idx && bt[start_idx + i]) + return false; + + return true; +} + +static int hns_roce_get_bt_num(u32 table_type, u32 hop_num) +{ + if (check_whether_bt_num_3(table_type, hop_num)) + return 3; + else if (check_whether_bt_num_2(table_type, hop_num)) + return 2; + else if (check_whether_bt_num_1(table_type, hop_num)) + return 1; + else + return 0; +} + +static int get_hem_table_config(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_mhop *mhop, + u32 type) +{ + struct device *dev = hr_dev->dev; + + switch (type) { + case HEM_TYPE_QPC: + mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; + mhop->hop_num = hr_dev->caps.qpc_hop_num; + break; + case HEM_TYPE_MTPT: + mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; + mhop->hop_num = hr_dev->caps.mpt_hop_num; + break; + case HEM_TYPE_CQC: + mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; + mhop->hop_num = hr_dev->caps.cqc_hop_num; + break; + case HEM_TYPE_SCCC: + mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.sccc_bt_num; + mhop->hop_num = hr_dev->caps.sccc_hop_num; + break; + case HEM_TYPE_QPC_TIMER: + mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num; + mhop->hop_num = hr_dev->caps.qpc_timer_hop_num; + break; + case HEM_TYPE_CQC_TIMER: + mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num; + mhop->hop_num = hr_dev->caps.cqc_timer_hop_num; + break; + case HEM_TYPE_SRQC: + mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; + mhop->hop_num = hr_dev->caps.srqc_hop_num; + break; + default: + dev_err(dev, "table %u not support multi-hop addressing!\n", + type); + return -EINVAL; + } + + return 0; +} + +int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long *obj, + struct hns_roce_hem_mhop *mhop) +{ + struct device *dev = hr_dev->dev; + u32 chunk_ba_num; + u32 chunk_size; + u32 table_idx; + u32 bt_num; + + if (get_hem_table_config(hr_dev, mhop, table->type)) + return -EINVAL; + + if (!obj) + return 0; + + /* + * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages. + * MTT/CQE alloc hem for bt pages. + */ + bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; + chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : + mhop->bt_chunk_size; + table_idx = (*obj & (table->num_obj - 1)) / + (chunk_size / table->obj_size); + switch (bt_num) { + case 3: + mhop->l2_idx = table_idx & (chunk_ba_num - 1); + mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); + mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num; + break; + case 2: + mhop->l1_idx = table_idx & (chunk_ba_num - 1); + mhop->l0_idx = table_idx / chunk_ba_num; + break; + case 1: + mhop->l0_idx = table_idx; + break; + default: + dev_err(dev, "table %u not support hop_num = %u!\n", + table->type, mhop->hop_num); + return -EINVAL; + } + if (mhop->l0_idx >= mhop->ba_l0_num) + mhop->l0_idx %= mhop->ba_l0_num; + + return 0; +} + +static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, + int npages, + unsigned long hem_alloc_size, + gfp_t gfp_mask) +{ + struct hns_roce_hem_chunk *chunk = NULL; + struct hns_roce_hem *hem; + struct scatterlist *mem; + int order; + void *buf; + + WARN_ON(gfp_mask & __GFP_HIGHMEM); + + hem = kmalloc(sizeof(*hem), + gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); + if (!hem) + return NULL; + + INIT_LIST_HEAD(&hem->chunk_list); + + order = get_order(hem_alloc_size); + + while (npages > 0) { + if (!chunk) { + chunk = kmalloc(sizeof(*chunk), + gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); + if (!chunk) + goto fail; + + sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN); + chunk->npages = 0; + chunk->nsg = 0; + memset(chunk->buf, 0, sizeof(chunk->buf)); + list_add_tail(&chunk->list, &hem->chunk_list); + } + + while (1 << order > npages) + --order; + + /* + * Alloc memory one time. If failed, don't alloc small block + * memory, directly return fail. + */ + mem = &chunk->mem[chunk->npages]; + buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, + &sg_dma_address(mem), gfp_mask); + if (!buf) + goto fail; + + chunk->buf[chunk->npages] = buf; + sg_dma_len(mem) = PAGE_SIZE << order; + + ++chunk->npages; + ++chunk->nsg; + npages -= 1 << order; + } + + return hem; + +fail: + hns_roce_free_hem(hr_dev, hem); + return NULL; +} + +void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) +{ + struct hns_roce_hem_chunk *chunk, *tmp; + int i; + + if (!hem) + return; + + list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { + for (i = 0; i < chunk->npages; ++i) + dma_free_coherent(hr_dev->dev, + sg_dma_len(&chunk->mem[i]), + chunk->buf[i], + sg_dma_address(&chunk->mem[i])); + kfree(chunk); + } + + kfree(hem); +} + +static int calc_hem_config(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj, + struct hns_roce_hem_mhop *mhop, + struct hns_roce_hem_index *index) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + unsigned long mhop_obj = obj; + u32 l0_idx, l1_idx, l2_idx; + u32 chunk_ba_num; + u32 bt_num; + int ret; + + ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop); + if (ret) + return ret; + + l0_idx = mhop->l0_idx; + l1_idx = mhop->l1_idx; + l2_idx = mhop->l2_idx; + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; + bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); + switch (bt_num) { + case 3: + index->l1 = l0_idx * chunk_ba_num + l1_idx; + index->l0 = l0_idx; + index->buf = l0_idx * chunk_ba_num * chunk_ba_num + + l1_idx * chunk_ba_num + l2_idx; + break; + case 2: + index->l0 = l0_idx; + index->buf = l0_idx * chunk_ba_num + l1_idx; + break; + case 1: + index->buf = l0_idx; + break; + default: + ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n", + table->type, mhop->hop_num); + return -EINVAL; + } + + if (unlikely(index->buf >= table->num_hem)) { + ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n", + table->type, index->buf, table->num_hem); + return -EINVAL; + } + + return 0; +} + +static void free_mhop_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + struct hns_roce_hem_mhop *mhop, + struct hns_roce_hem_index *index) +{ + u32 bt_size = mhop->bt_chunk_size; + struct device *dev = hr_dev->dev; + + if (index->inited & HEM_INDEX_BUF) { + hns_roce_free_hem(hr_dev, table->hem[index->buf]); + table->hem[index->buf] = NULL; + } + + if (index->inited & HEM_INDEX_L1) { + dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], + table->bt_l1_dma_addr[index->l1]); + table->bt_l1[index->l1] = NULL; + } + + if (index->inited & HEM_INDEX_L0) { + dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], + table->bt_l0_dma_addr[index->l0]); + table->bt_l0[index->l0] = NULL; + } +} + +static int alloc_mhop_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + struct hns_roce_hem_mhop *mhop, + struct hns_roce_hem_index *index) +{ + u32 bt_size = mhop->bt_chunk_size; + struct device *dev = hr_dev->dev; + struct hns_roce_hem_iter iter; + gfp_t flag; + u64 bt_ba; + u32 size; + int ret; + + /* alloc L1 BA's chunk */ + if ((check_whether_bt_num_3(table->type, mhop->hop_num) || + check_whether_bt_num_2(table->type, mhop->hop_num)) && + !table->bt_l0[index->l0]) { + table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, + &table->bt_l0_dma_addr[index->l0], + GFP_KERNEL); + if (!table->bt_l0[index->l0]) { + ret = -ENOMEM; + goto out; + } + index->inited |= HEM_INDEX_L0; + } + + /* alloc L2 BA's chunk */ + if (check_whether_bt_num_3(table->type, mhop->hop_num) && + !table->bt_l1[index->l1]) { + table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, + &table->bt_l1_dma_addr[index->l1], + GFP_KERNEL); + if (!table->bt_l1[index->l1]) { + ret = -ENOMEM; + goto err_alloc_hem; + } + index->inited |= HEM_INDEX_L1; + *(table->bt_l0[index->l0] + mhop->l1_idx) = + table->bt_l1_dma_addr[index->l1]; + } + + /* + * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. + * alloc bt space chunk for MTT/CQE. + */ + size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; + flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN; + table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, + size, flag); + if (!table->hem[index->buf]) { + ret = -ENOMEM; + goto err_alloc_hem; + } + + index->inited |= HEM_INDEX_BUF; + hns_roce_hem_first(table->hem[index->buf], &iter); + bt_ba = hns_roce_hem_addr(&iter); + if (table->type < HEM_TYPE_MTT) { + if (mhop->hop_num == 2) + *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; + else if (mhop->hop_num == 1) + *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; + } else if (mhop->hop_num == 2) { + *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; + } + + return 0; +err_alloc_hem: + free_mhop_hem(hr_dev, table, mhop, index); +out: + return ret; +} + +static int set_mhop_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj, + struct hns_roce_hem_mhop *mhop, + struct hns_roce_hem_index *index) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + int step_idx; + int ret = 0; + + if (index->inited & HEM_INDEX_L0) { + ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); + if (ret) { + ibdev_err(ibdev, "set HEM step 0 failed!\n"); + goto out; + } + } + + if (index->inited & HEM_INDEX_L1) { + ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); + if (ret) { + ibdev_err(ibdev, "set HEM step 1 failed!\n"); + goto out; + } + } + + if (index->inited & HEM_INDEX_BUF) { + if (mhop->hop_num == HNS_ROCE_HOP_NUM_0) + step_idx = 0; + else + step_idx = mhop->hop_num; + ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); + if (ret) + ibdev_err(ibdev, "set HEM step last failed!\n"); + } +out: + return ret; +} + +static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + unsigned long obj) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_hem_index index = {}; + struct hns_roce_hem_mhop mhop = {}; + int ret; + + ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); + if (ret) { + ibdev_err(ibdev, "calc hem config failed!\n"); + return ret; + } + + mutex_lock(&table->mutex); + if (table->hem[index.buf]) { + refcount_inc(&table->hem[index.buf]->refcount); + goto out; + } + + ret = alloc_mhop_hem(hr_dev, table, &mhop, &index); + if (ret) { + ibdev_err(ibdev, "alloc mhop hem failed!\n"); + goto out; + } + + /* set HEM base address to hardware */ + if (table->type < HEM_TYPE_MTT) { + ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index); + if (ret) { + ibdev_err(ibdev, "set HEM address to HW failed!\n"); + goto err_alloc; + } + } + + refcount_set(&table->hem[index.buf]->refcount, 1); + goto out; + +err_alloc: + free_mhop_hem(hr_dev, table, &mhop, &index); +out: + mutex_unlock(&table->mutex); + return ret; +} + +int hns_roce_table_get(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj) +{ + struct device *dev = hr_dev->dev; + unsigned long i; + int ret = 0; + + if (hns_roce_check_whether_mhop(hr_dev, table->type)) + return hns_roce_table_mhop_get(hr_dev, table, obj); + + i = (obj & (table->num_obj - 1)) / (table->table_chunk_size / + table->obj_size); + + mutex_lock(&table->mutex); + + if (table->hem[i]) { + refcount_inc(&table->hem[i]->refcount); + goto out; + } + + table->hem[i] = hns_roce_alloc_hem(hr_dev, + table->table_chunk_size >> PAGE_SHIFT, + table->table_chunk_size, + (table->lowmem ? GFP_KERNEL : + GFP_HIGHUSER) | __GFP_NOWARN); + if (!table->hem[i]) { + ret = -ENOMEM; + goto out; + } + + /* Set HEM base address(128K/page, pa) to Hardware */ + ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT); + if (ret) { + hns_roce_free_hem(hr_dev, table->hem[i]); + table->hem[i] = NULL; + dev_err(dev, "set HEM base address to HW failed, ret = %d.\n", + ret); + goto out; + } + + refcount_set(&table->hem[i]->refcount, 1); +out: + mutex_unlock(&table->mutex); + return ret; +} + +static void clear_mhop_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj, + struct hns_roce_hem_mhop *mhop, + struct hns_roce_hem_index *index) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + u32 hop_num = mhop->hop_num; + u32 chunk_ba_num; + int step_idx; + + index->inited = HEM_INDEX_BUF; + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; + if (check_whether_bt_num_2(table->type, hop_num)) { + if (hns_roce_check_hem_null(table->hem, index->buf, + chunk_ba_num, table->num_hem)) + index->inited |= HEM_INDEX_L0; + } else if (check_whether_bt_num_3(table->type, hop_num)) { + if (hns_roce_check_hem_null(table->hem, index->buf, + chunk_ba_num, table->num_hem)) { + index->inited |= HEM_INDEX_L1; + if (hns_roce_check_bt_null(table->bt_l1, index->l1, + chunk_ba_num)) + index->inited |= HEM_INDEX_L0; + } + } + + if (table->type < HEM_TYPE_MTT) { + if (hop_num == HNS_ROCE_HOP_NUM_0) + step_idx = 0; + else + step_idx = hop_num; + + if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx)) + ibdev_warn(ibdev, "failed to clear hop%u HEM.\n", hop_num); + + if (index->inited & HEM_INDEX_L1) + if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) + ibdev_warn(ibdev, "failed to clear HEM step 1.\n"); + + if (index->inited & HEM_INDEX_L0) + if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) + ibdev_warn(ibdev, "failed to clear HEM step 0.\n"); + } +} + +static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + unsigned long obj, + int check_refcount) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_hem_index index = {}; + struct hns_roce_hem_mhop mhop = {}; + int ret; + + ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); + if (ret) { + ibdev_err(ibdev, "calc hem config failed!\n"); + return; + } + + if (!check_refcount) + mutex_lock(&table->mutex); + else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, + &table->mutex)) + return; + + clear_mhop_hem(hr_dev, table, obj, &mhop, &index); + free_mhop_hem(hr_dev, table, &mhop, &index); + + mutex_unlock(&table->mutex); +} + +void hns_roce_table_put(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj) +{ + struct device *dev = hr_dev->dev; + unsigned long i; + + if (hns_roce_check_whether_mhop(hr_dev, table->type)) { + hns_roce_table_mhop_put(hr_dev, table, obj, 1); + return; + } + + i = (obj & (table->num_obj - 1)) / + (table->table_chunk_size / table->obj_size); + + if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, + &table->mutex)) + return; + + if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) + dev_warn(dev, "failed to clear HEM base address.\n"); + + hns_roce_free_hem(hr_dev, table->hem[i]); + table->hem[i] = NULL; + + mutex_unlock(&table->mutex); +} + +void *hns_roce_table_find(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, + unsigned long obj, dma_addr_t *dma_handle) +{ + struct hns_roce_hem_chunk *chunk; + struct hns_roce_hem_mhop mhop; + struct hns_roce_hem *hem; + unsigned long mhop_obj = obj; + unsigned long obj_per_chunk; + unsigned long idx_offset; + int offset, dma_offset; + void *addr = NULL; + u32 hem_idx = 0; + int length; + int i, j; + + if (!table->lowmem) + return NULL; + + mutex_lock(&table->mutex); + + if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { + obj_per_chunk = table->table_chunk_size / table->obj_size; + hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk]; + idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk; + dma_offset = offset = idx_offset * table->obj_size; + } else { + u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */ + + if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop)) + goto out; + /* mtt mhop */ + i = mhop.l0_idx; + j = mhop.l1_idx; + if (mhop.hop_num == 2) + hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j; + else if (mhop.hop_num == 1 || + mhop.hop_num == HNS_ROCE_HOP_NUM_0) + hem_idx = i; + + hem = table->hem[hem_idx]; + dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size % + mhop.bt_chunk_size; + if (mhop.hop_num == 2) + dma_offset = offset = 0; + } + + if (!hem) + goto out; + + list_for_each_entry(chunk, &hem->chunk_list, list) { + for (i = 0; i < chunk->npages; ++i) { + length = sg_dma_len(&chunk->mem[i]); + if (dma_handle && dma_offset >= 0) { + if (length > (u32)dma_offset) + *dma_handle = sg_dma_address( + &chunk->mem[i]) + dma_offset; + dma_offset -= length; + } + + if (length > (u32)offset) { + addr = chunk->buf[i] + offset; + goto out; + } + offset -= length; + } + } + +out: + mutex_unlock(&table->mutex); + return addr; +} + +int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, u32 type, + unsigned long obj_size, unsigned long nobj, + int use_lowmem) +{ + unsigned long obj_per_chunk; + unsigned long num_hem; + + if (!hns_roce_check_whether_mhop(hr_dev, type)) { + table->table_chunk_size = hr_dev->caps.chunk_sz; + obj_per_chunk = table->table_chunk_size / obj_size; + num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; + + table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); + if (!table->hem) + return -ENOMEM; + } else { + struct hns_roce_hem_mhop mhop = {}; + unsigned long buf_chunk_size; + unsigned long bt_chunk_size; + unsigned long bt_chunk_num; + unsigned long num_bt_l0 = 0; + u32 hop_num; + + if (get_hem_table_config(hr_dev, &mhop, type)) + return -EINVAL; + + buf_chunk_size = mhop.buf_chunk_size; + bt_chunk_size = mhop.bt_chunk_size; + num_bt_l0 = mhop.ba_l0_num; + hop_num = mhop.hop_num; + + obj_per_chunk = buf_chunk_size / obj_size; + num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; + bt_chunk_num = bt_chunk_size / BA_BYTE_LEN; + if (type >= HEM_TYPE_MTT) + num_bt_l0 = bt_chunk_num; + + table->hem = kcalloc(num_hem, sizeof(*table->hem), + GFP_KERNEL); + if (!table->hem) + goto err_kcalloc_hem_buf; + + if (check_whether_bt_num_3(type, hop_num)) { + unsigned long num_bt_l1; + + num_bt_l1 = (num_hem + bt_chunk_num - 1) / + bt_chunk_num; + table->bt_l1 = kcalloc(num_bt_l1, + sizeof(*table->bt_l1), + GFP_KERNEL); + if (!table->bt_l1) + goto err_kcalloc_bt_l1; + + table->bt_l1_dma_addr = kcalloc(num_bt_l1, + sizeof(*table->bt_l1_dma_addr), + GFP_KERNEL); + + if (!table->bt_l1_dma_addr) + goto err_kcalloc_l1_dma; + } + + if (check_whether_bt_num_2(type, hop_num) || + check_whether_bt_num_3(type, hop_num)) { + table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), + GFP_KERNEL); + if (!table->bt_l0) + goto err_kcalloc_bt_l0; + + table->bt_l0_dma_addr = kcalloc(num_bt_l0, + sizeof(*table->bt_l0_dma_addr), + GFP_KERNEL); + if (!table->bt_l0_dma_addr) + goto err_kcalloc_l0_dma; + } + } + + table->type = type; + table->num_hem = num_hem; + table->num_obj = nobj; + table->obj_size = obj_size; + table->lowmem = use_lowmem; + mutex_init(&table->mutex); + + return 0; + +err_kcalloc_l0_dma: + kfree(table->bt_l0); + table->bt_l0 = NULL; + +err_kcalloc_bt_l0: + kfree(table->bt_l1_dma_addr); + table->bt_l1_dma_addr = NULL; + +err_kcalloc_l1_dma: + kfree(table->bt_l1); + table->bt_l1 = NULL; + +err_kcalloc_bt_l1: + kfree(table->hem); + table->hem = NULL; + +err_kcalloc_hem_buf: + return -ENOMEM; +} + +static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table) +{ + struct hns_roce_hem_mhop mhop; + u32 buf_chunk_size; + u64 obj; + int i; + + if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) + return; + buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : + mhop.bt_chunk_size; + + for (i = 0; i < table->num_hem; ++i) { + obj = i * buf_chunk_size / table->obj_size; + if (table->hem[i]) + hns_roce_table_mhop_put(hr_dev, table, obj, 0); + } + + kfree(table->hem); + table->hem = NULL; + kfree(table->bt_l1); + table->bt_l1 = NULL; + kfree(table->bt_l1_dma_addr); + table->bt_l1_dma_addr = NULL; + kfree(table->bt_l0); + table->bt_l0 = NULL; + kfree(table->bt_l0_dma_addr); + table->bt_l0_dma_addr = NULL; +} + +void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table) +{ + struct device *dev = hr_dev->dev; + unsigned long i; + + if (hns_roce_check_whether_mhop(hr_dev, table->type)) { + hns_roce_cleanup_mhop_hem_table(hr_dev, table); + return; + } + + for (i = 0; i < table->num_hem; ++i) + if (table->hem[i]) { + if (hr_dev->hw->clear_hem(hr_dev, table, + i * table->table_chunk_size / table->obj_size, 0)) + dev_err(dev, "Clear HEM base address failed.\n"); + + hns_roce_free_hem(hr_dev, table->hem[i]); + } + + kfree(table->hem); +} + +void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) +{ + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) + hns_roce_cleanup_hem_table(hr_dev, + &hr_dev->srq_table.table); + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); + if (hr_dev->caps.qpc_timer_entry_sz) + hns_roce_cleanup_hem_table(hr_dev, + &hr_dev->qpc_timer_table); + if (hr_dev->caps.cqc_timer_entry_sz) + hns_roce_cleanup_hem_table(hr_dev, + &hr_dev->cqc_timer_table); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) + hns_roce_cleanup_hem_table(hr_dev, + &hr_dev->qp_table.sccc_table); + if (hr_dev->caps.trrl_entry_sz) + hns_roce_cleanup_hem_table(hr_dev, + &hr_dev->qp_table.trrl_table); + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); +} + +struct roce_hem_item { + struct list_head list; /* link all hems in the same bt level */ + struct list_head sibling; /* link all hems in last hop for mtt */ + void *addr; + dma_addr_t dma_addr; + size_t count; /* max ba numbers */ + int start; /* start buf offset in this hem */ + int end; /* end buf offset in this hem */ +}; + +static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev, + int start, int end, + int count, bool exist_bt, + int bt_level) +{ + struct roce_hem_item *hem; + + hem = kzalloc(sizeof(*hem), GFP_KERNEL); + if (!hem) + return NULL; + + if (exist_bt) { + hem->addr = dma_alloc_coherent(hr_dev->dev, + count * BA_BYTE_LEN, + &hem->dma_addr, GFP_KERNEL); + if (!hem->addr) { + kfree(hem); + return NULL; + } + } + + hem->count = count; + hem->start = start; + hem->end = end; + INIT_LIST_HEAD(&hem->list); + INIT_LIST_HEAD(&hem->sibling); + + return hem; +} + +static void hem_list_free_item(struct hns_roce_dev *hr_dev, + struct roce_hem_item *hem, bool exist_bt) +{ + if (exist_bt) + dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, + hem->addr, hem->dma_addr); + kfree(hem); +} + +static void hem_list_free_all(struct hns_roce_dev *hr_dev, + struct list_head *head, bool exist_bt) +{ + struct roce_hem_item *hem, *temp_hem; + + list_for_each_entry_safe(hem, temp_hem, head, list) { + list_del(&hem->list); + hem_list_free_item(hr_dev, hem, exist_bt); + } +} + +static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, + u64 table_addr) +{ + *(u64 *)(base_addr) = table_addr; +} + +/* assign L0 table address to hem from root bt */ +static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, + struct roce_hem_item *hem, void *cpu_addr, + u64 phy_addr) +{ + hem->addr = cpu_addr; + hem->dma_addr = (dma_addr_t)phy_addr; +} + +static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem, + int offset) +{ + return (hem->start <= offset && offset <= hem->end); +} + +static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list, + int page_offset) +{ + struct roce_hem_item *hem, *temp_hem; + struct roce_hem_item *found = NULL; + + list_for_each_entry_safe(hem, temp_hem, ba_list, list) { + if (hem_list_page_is_in_range(hem, page_offset)) { + found = hem; + break; + } + } + + return found; +} + +static bool hem_list_is_bottom_bt(int hopnum, int bt_level) +{ + /* + * hopnum base address table levels + * 0 L0(buf) + * 1 L0 -> buf + * 2 L0 -> L1 -> buf + * 3 L0 -> L1 -> L2 -> buf + */ + return bt_level >= (hopnum ? hopnum - 1 : hopnum); +} + +/** + * calc base address entries num + * @hopnum: num of mutihop addressing + * @bt_level: base address table level + * @unit: ba entries per bt page + */ +static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit) +{ + u32 step; + int max; + int i; + + if (hopnum <= bt_level) + return 0; + /* + * hopnum bt_level range + * 1 0 unit + * ------------ + * 2 0 unit * unit + * 2 1 unit + * ------------ + * 3 0 unit * unit * unit + * 3 1 unit * unit + * 3 2 unit + */ + step = 1; + max = hopnum - bt_level; + for (i = 0; i < max; i++) + step = step * unit; + + return step; +} + +/** + * calc the root ba entries which could cover all regions + * @regions: buf region array + * @region_cnt: array size of @regions + * @unit: ba entries per bt page + */ +int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions, + int region_cnt, int unit) +{ + struct hns_roce_buf_region *r; + int total = 0; + int step; + int i; + + for (i = 0; i < region_cnt; i++) { + r = (struct hns_roce_buf_region *)®ions[i]; + if (r->hopnum > 1) { + step = hem_list_calc_ba_range(r->hopnum, 1, unit); + if (step > 0) + total += (r->count + step - 1) / step; + } else { + total += r->count; + } + } + + return total; +} + +static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, + const struct hns_roce_buf_region *r, int unit, + int offset, struct list_head *mid_bt, + struct list_head *btm_bt) +{ + struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; + struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL]; + struct roce_hem_item *cur, *pre; + const int hopnum = r->hopnum; + int start_aligned; + int distance; + int ret = 0; + int max_ofs; + int level; + u32 step; + int end; + + if (hopnum <= 1) + return 0; + + if (hopnum > HNS_ROCE_MAX_BT_LEVEL) { + dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); + return -EINVAL; + } + + if (offset < r->offset) { + dev_err(hr_dev->dev, "invalid offset %d, min %u!\n", + offset, r->offset); + return -EINVAL; + } + + distance = offset - r->offset; + max_ofs = r->offset + r->count - 1; + for (level = 0; level < hopnum; level++) + INIT_LIST_HEAD(&temp_list[level]); + + /* config L1 bt to last bt and link them to corresponding parent */ + for (level = 1; level < hopnum; level++) { + cur = hem_list_search_item(&mid_bt[level], offset); + if (cur) { + hem_ptrs[level] = cur; + continue; + } + + step = hem_list_calc_ba_range(hopnum, level, unit); + if (step < 1) { + ret = -EINVAL; + goto err_exit; + } + + start_aligned = (distance / step) * step + r->offset; + end = min_t(int, start_aligned + step - 1, max_ofs); + cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, + true, level); + if (!cur) { + ret = -ENOMEM; + goto err_exit; + } + hem_ptrs[level] = cur; + list_add(&cur->list, &temp_list[level]); + if (hem_list_is_bottom_bt(hopnum, level)) + list_add(&cur->sibling, &temp_list[0]); + + /* link bt to parent bt */ + if (level > 1) { + pre = hem_ptrs[level - 1]; + step = (cur->start - pre->start) / step * BA_BYTE_LEN; + hem_list_link_bt(hr_dev, pre->addr + step, + cur->dma_addr); + } + } + + list_splice(&temp_list[0], btm_bt); + for (level = 1; level < hopnum; level++) + list_splice(&temp_list[level], &mid_bt[level]); + + return 0; + +err_exit: + for (level = 1; level < hopnum; level++) + hem_list_free_all(hr_dev, &temp_list[level], true); + + return ret; +} + +static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, int unit, + const struct hns_roce_buf_region *regions, + int region_cnt) +{ + struct list_head temp_list[HNS_ROCE_MAX_BT_REGION]; + struct roce_hem_item *hem, *temp_hem, *root_hem; + const struct hns_roce_buf_region *r; + struct list_head temp_root; + struct list_head temp_btm; + void *cpu_base; + u64 phy_base; + int ret = 0; + int ba_num; + int offset; + int total; + int step; + int i; + + r = ®ions[0]; + root_hem = hem_list_search_item(&hem_list->root_bt, r->offset); + if (root_hem) + return 0; + + ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit); + if (ba_num < 1) + return -ENOMEM; + + INIT_LIST_HEAD(&temp_root); + offset = r->offset; + /* indicate to last region */ + r = ®ions[region_cnt - 1]; + root_hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, + ba_num, true, 0); + if (!root_hem) + return -ENOMEM; + list_add(&root_hem->list, &temp_root); + + hem_list->root_ba = root_hem->dma_addr; + + INIT_LIST_HEAD(&temp_btm); + for (i = 0; i < region_cnt; i++) + INIT_LIST_HEAD(&temp_list[i]); + + total = 0; + for (i = 0; i < region_cnt && total < ba_num; i++) { + r = ®ions[i]; + if (!r->count) + continue; + + /* all regions's mid[x][0] shared the root_bt's trunk */ + cpu_base = root_hem->addr + total * BA_BYTE_LEN; + phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; + + /* if hopnum is 0 or 1, cut a new fake hem from the root bt + * which's address share to all regions. + */ + if (hem_list_is_bottom_bt(r->hopnum, 0)) { + hem = hem_list_alloc_item(hr_dev, r->offset, + r->offset + r->count - 1, + r->count, false, 0); + if (!hem) { + ret = -ENOMEM; + goto err_exit; + } + hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); + list_add(&hem->list, &temp_list[i]); + list_add(&hem->sibling, &temp_btm); + total += r->count; + } else { + step = hem_list_calc_ba_range(r->hopnum, 1, unit); + if (step < 1) { + ret = -EINVAL; + goto err_exit; + } + /* if exist mid bt, link L1 to L0 */ + list_for_each_entry_safe(hem, temp_hem, + &hem_list->mid_bt[i][1], list) { + offset = (hem->start - r->offset) / step * + BA_BYTE_LEN; + hem_list_link_bt(hr_dev, cpu_base + offset, + hem->dma_addr); + total++; + } + } + } + + list_splice(&temp_btm, &hem_list->btm_bt); + list_splice(&temp_root, &hem_list->root_bt); + for (i = 0; i < region_cnt; i++) + list_splice(&temp_list[i], &hem_list->mid_bt[i][0]); + + return 0; + +err_exit: + for (i = 0; i < region_cnt; i++) + hem_list_free_all(hr_dev, &temp_list[i], false); + + hem_list_free_all(hr_dev, &temp_root, true); + + return ret; +} + +/* construct the base address table and link them by address hop config */ +int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + const struct hns_roce_buf_region *regions, + int region_cnt, unsigned int bt_pg_shift) +{ + const struct hns_roce_buf_region *r; + int ofs, end; + int unit; + int ret; + int i; + + if (region_cnt > HNS_ROCE_MAX_BT_REGION) { + dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", + region_cnt); + return -EINVAL; + } + + unit = (1 << bt_pg_shift) / BA_BYTE_LEN; + for (i = 0; i < region_cnt; i++) { + r = ®ions[i]; + if (!r->count) + continue; + + end = r->offset + r->count; + for (ofs = r->offset; ofs < end; ofs += unit) { + ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, + hem_list->mid_bt[i], + &hem_list->btm_bt); + if (ret) { + dev_err(hr_dev->dev, + "alloc hem trunk fail ret=%d!\n", ret); + goto err_alloc; + } + } + } + + ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, + region_cnt); + if (ret) + dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret); + else + return 0; + +err_alloc: + hns_roce_hem_list_release(hr_dev, hem_list); + + return ret; +} + +void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list) +{ + int i, j; + + for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) + for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) + hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j], + j != 0); + + hem_list_free_all(hr_dev, &hem_list->root_bt, true); + INIT_LIST_HEAD(&hem_list->btm_bt); + hem_list->root_ba = 0; +} + +void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list) +{ + int i, j; + + INIT_LIST_HEAD(&hem_list->root_bt); + INIT_LIST_HEAD(&hem_list->btm_bt); + for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) + for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) + INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); +} + +void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + int offset, int *mtt_cnt, u64 *phy_addr) +{ + struct list_head *head = &hem_list->btm_bt; + struct roce_hem_item *hem, *temp_hem; + void *cpu_base = NULL; + u64 phy_base = 0; + int nr = 0; + + list_for_each_entry_safe(hem, temp_hem, head, sibling) { + if (hem_list_page_is_in_range(hem, offset)) { + nr = offset - hem->start; + cpu_base = hem->addr + nr * BA_BYTE_LEN; + phy_base = hem->dma_addr + nr * BA_BYTE_LEN; + nr = hem->end + 1 - offset; + break; + } + } + + if (mtt_cnt) + *mtt_cnt = nr; + + if (phy_addr) + *phy_addr = phy_base; + + return cpu_base; +} diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h new file mode 100644 index 000000000..b7617786b --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_hem.h @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _HNS_ROCE_HEM_H +#define _HNS_ROCE_HEM_H + +#define HEM_HOP_STEP_DIRECT 0xff + +enum { + /* MAP HEM(Hardware Entry Memory) */ + HEM_TYPE_QPC = 0, + HEM_TYPE_MTPT, + HEM_TYPE_CQC, + HEM_TYPE_SRQC, + HEM_TYPE_SCCC, + HEM_TYPE_QPC_TIMER, + HEM_TYPE_CQC_TIMER, + + /* UNMAP HEM */ + HEM_TYPE_MTT, + HEM_TYPE_CQE, + HEM_TYPE_SRQWQE, + HEM_TYPE_IDX, + HEM_TYPE_IRRL, + HEM_TYPE_TRRL, +}; + +#define HNS_ROCE_HEM_CHUNK_LEN \ + ((256 - sizeof(struct list_head) - 2 * sizeof(int)) / \ + (sizeof(struct scatterlist) + sizeof(void *))) + +#define check_whether_bt_num_3(type, hop_num) \ + (type < HEM_TYPE_MTT && hop_num == 2) + +#define check_whether_bt_num_2(type, hop_num) \ + ((type < HEM_TYPE_MTT && hop_num == 1) || \ + (type >= HEM_TYPE_MTT && hop_num == 2)) + +#define check_whether_bt_num_1(type, hop_num) \ + ((type < HEM_TYPE_MTT && hop_num == HNS_ROCE_HOP_NUM_0) || \ + (type >= HEM_TYPE_MTT && hop_num == 1) || \ + (type >= HEM_TYPE_MTT && hop_num == HNS_ROCE_HOP_NUM_0)) + +struct hns_roce_hem_chunk { + struct list_head list; + int npages; + int nsg; + struct scatterlist mem[HNS_ROCE_HEM_CHUNK_LEN]; + void *buf[HNS_ROCE_HEM_CHUNK_LEN]; +}; + +struct hns_roce_hem { + struct list_head chunk_list; + refcount_t refcount; +}; + +struct hns_roce_hem_iter { + struct hns_roce_hem *hem; + struct hns_roce_hem_chunk *chunk; + int page_idx; +}; + +struct hns_roce_hem_mhop { + u32 hop_num; + u32 buf_chunk_size; + u32 bt_chunk_size; + u32 ba_l0_num; + u32 l0_idx; /* level 0 base address table index */ + u32 l1_idx; /* level 1 base address table index */ + u32 l2_idx; /* level 2 base address table index */ +}; + +void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem); +int hns_roce_table_get(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj); +void hns_roce_table_put(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj); +void *hns_roce_table_find(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long obj, + dma_addr_t *dma_handle); +int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, u32 type, + unsigned long obj_size, unsigned long nobj, + int use_lowmem); +void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table); +void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev); +int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, unsigned long *obj, + struct hns_roce_hem_mhop *mhop); +bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type); + +void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list); +int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions, + int region_cnt, int unit); +int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + const struct hns_roce_buf_region *regions, + int region_cnt, unsigned int bt_pg_shift); +void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list); +void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + int offset, int *mtt_cnt, u64 *phy_addr); + +static inline void hns_roce_hem_first(struct hns_roce_hem *hem, + struct hns_roce_hem_iter *iter) +{ + iter->hem = hem; + iter->chunk = list_empty(&hem->chunk_list) ? NULL : + list_entry(hem->chunk_list.next, + struct hns_roce_hem_chunk, list); + iter->page_idx = 0; +} + +static inline int hns_roce_hem_last(struct hns_roce_hem_iter *iter) +{ + return !iter->chunk; +} + +static inline void hns_roce_hem_next(struct hns_roce_hem_iter *iter) +{ + if (++iter->page_idx >= iter->chunk->nsg) { + if (iter->chunk->list.next == &iter->hem->chunk_list) { + iter->chunk = NULL; + return; + } + + iter->chunk = list_entry(iter->chunk->list.next, + struct hns_roce_hem_chunk, list); + iter->page_idx = 0; + } +} + +static inline dma_addr_t hns_roce_hem_addr(struct hns_roce_hem_iter *iter) +{ + return sg_dma_address(&iter->chunk->mem[iter->page_idx]); +} + +#endif /* _HNS_ROCE_HEM_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c new file mode 100644 index 000000000..6f9b024d4 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -0,0 +1,4689 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/platform_device.h> +#include <linux/acpi.h> +#include <linux/etherdevice.h> +#include <linux/interrupt.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <rdma/ib_umem.h> +#include "hns_roce_common.h" +#include "hns_roce_device.h" +#include "hns_roce_cmd.h" +#include "hns_roce_hem.h" +#include "hns_roce_hw_v1.h" + +static void set_data_seg(struct hns_roce_wqe_data_seg *dseg, struct ib_sge *sg) +{ + dseg->lkey = cpu_to_le32(sg->lkey); + dseg->addr = cpu_to_le64(sg->addr); + dseg->len = cpu_to_le32(sg->length); +} + +static void set_raddr_seg(struct hns_roce_wqe_raddr_seg *rseg, u64 remote_addr, + u32 rkey) +{ + rseg->raddr = cpu_to_le64(remote_addr); + rseg->rkey = cpu_to_le32(rkey); + rseg->len = 0; +} + +static int hns_roce_v1_post_send(struct ib_qp *ibqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah); + struct hns_roce_ud_send_wqe *ud_sq_wqe = NULL; + struct hns_roce_wqe_ctrl_seg *ctrl = NULL; + struct hns_roce_wqe_data_seg *dseg = NULL; + struct hns_roce_qp *qp = to_hr_qp(ibqp); + struct device *dev = &hr_dev->pdev->dev; + struct hns_roce_sq_db sq_db = {}; + int ps_opcode, i; + unsigned long flags = 0; + void *wqe = NULL; + __le32 doorbell[2]; + int ret = 0; + int loopback; + u32 wqe_idx; + int nreq; + u8 *smac; + + if (unlikely(ibqp->qp_type != IB_QPT_GSI && + ibqp->qp_type != IB_QPT_RC)) { + dev_err(dev, "un-supported QP type\n"); + *bad_wr = NULL; + return -EOPNOTSUPP; + } + + spin_lock_irqsave(&qp->sq.lock, flags); + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { + ret = -ENOMEM; + *bad_wr = wr; + goto out; + } + + wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1); + + if (unlikely(wr->num_sge > qp->sq.max_gs)) { + dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n", + wr->num_sge, qp->sq.max_gs); + ret = -EINVAL; + *bad_wr = wr; + goto out; + } + + wqe = hns_roce_get_send_wqe(qp, wqe_idx); + qp->sq.wrid[wqe_idx] = wr->wr_id; + + /* Corresponding to the RC and RD type wqe process separately */ + if (ibqp->qp_type == IB_QPT_GSI) { + ud_sq_wqe = wqe; + roce_set_field(ud_sq_wqe->dmac_h, + UD_SEND_WQE_U32_4_DMAC_0_M, + UD_SEND_WQE_U32_4_DMAC_0_S, + ah->av.mac[0]); + roce_set_field(ud_sq_wqe->dmac_h, + UD_SEND_WQE_U32_4_DMAC_1_M, + UD_SEND_WQE_U32_4_DMAC_1_S, + ah->av.mac[1]); + roce_set_field(ud_sq_wqe->dmac_h, + UD_SEND_WQE_U32_4_DMAC_2_M, + UD_SEND_WQE_U32_4_DMAC_2_S, + ah->av.mac[2]); + roce_set_field(ud_sq_wqe->dmac_h, + UD_SEND_WQE_U32_4_DMAC_3_M, + UD_SEND_WQE_U32_4_DMAC_3_S, + ah->av.mac[3]); + + roce_set_field(ud_sq_wqe->u32_8, + UD_SEND_WQE_U32_8_DMAC_4_M, + UD_SEND_WQE_U32_8_DMAC_4_S, + ah->av.mac[4]); + roce_set_field(ud_sq_wqe->u32_8, + UD_SEND_WQE_U32_8_DMAC_5_M, + UD_SEND_WQE_U32_8_DMAC_5_S, + ah->av.mac[5]); + + smac = (u8 *)hr_dev->dev_addr[qp->port]; + loopback = ether_addr_equal_unaligned(ah->av.mac, + smac) ? 1 : 0; + roce_set_bit(ud_sq_wqe->u32_8, + UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S, + loopback); + + roce_set_field(ud_sq_wqe->u32_8, + UD_SEND_WQE_U32_8_OPERATION_TYPE_M, + UD_SEND_WQE_U32_8_OPERATION_TYPE_S, + HNS_ROCE_WQE_OPCODE_SEND); + roce_set_field(ud_sq_wqe->u32_8, + UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M, + UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S, + 2); + roce_set_bit(ud_sq_wqe->u32_8, + UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S, + 1); + + ud_sq_wqe->u32_8 |= (wr->send_flags & IB_SEND_SIGNALED ? + cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) | + (wr->send_flags & IB_SEND_SOLICITED ? + cpu_to_le32(HNS_ROCE_WQE_SE) : 0) | + ((wr->opcode == IB_WR_SEND_WITH_IMM) ? + cpu_to_le32(HNS_ROCE_WQE_IMM) : 0); + + roce_set_field(ud_sq_wqe->u32_16, + UD_SEND_WQE_U32_16_DEST_QP_M, + UD_SEND_WQE_U32_16_DEST_QP_S, + ud_wr(wr)->remote_qpn); + roce_set_field(ud_sq_wqe->u32_16, + UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M, + UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S, + ah->av.stat_rate); + + roce_set_field(ud_sq_wqe->u32_36, + UD_SEND_WQE_U32_36_FLOW_LABEL_M, + UD_SEND_WQE_U32_36_FLOW_LABEL_S, + ah->av.flowlabel); + roce_set_field(ud_sq_wqe->u32_36, + UD_SEND_WQE_U32_36_PRIORITY_M, + UD_SEND_WQE_U32_36_PRIORITY_S, + ah->av.sl); + roce_set_field(ud_sq_wqe->u32_36, + UD_SEND_WQE_U32_36_SGID_INDEX_M, + UD_SEND_WQE_U32_36_SGID_INDEX_S, + hns_get_gid_index(hr_dev, qp->phy_port, + ah->av.gid_index)); + + roce_set_field(ud_sq_wqe->u32_40, + UD_SEND_WQE_U32_40_HOP_LIMIT_M, + UD_SEND_WQE_U32_40_HOP_LIMIT_S, + ah->av.hop_limit); + roce_set_field(ud_sq_wqe->u32_40, + UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M, + UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S, + ah->av.tclass); + + memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN); + + ud_sq_wqe->va0_l = + cpu_to_le32((u32)wr->sg_list[0].addr); + ud_sq_wqe->va0_h = + cpu_to_le32((wr->sg_list[0].addr) >> 32); + ud_sq_wqe->l_key0 = + cpu_to_le32(wr->sg_list[0].lkey); + + ud_sq_wqe->va1_l = + cpu_to_le32((u32)wr->sg_list[1].addr); + ud_sq_wqe->va1_h = + cpu_to_le32((wr->sg_list[1].addr) >> 32); + ud_sq_wqe->l_key1 = + cpu_to_le32(wr->sg_list[1].lkey); + } else if (ibqp->qp_type == IB_QPT_RC) { + u32 tmp_len = 0; + + ctrl = wqe; + memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg)); + for (i = 0; i < wr->num_sge; i++) + tmp_len += wr->sg_list[i].length; + + ctrl->msg_length = + cpu_to_le32(le32_to_cpu(ctrl->msg_length) + tmp_len); + + ctrl->sgl_pa_h = 0; + ctrl->flag = 0; + + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_RDMA_WRITE_WITH_IMM: + ctrl->imm_data = wr->ex.imm_data; + break; + case IB_WR_SEND_WITH_INV: + ctrl->inv_key = + cpu_to_le32(wr->ex.invalidate_rkey); + break; + default: + ctrl->imm_data = 0; + break; + } + + /* Ctrl field, ctrl set type: sig, solic, imm, fence */ + /* SO wait for conforming application scenarios */ + ctrl->flag |= (wr->send_flags & IB_SEND_SIGNALED ? + cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) | + (wr->send_flags & IB_SEND_SOLICITED ? + cpu_to_le32(HNS_ROCE_WQE_SE) : 0) | + ((wr->opcode == IB_WR_SEND_WITH_IMM || + wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) ? + cpu_to_le32(HNS_ROCE_WQE_IMM) : 0) | + (wr->send_flags & IB_SEND_FENCE ? + (cpu_to_le32(HNS_ROCE_WQE_FENCE)) : 0); + + wqe += sizeof(struct hns_roce_wqe_ctrl_seg); + + switch (wr->opcode) { + case IB_WR_RDMA_READ: + ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ; + set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, + rdma_wr(wr)->rkey); + break; + case IB_WR_RDMA_WRITE: + case IB_WR_RDMA_WRITE_WITH_IMM: + ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE; + set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, + rdma_wr(wr)->rkey); + break; + case IB_WR_SEND: + case IB_WR_SEND_WITH_INV: + case IB_WR_SEND_WITH_IMM: + ps_opcode = HNS_ROCE_WQE_OPCODE_SEND; + break; + case IB_WR_LOCAL_INV: + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + case IB_WR_LSO: + default: + ps_opcode = HNS_ROCE_WQE_OPCODE_MASK; + break; + } + ctrl->flag |= cpu_to_le32(ps_opcode); + wqe += sizeof(struct hns_roce_wqe_raddr_seg); + + dseg = wqe; + if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) { + if (le32_to_cpu(ctrl->msg_length) > + hr_dev->caps.max_sq_inline) { + ret = -EINVAL; + *bad_wr = wr; + dev_err(dev, "inline len(1-%d)=%d, illegal", + ctrl->msg_length, + hr_dev->caps.max_sq_inline); + goto out; + } + for (i = 0; i < wr->num_sge; i++) { + memcpy(wqe, ((void *) (uintptr_t) + wr->sg_list[i].addr), + wr->sg_list[i].length); + wqe += wr->sg_list[i].length; + } + ctrl->flag |= cpu_to_le32(HNS_ROCE_WQE_INLINE); + } else { + /* sqe num is two */ + for (i = 0; i < wr->num_sge; i++) + set_data_seg(dseg + i, wr->sg_list + i); + + ctrl->flag |= cpu_to_le32(wr->num_sge << + HNS_ROCE_WQE_SGE_NUM_BIT); + } + } + } + +out: + /* Set DB return */ + if (likely(nreq)) { + qp->sq.head += nreq; + /* Memory barrier */ + wmb(); + + roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M, + SQ_DOORBELL_U32_4_SQ_HEAD_S, + (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1))); + roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SL_M, + SQ_DOORBELL_U32_4_SL_S, qp->sl); + roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M, + SQ_DOORBELL_U32_4_PORT_S, qp->phy_port); + roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M, + SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn); + roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1); + + doorbell[0] = sq_db.u32_4; + doorbell[1] = sq_db.u32_8; + + hns_roce_write64_k(doorbell, qp->sq.db_reg_l); + } + + spin_unlock_irqrestore(&qp->sq.lock, flags); + + return ret; +} + +static int hns_roce_v1_post_recv(struct ib_qp *ibqp, + const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct hns_roce_rq_wqe_ctrl *ctrl = NULL; + struct hns_roce_wqe_data_seg *scat = NULL; + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct device *dev = &hr_dev->pdev->dev; + struct hns_roce_rq_db rq_db = {}; + __le32 doorbell[2] = {0}; + unsigned long flags = 0; + unsigned int wqe_idx; + int ret = 0; + int nreq = 0; + int i = 0; + u32 reg_val; + + spin_lock_irqsave(&hr_qp->rq.lock, flags); + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + if (hns_roce_wq_overflow(&hr_qp->rq, nreq, + hr_qp->ibqp.recv_cq)) { + ret = -ENOMEM; + *bad_wr = wr; + goto out; + } + + wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1); + + if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) { + dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n", + wr->num_sge, hr_qp->rq.max_gs); + ret = -EINVAL; + *bad_wr = wr; + goto out; + } + + ctrl = hns_roce_get_recv_wqe(hr_qp, wqe_idx); + + roce_set_field(ctrl->rwqe_byte_12, + RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M, + RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S, + wr->num_sge); + + scat = (struct hns_roce_wqe_data_seg *)(ctrl + 1); + + for (i = 0; i < wr->num_sge; i++) + set_data_seg(scat + i, wr->sg_list + i); + + hr_qp->rq.wrid[wqe_idx] = wr->wr_id; + } + +out: + if (likely(nreq)) { + hr_qp->rq.head += nreq; + /* Memory barrier */ + wmb(); + + if (ibqp->qp_type == IB_QPT_GSI) { + __le32 tmp; + + /* SW update GSI rq header */ + reg_val = roce_read(to_hr_dev(ibqp->device), + ROCEE_QP1C_CFG3_0_REG + + QP1C_CFGN_OFFSET * hr_qp->phy_port); + tmp = cpu_to_le32(reg_val); + roce_set_field(tmp, + ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M, + ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S, + hr_qp->rq.head); + reg_val = le32_to_cpu(tmp); + roce_write(to_hr_dev(ibqp->device), + ROCEE_QP1C_CFG3_0_REG + + QP1C_CFGN_OFFSET * hr_qp->phy_port, reg_val); + } else { + roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M, + RQ_DOORBELL_U32_4_RQ_HEAD_S, + hr_qp->rq.head); + roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M, + RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn); + roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M, + RQ_DOORBELL_U32_8_CMD_S, 1); + roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S, + 1); + + doorbell[0] = rq_db.u32_4; + doorbell[1] = rq_db.u32_8; + + hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l); + } + } + spin_unlock_irqrestore(&hr_qp->rq.lock, flags); + + return ret; +} + +static void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev, + int sdb_mode, int odb_mode) +{ + __le32 tmp; + u32 val; + + val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); + tmp = cpu_to_le32(val); + roce_set_bit(tmp, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S, sdb_mode); + roce_set_bit(tmp, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S, odb_mode); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); +} + +static int hns_roce_v1_set_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, int obj, + int step_idx) +{ + spinlock_t *lock = &hr_dev->bt_cmd_lock; + struct device *dev = hr_dev->dev; + struct hns_roce_hem_iter iter; + void __iomem *bt_cmd; + __le32 bt_cmd_val[2]; + __le32 bt_cmd_h = 0; + unsigned long flags; + __le32 bt_cmd_l; + int ret = 0; + u64 bt_ba; + long end; + + /* Find the HEM(Hardware Entry Memory) entry */ + unsigned long i = (obj & (table->num_obj - 1)) / + (table->table_chunk_size / table->obj_size); + + switch (table->type) { + case HEM_TYPE_QPC: + case HEM_TYPE_MTPT: + case HEM_TYPE_CQC: + case HEM_TYPE_SRQC: + roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, + ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type); + break; + default: + return ret; + } + + roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M, + ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj); + roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0); + roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1); + + /* Currently iter only a chunk */ + for (hns_roce_hem_first(table->hem[i], &iter); + !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { + bt_ba = hns_roce_hem_addr(&iter) >> HNS_HW_PAGE_SHIFT; + + spin_lock_irqsave(lock, flags); + + bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; + + end = HW_SYNC_TIMEOUT_MSECS; + while (end > 0) { + if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT)) + break; + + mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); + end -= HW_SYNC_SLEEP_TIME_INTERVAL; + } + + if (end <= 0) { + dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); + spin_unlock_irqrestore(lock, flags); + return -EBUSY; + } + + bt_cmd_l = cpu_to_le32(bt_ba); + roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M, + ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, + upper_32_bits(bt_ba)); + + bt_cmd_val[0] = bt_cmd_l; + bt_cmd_val[1] = bt_cmd_h; + hns_roce_write64_k(bt_cmd_val, + hr_dev->reg_base + ROCEE_BT_CMD_L_REG); + spin_unlock_irqrestore(lock, flags); + } + + return ret; +} + +static void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev, u32 sdb_mode, + u32 odb_mode) +{ + __le32 tmp; + u32 val; + + /* Configure SDB/ODB extend mode */ + val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); + tmp = cpu_to_le32(val); + roce_set_bit(tmp, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S, sdb_mode); + roce_set_bit(tmp, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S, odb_mode); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); +} + +static void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept, + u32 sdb_alful) +{ + __le32 tmp; + u32 val; + + /* Configure SDB */ + val = roce_read(hr_dev, ROCEE_DB_SQ_WL_REG); + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M, + ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S, sdb_alful); + roce_set_field(tmp, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M, + ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S, sdb_alept); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_DB_SQ_WL_REG, val); +} + +static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept, + u32 odb_alful) +{ + __le32 tmp; + u32 val; + + /* Configure ODB */ + val = roce_read(hr_dev, ROCEE_DB_OTHERS_WL_REG); + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M, + ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S, odb_alful); + roce_set_field(tmp, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M, + ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S, odb_alept); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_DB_OTHERS_WL_REG, val); +} + +static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept, + u32 ext_sdb_alful) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_db_table *db = &priv->db_table; + struct device *dev = &hr_dev->pdev->dev; + dma_addr_t sdb_dma_addr; + __le32 tmp; + u32 val; + + /* Configure extend SDB threshold */ + roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept); + roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful); + + /* Configure extend SDB base addr */ + sdb_dma_addr = db->ext_db->sdb_buf_list->map; + roce_write(hr_dev, ROCEE_EXT_DB_SQ_REG, (u32)(sdb_dma_addr >> 12)); + + /* Configure extend SDB depth */ + val = roce_read(hr_dev, ROCEE_EXT_DB_SQ_H_REG); + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M, + ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S, + db->ext_db->esdb_dep); + /* + * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of + * using 4K page, and shift more 32 because of + * caculating the high 32 bit value evaluated to hardware. + */ + roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M, + ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S, sdb_dma_addr >> 44); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_EXT_DB_SQ_H_REG, val); + + dev_dbg(dev, "ext SDB depth: 0x%x\n", db->ext_db->esdb_dep); + dev_dbg(dev, "ext SDB threshold: empty: 0x%x, ful: 0x%x\n", + ext_sdb_alept, ext_sdb_alful); +} + +static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept, + u32 ext_odb_alful) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_db_table *db = &priv->db_table; + struct device *dev = &hr_dev->pdev->dev; + dma_addr_t odb_dma_addr; + __le32 tmp; + u32 val; + + /* Configure extend ODB threshold */ + roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept); + roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful); + + /* Configure extend ODB base addr */ + odb_dma_addr = db->ext_db->odb_buf_list->map; + roce_write(hr_dev, ROCEE_EXT_DB_OTH_REG, (u32)(odb_dma_addr >> 12)); + + /* Configure extend ODB depth */ + val = roce_read(hr_dev, ROCEE_EXT_DB_OTH_H_REG); + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M, + ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S, + db->ext_db->eodb_dep); + roce_set_field(tmp, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M, + ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S, + db->ext_db->eodb_dep); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_EXT_DB_OTH_H_REG, val); + + dev_dbg(dev, "ext ODB depth: 0x%x\n", db->ext_db->eodb_dep); + dev_dbg(dev, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n", + ext_odb_alept, ext_odb_alful); +} + +static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod, + u32 odb_ext_mod) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_db_table *db = &priv->db_table; + struct device *dev = &hr_dev->pdev->dev; + dma_addr_t sdb_dma_addr; + dma_addr_t odb_dma_addr; + int ret = 0; + + db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL); + if (!db->ext_db) + return -ENOMEM; + + if (sdb_ext_mod) { + db->ext_db->sdb_buf_list = kmalloc( + sizeof(*db->ext_db->sdb_buf_list), GFP_KERNEL); + if (!db->ext_db->sdb_buf_list) { + ret = -ENOMEM; + goto ext_sdb_buf_fail_out; + } + + db->ext_db->sdb_buf_list->buf = dma_alloc_coherent(dev, + HNS_ROCE_V1_EXT_SDB_SIZE, + &sdb_dma_addr, GFP_KERNEL); + if (!db->ext_db->sdb_buf_list->buf) { + ret = -ENOMEM; + goto alloc_sq_db_buf_fail; + } + db->ext_db->sdb_buf_list->map = sdb_dma_addr; + + db->ext_db->esdb_dep = ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH); + hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT, + HNS_ROCE_V1_EXT_SDB_ALFUL); + } else + hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT, + HNS_ROCE_V1_SDB_ALFUL); + + if (odb_ext_mod) { + db->ext_db->odb_buf_list = kmalloc( + sizeof(*db->ext_db->odb_buf_list), GFP_KERNEL); + if (!db->ext_db->odb_buf_list) { + ret = -ENOMEM; + goto ext_odb_buf_fail_out; + } + + db->ext_db->odb_buf_list->buf = dma_alloc_coherent(dev, + HNS_ROCE_V1_EXT_ODB_SIZE, + &odb_dma_addr, GFP_KERNEL); + if (!db->ext_db->odb_buf_list->buf) { + ret = -ENOMEM; + goto alloc_otr_db_buf_fail; + } + db->ext_db->odb_buf_list->map = odb_dma_addr; + + db->ext_db->eodb_dep = ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH); + hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT, + HNS_ROCE_V1_EXT_ODB_ALFUL); + } else + hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT, + HNS_ROCE_V1_ODB_ALFUL); + + hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod); + + return 0; + +alloc_otr_db_buf_fail: + kfree(db->ext_db->odb_buf_list); + +ext_odb_buf_fail_out: + if (sdb_ext_mod) { + dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE, + db->ext_db->sdb_buf_list->buf, + db->ext_db->sdb_buf_list->map); + } + +alloc_sq_db_buf_fail: + if (sdb_ext_mod) + kfree(db->ext_db->sdb_buf_list); + +ext_sdb_buf_fail_out: + kfree(db->ext_db); + return ret; +} + +static struct hns_roce_qp *hns_roce_v1_create_lp_qp(struct hns_roce_dev *hr_dev, + struct ib_pd *pd) +{ + struct device *dev = &hr_dev->pdev->dev; + struct ib_qp_init_attr init_attr; + struct ib_qp *qp; + + memset(&init_attr, 0, sizeof(struct ib_qp_init_attr)); + init_attr.qp_type = IB_QPT_RC; + init_attr.sq_sig_type = IB_SIGNAL_ALL_WR; + init_attr.cap.max_recv_wr = HNS_ROCE_MIN_WQE_NUM; + init_attr.cap.max_send_wr = HNS_ROCE_MIN_WQE_NUM; + + qp = hns_roce_create_qp(pd, &init_attr, NULL); + if (IS_ERR(qp)) { + dev_err(dev, "Create loop qp for mr free failed!"); + return NULL; + } + + return to_hr_qp(qp); +} + +static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_free_mr *free_mr = &priv->free_mr; + struct hns_roce_caps *caps = &hr_dev->caps; + struct ib_device *ibdev = &hr_dev->ib_dev; + struct device *dev = &hr_dev->pdev->dev; + struct ib_cq_init_attr cq_init_attr; + struct ib_qp_attr attr = { 0 }; + struct hns_roce_qp *hr_qp; + struct ib_cq *cq; + struct ib_pd *pd; + union ib_gid dgid; + __be64 subnet_prefix; + int attr_mask = 0; + int ret; + int i, j; + u8 queue_en[HNS_ROCE_V1_RESV_QP] = { 0 }; + u8 phy_port; + u8 port = 0; + u8 sl; + + /* Reserved cq for loop qp */ + cq_init_attr.cqe = HNS_ROCE_MIN_WQE_NUM * 2; + cq_init_attr.comp_vector = 0; + + cq = rdma_zalloc_drv_obj(ibdev, ib_cq); + if (!cq) + return -ENOMEM; + + ret = hns_roce_create_cq(cq, &cq_init_attr, NULL); + if (ret) { + dev_err(dev, "Create cq for reserved loop qp failed!"); + goto alloc_cq_failed; + } + free_mr->mr_free_cq = to_hr_cq(cq); + free_mr->mr_free_cq->ib_cq.device = &hr_dev->ib_dev; + free_mr->mr_free_cq->ib_cq.uobject = NULL; + free_mr->mr_free_cq->ib_cq.comp_handler = NULL; + free_mr->mr_free_cq->ib_cq.event_handler = NULL; + free_mr->mr_free_cq->ib_cq.cq_context = NULL; + atomic_set(&free_mr->mr_free_cq->ib_cq.usecnt, 0); + + pd = rdma_zalloc_drv_obj(ibdev, ib_pd); + if (!pd) { + ret = -ENOMEM; + goto alloc_mem_failed; + } + + pd->device = ibdev; + ret = hns_roce_alloc_pd(pd, NULL); + if (ret) + goto alloc_pd_failed; + + free_mr->mr_free_pd = to_hr_pd(pd); + free_mr->mr_free_pd->ibpd.device = &hr_dev->ib_dev; + free_mr->mr_free_pd->ibpd.uobject = NULL; + free_mr->mr_free_pd->ibpd.__internal_mr = NULL; + atomic_set(&free_mr->mr_free_pd->ibpd.usecnt, 0); + + attr.qp_access_flags = IB_ACCESS_REMOTE_WRITE; + attr.pkey_index = 0; + attr.min_rnr_timer = 0; + /* Disable read ability */ + attr.max_dest_rd_atomic = 0; + attr.max_rd_atomic = 0; + /* Use arbitrary values as rq_psn and sq_psn */ + attr.rq_psn = 0x0808; + attr.sq_psn = 0x0808; + attr.retry_cnt = 7; + attr.rnr_retry = 7; + attr.timeout = 0x12; + attr.path_mtu = IB_MTU_256; + attr.ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; + rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0); + rdma_ah_set_static_rate(&attr.ah_attr, 3); + + subnet_prefix = cpu_to_be64(0xfe80000000000000LL); + for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) { + phy_port = (i >= HNS_ROCE_MAX_PORTS) ? (i - 2) : + (i % HNS_ROCE_MAX_PORTS); + sl = i / HNS_ROCE_MAX_PORTS; + + for (j = 0; j < caps->num_ports; j++) { + if (hr_dev->iboe.phy_port[j] == phy_port) { + queue_en[i] = 1; + port = j; + break; + } + } + + if (!queue_en[i]) + continue; + + free_mr->mr_free_qp[i] = hns_roce_v1_create_lp_qp(hr_dev, pd); + if (!free_mr->mr_free_qp[i]) { + dev_err(dev, "Create loop qp failed!\n"); + ret = -ENOMEM; + goto create_lp_qp_failed; + } + hr_qp = free_mr->mr_free_qp[i]; + + hr_qp->port = port; + hr_qp->phy_port = phy_port; + hr_qp->ibqp.qp_type = IB_QPT_RC; + hr_qp->ibqp.device = &hr_dev->ib_dev; + hr_qp->ibqp.uobject = NULL; + atomic_set(&hr_qp->ibqp.usecnt, 0); + hr_qp->ibqp.pd = pd; + hr_qp->ibqp.recv_cq = cq; + hr_qp->ibqp.send_cq = cq; + + rdma_ah_set_port_num(&attr.ah_attr, port + 1); + rdma_ah_set_sl(&attr.ah_attr, sl); + attr.port_num = port + 1; + + attr.dest_qp_num = hr_qp->qpn; + memcpy(rdma_ah_retrieve_dmac(&attr.ah_attr), + hr_dev->dev_addr[port], + ETH_ALEN); + + memcpy(&dgid.raw, &subnet_prefix, sizeof(u64)); + memcpy(&dgid.raw[8], hr_dev->dev_addr[port], 3); + memcpy(&dgid.raw[13], hr_dev->dev_addr[port] + 3, 3); + dgid.raw[11] = 0xff; + dgid.raw[12] = 0xfe; + dgid.raw[8] ^= 2; + rdma_ah_set_dgid_raw(&attr.ah_attr, dgid.raw); + + ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask, + IB_QPS_RESET, IB_QPS_INIT); + if (ret) { + dev_err(dev, "modify qp failed(%d)!\n", ret); + goto create_lp_qp_failed; + } + + ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, IB_QP_DEST_QPN, + IB_QPS_INIT, IB_QPS_RTR); + if (ret) { + dev_err(dev, "modify qp failed(%d)!\n", ret); + goto create_lp_qp_failed; + } + + ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask, + IB_QPS_RTR, IB_QPS_RTS); + if (ret) { + dev_err(dev, "modify qp failed(%d)!\n", ret); + goto create_lp_qp_failed; + } + } + + return 0; + +create_lp_qp_failed: + for (i -= 1; i >= 0; i--) { + hr_qp = free_mr->mr_free_qp[i]; + if (hns_roce_v1_destroy_qp(&hr_qp->ibqp, NULL)) + dev_err(dev, "Destroy qp %d for mr free failed!\n", i); + } + + hns_roce_dealloc_pd(pd, NULL); + +alloc_pd_failed: + kfree(pd); + +alloc_mem_failed: + hns_roce_destroy_cq(cq, NULL); +alloc_cq_failed: + kfree(cq); + return ret; +} + +static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_free_mr *free_mr = &priv->free_mr; + struct device *dev = &hr_dev->pdev->dev; + struct hns_roce_qp *hr_qp; + int ret; + int i; + + for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) { + hr_qp = free_mr->mr_free_qp[i]; + if (!hr_qp) + continue; + + ret = hns_roce_v1_destroy_qp(&hr_qp->ibqp, NULL); + if (ret) + dev_err(dev, "Destroy qp %d for mr free failed(%d)!\n", + i, ret); + } + + hns_roce_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL); + kfree(&free_mr->mr_free_cq->ib_cq); + hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd, NULL); + kfree(&free_mr->mr_free_pd->ibpd); +} + +static int hns_roce_db_init(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_db_table *db = &priv->db_table; + struct device *dev = &hr_dev->pdev->dev; + u32 sdb_ext_mod; + u32 odb_ext_mod; + u32 sdb_evt_mod; + u32 odb_evt_mod; + int ret; + + memset(db, 0, sizeof(*db)); + + /* Default DB mode */ + sdb_ext_mod = HNS_ROCE_SDB_EXTEND_MODE; + odb_ext_mod = HNS_ROCE_ODB_EXTEND_MODE; + sdb_evt_mod = HNS_ROCE_SDB_NORMAL_MODE; + odb_evt_mod = HNS_ROCE_ODB_POLL_MODE; + + db->sdb_ext_mod = sdb_ext_mod; + db->odb_ext_mod = odb_ext_mod; + + /* Init extend DB */ + ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod); + if (ret) { + dev_err(dev, "Failed in extend DB configuration.\n"); + return ret; + } + + hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod); + + return 0; +} + +static void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct *work) +{ + struct hns_roce_recreate_lp_qp_work *lp_qp_work; + struct hns_roce_dev *hr_dev; + + lp_qp_work = container_of(work, struct hns_roce_recreate_lp_qp_work, + work); + hr_dev = to_hr_dev(lp_qp_work->ib_dev); + + hns_roce_v1_release_lp_qp(hr_dev); + + if (hns_roce_v1_rsv_lp_qp(hr_dev)) + dev_err(&hr_dev->pdev->dev, "create reserver qp failed\n"); + + if (lp_qp_work->comp_flag) + complete(lp_qp_work->comp); + + kfree(lp_qp_work); +} + +static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev) +{ + long end = HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS; + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_free_mr *free_mr = &priv->free_mr; + struct hns_roce_recreate_lp_qp_work *lp_qp_work; + struct device *dev = &hr_dev->pdev->dev; + struct completion comp; + + lp_qp_work = kzalloc(sizeof(struct hns_roce_recreate_lp_qp_work), + GFP_KERNEL); + if (!lp_qp_work) + return -ENOMEM; + + INIT_WORK(&(lp_qp_work->work), hns_roce_v1_recreate_lp_qp_work_fn); + + lp_qp_work->ib_dev = &(hr_dev->ib_dev); + lp_qp_work->comp = ∁ + lp_qp_work->comp_flag = 1; + + init_completion(lp_qp_work->comp); + + queue_work(free_mr->free_mr_wq, &(lp_qp_work->work)); + + while (end > 0) { + if (try_wait_for_completion(&comp)) + return 0; + msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE); + end -= HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE; + } + + lp_qp_work->comp_flag = 0; + if (try_wait_for_completion(&comp)) + return 0; + + dev_warn(dev, "recreate lp qp failed 20s timeout and return failed!\n"); + return -ETIMEDOUT; +} + +static int hns_roce_v1_send_lp_wqe(struct hns_roce_qp *hr_qp) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); + struct device *dev = &hr_dev->pdev->dev; + struct ib_send_wr send_wr; + const struct ib_send_wr *bad_wr; + int ret; + + memset(&send_wr, 0, sizeof(send_wr)); + send_wr.next = NULL; + send_wr.num_sge = 0; + send_wr.send_flags = 0; + send_wr.sg_list = NULL; + send_wr.wr_id = (unsigned long long)&send_wr; + send_wr.opcode = IB_WR_RDMA_WRITE; + + ret = hns_roce_v1_post_send(&hr_qp->ibqp, &send_wr, &bad_wr); + if (ret) { + dev_err(dev, "Post write wqe for mr free failed(%d)!", ret); + return ret; + } + + return 0; +} + +static void hns_roce_v1_mr_free_work_fn(struct work_struct *work) +{ + unsigned long end = + msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies; + struct hns_roce_mr_free_work *mr_work = + container_of(work, struct hns_roce_mr_free_work, work); + struct hns_roce_dev *hr_dev = to_hr_dev(mr_work->ib_dev); + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_free_mr *free_mr = &priv->free_mr; + struct hns_roce_cq *mr_free_cq = free_mr->mr_free_cq; + struct hns_roce_mr *hr_mr = mr_work->mr; + struct device *dev = &hr_dev->pdev->dev; + struct ib_wc wc[HNS_ROCE_V1_RESV_QP]; + struct hns_roce_qp *hr_qp; + int ne = 0; + int ret; + int i; + + for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) { + hr_qp = free_mr->mr_free_qp[i]; + if (!hr_qp) + continue; + ne++; + + ret = hns_roce_v1_send_lp_wqe(hr_qp); + if (ret) { + dev_err(dev, + "Send wqe (qp:0x%lx) for mr free failed(%d)!\n", + hr_qp->qpn, ret); + goto free_work; + } + } + + if (!ne) { + dev_err(dev, "Reserved loop qp is absent!\n"); + goto free_work; + } + + do { + ret = hns_roce_v1_poll_cq(&mr_free_cq->ib_cq, ne, wc); + if (ret < 0 && hr_qp) { + dev_err(dev, + "(qp:0x%lx) starts, Poll cqe failed(%d) for mr 0x%x free! Remain %d cqe\n", + hr_qp->qpn, ret, hr_mr->key, ne); + goto free_work; + } + ne -= ret; + usleep_range(HNS_ROCE_V1_FREE_MR_WAIT_VALUE * 1000, + (1 + HNS_ROCE_V1_FREE_MR_WAIT_VALUE) * 1000); + } while (ne && time_before_eq(jiffies, end)); + + if (ne != 0) + dev_err(dev, + "Poll cqe for mr 0x%x free timeout! Remain %d cqe\n", + hr_mr->key, ne); + +free_work: + if (mr_work->comp_flag) + complete(mr_work->comp); + kfree(mr_work); +} + +static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, + struct hns_roce_mr *mr, struct ib_udata *udata) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_free_mr *free_mr = &priv->free_mr; + long end = HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS; + struct device *dev = &hr_dev->pdev->dev; + struct hns_roce_mr_free_work *mr_work; + unsigned long start = jiffies; + struct completion comp; + int ret = 0; + + if (mr->enabled) { + if (hns_roce_hw_destroy_mpt(hr_dev, NULL, + key_to_hw_index(mr->key) & + (hr_dev->caps.num_mtpts - 1))) + dev_warn(dev, "DESTROY_MPT failed!\n"); + } + + mr_work = kzalloc(sizeof(*mr_work), GFP_KERNEL); + if (!mr_work) { + ret = -ENOMEM; + goto free_mr; + } + + INIT_WORK(&(mr_work->work), hns_roce_v1_mr_free_work_fn); + + mr_work->ib_dev = &(hr_dev->ib_dev); + mr_work->comp = ∁ + mr_work->comp_flag = 1; + mr_work->mr = (void *)mr; + init_completion(mr_work->comp); + + queue_work(free_mr->free_mr_wq, &(mr_work->work)); + + while (end > 0) { + if (try_wait_for_completion(&comp)) + goto free_mr; + msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE); + end -= HNS_ROCE_V1_FREE_MR_WAIT_VALUE; + } + + mr_work->comp_flag = 0; + if (try_wait_for_completion(&comp)) + goto free_mr; + + dev_warn(dev, "Free mr work 0x%x over 50s and failed!\n", mr->key); + ret = -ETIMEDOUT; + +free_mr: + dev_dbg(dev, "Free mr 0x%x use 0x%x us.\n", + mr->key, jiffies_to_usecs(jiffies) - jiffies_to_usecs(start)); + + hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, + key_to_hw_index(mr->key), 0); + hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr); + kfree(mr); + + return ret; +} + +static void hns_roce_db_free(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_db_table *db = &priv->db_table; + struct device *dev = &hr_dev->pdev->dev; + + if (db->sdb_ext_mod) { + dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE, + db->ext_db->sdb_buf_list->buf, + db->ext_db->sdb_buf_list->map); + kfree(db->ext_db->sdb_buf_list); + } + + if (db->odb_ext_mod) { + dma_free_coherent(dev, HNS_ROCE_V1_EXT_ODB_SIZE, + db->ext_db->odb_buf_list->buf, + db->ext_db->odb_buf_list->map); + kfree(db->ext_db->odb_buf_list); + } + + kfree(db->ext_db); +} + +static int hns_roce_raq_init(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_raq_table *raq = &priv->raq_table; + struct device *dev = &hr_dev->pdev->dev; + dma_addr_t addr; + int raq_shift; + __le32 tmp; + u32 val; + int ret; + + raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL); + if (!raq->e_raq_buf) + return -ENOMEM; + + raq->e_raq_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, + &addr, GFP_KERNEL); + if (!raq->e_raq_buf->buf) { + ret = -ENOMEM; + goto err_dma_alloc_raq; + } + raq->e_raq_buf->map = addr; + + /* Configure raq extended address. 48bit 4K align */ + roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq->e_raq_buf->map >> 12); + + /* Configure raq_shift */ + raq_shift = ilog2(HNS_ROCE_V1_RAQ_SIZE / HNS_ROCE_V1_RAQ_ENTRY); + val = roce_read(hr_dev, ROCEE_EXT_RAQ_H_REG); + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M, + ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S, raq_shift); + /* + * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of + * using 4K page, and shift more 32 because of + * caculating the high 32 bit value evaluated to hardware. + */ + roce_set_field(tmp, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M, + ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S, + raq->e_raq_buf->map >> 44); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_EXT_RAQ_H_REG, val); + dev_dbg(dev, "Configure raq_shift 0x%x.\n", val); + + /* Configure raq threshold */ + val = roce_read(hr_dev, ROCEE_RAQ_WL_REG); + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M, + ROCEE_RAQ_WL_ROCEE_RAQ_WL_S, + HNS_ROCE_V1_EXT_RAQ_WF); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_RAQ_WL_REG, val); + dev_dbg(dev, "Configure raq_wl 0x%x.\n", val); + + /* Enable extend raq */ + val = roce_read(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG); + tmp = cpu_to_le32(val); + roce_set_field(tmp, + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M, + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S, + POL_TIME_INTERVAL_VAL); + roce_set_bit(tmp, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE, 1); + roce_set_field(tmp, + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M, + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S, + 2); + roce_set_bit(tmp, + ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S, 1); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG, val); + dev_dbg(dev, "Configure WrmsPolTimeInterval 0x%x.\n", val); + + /* Enable raq drop */ + val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); + tmp = cpu_to_le32(val); + roce_set_bit(tmp, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S, 1); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); + dev_dbg(dev, "Configure GlbCfg = 0x%x.\n", val); + + return 0; + +err_dma_alloc_raq: + kfree(raq->e_raq_buf); + return ret; +} + +static void hns_roce_raq_free(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_raq_table *raq = &priv->raq_table; + struct device *dev = &hr_dev->pdev->dev; + + dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf, + raq->e_raq_buf->map); + kfree(raq->e_raq_buf); +} + +static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag) +{ + __le32 tmp; + u32 val; + + if (enable_flag) { + val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); + /* Open all ports */ + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_GLB_CFG_ROCEE_PORT_ST_M, + ROCEE_GLB_CFG_ROCEE_PORT_ST_S, + ALL_PORT_VAL_OPEN); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); + } else { + val = roce_read(hr_dev, ROCEE_GLB_CFG_REG); + /* Close all ports */ + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_GLB_CFG_ROCEE_PORT_ST_M, + ROCEE_GLB_CFG_ROCEE_PORT_ST_S, 0x0); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_GLB_CFG_REG, val); + } +} + +static int hns_roce_bt_init(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct device *dev = &hr_dev->pdev->dev; + int ret; + + priv->bt_table.qpc_buf.buf = dma_alloc_coherent(dev, + HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.qpc_buf.map, + GFP_KERNEL); + if (!priv->bt_table.qpc_buf.buf) + return -ENOMEM; + + priv->bt_table.mtpt_buf.buf = dma_alloc_coherent(dev, + HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.mtpt_buf.map, + GFP_KERNEL); + if (!priv->bt_table.mtpt_buf.buf) { + ret = -ENOMEM; + goto err_failed_alloc_mtpt_buf; + } + + priv->bt_table.cqc_buf.buf = dma_alloc_coherent(dev, + HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.cqc_buf.map, + GFP_KERNEL); + if (!priv->bt_table.cqc_buf.buf) { + ret = -ENOMEM; + goto err_failed_alloc_cqc_buf; + } + + return 0; + +err_failed_alloc_cqc_buf: + dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, + priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map); + +err_failed_alloc_mtpt_buf: + dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, + priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map); + + return ret; +} + +static void hns_roce_bt_free(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct device *dev = &hr_dev->pdev->dev; + + dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, + priv->bt_table.cqc_buf.buf, priv->bt_table.cqc_buf.map); + + dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, + priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map); + + dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE, + priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map); +} + +static int hns_roce_tptr_init(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf; + struct device *dev = &hr_dev->pdev->dev; + + /* + * This buffer will be used for CQ's tptr(tail pointer), also + * named ci(customer index). Every CQ will use 2 bytes to save + * cqe ci in hip06. Hardware will read this area to get new ci + * when the queue is almost full. + */ + tptr_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE, + &tptr_buf->map, GFP_KERNEL); + if (!tptr_buf->buf) + return -ENOMEM; + + hr_dev->tptr_dma_addr = tptr_buf->map; + hr_dev->tptr_size = HNS_ROCE_V1_TPTR_BUF_SIZE; + + return 0; +} + +static void hns_roce_tptr_free(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf; + struct device *dev = &hr_dev->pdev->dev; + + dma_free_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE, + tptr_buf->buf, tptr_buf->map); +} + +static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_free_mr *free_mr = &priv->free_mr; + struct device *dev = &hr_dev->pdev->dev; + int ret; + + free_mr->free_mr_wq = create_singlethread_workqueue("hns_roce_free_mr"); + if (!free_mr->free_mr_wq) { + dev_err(dev, "Create free mr workqueue failed!\n"); + return -ENOMEM; + } + + ret = hns_roce_v1_rsv_lp_qp(hr_dev); + if (ret) { + dev_err(dev, "Reserved loop qp failed(%d)!\n", ret); + flush_workqueue(free_mr->free_mr_wq); + destroy_workqueue(free_mr->free_mr_wq); + } + + return ret; +} + +static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_free_mr *free_mr = &priv->free_mr; + + flush_workqueue(free_mr->free_mr_wq); + destroy_workqueue(free_mr->free_mr_wq); + + hns_roce_v1_release_lp_qp(hr_dev); +} + +/** + * hns_roce_v1_reset - reset RoCE + * @hr_dev: RoCE device struct pointer + * @enable: true -- drop reset, false -- reset + * return 0 - success , negative --fail + */ +static int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset) +{ + struct device_node *dsaf_node; + struct device *dev = &hr_dev->pdev->dev; + struct device_node *np = dev->of_node; + struct fwnode_handle *fwnode; + int ret; + + /* check if this is DT/ACPI case */ + if (dev_of_node(dev)) { + dsaf_node = of_parse_phandle(np, "dsaf-handle", 0); + if (!dsaf_node) { + dev_err(dev, "could not find dsaf-handle\n"); + return -EINVAL; + } + fwnode = &dsaf_node->fwnode; + } else if (is_acpi_device_node(dev->fwnode)) { + struct fwnode_reference_args args; + + ret = acpi_node_get_property_reference(dev->fwnode, + "dsaf-handle", 0, &args); + if (ret) { + dev_err(dev, "could not find dsaf-handle\n"); + return ret; + } + fwnode = args.fwnode; + } else { + dev_err(dev, "cannot read data from DT or ACPI\n"); + return -ENXIO; + } + + ret = hns_dsaf_roce_reset(fwnode, false); + if (ret) + return ret; + + if (dereset) { + msleep(SLEEP_TIME_INTERVAL); + ret = hns_dsaf_roce_reset(fwnode, true); + } + + return ret; +} + +static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_caps *caps = &hr_dev->caps; + int i; + + hr_dev->vendor_id = roce_read(hr_dev, ROCEE_VENDOR_ID_REG); + hr_dev->vendor_part_id = roce_read(hr_dev, ROCEE_VENDOR_PART_ID_REG); + hr_dev->sys_image_guid = roce_read(hr_dev, ROCEE_SYS_IMAGE_GUID_L_REG) | + ((u64)roce_read(hr_dev, + ROCEE_SYS_IMAGE_GUID_H_REG) << 32); + hr_dev->hw_rev = HNS_ROCE_HW_VER1; + + caps->num_qps = HNS_ROCE_V1_MAX_QP_NUM; + caps->max_wqes = HNS_ROCE_V1_MAX_WQE_NUM; + caps->min_wqes = HNS_ROCE_MIN_WQE_NUM; + caps->num_cqs = HNS_ROCE_V1_MAX_CQ_NUM; + caps->min_cqes = HNS_ROCE_MIN_CQE_NUM; + caps->max_cqes = HNS_ROCE_V1_MAX_CQE_NUM; + caps->max_sq_sg = HNS_ROCE_V1_SG_NUM; + caps->max_rq_sg = HNS_ROCE_V1_SG_NUM; + caps->max_sq_inline = HNS_ROCE_V1_INLINE_SIZE; + caps->num_uars = HNS_ROCE_V1_UAR_NUM; + caps->phy_num_uars = HNS_ROCE_V1_PHY_UAR_NUM; + caps->num_aeq_vectors = HNS_ROCE_V1_AEQE_VEC_NUM; + caps->num_comp_vectors = HNS_ROCE_V1_COMP_VEC_NUM; + caps->num_other_vectors = HNS_ROCE_V1_ABNORMAL_VEC_NUM; + caps->num_mtpts = HNS_ROCE_V1_MAX_MTPT_NUM; + caps->num_mtt_segs = HNS_ROCE_V1_MAX_MTT_SEGS; + caps->num_pds = HNS_ROCE_V1_MAX_PD_NUM; + caps->max_qp_init_rdma = HNS_ROCE_V1_MAX_QP_INIT_RDMA; + caps->max_qp_dest_rdma = HNS_ROCE_V1_MAX_QP_DEST_RDMA; + caps->max_sq_desc_sz = HNS_ROCE_V1_MAX_SQ_DESC_SZ; + caps->max_rq_desc_sz = HNS_ROCE_V1_MAX_RQ_DESC_SZ; + caps->qpc_sz = HNS_ROCE_V1_QPC_SIZE; + caps->irrl_entry_sz = HNS_ROCE_V1_IRRL_ENTRY_SIZE; + caps->cqc_entry_sz = HNS_ROCE_V1_CQC_ENTRY_SIZE; + caps->mtpt_entry_sz = HNS_ROCE_V1_MTPT_ENTRY_SIZE; + caps->mtt_entry_sz = HNS_ROCE_V1_MTT_ENTRY_SIZE; + caps->cqe_sz = HNS_ROCE_V1_CQE_SIZE; + caps->page_size_cap = HNS_ROCE_V1_PAGE_SIZE_SUPPORT; + caps->reserved_lkey = 0; + caps->reserved_pds = 0; + caps->reserved_mrws = 1; + caps->reserved_uars = 0; + caps->reserved_cqs = 0; + caps->reserved_qps = 12; /* 2 SQP per port, six ports total 12 */ + caps->chunk_sz = HNS_ROCE_V1_TABLE_CHUNK_SIZE; + + for (i = 0; i < caps->num_ports; i++) + caps->pkey_table_len[i] = 1; + + for (i = 0; i < caps->num_ports; i++) { + /* Six ports shared 16 GID in v1 engine */ + if (i >= (HNS_ROCE_V1_GID_NUM % caps->num_ports)) + caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM / + caps->num_ports; + else + caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM / + caps->num_ports + 1; + } + + caps->ceqe_depth = HNS_ROCE_V1_COMP_EQE_NUM; + caps->aeqe_depth = HNS_ROCE_V1_ASYNC_EQE_NUM; + caps->local_ca_ack_delay = roce_read(hr_dev, ROCEE_ACK_DELAY_REG); + caps->max_mtu = IB_MTU_2048; + + return 0; +} + +static int hns_roce_v1_init(struct hns_roce_dev *hr_dev) +{ + int ret; + u32 val; + __le32 tmp; + struct device *dev = &hr_dev->pdev->dev; + + /* DMAE user config */ + val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG1_REG); + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M, + ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S, 0xf); + roce_set_field(tmp, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M, + ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S, + 1 << PAGES_SHIFT_16); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_DMAE_USER_CFG1_REG, val); + + val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG2_REG); + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M, + ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S, 0xf); + roce_set_field(tmp, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M, + ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S, + 1 << PAGES_SHIFT_16); + + ret = hns_roce_db_init(hr_dev); + if (ret) { + dev_err(dev, "doorbell init failed!\n"); + return ret; + } + + ret = hns_roce_raq_init(hr_dev); + if (ret) { + dev_err(dev, "raq init failed!\n"); + goto error_failed_raq_init; + } + + ret = hns_roce_bt_init(hr_dev); + if (ret) { + dev_err(dev, "bt init failed!\n"); + goto error_failed_bt_init; + } + + ret = hns_roce_tptr_init(hr_dev); + if (ret) { + dev_err(dev, "tptr init failed!\n"); + goto error_failed_tptr_init; + } + + ret = hns_roce_free_mr_init(hr_dev); + if (ret) { + dev_err(dev, "free mr init failed!\n"); + goto error_failed_free_mr_init; + } + + hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP); + + return 0; + +error_failed_free_mr_init: + hns_roce_tptr_free(hr_dev); + +error_failed_tptr_init: + hns_roce_bt_free(hr_dev); + +error_failed_bt_init: + hns_roce_raq_free(hr_dev); + +error_failed_raq_init: + hns_roce_db_free(hr_dev); + return ret; +} + +static void hns_roce_v1_exit(struct hns_roce_dev *hr_dev) +{ + hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN); + hns_roce_free_mr_free(hr_dev); + hns_roce_tptr_free(hr_dev); + hns_roce_bt_free(hr_dev); + hns_roce_raq_free(hr_dev); + hns_roce_db_free(hr_dev); +} + +static int hns_roce_v1_cmd_pending(struct hns_roce_dev *hr_dev) +{ + u32 status = readl(hr_dev->reg_base + ROCEE_MB6_REG); + + return (!!(status & (1 << HCR_GO_BIT))); +} + +static int hns_roce_v1_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, u32 in_modifier, u8 op_modifier, + u16 op, u16 token, int event) +{ + u32 __iomem *hcr = (u32 __iomem *)(hr_dev->reg_base + ROCEE_MB1_REG); + unsigned long end; + u32 val = 0; + __le32 tmp; + + end = msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS) + jiffies; + while (hns_roce_v1_cmd_pending(hr_dev)) { + if (time_after(jiffies, end)) { + dev_err(hr_dev->dev, "jiffies=%d end=%d\n", + (int)jiffies, (int)end); + return -EAGAIN; + } + cond_resched(); + } + + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_CMD_M, ROCEE_MB6_ROCEE_MB_CMD_S, + op); + roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_CMD_MDF_M, + ROCEE_MB6_ROCEE_MB_CMD_MDF_S, op_modifier); + roce_set_bit(tmp, ROCEE_MB6_ROCEE_MB_EVENT_S, event); + roce_set_bit(tmp, ROCEE_MB6_ROCEE_MB_HW_RUN_S, 1); + roce_set_field(tmp, ROCEE_MB6_ROCEE_MB_TOKEN_M, + ROCEE_MB6_ROCEE_MB_TOKEN_S, token); + + val = le32_to_cpu(tmp); + writeq(in_param, hcr + 0); + writeq(out_param, hcr + 2); + writel(in_modifier, hcr + 4); + /* Memory barrier */ + wmb(); + + writel(val, hcr + 5); + + return 0; +} + +static int hns_roce_v1_chk_mbox(struct hns_roce_dev *hr_dev, + unsigned long timeout) +{ + u8 __iomem *hcr = hr_dev->reg_base + ROCEE_MB1_REG; + unsigned long end; + u32 status = 0; + + end = msecs_to_jiffies(timeout) + jiffies; + while (hns_roce_v1_cmd_pending(hr_dev) && time_before(jiffies, end)) + cond_resched(); + + if (hns_roce_v1_cmd_pending(hr_dev)) { + dev_err(hr_dev->dev, "[cmd_poll]hw run cmd TIMEDOUT!\n"); + return -ETIMEDOUT; + } + + status = le32_to_cpu((__force __le32) + __raw_readl(hcr + HCR_STATUS_OFFSET)); + if ((status & STATUS_MASK) != 0x1) { + dev_err(hr_dev->dev, "mailbox status 0x%x!\n", status); + return -EBUSY; + } + + return 0; +} + +static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, + int gid_index, const union ib_gid *gid, + const struct ib_gid_attr *attr) +{ + unsigned long flags; + u32 *p = NULL; + u8 gid_idx; + + gid_idx = hns_get_gid_index(hr_dev, port, gid_index); + + spin_lock_irqsave(&hr_dev->iboe.lock, flags); + + p = (u32 *)&gid->raw[0]; + roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG + + (HNS_ROCE_V1_GID_NUM * gid_idx)); + + p = (u32 *)&gid->raw[4]; + roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_ML_0_REG + + (HNS_ROCE_V1_GID_NUM * gid_idx)); + + p = (u32 *)&gid->raw[8]; + roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_MH_0_REG + + (HNS_ROCE_V1_GID_NUM * gid_idx)); + + p = (u32 *)&gid->raw[0xc]; + roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG + + (HNS_ROCE_V1_GID_NUM * gid_idx)); + + spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); + + return 0; +} + +static int hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, + u8 *addr) +{ + u32 reg_smac_l; + u16 reg_smac_h; + __le32 tmp; + u16 *p_h; + u32 *p; + u32 val; + + /* + * When mac changed, loopback may fail + * because of smac not equal to dmac. + * We Need to release and create reserved qp again. + */ + if (hr_dev->hw->dereg_mr) { + int ret; + + ret = hns_roce_v1_recreate_lp_qp(hr_dev); + if (ret && ret != -ETIMEDOUT) + return ret; + } + + p = (u32 *)(&addr[0]); + reg_smac_l = *p; + roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_SMAC_L_0_REG + + PHY_PORT_OFFSET * phy_port); + + val = roce_read(hr_dev, + ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET); + tmp = cpu_to_le32(val); + p_h = (u16 *)(&addr[4]); + reg_smac_h = *p_h; + roce_set_field(tmp, ROCEE_SMAC_H_ROCEE_SMAC_H_M, + ROCEE_SMAC_H_ROCEE_SMAC_H_S, reg_smac_h); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET, + val); + + return 0; +} + +static void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port, + enum ib_mtu mtu) +{ + __le32 tmp; + u32 val; + + val = roce_read(hr_dev, + ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET); + tmp = cpu_to_le32(val); + roce_set_field(tmp, ROCEE_SMAC_H_ROCEE_PORT_MTU_M, + ROCEE_SMAC_H_ROCEE_PORT_MTU_S, mtu); + val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET, + val); +} + +static int hns_roce_v1_write_mtpt(struct hns_roce_dev *hr_dev, void *mb_buf, + struct hns_roce_mr *mr, + unsigned long mtpt_idx) +{ + u64 pages[HNS_ROCE_MAX_INNER_MTPT_NUM] = { 0 }; + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_v1_mpt_entry *mpt_entry; + dma_addr_t pbl_ba; + int count; + int i; + + /* MPT filled into mailbox buf */ + mpt_entry = (struct hns_roce_v1_mpt_entry *)mb_buf; + memset(mpt_entry, 0, sizeof(*mpt_entry)); + + roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_STATE_M, + MPT_BYTE_4_KEY_STATE_S, KEY_VALID); + roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_M, + MPT_BYTE_4_KEY_S, mr->key); + roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_PAGE_SIZE_M, + MPT_BYTE_4_PAGE_SIZE_S, MR_SIZE_4K); + roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_TYPE_S, 0); + roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_BIND_ENABLE_S, + (mr->access & IB_ACCESS_MW_BIND ? 1 : 0)); + roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_OWN_S, 0); + roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M, + MPT_BYTE_4_MEMORY_LOCATION_TYPE_S, mr->type); + roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_ATOMIC_S, 0); + roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_LOCAL_WRITE_S, + (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0)); + roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_WRITE_S, + (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0)); + roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_READ_S, + (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0)); + roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S, + 0); + roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_ADDRESS_TYPE_S, 0); + + roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M, + MPT_BYTE_12_PBL_ADDR_H_S, 0); + roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_MW_BIND_COUNTER_M, + MPT_BYTE_12_MW_BIND_COUNTER_S, 0); + + mpt_entry->virt_addr_l = cpu_to_le32((u32)mr->iova); + mpt_entry->virt_addr_h = cpu_to_le32((u32)(mr->iova >> 32)); + mpt_entry->length = cpu_to_le32((u32)mr->size); + + roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_PD_M, + MPT_BYTE_28_PD_S, mr->pd); + roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_L_KEY_IDX_L_M, + MPT_BYTE_28_L_KEY_IDX_L_S, mtpt_idx); + roce_set_field(mpt_entry->mpt_byte_64, MPT_BYTE_64_L_KEY_IDX_H_M, + MPT_BYTE_64_L_KEY_IDX_H_S, mtpt_idx >> MTPT_IDX_SHIFT); + + /* DMA memory register */ + if (mr->type == MR_TYPE_DMA) + return 0; + + count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages, + ARRAY_SIZE(pages), &pbl_ba); + if (count < 1) { + ibdev_err(ibdev, "failed to find PBL mtr, count = %d.", count); + return -ENOBUFS; + } + + /* Register user mr */ + for (i = 0; i < count; i++) { + switch (i) { + case 0: + mpt_entry->pa0_l = cpu_to_le32((u32)(pages[i])); + roce_set_field(mpt_entry->mpt_byte_36, + MPT_BYTE_36_PA0_H_M, + MPT_BYTE_36_PA0_H_S, + (u32)(pages[i] >> PAGES_SHIFT_32)); + break; + case 1: + roce_set_field(mpt_entry->mpt_byte_36, + MPT_BYTE_36_PA1_L_M, + MPT_BYTE_36_PA1_L_S, (u32)(pages[i])); + roce_set_field(mpt_entry->mpt_byte_40, + MPT_BYTE_40_PA1_H_M, + MPT_BYTE_40_PA1_H_S, + (u32)(pages[i] >> PAGES_SHIFT_24)); + break; + case 2: + roce_set_field(mpt_entry->mpt_byte_40, + MPT_BYTE_40_PA2_L_M, + MPT_BYTE_40_PA2_L_S, (u32)(pages[i])); + roce_set_field(mpt_entry->mpt_byte_44, + MPT_BYTE_44_PA2_H_M, + MPT_BYTE_44_PA2_H_S, + (u32)(pages[i] >> PAGES_SHIFT_16)); + break; + case 3: + roce_set_field(mpt_entry->mpt_byte_44, + MPT_BYTE_44_PA3_L_M, + MPT_BYTE_44_PA3_L_S, (u32)(pages[i])); + roce_set_field(mpt_entry->mpt_byte_48, + MPT_BYTE_48_PA3_H_M, + MPT_BYTE_48_PA3_H_S, + (u32)(pages[i] >> PAGES_SHIFT_8)); + break; + case 4: + mpt_entry->pa4_l = cpu_to_le32((u32)(pages[i])); + roce_set_field(mpt_entry->mpt_byte_56, + MPT_BYTE_56_PA4_H_M, + MPT_BYTE_56_PA4_H_S, + (u32)(pages[i] >> PAGES_SHIFT_32)); + break; + case 5: + roce_set_field(mpt_entry->mpt_byte_56, + MPT_BYTE_56_PA5_L_M, + MPT_BYTE_56_PA5_L_S, (u32)(pages[i])); + roce_set_field(mpt_entry->mpt_byte_60, + MPT_BYTE_60_PA5_H_M, + MPT_BYTE_60_PA5_H_S, + (u32)(pages[i] >> PAGES_SHIFT_24)); + break; + case 6: + roce_set_field(mpt_entry->mpt_byte_60, + MPT_BYTE_60_PA6_L_M, + MPT_BYTE_60_PA6_L_S, (u32)(pages[i])); + roce_set_field(mpt_entry->mpt_byte_64, + MPT_BYTE_64_PA6_H_M, + MPT_BYTE_64_PA6_H_S, + (u32)(pages[i] >> PAGES_SHIFT_16)); + break; + default: + break; + } + } + + mpt_entry->pbl_addr_l = cpu_to_le32(pbl_ba); + roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M, + MPT_BYTE_12_PBL_ADDR_H_S, upper_32_bits(pbl_ba)); + + return 0; +} + +static void *get_cqe(struct hns_roce_cq *hr_cq, int n) +{ + return hns_roce_buf_offset(hr_cq->mtr.kmem, n * HNS_ROCE_V1_CQE_SIZE); +} + +static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n) +{ + struct hns_roce_cqe *hr_cqe = get_cqe(hr_cq, n & hr_cq->ib_cq.cqe); + + /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */ + return (roce_get_bit(hr_cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S) ^ + !!(n & hr_cq->cq_depth)) ? hr_cqe : NULL; +} + +static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *hr_cq) +{ + return get_sw_cqe(hr_cq, hr_cq->cons_index); +} + +static void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index) +{ + __le32 doorbell[2]; + + doorbell[0] = cpu_to_le32(cons_index & ((hr_cq->cq_depth << 1) - 1)); + doorbell[1] = 0; + roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1); + roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M, + ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3); + roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M, + ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 0); + roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M, + ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq->cqn); + + hns_roce_write64_k(doorbell, hr_cq->cq_db_l); +} + +static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, + struct hns_roce_srq *srq) +{ + struct hns_roce_cqe *cqe, *dest; + u32 prod_index; + int nfreed = 0; + u8 owner_bit; + + for (prod_index = hr_cq->cons_index; get_sw_cqe(hr_cq, prod_index); + ++prod_index) { + if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe) + break; + } + + /* + * Now backwards through the CQ, removing CQ entries + * that match our QP by overwriting them with next entries. + */ + while ((int) --prod_index - (int) hr_cq->cons_index >= 0) { + cqe = get_cqe(hr_cq, prod_index & hr_cq->ib_cq.cqe); + if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M, + CQE_BYTE_16_LOCAL_QPN_S) & + HNS_ROCE_CQE_QPN_MASK) == qpn) { + /* In v1 engine, not support SRQ */ + ++nfreed; + } else if (nfreed) { + dest = get_cqe(hr_cq, (prod_index + nfreed) & + hr_cq->ib_cq.cqe); + owner_bit = roce_get_bit(dest->cqe_byte_4, + CQE_BYTE_4_OWNER_S); + memcpy(dest, cqe, sizeof(*cqe)); + roce_set_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S, + owner_bit); + } + } + + if (nfreed) { + hr_cq->cons_index += nfreed; + /* + * Make sure update of buffer contents is done before + * updating consumer index. + */ + wmb(); + + hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index); + } +} + +static void hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, + struct hns_roce_srq *srq) +{ + spin_lock_irq(&hr_cq->lock); + __hns_roce_v1_cq_clean(hr_cq, qpn, srq); + spin_unlock_irq(&hr_cq->lock); +} + +static void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev, + struct hns_roce_cq *hr_cq, void *mb_buf, + u64 *mtts, dma_addr_t dma_handle) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf; + struct hns_roce_cq_context *cq_context = mb_buf; + dma_addr_t tptr_dma_addr; + int offset; + + memset(cq_context, 0, sizeof(*cq_context)); + + /* Get the tptr for this CQ. */ + offset = hr_cq->cqn * HNS_ROCE_V1_TPTR_ENTRY_SIZE; + tptr_dma_addr = tptr_buf->map + offset; + hr_cq->tptr_addr = (u16 *)(tptr_buf->buf + offset); + + /* Register cq_context members */ + roce_set_field(cq_context->cqc_byte_4, + CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M, + CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S, CQ_STATE_VALID); + roce_set_field(cq_context->cqc_byte_4, CQ_CONTEXT_CQC_BYTE_4_CQN_M, + CQ_CONTEXT_CQC_BYTE_4_CQN_S, hr_cq->cqn); + + cq_context->cq_bt_l = cpu_to_le32((u32)dma_handle); + + roce_set_field(cq_context->cqc_byte_12, + CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M, + CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S, + ((u64)dma_handle >> 32)); + roce_set_field(cq_context->cqc_byte_12, + CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M, + CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S, + ilog2(hr_cq->cq_depth)); + roce_set_field(cq_context->cqc_byte_12, CQ_CONTEXT_CQC_BYTE_12_CEQN_M, + CQ_CONTEXT_CQC_BYTE_12_CEQN_S, hr_cq->vector); + + cq_context->cur_cqe_ba0_l = cpu_to_le32((u32)(mtts[0])); + + roce_set_field(cq_context->cqc_byte_20, + CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M, + CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S, (mtts[0]) >> 32); + /* Dedicated hardware, directly set 0 */ + roce_set_field(cq_context->cqc_byte_20, + CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M, + CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S, 0); + /** + * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of + * using 4K page, and shift more 32 because of + * caculating the high 32 bit value evaluated to hardware. + */ + roce_set_field(cq_context->cqc_byte_20, + CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M, + CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S, + tptr_dma_addr >> 44); + + cq_context->cqe_tptr_addr_l = cpu_to_le32((u32)(tptr_dma_addr >> 12)); + + roce_set_field(cq_context->cqc_byte_32, + CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M, + CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S, 0); + roce_set_bit(cq_context->cqc_byte_32, + CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S, 0); + roce_set_bit(cq_context->cqc_byte_32, + CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S, 0); + roce_set_bit(cq_context->cqc_byte_32, + CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S, 0); + roce_set_bit(cq_context->cqc_byte_32, + CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S, + 0); + /* The initial value of cq's ci is 0 */ + roce_set_field(cq_context->cqc_byte_32, + CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M, + CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0); +} + +static int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, + enum ib_cq_notify_flags flags) +{ + struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); + u32 notification_flag; + __le32 doorbell[2] = {}; + + notification_flag = (flags & IB_CQ_SOLICITED_MASK) == + IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL; + /* + * flags = 0; Notification Flag = 1, next + * flags = 1; Notification Flag = 0, solocited + */ + doorbell[0] = + cpu_to_le32(hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1)); + roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1); + roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M, + ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3); + roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M, + ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 1); + roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M, + ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, + hr_cq->cqn | notification_flag); + + hns_roce_write64_k(doorbell, hr_cq->cq_db_l); + + return 0; +} + +static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq, + struct hns_roce_qp **cur_qp, struct ib_wc *wc) +{ + int qpn; + int is_send; + u16 wqe_ctr; + u32 status; + u32 opcode; + struct hns_roce_cqe *cqe; + struct hns_roce_qp *hr_qp; + struct hns_roce_wq *wq; + struct hns_roce_wqe_ctrl_seg *sq_wqe; + struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); + struct device *dev = &hr_dev->pdev->dev; + + /* Find cqe according consumer index */ + cqe = next_cqe_sw(hr_cq); + if (!cqe) + return -EAGAIN; + + ++hr_cq->cons_index; + /* Memory barrier */ + rmb(); + /* 0->SQ, 1->RQ */ + is_send = !(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_SQ_RQ_FLAG_S)); + + /* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */ + if (roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M, + CQE_BYTE_16_LOCAL_QPN_S) <= 1) { + qpn = roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_PORT_NUM_M, + CQE_BYTE_20_PORT_NUM_S) + + roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M, + CQE_BYTE_16_LOCAL_QPN_S) * + HNS_ROCE_MAX_PORTS; + } else { + qpn = roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M, + CQE_BYTE_16_LOCAL_QPN_S); + } + + if (!*cur_qp || (qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->qpn) { + hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); + if (unlikely(!hr_qp)) { + dev_err(dev, "CQ %06lx with entry for unknown QPN %06x\n", + hr_cq->cqn, (qpn & HNS_ROCE_CQE_QPN_MASK)); + return -EINVAL; + } + + *cur_qp = hr_qp; + } + + wc->qp = &(*cur_qp)->ibqp; + wc->vendor_err = 0; + + status = roce_get_field(cqe->cqe_byte_4, + CQE_BYTE_4_STATUS_OF_THE_OPERATION_M, + CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) & + HNS_ROCE_CQE_STATUS_MASK; + switch (status) { + case HNS_ROCE_CQE_SUCCESS: + wc->status = IB_WC_SUCCESS; + break; + case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR: + wc->status = IB_WC_LOC_LEN_ERR; + break; + case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR: + wc->status = IB_WC_LOC_QP_OP_ERR; + break; + case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR: + wc->status = IB_WC_LOC_PROT_ERR; + break; + case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR: + wc->status = IB_WC_WR_FLUSH_ERR; + break; + case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR: + wc->status = IB_WC_MW_BIND_ERR; + break; + case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR: + wc->status = IB_WC_BAD_RESP_ERR; + break; + case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR: + wc->status = IB_WC_LOC_ACCESS_ERR; + break; + case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: + wc->status = IB_WC_REM_INV_REQ_ERR; + break; + case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR: + wc->status = IB_WC_REM_ACCESS_ERR; + break; + case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR: + wc->status = IB_WC_REM_OP_ERR; + break; + case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: + wc->status = IB_WC_RETRY_EXC_ERR; + break; + case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR: + wc->status = IB_WC_RNR_RETRY_EXC_ERR; + break; + default: + wc->status = IB_WC_GENERAL_ERR; + break; + } + + /* CQE status error, directly return */ + if (wc->status != IB_WC_SUCCESS) + return 0; + + if (is_send) { + /* SQ conrespond to CQE */ + sq_wqe = hns_roce_get_send_wqe(*cur_qp, + roce_get_field(cqe->cqe_byte_4, + CQE_BYTE_4_WQE_INDEX_M, + CQE_BYTE_4_WQE_INDEX_S) & + ((*cur_qp)->sq.wqe_cnt-1)); + switch (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_OPCODE_MASK) { + case HNS_ROCE_WQE_OPCODE_SEND: + wc->opcode = IB_WC_SEND; + break; + case HNS_ROCE_WQE_OPCODE_RDMA_READ: + wc->opcode = IB_WC_RDMA_READ; + wc->byte_len = le32_to_cpu(cqe->byte_cnt); + break; + case HNS_ROCE_WQE_OPCODE_RDMA_WRITE: + wc->opcode = IB_WC_RDMA_WRITE; + break; + case HNS_ROCE_WQE_OPCODE_LOCAL_INV: + wc->opcode = IB_WC_LOCAL_INV; + break; + case HNS_ROCE_WQE_OPCODE_UD_SEND: + wc->opcode = IB_WC_SEND; + break; + default: + wc->status = IB_WC_GENERAL_ERR; + break; + } + wc->wc_flags = (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_IMM ? + IB_WC_WITH_IMM : 0); + + wq = &(*cur_qp)->sq; + if ((*cur_qp)->sq_signal_bits) { + /* + * If sg_signal_bit is 1, + * firstly tail pointer updated to wqe + * which current cqe correspond to + */ + wqe_ctr = (u16)roce_get_field(cqe->cqe_byte_4, + CQE_BYTE_4_WQE_INDEX_M, + CQE_BYTE_4_WQE_INDEX_S); + wq->tail += (wqe_ctr - (u16)wq->tail) & + (wq->wqe_cnt - 1); + } + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; + } else { + /* RQ conrespond to CQE */ + wc->byte_len = le32_to_cpu(cqe->byte_cnt); + opcode = roce_get_field(cqe->cqe_byte_4, + CQE_BYTE_4_OPERATION_TYPE_M, + CQE_BYTE_4_OPERATION_TYPE_S) & + HNS_ROCE_CQE_OPCODE_MASK; + switch (opcode) { + case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE: + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; + wc->wc_flags = IB_WC_WITH_IMM; + wc->ex.imm_data = + cpu_to_be32(le32_to_cpu(cqe->immediate_data)); + break; + case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE: + if (roce_get_bit(cqe->cqe_byte_4, + CQE_BYTE_4_IMM_INDICATOR_S)) { + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_IMM; + wc->ex.imm_data = cpu_to_be32( + le32_to_cpu(cqe->immediate_data)); + } else { + wc->opcode = IB_WC_RECV; + wc->wc_flags = 0; + } + break; + default: + wc->status = IB_WC_GENERAL_ERR; + break; + } + + /* Update tail pointer, record wr_id */ + wq = &(*cur_qp)->rq; + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; + wc->sl = (u8)roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_SL_M, + CQE_BYTE_20_SL_S); + wc->src_qp = (u8)roce_get_field(cqe->cqe_byte_20, + CQE_BYTE_20_REMOTE_QPN_M, + CQE_BYTE_20_REMOTE_QPN_S); + wc->wc_flags |= (roce_get_bit(cqe->cqe_byte_20, + CQE_BYTE_20_GRH_PRESENT_S) ? + IB_WC_GRH : 0); + wc->pkey_index = (u16)roce_get_field(cqe->cqe_byte_28, + CQE_BYTE_28_P_KEY_IDX_M, + CQE_BYTE_28_P_KEY_IDX_S); + } + + return 0; +} + +int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) +{ + struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); + struct hns_roce_qp *cur_qp = NULL; + unsigned long flags; + int npolled; + int ret = 0; + + spin_lock_irqsave(&hr_cq->lock, flags); + + for (npolled = 0; npolled < num_entries; ++npolled) { + ret = hns_roce_v1_poll_one(hr_cq, &cur_qp, wc + npolled); + if (ret) + break; + } + + if (npolled) { + *hr_cq->tptr_addr = hr_cq->cons_index & + ((hr_cq->cq_depth << 1) - 1); + + /* Memroy barrier */ + wmb(); + hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index); + } + + spin_unlock_irqrestore(&hr_cq->lock, flags); + + if (ret == 0 || ret == -EAGAIN) + return npolled; + else + return ret; +} + +static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, int obj, + int step_idx) +{ + struct hns_roce_v1_priv *priv = hr_dev->priv; + struct device *dev = &hr_dev->pdev->dev; + long end = HW_SYNC_TIMEOUT_MSECS; + __le32 bt_cmd_val[2] = {0}; + unsigned long flags = 0; + void __iomem *bt_cmd; + u64 bt_ba = 0; + + switch (table->type) { + case HEM_TYPE_QPC: + bt_ba = priv->bt_table.qpc_buf.map >> 12; + break; + case HEM_TYPE_MTPT: + bt_ba = priv->bt_table.mtpt_buf.map >> 12; + break; + case HEM_TYPE_CQC: + bt_ba = priv->bt_table.cqc_buf.map >> 12; + break; + case HEM_TYPE_SRQC: + dev_dbg(dev, "HEM_TYPE_SRQC not support.\n"); + return -EINVAL; + default: + return 0; + } + roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, + ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type); + roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M, + ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj); + roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0); + roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1); + + spin_lock_irqsave(&hr_dev->bt_cmd_lock, flags); + + bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; + + while (1) { + if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) { + if (!end) { + dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); + spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, + flags); + return -EBUSY; + } + } else { + break; + } + mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); + end -= HW_SYNC_SLEEP_TIME_INTERVAL; + } + + bt_cmd_val[0] = cpu_to_le32(bt_ba); + roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M, + ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, bt_ba >> 32); + hns_roce_write64_k(bt_cmd_val, hr_dev->reg_base + ROCEE_BT_CMD_L_REG); + + spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags); + + return 0; +} + +static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev, + enum hns_roce_qp_state cur_state, + enum hns_roce_qp_state new_state, + struct hns_roce_qp_context *context, + struct hns_roce_qp *hr_qp) +{ + static const u16 + op[HNS_ROCE_QP_NUM_STATE][HNS_ROCE_QP_NUM_STATE] = { + [HNS_ROCE_QP_STATE_RST] = { + [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, + [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, + [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP, + }, + [HNS_ROCE_QP_STATE_INIT] = { + [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, + [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, + /* Note: In v1 engine, HW doesn't support RST2INIT. + * We use RST2INIT cmd instead of INIT2INIT. + */ + [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP, + [HNS_ROCE_QP_STATE_RTR] = HNS_ROCE_CMD_INIT2RTR_QP, + }, + [HNS_ROCE_QP_STATE_RTR] = { + [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, + [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, + [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTR2RTS_QP, + }, + [HNS_ROCE_QP_STATE_RTS] = { + [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, + [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, + [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTS2RTS_QP, + [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_RTS2SQD_QP, + }, + [HNS_ROCE_QP_STATE_SQD] = { + [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, + [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, + [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_SQD2RTS_QP, + [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_SQD2SQD_QP, + }, + [HNS_ROCE_QP_STATE_ERR] = { + [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP, + [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP, + } + }; + + struct hns_roce_cmd_mailbox *mailbox; + struct device *dev = &hr_dev->pdev->dev; + int ret; + + if (cur_state >= HNS_ROCE_QP_NUM_STATE || + new_state >= HNS_ROCE_QP_NUM_STATE || + !op[cur_state][new_state]) { + dev_err(dev, "[modify_qp]not support state %d to %d\n", + cur_state, new_state); + return -EINVAL; + } + + if (op[cur_state][new_state] == HNS_ROCE_CMD_2RST_QP) + return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2, + HNS_ROCE_CMD_2RST_QP, + HNS_ROCE_CMD_TIMEOUT_MSECS); + + if (op[cur_state][new_state] == HNS_ROCE_CMD_2ERR_QP) + return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2, + HNS_ROCE_CMD_2ERR_QP, + HNS_ROCE_CMD_TIMEOUT_MSECS); + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + memcpy(mailbox->buf, context, sizeof(*context)); + + ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0, + op[cur_state][new_state], + HNS_ROCE_CMD_TIMEOUT_MSECS); + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + return ret; +} + +static int find_wqe_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + u64 *sq_ba, u64 *rq_ba, dma_addr_t *bt_ba) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + int count; + + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, sq_ba, 1, bt_ba); + if (count < 1) { + ibdev_err(ibdev, "Failed to find SQ ba\n"); + return -ENOBUFS; + } + + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, rq_ba, + 1, NULL); + if (!count) { + ibdev_err(ibdev, "Failed to find RQ ba\n"); + return -ENOBUFS; + } + + return 0; +} + +static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, + int attr_mask, enum ib_qp_state cur_state, + enum ib_qp_state new_state) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct hns_roce_sqp_context *context; + dma_addr_t dma_handle = 0; + u32 __iomem *addr; + u64 sq_ba = 0; + u64 rq_ba = 0; + __le32 tmp; + u32 reg_val; + + context = kzalloc(sizeof(*context), GFP_KERNEL); + if (!context) + return -ENOMEM; + + /* Search QP buf's MTTs */ + if (find_wqe_mtt(hr_dev, hr_qp, &sq_ba, &rq_ba, &dma_handle)) + goto out; + + if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { + roce_set_field(context->qp1c_bytes_4, + QP1C_BYTES_4_SQ_WQE_SHIFT_M, + QP1C_BYTES_4_SQ_WQE_SHIFT_S, + ilog2((unsigned int)hr_qp->sq.wqe_cnt)); + roce_set_field(context->qp1c_bytes_4, + QP1C_BYTES_4_RQ_WQE_SHIFT_M, + QP1C_BYTES_4_RQ_WQE_SHIFT_S, + ilog2((unsigned int)hr_qp->rq.wqe_cnt)); + roce_set_field(context->qp1c_bytes_4, QP1C_BYTES_4_PD_M, + QP1C_BYTES_4_PD_S, to_hr_pd(ibqp->pd)->pdn); + + context->sq_rq_bt_l = cpu_to_le32(dma_handle); + roce_set_field(context->qp1c_bytes_12, + QP1C_BYTES_12_SQ_RQ_BT_H_M, + QP1C_BYTES_12_SQ_RQ_BT_H_S, + upper_32_bits(dma_handle)); + + roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M, + QP1C_BYTES_16_RQ_HEAD_S, hr_qp->rq.head); + roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_PORT_NUM_M, + QP1C_BYTES_16_PORT_NUM_S, hr_qp->phy_port); + roce_set_bit(context->qp1c_bytes_16, + QP1C_BYTES_16_SIGNALING_TYPE_S, + hr_qp->sq_signal_bits); + roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_BA_FLG_S, + 1); + roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_SQ_BA_FLG_S, + 1); + roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_QP1_ERR_S, + 0); + + roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_SQ_HEAD_M, + QP1C_BYTES_20_SQ_HEAD_S, hr_qp->sq.head); + roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_PKEY_IDX_M, + QP1C_BYTES_20_PKEY_IDX_S, attr->pkey_index); + + context->cur_rq_wqe_ba_l = cpu_to_le32(rq_ba); + + roce_set_field(context->qp1c_bytes_28, + QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M, + QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S, + upper_32_bits(rq_ba)); + roce_set_field(context->qp1c_bytes_28, + QP1C_BYTES_28_RQ_CUR_IDX_M, + QP1C_BYTES_28_RQ_CUR_IDX_S, 0); + + roce_set_field(context->qp1c_bytes_32, + QP1C_BYTES_32_RX_CQ_NUM_M, + QP1C_BYTES_32_RX_CQ_NUM_S, + to_hr_cq(ibqp->recv_cq)->cqn); + roce_set_field(context->qp1c_bytes_32, + QP1C_BYTES_32_TX_CQ_NUM_M, + QP1C_BYTES_32_TX_CQ_NUM_S, + to_hr_cq(ibqp->send_cq)->cqn); + + context->cur_sq_wqe_ba_l = cpu_to_le32(sq_ba); + + roce_set_field(context->qp1c_bytes_40, + QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M, + QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S, + upper_32_bits(sq_ba)); + roce_set_field(context->qp1c_bytes_40, + QP1C_BYTES_40_SQ_CUR_IDX_M, + QP1C_BYTES_40_SQ_CUR_IDX_S, 0); + + /* Copy context to QP1C register */ + addr = (u32 __iomem *)(hr_dev->reg_base + + ROCEE_QP1C_CFG0_0_REG + + hr_qp->phy_port * sizeof(*context)); + + writel(le32_to_cpu(context->qp1c_bytes_4), addr); + writel(le32_to_cpu(context->sq_rq_bt_l), addr + 1); + writel(le32_to_cpu(context->qp1c_bytes_12), addr + 2); + writel(le32_to_cpu(context->qp1c_bytes_16), addr + 3); + writel(le32_to_cpu(context->qp1c_bytes_20), addr + 4); + writel(le32_to_cpu(context->cur_rq_wqe_ba_l), addr + 5); + writel(le32_to_cpu(context->qp1c_bytes_28), addr + 6); + writel(le32_to_cpu(context->qp1c_bytes_32), addr + 7); + writel(le32_to_cpu(context->cur_sq_wqe_ba_l), addr + 8); + writel(le32_to_cpu(context->qp1c_bytes_40), addr + 9); + } + + /* Modify QP1C status */ + reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG + + hr_qp->phy_port * sizeof(*context)); + tmp = cpu_to_le32(reg_val); + roce_set_field(tmp, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M, + ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S, new_state); + reg_val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG + + hr_qp->phy_port * sizeof(*context), reg_val); + + hr_qp->state = new_state; + if (new_state == IB_QPS_RESET) { + hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn, + ibqp->srq ? to_hr_srq(ibqp->srq) : NULL); + if (ibqp->send_cq != ibqp->recv_cq) + hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq), + hr_qp->qpn, NULL); + + hr_qp->rq.head = 0; + hr_qp->rq.tail = 0; + hr_qp->sq.head = 0; + hr_qp->sq.tail = 0; + } + + kfree(context); + return 0; + +out: + kfree(context); + return -EINVAL; +} + +static bool check_qp_state(enum ib_qp_state cur_state, + enum ib_qp_state new_state) +{ + static const bool sm[][IB_QPS_ERR + 1] = { + [IB_QPS_RESET] = { [IB_QPS_RESET] = true, + [IB_QPS_INIT] = true }, + [IB_QPS_INIT] = { [IB_QPS_RESET] = true, + [IB_QPS_INIT] = true, + [IB_QPS_RTR] = true, + [IB_QPS_ERR] = true }, + [IB_QPS_RTR] = { [IB_QPS_RESET] = true, + [IB_QPS_RTS] = true, + [IB_QPS_ERR] = true }, + [IB_QPS_RTS] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }, + [IB_QPS_SQD] = {}, + [IB_QPS_SQE] = {}, + [IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true } + }; + + return sm[cur_state][new_state]; +} + +static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, + int attr_mask, enum ib_qp_state cur_state, + enum ib_qp_state new_state) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct device *dev = &hr_dev->pdev->dev; + struct hns_roce_qp_context *context; + const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); + dma_addr_t dma_handle_2 = 0; + dma_addr_t dma_handle = 0; + __le32 doorbell[2] = {0}; + u64 *mtts_2 = NULL; + int ret = -EINVAL; + u64 sq_ba = 0; + u64 rq_ba = 0; + int port; + u8 port_num; + u8 *dmac; + u8 *smac; + + if (!check_qp_state(cur_state, new_state)) { + ibdev_err(ibqp->device, + "not support QP(%u) status from %d to %d\n", + ibqp->qp_num, cur_state, new_state); + return -EINVAL; + } + + context = kzalloc(sizeof(*context), GFP_KERNEL); + if (!context) + return -ENOMEM; + + /* Search qp buf's mtts */ + if (find_wqe_mtt(hr_dev, hr_qp, &sq_ba, &rq_ba, &dma_handle)) + goto out; + + /* Search IRRL's mtts */ + mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, + hr_qp->qpn, &dma_handle_2); + if (mtts_2 == NULL) { + dev_err(dev, "qp irrl_table find failed\n"); + goto out; + } + + /* + * Reset to init + * Mandatory param: + * IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS + * Optional param: NA + */ + if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { + roce_set_field(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M, + QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S, + to_hr_qp_type(hr_qp->ibqp.qp_type)); + + roce_set_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0); + roce_set_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S, + !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ)); + roce_set_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S, + !!(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) + ); + roce_set_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S, + !!(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) + ); + roce_set_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1); + roce_set_field(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M, + QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S, + ilog2((unsigned int)hr_qp->sq.wqe_cnt)); + roce_set_field(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M, + QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S, + ilog2((unsigned int)hr_qp->rq.wqe_cnt)); + roce_set_field(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTES_4_PD_M, + QP_CONTEXT_QPC_BYTES_4_PD_S, + to_hr_pd(ibqp->pd)->pdn); + hr_qp->access_flags = attr->qp_access_flags; + roce_set_field(context->qpc_bytes_8, + QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M, + QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S, + to_hr_cq(ibqp->send_cq)->cqn); + roce_set_field(context->qpc_bytes_8, + QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M, + QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S, + to_hr_cq(ibqp->recv_cq)->cqn); + + if (ibqp->srq) + roce_set_field(context->qpc_bytes_12, + QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M, + QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S, + to_hr_srq(ibqp->srq)->srqn); + + roce_set_field(context->qpc_bytes_12, + QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M, + QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S, + attr->pkey_index); + hr_qp->pkey_index = attr->pkey_index; + roce_set_field(context->qpc_bytes_16, + QP_CONTEXT_QPC_BYTES_16_QP_NUM_M, + QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn); + } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { + roce_set_field(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M, + QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S, + to_hr_qp_type(hr_qp->ibqp.qp_type)); + roce_set_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0); + if (attr_mask & IB_QP_ACCESS_FLAGS) { + roce_set_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S, + !!(attr->qp_access_flags & + IB_ACCESS_REMOTE_READ)); + roce_set_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S, + !!(attr->qp_access_flags & + IB_ACCESS_REMOTE_WRITE)); + } else { + roce_set_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S, + !!(hr_qp->access_flags & + IB_ACCESS_REMOTE_READ)); + roce_set_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S, + !!(hr_qp->access_flags & + IB_ACCESS_REMOTE_WRITE)); + } + + roce_set_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1); + roce_set_field(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M, + QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S, + ilog2((unsigned int)hr_qp->sq.wqe_cnt)); + roce_set_field(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M, + QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S, + ilog2((unsigned int)hr_qp->rq.wqe_cnt)); + roce_set_field(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTES_4_PD_M, + QP_CONTEXT_QPC_BYTES_4_PD_S, + to_hr_pd(ibqp->pd)->pdn); + + roce_set_field(context->qpc_bytes_8, + QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M, + QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S, + to_hr_cq(ibqp->send_cq)->cqn); + roce_set_field(context->qpc_bytes_8, + QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M, + QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S, + to_hr_cq(ibqp->recv_cq)->cqn); + + if (ibqp->srq) + roce_set_field(context->qpc_bytes_12, + QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M, + QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S, + to_hr_srq(ibqp->srq)->srqn); + if (attr_mask & IB_QP_PKEY_INDEX) + roce_set_field(context->qpc_bytes_12, + QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M, + QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S, + attr->pkey_index); + else + roce_set_field(context->qpc_bytes_12, + QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M, + QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S, + hr_qp->pkey_index); + + roce_set_field(context->qpc_bytes_16, + QP_CONTEXT_QPC_BYTES_16_QP_NUM_M, + QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn); + } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { + if ((attr_mask & IB_QP_ALT_PATH) || + (attr_mask & IB_QP_ACCESS_FLAGS) || + (attr_mask & IB_QP_PKEY_INDEX) || + (attr_mask & IB_QP_QKEY)) { + dev_err(dev, "INIT2RTR attr_mask error\n"); + goto out; + } + + dmac = (u8 *)attr->ah_attr.roce.dmac; + + context->sq_rq_bt_l = cpu_to_le32(dma_handle); + roce_set_field(context->qpc_bytes_24, + QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M, + QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S, + upper_32_bits(dma_handle)); + roce_set_bit(context->qpc_bytes_24, + QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S, + 1); + roce_set_field(context->qpc_bytes_24, + QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M, + QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S, + attr->min_rnr_timer); + context->irrl_ba_l = cpu_to_le32((u32)(dma_handle_2)); + roce_set_field(context->qpc_bytes_32, + QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M, + QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S, + ((u32)(dma_handle_2 >> 32)) & + QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M); + roce_set_field(context->qpc_bytes_32, + QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M, + QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S, 0); + roce_set_bit(context->qpc_bytes_32, + QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S, + 1); + roce_set_bit(context->qpc_bytes_32, + QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S, + hr_qp->sq_signal_bits); + + port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : + hr_qp->port; + smac = (u8 *)hr_dev->dev_addr[port]; + /* when dmac equals smac or loop_idc is 1, it should loopback */ + if (ether_addr_equal_unaligned(dmac, smac) || + hr_dev->loop_idc == 0x1) + roce_set_bit(context->qpc_bytes_32, + QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S, 1); + + roce_set_bit(context->qpc_bytes_32, + QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S, + rdma_ah_get_ah_flags(&attr->ah_attr)); + roce_set_field(context->qpc_bytes_32, + QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M, + QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S, + ilog2((unsigned int)attr->max_dest_rd_atomic)); + + if (attr_mask & IB_QP_DEST_QPN) + roce_set_field(context->qpc_bytes_36, + QP_CONTEXT_QPC_BYTES_36_DEST_QP_M, + QP_CONTEXT_QPC_BYTES_36_DEST_QP_S, + attr->dest_qp_num); + + /* Configure GID index */ + port_num = rdma_ah_get_port_num(&attr->ah_attr); + roce_set_field(context->qpc_bytes_36, + QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M, + QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S, + hns_get_gid_index(hr_dev, + port_num - 1, + grh->sgid_index)); + + memcpy(&(context->dmac_l), dmac, 4); + + roce_set_field(context->qpc_bytes_44, + QP_CONTEXT_QPC_BYTES_44_DMAC_H_M, + QP_CONTEXT_QPC_BYTES_44_DMAC_H_S, + *((u16 *)(&dmac[4]))); + roce_set_field(context->qpc_bytes_44, + QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M, + QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S, + rdma_ah_get_static_rate(&attr->ah_attr)); + roce_set_field(context->qpc_bytes_44, + QP_CONTEXT_QPC_BYTES_44_HOPLMT_M, + QP_CONTEXT_QPC_BYTES_44_HOPLMT_S, + grh->hop_limit); + + roce_set_field(context->qpc_bytes_48, + QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M, + QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S, + grh->flow_label); + roce_set_field(context->qpc_bytes_48, + QP_CONTEXT_QPC_BYTES_48_TCLASS_M, + QP_CONTEXT_QPC_BYTES_48_TCLASS_S, + grh->traffic_class); + roce_set_field(context->qpc_bytes_48, + QP_CONTEXT_QPC_BYTES_48_MTU_M, + QP_CONTEXT_QPC_BYTES_48_MTU_S, attr->path_mtu); + + memcpy(context->dgid, grh->dgid.raw, + sizeof(grh->dgid.raw)); + + dev_dbg(dev, "dmac:%x :%lx\n", context->dmac_l, + roce_get_field(context->qpc_bytes_44, + QP_CONTEXT_QPC_BYTES_44_DMAC_H_M, + QP_CONTEXT_QPC_BYTES_44_DMAC_H_S)); + + roce_set_field(context->qpc_bytes_68, + QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M, + QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S, + hr_qp->rq.head); + roce_set_field(context->qpc_bytes_68, + QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M, + QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S, 0); + + context->cur_rq_wqe_ba_l = cpu_to_le32(rq_ba); + + roce_set_field(context->qpc_bytes_76, + QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M, + QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S, + upper_32_bits(rq_ba)); + roce_set_field(context->qpc_bytes_76, + QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M, + QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S, 0); + + context->rx_rnr_time = 0; + + roce_set_field(context->qpc_bytes_84, + QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M, + QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S, + attr->rq_psn - 1); + roce_set_field(context->qpc_bytes_84, + QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M, + QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S, 0); + + roce_set_field(context->qpc_bytes_88, + QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M, + QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S, + attr->rq_psn); + roce_set_bit(context->qpc_bytes_88, + QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S, 0); + roce_set_bit(context->qpc_bytes_88, + QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S, 0); + roce_set_field(context->qpc_bytes_88, + QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M, + QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S, + 0); + roce_set_field(context->qpc_bytes_88, + QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M, + QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S, + 0); + + context->dma_length = 0; + context->r_key = 0; + context->va_l = 0; + context->va_h = 0; + + roce_set_field(context->qpc_bytes_108, + QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M, + QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S, 0); + roce_set_bit(context->qpc_bytes_108, + QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S, 0); + roce_set_bit(context->qpc_bytes_108, + QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S, 0); + + roce_set_field(context->qpc_bytes_112, + QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M, + QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S, 0); + roce_set_field(context->qpc_bytes_112, + QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M, + QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S, 0); + + /* For chip resp ack */ + roce_set_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M, + QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S, + hr_qp->phy_port); + roce_set_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_SL_M, + QP_CONTEXT_QPC_BYTES_156_SL_S, + rdma_ah_get_sl(&attr->ah_attr)); + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) { + /* If exist optional param, return error */ + if ((attr_mask & IB_QP_ALT_PATH) || + (attr_mask & IB_QP_ACCESS_FLAGS) || + (attr_mask & IB_QP_QKEY) || + (attr_mask & IB_QP_PATH_MIG_STATE) || + (attr_mask & IB_QP_CUR_STATE) || + (attr_mask & IB_QP_MIN_RNR_TIMER)) { + dev_err(dev, "RTR2RTS attr_mask error\n"); + goto out; + } + + context->rx_cur_sq_wqe_ba_l = cpu_to_le32(sq_ba); + + roce_set_field(context->qpc_bytes_120, + QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M, + QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S, + upper_32_bits(sq_ba)); + + roce_set_field(context->qpc_bytes_124, + QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M, + QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S, 0); + roce_set_field(context->qpc_bytes_124, + QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M, + QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S, 0); + + roce_set_field(context->qpc_bytes_128, + QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M, + QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S, + attr->sq_psn); + roce_set_bit(context->qpc_bytes_128, + QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S, 0); + roce_set_field(context->qpc_bytes_128, + QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M, + QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S, + 0); + roce_set_bit(context->qpc_bytes_128, + QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S, 0); + + roce_set_field(context->qpc_bytes_132, + QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M, + QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S, 0); + roce_set_field(context->qpc_bytes_132, + QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M, + QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S, 0); + + roce_set_field(context->qpc_bytes_136, + QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M, + QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S, + attr->sq_psn); + roce_set_field(context->qpc_bytes_136, + QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M, + QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S, + attr->sq_psn); + + roce_set_field(context->qpc_bytes_140, + QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M, + QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S, + (attr->sq_psn >> SQ_PSN_SHIFT)); + roce_set_field(context->qpc_bytes_140, + QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M, + QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S, 0); + roce_set_bit(context->qpc_bytes_140, + QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S, 0); + + roce_set_field(context->qpc_bytes_148, + QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M, + QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S, 0); + roce_set_field(context->qpc_bytes_148, + QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M, + QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S, + attr->retry_cnt); + roce_set_field(context->qpc_bytes_148, + QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M, + QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S, + attr->rnr_retry); + roce_set_field(context->qpc_bytes_148, + QP_CONTEXT_QPC_BYTES_148_LSN_M, + QP_CONTEXT_QPC_BYTES_148_LSN_S, 0x100); + + context->rnr_retry = 0; + + roce_set_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M, + QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S, + attr->retry_cnt); + if (attr->timeout < 0x12) { + dev_info(dev, "ack timeout value(0x%x) must bigger than 0x12.\n", + attr->timeout); + roce_set_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M, + QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S, + 0x12); + } else { + roce_set_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M, + QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S, + attr->timeout); + } + roce_set_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M, + QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S, + attr->rnr_retry); + roce_set_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M, + QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S, + hr_qp->phy_port); + roce_set_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_SL_M, + QP_CONTEXT_QPC_BYTES_156_SL_S, + rdma_ah_get_sl(&attr->ah_attr)); + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + roce_set_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M, + QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S, + ilog2((unsigned int)attr->max_rd_atomic)); + roce_set_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M, + QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S, 0); + context->pkt_use_len = 0; + + roce_set_field(context->qpc_bytes_164, + QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M, + QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S, attr->sq_psn); + roce_set_field(context->qpc_bytes_164, + QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M, + QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S, 0); + + roce_set_field(context->qpc_bytes_168, + QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M, + QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S, + attr->sq_psn); + roce_set_field(context->qpc_bytes_168, + QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M, + QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S, 0); + roce_set_field(context->qpc_bytes_168, + QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M, + QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S, 0); + roce_set_bit(context->qpc_bytes_168, + QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S, 0); + roce_set_bit(context->qpc_bytes_168, + QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S, 0); + roce_set_bit(context->qpc_bytes_168, + QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S, 0); + context->sge_use_len = 0; + + roce_set_field(context->qpc_bytes_176, + QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M, + QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S, 0); + roce_set_field(context->qpc_bytes_176, + QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M, + QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S, + 0); + roce_set_field(context->qpc_bytes_180, + QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M, + QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S, 0); + roce_set_field(context->qpc_bytes_180, + QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M, + QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S, 0); + + context->tx_cur_sq_wqe_ba_l = cpu_to_le32(sq_ba); + + roce_set_field(context->qpc_bytes_188, + QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M, + QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S, + upper_32_bits(sq_ba)); + roce_set_bit(context->qpc_bytes_188, + QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S, 0); + roce_set_field(context->qpc_bytes_188, + QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M, + QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S, + 0); + } + + /* Every status migrate must change state */ + roce_set_field(context->qpc_bytes_144, + QP_CONTEXT_QPC_BYTES_144_QP_STATE_M, + QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, new_state); + + /* SW pass context to HW */ + ret = hns_roce_v1_qp_modify(hr_dev, to_hns_roce_state(cur_state), + to_hns_roce_state(new_state), context, + hr_qp); + if (ret) { + dev_err(dev, "hns_roce_qp_modify failed\n"); + goto out; + } + + /* + * Use rst2init to instead of init2init with drv, + * need to hw to flash RQ HEAD by DB again + */ + if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { + /* Memory barrier */ + wmb(); + + roce_set_field(doorbell[0], RQ_DOORBELL_U32_4_RQ_HEAD_M, + RQ_DOORBELL_U32_4_RQ_HEAD_S, hr_qp->rq.head); + roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_QPN_M, + RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn); + roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_CMD_M, + RQ_DOORBELL_U32_8_CMD_S, 1); + roce_set_bit(doorbell[1], RQ_DOORBELL_U32_8_HW_SYNC_S, 1); + + if (ibqp->uobject) { + hr_qp->rq.db_reg_l = hr_dev->reg_base + + hr_dev->odb_offset + + DB_REG_OFFSET * hr_dev->priv_uar.index; + } + + hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l); + } + + hr_qp->state = new_state; + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) + hr_qp->resp_depth = attr->max_dest_rd_atomic; + if (attr_mask & IB_QP_PORT) { + hr_qp->port = attr->port_num - 1; + hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; + } + + if (new_state == IB_QPS_RESET && !ibqp->uobject) { + hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn, + ibqp->srq ? to_hr_srq(ibqp->srq) : NULL); + if (ibqp->send_cq != ibqp->recv_cq) + hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq), + hr_qp->qpn, NULL); + + hr_qp->rq.head = 0; + hr_qp->rq.tail = 0; + hr_qp->sq.head = 0; + hr_qp->sq.tail = 0; + } +out: + kfree(context); + return ret; +} + +static int hns_roce_v1_modify_qp(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + enum ib_qp_state cur_state, + enum ib_qp_state new_state) +{ + + if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) + return hns_roce_v1_m_sqp(ibqp, attr, attr_mask, cur_state, + new_state); + else + return hns_roce_v1_m_qp(ibqp, attr, attr_mask, cur_state, + new_state); +} + +static enum ib_qp_state to_ib_qp_state(enum hns_roce_qp_state state) +{ + switch (state) { + case HNS_ROCE_QP_STATE_RST: + return IB_QPS_RESET; + case HNS_ROCE_QP_STATE_INIT: + return IB_QPS_INIT; + case HNS_ROCE_QP_STATE_RTR: + return IB_QPS_RTR; + case HNS_ROCE_QP_STATE_RTS: + return IB_QPS_RTS; + case HNS_ROCE_QP_STATE_SQD: + return IB_QPS_SQD; + case HNS_ROCE_QP_STATE_ERR: + return IB_QPS_ERR; + default: + return IB_QPS_ERR; + } +} + +static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_roce_qp_context *hr_context) +{ + struct hns_roce_cmd_mailbox *mailbox; + int ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0, + HNS_ROCE_CMD_QUERY_QP, + HNS_ROCE_CMD_TIMEOUT_MSECS); + if (!ret) + memcpy(hr_context, mailbox->buf, sizeof(*hr_context)); + else + dev_err(&hr_dev->pdev->dev, "QUERY QP cmd process error\n"); + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} + +static int hns_roce_v1_q_sqp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct hns_roce_sqp_context context; + u32 addr; + + mutex_lock(&hr_qp->mutex); + + if (hr_qp->state == IB_QPS_RESET) { + qp_attr->qp_state = IB_QPS_RESET; + goto done; + } + + addr = ROCEE_QP1C_CFG0_0_REG + + hr_qp->port * sizeof(struct hns_roce_sqp_context); + context.qp1c_bytes_4 = cpu_to_le32(roce_read(hr_dev, addr)); + context.sq_rq_bt_l = cpu_to_le32(roce_read(hr_dev, addr + 1)); + context.qp1c_bytes_12 = cpu_to_le32(roce_read(hr_dev, addr + 2)); + context.qp1c_bytes_16 = cpu_to_le32(roce_read(hr_dev, addr + 3)); + context.qp1c_bytes_20 = cpu_to_le32(roce_read(hr_dev, addr + 4)); + context.cur_rq_wqe_ba_l = cpu_to_le32(roce_read(hr_dev, addr + 5)); + context.qp1c_bytes_28 = cpu_to_le32(roce_read(hr_dev, addr + 6)); + context.qp1c_bytes_32 = cpu_to_le32(roce_read(hr_dev, addr + 7)); + context.cur_sq_wqe_ba_l = cpu_to_le32(roce_read(hr_dev, addr + 8)); + context.qp1c_bytes_40 = cpu_to_le32(roce_read(hr_dev, addr + 9)); + + hr_qp->state = roce_get_field(context.qp1c_bytes_4, + QP1C_BYTES_4_QP_STATE_M, + QP1C_BYTES_4_QP_STATE_S); + qp_attr->qp_state = hr_qp->state; + qp_attr->path_mtu = IB_MTU_256; + qp_attr->path_mig_state = IB_MIG_ARMED; + qp_attr->qkey = QKEY_VAL; + qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; + qp_attr->rq_psn = 0; + qp_attr->sq_psn = 0; + qp_attr->dest_qp_num = 1; + qp_attr->qp_access_flags = 6; + + qp_attr->pkey_index = roce_get_field(context.qp1c_bytes_20, + QP1C_BYTES_20_PKEY_IDX_M, + QP1C_BYTES_20_PKEY_IDX_S); + qp_attr->port_num = hr_qp->port + 1; + qp_attr->sq_draining = 0; + qp_attr->max_rd_atomic = 0; + qp_attr->max_dest_rd_atomic = 0; + qp_attr->min_rnr_timer = 0; + qp_attr->timeout = 0; + qp_attr->retry_cnt = 0; + qp_attr->rnr_retry = 0; + qp_attr->alt_timeout = 0; + +done: + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; + qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs; + qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; + qp_attr->cap.max_send_sge = hr_qp->sq.max_gs; + qp_attr->cap.max_inline_data = 0; + qp_init_attr->cap = qp_attr->cap; + qp_init_attr->create_flags = 0; + + mutex_unlock(&hr_qp->mutex); + + return 0; +} + +static int hns_roce_v1_q_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct device *dev = &hr_dev->pdev->dev; + struct hns_roce_qp_context *context; + int tmp_qp_state; + int ret = 0; + int state; + + context = kzalloc(sizeof(*context), GFP_KERNEL); + if (!context) + return -ENOMEM; + + memset(qp_attr, 0, sizeof(*qp_attr)); + memset(qp_init_attr, 0, sizeof(*qp_init_attr)); + + mutex_lock(&hr_qp->mutex); + + if (hr_qp->state == IB_QPS_RESET) { + qp_attr->qp_state = IB_QPS_RESET; + goto done; + } + + ret = hns_roce_v1_query_qpc(hr_dev, hr_qp, context); + if (ret) { + dev_err(dev, "query qpc error\n"); + ret = -EINVAL; + goto out; + } + + state = roce_get_field(context->qpc_bytes_144, + QP_CONTEXT_QPC_BYTES_144_QP_STATE_M, + QP_CONTEXT_QPC_BYTES_144_QP_STATE_S); + tmp_qp_state = (int)to_ib_qp_state((enum hns_roce_qp_state)state); + if (tmp_qp_state == -1) { + dev_err(dev, "to_ib_qp_state error\n"); + ret = -EINVAL; + goto out; + } + hr_qp->state = (u8)tmp_qp_state; + qp_attr->qp_state = (enum ib_qp_state)hr_qp->state; + qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->qpc_bytes_48, + QP_CONTEXT_QPC_BYTES_48_MTU_M, + QP_CONTEXT_QPC_BYTES_48_MTU_S); + qp_attr->path_mig_state = IB_MIG_ARMED; + qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; + if (hr_qp->ibqp.qp_type == IB_QPT_UD) + qp_attr->qkey = QKEY_VAL; + + qp_attr->rq_psn = roce_get_field(context->qpc_bytes_88, + QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M, + QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S); + qp_attr->sq_psn = (u32)roce_get_field(context->qpc_bytes_164, + QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M, + QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S); + qp_attr->dest_qp_num = (u8)roce_get_field(context->qpc_bytes_36, + QP_CONTEXT_QPC_BYTES_36_DEST_QP_M, + QP_CONTEXT_QPC_BYTES_36_DEST_QP_S); + qp_attr->qp_access_flags = ((roce_get_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S)) << 2) | + ((roce_get_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S)) << 1) | + ((roce_get_bit(context->qpc_bytes_4, + QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S)) << 3); + + if (hr_qp->ibqp.qp_type == IB_QPT_RC || + hr_qp->ibqp.qp_type == IB_QPT_UC) { + struct ib_global_route *grh = + rdma_ah_retrieve_grh(&qp_attr->ah_attr); + + rdma_ah_set_sl(&qp_attr->ah_attr, + roce_get_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_SL_M, + QP_CONTEXT_QPC_BYTES_156_SL_S)); + rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH); + grh->flow_label = + roce_get_field(context->qpc_bytes_48, + QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M, + QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S); + grh->sgid_index = + roce_get_field(context->qpc_bytes_36, + QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M, + QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S); + grh->hop_limit = + roce_get_field(context->qpc_bytes_44, + QP_CONTEXT_QPC_BYTES_44_HOPLMT_M, + QP_CONTEXT_QPC_BYTES_44_HOPLMT_S); + grh->traffic_class = + roce_get_field(context->qpc_bytes_48, + QP_CONTEXT_QPC_BYTES_48_TCLASS_M, + QP_CONTEXT_QPC_BYTES_48_TCLASS_S); + + memcpy(grh->dgid.raw, context->dgid, + sizeof(grh->dgid.raw)); + } + + qp_attr->pkey_index = roce_get_field(context->qpc_bytes_12, + QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M, + QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S); + qp_attr->port_num = hr_qp->port + 1; + qp_attr->sq_draining = 0; + qp_attr->max_rd_atomic = 1 << roce_get_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M, + QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S); + qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context->qpc_bytes_32, + QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M, + QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S); + qp_attr->min_rnr_timer = (u8)(roce_get_field(context->qpc_bytes_24, + QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M, + QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S)); + qp_attr->timeout = (u8)(roce_get_field(context->qpc_bytes_156, + QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M, + QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S)); + qp_attr->retry_cnt = roce_get_field(context->qpc_bytes_148, + QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M, + QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S); + qp_attr->rnr_retry = (u8)le32_to_cpu(context->rnr_retry); + +done: + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; + qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs; + + if (!ibqp->uobject) { + qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; + qp_attr->cap.max_send_sge = hr_qp->sq.max_gs; + } else { + qp_attr->cap.max_send_wr = 0; + qp_attr->cap.max_send_sge = 0; + } + + qp_init_attr->cap = qp_attr->cap; + +out: + mutex_unlock(&hr_qp->mutex); + kfree(context); + return ret; +} + +static int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + + return hr_qp->doorbell_qpn <= 1 ? + hns_roce_v1_q_sqp(ibqp, qp_attr, qp_attr_mask, qp_init_attr) : + hns_roce_v1_q_qp(ibqp, qp_attr, qp_attr_mask, qp_init_attr); +} + +int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct hns_roce_cq *send_cq, *recv_cq; + int ret; + + ret = hns_roce_v1_modify_qp(ibqp, NULL, 0, hr_qp->state, IB_QPS_RESET); + if (ret) + return ret; + + send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL; + recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL; + + hns_roce_lock_cqs(send_cq, recv_cq); + if (!udata) { + if (recv_cq) + __hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn, + (hr_qp->ibqp.srq ? + to_hr_srq(hr_qp->ibqp.srq) : + NULL)); + + if (send_cq && send_cq != recv_cq) + __hns_roce_v1_cq_clean(send_cq, hr_qp->qpn, NULL); + } + hns_roce_qp_remove(hr_dev, hr_qp); + hns_roce_unlock_cqs(send_cq, recv_cq); + + hns_roce_qp_destroy(hr_dev, hr_qp, udata); + + return 0; +} + +static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); + struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); + struct device *dev = &hr_dev->pdev->dev; + u32 cqe_cnt_ori; + u32 cqe_cnt_cur; + int wait_time = 0; + + /* + * Before freeing cq buffer, we need to ensure that the outstanding CQE + * have been written by checking the CQE counter. + */ + cqe_cnt_ori = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT); + while (1) { + if (roce_read(hr_dev, ROCEE_CAEP_CQE_WCMD_EMPTY) & + HNS_ROCE_CQE_WCMD_EMPTY_BIT) + break; + + cqe_cnt_cur = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT); + if ((cqe_cnt_cur - cqe_cnt_ori) >= HNS_ROCE_MIN_CQE_CNT) + break; + + msleep(HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS); + if (wait_time > HNS_ROCE_MAX_FREE_CQ_WAIT_CNT) { + dev_warn(dev, "Destroy cq 0x%lx timeout!\n", + hr_cq->cqn); + break; + } + wait_time++; + } + return 0; +} + +static void set_eq_cons_index_v1(struct hns_roce_eq *eq, int req_not) +{ + roce_raw_write((eq->cons_index & HNS_ROCE_V1_CONS_IDX_M) | + (req_not << eq->log_entries), eq->doorbell); +} + +static void hns_roce_v1_wq_catas_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, int qpn) +{ + struct device *dev = &hr_dev->pdev->dev; + + dev_warn(dev, "Local Work Queue Catastrophic Error.\n"); + switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, + HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { + case HNS_ROCE_LWQCE_QPC_ERROR: + dev_warn(dev, "QP %d, QPC error.\n", qpn); + break; + case HNS_ROCE_LWQCE_MTU_ERROR: + dev_warn(dev, "QP %d, MTU error.\n", qpn); + break; + case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR: + dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn); + break; + case HNS_ROCE_LWQCE_WQE_ADDR_ERROR: + dev_warn(dev, "QP %d, WQE addr error.\n", qpn); + break; + case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR: + dev_warn(dev, "QP %d, WQE shift error\n", qpn); + break; + case HNS_ROCE_LWQCE_SL_ERROR: + dev_warn(dev, "QP %d, SL error.\n", qpn); + break; + case HNS_ROCE_LWQCE_PORT_ERROR: + dev_warn(dev, "QP %d, port error.\n", qpn); + break; + default: + break; + } +} + +static void hns_roce_v1_local_wq_access_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, + int qpn) +{ + struct device *dev = &hr_dev->pdev->dev; + + dev_warn(dev, "Local Access Violation Work Queue Error.\n"); + switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, + HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { + case HNS_ROCE_LAVWQE_R_KEY_VIOLATION: + dev_warn(dev, "QP %d, R_key violation.\n", qpn); + break; + case HNS_ROCE_LAVWQE_LENGTH_ERROR: + dev_warn(dev, "QP %d, length error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_VA_ERROR: + dev_warn(dev, "QP %d, VA error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_PD_ERROR: + dev_err(dev, "QP %d, PD error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_RW_ACC_ERROR: + dev_warn(dev, "QP %d, rw acc error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_KEY_STATE_ERROR: + dev_warn(dev, "QP %d, key state error.\n", qpn); + break; + case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR: + dev_warn(dev, "QP %d, MR operation error.\n", qpn); + break; + default: + break; + } +} + +static void hns_roce_v1_qp_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, + int event_type) +{ + struct device *dev = &hr_dev->pdev->dev; + int phy_port; + int qpn; + + qpn = roce_get_field(aeqe->event.qp_event.qp, + HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M, + HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S); + phy_port = roce_get_field(aeqe->event.qp_event.qp, + HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M, + HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S); + if (qpn <= 1) + qpn = HNS_ROCE_MAX_PORTS * qpn + phy_port; + + switch (event_type) { + case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + dev_warn(dev, "Invalid Req Local Work Queue Error.\n" + "QP %d, phy_port %d.\n", qpn, phy_port); + break; + case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + hns_roce_v1_wq_catas_err_handle(hr_dev, aeqe, qpn); + break; + case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + hns_roce_v1_local_wq_access_err_handle(hr_dev, aeqe, qpn); + break; + default: + break; + } + + hns_roce_qp_event(hr_dev, qpn, event_type); +} + +static void hns_roce_v1_cq_err_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe, + int event_type) +{ + struct device *dev = &hr_dev->pdev->dev; + u32 cqn; + + cqn = roce_get_field(aeqe->event.cq_event.cq, + HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M, + HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S); + + switch (event_type) { + case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: + dev_warn(dev, "CQ 0x%x access err.\n", cqn); + break; + case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: + dev_warn(dev, "CQ 0x%x overflow\n", cqn); + break; + case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID: + dev_warn(dev, "CQ 0x%x ID invalid.\n", cqn); + break; + default: + break; + } + + hns_roce_cq_event(hr_dev, cqn, event_type); +} + +static void hns_roce_v1_db_overflow_handle(struct hns_roce_dev *hr_dev, + struct hns_roce_aeqe *aeqe) +{ + struct device *dev = &hr_dev->pdev->dev; + + switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M, + HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) { + case HNS_ROCE_DB_SUBTYPE_SDB_OVF: + dev_warn(dev, "SDB overflow.\n"); + break; + case HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF: + dev_warn(dev, "SDB almost overflow.\n"); + break; + case HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP: + dev_warn(dev, "SDB almost empty.\n"); + break; + case HNS_ROCE_DB_SUBTYPE_ODB_OVF: + dev_warn(dev, "ODB overflow.\n"); + break; + case HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF: + dev_warn(dev, "ODB almost overflow.\n"); + break; + case HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP: + dev_warn(dev, "SDB almost empty.\n"); + break; + default: + break; + } +} + +static struct hns_roce_aeqe *get_aeqe_v1(struct hns_roce_eq *eq, u32 entry) +{ + unsigned long off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQE_SIZE; + + return (struct hns_roce_aeqe *)((u8 *) + (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) + + off % HNS_ROCE_BA_SIZE); +} + +static struct hns_roce_aeqe *next_aeqe_sw_v1(struct hns_roce_eq *eq) +{ + struct hns_roce_aeqe *aeqe = get_aeqe_v1(eq, eq->cons_index); + + return (roce_get_bit(aeqe->asyn, HNS_ROCE_AEQE_U32_4_OWNER_S) ^ + !!(eq->cons_index & eq->entries)) ? aeqe : NULL; +} + +static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + struct device *dev = &hr_dev->pdev->dev; + struct hns_roce_aeqe *aeqe; + int aeqes_found = 0; + int event_type; + + while ((aeqe = next_aeqe_sw_v1(eq))) { + /* Make sure we read the AEQ entry after we have checked the + * ownership bit + */ + dma_rmb(); + + dev_dbg(dev, "aeqe = %pK, aeqe->asyn.event_type = 0x%lx\n", + aeqe, + roce_get_field(aeqe->asyn, + HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M, + HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S)); + event_type = roce_get_field(aeqe->asyn, + HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M, + HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S); + switch (event_type) { + case HNS_ROCE_EVENT_TYPE_PATH_MIG: + dev_warn(dev, "PATH MIG not supported\n"); + break; + case HNS_ROCE_EVENT_TYPE_COMM_EST: + dev_warn(dev, "COMMUNICATION established\n"); + break; + case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: + dev_warn(dev, "SQ DRAINED not supported\n"); + break; + case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: + dev_warn(dev, "PATH MIG failed\n"); + break; + case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + hns_roce_v1_qp_err_handle(hr_dev, aeqe, event_type); + break; + case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: + case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: + dev_warn(dev, "SRQ not support!\n"); + break; + case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: + case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: + case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID: + hns_roce_v1_cq_err_handle(hr_dev, aeqe, event_type); + break; + case HNS_ROCE_EVENT_TYPE_PORT_CHANGE: + dev_warn(dev, "port change.\n"); + break; + case HNS_ROCE_EVENT_TYPE_MB: + hns_roce_cmd_event(hr_dev, + le16_to_cpu(aeqe->event.cmd.token), + aeqe->event.cmd.status, + le64_to_cpu(aeqe->event.cmd.out_param + )); + break; + case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: + hns_roce_v1_db_overflow_handle(hr_dev, aeqe); + break; + case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW: + dev_warn(dev, "CEQ 0x%lx overflow.\n", + roce_get_field(aeqe->event.ce_event.ceqe, + HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M, + HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S)); + break; + default: + dev_warn(dev, "Unhandled event %d on EQ %d at idx %u.\n", + event_type, eq->eqn, eq->cons_index); + break; + } + + eq->cons_index++; + aeqes_found = 1; + + if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1) + eq->cons_index = 0; + } + + set_eq_cons_index_v1(eq, 0); + + return aeqes_found; +} + +static struct hns_roce_ceqe *get_ceqe_v1(struct hns_roce_eq *eq, u32 entry) +{ + unsigned long off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQE_SIZE; + + return (struct hns_roce_ceqe *)((u8 *) + (eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) + + off % HNS_ROCE_BA_SIZE); +} + +static struct hns_roce_ceqe *next_ceqe_sw_v1(struct hns_roce_eq *eq) +{ + struct hns_roce_ceqe *ceqe = get_ceqe_v1(eq, eq->cons_index); + + return (!!(roce_get_bit(ceqe->comp, + HNS_ROCE_CEQE_CEQE_COMP_OWNER_S))) ^ + (!!(eq->cons_index & eq->entries)) ? ceqe : NULL; +} + +static int hns_roce_v1_ceq_int(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + struct hns_roce_ceqe *ceqe; + int ceqes_found = 0; + u32 cqn; + + while ((ceqe = next_ceqe_sw_v1(eq))) { + /* Make sure we read CEQ entry after we have checked the + * ownership bit + */ + dma_rmb(); + + cqn = roce_get_field(ceqe->comp, + HNS_ROCE_CEQE_CEQE_COMP_CQN_M, + HNS_ROCE_CEQE_CEQE_COMP_CQN_S); + hns_roce_cq_completion(hr_dev, cqn); + + ++eq->cons_index; + ceqes_found = 1; + + if (eq->cons_index > + EQ_DEPTH_COEFF * hr_dev->caps.ceqe_depth - 1) + eq->cons_index = 0; + } + + set_eq_cons_index_v1(eq, 0); + + return ceqes_found; +} + +static irqreturn_t hns_roce_v1_msix_interrupt_eq(int irq, void *eq_ptr) +{ + struct hns_roce_eq *eq = eq_ptr; + struct hns_roce_dev *hr_dev = eq->hr_dev; + int int_work; + + if (eq->type_flag == HNS_ROCE_CEQ) + /* CEQ irq routine, CEQ is pulse irq, not clear */ + int_work = hns_roce_v1_ceq_int(hr_dev, eq); + else + /* AEQ irq routine, AEQ is pulse irq, not clear */ + int_work = hns_roce_v1_aeq_int(hr_dev, eq); + + return IRQ_RETVAL(int_work); +} + +static irqreturn_t hns_roce_v1_msix_interrupt_abn(int irq, void *dev_id) +{ + struct hns_roce_dev *hr_dev = dev_id; + struct device *dev = &hr_dev->pdev->dev; + int int_work = 0; + u32 caepaemask_val; + u32 cealmovf_val; + u32 caepaest_val; + u32 aeshift_val; + u32 ceshift_val; + u32 cemask_val; + __le32 tmp; + int i; + + /* + * Abnormal interrupt: + * AEQ overflow, ECC multi-bit err, CEQ overflow must clear + * interrupt, mask irq, clear irq, cancel mask operation + */ + aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG); + tmp = cpu_to_le32(aeshift_val); + + /* AEQE overflow */ + if (roce_get_bit(tmp, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S) == 1) { + dev_warn(dev, "AEQ overflow!\n"); + + /* Set mask */ + caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); + tmp = cpu_to_le32(caepaemask_val); + roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, + HNS_ROCE_INT_MASK_ENABLE); + caepaemask_val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val); + + /* Clear int state(INT_WC : write 1 clear) */ + caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG); + tmp = cpu_to_le32(caepaest_val); + roce_set_bit(tmp, ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S, 1); + caepaest_val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val); + + /* Clear mask */ + caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); + tmp = cpu_to_le32(caepaemask_val); + roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, + HNS_ROCE_INT_MASK_DISABLE); + caepaemask_val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val); + } + + /* CEQ almost overflow */ + for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) { + ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG + + i * CEQ_REG_OFFSET); + tmp = cpu_to_le32(ceshift_val); + + if (roce_get_bit(tmp, + ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S) == 1) { + dev_warn(dev, "CEQ[%d] almost overflow!\n", i); + int_work++; + + /* Set mask */ + cemask_val = roce_read(hr_dev, + ROCEE_CAEP_CE_IRQ_MASK_0_REG + + i * CEQ_REG_OFFSET); + tmp = cpu_to_le32(cemask_val); + roce_set_bit(tmp, + ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S, + HNS_ROCE_INT_MASK_ENABLE); + cemask_val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + + i * CEQ_REG_OFFSET, cemask_val); + + /* Clear int state(INT_WC : write 1 clear) */ + cealmovf_val = roce_read(hr_dev, + ROCEE_CAEP_CEQ_ALM_OVF_0_REG + + i * CEQ_REG_OFFSET); + tmp = cpu_to_le32(cealmovf_val); + roce_set_bit(tmp, + ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S, + 1); + cealmovf_val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG + + i * CEQ_REG_OFFSET, cealmovf_val); + + /* Clear mask */ + cemask_val = roce_read(hr_dev, + ROCEE_CAEP_CE_IRQ_MASK_0_REG + + i * CEQ_REG_OFFSET); + tmp = cpu_to_le32(cemask_val); + roce_set_bit(tmp, + ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S, + HNS_ROCE_INT_MASK_DISABLE); + cemask_val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + + i * CEQ_REG_OFFSET, cemask_val); + } + } + + /* ECC multi-bit error alarm */ + dev_warn(dev, "ECC UCERR ALARM: 0x%x, 0x%x, 0x%x\n", + roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG), + roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG), + roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG)); + + dev_warn(dev, "ECC CERR ALARM: 0x%x, 0x%x, 0x%x\n", + roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG), + roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG), + roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG)); + + return IRQ_RETVAL(int_work); +} + +static void hns_roce_v1_int_mask_enable(struct hns_roce_dev *hr_dev) +{ + u32 aemask_val; + int masken = 0; + __le32 tmp; + int i; + + /* AEQ INT */ + aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG); + tmp = cpu_to_le32(aemask_val); + roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S, + masken); + roce_set_bit(tmp, ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S, masken); + aemask_val = le32_to_cpu(tmp); + roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val); + + /* CEQ INT */ + for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) { + /* IRQ mask */ + roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG + + i * CEQ_REG_OFFSET, masken); + } +} + +static void hns_roce_v1_free_eq(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + int npages = (PAGE_ALIGN(eq->eqe_size * eq->entries) + + HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE; + int i; + + if (!eq->buf_list) + return; + + for (i = 0; i < npages; ++i) + dma_free_coherent(&hr_dev->pdev->dev, HNS_ROCE_BA_SIZE, + eq->buf_list[i].buf, eq->buf_list[i].map); + + kfree(eq->buf_list); +} + +static void hns_roce_v1_enable_eq(struct hns_roce_dev *hr_dev, int eq_num, + int enable_flag) +{ + void __iomem *eqc = hr_dev->eq_table.eqc_base[eq_num]; + __le32 tmp; + u32 val; + + val = readl(eqc); + tmp = cpu_to_le32(val); + + if (enable_flag) + roce_set_field(tmp, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, + HNS_ROCE_EQ_STAT_VALID); + else + roce_set_field(tmp, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, + HNS_ROCE_EQ_STAT_INVALID); + + val = le32_to_cpu(tmp); + writel(val, eqc); +} + +static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn]; + struct device *dev = &hr_dev->pdev->dev; + dma_addr_t tmp_dma_addr; + u32 eqcuridx_val = 0; + u32 eqconsindx_val; + u32 eqshift_val; + __le32 tmp2 = 0; + __le32 tmp1 = 0; + __le32 tmp = 0; + int num_bas; + int ret; + int i; + + num_bas = (PAGE_ALIGN(eq->entries * eq->eqe_size) + + HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE; + + if ((eq->entries * eq->eqe_size) > HNS_ROCE_BA_SIZE) { + dev_err(dev, "[error]eq buf %d gt ba size(%d) need bas=%d\n", + (eq->entries * eq->eqe_size), HNS_ROCE_BA_SIZE, + num_bas); + return -EINVAL; + } + + eq->buf_list = kcalloc(num_bas, sizeof(*eq->buf_list), GFP_KERNEL); + if (!eq->buf_list) + return -ENOMEM; + + for (i = 0; i < num_bas; ++i) { + eq->buf_list[i].buf = dma_alloc_coherent(dev, HNS_ROCE_BA_SIZE, + &tmp_dma_addr, + GFP_KERNEL); + if (!eq->buf_list[i].buf) { + ret = -ENOMEM; + goto err_out_free_pages; + } + + eq->buf_list[i].map = tmp_dma_addr; + } + eq->cons_index = 0; + roce_set_field(tmp, ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S, + HNS_ROCE_EQ_STAT_INVALID); + roce_set_field(tmp, ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M, + ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S, + eq->log_entries); + eqshift_val = le32_to_cpu(tmp); + writel(eqshift_val, eqc); + + /* Configure eq extended address 12~44bit */ + writel((u32)(eq->buf_list[0].map >> 12), eqc + 4); + + /* + * Configure eq extended address 45~49 bit. + * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of + * using 4K page, and shift more 32 because of + * caculating the high 32 bit value evaluated to hardware. + */ + roce_set_field(tmp1, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M, + ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S, + eq->buf_list[0].map >> 44); + roce_set_field(tmp1, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M, + ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0); + eqcuridx_val = le32_to_cpu(tmp1); + writel(eqcuridx_val, eqc + 8); + + /* Configure eq consumer index */ + roce_set_field(tmp2, ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M, + ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0); + eqconsindx_val = le32_to_cpu(tmp2); + writel(eqconsindx_val, eqc + 0xc); + + return 0; + +err_out_free_pages: + for (i -= 1; i >= 0; i--) + dma_free_coherent(dev, HNS_ROCE_BA_SIZE, eq->buf_list[i].buf, + eq->buf_list[i].map); + + kfree(eq->buf_list); + return ret; +} + +static int hns_roce_v1_init_eq_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; + struct device *dev = &hr_dev->pdev->dev; + struct hns_roce_eq *eq; + int irq_num; + int eq_num; + int ret; + int i, j; + + eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; + irq_num = eq_num + hr_dev->caps.num_other_vectors; + + eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); + if (!eq_table->eq) + return -ENOMEM; + + eq_table->eqc_base = kcalloc(eq_num, sizeof(*eq_table->eqc_base), + GFP_KERNEL); + if (!eq_table->eqc_base) { + ret = -ENOMEM; + goto err_eqc_base_alloc_fail; + } + + for (i = 0; i < eq_num; i++) { + eq = &eq_table->eq[i]; + eq->hr_dev = hr_dev; + eq->eqn = i; + eq->irq = hr_dev->irq[i]; + eq->log_page_size = PAGE_SHIFT; + + if (i < hr_dev->caps.num_comp_vectors) { + /* CEQ */ + eq_table->eqc_base[i] = hr_dev->reg_base + + ROCEE_CAEP_CEQC_SHIFT_0_REG + + CEQ_REG_OFFSET * i; + eq->type_flag = HNS_ROCE_CEQ; + eq->doorbell = hr_dev->reg_base + + ROCEE_CAEP_CEQC_CONS_IDX_0_REG + + CEQ_REG_OFFSET * i; + eq->entries = hr_dev->caps.ceqe_depth; + eq->log_entries = ilog2(eq->entries); + eq->eqe_size = HNS_ROCE_CEQE_SIZE; + } else { + /* AEQ */ + eq_table->eqc_base[i] = hr_dev->reg_base + + ROCEE_CAEP_AEQC_AEQE_SHIFT_REG; + eq->type_flag = HNS_ROCE_AEQ; + eq->doorbell = hr_dev->reg_base + + ROCEE_CAEP_AEQE_CONS_IDX_REG; + eq->entries = hr_dev->caps.aeqe_depth; + eq->log_entries = ilog2(eq->entries); + eq->eqe_size = HNS_ROCE_AEQE_SIZE; + } + } + + /* Disable irq */ + hns_roce_v1_int_mask_enable(hr_dev); + + /* Configure ce int interval */ + roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG, + HNS_ROCE_CEQ_DEFAULT_INTERVAL); + + /* Configure ce int burst num */ + roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG, + HNS_ROCE_CEQ_DEFAULT_BURST_NUM); + + for (i = 0; i < eq_num; i++) { + ret = hns_roce_v1_create_eq(hr_dev, &eq_table->eq[i]); + if (ret) { + dev_err(dev, "eq create failed\n"); + goto err_create_eq_fail; + } + } + + for (j = 0; j < irq_num; j++) { + if (j < eq_num) + ret = request_irq(hr_dev->irq[j], + hns_roce_v1_msix_interrupt_eq, 0, + hr_dev->irq_names[j], + &eq_table->eq[j]); + else + ret = request_irq(hr_dev->irq[j], + hns_roce_v1_msix_interrupt_abn, 0, + hr_dev->irq_names[j], hr_dev); + + if (ret) { + dev_err(dev, "request irq error!\n"); + goto err_request_irq_fail; + } + } + + for (i = 0; i < eq_num; i++) + hns_roce_v1_enable_eq(hr_dev, i, EQ_ENABLE); + + return 0; + +err_request_irq_fail: + for (j -= 1; j >= 0; j--) + free_irq(hr_dev->irq[j], &eq_table->eq[j]); + +err_create_eq_fail: + for (i -= 1; i >= 0; i--) + hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]); + + kfree(eq_table->eqc_base); + +err_eqc_base_alloc_fail: + kfree(eq_table->eq); + + return ret; +} + +static void hns_roce_v1_cleanup_eq_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; + int irq_num; + int eq_num; + int i; + + eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; + irq_num = eq_num + hr_dev->caps.num_other_vectors; + for (i = 0; i < eq_num; i++) { + /* Disable EQ */ + hns_roce_v1_enable_eq(hr_dev, i, EQ_DISABLE); + + free_irq(hr_dev->irq[i], &eq_table->eq[i]); + + hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]); + } + for (i = eq_num; i < irq_num; i++) + free_irq(hr_dev->irq[i], hr_dev); + + kfree(eq_table->eqc_base); + kfree(eq_table->eq); +} + +static const struct ib_device_ops hns_roce_v1_dev_ops = { + .destroy_qp = hns_roce_v1_destroy_qp, + .poll_cq = hns_roce_v1_poll_cq, + .post_recv = hns_roce_v1_post_recv, + .post_send = hns_roce_v1_post_send, + .query_qp = hns_roce_v1_query_qp, + .req_notify_cq = hns_roce_v1_req_notify_cq, +}; + +static const struct hns_roce_hw hns_roce_hw_v1 = { + .reset = hns_roce_v1_reset, + .hw_profile = hns_roce_v1_profile, + .hw_init = hns_roce_v1_init, + .hw_exit = hns_roce_v1_exit, + .post_mbox = hns_roce_v1_post_mbox, + .chk_mbox = hns_roce_v1_chk_mbox, + .set_gid = hns_roce_v1_set_gid, + .set_mac = hns_roce_v1_set_mac, + .set_mtu = hns_roce_v1_set_mtu, + .write_mtpt = hns_roce_v1_write_mtpt, + .write_cqc = hns_roce_v1_write_cqc, + .set_hem = hns_roce_v1_set_hem, + .clear_hem = hns_roce_v1_clear_hem, + .modify_qp = hns_roce_v1_modify_qp, + .query_qp = hns_roce_v1_query_qp, + .destroy_qp = hns_roce_v1_destroy_qp, + .post_send = hns_roce_v1_post_send, + .post_recv = hns_roce_v1_post_recv, + .req_notify_cq = hns_roce_v1_req_notify_cq, + .poll_cq = hns_roce_v1_poll_cq, + .dereg_mr = hns_roce_v1_dereg_mr, + .destroy_cq = hns_roce_v1_destroy_cq, + .init_eq = hns_roce_v1_init_eq_table, + .cleanup_eq = hns_roce_v1_cleanup_eq_table, + .hns_roce_dev_ops = &hns_roce_v1_dev_ops, +}; + +static const struct of_device_id hns_roce_of_match[] = { + { .compatible = "hisilicon,hns-roce-v1", .data = &hns_roce_hw_v1, }, + {}, +}; +MODULE_DEVICE_TABLE(of, hns_roce_of_match); + +static const struct acpi_device_id hns_roce_acpi_match[] = { + { "HISI00D1", (kernel_ulong_t)&hns_roce_hw_v1 }, + {}, +}; +MODULE_DEVICE_TABLE(acpi, hns_roce_acpi_match); + +static struct +platform_device *hns_roce_find_pdev(struct fwnode_handle *fwnode) +{ + struct device *dev; + + /* get the 'device' corresponding to the matching 'fwnode' */ + dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode); + /* get the platform device */ + return dev ? to_platform_device(dev) : NULL; +} + +static int hns_roce_get_cfg(struct hns_roce_dev *hr_dev) +{ + struct device *dev = &hr_dev->pdev->dev; + struct platform_device *pdev = NULL; + struct net_device *netdev = NULL; + struct device_node *net_node; + int port_cnt = 0; + u8 phy_port; + int ret; + int i; + + /* check if we are compatible with the underlying SoC */ + if (dev_of_node(dev)) { + const struct of_device_id *of_id; + + of_id = of_match_node(hns_roce_of_match, dev->of_node); + if (!of_id) { + dev_err(dev, "device is not compatible!\n"); + return -ENXIO; + } + hr_dev->hw = (const struct hns_roce_hw *)of_id->data; + if (!hr_dev->hw) { + dev_err(dev, "couldn't get H/W specific DT data!\n"); + return -ENXIO; + } + } else if (is_acpi_device_node(dev->fwnode)) { + const struct acpi_device_id *acpi_id; + + acpi_id = acpi_match_device(hns_roce_acpi_match, dev); + if (!acpi_id) { + dev_err(dev, "device is not compatible!\n"); + return -ENXIO; + } + hr_dev->hw = (const struct hns_roce_hw *) acpi_id->driver_data; + if (!hr_dev->hw) { + dev_err(dev, "couldn't get H/W specific ACPI data!\n"); + return -ENXIO; + } + } else { + dev_err(dev, "can't read compatibility data from DT or ACPI\n"); + return -ENXIO; + } + + /* get the mapped register base address */ + hr_dev->reg_base = devm_platform_ioremap_resource(hr_dev->pdev, 0); + if (IS_ERR(hr_dev->reg_base)) + return PTR_ERR(hr_dev->reg_base); + + /* read the node_guid of IB device from the DT or ACPI */ + ret = device_property_read_u8_array(dev, "node-guid", + (u8 *)&hr_dev->ib_dev.node_guid, + GUID_LEN); + if (ret) { + dev_err(dev, "couldn't get node_guid from DT or ACPI!\n"); + return ret; + } + + /* get the RoCE associated ethernet ports or netdevices */ + for (i = 0; i < HNS_ROCE_MAX_PORTS; i++) { + if (dev_of_node(dev)) { + net_node = of_parse_phandle(dev->of_node, "eth-handle", + i); + if (!net_node) + continue; + pdev = of_find_device_by_node(net_node); + } else if (is_acpi_device_node(dev->fwnode)) { + struct fwnode_reference_args args; + + ret = acpi_node_get_property_reference(dev->fwnode, + "eth-handle", + i, &args); + if (ret) + continue; + pdev = hns_roce_find_pdev(args.fwnode); + } else { + dev_err(dev, "cannot read data from DT or ACPI\n"); + return -ENXIO; + } + + if (pdev) { + netdev = platform_get_drvdata(pdev); + phy_port = (u8)i; + if (netdev) { + hr_dev->iboe.netdevs[port_cnt] = netdev; + hr_dev->iboe.phy_port[port_cnt] = phy_port; + } else { + dev_err(dev, "no netdev found with pdev %s\n", + pdev->name); + return -ENODEV; + } + port_cnt++; + } + } + + if (port_cnt == 0) { + dev_err(dev, "unable to get eth-handle for available ports!\n"); + return -EINVAL; + } + + hr_dev->caps.num_ports = port_cnt; + + /* cmd issue mode: 0 is poll, 1 is event */ + hr_dev->cmd_mod = 1; + hr_dev->loop_idc = 0; + hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; + hr_dev->odb_offset = ROCEE_DB_OTHERS_L_0_REG; + + /* read the interrupt names from the DT or ACPI */ + ret = device_property_read_string_array(dev, "interrupt-names", + hr_dev->irq_names, + HNS_ROCE_V1_MAX_IRQ_NUM); + if (ret < 0) { + dev_err(dev, "couldn't get interrupt names from DT or ACPI!\n"); + return ret; + } + + /* fetch the interrupt numbers */ + for (i = 0; i < HNS_ROCE_V1_MAX_IRQ_NUM; i++) { + hr_dev->irq[i] = platform_get_irq(hr_dev->pdev, i); + if (hr_dev->irq[i] <= 0) + return -EINVAL; + } + + return 0; +} + +/** + * hns_roce_probe - RoCE driver entrance + * @pdev: pointer to platform device + * Return : int + * + */ +static int hns_roce_probe(struct platform_device *pdev) +{ + int ret; + struct hns_roce_dev *hr_dev; + struct device *dev = &pdev->dev; + + hr_dev = ib_alloc_device(hns_roce_dev, ib_dev); + if (!hr_dev) + return -ENOMEM; + + hr_dev->priv = kzalloc(sizeof(struct hns_roce_v1_priv), GFP_KERNEL); + if (!hr_dev->priv) { + ret = -ENOMEM; + goto error_failed_kzalloc; + } + + hr_dev->pdev = pdev; + hr_dev->dev = dev; + platform_set_drvdata(pdev, hr_dev); + + if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64ULL)) && + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32ULL))) { + dev_err(dev, "Not usable DMA addressing mode\n"); + ret = -EIO; + goto error_failed_get_cfg; + } + + ret = hns_roce_get_cfg(hr_dev); + if (ret) { + dev_err(dev, "Get Configuration failed!\n"); + goto error_failed_get_cfg; + } + + ret = hns_roce_init(hr_dev); + if (ret) { + dev_err(dev, "RoCE engine init failed!\n"); + goto error_failed_get_cfg; + } + + return 0; + +error_failed_get_cfg: + kfree(hr_dev->priv); + +error_failed_kzalloc: + ib_dealloc_device(&hr_dev->ib_dev); + + return ret; +} + +/** + * hns_roce_remove - remove RoCE device + * @pdev: pointer to platform device + */ +static int hns_roce_remove(struct platform_device *pdev) +{ + struct hns_roce_dev *hr_dev = platform_get_drvdata(pdev); + + hns_roce_exit(hr_dev); + kfree(hr_dev->priv); + ib_dealloc_device(&hr_dev->ib_dev); + + return 0; +} + +static struct platform_driver hns_roce_driver = { + .probe = hns_roce_probe, + .remove = hns_roce_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = hns_roce_of_match, + .acpi_match_table = ACPI_PTR(hns_roce_acpi_match), + }, +}; + +module_platform_driver(hns_roce_driver); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); +MODULE_AUTHOR("Nenglong Zhao <zhaonenglong@hisilicon.com>"); +MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>"); +MODULE_DESCRIPTION("Hisilicon Hip06 Family RoCE Driver"); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h new file mode 100644 index 000000000..9ff1a41dd --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h @@ -0,0 +1,1104 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _HNS_ROCE_HW_V1_H +#define _HNS_ROCE_HW_V1_H + +#define CQ_STATE_VALID 2 + +#define HNS_ROCE_V1_MAX_PD_NUM 0x8000 +#define HNS_ROCE_V1_MAX_CQ_NUM 0x10000 +#define HNS_ROCE_V1_MAX_CQE_NUM 0x8000 + +#define HNS_ROCE_V1_MAX_QP_NUM 0x40000 +#define HNS_ROCE_V1_MAX_WQE_NUM 0x4000 + +#define HNS_ROCE_V1_MAX_MTPT_NUM 0x80000 + +#define HNS_ROCE_V1_MAX_MTT_SEGS 0x100000 + +#define HNS_ROCE_V1_MAX_QP_INIT_RDMA 128 +#define HNS_ROCE_V1_MAX_QP_DEST_RDMA 128 + +#define HNS_ROCE_V1_MAX_SQ_DESC_SZ 64 +#define HNS_ROCE_V1_MAX_RQ_DESC_SZ 64 +#define HNS_ROCE_V1_SG_NUM 2 +#define HNS_ROCE_V1_INLINE_SIZE 32 + +#define HNS_ROCE_V1_UAR_NUM 256 +#define HNS_ROCE_V1_PHY_UAR_NUM 8 + +#define HNS_ROCE_V1_GID_NUM 16 +#define HNS_ROCE_V1_RESV_QP 8 + +#define HNS_ROCE_V1_MAX_IRQ_NUM 34 +#define HNS_ROCE_V1_COMP_VEC_NUM 32 +#define HNS_ROCE_V1_AEQE_VEC_NUM 1 +#define HNS_ROCE_V1_ABNORMAL_VEC_NUM 1 + +#define HNS_ROCE_V1_COMP_EQE_NUM 0x8000 +#define HNS_ROCE_V1_ASYNC_EQE_NUM 0x400 + +#define HNS_ROCE_V1_QPC_SIZE 256 +#define HNS_ROCE_V1_IRRL_ENTRY_SIZE 8 +#define HNS_ROCE_V1_CQC_ENTRY_SIZE 64 +#define HNS_ROCE_V1_MTPT_ENTRY_SIZE 64 +#define HNS_ROCE_V1_MTT_ENTRY_SIZE 64 + +#define HNS_ROCE_V1_CQE_SIZE 32 +#define HNS_ROCE_V1_PAGE_SIZE_SUPPORT 0xFFFFF000 + +#define HNS_ROCE_V1_TABLE_CHUNK_SIZE (1 << 17) + +#define HNS_ROCE_V1_EXT_RAQ_WF 8 +#define HNS_ROCE_V1_RAQ_ENTRY 64 +#define HNS_ROCE_V1_RAQ_DEPTH 32768 +#define HNS_ROCE_V1_RAQ_SIZE (HNS_ROCE_V1_RAQ_ENTRY * HNS_ROCE_V1_RAQ_DEPTH) + +#define HNS_ROCE_V1_SDB_DEPTH 0x400 +#define HNS_ROCE_V1_ODB_DEPTH 0x400 + +#define HNS_ROCE_V1_DB_RSVD 0x80 + +#define HNS_ROCE_V1_SDB_ALEPT HNS_ROCE_V1_DB_RSVD +#define HNS_ROCE_V1_SDB_ALFUL (HNS_ROCE_V1_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD) +#define HNS_ROCE_V1_ODB_ALEPT HNS_ROCE_V1_DB_RSVD +#define HNS_ROCE_V1_ODB_ALFUL (HNS_ROCE_V1_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD) + +#define HNS_ROCE_V1_EXT_SDB_DEPTH 0x4000 +#define HNS_ROCE_V1_EXT_ODB_DEPTH 0x4000 +#define HNS_ROCE_V1_EXT_SDB_ENTRY 16 +#define HNS_ROCE_V1_EXT_ODB_ENTRY 16 +#define HNS_ROCE_V1_EXT_SDB_SIZE \ + (HNS_ROCE_V1_EXT_SDB_DEPTH * HNS_ROCE_V1_EXT_SDB_ENTRY) +#define HNS_ROCE_V1_EXT_ODB_SIZE \ + (HNS_ROCE_V1_EXT_ODB_DEPTH * HNS_ROCE_V1_EXT_ODB_ENTRY) + +#define HNS_ROCE_V1_EXT_SDB_ALEPT HNS_ROCE_V1_DB_RSVD +#define HNS_ROCE_V1_EXT_SDB_ALFUL \ + (HNS_ROCE_V1_EXT_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD) +#define HNS_ROCE_V1_EXT_ODB_ALEPT HNS_ROCE_V1_DB_RSVD +#define HNS_ROCE_V1_EXT_ODB_ALFUL \ + (HNS_ROCE_V1_EXT_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD) + +#define HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS 50000 +#define HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS 10000 +#define HNS_ROCE_V1_FREE_MR_WAIT_VALUE 5 +#define HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE 20 + +#define HNS_ROCE_BT_RSV_BUF_SIZE (1 << 17) + +#define HNS_ROCE_V1_TPTR_ENTRY_SIZE 2 +#define HNS_ROCE_V1_TPTR_BUF_SIZE \ + (HNS_ROCE_V1_TPTR_ENTRY_SIZE * HNS_ROCE_V1_MAX_CQ_NUM) + +#define HNS_ROCE_ODB_POLL_MODE 0 + +#define HNS_ROCE_SDB_NORMAL_MODE 0 +#define HNS_ROCE_SDB_EXTEND_MODE 1 + +#define HNS_ROCE_ODB_EXTEND_MODE 1 + +#define KEY_VALID 0x02 + +#define HNS_ROCE_CQE_QPN_MASK 0x3ffff +#define HNS_ROCE_CQE_STATUS_MASK 0x1f +#define HNS_ROCE_CQE_OPCODE_MASK 0xf + +#define HNS_ROCE_CQE_SUCCESS 0x00 +#define HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR 0x01 +#define HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR 0x02 +#define HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR 0x03 +#define HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR 0x04 +#define HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR 0x05 +#define HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR 0x06 +#define HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR 0x07 +#define HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR 0x08 +#define HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR 0x09 +#define HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR 0x0a +#define HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR 0x0b +#define HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR 0x0c + +#define QP1C_CFGN_OFFSET 0x28 +#define PHY_PORT_OFFSET 0x8 +#define MTPT_IDX_SHIFT 16 +#define ALL_PORT_VAL_OPEN 0x3f +#define POL_TIME_INTERVAL_VAL 0x80 +#define SLEEP_TIME_INTERVAL 20 +#define SQ_PSN_SHIFT 8 +#define QKEY_VAL 0x80010000 +#define SDB_INV_CNT_OFFSET 8 + +#define HNS_ROCE_CEQ_DEFAULT_INTERVAL 0x10 +#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM 0x10 + +#define HNS_ROCE_INT_MASK_DISABLE 0 +#define HNS_ROCE_INT_MASK_ENABLE 1 + +#define CEQ_REG_OFFSET 0x18 + +#define HNS_ROCE_CEQE_CEQE_COMP_OWNER_S 0 + +#define HNS_ROCE_V1_CONS_IDX_M GENMASK(15, 0) + +#define HNS_ROCE_CEQE_CEQE_COMP_CQN_S 16 +#define HNS_ROCE_CEQE_CEQE_COMP_CQN_M GENMASK(31, 16) + +#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S 16 +#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M GENMASK(23, 16) + +#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S 24 +#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M GENMASK(30, 24) + +#define HNS_ROCE_AEQE_U32_4_OWNER_S 31 + +#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S 0 +#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M GENMASK(23, 0) + +#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S 25 +#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M GENMASK(27, 25) + +#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S 0 +#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M GENMASK(15, 0) + +#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S 0 +#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M GENMASK(4, 0) + +struct hns_roce_cq_context { + __le32 cqc_byte_4; + __le32 cq_bt_l; + __le32 cqc_byte_12; + __le32 cur_cqe_ba0_l; + __le32 cqc_byte_20; + __le32 cqe_tptr_addr_l; + __le32 cur_cqe_ba1_l; + __le32 cqc_byte_32; +}; + +#define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S 0 +#define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M \ + (((1UL << 2) - 1) << CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S) + +#define CQ_CONTEXT_CQC_BYTE_4_CQN_S 16 +#define CQ_CONTEXT_CQC_BYTE_4_CQN_M \ + (((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_4_CQN_S) + +#define CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S 0 +#define CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M \ + (((1UL << 17) - 1) << CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S) + +#define CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S 20 +#define CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M \ + (((1UL << 4) - 1) << CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S) + +#define CQ_CONTEXT_CQC_BYTE_12_CEQN_S 24 +#define CQ_CONTEXT_CQC_BYTE_12_CEQN_M \ + (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_12_CEQN_S) + +#define CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S 0 +#define CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M \ + (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S) + +#define CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S 16 +#define CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M \ + (((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S) + +#define CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S 8 +#define CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M \ + (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S) + +#define CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S 0 +#define CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M \ + (((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S) + +#define CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S 9 + +#define CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S 8 +#define CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S 14 +#define CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S 15 + +#define CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S 16 +#define CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M \ + (((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S) + +struct hns_roce_cqe { + __le32 cqe_byte_4; + union { + __le32 r_key; + __le32 immediate_data; + }; + __le32 byte_cnt; + __le32 cqe_byte_16; + __le32 cqe_byte_20; + __le32 s_mac_l; + __le32 cqe_byte_28; + __le32 reserved; +}; + +#define CQE_BYTE_4_OWNER_S 7 +#define CQE_BYTE_4_SQ_RQ_FLAG_S 14 + +#define CQE_BYTE_4_STATUS_OF_THE_OPERATION_S 8 +#define CQE_BYTE_4_STATUS_OF_THE_OPERATION_M \ + (((1UL << 5) - 1) << CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) + +#define CQE_BYTE_4_WQE_INDEX_S 16 +#define CQE_BYTE_4_WQE_INDEX_M (((1UL << 14) - 1) << CQE_BYTE_4_WQE_INDEX_S) + +#define CQE_BYTE_4_OPERATION_TYPE_S 0 +#define CQE_BYTE_4_OPERATION_TYPE_M \ + (((1UL << 4) - 1) << CQE_BYTE_4_OPERATION_TYPE_S) + +#define CQE_BYTE_4_IMM_INDICATOR_S 15 + +#define CQE_BYTE_16_LOCAL_QPN_S 0 +#define CQE_BYTE_16_LOCAL_QPN_M (((1UL << 24) - 1) << CQE_BYTE_16_LOCAL_QPN_S) + +#define CQE_BYTE_20_PORT_NUM_S 26 +#define CQE_BYTE_20_PORT_NUM_M (((1UL << 3) - 1) << CQE_BYTE_20_PORT_NUM_S) + +#define CQE_BYTE_20_SL_S 24 +#define CQE_BYTE_20_SL_M (((1UL << 2) - 1) << CQE_BYTE_20_SL_S) + +#define CQE_BYTE_20_REMOTE_QPN_S 0 +#define CQE_BYTE_20_REMOTE_QPN_M \ + (((1UL << 24) - 1) << CQE_BYTE_20_REMOTE_QPN_S) + +#define CQE_BYTE_20_GRH_PRESENT_S 29 + +#define CQE_BYTE_28_P_KEY_IDX_S 16 +#define CQE_BYTE_28_P_KEY_IDX_M (((1UL << 16) - 1) << CQE_BYTE_28_P_KEY_IDX_S) + +#define CQ_DB_REQ_NOT_SOL 0 +#define CQ_DB_REQ_NOT (1 << 16) + +struct hns_roce_v1_mpt_entry { + __le32 mpt_byte_4; + __le32 pbl_addr_l; + __le32 mpt_byte_12; + __le32 virt_addr_l; + __le32 virt_addr_h; + __le32 length; + __le32 mpt_byte_28; + __le32 pa0_l; + __le32 mpt_byte_36; + __le32 mpt_byte_40; + __le32 mpt_byte_44; + __le32 mpt_byte_48; + __le32 pa4_l; + __le32 mpt_byte_56; + __le32 mpt_byte_60; + __le32 mpt_byte_64; +}; + +#define MPT_BYTE_4_KEY_STATE_S 0 +#define MPT_BYTE_4_KEY_STATE_M (((1UL << 2) - 1) << MPT_BYTE_4_KEY_STATE_S) + +#define MPT_BYTE_4_KEY_S 8 +#define MPT_BYTE_4_KEY_M (((1UL << 8) - 1) << MPT_BYTE_4_KEY_S) + +#define MPT_BYTE_4_PAGE_SIZE_S 16 +#define MPT_BYTE_4_PAGE_SIZE_M (((1UL << 2) - 1) << MPT_BYTE_4_PAGE_SIZE_S) + +#define MPT_BYTE_4_MW_TYPE_S 20 + +#define MPT_BYTE_4_MW_BIND_ENABLE_S 21 + +#define MPT_BYTE_4_OWN_S 22 + +#define MPT_BYTE_4_MEMORY_LOCATION_TYPE_S 24 +#define MPT_BYTE_4_MEMORY_LOCATION_TYPE_M \ + (((1UL << 2) - 1) << MPT_BYTE_4_MEMORY_LOCATION_TYPE_S) + +#define MPT_BYTE_4_REMOTE_ATOMIC_S 26 +#define MPT_BYTE_4_LOCAL_WRITE_S 27 +#define MPT_BYTE_4_REMOTE_WRITE_S 28 +#define MPT_BYTE_4_REMOTE_READ_S 29 +#define MPT_BYTE_4_REMOTE_INVAL_ENABLE_S 30 +#define MPT_BYTE_4_ADDRESS_TYPE_S 31 + +#define MPT_BYTE_12_PBL_ADDR_H_S 0 +#define MPT_BYTE_12_PBL_ADDR_H_M \ + (((1UL << 17) - 1) << MPT_BYTE_12_PBL_ADDR_H_S) + +#define MPT_BYTE_12_MW_BIND_COUNTER_S 17 +#define MPT_BYTE_12_MW_BIND_COUNTER_M \ + (((1UL << 15) - 1) << MPT_BYTE_12_MW_BIND_COUNTER_S) + +#define MPT_BYTE_28_PD_S 0 +#define MPT_BYTE_28_PD_M (((1UL << 16) - 1) << MPT_BYTE_28_PD_S) + +#define MPT_BYTE_28_L_KEY_IDX_L_S 16 +#define MPT_BYTE_28_L_KEY_IDX_L_M \ + (((1UL << 16) - 1) << MPT_BYTE_28_L_KEY_IDX_L_S) + +#define MPT_BYTE_36_PA0_H_S 0 +#define MPT_BYTE_36_PA0_H_M (((1UL << 5) - 1) << MPT_BYTE_36_PA0_H_S) + +#define MPT_BYTE_36_PA1_L_S 8 +#define MPT_BYTE_36_PA1_L_M (((1UL << 24) - 1) << MPT_BYTE_36_PA1_L_S) + +#define MPT_BYTE_40_PA1_H_S 0 +#define MPT_BYTE_40_PA1_H_M (((1UL << 13) - 1) << MPT_BYTE_40_PA1_H_S) + +#define MPT_BYTE_40_PA2_L_S 16 +#define MPT_BYTE_40_PA2_L_M (((1UL << 16) - 1) << MPT_BYTE_40_PA2_L_S) + +#define MPT_BYTE_44_PA2_H_S 0 +#define MPT_BYTE_44_PA2_H_M (((1UL << 21) - 1) << MPT_BYTE_44_PA2_H_S) + +#define MPT_BYTE_44_PA3_L_S 24 +#define MPT_BYTE_44_PA3_L_M (((1UL << 8) - 1) << MPT_BYTE_44_PA3_L_S) + +#define MPT_BYTE_48_PA3_H_S 0 +#define MPT_BYTE_48_PA3_H_M (((1UL << 29) - 1) << MPT_BYTE_48_PA3_H_S) + +#define MPT_BYTE_56_PA4_H_S 0 +#define MPT_BYTE_56_PA4_H_M (((1UL << 5) - 1) << MPT_BYTE_56_PA4_H_S) + +#define MPT_BYTE_56_PA5_L_S 8 +#define MPT_BYTE_56_PA5_L_M (((1UL << 24) - 1) << MPT_BYTE_56_PA5_L_S) + +#define MPT_BYTE_60_PA5_H_S 0 +#define MPT_BYTE_60_PA5_H_M (((1UL << 13) - 1) << MPT_BYTE_60_PA5_H_S) + +#define MPT_BYTE_60_PA6_L_S 16 +#define MPT_BYTE_60_PA6_L_M (((1UL << 16) - 1) << MPT_BYTE_60_PA6_L_S) + +#define MPT_BYTE_64_PA6_H_S 0 +#define MPT_BYTE_64_PA6_H_M (((1UL << 21) - 1) << MPT_BYTE_64_PA6_H_S) + +#define MPT_BYTE_64_L_KEY_IDX_H_S 24 +#define MPT_BYTE_64_L_KEY_IDX_H_M \ + (((1UL << 8) - 1) << MPT_BYTE_64_L_KEY_IDX_H_S) + +struct hns_roce_wqe_ctrl_seg { + __le32 sgl_pa_h; + __le32 flag; + union { + __be32 imm_data; + __le32 inv_key; + }; + __le32 msg_length; +}; + +struct hns_roce_wqe_data_seg { + __le64 addr; + __le32 lkey; + __le32 len; +}; + +struct hns_roce_wqe_raddr_seg { + __le32 rkey; + __le32 len; /* reserved */ + __le64 raddr; +}; + +struct hns_roce_rq_wqe_ctrl { + __le32 rwqe_byte_4; + __le32 rocee_sgl_ba_l; + __le32 rwqe_byte_12; + __le32 reserved[5]; +}; + +#define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S 16 +#define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M \ + (((1UL << 6) - 1) << RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S) + +#define HNS_ROCE_QP_DESTROY_TIMEOUT_MSECS 10000 + +#define GID_LEN 16 + +struct hns_roce_ud_send_wqe { + __le32 dmac_h; + __le32 u32_8; + __le32 immediate_data; + + __le32 u32_16; + union { + unsigned char dgid[GID_LEN]; + struct { + __le32 u32_20; + __le32 u32_24; + __le32 u32_28; + __le32 u32_32; + }; + }; + + __le32 u32_36; + __le32 u32_40; + + __le32 va0_l; + __le32 va0_h; + __le32 l_key0; + + __le32 va1_l; + __le32 va1_h; + __le32 l_key1; +}; + +#define UD_SEND_WQE_U32_4_DMAC_0_S 0 +#define UD_SEND_WQE_U32_4_DMAC_0_M \ + (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_0_S) + +#define UD_SEND_WQE_U32_4_DMAC_1_S 8 +#define UD_SEND_WQE_U32_4_DMAC_1_M \ + (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_1_S) + +#define UD_SEND_WQE_U32_4_DMAC_2_S 16 +#define UD_SEND_WQE_U32_4_DMAC_2_M \ + (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_2_S) + +#define UD_SEND_WQE_U32_4_DMAC_3_S 24 +#define UD_SEND_WQE_U32_4_DMAC_3_M \ + (((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_3_S) + +#define UD_SEND_WQE_U32_8_DMAC_4_S 0 +#define UD_SEND_WQE_U32_8_DMAC_4_M \ + (((1UL << 8) - 1) << UD_SEND_WQE_U32_8_DMAC_4_S) + +#define UD_SEND_WQE_U32_8_DMAC_5_S 8 +#define UD_SEND_WQE_U32_8_DMAC_5_M \ + (((1UL << 8) - 1) << UD_SEND_WQE_U32_8_DMAC_5_S) + +#define UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S 22 + +#define UD_SEND_WQE_U32_8_OPERATION_TYPE_S 16 +#define UD_SEND_WQE_U32_8_OPERATION_TYPE_M \ + (((1UL << 4) - 1) << UD_SEND_WQE_U32_8_OPERATION_TYPE_S) + +#define UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S 24 +#define UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M \ + (((1UL << 6) - 1) << UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S) + +#define UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S 31 + +#define UD_SEND_WQE_U32_16_DEST_QP_S 0 +#define UD_SEND_WQE_U32_16_DEST_QP_M \ + (((1UL << 24) - 1) << UD_SEND_WQE_U32_16_DEST_QP_S) + +#define UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S 24 +#define UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M \ + (((1UL << 8) - 1) << UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S) + +#define UD_SEND_WQE_U32_36_FLOW_LABEL_S 0 +#define UD_SEND_WQE_U32_36_FLOW_LABEL_M \ + (((1UL << 20) - 1) << UD_SEND_WQE_U32_36_FLOW_LABEL_S) + +#define UD_SEND_WQE_U32_36_PRIORITY_S 20 +#define UD_SEND_WQE_U32_36_PRIORITY_M \ + (((1UL << 4) - 1) << UD_SEND_WQE_U32_36_PRIORITY_S) + +#define UD_SEND_WQE_U32_36_SGID_INDEX_S 24 +#define UD_SEND_WQE_U32_36_SGID_INDEX_M \ + (((1UL << 8) - 1) << UD_SEND_WQE_U32_36_SGID_INDEX_S) + +#define UD_SEND_WQE_U32_40_HOP_LIMIT_S 0 +#define UD_SEND_WQE_U32_40_HOP_LIMIT_M \ + (((1UL << 8) - 1) << UD_SEND_WQE_U32_40_HOP_LIMIT_S) + +#define UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S 8 +#define UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M \ + (((1UL << 8) - 1) << UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S) + +struct hns_roce_sqp_context { + __le32 qp1c_bytes_4; + __le32 sq_rq_bt_l; + __le32 qp1c_bytes_12; + __le32 qp1c_bytes_16; + __le32 qp1c_bytes_20; + __le32 cur_rq_wqe_ba_l; + __le32 qp1c_bytes_28; + __le32 qp1c_bytes_32; + __le32 cur_sq_wqe_ba_l; + __le32 qp1c_bytes_40; +}; + +#define QP1C_BYTES_4_QP_STATE_S 0 +#define QP1C_BYTES_4_QP_STATE_M \ + (((1UL << 3) - 1) << QP1C_BYTES_4_QP_STATE_S) + +#define QP1C_BYTES_4_SQ_WQE_SHIFT_S 8 +#define QP1C_BYTES_4_SQ_WQE_SHIFT_M \ + (((1UL << 4) - 1) << QP1C_BYTES_4_SQ_WQE_SHIFT_S) + +#define QP1C_BYTES_4_RQ_WQE_SHIFT_S 12 +#define QP1C_BYTES_4_RQ_WQE_SHIFT_M \ + (((1UL << 4) - 1) << QP1C_BYTES_4_RQ_WQE_SHIFT_S) + +#define QP1C_BYTES_4_PD_S 16 +#define QP1C_BYTES_4_PD_M (((1UL << 16) - 1) << QP1C_BYTES_4_PD_S) + +#define QP1C_BYTES_12_SQ_RQ_BT_H_S 0 +#define QP1C_BYTES_12_SQ_RQ_BT_H_M \ + (((1UL << 17) - 1) << QP1C_BYTES_12_SQ_RQ_BT_H_S) + +#define QP1C_BYTES_16_RQ_HEAD_S 0 +#define QP1C_BYTES_16_RQ_HEAD_M (((1UL << 15) - 1) << QP1C_BYTES_16_RQ_HEAD_S) + +#define QP1C_BYTES_16_PORT_NUM_S 16 +#define QP1C_BYTES_16_PORT_NUM_M \ + (((1UL << 3) - 1) << QP1C_BYTES_16_PORT_NUM_S) + +#define QP1C_BYTES_16_SIGNALING_TYPE_S 27 +#define QP1C_BYTES_16_LOCAL_ENABLE_E2E_CREDIT_S 28 +#define QP1C_BYTES_16_RQ_BA_FLG_S 29 +#define QP1C_BYTES_16_SQ_BA_FLG_S 30 +#define QP1C_BYTES_16_QP1_ERR_S 31 + +#define QP1C_BYTES_20_SQ_HEAD_S 0 +#define QP1C_BYTES_20_SQ_HEAD_M (((1UL << 15) - 1) << QP1C_BYTES_20_SQ_HEAD_S) + +#define QP1C_BYTES_20_PKEY_IDX_S 16 +#define QP1C_BYTES_20_PKEY_IDX_M \ + (((1UL << 16) - 1) << QP1C_BYTES_20_PKEY_IDX_S) + +#define QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S 0 +#define QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M \ + (((1UL << 5) - 1) << QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S) + +#define QP1C_BYTES_28_RQ_CUR_IDX_S 16 +#define QP1C_BYTES_28_RQ_CUR_IDX_M \ + (((1UL << 15) - 1) << QP1C_BYTES_28_RQ_CUR_IDX_S) + +#define QP1C_BYTES_32_TX_CQ_NUM_S 0 +#define QP1C_BYTES_32_TX_CQ_NUM_M \ + (((1UL << 16) - 1) << QP1C_BYTES_32_TX_CQ_NUM_S) + +#define QP1C_BYTES_32_RX_CQ_NUM_S 16 +#define QP1C_BYTES_32_RX_CQ_NUM_M \ + (((1UL << 16) - 1) << QP1C_BYTES_32_RX_CQ_NUM_S) + +#define QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S 0 +#define QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M \ + (((1UL << 5) - 1) << QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S) + +#define QP1C_BYTES_40_SQ_CUR_IDX_S 16 +#define QP1C_BYTES_40_SQ_CUR_IDX_M \ + (((1UL << 15) - 1) << QP1C_BYTES_40_SQ_CUR_IDX_S) + +#define HNS_ROCE_WQE_INLINE (1UL<<31) +#define HNS_ROCE_WQE_SE (1UL<<30) + +#define HNS_ROCE_WQE_SGE_NUM_BIT 24 +#define HNS_ROCE_WQE_IMM (1UL<<23) +#define HNS_ROCE_WQE_FENCE (1UL<<21) +#define HNS_ROCE_WQE_CQ_NOTIFY (1UL<<20) + +#define HNS_ROCE_WQE_OPCODE_SEND (0<<16) +#define HNS_ROCE_WQE_OPCODE_RDMA_READ (1<<16) +#define HNS_ROCE_WQE_OPCODE_RDMA_WRITE (2<<16) +#define HNS_ROCE_WQE_OPCODE_LOCAL_INV (4<<16) +#define HNS_ROCE_WQE_OPCODE_UD_SEND (7<<16) +#define HNS_ROCE_WQE_OPCODE_MASK (15<<16) + +struct hns_roce_qp_context { + __le32 qpc_bytes_4; + __le32 qpc_bytes_8; + __le32 qpc_bytes_12; + __le32 qpc_bytes_16; + __le32 sq_rq_bt_l; + __le32 qpc_bytes_24; + __le32 irrl_ba_l; + __le32 qpc_bytes_32; + __le32 qpc_bytes_36; + __le32 dmac_l; + __le32 qpc_bytes_44; + __le32 qpc_bytes_48; + u8 dgid[16]; + __le32 qpc_bytes_68; + __le32 cur_rq_wqe_ba_l; + __le32 qpc_bytes_76; + __le32 rx_rnr_time; + __le32 qpc_bytes_84; + __le32 qpc_bytes_88; + union { + __le32 rx_sge_len; + __le32 dma_length; + }; + union { + __le32 rx_sge_num; + __le32 rx_send_pktn; + __le32 r_key; + }; + __le32 va_l; + __le32 va_h; + __le32 qpc_bytes_108; + __le32 qpc_bytes_112; + __le32 rx_cur_sq_wqe_ba_l; + __le32 qpc_bytes_120; + __le32 qpc_bytes_124; + __le32 qpc_bytes_128; + __le32 qpc_bytes_132; + __le32 qpc_bytes_136; + __le32 qpc_bytes_140; + __le32 qpc_bytes_144; + __le32 qpc_bytes_148; + union { + __le32 rnr_retry; + __le32 ack_time; + }; + __le32 qpc_bytes_156; + __le32 pkt_use_len; + __le32 qpc_bytes_164; + __le32 qpc_bytes_168; + union { + __le32 sge_use_len; + __le32 pa_use_len; + }; + __le32 qpc_bytes_176; + __le32 qpc_bytes_180; + __le32 tx_cur_sq_wqe_ba_l; + __le32 qpc_bytes_188; + __le32 rvd21; +}; + +#define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S 0 +#define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M \ + (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S) + +#define QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S 3 +#define QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S 4 +#define QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S 5 +#define QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S 6 +#define QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S 7 + +#define QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S 8 +#define QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M \ + (((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S) + +#define QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S 12 +#define QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M \ + (((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S) + +#define QP_CONTEXT_QPC_BYTES_4_PD_S 16 +#define QP_CONTEXT_QPC_BYTES_4_PD_M \ + (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_4_PD_S) + +#define QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S 0 +#define QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M \ + (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S) + +#define QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S 16 +#define QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M \ + (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S) + +#define QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S 0 +#define QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M \ + (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S) + +#define QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S 16 +#define QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M \ + (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S) + +#define QP_CONTEXT_QPC_BYTES_16_QP_NUM_S 0 +#define QP_CONTEXT_QPC_BYTES_16_QP_NUM_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_16_QP_NUM_S) + +#define QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S 0 +#define QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M \ + (((1UL << 17) - 1) << QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S) + +#define QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S 18 +#define QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M \ + (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S) + +#define QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S 23 + +#define QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S 0 +#define QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M \ + (((1UL << 17) - 1) << QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S) + +#define QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S 18 +#define QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M \ + (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S) + +#define QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S 20 +#define QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S 21 +#define QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S 22 +#define QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S 23 + +#define QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S 24 +#define QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M \ + (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S) + +#define QP_CONTEXT_QPC_BYTES_36_DEST_QP_S 0 +#define QP_CONTEXT_QPC_BYTES_36_DEST_QP_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_36_DEST_QP_S) + +#define QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S 24 +#define QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M \ + (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S) + +#define QP_CONTEXT_QPC_BYTES_44_DMAC_H_S 0 +#define QP_CONTEXT_QPC_BYTES_44_DMAC_H_M \ + (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_44_DMAC_H_S) + +#define QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S 16 +#define QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M \ + (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S) + +#define QP_CONTEXT_QPC_BYTES_44_HOPLMT_S 24 +#define QP_CONTEXT_QPC_BYTES_44_HOPLMT_M \ + (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_44_HOPLMT_S) + +#define QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S 0 +#define QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M \ + (((1UL << 20) - 1) << QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S) + +#define QP_CONTEXT_QPC_BYTES_48_TCLASS_S 20 +#define QP_CONTEXT_QPC_BYTES_48_TCLASS_M \ + (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_48_TCLASS_S) + +#define QP_CONTEXT_QPC_BYTES_48_MTU_S 28 +#define QP_CONTEXT_QPC_BYTES_48_MTU_M \ + (((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_48_MTU_S) + +#define QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S 0 +#define QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M \ + (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S) + +#define QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S 16 +#define QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M \ + (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S) + +#define QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S 0 +#define QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M \ + (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S) + +#define QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S 8 +#define QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S) + +#define QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S 0 +#define QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S) + +#define QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S 24 +#define QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M \ + (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S) + +#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S 0 +#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S) + +#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S 24 +#define QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S 25 + +#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S 26 +#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M \ + (((1UL << 2) - 1) << \ + QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S) + +#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S 29 +#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M \ + (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S) + +#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S 0 +#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S) + +#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S 24 +#define QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S 25 + +#define QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S 0 +#define QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S) + +#define QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S 24 +#define QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M \ + (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S) + +#define QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S 0 +#define QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M \ + (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S) + +#define QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S 0 +#define QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M \ + (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S) + +#define QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S 16 +#define QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M \ + (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S) + +#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S 0 +#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S) + +#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S 24 + +#define QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S 25 +#define QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M \ + (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S) + +#define QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S 27 + +#define QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S 0 +#define QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S) + +#define QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S 24 +#define QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M \ + (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S) + +#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S 0 +#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S) + +#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S 24 +#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M \ + (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S) + +#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S 0 +#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M \ + (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S) + +#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S 16 +#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M \ + (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S) + +#define QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S 31 + +#define QP_CONTEXT_QPC_BYTES_144_QP_STATE_S 0 +#define QP_CONTEXT_QPC_BYTES_144_QP_STATE_M \ + (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_144_QP_STATE_S) + +#define QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S 0 +#define QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M \ + (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S) + +#define QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S 2 +#define QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M \ + (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S) + +#define QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S 5 +#define QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M \ + (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S) + +#define QP_CONTEXT_QPC_BYTES_148_LSN_S 8 +#define QP_CONTEXT_QPC_BYTES_148_LSN_M \ + (((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_148_LSN_S) + +#define QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S 0 +#define QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M \ + (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S) + +#define QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S 3 +#define QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M \ + (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S) + +#define QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S 8 +#define QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M \ + (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S) + +#define QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S 11 +#define QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M \ + (((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S) + +#define QP_CONTEXT_QPC_BYTES_156_SL_S 14 +#define QP_CONTEXT_QPC_BYTES_156_SL_M \ + (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_156_SL_S) + +#define QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S 16 +#define QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M \ + (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S) + +#define QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S 24 +#define QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M \ + (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S) + +#define QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S 0 +#define QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S) + +#define QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S 24 +#define QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M \ + (((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S) + +#define QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S 0 +#define QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M \ + (((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S) + +#define QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S 24 +#define QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M \ + (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S) + +#define QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S 26 +#define QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M \ + (((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S) + +#define QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S 28 +#define QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S 29 +#define QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S 30 + +#define QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S 0 +#define QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M \ + (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S) + +#define QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S 16 +#define QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M \ + (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S) + +#define QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S 0 +#define QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M \ + (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S) + +#define QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S 16 +#define QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M \ + (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S) + +#define QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S 0 +#define QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M \ + (((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S) + +#define QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S 8 + +#define QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S 16 +#define QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M \ + (((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S) + +#define STATUS_MASK 0xff +#define GO_BIT_TIMEOUT_MSECS 10000 +#define HCR_STATUS_OFFSET 0x18 +#define HCR_GO_BIT 15 + +struct hns_roce_rq_db { + __le32 u32_4; + __le32 u32_8; +}; + +#define RQ_DOORBELL_U32_4_RQ_HEAD_S 0 +#define RQ_DOORBELL_U32_4_RQ_HEAD_M \ + (((1UL << 15) - 1) << RQ_DOORBELL_U32_4_RQ_HEAD_S) + +#define RQ_DOORBELL_U32_8_QPN_S 0 +#define RQ_DOORBELL_U32_8_QPN_M (((1UL << 24) - 1) << RQ_DOORBELL_U32_8_QPN_S) + +#define RQ_DOORBELL_U32_8_CMD_S 28 +#define RQ_DOORBELL_U32_8_CMD_M (((1UL << 3) - 1) << RQ_DOORBELL_U32_8_CMD_S) + +#define RQ_DOORBELL_U32_8_HW_SYNC_S 31 + +struct hns_roce_sq_db { + __le32 u32_4; + __le32 u32_8; +}; + +#define SQ_DOORBELL_U32_4_SQ_HEAD_S 0 +#define SQ_DOORBELL_U32_4_SQ_HEAD_M \ + (((1UL << 15) - 1) << SQ_DOORBELL_U32_4_SQ_HEAD_S) + +#define SQ_DOORBELL_U32_4_SL_S 16 +#define SQ_DOORBELL_U32_4_SL_M \ + (((1UL << 2) - 1) << SQ_DOORBELL_U32_4_SL_S) + +#define SQ_DOORBELL_U32_4_PORT_S 18 +#define SQ_DOORBELL_U32_4_PORT_M (((1UL << 3) - 1) << SQ_DOORBELL_U32_4_PORT_S) + +#define SQ_DOORBELL_U32_8_QPN_S 0 +#define SQ_DOORBELL_U32_8_QPN_M (((1UL << 24) - 1) << SQ_DOORBELL_U32_8_QPN_S) + +#define SQ_DOORBELL_HW_SYNC_S 31 + +struct hns_roce_ext_db { + int esdb_dep; + int eodb_dep; + struct hns_roce_buf_list *sdb_buf_list; + struct hns_roce_buf_list *odb_buf_list; +}; + +struct hns_roce_db_table { + int sdb_ext_mod; + int odb_ext_mod; + struct hns_roce_ext_db *ext_db; +}; + +#define HW_SYNC_SLEEP_TIME_INTERVAL 20 +#define HW_SYNC_TIMEOUT_MSECS (25 * HW_SYNC_SLEEP_TIME_INTERVAL) +#define BT_CMD_SYNC_SHIFT 31 +#define HNS_ROCE_BA_SIZE (32 * 4096) + +struct hns_roce_bt_table { + struct hns_roce_buf_list qpc_buf; + struct hns_roce_buf_list mtpt_buf; + struct hns_roce_buf_list cqc_buf; +}; + +struct hns_roce_tptr_table { + struct hns_roce_buf_list tptr_buf; +}; + +struct hns_roce_qp_work { + struct work_struct work; + struct ib_device *ib_dev; + struct hns_roce_qp *qp; + u32 db_wait_stage; + u32 sdb_issue_ptr; + u32 sdb_inv_cnt; + u32 sche_cnt; +}; + +struct hns_roce_mr_free_work { + struct work_struct work; + struct ib_device *ib_dev; + struct completion *comp; + int comp_flag; + void *mr; +}; + +struct hns_roce_recreate_lp_qp_work { + struct work_struct work; + struct ib_device *ib_dev; + struct completion *comp; + int comp_flag; +}; + +struct hns_roce_free_mr { + struct workqueue_struct *free_mr_wq; + struct hns_roce_qp *mr_free_qp[HNS_ROCE_V1_RESV_QP]; + struct hns_roce_cq *mr_free_cq; + struct hns_roce_pd *mr_free_pd; +}; + +struct hns_roce_v1_priv { + struct hns_roce_db_table db_table; + struct hns_roce_raq_table raq_table; + struct hns_roce_bt_table bt_table; + struct hns_roce_tptr_table tptr_table; + struct hns_roce_free_mr free_mr; +}; + +int hns_dsaf_roce_reset(struct fwnode_handle *dsaf_fwnode, bool dereset); +int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); +int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); + +#endif diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c new file mode 100644 index 000000000..518b38e91 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -0,0 +1,6465 @@ +/* + * Copyright (c) 2016-2017 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/acpi.h> +#include <linux/etherdevice.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/types.h> +#include <net/addrconf.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> +#include <rdma/ib_umem.h> +#include <rdma/uverbs_ioctl.h> + +#include "hnae3.h" +#include "hns_roce_common.h" +#include "hns_roce_device.h" +#include "hns_roce_cmd.h" +#include "hns_roce_hem.h" +#include "hns_roce_hw_v2.h" + +static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg, + struct ib_sge *sg) +{ + dseg->lkey = cpu_to_le32(sg->lkey); + dseg->addr = cpu_to_le64(sg->addr); + dseg->len = cpu_to_le32(sg->length); +} + +/* + * mapped-value = 1 + real-value + * The hns wr opcode real value is start from 0, In order to distinguish between + * initialized and uninitialized map values, we plus 1 to the actual value when + * defining the mapping, so that the validity can be identified by checking the + * mapped value is greater than 0. + */ +#define HR_OPC_MAP(ib_key, hr_key) \ + [IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key + +static const u32 hns_roce_op_code[] = { + HR_OPC_MAP(RDMA_WRITE, RDMA_WRITE), + HR_OPC_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE_WITH_IMM), + HR_OPC_MAP(SEND, SEND), + HR_OPC_MAP(SEND_WITH_IMM, SEND_WITH_IMM), + HR_OPC_MAP(RDMA_READ, RDMA_READ), + HR_OPC_MAP(ATOMIC_CMP_AND_SWP, ATOM_CMP_AND_SWAP), + HR_OPC_MAP(ATOMIC_FETCH_AND_ADD, ATOM_FETCH_AND_ADD), + HR_OPC_MAP(SEND_WITH_INV, SEND_WITH_INV), + HR_OPC_MAP(LOCAL_INV, LOCAL_INV), + HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP, ATOM_MSK_CMP_AND_SWAP), + HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD), + HR_OPC_MAP(REG_MR, FAST_REG_PMR), +}; + +static u32 to_hr_opcode(u32 ib_opcode) +{ + if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code)) + return HNS_ROCE_V2_WQE_OP_MASK; + + return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 : + HNS_ROCE_V2_WQE_OP_MASK; +} + +static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, + const struct ib_reg_wr *wr) +{ + struct hns_roce_wqe_frmr_seg *fseg = + (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); + struct hns_roce_mr *mr = to_hr_mr(wr->mr); + u64 pbl_ba; + + /* use ib_access_flags */ + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S, + wr->access & IB_ACCESS_MW_BIND ? 1 : 0); + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S, + wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RR_S, + wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0); + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RW_S, + wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0); + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_LW_S, + wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0); + + /* Data structure reuse may lead to confusion */ + pbl_ba = mr->pbl_mtr.hem_cfg.root_ba; + rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba)); + rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba)); + + rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff); + rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32); + rc_sq_wqe->rkey = cpu_to_le32(wr->key); + rc_sq_wqe->va = cpu_to_le64(wr->mr->iova); + + fseg->pbl_size = cpu_to_le32(mr->npages); + roce_set_field(fseg->mode_buf_pg_sz, + V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M, + V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S, + to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift)); + roce_set_bit(fseg->mode_buf_pg_sz, + V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0); +} + +static void set_atomic_seg(const struct ib_send_wr *wr, + struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, + unsigned int valid_num_sge) +{ + struct hns_roce_v2_wqe_data_seg *dseg = + (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); + struct hns_roce_wqe_atomic_seg *aseg = + (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg); + + set_data_seg_v2(dseg, wr->sg_list); + + if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { + aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap); + aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add); + } else { + aseg->fetchadd_swap_data = + cpu_to_le64(atomic_wr(wr)->compare_add); + aseg->cmp_data = 0; + } + + roce_set_field(rc_sq_wqe->byte_16, V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge); +} + +static unsigned int get_std_sge_num(struct hns_roce_qp *qp) +{ + if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) + return 0; + + return HNS_ROCE_SGE_IN_WQE; +} + +static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, + const struct ib_send_wr *wr, + unsigned int *sge_idx, u32 msg_len) +{ + struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev; + unsigned int left_len_in_pg; + unsigned int idx = *sge_idx; + unsigned int std_sge_num; + unsigned int i = 0; + unsigned int len; + void *addr; + void *dseg; + + std_sge_num = get_std_sge_num(qp); + if (msg_len > (qp->sq.max_gs - std_sge_num) * HNS_ROCE_SGE_SIZE) { + ibdev_err(ibdev, + "no enough extended sge space for inline data.\n"); + return -EINVAL; + } + + dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1)); + left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg; + len = wr->sg_list[0].length; + addr = (void *)(unsigned long)(wr->sg_list[0].addr); + + /* When copying data to extended sge space, the left length in page may + * not long enough for current user's sge. So the data should be + * splited into several parts, one in the first page, and the others in + * the subsequent pages. + */ + while (1) { + if (len <= left_len_in_pg) { + memcpy(dseg, addr, len); + + idx += len / HNS_ROCE_SGE_SIZE; + + i++; + if (i >= wr->num_sge) + break; + + left_len_in_pg -= len; + len = wr->sg_list[i].length; + addr = (void *)(unsigned long)(wr->sg_list[i].addr); + dseg += len; + } else { + memcpy(dseg, addr, left_len_in_pg); + + len -= left_len_in_pg; + addr += left_len_in_pg; + idx += left_len_in_pg / HNS_ROCE_SGE_SIZE; + dseg = hns_roce_get_extend_sge(qp, + idx & (qp->sge.sge_cnt - 1)); + left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT; + } + } + + *sge_idx = idx; + + return 0; +} + +static void set_extend_sge(struct hns_roce_qp *qp, struct ib_sge *sge, + unsigned int *sge_ind, unsigned int cnt) +{ + struct hns_roce_v2_wqe_data_seg *dseg; + unsigned int idx = *sge_ind; + + while (cnt > 0) { + dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1)); + if (likely(sge->length)) { + set_data_seg_v2(dseg, sge); + idx++; + cnt--; + } + sge++; + } + + *sge_ind = idx; +} + +static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); + int mtu = ib_mtu_enum_to_int(qp->path_mtu); + + if (mtu < 0 || len > qp->max_inline_data || len > mtu) { + ibdev_err(&hr_dev->ib_dev, + "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n", + len, qp->max_inline_data, mtu); + return false; + } + + return true; +} + +static int set_rc_inl(struct hns_roce_qp *qp, const struct ib_send_wr *wr, + struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, + unsigned int *sge_idx) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); + u32 msg_len = le32_to_cpu(rc_sq_wqe->msg_len); + struct ib_device *ibdev = &hr_dev->ib_dev; + unsigned int curr_idx = *sge_idx; + void *dseg = rc_sq_wqe; + unsigned int i; + int ret; + + if (unlikely(wr->opcode == IB_WR_RDMA_READ)) { + ibdev_err(ibdev, "invalid inline parameters!\n"); + return -EINVAL; + } + + if (!check_inl_data_len(qp, msg_len)) + return -EINVAL; + + dseg += sizeof(struct hns_roce_v2_rc_send_wqe); + + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S, 1); + + if (msg_len <= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) { + roce_set_bit(rc_sq_wqe->byte_20, + V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 0); + + for (i = 0; i < wr->num_sge; i++) { + memcpy(dseg, ((void *)wr->sg_list[i].addr), + wr->sg_list[i].length); + dseg += wr->sg_list[i].length; + } + } else { + roce_set_bit(rc_sq_wqe->byte_20, + V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S, 1); + + ret = fill_ext_sge_inl_data(qp, wr, &curr_idx, msg_len); + if (ret) + return ret; + + roce_set_field(rc_sq_wqe->byte_16, + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, + curr_idx - *sge_idx); + } + + *sge_idx = curr_idx; + + return 0; +} + +static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, + struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, + unsigned int *sge_ind, + unsigned int valid_num_sge) +{ + struct hns_roce_v2_wqe_data_seg *dseg = + (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); + struct hns_roce_qp *qp = to_hr_qp(ibqp); + int j = 0; + int i; + + roce_set_field(rc_sq_wqe->byte_20, + V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M, + V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S, + (*sge_ind) & (qp->sge.sge_cnt - 1)); + + if (wr->send_flags & IB_SEND_INLINE) + return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind); + + if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) { + for (i = 0; i < wr->num_sge; i++) { + if (likely(wr->sg_list[i].length)) { + set_data_seg_v2(dseg, wr->sg_list + i); + dseg++; + } + } + } else { + for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; i++) { + if (likely(wr->sg_list[i].length)) { + set_data_seg_v2(dseg, wr->sg_list + i); + dseg++; + j++; + } + } + + set_extend_sge(qp, wr->sg_list + i, sge_ind, + valid_num_sge - HNS_ROCE_SGE_IN_WQE); + } + + roce_set_field(rc_sq_wqe->byte_16, + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M, + V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge); + + return 0; +} + +static int check_send_valid(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct ib_qp *ibqp = &hr_qp->ibqp; + + if (unlikely(ibqp->qp_type != IB_QPT_RC && + ibqp->qp_type != IB_QPT_GSI && + ibqp->qp_type != IB_QPT_UD)) { + ibdev_err(ibdev, "Not supported QP(0x%x)type!\n", + ibqp->qp_type); + return -EOPNOTSUPP; + } else if (unlikely(hr_qp->state == IB_QPS_RESET || + hr_qp->state == IB_QPS_INIT || + hr_qp->state == IB_QPS_RTR)) { + ibdev_err(ibdev, "failed to post WQE, QP state %hhu!\n", + hr_qp->state); + return -EINVAL; + } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) { + ibdev_err(ibdev, "failed to post WQE, dev state %d!\n", + hr_dev->state); + return -EIO; + } + + return 0; +} + +static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr, + unsigned int *sge_len) +{ + unsigned int valid_num = 0; + unsigned int len = 0; + int i; + + for (i = 0; i < wr->num_sge; i++) { + if (likely(wr->sg_list[i].length)) { + len += wr->sg_list[i].length; + valid_num++; + } + } + + *sge_len = len; + return valid_num; +} + +static __le32 get_immtdata(const struct ib_send_wr *wr) +{ + switch (wr->opcode) { + case IB_WR_SEND_WITH_IMM: + case IB_WR_RDMA_WRITE_WITH_IMM: + return cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); + default: + return 0; + } +} + +static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, + const struct ib_send_wr *wr) +{ + u32 ib_op = wr->opcode; + + if (ib_op != IB_WR_SEND && ib_op != IB_WR_SEND_WITH_IMM) + return -EINVAL; + + ud_sq_wqe->immtdata = get_immtdata(wr); + + roce_set_field(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OPCODE_M, + V2_UD_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op)); + + return 0; +} + +static inline int set_ud_wqe(struct hns_roce_qp *qp, + const struct ib_send_wr *wr, + void *wqe, unsigned int *sge_idx, + unsigned int owner_bit) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); + struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah); + struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe; + unsigned int curr_idx = *sge_idx; + int valid_num_sge; + u32 msg_len = 0; + int ret; + + valid_num_sge = calc_wr_sge_num(wr, &msg_len); + memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe)); + + ret = set_ud_opcode(ud_sq_wqe, wr); + if (WARN_ON(ret)) + return ret; + + roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M, + V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]); + roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M, + V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]); + roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M, + V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]); + roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M, + V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]); + roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_4_M, + V2_UD_SEND_WQE_BYTE_48_DMAC_4_S, ah->av.mac[4]); + roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_5_M, + V2_UD_SEND_WQE_BYTE_48_DMAC_5_S, ah->av.mac[5]); + + ud_sq_wqe->msg_len = cpu_to_le32(msg_len); + + /* Set sig attr */ + roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_CQE_S, + (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0); + + /* Set se attr */ + roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_SE_S, + (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0); + + roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_OWNER_S, + owner_bit); + + roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_PD_M, + V2_UD_SEND_WQE_BYTE_16_PD_S, to_hr_pd(qp->ibqp.pd)->pdn); + + roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M, + V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge); + + roce_set_field(ud_sq_wqe->byte_20, + V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M, + V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S, + curr_idx & (qp->sge.sge_cnt - 1)); + + roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M, + V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, ah->av.udp_sport); + ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ? + qp->qkey : ud_wr(wr)->remote_qkey); + roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M, + V2_UD_SEND_WQE_BYTE_32_DQPN_S, ud_wr(wr)->remote_qpn); + + roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M, + V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit); + roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M, + V2_UD_SEND_WQE_BYTE_36_TCLASS_S, ah->av.tclass); + roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M, + V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel); + roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M, + V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl); + roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_PORTN_M, + V2_UD_SEND_WQE_BYTE_40_PORTN_S, qp->port); + + roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M, + V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S, ah->av.gid_index); + + if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08) { + roce_set_bit(ud_sq_wqe->byte_40, + V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S, + ah->av.vlan_en); + roce_set_field(ud_sq_wqe->byte_36, + V2_UD_SEND_WQE_BYTE_36_VLAN_M, + V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id); + } + + memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN_V2); + + set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge); + + *sge_idx = curr_idx; + + return 0; +} + +static int set_rc_opcode(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, + const struct ib_send_wr *wr) +{ + u32 ib_op = wr->opcode; + + rc_sq_wqe->immtdata = get_immtdata(wr); + + switch (ib_op) { + case IB_WR_RDMA_READ: + case IB_WR_RDMA_WRITE: + case IB_WR_RDMA_WRITE_WITH_IMM: + rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey); + rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr); + break; + case IB_WR_SEND: + case IB_WR_SEND_WITH_IMM: + break; + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey); + rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr); + break; + case IB_WR_REG_MR: + set_frmr_seg(rc_sq_wqe, reg_wr(wr)); + break; + case IB_WR_LOCAL_INV: + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SO_S, 1); + fallthrough; + case IB_WR_SEND_WITH_INV: + rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey); + break; + default: + return -EINVAL; + } + + roce_set_field(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OPCODE_M, + V2_RC_SEND_WQE_BYTE_4_OPCODE_S, to_hr_opcode(ib_op)); + + return 0; +} +static inline int set_rc_wqe(struct hns_roce_qp *qp, + const struct ib_send_wr *wr, + void *wqe, unsigned int *sge_idx, + unsigned int owner_bit) +{ + struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe; + unsigned int curr_idx = *sge_idx; + unsigned int valid_num_sge; + u32 msg_len = 0; + int ret; + + valid_num_sge = calc_wr_sge_num(wr, &msg_len); + memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe)); + + rc_sq_wqe->msg_len = cpu_to_le32(msg_len); + + ret = set_rc_opcode(rc_sq_wqe, wr); + if (WARN_ON(ret)) + return ret; + + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FENCE_S, + (wr->send_flags & IB_SEND_FENCE) ? 1 : 0); + + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SE_S, + (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0); + + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_CQE_S, + (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0); + + roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S, + owner_bit); + + if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || + wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) + set_atomic_seg(wr, rc_sq_wqe, valid_num_sge); + else if (wr->opcode != IB_WR_REG_MR) + ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe, + &curr_idx, valid_num_sge); + + *sge_idx = curr_idx; + + return ret; +} + +static inline void update_sq_db(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *qp) +{ + /* + * Hip08 hardware cannot flush the WQEs in SQ if the QP state + * gets into errored mode. Hence, as a workaround to this + * hardware limitation, driver needs to assist in flushing. But + * the flushing operation uses mailbox to convey the QP state to + * the hardware and which can sleep due to the mutex protection + * around the mailbox calls. Hence, use the deferred flush for + * now. + */ + if (qp->state == IB_QPS_ERR) { + if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) + init_flush_work(hr_dev, qp); + } else { + struct hns_roce_v2_db sq_db = {}; + + roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M, + V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn); + roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M, + V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB); + roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M, + V2_DB_PARAMETER_IDX_S, qp->sq.head); + roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M, + V2_DB_PARAMETER_SL_S, qp->sl); + + hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l); + } +} + +static int hns_roce_v2_post_send(struct ib_qp *ibqp, + const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_qp *qp = to_hr_qp(ibqp); + unsigned long flags = 0; + unsigned int owner_bit; + unsigned int sge_idx; + unsigned int wqe_idx; + void *wqe = NULL; + int nreq; + int ret; + + spin_lock_irqsave(&qp->sq.lock, flags); + + ret = check_send_valid(hr_dev, qp); + if (unlikely(ret)) { + *bad_wr = wr; + nreq = 0; + goto out; + } + + sge_idx = qp->next_sge; + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { + ret = -ENOMEM; + *bad_wr = wr; + goto out; + } + + wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1); + + if (unlikely(wr->num_sge > qp->sq.max_gs)) { + ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n", + wr->num_sge, qp->sq.max_gs); + ret = -EINVAL; + *bad_wr = wr; + goto out; + } + + wqe = hns_roce_get_send_wqe(qp, wqe_idx); + qp->sq.wrid[wqe_idx] = wr->wr_id; + owner_bit = + ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1); + + /* Corresponding to the QP type, wqe process separately */ + if (ibqp->qp_type == IB_QPT_GSI) + ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit); + else if (ibqp->qp_type == IB_QPT_RC) + ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit); + + if (unlikely(ret)) { + *bad_wr = wr; + goto out; + } + } + +out: + if (likely(nreq)) { + qp->sq.head += nreq; + qp->next_sge = sge_idx; + /* Memory barrier */ + wmb(); + update_sq_db(hr_dev, qp); + } + + spin_unlock_irqrestore(&qp->sq.lock, flags); + + return ret; +} + +static int check_recv_valid(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) + return -EIO; + else if (hr_qp->state == IB_QPS_RESET) + return -EINVAL; + + return 0; +} + +static int hns_roce_v2_post_recv(struct ib_qp *ibqp, + const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_v2_wqe_data_seg *dseg; + struct hns_roce_rinl_sge *sge_list; + unsigned long flags; + void *wqe = NULL; + u32 wqe_idx; + int nreq; + int ret; + int i; + + spin_lock_irqsave(&hr_qp->rq.lock, flags); + + ret = check_recv_valid(hr_dev, hr_qp); + if (unlikely(ret)) { + *bad_wr = wr; + nreq = 0; + goto out; + } + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq, + hr_qp->ibqp.recv_cq))) { + ret = -ENOMEM; + *bad_wr = wr; + goto out; + } + + wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1); + + if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) { + ibdev_err(ibdev, "num_sge = %d >= max_sge = %u.\n", + wr->num_sge, hr_qp->rq.max_gs); + ret = -EINVAL; + *bad_wr = wr; + goto out; + } + + wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx); + dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; + for (i = 0; i < wr->num_sge; i++) { + if (!wr->sg_list[i].length) + continue; + set_data_seg_v2(dseg, wr->sg_list + i); + dseg++; + } + + if (wr->num_sge < hr_qp->rq.max_gs) { + dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY); + dseg->addr = 0; + } + + /* rq support inline data */ + if (hr_qp->rq_inl_buf.wqe_cnt) { + sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list; + hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt = + (u32)wr->num_sge; + for (i = 0; i < wr->num_sge; i++) { + sge_list[i].addr = + (void *)(u64)wr->sg_list[i].addr; + sge_list[i].len = wr->sg_list[i].length; + } + } + + hr_qp->rq.wrid[wqe_idx] = wr->wr_id; + } + +out: + if (likely(nreq)) { + hr_qp->rq.head += nreq; + /* Memory barrier */ + wmb(); + + /* + * Hip08 hardware cannot flush the WQEs in RQ if the QP state + * gets into errored mode. Hence, as a workaround to this + * hardware limitation, driver needs to assist in flushing. But + * the flushing operation uses mailbox to convey the QP state to + * the hardware and which can sleep due to the mutex protection + * around the mailbox calls. Hence, use the deferred flush for + * now. + */ + if (hr_qp->state == IB_QPS_ERR) { + if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, + &hr_qp->flush_flag)) + init_flush_work(hr_dev, hr_qp); + } else { + *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff; + } + } + spin_unlock_irqrestore(&hr_qp->rq.lock, flags); + + return ret; +} + +static void *get_srq_wqe(struct hns_roce_srq *srq, int n) +{ + return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift); +} + +static void *get_idx_buf(struct hns_roce_idx_que *idx_que, int n) +{ + return hns_roce_buf_offset(idx_que->mtr.kmem, + n << idx_que->entry_shift); +} + +static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index) +{ + /* always called with interrupts disabled. */ + spin_lock(&srq->lock); + + bitmap_clear(srq->idx_que.bitmap, wqe_index, 1); + srq->tail++; + + spin_unlock(&srq->lock); +} + +static int find_empty_entry(struct hns_roce_idx_que *idx_que, + unsigned long size) +{ + int wqe_idx; + + if (unlikely(bitmap_full(idx_que->bitmap, size))) + return -ENOSPC; + + wqe_idx = find_first_zero_bit(idx_que->bitmap, size); + + bitmap_set(idx_que->bitmap, wqe_idx, 1); + + return wqe_idx; +} + +static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, + const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); + struct hns_roce_srq *srq = to_hr_srq(ibsrq); + struct hns_roce_v2_wqe_data_seg *dseg; + struct hns_roce_v2_db srq_db; + unsigned long flags; + __le32 *srq_idx; + int ret = 0; + int wqe_idx; + void *wqe; + int nreq; + int ind; + int i; + + spin_lock_irqsave(&srq->lock, flags); + + ind = srq->head & (srq->wqe_cnt - 1); + + for (nreq = 0; wr; ++nreq, wr = wr->next) { + if (unlikely(wr->num_sge >= srq->max_gs)) { + ret = -EINVAL; + *bad_wr = wr; + break; + } + + if (unlikely(srq->head == srq->tail)) { + ret = -ENOMEM; + *bad_wr = wr; + break; + } + + wqe_idx = find_empty_entry(&srq->idx_que, srq->wqe_cnt); + if (unlikely(wqe_idx < 0)) { + ret = -ENOMEM; + *bad_wr = wr; + break; + } + + wqe = get_srq_wqe(srq, wqe_idx); + dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; + + for (i = 0; i < wr->num_sge; ++i) { + dseg[i].len = cpu_to_le32(wr->sg_list[i].length); + dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey); + dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr); + } + + if (wr->num_sge < srq->max_gs) { + dseg[i].len = 0; + dseg[i].lkey = cpu_to_le32(0x100); + dseg[i].addr = 0; + } + + srq_idx = get_idx_buf(&srq->idx_que, ind); + *srq_idx = cpu_to_le32(wqe_idx); + + srq->wrid[wqe_idx] = wr->wr_id; + ind = (ind + 1) & (srq->wqe_cnt - 1); + } + + if (likely(nreq)) { + srq->head += nreq; + + /* + * Make sure that descriptors are written before + * doorbell record. + */ + wmb(); + + srq_db.byte_4 = + cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S | + (srq->srqn & V2_DB_BYTE_4_TAG_M)); + srq_db.parameter = + cpu_to_le32(srq->head & V2_DB_PARAMETER_IDX_M); + + hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l); + } + + spin_unlock_irqrestore(&srq->lock, flags); + + return ret; +} + +static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev, + unsigned long instance_stage, + unsigned long reset_stage) +{ + /* When hardware reset has been completed once or more, we should stop + * sending mailbox&cmq&doorbell to hardware. If now in .init_instance() + * function, we should exit with error. If now at HNAE3_INIT_CLIENT + * stage of soft reset process, we should exit with error, and then + * HNAE3_INIT_CLIENT related process can rollback the operation like + * notifing hardware to free resources, HNAE3_INIT_CLIENT related + * process will exit with error to notify NIC driver to reschedule soft + * reset process once again. + */ + hr_dev->is_reset = true; + hr_dev->dis_db = true; + + if (reset_stage == HNS_ROCE_STATE_RST_INIT || + instance_stage == HNS_ROCE_STATE_INIT) + return CMD_RST_PRC_EBUSY; + + return CMD_RST_PRC_SUCCESS; +} + +static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev, + unsigned long instance_stage, + unsigned long reset_stage) +{ +#define HW_RESET_TIMEOUT_US 1000000 +#define HW_RESET_SLEEP_US 1000 + + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + unsigned long val; + int ret; + + /* When hardware reset is detected, we should stop sending mailbox&cmq& + * doorbell to hardware. If now in .init_instance() function, we should + * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset + * process, we should exit with error, and then HNAE3_INIT_CLIENT + * related process can rollback the operation like notifing hardware to + * free resources, HNAE3_INIT_CLIENT related process will exit with + * error to notify NIC driver to reschedule soft reset process once + * again. + */ + hr_dev->dis_db = true; + + ret = read_poll_timeout(ops->ae_dev_reset_cnt, val, + val > hr_dev->reset_cnt, HW_RESET_SLEEP_US, + HW_RESET_TIMEOUT_US, false, handle); + if (!ret) + hr_dev->is_reset = true; + + if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT || + instance_stage == HNS_ROCE_STATE_INIT) + return CMD_RST_PRC_EBUSY; + + return CMD_RST_PRC_SUCCESS; +} + +static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + /* When software reset is detected at .init_instance() function, we + * should stop sending mailbox&cmq&doorbell to hardware, and exit + * with error. + */ + hr_dev->dis_db = true; + if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) + hr_dev->is_reset = true; + + return CMD_RST_PRC_EBUSY; +} + +static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + unsigned long instance_stage; /* the current instance stage */ + unsigned long reset_stage; /* the current reset stage */ + unsigned long reset_cnt; + bool sw_resetting; + bool hw_resetting; + + if (hr_dev->is_reset) + return CMD_RST_PRC_SUCCESS; + + /* Get information about reset from NIC driver or RoCE driver itself, + * the meaning of the following variables from NIC driver are described + * as below: + * reset_cnt -- The count value of completed hardware reset. + * hw_resetting -- Whether hardware device is resetting now. + * sw_resetting -- Whether NIC's software reset process is running now. + */ + instance_stage = handle->rinfo.instance_state; + reset_stage = handle->rinfo.reset_state; + reset_cnt = ops->ae_dev_reset_cnt(handle); + hw_resetting = ops->get_cmdq_stat(handle); + sw_resetting = ops->ae_dev_resetting(handle); + + if (reset_cnt != hr_dev->reset_cnt) + return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage, + reset_stage); + else if (hw_resetting) + return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage, + reset_stage); + else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) + return hns_roce_v2_cmd_sw_resetting(hr_dev); + + return 0; +} + +static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring) +{ + int ntu = ring->next_to_use; + int ntc = ring->next_to_clean; + int used = (ntu - ntc + ring->desc_num) % ring->desc_num; + + return ring->desc_num - used - 1; +} + +static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev, + struct hns_roce_v2_cmq_ring *ring) +{ + int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc); + + ring->desc = kzalloc(size, GFP_KERNEL); + if (!ring->desc) + return -ENOMEM; + + ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) { + ring->desc_dma_addr = 0; + kfree(ring->desc); + ring->desc = NULL; + return -ENOMEM; + } + + return 0; +} + +static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev, + struct hns_roce_v2_cmq_ring *ring) +{ + dma_unmap_single(hr_dev->dev, ring->desc_dma_addr, + ring->desc_num * sizeof(struct hns_roce_cmq_desc), + DMA_BIDIRECTIONAL); + + ring->desc_dma_addr = 0; + kfree(ring->desc); +} + +static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ? + &priv->cmq.csq : &priv->cmq.crq; + + ring->flag = ring_type; + ring->next_to_clean = 0; + ring->next_to_use = 0; + + return hns_roce_alloc_cmq_desc(hr_dev, ring); +} + +static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ? + &priv->cmq.csq : &priv->cmq.crq; + dma_addr_t dma = ring->desc_dma_addr; + + if (ring_type == TYPE_CSQ) { + roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma); + roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, + upper_32_bits(dma)); + roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG, + ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S); + roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0); + roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0); + } else { + roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma); + roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG, + upper_32_bits(dma)); + roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG, + ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S); + roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0); + roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0); + } +} + +static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + int ret; + + /* Setup the queue entries for command queue */ + priv->cmq.csq.desc_num = CMD_CSQ_DESC_NUM; + priv->cmq.crq.desc_num = CMD_CRQ_DESC_NUM; + + /* Setup the lock for command queue */ + spin_lock_init(&priv->cmq.csq.lock); + spin_lock_init(&priv->cmq.crq.lock); + + /* Setup Tx write back timeout */ + priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT; + + /* Init CSQ */ + ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ); + if (ret) { + dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret); + return ret; + } + + /* Init CRQ */ + ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ); + if (ret) { + dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret); + goto err_crq; + } + + /* Init CSQ REG */ + hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ); + + /* Init CRQ REG */ + hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ); + + return 0; + +err_crq: + hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); + + return ret; +} + +static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + + hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); + hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq); +} + +static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc, + enum hns_roce_opcode_type opcode, + bool is_read) +{ + memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc)); + desc->opcode = cpu_to_le16(opcode); + desc->flag = + cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN); + if (is_read) + desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR); + else + desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); +} + +static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev) +{ + u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG); + struct hns_roce_v2_priv *priv = hr_dev->priv; + + return head == priv->cmq.csq.next_to_use; +} + +static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq; + struct hns_roce_cmq_desc *desc; + u16 ntc = csq->next_to_clean; + u32 head; + int clean = 0; + + desc = &csq->desc[ntc]; + head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG); + while (head != ntc) { + memset(desc, 0, sizeof(*desc)); + ntc++; + if (ntc == csq->desc_num) + ntc = 0; + desc = &csq->desc[ntc]; + clean++; + } + csq->next_to_clean = ntc; + + return clean; +} + +static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev, + struct hns_roce_cmq_desc *desc, int num) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq; + struct hns_roce_cmq_desc *desc_to_use; + bool complete = false; + u32 timeout = 0; + int handle = 0; + u16 desc_ret; + int ret; + int ntc; + + spin_lock_bh(&csq->lock); + + if (num > hns_roce_cmq_space(csq)) { + spin_unlock_bh(&csq->lock); + return -EBUSY; + } + + /* + * Record the location of desc in the cmq for this time + * which will be use for hardware to write back + */ + ntc = csq->next_to_use; + + while (handle < num) { + desc_to_use = &csq->desc[csq->next_to_use]; + *desc_to_use = desc[handle]; + dev_dbg(hr_dev->dev, "set cmq desc:\n"); + csq->next_to_use++; + if (csq->next_to_use == csq->desc_num) + csq->next_to_use = 0; + handle++; + } + + /* Write to hardware */ + roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use); + + /* + * If the command is sync, wait for the firmware to write back, + * if multi descriptors to be sent, use the first one to check + */ + if (le16_to_cpu(desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) { + do { + if (hns_roce_cmq_csq_done(hr_dev)) + break; + udelay(1); + timeout++; + } while (timeout < priv->cmq.tx_timeout); + } + + if (hns_roce_cmq_csq_done(hr_dev)) { + complete = true; + handle = 0; + ret = 0; + while (handle < num) { + /* get the result of hardware write back */ + desc_to_use = &csq->desc[ntc]; + desc[handle] = *desc_to_use; + dev_dbg(hr_dev->dev, "Get cmq desc:\n"); + desc_ret = le16_to_cpu(desc[handle].retval); + if (unlikely(desc_ret != CMD_EXEC_SUCCESS)) + ret = -EIO; + priv->cmq.last_status = desc_ret; + ntc++; + handle++; + if (ntc == csq->desc_num) + ntc = 0; + } + } + + if (!complete) + ret = -EAGAIN; + + /* clean the command send queue */ + handle = hns_roce_cmq_csq_clean(hr_dev); + if (handle != num) + dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n", + handle, num); + + spin_unlock_bh(&csq->lock); + + return ret; +} + +static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, + struct hns_roce_cmq_desc *desc, int num) +{ + int retval; + int ret; + + ret = hns_roce_v2_rst_process_cmd(hr_dev); + if (ret == CMD_RST_PRC_SUCCESS) + return 0; + if (ret == CMD_RST_PRC_EBUSY) + return -EBUSY; + + ret = __hns_roce_cmq_send(hr_dev, desc, num); + if (ret) { + retval = hns_roce_v2_rst_process_cmd(hr_dev); + if (retval == CMD_RST_PRC_SUCCESS) + return 0; + else if (retval == CMD_RST_PRC_EBUSY) + return -EBUSY; + } + + return ret; +} + +static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_query_version *resp; + struct hns_roce_cmq_desc desc; + int ret; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + return ret; + + resp = (struct hns_roce_query_version *)desc.data; + hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version); + hr_dev->vendor_id = hr_dev->pci_dev->vendor; + + return 0; +} + +static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + unsigned long reset_cnt; + bool sw_resetting; + bool hw_resetting; + + reset_cnt = ops->ae_dev_reset_cnt(handle); + hw_resetting = ops->get_hw_reset_stat(handle); + sw_resetting = ops->ae_dev_resetting(handle); + + if (reset_cnt != hr_dev->reset_cnt || hw_resetting || sw_resetting) + return true; + + return false; +} + +static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval, + int flag) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + unsigned long instance_stage; + unsigned long reset_cnt; + unsigned long end; + bool sw_resetting; + bool hw_resetting; + + instance_stage = handle->rinfo.instance_state; + reset_cnt = ops->ae_dev_reset_cnt(handle); + hw_resetting = ops->get_hw_reset_stat(handle); + sw_resetting = ops->ae_dev_resetting(handle); + + if (reset_cnt != hr_dev->reset_cnt) { + hr_dev->dis_db = true; + hr_dev->is_reset = true; + dev_info(hr_dev->dev, "Func clear success after reset.\n"); + } else if (hw_resetting) { + hr_dev->dis_db = true; + + dev_warn(hr_dev->dev, + "Func clear is pending, device in resetting state.\n"); + end = HNS_ROCE_V2_HW_RST_TIMEOUT; + while (end) { + if (!ops->get_hw_reset_stat(handle)) { + hr_dev->is_reset = true; + dev_info(hr_dev->dev, + "Func clear success after reset.\n"); + return; + } + msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); + end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; + } + + dev_warn(hr_dev->dev, "Func clear failed.\n"); + } else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) { + hr_dev->dis_db = true; + + dev_warn(hr_dev->dev, + "Func clear is pending, device in resetting state.\n"); + end = HNS_ROCE_V2_HW_RST_TIMEOUT; + while (end) { + if (ops->ae_dev_reset_cnt(handle) != + hr_dev->reset_cnt) { + hr_dev->is_reset = true; + dev_info(hr_dev->dev, + "Func clear success after sw reset\n"); + return; + } + msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); + end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; + } + + dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n"); + } else { + if (retval && !flag) + dev_warn(hr_dev->dev, + "Func clear read failed, ret = %d.\n", retval); + + dev_warn(hr_dev->dev, "Func clear failed.\n"); + } +} +static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) +{ + bool fclr_write_fail_flag = false; + struct hns_roce_func_clear *resp; + struct hns_roce_cmq_desc desc; + unsigned long end; + int ret = 0; + + if (hns_roce_func_clr_chk_rst(hr_dev)) + goto out; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false); + resp = (struct hns_roce_func_clear *)desc.data; + + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) { + fclr_write_fail_flag = true; + dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n", + ret); + goto out; + } + + msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL); + end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS; + while (end) { + if (hns_roce_func_clr_chk_rst(hr_dev)) + goto out; + msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT); + end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, + true); + + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + continue; + + if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) { + hr_dev->is_reset = true; + return; + } + } + +out: + hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag); +} + +static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_query_fw_info *resp; + struct hns_roce_cmq_desc desc; + int ret; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + return ret; + + resp = (struct hns_roce_query_fw_info *)desc.data; + hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver)); + + return 0; +} + +static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cfg_global_param *req; + struct hns_roce_cmq_desc desc; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM, + false); + + req = (struct hns_roce_cfg_global_param *)desc.data; + memset(req, 0, sizeof(*req)); + roce_set_field(req->time_cfg_udp_port, + CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M, + CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8); + roce_set_field(req->time_cfg_udp_port, + CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M, + CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc[2]; + struct hns_roce_pf_res_a *req_a; + struct hns_roce_pf_res_b *req_b; + int ret; + int i; + + for (i = 0; i < 2; i++) { + hns_roce_cmq_setup_basic_desc(&desc[i], + HNS_ROCE_OPC_QUERY_PF_RES, true); + + if (i == 0) + desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + else + desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + } + + ret = hns_roce_cmq_send(hr_dev, desc, 2); + if (ret) + return ret; + + req_a = (struct hns_roce_pf_res_a *)desc[0].data; + req_b = (struct hns_roce_pf_res_b *)desc[1].data; + + hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num, + PF_RES_DATA_1_PF_QPC_BT_NUM_M, + PF_RES_DATA_1_PF_QPC_BT_NUM_S); + hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num, + PF_RES_DATA_2_PF_SRQC_BT_NUM_M, + PF_RES_DATA_2_PF_SRQC_BT_NUM_S); + hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num, + PF_RES_DATA_3_PF_CQC_BT_NUM_M, + PF_RES_DATA_3_PF_CQC_BT_NUM_S); + hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num, + PF_RES_DATA_4_PF_MPT_BT_NUM_M, + PF_RES_DATA_4_PF_MPT_BT_NUM_S); + + hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num, + PF_RES_DATA_3_PF_SL_NUM_M, + PF_RES_DATA_3_PF_SL_NUM_S); + hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num, + PF_RES_DATA_4_PF_SCCC_BT_NUM_M, + PF_RES_DATA_4_PF_SCCC_BT_NUM_S); + + return 0; +} + +static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_pf_timer_res_a *req_a; + struct hns_roce_cmq_desc desc; + int ret; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES, + true); + + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + return ret; + + req_a = (struct hns_roce_pf_timer_res_a *)desc.data; + + hr_dev->caps.qpc_timer_bt_num = + roce_get_field(req_a->qpc_timer_bt_idx_num, + PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M, + PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S); + hr_dev->caps.cqc_timer_bt_num = + roce_get_field(req_a->cqc_timer_bt_idx_num, + PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M, + PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S); + + return 0; +} + +static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id) +{ + struct hns_roce_cmq_desc desc; + struct hns_roce_vf_switch *swt; + int ret; + + swt = (struct hns_roce_vf_switch *)desc.data; + hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true); + swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL); + roce_set_field(swt->fun_id, VF_SWITCH_DATA_FUN_ID_VF_ID_M, + VF_SWITCH_DATA_FUN_ID_VF_ID_S, vf_id); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + return ret; + + desc.flag = + cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN); + desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); + roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1); + roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0); + roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc[2]; + struct hns_roce_vf_res_a *req_a; + struct hns_roce_vf_res_b *req_b; + int i; + + req_a = (struct hns_roce_vf_res_a *)desc[0].data; + req_b = (struct hns_roce_vf_res_b *)desc[1].data; + for (i = 0; i < 2; i++) { + hns_roce_cmq_setup_basic_desc(&desc[i], + HNS_ROCE_OPC_ALLOC_VF_RES, false); + + if (i == 0) + desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + else + desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + } + + roce_set_field(req_a->vf_qpc_bt_idx_num, + VF_RES_A_DATA_1_VF_QPC_BT_IDX_M, + VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0); + roce_set_field(req_a->vf_qpc_bt_idx_num, + VF_RES_A_DATA_1_VF_QPC_BT_NUM_M, + VF_RES_A_DATA_1_VF_QPC_BT_NUM_S, HNS_ROCE_VF_QPC_BT_NUM); + + roce_set_field(req_a->vf_srqc_bt_idx_num, + VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M, + VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0); + roce_set_field(req_a->vf_srqc_bt_idx_num, + VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M, + VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S, + HNS_ROCE_VF_SRQC_BT_NUM); + + roce_set_field(req_a->vf_cqc_bt_idx_num, + VF_RES_A_DATA_3_VF_CQC_BT_IDX_M, + VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0); + roce_set_field(req_a->vf_cqc_bt_idx_num, + VF_RES_A_DATA_3_VF_CQC_BT_NUM_M, + VF_RES_A_DATA_3_VF_CQC_BT_NUM_S, HNS_ROCE_VF_CQC_BT_NUM); + + roce_set_field(req_a->vf_mpt_bt_idx_num, + VF_RES_A_DATA_4_VF_MPT_BT_IDX_M, + VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0); + roce_set_field(req_a->vf_mpt_bt_idx_num, + VF_RES_A_DATA_4_VF_MPT_BT_NUM_M, + VF_RES_A_DATA_4_VF_MPT_BT_NUM_S, HNS_ROCE_VF_MPT_BT_NUM); + + roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_IDX_M, + VF_RES_A_DATA_5_VF_EQC_IDX_S, 0); + roce_set_field(req_a->vf_eqc_bt_idx_num, VF_RES_A_DATA_5_VF_EQC_NUM_M, + VF_RES_A_DATA_5_VF_EQC_NUM_S, HNS_ROCE_VF_EQC_NUM); + + roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_IDX_M, + VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0); + roce_set_field(req_b->vf_smac_idx_num, VF_RES_B_DATA_1_VF_SMAC_NUM_M, + VF_RES_B_DATA_1_VF_SMAC_NUM_S, HNS_ROCE_VF_SMAC_NUM); + + roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_IDX_M, + VF_RES_B_DATA_2_VF_SGID_IDX_S, 0); + roce_set_field(req_b->vf_sgid_idx_num, VF_RES_B_DATA_2_VF_SGID_NUM_M, + VF_RES_B_DATA_2_VF_SGID_NUM_S, HNS_ROCE_VF_SGID_NUM); + + roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_QID_IDX_M, + VF_RES_B_DATA_3_VF_QID_IDX_S, 0); + roce_set_field(req_b->vf_qid_idx_sl_num, VF_RES_B_DATA_3_VF_SL_NUM_M, + VF_RES_B_DATA_3_VF_SL_NUM_S, HNS_ROCE_VF_SL_NUM); + + roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M, + VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0); + roce_set_field(req_b->vf_sccc_idx_num, VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M, + VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S, + HNS_ROCE_VF_SCCC_BT_NUM); + + return hns_roce_cmq_send(hr_dev, desc, 2); +} + +static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev) +{ + u8 srqc_hop_num = hr_dev->caps.srqc_hop_num; + u8 qpc_hop_num = hr_dev->caps.qpc_hop_num; + u8 cqc_hop_num = hr_dev->caps.cqc_hop_num; + u8 mpt_hop_num = hr_dev->caps.mpt_hop_num; + u8 sccc_hop_num = hr_dev->caps.sccc_hop_num; + struct hns_roce_cfg_bt_attr *req; + struct hns_roce_cmq_desc desc; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false); + req = (struct hns_roce_cfg_bt_attr *)desc.data; + memset(req, 0, sizeof(*req)); + + roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M, + CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S, + hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET); + roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M, + CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S, + hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET); + roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M, + CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S, + qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num); + + roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M, + CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S, + hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET); + roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M, + CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S, + hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET); + roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M, + CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S, + srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num); + + roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M, + CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S, + hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET); + roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M, + CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S, + hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET); + roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M, + CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S, + cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num); + + roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M, + CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S, + hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET); + roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M, + CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S, + hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET); + roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M, + CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S, + mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num); + + roce_set_field(req->vf_sccc_cfg, + CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M, + CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S, + hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET); + roce_set_field(req->vf_sccc_cfg, + CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M, + CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S, + hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET); + roce_set_field(req->vf_sccc_cfg, + CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M, + CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S, + sccc_hop_num == + HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static void set_default_caps(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_caps *caps = &hr_dev->caps; + + caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM; + caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM; + caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM; + caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM; + caps->min_cqes = HNS_ROCE_MIN_CQE_NUM; + caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM; + caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM; + caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM; + caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM; + caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; + caps->num_uars = HNS_ROCE_V2_UAR_NUM; + caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM; + caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM; + caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM; + caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM; + caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM; + caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; + caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; + caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; + caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; + caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM; + caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA; + caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA; + caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ; + caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ; + caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ; + caps->qpc_sz = HNS_ROCE_V2_QPC_SZ; + caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ; + caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ; + caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ; + caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ; + caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ; + caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; + caps->idx_entry_sz = HNS_ROCE_V2_IDX_ENTRY_SZ; + caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE; + caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; + caps->reserved_lkey = 0; + caps->reserved_pds = 0; + caps->reserved_mrws = 1; + caps->reserved_uars = 0; + caps->reserved_cqs = 0; + caps->reserved_srqs = 0; + caps->reserved_qps = HNS_ROCE_V2_RSV_QPS; + + caps->qpc_ba_pg_sz = 0; + caps->qpc_buf_pg_sz = 0; + caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; + caps->srqc_ba_pg_sz = 0; + caps->srqc_buf_pg_sz = 0; + caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; + caps->cqc_ba_pg_sz = 0; + caps->cqc_buf_pg_sz = 0; + caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; + caps->mpt_ba_pg_sz = 0; + caps->mpt_buf_pg_sz = 0; + caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; + caps->mtt_ba_pg_sz = 0; + caps->mtt_buf_pg_sz = 0; + caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM; + caps->wqe_sq_hop_num = HNS_ROCE_SQWQE_HOP_NUM; + caps->wqe_sge_hop_num = HNS_ROCE_EXT_SGE_HOP_NUM; + caps->wqe_rq_hop_num = HNS_ROCE_RQWQE_HOP_NUM; + caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K; + caps->cqe_buf_pg_sz = 0; + caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; + caps->srqwqe_ba_pg_sz = 0; + caps->srqwqe_buf_pg_sz = 0; + caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM; + caps->idx_ba_pg_sz = 0; + caps->idx_buf_pg_sz = 0; + caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM; + caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE; + + caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR | + HNS_ROCE_CAP_FLAG_ROCE_V1_V2 | + HNS_ROCE_CAP_FLAG_RECORD_DB | + HNS_ROCE_CAP_FLAG_SQ_RECORD_DB; + + caps->pkey_table_len[0] = 1; + caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; + caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM; + caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM; + caps->aeqe_size = HNS_ROCE_AEQE_SIZE; + caps->ceqe_size = HNS_ROCE_CEQE_SIZE; + caps->local_ca_ack_delay = 0; + caps->max_mtu = IB_MTU_4096; + + caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR; + caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE; + + caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW | + HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR | + HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL; + + caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM; + caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ; + caps->qpc_timer_ba_pg_sz = 0; + caps->qpc_timer_buf_pg_sz = 0; + caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; + caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM; + caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; + caps->cqc_timer_ba_pg_sz = 0; + caps->cqc_timer_buf_pg_sz = 0; + caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; + + caps->sccc_sz = HNS_ROCE_V2_SCCC_SZ; + caps->sccc_ba_pg_sz = 0; + caps->sccc_buf_pg_sz = 0; + caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM; + + if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { + caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE; + caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE; + caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE; + caps->qpc_sz = HNS_ROCE_V3_QPC_SZ; + } +} + +static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num, + int *buf_page_size, int *bt_page_size, u32 hem_type) +{ + u64 obj_per_chunk; + u64 bt_chunk_size = PAGE_SIZE; + u64 buf_chunk_size = PAGE_SIZE; + u64 obj_per_chunk_default = buf_chunk_size / obj_size; + + *buf_page_size = 0; + *bt_page_size = 0; + + switch (hop_num) { + case 3: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case 2: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case 1: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case HNS_ROCE_HOP_NUM_0: + obj_per_chunk = ctx_bt_num * obj_per_chunk_default; + break; + default: + pr_err("table %u not support hop_num = %u!\n", hem_type, + hop_num); + return; + } + + if (hem_type >= HEM_TYPE_MTT) + *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); + else + *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); +} + +static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM]; + struct hns_roce_caps *caps = &hr_dev->caps; + struct hns_roce_query_pf_caps_a *resp_a; + struct hns_roce_query_pf_caps_b *resp_b; + struct hns_roce_query_pf_caps_c *resp_c; + struct hns_roce_query_pf_caps_d *resp_d; + struct hns_roce_query_pf_caps_e *resp_e; + int ctx_hop_num; + int pbl_hop_num; + int ret; + int i; + + for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) { + hns_roce_cmq_setup_basic_desc(&desc[i], + HNS_ROCE_OPC_QUERY_PF_CAPS_NUM, + true); + if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1)) + desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + else + desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + } + + ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM); + if (ret) + return ret; + + resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data; + resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data; + resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data; + resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data; + resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data; + + caps->local_ca_ack_delay = resp_a->local_ca_ack_delay; + caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg); + caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline); + caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg); + caps->max_extend_sg = le32_to_cpu(resp_a->max_extend_sg); + caps->num_qpc_timer = le16_to_cpu(resp_a->num_qpc_timer); + caps->num_cqc_timer = le16_to_cpu(resp_a->num_cqc_timer); + caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges); + caps->num_aeq_vectors = resp_a->num_aeq_vectors; + caps->num_other_vectors = resp_a->num_other_vectors; + caps->max_sq_desc_sz = resp_a->max_sq_desc_sz; + caps->max_rq_desc_sz = resp_a->max_rq_desc_sz; + caps->max_srq_desc_sz = resp_a->max_srq_desc_sz; + caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE; + + caps->mtpt_entry_sz = resp_b->mtpt_entry_sz; + caps->irrl_entry_sz = resp_b->irrl_entry_sz; + caps->trrl_entry_sz = resp_b->trrl_entry_sz; + caps->cqc_entry_sz = resp_b->cqc_entry_sz; + caps->srqc_entry_sz = resp_b->srqc_entry_sz; + caps->idx_entry_sz = resp_b->idx_entry_sz; + caps->sccc_sz = resp_b->sccc_sz; + caps->max_mtu = resp_b->max_mtu; + caps->qpc_sz = HNS_ROCE_V2_QPC_SZ; + caps->min_cqes = resp_b->min_cqes; + caps->min_wqes = resp_b->min_wqes; + caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap); + caps->pkey_table_len[0] = resp_b->pkey_table_len; + caps->phy_num_uars = resp_b->phy_num_uars; + ctx_hop_num = resp_b->ctx_hop_num; + pbl_hop_num = resp_b->pbl_hop_num; + + caps->num_pds = 1 << roce_get_field(resp_c->cap_flags_num_pds, + V2_QUERY_PF_CAPS_C_NUM_PDS_M, + V2_QUERY_PF_CAPS_C_NUM_PDS_S); + caps->flags = roce_get_field(resp_c->cap_flags_num_pds, + V2_QUERY_PF_CAPS_C_CAP_FLAGS_M, + V2_QUERY_PF_CAPS_C_CAP_FLAGS_S); + caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) << + HNS_ROCE_CAP_FLAGS_EX_SHIFT; + + caps->num_cqs = 1 << roce_get_field(resp_c->max_gid_num_cqs, + V2_QUERY_PF_CAPS_C_NUM_CQS_M, + V2_QUERY_PF_CAPS_C_NUM_CQS_S); + caps->gid_table_len[0] = roce_get_field(resp_c->max_gid_num_cqs, + V2_QUERY_PF_CAPS_C_MAX_GID_M, + V2_QUERY_PF_CAPS_C_MAX_GID_S); + caps->max_cqes = 1 << roce_get_field(resp_c->cq_depth, + V2_QUERY_PF_CAPS_C_CQ_DEPTH_M, + V2_QUERY_PF_CAPS_C_CQ_DEPTH_S); + caps->num_mtpts = 1 << roce_get_field(resp_c->num_mrws, + V2_QUERY_PF_CAPS_C_NUM_MRWS_M, + V2_QUERY_PF_CAPS_C_NUM_MRWS_S); + caps->num_qps = 1 << roce_get_field(resp_c->ord_num_qps, + V2_QUERY_PF_CAPS_C_NUM_QPS_M, + V2_QUERY_PF_CAPS_C_NUM_QPS_S); + caps->max_qp_init_rdma = roce_get_field(resp_c->ord_num_qps, + V2_QUERY_PF_CAPS_C_MAX_ORD_M, + V2_QUERY_PF_CAPS_C_MAX_ORD_S); + caps->max_qp_dest_rdma = caps->max_qp_init_rdma; + caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth); + caps->num_srqs = 1 << roce_get_field(resp_d->wq_hop_num_max_srqs, + V2_QUERY_PF_CAPS_D_NUM_SRQS_M, + V2_QUERY_PF_CAPS_D_NUM_SRQS_S); + caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth); + caps->ceqe_depth = 1 << roce_get_field(resp_d->num_ceqs_ceq_depth, + V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M, + V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S); + caps->num_comp_vectors = roce_get_field(resp_d->num_ceqs_ceq_depth, + V2_QUERY_PF_CAPS_D_NUM_CEQS_M, + V2_QUERY_PF_CAPS_D_NUM_CEQS_S); + caps->aeqe_depth = 1 << roce_get_field(resp_d->arm_st_aeq_depth, + V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M, + V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S); + caps->default_aeq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth, + V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_M, + V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_S); + caps->default_ceq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth, + V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_M, + V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_S); + caps->reserved_pds = roce_get_field(resp_d->num_uars_rsv_pds, + V2_QUERY_PF_CAPS_D_RSV_PDS_M, + V2_QUERY_PF_CAPS_D_RSV_PDS_S); + caps->num_uars = 1 << roce_get_field(resp_d->num_uars_rsv_pds, + V2_QUERY_PF_CAPS_D_NUM_UARS_M, + V2_QUERY_PF_CAPS_D_NUM_UARS_S); + caps->reserved_qps = roce_get_field(resp_d->rsv_uars_rsv_qps, + V2_QUERY_PF_CAPS_D_RSV_QPS_M, + V2_QUERY_PF_CAPS_D_RSV_QPS_S); + caps->reserved_uars = roce_get_field(resp_d->rsv_uars_rsv_qps, + V2_QUERY_PF_CAPS_D_RSV_UARS_M, + V2_QUERY_PF_CAPS_D_RSV_UARS_S); + caps->reserved_mrws = roce_get_field(resp_e->chunk_size_shift_rsv_mrws, + V2_QUERY_PF_CAPS_E_RSV_MRWS_M, + V2_QUERY_PF_CAPS_E_RSV_MRWS_S); + caps->chunk_sz = 1 << roce_get_field(resp_e->chunk_size_shift_rsv_mrws, + V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M, + V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S); + caps->reserved_cqs = roce_get_field(resp_e->rsv_cqs, + V2_QUERY_PF_CAPS_E_RSV_CQS_M, + V2_QUERY_PF_CAPS_E_RSV_CQS_S); + caps->reserved_srqs = roce_get_field(resp_e->rsv_srqs, + V2_QUERY_PF_CAPS_E_RSV_SRQS_M, + V2_QUERY_PF_CAPS_E_RSV_SRQS_S); + caps->reserved_lkey = roce_get_field(resp_e->rsv_lkey, + V2_QUERY_PF_CAPS_E_RSV_LKEYS_M, + V2_QUERY_PF_CAPS_E_RSV_LKEYS_S); + caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt); + caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period); + caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt); + caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period); + + caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ; + caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; + caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; + caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; + caps->ceqe_size = HNS_ROCE_CEQE_SIZE; + caps->aeqe_size = HNS_ROCE_AEQE_SIZE; + caps->mtt_ba_pg_sz = 0; + caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS; + caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; + caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; + + caps->qpc_hop_num = ctx_hop_num; + caps->srqc_hop_num = ctx_hop_num; + caps->cqc_hop_num = ctx_hop_num; + caps->mpt_hop_num = ctx_hop_num; + caps->mtt_hop_num = pbl_hop_num; + caps->cqe_hop_num = pbl_hop_num; + caps->srqwqe_hop_num = pbl_hop_num; + caps->idx_hop_num = pbl_hop_num; + caps->wqe_sq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs, + V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M, + V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S); + caps->wqe_sge_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs, + V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M, + V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S); + caps->wqe_rq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs, + V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M, + V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S); + + if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { + caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE; + caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE; + caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE; + caps->qpc_sz = HNS_ROCE_V3_QPC_SZ; + caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ; + } + + calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num, + caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz, + HEM_TYPE_QPC); + calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num, + caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz, + HEM_TYPE_MTPT); + calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num, + caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz, + HEM_TYPE_CQC); + calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, caps->srqc_hop_num, + caps->srqc_bt_num, &caps->srqc_buf_pg_sz, + &caps->srqc_ba_pg_sz, HEM_TYPE_SRQC); + + caps->sccc_hop_num = ctx_hop_num; + caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; + caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; + + calc_pg_sz(caps->num_qps, caps->sccc_sz, + caps->sccc_hop_num, caps->sccc_bt_num, + &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz, + HEM_TYPE_SCCC); + calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz, + caps->cqc_timer_hop_num, caps->cqc_timer_bt_num, + &caps->cqc_timer_buf_pg_sz, + &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER); + + calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num, + 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE); + calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz, + caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz, + &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE); + calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, caps->idx_hop_num, + 1, &caps->idx_buf_pg_sz, &caps->idx_ba_pg_sz, HEM_TYPE_IDX); + + if (!(caps->page_size_cap & PAGE_SIZE)) + caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; + + return 0; +} + +static int hns_roce_config_qpc_size(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc; + struct hns_roce_cfg_entry_size *cfg_size = + (struct hns_roce_cfg_entry_size *)desc.data; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE, + false); + + cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_QPC_SIZE); + cfg_size->size = cpu_to_le32(hr_dev->caps.qpc_sz); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static int hns_roce_config_sccc_size(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc; + struct hns_roce_cfg_entry_size *cfg_size = + (struct hns_roce_cfg_entry_size *)desc.data; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE, + false); + + cfg_size->type = cpu_to_le32(HNS_ROCE_CFG_SCCC_SIZE); + cfg_size->size = cpu_to_le32(hr_dev->caps.sccc_sz); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev) +{ + int ret; + + if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09) + return 0; + + ret = hns_roce_config_qpc_size(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret); + return ret; + } + + ret = hns_roce_config_sccc_size(hr_dev); + if (ret) + dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret); + + return ret; +} + +static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_caps *caps = &hr_dev->caps; + int ret; + + ret = hns_roce_cmq_query_hw_info(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n", + ret); + return ret; + } + + ret = hns_roce_query_fw_ver(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n", + ret); + return ret; + } + + ret = hns_roce_config_global_param(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n", + ret); + return ret; + } + + /* Get pf resource owned by every pf */ + ret = hns_roce_query_pf_resource(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n", + ret); + return ret; + } + + ret = hns_roce_query_pf_timer_resource(hr_dev); + if (ret) { + dev_err(hr_dev->dev, + "failed to query pf timer resource, ret = %d.\n", ret); + return ret; + } + + ret = hns_roce_set_vf_switch_param(hr_dev, 0); + if (ret) { + dev_err(hr_dev->dev, + "failed to set function switch param, ret = %d.\n", + ret); + return ret; + } + + hr_dev->vendor_part_id = hr_dev->pci_dev->device; + hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid); + + caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K; + caps->pbl_buf_pg_sz = 0; + caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM; + caps->eqe_ba_pg_sz = 0; + caps->eqe_buf_pg_sz = 0; + caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM; + caps->tsq_buf_pg_sz = 0; + + ret = hns_roce_query_pf_caps(hr_dev); + if (ret) + set_default_caps(hr_dev); + + ret = hns_roce_alloc_vf_resource(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n", + ret); + return ret; + } + + ret = hns_roce_v2_set_bt(hr_dev); + if (ret) { + dev_err(hr_dev->dev, + "Configure bt attribute fail, ret = %d.\n", ret); + return ret; + } + + /* Configure the size of QPC, SCCC, etc. */ + ret = hns_roce_config_entry_size(hr_dev); + + return ret; +} + +static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev, + enum hns_roce_link_table_type type) +{ + struct hns_roce_cmq_desc desc[2]; + struct hns_roce_cfg_llm_a *req_a = + (struct hns_roce_cfg_llm_a *)desc[0].data; + struct hns_roce_cfg_llm_b *req_b = + (struct hns_roce_cfg_llm_b *)desc[1].data; + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hns_roce_link_table *link_tbl; + struct hns_roce_link_table_entry *entry; + enum hns_roce_opcode_type opcode; + u32 page_num; + int i; + + switch (type) { + case TSQ_LINK_TABLE: + link_tbl = &priv->tsq; + opcode = HNS_ROCE_OPC_CFG_EXT_LLM; + break; + case TPQ_LINK_TABLE: + link_tbl = &priv->tpq; + opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM; + break; + default: + return -EINVAL; + } + + page_num = link_tbl->npages; + entry = link_tbl->table.buf; + + for (i = 0; i < 2; i++) { + hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false); + + if (i == 0) + desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + else + desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); + } + + req_a->base_addr_l = cpu_to_le32(link_tbl->table.map & 0xffffffff); + req_a->base_addr_h = cpu_to_le32(link_tbl->table.map >> 32); + roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_DEPTH_M, + CFG_LLM_QUE_DEPTH_S, link_tbl->npages); + roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_PGSZ_M, + CFG_LLM_QUE_PGSZ_S, link_tbl->pg_sz); + roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_INIT_EN_M, + CFG_LLM_INIT_EN_S, 1); + req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0); + req_a->head_ba_h_nxtptr = cpu_to_le32(entry[0].blk_ba1_nxt_ptr); + roce_set_field(req_a->head_ptr, CFG_LLM_HEAD_PTR_M, CFG_LLM_HEAD_PTR_S, + 0); + + req_b->tail_ba_l = cpu_to_le32(entry[page_num - 1].blk_ba0); + roce_set_field(req_b->tail_ba_h, CFG_LLM_TAIL_BA_H_M, + CFG_LLM_TAIL_BA_H_S, + entry[page_num - 1].blk_ba1_nxt_ptr & + HNS_ROCE_LINK_TABLE_BA1_M); + roce_set_field(req_b->tail_ptr, CFG_LLM_TAIL_PTR_M, CFG_LLM_TAIL_PTR_S, + (entry[page_num - 2].blk_ba1_nxt_ptr & + HNS_ROCE_LINK_TABLE_NXT_PTR_M) >> + HNS_ROCE_LINK_TABLE_NXT_PTR_S); + + return hns_roce_cmq_send(hr_dev, desc, 2); +} + +static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev, + enum hns_roce_link_table_type type) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hns_roce_link_table *link_tbl; + struct hns_roce_link_table_entry *entry; + struct device *dev = hr_dev->dev; + u32 buf_chk_sz; + dma_addr_t t; + int func_num = 1; + int pg_num_a; + int pg_num_b; + int pg_num; + int size; + int i; + + switch (type) { + case TSQ_LINK_TABLE: + link_tbl = &priv->tsq; + buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT); + pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz; + pg_num_b = hr_dev->caps.sl_num * 4 + 2; + break; + case TPQ_LINK_TABLE: + link_tbl = &priv->tpq; + buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz + PAGE_SHIFT); + pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz; + pg_num_b = 2 * 4 * func_num + 2; + break; + default: + return -EINVAL; + } + + pg_num = max(pg_num_a, pg_num_b); + size = pg_num * sizeof(struct hns_roce_link_table_entry); + + link_tbl->table.buf = dma_alloc_coherent(dev, size, + &link_tbl->table.map, + GFP_KERNEL); + if (!link_tbl->table.buf) + goto out; + + link_tbl->pg_list = kcalloc(pg_num, sizeof(*link_tbl->pg_list), + GFP_KERNEL); + if (!link_tbl->pg_list) + goto err_kcalloc_failed; + + entry = link_tbl->table.buf; + for (i = 0; i < pg_num; ++i) { + link_tbl->pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz, + &t, GFP_KERNEL); + if (!link_tbl->pg_list[i].buf) + goto err_alloc_buf_failed; + + link_tbl->pg_list[i].map = t; + + entry[i].blk_ba0 = (u32)(t >> 12); + entry[i].blk_ba1_nxt_ptr = (u32)(t >> 44); + + if (i < (pg_num - 1)) + entry[i].blk_ba1_nxt_ptr |= + (i + 1) << HNS_ROCE_LINK_TABLE_NXT_PTR_S; + } + link_tbl->npages = pg_num; + link_tbl->pg_sz = buf_chk_sz; + + return hns_roce_config_link_table(hr_dev, type); + +err_alloc_buf_failed: + for (i -= 1; i >= 0; i--) + dma_free_coherent(dev, buf_chk_sz, + link_tbl->pg_list[i].buf, + link_tbl->pg_list[i].map); + kfree(link_tbl->pg_list); + +err_kcalloc_failed: + dma_free_coherent(dev, size, link_tbl->table.buf, + link_tbl->table.map); + +out: + return -ENOMEM; +} + +static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev, + struct hns_roce_link_table *link_tbl) +{ + struct device *dev = hr_dev->dev; + int size; + int i; + + size = link_tbl->npages * sizeof(struct hns_roce_link_table_entry); + + for (i = 0; i < link_tbl->npages; ++i) + if (link_tbl->pg_list[i].buf) + dma_free_coherent(dev, link_tbl->pg_sz, + link_tbl->pg_list[i].buf, + link_tbl->pg_list[i].map); + kfree(link_tbl->pg_list); + + dma_free_coherent(dev, size, link_tbl->table.buf, + link_tbl->table.map); +} + +static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + int qpc_count, cqc_count; + int ret, i; + + /* TSQ includes SQ doorbell and ack doorbell */ + ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE); + if (ret) { + dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret); + return ret; + } + + ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE); + if (ret) { + dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret); + goto err_tpq_init_failed; + } + + /* Alloc memory for QPC Timer buffer space chunk */ + for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num; + qpc_count++) { + ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table, + qpc_count); + if (ret) { + dev_err(hr_dev->dev, "QPC Timer get failed\n"); + goto err_qpc_timer_failed; + } + } + + /* Alloc memory for CQC Timer buffer space chunk */ + for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num; + cqc_count++) { + ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table, + cqc_count); + if (ret) { + dev_err(hr_dev->dev, "CQC Timer get failed\n"); + goto err_cqc_timer_failed; + } + } + + return 0; + +err_cqc_timer_failed: + for (i = 0; i < cqc_count; i++) + hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i); + +err_qpc_timer_failed: + for (i = 0; i < qpc_count; i++) + hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i); + + hns_roce_free_link_table(hr_dev, &priv->tpq); + +err_tpq_init_failed: + hns_roce_free_link_table(hr_dev, &priv->tsq); + + return ret; +} + +static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + + hns_roce_function_clear(hr_dev); + + hns_roce_free_link_table(hr_dev, &priv->tpq); + hns_roce_free_link_table(hr_dev, &priv->tsq); +} + +static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_cmq_desc desc; + struct hns_roce_mbox_status *mb_st = + (struct hns_roce_mbox_status *)desc.data; + enum hns_roce_cmd_return_status status; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true); + + status = hns_roce_cmq_send(hr_dev, &desc, 1); + if (status) + return status; + + return le32_to_cpu(mb_st->mb_status_hw_run); +} + +static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev) +{ + u32 status = hns_roce_query_mbox_status(hr_dev); + + return status >> HNS_ROCE_HW_RUN_BIT_SHIFT; +} + +static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev) +{ + u32 status = hns_roce_query_mbox_status(hr_dev); + + return status & HNS_ROCE_HW_MB_STATUS_MASK; +} + +static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, u32 in_modifier, u8 op_modifier, + u16 op, u16 token, int event) +{ + struct hns_roce_cmq_desc desc; + struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false); + + mb->in_param_l = cpu_to_le32(in_param); + mb->in_param_h = cpu_to_le32(in_param >> 32); + mb->out_param_l = cpu_to_le32(out_param); + mb->out_param_h = cpu_to_le32(out_param >> 32); + mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op); + mb->token_event_en = cpu_to_le32(event << 16 | token); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param, + u64 out_param, u32 in_modifier, u8 op_modifier, + u16 op, u16 token, int event) +{ + struct device *dev = hr_dev->dev; + unsigned long end; + int ret; + + end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies; + while (hns_roce_v2_cmd_pending(hr_dev)) { + if (time_after(jiffies, end)) { + dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies, + (int)end); + return -EAGAIN; + } + cond_resched(); + } + + ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier, + op_modifier, op, token, event); + if (ret) + dev_err(dev, "Post mailbox fail(%d)\n", ret); + + return ret; +} + +static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev, + unsigned long timeout) +{ + struct device *dev = hr_dev->dev; + unsigned long end; + u32 status; + + end = msecs_to_jiffies(timeout) + jiffies; + while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end)) + cond_resched(); + + if (hns_roce_v2_cmd_pending(hr_dev)) { + dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n"); + return -ETIMEDOUT; + } + + status = hns_roce_v2_cmd_complete(hr_dev); + if (status != 0x1) { + if (status == CMD_RST_PRC_EBUSY) + return status; + + dev_err(dev, "mailbox status 0x%x!\n", status); + return -EBUSY; + } + + return 0; +} + +static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev, + int gid_index, const union ib_gid *gid, + enum hns_roce_sgid_type sgid_type) +{ + struct hns_roce_cmq_desc desc; + struct hns_roce_cfg_sgid_tb *sgid_tb = + (struct hns_roce_cfg_sgid_tb *)desc.data; + u32 *p; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false); + + roce_set_field(sgid_tb->table_idx_rsv, CFG_SGID_TB_TABLE_IDX_M, + CFG_SGID_TB_TABLE_IDX_S, gid_index); + roce_set_field(sgid_tb->vf_sgid_type_rsv, CFG_SGID_TB_VF_SGID_TYPE_M, + CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type); + + p = (u32 *)&gid->raw[0]; + sgid_tb->vf_sgid_l = cpu_to_le32(*p); + + p = (u32 *)&gid->raw[4]; + sgid_tb->vf_sgid_ml = cpu_to_le32(*p); + + p = (u32 *)&gid->raw[8]; + sgid_tb->vf_sgid_mh = cpu_to_le32(*p); + + p = (u32 *)&gid->raw[0xc]; + sgid_tb->vf_sgid_h = cpu_to_le32(*p); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port, + int gid_index, const union ib_gid *gid, + const struct ib_gid_attr *attr) +{ + enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1; + int ret; + + if (!gid || !attr) + return -EINVAL; + + if (attr->gid_type == IB_GID_TYPE_ROCE) + sgid_type = GID_TYPE_FLAG_ROCE_V1; + + if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + if (ipv6_addr_v4mapped((void *)gid)) + sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4; + else + sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6; + } + + ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type); + if (ret) + ibdev_err(&hr_dev->ib_dev, + "failed to configure sgid table, ret = %d!\n", + ret); + + return ret; +} + +static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, + u8 *addr) +{ + struct hns_roce_cmq_desc desc; + struct hns_roce_cfg_smac_tb *smac_tb = + (struct hns_roce_cfg_smac_tb *)desc.data; + u16 reg_smac_h; + u32 reg_smac_l; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false); + + reg_smac_l = *(u32 *)(&addr[0]); + reg_smac_h = *(u16 *)(&addr[4]); + + roce_set_field(smac_tb->tb_idx_rsv, CFG_SMAC_TB_IDX_M, + CFG_SMAC_TB_IDX_S, phy_port); + roce_set_field(smac_tb->vf_smac_h_rsv, CFG_SMAC_TB_VF_SMAC_H_M, + CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h); + smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l); + + return hns_roce_cmq_send(hr_dev, &desc, 1); +} + +static int set_mtpt_pbl(struct hns_roce_dev *hr_dev, + struct hns_roce_v2_mpt_entry *mpt_entry, + struct hns_roce_mr *mr) +{ + u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 }; + struct ib_device *ibdev = &hr_dev->ib_dev; + dma_addr_t pbl_ba; + int i, count; + + count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages, + min_t(int, ARRAY_SIZE(pages), mr->npages), + &pbl_ba); + if (count < 1) { + ibdev_err(ibdev, "failed to find PBL mtr, count = %d.\n", + count); + return -ENOBUFS; + } + + /* Aligned to the hardware address access unit */ + for (i = 0; i < count; i++) + pages[i] >>= 6; + + mpt_entry->pbl_size = cpu_to_le32(mr->npages); + mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> 3); + roce_set_field(mpt_entry->byte_48_mode_ba, + V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S, + upper_32_bits(pbl_ba >> 3)); + + mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0])); + roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M, + V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0])); + + mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1])); + roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M, + V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1])); + roce_set_field(mpt_entry->byte_64_buf_pa1, + V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M, + V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, + to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift)); + + return 0; +} + +static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev, + void *mb_buf, struct hns_roce_mr *mr, + unsigned long mtpt_idx) +{ + struct hns_roce_v2_mpt_entry *mpt_entry; + int ret; + + mpt_entry = mb_buf; + memset(mpt_entry, 0, sizeof(*mpt_entry)); + + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, + V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID); + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M, + V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num == + HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num); + roce_set_field(mpt_entry->byte_4_pd_hop_st, + V2_MPT_BYTE_4_PBL_BA_PG_SZ_M, + V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, + to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift)); + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, + V2_MPT_BYTE_4_PD_S, mr->pd); + + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S, + (mr->access & IB_ACCESS_MW_BIND ? 1 : 0)); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S, + mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S, + (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0)); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S, + (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0)); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, + (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0)); + + roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, + mr->type == MR_TYPE_MR ? 0 : 1); + roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_INNER_PA_VLD_S, + 1); + + mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size)); + mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size)); + mpt_entry->lkey = cpu_to_le32(mr->key); + mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova)); + mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova)); + + if (mr->type == MR_TYPE_DMA) + return 0; + + ret = set_mtpt_pbl(hr_dev, mpt_entry, mr); + + return ret; +} + +static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev, + struct hns_roce_mr *mr, int flags, + u32 pdn, int mr_access_flags, u64 iova, + u64 size, void *mb_buf) +{ + struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf; + int ret = 0; + + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, + V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID); + + if (flags & IB_MR_REREG_PD) { + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, + V2_MPT_BYTE_4_PD_S, pdn); + mr->pd = pdn; + } + + if (flags & IB_MR_REREG_ACCESS) { + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, + V2_MPT_BYTE_8_BIND_EN_S, + (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0)); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, + V2_MPT_BYTE_8_ATOMIC_EN_S, + mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S, + mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S, + mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, + mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0); + } + + if (flags & IB_MR_REREG_TRANS) { + mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova)); + mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova)); + mpt_entry->len_l = cpu_to_le32(lower_32_bits(size)); + mpt_entry->len_h = cpu_to_le32(upper_32_bits(size)); + + mr->iova = iova; + mr->size = size; + + ret = set_mtpt_pbl(hr_dev, mpt_entry, mr); + } + + return ret; +} + +static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev, + void *mb_buf, struct hns_roce_mr *mr) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_v2_mpt_entry *mpt_entry; + dma_addr_t pbl_ba = 0; + + mpt_entry = mb_buf; + memset(mpt_entry, 0, sizeof(*mpt_entry)); + + if (hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, NULL, 0, &pbl_ba) < 0) { + ibdev_err(ibdev, "failed to find frmr mtr.\n"); + return -ENOBUFS; + } + + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, + V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE); + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M, + V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1); + roce_set_field(mpt_entry->byte_4_pd_hop_st, + V2_MPT_BYTE_4_PBL_BA_PG_SZ_M, + V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, + to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift)); + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, + V2_MPT_BYTE_4_PD_S, mr->pd); + + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1); + + roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1); + roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0); + roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0); + roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1); + + mpt_entry->pbl_size = cpu_to_le32(mr->npages); + + mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> 3)); + roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M, + V2_MPT_BYTE_48_PBL_BA_H_S, + upper_32_bits(pbl_ba >> 3)); + + roce_set_field(mpt_entry->byte_64_buf_pa1, + V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M, + V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, + to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift)); + + return 0; +} + +static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw) +{ + struct hns_roce_v2_mpt_entry *mpt_entry; + + mpt_entry = mb_buf; + memset(mpt_entry, 0, sizeof(*mpt_entry)); + + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M, + V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE); + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M, + V2_MPT_BYTE_4_PD_S, mw->pdn); + roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M, + V2_MPT_BYTE_4_PBL_HOP_NUM_S, + mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : + mw->pbl_hop_num); + roce_set_field(mpt_entry->byte_4_pd_hop_st, + V2_MPT_BYTE_4_PBL_BA_PG_SZ_M, + V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, + mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET); + + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1); + roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S, 1); + + roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0); + roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1); + roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1); + roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S, + mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1); + + roce_set_field(mpt_entry->byte_64_buf_pa1, + V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M, + V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, + mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET); + + mpt_entry->lkey = cpu_to_le32(mw->rkey); + + return 0; +} + +static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n) +{ + return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size); +} + +static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n) +{ + struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe); + + /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */ + return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^ + !!(n & hr_cq->cq_depth)) ? cqe : NULL; +} + +static inline void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 ci) +{ + *hr_cq->set_ci_db = ci & V2_CQ_DB_PARAMETER_CONS_IDX_M; +} + +static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, + struct hns_roce_srq *srq) +{ + struct hns_roce_v2_cqe *cqe, *dest; + u32 prod_index; + int nfreed = 0; + int wqe_index; + u8 owner_bit; + + for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index); + ++prod_index) { + if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe) + break; + } + + /* + * Now backwards through the CQ, removing CQ entries + * that match our QP by overwriting them with next entries. + */ + while ((int) --prod_index - (int) hr_cq->cons_index >= 0) { + cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe); + if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M, + V2_CQE_BYTE_16_LCL_QPN_S) & + HNS_ROCE_V2_CQE_QPN_MASK) == qpn) { + if (srq && + roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) { + wqe_index = roce_get_field(cqe->byte_4, + V2_CQE_BYTE_4_WQE_INDX_M, + V2_CQE_BYTE_4_WQE_INDX_S); + hns_roce_free_srq_wqe(srq, wqe_index); + } + ++nfreed; + } else if (nfreed) { + dest = get_cqe_v2(hr_cq, (prod_index + nfreed) & + hr_cq->ib_cq.cqe); + owner_bit = roce_get_bit(dest->byte_4, + V2_CQE_BYTE_4_OWNER_S); + memcpy(dest, cqe, sizeof(*cqe)); + roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S, + owner_bit); + } + } + + if (nfreed) { + hr_cq->cons_index += nfreed; + /* + * Make sure update of buffer contents is done before + * updating consumer index. + */ + wmb(); + hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index); + } +} + +static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, + struct hns_roce_srq *srq) +{ + spin_lock_irq(&hr_cq->lock); + __hns_roce_v2_cq_clean(hr_cq, qpn, srq); + spin_unlock_irq(&hr_cq->lock); +} + +static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, + struct hns_roce_cq *hr_cq, void *mb_buf, + u64 *mtts, dma_addr_t dma_handle) +{ + struct hns_roce_v2_cq_context *cq_context; + + cq_context = mb_buf; + memset(cq_context, 0, sizeof(*cq_context)); + + roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M, + V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID); + roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M, + V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE); + roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M, + V2_CQC_BYTE_4_SHIFT_S, ilog2(hr_cq->cq_depth)); + roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M, + V2_CQC_BYTE_4_CEQN_S, hr_cq->vector); + + roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M, + V2_CQC_BYTE_8_CQN_S, hr_cq->cqn); + + roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQE_SIZE_M, + V2_CQC_BYTE_8_CQE_SIZE_S, hr_cq->cqe_size == + HNS_ROCE_V3_CQE_SIZE ? 1 : 0); + + cq_context->cqe_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0])); + + roce_set_field(cq_context->byte_16_hop_addr, + V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M, + V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S, + upper_32_bits(to_hr_hw_page_addr(mtts[0]))); + roce_set_field(cq_context->byte_16_hop_addr, + V2_CQC_BYTE_16_CQE_HOP_NUM_M, + V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num == + HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num); + + cq_context->cqe_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1])); + roce_set_field(cq_context->byte_24_pgsz_addr, + V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M, + V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S, + upper_32_bits(to_hr_hw_page_addr(mtts[1]))); + roce_set_field(cq_context->byte_24_pgsz_addr, + V2_CQC_BYTE_24_CQE_BA_PG_SZ_M, + V2_CQC_BYTE_24_CQE_BA_PG_SZ_S, + to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift)); + roce_set_field(cq_context->byte_24_pgsz_addr, + V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M, + V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S, + to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift)); + + cq_context->cqe_ba = cpu_to_le32(dma_handle >> 3); + + roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M, + V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3))); + + roce_set_bit(cq_context->byte_44_db_record, + V2_CQC_BYTE_44_DB_RECORD_EN_S, + (hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB) ? 1 : 0); + + roce_set_field(cq_context->byte_44_db_record, + V2_CQC_BYTE_44_DB_RECORD_ADDR_M, + V2_CQC_BYTE_44_DB_RECORD_ADDR_S, + ((u32)hr_cq->db.dma) >> 1); + cq_context->db_record_addr = cpu_to_le32(hr_cq->db.dma >> 32); + + roce_set_field(cq_context->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_MAX_CNT_M, + V2_CQC_BYTE_56_CQ_MAX_CNT_S, + HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM); + roce_set_field(cq_context->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_PERIOD_M, + V2_CQC_BYTE_56_CQ_PERIOD_S, + HNS_ROCE_V2_CQ_DEFAULT_INTERVAL); +} + +static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, + enum ib_cq_notify_flags flags) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); + struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); + u32 notification_flag; + __le32 doorbell[2]; + + doorbell[0] = 0; + doorbell[1] = 0; + + notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? + V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL; + /* + * flags = 0; Notification Flag = 1, next + * flags = 1; Notification Flag = 0, solocited + */ + roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S, + hr_cq->cqn); + roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S, + HNS_ROCE_V2_CQ_DB_NTR); + roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M, + V2_CQ_DB_PARAMETER_CONS_IDX_S, hr_cq->cons_index); + roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M, + V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3); + roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S, + notification_flag); + + hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l); + + return 0; +} + +static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, + struct hns_roce_qp **cur_qp, + struct ib_wc *wc) +{ + struct hns_roce_rinl_sge *sge_list; + u32 wr_num, wr_cnt, sge_num; + u32 sge_cnt, data_len, size; + void *wqe_buf; + + wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M, + V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff; + wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1); + + sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list; + sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt; + wqe_buf = hns_roce_get_recv_wqe(*cur_qp, wr_cnt); + data_len = wc->byte_len; + + for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) { + size = min(sge_list[sge_cnt].len, data_len); + memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size); + + data_len -= size; + wqe_buf += size; + } + + if (unlikely(data_len)) { + wc->status = IB_WC_LOC_LEN_ERR; + return -EAGAIN; + } + + return 0; +} + +static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq, + int num_entries, struct ib_wc *wc) +{ + unsigned int left; + int npolled = 0; + + left = wq->head - wq->tail; + if (left == 0) + return 0; + + left = min_t(unsigned int, (unsigned int)num_entries, left); + while (npolled < left) { + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + wc->status = IB_WC_WR_FLUSH_ERR; + wc->vendor_err = 0; + wc->qp = &hr_qp->ibqp; + + wq->tail++; + wc++; + npolled++; + } + + return npolled; +} + +static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries, + struct ib_wc *wc) +{ + struct hns_roce_qp *hr_qp; + int npolled = 0; + + list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) { + npolled += sw_comp(hr_qp, &hr_qp->sq, + num_entries - npolled, wc + npolled); + if (npolled >= num_entries) + goto out; + } + + list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) { + npolled += sw_comp(hr_qp, &hr_qp->rq, + num_entries - npolled, wc + npolled); + if (npolled >= num_entries) + goto out; + } + +out: + return npolled; +} + +static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, + struct hns_roce_cq *cq, struct hns_roce_v2_cqe *cqe, + struct ib_wc *wc) +{ + static const struct { + u32 cqe_status; + enum ib_wc_status wc_status; + } map[] = { + { HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS }, + { HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR }, + { HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR }, + { HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR }, + { HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR }, + { HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR }, + { HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR }, + { HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR }, + { HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR }, + { HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR }, + { HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR }, + { HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR, + IB_WC_RETRY_EXC_ERR }, + { HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR }, + { HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR }, + { HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR} + }; + + u32 cqe_status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M, + V2_CQE_BYTE_4_STATUS_S); + int i; + + wc->status = IB_WC_GENERAL_ERR; + for (i = 0; i < ARRAY_SIZE(map); i++) + if (cqe_status == map[i].cqe_status) { + wc->status = map[i].wc_status; + break; + } + + if (likely(wc->status == IB_WC_SUCCESS || + wc->status == IB_WC_WR_FLUSH_ERR)) + return; + + ibdev_err(&hr_dev->ib_dev, "error cqe status 0x%x:\n", cqe_status); + print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe, + cq->cqe_size, false); + + /* + * For hns ROCEE, GENERAL_ERR is an error type that is not defined in + * the standard protocol, the driver must ignore it and needn't to set + * the QP to an error state. + */ + if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR) + return; + + /* + * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state gets + * into errored mode. Hence, as a workaround to this hardware + * limitation, driver needs to assist in flushing. But the flushing + * operation uses mailbox to convey the QP state to the hardware and + * which can sleep due to the mutex protection around the mailbox calls. + * Hence, use the deferred flush for now. Once wc error detected, the + * flushing operation is needed. + */ + if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) + init_flush_work(hr_dev, qp); +} + +static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, + struct hns_roce_qp **cur_qp, struct ib_wc *wc) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); + struct hns_roce_srq *srq = NULL; + struct hns_roce_v2_cqe *cqe; + struct hns_roce_qp *hr_qp; + struct hns_roce_wq *wq; + int is_send; + u16 wqe_ctr; + u32 opcode; + int qpn; + int ret; + + /* Find cqe according to consumer index */ + cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index); + if (!cqe) + return -EAGAIN; + + ++hr_cq->cons_index; + /* Memory barrier */ + rmb(); + + /* 0->SQ, 1->RQ */ + is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S); + + qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M, + V2_CQE_BYTE_16_LCL_QPN_S); + + if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) { + hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); + if (unlikely(!hr_qp)) { + ibdev_err(&hr_dev->ib_dev, + "CQ %06lx with entry for unknown QPN %06x\n", + hr_cq->cqn, qpn & HNS_ROCE_V2_CQE_QPN_MASK); + return -EINVAL; + } + *cur_qp = hr_qp; + } + + wc->qp = &(*cur_qp)->ibqp; + wc->vendor_err = 0; + + if (is_send) { + wq = &(*cur_qp)->sq; + if ((*cur_qp)->sq_signal_bits) { + /* + * If sg_signal_bit is 1, + * firstly tail pointer updated to wqe + * which current cqe correspond to + */ + wqe_ctr = (u16)roce_get_field(cqe->byte_4, + V2_CQE_BYTE_4_WQE_INDX_M, + V2_CQE_BYTE_4_WQE_INDX_S); + wq->tail += (wqe_ctr - (u16)wq->tail) & + (wq->wqe_cnt - 1); + } + + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; + } else if ((*cur_qp)->ibqp.srq) { + srq = to_hr_srq((*cur_qp)->ibqp.srq); + wqe_ctr = (u16)roce_get_field(cqe->byte_4, + V2_CQE_BYTE_4_WQE_INDX_M, + V2_CQE_BYTE_4_WQE_INDX_S); + wc->wr_id = srq->wrid[wqe_ctr]; + hns_roce_free_srq_wqe(srq, wqe_ctr); + } else { + /* Update tail pointer, record wr_id */ + wq = &(*cur_qp)->rq; + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; + } + + get_cqe_status(hr_dev, *cur_qp, hr_cq, cqe, wc); + if (unlikely(wc->status != IB_WC_SUCCESS)) + return 0; + + if (is_send) { + wc->wc_flags = 0; + /* SQ corresponding to CQE */ + switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, + V2_CQE_BYTE_4_OPCODE_S) & 0x1f) { + case HNS_ROCE_V2_WQE_OP_SEND: + wc->opcode = IB_WC_SEND; + break; + case HNS_ROCE_V2_WQE_OP_SEND_WITH_INV: + wc->opcode = IB_WC_SEND; + break; + case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM: + wc->opcode = IB_WC_SEND; + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case HNS_ROCE_V2_WQE_OP_RDMA_READ: + wc->opcode = IB_WC_RDMA_READ; + wc->byte_len = le32_to_cpu(cqe->byte_cnt); + break; + case HNS_ROCE_V2_WQE_OP_RDMA_WRITE: + wc->opcode = IB_WC_RDMA_WRITE; + break; + case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RDMA_WRITE; + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case HNS_ROCE_V2_WQE_OP_LOCAL_INV: + wc->opcode = IB_WC_LOCAL_INV; + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + break; + case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP: + wc->opcode = IB_WC_COMP_SWAP; + wc->byte_len = 8; + break; + case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD: + wc->opcode = IB_WC_FETCH_ADD; + wc->byte_len = 8; + break; + case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP: + wc->opcode = IB_WC_MASKED_COMP_SWAP; + wc->byte_len = 8; + break; + case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD: + wc->opcode = IB_WC_MASKED_FETCH_ADD; + wc->byte_len = 8; + break; + case HNS_ROCE_V2_WQE_OP_FAST_REG_PMR: + wc->opcode = IB_WC_REG_MR; + break; + case HNS_ROCE_V2_WQE_OP_BIND_MW: + wc->opcode = IB_WC_REG_MR; + break; + default: + wc->status = IB_WC_GENERAL_ERR; + break; + } + } else { + /* RQ correspond to CQE */ + wc->byte_len = le32_to_cpu(cqe->byte_cnt); + + opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M, + V2_CQE_BYTE_4_OPCODE_S); + switch (opcode & 0x1f) { + case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM: + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; + wc->wc_flags = IB_WC_WITH_IMM; + wc->ex.imm_data = + cpu_to_be32(le32_to_cpu(cqe->immtdata)); + break; + case HNS_ROCE_V2_OPCODE_SEND: + wc->opcode = IB_WC_RECV; + wc->wc_flags = 0; + break; + case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_IMM; + wc->ex.imm_data = + cpu_to_be32(le32_to_cpu(cqe->immtdata)); + break; + case HNS_ROCE_V2_OPCODE_SEND_WITH_INV: + wc->opcode = IB_WC_RECV; + wc->wc_flags = IB_WC_WITH_INVALIDATE; + wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey); + break; + default: + wc->status = IB_WC_GENERAL_ERR; + break; + } + + if ((wc->qp->qp_type == IB_QPT_RC || + wc->qp->qp_type == IB_QPT_UC) && + (opcode == HNS_ROCE_V2_OPCODE_SEND || + opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM || + opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) && + (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) { + ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc); + if (unlikely(ret)) + return -EAGAIN; + } + + wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M, + V2_CQE_BYTE_32_SL_S); + wc->src_qp = (u8)roce_get_field(cqe->byte_32, + V2_CQE_BYTE_32_RMT_QPN_M, + V2_CQE_BYTE_32_RMT_QPN_S); + wc->slid = 0; + wc->wc_flags |= (roce_get_bit(cqe->byte_32, + V2_CQE_BYTE_32_GRH_S) ? + IB_WC_GRH : 0); + wc->port_num = roce_get_field(cqe->byte_32, + V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S); + wc->pkey_index = 0; + + if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) { + wc->vlan_id = (u16)roce_get_field(cqe->byte_28, + V2_CQE_BYTE_28_VID_M, + V2_CQE_BYTE_28_VID_S); + wc->wc_flags |= IB_WC_WITH_VLAN; + } else { + wc->vlan_id = 0xffff; + } + + wc->network_hdr_type = roce_get_field(cqe->byte_28, + V2_CQE_BYTE_28_PORT_TYPE_M, + V2_CQE_BYTE_28_PORT_TYPE_S); + } + + return 0; +} + +static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, + struct ib_wc *wc) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); + struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); + struct hns_roce_qp *cur_qp = NULL; + unsigned long flags; + int npolled; + + spin_lock_irqsave(&hr_cq->lock, flags); + + /* + * When the device starts to reset, the state is RST_DOWN. At this time, + * there may still be some valid CQEs in the hardware that are not + * polled. Therefore, it is not allowed to switch to the software mode + * immediately. When the state changes to UNINIT, CQE no longer exists + * in the hardware, and then switch to software mode. + */ + if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) { + npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc); + goto out; + } + + for (npolled = 0; npolled < num_entries; ++npolled) { + if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled)) + break; + } + + if (npolled) { + /* Memory barrier */ + wmb(); + hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index); + } + +out: + spin_unlock_irqrestore(&hr_cq->lock, flags); + + return npolled; +} + +static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, + int step_idx) +{ + int op; + + if (type == HEM_TYPE_SCCC && step_idx) + return -EINVAL; + + switch (type) { + case HEM_TYPE_QPC: + op = HNS_ROCE_CMD_WRITE_QPC_BT0; + break; + case HEM_TYPE_MTPT: + op = HNS_ROCE_CMD_WRITE_MPT_BT0; + break; + case HEM_TYPE_CQC: + op = HNS_ROCE_CMD_WRITE_CQC_BT0; + break; + case HEM_TYPE_SRQC: + op = HNS_ROCE_CMD_WRITE_SRQC_BT0; + break; + case HEM_TYPE_SCCC: + op = HNS_ROCE_CMD_WRITE_SCCC_BT0; + break; + case HEM_TYPE_QPC_TIMER: + op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0; + break; + case HEM_TYPE_CQC_TIMER: + op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0; + break; + default: + dev_warn(hr_dev->dev, + "table %u not to be written by mailbox!\n", type); + return -EINVAL; + } + + return op + step_idx; +} + +static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, u64 bt_ba, + u32 hem_type, int step_idx) +{ + struct hns_roce_cmd_mailbox *mailbox; + int ret; + int op; + + op = get_op_for_set_hem(hr_dev, hem_type, step_idx); + if (op < 0) + return 0; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj, + 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS); + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} + +static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, int obj, + int step_idx) +{ + struct hns_roce_hem_iter iter; + struct hns_roce_hem_mhop mhop; + struct hns_roce_hem *hem; + unsigned long mhop_obj = obj; + int i, j, k; + int ret = 0; + u64 hem_idx = 0; + u64 l1_idx = 0; + u64 bt_ba = 0; + u32 chunk_ba_num; + u32 hop_num; + + if (!hns_roce_check_whether_mhop(hr_dev, table->type)) + return 0; + + hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); + i = mhop.l0_idx; + j = mhop.l1_idx; + k = mhop.l2_idx; + hop_num = mhop.hop_num; + chunk_ba_num = mhop.bt_chunk_size / 8; + + if (hop_num == 2) { + hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num + + k; + l1_idx = i * chunk_ba_num + j; + } else if (hop_num == 1) { + hem_idx = i * chunk_ba_num + j; + } else if (hop_num == HNS_ROCE_HOP_NUM_0) { + hem_idx = i; + } + + if (table->type == HEM_TYPE_SCCC) + obj = mhop.l0_idx; + + if (check_whether_last_step(hop_num, step_idx)) { + hem = table->hem[hem_idx]; + for (hns_roce_hem_first(hem, &iter); + !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { + bt_ba = hns_roce_hem_addr(&iter); + ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, + step_idx); + } + } else { + if (step_idx == 0) + bt_ba = table->bt_l0_dma_addr[i]; + else if (step_idx == 1 && hop_num == 2) + bt_ba = table->bt_l1_dma_addr[l1_idx]; + + ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx); + } + + return ret; +} + +static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_table *table, int obj, + int step_idx) +{ + struct device *dev = hr_dev->dev; + struct hns_roce_cmd_mailbox *mailbox; + int ret; + u16 op = 0xff; + + if (!hns_roce_check_whether_mhop(hr_dev, table->type)) + return 0; + + switch (table->type) { + case HEM_TYPE_QPC: + op = HNS_ROCE_CMD_DESTROY_QPC_BT0; + break; + case HEM_TYPE_MTPT: + op = HNS_ROCE_CMD_DESTROY_MPT_BT0; + break; + case HEM_TYPE_CQC: + op = HNS_ROCE_CMD_DESTROY_CQC_BT0; + break; + case HEM_TYPE_SCCC: + case HEM_TYPE_QPC_TIMER: + case HEM_TYPE_CQC_TIMER: + break; + case HEM_TYPE_SRQC: + op = HNS_ROCE_CMD_DESTROY_SRQC_BT0; + break; + default: + dev_warn(dev, "table %u not to be destroyed by mailbox!\n", + table->type); + return 0; + } + + if (table->type == HEM_TYPE_SCCC || + table->type == HEM_TYPE_QPC_TIMER || + table->type == HEM_TYPE_CQC_TIMER) + return 0; + + op += step_idx; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + /* configure the tag and op */ + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op, + HNS_ROCE_CMD_TIMEOUT_MSECS); + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + return ret; +} + +static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask, + struct hns_roce_qp *hr_qp) +{ + struct hns_roce_cmd_mailbox *mailbox; + int qpc_size; + int ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + /* The qpc size of HIP08 is only 256B, which is half of HIP09 */ + qpc_size = hr_dev->caps.qpc_sz; + memcpy(mailbox->buf, context, qpc_size); + memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size); + + ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0, + HNS_ROCE_CMD_MODIFY_QPC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} + +static void set_access_flags(struct hns_roce_qp *hr_qp, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask, + const struct ib_qp_attr *attr, int attr_mask) +{ + u8 dest_rd_atomic; + u32 access_flags; + + dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ? + attr->max_dest_rd_atomic : hr_qp->resp_depth; + + access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ? + attr->qp_access_flags : hr_qp->atomic_rd_en; + + if (!dest_rd_atomic) + access_flags &= IB_ACCESS_REMOTE_WRITE; + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, + !!(access_flags & IB_ACCESS_REMOTE_READ)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, + !!(access_flags & IB_ACCESS_REMOTE_WRITE)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, + !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0); + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S, + !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S, 0); +} + +static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + roce_set_field(context->byte_4_sqpn_tst, + V2_QPC_BYTE_4_SGE_SHIFT_M, V2_QPC_BYTE_4_SGE_SHIFT_S, + to_hr_hem_entries_shift(hr_qp->sge.sge_cnt, + hr_qp->sge.sge_shift)); + + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, + ilog2(hr_qp->sq.wqe_cnt)); + + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, + ilog2(hr_qp->rq.wqe_cnt)); +} + +static void modify_qp_reset_to_init(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, + int attr_mask, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, + V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type)); + + roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, + V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn); + + roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, + V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn); + + roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M, + V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs)); + + set_qpc_wqe_cnt(hr_qp, context, qpc_mask); + + /* No VLAN need to set 0xFFF */ + roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M, + V2_QPC_BYTE_24_VLAN_ID_S, 0xfff); + + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) + roce_set_bit(context->byte_68_rq_db, + V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1); + + roce_set_field(context->byte_68_rq_db, + V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M, + V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, + ((u32)hr_qp->rdb.dma) >> 1); + context->rq_db_record_addr = cpu_to_le32(hr_qp->rdb.dma >> 32); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, + (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0); + + roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, + V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn); + if (ibqp->srq) { + roce_set_field(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, + to_hr_srq(ibqp->srq)->srqn); + roce_set_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQ_EN_S, 1); + } + + roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1); + + hr_qp->access_flags = attr->qp_access_flags; + roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, + V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn); +} + +static void modify_qp_init_to_init(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, + V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type)); + roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M, + V2_QPC_BYTE_4_TST_S, 0); + + if (attr_mask & IB_QP_ACCESS_FLAGS) { + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, + !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, + 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, + !!(attr->qp_access_flags & + IB_ACCESS_REMOTE_WRITE)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, + 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, + !!(attr->qp_access_flags & + IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, + 0); + roce_set_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_EXT_ATE_S, + !!(attr->qp_access_flags & + IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, + V2_QPC_BYTE_76_EXT_ATE_S, 0); + } else { + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, + !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, + 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, + !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, + 0); + + roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, + !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, + 0); + roce_set_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_EXT_ATE_S, + !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC)); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, + V2_QPC_BYTE_76_EXT_ATE_S, 0); + } + + roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, + V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn); + roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M, + V2_QPC_BYTE_16_PD_S, 0); + + roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, + V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn); + roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M, + V2_QPC_BYTE_80_RX_CQN_S, 0); + + roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, + V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn); + roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M, + V2_QPC_BYTE_252_TX_CQN_S, 0); + + if (ibqp->srq) { + roce_set_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQ_EN_S, 1); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQ_EN_S, 0); + roce_set_field(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, + to_hr_srq(ibqp->srq)->srqn); + roce_set_field(qpc_mask->byte_76_srqn_op_en, + V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0); + } + + roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, + V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn); + roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M, + V2_QPC_BYTE_4_SQPN_S, 0); + + if (attr_mask & IB_QP_DEST_QPN) { + roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M, + V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn); + roce_set_field(qpc_mask->byte_56_dqpn_err, + V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0); + } +} + +static int config_qp_rq_buf(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + u64 mtts[MTT_MIN_COUNT] = { 0 }; + u64 wqe_sge_ba; + int count; + + /* Search qp buf's mtts */ + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts, + MTT_MIN_COUNT, &wqe_sge_ba); + if (hr_qp->rq.wqe_cnt && count < 1) { + ibdev_err(&hr_dev->ib_dev, + "failed to find RQ WQE, QPN = 0x%lx.\n", hr_qp->qpn); + return -EINVAL; + } + + context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3); + qpc_mask->wqe_sge_ba = 0; + + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, + V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3)); + roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, + V2_QPC_BYTE_12_WQE_SGE_BA_S, 0); + + roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, + V2_QPC_BYTE_12_SQ_HOP_NUM_S, + to_hr_hem_hopnum(hr_dev->caps.wqe_sq_hop_num, + hr_qp->sq.wqe_cnt)); + roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, + V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0); + + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGE_HOP_NUM_M, + V2_QPC_BYTE_20_SGE_HOP_NUM_S, + to_hr_hem_hopnum(hr_dev->caps.wqe_sge_hop_num, + hr_qp->sge.sge_cnt)); + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGE_HOP_NUM_M, + V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0); + + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_RQ_HOP_NUM_M, + V2_QPC_BYTE_20_RQ_HOP_NUM_S, + to_hr_hem_hopnum(hr_dev->caps.wqe_rq_hop_num, + hr_qp->rq.wqe_cnt)); + + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_RQ_HOP_NUM_M, + V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0); + + roce_set_field(context->byte_16_buf_ba_pg_sz, + V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, + V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, + to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.ba_pg_shift)); + roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, + V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, + V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0); + + roce_set_field(context->byte_16_buf_ba_pg_sz, + V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M, + V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, + to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.buf_pg_shift)); + roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, + V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M, + V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0); + + context->rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0])); + qpc_mask->rq_cur_blk_addr = 0; + + roce_set_field(context->byte_92_srq_info, + V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, + upper_32_bits(to_hr_hw_page_addr(mtts[0]))); + roce_set_field(qpc_mask->byte_92_srq_info, + V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0); + + context->rq_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1])); + qpc_mask->rq_nxt_blk_addr = 0; + + roce_set_field(context->byte_104_rq_sge, + V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, + V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, + upper_32_bits(to_hr_hw_page_addr(mtts[1]))); + roce_set_field(qpc_mask->byte_104_rq_sge, + V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, + V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0); + + roce_set_field(context->byte_84_rq_ci_pi, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head); + roce_set_field(qpc_mask->byte_84_rq_ci_pi, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0); + + roce_set_field(qpc_mask->byte_84_rq_ci_pi, + V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M, + V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0); + + return 0; +} + +static int config_qp_sq_buf(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + u64 sge_cur_blk = 0; + u64 sq_cur_blk = 0; + int count; + + /* search qp buf's mtts */ + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL); + if (count < 1) { + ibdev_err(ibdev, "failed to find QP(0x%lx) SQ buf.\n", + hr_qp->qpn); + return -EINVAL; + } + if (hr_qp->sge.sge_cnt > 0) { + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, + hr_qp->sge.offset, + &sge_cur_blk, 1, NULL); + if (count < 1) { + ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n", + hr_qp->qpn); + return -EINVAL; + } + } + + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + context->sq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(sq_cur_blk)); + roce_set_field(context->byte_168_irrl_idx, + V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, + upper_32_bits(to_hr_hw_page_addr(sq_cur_blk))); + qpc_mask->sq_cur_blk_addr = 0; + roce_set_field(qpc_mask->byte_168_irrl_idx, + V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0); + + context->sq_cur_sge_blk_addr = + cpu_to_le32(to_hr_hw_page_addr(sge_cur_blk)); + roce_set_field(context->byte_184_irrl_idx, + V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, + V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, + upper_32_bits(to_hr_hw_page_addr(sge_cur_blk))); + qpc_mask->sq_cur_sge_blk_addr = 0; + roce_set_field(qpc_mask->byte_184_irrl_idx, + V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, + V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0); + + context->rx_sq_cur_blk_addr = + cpu_to_le32(to_hr_hw_page_addr(sq_cur_blk)); + roce_set_field(context->byte_232_irrl_sge, + V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, + upper_32_bits(to_hr_hw_page_addr(sq_cur_blk))); + qpc_mask->rx_sq_cur_blk_addr = 0; + roce_set_field(qpc_mask->byte_232_irrl_sge, + V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, + V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0); + + return 0; +} + +static inline enum ib_mtu get_mtu(struct ib_qp *ibqp, + const struct ib_qp_attr *attr) +{ + if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD) + return IB_MTU_4096; + + return attr->path_mtu; +} + +static int modify_qp_init_to_rtr(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct ib_device *ibdev = &hr_dev->ib_dev; + dma_addr_t trrl_ba; + dma_addr_t irrl_ba; + enum ib_mtu mtu; + u8 lp_pktn_ini; + u8 port_num; + u64 *mtts; + u8 *dmac; + u8 *smac; + int port; + int ret; + + ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask); + if (ret) { + ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret); + return ret; + } + + /* Search IRRL's mtts */ + mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, + hr_qp->qpn, &irrl_ba); + if (!mtts) { + ibdev_err(ibdev, "failed to find qp irrl_table.\n"); + return -EINVAL; + } + + /* Search TRRL's mtts */ + mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table, + hr_qp->qpn, &trrl_ba); + if (!mtts) { + ibdev_err(ibdev, "failed to find qp trrl_table.\n"); + return -EINVAL; + } + + if (attr_mask & IB_QP_ALT_PATH) { + ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n", + attr_mask); + return -EINVAL; + } + + roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M, + V2_QPC_BYTE_132_TRRL_BA_S, trrl_ba >> 4); + roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M, + V2_QPC_BYTE_132_TRRL_BA_S, 0); + context->trrl_ba = cpu_to_le32(trrl_ba >> (16 + 4)); + qpc_mask->trrl_ba = 0; + roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M, + V2_QPC_BYTE_140_TRRL_BA_S, + (u32)(trrl_ba >> (32 + 16 + 4))); + roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M, + V2_QPC_BYTE_140_TRRL_BA_S, 0); + + context->irrl_ba = cpu_to_le32(irrl_ba >> 6); + qpc_mask->irrl_ba = 0; + roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M, + V2_QPC_BYTE_208_IRRL_BA_S, + irrl_ba >> (32 + 6)); + roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M, + V2_QPC_BYTE_208_IRRL_BA_S, 0); + + roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1); + roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0); + + roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S, + hr_qp->sq_signal_bits); + roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S, + 0); + + port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port; + + smac = (u8 *)hr_dev->dev_addr[port]; + dmac = (u8 *)attr->ah_attr.roce.dmac; + /* when dmac equals smac or loop_idc is 1, it should loopback */ + if (ether_addr_equal_unaligned(dmac, smac) || + hr_dev->loop_idc == 0x1) { + roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1); + roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0); + } + + if (attr_mask & IB_QP_DEST_QPN) { + roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M, + V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num); + roce_set_field(qpc_mask->byte_56_dqpn_err, + V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0); + } + + /* Configure GID index */ + port_num = rdma_ah_get_port_num(&attr->ah_attr); + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, + hns_get_gid_index(hr_dev, port_num - 1, + grh->sgid_index)); + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0); + + memcpy(&(context->dmac), dmac, sizeof(u32)); + roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, + V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4]))); + qpc_mask->dmac = 0; + roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, + V2_QPC_BYTE_52_DMAC_S, 0); + + mtu = get_mtu(ibqp, attr); + hr_qp->path_mtu = mtu; + + if (attr_mask & IB_QP_PATH_MTU) { + roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M, + V2_QPC_BYTE_24_MTU_S, mtu); + roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M, + V2_QPC_BYTE_24_MTU_S, 0); + } + +#define MAX_LP_MSG_LEN 65536 + /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 64KB */ + lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / ib_mtu_enum_to_int(mtu)); + + roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, + V2_QPC_BYTE_56_LP_PKTN_INI_S, lp_pktn_ini); + roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, + V2_QPC_BYTE_56_LP_PKTN_INI_S, 0); + + /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */ + roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M, + V2_QPC_BYTE_172_ACK_REQ_FREQ_S, lp_pktn_ini); + roce_set_field(qpc_mask->byte_172_sq_psn, + V2_QPC_BYTE_172_ACK_REQ_FREQ_M, + V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0); + + roce_set_bit(qpc_mask->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0); + roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M, + V2_QPC_BYTE_96_RX_REQ_MSN_S, 0); + roce_set_field(qpc_mask->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M, + V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0); + + context->rq_rnr_timer = 0; + qpc_mask->rq_rnr_timer = 0; + + roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M, + V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0); + roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M, + V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0); + + /* rocee send 2^lp_sgen_ini segs every time */ + roce_set_field(context->byte_168_irrl_idx, + V2_QPC_BYTE_168_LP_SGEN_INI_M, + V2_QPC_BYTE_168_LP_SGEN_INI_S, 3); + roce_set_field(qpc_mask->byte_168_irrl_idx, + V2_QPC_BYTE_168_LP_SGEN_INI_M, + V2_QPC_BYTE_168_LP_SGEN_INI_S, 0); + + return 0; +} + +static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct ib_device *ibdev = &hr_dev->ib_dev; + int ret; + + /* Not support alternate path and path migration */ + if (attr_mask & (IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE)) { + ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask); + return -EINVAL; + } + + ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask); + if (ret) { + ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret); + return ret; + } + + /* + * Set some fields in context to zero, Because the default values + * of all fields in context are zero, we need not set them to 0 again. + * but we should set the relevant fields of context mask to 0. + */ + roce_set_field(qpc_mask->byte_232_irrl_sge, + V2_QPC_BYTE_232_IRRL_SGE_IDX_M, + V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0); + + roce_set_field(qpc_mask->byte_240_irrl_tail, + V2_QPC_BYTE_240_RX_ACK_MSN_M, + V2_QPC_BYTE_240_RX_ACK_MSN_S, 0); + + roce_set_field(qpc_mask->byte_248_ack_psn, + V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M, + V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0); + roce_set_bit(qpc_mask->byte_248_ack_psn, + V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0); + roce_set_field(qpc_mask->byte_248_ack_psn, + V2_QPC_BYTE_248_IRRL_PSN_M, + V2_QPC_BYTE_248_IRRL_PSN_S, 0); + + roce_set_field(qpc_mask->byte_240_irrl_tail, + V2_QPC_BYTE_240_IRRL_TAIL_REAL_M, + V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0); + + roce_set_field(qpc_mask->byte_220_retry_psn_msn, + V2_QPC_BYTE_220_RETRY_MSG_MSN_M, + V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0); + + roce_set_bit(qpc_mask->byte_248_ack_psn, + V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0); + + roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M, + V2_QPC_BYTE_212_CHECK_FLG_S, 0); + + roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M, + V2_QPC_BYTE_212_LSN_S, 0x100); + roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M, + V2_QPC_BYTE_212_LSN_S, 0); + + roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M, + V2_QPC_BYTE_196_IRRL_HEAD_S, 0); + + return 0; +} + +static inline u16 get_udp_sport(u32 fl, u32 lqpn, u32 rqpn) +{ + if (!fl) + fl = rdma_calc_flow_label(lqpn, rqpn); + + return rdma_flow_label_to_udp_sport(fl); +} + +static int hns_roce_v2_set_path(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, + int attr_mask, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct ib_device *ibdev = &hr_dev->ib_dev; + const struct ib_gid_attr *gid_attr = NULL; + int is_roce_protocol; + u16 vlan_id = 0xffff; + bool is_udp = false; + u8 ib_port; + u8 hr_port; + int ret; + + ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1; + hr_port = ib_port - 1; + is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) && + rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH; + + if (is_roce_protocol) { + gid_attr = attr->ah_attr.grh.sgid_attr; + ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL); + if (ret) + return ret; + + if (gid_attr) + is_udp = (gid_attr->gid_type == + IB_GID_TYPE_ROCE_UDP_ENCAP); + } + + if (vlan_id < VLAN_N_VID) { + roce_set_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1); + roce_set_bit(qpc_mask->byte_76_srqn_op_en, + V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0); + roce_set_bit(context->byte_168_irrl_idx, + V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1); + roce_set_bit(qpc_mask->byte_168_irrl_idx, + V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0); + } + + roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M, + V2_QPC_BYTE_24_VLAN_ID_S, vlan_id); + roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M, + V2_QPC_BYTE_24_VLAN_ID_S, 0); + + if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) { + ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n", + grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]); + return -EINVAL; + } + + if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) { + ibdev_err(ibdev, "ah attr is not RDMA roce type\n"); + return -EINVAL; + } + + roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M, + V2_QPC_BYTE_52_UDPSPN_S, + is_udp ? get_udp_sport(grh->flow_label, ibqp->qp_num, + attr->dest_qp_num) : 0); + + roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M, + V2_QPC_BYTE_52_UDPSPN_S, 0); + + roce_set_field(context->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, + grh->sgid_index); + + roce_set_field(qpc_mask->byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0); + + roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M, + V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit); + roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M, + V2_QPC_BYTE_24_HOP_LIMIT_S, 0); + + roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, + V2_QPC_BYTE_24_TC_S, get_tclass(&attr->ah_attr.grh)); + roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M, + V2_QPC_BYTE_24_TC_S, 0); + + roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M, + V2_QPC_BYTE_28_FL_S, grh->flow_label); + roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M, + V2_QPC_BYTE_28_FL_S, 0); + memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); + memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw)); + + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + if (unlikely(hr_qp->sl > MAX_SERVICE_LEVEL)) { + ibdev_err(ibdev, + "failed to fill QPC, sl (%d) shouldn't be larger than %d.\n", + hr_qp->sl, MAX_SERVICE_LEVEL); + return -EINVAL; + } + + roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, + V2_QPC_BYTE_28_SL_S, hr_qp->sl); + roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M, + V2_QPC_BYTE_28_SL_S, 0); + + return 0; +} + +static bool check_qp_state(enum ib_qp_state cur_state, + enum ib_qp_state new_state) +{ + static const bool sm[][IB_QPS_ERR + 1] = { + [IB_QPS_RESET] = { [IB_QPS_RESET] = true, + [IB_QPS_INIT] = true }, + [IB_QPS_INIT] = { [IB_QPS_RESET] = true, + [IB_QPS_INIT] = true, + [IB_QPS_RTR] = true, + [IB_QPS_ERR] = true }, + [IB_QPS_RTR] = { [IB_QPS_RESET] = true, + [IB_QPS_RTS] = true, + [IB_QPS_ERR] = true }, + [IB_QPS_RTS] = { [IB_QPS_RESET] = true, + [IB_QPS_RTS] = true, + [IB_QPS_ERR] = true }, + [IB_QPS_SQD] = {}, + [IB_QPS_SQE] = {}, + [IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true } + }; + + return sm[cur_state][new_state]; +} + +static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, + int attr_mask, + enum ib_qp_state cur_state, + enum ib_qp_state new_state, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + int ret = 0; + + if (!check_qp_state(cur_state, new_state)) { + ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n"); + return -EINVAL; + } + + if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { + memset(qpc_mask, 0, hr_dev->caps.qpc_sz); + modify_qp_reset_to_init(ibqp, attr, attr_mask, context, + qpc_mask); + } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { + modify_qp_init_to_init(ibqp, attr, attr_mask, context, + qpc_mask); + } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { + ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context, + qpc_mask); + } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) { + ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context, + qpc_mask); + } + + return ret; +} + +static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, + int attr_mask, + struct hns_roce_v2_qp_context *context, + struct hns_roce_v2_qp_context *qpc_mask) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + int ret = 0; + + if (attr_mask & IB_QP_AV) { + ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context, + qpc_mask); + if (ret) + return ret; + } + + if (attr_mask & IB_QP_TIMEOUT) { + if (attr->timeout < 31) { + roce_set_field(context->byte_28_at_fl, + V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S, + attr->timeout); + roce_set_field(qpc_mask->byte_28_at_fl, + V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S, + 0); + } else { + ibdev_warn(&hr_dev->ib_dev, + "Local ACK timeout shall be 0 to 30.\n"); + } + } + + if (attr_mask & IB_QP_RETRY_CNT) { + roce_set_field(context->byte_212_lsn, + V2_QPC_BYTE_212_RETRY_NUM_INIT_M, + V2_QPC_BYTE_212_RETRY_NUM_INIT_S, + attr->retry_cnt); + roce_set_field(qpc_mask->byte_212_lsn, + V2_QPC_BYTE_212_RETRY_NUM_INIT_M, + V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0); + + roce_set_field(context->byte_212_lsn, + V2_QPC_BYTE_212_RETRY_CNT_M, + V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt); + roce_set_field(qpc_mask->byte_212_lsn, + V2_QPC_BYTE_212_RETRY_CNT_M, + V2_QPC_BYTE_212_RETRY_CNT_S, 0); + } + + if (attr_mask & IB_QP_RNR_RETRY) { + roce_set_field(context->byte_244_rnr_rxack, + V2_QPC_BYTE_244_RNR_NUM_INIT_M, + V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry); + roce_set_field(qpc_mask->byte_244_rnr_rxack, + V2_QPC_BYTE_244_RNR_NUM_INIT_M, + V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0); + + roce_set_field(context->byte_244_rnr_rxack, + V2_QPC_BYTE_244_RNR_CNT_M, + V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry); + roce_set_field(qpc_mask->byte_244_rnr_rxack, + V2_QPC_BYTE_244_RNR_CNT_M, + V2_QPC_BYTE_244_RNR_CNT_S, 0); + } + + /* RC&UC&UD required attr */ + if (attr_mask & IB_QP_SQ_PSN) { + roce_set_field(context->byte_172_sq_psn, + V2_QPC_BYTE_172_SQ_CUR_PSN_M, + V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn); + roce_set_field(qpc_mask->byte_172_sq_psn, + V2_QPC_BYTE_172_SQ_CUR_PSN_M, + V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0); + + roce_set_field(context->byte_196_sq_psn, + V2_QPC_BYTE_196_SQ_MAX_PSN_M, + V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn); + roce_set_field(qpc_mask->byte_196_sq_psn, + V2_QPC_BYTE_196_SQ_MAX_PSN_M, + V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0); + + roce_set_field(context->byte_220_retry_psn_msn, + V2_QPC_BYTE_220_RETRY_MSG_PSN_M, + V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn); + roce_set_field(qpc_mask->byte_220_retry_psn_msn, + V2_QPC_BYTE_220_RETRY_MSG_PSN_M, + V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0); + + roce_set_field(context->byte_224_retry_msg, + V2_QPC_BYTE_224_RETRY_MSG_PSN_M, + V2_QPC_BYTE_224_RETRY_MSG_PSN_S, + attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S); + roce_set_field(qpc_mask->byte_224_retry_msg, + V2_QPC_BYTE_224_RETRY_MSG_PSN_M, + V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0); + + roce_set_field(context->byte_224_retry_msg, + V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M, + V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, + attr->sq_psn); + roce_set_field(qpc_mask->byte_224_retry_msg, + V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M, + V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0); + + roce_set_field(context->byte_244_rnr_rxack, + V2_QPC_BYTE_244_RX_ACK_EPSN_M, + V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn); + roce_set_field(qpc_mask->byte_244_rnr_rxack, + V2_QPC_BYTE_244_RX_ACK_EPSN_M, + V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0); + } + + if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) && + attr->max_dest_rd_atomic) { + roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M, + V2_QPC_BYTE_140_RR_MAX_S, + fls(attr->max_dest_rd_atomic - 1)); + roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M, + V2_QPC_BYTE_140_RR_MAX_S, 0); + } + + if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) { + roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M, + V2_QPC_BYTE_208_SR_MAX_S, + fls(attr->max_rd_atomic - 1)); + roce_set_field(qpc_mask->byte_208_irrl, + V2_QPC_BYTE_208_SR_MAX_M, + V2_QPC_BYTE_208_SR_MAX_S, 0); + } + + if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) + set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask); + + if (attr_mask & IB_QP_MIN_RNR_TIMER) { + roce_set_field(context->byte_80_rnr_rx_cqn, + V2_QPC_BYTE_80_MIN_RNR_TIME_M, + V2_QPC_BYTE_80_MIN_RNR_TIME_S, + attr->min_rnr_timer); + roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, + V2_QPC_BYTE_80_MIN_RNR_TIME_M, + V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0); + } + + /* RC&UC required attr */ + if (attr_mask & IB_QP_RQ_PSN) { + roce_set_field(context->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_EPSN_M, + V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn); + roce_set_field(qpc_mask->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_EPSN_M, + V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0); + + roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M, + V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1); + roce_set_field(qpc_mask->byte_152_raq, + V2_QPC_BYTE_152_RAQ_PSN_M, + V2_QPC_BYTE_152_RAQ_PSN_S, 0); + } + + if (attr_mask & IB_QP_QKEY) { + context->qkey_xrcd = cpu_to_le32(attr->qkey); + qpc_mask->qkey_xrcd = 0; + hr_qp->qkey = attr->qkey; + } + + return ret; +} + +static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, + int attr_mask) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + + if (attr_mask & IB_QP_ACCESS_FLAGS) + hr_qp->atomic_rd_en = attr->qp_access_flags; + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) + hr_qp->resp_depth = attr->max_dest_rd_atomic; + if (attr_mask & IB_QP_PORT) { + hr_qp->port = attr->port_num - 1; + hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; + } +} + +static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, + int attr_mask, enum ib_qp_state cur_state, + enum ib_qp_state new_state) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct hns_roce_v2_qp_context ctx[2]; + struct hns_roce_v2_qp_context *context = ctx; + struct hns_roce_v2_qp_context *qpc_mask = ctx + 1; + struct ib_device *ibdev = &hr_dev->ib_dev; + unsigned long sq_flag = 0; + unsigned long rq_flag = 0; + int ret; + + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + memset(context, 0, hr_dev->caps.qpc_sz); + memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz); + + ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state, + new_state, context, qpc_mask); + if (ret) + goto out; + + /* When QP state is err, SQ and RQ WQE should be flushed */ + if (new_state == IB_QPS_ERR) { + spin_lock_irqsave(&hr_qp->sq.lock, sq_flag); + hr_qp->state = IB_QPS_ERR; + roce_set_field(context->byte_160_sq_ci_pi, + V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, + V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, + hr_qp->sq.head); + roce_set_field(qpc_mask->byte_160_sq_ci_pi, + V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M, + V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0); + spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag); + + if (!ibqp->srq) { + spin_lock_irqsave(&hr_qp->rq.lock, rq_flag); + roce_set_field(context->byte_84_rq_ci_pi, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, + hr_qp->rq.head); + roce_set_field(qpc_mask->byte_84_rq_ci_pi, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M, + V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0); + spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag); + } + } + + /* Configure the optional fields */ + ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context, + qpc_mask); + if (ret) + goto out; + + roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S, + ibqp->srq ? 1 : 0); + roce_set_bit(qpc_mask->byte_108_rx_reqepsn, + V2_QPC_BYTE_108_INV_CREDIT_S, 0); + + /* Every status migrate must change state */ + roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M, + V2_QPC_BYTE_60_QP_ST_S, new_state); + roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M, + V2_QPC_BYTE_60_QP_ST_S, 0); + + /* SW pass context to HW */ + ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp); + if (ret) { + ibdev_err(ibdev, "failed to modify QP, ret = %d.\n", ret); + goto out; + } + + hr_qp->state = new_state; + + hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask); + + if (new_state == IB_QPS_RESET && !ibqp->uobject) { + hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn, + ibqp->srq ? to_hr_srq(ibqp->srq) : NULL); + if (ibqp->send_cq != ibqp->recv_cq) + hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), + hr_qp->qpn, NULL); + + hr_qp->rq.head = 0; + hr_qp->rq.tail = 0; + hr_qp->sq.head = 0; + hr_qp->sq.tail = 0; + hr_qp->next_sge = 0; + if (hr_qp->rq.wqe_cnt) + *hr_qp->rdb.db_record = 0; + } + +out: + return ret; +} + +static int to_ib_qp_st(enum hns_roce_v2_qp_state state) +{ + static const enum ib_qp_state map[] = { + [HNS_ROCE_QP_ST_RST] = IB_QPS_RESET, + [HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT, + [HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR, + [HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS, + [HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD, + [HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE, + [HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR, + [HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD + }; + + return (state < ARRAY_SIZE(map)) ? map[state] : -1; +} + +static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_roce_v2_qp_context *hr_context) +{ + struct hns_roce_cmd_mailbox *mailbox; + int ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0, + HNS_ROCE_CMD_QUERY_QPC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + if (ret) + goto out; + + memcpy(hr_context, mailbox->buf, hr_dev->caps.qpc_sz); + +out: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + return ret; +} + +static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + struct hns_roce_v2_qp_context context = {}; + struct ib_device *ibdev = &hr_dev->ib_dev; + int tmp_qp_state; + int state; + int ret; + + memset(qp_attr, 0, sizeof(*qp_attr)); + memset(qp_init_attr, 0, sizeof(*qp_init_attr)); + + mutex_lock(&hr_qp->mutex); + + if (hr_qp->state == IB_QPS_RESET) { + qp_attr->qp_state = IB_QPS_RESET; + ret = 0; + goto done; + } + + ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context); + if (ret) { + ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret); + ret = -EINVAL; + goto out; + } + + state = roce_get_field(context.byte_60_qpst_tempid, + V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S); + tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state); + if (tmp_qp_state == -1) { + ibdev_err(ibdev, "Illegal ib_qp_state\n"); + ret = -EINVAL; + goto out; + } + hr_qp->state = (u8)tmp_qp_state; + qp_attr->qp_state = (enum ib_qp_state)hr_qp->state; + qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context.byte_24_mtu_tc, + V2_QPC_BYTE_24_MTU_M, + V2_QPC_BYTE_24_MTU_S); + qp_attr->path_mig_state = IB_MIG_ARMED; + qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; + if (hr_qp->ibqp.qp_type == IB_QPT_UD) + qp_attr->qkey = le32_to_cpu(context.qkey_xrcd); + + qp_attr->rq_psn = roce_get_field(context.byte_108_rx_reqepsn, + V2_QPC_BYTE_108_RX_REQ_EPSN_M, + V2_QPC_BYTE_108_RX_REQ_EPSN_S); + qp_attr->sq_psn = (u32)roce_get_field(context.byte_172_sq_psn, + V2_QPC_BYTE_172_SQ_CUR_PSN_M, + V2_QPC_BYTE_172_SQ_CUR_PSN_S); + qp_attr->dest_qp_num = (u8)roce_get_field(context.byte_56_dqpn_err, + V2_QPC_BYTE_56_DQPN_M, + V2_QPC_BYTE_56_DQPN_S); + qp_attr->qp_access_flags = ((roce_get_bit(context.byte_76_srqn_op_en, + V2_QPC_BYTE_76_RRE_S)) << V2_QP_RRE_S) | + ((roce_get_bit(context.byte_76_srqn_op_en, + V2_QPC_BYTE_76_RWE_S)) << V2_QP_RWE_S) | + ((roce_get_bit(context.byte_76_srqn_op_en, + V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S); + + if (hr_qp->ibqp.qp_type == IB_QPT_RC || + hr_qp->ibqp.qp_type == IB_QPT_UC) { + struct ib_global_route *grh = + rdma_ah_retrieve_grh(&qp_attr->ah_attr); + + rdma_ah_set_sl(&qp_attr->ah_attr, + roce_get_field(context.byte_28_at_fl, + V2_QPC_BYTE_28_SL_M, + V2_QPC_BYTE_28_SL_S)); + grh->flow_label = roce_get_field(context.byte_28_at_fl, + V2_QPC_BYTE_28_FL_M, + V2_QPC_BYTE_28_FL_S); + grh->sgid_index = roce_get_field(context.byte_20_smac_sgid_idx, + V2_QPC_BYTE_20_SGID_IDX_M, + V2_QPC_BYTE_20_SGID_IDX_S); + grh->hop_limit = roce_get_field(context.byte_24_mtu_tc, + V2_QPC_BYTE_24_HOP_LIMIT_M, + V2_QPC_BYTE_24_HOP_LIMIT_S); + grh->traffic_class = roce_get_field(context.byte_24_mtu_tc, + V2_QPC_BYTE_24_TC_M, + V2_QPC_BYTE_24_TC_S); + + memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw)); + } + + qp_attr->port_num = hr_qp->port + 1; + qp_attr->sq_draining = 0; + qp_attr->max_rd_atomic = 1 << roce_get_field(context.byte_208_irrl, + V2_QPC_BYTE_208_SR_MAX_M, + V2_QPC_BYTE_208_SR_MAX_S); + qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context.byte_140_raq, + V2_QPC_BYTE_140_RR_MAX_M, + V2_QPC_BYTE_140_RR_MAX_S); + qp_attr->min_rnr_timer = (u8)roce_get_field(context.byte_80_rnr_rx_cqn, + V2_QPC_BYTE_80_MIN_RNR_TIME_M, + V2_QPC_BYTE_80_MIN_RNR_TIME_S); + qp_attr->timeout = (u8)roce_get_field(context.byte_28_at_fl, + V2_QPC_BYTE_28_AT_M, + V2_QPC_BYTE_28_AT_S); + qp_attr->retry_cnt = roce_get_field(context.byte_212_lsn, + V2_QPC_BYTE_212_RETRY_NUM_INIT_M, + V2_QPC_BYTE_212_RETRY_NUM_INIT_S); + qp_attr->rnr_retry = roce_get_field(context.byte_244_rnr_rxack, + V2_QPC_BYTE_244_RNR_NUM_INIT_M, + V2_QPC_BYTE_244_RNR_NUM_INIT_S); + +done: + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; + qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs; + + if (!ibqp->uobject) { + qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; + qp_attr->cap.max_send_sge = hr_qp->sq.max_gs; + } else { + qp_attr->cap.max_send_wr = 0; + qp_attr->cap.max_send_sge = 0; + } + + qp_init_attr->cap = qp_attr->cap; + qp_init_attr->sq_sig_type = hr_qp->sq_signal_bits; + +out: + mutex_unlock(&hr_qp->mutex); + return ret; +} + +static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct ib_udata *udata) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_cq *send_cq, *recv_cq; + unsigned long flags; + int ret = 0; + + if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) { + /* Modify qp to reset before destroying qp */ + ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0, + hr_qp->state, IB_QPS_RESET); + if (ret) + ibdev_err(ibdev, + "failed to modify QP to RST, ret = %d.\n", + ret); + } + + send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL; + recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL; + + spin_lock_irqsave(&hr_dev->qp_list_lock, flags); + hns_roce_lock_cqs(send_cq, recv_cq); + + if (!udata) { + if (recv_cq) + __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, + (hr_qp->ibqp.srq ? + to_hr_srq(hr_qp->ibqp.srq) : + NULL)); + + if (send_cq && send_cq != recv_cq) + __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL); + + } + + hns_roce_qp_remove(hr_dev, hr_qp); + + hns_roce_unlock_cqs(send_cq, recv_cq); + spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); + + return ret; +} + +static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + int ret; + + ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); + if (ret) + ibdev_err(&hr_dev->ib_dev, + "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n", + hr_qp->qpn, ret); + + hns_roce_qp_destroy(hr_dev, hr_qp, udata); + + return 0; +} + +static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_sccc_clr_done *resp; + struct hns_roce_sccc_clr *clr; + struct hns_roce_cmq_desc desc; + int ret, i; + + mutex_lock(&hr_dev->qp_table.scc_mutex); + + /* set scc ctx clear done flag */ + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) { + ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret); + goto out; + } + + /* clear scc context */ + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false); + clr = (struct hns_roce_sccc_clr *)desc.data; + clr->qpn = cpu_to_le32(hr_qp->qpn); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) { + ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret); + goto out; + } + + /* query scc context clear is done or not */ + resp = (struct hns_roce_sccc_clr_done *)desc.data; + for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) { + hns_roce_cmq_setup_basic_desc(&desc, + HNS_ROCE_OPC_QUERY_SCCC, true); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) { + ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n", + ret); + goto out; + } + + if (resp->clr_done) + goto out; + + msleep(20); + } + + ibdev_err(ibdev, "Query SCC clr done flag overtime.\n"); + ret = -ETIMEDOUT; + +out: + mutex_unlock(&hr_dev->qp_table.scc_mutex); + return ret; +} + +static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev, + struct hns_roce_srq *srq, u32 pdn, u16 xrcd, + u32 cqn, void *mb_buf, u64 *mtts_wqe, + u64 *mtts_idx, dma_addr_t dma_handle_wqe, + dma_addr_t dma_handle_idx) +{ + struct hns_roce_srq_context *srq_context; + + srq_context = mb_buf; + memset(srq_context, 0, sizeof(*srq_context)); + + roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M, + SRQC_BYTE_4_SRQ_ST_S, 1); + + roce_set_field(srq_context->byte_4_srqn_srqst, + SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M, + SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S, + to_hr_hem_hopnum(hr_dev->caps.srqwqe_hop_num, + srq->wqe_cnt)); + roce_set_field(srq_context->byte_4_srqn_srqst, + SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S, + ilog2(srq->wqe_cnt)); + + roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M, + SRQC_BYTE_4_SRQN_S, srq->srqn); + + roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M, + SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0); + + roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M, + SRQC_BYTE_12_SRQ_XRCD_S, xrcd); + + srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3)); + + roce_set_field(srq_context->byte_24_wqe_bt_ba, + SRQC_BYTE_24_SRQ_WQE_BT_BA_M, + SRQC_BYTE_24_SRQ_WQE_BT_BA_S, + dma_handle_wqe >> 35); + + roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M, + SRQC_BYTE_28_PD_S, pdn); + roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M, + SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 : + fls(srq->max_gs - 1)); + + srq_context->idx_bt_ba = cpu_to_le32(dma_handle_idx >> 3); + roce_set_field(srq_context->rsv_idx_bt_ba, + SRQC_BYTE_36_SRQ_IDX_BT_BA_M, + SRQC_BYTE_36_SRQ_IDX_BT_BA_S, + dma_handle_idx >> 35); + + srq_context->idx_cur_blk_addr = + cpu_to_le32(to_hr_hw_page_addr(mtts_idx[0])); + roce_set_field(srq_context->byte_44_idxbufpgsz_addr, + SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M, + SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S, + upper_32_bits(to_hr_hw_page_addr(mtts_idx[0]))); + roce_set_field(srq_context->byte_44_idxbufpgsz_addr, + SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M, + SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S, + to_hr_hem_hopnum(hr_dev->caps.idx_hop_num, + srq->wqe_cnt)); + + roce_set_field(srq_context->byte_44_idxbufpgsz_addr, + SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M, + SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S, + to_hr_hw_page_shift(srq->idx_que.mtr.hem_cfg.ba_pg_shift)); + roce_set_field(srq_context->byte_44_idxbufpgsz_addr, + SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M, + SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S, + to_hr_hw_page_shift(srq->idx_que.mtr.hem_cfg.buf_pg_shift)); + + srq_context->idx_nxt_blk_addr = + cpu_to_le32(to_hr_hw_page_addr(mtts_idx[1])); + roce_set_field(srq_context->rsv_idxnxtblkaddr, + SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M, + SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S, + upper_32_bits(to_hr_hw_page_addr(mtts_idx[1]))); + roce_set_field(srq_context->byte_56_xrc_cqn, + SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S, + cqn); + roce_set_field(srq_context->byte_56_xrc_cqn, + SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M, + SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S, + to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.ba_pg_shift)); + roce_set_field(srq_context->byte_56_xrc_cqn, + SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M, + SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S, + to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.buf_pg_shift)); + + roce_set_bit(srq_context->db_record_addr_record_en, + SRQC_BYTE_60_SRQ_RECORD_EN_S, 0); +} + +static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq, + struct ib_srq_attr *srq_attr, + enum ib_srq_attr_mask srq_attr_mask, + struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); + struct hns_roce_srq *srq = to_hr_srq(ibsrq); + struct hns_roce_srq_context *srq_context; + struct hns_roce_srq_context *srqc_mask; + struct hns_roce_cmd_mailbox *mailbox; + int ret; + + /* Resizing SRQs is not supported yet */ + if (srq_attr_mask & IB_SRQ_MAX_WR) + return -EINVAL; + + if (srq_attr_mask & IB_SRQ_LIMIT) { + if (srq_attr->srq_limit >= srq->wqe_cnt) + return -EINVAL; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + srq_context = mailbox->buf; + srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1; + + memset(srqc_mask, 0xff, sizeof(*srqc_mask)); + + roce_set_field(srq_context->byte_8_limit_wl, + SRQC_BYTE_8_SRQ_LIMIT_WL_M, + SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit); + roce_set_field(srqc_mask->byte_8_limit_wl, + SRQC_BYTE_8_SRQ_LIMIT_WL_M, + SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0); + + ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0, + HNS_ROCE_CMD_MODIFY_SRQC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + if (ret) { + ibdev_err(&hr_dev->ib_dev, + "failed to handle cmd of modifying SRQ, ret = %d.\n", + ret); + return ret; + } + } + + return 0; +} + +static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); + struct hns_roce_srq *srq = to_hr_srq(ibsrq); + struct hns_roce_srq_context *srq_context; + struct hns_roce_cmd_mailbox *mailbox; + int limit_wl; + int ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + srq_context = mailbox->buf; + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0, + HNS_ROCE_CMD_QUERY_SRQC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + if (ret) { + ibdev_err(&hr_dev->ib_dev, + "failed to process cmd of querying SRQ, ret = %d.\n", + ret); + goto out; + } + + limit_wl = roce_get_field(srq_context->byte_8_limit_wl, + SRQC_BYTE_8_SRQ_LIMIT_WL_M, + SRQC_BYTE_8_SRQ_LIMIT_WL_S); + + attr->srq_limit = limit_wl; + attr->max_wr = srq->wqe_cnt - 1; + attr->max_sge = srq->max_gs; + +out: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + return ret; +} + +static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(cq->device); + struct hns_roce_v2_cq_context *cq_context; + struct hns_roce_cq *hr_cq = to_hr_cq(cq); + struct hns_roce_v2_cq_context *cqc_mask; + struct hns_roce_cmd_mailbox *mailbox; + int ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + cq_context = mailbox->buf; + cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1; + + memset(cqc_mask, 0xff, sizeof(*cqc_mask)); + + roce_set_field(cq_context->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S, + cq_count); + roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S, + 0); + roce_set_field(cq_context->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S, + cq_period); + roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S, + 0); + + ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1, + HNS_ROCE_CMD_MODIFY_CQC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + if (ret) + ibdev_err(&hr_dev->ib_dev, + "failed to process cmd when modifying CQ, ret = %d.\n", + ret); + + return ret; +} + +static void hns_roce_irq_work_handle(struct work_struct *work) +{ + struct hns_roce_work *irq_work = + container_of(work, struct hns_roce_work, work); + struct ib_device *ibdev = &irq_work->hr_dev->ib_dev; + u32 qpn = irq_work->qpn; + u32 cqn = irq_work->cqn; + + switch (irq_work->event_type) { + case HNS_ROCE_EVENT_TYPE_PATH_MIG: + ibdev_info(ibdev, "Path migrated succeeded.\n"); + break; + case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: + ibdev_warn(ibdev, "Path migration failed.\n"); + break; + case HNS_ROCE_EVENT_TYPE_COMM_EST: + break; + case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: + ibdev_warn(ibdev, "Send queue drained.\n"); + break; + case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + ibdev_err(ibdev, "Local work queue 0x%x catast error, sub_event type is: %d\n", + qpn, irq_work->sub_type); + break; + case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + ibdev_err(ibdev, "Invalid request local work queue 0x%x error.\n", + qpn); + break; + case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + ibdev_err(ibdev, "Local access violation work queue 0x%x error, sub_event type is: %d\n", + qpn, irq_work->sub_type); + break; + case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: + ibdev_warn(ibdev, "SRQ limit reach.\n"); + break; + case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: + ibdev_warn(ibdev, "SRQ last wqe reach.\n"); + break; + case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + ibdev_err(ibdev, "SRQ catas error.\n"); + break; + case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: + ibdev_err(ibdev, "CQ 0x%x access err.\n", cqn); + break; + case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: + ibdev_warn(ibdev, "CQ 0x%x overflow\n", cqn); + break; + case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: + ibdev_warn(ibdev, "DB overflow.\n"); + break; + case HNS_ROCE_EVENT_TYPE_FLR: + ibdev_warn(ibdev, "Function level reset.\n"); + break; + default: + break; + } + + kfree(irq_work); +} + +static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq, + u32 qpn, u32 cqn) +{ + struct hns_roce_work *irq_work; + + irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC); + if (!irq_work) + return; + + INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle); + irq_work->hr_dev = hr_dev; + irq_work->qpn = qpn; + irq_work->cqn = cqn; + irq_work->event_type = eq->event_type; + irq_work->sub_type = eq->sub_type; + queue_work(hr_dev->irq_workq, &(irq_work->work)); +} + +static void set_eq_cons_index_v2(struct hns_roce_eq *eq) +{ + struct hns_roce_dev *hr_dev = eq->hr_dev; + __le32 doorbell[2] = {}; + + if (eq->type_flag == HNS_ROCE_AEQ) { + roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M, + HNS_ROCE_V2_EQ_DB_CMD_S, + eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? + HNS_ROCE_EQ_DB_CMD_AEQ : + HNS_ROCE_EQ_DB_CMD_AEQ_ARMED); + } else { + roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M, + HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn); + + roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M, + HNS_ROCE_V2_EQ_DB_CMD_S, + eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? + HNS_ROCE_EQ_DB_CMD_CEQ : + HNS_ROCE_EQ_DB_CMD_CEQ_ARMED); + } + + roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M, + HNS_ROCE_V2_EQ_DB_PARA_S, + (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M)); + + hns_roce_write64(hr_dev, doorbell, eq->doorbell); +} + +static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq) +{ + struct hns_roce_aeqe *aeqe; + + aeqe = hns_roce_buf_offset(eq->mtr.kmem, + (eq->cons_index & (eq->entries - 1)) * + eq->eqe_size); + + return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^ + !!(eq->cons_index & eq->entries)) ? aeqe : NULL; +} + +static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + struct device *dev = hr_dev->dev; + struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq); + int aeqe_found = 0; + int event_type; + int sub_type; + u32 srqn; + u32 qpn; + u32 cqn; + + while (aeqe) { + /* Make sure we read AEQ entry after we have checked the + * ownership bit + */ + dma_rmb(); + + event_type = roce_get_field(aeqe->asyn, + HNS_ROCE_V2_AEQE_EVENT_TYPE_M, + HNS_ROCE_V2_AEQE_EVENT_TYPE_S); + sub_type = roce_get_field(aeqe->asyn, + HNS_ROCE_V2_AEQE_SUB_TYPE_M, + HNS_ROCE_V2_AEQE_SUB_TYPE_S); + qpn = roce_get_field(aeqe->event.qp_event.qp, + HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M, + HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S); + cqn = roce_get_field(aeqe->event.cq_event.cq, + HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M, + HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S); + srqn = roce_get_field(aeqe->event.srq_event.srq, + HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M, + HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S); + + switch (event_type) { + case HNS_ROCE_EVENT_TYPE_PATH_MIG: + case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: + case HNS_ROCE_EVENT_TYPE_COMM_EST: + case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: + case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: + case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + hns_roce_qp_event(hr_dev, qpn, event_type); + break; + case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: + case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + hns_roce_srq_event(hr_dev, srqn, event_type); + break; + case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: + case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: + hns_roce_cq_event(hr_dev, cqn, event_type); + break; + case HNS_ROCE_EVENT_TYPE_MB: + hns_roce_cmd_event(hr_dev, + le16_to_cpu(aeqe->event.cmd.token), + aeqe->event.cmd.status, + le64_to_cpu(aeqe->event.cmd.out_param)); + break; + case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: + case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW: + case HNS_ROCE_EVENT_TYPE_FLR: + break; + default: + dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n", + event_type, eq->eqn, eq->cons_index); + break; + } + + eq->event_type = event_type; + eq->sub_type = sub_type; + ++eq->cons_index; + aeqe_found = 1; + + if (eq->cons_index > (2 * eq->entries - 1)) + eq->cons_index = 0; + + hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn); + + aeqe = next_aeqe_sw_v2(eq); + } + + set_eq_cons_index_v2(eq); + return aeqe_found; +} + +static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq) +{ + struct hns_roce_ceqe *ceqe; + + ceqe = hns_roce_buf_offset(eq->mtr.kmem, + (eq->cons_index & (eq->entries - 1)) * + eq->eqe_size); + + return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^ + (!!(eq->cons_index & eq->entries)) ? ceqe : NULL; +} + +static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq) +{ + struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq); + int ceqe_found = 0; + u32 cqn; + + while (ceqe) { + /* Make sure we read CEQ entry after we have checked the + * ownership bit + */ + dma_rmb(); + + cqn = roce_get_field(ceqe->comp, HNS_ROCE_V2_CEQE_COMP_CQN_M, + HNS_ROCE_V2_CEQE_COMP_CQN_S); + + hns_roce_cq_completion(hr_dev, cqn); + + ++eq->cons_index; + ceqe_found = 1; + + if (eq->cons_index > (EQ_DEPTH_COEFF * eq->entries - 1)) + eq->cons_index = 0; + + ceqe = next_ceqe_sw_v2(eq); + } + + set_eq_cons_index_v2(eq); + + return ceqe_found; +} + +static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr) +{ + struct hns_roce_eq *eq = eq_ptr; + struct hns_roce_dev *hr_dev = eq->hr_dev; + int int_work; + + if (eq->type_flag == HNS_ROCE_CEQ) + /* Completion event interrupt */ + int_work = hns_roce_v2_ceq_int(hr_dev, eq); + else + /* Asychronous event interrupt */ + int_work = hns_roce_v2_aeq_int(hr_dev, eq); + + return IRQ_RETVAL(int_work); +} + +static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id) +{ + struct hns_roce_dev *hr_dev = dev_id; + struct device *dev = hr_dev->dev; + int int_work = 0; + u32 int_st; + u32 int_en; + + /* Abnormal interrupt */ + int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG); + int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG); + + if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) { + struct pci_dev *pdev = hr_dev->pci_dev; + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); + const struct hnae3_ae_ops *ops = ae_dev->ops; + + dev_err(dev, "AEQ overflow!\n"); + + roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, + 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S); + + /* Set reset level for reset_event() */ + if (ops->set_default_reset_request) + ops->set_default_reset_request(ae_dev, + HNAE3_FUNC_RESET); + if (ops->reset_event) + ops->reset_event(pdev, NULL); + + int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S; + roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); + + int_work = 1; + } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) { + dev_err(dev, "BUS ERR!\n"); + + int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S; + roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); + + int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S; + roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); + + int_work = 1; + } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) { + dev_err(dev, "OTHER ERR!\n"); + + int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S; + roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st); + + int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S; + roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); + + int_work = 1; + } else + dev_err(dev, "There is no abnormal irq found!\n"); + + return IRQ_RETVAL(int_work); +} + +static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev, + int eq_num, int enable_flag) +{ + int i; + + if (enable_flag == EQ_ENABLE) { + for (i = 0; i < eq_num; i++) + roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + + i * EQ_REG_OFFSET, + HNS_ROCE_V2_VF_EVENT_INT_EN_M); + + roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, + HNS_ROCE_V2_VF_ABN_INT_EN_M); + roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, + HNS_ROCE_V2_VF_ABN_INT_CFG_M); + } else { + for (i = 0; i < eq_num; i++) + roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + + i * EQ_REG_OFFSET, + HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0); + + roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, + HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0); + roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, + HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0); + } +} + +static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn) +{ + struct device *dev = hr_dev->dev; + int ret; + + if (eqn < hr_dev->caps.num_comp_vectors) + ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M, + 0, HNS_ROCE_CMD_DESTROY_CEQC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + else + ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M, + 0, HNS_ROCE_CMD_DESTROY_AEQC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + if (ret) + dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn); +} + +static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) +{ + hns_roce_mtr_destroy(hr_dev, &eq->mtr); +} + +static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq, + void *mb_buf) +{ + u64 eqe_ba[MTT_MIN_COUNT] = { 0 }; + struct hns_roce_eq_context *eqc; + u64 bt_ba = 0; + int count; + + eqc = mb_buf; + memset(eqc, 0, sizeof(struct hns_roce_eq_context)); + + /* init eqc */ + eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG; + eq->cons_index = 0; + eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0; + eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0; + eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED; + eq->shift = ilog2((unsigned int)eq->entries); + + /* if not multi-hop, eqe buffer only use one trunk */ + count = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba, MTT_MIN_COUNT, + &bt_ba); + if (count < 1) { + dev_err(hr_dev->dev, "failed to find EQE mtr\n"); + return -ENOBUFS; + } + + /* set eqc state */ + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQ_ST_M, HNS_ROCE_EQC_EQ_ST_S, + HNS_ROCE_V2_EQ_STATE_VALID); + + /* set eqe hop num */ + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_HOP_NUM_M, + HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num); + + /* set eqc over_ignore */ + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_OVER_IGNORE_M, + HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore); + + /* set eqc coalesce */ + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_COALESCE_M, + HNS_ROCE_EQC_COALESCE_S, eq->coalesce); + + /* set eqc arm_state */ + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_ARM_ST_M, + HNS_ROCE_EQC_ARM_ST_S, eq->arm_st); + + /* set eqn */ + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQN_M, HNS_ROCE_EQC_EQN_S, + eq->eqn); + + /* set eqe_cnt */ + roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQE_CNT_M, + HNS_ROCE_EQC_EQE_CNT_S, HNS_ROCE_EQ_INIT_EQE_CNT); + + /* set eqe_ba_pg_sz */ + roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BA_PG_SZ_M, + HNS_ROCE_EQC_BA_PG_SZ_S, + to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift)); + + /* set eqe_buf_pg_sz */ + roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BUF_PG_SZ_M, + HNS_ROCE_EQC_BUF_PG_SZ_S, + to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift)); + + /* set eq_producer_idx */ + roce_set_field(eqc->byte_8, HNS_ROCE_EQC_PROD_INDX_M, + HNS_ROCE_EQC_PROD_INDX_S, HNS_ROCE_EQ_INIT_PROD_IDX); + + /* set eq_max_cnt */ + roce_set_field(eqc->byte_12, HNS_ROCE_EQC_MAX_CNT_M, + HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt); + + /* set eq_period */ + roce_set_field(eqc->byte_12, HNS_ROCE_EQC_PERIOD_M, + HNS_ROCE_EQC_PERIOD_S, eq->eq_period); + + /* set eqe_report_timer */ + roce_set_field(eqc->eqe_report_timer, HNS_ROCE_EQC_REPORT_TIMER_M, + HNS_ROCE_EQC_REPORT_TIMER_S, + HNS_ROCE_EQ_INIT_REPORT_TIMER); + + /* set bt_ba [34:3] */ + roce_set_field(eqc->eqe_ba0, HNS_ROCE_EQC_EQE_BA_L_M, + HNS_ROCE_EQC_EQE_BA_L_S, bt_ba >> 3); + + /* set bt_ba [64:35] */ + roce_set_field(eqc->eqe_ba1, HNS_ROCE_EQC_EQE_BA_H_M, + HNS_ROCE_EQC_EQE_BA_H_S, bt_ba >> 35); + + /* set eq shift */ + roce_set_field(eqc->byte_28, HNS_ROCE_EQC_SHIFT_M, HNS_ROCE_EQC_SHIFT_S, + eq->shift); + + /* set eq MSI_IDX */ + roce_set_field(eqc->byte_28, HNS_ROCE_EQC_MSI_INDX_M, + HNS_ROCE_EQC_MSI_INDX_S, HNS_ROCE_EQ_INIT_MSI_IDX); + + /* set cur_eqe_ba [27:12] */ + roce_set_field(eqc->byte_28, HNS_ROCE_EQC_CUR_EQE_BA_L_M, + HNS_ROCE_EQC_CUR_EQE_BA_L_S, eqe_ba[0] >> 12); + + /* set cur_eqe_ba [59:28] */ + roce_set_field(eqc->byte_32, HNS_ROCE_EQC_CUR_EQE_BA_M_M, + HNS_ROCE_EQC_CUR_EQE_BA_M_S, eqe_ba[0] >> 28); + + /* set cur_eqe_ba [63:60] */ + roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CUR_EQE_BA_H_M, + HNS_ROCE_EQC_CUR_EQE_BA_H_S, eqe_ba[0] >> 60); + + /* set eq consumer idx */ + roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CONS_INDX_M, + HNS_ROCE_EQC_CONS_INDX_S, HNS_ROCE_EQ_INIT_CONS_IDX); + + roce_set_field(eqc->byte_40, HNS_ROCE_EQC_NXT_EQE_BA_L_M, + HNS_ROCE_EQC_NXT_EQE_BA_L_S, eqe_ba[1] >> 12); + + roce_set_field(eqc->byte_44, HNS_ROCE_EQC_NXT_EQE_BA_H_M, + HNS_ROCE_EQC_NXT_EQE_BA_H_S, eqe_ba[1] >> 44); + + roce_set_field(eqc->byte_44, HNS_ROCE_EQC_EQE_SIZE_M, + HNS_ROCE_EQC_EQE_SIZE_S, + eq->eqe_size == HNS_ROCE_V3_EQE_SIZE ? 1 : 0); + + return 0; +} + +static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) +{ + struct hns_roce_buf_attr buf_attr = {}; + int err; + + if (hr_dev->caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0) + eq->hop_num = 0; + else + eq->hop_num = hr_dev->caps.eqe_hop_num; + + buf_attr.page_shift = hr_dev->caps.eqe_buf_pg_sz + HNS_HW_PAGE_SHIFT; + buf_attr.region[0].size = eq->entries * eq->eqe_size; + buf_attr.region[0].hopnum = eq->hop_num; + buf_attr.region_count = 1; + buf_attr.fixed_page = true; + + err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr, + hr_dev->caps.eqe_ba_pg_sz + + HNS_HW_PAGE_SHIFT, NULL, 0); + if (err) + dev_err(hr_dev->dev, "Failed to alloc EQE mtr, err %d\n", err); + + return err; +} + +static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev, + struct hns_roce_eq *eq, + unsigned int eq_cmd) +{ + struct hns_roce_cmd_mailbox *mailbox; + int ret; + + /* Allocate mailbox memory */ + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR_OR_NULL(mailbox)) + return -ENOMEM; + + ret = alloc_eq_buf(hr_dev, eq); + if (ret) + goto free_cmd_mbox; + + ret = config_eqc(hr_dev, eq, mailbox->buf); + if (ret) + goto err_cmd_mbox; + + ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0, + eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS); + if (ret) { + dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n"); + goto err_cmd_mbox; + } + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return 0; + +err_cmd_mbox: + free_eq_buf(hr_dev, eq); + +free_cmd_mbox: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} + +static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num, + int comp_num, int aeq_num, int other_num) +{ + struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; + int i, j; + int ret; + + for (i = 0; i < irq_num; i++) { + hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN, + GFP_KERNEL); + if (!hr_dev->irq_names[i]) { + ret = -ENOMEM; + goto err_kzalloc_failed; + } + } + + /* irq contains: abnormal + AEQ + CEQ */ + for (j = 0; j < other_num; j++) + snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, + "hns-abn-%d", j); + + for (j = other_num; j < (other_num + aeq_num); j++) + snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, + "hns-aeq-%d", j - other_num); + + for (j = (other_num + aeq_num); j < irq_num; j++) + snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, + "hns-ceq-%d", j - other_num - aeq_num); + + for (j = 0; j < irq_num; j++) { + if (j < other_num) + ret = request_irq(hr_dev->irq[j], + hns_roce_v2_msix_interrupt_abn, + 0, hr_dev->irq_names[j], hr_dev); + + else if (j < (other_num + comp_num)) + ret = request_irq(eq_table->eq[j - other_num].irq, + hns_roce_v2_msix_interrupt_eq, + 0, hr_dev->irq_names[j + aeq_num], + &eq_table->eq[j - other_num]); + else + ret = request_irq(eq_table->eq[j - other_num].irq, + hns_roce_v2_msix_interrupt_eq, + 0, hr_dev->irq_names[j - comp_num], + &eq_table->eq[j - other_num]); + if (ret) { + dev_err(hr_dev->dev, "Request irq error!\n"); + goto err_request_failed; + } + } + + return 0; + +err_request_failed: + for (j -= 1; j >= 0; j--) + if (j < other_num) + free_irq(hr_dev->irq[j], hr_dev); + else + free_irq(eq_table->eq[j - other_num].irq, + &eq_table->eq[j - other_num]); + +err_kzalloc_failed: + for (i -= 1; i >= 0; i--) + kfree(hr_dev->irq_names[i]); + + return ret; +} + +static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev) +{ + int irq_num; + int eq_num; + int i; + + eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; + irq_num = eq_num + hr_dev->caps.num_other_vectors; + + for (i = 0; i < hr_dev->caps.num_other_vectors; i++) + free_irq(hr_dev->irq[i], hr_dev); + + for (i = 0; i < eq_num; i++) + free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]); + + for (i = 0; i < irq_num; i++) + kfree(hr_dev->irq_names[i]); +} + +static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; + struct device *dev = hr_dev->dev; + struct hns_roce_eq *eq; + unsigned int eq_cmd; + int irq_num; + int eq_num; + int other_num; + int comp_num; + int aeq_num; + int i; + int ret; + + other_num = hr_dev->caps.num_other_vectors; + comp_num = hr_dev->caps.num_comp_vectors; + aeq_num = hr_dev->caps.num_aeq_vectors; + + eq_num = comp_num + aeq_num; + irq_num = eq_num + other_num; + + eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); + if (!eq_table->eq) + return -ENOMEM; + + /* create eq */ + for (i = 0; i < eq_num; i++) { + eq = &eq_table->eq[i]; + eq->hr_dev = hr_dev; + eq->eqn = i; + if (i < comp_num) { + /* CEQ */ + eq_cmd = HNS_ROCE_CMD_CREATE_CEQC; + eq->type_flag = HNS_ROCE_CEQ; + eq->entries = hr_dev->caps.ceqe_depth; + eq->eqe_size = hr_dev->caps.ceqe_size; + eq->irq = hr_dev->irq[i + other_num + aeq_num]; + eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM; + eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL; + } else { + /* AEQ */ + eq_cmd = HNS_ROCE_CMD_CREATE_AEQC; + eq->type_flag = HNS_ROCE_AEQ; + eq->entries = hr_dev->caps.aeqe_depth; + eq->eqe_size = hr_dev->caps.aeqe_size; + eq->irq = hr_dev->irq[i - comp_num + other_num]; + eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM; + eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL; + } + + ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd); + if (ret) { + dev_err(dev, "eq create failed.\n"); + goto err_create_eq_fail; + } + } + + /* enable irq */ + hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE); + + ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num, + aeq_num, other_num); + if (ret) { + dev_err(dev, "Request irq failed.\n"); + goto err_request_irq_fail; + } + + hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0); + if (!hr_dev->irq_workq) { + dev_err(dev, "Create irq workqueue failed!\n"); + ret = -ENOMEM; + goto err_create_wq_fail; + } + + return 0; + +err_create_wq_fail: + __hns_roce_free_irq(hr_dev); + +err_request_irq_fail: + hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE); + +err_create_eq_fail: + for (i -= 1; i >= 0; i--) + free_eq_buf(hr_dev, &eq_table->eq[i]); + kfree(eq_table->eq); + + return ret; +} + +static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; + int eq_num; + int i; + + eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; + + /* Disable irq */ + hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE); + + __hns_roce_free_irq(hr_dev); + + for (i = 0; i < eq_num; i++) { + hns_roce_v2_destroy_eqc(hr_dev, i); + + free_eq_buf(hr_dev, &eq_table->eq[i]); + } + + kfree(eq_table->eq); + + flush_workqueue(hr_dev->irq_workq); + destroy_workqueue(hr_dev->irq_workq); +} + +static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = { + .query_cqc_info = hns_roce_v2_query_cqc_info, +}; + +static const struct ib_device_ops hns_roce_v2_dev_ops = { + .destroy_qp = hns_roce_v2_destroy_qp, + .modify_cq = hns_roce_v2_modify_cq, + .poll_cq = hns_roce_v2_poll_cq, + .post_recv = hns_roce_v2_post_recv, + .post_send = hns_roce_v2_post_send, + .query_qp = hns_roce_v2_query_qp, + .req_notify_cq = hns_roce_v2_req_notify_cq, +}; + +static const struct ib_device_ops hns_roce_v2_dev_srq_ops = { + .modify_srq = hns_roce_v2_modify_srq, + .post_srq_recv = hns_roce_v2_post_srq_recv, + .query_srq = hns_roce_v2_query_srq, +}; + +static const struct hns_roce_hw hns_roce_hw_v2 = { + .cmq_init = hns_roce_v2_cmq_init, + .cmq_exit = hns_roce_v2_cmq_exit, + .hw_profile = hns_roce_v2_profile, + .hw_init = hns_roce_v2_init, + .hw_exit = hns_roce_v2_exit, + .post_mbox = hns_roce_v2_post_mbox, + .chk_mbox = hns_roce_v2_chk_mbox, + .rst_prc_mbox = hns_roce_v2_rst_process_cmd, + .set_gid = hns_roce_v2_set_gid, + .set_mac = hns_roce_v2_set_mac, + .write_mtpt = hns_roce_v2_write_mtpt, + .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt, + .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt, + .mw_write_mtpt = hns_roce_v2_mw_write_mtpt, + .write_cqc = hns_roce_v2_write_cqc, + .set_hem = hns_roce_v2_set_hem, + .clear_hem = hns_roce_v2_clear_hem, + .modify_qp = hns_roce_v2_modify_qp, + .query_qp = hns_roce_v2_query_qp, + .destroy_qp = hns_roce_v2_destroy_qp, + .qp_flow_control_init = hns_roce_v2_qp_flow_control_init, + .modify_cq = hns_roce_v2_modify_cq, + .post_send = hns_roce_v2_post_send, + .post_recv = hns_roce_v2_post_recv, + .req_notify_cq = hns_roce_v2_req_notify_cq, + .poll_cq = hns_roce_v2_poll_cq, + .init_eq = hns_roce_v2_init_eq_table, + .cleanup_eq = hns_roce_v2_cleanup_eq_table, + .write_srqc = hns_roce_v2_write_srqc, + .modify_srq = hns_roce_v2_modify_srq, + .query_srq = hns_roce_v2_query_srq, + .post_srq_recv = hns_roce_v2_post_srq_recv, + .hns_roce_dev_ops = &hns_roce_v2_dev_ops, + .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, +}; + +static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { + {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0}, + {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0}, + {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0}, + {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0}, + {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0}, + /* required last entry */ + {0, } +}; + +MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl); + +static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, + struct hnae3_handle *handle) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + int i; + + hr_dev->pci_dev = handle->pdev; + hr_dev->dev = &handle->pdev->dev; + hr_dev->hw = &hns_roce_hw_v2; + hr_dev->dfx = &hns_roce_dfx_hw_v2; + hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; + hr_dev->odb_offset = hr_dev->sdb_offset; + + /* Get info from NIC driver. */ + hr_dev->reg_base = handle->rinfo.roce_io_base; + hr_dev->caps.num_ports = 1; + hr_dev->iboe.netdevs[0] = handle->rinfo.netdev; + hr_dev->iboe.phy_port[0] = 0; + + addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid, + hr_dev->iboe.netdevs[0]->dev_addr); + + for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++) + hr_dev->irq[i] = pci_irq_vector(handle->pdev, + i + handle->rinfo.base_vector); + + /* cmd issue mode: 0 is poll, 1 is event */ + hr_dev->cmd_mod = 1; + hr_dev->loop_idc = 0; + + hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle); + priv->handle = handle; +} + +static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) +{ + struct hns_roce_dev *hr_dev; + int ret; + + hr_dev = ib_alloc_device(hns_roce_dev, ib_dev); + if (!hr_dev) + return -ENOMEM; + + hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL); + if (!hr_dev->priv) { + ret = -ENOMEM; + goto error_failed_kzalloc; + } + + hns_roce_hw_v2_get_cfg(hr_dev, handle); + + ret = hns_roce_init(hr_dev); + if (ret) { + dev_err(hr_dev->dev, "RoCE Engine init failed!\n"); + goto error_failed_get_cfg; + } + + handle->priv = hr_dev; + + return 0; + +error_failed_get_cfg: + kfree(hr_dev->priv); + +error_failed_kzalloc: + ib_dealloc_device(&hr_dev->ib_dev); + + return ret; +} + +static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, + bool reset) +{ + struct hns_roce_dev *hr_dev = handle->priv; + + if (!hr_dev) + return; + + handle->priv = NULL; + + hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT; + hns_roce_handle_device_err(hr_dev); + + hns_roce_exit(hr_dev); + kfree(hr_dev->priv); + ib_dealloc_device(&hr_dev->ib_dev); +} + +static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) +{ + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + const struct pci_device_id *id; + struct device *dev = &handle->pdev->dev; + int ret; + + handle->rinfo.instance_state = HNS_ROCE_STATE_INIT; + + if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) { + handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; + goto reset_chk_err; + } + + id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev); + if (!id) + return 0; + + ret = __hns_roce_hw_v2_init_instance(handle); + if (ret) { + handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; + dev_err(dev, "RoCE instance init failed! ret = %d\n", ret); + if (ops->ae_dev_resetting(handle) || + ops->get_hw_reset_stat(handle)) + goto reset_chk_err; + else + return ret; + } + + handle->rinfo.instance_state = HNS_ROCE_STATE_INITED; + + + return 0; + +reset_chk_err: + dev_err(dev, "Device is busy in resetting state.\n" + "please retry later.\n"); + + return -EBUSY; +} + +static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, + bool reset) +{ + if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) + return; + + handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT; + + __hns_roce_hw_v2_uninit_instance(handle, reset); + + handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; +} +static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle) +{ + struct hns_roce_dev *hr_dev; + + if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) { + set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state); + return 0; + } + + handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN; + clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state); + + hr_dev = handle->priv; + if (!hr_dev) + return 0; + + hr_dev->active = false; + hr_dev->dis_db = true; + hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN; + + return 0; +} + +static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle) +{ + struct device *dev = &handle->pdev->dev; + int ret; + + if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN, + &handle->rinfo.state)) { + handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED; + return 0; + } + + handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT; + + dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n"); + ret = __hns_roce_hw_v2_init_instance(handle); + if (ret) { + /* when reset notify type is HNAE3_INIT_CLIENT In reset notify + * callback function, RoCE Engine reinitialize. If RoCE reinit + * failed, we should inform NIC driver. + */ + handle->priv = NULL; + dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret); + } else { + handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED; + dev_info(dev, "Reset done, RoCE client reinit finished.\n"); + } + + return ret; +} + +static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle) +{ + if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state)) + return 0; + + handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT; + dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n"); + msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY); + __hns_roce_hw_v2_uninit_instance(handle, false); + + return 0; +} + +static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle, + enum hnae3_reset_notify_type type) +{ + int ret = 0; + + switch (type) { + case HNAE3_DOWN_CLIENT: + ret = hns_roce_hw_v2_reset_notify_down(handle); + break; + case HNAE3_INIT_CLIENT: + ret = hns_roce_hw_v2_reset_notify_init(handle); + break; + case HNAE3_UNINIT_CLIENT: + ret = hns_roce_hw_v2_reset_notify_uninit(handle); + break; + default: + break; + } + + return ret; +} + +static const struct hnae3_client_ops hns_roce_hw_v2_ops = { + .init_instance = hns_roce_hw_v2_init_instance, + .uninit_instance = hns_roce_hw_v2_uninit_instance, + .reset_notify = hns_roce_hw_v2_reset_notify, +}; + +static struct hnae3_client hns_roce_hw_v2_client = { + .name = "hns_roce_hw_v2", + .type = HNAE3_CLIENT_ROCE, + .ops = &hns_roce_hw_v2_ops, +}; + +static int __init hns_roce_hw_v2_init(void) +{ + return hnae3_register_client(&hns_roce_hw_v2_client); +} + +static void __exit hns_roce_hw_v2_exit(void) +{ + hnae3_unregister_client(&hns_roce_hw_v2_client); +} + +module_init(hns_roce_hw_v2_init); +module_exit(hns_roce_hw_v2_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); +MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>"); +MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>"); +MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver"); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h new file mode 100644 index 000000000..8948d2b55 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -0,0 +1,2000 @@ +/* + * Copyright (c) 2016-2017 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _HNS_ROCE_HW_V2_H +#define _HNS_ROCE_HW_V2_H + +#include <linux/bitops.h> + +#define HNS_ROCE_VF_QPC_BT_NUM 256 +#define HNS_ROCE_VF_SCCC_BT_NUM 64 +#define HNS_ROCE_VF_SRQC_BT_NUM 64 +#define HNS_ROCE_VF_CQC_BT_NUM 64 +#define HNS_ROCE_VF_MPT_BT_NUM 64 +#define HNS_ROCE_VF_EQC_NUM 64 +#define HNS_ROCE_VF_SMAC_NUM 32 +#define HNS_ROCE_VF_SGID_NUM 32 +#define HNS_ROCE_VF_SL_NUM 8 + +#define HNS_ROCE_V2_MAX_QP_NUM 0x100000 +#define HNS_ROCE_V2_MAX_QPC_TIMER_NUM 0x200 +#define HNS_ROCE_V2_MAX_WQE_NUM 0x8000 +#define HNS_ROCE_V2_MAX_SRQ 0x100000 +#define HNS_ROCE_V2_MAX_SRQ_WR 0x8000 +#define HNS_ROCE_V2_MAX_SRQ_SGE 64 +#define HNS_ROCE_V2_MAX_CQ_NUM 0x100000 +#define HNS_ROCE_V2_MAX_CQC_TIMER_NUM 0x100 +#define HNS_ROCE_V2_MAX_SRQ_NUM 0x100000 +#define HNS_ROCE_V2_MAX_CQE_NUM 0x400000 +#define HNS_ROCE_V2_MAX_SRQWQE_NUM 0x8000 +#define HNS_ROCE_V2_MAX_RQ_SGE_NUM 64 +#define HNS_ROCE_V2_MAX_SQ_SGE_NUM 64 +#define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM 0x200000 +#define HNS_ROCE_V2_MAX_SQ_INLINE 0x20 +#define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32 +#define HNS_ROCE_V2_UAR_NUM 256 +#define HNS_ROCE_V2_PHY_UAR_NUM 1 +#define HNS_ROCE_V2_MAX_IRQ_NUM 65 +#define HNS_ROCE_V2_COMP_VEC_NUM 63 +#define HNS_ROCE_V2_AEQE_VEC_NUM 1 +#define HNS_ROCE_V2_ABNORMAL_VEC_NUM 1 +#define HNS_ROCE_V2_MAX_MTPT_NUM 0x100000 +#define HNS_ROCE_V2_MAX_MTT_SEGS 0x1000000 +#define HNS_ROCE_V2_MAX_CQE_SEGS 0x1000000 +#define HNS_ROCE_V2_MAX_SRQWQE_SEGS 0x1000000 +#define HNS_ROCE_V2_MAX_IDX_SEGS 0x1000000 +#define HNS_ROCE_V2_MAX_PD_NUM 0x1000000 +#define HNS_ROCE_V2_MAX_QP_INIT_RDMA 128 +#define HNS_ROCE_V2_MAX_QP_DEST_RDMA 128 +#define HNS_ROCE_V2_MAX_SQ_DESC_SZ 64 +#define HNS_ROCE_V2_MAX_RQ_DESC_SZ 16 +#define HNS_ROCE_V2_MAX_SRQ_DESC_SZ 64 +#define HNS_ROCE_V2_IRRL_ENTRY_SZ 64 +#define HNS_ROCE_V2_TRRL_ENTRY_SZ 48 +#define HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ 100 +#define HNS_ROCE_V2_CQC_ENTRY_SZ 64 +#define HNS_ROCE_V2_SRQC_ENTRY_SZ 64 +#define HNS_ROCE_V2_MTPT_ENTRY_SZ 64 +#define HNS_ROCE_V2_MTT_ENTRY_SZ 64 +#define HNS_ROCE_V2_IDX_ENTRY_SZ 4 + +#define HNS_ROCE_V2_SCCC_SZ 32 +#define HNS_ROCE_V3_SCCC_SZ 64 + +#define HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ PAGE_SIZE +#define HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ PAGE_SIZE +#define HNS_ROCE_V2_PAGE_SIZE_SUPPORTED 0xFFFF000 +#define HNS_ROCE_V2_MAX_INNER_MTPT_NUM 2 +#define HNS_ROCE_INVALID_LKEY 0x100 +#define HNS_ROCE_CMQ_TX_TIMEOUT 30000 +#define HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE 2 +#define HNS_ROCE_V2_RSV_QPS 8 + +#define HNS_ROCE_V2_HW_RST_TIMEOUT 1000 +#define HNS_ROCE_V2_HW_RST_UNINT_DELAY 100 + +#define HNS_ROCE_V2_HW_RST_COMPLETION_WAIT 20 + +#define HNS_ROCE_CONTEXT_HOP_NUM 1 +#define HNS_ROCE_SCCC_HOP_NUM 1 +#define HNS_ROCE_MTT_HOP_NUM 1 +#define HNS_ROCE_CQE_HOP_NUM 1 +#define HNS_ROCE_SRQWQE_HOP_NUM 1 +#define HNS_ROCE_PBL_HOP_NUM 2 +#define HNS_ROCE_EQE_HOP_NUM 2 +#define HNS_ROCE_IDX_HOP_NUM 1 +#define HNS_ROCE_SQWQE_HOP_NUM 2 +#define HNS_ROCE_EXT_SGE_HOP_NUM 1 +#define HNS_ROCE_RQWQE_HOP_NUM 2 + +#define HNS_ROCE_BA_PG_SZ_SUPPORTED_256K 6 +#define HNS_ROCE_BA_PG_SZ_SUPPORTED_16K 2 +#define HNS_ROCE_V2_GID_INDEX_NUM 256 + +#define HNS_ROCE_V2_TABLE_CHUNK_SIZE (1 << 18) + +#define HNS_ROCE_CMD_FLAG_IN_VALID_SHIFT 0 +#define HNS_ROCE_CMD_FLAG_OUT_VALID_SHIFT 1 +#define HNS_ROCE_CMD_FLAG_NEXT_SHIFT 2 +#define HNS_ROCE_CMD_FLAG_WR_OR_RD_SHIFT 3 +#define HNS_ROCE_CMD_FLAG_NO_INTR_SHIFT 4 +#define HNS_ROCE_CMD_FLAG_ERR_INTR_SHIFT 5 + +#define HNS_ROCE_CMD_FLAG_IN BIT(HNS_ROCE_CMD_FLAG_IN_VALID_SHIFT) +#define HNS_ROCE_CMD_FLAG_OUT BIT(HNS_ROCE_CMD_FLAG_OUT_VALID_SHIFT) +#define HNS_ROCE_CMD_FLAG_NEXT BIT(HNS_ROCE_CMD_FLAG_NEXT_SHIFT) +#define HNS_ROCE_CMD_FLAG_WR BIT(HNS_ROCE_CMD_FLAG_WR_OR_RD_SHIFT) +#define HNS_ROCE_CMD_FLAG_NO_INTR BIT(HNS_ROCE_CMD_FLAG_NO_INTR_SHIFT) +#define HNS_ROCE_CMD_FLAG_ERR_INTR BIT(HNS_ROCE_CMD_FLAG_ERR_INTR_SHIFT) + +#define HNS_ROCE_CMQ_DESC_NUM_S 3 + +#define HNS_ROCE_CMQ_SCC_CLR_DONE_CNT 5 + +#define check_whether_last_step(hop_num, step_idx) \ + ((step_idx == 0 && hop_num == HNS_ROCE_HOP_NUM_0) || \ + (step_idx == 1 && hop_num == 1) || \ + (step_idx == 2 && hop_num == 2)) +#define HNS_ICL_SWITCH_CMD_ROCEE_SEL_SHIFT 0 +#define HNS_ICL_SWITCH_CMD_ROCEE_SEL BIT(HNS_ICL_SWITCH_CMD_ROCEE_SEL_SHIFT) + +#define CMD_CSQ_DESC_NUM 1024 +#define CMD_CRQ_DESC_NUM 1024 + +enum { + NO_ARMED = 0x0, + REG_NXT_CEQE = 0x2, + REG_NXT_SE_CEQE = 0x3 +}; + +#define V2_CQ_DB_REQ_NOT_SOL 0 +#define V2_CQ_DB_REQ_NOT 1 + +#define V2_CQ_STATE_VALID 1 +#define V2_QKEY_VAL 0x80010000 + +#define GID_LEN_V2 16 + +#define HNS_ROCE_V2_CQE_QPN_MASK 0xfffff + +enum { + HNS_ROCE_V2_WQE_OP_SEND = 0x0, + HNS_ROCE_V2_WQE_OP_SEND_WITH_INV = 0x1, + HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM = 0x2, + HNS_ROCE_V2_WQE_OP_RDMA_WRITE = 0x3, + HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM = 0x4, + HNS_ROCE_V2_WQE_OP_RDMA_READ = 0x5, + HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP = 0x6, + HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD = 0x7, + HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP = 0x8, + HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD = 0x9, + HNS_ROCE_V2_WQE_OP_FAST_REG_PMR = 0xa, + HNS_ROCE_V2_WQE_OP_LOCAL_INV = 0xb, + HNS_ROCE_V2_WQE_OP_BIND_MW = 0xc, + HNS_ROCE_V2_WQE_OP_MASK = 0x1f, +}; + +enum { + /* rq operations */ + HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM = 0x0, + HNS_ROCE_V2_OPCODE_SEND = 0x1, + HNS_ROCE_V2_OPCODE_SEND_WITH_IMM = 0x2, + HNS_ROCE_V2_OPCODE_SEND_WITH_INV = 0x3, +}; + +enum { + HNS_ROCE_V2_SQ_DB = 0x0, + HNS_ROCE_V2_RQ_DB = 0x1, + HNS_ROCE_V2_SRQ_DB = 0x2, + HNS_ROCE_V2_CQ_DB_PTR = 0x3, + HNS_ROCE_V2_CQ_DB_NTR = 0x4, +}; + +enum { + HNS_ROCE_CQE_V2_SUCCESS = 0x00, + HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR = 0x01, + HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR = 0x02, + HNS_ROCE_CQE_V2_LOCAL_PROT_ERR = 0x04, + HNS_ROCE_CQE_V2_WR_FLUSH_ERR = 0x05, + HNS_ROCE_CQE_V2_MW_BIND_ERR = 0x06, + HNS_ROCE_CQE_V2_BAD_RESP_ERR = 0x10, + HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR = 0x11, + HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR = 0x12, + HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR = 0x13, + HNS_ROCE_CQE_V2_REMOTE_OP_ERR = 0x14, + HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR = 0x15, + HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR = 0x16, + HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR = 0x22, + HNS_ROCE_CQE_V2_GENERAL_ERR = 0x23, + + HNS_ROCE_V2_CQE_STATUS_MASK = 0xff, +}; + +/* CMQ command */ +enum hns_roce_opcode_type { + HNS_QUERY_FW_VER = 0x0001, + HNS_ROCE_OPC_QUERY_HW_VER = 0x8000, + HNS_ROCE_OPC_CFG_GLOBAL_PARAM = 0x8001, + HNS_ROCE_OPC_ALLOC_PF_RES = 0x8004, + HNS_ROCE_OPC_QUERY_PF_RES = 0x8400, + HNS_ROCE_OPC_ALLOC_VF_RES = 0x8401, + HNS_ROCE_OPC_CFG_EXT_LLM = 0x8403, + HNS_ROCE_OPC_CFG_TMOUT_LLM = 0x8404, + HNS_ROCE_OPC_QUERY_PF_TIMER_RES = 0x8406, + HNS_ROCE_OPC_QUERY_PF_CAPS_NUM = 0x8408, + HNS_ROCE_OPC_CFG_ENTRY_SIZE = 0x8409, + HNS_ROCE_OPC_CFG_SGID_TB = 0x8500, + HNS_ROCE_OPC_CFG_SMAC_TB = 0x8501, + HNS_ROCE_OPC_POST_MB = 0x8504, + HNS_ROCE_OPC_QUERY_MB_ST = 0x8505, + HNS_ROCE_OPC_CFG_BT_ATTR = 0x8506, + HNS_ROCE_OPC_FUNC_CLEAR = 0x8508, + HNS_ROCE_OPC_CLR_SCCC = 0x8509, + HNS_ROCE_OPC_QUERY_SCCC = 0x850a, + HNS_ROCE_OPC_RESET_SCCC = 0x850b, + HNS_SWITCH_PARAMETER_CFG = 0x1033, +}; + +enum { + TYPE_CRQ, + TYPE_CSQ, +}; + +enum hns_roce_cmd_return_status { + CMD_EXEC_SUCCESS = 0, + CMD_NO_AUTH = 1, + CMD_NOT_EXEC = 2, + CMD_QUEUE_FULL = 3, +}; + +enum hns_roce_sgid_type { + GID_TYPE_FLAG_ROCE_V1 = 0, + GID_TYPE_FLAG_ROCE_V2_IPV4, + GID_TYPE_FLAG_ROCE_V2_IPV6, +}; + +struct hns_roce_v2_cq_context { + __le32 byte_4_pg_ceqn; + __le32 byte_8_cqn; + __le32 cqe_cur_blk_addr; + __le32 byte_16_hop_addr; + __le32 cqe_nxt_blk_addr; + __le32 byte_24_pgsz_addr; + __le32 byte_28_cq_pi; + __le32 byte_32_cq_ci; + __le32 cqe_ba; + __le32 byte_40_cqe_ba; + __le32 byte_44_db_record; + __le32 db_record_addr; + __le32 byte_52_cqe_cnt; + __le32 byte_56_cqe_period_maxcnt; + __le32 cqe_report_timer; + __le32 byte_64_se_cqe_idx; +}; +#define HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM 0x0 +#define HNS_ROCE_V2_CQ_DEFAULT_INTERVAL 0x0 + +#define V2_CQC_BYTE_4_CQ_ST_S 0 +#define V2_CQC_BYTE_4_CQ_ST_M GENMASK(1, 0) + +#define V2_CQC_BYTE_4_POLL_S 2 + +#define V2_CQC_BYTE_4_SE_S 3 + +#define V2_CQC_BYTE_4_OVER_IGNORE_S 4 + +#define V2_CQC_BYTE_4_COALESCE_S 5 + +#define V2_CQC_BYTE_4_ARM_ST_S 6 +#define V2_CQC_BYTE_4_ARM_ST_M GENMASK(7, 6) + +#define V2_CQC_BYTE_4_SHIFT_S 8 +#define V2_CQC_BYTE_4_SHIFT_M GENMASK(12, 8) + +#define V2_CQC_BYTE_4_CMD_SN_S 13 +#define V2_CQC_BYTE_4_CMD_SN_M GENMASK(14, 13) + +#define V2_CQC_BYTE_4_CEQN_S 15 +#define V2_CQC_BYTE_4_CEQN_M GENMASK(23, 15) + +#define V2_CQC_BYTE_4_PAGE_OFFSET_S 24 +#define V2_CQC_BYTE_4_PAGE_OFFSET_M GENMASK(31, 24) + +#define V2_CQC_BYTE_8_CQN_S 0 +#define V2_CQC_BYTE_8_CQN_M GENMASK(23, 0) + +#define V2_CQC_BYTE_8_CQE_SIZE_S 27 +#define V2_CQC_BYTE_8_CQE_SIZE_M GENMASK(28, 27) + +#define V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S 0 +#define V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M GENMASK(19, 0) + +#define V2_CQC_BYTE_16_CQE_HOP_NUM_S 30 +#define V2_CQC_BYTE_16_CQE_HOP_NUM_M GENMASK(31, 30) + +#define V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S 0 +#define V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M GENMASK(19, 0) + +#define V2_CQC_BYTE_24_CQE_BA_PG_SZ_S 24 +#define V2_CQC_BYTE_24_CQE_BA_PG_SZ_M GENMASK(27, 24) + +#define V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S 28 +#define V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M GENMASK(31, 28) + +#define V2_CQC_BYTE_28_CQ_PRODUCER_IDX_S 0 +#define V2_CQC_BYTE_28_CQ_PRODUCER_IDX_M GENMASK(23, 0) + +#define V2_CQC_BYTE_32_CQ_CONSUMER_IDX_S 0 +#define V2_CQC_BYTE_32_CQ_CONSUMER_IDX_M GENMASK(23, 0) + +#define V2_CQC_BYTE_40_CQE_BA_S 0 +#define V2_CQC_BYTE_40_CQE_BA_M GENMASK(28, 0) + +#define V2_CQC_BYTE_44_DB_RECORD_EN_S 0 + +#define V2_CQC_BYTE_44_DB_RECORD_ADDR_S 1 +#define V2_CQC_BYTE_44_DB_RECORD_ADDR_M GENMASK(31, 1) + +#define V2_CQC_BYTE_52_CQE_CNT_S 0 +#define V2_CQC_BYTE_52_CQE_CNT_M GENMASK(23, 0) + +#define V2_CQC_BYTE_56_CQ_MAX_CNT_S 0 +#define V2_CQC_BYTE_56_CQ_MAX_CNT_M GENMASK(15, 0) + +#define V2_CQC_BYTE_56_CQ_PERIOD_S 16 +#define V2_CQC_BYTE_56_CQ_PERIOD_M GENMASK(31, 16) + +#define V2_CQC_BYTE_64_SE_CQE_IDX_S 0 +#define V2_CQC_BYTE_64_SE_CQE_IDX_M GENMASK(23, 0) + +struct hns_roce_srq_context { + __le32 byte_4_srqn_srqst; + __le32 byte_8_limit_wl; + __le32 byte_12_xrcd; + __le32 byte_16_pi_ci; + __le32 wqe_bt_ba; + __le32 byte_24_wqe_bt_ba; + __le32 byte_28_rqws_pd; + __le32 idx_bt_ba; + __le32 rsv_idx_bt_ba; + __le32 idx_cur_blk_addr; + __le32 byte_44_idxbufpgsz_addr; + __le32 idx_nxt_blk_addr; + __le32 rsv_idxnxtblkaddr; + __le32 byte_56_xrc_cqn; + __le32 db_record_addr_record_en; + __le32 db_record_addr; +}; + +#define SRQC_BYTE_4_SRQ_ST_S 0 +#define SRQC_BYTE_4_SRQ_ST_M GENMASK(1, 0) + +#define SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S 2 +#define SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M GENMASK(3, 2) + +#define SRQC_BYTE_4_SRQ_SHIFT_S 4 +#define SRQC_BYTE_4_SRQ_SHIFT_M GENMASK(7, 4) + +#define SRQC_BYTE_4_SRQN_S 8 +#define SRQC_BYTE_4_SRQN_M GENMASK(31, 8) + +#define SRQC_BYTE_8_SRQ_LIMIT_WL_S 0 +#define SRQC_BYTE_8_SRQ_LIMIT_WL_M GENMASK(15, 0) + +#define SRQC_BYTE_12_SRQ_XRCD_S 0 +#define SRQC_BYTE_12_SRQ_XRCD_M GENMASK(23, 0) + +#define SRQC_BYTE_16_SRQ_PRODUCER_IDX_S 0 +#define SRQC_BYTE_16_SRQ_PRODUCER_IDX_M GENMASK(15, 0) + +#define SRQC_BYTE_16_SRQ_CONSUMER_IDX_S 0 +#define SRQC_BYTE_16_SRQ_CONSUMER_IDX_M GENMASK(31, 16) + +#define SRQC_BYTE_24_SRQ_WQE_BT_BA_S 0 +#define SRQC_BYTE_24_SRQ_WQE_BT_BA_M GENMASK(28, 0) + +#define SRQC_BYTE_28_PD_S 0 +#define SRQC_BYTE_28_PD_M GENMASK(23, 0) + +#define SRQC_BYTE_28_RQWS_S 24 +#define SRQC_BYTE_28_RQWS_M GENMASK(27, 24) + +#define SRQC_BYTE_36_SRQ_IDX_BT_BA_S 0 +#define SRQC_BYTE_36_SRQ_IDX_BT_BA_M GENMASK(28, 0) + +#define SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S 0 +#define SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M GENMASK(19, 0) + +#define SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S 22 +#define SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M GENMASK(23, 22) + +#define SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S 24 +#define SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M GENMASK(27, 24) + +#define SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S 28 +#define SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M GENMASK(31, 28) + +#define SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S 0 +#define SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M GENMASK(19, 0) + +#define SRQC_BYTE_56_SRQ_XRC_CQN_S 0 +#define SRQC_BYTE_56_SRQ_XRC_CQN_M GENMASK(23, 0) + +#define SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S 24 +#define SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M GENMASK(27, 24) + +#define SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S 28 +#define SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M GENMASK(31, 28) + +#define SRQC_BYTE_60_SRQ_RECORD_EN_S 0 + +#define SRQC_BYTE_60_SRQ_DB_RECORD_ADDR_S 1 +#define SRQC_BYTE_60_SRQ_DB_RECORD_ADDR_M GENMASK(31, 1) + +enum { + V2_MPT_ST_VALID = 0x1, + V2_MPT_ST_FREE = 0x2, +}; + +enum hns_roce_v2_qp_state { + HNS_ROCE_QP_ST_RST, + HNS_ROCE_QP_ST_INIT, + HNS_ROCE_QP_ST_RTR, + HNS_ROCE_QP_ST_RTS, + HNS_ROCE_QP_ST_SQD, + HNS_ROCE_QP_ST_SQER, + HNS_ROCE_QP_ST_ERR, + HNS_ROCE_QP_ST_SQ_DRAINING, + HNS_ROCE_QP_NUM_ST +}; + +struct hns_roce_v2_qp_context { + __le32 byte_4_sqpn_tst; + __le32 wqe_sge_ba; + __le32 byte_12_sq_hop; + __le32 byte_16_buf_ba_pg_sz; + __le32 byte_20_smac_sgid_idx; + __le32 byte_24_mtu_tc; + __le32 byte_28_at_fl; + u8 dgid[GID_LEN_V2]; + __le32 dmac; + __le32 byte_52_udpspn_dmac; + __le32 byte_56_dqpn_err; + __le32 byte_60_qpst_tempid; + __le32 qkey_xrcd; + __le32 byte_68_rq_db; + __le32 rq_db_record_addr; + __le32 byte_76_srqn_op_en; + __le32 byte_80_rnr_rx_cqn; + __le32 byte_84_rq_ci_pi; + __le32 rq_cur_blk_addr; + __le32 byte_92_srq_info; + __le32 byte_96_rx_reqmsn; + __le32 rq_nxt_blk_addr; + __le32 byte_104_rq_sge; + __le32 byte_108_rx_reqepsn; + __le32 rq_rnr_timer; + __le32 rx_msg_len; + __le32 rx_rkey_pkt_info; + __le64 rx_va; + __le32 byte_132_trrl; + __le32 trrl_ba; + __le32 byte_140_raq; + __le32 byte_144_raq; + __le32 byte_148_raq; + __le32 byte_152_raq; + __le32 byte_156_raq; + __le32 byte_160_sq_ci_pi; + __le32 sq_cur_blk_addr; + __le32 byte_168_irrl_idx; + __le32 byte_172_sq_psn; + __le32 byte_176_msg_pktn; + __le32 sq_cur_sge_blk_addr; + __le32 byte_184_irrl_idx; + __le32 cur_sge_offset; + __le32 byte_192_ext_sge; + __le32 byte_196_sq_psn; + __le32 byte_200_sq_max; + __le32 irrl_ba; + __le32 byte_208_irrl; + __le32 byte_212_lsn; + __le32 sq_timer; + __le32 byte_220_retry_psn_msn; + __le32 byte_224_retry_msg; + __le32 rx_sq_cur_blk_addr; + __le32 byte_232_irrl_sge; + __le32 irrl_cur_sge_offset; + __le32 byte_240_irrl_tail; + __le32 byte_244_rnr_rxack; + __le32 byte_248_ack_psn; + __le32 byte_252_err_txcqn; + __le32 byte_256_sqflush_rqcqe; + __le32 ext[64]; +}; + +#define V2_QPC_BYTE_4_TST_S 0 +#define V2_QPC_BYTE_4_TST_M GENMASK(2, 0) + +#define V2_QPC_BYTE_4_SGE_SHIFT_S 3 +#define V2_QPC_BYTE_4_SGE_SHIFT_M GENMASK(7, 3) + +#define V2_QPC_BYTE_4_SQPN_S 8 +#define V2_QPC_BYTE_4_SQPN_M GENMASK(31, 8) + +#define V2_QPC_BYTE_12_WQE_SGE_BA_S 0 +#define V2_QPC_BYTE_12_WQE_SGE_BA_M GENMASK(28, 0) + +#define V2_QPC_BYTE_12_SQ_HOP_NUM_S 29 +#define V2_QPC_BYTE_12_SQ_HOP_NUM_M GENMASK(30, 29) + +#define V2_QPC_BYTE_12_RSVD_LKEY_EN_S 31 + +#define V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S 0 +#define V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M GENMASK(3, 0) + +#define V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S 4 +#define V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M GENMASK(7, 4) + +#define V2_QPC_BYTE_16_PD_S 8 +#define V2_QPC_BYTE_16_PD_M GENMASK(31, 8) + +#define V2_QPC_BYTE_20_RQ_HOP_NUM_S 0 +#define V2_QPC_BYTE_20_RQ_HOP_NUM_M GENMASK(1, 0) + +#define V2_QPC_BYTE_20_SGE_HOP_NUM_S 2 +#define V2_QPC_BYTE_20_SGE_HOP_NUM_M GENMASK(3, 2) + +#define V2_QPC_BYTE_20_RQWS_S 4 +#define V2_QPC_BYTE_20_RQWS_M GENMASK(7, 4) + +#define V2_QPC_BYTE_20_SQ_SHIFT_S 8 +#define V2_QPC_BYTE_20_SQ_SHIFT_M GENMASK(11, 8) + +#define V2_QPC_BYTE_20_RQ_SHIFT_S 12 +#define V2_QPC_BYTE_20_RQ_SHIFT_M GENMASK(15, 12) + +#define V2_QPC_BYTE_20_SGID_IDX_S 16 +#define V2_QPC_BYTE_20_SGID_IDX_M GENMASK(23, 16) + +#define V2_QPC_BYTE_20_SMAC_IDX_S 24 +#define V2_QPC_BYTE_20_SMAC_IDX_M GENMASK(31, 24) + +#define V2_QPC_BYTE_24_HOP_LIMIT_S 0 +#define V2_QPC_BYTE_24_HOP_LIMIT_M GENMASK(7, 0) + +#define V2_QPC_BYTE_24_TC_S 8 +#define V2_QPC_BYTE_24_TC_M GENMASK(15, 8) + +#define V2_QPC_BYTE_24_VLAN_ID_S 16 +#define V2_QPC_BYTE_24_VLAN_ID_M GENMASK(27, 16) + +#define V2_QPC_BYTE_24_MTU_S 28 +#define V2_QPC_BYTE_24_MTU_M GENMASK(31, 28) + +#define V2_QPC_BYTE_28_FL_S 0 +#define V2_QPC_BYTE_28_FL_M GENMASK(19, 0) + +#define V2_QPC_BYTE_28_SL_S 20 +#define V2_QPC_BYTE_28_SL_M GENMASK(23, 20) + +#define V2_QPC_BYTE_28_CNP_TX_FLAG_S 24 + +#define V2_QPC_BYTE_28_CE_FLAG_S 25 + +#define V2_QPC_BYTE_28_LBI_S 26 + +#define V2_QPC_BYTE_28_AT_S 27 +#define V2_QPC_BYTE_28_AT_M GENMASK(31, 27) + +#define V2_QPC_BYTE_52_DMAC_S 0 +#define V2_QPC_BYTE_52_DMAC_M GENMASK(15, 0) + +#define V2_QPC_BYTE_52_UDPSPN_S 16 +#define V2_QPC_BYTE_52_UDPSPN_M GENMASK(31, 16) + +#define V2_QPC_BYTE_56_DQPN_S 0 +#define V2_QPC_BYTE_56_DQPN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_56_SQ_TX_ERR_S 24 +#define V2_QPC_BYTE_56_SQ_RX_ERR_S 25 +#define V2_QPC_BYTE_56_RQ_TX_ERR_S 26 +#define V2_QPC_BYTE_56_RQ_RX_ERR_S 27 + +#define V2_QPC_BYTE_56_LP_PKTN_INI_S 28 +#define V2_QPC_BYTE_56_LP_PKTN_INI_M GENMASK(31, 28) + +#define V2_QPC_BYTE_60_TEMPID_S 0 +#define V2_QPC_BYTE_60_TEMPID_M GENMASK(7, 0) + +#define V2_QPC_BYTE_60_SCC_TOKEN_S 8 +#define V2_QPC_BYTE_60_SCC_TOKEN_M GENMASK(26, 8) + +#define V2_QPC_BYTE_60_SQ_DB_DOING_S 27 + +#define V2_QPC_BYTE_60_RQ_DB_DOING_S 28 + +#define V2_QPC_BYTE_60_QP_ST_S 29 +#define V2_QPC_BYTE_60_QP_ST_M GENMASK(31, 29) + +#define V2_QPC_BYTE_68_RQ_RECORD_EN_S 0 + +#define V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S 1 +#define V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M GENMASK(31, 1) + +#define V2_QPC_BYTE_76_SRQN_S 0 +#define V2_QPC_BYTE_76_SRQN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_76_SRQ_EN_S 24 + +#define V2_QPC_BYTE_76_RRE_S 25 + +#define V2_QPC_BYTE_76_RWE_S 26 + +#define V2_QPC_BYTE_76_ATE_S 27 + +#define V2_QPC_BYTE_76_RQIE_S 28 +#define V2_QPC_BYTE_76_EXT_ATE_S 29 +#define V2_QPC_BYTE_76_RQ_VLAN_EN_S 30 +#define V2_QPC_BYTE_80_RX_CQN_S 0 +#define V2_QPC_BYTE_80_RX_CQN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_80_MIN_RNR_TIME_S 27 +#define V2_QPC_BYTE_80_MIN_RNR_TIME_M GENMASK(31, 27) + +#define V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S 0 +#define V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M GENMASK(15, 0) + +#define V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S 16 +#define V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M GENMASK(31, 16) + +#define V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S 0 +#define V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M GENMASK(19, 0) + +#define V2_QPC_BYTE_92_SRQ_INFO_S 20 +#define V2_QPC_BYTE_92_SRQ_INFO_M GENMASK(31, 20) + +#define V2_QPC_BYTE_96_RX_REQ_MSN_S 0 +#define V2_QPC_BYTE_96_RX_REQ_MSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S 0 +#define V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M GENMASK(19, 0) + +#define V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S 24 +#define V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M GENMASK(31, 24) + +#define V2_QPC_BYTE_108_INV_CREDIT_S 0 + +#define V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S 3 + +#define V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S 4 +#define V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M GENMASK(6, 4) + +#define V2_QPC_BYTE_108_RX_REQ_RNR_S 7 + +#define V2_QPC_BYTE_108_RX_REQ_EPSN_S 8 +#define V2_QPC_BYTE_108_RX_REQ_EPSN_M GENMASK(31, 8) + +#define V2_QPC_BYTE_132_TRRL_HEAD_MAX_S 0 +#define V2_QPC_BYTE_132_TRRL_HEAD_MAX_M GENMASK(7, 0) + +#define V2_QPC_BYTE_132_TRRL_TAIL_MAX_S 8 +#define V2_QPC_BYTE_132_TRRL_TAIL_MAX_M GENMASK(15, 8) + +#define V2_QPC_BYTE_132_TRRL_BA_S 16 +#define V2_QPC_BYTE_132_TRRL_BA_M GENMASK(31, 16) + +#define V2_QPC_BYTE_140_TRRL_BA_S 0 +#define V2_QPC_BYTE_140_TRRL_BA_M GENMASK(11, 0) + +#define V2_QPC_BYTE_140_RR_MAX_S 12 +#define V2_QPC_BYTE_140_RR_MAX_M GENMASK(14, 12) + +#define V2_QPC_BYTE_140_RQ_RTY_WAIT_DO_S 15 + +#define V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S 16 +#define V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M GENMASK(23, 16) + +#define V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S 24 +#define V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M GENMASK(31, 24) + +#define V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S 0 +#define V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_144_RAQ_CREDIT_S 25 +#define V2_QPC_BYTE_144_RAQ_CREDIT_M GENMASK(29, 25) + +#define V2_QPC_BYTE_144_RESP_RTY_FLG_S 31 + +#define V2_QPC_BYTE_148_RQ_MSN_S 0 +#define V2_QPC_BYTE_148_RQ_MSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_148_RAQ_SYNDROME_S 24 +#define V2_QPC_BYTE_148_RAQ_SYNDROME_M GENMASK(31, 24) + +#define V2_QPC_BYTE_152_RAQ_PSN_S 0 +#define V2_QPC_BYTE_152_RAQ_PSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S 24 +#define V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M GENMASK(31, 24) + +#define V2_QPC_BYTE_156_RAQ_USE_PKTN_S 0 +#define V2_QPC_BYTE_156_RAQ_USE_PKTN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S 0 +#define V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M GENMASK(15, 0) + +#define V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S 16 +#define V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M GENMASK(31, 16) + +#define V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S 0 +#define V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M GENMASK(19, 0) + +#define V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S 20 + +#define V2_QPC_BYTE_168_SQ_INVLD_FLG_S 21 + +#define V2_QPC_BYTE_168_LP_SGEN_INI_S 22 +#define V2_QPC_BYTE_168_LP_SGEN_INI_M GENMASK(23, 22) + +#define V2_QPC_BYTE_168_SQ_VLAN_EN_S 24 +#define V2_QPC_BYTE_168_POLL_DB_WAIT_DO_S 25 +#define V2_QPC_BYTE_168_SCC_TOKEN_FORBID_SQ_DEQ_S 26 +#define V2_QPC_BYTE_168_WAIT_ACK_TIMEOUT_S 27 +#define V2_QPC_BYTE_168_IRRL_IDX_LSB_S 28 +#define V2_QPC_BYTE_168_IRRL_IDX_LSB_M GENMASK(31, 28) + +#define V2_QPC_BYTE_172_ACK_REQ_FREQ_S 0 +#define V2_QPC_BYTE_172_ACK_REQ_FREQ_M GENMASK(5, 0) + +#define V2_QPC_BYTE_172_MSG_RNR_FLG_S 6 + +#define V2_QPC_BYTE_172_FRE_S 7 + +#define V2_QPC_BYTE_172_SQ_CUR_PSN_S 8 +#define V2_QPC_BYTE_172_SQ_CUR_PSN_M GENMASK(31, 8) + +#define V2_QPC_BYTE_176_MSG_USE_PKTN_S 0 +#define V2_QPC_BYTE_176_MSG_USE_PKTN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_176_IRRL_HEAD_PRE_S 24 +#define V2_QPC_BYTE_176_IRRL_HEAD_PRE_M GENMASK(31, 24) + +#define V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S 0 +#define V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M GENMASK(19, 0) + +#define V2_QPC_BYTE_184_IRRL_IDX_MSB_S 20 +#define V2_QPC_BYTE_184_IRRL_IDX_MSB_M GENMASK(31, 20) + +#define V2_QPC_BYTE_192_CUR_SGE_IDX_S 0 +#define V2_QPC_BYTE_192_CUR_SGE_IDX_M GENMASK(23, 0) + +#define V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S 24 +#define V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M GENMASK(31, 24) + +#define V2_QPC_BYTE_196_IRRL_HEAD_S 0 +#define V2_QPC_BYTE_196_IRRL_HEAD_M GENMASK(7, 0) + +#define V2_QPC_BYTE_196_SQ_MAX_PSN_S 8 +#define V2_QPC_BYTE_196_SQ_MAX_PSN_M GENMASK(31, 8) + +#define V2_QPC_BYTE_200_SQ_MAX_IDX_S 0 +#define V2_QPC_BYTE_200_SQ_MAX_IDX_M GENMASK(15, 0) + +#define V2_QPC_BYTE_200_LCL_OPERATED_CNT_S 16 +#define V2_QPC_BYTE_200_LCL_OPERATED_CNT_M GENMASK(31, 16) + +#define V2_QPC_BYTE_208_IRRL_BA_S 0 +#define V2_QPC_BYTE_208_IRRL_BA_M GENMASK(25, 0) + +#define V2_QPC_BYTE_208_PKT_RNR_FLG_S 26 + +#define V2_QPC_BYTE_208_PKT_RTY_FLG_S 27 + +#define V2_QPC_BYTE_208_RMT_E2E_S 28 + +#define V2_QPC_BYTE_208_SR_MAX_S 29 +#define V2_QPC_BYTE_208_SR_MAX_M GENMASK(31, 29) + +#define V2_QPC_BYTE_212_LSN_S 0 +#define V2_QPC_BYTE_212_LSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_212_RETRY_NUM_INIT_S 24 +#define V2_QPC_BYTE_212_RETRY_NUM_INIT_M GENMASK(26, 24) + +#define V2_QPC_BYTE_212_CHECK_FLG_S 27 +#define V2_QPC_BYTE_212_CHECK_FLG_M GENMASK(28, 27) + +#define V2_QPC_BYTE_212_RETRY_CNT_S 29 +#define V2_QPC_BYTE_212_RETRY_CNT_M GENMASK(31, 29) + +#define V2_QPC_BYTE_220_RETRY_MSG_MSN_S 0 +#define V2_QPC_BYTE_220_RETRY_MSG_MSN_M GENMASK(15, 0) + +#define V2_QPC_BYTE_220_RETRY_MSG_PSN_S 16 +#define V2_QPC_BYTE_220_RETRY_MSG_PSN_M GENMASK(31, 16) + +#define V2_QPC_BYTE_224_RETRY_MSG_PSN_S 0 +#define V2_QPC_BYTE_224_RETRY_MSG_PSN_M GENMASK(7, 0) + +#define V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S 8 +#define V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M GENMASK(31, 8) + +#define V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S 0 +#define V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M GENMASK(19, 0) + +#define V2_QPC_BYTE_232_IRRL_SGE_IDX_S 20 +#define V2_QPC_BYTE_232_IRRL_SGE_IDX_M GENMASK(28, 20) + +#define V2_QPC_BYTE_232_SO_LP_VLD_S 29 +#define V2_QPC_BYTE_232_FENCE_LP_VLD_S 30 +#define V2_QPC_BYTE_232_IRRL_LP_VLD_S 31 + +#define V2_QPC_BYTE_240_IRRL_TAIL_REAL_S 0 +#define V2_QPC_BYTE_240_IRRL_TAIL_REAL_M GENMASK(7, 0) + +#define V2_QPC_BYTE_240_IRRL_TAIL_RD_S 8 +#define V2_QPC_BYTE_240_IRRL_TAIL_RD_M GENMASK(15, 8) + +#define V2_QPC_BYTE_240_RX_ACK_MSN_S 16 +#define V2_QPC_BYTE_240_RX_ACK_MSN_M GENMASK(31, 16) + +#define V2_QPC_BYTE_244_RX_ACK_EPSN_S 0 +#define V2_QPC_BYTE_244_RX_ACK_EPSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_244_RNR_NUM_INIT_S 24 +#define V2_QPC_BYTE_244_RNR_NUM_INIT_M GENMASK(26, 24) + +#define V2_QPC_BYTE_244_RNR_CNT_S 27 +#define V2_QPC_BYTE_244_RNR_CNT_M GENMASK(29, 27) + +#define V2_QPC_BYTE_244_LCL_OP_FLG_S 30 +#define V2_QPC_BYTE_244_IRRL_RD_FLG_S 31 + +#define V2_QPC_BYTE_248_IRRL_PSN_S 0 +#define V2_QPC_BYTE_248_IRRL_PSN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_248_ACK_PSN_ERR_S 24 + +#define V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S 25 +#define V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M GENMASK(26, 25) + +#define V2_QPC_BYTE_248_IRRL_PSN_VLD_S 27 + +#define V2_QPC_BYTE_248_RNR_RETRY_FLAG_S 28 + +#define V2_QPC_BYTE_248_CQ_ERR_IND_S 31 + +#define V2_QPC_BYTE_252_TX_CQN_S 0 +#define V2_QPC_BYTE_252_TX_CQN_M GENMASK(23, 0) + +#define V2_QPC_BYTE_252_SIG_TYPE_S 24 + +#define V2_QPC_BYTE_252_ERR_TYPE_S 25 +#define V2_QPC_BYTE_252_ERR_TYPE_M GENMASK(31, 25) + +#define V2_QPC_BYTE_256_RQ_CQE_IDX_S 0 +#define V2_QPC_BYTE_256_RQ_CQE_IDX_M GENMASK(15, 0) + +#define V2_QPC_BYTE_256_SQ_FLUSH_IDX_S 16 +#define V2_QPC_BYTE_256_SQ_FLUSH_IDX_M GENMASK(31, 16) + +#define V2_QP_RWE_S 1 /* rdma write enable */ +#define V2_QP_RRE_S 2 /* rdma read enable */ +#define V2_QP_ATE_S 3 /* rdma atomic enable */ + +struct hns_roce_v2_cqe { + __le32 byte_4; + union { + __le32 rkey; + __le32 immtdata; + }; + __le32 byte_12; + __le32 byte_16; + __le32 byte_cnt; + u8 smac[4]; + __le32 byte_28; + __le32 byte_32; + __le32 rsv[8]; +}; + +#define V2_CQE_BYTE_4_OPCODE_S 0 +#define V2_CQE_BYTE_4_OPCODE_M GENMASK(4, 0) + +#define V2_CQE_BYTE_4_RQ_INLINE_S 5 + +#define V2_CQE_BYTE_4_S_R_S 6 + +#define V2_CQE_BYTE_4_OWNER_S 7 + +#define V2_CQE_BYTE_4_STATUS_S 8 +#define V2_CQE_BYTE_4_STATUS_M GENMASK(15, 8) + +#define V2_CQE_BYTE_4_WQE_INDX_S 16 +#define V2_CQE_BYTE_4_WQE_INDX_M GENMASK(31, 16) + +#define V2_CQE_BYTE_12_XRC_SRQN_S 0 +#define V2_CQE_BYTE_12_XRC_SRQN_M GENMASK(23, 0) + +#define V2_CQE_BYTE_16_LCL_QPN_S 0 +#define V2_CQE_BYTE_16_LCL_QPN_M GENMASK(23, 0) + +#define V2_CQE_BYTE_16_SUB_STATUS_S 24 +#define V2_CQE_BYTE_16_SUB_STATUS_M GENMASK(31, 24) + +#define V2_CQE_BYTE_28_SMAC_4_S 0 +#define V2_CQE_BYTE_28_SMAC_4_M GENMASK(7, 0) + +#define V2_CQE_BYTE_28_SMAC_5_S 8 +#define V2_CQE_BYTE_28_SMAC_5_M GENMASK(15, 8) + +#define V2_CQE_BYTE_28_PORT_TYPE_S 16 +#define V2_CQE_BYTE_28_PORT_TYPE_M GENMASK(17, 16) + +#define V2_CQE_BYTE_28_VID_S 18 +#define V2_CQE_BYTE_28_VID_M GENMASK(29, 18) + +#define V2_CQE_BYTE_28_VID_VLD_S 30 + +#define V2_CQE_BYTE_32_RMT_QPN_S 0 +#define V2_CQE_BYTE_32_RMT_QPN_M GENMASK(23, 0) + +#define V2_CQE_BYTE_32_SL_S 24 +#define V2_CQE_BYTE_32_SL_M GENMASK(26, 24) + +#define V2_CQE_BYTE_32_PORTN_S 27 +#define V2_CQE_BYTE_32_PORTN_M GENMASK(29, 27) + +#define V2_CQE_BYTE_32_GRH_S 30 + +#define V2_CQE_BYTE_32_LPK_S 31 + +struct hns_roce_v2_mpt_entry { + __le32 byte_4_pd_hop_st; + __le32 byte_8_mw_cnt_en; + __le32 byte_12_mw_pa; + __le32 bound_lkey; + __le32 len_l; + __le32 len_h; + __le32 lkey; + __le32 va_l; + __le32 va_h; + __le32 pbl_size; + __le32 pbl_ba_l; + __le32 byte_48_mode_ba; + __le32 pa0_l; + __le32 byte_56_pa0_h; + __le32 pa1_l; + __le32 byte_64_buf_pa1; +}; + +#define V2_MPT_BYTE_4_MPT_ST_S 0 +#define V2_MPT_BYTE_4_MPT_ST_M GENMASK(1, 0) + +#define V2_MPT_BYTE_4_PBL_HOP_NUM_S 2 +#define V2_MPT_BYTE_4_PBL_HOP_NUM_M GENMASK(3, 2) + +#define V2_MPT_BYTE_4_PBL_BA_PG_SZ_S 4 +#define V2_MPT_BYTE_4_PBL_BA_PG_SZ_M GENMASK(7, 4) + +#define V2_MPT_BYTE_4_PD_S 8 +#define V2_MPT_BYTE_4_PD_M GENMASK(31, 8) + +#define V2_MPT_BYTE_8_RA_EN_S 0 + +#define V2_MPT_BYTE_8_R_INV_EN_S 1 + +#define V2_MPT_BYTE_8_L_INV_EN_S 2 + +#define V2_MPT_BYTE_8_BIND_EN_S 3 + +#define V2_MPT_BYTE_8_ATOMIC_EN_S 4 + +#define V2_MPT_BYTE_8_RR_EN_S 5 + +#define V2_MPT_BYTE_8_RW_EN_S 6 + +#define V2_MPT_BYTE_8_LW_EN_S 7 + +#define V2_MPT_BYTE_8_MW_CNT_S 8 +#define V2_MPT_BYTE_8_MW_CNT_M GENMASK(31, 8) + +#define V2_MPT_BYTE_12_FRE_S 0 + +#define V2_MPT_BYTE_12_PA_S 1 + +#define V2_MPT_BYTE_12_MR_MW_S 4 + +#define V2_MPT_BYTE_12_BPD_S 5 + +#define V2_MPT_BYTE_12_BQP_S 6 + +#define V2_MPT_BYTE_12_INNER_PA_VLD_S 7 + +#define V2_MPT_BYTE_12_MW_BIND_QPN_S 8 +#define V2_MPT_BYTE_12_MW_BIND_QPN_M GENMASK(31, 8) + +#define V2_MPT_BYTE_48_PBL_BA_H_S 0 +#define V2_MPT_BYTE_48_PBL_BA_H_M GENMASK(28, 0) + +#define V2_MPT_BYTE_48_BLK_MODE_S 29 + +#define V2_MPT_BYTE_56_PA0_H_S 0 +#define V2_MPT_BYTE_56_PA0_H_M GENMASK(25, 0) + +#define V2_MPT_BYTE_64_PA1_H_S 0 +#define V2_MPT_BYTE_64_PA1_H_M GENMASK(25, 0) + +#define V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S 28 +#define V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M GENMASK(31, 28) + +#define V2_DB_BYTE_4_TAG_S 0 +#define V2_DB_BYTE_4_TAG_M GENMASK(23, 0) + +#define V2_DB_BYTE_4_CMD_S 24 +#define V2_DB_BYTE_4_CMD_M GENMASK(27, 24) + +#define V2_DB_PARAMETER_IDX_S 0 +#define V2_DB_PARAMETER_IDX_M GENMASK(15, 0) + +#define V2_DB_PARAMETER_SL_S 16 +#define V2_DB_PARAMETER_SL_M GENMASK(18, 16) + +#define V2_CQ_DB_BYTE_4_TAG_S 0 +#define V2_CQ_DB_BYTE_4_TAG_M GENMASK(23, 0) + +#define V2_CQ_DB_BYTE_4_CMD_S 24 +#define V2_CQ_DB_BYTE_4_CMD_M GENMASK(27, 24) + +#define V2_CQ_DB_PARAMETER_CONS_IDX_S 0 +#define V2_CQ_DB_PARAMETER_CONS_IDX_M GENMASK(23, 0) + +#define V2_CQ_DB_PARAMETER_CMD_SN_S 25 +#define V2_CQ_DB_PARAMETER_CMD_SN_M GENMASK(26, 25) + +#define V2_CQ_DB_PARAMETER_NOTIFY_S 24 + +struct hns_roce_v2_ud_send_wqe { + __le32 byte_4; + __le32 msg_len; + __le32 immtdata; + __le32 byte_16; + __le32 byte_20; + __le32 byte_24; + __le32 qkey; + __le32 byte_32; + __le32 byte_36; + __le32 byte_40; + __le32 dmac; + __le32 byte_48; + u8 dgid[GID_LEN_V2]; +}; + +#define V2_UD_SEND_WQE_BYTE_4_OPCODE_S 0 +#define V2_UD_SEND_WQE_BYTE_4_OPCODE_M GENMASK(4, 0) + +#define V2_UD_SEND_WQE_BYTE_4_OWNER_S 7 + +#define V2_UD_SEND_WQE_BYTE_4_CQE_S 8 + +#define V2_UD_SEND_WQE_BYTE_4_SE_S 11 + +#define V2_UD_SEND_WQE_BYTE_16_PD_S 0 +#define V2_UD_SEND_WQE_BYTE_16_PD_M GENMASK(23, 0) + +#define V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S 24 +#define V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M GENMASK(31, 24) + +#define V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S 0 +#define V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M GENMASK(23, 0) + +#define V2_UD_SEND_WQE_BYTE_24_UDPSPN_S 16 +#define V2_UD_SEND_WQE_BYTE_24_UDPSPN_M GENMASK(31, 16) + +#define V2_UD_SEND_WQE_BYTE_32_DQPN_S 0 +#define V2_UD_SEND_WQE_BYTE_32_DQPN_M GENMASK(23, 0) + +#define V2_UD_SEND_WQE_BYTE_36_VLAN_S 0 +#define V2_UD_SEND_WQE_BYTE_36_VLAN_M GENMASK(15, 0) + +#define V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S 16 +#define V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M GENMASK(23, 16) + +#define V2_UD_SEND_WQE_BYTE_36_TCLASS_S 24 +#define V2_UD_SEND_WQE_BYTE_36_TCLASS_M GENMASK(31, 24) + +#define V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S 0 +#define V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M GENMASK(19, 0) + +#define V2_UD_SEND_WQE_BYTE_40_SL_S 20 +#define V2_UD_SEND_WQE_BYTE_40_SL_M GENMASK(23, 20) + +#define V2_UD_SEND_WQE_BYTE_40_PORTN_S 24 +#define V2_UD_SEND_WQE_BYTE_40_PORTN_M GENMASK(26, 24) + +#define V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S 30 + +#define V2_UD_SEND_WQE_BYTE_40_LBI_S 31 + +#define V2_UD_SEND_WQE_DMAC_0_S 0 +#define V2_UD_SEND_WQE_DMAC_0_M GENMASK(7, 0) + +#define V2_UD_SEND_WQE_DMAC_1_S 8 +#define V2_UD_SEND_WQE_DMAC_1_M GENMASK(15, 8) + +#define V2_UD_SEND_WQE_DMAC_2_S 16 +#define V2_UD_SEND_WQE_DMAC_2_M GENMASK(23, 16) + +#define V2_UD_SEND_WQE_DMAC_3_S 24 +#define V2_UD_SEND_WQE_DMAC_3_M GENMASK(31, 24) + +#define V2_UD_SEND_WQE_BYTE_48_DMAC_4_S 0 +#define V2_UD_SEND_WQE_BYTE_48_DMAC_4_M GENMASK(7, 0) + +#define V2_UD_SEND_WQE_BYTE_48_DMAC_5_S 8 +#define V2_UD_SEND_WQE_BYTE_48_DMAC_5_M GENMASK(15, 8) + +#define V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S 16 +#define V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M GENMASK(23, 16) + +#define V2_UD_SEND_WQE_BYTE_48_SMAC_INDX_S 24 +#define V2_UD_SEND_WQE_BYTE_48_SMAC_INDX_M GENMASK(31, 24) + +struct hns_roce_v2_rc_send_wqe { + __le32 byte_4; + __le32 msg_len; + union { + __le32 inv_key; + __le32 immtdata; + }; + __le32 byte_16; + __le32 byte_20; + __le32 rkey; + __le64 va; +}; + +#define V2_RC_SEND_WQE_BYTE_4_OPCODE_S 0 +#define V2_RC_SEND_WQE_BYTE_4_OPCODE_M GENMASK(4, 0) + +#define V2_RC_SEND_WQE_BYTE_4_OWNER_S 7 + +#define V2_RC_SEND_WQE_BYTE_4_CQE_S 8 + +#define V2_RC_SEND_WQE_BYTE_4_FENCE_S 9 + +#define V2_RC_SEND_WQE_BYTE_4_SO_S 10 + +#define V2_RC_SEND_WQE_BYTE_4_SE_S 11 + +#define V2_RC_SEND_WQE_BYTE_4_INLINE_S 12 + +#define V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S 19 + +#define V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S 20 + +#define V2_RC_FRMR_WQE_BYTE_4_RR_S 21 + +#define V2_RC_FRMR_WQE_BYTE_4_RW_S 22 + +#define V2_RC_FRMR_WQE_BYTE_4_LW_S 23 + +#define V2_RC_SEND_WQE_BYTE_16_XRC_SRQN_S 0 +#define V2_RC_SEND_WQE_BYTE_16_XRC_SRQN_M GENMASK(23, 0) + +#define V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S 24 +#define V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M GENMASK(31, 24) + +#define V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S 0 +#define V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M GENMASK(23, 0) + +#define V2_RC_SEND_WQE_BYTE_20_INL_TYPE_S 31 + +struct hns_roce_wqe_frmr_seg { + __le32 pbl_size; + __le32 mode_buf_pg_sz; +}; + +#define V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S 4 +#define V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M GENMASK(7, 4) + +#define V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S 8 + +struct hns_roce_v2_wqe_data_seg { + __le32 len; + __le32 lkey; + __le64 addr; +}; + +struct hns_roce_v2_db { + __le32 byte_4; + __le32 parameter; +}; + +struct hns_roce_query_version { + __le16 rocee_vendor_id; + __le16 rocee_hw_version; + __le32 rsv[5]; +}; + +struct hns_roce_query_fw_info { + __le32 fw_ver; + __le32 rsv[5]; +}; + +struct hns_roce_func_clear { + __le32 rst_funcid_en; + __le32 func_done; + __le32 rsv[4]; +}; + +#define FUNC_CLEAR_RST_FUN_DONE_S 0 +/* Each physical function manages up to 248 virtual functions, it takes up to + * 100ms for each function to execute clear. If an abnormal reset occurs, it is + * executed twice at most, so it takes up to 249 * 2 * 100ms. + */ +#define HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS (249 * 2 * 100) +#define HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL 40 +#define HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT 20 + +struct hns_roce_cfg_llm_a { + __le32 base_addr_l; + __le32 base_addr_h; + __le32 depth_pgsz_init_en; + __le32 head_ba_l; + __le32 head_ba_h_nxtptr; + __le32 head_ptr; +}; + +#define CFG_LLM_QUE_DEPTH_S 0 +#define CFG_LLM_QUE_DEPTH_M GENMASK(12, 0) + +#define CFG_LLM_QUE_PGSZ_S 16 +#define CFG_LLM_QUE_PGSZ_M GENMASK(19, 16) + +#define CFG_LLM_INIT_EN_S 20 +#define CFG_LLM_INIT_EN_M GENMASK(20, 20) + +#define CFG_LLM_HEAD_PTR_S 0 +#define CFG_LLM_HEAD_PTR_M GENMASK(11, 0) + +struct hns_roce_cfg_llm_b { + __le32 tail_ba_l; + __le32 tail_ba_h; + __le32 tail_ptr; + __le32 rsv[3]; +}; + +#define CFG_LLM_TAIL_BA_H_S 0 +#define CFG_LLM_TAIL_BA_H_M GENMASK(19, 0) + +#define CFG_LLM_TAIL_PTR_S 0 +#define CFG_LLM_TAIL_PTR_M GENMASK(11, 0) + +struct hns_roce_cfg_global_param { + __le32 time_cfg_udp_port; + __le32 rsv[5]; +}; + +#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S 0 +#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M GENMASK(9, 0) + +#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S 16 +#define CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M GENMASK(31, 16) + +struct hns_roce_pf_res_a { + __le32 rsv; + __le32 qpc_bt_idx_num; + __le32 srqc_bt_idx_num; + __le32 cqc_bt_idx_num; + __le32 mpt_bt_idx_num; + __le32 eqc_bt_idx_num; +}; + +#define PF_RES_DATA_1_PF_QPC_BT_IDX_S 0 +#define PF_RES_DATA_1_PF_QPC_BT_IDX_M GENMASK(10, 0) + +#define PF_RES_DATA_1_PF_QPC_BT_NUM_S 16 +#define PF_RES_DATA_1_PF_QPC_BT_NUM_M GENMASK(27, 16) + +#define PF_RES_DATA_2_PF_SRQC_BT_IDX_S 0 +#define PF_RES_DATA_2_PF_SRQC_BT_IDX_M GENMASK(8, 0) + +#define PF_RES_DATA_2_PF_SRQC_BT_NUM_S 16 +#define PF_RES_DATA_2_PF_SRQC_BT_NUM_M GENMASK(25, 16) + +#define PF_RES_DATA_3_PF_CQC_BT_IDX_S 0 +#define PF_RES_DATA_3_PF_CQC_BT_IDX_M GENMASK(8, 0) + +#define PF_RES_DATA_3_PF_CQC_BT_NUM_S 16 +#define PF_RES_DATA_3_PF_CQC_BT_NUM_M GENMASK(25, 16) + +#define PF_RES_DATA_4_PF_MPT_BT_IDX_S 0 +#define PF_RES_DATA_4_PF_MPT_BT_IDX_M GENMASK(8, 0) + +#define PF_RES_DATA_4_PF_MPT_BT_NUM_S 16 +#define PF_RES_DATA_4_PF_MPT_BT_NUM_M GENMASK(25, 16) + +#define PF_RES_DATA_5_PF_EQC_BT_IDX_S 0 +#define PF_RES_DATA_5_PF_EQC_BT_IDX_M GENMASK(8, 0) + +#define PF_RES_DATA_5_PF_EQC_BT_NUM_S 16 +#define PF_RES_DATA_5_PF_EQC_BT_NUM_M GENMASK(25, 16) + +struct hns_roce_pf_res_b { + __le32 rsv0; + __le32 smac_idx_num; + __le32 sgid_idx_num; + __le32 qid_idx_sl_num; + __le32 sccc_bt_idx_num; + __le32 rsv; +}; + +#define PF_RES_DATA_1_PF_SMAC_IDX_S 0 +#define PF_RES_DATA_1_PF_SMAC_IDX_M GENMASK(7, 0) + +#define PF_RES_DATA_1_PF_SMAC_NUM_S 8 +#define PF_RES_DATA_1_PF_SMAC_NUM_M GENMASK(16, 8) + +#define PF_RES_DATA_2_PF_SGID_IDX_S 0 +#define PF_RES_DATA_2_PF_SGID_IDX_M GENMASK(7, 0) + +#define PF_RES_DATA_2_PF_SGID_NUM_S 8 +#define PF_RES_DATA_2_PF_SGID_NUM_M GENMASK(16, 8) + +#define PF_RES_DATA_3_PF_QID_IDX_S 0 +#define PF_RES_DATA_3_PF_QID_IDX_M GENMASK(9, 0) + +#define PF_RES_DATA_3_PF_SL_NUM_S 16 +#define PF_RES_DATA_3_PF_SL_NUM_M GENMASK(26, 16) + +#define PF_RES_DATA_4_PF_SCCC_BT_IDX_S 0 +#define PF_RES_DATA_4_PF_SCCC_BT_IDX_M GENMASK(8, 0) + +#define PF_RES_DATA_4_PF_SCCC_BT_NUM_S 9 +#define PF_RES_DATA_4_PF_SCCC_BT_NUM_M GENMASK(17, 9) + +struct hns_roce_pf_timer_res_a { + __le32 rsv0; + __le32 qpc_timer_bt_idx_num; + __le32 cqc_timer_bt_idx_num; + __le32 rsv[3]; +}; + +#define PF_RES_DATA_1_PF_QPC_TIMER_BT_IDX_S 0 +#define PF_RES_DATA_1_PF_QPC_TIMER_BT_IDX_M GENMASK(11, 0) + +#define PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S 16 +#define PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M GENMASK(28, 16) + +#define PF_RES_DATA_2_PF_CQC_TIMER_BT_IDX_S 0 +#define PF_RES_DATA_2_PF_CQC_TIMER_BT_IDX_M GENMASK(10, 0) + +#define PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S 16 +#define PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M GENMASK(27, 16) + +struct hns_roce_vf_res_a { + __le32 vf_id; + __le32 vf_qpc_bt_idx_num; + __le32 vf_srqc_bt_idx_num; + __le32 vf_cqc_bt_idx_num; + __le32 vf_mpt_bt_idx_num; + __le32 vf_eqc_bt_idx_num; +}; + +#define VF_RES_A_DATA_1_VF_QPC_BT_IDX_S 0 +#define VF_RES_A_DATA_1_VF_QPC_BT_IDX_M GENMASK(10, 0) + +#define VF_RES_A_DATA_1_VF_QPC_BT_NUM_S 16 +#define VF_RES_A_DATA_1_VF_QPC_BT_NUM_M GENMASK(27, 16) + +#define VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S 0 +#define VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M GENMASK(8, 0) + +#define VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S 16 +#define VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M GENMASK(25, 16) + +#define VF_RES_A_DATA_3_VF_CQC_BT_IDX_S 0 +#define VF_RES_A_DATA_3_VF_CQC_BT_IDX_M GENMASK(8, 0) + +#define VF_RES_A_DATA_3_VF_CQC_BT_NUM_S 16 +#define VF_RES_A_DATA_3_VF_CQC_BT_NUM_M GENMASK(25, 16) + +#define VF_RES_A_DATA_4_VF_MPT_BT_IDX_S 0 +#define VF_RES_A_DATA_4_VF_MPT_BT_IDX_M GENMASK(8, 0) + +#define VF_RES_A_DATA_4_VF_MPT_BT_NUM_S 16 +#define VF_RES_A_DATA_4_VF_MPT_BT_NUM_M GENMASK(25, 16) + +#define VF_RES_A_DATA_5_VF_EQC_IDX_S 0 +#define VF_RES_A_DATA_5_VF_EQC_IDX_M GENMASK(8, 0) + +#define VF_RES_A_DATA_5_VF_EQC_NUM_S 16 +#define VF_RES_A_DATA_5_VF_EQC_NUM_M GENMASK(25, 16) + +struct hns_roce_vf_res_b { + __le32 rsv0; + __le32 vf_smac_idx_num; + __le32 vf_sgid_idx_num; + __le32 vf_qid_idx_sl_num; + __le32 vf_sccc_idx_num; + __le32 rsv1; +}; + +#define VF_RES_B_DATA_0_VF_ID_S 0 +#define VF_RES_B_DATA_0_VF_ID_M GENMASK(7, 0) + +#define VF_RES_B_DATA_1_VF_SMAC_IDX_S 0 +#define VF_RES_B_DATA_1_VF_SMAC_IDX_M GENMASK(7, 0) + +#define VF_RES_B_DATA_1_VF_SMAC_NUM_S 8 +#define VF_RES_B_DATA_1_VF_SMAC_NUM_M GENMASK(16, 8) + +#define VF_RES_B_DATA_2_VF_SGID_IDX_S 0 +#define VF_RES_B_DATA_2_VF_SGID_IDX_M GENMASK(7, 0) + +#define VF_RES_B_DATA_2_VF_SGID_NUM_S 8 +#define VF_RES_B_DATA_2_VF_SGID_NUM_M GENMASK(16, 8) + +#define VF_RES_B_DATA_3_VF_QID_IDX_S 0 +#define VF_RES_B_DATA_3_VF_QID_IDX_M GENMASK(9, 0) + +#define VF_RES_B_DATA_3_VF_SL_NUM_S 16 +#define VF_RES_B_DATA_3_VF_SL_NUM_M GENMASK(19, 16) + +#define VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S 0 +#define VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M GENMASK(8, 0) + +#define VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S 9 +#define VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M GENMASK(17, 9) + +struct hns_roce_vf_switch { + __le32 rocee_sel; + __le32 fun_id; + __le32 cfg; + __le32 resv1; + __le32 resv2; + __le32 resv3; +}; + +#define VF_SWITCH_DATA_FUN_ID_VF_ID_S 3 +#define VF_SWITCH_DATA_FUN_ID_VF_ID_M GENMASK(10, 3) + +#define VF_SWITCH_DATA_CFG_ALW_LPBK_S 1 +#define VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S 2 +#define VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S 3 + +struct hns_roce_post_mbox { + __le32 in_param_l; + __le32 in_param_h; + __le32 out_param_l; + __le32 out_param_h; + __le32 cmd_tag; + __le32 token_event_en; +}; + +struct hns_roce_mbox_status { + __le32 mb_status_hw_run; + __le32 rsv[5]; +}; + +struct hns_roce_cfg_bt_attr { + __le32 vf_qpc_cfg; + __le32 vf_srqc_cfg; + __le32 vf_cqc_cfg; + __le32 vf_mpt_cfg; + __le32 vf_sccc_cfg; + __le32 rsv; +}; + +#define CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S 0 +#define CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M GENMASK(3, 0) + +#define CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S 4 +#define CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M GENMASK(7, 4) + +#define CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S 8 +#define CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M GENMASK(9, 8) + +#define CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S 0 +#define CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M GENMASK(3, 0) + +#define CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S 4 +#define CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M GENMASK(7, 4) + +#define CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S 8 +#define CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M GENMASK(9, 8) + +#define CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S 0 +#define CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M GENMASK(3, 0) + +#define CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S 4 +#define CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M GENMASK(7, 4) + +#define CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S 8 +#define CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M GENMASK(9, 8) + +#define CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S 0 +#define CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M GENMASK(3, 0) + +#define CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S 4 +#define CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M GENMASK(7, 4) + +#define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S 8 +#define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M GENMASK(9, 8) + +#define CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S 0 +#define CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M GENMASK(3, 0) + +#define CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S 4 +#define CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M GENMASK(7, 4) + +#define CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S 8 +#define CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M GENMASK(9, 8) + +struct hns_roce_cfg_sgid_tb { + __le32 table_idx_rsv; + __le32 vf_sgid_l; + __le32 vf_sgid_ml; + __le32 vf_sgid_mh; + __le32 vf_sgid_h; + __le32 vf_sgid_type_rsv; +}; + +enum { + HNS_ROCE_CFG_QPC_SIZE = BIT(0), + HNS_ROCE_CFG_SCCC_SIZE = BIT(1), +}; + +struct hns_roce_cfg_entry_size { + __le32 type; + __le32 rsv[4]; + __le32 size; +}; + +#define CFG_SGID_TB_TABLE_IDX_S 0 +#define CFG_SGID_TB_TABLE_IDX_M GENMASK(7, 0) + +#define CFG_SGID_TB_VF_SGID_TYPE_S 0 +#define CFG_SGID_TB_VF_SGID_TYPE_M GENMASK(1, 0) + +struct hns_roce_cfg_smac_tb { + __le32 tb_idx_rsv; + __le32 vf_smac_l; + __le32 vf_smac_h_rsv; + __le32 rsv[3]; +}; +#define CFG_SMAC_TB_IDX_S 0 +#define CFG_SMAC_TB_IDX_M GENMASK(7, 0) + +#define CFG_SMAC_TB_VF_SMAC_H_S 0 +#define CFG_SMAC_TB_VF_SMAC_H_M GENMASK(15, 0) + +#define HNS_ROCE_QUERY_PF_CAPS_CMD_NUM 5 +struct hns_roce_query_pf_caps_a { + u8 number_ports; + u8 local_ca_ack_delay; + __le16 max_sq_sg; + __le16 max_sq_inline; + __le16 max_rq_sg; + __le32 max_extend_sg; + __le16 num_qpc_timer; + __le16 num_cqc_timer; + __le16 max_srq_sges; + u8 num_aeq_vectors; + u8 num_other_vectors; + u8 max_sq_desc_sz; + u8 max_rq_desc_sz; + u8 max_srq_desc_sz; + u8 cqe_sz; +}; + +struct hns_roce_query_pf_caps_b { + u8 mtpt_entry_sz; + u8 irrl_entry_sz; + u8 trrl_entry_sz; + u8 cqc_entry_sz; + u8 srqc_entry_sz; + u8 idx_entry_sz; + u8 sccc_sz; + u8 max_mtu; + __le16 qpc_sz; + __le16 qpc_timer_entry_sz; + __le16 cqc_timer_entry_sz; + u8 min_cqes; + u8 min_wqes; + __le32 page_size_cap; + u8 pkey_table_len; + u8 phy_num_uars; + u8 ctx_hop_num; + u8 pbl_hop_num; +}; + +struct hns_roce_query_pf_caps_c { + __le32 cap_flags_num_pds; + __le32 max_gid_num_cqs; + __le32 cq_depth; + __le32 num_mrws; + __le32 ord_num_qps; + __le16 sq_depth; + __le16 rq_depth; +}; + +#define V2_QUERY_PF_CAPS_C_NUM_PDS_S 0 +#define V2_QUERY_PF_CAPS_C_NUM_PDS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_C_CAP_FLAGS_S 20 +#define V2_QUERY_PF_CAPS_C_CAP_FLAGS_M GENMASK(31, 20) + +#define V2_QUERY_PF_CAPS_C_NUM_CQS_S 0 +#define V2_QUERY_PF_CAPS_C_NUM_CQS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_C_MAX_GID_S 20 +#define V2_QUERY_PF_CAPS_C_MAX_GID_M GENMASK(28, 20) + +#define V2_QUERY_PF_CAPS_C_CQ_DEPTH_S 0 +#define V2_QUERY_PF_CAPS_C_CQ_DEPTH_M GENMASK(22, 0) + +#define V2_QUERY_PF_CAPS_C_NUM_MRWS_S 0 +#define V2_QUERY_PF_CAPS_C_NUM_MRWS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_C_NUM_QPS_S 0 +#define V2_QUERY_PF_CAPS_C_NUM_QPS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_C_MAX_ORD_S 20 +#define V2_QUERY_PF_CAPS_C_MAX_ORD_M GENMASK(27, 20) + +struct hns_roce_query_pf_caps_d { + __le32 wq_hop_num_max_srqs; + __le16 srq_depth; + __le16 cap_flags_ex; + __le32 num_ceqs_ceq_depth; + __le32 arm_st_aeq_depth; + __le32 num_uars_rsv_pds; + __le32 rsv_uars_rsv_qps; +}; +#define V2_QUERY_PF_CAPS_D_NUM_SRQS_S 0 +#define V2_QUERY_PF_CAPS_D_NUM_SRQS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S 20 +#define V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M GENMASK(21, 20) + +#define V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S 22 +#define V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M GENMASK(23, 22) + +#define V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S 24 +#define V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M GENMASK(25, 24) + + +#define V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S 0 +#define V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M GENMASK(21, 0) + +#define V2_QUERY_PF_CAPS_D_NUM_CEQS_S 22 +#define V2_QUERY_PF_CAPS_D_NUM_CEQS_M GENMASK(31, 22) + +#define V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S 0 +#define V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M GENMASK(21, 0) + +#define V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_S 22 +#define V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_M GENMASK(23, 22) + +#define V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_S 24 +#define V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_M GENMASK(25, 24) + +#define V2_QUERY_PF_CAPS_D_RSV_PDS_S 0 +#define V2_QUERY_PF_CAPS_D_RSV_PDS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_D_NUM_UARS_S 20 +#define V2_QUERY_PF_CAPS_D_NUM_UARS_M GENMASK(27, 20) + +#define V2_QUERY_PF_CAPS_D_RSV_QPS_S 0 +#define V2_QUERY_PF_CAPS_D_RSV_QPS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_D_RSV_UARS_S 20 +#define V2_QUERY_PF_CAPS_D_RSV_UARS_M GENMASK(27, 20) + +struct hns_roce_query_pf_caps_e { + __le32 chunk_size_shift_rsv_mrws; + __le32 rsv_cqs; + __le32 rsv_srqs; + __le32 rsv_lkey; + __le16 ceq_max_cnt; + __le16 ceq_period; + __le16 aeq_max_cnt; + __le16 aeq_period; +}; + +#define V2_QUERY_PF_CAPS_E_RSV_MRWS_S 0 +#define V2_QUERY_PF_CAPS_E_RSV_MRWS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S 20 +#define V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M GENMASK(31, 20) + +#define V2_QUERY_PF_CAPS_E_RSV_CQS_S 0 +#define V2_QUERY_PF_CAPS_E_RSV_CQS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_E_RSV_SRQS_S 0 +#define V2_QUERY_PF_CAPS_E_RSV_SRQS_M GENMASK(19, 0) + +#define V2_QUERY_PF_CAPS_E_RSV_LKEYS_S 0 +#define V2_QUERY_PF_CAPS_E_RSV_LKEYS_M GENMASK(19, 0) + +struct hns_roce_cmq_desc { + __le16 opcode; + __le16 flag; + __le16 retval; + __le16 rsv; + __le32 data[6]; +}; + +#define HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS 10000 + +#define HNS_ROCE_HW_RUN_BIT_SHIFT 31 +#define HNS_ROCE_HW_MB_STATUS_MASK 0xFF + +struct hns_roce_v2_cmq_ring { + dma_addr_t desc_dma_addr; + struct hns_roce_cmq_desc *desc; + u32 head; + u32 tail; + + u16 buf_size; + u16 desc_num; + int next_to_use; + int next_to_clean; + u8 flag; + spinlock_t lock; /* command queue lock */ +}; + +struct hns_roce_v2_cmq { + struct hns_roce_v2_cmq_ring csq; + struct hns_roce_v2_cmq_ring crq; + u16 tx_timeout; + u16 last_status; +}; + +enum hns_roce_link_table_type { + TSQ_LINK_TABLE, + TPQ_LINK_TABLE, +}; + +struct hns_roce_link_table { + struct hns_roce_buf_list table; + struct hns_roce_buf_list *pg_list; + u32 npages; + u32 pg_sz; +}; + +struct hns_roce_link_table_entry { + u32 blk_ba0; + u32 blk_ba1_nxt_ptr; +}; +#define HNS_ROCE_LINK_TABLE_BA1_S 0 +#define HNS_ROCE_LINK_TABLE_BA1_M GENMASK(19, 0) + +#define HNS_ROCE_LINK_TABLE_NXT_PTR_S 20 +#define HNS_ROCE_LINK_TABLE_NXT_PTR_M GENMASK(31, 20) + +struct hns_roce_v2_priv { + struct hnae3_handle *handle; + struct hns_roce_v2_cmq cmq; + struct hns_roce_link_table tsq; + struct hns_roce_link_table tpq; +}; + +struct hns_roce_eq_context { + __le32 byte_4; + __le32 byte_8; + __le32 byte_12; + __le32 eqe_report_timer; + __le32 eqe_ba0; + __le32 eqe_ba1; + __le32 byte_28; + __le32 byte_32; + __le32 byte_36; + __le32 byte_40; + __le32 byte_44; + __le32 rsv[5]; +}; + +#define HNS_ROCE_AEQ_DEFAULT_BURST_NUM 0x0 +#define HNS_ROCE_AEQ_DEFAULT_INTERVAL 0x0 +#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM 0x0 +#define HNS_ROCE_CEQ_DEFAULT_INTERVAL 0x0 + +#define HNS_ROCE_V2_EQ_STATE_INVALID 0 +#define HNS_ROCE_V2_EQ_STATE_VALID 1 +#define HNS_ROCE_V2_EQ_STATE_OVERFLOW 2 +#define HNS_ROCE_V2_EQ_STATE_FAILURE 3 + +#define HNS_ROCE_V2_EQ_OVER_IGNORE_0 0 +#define HNS_ROCE_V2_EQ_OVER_IGNORE_1 1 + +#define HNS_ROCE_V2_EQ_COALESCE_0 0 +#define HNS_ROCE_V2_EQ_COALESCE_1 1 + +#define HNS_ROCE_V2_EQ_FIRED 0 +#define HNS_ROCE_V2_EQ_ARMED 1 +#define HNS_ROCE_V2_EQ_ALWAYS_ARMED 3 + +#define HNS_ROCE_EQ_INIT_EQE_CNT 0 +#define HNS_ROCE_EQ_INIT_PROD_IDX 0 +#define HNS_ROCE_EQ_INIT_REPORT_TIMER 0 +#define HNS_ROCE_EQ_INIT_MSI_IDX 0 +#define HNS_ROCE_EQ_INIT_CONS_IDX 0 +#define HNS_ROCE_EQ_INIT_NXT_EQE_BA 0 + +#define HNS_ROCE_V2_CEQ_CEQE_OWNER_S 31 +#define HNS_ROCE_V2_AEQ_AEQE_OWNER_S 31 + +#define HNS_ROCE_V2_COMP_EQE_NUM 0x1000 +#define HNS_ROCE_V2_ASYNC_EQE_NUM 0x1000 + +#define HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S 0 +#define HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S 1 +#define HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S 2 + +#define HNS_ROCE_EQ_DB_CMD_AEQ 0x0 +#define HNS_ROCE_EQ_DB_CMD_AEQ_ARMED 0x1 +#define HNS_ROCE_EQ_DB_CMD_CEQ 0x2 +#define HNS_ROCE_EQ_DB_CMD_CEQ_ARMED 0x3 + +#define EQ_ENABLE 1 +#define EQ_DISABLE 0 + +#define EQ_REG_OFFSET 0x4 + +#define HNS_ROCE_INT_NAME_LEN 32 +#define HNS_ROCE_V2_EQN_M GENMASK(23, 0) + +#define HNS_ROCE_V2_CONS_IDX_M GENMASK(23, 0) + +#define HNS_ROCE_V2_VF_ABN_INT_EN_S 0 +#define HNS_ROCE_V2_VF_ABN_INT_EN_M GENMASK(0, 0) +#define HNS_ROCE_V2_VF_ABN_INT_ST_M GENMASK(2, 0) +#define HNS_ROCE_V2_VF_ABN_INT_CFG_M GENMASK(2, 0) +#define HNS_ROCE_V2_VF_EVENT_INT_EN_M GENMASK(0, 0) + +/* WORD0 */ +#define HNS_ROCE_EQC_EQ_ST_S 0 +#define HNS_ROCE_EQC_EQ_ST_M GENMASK(1, 0) + +#define HNS_ROCE_EQC_HOP_NUM_S 2 +#define HNS_ROCE_EQC_HOP_NUM_M GENMASK(3, 2) + +#define HNS_ROCE_EQC_OVER_IGNORE_S 4 +#define HNS_ROCE_EQC_OVER_IGNORE_M GENMASK(4, 4) + +#define HNS_ROCE_EQC_COALESCE_S 5 +#define HNS_ROCE_EQC_COALESCE_M GENMASK(5, 5) + +#define HNS_ROCE_EQC_ARM_ST_S 6 +#define HNS_ROCE_EQC_ARM_ST_M GENMASK(7, 6) + +#define HNS_ROCE_EQC_EQN_S 8 +#define HNS_ROCE_EQC_EQN_M GENMASK(15, 8) + +#define HNS_ROCE_EQC_EQE_CNT_S 16 +#define HNS_ROCE_EQC_EQE_CNT_M GENMASK(31, 16) + +/* WORD1 */ +#define HNS_ROCE_EQC_BA_PG_SZ_S 0 +#define HNS_ROCE_EQC_BA_PG_SZ_M GENMASK(3, 0) + +#define HNS_ROCE_EQC_BUF_PG_SZ_S 4 +#define HNS_ROCE_EQC_BUF_PG_SZ_M GENMASK(7, 4) + +#define HNS_ROCE_EQC_PROD_INDX_S 8 +#define HNS_ROCE_EQC_PROD_INDX_M GENMASK(31, 8) + +/* WORD2 */ +#define HNS_ROCE_EQC_MAX_CNT_S 0 +#define HNS_ROCE_EQC_MAX_CNT_M GENMASK(15, 0) + +#define HNS_ROCE_EQC_PERIOD_S 16 +#define HNS_ROCE_EQC_PERIOD_M GENMASK(31, 16) + +/* WORD3 */ +#define HNS_ROCE_EQC_REPORT_TIMER_S 0 +#define HNS_ROCE_EQC_REPORT_TIMER_M GENMASK(31, 0) + +/* WORD4 */ +#define HNS_ROCE_EQC_EQE_BA_L_S 0 +#define HNS_ROCE_EQC_EQE_BA_L_M GENMASK(31, 0) + +/* WORD5 */ +#define HNS_ROCE_EQC_EQE_BA_H_S 0 +#define HNS_ROCE_EQC_EQE_BA_H_M GENMASK(28, 0) + +/* WORD6 */ +#define HNS_ROCE_EQC_SHIFT_S 0 +#define HNS_ROCE_EQC_SHIFT_M GENMASK(7, 0) + +#define HNS_ROCE_EQC_MSI_INDX_S 8 +#define HNS_ROCE_EQC_MSI_INDX_M GENMASK(15, 8) + +#define HNS_ROCE_EQC_CUR_EQE_BA_L_S 16 +#define HNS_ROCE_EQC_CUR_EQE_BA_L_M GENMASK(31, 16) + +/* WORD7 */ +#define HNS_ROCE_EQC_CUR_EQE_BA_M_S 0 +#define HNS_ROCE_EQC_CUR_EQE_BA_M_M GENMASK(31, 0) + +/* WORD8 */ +#define HNS_ROCE_EQC_CUR_EQE_BA_H_S 0 +#define HNS_ROCE_EQC_CUR_EQE_BA_H_M GENMASK(3, 0) + +#define HNS_ROCE_EQC_CONS_INDX_S 8 +#define HNS_ROCE_EQC_CONS_INDX_M GENMASK(31, 8) + +/* WORD9 */ +#define HNS_ROCE_EQC_NXT_EQE_BA_L_S 0 +#define HNS_ROCE_EQC_NXT_EQE_BA_L_M GENMASK(31, 0) + +/* WORD10 */ +#define HNS_ROCE_EQC_NXT_EQE_BA_H_S 0 +#define HNS_ROCE_EQC_NXT_EQE_BA_H_M GENMASK(19, 0) + +#define HNS_ROCE_EQC_EQE_SIZE_S 20 +#define HNS_ROCE_EQC_EQE_SIZE_M GENMASK(21, 20) + +#define HNS_ROCE_V2_CEQE_COMP_CQN_S 0 +#define HNS_ROCE_V2_CEQE_COMP_CQN_M GENMASK(23, 0) + +#define HNS_ROCE_V2_AEQE_EVENT_TYPE_S 0 +#define HNS_ROCE_V2_AEQE_EVENT_TYPE_M GENMASK(7, 0) + +#define HNS_ROCE_V2_AEQE_SUB_TYPE_S 8 +#define HNS_ROCE_V2_AEQE_SUB_TYPE_M GENMASK(15, 8) + +#define HNS_ROCE_V2_EQ_DB_CMD_S 16 +#define HNS_ROCE_V2_EQ_DB_CMD_M GENMASK(17, 16) + +#define HNS_ROCE_V2_EQ_DB_TAG_S 0 +#define HNS_ROCE_V2_EQ_DB_TAG_M GENMASK(7, 0) + +#define HNS_ROCE_V2_EQ_DB_PARA_S 0 +#define HNS_ROCE_V2_EQ_DB_PARA_M GENMASK(23, 0) + +#define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S 0 +#define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M GENMASK(23, 0) + +#define MAX_SERVICE_LEVEL 0x7 + +struct hns_roce_wqe_atomic_seg { + __le64 fetchadd_swap_data; + __le64 cmp_data; +}; + +struct hns_roce_sccc_clr { + __le32 qpn; + __le32 rsv[5]; +}; + +struct hns_roce_sccc_clr_done { + __le32 clr_done; + __le32 rsv[5]; +}; + +int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn, + int *buffer); + +static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2], + void __iomem *dest) +{ + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (!hr_dev->dis_db && !ops->get_hw_reset_stat(handle)) + hns_roce_write64_k(val, dest); +} + +#endif diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c new file mode 100644 index 000000000..5a97b5a0b --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +// Copyright (c) 2019 Hisilicon Limited. + +#include "hnae3.h" +#include "hns_roce_device.h" +#include "hns_roce_cmd.h" +#include "hns_roce_hw_v2.h" + +int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn, + int *buffer) +{ + struct hns_roce_v2_cq_context *cq_context; + struct hns_roce_cmd_mailbox *mailbox; + int ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + cq_context = mailbox->buf; + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cqn, 0, + HNS_ROCE_CMD_QUERY_CQC, + HNS_ROCE_CMD_TIMEOUT_MSECS); + if (ret) { + dev_err(hr_dev->dev, "QUERY cqc cmd process error\n"); + goto err_mailbox; + } + + memcpy(buffer, cq_context, sizeof(*cq_context)); + +err_mailbox: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c new file mode 100644 index 000000000..90cbd15f6 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -0,0 +1,982 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include <linux/acpi.h> +#include <linux/of_platform.h> +#include <linux/module.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_smi.h> +#include <rdma/ib_user_verbs.h> +#include <rdma/ib_cache.h> +#include "hns_roce_common.h" +#include "hns_roce_device.h" +#include <rdma/hns-abi.h> +#include "hns_roce_hem.h" + +/** + * hns_get_gid_index - Get gid index. + * @hr_dev: pointer to structure hns_roce_dev. + * @port: port, value range: 0 ~ MAX + * @gid_index: gid_index, value range: 0 ~ MAX + * Description: + * N ports shared gids, allocation method as follow: + * GID[0][0], GID[1][0],.....GID[N - 1][0], + * GID[0][0], GID[1][0],.....GID[N - 1][0], + * And so on + */ +int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index) +{ + return gid_index * hr_dev->caps.num_ports + port; +} + +static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr) +{ + u8 phy_port; + u32 i = 0; + + if (!memcmp(hr_dev->dev_addr[port], addr, ETH_ALEN)) + return 0; + + for (i = 0; i < ETH_ALEN; i++) + hr_dev->dev_addr[port][i] = addr[i]; + + phy_port = hr_dev->iboe.phy_port[port]; + return hr_dev->hw->set_mac(hr_dev, phy_port, addr); +} + +static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); + u8 port = attr->port_num - 1; + int ret; + + if (port >= hr_dev->caps.num_ports) + return -EINVAL; + + ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &attr->gid, attr); + + return ret; +} + +static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); + struct ib_gid_attr zattr = {}; + u8 port = attr->port_num - 1; + int ret; + + if (port >= hr_dev->caps.num_ports) + return -EINVAL; + + ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &zgid, &zattr); + + return ret; +} + +static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port, + unsigned long event) +{ + struct device *dev = hr_dev->dev; + struct net_device *netdev; + int ret = 0; + + netdev = hr_dev->iboe.netdevs[port]; + if (!netdev) { + dev_err(dev, "Can't find netdev on port(%u)!\n", port); + return -ENODEV; + } + + switch (event) { + case NETDEV_UP: + case NETDEV_CHANGE: + case NETDEV_REGISTER: + case NETDEV_CHANGEADDR: + ret = hns_roce_set_mac(hr_dev, port, netdev->dev_addr); + break; + case NETDEV_DOWN: + /* + * In v1 engine, only support all ports closed together. + */ + break; + default: + dev_dbg(dev, "NETDEV event = 0x%x!\n", (u32)(event)); + break; + } + + return ret; +} + +static int hns_roce_netdev_event(struct notifier_block *self, + unsigned long event, void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); + struct hns_roce_ib_iboe *iboe = NULL; + struct hns_roce_dev *hr_dev = NULL; + int ret; + u8 port; + + hr_dev = container_of(self, struct hns_roce_dev, iboe.nb); + iboe = &hr_dev->iboe; + + for (port = 0; port < hr_dev->caps.num_ports; port++) { + if (dev == iboe->netdevs[port]) { + ret = handle_en_event(hr_dev, port, event); + if (ret) + return NOTIFY_DONE; + break; + } + } + + return NOTIFY_DONE; +} + +static int hns_roce_setup_mtu_mac(struct hns_roce_dev *hr_dev) +{ + int ret; + u8 i; + + for (i = 0; i < hr_dev->caps.num_ports; i++) { + if (hr_dev->hw->set_mtu) + hr_dev->hw->set_mtu(hr_dev, hr_dev->iboe.phy_port[i], + hr_dev->caps.max_mtu); + ret = hns_roce_set_mac(hr_dev, i, + hr_dev->iboe.netdevs[i]->dev_addr); + if (ret) + return ret; + } + + return 0; +} + +static int hns_roce_query_device(struct ib_device *ib_dev, + struct ib_device_attr *props, + struct ib_udata *uhw) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + + memset(props, 0, sizeof(*props)); + + props->fw_ver = hr_dev->caps.fw_ver; + props->sys_image_guid = cpu_to_be64(hr_dev->sys_image_guid); + props->max_mr_size = (u64)(~(0ULL)); + props->page_size_cap = hr_dev->caps.page_size_cap; + props->vendor_id = hr_dev->vendor_id; + props->vendor_part_id = hr_dev->vendor_part_id; + props->hw_ver = hr_dev->hw_rev; + props->max_qp = hr_dev->caps.num_qps; + props->max_qp_wr = hr_dev->caps.max_wqes; + props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT | + IB_DEVICE_RC_RNR_NAK_GEN; + props->max_send_sge = hr_dev->caps.max_sq_sg; + props->max_recv_sge = hr_dev->caps.max_rq_sg; + props->max_sge_rd = 1; + props->max_cq = hr_dev->caps.num_cqs; + props->max_cqe = hr_dev->caps.max_cqes; + props->max_mr = hr_dev->caps.num_mtpts; + props->max_pd = hr_dev->caps.num_pds; + props->max_qp_rd_atom = hr_dev->caps.max_qp_dest_rdma; + props->max_qp_init_rd_atom = hr_dev->caps.max_qp_init_rdma; + props->atomic_cap = hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_ATOMIC ? + IB_ATOMIC_HCA : IB_ATOMIC_NONE; + props->max_pkeys = 1; + props->local_ca_ack_delay = hr_dev->caps.local_ca_ack_delay; + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { + props->max_srq = hr_dev->caps.num_srqs; + props->max_srq_wr = hr_dev->caps.max_srq_wrs; + props->max_srq_sge = hr_dev->caps.max_srq_sges; + } + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR) { + props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; + props->max_fast_reg_page_list_len = HNS_ROCE_FRMR_MAX_PA; + } + + return 0; +} + +static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num, + struct ib_port_attr *props) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + struct device *dev = hr_dev->dev; + struct net_device *net_dev; + unsigned long flags; + enum ib_mtu mtu; + u8 port; + + port = port_num - 1; + + /* props being zeroed by the caller, avoid zeroing it here */ + + props->max_mtu = hr_dev->caps.max_mtu; + props->gid_tbl_len = hr_dev->caps.gid_table_len[port]; + props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | + IB_PORT_VENDOR_CLASS_SUP | + IB_PORT_BOOT_MGMT_SUP; + props->max_msg_sz = HNS_ROCE_MAX_MSG_LEN; + props->pkey_tbl_len = 1; + props->active_width = IB_WIDTH_4X; + props->active_speed = 1; + + spin_lock_irqsave(&hr_dev->iboe.lock, flags); + + net_dev = hr_dev->iboe.netdevs[port]; + if (!net_dev) { + spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); + dev_err(dev, "Find netdev %u failed!\n", port); + return -EINVAL; + } + + mtu = iboe_get_mtu(net_dev->mtu); + props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256; + props->state = netif_running(net_dev) && netif_carrier_ok(net_dev) ? + IB_PORT_ACTIVE : + IB_PORT_DOWN; + props->phys_state = props->state == IB_PORT_ACTIVE ? + IB_PORT_PHYS_STATE_LINK_UP : + IB_PORT_PHYS_STATE_DISABLED; + + spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); + + return 0; +} + +static enum rdma_link_layer hns_roce_get_link_layer(struct ib_device *device, + u8 port_num) +{ + return IB_LINK_LAYER_ETHERNET; +} + +static int hns_roce_query_pkey(struct ib_device *ib_dev, u8 port, u16 index, + u16 *pkey) +{ + if (index > 0) + return -EINVAL; + + *pkey = PKEY_ID; + + return 0; +} + +static int hns_roce_modify_device(struct ib_device *ib_dev, int mask, + struct ib_device_modify *props) +{ + unsigned long flags; + + if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) + return -EOPNOTSUPP; + + if (mask & IB_DEVICE_MODIFY_NODE_DESC) { + spin_lock_irqsave(&to_hr_dev(ib_dev)->sm_lock, flags); + memcpy(ib_dev->node_desc, props->node_desc, NODE_DESC_SIZE); + spin_unlock_irqrestore(&to_hr_dev(ib_dev)->sm_lock, flags); + } + + return 0; +} + +static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, + struct ib_udata *udata) +{ + int ret; + struct hns_roce_ucontext *context = to_hr_ucontext(uctx); + struct hns_roce_ib_alloc_ucontext_resp resp = {}; + struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device); + + if (!hr_dev->active) + return -EAGAIN; + + resp.qp_tab_size = hr_dev->caps.num_qps; + + ret = hns_roce_uar_alloc(hr_dev, &context->uar); + if (ret) + goto error_fail_uar_alloc; + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) { + INIT_LIST_HEAD(&context->page_list); + mutex_init(&context->page_mutex); + } + + resp.cqe_size = hr_dev->caps.cqe_sz; + + ret = ib_copy_to_udata(udata, &resp, + min(udata->outlen, sizeof(resp))); + if (ret) + goto error_fail_copy_to_udata; + + return 0; + +error_fail_copy_to_udata: + hns_roce_uar_free(hr_dev, &context->uar); + +error_fail_uar_alloc: + return ret; +} + +static void hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext) +{ + struct hns_roce_ucontext *context = to_hr_ucontext(ibcontext); + + hns_roce_uar_free(to_hr_dev(ibcontext->device), &context->uar); +} + +static int hns_roce_mmap(struct ib_ucontext *context, + struct vm_area_struct *vma) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(context->device); + + switch (vma->vm_pgoff) { + case 0: + return rdma_user_mmap_io(context, vma, + to_hr_ucontext(context)->uar.pfn, + PAGE_SIZE, + pgprot_device(vma->vm_page_prot), + NULL); + + /* vm_pgoff: 1 -- TPTR */ + case 1: + if (!hr_dev->tptr_dma_addr || !hr_dev->tptr_size) + return -EINVAL; + /* + * FIXME: using io_remap_pfn_range on the dma address returned + * by dma_alloc_coherent is totally wrong. + */ + return rdma_user_mmap_io(context, vma, + hr_dev->tptr_dma_addr >> PAGE_SHIFT, + hr_dev->tptr_size, + vma->vm_page_prot, + NULL); + + default: + return -EINVAL; + } +} + +static int hns_roce_port_immutable(struct ib_device *ib_dev, u8 port_num, + struct ib_port_immutable *immutable) +{ + struct ib_port_attr attr; + int ret; + + ret = ib_query_port(ib_dev, port_num, &attr); + if (ret) + return ret; + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; + + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; + if (to_hr_dev(ib_dev)->caps.flags & HNS_ROCE_CAP_FLAG_ROCE_V1_V2) + immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; + + return 0; +} + +static void hns_roce_disassociate_ucontext(struct ib_ucontext *ibcontext) +{ +} + +static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_ib_iboe *iboe = &hr_dev->iboe; + + hr_dev->active = false; + unregister_netdevice_notifier(&iboe->nb); + ib_unregister_device(&hr_dev->ib_dev); +} + +static const struct ib_device_ops hns_roce_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_HNS, + .uverbs_abi_ver = 1, + .uverbs_no_driver_id_binding = 1, + + .add_gid = hns_roce_add_gid, + .alloc_pd = hns_roce_alloc_pd, + .alloc_ucontext = hns_roce_alloc_ucontext, + .create_ah = hns_roce_create_ah, + .create_cq = hns_roce_create_cq, + .create_qp = hns_roce_create_qp, + .dealloc_pd = hns_roce_dealloc_pd, + .dealloc_ucontext = hns_roce_dealloc_ucontext, + .del_gid = hns_roce_del_gid, + .dereg_mr = hns_roce_dereg_mr, + .destroy_ah = hns_roce_destroy_ah, + .destroy_cq = hns_roce_destroy_cq, + .disassociate_ucontext = hns_roce_disassociate_ucontext, + .fill_res_cq_entry = hns_roce_fill_res_cq_entry, + .get_dma_mr = hns_roce_get_dma_mr, + .get_link_layer = hns_roce_get_link_layer, + .get_port_immutable = hns_roce_port_immutable, + .mmap = hns_roce_mmap, + .modify_device = hns_roce_modify_device, + .modify_qp = hns_roce_modify_qp, + .query_ah = hns_roce_query_ah, + .query_device = hns_roce_query_device, + .query_pkey = hns_roce_query_pkey, + .query_port = hns_roce_query_port, + .reg_user_mr = hns_roce_reg_user_mr, + + INIT_RDMA_OBJ_SIZE(ib_ah, hns_roce_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, hns_roce_cq, ib_cq), + INIT_RDMA_OBJ_SIZE(ib_pd, hns_roce_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_ucontext, hns_roce_ucontext, ibucontext), +}; + +static const struct ib_device_ops hns_roce_dev_mr_ops = { + .rereg_user_mr = hns_roce_rereg_user_mr, +}; + +static const struct ib_device_ops hns_roce_dev_mw_ops = { + .alloc_mw = hns_roce_alloc_mw, + .dealloc_mw = hns_roce_dealloc_mw, + + INIT_RDMA_OBJ_SIZE(ib_mw, hns_roce_mw, ibmw), +}; + +static const struct ib_device_ops hns_roce_dev_frmr_ops = { + .alloc_mr = hns_roce_alloc_mr, + .map_mr_sg = hns_roce_map_mr_sg, +}; + +static const struct ib_device_ops hns_roce_dev_srq_ops = { + .create_srq = hns_roce_create_srq, + .destroy_srq = hns_roce_destroy_srq, + + INIT_RDMA_OBJ_SIZE(ib_srq, hns_roce_srq, ibsrq), +}; + +static int hns_roce_register_device(struct hns_roce_dev *hr_dev) +{ + int ret; + struct hns_roce_ib_iboe *iboe = NULL; + struct ib_device *ib_dev = NULL; + struct device *dev = hr_dev->dev; + unsigned int i; + + iboe = &hr_dev->iboe; + spin_lock_init(&iboe->lock); + + ib_dev = &hr_dev->ib_dev; + + ib_dev->node_type = RDMA_NODE_IB_CA; + ib_dev->dev.parent = dev; + + ib_dev->phys_port_cnt = hr_dev->caps.num_ports; + ib_dev->local_dma_lkey = hr_dev->caps.reserved_lkey; + ib_dev->num_comp_vectors = hr_dev->caps.num_comp_vectors; + ib_dev->uverbs_cmd_mask = + (1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) | + (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) | + (1ULL << IB_USER_VERBS_CMD_QUERY_PORT) | + (1ULL << IB_USER_VERBS_CMD_ALLOC_PD) | + (1ULL << IB_USER_VERBS_CMD_DEALLOC_PD) | + (1ULL << IB_USER_VERBS_CMD_REG_MR) | + (1ULL << IB_USER_VERBS_CMD_DEREG_MR) | + (1ULL << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | + (1ULL << IB_USER_VERBS_CMD_CREATE_CQ) | + (1ULL << IB_USER_VERBS_CMD_DESTROY_CQ) | + (1ULL << IB_USER_VERBS_CMD_CREATE_QP) | + (1ULL << IB_USER_VERBS_CMD_MODIFY_QP) | + (1ULL << IB_USER_VERBS_CMD_QUERY_QP) | + (1ULL << IB_USER_VERBS_CMD_DESTROY_QP); + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_REREG_MR) { + ib_dev->uverbs_cmd_mask |= (1ULL << IB_USER_VERBS_CMD_REREG_MR); + ib_set_device_ops(ib_dev, &hns_roce_dev_mr_ops); + } + + /* MW */ + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_MW) { + ib_dev->uverbs_cmd_mask |= + (1ULL << IB_USER_VERBS_CMD_ALLOC_MW) | + (1ULL << IB_USER_VERBS_CMD_DEALLOC_MW); + ib_set_device_ops(ib_dev, &hns_roce_dev_mw_ops); + } + + /* FRMR */ + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR) + ib_set_device_ops(ib_dev, &hns_roce_dev_frmr_ops); + + /* SRQ */ + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { + ib_dev->uverbs_cmd_mask |= + (1ULL << IB_USER_VERBS_CMD_CREATE_SRQ) | + (1ULL << IB_USER_VERBS_CMD_MODIFY_SRQ) | + (1ULL << IB_USER_VERBS_CMD_QUERY_SRQ) | + (1ULL << IB_USER_VERBS_CMD_DESTROY_SRQ) | + (1ULL << IB_USER_VERBS_CMD_POST_SRQ_RECV); + ib_set_device_ops(ib_dev, &hns_roce_dev_srq_ops); + ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_srq_ops); + } + + ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops); + ib_set_device_ops(ib_dev, &hns_roce_dev_ops); + for (i = 0; i < hr_dev->caps.num_ports; i++) { + if (!hr_dev->iboe.netdevs[i]) + continue; + + ret = ib_device_set_netdev(ib_dev, hr_dev->iboe.netdevs[i], + i + 1); + if (ret) + return ret; + } + dma_set_max_seg_size(dev, UINT_MAX); + ret = ib_register_device(ib_dev, "hns_%d", dev); + if (ret) { + dev_err(dev, "ib_register_device failed!\n"); + return ret; + } + + ret = hns_roce_setup_mtu_mac(hr_dev); + if (ret) { + dev_err(dev, "setup_mtu_mac failed!\n"); + goto error_failed_setup_mtu_mac; + } + + iboe->nb.notifier_call = hns_roce_netdev_event; + ret = register_netdevice_notifier(&iboe->nb); + if (ret) { + dev_err(dev, "register_netdevice_notifier failed!\n"); + goto error_failed_setup_mtu_mac; + } + + hr_dev->active = true; + return 0; + +error_failed_setup_mtu_mac: + ib_unregister_device(ib_dev); + + return ret; +} + +static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) +{ + struct device *dev = hr_dev->dev; + int ret; + + ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table, + HEM_TYPE_MTPT, hr_dev->caps.mtpt_entry_sz, + hr_dev->caps.num_mtpts, 1); + if (ret) { + dev_err(dev, "Failed to init MTPT context memory, aborting.\n"); + return ret; + } + + ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.qp_table, + HEM_TYPE_QPC, hr_dev->caps.qpc_sz, + hr_dev->caps.num_qps, 1); + if (ret) { + dev_err(dev, "Failed to init QP context memory, aborting.\n"); + goto err_unmap_dmpt; + } + + ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.irrl_table, + HEM_TYPE_IRRL, + hr_dev->caps.irrl_entry_sz * + hr_dev->caps.max_qp_init_rdma, + hr_dev->caps.num_qps, 1); + if (ret) { + dev_err(dev, "Failed to init irrl_table memory, aborting.\n"); + goto err_unmap_qp; + } + + if (hr_dev->caps.trrl_entry_sz) { + ret = hns_roce_init_hem_table(hr_dev, + &hr_dev->qp_table.trrl_table, + HEM_TYPE_TRRL, + hr_dev->caps.trrl_entry_sz * + hr_dev->caps.max_qp_dest_rdma, + hr_dev->caps.num_qps, 1); + if (ret) { + dev_err(dev, + "Failed to init trrl_table memory, aborting.\n"); + goto err_unmap_irrl; + } + } + + ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cq_table.table, + HEM_TYPE_CQC, hr_dev->caps.cqc_entry_sz, + hr_dev->caps.num_cqs, 1); + if (ret) { + dev_err(dev, "Failed to init CQ context memory, aborting.\n"); + goto err_unmap_trrl; + } + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { + ret = hns_roce_init_hem_table(hr_dev, &hr_dev->srq_table.table, + HEM_TYPE_SRQC, + hr_dev->caps.srqc_entry_sz, + hr_dev->caps.num_srqs, 1); + if (ret) { + dev_err(dev, + "Failed to init SRQ context memory, aborting.\n"); + goto err_unmap_cq; + } + } + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { + ret = hns_roce_init_hem_table(hr_dev, + &hr_dev->qp_table.sccc_table, + HEM_TYPE_SCCC, + hr_dev->caps.sccc_sz, + hr_dev->caps.num_qps, 1); + if (ret) { + dev_err(dev, + "Failed to init SCC context memory, aborting.\n"); + goto err_unmap_srq; + } + } + + if (hr_dev->caps.qpc_timer_entry_sz) { + ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qpc_timer_table, + HEM_TYPE_QPC_TIMER, + hr_dev->caps.qpc_timer_entry_sz, + hr_dev->caps.num_qpc_timer, 1); + if (ret) { + dev_err(dev, + "Failed to init QPC timer memory, aborting.\n"); + goto err_unmap_ctx; + } + } + + if (hr_dev->caps.cqc_timer_entry_sz) { + ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cqc_timer_table, + HEM_TYPE_CQC_TIMER, + hr_dev->caps.cqc_timer_entry_sz, + hr_dev->caps.num_cqc_timer, 1); + if (ret) { + dev_err(dev, + "Failed to init CQC timer memory, aborting.\n"); + goto err_unmap_qpc_timer; + } + } + + return 0; + +err_unmap_qpc_timer: + if (hr_dev->caps.qpc_timer_entry_sz) + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qpc_timer_table); + +err_unmap_ctx: + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) + hns_roce_cleanup_hem_table(hr_dev, + &hr_dev->qp_table.sccc_table); +err_unmap_srq: + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->srq_table.table); + +err_unmap_cq: + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); + +err_unmap_trrl: + if (hr_dev->caps.trrl_entry_sz) + hns_roce_cleanup_hem_table(hr_dev, + &hr_dev->qp_table.trrl_table); + +err_unmap_irrl: + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); + +err_unmap_qp: + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); + +err_unmap_dmpt: + hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); + + return ret; +} + +/** + * hns_roce_setup_hca - setup host channel adapter + * @hr_dev: pointer to hns roce device + * Return : int + */ +static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) +{ + struct device *dev = hr_dev->dev; + int ret; + + spin_lock_init(&hr_dev->sm_lock); + spin_lock_init(&hr_dev->bt_cmd_lock); + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) { + INIT_LIST_HEAD(&hr_dev->pgdir_list); + mutex_init(&hr_dev->pgdir_mutex); + } + + ret = hns_roce_init_uar_table(hr_dev); + if (ret) { + dev_err(dev, "Failed to initialize uar table. aborting\n"); + return ret; + } + + ret = hns_roce_uar_alloc(hr_dev, &hr_dev->priv_uar); + if (ret) { + dev_err(dev, "Failed to allocate priv_uar.\n"); + goto err_uar_table_free; + } + + ret = hns_roce_init_pd_table(hr_dev); + if (ret) { + dev_err(dev, "Failed to init protected domain table.\n"); + goto err_uar_alloc_free; + } + + ret = hns_roce_init_mr_table(hr_dev); + if (ret) { + dev_err(dev, "Failed to init memory region table.\n"); + goto err_pd_table_free; + } + + ret = hns_roce_init_cq_table(hr_dev); + if (ret) { + dev_err(dev, "Failed to init completion queue table.\n"); + goto err_mr_table_free; + } + + ret = hns_roce_init_qp_table(hr_dev); + if (ret) { + dev_err(dev, "Failed to init queue pair table.\n"); + goto err_cq_table_free; + } + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { + ret = hns_roce_init_srq_table(hr_dev); + if (ret) { + dev_err(dev, + "Failed to init share receive queue table.\n"); + goto err_qp_table_free; + } + } + + return 0; + +err_qp_table_free: + hns_roce_cleanup_qp_table(hr_dev); + +err_cq_table_free: + hns_roce_cleanup_cq_table(hr_dev); + +err_mr_table_free: + hns_roce_cleanup_mr_table(hr_dev); + +err_pd_table_free: + hns_roce_cleanup_pd_table(hr_dev); + +err_uar_alloc_free: + hns_roce_uar_free(hr_dev, &hr_dev->priv_uar); + +err_uar_table_free: + hns_roce_cleanup_uar_table(hr_dev); + return ret; +} + +static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq) +{ + struct hns_roce_cq *hr_cq = to_hr_cq(cq); + unsigned long flags; + + spin_lock_irqsave(&hr_cq->lock, flags); + if (cq->comp_handler) { + if (!hr_cq->is_armed) { + hr_cq->is_armed = 1; + list_add_tail(&hr_cq->node, cq_list); + } + } + spin_unlock_irqrestore(&hr_cq->lock, flags); +} + +void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_qp *hr_qp; + struct hns_roce_cq *hr_cq; + struct list_head cq_list; + unsigned long flags_qp; + unsigned long flags; + + INIT_LIST_HEAD(&cq_list); + + spin_lock_irqsave(&hr_dev->qp_list_lock, flags); + list_for_each_entry(hr_qp, &hr_dev->qp_list, node) { + spin_lock_irqsave(&hr_qp->sq.lock, flags_qp); + if (hr_qp->sq.tail != hr_qp->sq.head) + check_and_get_armed_cq(&cq_list, hr_qp->ibqp.send_cq); + spin_unlock_irqrestore(&hr_qp->sq.lock, flags_qp); + + spin_lock_irqsave(&hr_qp->rq.lock, flags_qp); + if ((!hr_qp->ibqp.srq) && (hr_qp->rq.tail != hr_qp->rq.head)) + check_and_get_armed_cq(&cq_list, hr_qp->ibqp.recv_cq); + spin_unlock_irqrestore(&hr_qp->rq.lock, flags_qp); + } + + list_for_each_entry(hr_cq, &cq_list, node) + hns_roce_cq_completion(hr_dev, hr_cq->cqn); + + spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); +} + +int hns_roce_init(struct hns_roce_dev *hr_dev) +{ + struct device *dev = hr_dev->dev; + int ret; + + if (hr_dev->hw->reset) { + ret = hr_dev->hw->reset(hr_dev, true); + if (ret) { + dev_err(dev, "Reset RoCE engine failed!\n"); + return ret; + } + } + hr_dev->is_reset = false; + + if (hr_dev->hw->cmq_init) { + ret = hr_dev->hw->cmq_init(hr_dev); + if (ret) { + dev_err(dev, "Init RoCE Command Queue failed!\n"); + goto error_failed_cmq_init; + } + } + + ret = hr_dev->hw->hw_profile(hr_dev); + if (ret) { + dev_err(dev, "Get RoCE engine profile failed!\n"); + goto error_failed_cmd_init; + } + + ret = hns_roce_cmd_init(hr_dev); + if (ret) { + dev_err(dev, "cmd init failed!\n"); + goto error_failed_cmd_init; + } + + /* EQ depends on poll mode, event mode depends on EQ */ + ret = hr_dev->hw->init_eq(hr_dev); + if (ret) { + dev_err(dev, "eq init failed!\n"); + goto error_failed_eq_table; + } + + if (hr_dev->cmd_mod) { + ret = hns_roce_cmd_use_events(hr_dev); + if (ret) { + dev_warn(dev, + "Cmd event mode failed, set back to poll!\n"); + hns_roce_cmd_use_polling(hr_dev); + } + } + + ret = hns_roce_init_hem(hr_dev); + if (ret) { + dev_err(dev, "init HEM(Hardware Entry Memory) failed!\n"); + goto error_failed_init_hem; + } + + ret = hns_roce_setup_hca(hr_dev); + if (ret) { + dev_err(dev, "setup hca failed!\n"); + goto error_failed_setup_hca; + } + + if (hr_dev->hw->hw_init) { + ret = hr_dev->hw->hw_init(hr_dev); + if (ret) { + dev_err(dev, "hw_init failed!\n"); + goto error_failed_engine_init; + } + } + + INIT_LIST_HEAD(&hr_dev->qp_list); + spin_lock_init(&hr_dev->qp_list_lock); + + ret = hns_roce_register_device(hr_dev); + if (ret) + goto error_failed_register_device; + + return 0; + +error_failed_register_device: + if (hr_dev->hw->hw_exit) + hr_dev->hw->hw_exit(hr_dev); + +error_failed_engine_init: + hns_roce_cleanup_bitmap(hr_dev); + +error_failed_setup_hca: + hns_roce_cleanup_hem(hr_dev); + +error_failed_init_hem: + if (hr_dev->cmd_mod) + hns_roce_cmd_use_polling(hr_dev); + hr_dev->hw->cleanup_eq(hr_dev); + +error_failed_eq_table: + hns_roce_cmd_cleanup(hr_dev); + +error_failed_cmd_init: + if (hr_dev->hw->cmq_exit) + hr_dev->hw->cmq_exit(hr_dev); + +error_failed_cmq_init: + if (hr_dev->hw->reset) { + if (hr_dev->hw->reset(hr_dev, false)) + dev_err(dev, "Dereset RoCE engine failed!\n"); + } + + return ret; +} + +void hns_roce_exit(struct hns_roce_dev *hr_dev) +{ + hns_roce_unregister_device(hr_dev); + + if (hr_dev->hw->hw_exit) + hr_dev->hw->hw_exit(hr_dev); + hns_roce_cleanup_bitmap(hr_dev); + hns_roce_cleanup_hem(hr_dev); + + if (hr_dev->cmd_mod) + hns_roce_cmd_use_polling(hr_dev); + + hr_dev->hw->cleanup_eq(hr_dev); + hns_roce_cmd_cleanup(hr_dev); + if (hr_dev->hw->cmq_exit) + hr_dev->hw->cmq_exit(hr_dev); + if (hr_dev->hw->reset) + hr_dev->hw->reset(hr_dev, false); +} + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); +MODULE_AUTHOR("Nenglong Zhao <zhaonenglong@hisilicon.com>"); +MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>"); +MODULE_DESCRIPTION("HNS RoCE Driver"); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c new file mode 100644 index 000000000..d5b3b10e0 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -0,0 +1,1115 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/platform_device.h> +#include <linux/vmalloc.h> +#include <rdma/ib_umem.h> +#include "hns_roce_device.h" +#include "hns_roce_cmd.h" +#include "hns_roce_hem.h" + +static u32 hw_index_to_key(unsigned long ind) +{ + return (u32)(ind >> 24) | (ind << 8); +} + +unsigned long key_to_hw_index(u32 key) +{ + return (key << 24) | (key >> 8); +} + +static int hns_roce_hw_create_mpt(struct hns_roce_dev *hr_dev, + struct hns_roce_cmd_mailbox *mailbox, + unsigned long mpt_index) +{ + return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0, + HNS_ROCE_CMD_CREATE_MPT, + HNS_ROCE_CMD_TIMEOUT_MSECS); +} + +int hns_roce_hw_destroy_mpt(struct hns_roce_dev *hr_dev, + struct hns_roce_cmd_mailbox *mailbox, + unsigned long mpt_index) +{ + return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0, + mpt_index, !mailbox, HNS_ROCE_CMD_DESTROY_MPT, + HNS_ROCE_CMD_TIMEOUT_MSECS); +} + +static int alloc_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, + u32 pd, u64 iova, u64 size, u32 access) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + unsigned long obj = 0; + int err; + + /* Allocate a key for mr from mr_table */ + err = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &obj); + if (err) { + ibdev_err(ibdev, + "failed to alloc bitmap for MR key, ret = %d.\n", + err); + return -ENOMEM; + } + + mr->iova = iova; /* MR va starting addr */ + mr->size = size; /* MR addr range */ + mr->pd = pd; /* MR num */ + mr->access = access; /* MR access permit */ + mr->enabled = 0; /* MR active status */ + mr->key = hw_index_to_key(obj); /* MR key */ + + err = hns_roce_table_get(hr_dev, &hr_dev->mr_table.mtpt_table, obj); + if (err) { + ibdev_err(ibdev, "failed to alloc mtpt, ret = %d.\n", err); + goto err_free_bitmap; + } + + return 0; +err_free_bitmap: + hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, obj, BITMAP_NO_RR); + return err; +} + +static void free_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr) +{ + unsigned long obj = key_to_hw_index(mr->key); + + hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, obj); + hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, obj, BITMAP_NO_RR); +} + +static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, + size_t length, struct ib_udata *udata, u64 start, + int access) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + bool is_fast = mr->type == MR_TYPE_FRMR; + struct hns_roce_buf_attr buf_attr = {}; + int err; + + mr->pbl_hop_num = is_fast ? 1 : hr_dev->caps.pbl_hop_num; + buf_attr.page_shift = is_fast ? PAGE_SHIFT : + hr_dev->caps.pbl_buf_pg_sz + PAGE_SHIFT; + buf_attr.region[0].size = length; + buf_attr.region[0].hopnum = mr->pbl_hop_num; + buf_attr.region_count = 1; + buf_attr.fixed_page = true; + buf_attr.user_access = access; + /* fast MR's buffer is alloced before mapping, not at creation */ + buf_attr.mtt_only = is_fast; + + err = hns_roce_mtr_create(hr_dev, &mr->pbl_mtr, &buf_attr, + hr_dev->caps.pbl_ba_pg_sz + HNS_HW_PAGE_SHIFT, + udata, start); + if (err) + ibdev_err(ibdev, "failed to alloc pbl mtr, ret = %d.\n", err); + else + mr->npages = mr->pbl_mtr.hem_cfg.buf_pg_count; + + return err; +} + +static void free_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr) +{ + hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr); +} + +static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, + struct hns_roce_mr *mr) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + int ret; + + if (mr->enabled) { + ret = hns_roce_hw_destroy_mpt(hr_dev, NULL, + key_to_hw_index(mr->key) & + (hr_dev->caps.num_mtpts - 1)); + if (ret) + ibdev_warn(ibdev, "failed to destroy mpt, ret = %d.\n", + ret); + } + + free_mr_pbl(hr_dev, mr); + free_mr_key(hr_dev, mr); +} + +static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, + struct hns_roce_mr *mr) +{ + unsigned long mtpt_idx = key_to_hw_index(mr->key); + struct hns_roce_cmd_mailbox *mailbox; + struct device *dev = hr_dev->dev; + int ret; + + /* Allocate mailbox memory */ + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) { + ret = PTR_ERR(mailbox); + return ret; + } + + if (mr->type != MR_TYPE_FRMR) + ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr, + mtpt_idx); + else + ret = hr_dev->hw->frmr_write_mtpt(hr_dev, mailbox->buf, mr); + if (ret) { + dev_err(dev, "failed to write mtpt, ret = %d.\n", ret); + goto err_page; + } + + ret = hns_roce_hw_create_mpt(hr_dev, mailbox, + mtpt_idx & (hr_dev->caps.num_mtpts - 1)); + if (ret) { + dev_err(dev, "failed to create mpt, ret = %d.\n", ret); + goto err_page; + } + + mr->enabled = 1; + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return 0; + +err_page: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} + +int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; + int ret; + + ret = hns_roce_bitmap_init(&mr_table->mtpt_bitmap, + hr_dev->caps.num_mtpts, + hr_dev->caps.num_mtpts - 1, + hr_dev->caps.reserved_mrws, 0); + return ret; +} + +void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; + + hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap); +} + +struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); + struct hns_roce_mr *mr; + int ret; + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (mr == NULL) + return ERR_PTR(-ENOMEM); + + mr->type = MR_TYPE_DMA; + + /* Allocate memory region key */ + hns_roce_hem_list_init(&mr->pbl_mtr.hem_list); + ret = alloc_mr_key(hr_dev, mr, to_hr_pd(pd)->pdn, 0, 0, acc); + if (ret) + goto err_free; + + ret = hns_roce_mr_enable(to_hr_dev(pd->device), mr); + if (ret) + goto err_mr; + + mr->ibmr.rkey = mr->ibmr.lkey = mr->key; + + return &mr->ibmr; +err_mr: + free_mr_key(hr_dev, mr); + +err_free: + kfree(mr); + return ERR_PTR(ret); +} + +struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, + u64 virt_addr, int access_flags, + struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); + struct hns_roce_mr *mr; + int ret; + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); + + mr->type = MR_TYPE_MR; + ret = alloc_mr_key(hr_dev, mr, to_hr_pd(pd)->pdn, virt_addr, length, + access_flags); + if (ret) + goto err_alloc_mr; + + ret = alloc_mr_pbl(hr_dev, mr, length, udata, start, access_flags); + if (ret) + goto err_alloc_key; + + ret = hns_roce_mr_enable(hr_dev, mr); + if (ret) + goto err_alloc_pbl; + + mr->ibmr.rkey = mr->ibmr.lkey = mr->key; + + return &mr->ibmr; + +err_alloc_pbl: + free_mr_pbl(hr_dev, mr); +err_alloc_key: + free_mr_key(hr_dev, mr); +err_alloc_mr: + kfree(mr); + return ERR_PTR(ret); +} + +static int rereg_mr_trans(struct ib_mr *ibmr, int flags, + u64 start, u64 length, + u64 virt_addr, int mr_access_flags, + struct hns_roce_cmd_mailbox *mailbox, + u32 pdn, struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_mr *mr = to_hr_mr(ibmr); + int ret; + + free_mr_pbl(hr_dev, mr); + ret = alloc_mr_pbl(hr_dev, mr, length, udata, start, mr_access_flags); + if (ret) { + ibdev_err(ibdev, "failed to create mr PBL, ret = %d.\n", ret); + return ret; + } + + ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn, + mr_access_flags, virt_addr, + length, mailbox->buf); + if (ret) { + ibdev_err(ibdev, "failed to write mtpt, ret = %d.\n", ret); + free_mr_pbl(hr_dev, mr); + } + + return ret; +} + +int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, struct ib_pd *pd, + struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); + struct ib_device *ib_dev = &hr_dev->ib_dev; + struct hns_roce_mr *mr = to_hr_mr(ibmr); + struct hns_roce_cmd_mailbox *mailbox; + unsigned long mtpt_idx; + u32 pdn = 0; + int ret; + + if (!mr->enabled) + return -EINVAL; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1); + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, mtpt_idx, 0, + HNS_ROCE_CMD_QUERY_MPT, + HNS_ROCE_CMD_TIMEOUT_MSECS); + if (ret) + goto free_cmd_mbox; + + ret = hns_roce_hw_destroy_mpt(hr_dev, NULL, mtpt_idx); + if (ret) + ibdev_warn(ib_dev, "failed to destroy MPT, ret = %d.\n", ret); + + mr->enabled = 0; + + if (flags & IB_MR_REREG_PD) + pdn = to_hr_pd(pd)->pdn; + + if (flags & IB_MR_REREG_TRANS) { + ret = rereg_mr_trans(ibmr, flags, + start, length, + virt_addr, mr_access_flags, + mailbox, pdn, udata); + if (ret) + goto free_cmd_mbox; + } else { + ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn, + mr_access_flags, virt_addr, + length, mailbox->buf); + if (ret) + goto free_cmd_mbox; + } + + ret = hns_roce_hw_create_mpt(hr_dev, mailbox, mtpt_idx); + if (ret) { + ibdev_err(ib_dev, "failed to create MPT, ret = %d.\n", ret); + goto free_cmd_mbox; + } + + mr->enabled = 1; + if (flags & IB_MR_REREG_ACCESS) + mr->access = mr_access_flags; + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return 0; + +free_cmd_mbox: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} + +int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); + struct hns_roce_mr *mr = to_hr_mr(ibmr); + int ret = 0; + + if (hr_dev->hw->dereg_mr) { + ret = hr_dev->hw->dereg_mr(hr_dev, mr, udata); + } else { + hns_roce_mr_free(hr_dev, mr); + kfree(mr); + } + + return ret; +} + +struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, + u32 max_num_sg) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); + struct device *dev = hr_dev->dev; + struct hns_roce_mr *mr; + u64 length; + int ret; + + if (mr_type != IB_MR_TYPE_MEM_REG) + return ERR_PTR(-EINVAL); + + if (max_num_sg > HNS_ROCE_FRMR_MAX_PA) { + dev_err(dev, "max_num_sg larger than %d\n", + HNS_ROCE_FRMR_MAX_PA); + return ERR_PTR(-EINVAL); + } + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); + + mr->type = MR_TYPE_FRMR; + + /* Allocate memory region key */ + length = max_num_sg * (1 << PAGE_SHIFT); + ret = alloc_mr_key(hr_dev, mr, to_hr_pd(pd)->pdn, 0, length, 0); + if (ret) + goto err_free; + + ret = alloc_mr_pbl(hr_dev, mr, length, NULL, 0, 0); + if (ret) + goto err_key; + + ret = hns_roce_mr_enable(hr_dev, mr); + if (ret) + goto err_pbl; + + mr->ibmr.rkey = mr->ibmr.lkey = mr->key; + mr->ibmr.length = length; + + return &mr->ibmr; + +err_pbl: + free_mr_pbl(hr_dev, mr); +err_key: + free_mr_key(hr_dev, mr); +err_free: + kfree(mr); + return ERR_PTR(ret); +} + +static int hns_roce_set_page(struct ib_mr *ibmr, u64 addr) +{ + struct hns_roce_mr *mr = to_hr_mr(ibmr); + + if (likely(mr->npages < mr->pbl_mtr.hem_cfg.buf_pg_count)) { + mr->page_list[mr->npages++] = addr; + return 0; + } + + return -ENOBUFS; +} + +int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, + unsigned int *sg_offset) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_mr *mr = to_hr_mr(ibmr); + struct hns_roce_mtr *mtr = &mr->pbl_mtr; + int ret = 0; + + mr->npages = 0; + mr->page_list = kvcalloc(mr->pbl_mtr.hem_cfg.buf_pg_count, + sizeof(dma_addr_t), GFP_KERNEL); + if (!mr->page_list) + return ret; + + ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page); + if (ret < 1) { + ibdev_err(ibdev, "failed to store sg pages %u %u, cnt = %d.\n", + mr->npages, mr->pbl_mtr.hem_cfg.buf_pg_count, ret); + goto err_page_list; + } + + mtr->hem_cfg.region[0].offset = 0; + mtr->hem_cfg.region[0].count = mr->npages; + mtr->hem_cfg.region[0].hopnum = mr->pbl_hop_num; + mtr->hem_cfg.region_count = 1; + ret = hns_roce_mtr_map(hr_dev, mtr, mr->page_list, mr->npages); + if (ret) { + ibdev_err(ibdev, "failed to map sg mtr, ret = %d.\n", ret); + ret = 0; + } else { + mr->pbl_mtr.hem_cfg.buf_pg_shift = ilog2(ibmr->page_size); + ret = mr->npages; + } + +err_page_list: + kvfree(mr->page_list); + mr->page_list = NULL; + + return ret; +} + +static void hns_roce_mw_free(struct hns_roce_dev *hr_dev, + struct hns_roce_mw *mw) +{ + struct device *dev = hr_dev->dev; + int ret; + + if (mw->enabled) { + ret = hns_roce_hw_destroy_mpt(hr_dev, NULL, + key_to_hw_index(mw->rkey) & + (hr_dev->caps.num_mtpts - 1)); + if (ret) + dev_warn(dev, "MW DESTROY_MPT failed (%d)\n", ret); + + hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, + key_to_hw_index(mw->rkey)); + } + + hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, + key_to_hw_index(mw->rkey), BITMAP_NO_RR); +} + +static int hns_roce_mw_enable(struct hns_roce_dev *hr_dev, + struct hns_roce_mw *mw) +{ + struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; + struct hns_roce_cmd_mailbox *mailbox; + struct device *dev = hr_dev->dev; + unsigned long mtpt_idx = key_to_hw_index(mw->rkey); + int ret; + + /* prepare HEM entry memory */ + ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx); + if (ret) + return ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) { + ret = PTR_ERR(mailbox); + goto err_table; + } + + ret = hr_dev->hw->mw_write_mtpt(mailbox->buf, mw); + if (ret) { + dev_err(dev, "MW write mtpt fail!\n"); + goto err_page; + } + + ret = hns_roce_hw_create_mpt(hr_dev, mailbox, + mtpt_idx & (hr_dev->caps.num_mtpts - 1)); + if (ret) { + dev_err(dev, "MW CREATE_MPT failed (%d)\n", ret); + goto err_page; + } + + mw->enabled = 1; + + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return 0; + +err_page: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + +err_table: + hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx); + + return ret; +} + +int hns_roce_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibmw->device); + struct hns_roce_mw *mw = to_hr_mw(ibmw); + unsigned long index = 0; + int ret; + + /* Allocate a key for mw from bitmap */ + ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index); + if (ret) + return ret; + + mw->rkey = hw_index_to_key(index); + + ibmw->rkey = mw->rkey; + mw->pdn = to_hr_pd(ibmw->pd)->pdn; + mw->pbl_hop_num = hr_dev->caps.pbl_hop_num; + mw->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; + mw->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; + + ret = hns_roce_mw_enable(hr_dev, mw); + if (ret) + goto err_mw; + + return 0; + +err_mw: + hns_roce_mw_free(hr_dev, mw); + return ret; +} + +int hns_roce_dealloc_mw(struct ib_mw *ibmw) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibmw->device); + struct hns_roce_mw *mw = to_hr_mw(ibmw); + + hns_roce_mw_free(hr_dev, mw); + return 0; +} + +static int mtr_map_region(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + dma_addr_t *pages, struct hns_roce_buf_region *region) +{ + __le64 *mtts; + int offset; + int count; + int npage; + u64 addr; + int end; + int i; + + /* if hopnum is 0, buffer cannot store BAs, so skip write mtt */ + if (!region->hopnum) + return 0; + + offset = region->offset; + end = offset + region->count; + npage = 0; + while (offset < end) { + mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list, + offset, &count, NULL); + if (!mtts) + return -ENOBUFS; + + for (i = 0; i < count; i++) { + if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) + addr = to_hr_hw_page_addr(pages[npage]); + else + addr = pages[npage]; + + mtts[i] = cpu_to_le64(addr); + npage++; + } + offset += count; + } + + return 0; +} + +static inline bool mtr_has_mtt(struct hns_roce_buf_attr *attr) +{ + int i; + + for (i = 0; i < attr->region_count; i++) + if (attr->region[i].hopnum != HNS_ROCE_HOP_NUM_0 && + attr->region[i].hopnum > 0) + return true; + + /* because the mtr only one root base address, when hopnum is 0 means + * root base address equals the first buffer address, thus all alloced + * memory must in a continuous space accessed by direct mode. + */ + return false; +} + +static inline size_t mtr_bufs_size(struct hns_roce_buf_attr *attr) +{ + size_t size = 0; + int i; + + for (i = 0; i < attr->region_count; i++) + size += attr->region[i].size; + + return size; +} + +static inline size_t mtr_kmem_direct_size(bool is_direct, size_t alloc_size, + unsigned int page_shift) +{ + if (is_direct) + return ALIGN(alloc_size, 1 << page_shift); + else + return HNS_HW_DIRECT_PAGE_COUNT << page_shift; +} + +/* + * check the given pages in continuous address space + * Returns 0 on success, or the error page num. + */ +static inline int mtr_check_direct_pages(dma_addr_t *pages, int page_count, + unsigned int page_shift) +{ + size_t page_size = 1 << page_shift; + int i; + + for (i = 1; i < page_count; i++) + if (pages[i] - pages[i - 1] != page_size) + return i; + + return 0; +} + +static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) +{ + /* release user buffers */ + if (mtr->umem) { + ib_umem_release(mtr->umem); + mtr->umem = NULL; + } + + /* release kernel buffers */ + if (mtr->kmem) { + hns_roce_buf_free(hr_dev, mtr->kmem); + kfree(mtr->kmem); + mtr->kmem = NULL; + } +} + +static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + struct hns_roce_buf_attr *buf_attr, bool is_direct, + struct ib_udata *udata, unsigned long user_addr) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + unsigned int best_pg_shift; + int all_pg_count = 0; + size_t direct_size; + size_t total_size; + int ret; + + total_size = mtr_bufs_size(buf_attr); + if (total_size < 1) { + ibdev_err(ibdev, "Failed to check mtr size\n"); + return -EINVAL; + } + + if (udata) { + unsigned long pgsz_bitmap; + unsigned long page_size; + + mtr->kmem = NULL; + mtr->umem = ib_umem_get(ibdev, user_addr, total_size, + buf_attr->user_access); + if (IS_ERR_OR_NULL(mtr->umem)) { + ibdev_err(ibdev, "Failed to get umem, ret %ld\n", + PTR_ERR(mtr->umem)); + return -ENOMEM; + } + if (buf_attr->fixed_page) + pgsz_bitmap = 1 << buf_attr->page_shift; + else + pgsz_bitmap = GENMASK(buf_attr->page_shift, PAGE_SHIFT); + + page_size = ib_umem_find_best_pgsz(mtr->umem, pgsz_bitmap, + user_addr); + if (!page_size) + return -EINVAL; + best_pg_shift = order_base_2(page_size); + all_pg_count = ib_umem_num_dma_blocks(mtr->umem, page_size); + ret = 0; + } else { + mtr->umem = NULL; + mtr->kmem = kzalloc(sizeof(*mtr->kmem), GFP_KERNEL); + if (!mtr->kmem) { + ibdev_err(ibdev, "Failed to alloc kmem\n"); + return -ENOMEM; + } + direct_size = mtr_kmem_direct_size(is_direct, total_size, + buf_attr->page_shift); + ret = hns_roce_buf_alloc(hr_dev, total_size, direct_size, + mtr->kmem, buf_attr->page_shift); + if (ret) { + ibdev_err(ibdev, "Failed to alloc kmem, ret %d\n", ret); + goto err_alloc_mem; + } + best_pg_shift = buf_attr->page_shift; + all_pg_count = mtr->kmem->npages; + } + + /* must bigger than minimum hardware page shift */ + if (best_pg_shift < HNS_HW_PAGE_SHIFT || all_pg_count < 1) { + ret = -EINVAL; + ibdev_err(ibdev, "Failed to check mtr page shift %d count %d\n", + best_pg_shift, all_pg_count); + goto err_alloc_mem; + } + + mtr->hem_cfg.buf_pg_shift = best_pg_shift; + mtr->hem_cfg.buf_pg_count = all_pg_count; + + return 0; +err_alloc_mem: + mtr_free_bufs(hr_dev, mtr); + return ret; +} + +static int mtr_get_pages(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + dma_addr_t *pages, int count, unsigned int page_shift) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + int npage; + int err; + + if (mtr->umem) + npage = hns_roce_get_umem_bufs(hr_dev, pages, count, 0, + mtr->umem, page_shift); + else + npage = hns_roce_get_kmem_bufs(hr_dev, pages, count, 0, + mtr->kmem); + + if (mtr->hem_cfg.is_direct && npage > 1) { + err = mtr_check_direct_pages(pages, npage, page_shift); + if (err) { + ibdev_err(ibdev, "Failed to check %s direct page-%d\n", + mtr->umem ? "user" : "kernel", err); + npage = err; + } + } + + return npage; +} + +int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + dma_addr_t *pages, int page_cnt) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_buf_region *r; + int err; + int i; + + /* + * Only use the first page address as root ba when hopnum is 0, this + * is because the addresses of all pages are consecutive in this case. + */ + if (mtr->hem_cfg.is_direct) { + mtr->hem_cfg.root_ba = pages[0]; + return 0; + } + + for (i = 0; i < mtr->hem_cfg.region_count; i++) { + r = &mtr->hem_cfg.region[i]; + if (r->offset + r->count > page_cnt) { + err = -EINVAL; + ibdev_err(ibdev, + "failed to check mtr%u end %u + %u, max %u.\n", + i, r->offset, r->count, page_cnt); + return err; + } + + err = mtr_map_region(hr_dev, mtr, &pages[r->offset], r); + if (err) { + ibdev_err(ibdev, + "failed to map mtr%u offset %u, ret = %d.\n", + i, r->offset, err); + return err; + } + } + + return 0; +} + +int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr) +{ + struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg; + int start_index; + int mtt_count; + int total = 0; + __le64 *mtts; + int npage; + u64 addr; + int left; + + if (!mtt_buf || mtt_max < 1) + goto done; + + /* no mtt memory in direct mode, so just return the buffer address */ + if (cfg->is_direct) { + start_index = offset >> HNS_HW_PAGE_SHIFT; + for (mtt_count = 0; mtt_count < cfg->region_count && + total < mtt_max; mtt_count++) { + npage = cfg->region[mtt_count].offset; + if (npage < start_index) + continue; + + addr = cfg->root_ba + (npage << HNS_HW_PAGE_SHIFT); + if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) + mtt_buf[total] = to_hr_hw_page_addr(addr); + else + mtt_buf[total] = addr; + + total++; + } + + goto done; + } + + start_index = offset >> cfg->buf_pg_shift; + left = mtt_max; + while (left > 0) { + mtt_count = 0; + mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list, + start_index + total, + &mtt_count, NULL); + if (!mtts || !mtt_count) + goto done; + + npage = min(mtt_count, left); + left -= npage; + for (mtt_count = 0; mtt_count < npage; mtt_count++) + mtt_buf[total++] = le64_to_cpu(mtts[mtt_count]); + } + +done: + if (base_addr) + *base_addr = cfg->root_ba; + + return total; +} + +static int mtr_init_buf_cfg(struct hns_roce_dev *hr_dev, + struct hns_roce_buf_attr *attr, + struct hns_roce_hem_cfg *cfg, + unsigned int *buf_page_shift) +{ + struct hns_roce_buf_region *r; + unsigned int page_shift; + int page_cnt = 0; + size_t buf_size; + int region_cnt; + + if (cfg->is_direct) { + buf_size = cfg->buf_pg_count << cfg->buf_pg_shift; + page_cnt = DIV_ROUND_UP(buf_size, HNS_HW_PAGE_SIZE); + /* + * When HEM buffer use level-0 addressing, the page size equals + * the buffer size, and the the page size = 4K * 2^N. + */ + cfg->buf_pg_shift = HNS_HW_PAGE_SHIFT + order_base_2(page_cnt); + if (attr->region_count > 1) { + cfg->buf_pg_count = page_cnt; + page_shift = HNS_HW_PAGE_SHIFT; + } else { + cfg->buf_pg_count = 1; + page_shift = cfg->buf_pg_shift; + if (buf_size != 1 << page_shift) { + ibdev_err(&hr_dev->ib_dev, + "failed to check direct size %zu shift %d.\n", + buf_size, page_shift); + return -EINVAL; + } + } + } else { + page_shift = cfg->buf_pg_shift; + } + + /* convert buffer size to page index and page count */ + for (page_cnt = 0, region_cnt = 0; page_cnt < cfg->buf_pg_count && + region_cnt < attr->region_count && + region_cnt < ARRAY_SIZE(cfg->region); region_cnt++) { + r = &cfg->region[region_cnt]; + r->offset = page_cnt; + buf_size = hr_hw_page_align(attr->region[region_cnt].size); + r->count = DIV_ROUND_UP(buf_size, 1 << page_shift); + page_cnt += r->count; + r->hopnum = to_hr_hem_hopnum(attr->region[region_cnt].hopnum, + r->count); + } + + if (region_cnt < 1) { + ibdev_err(&hr_dev->ib_dev, + "failed to check mtr region count, pages = %d.\n", + cfg->buf_pg_count); + return -ENOBUFS; + } + + cfg->region_count = region_cnt; + *buf_page_shift = page_shift; + + return page_cnt; +} + +/** + * hns_roce_mtr_create - Create hns memory translate region. + * + * @mtr: memory translate region + * @buf_attr: buffer attribute for creating mtr + * @ba_page_shift: page shift for multi-hop base address table + * @udata: user space context, if it's NULL, means kernel space + * @user_addr: userspace virtual address to start at + */ +int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + struct hns_roce_buf_attr *buf_attr, + unsigned int ba_page_shift, struct ib_udata *udata, + unsigned long user_addr) +{ + struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg; + struct ib_device *ibdev = &hr_dev->ib_dev; + unsigned int buf_page_shift = 0; + dma_addr_t *pages = NULL; + int all_pg_cnt; + int get_pg_cnt; + int ret = 0; + + /* if disable mtt, all pages must in a continuous address range */ + cfg->is_direct = !mtr_has_mtt(buf_attr); + + /* if buffer only need mtt, just init the hem cfg */ + if (buf_attr->mtt_only) { + cfg->buf_pg_shift = buf_attr->page_shift; + cfg->buf_pg_count = mtr_bufs_size(buf_attr) >> + buf_attr->page_shift; + mtr->umem = NULL; + mtr->kmem = NULL; + } else { + ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, cfg->is_direct, + udata, user_addr); + if (ret) { + ibdev_err(ibdev, + "failed to alloc mtr bufs, ret = %d.\n", ret); + return ret; + } + } + + all_pg_cnt = mtr_init_buf_cfg(hr_dev, buf_attr, cfg, &buf_page_shift); + if (all_pg_cnt < 1) { + ret = -ENOBUFS; + ibdev_err(ibdev, "failed to init mtr buf cfg.\n"); + goto err_alloc_bufs; + } + + hns_roce_hem_list_init(&mtr->hem_list); + if (!cfg->is_direct) { + ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list, + cfg->region, cfg->region_count, + ba_page_shift); + if (ret) { + ibdev_err(ibdev, "failed to request mtr hem, ret = %d.\n", + ret); + goto err_alloc_bufs; + } + cfg->root_ba = mtr->hem_list.root_ba; + cfg->ba_pg_shift = ba_page_shift; + } else { + cfg->ba_pg_shift = cfg->buf_pg_shift; + } + + /* no buffer to map */ + if (buf_attr->mtt_only) + return 0; + + /* alloc a tmp array to store buffer's dma address */ + pages = kvcalloc(all_pg_cnt, sizeof(dma_addr_t), GFP_KERNEL); + if (!pages) { + ret = -ENOMEM; + ibdev_err(ibdev, "failed to alloc mtr page list %d.\n", + all_pg_cnt); + goto err_alloc_hem_list; + } + + get_pg_cnt = mtr_get_pages(hr_dev, mtr, pages, all_pg_cnt, + buf_page_shift); + if (get_pg_cnt != all_pg_cnt) { + ibdev_err(ibdev, "failed to get mtr page %d != %d.\n", + get_pg_cnt, all_pg_cnt); + ret = -ENOBUFS; + goto err_alloc_page_list; + } + + /* write buffer's dma address to BA table */ + ret = hns_roce_mtr_map(hr_dev, mtr, pages, all_pg_cnt); + if (ret) { + ibdev_err(ibdev, "failed to map mtr pages, ret = %d.\n", ret); + goto err_alloc_page_list; + } + + /* drop tmp array */ + kvfree(pages); + return 0; +err_alloc_page_list: + kvfree(pages); +err_alloc_hem_list: + hns_roce_hem_list_release(hr_dev, &mtr->hem_list); +err_alloc_bufs: + mtr_free_bufs(hr_dev, mtr); + return ret; +} + +void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) +{ + /* release multi-hop addressing resource */ + hns_roce_hem_list_release(hr_dev, &mtr->hem_list); + + /* free buffers */ + mtr_free_bufs(hr_dev, mtr); +} diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c new file mode 100644 index 000000000..012a769d6 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_pd.c @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/platform_device.h> +#include <linux/pci.h> +#include <uapi/rdma/hns-abi.h> +#include "hns_roce_device.h" + +static int hns_roce_pd_alloc(struct hns_roce_dev *hr_dev, unsigned long *pdn) +{ + return hns_roce_bitmap_alloc(&hr_dev->pd_bitmap, pdn) ? -ENOMEM : 0; +} + +static void hns_roce_pd_free(struct hns_roce_dev *hr_dev, unsigned long pdn) +{ + hns_roce_bitmap_free(&hr_dev->pd_bitmap, pdn, BITMAP_NO_RR); +} + +int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev) +{ + return hns_roce_bitmap_init(&hr_dev->pd_bitmap, hr_dev->caps.num_pds, + hr_dev->caps.num_pds - 1, + hr_dev->caps.reserved_pds, 0); +} + +void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev) +{ + hns_roce_bitmap_cleanup(&hr_dev->pd_bitmap); +} + +int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) +{ + struct ib_device *ib_dev = ibpd->device; + struct hns_roce_pd *pd = to_hr_pd(ibpd); + int ret; + + ret = hns_roce_pd_alloc(to_hr_dev(ib_dev), &pd->pdn); + if (ret) { + ibdev_err(ib_dev, "failed to alloc pd, ret = %d.\n", ret); + return ret; + } + + if (udata) { + struct hns_roce_ib_alloc_pd_resp resp = {.pdn = pd->pdn}; + + ret = ib_copy_to_udata(udata, &resp, + min(udata->outlen, sizeof(resp))); + if (ret) { + hns_roce_pd_free(to_hr_dev(ib_dev), pd->pdn); + ibdev_err(ib_dev, "failed to copy to udata, ret = %d\n", ret); + } + } + + return ret; +} + +int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) +{ + hns_roce_pd_free(to_hr_dev(pd->device), to_hr_pd(pd)->pdn); + return 0; +} + +int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) +{ + struct resource *res; + int ret; + + /* Using bitmap to manager UAR index */ + ret = hns_roce_bitmap_alloc(&hr_dev->uar_table.bitmap, &uar->logic_idx); + if (ret) + return -ENOMEM; + + if (uar->logic_idx > 0 && hr_dev->caps.phy_num_uars > 1) + uar->index = (uar->logic_idx - 1) % + (hr_dev->caps.phy_num_uars - 1) + 1; + else + uar->index = 0; + + if (!dev_is_pci(hr_dev->dev)) { + res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(&hr_dev->pdev->dev, "memory resource not found!\n"); + return -EINVAL; + } + uar->pfn = ((res->start) >> PAGE_SHIFT) + uar->index; + } else { + uar->pfn = ((pci_resource_start(hr_dev->pci_dev, 2)) + >> PAGE_SHIFT); + } + + return 0; +} + +void hns_roce_uar_free(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) +{ + hns_roce_bitmap_free(&hr_dev->uar_table.bitmap, uar->logic_idx, + BITMAP_NO_RR); +} + +int hns_roce_init_uar_table(struct hns_roce_dev *hr_dev) +{ + return hns_roce_bitmap_init(&hr_dev->uar_table.bitmap, + hr_dev->caps.num_uars, + hr_dev->caps.num_uars - 1, + hr_dev->caps.reserved_uars, 0); +} + +void hns_roce_cleanup_uar_table(struct hns_roce_dev *hr_dev) +{ + hns_roce_bitmap_cleanup(&hr_dev->uar_table.bitmap); +} diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c new file mode 100644 index 000000000..d1c07f1f8 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -0,0 +1,1320 @@ +/* + * Copyright (c) 2016 Hisilicon Limited. + * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <rdma/ib_addr.h> +#include <rdma/ib_umem.h> +#include <rdma/uverbs_ioctl.h> +#include "hns_roce_common.h" +#include "hns_roce_device.h" +#include "hns_roce_hem.h" +#include <rdma/hns-abi.h> + +static void flush_work_handle(struct work_struct *work) +{ + struct hns_roce_work *flush_work = container_of(work, + struct hns_roce_work, work); + struct hns_roce_qp *hr_qp = container_of(flush_work, + struct hns_roce_qp, flush_work); + struct device *dev = flush_work->hr_dev->dev; + struct ib_qp_attr attr; + int attr_mask; + int ret; + + attr_mask = IB_QP_STATE; + attr.qp_state = IB_QPS_ERR; + + if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) { + ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL); + if (ret) + dev_err(dev, "Modify QP to error state failed(%d) during CQE flush\n", + ret); + } + + /* + * make sure we signal QP destroy leg that flush QP was completed + * so that it can safely proceed ahead now and destroy QP + */ + if (atomic_dec_and_test(&hr_qp->refcount)) + complete(&hr_qp->free); +} + +void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +{ + struct hns_roce_work *flush_work = &hr_qp->flush_work; + + flush_work->hr_dev = hr_dev; + INIT_WORK(&flush_work->work, flush_work_handle); + atomic_inc(&hr_qp->refcount); + queue_work(hr_dev->irq_workq, &flush_work->work); +} + +void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) +{ + struct device *dev = hr_dev->dev; + struct hns_roce_qp *qp; + + xa_lock(&hr_dev->qp_table_xa); + qp = __hns_roce_qp_lookup(hr_dev, qpn); + if (qp) + atomic_inc(&qp->refcount); + xa_unlock(&hr_dev->qp_table_xa); + + if (!qp) { + dev_warn(dev, "Async event for bogus QP %08x\n", qpn); + return; + } + + if (hr_dev->hw_rev != HNS_ROCE_HW_VER1 && + (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR || + event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || + event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR)) { + qp->state = IB_QPS_ERR; + if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) + init_flush_work(hr_dev, qp); + } + + qp->event(qp, (enum hns_roce_event)event_type); + + if (atomic_dec_and_test(&qp->refcount)) + complete(&qp->free); +} + +static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, + enum hns_roce_event type) +{ + struct ib_qp *ibqp = &hr_qp->ibqp; + struct ib_event event; + + if (ibqp->event_handler) { + event.device = ibqp->device; + event.element.qp = ibqp; + switch (type) { + case HNS_ROCE_EVENT_TYPE_PATH_MIG: + event.event = IB_EVENT_PATH_MIG; + break; + case HNS_ROCE_EVENT_TYPE_COMM_EST: + event.event = IB_EVENT_COMM_EST; + break; + case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: + event.event = IB_EVENT_SQ_DRAINED; + break; + case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: + event.event = IB_EVENT_QP_LAST_WQE_REACHED; + break; + case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: + event.event = IB_EVENT_QP_FATAL; + break; + case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: + event.event = IB_EVENT_PATH_MIG_ERR; + break; + case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + event.event = IB_EVENT_QP_REQ_ERR; + break; + case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + event.event = IB_EVENT_QP_ACCESS_ERR; + break; + default: + dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n", + type, hr_qp->qpn); + return; + } + ibqp->event_handler(&event, ibqp->qp_context); + } +} + +static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +{ + unsigned long num = 0; + int ret; + + if (hr_qp->ibqp.qp_type == IB_QPT_GSI) { + /* when hw version is v1, the sqpn is allocated */ + if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) + num = HNS_ROCE_MAX_PORTS + + hr_dev->iboe.phy_port[hr_qp->port]; + else + num = 1; + + hr_qp->doorbell_qpn = 1; + } else { + ret = hns_roce_bitmap_alloc_range(&hr_dev->qp_table.bitmap, + 1, 1, &num); + if (ret) { + ibdev_err(&hr_dev->ib_dev, "Failed to alloc bitmap\n"); + return -ENOMEM; + } + + hr_qp->doorbell_qpn = (u32)num; + } + + hr_qp->qpn = num; + + return 0; +} + +enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state) +{ + switch (state) { + case IB_QPS_RESET: + return HNS_ROCE_QP_STATE_RST; + case IB_QPS_INIT: + return HNS_ROCE_QP_STATE_INIT; + case IB_QPS_RTR: + return HNS_ROCE_QP_STATE_RTR; + case IB_QPS_RTS: + return HNS_ROCE_QP_STATE_RTS; + case IB_QPS_SQD: + return HNS_ROCE_QP_STATE_SQD; + case IB_QPS_ERR: + return HNS_ROCE_QP_STATE_ERR; + default: + return HNS_ROCE_QP_NUM_STATE; + } +} + +static void add_qp_to_list(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct ib_cq *send_cq, struct ib_cq *recv_cq) +{ + struct hns_roce_cq *hr_send_cq, *hr_recv_cq; + unsigned long flags; + + hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL; + hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL; + + spin_lock_irqsave(&hr_dev->qp_list_lock, flags); + hns_roce_lock_cqs(hr_send_cq, hr_recv_cq); + + list_add_tail(&hr_qp->node, &hr_dev->qp_list); + if (hr_send_cq) + list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list); + if (hr_recv_cq) + list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list); + + hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq); + spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); +} + +static int hns_roce_qp_store(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct ib_qp_init_attr *init_attr) +{ + struct xarray *xa = &hr_dev->qp_table_xa; + int ret; + + if (!hr_qp->qpn) + return -EINVAL; + + ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL)); + if (ret) + dev_err(hr_dev->dev, "Failed to xa store for QPC\n"); + else + /* add QP to device's QP list for softwc */ + add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq, + init_attr->recv_cq); + + return ret; +} + +static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +{ + struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; + struct device *dev = hr_dev->dev; + int ret; + + if (!hr_qp->qpn) + return -EINVAL; + + /* In v1 engine, GSI QP context is saved in the RoCE hw's register */ + if (hr_qp->ibqp.qp_type == IB_QPT_GSI && + hr_dev->hw_rev == HNS_ROCE_HW_VER1) + return 0; + + /* Alloc memory for QPC */ + ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); + if (ret) { + dev_err(dev, "Failed to get QPC table\n"); + goto err_out; + } + + /* Alloc memory for IRRL */ + ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); + if (ret) { + dev_err(dev, "Failed to get IRRL table\n"); + goto err_put_qp; + } + + if (hr_dev->caps.trrl_entry_sz) { + /* Alloc memory for TRRL */ + ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, + hr_qp->qpn); + if (ret) { + dev_err(dev, "Failed to get TRRL table\n"); + goto err_put_irrl; + } + } + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { + /* Alloc memory for SCC CTX */ + ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, + hr_qp->qpn); + if (ret) { + dev_err(dev, "Failed to get SCC CTX table\n"); + goto err_put_trrl; + } + } + + return 0; + +err_put_trrl: + if (hr_dev->caps.trrl_entry_sz) + hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); + +err_put_irrl: + hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); + +err_put_qp: + hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); + +err_out: + return ret; +} + +void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +{ + struct xarray *xa = &hr_dev->qp_table_xa; + unsigned long flags; + + list_del(&hr_qp->node); + list_del(&hr_qp->sq_node); + list_del(&hr_qp->rq_node); + + xa_lock_irqsave(xa, flags); + __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1)); + xa_unlock_irqrestore(xa, flags); +} + +static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +{ + struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; + + /* In v1 engine, GSI QP context is saved in the RoCE hw's register */ + if (hr_qp->ibqp.qp_type == IB_QPT_GSI && + hr_dev->hw_rev == HNS_ROCE_HW_VER1) + return; + + if (hr_dev->caps.trrl_entry_sz) + hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); + hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); +} + +static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +{ + struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; + + if (hr_qp->ibqp.qp_type == IB_QPT_GSI) + return; + + if (hr_qp->qpn < hr_dev->caps.reserved_qps) + return; + + hns_roce_bitmap_free_range(&qp_table->bitmap, hr_qp->qpn, 1, BITMAP_RR); +} + +static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, + struct hns_roce_qp *hr_qp, int has_rq) +{ + u32 cnt; + + /* If srq exist, set zero for relative number of rq */ + if (!has_rq) { + hr_qp->rq.wqe_cnt = 0; + hr_qp->rq.max_gs = 0; + hr_qp->rq_inl_buf.wqe_cnt = 0; + cap->max_recv_wr = 0; + cap->max_recv_sge = 0; + + return 0; + } + + /* Check the validity of QP support capacity */ + if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes || + cap->max_recv_sge > hr_dev->caps.max_rq_sg) { + ibdev_err(&hr_dev->ib_dev, "RQ config error, depth=%u, sge=%d\n", + cap->max_recv_wr, cap->max_recv_sge); + return -EINVAL; + } + + cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes)); + if (cnt > hr_dev->caps.max_wqes) { + ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n", + cap->max_recv_wr); + return -EINVAL; + } + + hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge)); + + hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz * + hr_qp->rq.max_gs); + + hr_qp->rq.wqe_cnt = cnt; + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) + hr_qp->rq_inl_buf.wqe_cnt = cnt; + else + hr_qp->rq_inl_buf.wqe_cnt = 0; + + cap->max_recv_wr = cnt; + cap->max_recv_sge = hr_qp->rq.max_gs; + + return 0; +} + +static int set_extend_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt, + struct hns_roce_qp *hr_qp, + struct ib_qp_cap *cap) +{ + u32 cnt; + + cnt = max(1U, cap->max_send_sge); + if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) { + hr_qp->sq.max_gs = roundup_pow_of_two(cnt); + hr_qp->sge.sge_cnt = 0; + + return 0; + } + + hr_qp->sq.max_gs = cnt; + + /* UD sqwqe's sge use extend sge */ + if (hr_qp->ibqp.qp_type == IB_QPT_GSI || + hr_qp->ibqp.qp_type == IB_QPT_UD) { + cnt = roundup_pow_of_two(sq_wqe_cnt * hr_qp->sq.max_gs); + } else if (hr_qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) { + cnt = roundup_pow_of_two(sq_wqe_cnt * + (hr_qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE)); + } else { + cnt = 0; + } + + hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT; + + /* If the number of extended sge is not zero, they MUST use the + * space of HNS_HW_PAGE_SIZE at least. + */ + hr_qp->sge.sge_cnt = cnt ? + max(cnt, (u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE) : 0; + + return 0; +} + +static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, + struct ib_qp_cap *cap, + struct hns_roce_ib_create_qp *ucmd) +{ + u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); + u8 max_sq_stride = ilog2(roundup_sq_stride); + + /* Sanity check SQ size before proceeding */ + if (ucmd->log_sq_stride > max_sq_stride || + ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) { + ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n"); + return -EINVAL; + } + + if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { + ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n", + cap->max_send_sge); + return -EINVAL; + } + + return 0; +} + +static int set_user_sq_size(struct hns_roce_dev *hr_dev, + struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp, + struct hns_roce_ib_create_qp *ucmd) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + u32 cnt = 0; + int ret; + + if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) || + cnt > hr_dev->caps.max_wqes) + return -EINVAL; + + ret = check_sq_size_with_integrity(hr_dev, cap, ucmd); + if (ret) { + ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n", + ret); + return ret; + } + + ret = set_extend_sge_param(hr_dev, cnt, hr_qp, cap); + if (ret) + return ret; + + hr_qp->sq.wqe_shift = ucmd->log_sq_stride; + hr_qp->sq.wqe_cnt = cnt; + + return 0; +} + +static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_roce_buf_attr *buf_attr) +{ + int buf_size; + int idx = 0; + + hr_qp->buff_size = 0; + + /* SQ WQE */ + hr_qp->sq.offset = 0; + buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt, + hr_qp->sq.wqe_shift); + if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { + buf_attr->region[idx].size = buf_size; + buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num; + idx++; + hr_qp->buff_size += buf_size; + } + + /* extend SGE WQE in SQ */ + hr_qp->sge.offset = hr_qp->buff_size; + buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt, + hr_qp->sge.sge_shift); + if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { + buf_attr->region[idx].size = buf_size; + buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num; + idx++; + hr_qp->buff_size += buf_size; + } + + /* RQ WQE */ + hr_qp->rq.offset = hr_qp->buff_size; + buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt, + hr_qp->rq.wqe_shift); + if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { + buf_attr->region[idx].size = buf_size; + buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num; + idx++; + hr_qp->buff_size += buf_size; + } + + if (hr_qp->buff_size < 1) + return -EINVAL; + + buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; + buf_attr->fixed_page = true; + buf_attr->region_count = idx; + + return 0; +} + +static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, + struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + u32 cnt; + int ret; + + if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes || + cap->max_send_sge > hr_dev->caps.max_sq_sg) { + ibdev_err(ibdev, + "failed to check SQ WR or SGE num, ret = %d.\n", + -EINVAL); + return -EINVAL; + } + + cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes)); + if (cnt > hr_dev->caps.max_wqes) { + ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n", + cnt); + return -EINVAL; + } + + hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz); + hr_qp->sq.wqe_cnt = cnt; + + ret = set_extend_sge_param(hr_dev, cnt, hr_qp, cap); + if (ret) + return ret; + + /* sync the parameters of kernel QP to user's configuration */ + cap->max_send_wr = cnt; + cap->max_send_sge = hr_qp->sq.max_gs; + + return 0; +} + +static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr) +{ + if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr) + return 0; + + return 1; +} + +static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) +{ + if (attr->qp_type == IB_QPT_XRC_INI || + attr->qp_type == IB_QPT_XRC_TGT || attr->srq || + !attr->cap.max_recv_wr) + return 0; + + return 1; +} + +static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp, + struct ib_qp_init_attr *init_attr) +{ + u32 max_recv_sge = init_attr->cap.max_recv_sge; + u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt; + struct hns_roce_rinl_wqe *wqe_list; + int i; + + /* allocate recv inline buf */ + wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe), + GFP_KERNEL); + + if (!wqe_list) + goto err; + + /* Allocate a continuous buffer for all inline sge we need */ + wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge * + sizeof(struct hns_roce_rinl_sge)), + GFP_KERNEL); + if (!wqe_list[0].sg_list) + goto err_wqe_list; + + /* Assign buffers of sg_list to each inline wqe */ + for (i = 1; i < wqe_cnt; i++) + wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge]; + + hr_qp->rq_inl_buf.wqe_list = wqe_list; + + return 0; + +err_wqe_list: + kfree(wqe_list); + +err: + return -ENOMEM; +} + +static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) +{ + if (hr_qp->rq_inl_buf.wqe_list) + kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); + kfree(hr_qp->rq_inl_buf.wqe_list); +} + +static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, unsigned long addr) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_buf_attr buf_attr = {}; + int ret; + + if (!udata && hr_qp->rq_inl_buf.wqe_cnt) { + ret = alloc_rq_inline_buf(hr_qp, init_attr); + if (ret) { + ibdev_err(ibdev, + "failed to alloc inline buf, ret = %d.\n", + ret); + return ret; + } + } else { + hr_qp->rq_inl_buf.wqe_list = NULL; + } + + ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr); + if (ret) { + ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret); + goto err_inline; + } + ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr, + HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz, + udata, addr); + if (ret) { + ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret); + goto err_inline; + } + + return 0; +err_inline: + free_rq_inline_buf(hr_qp); + + return ret; +} + +static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) +{ + hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr); + free_rq_inline_buf(hr_qp); +} + +static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, + struct hns_roce_ib_create_qp_resp *resp, + struct hns_roce_ib_create_qp *ucmd) +{ + return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && + udata->outlen >= offsetofend(typeof(*resp), cap_flags) && + hns_roce_qp_has_sq(init_attr) && + udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr)); +} + +static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, + struct hns_roce_ib_create_qp_resp *resp) +{ + return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && + udata->outlen >= offsetofend(typeof(*resp), cap_flags) && + hns_roce_qp_has_rq(init_attr)); +} + +static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev, + struct ib_qp_init_attr *init_attr) +{ + return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && + hns_roce_qp_has_rq(init_attr)); +} + +static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, + struct hns_roce_ib_create_qp *ucmd, + struct hns_roce_ib_create_qp_resp *resp) +{ + struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( + udata, struct hns_roce_ucontext, ibucontext); + struct ib_device *ibdev = &hr_dev->ib_dev; + int ret; + + if (udata) { + if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) { + ret = hns_roce_db_map_user(uctx, udata, ucmd->sdb_addr, + &hr_qp->sdb); + if (ret) { + ibdev_err(ibdev, + "failed to map user SQ doorbell, ret = %d.\n", + ret); + goto err_out; + } + hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB; + } + + if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) { + ret = hns_roce_db_map_user(uctx, udata, ucmd->db_addr, + &hr_qp->rdb); + if (ret) { + ibdev_err(ibdev, + "failed to map user RQ doorbell, ret = %d.\n", + ret); + goto err_sdb; + } + hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; + } + } else { + /* QP doorbell register address */ + hr_qp->sq.db_reg_l = hr_dev->reg_base + hr_dev->sdb_offset + + DB_REG_OFFSET * hr_dev->priv_uar.index; + hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev->odb_offset + + DB_REG_OFFSET * hr_dev->priv_uar.index; + + if (kernel_qp_has_rdb(hr_dev, init_attr)) { + ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); + if (ret) { + ibdev_err(ibdev, + "failed to alloc kernel RQ doorbell, ret = %d.\n", + ret); + goto err_out; + } + *hr_qp->rdb.db_record = 0; + hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; + } + } + + return 0; +err_sdb: + if (udata && hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) + hns_roce_db_unmap_user(uctx, &hr_qp->sdb); +err_out: + return ret; +} + +static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_udata *udata) +{ + struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( + udata, struct hns_roce_ucontext, ibucontext); + + if (udata) { + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) + hns_roce_db_unmap_user(uctx, &hr_qp->rdb); + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) + hns_roce_db_unmap_user(uctx, &hr_qp->sdb); + } else { + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) + hns_roce_free_db(hr_dev, &hr_qp->rdb); + } +} + +static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + u64 *sq_wrid = NULL; + u64 *rq_wrid = NULL; + int ret; + + sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL); + if (ZERO_OR_NULL_PTR(sq_wrid)) { + ibdev_err(ibdev, "failed to alloc SQ wrid.\n"); + return -ENOMEM; + } + + if (hr_qp->rq.wqe_cnt) { + rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL); + if (ZERO_OR_NULL_PTR(rq_wrid)) { + ibdev_err(ibdev, "failed to alloc RQ wrid.\n"); + ret = -ENOMEM; + goto err_sq; + } + } + + hr_qp->sq.wrid = sq_wrid; + hr_qp->rq.wrid = rq_wrid; + return 0; +err_sq: + kfree(sq_wrid); + + return ret; +} + +static void free_kernel_wrid(struct hns_roce_qp *hr_qp) +{ + kfree(hr_qp->rq.wrid); + kfree(hr_qp->sq.wrid); +} + +static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, + struct hns_roce_ib_create_qp *ucmd) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + int ret; + + hr_qp->ibqp.qp_type = init_attr->qp_type; + + if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline) + init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline; + + hr_qp->max_inline_data = init_attr->cap.max_inline_data; + + if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) + hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; + else + hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR; + + ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp, + hns_roce_qp_has_rq(init_attr)); + if (ret) { + ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n", + ret); + return ret; + } + + if (udata) { + ret = ib_copy_from_udata(ucmd, udata, + min(udata->inlen, sizeof(*ucmd))); + if (ret) { + ibdev_err(ibdev, + "failed to copy QP ucmd, ret = %d\n", ret); + return ret; + } + + ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd); + if (ret) + ibdev_err(ibdev, + "failed to set user SQ size, ret = %d.\n", + ret); + } else { + if (init_attr->create_flags & + IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) { + ibdev_err(ibdev, "Failed to check multicast loopback\n"); + return -EINVAL; + } + + if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) { + ibdev_err(ibdev, "Failed to check ipoib ud lso\n"); + return -EINVAL; + } + + ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp); + if (ret) + ibdev_err(ibdev, + "failed to set kernel SQ size, ret = %d.\n", + ret); + } + + return ret; +} + +static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, + struct ib_pd *ib_pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, + struct hns_roce_qp *hr_qp) +{ + struct hns_roce_ib_create_qp_resp resp = {}; + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_ib_create_qp ucmd = {}; + int ret; + + mutex_init(&hr_qp->mutex); + spin_lock_init(&hr_qp->sq.lock); + spin_lock_init(&hr_qp->rq.lock); + + hr_qp->state = IB_QPS_RESET; + hr_qp->flush_flag = 0; + + ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd); + if (ret) { + ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret); + return ret; + } + + if (!udata) { + ret = alloc_kernel_wrid(hr_dev, hr_qp); + if (ret) { + ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n", + ret); + return ret; + } + } + + ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp); + if (ret) { + ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n", + ret); + goto err_wrid; + } + + ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr); + if (ret) { + ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret); + goto err_db; + } + + ret = alloc_qpn(hr_dev, hr_qp); + if (ret) { + ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret); + goto err_buf; + } + + ret = alloc_qpc(hr_dev, hr_qp); + if (ret) { + ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n", + ret); + goto err_qpn; + } + + ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr); + if (ret) { + ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret); + goto err_qpc; + } + + if (udata) { + resp.cap_flags = hr_qp->en_flags; + ret = ib_copy_to_udata(udata, &resp, + min(udata->outlen, sizeof(resp))); + if (ret) { + ibdev_err(ibdev, "copy qp resp failed!\n"); + goto err_store; + } + } + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { + ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); + if (ret) + goto err_store; + } + + hr_qp->ibqp.qp_num = hr_qp->qpn; + hr_qp->event = hns_roce_ib_qp_event; + atomic_set(&hr_qp->refcount, 1); + init_completion(&hr_qp->free); + + return 0; + +err_store: + hns_roce_qp_remove(hr_dev, hr_qp); +err_qpc: + free_qpc(hr_dev, hr_qp); +err_qpn: + free_qpn(hr_dev, hr_qp); +err_buf: + free_qp_buf(hr_dev, hr_qp); +err_db: + free_qp_db(hr_dev, hr_qp, udata); +err_wrid: + free_kernel_wrid(hr_qp); + return ret; +} + +void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, + struct ib_udata *udata) +{ + if (atomic_dec_and_test(&hr_qp->refcount)) + complete(&hr_qp->free); + wait_for_completion(&hr_qp->free); + + free_qpc(hr_dev, hr_qp); + free_qpn(hr_dev, hr_qp); + free_qp_buf(hr_dev, hr_qp); + free_kernel_wrid(hr_qp); + free_qp_db(hr_dev, hr_qp, udata); + + kfree(hr_qp); +} + +struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_qp *hr_qp; + int ret; + + switch (init_attr->qp_type) { + case IB_QPT_RC: + case IB_QPT_GSI: + break; + default: + ibdev_err(ibdev, "not support QP type %d\n", + init_attr->qp_type); + return ERR_PTR(-EOPNOTSUPP); + } + + hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); + if (!hr_qp) + return ERR_PTR(-ENOMEM); + + if (init_attr->qp_type == IB_QPT_GSI) { + hr_qp->port = init_attr->port_num - 1; + hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; + } + + ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); + if (ret) { + ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n", + init_attr->qp_type, ret); + ibdev_err(ibdev, "Create GSI QP failed!\n"); + kfree(hr_qp); + return ERR_PTR(ret); + } + return &hr_qp->ibqp; +} + +int to_hr_qp_type(int qp_type) +{ + int transport_type; + + if (qp_type == IB_QPT_RC) + transport_type = SERV_TYPE_RC; + else if (qp_type == IB_QPT_UC) + transport_type = SERV_TYPE_UC; + else if (qp_type == IB_QPT_UD) + transport_type = SERV_TYPE_UD; + else if (qp_type == IB_QPT_GSI) + transport_type = SERV_TYPE_UD; + else + transport_type = -1; + + return transport_type; +} + +static int check_mtu_validate(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct ib_qp_attr *attr, int attr_mask) +{ + enum ib_mtu active_mtu; + int p; + + p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; + active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); + + if ((hr_dev->caps.max_mtu >= IB_MTU_2048 && + attr->path_mtu > hr_dev->caps.max_mtu) || + attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { + ibdev_err(&hr_dev->ib_dev, + "attr path_mtu(%d)invalid while modify qp", + attr->path_mtu); + return -EINVAL; + } + + return 0; +} + +static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + int p; + + if ((attr_mask & IB_QP_PORT) && + (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { + ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n", + attr->port_num); + return -EINVAL; + } + + if (attr_mask & IB_QP_PKEY_INDEX) { + p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; + if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { + ibdev_err(&hr_dev->ib_dev, + "invalid attr, pkey_index = %u.\n", + attr->pkey_index); + return -EINVAL; + } + } + + if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && + attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { + ibdev_err(&hr_dev->ib_dev, + "invalid attr, max_rd_atomic = %u.\n", + attr->max_rd_atomic); + return -EINVAL; + } + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && + attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { + ibdev_err(&hr_dev->ib_dev, + "invalid attr, max_dest_rd_atomic = %u.\n", + attr->max_dest_rd_atomic); + return -EINVAL; + } + + if (attr_mask & IB_QP_PATH_MTU) + return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask); + + return 0; +} + +int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); + enum ib_qp_state cur_state, new_state; + int ret = -EINVAL; + + mutex_lock(&hr_qp->mutex); + + if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state) + goto out; + + cur_state = hr_qp->state; + new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; + + if (ibqp->uobject && + (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) { + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) { + hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr); + + if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) + hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); + } else { + ibdev_warn(&hr_dev->ib_dev, + "flush cqe is not supported in userspace!\n"); + goto out; + } + } + + if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, + attr_mask)) { + ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); + goto out; + } + + ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask); + if (ret) + goto out; + + if (cur_state == new_state && cur_state == IB_QPS_RESET) { + if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) { + ret = -EPERM; + ibdev_err(&hr_dev->ib_dev, + "RST2RST state is not supported\n"); + } else { + ret = 0; + } + + goto out; + } + + ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, + new_state); + +out: + mutex_unlock(&hr_qp->mutex); + + return ret; +} + +void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) + __acquires(&send_cq->lock) __acquires(&recv_cq->lock) +{ + if (unlikely(send_cq == NULL && recv_cq == NULL)) { + __acquire(&send_cq->lock); + __acquire(&recv_cq->lock); + } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { + spin_lock_irq(&send_cq->lock); + __acquire(&recv_cq->lock); + } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { + spin_lock_irq(&recv_cq->lock); + __acquire(&send_cq->lock); + } else if (send_cq == recv_cq) { + spin_lock_irq(&send_cq->lock); + __acquire(&recv_cq->lock); + } else if (send_cq->cqn < recv_cq->cqn) { + spin_lock_irq(&send_cq->lock); + spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); + } else { + spin_lock_irq(&recv_cq->lock); + spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); + } +} + +void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, + struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) + __releases(&recv_cq->lock) +{ + if (unlikely(send_cq == NULL && recv_cq == NULL)) { + __release(&recv_cq->lock); + __release(&send_cq->lock); + } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { + __release(&recv_cq->lock); + spin_unlock(&send_cq->lock); + } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { + __release(&send_cq->lock); + spin_unlock(&recv_cq->lock); + } else if (send_cq == recv_cq) { + __release(&recv_cq->lock); + spin_unlock_irq(&send_cq->lock); + } else if (send_cq->cqn < recv_cq->cqn) { + spin_unlock(&recv_cq->lock); + spin_unlock_irq(&send_cq->lock); + } else { + spin_unlock(&send_cq->lock); + spin_unlock_irq(&recv_cq->lock); + } +} + +static inline void *get_wqe(struct hns_roce_qp *hr_qp, int offset) +{ + return hns_roce_buf_offset(hr_qp->mtr.kmem, offset); +} + +void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, int n) +{ + return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); +} + +void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, int n) +{ + return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); +} + +void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, int n) +{ + return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift)); +} + +bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, + struct ib_cq *ib_cq) +{ + struct hns_roce_cq *hr_cq; + u32 cur; + + cur = hr_wq->head - hr_wq->tail; + if (likely(cur + nreq < hr_wq->wqe_cnt)) + return false; + + hr_cq = to_hr_cq(ib_cq); + spin_lock(&hr_cq->lock); + cur = hr_wq->head - hr_wq->tail; + spin_unlock(&hr_cq->lock); + + return cur + nreq >= hr_wq->wqe_cnt; +} + +int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; + int reserved_from_top = 0; + int reserved_from_bot; + int ret; + + mutex_init(&qp_table->scc_mutex); + xa_init(&hr_dev->qp_table_xa); + + reserved_from_bot = hr_dev->caps.reserved_qps; + + ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps, + hr_dev->caps.num_qps - 1, reserved_from_bot, + reserved_from_top); + if (ret) { + dev_err(hr_dev->dev, "qp bitmap init failed!error=%d\n", + ret); + return ret; + } + + return 0; +} + +void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) +{ + hns_roce_bitmap_cleanup(&hr_dev->qp_table.bitmap); +} diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c new file mode 100644 index 000000000..259444c0a --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +// Copyright (c) 2019 Hisilicon Limited. + +#include <rdma/rdma_cm.h> +#include <rdma/restrack.h> +#include <uapi/rdma/rdma_netlink.h> +#include "hnae3.h" +#include "hns_roce_common.h" +#include "hns_roce_device.h" +#include "hns_roce_hw_v2.h" + +static int hns_roce_fill_cq(struct sk_buff *msg, + struct hns_roce_v2_cq_context *context) +{ + if (rdma_nl_put_driver_u32(msg, "state", + roce_get_field(context->byte_4_pg_ceqn, + V2_CQC_BYTE_4_ARM_ST_M, + V2_CQC_BYTE_4_ARM_ST_S))) + goto err; + + if (rdma_nl_put_driver_u32(msg, "ceqn", + roce_get_field(context->byte_4_pg_ceqn, + V2_CQC_BYTE_4_CEQN_M, + V2_CQC_BYTE_4_CEQN_S))) + goto err; + + if (rdma_nl_put_driver_u32(msg, "cqn", + roce_get_field(context->byte_8_cqn, + V2_CQC_BYTE_8_CQN_M, + V2_CQC_BYTE_8_CQN_S))) + goto err; + + if (rdma_nl_put_driver_u32(msg, "hopnum", + roce_get_field(context->byte_16_hop_addr, + V2_CQC_BYTE_16_CQE_HOP_NUM_M, + V2_CQC_BYTE_16_CQE_HOP_NUM_S))) + goto err; + + if (rdma_nl_put_driver_u32( + msg, "pi", + roce_get_field(context->byte_28_cq_pi, + V2_CQC_BYTE_28_CQ_PRODUCER_IDX_M, + V2_CQC_BYTE_28_CQ_PRODUCER_IDX_S))) + goto err; + + if (rdma_nl_put_driver_u32( + msg, "ci", + roce_get_field(context->byte_32_cq_ci, + V2_CQC_BYTE_32_CQ_CONSUMER_IDX_M, + V2_CQC_BYTE_32_CQ_CONSUMER_IDX_S))) + goto err; + + if (rdma_nl_put_driver_u32( + msg, "coalesce", + roce_get_field(context->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_MAX_CNT_M, + V2_CQC_BYTE_56_CQ_MAX_CNT_S))) + goto err; + + if (rdma_nl_put_driver_u32( + msg, "period", + roce_get_field(context->byte_56_cqe_period_maxcnt, + V2_CQC_BYTE_56_CQ_PERIOD_M, + V2_CQC_BYTE_56_CQ_PERIOD_S))) + goto err; + + if (rdma_nl_put_driver_u32(msg, "cnt", + roce_get_field(context->byte_52_cqe_cnt, + V2_CQC_BYTE_52_CQE_CNT_M, + V2_CQC_BYTE_52_CQE_CNT_S))) + goto err; + + return 0; + +err: + return -EMSGSIZE; +} + +int hns_roce_fill_res_cq_entry(struct sk_buff *msg, + struct ib_cq *ib_cq) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); + struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); + struct hns_roce_v2_cq_context *context; + struct nlattr *table_attr; + int ret; + + if (!hr_dev->dfx->query_cqc_info) + return -EINVAL; + + context = kzalloc(sizeof(struct hns_roce_v2_cq_context), GFP_KERNEL); + if (!context) + return -ENOMEM; + + ret = hr_dev->dfx->query_cqc_info(hr_dev, hr_cq->cqn, (int *)context); + if (ret) + goto err; + + table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER); + if (!table_attr) { + ret = -EMSGSIZE; + goto err; + } + + if (hns_roce_fill_cq(msg, context)) { + ret = -EMSGSIZE; + goto err_cancel_table; + } + + nla_nest_end(msg, table_attr); + kfree(context); + + return 0; + +err_cancel_table: + nla_nest_cancel(msg, table_attr); +err: + kfree(context); + return ret; +} diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c new file mode 100644 index 000000000..02e2416b5 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2018 Hisilicon Limited. + */ + +#include <rdma/ib_umem.h> +#include <rdma/hns-abi.h> +#include "hns_roce_device.h" +#include "hns_roce_cmd.h" +#include "hns_roce_hem.h" + +void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type) +{ + struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; + struct hns_roce_srq *srq; + + xa_lock(&srq_table->xa); + srq = xa_load(&srq_table->xa, srqn & (hr_dev->caps.num_srqs - 1)); + if (srq) + atomic_inc(&srq->refcount); + xa_unlock(&srq_table->xa); + + if (!srq) { + dev_warn(hr_dev->dev, "Async event for bogus SRQ %08x\n", srqn); + return; + } + + srq->event(srq, event_type); + + if (atomic_dec_and_test(&srq->refcount)) + complete(&srq->free); +} + +static void hns_roce_ib_srq_event(struct hns_roce_srq *srq, + enum hns_roce_event event_type) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device); + struct ib_srq *ibsrq = &srq->ibsrq; + struct ib_event event; + + if (ibsrq->event_handler) { + event.device = ibsrq->device; + event.element.srq = ibsrq; + switch (event_type) { + case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: + event.event = IB_EVENT_SRQ_LIMIT_REACHED; + break; + case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: + event.event = IB_EVENT_SRQ_ERR; + break; + default: + dev_err(hr_dev->dev, + "hns_roce:Unexpected event type 0x%x on SRQ %06lx\n", + event_type, srq->srqn); + return; + } + + ibsrq->event_handler(&event, ibsrq->srq_context); + } +} + +static int hns_roce_hw_create_srq(struct hns_roce_dev *dev, + struct hns_roce_cmd_mailbox *mailbox, + unsigned long srq_num) +{ + return hns_roce_cmd_mbox(dev, mailbox->dma, 0, srq_num, 0, + HNS_ROCE_CMD_CREATE_SRQ, + HNS_ROCE_CMD_TIMEOUT_MSECS); +} + +static int hns_roce_hw_destroy_srq(struct hns_roce_dev *dev, + struct hns_roce_cmd_mailbox *mailbox, + unsigned long srq_num) +{ + return hns_roce_cmd_mbox(dev, 0, mailbox ? mailbox->dma : 0, srq_num, + mailbox ? 0 : 1, HNS_ROCE_CMD_DESTROY_SRQ, + HNS_ROCE_CMD_TIMEOUT_MSECS); +} + +static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, + u32 pdn, u32 cqn, u16 xrcd, u64 db_rec_addr) +{ + struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_cmd_mailbox *mailbox; + u64 mtts_wqe[MTT_MIN_COUNT] = { 0 }; + u64 mtts_idx[MTT_MIN_COUNT] = { 0 }; + dma_addr_t dma_handle_wqe = 0; + dma_addr_t dma_handle_idx = 0; + int ret; + + /* Get the physical address of srq buf */ + ret = hns_roce_mtr_find(hr_dev, &srq->buf_mtr, 0, mtts_wqe, + ARRAY_SIZE(mtts_wqe), &dma_handle_wqe); + if (ret < 1) { + ibdev_err(ibdev, "failed to find mtr for SRQ WQE, ret = %d.\n", + ret); + return -ENOBUFS; + } + + /* Get physical address of idx que buf */ + ret = hns_roce_mtr_find(hr_dev, &srq->idx_que.mtr, 0, mtts_idx, + ARRAY_SIZE(mtts_idx), &dma_handle_idx); + if (ret < 1) { + ibdev_err(ibdev, "failed to find mtr for SRQ idx, ret = %d.\n", + ret); + return -ENOBUFS; + } + + ret = hns_roce_bitmap_alloc(&srq_table->bitmap, &srq->srqn); + if (ret) { + ibdev_err(ibdev, + "failed to alloc SRQ number, ret = %d.\n", ret); + return -ENOMEM; + } + + ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn); + if (ret) { + ibdev_err(ibdev, "failed to get SRQC table, ret = %d.\n", ret); + goto err_out; + } + + ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL)); + if (ret) { + ibdev_err(ibdev, "failed to store SRQC, ret = %d.\n", ret); + goto err_put; + } + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR_OR_NULL(mailbox)) { + ret = -ENOMEM; + ibdev_err(ibdev, "failed to alloc mailbox for SRQC.\n"); + goto err_xa; + } + + hr_dev->hw->write_srqc(hr_dev, srq, pdn, xrcd, cqn, mailbox->buf, + mtts_wqe, mtts_idx, dma_handle_wqe, + dma_handle_idx); + + ret = hns_roce_hw_create_srq(hr_dev, mailbox, srq->srqn); + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + if (ret) { + ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret); + goto err_xa; + } + + atomic_set(&srq->refcount, 1); + init_completion(&srq->free); + return ret; + +err_xa: + xa_erase(&srq_table->xa, srq->srqn); + +err_put: + hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); + +err_out: + hns_roce_bitmap_free(&srq_table->bitmap, srq->srqn, BITMAP_NO_RR); + return ret; +} + +static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) +{ + struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; + int ret; + + ret = hns_roce_hw_destroy_srq(hr_dev, NULL, srq->srqn); + if (ret) + dev_err(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n", + ret, srq->srqn); + + xa_erase(&srq_table->xa, srq->srqn); + + if (atomic_dec_and_test(&srq->refcount)) + complete(&srq->free); + wait_for_completion(&srq->free); + + hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); + hns_roce_bitmap_free(&srq_table->bitmap, srq->srqn, BITMAP_NO_RR); +} + +static int alloc_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, + struct ib_udata *udata, unsigned long addr) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_buf_attr buf_attr = {}; + int err; + + srq->wqe_shift = ilog2(roundup_pow_of_two(max(HNS_ROCE_SGE_SIZE, + HNS_ROCE_SGE_SIZE * + srq->max_gs))); + + buf_attr.page_shift = hr_dev->caps.srqwqe_buf_pg_sz + HNS_HW_PAGE_SHIFT; + buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt, + srq->wqe_shift); + buf_attr.region[0].hopnum = hr_dev->caps.srqwqe_hop_num; + buf_attr.region_count = 1; + buf_attr.fixed_page = true; + + err = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr, + hr_dev->caps.srqwqe_ba_pg_sz + + HNS_HW_PAGE_SHIFT, udata, addr); + if (err) + ibdev_err(ibdev, + "failed to alloc SRQ buf mtr, ret = %d.\n", err); + + return err; +} + +static void free_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) +{ + hns_roce_mtr_destroy(hr_dev, &srq->buf_mtr); +} + +static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, + struct ib_udata *udata, unsigned long addr) +{ + struct hns_roce_idx_que *idx_que = &srq->idx_que; + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_buf_attr buf_attr = {}; + int err; + + srq->idx_que.entry_shift = ilog2(HNS_ROCE_IDX_QUE_ENTRY_SZ); + + buf_attr.page_shift = hr_dev->caps.idx_buf_pg_sz + HNS_HW_PAGE_SHIFT; + buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt, + srq->idx_que.entry_shift); + buf_attr.region[0].hopnum = hr_dev->caps.idx_hop_num; + buf_attr.region_count = 1; + buf_attr.fixed_page = true; + + err = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr, + hr_dev->caps.idx_ba_pg_sz + HNS_HW_PAGE_SHIFT, + udata, addr); + if (err) { + ibdev_err(ibdev, + "failed to alloc SRQ idx mtr, ret = %d.\n", err); + return err; + } + + if (!udata) { + idx_que->bitmap = bitmap_zalloc(srq->wqe_cnt, GFP_KERNEL); + if (!idx_que->bitmap) { + ibdev_err(ibdev, "failed to alloc SRQ idx bitmap.\n"); + err = -ENOMEM; + goto err_idx_mtr; + } + } + + return 0; +err_idx_mtr: + hns_roce_mtr_destroy(hr_dev, &idx_que->mtr); + + return err; +} + +static void free_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) +{ + struct hns_roce_idx_que *idx_que = &srq->idx_que; + + bitmap_free(idx_que->bitmap); + idx_que->bitmap = NULL; + hns_roce_mtr_destroy(hr_dev, &idx_que->mtr); +} + +static int alloc_srq_wrid(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) +{ + srq->head = 0; + srq->tail = srq->wqe_cnt - 1; + srq->wrid = kvmalloc_array(srq->wqe_cnt, sizeof(u64), GFP_KERNEL); + if (!srq->wrid) + return -ENOMEM; + + return 0; +} + +static void free_srq_wrid(struct hns_roce_srq *srq) +{ + kvfree(srq->wrid); + srq->wrid = NULL; +} + +int hns_roce_create_srq(struct ib_srq *ib_srq, + struct ib_srq_init_attr *init_attr, + struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq->device); + struct hns_roce_ib_create_srq_resp resp = {}; + struct hns_roce_srq *srq = to_hr_srq(ib_srq); + struct ib_device *ibdev = &hr_dev->ib_dev; + struct hns_roce_ib_create_srq ucmd = {}; + int ret; + u32 cqn; + + /* Check the actual SRQ wqe and SRQ sge num */ + if (init_attr->attr.max_wr >= hr_dev->caps.max_srq_wrs || + init_attr->attr.max_sge > hr_dev->caps.max_srq_sges) + return -EINVAL; + + mutex_init(&srq->mutex); + spin_lock_init(&srq->lock); + + srq->wqe_cnt = roundup_pow_of_two(init_attr->attr.max_wr + 1); + srq->max_gs = init_attr->attr.max_sge; + + if (udata) { + ret = ib_copy_from_udata(&ucmd, udata, + min(udata->inlen, sizeof(ucmd))); + if (ret) { + ibdev_err(ibdev, "failed to copy SRQ udata, ret = %d.\n", + ret); + return ret; + } + } + + ret = alloc_srq_buf(hr_dev, srq, udata, ucmd.buf_addr); + if (ret) { + ibdev_err(ibdev, + "failed to alloc SRQ buffer, ret = %d.\n", ret); + return ret; + } + + ret = alloc_srq_idx(hr_dev, srq, udata, ucmd.que_addr); + if (ret) { + ibdev_err(ibdev, "failed to alloc SRQ idx, ret = %d.\n", ret); + goto err_buf_alloc; + } + + if (!udata) { + ret = alloc_srq_wrid(hr_dev, srq); + if (ret) { + ibdev_err(ibdev, "failed to alloc SRQ wrid, ret = %d.\n", + ret); + goto err_idx_alloc; + } + } + + cqn = ib_srq_has_cq(init_attr->srq_type) ? + to_hr_cq(init_attr->ext.cq)->cqn : 0; + srq->db_reg_l = hr_dev->reg_base + SRQ_DB_REG; + + ret = alloc_srqc(hr_dev, srq, to_hr_pd(ib_srq->pd)->pdn, cqn, 0, 0); + if (ret) { + ibdev_err(ibdev, + "failed to alloc SRQ context, ret = %d.\n", ret); + goto err_wrid_alloc; + } + + srq->event = hns_roce_ib_srq_event; + resp.srqn = srq->srqn; + + if (udata) { + ret = ib_copy_to_udata(udata, &resp, + min(udata->outlen, sizeof(resp))); + if (ret) + goto err_srqc_alloc; + } + + return 0; + +err_srqc_alloc: + free_srqc(hr_dev, srq); +err_wrid_alloc: + free_srq_wrid(srq); +err_idx_alloc: + free_srq_idx(hr_dev, srq); +err_buf_alloc: + free_srq_buf(hr_dev, srq); + return ret; +} + +int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); + struct hns_roce_srq *srq = to_hr_srq(ibsrq); + + free_srqc(hr_dev, srq); + free_srq_idx(hr_dev, srq); + free_srq_wrid(srq); + free_srq_buf(hr_dev, srq); + return 0; +} + +int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; + + xa_init(&srq_table->xa); + + return hns_roce_bitmap_init(&srq_table->bitmap, hr_dev->caps.num_srqs, + hr_dev->caps.num_srqs - 1, + hr_dev->caps.reserved_srqs, 0); +} + +void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev) +{ + hns_roce_bitmap_cleanup(&hr_dev->srq_table.bitmap); +} |