summaryrefslogtreecommitdiffstats
path: root/drivers/vdpa/pds
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/vdpa/pds')
-rw-r--r--drivers/vdpa/pds/Makefile9
-rw-r--r--drivers/vdpa/pds/aux_drv.c140
-rw-r--r--drivers/vdpa/pds/aux_drv.h26
-rw-r--r--drivers/vdpa/pds/cmds.c185
-rw-r--r--drivers/vdpa/pds/cmds.h18
-rw-r--r--drivers/vdpa/pds/debugfs.c286
-rw-r--r--drivers/vdpa/pds/debugfs.h17
-rw-r--r--drivers/vdpa/pds/vdpa_dev.c844
-rw-r--r--drivers/vdpa/pds/vdpa_dev.h50
9 files changed, 1575 insertions, 0 deletions
diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile
new file mode 100644
index 0000000000..c2d314d461
--- /dev/null
+++ b/drivers/vdpa/pds/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright(c) 2023 Advanced Micro Devices, Inc
+
+obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o
+
+pds_vdpa-y := aux_drv.o \
+ cmds.o \
+ debugfs.o \
+ vdpa_dev.o
diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c
new file mode 100644
index 0000000000..186e9ee22e
--- /dev/null
+++ b/drivers/vdpa/pds/aux_drv.c
@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/auxiliary_bus.h>
+#include <linux/pci.h>
+#include <linux/vdpa.h>
+#include <linux/virtio_pci_modern.h>
+
+#include <linux/pds/pds_common.h>
+#include <linux/pds/pds_core_if.h>
+#include <linux/pds/pds_adminq.h>
+#include <linux/pds/pds_auxbus.h>
+
+#include "aux_drv.h"
+#include "debugfs.h"
+#include "vdpa_dev.h"
+
+static const struct auxiliary_device_id pds_vdpa_id_table[] = {
+ { .name = PDS_VDPA_DEV_NAME, },
+ {},
+};
+
+static int pds_vdpa_device_id_check(struct pci_dev *pdev)
+{
+ if (pdev->device != PCI_DEVICE_ID_PENSANDO_VDPA_VF ||
+ pdev->vendor != PCI_VENDOR_ID_PENSANDO)
+ return -ENODEV;
+
+ return PCI_DEVICE_ID_PENSANDO_VDPA_VF;
+}
+
+static int pds_vdpa_probe(struct auxiliary_device *aux_dev,
+ const struct auxiliary_device_id *id)
+
+{
+ struct pds_auxiliary_dev *padev =
+ container_of(aux_dev, struct pds_auxiliary_dev, aux_dev);
+ struct device *dev = &aux_dev->dev;
+ struct pds_vdpa_aux *vdpa_aux;
+ int err;
+
+ vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL);
+ if (!vdpa_aux)
+ return -ENOMEM;
+
+ vdpa_aux->padev = padev;
+ vdpa_aux->vf_id = pci_iov_vf_id(padev->vf_pdev);
+ auxiliary_set_drvdata(aux_dev, vdpa_aux);
+
+ /* Get device ident info and set up the vdpa_mgmt_dev */
+ err = pds_vdpa_get_mgmt_info(vdpa_aux);
+ if (err)
+ goto err_free_mem;
+
+ /* Find the virtio configuration */
+ vdpa_aux->vd_mdev.pci_dev = padev->vf_pdev;
+ vdpa_aux->vd_mdev.device_id_check = pds_vdpa_device_id_check;
+ vdpa_aux->vd_mdev.dma_mask = DMA_BIT_MASK(PDS_CORE_ADDR_LEN);
+ err = vp_modern_probe(&vdpa_aux->vd_mdev);
+ if (err) {
+ dev_err(dev, "Unable to probe for virtio configuration: %pe\n",
+ ERR_PTR(err));
+ goto err_free_mgmt_info;
+ }
+
+ /* Let vdpa know that we can provide devices */
+ err = vdpa_mgmtdev_register(&vdpa_aux->vdpa_mdev);
+ if (err) {
+ dev_err(dev, "%s: Failed to initialize vdpa_mgmt interface: %pe\n",
+ __func__, ERR_PTR(err));
+ goto err_free_virtio;
+ }
+
+ pds_vdpa_debugfs_add_pcidev(vdpa_aux);
+ pds_vdpa_debugfs_add_ident(vdpa_aux);
+
+ return 0;
+
+err_free_virtio:
+ vp_modern_remove(&vdpa_aux->vd_mdev);
+err_free_mgmt_info:
+ pci_free_irq_vectors(padev->vf_pdev);
+err_free_mem:
+ kfree(vdpa_aux);
+ auxiliary_set_drvdata(aux_dev, NULL);
+
+ return err;
+}
+
+static void pds_vdpa_remove(struct auxiliary_device *aux_dev)
+{
+ struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev);
+ struct device *dev = &aux_dev->dev;
+
+ vdpa_mgmtdev_unregister(&vdpa_aux->vdpa_mdev);
+ vp_modern_remove(&vdpa_aux->vd_mdev);
+ pci_free_irq_vectors(vdpa_aux->padev->vf_pdev);
+
+ pds_vdpa_debugfs_del_vdpadev(vdpa_aux);
+ kfree(vdpa_aux);
+ auxiliary_set_drvdata(aux_dev, NULL);
+
+ dev_info(dev, "Removed\n");
+}
+
+static struct auxiliary_driver pds_vdpa_driver = {
+ .name = PDS_DEV_TYPE_VDPA_STR,
+ .probe = pds_vdpa_probe,
+ .remove = pds_vdpa_remove,
+ .id_table = pds_vdpa_id_table,
+};
+
+static void __exit pds_vdpa_cleanup(void)
+{
+ auxiliary_driver_unregister(&pds_vdpa_driver);
+
+ pds_vdpa_debugfs_destroy();
+}
+module_exit(pds_vdpa_cleanup);
+
+static int __init pds_vdpa_init(void)
+{
+ int err;
+
+ pds_vdpa_debugfs_create();
+
+ err = auxiliary_driver_register(&pds_vdpa_driver);
+ if (err) {
+ pr_err("%s: aux driver register failed: %pe\n",
+ PDS_VDPA_DRV_NAME, ERR_PTR(err));
+ pds_vdpa_debugfs_destroy();
+ }
+
+ return err;
+}
+module_init(pds_vdpa_init);
+
+MODULE_DESCRIPTION(PDS_VDPA_DRV_DESCRIPTION);
+MODULE_AUTHOR("Advanced Micro Devices, Inc");
+MODULE_LICENSE("GPL");
diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h
new file mode 100644
index 0000000000..26b7534415
--- /dev/null
+++ b/drivers/vdpa/pds/aux_drv.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _AUX_DRV_H_
+#define _AUX_DRV_H_
+
+#include <linux/virtio_pci_modern.h>
+
+#define PDS_VDPA_DRV_DESCRIPTION "AMD/Pensando vDPA VF Device Driver"
+#define PDS_VDPA_DRV_NAME KBUILD_MODNAME
+
+struct pds_vdpa_aux {
+ struct pds_auxiliary_dev *padev;
+
+ struct vdpa_mgmt_dev vdpa_mdev;
+ struct pds_vdpa_device *pdsv;
+
+ struct pds_vdpa_ident ident;
+
+ int vf_id;
+ struct dentry *dentry;
+ struct virtio_pci_modern_device vd_mdev;
+
+ int nintrs;
+};
+#endif /* _AUX_DRV_H_ */
diff --git a/drivers/vdpa/pds/cmds.c b/drivers/vdpa/pds/cmds.c
new file mode 100644
index 0000000000..80863a41c3
--- /dev/null
+++ b/drivers/vdpa/pds/cmds.c
@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/vdpa.h>
+#include <linux/virtio_pci_modern.h>
+
+#include <linux/pds/pds_common.h>
+#include <linux/pds/pds_core_if.h>
+#include <linux/pds/pds_adminq.h>
+#include <linux/pds/pds_auxbus.h>
+
+#include "vdpa_dev.h"
+#include "aux_drv.h"
+#include "cmds.h"
+
+int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = &padev->aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_init.opcode = PDS_VDPA_CMD_INIT,
+ .vdpa_init.vdpa_index = pdsv->vdpa_index,
+ .vdpa_init.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ /* Initialize the vdpa/virtio device */
+ err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_init),
+ &comp, 0);
+ if (err)
+ dev_dbg(dev, "Failed to init hw, status %d: %pe\n",
+ comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = &padev->aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa.opcode = PDS_VDPA_CMD_RESET,
+ .vdpa.vdpa_index = pdsv->vdpa_index,
+ .vdpa.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa), &comp, 0);
+ if (err)
+ dev_dbg(dev, "Failed to reset hw, status %d: %pe\n",
+ comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_status(struct pds_vdpa_device *pdsv, u8 status)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = &padev->aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_status.opcode = PDS_VDPA_CMD_STATUS_UPDATE,
+ .vdpa_status.vdpa_index = pdsv->vdpa_index,
+ .vdpa_status.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_status.status = status,
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_status), &comp, 0);
+ if (err)
+ dev_dbg(dev, "Failed to set status to %#x, error status %d: %pe\n",
+ status, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = &padev->aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR,
+ .vdpa_setattr.vdpa_index = pdsv->vdpa_index,
+ .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_setattr.attr = PDS_VDPA_ATTR_MAC,
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ ether_addr_copy(cmd.vdpa_setattr.mac, mac);
+ err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_setattr),
+ &comp, 0);
+ if (err)
+ dev_dbg(dev, "Failed to set mac address %pM, status %d: %pe\n",
+ mac, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = &padev->aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR,
+ .vdpa_setattr.vdpa_index = pdsv->vdpa_index,
+ .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_setattr.attr = PDS_VDPA_ATTR_MAX_VQ_PAIRS,
+ .vdpa_setattr.max_vq_pairs = cpu_to_le16(max_vqp),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_setattr),
+ &comp, 0);
+ if (err)
+ dev_dbg(dev, "Failed to set max vq pairs %u, status %d: %pe\n",
+ max_vqp, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_init_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx,
+ struct pds_vdpa_vq_info *vq_info)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = &padev->aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_vq_init.opcode = PDS_VDPA_CMD_VQ_INIT,
+ .vdpa_vq_init.vdpa_index = pdsv->vdpa_index,
+ .vdpa_vq_init.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_vq_init.qid = cpu_to_le16(qid),
+ .vdpa_vq_init.len = cpu_to_le16(ilog2(vq_info->q_len)),
+ .vdpa_vq_init.desc_addr = cpu_to_le64(vq_info->desc_addr),
+ .vdpa_vq_init.avail_addr = cpu_to_le64(vq_info->avail_addr),
+ .vdpa_vq_init.used_addr = cpu_to_le64(vq_info->used_addr),
+ .vdpa_vq_init.intr_index = cpu_to_le16(qid),
+ .vdpa_vq_init.avail_index = cpu_to_le16(vq_info->avail_idx ^ invert_idx),
+ .vdpa_vq_init.used_index = cpu_to_le16(vq_info->used_idx ^ invert_idx),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ dev_dbg(dev, "%s: qid %d len %d desc_addr %#llx avail_addr %#llx used_addr %#llx\n",
+ __func__, qid, ilog2(vq_info->q_len),
+ vq_info->desc_addr, vq_info->avail_addr, vq_info->used_addr);
+
+ err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_vq_init),
+ &comp, 0);
+ if (err)
+ dev_dbg(dev, "Failed to init vq %d, status %d: %pe\n",
+ qid, comp.status, ERR_PTR(err));
+
+ return err;
+}
+
+int pds_vdpa_cmd_reset_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx,
+ struct pds_vdpa_vq_info *vq_info)
+{
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = &padev->aux_dev.dev;
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_vq_reset.opcode = PDS_VDPA_CMD_VQ_RESET,
+ .vdpa_vq_reset.vdpa_index = pdsv->vdpa_index,
+ .vdpa_vq_reset.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id),
+ .vdpa_vq_reset.qid = cpu_to_le16(qid),
+ };
+ union pds_core_adminq_comp comp = {};
+ int err;
+
+ err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_vq_reset),
+ &comp, 0);
+ if (err) {
+ dev_dbg(dev, "Failed to reset vq %d, status %d: %pe\n",
+ qid, comp.status, ERR_PTR(err));
+ return err;
+ }
+
+ vq_info->avail_idx = le16_to_cpu(comp.vdpa_vq_reset.avail_index) ^ invert_idx;
+ vq_info->used_idx = le16_to_cpu(comp.vdpa_vq_reset.used_index) ^ invert_idx;
+
+ return 0;
+}
diff --git a/drivers/vdpa/pds/cmds.h b/drivers/vdpa/pds/cmds.h
new file mode 100644
index 0000000000..e24d85cb8f
--- /dev/null
+++ b/drivers/vdpa/pds/cmds.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _VDPA_CMDS_H_
+#define _VDPA_CMDS_H_
+
+int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv);
+
+int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv);
+int pds_vdpa_cmd_set_status(struct pds_vdpa_device *pdsv, u8 status);
+int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac);
+int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp);
+int pds_vdpa_cmd_init_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx,
+ struct pds_vdpa_vq_info *vq_info);
+int pds_vdpa_cmd_reset_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx,
+ struct pds_vdpa_vq_info *vq_info);
+int pds_vdpa_cmd_set_features(struct pds_vdpa_device *pdsv, u64 features);
+#endif /* _VDPA_CMDS_H_ */
diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c
new file mode 100644
index 0000000000..c328e694f6
--- /dev/null
+++ b/drivers/vdpa/pds/debugfs.c
@@ -0,0 +1,286 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/pci.h>
+#include <linux/vdpa.h>
+
+#include <linux/pds/pds_common.h>
+#include <linux/pds/pds_core_if.h>
+#include <linux/pds/pds_adminq.h>
+#include <linux/pds/pds_auxbus.h>
+
+#include "aux_drv.h"
+#include "vdpa_dev.h"
+#include "debugfs.h"
+
+static struct dentry *dbfs_dir;
+
+void pds_vdpa_debugfs_create(void)
+{
+ dbfs_dir = debugfs_create_dir(PDS_VDPA_DRV_NAME, NULL);
+}
+
+void pds_vdpa_debugfs_destroy(void)
+{
+ debugfs_remove_recursive(dbfs_dir);
+ dbfs_dir = NULL;
+}
+
+#define PRINT_SBIT_NAME(__seq, __f, __name) \
+ do { \
+ if ((__f) & (__name)) \
+ seq_printf(__seq, " %s", &#__name[16]); \
+ } while (0)
+
+static void print_status_bits(struct seq_file *seq, u8 status)
+{
+ seq_puts(seq, "status:");
+ PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_ACKNOWLEDGE);
+ PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_DRIVER);
+ PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_DRIVER_OK);
+ PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_FEATURES_OK);
+ PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_NEEDS_RESET);
+ PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_FAILED);
+ seq_puts(seq, "\n");
+}
+
+static void print_feature_bits_all(struct seq_file *seq, u64 features)
+{
+ int i;
+
+ seq_puts(seq, "features:");
+
+ for (i = 0; i < (sizeof(u64) * 8); i++) {
+ u64 mask = BIT_ULL(i);
+
+ switch (features & mask) {
+ case BIT_ULL(VIRTIO_NET_F_CSUM):
+ seq_puts(seq, " VIRTIO_NET_F_CSUM");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_GUEST_CSUM):
+ seq_puts(seq, " VIRTIO_NET_F_GUEST_CSUM");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS):
+ seq_puts(seq, " VIRTIO_NET_F_CTRL_GUEST_OFFLOADS");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_MTU):
+ seq_puts(seq, " VIRTIO_NET_F_MTU");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_MAC):
+ seq_puts(seq, " VIRTIO_NET_F_MAC");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_GUEST_TSO4):
+ seq_puts(seq, " VIRTIO_NET_F_GUEST_TSO4");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_GUEST_TSO6):
+ seq_puts(seq, " VIRTIO_NET_F_GUEST_TSO6");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_GUEST_ECN):
+ seq_puts(seq, " VIRTIO_NET_F_GUEST_ECN");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_GUEST_UFO):
+ seq_puts(seq, " VIRTIO_NET_F_GUEST_UFO");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_HOST_TSO4):
+ seq_puts(seq, " VIRTIO_NET_F_HOST_TSO4");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_HOST_TSO6):
+ seq_puts(seq, " VIRTIO_NET_F_HOST_TSO6");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_HOST_ECN):
+ seq_puts(seq, " VIRTIO_NET_F_HOST_ECN");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_HOST_UFO):
+ seq_puts(seq, " VIRTIO_NET_F_HOST_UFO");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_MRG_RXBUF):
+ seq_puts(seq, " VIRTIO_NET_F_MRG_RXBUF");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_STATUS):
+ seq_puts(seq, " VIRTIO_NET_F_STATUS");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_CTRL_VQ):
+ seq_puts(seq, " VIRTIO_NET_F_CTRL_VQ");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_CTRL_RX):
+ seq_puts(seq, " VIRTIO_NET_F_CTRL_RX");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_CTRL_VLAN):
+ seq_puts(seq, " VIRTIO_NET_F_CTRL_VLAN");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA):
+ seq_puts(seq, " VIRTIO_NET_F_CTRL_RX_EXTRA");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE):
+ seq_puts(seq, " VIRTIO_NET_F_GUEST_ANNOUNCE");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_MQ):
+ seq_puts(seq, " VIRTIO_NET_F_MQ");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR):
+ seq_puts(seq, " VIRTIO_NET_F_CTRL_MAC_ADDR");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_HASH_REPORT):
+ seq_puts(seq, " VIRTIO_NET_F_HASH_REPORT");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_RSS):
+ seq_puts(seq, " VIRTIO_NET_F_RSS");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_RSC_EXT):
+ seq_puts(seq, " VIRTIO_NET_F_RSC_EXT");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_STANDBY):
+ seq_puts(seq, " VIRTIO_NET_F_STANDBY");
+ break;
+ case BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX):
+ seq_puts(seq, " VIRTIO_NET_F_SPEED_DUPLEX");
+ break;
+ case BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY):
+ seq_puts(seq, " VIRTIO_F_NOTIFY_ON_EMPTY");
+ break;
+ case BIT_ULL(VIRTIO_F_ANY_LAYOUT):
+ seq_puts(seq, " VIRTIO_F_ANY_LAYOUT");
+ break;
+ case BIT_ULL(VIRTIO_F_VERSION_1):
+ seq_puts(seq, " VIRTIO_F_VERSION_1");
+ break;
+ case BIT_ULL(VIRTIO_F_ACCESS_PLATFORM):
+ seq_puts(seq, " VIRTIO_F_ACCESS_PLATFORM");
+ break;
+ case BIT_ULL(VIRTIO_F_RING_PACKED):
+ seq_puts(seq, " VIRTIO_F_RING_PACKED");
+ break;
+ case BIT_ULL(VIRTIO_F_ORDER_PLATFORM):
+ seq_puts(seq, " VIRTIO_F_ORDER_PLATFORM");
+ break;
+ case BIT_ULL(VIRTIO_F_SR_IOV):
+ seq_puts(seq, " VIRTIO_F_SR_IOV");
+ break;
+ case 0:
+ break;
+ default:
+ seq_printf(seq, " bit_%d", i);
+ break;
+ }
+ }
+
+ seq_puts(seq, "\n");
+}
+
+void pds_vdpa_debugfs_add_pcidev(struct pds_vdpa_aux *vdpa_aux)
+{
+ vdpa_aux->dentry = debugfs_create_dir(pci_name(vdpa_aux->padev->vf_pdev), dbfs_dir);
+}
+
+static int identity_show(struct seq_file *seq, void *v)
+{
+ struct pds_vdpa_aux *vdpa_aux = seq->private;
+ struct vdpa_mgmt_dev *mgmt;
+ u64 hw_features;
+
+ seq_printf(seq, "aux_dev: %s\n",
+ dev_name(&vdpa_aux->padev->aux_dev.dev));
+
+ mgmt = &vdpa_aux->vdpa_mdev;
+ seq_printf(seq, "max_vqs: %d\n", mgmt->max_supported_vqs);
+ seq_printf(seq, "config_attr_mask: %#llx\n", mgmt->config_attr_mask);
+ hw_features = le64_to_cpu(vdpa_aux->ident.hw_features);
+ seq_printf(seq, "hw_features: %#llx\n", hw_features);
+ print_feature_bits_all(seq, hw_features);
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(identity);
+
+void pds_vdpa_debugfs_add_ident(struct pds_vdpa_aux *vdpa_aux)
+{
+ debugfs_create_file("identity", 0400, vdpa_aux->dentry,
+ vdpa_aux, &identity_fops);
+}
+
+static int config_show(struct seq_file *seq, void *v)
+{
+ struct pds_vdpa_device *pdsv = seq->private;
+ struct virtio_net_config vc;
+ u8 status;
+
+ memcpy_fromio(&vc, pdsv->vdpa_aux->vd_mdev.device,
+ sizeof(struct virtio_net_config));
+
+ seq_printf(seq, "mac: %pM\n", vc.mac);
+ seq_printf(seq, "max_virtqueue_pairs: %d\n",
+ __virtio16_to_cpu(true, vc.max_virtqueue_pairs));
+ seq_printf(seq, "mtu: %d\n", __virtio16_to_cpu(true, vc.mtu));
+ seq_printf(seq, "speed: %d\n", le32_to_cpu(vc.speed));
+ seq_printf(seq, "duplex: %d\n", vc.duplex);
+ seq_printf(seq, "rss_max_key_size: %d\n", vc.rss_max_key_size);
+ seq_printf(seq, "rss_max_indirection_table_length: %d\n",
+ le16_to_cpu(vc.rss_max_indirection_table_length));
+ seq_printf(seq, "supported_hash_types: %#x\n",
+ le32_to_cpu(vc.supported_hash_types));
+ seq_printf(seq, "vn_status: %#x\n",
+ __virtio16_to_cpu(true, vc.status));
+
+ status = vp_modern_get_status(&pdsv->vdpa_aux->vd_mdev);
+ seq_printf(seq, "dev_status: %#x\n", status);
+ print_status_bits(seq, status);
+ seq_printf(seq, "negotiated_features: %#llx\n", pdsv->negotiated_features);
+ print_feature_bits_all(seq, pdsv->negotiated_features);
+ seq_printf(seq, "vdpa_index: %d\n", pdsv->vdpa_index);
+ seq_printf(seq, "num_vqs: %d\n", pdsv->num_vqs);
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(config);
+
+static int vq_show(struct seq_file *seq, void *v)
+{
+ struct pds_vdpa_vq_info *vq = seq->private;
+
+ seq_printf(seq, "ready: %d\n", vq->ready);
+ seq_printf(seq, "desc_addr: %#llx\n", vq->desc_addr);
+ seq_printf(seq, "avail_addr: %#llx\n", vq->avail_addr);
+ seq_printf(seq, "used_addr: %#llx\n", vq->used_addr);
+ seq_printf(seq, "q_len: %d\n", vq->q_len);
+ seq_printf(seq, "qid: %d\n", vq->qid);
+
+ seq_printf(seq, "doorbell: %#llx\n", vq->doorbell);
+ seq_printf(seq, "avail_idx: %d\n", vq->avail_idx);
+ seq_printf(seq, "used_idx: %d\n", vq->used_idx);
+ seq_printf(seq, "irq: %d\n", vq->irq);
+ seq_printf(seq, "irq-name: %s\n", vq->irq_name);
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(vq);
+
+void pds_vdpa_debugfs_add_vdpadev(struct pds_vdpa_aux *vdpa_aux)
+{
+ int i;
+
+ debugfs_create_file("config", 0400, vdpa_aux->dentry, vdpa_aux->pdsv, &config_fops);
+
+ for (i = 0; i < vdpa_aux->pdsv->num_vqs; i++) {
+ char name[16];
+
+ snprintf(name, sizeof(name), "vq%02d", i);
+ debugfs_create_file(name, 0400, vdpa_aux->dentry,
+ &vdpa_aux->pdsv->vqs[i], &vq_fops);
+ }
+}
+
+void pds_vdpa_debugfs_del_vdpadev(struct pds_vdpa_aux *vdpa_aux)
+{
+ debugfs_remove_recursive(vdpa_aux->dentry);
+ vdpa_aux->dentry = NULL;
+}
+
+void pds_vdpa_debugfs_reset_vdpadev(struct pds_vdpa_aux *vdpa_aux)
+{
+ /* we don't keep track of the entries, so remove it all
+ * then rebuild the basics
+ */
+ pds_vdpa_debugfs_del_vdpadev(vdpa_aux);
+ pds_vdpa_debugfs_add_pcidev(vdpa_aux);
+ pds_vdpa_debugfs_add_ident(vdpa_aux);
+}
diff --git a/drivers/vdpa/pds/debugfs.h b/drivers/vdpa/pds/debugfs.h
new file mode 100644
index 0000000000..c088a4e8f1
--- /dev/null
+++ b/drivers/vdpa/pds/debugfs.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _PDS_VDPA_DEBUGFS_H_
+#define _PDS_VDPA_DEBUGFS_H_
+
+#include <linux/debugfs.h>
+
+void pds_vdpa_debugfs_create(void);
+void pds_vdpa_debugfs_destroy(void);
+void pds_vdpa_debugfs_add_pcidev(struct pds_vdpa_aux *vdpa_aux);
+void pds_vdpa_debugfs_add_ident(struct pds_vdpa_aux *vdpa_aux);
+void pds_vdpa_debugfs_add_vdpadev(struct pds_vdpa_aux *vdpa_aux);
+void pds_vdpa_debugfs_del_vdpadev(struct pds_vdpa_aux *vdpa_aux);
+void pds_vdpa_debugfs_reset_vdpadev(struct pds_vdpa_aux *vdpa_aux);
+
+#endif /* _PDS_VDPA_DEBUGFS_H_ */
diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c
new file mode 100644
index 0000000000..25c0fe5ec3
--- /dev/null
+++ b/drivers/vdpa/pds/vdpa_dev.c
@@ -0,0 +1,844 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/pci.h>
+#include <linux/vdpa.h>
+#include <uapi/linux/vdpa.h>
+#include <linux/virtio_pci_modern.h>
+
+#include <linux/pds/pds_common.h>
+#include <linux/pds/pds_core_if.h>
+#include <linux/pds/pds_adminq.h>
+#include <linux/pds/pds_auxbus.h>
+
+#include "vdpa_dev.h"
+#include "aux_drv.h"
+#include "cmds.h"
+#include "debugfs.h"
+
+static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev);
+
+static struct pds_vdpa_device *vdpa_to_pdsv(struct vdpa_device *vdpa_dev)
+{
+ return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev);
+}
+
+static int pds_vdpa_notify_handler(struct notifier_block *nb,
+ unsigned long ecode,
+ void *data)
+{
+ struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device, nb);
+ struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
+
+ dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);
+
+ if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) {
+ if (pdsv->config_cb.callback)
+ pdsv->config_cb.callback(pdsv->config_cb.private);
+ }
+
+ return 0;
+}
+
+static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv)
+{
+ struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
+ struct notifier_block *nb = &pdsv->nb;
+ int err;
+
+ if (!nb->notifier_call) {
+ nb->notifier_call = pds_vdpa_notify_handler;
+ err = pdsc_register_notify(nb);
+ if (err) {
+ nb->notifier_call = NULL;
+ dev_err(dev, "failed to register pds event handler: %ps\n",
+ ERR_PTR(err));
+ return -EINVAL;
+ }
+ dev_dbg(dev, "pds event handler registered\n");
+ }
+
+ return 0;
+}
+
+static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv)
+{
+ if (pdsv->nb.notifier_call) {
+ pdsc_unregister_notify(&pdsv->nb);
+ pdsv->nb.notifier_call = NULL;
+ }
+}
+
+static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
+ u64 desc_addr, u64 driver_addr, u64 device_addr)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+
+ pdsv->vqs[qid].desc_addr = desc_addr;
+ pdsv->vqs[qid].avail_addr = driver_addr;
+ pdsv->vqs[qid].used_addr = device_addr;
+
+ return 0;
+}
+
+static void pds_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+
+ pdsv->vqs[qid].q_len = num;
+}
+
+static void pds_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+
+ iowrite16(qid, pdsv->vqs[qid].notify);
+}
+
+static void pds_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
+ struct vdpa_callback *cb)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+
+ pdsv->vqs[qid].event_cb = *cb;
+}
+
+static irqreturn_t pds_vdpa_isr(int irq, void *data)
+{
+ struct pds_vdpa_vq_info *vq;
+
+ vq = data;
+ if (vq->event_cb.callback)
+ vq->event_cb.callback(vq->event_cb.private);
+
+ return IRQ_HANDLED;
+}
+
+static void pds_vdpa_release_irq(struct pds_vdpa_device *pdsv, int qid)
+{
+ if (pdsv->vqs[qid].irq == VIRTIO_MSI_NO_VECTOR)
+ return;
+
+ free_irq(pdsv->vqs[qid].irq, &pdsv->vqs[qid]);
+ pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;
+}
+
+static void pds_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+ struct device *dev = &pdsv->vdpa_dev.dev;
+ u64 driver_features;
+ u16 invert_idx = 0;
+ int err;
+
+ dev_dbg(dev, "%s: qid %d ready %d => %d\n",
+ __func__, qid, pdsv->vqs[qid].ready, ready);
+ if (ready == pdsv->vqs[qid].ready)
+ return;
+
+ driver_features = pds_vdpa_get_driver_features(vdpa_dev);
+ if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
+ invert_idx = PDS_VDPA_PACKED_INVERT_IDX;
+
+ if (ready) {
+ /* Pass vq setup info to DSC using adminq to gather up and
+ * send all info at once so FW can do its full set up in
+ * one easy operation
+ */
+ err = pds_vdpa_cmd_init_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]);
+ if (err) {
+ dev_err(dev, "Failed to init vq %d: %pe\n",
+ qid, ERR_PTR(err));
+ ready = false;
+ }
+ } else {
+ err = pds_vdpa_cmd_reset_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]);
+ if (err)
+ dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",
+ __func__, qid, ERR_PTR(err));
+ }
+
+ pdsv->vqs[qid].ready = ready;
+}
+
+static bool pds_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+
+ return pdsv->vqs[qid].ready;
+}
+
+static int pds_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
+ const struct vdpa_vq_state *state)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = &padev->aux_dev.dev;
+ u64 driver_features;
+ u16 avail;
+ u16 used;
+
+ if (pdsv->vqs[qid].ready) {
+ dev_err(dev, "Setting device position is denied while vq is enabled\n");
+ return -EINVAL;
+ }
+
+ driver_features = pds_vdpa_get_driver_features(vdpa_dev);
+ if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
+ avail = state->packed.last_avail_idx |
+ (state->packed.last_avail_counter << 15);
+ used = state->packed.last_used_idx |
+ (state->packed.last_used_counter << 15);
+
+ /* The avail and used index are stored with the packed wrap
+ * counter bit inverted. This way, in case set_vq_state is
+ * not called, the initial value can be set to zero prior to
+ * feature negotiation, and it is good for both packed and
+ * split vq.
+ */
+ avail ^= PDS_VDPA_PACKED_INVERT_IDX;
+ used ^= PDS_VDPA_PACKED_INVERT_IDX;
+ } else {
+ avail = state->split.avail_index;
+ /* state->split does not provide a used_index:
+ * the vq will be set to "empty" here, and the vq will read
+ * the current used index the next time the vq is kicked.
+ */
+ used = avail;
+ }
+
+ if (used != avail) {
+ dev_dbg(dev, "Setting used equal to avail, for interoperability\n");
+ used = avail;
+ }
+
+ pdsv->vqs[qid].avail_idx = avail;
+ pdsv->vqs[qid].used_idx = used;
+
+ return 0;
+}
+
+static int pds_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
+ struct vdpa_vq_state *state)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+ struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
+ struct device *dev = &padev->aux_dev.dev;
+ u64 driver_features;
+ u16 avail;
+ u16 used;
+
+ if (pdsv->vqs[qid].ready) {
+ dev_err(dev, "Getting device position is denied while vq is enabled\n");
+ return -EINVAL;
+ }
+
+ avail = pdsv->vqs[qid].avail_idx;
+ used = pdsv->vqs[qid].used_idx;
+
+ driver_features = pds_vdpa_get_driver_features(vdpa_dev);
+ if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
+ avail ^= PDS_VDPA_PACKED_INVERT_IDX;
+ used ^= PDS_VDPA_PACKED_INVERT_IDX;
+
+ state->packed.last_avail_idx = avail & 0x7fff;
+ state->packed.last_avail_counter = avail >> 15;
+ state->packed.last_used_idx = used & 0x7fff;
+ state->packed.last_used_counter = used >> 15;
+ } else {
+ state->split.avail_index = avail;
+ /* state->split does not provide a used_index. */
+ }
+
+ return 0;
+}
+
+static struct vdpa_notification_area
+pds_vdpa_get_vq_notification(struct vdpa_device *vdpa_dev, u16 qid)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+ struct virtio_pci_modern_device *vd_mdev;
+ struct vdpa_notification_area area;
+
+ area.addr = pdsv->vqs[qid].notify_pa;
+
+ vd_mdev = &pdsv->vdpa_aux->vd_mdev;
+ if (!vd_mdev->notify_offset_multiplier)
+ area.size = PDS_PAGE_SIZE;
+ else
+ area.size = vd_mdev->notify_offset_multiplier;
+
+ return area;
+}
+
+static int pds_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, u16 qid)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+
+ return pdsv->vqs[qid].irq;
+}
+
+static u32 pds_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
+{
+ return PDS_PAGE_SIZE;
+}
+
+static u32 pds_vdpa_get_vq_group(struct vdpa_device *vdpa_dev, u16 idx)
+{
+ return 0;
+}
+
+static u64 pds_vdpa_get_device_features(struct vdpa_device *vdpa_dev)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+
+ return pdsv->supported_features;
+}
+
+static int pds_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+ struct device *dev = &pdsv->vdpa_dev.dev;
+ u64 driver_features;
+ u64 nego_features;
+ u64 hw_features;
+ u64 missing;
+
+ if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)) && features) {
+ dev_err(dev, "VIRTIO_F_ACCESS_PLATFORM is not negotiated\n");
+ return -EOPNOTSUPP;
+ }
+
+ /* Check for valid feature bits */
+ nego_features = features & pdsv->supported_features;
+ missing = features & ~nego_features;
+ if (missing) {
+ dev_err(dev, "Can't support all requested features in %#llx, missing %#llx features\n",
+ features, missing);
+ return -EOPNOTSUPP;
+ }
+
+ driver_features = pds_vdpa_get_driver_features(vdpa_dev);
+ pdsv->negotiated_features = nego_features;
+ dev_dbg(dev, "%s: %#llx => %#llx\n",
+ __func__, driver_features, nego_features);
+
+ /* if we're faking the F_MAC, strip it before writing to device */
+ hw_features = le64_to_cpu(pdsv->vdpa_aux->ident.hw_features);
+ if (!(hw_features & BIT_ULL(VIRTIO_NET_F_MAC)))
+ nego_features &= ~BIT_ULL(VIRTIO_NET_F_MAC);
+
+ if (driver_features == nego_features)
+ return 0;
+
+ vp_modern_set_features(&pdsv->vdpa_aux->vd_mdev, nego_features);
+
+ return 0;
+}
+
+static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+
+ return pdsv->negotiated_features;
+}
+
+static void pds_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
+ struct vdpa_callback *cb)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+
+ pdsv->config_cb.callback = cb->callback;
+ pdsv->config_cb.private = cb->private;
+}
+
+static u16 pds_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+
+ /* qemu has assert() that vq_num_max <= VIRTQUEUE_MAX_SIZE (1024) */
+ return min_t(u16, 1024, BIT(le16_to_cpu(pdsv->vdpa_aux->ident.max_qlen)));
+}
+
+static u32 pds_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
+{
+ return VIRTIO_ID_NET;
+}
+
+static u32 pds_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)
+{
+ return PCI_VENDOR_ID_PENSANDO;
+}
+
+static u8 pds_vdpa_get_status(struct vdpa_device *vdpa_dev)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+
+ return vp_modern_get_status(&pdsv->vdpa_aux->vd_mdev);
+}
+
+static int pds_vdpa_request_irqs(struct pds_vdpa_device *pdsv)
+{
+ struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
+ struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux;
+ struct device *dev = &pdsv->vdpa_dev.dev;
+ int max_vq, nintrs, qid, err;
+
+ max_vq = vdpa_aux->vdpa_mdev.max_supported_vqs;
+
+ nintrs = pci_alloc_irq_vectors(pdev, max_vq, max_vq, PCI_IRQ_MSIX);
+ if (nintrs < 0) {
+ dev_err(dev, "Couldn't get %d msix vectors: %pe\n",
+ max_vq, ERR_PTR(nintrs));
+ return nintrs;
+ }
+
+ for (qid = 0; qid < pdsv->num_vqs; ++qid) {
+ int irq = pci_irq_vector(pdev, qid);
+
+ snprintf(pdsv->vqs[qid].irq_name, sizeof(pdsv->vqs[qid].irq_name),
+ "vdpa-%s-%d", dev_name(dev), qid);
+
+ err = request_irq(irq, pds_vdpa_isr, 0,
+ pdsv->vqs[qid].irq_name,
+ &pdsv->vqs[qid]);
+ if (err) {
+ dev_err(dev, "%s: no irq for qid %d: %pe\n",
+ __func__, qid, ERR_PTR(err));
+ goto err_release;
+ }
+
+ pdsv->vqs[qid].irq = irq;
+ }
+
+ vdpa_aux->nintrs = nintrs;
+
+ return 0;
+
+err_release:
+ while (qid--)
+ pds_vdpa_release_irq(pdsv, qid);
+
+ pci_free_irq_vectors(pdev);
+
+ vdpa_aux->nintrs = 0;
+
+ return err;
+}
+
+static void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv)
+{
+ struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
+ struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux;
+ int qid;
+
+ if (!vdpa_aux->nintrs)
+ return;
+
+ for (qid = 0; qid < pdsv->num_vqs; qid++)
+ pds_vdpa_release_irq(pdsv, qid);
+
+ pci_free_irq_vectors(pdev);
+
+ vdpa_aux->nintrs = 0;
+}
+
+static void pds_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+ struct device *dev = &pdsv->vdpa_dev.dev;
+ u8 old_status;
+ int i;
+
+ old_status = pds_vdpa_get_status(vdpa_dev);
+ dev_dbg(dev, "%s: old %#x new %#x\n", __func__, old_status, status);
+
+ if (status & ~old_status & VIRTIO_CONFIG_S_DRIVER_OK) {
+ if (pds_vdpa_request_irqs(pdsv))
+ status = old_status | VIRTIO_CONFIG_S_FAILED;
+ }
+
+ pds_vdpa_cmd_set_status(pdsv, status);
+
+ if (status == 0) {
+ struct vdpa_callback null_cb = { };
+
+ pds_vdpa_set_config_cb(vdpa_dev, &null_cb);
+ pds_vdpa_cmd_reset(pdsv);
+
+ for (i = 0; i < pdsv->num_vqs; i++) {
+ pdsv->vqs[i].avail_idx = 0;
+ pdsv->vqs[i].used_idx = 0;
+ }
+
+ pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
+ }
+
+ if (status & ~old_status & VIRTIO_CONFIG_S_FEATURES_OK) {
+ for (i = 0; i < pdsv->num_vqs; i++) {
+ pdsv->vqs[i].notify =
+ vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev,
+ i, &pdsv->vqs[i].notify_pa);
+ }
+ }
+
+ if (old_status & ~status & VIRTIO_CONFIG_S_DRIVER_OK)
+ pds_vdpa_release_irqs(pdsv);
+}
+
+static void pds_vdpa_init_vqs_entry(struct pds_vdpa_device *pdsv, int qid,
+ void __iomem *notify)
+{
+ memset(&pdsv->vqs[qid], 0, sizeof(pdsv->vqs[0]));
+ pdsv->vqs[qid].qid = qid;
+ pdsv->vqs[qid].pdsv = pdsv;
+ pdsv->vqs[qid].ready = false;
+ pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;
+ pdsv->vqs[qid].notify = notify;
+}
+
+static int pds_vdpa_reset(struct vdpa_device *vdpa_dev)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+ struct device *dev;
+ int err = 0;
+ u8 status;
+ int i;
+
+ dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
+ status = pds_vdpa_get_status(vdpa_dev);
+
+ if (status == 0)
+ return 0;
+
+ if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+ /* Reset the vqs */
+ for (i = 0; i < pdsv->num_vqs && !err; i++) {
+ err = pds_vdpa_cmd_reset_vq(pdsv, i, 0, &pdsv->vqs[i]);
+ if (err)
+ dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",
+ __func__, i, ERR_PTR(err));
+ }
+ }
+
+ pds_vdpa_set_status(vdpa_dev, 0);
+
+ if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+ /* Reset the vq info */
+ for (i = 0; i < pdsv->num_vqs && !err; i++)
+ pds_vdpa_init_vqs_entry(pdsv, i, pdsv->vqs[i].notify);
+ }
+
+ return 0;
+}
+
+static size_t pds_vdpa_get_config_size(struct vdpa_device *vdpa_dev)
+{
+ return sizeof(struct virtio_net_config);
+}
+
+static void pds_vdpa_get_config(struct vdpa_device *vdpa_dev,
+ unsigned int offset,
+ void *buf, unsigned int len)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+ void __iomem *device;
+
+ if (offset + len > sizeof(struct virtio_net_config)) {
+ WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len);
+ return;
+ }
+
+ device = pdsv->vdpa_aux->vd_mdev.device;
+ memcpy_fromio(buf, device + offset, len);
+}
+
+static void pds_vdpa_set_config(struct vdpa_device *vdpa_dev,
+ unsigned int offset, const void *buf,
+ unsigned int len)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+ void __iomem *device;
+
+ if (offset + len > sizeof(struct virtio_net_config)) {
+ WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len);
+ return;
+ }
+
+ device = pdsv->vdpa_aux->vd_mdev.device;
+ memcpy_toio(device + offset, buf, len);
+}
+
+static const struct vdpa_config_ops pds_vdpa_ops = {
+ .set_vq_address = pds_vdpa_set_vq_address,
+ .set_vq_num = pds_vdpa_set_vq_num,
+ .kick_vq = pds_vdpa_kick_vq,
+ .set_vq_cb = pds_vdpa_set_vq_cb,
+ .set_vq_ready = pds_vdpa_set_vq_ready,
+ .get_vq_ready = pds_vdpa_get_vq_ready,
+ .set_vq_state = pds_vdpa_set_vq_state,
+ .get_vq_state = pds_vdpa_get_vq_state,
+ .get_vq_notification = pds_vdpa_get_vq_notification,
+ .get_vq_irq = pds_vdpa_get_vq_irq,
+ .get_vq_align = pds_vdpa_get_vq_align,
+ .get_vq_group = pds_vdpa_get_vq_group,
+
+ .get_device_features = pds_vdpa_get_device_features,
+ .set_driver_features = pds_vdpa_set_driver_features,
+ .get_driver_features = pds_vdpa_get_driver_features,
+ .set_config_cb = pds_vdpa_set_config_cb,
+ .get_vq_num_max = pds_vdpa_get_vq_num_max,
+ .get_device_id = pds_vdpa_get_device_id,
+ .get_vendor_id = pds_vdpa_get_vendor_id,
+ .get_status = pds_vdpa_get_status,
+ .set_status = pds_vdpa_set_status,
+ .reset = pds_vdpa_reset,
+ .get_config_size = pds_vdpa_get_config_size,
+ .get_config = pds_vdpa_get_config,
+ .set_config = pds_vdpa_set_config,
+};
+static struct virtio_device_id pds_vdpa_id_table[] = {
+ {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
+ {0},
+};
+
+static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
+ const struct vdpa_dev_set_config *add_config)
+{
+ struct pds_vdpa_aux *vdpa_aux;
+ struct pds_vdpa_device *pdsv;
+ struct vdpa_mgmt_dev *mgmt;
+ u16 fw_max_vqs, vq_pairs;
+ struct device *dma_dev;
+ struct pci_dev *pdev;
+ struct device *dev;
+ int err;
+ int i;
+
+ vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
+ dev = &vdpa_aux->padev->aux_dev.dev;
+ mgmt = &vdpa_aux->vdpa_mdev;
+
+ if (vdpa_aux->pdsv) {
+ dev_warn(dev, "Multiple vDPA devices on a VF is not supported.\n");
+ return -EOPNOTSUPP;
+ }
+
+ pdsv = vdpa_alloc_device(struct pds_vdpa_device, vdpa_dev,
+ dev, &pds_vdpa_ops, 1, 1, name, false);
+ if (IS_ERR(pdsv)) {
+ dev_err(dev, "Failed to allocate vDPA structure: %pe\n", pdsv);
+ return PTR_ERR(pdsv);
+ }
+
+ vdpa_aux->pdsv = pdsv;
+ pdsv->vdpa_aux = vdpa_aux;
+
+ pdev = vdpa_aux->padev->vf_pdev;
+ dma_dev = &pdev->dev;
+ pdsv->vdpa_dev.dma_dev = dma_dev;
+
+ pdsv->supported_features = mgmt->supported_features;
+
+ if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
+ u64 unsupp_features =
+ add_config->device_features & ~pdsv->supported_features;
+
+ if (unsupp_features) {
+ dev_err(dev, "Unsupported features: %#llx\n", unsupp_features);
+ err = -EOPNOTSUPP;
+ goto err_unmap;
+ }
+
+ pdsv->supported_features = add_config->device_features;
+ }
+
+ err = pds_vdpa_cmd_reset(pdsv);
+ if (err) {
+ dev_err(dev, "Failed to reset hw: %pe\n", ERR_PTR(err));
+ goto err_unmap;
+ }
+
+ err = pds_vdpa_init_hw(pdsv);
+ if (err) {
+ dev_err(dev, "Failed to init hw: %pe\n", ERR_PTR(err));
+ goto err_unmap;
+ }
+
+ fw_max_vqs = le16_to_cpu(pdsv->vdpa_aux->ident.max_vqs);
+ vq_pairs = fw_max_vqs / 2;
+
+ /* Make sure we have the queues being requested */
+ if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MAX_VQP))
+ vq_pairs = add_config->net.max_vq_pairs;
+
+ pdsv->num_vqs = 2 * vq_pairs;
+ if (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))
+ pdsv->num_vqs++;
+
+ if (pdsv->num_vqs > fw_max_vqs) {
+ dev_err(dev, "%s: queue count requested %u greater than max %u\n",
+ __func__, pdsv->num_vqs, fw_max_vqs);
+ err = -ENOSPC;
+ goto err_unmap;
+ }
+
+ if (pdsv->num_vqs != fw_max_vqs) {
+ err = pds_vdpa_cmd_set_max_vq_pairs(pdsv, vq_pairs);
+ if (err) {
+ dev_err(dev, "Failed to set max_vq_pairs: %pe\n",
+ ERR_PTR(err));
+ goto err_unmap;
+ }
+ }
+
+ /* Set a mac, either from the user config if provided
+ * or use the device's mac if not 00:..:00
+ * or set a random mac
+ */
+ if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
+ ether_addr_copy(pdsv->mac, add_config->net.mac);
+ } else {
+ struct virtio_net_config __iomem *vc;
+
+ vc = pdsv->vdpa_aux->vd_mdev.device;
+ memcpy_fromio(pdsv->mac, vc->mac, sizeof(pdsv->mac));
+ if (is_zero_ether_addr(pdsv->mac) &&
+ (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_MAC))) {
+ eth_random_addr(pdsv->mac);
+ dev_info(dev, "setting random mac %pM\n", pdsv->mac);
+ }
+ }
+ pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
+
+ for (i = 0; i < pdsv->num_vqs; i++) {
+ void __iomem *notify;
+
+ notify = vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev,
+ i, &pdsv->vqs[i].notify_pa);
+ pds_vdpa_init_vqs_entry(pdsv, i, notify);
+ }
+
+ pdsv->vdpa_dev.mdev = &vdpa_aux->vdpa_mdev;
+
+ err = pds_vdpa_register_event_handler(pdsv);
+ if (err) {
+ dev_err(dev, "Failed to register for PDS events: %pe\n", ERR_PTR(err));
+ goto err_unmap;
+ }
+
+ /* We use the _vdpa_register_device() call rather than the
+ * vdpa_register_device() to avoid a deadlock because our
+ * dev_add() is called with the vdpa_dev_lock already set
+ * by vdpa_nl_cmd_dev_add_set_doit()
+ */
+ err = _vdpa_register_device(&pdsv->vdpa_dev, pdsv->num_vqs);
+ if (err) {
+ dev_err(dev, "Failed to register to vDPA bus: %pe\n", ERR_PTR(err));
+ goto err_unevent;
+ }
+
+ pds_vdpa_debugfs_add_vdpadev(vdpa_aux);
+
+ return 0;
+
+err_unevent:
+ pds_vdpa_unregister_event_handler(pdsv);
+err_unmap:
+ put_device(&pdsv->vdpa_dev.dev);
+ vdpa_aux->pdsv = NULL;
+ return err;
+}
+
+static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
+ struct vdpa_device *vdpa_dev)
+{
+ struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
+ struct pds_vdpa_aux *vdpa_aux;
+
+ pds_vdpa_unregister_event_handler(pdsv);
+
+ vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
+ _vdpa_unregister_device(vdpa_dev);
+
+ pds_vdpa_cmd_reset(vdpa_aux->pdsv);
+ pds_vdpa_debugfs_reset_vdpadev(vdpa_aux);
+
+ vdpa_aux->pdsv = NULL;
+
+ dev_info(&vdpa_aux->padev->aux_dev.dev, "Removed vdpa device\n");
+}
+
+static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
+ .dev_add = pds_vdpa_dev_add,
+ .dev_del = pds_vdpa_dev_del
+};
+
+int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
+{
+ union pds_core_adminq_cmd cmd = {
+ .vdpa_ident.opcode = PDS_VDPA_CMD_IDENT,
+ .vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id),
+ };
+ union pds_core_adminq_comp comp = {};
+ struct vdpa_mgmt_dev *mgmt;
+ struct pci_dev *pf_pdev;
+ struct device *pf_dev;
+ struct pci_dev *pdev;
+ dma_addr_t ident_pa;
+ struct device *dev;
+ u16 dev_intrs;
+ u16 max_vqs;
+ int err;
+
+ dev = &vdpa_aux->padev->aux_dev.dev;
+ pdev = vdpa_aux->padev->vf_pdev;
+ mgmt = &vdpa_aux->vdpa_mdev;
+
+ /* Get resource info through the PF's adminq. It is a block of info,
+ * so we need to map some memory for PF to make available to the
+ * firmware for writing the data.
+ */
+ pf_pdev = pci_physfn(vdpa_aux->padev->vf_pdev);
+ pf_dev = &pf_pdev->dev;
+ ident_pa = dma_map_single(pf_dev, &vdpa_aux->ident,
+ sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);
+ if (dma_mapping_error(pf_dev, ident_pa)) {
+ dev_err(dev, "Failed to map ident space\n");
+ return -ENOMEM;
+ }
+
+ cmd.vdpa_ident.ident_pa = cpu_to_le64(ident_pa);
+ cmd.vdpa_ident.len = cpu_to_le32(sizeof(vdpa_aux->ident));
+ err = pds_client_adminq_cmd(vdpa_aux->padev, &cmd,
+ sizeof(cmd.vdpa_ident), &comp, 0);
+ dma_unmap_single(pf_dev, ident_pa,
+ sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);
+ if (err) {
+ dev_err(dev, "Failed to ident hw, status %d: %pe\n",
+ comp.status, ERR_PTR(err));
+ return err;
+ }
+
+ max_vqs = le16_to_cpu(vdpa_aux->ident.max_vqs);
+ dev_intrs = pci_msix_vec_count(pdev);
+ dev_dbg(dev, "ident.max_vqs %d dev_intrs %d\n", max_vqs, dev_intrs);
+
+ max_vqs = min_t(u16, dev_intrs, max_vqs);
+ mgmt->max_supported_vqs = min_t(u16, PDS_VDPA_MAX_QUEUES, max_vqs);
+ vdpa_aux->nintrs = 0;
+
+ mgmt->ops = &pds_vdpa_mgmt_dev_ops;
+ mgmt->id_table = pds_vdpa_id_table;
+ mgmt->device = dev;
+ mgmt->supported_features = le64_to_cpu(vdpa_aux->ident.hw_features);
+
+ /* advertise F_MAC even if the device doesn't */
+ mgmt->supported_features |= BIT_ULL(VIRTIO_NET_F_MAC);
+
+ mgmt->config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);
+ mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
+ mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES);
+
+ return 0;
+}
diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h
new file mode 100644
index 0000000000..d984ba24a7
--- /dev/null
+++ b/drivers/vdpa/pds/vdpa_dev.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _VDPA_DEV_H_
+#define _VDPA_DEV_H_
+
+#include <linux/pci.h>
+#include <linux/vdpa.h>
+
+struct pds_vdpa_vq_info {
+ bool ready;
+ u64 desc_addr;
+ u64 avail_addr;
+ u64 used_addr;
+ u32 q_len;
+ u16 qid;
+ int irq;
+ char irq_name[32];
+
+ void __iomem *notify;
+ dma_addr_t notify_pa;
+
+ u64 doorbell;
+ u16 avail_idx;
+ u16 used_idx;
+
+ struct vdpa_callback event_cb;
+ struct pds_vdpa_device *pdsv;
+};
+
+#define PDS_VDPA_MAX_QUEUES 65
+#define PDS_VDPA_MAX_QLEN 32768
+struct pds_vdpa_device {
+ struct vdpa_device vdpa_dev;
+ struct pds_vdpa_aux *vdpa_aux;
+
+ struct pds_vdpa_vq_info vqs[PDS_VDPA_MAX_QUEUES];
+ u64 supported_features; /* supported device features */
+ u64 negotiated_features; /* negotiated features */
+ u8 vdpa_index; /* rsvd for future subdevice use */
+ u8 num_vqs; /* num vqs in use */
+ u8 mac[ETH_ALEN]; /* mac selected when the device was added */
+ struct vdpa_callback config_cb;
+ struct notifier_block nb;
+};
+
+#define PDS_VDPA_PACKED_INVERT_IDX 0x8000
+
+int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux);
+#endif /* _VDPA_DEV_H_ */
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-12 11:17:14 +0000