summaryrefslogtreecommitdiffstats
path: root/drivers/net/ethernet/pensando
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 18:49:45 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 18:49:45 +0000
commit2c3c1048746a4622d8c89a29670120dc8fab93c4 (patch)
tree848558de17fb3008cdf4d861b01ac7781903ce39 /drivers/net/ethernet/pensando
parentInitial commit. (diff)
downloadlinux-2c3c1048746a4622d8c89a29670120dc8fab93c4.tar.xz
linux-2c3c1048746a4622d8c89a29670120dc8fab93c4.zip
Adding upstream version 6.1.76.upstream/6.1.76
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'drivers/net/ethernet/pensando')
-rw-r--r--drivers/net/ethernet/pensando/Kconfig35
-rw-r--r--drivers/net/ethernet/pensando/Makefile6
-rw-r--r--drivers/net/ethernet/pensando/ionic/Makefile9
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic.h97
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_bus.h16
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c418
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_debugfs.c300
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_debugfs.h34
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_dev.c752
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_dev.h374
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_devlink.c106
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_devlink.h17
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_ethtool.c1003
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_ethtool.h9
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_fw.c196
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_if.h3092
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_lif.c3721
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_lif.h343
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_main.c738
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_phc.c647
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_regs.h136
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c619
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_rx_filter.h50
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_stats.c350
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_stats.h59
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_txrx.c1350
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_txrx.h20
27 files changed, 14497 insertions, 0 deletions
diff --git a/drivers/net/ethernet/pensando/Kconfig b/drivers/net/ethernet/pensando/Kconfig
new file mode 100644
index 000000000..3f7519e43
--- /dev/null
+++ b/drivers/net/ethernet/pensando/Kconfig
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2019 Pensando Systems, Inc
+#
+# Pensando device configuration
+#
+
+config NET_VENDOR_PENSANDO
+ bool "Pensando devices"
+ default y
+ help
+ If you have a network (Ethernet) card belonging to this class, say Y.
+
+ Note that the answer to this question doesn't directly affect the
+ kernel: saying N will just cause the configurator to skip all
+ the questions about Pensando cards. If you say Y, you will be asked
+ for your specific card in the following questions.
+
+if NET_VENDOR_PENSANDO
+
+config IONIC
+ tristate "Pensando Ethernet IONIC Support"
+ depends on 64BIT && PCI
+ depends on PTP_1588_CLOCK_OPTIONAL
+ select NET_DEVLINK
+ select DIMLIB
+ help
+ This enables the support for the Pensando family of Ethernet
+ adapters. More specific information on this driver can be
+ found in
+ <file:Documentation/networking/device_drivers/ethernet/pensando/ionic.rst>.
+
+ To compile this driver as a module, choose M here. The module
+ will be called ionic.
+
+endif # NET_VENDOR_PENSANDO
diff --git a/drivers/net/ethernet/pensando/Makefile b/drivers/net/ethernet/pensando/Makefile
new file mode 100644
index 000000000..21ce7499c
--- /dev/null
+++ b/drivers/net/ethernet/pensando/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for the Pensando network device drivers.
+#
+
+obj-$(CONFIG_IONIC) += ionic/
diff --git a/drivers/net/ethernet/pensando/ionic/Makefile b/drivers/net/ethernet/pensando/ionic/Makefile
new file mode 100644
index 000000000..4e7642a2d
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright(c) 2017 - 2019 Pensando Systems, Inc
+
+obj-$(CONFIG_IONIC) := ionic.o
+
+ionic-y := ionic_main.o ionic_bus_pci.o ionic_devlink.o ionic_dev.o \
+ ionic_debugfs.o ionic_lif.o ionic_rx_filter.o ionic_ethtool.o \
+ ionic_txrx.o ionic_stats.o ionic_fw.o
+ionic-$(CONFIG_PTP_1588_CLOCK) += ionic_phc.o
diff --git a/drivers/net/ethernet/pensando/ionic/ionic.h b/drivers/net/ethernet/pensando/ionic/ionic.h
new file mode 100644
index 000000000..602f4d45d
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#ifndef _IONIC_H_
+#define _IONIC_H_
+
+struct ionic_lif;
+
+#include "ionic_if.h"
+#include "ionic_dev.h"
+#include "ionic_devlink.h"
+
+#define IONIC_DRV_NAME "ionic"
+#define IONIC_DRV_DESCRIPTION "Pensando Ethernet NIC Driver"
+
+#define PCI_VENDOR_ID_PENSANDO 0x1dd8
+
+#define PCI_DEVICE_ID_PENSANDO_IONIC_ETH_PF 0x1002
+#define PCI_DEVICE_ID_PENSANDO_IONIC_ETH_VF 0x1003
+
+#define DEVCMD_TIMEOUT 5
+#define IONIC_ADMINQ_TIME_SLICE msecs_to_jiffies(100)
+
+#define IONIC_PHC_UPDATE_NS 10000000000 /* 10s in nanoseconds */
+#define NORMAL_PPB 1000000000 /* one billion parts per billion */
+#define SCALED_PPM (1000000ull << 16) /* 2^16 million parts per 2^16 million */
+
+struct ionic_vf {
+ u16 index;
+ u8 macaddr[6];
+ __le32 maxrate;
+ __le16 vlanid;
+ u8 spoofchk;
+ u8 trusted;
+ u8 linkstate;
+ dma_addr_t stats_pa;
+ struct ionic_lif_stats stats;
+};
+
+struct ionic {
+ struct pci_dev *pdev;
+ struct device *dev;
+ struct devlink_port dl_port;
+ struct ionic_dev idev;
+ struct mutex dev_cmd_lock; /* lock for dev_cmd operations */
+ struct dentry *dentry;
+ struct ionic_dev_bar bars[IONIC_BARS_MAX];
+ unsigned int num_bars;
+ struct ionic_identity ident;
+ struct ionic_lif *lif;
+ unsigned int nnqs_per_lif;
+ unsigned int neqs_per_lif;
+ unsigned int ntxqs_per_lif;
+ unsigned int nrxqs_per_lif;
+ unsigned int nintrs;
+ DECLARE_BITMAP(intrs, IONIC_INTR_CTRL_REGS_MAX);
+ struct work_struct nb_work;
+ struct notifier_block nb;
+ struct rw_semaphore vf_op_lock; /* lock for VF operations */
+ struct ionic_vf *vfs;
+ int num_vfs;
+ struct timer_list watchdog_timer;
+ int watchdog_period;
+};
+
+struct ionic_admin_ctx {
+ struct completion work;
+ union ionic_adminq_cmd cmd;
+ union ionic_adminq_comp comp;
+};
+
+int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx);
+int ionic_adminq_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx,
+ const int err, const bool do_msg);
+int ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx);
+int ionic_adminq_post_wait_nomsg(struct ionic_lif *lif, struct ionic_admin_ctx *ctx);
+void ionic_adminq_netdev_err_print(struct ionic_lif *lif, u8 opcode,
+ u8 status, int err);
+
+int ionic_dev_cmd_wait(struct ionic *ionic, unsigned long max_wait);
+int ionic_dev_cmd_wait_nomsg(struct ionic *ionic, unsigned long max_wait);
+void ionic_dev_cmd_dev_err_print(struct ionic *ionic, u8 opcode, u8 status,
+ int err);
+int ionic_set_dma_mask(struct ionic *ionic);
+int ionic_setup(struct ionic *ionic);
+
+int ionic_identify(struct ionic *ionic);
+int ionic_init(struct ionic *ionic);
+int ionic_reset(struct ionic *ionic);
+
+int ionic_port_identify(struct ionic *ionic);
+int ionic_port_init(struct ionic *ionic);
+int ionic_port_reset(struct ionic *ionic);
+
+const char *ionic_vf_attr_to_str(enum ionic_vf_attr attr);
+
+#endif /* _IONIC_H_ */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus.h b/drivers/net/ethernet/pensando/ionic/ionic_bus.h
new file mode 100644
index 000000000..2f4d08c64
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_bus.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#ifndef _IONIC_BUS_H_
+#define _IONIC_BUS_H_
+
+int ionic_bus_get_irq(struct ionic *ionic, unsigned int num);
+const char *ionic_bus_info(struct ionic *ionic);
+int ionic_bus_alloc_irq_vectors(struct ionic *ionic, unsigned int nintrs);
+void ionic_bus_free_irq_vectors(struct ionic *ionic);
+int ionic_bus_register_driver(void);
+void ionic_bus_unregister_driver(void);
+void __iomem *ionic_bus_map_dbpage(struct ionic *ionic, int page_num);
+void ionic_bus_unmap_dbpage(struct ionic *ionic, void __iomem *page);
+
+#endif /* _IONIC_BUS_H_ */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
new file mode 100644
index 000000000..ce436e973
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
@@ -0,0 +1,418 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/pci.h>
+
+#include "ionic.h"
+#include "ionic_bus.h"
+#include "ionic_lif.h"
+#include "ionic_debugfs.h"
+
+/* Supported devices */
+static const struct pci_device_id ionic_id_table[] = {
+ { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_PF) },
+ { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_VF) },
+ { 0, } /* end of table */
+};
+MODULE_DEVICE_TABLE(pci, ionic_id_table);
+
+int ionic_bus_get_irq(struct ionic *ionic, unsigned int num)
+{
+ return pci_irq_vector(ionic->pdev, num);
+}
+
+const char *ionic_bus_info(struct ionic *ionic)
+{
+ return pci_name(ionic->pdev);
+}
+
+int ionic_bus_alloc_irq_vectors(struct ionic *ionic, unsigned int nintrs)
+{
+ return pci_alloc_irq_vectors(ionic->pdev, nintrs, nintrs,
+ PCI_IRQ_MSIX);
+}
+
+void ionic_bus_free_irq_vectors(struct ionic *ionic)
+{
+ if (!ionic->nintrs)
+ return;
+
+ pci_free_irq_vectors(ionic->pdev);
+}
+
+static int ionic_map_bars(struct ionic *ionic)
+{
+ struct pci_dev *pdev = ionic->pdev;
+ struct device *dev = ionic->dev;
+ struct ionic_dev_bar *bars;
+ unsigned int i, j;
+
+ bars = ionic->bars;
+ ionic->num_bars = 0;
+
+ for (i = 0, j = 0; i < IONIC_BARS_MAX; i++) {
+ if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
+ continue;
+ bars[j].len = pci_resource_len(pdev, i);
+
+ /* only map the whole bar 0 */
+ if (j > 0) {
+ bars[j].vaddr = NULL;
+ } else {
+ bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);
+ if (!bars[j].vaddr) {
+ dev_err(dev,
+ "Cannot memory-map BAR %d, aborting\n",
+ i);
+ return -ENODEV;
+ }
+ }
+
+ bars[j].bus_addr = pci_resource_start(pdev, i);
+ bars[j].res_index = i;
+ ionic->num_bars++;
+ j++;
+ }
+
+ return 0;
+}
+
+static void ionic_unmap_bars(struct ionic *ionic)
+{
+ struct ionic_dev_bar *bars = ionic->bars;
+ unsigned int i;
+
+ for (i = 0; i < IONIC_BARS_MAX; i++) {
+ if (bars[i].vaddr) {
+ iounmap(bars[i].vaddr);
+ bars[i].bus_addr = 0;
+ bars[i].vaddr = NULL;
+ bars[i].len = 0;
+ }
+ }
+}
+
+void __iomem *ionic_bus_map_dbpage(struct ionic *ionic, int page_num)
+{
+ return pci_iomap_range(ionic->pdev,
+ ionic->bars[IONIC_PCI_BAR_DBELL].res_index,
+ (u64)page_num << PAGE_SHIFT, PAGE_SIZE);
+}
+
+void ionic_bus_unmap_dbpage(struct ionic *ionic, void __iomem *page)
+{
+ iounmap(page);
+}
+
+static void ionic_vf_dealloc_locked(struct ionic *ionic)
+{
+ struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR };
+ struct ionic_vf *v;
+ int i;
+
+ if (!ionic->vfs)
+ return;
+
+ for (i = ionic->num_vfs - 1; i >= 0; i--) {
+ v = &ionic->vfs[i];
+
+ if (v->stats_pa) {
+ vfc.stats_pa = 0;
+ (void)ionic_set_vf_config(ionic, i, &vfc);
+ dma_unmap_single(ionic->dev, v->stats_pa,
+ sizeof(v->stats), DMA_FROM_DEVICE);
+ v->stats_pa = 0;
+ }
+ }
+
+ kfree(ionic->vfs);
+ ionic->vfs = NULL;
+ ionic->num_vfs = 0;
+}
+
+static void ionic_vf_dealloc(struct ionic *ionic)
+{
+ down_write(&ionic->vf_op_lock);
+ ionic_vf_dealloc_locked(ionic);
+ up_write(&ionic->vf_op_lock);
+}
+
+static int ionic_vf_alloc(struct ionic *ionic, int num_vfs)
+{
+ struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR };
+ struct ionic_vf *v;
+ int err = 0;
+ int i;
+
+ down_write(&ionic->vf_op_lock);
+
+ ionic->vfs = kcalloc(num_vfs, sizeof(struct ionic_vf), GFP_KERNEL);
+ if (!ionic->vfs) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ for (i = 0; i < num_vfs; i++) {
+ v = &ionic->vfs[i];
+ v->stats_pa = dma_map_single(ionic->dev, &v->stats,
+ sizeof(v->stats), DMA_FROM_DEVICE);
+ if (dma_mapping_error(ionic->dev, v->stats_pa)) {
+ v->stats_pa = 0;
+ err = -ENODEV;
+ goto out;
+ }
+
+ ionic->num_vfs++;
+
+ /* ignore failures from older FW, we just won't get stats */
+ vfc.stats_pa = cpu_to_le64(v->stats_pa);
+ (void)ionic_set_vf_config(ionic, i, &vfc);
+ }
+
+out:
+ if (err)
+ ionic_vf_dealloc_locked(ionic);
+ up_write(&ionic->vf_op_lock);
+ return err;
+}
+
+static int ionic_sriov_configure(struct pci_dev *pdev, int num_vfs)
+{
+ struct ionic *ionic = pci_get_drvdata(pdev);
+ struct device *dev = ionic->dev;
+ int ret = 0;
+
+ if (ionic->lif &&
+ test_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state))
+ return -EBUSY;
+
+ if (num_vfs > 0) {
+ ret = pci_enable_sriov(pdev, num_vfs);
+ if (ret) {
+ dev_err(dev, "Cannot enable SRIOV: %d\n", ret);
+ goto out;
+ }
+
+ ret = ionic_vf_alloc(ionic, num_vfs);
+ if (ret) {
+ dev_err(dev, "Cannot alloc VFs: %d\n", ret);
+ pci_disable_sriov(pdev);
+ goto out;
+ }
+
+ ret = num_vfs;
+ } else {
+ pci_disable_sriov(pdev);
+ ionic_vf_dealloc(ionic);
+ }
+
+out:
+ return ret;
+}
+
+static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+ struct device *dev = &pdev->dev;
+ struct ionic *ionic;
+ int num_vfs;
+ int err;
+
+ ionic = ionic_devlink_alloc(dev);
+ if (!ionic)
+ return -ENOMEM;
+
+ ionic->pdev = pdev;
+ ionic->dev = dev;
+ pci_set_drvdata(pdev, ionic);
+ mutex_init(&ionic->dev_cmd_lock);
+
+ /* Query system for DMA addressing limitation for the device. */
+ err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(IONIC_ADDR_LEN));
+ if (err) {
+ dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting. err=%d\n",
+ err);
+ goto err_out_clear_drvdata;
+ }
+
+ ionic_debugfs_add_dev(ionic);
+
+ /* Setup PCI device */
+ err = pci_enable_device_mem(pdev);
+ if (err) {
+ dev_err(dev, "Cannot enable PCI device: %d, aborting\n", err);
+ goto err_out_debugfs_del_dev;
+ }
+
+ err = pci_request_regions(pdev, IONIC_DRV_NAME);
+ if (err) {
+ dev_err(dev, "Cannot request PCI regions: %d, aborting\n", err);
+ goto err_out_pci_disable_device;
+ }
+
+ pcie_print_link_status(pdev);
+
+ err = ionic_map_bars(ionic);
+ if (err)
+ goto err_out_pci_release_regions;
+
+ /* Configure the device */
+ err = ionic_setup(ionic);
+ if (err) {
+ dev_err(dev, "Cannot setup device: %d, aborting\n", err);
+ goto err_out_unmap_bars;
+ }
+ pci_set_master(pdev);
+
+ err = ionic_identify(ionic);
+ if (err) {
+ dev_err(dev, "Cannot identify device: %d, aborting\n", err);
+ goto err_out_teardown;
+ }
+ ionic_debugfs_add_ident(ionic);
+
+ err = ionic_init(ionic);
+ if (err) {
+ dev_err(dev, "Cannot init device: %d, aborting\n", err);
+ goto err_out_teardown;
+ }
+
+ /* Configure the ports */
+ err = ionic_port_identify(ionic);
+ if (err) {
+ dev_err(dev, "Cannot identify port: %d, aborting\n", err);
+ goto err_out_reset;
+ }
+
+ err = ionic_port_init(ionic);
+ if (err) {
+ dev_err(dev, "Cannot init port: %d, aborting\n", err);
+ goto err_out_reset;
+ }
+
+ /* Allocate and init the LIF */
+ err = ionic_lif_size(ionic);
+ if (err) {
+ dev_err(dev, "Cannot size LIF: %d, aborting\n", err);
+ goto err_out_port_reset;
+ }
+
+ err = ionic_lif_alloc(ionic);
+ if (err) {
+ dev_err(dev, "Cannot allocate LIF: %d, aborting\n", err);
+ goto err_out_free_irqs;
+ }
+
+ err = ionic_lif_init(ionic->lif);
+ if (err) {
+ dev_err(dev, "Cannot init LIF: %d, aborting\n", err);
+ goto err_out_free_lifs;
+ }
+
+ init_rwsem(&ionic->vf_op_lock);
+ num_vfs = pci_num_vf(pdev);
+ if (num_vfs) {
+ dev_info(dev, "%d VFs found already enabled\n", num_vfs);
+ err = ionic_vf_alloc(ionic, num_vfs);
+ if (err)
+ dev_err(dev, "Cannot enable existing VFs: %d\n", err);
+ }
+
+ err = ionic_devlink_register(ionic);
+ if (err) {
+ dev_err(dev, "Cannot register devlink: %d\n", err);
+ goto err_out_deinit_lifs;
+ }
+
+ err = ionic_lif_register(ionic->lif);
+ if (err) {
+ dev_err(dev, "Cannot register LIF: %d, aborting\n", err);
+ goto err_out_deregister_devlink;
+ }
+
+ mod_timer(&ionic->watchdog_timer,
+ round_jiffies(jiffies + ionic->watchdog_period));
+
+ return 0;
+
+err_out_deregister_devlink:
+ ionic_devlink_unregister(ionic);
+err_out_deinit_lifs:
+ ionic_vf_dealloc(ionic);
+ ionic_lif_deinit(ionic->lif);
+err_out_free_lifs:
+ ionic_lif_free(ionic->lif);
+ ionic->lif = NULL;
+err_out_free_irqs:
+ ionic_bus_free_irq_vectors(ionic);
+err_out_port_reset:
+ ionic_port_reset(ionic);
+err_out_reset:
+ ionic_reset(ionic);
+err_out_teardown:
+ pci_clear_master(pdev);
+ /* Don't fail the probe for these errors, keep
+ * the hw interface around for inspection
+ */
+ return 0;
+
+err_out_unmap_bars:
+ ionic_unmap_bars(ionic);
+err_out_pci_release_regions:
+ pci_release_regions(pdev);
+err_out_pci_disable_device:
+ pci_disable_device(pdev);
+err_out_debugfs_del_dev:
+ ionic_debugfs_del_dev(ionic);
+err_out_clear_drvdata:
+ mutex_destroy(&ionic->dev_cmd_lock);
+ ionic_devlink_free(ionic);
+
+ return err;
+}
+
+static void ionic_remove(struct pci_dev *pdev)
+{
+ struct ionic *ionic = pci_get_drvdata(pdev);
+
+ del_timer_sync(&ionic->watchdog_timer);
+
+ if (ionic->lif) {
+ ionic_lif_unregister(ionic->lif);
+ ionic_devlink_unregister(ionic);
+ ionic_lif_deinit(ionic->lif);
+ ionic_lif_free(ionic->lif);
+ ionic->lif = NULL;
+ ionic_bus_free_irq_vectors(ionic);
+ }
+
+ ionic_port_reset(ionic);
+ ionic_reset(ionic);
+ pci_clear_master(pdev);
+ ionic_unmap_bars(ionic);
+ pci_release_regions(pdev);
+ pci_disable_device(pdev);
+ ionic_debugfs_del_dev(ionic);
+ mutex_destroy(&ionic->dev_cmd_lock);
+ ionic_devlink_free(ionic);
+}
+
+static struct pci_driver ionic_driver = {
+ .name = IONIC_DRV_NAME,
+ .id_table = ionic_id_table,
+ .probe = ionic_probe,
+ .remove = ionic_remove,
+ .sriov_configure = ionic_sriov_configure,
+};
+
+int ionic_bus_register_driver(void)
+{
+ return pci_register_driver(&ionic_driver);
+}
+
+void ionic_bus_unregister_driver(void)
+{
+ pci_unregister_driver(&ionic_driver);
+}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_debugfs.c b/drivers/net/ethernet/pensando/ionic/ionic_debugfs.c
new file mode 100644
index 000000000..c58217027
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_debugfs.c
@@ -0,0 +1,300 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+
+#include "ionic.h"
+#include "ionic_bus.h"
+#include "ionic_lif.h"
+#include "ionic_debugfs.h"
+
+#ifdef CONFIG_DEBUG_FS
+
+static struct dentry *ionic_dir;
+
+void ionic_debugfs_create(void)
+{
+ ionic_dir = debugfs_create_dir(IONIC_DRV_NAME, NULL);
+}
+
+void ionic_debugfs_destroy(void)
+{
+ debugfs_remove_recursive(ionic_dir);
+}
+
+void ionic_debugfs_add_dev(struct ionic *ionic)
+{
+ ionic->dentry = debugfs_create_dir(ionic_bus_info(ionic), ionic_dir);
+}
+
+void ionic_debugfs_del_dev(struct ionic *ionic)
+{
+ debugfs_remove_recursive(ionic->dentry);
+ ionic->dentry = NULL;
+}
+
+static int identity_show(struct seq_file *seq, void *v)
+{
+ struct ionic *ionic = seq->private;
+ struct ionic_identity *ident;
+
+ ident = &ionic->ident;
+
+ seq_printf(seq, "nlifs: %d\n", ident->dev.nlifs);
+ seq_printf(seq, "nintrs: %d\n", ident->dev.nintrs);
+ seq_printf(seq, "ndbpgs_per_lif: %d\n", ident->dev.ndbpgs_per_lif);
+ seq_printf(seq, "intr_coal_mult: %d\n", ident->dev.intr_coal_mult);
+ seq_printf(seq, "intr_coal_div: %d\n", ident->dev.intr_coal_div);
+
+ seq_printf(seq, "max_ucast_filters: %d\n", ident->lif.eth.max_ucast_filters);
+ seq_printf(seq, "max_mcast_filters: %d\n", ident->lif.eth.max_mcast_filters);
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(identity);
+
+void ionic_debugfs_add_ident(struct ionic *ionic)
+{
+ debugfs_create_file("identity", 0400, ionic->dentry,
+ ionic, &identity_fops);
+}
+
+void ionic_debugfs_add_sizes(struct ionic *ionic)
+{
+ debugfs_create_u32("nlifs", 0400, ionic->dentry,
+ (u32 *)&ionic->ident.dev.nlifs);
+ debugfs_create_u32("nintrs", 0400, ionic->dentry, &ionic->nintrs);
+
+ debugfs_create_u32("ntxqs_per_lif", 0400, ionic->dentry,
+ (u32 *)&ionic->ident.lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
+ debugfs_create_u32("nrxqs_per_lif", 0400, ionic->dentry,
+ (u32 *)&ionic->ident.lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
+}
+
+static int q_tail_show(struct seq_file *seq, void *v)
+{
+ struct ionic_queue *q = seq->private;
+
+ seq_printf(seq, "%d\n", q->tail_idx);
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(q_tail);
+
+static int q_head_show(struct seq_file *seq, void *v)
+{
+ struct ionic_queue *q = seq->private;
+
+ seq_printf(seq, "%d\n", q->head_idx);
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(q_head);
+
+static int cq_tail_show(struct seq_file *seq, void *v)
+{
+ struct ionic_cq *cq = seq->private;
+
+ seq_printf(seq, "%d\n", cq->tail_idx);
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(cq_tail);
+
+static const struct debugfs_reg32 intr_ctrl_regs[] = {
+ { .name = "coal_init", .offset = 0, },
+ { .name = "mask", .offset = 4, },
+ { .name = "credits", .offset = 8, },
+ { .name = "mask_on_assert", .offset = 12, },
+ { .name = "coal_timer", .offset = 16, },
+};
+
+void ionic_debugfs_add_qcq(struct ionic_lif *lif, struct ionic_qcq *qcq)
+{
+ struct dentry *qcq_dentry, *q_dentry, *cq_dentry;
+ struct dentry *intr_dentry, *stats_dentry;
+ struct ionic_dev *idev = &lif->ionic->idev;
+ struct debugfs_regset32 *intr_ctrl_regset;
+ struct ionic_intr_info *intr = &qcq->intr;
+ struct debugfs_blob_wrapper *desc_blob;
+ struct device *dev = lif->ionic->dev;
+ struct ionic_queue *q = &qcq->q;
+ struct ionic_cq *cq = &qcq->cq;
+
+ qcq_dentry = debugfs_create_dir(q->name, lif->dentry);
+ if (IS_ERR_OR_NULL(qcq_dentry))
+ return;
+ qcq->dentry = qcq_dentry;
+
+ debugfs_create_x64("q_base_pa", 0400, qcq_dentry, &qcq->q_base_pa);
+ debugfs_create_x32("q_size", 0400, qcq_dentry, &qcq->q_size);
+ debugfs_create_x64("cq_base_pa", 0400, qcq_dentry, &qcq->cq_base_pa);
+ debugfs_create_x32("cq_size", 0400, qcq_dentry, &qcq->cq_size);
+ debugfs_create_x64("sg_base_pa", 0400, qcq_dentry, &qcq->sg_base_pa);
+ debugfs_create_x32("sg_size", 0400, qcq_dentry, &qcq->sg_size);
+
+ q_dentry = debugfs_create_dir("q", qcq->dentry);
+
+ debugfs_create_u32("index", 0400, q_dentry, &q->index);
+ debugfs_create_u32("num_descs", 0400, q_dentry, &q->num_descs);
+ debugfs_create_u32("desc_size", 0400, q_dentry, &q->desc_size);
+ debugfs_create_u32("pid", 0400, q_dentry, &q->pid);
+ debugfs_create_u32("qid", 0400, q_dentry, &q->hw_index);
+ debugfs_create_u32("qtype", 0400, q_dentry, &q->hw_type);
+ debugfs_create_u64("drop", 0400, q_dentry, &q->drop);
+
+ debugfs_create_file("tail", 0400, q_dentry, q, &q_tail_fops);
+ debugfs_create_file("head", 0400, q_dentry, q, &q_head_fops);
+
+ desc_blob = devm_kzalloc(dev, sizeof(*desc_blob), GFP_KERNEL);
+ if (!desc_blob)
+ return;
+ desc_blob->data = q->base;
+ desc_blob->size = (unsigned long)q->num_descs * q->desc_size;
+ debugfs_create_blob("desc_blob", 0400, q_dentry, desc_blob);
+
+ if (qcq->flags & IONIC_QCQ_F_SG) {
+ desc_blob = devm_kzalloc(dev, sizeof(*desc_blob), GFP_KERNEL);
+ if (!desc_blob)
+ return;
+ desc_blob->data = q->sg_base;
+ desc_blob->size = (unsigned long)q->num_descs * q->sg_desc_size;
+ debugfs_create_blob("sg_desc_blob", 0400, q_dentry,
+ desc_blob);
+ }
+
+ cq_dentry = debugfs_create_dir("cq", qcq->dentry);
+
+ debugfs_create_x64("base_pa", 0400, cq_dentry, &cq->base_pa);
+ debugfs_create_u32("num_descs", 0400, cq_dentry, &cq->num_descs);
+ debugfs_create_u32("desc_size", 0400, cq_dentry, &cq->desc_size);
+ debugfs_create_bool("done_color", 0400, cq_dentry, &cq->done_color);
+
+ debugfs_create_file("tail", 0400, cq_dentry, cq, &cq_tail_fops);
+
+ desc_blob = devm_kzalloc(dev, sizeof(*desc_blob), GFP_KERNEL);
+ if (!desc_blob)
+ return;
+ desc_blob->data = cq->base;
+ desc_blob->size = (unsigned long)cq->num_descs * cq->desc_size;
+ debugfs_create_blob("desc_blob", 0400, cq_dentry, desc_blob);
+
+ if (qcq->flags & IONIC_QCQ_F_INTR) {
+ intr_dentry = debugfs_create_dir("intr", qcq->dentry);
+
+ debugfs_create_u32("index", 0400, intr_dentry,
+ &intr->index);
+ debugfs_create_u32("vector", 0400, intr_dentry,
+ &intr->vector);
+ debugfs_create_u32("dim_coal_hw", 0400, intr_dentry,
+ &intr->dim_coal_hw);
+
+ intr_ctrl_regset = devm_kzalloc(dev, sizeof(*intr_ctrl_regset),
+ GFP_KERNEL);
+ if (!intr_ctrl_regset)
+ return;
+ intr_ctrl_regset->regs = intr_ctrl_regs;
+ intr_ctrl_regset->nregs = ARRAY_SIZE(intr_ctrl_regs);
+ intr_ctrl_regset->base = &idev->intr_ctrl[intr->index];
+
+ debugfs_create_regset32("intr_ctrl", 0400, intr_dentry,
+ intr_ctrl_regset);
+ }
+
+ if (qcq->flags & IONIC_QCQ_F_NOTIFYQ) {
+ stats_dentry = debugfs_create_dir("notifyblock", qcq->dentry);
+
+ debugfs_create_u64("eid", 0400, stats_dentry,
+ (u64 *)&lif->info->status.eid);
+ debugfs_create_u16("link_status", 0400, stats_dentry,
+ (u16 *)&lif->info->status.link_status);
+ debugfs_create_u32("link_speed", 0400, stats_dentry,
+ (u32 *)&lif->info->status.link_speed);
+ debugfs_create_u16("link_down_count", 0400, stats_dentry,
+ (u16 *)&lif->info->status.link_down_count);
+ }
+}
+
+static int netdev_show(struct seq_file *seq, void *v)
+{
+ struct net_device *netdev = seq->private;
+
+ seq_printf(seq, "%s\n", netdev->name);
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(netdev);
+
+static int lif_filters_show(struct seq_file *seq, void *v)
+{
+ struct ionic_lif *lif = seq->private;
+ struct ionic_rx_filter *f;
+ struct hlist_head *head;
+ struct hlist_node *tmp;
+ unsigned int i;
+
+ seq_puts(seq, "id flow state type filter\n");
+ spin_lock_bh(&lif->rx_filters.lock);
+ for (i = 0; i < IONIC_RX_FILTER_HLISTS; i++) {
+ head = &lif->rx_filters.by_id[i];
+ hlist_for_each_entry_safe(f, tmp, head, by_id) {
+ switch (le16_to_cpu(f->cmd.match)) {
+ case IONIC_RX_FILTER_MATCH_VLAN:
+ seq_printf(seq, "0x%04x 0x%08x 0x%02x vlan 0x%04x\n",
+ f->filter_id, f->flow_id, f->state,
+ le16_to_cpu(f->cmd.vlan.vlan));
+ break;
+ case IONIC_RX_FILTER_MATCH_MAC:
+ seq_printf(seq, "0x%04x 0x%08x 0x%02x mac %pM\n",
+ f->filter_id, f->flow_id, f->state,
+ f->cmd.mac.addr);
+ break;
+ case IONIC_RX_FILTER_MATCH_MAC_VLAN:
+ seq_printf(seq, "0x%04x 0x%08x 0x%02x macvl 0x%04x %pM\n",
+ f->filter_id, f->flow_id, f->state,
+ le16_to_cpu(f->cmd.vlan.vlan),
+ f->cmd.mac.addr);
+ break;
+ case IONIC_RX_FILTER_STEER_PKTCLASS:
+ seq_printf(seq, "0x%04x 0x%08x 0x%02x rxstr 0x%llx\n",
+ f->filter_id, f->flow_id, f->state,
+ le64_to_cpu(f->cmd.pkt_class));
+ break;
+ }
+ }
+ }
+ spin_unlock_bh(&lif->rx_filters.lock);
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(lif_filters);
+
+void ionic_debugfs_add_lif(struct ionic_lif *lif)
+{
+ struct dentry *lif_dentry;
+
+ lif_dentry = debugfs_create_dir(lif->name, lif->ionic->dentry);
+ if (IS_ERR_OR_NULL(lif_dentry))
+ return;
+ lif->dentry = lif_dentry;
+
+ debugfs_create_file("netdev", 0400, lif->dentry,
+ lif->netdev, &netdev_fops);
+ debugfs_create_file("filters", 0400, lif->dentry,
+ lif, &lif_filters_fops);
+}
+
+void ionic_debugfs_del_lif(struct ionic_lif *lif)
+{
+ debugfs_remove_recursive(lif->dentry);
+ lif->dentry = NULL;
+}
+
+void ionic_debugfs_del_qcq(struct ionic_qcq *qcq)
+{
+ debugfs_remove_recursive(qcq->dentry);
+ qcq->dentry = NULL;
+}
+
+#endif
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_debugfs.h b/drivers/net/ethernet/pensando/ionic/ionic_debugfs.h
new file mode 100644
index 000000000..c44ebde17
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_debugfs.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#ifndef _IONIC_DEBUGFS_H_
+#define _IONIC_DEBUGFS_H_
+
+#include <linux/debugfs.h>
+
+#ifdef CONFIG_DEBUG_FS
+
+void ionic_debugfs_create(void);
+void ionic_debugfs_destroy(void);
+void ionic_debugfs_add_dev(struct ionic *ionic);
+void ionic_debugfs_del_dev(struct ionic *ionic);
+void ionic_debugfs_add_ident(struct ionic *ionic);
+void ionic_debugfs_add_sizes(struct ionic *ionic);
+void ionic_debugfs_add_lif(struct ionic_lif *lif);
+void ionic_debugfs_add_qcq(struct ionic_lif *lif, struct ionic_qcq *qcq);
+void ionic_debugfs_del_lif(struct ionic_lif *lif);
+void ionic_debugfs_del_qcq(struct ionic_qcq *qcq);
+#else
+static inline void ionic_debugfs_create(void) { }
+static inline void ionic_debugfs_destroy(void) { }
+static inline void ionic_debugfs_add_dev(struct ionic *ionic) { }
+static inline void ionic_debugfs_del_dev(struct ionic *ionic) { }
+static inline void ionic_debugfs_add_ident(struct ionic *ionic) { }
+static inline void ionic_debugfs_add_sizes(struct ionic *ionic) { }
+static inline void ionic_debugfs_add_lif(struct ionic_lif *lif) { }
+static inline void ionic_debugfs_add_qcq(struct ionic_lif *lif, struct ionic_qcq *qcq) { }
+static inline void ionic_debugfs_del_lif(struct ionic_lif *lif) { }
+static inline void ionic_debugfs_del_qcq(struct ionic_qcq *qcq) { }
+#endif
+
+#endif /* _IONIC_DEBUGFS_H_ */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.c b/drivers/net/ethernet/pensando/ionic/ionic_dev.c
new file mode 100644
index 000000000..344a39246
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.c
@@ -0,0 +1,752 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/etherdevice.h>
+#include "ionic.h"
+#include "ionic_dev.h"
+#include "ionic_lif.h"
+
+static void ionic_watchdog_cb(struct timer_list *t)
+{
+ struct ionic *ionic = from_timer(ionic, t, watchdog_timer);
+ struct ionic_lif *lif = ionic->lif;
+ struct ionic_deferred_work *work;
+ int hb;
+
+ mod_timer(&ionic->watchdog_timer,
+ round_jiffies(jiffies + ionic->watchdog_period));
+
+ if (!lif)
+ return;
+
+ hb = ionic_heartbeat_check(ionic);
+ dev_dbg(ionic->dev, "%s: hb %d running %d UP %d\n",
+ __func__, hb, netif_running(lif->netdev),
+ test_bit(IONIC_LIF_F_UP, lif->state));
+
+ if (hb >= 0 &&
+ !test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
+
+ if (test_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state) &&
+ !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
+ work = kzalloc(sizeof(*work), GFP_ATOMIC);
+ if (!work) {
+ netdev_err(lif->netdev, "rxmode change dropped\n");
+ return;
+ }
+
+ work->type = IONIC_DW_TYPE_RX_MODE;
+ netdev_dbg(lif->netdev, "deferred: rx_mode\n");
+ ionic_lif_deferred_enqueue(&lif->deferred, work);
+ }
+}
+
+static void ionic_watchdog_init(struct ionic *ionic)
+{
+ struct ionic_dev *idev = &ionic->idev;
+
+ timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0);
+ ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ;
+
+ /* set times to ensure the first check will proceed */
+ atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ);
+ idev->last_hb_time = jiffies - 2 * ionic->watchdog_period;
+ /* init as ready, so no transition if the first check succeeds */
+ idev->last_fw_hb = 0;
+ idev->fw_hb_ready = true;
+ idev->fw_status_ready = true;
+ idev->fw_generation = IONIC_FW_STS_F_GENERATION &
+ ioread8(&idev->dev_info_regs->fw_status);
+}
+
+void ionic_init_devinfo(struct ionic *ionic)
+{
+ struct ionic_dev *idev = &ionic->idev;
+
+ idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type);
+ idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev);
+
+ memcpy_fromio(idev->dev_info.fw_version,
+ idev->dev_info_regs->fw_version,
+ IONIC_DEVINFO_FWVERS_BUFLEN);
+
+ memcpy_fromio(idev->dev_info.serial_num,
+ idev->dev_info_regs->serial_num,
+ IONIC_DEVINFO_SERIAL_BUFLEN);
+
+ idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0;
+ idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0;
+
+ dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version);
+}
+
+int ionic_dev_setup(struct ionic *ionic)
+{
+ struct ionic_dev_bar *bar = ionic->bars;
+ unsigned int num_bars = ionic->num_bars;
+ struct ionic_dev *idev = &ionic->idev;
+ struct device *dev = ionic->dev;
+ u32 sig;
+
+ /* BAR0: dev_cmd and interrupts */
+ if (num_bars < 1) {
+ dev_err(dev, "No bars found, aborting\n");
+ return -EFAULT;
+ }
+
+ if (bar->len < IONIC_BAR0_SIZE) {
+ dev_err(dev, "Resource bar size %lu too small, aborting\n",
+ bar->len);
+ return -EFAULT;
+ }
+
+ idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET;
+ idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET;
+ idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET;
+ idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET;
+
+ idev->hwstamp_regs = &idev->dev_info_regs->hwstamp;
+
+ sig = ioread32(&idev->dev_info_regs->signature);
+ if (sig != IONIC_DEV_INFO_SIGNATURE) {
+ dev_err(dev, "Incompatible firmware signature %x", sig);
+ return -EFAULT;
+ }
+
+ ionic_init_devinfo(ionic);
+
+ /* BAR1: doorbells */
+ bar++;
+ if (num_bars < 2) {
+ dev_err(dev, "Doorbell bar missing, aborting\n");
+ return -EFAULT;
+ }
+
+ ionic_watchdog_init(ionic);
+
+ idev->db_pages = bar->vaddr;
+ idev->phy_db_pages = bar->bus_addr;
+
+ return 0;
+}
+
+/* Devcmd Interface */
+bool ionic_is_fw_running(struct ionic_dev *idev)
+{
+ u8 fw_status = ioread8(&idev->dev_info_regs->fw_status);
+
+ /* firmware is useful only if the running bit is set and
+ * fw_status != 0xff (bad PCI read)
+ */
+ return (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING);
+}
+
+int ionic_heartbeat_check(struct ionic *ionic)
+{
+ unsigned long check_time, last_check_time;
+ struct ionic_dev *idev = &ionic->idev;
+ struct ionic_lif *lif = ionic->lif;
+ bool fw_status_ready = true;
+ bool fw_hb_ready;
+ u8 fw_generation;
+ u8 fw_status;
+ u32 fw_hb;
+
+ /* wait a least one second before testing again */
+ check_time = jiffies;
+ last_check_time = atomic_long_read(&idev->last_check_time);
+do_check_time:
+ if (time_before(check_time, last_check_time + HZ))
+ return 0;
+ if (!atomic_long_try_cmpxchg_relaxed(&idev->last_check_time,
+ &last_check_time, check_time)) {
+ /* if called concurrently, only the first should proceed. */
+ dev_dbg(ionic->dev, "%s: do_check_time again\n", __func__);
+ goto do_check_time;
+ }
+
+ fw_status = ioread8(&idev->dev_info_regs->fw_status);
+
+ /* If fw_status is not ready don't bother with the generation */
+ if (!ionic_is_fw_running(idev)) {
+ fw_status_ready = false;
+ } else {
+ fw_generation = fw_status & IONIC_FW_STS_F_GENERATION;
+ if (idev->fw_generation != fw_generation) {
+ dev_info(ionic->dev, "FW generation 0x%02x -> 0x%02x\n",
+ idev->fw_generation, fw_generation);
+
+ idev->fw_generation = fw_generation;
+
+ /* If the generation changed, the fw status is not
+ * ready so we need to trigger a fw-down cycle. After
+ * the down, the next watchdog will see the fw is up
+ * and the generation value stable, so will trigger
+ * the fw-up activity.
+ *
+ * If we had already moved to FW_RESET from a RESET event,
+ * it is possible that we never saw the fw_status go to 0,
+ * so we fake the current idev->fw_status_ready here to
+ * force the transition and get FW up again.
+ */
+ if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ idev->fw_status_ready = false; /* go to running */
+ else
+ fw_status_ready = false; /* go to down */
+ }
+ }
+
+ dev_dbg(ionic->dev, "fw_status 0x%02x ready %d idev->ready %d last_hb 0x%x state 0x%02lx\n",
+ fw_status, fw_status_ready, idev->fw_status_ready,
+ idev->last_fw_hb, lif->state[0]);
+
+ /* is this a transition? */
+ if (fw_status_ready != idev->fw_status_ready &&
+ !test_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
+ bool trigger = false;
+
+ idev->fw_status_ready = fw_status_ready;
+
+ if (!fw_status_ready &&
+ !test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
+ !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
+ dev_info(ionic->dev, "FW stopped 0x%02x\n", fw_status);
+ trigger = true;
+
+ } else if (fw_status_ready &&
+ test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
+ dev_info(ionic->dev, "FW running 0x%02x\n", fw_status);
+ trigger = true;
+ }
+
+ if (trigger) {
+ struct ionic_deferred_work *work;
+
+ work = kzalloc(sizeof(*work), GFP_ATOMIC);
+ if (work) {
+ work->type = IONIC_DW_TYPE_LIF_RESET;
+ work->fw_status = fw_status_ready;
+ ionic_lif_deferred_enqueue(&lif->deferred, work);
+ }
+ }
+ }
+
+ if (!idev->fw_status_ready)
+ return -ENXIO;
+
+ /* Because of some variability in the actual FW heartbeat, we
+ * wait longer than the DEVCMD_TIMEOUT before checking again.
+ */
+ last_check_time = idev->last_hb_time;
+ if (time_before(check_time, last_check_time + DEVCMD_TIMEOUT * 2 * HZ))
+ return 0;
+
+ fw_hb = ioread32(&idev->dev_info_regs->fw_heartbeat);
+ fw_hb_ready = fw_hb != idev->last_fw_hb;
+
+ /* early FW version had no heartbeat, so fake it */
+ if (!fw_hb_ready && !fw_hb)
+ fw_hb_ready = true;
+
+ dev_dbg(ionic->dev, "%s: fw_hb %u last_fw_hb %u ready %u\n",
+ __func__, fw_hb, idev->last_fw_hb, fw_hb_ready);
+
+ idev->last_fw_hb = fw_hb;
+
+ /* log a transition */
+ if (fw_hb_ready != idev->fw_hb_ready) {
+ idev->fw_hb_ready = fw_hb_ready;
+ if (!fw_hb_ready)
+ dev_info(ionic->dev, "FW heartbeat stalled at %d\n", fw_hb);
+ else
+ dev_info(ionic->dev, "FW heartbeat restored at %d\n", fw_hb);
+ }
+
+ if (!fw_hb_ready)
+ return -ENXIO;
+
+ idev->last_hb_time = check_time;
+
+ return 0;
+}
+
+u8 ionic_dev_cmd_status(struct ionic_dev *idev)
+{
+ return ioread8(&idev->dev_cmd_regs->comp.comp.status);
+}
+
+bool ionic_dev_cmd_done(struct ionic_dev *idev)
+{
+ return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE;
+}
+
+void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp)
+{
+ memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp));
+}
+
+void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd)
+{
+ memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd));
+ iowrite32(0, &idev->dev_cmd_regs->done);
+ iowrite32(1, &idev->dev_cmd_regs->doorbell);
+}
+
+/* Device commands */
+void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver)
+{
+ union ionic_dev_cmd cmd = {
+ .identify.opcode = IONIC_CMD_IDENTIFY,
+ .identify.ver = ver,
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_init(struct ionic_dev *idev)
+{
+ union ionic_dev_cmd cmd = {
+ .init.opcode = IONIC_CMD_INIT,
+ .init.type = 0,
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_reset(struct ionic_dev *idev)
+{
+ union ionic_dev_cmd cmd = {
+ .reset.opcode = IONIC_CMD_RESET,
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+/* Port commands */
+void ionic_dev_cmd_port_identify(struct ionic_dev *idev)
+{
+ union ionic_dev_cmd cmd = {
+ .port_init.opcode = IONIC_CMD_PORT_IDENTIFY,
+ .port_init.index = 0,
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_port_init(struct ionic_dev *idev)
+{
+ union ionic_dev_cmd cmd = {
+ .port_init.opcode = IONIC_CMD_PORT_INIT,
+ .port_init.index = 0,
+ .port_init.info_pa = cpu_to_le64(idev->port_info_pa),
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_port_reset(struct ionic_dev *idev)
+{
+ union ionic_dev_cmd cmd = {
+ .port_reset.opcode = IONIC_CMD_PORT_RESET,
+ .port_reset.index = 0,
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state)
+{
+ union ionic_dev_cmd cmd = {
+ .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
+ .port_setattr.index = 0,
+ .port_setattr.attr = IONIC_PORT_ATTR_STATE,
+ .port_setattr.state = state,
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed)
+{
+ union ionic_dev_cmd cmd = {
+ .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
+ .port_setattr.index = 0,
+ .port_setattr.attr = IONIC_PORT_ATTR_SPEED,
+ .port_setattr.speed = cpu_to_le32(speed),
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable)
+{
+ union ionic_dev_cmd cmd = {
+ .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
+ .port_setattr.index = 0,
+ .port_setattr.attr = IONIC_PORT_ATTR_AUTONEG,
+ .port_setattr.an_enable = an_enable,
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type)
+{
+ union ionic_dev_cmd cmd = {
+ .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
+ .port_setattr.index = 0,
+ .port_setattr.attr = IONIC_PORT_ATTR_FEC,
+ .port_setattr.fec_type = fec_type,
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type)
+{
+ union ionic_dev_cmd cmd = {
+ .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
+ .port_setattr.index = 0,
+ .port_setattr.attr = IONIC_PORT_ATTR_PAUSE,
+ .port_setattr.pause_type = pause_type,
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+/* VF commands */
+int ionic_set_vf_config(struct ionic *ionic, int vf,
+ struct ionic_vf_setattr_cmd *vfc)
+{
+ union ionic_dev_cmd cmd = {
+ .vf_setattr.opcode = IONIC_CMD_VF_SETATTR,
+ .vf_setattr.attr = vfc->attr,
+ .vf_setattr.vf_index = cpu_to_le16(vf),
+ };
+ int err;
+
+ memcpy(cmd.vf_setattr.pad, vfc->pad, sizeof(vfc->pad));
+
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_go(&ionic->idev, &cmd);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&ionic->dev_cmd_lock);
+
+ return err;
+}
+
+int ionic_dev_cmd_vf_getattr(struct ionic *ionic, int vf, u8 attr,
+ struct ionic_vf_getattr_comp *comp)
+{
+ union ionic_dev_cmd cmd = {
+ .vf_getattr.opcode = IONIC_CMD_VF_GETATTR,
+ .vf_getattr.attr = attr,
+ .vf_getattr.vf_index = cpu_to_le16(vf),
+ };
+ int err;
+
+ if (vf >= ionic->num_vfs)
+ return -EINVAL;
+
+ switch (attr) {
+ case IONIC_VF_ATTR_SPOOFCHK:
+ case IONIC_VF_ATTR_TRUST:
+ case IONIC_VF_ATTR_LINKSTATE:
+ case IONIC_VF_ATTR_MAC:
+ case IONIC_VF_ATTR_VLAN:
+ case IONIC_VF_ATTR_RATE:
+ break;
+ case IONIC_VF_ATTR_STATSADDR:
+ default:
+ return -EINVAL;
+ }
+
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_go(&ionic->idev, &cmd);
+ err = ionic_dev_cmd_wait_nomsg(ionic, DEVCMD_TIMEOUT);
+ memcpy_fromio(comp, &ionic->idev.dev_cmd_regs->comp.vf_getattr,
+ sizeof(*comp));
+ mutex_unlock(&ionic->dev_cmd_lock);
+
+ if (err && comp->status != IONIC_RC_ENOSUPP)
+ ionic_dev_cmd_dev_err_print(ionic, cmd.vf_getattr.opcode,
+ comp->status, err);
+
+ return err;
+}
+
+/* LIF commands */
+void ionic_dev_cmd_queue_identify(struct ionic_dev *idev,
+ u16 lif_type, u8 qtype, u8 qver)
+{
+ union ionic_dev_cmd cmd = {
+ .q_identify.opcode = IONIC_CMD_Q_IDENTIFY,
+ .q_identify.lif_type = cpu_to_le16(lif_type),
+ .q_identify.type = qtype,
+ .q_identify.ver = qver,
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver)
+{
+ union ionic_dev_cmd cmd = {
+ .lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY,
+ .lif_identify.type = type,
+ .lif_identify.ver = ver,
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index,
+ dma_addr_t info_pa)
+{
+ union ionic_dev_cmd cmd = {
+ .lif_init.opcode = IONIC_CMD_LIF_INIT,
+ .lif_init.index = cpu_to_le16(lif_index),
+ .lif_init.info_pa = cpu_to_le64(info_pa),
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index)
+{
+ union ionic_dev_cmd cmd = {
+ .lif_init.opcode = IONIC_CMD_LIF_RESET,
+ .lif_init.index = cpu_to_le16(lif_index),
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq,
+ u16 lif_index, u16 intr_index)
+{
+ struct ionic_queue *q = &qcq->q;
+ struct ionic_cq *cq = &qcq->cq;
+
+ union ionic_dev_cmd cmd = {
+ .q_init.opcode = IONIC_CMD_Q_INIT,
+ .q_init.lif_index = cpu_to_le16(lif_index),
+ .q_init.type = q->type,
+ .q_init.ver = qcq->q.lif->qtype_info[q->type].version,
+ .q_init.index = cpu_to_le32(q->index),
+ .q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
+ IONIC_QINIT_F_ENA),
+ .q_init.pid = cpu_to_le16(q->pid),
+ .q_init.intr_index = cpu_to_le16(intr_index),
+ .q_init.ring_size = ilog2(q->num_descs),
+ .q_init.ring_base = cpu_to_le64(q->base_pa),
+ .q_init.cq_ring_base = cpu_to_le64(cq->base_pa),
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+int ionic_db_page_num(struct ionic_lif *lif, int pid)
+{
+ return (lif->hw_index * lif->dbid_count) + pid;
+}
+
+int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq,
+ struct ionic_intr_info *intr,
+ unsigned int num_descs, size_t desc_size)
+{
+ unsigned int ring_size;
+
+ if (desc_size == 0 || !is_power_of_2(num_descs))
+ return -EINVAL;
+
+ ring_size = ilog2(num_descs);
+ if (ring_size < 2 || ring_size > 16)
+ return -EINVAL;
+
+ cq->lif = lif;
+ cq->bound_intr = intr;
+ cq->num_descs = num_descs;
+ cq->desc_size = desc_size;
+ cq->tail_idx = 0;
+ cq->done_color = 1;
+
+ return 0;
+}
+
+void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa)
+{
+ struct ionic_cq_info *cur;
+ unsigned int i;
+
+ cq->base = base;
+ cq->base_pa = base_pa;
+
+ for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++)
+ cur->cq_desc = base + (i * cq->desc_size);
+}
+
+void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q)
+{
+ cq->bound_q = q;
+}
+
+unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
+ ionic_cq_cb cb, ionic_cq_done_cb done_cb,
+ void *done_arg)
+{
+ struct ionic_cq_info *cq_info;
+ unsigned int work_done = 0;
+
+ if (work_to_do == 0)
+ return 0;
+
+ cq_info = &cq->info[cq->tail_idx];
+ while (cb(cq, cq_info)) {
+ if (cq->tail_idx == cq->num_descs - 1)
+ cq->done_color = !cq->done_color;
+ cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
+ cq_info = &cq->info[cq->tail_idx];
+
+ if (++work_done >= work_to_do)
+ break;
+ }
+
+ if (work_done && done_cb)
+ done_cb(done_arg);
+
+ return work_done;
+}
+
+int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
+ struct ionic_queue *q, unsigned int index, const char *name,
+ unsigned int num_descs, size_t desc_size,
+ size_t sg_desc_size, unsigned int pid)
+{
+ unsigned int ring_size;
+
+ if (desc_size == 0 || !is_power_of_2(num_descs))
+ return -EINVAL;
+
+ ring_size = ilog2(num_descs);
+ if (ring_size < 2 || ring_size > 16)
+ return -EINVAL;
+
+ q->lif = lif;
+ q->idev = idev;
+ q->index = index;
+ q->num_descs = num_descs;
+ q->desc_size = desc_size;
+ q->sg_desc_size = sg_desc_size;
+ q->tail_idx = 0;
+ q->head_idx = 0;
+ q->pid = pid;
+
+ snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index);
+
+ return 0;
+}
+
+void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
+{
+ struct ionic_desc_info *cur;
+ unsigned int i;
+
+ q->base = base;
+ q->base_pa = base_pa;
+
+ for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
+ cur->desc = base + (i * q->desc_size);
+}
+
+void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
+{
+ struct ionic_desc_info *cur;
+ unsigned int i;
+
+ q->sg_base = base;
+ q->sg_base_pa = base_pa;
+
+ for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
+ cur->sg_desc = base + (i * q->sg_desc_size);
+}
+
+void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
+ void *cb_arg)
+{
+ struct ionic_desc_info *desc_info;
+ struct ionic_lif *lif = q->lif;
+ struct device *dev = q->dev;
+
+ desc_info = &q->info[q->head_idx];
+ desc_info->cb = cb;
+ desc_info->cb_arg = cb_arg;
+
+ q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);
+
+ dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n",
+ q->lif->index, q->name, q->hw_type, q->hw_index,
+ q->head_idx, ring_doorbell);
+
+ if (ring_doorbell) {
+ ionic_dbell_ring(lif->kern_dbpage, q->hw_type,
+ q->dbval | q->head_idx);
+
+ q->dbell_jiffies = jiffies;
+
+ if (q_to_qcq(q)->napi_qcq)
+ mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
+ jiffies + IONIC_NAPI_DEADLINE);
+ }
+}
+
+static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos)
+{
+ unsigned int mask, tail, head;
+
+ mask = q->num_descs - 1;
+ tail = q->tail_idx;
+ head = q->head_idx;
+
+ return ((pos - tail) & mask) < ((head - tail) & mask);
+}
+
+void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
+ unsigned int stop_index)
+{
+ struct ionic_desc_info *desc_info;
+ ionic_desc_cb cb;
+ void *cb_arg;
+ u16 index;
+
+ /* check for empty queue */
+ if (q->tail_idx == q->head_idx)
+ return;
+
+ /* stop index must be for a descriptor that is not yet completed */
+ if (unlikely(!ionic_q_is_posted(q, stop_index)))
+ dev_err(q->dev,
+ "ionic stop is not posted %s stop %u tail %u head %u\n",
+ q->name, stop_index, q->tail_idx, q->head_idx);
+
+ do {
+ desc_info = &q->info[q->tail_idx];
+ index = q->tail_idx;
+ q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
+
+ cb = desc_info->cb;
+ cb_arg = desc_info->cb_arg;
+
+ desc_info->cb = NULL;
+ desc_info->cb_arg = NULL;
+
+ if (cb)
+ cb(q, desc_info, cq_info, cb_arg);
+ } while (index != stop_index);
+}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
new file mode 100644
index 000000000..13dfcf9f7
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
@@ -0,0 +1,374 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#ifndef _IONIC_DEV_H_
+#define _IONIC_DEV_H_
+
+#include <linux/atomic.h>
+#include <linux/mutex.h>
+#include <linux/workqueue.h>
+
+#include "ionic_if.h"
+#include "ionic_regs.h"
+
+#define IONIC_MAX_TX_DESC 8192
+#define IONIC_MAX_RX_DESC 16384
+#define IONIC_MIN_TXRX_DESC 64
+#define IONIC_DEF_TXRX_DESC 4096
+#define IONIC_RX_FILL_THRESHOLD 16
+#define IONIC_RX_FILL_DIV 8
+#define IONIC_LIFS_MAX 1024
+#define IONIC_WATCHDOG_SECS 5
+#define IONIC_ITR_COAL_USEC_DEFAULT 64
+
+#define IONIC_DEV_CMD_REG_VERSION 1
+#define IONIC_DEV_INFO_REG_COUNT 32
+#define IONIC_DEV_CMD_REG_COUNT 32
+
+#define IONIC_NAPI_DEADLINE (HZ / 200) /* 5ms */
+#define IONIC_ADMIN_DOORBELL_DEADLINE (HZ / 2) /* 500ms */
+#define IONIC_TX_DOORBELL_DEADLINE (HZ / 100) /* 10ms */
+#define IONIC_RX_MIN_DOORBELL_DEADLINE (HZ / 100) /* 10ms */
+#define IONIC_RX_MAX_DOORBELL_DEADLINE (HZ * 5) /* 5s */
+
+struct ionic_dev_bar {
+ void __iomem *vaddr;
+ phys_addr_t bus_addr;
+ unsigned long len;
+ int res_index;
+};
+
+#ifndef __CHECKER__
+/* Registers */
+static_assert(sizeof(struct ionic_intr) == 32);
+
+static_assert(sizeof(struct ionic_doorbell) == 8);
+static_assert(sizeof(struct ionic_intr_status) == 8);
+static_assert(sizeof(union ionic_dev_regs) == 4096);
+static_assert(sizeof(union ionic_dev_info_regs) == 2048);
+static_assert(sizeof(union ionic_dev_cmd_regs) == 2048);
+static_assert(sizeof(struct ionic_lif_stats) == 1024);
+
+static_assert(sizeof(struct ionic_admin_cmd) == 64);
+static_assert(sizeof(struct ionic_admin_comp) == 16);
+static_assert(sizeof(struct ionic_nop_cmd) == 64);
+static_assert(sizeof(struct ionic_nop_comp) == 16);
+
+/* Device commands */
+static_assert(sizeof(struct ionic_dev_identify_cmd) == 64);
+static_assert(sizeof(struct ionic_dev_identify_comp) == 16);
+static_assert(sizeof(struct ionic_dev_init_cmd) == 64);
+static_assert(sizeof(struct ionic_dev_init_comp) == 16);
+static_assert(sizeof(struct ionic_dev_reset_cmd) == 64);
+static_assert(sizeof(struct ionic_dev_reset_comp) == 16);
+static_assert(sizeof(struct ionic_dev_getattr_cmd) == 64);
+static_assert(sizeof(struct ionic_dev_getattr_comp) == 16);
+static_assert(sizeof(struct ionic_dev_setattr_cmd) == 64);
+static_assert(sizeof(struct ionic_dev_setattr_comp) == 16);
+static_assert(sizeof(struct ionic_lif_setphc_cmd) == 64);
+
+/* Port commands */
+static_assert(sizeof(struct ionic_port_identify_cmd) == 64);
+static_assert(sizeof(struct ionic_port_identify_comp) == 16);
+static_assert(sizeof(struct ionic_port_init_cmd) == 64);
+static_assert(sizeof(struct ionic_port_init_comp) == 16);
+static_assert(sizeof(struct ionic_port_reset_cmd) == 64);
+static_assert(sizeof(struct ionic_port_reset_comp) == 16);
+static_assert(sizeof(struct ionic_port_getattr_cmd) == 64);
+static_assert(sizeof(struct ionic_port_getattr_comp) == 16);
+static_assert(sizeof(struct ionic_port_setattr_cmd) == 64);
+static_assert(sizeof(struct ionic_port_setattr_comp) == 16);
+
+/* LIF commands */
+static_assert(sizeof(struct ionic_lif_init_cmd) == 64);
+static_assert(sizeof(struct ionic_lif_init_comp) == 16);
+static_assert(sizeof(struct ionic_lif_reset_cmd) == 64);
+static_assert(sizeof(ionic_lif_reset_comp) == 16);
+static_assert(sizeof(struct ionic_lif_getattr_cmd) == 64);
+static_assert(sizeof(struct ionic_lif_getattr_comp) == 16);
+static_assert(sizeof(struct ionic_lif_setattr_cmd) == 64);
+static_assert(sizeof(struct ionic_lif_setattr_comp) == 16);
+
+static_assert(sizeof(struct ionic_q_init_cmd) == 64);
+static_assert(sizeof(struct ionic_q_init_comp) == 16);
+static_assert(sizeof(struct ionic_q_control_cmd) == 64);
+static_assert(sizeof(ionic_q_control_comp) == 16);
+static_assert(sizeof(struct ionic_q_identify_cmd) == 64);
+static_assert(sizeof(struct ionic_q_identify_comp) == 16);
+
+static_assert(sizeof(struct ionic_rx_mode_set_cmd) == 64);
+static_assert(sizeof(ionic_rx_mode_set_comp) == 16);
+static_assert(sizeof(struct ionic_rx_filter_add_cmd) == 64);
+static_assert(sizeof(struct ionic_rx_filter_add_comp) == 16);
+static_assert(sizeof(struct ionic_rx_filter_del_cmd) == 64);
+static_assert(sizeof(ionic_rx_filter_del_comp) == 16);
+
+/* RDMA commands */
+static_assert(sizeof(struct ionic_rdma_reset_cmd) == 64);
+static_assert(sizeof(struct ionic_rdma_queue_cmd) == 64);
+
+/* Events */
+static_assert(sizeof(struct ionic_notifyq_cmd) == 4);
+static_assert(sizeof(union ionic_notifyq_comp) == 64);
+static_assert(sizeof(struct ionic_notifyq_event) == 64);
+static_assert(sizeof(struct ionic_link_change_event) == 64);
+static_assert(sizeof(struct ionic_reset_event) == 64);
+static_assert(sizeof(struct ionic_heartbeat_event) == 64);
+static_assert(sizeof(struct ionic_log_event) == 64);
+
+/* I/O */
+static_assert(sizeof(struct ionic_txq_desc) == 16);
+static_assert(sizeof(struct ionic_txq_sg_desc) == 128);
+static_assert(sizeof(struct ionic_txq_comp) == 16);
+
+static_assert(sizeof(struct ionic_rxq_desc) == 16);
+static_assert(sizeof(struct ionic_rxq_sg_desc) == 128);
+static_assert(sizeof(struct ionic_rxq_comp) == 16);
+
+/* SR/IOV */
+static_assert(sizeof(struct ionic_vf_setattr_cmd) == 64);
+static_assert(sizeof(struct ionic_vf_setattr_comp) == 16);
+static_assert(sizeof(struct ionic_vf_getattr_cmd) == 64);
+static_assert(sizeof(struct ionic_vf_getattr_comp) == 16);
+#endif /* __CHECKER__ */
+
+struct ionic_devinfo {
+ u8 asic_type;
+ u8 asic_rev;
+ char fw_version[IONIC_DEVINFO_FWVERS_BUFLEN + 1];
+ char serial_num[IONIC_DEVINFO_SERIAL_BUFLEN + 1];
+};
+
+struct ionic_dev {
+ union ionic_dev_info_regs __iomem *dev_info_regs;
+ union ionic_dev_cmd_regs __iomem *dev_cmd_regs;
+ struct ionic_hwstamp_regs __iomem *hwstamp_regs;
+
+ atomic_long_t last_check_time;
+ unsigned long last_hb_time;
+ u32 last_fw_hb;
+ bool fw_hb_ready;
+ bool fw_status_ready;
+ u8 fw_generation;
+
+ u64 __iomem *db_pages;
+ dma_addr_t phy_db_pages;
+
+ struct ionic_intr __iomem *intr_ctrl;
+ u64 __iomem *intr_status;
+
+ u32 port_info_sz;
+ struct ionic_port_info *port_info;
+ dma_addr_t port_info_pa;
+
+ struct ionic_devinfo dev_info;
+};
+
+struct ionic_cq_info {
+ union {
+ void *cq_desc;
+ struct ionic_admin_comp *admincq;
+ struct ionic_notifyq_event *notifyq;
+ };
+};
+
+struct ionic_queue;
+struct ionic_qcq;
+struct ionic_desc_info;
+
+typedef void (*ionic_desc_cb)(struct ionic_queue *q,
+ struct ionic_desc_info *desc_info,
+ struct ionic_cq_info *cq_info, void *cb_arg);
+
+#define IONIC_MAX_BUF_LEN ((u16)-1)
+#define IONIC_PAGE_SIZE PAGE_SIZE
+#define IONIC_PAGE_SPLIT_SZ (PAGE_SIZE / 2)
+#define IONIC_PAGE_GFP_MASK (GFP_ATOMIC | __GFP_NOWARN |\
+ __GFP_COMP | __GFP_MEMALLOC)
+
+struct ionic_buf_info {
+ struct page *page;
+ dma_addr_t dma_addr;
+ u32 page_offset;
+ u32 len;
+};
+
+#define IONIC_MAX_FRAGS (1 + IONIC_TX_MAX_SG_ELEMS_V1)
+
+struct ionic_desc_info {
+ union {
+ void *desc;
+ struct ionic_txq_desc *txq_desc;
+ struct ionic_rxq_desc *rxq_desc;
+ struct ionic_admin_cmd *adminq_desc;
+ };
+ union {
+ void *sg_desc;
+ struct ionic_txq_sg_desc *txq_sg_desc;
+ struct ionic_rxq_sg_desc *rxq_sgl_desc;
+ };
+ unsigned int bytes;
+ unsigned int nbufs;
+ struct ionic_buf_info bufs[IONIC_MAX_FRAGS];
+ ionic_desc_cb cb;
+ void *cb_arg;
+};
+
+#define IONIC_QUEUE_NAME_MAX_SZ 16
+
+struct ionic_queue {
+ struct device *dev;
+ struct ionic_lif *lif;
+ struct ionic_desc_info *info;
+ u64 dbval;
+ unsigned long dbell_deadline;
+ unsigned long dbell_jiffies;
+ u16 head_idx;
+ u16 tail_idx;
+ unsigned int index;
+ unsigned int num_descs;
+ unsigned int max_sg_elems;
+ u64 features;
+ u64 drop;
+ struct ionic_dev *idev;
+ unsigned int type;
+ unsigned int hw_index;
+ unsigned int hw_type;
+ union {
+ void *base;
+ struct ionic_txq_desc *txq;
+ struct ionic_rxq_desc *rxq;
+ struct ionic_admin_cmd *adminq;
+ };
+ union {
+ void *sg_base;
+ struct ionic_txq_sg_desc *txq_sgl;
+ struct ionic_rxq_sg_desc *rxq_sgl;
+ };
+ dma_addr_t base_pa;
+ dma_addr_t sg_base_pa;
+ unsigned int desc_size;
+ unsigned int sg_desc_size;
+ unsigned int pid;
+ char name[IONIC_QUEUE_NAME_MAX_SZ];
+} ____cacheline_aligned_in_smp;
+
+#define IONIC_INTR_INDEX_NOT_ASSIGNED -1
+#define IONIC_INTR_NAME_MAX_SZ 32
+
+struct ionic_intr_info {
+ char name[IONIC_INTR_NAME_MAX_SZ];
+ unsigned int index;
+ unsigned int vector;
+ u64 rearm_count;
+ unsigned int cpu;
+ cpumask_t affinity_mask;
+ u32 dim_coal_hw;
+};
+
+struct ionic_cq {
+ struct ionic_lif *lif;
+ struct ionic_cq_info *info;
+ struct ionic_queue *bound_q;
+ struct ionic_intr_info *bound_intr;
+ u16 tail_idx;
+ bool done_color;
+ unsigned int num_descs;
+ unsigned int desc_size;
+ void *base;
+ dma_addr_t base_pa;
+} ____cacheline_aligned_in_smp;
+
+struct ionic;
+
+static inline void ionic_intr_init(struct ionic_dev *idev,
+ struct ionic_intr_info *intr,
+ unsigned long index)
+{
+ ionic_intr_clean(idev->intr_ctrl, index);
+ intr->index = index;
+}
+
+static inline unsigned int ionic_q_space_avail(struct ionic_queue *q)
+{
+ unsigned int avail = q->tail_idx;
+
+ if (q->head_idx >= avail)
+ avail += q->num_descs - q->head_idx - 1;
+ else
+ avail -= q->head_idx + 1;
+
+ return avail;
+}
+
+static inline bool ionic_q_has_space(struct ionic_queue *q, unsigned int want)
+{
+ return ionic_q_space_avail(q) >= want;
+}
+
+void ionic_init_devinfo(struct ionic *ionic);
+int ionic_dev_setup(struct ionic *ionic);
+
+void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd);
+u8 ionic_dev_cmd_status(struct ionic_dev *idev);
+bool ionic_dev_cmd_done(struct ionic_dev *idev);
+void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp);
+
+void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver);
+void ionic_dev_cmd_init(struct ionic_dev *idev);
+void ionic_dev_cmd_reset(struct ionic_dev *idev);
+
+void ionic_dev_cmd_port_identify(struct ionic_dev *idev);
+void ionic_dev_cmd_port_init(struct ionic_dev *idev);
+void ionic_dev_cmd_port_reset(struct ionic_dev *idev);
+void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state);
+void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed);
+void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable);
+void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type);
+void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type);
+
+int ionic_set_vf_config(struct ionic *ionic, int vf,
+ struct ionic_vf_setattr_cmd *vfc);
+int ionic_dev_cmd_vf_getattr(struct ionic *ionic, int vf, u8 attr,
+ struct ionic_vf_getattr_comp *comp);
+void ionic_dev_cmd_queue_identify(struct ionic_dev *idev,
+ u16 lif_type, u8 qtype, u8 qver);
+void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver);
+void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index,
+ dma_addr_t addr);
+void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index);
+void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq,
+ u16 lif_index, u16 intr_index);
+
+int ionic_db_page_num(struct ionic_lif *lif, int pid);
+
+int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq,
+ struct ionic_intr_info *intr,
+ unsigned int num_descs, size_t desc_size);
+void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa);
+void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q);
+typedef bool (*ionic_cq_cb)(struct ionic_cq *cq, struct ionic_cq_info *cq_info);
+typedef void (*ionic_cq_done_cb)(void *done_arg);
+unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
+ ionic_cq_cb cb, ionic_cq_done_cb done_cb,
+ void *done_arg);
+
+int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
+ struct ionic_queue *q, unsigned int index, const char *name,
+ unsigned int num_descs, size_t desc_size,
+ size_t sg_desc_size, unsigned int pid);
+void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa);
+void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa);
+void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
+ void *cb_arg);
+void ionic_q_rewind(struct ionic_queue *q, struct ionic_desc_info *start);
+void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
+ unsigned int stop_index);
+int ionic_heartbeat_check(struct ionic *ionic);
+bool ionic_is_fw_running(struct ionic_dev *idev);
+
+bool ionic_adminq_poke_doorbell(struct ionic_queue *q);
+bool ionic_txq_poke_doorbell(struct ionic_queue *q);
+bool ionic_rxq_poke_doorbell(struct ionic_queue *q);
+
+#endif /* _IONIC_DEV_H_ */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_devlink.c b/drivers/net/ethernet/pensando/ionic/ionic_devlink.c
new file mode 100644
index 000000000..2696dac21
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_devlink.c
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+
+#include "ionic.h"
+#include "ionic_bus.h"
+#include "ionic_lif.h"
+#include "ionic_devlink.h"
+
+static int ionic_dl_flash_update(struct devlink *dl,
+ struct devlink_flash_update_params *params,
+ struct netlink_ext_ack *extack)
+{
+ struct ionic *ionic = devlink_priv(dl);
+
+ return ionic_firmware_update(ionic->lif, params->fw, extack);
+}
+
+static int ionic_dl_info_get(struct devlink *dl, struct devlink_info_req *req,
+ struct netlink_ext_ack *extack)
+{
+ struct ionic *ionic = devlink_priv(dl);
+ struct ionic_dev *idev = &ionic->idev;
+ char buf[16];
+ int err = 0;
+
+ err = devlink_info_driver_name_put(req, IONIC_DRV_NAME);
+ if (err)
+ return err;
+
+ err = devlink_info_version_running_put(req,
+ DEVLINK_INFO_VERSION_GENERIC_FW,
+ idev->dev_info.fw_version);
+ if (err)
+ return err;
+
+ snprintf(buf, sizeof(buf), "0x%x", idev->dev_info.asic_type);
+ err = devlink_info_version_fixed_put(req,
+ DEVLINK_INFO_VERSION_GENERIC_ASIC_ID,
+ buf);
+ if (err)
+ return err;
+
+ snprintf(buf, sizeof(buf), "0x%x", idev->dev_info.asic_rev);
+ err = devlink_info_version_fixed_put(req,
+ DEVLINK_INFO_VERSION_GENERIC_ASIC_REV,
+ buf);
+ if (err)
+ return err;
+
+ err = devlink_info_serial_number_put(req, idev->dev_info.serial_num);
+
+ return err;
+}
+
+static const struct devlink_ops ionic_dl_ops = {
+ .info_get = ionic_dl_info_get,
+ .flash_update = ionic_dl_flash_update,
+};
+
+struct ionic *ionic_devlink_alloc(struct device *dev)
+{
+ struct devlink *dl;
+
+ dl = devlink_alloc(&ionic_dl_ops, sizeof(struct ionic), dev);
+ if (!dl)
+ return NULL;
+
+ return devlink_priv(dl);
+}
+
+void ionic_devlink_free(struct ionic *ionic)
+{
+ struct devlink *dl = priv_to_devlink(ionic);
+
+ devlink_free(dl);
+}
+
+int ionic_devlink_register(struct ionic *ionic)
+{
+ struct devlink *dl = priv_to_devlink(ionic);
+ struct devlink_port_attrs attrs = {};
+ int err;
+
+ attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
+ devlink_port_attrs_set(&ionic->dl_port, &attrs);
+ err = devlink_port_register(dl, &ionic->dl_port, 0);
+ if (err) {
+ dev_err(ionic->dev, "devlink_port_register failed: %d\n", err);
+ return err;
+ }
+
+ devlink_port_type_eth_set(&ionic->dl_port, ionic->lif->netdev);
+ devlink_register(dl);
+ return 0;
+}
+
+void ionic_devlink_unregister(struct ionic *ionic)
+{
+ struct devlink *dl = priv_to_devlink(ionic);
+
+ devlink_unregister(dl);
+ devlink_port_unregister(&ionic->dl_port);
+}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_devlink.h b/drivers/net/ethernet/pensando/ionic/ionic_devlink.h
new file mode 100644
index 000000000..0a77e8e81
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_devlink.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#ifndef _IONIC_DEVLINK_H_
+#define _IONIC_DEVLINK_H_
+
+#include <net/devlink.h>
+
+int ionic_firmware_update(struct ionic_lif *lif, const struct firmware *fw,
+ struct netlink_ext_ack *extack);
+
+struct ionic *ionic_devlink_alloc(struct device *dev);
+void ionic_devlink_free(struct ionic *ionic);
+int ionic_devlink_register(struct ionic *ionic);
+void ionic_devlink_unregister(struct ionic *ionic);
+
+#endif /* _IONIC_DEVLINK_H_ */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
new file mode 100644
index 000000000..d7370fb60
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
@@ -0,0 +1,1003 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/sfp.h>
+
+#include "ionic.h"
+#include "ionic_bus.h"
+#include "ionic_lif.h"
+#include "ionic_ethtool.h"
+#include "ionic_stats.h"
+
+static void ionic_get_stats_strings(struct ionic_lif *lif, u8 *buf)
+{
+ u32 i;
+
+ for (i = 0; i < ionic_num_stats_grps; i++)
+ ionic_stats_groups[i].get_strings(lif, &buf);
+}
+
+static void ionic_get_stats(struct net_device *netdev,
+ struct ethtool_stats *stats, u64 *buf)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ u32 i;
+
+ if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return;
+
+ memset(buf, 0, stats->n_stats * sizeof(*buf));
+ for (i = 0; i < ionic_num_stats_grps; i++)
+ ionic_stats_groups[i].get_values(lif, &buf);
+}
+
+static int ionic_get_stats_count(struct ionic_lif *lif)
+{
+ int i, num_stats = 0;
+
+ for (i = 0; i < ionic_num_stats_grps; i++)
+ num_stats += ionic_stats_groups[i].get_count(lif);
+
+ return num_stats;
+}
+
+static int ionic_get_sset_count(struct net_device *netdev, int sset)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ int count = 0;
+
+ switch (sset) {
+ case ETH_SS_STATS:
+ count = ionic_get_stats_count(lif);
+ break;
+ }
+ return count;
+}
+
+static void ionic_get_strings(struct net_device *netdev,
+ u32 sset, u8 *buf)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+
+ switch (sset) {
+ case ETH_SS_STATS:
+ ionic_get_stats_strings(lif, buf);
+ break;
+ }
+}
+
+static void ionic_get_drvinfo(struct net_device *netdev,
+ struct ethtool_drvinfo *drvinfo)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+
+ strscpy(drvinfo->driver, IONIC_DRV_NAME, sizeof(drvinfo->driver));
+ strscpy(drvinfo->fw_version, ionic->idev.dev_info.fw_version,
+ sizeof(drvinfo->fw_version));
+ strscpy(drvinfo->bus_info, ionic_bus_info(ionic),
+ sizeof(drvinfo->bus_info));
+}
+
+static int ionic_get_regs_len(struct net_device *netdev)
+{
+ return (IONIC_DEV_INFO_REG_COUNT + IONIC_DEV_CMD_REG_COUNT) * sizeof(u32);
+}
+
+static void ionic_get_regs(struct net_device *netdev, struct ethtool_regs *regs,
+ void *p)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ unsigned int offset;
+ unsigned int size;
+
+ regs->version = IONIC_DEV_CMD_REG_VERSION;
+
+ offset = 0;
+ size = IONIC_DEV_INFO_REG_COUNT * sizeof(u32);
+ memcpy_fromio(p + offset, lif->ionic->idev.dev_info_regs->words, size);
+
+ offset += size;
+ size = IONIC_DEV_CMD_REG_COUNT * sizeof(u32);
+ memcpy_fromio(p + offset, lif->ionic->idev.dev_cmd_regs->words, size);
+}
+
+static int ionic_get_link_ksettings(struct net_device *netdev,
+ struct ethtool_link_ksettings *ks)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_dev *idev = &lif->ionic->idev;
+ int copper_seen = 0;
+
+ ethtool_link_ksettings_zero_link_mode(ks, supported);
+
+ if (!idev->port_info) {
+ netdev_err(netdev, "port_info not initialized\n");
+ return -EOPNOTSUPP;
+ }
+
+ /* The port_info data is found in a DMA space that the NIC keeps
+ * up-to-date, so there's no need to request the data from the
+ * NIC, we already have it in our memory space.
+ */
+
+ switch (le16_to_cpu(idev->port_info->status.xcvr.pid)) {
+ /* Copper */
+ case IONIC_XCVR_PID_QSFP_100G_CR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 100000baseCR4_Full);
+ copper_seen++;
+ break;
+ case IONIC_XCVR_PID_QSFP_40GBASE_CR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 40000baseCR4_Full);
+ copper_seen++;
+ break;
+ case IONIC_XCVR_PID_SFP_25GBASE_CR_S:
+ case IONIC_XCVR_PID_SFP_25GBASE_CR_L:
+ case IONIC_XCVR_PID_SFP_25GBASE_CR_N:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 25000baseCR_Full);
+ copper_seen++;
+ break;
+ case IONIC_XCVR_PID_SFP_10GBASE_AOC:
+ case IONIC_XCVR_PID_SFP_10GBASE_CU:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 10000baseCR_Full);
+ copper_seen++;
+ break;
+
+ /* Fibre */
+ case IONIC_XCVR_PID_QSFP_100G_SR4:
+ case IONIC_XCVR_PID_QSFP_100G_AOC:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 100000baseSR4_Full);
+ break;
+ case IONIC_XCVR_PID_QSFP_100G_CWDM4:
+ case IONIC_XCVR_PID_QSFP_100G_PSM4:
+ case IONIC_XCVR_PID_QSFP_100G_LR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 100000baseLR4_ER4_Full);
+ break;
+ case IONIC_XCVR_PID_QSFP_100G_ER4:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 100000baseLR4_ER4_Full);
+ break;
+ case IONIC_XCVR_PID_QSFP_40GBASE_SR4:
+ case IONIC_XCVR_PID_QSFP_40GBASE_AOC:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 40000baseSR4_Full);
+ break;
+ case IONIC_XCVR_PID_QSFP_40GBASE_LR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 40000baseLR4_Full);
+ break;
+ case IONIC_XCVR_PID_SFP_25GBASE_SR:
+ case IONIC_XCVR_PID_SFP_25GBASE_AOC:
+ case IONIC_XCVR_PID_SFP_25GBASE_ACC:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 25000baseSR_Full);
+ break;
+ case IONIC_XCVR_PID_SFP_10GBASE_SR:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 10000baseSR_Full);
+ break;
+ case IONIC_XCVR_PID_SFP_10GBASE_LR:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 10000baseLR_Full);
+ break;
+ case IONIC_XCVR_PID_SFP_10GBASE_LRM:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 10000baseLRM_Full);
+ break;
+ case IONIC_XCVR_PID_SFP_10GBASE_ER:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 10000baseER_Full);
+ break;
+ case IONIC_XCVR_PID_SFP_10GBASE_T:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 10000baseT_Full);
+ break;
+ case IONIC_XCVR_PID_SFP_1000BASE_T:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 1000baseT_Full);
+ break;
+ case IONIC_XCVR_PID_UNKNOWN:
+ /* This means there's no module plugged in */
+ break;
+ default:
+ dev_info(lif->ionic->dev, "unknown xcvr type pid=%d / 0x%x\n",
+ idev->port_info->status.xcvr.pid,
+ idev->port_info->status.xcvr.pid);
+ break;
+ }
+
+ linkmode_copy(ks->link_modes.advertising, ks->link_modes.supported);
+
+ ethtool_link_ksettings_add_link_mode(ks, supported, FEC_BASER);
+ ethtool_link_ksettings_add_link_mode(ks, supported, FEC_RS);
+ if (idev->port_info->config.fec_type == IONIC_PORT_FEC_TYPE_FC)
+ ethtool_link_ksettings_add_link_mode(ks, advertising, FEC_BASER);
+ else if (idev->port_info->config.fec_type == IONIC_PORT_FEC_TYPE_RS)
+ ethtool_link_ksettings_add_link_mode(ks, advertising, FEC_RS);
+
+ ethtool_link_ksettings_add_link_mode(ks, supported, FIBRE);
+ ethtool_link_ksettings_add_link_mode(ks, supported, Pause);
+
+ if (idev->port_info->status.xcvr.phy == IONIC_PHY_TYPE_COPPER ||
+ copper_seen)
+ ks->base.port = PORT_DA;
+ else if (idev->port_info->status.xcvr.phy == IONIC_PHY_TYPE_FIBER)
+ ks->base.port = PORT_FIBRE;
+ else
+ ks->base.port = PORT_NONE;
+
+ if (ks->base.port != PORT_NONE) {
+ ks->base.speed = le32_to_cpu(lif->info->status.link_speed);
+
+ if (le16_to_cpu(lif->info->status.link_status))
+ ks->base.duplex = DUPLEX_FULL;
+ else
+ ks->base.duplex = DUPLEX_UNKNOWN;
+
+ ethtool_link_ksettings_add_link_mode(ks, supported, Autoneg);
+
+ if (idev->port_info->config.an_enable) {
+ ethtool_link_ksettings_add_link_mode(ks, advertising,
+ Autoneg);
+ ks->base.autoneg = AUTONEG_ENABLE;
+ }
+ }
+
+ return 0;
+}
+
+static int ionic_set_link_ksettings(struct net_device *netdev,
+ const struct ethtool_link_ksettings *ks)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_dev *idev = &lif->ionic->idev;
+ struct ionic *ionic = lif->ionic;
+ int err = 0;
+
+ if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return -EBUSY;
+
+ /* set autoneg */
+ if (ks->base.autoneg != idev->port_info->config.an_enable) {
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_port_autoneg(idev, ks->base.autoneg);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&ionic->dev_cmd_lock);
+ if (err)
+ return err;
+ }
+
+ /* set speed */
+ if (ks->base.speed != le32_to_cpu(idev->port_info->config.speed)) {
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_port_speed(idev, ks->base.speed);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&ionic->dev_cmd_lock);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
+static void ionic_get_pauseparam(struct net_device *netdev,
+ struct ethtool_pauseparam *pause)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ u8 pause_type;
+
+ pause->autoneg = 0;
+
+ pause_type = lif->ionic->idev.port_info->config.pause_type;
+ if (pause_type) {
+ pause->rx_pause = (pause_type & IONIC_PAUSE_F_RX) ? 1 : 0;
+ pause->tx_pause = (pause_type & IONIC_PAUSE_F_TX) ? 1 : 0;
+ }
+}
+
+static int ionic_set_pauseparam(struct net_device *netdev,
+ struct ethtool_pauseparam *pause)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+ u32 requested_pause;
+ int err;
+
+ if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return -EBUSY;
+
+ if (pause->autoneg)
+ return -EOPNOTSUPP;
+
+ /* change both at the same time */
+ requested_pause = IONIC_PORT_PAUSE_TYPE_LINK;
+ if (pause->rx_pause)
+ requested_pause |= IONIC_PAUSE_F_RX;
+ if (pause->tx_pause)
+ requested_pause |= IONIC_PAUSE_F_TX;
+
+ if (requested_pause == lif->ionic->idev.port_info->config.pause_type)
+ return 0;
+
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_port_pause(&lif->ionic->idev, requested_pause);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&ionic->dev_cmd_lock);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+static int ionic_get_fecparam(struct net_device *netdev,
+ struct ethtool_fecparam *fec)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+
+ switch (lif->ionic->idev.port_info->config.fec_type) {
+ case IONIC_PORT_FEC_TYPE_NONE:
+ fec->active_fec = ETHTOOL_FEC_OFF;
+ break;
+ case IONIC_PORT_FEC_TYPE_RS:
+ fec->active_fec = ETHTOOL_FEC_RS;
+ break;
+ case IONIC_PORT_FEC_TYPE_FC:
+ fec->active_fec = ETHTOOL_FEC_BASER;
+ break;
+ }
+
+ fec->fec = ETHTOOL_FEC_OFF | ETHTOOL_FEC_RS | ETHTOOL_FEC_BASER;
+
+ return 0;
+}
+
+static int ionic_set_fecparam(struct net_device *netdev,
+ struct ethtool_fecparam *fec)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ u8 fec_type;
+ int ret = 0;
+
+ if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return -EBUSY;
+
+ if (lif->ionic->idev.port_info->config.an_enable) {
+ netdev_err(netdev, "FEC request not allowed while autoneg is enabled\n");
+ return -EINVAL;
+ }
+
+ switch (fec->fec) {
+ case ETHTOOL_FEC_NONE:
+ fec_type = IONIC_PORT_FEC_TYPE_NONE;
+ break;
+ case ETHTOOL_FEC_OFF:
+ fec_type = IONIC_PORT_FEC_TYPE_NONE;
+ break;
+ case ETHTOOL_FEC_RS:
+ fec_type = IONIC_PORT_FEC_TYPE_RS;
+ break;
+ case ETHTOOL_FEC_BASER:
+ fec_type = IONIC_PORT_FEC_TYPE_FC;
+ break;
+ case ETHTOOL_FEC_AUTO:
+ default:
+ netdev_err(netdev, "FEC request 0x%04x not supported\n",
+ fec->fec);
+ return -EINVAL;
+ }
+
+ if (fec_type != lif->ionic->idev.port_info->config.fec_type) {
+ mutex_lock(&lif->ionic->dev_cmd_lock);
+ ionic_dev_cmd_port_fec(&lif->ionic->idev, fec_type);
+ ret = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&lif->ionic->dev_cmd_lock);
+ }
+
+ return ret;
+}
+
+static int ionic_get_coalesce(struct net_device *netdev,
+ struct ethtool_coalesce *coalesce,
+ struct kernel_ethtool_coalesce *kernel_coal,
+ struct netlink_ext_ack *extack)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+
+ coalesce->tx_coalesce_usecs = lif->tx_coalesce_usecs;
+ coalesce->rx_coalesce_usecs = lif->rx_coalesce_usecs;
+
+ if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
+ coalesce->use_adaptive_tx_coalesce = test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
+ else
+ coalesce->use_adaptive_tx_coalesce = 0;
+
+ coalesce->use_adaptive_rx_coalesce = test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
+
+ return 0;
+}
+
+static int ionic_set_coalesce(struct net_device *netdev,
+ struct ethtool_coalesce *coalesce,
+ struct kernel_ethtool_coalesce *kernel_coal,
+ struct netlink_ext_ack *extack)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_identity *ident;
+ u32 rx_coal, rx_dim;
+ u32 tx_coal, tx_dim;
+ unsigned int i;
+
+ ident = &lif->ionic->ident;
+ if (ident->dev.intr_coal_div == 0) {
+ netdev_warn(netdev, "bad HW value in dev.intr_coal_div = %d\n",
+ ident->dev.intr_coal_div);
+ return -EIO;
+ }
+
+ /* Tx normally shares Rx interrupt, so only change Rx if not split */
+ if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) &&
+ (coalesce->tx_coalesce_usecs != lif->rx_coalesce_usecs ||
+ coalesce->use_adaptive_tx_coalesce)) {
+ netdev_warn(netdev, "only rx parameters can be changed\n");
+ return -EINVAL;
+ }
+
+ /* Convert the usec request to a HW usable value. If they asked
+ * for non-zero and it resolved to zero, bump it up
+ */
+ rx_coal = ionic_coal_usec_to_hw(lif->ionic, coalesce->rx_coalesce_usecs);
+ if (!rx_coal && coalesce->rx_coalesce_usecs)
+ rx_coal = 1;
+ tx_coal = ionic_coal_usec_to_hw(lif->ionic, coalesce->tx_coalesce_usecs);
+ if (!tx_coal && coalesce->tx_coalesce_usecs)
+ tx_coal = 1;
+
+ if (rx_coal > IONIC_INTR_CTRL_COAL_MAX ||
+ tx_coal > IONIC_INTR_CTRL_COAL_MAX)
+ return -ERANGE;
+
+ /* Save the new values */
+ lif->rx_coalesce_usecs = coalesce->rx_coalesce_usecs;
+ lif->rx_coalesce_hw = rx_coal;
+
+ if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
+ lif->tx_coalesce_usecs = coalesce->tx_coalesce_usecs;
+ else
+ lif->tx_coalesce_usecs = coalesce->rx_coalesce_usecs;
+ lif->tx_coalesce_hw = tx_coal;
+
+ if (coalesce->use_adaptive_rx_coalesce) {
+ set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
+ rx_dim = rx_coal;
+ } else {
+ clear_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
+ rx_dim = 0;
+ }
+
+ if (coalesce->use_adaptive_tx_coalesce) {
+ set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
+ tx_dim = tx_coal;
+ } else {
+ clear_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
+ tx_dim = 0;
+ }
+
+ if (test_bit(IONIC_LIF_F_UP, lif->state)) {
+ for (i = 0; i < lif->nxqs; i++) {
+ if (lif->rxqcqs[i]->flags & IONIC_QCQ_F_INTR) {
+ ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
+ lif->rxqcqs[i]->intr.index,
+ lif->rx_coalesce_hw);
+ lif->rxqcqs[i]->intr.dim_coal_hw = rx_dim;
+ }
+
+ if (lif->txqcqs[i]->flags & IONIC_QCQ_F_INTR) {
+ ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
+ lif->txqcqs[i]->intr.index,
+ lif->tx_coalesce_hw);
+ lif->txqcqs[i]->intr.dim_coal_hw = tx_dim;
+ }
+ }
+ }
+
+ return 0;
+}
+
+static void ionic_get_ringparam(struct net_device *netdev,
+ struct ethtool_ringparam *ring,
+ struct kernel_ethtool_ringparam *kernel_ring,
+ struct netlink_ext_ack *extack)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+
+ ring->tx_max_pending = IONIC_MAX_TX_DESC;
+ ring->tx_pending = lif->ntxq_descs;
+ ring->rx_max_pending = IONIC_MAX_RX_DESC;
+ ring->rx_pending = lif->nrxq_descs;
+}
+
+static int ionic_set_ringparam(struct net_device *netdev,
+ struct ethtool_ringparam *ring,
+ struct kernel_ethtool_ringparam *kernel_ring,
+ struct netlink_ext_ack *extack)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_queue_params qparam;
+ int err;
+
+ if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return -EBUSY;
+
+ ionic_init_queue_params(lif, &qparam);
+
+ if (ring->rx_mini_pending || ring->rx_jumbo_pending) {
+ netdev_info(netdev, "Changing jumbo or mini descriptors not supported\n");
+ return -EINVAL;
+ }
+
+ if (!is_power_of_2(ring->tx_pending) ||
+ !is_power_of_2(ring->rx_pending)) {
+ netdev_info(netdev, "Descriptor count must be a power of 2\n");
+ return -EINVAL;
+ }
+
+ /* if nothing to do return success */
+ if (ring->tx_pending == lif->ntxq_descs &&
+ ring->rx_pending == lif->nrxq_descs)
+ return 0;
+
+ if (ring->tx_pending != lif->ntxq_descs)
+ netdev_info(netdev, "Changing Tx ring size from %d to %d\n",
+ lif->ntxq_descs, ring->tx_pending);
+
+ if (ring->rx_pending != lif->nrxq_descs)
+ netdev_info(netdev, "Changing Rx ring size from %d to %d\n",
+ lif->nrxq_descs, ring->rx_pending);
+
+ /* if we're not running, just set the values and return */
+ if (!netif_running(lif->netdev)) {
+ lif->ntxq_descs = ring->tx_pending;
+ lif->nrxq_descs = ring->rx_pending;
+ return 0;
+ }
+
+ qparam.ntxq_descs = ring->tx_pending;
+ qparam.nrxq_descs = ring->rx_pending;
+
+ mutex_lock(&lif->queue_lock);
+ err = ionic_reconfigure_queues(lif, &qparam);
+ mutex_unlock(&lif->queue_lock);
+ if (err)
+ netdev_info(netdev, "Ring reconfiguration failed, changes canceled: %d\n", err);
+
+ return err;
+}
+
+static void ionic_get_channels(struct net_device *netdev,
+ struct ethtool_channels *ch)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+
+ /* report maximum channels */
+ ch->max_combined = lif->ionic->ntxqs_per_lif;
+ ch->max_rx = lif->ionic->ntxqs_per_lif / 2;
+ ch->max_tx = lif->ionic->ntxqs_per_lif / 2;
+
+ /* report current channels */
+ if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {
+ ch->rx_count = lif->nxqs;
+ ch->tx_count = lif->nxqs;
+ } else {
+ ch->combined_count = lif->nxqs;
+ }
+}
+
+static int ionic_set_channels(struct net_device *netdev,
+ struct ethtool_channels *ch)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_queue_params qparam;
+ int max_cnt;
+ int err;
+
+ if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return -EBUSY;
+
+ ionic_init_queue_params(lif, &qparam);
+
+ if (ch->rx_count != ch->tx_count) {
+ netdev_info(netdev, "The rx and tx count must be equal\n");
+ return -EINVAL;
+ }
+
+ if (ch->combined_count && ch->rx_count) {
+ netdev_info(netdev, "Use either combined or rx and tx, not both\n");
+ return -EINVAL;
+ }
+
+ max_cnt = lif->ionic->ntxqs_per_lif;
+ if (ch->combined_count) {
+ if (ch->combined_count > max_cnt)
+ return -EINVAL;
+
+ if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
+ netdev_info(lif->netdev, "Sharing queue interrupts\n");
+ else if (ch->combined_count == lif->nxqs)
+ return 0;
+
+ if (lif->nxqs != ch->combined_count)
+ netdev_info(netdev, "Changing queue count from %d to %d\n",
+ lif->nxqs, ch->combined_count);
+
+ qparam.nxqs = ch->combined_count;
+ qparam.intr_split = 0;
+ } else {
+ max_cnt /= 2;
+ if (ch->rx_count > max_cnt)
+ return -EINVAL;
+
+ if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
+ netdev_info(lif->netdev, "Splitting queue interrupts\n");
+ else if (ch->rx_count == lif->nxqs)
+ return 0;
+
+ if (lif->nxqs != ch->rx_count)
+ netdev_info(netdev, "Changing queue count from %d to %d\n",
+ lif->nxqs, ch->rx_count);
+
+ qparam.nxqs = ch->rx_count;
+ qparam.intr_split = 1;
+ }
+
+ /* if we're not running, just set the values and return */
+ if (!netif_running(lif->netdev)) {
+ lif->nxqs = qparam.nxqs;
+
+ if (qparam.intr_split) {
+ set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
+ } else {
+ clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
+ lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
+ lif->tx_coalesce_hw = lif->rx_coalesce_hw;
+ }
+ return 0;
+ }
+
+ mutex_lock(&lif->queue_lock);
+ err = ionic_reconfigure_queues(lif, &qparam);
+ mutex_unlock(&lif->queue_lock);
+ if (err)
+ netdev_info(netdev, "Queue reconfiguration failed, changes canceled: %d\n", err);
+
+ return err;
+}
+
+static int ionic_get_rxnfc(struct net_device *netdev,
+ struct ethtool_rxnfc *info, u32 *rules)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ int err = 0;
+
+ switch (info->cmd) {
+ case ETHTOOL_GRXRINGS:
+ info->data = lif->nxqs;
+ break;
+ default:
+ netdev_dbg(netdev, "Command parameter %d is not supported\n",
+ info->cmd);
+ err = -EOPNOTSUPP;
+ }
+
+ return err;
+}
+
+static u32 ionic_get_rxfh_indir_size(struct net_device *netdev)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+
+ return le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
+}
+
+static u32 ionic_get_rxfh_key_size(struct net_device *netdev)
+{
+ return IONIC_RSS_HASH_KEY_SIZE;
+}
+
+static int ionic_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key,
+ u8 *hfunc)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ unsigned int i, tbl_sz;
+
+ if (indir) {
+ tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
+ for (i = 0; i < tbl_sz; i++)
+ indir[i] = lif->rss_ind_tbl[i];
+ }
+
+ if (key)
+ memcpy(key, lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
+
+ if (hfunc)
+ *hfunc = ETH_RSS_HASH_TOP;
+
+ return 0;
+}
+
+static int ionic_set_rxfh(struct net_device *netdev, const u32 *indir,
+ const u8 *key, const u8 hfunc)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+
+ if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
+ return -EOPNOTSUPP;
+
+ return ionic_lif_rss_config(lif, lif->rss_types, key, indir);
+}
+
+static int ionic_set_tunable(struct net_device *dev,
+ const struct ethtool_tunable *tuna,
+ const void *data)
+{
+ struct ionic_lif *lif = netdev_priv(dev);
+
+ switch (tuna->id) {
+ case ETHTOOL_RX_COPYBREAK:
+ lif->rx_copybreak = *(u32 *)data;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+static int ionic_get_tunable(struct net_device *netdev,
+ const struct ethtool_tunable *tuna, void *data)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+
+ switch (tuna->id) {
+ case ETHTOOL_RX_COPYBREAK:
+ *(u32 *)data = lif->rx_copybreak;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+static int ionic_get_module_info(struct net_device *netdev,
+ struct ethtool_modinfo *modinfo)
+
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_dev *idev = &lif->ionic->idev;
+ struct ionic_xcvr_status *xcvr;
+ struct sfp_eeprom_base *sfp;
+
+ xcvr = &idev->port_info->status.xcvr;
+ sfp = (struct sfp_eeprom_base *) xcvr->sprom;
+
+ /* report the module data type and length */
+ switch (sfp->phys_id) {
+ case SFF8024_ID_SFP:
+ modinfo->type = ETH_MODULE_SFF_8079;
+ modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN;
+ break;
+ case SFF8024_ID_QSFP_8436_8636:
+ case SFF8024_ID_QSFP28_8636:
+ modinfo->type = ETH_MODULE_SFF_8436;
+ modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN;
+ break;
+ default:
+ netdev_info(netdev, "unknown xcvr type 0x%02x\n",
+ xcvr->sprom[0]);
+ modinfo->type = 0;
+ modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN;
+ break;
+ }
+
+ return 0;
+}
+
+static int ionic_get_module_eeprom(struct net_device *netdev,
+ struct ethtool_eeprom *ee,
+ u8 *data)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_dev *idev = &lif->ionic->idev;
+ struct ionic_xcvr_status *xcvr;
+ char tbuf[sizeof(xcvr->sprom)];
+ int count = 10;
+ u32 len;
+
+ /* The NIC keeps the module prom up-to-date in the DMA space
+ * so we can simply copy the module bytes into the data buffer.
+ */
+ xcvr = &idev->port_info->status.xcvr;
+ len = min_t(u32, sizeof(xcvr->sprom), ee->len);
+
+ do {
+ memcpy(data, xcvr->sprom, len);
+ memcpy(tbuf, xcvr->sprom, len);
+
+ /* Let's make sure we got a consistent copy */
+ if (!memcmp(data, tbuf, len))
+ break;
+
+ } while (--count);
+
+ if (!count)
+ return -ETIMEDOUT;
+
+ return 0;
+}
+
+static int ionic_get_ts_info(struct net_device *netdev,
+ struct ethtool_ts_info *info)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+ __le64 mask;
+
+ if (!lif->phc || !lif->phc->ptp)
+ return ethtool_op_get_ts_info(netdev, info);
+
+ info->phc_index = ptp_clock_index(lif->phc->ptp);
+
+ info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE |
+ SOF_TIMESTAMPING_RX_SOFTWARE |
+ SOF_TIMESTAMPING_SOFTWARE |
+ SOF_TIMESTAMPING_TX_HARDWARE |
+ SOF_TIMESTAMPING_RX_HARDWARE |
+ SOF_TIMESTAMPING_RAW_HARDWARE;
+
+ /* tx modes */
+
+ info->tx_types = BIT(HWTSTAMP_TX_OFF) |
+ BIT(HWTSTAMP_TX_ON);
+
+ mask = cpu_to_le64(BIT_ULL(IONIC_TXSTAMP_ONESTEP_SYNC));
+ if (ionic->ident.lif.eth.hwstamp_tx_modes & mask)
+ info->tx_types |= BIT(HWTSTAMP_TX_ONESTEP_SYNC);
+
+ mask = cpu_to_le64(BIT_ULL(IONIC_TXSTAMP_ONESTEP_P2P));
+ if (ionic->ident.lif.eth.hwstamp_tx_modes & mask)
+ info->tx_types |= BIT(HWTSTAMP_TX_ONESTEP_P2P);
+
+ /* rx filters */
+
+ info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) |
+ BIT(HWTSTAMP_FILTER_ALL);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_NTP_ALL);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_NTP_ALL);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP1_SYNC);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V1_L4_SYNC);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP1_DREQ);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP1_ALL);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V1_L4_EVENT);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP2_L4_SYNC);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V2_L4_SYNC);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP2_L4_DREQ);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP2_L4_ALL);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP2_L2_SYNC);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V2_L2_SYNC);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP2_L2_DREQ);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP2_L2_ALL);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP2_SYNC);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V2_SYNC);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP2_DREQ);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V2_DELAY_REQ);
+
+ mask = cpu_to_le64(IONIC_PKT_CLS_PTP2_ALL);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) == mask)
+ info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V2_EVENT);
+
+ return 0;
+}
+
+static int ionic_nway_reset(struct net_device *netdev)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+ int err = 0;
+
+ if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return -EBUSY;
+
+ /* flap the link to force auto-negotiation */
+
+ mutex_lock(&ionic->dev_cmd_lock);
+
+ ionic_dev_cmd_port_state(&ionic->idev, IONIC_PORT_ADMIN_STATE_DOWN);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+
+ if (!err) {
+ ionic_dev_cmd_port_state(&ionic->idev, IONIC_PORT_ADMIN_STATE_UP);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ }
+
+ mutex_unlock(&ionic->dev_cmd_lock);
+
+ return err;
+}
+
+static const struct ethtool_ops ionic_ethtool_ops = {
+ .supported_coalesce_params = ETHTOOL_COALESCE_USECS |
+ ETHTOOL_COALESCE_USE_ADAPTIVE_RX |
+ ETHTOOL_COALESCE_USE_ADAPTIVE_TX,
+ .get_drvinfo = ionic_get_drvinfo,
+ .get_regs_len = ionic_get_regs_len,
+ .get_regs = ionic_get_regs,
+ .get_link = ethtool_op_get_link,
+ .get_link_ksettings = ionic_get_link_ksettings,
+ .set_link_ksettings = ionic_set_link_ksettings,
+ .get_coalesce = ionic_get_coalesce,
+ .set_coalesce = ionic_set_coalesce,
+ .get_ringparam = ionic_get_ringparam,
+ .set_ringparam = ionic_set_ringparam,
+ .get_channels = ionic_get_channels,
+ .set_channels = ionic_set_channels,
+ .get_strings = ionic_get_strings,
+ .get_ethtool_stats = ionic_get_stats,
+ .get_sset_count = ionic_get_sset_count,
+ .get_rxnfc = ionic_get_rxnfc,
+ .get_rxfh_indir_size = ionic_get_rxfh_indir_size,
+ .get_rxfh_key_size = ionic_get_rxfh_key_size,
+ .get_rxfh = ionic_get_rxfh,
+ .set_rxfh = ionic_set_rxfh,
+ .get_tunable = ionic_get_tunable,
+ .set_tunable = ionic_set_tunable,
+ .get_module_info = ionic_get_module_info,
+ .get_module_eeprom = ionic_get_module_eeprom,
+ .get_pauseparam = ionic_get_pauseparam,
+ .set_pauseparam = ionic_set_pauseparam,
+ .get_fecparam = ionic_get_fecparam,
+ .set_fecparam = ionic_set_fecparam,
+ .get_ts_info = ionic_get_ts_info,
+ .nway_reset = ionic_nway_reset,
+};
+
+void ionic_ethtool_set_ops(struct net_device *netdev)
+{
+ netdev->ethtool_ops = &ionic_ethtool_ops;
+}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.h b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.h
new file mode 100644
index 000000000..38b91b1d7
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#ifndef _IONIC_ETHTOOL_H_
+#define _IONIC_ETHTOOL_H_
+
+void ionic_ethtool_set_ops(struct net_device *netdev);
+
+#endif /* _IONIC_ETHTOOL_H_ */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_fw.c b/drivers/net/ethernet/pensando/ionic/ionic_fw.c
new file mode 100644
index 000000000..5f40324cd
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_fw.c
@@ -0,0 +1,196 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2020 Pensando Systems, Inc */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/firmware.h>
+
+#include "ionic.h"
+#include "ionic_dev.h"
+#include "ionic_lif.h"
+#include "ionic_devlink.h"
+
+/* The worst case wait for the install activity is about 25 minutes when
+ * installing a new CPLD, which is very seldom. Normal is about 30-35
+ * seconds. Since the driver can't tell if a CPLD update will happen we
+ * set the timeout for the ugly case.
+ */
+#define IONIC_FW_INSTALL_TIMEOUT (25 * 60)
+#define IONIC_FW_SELECT_TIMEOUT 30
+
+/* Number of periodic log updates during fw file download */
+#define IONIC_FW_INTERVAL_FRACTION 32
+
+static void ionic_dev_cmd_firmware_download(struct ionic_dev *idev, u64 addr,
+ u32 offset, u32 length)
+{
+ union ionic_dev_cmd cmd = {
+ .fw_download.opcode = IONIC_CMD_FW_DOWNLOAD,
+ .fw_download.offset = cpu_to_le32(offset),
+ .fw_download.addr = cpu_to_le64(addr),
+ .fw_download.length = cpu_to_le32(length),
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+static void ionic_dev_cmd_firmware_install(struct ionic_dev *idev)
+{
+ union ionic_dev_cmd cmd = {
+ .fw_control.opcode = IONIC_CMD_FW_CONTROL,
+ .fw_control.oper = IONIC_FW_INSTALL_ASYNC
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+static void ionic_dev_cmd_firmware_activate(struct ionic_dev *idev, u8 slot)
+{
+ union ionic_dev_cmd cmd = {
+ .fw_control.opcode = IONIC_CMD_FW_CONTROL,
+ .fw_control.oper = IONIC_FW_ACTIVATE_ASYNC,
+ .fw_control.slot = slot
+ };
+
+ ionic_dev_cmd_go(idev, &cmd);
+}
+
+static int ionic_fw_status_long_wait(struct ionic *ionic,
+ const char *label,
+ unsigned long timeout,
+ u8 fw_cmd,
+ struct netlink_ext_ack *extack)
+{
+ union ionic_dev_cmd cmd = {
+ .fw_control.opcode = IONIC_CMD_FW_CONTROL,
+ .fw_control.oper = fw_cmd,
+ };
+ unsigned long start_time;
+ unsigned long end_time;
+ int err;
+
+ start_time = jiffies;
+ end_time = start_time + (timeout * HZ);
+ do {
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_go(&ionic->idev, &cmd);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&ionic->dev_cmd_lock);
+
+ msleep(20);
+ } while (time_before(jiffies, end_time) && (err == -EAGAIN || err == -ETIMEDOUT));
+
+ if (err == -EAGAIN || err == -ETIMEDOUT) {
+ NL_SET_ERR_MSG_MOD(extack, "Firmware wait timed out");
+ dev_err(ionic->dev, "DEV_CMD firmware wait %s timed out\n", label);
+ } else if (err) {
+ NL_SET_ERR_MSG_MOD(extack, "Firmware wait failed");
+ }
+
+ return err;
+}
+
+int ionic_firmware_update(struct ionic_lif *lif, const struct firmware *fw,
+ struct netlink_ext_ack *extack)
+{
+ struct ionic_dev *idev = &lif->ionic->idev;
+ struct net_device *netdev = lif->netdev;
+ struct ionic *ionic = lif->ionic;
+ union ionic_dev_cmd_comp comp;
+ u32 buf_sz, copy_sz, offset;
+ struct devlink *dl;
+ int next_interval;
+ int err = 0;
+ u8 fw_slot;
+
+ netdev_info(netdev, "Installing firmware\n");
+
+ dl = priv_to_devlink(ionic);
+ devlink_flash_update_status_notify(dl, "Preparing to flash", NULL, 0, 0);
+
+ buf_sz = sizeof(idev->dev_cmd_regs->data);
+
+ netdev_dbg(netdev,
+ "downloading firmware - size %d part_sz %d nparts %lu\n",
+ (int)fw->size, buf_sz, DIV_ROUND_UP(fw->size, buf_sz));
+
+ offset = 0;
+ next_interval = 0;
+ while (offset < fw->size) {
+ if (offset >= next_interval) {
+ devlink_flash_update_status_notify(dl, "Downloading", NULL,
+ offset, fw->size);
+ next_interval = offset + (fw->size / IONIC_FW_INTERVAL_FRACTION);
+ }
+
+ copy_sz = min_t(unsigned int, buf_sz, fw->size - offset);
+ mutex_lock(&ionic->dev_cmd_lock);
+ memcpy_toio(&idev->dev_cmd_regs->data, fw->data + offset, copy_sz);
+ ionic_dev_cmd_firmware_download(idev,
+ offsetof(union ionic_dev_cmd_regs, data),
+ offset, copy_sz);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&ionic->dev_cmd_lock);
+ if (err) {
+ netdev_err(netdev,
+ "download failed offset 0x%x addr 0x%lx len 0x%x\n",
+ offset, offsetof(union ionic_dev_cmd_regs, data),
+ copy_sz);
+ NL_SET_ERR_MSG_MOD(extack, "Segment download failed");
+ goto err_out;
+ }
+ offset += copy_sz;
+ }
+ devlink_flash_update_status_notify(dl, "Downloading", NULL,
+ fw->size, fw->size);
+
+ devlink_flash_update_timeout_notify(dl, "Installing", NULL,
+ IONIC_FW_INSTALL_TIMEOUT);
+
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_firmware_install(idev);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
+ fw_slot = comp.fw_control.slot;
+ mutex_unlock(&ionic->dev_cmd_lock);
+ if (err) {
+ NL_SET_ERR_MSG_MOD(extack, "Failed to start firmware install");
+ goto err_out;
+ }
+
+ err = ionic_fw_status_long_wait(ionic, "Installing",
+ IONIC_FW_INSTALL_TIMEOUT,
+ IONIC_FW_INSTALL_STATUS,
+ extack);
+ if (err)
+ goto err_out;
+
+ devlink_flash_update_timeout_notify(dl, "Selecting", NULL,
+ IONIC_FW_SELECT_TIMEOUT);
+
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_firmware_activate(idev, fw_slot);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&ionic->dev_cmd_lock);
+ if (err) {
+ NL_SET_ERR_MSG_MOD(extack, "Failed to start firmware select");
+ goto err_out;
+ }
+
+ err = ionic_fw_status_long_wait(ionic, "Selecting",
+ IONIC_FW_SELECT_TIMEOUT,
+ IONIC_FW_ACTIVATE_STATUS,
+ extack);
+ if (err)
+ goto err_out;
+
+ netdev_info(netdev, "Firmware update completed\n");
+
+err_out:
+ if (err)
+ devlink_flash_update_status_notify(dl, "Flash failed", NULL, 0, 0);
+ else
+ devlink_flash_update_status_notify(dl, "Flash done", NULL, 0, 0);
+ return err;
+}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_if.h b/drivers/net/ethernet/pensando/ionic/ionic_if.h
new file mode 100644
index 000000000..4a90f611c
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_if.h
@@ -0,0 +1,3092 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) OR BSD-2-Clause */
+/* Copyright (c) 2017-2020 Pensando Systems, Inc. All rights reserved. */
+
+#ifndef _IONIC_IF_H_
+#define _IONIC_IF_H_
+
+#define IONIC_DEV_INFO_SIGNATURE 0x44455649 /* 'DEVI' */
+#define IONIC_DEV_INFO_VERSION 1
+#define IONIC_IFNAMSIZ 16
+
+/**
+ * enum ionic_cmd_opcode - Device commands
+ */
+enum ionic_cmd_opcode {
+ IONIC_CMD_NOP = 0,
+
+ /* Device commands */
+ IONIC_CMD_IDENTIFY = 1,
+ IONIC_CMD_INIT = 2,
+ IONIC_CMD_RESET = 3,
+ IONIC_CMD_GETATTR = 4,
+ IONIC_CMD_SETATTR = 5,
+
+ /* Port commands */
+ IONIC_CMD_PORT_IDENTIFY = 10,
+ IONIC_CMD_PORT_INIT = 11,
+ IONIC_CMD_PORT_RESET = 12,
+ IONIC_CMD_PORT_GETATTR = 13,
+ IONIC_CMD_PORT_SETATTR = 14,
+
+ /* LIF commands */
+ IONIC_CMD_LIF_IDENTIFY = 20,
+ IONIC_CMD_LIF_INIT = 21,
+ IONIC_CMD_LIF_RESET = 22,
+ IONIC_CMD_LIF_GETATTR = 23,
+ IONIC_CMD_LIF_SETATTR = 24,
+ IONIC_CMD_LIF_SETPHC = 25,
+
+ IONIC_CMD_RX_MODE_SET = 30,
+ IONIC_CMD_RX_FILTER_ADD = 31,
+ IONIC_CMD_RX_FILTER_DEL = 32,
+
+ /* Queue commands */
+ IONIC_CMD_Q_IDENTIFY = 39,
+ IONIC_CMD_Q_INIT = 40,
+ IONIC_CMD_Q_CONTROL = 41,
+
+ /* RDMA commands */
+ IONIC_CMD_RDMA_RESET_LIF = 50,
+ IONIC_CMD_RDMA_CREATE_EQ = 51,
+ IONIC_CMD_RDMA_CREATE_CQ = 52,
+ IONIC_CMD_RDMA_CREATE_ADMINQ = 53,
+
+ /* SR/IOV commands */
+ IONIC_CMD_VF_GETATTR = 60,
+ IONIC_CMD_VF_SETATTR = 61,
+
+ /* QoS commands */
+ IONIC_CMD_QOS_CLASS_IDENTIFY = 240,
+ IONIC_CMD_QOS_CLASS_INIT = 241,
+ IONIC_CMD_QOS_CLASS_RESET = 242,
+ IONIC_CMD_QOS_CLASS_UPDATE = 243,
+ IONIC_CMD_QOS_CLEAR_STATS = 244,
+ IONIC_CMD_QOS_RESET = 245,
+
+ /* Firmware commands */
+ IONIC_CMD_FW_DOWNLOAD = 252,
+ IONIC_CMD_FW_CONTROL = 253,
+ IONIC_CMD_FW_DOWNLOAD_V1 = 254,
+ IONIC_CMD_FW_CONTROL_V1 = 255,
+};
+
+/**
+ * enum ionic_status_code - Device command return codes
+ */
+enum ionic_status_code {
+ IONIC_RC_SUCCESS = 0, /* Success */
+ IONIC_RC_EVERSION = 1, /* Incorrect version for request */
+ IONIC_RC_EOPCODE = 2, /* Invalid cmd opcode */
+ IONIC_RC_EIO = 3, /* I/O error */
+ IONIC_RC_EPERM = 4, /* Permission denied */
+ IONIC_RC_EQID = 5, /* Bad qid */
+ IONIC_RC_EQTYPE = 6, /* Bad qtype */
+ IONIC_RC_ENOENT = 7, /* No such element */
+ IONIC_RC_EINTR = 8, /* operation interrupted */
+ IONIC_RC_EAGAIN = 9, /* Try again */
+ IONIC_RC_ENOMEM = 10, /* Out of memory */
+ IONIC_RC_EFAULT = 11, /* Bad address */
+ IONIC_RC_EBUSY = 12, /* Device or resource busy */
+ IONIC_RC_EEXIST = 13, /* object already exists */
+ IONIC_RC_EINVAL = 14, /* Invalid argument */
+ IONIC_RC_ENOSPC = 15, /* No space left or alloc failure */
+ IONIC_RC_ERANGE = 16, /* Parameter out of range */
+ IONIC_RC_BAD_ADDR = 17, /* Descriptor contains a bad ptr */
+ IONIC_RC_DEV_CMD = 18, /* Device cmd attempted on AdminQ */
+ IONIC_RC_ENOSUPP = 19, /* Operation not supported */
+ IONIC_RC_ERROR = 29, /* Generic error */
+ IONIC_RC_ERDMA = 30, /* Generic RDMA error */
+ IONIC_RC_EVFID = 31, /* VF ID does not exist */
+ IONIC_RC_EBAD_FW = 32, /* FW file is invalid or corrupted */
+};
+
+enum ionic_notifyq_opcode {
+ IONIC_EVENT_LINK_CHANGE = 1,
+ IONIC_EVENT_RESET = 2,
+ IONIC_EVENT_HEARTBEAT = 3,
+ IONIC_EVENT_LOG = 4,
+ IONIC_EVENT_XCVR = 5,
+};
+
+/**
+ * struct ionic_admin_cmd - General admin command format
+ * @opcode: Opcode for the command
+ * @lif_index: LIF index
+ * @cmd_data: Opcode-specific command bytes
+ */
+struct ionic_admin_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 lif_index;
+ u8 cmd_data[60];
+};
+
+/**
+ * struct ionic_admin_comp - General admin command completion format
+ * @status: Status of the command (enum ionic_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @cmd_data: Command-specific bytes
+ * @color: Color bit (Always 0 for commands issued to the
+ * Device Cmd Registers)
+ */
+struct ionic_admin_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ u8 cmd_data[11];
+ u8 color;
+#define IONIC_COMP_COLOR_MASK 0x80
+};
+
+static inline u8 color_match(u8 color, u8 done_color)
+{
+ return (!!(color & IONIC_COMP_COLOR_MASK)) == done_color;
+}
+
+/**
+ * struct ionic_nop_cmd - NOP command
+ * @opcode: opcode
+ */
+struct ionic_nop_cmd {
+ u8 opcode;
+ u8 rsvd[63];
+};
+
+/**
+ * struct ionic_nop_comp - NOP command completion
+ * @status: Status of the command (enum ionic_status_code)
+ */
+struct ionic_nop_comp {
+ u8 status;
+ u8 rsvd[15];
+};
+
+/**
+ * struct ionic_dev_init_cmd - Device init command
+ * @opcode: opcode
+ * @type: Device type
+ */
+struct ionic_dev_init_cmd {
+ u8 opcode;
+ u8 type;
+ u8 rsvd[62];
+};
+
+/**
+ * struct ionic_dev_init_comp - Device init command completion
+ * @status: Status of the command (enum ionic_status_code)
+ */
+struct ionic_dev_init_comp {
+ u8 status;
+ u8 rsvd[15];
+};
+
+/**
+ * struct ionic_dev_reset_cmd - Device reset command
+ * @opcode: opcode
+ */
+struct ionic_dev_reset_cmd {
+ u8 opcode;
+ u8 rsvd[63];
+};
+
+/**
+ * struct ionic_dev_reset_comp - Reset command completion
+ * @status: Status of the command (enum ionic_status_code)
+ */
+struct ionic_dev_reset_comp {
+ u8 status;
+ u8 rsvd[15];
+};
+
+#define IONIC_IDENTITY_VERSION_1 1
+
+/**
+ * struct ionic_dev_identify_cmd - Driver/device identify command
+ * @opcode: opcode
+ * @ver: Highest version of identify supported by driver
+ */
+struct ionic_dev_identify_cmd {
+ u8 opcode;
+ u8 ver;
+ u8 rsvd[62];
+};
+
+/**
+ * struct ionic_dev_identify_comp - Driver/device identify command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @ver: Version of identify returned by device
+ */
+struct ionic_dev_identify_comp {
+ u8 status;
+ u8 ver;
+ u8 rsvd[14];
+};
+
+enum ionic_os_type {
+ IONIC_OS_TYPE_LINUX = 1,
+ IONIC_OS_TYPE_WIN = 2,
+ IONIC_OS_TYPE_DPDK = 3,
+ IONIC_OS_TYPE_FREEBSD = 4,
+ IONIC_OS_TYPE_IPXE = 5,
+ IONIC_OS_TYPE_ESXI = 6,
+};
+
+/**
+ * union ionic_drv_identity - driver identity information
+ * @os_type: OS type (see enum ionic_os_type)
+ * @os_dist: OS distribution, numeric format
+ * @os_dist_str: OS distribution, string format
+ * @kernel_ver: Kernel version, numeric format
+ * @kernel_ver_str: Kernel version, string format
+ * @driver_ver_str: Driver version, string format
+ */
+union ionic_drv_identity {
+ struct {
+ __le32 os_type;
+ __le32 os_dist;
+ char os_dist_str[128];
+ __le32 kernel_ver;
+ char kernel_ver_str[32];
+ char driver_ver_str[32];
+ };
+ __le32 words[478];
+};
+
+/**
+ * union ionic_dev_identity - device identity information
+ * @version: Version of device identify
+ * @type: Identify type (0 for now)
+ * @nports: Number of ports provisioned
+ * @nlifs: Number of LIFs provisioned
+ * @nintrs: Number of interrupts provisioned
+ * @ndbpgs_per_lif: Number of doorbell pages per LIF
+ * @intr_coal_mult: Interrupt coalescing multiplication factor
+ * Scale user-supplied interrupt coalescing
+ * value in usecs to device units using:
+ * device units = usecs * mult / div
+ * @intr_coal_div: Interrupt coalescing division factor
+ * Scale user-supplied interrupt coalescing
+ * value in usecs to device units using:
+ * device units = usecs * mult / div
+ * @eq_count: Number of shared event queues
+ * @hwstamp_mask: Bitmask for subtraction of hardware tick values.
+ * @hwstamp_mult: Hardware tick to nanosecond multiplier.
+ * @hwstamp_shift: Hardware tick to nanosecond divisor (power of two).
+ */
+union ionic_dev_identity {
+ struct {
+ u8 version;
+ u8 type;
+ u8 rsvd[2];
+ u8 nports;
+ u8 rsvd2[3];
+ __le32 nlifs;
+ __le32 nintrs;
+ __le32 ndbpgs_per_lif;
+ __le32 intr_coal_mult;
+ __le32 intr_coal_div;
+ __le32 eq_count;
+ __le64 hwstamp_mask;
+ __le32 hwstamp_mult;
+ __le32 hwstamp_shift;
+ };
+ __le32 words[478];
+};
+
+enum ionic_lif_type {
+ IONIC_LIF_TYPE_CLASSIC = 0,
+ IONIC_LIF_TYPE_MACVLAN = 1,
+ IONIC_LIF_TYPE_NETQUEUE = 2,
+};
+
+/**
+ * struct ionic_lif_identify_cmd - LIF identify command
+ * @opcode: opcode
+ * @type: LIF type (enum ionic_lif_type)
+ * @ver: Version of identify returned by device
+ */
+struct ionic_lif_identify_cmd {
+ u8 opcode;
+ u8 type;
+ u8 ver;
+ u8 rsvd[61];
+};
+
+/**
+ * struct ionic_lif_identify_comp - LIF identify command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @ver: Version of identify returned by device
+ */
+struct ionic_lif_identify_comp {
+ u8 status;
+ u8 ver;
+ u8 rsvd2[14];
+};
+
+/**
+ * enum ionic_lif_capability - LIF capabilities
+ * @IONIC_LIF_CAP_ETH: LIF supports Ethernet
+ * @IONIC_LIF_CAP_RDMA: LIF supports RDMA
+ */
+enum ionic_lif_capability {
+ IONIC_LIF_CAP_ETH = BIT(0),
+ IONIC_LIF_CAP_RDMA = BIT(1),
+};
+
+/**
+ * enum ionic_logical_qtype - Logical Queue Types
+ * @IONIC_QTYPE_ADMINQ: Administrative Queue
+ * @IONIC_QTYPE_NOTIFYQ: Notify Queue
+ * @IONIC_QTYPE_RXQ: Receive Queue
+ * @IONIC_QTYPE_TXQ: Transmit Queue
+ * @IONIC_QTYPE_EQ: Event Queue
+ * @IONIC_QTYPE_MAX: Max queue type supported
+ */
+enum ionic_logical_qtype {
+ IONIC_QTYPE_ADMINQ = 0,
+ IONIC_QTYPE_NOTIFYQ = 1,
+ IONIC_QTYPE_RXQ = 2,
+ IONIC_QTYPE_TXQ = 3,
+ IONIC_QTYPE_EQ = 4,
+ IONIC_QTYPE_MAX = 16,
+};
+
+/**
+ * enum ionic_q_feature - Common Features for most queue types
+ *
+ * Common features use bits 0-15. Per-queue-type features use higher bits.
+ *
+ * @IONIC_QIDENT_F_CQ: Queue has completion ring
+ * @IONIC_QIDENT_F_SG: Queue has scatter/gather ring
+ * @IONIC_QIDENT_F_EQ: Queue can use event queue
+ * @IONIC_QIDENT_F_CMB: Queue is in cmb bar
+ * @IONIC_Q_F_2X_DESC: Double main descriptor size
+ * @IONIC_Q_F_2X_CQ_DESC: Double cq descriptor size
+ * @IONIC_Q_F_2X_SG_DESC: Double sg descriptor size
+ * @IONIC_Q_F_4X_DESC: Quadruple main descriptor size
+ * @IONIC_Q_F_4X_CQ_DESC: Quadruple cq descriptor size
+ * @IONIC_Q_F_4X_SG_DESC: Quadruple sg descriptor size
+ */
+enum ionic_q_feature {
+ IONIC_QIDENT_F_CQ = BIT_ULL(0),
+ IONIC_QIDENT_F_SG = BIT_ULL(1),
+ IONIC_QIDENT_F_EQ = BIT_ULL(2),
+ IONIC_QIDENT_F_CMB = BIT_ULL(3),
+ IONIC_Q_F_2X_DESC = BIT_ULL(4),
+ IONIC_Q_F_2X_CQ_DESC = BIT_ULL(5),
+ IONIC_Q_F_2X_SG_DESC = BIT_ULL(6),
+ IONIC_Q_F_4X_DESC = BIT_ULL(7),
+ IONIC_Q_F_4X_CQ_DESC = BIT_ULL(8),
+ IONIC_Q_F_4X_SG_DESC = BIT_ULL(9),
+};
+
+/**
+ * enum ionic_rxq_feature - RXQ-specific Features
+ *
+ * Per-queue-type features use bits 16 and higher.
+ *
+ * @IONIC_RXQ_F_HWSTAMP: Queue supports Hardware Timestamping
+ */
+enum ionic_rxq_feature {
+ IONIC_RXQ_F_HWSTAMP = BIT_ULL(16),
+};
+
+/**
+ * enum ionic_txq_feature - TXQ-specific Features
+ *
+ * Per-queue-type features use bits 16 and higher.
+ *
+ * @IONIC_TXQ_F_HWSTAMP: Queue supports Hardware Timestamping
+ */
+enum ionic_txq_feature {
+ IONIC_TXQ_F_HWSTAMP = BIT(16),
+};
+
+/**
+ * struct ionic_hwstamp_bits - Hardware timestamp decoding bits
+ * @IONIC_HWSTAMP_INVALID: Invalid hardware timestamp value
+ * @IONIC_HWSTAMP_CQ_NEGOFFSET: Timestamp field negative offset
+ * from the base cq descriptor.
+ */
+enum ionic_hwstamp_bits {
+ IONIC_HWSTAMP_INVALID = ~0ull,
+ IONIC_HWSTAMP_CQ_NEGOFFSET = 8,
+};
+
+/**
+ * struct ionic_lif_logical_qtype - Descriptor of logical to HW queue type
+ * @qtype: Hardware Queue Type
+ * @qid_count: Number of Queue IDs of the logical type
+ * @qid_base: Minimum Queue ID of the logical type
+ */
+struct ionic_lif_logical_qtype {
+ u8 qtype;
+ u8 rsvd[3];
+ __le32 qid_count;
+ __le32 qid_base;
+};
+
+/**
+ * enum ionic_lif_state - LIF state
+ * @IONIC_LIF_DISABLE: LIF disabled
+ * @IONIC_LIF_ENABLE: LIF enabled
+ * @IONIC_LIF_QUIESCE: LIF Quiesced
+ */
+enum ionic_lif_state {
+ IONIC_LIF_QUIESCE = 0,
+ IONIC_LIF_ENABLE = 1,
+ IONIC_LIF_DISABLE = 2,
+};
+
+/**
+ * union ionic_lif_config - LIF configuration
+ * @state: LIF state (enum ionic_lif_state)
+ * @name: LIF name
+ * @mtu: MTU
+ * @mac: Station MAC address
+ * @vlan: Default Vlan ID
+ * @features: Features (enum ionic_eth_hw_features)
+ * @queue_count: Queue counts per queue-type
+ */
+union ionic_lif_config {
+ struct {
+ u8 state;
+ u8 rsvd[3];
+ char name[IONIC_IFNAMSIZ];
+ __le32 mtu;
+ u8 mac[6];
+ __le16 vlan;
+ __le64 features;
+ __le32 queue_count[IONIC_QTYPE_MAX];
+ } __packed;
+ __le32 words[64];
+};
+
+/**
+ * struct ionic_lif_identity - LIF identity information (type-specific)
+ *
+ * @capabilities: LIF capabilities
+ *
+ * @eth: Ethernet identify structure
+ * @version: Ethernet identify structure version
+ * @max_ucast_filters: Number of perfect unicast addresses supported
+ * @max_mcast_filters: Number of perfect multicast addresses supported
+ * @min_frame_size: Minimum size of frames to be sent
+ * @max_frame_size: Maximum size of frames to be sent
+ * @hwstamp_tx_modes: Bitmask of BIT_ULL(enum ionic_txstamp_mode)
+ * @hwstamp_rx_filters: Bitmask of enum ionic_pkt_class
+ * @config: LIF config struct with features, mtu, mac, q counts
+ *
+ * @rdma: RDMA identify structure
+ * @version: RDMA version of opcodes and queue descriptors
+ * @qp_opcodes: Number of RDMA queue pair opcodes supported
+ * @admin_opcodes: Number of RDMA admin opcodes supported
+ * @npts_per_lif: Page table size per LIF
+ * @nmrs_per_lif: Number of memory regions per LIF
+ * @nahs_per_lif: Number of address handles per LIF
+ * @max_stride: Max work request stride
+ * @cl_stride: Cache line stride
+ * @pte_stride: Page table entry stride
+ * @rrq_stride: Remote RQ work request stride
+ * @rsq_stride: Remote SQ work request stride
+ * @dcqcn_profiles: Number of DCQCN profiles
+ * @aq_qtype: RDMA Admin Qtype
+ * @sq_qtype: RDMA Send Qtype
+ * @rq_qtype: RDMA Receive Qtype
+ * @cq_qtype: RDMA Completion Qtype
+ * @eq_qtype: RDMA Event Qtype
+ */
+union ionic_lif_identity {
+ struct {
+ __le64 capabilities;
+
+ struct {
+ u8 version;
+ u8 rsvd[3];
+ __le32 max_ucast_filters;
+ __le32 max_mcast_filters;
+ __le16 rss_ind_tbl_sz;
+ __le32 min_frame_size;
+ __le32 max_frame_size;
+ u8 rsvd2[2];
+ __le64 hwstamp_tx_modes;
+ __le64 hwstamp_rx_filters;
+ u8 rsvd3[88];
+ union ionic_lif_config config;
+ } __packed eth;
+
+ struct {
+ u8 version;
+ u8 qp_opcodes;
+ u8 admin_opcodes;
+ u8 rsvd;
+ __le32 npts_per_lif;
+ __le32 nmrs_per_lif;
+ __le32 nahs_per_lif;
+ u8 max_stride;
+ u8 cl_stride;
+ u8 pte_stride;
+ u8 rrq_stride;
+ u8 rsq_stride;
+ u8 dcqcn_profiles;
+ u8 rsvd_dimensions[10];
+ struct ionic_lif_logical_qtype aq_qtype;
+ struct ionic_lif_logical_qtype sq_qtype;
+ struct ionic_lif_logical_qtype rq_qtype;
+ struct ionic_lif_logical_qtype cq_qtype;
+ struct ionic_lif_logical_qtype eq_qtype;
+ } __packed rdma;
+ } __packed;
+ __le32 words[478];
+};
+
+/**
+ * struct ionic_lif_init_cmd - LIF init command
+ * @opcode: Opcode
+ * @type: LIF type (enum ionic_lif_type)
+ * @index: LIF index
+ * @info_pa: Destination address for LIF info (struct ionic_lif_info)
+ */
+struct ionic_lif_init_cmd {
+ u8 opcode;
+ u8 type;
+ __le16 index;
+ __le32 rsvd;
+ __le64 info_pa;
+ u8 rsvd2[48];
+};
+
+/**
+ * struct ionic_lif_init_comp - LIF init command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @hw_index: Hardware index of the initialized LIF
+ */
+struct ionic_lif_init_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 hw_index;
+ u8 rsvd2[12];
+};
+
+/**
+ * struct ionic_q_identify_cmd - queue identify command
+ * @opcode: opcode
+ * @lif_type: LIF type (enum ionic_lif_type)
+ * @type: Logical queue type (enum ionic_logical_qtype)
+ * @ver: Highest queue type version that the driver supports
+ */
+struct ionic_q_identify_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 lif_type;
+ u8 type;
+ u8 ver;
+ u8 rsvd2[58];
+};
+
+/**
+ * struct ionic_q_identify_comp - queue identify command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @ver: Queue type version that can be used with FW
+ */
+struct ionic_q_identify_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ u8 ver;
+ u8 rsvd2[11];
+};
+
+/**
+ * union ionic_q_identity - queue identity information
+ * @version: Queue type version that can be used with FW
+ * @supported: Bitfield of queue versions, first bit = ver 0
+ * @features: Queue features (enum ionic_q_feature, etc)
+ * @desc_sz: Descriptor size
+ * @comp_sz: Completion descriptor size
+ * @sg_desc_sz: Scatter/Gather descriptor size
+ * @max_sg_elems: Maximum number of Scatter/Gather elements
+ * @sg_desc_stride: Number of Scatter/Gather elements per descriptor
+ */
+union ionic_q_identity {
+ struct {
+ u8 version;
+ u8 supported;
+ u8 rsvd[6];
+ __le64 features;
+ __le16 desc_sz;
+ __le16 comp_sz;
+ __le16 sg_desc_sz;
+ __le16 max_sg_elems;
+ __le16 sg_desc_stride;
+ };
+ __le32 words[478];
+};
+
+/**
+ * struct ionic_q_init_cmd - Queue init command
+ * @opcode: opcode
+ * @type: Logical queue type
+ * @ver: Queue type version
+ * @lif_index: LIF index
+ * @index: (LIF, qtype) relative admin queue index
+ * @intr_index: Interrupt control register index, or Event queue index
+ * @pid: Process ID
+ * @flags:
+ * IRQ: Interrupt requested on completion
+ * ENA: Enable the queue. If ENA=0 the queue is initialized
+ * but remains disabled, to be later enabled with the
+ * Queue Enable command. If ENA=1, then queue is
+ * initialized and then enabled.
+ * SG: Enable Scatter-Gather on the queue.
+ * in number of descs. The actual ring size is
+ * (1 << ring_size). For example, to
+ * select a ring size of 64 descriptors write
+ * ring_size = 6. The minimum ring_size value is 2
+ * for a ring size of 4 descriptors. The maximum
+ * ring_size value is 16 for a ring size of 64k
+ * descriptors. Values of ring_size <2 and >16 are
+ * reserved.
+ * EQ: Enable the Event Queue
+ * @cos: Class of service for this queue
+ * @ring_size: Queue ring size, encoded as a log2(size)
+ * @ring_base: Queue ring base address
+ * @cq_ring_base: Completion queue ring base address
+ * @sg_ring_base: Scatter/Gather ring base address
+ * @features: Mask of queue features to enable, if not in the flags above.
+ */
+struct ionic_q_init_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 lif_index;
+ u8 type;
+ u8 ver;
+ u8 rsvd1[2];
+ __le32 index;
+ __le16 pid;
+ __le16 intr_index;
+ __le16 flags;
+#define IONIC_QINIT_F_IRQ 0x01 /* Request interrupt on completion */
+#define IONIC_QINIT_F_ENA 0x02 /* Enable the queue */
+#define IONIC_QINIT_F_SG 0x04 /* Enable scatter/gather on the queue */
+#define IONIC_QINIT_F_EQ 0x08 /* Enable event queue */
+#define IONIC_QINIT_F_CMB 0x10 /* Enable cmb-based queue */
+#define IONIC_QINIT_F_DEBUG 0x80 /* Enable queue debugging */
+ u8 cos;
+ u8 ring_size;
+ __le64 ring_base;
+ __le64 cq_ring_base;
+ __le64 sg_ring_base;
+ u8 rsvd2[12];
+ __le64 features;
+} __packed;
+
+/**
+ * struct ionic_q_init_comp - Queue init command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @hw_index: Hardware Queue ID
+ * @hw_type: Hardware Queue type
+ * @color: Color
+ */
+struct ionic_q_init_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ __le32 hw_index;
+ u8 hw_type;
+ u8 rsvd2[6];
+ u8 color;
+};
+
+/* the device's internal addressing uses up to 52 bits */
+#define IONIC_ADDR_LEN 52
+#define IONIC_ADDR_MASK (BIT_ULL(IONIC_ADDR_LEN) - 1)
+
+enum ionic_txq_desc_opcode {
+ IONIC_TXQ_DESC_OPCODE_CSUM_NONE = 0,
+ IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL = 1,
+ IONIC_TXQ_DESC_OPCODE_CSUM_HW = 2,
+ IONIC_TXQ_DESC_OPCODE_TSO = 3,
+};
+
+/**
+ * struct ionic_txq_desc - Ethernet Tx queue descriptor format
+ * @cmd: Tx operation, see IONIC_TXQ_DESC_OPCODE_*:
+ *
+ * IONIC_TXQ_DESC_OPCODE_CSUM_NONE:
+ * Non-offload send. No segmentation,
+ * fragmentation or checksum calc/insertion is
+ * performed by device; packet is prepared
+ * to send by software stack and requires
+ * no further manipulation from device.
+ *
+ * IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL:
+ * Offload 16-bit L4 checksum
+ * calculation/insertion. The device will
+ * calculate the L4 checksum value and
+ * insert the result in the packet's L4
+ * header checksum field. The L4 checksum
+ * is calculated starting at @csum_start bytes
+ * into the packet to the end of the packet.
+ * The checksum insertion position is given
+ * in @csum_offset, which is the offset from
+ * @csum_start to the checksum field in the L4
+ * header. This feature is only applicable to
+ * protocols such as TCP, UDP and ICMP where a
+ * standard (i.e. the 'IP-style' checksum)
+ * one's complement 16-bit checksum is used,
+ * using an IP pseudo-header to seed the
+ * calculation. Software will preload the L4
+ * checksum field with the IP pseudo-header
+ * checksum.
+ *
+ * For tunnel encapsulation, @csum_start and
+ * @csum_offset refer to the inner L4
+ * header. Supported tunnels encapsulations
+ * are: IPIP, GRE, and UDP. If the @encap
+ * is clear, no further processing by the
+ * device is required; software will
+ * calculate the outer header checksums. If
+ * the @encap is set, the device will
+ * offload the outer header checksums using
+ * LCO (local checksum offload) (see
+ * Documentation/networking/checksum-offloads.rst
+ * for more info).
+ *
+ * IONIC_TXQ_DESC_OPCODE_CSUM_HW:
+ * Offload 16-bit checksum computation to hardware.
+ * If @csum_l3 is set then the packet's L3 checksum is
+ * updated. Similarly, if @csum_l4 is set the L4
+ * checksum is updated. If @encap is set then encap header
+ * checksums are also updated.
+ *
+ * IONIC_TXQ_DESC_OPCODE_TSO:
+ * Device performs TCP segmentation offload
+ * (TSO). @hdr_len is the number of bytes
+ * to the end of TCP header (the offset to
+ * the TCP payload). @mss is the desired
+ * MSS, the TCP payload length for each
+ * segment. The device will calculate/
+ * insert IP (IPv4 only) and TCP checksums
+ * for each segment. In the first data
+ * buffer containing the header template,
+ * the driver will set IPv4 checksum to 0
+ * and preload TCP checksum with the IP
+ * pseudo header calculated with IP length = 0.
+ *
+ * Supported tunnel encapsulations are IPIP,
+ * layer-3 GRE, and UDP. @hdr_len includes
+ * both outer and inner headers. The driver
+ * will set IPv4 checksum to zero and
+ * preload TCP checksum with IP pseudo
+ * header on the inner header.
+ *
+ * TCP ECN offload is supported. The device
+ * will set CWR flag in the first segment if
+ * CWR is set in the template header, and
+ * clear CWR in remaining segments.
+ * @flags:
+ * vlan:
+ * Insert an L2 VLAN header using @vlan_tci
+ * encap:
+ * Calculate encap header checksum
+ * csum_l3:
+ * Compute L3 header checksum
+ * csum_l4:
+ * Compute L4 header checksum
+ * tso_sot:
+ * TSO start
+ * tso_eot:
+ * TSO end
+ * @num_sg_elems: Number of scatter-gather elements in SG
+ * descriptor
+ * @addr: First data buffer's DMA address
+ * (Subsequent data buffers are on txq_sg_desc)
+ * @len: First data buffer's length, in bytes
+ * @vlan_tci: VLAN tag to insert in the packet (if requested
+ * by @V-bit). Includes .1p and .1q tags
+ * @hdr_len: Length of packet headers, including
+ * encapsulating outer header, if applicable
+ * Valid for opcodes IONIC_TXQ_DESC_OPCODE_CALC_CSUM and
+ * IONIC_TXQ_DESC_OPCODE_TSO. Should be set to zero for
+ * all other modes. For
+ * IONIC_TXQ_DESC_OPCODE_CALC_CSUM, @hdr_len is length
+ * of headers up to inner-most L4 header. For
+ * IONIC_TXQ_DESC_OPCODE_TSO, @hdr_len is up to
+ * inner-most L4 payload, so inclusive of
+ * inner-most L4 header.
+ * @mss: Desired MSS value for TSO; only applicable for
+ * IONIC_TXQ_DESC_OPCODE_TSO
+ * @csum_start: Offset from packet to first byte checked in L4 checksum
+ * @csum_offset: Offset from csum_start to L4 checksum field
+ */
+struct ionic_txq_desc {
+ __le64 cmd;
+#define IONIC_TXQ_DESC_OPCODE_MASK 0xf
+#define IONIC_TXQ_DESC_OPCODE_SHIFT 4
+#define IONIC_TXQ_DESC_FLAGS_MASK 0xf
+#define IONIC_TXQ_DESC_FLAGS_SHIFT 0
+#define IONIC_TXQ_DESC_NSGE_MASK 0xf
+#define IONIC_TXQ_DESC_NSGE_SHIFT 8
+#define IONIC_TXQ_DESC_ADDR_MASK (BIT_ULL(IONIC_ADDR_LEN) - 1)
+#define IONIC_TXQ_DESC_ADDR_SHIFT 12
+
+/* common flags */
+#define IONIC_TXQ_DESC_FLAG_VLAN 0x1
+#define IONIC_TXQ_DESC_FLAG_ENCAP 0x2
+
+/* flags for csum_hw opcode */
+#define IONIC_TXQ_DESC_FLAG_CSUM_L3 0x4
+#define IONIC_TXQ_DESC_FLAG_CSUM_L4 0x8
+
+/* flags for tso opcode */
+#define IONIC_TXQ_DESC_FLAG_TSO_SOT 0x4
+#define IONIC_TXQ_DESC_FLAG_TSO_EOT 0x8
+
+ __le16 len;
+ union {
+ __le16 vlan_tci;
+ __le16 hword0;
+ };
+ union {
+ __le16 csum_start;
+ __le16 hdr_len;
+ __le16 hword1;
+ };
+ union {
+ __le16 csum_offset;
+ __le16 mss;
+ __le16 hword2;
+ };
+};
+
+static inline u64 encode_txq_desc_cmd(u8 opcode, u8 flags,
+ u8 nsge, u64 addr)
+{
+ u64 cmd;
+
+ cmd = (opcode & IONIC_TXQ_DESC_OPCODE_MASK) << IONIC_TXQ_DESC_OPCODE_SHIFT;
+ cmd |= (flags & IONIC_TXQ_DESC_FLAGS_MASK) << IONIC_TXQ_DESC_FLAGS_SHIFT;
+ cmd |= (nsge & IONIC_TXQ_DESC_NSGE_MASK) << IONIC_TXQ_DESC_NSGE_SHIFT;
+ cmd |= (addr & IONIC_TXQ_DESC_ADDR_MASK) << IONIC_TXQ_DESC_ADDR_SHIFT;
+
+ return cmd;
+};
+
+static inline void decode_txq_desc_cmd(u64 cmd, u8 *opcode, u8 *flags,
+ u8 *nsge, u64 *addr)
+{
+ *opcode = (cmd >> IONIC_TXQ_DESC_OPCODE_SHIFT) & IONIC_TXQ_DESC_OPCODE_MASK;
+ *flags = (cmd >> IONIC_TXQ_DESC_FLAGS_SHIFT) & IONIC_TXQ_DESC_FLAGS_MASK;
+ *nsge = (cmd >> IONIC_TXQ_DESC_NSGE_SHIFT) & IONIC_TXQ_DESC_NSGE_MASK;
+ *addr = (cmd >> IONIC_TXQ_DESC_ADDR_SHIFT) & IONIC_TXQ_DESC_ADDR_MASK;
+};
+
+/**
+ * struct ionic_txq_sg_elem - Transmit scatter-gather (SG) descriptor element
+ * @addr: DMA address of SG element data buffer
+ * @len: Length of SG element data buffer, in bytes
+ */
+struct ionic_txq_sg_elem {
+ __le64 addr;
+ __le16 len;
+ __le16 rsvd[3];
+};
+
+/**
+ * struct ionic_txq_sg_desc - Transmit scatter-gather (SG) list
+ * @elems: Scatter-gather elements
+ */
+struct ionic_txq_sg_desc {
+#define IONIC_TX_MAX_SG_ELEMS 8
+#define IONIC_TX_SG_DESC_STRIDE 8
+ struct ionic_txq_sg_elem elems[IONIC_TX_MAX_SG_ELEMS];
+};
+
+struct ionic_txq_sg_desc_v1 {
+#define IONIC_TX_MAX_SG_ELEMS_V1 15
+#define IONIC_TX_SG_DESC_STRIDE_V1 16
+ struct ionic_txq_sg_elem elems[IONIC_TX_SG_DESC_STRIDE_V1];
+};
+
+/**
+ * struct ionic_txq_comp - Ethernet transmit queue completion descriptor
+ * @status: Status of the command (enum ionic_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @color: Color bit
+ */
+struct ionic_txq_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ u8 rsvd2[11];
+ u8 color;
+};
+
+enum ionic_rxq_desc_opcode {
+ IONIC_RXQ_DESC_OPCODE_SIMPLE = 0,
+ IONIC_RXQ_DESC_OPCODE_SG = 1,
+};
+
+/**
+ * struct ionic_rxq_desc - Ethernet Rx queue descriptor format
+ * @opcode: Rx operation, see IONIC_RXQ_DESC_OPCODE_*:
+ *
+ * IONIC_RXQ_DESC_OPCODE_SIMPLE:
+ * Receive full packet into data buffer
+ * starting at @addr. Results of
+ * receive, including actual bytes received,
+ * are recorded in Rx completion descriptor.
+ *
+ * @len: Data buffer's length, in bytes
+ * @addr: Data buffer's DMA address
+ */
+struct ionic_rxq_desc {
+ u8 opcode;
+ u8 rsvd[5];
+ __le16 len;
+ __le64 addr;
+};
+
+/**
+ * struct ionic_rxq_sg_elem - Receive scatter-gather (SG) descriptor element
+ * @addr: DMA address of SG element data buffer
+ * @len: Length of SG element data buffer, in bytes
+ */
+struct ionic_rxq_sg_elem {
+ __le64 addr;
+ __le16 len;
+ __le16 rsvd[3];
+};
+
+/**
+ * struct ionic_rxq_sg_desc - Receive scatter-gather (SG) list
+ * @elems: Scatter-gather elements
+ */
+struct ionic_rxq_sg_desc {
+#define IONIC_RX_MAX_SG_ELEMS 8
+#define IONIC_RX_SG_DESC_STRIDE 8
+ struct ionic_rxq_sg_elem elems[IONIC_RX_SG_DESC_STRIDE];
+};
+
+/**
+ * struct ionic_rxq_comp - Ethernet receive queue completion descriptor
+ * @status: Status of the command (enum ionic_status_code)
+ * @num_sg_elems: Number of SG elements used by this descriptor
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @rss_hash: 32-bit RSS hash
+ * @csum: 16-bit sum of the packet's L2 payload
+ * If the packet's L2 payload is odd length, an extra
+ * zero-value byte is included in the @csum calculation but
+ * not included in @len.
+ * @vlan_tci: VLAN tag stripped from the packet. Valid if @VLAN is
+ * set. Includes .1p and .1q tags.
+ * @len: Received packet length, in bytes. Excludes FCS.
+ * @csum_calc L2 payload checksum is computed or not
+ * @csum_flags: See IONIC_RXQ_COMP_CSUM_F_*:
+ *
+ * IONIC_RXQ_COMP_CSUM_F_TCP_OK:
+ * The TCP checksum calculated by the device
+ * matched the checksum in the receive packet's
+ * TCP header.
+ *
+ * IONIC_RXQ_COMP_CSUM_F_TCP_BAD:
+ * The TCP checksum calculated by the device did
+ * not match the checksum in the receive packet's
+ * TCP header.
+ *
+ * IONIC_RXQ_COMP_CSUM_F_UDP_OK:
+ * The UDP checksum calculated by the device
+ * matched the checksum in the receive packet's
+ * UDP header
+ *
+ * IONIC_RXQ_COMP_CSUM_F_UDP_BAD:
+ * The UDP checksum calculated by the device did
+ * not match the checksum in the receive packet's
+ * UDP header.
+ *
+ * IONIC_RXQ_COMP_CSUM_F_IP_OK:
+ * The IPv4 checksum calculated by the device
+ * matched the checksum in the receive packet's
+ * first IPv4 header. If the receive packet
+ * contains both a tunnel IPv4 header and a
+ * transport IPv4 header, the device validates the
+ * checksum for the both IPv4 headers.
+ *
+ * IONIC_RXQ_COMP_CSUM_F_IP_BAD:
+ * The IPv4 checksum calculated by the device did
+ * not match the checksum in the receive packet's
+ * first IPv4 header. If the receive packet
+ * contains both a tunnel IPv4 header and a
+ * transport IPv4 header, the device validates the
+ * checksum for both IP headers.
+ *
+ * IONIC_RXQ_COMP_CSUM_F_VLAN:
+ * The VLAN header was stripped and placed in @vlan_tci.
+ *
+ * IONIC_RXQ_COMP_CSUM_F_CALC:
+ * The checksum was calculated by the device.
+ *
+ * @pkt_type_color: Packet type and color bit; see IONIC_RXQ_COMP_PKT_TYPE_MASK
+ */
+struct ionic_rxq_comp {
+ u8 status;
+ u8 num_sg_elems;
+ __le16 comp_index;
+ __le32 rss_hash;
+ __le16 csum;
+ __le16 vlan_tci;
+ __le16 len;
+ u8 csum_flags;
+#define IONIC_RXQ_COMP_CSUM_F_TCP_OK 0x01
+#define IONIC_RXQ_COMP_CSUM_F_TCP_BAD 0x02
+#define IONIC_RXQ_COMP_CSUM_F_UDP_OK 0x04
+#define IONIC_RXQ_COMP_CSUM_F_UDP_BAD 0x08
+#define IONIC_RXQ_COMP_CSUM_F_IP_OK 0x10
+#define IONIC_RXQ_COMP_CSUM_F_IP_BAD 0x20
+#define IONIC_RXQ_COMP_CSUM_F_VLAN 0x40
+#define IONIC_RXQ_COMP_CSUM_F_CALC 0x80
+ u8 pkt_type_color;
+#define IONIC_RXQ_COMP_PKT_TYPE_MASK 0x7f
+};
+
+enum ionic_pkt_type {
+ IONIC_PKT_TYPE_NON_IP = 0x00,
+ IONIC_PKT_TYPE_IPV4 = 0x01,
+ IONIC_PKT_TYPE_IPV4_TCP = 0x03,
+ IONIC_PKT_TYPE_IPV4_UDP = 0x05,
+ IONIC_PKT_TYPE_IPV6 = 0x08,
+ IONIC_PKT_TYPE_IPV6_TCP = 0x18,
+ IONIC_PKT_TYPE_IPV6_UDP = 0x28,
+ /* below types are only used if encap offloads are enabled on lif */
+ IONIC_PKT_TYPE_ENCAP_NON_IP = 0x40,
+ IONIC_PKT_TYPE_ENCAP_IPV4 = 0x41,
+ IONIC_PKT_TYPE_ENCAP_IPV4_TCP = 0x43,
+ IONIC_PKT_TYPE_ENCAP_IPV4_UDP = 0x45,
+ IONIC_PKT_TYPE_ENCAP_IPV6 = 0x48,
+ IONIC_PKT_TYPE_ENCAP_IPV6_TCP = 0x58,
+ IONIC_PKT_TYPE_ENCAP_IPV6_UDP = 0x68,
+};
+
+enum ionic_eth_hw_features {
+ IONIC_ETH_HW_VLAN_TX_TAG = BIT(0),
+ IONIC_ETH_HW_VLAN_RX_STRIP = BIT(1),
+ IONIC_ETH_HW_VLAN_RX_FILTER = BIT(2),
+ IONIC_ETH_HW_RX_HASH = BIT(3),
+ IONIC_ETH_HW_RX_CSUM = BIT(4),
+ IONIC_ETH_HW_TX_SG = BIT(5),
+ IONIC_ETH_HW_RX_SG = BIT(6),
+ IONIC_ETH_HW_TX_CSUM = BIT(7),
+ IONIC_ETH_HW_TSO = BIT(8),
+ IONIC_ETH_HW_TSO_IPV6 = BIT(9),
+ IONIC_ETH_HW_TSO_ECN = BIT(10),
+ IONIC_ETH_HW_TSO_GRE = BIT(11),
+ IONIC_ETH_HW_TSO_GRE_CSUM = BIT(12),
+ IONIC_ETH_HW_TSO_IPXIP4 = BIT(13),
+ IONIC_ETH_HW_TSO_IPXIP6 = BIT(14),
+ IONIC_ETH_HW_TSO_UDP = BIT(15),
+ IONIC_ETH_HW_TSO_UDP_CSUM = BIT(16),
+ IONIC_ETH_HW_RX_CSUM_GENEVE = BIT(17),
+ IONIC_ETH_HW_TX_CSUM_GENEVE = BIT(18),
+ IONIC_ETH_HW_TSO_GENEVE = BIT(19),
+ IONIC_ETH_HW_TIMESTAMP = BIT(20),
+};
+
+/**
+ * enum ionic_pkt_class - Packet classification mask.
+ *
+ * Used with rx steering filter, packets indicated by the mask can be steered
+ * toward a specific receive queue.
+ *
+ * @IONIC_PKT_CLS_NTP_ALL: All NTP packets.
+ * @IONIC_PKT_CLS_PTP1_SYNC: PTPv1 sync
+ * @IONIC_PKT_CLS_PTP1_DREQ: PTPv1 delay-request
+ * @IONIC_PKT_CLS_PTP1_ALL: PTPv1 all packets
+ * @IONIC_PKT_CLS_PTP2_L4_SYNC: PTPv2-UDP sync
+ * @IONIC_PKT_CLS_PTP2_L4_DREQ: PTPv2-UDP delay-request
+ * @IONIC_PKT_CLS_PTP2_L4_ALL: PTPv2-UDP all packets
+ * @IONIC_PKT_CLS_PTP2_L2_SYNC: PTPv2-ETH sync
+ * @IONIC_PKT_CLS_PTP2_L2_DREQ: PTPv2-ETH delay-request
+ * @IONIC_PKT_CLS_PTP2_L2_ALL: PTPv2-ETH all packets
+ * @IONIC_PKT_CLS_PTP2_SYNC: PTPv2 sync
+ * @IONIC_PKT_CLS_PTP2_DREQ: PTPv2 delay-request
+ * @IONIC_PKT_CLS_PTP2_ALL: PTPv2 all packets
+ * @IONIC_PKT_CLS_PTP_SYNC: PTP sync
+ * @IONIC_PKT_CLS_PTP_DREQ: PTP delay-request
+ * @IONIC_PKT_CLS_PTP_ALL: PTP all packets
+ */
+enum ionic_pkt_class {
+ IONIC_PKT_CLS_NTP_ALL = BIT(0),
+
+ IONIC_PKT_CLS_PTP1_SYNC = BIT(1),
+ IONIC_PKT_CLS_PTP1_DREQ = BIT(2),
+ IONIC_PKT_CLS_PTP1_ALL = BIT(3) |
+ IONIC_PKT_CLS_PTP1_SYNC | IONIC_PKT_CLS_PTP1_DREQ,
+
+ IONIC_PKT_CLS_PTP2_L4_SYNC = BIT(4),
+ IONIC_PKT_CLS_PTP2_L4_DREQ = BIT(5),
+ IONIC_PKT_CLS_PTP2_L4_ALL = BIT(6) |
+ IONIC_PKT_CLS_PTP2_L4_SYNC | IONIC_PKT_CLS_PTP2_L4_DREQ,
+
+ IONIC_PKT_CLS_PTP2_L2_SYNC = BIT(7),
+ IONIC_PKT_CLS_PTP2_L2_DREQ = BIT(8),
+ IONIC_PKT_CLS_PTP2_L2_ALL = BIT(9) |
+ IONIC_PKT_CLS_PTP2_L2_SYNC | IONIC_PKT_CLS_PTP2_L2_DREQ,
+
+ IONIC_PKT_CLS_PTP2_SYNC =
+ IONIC_PKT_CLS_PTP2_L4_SYNC | IONIC_PKT_CLS_PTP2_L2_SYNC,
+ IONIC_PKT_CLS_PTP2_DREQ =
+ IONIC_PKT_CLS_PTP2_L4_DREQ | IONIC_PKT_CLS_PTP2_L2_DREQ,
+ IONIC_PKT_CLS_PTP2_ALL =
+ IONIC_PKT_CLS_PTP2_L4_ALL | IONIC_PKT_CLS_PTP2_L2_ALL,
+
+ IONIC_PKT_CLS_PTP_SYNC =
+ IONIC_PKT_CLS_PTP1_SYNC | IONIC_PKT_CLS_PTP2_SYNC,
+ IONIC_PKT_CLS_PTP_DREQ =
+ IONIC_PKT_CLS_PTP1_DREQ | IONIC_PKT_CLS_PTP2_DREQ,
+ IONIC_PKT_CLS_PTP_ALL =
+ IONIC_PKT_CLS_PTP1_ALL | IONIC_PKT_CLS_PTP2_ALL,
+};
+
+/**
+ * struct ionic_q_control_cmd - Queue control command
+ * @opcode: opcode
+ * @type: Queue type
+ * @lif_index: LIF index
+ * @index: Queue index
+ * @oper: Operation (enum ionic_q_control_oper)
+ */
+struct ionic_q_control_cmd {
+ u8 opcode;
+ u8 type;
+ __le16 lif_index;
+ __le32 index;
+ u8 oper;
+ u8 rsvd[55];
+};
+
+typedef struct ionic_admin_comp ionic_q_control_comp;
+
+enum q_control_oper {
+ IONIC_Q_DISABLE = 0,
+ IONIC_Q_ENABLE = 1,
+ IONIC_Q_HANG_RESET = 2,
+};
+
+/**
+ * enum ionic_phy_type - Physical connection type
+ * @IONIC_PHY_TYPE_NONE: No PHY installed
+ * @IONIC_PHY_TYPE_COPPER: Copper PHY
+ * @IONIC_PHY_TYPE_FIBER: Fiber PHY
+ */
+enum ionic_phy_type {
+ IONIC_PHY_TYPE_NONE = 0,
+ IONIC_PHY_TYPE_COPPER = 1,
+ IONIC_PHY_TYPE_FIBER = 2,
+};
+
+/**
+ * enum ionic_xcvr_state - Transceiver status
+ * @IONIC_XCVR_STATE_REMOVED: Transceiver removed
+ * @IONIC_XCVR_STATE_INSERTED: Transceiver inserted
+ * @IONIC_XCVR_STATE_PENDING: Transceiver pending
+ * @IONIC_XCVR_STATE_SPROM_READ: Transceiver data read
+ * @IONIC_XCVR_STATE_SPROM_READ_ERR: Transceiver data read error
+ */
+enum ionic_xcvr_state {
+ IONIC_XCVR_STATE_REMOVED = 0,
+ IONIC_XCVR_STATE_INSERTED = 1,
+ IONIC_XCVR_STATE_PENDING = 2,
+ IONIC_XCVR_STATE_SPROM_READ = 3,
+ IONIC_XCVR_STATE_SPROM_READ_ERR = 4,
+};
+
+/**
+ * enum ionic_xcvr_pid - Supported link modes
+ */
+enum ionic_xcvr_pid {
+ IONIC_XCVR_PID_UNKNOWN = 0,
+
+ /* CU */
+ IONIC_XCVR_PID_QSFP_100G_CR4 = 1,
+ IONIC_XCVR_PID_QSFP_40GBASE_CR4 = 2,
+ IONIC_XCVR_PID_SFP_25GBASE_CR_S = 3,
+ IONIC_XCVR_PID_SFP_25GBASE_CR_L = 4,
+ IONIC_XCVR_PID_SFP_25GBASE_CR_N = 5,
+
+ /* Fiber */
+ IONIC_XCVR_PID_QSFP_100G_AOC = 50,
+ IONIC_XCVR_PID_QSFP_100G_ACC = 51,
+ IONIC_XCVR_PID_QSFP_100G_SR4 = 52,
+ IONIC_XCVR_PID_QSFP_100G_LR4 = 53,
+ IONIC_XCVR_PID_QSFP_100G_ER4 = 54,
+ IONIC_XCVR_PID_QSFP_40GBASE_ER4 = 55,
+ IONIC_XCVR_PID_QSFP_40GBASE_SR4 = 56,
+ IONIC_XCVR_PID_QSFP_40GBASE_LR4 = 57,
+ IONIC_XCVR_PID_QSFP_40GBASE_AOC = 58,
+ IONIC_XCVR_PID_SFP_25GBASE_SR = 59,
+ IONIC_XCVR_PID_SFP_25GBASE_LR = 60,
+ IONIC_XCVR_PID_SFP_25GBASE_ER = 61,
+ IONIC_XCVR_PID_SFP_25GBASE_AOC = 62,
+ IONIC_XCVR_PID_SFP_10GBASE_SR = 63,
+ IONIC_XCVR_PID_SFP_10GBASE_LR = 64,
+ IONIC_XCVR_PID_SFP_10GBASE_LRM = 65,
+ IONIC_XCVR_PID_SFP_10GBASE_ER = 66,
+ IONIC_XCVR_PID_SFP_10GBASE_AOC = 67,
+ IONIC_XCVR_PID_SFP_10GBASE_CU = 68,
+ IONIC_XCVR_PID_QSFP_100G_CWDM4 = 69,
+ IONIC_XCVR_PID_QSFP_100G_PSM4 = 70,
+ IONIC_XCVR_PID_SFP_25GBASE_ACC = 71,
+ IONIC_XCVR_PID_SFP_10GBASE_T = 72,
+ IONIC_XCVR_PID_SFP_1000BASE_T = 73,
+};
+
+/**
+ * enum ionic_port_type - Port types
+ * @IONIC_PORT_TYPE_NONE: Port type not configured
+ * @IONIC_PORT_TYPE_ETH: Port carries ethernet traffic (inband)
+ * @IONIC_PORT_TYPE_MGMT: Port carries mgmt traffic (out-of-band)
+ */
+enum ionic_port_type {
+ IONIC_PORT_TYPE_NONE = 0,
+ IONIC_PORT_TYPE_ETH = 1,
+ IONIC_PORT_TYPE_MGMT = 2,
+};
+
+/**
+ * enum ionic_port_admin_state - Port config state
+ * @IONIC_PORT_ADMIN_STATE_NONE: Port admin state not configured
+ * @IONIC_PORT_ADMIN_STATE_DOWN: Port admin disabled
+ * @IONIC_PORT_ADMIN_STATE_UP: Port admin enabled
+ */
+enum ionic_port_admin_state {
+ IONIC_PORT_ADMIN_STATE_NONE = 0,
+ IONIC_PORT_ADMIN_STATE_DOWN = 1,
+ IONIC_PORT_ADMIN_STATE_UP = 2,
+};
+
+/**
+ * enum ionic_port_oper_status - Port operational status
+ * @IONIC_PORT_OPER_STATUS_NONE: Port disabled
+ * @IONIC_PORT_OPER_STATUS_UP: Port link status up
+ * @IONIC_PORT_OPER_STATUS_DOWN: Port link status down
+ */
+enum ionic_port_oper_status {
+ IONIC_PORT_OPER_STATUS_NONE = 0,
+ IONIC_PORT_OPER_STATUS_UP = 1,
+ IONIC_PORT_OPER_STATUS_DOWN = 2,
+};
+
+/**
+ * enum ionic_port_fec_type - Ethernet Forward error correction (FEC) modes
+ * @IONIC_PORT_FEC_TYPE_NONE: FEC Disabled
+ * @IONIC_PORT_FEC_TYPE_FC: FireCode FEC
+ * @IONIC_PORT_FEC_TYPE_RS: ReedSolomon FEC
+ */
+enum ionic_port_fec_type {
+ IONIC_PORT_FEC_TYPE_NONE = 0,
+ IONIC_PORT_FEC_TYPE_FC = 1,
+ IONIC_PORT_FEC_TYPE_RS = 2,
+};
+
+/**
+ * enum ionic_port_pause_type - Ethernet pause (flow control) modes
+ * @IONIC_PORT_PAUSE_TYPE_NONE: Disable Pause
+ * @IONIC_PORT_PAUSE_TYPE_LINK: Link level pause
+ * @IONIC_PORT_PAUSE_TYPE_PFC: Priority-Flow Control
+ */
+enum ionic_port_pause_type {
+ IONIC_PORT_PAUSE_TYPE_NONE = 0,
+ IONIC_PORT_PAUSE_TYPE_LINK = 1,
+ IONIC_PORT_PAUSE_TYPE_PFC = 2,
+};
+
+/**
+ * enum ionic_port_loopback_mode - Loopback modes
+ * @IONIC_PORT_LOOPBACK_MODE_NONE: Disable loopback
+ * @IONIC_PORT_LOOPBACK_MODE_MAC: MAC loopback
+ * @IONIC_PORT_LOOPBACK_MODE_PHY: PHY/SerDes loopback
+ */
+enum ionic_port_loopback_mode {
+ IONIC_PORT_LOOPBACK_MODE_NONE = 0,
+ IONIC_PORT_LOOPBACK_MODE_MAC = 1,
+ IONIC_PORT_LOOPBACK_MODE_PHY = 2,
+};
+
+/**
+ * struct ionic_xcvr_status - Transceiver Status information
+ * @state: Transceiver status (enum ionic_xcvr_state)
+ * @phy: Physical connection type (enum ionic_phy_type)
+ * @pid: Transceiver link mode (enum ionic_xcvr_pid)
+ * @sprom: Transceiver sprom contents
+ */
+struct ionic_xcvr_status {
+ u8 state;
+ u8 phy;
+ __le16 pid;
+ u8 sprom[256];
+};
+
+/**
+ * union ionic_port_config - Port configuration
+ * @speed: port speed (in Mbps)
+ * @mtu: mtu
+ * @state: port admin state (enum ionic_port_admin_state)
+ * @an_enable: autoneg enable
+ * @fec_type: fec type (enum ionic_port_fec_type)
+ * @pause_type: pause type (enum ionic_port_pause_type)
+ * @loopback_mode: loopback mode (enum ionic_port_loopback_mode)
+ */
+union ionic_port_config {
+ struct {
+#define IONIC_SPEED_100G 100000 /* 100G in Mbps */
+#define IONIC_SPEED_50G 50000 /* 50G in Mbps */
+#define IONIC_SPEED_40G 40000 /* 40G in Mbps */
+#define IONIC_SPEED_25G 25000 /* 25G in Mbps */
+#define IONIC_SPEED_10G 10000 /* 10G in Mbps */
+#define IONIC_SPEED_1G 1000 /* 1G in Mbps */
+ __le32 speed;
+ __le32 mtu;
+ u8 state;
+ u8 an_enable;
+ u8 fec_type;
+#define IONIC_PAUSE_TYPE_MASK 0x0f
+#define IONIC_PAUSE_FLAGS_MASK 0xf0
+#define IONIC_PAUSE_F_TX 0x10
+#define IONIC_PAUSE_F_RX 0x20
+ u8 pause_type;
+ u8 loopback_mode;
+ };
+ __le32 words[64];
+};
+
+/**
+ * struct ionic_port_status - Port Status information
+ * @status: link status (enum ionic_port_oper_status)
+ * @id: port id
+ * @speed: link speed (in Mbps)
+ * @link_down_count: number of times link went from up to down
+ * @fec_type: fec type (enum ionic_port_fec_type)
+ * @xcvr: transceiver status
+ */
+struct ionic_port_status {
+ __le32 id;
+ __le32 speed;
+ u8 status;
+ __le16 link_down_count;
+ u8 fec_type;
+ u8 rsvd[48];
+ struct ionic_xcvr_status xcvr;
+} __packed;
+
+/**
+ * struct ionic_port_identify_cmd - Port identify command
+ * @opcode: opcode
+ * @index: port index
+ * @ver: Highest version of identify supported by driver
+ */
+struct ionic_port_identify_cmd {
+ u8 opcode;
+ u8 index;
+ u8 ver;
+ u8 rsvd[61];
+};
+
+/**
+ * struct ionic_port_identify_comp - Port identify command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @ver: Version of identify returned by device
+ */
+struct ionic_port_identify_comp {
+ u8 status;
+ u8 ver;
+ u8 rsvd[14];
+};
+
+/**
+ * struct ionic_port_init_cmd - Port initialization command
+ * @opcode: opcode
+ * @index: port index
+ * @info_pa: destination address for port info (struct ionic_port_info)
+ */
+struct ionic_port_init_cmd {
+ u8 opcode;
+ u8 index;
+ u8 rsvd[6];
+ __le64 info_pa;
+ u8 rsvd2[48];
+};
+
+/**
+ * struct ionic_port_init_comp - Port initialization command completion
+ * @status: Status of the command (enum ionic_status_code)
+ */
+struct ionic_port_init_comp {
+ u8 status;
+ u8 rsvd[15];
+};
+
+/**
+ * struct ionic_port_reset_cmd - Port reset command
+ * @opcode: opcode
+ * @index: port index
+ */
+struct ionic_port_reset_cmd {
+ u8 opcode;
+ u8 index;
+ u8 rsvd[62];
+};
+
+/**
+ * struct ionic_port_reset_comp - Port reset command completion
+ * @status: Status of the command (enum ionic_status_code)
+ */
+struct ionic_port_reset_comp {
+ u8 status;
+ u8 rsvd[15];
+};
+
+/**
+ * enum ionic_stats_ctl_cmd - List of commands for stats control
+ * @IONIC_STATS_CTL_RESET: Reset statistics
+ */
+enum ionic_stats_ctl_cmd {
+ IONIC_STATS_CTL_RESET = 0,
+};
+
+/**
+ * enum ionic_txstamp_mode - List of TX Timestamping Modes
+ * @IONIC_TXSTAMP_OFF: Disable TX hardware timetamping.
+ * @IONIC_TXSTAMP_ON: Enable local TX hardware timetamping.
+ * @IONIC_TXSTAMP_ONESTEP_SYNC: Modify TX PTP Sync packets.
+ * @IONIC_TXSTAMP_ONESTEP_P2P: Modify TX PTP Sync and PDelayResp.
+ */
+enum ionic_txstamp_mode {
+ IONIC_TXSTAMP_OFF = 0,
+ IONIC_TXSTAMP_ON = 1,
+ IONIC_TXSTAMP_ONESTEP_SYNC = 2,
+ IONIC_TXSTAMP_ONESTEP_P2P = 3,
+};
+
+/**
+ * enum ionic_port_attr - List of device attributes
+ * @IONIC_PORT_ATTR_STATE: Port state attribute
+ * @IONIC_PORT_ATTR_SPEED: Port speed attribute
+ * @IONIC_PORT_ATTR_MTU: Port MTU attribute
+ * @IONIC_PORT_ATTR_AUTONEG: Port autonegotiation attribute
+ * @IONIC_PORT_ATTR_FEC: Port FEC attribute
+ * @IONIC_PORT_ATTR_PAUSE: Port pause attribute
+ * @IONIC_PORT_ATTR_LOOPBACK: Port loopback attribute
+ * @IONIC_PORT_ATTR_STATS_CTRL: Port statistics control attribute
+ */
+enum ionic_port_attr {
+ IONIC_PORT_ATTR_STATE = 0,
+ IONIC_PORT_ATTR_SPEED = 1,
+ IONIC_PORT_ATTR_MTU = 2,
+ IONIC_PORT_ATTR_AUTONEG = 3,
+ IONIC_PORT_ATTR_FEC = 4,
+ IONIC_PORT_ATTR_PAUSE = 5,
+ IONIC_PORT_ATTR_LOOPBACK = 6,
+ IONIC_PORT_ATTR_STATS_CTRL = 7,
+};
+
+/**
+ * struct ionic_port_setattr_cmd - Set port attributes on the NIC
+ * @opcode: Opcode
+ * @index: Port index
+ * @attr: Attribute type (enum ionic_port_attr)
+ * @state: Port state
+ * @speed: Port speed
+ * @mtu: Port MTU
+ * @an_enable: Port autonegotiation setting
+ * @fec_type: Port FEC type setting
+ * @pause_type: Port pause type setting
+ * @loopback_mode: Port loopback mode
+ * @stats_ctl: Port stats setting
+ */
+struct ionic_port_setattr_cmd {
+ u8 opcode;
+ u8 index;
+ u8 attr;
+ u8 rsvd;
+ union {
+ u8 state;
+ __le32 speed;
+ __le32 mtu;
+ u8 an_enable;
+ u8 fec_type;
+ u8 pause_type;
+ u8 loopback_mode;
+ u8 stats_ctl;
+ u8 rsvd2[60];
+ };
+};
+
+/**
+ * struct ionic_port_setattr_comp - Port set attr command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @color: Color bit
+ */
+struct ionic_port_setattr_comp {
+ u8 status;
+ u8 rsvd[14];
+ u8 color;
+};
+
+/**
+ * struct ionic_port_getattr_cmd - Get port attributes from the NIC
+ * @opcode: Opcode
+ * @index: port index
+ * @attr: Attribute type (enum ionic_port_attr)
+ */
+struct ionic_port_getattr_cmd {
+ u8 opcode;
+ u8 index;
+ u8 attr;
+ u8 rsvd[61];
+};
+
+/**
+ * struct ionic_port_getattr_comp - Port get attr command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @state: Port state
+ * @speed: Port speed
+ * @mtu: Port MTU
+ * @an_enable: Port autonegotiation setting
+ * @fec_type: Port FEC type setting
+ * @pause_type: Port pause type setting
+ * @loopback_mode: Port loopback mode
+ * @color: Color bit
+ */
+struct ionic_port_getattr_comp {
+ u8 status;
+ u8 rsvd[3];
+ union {
+ u8 state;
+ __le32 speed;
+ __le32 mtu;
+ u8 an_enable;
+ u8 fec_type;
+ u8 pause_type;
+ u8 loopback_mode;
+ u8 rsvd2[11];
+ } __packed;
+ u8 color;
+};
+
+/**
+ * struct ionic_lif_status - LIF status register
+ * @eid: most recent NotifyQ event id
+ * @port_num: port the LIF is connected to
+ * @link_status: port status (enum ionic_port_oper_status)
+ * @link_speed: speed of link in Mbps
+ * @link_down_count: number of times link went from up to down
+ */
+struct ionic_lif_status {
+ __le64 eid;
+ u8 port_num;
+ u8 rsvd;
+ __le16 link_status;
+ __le32 link_speed; /* units of 1Mbps: eg 10000 = 10Gbps */
+ __le16 link_down_count;
+ u8 rsvd2[46];
+};
+
+/**
+ * struct ionic_lif_reset_cmd - LIF reset command
+ * @opcode: opcode
+ * @index: LIF index
+ */
+struct ionic_lif_reset_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 index;
+ __le32 rsvd2[15];
+};
+
+typedef struct ionic_admin_comp ionic_lif_reset_comp;
+
+enum ionic_dev_state {
+ IONIC_DEV_DISABLE = 0,
+ IONIC_DEV_ENABLE = 1,
+ IONIC_DEV_HANG_RESET = 2,
+};
+
+/**
+ * enum ionic_dev_attr - List of device attributes
+ * @IONIC_DEV_ATTR_STATE: Device state attribute
+ * @IONIC_DEV_ATTR_NAME: Device name attribute
+ * @IONIC_DEV_ATTR_FEATURES: Device feature attributes
+ */
+enum ionic_dev_attr {
+ IONIC_DEV_ATTR_STATE = 0,
+ IONIC_DEV_ATTR_NAME = 1,
+ IONIC_DEV_ATTR_FEATURES = 2,
+};
+
+/**
+ * struct ionic_dev_setattr_cmd - Set Device attributes on the NIC
+ * @opcode: Opcode
+ * @attr: Attribute type (enum ionic_dev_attr)
+ * @state: Device state (enum ionic_dev_state)
+ * @name: The bus info, e.g. PCI slot-device-function, 0 terminated
+ * @features: Device features
+ */
+struct ionic_dev_setattr_cmd {
+ u8 opcode;
+ u8 attr;
+ __le16 rsvd;
+ union {
+ u8 state;
+ char name[IONIC_IFNAMSIZ];
+ __le64 features;
+ u8 rsvd2[60];
+ } __packed;
+};
+
+/**
+ * struct ionic_dev_setattr_comp - Device set attr command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @features: Device features
+ * @color: Color bit
+ */
+struct ionic_dev_setattr_comp {
+ u8 status;
+ u8 rsvd[3];
+ union {
+ __le64 features;
+ u8 rsvd2[11];
+ } __packed;
+ u8 color;
+};
+
+/**
+ * struct ionic_dev_getattr_cmd - Get Device attributes from the NIC
+ * @opcode: opcode
+ * @attr: Attribute type (enum ionic_dev_attr)
+ */
+struct ionic_dev_getattr_cmd {
+ u8 opcode;
+ u8 attr;
+ u8 rsvd[62];
+};
+
+/**
+ * struct ionic_dev_setattr_comp - Device set attr command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @features: Device features
+ * @color: Color bit
+ */
+struct ionic_dev_getattr_comp {
+ u8 status;
+ u8 rsvd[3];
+ union {
+ __le64 features;
+ u8 rsvd2[11];
+ } __packed;
+ u8 color;
+};
+
+/**
+ * RSS parameters
+ */
+#define IONIC_RSS_HASH_KEY_SIZE 40
+
+enum ionic_rss_hash_types {
+ IONIC_RSS_TYPE_IPV4 = BIT(0),
+ IONIC_RSS_TYPE_IPV4_TCP = BIT(1),
+ IONIC_RSS_TYPE_IPV4_UDP = BIT(2),
+ IONIC_RSS_TYPE_IPV6 = BIT(3),
+ IONIC_RSS_TYPE_IPV6_TCP = BIT(4),
+ IONIC_RSS_TYPE_IPV6_UDP = BIT(5),
+};
+
+/**
+ * enum ionic_lif_attr - List of LIF attributes
+ * @IONIC_LIF_ATTR_STATE: LIF state attribute
+ * @IONIC_LIF_ATTR_NAME: LIF name attribute
+ * @IONIC_LIF_ATTR_MTU: LIF MTU attribute
+ * @IONIC_LIF_ATTR_MAC: LIF MAC attribute
+ * @IONIC_LIF_ATTR_FEATURES: LIF features attribute
+ * @IONIC_LIF_ATTR_RSS: LIF RSS attribute
+ * @IONIC_LIF_ATTR_STATS_CTRL: LIF statistics control attribute
+ * @IONIC_LIF_ATTR_TXSTAMP: LIF TX timestamping mode
+ */
+enum ionic_lif_attr {
+ IONIC_LIF_ATTR_STATE = 0,
+ IONIC_LIF_ATTR_NAME = 1,
+ IONIC_LIF_ATTR_MTU = 2,
+ IONIC_LIF_ATTR_MAC = 3,
+ IONIC_LIF_ATTR_FEATURES = 4,
+ IONIC_LIF_ATTR_RSS = 5,
+ IONIC_LIF_ATTR_STATS_CTRL = 6,
+ IONIC_LIF_ATTR_TXSTAMP = 7,
+};
+
+/**
+ * struct ionic_lif_setattr_cmd - Set LIF attributes on the NIC
+ * @opcode: Opcode
+ * @attr: Attribute type (enum ionic_lif_attr)
+ * @index: LIF index
+ * @state: LIF state (enum ionic_lif_state)
+ * @name: The netdev name string, 0 terminated
+ * @mtu: Mtu
+ * @mac: Station mac
+ * @features: Features (enum ionic_eth_hw_features)
+ * @rss: RSS properties
+ * @types: The hash types to enable (see rss_hash_types)
+ * @key: The hash secret key
+ * @addr: Address for the indirection table shared memory
+ * @stats_ctl: stats control commands (enum ionic_stats_ctl_cmd)
+ * @txstamp: TX Timestamping Mode (enum ionic_txstamp_mode)
+ */
+struct ionic_lif_setattr_cmd {
+ u8 opcode;
+ u8 attr;
+ __le16 index;
+ union {
+ u8 state;
+ char name[IONIC_IFNAMSIZ];
+ __le32 mtu;
+ u8 mac[6];
+ __le64 features;
+ struct {
+ __le16 types;
+ u8 key[IONIC_RSS_HASH_KEY_SIZE];
+ u8 rsvd[6];
+ __le64 addr;
+ } rss;
+ u8 stats_ctl;
+ __le16 txstamp_mode;
+ u8 rsvd[60];
+ } __packed;
+};
+
+/**
+ * struct ionic_lif_setattr_comp - LIF set attr command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @features: features (enum ionic_eth_hw_features)
+ * @color: Color bit
+ */
+struct ionic_lif_setattr_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ union {
+ __le64 features;
+ u8 rsvd2[11];
+ } __packed;
+ u8 color;
+};
+
+/**
+ * struct ionic_lif_getattr_cmd - Get LIF attributes from the NIC
+ * @opcode: Opcode
+ * @attr: Attribute type (enum ionic_lif_attr)
+ * @index: LIF index
+ */
+struct ionic_lif_getattr_cmd {
+ u8 opcode;
+ u8 attr;
+ __le16 index;
+ u8 rsvd[60];
+};
+
+/**
+ * struct ionic_lif_getattr_comp - LIF get attr command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @state: LIF state (enum ionic_lif_state)
+ * @name: The netdev name string, 0 terminated
+ * @mtu: Mtu
+ * @mac: Station mac
+ * @features: Features (enum ionic_eth_hw_features)
+ * @txstamp: TX Timestamping Mode (enum ionic_txstamp_mode)
+ * @color: Color bit
+ */
+struct ionic_lif_getattr_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ union {
+ u8 state;
+ __le32 mtu;
+ u8 mac[6];
+ __le64 features;
+ __le16 txstamp_mode;
+ u8 rsvd2[11];
+ } __packed;
+ u8 color;
+};
+
+/**
+ * struct ionic_lif_setphc_cmd - Set LIF PTP Hardware Clock
+ * @opcode: Opcode
+ * @lif_index: LIF index
+ * @tick: Hardware stamp tick of an instant in time.
+ * @nsec: Nanosecond stamp of the same instant.
+ * @frac: Fractional nanoseconds at the same instant.
+ * @mult: Cycle to nanosecond multiplier.
+ * @shift: Cycle to nanosecond divisor (power of two).
+ */
+struct ionic_lif_setphc_cmd {
+ u8 opcode;
+ u8 rsvd1;
+ __le16 lif_index;
+ u8 rsvd2[4];
+ __le64 tick;
+ __le64 nsec;
+ __le64 frac;
+ __le32 mult;
+ __le32 shift;
+ u8 rsvd3[24];
+};
+
+enum ionic_rx_mode {
+ IONIC_RX_MODE_F_UNICAST = BIT(0),
+ IONIC_RX_MODE_F_MULTICAST = BIT(1),
+ IONIC_RX_MODE_F_BROADCAST = BIT(2),
+ IONIC_RX_MODE_F_PROMISC = BIT(3),
+ IONIC_RX_MODE_F_ALLMULTI = BIT(4),
+ IONIC_RX_MODE_F_RDMA_SNIFFER = BIT(5),
+};
+
+/**
+ * struct ionic_rx_mode_set_cmd - Set LIF's Rx mode command
+ * @opcode: opcode
+ * @lif_index: LIF index
+ * @rx_mode: Rx mode flags:
+ * IONIC_RX_MODE_F_UNICAST: Accept known unicast packets
+ * IONIC_RX_MODE_F_MULTICAST: Accept known multicast packets
+ * IONIC_RX_MODE_F_BROADCAST: Accept broadcast packets
+ * IONIC_RX_MODE_F_PROMISC: Accept any packets
+ * IONIC_RX_MODE_F_ALLMULTI: Accept any multicast packets
+ * IONIC_RX_MODE_F_RDMA_SNIFFER: Sniff RDMA packets
+ */
+struct ionic_rx_mode_set_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 lif_index;
+ __le16 rx_mode;
+ __le16 rsvd2[29];
+};
+
+typedef struct ionic_admin_comp ionic_rx_mode_set_comp;
+
+enum ionic_rx_filter_match_type {
+ IONIC_RX_FILTER_MATCH_VLAN = 0x0,
+ IONIC_RX_FILTER_MATCH_MAC = 0x1,
+ IONIC_RX_FILTER_MATCH_MAC_VLAN = 0x2,
+ IONIC_RX_FILTER_STEER_PKTCLASS = 0x10,
+};
+
+/**
+ * struct ionic_rx_filter_add_cmd - Add LIF Rx filter command
+ * @opcode: opcode
+ * @qtype: Queue type
+ * @lif_index: LIF index
+ * @qid: Queue ID
+ * @match: Rx filter match type (see IONIC_RX_FILTER_MATCH_xxx)
+ * @vlan: VLAN filter
+ * @vlan: VLAN ID
+ * @mac: MAC filter
+ * @addr: MAC address (network-byte order)
+ * @mac_vlan: MACVLAN filter
+ * @vlan: VLAN ID
+ * @addr: MAC address (network-byte order)
+ * @pkt_class: Packet classification filter
+ */
+struct ionic_rx_filter_add_cmd {
+ u8 opcode;
+ u8 qtype;
+ __le16 lif_index;
+ __le32 qid;
+ __le16 match;
+ union {
+ struct {
+ __le16 vlan;
+ } vlan;
+ struct {
+ u8 addr[6];
+ } mac;
+ struct {
+ __le16 vlan;
+ u8 addr[6];
+ } mac_vlan;
+ __le64 pkt_class;
+ u8 rsvd[54];
+ } __packed;
+};
+
+/**
+ * struct ionic_rx_filter_add_comp - Add LIF Rx filter command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @filter_id: Filter ID
+ * @color: Color bit
+ */
+struct ionic_rx_filter_add_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ __le32 filter_id;
+ u8 rsvd2[7];
+ u8 color;
+};
+
+/**
+ * struct ionic_rx_filter_del_cmd - Delete LIF Rx filter command
+ * @opcode: opcode
+ * @lif_index: LIF index
+ * @filter_id: Filter ID
+ */
+struct ionic_rx_filter_del_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 lif_index;
+ __le32 filter_id;
+ u8 rsvd2[56];
+};
+
+typedef struct ionic_admin_comp ionic_rx_filter_del_comp;
+
+enum ionic_vf_attr {
+ IONIC_VF_ATTR_SPOOFCHK = 1,
+ IONIC_VF_ATTR_TRUST = 2,
+ IONIC_VF_ATTR_MAC = 3,
+ IONIC_VF_ATTR_LINKSTATE = 4,
+ IONIC_VF_ATTR_VLAN = 5,
+ IONIC_VF_ATTR_RATE = 6,
+ IONIC_VF_ATTR_STATSADDR = 7,
+};
+
+/**
+ * enum ionic_vf_link_status - Virtual Function link status
+ * @IONIC_VF_LINK_STATUS_AUTO: Use link state of the uplink
+ * @IONIC_VF_LINK_STATUS_UP: Link always up
+ * @IONIC_VF_LINK_STATUS_DOWN: Link always down
+ */
+enum ionic_vf_link_status {
+ IONIC_VF_LINK_STATUS_AUTO = 0,
+ IONIC_VF_LINK_STATUS_UP = 1,
+ IONIC_VF_LINK_STATUS_DOWN = 2,
+};
+
+/**
+ * struct ionic_vf_setattr_cmd - Set VF attributes on the NIC
+ * @opcode: Opcode
+ * @attr: Attribute type (enum ionic_vf_attr)
+ * @vf_index: VF index
+ * @macaddr: mac address
+ * @vlanid: vlan ID
+ * @maxrate: max Tx rate in Mbps
+ * @spoofchk: enable address spoof checking
+ * @trust: enable VF trust
+ * @linkstate: set link up or down
+ * @stats_pa: set DMA address for VF stats
+ */
+struct ionic_vf_setattr_cmd {
+ u8 opcode;
+ u8 attr;
+ __le16 vf_index;
+ union {
+ u8 macaddr[6];
+ __le16 vlanid;
+ __le32 maxrate;
+ u8 spoofchk;
+ u8 trust;
+ u8 linkstate;
+ __le64 stats_pa;
+ u8 pad[60];
+ } __packed;
+};
+
+struct ionic_vf_setattr_comp {
+ u8 status;
+ u8 attr;
+ __le16 vf_index;
+ __le16 comp_index;
+ u8 rsvd[9];
+ u8 color;
+};
+
+/**
+ * struct ionic_vf_getattr_cmd - Get VF attributes from the NIC
+ * @opcode: Opcode
+ * @attr: Attribute type (enum ionic_vf_attr)
+ * @vf_index: VF index
+ */
+struct ionic_vf_getattr_cmd {
+ u8 opcode;
+ u8 attr;
+ __le16 vf_index;
+ u8 rsvd[60];
+};
+
+struct ionic_vf_getattr_comp {
+ u8 status;
+ u8 attr;
+ __le16 vf_index;
+ union {
+ u8 macaddr[6];
+ __le16 vlanid;
+ __le32 maxrate;
+ u8 spoofchk;
+ u8 trust;
+ u8 linkstate;
+ __le64 stats_pa;
+ u8 pad[11];
+ } __packed;
+ u8 color;
+};
+
+/**
+ * struct ionic_qos_identify_cmd - QoS identify command
+ * @opcode: opcode
+ * @ver: Highest version of identify supported by driver
+ *
+ */
+struct ionic_qos_identify_cmd {
+ u8 opcode;
+ u8 ver;
+ u8 rsvd[62];
+};
+
+/**
+ * struct ionic_qos_identify_comp - QoS identify command completion
+ * @status: Status of the command (enum ionic_status_code)
+ * @ver: Version of identify returned by device
+ */
+struct ionic_qos_identify_comp {
+ u8 status;
+ u8 ver;
+ u8 rsvd[14];
+};
+
+#define IONIC_QOS_TC_MAX 8
+#define IONIC_QOS_ALL_TC 0xFF
+/* Capri max supported, should be renamed. */
+#define IONIC_QOS_CLASS_MAX 7
+#define IONIC_QOS_PCP_MAX 8
+#define IONIC_QOS_CLASS_NAME_SZ 32
+#define IONIC_QOS_DSCP_MAX 64
+#define IONIC_QOS_ALL_PCP 0xFF
+#define IONIC_DSCP_BLOCK_SIZE 8
+
+/**
+ * enum ionic_qos_class
+ */
+enum ionic_qos_class {
+ IONIC_QOS_CLASS_DEFAULT = 0,
+ IONIC_QOS_CLASS_USER_DEFINED_1 = 1,
+ IONIC_QOS_CLASS_USER_DEFINED_2 = 2,
+ IONIC_QOS_CLASS_USER_DEFINED_3 = 3,
+ IONIC_QOS_CLASS_USER_DEFINED_4 = 4,
+ IONIC_QOS_CLASS_USER_DEFINED_5 = 5,
+ IONIC_QOS_CLASS_USER_DEFINED_6 = 6,
+};
+
+/**
+ * enum ionic_qos_class_type - Traffic classification criteria
+ * @IONIC_QOS_CLASS_TYPE_NONE: No QoS
+ * @IONIC_QOS_CLASS_TYPE_PCP: Dot1Q PCP
+ * @IONIC_QOS_CLASS_TYPE_DSCP: IP DSCP
+ */
+enum ionic_qos_class_type {
+ IONIC_QOS_CLASS_TYPE_NONE = 0,
+ IONIC_QOS_CLASS_TYPE_PCP = 1,
+ IONIC_QOS_CLASS_TYPE_DSCP = 2,
+};
+
+/**
+ * enum ionic_qos_sched_type - QoS class scheduling type
+ * @IONIC_QOS_SCHED_TYPE_STRICT: Strict priority
+ * @IONIC_QOS_SCHED_TYPE_DWRR: Deficit weighted round-robin
+ */
+enum ionic_qos_sched_type {
+ IONIC_QOS_SCHED_TYPE_STRICT = 0,
+ IONIC_QOS_SCHED_TYPE_DWRR = 1,
+};
+
+/**
+ * union ionic_qos_config - QoS configuration structure
+ * @flags: Configuration flags
+ * IONIC_QOS_CONFIG_F_ENABLE enable
+ * IONIC_QOS_CONFIG_F_NO_DROP drop/nodrop
+ * IONIC_QOS_CONFIG_F_RW_DOT1Q_PCP enable dot1q pcp rewrite
+ * IONIC_QOS_CONFIG_F_RW_IP_DSCP enable ip dscp rewrite
+ * IONIC_QOS_CONFIG_F_NON_DISRUPTIVE Non-disruptive TC update
+ * @sched_type: QoS class scheduling type (enum ionic_qos_sched_type)
+ * @class_type: QoS class type (enum ionic_qos_class_type)
+ * @pause_type: QoS pause type (enum ionic_qos_pause_type)
+ * @name: QoS class name
+ * @mtu: MTU of the class
+ * @pfc_cos: Priority-Flow Control class of service
+ * @dwrr_weight: QoS class scheduling weight
+ * @strict_rlmt: Rate limit for strict priority scheduling
+ * @rw_dot1q_pcp: Rewrite dot1q pcp to value (valid iff F_RW_DOT1Q_PCP)
+ * @rw_ip_dscp: Rewrite ip dscp to value (valid iff F_RW_IP_DSCP)
+ * @dot1q_pcp: Dot1q pcp value
+ * @ndscp: Number of valid dscp values in the ip_dscp field
+ * @ip_dscp: IP dscp values
+ */
+union ionic_qos_config {
+ struct {
+#define IONIC_QOS_CONFIG_F_ENABLE BIT(0)
+#define IONIC_QOS_CONFIG_F_NO_DROP BIT(1)
+/* Used to rewrite PCP or DSCP value. */
+#define IONIC_QOS_CONFIG_F_RW_DOT1Q_PCP BIT(2)
+#define IONIC_QOS_CONFIG_F_RW_IP_DSCP BIT(3)
+/* Non-disruptive TC update */
+#define IONIC_QOS_CONFIG_F_NON_DISRUPTIVE BIT(4)
+ u8 flags;
+ u8 sched_type;
+ u8 class_type;
+ u8 pause_type;
+ char name[IONIC_QOS_CLASS_NAME_SZ];
+ __le32 mtu;
+ /* flow control */
+ u8 pfc_cos;
+ /* scheduler */
+ union {
+ u8 dwrr_weight;
+ __le64 strict_rlmt;
+ };
+ /* marking */
+ /* Used to rewrite PCP or DSCP value. */
+ union {
+ u8 rw_dot1q_pcp;
+ u8 rw_ip_dscp;
+ };
+ /* classification */
+ union {
+ u8 dot1q_pcp;
+ struct {
+ u8 ndscp;
+ u8 ip_dscp[IONIC_QOS_DSCP_MAX];
+ };
+ };
+ };
+ __le32 words[64];
+};
+
+/**
+ * union ionic_qos_identity - QoS identity structure
+ * @version: Version of the identify structure
+ * @type: QoS system type
+ * @nclasses: Number of usable QoS classes
+ * @config: Current configuration of classes
+ */
+union ionic_qos_identity {
+ struct {
+ u8 version;
+ u8 type;
+ u8 rsvd[62];
+ union ionic_qos_config config[IONIC_QOS_CLASS_MAX];
+ };
+ __le32 words[478];
+};
+
+/**
+ * struct ionic_qos_init_cmd - QoS config init command
+ * @opcode: Opcode
+ * @group: QoS class id
+ * @info_pa: destination address for qos info
+ */
+struct ionic_qos_init_cmd {
+ u8 opcode;
+ u8 group;
+ u8 rsvd[6];
+ __le64 info_pa;
+ u8 rsvd1[48];
+};
+
+typedef struct ionic_admin_comp ionic_qos_init_comp;
+
+/**
+ * struct ionic_qos_reset_cmd - QoS config reset command
+ * @opcode: Opcode
+ * @group: QoS class id
+ */
+struct ionic_qos_reset_cmd {
+ u8 opcode;
+ u8 group;
+ u8 rsvd[62];
+};
+
+/**
+ * struct ionic_qos_clear_port_stats_cmd - Qos config reset command
+ * @opcode: Opcode
+ */
+struct ionic_qos_clear_stats_cmd {
+ u8 opcode;
+ u8 group_bitmap;
+ u8 rsvd[62];
+};
+
+typedef struct ionic_admin_comp ionic_qos_reset_comp;
+
+/**
+ * struct ionic_fw_download_cmd - Firmware download command
+ * @opcode: opcode
+ * @addr: dma address of the firmware buffer
+ * @offset: offset of the firmware buffer within the full image
+ * @length: number of valid bytes in the firmware buffer
+ */
+struct ionic_fw_download_cmd {
+ u8 opcode;
+ u8 rsvd[3];
+ __le32 offset;
+ __le64 addr;
+ __le32 length;
+};
+
+typedef struct ionic_admin_comp ionic_fw_download_comp;
+
+/**
+ * enum ionic_fw_control_oper - FW control operations
+ * @IONIC_FW_RESET: Reset firmware
+ * @IONIC_FW_INSTALL: Install firmware
+ * @IONIC_FW_ACTIVATE: Activate firmware
+ * @IONIC_FW_INSTALL_ASYNC: Install firmware asynchronously
+ * @IONIC_FW_INSTALL_STATUS: Firmware installation status
+ * @IONIC_FW_ACTIVATE_ASYNC: Activate firmware asynchronously
+ * @IONIC_FW_ACTIVATE_STATUS: Firmware activate status
+ */
+enum ionic_fw_control_oper {
+ IONIC_FW_RESET = 0,
+ IONIC_FW_INSTALL = 1,
+ IONIC_FW_ACTIVATE = 2,
+ IONIC_FW_INSTALL_ASYNC = 3,
+ IONIC_FW_INSTALL_STATUS = 4,
+ IONIC_FW_ACTIVATE_ASYNC = 5,
+ IONIC_FW_ACTIVATE_STATUS = 6,
+ IONIC_FW_UPDATE_CLEANUP = 7,
+};
+
+/**
+ * struct ionic_fw_control_cmd - Firmware control command
+ * @opcode: opcode
+ * @oper: firmware control operation (enum ionic_fw_control_oper)
+ * @slot: slot to activate
+ */
+struct ionic_fw_control_cmd {
+ u8 opcode;
+ u8 rsvd[3];
+ u8 oper;
+ u8 slot;
+ u8 rsvd1[58];
+};
+
+/**
+ * struct ionic_fw_control_comp - Firmware control copletion
+ * @status: Status of the command (enum ionic_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @slot: Slot where the firmware was installed
+ * @color: Color bit
+ */
+struct ionic_fw_control_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ u8 slot;
+ u8 rsvd1[10];
+ u8 color;
+};
+
+/******************************************************************
+ ******************* RDMA Commands ********************************
+ ******************************************************************/
+
+/**
+ * struct ionic_rdma_reset_cmd - Reset RDMA LIF cmd
+ * @opcode: opcode
+ * @lif_index: LIF index
+ *
+ * There is no RDMA specific dev command completion struct. Completion uses
+ * the common struct ionic_admin_comp. Only the status is indicated.
+ * Nonzero status means the LIF does not support RDMA.
+ **/
+struct ionic_rdma_reset_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 lif_index;
+ u8 rsvd2[60];
+};
+
+/**
+ * struct ionic_rdma_queue_cmd - Create RDMA Queue command
+ * @opcode: opcode, 52, 53
+ * @lif_index: LIF index
+ * @qid_ver: (qid | (RDMA version << 24))
+ * @cid: intr, eq_id, or cq_id
+ * @dbid: doorbell page id
+ * @depth_log2: log base two of queue depth
+ * @stride_log2: log base two of queue stride
+ * @dma_addr: address of the queue memory
+ *
+ * The same command struct is used to create an RDMA event queue, completion
+ * queue, or RDMA admin queue. The cid is an interrupt number for an event
+ * queue, an event queue id for a completion queue, or a completion queue id
+ * for an RDMA admin queue.
+ *
+ * The queue created via a dev command must be contiguous in dma space.
+ *
+ * The dev commands are intended only to be used during driver initialization,
+ * to create queues supporting the RDMA admin queue. Other queues, and other
+ * types of RDMA resources like memory regions, will be created and registered
+ * via the RDMA admin queue, and will support a more complete interface
+ * providing scatter gather lists for larger, scattered queue buffers and
+ * memory registration.
+ *
+ * There is no RDMA specific dev command completion struct. Completion uses
+ * the common struct ionic_admin_comp. Only the status is indicated.
+ **/
+struct ionic_rdma_queue_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 lif_index;
+ __le32 qid_ver;
+ __le32 cid;
+ __le16 dbid;
+ u8 depth_log2;
+ u8 stride_log2;
+ __le64 dma_addr;
+ u8 rsvd2[40];
+};
+
+/******************************************************************
+ ******************* Notify Events ********************************
+ ******************************************************************/
+
+/**
+ * struct ionic_notifyq_event - Generic event reporting structure
+ * @eid: event number
+ * @ecode: event code
+ * @data: unspecified data about the event
+ *
+ * This is the generic event report struct from which the other
+ * actual events will be formed.
+ */
+struct ionic_notifyq_event {
+ __le64 eid;
+ __le16 ecode;
+ u8 data[54];
+};
+
+/**
+ * struct ionic_link_change_event - Link change event notification
+ * @eid: event number
+ * @ecode: event code = IONIC_EVENT_LINK_CHANGE
+ * @link_status: link up/down, with error bits (enum ionic_port_status)
+ * @link_speed: speed of the network link
+ *
+ * Sent when the network link state changes between UP and DOWN
+ */
+struct ionic_link_change_event {
+ __le64 eid;
+ __le16 ecode;
+ __le16 link_status;
+ __le32 link_speed; /* units of 1Mbps: e.g. 10000 = 10Gbps */
+ u8 rsvd[48];
+};
+
+/**
+ * struct ionic_reset_event - Reset event notification
+ * @eid: event number
+ * @ecode: event code = IONIC_EVENT_RESET
+ * @reset_code: reset type
+ * @state: 0=pending, 1=complete, 2=error
+ *
+ * Sent when the NIC or some subsystem is going to be or
+ * has been reset.
+ */
+struct ionic_reset_event {
+ __le64 eid;
+ __le16 ecode;
+ u8 reset_code;
+ u8 state;
+ u8 rsvd[52];
+};
+
+/**
+ * struct ionic_heartbeat_event - Sent periodically by NIC to indicate health
+ * @eid: event number
+ * @ecode: event code = IONIC_EVENT_HEARTBEAT
+ */
+struct ionic_heartbeat_event {
+ __le64 eid;
+ __le16 ecode;
+ u8 rsvd[54];
+};
+
+/**
+ * struct ionic_log_event - Sent to notify the driver of an internal error
+ * @eid: event number
+ * @ecode: event code = IONIC_EVENT_LOG
+ * @data: log data
+ */
+struct ionic_log_event {
+ __le64 eid;
+ __le16 ecode;
+ u8 data[54];
+};
+
+/**
+ * struct ionic_xcvr_event - Transceiver change event
+ * @eid: event number
+ * @ecode: event code = IONIC_EVENT_XCVR
+ */
+struct ionic_xcvr_event {
+ __le64 eid;
+ __le16 ecode;
+ u8 rsvd[54];
+};
+
+/**
+ * struct ionic_port_stats - Port statistics structure
+ */
+struct ionic_port_stats {
+ __le64 frames_rx_ok;
+ __le64 frames_rx_all;
+ __le64 frames_rx_bad_fcs;
+ __le64 frames_rx_bad_all;
+ __le64 octets_rx_ok;
+ __le64 octets_rx_all;
+ __le64 frames_rx_unicast;
+ __le64 frames_rx_multicast;
+ __le64 frames_rx_broadcast;
+ __le64 frames_rx_pause;
+ __le64 frames_rx_bad_length;
+ __le64 frames_rx_undersized;
+ __le64 frames_rx_oversized;
+ __le64 frames_rx_fragments;
+ __le64 frames_rx_jabber;
+ __le64 frames_rx_pripause;
+ __le64 frames_rx_stomped_crc;
+ __le64 frames_rx_too_long;
+ __le64 frames_rx_vlan_good;
+ __le64 frames_rx_dropped;
+ __le64 frames_rx_less_than_64b;
+ __le64 frames_rx_64b;
+ __le64 frames_rx_65b_127b;
+ __le64 frames_rx_128b_255b;
+ __le64 frames_rx_256b_511b;
+ __le64 frames_rx_512b_1023b;
+ __le64 frames_rx_1024b_1518b;
+ __le64 frames_rx_1519b_2047b;
+ __le64 frames_rx_2048b_4095b;
+ __le64 frames_rx_4096b_8191b;
+ __le64 frames_rx_8192b_9215b;
+ __le64 frames_rx_other;
+ __le64 frames_tx_ok;
+ __le64 frames_tx_all;
+ __le64 frames_tx_bad;
+ __le64 octets_tx_ok;
+ __le64 octets_tx_total;
+ __le64 frames_tx_unicast;
+ __le64 frames_tx_multicast;
+ __le64 frames_tx_broadcast;
+ __le64 frames_tx_pause;
+ __le64 frames_tx_pripause;
+ __le64 frames_tx_vlan;
+ __le64 frames_tx_less_than_64b;
+ __le64 frames_tx_64b;
+ __le64 frames_tx_65b_127b;
+ __le64 frames_tx_128b_255b;
+ __le64 frames_tx_256b_511b;
+ __le64 frames_tx_512b_1023b;
+ __le64 frames_tx_1024b_1518b;
+ __le64 frames_tx_1519b_2047b;
+ __le64 frames_tx_2048b_4095b;
+ __le64 frames_tx_4096b_8191b;
+ __le64 frames_tx_8192b_9215b;
+ __le64 frames_tx_other;
+ __le64 frames_tx_pri_0;
+ __le64 frames_tx_pri_1;
+ __le64 frames_tx_pri_2;
+ __le64 frames_tx_pri_3;
+ __le64 frames_tx_pri_4;
+ __le64 frames_tx_pri_5;
+ __le64 frames_tx_pri_6;
+ __le64 frames_tx_pri_7;
+ __le64 frames_rx_pri_0;
+ __le64 frames_rx_pri_1;
+ __le64 frames_rx_pri_2;
+ __le64 frames_rx_pri_3;
+ __le64 frames_rx_pri_4;
+ __le64 frames_rx_pri_5;
+ __le64 frames_rx_pri_6;
+ __le64 frames_rx_pri_7;
+ __le64 tx_pripause_0_1us_count;
+ __le64 tx_pripause_1_1us_count;
+ __le64 tx_pripause_2_1us_count;
+ __le64 tx_pripause_3_1us_count;
+ __le64 tx_pripause_4_1us_count;
+ __le64 tx_pripause_5_1us_count;
+ __le64 tx_pripause_6_1us_count;
+ __le64 tx_pripause_7_1us_count;
+ __le64 rx_pripause_0_1us_count;
+ __le64 rx_pripause_1_1us_count;
+ __le64 rx_pripause_2_1us_count;
+ __le64 rx_pripause_3_1us_count;
+ __le64 rx_pripause_4_1us_count;
+ __le64 rx_pripause_5_1us_count;
+ __le64 rx_pripause_6_1us_count;
+ __le64 rx_pripause_7_1us_count;
+ __le64 rx_pause_1us_count;
+ __le64 frames_tx_truncated;
+};
+
+struct ionic_mgmt_port_stats {
+ __le64 frames_rx_ok;
+ __le64 frames_rx_all;
+ __le64 frames_rx_bad_fcs;
+ __le64 frames_rx_bad_all;
+ __le64 octets_rx_ok;
+ __le64 octets_rx_all;
+ __le64 frames_rx_unicast;
+ __le64 frames_rx_multicast;
+ __le64 frames_rx_broadcast;
+ __le64 frames_rx_pause;
+ __le64 frames_rx_bad_length;
+ __le64 frames_rx_undersized;
+ __le64 frames_rx_oversized;
+ __le64 frames_rx_fragments;
+ __le64 frames_rx_jabber;
+ __le64 frames_rx_64b;
+ __le64 frames_rx_65b_127b;
+ __le64 frames_rx_128b_255b;
+ __le64 frames_rx_256b_511b;
+ __le64 frames_rx_512b_1023b;
+ __le64 frames_rx_1024b_1518b;
+ __le64 frames_rx_gt_1518b;
+ __le64 frames_rx_fifo_full;
+ __le64 frames_tx_ok;
+ __le64 frames_tx_all;
+ __le64 frames_tx_bad;
+ __le64 octets_tx_ok;
+ __le64 octets_tx_total;
+ __le64 frames_tx_unicast;
+ __le64 frames_tx_multicast;
+ __le64 frames_tx_broadcast;
+ __le64 frames_tx_pause;
+};
+
+enum ionic_pb_buffer_drop_stats {
+ IONIC_BUFFER_INTRINSIC_DROP = 0,
+ IONIC_BUFFER_DISCARDED,
+ IONIC_BUFFER_ADMITTED,
+ IONIC_BUFFER_OUT_OF_CELLS_DROP,
+ IONIC_BUFFER_OUT_OF_CELLS_DROP_2,
+ IONIC_BUFFER_OUT_OF_CREDIT_DROP,
+ IONIC_BUFFER_TRUNCATION_DROP,
+ IONIC_BUFFER_PORT_DISABLED_DROP,
+ IONIC_BUFFER_COPY_TO_CPU_TAIL_DROP,
+ IONIC_BUFFER_SPAN_TAIL_DROP,
+ IONIC_BUFFER_MIN_SIZE_VIOLATION_DROP,
+ IONIC_BUFFER_ENQUEUE_ERROR_DROP,
+ IONIC_BUFFER_INVALID_PORT_DROP,
+ IONIC_BUFFER_INVALID_OUTPUT_QUEUE_DROP,
+ IONIC_BUFFER_DROP_MAX,
+};
+
+enum ionic_oflow_drop_stats {
+ IONIC_OFLOW_OCCUPANCY_DROP,
+ IONIC_OFLOW_EMERGENCY_STOP_DROP,
+ IONIC_OFLOW_WRITE_BUFFER_ACK_FILL_UP_DROP,
+ IONIC_OFLOW_WRITE_BUFFER_ACK_FULL_DROP,
+ IONIC_OFLOW_WRITE_BUFFER_FULL_DROP,
+ IONIC_OFLOW_CONTROL_FIFO_FULL_DROP,
+ IONIC_OFLOW_DROP_MAX,
+};
+
+/**
+ * struct port_pb_stats - packet buffers system stats
+ * uses ionic_pb_buffer_drop_stats for drop_counts[]
+ */
+struct ionic_port_pb_stats {
+ __le64 sop_count_in;
+ __le64 eop_count_in;
+ __le64 sop_count_out;
+ __le64 eop_count_out;
+ __le64 drop_counts[IONIC_BUFFER_DROP_MAX];
+ __le64 input_queue_buffer_occupancy[IONIC_QOS_TC_MAX];
+ __le64 input_queue_port_monitor[IONIC_QOS_TC_MAX];
+ __le64 output_queue_port_monitor[IONIC_QOS_TC_MAX];
+ __le64 oflow_drop_counts[IONIC_OFLOW_DROP_MAX];
+ __le64 input_queue_good_pkts_in[IONIC_QOS_TC_MAX];
+ __le64 input_queue_good_pkts_out[IONIC_QOS_TC_MAX];
+ __le64 input_queue_err_pkts_in[IONIC_QOS_TC_MAX];
+ __le64 input_queue_fifo_depth[IONIC_QOS_TC_MAX];
+ __le64 input_queue_max_fifo_depth[IONIC_QOS_TC_MAX];
+ __le64 input_queue_peak_occupancy[IONIC_QOS_TC_MAX];
+ __le64 output_queue_buffer_occupancy[IONIC_QOS_TC_MAX];
+};
+
+/**
+ * struct ionic_port_identity - port identity structure
+ * @version: identity structure version
+ * @type: type of port (enum ionic_port_type)
+ * @num_lanes: number of lanes for the port
+ * @autoneg: autoneg supported
+ * @min_frame_size: minimum frame size supported
+ * @max_frame_size: maximum frame size supported
+ * @fec_type: supported fec types
+ * @pause_type: supported pause types
+ * @loopback_mode: supported loopback mode
+ * @speeds: supported speeds
+ * @config: current port configuration
+ */
+union ionic_port_identity {
+ struct {
+ u8 version;
+ u8 type;
+ u8 num_lanes;
+ u8 autoneg;
+ __le32 min_frame_size;
+ __le32 max_frame_size;
+ u8 fec_type[4];
+ u8 pause_type[2];
+ u8 loopback_mode[2];
+ __le32 speeds[16];
+ u8 rsvd2[44];
+ union ionic_port_config config;
+ };
+ __le32 words[478];
+};
+
+/**
+ * struct ionic_port_info - port info structure
+ * @config: Port configuration data
+ * @status: Port status data
+ * @stats: Port statistics data
+ * @mgmt_stats: Port management statistics data
+ * @port_pb_drop_stats: uplink pb drop stats
+ */
+struct ionic_port_info {
+ union ionic_port_config config;
+ struct ionic_port_status status;
+ union {
+ struct ionic_port_stats stats;
+ struct ionic_mgmt_port_stats mgmt_stats;
+ };
+ /* room for pb_stats to start at 2k offset */
+ u8 rsvd[760];
+ struct ionic_port_pb_stats pb_stats;
+};
+
+/**
+ * struct ionic_lif_stats - LIF statistics structure
+ */
+struct ionic_lif_stats {
+ /* RX */
+ __le64 rx_ucast_bytes;
+ __le64 rx_ucast_packets;
+ __le64 rx_mcast_bytes;
+ __le64 rx_mcast_packets;
+ __le64 rx_bcast_bytes;
+ __le64 rx_bcast_packets;
+ __le64 rsvd0;
+ __le64 rsvd1;
+ /* RX drops */
+ __le64 rx_ucast_drop_bytes;
+ __le64 rx_ucast_drop_packets;
+ __le64 rx_mcast_drop_bytes;
+ __le64 rx_mcast_drop_packets;
+ __le64 rx_bcast_drop_bytes;
+ __le64 rx_bcast_drop_packets;
+ __le64 rx_dma_error;
+ __le64 rsvd2;
+ /* TX */
+ __le64 tx_ucast_bytes;
+ __le64 tx_ucast_packets;
+ __le64 tx_mcast_bytes;
+ __le64 tx_mcast_packets;
+ __le64 tx_bcast_bytes;
+ __le64 tx_bcast_packets;
+ __le64 rsvd3;
+ __le64 rsvd4;
+ /* TX drops */
+ __le64 tx_ucast_drop_bytes;
+ __le64 tx_ucast_drop_packets;
+ __le64 tx_mcast_drop_bytes;
+ __le64 tx_mcast_drop_packets;
+ __le64 tx_bcast_drop_bytes;
+ __le64 tx_bcast_drop_packets;
+ __le64 tx_dma_error;
+ __le64 rsvd5;
+ /* Rx Queue/Ring drops */
+ __le64 rx_queue_disabled;
+ __le64 rx_queue_empty;
+ __le64 rx_queue_error;
+ __le64 rx_desc_fetch_error;
+ __le64 rx_desc_data_error;
+ __le64 rsvd6;
+ __le64 rsvd7;
+ __le64 rsvd8;
+ /* Tx Queue/Ring drops */
+ __le64 tx_queue_disabled;
+ __le64 tx_queue_error;
+ __le64 tx_desc_fetch_error;
+ __le64 tx_desc_data_error;
+ __le64 tx_queue_empty;
+ __le64 rsvd10;
+ __le64 rsvd11;
+ __le64 rsvd12;
+
+ /* RDMA/ROCE TX */
+ __le64 tx_rdma_ucast_bytes;
+ __le64 tx_rdma_ucast_packets;
+ __le64 tx_rdma_mcast_bytes;
+ __le64 tx_rdma_mcast_packets;
+ __le64 tx_rdma_cnp_packets;
+ __le64 rsvd13;
+ __le64 rsvd14;
+ __le64 rsvd15;
+
+ /* RDMA/ROCE RX */
+ __le64 rx_rdma_ucast_bytes;
+ __le64 rx_rdma_ucast_packets;
+ __le64 rx_rdma_mcast_bytes;
+ __le64 rx_rdma_mcast_packets;
+ __le64 rx_rdma_cnp_packets;
+ __le64 rx_rdma_ecn_packets;
+ __le64 rsvd16;
+ __le64 rsvd17;
+
+ __le64 rsvd18;
+ __le64 rsvd19;
+ __le64 rsvd20;
+ __le64 rsvd21;
+ __le64 rsvd22;
+ __le64 rsvd23;
+ __le64 rsvd24;
+ __le64 rsvd25;
+
+ __le64 rsvd26;
+ __le64 rsvd27;
+ __le64 rsvd28;
+ __le64 rsvd29;
+ __le64 rsvd30;
+ __le64 rsvd31;
+ __le64 rsvd32;
+ __le64 rsvd33;
+
+ __le64 rsvd34;
+ __le64 rsvd35;
+ __le64 rsvd36;
+ __le64 rsvd37;
+ __le64 rsvd38;
+ __le64 rsvd39;
+ __le64 rsvd40;
+ __le64 rsvd41;
+
+ __le64 rsvd42;
+ __le64 rsvd43;
+ __le64 rsvd44;
+ __le64 rsvd45;
+ __le64 rsvd46;
+ __le64 rsvd47;
+ __le64 rsvd48;
+ __le64 rsvd49;
+
+ /* RDMA/ROCE REQ Error/Debugs (768 - 895) */
+ __le64 rdma_req_rx_pkt_seq_err;
+ __le64 rdma_req_rx_rnr_retry_err;
+ __le64 rdma_req_rx_remote_access_err;
+ __le64 rdma_req_rx_remote_inv_req_err;
+ __le64 rdma_req_rx_remote_oper_err;
+ __le64 rdma_req_rx_implied_nak_seq_err;
+ __le64 rdma_req_rx_cqe_err;
+ __le64 rdma_req_rx_cqe_flush_err;
+
+ __le64 rdma_req_rx_dup_responses;
+ __le64 rdma_req_rx_invalid_packets;
+ __le64 rdma_req_tx_local_access_err;
+ __le64 rdma_req_tx_local_oper_err;
+ __le64 rdma_req_tx_memory_mgmt_err;
+ __le64 rsvd52;
+ __le64 rsvd53;
+ __le64 rsvd54;
+
+ /* RDMA/ROCE RESP Error/Debugs (896 - 1023) */
+ __le64 rdma_resp_rx_dup_requests;
+ __le64 rdma_resp_rx_out_of_buffer;
+ __le64 rdma_resp_rx_out_of_seq_pkts;
+ __le64 rdma_resp_rx_cqe_err;
+ __le64 rdma_resp_rx_cqe_flush_err;
+ __le64 rdma_resp_rx_local_len_err;
+ __le64 rdma_resp_rx_inv_request_err;
+ __le64 rdma_resp_rx_local_qp_oper_err;
+
+ __le64 rdma_resp_rx_out_of_atomic_resource;
+ __le64 rdma_resp_tx_pkt_seq_err;
+ __le64 rdma_resp_tx_remote_inv_req_err;
+ __le64 rdma_resp_tx_remote_access_err;
+ __le64 rdma_resp_tx_remote_oper_err;
+ __le64 rdma_resp_tx_rnr_retry_err;
+ __le64 rsvd57;
+ __le64 rsvd58;
+};
+
+/**
+ * struct ionic_lif_info - LIF info structure
+ * @config: LIF configuration structure
+ * @status: LIF status structure
+ * @stats: LIF statistics structure
+ */
+struct ionic_lif_info {
+ union ionic_lif_config config;
+ struct ionic_lif_status status;
+ struct ionic_lif_stats stats;
+};
+
+union ionic_dev_cmd {
+ u32 words[16];
+ struct ionic_admin_cmd cmd;
+ struct ionic_nop_cmd nop;
+
+ struct ionic_dev_identify_cmd identify;
+ struct ionic_dev_init_cmd init;
+ struct ionic_dev_reset_cmd reset;
+ struct ionic_dev_getattr_cmd getattr;
+ struct ionic_dev_setattr_cmd setattr;
+
+ struct ionic_port_identify_cmd port_identify;
+ struct ionic_port_init_cmd port_init;
+ struct ionic_port_reset_cmd port_reset;
+ struct ionic_port_getattr_cmd port_getattr;
+ struct ionic_port_setattr_cmd port_setattr;
+
+ struct ionic_vf_setattr_cmd vf_setattr;
+ struct ionic_vf_getattr_cmd vf_getattr;
+
+ struct ionic_lif_identify_cmd lif_identify;
+ struct ionic_lif_init_cmd lif_init;
+ struct ionic_lif_reset_cmd lif_reset;
+
+ struct ionic_qos_identify_cmd qos_identify;
+ struct ionic_qos_init_cmd qos_init;
+ struct ionic_qos_reset_cmd qos_reset;
+ struct ionic_qos_clear_stats_cmd qos_clear_stats;
+
+ struct ionic_q_identify_cmd q_identify;
+ struct ionic_q_init_cmd q_init;
+ struct ionic_q_control_cmd q_control;
+
+ struct ionic_fw_download_cmd fw_download;
+ struct ionic_fw_control_cmd fw_control;
+};
+
+union ionic_dev_cmd_comp {
+ u32 words[4];
+ u8 status;
+ struct ionic_admin_comp comp;
+ struct ionic_nop_comp nop;
+
+ struct ionic_dev_identify_comp identify;
+ struct ionic_dev_init_comp init;
+ struct ionic_dev_reset_comp reset;
+ struct ionic_dev_getattr_comp getattr;
+ struct ionic_dev_setattr_comp setattr;
+
+ struct ionic_port_identify_comp port_identify;
+ struct ionic_port_init_comp port_init;
+ struct ionic_port_reset_comp port_reset;
+ struct ionic_port_getattr_comp port_getattr;
+ struct ionic_port_setattr_comp port_setattr;
+
+ struct ionic_vf_setattr_comp vf_setattr;
+ struct ionic_vf_getattr_comp vf_getattr;
+
+ struct ionic_lif_identify_comp lif_identify;
+ struct ionic_lif_init_comp lif_init;
+ ionic_lif_reset_comp lif_reset;
+
+ struct ionic_qos_identify_comp qos_identify;
+ ionic_qos_init_comp qos_init;
+ ionic_qos_reset_comp qos_reset;
+
+ struct ionic_q_identify_comp q_identify;
+ struct ionic_q_init_comp q_init;
+
+ ionic_fw_download_comp fw_download;
+ struct ionic_fw_control_comp fw_control;
+};
+
+/**
+ * struct ionic_hwstamp_regs - Hardware current timestamp registers
+ * @tick_low: Low 32 bits of hardware timestamp
+ * @tick_high: High 32 bits of hardware timestamp
+ */
+struct ionic_hwstamp_regs {
+ u32 tick_low;
+ u32 tick_high;
+};
+
+/**
+ * union ionic_dev_info_regs - Device info register format (read-only)
+ * @signature: Signature value of 0x44455649 ('DEVI')
+ * @version: Current version of info
+ * @asic_type: Asic type
+ * @asic_rev: Asic revision
+ * @fw_status: Firmware status
+ * bit 0 - 1 = fw running
+ * bit 4-7 - 4 bit generation number, changes on fw restart
+ * @fw_heartbeat: Firmware heartbeat counter
+ * @serial_num: Serial number
+ * @fw_version: Firmware version
+ * @hwstamp_regs: Hardware current timestamp registers
+ */
+union ionic_dev_info_regs {
+#define IONIC_DEVINFO_FWVERS_BUFLEN 32
+#define IONIC_DEVINFO_SERIAL_BUFLEN 32
+ struct {
+ u32 signature;
+ u8 version;
+ u8 asic_type;
+ u8 asic_rev;
+#define IONIC_FW_STS_F_RUNNING 0x01
+#define IONIC_FW_STS_F_GENERATION 0xF0
+ u8 fw_status;
+ u32 fw_heartbeat;
+ char fw_version[IONIC_DEVINFO_FWVERS_BUFLEN];
+ char serial_num[IONIC_DEVINFO_SERIAL_BUFLEN];
+ u8 rsvd_pad1024[948];
+ struct ionic_hwstamp_regs hwstamp;
+ };
+ u32 words[512];
+};
+
+/**
+ * union ionic_dev_cmd_regs - Device command register format (read-write)
+ * @doorbell: Device Cmd Doorbell, write-only
+ * Write a 1 to signal device to process cmd,
+ * poll done for completion.
+ * @done: Done indicator, bit 0 == 1 when command is complete
+ * @cmd: Opcode-specific command bytes
+ * @comp: Opcode-specific response bytes
+ * @data: Opcode-specific side-data
+ */
+union ionic_dev_cmd_regs {
+ struct {
+ u32 doorbell;
+ u32 done;
+ union ionic_dev_cmd cmd;
+ union ionic_dev_cmd_comp comp;
+ u8 rsvd[48];
+ u32 data[478];
+ } __packed;
+ u32 words[512];
+};
+
+/**
+ * union ionic_dev_regs - Device register format for bar 0 page 0
+ * @info: Device info registers
+ * @devcmd: Device command registers
+ */
+union ionic_dev_regs {
+ struct {
+ union ionic_dev_info_regs info;
+ union ionic_dev_cmd_regs devcmd;
+ } __packed;
+ __le32 words[1024];
+};
+
+union ionic_adminq_cmd {
+ struct ionic_admin_cmd cmd;
+ struct ionic_nop_cmd nop;
+ struct ionic_q_identify_cmd q_identify;
+ struct ionic_q_init_cmd q_init;
+ struct ionic_q_control_cmd q_control;
+ struct ionic_lif_setattr_cmd lif_setattr;
+ struct ionic_lif_getattr_cmd lif_getattr;
+ struct ionic_lif_setphc_cmd lif_setphc;
+ struct ionic_rx_mode_set_cmd rx_mode_set;
+ struct ionic_rx_filter_add_cmd rx_filter_add;
+ struct ionic_rx_filter_del_cmd rx_filter_del;
+ struct ionic_rdma_reset_cmd rdma_reset;
+ struct ionic_rdma_queue_cmd rdma_queue;
+ struct ionic_fw_download_cmd fw_download;
+ struct ionic_fw_control_cmd fw_control;
+};
+
+union ionic_adminq_comp {
+ struct ionic_admin_comp comp;
+ struct ionic_nop_comp nop;
+ struct ionic_q_identify_comp q_identify;
+ struct ionic_q_init_comp q_init;
+ struct ionic_lif_setattr_comp lif_setattr;
+ struct ionic_lif_getattr_comp lif_getattr;
+ struct ionic_admin_comp lif_setphc;
+ struct ionic_rx_filter_add_comp rx_filter_add;
+ struct ionic_fw_control_comp fw_control;
+};
+
+#define IONIC_BARS_MAX 6
+#define IONIC_PCI_BAR_DBELL 1
+
+/* BAR0 */
+#define IONIC_BAR0_SIZE 0x8000
+
+#define IONIC_BAR0_DEV_INFO_REGS_OFFSET 0x0000
+#define IONIC_BAR0_DEV_CMD_REGS_OFFSET 0x0800
+#define IONIC_BAR0_DEV_CMD_DATA_REGS_OFFSET 0x0c00
+#define IONIC_BAR0_INTR_STATUS_OFFSET 0x1000
+#define IONIC_BAR0_INTR_CTRL_OFFSET 0x2000
+#define IONIC_DEV_CMD_DONE 0x00000001
+
+#define IONIC_ASIC_TYPE_CAPRI 0
+
+/**
+ * struct ionic_doorbell - Doorbell register layout
+ * @p_index: Producer index
+ * @ring: Selects the specific ring of the queue to update
+ * Type-specific meaning:
+ * ring=0: Default producer/consumer queue
+ * ring=1: (CQ, EQ) Re-Arm queue. RDMA CQs
+ * send events to EQs when armed. EQs send
+ * interrupts when armed.
+ * @qid_lo: Queue destination for the producer index and flags (low bits)
+ * @qid_hi: Queue destination for the producer index and flags (high bits)
+ */
+struct ionic_doorbell {
+ __le16 p_index;
+ u8 ring;
+ u8 qid_lo;
+ __le16 qid_hi;
+ u16 rsvd2;
+};
+
+struct ionic_intr_status {
+ u32 status[2];
+};
+
+struct ionic_notifyq_cmd {
+ __le32 data; /* Not used but needed for qcq structure */
+};
+
+union ionic_notifyq_comp {
+ struct ionic_notifyq_event event;
+ struct ionic_link_change_event link_change;
+ struct ionic_reset_event reset;
+ struct ionic_heartbeat_event heartbeat;
+ struct ionic_log_event log;
+};
+
+/* Deprecate */
+struct ionic_identity {
+ union ionic_drv_identity drv;
+ union ionic_dev_identity dev;
+ union ionic_lif_identity lif;
+ union ionic_port_identity port;
+ union ionic_qos_identity qos;
+ union ionic_q_identity txq;
+};
+
+#endif /* _IONIC_IF_H_ */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
new file mode 100644
index 000000000..f7634884c
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
@@ -0,0 +1,3721 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#include <linux/ethtool.h>
+#include <linux/printk.h>
+#include <linux/dynamic_debug.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/rtnetlink.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/cpumask.h>
+#include <linux/crash_dump.h>
+#include <linux/vmalloc.h>
+
+#include "ionic.h"
+#include "ionic_bus.h"
+#include "ionic_dev.h"
+#include "ionic_lif.h"
+#include "ionic_txrx.h"
+#include "ionic_ethtool.h"
+#include "ionic_debugfs.h"
+
+/* queuetype support level */
+static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
+ [IONIC_QTYPE_ADMINQ] = 0, /* 0 = Base version with CQ support */
+ [IONIC_QTYPE_NOTIFYQ] = 0, /* 0 = Base version */
+ [IONIC_QTYPE_RXQ] = 0, /* 0 = Base version with CQ+SG support */
+ [IONIC_QTYPE_TXQ] = 1, /* 0 = Base version with CQ+SG support
+ * 1 = ... with Tx SG version 1
+ */
+};
+
+static void ionic_link_status_check(struct ionic_lif *lif);
+static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
+static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
+static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
+
+static void ionic_txrx_deinit(struct ionic_lif *lif);
+static int ionic_txrx_init(struct ionic_lif *lif);
+static int ionic_start_queues(struct ionic_lif *lif);
+static void ionic_stop_queues(struct ionic_lif *lif);
+static void ionic_lif_queue_identify(struct ionic_lif *lif);
+
+static void ionic_dim_work(struct work_struct *work)
+{
+ struct dim *dim = container_of(work, struct dim, work);
+ struct ionic_intr_info *intr;
+ struct dim_cq_moder cur_moder;
+ struct ionic_qcq *qcq;
+ struct ionic_lif *lif;
+ u32 new_coal;
+
+ cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
+ qcq = container_of(dim, struct ionic_qcq, dim);
+ lif = qcq->q.lif;
+ new_coal = ionic_coal_usec_to_hw(lif->ionic, cur_moder.usec);
+ new_coal = new_coal ? new_coal : 1;
+
+ intr = &qcq->intr;
+ if (intr->dim_coal_hw != new_coal) {
+ intr->dim_coal_hw = new_coal;
+
+ ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
+ intr->index, intr->dim_coal_hw);
+ }
+
+ dim->state = DIM_START_MEASURE;
+}
+
+static void ionic_lif_deferred_work(struct work_struct *work)
+{
+ struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
+ struct ionic_deferred *def = &lif->deferred;
+ struct ionic_deferred_work *w = NULL;
+
+ do {
+ spin_lock_bh(&def->lock);
+ if (!list_empty(&def->list)) {
+ w = list_first_entry(&def->list,
+ struct ionic_deferred_work, list);
+ list_del(&w->list);
+ }
+ spin_unlock_bh(&def->lock);
+
+ if (!w)
+ break;
+
+ switch (w->type) {
+ case IONIC_DW_TYPE_RX_MODE:
+ ionic_lif_rx_mode(lif);
+ break;
+ case IONIC_DW_TYPE_LINK_STATUS:
+ ionic_link_status_check(lif);
+ break;
+ case IONIC_DW_TYPE_LIF_RESET:
+ if (w->fw_status) {
+ ionic_lif_handle_fw_up(lif);
+ } else {
+ ionic_lif_handle_fw_down(lif);
+
+ /* Fire off another watchdog to see
+ * if the FW is already back rather than
+ * waiting another whole cycle
+ */
+ mod_timer(&lif->ionic->watchdog_timer, jiffies + 1);
+ }
+ break;
+ default:
+ break;
+ }
+ kfree(w);
+ w = NULL;
+ } while (true);
+}
+
+void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
+ struct ionic_deferred_work *work)
+{
+ spin_lock_bh(&def->lock);
+ list_add_tail(&work->list, &def->list);
+ spin_unlock_bh(&def->lock);
+ schedule_work(&def->work);
+}
+
+static void ionic_link_status_check(struct ionic_lif *lif)
+{
+ struct net_device *netdev = lif->netdev;
+ u16 link_status;
+ bool link_up;
+
+ if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
+ return;
+
+ /* Don't put carrier back up if we're in a broken state */
+ if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
+ clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
+ return;
+ }
+
+ link_status = le16_to_cpu(lif->info->status.link_status);
+ link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
+
+ if (link_up) {
+ int err = 0;
+
+ if (netdev->flags & IFF_UP && netif_running(netdev)) {
+ mutex_lock(&lif->queue_lock);
+ err = ionic_start_queues(lif);
+ if (err && err != -EBUSY) {
+ netdev_err(lif->netdev,
+ "Failed to start queues: %d\n", err);
+ set_bit(IONIC_LIF_F_BROKEN, lif->state);
+ netif_carrier_off(lif->netdev);
+ }
+ mutex_unlock(&lif->queue_lock);
+ }
+
+ if (!err && !netif_carrier_ok(netdev)) {
+ ionic_port_identify(lif->ionic);
+ netdev_info(netdev, "Link up - %d Gbps\n",
+ le32_to_cpu(lif->info->status.link_speed) / 1000);
+ netif_carrier_on(netdev);
+ }
+ } else {
+ if (netif_carrier_ok(netdev)) {
+ netdev_info(netdev, "Link down\n");
+ netif_carrier_off(netdev);
+ }
+
+ if (netdev->flags & IFF_UP && netif_running(netdev)) {
+ mutex_lock(&lif->queue_lock);
+ ionic_stop_queues(lif);
+ mutex_unlock(&lif->queue_lock);
+ }
+ }
+
+ clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
+}
+
+void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
+{
+ struct ionic_deferred_work *work;
+
+ /* we only need one request outstanding at a time */
+ if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
+ return;
+
+ if (!can_sleep) {
+ work = kzalloc(sizeof(*work), GFP_ATOMIC);
+ if (!work) {
+ clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
+ return;
+ }
+
+ work->type = IONIC_DW_TYPE_LINK_STATUS;
+ ionic_lif_deferred_enqueue(&lif->deferred, work);
+ } else {
+ ionic_link_status_check(lif);
+ }
+}
+
+static void ionic_napi_deadline(struct timer_list *timer)
+{
+ struct ionic_qcq *qcq = container_of(timer, struct ionic_qcq, napi_deadline);
+
+ napi_schedule(&qcq->napi);
+}
+
+static irqreturn_t ionic_isr(int irq, void *data)
+{
+ struct napi_struct *napi = data;
+
+ napi_schedule_irqoff(napi);
+
+ return IRQ_HANDLED;
+}
+
+static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
+{
+ struct ionic_intr_info *intr = &qcq->intr;
+ struct device *dev = lif->ionic->dev;
+ struct ionic_queue *q = &qcq->q;
+ const char *name;
+
+ if (lif->registered)
+ name = lif->netdev->name;
+ else
+ name = dev_name(dev);
+
+ snprintf(intr->name, sizeof(intr->name),
+ "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
+
+ return devm_request_irq(dev, intr->vector, ionic_isr,
+ 0, intr->name, &qcq->napi);
+}
+
+static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
+{
+ struct ionic *ionic = lif->ionic;
+ int index;
+
+ index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
+ if (index == ionic->nintrs) {
+ netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
+ __func__, index, ionic->nintrs);
+ return -ENOSPC;
+ }
+
+ set_bit(index, ionic->intrs);
+ ionic_intr_init(&ionic->idev, intr, index);
+
+ return 0;
+}
+
+static void ionic_intr_free(struct ionic *ionic, int index)
+{
+ if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
+ clear_bit(index, ionic->intrs);
+}
+
+static int ionic_qcq_enable(struct ionic_qcq *qcq)
+{
+ struct ionic_queue *q = &qcq->q;
+ struct ionic_lif *lif = q->lif;
+ struct ionic_dev *idev;
+ struct device *dev;
+
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.q_control = {
+ .opcode = IONIC_CMD_Q_CONTROL,
+ .lif_index = cpu_to_le16(lif->index),
+ .type = q->type,
+ .index = cpu_to_le32(q->index),
+ .oper = IONIC_Q_ENABLE,
+ },
+ };
+ int ret;
+
+ idev = &lif->ionic->idev;
+ dev = lif->ionic->dev;
+
+ dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
+ ctx.cmd.q_control.index, ctx.cmd.q_control.type);
+
+ if (qcq->flags & IONIC_QCQ_F_INTR)
+ ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
+
+ ret = ionic_adminq_post_wait(lif, &ctx);
+ if (ret)
+ return ret;
+
+ if (qcq->napi.poll)
+ napi_enable(&qcq->napi);
+
+ if (qcq->flags & IONIC_QCQ_F_INTR) {
+ irq_set_affinity_hint(qcq->intr.vector,
+ &qcq->intr.affinity_mask);
+ ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
+ IONIC_INTR_MASK_CLEAR);
+ }
+
+ return 0;
+}
+
+static int ionic_qcq_disable(struct ionic_lif *lif, struct ionic_qcq *qcq, int fw_err)
+{
+ struct ionic_queue *q;
+
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.q_control = {
+ .opcode = IONIC_CMD_Q_CONTROL,
+ .oper = IONIC_Q_DISABLE,
+ },
+ };
+
+ if (!qcq) {
+ netdev_err(lif->netdev, "%s: bad qcq\n", __func__);
+ return -ENXIO;
+ }
+
+ q = &qcq->q;
+
+ if (qcq->flags & IONIC_QCQ_F_INTR) {
+ struct ionic_dev *idev = &lif->ionic->idev;
+
+ cancel_work_sync(&qcq->dim.work);
+ ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
+ IONIC_INTR_MASK_SET);
+ synchronize_irq(qcq->intr.vector);
+ irq_set_affinity_hint(qcq->intr.vector, NULL);
+ napi_disable(&qcq->napi);
+ del_timer_sync(&qcq->napi_deadline);
+ }
+
+ /* If there was a previous fw communcation error, don't bother with
+ * sending the adminq command and just return the same error value.
+ */
+ if (fw_err == -ETIMEDOUT || fw_err == -ENXIO)
+ return fw_err;
+
+ ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
+ ctx.cmd.q_control.type = q->type;
+ ctx.cmd.q_control.index = cpu_to_le32(q->index);
+ dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
+ ctx.cmd.q_control.index, ctx.cmd.q_control.type);
+
+ return ionic_adminq_post_wait(lif, &ctx);
+}
+
+static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
+{
+ struct ionic_dev *idev = &lif->ionic->idev;
+
+ if (!qcq)
+ return;
+
+ if (!(qcq->flags & IONIC_QCQ_F_INITED))
+ return;
+
+ if (qcq->flags & IONIC_QCQ_F_INTR) {
+ ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
+ IONIC_INTR_MASK_SET);
+ netif_napi_del(&qcq->napi);
+ }
+
+ qcq->flags &= ~IONIC_QCQ_F_INITED;
+}
+
+static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
+{
+ if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
+ return;
+
+ irq_set_affinity_hint(qcq->intr.vector, NULL);
+ devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
+ qcq->intr.vector = 0;
+ ionic_intr_free(lif->ionic, qcq->intr.index);
+ qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
+}
+
+static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
+{
+ struct device *dev = lif->ionic->dev;
+
+ if (!qcq)
+ return;
+
+ ionic_debugfs_del_qcq(qcq);
+
+ if (qcq->q_base) {
+ dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
+ qcq->q_base = NULL;
+ qcq->q_base_pa = 0;
+ }
+
+ if (qcq->cq_base) {
+ dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
+ qcq->cq_base = NULL;
+ qcq->cq_base_pa = 0;
+ }
+
+ if (qcq->sg_base) {
+ dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
+ qcq->sg_base = NULL;
+ qcq->sg_base_pa = 0;
+ }
+
+ ionic_qcq_intr_free(lif, qcq);
+
+ if (qcq->cq.info) {
+ vfree(qcq->cq.info);
+ qcq->cq.info = NULL;
+ }
+ if (qcq->q.info) {
+ vfree(qcq->q.info);
+ qcq->q.info = NULL;
+ }
+}
+
+static void ionic_qcqs_free(struct ionic_lif *lif)
+{
+ struct device *dev = lif->ionic->dev;
+ struct ionic_qcq *adminqcq;
+ unsigned long irqflags;
+
+ if (lif->notifyqcq) {
+ ionic_qcq_free(lif, lif->notifyqcq);
+ devm_kfree(dev, lif->notifyqcq);
+ lif->notifyqcq = NULL;
+ }
+
+ if (lif->adminqcq) {
+ spin_lock_irqsave(&lif->adminq_lock, irqflags);
+ adminqcq = READ_ONCE(lif->adminqcq);
+ lif->adminqcq = NULL;
+ spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
+ if (adminqcq) {
+ ionic_qcq_free(lif, adminqcq);
+ devm_kfree(dev, adminqcq);
+ }
+ }
+
+ if (lif->rxqcqs) {
+ devm_kfree(dev, lif->rxqstats);
+ lif->rxqstats = NULL;
+ devm_kfree(dev, lif->rxqcqs);
+ lif->rxqcqs = NULL;
+ }
+
+ if (lif->txqcqs) {
+ devm_kfree(dev, lif->txqstats);
+ lif->txqstats = NULL;
+ devm_kfree(dev, lif->txqcqs);
+ lif->txqcqs = NULL;
+ }
+}
+
+static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
+ struct ionic_qcq *n_qcq)
+{
+ n_qcq->intr.vector = src_qcq->intr.vector;
+ n_qcq->intr.index = src_qcq->intr.index;
+ n_qcq->napi_qcq = src_qcq->napi_qcq;
+}
+
+static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
+{
+ int err;
+
+ if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
+ qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
+ return 0;
+ }
+
+ err = ionic_intr_alloc(lif, &qcq->intr);
+ if (err) {
+ netdev_warn(lif->netdev, "no intr for %s: %d\n",
+ qcq->q.name, err);
+ goto err_out;
+ }
+
+ err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
+ if (err < 0) {
+ netdev_warn(lif->netdev, "no vector for %s: %d\n",
+ qcq->q.name, err);
+ goto err_out_free_intr;
+ }
+ qcq->intr.vector = err;
+ ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
+ IONIC_INTR_MASK_SET);
+
+ err = ionic_request_irq(lif, qcq);
+ if (err) {
+ netdev_warn(lif->netdev, "irq request failed %d\n", err);
+ goto err_out_free_intr;
+ }
+
+ /* try to get the irq on the local numa node first */
+ qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
+ dev_to_node(lif->ionic->dev));
+ if (qcq->intr.cpu != -1)
+ cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
+
+ netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
+ return 0;
+
+err_out_free_intr:
+ ionic_intr_free(lif->ionic, qcq->intr.index);
+err_out:
+ return err;
+}
+
+static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
+ unsigned int index,
+ const char *name, unsigned int flags,
+ unsigned int num_descs, unsigned int desc_size,
+ unsigned int cq_desc_size,
+ unsigned int sg_desc_size,
+ unsigned int pid, struct ionic_qcq **qcq)
+{
+ struct ionic_dev *idev = &lif->ionic->idev;
+ struct device *dev = lif->ionic->dev;
+ void *q_base, *cq_base, *sg_base;
+ dma_addr_t cq_base_pa = 0;
+ dma_addr_t sg_base_pa = 0;
+ dma_addr_t q_base_pa = 0;
+ struct ionic_qcq *new;
+ int err;
+
+ *qcq = NULL;
+
+ new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
+ if (!new) {
+ netdev_err(lif->netdev, "Cannot allocate queue structure\n");
+ err = -ENOMEM;
+ goto err_out;
+ }
+
+ new->q.dev = dev;
+ new->flags = flags;
+
+ new->q.info = vzalloc(num_descs * sizeof(*new->q.info));
+ if (!new->q.info) {
+ netdev_err(lif->netdev, "Cannot allocate queue info\n");
+ err = -ENOMEM;
+ goto err_out_free_qcq;
+ }
+
+ new->q.type = type;
+ new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
+
+ err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
+ desc_size, sg_desc_size, pid);
+ if (err) {
+ netdev_err(lif->netdev, "Cannot initialize queue\n");
+ goto err_out_free_q_info;
+ }
+
+ err = ionic_alloc_qcq_interrupt(lif, new);
+ if (err)
+ goto err_out;
+
+ new->cq.info = vzalloc(num_descs * sizeof(*new->cq.info));
+ if (!new->cq.info) {
+ netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
+ err = -ENOMEM;
+ goto err_out_free_irq;
+ }
+
+ err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
+ if (err) {
+ netdev_err(lif->netdev, "Cannot initialize completion queue\n");
+ goto err_out_free_cq_info;
+ }
+
+ if (flags & IONIC_QCQ_F_NOTIFYQ) {
+ int q_size, cq_size;
+
+ /* q & cq need to be contiguous in case of notifyq */
+ q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
+ cq_size = ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
+
+ new->q_size = PAGE_SIZE + q_size + cq_size;
+ new->q_base = dma_alloc_coherent(dev, new->q_size,
+ &new->q_base_pa, GFP_KERNEL);
+ if (!new->q_base) {
+ netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
+ err = -ENOMEM;
+ goto err_out_free_cq_info;
+ }
+ q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
+ q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
+ ionic_q_map(&new->q, q_base, q_base_pa);
+
+ cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
+ cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
+ ionic_cq_map(&new->cq, cq_base, cq_base_pa);
+ ionic_cq_bind(&new->cq, &new->q);
+ } else {
+ new->q_size = PAGE_SIZE + (num_descs * desc_size);
+ new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
+ GFP_KERNEL);
+ if (!new->q_base) {
+ netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
+ err = -ENOMEM;
+ goto err_out_free_cq_info;
+ }
+ q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
+ q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
+ ionic_q_map(&new->q, q_base, q_base_pa);
+
+ new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
+ new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
+ GFP_KERNEL);
+ if (!new->cq_base) {
+ netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
+ err = -ENOMEM;
+ goto err_out_free_q;
+ }
+ cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
+ cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
+ ionic_cq_map(&new->cq, cq_base, cq_base_pa);
+ ionic_cq_bind(&new->cq, &new->q);
+ }
+
+ if (flags & IONIC_QCQ_F_SG) {
+ new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
+ new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
+ GFP_KERNEL);
+ if (!new->sg_base) {
+ netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
+ err = -ENOMEM;
+ goto err_out_free_cq;
+ }
+ sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
+ sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
+ ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
+ }
+
+ INIT_WORK(&new->dim.work, ionic_dim_work);
+ new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
+
+ *qcq = new;
+
+ return 0;
+
+err_out_free_cq:
+ dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
+err_out_free_q:
+ dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
+err_out_free_cq_info:
+ vfree(new->cq.info);
+err_out_free_irq:
+ if (flags & IONIC_QCQ_F_INTR) {
+ devm_free_irq(dev, new->intr.vector, &new->napi);
+ ionic_intr_free(lif->ionic, new->intr.index);
+ }
+err_out_free_q_info:
+ vfree(new->q.info);
+err_out_free_qcq:
+ devm_kfree(dev, new);
+err_out:
+ dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
+ return err;
+}
+
+static int ionic_qcqs_alloc(struct ionic_lif *lif)
+{
+ struct device *dev = lif->ionic->dev;
+ unsigned int flags;
+ int err;
+
+ flags = IONIC_QCQ_F_INTR;
+ err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
+ IONIC_ADMINQ_LENGTH,
+ sizeof(struct ionic_admin_cmd),
+ sizeof(struct ionic_admin_comp),
+ 0, lif->kern_pid, &lif->adminqcq);
+ if (err)
+ return err;
+ ionic_debugfs_add_qcq(lif, lif->adminqcq);
+
+ if (lif->ionic->nnqs_per_lif) {
+ flags = IONIC_QCQ_F_NOTIFYQ;
+ err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
+ flags, IONIC_NOTIFYQ_LENGTH,
+ sizeof(struct ionic_notifyq_cmd),
+ sizeof(union ionic_notifyq_comp),
+ 0, lif->kern_pid, &lif->notifyqcq);
+ if (err)
+ goto err_out;
+ ionic_debugfs_add_qcq(lif, lif->notifyqcq);
+
+ /* Let the notifyq ride on the adminq interrupt */
+ ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
+ }
+
+ err = -ENOMEM;
+ lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
+ sizeof(*lif->txqcqs), GFP_KERNEL);
+ if (!lif->txqcqs)
+ goto err_out;
+ lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
+ sizeof(*lif->rxqcqs), GFP_KERNEL);
+ if (!lif->rxqcqs)
+ goto err_out;
+
+ lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
+ sizeof(*lif->txqstats), GFP_KERNEL);
+ if (!lif->txqstats)
+ goto err_out;
+ lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
+ sizeof(*lif->rxqstats), GFP_KERNEL);
+ if (!lif->rxqstats)
+ goto err_out;
+
+ return 0;
+
+err_out:
+ ionic_qcqs_free(lif);
+ return err;
+}
+
+static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
+{
+ qcq->q.tail_idx = 0;
+ qcq->q.head_idx = 0;
+ qcq->cq.tail_idx = 0;
+ qcq->cq.done_color = 1;
+ memset(qcq->q_base, 0, qcq->q_size);
+ memset(qcq->cq_base, 0, qcq->cq_size);
+ memset(qcq->sg_base, 0, qcq->sg_size);
+}
+
+static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
+{
+ struct device *dev = lif->ionic->dev;
+ struct ionic_queue *q = &qcq->q;
+ struct ionic_cq *cq = &qcq->cq;
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.q_init = {
+ .opcode = IONIC_CMD_Q_INIT,
+ .lif_index = cpu_to_le16(lif->index),
+ .type = q->type,
+ .ver = lif->qtype_info[q->type].version,
+ .index = cpu_to_le32(q->index),
+ .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
+ IONIC_QINIT_F_SG),
+ .pid = cpu_to_le16(q->pid),
+ .ring_size = ilog2(q->num_descs),
+ .ring_base = cpu_to_le64(q->base_pa),
+ .cq_ring_base = cpu_to_le64(cq->base_pa),
+ .sg_ring_base = cpu_to_le64(q->sg_base_pa),
+ .features = cpu_to_le64(q->features),
+ },
+ };
+ unsigned int intr_index;
+ int err;
+
+ intr_index = qcq->intr.index;
+
+ ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index);
+
+ dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
+ dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
+ dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
+ dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
+ dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
+ dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
+ dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
+
+ ionic_qcq_sanitize(qcq);
+
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err)
+ return err;
+
+ q->hw_type = ctx.comp.q_init.hw_type;
+ q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
+ q->dbval = IONIC_DBELL_QID(q->hw_index);
+
+ dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
+ dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
+
+ q->dbell_deadline = IONIC_TX_DOORBELL_DEADLINE;
+ q->dbell_jiffies = jiffies;
+
+ if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {
+ netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);
+ qcq->napi_qcq = qcq;
+ timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
+ }
+
+ qcq->flags |= IONIC_QCQ_F_INITED;
+
+ return 0;
+}
+
+static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
+{
+ struct device *dev = lif->ionic->dev;
+ struct ionic_queue *q = &qcq->q;
+ struct ionic_cq *cq = &qcq->cq;
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.q_init = {
+ .opcode = IONIC_CMD_Q_INIT,
+ .lif_index = cpu_to_le16(lif->index),
+ .type = q->type,
+ .ver = lif->qtype_info[q->type].version,
+ .index = cpu_to_le32(q->index),
+ .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
+ IONIC_QINIT_F_SG),
+ .intr_index = cpu_to_le16(cq->bound_intr->index),
+ .pid = cpu_to_le16(q->pid),
+ .ring_size = ilog2(q->num_descs),
+ .ring_base = cpu_to_le64(q->base_pa),
+ .cq_ring_base = cpu_to_le64(cq->base_pa),
+ .sg_ring_base = cpu_to_le64(q->sg_base_pa),
+ .features = cpu_to_le64(q->features),
+ },
+ };
+ int err;
+
+ dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
+ dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
+ dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
+ dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
+ dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
+ dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
+ dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
+
+ ionic_qcq_sanitize(qcq);
+
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err)
+ return err;
+
+ q->hw_type = ctx.comp.q_init.hw_type;
+ q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
+ q->dbval = IONIC_DBELL_QID(q->hw_index);
+
+ dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
+ dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
+
+ q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
+ q->dbell_jiffies = jiffies;
+
+ if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
+ netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);
+ else
+ netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);
+
+ qcq->napi_qcq = qcq;
+ timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
+
+ qcq->flags |= IONIC_QCQ_F_INITED;
+
+ return 0;
+}
+
+int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
+{
+ unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
+ unsigned int txq_i, flags;
+ struct ionic_qcq *txq;
+ u64 features;
+ int err;
+
+ if (lif->hwstamp_txq)
+ return 0;
+
+ features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
+
+ num_desc = IONIC_MIN_TXRX_DESC;
+ desc_sz = sizeof(struct ionic_txq_desc);
+ comp_sz = 2 * sizeof(struct ionic_txq_comp);
+
+ if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
+ lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
+ sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
+ else
+ sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
+
+ txq_i = lif->ionic->ntxqs_per_lif;
+ flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
+
+ err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
+ num_desc, desc_sz, comp_sz, sg_desc_sz,
+ lif->kern_pid, &txq);
+ if (err)
+ goto err_qcq_alloc;
+
+ txq->q.features = features;
+
+ ionic_link_qcq_interrupts(lif->adminqcq, txq);
+ ionic_debugfs_add_qcq(lif, txq);
+
+ lif->hwstamp_txq = txq;
+
+ if (netif_running(lif->netdev)) {
+ err = ionic_lif_txq_init(lif, txq);
+ if (err)
+ goto err_qcq_init;
+
+ if (test_bit(IONIC_LIF_F_UP, lif->state)) {
+ err = ionic_qcq_enable(txq);
+ if (err)
+ goto err_qcq_enable;
+ }
+ }
+
+ return 0;
+
+err_qcq_enable:
+ ionic_lif_qcq_deinit(lif, txq);
+err_qcq_init:
+ lif->hwstamp_txq = NULL;
+ ionic_debugfs_del_qcq(txq);
+ ionic_qcq_free(lif, txq);
+ devm_kfree(lif->ionic->dev, txq);
+err_qcq_alloc:
+ return err;
+}
+
+int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
+{
+ unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
+ unsigned int rxq_i, flags;
+ struct ionic_qcq *rxq;
+ u64 features;
+ int err;
+
+ if (lif->hwstamp_rxq)
+ return 0;
+
+ features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
+
+ num_desc = IONIC_MIN_TXRX_DESC;
+ desc_sz = sizeof(struct ionic_rxq_desc);
+ comp_sz = 2 * sizeof(struct ionic_rxq_comp);
+ sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
+
+ rxq_i = lif->ionic->nrxqs_per_lif;
+ flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
+
+ err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
+ num_desc, desc_sz, comp_sz, sg_desc_sz,
+ lif->kern_pid, &rxq);
+ if (err)
+ goto err_qcq_alloc;
+
+ rxq->q.features = features;
+
+ ionic_link_qcq_interrupts(lif->adminqcq, rxq);
+ ionic_debugfs_add_qcq(lif, rxq);
+
+ lif->hwstamp_rxq = rxq;
+
+ if (netif_running(lif->netdev)) {
+ err = ionic_lif_rxq_init(lif, rxq);
+ if (err)
+ goto err_qcq_init;
+
+ if (test_bit(IONIC_LIF_F_UP, lif->state)) {
+ ionic_rx_fill(&rxq->q);
+ err = ionic_qcq_enable(rxq);
+ if (err)
+ goto err_qcq_enable;
+ }
+ }
+
+ return 0;
+
+err_qcq_enable:
+ ionic_lif_qcq_deinit(lif, rxq);
+err_qcq_init:
+ lif->hwstamp_rxq = NULL;
+ ionic_debugfs_del_qcq(rxq);
+ ionic_qcq_free(lif, rxq);
+ devm_kfree(lif->ionic->dev, rxq);
+err_qcq_alloc:
+ return err;
+}
+
+int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
+{
+ struct ionic_queue_params qparam;
+
+ ionic_init_queue_params(lif, &qparam);
+
+ if (rx_all)
+ qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
+ else
+ qparam.rxq_features = 0;
+
+ /* if we're not running, just set the values and return */
+ if (!netif_running(lif->netdev)) {
+ lif->rxq_features = qparam.rxq_features;
+ return 0;
+ }
+
+ return ionic_reconfigure_queues(lif, &qparam);
+}
+
+int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
+{
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.lif_setattr = {
+ .opcode = IONIC_CMD_LIF_SETATTR,
+ .index = cpu_to_le16(lif->index),
+ .attr = IONIC_LIF_ATTR_TXSTAMP,
+ .txstamp_mode = cpu_to_le16(txstamp_mode),
+ },
+ };
+
+ return ionic_adminq_post_wait(lif, &ctx);
+}
+
+static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
+{
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.rx_filter_del = {
+ .opcode = IONIC_CMD_RX_FILTER_DEL,
+ .lif_index = cpu_to_le16(lif->index),
+ },
+ };
+ struct ionic_rx_filter *f;
+ u32 filter_id;
+ int err;
+
+ spin_lock_bh(&lif->rx_filters.lock);
+
+ f = ionic_rx_filter_rxsteer(lif);
+ if (!f) {
+ spin_unlock_bh(&lif->rx_filters.lock);
+ return;
+ }
+
+ filter_id = f->filter_id;
+ ionic_rx_filter_free(lif, f);
+
+ spin_unlock_bh(&lif->rx_filters.lock);
+
+ netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
+
+ ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
+
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err && err != -EEXIST)
+ netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
+}
+
+static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
+{
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.rx_filter_add = {
+ .opcode = IONIC_CMD_RX_FILTER_ADD,
+ .lif_index = cpu_to_le16(lif->index),
+ .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
+ .pkt_class = cpu_to_le64(pkt_class),
+ },
+ };
+ u8 qtype;
+ u32 qid;
+ int err;
+
+ if (!lif->hwstamp_rxq)
+ return -EINVAL;
+
+ qtype = lif->hwstamp_rxq->q.type;
+ ctx.cmd.rx_filter_add.qtype = qtype;
+
+ qid = lif->hwstamp_rxq->q.index;
+ ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
+
+ netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err && err != -EEXIST)
+ return err;
+
+ spin_lock_bh(&lif->rx_filters.lock);
+ err = ionic_rx_filter_save(lif, 0, qid, 0, &ctx, IONIC_FILTER_STATE_SYNCED);
+ spin_unlock_bh(&lif->rx_filters.lock);
+
+ return err;
+}
+
+int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
+{
+ ionic_lif_del_hwstamp_rxfilt(lif);
+
+ if (!pkt_class)
+ return 0;
+
+ return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
+}
+
+static bool ionic_notifyq_service(struct ionic_cq *cq,
+ struct ionic_cq_info *cq_info)
+{
+ union ionic_notifyq_comp *comp = cq_info->cq_desc;
+ struct ionic_deferred_work *work;
+ struct net_device *netdev;
+ struct ionic_queue *q;
+ struct ionic_lif *lif;
+ u64 eid;
+
+ q = cq->bound_q;
+ lif = q->info[0].cb_arg;
+ netdev = lif->netdev;
+ eid = le64_to_cpu(comp->event.eid);
+
+ /* Have we run out of new completions to process? */
+ if ((s64)(eid - lif->last_eid) <= 0)
+ return false;
+
+ lif->last_eid = eid;
+
+ dev_dbg(lif->ionic->dev, "notifyq event:\n");
+ dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
+ comp, sizeof(*comp), true);
+
+ switch (le16_to_cpu(comp->event.ecode)) {
+ case IONIC_EVENT_LINK_CHANGE:
+ ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
+ break;
+ case IONIC_EVENT_RESET:
+ if (lif->ionic->idev.fw_status_ready &&
+ !test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
+ !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
+ work = kzalloc(sizeof(*work), GFP_ATOMIC);
+ if (!work) {
+ netdev_err(lif->netdev, "Reset event dropped\n");
+ clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
+ } else {
+ work->type = IONIC_DW_TYPE_LIF_RESET;
+ ionic_lif_deferred_enqueue(&lif->deferred, work);
+ }
+ }
+ break;
+ default:
+ netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
+ comp->event.ecode, eid);
+ break;
+ }
+
+ return true;
+}
+
+static bool ionic_adminq_service(struct ionic_cq *cq,
+ struct ionic_cq_info *cq_info)
+{
+ struct ionic_admin_comp *comp = cq_info->cq_desc;
+
+ if (!color_match(comp->color, cq->done_color))
+ return false;
+
+ ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
+
+ return true;
+}
+
+static int ionic_adminq_napi(struct napi_struct *napi, int budget)
+{
+ struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
+ struct ionic_lif *lif = napi_to_cq(napi)->lif;
+ struct ionic_dev *idev = &lif->ionic->idev;
+ unsigned long irqflags;
+ unsigned int flags = 0;
+ bool resched = false;
+ int rx_work = 0;
+ int tx_work = 0;
+ int n_work = 0;
+ int a_work = 0;
+ int work_done;
+ int credits;
+
+ if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
+ n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
+ ionic_notifyq_service, NULL, NULL);
+
+ spin_lock_irqsave(&lif->adminq_lock, irqflags);
+ if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
+ a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
+ ionic_adminq_service, NULL, NULL);
+ spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
+
+ if (lif->hwstamp_rxq)
+ rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
+ ionic_rx_service, NULL, NULL);
+
+ if (lif->hwstamp_txq)
+ tx_work = ionic_cq_service(&lif->hwstamp_txq->cq, budget,
+ ionic_tx_service, NULL, NULL);
+
+ work_done = max(max(n_work, a_work), max(rx_work, tx_work));
+ if (work_done < budget && napi_complete_done(napi, work_done)) {
+ flags |= IONIC_INTR_CRED_UNMASK;
+ intr->rearm_count++;
+ }
+
+ if (work_done || flags) {
+ flags |= IONIC_INTR_CRED_RESET_COALESCE;
+ credits = n_work + a_work + rx_work + tx_work;
+ ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
+ }
+
+ if (!a_work && ionic_adminq_poke_doorbell(&lif->adminqcq->q))
+ resched = true;
+ if (lif->hwstamp_rxq && !rx_work && ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q))
+ resched = true;
+ if (lif->hwstamp_txq && !tx_work && ionic_txq_poke_doorbell(&lif->hwstamp_txq->q))
+ resched = true;
+ if (resched)
+ mod_timer(&lif->adminqcq->napi_deadline,
+ jiffies + IONIC_NAPI_DEADLINE);
+
+ return work_done;
+}
+
+void ionic_get_stats64(struct net_device *netdev,
+ struct rtnl_link_stats64 *ns)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_lif_stats *ls;
+
+ memset(ns, 0, sizeof(*ns));
+ ls = &lif->info->stats;
+
+ ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
+ le64_to_cpu(ls->rx_mcast_packets) +
+ le64_to_cpu(ls->rx_bcast_packets);
+
+ ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
+ le64_to_cpu(ls->tx_mcast_packets) +
+ le64_to_cpu(ls->tx_bcast_packets);
+
+ ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
+ le64_to_cpu(ls->rx_mcast_bytes) +
+ le64_to_cpu(ls->rx_bcast_bytes);
+
+ ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
+ le64_to_cpu(ls->tx_mcast_bytes) +
+ le64_to_cpu(ls->tx_bcast_bytes);
+
+ ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
+ le64_to_cpu(ls->rx_mcast_drop_packets) +
+ le64_to_cpu(ls->rx_bcast_drop_packets);
+
+ ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
+ le64_to_cpu(ls->tx_mcast_drop_packets) +
+ le64_to_cpu(ls->tx_bcast_drop_packets);
+
+ ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
+
+ ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
+
+ ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
+ le64_to_cpu(ls->rx_queue_disabled) +
+ le64_to_cpu(ls->rx_desc_fetch_error) +
+ le64_to_cpu(ls->rx_desc_data_error);
+
+ ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
+ le64_to_cpu(ls->tx_queue_disabled) +
+ le64_to_cpu(ls->tx_desc_fetch_error) +
+ le64_to_cpu(ls->tx_desc_data_error);
+
+ ns->rx_errors = ns->rx_over_errors +
+ ns->rx_missed_errors;
+
+ ns->tx_errors = ns->tx_aborted_errors;
+}
+
+static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
+{
+ return ionic_lif_list_addr(netdev_priv(netdev), addr, ADD_ADDR);
+}
+
+static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
+{
+ /* Don't delete our own address from the uc list */
+ if (ether_addr_equal(addr, netdev->dev_addr))
+ return 0;
+
+ return ionic_lif_list_addr(netdev_priv(netdev), addr, DEL_ADDR);
+}
+
+void ionic_lif_rx_mode(struct ionic_lif *lif)
+{
+ struct net_device *netdev = lif->netdev;
+ unsigned int nfilters;
+ unsigned int nd_flags;
+ char buf[128];
+ u16 rx_mode;
+ int i;
+#define REMAIN(__x) (sizeof(buf) - (__x))
+
+ mutex_lock(&lif->config_lock);
+
+ /* grab the flags once for local use */
+ nd_flags = netdev->flags;
+
+ rx_mode = IONIC_RX_MODE_F_UNICAST;
+ rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
+ rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
+ rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
+ rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
+
+ /* sync the filters */
+ ionic_rx_filter_sync(lif);
+
+ /* check for overflow state
+ * if so, we track that we overflowed and enable NIC PROMISC
+ * else if the overflow is set and not needed
+ * we remove our overflow flag and check the netdev flags
+ * to see if we can disable NIC PROMISC
+ */
+ nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
+
+ if (((lif->nucast + lif->nmcast) >= nfilters) ||
+ (lif->max_vlans && lif->nvlans >= lif->max_vlans)) {
+ rx_mode |= IONIC_RX_MODE_F_PROMISC;
+ rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
+ } else {
+ if (!(nd_flags & IFF_PROMISC))
+ rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
+ if (!(nd_flags & IFF_ALLMULTI))
+ rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
+ }
+
+ i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
+ lif->rx_mode, rx_mode);
+ if (rx_mode & IONIC_RX_MODE_F_UNICAST)
+ i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
+ if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
+ i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
+ if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
+ i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
+ if (rx_mode & IONIC_RX_MODE_F_PROMISC)
+ i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
+ if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
+ i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
+ if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)
+ i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");
+ netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);
+
+ if (lif->rx_mode != rx_mode) {
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.rx_mode_set = {
+ .opcode = IONIC_CMD_RX_MODE_SET,
+ .lif_index = cpu_to_le16(lif->index),
+ },
+ };
+ int err;
+
+ ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err)
+ netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",
+ rx_mode, err);
+ else
+ lif->rx_mode = rx_mode;
+ }
+
+ mutex_unlock(&lif->config_lock);
+}
+
+static void ionic_ndo_set_rx_mode(struct net_device *netdev)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_deferred_work *work;
+
+ /* Sync the kernel filter list with the driver filter list */
+ __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
+ __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
+
+ /* Shove off the rest of the rxmode work to the work task
+ * which will include syncing the filters to the firmware.
+ */
+ work = kzalloc(sizeof(*work), GFP_ATOMIC);
+ if (!work) {
+ netdev_err(lif->netdev, "rxmode change dropped\n");
+ return;
+ }
+ work->type = IONIC_DW_TYPE_RX_MODE;
+ netdev_dbg(lif->netdev, "deferred: rx_mode\n");
+ ionic_lif_deferred_enqueue(&lif->deferred, work);
+}
+
+static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
+{
+ u64 wanted = 0;
+
+ if (features & NETIF_F_HW_VLAN_CTAG_TX)
+ wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
+ if (features & NETIF_F_HW_VLAN_CTAG_RX)
+ wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
+ if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
+ wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
+ if (features & NETIF_F_RXHASH)
+ wanted |= IONIC_ETH_HW_RX_HASH;
+ if (features & NETIF_F_RXCSUM)
+ wanted |= IONIC_ETH_HW_RX_CSUM;
+ if (features & NETIF_F_SG)
+ wanted |= IONIC_ETH_HW_TX_SG;
+ if (features & NETIF_F_HW_CSUM)
+ wanted |= IONIC_ETH_HW_TX_CSUM;
+ if (features & NETIF_F_TSO)
+ wanted |= IONIC_ETH_HW_TSO;
+ if (features & NETIF_F_TSO6)
+ wanted |= IONIC_ETH_HW_TSO_IPV6;
+ if (features & NETIF_F_TSO_ECN)
+ wanted |= IONIC_ETH_HW_TSO_ECN;
+ if (features & NETIF_F_GSO_GRE)
+ wanted |= IONIC_ETH_HW_TSO_GRE;
+ if (features & NETIF_F_GSO_GRE_CSUM)
+ wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
+ if (features & NETIF_F_GSO_IPXIP4)
+ wanted |= IONIC_ETH_HW_TSO_IPXIP4;
+ if (features & NETIF_F_GSO_IPXIP6)
+ wanted |= IONIC_ETH_HW_TSO_IPXIP6;
+ if (features & NETIF_F_GSO_UDP_TUNNEL)
+ wanted |= IONIC_ETH_HW_TSO_UDP;
+ if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
+ wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
+
+ return cpu_to_le64(wanted);
+}
+
+static int ionic_set_nic_features(struct ionic_lif *lif,
+ netdev_features_t features)
+{
+ struct device *dev = lif->ionic->dev;
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.lif_setattr = {
+ .opcode = IONIC_CMD_LIF_SETATTR,
+ .index = cpu_to_le16(lif->index),
+ .attr = IONIC_LIF_ATTR_FEATURES,
+ },
+ };
+ u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
+ IONIC_ETH_HW_VLAN_RX_STRIP |
+ IONIC_ETH_HW_VLAN_RX_FILTER;
+ u64 old_hw_features;
+ int err;
+
+ ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
+
+ if (lif->phc)
+ ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
+
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err)
+ return err;
+
+ old_hw_features = lif->hw_features;
+ lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
+ ctx.comp.lif_setattr.features);
+
+ if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
+ ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
+
+ if ((vlan_flags & le64_to_cpu(ctx.cmd.lif_setattr.features)) &&
+ !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
+ dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
+
+ if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
+ dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
+ if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
+ dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
+ if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
+ dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
+ if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
+ dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
+ if (lif->hw_features & IONIC_ETH_HW_TX_SG)
+ dev_dbg(dev, "feature ETH_HW_TX_SG\n");
+ if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
+ dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
+ if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
+ dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
+ if (lif->hw_features & IONIC_ETH_HW_TSO)
+ dev_dbg(dev, "feature ETH_HW_TSO\n");
+ if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
+ dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
+ if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
+ dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
+ if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
+ dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
+ if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
+ dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
+ if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
+ dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
+ if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
+ dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
+ if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
+ dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
+ if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
+ dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
+ if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
+ dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
+
+ return 0;
+}
+
+static int ionic_init_nic_features(struct ionic_lif *lif)
+{
+ struct net_device *netdev = lif->netdev;
+ netdev_features_t features;
+ int err;
+
+ /* set up what we expect to support by default */
+ features = NETIF_F_HW_VLAN_CTAG_TX |
+ NETIF_F_HW_VLAN_CTAG_RX |
+ NETIF_F_HW_VLAN_CTAG_FILTER |
+ NETIF_F_SG |
+ NETIF_F_HW_CSUM |
+ NETIF_F_RXCSUM |
+ NETIF_F_TSO |
+ NETIF_F_TSO6 |
+ NETIF_F_TSO_ECN;
+
+ if (lif->nxqs > 1)
+ features |= NETIF_F_RXHASH;
+
+ err = ionic_set_nic_features(lif, features);
+ if (err)
+ return err;
+
+ /* tell the netdev what we actually can support */
+ netdev->features |= NETIF_F_HIGHDMA;
+
+ if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
+ netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
+ if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
+ netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
+ if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
+ netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
+ if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
+ netdev->hw_features |= NETIF_F_RXHASH;
+ if (lif->hw_features & IONIC_ETH_HW_TX_SG)
+ netdev->hw_features |= NETIF_F_SG;
+
+ if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
+ netdev->hw_enc_features |= NETIF_F_HW_CSUM;
+ if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
+ netdev->hw_enc_features |= NETIF_F_RXCSUM;
+ if (lif->hw_features & IONIC_ETH_HW_TSO)
+ netdev->hw_enc_features |= NETIF_F_TSO;
+ if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
+ netdev->hw_enc_features |= NETIF_F_TSO6;
+ if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
+ netdev->hw_enc_features |= NETIF_F_TSO_ECN;
+ if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
+ netdev->hw_enc_features |= NETIF_F_GSO_GRE;
+ if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
+ netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
+ if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
+ netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
+ if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
+ netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
+ if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
+ netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
+ if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
+ netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
+
+ netdev->hw_features |= netdev->hw_enc_features;
+ netdev->features |= netdev->hw_features;
+ netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
+
+ netdev->priv_flags |= IFF_UNICAST_FLT |
+ IFF_LIVE_ADDR_CHANGE;
+
+ return 0;
+}
+
+static int ionic_set_features(struct net_device *netdev,
+ netdev_features_t features)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ int err;
+
+ netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
+ __func__, (u64)lif->netdev->features, (u64)features);
+
+ err = ionic_set_nic_features(lif, features);
+
+ return err;
+}
+
+static int ionic_set_attr_mac(struct ionic_lif *lif, u8 *mac)
+{
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.lif_setattr = {
+ .opcode = IONIC_CMD_LIF_SETATTR,
+ .index = cpu_to_le16(lif->index),
+ .attr = IONIC_LIF_ATTR_MAC,
+ },
+ };
+
+ ether_addr_copy(ctx.cmd.lif_setattr.mac, mac);
+ return ionic_adminq_post_wait(lif, &ctx);
+}
+
+static int ionic_get_attr_mac(struct ionic_lif *lif, u8 *mac_addr)
+{
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.lif_getattr = {
+ .opcode = IONIC_CMD_LIF_GETATTR,
+ .index = cpu_to_le16(lif->index),
+ .attr = IONIC_LIF_ATTR_MAC,
+ },
+ };
+ int err;
+
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err)
+ return err;
+
+ ether_addr_copy(mac_addr, ctx.comp.lif_getattr.mac);
+ return 0;
+}
+
+static int ionic_program_mac(struct ionic_lif *lif, u8 *mac)
+{
+ u8 get_mac[ETH_ALEN];
+ int err;
+
+ err = ionic_set_attr_mac(lif, mac);
+ if (err)
+ return err;
+
+ err = ionic_get_attr_mac(lif, get_mac);
+ if (err)
+ return err;
+
+ /* To deal with older firmware that silently ignores the set attr mac:
+ * doesn't actually change the mac and doesn't return an error, so we
+ * do the get attr to verify whether or not the set actually happened
+ */
+ if (!ether_addr_equal(get_mac, mac))
+ return 1;
+
+ return 0;
+}
+
+static int ionic_set_mac_address(struct net_device *netdev, void *sa)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct sockaddr *addr = sa;
+ u8 *mac;
+ int err;
+
+ mac = (u8 *)addr->sa_data;
+ if (ether_addr_equal(netdev->dev_addr, mac))
+ return 0;
+
+ err = ionic_program_mac(lif, mac);
+ if (err < 0)
+ return err;
+
+ if (err > 0)
+ netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n",
+ __func__);
+
+ err = eth_prepare_mac_addr_change(netdev, addr);
+ if (err)
+ return err;
+
+ if (!is_zero_ether_addr(netdev->dev_addr)) {
+ netdev_info(netdev, "deleting mac addr %pM\n",
+ netdev->dev_addr);
+ ionic_lif_addr_del(netdev_priv(netdev), netdev->dev_addr);
+ }
+
+ eth_commit_mac_addr_change(netdev, addr);
+ netdev_info(netdev, "updating mac addr %pM\n", mac);
+
+ return ionic_lif_addr_add(netdev_priv(netdev), mac);
+}
+
+static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
+{
+ /* Stop and clean the queues before reconfiguration */
+ netif_device_detach(lif->netdev);
+ ionic_stop_queues(lif);
+ ionic_txrx_deinit(lif);
+}
+
+static int ionic_start_queues_reconfig(struct ionic_lif *lif)
+{
+ int err;
+
+ /* Re-init the queues after reconfiguration */
+
+ /* The only way txrx_init can fail here is if communication
+ * with FW is suddenly broken. There's not much we can do
+ * at this point - error messages have already been printed,
+ * so we can continue on and the user can eventually do a
+ * DOWN and UP to try to reset and clear the issue.
+ */
+ err = ionic_txrx_init(lif);
+ ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
+ netif_device_attach(lif->netdev);
+
+ return err;
+}
+
+static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.lif_setattr = {
+ .opcode = IONIC_CMD_LIF_SETATTR,
+ .index = cpu_to_le16(lif->index),
+ .attr = IONIC_LIF_ATTR_MTU,
+ .mtu = cpu_to_le32(new_mtu),
+ },
+ };
+ int err;
+
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err)
+ return err;
+
+ /* if we're not running, nothing more to do */
+ if (!netif_running(netdev)) {
+ netdev->mtu = new_mtu;
+ return 0;
+ }
+
+ mutex_lock(&lif->queue_lock);
+ ionic_stop_queues_reconfig(lif);
+ netdev->mtu = new_mtu;
+ err = ionic_start_queues_reconfig(lif);
+ mutex_unlock(&lif->queue_lock);
+
+ return err;
+}
+
+static void ionic_tx_timeout_work(struct work_struct *ws)
+{
+ struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
+
+ if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return;
+
+ /* if we were stopped before this scheduled job was launched,
+ * don't bother the queues as they are already stopped.
+ */
+ if (!netif_running(lif->netdev))
+ return;
+
+ mutex_lock(&lif->queue_lock);
+ ionic_stop_queues_reconfig(lif);
+ ionic_start_queues_reconfig(lif);
+ mutex_unlock(&lif->queue_lock);
+}
+
+static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+
+ netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
+ schedule_work(&lif->tx_timeout_work);
+}
+
+static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
+ u16 vid)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ int err;
+
+ err = ionic_lif_vlan_add(lif, vid);
+ if (err)
+ return err;
+
+ ionic_lif_rx_mode(lif);
+
+ return 0;
+}
+
+static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
+ u16 vid)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ int err;
+
+ err = ionic_lif_vlan_del(lif, vid);
+ if (err)
+ return err;
+
+ ionic_lif_rx_mode(lif);
+
+ return 0;
+}
+
+int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
+ const u8 *key, const u32 *indir)
+{
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.lif_setattr = {
+ .opcode = IONIC_CMD_LIF_SETATTR,
+ .attr = IONIC_LIF_ATTR_RSS,
+ .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
+ },
+ };
+ unsigned int i, tbl_sz;
+
+ if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
+ lif->rss_types = types;
+ ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
+ }
+
+ if (key)
+ memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
+
+ if (indir) {
+ tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
+ for (i = 0; i < tbl_sz; i++)
+ lif->rss_ind_tbl[i] = indir[i];
+ }
+
+ memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
+ IONIC_RSS_HASH_KEY_SIZE);
+
+ return ionic_adminq_post_wait(lif, &ctx);
+}
+
+static int ionic_lif_rss_init(struct ionic_lif *lif)
+{
+ unsigned int tbl_sz;
+ unsigned int i;
+
+ lif->rss_types = IONIC_RSS_TYPE_IPV4 |
+ IONIC_RSS_TYPE_IPV4_TCP |
+ IONIC_RSS_TYPE_IPV4_UDP |
+ IONIC_RSS_TYPE_IPV6 |
+ IONIC_RSS_TYPE_IPV6_TCP |
+ IONIC_RSS_TYPE_IPV6_UDP;
+
+ /* Fill indirection table with 'default' values */
+ tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
+ for (i = 0; i < tbl_sz; i++)
+ lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
+
+ return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
+}
+
+static void ionic_lif_rss_deinit(struct ionic_lif *lif)
+{
+ int tbl_sz;
+
+ tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
+ memset(lif->rss_ind_tbl, 0, tbl_sz);
+ memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
+
+ ionic_lif_rss_config(lif, 0x0, NULL, NULL);
+}
+
+static void ionic_lif_quiesce(struct ionic_lif *lif)
+{
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.lif_setattr = {
+ .opcode = IONIC_CMD_LIF_SETATTR,
+ .index = cpu_to_le16(lif->index),
+ .attr = IONIC_LIF_ATTR_STATE,
+ .state = IONIC_LIF_QUIESCE,
+ },
+ };
+ int err;
+
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err)
+ netdev_dbg(lif->netdev, "lif quiesce failed %d\n", err);
+}
+
+static void ionic_txrx_disable(struct ionic_lif *lif)
+{
+ unsigned int i;
+ int err = 0;
+
+ if (lif->txqcqs) {
+ for (i = 0; i < lif->nxqs; i++)
+ err = ionic_qcq_disable(lif, lif->txqcqs[i], err);
+ }
+
+ if (lif->hwstamp_txq)
+ err = ionic_qcq_disable(lif, lif->hwstamp_txq, err);
+
+ if (lif->rxqcqs) {
+ for (i = 0; i < lif->nxqs; i++)
+ err = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
+ }
+
+ if (lif->hwstamp_rxq)
+ err = ionic_qcq_disable(lif, lif->hwstamp_rxq, err);
+
+ ionic_lif_quiesce(lif);
+}
+
+static void ionic_txrx_deinit(struct ionic_lif *lif)
+{
+ unsigned int i;
+
+ if (lif->txqcqs) {
+ for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
+ ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
+ ionic_tx_flush(&lif->txqcqs[i]->cq);
+ ionic_tx_empty(&lif->txqcqs[i]->q);
+ }
+ }
+
+ if (lif->rxqcqs) {
+ for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
+ ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
+ ionic_rx_empty(&lif->rxqcqs[i]->q);
+ }
+ }
+ lif->rx_mode = 0;
+
+ if (lif->hwstamp_txq) {
+ ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
+ ionic_tx_flush(&lif->hwstamp_txq->cq);
+ ionic_tx_empty(&lif->hwstamp_txq->q);
+ }
+
+ if (lif->hwstamp_rxq) {
+ ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
+ ionic_rx_empty(&lif->hwstamp_rxq->q);
+ }
+}
+
+static void ionic_txrx_free(struct ionic_lif *lif)
+{
+ unsigned int i;
+
+ if (lif->txqcqs) {
+ for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
+ ionic_qcq_free(lif, lif->txqcqs[i]);
+ devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
+ lif->txqcqs[i] = NULL;
+ }
+ }
+
+ if (lif->rxqcqs) {
+ for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
+ ionic_qcq_free(lif, lif->rxqcqs[i]);
+ devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
+ lif->rxqcqs[i] = NULL;
+ }
+ }
+
+ if (lif->hwstamp_txq) {
+ ionic_qcq_free(lif, lif->hwstamp_txq);
+ devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
+ lif->hwstamp_txq = NULL;
+ }
+
+ if (lif->hwstamp_rxq) {
+ ionic_qcq_free(lif, lif->hwstamp_rxq);
+ devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
+ lif->hwstamp_rxq = NULL;
+ }
+}
+
+static int ionic_txrx_alloc(struct ionic_lif *lif)
+{
+ unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
+ unsigned int flags, i;
+ int err = 0;
+
+ num_desc = lif->ntxq_descs;
+ desc_sz = sizeof(struct ionic_txq_desc);
+ comp_sz = sizeof(struct ionic_txq_comp);
+
+ if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
+ lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
+ sizeof(struct ionic_txq_sg_desc_v1))
+ sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
+ else
+ sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
+
+ flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
+ if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
+ flags |= IONIC_QCQ_F_INTR;
+ for (i = 0; i < lif->nxqs; i++) {
+ err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
+ num_desc, desc_sz, comp_sz, sg_desc_sz,
+ lif->kern_pid, &lif->txqcqs[i]);
+ if (err)
+ goto err_out;
+
+ if (flags & IONIC_QCQ_F_INTR) {
+ ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
+ lif->txqcqs[i]->intr.index,
+ lif->tx_coalesce_hw);
+ if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
+ lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
+ }
+
+ ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
+ }
+
+ flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
+
+ num_desc = lif->nrxq_descs;
+ desc_sz = sizeof(struct ionic_rxq_desc);
+ comp_sz = sizeof(struct ionic_rxq_comp);
+ sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
+
+ if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
+ comp_sz *= 2;
+
+ for (i = 0; i < lif->nxqs; i++) {
+ err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
+ num_desc, desc_sz, comp_sz, sg_desc_sz,
+ lif->kern_pid, &lif->rxqcqs[i]);
+ if (err)
+ goto err_out;
+
+ lif->rxqcqs[i]->q.features = lif->rxq_features;
+
+ ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
+ lif->rxqcqs[i]->intr.index,
+ lif->rx_coalesce_hw);
+ if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
+ lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
+
+ if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
+ ionic_link_qcq_interrupts(lif->rxqcqs[i],
+ lif->txqcqs[i]);
+
+ ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
+ }
+
+ return 0;
+
+err_out:
+ ionic_txrx_free(lif);
+
+ return err;
+}
+
+static int ionic_txrx_init(struct ionic_lif *lif)
+{
+ unsigned int i;
+ int err;
+
+ for (i = 0; i < lif->nxqs; i++) {
+ err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
+ if (err)
+ goto err_out;
+
+ err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
+ if (err) {
+ ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
+ goto err_out;
+ }
+ }
+
+ if (lif->netdev->features & NETIF_F_RXHASH)
+ ionic_lif_rss_init(lif);
+
+ ionic_lif_rx_mode(lif);
+
+ return 0;
+
+err_out:
+ while (i--) {
+ ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
+ ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
+ }
+
+ return err;
+}
+
+static int ionic_txrx_enable(struct ionic_lif *lif)
+{
+ int derr = 0;
+ int i, err;
+
+ for (i = 0; i < lif->nxqs; i++) {
+ if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
+ dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
+ err = -ENXIO;
+ goto err_out;
+ }
+
+ ionic_rx_fill(&lif->rxqcqs[i]->q);
+ err = ionic_qcq_enable(lif->rxqcqs[i]);
+ if (err)
+ goto err_out;
+
+ err = ionic_qcq_enable(lif->txqcqs[i]);
+ if (err) {
+ derr = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
+ goto err_out;
+ }
+ }
+
+ if (lif->hwstamp_rxq) {
+ ionic_rx_fill(&lif->hwstamp_rxq->q);
+ err = ionic_qcq_enable(lif->hwstamp_rxq);
+ if (err)
+ goto err_out_hwstamp_rx;
+ }
+
+ if (lif->hwstamp_txq) {
+ err = ionic_qcq_enable(lif->hwstamp_txq);
+ if (err)
+ goto err_out_hwstamp_tx;
+ }
+
+ return 0;
+
+err_out_hwstamp_tx:
+ if (lif->hwstamp_rxq)
+ derr = ionic_qcq_disable(lif, lif->hwstamp_rxq, derr);
+err_out_hwstamp_rx:
+ i = lif->nxqs;
+err_out:
+ while (i--) {
+ derr = ionic_qcq_disable(lif, lif->txqcqs[i], derr);
+ derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr);
+ }
+
+ return err;
+}
+
+static int ionic_start_queues(struct ionic_lif *lif)
+{
+ int err;
+
+ if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
+ return -EIO;
+
+ if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return -EBUSY;
+
+ if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
+ return 0;
+
+ err = ionic_txrx_enable(lif);
+ if (err) {
+ clear_bit(IONIC_LIF_F_UP, lif->state);
+ return err;
+ }
+ netif_tx_wake_all_queues(lif->netdev);
+
+ return 0;
+}
+
+static int ionic_open(struct net_device *netdev)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ int err;
+
+ /* If recovering from a broken state, clear the bit and we'll try again */
+ if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
+ netdev_info(netdev, "clearing broken state\n");
+
+ mutex_lock(&lif->queue_lock);
+
+ err = ionic_txrx_alloc(lif);
+ if (err)
+ goto err_unlock;
+
+ err = ionic_txrx_init(lif);
+ if (err)
+ goto err_txrx_free;
+
+ err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
+ if (err)
+ goto err_txrx_deinit;
+
+ err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
+ if (err)
+ goto err_txrx_deinit;
+
+ /* don't start the queues until we have link */
+ if (netif_carrier_ok(netdev)) {
+ err = ionic_start_queues(lif);
+ if (err)
+ goto err_txrx_deinit;
+ }
+
+ /* If hardware timestamping is enabled, but the queues were freed by
+ * ionic_stop, those need to be reallocated and initialized, too.
+ */
+ ionic_lif_hwstamp_recreate_queues(lif);
+
+ mutex_unlock(&lif->queue_lock);
+
+ return 0;
+
+err_txrx_deinit:
+ ionic_txrx_deinit(lif);
+err_txrx_free:
+ ionic_txrx_free(lif);
+err_unlock:
+ mutex_unlock(&lif->queue_lock);
+ return err;
+}
+
+static void ionic_stop_queues(struct ionic_lif *lif)
+{
+ if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
+ return;
+
+ netif_tx_disable(lif->netdev);
+ ionic_txrx_disable(lif);
+}
+
+static int ionic_stop(struct net_device *netdev)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+
+ if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return 0;
+
+ mutex_lock(&lif->queue_lock);
+ ionic_stop_queues(lif);
+ ionic_txrx_deinit(lif);
+ ionic_txrx_free(lif);
+ mutex_unlock(&lif->queue_lock);
+
+ return 0;
+}
+
+static int ionic_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+
+ switch (cmd) {
+ case SIOCSHWTSTAMP:
+ return ionic_lif_hwstamp_set(lif, ifr);
+ case SIOCGHWTSTAMP:
+ return ionic_lif_hwstamp_get(lif, ifr);
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static int ionic_update_cached_vf_config(struct ionic *ionic, int vf)
+{
+ struct ionic_vf_getattr_comp comp = { 0 };
+ int err;
+ u8 attr;
+
+ attr = IONIC_VF_ATTR_VLAN;
+ err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
+ if (err && comp.status != IONIC_RC_ENOSUPP)
+ goto err_out;
+ if (!err)
+ ionic->vfs[vf].vlanid = comp.vlanid;
+
+ attr = IONIC_VF_ATTR_SPOOFCHK;
+ err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
+ if (err && comp.status != IONIC_RC_ENOSUPP)
+ goto err_out;
+ if (!err)
+ ionic->vfs[vf].spoofchk = comp.spoofchk;
+
+ attr = IONIC_VF_ATTR_LINKSTATE;
+ err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
+ if (err && comp.status != IONIC_RC_ENOSUPP)
+ goto err_out;
+ if (!err) {
+ switch (comp.linkstate) {
+ case IONIC_VF_LINK_STATUS_UP:
+ ionic->vfs[vf].linkstate = IFLA_VF_LINK_STATE_ENABLE;
+ break;
+ case IONIC_VF_LINK_STATUS_DOWN:
+ ionic->vfs[vf].linkstate = IFLA_VF_LINK_STATE_DISABLE;
+ break;
+ case IONIC_VF_LINK_STATUS_AUTO:
+ ionic->vfs[vf].linkstate = IFLA_VF_LINK_STATE_AUTO;
+ break;
+ default:
+ dev_warn(ionic->dev, "Unexpected link state %u\n", comp.linkstate);
+ break;
+ }
+ }
+
+ attr = IONIC_VF_ATTR_RATE;
+ err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
+ if (err && comp.status != IONIC_RC_ENOSUPP)
+ goto err_out;
+ if (!err)
+ ionic->vfs[vf].maxrate = comp.maxrate;
+
+ attr = IONIC_VF_ATTR_TRUST;
+ err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
+ if (err && comp.status != IONIC_RC_ENOSUPP)
+ goto err_out;
+ if (!err)
+ ionic->vfs[vf].trusted = comp.trust;
+
+ attr = IONIC_VF_ATTR_MAC;
+ err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
+ if (err && comp.status != IONIC_RC_ENOSUPP)
+ goto err_out;
+ if (!err)
+ ether_addr_copy(ionic->vfs[vf].macaddr, comp.macaddr);
+
+err_out:
+ if (err)
+ dev_err(ionic->dev, "Failed to get %s for VF %d\n",
+ ionic_vf_attr_to_str(attr), vf);
+
+ return err;
+}
+
+static int ionic_get_vf_config(struct net_device *netdev,
+ int vf, struct ifla_vf_info *ivf)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+ int ret = 0;
+
+ if (!netif_device_present(netdev))
+ return -EBUSY;
+
+ down_read(&ionic->vf_op_lock);
+
+ if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
+ ret = -EINVAL;
+ } else {
+ ivf->vf = vf;
+ ivf->qos = 0;
+
+ ret = ionic_update_cached_vf_config(ionic, vf);
+ if (!ret) {
+ ivf->vlan = le16_to_cpu(ionic->vfs[vf].vlanid);
+ ivf->spoofchk = ionic->vfs[vf].spoofchk;
+ ivf->linkstate = ionic->vfs[vf].linkstate;
+ ivf->max_tx_rate = le32_to_cpu(ionic->vfs[vf].maxrate);
+ ivf->trusted = ionic->vfs[vf].trusted;
+ ether_addr_copy(ivf->mac, ionic->vfs[vf].macaddr);
+ }
+ }
+
+ up_read(&ionic->vf_op_lock);
+ return ret;
+}
+
+static int ionic_get_vf_stats(struct net_device *netdev, int vf,
+ struct ifla_vf_stats *vf_stats)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+ struct ionic_lif_stats *vs;
+ int ret = 0;
+
+ if (!netif_device_present(netdev))
+ return -EBUSY;
+
+ down_read(&ionic->vf_op_lock);
+
+ if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
+ ret = -EINVAL;
+ } else {
+ memset(vf_stats, 0, sizeof(*vf_stats));
+ vs = &ionic->vfs[vf].stats;
+
+ vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
+ vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
+ vf_stats->rx_bytes = le64_to_cpu(vs->rx_ucast_bytes);
+ vf_stats->tx_bytes = le64_to_cpu(vs->tx_ucast_bytes);
+ vf_stats->broadcast = le64_to_cpu(vs->rx_bcast_packets);
+ vf_stats->multicast = le64_to_cpu(vs->rx_mcast_packets);
+ vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
+ le64_to_cpu(vs->rx_mcast_drop_packets) +
+ le64_to_cpu(vs->rx_bcast_drop_packets);
+ vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
+ le64_to_cpu(vs->tx_mcast_drop_packets) +
+ le64_to_cpu(vs->tx_bcast_drop_packets);
+ }
+
+ up_read(&ionic->vf_op_lock);
+ return ret;
+}
+
+static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
+{
+ struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_MAC };
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+ int ret;
+
+ if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
+ return -EINVAL;
+
+ if (!netif_device_present(netdev))
+ return -EBUSY;
+
+ down_write(&ionic->vf_op_lock);
+
+ if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
+ ret = -EINVAL;
+ } else {
+ ether_addr_copy(vfc.macaddr, mac);
+ dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
+ __func__, vf, vfc.macaddr);
+
+ ret = ionic_set_vf_config(ionic, vf, &vfc);
+ if (!ret)
+ ether_addr_copy(ionic->vfs[vf].macaddr, mac);
+ }
+
+ up_write(&ionic->vf_op_lock);
+ return ret;
+}
+
+static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
+ u8 qos, __be16 proto)
+{
+ struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_VLAN };
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+ int ret;
+
+ /* until someday when we support qos */
+ if (qos)
+ return -EINVAL;
+
+ if (vlan > 4095)
+ return -EINVAL;
+
+ if (proto != htons(ETH_P_8021Q))
+ return -EPROTONOSUPPORT;
+
+ if (!netif_device_present(netdev))
+ return -EBUSY;
+
+ down_write(&ionic->vf_op_lock);
+
+ if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
+ ret = -EINVAL;
+ } else {
+ vfc.vlanid = cpu_to_le16(vlan);
+ dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
+ __func__, vf, le16_to_cpu(vfc.vlanid));
+
+ ret = ionic_set_vf_config(ionic, vf, &vfc);
+ if (!ret)
+ ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
+ }
+
+ up_write(&ionic->vf_op_lock);
+ return ret;
+}
+
+static int ionic_set_vf_rate(struct net_device *netdev, int vf,
+ int tx_min, int tx_max)
+{
+ struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_RATE };
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+ int ret;
+
+ /* setting the min just seems silly */
+ if (tx_min)
+ return -EINVAL;
+
+ if (!netif_device_present(netdev))
+ return -EBUSY;
+
+ down_write(&ionic->vf_op_lock);
+
+ if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
+ ret = -EINVAL;
+ } else {
+ vfc.maxrate = cpu_to_le32(tx_max);
+ dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
+ __func__, vf, le32_to_cpu(vfc.maxrate));
+
+ ret = ionic_set_vf_config(ionic, vf, &vfc);
+ if (!ret)
+ lif->ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
+ }
+
+ up_write(&ionic->vf_op_lock);
+ return ret;
+}
+
+static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
+{
+ struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_SPOOFCHK };
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+ int ret;
+
+ if (!netif_device_present(netdev))
+ return -EBUSY;
+
+ down_write(&ionic->vf_op_lock);
+
+ if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
+ ret = -EINVAL;
+ } else {
+ vfc.spoofchk = set;
+ dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
+ __func__, vf, vfc.spoofchk);
+
+ ret = ionic_set_vf_config(ionic, vf, &vfc);
+ if (!ret)
+ ionic->vfs[vf].spoofchk = set;
+ }
+
+ up_write(&ionic->vf_op_lock);
+ return ret;
+}
+
+static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
+{
+ struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_TRUST };
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+ int ret;
+
+ if (!netif_device_present(netdev))
+ return -EBUSY;
+
+ down_write(&ionic->vf_op_lock);
+
+ if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
+ ret = -EINVAL;
+ } else {
+ vfc.trust = set;
+ dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
+ __func__, vf, vfc.trust);
+
+ ret = ionic_set_vf_config(ionic, vf, &vfc);
+ if (!ret)
+ ionic->vfs[vf].trusted = set;
+ }
+
+ up_write(&ionic->vf_op_lock);
+ return ret;
+}
+
+static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
+{
+ struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_LINKSTATE };
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic *ionic = lif->ionic;
+ u8 vfls;
+ int ret;
+
+ switch (set) {
+ case IFLA_VF_LINK_STATE_ENABLE:
+ vfls = IONIC_VF_LINK_STATUS_UP;
+ break;
+ case IFLA_VF_LINK_STATE_DISABLE:
+ vfls = IONIC_VF_LINK_STATUS_DOWN;
+ break;
+ case IFLA_VF_LINK_STATE_AUTO:
+ vfls = IONIC_VF_LINK_STATUS_AUTO;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (!netif_device_present(netdev))
+ return -EBUSY;
+
+ down_write(&ionic->vf_op_lock);
+
+ if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
+ ret = -EINVAL;
+ } else {
+ vfc.linkstate = vfls;
+ dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
+ __func__, vf, vfc.linkstate);
+
+ ret = ionic_set_vf_config(ionic, vf, &vfc);
+ if (!ret)
+ ionic->vfs[vf].linkstate = set;
+ }
+
+ up_write(&ionic->vf_op_lock);
+ return ret;
+}
+
+static const struct net_device_ops ionic_netdev_ops = {
+ .ndo_open = ionic_open,
+ .ndo_stop = ionic_stop,
+ .ndo_eth_ioctl = ionic_eth_ioctl,
+ .ndo_start_xmit = ionic_start_xmit,
+ .ndo_get_stats64 = ionic_get_stats64,
+ .ndo_set_rx_mode = ionic_ndo_set_rx_mode,
+ .ndo_set_features = ionic_set_features,
+ .ndo_set_mac_address = ionic_set_mac_address,
+ .ndo_validate_addr = eth_validate_addr,
+ .ndo_tx_timeout = ionic_tx_timeout,
+ .ndo_change_mtu = ionic_change_mtu,
+ .ndo_vlan_rx_add_vid = ionic_vlan_rx_add_vid,
+ .ndo_vlan_rx_kill_vid = ionic_vlan_rx_kill_vid,
+ .ndo_set_vf_vlan = ionic_set_vf_vlan,
+ .ndo_set_vf_trust = ionic_set_vf_trust,
+ .ndo_set_vf_mac = ionic_set_vf_mac,
+ .ndo_set_vf_rate = ionic_set_vf_rate,
+ .ndo_set_vf_spoofchk = ionic_set_vf_spoofchk,
+ .ndo_get_vf_config = ionic_get_vf_config,
+ .ndo_set_vf_link_state = ionic_set_vf_link_state,
+ .ndo_get_vf_stats = ionic_get_vf_stats,
+};
+
+static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
+{
+ /* only swapping the queues, not the napi, flags, or other stuff */
+ swap(a->q.features, b->q.features);
+ swap(a->q.num_descs, b->q.num_descs);
+ swap(a->q.desc_size, b->q.desc_size);
+ swap(a->q.base, b->q.base);
+ swap(a->q.base_pa, b->q.base_pa);
+ swap(a->q.info, b->q.info);
+ swap(a->q_base, b->q_base);
+ swap(a->q_base_pa, b->q_base_pa);
+ swap(a->q_size, b->q_size);
+
+ swap(a->q.sg_desc_size, b->q.sg_desc_size);
+ swap(a->q.sg_base, b->q.sg_base);
+ swap(a->q.sg_base_pa, b->q.sg_base_pa);
+ swap(a->sg_base, b->sg_base);
+ swap(a->sg_base_pa, b->sg_base_pa);
+ swap(a->sg_size, b->sg_size);
+
+ swap(a->cq.num_descs, b->cq.num_descs);
+ swap(a->cq.desc_size, b->cq.desc_size);
+ swap(a->cq.base, b->cq.base);
+ swap(a->cq.base_pa, b->cq.base_pa);
+ swap(a->cq.info, b->cq.info);
+ swap(a->cq_base, b->cq_base);
+ swap(a->cq_base_pa, b->cq_base_pa);
+ swap(a->cq_size, b->cq_size);
+
+ ionic_debugfs_del_qcq(a);
+ ionic_debugfs_add_qcq(a->q.lif, a);
+}
+
+int ionic_reconfigure_queues(struct ionic_lif *lif,
+ struct ionic_queue_params *qparam)
+{
+ unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
+ struct ionic_qcq **tx_qcqs = NULL;
+ struct ionic_qcq **rx_qcqs = NULL;
+ unsigned int flags, i;
+ int err = 0;
+
+ /* allocate temporary qcq arrays to hold new queue structs */
+ if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
+ tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
+ sizeof(struct ionic_qcq *), GFP_KERNEL);
+ if (!tx_qcqs) {
+ err = -ENOMEM;
+ goto err_out;
+ }
+ }
+ if (qparam->nxqs != lif->nxqs ||
+ qparam->nrxq_descs != lif->nrxq_descs ||
+ qparam->rxq_features != lif->rxq_features) {
+ rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
+ sizeof(struct ionic_qcq *), GFP_KERNEL);
+ if (!rx_qcqs) {
+ err = -ENOMEM;
+ goto err_out;
+ }
+ }
+
+ /* allocate new desc_info and rings, but leave the interrupt setup
+ * until later so as to not mess with the still-running queues
+ */
+ if (tx_qcqs) {
+ num_desc = qparam->ntxq_descs;
+ desc_sz = sizeof(struct ionic_txq_desc);
+ comp_sz = sizeof(struct ionic_txq_comp);
+
+ if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
+ lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
+ sizeof(struct ionic_txq_sg_desc_v1))
+ sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
+ else
+ sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
+
+ for (i = 0; i < qparam->nxqs; i++) {
+ flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
+ err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
+ num_desc, desc_sz, comp_sz, sg_desc_sz,
+ lif->kern_pid, &tx_qcqs[i]);
+ if (err)
+ goto err_out;
+ }
+ }
+
+ if (rx_qcqs) {
+ num_desc = qparam->nrxq_descs;
+ desc_sz = sizeof(struct ionic_rxq_desc);
+ comp_sz = sizeof(struct ionic_rxq_comp);
+ sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
+
+ if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
+ comp_sz *= 2;
+
+ for (i = 0; i < qparam->nxqs; i++) {
+ flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
+ err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
+ num_desc, desc_sz, comp_sz, sg_desc_sz,
+ lif->kern_pid, &rx_qcqs[i]);
+ if (err)
+ goto err_out;
+
+ rx_qcqs[i]->q.features = qparam->rxq_features;
+ }
+ }
+
+ /* stop and clean the queues */
+ ionic_stop_queues_reconfig(lif);
+
+ if (qparam->nxqs != lif->nxqs) {
+ err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
+ if (err)
+ goto err_out_reinit_unlock;
+ err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
+ if (err) {
+ netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
+ goto err_out_reinit_unlock;
+ }
+ }
+
+ /* swap new desc_info and rings, keeping existing interrupt config */
+ if (tx_qcqs) {
+ lif->ntxq_descs = qparam->ntxq_descs;
+ for (i = 0; i < qparam->nxqs; i++)
+ ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
+ }
+
+ if (rx_qcqs) {
+ lif->nrxq_descs = qparam->nrxq_descs;
+ for (i = 0; i < qparam->nxqs; i++)
+ ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
+ }
+
+ /* if we need to change the interrupt layout, this is the time */
+ if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
+ qparam->nxqs != lif->nxqs) {
+ if (qparam->intr_split) {
+ set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
+ } else {
+ clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
+ lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
+ lif->tx_coalesce_hw = lif->rx_coalesce_hw;
+ }
+
+ /* clear existing interrupt assignments */
+ for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
+ ionic_qcq_intr_free(lif, lif->txqcqs[i]);
+ ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
+ }
+
+ /* re-assign the interrupts */
+ for (i = 0; i < qparam->nxqs; i++) {
+ lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
+ err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
+ ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
+ lif->rxqcqs[i]->intr.index,
+ lif->rx_coalesce_hw);
+
+ if (qparam->intr_split) {
+ lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
+ err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
+ ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
+ lif->txqcqs[i]->intr.index,
+ lif->tx_coalesce_hw);
+ if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
+ lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
+ } else {
+ lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
+ ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
+ }
+ }
+ }
+
+ /* now we can rework the debugfs mappings */
+ if (tx_qcqs) {
+ for (i = 0; i < qparam->nxqs; i++) {
+ ionic_debugfs_del_qcq(lif->txqcqs[i]);
+ ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
+ }
+ }
+
+ if (rx_qcqs) {
+ for (i = 0; i < qparam->nxqs; i++) {
+ ionic_debugfs_del_qcq(lif->rxqcqs[i]);
+ ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
+ }
+ }
+
+ swap(lif->nxqs, qparam->nxqs);
+ swap(lif->rxq_features, qparam->rxq_features);
+
+err_out_reinit_unlock:
+ /* re-init the queues, but don't lose an error code */
+ if (err)
+ ionic_start_queues_reconfig(lif);
+ else
+ err = ionic_start_queues_reconfig(lif);
+
+err_out:
+ /* free old allocs without cleaning intr */
+ for (i = 0; i < qparam->nxqs; i++) {
+ if (tx_qcqs && tx_qcqs[i]) {
+ tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
+ ionic_qcq_free(lif, tx_qcqs[i]);
+ devm_kfree(lif->ionic->dev, tx_qcqs[i]);
+ tx_qcqs[i] = NULL;
+ }
+ if (rx_qcqs && rx_qcqs[i]) {
+ rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
+ ionic_qcq_free(lif, rx_qcqs[i]);
+ devm_kfree(lif->ionic->dev, rx_qcqs[i]);
+ rx_qcqs[i] = NULL;
+ }
+ }
+
+ /* free q array */
+ if (rx_qcqs) {
+ devm_kfree(lif->ionic->dev, rx_qcqs);
+ rx_qcqs = NULL;
+ }
+ if (tx_qcqs) {
+ devm_kfree(lif->ionic->dev, tx_qcqs);
+ tx_qcqs = NULL;
+ }
+
+ /* clean the unused dma and info allocations when new set is smaller
+ * than the full array, but leave the qcq shells in place
+ */
+ for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
+ if (lif->txqcqs && lif->txqcqs[i]) {
+ lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
+ ionic_qcq_free(lif, lif->txqcqs[i]);
+ }
+
+ if (lif->rxqcqs && lif->rxqcqs[i]) {
+ lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
+ ionic_qcq_free(lif, lif->rxqcqs[i]);
+ }
+ }
+
+ if (err)
+ netdev_info(lif->netdev, "%s: failed %d\n", __func__, err);
+
+ return err;
+}
+
+int ionic_lif_alloc(struct ionic *ionic)
+{
+ struct device *dev = ionic->dev;
+ union ionic_lif_identity *lid;
+ struct net_device *netdev;
+ struct ionic_lif *lif;
+ int tbl_sz;
+ int err;
+
+ lid = kzalloc(sizeof(*lid), GFP_KERNEL);
+ if (!lid)
+ return -ENOMEM;
+
+ netdev = alloc_etherdev_mqs(sizeof(*lif),
+ ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
+ if (!netdev) {
+ dev_err(dev, "Cannot allocate netdev, aborting\n");
+ err = -ENOMEM;
+ goto err_out_free_lid;
+ }
+
+ SET_NETDEV_DEV(netdev, dev);
+
+ lif = netdev_priv(netdev);
+ lif->netdev = netdev;
+ ionic->lif = lif;
+ netdev->netdev_ops = &ionic_netdev_ops;
+ ionic_ethtool_set_ops(netdev);
+
+ netdev->watchdog_timeo = 2 * HZ;
+ netif_carrier_off(netdev);
+
+ lif->identity = lid;
+ lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
+ err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
+ if (err) {
+ dev_err(ionic->dev, "Cannot identify type %d: %d\n",
+ lif->lif_type, err);
+ goto err_out_free_netdev;
+ }
+ lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
+ le32_to_cpu(lif->identity->eth.min_frame_size));
+ lif->netdev->max_mtu =
+ le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
+
+ lif->neqs = ionic->neqs_per_lif;
+ lif->nxqs = ionic->ntxqs_per_lif;
+
+ lif->ionic = ionic;
+ lif->index = 0;
+
+ if (is_kdump_kernel()) {
+ lif->ntxq_descs = IONIC_MIN_TXRX_DESC;
+ lif->nrxq_descs = IONIC_MIN_TXRX_DESC;
+ } else {
+ lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
+ lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
+ }
+
+ /* Convert the default coalesce value to actual hw resolution */
+ lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
+ lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
+ lif->rx_coalesce_usecs);
+ lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
+ lif->tx_coalesce_hw = lif->rx_coalesce_hw;
+ set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
+ set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
+
+ snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
+
+ mutex_init(&lif->queue_lock);
+ mutex_init(&lif->config_lock);
+
+ spin_lock_init(&lif->adminq_lock);
+
+ spin_lock_init(&lif->deferred.lock);
+ INIT_LIST_HEAD(&lif->deferred.list);
+ INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
+
+ /* allocate lif info */
+ lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
+ lif->info = dma_alloc_coherent(dev, lif->info_sz,
+ &lif->info_pa, GFP_KERNEL);
+ if (!lif->info) {
+ dev_err(dev, "Failed to allocate lif info, aborting\n");
+ err = -ENOMEM;
+ goto err_out_free_mutex;
+ }
+
+ ionic_debugfs_add_lif(lif);
+
+ /* allocate control queues and txrx queue arrays */
+ ionic_lif_queue_identify(lif);
+ err = ionic_qcqs_alloc(lif);
+ if (err)
+ goto err_out_free_lif_info;
+
+ /* allocate rss indirection table */
+ tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
+ lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
+ lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
+ &lif->rss_ind_tbl_pa,
+ GFP_KERNEL);
+
+ if (!lif->rss_ind_tbl) {
+ err = -ENOMEM;
+ dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
+ goto err_out_free_qcqs;
+ }
+ netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
+
+ ionic_lif_alloc_phc(lif);
+
+ return 0;
+
+err_out_free_qcqs:
+ ionic_qcqs_free(lif);
+err_out_free_lif_info:
+ dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
+ lif->info = NULL;
+ lif->info_pa = 0;
+err_out_free_mutex:
+ mutex_destroy(&lif->config_lock);
+ mutex_destroy(&lif->queue_lock);
+err_out_free_netdev:
+ free_netdev(lif->netdev);
+ lif = NULL;
+err_out_free_lid:
+ kfree(lid);
+
+ return err;
+}
+
+static void ionic_lif_reset(struct ionic_lif *lif)
+{
+ struct ionic_dev *idev = &lif->ionic->idev;
+
+ mutex_lock(&lif->ionic->dev_cmd_lock);
+ ionic_dev_cmd_lif_reset(idev, lif->index);
+ ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&lif->ionic->dev_cmd_lock);
+}
+
+static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
+{
+ struct ionic *ionic = lif->ionic;
+
+ if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return;
+
+ dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
+
+ netif_device_detach(lif->netdev);
+
+ mutex_lock(&lif->queue_lock);
+ if (test_bit(IONIC_LIF_F_UP, lif->state)) {
+ dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
+ ionic_stop_queues(lif);
+ }
+
+ if (netif_running(lif->netdev)) {
+ ionic_txrx_deinit(lif);
+ ionic_txrx_free(lif);
+ }
+ ionic_lif_deinit(lif);
+ ionic_reset(ionic);
+ ionic_qcqs_free(lif);
+
+ mutex_unlock(&lif->queue_lock);
+
+ clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
+ dev_info(ionic->dev, "FW Down: LIFs stopped\n");
+}
+
+static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
+{
+ struct ionic *ionic = lif->ionic;
+ int err;
+
+ if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ return;
+
+ dev_info(ionic->dev, "FW Up: restarting LIFs\n");
+
+ ionic_init_devinfo(ionic);
+ err = ionic_identify(ionic);
+ if (err)
+ goto err_out;
+ err = ionic_port_identify(ionic);
+ if (err)
+ goto err_out;
+ err = ionic_port_init(ionic);
+ if (err)
+ goto err_out;
+
+ mutex_lock(&lif->queue_lock);
+
+ if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
+ dev_info(ionic->dev, "FW Up: clearing broken state\n");
+
+ err = ionic_qcqs_alloc(lif);
+ if (err)
+ goto err_unlock;
+
+ err = ionic_lif_init(lif);
+ if (err)
+ goto err_qcqs_free;
+
+ if (lif->registered)
+ ionic_lif_set_netdev_info(lif);
+
+ ionic_rx_filter_replay(lif);
+
+ if (netif_running(lif->netdev)) {
+ err = ionic_txrx_alloc(lif);
+ if (err)
+ goto err_lifs_deinit;
+
+ err = ionic_txrx_init(lif);
+ if (err)
+ goto err_txrx_free;
+ }
+
+ mutex_unlock(&lif->queue_lock);
+
+ clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
+ ionic_link_status_check_request(lif, CAN_SLEEP);
+ netif_device_attach(lif->netdev);
+ dev_info(ionic->dev, "FW Up: LIFs restarted\n");
+
+ /* restore the hardware timestamping queues */
+ ionic_lif_hwstamp_replay(lif);
+
+ return;
+
+err_txrx_free:
+ ionic_txrx_free(lif);
+err_lifs_deinit:
+ ionic_lif_deinit(lif);
+err_qcqs_free:
+ ionic_qcqs_free(lif);
+err_unlock:
+ mutex_unlock(&lif->queue_lock);
+err_out:
+ dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
+}
+
+void ionic_lif_free(struct ionic_lif *lif)
+{
+ struct device *dev = lif->ionic->dev;
+
+ ionic_lif_free_phc(lif);
+
+ /* free rss indirection table */
+ dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
+ lif->rss_ind_tbl_pa);
+ lif->rss_ind_tbl = NULL;
+ lif->rss_ind_tbl_pa = 0;
+
+ /* free queues */
+ ionic_qcqs_free(lif);
+ if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ ionic_lif_reset(lif);
+
+ /* free lif info */
+ kfree(lif->identity);
+ dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
+ lif->info = NULL;
+ lif->info_pa = 0;
+
+ /* unmap doorbell page */
+ ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
+ lif->kern_dbpage = NULL;
+
+ mutex_destroy(&lif->config_lock);
+ mutex_destroy(&lif->queue_lock);
+
+ /* free netdev & lif */
+ ionic_debugfs_del_lif(lif);
+ free_netdev(lif->netdev);
+}
+
+void ionic_lif_deinit(struct ionic_lif *lif)
+{
+ if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
+ return;
+
+ if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
+ cancel_work_sync(&lif->deferred.work);
+ cancel_work_sync(&lif->tx_timeout_work);
+ ionic_rx_filters_deinit(lif);
+ if (lif->netdev->features & NETIF_F_RXHASH)
+ ionic_lif_rss_deinit(lif);
+ }
+
+ napi_disable(&lif->adminqcq->napi);
+ ionic_lif_qcq_deinit(lif, lif->notifyqcq);
+ ionic_lif_qcq_deinit(lif, lif->adminqcq);
+
+ ionic_lif_reset(lif);
+}
+
+static int ionic_lif_adminq_init(struct ionic_lif *lif)
+{
+ struct device *dev = lif->ionic->dev;
+ struct ionic_q_init_comp comp;
+ struct ionic_dev *idev;
+ struct ionic_qcq *qcq;
+ struct ionic_queue *q;
+ int err;
+
+ idev = &lif->ionic->idev;
+ qcq = lif->adminqcq;
+ q = &qcq->q;
+
+ mutex_lock(&lif->ionic->dev_cmd_lock);
+ ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
+ err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
+ ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
+ mutex_unlock(&lif->ionic->dev_cmd_lock);
+ if (err) {
+ netdev_err(lif->netdev, "adminq init failed %d\n", err);
+ return err;
+ }
+
+ q->hw_type = comp.hw_type;
+ q->hw_index = le32_to_cpu(comp.hw_index);
+ q->dbval = IONIC_DBELL_QID(q->hw_index);
+
+ dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
+ dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
+
+ q->dbell_deadline = IONIC_ADMIN_DOORBELL_DEADLINE;
+ q->dbell_jiffies = jiffies;
+
+ netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);
+
+ qcq->napi_qcq = qcq;
+ timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
+
+ napi_enable(&qcq->napi);
+
+ if (qcq->flags & IONIC_QCQ_F_INTR)
+ ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
+ IONIC_INTR_MASK_CLEAR);
+
+ qcq->flags |= IONIC_QCQ_F_INITED;
+
+ return 0;
+}
+
+static int ionic_lif_notifyq_init(struct ionic_lif *lif)
+{
+ struct ionic_qcq *qcq = lif->notifyqcq;
+ struct device *dev = lif->ionic->dev;
+ struct ionic_queue *q = &qcq->q;
+ int err;
+
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.q_init = {
+ .opcode = IONIC_CMD_Q_INIT,
+ .lif_index = cpu_to_le16(lif->index),
+ .type = q->type,
+ .ver = lif->qtype_info[q->type].version,
+ .index = cpu_to_le32(q->index),
+ .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
+ IONIC_QINIT_F_ENA),
+ .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
+ .pid = cpu_to_le16(q->pid),
+ .ring_size = ilog2(q->num_descs),
+ .ring_base = cpu_to_le64(q->base_pa),
+ }
+ };
+
+ dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
+ dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
+ dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
+ dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
+
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err)
+ return err;
+
+ lif->last_eid = 0;
+ q->hw_type = ctx.comp.q_init.hw_type;
+ q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
+ q->dbval = IONIC_DBELL_QID(q->hw_index);
+
+ dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
+ dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
+
+ /* preset the callback info */
+ q->info[0].cb_arg = lif;
+
+ qcq->flags |= IONIC_QCQ_F_INITED;
+
+ return 0;
+}
+
+static int ionic_station_set(struct ionic_lif *lif)
+{
+ struct net_device *netdev = lif->netdev;
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.lif_getattr = {
+ .opcode = IONIC_CMD_LIF_GETATTR,
+ .index = cpu_to_le16(lif->index),
+ .attr = IONIC_LIF_ATTR_MAC,
+ },
+ };
+ u8 mac_address[ETH_ALEN];
+ struct sockaddr addr;
+ int err;
+
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err)
+ return err;
+ netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
+ ctx.comp.lif_getattr.mac);
+ ether_addr_copy(mac_address, ctx.comp.lif_getattr.mac);
+
+ if (is_zero_ether_addr(mac_address)) {
+ eth_hw_addr_random(netdev);
+ netdev_dbg(netdev, "Random Mac generated: %pM\n", netdev->dev_addr);
+ ether_addr_copy(mac_address, netdev->dev_addr);
+
+ err = ionic_program_mac(lif, mac_address);
+ if (err < 0)
+ return err;
+
+ if (err > 0) {
+ netdev_dbg(netdev, "%s:SET/GET ATTR Mac are not same-due to old FW running\n",
+ __func__);
+ return 0;
+ }
+ }
+
+ if (!is_zero_ether_addr(netdev->dev_addr)) {
+ /* If the netdev mac is non-zero and doesn't match the default
+ * device address, it was set by something earlier and we're
+ * likely here again after a fw-upgrade reset. We need to be
+ * sure the netdev mac is in our filter list.
+ */
+ if (!ether_addr_equal(mac_address, netdev->dev_addr))
+ ionic_lif_addr_add(lif, netdev->dev_addr);
+ } else {
+ /* Update the netdev mac with the device's mac */
+ ether_addr_copy(addr.sa_data, mac_address);
+ addr.sa_family = AF_INET;
+ err = eth_prepare_mac_addr_change(netdev, &addr);
+ if (err) {
+ netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
+ addr.sa_data, err);
+ return 0;
+ }
+
+ eth_commit_mac_addr_change(netdev, &addr);
+ }
+
+ netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
+ netdev->dev_addr);
+ ionic_lif_addr_add(lif, netdev->dev_addr);
+
+ return 0;
+}
+
+int ionic_lif_init(struct ionic_lif *lif)
+{
+ struct ionic_dev *idev = &lif->ionic->idev;
+ struct device *dev = lif->ionic->dev;
+ struct ionic_lif_init_comp comp;
+ int dbpage_num;
+ int err;
+
+ mutex_lock(&lif->ionic->dev_cmd_lock);
+ ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
+ err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
+ ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
+ mutex_unlock(&lif->ionic->dev_cmd_lock);
+ if (err)
+ return err;
+
+ lif->hw_index = le16_to_cpu(comp.hw_index);
+
+ /* now that we have the hw_index we can figure out our doorbell page */
+ lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
+ if (!lif->dbid_count) {
+ dev_err(dev, "No doorbell pages, aborting\n");
+ return -EINVAL;
+ }
+
+ lif->kern_pid = 0;
+ dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
+ lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
+ if (!lif->kern_dbpage) {
+ dev_err(dev, "Cannot map dbpage, aborting\n");
+ return -ENOMEM;
+ }
+
+ err = ionic_lif_adminq_init(lif);
+ if (err)
+ goto err_out_adminq_deinit;
+
+ if (lif->ionic->nnqs_per_lif) {
+ err = ionic_lif_notifyq_init(lif);
+ if (err)
+ goto err_out_notifyq_deinit;
+ }
+
+ err = ionic_init_nic_features(lif);
+ if (err)
+ goto err_out_notifyq_deinit;
+
+ if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
+ err = ionic_rx_filters_init(lif);
+ if (err)
+ goto err_out_notifyq_deinit;
+ }
+
+ err = ionic_station_set(lif);
+ if (err)
+ goto err_out_notifyq_deinit;
+
+ lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
+
+ set_bit(IONIC_LIF_F_INITED, lif->state);
+
+ INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
+
+ return 0;
+
+err_out_notifyq_deinit:
+ napi_disable(&lif->adminqcq->napi);
+ ionic_lif_qcq_deinit(lif, lif->notifyqcq);
+err_out_adminq_deinit:
+ ionic_lif_qcq_deinit(lif, lif->adminqcq);
+ ionic_lif_reset(lif);
+ ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
+ lif->kern_dbpage = NULL;
+
+ return err;
+}
+
+static void ionic_lif_notify_work(struct work_struct *ws)
+{
+}
+
+static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
+{
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.lif_setattr = {
+ .opcode = IONIC_CMD_LIF_SETATTR,
+ .index = cpu_to_le16(lif->index),
+ .attr = IONIC_LIF_ATTR_NAME,
+ },
+ };
+
+ strscpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
+ sizeof(ctx.cmd.lif_setattr.name));
+
+ ionic_adminq_post_wait(lif, &ctx);
+}
+
+static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
+{
+ if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
+ return NULL;
+
+ return netdev_priv(netdev);
+}
+
+static int ionic_lif_notify(struct notifier_block *nb,
+ unsigned long event, void *info)
+{
+ struct net_device *ndev = netdev_notifier_info_to_dev(info);
+ struct ionic *ionic = container_of(nb, struct ionic, nb);
+ struct ionic_lif *lif = ionic_netdev_lif(ndev);
+
+ if (!lif || lif->ionic != ionic)
+ return NOTIFY_DONE;
+
+ switch (event) {
+ case NETDEV_CHANGENAME:
+ ionic_lif_set_netdev_info(lif);
+ break;
+ }
+
+ return NOTIFY_DONE;
+}
+
+int ionic_lif_register(struct ionic_lif *lif)
+{
+ int err;
+
+ ionic_lif_register_phc(lif);
+
+ INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
+
+ lif->ionic->nb.notifier_call = ionic_lif_notify;
+
+ err = register_netdevice_notifier(&lif->ionic->nb);
+ if (err)
+ lif->ionic->nb.notifier_call = NULL;
+
+ /* only register LIF0 for now */
+ err = register_netdev(lif->netdev);
+ if (err) {
+ dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
+ ionic_lif_unregister_phc(lif);
+ return err;
+ }
+
+ ionic_link_status_check_request(lif, CAN_SLEEP);
+ lif->registered = true;
+ ionic_lif_set_netdev_info(lif);
+
+ return 0;
+}
+
+void ionic_lif_unregister(struct ionic_lif *lif)
+{
+ if (lif->ionic->nb.notifier_call) {
+ unregister_netdevice_notifier(&lif->ionic->nb);
+ cancel_work_sync(&lif->ionic->nb_work);
+ lif->ionic->nb.notifier_call = NULL;
+ }
+
+ if (lif->netdev->reg_state == NETREG_REGISTERED)
+ unregister_netdev(lif->netdev);
+
+ ionic_lif_unregister_phc(lif);
+
+ lif->registered = false;
+}
+
+static void ionic_lif_queue_identify(struct ionic_lif *lif)
+{
+ union ionic_q_identity __iomem *q_ident;
+ struct ionic *ionic = lif->ionic;
+ struct ionic_dev *idev;
+ int qtype;
+ int err;
+
+ idev = &lif->ionic->idev;
+ q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
+
+ for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
+ struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
+
+ /* filter out the ones we know about */
+ switch (qtype) {
+ case IONIC_QTYPE_ADMINQ:
+ case IONIC_QTYPE_NOTIFYQ:
+ case IONIC_QTYPE_RXQ:
+ case IONIC_QTYPE_TXQ:
+ break;
+ default:
+ continue;
+ }
+
+ memset(qti, 0, sizeof(*qti));
+
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
+ ionic_qtype_versions[qtype]);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ if (!err) {
+ qti->version = readb(&q_ident->version);
+ qti->supported = readb(&q_ident->supported);
+ qti->features = readq(&q_ident->features);
+ qti->desc_sz = readw(&q_ident->desc_sz);
+ qti->comp_sz = readw(&q_ident->comp_sz);
+ qti->sg_desc_sz = readw(&q_ident->sg_desc_sz);
+ qti->max_sg_elems = readw(&q_ident->max_sg_elems);
+ qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
+ }
+ mutex_unlock(&ionic->dev_cmd_lock);
+
+ if (err == -EINVAL) {
+ dev_err(ionic->dev, "qtype %d not supported\n", qtype);
+ continue;
+ } else if (err == -EIO) {
+ dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
+ return;
+ } else if (err) {
+ dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
+ qtype, err);
+ return;
+ }
+
+ dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
+ qtype, qti->version);
+ dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
+ qtype, qti->supported);
+ dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
+ qtype, qti->features);
+ dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
+ qtype, qti->desc_sz);
+ dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
+ qtype, qti->comp_sz);
+ dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
+ qtype, qti->sg_desc_sz);
+ dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
+ qtype, qti->max_sg_elems);
+ dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
+ qtype, qti->sg_desc_stride);
+ }
+}
+
+int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
+ union ionic_lif_identity *lid)
+{
+ struct ionic_dev *idev = &ionic->idev;
+ size_t sz;
+ int err;
+
+ sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
+
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
+ mutex_unlock(&ionic->dev_cmd_lock);
+ if (err)
+ return (err);
+
+ dev_dbg(ionic->dev, "capabilities 0x%llx\n",
+ le64_to_cpu(lid->capabilities));
+
+ dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
+ le32_to_cpu(lid->eth.max_ucast_filters));
+ dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
+ le32_to_cpu(lid->eth.max_mcast_filters));
+ dev_dbg(ionic->dev, "eth.features 0x%llx\n",
+ le64_to_cpu(lid->eth.config.features));
+ dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
+ le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
+ dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
+ le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
+ dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
+ le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
+ dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
+ le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
+ dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
+ dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
+ dev_dbg(ionic->dev, "eth.config.mtu %d\n",
+ le32_to_cpu(lid->eth.config.mtu));
+
+ return 0;
+}
+
+int ionic_lif_size(struct ionic *ionic)
+{
+ struct ionic_identity *ident = &ionic->ident;
+ unsigned int nintrs, dev_nintrs;
+ union ionic_lif_config *lc;
+ unsigned int ntxqs_per_lif;
+ unsigned int nrxqs_per_lif;
+ unsigned int neqs_per_lif;
+ unsigned int nnqs_per_lif;
+ unsigned int nxqs, neqs;
+ unsigned int min_intrs;
+ int err;
+
+ /* retrieve basic values from FW */
+ lc = &ident->lif.eth.config;
+ dev_nintrs = le32_to_cpu(ident->dev.nintrs);
+ neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
+ nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
+ ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
+ nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
+
+ /* limit values to play nice with kdump */
+ if (is_kdump_kernel()) {
+ dev_nintrs = 2;
+ neqs_per_lif = 0;
+ nnqs_per_lif = 0;
+ ntxqs_per_lif = 1;
+ nrxqs_per_lif = 1;
+ }
+
+ /* reserve last queue id for hardware timestamping */
+ if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
+ if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
+ lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
+ } else {
+ ntxqs_per_lif -= 1;
+ nrxqs_per_lif -= 1;
+ }
+ }
+
+ nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
+ nxqs = min(nxqs, num_online_cpus());
+ neqs = min(neqs_per_lif, num_online_cpus());
+
+try_again:
+ /* interrupt usage:
+ * 1 for master lif adminq/notifyq
+ * 1 for each CPU for master lif TxRx queue pairs
+ * whatever's left is for RDMA queues
+ */
+ nintrs = 1 + nxqs + neqs;
+ min_intrs = 2; /* adminq + 1 TxRx queue pair */
+
+ if (nintrs > dev_nintrs)
+ goto try_fewer;
+
+ err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
+ if (err < 0 && err != -ENOSPC) {
+ dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
+ return err;
+ }
+ if (err == -ENOSPC)
+ goto try_fewer;
+
+ if (err != nintrs) {
+ ionic_bus_free_irq_vectors(ionic);
+ goto try_fewer;
+ }
+
+ ionic->nnqs_per_lif = nnqs_per_lif;
+ ionic->neqs_per_lif = neqs;
+ ionic->ntxqs_per_lif = nxqs;
+ ionic->nrxqs_per_lif = nxqs;
+ ionic->nintrs = nintrs;
+
+ ionic_debugfs_add_sizes(ionic);
+
+ return 0;
+
+try_fewer:
+ if (nnqs_per_lif > 1) {
+ nnqs_per_lif >>= 1;
+ goto try_again;
+ }
+ if (neqs > 1) {
+ neqs >>= 1;
+ goto try_again;
+ }
+ if (nxqs > 1) {
+ nxqs >>= 1;
+ goto try_again;
+ }
+ dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
+ return -ENOSPC;
+}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.h b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
new file mode 100644
index 000000000..734519895
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
@@ -0,0 +1,343 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#ifndef _IONIC_LIF_H_
+#define _IONIC_LIF_H_
+
+#include <linux/ptp_clock_kernel.h>
+#include <linux/timecounter.h>
+#include <uapi/linux/net_tstamp.h>
+#include <linux/dim.h>
+#include <linux/pci.h>
+#include "ionic_rx_filter.h"
+
+#define IONIC_ADMINQ_LENGTH 16 /* must be a power of two */
+#define IONIC_NOTIFYQ_LENGTH 64 /* must be a power of two */
+
+#define ADD_ADDR true
+#define DEL_ADDR false
+#define CAN_SLEEP true
+#define CAN_NOT_SLEEP false
+
+#define IONIC_RX_COPYBREAK_DEFAULT 256
+#define IONIC_TX_BUDGET_DEFAULT 256
+
+struct ionic_tx_stats {
+ u64 pkts;
+ u64 bytes;
+ u64 csum_none;
+ u64 csum;
+ u64 tso;
+ u64 tso_bytes;
+ u64 frags;
+ u64 vlan_inserted;
+ u64 clean;
+ u64 linearize;
+ u64 crc32_csum;
+ u64 dma_map_err;
+ u64 hwstamp_valid;
+ u64 hwstamp_invalid;
+};
+
+struct ionic_rx_stats {
+ u64 pkts;
+ u64 bytes;
+ u64 csum_none;
+ u64 csum_complete;
+ u64 dropped;
+ u64 vlan_stripped;
+ u64 csum_error;
+ u64 dma_map_err;
+ u64 alloc_err;
+ u64 hwstamp_valid;
+ u64 hwstamp_invalid;
+};
+
+#define IONIC_QCQ_F_INITED BIT(0)
+#define IONIC_QCQ_F_SG BIT(1)
+#define IONIC_QCQ_F_INTR BIT(2)
+#define IONIC_QCQ_F_TX_STATS BIT(3)
+#define IONIC_QCQ_F_RX_STATS BIT(4)
+#define IONIC_QCQ_F_NOTIFYQ BIT(5)
+
+struct ionic_qcq {
+ void *q_base;
+ dma_addr_t q_base_pa;
+ u32 q_size;
+ void *cq_base;
+ dma_addr_t cq_base_pa;
+ u32 cq_size;
+ void *sg_base;
+ dma_addr_t sg_base_pa;
+ u32 sg_size;
+ struct dim dim;
+ struct ionic_queue q;
+ struct ionic_cq cq;
+ struct ionic_intr_info intr;
+ struct timer_list napi_deadline;
+ struct napi_struct napi;
+ unsigned int flags;
+ struct ionic_qcq *napi_qcq;
+ struct dentry *dentry;
+};
+
+#define q_to_qcq(q) container_of(q, struct ionic_qcq, q)
+#define q_to_tx_stats(q) (&(q)->lif->txqstats[(q)->index])
+#define q_to_rx_stats(q) (&(q)->lif->rxqstats[(q)->index])
+#define napi_to_qcq(napi) container_of(napi, struct ionic_qcq, napi)
+#define napi_to_cq(napi) (&napi_to_qcq(napi)->cq)
+
+enum ionic_deferred_work_type {
+ IONIC_DW_TYPE_RX_MODE,
+ IONIC_DW_TYPE_LINK_STATUS,
+ IONIC_DW_TYPE_LIF_RESET,
+};
+
+struct ionic_deferred_work {
+ struct list_head list;
+ enum ionic_deferred_work_type type;
+ union {
+ u8 addr[ETH_ALEN];
+ u8 fw_status;
+ };
+};
+
+struct ionic_deferred {
+ spinlock_t lock; /* lock for deferred work list */
+ struct list_head list;
+ struct work_struct work;
+};
+
+struct ionic_lif_sw_stats {
+ u64 tx_packets;
+ u64 tx_bytes;
+ u64 rx_packets;
+ u64 rx_bytes;
+ u64 tx_tso;
+ u64 tx_tso_bytes;
+ u64 tx_csum_none;
+ u64 tx_csum;
+ u64 rx_csum_none;
+ u64 rx_csum_complete;
+ u64 rx_csum_error;
+ u64 tx_hwstamp_valid;
+ u64 tx_hwstamp_invalid;
+ u64 rx_hwstamp_valid;
+ u64 rx_hwstamp_invalid;
+ u64 hw_tx_dropped;
+ u64 hw_rx_dropped;
+ u64 hw_rx_over_errors;
+ u64 hw_rx_missed_errors;
+ u64 hw_tx_aborted_errors;
+};
+
+enum ionic_lif_state_flags {
+ IONIC_LIF_F_INITED,
+ IONIC_LIF_F_UP,
+ IONIC_LIF_F_LINK_CHECK_REQUESTED,
+ IONIC_LIF_F_FILTER_SYNC_NEEDED,
+ IONIC_LIF_F_FW_RESET,
+ IONIC_LIF_F_FW_STOPPING,
+ IONIC_LIF_F_SPLIT_INTR,
+ IONIC_LIF_F_BROKEN,
+ IONIC_LIF_F_TX_DIM_INTR,
+ IONIC_LIF_F_RX_DIM_INTR,
+
+ /* leave this as last */
+ IONIC_LIF_F_STATE_SIZE
+};
+
+struct ionic_qtype_info {
+ u8 version;
+ u8 supported;
+ u64 features;
+ u16 desc_sz;
+ u16 comp_sz;
+ u16 sg_desc_sz;
+ u16 max_sg_elems;
+ u16 sg_desc_stride;
+};
+
+struct ionic_phc;
+
+#define IONIC_LIF_NAME_MAX_SZ 32
+struct ionic_lif {
+ struct net_device *netdev;
+ DECLARE_BITMAP(state, IONIC_LIF_F_STATE_SIZE);
+ struct ionic *ionic;
+ unsigned int index;
+ unsigned int hw_index;
+ struct mutex queue_lock; /* lock for queue structures */
+ struct mutex config_lock; /* lock for config actions */
+ spinlock_t adminq_lock; /* lock for AdminQ operations */
+ struct ionic_qcq *adminqcq;
+ struct ionic_qcq *notifyqcq;
+ struct ionic_qcq **txqcqs;
+ struct ionic_qcq *hwstamp_txq;
+ struct ionic_tx_stats *txqstats;
+ struct ionic_qcq **rxqcqs;
+ struct ionic_qcq *hwstamp_rxq;
+ struct ionic_rx_stats *rxqstats;
+ struct ionic_deferred deferred;
+ struct work_struct tx_timeout_work;
+ u64 last_eid;
+ unsigned int kern_pid;
+ u64 __iomem *kern_dbpage;
+ unsigned int neqs;
+ unsigned int nxqs;
+ unsigned int ntxq_descs;
+ unsigned int nrxq_descs;
+ u32 rx_copybreak;
+ u64 rxq_features;
+ u16 rx_mode;
+ u64 hw_features;
+ bool registered;
+ u16 lif_type;
+ unsigned int nmcast;
+ unsigned int nucast;
+ unsigned int nvlans;
+ unsigned int max_vlans;
+ char name[IONIC_LIF_NAME_MAX_SZ];
+
+ union ionic_lif_identity *identity;
+ struct ionic_lif_info *info;
+ dma_addr_t info_pa;
+ u32 info_sz;
+ struct ionic_qtype_info qtype_info[IONIC_QTYPE_MAX];
+
+ u16 rss_types;
+ u8 rss_hash_key[IONIC_RSS_HASH_KEY_SIZE];
+ u8 *rss_ind_tbl;
+ dma_addr_t rss_ind_tbl_pa;
+ u32 rss_ind_tbl_sz;
+
+ struct ionic_rx_filters rx_filters;
+ u32 rx_coalesce_usecs; /* what the user asked for */
+ u32 rx_coalesce_hw; /* what the hw is using */
+ u32 tx_coalesce_usecs; /* what the user asked for */
+ u32 tx_coalesce_hw; /* what the hw is using */
+ unsigned int dbid_count;
+
+ struct ionic_phc *phc;
+
+ struct dentry *dentry;
+};
+
+struct ionic_phc {
+ spinlock_t lock; /* lock for cc and tc */
+ struct cyclecounter cc;
+ struct timecounter tc;
+
+ struct mutex config_lock; /* lock for ts_config */
+ struct hwtstamp_config ts_config;
+ u64 ts_config_rx_filt;
+ u32 ts_config_tx_mode;
+
+ u32 init_cc_mult;
+ long aux_work_delay;
+
+ struct ptp_clock_info ptp_info;
+ struct ptp_clock *ptp;
+ struct ionic_lif *lif;
+};
+
+struct ionic_queue_params {
+ unsigned int nxqs;
+ unsigned int ntxq_descs;
+ unsigned int nrxq_descs;
+ unsigned int intr_split;
+ u64 rxq_features;
+};
+
+static inline void ionic_init_queue_params(struct ionic_lif *lif,
+ struct ionic_queue_params *qparam)
+{
+ qparam->nxqs = lif->nxqs;
+ qparam->ntxq_descs = lif->ntxq_descs;
+ qparam->nrxq_descs = lif->nrxq_descs;
+ qparam->intr_split = test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
+ qparam->rxq_features = lif->rxq_features;
+}
+
+static inline u32 ionic_coal_usec_to_hw(struct ionic *ionic, u32 usecs)
+{
+ u32 mult = le32_to_cpu(ionic->ident.dev.intr_coal_mult);
+ u32 div = le32_to_cpu(ionic->ident.dev.intr_coal_div);
+
+ /* Div-by-zero should never be an issue, but check anyway */
+ if (!div || !mult)
+ return 0;
+
+ /* Round up in case usecs is close to the next hw unit */
+ usecs += (div / mult) >> 1;
+
+ /* Convert from usecs to device units */
+ return (usecs * mult) / div;
+}
+
+void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep);
+void ionic_get_stats64(struct net_device *netdev,
+ struct rtnl_link_stats64 *ns);
+void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
+ struct ionic_deferred_work *work);
+int ionic_lif_alloc(struct ionic *ionic);
+int ionic_lif_init(struct ionic_lif *lif);
+void ionic_lif_free(struct ionic_lif *lif);
+void ionic_lif_deinit(struct ionic_lif *lif);
+
+int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr);
+int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr);
+
+int ionic_lif_register(struct ionic_lif *lif);
+void ionic_lif_unregister(struct ionic_lif *lif);
+int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
+ union ionic_lif_identity *lif_ident);
+int ionic_lif_size(struct ionic *ionic);
+
+#if IS_ENABLED(CONFIG_PTP_1588_CLOCK)
+void ionic_lif_hwstamp_replay(struct ionic_lif *lif);
+void ionic_lif_hwstamp_recreate_queues(struct ionic_lif *lif);
+int ionic_lif_hwstamp_set(struct ionic_lif *lif, struct ifreq *ifr);
+int ionic_lif_hwstamp_get(struct ionic_lif *lif, struct ifreq *ifr);
+ktime_t ionic_lif_phc_ktime(struct ionic_lif *lif, u64 counter);
+void ionic_lif_register_phc(struct ionic_lif *lif);
+void ionic_lif_unregister_phc(struct ionic_lif *lif);
+void ionic_lif_alloc_phc(struct ionic_lif *lif);
+void ionic_lif_free_phc(struct ionic_lif *lif);
+#else
+static inline void ionic_lif_hwstamp_replay(struct ionic_lif *lif) {}
+static inline void ionic_lif_hwstamp_recreate_queues(struct ionic_lif *lif) {}
+
+static inline int ionic_lif_hwstamp_set(struct ionic_lif *lif, struct ifreq *ifr)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline int ionic_lif_hwstamp_get(struct ionic_lif *lif, struct ifreq *ifr)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline ktime_t ionic_lif_phc_ktime(struct ionic_lif *lif, u64 counter)
+{
+ return ns_to_ktime(0);
+}
+
+static inline void ionic_lif_register_phc(struct ionic_lif *lif) {}
+static inline void ionic_lif_unregister_phc(struct ionic_lif *lif) {}
+static inline void ionic_lif_alloc_phc(struct ionic_lif *lif) {}
+static inline void ionic_lif_free_phc(struct ionic_lif *lif) {}
+#endif
+
+int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif);
+int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif);
+int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all);
+int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode);
+int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class);
+
+int ionic_lif_rss_config(struct ionic_lif *lif, u16 types,
+ const u8 *key, const u32 *indir);
+void ionic_lif_rx_mode(struct ionic_lif *lif);
+int ionic_reconfigure_queues(struct ionic_lif *lif,
+ struct ionic_queue_params *qparam);
+#endif /* _IONIC_LIF_H_ */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_main.c b/drivers/net/ethernet/pensando/ionic/ionic_main.c
new file mode 100644
index 000000000..79272f5f3
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_main.c
@@ -0,0 +1,738 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#include <linux/printk.h>
+#include <linux/dynamic_debug.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/utsname.h>
+#include <generated/utsrelease.h>
+#include <linux/ctype.h>
+
+#include "ionic.h"
+#include "ionic_bus.h"
+#include "ionic_lif.h"
+#include "ionic_debugfs.h"
+
+MODULE_DESCRIPTION(IONIC_DRV_DESCRIPTION);
+MODULE_AUTHOR("Pensando Systems, Inc");
+MODULE_LICENSE("GPL");
+
+static const char *ionic_error_to_str(enum ionic_status_code code)
+{
+ switch (code) {
+ case IONIC_RC_SUCCESS:
+ return "IONIC_RC_SUCCESS";
+ case IONIC_RC_EVERSION:
+ return "IONIC_RC_EVERSION";
+ case IONIC_RC_EOPCODE:
+ return "IONIC_RC_EOPCODE";
+ case IONIC_RC_EIO:
+ return "IONIC_RC_EIO";
+ case IONIC_RC_EPERM:
+ return "IONIC_RC_EPERM";
+ case IONIC_RC_EQID:
+ return "IONIC_RC_EQID";
+ case IONIC_RC_EQTYPE:
+ return "IONIC_RC_EQTYPE";
+ case IONIC_RC_ENOENT:
+ return "IONIC_RC_ENOENT";
+ case IONIC_RC_EINTR:
+ return "IONIC_RC_EINTR";
+ case IONIC_RC_EAGAIN:
+ return "IONIC_RC_EAGAIN";
+ case IONIC_RC_ENOMEM:
+ return "IONIC_RC_ENOMEM";
+ case IONIC_RC_EFAULT:
+ return "IONIC_RC_EFAULT";
+ case IONIC_RC_EBUSY:
+ return "IONIC_RC_EBUSY";
+ case IONIC_RC_EEXIST:
+ return "IONIC_RC_EEXIST";
+ case IONIC_RC_EINVAL:
+ return "IONIC_RC_EINVAL";
+ case IONIC_RC_ENOSPC:
+ return "IONIC_RC_ENOSPC";
+ case IONIC_RC_ERANGE:
+ return "IONIC_RC_ERANGE";
+ case IONIC_RC_BAD_ADDR:
+ return "IONIC_RC_BAD_ADDR";
+ case IONIC_RC_DEV_CMD:
+ return "IONIC_RC_DEV_CMD";
+ case IONIC_RC_ENOSUPP:
+ return "IONIC_RC_ENOSUPP";
+ case IONIC_RC_ERROR:
+ return "IONIC_RC_ERROR";
+ case IONIC_RC_ERDMA:
+ return "IONIC_RC_ERDMA";
+ case IONIC_RC_EBAD_FW:
+ return "IONIC_RC_EBAD_FW";
+ default:
+ return "IONIC_RC_UNKNOWN";
+ }
+}
+
+static int ionic_error_to_errno(enum ionic_status_code code)
+{
+ switch (code) {
+ case IONIC_RC_SUCCESS:
+ return 0;
+ case IONIC_RC_EVERSION:
+ case IONIC_RC_EQTYPE:
+ case IONIC_RC_EQID:
+ case IONIC_RC_EINVAL:
+ case IONIC_RC_ENOSUPP:
+ return -EINVAL;
+ case IONIC_RC_EPERM:
+ return -EPERM;
+ case IONIC_RC_ENOENT:
+ return -ENOENT;
+ case IONIC_RC_EAGAIN:
+ return -EAGAIN;
+ case IONIC_RC_ENOMEM:
+ return -ENOMEM;
+ case IONIC_RC_EFAULT:
+ return -EFAULT;
+ case IONIC_RC_EBUSY:
+ return -EBUSY;
+ case IONIC_RC_EEXIST:
+ return -EEXIST;
+ case IONIC_RC_ENOSPC:
+ return -ENOSPC;
+ case IONIC_RC_ERANGE:
+ return -ERANGE;
+ case IONIC_RC_BAD_ADDR:
+ return -EFAULT;
+ case IONIC_RC_EOPCODE:
+ case IONIC_RC_EINTR:
+ case IONIC_RC_DEV_CMD:
+ case IONIC_RC_ERROR:
+ case IONIC_RC_ERDMA:
+ case IONIC_RC_EIO:
+ default:
+ return -EIO;
+ }
+}
+
+static const char *ionic_opcode_to_str(enum ionic_cmd_opcode opcode)
+{
+ switch (opcode) {
+ case IONIC_CMD_NOP:
+ return "IONIC_CMD_NOP";
+ case IONIC_CMD_INIT:
+ return "IONIC_CMD_INIT";
+ case IONIC_CMD_RESET:
+ return "IONIC_CMD_RESET";
+ case IONIC_CMD_IDENTIFY:
+ return "IONIC_CMD_IDENTIFY";
+ case IONIC_CMD_GETATTR:
+ return "IONIC_CMD_GETATTR";
+ case IONIC_CMD_SETATTR:
+ return "IONIC_CMD_SETATTR";
+ case IONIC_CMD_PORT_IDENTIFY:
+ return "IONIC_CMD_PORT_IDENTIFY";
+ case IONIC_CMD_PORT_INIT:
+ return "IONIC_CMD_PORT_INIT";
+ case IONIC_CMD_PORT_RESET:
+ return "IONIC_CMD_PORT_RESET";
+ case IONIC_CMD_PORT_GETATTR:
+ return "IONIC_CMD_PORT_GETATTR";
+ case IONIC_CMD_PORT_SETATTR:
+ return "IONIC_CMD_PORT_SETATTR";
+ case IONIC_CMD_LIF_INIT:
+ return "IONIC_CMD_LIF_INIT";
+ case IONIC_CMD_LIF_RESET:
+ return "IONIC_CMD_LIF_RESET";
+ case IONIC_CMD_LIF_IDENTIFY:
+ return "IONIC_CMD_LIF_IDENTIFY";
+ case IONIC_CMD_LIF_SETATTR:
+ return "IONIC_CMD_LIF_SETATTR";
+ case IONIC_CMD_LIF_GETATTR:
+ return "IONIC_CMD_LIF_GETATTR";
+ case IONIC_CMD_LIF_SETPHC:
+ return "IONIC_CMD_LIF_SETPHC";
+ case IONIC_CMD_RX_MODE_SET:
+ return "IONIC_CMD_RX_MODE_SET";
+ case IONIC_CMD_RX_FILTER_ADD:
+ return "IONIC_CMD_RX_FILTER_ADD";
+ case IONIC_CMD_RX_FILTER_DEL:
+ return "IONIC_CMD_RX_FILTER_DEL";
+ case IONIC_CMD_Q_IDENTIFY:
+ return "IONIC_CMD_Q_IDENTIFY";
+ case IONIC_CMD_Q_INIT:
+ return "IONIC_CMD_Q_INIT";
+ case IONIC_CMD_Q_CONTROL:
+ return "IONIC_CMD_Q_CONTROL";
+ case IONIC_CMD_RDMA_RESET_LIF:
+ return "IONIC_CMD_RDMA_RESET_LIF";
+ case IONIC_CMD_RDMA_CREATE_EQ:
+ return "IONIC_CMD_RDMA_CREATE_EQ";
+ case IONIC_CMD_RDMA_CREATE_CQ:
+ return "IONIC_CMD_RDMA_CREATE_CQ";
+ case IONIC_CMD_RDMA_CREATE_ADMINQ:
+ return "IONIC_CMD_RDMA_CREATE_ADMINQ";
+ case IONIC_CMD_FW_DOWNLOAD:
+ return "IONIC_CMD_FW_DOWNLOAD";
+ case IONIC_CMD_FW_CONTROL:
+ return "IONIC_CMD_FW_CONTROL";
+ case IONIC_CMD_FW_DOWNLOAD_V1:
+ return "IONIC_CMD_FW_DOWNLOAD_V1";
+ case IONIC_CMD_FW_CONTROL_V1:
+ return "IONIC_CMD_FW_CONTROL_V1";
+ case IONIC_CMD_VF_GETATTR:
+ return "IONIC_CMD_VF_GETATTR";
+ case IONIC_CMD_VF_SETATTR:
+ return "IONIC_CMD_VF_SETATTR";
+ default:
+ return "DEVCMD_UNKNOWN";
+ }
+}
+
+const char *ionic_vf_attr_to_str(enum ionic_vf_attr attr)
+{
+ switch (attr) {
+ case IONIC_VF_ATTR_SPOOFCHK:
+ return "IONIC_VF_ATTR_SPOOFCHK";
+ case IONIC_VF_ATTR_TRUST:
+ return "IONIC_VF_ATTR_TRUST";
+ case IONIC_VF_ATTR_LINKSTATE:
+ return "IONIC_VF_ATTR_LINKSTATE";
+ case IONIC_VF_ATTR_MAC:
+ return "IONIC_VF_ATTR_MAC";
+ case IONIC_VF_ATTR_VLAN:
+ return "IONIC_VF_ATTR_VLAN";
+ case IONIC_VF_ATTR_RATE:
+ return "IONIC_VF_ATTR_RATE";
+ case IONIC_VF_ATTR_STATSADDR:
+ return "IONIC_VF_ATTR_STATSADDR";
+ default:
+ return "IONIC_VF_ATTR_UNKNOWN";
+ }
+}
+
+static void ionic_adminq_flush(struct ionic_lif *lif)
+{
+ struct ionic_desc_info *desc_info;
+ unsigned long irqflags;
+ struct ionic_queue *q;
+
+ spin_lock_irqsave(&lif->adminq_lock, irqflags);
+ if (!lif->adminqcq) {
+ spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
+ return;
+ }
+
+ q = &lif->adminqcq->q;
+
+ while (q->tail_idx != q->head_idx) {
+ desc_info = &q->info[q->tail_idx];
+ memset(desc_info->desc, 0, sizeof(union ionic_adminq_cmd));
+ desc_info->cb = NULL;
+ desc_info->cb_arg = NULL;
+ q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
+ }
+ spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
+}
+
+void ionic_adminq_netdev_err_print(struct ionic_lif *lif, u8 opcode,
+ u8 status, int err)
+{
+ const char *stat_str;
+
+ stat_str = (err == -ETIMEDOUT) ? "TIMEOUT" :
+ ionic_error_to_str(status);
+
+ netdev_err(lif->netdev, "%s (%d) failed: %s (%d)\n",
+ ionic_opcode_to_str(opcode), opcode, stat_str, err);
+}
+
+static int ionic_adminq_check_err(struct ionic_lif *lif,
+ struct ionic_admin_ctx *ctx,
+ const bool timeout,
+ const bool do_msg)
+{
+ int err = 0;
+
+ if (ctx->comp.comp.status || timeout) {
+ err = timeout ? -ETIMEDOUT :
+ ionic_error_to_errno(ctx->comp.comp.status);
+
+ if (do_msg)
+ ionic_adminq_netdev_err_print(lif, ctx->cmd.cmd.opcode,
+ ctx->comp.comp.status, err);
+
+ if (timeout)
+ ionic_adminq_flush(lif);
+ }
+
+ return err;
+}
+
+static void ionic_adminq_cb(struct ionic_queue *q,
+ struct ionic_desc_info *desc_info,
+ struct ionic_cq_info *cq_info, void *cb_arg)
+{
+ struct ionic_admin_ctx *ctx = cb_arg;
+ struct ionic_admin_comp *comp;
+
+ if (!ctx)
+ return;
+
+ comp = cq_info->cq_desc;
+
+ memcpy(&ctx->comp, comp, sizeof(*comp));
+
+ dev_dbg(q->dev, "comp admin queue command:\n");
+ dynamic_hex_dump("comp ", DUMP_PREFIX_OFFSET, 16, 1,
+ &ctx->comp, sizeof(ctx->comp), true);
+
+ complete_all(&ctx->work);
+}
+
+bool ionic_adminq_poke_doorbell(struct ionic_queue *q)
+{
+ struct ionic_lif *lif = q->lif;
+ unsigned long now, then, dif;
+ unsigned long irqflags;
+
+ spin_lock_irqsave(&lif->adminq_lock, irqflags);
+
+ if (q->tail_idx == q->head_idx) {
+ spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
+ return false;
+ }
+
+ now = READ_ONCE(jiffies);
+ then = q->dbell_jiffies;
+ dif = now - then;
+
+ if (dif > q->dbell_deadline) {
+ ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
+ q->dbval | q->head_idx);
+
+ q->dbell_jiffies = now;
+ }
+
+ spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
+
+ return true;
+}
+
+int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
+{
+ struct ionic_desc_info *desc_info;
+ unsigned long irqflags;
+ struct ionic_queue *q;
+ int err = 0;
+
+ spin_lock_irqsave(&lif->adminq_lock, irqflags);
+ if (!lif->adminqcq) {
+ spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
+ return -EIO;
+ }
+
+ q = &lif->adminqcq->q;
+
+ if (!ionic_q_has_space(q, 1)) {
+ err = -ENOSPC;
+ goto err_out;
+ }
+
+ err = ionic_heartbeat_check(lif->ionic);
+ if (err)
+ goto err_out;
+
+ desc_info = &q->info[q->head_idx];
+ memcpy(desc_info->desc, &ctx->cmd, sizeof(ctx->cmd));
+
+ dev_dbg(&lif->netdev->dev, "post admin queue command:\n");
+ dynamic_hex_dump("cmd ", DUMP_PREFIX_OFFSET, 16, 1,
+ &ctx->cmd, sizeof(ctx->cmd), true);
+
+ ionic_q_post(q, true, ionic_adminq_cb, ctx);
+
+err_out:
+ spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
+
+ return err;
+}
+
+int ionic_adminq_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx,
+ const int err, const bool do_msg)
+{
+ struct net_device *netdev = lif->netdev;
+ unsigned long time_limit;
+ unsigned long time_start;
+ unsigned long time_done;
+ unsigned long remaining;
+ const char *name;
+
+ name = ionic_opcode_to_str(ctx->cmd.cmd.opcode);
+
+ if (err) {
+ if (do_msg && !test_bit(IONIC_LIF_F_FW_RESET, lif->state))
+ netdev_err(netdev, "Posting of %s (%d) failed: %d\n",
+ name, ctx->cmd.cmd.opcode, err);
+ ctx->comp.comp.status = IONIC_RC_ERROR;
+ return err;
+ }
+
+ time_start = jiffies;
+ time_limit = time_start + HZ * (ulong)DEVCMD_TIMEOUT;
+ do {
+ remaining = wait_for_completion_timeout(&ctx->work,
+ IONIC_ADMINQ_TIME_SLICE);
+
+ /* check for done */
+ if (remaining)
+ break;
+
+ /* force a check of FW status and break out if FW reset */
+ (void)ionic_heartbeat_check(lif->ionic);
+ if ((test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
+ !lif->ionic->idev.fw_status_ready) ||
+ test_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
+ if (do_msg)
+ netdev_warn(netdev, "%s (%d) interrupted, FW in reset\n",
+ name, ctx->cmd.cmd.opcode);
+ ctx->comp.comp.status = IONIC_RC_ERROR;
+ return -ENXIO;
+ }
+
+ } while (time_before(jiffies, time_limit));
+ time_done = jiffies;
+
+ dev_dbg(lif->ionic->dev, "%s: elapsed %d msecs\n",
+ __func__, jiffies_to_msecs(time_done - time_start));
+
+ return ionic_adminq_check_err(lif, ctx,
+ time_after_eq(time_done, time_limit),
+ do_msg);
+}
+
+int ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
+{
+ int err;
+
+ err = ionic_adminq_post(lif, ctx);
+
+ return ionic_adminq_wait(lif, ctx, err, true);
+}
+
+int ionic_adminq_post_wait_nomsg(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
+{
+ int err;
+
+ err = ionic_adminq_post(lif, ctx);
+
+ return ionic_adminq_wait(lif, ctx, err, false);
+}
+
+static void ionic_dev_cmd_clean(struct ionic *ionic)
+{
+ struct ionic_dev *idev = &ionic->idev;
+
+ iowrite32(0, &idev->dev_cmd_regs->doorbell);
+ memset_io(&idev->dev_cmd_regs->cmd, 0, sizeof(idev->dev_cmd_regs->cmd));
+}
+
+void ionic_dev_cmd_dev_err_print(struct ionic *ionic, u8 opcode, u8 status,
+ int err)
+{
+ const char *stat_str;
+
+ stat_str = (err == -ETIMEDOUT) ? "TIMEOUT" :
+ ionic_error_to_str(status);
+
+ dev_err(ionic->dev, "DEV_CMD %s (%d) error, %s (%d) failed\n",
+ ionic_opcode_to_str(opcode), opcode, stat_str, err);
+}
+
+static int __ionic_dev_cmd_wait(struct ionic *ionic, unsigned long max_seconds,
+ const bool do_msg)
+{
+ struct ionic_dev *idev = &ionic->idev;
+ unsigned long start_time;
+ unsigned long max_wait;
+ unsigned long duration;
+ int done = 0;
+ bool fw_up;
+ int opcode;
+ int err;
+
+ /* Wait for dev cmd to complete, retrying if we get EAGAIN,
+ * but don't wait any longer than max_seconds.
+ */
+ max_wait = jiffies + (max_seconds * HZ);
+try_again:
+ opcode = readb(&idev->dev_cmd_regs->cmd.cmd.opcode);
+ start_time = jiffies;
+ for (fw_up = ionic_is_fw_running(idev);
+ !done && fw_up && time_before(jiffies, max_wait);
+ fw_up = ionic_is_fw_running(idev)) {
+ done = ionic_dev_cmd_done(idev);
+ if (done)
+ break;
+ usleep_range(100, 200);
+ }
+ duration = jiffies - start_time;
+
+ dev_dbg(ionic->dev, "DEVCMD %s (%d) done=%d took %ld secs (%ld jiffies)\n",
+ ionic_opcode_to_str(opcode), opcode,
+ done, duration / HZ, duration);
+
+ if (!done && !fw_up) {
+ ionic_dev_cmd_clean(ionic);
+ dev_warn(ionic->dev, "DEVCMD %s (%d) interrupted - FW is down\n",
+ ionic_opcode_to_str(opcode), opcode);
+ return -ENXIO;
+ }
+
+ if (!done && !time_before(jiffies, max_wait)) {
+ ionic_dev_cmd_clean(ionic);
+ dev_warn(ionic->dev, "DEVCMD %s (%d) timeout after %ld secs\n",
+ ionic_opcode_to_str(opcode), opcode, max_seconds);
+ return -ETIMEDOUT;
+ }
+
+ err = ionic_dev_cmd_status(&ionic->idev);
+ if (err) {
+ if (err == IONIC_RC_EAGAIN &&
+ time_before(jiffies, (max_wait - HZ))) {
+ dev_dbg(ionic->dev, "DEV_CMD %s (%d), %s (%d) retrying...\n",
+ ionic_opcode_to_str(opcode), opcode,
+ ionic_error_to_str(err), err);
+
+ iowrite32(0, &idev->dev_cmd_regs->done);
+ msleep(1000);
+ iowrite32(1, &idev->dev_cmd_regs->doorbell);
+ goto try_again;
+ }
+
+ if (!(opcode == IONIC_CMD_FW_CONTROL && err == IONIC_RC_EAGAIN))
+ if (do_msg)
+ ionic_dev_cmd_dev_err_print(ionic, opcode, err,
+ ionic_error_to_errno(err));
+
+ return ionic_error_to_errno(err);
+ }
+
+ ionic_dev_cmd_clean(ionic);
+
+ return 0;
+}
+
+int ionic_dev_cmd_wait(struct ionic *ionic, unsigned long max_seconds)
+{
+ return __ionic_dev_cmd_wait(ionic, max_seconds, true);
+}
+
+int ionic_dev_cmd_wait_nomsg(struct ionic *ionic, unsigned long max_seconds)
+{
+ return __ionic_dev_cmd_wait(ionic, max_seconds, false);
+}
+
+int ionic_setup(struct ionic *ionic)
+{
+ int err;
+
+ err = ionic_dev_setup(ionic);
+ if (err)
+ return err;
+ ionic_reset(ionic);
+
+ return 0;
+}
+
+int ionic_identify(struct ionic *ionic)
+{
+ struct ionic_identity *ident = &ionic->ident;
+ struct ionic_dev *idev = &ionic->idev;
+ size_t sz;
+ int err;
+
+ memset(ident, 0, sizeof(*ident));
+
+ ident->drv.os_type = cpu_to_le32(IONIC_OS_TYPE_LINUX);
+ strncpy(ident->drv.driver_ver_str, UTS_RELEASE,
+ sizeof(ident->drv.driver_ver_str) - 1);
+
+ mutex_lock(&ionic->dev_cmd_lock);
+
+ sz = min(sizeof(ident->drv), sizeof(idev->dev_cmd_regs->data));
+ memcpy_toio(&idev->dev_cmd_regs->data, &ident->drv, sz);
+
+ ionic_dev_cmd_identify(idev, IONIC_IDENTITY_VERSION_1);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ if (!err) {
+ sz = min(sizeof(ident->dev), sizeof(idev->dev_cmd_regs->data));
+ memcpy_fromio(&ident->dev, &idev->dev_cmd_regs->data, sz);
+ }
+ mutex_unlock(&ionic->dev_cmd_lock);
+
+ if (err) {
+ dev_err(ionic->dev, "Cannot identify ionic: %d\n", err);
+ goto err_out;
+ }
+
+ if (isprint(idev->dev_info.fw_version[0]) &&
+ isascii(idev->dev_info.fw_version[0]))
+ dev_info(ionic->dev, "FW: %.*s\n",
+ (int)(sizeof(idev->dev_info.fw_version) - 1),
+ idev->dev_info.fw_version);
+ else
+ dev_info(ionic->dev, "FW: (invalid string) 0x%02x 0x%02x 0x%02x 0x%02x ...\n",
+ (u8)idev->dev_info.fw_version[0],
+ (u8)idev->dev_info.fw_version[1],
+ (u8)idev->dev_info.fw_version[2],
+ (u8)idev->dev_info.fw_version[3]);
+
+ err = ionic_lif_identify(ionic, IONIC_LIF_TYPE_CLASSIC,
+ &ionic->ident.lif);
+ if (err) {
+ dev_err(ionic->dev, "Cannot identify LIFs: %d\n", err);
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ return err;
+}
+
+int ionic_init(struct ionic *ionic)
+{
+ struct ionic_dev *idev = &ionic->idev;
+ int err;
+
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_init(idev);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&ionic->dev_cmd_lock);
+
+ return err;
+}
+
+int ionic_reset(struct ionic *ionic)
+{
+ struct ionic_dev *idev = &ionic->idev;
+ int err;
+
+ if (!ionic_is_fw_running(idev))
+ return 0;
+
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_reset(idev);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&ionic->dev_cmd_lock);
+
+ return err;
+}
+
+int ionic_port_identify(struct ionic *ionic)
+{
+ struct ionic_identity *ident = &ionic->ident;
+ struct ionic_dev *idev = &ionic->idev;
+ size_t sz;
+ int err;
+
+ mutex_lock(&ionic->dev_cmd_lock);
+
+ ionic_dev_cmd_port_identify(idev);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ if (!err) {
+ sz = min(sizeof(ident->port), sizeof(idev->dev_cmd_regs->data));
+ memcpy_fromio(&ident->port, &idev->dev_cmd_regs->data, sz);
+ }
+
+ mutex_unlock(&ionic->dev_cmd_lock);
+
+ return err;
+}
+
+int ionic_port_init(struct ionic *ionic)
+{
+ struct ionic_identity *ident = &ionic->ident;
+ struct ionic_dev *idev = &ionic->idev;
+ size_t sz;
+ int err;
+
+ if (!idev->port_info) {
+ idev->port_info_sz = ALIGN(sizeof(*idev->port_info), PAGE_SIZE);
+ idev->port_info = dma_alloc_coherent(ionic->dev,
+ idev->port_info_sz,
+ &idev->port_info_pa,
+ GFP_KERNEL);
+ if (!idev->port_info)
+ return -ENOMEM;
+ }
+
+ sz = min(sizeof(ident->port.config), sizeof(idev->dev_cmd_regs->data));
+
+ mutex_lock(&ionic->dev_cmd_lock);
+
+ memcpy_toio(&idev->dev_cmd_regs->data, &ident->port.config, sz);
+ ionic_dev_cmd_port_init(idev);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+
+ ionic_dev_cmd_port_state(&ionic->idev, IONIC_PORT_ADMIN_STATE_UP);
+ (void)ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+
+ mutex_unlock(&ionic->dev_cmd_lock);
+ if (err) {
+ dev_err(ionic->dev, "Failed to init port\n");
+ dma_free_coherent(ionic->dev, idev->port_info_sz,
+ idev->port_info, idev->port_info_pa);
+ idev->port_info = NULL;
+ idev->port_info_pa = 0;
+ }
+
+ return err;
+}
+
+int ionic_port_reset(struct ionic *ionic)
+{
+ struct ionic_dev *idev = &ionic->idev;
+ int err = 0;
+
+ if (!idev->port_info)
+ return 0;
+
+ if (ionic_is_fw_running(idev)) {
+ mutex_lock(&ionic->dev_cmd_lock);
+ ionic_dev_cmd_port_reset(idev);
+ err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
+ mutex_unlock(&ionic->dev_cmd_lock);
+ }
+
+ dma_free_coherent(ionic->dev, idev->port_info_sz,
+ idev->port_info, idev->port_info_pa);
+
+ idev->port_info = NULL;
+ idev->port_info_pa = 0;
+
+ return err;
+}
+
+static int __init ionic_init_module(void)
+{
+ int ret;
+
+ ionic_debugfs_create();
+ ret = ionic_bus_register_driver();
+ if (ret)
+ ionic_debugfs_destroy();
+
+ return ret;
+}
+
+static void __exit ionic_cleanup_module(void)
+{
+ ionic_bus_unregister_driver();
+ ionic_debugfs_destroy();
+
+ pr_info("%s removed\n", IONIC_DRV_NAME);
+}
+
+module_init(ionic_init_module);
+module_exit(ionic_cleanup_module);
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_phc.c b/drivers/net/ethernet/pensando/ionic/ionic_phc.c
new file mode 100644
index 000000000..887046838
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_phc.c
@@ -0,0 +1,647 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 - 2021 Pensando Systems, Inc */
+
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+
+#include "ionic.h"
+#include "ionic_bus.h"
+#include "ionic_lif.h"
+#include "ionic_ethtool.h"
+
+static int ionic_hwstamp_tx_mode(int config_tx_type)
+{
+ switch (config_tx_type) {
+ case HWTSTAMP_TX_OFF:
+ return IONIC_TXSTAMP_OFF;
+ case HWTSTAMP_TX_ON:
+ return IONIC_TXSTAMP_ON;
+ case HWTSTAMP_TX_ONESTEP_SYNC:
+ return IONIC_TXSTAMP_ONESTEP_SYNC;
+ case HWTSTAMP_TX_ONESTEP_P2P:
+ return IONIC_TXSTAMP_ONESTEP_P2P;
+ default:
+ return -ERANGE;
+ }
+}
+
+static u64 ionic_hwstamp_rx_filt(int config_rx_filter)
+{
+ switch (config_rx_filter) {
+ case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
+ return IONIC_PKT_CLS_PTP1_ALL;
+ case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
+ return IONIC_PKT_CLS_PTP1_SYNC;
+ case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
+ return IONIC_PKT_CLS_PTP1_SYNC | IONIC_PKT_CLS_PTP1_DREQ;
+
+ case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
+ return IONIC_PKT_CLS_PTP2_L4_ALL;
+ case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
+ return IONIC_PKT_CLS_PTP2_L4_SYNC;
+ case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
+ return IONIC_PKT_CLS_PTP2_L4_SYNC | IONIC_PKT_CLS_PTP2_L4_DREQ;
+
+ case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
+ return IONIC_PKT_CLS_PTP2_L2_ALL;
+ case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
+ return IONIC_PKT_CLS_PTP2_L2_SYNC;
+ case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
+ return IONIC_PKT_CLS_PTP2_L2_SYNC | IONIC_PKT_CLS_PTP2_L2_DREQ;
+
+ case HWTSTAMP_FILTER_PTP_V2_EVENT:
+ return IONIC_PKT_CLS_PTP2_ALL;
+ case HWTSTAMP_FILTER_PTP_V2_SYNC:
+ return IONIC_PKT_CLS_PTP2_SYNC;
+ case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
+ return IONIC_PKT_CLS_PTP2_SYNC | IONIC_PKT_CLS_PTP2_DREQ;
+
+ case HWTSTAMP_FILTER_NTP_ALL:
+ return IONIC_PKT_CLS_NTP_ALL;
+
+ default:
+ return 0;
+ }
+}
+
+static int ionic_lif_hwstamp_set_ts_config(struct ionic_lif *lif,
+ struct hwtstamp_config *new_ts)
+{
+ struct ionic *ionic = lif->ionic;
+ struct hwtstamp_config *config;
+ struct hwtstamp_config ts;
+ int tx_mode = 0;
+ u64 rx_filt = 0;
+ int err, err2;
+ bool rx_all;
+ __le64 mask;
+
+ if (!lif->phc || !lif->phc->ptp)
+ return -EOPNOTSUPP;
+
+ mutex_lock(&lif->phc->config_lock);
+
+ if (new_ts) {
+ config = new_ts;
+ } else {
+ /* If called with new_ts == NULL, replay the previous request
+ * primarily for recovery after a FW_RESET.
+ * We saved the previous configuration request info, so copy
+ * the previous request for reference, clear the current state
+ * to match the device's reset state, and run with it.
+ */
+ config = &ts;
+ memcpy(config, &lif->phc->ts_config, sizeof(*config));
+ memset(&lif->phc->ts_config, 0, sizeof(lif->phc->ts_config));
+ lif->phc->ts_config_tx_mode = 0;
+ lif->phc->ts_config_rx_filt = 0;
+ }
+
+ tx_mode = ionic_hwstamp_tx_mode(config->tx_type);
+ if (tx_mode < 0) {
+ err = tx_mode;
+ goto err_queues;
+ }
+
+ mask = cpu_to_le64(BIT_ULL(tx_mode));
+ if ((ionic->ident.lif.eth.hwstamp_tx_modes & mask) != mask) {
+ err = -ERANGE;
+ goto err_queues;
+ }
+
+ rx_filt = ionic_hwstamp_rx_filt(config->rx_filter);
+ rx_all = config->rx_filter != HWTSTAMP_FILTER_NONE && !rx_filt;
+
+ mask = cpu_to_le64(rx_filt);
+ if ((ionic->ident.lif.eth.hwstamp_rx_filters & mask) != mask) {
+ rx_filt = 0;
+ rx_all = true;
+ config->rx_filter = HWTSTAMP_FILTER_ALL;
+ }
+
+ dev_dbg(ionic->dev, "%s: config_rx_filter %d rx_filt %#llx rx_all %d\n",
+ __func__, config->rx_filter, rx_filt, rx_all);
+
+ if (tx_mode) {
+ err = ionic_lif_create_hwstamp_txq(lif);
+ if (err)
+ goto err_queues;
+ }
+
+ if (rx_filt) {
+ err = ionic_lif_create_hwstamp_rxq(lif);
+ if (err)
+ goto err_queues;
+ }
+
+ if (tx_mode != lif->phc->ts_config_tx_mode) {
+ err = ionic_lif_set_hwstamp_txmode(lif, tx_mode);
+ if (err)
+ goto err_txmode;
+ }
+
+ if (rx_filt != lif->phc->ts_config_rx_filt) {
+ err = ionic_lif_set_hwstamp_rxfilt(lif, rx_filt);
+ if (err)
+ goto err_rxfilt;
+ }
+
+ if (rx_all != (lif->phc->ts_config.rx_filter == HWTSTAMP_FILTER_ALL)) {
+ err = ionic_lif_config_hwstamp_rxq_all(lif, rx_all);
+ if (err)
+ goto err_rxall;
+ }
+
+ memcpy(&lif->phc->ts_config, config, sizeof(*config));
+ lif->phc->ts_config_rx_filt = rx_filt;
+ lif->phc->ts_config_tx_mode = tx_mode;
+
+ mutex_unlock(&lif->phc->config_lock);
+
+ return 0;
+
+err_rxall:
+ if (rx_filt != lif->phc->ts_config_rx_filt) {
+ rx_filt = lif->phc->ts_config_rx_filt;
+ err2 = ionic_lif_set_hwstamp_rxfilt(lif, rx_filt);
+ if (err2)
+ dev_err(ionic->dev,
+ "Failed to revert rx timestamp filter: %d\n", err2);
+ }
+err_rxfilt:
+ if (tx_mode != lif->phc->ts_config_tx_mode) {
+ tx_mode = lif->phc->ts_config_tx_mode;
+ err2 = ionic_lif_set_hwstamp_txmode(lif, tx_mode);
+ if (err2)
+ dev_err(ionic->dev,
+ "Failed to revert tx timestamp mode: %d\n", err2);
+ }
+err_txmode:
+ /* special queues remain allocated, just unused */
+err_queues:
+ mutex_unlock(&lif->phc->config_lock);
+ return err;
+}
+
+int ionic_lif_hwstamp_set(struct ionic_lif *lif, struct ifreq *ifr)
+{
+ struct hwtstamp_config config;
+ int err;
+
+ if (!lif->phc || !lif->phc->ptp)
+ return -EOPNOTSUPP;
+
+ if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
+ return -EFAULT;
+
+ mutex_lock(&lif->queue_lock);
+ err = ionic_lif_hwstamp_set_ts_config(lif, &config);
+ mutex_unlock(&lif->queue_lock);
+ if (err) {
+ netdev_info(lif->netdev, "hwstamp set failed: %d\n", err);
+ return err;
+ }
+
+ if (copy_to_user(ifr->ifr_data, &config, sizeof(config)))
+ return -EFAULT;
+
+ return 0;
+}
+
+void ionic_lif_hwstamp_replay(struct ionic_lif *lif)
+{
+ int err;
+
+ if (!lif->phc || !lif->phc->ptp)
+ return;
+
+ mutex_lock(&lif->queue_lock);
+ err = ionic_lif_hwstamp_set_ts_config(lif, NULL);
+ mutex_unlock(&lif->queue_lock);
+ if (err)
+ netdev_info(lif->netdev, "hwstamp replay failed: %d\n", err);
+}
+
+void ionic_lif_hwstamp_recreate_queues(struct ionic_lif *lif)
+{
+ int err;
+
+ if (!lif->phc || !lif->phc->ptp)
+ return;
+
+ mutex_lock(&lif->phc->config_lock);
+
+ if (lif->phc->ts_config_tx_mode) {
+ err = ionic_lif_create_hwstamp_txq(lif);
+ if (err)
+ netdev_info(lif->netdev, "hwstamp recreate txq failed: %d\n", err);
+ }
+
+ if (lif->phc->ts_config_rx_filt) {
+ err = ionic_lif_create_hwstamp_rxq(lif);
+ if (err)
+ netdev_info(lif->netdev, "hwstamp recreate rxq failed: %d\n", err);
+ }
+
+ mutex_unlock(&lif->phc->config_lock);
+}
+
+int ionic_lif_hwstamp_get(struct ionic_lif *lif, struct ifreq *ifr)
+{
+ struct hwtstamp_config config;
+
+ if (!lif->phc || !lif->phc->ptp)
+ return -EOPNOTSUPP;
+
+ mutex_lock(&lif->phc->config_lock);
+ memcpy(&config, &lif->phc->ts_config, sizeof(config));
+ mutex_unlock(&lif->phc->config_lock);
+
+ if (copy_to_user(ifr->ifr_data, &config, sizeof(config)))
+ return -EFAULT;
+ return 0;
+}
+
+static u64 ionic_hwstamp_read(struct ionic *ionic,
+ struct ptp_system_timestamp *sts)
+{
+ u32 tick_high_before, tick_high, tick_low;
+
+ /* read and discard low part to defeat hw staging of high part */
+ (void)ioread32(&ionic->idev.hwstamp_regs->tick_low);
+
+ tick_high_before = ioread32(&ionic->idev.hwstamp_regs->tick_high);
+
+ ptp_read_system_prets(sts);
+ tick_low = ioread32(&ionic->idev.hwstamp_regs->tick_low);
+ ptp_read_system_postts(sts);
+
+ tick_high = ioread32(&ionic->idev.hwstamp_regs->tick_high);
+
+ /* If tick_high changed, re-read tick_low once more. Assume tick_high
+ * cannot change again so soon as in the span of re-reading tick_low.
+ */
+ if (tick_high != tick_high_before) {
+ ptp_read_system_prets(sts);
+ tick_low = ioread32(&ionic->idev.hwstamp_regs->tick_low);
+ ptp_read_system_postts(sts);
+ }
+
+ return (u64)tick_low | ((u64)tick_high << 32);
+}
+
+static u64 ionic_cc_read(const struct cyclecounter *cc)
+{
+ struct ionic_phc *phc = container_of(cc, struct ionic_phc, cc);
+ struct ionic *ionic = phc->lif->ionic;
+
+ return ionic_hwstamp_read(ionic, NULL);
+}
+
+static int ionic_setphc_cmd(struct ionic_phc *phc, struct ionic_admin_ctx *ctx)
+{
+ ctx->work = COMPLETION_INITIALIZER_ONSTACK(ctx->work);
+
+ ctx->cmd.lif_setphc.opcode = IONIC_CMD_LIF_SETPHC;
+ ctx->cmd.lif_setphc.lif_index = cpu_to_le16(phc->lif->index);
+
+ ctx->cmd.lif_setphc.tick = cpu_to_le64(phc->tc.cycle_last);
+ ctx->cmd.lif_setphc.nsec = cpu_to_le64(phc->tc.nsec);
+ ctx->cmd.lif_setphc.frac = cpu_to_le64(phc->tc.frac);
+ ctx->cmd.lif_setphc.mult = cpu_to_le32(phc->cc.mult);
+ ctx->cmd.lif_setphc.shift = cpu_to_le32(phc->cc.shift);
+
+ return ionic_adminq_post(phc->lif, ctx);
+}
+
+static int ionic_phc_adjfine(struct ptp_clock_info *info, long scaled_ppm)
+{
+ struct ionic_phc *phc = container_of(info, struct ionic_phc, ptp_info);
+ struct ionic_admin_ctx ctx = {};
+ unsigned long irqflags;
+ s64 adj;
+ int err;
+
+ /* Reject phc adjustments during device upgrade */
+ if (test_bit(IONIC_LIF_F_FW_RESET, phc->lif->state))
+ return -EBUSY;
+
+ /* Adjustment value scaled by 2^16 million */
+ adj = (s64)scaled_ppm * phc->init_cc_mult;
+
+ /* Adjustment value to scale */
+ adj /= (s64)SCALED_PPM;
+
+ /* Final adjusted multiplier */
+ adj += phc->init_cc_mult;
+
+ spin_lock_irqsave(&phc->lock, irqflags);
+
+ /* update the point-in-time basis to now, before adjusting the rate */
+ timecounter_read(&phc->tc);
+ phc->cc.mult = adj;
+
+ /* Setphc commands are posted in-order, sequenced by phc->lock. We
+ * need to drop the lock before waiting for the command to complete.
+ */
+ err = ionic_setphc_cmd(phc, &ctx);
+
+ spin_unlock_irqrestore(&phc->lock, irqflags);
+
+ return ionic_adminq_wait(phc->lif, &ctx, err, true);
+}
+
+static int ionic_phc_adjtime(struct ptp_clock_info *info, s64 delta)
+{
+ struct ionic_phc *phc = container_of(info, struct ionic_phc, ptp_info);
+ struct ionic_admin_ctx ctx = {};
+ unsigned long irqflags;
+ int err;
+
+ /* Reject phc adjustments during device upgrade */
+ if (test_bit(IONIC_LIF_F_FW_RESET, phc->lif->state))
+ return -EBUSY;
+
+ spin_lock_irqsave(&phc->lock, irqflags);
+
+ timecounter_adjtime(&phc->tc, delta);
+
+ /* Setphc commands are posted in-order, sequenced by phc->lock. We
+ * need to drop the lock before waiting for the command to complete.
+ */
+ err = ionic_setphc_cmd(phc, &ctx);
+
+ spin_unlock_irqrestore(&phc->lock, irqflags);
+
+ return ionic_adminq_wait(phc->lif, &ctx, err, true);
+}
+
+static int ionic_phc_settime64(struct ptp_clock_info *info,
+ const struct timespec64 *ts)
+{
+ struct ionic_phc *phc = container_of(info, struct ionic_phc, ptp_info);
+ struct ionic_admin_ctx ctx = {};
+ unsigned long irqflags;
+ int err;
+ u64 ns;
+
+ /* Reject phc adjustments during device upgrade */
+ if (test_bit(IONIC_LIF_F_FW_RESET, phc->lif->state))
+ return -EBUSY;
+
+ ns = timespec64_to_ns(ts);
+
+ spin_lock_irqsave(&phc->lock, irqflags);
+
+ timecounter_init(&phc->tc, &phc->cc, ns);
+
+ /* Setphc commands are posted in-order, sequenced by phc->lock. We
+ * need to drop the lock before waiting for the command to complete.
+ */
+ err = ionic_setphc_cmd(phc, &ctx);
+
+ spin_unlock_irqrestore(&phc->lock, irqflags);
+
+ return ionic_adminq_wait(phc->lif, &ctx, err, true);
+}
+
+static int ionic_phc_gettimex64(struct ptp_clock_info *info,
+ struct timespec64 *ts,
+ struct ptp_system_timestamp *sts)
+{
+ struct ionic_phc *phc = container_of(info, struct ionic_phc, ptp_info);
+ struct ionic *ionic = phc->lif->ionic;
+ unsigned long irqflags;
+ u64 tick, ns;
+
+ /* Do not attempt to read device time during upgrade */
+ if (test_bit(IONIC_LIF_F_FW_RESET, phc->lif->state))
+ return -EBUSY;
+
+ spin_lock_irqsave(&phc->lock, irqflags);
+
+ tick = ionic_hwstamp_read(ionic, sts);
+
+ ns = timecounter_cyc2time(&phc->tc, tick);
+
+ spin_unlock_irqrestore(&phc->lock, irqflags);
+
+ *ts = ns_to_timespec64(ns);
+
+ return 0;
+}
+
+static long ionic_phc_aux_work(struct ptp_clock_info *info)
+{
+ struct ionic_phc *phc = container_of(info, struct ionic_phc, ptp_info);
+ struct ionic_admin_ctx ctx = {};
+ unsigned long irqflags;
+ int err;
+
+ /* Do not update phc during device upgrade, but keep polling to resume
+ * after upgrade. Since we don't update the point in time basis, there
+ * is no expectation that we are maintaining the phc time during the
+ * upgrade. After upgrade, it will need to be readjusted back to the
+ * correct time by the ptp daemon.
+ */
+ if (test_bit(IONIC_LIF_F_FW_RESET, phc->lif->state))
+ return phc->aux_work_delay;
+
+ spin_lock_irqsave(&phc->lock, irqflags);
+
+ /* update point-in-time basis to now */
+ timecounter_read(&phc->tc);
+
+ /* Setphc commands are posted in-order, sequenced by phc->lock. We
+ * need to drop the lock before waiting for the command to complete.
+ */
+ err = ionic_setphc_cmd(phc, &ctx);
+
+ spin_unlock_irqrestore(&phc->lock, irqflags);
+
+ ionic_adminq_wait(phc->lif, &ctx, err, true);
+
+ return phc->aux_work_delay;
+}
+
+ktime_t ionic_lif_phc_ktime(struct ionic_lif *lif, u64 tick)
+{
+ unsigned long irqflags;
+ u64 ns;
+
+ if (!lif->phc)
+ return 0;
+
+ spin_lock_irqsave(&lif->phc->lock, irqflags);
+ ns = timecounter_cyc2time(&lif->phc->tc, tick);
+ spin_unlock_irqrestore(&lif->phc->lock, irqflags);
+
+ return ns_to_ktime(ns);
+}
+
+static const struct ptp_clock_info ionic_ptp_info = {
+ .owner = THIS_MODULE,
+ .name = "ionic_ptp",
+ .adjfine = ionic_phc_adjfine,
+ .adjtime = ionic_phc_adjtime,
+ .gettimex64 = ionic_phc_gettimex64,
+ .settime64 = ionic_phc_settime64,
+ .do_aux_work = ionic_phc_aux_work,
+};
+
+void ionic_lif_register_phc(struct ionic_lif *lif)
+{
+ if (!lif->phc || !(lif->hw_features & IONIC_ETH_HW_TIMESTAMP))
+ return;
+
+ lif->phc->ptp = ptp_clock_register(&lif->phc->ptp_info, lif->ionic->dev);
+
+ if (IS_ERR(lif->phc->ptp)) {
+ dev_warn(lif->ionic->dev, "Cannot register phc device: %ld\n",
+ PTR_ERR(lif->phc->ptp));
+
+ lif->phc->ptp = NULL;
+ }
+
+ if (lif->phc->ptp)
+ ptp_schedule_worker(lif->phc->ptp, lif->phc->aux_work_delay);
+}
+
+void ionic_lif_unregister_phc(struct ionic_lif *lif)
+{
+ if (!lif->phc || !lif->phc->ptp)
+ return;
+
+ ptp_clock_unregister(lif->phc->ptp);
+
+ lif->phc->ptp = NULL;
+}
+
+void ionic_lif_alloc_phc(struct ionic_lif *lif)
+{
+ struct ionic *ionic = lif->ionic;
+ struct ionic_phc *phc;
+ u64 delay, diff, mult;
+ u64 frac = 0;
+ u64 features;
+ u32 shift;
+
+ if (!ionic->idev.hwstamp_regs)
+ return;
+
+ features = le64_to_cpu(ionic->ident.lif.eth.config.features);
+ if (!(features & IONIC_ETH_HW_TIMESTAMP))
+ return;
+
+ phc = devm_kzalloc(ionic->dev, sizeof(*phc), GFP_KERNEL);
+ if (!phc)
+ return;
+
+ phc->lif = lif;
+
+ phc->cc.read = ionic_cc_read;
+ phc->cc.mask = le64_to_cpu(ionic->ident.dev.hwstamp_mask);
+ phc->cc.mult = le32_to_cpu(ionic->ident.dev.hwstamp_mult);
+ phc->cc.shift = le32_to_cpu(ionic->ident.dev.hwstamp_shift);
+
+ if (!phc->cc.mult) {
+ dev_err(lif->ionic->dev,
+ "Invalid device PHC mask multiplier %u, disabling HW timestamp support\n",
+ phc->cc.mult);
+ devm_kfree(lif->ionic->dev, phc);
+ lif->phc = NULL;
+ return;
+ }
+
+ dev_dbg(lif->ionic->dev, "Device PHC mask %#llx mult %u shift %u\n",
+ phc->cc.mask, phc->cc.mult, phc->cc.shift);
+
+ spin_lock_init(&phc->lock);
+ mutex_init(&phc->config_lock);
+
+ /* max ticks is limited by the multiplier, or by the update period. */
+ if (phc->cc.shift + 2 + ilog2(IONIC_PHC_UPDATE_NS) >= 64) {
+ /* max ticks that do not overflow when multiplied by max
+ * adjusted multiplier (twice the initial multiplier)
+ */
+ diff = U64_MAX / phc->cc.mult / 2;
+ } else {
+ /* approx ticks at four times the update period */
+ diff = (u64)IONIC_PHC_UPDATE_NS << (phc->cc.shift + 2);
+ diff = DIV_ROUND_UP(diff, phc->cc.mult);
+ }
+
+ /* transform to bitmask */
+ diff |= diff >> 1;
+ diff |= diff >> 2;
+ diff |= diff >> 4;
+ diff |= diff >> 8;
+ diff |= diff >> 16;
+ diff |= diff >> 32;
+
+ /* constrain to the hardware bitmask, and use this as the bitmask */
+ diff &= phc->cc.mask;
+ phc->cc.mask = diff;
+
+ /* the wrap period is now defined by diff (or phc->cc.mask)
+ *
+ * we will update the time basis at about 1/4 the wrap period, so
+ * should not see a difference of more than +/- diff/4.
+ *
+ * this is sufficient not see a difference of more than +/- diff/2, as
+ * required by timecounter_cyc2time, to detect an old time stamp.
+ *
+ * adjust the initial multiplier, being careful to avoid overflow:
+ * - do not overflow 63 bits: init_cc_mult * SCALED_PPM
+ * - do not overflow 64 bits: max_mult * (diff / 2)
+ *
+ * we want to increase the initial multiplier as much as possible, to
+ * allow for more precise adjustment in ionic_phc_adjfine.
+ *
+ * only adjust the multiplier if we can double it or more.
+ */
+ mult = U64_MAX / 2 / max(diff / 2, SCALED_PPM);
+ shift = mult / phc->cc.mult;
+ if (shift >= 2) {
+ /* initial multiplier will be 2^n of hardware cc.mult */
+ shift = fls(shift);
+ /* increase cc.mult and cc.shift by the same 2^n and n. */
+ phc->cc.mult <<= shift;
+ phc->cc.shift += shift;
+ }
+
+ dev_dbg(lif->ionic->dev, "Initial PHC mask %#llx mult %u shift %u\n",
+ phc->cc.mask, phc->cc.mult, phc->cc.shift);
+
+ /* frequency adjustments are relative to the initial multiplier */
+ phc->init_cc_mult = phc->cc.mult;
+
+ timecounter_init(&phc->tc, &phc->cc, ktime_get_real_ns());
+
+ /* Update cycle_last at 1/4 the wrap period, or IONIC_PHC_UPDATE_NS */
+ delay = min_t(u64, IONIC_PHC_UPDATE_NS,
+ cyclecounter_cyc2ns(&phc->cc, diff / 4, 0, &frac));
+ dev_dbg(lif->ionic->dev, "Work delay %llu ms\n", delay / NSEC_PER_MSEC);
+
+ phc->aux_work_delay = nsecs_to_jiffies(delay);
+
+ phc->ptp_info = ionic_ptp_info;
+
+ /* We have allowed to adjust the multiplier up to +/- 1 part per 1.
+ * Here expressed as NORMAL_PPB (1 billion parts per billion).
+ */
+ phc->ptp_info.max_adj = NORMAL_PPB;
+
+ lif->phc = phc;
+}
+
+void ionic_lif_free_phc(struct ionic_lif *lif)
+{
+ if (!lif->phc)
+ return;
+
+ mutex_destroy(&lif->phc->config_lock);
+
+ devm_kfree(lif->ionic->dev, lif->phc);
+ lif->phc = NULL;
+}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_regs.h b/drivers/net/ethernet/pensando/ionic/ionic_regs.h
new file mode 100644
index 000000000..2e174f45c
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_regs.h
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) OR BSD-2-Clause */
+/* Copyright (c) 2018-2019 Pensando Systems, Inc. All rights reserved. */
+
+#ifndef IONIC_REGS_H
+#define IONIC_REGS_H
+
+#include <linux/io.h>
+
+/** struct ionic_intr - interrupt control register set.
+ * @coal_init: coalesce timer initial value.
+ * @mask: interrupt mask value.
+ * @credits: interrupt credit count and return.
+ * @mask_assert: interrupt mask value on assert.
+ * @coal: coalesce timer time remaining.
+ */
+struct ionic_intr {
+ u32 coal_init;
+ u32 mask;
+ u32 credits;
+ u32 mask_assert;
+ u32 coal;
+ u32 rsvd[3];
+};
+
+#define IONIC_INTR_CTRL_REGS_MAX 2048
+#define IONIC_INTR_CTRL_COAL_MAX 0x3F
+
+/** enum ionic_intr_mask_vals - valid values for mask and mask_assert.
+ * @IONIC_INTR_MASK_CLEAR: unmask interrupt.
+ * @IONIC_INTR_MASK_SET: mask interrupt.
+ */
+enum ionic_intr_mask_vals {
+ IONIC_INTR_MASK_CLEAR = 0,
+ IONIC_INTR_MASK_SET = 1,
+};
+
+/** enum ionic_intr_credits_bits - bitwise composition of credits values.
+ * @IONIC_INTR_CRED_COUNT: bit mask of credit count, no shift needed.
+ * @IONIC_INTR_CRED_COUNT_SIGNED: bit mask of credit count, including sign bit.
+ * @IONIC_INTR_CRED_UNMASK: unmask the interrupt.
+ * @IONIC_INTR_CRED_RESET_COALESCE: reset the coalesce timer.
+ * @IONIC_INTR_CRED_REARM: unmask the and reset the timer.
+ */
+enum ionic_intr_credits_bits {
+ IONIC_INTR_CRED_COUNT = 0x7fffu,
+ IONIC_INTR_CRED_COUNT_SIGNED = 0xffffu,
+ IONIC_INTR_CRED_UNMASK = 0x10000u,
+ IONIC_INTR_CRED_RESET_COALESCE = 0x20000u,
+ IONIC_INTR_CRED_REARM = (IONIC_INTR_CRED_UNMASK |
+ IONIC_INTR_CRED_RESET_COALESCE),
+};
+
+static inline void ionic_intr_coal_init(struct ionic_intr __iomem *intr_ctrl,
+ int intr_idx, u32 coal)
+{
+ iowrite32(coal, &intr_ctrl[intr_idx].coal_init);
+}
+
+static inline void ionic_intr_mask(struct ionic_intr __iomem *intr_ctrl,
+ int intr_idx, u32 mask)
+{
+ iowrite32(mask, &intr_ctrl[intr_idx].mask);
+}
+
+static inline void ionic_intr_credits(struct ionic_intr __iomem *intr_ctrl,
+ int intr_idx, u32 cred, u32 flags)
+{
+ if (WARN_ON_ONCE(cred > IONIC_INTR_CRED_COUNT)) {
+ cred = ioread32(&intr_ctrl[intr_idx].credits);
+ cred &= IONIC_INTR_CRED_COUNT_SIGNED;
+ }
+
+ iowrite32(cred | flags, &intr_ctrl[intr_idx].credits);
+}
+
+static inline void ionic_intr_clean(struct ionic_intr __iomem *intr_ctrl,
+ int intr_idx)
+{
+ u32 cred;
+
+ cred = ioread32(&intr_ctrl[intr_idx].credits);
+ cred &= IONIC_INTR_CRED_COUNT_SIGNED;
+ cred |= IONIC_INTR_CRED_RESET_COALESCE;
+ iowrite32(cred, &intr_ctrl[intr_idx].credits);
+}
+
+static inline void ionic_intr_mask_assert(struct ionic_intr __iomem *intr_ctrl,
+ int intr_idx, u32 mask)
+{
+ iowrite32(mask, &intr_ctrl[intr_idx].mask_assert);
+}
+
+/** enum ionic_dbell_bits - bitwise composition of dbell values.
+ *
+ * @IONIC_DBELL_QID_MASK: unshifted mask of valid queue id bits.
+ * @IONIC_DBELL_QID_SHIFT: queue id shift amount in dbell value.
+ * @IONIC_DBELL_QID: macro to build QID component of dbell value.
+ *
+ * @IONIC_DBELL_RING_MASK: unshifted mask of valid ring bits.
+ * @IONIC_DBELL_RING_SHIFT: ring shift amount in dbell value.
+ * @IONIC_DBELL_RING: macro to build ring component of dbell value.
+ *
+ * @IONIC_DBELL_RING_0: ring zero dbell component value.
+ * @IONIC_DBELL_RING_1: ring one dbell component value.
+ * @IONIC_DBELL_RING_2: ring two dbell component value.
+ * @IONIC_DBELL_RING_3: ring three dbell component value.
+ *
+ * @IONIC_DBELL_INDEX_MASK: bit mask of valid index bits, no shift needed.
+ */
+enum ionic_dbell_bits {
+ IONIC_DBELL_QID_MASK = 0xffffff,
+ IONIC_DBELL_QID_SHIFT = 24,
+
+#define IONIC_DBELL_QID(n) \
+ (((u64)(n) & IONIC_DBELL_QID_MASK) << IONIC_DBELL_QID_SHIFT)
+
+ IONIC_DBELL_RING_MASK = 0x7,
+ IONIC_DBELL_RING_SHIFT = 16,
+
+#define IONIC_DBELL_RING(n) \
+ (((u64)(n) & IONIC_DBELL_RING_MASK) << IONIC_DBELL_RING_SHIFT)
+
+ IONIC_DBELL_RING_0 = 0,
+ IONIC_DBELL_RING_1 = IONIC_DBELL_RING(1),
+ IONIC_DBELL_RING_2 = IONIC_DBELL_RING(2),
+ IONIC_DBELL_RING_3 = IONIC_DBELL_RING(3),
+
+ IONIC_DBELL_INDEX_MASK = 0xffff,
+};
+
+static inline void ionic_dbell_ring(u64 __iomem *db_page, int qtype, u64 val)
+{
+ writeq(val, &db_page[qtype]);
+}
+
+#endif /* IONIC_REGS_H */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c b/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c
new file mode 100644
index 000000000..b7363376d
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c
@@ -0,0 +1,619 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#include <linux/netdevice.h>
+#include <linux/dynamic_debug.h>
+#include <linux/etherdevice.h>
+#include <linux/list.h>
+
+#include "ionic.h"
+#include "ionic_lif.h"
+#include "ionic_rx_filter.h"
+
+void ionic_rx_filter_free(struct ionic_lif *lif, struct ionic_rx_filter *f)
+{
+ struct device *dev = lif->ionic->dev;
+
+ hlist_del(&f->by_id);
+ hlist_del(&f->by_hash);
+ devm_kfree(dev, f);
+}
+
+void ionic_rx_filter_replay(struct ionic_lif *lif)
+{
+ struct ionic_rx_filter_add_cmd *ac;
+ struct hlist_head new_id_list;
+ struct ionic_admin_ctx ctx;
+ struct ionic_rx_filter *f;
+ struct hlist_head *head;
+ struct hlist_node *tmp;
+ unsigned int key;
+ unsigned int i;
+ int err;
+
+ INIT_HLIST_HEAD(&new_id_list);
+ ac = &ctx.cmd.rx_filter_add;
+
+ for (i = 0; i < IONIC_RX_FILTER_HLISTS; i++) {
+ head = &lif->rx_filters.by_id[i];
+ hlist_for_each_entry_safe(f, tmp, head, by_id) {
+ ctx.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work);
+ memcpy(ac, &f->cmd, sizeof(f->cmd));
+ dev_dbg(&lif->netdev->dev, "replay filter command:\n");
+ dynamic_hex_dump("cmd ", DUMP_PREFIX_OFFSET, 16, 1,
+ &ctx.cmd, sizeof(ctx.cmd), true);
+
+ err = ionic_adminq_post_wait(lif, &ctx);
+ if (err) {
+ switch (le16_to_cpu(ac->match)) {
+ case IONIC_RX_FILTER_MATCH_VLAN:
+ netdev_info(lif->netdev, "Replay failed - %d: vlan %d\n",
+ err,
+ le16_to_cpu(ac->vlan.vlan));
+ break;
+ case IONIC_RX_FILTER_MATCH_MAC:
+ netdev_info(lif->netdev, "Replay failed - %d: mac %pM\n",
+ err, ac->mac.addr);
+ break;
+ case IONIC_RX_FILTER_MATCH_MAC_VLAN:
+ netdev_info(lif->netdev, "Replay failed - %d: vlan %d mac %pM\n",
+ err,
+ le16_to_cpu(ac->vlan.vlan),
+ ac->mac.addr);
+ break;
+ }
+ spin_lock_bh(&lif->rx_filters.lock);
+ ionic_rx_filter_free(lif, f);
+ spin_unlock_bh(&lif->rx_filters.lock);
+
+ continue;
+ }
+
+ /* remove from old id list, save new id in tmp list */
+ spin_lock_bh(&lif->rx_filters.lock);
+ hlist_del(&f->by_id);
+ spin_unlock_bh(&lif->rx_filters.lock);
+ f->filter_id = le32_to_cpu(ctx.comp.rx_filter_add.filter_id);
+ hlist_add_head(&f->by_id, &new_id_list);
+ }
+ }
+
+ /* rebuild the by_id hash lists with the new filter ids */
+ spin_lock_bh(&lif->rx_filters.lock);
+ hlist_for_each_entry_safe(f, tmp, &new_id_list, by_id) {
+ key = f->filter_id & IONIC_RX_FILTER_HLISTS_MASK;
+ head = &lif->rx_filters.by_id[key];
+ hlist_add_head(&f->by_id, head);
+ }
+ spin_unlock_bh(&lif->rx_filters.lock);
+}
+
+int ionic_rx_filters_init(struct ionic_lif *lif)
+{
+ unsigned int i;
+
+ spin_lock_init(&lif->rx_filters.lock);
+
+ spin_lock_bh(&lif->rx_filters.lock);
+ for (i = 0; i < IONIC_RX_FILTER_HLISTS; i++) {
+ INIT_HLIST_HEAD(&lif->rx_filters.by_hash[i]);
+ INIT_HLIST_HEAD(&lif->rx_filters.by_id[i]);
+ }
+ spin_unlock_bh(&lif->rx_filters.lock);
+
+ return 0;
+}
+
+void ionic_rx_filters_deinit(struct ionic_lif *lif)
+{
+ struct ionic_rx_filter *f;
+ struct hlist_head *head;
+ struct hlist_node *tmp;
+ unsigned int i;
+
+ spin_lock_bh(&lif->rx_filters.lock);
+ for (i = 0; i < IONIC_RX_FILTER_HLISTS; i++) {
+ head = &lif->rx_filters.by_id[i];
+ hlist_for_each_entry_safe(f, tmp, head, by_id)
+ ionic_rx_filter_free(lif, f);
+ }
+ spin_unlock_bh(&lif->rx_filters.lock);
+}
+
+int ionic_rx_filter_save(struct ionic_lif *lif, u32 flow_id, u16 rxq_index,
+ u32 hash, struct ionic_admin_ctx *ctx,
+ enum ionic_filter_state state)
+{
+ struct device *dev = lif->ionic->dev;
+ struct ionic_rx_filter_add_cmd *ac;
+ struct ionic_rx_filter *f = NULL;
+ struct hlist_head *head;
+ unsigned int key;
+
+ ac = &ctx->cmd.rx_filter_add;
+
+ switch (le16_to_cpu(ac->match)) {
+ case IONIC_RX_FILTER_MATCH_VLAN:
+ key = le16_to_cpu(ac->vlan.vlan);
+ f = ionic_rx_filter_by_vlan(lif, le16_to_cpu(ac->vlan.vlan));
+ break;
+ case IONIC_RX_FILTER_MATCH_MAC:
+ key = *(u32 *)ac->mac.addr;
+ f = ionic_rx_filter_by_addr(lif, ac->mac.addr);
+ break;
+ case IONIC_RX_FILTER_MATCH_MAC_VLAN:
+ key = le16_to_cpu(ac->mac_vlan.vlan);
+ break;
+ case IONIC_RX_FILTER_STEER_PKTCLASS:
+ key = 0;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (f) {
+ /* remove from current linking so we can refresh it */
+ hlist_del(&f->by_id);
+ hlist_del(&f->by_hash);
+ } else {
+ f = devm_kzalloc(dev, sizeof(*f), GFP_ATOMIC);
+ if (!f)
+ return -ENOMEM;
+ }
+
+ f->flow_id = flow_id;
+ f->filter_id = le32_to_cpu(ctx->comp.rx_filter_add.filter_id);
+ f->state = state;
+ f->rxq_index = rxq_index;
+ memcpy(&f->cmd, ac, sizeof(f->cmd));
+ netdev_dbg(lif->netdev, "rx_filter add filter_id %d\n", f->filter_id);
+
+ INIT_HLIST_NODE(&f->by_hash);
+ INIT_HLIST_NODE(&f->by_id);
+
+ key = hash_32(key, IONIC_RX_FILTER_HASH_BITS);
+ head = &lif->rx_filters.by_hash[key];
+ hlist_add_head(&f->by_hash, head);
+
+ key = f->filter_id & IONIC_RX_FILTER_HLISTS_MASK;
+ head = &lif->rx_filters.by_id[key];
+ hlist_add_head(&f->by_id, head);
+
+ return 0;
+}
+
+struct ionic_rx_filter *ionic_rx_filter_by_vlan(struct ionic_lif *lif, u16 vid)
+{
+ struct ionic_rx_filter *f;
+ struct hlist_head *head;
+ unsigned int key;
+
+ key = hash_32(vid, IONIC_RX_FILTER_HASH_BITS);
+ head = &lif->rx_filters.by_hash[key];
+
+ hlist_for_each_entry(f, head, by_hash) {
+ if (le16_to_cpu(f->cmd.match) != IONIC_RX_FILTER_MATCH_VLAN)
+ continue;
+ if (le16_to_cpu(f->cmd.vlan.vlan) == vid)
+ return f;
+ }
+
+ return NULL;
+}
+
+struct ionic_rx_filter *ionic_rx_filter_by_addr(struct ionic_lif *lif,
+ const u8 *addr)
+{
+ struct ionic_rx_filter *f;
+ struct hlist_head *head;
+ unsigned int key;
+
+ key = hash_32(*(u32 *)addr, IONIC_RX_FILTER_HASH_BITS);
+ head = &lif->rx_filters.by_hash[key];
+
+ hlist_for_each_entry(f, head, by_hash) {
+ if (le16_to_cpu(f->cmd.match) != IONIC_RX_FILTER_MATCH_MAC)
+ continue;
+ if (memcmp(addr, f->cmd.mac.addr, ETH_ALEN) == 0)
+ return f;
+ }
+
+ return NULL;
+}
+
+struct ionic_rx_filter *ionic_rx_filter_rxsteer(struct ionic_lif *lif)
+{
+ struct ionic_rx_filter *f;
+ struct hlist_head *head;
+ unsigned int key;
+
+ key = hash_32(0, IONIC_RX_FILTER_HASH_BITS);
+ head = &lif->rx_filters.by_hash[key];
+
+ hlist_for_each_entry(f, head, by_hash) {
+ if (le16_to_cpu(f->cmd.match) != IONIC_RX_FILTER_STEER_PKTCLASS)
+ continue;
+ return f;
+ }
+
+ return NULL;
+}
+
+static struct ionic_rx_filter *ionic_rx_filter_find(struct ionic_lif *lif,
+ struct ionic_rx_filter_add_cmd *ac)
+{
+ switch (le16_to_cpu(ac->match)) {
+ case IONIC_RX_FILTER_MATCH_VLAN:
+ return ionic_rx_filter_by_vlan(lif, le16_to_cpu(ac->vlan.vlan));
+ case IONIC_RX_FILTER_MATCH_MAC:
+ return ionic_rx_filter_by_addr(lif, ac->mac.addr);
+ default:
+ netdev_err(lif->netdev, "unsupported filter match %d",
+ le16_to_cpu(ac->match));
+ return NULL;
+ }
+}
+
+int ionic_lif_list_addr(struct ionic_lif *lif, const u8 *addr, bool mode)
+{
+ struct ionic_rx_filter *f;
+ int err;
+
+ spin_lock_bh(&lif->rx_filters.lock);
+
+ f = ionic_rx_filter_by_addr(lif, addr);
+ if (mode == ADD_ADDR && !f) {
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.rx_filter_add = {
+ .opcode = IONIC_CMD_RX_FILTER_ADD,
+ .lif_index = cpu_to_le16(lif->index),
+ .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC),
+ },
+ };
+
+ memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, ETH_ALEN);
+ err = ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx,
+ IONIC_FILTER_STATE_NEW);
+ if (err) {
+ spin_unlock_bh(&lif->rx_filters.lock);
+ return err;
+ }
+
+ } else if (mode == ADD_ADDR && f) {
+ if (f->state == IONIC_FILTER_STATE_OLD)
+ f->state = IONIC_FILTER_STATE_SYNCED;
+
+ } else if (mode == DEL_ADDR && f) {
+ if (f->state == IONIC_FILTER_STATE_NEW)
+ ionic_rx_filter_free(lif, f);
+ else if (f->state == IONIC_FILTER_STATE_SYNCED)
+ f->state = IONIC_FILTER_STATE_OLD;
+ } else if (mode == DEL_ADDR && !f) {
+ spin_unlock_bh(&lif->rx_filters.lock);
+ return -ENOENT;
+ }
+
+ spin_unlock_bh(&lif->rx_filters.lock);
+
+ set_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state);
+
+ return 0;
+}
+
+static int ionic_lif_filter_add(struct ionic_lif *lif,
+ struct ionic_rx_filter_add_cmd *ac)
+{
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ };
+ struct ionic_rx_filter *f;
+ int nfilters;
+ int err = 0;
+
+ ctx.cmd.rx_filter_add = *ac;
+ ctx.cmd.rx_filter_add.opcode = IONIC_CMD_RX_FILTER_ADD,
+ ctx.cmd.rx_filter_add.lif_index = cpu_to_le16(lif->index),
+
+ spin_lock_bh(&lif->rx_filters.lock);
+ f = ionic_rx_filter_find(lif, &ctx.cmd.rx_filter_add);
+ if (f) {
+ /* don't bother if we already have it and it is sync'd */
+ if (f->state == IONIC_FILTER_STATE_SYNCED) {
+ spin_unlock_bh(&lif->rx_filters.lock);
+ return 0;
+ }
+
+ /* mark preemptively as sync'd to block any parallel attempts */
+ f->state = IONIC_FILTER_STATE_SYNCED;
+ } else {
+ /* save as SYNCED to catch any DEL requests while processing */
+ err = ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx,
+ IONIC_FILTER_STATE_SYNCED);
+ }
+ spin_unlock_bh(&lif->rx_filters.lock);
+ if (err)
+ return err;
+
+ /* Don't bother with the write to FW if we know there's no room,
+ * we can try again on the next sync attempt.
+ * Since the FW doesn't have a way to tell us the vlan limit,
+ * we start max_vlans at 0 until we hit the ENOSPC error.
+ */
+ switch (le16_to_cpu(ctx.cmd.rx_filter_add.match)) {
+ case IONIC_RX_FILTER_MATCH_VLAN:
+ netdev_dbg(lif->netdev, "%s: rx_filter add VLAN %d\n",
+ __func__, ctx.cmd.rx_filter_add.vlan.vlan);
+ if (lif->max_vlans && lif->nvlans >= lif->max_vlans)
+ err = -ENOSPC;
+ break;
+ case IONIC_RX_FILTER_MATCH_MAC:
+ netdev_dbg(lif->netdev, "%s: rx_filter add ADDR %pM\n",
+ __func__, ctx.cmd.rx_filter_add.mac.addr);
+ nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
+ if ((lif->nucast + lif->nmcast) >= nfilters)
+ err = -ENOSPC;
+ break;
+ }
+
+ if (err != -ENOSPC)
+ err = ionic_adminq_post_wait_nomsg(lif, &ctx);
+
+ spin_lock_bh(&lif->rx_filters.lock);
+
+ if (err && err != -EEXIST) {
+ /* set the state back to NEW so we can try again later */
+ f = ionic_rx_filter_find(lif, &ctx.cmd.rx_filter_add);
+ if (f && f->state == IONIC_FILTER_STATE_SYNCED) {
+ f->state = IONIC_FILTER_STATE_NEW;
+
+ /* If -ENOSPC we won't waste time trying to sync again
+ * until there is a delete that might make room
+ */
+ if (err != -ENOSPC)
+ set_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state);
+ }
+
+ spin_unlock_bh(&lif->rx_filters.lock);
+
+ /* store the max_vlans limit that we found */
+ if (err == -ENOSPC &&
+ le16_to_cpu(ctx.cmd.rx_filter_add.match) == IONIC_RX_FILTER_MATCH_VLAN)
+ lif->max_vlans = lif->nvlans;
+
+ /* Prevent unnecessary error messages on recoverable
+ * errors as the filter will get retried on the next
+ * sync attempt.
+ */
+ switch (err) {
+ case -ENOSPC:
+ case -ENXIO:
+ case -ETIMEDOUT:
+ case -EAGAIN:
+ case -EBUSY:
+ return 0;
+ default:
+ break;
+ }
+
+ ionic_adminq_netdev_err_print(lif, ctx.cmd.cmd.opcode,
+ ctx.comp.comp.status, err);
+ switch (le16_to_cpu(ctx.cmd.rx_filter_add.match)) {
+ case IONIC_RX_FILTER_MATCH_VLAN:
+ netdev_info(lif->netdev, "rx_filter add failed: VLAN %d\n",
+ ctx.cmd.rx_filter_add.vlan.vlan);
+ break;
+ case IONIC_RX_FILTER_MATCH_MAC:
+ netdev_info(lif->netdev, "rx_filter add failed: ADDR %pM\n",
+ ctx.cmd.rx_filter_add.mac.addr);
+ break;
+ }
+
+ return err;
+ }
+
+ switch (le16_to_cpu(ctx.cmd.rx_filter_add.match)) {
+ case IONIC_RX_FILTER_MATCH_VLAN:
+ lif->nvlans++;
+ break;
+ case IONIC_RX_FILTER_MATCH_MAC:
+ if (is_multicast_ether_addr(ctx.cmd.rx_filter_add.mac.addr))
+ lif->nmcast++;
+ else
+ lif->nucast++;
+ break;
+ }
+
+ f = ionic_rx_filter_find(lif, &ctx.cmd.rx_filter_add);
+ if (f && f->state == IONIC_FILTER_STATE_OLD) {
+ /* Someone requested a delete while we were adding
+ * so update the filter info with the results from the add
+ * and the data will be there for the delete on the next
+ * sync cycle.
+ */
+ err = ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx,
+ IONIC_FILTER_STATE_OLD);
+ } else {
+ err = ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx,
+ IONIC_FILTER_STATE_SYNCED);
+ }
+
+ spin_unlock_bh(&lif->rx_filters.lock);
+
+ return err;
+}
+
+int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr)
+{
+ struct ionic_rx_filter_add_cmd ac = {
+ .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC),
+ };
+
+ memcpy(&ac.mac.addr, addr, ETH_ALEN);
+
+ return ionic_lif_filter_add(lif, &ac);
+}
+
+int ionic_lif_vlan_add(struct ionic_lif *lif, const u16 vid)
+{
+ struct ionic_rx_filter_add_cmd ac = {
+ .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_VLAN),
+ .vlan.vlan = cpu_to_le16(vid),
+ };
+
+ return ionic_lif_filter_add(lif, &ac);
+}
+
+static int ionic_lif_filter_del(struct ionic_lif *lif,
+ struct ionic_rx_filter_add_cmd *ac)
+{
+ struct ionic_admin_ctx ctx = {
+ .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
+ .cmd.rx_filter_del = {
+ .opcode = IONIC_CMD_RX_FILTER_DEL,
+ .lif_index = cpu_to_le16(lif->index),
+ },
+ };
+ struct ionic_rx_filter *f;
+ int state;
+ int err;
+
+ spin_lock_bh(&lif->rx_filters.lock);
+ f = ionic_rx_filter_find(lif, ac);
+ if (!f) {
+ spin_unlock_bh(&lif->rx_filters.lock);
+ return -ENOENT;
+ }
+
+ switch (le16_to_cpu(ac->match)) {
+ case IONIC_RX_FILTER_MATCH_VLAN:
+ netdev_dbg(lif->netdev, "%s: rx_filter del VLAN %d id %d\n",
+ __func__, ac->vlan.vlan, f->filter_id);
+ lif->nvlans--;
+ break;
+ case IONIC_RX_FILTER_MATCH_MAC:
+ netdev_dbg(lif->netdev, "%s: rx_filter del ADDR %pM id %d\n",
+ __func__, ac->mac.addr, f->filter_id);
+ if (is_multicast_ether_addr(ac->mac.addr) && lif->nmcast)
+ lif->nmcast--;
+ else if (!is_multicast_ether_addr(ac->mac.addr) && lif->nucast)
+ lif->nucast--;
+ break;
+ }
+
+ state = f->state;
+ ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
+ ionic_rx_filter_free(lif, f);
+
+ spin_unlock_bh(&lif->rx_filters.lock);
+
+ if (state != IONIC_FILTER_STATE_NEW) {
+ err = ionic_adminq_post_wait_nomsg(lif, &ctx);
+
+ switch (err) {
+ /* ignore these errors */
+ case -EEXIST:
+ case -ENXIO:
+ case -ETIMEDOUT:
+ case -EAGAIN:
+ case -EBUSY:
+ case 0:
+ break;
+ default:
+ ionic_adminq_netdev_err_print(lif, ctx.cmd.cmd.opcode,
+ ctx.comp.comp.status, err);
+ return err;
+ }
+ }
+
+ return 0;
+}
+
+int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr)
+{
+ struct ionic_rx_filter_add_cmd ac = {
+ .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC),
+ };
+
+ memcpy(&ac.mac.addr, addr, ETH_ALEN);
+
+ return ionic_lif_filter_del(lif, &ac);
+}
+
+int ionic_lif_vlan_del(struct ionic_lif *lif, const u16 vid)
+{
+ struct ionic_rx_filter_add_cmd ac = {
+ .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_VLAN),
+ .vlan.vlan = cpu_to_le16(vid),
+ };
+
+ return ionic_lif_filter_del(lif, &ac);
+}
+
+struct sync_item {
+ struct list_head list;
+ struct ionic_rx_filter f;
+};
+
+void ionic_rx_filter_sync(struct ionic_lif *lif)
+{
+ struct device *dev = lif->ionic->dev;
+ struct list_head sync_add_list;
+ struct list_head sync_del_list;
+ struct sync_item *sync_item;
+ struct ionic_rx_filter *f;
+ struct hlist_head *head;
+ struct hlist_node *tmp;
+ struct sync_item *spos;
+ unsigned int i;
+
+ INIT_LIST_HEAD(&sync_add_list);
+ INIT_LIST_HEAD(&sync_del_list);
+
+ clear_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state);
+
+ /* Copy the filters to be added and deleted
+ * into a separate local list that needs no locking.
+ */
+ spin_lock_bh(&lif->rx_filters.lock);
+ for (i = 0; i < IONIC_RX_FILTER_HLISTS; i++) {
+ head = &lif->rx_filters.by_id[i];
+ hlist_for_each_entry_safe(f, tmp, head, by_id) {
+ if (f->state == IONIC_FILTER_STATE_NEW ||
+ f->state == IONIC_FILTER_STATE_OLD) {
+ sync_item = devm_kzalloc(dev, sizeof(*sync_item),
+ GFP_ATOMIC);
+ if (!sync_item)
+ goto loop_out;
+
+ sync_item->f = *f;
+
+ if (f->state == IONIC_FILTER_STATE_NEW)
+ list_add(&sync_item->list, &sync_add_list);
+ else
+ list_add(&sync_item->list, &sync_del_list);
+ }
+ }
+ }
+loop_out:
+ spin_unlock_bh(&lif->rx_filters.lock);
+
+ /* If the add or delete fails, it won't get marked as sync'd
+ * and will be tried again in the next sync action.
+ * Do the deletes first in case we're in an overflow state and
+ * they can clear room for some new filters
+ */
+ list_for_each_entry_safe(sync_item, spos, &sync_del_list, list) {
+ (void)ionic_lif_filter_del(lif, &sync_item->f.cmd);
+
+ list_del(&sync_item->list);
+ devm_kfree(dev, sync_item);
+ }
+
+ list_for_each_entry_safe(sync_item, spos, &sync_add_list, list) {
+ (void)ionic_lif_filter_add(lif, &sync_item->f.cmd);
+
+ list_del(&sync_item->list);
+ devm_kfree(dev, sync_item);
+ }
+}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.h b/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.h
new file mode 100644
index 000000000..87b2666f2
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#ifndef _IONIC_RX_FILTER_H_
+#define _IONIC_RX_FILTER_H_
+
+#define IONIC_RXQ_INDEX_ANY (0xFFFF)
+
+enum ionic_filter_state {
+ IONIC_FILTER_STATE_SYNCED,
+ IONIC_FILTER_STATE_NEW,
+ IONIC_FILTER_STATE_OLD,
+};
+
+struct ionic_rx_filter {
+ u32 flow_id;
+ u32 filter_id;
+ u16 rxq_index;
+ enum ionic_filter_state state;
+ struct ionic_rx_filter_add_cmd cmd;
+ struct hlist_node by_hash;
+ struct hlist_node by_id;
+};
+
+#define IONIC_RX_FILTER_HASH_BITS 10
+#define IONIC_RX_FILTER_HLISTS BIT(IONIC_RX_FILTER_HASH_BITS)
+#define IONIC_RX_FILTER_HLISTS_MASK (IONIC_RX_FILTER_HLISTS - 1)
+struct ionic_rx_filters {
+ spinlock_t lock; /* filter list lock */
+ struct hlist_head by_hash[IONIC_RX_FILTER_HLISTS]; /* by skb hash */
+ struct hlist_head by_id[IONIC_RX_FILTER_HLISTS]; /* by filter_id */
+};
+
+void ionic_rx_filter_free(struct ionic_lif *lif, struct ionic_rx_filter *f);
+void ionic_rx_filter_replay(struct ionic_lif *lif);
+int ionic_rx_filters_init(struct ionic_lif *lif);
+void ionic_rx_filters_deinit(struct ionic_lif *lif);
+int ionic_rx_filter_save(struct ionic_lif *lif, u32 flow_id, u16 rxq_index,
+ u32 hash, struct ionic_admin_ctx *ctx,
+ enum ionic_filter_state state);
+struct ionic_rx_filter *ionic_rx_filter_by_vlan(struct ionic_lif *lif, u16 vid);
+struct ionic_rx_filter *ionic_rx_filter_by_addr(struct ionic_lif *lif, const u8 *addr);
+struct ionic_rx_filter *ionic_rx_filter_rxsteer(struct ionic_lif *lif);
+void ionic_rx_filter_sync(struct ionic_lif *lif);
+int ionic_lif_list_addr(struct ionic_lif *lif, const u8 *addr, bool mode);
+int ionic_rx_filters_need_sync(struct ionic_lif *lif);
+int ionic_lif_vlan_add(struct ionic_lif *lif, const u16 vid);
+int ionic_lif_vlan_del(struct ionic_lif *lif, const u16 vid);
+
+#endif /* _IONIC_RX_FILTER_H_ */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_stats.c b/drivers/net/ethernet/pensando/ionic/ionic_stats.c
new file mode 100644
index 000000000..9859a4432
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_stats.c
@@ -0,0 +1,350 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#include <linux/ethtool.h>
+#include <linux/kernel.h>
+#include <linux/mutex.h>
+#include <linux/netdevice.h>
+
+#include "ionic.h"
+#include "ionic_lif.h"
+#include "ionic_stats.h"
+
+static const struct ionic_stat_desc ionic_lif_stats_desc[] = {
+ IONIC_LIF_STAT_DESC(tx_packets),
+ IONIC_LIF_STAT_DESC(tx_bytes),
+ IONIC_LIF_STAT_DESC(rx_packets),
+ IONIC_LIF_STAT_DESC(rx_bytes),
+ IONIC_LIF_STAT_DESC(tx_tso),
+ IONIC_LIF_STAT_DESC(tx_tso_bytes),
+ IONIC_LIF_STAT_DESC(tx_csum_none),
+ IONIC_LIF_STAT_DESC(tx_csum),
+ IONIC_LIF_STAT_DESC(rx_csum_none),
+ IONIC_LIF_STAT_DESC(rx_csum_complete),
+ IONIC_LIF_STAT_DESC(rx_csum_error),
+ IONIC_LIF_STAT_DESC(hw_tx_dropped),
+ IONIC_LIF_STAT_DESC(hw_rx_dropped),
+ IONIC_LIF_STAT_DESC(hw_rx_over_errors),
+ IONIC_LIF_STAT_DESC(hw_rx_missed_errors),
+ IONIC_LIF_STAT_DESC(hw_tx_aborted_errors),
+};
+
+static const struct ionic_stat_desc ionic_port_stats_desc[] = {
+ IONIC_PORT_STAT_DESC(frames_rx_ok),
+ IONIC_PORT_STAT_DESC(frames_rx_all),
+ IONIC_PORT_STAT_DESC(frames_rx_bad_fcs),
+ IONIC_PORT_STAT_DESC(frames_rx_bad_all),
+ IONIC_PORT_STAT_DESC(octets_rx_ok),
+ IONIC_PORT_STAT_DESC(octets_rx_all),
+ IONIC_PORT_STAT_DESC(frames_rx_unicast),
+ IONIC_PORT_STAT_DESC(frames_rx_multicast),
+ IONIC_PORT_STAT_DESC(frames_rx_broadcast),
+ IONIC_PORT_STAT_DESC(frames_rx_pause),
+ IONIC_PORT_STAT_DESC(frames_rx_bad_length),
+ IONIC_PORT_STAT_DESC(frames_rx_undersized),
+ IONIC_PORT_STAT_DESC(frames_rx_oversized),
+ IONIC_PORT_STAT_DESC(frames_rx_fragments),
+ IONIC_PORT_STAT_DESC(frames_rx_jabber),
+ IONIC_PORT_STAT_DESC(frames_rx_pripause),
+ IONIC_PORT_STAT_DESC(frames_rx_stomped_crc),
+ IONIC_PORT_STAT_DESC(frames_rx_too_long),
+ IONIC_PORT_STAT_DESC(frames_rx_vlan_good),
+ IONIC_PORT_STAT_DESC(frames_rx_dropped),
+ IONIC_PORT_STAT_DESC(frames_rx_less_than_64b),
+ IONIC_PORT_STAT_DESC(frames_rx_64b),
+ IONIC_PORT_STAT_DESC(frames_rx_65b_127b),
+ IONIC_PORT_STAT_DESC(frames_rx_128b_255b),
+ IONIC_PORT_STAT_DESC(frames_rx_256b_511b),
+ IONIC_PORT_STAT_DESC(frames_rx_512b_1023b),
+ IONIC_PORT_STAT_DESC(frames_rx_1024b_1518b),
+ IONIC_PORT_STAT_DESC(frames_rx_1519b_2047b),
+ IONIC_PORT_STAT_DESC(frames_rx_2048b_4095b),
+ IONIC_PORT_STAT_DESC(frames_rx_4096b_8191b),
+ IONIC_PORT_STAT_DESC(frames_rx_8192b_9215b),
+ IONIC_PORT_STAT_DESC(frames_rx_other),
+ IONIC_PORT_STAT_DESC(frames_tx_ok),
+ IONIC_PORT_STAT_DESC(frames_tx_all),
+ IONIC_PORT_STAT_DESC(frames_tx_bad),
+ IONIC_PORT_STAT_DESC(octets_tx_ok),
+ IONIC_PORT_STAT_DESC(octets_tx_total),
+ IONIC_PORT_STAT_DESC(frames_tx_unicast),
+ IONIC_PORT_STAT_DESC(frames_tx_multicast),
+ IONIC_PORT_STAT_DESC(frames_tx_broadcast),
+ IONIC_PORT_STAT_DESC(frames_tx_pause),
+ IONIC_PORT_STAT_DESC(frames_tx_pripause),
+ IONIC_PORT_STAT_DESC(frames_tx_vlan),
+ IONIC_PORT_STAT_DESC(frames_tx_less_than_64b),
+ IONIC_PORT_STAT_DESC(frames_tx_64b),
+ IONIC_PORT_STAT_DESC(frames_tx_65b_127b),
+ IONIC_PORT_STAT_DESC(frames_tx_128b_255b),
+ IONIC_PORT_STAT_DESC(frames_tx_256b_511b),
+ IONIC_PORT_STAT_DESC(frames_tx_512b_1023b),
+ IONIC_PORT_STAT_DESC(frames_tx_1024b_1518b),
+ IONIC_PORT_STAT_DESC(frames_tx_1519b_2047b),
+ IONIC_PORT_STAT_DESC(frames_tx_2048b_4095b),
+ IONIC_PORT_STAT_DESC(frames_tx_4096b_8191b),
+ IONIC_PORT_STAT_DESC(frames_tx_8192b_9215b),
+ IONIC_PORT_STAT_DESC(frames_tx_other),
+ IONIC_PORT_STAT_DESC(frames_tx_pri_0),
+ IONIC_PORT_STAT_DESC(frames_tx_pri_1),
+ IONIC_PORT_STAT_DESC(frames_tx_pri_2),
+ IONIC_PORT_STAT_DESC(frames_tx_pri_3),
+ IONIC_PORT_STAT_DESC(frames_tx_pri_4),
+ IONIC_PORT_STAT_DESC(frames_tx_pri_5),
+ IONIC_PORT_STAT_DESC(frames_tx_pri_6),
+ IONIC_PORT_STAT_DESC(frames_tx_pri_7),
+ IONIC_PORT_STAT_DESC(frames_rx_pri_0),
+ IONIC_PORT_STAT_DESC(frames_rx_pri_1),
+ IONIC_PORT_STAT_DESC(frames_rx_pri_2),
+ IONIC_PORT_STAT_DESC(frames_rx_pri_3),
+ IONIC_PORT_STAT_DESC(frames_rx_pri_4),
+ IONIC_PORT_STAT_DESC(frames_rx_pri_5),
+ IONIC_PORT_STAT_DESC(frames_rx_pri_6),
+ IONIC_PORT_STAT_DESC(frames_rx_pri_7),
+ IONIC_PORT_STAT_DESC(tx_pripause_0_1us_count),
+ IONIC_PORT_STAT_DESC(tx_pripause_1_1us_count),
+ IONIC_PORT_STAT_DESC(tx_pripause_2_1us_count),
+ IONIC_PORT_STAT_DESC(tx_pripause_3_1us_count),
+ IONIC_PORT_STAT_DESC(tx_pripause_4_1us_count),
+ IONIC_PORT_STAT_DESC(tx_pripause_5_1us_count),
+ IONIC_PORT_STAT_DESC(tx_pripause_6_1us_count),
+ IONIC_PORT_STAT_DESC(tx_pripause_7_1us_count),
+ IONIC_PORT_STAT_DESC(rx_pripause_0_1us_count),
+ IONIC_PORT_STAT_DESC(rx_pripause_1_1us_count),
+ IONIC_PORT_STAT_DESC(rx_pripause_2_1us_count),
+ IONIC_PORT_STAT_DESC(rx_pripause_3_1us_count),
+ IONIC_PORT_STAT_DESC(rx_pripause_4_1us_count),
+ IONIC_PORT_STAT_DESC(rx_pripause_5_1us_count),
+ IONIC_PORT_STAT_DESC(rx_pripause_6_1us_count),
+ IONIC_PORT_STAT_DESC(rx_pripause_7_1us_count),
+ IONIC_PORT_STAT_DESC(rx_pause_1us_count),
+ IONIC_PORT_STAT_DESC(frames_tx_truncated),
+};
+
+static const struct ionic_stat_desc ionic_tx_stats_desc[] = {
+ IONIC_TX_STAT_DESC(pkts),
+ IONIC_TX_STAT_DESC(bytes),
+ IONIC_TX_STAT_DESC(clean),
+ IONIC_TX_STAT_DESC(dma_map_err),
+ IONIC_TX_STAT_DESC(linearize),
+ IONIC_TX_STAT_DESC(frags),
+ IONIC_TX_STAT_DESC(tso),
+ IONIC_TX_STAT_DESC(tso_bytes),
+ IONIC_TX_STAT_DESC(hwstamp_valid),
+ IONIC_TX_STAT_DESC(hwstamp_invalid),
+ IONIC_TX_STAT_DESC(csum_none),
+ IONIC_TX_STAT_DESC(csum),
+ IONIC_TX_STAT_DESC(vlan_inserted),
+};
+
+static const struct ionic_stat_desc ionic_rx_stats_desc[] = {
+ IONIC_RX_STAT_DESC(pkts),
+ IONIC_RX_STAT_DESC(bytes),
+ IONIC_RX_STAT_DESC(dma_map_err),
+ IONIC_RX_STAT_DESC(alloc_err),
+ IONIC_RX_STAT_DESC(csum_none),
+ IONIC_RX_STAT_DESC(csum_complete),
+ IONIC_RX_STAT_DESC(csum_error),
+ IONIC_RX_STAT_DESC(hwstamp_valid),
+ IONIC_RX_STAT_DESC(hwstamp_invalid),
+ IONIC_RX_STAT_DESC(dropped),
+ IONIC_RX_STAT_DESC(vlan_stripped),
+};
+
+#define IONIC_NUM_LIF_STATS ARRAY_SIZE(ionic_lif_stats_desc)
+#define IONIC_NUM_PORT_STATS ARRAY_SIZE(ionic_port_stats_desc)
+#define IONIC_NUM_TX_STATS ARRAY_SIZE(ionic_tx_stats_desc)
+#define IONIC_NUM_RX_STATS ARRAY_SIZE(ionic_rx_stats_desc)
+
+#define MAX_Q(lif) ((lif)->netdev->real_num_tx_queues)
+
+static void ionic_add_lif_txq_stats(struct ionic_lif *lif, int q_num,
+ struct ionic_lif_sw_stats *stats)
+{
+ struct ionic_tx_stats *txstats = &lif->txqstats[q_num];
+
+ stats->tx_packets += txstats->pkts;
+ stats->tx_bytes += txstats->bytes;
+ stats->tx_tso += txstats->tso;
+ stats->tx_tso_bytes += txstats->tso_bytes;
+ stats->tx_csum_none += txstats->csum_none;
+ stats->tx_csum += txstats->csum;
+ stats->tx_hwstamp_valid += txstats->hwstamp_valid;
+ stats->tx_hwstamp_invalid += txstats->hwstamp_invalid;
+}
+
+static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num,
+ struct ionic_lif_sw_stats *stats)
+{
+ struct ionic_rx_stats *rxstats = &lif->rxqstats[q_num];
+
+ stats->rx_packets += rxstats->pkts;
+ stats->rx_bytes += rxstats->bytes;
+ stats->rx_csum_none += rxstats->csum_none;
+ stats->rx_csum_complete += rxstats->csum_complete;
+ stats->rx_csum_error += rxstats->csum_error;
+ stats->rx_hwstamp_valid += rxstats->hwstamp_valid;
+ stats->rx_hwstamp_invalid += rxstats->hwstamp_invalid;
+}
+
+static void ionic_get_lif_stats(struct ionic_lif *lif,
+ struct ionic_lif_sw_stats *stats)
+{
+ struct rtnl_link_stats64 ns;
+ int q_num;
+
+ memset(stats, 0, sizeof(*stats));
+
+ for (q_num = 0; q_num < MAX_Q(lif); q_num++) {
+ ionic_add_lif_txq_stats(lif, q_num, stats);
+ ionic_add_lif_rxq_stats(lif, q_num, stats);
+ }
+
+ if (lif->hwstamp_txq)
+ ionic_add_lif_txq_stats(lif, lif->hwstamp_txq->q.index, stats);
+
+ if (lif->hwstamp_rxq)
+ ionic_add_lif_rxq_stats(lif, lif->hwstamp_rxq->q.index, stats);
+
+ ionic_get_stats64(lif->netdev, &ns);
+ stats->hw_tx_dropped = ns.tx_dropped;
+ stats->hw_rx_dropped = ns.rx_dropped;
+ stats->hw_rx_over_errors = ns.rx_over_errors;
+ stats->hw_rx_missed_errors = ns.rx_missed_errors;
+ stats->hw_tx_aborted_errors = ns.tx_aborted_errors;
+}
+
+static u64 ionic_sw_stats_get_count(struct ionic_lif *lif)
+{
+ u64 total = 0, tx_queues = MAX_Q(lif), rx_queues = MAX_Q(lif);
+
+ if (lif->hwstamp_txq)
+ tx_queues += 1;
+
+ if (lif->hwstamp_rxq)
+ rx_queues += 1;
+
+ total += IONIC_NUM_LIF_STATS;
+ total += IONIC_NUM_PORT_STATS;
+
+ total += tx_queues * IONIC_NUM_TX_STATS;
+ total += rx_queues * IONIC_NUM_RX_STATS;
+
+ return total;
+}
+
+static void ionic_sw_stats_get_tx_strings(struct ionic_lif *lif, u8 **buf,
+ int q_num)
+{
+ int i;
+
+ for (i = 0; i < IONIC_NUM_TX_STATS; i++)
+ ethtool_sprintf(buf, "tx_%d_%s", q_num,
+ ionic_tx_stats_desc[i].name);
+}
+
+static void ionic_sw_stats_get_rx_strings(struct ionic_lif *lif, u8 **buf,
+ int q_num)
+{
+ int i;
+
+ for (i = 0; i < IONIC_NUM_RX_STATS; i++)
+ ethtool_sprintf(buf, "rx_%d_%s", q_num,
+ ionic_rx_stats_desc[i].name);
+}
+
+static void ionic_sw_stats_get_strings(struct ionic_lif *lif, u8 **buf)
+{
+ int i, q_num;
+
+ for (i = 0; i < IONIC_NUM_LIF_STATS; i++)
+ ethtool_sprintf(buf, ionic_lif_stats_desc[i].name);
+
+ for (i = 0; i < IONIC_NUM_PORT_STATS; i++)
+ ethtool_sprintf(buf, ionic_port_stats_desc[i].name);
+
+ for (q_num = 0; q_num < MAX_Q(lif); q_num++)
+ ionic_sw_stats_get_tx_strings(lif, buf, q_num);
+
+ if (lif->hwstamp_txq)
+ ionic_sw_stats_get_tx_strings(lif, buf, lif->hwstamp_txq->q.index);
+
+ for (q_num = 0; q_num < MAX_Q(lif); q_num++)
+ ionic_sw_stats_get_rx_strings(lif, buf, q_num);
+
+ if (lif->hwstamp_rxq)
+ ionic_sw_stats_get_rx_strings(lif, buf, lif->hwstamp_rxq->q.index);
+}
+
+static void ionic_sw_stats_get_txq_values(struct ionic_lif *lif, u64 **buf,
+ int q_num)
+{
+ struct ionic_tx_stats *txstats;
+ int i;
+
+ txstats = &lif->txqstats[q_num];
+
+ for (i = 0; i < IONIC_NUM_TX_STATS; i++) {
+ **buf = IONIC_READ_STAT64(txstats, &ionic_tx_stats_desc[i]);
+ (*buf)++;
+ }
+}
+
+static void ionic_sw_stats_get_rxq_values(struct ionic_lif *lif, u64 **buf,
+ int q_num)
+{
+ struct ionic_rx_stats *rxstats;
+ int i;
+
+ rxstats = &lif->rxqstats[q_num];
+
+ for (i = 0; i < IONIC_NUM_RX_STATS; i++) {
+ **buf = IONIC_READ_STAT64(rxstats, &ionic_rx_stats_desc[i]);
+ (*buf)++;
+ }
+}
+
+static void ionic_sw_stats_get_values(struct ionic_lif *lif, u64 **buf)
+{
+ struct ionic_port_stats *port_stats;
+ struct ionic_lif_sw_stats lif_stats;
+ int i, q_num;
+
+ ionic_get_lif_stats(lif, &lif_stats);
+
+ for (i = 0; i < IONIC_NUM_LIF_STATS; i++) {
+ **buf = IONIC_READ_STAT64(&lif_stats, &ionic_lif_stats_desc[i]);
+ (*buf)++;
+ }
+
+ port_stats = &lif->ionic->idev.port_info->stats;
+ for (i = 0; i < IONIC_NUM_PORT_STATS; i++) {
+ **buf = IONIC_READ_STAT_LE64(port_stats,
+ &ionic_port_stats_desc[i]);
+ (*buf)++;
+ }
+
+ for (q_num = 0; q_num < MAX_Q(lif); q_num++)
+ ionic_sw_stats_get_txq_values(lif, buf, q_num);
+
+ if (lif->hwstamp_txq)
+ ionic_sw_stats_get_txq_values(lif, buf, lif->hwstamp_txq->q.index);
+
+ for (q_num = 0; q_num < MAX_Q(lif); q_num++)
+ ionic_sw_stats_get_rxq_values(lif, buf, q_num);
+
+ if (lif->hwstamp_rxq)
+ ionic_sw_stats_get_rxq_values(lif, buf, lif->hwstamp_rxq->q.index);
+}
+
+const struct ionic_stats_group_intf ionic_stats_groups[] = {
+ /* SW Stats group */
+ {
+ .get_strings = ionic_sw_stats_get_strings,
+ .get_values = ionic_sw_stats_get_values,
+ .get_count = ionic_sw_stats_get_count,
+ },
+ /* Add more stat groups here */
+};
+
+const int ionic_num_stats_grps = ARRAY_SIZE(ionic_stats_groups);
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_stats.h b/drivers/net/ethernet/pensando/ionic/ionic_stats.h
new file mode 100644
index 000000000..2a725834f
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_stats.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#ifndef _IONIC_STATS_H_
+#define _IONIC_STATS_H_
+
+#define IONIC_STAT_TO_OFFSET(type, stat_name) (offsetof(type, stat_name))
+
+#define IONIC_STAT_DESC(type, stat_name) { \
+ .name = #stat_name, \
+ .offset = IONIC_STAT_TO_OFFSET(type, stat_name) \
+}
+
+#define IONIC_PORT_STAT_DESC(stat_name) \
+ IONIC_STAT_DESC(struct ionic_port_stats, stat_name)
+
+#define IONIC_LIF_STAT_DESC(stat_name) \
+ IONIC_STAT_DESC(struct ionic_lif_sw_stats, stat_name)
+
+#define IONIC_TX_STAT_DESC(stat_name) \
+ IONIC_STAT_DESC(struct ionic_tx_stats, stat_name)
+
+#define IONIC_RX_STAT_DESC(stat_name) \
+ IONIC_STAT_DESC(struct ionic_rx_stats, stat_name)
+
+#define IONIC_TX_Q_STAT_DESC(stat_name) \
+ IONIC_STAT_DESC(struct ionic_queue, stat_name)
+
+#define IONIC_CQ_STAT_DESC(stat_name) \
+ IONIC_STAT_DESC(struct ionic_cq, stat_name)
+
+#define IONIC_INTR_STAT_DESC(stat_name) \
+ IONIC_STAT_DESC(struct ionic_intr_info, stat_name)
+
+#define IONIC_NAPI_STAT_DESC(stat_name) \
+ IONIC_STAT_DESC(struct ionic_napi_stats, stat_name)
+
+/* Interface structure for a particalar stats group */
+struct ionic_stats_group_intf {
+ void (*get_strings)(struct ionic_lif *lif, u8 **buf);
+ void (*get_values)(struct ionic_lif *lif, u64 **buf);
+ u64 (*get_count)(struct ionic_lif *lif);
+};
+
+extern const struct ionic_stats_group_intf ionic_stats_groups[];
+extern const int ionic_num_stats_grps;
+
+#define IONIC_READ_STAT64(base_ptr, desc_ptr) \
+ (*((u64 *)(((u8 *)(base_ptr)) + (desc_ptr)->offset)))
+
+#define IONIC_READ_STAT_LE64(base_ptr, desc_ptr) \
+ __le64_to_cpu(*((__le64 *)(((u8 *)(base_ptr)) + (desc_ptr)->offset)))
+
+struct ionic_stat_desc {
+ char name[ETH_GSTRING_LEN];
+ u64 offset;
+};
+
+#endif /* _IONIC_STATS_H_ */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
new file mode 100644
index 000000000..4684b9f19
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -0,0 +1,1350 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/if_vlan.h>
+#include <net/ip6_checksum.h>
+
+#include "ionic.h"
+#include "ionic_lif.h"
+#include "ionic_txrx.h"
+
+static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
+ ionic_desc_cb cb_func, void *cb_arg)
+{
+ ionic_q_post(q, ring_dbell, cb_func, cb_arg);
+}
+
+static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell,
+ ionic_desc_cb cb_func, void *cb_arg)
+{
+ ionic_q_post(q, ring_dbell, cb_func, cb_arg);
+}
+
+bool ionic_txq_poke_doorbell(struct ionic_queue *q)
+{
+ unsigned long now, then, dif;
+ struct netdev_queue *netdev_txq;
+ struct net_device *netdev;
+
+ netdev = q->lif->netdev;
+ netdev_txq = netdev_get_tx_queue(netdev, q->index);
+
+ HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id());
+
+ if (q->tail_idx == q->head_idx) {
+ HARD_TX_UNLOCK(netdev, netdev_txq);
+ return false;
+ }
+
+ now = READ_ONCE(jiffies);
+ then = q->dbell_jiffies;
+ dif = now - then;
+
+ if (dif > q->dbell_deadline) {
+ ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
+ q->dbval | q->head_idx);
+
+ q->dbell_jiffies = now;
+ }
+
+ HARD_TX_UNLOCK(netdev, netdev_txq);
+
+ return true;
+}
+
+bool ionic_rxq_poke_doorbell(struct ionic_queue *q)
+{
+ unsigned long now, then, dif;
+
+ /* no lock, called from rx napi or txrx napi, nothing else can fill */
+
+ if (q->tail_idx == q->head_idx)
+ return false;
+
+ now = READ_ONCE(jiffies);
+ then = q->dbell_jiffies;
+ dif = now - then;
+
+ if (dif > q->dbell_deadline) {
+ ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
+ q->dbval | q->head_idx);
+
+ q->dbell_jiffies = now;
+
+ dif = 2 * q->dbell_deadline;
+ if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE)
+ dif = IONIC_RX_MAX_DOORBELL_DEADLINE;
+
+ q->dbell_deadline = dif;
+ }
+
+ return true;
+}
+
+static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)
+{
+ return netdev_get_tx_queue(q->lif->netdev, q->index);
+}
+
+static int ionic_rx_page_alloc(struct ionic_queue *q,
+ struct ionic_buf_info *buf_info)
+{
+ struct net_device *netdev = q->lif->netdev;
+ struct ionic_rx_stats *stats;
+ struct device *dev;
+ struct page *page;
+
+ dev = q->dev;
+ stats = q_to_rx_stats(q);
+
+ if (unlikely(!buf_info)) {
+ net_err_ratelimited("%s: %s invalid buf_info in alloc\n",
+ netdev->name, q->name);
+ return -EINVAL;
+ }
+
+ page = alloc_pages(IONIC_PAGE_GFP_MASK, 0);
+ if (unlikely(!page)) {
+ net_err_ratelimited("%s: %s page alloc failed\n",
+ netdev->name, q->name);
+ stats->alloc_err++;
+ return -ENOMEM;
+ }
+
+ buf_info->dma_addr = dma_map_page(dev, page, 0,
+ IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
+ if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) {
+ __free_pages(page, 0);
+ net_err_ratelimited("%s: %s dma map failed\n",
+ netdev->name, q->name);
+ stats->dma_map_err++;
+ return -EIO;
+ }
+
+ buf_info->page = page;
+ buf_info->page_offset = 0;
+
+ return 0;
+}
+
+static void ionic_rx_page_free(struct ionic_queue *q,
+ struct ionic_buf_info *buf_info)
+{
+ struct net_device *netdev = q->lif->netdev;
+ struct device *dev = q->dev;
+
+ if (unlikely(!buf_info)) {
+ net_err_ratelimited("%s: %s invalid buf_info in free\n",
+ netdev->name, q->name);
+ return;
+ }
+
+ if (!buf_info->page)
+ return;
+
+ dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
+ __free_pages(buf_info->page, 0);
+ buf_info->page = NULL;
+}
+
+static bool ionic_rx_buf_recycle(struct ionic_queue *q,
+ struct ionic_buf_info *buf_info, u32 used)
+{
+ u32 size;
+
+ /* don't re-use pages allocated in low-mem condition */
+ if (page_is_pfmemalloc(buf_info->page))
+ return false;
+
+ /* don't re-use buffers from non-local numa nodes */
+ if (page_to_nid(buf_info->page) != numa_mem_id())
+ return false;
+
+ size = ALIGN(used, IONIC_PAGE_SPLIT_SZ);
+ buf_info->page_offset += size;
+ if (buf_info->page_offset >= IONIC_PAGE_SIZE)
+ return false;
+
+ get_page(buf_info->page);
+
+ return true;
+}
+
+static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
+ struct ionic_desc_info *desc_info,
+ struct ionic_rxq_comp *comp)
+{
+ struct net_device *netdev = q->lif->netdev;
+ struct ionic_buf_info *buf_info;
+ struct ionic_rx_stats *stats;
+ struct device *dev = q->dev;
+ struct sk_buff *skb;
+ unsigned int i;
+ u16 frag_len;
+ u16 len;
+
+ stats = q_to_rx_stats(q);
+
+ buf_info = &desc_info->bufs[0];
+ len = le16_to_cpu(comp->len);
+
+ prefetchw(buf_info->page);
+
+ skb = napi_get_frags(&q_to_qcq(q)->napi);
+ if (unlikely(!skb)) {
+ net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
+ netdev->name, q->name);
+ stats->alloc_err++;
+ return NULL;
+ }
+
+ i = comp->num_sg_elems + 1;
+ do {
+ if (unlikely(!buf_info->page)) {
+ dev_kfree_skb(skb);
+ return NULL;
+ }
+
+ frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN,
+ IONIC_PAGE_SIZE - buf_info->page_offset));
+ len -= frag_len;
+
+ dma_sync_single_for_cpu(dev,
+ buf_info->dma_addr + buf_info->page_offset,
+ frag_len, DMA_FROM_DEVICE);
+
+ skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
+ buf_info->page, buf_info->page_offset, frag_len,
+ IONIC_PAGE_SIZE);
+
+ if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) {
+ dma_unmap_page(dev, buf_info->dma_addr,
+ IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
+ buf_info->page = NULL;
+ }
+
+ buf_info++;
+
+ i--;
+ } while (i > 0);
+
+ return skb;
+}
+
+static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
+ struct ionic_desc_info *desc_info,
+ struct ionic_rxq_comp *comp)
+{
+ struct net_device *netdev = q->lif->netdev;
+ struct ionic_buf_info *buf_info;
+ struct ionic_rx_stats *stats;
+ struct device *dev = q->dev;
+ struct sk_buff *skb;
+ u16 len;
+
+ stats = q_to_rx_stats(q);
+
+ buf_info = &desc_info->bufs[0];
+ len = le16_to_cpu(comp->len);
+
+ skb = napi_alloc_skb(&q_to_qcq(q)->napi, len);
+ if (unlikely(!skb)) {
+ net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
+ netdev->name, q->name);
+ stats->alloc_err++;
+ return NULL;
+ }
+
+ if (unlikely(!buf_info->page)) {
+ dev_kfree_skb(skb);
+ return NULL;
+ }
+
+ dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset,
+ len, DMA_FROM_DEVICE);
+ skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len);
+ dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset,
+ len, DMA_FROM_DEVICE);
+
+ skb_put(skb, len);
+ skb->protocol = eth_type_trans(skb, q->lif->netdev);
+
+ return skb;
+}
+
+static void ionic_rx_clean(struct ionic_queue *q,
+ struct ionic_desc_info *desc_info,
+ struct ionic_cq_info *cq_info,
+ void *cb_arg)
+{
+ struct net_device *netdev = q->lif->netdev;
+ struct ionic_qcq *qcq = q_to_qcq(q);
+ struct ionic_rx_stats *stats;
+ struct ionic_rxq_comp *comp;
+ struct sk_buff *skb;
+
+ comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp);
+
+ stats = q_to_rx_stats(q);
+
+ if (comp->status) {
+ stats->dropped++;
+ return;
+ }
+
+ stats->pkts++;
+ stats->bytes += le16_to_cpu(comp->len);
+
+ if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
+ skb = ionic_rx_copybreak(q, desc_info, comp);
+ else
+ skb = ionic_rx_frags(q, desc_info, comp);
+
+ if (unlikely(!skb)) {
+ stats->dropped++;
+ return;
+ }
+
+ skb_record_rx_queue(skb, q->index);
+
+ if (likely(netdev->features & NETIF_F_RXHASH)) {
+ switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
+ case IONIC_PKT_TYPE_IPV4:
+ case IONIC_PKT_TYPE_IPV6:
+ skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
+ PKT_HASH_TYPE_L3);
+ break;
+ case IONIC_PKT_TYPE_IPV4_TCP:
+ case IONIC_PKT_TYPE_IPV6_TCP:
+ case IONIC_PKT_TYPE_IPV4_UDP:
+ case IONIC_PKT_TYPE_IPV6_UDP:
+ skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
+ PKT_HASH_TYPE_L4);
+ break;
+ }
+ }
+
+ if (likely(netdev->features & NETIF_F_RXCSUM) &&
+ (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) {
+ skb->ip_summed = CHECKSUM_COMPLETE;
+ skb->csum = (__force __wsum)le16_to_cpu(comp->csum);
+ stats->csum_complete++;
+ } else {
+ stats->csum_none++;
+ }
+
+ if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
+ (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) ||
+ (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)))
+ stats->csum_error++;
+
+ if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
+ (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) {
+ __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
+ le16_to_cpu(comp->vlan_tci));
+ stats->vlan_stripped++;
+ }
+
+ if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) {
+ __le64 *cq_desc_hwstamp;
+ u64 hwstamp;
+
+ cq_desc_hwstamp =
+ cq_info->cq_desc +
+ qcq->cq.desc_size -
+ sizeof(struct ionic_rxq_comp) -
+ IONIC_HWSTAMP_CQ_NEGOFFSET;
+
+ hwstamp = le64_to_cpu(*cq_desc_hwstamp);
+
+ if (hwstamp != IONIC_HWSTAMP_INVALID) {
+ skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
+ stats->hwstamp_valid++;
+ } else {
+ stats->hwstamp_invalid++;
+ }
+ }
+
+ if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
+ napi_gro_receive(&qcq->napi, skb);
+ else
+ napi_gro_frags(&qcq->napi);
+}
+
+bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
+{
+ struct ionic_queue *q = cq->bound_q;
+ struct ionic_desc_info *desc_info;
+ struct ionic_rxq_comp *comp;
+
+ comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
+
+ if (!color_match(comp->pkt_type_color, cq->done_color))
+ return false;
+
+ /* check for empty queue */
+ if (q->tail_idx == q->head_idx)
+ return false;
+
+ if (q->tail_idx != le16_to_cpu(comp->comp_index))
+ return false;
+
+ desc_info = &q->info[q->tail_idx];
+ q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
+
+ /* clean the related q entry, only one per qc completion */
+ ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg);
+
+ desc_info->cb = NULL;
+ desc_info->cb_arg = NULL;
+
+ return true;
+}
+
+void ionic_rx_fill(struct ionic_queue *q)
+{
+ struct net_device *netdev = q->lif->netdev;
+ struct ionic_desc_info *desc_info;
+ struct ionic_rxq_sg_desc *sg_desc;
+ struct ionic_rxq_sg_elem *sg_elem;
+ struct ionic_buf_info *buf_info;
+ unsigned int fill_threshold;
+ struct ionic_rxq_desc *desc;
+ unsigned int remain_len;
+ unsigned int frag_len;
+ unsigned int nfrags;
+ unsigned int n_fill;
+ unsigned int i, j;
+ unsigned int len;
+
+ n_fill = ionic_q_space_avail(q);
+
+ fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD,
+ q->num_descs / IONIC_RX_FILL_DIV);
+ if (n_fill < fill_threshold)
+ return;
+
+ len = netdev->mtu + ETH_HLEN + VLAN_HLEN;
+
+ for (i = n_fill; i; i--) {
+ nfrags = 0;
+ remain_len = len;
+ desc_info = &q->info[q->head_idx];
+ desc = desc_info->desc;
+ buf_info = &desc_info->bufs[0];
+
+ if (!buf_info->page) { /* alloc a new buffer? */
+ if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
+ desc->addr = 0;
+ desc->len = 0;
+ return;
+ }
+ }
+
+ /* fill main descriptor - buf[0] */
+ desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
+ frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN,
+ IONIC_PAGE_SIZE - buf_info->page_offset));
+ desc->len = cpu_to_le16(frag_len);
+ remain_len -= frag_len;
+ buf_info++;
+ nfrags++;
+
+ /* fill sg descriptors - buf[1..n] */
+ sg_desc = desc_info->sg_desc;
+ for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) {
+ sg_elem = &sg_desc->elems[j];
+ if (!buf_info->page) { /* alloc a new sg buffer? */
+ if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
+ sg_elem->addr = 0;
+ sg_elem->len = 0;
+ return;
+ }
+ }
+
+ sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
+ frag_len = min_t(u16, remain_len, min_t(u32, IONIC_MAX_BUF_LEN,
+ IONIC_PAGE_SIZE -
+ buf_info->page_offset));
+ sg_elem->len = cpu_to_le16(frag_len);
+ remain_len -= frag_len;
+ buf_info++;
+ nfrags++;
+ }
+
+ /* clear end sg element as a sentinel */
+ if (j < q->max_sg_elems) {
+ sg_elem = &sg_desc->elems[j];
+ memset(sg_elem, 0, sizeof(*sg_elem));
+ }
+
+ desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
+ IONIC_RXQ_DESC_OPCODE_SIMPLE;
+ desc_info->nbufs = nfrags;
+
+ ionic_rxq_post(q, false, ionic_rx_clean, NULL);
+ }
+
+ ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
+ q->dbval | q->head_idx);
+
+ q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
+ q->dbell_jiffies = jiffies;
+
+ mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
+ jiffies + IONIC_NAPI_DEADLINE);
+}
+
+void ionic_rx_empty(struct ionic_queue *q)
+{
+ struct ionic_desc_info *desc_info;
+ struct ionic_buf_info *buf_info;
+ unsigned int i, j;
+
+ for (i = 0; i < q->num_descs; i++) {
+ desc_info = &q->info[i];
+ for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) {
+ buf_info = &desc_info->bufs[j];
+ if (buf_info->page)
+ ionic_rx_page_free(q, buf_info);
+ }
+
+ desc_info->nbufs = 0;
+ desc_info->cb = NULL;
+ desc_info->cb_arg = NULL;
+ }
+
+ q->head_idx = 0;
+ q->tail_idx = 0;
+}
+
+static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode)
+{
+ struct dim_sample dim_sample;
+ struct ionic_lif *lif;
+ unsigned int qi;
+ u64 pkts, bytes;
+
+ if (!qcq->intr.dim_coal_hw)
+ return;
+
+ lif = qcq->q.lif;
+ qi = qcq->cq.bound_q->index;
+
+ switch (napi_mode) {
+ case IONIC_LIF_F_TX_DIM_INTR:
+ pkts = lif->txqstats[qi].pkts;
+ bytes = lif->txqstats[qi].bytes;
+ break;
+ case IONIC_LIF_F_RX_DIM_INTR:
+ pkts = lif->rxqstats[qi].pkts;
+ bytes = lif->rxqstats[qi].bytes;
+ break;
+ default:
+ pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts;
+ bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes;
+ break;
+ }
+
+ dim_update_sample(qcq->cq.bound_intr->rearm_count,
+ pkts, bytes, &dim_sample);
+
+ net_dim(&qcq->dim, dim_sample);
+}
+
+int ionic_tx_napi(struct napi_struct *napi, int budget)
+{
+ struct ionic_qcq *qcq = napi_to_qcq(napi);
+ struct ionic_cq *cq = napi_to_cq(napi);
+ struct ionic_dev *idev;
+ struct ionic_lif *lif;
+ u32 work_done = 0;
+ u32 flags = 0;
+
+ lif = cq->bound_q->lif;
+ idev = &lif->ionic->idev;
+
+ work_done = ionic_cq_service(cq, budget,
+ ionic_tx_service, NULL, NULL);
+
+ if (work_done < budget && napi_complete_done(napi, work_done)) {
+ ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR);
+ flags |= IONIC_INTR_CRED_UNMASK;
+ cq->bound_intr->rearm_count++;
+ }
+
+ if (work_done || flags) {
+ flags |= IONIC_INTR_CRED_RESET_COALESCE;
+ ionic_intr_credits(idev->intr_ctrl,
+ cq->bound_intr->index,
+ work_done, flags);
+ }
+
+ if (!work_done && ionic_txq_poke_doorbell(&qcq->q))
+ mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
+
+ return work_done;
+}
+
+int ionic_rx_napi(struct napi_struct *napi, int budget)
+{
+ struct ionic_qcq *qcq = napi_to_qcq(napi);
+ struct ionic_cq *cq = napi_to_cq(napi);
+ struct ionic_dev *idev;
+ struct ionic_lif *lif;
+ u32 work_done = 0;
+ u32 flags = 0;
+
+ lif = cq->bound_q->lif;
+ idev = &lif->ionic->idev;
+
+ work_done = ionic_cq_service(cq, budget,
+ ionic_rx_service, NULL, NULL);
+
+ ionic_rx_fill(cq->bound_q);
+
+ if (work_done < budget && napi_complete_done(napi, work_done)) {
+ ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR);
+ flags |= IONIC_INTR_CRED_UNMASK;
+ cq->bound_intr->rearm_count++;
+ }
+
+ if (work_done || flags) {
+ flags |= IONIC_INTR_CRED_RESET_COALESCE;
+ ionic_intr_credits(idev->intr_ctrl,
+ cq->bound_intr->index,
+ work_done, flags);
+ }
+
+ if (!work_done && ionic_rxq_poke_doorbell(&qcq->q))
+ mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
+
+ return work_done;
+}
+
+int ionic_txrx_napi(struct napi_struct *napi, int budget)
+{
+ struct ionic_qcq *rxqcq = napi_to_qcq(napi);
+ struct ionic_cq *rxcq = napi_to_cq(napi);
+ unsigned int qi = rxcq->bound_q->index;
+ struct ionic_qcq *txqcq;
+ struct ionic_dev *idev;
+ struct ionic_lif *lif;
+ struct ionic_cq *txcq;
+ bool resched = false;
+ u32 rx_work_done = 0;
+ u32 tx_work_done = 0;
+ u32 flags = 0;
+
+ lif = rxcq->bound_q->lif;
+ idev = &lif->ionic->idev;
+ txqcq = lif->txqcqs[qi];
+ txcq = &lif->txqcqs[qi]->cq;
+
+ tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT,
+ ionic_tx_service, NULL, NULL);
+
+ rx_work_done = ionic_cq_service(rxcq, budget,
+ ionic_rx_service, NULL, NULL);
+
+ ionic_rx_fill(rxcq->bound_q);
+
+ if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {
+ ionic_dim_update(rxqcq, 0);
+ flags |= IONIC_INTR_CRED_UNMASK;
+ rxcq->bound_intr->rearm_count++;
+ }
+
+ if (rx_work_done || flags) {
+ flags |= IONIC_INTR_CRED_RESET_COALESCE;
+ ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index,
+ tx_work_done + rx_work_done, flags);
+ }
+
+ if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q))
+ resched = true;
+ if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q))
+ resched = true;
+ if (resched)
+ mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
+
+ return rx_work_done;
+}
+
+static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
+ void *data, size_t len)
+{
+ struct ionic_tx_stats *stats = q_to_tx_stats(q);
+ struct device *dev = q->dev;
+ dma_addr_t dma_addr;
+
+ dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
+ if (dma_mapping_error(dev, dma_addr)) {
+ net_warn_ratelimited("%s: DMA single map failed on %s!\n",
+ q->lif->netdev->name, q->name);
+ stats->dma_map_err++;
+ return 0;
+ }
+ return dma_addr;
+}
+
+static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
+ const skb_frag_t *frag,
+ size_t offset, size_t len)
+{
+ struct ionic_tx_stats *stats = q_to_tx_stats(q);
+ struct device *dev = q->dev;
+ dma_addr_t dma_addr;
+
+ dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
+ if (dma_mapping_error(dev, dma_addr)) {
+ net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
+ q->lif->netdev->name, q->name);
+ stats->dma_map_err++;
+ }
+ return dma_addr;
+}
+
+static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb,
+ struct ionic_desc_info *desc_info)
+{
+ struct ionic_buf_info *buf_info = desc_info->bufs;
+ struct ionic_tx_stats *stats = q_to_tx_stats(q);
+ struct device *dev = q->dev;
+ dma_addr_t dma_addr;
+ unsigned int nfrags;
+ skb_frag_t *frag;
+ int frag_idx;
+
+ dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
+ if (dma_mapping_error(dev, dma_addr)) {
+ stats->dma_map_err++;
+ return -EIO;
+ }
+ buf_info->dma_addr = dma_addr;
+ buf_info->len = skb_headlen(skb);
+ buf_info++;
+
+ frag = skb_shinfo(skb)->frags;
+ nfrags = skb_shinfo(skb)->nr_frags;
+ for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) {
+ dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
+ if (dma_mapping_error(dev, dma_addr)) {
+ stats->dma_map_err++;
+ goto dma_fail;
+ }
+ buf_info->dma_addr = dma_addr;
+ buf_info->len = skb_frag_size(frag);
+ buf_info++;
+ }
+
+ desc_info->nbufs = 1 + nfrags;
+
+ return 0;
+
+dma_fail:
+ /* unwind the frag mappings and the head mapping */
+ while (frag_idx > 0) {
+ frag_idx--;
+ buf_info--;
+ dma_unmap_page(dev, buf_info->dma_addr,
+ buf_info->len, DMA_TO_DEVICE);
+ }
+ dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE);
+ return -EIO;
+}
+
+static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
+ struct ionic_desc_info *desc_info)
+{
+ struct ionic_buf_info *buf_info = desc_info->bufs;
+ struct device *dev = q->dev;
+ unsigned int i;
+
+ if (!desc_info->nbufs)
+ return;
+
+ dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr,
+ buf_info->len, DMA_TO_DEVICE);
+ buf_info++;
+ for (i = 1; i < desc_info->nbufs; i++, buf_info++)
+ dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr,
+ buf_info->len, DMA_TO_DEVICE);
+
+ desc_info->nbufs = 0;
+}
+
+static void ionic_tx_clean(struct ionic_queue *q,
+ struct ionic_desc_info *desc_info,
+ struct ionic_cq_info *cq_info,
+ void *cb_arg)
+{
+ struct ionic_tx_stats *stats = q_to_tx_stats(q);
+ struct ionic_qcq *qcq = q_to_qcq(q);
+ struct sk_buff *skb = cb_arg;
+ u16 qi;
+
+ ionic_tx_desc_unmap_bufs(q, desc_info);
+
+ if (!skb)
+ return;
+
+ qi = skb_get_queue_mapping(skb);
+
+ if (unlikely(q->features & IONIC_TXQ_F_HWSTAMP)) {
+ if (cq_info) {
+ struct skb_shared_hwtstamps hwts = {};
+ __le64 *cq_desc_hwstamp;
+ u64 hwstamp;
+
+ cq_desc_hwstamp =
+ cq_info->cq_desc +
+ qcq->cq.desc_size -
+ sizeof(struct ionic_txq_comp) -
+ IONIC_HWSTAMP_CQ_NEGOFFSET;
+
+ hwstamp = le64_to_cpu(*cq_desc_hwstamp);
+
+ if (hwstamp != IONIC_HWSTAMP_INVALID) {
+ hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp);
+
+ skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+ skb_tstamp_tx(skb, &hwts);
+
+ stats->hwstamp_valid++;
+ } else {
+ stats->hwstamp_invalid++;
+ }
+ }
+
+ } else if (unlikely(__netif_subqueue_stopped(q->lif->netdev, qi))) {
+ netif_wake_subqueue(q->lif->netdev, qi);
+ }
+
+ desc_info->bytes = skb->len;
+ stats->clean++;
+
+ dev_consume_skb_any(skb);
+}
+
+bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
+{
+ struct ionic_queue *q = cq->bound_q;
+ struct ionic_desc_info *desc_info;
+ struct ionic_txq_comp *comp;
+ int bytes = 0;
+ int pkts = 0;
+ u16 index;
+
+ comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
+
+ if (!color_match(comp->color, cq->done_color))
+ return false;
+
+ /* clean the related q entries, there could be
+ * several q entries completed for each cq completion
+ */
+ do {
+ desc_info = &q->info[q->tail_idx];
+ desc_info->bytes = 0;
+ index = q->tail_idx;
+ q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
+ ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg);
+ if (desc_info->cb_arg) {
+ pkts++;
+ bytes += desc_info->bytes;
+ }
+ desc_info->cb = NULL;
+ desc_info->cb_arg = NULL;
+ } while (index != le16_to_cpu(comp->comp_index));
+
+ if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
+ netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);
+
+ return true;
+}
+
+void ionic_tx_flush(struct ionic_cq *cq)
+{
+ struct ionic_dev *idev = &cq->lif->ionic->idev;
+ u32 work_done;
+
+ work_done = ionic_cq_service(cq, cq->num_descs,
+ ionic_tx_service, NULL, NULL);
+ if (work_done)
+ ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
+ work_done, IONIC_INTR_CRED_RESET_COALESCE);
+}
+
+void ionic_tx_empty(struct ionic_queue *q)
+{
+ struct ionic_desc_info *desc_info;
+ int bytes = 0;
+ int pkts = 0;
+
+ /* walk the not completed tx entries, if any */
+ while (q->head_idx != q->tail_idx) {
+ desc_info = &q->info[q->tail_idx];
+ desc_info->bytes = 0;
+ q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
+ ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg);
+ if (desc_info->cb_arg) {
+ pkts++;
+ bytes += desc_info->bytes;
+ }
+ desc_info->cb = NULL;
+ desc_info->cb_arg = NULL;
+ }
+
+ if (pkts && bytes && !unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
+ netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);
+}
+
+static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
+{
+ int err;
+
+ err = skb_cow_head(skb, 0);
+ if (err)
+ return err;
+
+ if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
+ inner_ip_hdr(skb)->check = 0;
+ inner_tcp_hdr(skb)->check =
+ ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
+ inner_ip_hdr(skb)->daddr,
+ 0, IPPROTO_TCP, 0);
+ } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
+ inner_tcp_hdr(skb)->check =
+ ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
+ &inner_ipv6_hdr(skb)->daddr,
+ 0, IPPROTO_TCP, 0);
+ }
+
+ return 0;
+}
+
+static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
+{
+ int err;
+
+ err = skb_cow_head(skb, 0);
+ if (err)
+ return err;
+
+ if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
+ ip_hdr(skb)->check = 0;
+ tcp_hdr(skb)->check =
+ ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
+ ip_hdr(skb)->daddr,
+ 0, IPPROTO_TCP, 0);
+ } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
+ tcp_v6_gso_csum_prep(skb);
+ }
+
+ return 0;
+}
+
+static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
+ struct sk_buff *skb,
+ dma_addr_t addr, u8 nsge, u16 len,
+ unsigned int hdrlen, unsigned int mss,
+ bool outer_csum,
+ u16 vlan_tci, bool has_vlan,
+ bool start, bool done)
+{
+ u8 flags = 0;
+ u64 cmd;
+
+ flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
+ flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
+ flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
+ flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
+
+ cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr);
+ desc->cmd = cpu_to_le64(cmd);
+ desc->len = cpu_to_le16(len);
+ desc->vlan_tci = cpu_to_le16(vlan_tci);
+ desc->hdr_len = cpu_to_le16(hdrlen);
+ desc->mss = cpu_to_le16(mss);
+
+ if (start) {
+ skb_tx_timestamp(skb);
+ if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
+ netdev_tx_sent_queue(q_to_ndq(q), skb->len);
+ ionic_txq_post(q, false, ionic_tx_clean, skb);
+ } else {
+ ionic_txq_post(q, done, NULL, NULL);
+ }
+}
+
+static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
+{
+ struct ionic_tx_stats *stats = q_to_tx_stats(q);
+ struct ionic_desc_info *desc_info;
+ struct ionic_buf_info *buf_info;
+ struct ionic_txq_sg_elem *elem;
+ struct ionic_txq_desc *desc;
+ unsigned int chunk_len;
+ unsigned int frag_rem;
+ unsigned int tso_rem;
+ unsigned int seg_rem;
+ dma_addr_t desc_addr;
+ dma_addr_t frag_addr;
+ unsigned int hdrlen;
+ unsigned int len;
+ unsigned int mss;
+ bool start, done;
+ bool outer_csum;
+ bool has_vlan;
+ u16 desc_len;
+ u8 desc_nsge;
+ u16 vlan_tci;
+ bool encap;
+ int err;
+
+ desc_info = &q->info[q->head_idx];
+ buf_info = desc_info->bufs;
+
+ if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
+ return -EIO;
+
+ len = skb->len;
+ mss = skb_shinfo(skb)->gso_size;
+ outer_csum = (skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM) ||
+ (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
+ has_vlan = !!skb_vlan_tag_present(skb);
+ vlan_tci = skb_vlan_tag_get(skb);
+ encap = skb->encapsulation;
+
+ /* Preload inner-most TCP csum field with IP pseudo hdr
+ * calculated with IP length set to zero. HW will later
+ * add in length to each TCP segment resulting from the TSO.
+ */
+
+ if (encap)
+ err = ionic_tx_tcp_inner_pseudo_csum(skb);
+ else
+ err = ionic_tx_tcp_pseudo_csum(skb);
+ if (err) {
+ /* clean up mapping from ionic_tx_map_skb */
+ ionic_tx_desc_unmap_bufs(q, desc_info);
+ return err;
+ }
+
+ if (encap)
+ hdrlen = skb_inner_tcp_all_headers(skb);
+ else
+ hdrlen = skb_tcp_all_headers(skb);
+
+ tso_rem = len;
+ seg_rem = min(tso_rem, hdrlen + mss);
+
+ frag_addr = 0;
+ frag_rem = 0;
+
+ start = true;
+
+ while (tso_rem > 0) {
+ desc = NULL;
+ elem = NULL;
+ desc_addr = 0;
+ desc_len = 0;
+ desc_nsge = 0;
+ /* use fragments until we have enough to post a single descriptor */
+ while (seg_rem > 0) {
+ /* if the fragment is exhausted then move to the next one */
+ if (frag_rem == 0) {
+ /* grab the next fragment */
+ frag_addr = buf_info->dma_addr;
+ frag_rem = buf_info->len;
+ buf_info++;
+ }
+ chunk_len = min(frag_rem, seg_rem);
+ if (!desc) {
+ /* fill main descriptor */
+ desc = desc_info->txq_desc;
+ elem = desc_info->txq_sg_desc->elems;
+ desc_addr = frag_addr;
+ desc_len = chunk_len;
+ } else {
+ /* fill sg descriptor */
+ elem->addr = cpu_to_le64(frag_addr);
+ elem->len = cpu_to_le16(chunk_len);
+ elem++;
+ desc_nsge++;
+ }
+ frag_addr += chunk_len;
+ frag_rem -= chunk_len;
+ tso_rem -= chunk_len;
+ seg_rem -= chunk_len;
+ }
+ seg_rem = min(tso_rem, mss);
+ done = (tso_rem == 0);
+ /* post descriptor */
+ ionic_tx_tso_post(q, desc, skb,
+ desc_addr, desc_nsge, desc_len,
+ hdrlen, mss, outer_csum, vlan_tci, has_vlan,
+ start, done);
+ start = false;
+ /* Buffer information is stored with the first tso descriptor */
+ desc_info = &q->info[q->head_idx];
+ desc_info->nbufs = 0;
+ }
+
+ stats->pkts += DIV_ROUND_UP(len - hdrlen, mss);
+ stats->bytes += len;
+ stats->tso++;
+ stats->tso_bytes = len;
+
+ return 0;
+}
+
+static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb,
+ struct ionic_desc_info *desc_info)
+{
+ struct ionic_txq_desc *desc = desc_info->txq_desc;
+ struct ionic_buf_info *buf_info = desc_info->bufs;
+ struct ionic_tx_stats *stats = q_to_tx_stats(q);
+ bool has_vlan;
+ u8 flags = 0;
+ bool encap;
+ u64 cmd;
+
+ has_vlan = !!skb_vlan_tag_present(skb);
+ encap = skb->encapsulation;
+
+ flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
+ flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
+
+ cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL,
+ flags, skb_shinfo(skb)->nr_frags,
+ buf_info->dma_addr);
+ desc->cmd = cpu_to_le64(cmd);
+ desc->len = cpu_to_le16(buf_info->len);
+ if (has_vlan) {
+ desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
+ stats->vlan_inserted++;
+ } else {
+ desc->vlan_tci = 0;
+ }
+ desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
+ desc->csum_offset = cpu_to_le16(skb->csum_offset);
+
+ if (skb_csum_is_sctp(skb))
+ stats->crc32_csum++;
+ else
+ stats->csum++;
+}
+
+static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb,
+ struct ionic_desc_info *desc_info)
+{
+ struct ionic_txq_desc *desc = desc_info->txq_desc;
+ struct ionic_buf_info *buf_info = desc_info->bufs;
+ struct ionic_tx_stats *stats = q_to_tx_stats(q);
+ bool has_vlan;
+ u8 flags = 0;
+ bool encap;
+ u64 cmd;
+
+ has_vlan = !!skb_vlan_tag_present(skb);
+ encap = skb->encapsulation;
+
+ flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
+ flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
+
+ cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
+ flags, skb_shinfo(skb)->nr_frags,
+ buf_info->dma_addr);
+ desc->cmd = cpu_to_le64(cmd);
+ desc->len = cpu_to_le16(buf_info->len);
+ if (has_vlan) {
+ desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
+ stats->vlan_inserted++;
+ } else {
+ desc->vlan_tci = 0;
+ }
+ desc->csum_start = 0;
+ desc->csum_offset = 0;
+
+ stats->csum_none++;
+}
+
+static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb,
+ struct ionic_desc_info *desc_info)
+{
+ struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc;
+ struct ionic_buf_info *buf_info = &desc_info->bufs[1];
+ struct ionic_txq_sg_elem *elem = sg_desc->elems;
+ struct ionic_tx_stats *stats = q_to_tx_stats(q);
+ unsigned int i;
+
+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) {
+ elem->addr = cpu_to_le64(buf_info->dma_addr);
+ elem->len = cpu_to_le16(buf_info->len);
+ }
+
+ stats->frags += skb_shinfo(skb)->nr_frags;
+}
+
+static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb)
+{
+ struct ionic_desc_info *desc_info = &q->info[q->head_idx];
+ struct ionic_tx_stats *stats = q_to_tx_stats(q);
+
+ if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
+ return -EIO;
+
+ /* set up the initial descriptor */
+ if (skb->ip_summed == CHECKSUM_PARTIAL)
+ ionic_tx_calc_csum(q, skb, desc_info);
+ else
+ ionic_tx_calc_no_csum(q, skb, desc_info);
+
+ /* add frags */
+ ionic_tx_skb_frags(q, skb, desc_info);
+
+ skb_tx_timestamp(skb);
+ stats->pkts++;
+ stats->bytes += skb->len;
+
+ if (!unlikely(q->features & IONIC_TXQ_F_HWSTAMP))
+ netdev_tx_sent_queue(q_to_ndq(q), skb->len);
+ ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
+
+ return 0;
+}
+
+static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
+{
+ struct ionic_tx_stats *stats = q_to_tx_stats(q);
+ int ndescs;
+ int err;
+
+ /* Each desc is mss long max, so a descriptor for each gso_seg */
+ if (skb_is_gso(skb))
+ ndescs = skb_shinfo(skb)->gso_segs;
+ else
+ ndescs = 1;
+
+ /* If non-TSO, just need 1 desc and nr_frags sg elems */
+ if (skb_shinfo(skb)->nr_frags <= q->max_sg_elems)
+ return ndescs;
+
+ /* Too many frags, so linearize */
+ err = skb_linearize(skb);
+ if (err)
+ return err;
+
+ stats->linearize++;
+
+ return ndescs;
+}
+
+static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs)
+{
+ int stopped = 0;
+
+ if (unlikely(!ionic_q_has_space(q, ndescs))) {
+ netif_stop_subqueue(q->lif->netdev, q->index);
+ stopped = 1;
+
+ /* Might race with ionic_tx_clean, check again */
+ smp_rmb();
+ if (ionic_q_has_space(q, ndescs)) {
+ netif_wake_subqueue(q->lif->netdev, q->index);
+ stopped = 0;
+ }
+ }
+
+ return stopped;
+}
+
+static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb,
+ struct net_device *netdev)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_queue *q = &lif->hwstamp_txq->q;
+ int err, ndescs;
+
+ /* Does not stop/start txq, because we post to a separate tx queue
+ * for timestamping, and if a packet can't be posted immediately to
+ * the timestamping queue, it is dropped.
+ */
+
+ ndescs = ionic_tx_descs_needed(q, skb);
+ if (unlikely(ndescs < 0))
+ goto err_out_drop;
+
+ if (unlikely(!ionic_q_has_space(q, ndescs)))
+ goto err_out_drop;
+
+ skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP;
+ if (skb_is_gso(skb))
+ err = ionic_tx_tso(q, skb);
+ else
+ err = ionic_tx(q, skb);
+
+ if (err)
+ goto err_out_drop;
+
+ return NETDEV_TX_OK;
+
+err_out_drop:
+ q->drop++;
+ dev_kfree_skb(skb);
+ return NETDEV_TX_OK;
+}
+
+netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
+{
+ u16 queue_index = skb_get_queue_mapping(skb);
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_queue *q;
+ int ndescs;
+ int err;
+
+ if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) {
+ dev_kfree_skb(skb);
+ return NETDEV_TX_OK;
+ }
+
+ if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
+ if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode)
+ return ionic_start_hwstamp_xmit(skb, netdev);
+
+ if (unlikely(queue_index >= lif->nxqs))
+ queue_index = 0;
+ q = &lif->txqcqs[queue_index]->q;
+
+ ndescs = ionic_tx_descs_needed(q, skb);
+ if (ndescs < 0)
+ goto err_out_drop;
+
+ if (unlikely(ionic_maybe_stop_tx(q, ndescs)))
+ return NETDEV_TX_BUSY;
+
+ if (skb_is_gso(skb))
+ err = ionic_tx_tso(q, skb);
+ else
+ err = ionic_tx(q, skb);
+
+ if (err)
+ goto err_out_drop;
+
+ /* Stop the queue if there aren't descriptors for the next packet.
+ * Since our SG lists per descriptor take care of most of the possible
+ * fragmentation, we don't need to have many descriptors available.
+ */
+ ionic_maybe_stop_tx(q, 4);
+
+ return NETDEV_TX_OK;
+
+err_out_drop:
+ q->drop++;
+ dev_kfree_skb(skb);
+ return NETDEV_TX_OK;
+}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.h b/drivers/net/ethernet/pensando/ionic/ionic_txrx.h
new file mode 100644
index 000000000..d7cbaad8a
--- /dev/null
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
+
+#ifndef _IONIC_TXRX_H_
+#define _IONIC_TXRX_H_
+
+void ionic_tx_flush(struct ionic_cq *cq);
+
+void ionic_rx_fill(struct ionic_queue *q);
+void ionic_rx_empty(struct ionic_queue *q);
+void ionic_tx_empty(struct ionic_queue *q);
+int ionic_rx_napi(struct napi_struct *napi, int budget);
+int ionic_tx_napi(struct napi_struct *napi, int budget);
+int ionic_txrx_napi(struct napi_struct *napi, int budget);
+netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev);
+
+bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info);
+bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info);
+
+#endif /* _IONIC_TXRX_H_ */