diff options
Diffstat (limited to 'drivers/net/ethernet/chelsio')
136 files changed, 115832 insertions, 0 deletions
diff --git a/drivers/net/ethernet/chelsio/Kconfig b/drivers/net/ethernet/chelsio/Kconfig new file mode 100644 index 000000000..c931ec8ca --- /dev/null +++ b/drivers/net/ethernet/chelsio/Kconfig @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Chelsio device configuration +# + +config NET_VENDOR_CHELSIO + bool "Chelsio devices" + default y + depends on PCI + help + If you have a network (Ethernet) card belonging to this class, say Y. + + Note that the answer to this question doesn't directly affect the + kernel: saying N will just cause the configurator to skip all + the questions about Chelsio devices. If you say Y, you will be asked for + your specific card in the following questions. + +if NET_VENDOR_CHELSIO + +config CHELSIO_T1 + tristate "Chelsio 10Gb Ethernet support" + depends on PCI + select CRC32 + select MDIO + help + This driver supports Chelsio gigabit and 10-gigabit + Ethernet cards. More information about adapter features and + performance tuning is in + <file:Documentation/networking/device_drivers/ethernet/chelsio/cxgb.rst>. + + For general information about Chelsio and our products, visit + our website at <http://www.chelsio.com>. + + For customer support, please visit our customer support page at + <http://www.chelsio.com/support.html>. + + Please send feedback to <linux-bugs@chelsio.com>. + + To compile this driver as a module, choose M here: the module + will be called cxgb. + +config CHELSIO_T1_1G + bool "Chelsio gigabit Ethernet support" + depends on CHELSIO_T1 + help + Enables support for Chelsio's gigabit Ethernet PCI cards. If you + are using only 10G cards say 'N' here. + +config CHELSIO_T3 + tristate "Chelsio Communications T3 10Gb Ethernet support" + depends on PCI && INET + select FW_LOADER + select MDIO + help + This driver supports Chelsio T3-based gigabit and 10Gb Ethernet + adapters. + + For general information about Chelsio and our products, visit + our website at <http://www.chelsio.com>. + + For customer support, please visit our customer support page at + <http://www.chelsio.com/support.html>. + + Please send feedback to <linux-bugs@chelsio.com>. + + To compile this driver as a module, choose M here: the module + will be called cxgb3. + +config CHELSIO_T4 + tristate "Chelsio Communications T4/T5/T6 Ethernet support" + depends on PCI && (IPV6 || IPV6=n) && (TLS || TLS=n) + depends on PTP_1588_CLOCK_OPTIONAL + select FW_LOADER + select MDIO + select ZLIB_DEFLATE + help + This driver supports Chelsio T4, T5 & T6 based gigabit, 10Gb Ethernet + adapter and T5/T6 based 40Gb and T6 based 25Gb, 50Gb and 100Gb + Ethernet adapters. + + For general information about Chelsio and our products, visit + our website at <http://www.chelsio.com>. + + For customer support, please visit our customer support page at + <http://www.chelsio.com/support.html>. + + Please send feedback to <linux-bugs@chelsio.com>. + + To compile this driver as a module choose M here; the module + will be called cxgb4. + +config CHELSIO_T4_DCB + bool "Data Center Bridging (DCB) Support for Chelsio T4/T5/T6 cards" + default n + depends on CHELSIO_T4 && DCB + help + Enable DCB support through rtNetlink interface. + Say Y here if you want to enable Data Center Bridging (DCB) support + in the driver. + + If unsure, say N. + +config CHELSIO_T4_FCOE + bool "Fibre Channel over Ethernet (FCoE) Support for Chelsio T5 cards" + default n + depends on CHELSIO_T4 && CHELSIO_T4_DCB && FCOE + help + Enable FCoE offload features. + Say Y here if you want to enable Fibre Channel over Ethernet (FCoE) support + in the driver. + + If unsure, say N. + +config CHELSIO_T4VF + tristate "Chelsio Communications T4/T5/T6 Virtual Function Ethernet support" + depends on PCI + help + This driver supports Chelsio T4, T5 & T6 based gigabit, 10Gb Ethernet + adapters and T5/T6 based 40Gb and T6 based 25Gb, 50Gb and 100Gb + Ethernet adapters with PCI-E SR-IOV Virtual Functions. + + For general information about Chelsio and our products, visit + our website at <http://www.chelsio.com>. + + For customer support, please visit our customer support page at + <http://www.chelsio.com/support.html>. + + Please send feedback to <linux-bugs@chelsio.com>. + + To compile this driver as a module choose M here; the module + will be called cxgb4vf. + +config CHELSIO_LIB + tristate + help + Common library for Chelsio drivers. + +source "drivers/net/ethernet/chelsio/inline_crypto/Kconfig" + +endif # NET_VENDOR_CHELSIO diff --git a/drivers/net/ethernet/chelsio/Makefile b/drivers/net/ethernet/chelsio/Makefile new file mode 100644 index 000000000..1a6fd8b2b --- /dev/null +++ b/drivers/net/ethernet/chelsio/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Chelsio network device drivers. +# + +obj-$(CONFIG_CHELSIO_T1) += cxgb/ +obj-$(CONFIG_CHELSIO_T3) += cxgb3/ +obj-$(CONFIG_CHELSIO_T4) += cxgb4/ +obj-$(CONFIG_CHELSIO_T4VF) += cxgb4vf/ +obj-$(CONFIG_CHELSIO_LIB) += libcxgb/ +obj-$(CONFIG_CHELSIO_INLINE_CRYPTO) += inline_crypto/ diff --git a/drivers/net/ethernet/chelsio/cxgb/Makefile b/drivers/net/ethernet/chelsio/cxgb/Makefile new file mode 100644 index 000000000..8008282a2 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Chelsio T1 driver +# + +obj-$(CONFIG_CHELSIO_T1) += cxgb.o + +cxgb-$(CONFIG_CHELSIO_T1_1G) += mv88e1xxx.o vsc7326.o +cxgb-objs := cxgb2.o espi.o tp.o pm3393.o sge.o subr.o \ + mv88x201x.o my3126.o $(cxgb-y) diff --git a/drivers/net/ethernet/chelsio/cxgb/common.h b/drivers/net/ethernet/chelsio/cxgb/common.h new file mode 100644 index 000000000..e56eff701 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/common.h @@ -0,0 +1,343 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/***************************************************************************** + * * + * File: common.h * + * $Revision: 1.21 $ * + * $Date: 2005/06/22 00:43:25 $ * + * Description: * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#define pr_fmt(fmt) "cxgb: " fmt + +#ifndef _CXGB_COMMON_H_ +#define _CXGB_COMMON_H_ + +#include <linux/module.h> +#include <linux/netdevice.h> +#include <linux/types.h> +#include <linux/delay.h> +#include <linux/pci.h> +#include <linux/ethtool.h> +#include <linux/if_vlan.h> +#include <linux/mdio.h> +#include <linux/crc32.h> +#include <linux/slab.h> +#include <asm/io.h> +#include <linux/pci_ids.h> + +#define DRV_DESCRIPTION "Chelsio 10Gb Ethernet Driver" +#define DRV_NAME "cxgb" + +#define CH_DEVICE(devid, ssid, idx) \ + { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, ssid, 0, 0, idx } + +#define SUPPORTED_PAUSE (1 << 13) +#define SUPPORTED_LOOPBACK (1 << 15) + +#define ADVERTISED_PAUSE (1 << 13) +#define ADVERTISED_ASYM_PAUSE (1 << 14) + +typedef struct adapter adapter_t; + +struct t1_rx_mode { + struct net_device *dev; +}; + +#define t1_rx_mode_promisc(rm) (rm->dev->flags & IFF_PROMISC) +#define t1_rx_mode_allmulti(rm) (rm->dev->flags & IFF_ALLMULTI) +#define t1_rx_mode_mc_cnt(rm) (netdev_mc_count(rm->dev)) +#define t1_get_netdev(rm) (rm->dev) + +#define MAX_NPORTS 4 +#define PORT_MASK ((1 << MAX_NPORTS) - 1) +#define NMTUS 8 +#define TCB_SIZE 128 + +#define SPEED_INVALID 0xffff +#define DUPLEX_INVALID 0xff + +/* Max frame size PM3393 can handle. Includes Ethernet header and CRC. */ +#define PM3393_MAX_FRAME_SIZE 9600 + +#define VSC7326_MAX_MTU 9600 + +enum { + CHBT_BOARD_N110, + CHBT_BOARD_N210, + CHBT_BOARD_7500, + CHBT_BOARD_8000, + CHBT_BOARD_CHT101, + CHBT_BOARD_CHT110, + CHBT_BOARD_CHT210, + CHBT_BOARD_CHT204, + CHBT_BOARD_CHT204V, + CHBT_BOARD_CHT204E, + CHBT_BOARD_CHN204, + CHBT_BOARD_COUGAR, + CHBT_BOARD_6800, + CHBT_BOARD_SIMUL, +}; + +enum { + CHBT_TERM_FPGA, + CHBT_TERM_T1, + CHBT_TERM_T2, + CHBT_TERM_T3 +}; + +enum { + CHBT_MAC_CHELSIO_A, + CHBT_MAC_IXF1010, + CHBT_MAC_PM3393, + CHBT_MAC_VSC7321, + CHBT_MAC_DUMMY +}; + +enum { + CHBT_PHY_88E1041, + CHBT_PHY_88E1111, + CHBT_PHY_88X2010, + CHBT_PHY_XPAK, + CHBT_PHY_MY3126, + CHBT_PHY_8244, + CHBT_PHY_DUMMY +}; + +enum { + PAUSE_RX = 1 << 0, + PAUSE_TX = 1 << 1, + PAUSE_AUTONEG = 1 << 2 +}; + +/* Revisions of T1 chip */ +enum { + TERM_T1A = 0, + TERM_T1B = 1, + TERM_T2 = 3 +}; + +struct sge_params { + unsigned int cmdQ_size[2]; + unsigned int freelQ_size[2]; + unsigned int large_buf_capacity; + unsigned int rx_coalesce_usecs; + unsigned int last_rx_coalesce_raw; + unsigned int default_rx_coalesce_usecs; + unsigned int sample_interval_usecs; + unsigned int coalesce_enable; + unsigned int polling; +}; + +struct chelsio_pci_params { + unsigned short speed; + unsigned char width; + unsigned char is_pcix; +}; + +struct tp_params { + unsigned int pm_size; + unsigned int cm_size; + unsigned int pm_rx_base; + unsigned int pm_tx_base; + unsigned int pm_rx_pg_size; + unsigned int pm_tx_pg_size; + unsigned int pm_rx_num_pgs; + unsigned int pm_tx_num_pgs; + unsigned int rx_coalescing_size; + unsigned int use_5tuple_mode; +}; + +struct mc5_params { + unsigned int mode; /* selects MC5 width */ + unsigned int nservers; /* size of server region */ + unsigned int nroutes; /* size of routing region */ +}; + +/* Default MC5 region sizes */ +#define DEFAULT_SERVER_REGION_LEN 256 +#define DEFAULT_RT_REGION_LEN 1024 + +struct adapter_params { + struct sge_params sge; + struct mc5_params mc5; + struct tp_params tp; + struct chelsio_pci_params pci; + + const struct board_info *brd_info; + + unsigned short mtus[NMTUS]; + unsigned int nports; /* # of ethernet ports */ + unsigned int stats_update_period; + unsigned short chip_revision; + unsigned char chip_version; + unsigned char is_asic; + unsigned char has_msi; +}; + +struct link_config { + unsigned int supported; /* link capabilities */ + unsigned int advertising; /* advertised capabilities */ + unsigned short requested_speed; /* speed user has requested */ + unsigned short speed; /* actual link speed */ + unsigned char requested_duplex; /* duplex user has requested */ + unsigned char duplex; /* actual link duplex */ + unsigned char requested_fc; /* flow control user has requested */ + unsigned char fc; /* actual link flow control */ + unsigned char autoneg; /* autonegotiating? */ +}; + +struct cmac; +struct cphy; + +struct port_info { + struct net_device *dev; + struct cmac *mac; + struct cphy *phy; + struct link_config link_config; +}; + +struct sge; +struct peespi; + +struct adapter { + u8 __iomem *regs; + struct pci_dev *pdev; + unsigned long registered_device_map; + unsigned long open_device_map; + unsigned long flags; + + const char *name; + int msg_enable; + u32 mmio_len; + + struct adapter_params params; + + /* Terminator modules. */ + struct sge *sge; + struct peespi *espi; + struct petp *tp; + + struct napi_struct napi; + struct port_info port[MAX_NPORTS]; + struct delayed_work stats_update_task; + struct timer_list stats_update_timer; + + spinlock_t tpi_lock; + spinlock_t work_lock; + spinlock_t mac_lock; + + /* guards async operations */ + spinlock_t async_lock ____cacheline_aligned; + u32 pending_thread_intr; + u32 slow_intr_mask; + int t1powersave; +}; + +enum { /* adapter flags */ + FULL_INIT_DONE = 1 << 0, +}; + +struct mdio_ops; +struct gmac; +struct gphy; + +struct board_info { + unsigned char board; + unsigned char port_number; + unsigned long caps; + unsigned char chip_term; + unsigned char chip_mac; + unsigned char chip_phy; + unsigned int clock_core; + unsigned int clock_mc3; + unsigned int clock_mc4; + unsigned int espi_nports; + unsigned int clock_elmer0; + unsigned char mdio_mdien; + unsigned char mdio_mdiinv; + unsigned char mdio_mdc; + unsigned char mdio_phybaseaddr; + const struct gmac *gmac; + const struct gphy *gphy; + const struct mdio_ops *mdio_ops; + const char *desc; +}; + +static inline int t1_is_asic(const adapter_t *adapter) +{ + return adapter->params.is_asic; +} + +extern const struct pci_device_id t1_pci_tbl[]; + +static inline int adapter_matches_type(const adapter_t *adapter, + int version, int revision) +{ + return adapter->params.chip_version == version && + adapter->params.chip_revision == revision; +} + +#define t1_is_T1B(adap) adapter_matches_type(adap, CHBT_TERM_T1, TERM_T1B) +#define is_T2(adap) adapter_matches_type(adap, CHBT_TERM_T2, TERM_T2) + +/* Returns true if an adapter supports VLAN acceleration and TSO */ +static inline int vlan_tso_capable(const adapter_t *adapter) +{ + return !t1_is_T1B(adapter); +} + +#define for_each_port(adapter, iter) \ + for (iter = 0; iter < (adapter)->params.nports; ++iter) + +#define board_info(adapter) ((adapter)->params.brd_info) +#define is_10G(adapter) (board_info(adapter)->caps & SUPPORTED_10000baseT_Full) + +static inline unsigned int core_ticks_per_usec(const adapter_t *adap) +{ + return board_info(adap)->clock_core / 1000000; +} + +int __t1_tpi_read(adapter_t *adapter, u32 addr, u32 *valp); +int __t1_tpi_write(adapter_t *adapter, u32 addr, u32 value); +int t1_tpi_write(adapter_t *adapter, u32 addr, u32 value); +int t1_tpi_read(adapter_t *adapter, u32 addr, u32 *value); + +void t1_interrupts_enable(adapter_t *adapter); +void t1_interrupts_disable(adapter_t *adapter); +void t1_interrupts_clear(adapter_t *adapter); +int t1_elmer0_ext_intr_handler(adapter_t *adapter); +irqreturn_t t1_slow_intr_handler(adapter_t *adapter); + +int t1_link_start(struct cphy *phy, struct cmac *mac, struct link_config *lc); +const struct board_info *t1_get_board_info(unsigned int board_id); +const struct board_info *t1_get_board_info_from_ids(unsigned int devid, + unsigned short ssid); +int t1_seeprom_read(adapter_t *adapter, u32 addr, __le32 *data); +int t1_get_board_rev(adapter_t *adapter, const struct board_info *bi, + struct adapter_params *p); +int t1_init_hw_modules(adapter_t *adapter); +int t1_init_sw_modules(adapter_t *adapter, const struct board_info *bi); +void t1_free_sw_modules(adapter_t *adapter); +void t1_link_changed(adapter_t *adapter, int port_id); +void t1_link_negotiated(adapter_t *adapter, int port_id, int link_stat, + int speed, int duplex, int pause); +#endif /* _CXGB_COMMON_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb/cphy.h b/drivers/net/ethernet/chelsio/cxgb/cphy.h new file mode 100644 index 000000000..12639b688 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/cphy.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/***************************************************************************** + * * + * File: cphy.h * + * $Revision: 1.7 $ * + * $Date: 2005/06/21 18:29:47 $ * + * Description: * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#ifndef _CXGB_CPHY_H_ +#define _CXGB_CPHY_H_ + +#include "common.h" + +struct mdio_ops { + void (*init)(adapter_t *adapter, const struct board_info *bi); + int (*read)(struct net_device *dev, int phy_addr, int mmd_addr, + u16 reg_addr); + int (*write)(struct net_device *dev, int phy_addr, int mmd_addr, + u16 reg_addr, u16 val); + unsigned mode_support; +}; + +/* PHY interrupt types */ +enum { + cphy_cause_link_change = 0x1, + cphy_cause_error = 0x2, + cphy_cause_fifo_error = 0x3 +}; + +enum { + PHY_LINK_UP = 0x1, + PHY_AUTONEG_RDY = 0x2, + PHY_AUTONEG_EN = 0x4 +}; + +struct cphy; + +/* PHY operations */ +struct cphy_ops { + void (*destroy)(struct cphy *); + int (*reset)(struct cphy *, int wait); + + int (*interrupt_enable)(struct cphy *); + int (*interrupt_disable)(struct cphy *); + int (*interrupt_clear)(struct cphy *); + int (*interrupt_handler)(struct cphy *); + + int (*autoneg_enable)(struct cphy *); + int (*autoneg_disable)(struct cphy *); + int (*autoneg_restart)(struct cphy *); + + int (*advertise)(struct cphy *phy, unsigned int advertise_map); + int (*set_loopback)(struct cphy *, int on); + int (*set_speed_duplex)(struct cphy *phy, int speed, int duplex); + int (*get_link_status)(struct cphy *phy, int *link_ok, int *speed, + int *duplex, int *fc); + + u32 mmds; +}; + +/* A PHY instance */ +struct cphy { + int state; /* Link status state machine */ + adapter_t *adapter; /* associated adapter */ + + struct delayed_work phy_update; + + u16 bmsr; + int count; + int act_count; + int act_on; + + u32 elmer_gpo; + + const struct cphy_ops *ops; /* PHY operations */ + struct mdio_if_info mdio; + struct cphy_instance *instance; +}; + +/* Convenience MDIO read/write wrappers */ +static inline int cphy_mdio_read(struct cphy *cphy, int mmd, int reg, + unsigned int *valp) +{ + int rc = cphy->mdio.mdio_read(cphy->mdio.dev, cphy->mdio.prtad, mmd, + reg); + *valp = (rc >= 0) ? rc : -1; + return (rc >= 0) ? 0 : rc; +} + +static inline int cphy_mdio_write(struct cphy *cphy, int mmd, int reg, + unsigned int val) +{ + return cphy->mdio.mdio_write(cphy->mdio.dev, cphy->mdio.prtad, mmd, + reg, val); +} + +static inline int simple_mdio_read(struct cphy *cphy, int reg, + unsigned int *valp) +{ + return cphy_mdio_read(cphy, MDIO_DEVAD_NONE, reg, valp); +} + +static inline int simple_mdio_write(struct cphy *cphy, int reg, + unsigned int val) +{ + return cphy_mdio_write(cphy, MDIO_DEVAD_NONE, reg, val); +} + +/* Convenience initializer */ +static inline void cphy_init(struct cphy *phy, struct net_device *dev, + int phy_addr, const struct cphy_ops *phy_ops, + const struct mdio_ops *mdio_ops) +{ + struct adapter *adapter = netdev_priv(dev); + phy->adapter = adapter; + phy->ops = phy_ops; + if (mdio_ops) { + phy->mdio.prtad = phy_addr; + phy->mdio.mmds = phy_ops->mmds; + phy->mdio.mode_support = mdio_ops->mode_support; + phy->mdio.mdio_read = mdio_ops->read; + phy->mdio.mdio_write = mdio_ops->write; + } + phy->mdio.dev = dev; +} + +/* Operations of the PHY-instance factory */ +struct gphy { + /* Construct a PHY instance with the given PHY address */ + struct cphy *(*create)(struct net_device *dev, int phy_addr, + const struct mdio_ops *mdio_ops); + + /* + * Reset the PHY chip. This resets the whole PHY chip, not individual + * ports. + */ + int (*reset)(adapter_t *adapter); +}; + +extern const struct gphy t1_my3126_ops; +extern const struct gphy t1_mv88e1xxx_ops; +extern const struct gphy t1_vsc8244_ops; +extern const struct gphy t1_mv88x201x_ops; + +#endif /* _CXGB_CPHY_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb/cpl5_cmd.h b/drivers/net/ethernet/chelsio/cxgb/cpl5_cmd.h new file mode 100644 index 000000000..a30fb4071 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/cpl5_cmd.h @@ -0,0 +1,628 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/***************************************************************************** + * * + * File: cpl5_cmd.h * + * $Revision: 1.6 $ * + * $Date: 2005/06/21 18:29:47 $ * + * Description: * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#ifndef _CXGB_CPL5_CMD_H_ +#define _CXGB_CPL5_CMD_H_ + +#include <asm/byteorder.h> + +#if !defined(__LITTLE_ENDIAN_BITFIELD) && !defined(__BIG_ENDIAN_BITFIELD) +#error "Adjust your <asm/byteorder.h> defines" +#endif + +enum CPL_opcode { + CPL_PASS_OPEN_REQ = 0x1, + CPL_PASS_OPEN_RPL = 0x2, + CPL_PASS_ESTABLISH = 0x3, + CPL_PASS_ACCEPT_REQ = 0xE, + CPL_PASS_ACCEPT_RPL = 0x4, + CPL_ACT_OPEN_REQ = 0x5, + CPL_ACT_OPEN_RPL = 0x6, + CPL_CLOSE_CON_REQ = 0x7, + CPL_CLOSE_CON_RPL = 0x8, + CPL_CLOSE_LISTSRV_REQ = 0x9, + CPL_CLOSE_LISTSRV_RPL = 0xA, + CPL_ABORT_REQ = 0xB, + CPL_ABORT_RPL = 0xC, + CPL_PEER_CLOSE = 0xD, + CPL_ACT_ESTABLISH = 0x17, + + CPL_GET_TCB = 0x24, + CPL_GET_TCB_RPL = 0x25, + CPL_SET_TCB = 0x26, + CPL_SET_TCB_FIELD = 0x27, + CPL_SET_TCB_RPL = 0x28, + CPL_PCMD = 0x29, + + CPL_PCMD_READ = 0x31, + CPL_PCMD_READ_RPL = 0x32, + + + CPL_RX_DATA = 0xA0, + CPL_RX_DATA_DDP = 0xA1, + CPL_RX_DATA_ACK = 0xA3, + CPL_RX_PKT = 0xAD, + CPL_RX_ISCSI_HDR = 0xAF, + CPL_TX_DATA_ACK = 0xB0, + CPL_TX_DATA = 0xB1, + CPL_TX_PKT = 0xB2, + CPL_TX_PKT_LSO = 0xB6, + + CPL_RTE_DELETE_REQ = 0xC0, + CPL_RTE_DELETE_RPL = 0xC1, + CPL_RTE_WRITE_REQ = 0xC2, + CPL_RTE_WRITE_RPL = 0xD3, + CPL_RTE_READ_REQ = 0xC3, + CPL_RTE_READ_RPL = 0xC4, + CPL_L2T_WRITE_REQ = 0xC5, + CPL_L2T_WRITE_RPL = 0xD4, + CPL_L2T_READ_REQ = 0xC6, + CPL_L2T_READ_RPL = 0xC7, + CPL_SMT_WRITE_REQ = 0xC8, + CPL_SMT_WRITE_RPL = 0xD5, + CPL_SMT_READ_REQ = 0xC9, + CPL_SMT_READ_RPL = 0xCA, + CPL_ARP_MISS_REQ = 0xCD, + CPL_ARP_MISS_RPL = 0xCE, + CPL_MIGRATE_C2T_REQ = 0xDC, + CPL_MIGRATE_C2T_RPL = 0xDD, + CPL_ERROR = 0xD7, + + /* internal: driver -> TOM */ + CPL_MSS_CHANGE = 0xE1 +}; + +#define NUM_CPL_CMDS 256 + +enum CPL_error { + CPL_ERR_NONE = 0, + CPL_ERR_TCAM_PARITY = 1, + CPL_ERR_TCAM_FULL = 3, + CPL_ERR_CONN_RESET = 20, + CPL_ERR_CONN_EXIST = 22, + CPL_ERR_ARP_MISS = 23, + CPL_ERR_BAD_SYN = 24, + CPL_ERR_CONN_TIMEDOUT = 30, + CPL_ERR_XMIT_TIMEDOUT = 31, + CPL_ERR_PERSIST_TIMEDOUT = 32, + CPL_ERR_FINWAIT2_TIMEDOUT = 33, + CPL_ERR_KEEPALIVE_TIMEDOUT = 34, + CPL_ERR_ABORT_FAILED = 42, + CPL_ERR_GENERAL = 99 +}; + +enum { + CPL_CONN_POLICY_AUTO = 0, + CPL_CONN_POLICY_ASK = 1, + CPL_CONN_POLICY_DENY = 3 +}; + +enum { + ULP_MODE_NONE = 0, + ULP_MODE_TCPDDP = 1, + ULP_MODE_ISCSI = 2, + ULP_MODE_IWARP = 3, + ULP_MODE_SSL = 4 +}; + +enum { + CPL_PASS_OPEN_ACCEPT, + CPL_PASS_OPEN_REJECT +}; + +enum { + CPL_ABORT_SEND_RST = 0, + CPL_ABORT_NO_RST, + CPL_ABORT_POST_CLOSE_REQ = 2 +}; + +enum { // TX_PKT_LSO ethernet types + CPL_ETH_II, + CPL_ETH_II_VLAN, + CPL_ETH_802_3, + CPL_ETH_802_3_VLAN +}; + +union opcode_tid { + u32 opcode_tid; + u8 opcode; +}; + +#define S_OPCODE 24 +#define V_OPCODE(x) ((x) << S_OPCODE) +#define G_OPCODE(x) (((x) >> S_OPCODE) & 0xFF) +#define G_TID(x) ((x) & 0xFFFFFF) + +/* tid is assumed to be 24-bits */ +#define MK_OPCODE_TID(opcode, tid) (V_OPCODE(opcode) | (tid)) + +#define OPCODE_TID(cmd) ((cmd)->ot.opcode_tid) + +/* extract the TID from a CPL command */ +#define GET_TID(cmd) (G_TID(ntohl(OPCODE_TID(cmd)))) + +struct tcp_options { + u16 mss; + u8 wsf; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 rsvd:4; + u8 ecn:1; + u8 sack:1; + u8 tstamp:1; +#else + u8 tstamp:1; + u8 sack:1; + u8 ecn:1; + u8 rsvd:4; +#endif +}; + +struct cpl_pass_open_req { + union opcode_tid ot; + u16 local_port; + u16 peer_port; + u32 local_ip; + u32 peer_ip; + u32 opt0h; + u32 opt0l; + u32 peer_netmask; + u32 opt1; +}; + +struct cpl_pass_open_rpl { + union opcode_tid ot; + u16 local_port; + u16 peer_port; + u32 local_ip; + u32 peer_ip; + u8 resvd[7]; + u8 status; +}; + +struct cpl_pass_establish { + union opcode_tid ot; + u16 local_port; + u16 peer_port; + u32 local_ip; + u32 peer_ip; + u32 tos_tid; + u8 l2t_idx; + u8 rsvd[3]; + u32 snd_isn; + u32 rcv_isn; +}; + +struct cpl_pass_accept_req { + union opcode_tid ot; + u16 local_port; + u16 peer_port; + u32 local_ip; + u32 peer_ip; + u32 tos_tid; + struct tcp_options tcp_options; + u8 dst_mac[6]; + u16 vlan_tag; + u8 src_mac[6]; + u8 rsvd[2]; + u32 rcv_isn; + u32 unknown_tcp_options; +}; + +struct cpl_pass_accept_rpl { + union opcode_tid ot; + u32 rsvd0; + u32 rsvd1; + u32 peer_ip; + u32 opt0h; + union { + u32 opt0l; + struct { + u8 rsvd[3]; + u8 status; + }; + }; +}; + +struct cpl_act_open_req { + union opcode_tid ot; + u16 local_port; + u16 peer_port; + u32 local_ip; + u32 peer_ip; + u32 opt0h; + u32 opt0l; + u32 iff_vlantag; + u32 rsvd; +}; + +struct cpl_act_open_rpl { + union opcode_tid ot; + u16 local_port; + u16 peer_port; + u32 local_ip; + u32 peer_ip; + u32 new_tid; + u8 rsvd[3]; + u8 status; +}; + +struct cpl_act_establish { + union opcode_tid ot; + u16 local_port; + u16 peer_port; + u32 local_ip; + u32 peer_ip; + u32 tos_tid; + u32 rsvd; + u32 snd_isn; + u32 rcv_isn; +}; + +struct cpl_get_tcb { + union opcode_tid ot; + u32 rsvd; +}; + +struct cpl_get_tcb_rpl { + union opcode_tid ot; + u16 len; + u8 rsvd; + u8 status; +}; + +struct cpl_set_tcb { + union opcode_tid ot; + u16 len; + u16 rsvd; +}; + +struct cpl_set_tcb_field { + union opcode_tid ot; + u8 rsvd[3]; + u8 offset; + u32 mask; + u32 val; +}; + +struct cpl_set_tcb_rpl { + union opcode_tid ot; + u8 rsvd[3]; + u8 status; +}; + +struct cpl_pcmd { + union opcode_tid ot; + u16 dlen_in; + u16 dlen_out; + u32 pcmd_parm[2]; +}; + +struct cpl_pcmd_read { + union opcode_tid ot; + u32 rsvd1; + u16 rsvd2; + u32 addr; + u16 len; +}; + +struct cpl_pcmd_read_rpl { + union opcode_tid ot; + u16 len; +}; + +struct cpl_close_con_req { + union opcode_tid ot; + u32 rsvd; +}; + +struct cpl_close_con_rpl { + union opcode_tid ot; + u8 rsvd[3]; + u8 status; + u32 snd_nxt; + u32 rcv_nxt; +}; + +struct cpl_close_listserv_req { + union opcode_tid ot; + u32 rsvd; +}; + +struct cpl_close_listserv_rpl { + union opcode_tid ot; + u8 rsvd[3]; + u8 status; +}; + +struct cpl_abort_req { + union opcode_tid ot; + u32 rsvd0; + u8 rsvd1; + u8 cmd; + u8 rsvd2[6]; +}; + +struct cpl_abort_rpl { + union opcode_tid ot; + u32 rsvd0; + u8 rsvd1; + u8 status; + u8 rsvd2[6]; +}; + +struct cpl_peer_close { + union opcode_tid ot; + u32 rsvd; +}; + +struct cpl_tx_data { + union opcode_tid ot; + u32 len; + u32 rsvd0; + u16 urg; + u16 flags; +}; + +struct cpl_tx_data_ack { + union opcode_tid ot; + u32 ack_seq; +}; + +struct cpl_rx_data { + union opcode_tid ot; + u32 len; + u32 seq; + u16 urg; + u8 rsvd; + u8 status; +}; + +struct cpl_rx_data_ack { + union opcode_tid ot; + u32 credit; +}; + +struct cpl_rx_data_ddp { + union opcode_tid ot; + u32 len; + u32 seq; + u32 nxt_seq; + u32 ulp_crc; + u16 ddp_status; + u8 rsvd; + u8 status; +}; + +/* + * We want this header's alignment to be no more stringent than 2-byte aligned. + * All fields are u8 or u16 except for the length. However that field is not + * used so we break it into 2 16-bit parts to easily meet our alignment needs. + */ +struct cpl_tx_pkt { + u8 opcode; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 iff:4; + u8 ip_csum_dis:1; + u8 l4_csum_dis:1; + u8 vlan_valid:1; + u8 rsvd:1; +#else + u8 rsvd:1; + u8 vlan_valid:1; + u8 l4_csum_dis:1; + u8 ip_csum_dis:1; + u8 iff:4; +#endif + u16 vlan; + u16 len_hi; + u16 len_lo; +}; + +struct cpl_tx_pkt_lso { + u8 opcode; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 iff:4; + u8 ip_csum_dis:1; + u8 l4_csum_dis:1; + u8 vlan_valid:1; + u8 :1; +#else + u8 :1; + u8 vlan_valid:1; + u8 l4_csum_dis:1; + u8 ip_csum_dis:1; + u8 iff:4; +#endif + u16 vlan; + __be32 len; + + u8 rsvd[5]; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 tcp_hdr_words:4; + u8 ip_hdr_words:4; +#else + u8 ip_hdr_words:4; + u8 tcp_hdr_words:4; +#endif + __be16 eth_type_mss; +}; + +struct cpl_rx_pkt { + u8 opcode; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 iff:4; + u8 csum_valid:1; + u8 bad_pkt:1; + u8 vlan_valid:1; + u8 rsvd:1; +#else + u8 rsvd:1; + u8 vlan_valid:1; + u8 bad_pkt:1; + u8 csum_valid:1; + u8 iff:4; +#endif + u16 csum; + u16 vlan; + u16 len; +}; + +struct cpl_l2t_write_req { + union opcode_tid ot; + u32 params; + u8 rsvd1[2]; + u8 dst_mac[6]; +}; + +struct cpl_l2t_write_rpl { + union opcode_tid ot; + u8 status; + u8 rsvd[3]; +}; + +struct cpl_l2t_read_req { + union opcode_tid ot; + u8 rsvd[3]; + u8 l2t_idx; +}; + +struct cpl_l2t_read_rpl { + union opcode_tid ot; + u32 params; + u8 rsvd1[2]; + u8 dst_mac[6]; +}; + +struct cpl_smt_write_req { + union opcode_tid ot; + u8 rsvd0; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 rsvd1:1; + u8 mtu_idx:3; + u8 iff:4; +#else + u8 iff:4; + u8 mtu_idx:3; + u8 rsvd1:1; +#endif + u16 rsvd2; + u16 rsvd3; + u8 src_mac1[6]; + u16 rsvd4; + u8 src_mac0[6]; +}; + +struct cpl_smt_write_rpl { + union opcode_tid ot; + u8 status; + u8 rsvd[3]; +}; + +struct cpl_smt_read_req { + union opcode_tid ot; + u8 rsvd0; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 rsvd1:4; + u8 iff:4; +#else + u8 iff:4; + u8 rsvd1:4; +#endif + u16 rsvd2; +}; + +struct cpl_smt_read_rpl { + union opcode_tid ot; + u8 status; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 rsvd1:1; + u8 mtu_idx:3; + u8 rsvd0:4; +#else + u8 rsvd0:4; + u8 mtu_idx:3; + u8 rsvd1:1; +#endif + u16 rsvd2; + u16 rsvd3; + u8 src_mac1[6]; + u16 rsvd4; + u8 src_mac0[6]; +}; + +struct cpl_rte_delete_req { + union opcode_tid ot; + u32 params; +}; + +struct cpl_rte_delete_rpl { + union opcode_tid ot; + u8 status; + u8 rsvd[3]; +}; + +struct cpl_rte_write_req { + union opcode_tid ot; + u32 params; + u32 netmask; + u32 faddr; +}; + +struct cpl_rte_write_rpl { + union opcode_tid ot; + u8 status; + u8 rsvd[3]; +}; + +struct cpl_rte_read_req { + union opcode_tid ot; + u32 params; +}; + +struct cpl_rte_read_rpl { + union opcode_tid ot; + u8 status; + u8 rsvd0[2]; + u8 l2t_idx; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 rsvd1:7; + u8 select:1; +#else + u8 select:1; + u8 rsvd1:7; +#endif + u8 rsvd2[3]; + u32 addr; +}; + +struct cpl_mss_change { + union opcode_tid ot; + u32 mss; +}; + +#endif /* _CXGB_CPL5_CMD_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb/cxgb2.c b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c new file mode 100644 index 000000000..d2286adf0 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c @@ -0,0 +1,1329 @@ +/***************************************************************************** + * * + * File: cxgb2.c * + * $Revision: 1.25 $ * + * $Date: 2005/06/22 00:43:25 $ * + * Description: * + * Chelsio 10Gb Ethernet Driver. * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License, version 2, as * + * published by the Free Software Foundation. * + * * + * You should have received a copy of the GNU General Public License along * + * with this program; if not, see <http://www.gnu.org/licenses/>. * + * * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED * + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#include "common.h" +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> +#include <linux/if_vlan.h> +#include <linux/mii.h> +#include <linux/sockios.h> +#include <linux/dma-mapping.h> +#include <linux/uaccess.h> + +#include "cpl5_cmd.h" +#include "regs.h" +#include "gmac.h" +#include "cphy.h" +#include "sge.h" +#include "tp.h" +#include "espi.h" +#include "elmer0.h" + +#include <linux/workqueue.h> + +static inline void schedule_mac_stats_update(struct adapter *ap, int secs) +{ + schedule_delayed_work(&ap->stats_update_task, secs * HZ); +} + +static inline void cancel_mac_stats_update(struct adapter *ap) +{ + cancel_delayed_work(&ap->stats_update_task); +} + +#define MAX_CMDQ_ENTRIES 16384 +#define MAX_CMDQ1_ENTRIES 1024 +#define MAX_RX_BUFFERS 16384 +#define MAX_RX_JUMBO_BUFFERS 16384 +#define MAX_TX_BUFFERS_HIGH 16384U +#define MAX_TX_BUFFERS_LOW 1536U +#define MAX_TX_BUFFERS 1460U +#define MIN_FL_ENTRIES 32 + +#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \ + NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\ + NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) + +/* + * The EEPROM is actually bigger but only the first few bytes are used so we + * only report those. + */ +#define EEPROM_SIZE 32 + +MODULE_DESCRIPTION(DRV_DESCRIPTION); +MODULE_AUTHOR("Chelsio Communications"); +MODULE_LICENSE("GPL"); + +static int dflt_msg_enable = DFLT_MSG_ENABLE; + +module_param(dflt_msg_enable, int, 0); +MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T1 default message enable bitmap"); + +#define HCLOCK 0x0 +#define LCLOCK 0x1 + +/* T1 cards powersave mode */ +static int t1_clock(struct adapter *adapter, int mode); +static int t1powersave = 1; /* HW default is powersave mode. */ + +module_param(t1powersave, int, 0); +MODULE_PARM_DESC(t1powersave, "Enable/Disable T1 powersaving mode"); + +static int disable_msi = 0; +module_param(disable_msi, int, 0); +MODULE_PARM_DESC(disable_msi, "Disable Message Signaled Interrupt (MSI)"); + +/* + * Setup MAC to receive the types of packets we want. + */ +static void t1_set_rxmode(struct net_device *dev) +{ + struct adapter *adapter = dev->ml_priv; + struct cmac *mac = adapter->port[dev->if_port].mac; + struct t1_rx_mode rm; + + rm.dev = dev; + mac->ops->set_rx_mode(mac, &rm); +} + +static void link_report(struct port_info *p) +{ + if (!netif_carrier_ok(p->dev)) + netdev_info(p->dev, "link down\n"); + else { + const char *s = "10Mbps"; + + switch (p->link_config.speed) { + case SPEED_10000: s = "10Gbps"; break; + case SPEED_1000: s = "1000Mbps"; break; + case SPEED_100: s = "100Mbps"; break; + } + + netdev_info(p->dev, "link up, %s, %s-duplex\n", + s, p->link_config.duplex == DUPLEX_FULL + ? "full" : "half"); + } +} + +void t1_link_negotiated(struct adapter *adapter, int port_id, int link_stat, + int speed, int duplex, int pause) +{ + struct port_info *p = &adapter->port[port_id]; + + if (link_stat != netif_carrier_ok(p->dev)) { + if (link_stat) + netif_carrier_on(p->dev); + else + netif_carrier_off(p->dev); + link_report(p); + + /* multi-ports: inform toe */ + if ((speed > 0) && (adapter->params.nports > 1)) { + unsigned int sched_speed = 10; + switch (speed) { + case SPEED_1000: + sched_speed = 1000; + break; + case SPEED_100: + sched_speed = 100; + break; + case SPEED_10: + sched_speed = 10; + break; + } + t1_sched_update_parms(adapter->sge, port_id, 0, sched_speed); + } + } +} + +static void link_start(struct port_info *p) +{ + struct cmac *mac = p->mac; + + mac->ops->reset(mac); + if (mac->ops->macaddress_set) + mac->ops->macaddress_set(mac, p->dev->dev_addr); + t1_set_rxmode(p->dev); + t1_link_start(p->phy, mac, &p->link_config); + mac->ops->enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); +} + +static void enable_hw_csum(struct adapter *adapter) +{ + if (adapter->port[0].dev->hw_features & NETIF_F_TSO) + t1_tp_set_ip_checksum_offload(adapter->tp, 1); /* for TSO only */ + t1_tp_set_tcp_checksum_offload(adapter->tp, 1); +} + +/* + * Things to do upon first use of a card. + * This must run with the rtnl lock held. + */ +static int cxgb_up(struct adapter *adapter) +{ + int err = 0; + + if (!(adapter->flags & FULL_INIT_DONE)) { + err = t1_init_hw_modules(adapter); + if (err) + goto out_err; + + enable_hw_csum(adapter); + adapter->flags |= FULL_INIT_DONE; + } + + t1_interrupts_clear(adapter); + + adapter->params.has_msi = !disable_msi && !pci_enable_msi(adapter->pdev); + err = request_threaded_irq(adapter->pdev->irq, t1_interrupt, + t1_interrupt_thread, + adapter->params.has_msi ? 0 : IRQF_SHARED, + adapter->name, adapter); + if (err) { + if (adapter->params.has_msi) + pci_disable_msi(adapter->pdev); + + goto out_err; + } + + t1_sge_start(adapter->sge); + t1_interrupts_enable(adapter); +out_err: + return err; +} + +/* + * Release resources when all the ports have been stopped. + */ +static void cxgb_down(struct adapter *adapter) +{ + t1_sge_stop(adapter->sge); + t1_interrupts_disable(adapter); + free_irq(adapter->pdev->irq, adapter); + if (adapter->params.has_msi) + pci_disable_msi(adapter->pdev); +} + +static int cxgb_open(struct net_device *dev) +{ + int err; + struct adapter *adapter = dev->ml_priv; + int other_ports = adapter->open_device_map & PORT_MASK; + + napi_enable(&adapter->napi); + if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0) { + napi_disable(&adapter->napi); + return err; + } + + __set_bit(dev->if_port, &adapter->open_device_map); + link_start(&adapter->port[dev->if_port]); + netif_start_queue(dev); + if (!other_ports && adapter->params.stats_update_period) + schedule_mac_stats_update(adapter, + adapter->params.stats_update_period); + + t1_vlan_mode(adapter, dev->features); + return 0; +} + +static int cxgb_close(struct net_device *dev) +{ + struct adapter *adapter = dev->ml_priv; + struct port_info *p = &adapter->port[dev->if_port]; + struct cmac *mac = p->mac; + + netif_stop_queue(dev); + napi_disable(&adapter->napi); + mac->ops->disable(mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); + netif_carrier_off(dev); + + clear_bit(dev->if_port, &adapter->open_device_map); + if (adapter->params.stats_update_period && + !(adapter->open_device_map & PORT_MASK)) { + /* Stop statistics accumulation. */ + smp_mb__after_atomic(); + spin_lock(&adapter->work_lock); /* sync with update task */ + spin_unlock(&adapter->work_lock); + cancel_mac_stats_update(adapter); + } + + if (!adapter->open_device_map) + cxgb_down(adapter); + return 0; +} + +static struct net_device_stats *t1_get_stats(struct net_device *dev) +{ + struct adapter *adapter = dev->ml_priv; + struct port_info *p = &adapter->port[dev->if_port]; + struct net_device_stats *ns = &dev->stats; + const struct cmac_statistics *pstats; + + /* Do a full update of the MAC stats */ + pstats = p->mac->ops->statistics_update(p->mac, + MAC_STATS_UPDATE_FULL); + + ns->tx_packets = pstats->TxUnicastFramesOK + + pstats->TxMulticastFramesOK + pstats->TxBroadcastFramesOK; + + ns->rx_packets = pstats->RxUnicastFramesOK + + pstats->RxMulticastFramesOK + pstats->RxBroadcastFramesOK; + + ns->tx_bytes = pstats->TxOctetsOK; + ns->rx_bytes = pstats->RxOctetsOK; + + ns->tx_errors = pstats->TxLateCollisions + pstats->TxLengthErrors + + pstats->TxUnderrun + pstats->TxFramesAbortedDueToXSCollisions; + ns->rx_errors = pstats->RxDataErrors + pstats->RxJabberErrors + + pstats->RxFCSErrors + pstats->RxAlignErrors + + pstats->RxSequenceErrors + pstats->RxFrameTooLongErrors + + pstats->RxSymbolErrors + pstats->RxRuntErrors; + + ns->multicast = pstats->RxMulticastFramesOK; + ns->collisions = pstats->TxTotalCollisions; + + /* detailed rx_errors */ + ns->rx_length_errors = pstats->RxFrameTooLongErrors + + pstats->RxJabberErrors; + ns->rx_over_errors = 0; + ns->rx_crc_errors = pstats->RxFCSErrors; + ns->rx_frame_errors = pstats->RxAlignErrors; + ns->rx_fifo_errors = 0; + ns->rx_missed_errors = 0; + + /* detailed tx_errors */ + ns->tx_aborted_errors = pstats->TxFramesAbortedDueToXSCollisions; + ns->tx_carrier_errors = 0; + ns->tx_fifo_errors = pstats->TxUnderrun; + ns->tx_heartbeat_errors = 0; + ns->tx_window_errors = pstats->TxLateCollisions; + return ns; +} + +static u32 get_msglevel(struct net_device *dev) +{ + struct adapter *adapter = dev->ml_priv; + + return adapter->msg_enable; +} + +static void set_msglevel(struct net_device *dev, u32 val) +{ + struct adapter *adapter = dev->ml_priv; + + adapter->msg_enable = val; +} + +static const char stats_strings[][ETH_GSTRING_LEN] = { + "TxOctetsOK", + "TxOctetsBad", + "TxUnicastFramesOK", + "TxMulticastFramesOK", + "TxBroadcastFramesOK", + "TxPauseFrames", + "TxFramesWithDeferredXmissions", + "TxLateCollisions", + "TxTotalCollisions", + "TxFramesAbortedDueToXSCollisions", + "TxUnderrun", + "TxLengthErrors", + "TxInternalMACXmitError", + "TxFramesWithExcessiveDeferral", + "TxFCSErrors", + "TxJumboFramesOk", + "TxJumboOctetsOk", + + "RxOctetsOK", + "RxOctetsBad", + "RxUnicastFramesOK", + "RxMulticastFramesOK", + "RxBroadcastFramesOK", + "RxPauseFrames", + "RxFCSErrors", + "RxAlignErrors", + "RxSymbolErrors", + "RxDataErrors", + "RxSequenceErrors", + "RxRuntErrors", + "RxJabberErrors", + "RxInternalMACRcvError", + "RxInRangeLengthErrors", + "RxOutOfRangeLengthField", + "RxFrameTooLongErrors", + "RxJumboFramesOk", + "RxJumboOctetsOk", + + /* Port stats */ + "RxCsumGood", + "TxCsumOffload", + "TxTso", + "RxVlan", + "TxVlan", + "TxNeedHeadroom", + + /* Interrupt stats */ + "rx drops", + "pure_rsps", + "unhandled irqs", + "respQ_empty", + "respQ_overflow", + "freelistQ_empty", + "pkt_too_big", + "pkt_mismatch", + "cmdQ_full0", + "cmdQ_full1", + + "espi_DIP2ParityErr", + "espi_DIP4Err", + "espi_RxDrops", + "espi_TxDrops", + "espi_RxOvfl", + "espi_ParityErr" +}; + +#define T2_REGMAP_SIZE (3 * 1024) + +static int get_regs_len(struct net_device *dev) +{ + return T2_REGMAP_SIZE; +} + +static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) +{ + struct adapter *adapter = dev->ml_priv; + + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(adapter->pdev), + sizeof(info->bus_info)); +} + +static int get_sset_count(struct net_device *dev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return ARRAY_SIZE(stats_strings); + default: + return -EOPNOTSUPP; + } +} + +static void get_strings(struct net_device *dev, u32 stringset, u8 *data) +{ + if (stringset == ETH_SS_STATS) + memcpy(data, stats_strings, sizeof(stats_strings)); +} + +static void get_stats(struct net_device *dev, struct ethtool_stats *stats, + u64 *data) +{ + struct adapter *adapter = dev->ml_priv; + struct cmac *mac = adapter->port[dev->if_port].mac; + const struct cmac_statistics *s; + const struct sge_intr_counts *t; + struct sge_port_stats ss; + + s = mac->ops->statistics_update(mac, MAC_STATS_UPDATE_FULL); + t = t1_sge_get_intr_counts(adapter->sge); + t1_sge_get_port_stats(adapter->sge, dev->if_port, &ss); + + *data++ = s->TxOctetsOK; + *data++ = s->TxOctetsBad; + *data++ = s->TxUnicastFramesOK; + *data++ = s->TxMulticastFramesOK; + *data++ = s->TxBroadcastFramesOK; + *data++ = s->TxPauseFrames; + *data++ = s->TxFramesWithDeferredXmissions; + *data++ = s->TxLateCollisions; + *data++ = s->TxTotalCollisions; + *data++ = s->TxFramesAbortedDueToXSCollisions; + *data++ = s->TxUnderrun; + *data++ = s->TxLengthErrors; + *data++ = s->TxInternalMACXmitError; + *data++ = s->TxFramesWithExcessiveDeferral; + *data++ = s->TxFCSErrors; + *data++ = s->TxJumboFramesOK; + *data++ = s->TxJumboOctetsOK; + + *data++ = s->RxOctetsOK; + *data++ = s->RxOctetsBad; + *data++ = s->RxUnicastFramesOK; + *data++ = s->RxMulticastFramesOK; + *data++ = s->RxBroadcastFramesOK; + *data++ = s->RxPauseFrames; + *data++ = s->RxFCSErrors; + *data++ = s->RxAlignErrors; + *data++ = s->RxSymbolErrors; + *data++ = s->RxDataErrors; + *data++ = s->RxSequenceErrors; + *data++ = s->RxRuntErrors; + *data++ = s->RxJabberErrors; + *data++ = s->RxInternalMACRcvError; + *data++ = s->RxInRangeLengthErrors; + *data++ = s->RxOutOfRangeLengthField; + *data++ = s->RxFrameTooLongErrors; + *data++ = s->RxJumboFramesOK; + *data++ = s->RxJumboOctetsOK; + + *data++ = ss.rx_cso_good; + *data++ = ss.tx_cso; + *data++ = ss.tx_tso; + *data++ = ss.vlan_xtract; + *data++ = ss.vlan_insert; + *data++ = ss.tx_need_hdrroom; + + *data++ = t->rx_drops; + *data++ = t->pure_rsps; + *data++ = t->unhandled_irqs; + *data++ = t->respQ_empty; + *data++ = t->respQ_overflow; + *data++ = t->freelistQ_empty; + *data++ = t->pkt_too_big; + *data++ = t->pkt_mismatch; + *data++ = t->cmdQ_full[0]; + *data++ = t->cmdQ_full[1]; + + if (adapter->espi) { + const struct espi_intr_counts *e; + + e = t1_espi_get_intr_counts(adapter->espi); + *data++ = e->DIP2_parity_err; + *data++ = e->DIP4_err; + *data++ = e->rx_drops; + *data++ = e->tx_drops; + *data++ = e->rx_ovflw; + *data++ = e->parity_err; + } +} + +static inline void reg_block_dump(struct adapter *ap, void *buf, + unsigned int start, unsigned int end) +{ + u32 *p = buf + start; + + for ( ; start <= end; start += sizeof(u32)) + *p++ = readl(ap->regs + start); +} + +static void get_regs(struct net_device *dev, struct ethtool_regs *regs, + void *buf) +{ + struct adapter *ap = dev->ml_priv; + + /* + * Version scheme: bits 0..9: chip version, bits 10..15: chip revision + */ + regs->version = 2; + + memset(buf, 0, T2_REGMAP_SIZE); + reg_block_dump(ap, buf, 0, A_SG_RESPACCUTIMER); + reg_block_dump(ap, buf, A_MC3_CFG, A_MC4_INT_CAUSE); + reg_block_dump(ap, buf, A_TPI_ADDR, A_TPI_PAR); + reg_block_dump(ap, buf, A_TP_IN_CONFIG, A_TP_TX_DROP_COUNT); + reg_block_dump(ap, buf, A_RAT_ROUTE_CONTROL, A_RAT_INTR_CAUSE); + reg_block_dump(ap, buf, A_CSPI_RX_AE_WM, A_CSPI_INTR_ENABLE); + reg_block_dump(ap, buf, A_ESPI_SCH_TOKEN0, A_ESPI_GOSTAT); + reg_block_dump(ap, buf, A_ULP_ULIMIT, A_ULP_PIO_CTRL); + reg_block_dump(ap, buf, A_PL_ENABLE, A_PL_CAUSE); + reg_block_dump(ap, buf, A_MC5_CONFIG, A_MC5_MASK_WRITE_CMD); +} + +static int get_link_ksettings(struct net_device *dev, + struct ethtool_link_ksettings *cmd) +{ + struct adapter *adapter = dev->ml_priv; + struct port_info *p = &adapter->port[dev->if_port]; + u32 supported, advertising; + + supported = p->link_config.supported; + advertising = p->link_config.advertising; + + if (netif_carrier_ok(dev)) { + cmd->base.speed = p->link_config.speed; + cmd->base.duplex = p->link_config.duplex; + } else { + cmd->base.speed = SPEED_UNKNOWN; + cmd->base.duplex = DUPLEX_UNKNOWN; + } + + cmd->base.port = (supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE; + cmd->base.phy_address = p->phy->mdio.prtad; + cmd->base.autoneg = p->link_config.autoneg; + + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, + supported); + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, + advertising); + + return 0; +} + +static int speed_duplex_to_caps(int speed, int duplex) +{ + int cap = 0; + + switch (speed) { + case SPEED_10: + if (duplex == DUPLEX_FULL) + cap = SUPPORTED_10baseT_Full; + else + cap = SUPPORTED_10baseT_Half; + break; + case SPEED_100: + if (duplex == DUPLEX_FULL) + cap = SUPPORTED_100baseT_Full; + else + cap = SUPPORTED_100baseT_Half; + break; + case SPEED_1000: + if (duplex == DUPLEX_FULL) + cap = SUPPORTED_1000baseT_Full; + else + cap = SUPPORTED_1000baseT_Half; + break; + case SPEED_10000: + if (duplex == DUPLEX_FULL) + cap = SUPPORTED_10000baseT_Full; + } + return cap; +} + +#define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \ + ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \ + ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \ + ADVERTISED_10000baseT_Full) + +static int set_link_ksettings(struct net_device *dev, + const struct ethtool_link_ksettings *cmd) +{ + struct adapter *adapter = dev->ml_priv; + struct port_info *p = &adapter->port[dev->if_port]; + struct link_config *lc = &p->link_config; + u32 advertising; + + ethtool_convert_link_mode_to_legacy_u32(&advertising, + cmd->link_modes.advertising); + + if (!(lc->supported & SUPPORTED_Autoneg)) + return -EOPNOTSUPP; /* can't change speed/duplex */ + + if (cmd->base.autoneg == AUTONEG_DISABLE) { + u32 speed = cmd->base.speed; + int cap = speed_duplex_to_caps(speed, cmd->base.duplex); + + if (!(lc->supported & cap) || (speed == SPEED_1000)) + return -EINVAL; + lc->requested_speed = speed; + lc->requested_duplex = cmd->base.duplex; + lc->advertising = 0; + } else { + advertising &= ADVERTISED_MASK; + if (advertising & (advertising - 1)) + advertising = lc->supported; + advertising &= lc->supported; + if (!advertising) + return -EINVAL; + lc->requested_speed = SPEED_INVALID; + lc->requested_duplex = DUPLEX_INVALID; + lc->advertising = advertising | ADVERTISED_Autoneg; + } + lc->autoneg = cmd->base.autoneg; + if (netif_running(dev)) + t1_link_start(p->phy, p->mac, lc); + return 0; +} + +static void get_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *epause) +{ + struct adapter *adapter = dev->ml_priv; + struct port_info *p = &adapter->port[dev->if_port]; + + epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0; + epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0; + epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0; +} + +static int set_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *epause) +{ + struct adapter *adapter = dev->ml_priv; + struct port_info *p = &adapter->port[dev->if_port]; + struct link_config *lc = &p->link_config; + + if (epause->autoneg == AUTONEG_DISABLE) + lc->requested_fc = 0; + else if (lc->supported & SUPPORTED_Autoneg) + lc->requested_fc = PAUSE_AUTONEG; + else + return -EINVAL; + + if (epause->rx_pause) + lc->requested_fc |= PAUSE_RX; + if (epause->tx_pause) + lc->requested_fc |= PAUSE_TX; + if (lc->autoneg == AUTONEG_ENABLE) { + if (netif_running(dev)) + t1_link_start(p->phy, p->mac, lc); + } else { + lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); + if (netif_running(dev)) + p->mac->ops->set_speed_duplex_fc(p->mac, -1, -1, + lc->fc); + } + return 0; +} + +static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) +{ + struct adapter *adapter = dev->ml_priv; + int jumbo_fl = t1_is_T1B(adapter) ? 1 : 0; + + e->rx_max_pending = MAX_RX_BUFFERS; + e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS; + e->tx_max_pending = MAX_CMDQ_ENTRIES; + + e->rx_pending = adapter->params.sge.freelQ_size[!jumbo_fl]; + e->rx_jumbo_pending = adapter->params.sge.freelQ_size[jumbo_fl]; + e->tx_pending = adapter->params.sge.cmdQ_size[0]; +} + +static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) +{ + struct adapter *adapter = dev->ml_priv; + int jumbo_fl = t1_is_T1B(adapter) ? 1 : 0; + + if (e->rx_pending > MAX_RX_BUFFERS || e->rx_mini_pending || + e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS || + e->tx_pending > MAX_CMDQ_ENTRIES || + e->rx_pending < MIN_FL_ENTRIES || + e->rx_jumbo_pending < MIN_FL_ENTRIES || + e->tx_pending < (adapter->params.nports + 1) * (MAX_SKB_FRAGS + 1)) + return -EINVAL; + + if (adapter->flags & FULL_INIT_DONE) + return -EBUSY; + + adapter->params.sge.freelQ_size[!jumbo_fl] = e->rx_pending; + adapter->params.sge.freelQ_size[jumbo_fl] = e->rx_jumbo_pending; + adapter->params.sge.cmdQ_size[0] = e->tx_pending; + adapter->params.sge.cmdQ_size[1] = e->tx_pending > MAX_CMDQ1_ENTRIES ? + MAX_CMDQ1_ENTRIES : e->tx_pending; + return 0; +} + +static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct adapter *adapter = dev->ml_priv; + + adapter->params.sge.rx_coalesce_usecs = c->rx_coalesce_usecs; + adapter->params.sge.coalesce_enable = c->use_adaptive_rx_coalesce; + adapter->params.sge.sample_interval_usecs = c->rate_sample_interval; + t1_sge_set_coalesce_params(adapter->sge, &adapter->params.sge); + return 0; +} + +static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct adapter *adapter = dev->ml_priv; + + c->rx_coalesce_usecs = adapter->params.sge.rx_coalesce_usecs; + c->rate_sample_interval = adapter->params.sge.sample_interval_usecs; + c->use_adaptive_rx_coalesce = adapter->params.sge.coalesce_enable; + return 0; +} + +static int get_eeprom_len(struct net_device *dev) +{ + struct adapter *adapter = dev->ml_priv; + + return t1_is_asic(adapter) ? EEPROM_SIZE : 0; +} + +#define EEPROM_MAGIC(ap) \ + (PCI_VENDOR_ID_CHELSIO | ((ap)->params.chip_version << 16)) + +static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e, + u8 *data) +{ + int i; + u8 buf[EEPROM_SIZE] __attribute__((aligned(4))); + struct adapter *adapter = dev->ml_priv; + + e->magic = EEPROM_MAGIC(adapter); + for (i = e->offset & ~3; i < e->offset + e->len; i += sizeof(u32)) + t1_seeprom_read(adapter, i, (__le32 *)&buf[i]); + memcpy(data, buf + e->offset, e->len); + return 0; +} + +static const struct ethtool_ops t1_ethtool_ops = { + .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS | + ETHTOOL_COALESCE_USE_ADAPTIVE_RX | + ETHTOOL_COALESCE_RATE_SAMPLE_INTERVAL, + .get_drvinfo = get_drvinfo, + .get_msglevel = get_msglevel, + .set_msglevel = set_msglevel, + .get_ringparam = get_sge_param, + .set_ringparam = set_sge_param, + .get_coalesce = get_coalesce, + .set_coalesce = set_coalesce, + .get_eeprom_len = get_eeprom_len, + .get_eeprom = get_eeprom, + .get_pauseparam = get_pauseparam, + .set_pauseparam = set_pauseparam, + .get_link = ethtool_op_get_link, + .get_strings = get_strings, + .get_sset_count = get_sset_count, + .get_ethtool_stats = get_stats, + .get_regs_len = get_regs_len, + .get_regs = get_regs, + .get_link_ksettings = get_link_ksettings, + .set_link_ksettings = set_link_ksettings, +}; + +static int t1_ioctl(struct net_device *dev, struct ifreq *req, int cmd) +{ + struct adapter *adapter = dev->ml_priv; + struct mdio_if_info *mdio = &adapter->port[dev->if_port].phy->mdio; + + return mdio_mii_ioctl(mdio, if_mii(req), cmd); +} + +static int t1_change_mtu(struct net_device *dev, int new_mtu) +{ + int ret; + struct adapter *adapter = dev->ml_priv; + struct cmac *mac = adapter->port[dev->if_port].mac; + + if (!mac->ops->set_mtu) + return -EOPNOTSUPP; + if ((ret = mac->ops->set_mtu(mac, new_mtu))) + return ret; + dev->mtu = new_mtu; + return 0; +} + +static int t1_set_mac_addr(struct net_device *dev, void *p) +{ + struct adapter *adapter = dev->ml_priv; + struct cmac *mac = adapter->port[dev->if_port].mac; + struct sockaddr *addr = p; + + if (!mac->ops->macaddress_set) + return -EOPNOTSUPP; + + eth_hw_addr_set(dev, addr->sa_data); + mac->ops->macaddress_set(mac, dev->dev_addr); + return 0; +} + +static netdev_features_t t1_fix_features(struct net_device *dev, + netdev_features_t features) +{ + /* + * Since there is no support for separate rx/tx vlan accel + * enable/disable make sure tx flag is always in same state as rx. + */ + if (features & NETIF_F_HW_VLAN_CTAG_RX) + features |= NETIF_F_HW_VLAN_CTAG_TX; + else + features &= ~NETIF_F_HW_VLAN_CTAG_TX; + + return features; +} + +static int t1_set_features(struct net_device *dev, netdev_features_t features) +{ + netdev_features_t changed = dev->features ^ features; + struct adapter *adapter = dev->ml_priv; + + if (changed & NETIF_F_HW_VLAN_CTAG_RX) + t1_vlan_mode(adapter, features); + + return 0; +} +#ifdef CONFIG_NET_POLL_CONTROLLER +static void t1_netpoll(struct net_device *dev) +{ + unsigned long flags; + struct adapter *adapter = dev->ml_priv; + + local_irq_save(flags); + t1_interrupt(adapter->pdev->irq, adapter); + local_irq_restore(flags); +} +#endif + +/* + * Periodic accumulation of MAC statistics. This is used only if the MAC + * does not have any other way to prevent stats counter overflow. + */ +static void mac_stats_task(struct work_struct *work) +{ + int i; + struct adapter *adapter = + container_of(work, struct adapter, stats_update_task.work); + + for_each_port(adapter, i) { + struct port_info *p = &adapter->port[i]; + + if (netif_running(p->dev)) + p->mac->ops->statistics_update(p->mac, + MAC_STATS_UPDATE_FAST); + } + + /* Schedule the next statistics update if any port is active. */ + spin_lock(&adapter->work_lock); + if (adapter->open_device_map & PORT_MASK) + schedule_mac_stats_update(adapter, + adapter->params.stats_update_period); + spin_unlock(&adapter->work_lock); +} + +static const struct net_device_ops cxgb_netdev_ops = { + .ndo_open = cxgb_open, + .ndo_stop = cxgb_close, + .ndo_start_xmit = t1_start_xmit, + .ndo_get_stats = t1_get_stats, + .ndo_validate_addr = eth_validate_addr, + .ndo_set_rx_mode = t1_set_rxmode, + .ndo_eth_ioctl = t1_ioctl, + .ndo_change_mtu = t1_change_mtu, + .ndo_set_mac_address = t1_set_mac_addr, + .ndo_fix_features = t1_fix_features, + .ndo_set_features = t1_set_features, +#ifdef CONFIG_NET_POLL_CONTROLLER + .ndo_poll_controller = t1_netpoll, +#endif +}; + +static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + unsigned long mmio_start, mmio_len; + const struct board_info *bi; + struct adapter *adapter = NULL; + struct port_info *pi; + int i, err; + + err = pci_enable_device(pdev); + if (err) + return err; + + if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { + pr_err("%s: cannot find PCI device memory base address\n", + pci_name(pdev)); + err = -ENODEV; + goto out_disable_pdev; + } + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + pr_err("%s: no usable DMA configuration\n", pci_name(pdev)); + goto out_disable_pdev; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + pr_err("%s: cannot obtain PCI resources\n", pci_name(pdev)); + goto out_disable_pdev; + } + + pci_set_master(pdev); + + mmio_start = pci_resource_start(pdev, 0); + mmio_len = pci_resource_len(pdev, 0); + bi = t1_get_board_info(ent->driver_data); + + for (i = 0; i < bi->port_number; ++i) { + struct net_device *netdev; + + netdev = alloc_etherdev(adapter ? 0 : sizeof(*adapter)); + if (!netdev) { + err = -ENOMEM; + goto out_free_dev; + } + + SET_NETDEV_DEV(netdev, &pdev->dev); + + if (!adapter) { + adapter = netdev_priv(netdev); + adapter->pdev = pdev; + adapter->port[0].dev = netdev; /* so we don't leak it */ + + adapter->regs = ioremap(mmio_start, mmio_len); + if (!adapter->regs) { + pr_err("%s: cannot map device registers\n", + pci_name(pdev)); + err = -ENOMEM; + goto out_free_dev; + } + + if (t1_get_board_rev(adapter, bi, &adapter->params)) { + err = -ENODEV; /* Can't handle this chip rev */ + goto out_free_dev; + } + + adapter->name = pci_name(pdev); + adapter->msg_enable = dflt_msg_enable; + adapter->mmio_len = mmio_len; + + spin_lock_init(&adapter->tpi_lock); + spin_lock_init(&adapter->work_lock); + spin_lock_init(&adapter->async_lock); + spin_lock_init(&adapter->mac_lock); + + INIT_DELAYED_WORK(&adapter->stats_update_task, + mac_stats_task); + + pci_set_drvdata(pdev, netdev); + } + + pi = &adapter->port[i]; + pi->dev = netdev; + netif_carrier_off(netdev); + netdev->irq = pdev->irq; + netdev->if_port = i; + netdev->mem_start = mmio_start; + netdev->mem_end = mmio_start + mmio_len - 1; + netdev->ml_priv = adapter; + netdev->hw_features |= NETIF_F_SG | NETIF_F_IP_CSUM | + NETIF_F_RXCSUM; + netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM | + NETIF_F_RXCSUM | NETIF_F_LLTX | NETIF_F_HIGHDMA; + + if (vlan_tso_capable(adapter)) { + netdev->features |= + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_HW_VLAN_CTAG_RX; + netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; + + /* T204: disable TSO */ + if (!(is_T2(adapter)) || bi->port_number != 4) { + netdev->hw_features |= NETIF_F_TSO; + netdev->features |= NETIF_F_TSO; + } + } + + netdev->netdev_ops = &cxgb_netdev_ops; + netdev->hard_header_len += (netdev->hw_features & NETIF_F_TSO) ? + sizeof(struct cpl_tx_pkt_lso) : sizeof(struct cpl_tx_pkt); + + netif_napi_add(netdev, &adapter->napi, t1_poll); + + netdev->ethtool_ops = &t1_ethtool_ops; + + switch (bi->board) { + case CHBT_BOARD_CHT110: + case CHBT_BOARD_N110: + case CHBT_BOARD_N210: + case CHBT_BOARD_CHT210: + netdev->max_mtu = PM3393_MAX_FRAME_SIZE - + (ETH_HLEN + ETH_FCS_LEN); + break; + case CHBT_BOARD_CHN204: + netdev->max_mtu = VSC7326_MAX_MTU; + break; + default: + netdev->max_mtu = ETH_DATA_LEN; + break; + } + } + + if (t1_init_sw_modules(adapter, bi) < 0) { + err = -ENODEV; + goto out_free_dev; + } + + /* + * The card is now ready to go. If any errors occur during device + * registration we do not fail the whole card but rather proceed only + * with the ports we manage to register successfully. However we must + * register at least one net device. + */ + for (i = 0; i < bi->port_number; ++i) { + err = register_netdev(adapter->port[i].dev); + if (err) + pr_warn("%s: cannot register net device %s, skipping\n", + pci_name(pdev), adapter->port[i].dev->name); + else { + /* + * Change the name we use for messages to the name of + * the first successfully registered interface. + */ + if (!adapter->registered_device_map) + adapter->name = adapter->port[i].dev->name; + + __set_bit(i, &adapter->registered_device_map); + } + } + if (!adapter->registered_device_map) { + pr_err("%s: could not register any net devices\n", + pci_name(pdev)); + err = -EINVAL; + goto out_release_adapter_res; + } + + pr_info("%s: %s (rev %d), %s %dMHz/%d-bit\n", + adapter->name, bi->desc, adapter->params.chip_revision, + adapter->params.pci.is_pcix ? "PCIX" : "PCI", + adapter->params.pci.speed, adapter->params.pci.width); + + /* + * Set the T1B ASIC and memory clocks. + */ + if (t1powersave) + adapter->t1powersave = LCLOCK; /* HW default is powersave mode. */ + else + adapter->t1powersave = HCLOCK; + if (t1_is_T1B(adapter)) + t1_clock(adapter, t1powersave); + + return 0; + +out_release_adapter_res: + t1_free_sw_modules(adapter); +out_free_dev: + if (adapter) { + if (adapter->regs) + iounmap(adapter->regs); + for (i = bi->port_number - 1; i >= 0; --i) + if (adapter->port[i].dev) + free_netdev(adapter->port[i].dev); + } + pci_release_regions(pdev); +out_disable_pdev: + pci_disable_device(pdev); + return err; +} + +static void bit_bang(struct adapter *adapter, int bitdata, int nbits) +{ + int data; + int i; + u32 val; + + enum { + S_CLOCK = 1 << 3, + S_DATA = 1 << 4 + }; + + for (i = (nbits - 1); i > -1; i--) { + + udelay(50); + + data = ((bitdata >> i) & 0x1); + __t1_tpi_read(adapter, A_ELMER0_GPO, &val); + + if (data) + val |= S_DATA; + else + val &= ~S_DATA; + + udelay(50); + + /* Set SCLOCK low */ + val &= ~S_CLOCK; + __t1_tpi_write(adapter, A_ELMER0_GPO, val); + + udelay(50); + + /* Write SCLOCK high */ + val |= S_CLOCK; + __t1_tpi_write(adapter, A_ELMER0_GPO, val); + + } +} + +static int t1_clock(struct adapter *adapter, int mode) +{ + u32 val; + int M_CORE_VAL; + int M_MEM_VAL; + + enum { + M_CORE_BITS = 9, + T_CORE_VAL = 0, + T_CORE_BITS = 2, + N_CORE_VAL = 0, + N_CORE_BITS = 2, + M_MEM_BITS = 9, + T_MEM_VAL = 0, + T_MEM_BITS = 2, + N_MEM_VAL = 0, + N_MEM_BITS = 2, + NP_LOAD = 1 << 17, + S_LOAD_MEM = 1 << 5, + S_LOAD_CORE = 1 << 6, + S_CLOCK = 1 << 3 + }; + + if (!t1_is_T1B(adapter)) + return -ENODEV; /* Can't re-clock this chip. */ + + if (mode & 2) + return 0; /* show current mode. */ + + if ((adapter->t1powersave & 1) == (mode & 1)) + return -EALREADY; /* ASIC already running in mode. */ + + if ((mode & 1) == HCLOCK) { + M_CORE_VAL = 0x14; + M_MEM_VAL = 0x18; + adapter->t1powersave = HCLOCK; /* overclock */ + } else { + M_CORE_VAL = 0xe; + M_MEM_VAL = 0x10; + adapter->t1powersave = LCLOCK; /* underclock */ + } + + /* Don't interrupt this serial stream! */ + spin_lock(&adapter->tpi_lock); + + /* Initialize for ASIC core */ + __t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val |= NP_LOAD; + udelay(50); + __t1_tpi_write(adapter, A_ELMER0_GPO, val); + udelay(50); + __t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val &= ~S_LOAD_CORE; + val &= ~S_CLOCK; + __t1_tpi_write(adapter, A_ELMER0_GPO, val); + udelay(50); + + /* Serial program the ASIC clock synthesizer */ + bit_bang(adapter, T_CORE_VAL, T_CORE_BITS); + bit_bang(adapter, N_CORE_VAL, N_CORE_BITS); + bit_bang(adapter, M_CORE_VAL, M_CORE_BITS); + udelay(50); + + /* Finish ASIC core */ + __t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val |= S_LOAD_CORE; + udelay(50); + __t1_tpi_write(adapter, A_ELMER0_GPO, val); + udelay(50); + __t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val &= ~S_LOAD_CORE; + udelay(50); + __t1_tpi_write(adapter, A_ELMER0_GPO, val); + udelay(50); + + /* Initialize for memory */ + __t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val |= NP_LOAD; + udelay(50); + __t1_tpi_write(adapter, A_ELMER0_GPO, val); + udelay(50); + __t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val &= ~S_LOAD_MEM; + val &= ~S_CLOCK; + udelay(50); + __t1_tpi_write(adapter, A_ELMER0_GPO, val); + udelay(50); + + /* Serial program the memory clock synthesizer */ + bit_bang(adapter, T_MEM_VAL, T_MEM_BITS); + bit_bang(adapter, N_MEM_VAL, N_MEM_BITS); + bit_bang(adapter, M_MEM_VAL, M_MEM_BITS); + udelay(50); + + /* Finish memory */ + __t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val |= S_LOAD_MEM; + udelay(50); + __t1_tpi_write(adapter, A_ELMER0_GPO, val); + udelay(50); + __t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val &= ~S_LOAD_MEM; + udelay(50); + __t1_tpi_write(adapter, A_ELMER0_GPO, val); + + spin_unlock(&adapter->tpi_lock); + + return 0; +} + +static inline void t1_sw_reset(struct pci_dev *pdev) +{ + pci_write_config_dword(pdev, A_PCICFG_PM_CSR, 3); + pci_write_config_dword(pdev, A_PCICFG_PM_CSR, 0); +} + +static void remove_one(struct pci_dev *pdev) +{ + struct net_device *dev = pci_get_drvdata(pdev); + struct adapter *adapter = dev->ml_priv; + int i; + + for_each_port(adapter, i) { + if (test_bit(i, &adapter->registered_device_map)) + unregister_netdev(adapter->port[i].dev); + } + + t1_free_sw_modules(adapter); + iounmap(adapter->regs); + + while (--i >= 0) { + if (adapter->port[i].dev) + free_netdev(adapter->port[i].dev); + } + + pci_release_regions(pdev); + pci_disable_device(pdev); + t1_sw_reset(pdev); +} + +static struct pci_driver cxgb_pci_driver = { + .name = DRV_NAME, + .id_table = t1_pci_tbl, + .probe = init_one, + .remove = remove_one, +}; + +module_pci_driver(cxgb_pci_driver); diff --git a/drivers/net/ethernet/chelsio/cxgb/elmer0.h b/drivers/net/ethernet/chelsio/cxgb/elmer0.h new file mode 100644 index 000000000..0427e894c --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/elmer0.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/***************************************************************************** + * * + * File: elmer0.h * + * $Revision: 1.6 $ * + * $Date: 2005/06/21 22:49:43 $ * + * Description: * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#ifndef _CXGB_ELMER0_H_ +#define _CXGB_ELMER0_H_ + +/* ELMER0 flavors */ +enum { + ELMER0_XC2S300E_6FT256_C, + ELMER0_XC2S100E_6TQ144_C +}; + +/* ELMER0 registers */ +#define A_ELMER0_VERSION 0x100000 +#define A_ELMER0_PHY_CFG 0x100004 +#define A_ELMER0_INT_ENABLE 0x100008 +#define A_ELMER0_INT_CAUSE 0x10000c +#define A_ELMER0_GPI_CFG 0x100010 +#define A_ELMER0_GPI_STAT 0x100014 +#define A_ELMER0_GPO 0x100018 +#define A_ELMER0_PORT0_MI1_CFG 0x400000 + +#define S_MI1_MDI_ENABLE 0 +#define V_MI1_MDI_ENABLE(x) ((x) << S_MI1_MDI_ENABLE) +#define F_MI1_MDI_ENABLE V_MI1_MDI_ENABLE(1U) + +#define S_MI1_MDI_INVERT 1 +#define V_MI1_MDI_INVERT(x) ((x) << S_MI1_MDI_INVERT) +#define F_MI1_MDI_INVERT V_MI1_MDI_INVERT(1U) + +#define S_MI1_PREAMBLE_ENABLE 2 +#define V_MI1_PREAMBLE_ENABLE(x) ((x) << S_MI1_PREAMBLE_ENABLE) +#define F_MI1_PREAMBLE_ENABLE V_MI1_PREAMBLE_ENABLE(1U) + +#define S_MI1_SOF 3 +#define M_MI1_SOF 0x3 +#define V_MI1_SOF(x) ((x) << S_MI1_SOF) +#define G_MI1_SOF(x) (((x) >> S_MI1_SOF) & M_MI1_SOF) + +#define S_MI1_CLK_DIV 5 +#define M_MI1_CLK_DIV 0xff +#define V_MI1_CLK_DIV(x) ((x) << S_MI1_CLK_DIV) +#define G_MI1_CLK_DIV(x) (((x) >> S_MI1_CLK_DIV) & M_MI1_CLK_DIV) + +#define A_ELMER0_PORT0_MI1_ADDR 0x400004 + +#define S_MI1_REG_ADDR 0 +#define M_MI1_REG_ADDR 0x1f +#define V_MI1_REG_ADDR(x) ((x) << S_MI1_REG_ADDR) +#define G_MI1_REG_ADDR(x) (((x) >> S_MI1_REG_ADDR) & M_MI1_REG_ADDR) + +#define S_MI1_PHY_ADDR 5 +#define M_MI1_PHY_ADDR 0x1f +#define V_MI1_PHY_ADDR(x) ((x) << S_MI1_PHY_ADDR) +#define G_MI1_PHY_ADDR(x) (((x) >> S_MI1_PHY_ADDR) & M_MI1_PHY_ADDR) + +#define A_ELMER0_PORT0_MI1_DATA 0x400008 + +#define S_MI1_DATA 0 +#define M_MI1_DATA 0xffff +#define V_MI1_DATA(x) ((x) << S_MI1_DATA) +#define G_MI1_DATA(x) (((x) >> S_MI1_DATA) & M_MI1_DATA) + +#define A_ELMER0_PORT0_MI1_OP 0x40000c + +#define S_MI1_OP 0 +#define M_MI1_OP 0x3 +#define V_MI1_OP(x) ((x) << S_MI1_OP) +#define G_MI1_OP(x) (((x) >> S_MI1_OP) & M_MI1_OP) + +#define S_MI1_ADDR_AUTOINC 2 +#define V_MI1_ADDR_AUTOINC(x) ((x) << S_MI1_ADDR_AUTOINC) +#define F_MI1_ADDR_AUTOINC V_MI1_ADDR_AUTOINC(1U) + +#define S_MI1_OP_BUSY 31 +#define V_MI1_OP_BUSY(x) ((x) << S_MI1_OP_BUSY) +#define F_MI1_OP_BUSY V_MI1_OP_BUSY(1U) + +#define A_ELMER0_PORT1_MI1_CFG 0x500000 +#define A_ELMER0_PORT1_MI1_ADDR 0x500004 +#define A_ELMER0_PORT1_MI1_DATA 0x500008 +#define A_ELMER0_PORT1_MI1_OP 0x50000c +#define A_ELMER0_PORT2_MI1_CFG 0x600000 +#define A_ELMER0_PORT2_MI1_ADDR 0x600004 +#define A_ELMER0_PORT2_MI1_DATA 0x600008 +#define A_ELMER0_PORT2_MI1_OP 0x60000c +#define A_ELMER0_PORT3_MI1_CFG 0x700000 +#define A_ELMER0_PORT3_MI1_ADDR 0x700004 +#define A_ELMER0_PORT3_MI1_DATA 0x700008 +#define A_ELMER0_PORT3_MI1_OP 0x70000c + +/* Simple bit definition for GPI and GP0 registers. */ +#define ELMER0_GP_BIT0 0x0001 +#define ELMER0_GP_BIT1 0x0002 +#define ELMER0_GP_BIT2 0x0004 +#define ELMER0_GP_BIT3 0x0008 +#define ELMER0_GP_BIT4 0x0010 +#define ELMER0_GP_BIT5 0x0020 +#define ELMER0_GP_BIT6 0x0040 +#define ELMER0_GP_BIT7 0x0080 +#define ELMER0_GP_BIT8 0x0100 +#define ELMER0_GP_BIT9 0x0200 +#define ELMER0_GP_BIT10 0x0400 +#define ELMER0_GP_BIT11 0x0800 +#define ELMER0_GP_BIT12 0x1000 +#define ELMER0_GP_BIT13 0x2000 +#define ELMER0_GP_BIT14 0x4000 +#define ELMER0_GP_BIT15 0x8000 +#define ELMER0_GP_BIT16 0x10000 +#define ELMER0_GP_BIT17 0x20000 +#define ELMER0_GP_BIT18 0x40000 +#define ELMER0_GP_BIT19 0x80000 + +#define MI1_OP_DIRECT_WRITE 1 +#define MI1_OP_DIRECT_READ 2 + +#define MI1_OP_INDIRECT_ADDRESS 0 +#define MI1_OP_INDIRECT_WRITE 1 +#define MI1_OP_INDIRECT_READ_INC 2 +#define MI1_OP_INDIRECT_READ 3 + +#endif /* _CXGB_ELMER0_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb/espi.c b/drivers/net/ethernet/chelsio/cxgb/espi.c new file mode 100644 index 000000000..ef7056943 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/espi.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: GPL-2.0-only +/***************************************************************************** + * * + * File: espi.c * + * $Revision: 1.14 $ * + * $Date: 2005/05/14 00:59:32 $ * + * Description: * + * Ethernet SPI functionality. * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#include "common.h" +#include "regs.h" +#include "espi.h" + +struct peespi { + adapter_t *adapter; + struct espi_intr_counts intr_cnt; + u32 misc_ctrl; + spinlock_t lock; +}; + +#define ESPI_INTR_MASK (F_DIP4ERR | F_RXDROP | F_TXDROP | F_RXOVERFLOW | \ + F_RAMPARITYERR | F_DIP2PARITYERR) +#define MON_MASK (V_MONITORED_PORT_NUM(3) | F_MONITORED_DIRECTION \ + | F_MONITORED_INTERFACE) + +#define TRICN_CNFG 14 +#define TRICN_CMD_READ 0x11 +#define TRICN_CMD_WRITE 0x21 +#define TRICN_CMD_ATTEMPTS 10 + +static int tricn_write(adapter_t *adapter, int bundle_addr, int module_addr, + int ch_addr, int reg_offset, u32 wr_data) +{ + int busy, attempts = TRICN_CMD_ATTEMPTS; + + writel(V_WRITE_DATA(wr_data) | + V_REGISTER_OFFSET(reg_offset) | + V_CHANNEL_ADDR(ch_addr) | V_MODULE_ADDR(module_addr) | + V_BUNDLE_ADDR(bundle_addr) | + V_SPI4_COMMAND(TRICN_CMD_WRITE), + adapter->regs + A_ESPI_CMD_ADDR); + writel(0, adapter->regs + A_ESPI_GOSTAT); + + do { + busy = readl(adapter->regs + A_ESPI_GOSTAT) & F_ESPI_CMD_BUSY; + } while (busy && --attempts); + + if (busy) + pr_err("%s: TRICN write timed out\n", adapter->name); + + return busy; +} + +static int tricn_init(adapter_t *adapter) +{ + int i, sme = 1; + + if (!(readl(adapter->regs + A_ESPI_RX_RESET) & F_RX_CLK_STATUS)) { + pr_err("%s: ESPI clock not ready\n", adapter->name); + return -1; + } + + writel(F_ESPI_RX_CORE_RST, adapter->regs + A_ESPI_RX_RESET); + + if (sme) { + tricn_write(adapter, 0, 0, 0, TRICN_CNFG, 0x81); + tricn_write(adapter, 0, 1, 0, TRICN_CNFG, 0x81); + tricn_write(adapter, 0, 2, 0, TRICN_CNFG, 0x81); + } + for (i = 1; i <= 8; i++) + tricn_write(adapter, 0, 0, i, TRICN_CNFG, 0xf1); + for (i = 1; i <= 2; i++) + tricn_write(adapter, 0, 1, i, TRICN_CNFG, 0xf1); + for (i = 1; i <= 3; i++) + tricn_write(adapter, 0, 2, i, TRICN_CNFG, 0xe1); + tricn_write(adapter, 0, 2, 4, TRICN_CNFG, 0xf1); + tricn_write(adapter, 0, 2, 5, TRICN_CNFG, 0xe1); + tricn_write(adapter, 0, 2, 6, TRICN_CNFG, 0xf1); + tricn_write(adapter, 0, 2, 7, TRICN_CNFG, 0x80); + tricn_write(adapter, 0, 2, 8, TRICN_CNFG, 0xf1); + + writel(F_ESPI_RX_CORE_RST | F_ESPI_RX_LNK_RST, + adapter->regs + A_ESPI_RX_RESET); + + return 0; +} + +void t1_espi_intr_enable(struct peespi *espi) +{ + u32 enable, pl_intr = readl(espi->adapter->regs + A_PL_ENABLE); + + /* + * Cannot enable ESPI interrupts on T1B because HW asserts the + * interrupt incorrectly, namely the driver gets ESPI interrupts + * but no data is actually dropped (can verify this reading the ESPI + * drop registers). Also, once the ESPI interrupt is asserted it + * cannot be cleared (HW bug). + */ + enable = t1_is_T1B(espi->adapter) ? 0 : ESPI_INTR_MASK; + writel(enable, espi->adapter->regs + A_ESPI_INTR_ENABLE); + writel(pl_intr | F_PL_INTR_ESPI, espi->adapter->regs + A_PL_ENABLE); +} + +void t1_espi_intr_clear(struct peespi *espi) +{ + readl(espi->adapter->regs + A_ESPI_DIP2_ERR_COUNT); + writel(0xffffffff, espi->adapter->regs + A_ESPI_INTR_STATUS); + writel(F_PL_INTR_ESPI, espi->adapter->regs + A_PL_CAUSE); +} + +void t1_espi_intr_disable(struct peespi *espi) +{ + u32 pl_intr = readl(espi->adapter->regs + A_PL_ENABLE); + + writel(0, espi->adapter->regs + A_ESPI_INTR_ENABLE); + writel(pl_intr & ~F_PL_INTR_ESPI, espi->adapter->regs + A_PL_ENABLE); +} + +int t1_espi_intr_handler(struct peespi *espi) +{ + u32 status = readl(espi->adapter->regs + A_ESPI_INTR_STATUS); + + if (status & F_DIP4ERR) + espi->intr_cnt.DIP4_err++; + if (status & F_RXDROP) + espi->intr_cnt.rx_drops++; + if (status & F_TXDROP) + espi->intr_cnt.tx_drops++; + if (status & F_RXOVERFLOW) + espi->intr_cnt.rx_ovflw++; + if (status & F_RAMPARITYERR) + espi->intr_cnt.parity_err++; + if (status & F_DIP2PARITYERR) { + espi->intr_cnt.DIP2_parity_err++; + + /* + * Must read the error count to clear the interrupt + * that it causes. + */ + readl(espi->adapter->regs + A_ESPI_DIP2_ERR_COUNT); + } + + /* + * For T1B we need to write 1 to clear ESPI interrupts. For T2+ we + * write the status as is. + */ + if (status && t1_is_T1B(espi->adapter)) + status = 1; + writel(status, espi->adapter->regs + A_ESPI_INTR_STATUS); + return 0; +} + +const struct espi_intr_counts *t1_espi_get_intr_counts(struct peespi *espi) +{ + return &espi->intr_cnt; +} + +static void espi_setup_for_pm3393(adapter_t *adapter) +{ + u32 wmark = t1_is_T1B(adapter) ? 0x4000 : 0x3200; + + writel(0x1f4, adapter->regs + A_ESPI_SCH_TOKEN0); + writel(0x1f4, adapter->regs + A_ESPI_SCH_TOKEN1); + writel(0x1f4, adapter->regs + A_ESPI_SCH_TOKEN2); + writel(0x1f4, adapter->regs + A_ESPI_SCH_TOKEN3); + writel(0x100, adapter->regs + A_ESPI_RX_FIFO_ALMOST_EMPTY_WATERMARK); + writel(wmark, adapter->regs + A_ESPI_RX_FIFO_ALMOST_FULL_WATERMARK); + writel(3, adapter->regs + A_ESPI_CALENDAR_LENGTH); + writel(0x08000008, adapter->regs + A_ESPI_TRAIN); + writel(V_RX_NPORTS(1) | V_TX_NPORTS(1), adapter->regs + A_PORT_CONFIG); +} + +static void espi_setup_for_vsc7321(adapter_t *adapter) +{ + writel(0x1f4, adapter->regs + A_ESPI_SCH_TOKEN0); + writel(0x1f401f4, adapter->regs + A_ESPI_SCH_TOKEN1); + writel(0x1f4, adapter->regs + A_ESPI_SCH_TOKEN2); + writel(0xa00, adapter->regs + A_ESPI_RX_FIFO_ALMOST_FULL_WATERMARK); + writel(0x1ff, adapter->regs + A_ESPI_RX_FIFO_ALMOST_EMPTY_WATERMARK); + writel(1, adapter->regs + A_ESPI_CALENDAR_LENGTH); + writel(V_RX_NPORTS(4) | V_TX_NPORTS(4), adapter->regs + A_PORT_CONFIG); + + writel(0x08000008, adapter->regs + A_ESPI_TRAIN); +} + +/* + * Note that T1B requires at least 2 ports for IXF1010 due to a HW bug. + */ +static void espi_setup_for_ixf1010(adapter_t *adapter, int nports) +{ + writel(1, adapter->regs + A_ESPI_CALENDAR_LENGTH); + if (nports == 4) { + if (is_T2(adapter)) { + writel(0xf00, adapter->regs + A_ESPI_RX_FIFO_ALMOST_FULL_WATERMARK); + writel(0x3c0, adapter->regs + A_ESPI_RX_FIFO_ALMOST_EMPTY_WATERMARK); + } else { + writel(0x7ff, adapter->regs + A_ESPI_RX_FIFO_ALMOST_FULL_WATERMARK); + writel(0x1ff, adapter->regs + A_ESPI_RX_FIFO_ALMOST_EMPTY_WATERMARK); + } + } else { + writel(0x1fff, adapter->regs + A_ESPI_RX_FIFO_ALMOST_FULL_WATERMARK); + writel(0x7ff, adapter->regs + A_ESPI_RX_FIFO_ALMOST_EMPTY_WATERMARK); + } + writel(V_RX_NPORTS(nports) | V_TX_NPORTS(nports), adapter->regs + A_PORT_CONFIG); + +} + +int t1_espi_init(struct peespi *espi, int mac_type, int nports) +{ + u32 status_enable_extra = 0; + adapter_t *adapter = espi->adapter; + + /* Disable ESPI training. MACs that can handle it enable it below. */ + writel(0, adapter->regs + A_ESPI_TRAIN); + + if (is_T2(adapter)) { + writel(V_OUT_OF_SYNC_COUNT(4) | + V_DIP2_PARITY_ERR_THRES(3) | + V_DIP4_THRES(1), adapter->regs + A_ESPI_MISC_CONTROL); + writel(nports == 4 ? 0x200040 : 0x1000080, + adapter->regs + A_ESPI_MAXBURST1_MAXBURST2); + } else + writel(0x800100, adapter->regs + A_ESPI_MAXBURST1_MAXBURST2); + + if (mac_type == CHBT_MAC_PM3393) + espi_setup_for_pm3393(adapter); + else if (mac_type == CHBT_MAC_VSC7321) + espi_setup_for_vsc7321(adapter); + else if (mac_type == CHBT_MAC_IXF1010) { + status_enable_extra = F_INTEL1010MODE; + espi_setup_for_ixf1010(adapter, nports); + } else + return -1; + + writel(status_enable_extra | F_RXSTATUSENABLE, + adapter->regs + A_ESPI_FIFO_STATUS_ENABLE); + + if (is_T2(adapter)) { + tricn_init(adapter); + /* + * Always position the control at the 1st port egress IN + * (sop,eop) counter to reduce PIOs for T/N210 workaround. + */ + espi->misc_ctrl = readl(adapter->regs + A_ESPI_MISC_CONTROL); + espi->misc_ctrl &= ~MON_MASK; + espi->misc_ctrl |= F_MONITORED_DIRECTION; + if (adapter->params.nports == 1) + espi->misc_ctrl |= F_MONITORED_INTERFACE; + writel(espi->misc_ctrl, adapter->regs + A_ESPI_MISC_CONTROL); + spin_lock_init(&espi->lock); + } + + return 0; +} + +void t1_espi_destroy(struct peespi *espi) +{ + kfree(espi); +} + +struct peespi *t1_espi_create(adapter_t *adapter) +{ + struct peespi *espi = kzalloc(sizeof(*espi), GFP_KERNEL); + + if (espi) + espi->adapter = adapter; + return espi; +} + +#if 0 +void t1_espi_set_misc_ctrl(adapter_t *adapter, u32 val) +{ + struct peespi *espi = adapter->espi; + + if (!is_T2(adapter)) + return; + spin_lock(&espi->lock); + espi->misc_ctrl = (val & ~MON_MASK) | + (espi->misc_ctrl & MON_MASK); + writel(espi->misc_ctrl, adapter->regs + A_ESPI_MISC_CONTROL); + spin_unlock(&espi->lock); +} +#endif /* 0 */ + +u32 t1_espi_get_mon(adapter_t *adapter, u32 addr, u8 wait) +{ + struct peespi *espi = adapter->espi; + u32 sel; + + if (!is_T2(adapter)) + return 0; + + sel = V_MONITORED_PORT_NUM((addr & 0x3c) >> 2); + if (!wait) { + if (!spin_trylock(&espi->lock)) + return 0; + } else + spin_lock(&espi->lock); + + if ((sel != (espi->misc_ctrl & MON_MASK))) { + writel(((espi->misc_ctrl & ~MON_MASK) | sel), + adapter->regs + A_ESPI_MISC_CONTROL); + sel = readl(adapter->regs + A_ESPI_SCH_TOKEN3); + writel(espi->misc_ctrl, adapter->regs + A_ESPI_MISC_CONTROL); + } else + sel = readl(adapter->regs + A_ESPI_SCH_TOKEN3); + spin_unlock(&espi->lock); + return sel; +} + +/* + * This function is for T204 only. + * compare with t1_espi_get_mon(), it reads espiInTxSop[0 ~ 3] in + * one shot, since there is no per port counter on the out side. + */ +int t1_espi_get_mon_t204(adapter_t *adapter, u32 *valp, u8 wait) +{ + struct peespi *espi = adapter->espi; + u8 i, nport = (u8)adapter->params.nports; + + if (!wait) { + if (!spin_trylock(&espi->lock)) + return -1; + } else + spin_lock(&espi->lock); + + if ((espi->misc_ctrl & MON_MASK) != F_MONITORED_DIRECTION) { + espi->misc_ctrl = (espi->misc_ctrl & ~MON_MASK) | + F_MONITORED_DIRECTION; + writel(espi->misc_ctrl, adapter->regs + A_ESPI_MISC_CONTROL); + } + for (i = 0 ; i < nport; i++, valp++) { + if (i) { + writel(espi->misc_ctrl | V_MONITORED_PORT_NUM(i), + adapter->regs + A_ESPI_MISC_CONTROL); + } + *valp = readl(adapter->regs + A_ESPI_SCH_TOKEN3); + } + + writel(espi->misc_ctrl, adapter->regs + A_ESPI_MISC_CONTROL); + spin_unlock(&espi->lock); + return 0; +} diff --git a/drivers/net/ethernet/chelsio/cxgb/espi.h b/drivers/net/ethernet/chelsio/cxgb/espi.h new file mode 100644 index 000000000..f588e9f3b --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/espi.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/***************************************************************************** + * * + * File: espi.h * + * $Revision: 1.7 $ * + * $Date: 2005/06/21 18:29:47 $ * + * Description: * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#ifndef _CXGB_ESPI_H_ +#define _CXGB_ESPI_H_ + +#include "common.h" + +struct espi_intr_counts { + unsigned int DIP4_err; + unsigned int rx_drops; + unsigned int tx_drops; + unsigned int rx_ovflw; + unsigned int parity_err; + unsigned int DIP2_parity_err; +}; + +struct peespi; + +struct peespi *t1_espi_create(adapter_t *adapter); +void t1_espi_destroy(struct peespi *espi); +int t1_espi_init(struct peespi *espi, int mac_type, int nports); + +void t1_espi_intr_enable(struct peespi *); +void t1_espi_intr_clear(struct peespi *); +void t1_espi_intr_disable(struct peespi *); +int t1_espi_intr_handler(struct peespi *); +const struct espi_intr_counts *t1_espi_get_intr_counts(struct peespi *espi); + +u32 t1_espi_get_mon(adapter_t *adapter, u32 addr, u8 wait); +int t1_espi_get_mon_t204(adapter_t *, u32 *, u8); + +#endif /* _CXGB_ESPI_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb/fpga_defs.h b/drivers/net/ethernet/chelsio/cxgb/fpga_defs.h new file mode 100644 index 000000000..e9c65d812 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/fpga_defs.h @@ -0,0 +1,233 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* $Date: 2005/03/07 23:59:05 $ $RCSfile: fpga_defs.h,v $ $Revision: 1.4 $ */ + +/* + * FPGA specific definitions + */ + +#ifndef __CHELSIO_FPGA_DEFS_H__ +#define __CHELSIO_FPGA_DEFS_H__ + +#define FPGA_PCIX_ADDR_VERSION 0xA08 +#define FPGA_PCIX_ADDR_STAT 0xA0C + +/* FPGA master interrupt Cause/Enable bits */ +#define FPGA_PCIX_INTERRUPT_SGE_ERROR 0x1 +#define FPGA_PCIX_INTERRUPT_SGE_DATA 0x2 +#define FPGA_PCIX_INTERRUPT_TP 0x4 +#define FPGA_PCIX_INTERRUPT_MC3 0x8 +#define FPGA_PCIX_INTERRUPT_GMAC 0x10 +#define FPGA_PCIX_INTERRUPT_PCIX 0x20 + +/* TP interrupt register addresses */ +#define FPGA_TP_ADDR_INTERRUPT_ENABLE 0xA10 +#define FPGA_TP_ADDR_INTERRUPT_CAUSE 0xA14 +#define FPGA_TP_ADDR_VERSION 0xA18 + +/* TP interrupt Cause/Enable bits */ +#define FPGA_TP_INTERRUPT_MC4 0x1 +#define FPGA_TP_INTERRUPT_MC5 0x2 + +/* + * PM interrupt register addresses + */ +#define FPGA_MC3_REG_INTRENABLE 0xA20 +#define FPGA_MC3_REG_INTRCAUSE 0xA24 +#define FPGA_MC3_REG_VERSION 0xA28 + +/* + * GMAC interrupt register addresses + */ +#define FPGA_GMAC_ADDR_INTERRUPT_ENABLE 0xA30 +#define FPGA_GMAC_ADDR_INTERRUPT_CAUSE 0xA34 +#define FPGA_GMAC_ADDR_VERSION 0xA38 + +/* GMAC Cause/Enable bits */ +#define FPGA_GMAC_INTERRUPT_PORT0 0x1 +#define FPGA_GMAC_INTERRUPT_PORT1 0x2 +#define FPGA_GMAC_INTERRUPT_PORT2 0x4 +#define FPGA_GMAC_INTERRUPT_PORT3 0x8 + +/* MI0 registers */ +#define A_MI0_CLK 0xb00 + +#define S_MI0_CLK_DIV 0 +#define M_MI0_CLK_DIV 0xff +#define V_MI0_CLK_DIV(x) ((x) << S_MI0_CLK_DIV) +#define G_MI0_CLK_DIV(x) (((x) >> S_MI0_CLK_DIV) & M_MI0_CLK_DIV) + +#define S_MI0_CLK_CNT 8 +#define M_MI0_CLK_CNT 0xff +#define V_MI0_CLK_CNT(x) ((x) << S_MI0_CLK_CNT) +#define G_MI0_CLK_CNT(x) (((x) >> S_MI0_CLK_CNT) & M_MI0_CLK_CNT) + +#define A_MI0_CSR 0xb04 + +#define S_MI0_CSR_POLL 0 +#define V_MI0_CSR_POLL(x) ((x) << S_MI0_CSR_POLL) +#define F_MI0_CSR_POLL V_MI0_CSR_POLL(1U) + +#define S_MI0_PREAMBLE 1 +#define V_MI0_PREAMBLE(x) ((x) << S_MI0_PREAMBLE) +#define F_MI0_PREAMBLE V_MI0_PREAMBLE(1U) + +#define S_MI0_INTR_ENABLE 2 +#define V_MI0_INTR_ENABLE(x) ((x) << S_MI0_INTR_ENABLE) +#define F_MI0_INTR_ENABLE V_MI0_INTR_ENABLE(1U) + +#define S_MI0_BUSY 3 +#define V_MI0_BUSY(x) ((x) << S_MI0_BUSY) +#define F_MI0_BUSY V_MI0_BUSY(1U) + +#define S_MI0_MDIO 4 +#define V_MI0_MDIO(x) ((x) << S_MI0_MDIO) +#define F_MI0_MDIO V_MI0_MDIO(1U) + +#define A_MI0_ADDR 0xb08 + +#define S_MI0_PHY_REG_ADDR 0 +#define M_MI0_PHY_REG_ADDR 0x1f +#define V_MI0_PHY_REG_ADDR(x) ((x) << S_MI0_PHY_REG_ADDR) +#define G_MI0_PHY_REG_ADDR(x) (((x) >> S_MI0_PHY_REG_ADDR) & M_MI0_PHY_REG_ADDR) + +#define S_MI0_PHY_ADDR 5 +#define M_MI0_PHY_ADDR 0x1f +#define V_MI0_PHY_ADDR(x) ((x) << S_MI0_PHY_ADDR) +#define G_MI0_PHY_ADDR(x) (((x) >> S_MI0_PHY_ADDR) & M_MI0_PHY_ADDR) + +#define A_MI0_DATA_EXT 0xb0c +#define A_MI0_DATA_INT 0xb10 + +/* GMAC registers */ +#define A_GMAC_MACID_LO 0x28 +#define A_GMAC_MACID_HI 0x2c +#define A_GMAC_CSR 0x30 + +#define S_INTERFACE 0 +#define M_INTERFACE 0x3 +#define V_INTERFACE(x) ((x) << S_INTERFACE) +#define G_INTERFACE(x) (((x) >> S_INTERFACE) & M_INTERFACE) + +#define S_MAC_TX_ENABLE 2 +#define V_MAC_TX_ENABLE(x) ((x) << S_MAC_TX_ENABLE) +#define F_MAC_TX_ENABLE V_MAC_TX_ENABLE(1U) + +#define S_MAC_RX_ENABLE 3 +#define V_MAC_RX_ENABLE(x) ((x) << S_MAC_RX_ENABLE) +#define F_MAC_RX_ENABLE V_MAC_RX_ENABLE(1U) + +#define S_MAC_LB_ENABLE 4 +#define V_MAC_LB_ENABLE(x) ((x) << S_MAC_LB_ENABLE) +#define F_MAC_LB_ENABLE V_MAC_LB_ENABLE(1U) + +#define S_MAC_SPEED 5 +#define M_MAC_SPEED 0x3 +#define V_MAC_SPEED(x) ((x) << S_MAC_SPEED) +#define G_MAC_SPEED(x) (((x) >> S_MAC_SPEED) & M_MAC_SPEED) + +#define S_MAC_HD_FC_ENABLE 7 +#define V_MAC_HD_FC_ENABLE(x) ((x) << S_MAC_HD_FC_ENABLE) +#define F_MAC_HD_FC_ENABLE V_MAC_HD_FC_ENABLE(1U) + +#define S_MAC_HALF_DUPLEX 8 +#define V_MAC_HALF_DUPLEX(x) ((x) << S_MAC_HALF_DUPLEX) +#define F_MAC_HALF_DUPLEX V_MAC_HALF_DUPLEX(1U) + +#define S_MAC_PROMISC 9 +#define V_MAC_PROMISC(x) ((x) << S_MAC_PROMISC) +#define F_MAC_PROMISC V_MAC_PROMISC(1U) + +#define S_MAC_MC_ENABLE 10 +#define V_MAC_MC_ENABLE(x) ((x) << S_MAC_MC_ENABLE) +#define F_MAC_MC_ENABLE V_MAC_MC_ENABLE(1U) + +#define S_MAC_RESET 11 +#define V_MAC_RESET(x) ((x) << S_MAC_RESET) +#define F_MAC_RESET V_MAC_RESET(1U) + +#define S_MAC_RX_PAUSE_ENABLE 12 +#define V_MAC_RX_PAUSE_ENABLE(x) ((x) << S_MAC_RX_PAUSE_ENABLE) +#define F_MAC_RX_PAUSE_ENABLE V_MAC_RX_PAUSE_ENABLE(1U) + +#define S_MAC_TX_PAUSE_ENABLE 13 +#define V_MAC_TX_PAUSE_ENABLE(x) ((x) << S_MAC_TX_PAUSE_ENABLE) +#define F_MAC_TX_PAUSE_ENABLE V_MAC_TX_PAUSE_ENABLE(1U) + +#define S_MAC_LWM_ENABLE 14 +#define V_MAC_LWM_ENABLE(x) ((x) << S_MAC_LWM_ENABLE) +#define F_MAC_LWM_ENABLE V_MAC_LWM_ENABLE(1U) + +#define S_MAC_MAGIC_PKT_ENABLE 15 +#define V_MAC_MAGIC_PKT_ENABLE(x) ((x) << S_MAC_MAGIC_PKT_ENABLE) +#define F_MAC_MAGIC_PKT_ENABLE V_MAC_MAGIC_PKT_ENABLE(1U) + +#define S_MAC_ISL_ENABLE 16 +#define V_MAC_ISL_ENABLE(x) ((x) << S_MAC_ISL_ENABLE) +#define F_MAC_ISL_ENABLE V_MAC_ISL_ENABLE(1U) + +#define S_MAC_JUMBO_ENABLE 17 +#define V_MAC_JUMBO_ENABLE(x) ((x) << S_MAC_JUMBO_ENABLE) +#define F_MAC_JUMBO_ENABLE V_MAC_JUMBO_ENABLE(1U) + +#define S_MAC_RX_PAD_ENABLE 18 +#define V_MAC_RX_PAD_ENABLE(x) ((x) << S_MAC_RX_PAD_ENABLE) +#define F_MAC_RX_PAD_ENABLE V_MAC_RX_PAD_ENABLE(1U) + +#define S_MAC_RX_CRC_ENABLE 19 +#define V_MAC_RX_CRC_ENABLE(x) ((x) << S_MAC_RX_CRC_ENABLE) +#define F_MAC_RX_CRC_ENABLE V_MAC_RX_CRC_ENABLE(1U) + +#define A_GMAC_IFS 0x34 + +#define S_MAC_IFS2 0 +#define M_MAC_IFS2 0x3f +#define V_MAC_IFS2(x) ((x) << S_MAC_IFS2) +#define G_MAC_IFS2(x) (((x) >> S_MAC_IFS2) & M_MAC_IFS2) + +#define S_MAC_IFS1 8 +#define M_MAC_IFS1 0x7f +#define V_MAC_IFS1(x) ((x) << S_MAC_IFS1) +#define G_MAC_IFS1(x) (((x) >> S_MAC_IFS1) & M_MAC_IFS1) + +#define A_GMAC_JUMBO_FRAME_LEN 0x38 +#define A_GMAC_LNK_DLY 0x3c +#define A_GMAC_PAUSETIME 0x40 +#define A_GMAC_MCAST_LO 0x44 +#define A_GMAC_MCAST_HI 0x48 +#define A_GMAC_MCAST_MASK_LO 0x4c +#define A_GMAC_MCAST_MASK_HI 0x50 +#define A_GMAC_RMT_CNT 0x54 +#define A_GMAC_RMT_DATA 0x58 +#define A_GMAC_BACKOFF_SEED 0x5c +#define A_GMAC_TXF_THRES 0x60 + +#define S_TXF_READ_THRESHOLD 0 +#define M_TXF_READ_THRESHOLD 0xff +#define V_TXF_READ_THRESHOLD(x) ((x) << S_TXF_READ_THRESHOLD) +#define G_TXF_READ_THRESHOLD(x) (((x) >> S_TXF_READ_THRESHOLD) & M_TXF_READ_THRESHOLD) + +#define S_TXF_WRITE_THRESHOLD 16 +#define M_TXF_WRITE_THRESHOLD 0xff +#define V_TXF_WRITE_THRESHOLD(x) ((x) << S_TXF_WRITE_THRESHOLD) +#define G_TXF_WRITE_THRESHOLD(x) (((x) >> S_TXF_WRITE_THRESHOLD) & M_TXF_WRITE_THRESHOLD) + +#define MAC_REG_BASE 0x600 +#define MAC_REG_ADDR(idx, reg) (MAC_REG_BASE + (idx) * 128 + (reg)) + +#define MAC_REG_IDLO(idx) MAC_REG_ADDR(idx, A_GMAC_MACID_LO) +#define MAC_REG_IDHI(idx) MAC_REG_ADDR(idx, A_GMAC_MACID_HI) +#define MAC_REG_CSR(idx) MAC_REG_ADDR(idx, A_GMAC_CSR) +#define MAC_REG_IFS(idx) MAC_REG_ADDR(idx, A_GMAC_IFS) +#define MAC_REG_LARGEFRAMELENGTH(idx) MAC_REG_ADDR(idx, A_GMAC_JUMBO_FRAME_LEN) +#define MAC_REG_LINKDLY(idx) MAC_REG_ADDR(idx, A_GMAC_LNK_DLY) +#define MAC_REG_PAUSETIME(idx) MAC_REG_ADDR(idx, A_GMAC_PAUSETIME) +#define MAC_REG_CASTLO(idx) MAC_REG_ADDR(idx, A_GMAC_MCAST_LO) +#define MAC_REG_MCASTHI(idx) MAC_REG_ADDR(idx, A_GMAC_MCAST_HI) +#define MAC_REG_CASTMASKLO(idx) MAC_REG_ADDR(idx, A_GMAC_MCAST_MASK_LO) +#define MAC_REG_MCASTMASKHI(idx) MAC_REG_ADDR(idx, A_GMAC_MCAST_MASK_HI) +#define MAC_REG_RMCNT(idx) MAC_REG_ADDR(idx, A_GMAC_RMT_CNT) +#define MAC_REG_RMDATA(idx) MAC_REG_ADDR(idx, A_GMAC_RMT_DATA) +#define MAC_REG_GMRANDBACKOFFSEED(idx) MAC_REG_ADDR(idx, A_GMAC_BACKOFF_SEED) +#define MAC_REG_TXFTHRESHOLDS(idx) MAC_REG_ADDR(idx, A_GMAC_TXF_THRES) + +#endif diff --git a/drivers/net/ethernet/chelsio/cxgb/gmac.h b/drivers/net/ethernet/chelsio/cxgb/gmac.h new file mode 100644 index 000000000..96077da1e --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/gmac.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/***************************************************************************** + * * + * File: gmac.h * + * $Revision: 1.6 $ * + * $Date: 2005/06/21 18:29:47 $ * + * Description: * + * Generic MAC functionality. * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#ifndef _CXGB_GMAC_H_ +#define _CXGB_GMAC_H_ + +#include "common.h" + +enum { + MAC_STATS_UPDATE_FAST, + MAC_STATS_UPDATE_FULL +}; + +enum { + MAC_DIRECTION_RX = 1, + MAC_DIRECTION_TX = 2 +}; + +struct cmac_statistics { + /* Transmit */ + u64 TxOctetsOK; + u64 TxOctetsBad; + u64 TxUnicastFramesOK; + u64 TxMulticastFramesOK; + u64 TxBroadcastFramesOK; + u64 TxPauseFrames; + u64 TxFramesWithDeferredXmissions; + u64 TxLateCollisions; + u64 TxTotalCollisions; + u64 TxFramesAbortedDueToXSCollisions; + u64 TxUnderrun; + u64 TxLengthErrors; + u64 TxInternalMACXmitError; + u64 TxFramesWithExcessiveDeferral; + u64 TxFCSErrors; + u64 TxJumboFramesOK; + u64 TxJumboOctetsOK; + + /* Receive */ + u64 RxOctetsOK; + u64 RxOctetsBad; + u64 RxUnicastFramesOK; + u64 RxMulticastFramesOK; + u64 RxBroadcastFramesOK; + u64 RxPauseFrames; + u64 RxFCSErrors; + u64 RxAlignErrors; + u64 RxSymbolErrors; + u64 RxDataErrors; + u64 RxSequenceErrors; + u64 RxRuntErrors; + u64 RxJabberErrors; + u64 RxInternalMACRcvError; + u64 RxInRangeLengthErrors; + u64 RxOutOfRangeLengthField; + u64 RxFrameTooLongErrors; + u64 RxJumboFramesOK; + u64 RxJumboOctetsOK; +}; + +struct cmac_ops { + void (*destroy)(struct cmac *); + int (*reset)(struct cmac *); + int (*interrupt_enable)(struct cmac *); + int (*interrupt_disable)(struct cmac *); + int (*interrupt_clear)(struct cmac *); + int (*interrupt_handler)(struct cmac *); + + int (*enable)(struct cmac *, int); + int (*disable)(struct cmac *, int); + + int (*loopback_enable)(struct cmac *); + int (*loopback_disable)(struct cmac *); + + int (*set_mtu)(struct cmac *, int mtu); + int (*set_rx_mode)(struct cmac *, struct t1_rx_mode *rm); + + int (*set_speed_duplex_fc)(struct cmac *, int speed, int duplex, int fc); + int (*get_speed_duplex_fc)(struct cmac *, int *speed, int *duplex, + int *fc); + + const struct cmac_statistics *(*statistics_update)(struct cmac *, int); + + int (*macaddress_get)(struct cmac *, u8 mac_addr[6]); + int (*macaddress_set)(struct cmac *, const u8 mac_addr[6]); +}; + +typedef struct _cmac_instance cmac_instance; + +struct cmac { + struct cmac_statistics stats; + adapter_t *adapter; + const struct cmac_ops *ops; + cmac_instance *instance; +}; + +struct gmac { + unsigned int stats_update_period; + struct cmac *(*create)(adapter_t *adapter, int index); + int (*reset)(adapter_t *); +}; + +extern const struct gmac t1_pm3393_ops; +extern const struct gmac t1_vsc7326_ops; + +#endif /* _CXGB_GMAC_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb/mv88e1xxx.c b/drivers/net/ethernet/chelsio/cxgb/mv88e1xxx.c new file mode 100644 index 000000000..30b003484 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/mv88e1xxx.c @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-2.0 +/* $Date: 2005/10/24 23:18:13 $ $RCSfile: mv88e1xxx.c,v $ $Revision: 1.49 $ */ +#include "common.h" +#include "mv88e1xxx.h" +#include "cphy.h" +#include "elmer0.h" + +/* MV88E1XXX MDI crossover register values */ +#define CROSSOVER_MDI 0 +#define CROSSOVER_MDIX 1 +#define CROSSOVER_AUTO 3 + +#define INTR_ENABLE_MASK 0x6CA0 + +/* + * Set the bits given by 'bitval' in PHY register 'reg'. + */ +static void mdio_set_bit(struct cphy *cphy, int reg, u32 bitval) +{ + u32 val; + + (void) simple_mdio_read(cphy, reg, &val); + (void) simple_mdio_write(cphy, reg, val | bitval); +} + +/* + * Clear the bits given by 'bitval' in PHY register 'reg'. + */ +static void mdio_clear_bit(struct cphy *cphy, int reg, u32 bitval) +{ + u32 val; + + (void) simple_mdio_read(cphy, reg, &val); + (void) simple_mdio_write(cphy, reg, val & ~bitval); +} + +/* + * NAME: phy_reset + * + * DESC: Reset the given PHY's port. NOTE: This is not a global + * chip reset. + * + * PARAMS: cphy - Pointer to PHY instance data. + * + * RETURN: 0 - Successful reset. + * -1 - Timeout. + */ +static int mv88e1xxx_reset(struct cphy *cphy, int wait) +{ + u32 ctl; + int time_out = 1000; + + mdio_set_bit(cphy, MII_BMCR, BMCR_RESET); + + do { + (void) simple_mdio_read(cphy, MII_BMCR, &ctl); + ctl &= BMCR_RESET; + if (ctl) + udelay(1); + } while (ctl && --time_out); + + return ctl ? -1 : 0; +} + +static int mv88e1xxx_interrupt_enable(struct cphy *cphy) +{ + /* Enable PHY interrupts. */ + (void) simple_mdio_write(cphy, MV88E1XXX_INTERRUPT_ENABLE_REGISTER, + INTR_ENABLE_MASK); + + /* Enable Marvell interrupts through Elmer0. */ + if (t1_is_asic(cphy->adapter)) { + u32 elmer; + + t1_tpi_read(cphy->adapter, A_ELMER0_INT_ENABLE, &elmer); + elmer |= ELMER0_GP_BIT1; + if (is_T2(cphy->adapter)) + elmer |= ELMER0_GP_BIT2 | ELMER0_GP_BIT3 | ELMER0_GP_BIT4; + t1_tpi_write(cphy->adapter, A_ELMER0_INT_ENABLE, elmer); + } + return 0; +} + +static int mv88e1xxx_interrupt_disable(struct cphy *cphy) +{ + /* Disable all phy interrupts. */ + (void) simple_mdio_write(cphy, MV88E1XXX_INTERRUPT_ENABLE_REGISTER, 0); + + /* Disable Marvell interrupts through Elmer0. */ + if (t1_is_asic(cphy->adapter)) { + u32 elmer; + + t1_tpi_read(cphy->adapter, A_ELMER0_INT_ENABLE, &elmer); + elmer &= ~ELMER0_GP_BIT1; + if (is_T2(cphy->adapter)) + elmer &= ~(ELMER0_GP_BIT2|ELMER0_GP_BIT3|ELMER0_GP_BIT4); + t1_tpi_write(cphy->adapter, A_ELMER0_INT_ENABLE, elmer); + } + return 0; +} + +static int mv88e1xxx_interrupt_clear(struct cphy *cphy) +{ + u32 elmer; + + /* Clear PHY interrupts by reading the register. */ + (void) simple_mdio_read(cphy, + MV88E1XXX_INTERRUPT_STATUS_REGISTER, &elmer); + + /* Clear Marvell interrupts through Elmer0. */ + if (t1_is_asic(cphy->adapter)) { + t1_tpi_read(cphy->adapter, A_ELMER0_INT_CAUSE, &elmer); + elmer |= ELMER0_GP_BIT1; + if (is_T2(cphy->adapter)) + elmer |= ELMER0_GP_BIT2|ELMER0_GP_BIT3|ELMER0_GP_BIT4; + t1_tpi_write(cphy->adapter, A_ELMER0_INT_CAUSE, elmer); + } + return 0; +} + +/* + * Set the PHY speed and duplex. This also disables auto-negotiation, except + * for 1Gb/s, where auto-negotiation is mandatory. + */ +static int mv88e1xxx_set_speed_duplex(struct cphy *phy, int speed, int duplex) +{ + u32 ctl; + + (void) simple_mdio_read(phy, MII_BMCR, &ctl); + if (speed >= 0) { + ctl &= ~(BMCR_SPEED100 | BMCR_SPEED1000 | BMCR_ANENABLE); + if (speed == SPEED_100) + ctl |= BMCR_SPEED100; + else if (speed == SPEED_1000) + ctl |= BMCR_SPEED1000; + } + if (duplex >= 0) { + ctl &= ~(BMCR_FULLDPLX | BMCR_ANENABLE); + if (duplex == DUPLEX_FULL) + ctl |= BMCR_FULLDPLX; + } + if (ctl & BMCR_SPEED1000) /* auto-negotiation required for 1Gb/s */ + ctl |= BMCR_ANENABLE; + (void) simple_mdio_write(phy, MII_BMCR, ctl); + return 0; +} + +static int mv88e1xxx_crossover_set(struct cphy *cphy, int crossover) +{ + u32 data32; + + (void) simple_mdio_read(cphy, + MV88E1XXX_SPECIFIC_CNTRL_REGISTER, &data32); + data32 &= ~V_PSCR_MDI_XOVER_MODE(M_PSCR_MDI_XOVER_MODE); + data32 |= V_PSCR_MDI_XOVER_MODE(crossover); + (void) simple_mdio_write(cphy, + MV88E1XXX_SPECIFIC_CNTRL_REGISTER, data32); + return 0; +} + +static int mv88e1xxx_autoneg_enable(struct cphy *cphy) +{ + u32 ctl; + + (void) mv88e1xxx_crossover_set(cphy, CROSSOVER_AUTO); + + (void) simple_mdio_read(cphy, MII_BMCR, &ctl); + /* restart autoneg for change to take effect */ + ctl |= BMCR_ANENABLE | BMCR_ANRESTART; + (void) simple_mdio_write(cphy, MII_BMCR, ctl); + return 0; +} + +static int mv88e1xxx_autoneg_disable(struct cphy *cphy) +{ + u32 ctl; + + /* + * Crossover *must* be set to manual in order to disable auto-neg. + * The Alaska FAQs document highlights this point. + */ + (void) mv88e1xxx_crossover_set(cphy, CROSSOVER_MDI); + + /* + * Must include autoneg reset when disabling auto-neg. This + * is described in the Alaska FAQ document. + */ + (void) simple_mdio_read(cphy, MII_BMCR, &ctl); + ctl &= ~BMCR_ANENABLE; + (void) simple_mdio_write(cphy, MII_BMCR, ctl | BMCR_ANRESTART); + return 0; +} + +static int mv88e1xxx_autoneg_restart(struct cphy *cphy) +{ + mdio_set_bit(cphy, MII_BMCR, BMCR_ANRESTART); + return 0; +} + +static int mv88e1xxx_advertise(struct cphy *phy, unsigned int advertise_map) +{ + u32 val = 0; + + if (advertise_map & + (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) { + (void) simple_mdio_read(phy, MII_GBCR, &val); + val &= ~(GBCR_ADV_1000HALF | GBCR_ADV_1000FULL); + if (advertise_map & ADVERTISED_1000baseT_Half) + val |= GBCR_ADV_1000HALF; + if (advertise_map & ADVERTISED_1000baseT_Full) + val |= GBCR_ADV_1000FULL; + } + (void) simple_mdio_write(phy, MII_GBCR, val); + + val = 1; + if (advertise_map & ADVERTISED_10baseT_Half) + val |= ADVERTISE_10HALF; + if (advertise_map & ADVERTISED_10baseT_Full) + val |= ADVERTISE_10FULL; + if (advertise_map & ADVERTISED_100baseT_Half) + val |= ADVERTISE_100HALF; + if (advertise_map & ADVERTISED_100baseT_Full) + val |= ADVERTISE_100FULL; + if (advertise_map & ADVERTISED_PAUSE) + val |= ADVERTISE_PAUSE; + if (advertise_map & ADVERTISED_ASYM_PAUSE) + val |= ADVERTISE_PAUSE_ASYM; + (void) simple_mdio_write(phy, MII_ADVERTISE, val); + return 0; +} + +static int mv88e1xxx_set_loopback(struct cphy *cphy, int on) +{ + if (on) + mdio_set_bit(cphy, MII_BMCR, BMCR_LOOPBACK); + else + mdio_clear_bit(cphy, MII_BMCR, BMCR_LOOPBACK); + return 0; +} + +static int mv88e1xxx_get_link_status(struct cphy *cphy, int *link_ok, + int *speed, int *duplex, int *fc) +{ + u32 status; + int sp = -1, dplx = -1, pause = 0; + + (void) simple_mdio_read(cphy, + MV88E1XXX_SPECIFIC_STATUS_REGISTER, &status); + if ((status & V_PSSR_STATUS_RESOLVED) != 0) { + if (status & V_PSSR_RX_PAUSE) + pause |= PAUSE_RX; + if (status & V_PSSR_TX_PAUSE) + pause |= PAUSE_TX; + dplx = (status & V_PSSR_DUPLEX) ? DUPLEX_FULL : DUPLEX_HALF; + sp = G_PSSR_SPEED(status); + if (sp == 0) + sp = SPEED_10; + else if (sp == 1) + sp = SPEED_100; + else + sp = SPEED_1000; + } + if (link_ok) + *link_ok = (status & V_PSSR_LINK) != 0; + if (speed) + *speed = sp; + if (duplex) + *duplex = dplx; + if (fc) + *fc = pause; + return 0; +} + +static int mv88e1xxx_downshift_set(struct cphy *cphy, int downshift_enable) +{ + u32 val; + + (void) simple_mdio_read(cphy, + MV88E1XXX_EXT_PHY_SPECIFIC_CNTRL_REGISTER, &val); + + /* + * Set the downshift counter to 2 so we try to establish Gb link + * twice before downshifting. + */ + val &= ~(V_DOWNSHIFT_ENABLE | V_DOWNSHIFT_CNT(M_DOWNSHIFT_CNT)); + + if (downshift_enable) + val |= V_DOWNSHIFT_ENABLE | V_DOWNSHIFT_CNT(2); + (void) simple_mdio_write(cphy, + MV88E1XXX_EXT_PHY_SPECIFIC_CNTRL_REGISTER, val); + return 0; +} + +static int mv88e1xxx_interrupt_handler(struct cphy *cphy) +{ + int cphy_cause = 0; + u32 status; + + /* + * Loop until cause reads zero. Need to handle bouncing interrupts. + */ + while (1) { + u32 cause; + + (void) simple_mdio_read(cphy, + MV88E1XXX_INTERRUPT_STATUS_REGISTER, + &cause); + cause &= INTR_ENABLE_MASK; + if (!cause) + break; + + if (cause & MV88E1XXX_INTR_LINK_CHNG) { + (void) simple_mdio_read(cphy, + MV88E1XXX_SPECIFIC_STATUS_REGISTER, &status); + + if (status & MV88E1XXX_INTR_LINK_CHNG) + cphy->state |= PHY_LINK_UP; + else { + cphy->state &= ~PHY_LINK_UP; + if (cphy->state & PHY_AUTONEG_EN) + cphy->state &= ~PHY_AUTONEG_RDY; + cphy_cause |= cphy_cause_link_change; + } + } + + if (cause & MV88E1XXX_INTR_AUTONEG_DONE) + cphy->state |= PHY_AUTONEG_RDY; + + if ((cphy->state & (PHY_LINK_UP | PHY_AUTONEG_RDY)) == + (PHY_LINK_UP | PHY_AUTONEG_RDY)) + cphy_cause |= cphy_cause_link_change; + } + return cphy_cause; +} + +static void mv88e1xxx_destroy(struct cphy *cphy) +{ + kfree(cphy); +} + +static const struct cphy_ops mv88e1xxx_ops = { + .destroy = mv88e1xxx_destroy, + .reset = mv88e1xxx_reset, + .interrupt_enable = mv88e1xxx_interrupt_enable, + .interrupt_disable = mv88e1xxx_interrupt_disable, + .interrupt_clear = mv88e1xxx_interrupt_clear, + .interrupt_handler = mv88e1xxx_interrupt_handler, + .autoneg_enable = mv88e1xxx_autoneg_enable, + .autoneg_disable = mv88e1xxx_autoneg_disable, + .autoneg_restart = mv88e1xxx_autoneg_restart, + .advertise = mv88e1xxx_advertise, + .set_loopback = mv88e1xxx_set_loopback, + .set_speed_duplex = mv88e1xxx_set_speed_duplex, + .get_link_status = mv88e1xxx_get_link_status, +}; + +static struct cphy *mv88e1xxx_phy_create(struct net_device *dev, int phy_addr, + const struct mdio_ops *mdio_ops) +{ + struct adapter *adapter = netdev_priv(dev); + struct cphy *cphy = kzalloc(sizeof(*cphy), GFP_KERNEL); + + if (!cphy) + return NULL; + + cphy_init(cphy, dev, phy_addr, &mv88e1xxx_ops, mdio_ops); + + /* Configure particular PHY's to run in a different mode. */ + if ((board_info(adapter)->caps & SUPPORTED_TP) && + board_info(adapter)->chip_phy == CHBT_PHY_88E1111) { + /* + * Configure the PHY transmitter as class A to reduce EMI. + */ + (void) simple_mdio_write(cphy, + MV88E1XXX_EXTENDED_ADDR_REGISTER, 0xB); + (void) simple_mdio_write(cphy, + MV88E1XXX_EXTENDED_REGISTER, 0x8004); + } + (void) mv88e1xxx_downshift_set(cphy, 1); /* Enable downshift */ + + /* LED */ + if (is_T2(adapter)) { + (void) simple_mdio_write(cphy, + MV88E1XXX_LED_CONTROL_REGISTER, 0x1); + } + + return cphy; +} + +static int mv88e1xxx_phy_reset(adapter_t* adapter) +{ + return 0; +} + +const struct gphy t1_mv88e1xxx_ops = { + .create = mv88e1xxx_phy_create, + .reset = mv88e1xxx_phy_reset +}; diff --git a/drivers/net/ethernet/chelsio/cxgb/mv88e1xxx.h b/drivers/net/ethernet/chelsio/cxgb/mv88e1xxx.h new file mode 100644 index 000000000..11bf0f7c2 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/mv88e1xxx.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* $Date: 2005/03/07 23:59:05 $ $RCSfile: mv88e1xxx.h,v $ $Revision: 1.13 $ */ +#ifndef CHELSIO_MV8E1XXX_H +#define CHELSIO_MV8E1XXX_H + +#ifndef BMCR_SPEED1000 +# define BMCR_SPEED1000 0x40 +#endif + +#ifndef ADVERTISE_PAUSE +# define ADVERTISE_PAUSE 0x400 +#endif +#ifndef ADVERTISE_PAUSE_ASYM +# define ADVERTISE_PAUSE_ASYM 0x800 +#endif + +/* Gigabit MII registers */ +#define MII_GBCR 9 /* 1000Base-T control register */ +#define MII_GBSR 10 /* 1000Base-T status register */ + +/* 1000Base-T control register fields */ +#define GBCR_ADV_1000HALF 0x100 +#define GBCR_ADV_1000FULL 0x200 +#define GBCR_PREFER_MASTER 0x400 +#define GBCR_MANUAL_AS_MASTER 0x800 +#define GBCR_MANUAL_CONFIG_ENABLE 0x1000 + +/* 1000Base-T status register fields */ +#define GBSR_LP_1000HALF 0x400 +#define GBSR_LP_1000FULL 0x800 +#define GBSR_REMOTE_OK 0x1000 +#define GBSR_LOCAL_OK 0x2000 +#define GBSR_LOCAL_MASTER 0x4000 +#define GBSR_MASTER_FAULT 0x8000 + +/* Marvell PHY interrupt status bits. */ +#define MV88E1XXX_INTR_JABBER 0x0001 +#define MV88E1XXX_INTR_POLARITY_CHNG 0x0002 +#define MV88E1XXX_INTR_ENG_DETECT_CHNG 0x0010 +#define MV88E1XXX_INTR_DOWNSHIFT 0x0020 +#define MV88E1XXX_INTR_MDI_XOVER_CHNG 0x0040 +#define MV88E1XXX_INTR_FIFO_OVER_UNDER 0x0080 +#define MV88E1XXX_INTR_FALSE_CARRIER 0x0100 +#define MV88E1XXX_INTR_SYMBOL_ERROR 0x0200 +#define MV88E1XXX_INTR_LINK_CHNG 0x0400 +#define MV88E1XXX_INTR_AUTONEG_DONE 0x0800 +#define MV88E1XXX_INTR_PAGE_RECV 0x1000 +#define MV88E1XXX_INTR_DUPLEX_CHNG 0x2000 +#define MV88E1XXX_INTR_SPEED_CHNG 0x4000 +#define MV88E1XXX_INTR_AUTONEG_ERR 0x8000 + +/* Marvell PHY specific registers. */ +#define MV88E1XXX_SPECIFIC_CNTRL_REGISTER 16 +#define MV88E1XXX_SPECIFIC_STATUS_REGISTER 17 +#define MV88E1XXX_INTERRUPT_ENABLE_REGISTER 18 +#define MV88E1XXX_INTERRUPT_STATUS_REGISTER 19 +#define MV88E1XXX_EXT_PHY_SPECIFIC_CNTRL_REGISTER 20 +#define MV88E1XXX_RECV_ERR_CNTR_REGISTER 21 +#define MV88E1XXX_RES_REGISTER 22 +#define MV88E1XXX_GLOBAL_STATUS_REGISTER 23 +#define MV88E1XXX_LED_CONTROL_REGISTER 24 +#define MV88E1XXX_MANUAL_LED_OVERRIDE_REGISTER 25 +#define MV88E1XXX_EXT_PHY_SPECIFIC_CNTRL_2_REGISTER 26 +#define MV88E1XXX_EXT_PHY_SPECIFIC_STATUS_REGISTER 27 +#define MV88E1XXX_VIRTUAL_CABLE_TESTER_REGISTER 28 +#define MV88E1XXX_EXTENDED_ADDR_REGISTER 29 +#define MV88E1XXX_EXTENDED_REGISTER 30 + +/* PHY specific control register fields */ +#define S_PSCR_MDI_XOVER_MODE 5 +#define M_PSCR_MDI_XOVER_MODE 0x3 +#define V_PSCR_MDI_XOVER_MODE(x) ((x) << S_PSCR_MDI_XOVER_MODE) +#define G_PSCR_MDI_XOVER_MODE(x) (((x) >> S_PSCR_MDI_XOVER_MODE) & M_PSCR_MDI_XOVER_MODE) + +/* Extended PHY specific control register fields */ +#define S_DOWNSHIFT_ENABLE 8 +#define V_DOWNSHIFT_ENABLE (1 << S_DOWNSHIFT_ENABLE) + +#define S_DOWNSHIFT_CNT 9 +#define M_DOWNSHIFT_CNT 0x7 +#define V_DOWNSHIFT_CNT(x) ((x) << S_DOWNSHIFT_CNT) +#define G_DOWNSHIFT_CNT(x) (((x) >> S_DOWNSHIFT_CNT) & M_DOWNSHIFT_CNT) + +/* PHY specific status register fields */ +#define S_PSSR_JABBER 0 +#define V_PSSR_JABBER (1 << S_PSSR_JABBER) + +#define S_PSSR_POLARITY 1 +#define V_PSSR_POLARITY (1 << S_PSSR_POLARITY) + +#define S_PSSR_RX_PAUSE 2 +#define V_PSSR_RX_PAUSE (1 << S_PSSR_RX_PAUSE) + +#define S_PSSR_TX_PAUSE 3 +#define V_PSSR_TX_PAUSE (1 << S_PSSR_TX_PAUSE) + +#define S_PSSR_ENERGY_DETECT 4 +#define V_PSSR_ENERGY_DETECT (1 << S_PSSR_ENERGY_DETECT) + +#define S_PSSR_DOWNSHIFT_STATUS 5 +#define V_PSSR_DOWNSHIFT_STATUS (1 << S_PSSR_DOWNSHIFT_STATUS) + +#define S_PSSR_MDI 6 +#define V_PSSR_MDI (1 << S_PSSR_MDI) + +#define S_PSSR_CABLE_LEN 7 +#define M_PSSR_CABLE_LEN 0x7 +#define V_PSSR_CABLE_LEN(x) ((x) << S_PSSR_CABLE_LEN) +#define G_PSSR_CABLE_LEN(x) (((x) >> S_PSSR_CABLE_LEN) & M_PSSR_CABLE_LEN) + +#define S_PSSR_LINK 10 +#define V_PSSR_LINK (1 << S_PSSR_LINK) + +#define S_PSSR_STATUS_RESOLVED 11 +#define V_PSSR_STATUS_RESOLVED (1 << S_PSSR_STATUS_RESOLVED) + +#define S_PSSR_PAGE_RECEIVED 12 +#define V_PSSR_PAGE_RECEIVED (1 << S_PSSR_PAGE_RECEIVED) + +#define S_PSSR_DUPLEX 13 +#define V_PSSR_DUPLEX (1 << S_PSSR_DUPLEX) + +#define S_PSSR_SPEED 14 +#define M_PSSR_SPEED 0x3 +#define V_PSSR_SPEED(x) ((x) << S_PSSR_SPEED) +#define G_PSSR_SPEED(x) (((x) >> S_PSSR_SPEED) & M_PSSR_SPEED) + +#endif diff --git a/drivers/net/ethernet/chelsio/cxgb/mv88x201x.c b/drivers/net/ethernet/chelsio/cxgb/mv88x201x.c new file mode 100644 index 000000000..556c8ad68 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/mv88x201x.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0-only +/***************************************************************************** + * * + * File: mv88x201x.c * + * $Revision: 1.12 $ * + * $Date: 2005/04/15 19:27:14 $ * + * Description: * + * Marvell PHY (mv88x201x) functionality. * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#include "cphy.h" +#include "elmer0.h" + +/* + * The 88x2010 Rev C. requires some link status registers * to be read + * twice in order to get the right values. Future * revisions will fix + * this problem and then this macro * can disappear. + */ +#define MV88x2010_LINK_STATUS_BUGS 1 + +static int led_init(struct cphy *cphy) +{ + /* Setup the LED registers so we can turn on/off. + * Writing these bits maps control to another + * register. mmd(0x1) addr(0x7) + */ + cphy_mdio_write(cphy, MDIO_MMD_PCS, 0x8304, 0xdddd); + return 0; +} + +static int led_link(struct cphy *cphy, u32 do_enable) +{ + u32 led = 0; +#define LINK_ENABLE_BIT 0x1 + + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_CTRL2, &led); + + if (do_enable & LINK_ENABLE_BIT) { + led |= LINK_ENABLE_BIT; + cphy_mdio_write(cphy, MDIO_MMD_PMAPMD, MDIO_CTRL2, led); + } else { + led &= ~LINK_ENABLE_BIT; + cphy_mdio_write(cphy, MDIO_MMD_PMAPMD, MDIO_CTRL2, led); + } + return 0; +} + +/* Port Reset */ +static int mv88x201x_reset(struct cphy *cphy, int wait) +{ + /* This can be done through registers. It is not required since + * a full chip reset is used. + */ + return 0; +} + +static int mv88x201x_interrupt_enable(struct cphy *cphy) +{ + /* Enable PHY LASI interrupts. */ + cphy_mdio_write(cphy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_CTRL, + MDIO_PMA_LASI_LSALARM); + + /* Enable Marvell interrupts through Elmer0. */ + if (t1_is_asic(cphy->adapter)) { + u32 elmer; + + t1_tpi_read(cphy->adapter, A_ELMER0_INT_ENABLE, &elmer); + elmer |= ELMER0_GP_BIT6; + t1_tpi_write(cphy->adapter, A_ELMER0_INT_ENABLE, elmer); + } + return 0; +} + +static int mv88x201x_interrupt_disable(struct cphy *cphy) +{ + /* Disable PHY LASI interrupts. */ + cphy_mdio_write(cphy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_CTRL, 0x0); + + /* Disable Marvell interrupts through Elmer0. */ + if (t1_is_asic(cphy->adapter)) { + u32 elmer; + + t1_tpi_read(cphy->adapter, A_ELMER0_INT_ENABLE, &elmer); + elmer &= ~ELMER0_GP_BIT6; + t1_tpi_write(cphy->adapter, A_ELMER0_INT_ENABLE, elmer); + } + return 0; +} + +static int mv88x201x_interrupt_clear(struct cphy *cphy) +{ + u32 elmer; + u32 val; + +#ifdef MV88x2010_LINK_STATUS_BUGS + /* Required to read twice before clear takes affect. */ + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_RXSTAT, &val); + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_TXSTAT, &val); + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_STAT, &val); + + /* Read this register after the others above it else + * the register doesn't clear correctly. + */ + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_STAT1, &val); +#endif + + /* Clear link status. */ + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_STAT1, &val); + /* Clear PHY LASI interrupts. */ + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_STAT, &val); + +#ifdef MV88x2010_LINK_STATUS_BUGS + /* Do it again. */ + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_RXSTAT, &val); + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_TXSTAT, &val); +#endif + + /* Clear Marvell interrupts through Elmer0. */ + if (t1_is_asic(cphy->adapter)) { + t1_tpi_read(cphy->adapter, A_ELMER0_INT_CAUSE, &elmer); + elmer |= ELMER0_GP_BIT6; + t1_tpi_write(cphy->adapter, A_ELMER0_INT_CAUSE, elmer); + } + return 0; +} + +static int mv88x201x_interrupt_handler(struct cphy *cphy) +{ + /* Clear interrupts */ + mv88x201x_interrupt_clear(cphy); + + /* We have only enabled link change interrupts and so + * cphy_cause must be a link change interrupt. + */ + return cphy_cause_link_change; +} + +static int mv88x201x_set_loopback(struct cphy *cphy, int on) +{ + return 0; +} + +static int mv88x201x_get_link_status(struct cphy *cphy, int *link_ok, + int *speed, int *duplex, int *fc) +{ + u32 val = 0; + + if (link_ok) { + /* Read link status. */ + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_STAT1, &val); + val &= MDIO_STAT1_LSTATUS; + *link_ok = (val == MDIO_STAT1_LSTATUS); + /* Turn on/off Link LED */ + led_link(cphy, *link_ok); + } + if (speed) + *speed = SPEED_10000; + if (duplex) + *duplex = DUPLEX_FULL; + if (fc) + *fc = PAUSE_RX | PAUSE_TX; + return 0; +} + +static void mv88x201x_destroy(struct cphy *cphy) +{ + kfree(cphy); +} + +static const struct cphy_ops mv88x201x_ops = { + .destroy = mv88x201x_destroy, + .reset = mv88x201x_reset, + .interrupt_enable = mv88x201x_interrupt_enable, + .interrupt_disable = mv88x201x_interrupt_disable, + .interrupt_clear = mv88x201x_interrupt_clear, + .interrupt_handler = mv88x201x_interrupt_handler, + .get_link_status = mv88x201x_get_link_status, + .set_loopback = mv88x201x_set_loopback, + .mmds = (MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS | + MDIO_DEVS_PHYXS | MDIO_DEVS_WIS), +}; + +static struct cphy *mv88x201x_phy_create(struct net_device *dev, int phy_addr, + const struct mdio_ops *mdio_ops) +{ + u32 val; + struct cphy *cphy = kzalloc(sizeof(*cphy), GFP_KERNEL); + + if (!cphy) + return NULL; + + cphy_init(cphy, dev, phy_addr, &mv88x201x_ops, mdio_ops); + + /* Commands the PHY to enable XFP's clock. */ + cphy_mdio_read(cphy, MDIO_MMD_PCS, 0x8300, &val); + cphy_mdio_write(cphy, MDIO_MMD_PCS, 0x8300, val | 1); + + /* Clear link status. Required because of a bug in the PHY. */ + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_STAT2, &val); + cphy_mdio_read(cphy, MDIO_MMD_PCS, MDIO_STAT2, &val); + + /* Allows for Link,Ack LED turn on/off */ + led_init(cphy); + return cphy; +} + +/* Chip Reset */ +static int mv88x201x_phy_reset(adapter_t *adapter) +{ + u32 val; + + t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val &= ~4; + t1_tpi_write(adapter, A_ELMER0_GPO, val); + msleep(100); + + t1_tpi_write(adapter, A_ELMER0_GPO, val | 4); + msleep(1000); + + /* Now lets enable the Laser. Delay 100us */ + t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val |= 0x8000; + t1_tpi_write(adapter, A_ELMER0_GPO, val); + udelay(100); + return 0; +} + +const struct gphy t1_mv88x201x_ops = { + .create = mv88x201x_phy_create, + .reset = mv88x201x_phy_reset +}; diff --git a/drivers/net/ethernet/chelsio/cxgb/my3126.c b/drivers/net/ethernet/chelsio/cxgb/my3126.c new file mode 100644 index 000000000..60aa45b37 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/my3126.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0 +/* $Date: 2005/11/12 02:13:49 $ $RCSfile: my3126.c,v $ $Revision: 1.15 $ */ +#include "cphy.h" +#include "elmer0.h" +#include "suni1x10gexp_regs.h" + +/* Port Reset */ +static int my3126_reset(struct cphy *cphy, int wait) +{ + /* + * This can be done through registers. It is not required since + * a full chip reset is used. + */ + return 0; +} + +static int my3126_interrupt_enable(struct cphy *cphy) +{ + schedule_delayed_work(&cphy->phy_update, HZ/30); + t1_tpi_read(cphy->adapter, A_ELMER0_GPO, &cphy->elmer_gpo); + return 0; +} + +static int my3126_interrupt_disable(struct cphy *cphy) +{ + cancel_delayed_work_sync(&cphy->phy_update); + return 0; +} + +static int my3126_interrupt_clear(struct cphy *cphy) +{ + return 0; +} + +#define OFFSET(REG_ADDR) (REG_ADDR << 2) + +static int my3126_interrupt_handler(struct cphy *cphy) +{ + u32 val; + u16 val16; + u16 status; + u32 act_count; + adapter_t *adapter; + adapter = cphy->adapter; + + if (cphy->count == 50) { + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_STAT1, &val); + val16 = (u16) val; + status = cphy->bmsr ^ val16; + + if (status & MDIO_STAT1_LSTATUS) + t1_link_changed(adapter, 0); + cphy->bmsr = val16; + + /* We have only enabled link change interrupts so it + must be that + */ + cphy->count = 0; + } + + t1_tpi_write(adapter, OFFSET(SUNI1x10GEXP_REG_MSTAT_CONTROL), + SUNI1x10GEXP_BITMSK_MSTAT_SNAP); + t1_tpi_read(adapter, + OFFSET(SUNI1x10GEXP_REG_MSTAT_COUNTER_1_LOW), &act_count); + t1_tpi_read(adapter, + OFFSET(SUNI1x10GEXP_REG_MSTAT_COUNTER_33_LOW), &val); + act_count += val; + + /* Populate elmer_gpo with the register value */ + t1_tpi_read(adapter, A_ELMER0_GPO, &val); + cphy->elmer_gpo = val; + + if ( (val & (1 << 8)) || (val & (1 << 19)) || + (cphy->act_count == act_count) || cphy->act_on ) { + if (is_T2(adapter)) + val |= (1 << 9); + else if (t1_is_T1B(adapter)) + val |= (1 << 20); + cphy->act_on = 0; + } else { + if (is_T2(adapter)) + val &= ~(1 << 9); + else if (t1_is_T1B(adapter)) + val &= ~(1 << 20); + cphy->act_on = 1; + } + + t1_tpi_write(adapter, A_ELMER0_GPO, val); + + cphy->elmer_gpo = val; + cphy->act_count = act_count; + cphy->count++; + + return cphy_cause_link_change; +} + +static void my3126_poll(struct work_struct *work) +{ + struct cphy *cphy = container_of(work, struct cphy, phy_update.work); + + my3126_interrupt_handler(cphy); +} + +static int my3126_set_loopback(struct cphy *cphy, int on) +{ + return 0; +} + +/* To check the activity LED */ +static int my3126_get_link_status(struct cphy *cphy, + int *link_ok, int *speed, int *duplex, int *fc) +{ + u32 val; + u16 val16; + adapter_t *adapter; + + adapter = cphy->adapter; + cphy_mdio_read(cphy, MDIO_MMD_PMAPMD, MDIO_STAT1, &val); + val16 = (u16) val; + + /* Populate elmer_gpo with the register value */ + t1_tpi_read(adapter, A_ELMER0_GPO, &val); + cphy->elmer_gpo = val; + + *link_ok = (val16 & MDIO_STAT1_LSTATUS); + + if (*link_ok) { + /* Turn on the LED. */ + if (is_T2(adapter)) + val &= ~(1 << 8); + else if (t1_is_T1B(adapter)) + val &= ~(1 << 19); + } else { + /* Turn off the LED. */ + if (is_T2(adapter)) + val |= (1 << 8); + else if (t1_is_T1B(adapter)) + val |= (1 << 19); + } + + t1_tpi_write(adapter, A_ELMER0_GPO, val); + cphy->elmer_gpo = val; + *speed = SPEED_10000; + *duplex = DUPLEX_FULL; + + /* need to add flow control */ + if (fc) + *fc = PAUSE_RX | PAUSE_TX; + + return 0; +} + +static void my3126_destroy(struct cphy *cphy) +{ + kfree(cphy); +} + +static const struct cphy_ops my3126_ops = { + .destroy = my3126_destroy, + .reset = my3126_reset, + .interrupt_enable = my3126_interrupt_enable, + .interrupt_disable = my3126_interrupt_disable, + .interrupt_clear = my3126_interrupt_clear, + .interrupt_handler = my3126_interrupt_handler, + .get_link_status = my3126_get_link_status, + .set_loopback = my3126_set_loopback, + .mmds = (MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS | + MDIO_DEVS_PHYXS), +}; + +static struct cphy *my3126_phy_create(struct net_device *dev, + int phy_addr, const struct mdio_ops *mdio_ops) +{ + struct cphy *cphy = kzalloc(sizeof (*cphy), GFP_KERNEL); + + if (!cphy) + return NULL; + + cphy_init(cphy, dev, phy_addr, &my3126_ops, mdio_ops); + INIT_DELAYED_WORK(&cphy->phy_update, my3126_poll); + cphy->bmsr = 0; + + return cphy; +} + +/* Chip Reset */ +static int my3126_phy_reset(adapter_t * adapter) +{ + u32 val; + + t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val &= ~4; + t1_tpi_write(adapter, A_ELMER0_GPO, val); + msleep(100); + + t1_tpi_write(adapter, A_ELMER0_GPO, val | 4); + msleep(1000); + + /* Now lets enable the Laser. Delay 100us */ + t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val |= 0x8000; + t1_tpi_write(adapter, A_ELMER0_GPO, val); + udelay(100); + return 0; +} + +const struct gphy t1_my3126_ops = { + .create = my3126_phy_create, + .reset = my3126_phy_reset +}; diff --git a/drivers/net/ethernet/chelsio/cxgb/pm3393.c b/drivers/net/ethernet/chelsio/cxgb/pm3393.c new file mode 100644 index 000000000..cbfa03d56 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/pm3393.c @@ -0,0 +1,780 @@ +// SPDX-License-Identifier: GPL-2.0-only +/***************************************************************************** + * * + * File: pm3393.c * + * $Revision: 1.16 $ * + * $Date: 2005/05/14 00:59:32 $ * + * Description: * + * PMC/SIERRA (pm3393) MAC-PHY functionality. * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#include "common.h" +#include "regs.h" +#include "gmac.h" +#include "elmer0.h" +#include "suni1x10gexp_regs.h" + +#include <linux/crc32.h> +#include <linux/slab.h> + +#define OFFSET(REG_ADDR) ((REG_ADDR) << 2) + +#define IPG 12 +#define TXXG_CONF1_VAL ((IPG << SUNI1x10GEXP_BITOFF_TXXG_IPGT) | \ + SUNI1x10GEXP_BITMSK_TXXG_32BIT_ALIGN | SUNI1x10GEXP_BITMSK_TXXG_CRCEN | \ + SUNI1x10GEXP_BITMSK_TXXG_PADEN) +#define RXXG_CONF1_VAL (SUNI1x10GEXP_BITMSK_RXXG_PUREP | 0x14 | \ + SUNI1x10GEXP_BITMSK_RXXG_FLCHK | SUNI1x10GEXP_BITMSK_RXXG_CRC_STRIP) + +/* Update statistics every 15 minutes */ +#define STATS_TICK_SECS (15 * 60) + +enum { /* RMON registers */ + RxOctetsReceivedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_1_LOW, + RxUnicastFramesReceivedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_4_LOW, + RxMulticastFramesReceivedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_5_LOW, + RxBroadcastFramesReceivedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_6_LOW, + RxPAUSEMACCtrlFramesReceived = SUNI1x10GEXP_REG_MSTAT_COUNTER_8_LOW, + RxFrameCheckSequenceErrors = SUNI1x10GEXP_REG_MSTAT_COUNTER_10_LOW, + RxFramesLostDueToInternalMACErrors = SUNI1x10GEXP_REG_MSTAT_COUNTER_11_LOW, + RxSymbolErrors = SUNI1x10GEXP_REG_MSTAT_COUNTER_12_LOW, + RxInRangeLengthErrors = SUNI1x10GEXP_REG_MSTAT_COUNTER_13_LOW, + RxFramesTooLongErrors = SUNI1x10GEXP_REG_MSTAT_COUNTER_15_LOW, + RxJabbers = SUNI1x10GEXP_REG_MSTAT_COUNTER_16_LOW, + RxFragments = SUNI1x10GEXP_REG_MSTAT_COUNTER_17_LOW, + RxUndersizedFrames = SUNI1x10GEXP_REG_MSTAT_COUNTER_18_LOW, + RxJumboFramesReceivedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_25_LOW, + RxJumboOctetsReceivedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_26_LOW, + + TxOctetsTransmittedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_33_LOW, + TxFramesLostDueToInternalMACTransmissionError = SUNI1x10GEXP_REG_MSTAT_COUNTER_35_LOW, + TxTransmitSystemError = SUNI1x10GEXP_REG_MSTAT_COUNTER_36_LOW, + TxUnicastFramesTransmittedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_38_LOW, + TxMulticastFramesTransmittedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_40_LOW, + TxBroadcastFramesTransmittedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_42_LOW, + TxPAUSEMACCtrlFramesTransmitted = SUNI1x10GEXP_REG_MSTAT_COUNTER_43_LOW, + TxJumboFramesReceivedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_51_LOW, + TxJumboOctetsReceivedOK = SUNI1x10GEXP_REG_MSTAT_COUNTER_52_LOW +}; + +struct _cmac_instance { + u8 enabled; + u8 fc; + u8 mac_addr[6]; +}; + +static int pmread(struct cmac *cmac, u32 reg, u32 * data32) +{ + t1_tpi_read(cmac->adapter, OFFSET(reg), data32); + return 0; +} + +static int pmwrite(struct cmac *cmac, u32 reg, u32 data32) +{ + t1_tpi_write(cmac->adapter, OFFSET(reg), data32); + return 0; +} + +/* Port reset. */ +static int pm3393_reset(struct cmac *cmac) +{ + return 0; +} + +/* + * Enable interrupts for the PM3393 + * + * 1. Enable PM3393 BLOCK interrupts. + * 2. Enable PM3393 Master Interrupt bit(INTE) + * 3. Enable ELMER's PM3393 bit. + * 4. Enable Terminator external interrupt. + */ +static int pm3393_interrupt_enable(struct cmac *cmac) +{ + u32 pl_intr; + + /* PM3393 - Enabling all hardware block interrupts. + */ + pmwrite(cmac, SUNI1x10GEXP_REG_SERDES_3125_INTERRUPT_ENABLE, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_XRF_INTERRUPT_ENABLE, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_XRF_DIAG_INTERRUPT_ENABLE, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_RXOAM_INTERRUPT_ENABLE, 0xffff); + + /* Don't interrupt on statistics overflow, we are polling */ + pmwrite(cmac, SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_0, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_1, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_2, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_3, 0); + + pmwrite(cmac, SUNI1x10GEXP_REG_IFLX_FIFO_OVERFLOW_ENABLE, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_PL4ODP_INTERRUPT_MASK, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_XTEF_INTERRUPT_ENABLE, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_TXOAM_INTERRUPT_ENABLE, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_CONFIG_3, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_PL4IO_LOCK_DETECT_MASK, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_TXXG_CONFIG_3, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_PL4IDU_INTERRUPT_MASK, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_EFLX_FIFO_OVERFLOW_ERROR_ENABLE, 0xffff); + + /* PM3393 - Global interrupt enable + */ + /* TBD XXX Disable for now until we figure out why error interrupts keep asserting. */ + pmwrite(cmac, SUNI1x10GEXP_REG_GLOBAL_INTERRUPT_ENABLE, + 0 /*SUNI1x10GEXP_BITMSK_TOP_INTE */ ); + + /* TERMINATOR - PL_INTERUPTS_EXT */ + pl_intr = readl(cmac->adapter->regs + A_PL_ENABLE); + pl_intr |= F_PL_INTR_EXT; + writel(pl_intr, cmac->adapter->regs + A_PL_ENABLE); + return 0; +} + +static int pm3393_interrupt_disable(struct cmac *cmac) +{ + u32 elmer; + + /* PM3393 - Enabling HW interrupt blocks. */ + pmwrite(cmac, SUNI1x10GEXP_REG_SERDES_3125_INTERRUPT_ENABLE, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_XRF_INTERRUPT_ENABLE, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_XRF_DIAG_INTERRUPT_ENABLE, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_RXOAM_INTERRUPT_ENABLE, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_0, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_1, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_2, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_3, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_IFLX_FIFO_OVERFLOW_ENABLE, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_PL4ODP_INTERRUPT_MASK, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_XTEF_INTERRUPT_ENABLE, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_TXOAM_INTERRUPT_ENABLE, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_CONFIG_3, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_PL4IO_LOCK_DETECT_MASK, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_TXXG_CONFIG_3, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_PL4IDU_INTERRUPT_MASK, 0); + pmwrite(cmac, SUNI1x10GEXP_REG_EFLX_FIFO_OVERFLOW_ERROR_ENABLE, 0); + + /* PM3393 - Global interrupt enable */ + pmwrite(cmac, SUNI1x10GEXP_REG_GLOBAL_INTERRUPT_ENABLE, 0); + + /* ELMER - External chip interrupts. */ + t1_tpi_read(cmac->adapter, A_ELMER0_INT_ENABLE, &elmer); + elmer &= ~ELMER0_GP_BIT1; + t1_tpi_write(cmac->adapter, A_ELMER0_INT_ENABLE, elmer); + + /* TERMINATOR - PL_INTERUPTS_EXT */ + /* DO NOT DISABLE TERMINATOR's EXTERNAL INTERRUPTS. ANOTHER CHIP + * COULD WANT THEM ENABLED. We disable PM3393 at the ELMER level. + */ + + return 0; +} + +static int pm3393_interrupt_clear(struct cmac *cmac) +{ + u32 elmer; + u32 pl_intr; + u32 val32; + + /* PM3393 - Clearing HW interrupt blocks. Note, this assumes + * bit WCIMODE=0 for a clear-on-read. + */ + pmread(cmac, SUNI1x10GEXP_REG_SERDES_3125_INTERRUPT_STATUS, &val32); + pmread(cmac, SUNI1x10GEXP_REG_XRF_INTERRUPT_STATUS, &val32); + pmread(cmac, SUNI1x10GEXP_REG_XRF_DIAG_INTERRUPT_STATUS, &val32); + pmread(cmac, SUNI1x10GEXP_REG_RXOAM_INTERRUPT_STATUS, &val32); + pmread(cmac, SUNI1x10GEXP_REG_PL4ODP_INTERRUPT, &val32); + pmread(cmac, SUNI1x10GEXP_REG_XTEF_INTERRUPT_STATUS, &val32); + pmread(cmac, SUNI1x10GEXP_REG_IFLX_FIFO_OVERFLOW_INTERRUPT, &val32); + pmread(cmac, SUNI1x10GEXP_REG_TXOAM_INTERRUPT_STATUS, &val32); + pmread(cmac, SUNI1x10GEXP_REG_RXXG_INTERRUPT, &val32); + pmread(cmac, SUNI1x10GEXP_REG_TXXG_INTERRUPT, &val32); + pmread(cmac, SUNI1x10GEXP_REG_PL4IDU_INTERRUPT, &val32); + pmread(cmac, SUNI1x10GEXP_REG_EFLX_FIFO_OVERFLOW_ERROR_INDICATION, + &val32); + pmread(cmac, SUNI1x10GEXP_REG_PL4IO_LOCK_DETECT_STATUS, &val32); + pmread(cmac, SUNI1x10GEXP_REG_PL4IO_LOCK_DETECT_CHANGE, &val32); + + /* PM3393 - Global interrupt status + */ + pmread(cmac, SUNI1x10GEXP_REG_MASTER_INTERRUPT_STATUS, &val32); + + /* ELMER - External chip interrupts. + */ + t1_tpi_read(cmac->adapter, A_ELMER0_INT_CAUSE, &elmer); + elmer |= ELMER0_GP_BIT1; + t1_tpi_write(cmac->adapter, A_ELMER0_INT_CAUSE, elmer); + + /* TERMINATOR - PL_INTERUPTS_EXT + */ + pl_intr = readl(cmac->adapter->regs + A_PL_CAUSE); + pl_intr |= F_PL_INTR_EXT; + writel(pl_intr, cmac->adapter->regs + A_PL_CAUSE); + + return 0; +} + +/* Interrupt handler */ +static int pm3393_interrupt_handler(struct cmac *cmac) +{ + u32 master_intr_status; + + /* Read the master interrupt status register. */ + pmread(cmac, SUNI1x10GEXP_REG_MASTER_INTERRUPT_STATUS, + &master_intr_status); + if (netif_msg_intr(cmac->adapter)) + dev_dbg(&cmac->adapter->pdev->dev, "PM3393 intr cause 0x%x\n", + master_intr_status); + + /* TBD XXX Lets just clear everything for now */ + pm3393_interrupt_clear(cmac); + + return 0; +} + +static int pm3393_enable(struct cmac *cmac, int which) +{ + if (which & MAC_DIRECTION_RX) + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_CONFIG_1, + (RXXG_CONF1_VAL | SUNI1x10GEXP_BITMSK_RXXG_RXEN)); + + if (which & MAC_DIRECTION_TX) { + u32 val = TXXG_CONF1_VAL | SUNI1x10GEXP_BITMSK_TXXG_TXEN0; + + if (cmac->instance->fc & PAUSE_RX) + val |= SUNI1x10GEXP_BITMSK_TXXG_FCRX; + if (cmac->instance->fc & PAUSE_TX) + val |= SUNI1x10GEXP_BITMSK_TXXG_FCTX; + pmwrite(cmac, SUNI1x10GEXP_REG_TXXG_CONFIG_1, val); + } + + cmac->instance->enabled |= which; + return 0; +} + +static int pm3393_enable_port(struct cmac *cmac, int which) +{ + /* Clear port statistics */ + pmwrite(cmac, SUNI1x10GEXP_REG_MSTAT_CONTROL, + SUNI1x10GEXP_BITMSK_MSTAT_CLEAR); + udelay(2); + memset(&cmac->stats, 0, sizeof(struct cmac_statistics)); + + pm3393_enable(cmac, which); + + /* + * XXX This should be done by the PHY and preferably not at all. + * The PHY doesn't give us link status indication on its own so have + * the link management code query it instead. + */ + t1_link_changed(cmac->adapter, 0); + return 0; +} + +static int pm3393_disable(struct cmac *cmac, int which) +{ + if (which & MAC_DIRECTION_RX) + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_CONFIG_1, RXXG_CONF1_VAL); + if (which & MAC_DIRECTION_TX) + pmwrite(cmac, SUNI1x10GEXP_REG_TXXG_CONFIG_1, TXXG_CONF1_VAL); + + /* + * The disable is graceful. Give the PM3393 time. Can't wait very + * long here, we may be holding locks. + */ + udelay(20); + + cmac->instance->enabled &= ~which; + return 0; +} + +static int pm3393_loopback_enable(struct cmac *cmac) +{ + return 0; +} + +static int pm3393_loopback_disable(struct cmac *cmac) +{ + return 0; +} + +static int pm3393_set_mtu(struct cmac *cmac, int mtu) +{ + int enabled = cmac->instance->enabled; + + mtu += ETH_HLEN + ETH_FCS_LEN; + + /* Disable Rx/Tx MAC before configuring it. */ + if (enabled) + pm3393_disable(cmac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); + + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_MAX_FRAME_LENGTH, mtu); + pmwrite(cmac, SUNI1x10GEXP_REG_TXXG_MAX_FRAME_SIZE, mtu); + + if (enabled) + pm3393_enable(cmac, enabled); + return 0; +} + +static int pm3393_set_rx_mode(struct cmac *cmac, struct t1_rx_mode *rm) +{ + int enabled = cmac->instance->enabled & MAC_DIRECTION_RX; + u32 rx_mode; + + /* Disable MAC RX before reconfiguring it */ + if (enabled) + pm3393_disable(cmac, MAC_DIRECTION_RX); + + pmread(cmac, SUNI1x10GEXP_REG_RXXG_ADDRESS_FILTER_CONTROL_2, &rx_mode); + rx_mode &= ~(SUNI1x10GEXP_BITMSK_RXXG_PMODE | + SUNI1x10GEXP_BITMSK_RXXG_MHASH_EN); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_ADDRESS_FILTER_CONTROL_2, + (u16)rx_mode); + + if (t1_rx_mode_promisc(rm)) { + /* Promiscuous mode. */ + rx_mode |= SUNI1x10GEXP_BITMSK_RXXG_PMODE; + } + if (t1_rx_mode_allmulti(rm)) { + /* Accept all multicast. */ + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_LOW, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_MIDLOW, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_MIDHIGH, 0xffff); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_HIGH, 0xffff); + rx_mode |= SUNI1x10GEXP_BITMSK_RXXG_MHASH_EN; + } else if (t1_rx_mode_mc_cnt(rm)) { + /* Accept one or more multicast(s). */ + struct netdev_hw_addr *ha; + int bit; + u16 mc_filter[4] = { 0, }; + + netdev_for_each_mc_addr(ha, t1_get_netdev(rm)) { + /* bit[23:28] */ + bit = (ether_crc(ETH_ALEN, ha->addr) >> 23) & 0x3f; + mc_filter[bit >> 4] |= 1 << (bit & 0xf); + } + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_LOW, mc_filter[0]); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_MIDLOW, mc_filter[1]); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_MIDHIGH, mc_filter[2]); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_HIGH, mc_filter[3]); + rx_mode |= SUNI1x10GEXP_BITMSK_RXXG_MHASH_EN; + } + + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_ADDRESS_FILTER_CONTROL_2, (u16)rx_mode); + + if (enabled) + pm3393_enable(cmac, MAC_DIRECTION_RX); + + return 0; +} + +static int pm3393_get_speed_duplex_fc(struct cmac *cmac, int *speed, + int *duplex, int *fc) +{ + if (speed) + *speed = SPEED_10000; + if (duplex) + *duplex = DUPLEX_FULL; + if (fc) + *fc = cmac->instance->fc; + return 0; +} + +static int pm3393_set_speed_duplex_fc(struct cmac *cmac, int speed, int duplex, + int fc) +{ + if (speed >= 0 && speed != SPEED_10000) + return -1; + if (duplex >= 0 && duplex != DUPLEX_FULL) + return -1; + if (fc & ~(PAUSE_TX | PAUSE_RX)) + return -1; + + if (fc != cmac->instance->fc) { + cmac->instance->fc = (u8) fc; + if (cmac->instance->enabled & MAC_DIRECTION_TX) + pm3393_enable(cmac, MAC_DIRECTION_TX); + } + return 0; +} + +#define RMON_UPDATE(mac, name, stat_name) \ +{ \ + t1_tpi_read((mac)->adapter, OFFSET(name), &val0); \ + t1_tpi_read((mac)->adapter, OFFSET((name)+1), &val1); \ + t1_tpi_read((mac)->adapter, OFFSET((name)+2), &val2); \ + (mac)->stats.stat_name = (u64)(val0 & 0xffff) | \ + ((u64)(val1 & 0xffff) << 16) | \ + ((u64)(val2 & 0xff) << 32) | \ + ((mac)->stats.stat_name & \ + 0xffffff0000000000ULL); \ + if (ro & \ + (1ULL << ((name - SUNI1x10GEXP_REG_MSTAT_COUNTER_0_LOW) >> 2))) \ + (mac)->stats.stat_name += 1ULL << 40; \ +} + +static const struct cmac_statistics *pm3393_update_statistics(struct cmac *mac, + int flag) +{ + u64 ro; + u32 val0, val1, val2, val3; + + /* Snap the counters */ + pmwrite(mac, SUNI1x10GEXP_REG_MSTAT_CONTROL, + SUNI1x10GEXP_BITMSK_MSTAT_SNAP); + + /* Counter rollover, clear on read */ + pmread(mac, SUNI1x10GEXP_REG_MSTAT_COUNTER_ROLLOVER_0, &val0); + pmread(mac, SUNI1x10GEXP_REG_MSTAT_COUNTER_ROLLOVER_1, &val1); + pmread(mac, SUNI1x10GEXP_REG_MSTAT_COUNTER_ROLLOVER_2, &val2); + pmread(mac, SUNI1x10GEXP_REG_MSTAT_COUNTER_ROLLOVER_3, &val3); + ro = ((u64)val0 & 0xffff) | (((u64)val1 & 0xffff) << 16) | + (((u64)val2 & 0xffff) << 32) | (((u64)val3 & 0xffff) << 48); + + /* Rx stats */ + RMON_UPDATE(mac, RxOctetsReceivedOK, RxOctetsOK); + RMON_UPDATE(mac, RxUnicastFramesReceivedOK, RxUnicastFramesOK); + RMON_UPDATE(mac, RxMulticastFramesReceivedOK, RxMulticastFramesOK); + RMON_UPDATE(mac, RxBroadcastFramesReceivedOK, RxBroadcastFramesOK); + RMON_UPDATE(mac, RxPAUSEMACCtrlFramesReceived, RxPauseFrames); + RMON_UPDATE(mac, RxFrameCheckSequenceErrors, RxFCSErrors); + RMON_UPDATE(mac, RxFramesLostDueToInternalMACErrors, + RxInternalMACRcvError); + RMON_UPDATE(mac, RxSymbolErrors, RxSymbolErrors); + RMON_UPDATE(mac, RxInRangeLengthErrors, RxInRangeLengthErrors); + RMON_UPDATE(mac, RxFramesTooLongErrors , RxFrameTooLongErrors); + RMON_UPDATE(mac, RxJabbers, RxJabberErrors); + RMON_UPDATE(mac, RxFragments, RxRuntErrors); + RMON_UPDATE(mac, RxUndersizedFrames, RxRuntErrors); + RMON_UPDATE(mac, RxJumboFramesReceivedOK, RxJumboFramesOK); + RMON_UPDATE(mac, RxJumboOctetsReceivedOK, RxJumboOctetsOK); + + /* Tx stats */ + RMON_UPDATE(mac, TxOctetsTransmittedOK, TxOctetsOK); + RMON_UPDATE(mac, TxFramesLostDueToInternalMACTransmissionError, + TxInternalMACXmitError); + RMON_UPDATE(mac, TxTransmitSystemError, TxFCSErrors); + RMON_UPDATE(mac, TxUnicastFramesTransmittedOK, TxUnicastFramesOK); + RMON_UPDATE(mac, TxMulticastFramesTransmittedOK, TxMulticastFramesOK); + RMON_UPDATE(mac, TxBroadcastFramesTransmittedOK, TxBroadcastFramesOK); + RMON_UPDATE(mac, TxPAUSEMACCtrlFramesTransmitted, TxPauseFrames); + RMON_UPDATE(mac, TxJumboFramesReceivedOK, TxJumboFramesOK); + RMON_UPDATE(mac, TxJumboOctetsReceivedOK, TxJumboOctetsOK); + + return &mac->stats; +} + +static int pm3393_macaddress_get(struct cmac *cmac, u8 mac_addr[6]) +{ + memcpy(mac_addr, cmac->instance->mac_addr, ETH_ALEN); + return 0; +} + +static int pm3393_macaddress_set(struct cmac *cmac, const u8 ma[6]) +{ + u32 val, lo, mid, hi, enabled = cmac->instance->enabled; + + /* + * MAC addr: 00:07:43:00:13:09 + * + * ma[5] = 0x09 + * ma[4] = 0x13 + * ma[3] = 0x00 + * ma[2] = 0x43 + * ma[1] = 0x07 + * ma[0] = 0x00 + * + * The PM3393 requires byte swapping and reverse order entry + * when programming MAC addresses: + * + * low_bits[15:0] = ma[1]:ma[0] + * mid_bits[31:16] = ma[3]:ma[2] + * high_bits[47:32] = ma[5]:ma[4] + */ + + /* Store local copy */ + memcpy(cmac->instance->mac_addr, ma, ETH_ALEN); + + lo = ((u32) ma[1] << 8) | (u32) ma[0]; + mid = ((u32) ma[3] << 8) | (u32) ma[2]; + hi = ((u32) ma[5] << 8) | (u32) ma[4]; + + /* Disable Rx/Tx MAC before configuring it. */ + if (enabled) + pm3393_disable(cmac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); + + /* Set RXXG Station Address */ + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_SA_15_0, lo); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_SA_31_16, mid); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_SA_47_32, hi); + + /* Set TXXG Station Address */ + pmwrite(cmac, SUNI1x10GEXP_REG_TXXG_SA_15_0, lo); + pmwrite(cmac, SUNI1x10GEXP_REG_TXXG_SA_31_16, mid); + pmwrite(cmac, SUNI1x10GEXP_REG_TXXG_SA_47_32, hi); + + /* Setup Exact Match Filter 1 with our MAC address + * + * Must disable exact match filter before configuring it. + */ + pmread(cmac, SUNI1x10GEXP_REG_RXXG_ADDRESS_FILTER_CONTROL_0, &val); + val &= 0xff0f; + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_ADDRESS_FILTER_CONTROL_0, val); + + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_1_LOW, lo); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_1_MID, mid); + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_1_HIGH, hi); + + val |= 0x0090; + pmwrite(cmac, SUNI1x10GEXP_REG_RXXG_ADDRESS_FILTER_CONTROL_0, val); + + if (enabled) + pm3393_enable(cmac, enabled); + return 0; +} + +static void pm3393_destroy(struct cmac *cmac) +{ + kfree(cmac); +} + +static const struct cmac_ops pm3393_ops = { + .destroy = pm3393_destroy, + .reset = pm3393_reset, + .interrupt_enable = pm3393_interrupt_enable, + .interrupt_disable = pm3393_interrupt_disable, + .interrupt_clear = pm3393_interrupt_clear, + .interrupt_handler = pm3393_interrupt_handler, + .enable = pm3393_enable_port, + .disable = pm3393_disable, + .loopback_enable = pm3393_loopback_enable, + .loopback_disable = pm3393_loopback_disable, + .set_mtu = pm3393_set_mtu, + .set_rx_mode = pm3393_set_rx_mode, + .get_speed_duplex_fc = pm3393_get_speed_duplex_fc, + .set_speed_duplex_fc = pm3393_set_speed_duplex_fc, + .statistics_update = pm3393_update_statistics, + .macaddress_get = pm3393_macaddress_get, + .macaddress_set = pm3393_macaddress_set +}; + +static struct cmac *pm3393_mac_create(adapter_t *adapter, int index) +{ + struct cmac *cmac; + + cmac = kzalloc(sizeof(*cmac) + sizeof(cmac_instance), GFP_KERNEL); + if (!cmac) + return NULL; + + cmac->ops = &pm3393_ops; + cmac->instance = (cmac_instance *) (cmac + 1); + cmac->adapter = adapter; + cmac->instance->fc = PAUSE_TX | PAUSE_RX; + + t1_tpi_write(adapter, OFFSET(0x0001), 0x00008000); + t1_tpi_write(adapter, OFFSET(0x0001), 0x00000000); + t1_tpi_write(adapter, OFFSET(0x2308), 0x00009800); + t1_tpi_write(adapter, OFFSET(0x2305), 0x00001001); /* PL4IO Enable */ + t1_tpi_write(adapter, OFFSET(0x2320), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x2321), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x2322), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x2323), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x2324), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x2325), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x2326), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x2327), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x2328), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x2329), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x232a), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x232b), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x232c), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x232d), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x232e), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x232f), 0x00008800); + t1_tpi_write(adapter, OFFSET(0x230d), 0x00009c00); + t1_tpi_write(adapter, OFFSET(0x2304), 0x00000202); /* PL4IO Calendar Repetitions */ + + t1_tpi_write(adapter, OFFSET(0x3200), 0x00008080); /* EFLX Enable */ + t1_tpi_write(adapter, OFFSET(0x3210), 0x00000000); /* EFLX Channel Deprovision */ + t1_tpi_write(adapter, OFFSET(0x3203), 0x00000000); /* EFLX Low Limit */ + t1_tpi_write(adapter, OFFSET(0x3204), 0x00000040); /* EFLX High Limit */ + t1_tpi_write(adapter, OFFSET(0x3205), 0x000002cc); /* EFLX Almost Full */ + t1_tpi_write(adapter, OFFSET(0x3206), 0x00000199); /* EFLX Almost Empty */ + t1_tpi_write(adapter, OFFSET(0x3207), 0x00000240); /* EFLX Cut Through Threshold */ + t1_tpi_write(adapter, OFFSET(0x3202), 0x00000000); /* EFLX Indirect Register Update */ + t1_tpi_write(adapter, OFFSET(0x3210), 0x00000001); /* EFLX Channel Provision */ + t1_tpi_write(adapter, OFFSET(0x3208), 0x0000ffff); /* EFLX Undocumented */ + t1_tpi_write(adapter, OFFSET(0x320a), 0x0000ffff); /* EFLX Undocumented */ + t1_tpi_write(adapter, OFFSET(0x320c), 0x0000ffff); /* EFLX enable overflow interrupt The other bit are undocumented */ + t1_tpi_write(adapter, OFFSET(0x320e), 0x0000ffff); /* EFLX Undocumented */ + + t1_tpi_write(adapter, OFFSET(0x2200), 0x0000c000); /* IFLX Configuration - enable */ + t1_tpi_write(adapter, OFFSET(0x2201), 0x00000000); /* IFLX Channel Deprovision */ + t1_tpi_write(adapter, OFFSET(0x220e), 0x00000000); /* IFLX Low Limit */ + t1_tpi_write(adapter, OFFSET(0x220f), 0x00000100); /* IFLX High Limit */ + t1_tpi_write(adapter, OFFSET(0x2210), 0x00000c00); /* IFLX Almost Full Limit */ + t1_tpi_write(adapter, OFFSET(0x2211), 0x00000599); /* IFLX Almost Empty Limit */ + t1_tpi_write(adapter, OFFSET(0x220d), 0x00000000); /* IFLX Indirect Register Update */ + t1_tpi_write(adapter, OFFSET(0x2201), 0x00000001); /* IFLX Channel Provision */ + t1_tpi_write(adapter, OFFSET(0x2203), 0x0000ffff); /* IFLX Undocumented */ + t1_tpi_write(adapter, OFFSET(0x2205), 0x0000ffff); /* IFLX Undocumented */ + t1_tpi_write(adapter, OFFSET(0x2209), 0x0000ffff); /* IFLX Enable overflow interrupt. The other bit are undocumented */ + + t1_tpi_write(adapter, OFFSET(0x2241), 0xfffffffe); /* PL4MOS Undocumented */ + t1_tpi_write(adapter, OFFSET(0x2242), 0x0000ffff); /* PL4MOS Undocumented */ + t1_tpi_write(adapter, OFFSET(0x2243), 0x00000008); /* PL4MOS Starving Burst Size */ + t1_tpi_write(adapter, OFFSET(0x2244), 0x00000008); /* PL4MOS Hungry Burst Size */ + t1_tpi_write(adapter, OFFSET(0x2245), 0x00000008); /* PL4MOS Transfer Size */ + t1_tpi_write(adapter, OFFSET(0x2240), 0x00000005); /* PL4MOS Disable */ + + t1_tpi_write(adapter, OFFSET(0x2280), 0x00002103); /* PL4ODP Training Repeat and SOP rule */ + t1_tpi_write(adapter, OFFSET(0x2284), 0x00000000); /* PL4ODP MAX_T setting */ + + t1_tpi_write(adapter, OFFSET(0x3280), 0x00000087); /* PL4IDU Enable data forward, port state machine. Set ALLOW_NON_ZERO_OLB */ + t1_tpi_write(adapter, OFFSET(0x3282), 0x0000001f); /* PL4IDU Enable Dip4 check error interrupts */ + + t1_tpi_write(adapter, OFFSET(0x3040), 0x0c32); /* # TXXG Config */ + /* For T1 use timer based Mac flow control. */ + t1_tpi_write(adapter, OFFSET(0x304d), 0x8000); + t1_tpi_write(adapter, OFFSET(0x2040), 0x059c); /* # RXXG Config */ + t1_tpi_write(adapter, OFFSET(0x2049), 0x0001); /* # RXXG Cut Through */ + t1_tpi_write(adapter, OFFSET(0x2070), 0x0000); /* # Disable promiscuous mode */ + + /* Setup Exact Match Filter 0 to allow broadcast packets. + */ + t1_tpi_write(adapter, OFFSET(0x206e), 0x0000); /* # Disable Match Enable bit */ + t1_tpi_write(adapter, OFFSET(0x204a), 0xffff); /* # low addr */ + t1_tpi_write(adapter, OFFSET(0x204b), 0xffff); /* # mid addr */ + t1_tpi_write(adapter, OFFSET(0x204c), 0xffff); /* # high addr */ + t1_tpi_write(adapter, OFFSET(0x206e), 0x0009); /* # Enable Match Enable bit */ + + t1_tpi_write(adapter, OFFSET(0x0003), 0x0000); /* # NO SOP/ PAD_EN setup */ + t1_tpi_write(adapter, OFFSET(0x0100), 0x0ff0); /* # RXEQB disabled */ + t1_tpi_write(adapter, OFFSET(0x0101), 0x0f0f); /* # No Preemphasis */ + + return cmac; +} + +static int pm3393_mac_reset(adapter_t * adapter) +{ + u32 val; + u32 x; + u32 is_pl4_reset_finished; + u32 is_pl4_outof_lock; + u32 is_xaui_mabc_pll_locked; + u32 successful_reset; + int i; + + /* The following steps are required to properly reset + * the PM3393. This information is provided in the + * PM3393 datasheet (Issue 2: November 2002) + * section 13.1 -- Device Reset. + * + * The PM3393 has three types of components that are + * individually reset: + * + * DRESETB - Digital circuitry + * PL4_ARESETB - PL4 analog circuitry + * XAUI_ARESETB - XAUI bus analog circuitry + * + * Steps to reset PM3393 using RSTB pin: + * + * 1. Assert RSTB pin low ( write 0 ) + * 2. Wait at least 1ms to initiate a complete initialization of device. + * 3. Wait until all external clocks and REFSEL are stable. + * 4. Wait minimum of 1ms. (after external clocks and REFEL are stable) + * 5. De-assert RSTB ( write 1 ) + * 6. Wait until internal timers to expires after ~14ms. + * - Allows analog clock synthesizer(PL4CSU) to stabilize to + * selected reference frequency before allowing the digital + * portion of the device to operate. + * 7. Wait at least 200us for XAUI interface to stabilize. + * 8. Verify the PM3393 came out of reset successfully. + * Set successful reset flag if everything worked else try again + * a few more times. + */ + + successful_reset = 0; + for (i = 0; i < 3 && !successful_reset; i++) { + /* 1 */ + t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val &= ~1; + t1_tpi_write(adapter, A_ELMER0_GPO, val); + + /* 2 */ + msleep(1); + + /* 3 */ + msleep(1); + + /* 4 */ + msleep(2 /*1 extra ms for safety */ ); + + /* 5 */ + val |= 1; + t1_tpi_write(adapter, A_ELMER0_GPO, val); + + /* 6 */ + msleep(15 /*1 extra ms for safety */ ); + + /* 7 */ + msleep(1); + + /* 8 */ + + /* Has PL4 analog block come out of reset correctly? */ + t1_tpi_read(adapter, OFFSET(SUNI1x10GEXP_REG_DEVICE_STATUS), &val); + is_pl4_reset_finished = (val & SUNI1x10GEXP_BITMSK_TOP_EXPIRED); + + /* TBD XXX SUNI1x10GEXP_BITMSK_TOP_PL4_IS_DOOL gets locked later in the init sequence + * figure out why? */ + + /* Have all PL4 block clocks locked? */ + x = (SUNI1x10GEXP_BITMSK_TOP_PL4_ID_DOOL + /*| SUNI1x10GEXP_BITMSK_TOP_PL4_IS_DOOL */ | + SUNI1x10GEXP_BITMSK_TOP_PL4_ID_ROOL | + SUNI1x10GEXP_BITMSK_TOP_PL4_IS_ROOL | + SUNI1x10GEXP_BITMSK_TOP_PL4_OUT_ROOL); + is_pl4_outof_lock = (val & x); + + /* ??? If this fails, might be able to software reset the XAUI part + * and try to recover... thus saving us from doing another HW reset */ + /* Has the XAUI MABC PLL circuitry stablized? */ + is_xaui_mabc_pll_locked = + (val & SUNI1x10GEXP_BITMSK_TOP_SXRA_EXPIRED); + + successful_reset = (is_pl4_reset_finished && !is_pl4_outof_lock + && is_xaui_mabc_pll_locked); + + if (netif_msg_hw(adapter)) + dev_dbg(&adapter->pdev->dev, + "PM3393 HW reset %d: pl4_reset 0x%x, val 0x%x, " + "is_pl4_outof_lock 0x%x, xaui_locked 0x%x\n", + i, is_pl4_reset_finished, val, + is_pl4_outof_lock, is_xaui_mabc_pll_locked); + } + return successful_reset ? 0 : 1; +} + +const struct gmac t1_pm3393_ops = { + .stats_update_period = STATS_TICK_SECS, + .create = pm3393_mac_create, + .reset = pm3393_mac_reset, +}; diff --git a/drivers/net/ethernet/chelsio/cxgb/regs.h b/drivers/net/ethernet/chelsio/cxgb/regs.h new file mode 100644 index 000000000..f751e680c --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/regs.h @@ -0,0 +1,2158 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/***************************************************************************** + * * + * File: regs.h * + * $Revision: 1.8 $ * + * $Date: 2005/06/21 18:29:48 $ * + * Description: * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#ifndef _CXGB_REGS_H_ +#define _CXGB_REGS_H_ + +/* SGE registers */ +#define A_SG_CONTROL 0x0 + +#define S_CMDQ0_ENABLE 0 +#define V_CMDQ0_ENABLE(x) ((x) << S_CMDQ0_ENABLE) +#define F_CMDQ0_ENABLE V_CMDQ0_ENABLE(1U) + +#define S_CMDQ1_ENABLE 1 +#define V_CMDQ1_ENABLE(x) ((x) << S_CMDQ1_ENABLE) +#define F_CMDQ1_ENABLE V_CMDQ1_ENABLE(1U) + +#define S_FL0_ENABLE 2 +#define V_FL0_ENABLE(x) ((x) << S_FL0_ENABLE) +#define F_FL0_ENABLE V_FL0_ENABLE(1U) + +#define S_FL1_ENABLE 3 +#define V_FL1_ENABLE(x) ((x) << S_FL1_ENABLE) +#define F_FL1_ENABLE V_FL1_ENABLE(1U) + +#define S_CPL_ENABLE 4 +#define V_CPL_ENABLE(x) ((x) << S_CPL_ENABLE) +#define F_CPL_ENABLE V_CPL_ENABLE(1U) + +#define S_RESPONSE_QUEUE_ENABLE 5 +#define V_RESPONSE_QUEUE_ENABLE(x) ((x) << S_RESPONSE_QUEUE_ENABLE) +#define F_RESPONSE_QUEUE_ENABLE V_RESPONSE_QUEUE_ENABLE(1U) + +#define S_CMDQ_PRIORITY 6 +#define M_CMDQ_PRIORITY 0x3 +#define V_CMDQ_PRIORITY(x) ((x) << S_CMDQ_PRIORITY) +#define G_CMDQ_PRIORITY(x) (((x) >> S_CMDQ_PRIORITY) & M_CMDQ_PRIORITY) + +#define S_DISABLE_CMDQ0_GTS 8 +#define V_DISABLE_CMDQ0_GTS(x) ((x) << S_DISABLE_CMDQ0_GTS) +#define F_DISABLE_CMDQ0_GTS V_DISABLE_CMDQ0_GTS(1U) + +#define S_DISABLE_CMDQ1_GTS 9 +#define V_DISABLE_CMDQ1_GTS(x) ((x) << S_DISABLE_CMDQ1_GTS) +#define F_DISABLE_CMDQ1_GTS V_DISABLE_CMDQ1_GTS(1U) + +#define S_DISABLE_FL0_GTS 10 +#define V_DISABLE_FL0_GTS(x) ((x) << S_DISABLE_FL0_GTS) +#define F_DISABLE_FL0_GTS V_DISABLE_FL0_GTS(1U) + +#define S_DISABLE_FL1_GTS 11 +#define V_DISABLE_FL1_GTS(x) ((x) << S_DISABLE_FL1_GTS) +#define F_DISABLE_FL1_GTS V_DISABLE_FL1_GTS(1U) + +#define S_ENABLE_BIG_ENDIAN 12 +#define V_ENABLE_BIG_ENDIAN(x) ((x) << S_ENABLE_BIG_ENDIAN) +#define F_ENABLE_BIG_ENDIAN V_ENABLE_BIG_ENDIAN(1U) + +#define S_FL_SELECTION_CRITERIA 13 +#define V_FL_SELECTION_CRITERIA(x) ((x) << S_FL_SELECTION_CRITERIA) +#define F_FL_SELECTION_CRITERIA V_FL_SELECTION_CRITERIA(1U) + +#define S_ISCSI_COALESCE 14 +#define V_ISCSI_COALESCE(x) ((x) << S_ISCSI_COALESCE) +#define F_ISCSI_COALESCE V_ISCSI_COALESCE(1U) + +#define S_RX_PKT_OFFSET 15 +#define M_RX_PKT_OFFSET 0x7 +#define V_RX_PKT_OFFSET(x) ((x) << S_RX_PKT_OFFSET) +#define G_RX_PKT_OFFSET(x) (((x) >> S_RX_PKT_OFFSET) & M_RX_PKT_OFFSET) + +#define S_VLAN_XTRACT 18 +#define V_VLAN_XTRACT(x) ((x) << S_VLAN_XTRACT) +#define F_VLAN_XTRACT V_VLAN_XTRACT(1U) + +#define A_SG_DOORBELL 0x4 +#define A_SG_CMD0BASELWR 0x8 +#define A_SG_CMD0BASEUPR 0xc +#define A_SG_CMD1BASELWR 0x10 +#define A_SG_CMD1BASEUPR 0x14 +#define A_SG_FL0BASELWR 0x18 +#define A_SG_FL0BASEUPR 0x1c +#define A_SG_FL1BASELWR 0x20 +#define A_SG_FL1BASEUPR 0x24 +#define A_SG_CMD0SIZE 0x28 + +#define S_CMDQ0_SIZE 0 +#define M_CMDQ0_SIZE 0x1ffff +#define V_CMDQ0_SIZE(x) ((x) << S_CMDQ0_SIZE) +#define G_CMDQ0_SIZE(x) (((x) >> S_CMDQ0_SIZE) & M_CMDQ0_SIZE) + +#define A_SG_FL0SIZE 0x2c + +#define S_FL0_SIZE 0 +#define M_FL0_SIZE 0x1ffff +#define V_FL0_SIZE(x) ((x) << S_FL0_SIZE) +#define G_FL0_SIZE(x) (((x) >> S_FL0_SIZE) & M_FL0_SIZE) + +#define A_SG_RSPSIZE 0x30 + +#define S_RESPQ_SIZE 0 +#define M_RESPQ_SIZE 0x1ffff +#define V_RESPQ_SIZE(x) ((x) << S_RESPQ_SIZE) +#define G_RESPQ_SIZE(x) (((x) >> S_RESPQ_SIZE) & M_RESPQ_SIZE) + +#define A_SG_RSPBASELWR 0x34 +#define A_SG_RSPBASEUPR 0x38 +#define A_SG_FLTHRESHOLD 0x3c + +#define S_FL_THRESHOLD 0 +#define M_FL_THRESHOLD 0xffff +#define V_FL_THRESHOLD(x) ((x) << S_FL_THRESHOLD) +#define G_FL_THRESHOLD(x) (((x) >> S_FL_THRESHOLD) & M_FL_THRESHOLD) + +#define A_SG_RSPQUEUECREDIT 0x40 + +#define S_RESPQ_CREDIT 0 +#define M_RESPQ_CREDIT 0x1ffff +#define V_RESPQ_CREDIT(x) ((x) << S_RESPQ_CREDIT) +#define G_RESPQ_CREDIT(x) (((x) >> S_RESPQ_CREDIT) & M_RESPQ_CREDIT) + +#define A_SG_SLEEPING 0x48 + +#define S_SLEEPING 0 +#define M_SLEEPING 0xffff +#define V_SLEEPING(x) ((x) << S_SLEEPING) +#define G_SLEEPING(x) (((x) >> S_SLEEPING) & M_SLEEPING) + +#define A_SG_INTRTIMER 0x4c + +#define S_INTERRUPT_TIMER_COUNT 0 +#define M_INTERRUPT_TIMER_COUNT 0xffffff +#define V_INTERRUPT_TIMER_COUNT(x) ((x) << S_INTERRUPT_TIMER_COUNT) +#define G_INTERRUPT_TIMER_COUNT(x) (((x) >> S_INTERRUPT_TIMER_COUNT) & M_INTERRUPT_TIMER_COUNT) + +#define A_SG_CMD0PTR 0x50 + +#define S_CMDQ0_POINTER 0 +#define M_CMDQ0_POINTER 0xffff +#define V_CMDQ0_POINTER(x) ((x) << S_CMDQ0_POINTER) +#define G_CMDQ0_POINTER(x) (((x) >> S_CMDQ0_POINTER) & M_CMDQ0_POINTER) + +#define S_CURRENT_GENERATION_BIT 16 +#define V_CURRENT_GENERATION_BIT(x) ((x) << S_CURRENT_GENERATION_BIT) +#define F_CURRENT_GENERATION_BIT V_CURRENT_GENERATION_BIT(1U) + +#define A_SG_CMD1PTR 0x54 + +#define S_CMDQ1_POINTER 0 +#define M_CMDQ1_POINTER 0xffff +#define V_CMDQ1_POINTER(x) ((x) << S_CMDQ1_POINTER) +#define G_CMDQ1_POINTER(x) (((x) >> S_CMDQ1_POINTER) & M_CMDQ1_POINTER) + +#define A_SG_FL0PTR 0x58 + +#define S_FL0_POINTER 0 +#define M_FL0_POINTER 0xffff +#define V_FL0_POINTER(x) ((x) << S_FL0_POINTER) +#define G_FL0_POINTER(x) (((x) >> S_FL0_POINTER) & M_FL0_POINTER) + +#define A_SG_FL1PTR 0x5c + +#define S_FL1_POINTER 0 +#define M_FL1_POINTER 0xffff +#define V_FL1_POINTER(x) ((x) << S_FL1_POINTER) +#define G_FL1_POINTER(x) (((x) >> S_FL1_POINTER) & M_FL1_POINTER) + +#define A_SG_VERSION 0x6c + +#define S_DAY 0 +#define M_DAY 0x1f +#define V_DAY(x) ((x) << S_DAY) +#define G_DAY(x) (((x) >> S_DAY) & M_DAY) + +#define S_MONTH 5 +#define M_MONTH 0xf +#define V_MONTH(x) ((x) << S_MONTH) +#define G_MONTH(x) (((x) >> S_MONTH) & M_MONTH) + +#define A_SG_CMD1SIZE 0xb0 + +#define S_CMDQ1_SIZE 0 +#define M_CMDQ1_SIZE 0x1ffff +#define V_CMDQ1_SIZE(x) ((x) << S_CMDQ1_SIZE) +#define G_CMDQ1_SIZE(x) (((x) >> S_CMDQ1_SIZE) & M_CMDQ1_SIZE) + +#define A_SG_FL1SIZE 0xb4 + +#define S_FL1_SIZE 0 +#define M_FL1_SIZE 0x1ffff +#define V_FL1_SIZE(x) ((x) << S_FL1_SIZE) +#define G_FL1_SIZE(x) (((x) >> S_FL1_SIZE) & M_FL1_SIZE) + +#define A_SG_INT_ENABLE 0xb8 + +#define S_RESPQ_EXHAUSTED 0 +#define V_RESPQ_EXHAUSTED(x) ((x) << S_RESPQ_EXHAUSTED) +#define F_RESPQ_EXHAUSTED V_RESPQ_EXHAUSTED(1U) + +#define S_RESPQ_OVERFLOW 1 +#define V_RESPQ_OVERFLOW(x) ((x) << S_RESPQ_OVERFLOW) +#define F_RESPQ_OVERFLOW V_RESPQ_OVERFLOW(1U) + +#define S_FL_EXHAUSTED 2 +#define V_FL_EXHAUSTED(x) ((x) << S_FL_EXHAUSTED) +#define F_FL_EXHAUSTED V_FL_EXHAUSTED(1U) + +#define S_PACKET_TOO_BIG 3 +#define V_PACKET_TOO_BIG(x) ((x) << S_PACKET_TOO_BIG) +#define F_PACKET_TOO_BIG V_PACKET_TOO_BIG(1U) + +#define S_PACKET_MISMATCH 4 +#define V_PACKET_MISMATCH(x) ((x) << S_PACKET_MISMATCH) +#define F_PACKET_MISMATCH V_PACKET_MISMATCH(1U) + +#define A_SG_INT_CAUSE 0xbc +#define A_SG_RESPACCUTIMER 0xc0 + +/* MC3 registers */ +#define A_MC3_CFG 0x100 + +#define S_CLK_ENABLE 0 +#define V_CLK_ENABLE(x) ((x) << S_CLK_ENABLE) +#define F_CLK_ENABLE V_CLK_ENABLE(1U) + +#define S_READY 1 +#define V_READY(x) ((x) << S_READY) +#define F_READY V_READY(1U) + +#define S_READ_TO_WRITE_DELAY 2 +#define M_READ_TO_WRITE_DELAY 0x7 +#define V_READ_TO_WRITE_DELAY(x) ((x) << S_READ_TO_WRITE_DELAY) +#define G_READ_TO_WRITE_DELAY(x) (((x) >> S_READ_TO_WRITE_DELAY) & M_READ_TO_WRITE_DELAY) + +#define S_WRITE_TO_READ_DELAY 5 +#define M_WRITE_TO_READ_DELAY 0x7 +#define V_WRITE_TO_READ_DELAY(x) ((x) << S_WRITE_TO_READ_DELAY) +#define G_WRITE_TO_READ_DELAY(x) (((x) >> S_WRITE_TO_READ_DELAY) & M_WRITE_TO_READ_DELAY) + +#define S_MC3_BANK_CYCLE 8 +#define M_MC3_BANK_CYCLE 0xf +#define V_MC3_BANK_CYCLE(x) ((x) << S_MC3_BANK_CYCLE) +#define G_MC3_BANK_CYCLE(x) (((x) >> S_MC3_BANK_CYCLE) & M_MC3_BANK_CYCLE) + +#define S_REFRESH_CYCLE 12 +#define M_REFRESH_CYCLE 0xf +#define V_REFRESH_CYCLE(x) ((x) << S_REFRESH_CYCLE) +#define G_REFRESH_CYCLE(x) (((x) >> S_REFRESH_CYCLE) & M_REFRESH_CYCLE) + +#define S_PRECHARGE_CYCLE 16 +#define M_PRECHARGE_CYCLE 0x3 +#define V_PRECHARGE_CYCLE(x) ((x) << S_PRECHARGE_CYCLE) +#define G_PRECHARGE_CYCLE(x) (((x) >> S_PRECHARGE_CYCLE) & M_PRECHARGE_CYCLE) + +#define S_ACTIVE_TO_READ_WRITE_DELAY 18 +#define V_ACTIVE_TO_READ_WRITE_DELAY(x) ((x) << S_ACTIVE_TO_READ_WRITE_DELAY) +#define F_ACTIVE_TO_READ_WRITE_DELAY V_ACTIVE_TO_READ_WRITE_DELAY(1U) + +#define S_ACTIVE_TO_PRECHARGE_DELAY 19 +#define M_ACTIVE_TO_PRECHARGE_DELAY 0x7 +#define V_ACTIVE_TO_PRECHARGE_DELAY(x) ((x) << S_ACTIVE_TO_PRECHARGE_DELAY) +#define G_ACTIVE_TO_PRECHARGE_DELAY(x) (((x) >> S_ACTIVE_TO_PRECHARGE_DELAY) & M_ACTIVE_TO_PRECHARGE_DELAY) + +#define S_WRITE_RECOVERY_DELAY 22 +#define M_WRITE_RECOVERY_DELAY 0x3 +#define V_WRITE_RECOVERY_DELAY(x) ((x) << S_WRITE_RECOVERY_DELAY) +#define G_WRITE_RECOVERY_DELAY(x) (((x) >> S_WRITE_RECOVERY_DELAY) & M_WRITE_RECOVERY_DELAY) + +#define S_DENSITY 24 +#define M_DENSITY 0x3 +#define V_DENSITY(x) ((x) << S_DENSITY) +#define G_DENSITY(x) (((x) >> S_DENSITY) & M_DENSITY) + +#define S_ORGANIZATION 26 +#define V_ORGANIZATION(x) ((x) << S_ORGANIZATION) +#define F_ORGANIZATION V_ORGANIZATION(1U) + +#define S_BANKS 27 +#define V_BANKS(x) ((x) << S_BANKS) +#define F_BANKS V_BANKS(1U) + +#define S_UNREGISTERED 28 +#define V_UNREGISTERED(x) ((x) << S_UNREGISTERED) +#define F_UNREGISTERED V_UNREGISTERED(1U) + +#define S_MC3_WIDTH 29 +#define M_MC3_WIDTH 0x3 +#define V_MC3_WIDTH(x) ((x) << S_MC3_WIDTH) +#define G_MC3_WIDTH(x) (((x) >> S_MC3_WIDTH) & M_MC3_WIDTH) + +#define S_MC3_SLOW 31 +#define V_MC3_SLOW(x) ((x) << S_MC3_SLOW) +#define F_MC3_SLOW V_MC3_SLOW(1U) + +#define A_MC3_MODE 0x104 + +#define S_MC3_MODE 0 +#define M_MC3_MODE 0x3fff +#define V_MC3_MODE(x) ((x) << S_MC3_MODE) +#define G_MC3_MODE(x) (((x) >> S_MC3_MODE) & M_MC3_MODE) + +#define S_BUSY 31 +#define V_BUSY(x) ((x) << S_BUSY) +#define F_BUSY V_BUSY(1U) + +#define A_MC3_EXT_MODE 0x108 + +#define S_MC3_EXTENDED_MODE 0 +#define M_MC3_EXTENDED_MODE 0x3fff +#define V_MC3_EXTENDED_MODE(x) ((x) << S_MC3_EXTENDED_MODE) +#define G_MC3_EXTENDED_MODE(x) (((x) >> S_MC3_EXTENDED_MODE) & M_MC3_EXTENDED_MODE) + +#define A_MC3_PRECHARG 0x10c +#define A_MC3_REFRESH 0x110 + +#define S_REFRESH_ENABLE 0 +#define V_REFRESH_ENABLE(x) ((x) << S_REFRESH_ENABLE) +#define F_REFRESH_ENABLE V_REFRESH_ENABLE(1U) + +#define S_REFRESH_DIVISOR 1 +#define M_REFRESH_DIVISOR 0x3fff +#define V_REFRESH_DIVISOR(x) ((x) << S_REFRESH_DIVISOR) +#define G_REFRESH_DIVISOR(x) (((x) >> S_REFRESH_DIVISOR) & M_REFRESH_DIVISOR) + +#define A_MC3_STROBE 0x114 + +#define S_MASTER_DLL_RESET 0 +#define V_MASTER_DLL_RESET(x) ((x) << S_MASTER_DLL_RESET) +#define F_MASTER_DLL_RESET V_MASTER_DLL_RESET(1U) + +#define S_MASTER_DLL_TAP_COUNT 1 +#define M_MASTER_DLL_TAP_COUNT 0xff +#define V_MASTER_DLL_TAP_COUNT(x) ((x) << S_MASTER_DLL_TAP_COUNT) +#define G_MASTER_DLL_TAP_COUNT(x) (((x) >> S_MASTER_DLL_TAP_COUNT) & M_MASTER_DLL_TAP_COUNT) + +#define S_MASTER_DLL_LOCKED 9 +#define V_MASTER_DLL_LOCKED(x) ((x) << S_MASTER_DLL_LOCKED) +#define F_MASTER_DLL_LOCKED V_MASTER_DLL_LOCKED(1U) + +#define S_MASTER_DLL_MAX_TAP_COUNT 10 +#define V_MASTER_DLL_MAX_TAP_COUNT(x) ((x) << S_MASTER_DLL_MAX_TAP_COUNT) +#define F_MASTER_DLL_MAX_TAP_COUNT V_MASTER_DLL_MAX_TAP_COUNT(1U) + +#define S_MASTER_DLL_TAP_COUNT_OFFSET 11 +#define M_MASTER_DLL_TAP_COUNT_OFFSET 0x3f +#define V_MASTER_DLL_TAP_COUNT_OFFSET(x) ((x) << S_MASTER_DLL_TAP_COUNT_OFFSET) +#define G_MASTER_DLL_TAP_COUNT_OFFSET(x) (((x) >> S_MASTER_DLL_TAP_COUNT_OFFSET) & M_MASTER_DLL_TAP_COUNT_OFFSET) + +#define S_SLAVE_DLL_RESET 11 +#define V_SLAVE_DLL_RESET(x) ((x) << S_SLAVE_DLL_RESET) +#define F_SLAVE_DLL_RESET V_SLAVE_DLL_RESET(1U) + +#define S_SLAVE_DLL_DELTA 12 +#define M_SLAVE_DLL_DELTA 0xf +#define V_SLAVE_DLL_DELTA(x) ((x) << S_SLAVE_DLL_DELTA) +#define G_SLAVE_DLL_DELTA(x) (((x) >> S_SLAVE_DLL_DELTA) & M_SLAVE_DLL_DELTA) + +#define S_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT 17 +#define M_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT 0x3f +#define V_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT(x) ((x) << S_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT) +#define G_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT(x) (((x) >> S_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT) & M_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT) + +#define S_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT_ENABLE 23 +#define V_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT_ENABLE(x) ((x) << S_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT_ENABLE) +#define F_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT_ENABLE V_SLAVE_DELAY_LINE_MANUAL_TAP_COUNT_ENABLE(1U) + +#define S_SLAVE_DELAY_LINE_TAP_COUNT 24 +#define M_SLAVE_DELAY_LINE_TAP_COUNT 0x3f +#define V_SLAVE_DELAY_LINE_TAP_COUNT(x) ((x) << S_SLAVE_DELAY_LINE_TAP_COUNT) +#define G_SLAVE_DELAY_LINE_TAP_COUNT(x) (((x) >> S_SLAVE_DELAY_LINE_TAP_COUNT) & M_SLAVE_DELAY_LINE_TAP_COUNT) + +#define A_MC3_ECC_CNTL 0x118 + +#define S_ECC_GENERATION_ENABLE 0 +#define V_ECC_GENERATION_ENABLE(x) ((x) << S_ECC_GENERATION_ENABLE) +#define F_ECC_GENERATION_ENABLE V_ECC_GENERATION_ENABLE(1U) + +#define S_ECC_CHECK_ENABLE 1 +#define V_ECC_CHECK_ENABLE(x) ((x) << S_ECC_CHECK_ENABLE) +#define F_ECC_CHECK_ENABLE V_ECC_CHECK_ENABLE(1U) + +#define S_CORRECTABLE_ERROR_COUNT 2 +#define M_CORRECTABLE_ERROR_COUNT 0xff +#define V_CORRECTABLE_ERROR_COUNT(x) ((x) << S_CORRECTABLE_ERROR_COUNT) +#define G_CORRECTABLE_ERROR_COUNT(x) (((x) >> S_CORRECTABLE_ERROR_COUNT) & M_CORRECTABLE_ERROR_COUNT) + +#define S_UNCORRECTABLE_ERROR_COUNT 10 +#define M_UNCORRECTABLE_ERROR_COUNT 0xff +#define V_UNCORRECTABLE_ERROR_COUNT(x) ((x) << S_UNCORRECTABLE_ERROR_COUNT) +#define G_UNCORRECTABLE_ERROR_COUNT(x) (((x) >> S_UNCORRECTABLE_ERROR_COUNT) & M_UNCORRECTABLE_ERROR_COUNT) + +#define A_MC3_CE_ADDR 0x11c + +#define S_MC3_CE_ADDR 4 +#define M_MC3_CE_ADDR 0xfffffff +#define V_MC3_CE_ADDR(x) ((x) << S_MC3_CE_ADDR) +#define G_MC3_CE_ADDR(x) (((x) >> S_MC3_CE_ADDR) & M_MC3_CE_ADDR) + +#define A_MC3_CE_DATA0 0x120 +#define A_MC3_CE_DATA1 0x124 +#define A_MC3_CE_DATA2 0x128 +#define A_MC3_CE_DATA3 0x12c +#define A_MC3_CE_DATA4 0x130 +#define A_MC3_UE_ADDR 0x134 + +#define S_MC3_UE_ADDR 4 +#define M_MC3_UE_ADDR 0xfffffff +#define V_MC3_UE_ADDR(x) ((x) << S_MC3_UE_ADDR) +#define G_MC3_UE_ADDR(x) (((x) >> S_MC3_UE_ADDR) & M_MC3_UE_ADDR) + +#define A_MC3_UE_DATA0 0x138 +#define A_MC3_UE_DATA1 0x13c +#define A_MC3_UE_DATA2 0x140 +#define A_MC3_UE_DATA3 0x144 +#define A_MC3_UE_DATA4 0x148 +#define A_MC3_BD_ADDR 0x14c +#define A_MC3_BD_DATA0 0x150 +#define A_MC3_BD_DATA1 0x154 +#define A_MC3_BD_DATA2 0x158 +#define A_MC3_BD_DATA3 0x15c +#define A_MC3_BD_DATA4 0x160 +#define A_MC3_BD_OP 0x164 + +#define S_BACK_DOOR_OPERATION 0 +#define V_BACK_DOOR_OPERATION(x) ((x) << S_BACK_DOOR_OPERATION) +#define F_BACK_DOOR_OPERATION V_BACK_DOOR_OPERATION(1U) + +#define A_MC3_BIST_ADDR_BEG 0x168 +#define A_MC3_BIST_ADDR_END 0x16c +#define A_MC3_BIST_DATA 0x170 +#define A_MC3_BIST_OP 0x174 + +#define S_OP 0 +#define V_OP(x) ((x) << S_OP) +#define F_OP V_OP(1U) + +#define S_DATA_PATTERN 1 +#define M_DATA_PATTERN 0x3 +#define V_DATA_PATTERN(x) ((x) << S_DATA_PATTERN) +#define G_DATA_PATTERN(x) (((x) >> S_DATA_PATTERN) & M_DATA_PATTERN) + +#define S_CONTINUOUS 3 +#define V_CONTINUOUS(x) ((x) << S_CONTINUOUS) +#define F_CONTINUOUS V_CONTINUOUS(1U) + +#define A_MC3_INT_ENABLE 0x178 + +#define S_MC3_CORR_ERR 0 +#define V_MC3_CORR_ERR(x) ((x) << S_MC3_CORR_ERR) +#define F_MC3_CORR_ERR V_MC3_CORR_ERR(1U) + +#define S_MC3_UNCORR_ERR 1 +#define V_MC3_UNCORR_ERR(x) ((x) << S_MC3_UNCORR_ERR) +#define F_MC3_UNCORR_ERR V_MC3_UNCORR_ERR(1U) + +#define S_MC3_PARITY_ERR 2 +#define M_MC3_PARITY_ERR 0xff +#define V_MC3_PARITY_ERR(x) ((x) << S_MC3_PARITY_ERR) +#define G_MC3_PARITY_ERR(x) (((x) >> S_MC3_PARITY_ERR) & M_MC3_PARITY_ERR) + +#define S_MC3_ADDR_ERR 10 +#define V_MC3_ADDR_ERR(x) ((x) << S_MC3_ADDR_ERR) +#define F_MC3_ADDR_ERR V_MC3_ADDR_ERR(1U) + +#define A_MC3_INT_CAUSE 0x17c + +/* MC4 registers */ +#define A_MC4_CFG 0x180 + +#define S_POWER_UP 0 +#define V_POWER_UP(x) ((x) << S_POWER_UP) +#define F_POWER_UP V_POWER_UP(1U) + +#define S_MC4_BANK_CYCLE 8 +#define M_MC4_BANK_CYCLE 0x7 +#define V_MC4_BANK_CYCLE(x) ((x) << S_MC4_BANK_CYCLE) +#define G_MC4_BANK_CYCLE(x) (((x) >> S_MC4_BANK_CYCLE) & M_MC4_BANK_CYCLE) + +#define S_MC4_NARROW 24 +#define V_MC4_NARROW(x) ((x) << S_MC4_NARROW) +#define F_MC4_NARROW V_MC4_NARROW(1U) + +#define S_MC4_SLOW 25 +#define V_MC4_SLOW(x) ((x) << S_MC4_SLOW) +#define F_MC4_SLOW V_MC4_SLOW(1U) + +#define S_MC4A_WIDTH 24 +#define M_MC4A_WIDTH 0x3 +#define V_MC4A_WIDTH(x) ((x) << S_MC4A_WIDTH) +#define G_MC4A_WIDTH(x) (((x) >> S_MC4A_WIDTH) & M_MC4A_WIDTH) + +#define S_MC4A_SLOW 26 +#define V_MC4A_SLOW(x) ((x) << S_MC4A_SLOW) +#define F_MC4A_SLOW V_MC4A_SLOW(1U) + +#define A_MC4_MODE 0x184 + +#define S_MC4_MODE 0 +#define M_MC4_MODE 0x7fff +#define V_MC4_MODE(x) ((x) << S_MC4_MODE) +#define G_MC4_MODE(x) (((x) >> S_MC4_MODE) & M_MC4_MODE) + +#define A_MC4_EXT_MODE 0x188 + +#define S_MC4_EXTENDED_MODE 0 +#define M_MC4_EXTENDED_MODE 0x7fff +#define V_MC4_EXTENDED_MODE(x) ((x) << S_MC4_EXTENDED_MODE) +#define G_MC4_EXTENDED_MODE(x) (((x) >> S_MC4_EXTENDED_MODE) & M_MC4_EXTENDED_MODE) + +#define A_MC4_REFRESH 0x190 +#define A_MC4_STROBE 0x194 +#define A_MC4_ECC_CNTL 0x198 +#define A_MC4_CE_ADDR 0x19c + +#define S_MC4_CE_ADDR 4 +#define M_MC4_CE_ADDR 0xffffff +#define V_MC4_CE_ADDR(x) ((x) << S_MC4_CE_ADDR) +#define G_MC4_CE_ADDR(x) (((x) >> S_MC4_CE_ADDR) & M_MC4_CE_ADDR) + +#define A_MC4_CE_DATA0 0x1a0 +#define A_MC4_CE_DATA1 0x1a4 +#define A_MC4_CE_DATA2 0x1a8 +#define A_MC4_CE_DATA3 0x1ac +#define A_MC4_CE_DATA4 0x1b0 +#define A_MC4_UE_ADDR 0x1b4 + +#define S_MC4_UE_ADDR 4 +#define M_MC4_UE_ADDR 0xffffff +#define V_MC4_UE_ADDR(x) ((x) << S_MC4_UE_ADDR) +#define G_MC4_UE_ADDR(x) (((x) >> S_MC4_UE_ADDR) & M_MC4_UE_ADDR) + +#define A_MC4_UE_DATA0 0x1b8 +#define A_MC4_UE_DATA1 0x1bc +#define A_MC4_UE_DATA2 0x1c0 +#define A_MC4_UE_DATA3 0x1c4 +#define A_MC4_UE_DATA4 0x1c8 +#define A_MC4_BD_ADDR 0x1cc + +#define S_MC4_BACK_DOOR_ADDR 0 +#define M_MC4_BACK_DOOR_ADDR 0xfffffff +#define V_MC4_BACK_DOOR_ADDR(x) ((x) << S_MC4_BACK_DOOR_ADDR) +#define G_MC4_BACK_DOOR_ADDR(x) (((x) >> S_MC4_BACK_DOOR_ADDR) & M_MC4_BACK_DOOR_ADDR) + +#define A_MC4_BD_DATA0 0x1d0 +#define A_MC4_BD_DATA1 0x1d4 +#define A_MC4_BD_DATA2 0x1d8 +#define A_MC4_BD_DATA3 0x1dc +#define A_MC4_BD_DATA4 0x1e0 +#define A_MC4_BD_OP 0x1e4 + +#define S_OPERATION 0 +#define V_OPERATION(x) ((x) << S_OPERATION) +#define F_OPERATION V_OPERATION(1U) + +#define A_MC4_BIST_ADDR_BEG 0x1e8 +#define A_MC4_BIST_ADDR_END 0x1ec +#define A_MC4_BIST_DATA 0x1f0 +#define A_MC4_BIST_OP 0x1f4 +#define A_MC4_INT_ENABLE 0x1f8 + +#define S_MC4_CORR_ERR 0 +#define V_MC4_CORR_ERR(x) ((x) << S_MC4_CORR_ERR) +#define F_MC4_CORR_ERR V_MC4_CORR_ERR(1U) + +#define S_MC4_UNCORR_ERR 1 +#define V_MC4_UNCORR_ERR(x) ((x) << S_MC4_UNCORR_ERR) +#define F_MC4_UNCORR_ERR V_MC4_UNCORR_ERR(1U) + +#define S_MC4_ADDR_ERR 2 +#define V_MC4_ADDR_ERR(x) ((x) << S_MC4_ADDR_ERR) +#define F_MC4_ADDR_ERR V_MC4_ADDR_ERR(1U) + +#define A_MC4_INT_CAUSE 0x1fc + +/* TPI registers */ +#define A_TPI_ADDR 0x280 + +#define S_TPI_ADDRESS 0 +#define M_TPI_ADDRESS 0xffffff +#define V_TPI_ADDRESS(x) ((x) << S_TPI_ADDRESS) +#define G_TPI_ADDRESS(x) (((x) >> S_TPI_ADDRESS) & M_TPI_ADDRESS) + +#define A_TPI_WR_DATA 0x284 +#define A_TPI_RD_DATA 0x288 +#define A_TPI_CSR 0x28c + +#define S_TPIWR 0 +#define V_TPIWR(x) ((x) << S_TPIWR) +#define F_TPIWR V_TPIWR(1U) + +#define S_TPIRDY 1 +#define V_TPIRDY(x) ((x) << S_TPIRDY) +#define F_TPIRDY V_TPIRDY(1U) + +#define S_INT_DIR 31 +#define V_INT_DIR(x) ((x) << S_INT_DIR) +#define F_INT_DIR V_INT_DIR(1U) + +#define A_TPI_PAR 0x29c + +#define S_TPIPAR 0 +#define M_TPIPAR 0x7f +#define V_TPIPAR(x) ((x) << S_TPIPAR) +#define G_TPIPAR(x) (((x) >> S_TPIPAR) & M_TPIPAR) + + +/* TP registers */ +#define A_TP_IN_CONFIG 0x300 + +#define S_TP_IN_CSPI_TUNNEL 0 +#define V_TP_IN_CSPI_TUNNEL(x) ((x) << S_TP_IN_CSPI_TUNNEL) +#define F_TP_IN_CSPI_TUNNEL V_TP_IN_CSPI_TUNNEL(1U) + +#define S_TP_IN_CSPI_ETHERNET 1 +#define V_TP_IN_CSPI_ETHERNET(x) ((x) << S_TP_IN_CSPI_ETHERNET) +#define F_TP_IN_CSPI_ETHERNET V_TP_IN_CSPI_ETHERNET(1U) + +#define S_TP_IN_CSPI_CPL 3 +#define V_TP_IN_CSPI_CPL(x) ((x) << S_TP_IN_CSPI_CPL) +#define F_TP_IN_CSPI_CPL V_TP_IN_CSPI_CPL(1U) + +#define S_TP_IN_CSPI_POS 4 +#define V_TP_IN_CSPI_POS(x) ((x) << S_TP_IN_CSPI_POS) +#define F_TP_IN_CSPI_POS V_TP_IN_CSPI_POS(1U) + +#define S_TP_IN_CSPI_CHECK_IP_CSUM 5 +#define V_TP_IN_CSPI_CHECK_IP_CSUM(x) ((x) << S_TP_IN_CSPI_CHECK_IP_CSUM) +#define F_TP_IN_CSPI_CHECK_IP_CSUM V_TP_IN_CSPI_CHECK_IP_CSUM(1U) + +#define S_TP_IN_CSPI_CHECK_TCP_CSUM 6 +#define V_TP_IN_CSPI_CHECK_TCP_CSUM(x) ((x) << S_TP_IN_CSPI_CHECK_TCP_CSUM) +#define F_TP_IN_CSPI_CHECK_TCP_CSUM V_TP_IN_CSPI_CHECK_TCP_CSUM(1U) + +#define S_TP_IN_ESPI_TUNNEL 7 +#define V_TP_IN_ESPI_TUNNEL(x) ((x) << S_TP_IN_ESPI_TUNNEL) +#define F_TP_IN_ESPI_TUNNEL V_TP_IN_ESPI_TUNNEL(1U) + +#define S_TP_IN_ESPI_ETHERNET 8 +#define V_TP_IN_ESPI_ETHERNET(x) ((x) << S_TP_IN_ESPI_ETHERNET) +#define F_TP_IN_ESPI_ETHERNET V_TP_IN_ESPI_ETHERNET(1U) + +#define S_TP_IN_ESPI_CPL 10 +#define V_TP_IN_ESPI_CPL(x) ((x) << S_TP_IN_ESPI_CPL) +#define F_TP_IN_ESPI_CPL V_TP_IN_ESPI_CPL(1U) + +#define S_TP_IN_ESPI_POS 11 +#define V_TP_IN_ESPI_POS(x) ((x) << S_TP_IN_ESPI_POS) +#define F_TP_IN_ESPI_POS V_TP_IN_ESPI_POS(1U) + +#define S_TP_IN_ESPI_CHECK_IP_CSUM 12 +#define V_TP_IN_ESPI_CHECK_IP_CSUM(x) ((x) << S_TP_IN_ESPI_CHECK_IP_CSUM) +#define F_TP_IN_ESPI_CHECK_IP_CSUM V_TP_IN_ESPI_CHECK_IP_CSUM(1U) + +#define S_TP_IN_ESPI_CHECK_TCP_CSUM 13 +#define V_TP_IN_ESPI_CHECK_TCP_CSUM(x) ((x) << S_TP_IN_ESPI_CHECK_TCP_CSUM) +#define F_TP_IN_ESPI_CHECK_TCP_CSUM V_TP_IN_ESPI_CHECK_TCP_CSUM(1U) + +#define S_OFFLOAD_DISABLE 14 +#define V_OFFLOAD_DISABLE(x) ((x) << S_OFFLOAD_DISABLE) +#define F_OFFLOAD_DISABLE V_OFFLOAD_DISABLE(1U) + +#define A_TP_OUT_CONFIG 0x304 + +#define S_TP_OUT_C_ETH 0 +#define V_TP_OUT_C_ETH(x) ((x) << S_TP_OUT_C_ETH) +#define F_TP_OUT_C_ETH V_TP_OUT_C_ETH(1U) + +#define S_TP_OUT_CSPI_CPL 2 +#define V_TP_OUT_CSPI_CPL(x) ((x) << S_TP_OUT_CSPI_CPL) +#define F_TP_OUT_CSPI_CPL V_TP_OUT_CSPI_CPL(1U) + +#define S_TP_OUT_CSPI_POS 3 +#define V_TP_OUT_CSPI_POS(x) ((x) << S_TP_OUT_CSPI_POS) +#define F_TP_OUT_CSPI_POS V_TP_OUT_CSPI_POS(1U) + +#define S_TP_OUT_CSPI_GENERATE_IP_CSUM 4 +#define V_TP_OUT_CSPI_GENERATE_IP_CSUM(x) ((x) << S_TP_OUT_CSPI_GENERATE_IP_CSUM) +#define F_TP_OUT_CSPI_GENERATE_IP_CSUM V_TP_OUT_CSPI_GENERATE_IP_CSUM(1U) + +#define S_TP_OUT_CSPI_GENERATE_TCP_CSUM 5 +#define V_TP_OUT_CSPI_GENERATE_TCP_CSUM(x) ((x) << S_TP_OUT_CSPI_GENERATE_TCP_CSUM) +#define F_TP_OUT_CSPI_GENERATE_TCP_CSUM V_TP_OUT_CSPI_GENERATE_TCP_CSUM(1U) + +#define S_TP_OUT_ESPI_ETHERNET 6 +#define V_TP_OUT_ESPI_ETHERNET(x) ((x) << S_TP_OUT_ESPI_ETHERNET) +#define F_TP_OUT_ESPI_ETHERNET V_TP_OUT_ESPI_ETHERNET(1U) + +#define S_TP_OUT_ESPI_TAG_ETHERNET 7 +#define V_TP_OUT_ESPI_TAG_ETHERNET(x) ((x) << S_TP_OUT_ESPI_TAG_ETHERNET) +#define F_TP_OUT_ESPI_TAG_ETHERNET V_TP_OUT_ESPI_TAG_ETHERNET(1U) + +#define S_TP_OUT_ESPI_CPL 8 +#define V_TP_OUT_ESPI_CPL(x) ((x) << S_TP_OUT_ESPI_CPL) +#define F_TP_OUT_ESPI_CPL V_TP_OUT_ESPI_CPL(1U) + +#define S_TP_OUT_ESPI_POS 9 +#define V_TP_OUT_ESPI_POS(x) ((x) << S_TP_OUT_ESPI_POS) +#define F_TP_OUT_ESPI_POS V_TP_OUT_ESPI_POS(1U) + +#define S_TP_OUT_ESPI_GENERATE_IP_CSUM 10 +#define V_TP_OUT_ESPI_GENERATE_IP_CSUM(x) ((x) << S_TP_OUT_ESPI_GENERATE_IP_CSUM) +#define F_TP_OUT_ESPI_GENERATE_IP_CSUM V_TP_OUT_ESPI_GENERATE_IP_CSUM(1U) + +#define S_TP_OUT_ESPI_GENERATE_TCP_CSUM 11 +#define V_TP_OUT_ESPI_GENERATE_TCP_CSUM(x) ((x) << S_TP_OUT_ESPI_GENERATE_TCP_CSUM) +#define F_TP_OUT_ESPI_GENERATE_TCP_CSUM V_TP_OUT_ESPI_GENERATE_TCP_CSUM(1U) + +#define A_TP_GLOBAL_CONFIG 0x308 + +#define S_IP_TTL 0 +#define M_IP_TTL 0xff +#define V_IP_TTL(x) ((x) << S_IP_TTL) +#define G_IP_TTL(x) (((x) >> S_IP_TTL) & M_IP_TTL) + +#define S_TCAM_SERVER_REGION_USAGE 8 +#define M_TCAM_SERVER_REGION_USAGE 0x3 +#define V_TCAM_SERVER_REGION_USAGE(x) ((x) << S_TCAM_SERVER_REGION_USAGE) +#define G_TCAM_SERVER_REGION_USAGE(x) (((x) >> S_TCAM_SERVER_REGION_USAGE) & M_TCAM_SERVER_REGION_USAGE) + +#define S_QOS_MAPPING 10 +#define V_QOS_MAPPING(x) ((x) << S_QOS_MAPPING) +#define F_QOS_MAPPING V_QOS_MAPPING(1U) + +#define S_TCP_CSUM 11 +#define V_TCP_CSUM(x) ((x) << S_TCP_CSUM) +#define F_TCP_CSUM V_TCP_CSUM(1U) + +#define S_UDP_CSUM 12 +#define V_UDP_CSUM(x) ((x) << S_UDP_CSUM) +#define F_UDP_CSUM V_UDP_CSUM(1U) + +#define S_IP_CSUM 13 +#define V_IP_CSUM(x) ((x) << S_IP_CSUM) +#define F_IP_CSUM V_IP_CSUM(1U) + +#define S_IP_ID_SPLIT 14 +#define V_IP_ID_SPLIT(x) ((x) << S_IP_ID_SPLIT) +#define F_IP_ID_SPLIT V_IP_ID_SPLIT(1U) + +#define S_PATH_MTU 15 +#define V_PATH_MTU(x) ((x) << S_PATH_MTU) +#define F_PATH_MTU V_PATH_MTU(1U) + +#define S_5TUPLE_LOOKUP 17 +#define M_5TUPLE_LOOKUP 0x3 +#define V_5TUPLE_LOOKUP(x) ((x) << S_5TUPLE_LOOKUP) +#define G_5TUPLE_LOOKUP(x) (((x) >> S_5TUPLE_LOOKUP) & M_5TUPLE_LOOKUP) + +#define S_IP_FRAGMENT_DROP 19 +#define V_IP_FRAGMENT_DROP(x) ((x) << S_IP_FRAGMENT_DROP) +#define F_IP_FRAGMENT_DROP V_IP_FRAGMENT_DROP(1U) + +#define S_PING_DROP 20 +#define V_PING_DROP(x) ((x) << S_PING_DROP) +#define F_PING_DROP V_PING_DROP(1U) + +#define S_PROTECT_MODE 21 +#define V_PROTECT_MODE(x) ((x) << S_PROTECT_MODE) +#define F_PROTECT_MODE V_PROTECT_MODE(1U) + +#define S_SYN_COOKIE_ALGORITHM 22 +#define V_SYN_COOKIE_ALGORITHM(x) ((x) << S_SYN_COOKIE_ALGORITHM) +#define F_SYN_COOKIE_ALGORITHM V_SYN_COOKIE_ALGORITHM(1U) + +#define S_ATTACK_FILTER 23 +#define V_ATTACK_FILTER(x) ((x) << S_ATTACK_FILTER) +#define F_ATTACK_FILTER V_ATTACK_FILTER(1U) + +#define S_INTERFACE_TYPE 24 +#define V_INTERFACE_TYPE(x) ((x) << S_INTERFACE_TYPE) +#define F_INTERFACE_TYPE V_INTERFACE_TYPE(1U) + +#define S_DISABLE_RX_FLOW_CONTROL 25 +#define V_DISABLE_RX_FLOW_CONTROL(x) ((x) << S_DISABLE_RX_FLOW_CONTROL) +#define F_DISABLE_RX_FLOW_CONTROL V_DISABLE_RX_FLOW_CONTROL(1U) + +#define S_SYN_COOKIE_PARAMETER 26 +#define M_SYN_COOKIE_PARAMETER 0x3f +#define V_SYN_COOKIE_PARAMETER(x) ((x) << S_SYN_COOKIE_PARAMETER) +#define G_SYN_COOKIE_PARAMETER(x) (((x) >> S_SYN_COOKIE_PARAMETER) & M_SYN_COOKIE_PARAMETER) + +#define A_TP_GLOBAL_RX_CREDITS 0x30c +#define A_TP_CM_SIZE 0x310 +#define A_TP_CM_MM_BASE 0x314 + +#define S_CM_MEMMGR_BASE 0 +#define M_CM_MEMMGR_BASE 0xfffffff +#define V_CM_MEMMGR_BASE(x) ((x) << S_CM_MEMMGR_BASE) +#define G_CM_MEMMGR_BASE(x) (((x) >> S_CM_MEMMGR_BASE) & M_CM_MEMMGR_BASE) + +#define A_TP_CM_TIMER_BASE 0x318 + +#define S_CM_TIMER_BASE 0 +#define M_CM_TIMER_BASE 0xfffffff +#define V_CM_TIMER_BASE(x) ((x) << S_CM_TIMER_BASE) +#define G_CM_TIMER_BASE(x) (((x) >> S_CM_TIMER_BASE) & M_CM_TIMER_BASE) + +#define A_TP_PM_SIZE 0x31c +#define A_TP_PM_TX_BASE 0x320 +#define A_TP_PM_DEFRAG_BASE 0x324 +#define A_TP_PM_RX_BASE 0x328 +#define A_TP_PM_RX_PG_SIZE 0x32c +#define A_TP_PM_RX_MAX_PGS 0x330 +#define A_TP_PM_TX_PG_SIZE 0x334 +#define A_TP_PM_TX_MAX_PGS 0x338 +#define A_TP_TCP_OPTIONS 0x340 + +#define S_TIMESTAMP 0 +#define M_TIMESTAMP 0x3 +#define V_TIMESTAMP(x) ((x) << S_TIMESTAMP) +#define G_TIMESTAMP(x) (((x) >> S_TIMESTAMP) & M_TIMESTAMP) + +#define S_WINDOW_SCALE 2 +#define M_WINDOW_SCALE 0x3 +#define V_WINDOW_SCALE(x) ((x) << S_WINDOW_SCALE) +#define G_WINDOW_SCALE(x) (((x) >> S_WINDOW_SCALE) & M_WINDOW_SCALE) + +#define S_SACK 4 +#define M_SACK 0x3 +#define V_SACK(x) ((x) << S_SACK) +#define G_SACK(x) (((x) >> S_SACK) & M_SACK) + +#define S_ECN 6 +#define M_ECN 0x3 +#define V_ECN(x) ((x) << S_ECN) +#define G_ECN(x) (((x) >> S_ECN) & M_ECN) + +#define S_SACK_ALGORITHM 8 +#define M_SACK_ALGORITHM 0x3 +#define V_SACK_ALGORITHM(x) ((x) << S_SACK_ALGORITHM) +#define G_SACK_ALGORITHM(x) (((x) >> S_SACK_ALGORITHM) & M_SACK_ALGORITHM) + +#define S_MSS 10 +#define V_MSS(x) ((x) << S_MSS) +#define F_MSS V_MSS(1U) + +#define S_DEFAULT_PEER_MSS 16 +#define M_DEFAULT_PEER_MSS 0xffff +#define V_DEFAULT_PEER_MSS(x) ((x) << S_DEFAULT_PEER_MSS) +#define G_DEFAULT_PEER_MSS(x) (((x) >> S_DEFAULT_PEER_MSS) & M_DEFAULT_PEER_MSS) + +#define A_TP_DACK_CONFIG 0x344 + +#define S_DACK_MODE 0 +#define V_DACK_MODE(x) ((x) << S_DACK_MODE) +#define F_DACK_MODE V_DACK_MODE(1U) + +#define S_DACK_AUTO_MGMT 1 +#define V_DACK_AUTO_MGMT(x) ((x) << S_DACK_AUTO_MGMT) +#define F_DACK_AUTO_MGMT V_DACK_AUTO_MGMT(1U) + +#define S_DACK_AUTO_CAREFUL 2 +#define V_DACK_AUTO_CAREFUL(x) ((x) << S_DACK_AUTO_CAREFUL) +#define F_DACK_AUTO_CAREFUL V_DACK_AUTO_CAREFUL(1U) + +#define S_DACK_MSS_SELECTOR 3 +#define M_DACK_MSS_SELECTOR 0x3 +#define V_DACK_MSS_SELECTOR(x) ((x) << S_DACK_MSS_SELECTOR) +#define G_DACK_MSS_SELECTOR(x) (((x) >> S_DACK_MSS_SELECTOR) & M_DACK_MSS_SELECTOR) + +#define S_DACK_BYTE_THRESHOLD 5 +#define M_DACK_BYTE_THRESHOLD 0xfffff +#define V_DACK_BYTE_THRESHOLD(x) ((x) << S_DACK_BYTE_THRESHOLD) +#define G_DACK_BYTE_THRESHOLD(x) (((x) >> S_DACK_BYTE_THRESHOLD) & M_DACK_BYTE_THRESHOLD) + +#define A_TP_PC_CONFIG 0x348 + +#define S_TP_ACCESS_LATENCY 0 +#define M_TP_ACCESS_LATENCY 0xf +#define V_TP_ACCESS_LATENCY(x) ((x) << S_TP_ACCESS_LATENCY) +#define G_TP_ACCESS_LATENCY(x) (((x) >> S_TP_ACCESS_LATENCY) & M_TP_ACCESS_LATENCY) + +#define S_HELD_FIN_DISABLE 4 +#define V_HELD_FIN_DISABLE(x) ((x) << S_HELD_FIN_DISABLE) +#define F_HELD_FIN_DISABLE V_HELD_FIN_DISABLE(1U) + +#define S_DDP_FC_ENABLE 5 +#define V_DDP_FC_ENABLE(x) ((x) << S_DDP_FC_ENABLE) +#define F_DDP_FC_ENABLE V_DDP_FC_ENABLE(1U) + +#define S_RDMA_ERR_ENABLE 6 +#define V_RDMA_ERR_ENABLE(x) ((x) << S_RDMA_ERR_ENABLE) +#define F_RDMA_ERR_ENABLE V_RDMA_ERR_ENABLE(1U) + +#define S_FAST_PDU_DELIVERY 7 +#define V_FAST_PDU_DELIVERY(x) ((x) << S_FAST_PDU_DELIVERY) +#define F_FAST_PDU_DELIVERY V_FAST_PDU_DELIVERY(1U) + +#define S_CLEAR_FIN 8 +#define V_CLEAR_FIN(x) ((x) << S_CLEAR_FIN) +#define F_CLEAR_FIN V_CLEAR_FIN(1U) + +#define S_DIS_TX_FILL_WIN_PUSH 12 +#define V_DIS_TX_FILL_WIN_PUSH(x) ((x) << S_DIS_TX_FILL_WIN_PUSH) +#define F_DIS_TX_FILL_WIN_PUSH V_DIS_TX_FILL_WIN_PUSH(1U) + +#define S_TP_PC_REV 30 +#define M_TP_PC_REV 0x3 +#define V_TP_PC_REV(x) ((x) << S_TP_PC_REV) +#define G_TP_PC_REV(x) (((x) >> S_TP_PC_REV) & M_TP_PC_REV) + +#define A_TP_BACKOFF0 0x350 + +#define S_ELEMENT0 0 +#define M_ELEMENT0 0xff +#define V_ELEMENT0(x) ((x) << S_ELEMENT0) +#define G_ELEMENT0(x) (((x) >> S_ELEMENT0) & M_ELEMENT0) + +#define S_ELEMENT1 8 +#define M_ELEMENT1 0xff +#define V_ELEMENT1(x) ((x) << S_ELEMENT1) +#define G_ELEMENT1(x) (((x) >> S_ELEMENT1) & M_ELEMENT1) + +#define S_ELEMENT2 16 +#define M_ELEMENT2 0xff +#define V_ELEMENT2(x) ((x) << S_ELEMENT2) +#define G_ELEMENT2(x) (((x) >> S_ELEMENT2) & M_ELEMENT2) + +#define S_ELEMENT3 24 +#define M_ELEMENT3 0xff +#define V_ELEMENT3(x) ((x) << S_ELEMENT3) +#define G_ELEMENT3(x) (((x) >> S_ELEMENT3) & M_ELEMENT3) + +#define A_TP_BACKOFF1 0x354 +#define A_TP_BACKOFF2 0x358 +#define A_TP_BACKOFF3 0x35c +#define A_TP_PARA_REG0 0x360 + +#define S_VAR_MULT 0 +#define M_VAR_MULT 0xf +#define V_VAR_MULT(x) ((x) << S_VAR_MULT) +#define G_VAR_MULT(x) (((x) >> S_VAR_MULT) & M_VAR_MULT) + +#define S_VAR_GAIN 4 +#define M_VAR_GAIN 0xf +#define V_VAR_GAIN(x) ((x) << S_VAR_GAIN) +#define G_VAR_GAIN(x) (((x) >> S_VAR_GAIN) & M_VAR_GAIN) + +#define S_SRTT_GAIN 8 +#define M_SRTT_GAIN 0xf +#define V_SRTT_GAIN(x) ((x) << S_SRTT_GAIN) +#define G_SRTT_GAIN(x) (((x) >> S_SRTT_GAIN) & M_SRTT_GAIN) + +#define S_RTTVAR_INIT 12 +#define M_RTTVAR_INIT 0xf +#define V_RTTVAR_INIT(x) ((x) << S_RTTVAR_INIT) +#define G_RTTVAR_INIT(x) (((x) >> S_RTTVAR_INIT) & M_RTTVAR_INIT) + +#define S_DUP_THRESH 20 +#define M_DUP_THRESH 0xf +#define V_DUP_THRESH(x) ((x) << S_DUP_THRESH) +#define G_DUP_THRESH(x) (((x) >> S_DUP_THRESH) & M_DUP_THRESH) + +#define S_INIT_CONG_WIN 24 +#define M_INIT_CONG_WIN 0x7 +#define V_INIT_CONG_WIN(x) ((x) << S_INIT_CONG_WIN) +#define G_INIT_CONG_WIN(x) (((x) >> S_INIT_CONG_WIN) & M_INIT_CONG_WIN) + +#define A_TP_PARA_REG1 0x364 + +#define S_INITIAL_SLOW_START_THRESHOLD 0 +#define M_INITIAL_SLOW_START_THRESHOLD 0xffff +#define V_INITIAL_SLOW_START_THRESHOLD(x) ((x) << S_INITIAL_SLOW_START_THRESHOLD) +#define G_INITIAL_SLOW_START_THRESHOLD(x) (((x) >> S_INITIAL_SLOW_START_THRESHOLD) & M_INITIAL_SLOW_START_THRESHOLD) + +#define S_RECEIVE_BUFFER_SIZE 16 +#define M_RECEIVE_BUFFER_SIZE 0xffff +#define V_RECEIVE_BUFFER_SIZE(x) ((x) << S_RECEIVE_BUFFER_SIZE) +#define G_RECEIVE_BUFFER_SIZE(x) (((x) >> S_RECEIVE_BUFFER_SIZE) & M_RECEIVE_BUFFER_SIZE) + +#define A_TP_PARA_REG2 0x368 + +#define S_RX_COALESCE_SIZE 0 +#define M_RX_COALESCE_SIZE 0xffff +#define V_RX_COALESCE_SIZE(x) ((x) << S_RX_COALESCE_SIZE) +#define G_RX_COALESCE_SIZE(x) (((x) >> S_RX_COALESCE_SIZE) & M_RX_COALESCE_SIZE) + +#define S_MAX_RX_SIZE 16 +#define M_MAX_RX_SIZE 0xffff +#define V_MAX_RX_SIZE(x) ((x) << S_MAX_RX_SIZE) +#define G_MAX_RX_SIZE(x) (((x) >> S_MAX_RX_SIZE) & M_MAX_RX_SIZE) + +#define A_TP_PARA_REG3 0x36c + +#define S_RX_COALESCING_PSH_DELIVER 0 +#define V_RX_COALESCING_PSH_DELIVER(x) ((x) << S_RX_COALESCING_PSH_DELIVER) +#define F_RX_COALESCING_PSH_DELIVER V_RX_COALESCING_PSH_DELIVER(1U) + +#define S_RX_COALESCING_ENABLE 1 +#define V_RX_COALESCING_ENABLE(x) ((x) << S_RX_COALESCING_ENABLE) +#define F_RX_COALESCING_ENABLE V_RX_COALESCING_ENABLE(1U) + +#define S_TAHOE_ENABLE 2 +#define V_TAHOE_ENABLE(x) ((x) << S_TAHOE_ENABLE) +#define F_TAHOE_ENABLE V_TAHOE_ENABLE(1U) + +#define S_MAX_REORDER_FRAGMENTS 12 +#define M_MAX_REORDER_FRAGMENTS 0x7 +#define V_MAX_REORDER_FRAGMENTS(x) ((x) << S_MAX_REORDER_FRAGMENTS) +#define G_MAX_REORDER_FRAGMENTS(x) (((x) >> S_MAX_REORDER_FRAGMENTS) & M_MAX_REORDER_FRAGMENTS) + +#define A_TP_TIMER_RESOLUTION 0x390 + +#define S_DELAYED_ACK_TIMER_RESOLUTION 0 +#define M_DELAYED_ACK_TIMER_RESOLUTION 0x3f +#define V_DELAYED_ACK_TIMER_RESOLUTION(x) ((x) << S_DELAYED_ACK_TIMER_RESOLUTION) +#define G_DELAYED_ACK_TIMER_RESOLUTION(x) (((x) >> S_DELAYED_ACK_TIMER_RESOLUTION) & M_DELAYED_ACK_TIMER_RESOLUTION) + +#define S_GENERIC_TIMER_RESOLUTION 16 +#define M_GENERIC_TIMER_RESOLUTION 0x3f +#define V_GENERIC_TIMER_RESOLUTION(x) ((x) << S_GENERIC_TIMER_RESOLUTION) +#define G_GENERIC_TIMER_RESOLUTION(x) (((x) >> S_GENERIC_TIMER_RESOLUTION) & M_GENERIC_TIMER_RESOLUTION) + +#define A_TP_2MSL 0x394 + +#define S_2MSL 0 +#define M_2MSL 0x3fffffff +#define V_2MSL(x) ((x) << S_2MSL) +#define G_2MSL(x) (((x) >> S_2MSL) & M_2MSL) + +#define A_TP_RXT_MIN 0x398 + +#define S_RETRANSMIT_TIMER_MIN 0 +#define M_RETRANSMIT_TIMER_MIN 0xffff +#define V_RETRANSMIT_TIMER_MIN(x) ((x) << S_RETRANSMIT_TIMER_MIN) +#define G_RETRANSMIT_TIMER_MIN(x) (((x) >> S_RETRANSMIT_TIMER_MIN) & M_RETRANSMIT_TIMER_MIN) + +#define A_TP_RXT_MAX 0x39c + +#define S_RETRANSMIT_TIMER_MAX 0 +#define M_RETRANSMIT_TIMER_MAX 0x3fffffff +#define V_RETRANSMIT_TIMER_MAX(x) ((x) << S_RETRANSMIT_TIMER_MAX) +#define G_RETRANSMIT_TIMER_MAX(x) (((x) >> S_RETRANSMIT_TIMER_MAX) & M_RETRANSMIT_TIMER_MAX) + +#define A_TP_PERS_MIN 0x3a0 + +#define S_PERSIST_TIMER_MIN 0 +#define M_PERSIST_TIMER_MIN 0xffff +#define V_PERSIST_TIMER_MIN(x) ((x) << S_PERSIST_TIMER_MIN) +#define G_PERSIST_TIMER_MIN(x) (((x) >> S_PERSIST_TIMER_MIN) & M_PERSIST_TIMER_MIN) + +#define A_TP_PERS_MAX 0x3a4 + +#define S_PERSIST_TIMER_MAX 0 +#define M_PERSIST_TIMER_MAX 0x3fffffff +#define V_PERSIST_TIMER_MAX(x) ((x) << S_PERSIST_TIMER_MAX) +#define G_PERSIST_TIMER_MAX(x) (((x) >> S_PERSIST_TIMER_MAX) & M_PERSIST_TIMER_MAX) + +#define A_TP_KEEP_IDLE 0x3ac + +#define S_KEEP_ALIVE_IDLE_TIME 0 +#define M_KEEP_ALIVE_IDLE_TIME 0x3fffffff +#define V_KEEP_ALIVE_IDLE_TIME(x) ((x) << S_KEEP_ALIVE_IDLE_TIME) +#define G_KEEP_ALIVE_IDLE_TIME(x) (((x) >> S_KEEP_ALIVE_IDLE_TIME) & M_KEEP_ALIVE_IDLE_TIME) + +#define A_TP_KEEP_INTVL 0x3b0 + +#define S_KEEP_ALIVE_INTERVAL_TIME 0 +#define M_KEEP_ALIVE_INTERVAL_TIME 0x3fffffff +#define V_KEEP_ALIVE_INTERVAL_TIME(x) ((x) << S_KEEP_ALIVE_INTERVAL_TIME) +#define G_KEEP_ALIVE_INTERVAL_TIME(x) (((x) >> S_KEEP_ALIVE_INTERVAL_TIME) & M_KEEP_ALIVE_INTERVAL_TIME) + +#define A_TP_INIT_SRTT 0x3b4 + +#define S_INITIAL_SRTT 0 +#define M_INITIAL_SRTT 0xffff +#define V_INITIAL_SRTT(x) ((x) << S_INITIAL_SRTT) +#define G_INITIAL_SRTT(x) (((x) >> S_INITIAL_SRTT) & M_INITIAL_SRTT) + +#define A_TP_DACK_TIME 0x3b8 + +#define S_DELAYED_ACK_TIME 0 +#define M_DELAYED_ACK_TIME 0x7ff +#define V_DELAYED_ACK_TIME(x) ((x) << S_DELAYED_ACK_TIME) +#define G_DELAYED_ACK_TIME(x) (((x) >> S_DELAYED_ACK_TIME) & M_DELAYED_ACK_TIME) + +#define A_TP_FINWAIT2_TIME 0x3bc + +#define S_FINWAIT2_TIME 0 +#define M_FINWAIT2_TIME 0x3fffffff +#define V_FINWAIT2_TIME(x) ((x) << S_FINWAIT2_TIME) +#define G_FINWAIT2_TIME(x) (((x) >> S_FINWAIT2_TIME) & M_FINWAIT2_TIME) + +#define A_TP_FAST_FINWAIT2_TIME 0x3c0 + +#define S_FAST_FINWAIT2_TIME 0 +#define M_FAST_FINWAIT2_TIME 0x3fffffff +#define V_FAST_FINWAIT2_TIME(x) ((x) << S_FAST_FINWAIT2_TIME) +#define G_FAST_FINWAIT2_TIME(x) (((x) >> S_FAST_FINWAIT2_TIME) & M_FAST_FINWAIT2_TIME) + +#define A_TP_SHIFT_CNT 0x3c4 + +#define S_KEEPALIVE_MAX 0 +#define M_KEEPALIVE_MAX 0xff +#define V_KEEPALIVE_MAX(x) ((x) << S_KEEPALIVE_MAX) +#define G_KEEPALIVE_MAX(x) (((x) >> S_KEEPALIVE_MAX) & M_KEEPALIVE_MAX) + +#define S_WINDOWPROBE_MAX 8 +#define M_WINDOWPROBE_MAX 0xff +#define V_WINDOWPROBE_MAX(x) ((x) << S_WINDOWPROBE_MAX) +#define G_WINDOWPROBE_MAX(x) (((x) >> S_WINDOWPROBE_MAX) & M_WINDOWPROBE_MAX) + +#define S_RETRANSMISSION_MAX 16 +#define M_RETRANSMISSION_MAX 0xff +#define V_RETRANSMISSION_MAX(x) ((x) << S_RETRANSMISSION_MAX) +#define G_RETRANSMISSION_MAX(x) (((x) >> S_RETRANSMISSION_MAX) & M_RETRANSMISSION_MAX) + +#define S_SYN_MAX 24 +#define M_SYN_MAX 0xff +#define V_SYN_MAX(x) ((x) << S_SYN_MAX) +#define G_SYN_MAX(x) (((x) >> S_SYN_MAX) & M_SYN_MAX) + +#define A_TP_QOS_REG0 0x3e0 + +#define S_L3_VALUE 0 +#define M_L3_VALUE 0x3f +#define V_L3_VALUE(x) ((x) << S_L3_VALUE) +#define G_L3_VALUE(x) (((x) >> S_L3_VALUE) & M_L3_VALUE) + +#define A_TP_QOS_REG1 0x3e4 +#define A_TP_QOS_REG2 0x3e8 +#define A_TP_QOS_REG3 0x3ec +#define A_TP_QOS_REG4 0x3f0 +#define A_TP_QOS_REG5 0x3f4 +#define A_TP_QOS_REG6 0x3f8 +#define A_TP_QOS_REG7 0x3fc +#define A_TP_MTU_REG0 0x404 +#define A_TP_MTU_REG1 0x408 +#define A_TP_MTU_REG2 0x40c +#define A_TP_MTU_REG3 0x410 +#define A_TP_MTU_REG4 0x414 +#define A_TP_MTU_REG5 0x418 +#define A_TP_MTU_REG6 0x41c +#define A_TP_MTU_REG7 0x420 +#define A_TP_RESET 0x44c + +#define S_TP_RESET 0 +#define V_TP_RESET(x) ((x) << S_TP_RESET) +#define F_TP_RESET V_TP_RESET(1U) + +#define S_CM_MEMMGR_INIT 1 +#define V_CM_MEMMGR_INIT(x) ((x) << S_CM_MEMMGR_INIT) +#define F_CM_MEMMGR_INIT V_CM_MEMMGR_INIT(1U) + +#define A_TP_MIB_INDEX 0x450 +#define A_TP_MIB_DATA 0x454 +#define A_TP_SYNC_TIME_HI 0x458 +#define A_TP_SYNC_TIME_LO 0x45c +#define A_TP_CM_MM_RX_FLST_BASE 0x460 + +#define S_CM_MEMMGR_RX_FREE_LIST_BASE 0 +#define M_CM_MEMMGR_RX_FREE_LIST_BASE 0xfffffff +#define V_CM_MEMMGR_RX_FREE_LIST_BASE(x) ((x) << S_CM_MEMMGR_RX_FREE_LIST_BASE) +#define G_CM_MEMMGR_RX_FREE_LIST_BASE(x) (((x) >> S_CM_MEMMGR_RX_FREE_LIST_BASE) & M_CM_MEMMGR_RX_FREE_LIST_BASE) + +#define A_TP_CM_MM_TX_FLST_BASE 0x464 + +#define S_CM_MEMMGR_TX_FREE_LIST_BASE 0 +#define M_CM_MEMMGR_TX_FREE_LIST_BASE 0xfffffff +#define V_CM_MEMMGR_TX_FREE_LIST_BASE(x) ((x) << S_CM_MEMMGR_TX_FREE_LIST_BASE) +#define G_CM_MEMMGR_TX_FREE_LIST_BASE(x) (((x) >> S_CM_MEMMGR_TX_FREE_LIST_BASE) & M_CM_MEMMGR_TX_FREE_LIST_BASE) + +#define A_TP_CM_MM_P_FLST_BASE 0x468 + +#define S_CM_MEMMGR_PSTRUCT_FREE_LIST_BASE 0 +#define M_CM_MEMMGR_PSTRUCT_FREE_LIST_BASE 0xfffffff +#define V_CM_MEMMGR_PSTRUCT_FREE_LIST_BASE(x) ((x) << S_CM_MEMMGR_PSTRUCT_FREE_LIST_BASE) +#define G_CM_MEMMGR_PSTRUCT_FREE_LIST_BASE(x) (((x) >> S_CM_MEMMGR_PSTRUCT_FREE_LIST_BASE) & M_CM_MEMMGR_PSTRUCT_FREE_LIST_BASE) + +#define A_TP_CM_MM_MAX_P 0x46c + +#define S_CM_MEMMGR_MAX_PSTRUCT 0 +#define M_CM_MEMMGR_MAX_PSTRUCT 0xfffffff +#define V_CM_MEMMGR_MAX_PSTRUCT(x) ((x) << S_CM_MEMMGR_MAX_PSTRUCT) +#define G_CM_MEMMGR_MAX_PSTRUCT(x) (((x) >> S_CM_MEMMGR_MAX_PSTRUCT) & M_CM_MEMMGR_MAX_PSTRUCT) + +#define A_TP_INT_ENABLE 0x470 + +#define S_TX_FREE_LIST_EMPTY 0 +#define V_TX_FREE_LIST_EMPTY(x) ((x) << S_TX_FREE_LIST_EMPTY) +#define F_TX_FREE_LIST_EMPTY V_TX_FREE_LIST_EMPTY(1U) + +#define S_RX_FREE_LIST_EMPTY 1 +#define V_RX_FREE_LIST_EMPTY(x) ((x) << S_RX_FREE_LIST_EMPTY) +#define F_RX_FREE_LIST_EMPTY V_RX_FREE_LIST_EMPTY(1U) + +#define A_TP_INT_CAUSE 0x474 +#define A_TP_TIMER_SEPARATOR 0x4a4 + +#define S_DISABLE_PAST_TIMER_INSERTION 0 +#define V_DISABLE_PAST_TIMER_INSERTION(x) ((x) << S_DISABLE_PAST_TIMER_INSERTION) +#define F_DISABLE_PAST_TIMER_INSERTION V_DISABLE_PAST_TIMER_INSERTION(1U) + +#define S_MODULATION_TIMER_SEPARATOR 1 +#define M_MODULATION_TIMER_SEPARATOR 0x7fff +#define V_MODULATION_TIMER_SEPARATOR(x) ((x) << S_MODULATION_TIMER_SEPARATOR) +#define G_MODULATION_TIMER_SEPARATOR(x) (((x) >> S_MODULATION_TIMER_SEPARATOR) & M_MODULATION_TIMER_SEPARATOR) + +#define S_GLOBAL_TIMER_SEPARATOR 16 +#define M_GLOBAL_TIMER_SEPARATOR 0xffff +#define V_GLOBAL_TIMER_SEPARATOR(x) ((x) << S_GLOBAL_TIMER_SEPARATOR) +#define G_GLOBAL_TIMER_SEPARATOR(x) (((x) >> S_GLOBAL_TIMER_SEPARATOR) & M_GLOBAL_TIMER_SEPARATOR) + +#define A_TP_CM_FC_MODE 0x4b0 +#define A_TP_PC_CONGESTION_CNTL 0x4b4 +#define A_TP_TX_DROP_CONFIG 0x4b8 + +#define S_ENABLE_TX_DROP 31 +#define V_ENABLE_TX_DROP(x) ((x) << S_ENABLE_TX_DROP) +#define F_ENABLE_TX_DROP V_ENABLE_TX_DROP(1U) + +#define S_ENABLE_TX_ERROR 30 +#define V_ENABLE_TX_ERROR(x) ((x) << S_ENABLE_TX_ERROR) +#define F_ENABLE_TX_ERROR V_ENABLE_TX_ERROR(1U) + +#define S_DROP_TICKS_CNT 4 +#define M_DROP_TICKS_CNT 0x3ffffff +#define V_DROP_TICKS_CNT(x) ((x) << S_DROP_TICKS_CNT) +#define G_DROP_TICKS_CNT(x) (((x) >> S_DROP_TICKS_CNT) & M_DROP_TICKS_CNT) + +#define S_NUM_PKTS_DROPPED 0 +#define M_NUM_PKTS_DROPPED 0xf +#define V_NUM_PKTS_DROPPED(x) ((x) << S_NUM_PKTS_DROPPED) +#define G_NUM_PKTS_DROPPED(x) (((x) >> S_NUM_PKTS_DROPPED) & M_NUM_PKTS_DROPPED) + +#define A_TP_TX_DROP_COUNT 0x4bc + +/* RAT registers */ +#define A_RAT_ROUTE_CONTROL 0x580 + +#define S_USE_ROUTE_TABLE 0 +#define V_USE_ROUTE_TABLE(x) ((x) << S_USE_ROUTE_TABLE) +#define F_USE_ROUTE_TABLE V_USE_ROUTE_TABLE(1U) + +#define S_ENABLE_CSPI 1 +#define V_ENABLE_CSPI(x) ((x) << S_ENABLE_CSPI) +#define F_ENABLE_CSPI V_ENABLE_CSPI(1U) + +#define S_ENABLE_PCIX 2 +#define V_ENABLE_PCIX(x) ((x) << S_ENABLE_PCIX) +#define F_ENABLE_PCIX V_ENABLE_PCIX(1U) + +#define A_RAT_ROUTE_TABLE_INDEX 0x584 + +#define S_ROUTE_TABLE_INDEX 0 +#define M_ROUTE_TABLE_INDEX 0xf +#define V_ROUTE_TABLE_INDEX(x) ((x) << S_ROUTE_TABLE_INDEX) +#define G_ROUTE_TABLE_INDEX(x) (((x) >> S_ROUTE_TABLE_INDEX) & M_ROUTE_TABLE_INDEX) + +#define A_RAT_ROUTE_TABLE_DATA 0x588 +#define A_RAT_NO_ROUTE 0x58c + +#define S_CPL_OPCODE 0 +#define M_CPL_OPCODE 0xff +#define V_CPL_OPCODE(x) ((x) << S_CPL_OPCODE) +#define G_CPL_OPCODE(x) (((x) >> S_CPL_OPCODE) & M_CPL_OPCODE) + +#define A_RAT_INTR_ENABLE 0x590 + +#define S_ZEROROUTEERROR 0 +#define V_ZEROROUTEERROR(x) ((x) << S_ZEROROUTEERROR) +#define F_ZEROROUTEERROR V_ZEROROUTEERROR(1U) + +#define S_CSPIFRAMINGERROR 1 +#define V_CSPIFRAMINGERROR(x) ((x) << S_CSPIFRAMINGERROR) +#define F_CSPIFRAMINGERROR V_CSPIFRAMINGERROR(1U) + +#define S_SGEFRAMINGERROR 2 +#define V_SGEFRAMINGERROR(x) ((x) << S_SGEFRAMINGERROR) +#define F_SGEFRAMINGERROR V_SGEFRAMINGERROR(1U) + +#define S_TPFRAMINGERROR 3 +#define V_TPFRAMINGERROR(x) ((x) << S_TPFRAMINGERROR) +#define F_TPFRAMINGERROR V_TPFRAMINGERROR(1U) + +#define A_RAT_INTR_CAUSE 0x594 + +/* CSPI registers */ +#define A_CSPI_RX_AE_WM 0x810 +#define A_CSPI_RX_AF_WM 0x814 +#define A_CSPI_CALENDAR_LEN 0x818 + +#define S_CALENDARLENGTH 0 +#define M_CALENDARLENGTH 0xffff +#define V_CALENDARLENGTH(x) ((x) << S_CALENDARLENGTH) +#define G_CALENDARLENGTH(x) (((x) >> S_CALENDARLENGTH) & M_CALENDARLENGTH) + +#define A_CSPI_FIFO_STATUS_ENABLE 0x820 + +#define S_FIFOSTATUSENABLE 0 +#define V_FIFOSTATUSENABLE(x) ((x) << S_FIFOSTATUSENABLE) +#define F_FIFOSTATUSENABLE V_FIFOSTATUSENABLE(1U) + +#define A_CSPI_MAXBURST1_MAXBURST2 0x828 + +#define S_MAXBURST1 0 +#define M_MAXBURST1 0xffff +#define V_MAXBURST1(x) ((x) << S_MAXBURST1) +#define G_MAXBURST1(x) (((x) >> S_MAXBURST1) & M_MAXBURST1) + +#define S_MAXBURST2 16 +#define M_MAXBURST2 0xffff +#define V_MAXBURST2(x) ((x) << S_MAXBURST2) +#define G_MAXBURST2(x) (((x) >> S_MAXBURST2) & M_MAXBURST2) + +#define A_CSPI_TRAIN 0x82c + +#define S_CSPI_TRAIN_ALPHA 0 +#define M_CSPI_TRAIN_ALPHA 0xffff +#define V_CSPI_TRAIN_ALPHA(x) ((x) << S_CSPI_TRAIN_ALPHA) +#define G_CSPI_TRAIN_ALPHA(x) (((x) >> S_CSPI_TRAIN_ALPHA) & M_CSPI_TRAIN_ALPHA) + +#define S_CSPI_TRAIN_DATA_MAXT 16 +#define M_CSPI_TRAIN_DATA_MAXT 0xffff +#define V_CSPI_TRAIN_DATA_MAXT(x) ((x) << S_CSPI_TRAIN_DATA_MAXT) +#define G_CSPI_TRAIN_DATA_MAXT(x) (((x) >> S_CSPI_TRAIN_DATA_MAXT) & M_CSPI_TRAIN_DATA_MAXT) + +#define A_CSPI_INTR_STATUS 0x848 + +#define S_DIP4ERR 0 +#define V_DIP4ERR(x) ((x) << S_DIP4ERR) +#define F_DIP4ERR V_DIP4ERR(1U) + +#define S_RXDROP 1 +#define V_RXDROP(x) ((x) << S_RXDROP) +#define F_RXDROP V_RXDROP(1U) + +#define S_TXDROP 2 +#define V_TXDROP(x) ((x) << S_TXDROP) +#define F_TXDROP V_TXDROP(1U) + +#define S_RXOVERFLOW 3 +#define V_RXOVERFLOW(x) ((x) << S_RXOVERFLOW) +#define F_RXOVERFLOW V_RXOVERFLOW(1U) + +#define S_RAMPARITYERR 4 +#define V_RAMPARITYERR(x) ((x) << S_RAMPARITYERR) +#define F_RAMPARITYERR V_RAMPARITYERR(1U) + +#define A_CSPI_INTR_ENABLE 0x84c + +/* ESPI registers */ +#define A_ESPI_SCH_TOKEN0 0x880 + +#define S_SCHTOKEN0 0 +#define M_SCHTOKEN0 0xffff +#define V_SCHTOKEN0(x) ((x) << S_SCHTOKEN0) +#define G_SCHTOKEN0(x) (((x) >> S_SCHTOKEN0) & M_SCHTOKEN0) + +#define A_ESPI_SCH_TOKEN1 0x884 + +#define S_SCHTOKEN1 0 +#define M_SCHTOKEN1 0xffff +#define V_SCHTOKEN1(x) ((x) << S_SCHTOKEN1) +#define G_SCHTOKEN1(x) (((x) >> S_SCHTOKEN1) & M_SCHTOKEN1) + +#define A_ESPI_SCH_TOKEN2 0x888 + +#define S_SCHTOKEN2 0 +#define M_SCHTOKEN2 0xffff +#define V_SCHTOKEN2(x) ((x) << S_SCHTOKEN2) +#define G_SCHTOKEN2(x) (((x) >> S_SCHTOKEN2) & M_SCHTOKEN2) + +#define A_ESPI_SCH_TOKEN3 0x88c + +#define S_SCHTOKEN3 0 +#define M_SCHTOKEN3 0xffff +#define V_SCHTOKEN3(x) ((x) << S_SCHTOKEN3) +#define G_SCHTOKEN3(x) (((x) >> S_SCHTOKEN3) & M_SCHTOKEN3) + +#define A_ESPI_RX_FIFO_ALMOST_EMPTY_WATERMARK 0x890 + +#define S_ALMOSTEMPTY 0 +#define M_ALMOSTEMPTY 0xffff +#define V_ALMOSTEMPTY(x) ((x) << S_ALMOSTEMPTY) +#define G_ALMOSTEMPTY(x) (((x) >> S_ALMOSTEMPTY) & M_ALMOSTEMPTY) + +#define A_ESPI_RX_FIFO_ALMOST_FULL_WATERMARK 0x894 + +#define S_ALMOSTFULL 0 +#define M_ALMOSTFULL 0xffff +#define V_ALMOSTFULL(x) ((x) << S_ALMOSTFULL) +#define G_ALMOSTFULL(x) (((x) >> S_ALMOSTFULL) & M_ALMOSTFULL) + +#define A_ESPI_CALENDAR_LENGTH 0x898 +#define A_PORT_CONFIG 0x89c + +#define S_RX_NPORTS 0 +#define M_RX_NPORTS 0xff +#define V_RX_NPORTS(x) ((x) << S_RX_NPORTS) +#define G_RX_NPORTS(x) (((x) >> S_RX_NPORTS) & M_RX_NPORTS) + +#define S_TX_NPORTS 8 +#define M_TX_NPORTS 0xff +#define V_TX_NPORTS(x) ((x) << S_TX_NPORTS) +#define G_TX_NPORTS(x) (((x) >> S_TX_NPORTS) & M_TX_NPORTS) + +#define A_ESPI_FIFO_STATUS_ENABLE 0x8a0 + +#define S_RXSTATUSENABLE 0 +#define V_RXSTATUSENABLE(x) ((x) << S_RXSTATUSENABLE) +#define F_RXSTATUSENABLE V_RXSTATUSENABLE(1U) + +#define S_TXDROPENABLE 1 +#define V_TXDROPENABLE(x) ((x) << S_TXDROPENABLE) +#define F_TXDROPENABLE V_TXDROPENABLE(1U) + +#define S_RXENDIANMODE 2 +#define V_RXENDIANMODE(x) ((x) << S_RXENDIANMODE) +#define F_RXENDIANMODE V_RXENDIANMODE(1U) + +#define S_TXENDIANMODE 3 +#define V_TXENDIANMODE(x) ((x) << S_TXENDIANMODE) +#define F_TXENDIANMODE V_TXENDIANMODE(1U) + +#define S_INTEL1010MODE 4 +#define V_INTEL1010MODE(x) ((x) << S_INTEL1010MODE) +#define F_INTEL1010MODE V_INTEL1010MODE(1U) + +#define A_ESPI_MAXBURST1_MAXBURST2 0x8a8 +#define A_ESPI_TRAIN 0x8ac + +#define S_MAXTRAINALPHA 0 +#define M_MAXTRAINALPHA 0xffff +#define V_MAXTRAINALPHA(x) ((x) << S_MAXTRAINALPHA) +#define G_MAXTRAINALPHA(x) (((x) >> S_MAXTRAINALPHA) & M_MAXTRAINALPHA) + +#define S_MAXTRAINDATA 16 +#define M_MAXTRAINDATA 0xffff +#define V_MAXTRAINDATA(x) ((x) << S_MAXTRAINDATA) +#define G_MAXTRAINDATA(x) (((x) >> S_MAXTRAINDATA) & M_MAXTRAINDATA) + +#define A_RAM_STATUS 0x8b0 + +#define S_RXFIFOPARITYERROR 0 +#define M_RXFIFOPARITYERROR 0x3ff +#define V_RXFIFOPARITYERROR(x) ((x) << S_RXFIFOPARITYERROR) +#define G_RXFIFOPARITYERROR(x) (((x) >> S_RXFIFOPARITYERROR) & M_RXFIFOPARITYERROR) + +#define S_TXFIFOPARITYERROR 10 +#define M_TXFIFOPARITYERROR 0x3ff +#define V_TXFIFOPARITYERROR(x) ((x) << S_TXFIFOPARITYERROR) +#define G_TXFIFOPARITYERROR(x) (((x) >> S_TXFIFOPARITYERROR) & M_TXFIFOPARITYERROR) + +#define S_RXFIFOOVERFLOW 20 +#define M_RXFIFOOVERFLOW 0x3ff +#define V_RXFIFOOVERFLOW(x) ((x) << S_RXFIFOOVERFLOW) +#define G_RXFIFOOVERFLOW(x) (((x) >> S_RXFIFOOVERFLOW) & M_RXFIFOOVERFLOW) + +#define A_TX_DROP_COUNT0 0x8b4 + +#define S_TXPORT0DROPCNT 0 +#define M_TXPORT0DROPCNT 0xffff +#define V_TXPORT0DROPCNT(x) ((x) << S_TXPORT0DROPCNT) +#define G_TXPORT0DROPCNT(x) (((x) >> S_TXPORT0DROPCNT) & M_TXPORT0DROPCNT) + +#define S_TXPORT1DROPCNT 16 +#define M_TXPORT1DROPCNT 0xffff +#define V_TXPORT1DROPCNT(x) ((x) << S_TXPORT1DROPCNT) +#define G_TXPORT1DROPCNT(x) (((x) >> S_TXPORT1DROPCNT) & M_TXPORT1DROPCNT) + +#define A_TX_DROP_COUNT1 0x8b8 + +#define S_TXPORT2DROPCNT 0 +#define M_TXPORT2DROPCNT 0xffff +#define V_TXPORT2DROPCNT(x) ((x) << S_TXPORT2DROPCNT) +#define G_TXPORT2DROPCNT(x) (((x) >> S_TXPORT2DROPCNT) & M_TXPORT2DROPCNT) + +#define S_TXPORT3DROPCNT 16 +#define M_TXPORT3DROPCNT 0xffff +#define V_TXPORT3DROPCNT(x) ((x) << S_TXPORT3DROPCNT) +#define G_TXPORT3DROPCNT(x) (((x) >> S_TXPORT3DROPCNT) & M_TXPORT3DROPCNT) + +#define A_RX_DROP_COUNT0 0x8bc + +#define S_RXPORT0DROPCNT 0 +#define M_RXPORT0DROPCNT 0xffff +#define V_RXPORT0DROPCNT(x) ((x) << S_RXPORT0DROPCNT) +#define G_RXPORT0DROPCNT(x) (((x) >> S_RXPORT0DROPCNT) & M_RXPORT0DROPCNT) + +#define S_RXPORT1DROPCNT 16 +#define M_RXPORT1DROPCNT 0xffff +#define V_RXPORT1DROPCNT(x) ((x) << S_RXPORT1DROPCNT) +#define G_RXPORT1DROPCNT(x) (((x) >> S_RXPORT1DROPCNT) & M_RXPORT1DROPCNT) + +#define A_RX_DROP_COUNT1 0x8c0 + +#define S_RXPORT2DROPCNT 0 +#define M_RXPORT2DROPCNT 0xffff +#define V_RXPORT2DROPCNT(x) ((x) << S_RXPORT2DROPCNT) +#define G_RXPORT2DROPCNT(x) (((x) >> S_RXPORT2DROPCNT) & M_RXPORT2DROPCNT) + +#define S_RXPORT3DROPCNT 16 +#define M_RXPORT3DROPCNT 0xffff +#define V_RXPORT3DROPCNT(x) ((x) << S_RXPORT3DROPCNT) +#define G_RXPORT3DROPCNT(x) (((x) >> S_RXPORT3DROPCNT) & M_RXPORT3DROPCNT) + +#define A_DIP4_ERROR_COUNT 0x8c4 + +#define S_DIP4ERRORCNT 0 +#define M_DIP4ERRORCNT 0xfff +#define V_DIP4ERRORCNT(x) ((x) << S_DIP4ERRORCNT) +#define G_DIP4ERRORCNT(x) (((x) >> S_DIP4ERRORCNT) & M_DIP4ERRORCNT) + +#define S_DIP4ERRORCNTSHADOW 12 +#define M_DIP4ERRORCNTSHADOW 0xfff +#define V_DIP4ERRORCNTSHADOW(x) ((x) << S_DIP4ERRORCNTSHADOW) +#define G_DIP4ERRORCNTSHADOW(x) (((x) >> S_DIP4ERRORCNTSHADOW) & M_DIP4ERRORCNTSHADOW) + +#define S_TRICN_RX_TRAIN_ERR 24 +#define V_TRICN_RX_TRAIN_ERR(x) ((x) << S_TRICN_RX_TRAIN_ERR) +#define F_TRICN_RX_TRAIN_ERR V_TRICN_RX_TRAIN_ERR(1U) + +#define S_TRICN_RX_TRAINING 25 +#define V_TRICN_RX_TRAINING(x) ((x) << S_TRICN_RX_TRAINING) +#define F_TRICN_RX_TRAINING V_TRICN_RX_TRAINING(1U) + +#define S_TRICN_RX_TRAIN_OK 26 +#define V_TRICN_RX_TRAIN_OK(x) ((x) << S_TRICN_RX_TRAIN_OK) +#define F_TRICN_RX_TRAIN_OK V_TRICN_RX_TRAIN_OK(1U) + +#define A_ESPI_INTR_STATUS 0x8c8 + +#define S_DIP2PARITYERR 5 +#define V_DIP2PARITYERR(x) ((x) << S_DIP2PARITYERR) +#define F_DIP2PARITYERR V_DIP2PARITYERR(1U) + +#define A_ESPI_INTR_ENABLE 0x8cc +#define A_RX_DROP_THRESHOLD 0x8d0 +#define A_ESPI_RX_RESET 0x8ec + +#define S_ESPI_RX_LNK_RST 0 +#define V_ESPI_RX_LNK_RST(x) ((x) << S_ESPI_RX_LNK_RST) +#define F_ESPI_RX_LNK_RST V_ESPI_RX_LNK_RST(1U) + +#define S_ESPI_RX_CORE_RST 1 +#define V_ESPI_RX_CORE_RST(x) ((x) << S_ESPI_RX_CORE_RST) +#define F_ESPI_RX_CORE_RST V_ESPI_RX_CORE_RST(1U) + +#define S_RX_CLK_STATUS 2 +#define V_RX_CLK_STATUS(x) ((x) << S_RX_CLK_STATUS) +#define F_RX_CLK_STATUS V_RX_CLK_STATUS(1U) + +#define A_ESPI_MISC_CONTROL 0x8f0 + +#define S_OUT_OF_SYNC_COUNT 0 +#define M_OUT_OF_SYNC_COUNT 0xf +#define V_OUT_OF_SYNC_COUNT(x) ((x) << S_OUT_OF_SYNC_COUNT) +#define G_OUT_OF_SYNC_COUNT(x) (((x) >> S_OUT_OF_SYNC_COUNT) & M_OUT_OF_SYNC_COUNT) + +#define S_DIP2_COUNT_MODE_ENABLE 4 +#define V_DIP2_COUNT_MODE_ENABLE(x) ((x) << S_DIP2_COUNT_MODE_ENABLE) +#define F_DIP2_COUNT_MODE_ENABLE V_DIP2_COUNT_MODE_ENABLE(1U) + +#define S_DIP2_PARITY_ERR_THRES 5 +#define M_DIP2_PARITY_ERR_THRES 0xf +#define V_DIP2_PARITY_ERR_THRES(x) ((x) << S_DIP2_PARITY_ERR_THRES) +#define G_DIP2_PARITY_ERR_THRES(x) (((x) >> S_DIP2_PARITY_ERR_THRES) & M_DIP2_PARITY_ERR_THRES) + +#define S_DIP4_THRES 9 +#define M_DIP4_THRES 0xfff +#define V_DIP4_THRES(x) ((x) << S_DIP4_THRES) +#define G_DIP4_THRES(x) (((x) >> S_DIP4_THRES) & M_DIP4_THRES) + +#define S_DIP4_THRES_ENABLE 21 +#define V_DIP4_THRES_ENABLE(x) ((x) << S_DIP4_THRES_ENABLE) +#define F_DIP4_THRES_ENABLE V_DIP4_THRES_ENABLE(1U) + +#define S_FORCE_DISABLE_STATUS 22 +#define V_FORCE_DISABLE_STATUS(x) ((x) << S_FORCE_DISABLE_STATUS) +#define F_FORCE_DISABLE_STATUS V_FORCE_DISABLE_STATUS(1U) + +#define S_DYNAMIC_DESKEW 23 +#define V_DYNAMIC_DESKEW(x) ((x) << S_DYNAMIC_DESKEW) +#define F_DYNAMIC_DESKEW V_DYNAMIC_DESKEW(1U) + +#define S_MONITORED_PORT_NUM 25 +#define M_MONITORED_PORT_NUM 0x3 +#define V_MONITORED_PORT_NUM(x) ((x) << S_MONITORED_PORT_NUM) +#define G_MONITORED_PORT_NUM(x) (((x) >> S_MONITORED_PORT_NUM) & M_MONITORED_PORT_NUM) + +#define S_MONITORED_DIRECTION 27 +#define V_MONITORED_DIRECTION(x) ((x) << S_MONITORED_DIRECTION) +#define F_MONITORED_DIRECTION V_MONITORED_DIRECTION(1U) + +#define S_MONITORED_INTERFACE 28 +#define V_MONITORED_INTERFACE(x) ((x) << S_MONITORED_INTERFACE) +#define F_MONITORED_INTERFACE V_MONITORED_INTERFACE(1U) + +#define A_ESPI_DIP2_ERR_COUNT 0x8f4 + +#define S_DIP2_ERR_CNT 0 +#define M_DIP2_ERR_CNT 0xf +#define V_DIP2_ERR_CNT(x) ((x) << S_DIP2_ERR_CNT) +#define G_DIP2_ERR_CNT(x) (((x) >> S_DIP2_ERR_CNT) & M_DIP2_ERR_CNT) + +#define A_ESPI_CMD_ADDR 0x8f8 + +#define S_WRITE_DATA 0 +#define M_WRITE_DATA 0xff +#define V_WRITE_DATA(x) ((x) << S_WRITE_DATA) +#define G_WRITE_DATA(x) (((x) >> S_WRITE_DATA) & M_WRITE_DATA) + +#define S_REGISTER_OFFSET 8 +#define M_REGISTER_OFFSET 0xf +#define V_REGISTER_OFFSET(x) ((x) << S_REGISTER_OFFSET) +#define G_REGISTER_OFFSET(x) (((x) >> S_REGISTER_OFFSET) & M_REGISTER_OFFSET) + +#define S_CHANNEL_ADDR 12 +#define M_CHANNEL_ADDR 0xf +#define V_CHANNEL_ADDR(x) ((x) << S_CHANNEL_ADDR) +#define G_CHANNEL_ADDR(x) (((x) >> S_CHANNEL_ADDR) & M_CHANNEL_ADDR) + +#define S_MODULE_ADDR 16 +#define M_MODULE_ADDR 0x3 +#define V_MODULE_ADDR(x) ((x) << S_MODULE_ADDR) +#define G_MODULE_ADDR(x) (((x) >> S_MODULE_ADDR) & M_MODULE_ADDR) + +#define S_BUNDLE_ADDR 20 +#define M_BUNDLE_ADDR 0x3 +#define V_BUNDLE_ADDR(x) ((x) << S_BUNDLE_ADDR) +#define G_BUNDLE_ADDR(x) (((x) >> S_BUNDLE_ADDR) & M_BUNDLE_ADDR) + +#define S_SPI4_COMMAND 24 +#define M_SPI4_COMMAND 0xff +#define V_SPI4_COMMAND(x) ((x) << S_SPI4_COMMAND) +#define G_SPI4_COMMAND(x) (((x) >> S_SPI4_COMMAND) & M_SPI4_COMMAND) + +#define A_ESPI_GOSTAT 0x8fc + +#define S_READ_DATA 0 +#define M_READ_DATA 0xff +#define V_READ_DATA(x) ((x) << S_READ_DATA) +#define G_READ_DATA(x) (((x) >> S_READ_DATA) & M_READ_DATA) + +#define S_ESPI_CMD_BUSY 8 +#define V_ESPI_CMD_BUSY(x) ((x) << S_ESPI_CMD_BUSY) +#define F_ESPI_CMD_BUSY V_ESPI_CMD_BUSY(1U) + +#define S_ERROR_ACK 9 +#define V_ERROR_ACK(x) ((x) << S_ERROR_ACK) +#define F_ERROR_ACK V_ERROR_ACK(1U) + +#define S_UNMAPPED_ERR 10 +#define V_UNMAPPED_ERR(x) ((x) << S_UNMAPPED_ERR) +#define F_UNMAPPED_ERR V_UNMAPPED_ERR(1U) + +#define S_TRANSACTION_TIMER 16 +#define M_TRANSACTION_TIMER 0xff +#define V_TRANSACTION_TIMER(x) ((x) << S_TRANSACTION_TIMER) +#define G_TRANSACTION_TIMER(x) (((x) >> S_TRANSACTION_TIMER) & M_TRANSACTION_TIMER) + + +/* ULP registers */ +#define A_ULP_ULIMIT 0x980 +#define A_ULP_TAGMASK 0x984 +#define A_ULP_HREG_INDEX 0x988 +#define A_ULP_HREG_DATA 0x98c +#define A_ULP_INT_ENABLE 0x990 +#define A_ULP_INT_CAUSE 0x994 + +#define S_HREG_PAR_ERR 0 +#define V_HREG_PAR_ERR(x) ((x) << S_HREG_PAR_ERR) +#define F_HREG_PAR_ERR V_HREG_PAR_ERR(1U) + +#define S_EGRS_DATA_PAR_ERR 1 +#define V_EGRS_DATA_PAR_ERR(x) ((x) << S_EGRS_DATA_PAR_ERR) +#define F_EGRS_DATA_PAR_ERR V_EGRS_DATA_PAR_ERR(1U) + +#define S_INGRS_DATA_PAR_ERR 2 +#define V_INGRS_DATA_PAR_ERR(x) ((x) << S_INGRS_DATA_PAR_ERR) +#define F_INGRS_DATA_PAR_ERR V_INGRS_DATA_PAR_ERR(1U) + +#define S_PM_INTR 3 +#define V_PM_INTR(x) ((x) << S_PM_INTR) +#define F_PM_INTR V_PM_INTR(1U) + +#define S_PM_E2C_SYNC_ERR 4 +#define V_PM_E2C_SYNC_ERR(x) ((x) << S_PM_E2C_SYNC_ERR) +#define F_PM_E2C_SYNC_ERR V_PM_E2C_SYNC_ERR(1U) + +#define S_PM_C2E_SYNC_ERR 5 +#define V_PM_C2E_SYNC_ERR(x) ((x) << S_PM_C2E_SYNC_ERR) +#define F_PM_C2E_SYNC_ERR V_PM_C2E_SYNC_ERR(1U) + +#define S_PM_E2C_EMPTY_ERR 6 +#define V_PM_E2C_EMPTY_ERR(x) ((x) << S_PM_E2C_EMPTY_ERR) +#define F_PM_E2C_EMPTY_ERR V_PM_E2C_EMPTY_ERR(1U) + +#define S_PM_C2E_EMPTY_ERR 7 +#define V_PM_C2E_EMPTY_ERR(x) ((x) << S_PM_C2E_EMPTY_ERR) +#define F_PM_C2E_EMPTY_ERR V_PM_C2E_EMPTY_ERR(1U) + +#define S_PM_PAR_ERR 8 +#define M_PM_PAR_ERR 0xffff +#define V_PM_PAR_ERR(x) ((x) << S_PM_PAR_ERR) +#define G_PM_PAR_ERR(x) (((x) >> S_PM_PAR_ERR) & M_PM_PAR_ERR) + +#define S_PM_E2C_WRT_FULL 24 +#define V_PM_E2C_WRT_FULL(x) ((x) << S_PM_E2C_WRT_FULL) +#define F_PM_E2C_WRT_FULL V_PM_E2C_WRT_FULL(1U) + +#define S_PM_C2E_WRT_FULL 25 +#define V_PM_C2E_WRT_FULL(x) ((x) << S_PM_C2E_WRT_FULL) +#define F_PM_C2E_WRT_FULL V_PM_C2E_WRT_FULL(1U) + +#define A_ULP_PIO_CTRL 0x998 + +/* PL registers */ +#define A_PL_ENABLE 0xa00 + +#define S_PL_INTR_SGE_ERR 0 +#define V_PL_INTR_SGE_ERR(x) ((x) << S_PL_INTR_SGE_ERR) +#define F_PL_INTR_SGE_ERR V_PL_INTR_SGE_ERR(1U) + +#define S_PL_INTR_SGE_DATA 1 +#define V_PL_INTR_SGE_DATA(x) ((x) << S_PL_INTR_SGE_DATA) +#define F_PL_INTR_SGE_DATA V_PL_INTR_SGE_DATA(1U) + +#define S_PL_INTR_MC3 2 +#define V_PL_INTR_MC3(x) ((x) << S_PL_INTR_MC3) +#define F_PL_INTR_MC3 V_PL_INTR_MC3(1U) + +#define S_PL_INTR_MC4 3 +#define V_PL_INTR_MC4(x) ((x) << S_PL_INTR_MC4) +#define F_PL_INTR_MC4 V_PL_INTR_MC4(1U) + +#define S_PL_INTR_MC5 4 +#define V_PL_INTR_MC5(x) ((x) << S_PL_INTR_MC5) +#define F_PL_INTR_MC5 V_PL_INTR_MC5(1U) + +#define S_PL_INTR_RAT 5 +#define V_PL_INTR_RAT(x) ((x) << S_PL_INTR_RAT) +#define F_PL_INTR_RAT V_PL_INTR_RAT(1U) + +#define S_PL_INTR_TP 6 +#define V_PL_INTR_TP(x) ((x) << S_PL_INTR_TP) +#define F_PL_INTR_TP V_PL_INTR_TP(1U) + +#define S_PL_INTR_ULP 7 +#define V_PL_INTR_ULP(x) ((x) << S_PL_INTR_ULP) +#define F_PL_INTR_ULP V_PL_INTR_ULP(1U) + +#define S_PL_INTR_ESPI 8 +#define V_PL_INTR_ESPI(x) ((x) << S_PL_INTR_ESPI) +#define F_PL_INTR_ESPI V_PL_INTR_ESPI(1U) + +#define S_PL_INTR_CSPI 9 +#define V_PL_INTR_CSPI(x) ((x) << S_PL_INTR_CSPI) +#define F_PL_INTR_CSPI V_PL_INTR_CSPI(1U) + +#define S_PL_INTR_PCIX 10 +#define V_PL_INTR_PCIX(x) ((x) << S_PL_INTR_PCIX) +#define F_PL_INTR_PCIX V_PL_INTR_PCIX(1U) + +#define S_PL_INTR_EXT 11 +#define V_PL_INTR_EXT(x) ((x) << S_PL_INTR_EXT) +#define F_PL_INTR_EXT V_PL_INTR_EXT(1U) + +#define A_PL_CAUSE 0xa04 + +/* MC5 registers */ +#define A_MC5_CONFIG 0xc04 + +#define S_MODE 0 +#define V_MODE(x) ((x) << S_MODE) +#define F_MODE V_MODE(1U) + +#define S_TCAM_RESET 1 +#define V_TCAM_RESET(x) ((x) << S_TCAM_RESET) +#define F_TCAM_RESET V_TCAM_RESET(1U) + +#define S_TCAM_READY 2 +#define V_TCAM_READY(x) ((x) << S_TCAM_READY) +#define F_TCAM_READY V_TCAM_READY(1U) + +#define S_DBGI_ENABLE 4 +#define V_DBGI_ENABLE(x) ((x) << S_DBGI_ENABLE) +#define F_DBGI_ENABLE V_DBGI_ENABLE(1U) + +#define S_M_BUS_ENABLE 5 +#define V_M_BUS_ENABLE(x) ((x) << S_M_BUS_ENABLE) +#define F_M_BUS_ENABLE V_M_BUS_ENABLE(1U) + +#define S_PARITY_ENABLE 6 +#define V_PARITY_ENABLE(x) ((x) << S_PARITY_ENABLE) +#define F_PARITY_ENABLE V_PARITY_ENABLE(1U) + +#define S_SYN_ISSUE_MODE 7 +#define M_SYN_ISSUE_MODE 0x3 +#define V_SYN_ISSUE_MODE(x) ((x) << S_SYN_ISSUE_MODE) +#define G_SYN_ISSUE_MODE(x) (((x) >> S_SYN_ISSUE_MODE) & M_SYN_ISSUE_MODE) + +#define S_BUILD 16 +#define V_BUILD(x) ((x) << S_BUILD) +#define F_BUILD V_BUILD(1U) + +#define S_COMPRESSION_ENABLE 17 +#define V_COMPRESSION_ENABLE(x) ((x) << S_COMPRESSION_ENABLE) +#define F_COMPRESSION_ENABLE V_COMPRESSION_ENABLE(1U) + +#define S_NUM_LIP 18 +#define M_NUM_LIP 0x3f +#define V_NUM_LIP(x) ((x) << S_NUM_LIP) +#define G_NUM_LIP(x) (((x) >> S_NUM_LIP) & M_NUM_LIP) + +#define S_TCAM_PART_CNT 24 +#define M_TCAM_PART_CNT 0x3 +#define V_TCAM_PART_CNT(x) ((x) << S_TCAM_PART_CNT) +#define G_TCAM_PART_CNT(x) (((x) >> S_TCAM_PART_CNT) & M_TCAM_PART_CNT) + +#define S_TCAM_PART_TYPE 26 +#define M_TCAM_PART_TYPE 0x3 +#define V_TCAM_PART_TYPE(x) ((x) << S_TCAM_PART_TYPE) +#define G_TCAM_PART_TYPE(x) (((x) >> S_TCAM_PART_TYPE) & M_TCAM_PART_TYPE) + +#define S_TCAM_PART_SIZE 28 +#define M_TCAM_PART_SIZE 0x3 +#define V_TCAM_PART_SIZE(x) ((x) << S_TCAM_PART_SIZE) +#define G_TCAM_PART_SIZE(x) (((x) >> S_TCAM_PART_SIZE) & M_TCAM_PART_SIZE) + +#define S_TCAM_PART_TYPE_HI 30 +#define V_TCAM_PART_TYPE_HI(x) ((x) << S_TCAM_PART_TYPE_HI) +#define F_TCAM_PART_TYPE_HI V_TCAM_PART_TYPE_HI(1U) + +#define A_MC5_SIZE 0xc08 + +#define S_SIZE 0 +#define M_SIZE 0x3fffff +#define V_SIZE(x) ((x) << S_SIZE) +#define G_SIZE(x) (((x) >> S_SIZE) & M_SIZE) + +#define A_MC5_ROUTING_TABLE_INDEX 0xc0c + +#define S_START_OF_ROUTING_TABLE 0 +#define M_START_OF_ROUTING_TABLE 0x3fffff +#define V_START_OF_ROUTING_TABLE(x) ((x) << S_START_OF_ROUTING_TABLE) +#define G_START_OF_ROUTING_TABLE(x) (((x) >> S_START_OF_ROUTING_TABLE) & M_START_OF_ROUTING_TABLE) + +#define A_MC5_SERVER_INDEX 0xc14 + +#define S_START_OF_SERVER_INDEX 0 +#define M_START_OF_SERVER_INDEX 0x3fffff +#define V_START_OF_SERVER_INDEX(x) ((x) << S_START_OF_SERVER_INDEX) +#define G_START_OF_SERVER_INDEX(x) (((x) >> S_START_OF_SERVER_INDEX) & M_START_OF_SERVER_INDEX) + +#define A_MC5_LIP_RAM_ADDR 0xc18 + +#define S_LOCAL_IP_RAM_ADDR 0 +#define M_LOCAL_IP_RAM_ADDR 0x3f +#define V_LOCAL_IP_RAM_ADDR(x) ((x) << S_LOCAL_IP_RAM_ADDR) +#define G_LOCAL_IP_RAM_ADDR(x) (((x) >> S_LOCAL_IP_RAM_ADDR) & M_LOCAL_IP_RAM_ADDR) + +#define S_RAM_WRITE_ENABLE 8 +#define V_RAM_WRITE_ENABLE(x) ((x) << S_RAM_WRITE_ENABLE) +#define F_RAM_WRITE_ENABLE V_RAM_WRITE_ENABLE(1U) + +#define A_MC5_LIP_RAM_DATA 0xc1c +#define A_MC5_RSP_LATENCY 0xc20 + +#define S_SEARCH_RESPONSE_LATENCY 0 +#define M_SEARCH_RESPONSE_LATENCY 0x1f +#define V_SEARCH_RESPONSE_LATENCY(x) ((x) << S_SEARCH_RESPONSE_LATENCY) +#define G_SEARCH_RESPONSE_LATENCY(x) (((x) >> S_SEARCH_RESPONSE_LATENCY) & M_SEARCH_RESPONSE_LATENCY) + +#define S_LEARN_RESPONSE_LATENCY 8 +#define M_LEARN_RESPONSE_LATENCY 0x1f +#define V_LEARN_RESPONSE_LATENCY(x) ((x) << S_LEARN_RESPONSE_LATENCY) +#define G_LEARN_RESPONSE_LATENCY(x) (((x) >> S_LEARN_RESPONSE_LATENCY) & M_LEARN_RESPONSE_LATENCY) + +#define A_MC5_PARITY_LATENCY 0xc24 + +#define S_SRCHLAT 0 +#define M_SRCHLAT 0x1f +#define V_SRCHLAT(x) ((x) << S_SRCHLAT) +#define G_SRCHLAT(x) (((x) >> S_SRCHLAT) & M_SRCHLAT) + +#define S_PARLAT 8 +#define M_PARLAT 0x1f +#define V_PARLAT(x) ((x) << S_PARLAT) +#define G_PARLAT(x) (((x) >> S_PARLAT) & M_PARLAT) + +#define A_MC5_WR_LRN_VERIFY 0xc28 + +#define S_POVEREN 0 +#define V_POVEREN(x) ((x) << S_POVEREN) +#define F_POVEREN V_POVEREN(1U) + +#define S_LRNVEREN 1 +#define V_LRNVEREN(x) ((x) << S_LRNVEREN) +#define F_LRNVEREN V_LRNVEREN(1U) + +#define S_VWVEREN 2 +#define V_VWVEREN(x) ((x) << S_VWVEREN) +#define F_VWVEREN V_VWVEREN(1U) + +#define A_MC5_PART_ID_INDEX 0xc2c + +#define S_IDINDEX 0 +#define M_IDINDEX 0xf +#define V_IDINDEX(x) ((x) << S_IDINDEX) +#define G_IDINDEX(x) (((x) >> S_IDINDEX) & M_IDINDEX) + +#define A_MC5_RESET_MAX 0xc30 + +#define S_RSTMAX 0 +#define M_RSTMAX 0x1ff +#define V_RSTMAX(x) ((x) << S_RSTMAX) +#define G_RSTMAX(x) (((x) >> S_RSTMAX) & M_RSTMAX) + +#define A_MC5_INT_ENABLE 0xc40 + +#define S_MC5_INT_HIT_OUT_ACTIVE_REGION_ERR 0 +#define V_MC5_INT_HIT_OUT_ACTIVE_REGION_ERR(x) ((x) << S_MC5_INT_HIT_OUT_ACTIVE_REGION_ERR) +#define F_MC5_INT_HIT_OUT_ACTIVE_REGION_ERR V_MC5_INT_HIT_OUT_ACTIVE_REGION_ERR(1U) + +#define S_MC5_INT_HIT_IN_ACTIVE_REGION_ERR 1 +#define V_MC5_INT_HIT_IN_ACTIVE_REGION_ERR(x) ((x) << S_MC5_INT_HIT_IN_ACTIVE_REGION_ERR) +#define F_MC5_INT_HIT_IN_ACTIVE_REGION_ERR V_MC5_INT_HIT_IN_ACTIVE_REGION_ERR(1U) + +#define S_MC5_INT_HIT_IN_RT_REGION_ERR 2 +#define V_MC5_INT_HIT_IN_RT_REGION_ERR(x) ((x) << S_MC5_INT_HIT_IN_RT_REGION_ERR) +#define F_MC5_INT_HIT_IN_RT_REGION_ERR V_MC5_INT_HIT_IN_RT_REGION_ERR(1U) + +#define S_MC5_INT_MISS_ERR 3 +#define V_MC5_INT_MISS_ERR(x) ((x) << S_MC5_INT_MISS_ERR) +#define F_MC5_INT_MISS_ERR V_MC5_INT_MISS_ERR(1U) + +#define S_MC5_INT_LIP0_ERR 4 +#define V_MC5_INT_LIP0_ERR(x) ((x) << S_MC5_INT_LIP0_ERR) +#define F_MC5_INT_LIP0_ERR V_MC5_INT_LIP0_ERR(1U) + +#define S_MC5_INT_LIP_MISS_ERR 5 +#define V_MC5_INT_LIP_MISS_ERR(x) ((x) << S_MC5_INT_LIP_MISS_ERR) +#define F_MC5_INT_LIP_MISS_ERR V_MC5_INT_LIP_MISS_ERR(1U) + +#define S_MC5_INT_PARITY_ERR 6 +#define V_MC5_INT_PARITY_ERR(x) ((x) << S_MC5_INT_PARITY_ERR) +#define F_MC5_INT_PARITY_ERR V_MC5_INT_PARITY_ERR(1U) + +#define S_MC5_INT_ACTIVE_REGION_FULL 7 +#define V_MC5_INT_ACTIVE_REGION_FULL(x) ((x) << S_MC5_INT_ACTIVE_REGION_FULL) +#define F_MC5_INT_ACTIVE_REGION_FULL V_MC5_INT_ACTIVE_REGION_FULL(1U) + +#define S_MC5_INT_NFA_SRCH_ERR 8 +#define V_MC5_INT_NFA_SRCH_ERR(x) ((x) << S_MC5_INT_NFA_SRCH_ERR) +#define F_MC5_INT_NFA_SRCH_ERR V_MC5_INT_NFA_SRCH_ERR(1U) + +#define S_MC5_INT_SYN_COOKIE 9 +#define V_MC5_INT_SYN_COOKIE(x) ((x) << S_MC5_INT_SYN_COOKIE) +#define F_MC5_INT_SYN_COOKIE V_MC5_INT_SYN_COOKIE(1U) + +#define S_MC5_INT_SYN_COOKIE_BAD 10 +#define V_MC5_INT_SYN_COOKIE_BAD(x) ((x) << S_MC5_INT_SYN_COOKIE_BAD) +#define F_MC5_INT_SYN_COOKIE_BAD V_MC5_INT_SYN_COOKIE_BAD(1U) + +#define S_MC5_INT_SYN_COOKIE_OFF 11 +#define V_MC5_INT_SYN_COOKIE_OFF(x) ((x) << S_MC5_INT_SYN_COOKIE_OFF) +#define F_MC5_INT_SYN_COOKIE_OFF V_MC5_INT_SYN_COOKIE_OFF(1U) + +#define S_MC5_INT_UNKNOWN_CMD 15 +#define V_MC5_INT_UNKNOWN_CMD(x) ((x) << S_MC5_INT_UNKNOWN_CMD) +#define F_MC5_INT_UNKNOWN_CMD V_MC5_INT_UNKNOWN_CMD(1U) + +#define S_MC5_INT_REQUESTQ_PARITY_ERR 16 +#define V_MC5_INT_REQUESTQ_PARITY_ERR(x) ((x) << S_MC5_INT_REQUESTQ_PARITY_ERR) +#define F_MC5_INT_REQUESTQ_PARITY_ERR V_MC5_INT_REQUESTQ_PARITY_ERR(1U) + +#define S_MC5_INT_DISPATCHQ_PARITY_ERR 17 +#define V_MC5_INT_DISPATCHQ_PARITY_ERR(x) ((x) << S_MC5_INT_DISPATCHQ_PARITY_ERR) +#define F_MC5_INT_DISPATCHQ_PARITY_ERR V_MC5_INT_DISPATCHQ_PARITY_ERR(1U) + +#define S_MC5_INT_DEL_ACT_EMPTY 18 +#define V_MC5_INT_DEL_ACT_EMPTY(x) ((x) << S_MC5_INT_DEL_ACT_EMPTY) +#define F_MC5_INT_DEL_ACT_EMPTY V_MC5_INT_DEL_ACT_EMPTY(1U) + +#define A_MC5_INT_CAUSE 0xc44 +#define A_MC5_INT_TID 0xc48 +#define A_MC5_INT_PTID 0xc4c +#define A_MC5_DBGI_CONFIG 0xc74 +#define A_MC5_DBGI_REQ_CMD 0xc78 + +#define S_CMDMODE 0 +#define M_CMDMODE 0x7 +#define V_CMDMODE(x) ((x) << S_CMDMODE) +#define G_CMDMODE(x) (((x) >> S_CMDMODE) & M_CMDMODE) + +#define S_SADRSEL 4 +#define V_SADRSEL(x) ((x) << S_SADRSEL) +#define F_SADRSEL V_SADRSEL(1U) + +#define S_WRITE_BURST_SIZE 22 +#define M_WRITE_BURST_SIZE 0x3ff +#define V_WRITE_BURST_SIZE(x) ((x) << S_WRITE_BURST_SIZE) +#define G_WRITE_BURST_SIZE(x) (((x) >> S_WRITE_BURST_SIZE) & M_WRITE_BURST_SIZE) + +#define A_MC5_DBGI_REQ_ADDR0 0xc7c +#define A_MC5_DBGI_REQ_ADDR1 0xc80 +#define A_MC5_DBGI_REQ_ADDR2 0xc84 +#define A_MC5_DBGI_REQ_DATA0 0xc88 +#define A_MC5_DBGI_REQ_DATA1 0xc8c +#define A_MC5_DBGI_REQ_DATA2 0xc90 +#define A_MC5_DBGI_REQ_DATA3 0xc94 +#define A_MC5_DBGI_REQ_DATA4 0xc98 +#define A_MC5_DBGI_REQ_MASK0 0xc9c +#define A_MC5_DBGI_REQ_MASK1 0xca0 +#define A_MC5_DBGI_REQ_MASK2 0xca4 +#define A_MC5_DBGI_REQ_MASK3 0xca8 +#define A_MC5_DBGI_REQ_MASK4 0xcac +#define A_MC5_DBGI_RSP_STATUS 0xcb0 + +#define S_DBGI_RSP_VALID 0 +#define V_DBGI_RSP_VALID(x) ((x) << S_DBGI_RSP_VALID) +#define F_DBGI_RSP_VALID V_DBGI_RSP_VALID(1U) + +#define S_DBGI_RSP_HIT 1 +#define V_DBGI_RSP_HIT(x) ((x) << S_DBGI_RSP_HIT) +#define F_DBGI_RSP_HIT V_DBGI_RSP_HIT(1U) + +#define S_DBGI_RSP_ERR 2 +#define V_DBGI_RSP_ERR(x) ((x) << S_DBGI_RSP_ERR) +#define F_DBGI_RSP_ERR V_DBGI_RSP_ERR(1U) + +#define S_DBGI_RSP_ERR_REASON 8 +#define M_DBGI_RSP_ERR_REASON 0x7 +#define V_DBGI_RSP_ERR_REASON(x) ((x) << S_DBGI_RSP_ERR_REASON) +#define G_DBGI_RSP_ERR_REASON(x) (((x) >> S_DBGI_RSP_ERR_REASON) & M_DBGI_RSP_ERR_REASON) + +#define A_MC5_DBGI_RSP_DATA0 0xcb4 +#define A_MC5_DBGI_RSP_DATA1 0xcb8 +#define A_MC5_DBGI_RSP_DATA2 0xcbc +#define A_MC5_DBGI_RSP_DATA3 0xcc0 +#define A_MC5_DBGI_RSP_DATA4 0xcc4 +#define A_MC5_DBGI_RSP_LAST_CMD 0xcc8 +#define A_MC5_POPEN_DATA_WR_CMD 0xccc +#define A_MC5_POPEN_MASK_WR_CMD 0xcd0 +#define A_MC5_AOPEN_SRCH_CMD 0xcd4 +#define A_MC5_AOPEN_LRN_CMD 0xcd8 +#define A_MC5_SYN_SRCH_CMD 0xcdc +#define A_MC5_SYN_LRN_CMD 0xce0 +#define A_MC5_ACK_SRCH_CMD 0xce4 +#define A_MC5_ACK_LRN_CMD 0xce8 +#define A_MC5_ILOOKUP_CMD 0xcec +#define A_MC5_ELOOKUP_CMD 0xcf0 +#define A_MC5_DATA_WRITE_CMD 0xcf4 +#define A_MC5_DATA_READ_CMD 0xcf8 +#define A_MC5_MASK_WRITE_CMD 0xcfc + +/* PCICFG registers */ +#define A_PCICFG_PM_CSR 0x44 +#define A_PCICFG_VPD_ADDR 0x4a + +#define S_VPD_ADDR 0 +#define M_VPD_ADDR 0x7fff +#define V_VPD_ADDR(x) ((x) << S_VPD_ADDR) +#define G_VPD_ADDR(x) (((x) >> S_VPD_ADDR) & M_VPD_ADDR) + +#define S_VPD_OP_FLAG 15 +#define V_VPD_OP_FLAG(x) ((x) << S_VPD_OP_FLAG) +#define F_VPD_OP_FLAG V_VPD_OP_FLAG(1U) + +#define A_PCICFG_VPD_DATA 0x4c +#define A_PCICFG_PCIX_CMD 0x60 +#define A_PCICFG_INTR_ENABLE 0xf4 + +#define S_MASTER_PARITY_ERR 0 +#define V_MASTER_PARITY_ERR(x) ((x) << S_MASTER_PARITY_ERR) +#define F_MASTER_PARITY_ERR V_MASTER_PARITY_ERR(1U) + +#define S_SIG_TARGET_ABORT 1 +#define V_SIG_TARGET_ABORT(x) ((x) << S_SIG_TARGET_ABORT) +#define F_SIG_TARGET_ABORT V_SIG_TARGET_ABORT(1U) + +#define S_RCV_TARGET_ABORT 2 +#define V_RCV_TARGET_ABORT(x) ((x) << S_RCV_TARGET_ABORT) +#define F_RCV_TARGET_ABORT V_RCV_TARGET_ABORT(1U) + +#define S_RCV_MASTER_ABORT 3 +#define V_RCV_MASTER_ABORT(x) ((x) << S_RCV_MASTER_ABORT) +#define F_RCV_MASTER_ABORT V_RCV_MASTER_ABORT(1U) + +#define S_SIG_SYS_ERR 4 +#define V_SIG_SYS_ERR(x) ((x) << S_SIG_SYS_ERR) +#define F_SIG_SYS_ERR V_SIG_SYS_ERR(1U) + +#define S_DET_PARITY_ERR 5 +#define V_DET_PARITY_ERR(x) ((x) << S_DET_PARITY_ERR) +#define F_DET_PARITY_ERR V_DET_PARITY_ERR(1U) + +#define S_PIO_PARITY_ERR 6 +#define V_PIO_PARITY_ERR(x) ((x) << S_PIO_PARITY_ERR) +#define F_PIO_PARITY_ERR V_PIO_PARITY_ERR(1U) + +#define S_WF_PARITY_ERR 7 +#define V_WF_PARITY_ERR(x) ((x) << S_WF_PARITY_ERR) +#define F_WF_PARITY_ERR V_WF_PARITY_ERR(1U) + +#define S_RF_PARITY_ERR 8 +#define M_RF_PARITY_ERR 0x3 +#define V_RF_PARITY_ERR(x) ((x) << S_RF_PARITY_ERR) +#define G_RF_PARITY_ERR(x) (((x) >> S_RF_PARITY_ERR) & M_RF_PARITY_ERR) + +#define S_CF_PARITY_ERR 10 +#define M_CF_PARITY_ERR 0x3 +#define V_CF_PARITY_ERR(x) ((x) << S_CF_PARITY_ERR) +#define G_CF_PARITY_ERR(x) (((x) >> S_CF_PARITY_ERR) & M_CF_PARITY_ERR) + +#define A_PCICFG_INTR_CAUSE 0xf8 +#define A_PCICFG_MODE 0xfc + +#define S_PCI_MODE_64BIT 0 +#define V_PCI_MODE_64BIT(x) ((x) << S_PCI_MODE_64BIT) +#define F_PCI_MODE_64BIT V_PCI_MODE_64BIT(1U) + +#define S_PCI_MODE_66MHZ 1 +#define V_PCI_MODE_66MHZ(x) ((x) << S_PCI_MODE_66MHZ) +#define F_PCI_MODE_66MHZ V_PCI_MODE_66MHZ(1U) + +#define S_PCI_MODE_PCIX_INITPAT 2 +#define M_PCI_MODE_PCIX_INITPAT 0x7 +#define V_PCI_MODE_PCIX_INITPAT(x) ((x) << S_PCI_MODE_PCIX_INITPAT) +#define G_PCI_MODE_PCIX_INITPAT(x) (((x) >> S_PCI_MODE_PCIX_INITPAT) & M_PCI_MODE_PCIX_INITPAT) + +#define S_PCI_MODE_PCIX 5 +#define V_PCI_MODE_PCIX(x) ((x) << S_PCI_MODE_PCIX) +#define F_PCI_MODE_PCIX V_PCI_MODE_PCIX(1U) + +#define S_PCI_MODE_CLK 6 +#define M_PCI_MODE_CLK 0x3 +#define V_PCI_MODE_CLK(x) ((x) << S_PCI_MODE_CLK) +#define G_PCI_MODE_CLK(x) (((x) >> S_PCI_MODE_CLK) & M_PCI_MODE_CLK) + +#endif /* _CXGB_REGS_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb/sge.c b/drivers/net/ethernet/chelsio/cxgb/sge.c new file mode 100644 index 000000000..861edff5e --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/sge.c @@ -0,0 +1,2156 @@ +// SPDX-License-Identifier: GPL-2.0-only +/***************************************************************************** + * * + * File: sge.c * + * $Revision: 1.26 $ * + * $Date: 2005/06/21 18:29:48 $ * + * Description: * + * DMA engine. * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#include "common.h" + +#include <linux/types.h> +#include <linux/errno.h> +#include <linux/pci.h> +#include <linux/ktime.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> +#include <linux/if_vlan.h> +#include <linux/skbuff.h> +#include <linux/mm.h> +#include <linux/tcp.h> +#include <linux/ip.h> +#include <linux/in.h> +#include <linux/if_arp.h> +#include <linux/slab.h> +#include <linux/prefetch.h> + +#include "cpl5_cmd.h" +#include "sge.h" +#include "regs.h" +#include "espi.h" + +/* This belongs in if_ether.h */ +#define ETH_P_CPL5 0xf + +#define SGE_CMDQ_N 2 +#define SGE_FREELQ_N 2 +#define SGE_CMDQ0_E_N 1024 +#define SGE_CMDQ1_E_N 128 +#define SGE_FREEL_SIZE 4096 +#define SGE_JUMBO_FREEL_SIZE 512 +#define SGE_FREEL_REFILL_THRESH 16 +#define SGE_RESPQ_E_N 1024 +#define SGE_INTRTIMER_NRES 1000 +#define SGE_RX_SM_BUF_SIZE 1536 +#define SGE_TX_DESC_MAX_PLEN 16384 + +#define SGE_RESPQ_REPLENISH_THRES (SGE_RESPQ_E_N / 4) + +/* + * Period of the TX buffer reclaim timer. This timer does not need to run + * frequently as TX buffers are usually reclaimed by new TX packets. + */ +#define TX_RECLAIM_PERIOD (HZ / 4) + +#define M_CMD_LEN 0x7fffffff +#define V_CMD_LEN(v) (v) +#define G_CMD_LEN(v) ((v) & M_CMD_LEN) +#define V_CMD_GEN1(v) ((v) << 31) +#define V_CMD_GEN2(v) (v) +#define F_CMD_DATAVALID (1 << 1) +#define F_CMD_SOP (1 << 2) +#define V_CMD_EOP(v) ((v) << 3) + +/* + * Command queue, receive buffer list, and response queue descriptors. + */ +#if defined(__BIG_ENDIAN_BITFIELD) +struct cmdQ_e { + u32 addr_lo; + u32 len_gen; + u32 flags; + u32 addr_hi; +}; + +struct freelQ_e { + u32 addr_lo; + u32 len_gen; + u32 gen2; + u32 addr_hi; +}; + +struct respQ_e { + u32 Qsleeping : 4; + u32 Cmdq1CreditReturn : 5; + u32 Cmdq1DmaComplete : 5; + u32 Cmdq0CreditReturn : 5; + u32 Cmdq0DmaComplete : 5; + u32 FreelistQid : 2; + u32 CreditValid : 1; + u32 DataValid : 1; + u32 Offload : 1; + u32 Eop : 1; + u32 Sop : 1; + u32 GenerationBit : 1; + u32 BufferLength; +}; +#elif defined(__LITTLE_ENDIAN_BITFIELD) +struct cmdQ_e { + u32 len_gen; + u32 addr_lo; + u32 addr_hi; + u32 flags; +}; + +struct freelQ_e { + u32 len_gen; + u32 addr_lo; + u32 addr_hi; + u32 gen2; +}; + +struct respQ_e { + u32 BufferLength; + u32 GenerationBit : 1; + u32 Sop : 1; + u32 Eop : 1; + u32 Offload : 1; + u32 DataValid : 1; + u32 CreditValid : 1; + u32 FreelistQid : 2; + u32 Cmdq0DmaComplete : 5; + u32 Cmdq0CreditReturn : 5; + u32 Cmdq1DmaComplete : 5; + u32 Cmdq1CreditReturn : 5; + u32 Qsleeping : 4; +} ; +#endif + +/* + * SW Context Command and Freelist Queue Descriptors + */ +struct cmdQ_ce { + struct sk_buff *skb; + DEFINE_DMA_UNMAP_ADDR(dma_addr); + DEFINE_DMA_UNMAP_LEN(dma_len); +}; + +struct freelQ_ce { + struct sk_buff *skb; + DEFINE_DMA_UNMAP_ADDR(dma_addr); + DEFINE_DMA_UNMAP_LEN(dma_len); +}; + +/* + * SW command, freelist and response rings + */ +struct cmdQ { + unsigned long status; /* HW DMA fetch status */ + unsigned int in_use; /* # of in-use command descriptors */ + unsigned int size; /* # of descriptors */ + unsigned int processed; /* total # of descs HW has processed */ + unsigned int cleaned; /* total # of descs SW has reclaimed */ + unsigned int stop_thres; /* SW TX queue suspend threshold */ + u16 pidx; /* producer index (SW) */ + u16 cidx; /* consumer index (HW) */ + u8 genbit; /* current generation (=valid) bit */ + u8 sop; /* is next entry start of packet? */ + struct cmdQ_e *entries; /* HW command descriptor Q */ + struct cmdQ_ce *centries; /* SW command context descriptor Q */ + dma_addr_t dma_addr; /* DMA addr HW command descriptor Q */ + spinlock_t lock; /* Lock to protect cmdQ enqueuing */ +}; + +struct freelQ { + unsigned int credits; /* # of available RX buffers */ + unsigned int size; /* free list capacity */ + u16 pidx; /* producer index (SW) */ + u16 cidx; /* consumer index (HW) */ + u16 rx_buffer_size; /* Buffer size on this free list */ + u16 dma_offset; /* DMA offset to align IP headers */ + u16 recycleq_idx; /* skb recycle q to use */ + u8 genbit; /* current generation (=valid) bit */ + struct freelQ_e *entries; /* HW freelist descriptor Q */ + struct freelQ_ce *centries; /* SW freelist context descriptor Q */ + dma_addr_t dma_addr; /* DMA addr HW freelist descriptor Q */ +}; + +struct respQ { + unsigned int credits; /* credits to be returned to SGE */ + unsigned int size; /* # of response Q descriptors */ + u16 cidx; /* consumer index (SW) */ + u8 genbit; /* current generation(=valid) bit */ + struct respQ_e *entries; /* HW response descriptor Q */ + dma_addr_t dma_addr; /* DMA addr HW response descriptor Q */ +}; + +/* Bit flags for cmdQ.status */ +enum { + CMDQ_STAT_RUNNING = 1, /* fetch engine is running */ + CMDQ_STAT_LAST_PKT_DB = 2 /* last packet rung the doorbell */ +}; + +/* T204 TX SW scheduler */ + +/* Per T204 TX port */ +struct sched_port { + unsigned int avail; /* available bits - quota */ + unsigned int drain_bits_per_1024ns; /* drain rate */ + unsigned int speed; /* drain rate, mbps */ + unsigned int mtu; /* mtu size */ + struct sk_buff_head skbq; /* pending skbs */ +}; + +/* Per T204 device */ +struct sched { + ktime_t last_updated; /* last time quotas were computed */ + unsigned int max_avail; /* max bits to be sent to any port */ + unsigned int port; /* port index (round robin ports) */ + unsigned int num; /* num skbs in per port queues */ + struct sched_port p[MAX_NPORTS]; + struct tasklet_struct sched_tsk;/* tasklet used to run scheduler */ + struct sge *sge; +}; + +static void restart_sched(struct tasklet_struct *t); + + +/* + * Main SGE data structure + * + * Interrupts are handled by a single CPU and it is likely that on a MP system + * the application is migrated to another CPU. In that scenario, we try to + * separate the RX(in irq context) and TX state in order to decrease memory + * contention. + */ +struct sge { + struct adapter *adapter; /* adapter backpointer */ + struct net_device *netdev; /* netdevice backpointer */ + struct freelQ freelQ[SGE_FREELQ_N]; /* buffer free lists */ + struct respQ respQ; /* response Q */ + unsigned long stopped_tx_queues; /* bitmap of suspended Tx queues */ + unsigned int rx_pkt_pad; /* RX padding for L2 packets */ + unsigned int jumbo_fl; /* jumbo freelist Q index */ + unsigned int intrtimer_nres; /* no-resource interrupt timer */ + unsigned int fixed_intrtimer;/* non-adaptive interrupt timer */ + struct timer_list tx_reclaim_timer; /* reclaims TX buffers */ + struct timer_list espibug_timer; + unsigned long espibug_timeout; + struct sk_buff *espibug_skb[MAX_NPORTS]; + u32 sge_control; /* shadow value of sge control reg */ + struct sge_intr_counts stats; + struct sge_port_stats __percpu *port_stats[MAX_NPORTS]; + struct sched *tx_sched; + struct cmdQ cmdQ[SGE_CMDQ_N] ____cacheline_aligned_in_smp; +}; + +static const u8 ch_mac_addr[ETH_ALEN] = { + 0x0, 0x7, 0x43, 0x0, 0x0, 0x0 +}; + +/* + * stop tasklet and free all pending skb's + */ +static void tx_sched_stop(struct sge *sge) +{ + struct sched *s = sge->tx_sched; + int i; + + tasklet_kill(&s->sched_tsk); + + for (i = 0; i < MAX_NPORTS; i++) + __skb_queue_purge(&s->p[s->port].skbq); +} + +/* + * t1_sched_update_parms() is called when the MTU or link speed changes. It + * re-computes scheduler parameters to scope with the change. + */ +unsigned int t1_sched_update_parms(struct sge *sge, unsigned int port, + unsigned int mtu, unsigned int speed) +{ + struct sched *s = sge->tx_sched; + struct sched_port *p = &s->p[port]; + unsigned int max_avail_segs; + + pr_debug("%s mtu=%d speed=%d\n", __func__, mtu, speed); + if (speed) + p->speed = speed; + if (mtu) + p->mtu = mtu; + + if (speed || mtu) { + unsigned long long drain = 1024ULL * p->speed * (p->mtu - 40); + do_div(drain, (p->mtu + 50) * 1000); + p->drain_bits_per_1024ns = (unsigned int) drain; + + if (p->speed < 1000) + p->drain_bits_per_1024ns = + 90 * p->drain_bits_per_1024ns / 100; + } + + if (board_info(sge->adapter)->board == CHBT_BOARD_CHT204) { + p->drain_bits_per_1024ns -= 16; + s->max_avail = max(4096U, p->mtu + 16 + 14 + 4); + max_avail_segs = max(1U, 4096 / (p->mtu - 40)); + } else { + s->max_avail = 16384; + max_avail_segs = max(1U, 9000 / (p->mtu - 40)); + } + + pr_debug("t1_sched_update_parms: mtu %u speed %u max_avail %u " + "max_avail_segs %u drain_bits_per_1024ns %u\n", p->mtu, + p->speed, s->max_avail, max_avail_segs, + p->drain_bits_per_1024ns); + + return max_avail_segs * (p->mtu - 40); +} + +#if 0 + +/* + * t1_sched_max_avail_bytes() tells the scheduler the maximum amount of + * data that can be pushed per port. + */ +void t1_sched_set_max_avail_bytes(struct sge *sge, unsigned int val) +{ + struct sched *s = sge->tx_sched; + unsigned int i; + + s->max_avail = val; + for (i = 0; i < MAX_NPORTS; i++) + t1_sched_update_parms(sge, i, 0, 0); +} + +/* + * t1_sched_set_drain_bits_per_us() tells the scheduler at which rate a port + * is draining. + */ +void t1_sched_set_drain_bits_per_us(struct sge *sge, unsigned int port, + unsigned int val) +{ + struct sched *s = sge->tx_sched; + struct sched_port *p = &s->p[port]; + p->drain_bits_per_1024ns = val * 1024 / 1000; + t1_sched_update_parms(sge, port, 0, 0); +} + +#endif /* 0 */ + +/* + * tx_sched_init() allocates resources and does basic initialization. + */ +static int tx_sched_init(struct sge *sge) +{ + struct sched *s; + int i; + + s = kzalloc(sizeof (struct sched), GFP_KERNEL); + if (!s) + return -ENOMEM; + + pr_debug("tx_sched_init\n"); + tasklet_setup(&s->sched_tsk, restart_sched); + s->sge = sge; + sge->tx_sched = s; + + for (i = 0; i < MAX_NPORTS; i++) { + skb_queue_head_init(&s->p[i].skbq); + t1_sched_update_parms(sge, i, 1500, 1000); + } + + return 0; +} + +/* + * sched_update_avail() computes the delta since the last time it was called + * and updates the per port quota (number of bits that can be sent to the any + * port). + */ +static inline int sched_update_avail(struct sge *sge) +{ + struct sched *s = sge->tx_sched; + ktime_t now = ktime_get(); + unsigned int i; + long long delta_time_ns; + + delta_time_ns = ktime_to_ns(ktime_sub(now, s->last_updated)); + + pr_debug("sched_update_avail delta=%lld\n", delta_time_ns); + if (delta_time_ns < 15000) + return 0; + + for (i = 0; i < MAX_NPORTS; i++) { + struct sched_port *p = &s->p[i]; + unsigned int delta_avail; + + delta_avail = (p->drain_bits_per_1024ns * delta_time_ns) >> 13; + p->avail = min(p->avail + delta_avail, s->max_avail); + } + + s->last_updated = now; + + return 1; +} + +/* + * sched_skb() is called from two different places. In the tx path, any + * packet generating load on an output port will call sched_skb() + * (skb != NULL). In addition, sched_skb() is called from the irq/soft irq + * context (skb == NULL). + * The scheduler only returns a skb (which will then be sent) if the + * length of the skb is <= the current quota of the output port. + */ +static struct sk_buff *sched_skb(struct sge *sge, struct sk_buff *skb, + unsigned int credits) +{ + struct sched *s = sge->tx_sched; + struct sk_buff_head *skbq; + unsigned int i, len, update = 1; + + pr_debug("sched_skb %p\n", skb); + if (!skb) { + if (!s->num) + return NULL; + } else { + skbq = &s->p[skb->dev->if_port].skbq; + __skb_queue_tail(skbq, skb); + s->num++; + skb = NULL; + } + + if (credits < MAX_SKB_FRAGS + 1) + goto out; + +again: + for (i = 0; i < MAX_NPORTS; i++) { + s->port = (s->port + 1) & (MAX_NPORTS - 1); + skbq = &s->p[s->port].skbq; + + skb = skb_peek(skbq); + + if (!skb) + continue; + + len = skb->len; + if (len <= s->p[s->port].avail) { + s->p[s->port].avail -= len; + s->num--; + __skb_unlink(skb, skbq); + goto out; + } + skb = NULL; + } + + if (update-- && sched_update_avail(sge)) + goto again; + +out: + /* If there are more pending skbs, we use the hardware to schedule us + * again. + */ + if (s->num && !skb) { + struct cmdQ *q = &sge->cmdQ[0]; + clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); + if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) { + set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); + writel(F_CMDQ0_ENABLE, sge->adapter->regs + A_SG_DOORBELL); + } + } + pr_debug("sched_skb ret %p\n", skb); + + return skb; +} + +/* + * PIO to indicate that memory mapped Q contains valid descriptor(s). + */ +static inline void doorbell_pio(struct adapter *adapter, u32 val) +{ + wmb(); + writel(val, adapter->regs + A_SG_DOORBELL); +} + +/* + * Frees all RX buffers on the freelist Q. The caller must make sure that + * the SGE is turned off before calling this function. + */ +static void free_freelQ_buffers(struct pci_dev *pdev, struct freelQ *q) +{ + unsigned int cidx = q->cidx; + + while (q->credits--) { + struct freelQ_ce *ce = &q->centries[cidx]; + + dma_unmap_single(&pdev->dev, dma_unmap_addr(ce, dma_addr), + dma_unmap_len(ce, dma_len), DMA_FROM_DEVICE); + dev_kfree_skb(ce->skb); + ce->skb = NULL; + if (++cidx == q->size) + cidx = 0; + } +} + +/* + * Free RX free list and response queue resources. + */ +static void free_rx_resources(struct sge *sge) +{ + struct pci_dev *pdev = sge->adapter->pdev; + unsigned int size, i; + + if (sge->respQ.entries) { + size = sizeof(struct respQ_e) * sge->respQ.size; + dma_free_coherent(&pdev->dev, size, sge->respQ.entries, + sge->respQ.dma_addr); + } + + for (i = 0; i < SGE_FREELQ_N; i++) { + struct freelQ *q = &sge->freelQ[i]; + + if (q->centries) { + free_freelQ_buffers(pdev, q); + kfree(q->centries); + } + if (q->entries) { + size = sizeof(struct freelQ_e) * q->size; + dma_free_coherent(&pdev->dev, size, q->entries, + q->dma_addr); + } + } +} + +/* + * Allocates basic RX resources, consisting of memory mapped freelist Qs and a + * response queue. + */ +static int alloc_rx_resources(struct sge *sge, struct sge_params *p) +{ + struct pci_dev *pdev = sge->adapter->pdev; + unsigned int size, i; + + for (i = 0; i < SGE_FREELQ_N; i++) { + struct freelQ *q = &sge->freelQ[i]; + + q->genbit = 1; + q->size = p->freelQ_size[i]; + q->dma_offset = sge->rx_pkt_pad ? 0 : NET_IP_ALIGN; + size = sizeof(struct freelQ_e) * q->size; + q->entries = dma_alloc_coherent(&pdev->dev, size, + &q->dma_addr, GFP_KERNEL); + if (!q->entries) + goto err_no_mem; + + size = sizeof(struct freelQ_ce) * q->size; + q->centries = kzalloc(size, GFP_KERNEL); + if (!q->centries) + goto err_no_mem; + } + + /* + * Calculate the buffer sizes for the two free lists. FL0 accommodates + * regular sized Ethernet frames, FL1 is sized not to exceed 16K, + * including all the sk_buff overhead. + * + * Note: For T2 FL0 and FL1 are reversed. + */ + sge->freelQ[!sge->jumbo_fl].rx_buffer_size = SGE_RX_SM_BUF_SIZE + + sizeof(struct cpl_rx_data) + + sge->freelQ[!sge->jumbo_fl].dma_offset; + + size = (16 * 1024) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + + sge->freelQ[sge->jumbo_fl].rx_buffer_size = size; + + /* + * Setup which skb recycle Q should be used when recycling buffers from + * each free list. + */ + sge->freelQ[!sge->jumbo_fl].recycleq_idx = 0; + sge->freelQ[sge->jumbo_fl].recycleq_idx = 1; + + sge->respQ.genbit = 1; + sge->respQ.size = SGE_RESPQ_E_N; + sge->respQ.credits = 0; + size = sizeof(struct respQ_e) * sge->respQ.size; + sge->respQ.entries = + dma_alloc_coherent(&pdev->dev, size, &sge->respQ.dma_addr, + GFP_KERNEL); + if (!sge->respQ.entries) + goto err_no_mem; + return 0; + +err_no_mem: + free_rx_resources(sge); + return -ENOMEM; +} + +/* + * Reclaims n TX descriptors and frees the buffers associated with them. + */ +static void free_cmdQ_buffers(struct sge *sge, struct cmdQ *q, unsigned int n) +{ + struct cmdQ_ce *ce; + struct pci_dev *pdev = sge->adapter->pdev; + unsigned int cidx = q->cidx; + + q->in_use -= n; + ce = &q->centries[cidx]; + while (n--) { + if (likely(dma_unmap_len(ce, dma_len))) { + dma_unmap_single(&pdev->dev, + dma_unmap_addr(ce, dma_addr), + dma_unmap_len(ce, dma_len), + DMA_TO_DEVICE); + if (q->sop) + q->sop = 0; + } + if (ce->skb) { + dev_kfree_skb_any(ce->skb); + q->sop = 1; + } + ce++; + if (++cidx == q->size) { + cidx = 0; + ce = q->centries; + } + } + q->cidx = cidx; +} + +/* + * Free TX resources. + * + * Assumes that SGE is stopped and all interrupts are disabled. + */ +static void free_tx_resources(struct sge *sge) +{ + struct pci_dev *pdev = sge->adapter->pdev; + unsigned int size, i; + + for (i = 0; i < SGE_CMDQ_N; i++) { + struct cmdQ *q = &sge->cmdQ[i]; + + if (q->centries) { + if (q->in_use) + free_cmdQ_buffers(sge, q, q->in_use); + kfree(q->centries); + } + if (q->entries) { + size = sizeof(struct cmdQ_e) * q->size; + dma_free_coherent(&pdev->dev, size, q->entries, + q->dma_addr); + } + } +} + +/* + * Allocates basic TX resources, consisting of memory mapped command Qs. + */ +static int alloc_tx_resources(struct sge *sge, struct sge_params *p) +{ + struct pci_dev *pdev = sge->adapter->pdev; + unsigned int size, i; + + for (i = 0; i < SGE_CMDQ_N; i++) { + struct cmdQ *q = &sge->cmdQ[i]; + + q->genbit = 1; + q->sop = 1; + q->size = p->cmdQ_size[i]; + q->in_use = 0; + q->status = 0; + q->processed = q->cleaned = 0; + q->stop_thres = 0; + spin_lock_init(&q->lock); + size = sizeof(struct cmdQ_e) * q->size; + q->entries = dma_alloc_coherent(&pdev->dev, size, + &q->dma_addr, GFP_KERNEL); + if (!q->entries) + goto err_no_mem; + + size = sizeof(struct cmdQ_ce) * q->size; + q->centries = kzalloc(size, GFP_KERNEL); + if (!q->centries) + goto err_no_mem; + } + + /* + * CommandQ 0 handles Ethernet and TOE packets, while queue 1 is TOE + * only. For queue 0 set the stop threshold so we can handle one more + * packet from each port, plus reserve an additional 24 entries for + * Ethernet packets only. Queue 1 never suspends nor do we reserve + * space for Ethernet packets. + */ + sge->cmdQ[0].stop_thres = sge->adapter->params.nports * + (MAX_SKB_FRAGS + 1); + return 0; + +err_no_mem: + free_tx_resources(sge); + return -ENOMEM; +} + +static inline void setup_ring_params(struct adapter *adapter, u64 addr, + u32 size, int base_reg_lo, + int base_reg_hi, int size_reg) +{ + writel((u32)addr, adapter->regs + base_reg_lo); + writel(addr >> 32, adapter->regs + base_reg_hi); + writel(size, adapter->regs + size_reg); +} + +/* + * Enable/disable VLAN acceleration. + */ +void t1_vlan_mode(struct adapter *adapter, netdev_features_t features) +{ + struct sge *sge = adapter->sge; + + if (features & NETIF_F_HW_VLAN_CTAG_RX) + sge->sge_control |= F_VLAN_XTRACT; + else + sge->sge_control &= ~F_VLAN_XTRACT; + if (adapter->open_device_map) { + writel(sge->sge_control, adapter->regs + A_SG_CONTROL); + readl(adapter->regs + A_SG_CONTROL); /* flush */ + } +} + +/* + * Programs the various SGE registers. However, the engine is not yet enabled, + * but sge->sge_control is setup and ready to go. + */ +static void configure_sge(struct sge *sge, struct sge_params *p) +{ + struct adapter *ap = sge->adapter; + + writel(0, ap->regs + A_SG_CONTROL); + setup_ring_params(ap, sge->cmdQ[0].dma_addr, sge->cmdQ[0].size, + A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE); + setup_ring_params(ap, sge->cmdQ[1].dma_addr, sge->cmdQ[1].size, + A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE); + setup_ring_params(ap, sge->freelQ[0].dma_addr, + sge->freelQ[0].size, A_SG_FL0BASELWR, + A_SG_FL0BASEUPR, A_SG_FL0SIZE); + setup_ring_params(ap, sge->freelQ[1].dma_addr, + sge->freelQ[1].size, A_SG_FL1BASELWR, + A_SG_FL1BASEUPR, A_SG_FL1SIZE); + + /* The threshold comparison uses <. */ + writel(SGE_RX_SM_BUF_SIZE + 1, ap->regs + A_SG_FLTHRESHOLD); + + setup_ring_params(ap, sge->respQ.dma_addr, sge->respQ.size, + A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE); + writel((u32)sge->respQ.size - 1, ap->regs + A_SG_RSPQUEUECREDIT); + + sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE | + F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE | + V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE | + V_RX_PKT_OFFSET(sge->rx_pkt_pad); + +#if defined(__BIG_ENDIAN_BITFIELD) + sge->sge_control |= F_ENABLE_BIG_ENDIAN; +#endif + + /* Initialize no-resource timer */ + sge->intrtimer_nres = SGE_INTRTIMER_NRES * core_ticks_per_usec(ap); + + t1_sge_set_coalesce_params(sge, p); +} + +/* + * Return the payload capacity of the jumbo free-list buffers. + */ +static inline unsigned int jumbo_payload_capacity(const struct sge *sge) +{ + return sge->freelQ[sge->jumbo_fl].rx_buffer_size - + sge->freelQ[sge->jumbo_fl].dma_offset - + sizeof(struct cpl_rx_data); +} + +/* + * Frees all SGE related resources and the sge structure itself + */ +void t1_sge_destroy(struct sge *sge) +{ + int i; + + for_each_port(sge->adapter, i) + free_percpu(sge->port_stats[i]); + + kfree(sge->tx_sched); + free_tx_resources(sge); + free_rx_resources(sge); + kfree(sge); +} + +/* + * Allocates new RX buffers on the freelist Q (and tracks them on the freelist + * context Q) until the Q is full or alloc_skb fails. + * + * It is possible that the generation bits already match, indicating that the + * buffer is already valid and nothing needs to be done. This happens when we + * copied a received buffer into a new sk_buff during the interrupt processing. + * + * If the SGE doesn't automatically align packets properly (!sge->rx_pkt_pad), + * we specify a RX_OFFSET in order to make sure that the IP header is 4B + * aligned. + */ +static void refill_free_list(struct sge *sge, struct freelQ *q) +{ + struct pci_dev *pdev = sge->adapter->pdev; + struct freelQ_ce *ce = &q->centries[q->pidx]; + struct freelQ_e *e = &q->entries[q->pidx]; + unsigned int dma_len = q->rx_buffer_size - q->dma_offset; + + while (q->credits < q->size) { + struct sk_buff *skb; + dma_addr_t mapping; + + skb = dev_alloc_skb(q->rx_buffer_size); + if (!skb) + break; + + skb_reserve(skb, q->dma_offset); + mapping = dma_map_single(&pdev->dev, skb->data, dma_len, + DMA_FROM_DEVICE); + skb_reserve(skb, sge->rx_pkt_pad); + + ce->skb = skb; + dma_unmap_addr_set(ce, dma_addr, mapping); + dma_unmap_len_set(ce, dma_len, dma_len); + e->addr_lo = (u32)mapping; + e->addr_hi = (u64)mapping >> 32; + e->len_gen = V_CMD_LEN(dma_len) | V_CMD_GEN1(q->genbit); + wmb(); + e->gen2 = V_CMD_GEN2(q->genbit); + + e++; + ce++; + if (++q->pidx == q->size) { + q->pidx = 0; + q->genbit ^= 1; + ce = q->centries; + e = q->entries; + } + q->credits++; + } +} + +/* + * Calls refill_free_list for both free lists. If we cannot fill at least 1/4 + * of both rings, we go into 'few interrupt mode' in order to give the system + * time to free up resources. + */ +static void freelQs_empty(struct sge *sge) +{ + struct adapter *adapter = sge->adapter; + u32 irq_reg = readl(adapter->regs + A_SG_INT_ENABLE); + u32 irqholdoff_reg; + + refill_free_list(sge, &sge->freelQ[0]); + refill_free_list(sge, &sge->freelQ[1]); + + if (sge->freelQ[0].credits > (sge->freelQ[0].size >> 2) && + sge->freelQ[1].credits > (sge->freelQ[1].size >> 2)) { + irq_reg |= F_FL_EXHAUSTED; + irqholdoff_reg = sge->fixed_intrtimer; + } else { + /* Clear the F_FL_EXHAUSTED interrupts for now */ + irq_reg &= ~F_FL_EXHAUSTED; + irqholdoff_reg = sge->intrtimer_nres; + } + writel(irqholdoff_reg, adapter->regs + A_SG_INTRTIMER); + writel(irq_reg, adapter->regs + A_SG_INT_ENABLE); + + /* We reenable the Qs to force a freelist GTS interrupt later */ + doorbell_pio(adapter, F_FL0_ENABLE | F_FL1_ENABLE); +} + +#define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA) +#define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH) +#define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \ + F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH) + +/* + * Disable SGE Interrupts + */ +void t1_sge_intr_disable(struct sge *sge) +{ + u32 val = readl(sge->adapter->regs + A_PL_ENABLE); + + writel(val & ~SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_ENABLE); + writel(0, sge->adapter->regs + A_SG_INT_ENABLE); +} + +/* + * Enable SGE interrupts. + */ +void t1_sge_intr_enable(struct sge *sge) +{ + u32 en = SGE_INT_ENABLE; + u32 val = readl(sge->adapter->regs + A_PL_ENABLE); + + if (sge->adapter->port[0].dev->hw_features & NETIF_F_TSO) + en &= ~F_PACKET_TOO_BIG; + writel(en, sge->adapter->regs + A_SG_INT_ENABLE); + writel(val | SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_ENABLE); +} + +/* + * Clear SGE interrupts. + */ +void t1_sge_intr_clear(struct sge *sge) +{ + writel(SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_CAUSE); + writel(0xffffffff, sge->adapter->regs + A_SG_INT_CAUSE); +} + +/* + * SGE 'Error' interrupt handler + */ +bool t1_sge_intr_error_handler(struct sge *sge) +{ + struct adapter *adapter = sge->adapter; + u32 cause = readl(adapter->regs + A_SG_INT_CAUSE); + bool wake = false; + + if (adapter->port[0].dev->hw_features & NETIF_F_TSO) + cause &= ~F_PACKET_TOO_BIG; + if (cause & F_RESPQ_EXHAUSTED) + sge->stats.respQ_empty++; + if (cause & F_RESPQ_OVERFLOW) { + sge->stats.respQ_overflow++; + pr_alert("%s: SGE response queue overflow\n", + adapter->name); + } + if (cause & F_FL_EXHAUSTED) { + sge->stats.freelistQ_empty++; + freelQs_empty(sge); + } + if (cause & F_PACKET_TOO_BIG) { + sge->stats.pkt_too_big++; + pr_alert("%s: SGE max packet size exceeded\n", + adapter->name); + } + if (cause & F_PACKET_MISMATCH) { + sge->stats.pkt_mismatch++; + pr_alert("%s: SGE packet mismatch\n", adapter->name); + } + if (cause & SGE_INT_FATAL) { + t1_interrupts_disable(adapter); + adapter->pending_thread_intr |= F_PL_INTR_SGE_ERR; + wake = true; + } + + writel(cause, adapter->regs + A_SG_INT_CAUSE); + return wake; +} + +const struct sge_intr_counts *t1_sge_get_intr_counts(const struct sge *sge) +{ + return &sge->stats; +} + +void t1_sge_get_port_stats(const struct sge *sge, int port, + struct sge_port_stats *ss) +{ + int cpu; + + memset(ss, 0, sizeof(*ss)); + for_each_possible_cpu(cpu) { + struct sge_port_stats *st = per_cpu_ptr(sge->port_stats[port], cpu); + + ss->rx_cso_good += st->rx_cso_good; + ss->tx_cso += st->tx_cso; + ss->tx_tso += st->tx_tso; + ss->tx_need_hdrroom += st->tx_need_hdrroom; + ss->vlan_xtract += st->vlan_xtract; + ss->vlan_insert += st->vlan_insert; + } +} + +/** + * recycle_fl_buf - recycle a free list buffer + * @fl: the free list + * @idx: index of buffer to recycle + * + * Recycles the specified buffer on the given free list by adding it at + * the next available slot on the list. + */ +static void recycle_fl_buf(struct freelQ *fl, int idx) +{ + struct freelQ_e *from = &fl->entries[idx]; + struct freelQ_e *to = &fl->entries[fl->pidx]; + + fl->centries[fl->pidx] = fl->centries[idx]; + to->addr_lo = from->addr_lo; + to->addr_hi = from->addr_hi; + to->len_gen = G_CMD_LEN(from->len_gen) | V_CMD_GEN1(fl->genbit); + wmb(); + to->gen2 = V_CMD_GEN2(fl->genbit); + fl->credits++; + + if (++fl->pidx == fl->size) { + fl->pidx = 0; + fl->genbit ^= 1; + } +} + +static int copybreak __read_mostly = 256; +module_param(copybreak, int, 0); +MODULE_PARM_DESC(copybreak, "Receive copy threshold"); + +/** + * get_packet - return the next ingress packet buffer + * @adapter: the adapter that received the packet + * @fl: the SGE free list holding the packet + * @len: the actual packet length, excluding any SGE padding + * + * Get the next packet from a free list and complete setup of the + * sk_buff. If the packet is small we make a copy and recycle the + * original buffer, otherwise we use the original buffer itself. If a + * positive drop threshold is supplied packets are dropped and their + * buffers recycled if (a) the number of remaining buffers is under the + * threshold and the packet is too big to copy, or (b) the packet should + * be copied but there is no memory for the copy. + */ +static inline struct sk_buff *get_packet(struct adapter *adapter, + struct freelQ *fl, unsigned int len) +{ + const struct freelQ_ce *ce = &fl->centries[fl->cidx]; + struct pci_dev *pdev = adapter->pdev; + struct sk_buff *skb; + + if (len < copybreak) { + skb = napi_alloc_skb(&adapter->napi, len); + if (!skb) + goto use_orig_buf; + + skb_put(skb, len); + dma_sync_single_for_cpu(&pdev->dev, + dma_unmap_addr(ce, dma_addr), + dma_unmap_len(ce, dma_len), + DMA_FROM_DEVICE); + skb_copy_from_linear_data(ce->skb, skb->data, len); + dma_sync_single_for_device(&pdev->dev, + dma_unmap_addr(ce, dma_addr), + dma_unmap_len(ce, dma_len), + DMA_FROM_DEVICE); + recycle_fl_buf(fl, fl->cidx); + return skb; + } + +use_orig_buf: + if (fl->credits < 2) { + recycle_fl_buf(fl, fl->cidx); + return NULL; + } + + dma_unmap_single(&pdev->dev, dma_unmap_addr(ce, dma_addr), + dma_unmap_len(ce, dma_len), DMA_FROM_DEVICE); + skb = ce->skb; + prefetch(skb->data); + + skb_put(skb, len); + return skb; +} + +/** + * unexpected_offload - handle an unexpected offload packet + * @adapter: the adapter + * @fl: the free list that received the packet + * + * Called when we receive an unexpected offload packet (e.g., the TOE + * function is disabled or the card is a NIC). Prints a message and + * recycles the buffer. + */ +static void unexpected_offload(struct adapter *adapter, struct freelQ *fl) +{ + struct freelQ_ce *ce = &fl->centries[fl->cidx]; + struct sk_buff *skb = ce->skb; + + dma_sync_single_for_cpu(&adapter->pdev->dev, + dma_unmap_addr(ce, dma_addr), + dma_unmap_len(ce, dma_len), DMA_FROM_DEVICE); + pr_err("%s: unexpected offload packet, cmd %u\n", + adapter->name, *skb->data); + recycle_fl_buf(fl, fl->cidx); +} + +/* + * T1/T2 SGE limits the maximum DMA size per TX descriptor to + * SGE_TX_DESC_MAX_PLEN (16KB). If the PAGE_SIZE is larger than 16KB, the + * stack might send more than SGE_TX_DESC_MAX_PLEN in a contiguous manner. + * Note that the *_large_page_tx_descs stuff will be optimized out when + * PAGE_SIZE <= SGE_TX_DESC_MAX_PLEN. + * + * compute_large_page_descs() computes how many additional descriptors are + * required to break down the stack's request. + */ +static inline unsigned int compute_large_page_tx_descs(struct sk_buff *skb) +{ + unsigned int count = 0; + + if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN) { + unsigned int nfrags = skb_shinfo(skb)->nr_frags; + unsigned int i, len = skb_headlen(skb); + while (len > SGE_TX_DESC_MAX_PLEN) { + count++; + len -= SGE_TX_DESC_MAX_PLEN; + } + for (i = 0; nfrags--; i++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + len = skb_frag_size(frag); + while (len > SGE_TX_DESC_MAX_PLEN) { + count++; + len -= SGE_TX_DESC_MAX_PLEN; + } + } + } + return count; +} + +/* + * Write a cmdQ entry. + * + * Since this function writes the 'flags' field, it must not be used to + * write the first cmdQ entry. + */ +static inline void write_tx_desc(struct cmdQ_e *e, dma_addr_t mapping, + unsigned int len, unsigned int gen, + unsigned int eop) +{ + BUG_ON(len > SGE_TX_DESC_MAX_PLEN); + + e->addr_lo = (u32)mapping; + e->addr_hi = (u64)mapping >> 32; + e->len_gen = V_CMD_LEN(len) | V_CMD_GEN1(gen); + e->flags = F_CMD_DATAVALID | V_CMD_EOP(eop) | V_CMD_GEN2(gen); +} + +/* + * See comment for previous function. + * + * write_tx_descs_large_page() writes additional SGE tx descriptors if + * *desc_len exceeds HW's capability. + */ +static inline unsigned int write_large_page_tx_descs(unsigned int pidx, + struct cmdQ_e **e, + struct cmdQ_ce **ce, + unsigned int *gen, + dma_addr_t *desc_mapping, + unsigned int *desc_len, + unsigned int nfrags, + struct cmdQ *q) +{ + if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN) { + struct cmdQ_e *e1 = *e; + struct cmdQ_ce *ce1 = *ce; + + while (*desc_len > SGE_TX_DESC_MAX_PLEN) { + *desc_len -= SGE_TX_DESC_MAX_PLEN; + write_tx_desc(e1, *desc_mapping, SGE_TX_DESC_MAX_PLEN, + *gen, nfrags == 0 && *desc_len == 0); + ce1->skb = NULL; + dma_unmap_len_set(ce1, dma_len, 0); + *desc_mapping += SGE_TX_DESC_MAX_PLEN; + if (*desc_len) { + ce1++; + e1++; + if (++pidx == q->size) { + pidx = 0; + *gen ^= 1; + ce1 = q->centries; + e1 = q->entries; + } + } + } + *e = e1; + *ce = ce1; + } + return pidx; +} + +/* + * Write the command descriptors to transmit the given skb starting at + * descriptor pidx with the given generation. + */ +static inline void write_tx_descs(struct adapter *adapter, struct sk_buff *skb, + unsigned int pidx, unsigned int gen, + struct cmdQ *q) +{ + dma_addr_t mapping, desc_mapping; + struct cmdQ_e *e, *e1; + struct cmdQ_ce *ce; + unsigned int i, flags, first_desc_len, desc_len, + nfrags = skb_shinfo(skb)->nr_frags; + + e = e1 = &q->entries[pidx]; + ce = &q->centries[pidx]; + + mapping = dma_map_single(&adapter->pdev->dev, skb->data, + skb_headlen(skb), DMA_TO_DEVICE); + + desc_mapping = mapping; + desc_len = skb_headlen(skb); + + flags = F_CMD_DATAVALID | F_CMD_SOP | + V_CMD_EOP(nfrags == 0 && desc_len <= SGE_TX_DESC_MAX_PLEN) | + V_CMD_GEN2(gen); + first_desc_len = (desc_len <= SGE_TX_DESC_MAX_PLEN) ? + desc_len : SGE_TX_DESC_MAX_PLEN; + e->addr_lo = (u32)desc_mapping; + e->addr_hi = (u64)desc_mapping >> 32; + e->len_gen = V_CMD_LEN(first_desc_len) | V_CMD_GEN1(gen); + ce->skb = NULL; + dma_unmap_len_set(ce, dma_len, 0); + + if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN && + desc_len > SGE_TX_DESC_MAX_PLEN) { + desc_mapping += first_desc_len; + desc_len -= first_desc_len; + e1++; + ce++; + if (++pidx == q->size) { + pidx = 0; + gen ^= 1; + e1 = q->entries; + ce = q->centries; + } + pidx = write_large_page_tx_descs(pidx, &e1, &ce, &gen, + &desc_mapping, &desc_len, + nfrags, q); + + if (likely(desc_len)) + write_tx_desc(e1, desc_mapping, desc_len, gen, + nfrags == 0); + } + + ce->skb = NULL; + dma_unmap_addr_set(ce, dma_addr, mapping); + dma_unmap_len_set(ce, dma_len, skb_headlen(skb)); + + for (i = 0; nfrags--; i++) { + skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + e1++; + ce++; + if (++pidx == q->size) { + pidx = 0; + gen ^= 1; + e1 = q->entries; + ce = q->centries; + } + + mapping = skb_frag_dma_map(&adapter->pdev->dev, frag, 0, + skb_frag_size(frag), DMA_TO_DEVICE); + desc_mapping = mapping; + desc_len = skb_frag_size(frag); + + pidx = write_large_page_tx_descs(pidx, &e1, &ce, &gen, + &desc_mapping, &desc_len, + nfrags, q); + if (likely(desc_len)) + write_tx_desc(e1, desc_mapping, desc_len, gen, + nfrags == 0); + ce->skb = NULL; + dma_unmap_addr_set(ce, dma_addr, mapping); + dma_unmap_len_set(ce, dma_len, skb_frag_size(frag)); + } + ce->skb = skb; + wmb(); + e->flags = flags; +} + +/* + * Clean up completed Tx buffers. + */ +static inline void reclaim_completed_tx(struct sge *sge, struct cmdQ *q) +{ + unsigned int reclaim = q->processed - q->cleaned; + + if (reclaim) { + pr_debug("reclaim_completed_tx processed:%d cleaned:%d\n", + q->processed, q->cleaned); + free_cmdQ_buffers(sge, q, reclaim); + q->cleaned += reclaim; + } +} + +/* + * Called from tasklet. Checks the scheduler for any + * pending skbs that can be sent. + */ +static void restart_sched(struct tasklet_struct *t) +{ + struct sched *s = from_tasklet(s, t, sched_tsk); + struct sge *sge = s->sge; + struct adapter *adapter = sge->adapter; + struct cmdQ *q = &sge->cmdQ[0]; + struct sk_buff *skb; + unsigned int credits, queued_skb = 0; + + spin_lock(&q->lock); + reclaim_completed_tx(sge, q); + + credits = q->size - q->in_use; + pr_debug("restart_sched credits=%d\n", credits); + while ((skb = sched_skb(sge, NULL, credits)) != NULL) { + unsigned int genbit, pidx, count; + count = 1 + skb_shinfo(skb)->nr_frags; + count += compute_large_page_tx_descs(skb); + q->in_use += count; + genbit = q->genbit; + pidx = q->pidx; + q->pidx += count; + if (q->pidx >= q->size) { + q->pidx -= q->size; + q->genbit ^= 1; + } + write_tx_descs(adapter, skb, pidx, genbit, q); + credits = q->size - q->in_use; + queued_skb = 1; + } + + if (queued_skb) { + clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); + if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) { + set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); + writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL); + } + } + spin_unlock(&q->lock); +} + +/** + * sge_rx - process an ingress ethernet packet + * @sge: the sge structure + * @fl: the free list that contains the packet buffer + * @len: the packet length + * + * Process an ingress ethernet packet and deliver it to the stack. + */ +static void sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len) +{ + struct sk_buff *skb; + const struct cpl_rx_pkt *p; + struct adapter *adapter = sge->adapter; + struct sge_port_stats *st; + struct net_device *dev; + + skb = get_packet(adapter, fl, len - sge->rx_pkt_pad); + if (unlikely(!skb)) { + sge->stats.rx_drops++; + return; + } + + p = (const struct cpl_rx_pkt *) skb->data; + if (p->iff >= adapter->params.nports) { + kfree_skb(skb); + return; + } + __skb_pull(skb, sizeof(*p)); + + st = this_cpu_ptr(sge->port_stats[p->iff]); + dev = adapter->port[p->iff].dev; + + skb->protocol = eth_type_trans(skb, dev); + if ((dev->features & NETIF_F_RXCSUM) && p->csum == 0xffff && + skb->protocol == htons(ETH_P_IP) && + (skb->data[9] == IPPROTO_TCP || skb->data[9] == IPPROTO_UDP)) { + ++st->rx_cso_good; + skb->ip_summed = CHECKSUM_UNNECESSARY; + } else + skb_checksum_none_assert(skb); + + if (p->vlan_valid) { + st->vlan_xtract++; + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(p->vlan)); + } + netif_receive_skb(skb); +} + +/* + * Returns true if a command queue has enough available descriptors that + * we can resume Tx operation after temporarily disabling its packet queue. + */ +static inline int enough_free_Tx_descs(const struct cmdQ *q) +{ + unsigned int r = q->processed - q->cleaned; + + return q->in_use - r < (q->size >> 1); +} + +/* + * Called when sufficient space has become available in the SGE command queues + * after the Tx packet schedulers have been suspended to restart the Tx path. + */ +static void restart_tx_queues(struct sge *sge) +{ + struct adapter *adap = sge->adapter; + int i; + + if (!enough_free_Tx_descs(&sge->cmdQ[0])) + return; + + for_each_port(adap, i) { + struct net_device *nd = adap->port[i].dev; + + if (test_and_clear_bit(nd->if_port, &sge->stopped_tx_queues) && + netif_running(nd)) { + sge->stats.cmdQ_restarted[2]++; + netif_wake_queue(nd); + } + } +} + +/* + * update_tx_info is called from the interrupt handler/NAPI to return cmdQ0 + * information. + */ +static unsigned int update_tx_info(struct adapter *adapter, + unsigned int flags, + unsigned int pr0) +{ + struct sge *sge = adapter->sge; + struct cmdQ *cmdq = &sge->cmdQ[0]; + + cmdq->processed += pr0; + if (flags & (F_FL0_ENABLE | F_FL1_ENABLE)) { + freelQs_empty(sge); + flags &= ~(F_FL0_ENABLE | F_FL1_ENABLE); + } + if (flags & F_CMDQ0_ENABLE) { + clear_bit(CMDQ_STAT_RUNNING, &cmdq->status); + + if (cmdq->cleaned + cmdq->in_use != cmdq->processed && + !test_and_set_bit(CMDQ_STAT_LAST_PKT_DB, &cmdq->status)) { + set_bit(CMDQ_STAT_RUNNING, &cmdq->status); + writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL); + } + if (sge->tx_sched) + tasklet_hi_schedule(&sge->tx_sched->sched_tsk); + + flags &= ~F_CMDQ0_ENABLE; + } + + if (unlikely(sge->stopped_tx_queues != 0)) + restart_tx_queues(sge); + + return flags; +} + +/* + * Process SGE responses, up to the supplied budget. Returns the number of + * responses processed. A negative budget is effectively unlimited. + */ +static int process_responses(struct adapter *adapter, int budget) +{ + struct sge *sge = adapter->sge; + struct respQ *q = &sge->respQ; + struct respQ_e *e = &q->entries[q->cidx]; + int done = 0; + unsigned int flags = 0; + unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0}; + + while (done < budget && e->GenerationBit == q->genbit) { + flags |= e->Qsleeping; + + cmdq_processed[0] += e->Cmdq0CreditReturn; + cmdq_processed[1] += e->Cmdq1CreditReturn; + + /* We batch updates to the TX side to avoid cacheline + * ping-pong of TX state information on MP where the sender + * might run on a different CPU than this function... + */ + if (unlikely((flags & F_CMDQ0_ENABLE) || cmdq_processed[0] > 64)) { + flags = update_tx_info(adapter, flags, cmdq_processed[0]); + cmdq_processed[0] = 0; + } + + if (unlikely(cmdq_processed[1] > 16)) { + sge->cmdQ[1].processed += cmdq_processed[1]; + cmdq_processed[1] = 0; + } + + if (likely(e->DataValid)) { + struct freelQ *fl = &sge->freelQ[e->FreelistQid]; + + BUG_ON(!e->Sop || !e->Eop); + if (unlikely(e->Offload)) + unexpected_offload(adapter, fl); + else + sge_rx(sge, fl, e->BufferLength); + + ++done; + + /* + * Note: this depends on each packet consuming a + * single free-list buffer; cf. the BUG above. + */ + if (++fl->cidx == fl->size) + fl->cidx = 0; + prefetch(fl->centries[fl->cidx].skb); + + if (unlikely(--fl->credits < + fl->size - SGE_FREEL_REFILL_THRESH)) + refill_free_list(sge, fl); + } else + sge->stats.pure_rsps++; + + e++; + if (unlikely(++q->cidx == q->size)) { + q->cidx = 0; + q->genbit ^= 1; + e = q->entries; + } + prefetch(e); + + if (++q->credits > SGE_RESPQ_REPLENISH_THRES) { + writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT); + q->credits = 0; + } + } + + flags = update_tx_info(adapter, flags, cmdq_processed[0]); + sge->cmdQ[1].processed += cmdq_processed[1]; + + return done; +} + +static inline int responses_pending(const struct adapter *adapter) +{ + const struct respQ *Q = &adapter->sge->respQ; + const struct respQ_e *e = &Q->entries[Q->cidx]; + + return e->GenerationBit == Q->genbit; +} + +/* + * A simpler version of process_responses() that handles only pure (i.e., + * non data-carrying) responses. Such respones are too light-weight to justify + * calling a softirq when using NAPI, so we handle them specially in hard + * interrupt context. The function is called with a pointer to a response, + * which the caller must ensure is a valid pure response. Returns 1 if it + * encounters a valid data-carrying response, 0 otherwise. + */ +static int process_pure_responses(struct adapter *adapter) +{ + struct sge *sge = adapter->sge; + struct respQ *q = &sge->respQ; + struct respQ_e *e = &q->entries[q->cidx]; + const struct freelQ *fl = &sge->freelQ[e->FreelistQid]; + unsigned int flags = 0; + unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0}; + + prefetch(fl->centries[fl->cidx].skb); + if (e->DataValid) + return 1; + + do { + flags |= e->Qsleeping; + + cmdq_processed[0] += e->Cmdq0CreditReturn; + cmdq_processed[1] += e->Cmdq1CreditReturn; + + e++; + if (unlikely(++q->cidx == q->size)) { + q->cidx = 0; + q->genbit ^= 1; + e = q->entries; + } + prefetch(e); + + if (++q->credits > SGE_RESPQ_REPLENISH_THRES) { + writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT); + q->credits = 0; + } + sge->stats.pure_rsps++; + } while (e->GenerationBit == q->genbit && !e->DataValid); + + flags = update_tx_info(adapter, flags, cmdq_processed[0]); + sge->cmdQ[1].processed += cmdq_processed[1]; + + return e->GenerationBit == q->genbit; +} + +/* + * Handler for new data events when using NAPI. This does not need any locking + * or protection from interrupts as data interrupts are off at this point and + * other adapter interrupts do not interfere. + */ +int t1_poll(struct napi_struct *napi, int budget) +{ + struct adapter *adapter = container_of(napi, struct adapter, napi); + int work_done = process_responses(adapter, budget); + + if (likely(work_done < budget)) { + napi_complete_done(napi, work_done); + writel(adapter->sge->respQ.cidx, + adapter->regs + A_SG_SLEEPING); + } + return work_done; +} + +irqreturn_t t1_interrupt_thread(int irq, void *data) +{ + struct adapter *adapter = data; + u32 pending_thread_intr; + + spin_lock_irq(&adapter->async_lock); + pending_thread_intr = adapter->pending_thread_intr; + adapter->pending_thread_intr = 0; + spin_unlock_irq(&adapter->async_lock); + + if (!pending_thread_intr) + return IRQ_NONE; + + if (pending_thread_intr & F_PL_INTR_EXT) + t1_elmer0_ext_intr_handler(adapter); + + /* This error is fatal, interrupts remain off */ + if (pending_thread_intr & F_PL_INTR_SGE_ERR) { + pr_alert("%s: encountered fatal error, operation suspended\n", + adapter->name); + t1_sge_stop(adapter->sge); + return IRQ_HANDLED; + } + + spin_lock_irq(&adapter->async_lock); + adapter->slow_intr_mask |= F_PL_INTR_EXT; + + writel(F_PL_INTR_EXT, adapter->regs + A_PL_CAUSE); + writel(adapter->slow_intr_mask | F_PL_INTR_SGE_DATA, + adapter->regs + A_PL_ENABLE); + spin_unlock_irq(&adapter->async_lock); + + return IRQ_HANDLED; +} + +irqreturn_t t1_interrupt(int irq, void *data) +{ + struct adapter *adapter = data; + struct sge *sge = adapter->sge; + irqreturn_t handled; + + if (likely(responses_pending(adapter))) { + writel(F_PL_INTR_SGE_DATA, adapter->regs + A_PL_CAUSE); + + if (napi_schedule_prep(&adapter->napi)) { + if (process_pure_responses(adapter)) + __napi_schedule(&adapter->napi); + else { + /* no data, no NAPI needed */ + writel(sge->respQ.cidx, adapter->regs + A_SG_SLEEPING); + /* undo schedule_prep */ + napi_enable(&adapter->napi); + } + } + return IRQ_HANDLED; + } + + spin_lock(&adapter->async_lock); + handled = t1_slow_intr_handler(adapter); + spin_unlock(&adapter->async_lock); + + if (handled == IRQ_NONE) + sge->stats.unhandled_irqs++; + + return handled; +} + +/* + * Enqueues the sk_buff onto the cmdQ[qid] and has hardware fetch it. + * + * The code figures out how many entries the sk_buff will require in the + * cmdQ and updates the cmdQ data structure with the state once the enqueue + * has complete. Then, it doesn't access the global structure anymore, but + * uses the corresponding fields on the stack. In conjunction with a spinlock + * around that code, we can make the function reentrant without holding the + * lock when we actually enqueue (which might be expensive, especially on + * architectures with IO MMUs). + * + * This runs with softirqs disabled. + */ +static int t1_sge_tx(struct sk_buff *skb, struct adapter *adapter, + unsigned int qid, struct net_device *dev) +{ + struct sge *sge = adapter->sge; + struct cmdQ *q = &sge->cmdQ[qid]; + unsigned int credits, pidx, genbit, count, use_sched_skb = 0; + + spin_lock(&q->lock); + + reclaim_completed_tx(sge, q); + + pidx = q->pidx; + credits = q->size - q->in_use; + count = 1 + skb_shinfo(skb)->nr_frags; + count += compute_large_page_tx_descs(skb); + + /* Ethernet packet */ + if (unlikely(credits < count)) { + if (!netif_queue_stopped(dev)) { + netif_stop_queue(dev); + set_bit(dev->if_port, &sge->stopped_tx_queues); + sge->stats.cmdQ_full[2]++; + pr_err("%s: Tx ring full while queue awake!\n", + adapter->name); + } + spin_unlock(&q->lock); + return NETDEV_TX_BUSY; + } + + if (unlikely(credits - count < q->stop_thres)) { + netif_stop_queue(dev); + set_bit(dev->if_port, &sge->stopped_tx_queues); + sge->stats.cmdQ_full[2]++; + } + + /* T204 cmdQ0 skbs that are destined for a certain port have to go + * through the scheduler. + */ + if (sge->tx_sched && !qid && skb->dev) { +use_sched: + use_sched_skb = 1; + /* Note that the scheduler might return a different skb than + * the one passed in. + */ + skb = sched_skb(sge, skb, credits); + if (!skb) { + spin_unlock(&q->lock); + return NETDEV_TX_OK; + } + pidx = q->pidx; + count = 1 + skb_shinfo(skb)->nr_frags; + count += compute_large_page_tx_descs(skb); + } + + q->in_use += count; + genbit = q->genbit; + pidx = q->pidx; + q->pidx += count; + if (q->pidx >= q->size) { + q->pidx -= q->size; + q->genbit ^= 1; + } + spin_unlock(&q->lock); + + write_tx_descs(adapter, skb, pidx, genbit, q); + + /* + * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring + * the doorbell if the Q is asleep. There is a natural race, where + * the hardware is going to sleep just after we checked, however, + * then the interrupt handler will detect the outstanding TX packet + * and ring the doorbell for us. + */ + if (qid) + doorbell_pio(adapter, F_CMDQ1_ENABLE); + else { + clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); + if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) { + set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); + writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL); + } + } + + if (use_sched_skb) { + if (spin_trylock(&q->lock)) { + credits = q->size - q->in_use; + skb = NULL; + goto use_sched; + } + } + return NETDEV_TX_OK; +} + +#define MK_ETH_TYPE_MSS(type, mss) (((mss) & 0x3FFF) | ((type) << 14)) + +/* + * eth_hdr_len - return the length of an Ethernet header + * @data: pointer to the start of the Ethernet header + * + * Returns the length of an Ethernet header, including optional VLAN tag. + */ +static inline int eth_hdr_len(const void *data) +{ + const struct ethhdr *e = data; + + return e->h_proto == htons(ETH_P_8021Q) ? VLAN_ETH_HLEN : ETH_HLEN; +} + +/* + * Adds the CPL header to the sk_buff and passes it to t1_sge_tx. + */ +netdev_tx_t t1_start_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct adapter *adapter = dev->ml_priv; + struct sge *sge = adapter->sge; + struct sge_port_stats *st = this_cpu_ptr(sge->port_stats[dev->if_port]); + struct cpl_tx_pkt *cpl; + struct sk_buff *orig_skb = skb; + int ret; + + if (skb->protocol == htons(ETH_P_CPL5)) + goto send; + + /* + * We are using a non-standard hard_header_len. + * Allocate more header room in the rare cases it is not big enough. + */ + if (unlikely(skb_headroom(skb) < dev->hard_header_len - ETH_HLEN)) { + skb = skb_realloc_headroom(skb, sizeof(struct cpl_tx_pkt_lso)); + ++st->tx_need_hdrroom; + dev_kfree_skb_any(orig_skb); + if (!skb) + return NETDEV_TX_OK; + } + + if (skb_shinfo(skb)->gso_size) { + int eth_type; + struct cpl_tx_pkt_lso *hdr; + + ++st->tx_tso; + + eth_type = skb_network_offset(skb) == ETH_HLEN ? + CPL_ETH_II : CPL_ETH_II_VLAN; + + hdr = skb_push(skb, sizeof(*hdr)); + hdr->opcode = CPL_TX_PKT_LSO; + hdr->ip_csum_dis = hdr->l4_csum_dis = 0; + hdr->ip_hdr_words = ip_hdr(skb)->ihl; + hdr->tcp_hdr_words = tcp_hdr(skb)->doff; + hdr->eth_type_mss = htons(MK_ETH_TYPE_MSS(eth_type, + skb_shinfo(skb)->gso_size)); + hdr->len = htonl(skb->len - sizeof(*hdr)); + cpl = (struct cpl_tx_pkt *)hdr; + } else { + /* + * Packets shorter than ETH_HLEN can break the MAC, drop them + * early. Also, we may get oversized packets because some + * parts of the kernel don't handle our unusual hard_header_len + * right, drop those too. + */ + if (unlikely(skb->len < ETH_HLEN || + skb->len > dev->mtu + eth_hdr_len(skb->data))) { + netdev_dbg(dev, "packet size %d hdr %d mtu%d\n", + skb->len, eth_hdr_len(skb->data), dev->mtu); + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + + if (skb->ip_summed == CHECKSUM_PARTIAL && + ip_hdr(skb)->protocol == IPPROTO_UDP) { + if (unlikely(skb_checksum_help(skb))) { + netdev_dbg(dev, "unable to do udp checksum\n"); + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + } + + /* Hmmm, assuming to catch the gratious arp... and we'll use + * it to flush out stuck espi packets... + */ + if ((unlikely(!adapter->sge->espibug_skb[dev->if_port]))) { + if (skb->protocol == htons(ETH_P_ARP) && + arp_hdr(skb)->ar_op == htons(ARPOP_REQUEST)) { + adapter->sge->espibug_skb[dev->if_port] = skb; + /* We want to re-use this skb later. We + * simply bump the reference count and it + * will not be freed... + */ + skb = skb_get(skb); + } + } + + cpl = __skb_push(skb, sizeof(*cpl)); + cpl->opcode = CPL_TX_PKT; + cpl->ip_csum_dis = 1; /* SW calculates IP csum */ + cpl->l4_csum_dis = skb->ip_summed == CHECKSUM_PARTIAL ? 0 : 1; + /* the length field isn't used so don't bother setting it */ + + st->tx_cso += (skb->ip_summed == CHECKSUM_PARTIAL); + } + cpl->iff = dev->if_port; + + if (skb_vlan_tag_present(skb)) { + cpl->vlan_valid = 1; + cpl->vlan = htons(skb_vlan_tag_get(skb)); + st->vlan_insert++; + } else + cpl->vlan_valid = 0; + +send: + ret = t1_sge_tx(skb, adapter, 0, dev); + + /* If transmit busy, and we reallocated skb's due to headroom limit, + * then silently discard to avoid leak. + */ + if (unlikely(ret != NETDEV_TX_OK && skb != orig_skb)) { + dev_kfree_skb_any(skb); + ret = NETDEV_TX_OK; + } + return ret; +} + +/* + * Callback for the Tx buffer reclaim timer. Runs with softirqs disabled. + */ +static void sge_tx_reclaim_cb(struct timer_list *t) +{ + int i; + struct sge *sge = from_timer(sge, t, tx_reclaim_timer); + + for (i = 0; i < SGE_CMDQ_N; ++i) { + struct cmdQ *q = &sge->cmdQ[i]; + + if (!spin_trylock(&q->lock)) + continue; + + reclaim_completed_tx(sge, q); + if (i == 0 && q->in_use) { /* flush pending credits */ + writel(F_CMDQ0_ENABLE, sge->adapter->regs + A_SG_DOORBELL); + } + spin_unlock(&q->lock); + } + mod_timer(&sge->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD); +} + +/* + * Propagate changes of the SGE coalescing parameters to the HW. + */ +int t1_sge_set_coalesce_params(struct sge *sge, struct sge_params *p) +{ + sge->fixed_intrtimer = p->rx_coalesce_usecs * + core_ticks_per_usec(sge->adapter); + writel(sge->fixed_intrtimer, sge->adapter->regs + A_SG_INTRTIMER); + return 0; +} + +/* + * Allocates both RX and TX resources and configures the SGE. However, + * the hardware is not enabled yet. + */ +int t1_sge_configure(struct sge *sge, struct sge_params *p) +{ + if (alloc_rx_resources(sge, p)) + return -ENOMEM; + if (alloc_tx_resources(sge, p)) { + free_rx_resources(sge); + return -ENOMEM; + } + configure_sge(sge, p); + + /* + * Now that we have sized the free lists calculate the payload + * capacity of the large buffers. Other parts of the driver use + * this to set the max offload coalescing size so that RX packets + * do not overflow our large buffers. + */ + p->large_buf_capacity = jumbo_payload_capacity(sge); + return 0; +} + +/* + * Disables the DMA engine. + */ +void t1_sge_stop(struct sge *sge) +{ + int i; + writel(0, sge->adapter->regs + A_SG_CONTROL); + readl(sge->adapter->regs + A_SG_CONTROL); /* flush */ + + if (is_T2(sge->adapter)) + del_timer_sync(&sge->espibug_timer); + + del_timer_sync(&sge->tx_reclaim_timer); + if (sge->tx_sched) + tx_sched_stop(sge); + + for (i = 0; i < MAX_NPORTS; i++) + kfree_skb(sge->espibug_skb[i]); +} + +/* + * Enables the DMA engine. + */ +void t1_sge_start(struct sge *sge) +{ + refill_free_list(sge, &sge->freelQ[0]); + refill_free_list(sge, &sge->freelQ[1]); + + writel(sge->sge_control, sge->adapter->regs + A_SG_CONTROL); + doorbell_pio(sge->adapter, F_FL0_ENABLE | F_FL1_ENABLE); + readl(sge->adapter->regs + A_SG_CONTROL); /* flush */ + + mod_timer(&sge->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD); + + if (is_T2(sge->adapter)) + mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout); +} + +/* + * Callback for the T2 ESPI 'stuck packet feature' workaorund + */ +static void espibug_workaround_t204(struct timer_list *t) +{ + struct sge *sge = from_timer(sge, t, espibug_timer); + struct adapter *adapter = sge->adapter; + unsigned int nports = adapter->params.nports; + u32 seop[MAX_NPORTS]; + + if (adapter->open_device_map & PORT_MASK) { + int i; + + if (t1_espi_get_mon_t204(adapter, &(seop[0]), 0) < 0) + return; + + for (i = 0; i < nports; i++) { + struct sk_buff *skb = sge->espibug_skb[i]; + + if (!netif_running(adapter->port[i].dev) || + netif_queue_stopped(adapter->port[i].dev) || + !seop[i] || ((seop[i] & 0xfff) != 0) || !skb) + continue; + + if (!skb->cb[0]) { + skb_copy_to_linear_data_offset(skb, + sizeof(struct cpl_tx_pkt), + ch_mac_addr, + ETH_ALEN); + skb_copy_to_linear_data_offset(skb, + skb->len - 10, + ch_mac_addr, + ETH_ALEN); + skb->cb[0] = 0xff; + } + + /* bump the reference count to avoid freeing of + * the skb once the DMA has completed. + */ + skb = skb_get(skb); + t1_sge_tx(skb, adapter, 0, adapter->port[i].dev); + } + } + mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout); +} + +static void espibug_workaround(struct timer_list *t) +{ + struct sge *sge = from_timer(sge, t, espibug_timer); + struct adapter *adapter = sge->adapter; + + if (netif_running(adapter->port[0].dev)) { + struct sk_buff *skb = sge->espibug_skb[0]; + u32 seop = t1_espi_get_mon(adapter, 0x930, 0); + + if ((seop & 0xfff0fff) == 0xfff && skb) { + if (!skb->cb[0]) { + skb_copy_to_linear_data_offset(skb, + sizeof(struct cpl_tx_pkt), + ch_mac_addr, + ETH_ALEN); + skb_copy_to_linear_data_offset(skb, + skb->len - 10, + ch_mac_addr, + ETH_ALEN); + skb->cb[0] = 0xff; + } + + /* bump the reference count to avoid freeing of the + * skb once the DMA has completed. + */ + skb = skb_get(skb); + t1_sge_tx(skb, adapter, 0, adapter->port[0].dev); + } + } + mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout); +} + +/* + * Creates a t1_sge structure and returns suggested resource parameters. + */ +struct sge *t1_sge_create(struct adapter *adapter, struct sge_params *p) +{ + struct sge *sge = kzalloc(sizeof(*sge), GFP_KERNEL); + int i; + + if (!sge) + return NULL; + + sge->adapter = adapter; + sge->netdev = adapter->port[0].dev; + sge->rx_pkt_pad = t1_is_T1B(adapter) ? 0 : 2; + sge->jumbo_fl = t1_is_T1B(adapter) ? 1 : 0; + + for_each_port(adapter, i) { + sge->port_stats[i] = alloc_percpu(struct sge_port_stats); + if (!sge->port_stats[i]) + goto nomem_port; + } + + timer_setup(&sge->tx_reclaim_timer, sge_tx_reclaim_cb, 0); + + if (is_T2(sge->adapter)) { + timer_setup(&sge->espibug_timer, + adapter->params.nports > 1 ? espibug_workaround_t204 : espibug_workaround, + 0); + + if (adapter->params.nports > 1) + tx_sched_init(sge); + + sge->espibug_timeout = 1; + /* for T204, every 10ms */ + if (adapter->params.nports > 1) + sge->espibug_timeout = HZ/100; + } + + + p->cmdQ_size[0] = SGE_CMDQ0_E_N; + p->cmdQ_size[1] = SGE_CMDQ1_E_N; + p->freelQ_size[!sge->jumbo_fl] = SGE_FREEL_SIZE; + p->freelQ_size[sge->jumbo_fl] = SGE_JUMBO_FREEL_SIZE; + if (sge->tx_sched) { + if (board_info(sge->adapter)->board == CHBT_BOARD_CHT204) + p->rx_coalesce_usecs = 15; + else + p->rx_coalesce_usecs = 50; + } else + p->rx_coalesce_usecs = 50; + + p->coalesce_enable = 0; + p->sample_interval_usecs = 0; + + return sge; +nomem_port: + while (i >= 0) { + free_percpu(sge->port_stats[i]); + --i; + } + kfree(sge); + return NULL; + +} diff --git a/drivers/net/ethernet/chelsio/cxgb/sge.h b/drivers/net/ethernet/chelsio/cxgb/sge.h new file mode 100644 index 000000000..f7e6f6404 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/sge.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/***************************************************************************** + * * + * File: sge.h * + * $Revision: 1.11 $ * + * $Date: 2005/06/21 22:10:55 $ * + * Description: * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#ifndef _CXGB_SGE_H_ +#define _CXGB_SGE_H_ + +#include <linux/types.h> +#include <linux/interrupt.h> +#include <asm/byteorder.h> + +struct sge_intr_counts { + unsigned int rx_drops; /* # of packets dropped due to no mem */ + unsigned int pure_rsps; /* # of non-payload responses */ + unsigned int unhandled_irqs; /* # of unhandled interrupts */ + unsigned int respQ_empty; /* # times respQ empty */ + unsigned int respQ_overflow; /* # respQ overflow (fatal) */ + unsigned int freelistQ_empty; /* # times freelist empty */ + unsigned int pkt_too_big; /* packet too large (fatal) */ + unsigned int pkt_mismatch; + unsigned int cmdQ_full[3]; /* not HW IRQ, host cmdQ[] full */ + unsigned int cmdQ_restarted[3];/* # of times cmdQ X was restarted */ +}; + +struct sge_port_stats { + u64 rx_cso_good; /* # of successful RX csum offloads */ + u64 tx_cso; /* # of TX checksum offloads */ + u64 tx_tso; /* # of TSO requests */ + u64 vlan_xtract; /* # of VLAN tag extractions */ + u64 vlan_insert; /* # of VLAN tag insertions */ + u64 tx_need_hdrroom; /* # of TX skbs in need of more header room */ +}; + +struct sk_buff; +struct net_device; +struct adapter; +struct sge_params; +struct sge; + +struct sge *t1_sge_create(struct adapter *, struct sge_params *); +int t1_sge_configure(struct sge *, struct sge_params *); +int t1_sge_set_coalesce_params(struct sge *, struct sge_params *); +void t1_sge_destroy(struct sge *); +irqreturn_t t1_interrupt_thread(int irq, void *data); +irqreturn_t t1_interrupt(int irq, void *cookie); +int t1_poll(struct napi_struct *, int); + +netdev_tx_t t1_start_xmit(struct sk_buff *skb, struct net_device *dev); +void t1_vlan_mode(struct adapter *adapter, netdev_features_t features); +void t1_sge_start(struct sge *); +void t1_sge_stop(struct sge *); +bool t1_sge_intr_error_handler(struct sge *sge); +void t1_sge_intr_enable(struct sge *); +void t1_sge_intr_disable(struct sge *); +void t1_sge_intr_clear(struct sge *); +const struct sge_intr_counts *t1_sge_get_intr_counts(const struct sge *sge); +void t1_sge_get_port_stats(const struct sge *sge, int port, struct sge_port_stats *); +unsigned int t1_sched_update_parms(struct sge *, unsigned int, unsigned int, + unsigned int); + +#endif /* _CXGB_SGE_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb/subr.c b/drivers/net/ethernet/chelsio/cxgb/subr.c new file mode 100644 index 000000000..367a9e458 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/subr.c @@ -0,0 +1,1145 @@ +// SPDX-License-Identifier: GPL-2.0-only +/***************************************************************************** + * * + * File: subr.c * + * $Revision: 1.27 $ * + * $Date: 2005/06/22 01:08:36 $ * + * Description: * + * Various subroutines (intr,pio,etc.) used by Chelsio 10G Ethernet driver. * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * + * All rights reserved. * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: Dimitrios Michailidis <dm@chelsio.com> * + * Tina Yang <tainay@chelsio.com> * + * Felix Marti <felix@chelsio.com> * + * Scott Bardone <sbardone@chelsio.com> * + * Kurt Ottaway <kottaway@chelsio.com> * + * Frank DiMambro <frank@chelsio.com> * + * * + * History: * + * * + ****************************************************************************/ + +#include "common.h" +#include "elmer0.h" +#include "regs.h" +#include "gmac.h" +#include "cphy.h" +#include "sge.h" +#include "tp.h" +#include "espi.h" + +/** + * t1_wait_op_done - wait until an operation is completed + * @adapter: the adapter performing the operation + * @reg: the register to check for completion + * @mask: a single-bit field within @reg that indicates completion + * @polarity: the value of the field when the operation is completed + * @attempts: number of check iterations + * @delay: delay in usecs between iterations + * + * Wait until an operation is completed by checking a bit in a register + * up to @attempts times. Returns %0 if the operation completes and %1 + * otherwise. + */ +static int t1_wait_op_done(adapter_t *adapter, int reg, u32 mask, int polarity, + int attempts, int delay) +{ + while (1) { + u32 val = readl(adapter->regs + reg) & mask; + + if (!!val == polarity) + return 0; + if (--attempts == 0) + return 1; + if (delay) + udelay(delay); + } +} + +#define TPI_ATTEMPTS 50 + +/* + * Write a register over the TPI interface (unlocked and locked versions). + */ +int __t1_tpi_write(adapter_t *adapter, u32 addr, u32 value) +{ + int tpi_busy; + + writel(addr, adapter->regs + A_TPI_ADDR); + writel(value, adapter->regs + A_TPI_WR_DATA); + writel(F_TPIWR, adapter->regs + A_TPI_CSR); + + tpi_busy = t1_wait_op_done(adapter, A_TPI_CSR, F_TPIRDY, 1, + TPI_ATTEMPTS, 3); + if (tpi_busy) + pr_alert("%s: TPI write to 0x%x failed\n", + adapter->name, addr); + return tpi_busy; +} + +int t1_tpi_write(adapter_t *adapter, u32 addr, u32 value) +{ + int ret; + + spin_lock(&adapter->tpi_lock); + ret = __t1_tpi_write(adapter, addr, value); + spin_unlock(&adapter->tpi_lock); + return ret; +} + +/* + * Read a register over the TPI interface (unlocked and locked versions). + */ +int __t1_tpi_read(adapter_t *adapter, u32 addr, u32 *valp) +{ + int tpi_busy; + + writel(addr, adapter->regs + A_TPI_ADDR); + writel(0, adapter->regs + A_TPI_CSR); + + tpi_busy = t1_wait_op_done(adapter, A_TPI_CSR, F_TPIRDY, 1, + TPI_ATTEMPTS, 3); + if (tpi_busy) + pr_alert("%s: TPI read from 0x%x failed\n", + adapter->name, addr); + else + *valp = readl(adapter->regs + A_TPI_RD_DATA); + return tpi_busy; +} + +int t1_tpi_read(adapter_t *adapter, u32 addr, u32 *valp) +{ + int ret; + + spin_lock(&adapter->tpi_lock); + ret = __t1_tpi_read(adapter, addr, valp); + spin_unlock(&adapter->tpi_lock); + return ret; +} + +/* + * Set a TPI parameter. + */ +static void t1_tpi_par(adapter_t *adapter, u32 value) +{ + writel(V_TPIPAR(value), adapter->regs + A_TPI_PAR); +} + +/* + * Called when a port's link settings change to propagate the new values to the + * associated PHY and MAC. After performing the common tasks it invokes an + * OS-specific handler. + */ +void t1_link_changed(adapter_t *adapter, int port_id) +{ + int link_ok, speed, duplex, fc; + struct cphy *phy = adapter->port[port_id].phy; + struct link_config *lc = &adapter->port[port_id].link_config; + + phy->ops->get_link_status(phy, &link_ok, &speed, &duplex, &fc); + + lc->speed = speed < 0 ? SPEED_INVALID : speed; + lc->duplex = duplex < 0 ? DUPLEX_INVALID : duplex; + if (!(lc->requested_fc & PAUSE_AUTONEG)) + fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); + + if (link_ok && speed >= 0 && lc->autoneg == AUTONEG_ENABLE) { + /* Set MAC speed, duplex, and flow control to match PHY. */ + struct cmac *mac = adapter->port[port_id].mac; + + mac->ops->set_speed_duplex_fc(mac, speed, duplex, fc); + lc->fc = (unsigned char)fc; + } + t1_link_negotiated(adapter, port_id, link_ok, speed, duplex, fc); +} + +static bool t1_pci_intr_handler(adapter_t *adapter) +{ + u32 pcix_cause; + + pci_read_config_dword(adapter->pdev, A_PCICFG_INTR_CAUSE, &pcix_cause); + + if (pcix_cause) { + pci_write_config_dword(adapter->pdev, A_PCICFG_INTR_CAUSE, + pcix_cause); + /* PCI errors are fatal */ + t1_interrupts_disable(adapter); + adapter->pending_thread_intr |= F_PL_INTR_SGE_ERR; + pr_alert("%s: PCI error encountered.\n", adapter->name); + return true; + } + return false; +} + +#ifdef CONFIG_CHELSIO_T1_1G +#include "fpga_defs.h" + +/* + * PHY interrupt handler for FPGA boards. + */ +static int fpga_phy_intr_handler(adapter_t *adapter) +{ + int p; + u32 cause = readl(adapter->regs + FPGA_GMAC_ADDR_INTERRUPT_CAUSE); + + for_each_port(adapter, p) + if (cause & (1 << p)) { + struct cphy *phy = adapter->port[p].phy; + int phy_cause = phy->ops->interrupt_handler(phy); + + if (phy_cause & cphy_cause_link_change) + t1_link_changed(adapter, p); + } + writel(cause, adapter->regs + FPGA_GMAC_ADDR_INTERRUPT_CAUSE); + return 0; +} + +/* + * Slow path interrupt handler for FPGAs. + */ +static irqreturn_t fpga_slow_intr(adapter_t *adapter) +{ + u32 cause = readl(adapter->regs + A_PL_CAUSE); + irqreturn_t ret = IRQ_NONE; + + cause &= ~F_PL_INTR_SGE_DATA; + if (cause & F_PL_INTR_SGE_ERR) { + if (t1_sge_intr_error_handler(adapter->sge)) + ret = IRQ_WAKE_THREAD; + } + + if (cause & FPGA_PCIX_INTERRUPT_GMAC) + fpga_phy_intr_handler(adapter); + + if (cause & FPGA_PCIX_INTERRUPT_TP) { + /* + * FPGA doesn't support MC4 interrupts and it requires + * this odd layer of indirection for MC5. + */ + u32 tp_cause = readl(adapter->regs + FPGA_TP_ADDR_INTERRUPT_CAUSE); + + /* Clear TP interrupt */ + writel(tp_cause, adapter->regs + FPGA_TP_ADDR_INTERRUPT_CAUSE); + } + if (cause & FPGA_PCIX_INTERRUPT_PCIX) { + if (t1_pci_intr_handler(adapter)) + ret = IRQ_WAKE_THREAD; + } + + /* Clear the interrupts just processed. */ + if (cause) + writel(cause, adapter->regs + A_PL_CAUSE); + + if (ret != IRQ_NONE) + return ret; + + return cause == 0 ? IRQ_NONE : IRQ_HANDLED; +} +#endif + +/* + * Wait until Elmer's MI1 interface is ready for new operations. + */ +static int mi1_wait_until_ready(adapter_t *adapter, int mi1_reg) +{ + int attempts = 100, busy; + + do { + u32 val; + + __t1_tpi_read(adapter, mi1_reg, &val); + busy = val & F_MI1_OP_BUSY; + if (busy) + udelay(10); + } while (busy && --attempts); + if (busy) + pr_alert("%s: MDIO operation timed out\n", adapter->name); + return busy; +} + +/* + * MI1 MDIO initialization. + */ +static void mi1_mdio_init(adapter_t *adapter, const struct board_info *bi) +{ + u32 clkdiv = bi->clock_elmer0 / (2 * bi->mdio_mdc) - 1; + u32 val = F_MI1_PREAMBLE_ENABLE | V_MI1_MDI_INVERT(bi->mdio_mdiinv) | + V_MI1_MDI_ENABLE(bi->mdio_mdien) | V_MI1_CLK_DIV(clkdiv); + + if (!(bi->caps & SUPPORTED_10000baseT_Full)) + val |= V_MI1_SOF(1); + t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_CFG, val); +} + +#if defined(CONFIG_CHELSIO_T1_1G) +/* + * Elmer MI1 MDIO read/write operations. + */ +static int mi1_mdio_read(struct net_device *dev, int phy_addr, int mmd_addr, + u16 reg_addr) +{ + struct adapter *adapter = dev->ml_priv; + u32 addr = V_MI1_REG_ADDR(reg_addr) | V_MI1_PHY_ADDR(phy_addr); + unsigned int val; + + spin_lock(&adapter->tpi_lock); + __t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_ADDR, addr); + __t1_tpi_write(adapter, + A_ELMER0_PORT0_MI1_OP, MI1_OP_DIRECT_READ); + mi1_wait_until_ready(adapter, A_ELMER0_PORT0_MI1_OP); + __t1_tpi_read(adapter, A_ELMER0_PORT0_MI1_DATA, &val); + spin_unlock(&adapter->tpi_lock); + return val; +} + +static int mi1_mdio_write(struct net_device *dev, int phy_addr, int mmd_addr, + u16 reg_addr, u16 val) +{ + struct adapter *adapter = dev->ml_priv; + u32 addr = V_MI1_REG_ADDR(reg_addr) | V_MI1_PHY_ADDR(phy_addr); + + spin_lock(&adapter->tpi_lock); + __t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_ADDR, addr); + __t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_DATA, val); + __t1_tpi_write(adapter, + A_ELMER0_PORT0_MI1_OP, MI1_OP_DIRECT_WRITE); + mi1_wait_until_ready(adapter, A_ELMER0_PORT0_MI1_OP); + spin_unlock(&adapter->tpi_lock); + return 0; +} + +static const struct mdio_ops mi1_mdio_ops = { + .init = mi1_mdio_init, + .read = mi1_mdio_read, + .write = mi1_mdio_write, + .mode_support = MDIO_SUPPORTS_C22 +}; + +#endif + +static int mi1_mdio_ext_read(struct net_device *dev, int phy_addr, int mmd_addr, + u16 reg_addr) +{ + struct adapter *adapter = dev->ml_priv; + u32 addr = V_MI1_REG_ADDR(mmd_addr) | V_MI1_PHY_ADDR(phy_addr); + unsigned int val; + + spin_lock(&adapter->tpi_lock); + + /* Write the address we want. */ + __t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_ADDR, addr); + __t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_DATA, reg_addr); + __t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_OP, + MI1_OP_INDIRECT_ADDRESS); + mi1_wait_until_ready(adapter, A_ELMER0_PORT0_MI1_OP); + + /* Write the operation we want. */ + __t1_tpi_write(adapter, + A_ELMER0_PORT0_MI1_OP, MI1_OP_INDIRECT_READ); + mi1_wait_until_ready(adapter, A_ELMER0_PORT0_MI1_OP); + + /* Read the data. */ + __t1_tpi_read(adapter, A_ELMER0_PORT0_MI1_DATA, &val); + spin_unlock(&adapter->tpi_lock); + return val; +} + +static int mi1_mdio_ext_write(struct net_device *dev, int phy_addr, + int mmd_addr, u16 reg_addr, u16 val) +{ + struct adapter *adapter = dev->ml_priv; + u32 addr = V_MI1_REG_ADDR(mmd_addr) | V_MI1_PHY_ADDR(phy_addr); + + spin_lock(&adapter->tpi_lock); + + /* Write the address we want. */ + __t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_ADDR, addr); + __t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_DATA, reg_addr); + __t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_OP, + MI1_OP_INDIRECT_ADDRESS); + mi1_wait_until_ready(adapter, A_ELMER0_PORT0_MI1_OP); + + /* Write the data. */ + __t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_DATA, val); + __t1_tpi_write(adapter, A_ELMER0_PORT0_MI1_OP, MI1_OP_INDIRECT_WRITE); + mi1_wait_until_ready(adapter, A_ELMER0_PORT0_MI1_OP); + spin_unlock(&adapter->tpi_lock); + return 0; +} + +static const struct mdio_ops mi1_mdio_ext_ops = { + .init = mi1_mdio_init, + .read = mi1_mdio_ext_read, + .write = mi1_mdio_ext_write, + .mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22 +}; + +enum { + CH_BRD_T110_1CU, + CH_BRD_N110_1F, + CH_BRD_N210_1F, + CH_BRD_T210_1F, + CH_BRD_T210_1CU, + CH_BRD_N204_4CU, +}; + +static const struct board_info t1_board[] = { + { + .board = CHBT_BOARD_CHT110, + .port_number = 1, + .caps = SUPPORTED_10000baseT_Full, + .chip_term = CHBT_TERM_T1, + .chip_mac = CHBT_MAC_PM3393, + .chip_phy = CHBT_PHY_MY3126, + .clock_core = 125000000, + .clock_mc3 = 150000000, + .clock_mc4 = 125000000, + .espi_nports = 1, + .clock_elmer0 = 44, + .mdio_mdien = 1, + .mdio_mdiinv = 1, + .mdio_mdc = 1, + .mdio_phybaseaddr = 1, + .gmac = &t1_pm3393_ops, + .gphy = &t1_my3126_ops, + .mdio_ops = &mi1_mdio_ext_ops, + .desc = "Chelsio T110 1x10GBase-CX4 TOE", + }, + + { + .board = CHBT_BOARD_N110, + .port_number = 1, + .caps = SUPPORTED_10000baseT_Full | SUPPORTED_FIBRE, + .chip_term = CHBT_TERM_T1, + .chip_mac = CHBT_MAC_PM3393, + .chip_phy = CHBT_PHY_88X2010, + .clock_core = 125000000, + .espi_nports = 1, + .clock_elmer0 = 44, + .mdio_mdien = 0, + .mdio_mdiinv = 0, + .mdio_mdc = 1, + .mdio_phybaseaddr = 0, + .gmac = &t1_pm3393_ops, + .gphy = &t1_mv88x201x_ops, + .mdio_ops = &mi1_mdio_ext_ops, + .desc = "Chelsio N110 1x10GBaseX NIC", + }, + + { + .board = CHBT_BOARD_N210, + .port_number = 1, + .caps = SUPPORTED_10000baseT_Full | SUPPORTED_FIBRE, + .chip_term = CHBT_TERM_T2, + .chip_mac = CHBT_MAC_PM3393, + .chip_phy = CHBT_PHY_88X2010, + .clock_core = 125000000, + .espi_nports = 1, + .clock_elmer0 = 44, + .mdio_mdien = 0, + .mdio_mdiinv = 0, + .mdio_mdc = 1, + .mdio_phybaseaddr = 0, + .gmac = &t1_pm3393_ops, + .gphy = &t1_mv88x201x_ops, + .mdio_ops = &mi1_mdio_ext_ops, + .desc = "Chelsio N210 1x10GBaseX NIC", + }, + + { + .board = CHBT_BOARD_CHT210, + .port_number = 1, + .caps = SUPPORTED_10000baseT_Full, + .chip_term = CHBT_TERM_T2, + .chip_mac = CHBT_MAC_PM3393, + .chip_phy = CHBT_PHY_88X2010, + .clock_core = 125000000, + .clock_mc3 = 133000000, + .clock_mc4 = 125000000, + .espi_nports = 1, + .clock_elmer0 = 44, + .mdio_mdien = 0, + .mdio_mdiinv = 0, + .mdio_mdc = 1, + .mdio_phybaseaddr = 0, + .gmac = &t1_pm3393_ops, + .gphy = &t1_mv88x201x_ops, + .mdio_ops = &mi1_mdio_ext_ops, + .desc = "Chelsio T210 1x10GBaseX TOE", + }, + + { + .board = CHBT_BOARD_CHT210, + .port_number = 1, + .caps = SUPPORTED_10000baseT_Full, + .chip_term = CHBT_TERM_T2, + .chip_mac = CHBT_MAC_PM3393, + .chip_phy = CHBT_PHY_MY3126, + .clock_core = 125000000, + .clock_mc3 = 133000000, + .clock_mc4 = 125000000, + .espi_nports = 1, + .clock_elmer0 = 44, + .mdio_mdien = 1, + .mdio_mdiinv = 1, + .mdio_mdc = 1, + .mdio_phybaseaddr = 1, + .gmac = &t1_pm3393_ops, + .gphy = &t1_my3126_ops, + .mdio_ops = &mi1_mdio_ext_ops, + .desc = "Chelsio T210 1x10GBase-CX4 TOE", + }, + +#ifdef CONFIG_CHELSIO_T1_1G + { + .board = CHBT_BOARD_CHN204, + .port_number = 4, + .caps = SUPPORTED_10baseT_Half | SUPPORTED_10baseT_Full + | SUPPORTED_100baseT_Half | SUPPORTED_100baseT_Full + | SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | + SUPPORTED_PAUSE | SUPPORTED_TP, + .chip_term = CHBT_TERM_T2, + .chip_mac = CHBT_MAC_VSC7321, + .chip_phy = CHBT_PHY_88E1111, + .clock_core = 100000000, + .espi_nports = 4, + .clock_elmer0 = 44, + .mdio_mdien = 0, + .mdio_mdiinv = 0, + .mdio_mdc = 0, + .mdio_phybaseaddr = 4, + .gmac = &t1_vsc7326_ops, + .gphy = &t1_mv88e1xxx_ops, + .mdio_ops = &mi1_mdio_ops, + .desc = "Chelsio N204 4x100/1000BaseT NIC", + }, +#endif + +}; + +const struct pci_device_id t1_pci_tbl[] = { + CH_DEVICE(8, 0, CH_BRD_T110_1CU), + CH_DEVICE(8, 1, CH_BRD_T110_1CU), + CH_DEVICE(7, 0, CH_BRD_N110_1F), + CH_DEVICE(10, 1, CH_BRD_N210_1F), + CH_DEVICE(11, 1, CH_BRD_T210_1F), + CH_DEVICE(14, 1, CH_BRD_T210_1CU), + CH_DEVICE(16, 1, CH_BRD_N204_4CU), + { 0 } +}; + +MODULE_DEVICE_TABLE(pci, t1_pci_tbl); + +/* + * Return the board_info structure with a given index. Out-of-range indices + * return NULL. + */ +const struct board_info *t1_get_board_info(unsigned int board_id) +{ + return board_id < ARRAY_SIZE(t1_board) ? &t1_board[board_id] : NULL; +} + +struct chelsio_vpd_t { + u32 format_version; + u8 serial_number[16]; + u8 mac_base_address[6]; + u8 pad[2]; /* make multiple-of-4 size requirement explicit */ +}; + +#define EEPROMSIZE (8 * 1024) +#define EEPROM_MAX_POLL 4 + +/* + * Read SEEPROM. A zero is written to the flag register when the address is + * written to the Control register. The hardware device will set the flag to a + * one when 4B have been transferred to the Data register. + */ +int t1_seeprom_read(adapter_t *adapter, u32 addr, __le32 *data) +{ + int i = EEPROM_MAX_POLL; + u16 val; + u32 v; + + if (addr >= EEPROMSIZE || (addr & 3)) + return -EINVAL; + + pci_write_config_word(adapter->pdev, A_PCICFG_VPD_ADDR, (u16)addr); + do { + udelay(50); + pci_read_config_word(adapter->pdev, A_PCICFG_VPD_ADDR, &val); + } while (!(val & F_VPD_OP_FLAG) && --i); + + if (!(val & F_VPD_OP_FLAG)) { + pr_err("%s: reading EEPROM address 0x%x failed\n", + adapter->name, addr); + return -EIO; + } + pci_read_config_dword(adapter->pdev, A_PCICFG_VPD_DATA, &v); + *data = cpu_to_le32(v); + return 0; +} + +static int t1_eeprom_vpd_get(adapter_t *adapter, struct chelsio_vpd_t *vpd) +{ + int addr, ret = 0; + + for (addr = 0; !ret && addr < sizeof(*vpd); addr += sizeof(u32)) + ret = t1_seeprom_read(adapter, addr, + (__le32 *)((u8 *)vpd + addr)); + + return ret; +} + +/* + * Read a port's MAC address from the VPD ROM. + */ +static int vpd_macaddress_get(adapter_t *adapter, int index, u8 mac_addr[]) +{ + struct chelsio_vpd_t vpd; + + if (t1_eeprom_vpd_get(adapter, &vpd)) + return 1; + memcpy(mac_addr, vpd.mac_base_address, 5); + mac_addr[5] = vpd.mac_base_address[5] + index; + return 0; +} + +/* + * Set up the MAC/PHY according to the requested link settings. + * + * If the PHY can auto-negotiate first decide what to advertise, then + * enable/disable auto-negotiation as desired and reset. + * + * If the PHY does not auto-negotiate we just reset it. + * + * If auto-negotiation is off set the MAC to the proper speed/duplex/FC, + * otherwise do it later based on the outcome of auto-negotiation. + */ +int t1_link_start(struct cphy *phy, struct cmac *mac, struct link_config *lc) +{ + unsigned int fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); + + if (lc->supported & SUPPORTED_Autoneg) { + lc->advertising &= ~(ADVERTISED_ASYM_PAUSE | ADVERTISED_PAUSE); + if (fc) { + if (fc == ((PAUSE_RX | PAUSE_TX) & + (mac->adapter->params.nports < 2))) + lc->advertising |= ADVERTISED_PAUSE; + else { + lc->advertising |= ADVERTISED_ASYM_PAUSE; + if (fc == PAUSE_RX) + lc->advertising |= ADVERTISED_PAUSE; + } + } + phy->ops->advertise(phy, lc->advertising); + + if (lc->autoneg == AUTONEG_DISABLE) { + lc->speed = lc->requested_speed; + lc->duplex = lc->requested_duplex; + lc->fc = (unsigned char)fc; + mac->ops->set_speed_duplex_fc(mac, lc->speed, + lc->duplex, fc); + /* Also disables autoneg */ + phy->state = PHY_AUTONEG_RDY; + phy->ops->set_speed_duplex(phy, lc->speed, lc->duplex); + phy->ops->reset(phy, 0); + } else { + phy->state = PHY_AUTONEG_EN; + phy->ops->autoneg_enable(phy); /* also resets PHY */ + } + } else { + phy->state = PHY_AUTONEG_RDY; + mac->ops->set_speed_duplex_fc(mac, -1, -1, fc); + lc->fc = (unsigned char)fc; + phy->ops->reset(phy, 0); + } + return 0; +} + +/* + * External interrupt handler for boards using elmer0. + */ +int t1_elmer0_ext_intr_handler(adapter_t *adapter) +{ + struct cphy *phy; + int phy_cause; + u32 cause; + + t1_tpi_read(adapter, A_ELMER0_INT_CAUSE, &cause); + + switch (board_info(adapter)->board) { +#ifdef CONFIG_CHELSIO_T1_1G + case CHBT_BOARD_CHT204: + case CHBT_BOARD_CHT204E: + case CHBT_BOARD_CHN204: + case CHBT_BOARD_CHT204V: { + int i, port_bit; + for_each_port(adapter, i) { + port_bit = i + 1; + if (!(cause & (1 << port_bit))) + continue; + + phy = adapter->port[i].phy; + phy_cause = phy->ops->interrupt_handler(phy); + if (phy_cause & cphy_cause_link_change) + t1_link_changed(adapter, i); + } + break; + } + case CHBT_BOARD_CHT101: + if (cause & ELMER0_GP_BIT1) { /* Marvell 88E1111 interrupt */ + phy = adapter->port[0].phy; + phy_cause = phy->ops->interrupt_handler(phy); + if (phy_cause & cphy_cause_link_change) + t1_link_changed(adapter, 0); + } + break; + case CHBT_BOARD_7500: { + int p; + /* + * Elmer0's interrupt cause isn't useful here because there is + * only one bit that can be set for all 4 ports. This means + * we are forced to check every PHY's interrupt status + * register to see who initiated the interrupt. + */ + for_each_port(adapter, p) { + phy = adapter->port[p].phy; + phy_cause = phy->ops->interrupt_handler(phy); + if (phy_cause & cphy_cause_link_change) + t1_link_changed(adapter, p); + } + break; + } +#endif + case CHBT_BOARD_CHT210: + case CHBT_BOARD_N210: + case CHBT_BOARD_N110: + if (cause & ELMER0_GP_BIT6) { /* Marvell 88x2010 interrupt */ + phy = adapter->port[0].phy; + phy_cause = phy->ops->interrupt_handler(phy); + if (phy_cause & cphy_cause_link_change) + t1_link_changed(adapter, 0); + } + break; + case CHBT_BOARD_8000: + case CHBT_BOARD_CHT110: + if (netif_msg_intr(adapter)) + dev_dbg(&adapter->pdev->dev, + "External interrupt cause 0x%x\n", cause); + if (cause & ELMER0_GP_BIT1) { /* PMC3393 INTB */ + struct cmac *mac = adapter->port[0].mac; + + mac->ops->interrupt_handler(mac); + } + if (cause & ELMER0_GP_BIT5) { /* XPAK MOD_DETECT */ + u32 mod_detect; + + t1_tpi_read(adapter, + A_ELMER0_GPI_STAT, &mod_detect); + if (netif_msg_link(adapter)) + dev_info(&adapter->pdev->dev, "XPAK %s\n", + mod_detect ? "removed" : "inserted"); + } + break; + } + t1_tpi_write(adapter, A_ELMER0_INT_CAUSE, cause); + return 0; +} + +/* Enables all interrupts. */ +void t1_interrupts_enable(adapter_t *adapter) +{ + unsigned int i; + + adapter->slow_intr_mask = F_PL_INTR_SGE_ERR | F_PL_INTR_TP; + + t1_sge_intr_enable(adapter->sge); + t1_tp_intr_enable(adapter->tp); + if (adapter->espi) { + adapter->slow_intr_mask |= F_PL_INTR_ESPI; + t1_espi_intr_enable(adapter->espi); + } + + /* Enable MAC/PHY interrupts for each port. */ + for_each_port(adapter, i) { + adapter->port[i].mac->ops->interrupt_enable(adapter->port[i].mac); + adapter->port[i].phy->ops->interrupt_enable(adapter->port[i].phy); + } + + /* Enable PCIX & external chip interrupts on ASIC boards. */ + if (t1_is_asic(adapter)) { + u32 pl_intr = readl(adapter->regs + A_PL_ENABLE); + + /* PCI-X interrupts */ + pci_write_config_dword(adapter->pdev, A_PCICFG_INTR_ENABLE, + 0xffffffff); + + adapter->slow_intr_mask |= F_PL_INTR_EXT | F_PL_INTR_PCIX; + pl_intr |= F_PL_INTR_EXT | F_PL_INTR_PCIX; + writel(pl_intr, adapter->regs + A_PL_ENABLE); + } +} + +/* Disables all interrupts. */ +void t1_interrupts_disable(adapter_t* adapter) +{ + unsigned int i; + + t1_sge_intr_disable(adapter->sge); + t1_tp_intr_disable(adapter->tp); + if (adapter->espi) + t1_espi_intr_disable(adapter->espi); + + /* Disable MAC/PHY interrupts for each port. */ + for_each_port(adapter, i) { + adapter->port[i].mac->ops->interrupt_disable(adapter->port[i].mac); + adapter->port[i].phy->ops->interrupt_disable(adapter->port[i].phy); + } + + /* Disable PCIX & external chip interrupts. */ + if (t1_is_asic(adapter)) + writel(0, adapter->regs + A_PL_ENABLE); + + /* PCI-X interrupts */ + pci_write_config_dword(adapter->pdev, A_PCICFG_INTR_ENABLE, 0); + + adapter->slow_intr_mask = 0; +} + +/* Clears all interrupts */ +void t1_interrupts_clear(adapter_t* adapter) +{ + unsigned int i; + + t1_sge_intr_clear(adapter->sge); + t1_tp_intr_clear(adapter->tp); + if (adapter->espi) + t1_espi_intr_clear(adapter->espi); + + /* Clear MAC/PHY interrupts for each port. */ + for_each_port(adapter, i) { + adapter->port[i].mac->ops->interrupt_clear(adapter->port[i].mac); + adapter->port[i].phy->ops->interrupt_clear(adapter->port[i].phy); + } + + /* Enable interrupts for external devices. */ + if (t1_is_asic(adapter)) { + u32 pl_intr = readl(adapter->regs + A_PL_CAUSE); + + writel(pl_intr | F_PL_INTR_EXT | F_PL_INTR_PCIX, + adapter->regs + A_PL_CAUSE); + } + + /* PCI-X interrupts */ + pci_write_config_dword(adapter->pdev, A_PCICFG_INTR_CAUSE, 0xffffffff); +} + +/* + * Slow path interrupt handler for ASICs. + */ +static irqreturn_t asic_slow_intr(adapter_t *adapter) +{ + u32 cause = readl(adapter->regs + A_PL_CAUSE); + irqreturn_t ret = IRQ_HANDLED; + + cause &= adapter->slow_intr_mask; + if (!cause) + return IRQ_NONE; + if (cause & F_PL_INTR_SGE_ERR) { + if (t1_sge_intr_error_handler(adapter->sge)) + ret = IRQ_WAKE_THREAD; + } + if (cause & F_PL_INTR_TP) + t1_tp_intr_handler(adapter->tp); + if (cause & F_PL_INTR_ESPI) + t1_espi_intr_handler(adapter->espi); + if (cause & F_PL_INTR_PCIX) { + if (t1_pci_intr_handler(adapter)) + ret = IRQ_WAKE_THREAD; + } + if (cause & F_PL_INTR_EXT) { + /* Wake the threaded interrupt to handle external interrupts as + * we require a process context. We disable EXT interrupts in + * the interim and let the thread reenable them when it's done. + */ + adapter->pending_thread_intr |= F_PL_INTR_EXT; + adapter->slow_intr_mask &= ~F_PL_INTR_EXT; + writel(adapter->slow_intr_mask | F_PL_INTR_SGE_DATA, + adapter->regs + A_PL_ENABLE); + ret = IRQ_WAKE_THREAD; + } + + /* Clear the interrupts just processed. */ + writel(cause, adapter->regs + A_PL_CAUSE); + readl(adapter->regs + A_PL_CAUSE); /* flush writes */ + return ret; +} + +irqreturn_t t1_slow_intr_handler(adapter_t *adapter) +{ +#ifdef CONFIG_CHELSIO_T1_1G + if (!t1_is_asic(adapter)) + return fpga_slow_intr(adapter); +#endif + return asic_slow_intr(adapter); +} + +/* Power sequencing is a work-around for Intel's XPAKs. */ +static void power_sequence_xpak(adapter_t* adapter) +{ + u32 mod_detect; + u32 gpo; + + /* Check for XPAK */ + t1_tpi_read(adapter, A_ELMER0_GPI_STAT, &mod_detect); + if (!(ELMER0_GP_BIT5 & mod_detect)) { + /* XPAK is present */ + t1_tpi_read(adapter, A_ELMER0_GPO, &gpo); + gpo |= ELMER0_GP_BIT18; + t1_tpi_write(adapter, A_ELMER0_GPO, gpo); + } +} + +int t1_get_board_rev(adapter_t *adapter, const struct board_info *bi, + struct adapter_params *p) +{ + p->chip_version = bi->chip_term; + p->is_asic = (p->chip_version != CHBT_TERM_FPGA); + if (p->chip_version == CHBT_TERM_T1 || + p->chip_version == CHBT_TERM_T2 || + p->chip_version == CHBT_TERM_FPGA) { + u32 val = readl(adapter->regs + A_TP_PC_CONFIG); + + val = G_TP_PC_REV(val); + if (val == 2) + p->chip_revision = TERM_T1B; + else if (val == 3) + p->chip_revision = TERM_T2; + else + return -1; + } else + return -1; + return 0; +} + +/* + * Enable board components other than the Chelsio chip, such as external MAC + * and PHY. + */ +static int board_init(adapter_t *adapter, const struct board_info *bi) +{ + switch (bi->board) { + case CHBT_BOARD_8000: + case CHBT_BOARD_N110: + case CHBT_BOARD_N210: + case CHBT_BOARD_CHT210: + t1_tpi_par(adapter, 0xf); + t1_tpi_write(adapter, A_ELMER0_GPO, 0x800); + break; + case CHBT_BOARD_CHT110: + t1_tpi_par(adapter, 0xf); + t1_tpi_write(adapter, A_ELMER0_GPO, 0x1800); + + /* TBD XXX Might not need. This fixes a problem + * described in the Intel SR XPAK errata. + */ + power_sequence_xpak(adapter); + break; +#ifdef CONFIG_CHELSIO_T1_1G + case CHBT_BOARD_CHT204E: + /* add config space write here */ + case CHBT_BOARD_CHT204: + case CHBT_BOARD_CHT204V: + case CHBT_BOARD_CHN204: + t1_tpi_par(adapter, 0xf); + t1_tpi_write(adapter, A_ELMER0_GPO, 0x804); + break; + case CHBT_BOARD_CHT101: + case CHBT_BOARD_7500: + t1_tpi_par(adapter, 0xf); + t1_tpi_write(adapter, A_ELMER0_GPO, 0x1804); + break; +#endif + } + return 0; +} + +/* + * Initialize and configure the Terminator HW modules. Note that external + * MAC and PHYs are initialized separately. + */ +int t1_init_hw_modules(adapter_t *adapter) +{ + int err = -EIO; + const struct board_info *bi = board_info(adapter); + + if (!bi->clock_mc4) { + u32 val = readl(adapter->regs + A_MC4_CFG); + + writel(val | F_READY | F_MC4_SLOW, adapter->regs + A_MC4_CFG); + writel(F_M_BUS_ENABLE | F_TCAM_RESET, + adapter->regs + A_MC5_CONFIG); + } + + if (adapter->espi && t1_espi_init(adapter->espi, bi->chip_mac, + bi->espi_nports)) + goto out_err; + + if (t1_tp_reset(adapter->tp, &adapter->params.tp, bi->clock_core)) + goto out_err; + + err = t1_sge_configure(adapter->sge, &adapter->params.sge); + if (err) + goto out_err; + + err = 0; +out_err: + return err; +} + +/* + * Determine a card's PCI mode. + */ +static void get_pci_mode(adapter_t *adapter, struct chelsio_pci_params *p) +{ + static const unsigned short speed_map[] = { 33, 66, 100, 133 }; + u32 pci_mode; + + pci_read_config_dword(adapter->pdev, A_PCICFG_MODE, &pci_mode); + p->speed = speed_map[G_PCI_MODE_CLK(pci_mode)]; + p->width = (pci_mode & F_PCI_MODE_64BIT) ? 64 : 32; + p->is_pcix = (pci_mode & F_PCI_MODE_PCIX) != 0; +} + +/* + * Release the structures holding the SW per-Terminator-HW-module state. + */ +void t1_free_sw_modules(adapter_t *adapter) +{ + unsigned int i; + + for_each_port(adapter, i) { + struct cmac *mac = adapter->port[i].mac; + struct cphy *phy = adapter->port[i].phy; + + if (mac) + mac->ops->destroy(mac); + if (phy) + phy->ops->destroy(phy); + } + + if (adapter->sge) + t1_sge_destroy(adapter->sge); + if (adapter->tp) + t1_tp_destroy(adapter->tp); + if (adapter->espi) + t1_espi_destroy(adapter->espi); +} + +static void init_link_config(struct link_config *lc, + const struct board_info *bi) +{ + lc->supported = bi->caps; + lc->requested_speed = lc->speed = SPEED_INVALID; + lc->requested_duplex = lc->duplex = DUPLEX_INVALID; + lc->requested_fc = lc->fc = PAUSE_RX | PAUSE_TX; + if (lc->supported & SUPPORTED_Autoneg) { + lc->advertising = lc->supported; + lc->autoneg = AUTONEG_ENABLE; + lc->requested_fc |= PAUSE_AUTONEG; + } else { + lc->advertising = 0; + lc->autoneg = AUTONEG_DISABLE; + } +} + +/* + * Allocate and initialize the data structures that hold the SW state of + * the Terminator HW modules. + */ +int t1_init_sw_modules(adapter_t *adapter, const struct board_info *bi) +{ + unsigned int i; + + adapter->params.brd_info = bi; + adapter->params.nports = bi->port_number; + adapter->params.stats_update_period = bi->gmac->stats_update_period; + + adapter->sge = t1_sge_create(adapter, &adapter->params.sge); + if (!adapter->sge) { + pr_err("%s: SGE initialization failed\n", + adapter->name); + goto error; + } + + if (bi->espi_nports && !(adapter->espi = t1_espi_create(adapter))) { + pr_err("%s: ESPI initialization failed\n", + adapter->name); + goto error; + } + + adapter->tp = t1_tp_create(adapter, &adapter->params.tp); + if (!adapter->tp) { + pr_err("%s: TP initialization failed\n", + adapter->name); + goto error; + } + + board_init(adapter, bi); + bi->mdio_ops->init(adapter, bi); + if (bi->gphy->reset) + bi->gphy->reset(adapter); + if (bi->gmac->reset) + bi->gmac->reset(adapter); + + for_each_port(adapter, i) { + u8 hw_addr[6]; + struct cmac *mac; + int phy_addr = bi->mdio_phybaseaddr + i; + + adapter->port[i].phy = bi->gphy->create(adapter->port[i].dev, + phy_addr, bi->mdio_ops); + if (!adapter->port[i].phy) { + pr_err("%s: PHY %d initialization failed\n", + adapter->name, i); + goto error; + } + + adapter->port[i].mac = mac = bi->gmac->create(adapter, i); + if (!mac) { + pr_err("%s: MAC %d initialization failed\n", + adapter->name, i); + goto error; + } + + /* + * Get the port's MAC addresses either from the EEPROM if one + * exists or the one hardcoded in the MAC. + */ + if (!t1_is_asic(adapter) || bi->chip_mac == CHBT_MAC_DUMMY) + mac->ops->macaddress_get(mac, hw_addr); + else if (vpd_macaddress_get(adapter, i, hw_addr)) { + pr_err("%s: could not read MAC address from VPD ROM\n", + adapter->port[i].dev->name); + goto error; + } + eth_hw_addr_set(adapter->port[i].dev, hw_addr); + init_link_config(&adapter->port[i].link_config, bi); + } + + get_pci_mode(adapter, &adapter->params.pci); + t1_interrupts_clear(adapter); + return 0; + +error: + t1_free_sw_modules(adapter); + return -1; +} diff --git a/drivers/net/ethernet/chelsio/cxgb/suni1x10gexp_regs.h b/drivers/net/ethernet/chelsio/cxgb/suni1x10gexp_regs.h new file mode 100644 index 000000000..4c8831706 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/suni1x10gexp_regs.h @@ -0,0 +1,1632 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/***************************************************************************** + * * + * File: suni1x10gexp_regs.h * + * $Revision: 1.9 $ * + * $Date: 2005/06/22 00:17:04 $ * + * Description: * + * PMC/SIERRA (pm3393) MAC-PHY functionality. * + * part of the Chelsio 10Gb Ethernet Driver. * + * * + * * + * http://www.chelsio.com * + * * + * Maintainers: maintainers@chelsio.com * + * * + * Authors: PMC/SIERRA * + * * + * History: * + * * + ****************************************************************************/ + +#ifndef _CXGB_SUNI1x10GEXP_REGS_H_ +#define _CXGB_SUNI1x10GEXP_REGS_H_ + +/* +** Space allocated for each Exact Match Filter +** There are 8 filter configurations +*/ +#define SUNI1x10GEXP_REG_SIZEOF_MAC_FILTER 0x0003 + +#define mSUNI1x10GEXP_MAC_FILTER_OFFSET(filterId) ( (filterId) * SUNI1x10GEXP_REG_SIZEOF_MAC_FILTER ) + +/* +** Space allocated for VLAN-Id Filter +** There are 8 filter configurations +*/ +#define SUNI1x10GEXP_REG_SIZEOF_MAC_VID_FILTER 0x0001 + +#define mSUNI1x10GEXP_MAC_VID_FILTER_OFFSET(filterId) ( (filterId) * SUNI1x10GEXP_REG_SIZEOF_MAC_VID_FILTER ) + +/* +** Space allocated for each MSTAT Counter +*/ +#define SUNI1x10GEXP_REG_SIZEOF_MSTAT_COUNT 0x0004 + +#define mSUNI1x10GEXP_MSTAT_COUNT_OFFSET(countId) ( (countId) * SUNI1x10GEXP_REG_SIZEOF_MSTAT_COUNT ) + + +/******************************************************************************/ +/** S/UNI-1x10GE-XP REGISTER ADDRESS MAP **/ +/******************************************************************************/ +/* Refer to the Register Bit Masks bellow for the naming of each register and */ +/* to the S/UNI-1x10GE-XP Data Sheet for the signification of each bit */ +/******************************************************************************/ + + +#define SUNI1x10GEXP_REG_IDENTIFICATION 0x0000 +#define SUNI1x10GEXP_REG_PRODUCT_REVISION 0x0001 +#define SUNI1x10GEXP_REG_CONFIG_AND_RESET_CONTROL 0x0002 +#define SUNI1x10GEXP_REG_LOOPBACK_MISC_CTRL 0x0003 +#define SUNI1x10GEXP_REG_DEVICE_STATUS 0x0004 +#define SUNI1x10GEXP_REG_GLOBAL_PERFORMANCE_MONITOR_UPDATE 0x0005 + +#define SUNI1x10GEXP_REG_MDIO_COMMAND 0x0006 +#define SUNI1x10GEXP_REG_MDIO_INTERRUPT_ENABLE 0x0007 +#define SUNI1x10GEXP_REG_MDIO_INTERRUPT_STATUS 0x0008 +#define SUNI1x10GEXP_REG_MMD_PHY_ADDRESS 0x0009 +#define SUNI1x10GEXP_REG_MMD_CONTROL_ADDRESS_DATA 0x000A +#define SUNI1x10GEXP_REG_MDIO_READ_STATUS_DATA 0x000B + +#define SUNI1x10GEXP_REG_OAM_INTF_CTRL 0x000C +#define SUNI1x10GEXP_REG_MASTER_INTERRUPT_STATUS 0x000D +#define SUNI1x10GEXP_REG_GLOBAL_INTERRUPT_ENABLE 0x000E +#define SUNI1x10GEXP_REG_FREE 0x000F + +#define SUNI1x10GEXP_REG_XTEF_MISC_CTRL 0x0010 +#define SUNI1x10GEXP_REG_XRF_MISC_CTRL 0x0011 + +#define SUNI1x10GEXP_REG_SERDES_3125_CONFIG_1 0x0100 +#define SUNI1x10GEXP_REG_SERDES_3125_CONFIG_2 0x0101 +#define SUNI1x10GEXP_REG_SERDES_3125_INTERRUPT_ENABLE 0x0102 +#define SUNI1x10GEXP_REG_SERDES_3125_INTERRUPT_VISIBLE 0x0103 +#define SUNI1x10GEXP_REG_SERDES_3125_INTERRUPT_STATUS 0x0104 +#define SUNI1x10GEXP_REG_SERDES_3125_TEST_CONFIG 0x0107 + +#define SUNI1x10GEXP_REG_RXXG_CONFIG_1 0x2040 +#define SUNI1x10GEXP_REG_RXXG_CONFIG_2 0x2041 +#define SUNI1x10GEXP_REG_RXXG_CONFIG_3 0x2042 +#define SUNI1x10GEXP_REG_RXXG_INTERRUPT 0x2043 +#define SUNI1x10GEXP_REG_RXXG_MAX_FRAME_LENGTH 0x2045 +#define SUNI1x10GEXP_REG_RXXG_SA_15_0 0x2046 +#define SUNI1x10GEXP_REG_RXXG_SA_31_16 0x2047 +#define SUNI1x10GEXP_REG_RXXG_SA_47_32 0x2048 +#define SUNI1x10GEXP_REG_RXXG_RECEIVE_FIFO_THRESHOLD 0x2049 +#define mSUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_LOW(filterId) (0x204A + mSUNI1x10GEXP_MAC_FILTER_OFFSET(filterId)) +#define mSUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_MID(filterId) (0x204B + mSUNI1x10GEXP_MAC_FILTER_OFFSET(filterId)) +#define mSUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_HIGH(filterId)(0x204C + mSUNI1x10GEXP_MAC_FILTER_OFFSET(filterId)) +#define mSUNI1x10GEXP_REG_RXXG_EXACT_MATCH_VID(filterId) (0x2062 + mSUNI1x10GEXP_MAC_VID_FILTER_OFFSET(filterId)) +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_0_LOW 0x204A +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_0_MID 0x204B +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_0_HIGH 0x204C +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_1_LOW 0x204D +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_1_MID 0x204E +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_1_HIGH 0x204F +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_2_LOW 0x2050 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_2_MID 0x2051 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_2_HIGH 0x2052 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_3_LOW 0x2053 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_3_MID 0x2054 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_3_HIGH 0x2055 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_4_LOW 0x2056 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_4_MID 0x2057 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_4_HIGH 0x2058 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_5_LOW 0x2059 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_5_MID 0x205A +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_5_HIGH 0x205B +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_6_LOW 0x205C +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_6_MID 0x205D +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_6_HIGH 0x205E +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_7_LOW 0x205F +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_7_MID 0x2060 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_ADDR_7_HIGH 0x2061 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_VID_0 0x2062 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_VID_1 0x2063 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_VID_2 0x2064 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_VID_3 0x2065 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_VID_4 0x2066 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_VID_5 0x2067 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_VID_6 0x2068 +#define SUNI1x10GEXP_REG_RXXG_EXACT_MATCH_VID_7 0x2069 +#define SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_LOW 0x206A +#define SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_MIDLOW 0x206B +#define SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_MIDHIGH 0x206C +#define SUNI1x10GEXP_REG_RXXG_MULTICAST_HASH_HIGH 0x206D +#define SUNI1x10GEXP_REG_RXXG_ADDRESS_FILTER_CONTROL_0 0x206E +#define SUNI1x10GEXP_REG_RXXG_ADDRESS_FILTER_CONTROL_1 0x206F +#define SUNI1x10GEXP_REG_RXXG_ADDRESS_FILTER_CONTROL_2 0x2070 + +#define SUNI1x10GEXP_REG_XRF_PATTERN_GEN_CTRL 0x2081 +#define SUNI1x10GEXP_REG_XRF_8BTB_ERR_COUNT_LANE_0 0x2084 +#define SUNI1x10GEXP_REG_XRF_8BTB_ERR_COUNT_LANE_1 0x2085 +#define SUNI1x10GEXP_REG_XRF_8BTB_ERR_COUNT_LANE_2 0x2086 +#define SUNI1x10GEXP_REG_XRF_8BTB_ERR_COUNT_LANE_3 0x2087 +#define SUNI1x10GEXP_REG_XRF_INTERRUPT_ENABLE 0x2088 +#define SUNI1x10GEXP_REG_XRF_INTERRUPT_STATUS 0x2089 +#define SUNI1x10GEXP_REG_XRF_ERR_STATUS 0x208A +#define SUNI1x10GEXP_REG_XRF_DIAG_INTERRUPT_ENABLE 0x208B +#define SUNI1x10GEXP_REG_XRF_DIAG_INTERRUPT_STATUS 0x208C +#define SUNI1x10GEXP_REG_XRF_CODE_ERR_THRES 0x2092 + +#define SUNI1x10GEXP_REG_RXOAM_CONFIG 0x20C0 +#define SUNI1x10GEXP_REG_RXOAM_FILTER_1_CONFIG 0x20C1 +#define SUNI1x10GEXP_REG_RXOAM_FILTER_2_CONFIG 0x20C2 +#define SUNI1x10GEXP_REG_RXOAM_CONFIG_2 0x20C3 +#define SUNI1x10GEXP_REG_RXOAM_HEC_CONFIG 0x20C4 +#define SUNI1x10GEXP_REG_RXOAM_HEC_ERR_THRES 0x20C5 +#define SUNI1x10GEXP_REG_RXOAM_INTERRUPT_ENABLE 0x20C7 +#define SUNI1x10GEXP_REG_RXOAM_INTERRUPT_STATUS 0x20C8 +#define SUNI1x10GEXP_REG_RXOAM_STATUS 0x20C9 +#define SUNI1x10GEXP_REG_RXOAM_HEC_ERR_COUNT 0x20CA +#define SUNI1x10GEXP_REG_RXOAM_FIFO_OVERFLOW_COUNT 0x20CB +#define SUNI1x10GEXP_REG_RXOAM_FILTER_MISMATCH_COUNT_LSB 0x20CC +#define SUNI1x10GEXP_REG_RXOAM_FILTER_MISMATCH_COUNT_MSB 0x20CD +#define SUNI1x10GEXP_REG_RXOAM_FILTER_1_MISMATCH_COUNT_LSB 0x20CE +#define SUNI1x10GEXP_REG_RXOAM_FILTER_1_MISMATCH_COUNT_MSB 0x20CF +#define SUNI1x10GEXP_REG_RXOAM_FILTER_2_MISMATCH_COUNT_LSB 0x20D0 +#define SUNI1x10GEXP_REG_RXOAM_FILTER_2_MISMATCH_COUNT_MSB 0x20D1 +#define SUNI1x10GEXP_REG_RXOAM_OAM_EXTRACT_COUNT_LSB 0x20D2 +#define SUNI1x10GEXP_REG_RXOAM_OAM_EXTRACT_COUNT_MSB 0x20D3 +#define SUNI1x10GEXP_REG_RXOAM_MINI_PACKET_COUNT_LSB 0x20D4 +#define SUNI1x10GEXP_REG_RXOAM_MINI_PACKET_COUNT_MSB 0x20D5 +#define SUNI1x10GEXP_REG_RXOAM_FILTER_MISMATCH_THRES_LSB 0x20D6 +#define SUNI1x10GEXP_REG_RXOAM_FILTER_MISMATCH_THRES_MSB 0x20D7 + +#define SUNI1x10GEXP_REG_MSTAT_CONTROL 0x2100 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_ROLLOVER_0 0x2101 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_ROLLOVER_1 0x2102 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_ROLLOVER_2 0x2103 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_ROLLOVER_3 0x2104 +#define SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_0 0x2105 +#define SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_1 0x2106 +#define SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_2 0x2107 +#define SUNI1x10GEXP_REG_MSTAT_INTERRUPT_MASK_3 0x2108 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_WRITE_ADDRESS 0x2109 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_WRITE_DATA_LOW 0x210A +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_WRITE_DATA_MIDDLE 0x210B +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_WRITE_DATA_HIGH 0x210C +#define mSUNI1x10GEXP_REG_MSTAT_COUNTER_LOW(countId) (0x2110 + mSUNI1x10GEXP_MSTAT_COUNT_OFFSET(countId)) +#define mSUNI1x10GEXP_REG_MSTAT_COUNTER_MID(countId) (0x2111 + mSUNI1x10GEXP_MSTAT_COUNT_OFFSET(countId)) +#define mSUNI1x10GEXP_REG_MSTAT_COUNTER_HIGH(countId) (0x2112 + mSUNI1x10GEXP_MSTAT_COUNT_OFFSET(countId)) +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_0_LOW 0x2110 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_0_MID 0x2111 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_0_HIGH 0x2112 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_0_RESVD 0x2113 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_1_LOW 0x2114 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_1_MID 0x2115 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_1_HIGH 0x2116 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_1_RESVD 0x2117 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_2_LOW 0x2118 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_2_MID 0x2119 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_2_HIGH 0x211A +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_2_RESVD 0x211B +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_3_LOW 0x211C +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_3_MID 0x211D +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_3_HIGH 0x211E +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_3_RESVD 0x211F +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_4_LOW 0x2120 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_4_MID 0x2121 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_4_HIGH 0x2122 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_4_RESVD 0x2123 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_5_LOW 0x2124 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_5_MID 0x2125 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_5_HIGH 0x2126 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_5_RESVD 0x2127 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_6_LOW 0x2128 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_6_MID 0x2129 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_6_HIGH 0x212A +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_6_RESVD 0x212B +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_7_LOW 0x212C +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_7_MID 0x212D +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_7_HIGH 0x212E +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_7_RESVD 0x212F +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_8_LOW 0x2130 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_8_MID 0x2131 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_8_HIGH 0x2132 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_8_RESVD 0x2133 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_9_LOW 0x2134 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_9_MID 0x2135 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_9_HIGH 0x2136 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_9_RESVD 0x2137 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_10_LOW 0x2138 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_10_MID 0x2139 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_10_HIGH 0x213A +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_10_RESVD 0x213B +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_11_LOW 0x213C +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_11_MID 0x213D +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_11_HIGH 0x213E +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_11_RESVD 0x213F +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_12_LOW 0x2140 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_12_MID 0x2141 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_12_HIGH 0x2142 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_12_RESVD 0x2143 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_13_LOW 0x2144 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_13_MID 0x2145 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_13_HIGH 0x2146 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_13_RESVD 0x2147 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_14_LOW 0x2148 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_14_MID 0x2149 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_14_HIGH 0x214A +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_14_RESVD 0x214B +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_15_LOW 0x214C +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_15_MID 0x214D +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_15_HIGH 0x214E +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_15_RESVD 0x214F +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_16_LOW 0x2150 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_16_MID 0x2151 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_16_HIGH 0x2152 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_16_RESVD 0x2153 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_17_LOW 0x2154 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_17_MID 0x2155 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_17_HIGH 0x2156 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_17_RESVD 0x2157 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_18_LOW 0x2158 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_18_MID 0x2159 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_18_HIGH 0x215A +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_18_RESVD 0x215B +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_19_LOW 0x215C +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_19_MID 0x215D +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_19_HIGH 0x215E +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_19_RESVD 0x215F +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_20_LOW 0x2160 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_20_MID 0x2161 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_20_HIGH 0x2162 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_20_RESVD 0x2163 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_21_LOW 0x2164 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_21_MID 0x2165 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_21_HIGH 0x2166 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_21_RESVD 0x2167 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_22_LOW 0x2168 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_22_MID 0x2169 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_22_HIGH 0x216A +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_22_RESVD 0x216B +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_23_LOW 0x216C +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_23_MID 0x216D +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_23_HIGH 0x216E +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_23_RESVD 0x216F +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_24_LOW 0x2170 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_24_MID 0x2171 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_24_HIGH 0x2172 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_24_RESVD 0x2173 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_25_LOW 0x2174 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_25_MID 0x2175 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_25_HIGH 0x2176 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_25_RESVD 0x2177 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_26_LOW 0x2178 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_26_MID 0x2179 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_26_HIGH 0x217a +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_26_RESVD 0x217b +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_27_LOW 0x217c +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_27_MID 0x217d +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_27_HIGH 0x217e +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_27_RESVD 0x217f +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_28_LOW 0x2180 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_28_MID 0x2181 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_28_HIGH 0x2182 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_28_RESVD 0x2183 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_29_LOW 0x2184 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_29_MID 0x2185 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_29_HIGH 0x2186 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_29_RESVD 0x2187 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_30_LOW 0x2188 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_30_MID 0x2189 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_30_HIGH 0x218A +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_30_RESVD 0x218B +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_31_LOW 0x218C +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_31_MID 0x218D +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_31_HIGH 0x218E +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_31_RESVD 0x218F +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_32_LOW 0x2190 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_32_MID 0x2191 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_32_HIGH 0x2192 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_32_RESVD 0x2193 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_33_LOW 0x2194 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_33_MID 0x2195 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_33_HIGH 0x2196 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_33_RESVD 0x2197 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_34_LOW 0x2198 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_34_MID 0x2199 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_34_HIGH 0x219A +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_34_RESVD 0x219B +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_35_LOW 0x219C +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_35_MID 0x219D +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_35_HIGH 0x219E +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_35_RESVD 0x219F +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_36_LOW 0x21A0 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_36_MID 0x21A1 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_36_HIGH 0x21A2 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_36_RESVD 0x21A3 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_37_LOW 0x21A4 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_37_MID 0x21A5 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_37_HIGH 0x21A6 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_37_RESVD 0x21A7 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_38_LOW 0x21A8 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_38_MID 0x21A9 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_38_HIGH 0x21AA +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_38_RESVD 0x21AB +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_39_LOW 0x21AC +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_39_MID 0x21AD +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_39_HIGH 0x21AE +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_39_RESVD 0x21AF +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_40_LOW 0x21B0 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_40_MID 0x21B1 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_40_HIGH 0x21B2 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_40_RESVD 0x21B3 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_41_LOW 0x21B4 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_41_MID 0x21B5 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_41_HIGH 0x21B6 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_41_RESVD 0x21B7 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_42_LOW 0x21B8 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_42_MID 0x21B9 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_42_HIGH 0x21BA +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_42_RESVD 0x21BB +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_43_LOW 0x21BC +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_43_MID 0x21BD +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_43_HIGH 0x21BE +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_43_RESVD 0x21BF +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_44_LOW 0x21C0 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_44_MID 0x21C1 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_44_HIGH 0x21C2 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_44_RESVD 0x21C3 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_45_LOW 0x21C4 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_45_MID 0x21C5 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_45_HIGH 0x21C6 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_45_RESVD 0x21C7 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_46_LOW 0x21C8 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_46_MID 0x21C9 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_46_HIGH 0x21CA +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_46_RESVD 0x21CB +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_47_LOW 0x21CC +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_47_MID 0x21CD +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_47_HIGH 0x21CE +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_47_RESVD 0x21CF +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_48_LOW 0x21D0 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_48_MID 0x21D1 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_48_HIGH 0x21D2 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_48_RESVD 0x21D3 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_49_LOW 0x21D4 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_49_MID 0x21D5 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_49_HIGH 0x21D6 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_49_RESVD 0x21D7 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_50_LOW 0x21D8 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_50_MID 0x21D9 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_50_HIGH 0x21DA +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_50_RESVD 0x21DB +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_51_LOW 0x21DC +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_51_MID 0x21DD +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_51_HIGH 0x21DE +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_51_RESVD 0x21DF +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_52_LOW 0x21E0 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_52_MID 0x21E1 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_52_HIGH 0x21E2 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_52_RESVD 0x21E3 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_53_LOW 0x21E4 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_53_MID 0x21E5 +#define SUNI1x10GEXP_REG_MSTAT_COUNTER_53_HIGH 0x21E6 +#define SUNI1x10GEXP_CNTR_MAC_ETHERNET_NUM 51 + +#define SUNI1x10GEXP_REG_IFLX_GLOBAL_CONFIG 0x2200 +#define SUNI1x10GEXP_REG_IFLX_CHANNEL_PROVISION 0x2201 +#define SUNI1x10GEXP_REG_IFLX_FIFO_OVERFLOW_ENABLE 0x2209 +#define SUNI1x10GEXP_REG_IFLX_FIFO_OVERFLOW_INTERRUPT 0x220A +#define SUNI1x10GEXP_REG_IFLX_INDIR_CHANNEL_ADDRESS 0x220D +#define SUNI1x10GEXP_REG_IFLX_INDIR_LOGICAL_FIFO_LOW_LIMIT_PROVISION 0x220E +#define SUNI1x10GEXP_REG_IFLX_INDIR_LOGICAL_FIFO_HIGH_LIMIT 0x220F +#define SUNI1x10GEXP_REG_IFLX_INDIR_FULL_ALMOST_FULL_STATUS_LIMIT 0x2210 +#define SUNI1x10GEXP_REG_IFLX_INDIR_EMPTY_ALMOST_EMPTY_STATUS_LIMIT 0x2211 + +#define SUNI1x10GEXP_REG_PL4MOS_CONFIG 0x2240 +#define SUNI1x10GEXP_REG_PL4MOS_MASK 0x2241 +#define SUNI1x10GEXP_REG_PL4MOS_FAIRNESS_MASKING 0x2242 +#define SUNI1x10GEXP_REG_PL4MOS_MAXBURST1 0x2243 +#define SUNI1x10GEXP_REG_PL4MOS_MAXBURST2 0x2244 +#define SUNI1x10GEXP_REG_PL4MOS_TRANSFER_SIZE 0x2245 + +#define SUNI1x10GEXP_REG_PL4ODP_CONFIG 0x2280 +#define SUNI1x10GEXP_REG_PL4ODP_INTERRUPT_MASK 0x2282 +#define SUNI1x10GEXP_REG_PL4ODP_INTERRUPT 0x2283 +#define SUNI1x10GEXP_REG_PL4ODP_CONFIG_MAX_T 0x2284 + +#define SUNI1x10GEXP_REG_PL4IO_LOCK_DETECT_STATUS 0x2300 +#define SUNI1x10GEXP_REG_PL4IO_LOCK_DETECT_CHANGE 0x2301 +#define SUNI1x10GEXP_REG_PL4IO_LOCK_DETECT_MASK 0x2302 +#define SUNI1x10GEXP_REG_PL4IO_LOCK_DETECT_LIMITS 0x2303 +#define SUNI1x10GEXP_REG_PL4IO_CALENDAR_REPETITIONS 0x2304 +#define SUNI1x10GEXP_REG_PL4IO_CONFIG 0x2305 + +#define SUNI1x10GEXP_REG_TXXG_CONFIG_1 0x3040 +#define SUNI1x10GEXP_REG_TXXG_CONFIG_2 0x3041 +#define SUNI1x10GEXP_REG_TXXG_CONFIG_3 0x3042 +#define SUNI1x10GEXP_REG_TXXG_INTERRUPT 0x3043 +#define SUNI1x10GEXP_REG_TXXG_STATUS 0x3044 +#define SUNI1x10GEXP_REG_TXXG_MAX_FRAME_SIZE 0x3045 +#define SUNI1x10GEXP_REG_TXXG_MIN_FRAME_SIZE 0x3046 +#define SUNI1x10GEXP_REG_TXXG_SA_15_0 0x3047 +#define SUNI1x10GEXP_REG_TXXG_SA_31_16 0x3048 +#define SUNI1x10GEXP_REG_TXXG_SA_47_32 0x3049 +#define SUNI1x10GEXP_REG_TXXG_PAUSE_TIMER 0x304D +#define SUNI1x10GEXP_REG_TXXG_PAUSE_TIMER_INTERVAL 0x304E +#define SUNI1x10GEXP_REG_TXXG_FILTER_ERROR_COUNTER 0x3051 +#define SUNI1x10GEXP_REG_TXXG_PAUSE_QUANTUM_CONFIG 0x3052 + +#define SUNI1x10GEXP_REG_XTEF_CTRL 0x3080 +#define SUNI1x10GEXP_REG_XTEF_INTERRUPT_STATUS 0x3084 +#define SUNI1x10GEXP_REG_XTEF_INTERRUPT_ENABLE 0x3085 +#define SUNI1x10GEXP_REG_XTEF_VISIBILITY 0x3086 + +#define SUNI1x10GEXP_REG_TXOAM_OAM_CONFIG 0x30C0 +#define SUNI1x10GEXP_REG_TXOAM_MINI_RATE_CONFIG 0x30C1 +#define SUNI1x10GEXP_REG_TXOAM_MINI_GAP_FIFO_CONFIG 0x30C2 +#define SUNI1x10GEXP_REG_TXOAM_P1P2_STATIC_VALUES 0x30C3 +#define SUNI1x10GEXP_REG_TXOAM_P3P4_STATIC_VALUES 0x30C4 +#define SUNI1x10GEXP_REG_TXOAM_P5P6_STATIC_VALUES 0x30C5 +#define SUNI1x10GEXP_REG_TXOAM_INTERRUPT_ENABLE 0x30C6 +#define SUNI1x10GEXP_REG_TXOAM_INTERRUPT_STATUS 0x30C7 +#define SUNI1x10GEXP_REG_TXOAM_INSERT_COUNT_LSB 0x30C8 +#define SUNI1x10GEXP_REG_TXOAM_INSERT_COUNT_MSB 0x30C9 +#define SUNI1x10GEXP_REG_TXOAM_OAM_MINI_COUNT_LSB 0x30CA +#define SUNI1x10GEXP_REG_TXOAM_OAM_MINI_COUNT_MSB 0x30CB +#define SUNI1x10GEXP_REG_TXOAM_P1P2_MINI_MASK 0x30CC +#define SUNI1x10GEXP_REG_TXOAM_P3P4_MINI_MASK 0x30CD +#define SUNI1x10GEXP_REG_TXOAM_P5P6_MINI_MASK 0x30CE +#define SUNI1x10GEXP_REG_TXOAM_COSET 0x30CF +#define SUNI1x10GEXP_REG_TXOAM_EMPTY_FIFO_INS_OP_CNT_LSB 0x30D0 +#define SUNI1x10GEXP_REG_TXOAM_EMPTY_FIFO_INS_OP_CNT_MSB 0x30D1 +#define SUNI1x10GEXP_REG_TXOAM_STATIC_VALUE_MINI_COUNT_LSB 0x30D2 +#define SUNI1x10GEXP_REG_TXOAM_STATIC_VALUE_MINI_COUNT_MSB 0x30D3 + + +#define SUNI1x10GEXP_REG_EFLX_GLOBAL_CONFIG 0x3200 +#define SUNI1x10GEXP_REG_EFLX_ERCU_GLOBAL_STATUS 0x3201 +#define SUNI1x10GEXP_REG_EFLX_INDIR_CHANNEL_ADDRESS 0x3202 +#define SUNI1x10GEXP_REG_EFLX_INDIR_FIFO_LOW_LIMIT 0x3203 +#define SUNI1x10GEXP_REG_EFLX_INDIR_FIFO_HIGH_LIMIT 0x3204 +#define SUNI1x10GEXP_REG_EFLX_INDIR_FULL_ALMOST_FULL_STATUS_AND_LIMIT 0x3205 +#define SUNI1x10GEXP_REG_EFLX_INDIR_EMPTY_ALMOST_EMPTY_STATUS_AND_LIMIT 0x3206 +#define SUNI1x10GEXP_REG_EFLX_INDIR_FIFO_CUT_THROUGH_THRESHOLD 0x3207 +#define SUNI1x10GEXP_REG_EFLX_FIFO_OVERFLOW_ERROR_ENABLE 0x320C +#define SUNI1x10GEXP_REG_EFLX_FIFO_OVERFLOW_ERROR_INDICATION 0x320D +#define SUNI1x10GEXP_REG_EFLX_CHANNEL_PROVISION 0x3210 + +#define SUNI1x10GEXP_REG_PL4IDU_CONFIG 0x3280 +#define SUNI1x10GEXP_REG_PL4IDU_INTERRUPT_MASK 0x3282 +#define SUNI1x10GEXP_REG_PL4IDU_INTERRUPT 0x3283 + + +/*----------------------------------------*/ +#define SUNI1x10GEXP_REG_MAX_OFFSET 0x3480 + +/******************************************************************************/ +/* -- End register offset definitions -- */ +/******************************************************************************/ + +/******************************************************************************/ +/** SUNI-1x10GE-XP REGISTER BIT MASKS **/ +/******************************************************************************/ + +#define SUNI1x10GEXP_BITMSK_BITS_1 0x00001 +#define SUNI1x10GEXP_BITMSK_BITS_2 0x00003 +#define SUNI1x10GEXP_BITMSK_BITS_3 0x00007 +#define SUNI1x10GEXP_BITMSK_BITS_4 0x0000f +#define SUNI1x10GEXP_BITMSK_BITS_5 0x0001f +#define SUNI1x10GEXP_BITMSK_BITS_6 0x0003f +#define SUNI1x10GEXP_BITMSK_BITS_7 0x0007f +#define SUNI1x10GEXP_BITMSK_BITS_8 0x000ff +#define SUNI1x10GEXP_BITMSK_BITS_9 0x001ff +#define SUNI1x10GEXP_BITMSK_BITS_10 0x003ff +#define SUNI1x10GEXP_BITMSK_BITS_11 0x007ff +#define SUNI1x10GEXP_BITMSK_BITS_12 0x00fff +#define SUNI1x10GEXP_BITMSK_BITS_13 0x01fff +#define SUNI1x10GEXP_BITMSK_BITS_14 0x03fff +#define SUNI1x10GEXP_BITMSK_BITS_15 0x07fff +#define SUNI1x10GEXP_BITMSK_BITS_16 0x0ffff + +#define mSUNI1x10GEXP_CLR_MSBITS_1(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_15) +#define mSUNI1x10GEXP_CLR_MSBITS_2(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_14) +#define mSUNI1x10GEXP_CLR_MSBITS_3(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_13) +#define mSUNI1x10GEXP_CLR_MSBITS_4(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_12) +#define mSUNI1x10GEXP_CLR_MSBITS_5(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_11) +#define mSUNI1x10GEXP_CLR_MSBITS_6(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_10) +#define mSUNI1x10GEXP_CLR_MSBITS_7(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_9) +#define mSUNI1x10GEXP_CLR_MSBITS_8(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_8) +#define mSUNI1x10GEXP_CLR_MSBITS_9(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_7) +#define mSUNI1x10GEXP_CLR_MSBITS_10(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_6) +#define mSUNI1x10GEXP_CLR_MSBITS_11(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_5) +#define mSUNI1x10GEXP_CLR_MSBITS_12(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_4) +#define mSUNI1x10GEXP_CLR_MSBITS_13(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_3) +#define mSUNI1x10GEXP_CLR_MSBITS_14(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_2) +#define mSUNI1x10GEXP_CLR_MSBITS_15(v) ((v) & SUNI1x10GEXP_BITMSK_BITS_1) + +#define mSUNI1x10GEXP_GET_BIT(val, bitMsk) (((val)&(bitMsk)) ? 1:0) + + + +/*---------------------------------------------------------------------------- + * Register 0x0001: S/UNI-1x10GE-XP Product Revision + * Bit 3-0 REVISION + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_REVISION 0x000F + +/*---------------------------------------------------------------------------- + * Register 0x0002: S/UNI-1x10GE-XP Configuration and Reset Control + * Bit 2 XAUI_ARESETB + * Bit 1 PL4_ARESETB + * Bit 0 DRESETB + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_XAUI_ARESET 0x0004 +#define SUNI1x10GEXP_BITMSK_PL4_ARESET 0x0002 +#define SUNI1x10GEXP_BITMSK_DRESETB 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x0003: S/UNI-1x10GE-XP Loop Back and Miscellaneous Control + * Bit 11 PL4IO_OUTCLKSEL + * Bit 9 SYSPCSLB + * Bit 8 LINEPCSLB + * Bit 7 MSTAT_BYPASS + * Bit 6 RXXG_BYPASS + * Bit 5 TXXG_BYPASS + * Bit 4 SOP_PAD_EN + * Bit 1 LOS_INV + * Bit 0 OVERRIDE_LOS + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4IO_OUTCLKSEL 0x0800 +#define SUNI1x10GEXP_BITMSK_SYSPCSLB 0x0200 +#define SUNI1x10GEXP_BITMSK_LINEPCSLB 0x0100 +#define SUNI1x10GEXP_BITMSK_MSTAT_BYPASS 0x0080 +#define SUNI1x10GEXP_BITMSK_RXXG_BYPASS 0x0040 +#define SUNI1x10GEXP_BITMSK_TXXG_BYPASS 0x0020 +#define SUNI1x10GEXP_BITMSK_SOP_PAD_EN 0x0010 +#define SUNI1x10GEXP_BITMSK_LOS_INV 0x0002 +#define SUNI1x10GEXP_BITMSK_OVERRIDE_LOS 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x0004: S/UNI-1x10GE-XP Device Status + * Bit 9 TOP_SXRA_EXPIRED + * Bit 8 TOP_MDIO_BUSY + * Bit 7 TOP_DTRB + * Bit 6 TOP_EXPIRED + * Bit 5 TOP_PAUSED + * Bit 4 TOP_PL4_ID_DOOL + * Bit 3 TOP_PL4_IS_DOOL + * Bit 2 TOP_PL4_ID_ROOL + * Bit 1 TOP_PL4_IS_ROOL + * Bit 0 TOP_PL4_OUT_ROOL + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TOP_SXRA_EXPIRED 0x0200 +#define SUNI1x10GEXP_BITMSK_TOP_MDIO_BUSY 0x0100 +#define SUNI1x10GEXP_BITMSK_TOP_DTRB 0x0080 +#define SUNI1x10GEXP_BITMSK_TOP_EXPIRED 0x0040 +#define SUNI1x10GEXP_BITMSK_TOP_PAUSED 0x0020 +#define SUNI1x10GEXP_BITMSK_TOP_PL4_ID_DOOL 0x0010 +#define SUNI1x10GEXP_BITMSK_TOP_PL4_IS_DOOL 0x0008 +#define SUNI1x10GEXP_BITMSK_TOP_PL4_ID_ROOL 0x0004 +#define SUNI1x10GEXP_BITMSK_TOP_PL4_IS_ROOL 0x0002 +#define SUNI1x10GEXP_BITMSK_TOP_PL4_OUT_ROOL 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x0005: Global Performance Update and Clock Monitors + * Bit 15 TIP + * Bit 8 XAUI_REF_CLKA + * Bit 7 RXLANE3CLKA + * Bit 6 RXLANE2CLKA + * Bit 5 RXLANE1CLKA + * Bit 4 RXLANE0CLKA + * Bit 3 CSUCLKA + * Bit 2 TDCLKA + * Bit 1 RSCLKA + * Bit 0 RDCLKA + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TIP 0x8000 +#define SUNI1x10GEXP_BITMSK_XAUI_REF_CLKA 0x0100 +#define SUNI1x10GEXP_BITMSK_RXLANE3CLKA 0x0080 +#define SUNI1x10GEXP_BITMSK_RXLANE2CLKA 0x0040 +#define SUNI1x10GEXP_BITMSK_RXLANE1CLKA 0x0020 +#define SUNI1x10GEXP_BITMSK_RXLANE0CLKA 0x0010 +#define SUNI1x10GEXP_BITMSK_CSUCLKA 0x0008 +#define SUNI1x10GEXP_BITMSK_TDCLKA 0x0004 +#define SUNI1x10GEXP_BITMSK_RSCLKA 0x0002 +#define SUNI1x10GEXP_BITMSK_RDCLKA 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x0006: MDIO Command + * Bit 4 MDIO_RDINC + * Bit 3 MDIO_RSTAT + * Bit 2 MDIO_LCTLD + * Bit 1 MDIO_LCTLA + * Bit 0 MDIO_SPRE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_MDIO_RDINC 0x0010 +#define SUNI1x10GEXP_BITMSK_MDIO_RSTAT 0x0008 +#define SUNI1x10GEXP_BITMSK_MDIO_LCTLD 0x0004 +#define SUNI1x10GEXP_BITMSK_MDIO_LCTLA 0x0002 +#define SUNI1x10GEXP_BITMSK_MDIO_SPRE 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x0007: MDIO Interrupt Enable + * Bit 0 MDIO_BUSY_EN + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_MDIO_BUSY_EN 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x0008: MDIO Interrupt Status + * Bit 0 MDIO_BUSYI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_MDIO_BUSYI 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x0009: MMD PHY Address + * Bit 12-8 MDIO_DEVADR + * Bit 4-0 MDIO_PRTADR + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_MDIO_DEVADR 0x1F00 +#define SUNI1x10GEXP_BITOFF_MDIO_DEVADR 8 +#define SUNI1x10GEXP_BITMSK_MDIO_PRTADR 0x001F +#define SUNI1x10GEXP_BITOFF_MDIO_PRTADR 0 + +/*---------------------------------------------------------------------------- + * Register 0x000C: OAM Interface Control + * Bit 6 MDO_OD_ENB + * Bit 5 MDI_INV + * Bit 4 MDI_SEL + * Bit 3 RXOAMEN + * Bit 2 RXOAMCLKEN + * Bit 1 TXOAMEN + * Bit 0 TXOAMCLKEN + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_MDO_OD_ENB 0x0040 +#define SUNI1x10GEXP_BITMSK_MDI_INV 0x0020 +#define SUNI1x10GEXP_BITMSK_MDI_SEL 0x0010 +#define SUNI1x10GEXP_BITMSK_RXOAMEN 0x0008 +#define SUNI1x10GEXP_BITMSK_RXOAMCLKEN 0x0004 +#define SUNI1x10GEXP_BITMSK_TXOAMEN 0x0002 +#define SUNI1x10GEXP_BITMSK_TXOAMCLKEN 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x000D: S/UNI-1x10GE-XP Master Interrupt Status + * Bit 15 TOP_PL4IO_INT + * Bit 14 TOP_IRAM_INT + * Bit 13 TOP_ERAM_INT + * Bit 12 TOP_XAUI_INT + * Bit 11 TOP_MSTAT_INT + * Bit 10 TOP_RXXG_INT + * Bit 9 TOP_TXXG_INT + * Bit 8 TOP_XRF_INT + * Bit 7 TOP_XTEF_INT + * Bit 6 TOP_MDIO_BUSY_INT + * Bit 5 TOP_RXOAM_INT + * Bit 4 TOP_TXOAM_INT + * Bit 3 TOP_IFLX_INT + * Bit 2 TOP_EFLX_INT + * Bit 1 TOP_PL4ODP_INT + * Bit 0 TOP_PL4IDU_INT + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TOP_PL4IO_INT 0x8000 +#define SUNI1x10GEXP_BITMSK_TOP_IRAM_INT 0x4000 +#define SUNI1x10GEXP_BITMSK_TOP_ERAM_INT 0x2000 +#define SUNI1x10GEXP_BITMSK_TOP_XAUI_INT 0x1000 +#define SUNI1x10GEXP_BITMSK_TOP_MSTAT_INT 0x0800 +#define SUNI1x10GEXP_BITMSK_TOP_RXXG_INT 0x0400 +#define SUNI1x10GEXP_BITMSK_TOP_TXXG_INT 0x0200 +#define SUNI1x10GEXP_BITMSK_TOP_XRF_INT 0x0100 +#define SUNI1x10GEXP_BITMSK_TOP_XTEF_INT 0x0080 +#define SUNI1x10GEXP_BITMSK_TOP_MDIO_BUSY_INT 0x0040 +#define SUNI1x10GEXP_BITMSK_TOP_RXOAM_INT 0x0020 +#define SUNI1x10GEXP_BITMSK_TOP_TXOAM_INT 0x0010 +#define SUNI1x10GEXP_BITMSK_TOP_IFLX_INT 0x0008 +#define SUNI1x10GEXP_BITMSK_TOP_EFLX_INT 0x0004 +#define SUNI1x10GEXP_BITMSK_TOP_PL4ODP_INT 0x0002 +#define SUNI1x10GEXP_BITMSK_TOP_PL4IDU_INT 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x000E:PM3393 Global interrupt enable + * Bit 15 TOP_INTE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TOP_INTE 0x8000 + +/*---------------------------------------------------------------------------- + * Register 0x0010: XTEF Miscellaneous Control + * Bit 7 RF_VAL + * Bit 6 RF_OVERRIDE + * Bit 5 LF_VAL + * Bit 4 LF_OVERRIDE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RF_VAL 0x0080 +#define SUNI1x10GEXP_BITMSK_RF_OVERRIDE 0x0040 +#define SUNI1x10GEXP_BITMSK_LF_VAL 0x0020 +#define SUNI1x10GEXP_BITMSK_LF_OVERRIDE 0x0010 +#define SUNI1x10GEXP_BITMSK_LFRF_OVERRIDE_VAL 0x00F0 + +/*---------------------------------------------------------------------------- + * Register 0x0011: XRF Miscellaneous Control + * Bit 6-4 EN_IDLE_REP + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EN_IDLE_REP 0x0070 + +/*---------------------------------------------------------------------------- + * Register 0x0100: SERDES 3125 Configuration Register 1 + * Bit 10 RXEQB_3 + * Bit 8 RXEQB_2 + * Bit 6 RXEQB_1 + * Bit 4 RXEQB_0 + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXEQB 0x0FF0 +#define SUNI1x10GEXP_BITOFF_RXEQB_3 10 +#define SUNI1x10GEXP_BITOFF_RXEQB_2 8 +#define SUNI1x10GEXP_BITOFF_RXEQB_1 6 +#define SUNI1x10GEXP_BITOFF_RXEQB_0 4 + +/*---------------------------------------------------------------------------- + * Register 0x0101: SERDES 3125 Configuration Register 2 + * Bit 12 YSEL + * Bit 7 PRE_EMPH_3 + * Bit 6 PRE_EMPH_2 + * Bit 5 PRE_EMPH_1 + * Bit 4 PRE_EMPH_0 + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_YSEL 0x1000 +#define SUNI1x10GEXP_BITMSK_PRE_EMPH 0x00F0 +#define SUNI1x10GEXP_BITMSK_PRE_EMPH_3 0x0080 +#define SUNI1x10GEXP_BITMSK_PRE_EMPH_2 0x0040 +#define SUNI1x10GEXP_BITMSK_PRE_EMPH_1 0x0020 +#define SUNI1x10GEXP_BITMSK_PRE_EMPH_0 0x0010 + +/*---------------------------------------------------------------------------- + * Register 0x0102: SERDES 3125 Interrupt Enable Register + * Bit 3 LASIE + * Bit 2 SPLL_RAE + * Bit 1 MPLL_RAE + * Bit 0 PLL_LOCKE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_LASIE 0x0008 +#define SUNI1x10GEXP_BITMSK_SPLL_RAE 0x0004 +#define SUNI1x10GEXP_BITMSK_MPLL_RAE 0x0002 +#define SUNI1x10GEXP_BITMSK_PLL_LOCKE 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x0103: SERDES 3125 Interrupt Visibility Register + * Bit 3 LASIV + * Bit 2 SPLL_RAV + * Bit 1 MPLL_RAV + * Bit 0 PLL_LOCKV + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_LASIV 0x0008 +#define SUNI1x10GEXP_BITMSK_SPLL_RAV 0x0004 +#define SUNI1x10GEXP_BITMSK_MPLL_RAV 0x0002 +#define SUNI1x10GEXP_BITMSK_PLL_LOCKV 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x0104: SERDES 3125 Interrupt Status Register + * Bit 3 LASII + * Bit 2 SPLL_RAI + * Bit 1 MPLL_RAI + * Bit 0 PLL_LOCKI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_LASII 0x0008 +#define SUNI1x10GEXP_BITMSK_SPLL_RAI 0x0004 +#define SUNI1x10GEXP_BITMSK_MPLL_RAI 0x0002 +#define SUNI1x10GEXP_BITMSK_PLL_LOCKI 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x0107: SERDES 3125 Test Configuration + * Bit 12 DUALTX + * Bit 10 HC_1 + * Bit 9 HC_0 + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_DUALTX 0x1000 +#define SUNI1x10GEXP_BITMSK_HC 0x0600 +#define SUNI1x10GEXP_BITOFF_HC_0 9 + +/*---------------------------------------------------------------------------- + * Register 0x2040: RXXG Configuration 1 + * Bit 15 RXXG_RXEN + * Bit 14 RXXG_ROCF + * Bit 13 RXXG_PAD_STRIP + * Bit 10 RXXG_PUREP + * Bit 9 RXXG_LONGP + * Bit 8 RXXG_PARF + * Bit 7 RXXG_FLCHK + * Bit 5 RXXG_PASS_CTRL + * Bit 3 RXXG_CRC_STRIP + * Bit 2-0 RXXG_MIFG + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXXG_RXEN 0x8000 +#define SUNI1x10GEXP_BITMSK_RXXG_ROCF 0x4000 +#define SUNI1x10GEXP_BITMSK_RXXG_PAD_STRIP 0x2000 +#define SUNI1x10GEXP_BITMSK_RXXG_PUREP 0x0400 +#define SUNI1x10GEXP_BITMSK_RXXG_LONGP 0x0200 +#define SUNI1x10GEXP_BITMSK_RXXG_PARF 0x0100 +#define SUNI1x10GEXP_BITMSK_RXXG_FLCHK 0x0080 +#define SUNI1x10GEXP_BITMSK_RXXG_PASS_CTRL 0x0020 +#define SUNI1x10GEXP_BITMSK_RXXG_CRC_STRIP 0x0008 + +/*---------------------------------------------------------------------------- + * Register 0x02041: RXXG Configuration 2 + * Bit 7-0 RXXG_HDRSIZE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXXG_HDRSIZE 0x00FF + +/*---------------------------------------------------------------------------- + * Register 0x2042: RXXG Configuration 3 + * Bit 15 RXXG_MIN_LERRE + * Bit 14 RXXG_MAX_LERRE + * Bit 12 RXXG_LINE_ERRE + * Bit 10 RXXG_RX_OVRE + * Bit 9 RXXG_ADR_FILTERE + * Bit 8 RXXG_ERR_FILTERE + * Bit 5 RXXG_PRMB_ERRE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXXG_MIN_LERRE 0x8000 +#define SUNI1x10GEXP_BITMSK_RXXG_MAX_LERRE 0x4000 +#define SUNI1x10GEXP_BITMSK_RXXG_LINE_ERRE 0x1000 +#define SUNI1x10GEXP_BITMSK_RXXG_RX_OVRE 0x0400 +#define SUNI1x10GEXP_BITMSK_RXXG_ADR_FILTERE 0x0200 +#define SUNI1x10GEXP_BITMSK_RXXG_ERR_FILTERRE 0x0100 +#define SUNI1x10GEXP_BITMSK_RXXG_PRMB_ERRE 0x0020 + +/*---------------------------------------------------------------------------- + * Register 0x2043: RXXG Interrupt + * Bit 15 RXXG_MIN_LERRI + * Bit 14 RXXG_MAX_LERRI + * Bit 12 RXXG_LINE_ERRI + * Bit 10 RXXG_RX_OVRI + * Bit 9 RXXG_ADR_FILTERI + * Bit 8 RXXG_ERR_FILTERI + * Bit 5 RXXG_PRMB_ERRE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXXG_MIN_LERRI 0x8000 +#define SUNI1x10GEXP_BITMSK_RXXG_MAX_LERRI 0x4000 +#define SUNI1x10GEXP_BITMSK_RXXG_LINE_ERRI 0x1000 +#define SUNI1x10GEXP_BITMSK_RXXG_RX_OVRI 0x0400 +#define SUNI1x10GEXP_BITMSK_RXXG_ADR_FILTERI 0x0200 +#define SUNI1x10GEXP_BITMSK_RXXG_ERR_FILTERI 0x0100 +#define SUNI1x10GEXP_BITMSK_RXXG_PRMB_ERRE 0x0020 + +/*---------------------------------------------------------------------------- + * Register 0x2049: RXXG Receive FIFO Threshold + * Bit 2-0 RXXG_CUT_THRU + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXXG_CUT_THRU 0x0007 +#define SUNI1x10GEXP_BITOFF_RXXG_CUT_THRU 0 + +/*---------------------------------------------------------------------------- + * Register 0x2062H - 0x2069: RXXG Exact Match VID + * Bit 11-0 RXXG_VID_MATCH + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXXG_VID_MATCH 0x0FFF +#define SUNI1x10GEXP_BITOFF_RXXG_VID_MATCH 0 + +/*---------------------------------------------------------------------------- + * Register 0x206EH - 0x206F: RXXG Address Filter Control + * Bit 3 RXXG_FORWARD_ENABLE + * Bit 2 RXXG_VLAN_ENABLE + * Bit 1 RXXG_SRC_ADDR + * Bit 0 RXXG_MATCH_ENABLE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXXG_FORWARD_ENABLE 0x0008 +#define SUNI1x10GEXP_BITMSK_RXXG_VLAN_ENABLE 0x0004 +#define SUNI1x10GEXP_BITMSK_RXXG_SRC_ADDR 0x0002 +#define SUNI1x10GEXP_BITMSK_RXXG_MATCH_ENABLE 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x2070: RXXG Address Filter Control 2 + * Bit 1 RXXG_PMODE + * Bit 0 RXXG_MHASH_EN + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXXG_PMODE 0x0002 +#define SUNI1x10GEXP_BITMSK_RXXG_MHASH_EN 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x2081: XRF Control Register 2 + * Bit 6 EN_PKT_GEN + * Bit 4-2 PATT + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EN_PKT_GEN 0x0040 +#define SUNI1x10GEXP_BITMSK_PATT 0x001C +#define SUNI1x10GEXP_BITOFF_PATT 2 + +/*---------------------------------------------------------------------------- + * Register 0x2088: XRF Interrupt Enable + * Bit 12-9 LANE_HICERE + * Bit 8-5 HS_SD_LANEE + * Bit 4 ALIGN_STATUS_ERRE + * Bit 3-0 LANE_SYNC_STAT_ERRE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_LANE_HICERE 0x1E00 +#define SUNI1x10GEXP_BITOFF_LANE_HICERE 9 +#define SUNI1x10GEXP_BITMSK_HS_SD_LANEE 0x01E0 +#define SUNI1x10GEXP_BITOFF_HS_SD_LANEE 5 +#define SUNI1x10GEXP_BITMSK_ALIGN_STATUS_ERRE 0x0010 +#define SUNI1x10GEXP_BITMSK_LANE_SYNC_STAT_ERRE 0x000F +#define SUNI1x10GEXP_BITOFF_LANE_SYNC_STAT_ERRE 0 + +/*---------------------------------------------------------------------------- + * Register 0x2089: XRF Interrupt Status + * Bit 12-9 LANE_HICERI + * Bit 8-5 HS_SD_LANEI + * Bit 4 ALIGN_STATUS_ERRI + * Bit 3-0 LANE_SYNC_STAT_ERRI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_LANE_HICERI 0x1E00 +#define SUNI1x10GEXP_BITOFF_LANE_HICERI 9 +#define SUNI1x10GEXP_BITMSK_HS_SD_LANEI 0x01E0 +#define SUNI1x10GEXP_BITOFF_HS_SD_LANEI 5 +#define SUNI1x10GEXP_BITMSK_ALIGN_STATUS_ERRI 0x0010 +#define SUNI1x10GEXP_BITMSK_LANE_SYNC_STAT_ERRI 0x000F +#define SUNI1x10GEXP_BITOFF_LANE_SYNC_STAT_ERRI 0 + +/*---------------------------------------------------------------------------- + * Register 0x208A: XRF Error Status + * Bit 8-5 HS_SD_LANE + * Bit 4 ALIGN_STATUS_ERR + * Bit 3-0 LANE_SYNC_STAT_ERR + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_HS_SD_LANE3 0x0100 +#define SUNI1x10GEXP_BITMSK_HS_SD_LANE2 0x0080 +#define SUNI1x10GEXP_BITMSK_HS_SD_LANE1 0x0040 +#define SUNI1x10GEXP_BITMSK_HS_SD_LANE0 0x0020 +#define SUNI1x10GEXP_BITMSK_ALIGN_STATUS_ERR 0x0010 +#define SUNI1x10GEXP_BITMSK_LANE3_SYNC_STAT_ERR 0x0008 +#define SUNI1x10GEXP_BITMSK_LANE2_SYNC_STAT_ERR 0x0004 +#define SUNI1x10GEXP_BITMSK_LANE1_SYNC_STAT_ERR 0x0002 +#define SUNI1x10GEXP_BITMSK_LANE0_SYNC_STAT_ERR 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x208B: XRF Diagnostic Interrupt Enable + * Bit 7-4 LANE_OVERRUNE + * Bit 3-0 LANE_UNDERRUNE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_LANE_OVERRUNE 0x00F0 +#define SUNI1x10GEXP_BITOFF_LANE_OVERRUNE 4 +#define SUNI1x10GEXP_BITMSK_LANE_UNDERRUNE 0x000F +#define SUNI1x10GEXP_BITOFF_LANE_UNDERRUNE 0 + +/*---------------------------------------------------------------------------- + * Register 0x208C: XRF Diagnostic Interrupt Status + * Bit 7-4 LANE_OVERRUNI + * Bit 3-0 LANE_UNDERRUNI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_LANE_OVERRUNI 0x00F0 +#define SUNI1x10GEXP_BITOFF_LANE_OVERRUNI 4 +#define SUNI1x10GEXP_BITMSK_LANE_UNDERRUNI 0x000F +#define SUNI1x10GEXP_BITOFF_LANE_UNDERRUNI 0 + +/*---------------------------------------------------------------------------- + * Register 0x20C0: RXOAM Configuration + * Bit 15 RXOAM_BUSY + * Bit 14-12 RXOAM_F2_SEL + * Bit 10-8 RXOAM_F1_SEL + * Bit 7-6 RXOAM_FILTER_CTRL + * Bit 5-0 RXOAM_PX_EN + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXOAM_BUSY 0x8000 +#define SUNI1x10GEXP_BITMSK_RXOAM_F2_SEL 0x7000 +#define SUNI1x10GEXP_BITOFF_RXOAM_F2_SEL 12 +#define SUNI1x10GEXP_BITMSK_RXOAM_F1_SEL 0x0700 +#define SUNI1x10GEXP_BITOFF_RXOAM_F1_SEL 8 +#define SUNI1x10GEXP_BITMSK_RXOAM_FILTER_CTRL 0x00C0 +#define SUNI1x10GEXP_BITOFF_RXOAM_FILTER_CTRL 6 +#define SUNI1x10GEXP_BITMSK_RXOAM_PX_EN 0x003F +#define SUNI1x10GEXP_BITOFF_RXOAM_PX_EN 0 + +/*---------------------------------------------------------------------------- + * Register 0x20C1,0x20C2: RXOAM Filter Configuration + * Bit 15-8 RXOAM_FX_MASK + * Bit 7-0 RXOAM_FX_VAL + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXOAM_FX_MASK 0xFF00 +#define SUNI1x10GEXP_BITOFF_RXOAM_FX_MASK 8 +#define SUNI1x10GEXP_BITMSK_RXOAM_FX_VAL 0x00FF +#define SUNI1x10GEXP_BITOFF_RXOAM_FX_VAl 0 + +/*---------------------------------------------------------------------------- + * Register 0x20C3: RXOAM Configuration Register 2 + * Bit 13 RXOAM_REC_BYTE_VAL + * Bit 11-10 RXOAM_BYPASS_MODE + * Bit 5-0 RXOAM_PX_CLEAR + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXOAM_REC_BYTE_VAL 0x2000 +#define SUNI1x10GEXP_BITMSK_RXOAM_BYPASS_MODE 0x0C00 +#define SUNI1x10GEXP_BITOFF_RXOAM_BYPASS_MODE 10 +#define SUNI1x10GEXP_BITMSK_RXOAM_PX_CLEAR 0x003F +#define SUNI1x10GEXP_BITOFF_RXOAM_PX_CLEAR 0 + +/*---------------------------------------------------------------------------- + * Register 0x20C4: RXOAM HEC Configuration + * Bit 15-8 RXOAM_COSET + * Bit 2 RXOAM_HEC_ERR_PKT + * Bit 0 RXOAM_HEC_EN + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXOAM_COSET 0xFF00 +#define SUNI1x10GEXP_BITOFF_RXOAM_COSET 8 +#define SUNI1x10GEXP_BITMSK_RXOAM_HEC_ERR_PKT 0x0004 +#define SUNI1x10GEXP_BITMSK_RXOAM_HEC_EN 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x20C7: RXOAM Interrupt Enable + * Bit 10 RXOAM_FILTER_THRSHE + * Bit 9 RXOAM_OAM_ERRE + * Bit 8 RXOAM_HECE_THRSHE + * Bit 7 RXOAM_SOPE + * Bit 6 RXOAM_RFE + * Bit 5 RXOAM_LFE + * Bit 4 RXOAM_DV_ERRE + * Bit 3 RXOAM_DATA_INVALIDE + * Bit 2 RXOAM_FILTER_DROPE + * Bit 1 RXOAM_HECE + * Bit 0 RXOAM_OFLE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXOAM_FILTER_THRSHE 0x0400 +#define SUNI1x10GEXP_BITMSK_RXOAM_OAM_ERRE 0x0200 +#define SUNI1x10GEXP_BITMSK_RXOAM_HECE_THRSHE 0x0100 +#define SUNI1x10GEXP_BITMSK_RXOAM_SOPE 0x0080 +#define SUNI1x10GEXP_BITMSK_RXOAM_RFE 0x0040 +#define SUNI1x10GEXP_BITMSK_RXOAM_LFE 0x0020 +#define SUNI1x10GEXP_BITMSK_RXOAM_DV_ERRE 0x0010 +#define SUNI1x10GEXP_BITMSK_RXOAM_DATA_INVALIDE 0x0008 +#define SUNI1x10GEXP_BITMSK_RXOAM_FILTER_DROPE 0x0004 +#define SUNI1x10GEXP_BITMSK_RXOAM_HECE 0x0002 +#define SUNI1x10GEXP_BITMSK_RXOAM_OFLE 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x20C8: RXOAM Interrupt Status + * Bit 10 RXOAM_FILTER_THRSHI + * Bit 9 RXOAM_OAM_ERRI + * Bit 8 RXOAM_HECE_THRSHI + * Bit 7 RXOAM_SOPI + * Bit 6 RXOAM_RFI + * Bit 5 RXOAM_LFI + * Bit 4 RXOAM_DV_ERRI + * Bit 3 RXOAM_DATA_INVALIDI + * Bit 2 RXOAM_FILTER_DROPI + * Bit 1 RXOAM_HECI + * Bit 0 RXOAM_OFLI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXOAM_FILTER_THRSHI 0x0400 +#define SUNI1x10GEXP_BITMSK_RXOAM_OAM_ERRI 0x0200 +#define SUNI1x10GEXP_BITMSK_RXOAM_HECE_THRSHI 0x0100 +#define SUNI1x10GEXP_BITMSK_RXOAM_SOPI 0x0080 +#define SUNI1x10GEXP_BITMSK_RXOAM_RFI 0x0040 +#define SUNI1x10GEXP_BITMSK_RXOAM_LFI 0x0020 +#define SUNI1x10GEXP_BITMSK_RXOAM_DV_ERRI 0x0010 +#define SUNI1x10GEXP_BITMSK_RXOAM_DATA_INVALIDI 0x0008 +#define SUNI1x10GEXP_BITMSK_RXOAM_FILTER_DROPI 0x0004 +#define SUNI1x10GEXP_BITMSK_RXOAM_HECI 0x0002 +#define SUNI1x10GEXP_BITMSK_RXOAM_OFLI 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x20C9: RXOAM Status + * Bit 10 RXOAM_FILTER_THRSHV + * Bit 8 RXOAM_HECE_THRSHV + * Bit 6 RXOAM_RFV + * Bit 5 RXOAM_LFV + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_RXOAM_FILTER_THRSHV 0x0400 +#define SUNI1x10GEXP_BITMSK_RXOAM_HECE_THRSHV 0x0100 +#define SUNI1x10GEXP_BITMSK_RXOAM_RFV 0x0040 +#define SUNI1x10GEXP_BITMSK_RXOAM_LFV 0x0020 + +/*---------------------------------------------------------------------------- + * Register 0x2100: MSTAT Control + * Bit 2 MSTAT_WRITE + * Bit 1 MSTAT_CLEAR + * Bit 0 MSTAT_SNAP + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_MSTAT_WRITE 0x0004 +#define SUNI1x10GEXP_BITMSK_MSTAT_CLEAR 0x0002 +#define SUNI1x10GEXP_BITMSK_MSTAT_SNAP 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x2109: MSTAT Counter Write Address + * Bit 5-0 MSTAT_WRITE_ADDRESS + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_MSTAT_WRITE_ADDRESS 0x003F +#define SUNI1x10GEXP_BITOFF_MSTAT_WRITE_ADDRESS 0 + +/*---------------------------------------------------------------------------- + * Register 0x2200: IFLX Global Configuration Register + * Bit 15 IFLX_IRCU_ENABLE + * Bit 14 IFLX_IDSWT_ENABLE + * Bit 13-0 IFLX_IFD_CNT + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_IFLX_IRCU_ENABLE 0x8000 +#define SUNI1x10GEXP_BITMSK_IFLX_IDSWT_ENABLE 0x4000 +#define SUNI1x10GEXP_BITMSK_IFLX_IFD_CNT 0x3FFF +#define SUNI1x10GEXP_BITOFF_IFLX_IFD_CNT 0 + +/*---------------------------------------------------------------------------- + * Register 0x2209: IFLX FIFO Overflow Enable + * Bit 0 IFLX_OVFE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_IFLX_OVFE 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x220A: IFLX FIFO Overflow Interrupt + * Bit 0 IFLX_OVFI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_IFLX_OVFI 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x220D: IFLX Indirect Channel Address + * Bit 15 IFLX_BUSY + * Bit 14 IFLX_RWB + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_IFLX_BUSY 0x8000 +#define SUNI1x10GEXP_BITMSK_IFLX_RWB 0x4000 + +/*---------------------------------------------------------------------------- + * Register 0x220E: IFLX Indirect Logical FIFO Low Limit & Provision + * Bit 9-0 IFLX_LOLIM + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_IFLX_LOLIM 0x03FF +#define SUNI1x10GEXP_BITOFF_IFLX_LOLIM 0 + +/*---------------------------------------------------------------------------- + * Register 0x220F: IFLX Indirect Logical FIFO High Limit + * Bit 9-0 IFLX_HILIM + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_IFLX_HILIM 0x03FF +#define SUNI1x10GEXP_BITOFF_IFLX_HILIM 0 + +/*---------------------------------------------------------------------------- + * Register 0x2210: IFLX Indirect Full/Almost Full Status & Limit + * Bit 15 IFLX_FULL + * Bit 14 IFLX_AFULL + * Bit 13-0 IFLX_AFTH + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_IFLX_FULL 0x8000 +#define SUNI1x10GEXP_BITMSK_IFLX_AFULL 0x4000 +#define SUNI1x10GEXP_BITMSK_IFLX_AFTH 0x3FFF +#define SUNI1x10GEXP_BITOFF_IFLX_AFTH 0 + +/*---------------------------------------------------------------------------- + * Register 0x2211: IFLX Indirect Empty/Almost Empty Status & Limit + * Bit 15 IFLX_EMPTY + * Bit 14 IFLX_AEMPTY + * Bit 13-0 IFLX_AETH + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_IFLX_EMPTY 0x8000 +#define SUNI1x10GEXP_BITMSK_IFLX_AEMPTY 0x4000 +#define SUNI1x10GEXP_BITMSK_IFLX_AETH 0x3FFF +#define SUNI1x10GEXP_BITOFF_IFLX_AETH 0 + +/*---------------------------------------------------------------------------- + * Register 0x2240: PL4MOS Configuration Register + * Bit 3 PL4MOS_RE_INIT + * Bit 2 PL4MOS_EN + * Bit 1 PL4MOS_NO_STATUS + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4MOS_RE_INIT 0x0008 +#define SUNI1x10GEXP_BITMSK_PL4MOS_EN 0x0004 +#define SUNI1x10GEXP_BITMSK_PL4MOS_NO_STATUS 0x0002 + +/*---------------------------------------------------------------------------- + * Register 0x2243: PL4MOS MaxBurst1 Register + * Bit 11-0 PL4MOS_MAX_BURST1 + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4MOS_MAX_BURST1 0x0FFF +#define SUNI1x10GEXP_BITOFF_PL4MOS_MAX_BURST1 0 + +/*---------------------------------------------------------------------------- + * Register 0x2244: PL4MOS MaxBurst2 Register + * Bit 11-0 PL4MOS_MAX_BURST2 + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4MOS_MAX_BURST2 0x0FFF +#define SUNI1x10GEXP_BITOFF_PL4MOS_MAX_BURST2 0 + +/*---------------------------------------------------------------------------- + * Register 0x2245: PL4MOS Transfer Size Register + * Bit 7-0 PL4MOS_MAX_TRANSFER + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4MOS_MAX_TRANSFER 0x00FF +#define SUNI1x10GEXP_BITOFF_PL4MOS_MAX_TRANSFER 0 + +/*---------------------------------------------------------------------------- + * Register 0x2280: PL4ODP Configuration + * Bit 15-12 PL4ODP_REPEAT_T + * Bit 8 PL4ODP_SOP_RULE + * Bit 1 PL4ODP_EN_PORTS + * Bit 0 PL4ODP_EN_DFWD + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4ODP_REPEAT_T 0xF000 +#define SUNI1x10GEXP_BITOFF_PL4ODP_REPEAT_T 12 +#define SUNI1x10GEXP_BITMSK_PL4ODP_SOP_RULE 0x0100 +#define SUNI1x10GEXP_BITMSK_PL4ODP_EN_PORTS 0x0002 +#define SUNI1x10GEXP_BITMSK_PL4ODP_EN_DFWD 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x2282: PL4ODP Interrupt Mask + * Bit 0 PL4ODP_OUT_DISE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4ODP_OUT_DISE 0x0001 + + + +#define SUNI1x10GEXP_BITMSK_PL4ODP_PPE_EOPEOBE 0x0080 +#define SUNI1x10GEXP_BITMSK_PL4ODP_PPE_ERREOPE 0x0040 +#define SUNI1x10GEXP_BITMSK_PL4ODP_PPE_MEOPE 0x0008 +#define SUNI1x10GEXP_BITMSK_PL4ODP_PPE_MSOPE 0x0004 +#define SUNI1x10GEXP_BITMSK_PL4ODP_ES_OVRE 0x0002 + + +/*---------------------------------------------------------------------------- + * Register 0x2283: PL4ODP Interrupt + * Bit 0 PL4ODP_OUT_DISI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4ODP_OUT_DISI 0x0001 + + + +#define SUNI1x10GEXP_BITMSK_PL4ODP_PPE_EOPEOBI 0x0080 +#define SUNI1x10GEXP_BITMSK_PL4ODP_PPE_ERREOPI 0x0040 +#define SUNI1x10GEXP_BITMSK_PL4ODP_PPE_MEOPI 0x0008 +#define SUNI1x10GEXP_BITMSK_PL4ODP_PPE_MSOPI 0x0004 +#define SUNI1x10GEXP_BITMSK_PL4ODP_ES_OVRI 0x0002 + +/*---------------------------------------------------------------------------- + * Register 0x2300: PL4IO Lock Detect Status + * Bit 15 PL4IO_OUT_ROOLV + * Bit 12 PL4IO_IS_ROOLV + * Bit 11 PL4IO_DIP2_ERRV + * Bit 8 PL4IO_ID_ROOLV + * Bit 4 PL4IO_IS_DOOLV + * Bit 0 PL4IO_ID_DOOLV + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4IO_OUT_ROOLV 0x8000 +#define SUNI1x10GEXP_BITMSK_PL4IO_IS_ROOLV 0x1000 +#define SUNI1x10GEXP_BITMSK_PL4IO_DIP2_ERRV 0x0800 +#define SUNI1x10GEXP_BITMSK_PL4IO_ID_ROOLV 0x0100 +#define SUNI1x10GEXP_BITMSK_PL4IO_IS_DOOLV 0x0010 +#define SUNI1x10GEXP_BITMSK_PL4IO_ID_DOOLV 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x2301: PL4IO Lock Detect Change + * Bit 15 PL4IO_OUT_ROOLI + * Bit 12 PL4IO_IS_ROOLI + * Bit 11 PL4IO_DIP2_ERRI + * Bit 8 PL4IO_ID_ROOLI + * Bit 4 PL4IO_IS_DOOLI + * Bit 0 PL4IO_ID_DOOLI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4IO_OUT_ROOLI 0x8000 +#define SUNI1x10GEXP_BITMSK_PL4IO_IS_ROOLI 0x1000 +#define SUNI1x10GEXP_BITMSK_PL4IO_DIP2_ERRI 0x0800 +#define SUNI1x10GEXP_BITMSK_PL4IO_ID_ROOLI 0x0100 +#define SUNI1x10GEXP_BITMSK_PL4IO_IS_DOOLI 0x0010 +#define SUNI1x10GEXP_BITMSK_PL4IO_ID_DOOLI 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x2302: PL4IO Lock Detect Mask + * Bit 15 PL4IO_OUT_ROOLE + * Bit 12 PL4IO_IS_ROOLE + * Bit 11 PL4IO_DIP2_ERRE + * Bit 8 PL4IO_ID_ROOLE + * Bit 4 PL4IO_IS_DOOLE + * Bit 0 PL4IO_ID_DOOLE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4IO_OUT_ROOLE 0x8000 +#define SUNI1x10GEXP_BITMSK_PL4IO_IS_ROOLE 0x1000 +#define SUNI1x10GEXP_BITMSK_PL4IO_DIP2_ERRE 0x0800 +#define SUNI1x10GEXP_BITMSK_PL4IO_ID_ROOLE 0x0100 +#define SUNI1x10GEXP_BITMSK_PL4IO_IS_DOOLE 0x0010 +#define SUNI1x10GEXP_BITMSK_PL4IO_ID_DOOLE 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x2303: PL4IO Lock Detect Limits + * Bit 15-8 PL4IO_REF_LIMIT + * Bit 7-0 PL4IO_TRAN_LIMIT + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4IO_REF_LIMIT 0xFF00 +#define SUNI1x10GEXP_BITOFF_PL4IO_REF_LIMIT 8 +#define SUNI1x10GEXP_BITMSK_PL4IO_TRAN_LIMIT 0x00FF +#define SUNI1x10GEXP_BITOFF_PL4IO_TRAN_LIMIT 0 + +/*---------------------------------------------------------------------------- + * Register 0x2304: PL4IO Calendar Repetitions + * Bit 15-8 PL4IO_IN_MUL + * Bit 7-0 PL4IO_OUT_MUL + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4IO_IN_MUL 0xFF00 +#define SUNI1x10GEXP_BITOFF_PL4IO_IN_MUL 8 +#define SUNI1x10GEXP_BITMSK_PL4IO_OUT_MUL 0x00FF +#define SUNI1x10GEXP_BITOFF_PL4IO_OUT_MUL 0 + +/*---------------------------------------------------------------------------- + * Register 0x2305: PL4IO Configuration + * Bit 15 PL4IO_DIP2_ERR_CHK + * Bit 11 PL4IO_ODAT_DIS + * Bit 10 PL4IO_TRAIN_DIS + * Bit 9 PL4IO_OSTAT_DIS + * Bit 8 PL4IO_ISTAT_DIS + * Bit 7 PL4IO_NO_ISTAT + * Bit 6 PL4IO_STAT_OUTSEL + * Bit 5 PL4IO_INSEL + * Bit 4 PL4IO_DLSEL + * Bit 1-0 PL4IO_OUTSEL + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4IO_DIP2_ERR_CHK 0x8000 +#define SUNI1x10GEXP_BITMSK_PL4IO_ODAT_DIS 0x0800 +#define SUNI1x10GEXP_BITMSK_PL4IO_TRAIN_DIS 0x0400 +#define SUNI1x10GEXP_BITMSK_PL4IO_OSTAT_DIS 0x0200 +#define SUNI1x10GEXP_BITMSK_PL4IO_ISTAT_DIS 0x0100 +#define SUNI1x10GEXP_BITMSK_PL4IO_NO_ISTAT 0x0080 +#define SUNI1x10GEXP_BITMSK_PL4IO_STAT_OUTSEL 0x0040 +#define SUNI1x10GEXP_BITMSK_PL4IO_INSEL 0x0020 +#define SUNI1x10GEXP_BITMSK_PL4IO_DLSEL 0x0010 +#define SUNI1x10GEXP_BITMSK_PL4IO_OUTSEL 0x0003 +#define SUNI1x10GEXP_BITOFF_PL4IO_OUTSEL 0 + +/*---------------------------------------------------------------------------- + * Register 0x3040: TXXG Configuration Register 1 + * Bit 15 TXXG_TXEN0 + * Bit 13 TXXG_HOSTPAUSE + * Bit 12-7 TXXG_IPGT + * Bit 5 TXXG_32BIT_ALIGN + * Bit 4 TXXG_CRCEN + * Bit 3 TXXG_FCTX + * Bit 2 TXXG_FCRX + * Bit 1 TXXG_PADEN + * Bit 0 TXXG_SPRE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXXG_TXEN0 0x8000 +#define SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE 0x2000 +#define SUNI1x10GEXP_BITMSK_TXXG_IPGT 0x1F80 +#define SUNI1x10GEXP_BITOFF_TXXG_IPGT 7 +#define SUNI1x10GEXP_BITMSK_TXXG_32BIT_ALIGN 0x0020 +#define SUNI1x10GEXP_BITMSK_TXXG_CRCEN 0x0010 +#define SUNI1x10GEXP_BITMSK_TXXG_FCTX 0x0008 +#define SUNI1x10GEXP_BITMSK_TXXG_FCRX 0x0004 +#define SUNI1x10GEXP_BITMSK_TXXG_PADEN 0x0002 +#define SUNI1x10GEXP_BITMSK_TXXG_SPRE 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x3041: TXXG Configuration Register 2 + * Bit 7-0 TXXG_HDRSIZE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXXG_HDRSIZE 0x00FF + +/*---------------------------------------------------------------------------- + * Register 0x3042: TXXG Configuration Register 3 + * Bit 15 TXXG_FIFO_ERRE + * Bit 14 TXXG_FIFO_UDRE + * Bit 13 TXXG_MAX_LERRE + * Bit 12 TXXG_MIN_LERRE + * Bit 11 TXXG_XFERE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXXG_FIFO_ERRE 0x8000 +#define SUNI1x10GEXP_BITMSK_TXXG_FIFO_UDRE 0x4000 +#define SUNI1x10GEXP_BITMSK_TXXG_MAX_LERRE 0x2000 +#define SUNI1x10GEXP_BITMSK_TXXG_MIN_LERRE 0x1000 +#define SUNI1x10GEXP_BITMSK_TXXG_XFERE 0x0800 + +/*---------------------------------------------------------------------------- + * Register 0x3043: TXXG Interrupt + * Bit 15 TXXG_FIFO_ERRI + * Bit 14 TXXG_FIFO_UDRI + * Bit 13 TXXG_MAX_LERRI + * Bit 12 TXXG_MIN_LERRI + * Bit 11 TXXG_XFERI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXXG_FIFO_ERRI 0x8000 +#define SUNI1x10GEXP_BITMSK_TXXG_FIFO_UDRI 0x4000 +#define SUNI1x10GEXP_BITMSK_TXXG_MAX_LERRI 0x2000 +#define SUNI1x10GEXP_BITMSK_TXXG_MIN_LERRI 0x1000 +#define SUNI1x10GEXP_BITMSK_TXXG_XFERI 0x0800 + +/*---------------------------------------------------------------------------- + * Register 0x3044: TXXG Status Register + * Bit 1 TXXG_TXACTIVE + * Bit 0 TXXG_PAUSED + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXXG_TXACTIVE 0x0002 +#define SUNI1x10GEXP_BITMSK_TXXG_PAUSED 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x3046: TXXG TX_MINFR - Transmit Min Frame Size Register + * Bit 7-0 TXXG_TX_MINFR + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXXG_TX_MINFR 0x00FF +#define SUNI1x10GEXP_BITOFF_TXXG_TX_MINFR 0 + +/*---------------------------------------------------------------------------- + * Register 0x3052: TXXG Pause Quantum Value Configuration Register + * Bit 7-0 TXXG_FC_PAUSE_QNTM + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXXG_FC_PAUSE_QNTM 0x00FF +#define SUNI1x10GEXP_BITOFF_TXXG_FC_PAUSE_QNTM 0 + +/*---------------------------------------------------------------------------- + * Register 0x3080: XTEF Control + * Bit 3-0 XTEF_FORCE_PARITY_ERR + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_XTEF_FORCE_PARITY_ERR 0x000F +#define SUNI1x10GEXP_BITOFF_XTEF_FORCE_PARITY_ERR 0 + +/*---------------------------------------------------------------------------- + * Register 0x3084: XTEF Interrupt Event Register + * Bit 0 XTEF_LOST_SYNCI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_XTEF_LOST_SYNCI 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x3085: XTEF Interrupt Enable Register + * Bit 0 XTEF_LOST_SYNCE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_XTEF_LOST_SYNCE 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x3086: XTEF Visibility Register + * Bit 0 XTEF_LOST_SYNCV + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_XTEF_LOST_SYNCV 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x30C0: TXOAM OAM Configuration + * Bit 15 TXOAM_HEC_EN + * Bit 14 TXOAM_EMPTYCODE_EN + * Bit 13 TXOAM_FORCE_IDLE + * Bit 12 TXOAM_IGNORE_IDLE + * Bit 11-6 TXOAM_PX_OVERWRITE + * Bit 5-0 TXOAM_PX_SEL + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXOAM_HEC_EN 0x8000 +#define SUNI1x10GEXP_BITMSK_TXOAM_EMPTYCODE_EN 0x4000 +#define SUNI1x10GEXP_BITMSK_TXOAM_FORCE_IDLE 0x2000 +#define SUNI1x10GEXP_BITMSK_TXOAM_IGNORE_IDLE 0x1000 +#define SUNI1x10GEXP_BITMSK_TXOAM_PX_OVERWRITE 0x0FC0 +#define SUNI1x10GEXP_BITOFF_TXOAM_PX_OVERWRITE 6 +#define SUNI1x10GEXP_BITMSK_TXOAM_PX_SEL 0x003F +#define SUNI1x10GEXP_BITOFF_TXOAM_PX_SEL 0 + +/*---------------------------------------------------------------------------- + * Register 0x30C1: TXOAM Mini-Packet Rate Configuration + * Bit 15 TXOAM_MINIDIS + * Bit 14 TXOAM_BUSY + * Bit 13 TXOAM_TRANS_EN + * Bit 10-0 TXOAM_MINIRATE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXOAM_MINIDIS 0x8000 +#define SUNI1x10GEXP_BITMSK_TXOAM_BUSY 0x4000 +#define SUNI1x10GEXP_BITMSK_TXOAM_TRANS_EN 0x2000 +#define SUNI1x10GEXP_BITMSK_TXOAM_MINIRATE 0x07FF + +/*---------------------------------------------------------------------------- + * Register 0x30C2: TXOAM Mini-Packet Gap and FIFO Configuration + * Bit 13-10 TXOAM_FTHRESH + * Bit 9-6 TXOAM_MINIPOST + * Bit 5-0 TXOAM_MINIPRE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXOAM_FTHRESH 0x3C00 +#define SUNI1x10GEXP_BITOFF_TXOAM_FTHRESH 10 +#define SUNI1x10GEXP_BITMSK_TXOAM_MINIPOST 0x03C0 +#define SUNI1x10GEXP_BITOFF_TXOAM_MINIPOST 6 +#define SUNI1x10GEXP_BITMSK_TXOAM_MINIPRE 0x003F + +/*---------------------------------------------------------------------------- + * Register 0x30C6: TXOAM Interrupt Enable + * Bit 2 TXOAM_SOP_ERRE + * Bit 1 TXOAM_OFLE + * Bit 0 TXOAM_ERRE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXOAM_SOP_ERRE 0x0004 +#define SUNI1x10GEXP_BITMSK_TXOAM_OFLE 0x0002 +#define SUNI1x10GEXP_BITMSK_TXOAM_ERRE 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x30C7: TXOAM Interrupt Status + * Bit 2 TXOAM_SOP_ERRI + * Bit 1 TXOAM_OFLI + * Bit 0 TXOAM_ERRI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXOAM_SOP_ERRI 0x0004 +#define SUNI1x10GEXP_BITMSK_TXOAM_OFLI 0x0002 +#define SUNI1x10GEXP_BITMSK_TXOAM_ERRI 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x30CF: TXOAM Coset + * Bit 7-0 TXOAM_COSET + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_TXOAM_COSET 0x00FF + +/*---------------------------------------------------------------------------- + * Register 0x3200: EFLX Global Configuration + * Bit 15 EFLX_ERCU_EN + * Bit 7 EFLX_EN_EDSWT + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EFLX_ERCU_EN 0x8000 +#define SUNI1x10GEXP_BITMSK_EFLX_EN_EDSWT 0x0080 + +/*---------------------------------------------------------------------------- + * Register 0x3201: EFLX ERCU Global Status + * Bit 13 EFLX_OVF_ERR + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EFLX_OVF_ERR 0x2000 + +/*---------------------------------------------------------------------------- + * Register 0x3202: EFLX Indirect Channel Address + * Bit 15 EFLX_BUSY + * Bit 14 EFLX_RDWRB + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EFLX_BUSY 0x8000 +#define SUNI1x10GEXP_BITMSK_EFLX_RDWRB 0x4000 + +/*---------------------------------------------------------------------------- + * Register 0x3203: EFLX Indirect Logical FIFO Low Limit + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EFLX_LOLIM 0x03FF +#define SUNI1x10GEXP_BITOFF_EFLX_LOLIM 0 + +/*---------------------------------------------------------------------------- + * Register 0x3204: EFLX Indirect Logical FIFO High Limit + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EFLX_HILIM 0x03FF +#define SUNI1x10GEXP_BITOFF_EFLX_HILIM 0 + +/*---------------------------------------------------------------------------- + * Register 0x3205: EFLX Indirect Full/Almost-Full Status and Limit + * Bit 15 EFLX_FULL + * Bit 14 EFLX_AFULL + * Bit 13-0 EFLX_AFTH + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EFLX_FULL 0x8000 +#define SUNI1x10GEXP_BITMSK_EFLX_AFULL 0x4000 +#define SUNI1x10GEXP_BITMSK_EFLX_AFTH 0x3FFF +#define SUNI1x10GEXP_BITOFF_EFLX_AFTH 0 + +/*---------------------------------------------------------------------------- + * Register 0x3206: EFLX Indirect Empty/Almost-Empty Status and Limit + * Bit 15 EFLX_EMPTY + * Bit 14 EFLX_AEMPTY + * Bit 13-0 EFLX_AETH + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EFLX_EMPTY 0x8000 +#define SUNI1x10GEXP_BITMSK_EFLX_AEMPTY 0x4000 +#define SUNI1x10GEXP_BITMSK_EFLX_AETH 0x3FFF +#define SUNI1x10GEXP_BITOFF_EFLX_AETH 0 + +/*---------------------------------------------------------------------------- + * Register 0x3207: EFLX Indirect FIFO Cut-Through Threshold + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EFLX_CUT_THRU 0x3FFF +#define SUNI1x10GEXP_BITOFF_EFLX_CUT_THRU 0 + +/*---------------------------------------------------------------------------- + * Register 0x320C: EFLX FIFO Overflow Error Enable + * Bit 0 EFLX_OVFE + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EFLX_OVFE 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x320D: EFLX FIFO Overflow Error Indication + * Bit 0 EFLX_OVFI + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EFLX_OVFI 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x3210: EFLX Channel Provision + * Bit 0 EFLX_PROV + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_EFLX_PROV 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x3280: PL4IDU Configuration + * Bit 2 PL4IDU_SYNCH_ON_TRAIN + * Bit 1 PL4IDU_EN_PORTS + * Bit 0 PL4IDU_EN_DFWD + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4IDU_SYNCH_ON_TRAIN 0x0004 +#define SUNI1x10GEXP_BITMSK_PL4IDU_EN_PORTS 0x0002 +#define SUNI1x10GEXP_BITMSK_PL4IDU_EN_DFWD 0x0001 + +/*---------------------------------------------------------------------------- + * Register 0x3282: PL4IDU Interrupt Mask + * Bit 1 PL4IDU_DIP4E + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4IDU_DIP4E 0x0002 + +/*---------------------------------------------------------------------------- + * Register 0x3283: PL4IDU Interrupt + * Bit 1 PL4IDU_DIP4I + *----------------------------------------------------------------------------*/ +#define SUNI1x10GEXP_BITMSK_PL4IDU_DIP4I 0x0002 + +#endif /* _CXGB_SUNI1x10GEXP_REGS_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb/tp.c b/drivers/net/ethernet/chelsio/cxgb/tp.c new file mode 100644 index 000000000..4337cee07 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/tp.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0 +/* $Date: 2006/02/07 04:21:54 $ $RCSfile: tp.c,v $ $Revision: 1.73 $ */ +#include "common.h" +#include "regs.h" +#include "tp.h" +#ifdef CONFIG_CHELSIO_T1_1G +#include "fpga_defs.h" +#endif + +struct petp { + adapter_t *adapter; +}; + +/* Pause deadlock avoidance parameters */ +#define DROP_MSEC 16 +#define DROP_PKTS_CNT 1 + +static void tp_init(adapter_t * ap, const struct tp_params *p, + unsigned int tp_clk) +{ + u32 val; + + if (!t1_is_asic(ap)) + return; + + val = F_TP_IN_CSPI_CPL | F_TP_IN_CSPI_CHECK_IP_CSUM | + F_TP_IN_CSPI_CHECK_TCP_CSUM | F_TP_IN_ESPI_ETHERNET; + if (!p->pm_size) + val |= F_OFFLOAD_DISABLE; + else + val |= F_TP_IN_ESPI_CHECK_IP_CSUM | F_TP_IN_ESPI_CHECK_TCP_CSUM; + writel(val, ap->regs + A_TP_IN_CONFIG); + writel(F_TP_OUT_CSPI_CPL | + F_TP_OUT_ESPI_ETHERNET | + F_TP_OUT_ESPI_GENERATE_IP_CSUM | + F_TP_OUT_ESPI_GENERATE_TCP_CSUM, ap->regs + A_TP_OUT_CONFIG); + writel(V_IP_TTL(64) | + F_PATH_MTU /* IP DF bit */ | + V_5TUPLE_LOOKUP(p->use_5tuple_mode) | + V_SYN_COOKIE_PARAMETER(29), ap->regs + A_TP_GLOBAL_CONFIG); + /* + * Enable pause frame deadlock prevention. + */ + if (is_T2(ap) && ap->params.nports > 1) { + u32 drop_ticks = DROP_MSEC * (tp_clk / 1000); + + writel(F_ENABLE_TX_DROP | F_ENABLE_TX_ERROR | + V_DROP_TICKS_CNT(drop_ticks) | + V_NUM_PKTS_DROPPED(DROP_PKTS_CNT), + ap->regs + A_TP_TX_DROP_CONFIG); + } +} + +void t1_tp_destroy(struct petp *tp) +{ + kfree(tp); +} + +struct petp *t1_tp_create(adapter_t *adapter, struct tp_params *p) +{ + struct petp *tp = kzalloc(sizeof(*tp), GFP_KERNEL); + + if (!tp) + return NULL; + + tp->adapter = adapter; + + return tp; +} + +void t1_tp_intr_enable(struct petp *tp) +{ + u32 tp_intr = readl(tp->adapter->regs + A_PL_ENABLE); + +#ifdef CONFIG_CHELSIO_T1_1G + if (!t1_is_asic(tp->adapter)) { + /* FPGA */ + writel(0xffffffff, + tp->adapter->regs + FPGA_TP_ADDR_INTERRUPT_ENABLE); + writel(tp_intr | FPGA_PCIX_INTERRUPT_TP, + tp->adapter->regs + A_PL_ENABLE); + } else +#endif + { + /* We don't use any TP interrupts */ + writel(0, tp->adapter->regs + A_TP_INT_ENABLE); + writel(tp_intr | F_PL_INTR_TP, + tp->adapter->regs + A_PL_ENABLE); + } +} + +void t1_tp_intr_disable(struct petp *tp) +{ + u32 tp_intr = readl(tp->adapter->regs + A_PL_ENABLE); + +#ifdef CONFIG_CHELSIO_T1_1G + if (!t1_is_asic(tp->adapter)) { + /* FPGA */ + writel(0, tp->adapter->regs + FPGA_TP_ADDR_INTERRUPT_ENABLE); + writel(tp_intr & ~FPGA_PCIX_INTERRUPT_TP, + tp->adapter->regs + A_PL_ENABLE); + } else +#endif + { + writel(0, tp->adapter->regs + A_TP_INT_ENABLE); + writel(tp_intr & ~F_PL_INTR_TP, + tp->adapter->regs + A_PL_ENABLE); + } +} + +void t1_tp_intr_clear(struct petp *tp) +{ +#ifdef CONFIG_CHELSIO_T1_1G + if (!t1_is_asic(tp->adapter)) { + writel(0xffffffff, + tp->adapter->regs + FPGA_TP_ADDR_INTERRUPT_CAUSE); + writel(FPGA_PCIX_INTERRUPT_TP, tp->adapter->regs + A_PL_CAUSE); + return; + } +#endif + writel(0xffffffff, tp->adapter->regs + A_TP_INT_CAUSE); + writel(F_PL_INTR_TP, tp->adapter->regs + A_PL_CAUSE); +} + +int t1_tp_intr_handler(struct petp *tp) +{ + u32 cause; + +#ifdef CONFIG_CHELSIO_T1_1G + /* FPGA doesn't support TP interrupts. */ + if (!t1_is_asic(tp->adapter)) + return 1; +#endif + + cause = readl(tp->adapter->regs + A_TP_INT_CAUSE); + writel(cause, tp->adapter->regs + A_TP_INT_CAUSE); + return 0; +} + +static void set_csum_offload(struct petp *tp, u32 csum_bit, int enable) +{ + u32 val = readl(tp->adapter->regs + A_TP_GLOBAL_CONFIG); + + if (enable) + val |= csum_bit; + else + val &= ~csum_bit; + writel(val, tp->adapter->regs + A_TP_GLOBAL_CONFIG); +} + +void t1_tp_set_ip_checksum_offload(struct petp *tp, int enable) +{ + set_csum_offload(tp, F_IP_CSUM, enable); +} + +void t1_tp_set_tcp_checksum_offload(struct petp *tp, int enable) +{ + set_csum_offload(tp, F_TCP_CSUM, enable); +} + +/* + * Initialize TP state. tp_params contains initial settings for some TP + * parameters, particularly the one-time PM and CM settings. + */ +int t1_tp_reset(struct petp *tp, struct tp_params *p, unsigned int tp_clk) +{ + adapter_t *adapter = tp->adapter; + + tp_init(adapter, p, tp_clk); + writel(F_TP_RESET, adapter->regs + A_TP_RESET); + return 0; +} diff --git a/drivers/net/ethernet/chelsio/cxgb/tp.h b/drivers/net/ethernet/chelsio/cxgb/tp.h new file mode 100644 index 000000000..ba15675d5 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/tp.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* $Date: 2005/03/07 23:59:05 $ $RCSfile: tp.h,v $ $Revision: 1.20 $ */ +#ifndef CHELSIO_TP_H +#define CHELSIO_TP_H + +#include "common.h" + +#define TP_MAX_RX_COALESCING_SIZE 16224U + +struct tp_mib_statistics { + + /* IP */ + u32 ipInReceive_hi; + u32 ipInReceive_lo; + u32 ipInHdrErrors_hi; + u32 ipInHdrErrors_lo; + u32 ipInAddrErrors_hi; + u32 ipInAddrErrors_lo; + u32 ipInUnknownProtos_hi; + u32 ipInUnknownProtos_lo; + u32 ipInDiscards_hi; + u32 ipInDiscards_lo; + u32 ipInDelivers_hi; + u32 ipInDelivers_lo; + u32 ipOutRequests_hi; + u32 ipOutRequests_lo; + u32 ipOutDiscards_hi; + u32 ipOutDiscards_lo; + u32 ipOutNoRoutes_hi; + u32 ipOutNoRoutes_lo; + u32 ipReasmTimeout; + u32 ipReasmReqds; + u32 ipReasmOKs; + u32 ipReasmFails; + + u32 reserved[8]; + + /* TCP */ + u32 tcpActiveOpens; + u32 tcpPassiveOpens; + u32 tcpAttemptFails; + u32 tcpEstabResets; + u32 tcpOutRsts; + u32 tcpCurrEstab; + u32 tcpInSegs_hi; + u32 tcpInSegs_lo; + u32 tcpOutSegs_hi; + u32 tcpOutSegs_lo; + u32 tcpRetransSeg_hi; + u32 tcpRetransSeg_lo; + u32 tcpInErrs_hi; + u32 tcpInErrs_lo; + u32 tcpRtoMin; + u32 tcpRtoMax; +}; + +struct petp; +struct tp_params; + +struct petp *t1_tp_create(adapter_t *adapter, struct tp_params *p); +void t1_tp_destroy(struct petp *tp); + +void t1_tp_intr_disable(struct petp *tp); +void t1_tp_intr_enable(struct petp *tp); +void t1_tp_intr_clear(struct petp *tp); +int t1_tp_intr_handler(struct petp *tp); + +void t1_tp_get_mib_statistics(adapter_t *adap, struct tp_mib_statistics *tps); +void t1_tp_set_tcp_checksum_offload(struct petp *tp, int enable); +void t1_tp_set_ip_checksum_offload(struct petp *tp, int enable); +int t1_tp_set_coalescing_size(struct petp *tp, unsigned int size); +int t1_tp_reset(struct petp *tp, struct tp_params *p, unsigned int tp_clk); +#endif diff --git a/drivers/net/ethernet/chelsio/cxgb/vsc7326.c b/drivers/net/ethernet/chelsio/cxgb/vsc7326.c new file mode 100644 index 000000000..2ad3efb55 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/vsc7326.c @@ -0,0 +1,726 @@ +// SPDX-License-Identifier: GPL-2.0 +/* $Date: 2006/04/28 19:20:06 $ $RCSfile: vsc7326.c,v $ $Revision: 1.19 $ */ + +/* Driver for Vitesse VSC7326 (Schaumburg) MAC */ + +#include "gmac.h" +#include "elmer0.h" +#include "vsc7326_reg.h" + +/* Update fast changing statistics every 15 seconds */ +#define STATS_TICK_SECS 15 +/* 30 minutes for full statistics update */ +#define MAJOR_UPDATE_TICKS (1800 / STATS_TICK_SECS) + +/* The egress WM value 0x01a01fff should be used only when the + * interface is down (MAC port disabled). This is a workaround + * for disabling the T2/MAC flow-control. When the interface is + * enabled, the WM value should be set to 0x014a03F0. + */ +#define WM_DISABLE 0x01a01fff +#define WM_ENABLE 0x014a03F0 + +struct init_table { + u32 addr; + u32 data; +}; + +struct _cmac_instance { + u32 index; + u32 ticks; +}; + +#define INITBLOCK_SLEEP 0xffffffff + +static void vsc_read(adapter_t *adapter, u32 addr, u32 *val) +{ + u32 status, vlo, vhi; + int i; + + spin_lock_bh(&adapter->mac_lock); + t1_tpi_read(adapter, (addr << 2) + 4, &vlo); + i = 0; + do { + t1_tpi_read(adapter, (REG_LOCAL_STATUS << 2) + 4, &vlo); + t1_tpi_read(adapter, REG_LOCAL_STATUS << 2, &vhi); + status = (vhi << 16) | vlo; + i++; + } while (((status & 1) == 0) && (i < 50)); + if (i == 50) + pr_err("Invalid tpi read from MAC, breaking loop.\n"); + + t1_tpi_read(adapter, (REG_LOCAL_DATA << 2) + 4, &vlo); + t1_tpi_read(adapter, REG_LOCAL_DATA << 2, &vhi); + + *val = (vhi << 16) | vlo; + + /* pr_err("rd: block: 0x%x sublock: 0x%x reg: 0x%x data: 0x%x\n", + ((addr&0xe000)>>13), ((addr&0x1e00)>>9), + ((addr&0x01fe)>>1), *val); */ + spin_unlock_bh(&adapter->mac_lock); +} + +static void vsc_write(adapter_t *adapter, u32 addr, u32 data) +{ + spin_lock_bh(&adapter->mac_lock); + t1_tpi_write(adapter, (addr << 2) + 4, data & 0xFFFF); + t1_tpi_write(adapter, addr << 2, (data >> 16) & 0xFFFF); + /* pr_err("wr: block: 0x%x sublock: 0x%x reg: 0x%x data: 0x%x\n", + ((addr&0xe000)>>13), ((addr&0x1e00)>>9), + ((addr&0x01fe)>>1), data); */ + spin_unlock_bh(&adapter->mac_lock); +} + +/* Hard reset the MAC. This wipes out *all* configuration. */ +static void vsc7326_full_reset(adapter_t* adapter) +{ + u32 val; + u32 result = 0xffff; + + t1_tpi_read(adapter, A_ELMER0_GPO, &val); + val &= ~1; + t1_tpi_write(adapter, A_ELMER0_GPO, val); + udelay(2); + val |= 0x1; /* Enable mac MAC itself */ + val |= 0x800; /* Turn off the red LED */ + t1_tpi_write(adapter, A_ELMER0_GPO, val); + mdelay(1); + vsc_write(adapter, REG_SW_RESET, 0x80000001); + do { + mdelay(1); + vsc_read(adapter, REG_SW_RESET, &result); + } while (result != 0x0); +} + +static struct init_table vsc7326_reset[] = { + { REG_IFACE_MODE, 0x00000000 }, + { REG_CRC_CFG, 0x00000020 }, + { REG_PLL_CLK_SPEED, 0x00050c00 }, + { REG_PLL_CLK_SPEED, 0x00050c00 }, + { REG_MSCH, 0x00002f14 }, + { REG_SPI4_MISC, 0x00040409 }, + { REG_SPI4_DESKEW, 0x00080000 }, + { REG_SPI4_ING_SETUP2, 0x08080004 }, + { REG_SPI4_ING_SETUP0, 0x04111004 }, + { REG_SPI4_EGR_SETUP0, 0x80001a04 }, + { REG_SPI4_ING_SETUP1, 0x02010000 }, + { REG_AGE_INC(0), 0x00000000 }, + { REG_AGE_INC(1), 0x00000000 }, + { REG_ING_CONTROL, 0x0a200011 }, + { REG_EGR_CONTROL, 0xa0010091 }, +}; + +static struct init_table vsc7326_portinit[4][22] = { + { /* Port 0 */ + /* FIFO setup */ + { REG_DBG(0), 0x000004f0 }, + { REG_HDX(0), 0x00073101 }, + { REG_TEST(0,0), 0x00000022 }, + { REG_TEST(1,0), 0x00000022 }, + { REG_TOP_BOTTOM(0,0), 0x003f0000 }, + { REG_TOP_BOTTOM(1,0), 0x00120000 }, + { REG_HIGH_LOW_WM(0,0), 0x07460757 }, + { REG_HIGH_LOW_WM(1,0), WM_DISABLE }, + { REG_CT_THRHLD(0,0), 0x00000000 }, + { REG_CT_THRHLD(1,0), 0x00000000 }, + { REG_BUCKE(0), 0x0002ffff }, + { REG_BUCKI(0), 0x0002ffff }, + { REG_TEST(0,0), 0x00000020 }, + { REG_TEST(1,0), 0x00000020 }, + /* Port config */ + { REG_MAX_LEN(0), 0x00002710 }, + { REG_PORT_FAIL(0), 0x00000002 }, + { REG_NORMALIZER(0), 0x00000a64 }, + { REG_DENORM(0), 0x00000010 }, + { REG_STICK_BIT(0), 0x03baa370 }, + { REG_DEV_SETUP(0), 0x00000083 }, + { REG_DEV_SETUP(0), 0x00000082 }, + { REG_MODE_CFG(0), 0x0200259f }, + }, + { /* Port 1 */ + /* FIFO setup */ + { REG_DBG(1), 0x000004f0 }, + { REG_HDX(1), 0x00073101 }, + { REG_TEST(0,1), 0x00000022 }, + { REG_TEST(1,1), 0x00000022 }, + { REG_TOP_BOTTOM(0,1), 0x007e003f }, + { REG_TOP_BOTTOM(1,1), 0x00240012 }, + { REG_HIGH_LOW_WM(0,1), 0x07460757 }, + { REG_HIGH_LOW_WM(1,1), WM_DISABLE }, + { REG_CT_THRHLD(0,1), 0x00000000 }, + { REG_CT_THRHLD(1,1), 0x00000000 }, + { REG_BUCKE(1), 0x0002ffff }, + { REG_BUCKI(1), 0x0002ffff }, + { REG_TEST(0,1), 0x00000020 }, + { REG_TEST(1,1), 0x00000020 }, + /* Port config */ + { REG_MAX_LEN(1), 0x00002710 }, + { REG_PORT_FAIL(1), 0x00000002 }, + { REG_NORMALIZER(1), 0x00000a64 }, + { REG_DENORM(1), 0x00000010 }, + { REG_STICK_BIT(1), 0x03baa370 }, + { REG_DEV_SETUP(1), 0x00000083 }, + { REG_DEV_SETUP(1), 0x00000082 }, + { REG_MODE_CFG(1), 0x0200259f }, + }, + { /* Port 2 */ + /* FIFO setup */ + { REG_DBG(2), 0x000004f0 }, + { REG_HDX(2), 0x00073101 }, + { REG_TEST(0,2), 0x00000022 }, + { REG_TEST(1,2), 0x00000022 }, + { REG_TOP_BOTTOM(0,2), 0x00bd007e }, + { REG_TOP_BOTTOM(1,2), 0x00360024 }, + { REG_HIGH_LOW_WM(0,2), 0x07460757 }, + { REG_HIGH_LOW_WM(1,2), WM_DISABLE }, + { REG_CT_THRHLD(0,2), 0x00000000 }, + { REG_CT_THRHLD(1,2), 0x00000000 }, + { REG_BUCKE(2), 0x0002ffff }, + { REG_BUCKI(2), 0x0002ffff }, + { REG_TEST(0,2), 0x00000020 }, + { REG_TEST(1,2), 0x00000020 }, + /* Port config */ + { REG_MAX_LEN(2), 0x00002710 }, + { REG_PORT_FAIL(2), 0x00000002 }, + { REG_NORMALIZER(2), 0x00000a64 }, + { REG_DENORM(2), 0x00000010 }, + { REG_STICK_BIT(2), 0x03baa370 }, + { REG_DEV_SETUP(2), 0x00000083 }, + { REG_DEV_SETUP(2), 0x00000082 }, + { REG_MODE_CFG(2), 0x0200259f }, + }, + { /* Port 3 */ + /* FIFO setup */ + { REG_DBG(3), 0x000004f0 }, + { REG_HDX(3), 0x00073101 }, + { REG_TEST(0,3), 0x00000022 }, + { REG_TEST(1,3), 0x00000022 }, + { REG_TOP_BOTTOM(0,3), 0x00fc00bd }, + { REG_TOP_BOTTOM(1,3), 0x00480036 }, + { REG_HIGH_LOW_WM(0,3), 0x07460757 }, + { REG_HIGH_LOW_WM(1,3), WM_DISABLE }, + { REG_CT_THRHLD(0,3), 0x00000000 }, + { REG_CT_THRHLD(1,3), 0x00000000 }, + { REG_BUCKE(3), 0x0002ffff }, + { REG_BUCKI(3), 0x0002ffff }, + { REG_TEST(0,3), 0x00000020 }, + { REG_TEST(1,3), 0x00000020 }, + /* Port config */ + { REG_MAX_LEN(3), 0x00002710 }, + { REG_PORT_FAIL(3), 0x00000002 }, + { REG_NORMALIZER(3), 0x00000a64 }, + { REG_DENORM(3), 0x00000010 }, + { REG_STICK_BIT(3), 0x03baa370 }, + { REG_DEV_SETUP(3), 0x00000083 }, + { REG_DEV_SETUP(3), 0x00000082 }, + { REG_MODE_CFG(3), 0x0200259f }, + }, +}; + +static void run_table(adapter_t *adapter, struct init_table *ib, int len) +{ + int i; + + for (i = 0; i < len; i++) { + if (ib[i].addr == INITBLOCK_SLEEP) { + udelay( ib[i].data ); + pr_err("sleep %d us\n",ib[i].data); + } else + vsc_write( adapter, ib[i].addr, ib[i].data ); + } +} + +static int bist_rd(adapter_t *adapter, int moduleid, int address) +{ + int data = 0; + u32 result = 0; + + if ((address != 0x0) && + (address != 0x1) && + (address != 0x2) && + (address != 0xd) && + (address != 0xe)) + pr_err("No bist address: 0x%x\n", address); + + data = ((0x00 << 24) | ((address & 0xff) << 16) | (0x00 << 8) | + ((moduleid & 0xff) << 0)); + vsc_write(adapter, REG_RAM_BIST_CMD, data); + + udelay(10); + + vsc_read(adapter, REG_RAM_BIST_RESULT, &result); + if ((result & (1 << 9)) != 0x0) + pr_err("Still in bist read: 0x%x\n", result); + else if ((result & (1 << 8)) != 0x0) + pr_err("bist read error: 0x%x\n", result); + + return result & 0xff; +} + +static int bist_wr(adapter_t *adapter, int moduleid, int address, int value) +{ + int data = 0; + u32 result = 0; + + if ((address != 0x0) && + (address != 0x1) && + (address != 0x2) && + (address != 0xd) && + (address != 0xe)) + pr_err("No bist address: 0x%x\n", address); + + if (value > 255) + pr_err("Suspicious write out of range value: 0x%x\n", value); + + data = ((0x01 << 24) | ((address & 0xff) << 16) | (value << 8) | + ((moduleid & 0xff) << 0)); + vsc_write(adapter, REG_RAM_BIST_CMD, data); + + udelay(5); + + vsc_read(adapter, REG_RAM_BIST_CMD, &result); + if ((result & (1 << 27)) != 0x0) + pr_err("Still in bist write: 0x%x\n", result); + else if ((result & (1 << 26)) != 0x0) + pr_err("bist write error: 0x%x\n", result); + + return 0; +} + +static int run_bist(adapter_t *adapter, int moduleid) +{ + /*run bist*/ + (void) bist_wr(adapter,moduleid, 0x00, 0x02); + (void) bist_wr(adapter,moduleid, 0x01, 0x01); + + return 0; +} + +static int check_bist(adapter_t *adapter, int moduleid) +{ + int result=0; + int column=0; + /*check bist*/ + result = bist_rd(adapter,moduleid, 0x02); + column = ((bist_rd(adapter,moduleid, 0x0e)<<8) + + (bist_rd(adapter,moduleid, 0x0d))); + if ((result & 3) != 0x3) + pr_err("Result: 0x%x BIST error in ram %d, column: 0x%04x\n", + result, moduleid, column); + return 0; +} + +static int enable_mem(adapter_t *adapter, int moduleid) +{ + /*enable mem*/ + (void) bist_wr(adapter,moduleid, 0x00, 0x00); + return 0; +} + +static int run_bist_all(adapter_t *adapter) +{ + int port = 0; + u32 val = 0; + + vsc_write(adapter, REG_MEM_BIST, 0x5); + vsc_read(adapter, REG_MEM_BIST, &val); + + for (port = 0; port < 12; port++) + vsc_write(adapter, REG_DEV_SETUP(port), 0x0); + + udelay(300); + vsc_write(adapter, REG_SPI4_MISC, 0x00040409); + udelay(300); + + (void) run_bist(adapter,13); + (void) run_bist(adapter,14); + (void) run_bist(adapter,20); + (void) run_bist(adapter,21); + mdelay(200); + (void) check_bist(adapter,13); + (void) check_bist(adapter,14); + (void) check_bist(adapter,20); + (void) check_bist(adapter,21); + udelay(100); + (void) enable_mem(adapter,13); + (void) enable_mem(adapter,14); + (void) enable_mem(adapter,20); + (void) enable_mem(adapter,21); + udelay(300); + vsc_write(adapter, REG_SPI4_MISC, 0x60040400); + udelay(300); + for (port = 0; port < 12; port++) + vsc_write(adapter, REG_DEV_SETUP(port), 0x1); + + udelay(300); + vsc_write(adapter, REG_MEM_BIST, 0x0); + mdelay(10); + return 0; +} + +static int mac_intr_handler(struct cmac *mac) +{ + return 0; +} + +static int mac_intr_enable(struct cmac *mac) +{ + return 0; +} + +static int mac_intr_disable(struct cmac *mac) +{ + return 0; +} + +static int mac_intr_clear(struct cmac *mac) +{ + return 0; +} + +/* Expect MAC address to be in network byte order. */ +static int mac_set_address(struct cmac* mac, const u8 addr[6]) +{ + u32 val; + int port = mac->instance->index; + + vsc_write(mac->adapter, REG_MAC_LOW_ADDR(port), + (addr[3] << 16) | (addr[4] << 8) | addr[5]); + vsc_write(mac->adapter, REG_MAC_HIGH_ADDR(port), + (addr[0] << 16) | (addr[1] << 8) | addr[2]); + + vsc_read(mac->adapter, REG_ING_FFILT_UM_EN, &val); + val &= ~0xf0000000; + vsc_write(mac->adapter, REG_ING_FFILT_UM_EN, val | (port << 28)); + + vsc_write(mac->adapter, REG_ING_FFILT_MASK0, + 0xffff0000 | (addr[4] << 8) | addr[5]); + vsc_write(mac->adapter, REG_ING_FFILT_MASK1, + 0xffff0000 | (addr[2] << 8) | addr[3]); + vsc_write(mac->adapter, REG_ING_FFILT_MASK2, + 0xffff0000 | (addr[0] << 8) | addr[1]); + return 0; +} + +static int mac_get_address(struct cmac *mac, u8 addr[6]) +{ + u32 addr_lo, addr_hi; + int port = mac->instance->index; + + vsc_read(mac->adapter, REG_MAC_LOW_ADDR(port), &addr_lo); + vsc_read(mac->adapter, REG_MAC_HIGH_ADDR(port), &addr_hi); + + addr[0] = (u8) (addr_hi >> 16); + addr[1] = (u8) (addr_hi >> 8); + addr[2] = (u8) addr_hi; + addr[3] = (u8) (addr_lo >> 16); + addr[4] = (u8) (addr_lo >> 8); + addr[5] = (u8) addr_lo; + return 0; +} + +/* This is intended to reset a port, not the whole MAC */ +static int mac_reset(struct cmac *mac) +{ + int index = mac->instance->index; + + run_table(mac->adapter, vsc7326_portinit[index], + ARRAY_SIZE(vsc7326_portinit[index])); + + return 0; +} + +static int mac_set_rx_mode(struct cmac *mac, struct t1_rx_mode *rm) +{ + u32 v; + int port = mac->instance->index; + + vsc_read(mac->adapter, REG_ING_FFILT_UM_EN, &v); + v |= 1 << 12; + + if (t1_rx_mode_promisc(rm)) + v &= ~(1 << (port + 16)); + else + v |= 1 << (port + 16); + + vsc_write(mac->adapter, REG_ING_FFILT_UM_EN, v); + return 0; +} + +static int mac_set_mtu(struct cmac *mac, int mtu) +{ + int port = mac->instance->index; + + /* max_len includes header and FCS */ + vsc_write(mac->adapter, REG_MAX_LEN(port), mtu + 14 + 4); + return 0; +} + +static int mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, + int fc) +{ + u32 v; + int enable, port = mac->instance->index; + + if (speed >= 0 && speed != SPEED_10 && speed != SPEED_100 && + speed != SPEED_1000) + return -1; + if (duplex > 0 && duplex != DUPLEX_FULL) + return -1; + + if (speed >= 0) { + vsc_read(mac->adapter, REG_MODE_CFG(port), &v); + enable = v & 3; /* save tx/rx enables */ + v &= ~0xf; + v |= 4; /* full duplex */ + if (speed == SPEED_1000) + v |= 8; /* GigE */ + enable |= v; + vsc_write(mac->adapter, REG_MODE_CFG(port), v); + + if (speed == SPEED_1000) + v = 0x82; + else if (speed == SPEED_100) + v = 0x84; + else /* SPEED_10 */ + v = 0x86; + vsc_write(mac->adapter, REG_DEV_SETUP(port), v | 1); /* reset */ + vsc_write(mac->adapter, REG_DEV_SETUP(port), v); + vsc_read(mac->adapter, REG_DBG(port), &v); + v &= ~0xff00; + if (speed == SPEED_1000) + v |= 0x400; + else if (speed == SPEED_100) + v |= 0x2000; + else /* SPEED_10 */ + v |= 0xff00; + vsc_write(mac->adapter, REG_DBG(port), v); + + vsc_write(mac->adapter, REG_TX_IFG(port), + speed == SPEED_1000 ? 5 : 0x11); + if (duplex == DUPLEX_HALF) + enable = 0x0; /* 100 or 10 */ + else if (speed == SPEED_1000) + enable = 0xc; + else /* SPEED_100 or 10 */ + enable = 0x4; + enable |= 0x9 << 10; /* IFG1 */ + enable |= 0x6 << 6; /* IFG2 */ + enable |= 0x1 << 4; /* VLAN */ + enable |= 0x3; /* RX/TX EN */ + vsc_write(mac->adapter, REG_MODE_CFG(port), enable); + + } + + vsc_read(mac->adapter, REG_PAUSE_CFG(port), &v); + v &= 0xfff0ffff; + v |= 0x20000; /* xon/xoff */ + if (fc & PAUSE_RX) + v |= 0x40000; + if (fc & PAUSE_TX) + v |= 0x80000; + if (fc == (PAUSE_RX | PAUSE_TX)) + v |= 0x10000; + vsc_write(mac->adapter, REG_PAUSE_CFG(port), v); + return 0; +} + +static int mac_enable(struct cmac *mac, int which) +{ + u32 val; + int port = mac->instance->index; + + /* Write the correct WM value when the port is enabled. */ + vsc_write(mac->adapter, REG_HIGH_LOW_WM(1,port), WM_ENABLE); + + vsc_read(mac->adapter, REG_MODE_CFG(port), &val); + if (which & MAC_DIRECTION_RX) + val |= 0x2; + if (which & MAC_DIRECTION_TX) + val |= 1; + vsc_write(mac->adapter, REG_MODE_CFG(port), val); + return 0; +} + +static int mac_disable(struct cmac *mac, int which) +{ + u32 val; + int i, port = mac->instance->index; + + /* Reset the port, this also writes the correct WM value */ + mac_reset(mac); + + vsc_read(mac->adapter, REG_MODE_CFG(port), &val); + if (which & MAC_DIRECTION_RX) + val &= ~0x2; + if (which & MAC_DIRECTION_TX) + val &= ~0x1; + vsc_write(mac->adapter, REG_MODE_CFG(port), val); + vsc_read(mac->adapter, REG_MODE_CFG(port), &val); + + /* Clear stats */ + for (i = 0; i <= 0x3a; ++i) + vsc_write(mac->adapter, CRA(4, port, i), 0); + + /* Clear software counters */ + memset(&mac->stats, 0, sizeof(struct cmac_statistics)); + + return 0; +} + +static void rmon_update(struct cmac *mac, unsigned int addr, u64 *stat) +{ + u32 v, lo; + + vsc_read(mac->adapter, addr, &v); + lo = *stat; + *stat = *stat - lo + v; + + if (v == 0) + return; + + if (v < lo) + *stat += (1ULL << 32); +} + +static void port_stats_update(struct cmac *mac) +{ + struct { + unsigned int reg; + unsigned int offset; + } hw_stats[] = { + +#define HW_STAT(reg, stat_name) \ + { reg, offsetof(struct cmac_statistics, stat_name) / sizeof(u64) } + + /* Rx stats */ + HW_STAT(RxUnicast, RxUnicastFramesOK), + HW_STAT(RxMulticast, RxMulticastFramesOK), + HW_STAT(RxBroadcast, RxBroadcastFramesOK), + HW_STAT(Crc, RxFCSErrors), + HW_STAT(RxAlignment, RxAlignErrors), + HW_STAT(RxOversize, RxFrameTooLongErrors), + HW_STAT(RxPause, RxPauseFrames), + HW_STAT(RxJabbers, RxJabberErrors), + HW_STAT(RxFragments, RxRuntErrors), + HW_STAT(RxUndersize, RxRuntErrors), + HW_STAT(RxSymbolCarrier, RxSymbolErrors), + HW_STAT(RxSize1519ToMax, RxJumboFramesOK), + + /* Tx stats (skip collision stats as we are full-duplex only) */ + HW_STAT(TxUnicast, TxUnicastFramesOK), + HW_STAT(TxMulticast, TxMulticastFramesOK), + HW_STAT(TxBroadcast, TxBroadcastFramesOK), + HW_STAT(TxPause, TxPauseFrames), + HW_STAT(TxUnderrun, TxUnderrun), + HW_STAT(TxSize1519ToMax, TxJumboFramesOK), + }, *p = hw_stats; + unsigned int port = mac->instance->index; + u64 *stats = (u64 *)&mac->stats; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(hw_stats); i++) + rmon_update(mac, CRA(0x4, port, p->reg), stats + p->offset); + + rmon_update(mac, REG_TX_OK_BYTES(port), &mac->stats.TxOctetsOK); + rmon_update(mac, REG_RX_OK_BYTES(port), &mac->stats.RxOctetsOK); + rmon_update(mac, REG_RX_BAD_BYTES(port), &mac->stats.RxOctetsBad); +} + +/* + * This function is called periodically to accumulate the current values of the + * RMON counters into the port statistics. Since the counters are only 32 bits + * some of them can overflow in less than a minute at GigE speeds, so this + * function should be called every 30 seconds or so. + * + * To cut down on reading costs we update only the octet counters at each tick + * and do a full update at major ticks, which can be every 30 minutes or more. + */ +static const struct cmac_statistics *mac_update_statistics(struct cmac *mac, + int flag) +{ + if (flag == MAC_STATS_UPDATE_FULL || + mac->instance->ticks >= MAJOR_UPDATE_TICKS) { + port_stats_update(mac); + mac->instance->ticks = 0; + } else { + int port = mac->instance->index; + + rmon_update(mac, REG_RX_OK_BYTES(port), + &mac->stats.RxOctetsOK); + rmon_update(mac, REG_RX_BAD_BYTES(port), + &mac->stats.RxOctetsBad); + rmon_update(mac, REG_TX_OK_BYTES(port), + &mac->stats.TxOctetsOK); + mac->instance->ticks++; + } + return &mac->stats; +} + +static void mac_destroy(struct cmac *mac) +{ + kfree(mac); +} + +static const struct cmac_ops vsc7326_ops = { + .destroy = mac_destroy, + .reset = mac_reset, + .interrupt_handler = mac_intr_handler, + .interrupt_enable = mac_intr_enable, + .interrupt_disable = mac_intr_disable, + .interrupt_clear = mac_intr_clear, + .enable = mac_enable, + .disable = mac_disable, + .set_mtu = mac_set_mtu, + .set_rx_mode = mac_set_rx_mode, + .set_speed_duplex_fc = mac_set_speed_duplex_fc, + .statistics_update = mac_update_statistics, + .macaddress_get = mac_get_address, + .macaddress_set = mac_set_address, +}; + +static struct cmac *vsc7326_mac_create(adapter_t *adapter, int index) +{ + struct cmac *mac; + u32 val; + int i; + + mac = kzalloc(sizeof(*mac) + sizeof(cmac_instance), GFP_KERNEL); + if (!mac) + return NULL; + + mac->ops = &vsc7326_ops; + mac->instance = (cmac_instance *)(mac + 1); + mac->adapter = adapter; + + mac->instance->index = index; + mac->instance->ticks = 0; + + i = 0; + do { + u32 vhi, vlo; + + vhi = vlo = 0; + t1_tpi_read(adapter, (REG_LOCAL_STATUS << 2) + 4, &vlo); + udelay(1); + t1_tpi_read(adapter, REG_LOCAL_STATUS << 2, &vhi); + udelay(5); + val = (vhi << 16) | vlo; + } while ((++i < 10000) && (val == 0xffffffff)); + + return mac; +} + +static int vsc7326_mac_reset(adapter_t *adapter) +{ + vsc7326_full_reset(adapter); + (void) run_bist_all(adapter); + run_table(adapter, vsc7326_reset, ARRAY_SIZE(vsc7326_reset)); + return 0; +} + +const struct gmac t1_vsc7326_ops = { + .stats_update_period = STATS_TICK_SECS, + .create = vsc7326_mac_create, + .reset = vsc7326_mac_reset, +}; diff --git a/drivers/net/ethernet/chelsio/cxgb/vsc7326_reg.h b/drivers/net/ethernet/chelsio/cxgb/vsc7326_reg.h new file mode 100644 index 000000000..04503857c --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb/vsc7326_reg.h @@ -0,0 +1,298 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* $Date: 2006/04/28 19:20:17 $ $RCSfile: vsc7326_reg.h,v $ $Revision: 1.5 $ */ +#ifndef _VSC7321_REG_H_ +#define _VSC7321_REG_H_ + +/* Register definitions for Vitesse VSC7321 (Meigs II) MAC + * + * Straight off the data sheet, VMDS-10038 Rev 2.0 and + * PD0011-01-14-Meigs-II 2002-12-12 + */ + +/* Just 'cause it's in here doesn't mean it's used. */ + +#define CRA(blk,sub,adr) ((((blk) & 0x7) << 13) | (((sub) & 0xf) << 9) | (((adr) & 0xff) << 1)) + +/* System and CPU comm's registers */ +#define REG_CHIP_ID CRA(0x7,0xf,0x00) /* Chip ID */ +#define REG_BLADE_ID CRA(0x7,0xf,0x01) /* Blade ID */ +#define REG_SW_RESET CRA(0x7,0xf,0x02) /* Global Soft Reset */ +#define REG_MEM_BIST CRA(0x7,0xf,0x04) /* mem */ +#define REG_IFACE_MODE CRA(0x7,0xf,0x07) /* Interface mode */ +#define REG_MSCH CRA(0x7,0x2,0x06) /* CRC error count */ +#define REG_CRC_CNT CRA(0x7,0x2,0x0a) /* CRC error count */ +#define REG_CRC_CFG CRA(0x7,0x2,0x0b) /* CRC config */ +#define REG_SI_TRANSFER_SEL CRA(0x7,0xf,0x18) /* SI Transfer Select */ +#define REG_PLL_CLK_SPEED CRA(0x7,0xf,0x19) /* Clock Speed Selection */ +#define REG_SYS_CLK_SELECT CRA(0x7,0xf,0x1c) /* System Clock Select */ +#define REG_GPIO_CTRL CRA(0x7,0xf,0x1d) /* GPIO Control */ +#define REG_GPIO_OUT CRA(0x7,0xf,0x1e) /* GPIO Out */ +#define REG_GPIO_IN CRA(0x7,0xf,0x1f) /* GPIO In */ +#define REG_CPU_TRANSFER_SEL CRA(0x7,0xf,0x20) /* CPU Transfer Select */ +#define REG_LOCAL_DATA CRA(0x7,0xf,0xfe) /* Local CPU Data Register */ +#define REG_LOCAL_STATUS CRA(0x7,0xf,0xff) /* Local CPU Status Register */ + +/* Aggregator registers */ +#define REG_AGGR_SETUP CRA(0x7,0x1,0x00) /* Aggregator Setup */ +#define REG_PMAP_TABLE CRA(0x7,0x1,0x01) /* Port map table */ +#define REG_MPLS_BIT0 CRA(0x7,0x1,0x08) /* MPLS bit0 position */ +#define REG_MPLS_BIT1 CRA(0x7,0x1,0x09) /* MPLS bit1 position */ +#define REG_MPLS_BIT2 CRA(0x7,0x1,0x0a) /* MPLS bit2 position */ +#define REG_MPLS_BIT3 CRA(0x7,0x1,0x0b) /* MPLS bit3 position */ +#define REG_MPLS_BITMASK CRA(0x7,0x1,0x0c) /* MPLS bit mask */ +#define REG_PRE_BIT0POS CRA(0x7,0x1,0x10) /* Preamble bit0 position */ +#define REG_PRE_BIT1POS CRA(0x7,0x1,0x11) /* Preamble bit1 position */ +#define REG_PRE_BIT2POS CRA(0x7,0x1,0x12) /* Preamble bit2 position */ +#define REG_PRE_BIT3POS CRA(0x7,0x1,0x13) /* Preamble bit3 position */ +#define REG_PRE_ERR_CNT CRA(0x7,0x1,0x14) /* Preamble parity error count */ + +/* BIST registers */ +/*#define REG_RAM_BIST_CMD CRA(0x7,0x2,0x00)*/ /* RAM BIST Command Register */ +/*#define REG_RAM_BIST_RESULT CRA(0x7,0x2,0x01)*/ /* RAM BIST Read Status/Result */ +#define REG_RAM_BIST_CMD CRA(0x7,0x1,0x00) /* RAM BIST Command Register */ +#define REG_RAM_BIST_RESULT CRA(0x7,0x1,0x01) /* RAM BIST Read Status/Result */ +#define BIST_PORT_SELECT 0x00 /* BIST port select */ +#define BIST_COMMAND 0x01 /* BIST enable/disable */ +#define BIST_STATUS 0x02 /* BIST operation status */ +#define BIST_ERR_CNT_LSB 0x03 /* BIST error count lo 8b */ +#define BIST_ERR_CNT_MSB 0x04 /* BIST error count hi 8b */ +#define BIST_ERR_SEL_LSB 0x05 /* BIST error select lo 8b */ +#define BIST_ERR_SEL_MSB 0x06 /* BIST error select hi 8b */ +#define BIST_ERROR_STATE 0x07 /* BIST engine internal state */ +#define BIST_ERR_ADR0 0x08 /* BIST error address lo 8b */ +#define BIST_ERR_ADR1 0x09 /* BIST error address lomid 8b */ +#define BIST_ERR_ADR2 0x0a /* BIST error address himid 8b */ +#define BIST_ERR_ADR3 0x0b /* BIST error address hi 8b */ + +/* FIFO registers + * ie = 0 for ingress, 1 for egress + * fn = FIFO number, 0-9 + */ +#define REG_TEST(ie,fn) CRA(0x2,ie&1,0x00+fn) /* Mode & Test Register */ +#define REG_TOP_BOTTOM(ie,fn) CRA(0x2,ie&1,0x10+fn) /* FIFO Buffer Top & Bottom */ +#define REG_TAIL(ie,fn) CRA(0x2,ie&1,0x20+fn) /* FIFO Write Pointer */ +#define REG_HEAD(ie,fn) CRA(0x2,ie&1,0x30+fn) /* FIFO Read Pointer */ +#define REG_HIGH_LOW_WM(ie,fn) CRA(0x2,ie&1,0x40+fn) /* Flow Control Water Marks */ +#define REG_CT_THRHLD(ie,fn) CRA(0x2,ie&1,0x50+fn) /* Cut Through Threshold */ +#define REG_FIFO_DROP_CNT(ie,fn) CRA(0x2,ie&1,0x60+fn) /* Drop & CRC Error Counter */ +#define REG_DEBUG_BUF_CNT(ie,fn) CRA(0x2,ie&1,0x70+fn) /* Input Side Debug Counter */ +#define REG_BUCKI(fn) CRA(0x2,2,0x20+fn) /* Input Side Debug Counter */ +#define REG_BUCKE(fn) CRA(0x2,3,0x20+fn) /* Input Side Debug Counter */ + +/* Traffic shaper buckets + * ie = 0 for ingress, 1 for egress + * bn = bucket number 0-10 (yes, 11 buckets) + */ +/* OK, this one's kinda ugly. Some hardware designers are perverse. */ +#define REG_TRAFFIC_SHAPER_BUCKET(ie,bn) CRA(0x2,ie&1,0x0a + (bn>7) | ((bn&7)<<4)) +#define REG_TRAFFIC_SHAPER_CONTROL(ie) CRA(0x2,ie&1,0x3b) + +#define REG_SRAM_ADR(ie) CRA(0x2,ie&1,0x0e) /* FIFO SRAM address */ +#define REG_SRAM_WR_STRB(ie) CRA(0x2,ie&1,0x1e) /* FIFO SRAM write strobe */ +#define REG_SRAM_RD_STRB(ie) CRA(0x2,ie&1,0x2e) /* FIFO SRAM read strobe */ +#define REG_SRAM_DATA_0(ie) CRA(0x2,ie&1,0x3e) /* FIFO SRAM data lo 8b */ +#define REG_SRAM_DATA_1(ie) CRA(0x2,ie&1,0x4e) /* FIFO SRAM data lomid 8b */ +#define REG_SRAM_DATA_2(ie) CRA(0x2,ie&1,0x5e) /* FIFO SRAM data himid 8b */ +#define REG_SRAM_DATA_3(ie) CRA(0x2,ie&1,0x6e) /* FIFO SRAM data hi 8b */ +#define REG_SRAM_DATA_BLK_TYPE(ie) CRA(0x2,ie&1,0x7e) /* FIFO SRAM tag */ +/* REG_ING_CONTROL equals REG_CONTROL with ie = 0, likewise REG_EGR_CONTROL is ie = 1 */ +#define REG_CONTROL(ie) CRA(0x2,ie&1,0x0f) /* FIFO control */ +#define REG_ING_CONTROL CRA(0x2,0x0,0x0f) /* Ingress control (alias) */ +#define REG_EGR_CONTROL CRA(0x2,0x1,0x0f) /* Egress control (alias) */ +#define REG_AGE_TIMER(ie) CRA(0x2,ie&1,0x1f) /* Aging timer */ +#define REG_AGE_INC(ie) CRA(0x2,ie&1,0x2f) /* Aging increment */ +#define DEBUG_OUT(ie) CRA(0x2,ie&1,0x3f) /* Output debug counter control */ +#define DEBUG_CNT(ie) CRA(0x2,ie&1,0x4f) /* Output debug counter */ + +/* SPI4 interface */ +#define REG_SPI4_MISC CRA(0x5,0x0,0x00) /* Misc Register */ +#define REG_SPI4_STATUS CRA(0x5,0x0,0x01) /* CML Status */ +#define REG_SPI4_ING_SETUP0 CRA(0x5,0x0,0x02) /* Ingress Status Channel Setup */ +#define REG_SPI4_ING_SETUP1 CRA(0x5,0x0,0x03) /* Ingress Data Training Setup */ +#define REG_SPI4_ING_SETUP2 CRA(0x5,0x0,0x04) /* Ingress Data Burst Size Setup */ +#define REG_SPI4_EGR_SETUP0 CRA(0x5,0x0,0x05) /* Egress Status Channel Setup */ +#define REG_SPI4_DBG_CNT(n) CRA(0x5,0x0,0x10+n) /* Debug counters 0-9 */ +#define REG_SPI4_DBG_SETUP CRA(0x5,0x0,0x1A) /* Debug counters setup */ +#define REG_SPI4_TEST CRA(0x5,0x0,0x20) /* Test Setup Register */ +#define REG_TPGEN_UP0 CRA(0x5,0x0,0x21) /* Test Pattern generator user pattern 0 */ +#define REG_TPGEN_UP1 CRA(0x5,0x0,0x22) /* Test Pattern generator user pattern 1 */ +#define REG_TPCHK_UP0 CRA(0x5,0x0,0x23) /* Test Pattern checker user pattern 0 */ +#define REG_TPCHK_UP1 CRA(0x5,0x0,0x24) /* Test Pattern checker user pattern 1 */ +#define REG_TPSAM_P0 CRA(0x5,0x0,0x25) /* Sampled pattern 0 */ +#define REG_TPSAM_P1 CRA(0x5,0x0,0x26) /* Sampled pattern 1 */ +#define REG_TPERR_CNT CRA(0x5,0x0,0x27) /* Pattern checker error counter */ +#define REG_SPI4_STICKY CRA(0x5,0x0,0x30) /* Sticky bits register */ +#define REG_SPI4_DBG_INH CRA(0x5,0x0,0x31) /* Core egress & ingress inhibit */ +#define REG_SPI4_DBG_STATUS CRA(0x5,0x0,0x32) /* Sampled ingress status */ +#define REG_SPI4_DBG_GRANT CRA(0x5,0x0,0x33) /* Ingress cranted credit value */ + +#define REG_SPI4_DESKEW CRA(0x5,0x0,0x43) /* Ingress cranted credit value */ + +/* 10GbE MAC Block Registers */ +/* Note that those registers that are exactly the same for 10GbE as for + * tri-speed are only defined with the version that needs a port number. + * Pass 0xa in those cases. + * + * Also note that despite the presence of a MAC address register, this part + * does no ingress MAC address filtering. That register is used only for + * pause frame detection and generation. + */ +/* 10GbE specific, and different from tri-speed */ +#define REG_MISC_10G CRA(0x1,0xa,0x00) /* Misc 10GbE setup */ +#define REG_PAUSE_10G CRA(0x1,0xa,0x01) /* Pause register */ +#define REG_NORMALIZER_10G CRA(0x1,0xa,0x05) /* 10G normalizer */ +#define REG_STICKY_RX CRA(0x1,0xa,0x06) /* RX debug register */ +#define REG_DENORM_10G CRA(0x1,0xa,0x07) /* Denormalizer */ +#define REG_STICKY_TX CRA(0x1,0xa,0x08) /* TX sticky bits */ +#define REG_MAX_RXHIGH CRA(0x1,0xa,0x0a) /* XGMII lane 0-3 debug */ +#define REG_MAX_RXLOW CRA(0x1,0xa,0x0b) /* XGMII lane 4-7 debug */ +#define REG_MAC_TX_STICKY CRA(0x1,0xa,0x0c) /* MAC Tx state sticky debug */ +#define REG_MAC_TX_RUNNING CRA(0x1,0xa,0x0d) /* MAC Tx state running debug */ +#define REG_TX_ABORT_AGE CRA(0x1,0xa,0x14) /* Aged Tx frames discarded */ +#define REG_TX_ABORT_SHORT CRA(0x1,0xa,0x15) /* Short Tx frames discarded */ +#define REG_TX_ABORT_TAXI CRA(0x1,0xa,0x16) /* Taxi error frames discarded */ +#define REG_TX_ABORT_UNDERRUN CRA(0x1,0xa,0x17) /* Tx Underrun abort counter */ +#define REG_TX_DENORM_DISCARD CRA(0x1,0xa,0x18) /* Tx denormalizer discards */ +#define REG_XAUI_STAT_A CRA(0x1,0xa,0x20) /* XAUI status A */ +#define REG_XAUI_STAT_B CRA(0x1,0xa,0x21) /* XAUI status B */ +#define REG_XAUI_STAT_C CRA(0x1,0xa,0x22) /* XAUI status C */ +#define REG_XAUI_CONF_A CRA(0x1,0xa,0x23) /* XAUI configuration A */ +#define REG_XAUI_CONF_B CRA(0x1,0xa,0x24) /* XAUI configuration B */ +#define REG_XAUI_CODE_GRP_CNT CRA(0x1,0xa,0x25) /* XAUI code group error count */ +#define REG_XAUI_CONF_TEST_A CRA(0x1,0xa,0x26) /* XAUI test register A */ +#define REG_PDERRCNT CRA(0x1,0xa,0x27) /* XAUI test register B */ + +/* pn = port number 0-9 for tri-speed, 10 for 10GbE */ +/* Both tri-speed and 10GbE */ +#define REG_MAX_LEN(pn) CRA(0x1,pn,0x02) /* Max length */ +#define REG_MAC_HIGH_ADDR(pn) CRA(0x1,pn,0x03) /* Upper 24 bits of MAC addr */ +#define REG_MAC_LOW_ADDR(pn) CRA(0x1,pn,0x04) /* Lower 24 bits of MAC addr */ + +/* tri-speed only + * pn = port number, 0-9 + */ +#define REG_MODE_CFG(pn) CRA(0x1,pn,0x00) /* Mode configuration */ +#define REG_PAUSE_CFG(pn) CRA(0x1,pn,0x01) /* Pause configuration */ +#define REG_NORMALIZER(pn) CRA(0x1,pn,0x05) /* Normalizer */ +#define REG_TBI_STATUS(pn) CRA(0x1,pn,0x06) /* TBI status */ +#define REG_PCS_STATUS_DBG(pn) CRA(0x1,pn,0x07) /* PCS status debug */ +#define REG_PCS_CTRL(pn) CRA(0x1,pn,0x08) /* PCS control */ +#define REG_TBI_CONFIG(pn) CRA(0x1,pn,0x09) /* TBI configuration */ +#define REG_STICK_BIT(pn) CRA(0x1,pn,0x0a) /* Sticky bits */ +#define REG_DEV_SETUP(pn) CRA(0x1,pn,0x0b) /* MAC clock/reset setup */ +#define REG_DROP_CNT(pn) CRA(0x1,pn,0x0c) /* Drop counter */ +#define REG_PORT_POS(pn) CRA(0x1,pn,0x0d) /* Preamble port position */ +#define REG_PORT_FAIL(pn) CRA(0x1,pn,0x0e) /* Preamble port position */ +#define REG_SERDES_CONF(pn) CRA(0x1,pn,0x0f) /* SerDes configuration */ +#define REG_SERDES_TEST(pn) CRA(0x1,pn,0x10) /* SerDes test */ +#define REG_SERDES_STAT(pn) CRA(0x1,pn,0x11) /* SerDes status */ +#define REG_SERDES_COM_CNT(pn) CRA(0x1,pn,0x12) /* SerDes comma counter */ +#define REG_DENORM(pn) CRA(0x1,pn,0x15) /* Frame denormalization */ +#define REG_DBG(pn) CRA(0x1,pn,0x16) /* Device 1G debug */ +#define REG_TX_IFG(pn) CRA(0x1,pn,0x18) /* Tx IFG config */ +#define REG_HDX(pn) CRA(0x1,pn,0x19) /* Half-duplex config */ + +/* Statistics */ +/* CRA(0x4,pn,reg) */ +/* reg below */ +/* pn = port number, 0-a, a = 10GbE */ + +enum { + RxInBytes = 0x00, // # Rx in octets + RxSymbolCarrier = 0x01, // Frames w/ symbol errors + RxPause = 0x02, // # pause frames received + RxUnsupOpcode = 0x03, // # control frames with unsupported opcode + RxOkBytes = 0x04, // # octets in good frames + RxBadBytes = 0x05, // # octets in bad frames + RxUnicast = 0x06, // # good unicast frames + RxMulticast = 0x07, // # good multicast frames + RxBroadcast = 0x08, // # good broadcast frames + Crc = 0x09, // # frames w/ bad CRC only + RxAlignment = 0x0a, // # frames w/ alignment err + RxUndersize = 0x0b, // # frames undersize + RxFragments = 0x0c, // # frames undersize w/ crc err + RxInRangeLengthError = 0x0d, // # frames with length error + RxOutOfRangeError = 0x0e, // # frames with illegal length field + RxOversize = 0x0f, // # frames oversize + RxJabbers = 0x10, // # frames oversize w/ crc err + RxSize64 = 0x11, // # frames 64 octets long + RxSize65To127 = 0x12, // # frames 65-127 octets + RxSize128To255 = 0x13, // # frames 128-255 + RxSize256To511 = 0x14, // # frames 256-511 + RxSize512To1023 = 0x15, // # frames 512-1023 + RxSize1024To1518 = 0x16, // # frames 1024-1518 + RxSize1519ToMax = 0x17, // # frames 1519-max + + TxOutBytes = 0x18, // # octets tx + TxPause = 0x19, // # pause frames sent + TxOkBytes = 0x1a, // # octets tx OK + TxUnicast = 0x1b, // # frames unicast + TxMulticast = 0x1c, // # frames multicast + TxBroadcast = 0x1d, // # frames broadcast + TxMultipleColl = 0x1e, // # frames tx after multiple collisions + TxLateColl = 0x1f, // # late collisions detected + TxXcoll = 0x20, // # frames lost, excessive collisions + TxDefer = 0x21, // # frames deferred on first tx attempt + TxXdefer = 0x22, // # frames excessively deferred + TxCsense = 0x23, // carrier sense errors at frame end + TxSize64 = 0x24, // # frames 64 octets long + TxSize65To127 = 0x25, // # frames 65-127 octets + TxSize128To255 = 0x26, // # frames 128-255 + TxSize256To511 = 0x27, // # frames 256-511 + TxSize512To1023 = 0x28, // # frames 512-1023 + TxSize1024To1518 = 0x29, // # frames 1024-1518 + TxSize1519ToMax = 0x2a, // # frames 1519-max + TxSingleColl = 0x2b, // # frames tx after single collision + TxBackoff2 = 0x2c, // # frames tx ok after 2 backoffs/collisions + TxBackoff3 = 0x2d, // after 3 backoffs/collisions + TxBackoff4 = 0x2e, // after 4 + TxBackoff5 = 0x2f, // after 5 + TxBackoff6 = 0x30, // after 6 + TxBackoff7 = 0x31, // after 7 + TxBackoff8 = 0x32, // after 8 + TxBackoff9 = 0x33, // after 9 + TxBackoff10 = 0x34, // after 10 + TxBackoff11 = 0x35, // after 11 + TxBackoff12 = 0x36, // after 12 + TxBackoff13 = 0x37, // after 13 + TxBackoff14 = 0x38, // after 14 + TxBackoff15 = 0x39, // after 15 + TxUnderrun = 0x3a, // # frames dropped from underrun + // Hole. See REG_RX_XGMII_PROT_ERR below. + RxIpgShrink = 0x3c, // # of IPG shrinks detected + // Duplicate. See REG_STAT_STICKY10G below. + StatSticky1G = 0x3e, // tri-speed sticky bits + StatInit = 0x3f // Clear all statistics +}; + +#define REG_RX_XGMII_PROT_ERR CRA(0x4,0xa,0x3b) /* # protocol errors detected on XGMII interface */ +#define REG_STAT_STICKY10G CRA(0x4,0xa,StatSticky1G) /* 10GbE sticky bits */ + +#define REG_RX_OK_BYTES(pn) CRA(0x4,pn,RxOkBytes) +#define REG_RX_BAD_BYTES(pn) CRA(0x4,pn,RxBadBytes) +#define REG_TX_OK_BYTES(pn) CRA(0x4,pn,TxOkBytes) + +/* MII-Management Block registers */ +/* These are for MII-M interface 0, which is the bidirectional LVTTL one. If + * we hooked up to the one with separate directions, the middle 0x0 needs to + * change to 0x1. And the current errata states that MII-M 1 doesn't work. + */ + +#define REG_MIIM_STATUS CRA(0x3,0x0,0x00) /* MII-M Status */ +#define REG_MIIM_CMD CRA(0x3,0x0,0x01) /* MII-M Command */ +#define REG_MIIM_DATA CRA(0x3,0x0,0x02) /* MII-M Data */ +#define REG_MIIM_PRESCALE CRA(0x3,0x0,0x03) /* MII-M MDC Prescale */ + +#define REG_ING_FFILT_UM_EN CRA(0x2, 0, 0xd) +#define REG_ING_FFILT_BE_EN CRA(0x2, 0, 0x1d) +#define REG_ING_FFILT_VAL0 CRA(0x2, 0, 0x2d) +#define REG_ING_FFILT_VAL1 CRA(0x2, 0, 0x3d) +#define REG_ING_FFILT_MASK0 CRA(0x2, 0, 0x4d) +#define REG_ING_FFILT_MASK1 CRA(0x2, 0, 0x5d) +#define REG_ING_FFILT_MASK2 CRA(0x2, 0, 0x6d) +#define REG_ING_FFILT_ETYPE CRA(0x2, 0, 0x7d) + + +/* Whew. */ + +#endif diff --git a/drivers/net/ethernet/chelsio/cxgb3/Makefile b/drivers/net/ethernet/chelsio/cxgb3/Makefile new file mode 100644 index 000000000..f65f0d93b --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Chelsio T3 driver +# + +obj-$(CONFIG_CHELSIO_T3) += cxgb3.o + +cxgb3-objs := cxgb3_main.o ael1002.o vsc8211.o t3_hw.o mc5.o \ + xgmac.o sge.o l2t.o cxgb3_offload.o aq100x.o diff --git a/drivers/net/ethernet/chelsio/cxgb3/adapter.h b/drivers/net/ethernet/chelsio/cxgb3/adapter.h new file mode 100644 index 000000000..6d682b7c7 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/adapter.h @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* This file should not be included directly. Include common.h instead. */ + +#ifndef __T3_ADAPTER_H__ +#define __T3_ADAPTER_H__ + +#include <linux/pci.h> +#include <linux/spinlock.h> +#include <linux/interrupt.h> +#include <linux/timer.h> +#include <linux/cache.h> +#include <linux/mutex.h> +#include <linux/bitops.h> +#include "t3cdev.h" +#include <asm/io.h> + +struct adapter; +struct sge_qset; +struct port_info; + +enum mac_idx_types { + LAN_MAC_IDX = 0, + SAN_MAC_IDX, + + MAX_MAC_IDX +}; + +struct iscsi_config { + __u8 mac_addr[ETH_ALEN]; + __u32 flags; + int (*send)(struct port_info *pi, struct sk_buff **skb); + int (*recv)(struct port_info *pi, struct sk_buff *skb); +}; + +struct port_info { + struct adapter *adapter; + struct sge_qset *qs; + u8 port_id; + u8 nqsets; + u8 first_qset; + struct cphy phy; + struct cmac mac; + struct link_config link_config; + int activity; + __be32 iscsi_ipv4addr; + struct iscsi_config iscsic; + + int link_fault; /* link fault was detected */ +}; + +enum { /* adapter flags */ + FULL_INIT_DONE = (1 << 0), + USING_MSI = (1 << 1), + USING_MSIX = (1 << 2), + QUEUES_BOUND = (1 << 3), + TP_PARITY_INIT = (1 << 4), + NAPI_INIT = (1 << 5), +}; + +struct fl_pg_chunk { + struct page *page; + void *va; + unsigned int offset; + unsigned long *p_cnt; + dma_addr_t mapping; +}; + +struct rx_desc; +struct rx_sw_desc; + +struct sge_fl { /* SGE per free-buffer list state */ + unsigned int buf_size; /* size of each Rx buffer */ + unsigned int credits; /* # of available Rx buffers */ + unsigned int pend_cred; /* new buffers since last FL DB ring */ + unsigned int size; /* capacity of free list */ + unsigned int cidx; /* consumer index */ + unsigned int pidx; /* producer index */ + unsigned int gen; /* free list generation */ + struct fl_pg_chunk pg_chunk;/* page chunk cache */ + unsigned int use_pages; /* whether FL uses pages or sk_buffs */ + unsigned int order; /* order of page allocations */ + unsigned int alloc_size; /* size of allocated buffer */ + struct rx_desc *desc; /* address of HW Rx descriptor ring */ + struct rx_sw_desc *sdesc; /* address of SW Rx descriptor ring */ + dma_addr_t phys_addr; /* physical address of HW ring start */ + unsigned int cntxt_id; /* SGE context id for the free list */ + unsigned long empty; /* # of times queue ran out of buffers */ + unsigned long alloc_failed; /* # of times buffer allocation failed */ +}; + +/* + * Bundle size for grouping offload RX packets for delivery to the stack. + * Don't make this too big as we do prefetch on each packet in a bundle. + */ +# define RX_BUNDLE_SIZE 8 + +struct rsp_desc; + +struct sge_rspq { /* state for an SGE response queue */ + unsigned int credits; /* # of pending response credits */ + unsigned int size; /* capacity of response queue */ + unsigned int cidx; /* consumer index */ + unsigned int gen; /* current generation bit */ + unsigned int polling; /* is the queue serviced through NAPI? */ + unsigned int holdoff_tmr; /* interrupt holdoff timer in 100ns */ + unsigned int next_holdoff; /* holdoff time for next interrupt */ + unsigned int rx_recycle_buf; /* whether recycling occurred + within current sop-eop */ + struct rsp_desc *desc; /* address of HW response ring */ + dma_addr_t phys_addr; /* physical address of the ring */ + unsigned int cntxt_id; /* SGE context id for the response q */ + spinlock_t lock; /* guards response processing */ + struct sk_buff_head rx_queue; /* offload packet receive queue */ + struct sk_buff *pg_skb; /* used to build frag list in napi handler */ + + unsigned long offload_pkts; + unsigned long offload_bundles; + unsigned long eth_pkts; /* # of ethernet packets */ + unsigned long pure_rsps; /* # of pure (non-data) responses */ + unsigned long imm_data; /* responses with immediate data */ + unsigned long rx_drops; /* # of packets dropped due to no mem */ + unsigned long async_notif; /* # of asynchronous notification events */ + unsigned long empty; /* # of times queue ran out of credits */ + unsigned long nomem; /* # of responses deferred due to no mem */ + unsigned long unhandled_irqs; /* # of spurious intrs */ + unsigned long starved; + unsigned long restarted; +}; + +struct tx_desc; +struct tx_sw_desc; + +struct sge_txq { /* state for an SGE Tx queue */ + unsigned long flags; /* HW DMA fetch status */ + unsigned int in_use; /* # of in-use Tx descriptors */ + unsigned int size; /* # of descriptors */ + unsigned int processed; /* total # of descs HW has processed */ + unsigned int cleaned; /* total # of descs SW has reclaimed */ + unsigned int stop_thres; /* SW TX queue suspend threshold */ + unsigned int cidx; /* consumer index */ + unsigned int pidx; /* producer index */ + unsigned int gen; /* current value of generation bit */ + unsigned int unacked; /* Tx descriptors used since last COMPL */ + struct tx_desc *desc; /* address of HW Tx descriptor ring */ + struct tx_sw_desc *sdesc; /* address of SW Tx descriptor ring */ + spinlock_t lock; /* guards enqueueing of new packets */ + unsigned int token; /* WR token */ + dma_addr_t phys_addr; /* physical address of the ring */ + struct sk_buff_head sendq; /* List of backpressured offload packets */ + struct work_struct qresume_task; /* restarts the queue */ + unsigned int cntxt_id; /* SGE context id for the Tx q */ + unsigned long stops; /* # of times q has been stopped */ + unsigned long restarts; /* # of queue restarts */ +}; + +enum { /* per port SGE statistics */ + SGE_PSTAT_TSO, /* # of TSO requests */ + SGE_PSTAT_RX_CSUM_GOOD, /* # of successful RX csum offloads */ + SGE_PSTAT_TX_CSUM, /* # of TX checksum offloads */ + SGE_PSTAT_VLANEX, /* # of VLAN tag extractions */ + SGE_PSTAT_VLANINS, /* # of VLAN tag insertions */ + + SGE_PSTAT_MAX /* must be last */ +}; + +struct napi_gro_fraginfo; + +struct sge_qset { /* an SGE queue set */ + struct adapter *adap; + struct napi_struct napi; + struct sge_rspq rspq; + struct sge_fl fl[SGE_RXQ_PER_SET]; + struct sge_txq txq[SGE_TXQ_PER_SET]; + int nomem; + void *lro_va; + struct net_device *netdev; + struct netdev_queue *tx_q; /* associated netdev TX queue */ + unsigned long txq_stopped; /* which Tx queues are stopped */ + struct timer_list tx_reclaim_timer; /* reclaims TX buffers */ + struct timer_list rx_reclaim_timer; /* reclaims RX buffers */ + unsigned long port_stats[SGE_PSTAT_MAX]; +} ____cacheline_aligned; + +struct sge { + struct sge_qset qs[SGE_QSETS]; + spinlock_t reg_lock; /* guards non-atomic SGE registers (eg context) */ +}; + +struct adapter { + struct t3cdev tdev; + struct list_head adapter_list; + void __iomem *regs; + struct pci_dev *pdev; + unsigned long registered_device_map; + unsigned long open_device_map; + unsigned long flags; + + const char *name; + int msg_enable; + unsigned int mmio_len; + + struct adapter_params params; + unsigned int slow_intr_mask; + unsigned long irq_stats[IRQ_NUM_STATS]; + + int msix_nvectors; + struct { + unsigned short vec; + char desc[22]; + } msix_info[SGE_QSETS + 1]; + + /* T3 modules */ + struct sge sge; + struct mc7 pmrx; + struct mc7 pmtx; + struct mc7 cm; + struct mc5 mc5; + + struct net_device *port[MAX_NPORTS]; + unsigned int check_task_cnt; + struct delayed_work adap_check_task; + struct work_struct ext_intr_handler_task; + struct work_struct fatal_error_handler_task; + struct work_struct link_fault_handler_task; + + struct work_struct db_full_task; + struct work_struct db_empty_task; + struct work_struct db_drop_task; + + struct dentry *debugfs_root; + + struct mutex mdio_lock; + spinlock_t stats_lock; + spinlock_t work_lock; + + struct sk_buff *nofail_skb; +}; + +static inline u32 t3_read_reg(struct adapter *adapter, u32 reg_addr) +{ + u32 val = readl(adapter->regs + reg_addr); + + CH_DBG(adapter, MMIO, "read register 0x%x value 0x%x\n", reg_addr, val); + return val; +} + +static inline void t3_write_reg(struct adapter *adapter, u32 reg_addr, u32 val) +{ + CH_DBG(adapter, MMIO, "setting register 0x%x to 0x%x\n", reg_addr, val); + writel(val, adapter->regs + reg_addr); +} + +static inline struct port_info *adap2pinfo(struct adapter *adap, int idx) +{ + return netdev_priv(adap->port[idx]); +} + +static inline int phy2portid(struct cphy *phy) +{ + struct adapter *adap = phy->adapter; + struct port_info *port0 = adap2pinfo(adap, 0); + + return &port0->phy == phy ? 0 : 1; +} + +#define OFFLOAD_DEVMAP_BIT 15 + +#define tdev2adap(d) container_of(d, struct adapter, tdev) + +static inline int offload_running(struct adapter *adapter) +{ + return test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map); +} + +int t3_offload_tx(struct t3cdev *tdev, struct sk_buff *skb); + +void t3_os_ext_intr_handler(struct adapter *adapter); +void t3_os_link_changed(struct adapter *adapter, int port_id, int link_status, + int speed, int duplex, int fc); +void t3_os_phymod_changed(struct adapter *adap, int port_id); +void t3_os_link_fault(struct adapter *adapter, int port_id, int state); +void t3_os_link_fault_handler(struct adapter *adapter, int port_id); + +void t3_sge_start(struct adapter *adap); +void t3_sge_stop_dma(struct adapter *adap); +void t3_sge_stop(struct adapter *adap); +void t3_start_sge_timers(struct adapter *adap); +void t3_stop_sge_timers(struct adapter *adap); +void t3_free_sge_resources(struct adapter *adap); +void t3_sge_err_intr_handler(struct adapter *adapter); +irq_handler_t t3_intr_handler(struct adapter *adap, int polling); +netdev_tx_t t3_eth_xmit(struct sk_buff *skb, struct net_device *dev); +int t3_mgmt_tx(struct adapter *adap, struct sk_buff *skb); +void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p); +int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports, + int irq_vec_idx, const struct qset_params *p, + int ntxq, struct net_device *dev, + struct netdev_queue *netdevq); +extern struct workqueue_struct *cxgb3_wq; + +int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size); + +#endif /* __T3_ADAPTER_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb3/ael1002.c b/drivers/net/ethernet/chelsio/cxgb3/ael1002.c new file mode 100644 index 000000000..9d591f0dd --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/ael1002.c @@ -0,0 +1,936 @@ +/* + * Copyright (c) 2005-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "common.h" +#include "regs.h" + +enum { + AEL100X_TX_CONFIG1 = 0xc002, + AEL1002_PWR_DOWN_HI = 0xc011, + AEL1002_PWR_DOWN_LO = 0xc012, + AEL1002_XFI_EQL = 0xc015, + AEL1002_LB_EN = 0xc017, + AEL_OPT_SETTINGS = 0xc017, + AEL_I2C_CTRL = 0xc30a, + AEL_I2C_DATA = 0xc30b, + AEL_I2C_STAT = 0xc30c, + AEL2005_GPIO_CTRL = 0xc214, + AEL2005_GPIO_STAT = 0xc215, + + AEL2020_GPIO_INTR = 0xc103, /* Latch High (LH) */ + AEL2020_GPIO_CTRL = 0xc108, /* Store Clear (SC) */ + AEL2020_GPIO_STAT = 0xc10c, /* Read Only (RO) */ + AEL2020_GPIO_CFG = 0xc110, /* Read Write (RW) */ + + AEL2020_GPIO_SDA = 0, /* IN: i2c serial data */ + AEL2020_GPIO_MODDET = 1, /* IN: Module Detect */ + AEL2020_GPIO_0 = 3, /* IN: unassigned */ + AEL2020_GPIO_1 = 2, /* OUT: unassigned */ + AEL2020_GPIO_LSTAT = AEL2020_GPIO_1, /* wired to link status LED */ +}; + +enum { edc_none, edc_sr, edc_twinax }; + +/* PHY module I2C device address */ +enum { + MODULE_DEV_ADDR = 0xa0, + SFF_DEV_ADDR = 0xa2, +}; + +/* PHY transceiver type */ +enum { + phy_transtype_unknown = 0, + phy_transtype_sfp = 3, + phy_transtype_xfp = 6, +}; + +#define AEL2005_MODDET_IRQ 4 + +struct reg_val { + unsigned short mmd_addr; + unsigned short reg_addr; + unsigned short clear_bits; + unsigned short set_bits; +}; + +static int set_phy_regs(struct cphy *phy, const struct reg_val *rv) +{ + int err; + + for (err = 0; rv->mmd_addr && !err; rv++) { + if (rv->clear_bits == 0xffff) + err = t3_mdio_write(phy, rv->mmd_addr, rv->reg_addr, + rv->set_bits); + else + err = t3_mdio_change_bits(phy, rv->mmd_addr, + rv->reg_addr, rv->clear_bits, + rv->set_bits); + } + return err; +} + +static void ael100x_txon(struct cphy *phy) +{ + int tx_on_gpio = + phy->mdio.prtad == 0 ? F_GPIO7_OUT_VAL : F_GPIO2_OUT_VAL; + + msleep(100); + t3_set_reg_field(phy->adapter, A_T3DBG_GPIO_EN, 0, tx_on_gpio); + msleep(30); +} + +/* + * Read an 8-bit word from a device attached to the PHY's i2c bus. + */ +static int ael_i2c_rd(struct cphy *phy, int dev_addr, int word_addr) +{ + int i, err; + unsigned int stat, data; + + err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, AEL_I2C_CTRL, + (dev_addr << 8) | (1 << 8) | word_addr); + if (err) + return err; + + for (i = 0; i < 200; i++) { + msleep(1); + err = t3_mdio_read(phy, MDIO_MMD_PMAPMD, AEL_I2C_STAT, &stat); + if (err) + return err; + if ((stat & 3) == 1) { + err = t3_mdio_read(phy, MDIO_MMD_PMAPMD, AEL_I2C_DATA, + &data); + if (err) + return err; + return data >> 8; + } + } + CH_WARN(phy->adapter, "PHY %u i2c read of dev.addr %#x.%#x timed out\n", + phy->mdio.prtad, dev_addr, word_addr); + return -ETIMEDOUT; +} + +static int ael1002_power_down(struct cphy *phy, int enable) +{ + int err; + + err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, MDIO_PMA_TXDIS, !!enable); + if (!err) + err = mdio_set_flag(&phy->mdio, phy->mdio.prtad, + MDIO_MMD_PMAPMD, MDIO_CTRL1, + MDIO_CTRL1_LPOWER, enable); + return err; +} + +static int ael1002_reset(struct cphy *phy, int wait) +{ + int err; + + if ((err = ael1002_power_down(phy, 0)) || + (err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, AEL100X_TX_CONFIG1, 1)) || + (err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, AEL1002_PWR_DOWN_HI, 0)) || + (err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, AEL1002_PWR_DOWN_LO, 0)) || + (err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, AEL1002_XFI_EQL, 0x18)) || + (err = t3_mdio_change_bits(phy, MDIO_MMD_PMAPMD, AEL1002_LB_EN, + 0, 1 << 5))) + return err; + return 0; +} + +static int ael1002_intr_noop(struct cphy *phy) +{ + return 0; +} + +/* + * Get link status for a 10GBASE-R device. + */ +static int get_link_status_r(struct cphy *phy, int *link_ok, int *speed, + int *duplex, int *fc) +{ + if (link_ok) { + unsigned int stat0, stat1, stat2; + int err = t3_mdio_read(phy, MDIO_MMD_PMAPMD, + MDIO_PMA_RXDET, &stat0); + + if (!err) + err = t3_mdio_read(phy, MDIO_MMD_PCS, + MDIO_PCS_10GBRT_STAT1, &stat1); + if (!err) + err = t3_mdio_read(phy, MDIO_MMD_PHYXS, + MDIO_PHYXS_LNSTAT, &stat2); + if (err) + return err; + *link_ok = (stat0 & stat1 & (stat2 >> 12)) & 1; + } + if (speed) + *speed = SPEED_10000; + if (duplex) + *duplex = DUPLEX_FULL; + return 0; +} + +static const struct cphy_ops ael1002_ops = { + .reset = ael1002_reset, + .intr_enable = ael1002_intr_noop, + .intr_disable = ael1002_intr_noop, + .intr_clear = ael1002_intr_noop, + .intr_handler = ael1002_intr_noop, + .get_link_status = get_link_status_r, + .power_down = ael1002_power_down, + .mmds = MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS | MDIO_DEVS_PHYXS, +}; + +int t3_ael1002_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops) +{ + cphy_init(phy, adapter, phy_addr, &ael1002_ops, mdio_ops, + SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_FIBRE, + "10GBASE-R"); + ael100x_txon(phy); + return 0; +} + +static int ael1006_reset(struct cphy *phy, int wait) +{ + return t3_phy_reset(phy, MDIO_MMD_PMAPMD, wait); +} + +static const struct cphy_ops ael1006_ops = { + .reset = ael1006_reset, + .intr_enable = t3_phy_lasi_intr_enable, + .intr_disable = t3_phy_lasi_intr_disable, + .intr_clear = t3_phy_lasi_intr_clear, + .intr_handler = t3_phy_lasi_intr_handler, + .get_link_status = get_link_status_r, + .power_down = ael1002_power_down, + .mmds = MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS | MDIO_DEVS_PHYXS, +}; + +int t3_ael1006_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops) +{ + cphy_init(phy, adapter, phy_addr, &ael1006_ops, mdio_ops, + SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_FIBRE, + "10GBASE-SR"); + ael100x_txon(phy); + return 0; +} + +/* + * Decode our module type. + */ +static int ael2xxx_get_module_type(struct cphy *phy, int delay_ms) +{ + int v; + + if (delay_ms) + msleep(delay_ms); + + /* see SFF-8472 for below */ + v = ael_i2c_rd(phy, MODULE_DEV_ADDR, 3); + if (v < 0) + return v; + + if (v == 0x10) + return phy_modtype_sr; + if (v == 0x20) + return phy_modtype_lr; + if (v == 0x40) + return phy_modtype_lrm; + + v = ael_i2c_rd(phy, MODULE_DEV_ADDR, 6); + if (v < 0) + return v; + if (v != 4) + goto unknown; + + v = ael_i2c_rd(phy, MODULE_DEV_ADDR, 10); + if (v < 0) + return v; + + if (v & 0x80) { + v = ael_i2c_rd(phy, MODULE_DEV_ADDR, 0x12); + if (v < 0) + return v; + return v > 10 ? phy_modtype_twinax_long : phy_modtype_twinax; + } +unknown: + return phy_modtype_unknown; +} + +/* + * Code to support the Aeluros/NetLogic 2005 10Gb PHY. + */ +static int ael2005_setup_sr_edc(struct cphy *phy) +{ + static const struct reg_val regs[] = { + { MDIO_MMD_PMAPMD, 0xc003, 0xffff, 0x181 }, + { MDIO_MMD_PMAPMD, 0xc010, 0xffff, 0x448a }, + { MDIO_MMD_PMAPMD, 0xc04a, 0xffff, 0x5200 }, + { 0, 0, 0, 0 } + }; + + int i, err; + + err = set_phy_regs(phy, regs); + if (err) + return err; + + msleep(50); + + if (phy->priv != edc_sr) + err = t3_get_edc_fw(phy, EDC_OPT_AEL2005, + EDC_OPT_AEL2005_SIZE); + if (err) + return err; + + for (i = 0; i < EDC_OPT_AEL2005_SIZE / sizeof(u16) && !err; i += 2) + err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, + phy->phy_cache[i], + phy->phy_cache[i + 1]); + if (!err) + phy->priv = edc_sr; + return err; +} + +static int ael2005_setup_twinax_edc(struct cphy *phy, int modtype) +{ + static const struct reg_val regs[] = { + { MDIO_MMD_PMAPMD, 0xc04a, 0xffff, 0x5a00 }, + { 0, 0, 0, 0 } + }; + static const struct reg_val preemphasis[] = { + { MDIO_MMD_PMAPMD, 0xc014, 0xffff, 0xfe16 }, + { MDIO_MMD_PMAPMD, 0xc015, 0xffff, 0xa000 }, + { 0, 0, 0, 0 } + }; + int i, err; + + err = set_phy_regs(phy, regs); + if (!err && modtype == phy_modtype_twinax_long) + err = set_phy_regs(phy, preemphasis); + if (err) + return err; + + msleep(50); + + if (phy->priv != edc_twinax) + err = t3_get_edc_fw(phy, EDC_TWX_AEL2005, + EDC_TWX_AEL2005_SIZE); + if (err) + return err; + + for (i = 0; i < EDC_TWX_AEL2005_SIZE / sizeof(u16) && !err; i += 2) + err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, + phy->phy_cache[i], + phy->phy_cache[i + 1]); + if (!err) + phy->priv = edc_twinax; + return err; +} + +static int ael2005_get_module_type(struct cphy *phy, int delay_ms) +{ + int v; + unsigned int stat; + + v = t3_mdio_read(phy, MDIO_MMD_PMAPMD, AEL2005_GPIO_CTRL, &stat); + if (v) + return v; + + if (stat & (1 << 8)) /* module absent */ + return phy_modtype_none; + + return ael2xxx_get_module_type(phy, delay_ms); +} + +static int ael2005_intr_enable(struct cphy *phy) +{ + int err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, AEL2005_GPIO_CTRL, 0x200); + return err ? err : t3_phy_lasi_intr_enable(phy); +} + +static int ael2005_intr_disable(struct cphy *phy) +{ + int err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, AEL2005_GPIO_CTRL, 0x100); + return err ? err : t3_phy_lasi_intr_disable(phy); +} + +static int ael2005_intr_clear(struct cphy *phy) +{ + int err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, AEL2005_GPIO_CTRL, 0xd00); + return err ? err : t3_phy_lasi_intr_clear(phy); +} + +static int ael2005_reset(struct cphy *phy, int wait) +{ + static const struct reg_val regs0[] = { + { MDIO_MMD_PMAPMD, 0xc001, 0, 1 << 5 }, + { MDIO_MMD_PMAPMD, 0xc017, 0, 1 << 5 }, + { MDIO_MMD_PMAPMD, 0xc013, 0xffff, 0xf341 }, + { MDIO_MMD_PMAPMD, 0xc210, 0xffff, 0x8000 }, + { MDIO_MMD_PMAPMD, 0xc210, 0xffff, 0x8100 }, + { MDIO_MMD_PMAPMD, 0xc210, 0xffff, 0x8000 }, + { MDIO_MMD_PMAPMD, 0xc210, 0xffff, 0 }, + { 0, 0, 0, 0 } + }; + static const struct reg_val regs1[] = { + { MDIO_MMD_PMAPMD, 0xca00, 0xffff, 0x0080 }, + { MDIO_MMD_PMAPMD, 0xca12, 0xffff, 0 }, + { 0, 0, 0, 0 } + }; + + int err; + unsigned int lasi_ctrl; + + err = t3_mdio_read(phy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_CTRL, + &lasi_ctrl); + if (err) + return err; + + err = t3_phy_reset(phy, MDIO_MMD_PMAPMD, 0); + if (err) + return err; + + msleep(125); + phy->priv = edc_none; + err = set_phy_regs(phy, regs0); + if (err) + return err; + + msleep(50); + + err = ael2005_get_module_type(phy, 0); + if (err < 0) + return err; + phy->modtype = err; + + if (err == phy_modtype_twinax || err == phy_modtype_twinax_long) + err = ael2005_setup_twinax_edc(phy, err); + else + err = ael2005_setup_sr_edc(phy); + if (err) + return err; + + err = set_phy_regs(phy, regs1); + if (err) + return err; + + /* reset wipes out interrupts, reenable them if they were on */ + if (lasi_ctrl & 1) + err = ael2005_intr_enable(phy); + return err; +} + +static int ael2005_intr_handler(struct cphy *phy) +{ + unsigned int stat; + int ret, edc_needed, cause = 0; + + ret = t3_mdio_read(phy, MDIO_MMD_PMAPMD, AEL2005_GPIO_STAT, &stat); + if (ret) + return ret; + + if (stat & AEL2005_MODDET_IRQ) { + ret = t3_mdio_write(phy, MDIO_MMD_PMAPMD, AEL2005_GPIO_CTRL, + 0xd00); + if (ret) + return ret; + + /* modules have max 300 ms init time after hot plug */ + ret = ael2005_get_module_type(phy, 300); + if (ret < 0) + return ret; + + phy->modtype = ret; + if (ret == phy_modtype_none) + edc_needed = phy->priv; /* on unplug retain EDC */ + else if (ret == phy_modtype_twinax || + ret == phy_modtype_twinax_long) + edc_needed = edc_twinax; + else + edc_needed = edc_sr; + + if (edc_needed != phy->priv) { + ret = ael2005_reset(phy, 0); + return ret ? ret : cphy_cause_module_change; + } + cause = cphy_cause_module_change; + } + + ret = t3_phy_lasi_intr_handler(phy); + if (ret < 0) + return ret; + + ret |= cause; + return ret ? ret : cphy_cause_link_change; +} + +static const struct cphy_ops ael2005_ops = { + .reset = ael2005_reset, + .intr_enable = ael2005_intr_enable, + .intr_disable = ael2005_intr_disable, + .intr_clear = ael2005_intr_clear, + .intr_handler = ael2005_intr_handler, + .get_link_status = get_link_status_r, + .power_down = ael1002_power_down, + .mmds = MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS | MDIO_DEVS_PHYXS, +}; + +int t3_ael2005_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops) +{ + cphy_init(phy, adapter, phy_addr, &ael2005_ops, mdio_ops, + SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_FIBRE | + SUPPORTED_IRQ, "10GBASE-R"); + msleep(125); + return t3_mdio_change_bits(phy, MDIO_MMD_PMAPMD, AEL_OPT_SETTINGS, 0, + 1 << 5); +} + +/* + * Setup EDC and other parameters for operation with an optical module. + */ +static int ael2020_setup_sr_edc(struct cphy *phy) +{ + static const struct reg_val regs[] = { + /* set CDR offset to 10 */ + { MDIO_MMD_PMAPMD, 0xcc01, 0xffff, 0x488a }, + + /* adjust 10G RX bias current */ + { MDIO_MMD_PMAPMD, 0xcb1b, 0xffff, 0x0200 }, + { MDIO_MMD_PMAPMD, 0xcb1c, 0xffff, 0x00f0 }, + { MDIO_MMD_PMAPMD, 0xcc06, 0xffff, 0x00e0 }, + + /* end */ + { 0, 0, 0, 0 } + }; + int err; + + err = set_phy_regs(phy, regs); + msleep(50); + if (err) + return err; + + phy->priv = edc_sr; + return 0; +} + +/* + * Setup EDC and other parameters for operation with an TWINAX module. + */ +static int ael2020_setup_twinax_edc(struct cphy *phy, int modtype) +{ + /* set uC to 40MHz */ + static const struct reg_val uCclock40MHz[] = { + { MDIO_MMD_PMAPMD, 0xff28, 0xffff, 0x4001 }, + { MDIO_MMD_PMAPMD, 0xff2a, 0xffff, 0x0002 }, + { 0, 0, 0, 0 } + }; + + /* activate uC clock */ + static const struct reg_val uCclockActivate[] = { + { MDIO_MMD_PMAPMD, 0xd000, 0xffff, 0x5200 }, + { 0, 0, 0, 0 } + }; + + /* set PC to start of SRAM and activate uC */ + static const struct reg_val uCactivate[] = { + { MDIO_MMD_PMAPMD, 0xd080, 0xffff, 0x0100 }, + { MDIO_MMD_PMAPMD, 0xd092, 0xffff, 0x0000 }, + { 0, 0, 0, 0 } + }; + int i, err; + + /* set uC clock and activate it */ + err = set_phy_regs(phy, uCclock40MHz); + msleep(500); + if (err) + return err; + err = set_phy_regs(phy, uCclockActivate); + msleep(500); + if (err) + return err; + + if (phy->priv != edc_twinax) + err = t3_get_edc_fw(phy, EDC_TWX_AEL2020, + EDC_TWX_AEL2020_SIZE); + if (err) + return err; + + for (i = 0; i < EDC_TWX_AEL2020_SIZE / sizeof(u16) && !err; i += 2) + err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, + phy->phy_cache[i], + phy->phy_cache[i + 1]); + /* activate uC */ + err = set_phy_regs(phy, uCactivate); + if (!err) + phy->priv = edc_twinax; + return err; +} + +/* + * Return Module Type. + */ +static int ael2020_get_module_type(struct cphy *phy, int delay_ms) +{ + int v; + unsigned int stat; + + v = t3_mdio_read(phy, MDIO_MMD_PMAPMD, AEL2020_GPIO_STAT, &stat); + if (v) + return v; + + if (stat & (0x1 << (AEL2020_GPIO_MODDET*4))) { + /* module absent */ + return phy_modtype_none; + } + + return ael2xxx_get_module_type(phy, delay_ms); +} + +/* + * Enable PHY interrupts. We enable "Module Detection" interrupts (on any + * state transition) and then generic Link Alarm Status Interrupt (LASI). + */ +static int ael2020_intr_enable(struct cphy *phy) +{ + static const struct reg_val regs[] = { + /* output Module's Loss Of Signal (LOS) to LED */ + { MDIO_MMD_PMAPMD, AEL2020_GPIO_CFG+AEL2020_GPIO_LSTAT, + 0xffff, 0x4 }, + { MDIO_MMD_PMAPMD, AEL2020_GPIO_CTRL, + 0xffff, 0x8 << (AEL2020_GPIO_LSTAT*4) }, + + /* enable module detect status change interrupts */ + { MDIO_MMD_PMAPMD, AEL2020_GPIO_CTRL, + 0xffff, 0x2 << (AEL2020_GPIO_MODDET*4) }, + + /* end */ + { 0, 0, 0, 0 } + }; + int err, link_ok = 0; + + /* set up "link status" LED and enable module change interrupts */ + err = set_phy_regs(phy, regs); + if (err) + return err; + + err = get_link_status_r(phy, &link_ok, NULL, NULL, NULL); + if (err) + return err; + if (link_ok) + t3_link_changed(phy->adapter, + phy2portid(phy)); + + err = t3_phy_lasi_intr_enable(phy); + if (err) + return err; + + return 0; +} + +/* + * Disable PHY interrupts. The mirror of the above ... + */ +static int ael2020_intr_disable(struct cphy *phy) +{ + static const struct reg_val regs[] = { + /* reset "link status" LED to "off" */ + { MDIO_MMD_PMAPMD, AEL2020_GPIO_CTRL, + 0xffff, 0xb << (AEL2020_GPIO_LSTAT*4) }, + + /* disable module detect status change interrupts */ + { MDIO_MMD_PMAPMD, AEL2020_GPIO_CTRL, + 0xffff, 0x1 << (AEL2020_GPIO_MODDET*4) }, + + /* end */ + { 0, 0, 0, 0 } + }; + int err; + + /* turn off "link status" LED and disable module change interrupts */ + err = set_phy_regs(phy, regs); + if (err) + return err; + + return t3_phy_lasi_intr_disable(phy); +} + +/* + * Clear PHY interrupt state. + */ +static int ael2020_intr_clear(struct cphy *phy) +{ + /* + * The GPIO Interrupt register on the AEL2020 is a "Latching High" + * (LH) register which is cleared to the current state when it's read. + * Thus, we simply read the register and discard the result. + */ + unsigned int stat; + int err = t3_mdio_read(phy, MDIO_MMD_PMAPMD, AEL2020_GPIO_INTR, &stat); + return err ? err : t3_phy_lasi_intr_clear(phy); +} + +static const struct reg_val ael2020_reset_regs[] = { + /* Erratum #2: CDRLOL asserted, causing PMA link down status */ + { MDIO_MMD_PMAPMD, 0xc003, 0xffff, 0x3101 }, + + /* force XAUI to send LF when RX_LOS is asserted */ + { MDIO_MMD_PMAPMD, 0xcd40, 0xffff, 0x0001 }, + + /* allow writes to transceiver module EEPROM on i2c bus */ + { MDIO_MMD_PMAPMD, 0xff02, 0xffff, 0x0023 }, + { MDIO_MMD_PMAPMD, 0xff03, 0xffff, 0x0000 }, + { MDIO_MMD_PMAPMD, 0xff04, 0xffff, 0x0000 }, + + /* end */ + { 0, 0, 0, 0 } +}; +/* + * Reset the PHY and put it into a canonical operating state. + */ +static int ael2020_reset(struct cphy *phy, int wait) +{ + int err; + unsigned int lasi_ctrl; + + /* grab current interrupt state */ + err = t3_mdio_read(phy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_CTRL, + &lasi_ctrl); + if (err) + return err; + + err = t3_phy_reset(phy, MDIO_MMD_PMAPMD, 125); + if (err) + return err; + msleep(100); + + /* basic initialization for all module types */ + phy->priv = edc_none; + err = set_phy_regs(phy, ael2020_reset_regs); + if (err) + return err; + + /* determine module type and perform appropriate initialization */ + err = ael2020_get_module_type(phy, 0); + if (err < 0) + return err; + phy->modtype = (u8)err; + if (err == phy_modtype_twinax || err == phy_modtype_twinax_long) + err = ael2020_setup_twinax_edc(phy, err); + else + err = ael2020_setup_sr_edc(phy); + if (err) + return err; + + /* reset wipes out interrupts, reenable them if they were on */ + if (lasi_ctrl & 1) + err = ael2005_intr_enable(phy); + return err; +} + +/* + * Handle a PHY interrupt. + */ +static int ael2020_intr_handler(struct cphy *phy) +{ + unsigned int stat; + int ret, edc_needed, cause = 0; + + ret = t3_mdio_read(phy, MDIO_MMD_PMAPMD, AEL2020_GPIO_INTR, &stat); + if (ret) + return ret; + + if (stat & (0x1 << AEL2020_GPIO_MODDET)) { + /* modules have max 300 ms init time after hot plug */ + ret = ael2020_get_module_type(phy, 300); + if (ret < 0) + return ret; + + phy->modtype = (u8)ret; + if (ret == phy_modtype_none) + edc_needed = phy->priv; /* on unplug retain EDC */ + else if (ret == phy_modtype_twinax || + ret == phy_modtype_twinax_long) + edc_needed = edc_twinax; + else + edc_needed = edc_sr; + + if (edc_needed != phy->priv) { + ret = ael2020_reset(phy, 0); + return ret ? ret : cphy_cause_module_change; + } + cause = cphy_cause_module_change; + } + + ret = t3_phy_lasi_intr_handler(phy); + if (ret < 0) + return ret; + + ret |= cause; + return ret ? ret : cphy_cause_link_change; +} + +static const struct cphy_ops ael2020_ops = { + .reset = ael2020_reset, + .intr_enable = ael2020_intr_enable, + .intr_disable = ael2020_intr_disable, + .intr_clear = ael2020_intr_clear, + .intr_handler = ael2020_intr_handler, + .get_link_status = get_link_status_r, + .power_down = ael1002_power_down, + .mmds = MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS | MDIO_DEVS_PHYXS, +}; + +int t3_ael2020_phy_prep(struct cphy *phy, struct adapter *adapter, int phy_addr, + const struct mdio_ops *mdio_ops) +{ + cphy_init(phy, adapter, phy_addr, &ael2020_ops, mdio_ops, + SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_FIBRE | + SUPPORTED_IRQ, "10GBASE-R"); + msleep(125); + + return set_phy_regs(phy, ael2020_reset_regs); +} + +/* + * Get link status for a 10GBASE-X device. + */ +static int get_link_status_x(struct cphy *phy, int *link_ok, int *speed, + int *duplex, int *fc) +{ + if (link_ok) { + unsigned int stat0, stat1, stat2; + int err = t3_mdio_read(phy, MDIO_MMD_PMAPMD, + MDIO_PMA_RXDET, &stat0); + + if (!err) + err = t3_mdio_read(phy, MDIO_MMD_PCS, + MDIO_PCS_10GBX_STAT1, &stat1); + if (!err) + err = t3_mdio_read(phy, MDIO_MMD_PHYXS, + MDIO_PHYXS_LNSTAT, &stat2); + if (err) + return err; + *link_ok = (stat0 & (stat1 >> 12) & (stat2 >> 12)) & 1; + } + if (speed) + *speed = SPEED_10000; + if (duplex) + *duplex = DUPLEX_FULL; + return 0; +} + +static const struct cphy_ops qt2045_ops = { + .reset = ael1006_reset, + .intr_enable = t3_phy_lasi_intr_enable, + .intr_disable = t3_phy_lasi_intr_disable, + .intr_clear = t3_phy_lasi_intr_clear, + .intr_handler = t3_phy_lasi_intr_handler, + .get_link_status = get_link_status_x, + .power_down = ael1002_power_down, + .mmds = MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS | MDIO_DEVS_PHYXS, +}; + +int t3_qt2045_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops) +{ + unsigned int stat; + + cphy_init(phy, adapter, phy_addr, &qt2045_ops, mdio_ops, + SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_TP, + "10GBASE-CX4"); + + /* + * Some cards where the PHY is supposed to be at address 0 actually + * have it at 1. + */ + if (!phy_addr && + !t3_mdio_read(phy, MDIO_MMD_PMAPMD, MDIO_STAT1, &stat) && + stat == 0xffff) + phy->mdio.prtad = 1; + return 0; +} + +static int xaui_direct_reset(struct cphy *phy, int wait) +{ + return 0; +} + +static int xaui_direct_get_link_status(struct cphy *phy, int *link_ok, + int *speed, int *duplex, int *fc) +{ + if (link_ok) { + unsigned int status; + int prtad = phy->mdio.prtad; + + status = t3_read_reg(phy->adapter, + XGM_REG(A_XGM_SERDES_STAT0, prtad)) | + t3_read_reg(phy->adapter, + XGM_REG(A_XGM_SERDES_STAT1, prtad)) | + t3_read_reg(phy->adapter, + XGM_REG(A_XGM_SERDES_STAT2, prtad)) | + t3_read_reg(phy->adapter, + XGM_REG(A_XGM_SERDES_STAT3, prtad)); + *link_ok = !(status & F_LOWSIG0); + } + if (speed) + *speed = SPEED_10000; + if (duplex) + *duplex = DUPLEX_FULL; + return 0; +} + +static int xaui_direct_power_down(struct cphy *phy, int enable) +{ + return 0; +} + +static const struct cphy_ops xaui_direct_ops = { + .reset = xaui_direct_reset, + .intr_enable = ael1002_intr_noop, + .intr_disable = ael1002_intr_noop, + .intr_clear = ael1002_intr_noop, + .intr_handler = ael1002_intr_noop, + .get_link_status = xaui_direct_get_link_status, + .power_down = xaui_direct_power_down, +}; + +int t3_xaui_direct_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops) +{ + cphy_init(phy, adapter, phy_addr, &xaui_direct_ops, mdio_ops, + SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_TP, + "10GBASE-CX4"); + return 0; +} diff --git a/drivers/net/ethernet/chelsio/cxgb3/aq100x.c b/drivers/net/ethernet/chelsio/cxgb3/aq100x.c new file mode 100644 index 000000000..6af5d200e --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/aq100x.c @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2005-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "common.h" +#include "regs.h" + +enum { + /* MDIO_DEV_PMA_PMD registers */ + AQ_LINK_STAT = 0xe800, + AQ_IMASK_PMA = 0xf000, + + /* MDIO_DEV_XGXS registers */ + AQ_XAUI_RX_CFG = 0xc400, + AQ_XAUI_TX_CFG = 0xe400, + + /* MDIO_DEV_ANEG registers */ + AQ_1G_CTRL = 0xc400, + AQ_ANEG_STAT = 0xc800, + + /* MDIO_DEV_VEND1 registers */ + AQ_FW_VERSION = 0x0020, + AQ_IFLAG_GLOBAL = 0xfc00, + AQ_IMASK_GLOBAL = 0xff00, +}; + +enum { + IMASK_PMA = 1 << 2, + IMASK_GLOBAL = 1 << 15, + ADV_1G_FULL = 1 << 15, + ADV_1G_HALF = 1 << 14, + ADV_10G_FULL = 1 << 12, + AQ_RESET = (1 << 14) | (1 << 15), + AQ_LOWPOWER = 1 << 12, +}; + +static int aq100x_reset(struct cphy *phy, int wait) +{ + /* + * Ignore the caller specified wait time; always wait for the reset to + * complete. Can take up to 3s. + */ + int err = t3_phy_reset(phy, MDIO_MMD_VEND1, 3000); + + if (err) + CH_WARN(phy->adapter, "PHY%d: reset failed (0x%x).\n", + phy->mdio.prtad, err); + + return err; +} + +static int aq100x_intr_enable(struct cphy *phy) +{ + int err = t3_mdio_write(phy, MDIO_MMD_PMAPMD, AQ_IMASK_PMA, IMASK_PMA); + if (err) + return err; + + err = t3_mdio_write(phy, MDIO_MMD_VEND1, AQ_IMASK_GLOBAL, IMASK_GLOBAL); + return err; +} + +static int aq100x_intr_disable(struct cphy *phy) +{ + return t3_mdio_write(phy, MDIO_MMD_VEND1, AQ_IMASK_GLOBAL, 0); +} + +static int aq100x_intr_clear(struct cphy *phy) +{ + unsigned int v; + + t3_mdio_read(phy, MDIO_MMD_VEND1, AQ_IFLAG_GLOBAL, &v); + t3_mdio_read(phy, MDIO_MMD_PMAPMD, MDIO_STAT1, &v); + + return 0; +} + +static int aq100x_intr_handler(struct cphy *phy) +{ + int err; + unsigned int cause, v; + + err = t3_mdio_read(phy, MDIO_MMD_VEND1, AQ_IFLAG_GLOBAL, &cause); + if (err) + return err; + + /* Read (and reset) the latching version of the status */ + t3_mdio_read(phy, MDIO_MMD_PMAPMD, MDIO_STAT1, &v); + + return cphy_cause_link_change; +} + +static int aq100x_power_down(struct cphy *phy, int off) +{ + return mdio_set_flag(&phy->mdio, phy->mdio.prtad, + MDIO_MMD_PMAPMD, MDIO_CTRL1, + MDIO_CTRL1_LPOWER, off); +} + +static int aq100x_autoneg_enable(struct cphy *phy) +{ + int err; + + err = aq100x_power_down(phy, 0); + if (!err) + err = mdio_set_flag(&phy->mdio, phy->mdio.prtad, + MDIO_MMD_AN, MDIO_CTRL1, + BMCR_ANENABLE | BMCR_ANRESTART, 1); + + return err; +} + +static int aq100x_autoneg_restart(struct cphy *phy) +{ + int err; + + err = aq100x_power_down(phy, 0); + if (!err) + err = mdio_set_flag(&phy->mdio, phy->mdio.prtad, + MDIO_MMD_AN, MDIO_CTRL1, + BMCR_ANENABLE | BMCR_ANRESTART, 1); + + return err; +} + +static int aq100x_advertise(struct cphy *phy, unsigned int advertise_map) +{ + unsigned int adv; + int err; + + /* 10G advertisement */ + adv = 0; + if (advertise_map & ADVERTISED_10000baseT_Full) + adv |= ADV_10G_FULL; + err = t3_mdio_change_bits(phy, MDIO_MMD_AN, MDIO_AN_10GBT_CTRL, + ADV_10G_FULL, adv); + if (err) + return err; + + /* 1G advertisement */ + adv = 0; + if (advertise_map & ADVERTISED_1000baseT_Full) + adv |= ADV_1G_FULL; + if (advertise_map & ADVERTISED_1000baseT_Half) + adv |= ADV_1G_HALF; + err = t3_mdio_change_bits(phy, MDIO_MMD_AN, AQ_1G_CTRL, + ADV_1G_FULL | ADV_1G_HALF, adv); + if (err) + return err; + + /* 100M, pause advertisement */ + adv = 0; + if (advertise_map & ADVERTISED_100baseT_Half) + adv |= ADVERTISE_100HALF; + if (advertise_map & ADVERTISED_100baseT_Full) + adv |= ADVERTISE_100FULL; + if (advertise_map & ADVERTISED_Pause) + adv |= ADVERTISE_PAUSE_CAP; + if (advertise_map & ADVERTISED_Asym_Pause) + adv |= ADVERTISE_PAUSE_ASYM; + err = t3_mdio_change_bits(phy, MDIO_MMD_AN, MDIO_AN_ADVERTISE, + 0xfe0, adv); + + return err; +} + +static int aq100x_set_loopback(struct cphy *phy, int mmd, int dir, int enable) +{ + return mdio_set_flag(&phy->mdio, phy->mdio.prtad, + MDIO_MMD_PMAPMD, MDIO_CTRL1, + BMCR_LOOPBACK, enable); +} + +static int aq100x_set_speed_duplex(struct cphy *phy, int speed, int duplex) +{ + /* no can do */ + return -1; +} + +static int aq100x_get_link_status(struct cphy *phy, int *link_ok, + int *speed, int *duplex, int *fc) +{ + int err; + unsigned int v; + + if (link_ok) { + err = t3_mdio_read(phy, MDIO_MMD_PMAPMD, AQ_LINK_STAT, &v); + if (err) + return err; + + *link_ok = v & 1; + if (!*link_ok) + return 0; + } + + err = t3_mdio_read(phy, MDIO_MMD_AN, AQ_ANEG_STAT, &v); + if (err) + return err; + + if (speed) { + switch (v & 0x6) { + case 0x6: + *speed = SPEED_10000; + break; + case 0x4: + *speed = SPEED_1000; + break; + case 0x2: + *speed = SPEED_100; + break; + case 0x0: + *speed = SPEED_10; + break; + } + } + + if (duplex) + *duplex = v & 1 ? DUPLEX_FULL : DUPLEX_HALF; + + return 0; +} + +static const struct cphy_ops aq100x_ops = { + .reset = aq100x_reset, + .intr_enable = aq100x_intr_enable, + .intr_disable = aq100x_intr_disable, + .intr_clear = aq100x_intr_clear, + .intr_handler = aq100x_intr_handler, + .autoneg_enable = aq100x_autoneg_enable, + .autoneg_restart = aq100x_autoneg_restart, + .advertise = aq100x_advertise, + .set_loopback = aq100x_set_loopback, + .set_speed_duplex = aq100x_set_speed_duplex, + .get_link_status = aq100x_get_link_status, + .power_down = aq100x_power_down, + .mmds = MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS | MDIO_DEVS_PHYXS, +}; + +int t3_aq100x_phy_prep(struct cphy *phy, struct adapter *adapter, int phy_addr, + const struct mdio_ops *mdio_ops) +{ + unsigned int v, v2, gpio, wait; + int err; + + cphy_init(phy, adapter, phy_addr, &aq100x_ops, mdio_ops, + SUPPORTED_1000baseT_Full | SUPPORTED_10000baseT_Full | + SUPPORTED_TP | SUPPORTED_Autoneg | SUPPORTED_AUI, + "1000/10GBASE-T"); + + /* + * The PHY has been out of reset ever since the system powered up. So + * we do a hard reset over here. + */ + gpio = phy_addr ? F_GPIO10_OUT_VAL : F_GPIO6_OUT_VAL; + t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, gpio, 0); + msleep(1); + t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, gpio, gpio); + + /* + * Give it enough time to load the firmware and get ready for mdio. + */ + msleep(1000); + wait = 500; /* in 10ms increments */ + do { + err = t3_mdio_read(phy, MDIO_MMD_VEND1, MDIO_CTRL1, &v); + if (err || v == 0xffff) { + + /* Allow prep_adapter to succeed when ffff is read */ + + CH_WARN(adapter, "PHY%d: reset failed (0x%x, 0x%x).\n", + phy_addr, err, v); + goto done; + } + + v &= AQ_RESET; + if (v) + msleep(10); + } while (v && --wait); + if (v) { + CH_WARN(adapter, "PHY%d: reset timed out (0x%x).\n", + phy_addr, v); + + goto done; /* let prep_adapter succeed */ + } + + /* Datasheet says 3s max but this has been observed */ + wait = (500 - wait) * 10 + 1000; + if (wait > 3000) + CH_WARN(adapter, "PHY%d: reset took %ums\n", phy_addr, wait); + + /* Firmware version check. */ + t3_mdio_read(phy, MDIO_MMD_VEND1, AQ_FW_VERSION, &v); + if (v != 101) + CH_WARN(adapter, "PHY%d: unsupported firmware %d\n", + phy_addr, v); + + /* + * The PHY should start in really-low-power mode. Prepare it for normal + * operations. + */ + err = t3_mdio_read(phy, MDIO_MMD_VEND1, MDIO_CTRL1, &v); + if (err) + return err; + if (v & AQ_LOWPOWER) { + err = t3_mdio_change_bits(phy, MDIO_MMD_VEND1, MDIO_CTRL1, + AQ_LOWPOWER, 0); + if (err) + return err; + msleep(10); + } else + CH_WARN(adapter, "PHY%d does not start in low power mode.\n", + phy_addr); + + /* + * Verify XAUI settings, but let prep succeed no matter what. + */ + v = v2 = 0; + t3_mdio_read(phy, MDIO_MMD_PHYXS, AQ_XAUI_RX_CFG, &v); + t3_mdio_read(phy, MDIO_MMD_PHYXS, AQ_XAUI_TX_CFG, &v2); + if (v != 0x1b || v2 != 0x1b) + CH_WARN(adapter, + "PHY%d: incorrect XAUI settings (0x%x, 0x%x).\n", + phy_addr, v, v2); + +done: + return err; +} diff --git a/drivers/net/ethernet/chelsio/cxgb3/common.h b/drivers/net/ethernet/chelsio/cxgb3/common.h new file mode 100644 index 000000000..ecd025dda --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/common.h @@ -0,0 +1,773 @@ +/* + * Copyright (c) 2005-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef __CHELSIO_COMMON_H +#define __CHELSIO_COMMON_H + +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/ctype.h> +#include <linux/delay.h> +#include <linux/netdevice.h> +#include <linux/ethtool.h> +#include <linux/mdio.h> +#include "version.h" + +#define CH_ERR(adap, fmt, ...) dev_err(&adap->pdev->dev, fmt, ##__VA_ARGS__) +#define CH_WARN(adap, fmt, ...) dev_warn(&adap->pdev->dev, fmt, ##__VA_ARGS__) +#define CH_ALERT(adap, fmt, ...) dev_alert(&adap->pdev->dev, fmt, ##__VA_ARGS__) + +/* + * More powerful macro that selectively prints messages based on msg_enable. + * For info and debugging messages. + */ +#define CH_MSG(adapter, level, category, fmt, ...) do { \ + if ((adapter)->msg_enable & NETIF_MSG_##category) \ + dev_printk(KERN_##level, &adapter->pdev->dev, fmt, \ + ## __VA_ARGS__); \ +} while (0) + +#ifdef DEBUG +# define CH_DBG(adapter, category, fmt, ...) \ + CH_MSG(adapter, DEBUG, category, fmt, ## __VA_ARGS__) +#else +# define CH_DBG(adapter, category, fmt, ...) +#endif + +/* Additional NETIF_MSG_* categories */ +#define NETIF_MSG_MMIO 0x8000000 + +enum { + MAX_NPORTS = 2, /* max # of ports */ + MAX_FRAME_SIZE = 10240, /* max MAC frame size, including header + FCS */ + EEPROMSIZE = 8192, /* Serial EEPROM size */ + SERNUM_LEN = 16, /* Serial # length */ + RSS_TABLE_SIZE = 64, /* size of RSS lookup and mapping tables */ + TCB_SIZE = 128, /* TCB size */ + NMTUS = 16, /* size of MTU table */ + NCCTRL_WIN = 32, /* # of congestion control windows */ + PROTO_SRAM_LINES = 128, /* size of TP sram */ +}; + +#define MAX_RX_COALESCING_LEN 12288U + +enum { + PAUSE_RX = 1 << 0, + PAUSE_TX = 1 << 1, + PAUSE_AUTONEG = 1 << 2 +}; + +enum { + SUPPORTED_IRQ = 1 << 24 +}; + +enum { /* adapter interrupt-maintained statistics */ + STAT_ULP_CH0_PBL_OOB, + STAT_ULP_CH1_PBL_OOB, + STAT_PCI_CORR_ECC, + + IRQ_NUM_STATS /* keep last */ +}; + +#define TP_VERSION_MAJOR 1 +#define TP_VERSION_MINOR 1 +#define TP_VERSION_MICRO 0 + +#define S_TP_VERSION_MAJOR 16 +#define M_TP_VERSION_MAJOR 0xFF +#define V_TP_VERSION_MAJOR(x) ((x) << S_TP_VERSION_MAJOR) +#define G_TP_VERSION_MAJOR(x) \ + (((x) >> S_TP_VERSION_MAJOR) & M_TP_VERSION_MAJOR) + +#define S_TP_VERSION_MINOR 8 +#define M_TP_VERSION_MINOR 0xFF +#define V_TP_VERSION_MINOR(x) ((x) << S_TP_VERSION_MINOR) +#define G_TP_VERSION_MINOR(x) \ + (((x) >> S_TP_VERSION_MINOR) & M_TP_VERSION_MINOR) + +#define S_TP_VERSION_MICRO 0 +#define M_TP_VERSION_MICRO 0xFF +#define V_TP_VERSION_MICRO(x) ((x) << S_TP_VERSION_MICRO) +#define G_TP_VERSION_MICRO(x) \ + (((x) >> S_TP_VERSION_MICRO) & M_TP_VERSION_MICRO) + +enum { + SGE_QSETS = 8, /* # of SGE Tx/Rx/RspQ sets */ + SGE_RXQ_PER_SET = 2, /* # of Rx queues per set */ + SGE_TXQ_PER_SET = 3 /* # of Tx queues per set */ +}; + +enum sge_context_type { /* SGE egress context types */ + SGE_CNTXT_RDMA = 0, + SGE_CNTXT_ETH = 2, + SGE_CNTXT_OFLD = 4, + SGE_CNTXT_CTRL = 5 +}; + +enum { + AN_PKT_SIZE = 32, /* async notification packet size */ + IMMED_PKT_SIZE = 48 /* packet size for immediate data */ +}; + +struct sg_ent { /* SGE scatter/gather entry */ + __be32 len[2]; + __be64 addr[2]; +}; + +#ifndef SGE_NUM_GENBITS +/* Must be 1 or 2 */ +# define SGE_NUM_GENBITS 2 +#endif + +#define TX_DESC_FLITS 16U +#define WR_FLITS (TX_DESC_FLITS + 1 - SGE_NUM_GENBITS) + +struct cphy; +struct adapter; + +struct mdio_ops { + int (*read)(struct net_device *dev, int phy_addr, int mmd_addr, + u16 reg_addr); + int (*write)(struct net_device *dev, int phy_addr, int mmd_addr, + u16 reg_addr, u16 val); + unsigned mode_support; +}; + +struct adapter_info { + unsigned char nports0; /* # of ports on channel 0 */ + unsigned char nports1; /* # of ports on channel 1 */ + unsigned char phy_base_addr; /* MDIO PHY base address */ + unsigned int gpio_out; /* GPIO output settings */ + unsigned char gpio_intr[MAX_NPORTS]; /* GPIO PHY IRQ pins */ + unsigned long caps; /* adapter capabilities */ + const struct mdio_ops *mdio_ops; /* MDIO operations */ + const char *desc; /* product description */ +}; + +struct mc5_stats { + unsigned long parity_err; + unsigned long active_rgn_full; + unsigned long nfa_srch_err; + unsigned long unknown_cmd; + unsigned long reqq_parity_err; + unsigned long dispq_parity_err; + unsigned long del_act_empty; +}; + +struct mc7_stats { + unsigned long corr_err; + unsigned long uncorr_err; + unsigned long parity_err; + unsigned long addr_err; +}; + +struct mac_stats { + u64 tx_octets; /* total # of octets in good frames */ + u64 tx_octets_bad; /* total # of octets in error frames */ + u64 tx_frames; /* all good frames */ + u64 tx_mcast_frames; /* good multicast frames */ + u64 tx_bcast_frames; /* good broadcast frames */ + u64 tx_pause; /* # of transmitted pause frames */ + u64 tx_deferred; /* frames with deferred transmissions */ + u64 tx_late_collisions; /* # of late collisions */ + u64 tx_total_collisions; /* # of total collisions */ + u64 tx_excess_collisions; /* frame errors from excessive collissions */ + u64 tx_underrun; /* # of Tx FIFO underruns */ + u64 tx_len_errs; /* # of Tx length errors */ + u64 tx_mac_internal_errs; /* # of internal MAC errors on Tx */ + u64 tx_excess_deferral; /* # of frames with excessive deferral */ + u64 tx_fcs_errs; /* # of frames with bad FCS */ + + u64 tx_frames_64; /* # of Tx frames in a particular range */ + u64 tx_frames_65_127; + u64 tx_frames_128_255; + u64 tx_frames_256_511; + u64 tx_frames_512_1023; + u64 tx_frames_1024_1518; + u64 tx_frames_1519_max; + + u64 rx_octets; /* total # of octets in good frames */ + u64 rx_octets_bad; /* total # of octets in error frames */ + u64 rx_frames; /* all good frames */ + u64 rx_mcast_frames; /* good multicast frames */ + u64 rx_bcast_frames; /* good broadcast frames */ + u64 rx_pause; /* # of received pause frames */ + u64 rx_fcs_errs; /* # of received frames with bad FCS */ + u64 rx_align_errs; /* alignment errors */ + u64 rx_symbol_errs; /* symbol errors */ + u64 rx_data_errs; /* data errors */ + u64 rx_sequence_errs; /* sequence errors */ + u64 rx_runt; /* # of runt frames */ + u64 rx_jabber; /* # of jabber frames */ + u64 rx_short; /* # of short frames */ + u64 rx_too_long; /* # of oversized frames */ + u64 rx_mac_internal_errs; /* # of internal MAC errors on Rx */ + + u64 rx_frames_64; /* # of Rx frames in a particular range */ + u64 rx_frames_65_127; + u64 rx_frames_128_255; + u64 rx_frames_256_511; + u64 rx_frames_512_1023; + u64 rx_frames_1024_1518; + u64 rx_frames_1519_max; + + u64 rx_cong_drops; /* # of Rx drops due to SGE congestion */ + + unsigned long tx_fifo_parity_err; + unsigned long rx_fifo_parity_err; + unsigned long tx_fifo_urun; + unsigned long rx_fifo_ovfl; + unsigned long serdes_signal_loss; + unsigned long xaui_pcs_ctc_err; + unsigned long xaui_pcs_align_change; + + unsigned long num_toggled; /* # times toggled TxEn due to stuck TX */ + unsigned long num_resets; /* # times reset due to stuck TX */ + + unsigned long link_faults; /* # detected link faults */ +}; + +struct tp_mib_stats { + u32 ipInReceive_hi; + u32 ipInReceive_lo; + u32 ipInHdrErrors_hi; + u32 ipInHdrErrors_lo; + u32 ipInAddrErrors_hi; + u32 ipInAddrErrors_lo; + u32 ipInUnknownProtos_hi; + u32 ipInUnknownProtos_lo; + u32 ipInDiscards_hi; + u32 ipInDiscards_lo; + u32 ipInDelivers_hi; + u32 ipInDelivers_lo; + u32 ipOutRequests_hi; + u32 ipOutRequests_lo; + u32 ipOutDiscards_hi; + u32 ipOutDiscards_lo; + u32 ipOutNoRoutes_hi; + u32 ipOutNoRoutes_lo; + u32 ipReasmTimeout; + u32 ipReasmReqds; + u32 ipReasmOKs; + u32 ipReasmFails; + + u32 reserved[8]; + + u32 tcpActiveOpens; + u32 tcpPassiveOpens; + u32 tcpAttemptFails; + u32 tcpEstabResets; + u32 tcpOutRsts; + u32 tcpCurrEstab; + u32 tcpInSegs_hi; + u32 tcpInSegs_lo; + u32 tcpOutSegs_hi; + u32 tcpOutSegs_lo; + u32 tcpRetransSeg_hi; + u32 tcpRetransSeg_lo; + u32 tcpInErrs_hi; + u32 tcpInErrs_lo; + u32 tcpRtoMin; + u32 tcpRtoMax; +}; + +struct tp_params { + unsigned int nchan; /* # of channels */ + unsigned int pmrx_size; /* total PMRX capacity */ + unsigned int pmtx_size; /* total PMTX capacity */ + unsigned int cm_size; /* total CM capacity */ + unsigned int chan_rx_size; /* per channel Rx size */ + unsigned int chan_tx_size; /* per channel Tx size */ + unsigned int rx_pg_size; /* Rx page size */ + unsigned int tx_pg_size; /* Tx page size */ + unsigned int rx_num_pgs; /* # of Rx pages */ + unsigned int tx_num_pgs; /* # of Tx pages */ + unsigned int ntimer_qs; /* # of timer queues */ +}; + +struct qset_params { /* SGE queue set parameters */ + unsigned int polling; /* polling/interrupt service for rspq */ + unsigned int coalesce_usecs; /* irq coalescing timer */ + unsigned int rspq_size; /* # of entries in response queue */ + unsigned int fl_size; /* # of entries in regular free list */ + unsigned int jumbo_size; /* # of entries in jumbo free list */ + unsigned int txq_size[SGE_TXQ_PER_SET]; /* Tx queue sizes */ + unsigned int cong_thres; /* FL congestion threshold */ + unsigned int vector; /* Interrupt (line or vector) number */ +}; + +struct sge_params { + unsigned int max_pkt_size; /* max offload pkt size */ + struct qset_params qset[SGE_QSETS]; +}; + +struct mc5_params { + unsigned int mode; /* selects MC5 width */ + unsigned int nservers; /* size of server region */ + unsigned int nfilters; /* size of filter region */ + unsigned int nroutes; /* size of routing region */ +}; + +/* Default MC5 region sizes */ +enum { + DEFAULT_NSERVERS = 512, + DEFAULT_NFILTERS = 128 +}; + +/* MC5 modes, these must be non-0 */ +enum { + MC5_MODE_144_BIT = 1, + MC5_MODE_72_BIT = 2 +}; + +/* MC5 min active region size */ +enum { MC5_MIN_TIDS = 16 }; + +struct vpd_params { + unsigned int cclk; + unsigned int mclk; + unsigned int uclk; + unsigned int mdc; + unsigned int mem_timing; + u8 sn[SERNUM_LEN + 1]; + u8 eth_base[6]; + u8 port_type[MAX_NPORTS]; + unsigned short xauicfg[2]; +}; + +struct pci_params { + unsigned int vpd_cap_addr; + unsigned short speed; + unsigned char width; + unsigned char variant; +}; + +enum { + PCI_VARIANT_PCI, + PCI_VARIANT_PCIX_MODE1_PARITY, + PCI_VARIANT_PCIX_MODE1_ECC, + PCI_VARIANT_PCIX_266_MODE2, + PCI_VARIANT_PCIE +}; + +struct adapter_params { + struct sge_params sge; + struct mc5_params mc5; + struct tp_params tp; + struct vpd_params vpd; + struct pci_params pci; + + const struct adapter_info *info; + + unsigned short mtus[NMTUS]; + unsigned short a_wnd[NCCTRL_WIN]; + unsigned short b_wnd[NCCTRL_WIN]; + + unsigned int nports; /* # of ethernet ports */ + unsigned int chan_map; /* bitmap of in-use Tx channels */ + unsigned int stats_update_period; /* MAC stats accumulation period */ + unsigned int linkpoll_period; /* link poll period in 0.1s */ + unsigned int rev; /* chip revision */ + unsigned int offload; +}; + +enum { /* chip revisions */ + T3_REV_A = 0, + T3_REV_B = 2, + T3_REV_B2 = 3, + T3_REV_C = 4, +}; + +struct trace_params { + u32 sip; + u32 sip_mask; + u32 dip; + u32 dip_mask; + u16 sport; + u16 sport_mask; + u16 dport; + u16 dport_mask; + u32 vlan:12; + u32 vlan_mask:12; + u32 intf:4; + u32 intf_mask:4; + u8 proto; + u8 proto_mask; +}; + +struct link_config { + unsigned int supported; /* link capabilities */ + unsigned int advertising; /* advertised capabilities */ + unsigned short requested_speed; /* speed user has requested */ + unsigned short speed; /* actual link speed */ + unsigned char requested_duplex; /* duplex user has requested */ + unsigned char duplex; /* actual link duplex */ + unsigned char requested_fc; /* flow control user has requested */ + unsigned char fc; /* actual link flow control */ + unsigned char autoneg; /* autonegotiating? */ + unsigned int link_ok; /* link up? */ +}; + +#define SPEED_INVALID 0xffff +#define DUPLEX_INVALID 0xff + +struct mc5 { + struct adapter *adapter; + unsigned int tcam_size; + unsigned char part_type; + unsigned char parity_enabled; + unsigned char mode; + struct mc5_stats stats; +}; + +static inline unsigned int t3_mc5_size(const struct mc5 *p) +{ + return p->tcam_size; +} + +struct mc7 { + struct adapter *adapter; /* backpointer to adapter */ + unsigned int size; /* memory size in bytes */ + unsigned int width; /* MC7 interface width */ + unsigned int offset; /* register address offset for MC7 instance */ + const char *name; /* name of MC7 instance */ + struct mc7_stats stats; /* MC7 statistics */ +}; + +static inline unsigned int t3_mc7_size(const struct mc7 *p) +{ + return p->size; +} + +struct cmac { + struct adapter *adapter; + unsigned int offset; + unsigned int nucast; /* # of address filters for unicast MACs */ + unsigned int tx_tcnt; + unsigned int tx_xcnt; + u64 tx_mcnt; + unsigned int rx_xcnt; + unsigned int rx_ocnt; + u64 rx_mcnt; + unsigned int toggle_cnt; + unsigned int txen; + u64 rx_pause; + struct mac_stats stats; +}; + +enum { + MAC_DIRECTION_RX = 1, + MAC_DIRECTION_TX = 2, + MAC_RXFIFO_SIZE = 32768 +}; + +/* PHY loopback direction */ +enum { + PHY_LOOPBACK_TX = 1, + PHY_LOOPBACK_RX = 2 +}; + +/* PHY interrupt types */ +enum { + cphy_cause_link_change = 1, + cphy_cause_fifo_error = 2, + cphy_cause_module_change = 4, +}; + +/* PHY module types */ +enum { + phy_modtype_none, + phy_modtype_sr, + phy_modtype_lr, + phy_modtype_lrm, + phy_modtype_twinax, + phy_modtype_twinax_long, + phy_modtype_unknown +}; + +/* PHY operations */ +struct cphy_ops { + int (*reset)(struct cphy *phy, int wait); + + int (*intr_enable)(struct cphy *phy); + int (*intr_disable)(struct cphy *phy); + int (*intr_clear)(struct cphy *phy); + int (*intr_handler)(struct cphy *phy); + + int (*autoneg_enable)(struct cphy *phy); + int (*autoneg_restart)(struct cphy *phy); + + int (*advertise)(struct cphy *phy, unsigned int advertise_map); + int (*set_loopback)(struct cphy *phy, int mmd, int dir, int enable); + int (*set_speed_duplex)(struct cphy *phy, int speed, int duplex); + int (*get_link_status)(struct cphy *phy, int *link_ok, int *speed, + int *duplex, int *fc); + int (*power_down)(struct cphy *phy, int enable); + + u32 mmds; +}; +enum { + EDC_OPT_AEL2005 = 0, + EDC_OPT_AEL2005_SIZE = 1084, + EDC_TWX_AEL2005 = 1, + EDC_TWX_AEL2005_SIZE = 1464, + EDC_TWX_AEL2020 = 2, + EDC_TWX_AEL2020_SIZE = 1628, + EDC_MAX_SIZE = EDC_TWX_AEL2020_SIZE, /* Max cache size */ +}; + +/* A PHY instance */ +struct cphy { + u8 modtype; /* PHY module type */ + short priv; /* scratch pad */ + unsigned int caps; /* PHY capabilities */ + struct adapter *adapter; /* associated adapter */ + const char *desc; /* PHY description */ + unsigned long fifo_errors; /* FIFO over/under-flows */ + const struct cphy_ops *ops; /* PHY operations */ + struct mdio_if_info mdio; + u16 phy_cache[EDC_MAX_SIZE]; /* EDC cache */ +}; + +/* Convenience MDIO read/write wrappers */ +static inline int t3_mdio_read(struct cphy *phy, int mmd, int reg, + unsigned int *valp) +{ + int rc = phy->mdio.mdio_read(phy->mdio.dev, phy->mdio.prtad, mmd, reg); + *valp = (rc >= 0) ? rc : -1; + return (rc >= 0) ? 0 : rc; +} + +static inline int t3_mdio_write(struct cphy *phy, int mmd, int reg, + unsigned int val) +{ + return phy->mdio.mdio_write(phy->mdio.dev, phy->mdio.prtad, mmd, + reg, val); +} + +/* Convenience initializer */ +static inline void cphy_init(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct cphy_ops *phy_ops, + const struct mdio_ops *mdio_ops, + unsigned int caps, const char *desc) +{ + phy->caps = caps; + phy->adapter = adapter; + phy->desc = desc; + phy->ops = phy_ops; + if (mdio_ops) { + phy->mdio.prtad = phy_addr; + phy->mdio.mmds = phy_ops->mmds; + phy->mdio.mode_support = mdio_ops->mode_support; + phy->mdio.mdio_read = mdio_ops->read; + phy->mdio.mdio_write = mdio_ops->write; + } +} + +/* Accumulate MAC statistics every 180 seconds. For 1G we multiply by 10. */ +#define MAC_STATS_ACCUM_SECS 180 + +#define XGM_REG(reg_addr, idx) \ + ((reg_addr) + (idx) * (XGMAC0_1_BASE_ADDR - XGMAC0_0_BASE_ADDR)) + +struct addr_val_pair { + unsigned int reg_addr; + unsigned int val; +}; + +#include "adapter.h" + +#ifndef PCI_VENDOR_ID_CHELSIO +# define PCI_VENDOR_ID_CHELSIO 0x1425 +#endif + +#define for_each_port(adapter, iter) \ + for (iter = 0; iter < (adapter)->params.nports; ++iter) + +#define adapter_info(adap) ((adap)->params.info) + +static inline int uses_xaui(const struct adapter *adap) +{ + return adapter_info(adap)->caps & SUPPORTED_AUI; +} + +static inline int is_10G(const struct adapter *adap) +{ + return adapter_info(adap)->caps & SUPPORTED_10000baseT_Full; +} + +static inline int is_offload(const struct adapter *adap) +{ + return adap->params.offload; +} + +static inline unsigned int core_ticks_per_usec(const struct adapter *adap) +{ + return adap->params.vpd.cclk / 1000; +} + +static inline unsigned int is_pcie(const struct adapter *adap) +{ + return adap->params.pci.variant == PCI_VARIANT_PCIE; +} + +void t3_set_reg_field(struct adapter *adap, unsigned int addr, u32 mask, + u32 val); +void t3_write_regs(struct adapter *adapter, const struct addr_val_pair *p, + int n, unsigned int offset); +int t3_wait_op_done_val(struct adapter *adapter, int reg, u32 mask, + int polarity, int attempts, int delay, u32 *valp); +static inline int t3_wait_op_done(struct adapter *adapter, int reg, u32 mask, + int polarity, int attempts, int delay) +{ + return t3_wait_op_done_val(adapter, reg, mask, polarity, attempts, + delay, NULL); +} +int t3_mdio_change_bits(struct cphy *phy, int mmd, int reg, unsigned int clear, + unsigned int set); +int t3_phy_reset(struct cphy *phy, int mmd, int wait); +int t3_phy_advertise(struct cphy *phy, unsigned int advert); +int t3_phy_advertise_fiber(struct cphy *phy, unsigned int advert); +int t3_set_phy_speed_duplex(struct cphy *phy, int speed, int duplex); +int t3_phy_lasi_intr_enable(struct cphy *phy); +int t3_phy_lasi_intr_disable(struct cphy *phy); +int t3_phy_lasi_intr_clear(struct cphy *phy); +int t3_phy_lasi_intr_handler(struct cphy *phy); + +void t3_intr_enable(struct adapter *adapter); +void t3_intr_disable(struct adapter *adapter); +void t3_intr_clear(struct adapter *adapter); +void t3_xgm_intr_enable(struct adapter *adapter, int idx); +void t3_xgm_intr_disable(struct adapter *adapter, int idx); +void t3_port_intr_enable(struct adapter *adapter, int idx); +void t3_port_intr_disable(struct adapter *adapter, int idx); +int t3_slow_intr_handler(struct adapter *adapter); +int t3_phy_intr_handler(struct adapter *adapter); + +void t3_link_changed(struct adapter *adapter, int port_id); +void t3_link_fault(struct adapter *adapter, int port_id); +int t3_link_start(struct cphy *phy, struct cmac *mac, struct link_config *lc); +const struct adapter_info *t3_get_adapter_info(unsigned int board_id); +int t3_seeprom_wp(struct adapter *adapter, int enable); +int t3_get_tp_version(struct adapter *adapter, u32 *vers); +int t3_check_tpsram_version(struct adapter *adapter); +int t3_check_tpsram(struct adapter *adapter, const u8 *tp_ram, + unsigned int size); +int t3_set_proto_sram(struct adapter *adap, const u8 *data); +int t3_load_fw(struct adapter *adapter, const u8 * fw_data, unsigned int size); +int t3_get_fw_version(struct adapter *adapter, u32 *vers); +int t3_check_fw_version(struct adapter *adapter); +int t3_init_hw(struct adapter *adapter, u32 fw_params); +int t3_reset_adapter(struct adapter *adapter); +int t3_prep_adapter(struct adapter *adapter, const struct adapter_info *ai, + int reset); +int t3_replay_prep_adapter(struct adapter *adapter); +void t3_led_ready(struct adapter *adapter); +void t3_fatal_err(struct adapter *adapter); +void t3_set_vlan_accel(struct adapter *adapter, unsigned int ports, int on); +void t3_config_rss(struct adapter *adapter, unsigned int rss_config, + const u8 * cpus, const u16 *rspq); +int t3_cim_ctl_blk_read(struct adapter *adap, unsigned int addr, + unsigned int n, unsigned int *valp); +int t3_mc7_bd_read(struct mc7 *mc7, unsigned int start, unsigned int n, + u64 *buf); + +int t3_mac_reset(struct cmac *mac); +void t3b_pcs_reset(struct cmac *mac); +void t3_mac_disable_exact_filters(struct cmac *mac); +void t3_mac_enable_exact_filters(struct cmac *mac); +int t3_mac_enable(struct cmac *mac, int which); +int t3_mac_disable(struct cmac *mac, int which); +int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu); +int t3_mac_set_rx_mode(struct cmac *mac, struct net_device *dev); +int t3_mac_set_address(struct cmac *mac, unsigned int idx, const u8 addr[6]); +int t3_mac_set_num_ucast(struct cmac *mac, int n); +const struct mac_stats *t3_mac_update_stats(struct cmac *mac); +int t3_mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, int fc); +int t3b2_mac_watchdog_task(struct cmac *mac); + +void t3_mc5_prep(struct adapter *adapter, struct mc5 *mc5, int mode); +int t3_mc5_init(struct mc5 *mc5, unsigned int nservers, unsigned int nfilters, + unsigned int nroutes); +void t3_mc5_intr_handler(struct mc5 *mc5); + +void t3_tp_set_offload_mode(struct adapter *adap, int enable); +void t3_tp_get_mib_stats(struct adapter *adap, struct tp_mib_stats *tps); +void t3_load_mtus(struct adapter *adap, unsigned short mtus[NMTUS], + unsigned short alpha[NCCTRL_WIN], + unsigned short beta[NCCTRL_WIN], unsigned short mtu_cap); +void t3_config_trace_filter(struct adapter *adapter, + const struct trace_params *tp, int filter_index, + int invert, int enable); +int t3_config_sched(struct adapter *adap, unsigned int kbps, int sched); + +void t3_sge_prep(struct adapter *adap, struct sge_params *p); +void t3_sge_init(struct adapter *adap, struct sge_params *p); +int t3_sge_init_ecntxt(struct adapter *adapter, unsigned int id, int gts_enable, + enum sge_context_type type, int respq, u64 base_addr, + unsigned int size, unsigned int token, int gen, + unsigned int cidx); +int t3_sge_init_flcntxt(struct adapter *adapter, unsigned int id, + int gts_enable, u64 base_addr, unsigned int size, + unsigned int esize, unsigned int cong_thres, int gen, + unsigned int cidx); +int t3_sge_init_rspcntxt(struct adapter *adapter, unsigned int id, + int irq_vec_idx, u64 base_addr, unsigned int size, + unsigned int fl_thres, int gen, unsigned int cidx); +int t3_sge_init_cqcntxt(struct adapter *adapter, unsigned int id, u64 base_addr, + unsigned int size, int rspq, int ovfl_mode, + unsigned int credits, unsigned int credit_thres); +int t3_sge_enable_ecntxt(struct adapter *adapter, unsigned int id, int enable); +int t3_sge_disable_fl(struct adapter *adapter, unsigned int id); +int t3_sge_disable_rspcntxt(struct adapter *adapter, unsigned int id); +int t3_sge_disable_cqcntxt(struct adapter *adapter, unsigned int id); +int t3_sge_cqcntxt_op(struct adapter *adapter, unsigned int id, unsigned int op, + unsigned int credits); + +int t3_vsc8211_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops); +int t3_ael1002_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops); +int t3_ael1006_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops); +int t3_ael2005_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops); +int t3_ael2020_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops); +int t3_qt2045_phy_prep(struct cphy *phy, struct adapter *adapter, int phy_addr, + const struct mdio_ops *mdio_ops); +int t3_xaui_direct_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops); +int t3_aq100x_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops); + +extern struct workqueue_struct *cxgb3_wq; +#endif /* __CHELSIO_COMMON_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_ctl_defs.h b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_ctl_defs.h new file mode 100644 index 000000000..369fe711f --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_ctl_defs.h @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef _CXGB3_OFFLOAD_CTL_DEFS_H +#define _CXGB3_OFFLOAD_CTL_DEFS_H + +enum { + GET_MAX_OUTSTANDING_WR = 0, + GET_TX_MAX_CHUNK = 1, + GET_TID_RANGE = 2, + GET_STID_RANGE = 3, + GET_RTBL_RANGE = 4, + GET_L2T_CAPACITY = 5, + GET_MTUS = 6, + GET_WR_LEN = 7, + GET_IFF_FROM_MAC = 8, + GET_DDP_PARAMS = 9, + GET_PORTS = 10, + + ULP_ISCSI_GET_PARAMS = 11, + ULP_ISCSI_SET_PARAMS = 12, + + RDMA_GET_PARAMS = 13, + RDMA_CQ_OP = 14, + RDMA_CQ_SETUP = 15, + RDMA_CQ_DISABLE = 16, + RDMA_CTRL_QP_SETUP = 17, + RDMA_GET_MEM = 18, + RDMA_GET_MIB = 19, + + GET_RX_PAGE_INFO = 50, + GET_ISCSI_IPV4ADDR = 51, + + GET_EMBEDDED_INFO = 70, +}; + +/* + * Structure used to describe a TID range. Valid TIDs are [base, base+num). + */ +struct tid_range { + unsigned int base; /* first TID */ + unsigned int num; /* number of TIDs in range */ +}; + +/* + * Structure used to request the size and contents of the MTU table. + */ +struct mtutab { + unsigned int size; /* # of entries in the MTU table */ + const unsigned short *mtus; /* the MTU table values */ +}; + +struct net_device; + +/* + * Structure used to request the adapter net_device owning a given MAC address. + */ +struct iff_mac { + struct net_device *dev; /* the net_device */ + const unsigned char *mac_addr; /* MAC address to lookup */ + u16 vlan_tag; +}; + +/* Structure used to request a port's iSCSI IPv4 address */ +struct iscsi_ipv4addr { + struct net_device *dev; /* the net_device */ + __be32 ipv4addr; /* the return iSCSI IPv4 address */ +}; + +struct pci_dev; + +/* + * Structure used to request the TCP DDP parameters. + */ +struct ddp_params { + unsigned int llimit; /* TDDP region start address */ + unsigned int ulimit; /* TDDP region end address */ + unsigned int tag_mask; /* TDDP tag mask */ + struct pci_dev *pdev; +}; + +struct adap_ports { + unsigned int nports; /* number of ports on this adapter */ + struct net_device *lldevs[2]; +}; + +/* + * Structure used to return information to the iscsi layer. + */ +struct ulp_iscsi_info { + unsigned int offset; + unsigned int llimit; + unsigned int ulimit; + unsigned int tagmask; + u8 pgsz_factor[4]; + unsigned int max_rxsz; + unsigned int max_txsz; + struct pci_dev *pdev; +}; + +/* + * Structure used to return information to the RDMA layer. + */ +struct rdma_info { + unsigned int tpt_base; /* TPT base address */ + unsigned int tpt_top; /* TPT last entry address */ + unsigned int pbl_base; /* PBL base address */ + unsigned int pbl_top; /* PBL last entry address */ + unsigned int rqt_base; /* RQT base address */ + unsigned int rqt_top; /* RQT last entry address */ + unsigned int udbell_len; /* user doorbell region length */ + unsigned long udbell_physbase; /* user doorbell physical start addr */ + void __iomem *kdb_addr; /* kernel doorbell register address */ + struct pci_dev *pdev; /* associated PCI device */ +}; + +/* + * Structure used to request an operation on an RDMA completion queue. + */ +struct rdma_cq_op { + unsigned int id; + unsigned int op; + unsigned int credits; +}; + +/* + * Structure used to setup RDMA completion queues. + */ +struct rdma_cq_setup { + unsigned int id; + unsigned long long base_addr; + unsigned int size; + unsigned int credits; + unsigned int credit_thres; + unsigned int ovfl_mode; +}; + +/* + * Structure used to setup the RDMA control egress context. + */ +struct rdma_ctrlqp_setup { + unsigned long long base_addr; + unsigned int size; +}; + +/* + * Offload TX/RX page information. + */ +struct ofld_page_info { + unsigned int page_size; /* Page size, should be a power of 2 */ + unsigned int num; /* Number of pages */ +}; + +/* + * Structure used to get firmware and protocol engine versions. + */ +struct ch_embedded_info { + u32 fw_vers; + u32 tp_vers; +}; +#endif /* _CXGB3_OFFLOAD_CTL_DEFS_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_defs.h b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_defs.h new file mode 100644 index 000000000..f04e81f33 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_defs.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2006-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef _CHELSIO_DEFS_H +#define _CHELSIO_DEFS_H + +#include <linux/skbuff.h> +#include <net/tcp.h> + +#include "t3cdev.h" + +#include "cxgb3_offload.h" + +#define VALIDATE_TID 1 + +/* + * Map an ATID or STID to their entries in the corresponding TID tables. + */ +static inline union active_open_entry *atid2entry(const struct tid_info *t, + unsigned int atid) +{ + return &t->atid_tab[atid - t->atid_base]; +} + +static inline union listen_entry *stid2entry(const struct tid_info *t, + unsigned int stid) +{ + return &t->stid_tab[stid - t->stid_base]; +} + +/* + * Find the connection corresponding to a TID. + */ +static inline struct t3c_tid_entry *lookup_tid(const struct tid_info *t, + unsigned int tid) +{ + struct t3c_tid_entry *t3c_tid = tid < t->ntids ? + &(t->tid_tab[tid]) : NULL; + + return (t3c_tid && t3c_tid->client) ? t3c_tid : NULL; +} + +/* + * Find the connection corresponding to a server TID. + */ +static inline struct t3c_tid_entry *lookup_stid(const struct tid_info *t, + unsigned int tid) +{ + union listen_entry *e; + + if (tid < t->stid_base || tid >= t->stid_base + t->nstids) + return NULL; + + e = stid2entry(t, tid); + if ((void *)e->next >= (void *)t->tid_tab && + (void *)e->next < (void *)&t->atid_tab[t->natids]) + return NULL; + + return &e->t3c_tid; +} + +/* + * Find the connection corresponding to an active-open TID. + */ +static inline struct t3c_tid_entry *lookup_atid(const struct tid_info *t, + unsigned int tid) +{ + union active_open_entry *e; + + if (tid < t->atid_base || tid >= t->atid_base + t->natids) + return NULL; + + e = atid2entry(t, tid); + if ((void *)e->next >= (void *)t->tid_tab && + (void *)e->next < (void *)&t->atid_tab[t->natids]) + return NULL; + + return &e->t3c_tid; +} + +int attach_t3cdev(struct t3cdev *dev); +void detach_t3cdev(struct t3cdev *dev); +#endif diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_ioctl.h b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_ioctl.h new file mode 100644 index 000000000..401827b82 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_ioctl.h @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef __CHIOCTL_H__ +#define __CHIOCTL_H__ + +/* + * Ioctl commands specific to this driver. + */ +enum { + CHELSIO_GETMTUTAB = 1029, + CHELSIO_SETMTUTAB = 1030, + CHELSIO_SET_PM = 1032, + CHELSIO_GET_PM = 1033, + CHELSIO_GET_MEM = 1038, + CHELSIO_LOAD_FW = 1041, + CHELSIO_SET_TRACE_FILTER = 1044, + CHELSIO_SET_QSET_PARAMS = 1045, + CHELSIO_GET_QSET_PARAMS = 1046, + CHELSIO_SET_QSET_NUM = 1047, + CHELSIO_GET_QSET_NUM = 1048, +}; + +struct ch_reg { + uint32_t cmd; + uint32_t addr; + uint32_t val; +}; + +struct ch_cntxt { + uint32_t cmd; + uint32_t cntxt_type; + uint32_t cntxt_id; + uint32_t data[4]; +}; + +/* context types */ +enum { CNTXT_TYPE_EGRESS, CNTXT_TYPE_FL, CNTXT_TYPE_RSP, CNTXT_TYPE_CQ }; + +struct ch_desc { + uint32_t cmd; + uint32_t queue_num; + uint32_t idx; + uint32_t size; + uint8_t data[128]; +}; + +struct ch_mem_range { + uint32_t cmd; + uint32_t mem_id; + uint32_t addr; + uint32_t len; + uint32_t version; + uint8_t buf[]; +}; + +struct ch_qset_params { + uint32_t cmd; + uint32_t qset_idx; + int32_t txq_size[3]; + int32_t rspq_size; + int32_t fl_size[2]; + int32_t intr_lat; + int32_t polling; + int32_t lro; + int32_t cong_thres; + int32_t vector; + int32_t qnum; +}; + +struct ch_pktsched_params { + uint32_t cmd; + uint8_t sched; + uint8_t idx; + uint8_t min; + uint8_t max; + uint8_t binding; +}; + +#ifndef TCB_SIZE +# define TCB_SIZE 128 +#endif + +/* TCB size in 32-bit words */ +#define TCB_WORDS (TCB_SIZE / 4) + +enum { MEM_CM, MEM_PMRX, MEM_PMTX }; /* ch_mem_range.mem_id values */ + +struct ch_mtus { + uint32_t cmd; + uint32_t nmtus; + uint16_t mtus[NMTUS]; +}; + +struct ch_pm { + uint32_t cmd; + uint32_t tx_pg_sz; + uint32_t tx_num_pg; + uint32_t rx_pg_sz; + uint32_t rx_num_pg; + uint32_t pm_total; +}; + +struct ch_tcam { + uint32_t cmd; + uint32_t tcam_size; + uint32_t nservers; + uint32_t nroutes; + uint32_t nfilters; +}; + +struct ch_tcb { + uint32_t cmd; + uint32_t tcb_index; + uint32_t tcb_data[TCB_WORDS]; +}; + +struct ch_tcam_word { + uint32_t cmd; + uint32_t addr; + uint32_t buf[3]; +}; + +struct ch_trace { + uint32_t cmd; + uint32_t sip; + uint32_t sip_mask; + uint32_t dip; + uint32_t dip_mask; + uint16_t sport; + uint16_t sport_mask; + uint16_t dport; + uint16_t dport_mask; + uint32_t vlan:12; + uint32_t vlan_mask:12; + uint32_t intf:4; + uint32_t intf_mask:4; + uint8_t proto; + uint8_t proto_mask; + uint8_t invert_match:1; + uint8_t config_tx:1; + uint8_t config_rx:1; + uint8_t trace_tx:1; + uint8_t trace_rx:1; +}; + +#define SIOCCHIOCTL SIOCDEVPRIVATE + +#endif diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c new file mode 100644 index 000000000..9b84c8d8d --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c @@ -0,0 +1,3472 @@ +/* + * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/module.h> +#include <linux/init.h> +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> +#include <linux/if_vlan.h> +#include <linux/mdio.h> +#include <linux/sockios.h> +#include <linux/workqueue.h> +#include <linux/proc_fs.h> +#include <linux/rtnetlink.h> +#include <linux/firmware.h> +#include <linux/log2.h> +#include <linux/stringify.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/nospec.h> + +#include "common.h" +#include "cxgb3_ioctl.h" +#include "regs.h" +#include "cxgb3_offload.h" +#include "version.h" + +#include "cxgb3_ctl_defs.h" +#include "t3_cpl.h" +#include "firmware_exports.h" + +enum { + MAX_TXQ_ENTRIES = 16384, + MAX_CTRL_TXQ_ENTRIES = 1024, + MAX_RSPQ_ENTRIES = 16384, + MAX_RX_BUFFERS = 16384, + MAX_RX_JUMBO_BUFFERS = 16384, + MIN_TXQ_ENTRIES = 4, + MIN_CTRL_TXQ_ENTRIES = 4, + MIN_RSPQ_ENTRIES = 32, + MIN_FL_ENTRIES = 32 +}; + +#define PORT_MASK ((1 << MAX_NPORTS) - 1) + +#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \ + NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\ + NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) + +#define EEPROM_MAGIC 0x38E2F10C + +#define CH_DEVICE(devid, idx) \ + { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx } + +static const struct pci_device_id cxgb3_pci_tbl[] = { + CH_DEVICE(0x20, 0), /* PE9000 */ + CH_DEVICE(0x21, 1), /* T302E */ + CH_DEVICE(0x22, 2), /* T310E */ + CH_DEVICE(0x23, 3), /* T320X */ + CH_DEVICE(0x24, 1), /* T302X */ + CH_DEVICE(0x25, 3), /* T320E */ + CH_DEVICE(0x26, 2), /* T310X */ + CH_DEVICE(0x30, 2), /* T3B10 */ + CH_DEVICE(0x31, 3), /* T3B20 */ + CH_DEVICE(0x32, 1), /* T3B02 */ + CH_DEVICE(0x35, 6), /* T3C20-derived T3C10 */ + CH_DEVICE(0x36, 3), /* S320E-CR */ + CH_DEVICE(0x37, 7), /* N320E-G2 */ + {0,} +}; + +MODULE_DESCRIPTION(DRV_DESC); +MODULE_AUTHOR("Chelsio Communications"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl); + +static int dflt_msg_enable = DFLT_MSG_ENABLE; + +module_param(dflt_msg_enable, int, 0644); +MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap"); + +/* + * The driver uses the best interrupt scheme available on a platform in the + * order MSI-X, MSI, legacy pin interrupts. This parameter determines which + * of these schemes the driver may consider as follows: + * + * msi = 2: choose from among all three options + * msi = 1: only consider MSI and pin interrupts + * msi = 0: force pin interrupts + */ +static int msi = 2; + +module_param(msi, int, 0644); +MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X"); + +/* + * The driver enables offload as a default. + * To disable it, use ofld_disable = 1. + */ + +static int ofld_disable = 0; + +module_param(ofld_disable, int, 0644); +MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not"); + +/* + * We have work elements that we need to cancel when an interface is taken + * down. Normally the work elements would be executed by keventd but that + * can deadlock because of linkwatch. If our close method takes the rtnl + * lock and linkwatch is ahead of our work elements in keventd, linkwatch + * will block keventd as it needs the rtnl lock, and we'll deadlock waiting + * for our work to complete. Get our own work queue to solve this. + */ +struct workqueue_struct *cxgb3_wq; + +/** + * link_report - show link status and link speed/duplex + * @dev: the port whose settings are to be reported + * + * Shows the link status, speed, and duplex of a port. + */ +static void link_report(struct net_device *dev) +{ + if (!netif_carrier_ok(dev)) + netdev_info(dev, "link down\n"); + else { + const char *s = "10Mbps"; + const struct port_info *p = netdev_priv(dev); + + switch (p->link_config.speed) { + case SPEED_10000: + s = "10Gbps"; + break; + case SPEED_1000: + s = "1000Mbps"; + break; + case SPEED_100: + s = "100Mbps"; + break; + } + + netdev_info(dev, "link up, %s, %s-duplex\n", + s, p->link_config.duplex == DUPLEX_FULL + ? "full" : "half"); + } +} + +static void enable_tx_fifo_drain(struct adapter *adapter, + struct port_info *pi) +{ + t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0, + F_ENDROPPKT); + t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0); + t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN); + t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN); +} + +static void disable_tx_fifo_drain(struct adapter *adapter, + struct port_info *pi) +{ + t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, + F_ENDROPPKT, 0); +} + +void t3_os_link_fault(struct adapter *adap, int port_id, int state) +{ + struct net_device *dev = adap->port[port_id]; + struct port_info *pi = netdev_priv(dev); + + if (state == netif_carrier_ok(dev)) + return; + + if (state) { + struct cmac *mac = &pi->mac; + + netif_carrier_on(dev); + + disable_tx_fifo_drain(adap, pi); + + /* Clear local faults */ + t3_xgm_intr_disable(adap, pi->port_id); + t3_read_reg(adap, A_XGM_INT_STATUS + + pi->mac.offset); + t3_write_reg(adap, + A_XGM_INT_CAUSE + pi->mac.offset, + F_XGM_INT); + + t3_set_reg_field(adap, + A_XGM_INT_ENABLE + + pi->mac.offset, + F_XGM_INT, F_XGM_INT); + t3_xgm_intr_enable(adap, pi->port_id); + + t3_mac_enable(mac, MAC_DIRECTION_TX); + } else { + netif_carrier_off(dev); + + /* Flush TX FIFO */ + enable_tx_fifo_drain(adap, pi); + } + link_report(dev); +} + +/** + * t3_os_link_changed - handle link status changes + * @adapter: the adapter associated with the link change + * @port_id: the port index whose limk status has changed + * @link_stat: the new status of the link + * @speed: the new speed setting + * @duplex: the new duplex setting + * @pause: the new flow-control setting + * + * This is the OS-dependent handler for link status changes. The OS + * neutral handler takes care of most of the processing for these events, + * then calls this handler for any OS-specific processing. + */ +void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat, + int speed, int duplex, int pause) +{ + struct net_device *dev = adapter->port[port_id]; + struct port_info *pi = netdev_priv(dev); + struct cmac *mac = &pi->mac; + + /* Skip changes from disabled ports. */ + if (!netif_running(dev)) + return; + + if (link_stat != netif_carrier_ok(dev)) { + if (link_stat) { + disable_tx_fifo_drain(adapter, pi); + + t3_mac_enable(mac, MAC_DIRECTION_RX); + + /* Clear local faults */ + t3_xgm_intr_disable(adapter, pi->port_id); + t3_read_reg(adapter, A_XGM_INT_STATUS + + pi->mac.offset); + t3_write_reg(adapter, + A_XGM_INT_CAUSE + pi->mac.offset, + F_XGM_INT); + + t3_set_reg_field(adapter, + A_XGM_INT_ENABLE + pi->mac.offset, + F_XGM_INT, F_XGM_INT); + t3_xgm_intr_enable(adapter, pi->port_id); + + netif_carrier_on(dev); + } else { + netif_carrier_off(dev); + + t3_xgm_intr_disable(adapter, pi->port_id); + t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset); + t3_set_reg_field(adapter, + A_XGM_INT_ENABLE + pi->mac.offset, + F_XGM_INT, 0); + + if (is_10G(adapter)) + pi->phy.ops->power_down(&pi->phy, 1); + + t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset); + t3_mac_disable(mac, MAC_DIRECTION_RX); + t3_link_start(&pi->phy, mac, &pi->link_config); + + /* Flush TX FIFO */ + enable_tx_fifo_drain(adapter, pi); + } + + link_report(dev); + } +} + +/** + * t3_os_phymod_changed - handle PHY module changes + * @adap: the adapter associated with the link change + * @port_id: the port index whose limk status has changed + * + * This is the OS-dependent handler for PHY module changes. It is + * invoked when a PHY module is removed or inserted for any OS-specific + * processing. + */ +void t3_os_phymod_changed(struct adapter *adap, int port_id) +{ + static const char *mod_str[] = { + NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown" + }; + + const struct net_device *dev = adap->port[port_id]; + const struct port_info *pi = netdev_priv(dev); + + if (pi->phy.modtype == phy_modtype_none) + netdev_info(dev, "PHY module unplugged\n"); + else + netdev_info(dev, "%s PHY module inserted\n", + mod_str[pi->phy.modtype]); +} + +static void cxgb_set_rxmode(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + + t3_mac_set_rx_mode(&pi->mac, dev); +} + +/** + * link_start - enable a port + * @dev: the device to enable + * + * Performs the MAC and PHY actions needed to enable a port. + */ +static void link_start(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct cmac *mac = &pi->mac; + + t3_mac_reset(mac); + t3_mac_set_num_ucast(mac, MAX_MAC_IDX); + t3_mac_set_mtu(mac, dev->mtu); + t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr); + t3_mac_set_address(mac, SAN_MAC_IDX, pi->iscsic.mac_addr); + t3_mac_set_rx_mode(mac, dev); + t3_link_start(&pi->phy, mac, &pi->link_config); + t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); +} + +static inline void cxgb_disable_msi(struct adapter *adapter) +{ + if (adapter->flags & USING_MSIX) { + pci_disable_msix(adapter->pdev); + adapter->flags &= ~USING_MSIX; + } else if (adapter->flags & USING_MSI) { + pci_disable_msi(adapter->pdev); + adapter->flags &= ~USING_MSI; + } +} + +/* + * Interrupt handler for asynchronous events used with MSI-X. + */ +static irqreturn_t t3_async_intr_handler(int irq, void *cookie) +{ + t3_slow_intr_handler(cookie); + return IRQ_HANDLED; +} + +/* + * Name the MSI-X interrupts. + */ +static void name_msix_vecs(struct adapter *adap) +{ + int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1; + + snprintf(adap->msix_info[0].desc, n, "%s", adap->name); + adap->msix_info[0].desc[n] = 0; + + for_each_port(adap, j) { + struct net_device *d = adap->port[j]; + const struct port_info *pi = netdev_priv(d); + + for (i = 0; i < pi->nqsets; i++, msi_idx++) { + snprintf(adap->msix_info[msi_idx].desc, n, + "%s-%d", d->name, pi->first_qset + i); + adap->msix_info[msi_idx].desc[n] = 0; + } + } +} + +static int request_msix_data_irqs(struct adapter *adap) +{ + int i, j, err, qidx = 0; + + for_each_port(adap, i) { + int nqsets = adap2pinfo(adap, i)->nqsets; + + for (j = 0; j < nqsets; ++j) { + err = request_irq(adap->msix_info[qidx + 1].vec, + t3_intr_handler(adap, + adap->sge.qs[qidx]. + rspq.polling), 0, + adap->msix_info[qidx + 1].desc, + &adap->sge.qs[qidx]); + if (err) { + while (--qidx >= 0) + free_irq(adap->msix_info[qidx + 1].vec, + &adap->sge.qs[qidx]); + return err; + } + qidx++; + } + } + return 0; +} + +static void free_irq_resources(struct adapter *adapter) +{ + if (adapter->flags & USING_MSIX) { + int i, n = 0; + + free_irq(adapter->msix_info[0].vec, adapter); + for_each_port(adapter, i) + n += adap2pinfo(adapter, i)->nqsets; + + for (i = 0; i < n; ++i) + free_irq(adapter->msix_info[i + 1].vec, + &adapter->sge.qs[i]); + } else + free_irq(adapter->pdev->irq, adapter); +} + +static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt, + unsigned long n) +{ + int attempts = 10; + + while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) { + if (!--attempts) + return -ETIMEDOUT; + msleep(10); + } + return 0; +} + +static int init_tp_parity(struct adapter *adap) +{ + int i; + struct sk_buff *skb; + struct cpl_set_tcb_field *greq; + unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts; + + t3_tp_set_offload_mode(adap, 1); + + for (i = 0; i < 16; i++) { + struct cpl_smt_write_req *req; + + skb = alloc_skb(sizeof(*req), GFP_KERNEL); + if (!skb) + skb = adap->nofail_skb; + if (!skb) + goto alloc_skb_fail; + + req = __skb_put_zero(skb, sizeof(*req)); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); + req->mtu_idx = NMTUS - 1; + req->iff = i; + t3_mgmt_tx(adap, skb); + if (skb == adap->nofail_skb) { + await_mgmt_replies(adap, cnt, i + 1); + adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL); + if (!adap->nofail_skb) + goto alloc_skb_fail; + } + } + + for (i = 0; i < 2048; i++) { + struct cpl_l2t_write_req *req; + + skb = alloc_skb(sizeof(*req), GFP_KERNEL); + if (!skb) + skb = adap->nofail_skb; + if (!skb) + goto alloc_skb_fail; + + req = __skb_put_zero(skb, sizeof(*req)); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); + req->params = htonl(V_L2T_W_IDX(i)); + t3_mgmt_tx(adap, skb); + if (skb == adap->nofail_skb) { + await_mgmt_replies(adap, cnt, 16 + i + 1); + adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL); + if (!adap->nofail_skb) + goto alloc_skb_fail; + } + } + + for (i = 0; i < 2048; i++) { + struct cpl_rte_write_req *req; + + skb = alloc_skb(sizeof(*req), GFP_KERNEL); + if (!skb) + skb = adap->nofail_skb; + if (!skb) + goto alloc_skb_fail; + + req = __skb_put_zero(skb, sizeof(*req)); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); + req->l2t_idx = htonl(V_L2T_W_IDX(i)); + t3_mgmt_tx(adap, skb); + if (skb == adap->nofail_skb) { + await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1); + adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL); + if (!adap->nofail_skb) + goto alloc_skb_fail; + } + } + + skb = alloc_skb(sizeof(*greq), GFP_KERNEL); + if (!skb) + skb = adap->nofail_skb; + if (!skb) + goto alloc_skb_fail; + + greq = __skb_put_zero(skb, sizeof(*greq)); + greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); + greq->mask = cpu_to_be64(1); + t3_mgmt_tx(adap, skb); + + i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); + if (skb == adap->nofail_skb) { + i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1); + adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL); + } + + t3_tp_set_offload_mode(adap, 0); + return i; + +alloc_skb_fail: + t3_tp_set_offload_mode(adap, 0); + return -ENOMEM; +} + +/** + * setup_rss - configure RSS + * @adap: the adapter + * + * Sets up RSS to distribute packets to multiple receive queues. We + * configure the RSS CPU lookup table to distribute to the number of HW + * receive queues, and the response queue lookup table to narrow that + * down to the response queues actually configured for each port. + * We always configure the RSS mapping for two ports since the mapping + * table has plenty of entries. + */ +static void setup_rss(struct adapter *adap) +{ + int i; + unsigned int nq0 = adap2pinfo(adap, 0)->nqsets; + unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1; + u8 cpus[SGE_QSETS + 1]; + u16 rspq_map[RSS_TABLE_SIZE + 1]; + + for (i = 0; i < SGE_QSETS; ++i) + cpus[i] = i; + cpus[SGE_QSETS] = 0xff; /* terminator */ + + for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) { + rspq_map[i] = i % nq0; + rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0; + } + rspq_map[RSS_TABLE_SIZE] = 0xffff; /* terminator */ + + t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN | + F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | + V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map); +} + +static void ring_dbs(struct adapter *adap) +{ + int i, j; + + for (i = 0; i < SGE_QSETS; i++) { + struct sge_qset *qs = &adap->sge.qs[i]; + + if (qs->adap) + for (j = 0; j < SGE_TXQ_PER_SET; j++) + t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(qs->txq[j].cntxt_id)); + } +} + +static void init_napi(struct adapter *adap) +{ + int i; + + for (i = 0; i < SGE_QSETS; i++) { + struct sge_qset *qs = &adap->sge.qs[i]; + + if (qs->adap) + netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll); + } + + /* + * netif_napi_add() can be called only once per napi_struct because it + * adds each new napi_struct to a list. Be careful not to call it a + * second time, e.g., during EEH recovery, by making a note of it. + */ + adap->flags |= NAPI_INIT; +} + +/* + * Wait until all NAPI handlers are descheduled. This includes the handlers of + * both netdevices representing interfaces and the dummy ones for the extra + * queues. + */ +static void quiesce_rx(struct adapter *adap) +{ + int i; + + for (i = 0; i < SGE_QSETS; i++) + if (adap->sge.qs[i].adap) + napi_disable(&adap->sge.qs[i].napi); +} + +static void enable_all_napi(struct adapter *adap) +{ + int i; + for (i = 0; i < SGE_QSETS; i++) + if (adap->sge.qs[i].adap) + napi_enable(&adap->sge.qs[i].napi); +} + +/** + * setup_sge_qsets - configure SGE Tx/Rx/response queues + * @adap: the adapter + * + * Determines how many sets of SGE queues to use and initializes them. + * We support multiple queue sets per port if we have MSI-X, otherwise + * just one queue set per port. + */ +static int setup_sge_qsets(struct adapter *adap) +{ + int i, j, err, irq_idx = 0, qset_idx = 0; + unsigned int ntxq = SGE_TXQ_PER_SET; + + if (adap->params.rev > 0 && !(adap->flags & USING_MSI)) + irq_idx = -1; + + for_each_port(adap, i) { + struct net_device *dev = adap->port[i]; + struct port_info *pi = netdev_priv(dev); + + pi->qs = &adap->sge.qs[pi->first_qset]; + for (j = 0; j < pi->nqsets; ++j, ++qset_idx) { + err = t3_sge_alloc_qset(adap, qset_idx, 1, + (adap->flags & USING_MSIX) ? qset_idx + 1 : + irq_idx, + &adap->params.sge.qset[qset_idx], ntxq, dev, + netdev_get_tx_queue(dev, j)); + if (err) { + t3_free_sge_resources(adap); + return err; + } + } + } + + return 0; +} + +static ssize_t attr_show(struct device *d, char *buf, + ssize_t(*format) (struct net_device *, char *)) +{ + ssize_t len; + + /* Synchronize with ioctls that may shut down the device */ + rtnl_lock(); + len = (*format) (to_net_dev(d), buf); + rtnl_unlock(); + return len; +} + +static ssize_t attr_store(struct device *d, + const char *buf, size_t len, + ssize_t(*set) (struct net_device *, unsigned int), + unsigned int min_val, unsigned int max_val) +{ + ssize_t ret; + unsigned int val; + + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + + ret = kstrtouint(buf, 0, &val); + if (ret) + return ret; + if (val < min_val || val > max_val) + return -EINVAL; + + rtnl_lock(); + ret = (*set) (to_net_dev(d), val); + if (!ret) + ret = len; + rtnl_unlock(); + return ret; +} + +#define CXGB3_SHOW(name, val_expr) \ +static ssize_t format_##name(struct net_device *dev, char *buf) \ +{ \ + struct port_info *pi = netdev_priv(dev); \ + struct adapter *adap = pi->adapter; \ + return sprintf(buf, "%u\n", val_expr); \ +} \ +static ssize_t show_##name(struct device *d, struct device_attribute *attr, \ + char *buf) \ +{ \ + return attr_show(d, buf, format_##name); \ +} + +static ssize_t set_nfilters(struct net_device *dev, unsigned int val) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0; + + if (adap->flags & FULL_INIT_DONE) + return -EBUSY; + if (val && adap->params.rev == 0) + return -EINVAL; + if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers - + min_tids) + return -EINVAL; + adap->params.mc5.nfilters = val; + return 0; +} + +static ssize_t store_nfilters(struct device *d, struct device_attribute *attr, + const char *buf, size_t len) +{ + return attr_store(d, buf, len, set_nfilters, 0, ~0); +} + +static ssize_t set_nservers(struct net_device *dev, unsigned int val) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + + if (adap->flags & FULL_INIT_DONE) + return -EBUSY; + if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters - + MC5_MIN_TIDS) + return -EINVAL; + adap->params.mc5.nservers = val; + return 0; +} + +static ssize_t store_nservers(struct device *d, struct device_attribute *attr, + const char *buf, size_t len) +{ + return attr_store(d, buf, len, set_nservers, 0, ~0); +} + +#define CXGB3_ATTR_R(name, val_expr) \ +CXGB3_SHOW(name, val_expr) \ +static DEVICE_ATTR(name, 0444, show_##name, NULL) + +#define CXGB3_ATTR_RW(name, val_expr, store_method) \ +CXGB3_SHOW(name, val_expr) \ +static DEVICE_ATTR(name, 0644, show_##name, store_method) + +CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5)); +CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters); +CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers); + +static struct attribute *cxgb3_attrs[] = { + &dev_attr_cam_size.attr, + &dev_attr_nfilters.attr, + &dev_attr_nservers.attr, + NULL +}; + +static const struct attribute_group cxgb3_attr_group = { + .attrs = cxgb3_attrs, +}; + +static ssize_t tm_attr_show(struct device *d, + char *buf, int sched) +{ + struct port_info *pi = netdev_priv(to_net_dev(d)); + struct adapter *adap = pi->adapter; + unsigned int v, addr, bpt, cpt; + ssize_t len; + + addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2; + rtnl_lock(); + t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr); + v = t3_read_reg(adap, A_TP_TM_PIO_DATA); + if (sched & 1) + v >>= 16; + bpt = (v >> 8) & 0xff; + cpt = v & 0xff; + if (!cpt) + len = sprintf(buf, "disabled\n"); + else { + v = (adap->params.vpd.cclk * 1000) / cpt; + len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125); + } + rtnl_unlock(); + return len; +} + +static ssize_t tm_attr_store(struct device *d, + const char *buf, size_t len, int sched) +{ + struct port_info *pi = netdev_priv(to_net_dev(d)); + struct adapter *adap = pi->adapter; + unsigned int val; + ssize_t ret; + + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + + ret = kstrtouint(buf, 0, &val); + if (ret) + return ret; + if (val > 10000000) + return -EINVAL; + + rtnl_lock(); + ret = t3_config_sched(adap, val, sched); + if (!ret) + ret = len; + rtnl_unlock(); + return ret; +} + +#define TM_ATTR(name, sched) \ +static ssize_t show_##name(struct device *d, struct device_attribute *attr, \ + char *buf) \ +{ \ + return tm_attr_show(d, buf, sched); \ +} \ +static ssize_t store_##name(struct device *d, struct device_attribute *attr, \ + const char *buf, size_t len) \ +{ \ + return tm_attr_store(d, buf, len, sched); \ +} \ +static DEVICE_ATTR(name, 0644, show_##name, store_##name) + +TM_ATTR(sched0, 0); +TM_ATTR(sched1, 1); +TM_ATTR(sched2, 2); +TM_ATTR(sched3, 3); +TM_ATTR(sched4, 4); +TM_ATTR(sched5, 5); +TM_ATTR(sched6, 6); +TM_ATTR(sched7, 7); + +static struct attribute *offload_attrs[] = { + &dev_attr_sched0.attr, + &dev_attr_sched1.attr, + &dev_attr_sched2.attr, + &dev_attr_sched3.attr, + &dev_attr_sched4.attr, + &dev_attr_sched5.attr, + &dev_attr_sched6.attr, + &dev_attr_sched7.attr, + NULL +}; + +static const struct attribute_group offload_attr_group = { + .attrs = offload_attrs, +}; + +/* + * Sends an sk_buff to an offload queue driver + * after dealing with any active network taps. + */ +static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb) +{ + int ret; + + local_bh_disable(); + ret = t3_offload_tx(tdev, skb); + local_bh_enable(); + return ret; +} + +static int write_smt_entry(struct adapter *adapter, int idx) +{ + struct cpl_smt_write_req *req; + struct port_info *pi = netdev_priv(adapter->port[idx]); + struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL); + + if (!skb) + return -ENOMEM; + + req = __skb_put(skb, sizeof(*req)); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); + req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ + req->iff = idx; + memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN); + memcpy(req->src_mac1, pi->iscsic.mac_addr, ETH_ALEN); + skb->priority = 1; + offload_tx(&adapter->tdev, skb); + return 0; +} + +static int init_smt(struct adapter *adapter) +{ + int i; + + for_each_port(adapter, i) + write_smt_entry(adapter, i); + return 0; +} + +static void init_port_mtus(struct adapter *adapter) +{ + unsigned int mtus = adapter->port[0]->mtu; + + if (adapter->port[1]) + mtus |= adapter->port[1]->mtu << 16; + t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus); +} + +static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo, + int hi, int port) +{ + struct sk_buff *skb; + struct mngt_pktsched_wr *req; + int ret; + + skb = alloc_skb(sizeof(*req), GFP_KERNEL); + if (!skb) + skb = adap->nofail_skb; + if (!skb) + return -ENOMEM; + + req = skb_put(skb, sizeof(*req)); + req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); + req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; + req->sched = sched; + req->idx = qidx; + req->min = lo; + req->max = hi; + req->binding = port; + ret = t3_mgmt_tx(adap, skb); + if (skb == adap->nofail_skb) { + adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field), + GFP_KERNEL); + if (!adap->nofail_skb) + ret = -ENOMEM; + } + + return ret; +} + +static int bind_qsets(struct adapter *adap) +{ + int i, j, err = 0; + + for_each_port(adap, i) { + const struct port_info *pi = adap2pinfo(adap, i); + + for (j = 0; j < pi->nqsets; ++j) { + int ret = send_pktsched_cmd(adap, 1, + pi->first_qset + j, -1, + -1, i); + if (ret) + err = ret; + } + } + + return err; +} + +#define FW_VERSION __stringify(FW_VERSION_MAJOR) "." \ + __stringify(FW_VERSION_MINOR) "." __stringify(FW_VERSION_MICRO) +#define FW_FNAME "cxgb3/t3fw-" FW_VERSION ".bin" +#define TPSRAM_VERSION __stringify(TP_VERSION_MAJOR) "." \ + __stringify(TP_VERSION_MINOR) "." __stringify(TP_VERSION_MICRO) +#define TPSRAM_NAME "cxgb3/t3%c_psram-" TPSRAM_VERSION ".bin" +#define AEL2005_OPT_EDC_NAME "cxgb3/ael2005_opt_edc.bin" +#define AEL2005_TWX_EDC_NAME "cxgb3/ael2005_twx_edc.bin" +#define AEL2020_TWX_EDC_NAME "cxgb3/ael2020_twx_edc.bin" +MODULE_FIRMWARE(FW_FNAME); +MODULE_FIRMWARE("cxgb3/t3b_psram-" TPSRAM_VERSION ".bin"); +MODULE_FIRMWARE("cxgb3/t3c_psram-" TPSRAM_VERSION ".bin"); +MODULE_FIRMWARE(AEL2005_OPT_EDC_NAME); +MODULE_FIRMWARE(AEL2005_TWX_EDC_NAME); +MODULE_FIRMWARE(AEL2020_TWX_EDC_NAME); + +static inline const char *get_edc_fw_name(int edc_idx) +{ + const char *fw_name = NULL; + + switch (edc_idx) { + case EDC_OPT_AEL2005: + fw_name = AEL2005_OPT_EDC_NAME; + break; + case EDC_TWX_AEL2005: + fw_name = AEL2005_TWX_EDC_NAME; + break; + case EDC_TWX_AEL2020: + fw_name = AEL2020_TWX_EDC_NAME; + break; + } + return fw_name; +} + +int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size) +{ + struct adapter *adapter = phy->adapter; + const struct firmware *fw; + const char *fw_name; + u32 csum; + const __be32 *p; + u16 *cache = phy->phy_cache; + int i, ret = -EINVAL; + + fw_name = get_edc_fw_name(edc_idx); + if (fw_name) + ret = request_firmware(&fw, fw_name, &adapter->pdev->dev); + if (ret < 0) { + dev_err(&adapter->pdev->dev, + "could not upgrade firmware: unable to load %s\n", + fw_name); + return ret; + } + + /* check size, take checksum in account */ + if (fw->size > size + 4) { + CH_ERR(adapter, "firmware image too large %u, expected %d\n", + (unsigned int)fw->size, size + 4); + ret = -EINVAL; + } + + /* compute checksum */ + p = (const __be32 *)fw->data; + for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++) + csum += ntohl(p[i]); + + if (csum != 0xffffffff) { + CH_ERR(adapter, "corrupted firmware image, checksum %u\n", + csum); + ret = -EINVAL; + } + + for (i = 0; i < size / 4 ; i++) { + *cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16; + *cache++ = be32_to_cpu(p[i]) & 0xffff; + } + + release_firmware(fw); + + return ret; +} + +static int upgrade_fw(struct adapter *adap) +{ + int ret; + const struct firmware *fw; + struct device *dev = &adap->pdev->dev; + + ret = request_firmware(&fw, FW_FNAME, dev); + if (ret < 0) { + dev_err(dev, "could not upgrade firmware: unable to load %s\n", + FW_FNAME); + return ret; + } + ret = t3_load_fw(adap, fw->data, fw->size); + release_firmware(fw); + + if (ret == 0) + dev_info(dev, "successful upgrade to firmware %d.%d.%d\n", + FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); + else + dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n", + FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO); + + return ret; +} + +static inline char t3rev2char(struct adapter *adapter) +{ + char rev = 0; + + switch(adapter->params.rev) { + case T3_REV_B: + case T3_REV_B2: + rev = 'b'; + break; + case T3_REV_C: + rev = 'c'; + break; + } + return rev; +} + +static int update_tpsram(struct adapter *adap) +{ + const struct firmware *tpsram; + char buf[64]; + struct device *dev = &adap->pdev->dev; + int ret; + char rev; + + rev = t3rev2char(adap); + if (!rev) + return 0; + + snprintf(buf, sizeof(buf), TPSRAM_NAME, rev); + + ret = request_firmware(&tpsram, buf, dev); + if (ret < 0) { + dev_err(dev, "could not load TP SRAM: unable to load %s\n", + buf); + return ret; + } + + ret = t3_check_tpsram(adap, tpsram->data, tpsram->size); + if (ret) + goto release_tpsram; + + ret = t3_set_proto_sram(adap, tpsram->data); + if (ret == 0) + dev_info(dev, + "successful update of protocol engine " + "to %d.%d.%d\n", + TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); + else + dev_err(dev, "failed to update of protocol engine %d.%d.%d\n", + TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO); + if (ret) + dev_err(dev, "loading protocol SRAM failed\n"); + +release_tpsram: + release_firmware(tpsram); + + return ret; +} + +/** + * t3_synchronize_rx - wait for current Rx processing on a port to complete + * @adap: the adapter + * @p: the port + * + * Ensures that current Rx processing on any of the queues associated with + * the given port completes before returning. We do this by acquiring and + * releasing the locks of the response queues associated with the port. + */ +static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p) +{ + int i; + + for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { + struct sge_rspq *q = &adap->sge.qs[i].rspq; + + spin_lock_irq(&q->lock); + spin_unlock_irq(&q->lock); + } +} + +static void cxgb_vlan_mode(struct net_device *dev, netdev_features_t features) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + if (adapter->params.rev > 0) { + t3_set_vlan_accel(adapter, 1 << pi->port_id, + features & NETIF_F_HW_VLAN_CTAG_RX); + } else { + /* single control for all ports */ + unsigned int i, have_vlans = features & NETIF_F_HW_VLAN_CTAG_RX; + + for_each_port(adapter, i) + have_vlans |= + adapter->port[i]->features & + NETIF_F_HW_VLAN_CTAG_RX; + + t3_set_vlan_accel(adapter, 1, have_vlans); + } + t3_synchronize_rx(adapter, pi); +} + +/** + * cxgb_up - enable the adapter + * @adap: adapter being enabled + * + * Called when the first port is enabled, this function performs the + * actions necessary to make an adapter operational, such as completing + * the initialization of HW modules, and enabling interrupts. + * + * Must be called with the rtnl lock held. + */ +static int cxgb_up(struct adapter *adap) +{ + int i, err; + + if (!(adap->flags & FULL_INIT_DONE)) { + err = t3_check_fw_version(adap); + if (err == -EINVAL) { + err = upgrade_fw(adap); + CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n", + FW_VERSION_MAJOR, FW_VERSION_MINOR, + FW_VERSION_MICRO, err ? "failed" : "succeeded"); + } + + err = t3_check_tpsram_version(adap); + if (err == -EINVAL) { + err = update_tpsram(adap); + CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n", + TP_VERSION_MAJOR, TP_VERSION_MINOR, + TP_VERSION_MICRO, err ? "failed" : "succeeded"); + } + + /* + * Clear interrupts now to catch errors if t3_init_hw fails. + * We clear them again later as initialization may trigger + * conditions that can interrupt. + */ + t3_intr_clear(adap); + + err = t3_init_hw(adap, 0); + if (err) + goto out; + + t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT); + t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); + + err = setup_sge_qsets(adap); + if (err) + goto out; + + for_each_port(adap, i) + cxgb_vlan_mode(adap->port[i], adap->port[i]->features); + + setup_rss(adap); + if (!(adap->flags & NAPI_INIT)) + init_napi(adap); + + t3_start_sge_timers(adap); + adap->flags |= FULL_INIT_DONE; + } + + t3_intr_clear(adap); + + if (adap->flags & USING_MSIX) { + name_msix_vecs(adap); + err = request_irq(adap->msix_info[0].vec, + t3_async_intr_handler, 0, + adap->msix_info[0].desc, adap); + if (err) + goto irq_err; + + err = request_msix_data_irqs(adap); + if (err) { + free_irq(adap->msix_info[0].vec, adap); + goto irq_err; + } + } else { + err = request_irq(adap->pdev->irq, + t3_intr_handler(adap, adap->sge.qs[0].rspq.polling), + (adap->flags & USING_MSI) ? 0 : IRQF_SHARED, + adap->name, adap); + if (err) + goto irq_err; + } + + enable_all_napi(adap); + t3_sge_start(adap); + t3_intr_enable(adap); + + if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) && + is_offload(adap) && init_tp_parity(adap) == 0) + adap->flags |= TP_PARITY_INIT; + + if (adap->flags & TP_PARITY_INIT) { + t3_write_reg(adap, A_TP_INT_CAUSE, + F_CMCACHEPERR | F_ARPLUTPERR); + t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff); + } + + if (!(adap->flags & QUEUES_BOUND)) { + int ret = bind_qsets(adap); + + if (ret < 0) { + CH_ERR(adap, "failed to bind qsets, err %d\n", ret); + t3_intr_disable(adap); + quiesce_rx(adap); + free_irq_resources(adap); + err = ret; + goto out; + } + adap->flags |= QUEUES_BOUND; + } + +out: + return err; +irq_err: + CH_ERR(adap, "request_irq failed, err %d\n", err); + goto out; +} + +/* + * Release resources when all the ports and offloading have been stopped. + */ +static void cxgb_down(struct adapter *adapter, int on_wq) +{ + t3_sge_stop(adapter); + spin_lock_irq(&adapter->work_lock); /* sync with PHY intr task */ + t3_intr_disable(adapter); + spin_unlock_irq(&adapter->work_lock); + + free_irq_resources(adapter); + quiesce_rx(adapter); + t3_sge_stop(adapter); + if (!on_wq) + flush_workqueue(cxgb3_wq);/* wait for external IRQ handler */ +} + +static void schedule_chk_task(struct adapter *adap) +{ + unsigned int timeo; + + timeo = adap->params.linkpoll_period ? + (HZ * adap->params.linkpoll_period) / 10 : + adap->params.stats_update_period * HZ; + if (timeo) + queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo); +} + +static int offload_open(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + struct t3cdev *tdev = dev2t3cdev(dev); + int adap_up = adapter->open_device_map & PORT_MASK; + int err; + + if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) + return 0; + + if (!adap_up && (err = cxgb_up(adapter)) < 0) + goto out; + + t3_tp_set_offload_mode(adapter, 1); + tdev->lldev = adapter->port[0]; + err = cxgb3_offload_activate(adapter); + if (err) + goto out; + + init_port_mtus(adapter); + t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd, + adapter->params.b_wnd, + adapter->params.rev == 0 ? + adapter->port[0]->mtu : 0xffff); + init_smt(adapter); + + if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group)) + dev_dbg(&dev->dev, "cannot create sysfs group\n"); + + /* Call back all registered clients */ + cxgb3_add_clients(tdev); + +out: + /* restore them in case the offload module has changed them */ + if (err) { + t3_tp_set_offload_mode(adapter, 0); + clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map); + cxgb3_set_dummy_ops(tdev); + } + return err; +} + +static int offload_close(struct t3cdev *tdev) +{ + struct adapter *adapter = tdev2adap(tdev); + struct t3c_data *td = T3C_DATA(tdev); + + if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) + return 0; + + /* Call back all registered clients */ + cxgb3_remove_clients(tdev); + + sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group); + + /* Flush work scheduled while releasing TIDs */ + flush_work(&td->tid_release_task); + + tdev->lldev = NULL; + cxgb3_set_dummy_ops(tdev); + t3_tp_set_offload_mode(adapter, 0); + clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map); + + if (!adapter->open_device_map) + cxgb_down(adapter, 0); + + cxgb3_offload_deactivate(adapter); + return 0; +} + +static int cxgb_open(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + int other_ports = adapter->open_device_map & PORT_MASK; + int err; + + if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0) + return err; + + set_bit(pi->port_id, &adapter->open_device_map); + if (is_offload(adapter) && !ofld_disable) { + err = offload_open(dev); + if (err) + pr_warn("Could not initialize offload capabilities\n"); + } + + netif_set_real_num_tx_queues(dev, pi->nqsets); + err = netif_set_real_num_rx_queues(dev, pi->nqsets); + if (err) + return err; + link_start(dev); + t3_port_intr_enable(adapter, pi->port_id); + netif_tx_start_all_queues(dev); + if (!other_ports) + schedule_chk_task(adapter); + + cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id); + return 0; +} + +static int __cxgb_close(struct net_device *dev, int on_wq) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + + if (!adapter->open_device_map) + return 0; + + /* Stop link fault interrupts */ + t3_xgm_intr_disable(adapter, pi->port_id); + t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset); + + t3_port_intr_disable(adapter, pi->port_id); + netif_tx_stop_all_queues(dev); + pi->phy.ops->power_down(&pi->phy, 1); + netif_carrier_off(dev); + t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX); + + spin_lock_irq(&adapter->work_lock); /* sync with update task */ + clear_bit(pi->port_id, &adapter->open_device_map); + spin_unlock_irq(&adapter->work_lock); + + if (!(adapter->open_device_map & PORT_MASK)) + cancel_delayed_work_sync(&adapter->adap_check_task); + + if (!adapter->open_device_map) + cxgb_down(adapter, on_wq); + + cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id); + return 0; +} + +static int cxgb_close(struct net_device *dev) +{ + return __cxgb_close(dev, 0); +} + +static struct net_device_stats *cxgb_get_stats(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + struct net_device_stats *ns = &dev->stats; + const struct mac_stats *pstats; + + spin_lock(&adapter->stats_lock); + pstats = t3_mac_update_stats(&pi->mac); + spin_unlock(&adapter->stats_lock); + + ns->tx_bytes = pstats->tx_octets; + ns->tx_packets = pstats->tx_frames; + ns->rx_bytes = pstats->rx_octets; + ns->rx_packets = pstats->rx_frames; + ns->multicast = pstats->rx_mcast_frames; + + ns->tx_errors = pstats->tx_underrun; + ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs + + pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short + + pstats->rx_fifo_ovfl; + + /* detailed rx_errors */ + ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long; + ns->rx_over_errors = 0; + ns->rx_crc_errors = pstats->rx_fcs_errs; + ns->rx_frame_errors = pstats->rx_symbol_errs; + ns->rx_fifo_errors = pstats->rx_fifo_ovfl; + ns->rx_missed_errors = pstats->rx_cong_drops; + + /* detailed tx_errors */ + ns->tx_aborted_errors = 0; + ns->tx_carrier_errors = 0; + ns->tx_fifo_errors = pstats->tx_underrun; + ns->tx_heartbeat_errors = 0; + ns->tx_window_errors = 0; + return ns; +} + +static u32 get_msglevel(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + return adapter->msg_enable; +} + +static void set_msglevel(struct net_device *dev, u32 val) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + adapter->msg_enable = val; +} + +static const char stats_strings[][ETH_GSTRING_LEN] = { + "TxOctetsOK ", + "TxFramesOK ", + "TxMulticastFramesOK", + "TxBroadcastFramesOK", + "TxPauseFrames ", + "TxUnderrun ", + "TxExtUnderrun ", + + "TxFrames64 ", + "TxFrames65To127 ", + "TxFrames128To255 ", + "TxFrames256To511 ", + "TxFrames512To1023 ", + "TxFrames1024To1518 ", + "TxFrames1519ToMax ", + + "RxOctetsOK ", + "RxFramesOK ", + "RxMulticastFramesOK", + "RxBroadcastFramesOK", + "RxPauseFrames ", + "RxFCSErrors ", + "RxSymbolErrors ", + "RxShortErrors ", + "RxJabberErrors ", + "RxLengthErrors ", + "RxFIFOoverflow ", + + "RxFrames64 ", + "RxFrames65To127 ", + "RxFrames128To255 ", + "RxFrames256To511 ", + "RxFrames512To1023 ", + "RxFrames1024To1518 ", + "RxFrames1519ToMax ", + + "PhyFIFOErrors ", + "TSO ", + "VLANextractions ", + "VLANinsertions ", + "TxCsumOffload ", + "RxCsumGood ", + "LroAggregated ", + "LroFlushed ", + "LroNoDesc ", + "RxDrops ", + + "CheckTXEnToggled ", + "CheckResets ", + + "LinkFaults ", +}; + +static int get_sset_count(struct net_device *dev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return ARRAY_SIZE(stats_strings); + default: + return -EOPNOTSUPP; + } +} + +#define T3_REGMAP_SIZE (3 * 1024) + +static int get_regs_len(struct net_device *dev) +{ + return T3_REGMAP_SIZE; +} + +static int get_eeprom_len(struct net_device *dev) +{ + return EEPROMSIZE; +} + +static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + u32 fw_vers = 0; + u32 tp_vers = 0; + + spin_lock(&adapter->stats_lock); + t3_get_fw_version(adapter, &fw_vers); + t3_get_tp_version(adapter, &tp_vers); + spin_unlock(&adapter->stats_lock); + + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(adapter->pdev), + sizeof(info->bus_info)); + if (fw_vers) + snprintf(info->fw_version, sizeof(info->fw_version), + "%s %u.%u.%u TP %u.%u.%u", + G_FW_VERSION_TYPE(fw_vers) ? "T" : "N", + G_FW_VERSION_MAJOR(fw_vers), + G_FW_VERSION_MINOR(fw_vers), + G_FW_VERSION_MICRO(fw_vers), + G_TP_VERSION_MAJOR(tp_vers), + G_TP_VERSION_MINOR(tp_vers), + G_TP_VERSION_MICRO(tp_vers)); +} + +static void get_strings(struct net_device *dev, u32 stringset, u8 * data) +{ + if (stringset == ETH_SS_STATS) + memcpy(data, stats_strings, sizeof(stats_strings)); +} + +static unsigned long collect_sge_port_stats(struct adapter *adapter, + struct port_info *p, int idx) +{ + int i; + unsigned long tot = 0; + + for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i) + tot += adapter->sge.qs[i].port_stats[idx]; + return tot; +} + +static void get_stats(struct net_device *dev, struct ethtool_stats *stats, + u64 *data) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + const struct mac_stats *s; + + spin_lock(&adapter->stats_lock); + s = t3_mac_update_stats(&pi->mac); + spin_unlock(&adapter->stats_lock); + + *data++ = s->tx_octets; + *data++ = s->tx_frames; + *data++ = s->tx_mcast_frames; + *data++ = s->tx_bcast_frames; + *data++ = s->tx_pause; + *data++ = s->tx_underrun; + *data++ = s->tx_fifo_urun; + + *data++ = s->tx_frames_64; + *data++ = s->tx_frames_65_127; + *data++ = s->tx_frames_128_255; + *data++ = s->tx_frames_256_511; + *data++ = s->tx_frames_512_1023; + *data++ = s->tx_frames_1024_1518; + *data++ = s->tx_frames_1519_max; + + *data++ = s->rx_octets; + *data++ = s->rx_frames; + *data++ = s->rx_mcast_frames; + *data++ = s->rx_bcast_frames; + *data++ = s->rx_pause; + *data++ = s->rx_fcs_errs; + *data++ = s->rx_symbol_errs; + *data++ = s->rx_short; + *data++ = s->rx_jabber; + *data++ = s->rx_too_long; + *data++ = s->rx_fifo_ovfl; + + *data++ = s->rx_frames_64; + *data++ = s->rx_frames_65_127; + *data++ = s->rx_frames_128_255; + *data++ = s->rx_frames_256_511; + *data++ = s->rx_frames_512_1023; + *data++ = s->rx_frames_1024_1518; + *data++ = s->rx_frames_1519_max; + + *data++ = pi->phy.fifo_errors; + + *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO); + *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX); + *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS); + *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM); + *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD); + *data++ = 0; + *data++ = 0; + *data++ = 0; + *data++ = s->rx_cong_drops; + + *data++ = s->num_toggled; + *data++ = s->num_resets; + + *data++ = s->link_faults; +} + +static inline void reg_block_dump(struct adapter *ap, void *buf, + unsigned int start, unsigned int end) +{ + u32 *p = buf + start; + + for (; start <= end; start += sizeof(u32)) + *p++ = t3_read_reg(ap, start); +} + +static void get_regs(struct net_device *dev, struct ethtool_regs *regs, + void *buf) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *ap = pi->adapter; + + /* + * Version scheme: + * bits 0..9: chip version + * bits 10..15: chip revision + * bit 31: set for PCIe cards + */ + regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31); + + /* + * We skip the MAC statistics registers because they are clear-on-read. + * Also reading multi-register stats would need to synchronize with the + * periodic mac stats accumulation. Hard to justify the complexity. + */ + memset(buf, 0, T3_REGMAP_SIZE); + reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN); + reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT); + reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE); + reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA); + reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3); + reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0, + XGM_REG(A_XGM_SERDES_STAT3, 1)); + reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1), + XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1)); +} + +static int restart_autoneg(struct net_device *dev) +{ + struct port_info *p = netdev_priv(dev); + + if (!netif_running(dev)) + return -EAGAIN; + if (p->link_config.autoneg != AUTONEG_ENABLE) + return -EINVAL; + p->phy.ops->autoneg_restart(&p->phy); + return 0; +} + +static int set_phys_id(struct net_device *dev, + enum ethtool_phys_id_state state) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + switch (state) { + case ETHTOOL_ID_ACTIVE: + return 1; /* cycle on/off once per second */ + + case ETHTOOL_ID_OFF: + t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, 0); + break; + + case ETHTOOL_ID_ON: + case ETHTOOL_ID_INACTIVE: + t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, + F_GPIO0_OUT_VAL); + } + + return 0; +} + +static int get_link_ksettings(struct net_device *dev, + struct ethtool_link_ksettings *cmd) +{ + struct port_info *p = netdev_priv(dev); + u32 supported; + + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, + p->link_config.supported); + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, + p->link_config.advertising); + + if (netif_carrier_ok(dev)) { + cmd->base.speed = p->link_config.speed; + cmd->base.duplex = p->link_config.duplex; + } else { + cmd->base.speed = SPEED_UNKNOWN; + cmd->base.duplex = DUPLEX_UNKNOWN; + } + + ethtool_convert_link_mode_to_legacy_u32(&supported, + cmd->link_modes.supported); + + cmd->base.port = (supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE; + cmd->base.phy_address = p->phy.mdio.prtad; + cmd->base.autoneg = p->link_config.autoneg; + return 0; +} + +static int speed_duplex_to_caps(int speed, int duplex) +{ + int cap = 0; + + switch (speed) { + case SPEED_10: + if (duplex == DUPLEX_FULL) + cap = SUPPORTED_10baseT_Full; + else + cap = SUPPORTED_10baseT_Half; + break; + case SPEED_100: + if (duplex == DUPLEX_FULL) + cap = SUPPORTED_100baseT_Full; + else + cap = SUPPORTED_100baseT_Half; + break; + case SPEED_1000: + if (duplex == DUPLEX_FULL) + cap = SUPPORTED_1000baseT_Full; + else + cap = SUPPORTED_1000baseT_Half; + break; + case SPEED_10000: + if (duplex == DUPLEX_FULL) + cap = SUPPORTED_10000baseT_Full; + } + return cap; +} + +#define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \ + ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \ + ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \ + ADVERTISED_10000baseT_Full) + +static int set_link_ksettings(struct net_device *dev, + const struct ethtool_link_ksettings *cmd) +{ + struct port_info *p = netdev_priv(dev); + struct link_config *lc = &p->link_config; + u32 advertising; + + ethtool_convert_link_mode_to_legacy_u32(&advertising, + cmd->link_modes.advertising); + + if (!(lc->supported & SUPPORTED_Autoneg)) { + /* + * PHY offers a single speed/duplex. See if that's what's + * being requested. + */ + if (cmd->base.autoneg == AUTONEG_DISABLE) { + u32 speed = cmd->base.speed; + int cap = speed_duplex_to_caps(speed, cmd->base.duplex); + if (lc->supported & cap) + return 0; + } + return -EINVAL; + } + + if (cmd->base.autoneg == AUTONEG_DISABLE) { + u32 speed = cmd->base.speed; + int cap = speed_duplex_to_caps(speed, cmd->base.duplex); + + if (!(lc->supported & cap) || (speed == SPEED_1000)) + return -EINVAL; + lc->requested_speed = speed; + lc->requested_duplex = cmd->base.duplex; + lc->advertising = 0; + } else { + advertising &= ADVERTISED_MASK; + advertising &= lc->supported; + if (!advertising) + return -EINVAL; + lc->requested_speed = SPEED_INVALID; + lc->requested_duplex = DUPLEX_INVALID; + lc->advertising = advertising | ADVERTISED_Autoneg; + } + lc->autoneg = cmd->base.autoneg; + if (netif_running(dev)) + t3_link_start(&p->phy, &p->mac, lc); + return 0; +} + +static void get_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *epause) +{ + struct port_info *p = netdev_priv(dev); + + epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0; + epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0; + epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0; +} + +static int set_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *epause) +{ + struct port_info *p = netdev_priv(dev); + struct link_config *lc = &p->link_config; + + if (epause->autoneg == AUTONEG_DISABLE) + lc->requested_fc = 0; + else if (lc->supported & SUPPORTED_Autoneg) + lc->requested_fc = PAUSE_AUTONEG; + else + return -EINVAL; + + if (epause->rx_pause) + lc->requested_fc |= PAUSE_RX; + if (epause->tx_pause) + lc->requested_fc |= PAUSE_TX; + if (lc->autoneg == AUTONEG_ENABLE) { + if (netif_running(dev)) + t3_link_start(&p->phy, &p->mac, lc); + } else { + lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); + if (netif_running(dev)) + t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc); + } + return 0; +} + +static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset]; + + e->rx_max_pending = MAX_RX_BUFFERS; + e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS; + e->tx_max_pending = MAX_TXQ_ENTRIES; + + e->rx_pending = q->fl_size; + e->rx_mini_pending = q->rspq_size; + e->rx_jumbo_pending = q->jumbo_size; + e->tx_pending = q->txq_size[0]; +} + +static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + struct qset_params *q; + int i; + + if (e->rx_pending > MAX_RX_BUFFERS || + e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS || + e->tx_pending > MAX_TXQ_ENTRIES || + e->rx_mini_pending > MAX_RSPQ_ENTRIES || + e->rx_mini_pending < MIN_RSPQ_ENTRIES || + e->rx_pending < MIN_FL_ENTRIES || + e->rx_jumbo_pending < MIN_FL_ENTRIES || + e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES) + return -EINVAL; + + if (adapter->flags & FULL_INIT_DONE) + return -EBUSY; + + q = &adapter->params.sge.qset[pi->first_qset]; + for (i = 0; i < pi->nqsets; ++i, ++q) { + q->rspq_size = e->rx_mini_pending; + q->fl_size = e->rx_pending; + q->jumbo_size = e->rx_jumbo_pending; + q->txq_size[0] = e->tx_pending; + q->txq_size[1] = e->tx_pending; + q->txq_size[2] = e->tx_pending; + } + return 0; +} + +static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + struct qset_params *qsp; + struct sge_qset *qs; + int i; + + if (c->rx_coalesce_usecs * 10 > M_NEWTIMER) + return -EINVAL; + + for (i = 0; i < pi->nqsets; i++) { + qsp = &adapter->params.sge.qset[i]; + qs = &adapter->sge.qs[i]; + qsp->coalesce_usecs = c->rx_coalesce_usecs; + t3_update_qset_coalesce(qs, qsp); + } + + return 0; +} + +static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + struct qset_params *q = adapter->params.sge.qset; + + c->rx_coalesce_usecs = q->coalesce_usecs; + return 0; +} + +static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e, + u8 * data) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + int cnt; + + e->magic = EEPROM_MAGIC; + cnt = pci_read_vpd(adapter->pdev, e->offset, e->len, data); + if (cnt < 0) + return cnt; + + e->len = cnt; + + return 0; +} + +static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, + u8 * data) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + u32 aligned_offset, aligned_len; + u8 *buf; + int err; + + if (eeprom->magic != EEPROM_MAGIC) + return -EINVAL; + + aligned_offset = eeprom->offset & ~3; + aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3; + + if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) { + buf = kmalloc(aligned_len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + err = pci_read_vpd(adapter->pdev, aligned_offset, aligned_len, + buf); + if (err < 0) + goto out; + memcpy(buf + (eeprom->offset & 3), data, eeprom->len); + } else + buf = data; + + err = t3_seeprom_wp(adapter, 0); + if (err) + goto out; + + err = pci_write_vpd(adapter->pdev, aligned_offset, aligned_len, buf); + if (err >= 0) + err = t3_seeprom_wp(adapter, 1); +out: + if (buf != data) + kfree(buf); + return err < 0 ? err : 0; +} + +static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol) +{ + wol->supported = 0; + wol->wolopts = 0; + memset(&wol->sopass, 0, sizeof(wol->sopass)); +} + +static const struct ethtool_ops cxgb_ethtool_ops = { + .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS, + .get_drvinfo = get_drvinfo, + .get_msglevel = get_msglevel, + .set_msglevel = set_msglevel, + .get_ringparam = get_sge_param, + .set_ringparam = set_sge_param, + .get_coalesce = get_coalesce, + .set_coalesce = set_coalesce, + .get_eeprom_len = get_eeprom_len, + .get_eeprom = get_eeprom, + .set_eeprom = set_eeprom, + .get_pauseparam = get_pauseparam, + .set_pauseparam = set_pauseparam, + .get_link = ethtool_op_get_link, + .get_strings = get_strings, + .set_phys_id = set_phys_id, + .nway_reset = restart_autoneg, + .get_sset_count = get_sset_count, + .get_ethtool_stats = get_stats, + .get_regs_len = get_regs_len, + .get_regs = get_regs, + .get_wol = get_wol, + .get_link_ksettings = get_link_ksettings, + .set_link_ksettings = set_link_ksettings, +}; + +static int in_range(int val, int lo, int hi) +{ + return val < 0 || (val <= hi && val >= lo); +} + +static int cxgb_siocdevprivate(struct net_device *dev, + struct ifreq *ifreq, + void __user *useraddr, + int cmd) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + int ret; + + if (cmd != SIOCCHIOCTL) + return -EOPNOTSUPP; + + if (copy_from_user(&cmd, useraddr, sizeof(cmd))) + return -EFAULT; + + switch (cmd) { + case CHELSIO_SET_QSET_PARAMS:{ + int i; + struct qset_params *q; + struct ch_qset_params t; + int q1 = pi->first_qset; + int nqsets = pi->nqsets; + + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + if (copy_from_user(&t, useraddr, sizeof(t))) + return -EFAULT; + if (t.cmd != CHELSIO_SET_QSET_PARAMS) + return -EINVAL; + if (t.qset_idx >= SGE_QSETS) + return -EINVAL; + if (!in_range(t.intr_lat, 0, M_NEWTIMER) || + !in_range(t.cong_thres, 0, 255) || + !in_range(t.txq_size[0], MIN_TXQ_ENTRIES, + MAX_TXQ_ENTRIES) || + !in_range(t.txq_size[1], MIN_TXQ_ENTRIES, + MAX_TXQ_ENTRIES) || + !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES, + MAX_CTRL_TXQ_ENTRIES) || + !in_range(t.fl_size[0], MIN_FL_ENTRIES, + MAX_RX_BUFFERS) || + !in_range(t.fl_size[1], MIN_FL_ENTRIES, + MAX_RX_JUMBO_BUFFERS) || + !in_range(t.rspq_size, MIN_RSPQ_ENTRIES, + MAX_RSPQ_ENTRIES)) + return -EINVAL; + + if ((adapter->flags & FULL_INIT_DONE) && + (t.rspq_size >= 0 || t.fl_size[0] >= 0 || + t.fl_size[1] >= 0 || t.txq_size[0] >= 0 || + t.txq_size[1] >= 0 || t.txq_size[2] >= 0 || + t.polling >= 0 || t.cong_thres >= 0)) + return -EBUSY; + + /* Allow setting of any available qset when offload enabled */ + if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) { + q1 = 0; + for_each_port(adapter, i) { + pi = adap2pinfo(adapter, i); + nqsets += pi->first_qset + pi->nqsets; + } + } + + if (t.qset_idx < q1) + return -EINVAL; + if (t.qset_idx > q1 + nqsets - 1) + return -EINVAL; + + q = &adapter->params.sge.qset[t.qset_idx]; + + if (t.rspq_size >= 0) + q->rspq_size = t.rspq_size; + if (t.fl_size[0] >= 0) + q->fl_size = t.fl_size[0]; + if (t.fl_size[1] >= 0) + q->jumbo_size = t.fl_size[1]; + if (t.txq_size[0] >= 0) + q->txq_size[0] = t.txq_size[0]; + if (t.txq_size[1] >= 0) + q->txq_size[1] = t.txq_size[1]; + if (t.txq_size[2] >= 0) + q->txq_size[2] = t.txq_size[2]; + if (t.cong_thres >= 0) + q->cong_thres = t.cong_thres; + if (t.intr_lat >= 0) { + struct sge_qset *qs = + &adapter->sge.qs[t.qset_idx]; + + q->coalesce_usecs = t.intr_lat; + t3_update_qset_coalesce(qs, q); + } + if (t.polling >= 0) { + if (adapter->flags & USING_MSIX) + q->polling = t.polling; + else { + /* No polling with INTx for T3A */ + if (adapter->params.rev == 0 && + !(adapter->flags & USING_MSI)) + t.polling = 0; + + for (i = 0; i < SGE_QSETS; i++) { + q = &adapter->params.sge. + qset[i]; + q->polling = t.polling; + } + } + } + + if (t.lro >= 0) { + if (t.lro) + dev->wanted_features |= NETIF_F_GRO; + else + dev->wanted_features &= ~NETIF_F_GRO; + netdev_update_features(dev); + } + + break; + } + case CHELSIO_GET_QSET_PARAMS:{ + struct qset_params *q; + struct ch_qset_params t; + int q1 = pi->first_qset; + int nqsets = pi->nqsets; + int i; + + if (copy_from_user(&t, useraddr, sizeof(t))) + return -EFAULT; + + if (t.cmd != CHELSIO_GET_QSET_PARAMS) + return -EINVAL; + + /* Display qsets for all ports when offload enabled */ + if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) { + q1 = 0; + for_each_port(adapter, i) { + pi = adap2pinfo(adapter, i); + nqsets = pi->first_qset + pi->nqsets; + } + } + + if (t.qset_idx >= nqsets) + return -EINVAL; + t.qset_idx = array_index_nospec(t.qset_idx, nqsets); + + q = &adapter->params.sge.qset[q1 + t.qset_idx]; + t.rspq_size = q->rspq_size; + t.txq_size[0] = q->txq_size[0]; + t.txq_size[1] = q->txq_size[1]; + t.txq_size[2] = q->txq_size[2]; + t.fl_size[0] = q->fl_size; + t.fl_size[1] = q->jumbo_size; + t.polling = q->polling; + t.lro = !!(dev->features & NETIF_F_GRO); + t.intr_lat = q->coalesce_usecs; + t.cong_thres = q->cong_thres; + t.qnum = q1; + + if (adapter->flags & USING_MSIX) + t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec; + else + t.vector = adapter->pdev->irq; + + if (copy_to_user(useraddr, &t, sizeof(t))) + return -EFAULT; + break; + } + case CHELSIO_SET_QSET_NUM:{ + struct ch_reg edata; + unsigned int i, first_qset = 0, other_qsets = 0; + + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + if (adapter->flags & FULL_INIT_DONE) + return -EBUSY; + if (copy_from_user(&edata, useraddr, sizeof(edata))) + return -EFAULT; + if (edata.cmd != CHELSIO_SET_QSET_NUM) + return -EINVAL; + if (edata.val < 1 || + (edata.val > 1 && !(adapter->flags & USING_MSIX))) + return -EINVAL; + + for_each_port(adapter, i) + if (adapter->port[i] && adapter->port[i] != dev) + other_qsets += adap2pinfo(adapter, i)->nqsets; + + if (edata.val + other_qsets > SGE_QSETS) + return -EINVAL; + + pi->nqsets = edata.val; + + for_each_port(adapter, i) + if (adapter->port[i]) { + pi = adap2pinfo(adapter, i); + pi->first_qset = first_qset; + first_qset += pi->nqsets; + } + break; + } + case CHELSIO_GET_QSET_NUM:{ + struct ch_reg edata; + + memset(&edata, 0, sizeof(struct ch_reg)); + + edata.cmd = CHELSIO_GET_QSET_NUM; + edata.val = pi->nqsets; + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + break; + } + case CHELSIO_LOAD_FW:{ + u8 *fw_data; + struct ch_mem_range t; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + if (copy_from_user(&t, useraddr, sizeof(t))) + return -EFAULT; + if (t.cmd != CHELSIO_LOAD_FW) + return -EINVAL; + /* Check t.len sanity ? */ + fw_data = memdup_user(useraddr + sizeof(t), t.len); + if (IS_ERR(fw_data)) + return PTR_ERR(fw_data); + + ret = t3_load_fw(adapter, fw_data, t.len); + kfree(fw_data); + if (ret) + return ret; + break; + } + case CHELSIO_SETMTUTAB:{ + struct ch_mtus m; + int i; + + if (!is_offload(adapter)) + return -EOPNOTSUPP; + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + if (offload_running(adapter)) + return -EBUSY; + if (copy_from_user(&m, useraddr, sizeof(m))) + return -EFAULT; + if (m.cmd != CHELSIO_SETMTUTAB) + return -EINVAL; + if (m.nmtus != NMTUS) + return -EINVAL; + if (m.mtus[0] < 81) /* accommodate SACK */ + return -EINVAL; + + /* MTUs must be in ascending order */ + for (i = 1; i < NMTUS; ++i) + if (m.mtus[i] < m.mtus[i - 1]) + return -EINVAL; + + memcpy(adapter->params.mtus, m.mtus, + sizeof(adapter->params.mtus)); + break; + } + case CHELSIO_GET_PM:{ + struct tp_params *p = &adapter->params.tp; + struct ch_pm m = {.cmd = CHELSIO_GET_PM }; + + if (!is_offload(adapter)) + return -EOPNOTSUPP; + m.tx_pg_sz = p->tx_pg_size; + m.tx_num_pg = p->tx_num_pgs; + m.rx_pg_sz = p->rx_pg_size; + m.rx_num_pg = p->rx_num_pgs; + m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan; + if (copy_to_user(useraddr, &m, sizeof(m))) + return -EFAULT; + break; + } + case CHELSIO_SET_PM:{ + struct ch_pm m; + struct tp_params *p = &adapter->params.tp; + + if (!is_offload(adapter)) + return -EOPNOTSUPP; + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + if (adapter->flags & FULL_INIT_DONE) + return -EBUSY; + if (copy_from_user(&m, useraddr, sizeof(m))) + return -EFAULT; + if (m.cmd != CHELSIO_SET_PM) + return -EINVAL; + if (!is_power_of_2(m.rx_pg_sz) || + !is_power_of_2(m.tx_pg_sz)) + return -EINVAL; /* not power of 2 */ + if (!(m.rx_pg_sz & 0x14000)) + return -EINVAL; /* not 16KB or 64KB */ + if (!(m.tx_pg_sz & 0x1554000)) + return -EINVAL; + if (m.tx_num_pg == -1) + m.tx_num_pg = p->tx_num_pgs; + if (m.rx_num_pg == -1) + m.rx_num_pg = p->rx_num_pgs; + if (m.tx_num_pg % 24 || m.rx_num_pg % 24) + return -EINVAL; + if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size || + m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size) + return -EINVAL; + p->rx_pg_size = m.rx_pg_sz; + p->tx_pg_size = m.tx_pg_sz; + p->rx_num_pgs = m.rx_num_pg; + p->tx_num_pgs = m.tx_num_pg; + break; + } + case CHELSIO_GET_MEM:{ + struct ch_mem_range t; + struct mc7 *mem; + u64 buf[32]; + + if (!is_offload(adapter)) + return -EOPNOTSUPP; + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + if (!(adapter->flags & FULL_INIT_DONE)) + return -EIO; /* need the memory controllers */ + if (copy_from_user(&t, useraddr, sizeof(t))) + return -EFAULT; + if (t.cmd != CHELSIO_GET_MEM) + return -EINVAL; + if ((t.addr & 7) || (t.len & 7)) + return -EINVAL; + if (t.mem_id == MEM_CM) + mem = &adapter->cm; + else if (t.mem_id == MEM_PMRX) + mem = &adapter->pmrx; + else if (t.mem_id == MEM_PMTX) + mem = &adapter->pmtx; + else + return -EINVAL; + + /* + * Version scheme: + * bits 0..9: chip version + * bits 10..15: chip revision + */ + t.version = 3 | (adapter->params.rev << 10); + if (copy_to_user(useraddr, &t, sizeof(t))) + return -EFAULT; + + /* + * Read 256 bytes at a time as len can be large and we don't + * want to use huge intermediate buffers. + */ + useraddr += sizeof(t); /* advance to start of buffer */ + while (t.len) { + unsigned int chunk = + min_t(unsigned int, t.len, sizeof(buf)); + + ret = + t3_mc7_bd_read(mem, t.addr / 8, chunk / 8, + buf); + if (ret) + return ret; + if (copy_to_user(useraddr, buf, chunk)) + return -EFAULT; + useraddr += chunk; + t.addr += chunk; + t.len -= chunk; + } + break; + } + case CHELSIO_SET_TRACE_FILTER:{ + struct ch_trace t; + const struct trace_params *tp; + + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + if (!offload_running(adapter)) + return -EAGAIN; + if (copy_from_user(&t, useraddr, sizeof(t))) + return -EFAULT; + if (t.cmd != CHELSIO_SET_TRACE_FILTER) + return -EINVAL; + + tp = (const struct trace_params *)&t.sip; + if (t.config_tx) + t3_config_trace_filter(adapter, tp, 0, + t.invert_match, + t.trace_tx); + if (t.config_rx) + t3_config_trace_filter(adapter, tp, 1, + t.invert_match, + t.trace_rx); + break; + } + default: + return -EOPNOTSUPP; + } + return 0; +} + +static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd) +{ + struct mii_ioctl_data *data = if_mii(req); + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + switch (cmd) { + case SIOCGMIIREG: + case SIOCSMIIREG: + /* Convert phy_id from older PRTAD/DEVAD format */ + if (is_10G(adapter) && + !mdio_phy_id_is_c45(data->phy_id) && + (data->phy_id & 0x1f00) && + !(data->phy_id & 0xe0e0)) + data->phy_id = mdio_phy_id_c45(data->phy_id >> 8, + data->phy_id & 0x1f); + fallthrough; + case SIOCGMIIPHY: + return mdio_mii_ioctl(&pi->phy.mdio, data, cmd); + default: + return -EOPNOTSUPP; + } +} + +static int cxgb_change_mtu(struct net_device *dev, int new_mtu) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + int ret; + + if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu))) + return ret; + dev->mtu = new_mtu; + init_port_mtus(adapter); + if (adapter->params.rev == 0 && offload_running(adapter)) + t3_load_mtus(adapter, adapter->params.mtus, + adapter->params.a_wnd, adapter->params.b_wnd, + adapter->port[0]->mtu); + return 0; +} + +static int cxgb_set_mac_addr(struct net_device *dev, void *p) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + struct sockaddr *addr = p; + + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + + eth_hw_addr_set(dev, addr->sa_data); + t3_mac_set_address(&pi->mac, LAN_MAC_IDX, dev->dev_addr); + if (offload_running(adapter)) + write_smt_entry(adapter, pi->port_id); + return 0; +} + +static netdev_features_t cxgb_fix_features(struct net_device *dev, + netdev_features_t features) +{ + /* + * Since there is no support for separate rx/tx vlan accel + * enable/disable make sure tx flag is always in same state as rx. + */ + if (features & NETIF_F_HW_VLAN_CTAG_RX) + features |= NETIF_F_HW_VLAN_CTAG_TX; + else + features &= ~NETIF_F_HW_VLAN_CTAG_TX; + + return features; +} + +static int cxgb_set_features(struct net_device *dev, netdev_features_t features) +{ + netdev_features_t changed = dev->features ^ features; + + if (changed & NETIF_F_HW_VLAN_CTAG_RX) + cxgb_vlan_mode(dev, features); + + return 0; +} + +#ifdef CONFIG_NET_POLL_CONTROLLER +static void cxgb_netpoll(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + int qidx; + + for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) { + struct sge_qset *qs = &adapter->sge.qs[qidx]; + void *source; + + if (adapter->flags & USING_MSIX) + source = qs; + else + source = adapter; + + t3_intr_handler(adapter, qs->rspq.polling) (0, source); + } +} +#endif + +/* + * Periodic accumulation of MAC statistics. + */ +static void mac_stats_update(struct adapter *adapter) +{ + int i; + + for_each_port(adapter, i) { + struct net_device *dev = adapter->port[i]; + struct port_info *p = netdev_priv(dev); + + if (netif_running(dev)) { + spin_lock(&adapter->stats_lock); + t3_mac_update_stats(&p->mac); + spin_unlock(&adapter->stats_lock); + } + } +} + +static void check_link_status(struct adapter *adapter) +{ + int i; + + for_each_port(adapter, i) { + struct net_device *dev = adapter->port[i]; + struct port_info *p = netdev_priv(dev); + int link_fault; + + spin_lock_irq(&adapter->work_lock); + link_fault = p->link_fault; + spin_unlock_irq(&adapter->work_lock); + + if (link_fault) { + t3_link_fault(adapter, i); + continue; + } + + if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) { + t3_xgm_intr_disable(adapter, i); + t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset); + + t3_link_changed(adapter, i); + t3_xgm_intr_enable(adapter, i); + } + } +} + +static void check_t3b2_mac(struct adapter *adapter) +{ + int i; + + if (!rtnl_trylock()) /* synchronize with ifdown */ + return; + + for_each_port(adapter, i) { + struct net_device *dev = adapter->port[i]; + struct port_info *p = netdev_priv(dev); + int status; + + if (!netif_running(dev)) + continue; + + status = 0; + if (netif_running(dev) && netif_carrier_ok(dev)) + status = t3b2_mac_watchdog_task(&p->mac); + if (status == 1) + p->mac.stats.num_toggled++; + else if (status == 2) { + struct cmac *mac = &p->mac; + + t3_mac_set_mtu(mac, dev->mtu); + t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr); + cxgb_set_rxmode(dev); + t3_link_start(&p->phy, mac, &p->link_config); + t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); + t3_port_intr_enable(adapter, p->port_id); + p->mac.stats.num_resets++; + } + } + rtnl_unlock(); +} + + +static void t3_adap_check_task(struct work_struct *work) +{ + struct adapter *adapter = container_of(work, struct adapter, + adap_check_task.work); + const struct adapter_params *p = &adapter->params; + int port; + unsigned int v, status, reset; + + adapter->check_task_cnt++; + + check_link_status(adapter); + + /* Accumulate MAC stats if needed */ + if (!p->linkpoll_period || + (adapter->check_task_cnt * p->linkpoll_period) / 10 >= + p->stats_update_period) { + mac_stats_update(adapter); + adapter->check_task_cnt = 0; + } + + if (p->rev == T3_REV_B2) + check_t3b2_mac(adapter); + + /* + * Scan the XGMAC's to check for various conditions which we want to + * monitor in a periodic polling manner rather than via an interrupt + * condition. This is used for conditions which would otherwise flood + * the system with interrupts and we only really need to know that the + * conditions are "happening" ... For each condition we count the + * detection of the condition and reset it for the next polling loop. + */ + for_each_port(adapter, port) { + struct cmac *mac = &adap2pinfo(adapter, port)->mac; + u32 cause; + + cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset); + reset = 0; + if (cause & F_RXFIFO_OVERFLOW) { + mac->stats.rx_fifo_ovfl++; + reset |= F_RXFIFO_OVERFLOW; + } + + t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset); + } + + /* + * We do the same as above for FL_EMPTY interrupts. + */ + status = t3_read_reg(adapter, A_SG_INT_CAUSE); + reset = 0; + + if (status & F_FLEMPTY) { + struct sge_qset *qs = &adapter->sge.qs[0]; + int i = 0; + + reset |= F_FLEMPTY; + + v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) & + 0xffff; + + while (v) { + qs->fl[i].empty += (v & 1); + if (i) + qs++; + i ^= 1; + v >>= 1; + } + } + + t3_write_reg(adapter, A_SG_INT_CAUSE, reset); + + /* Schedule the next check update if any port is active. */ + spin_lock_irq(&adapter->work_lock); + if (adapter->open_device_map & PORT_MASK) + schedule_chk_task(adapter); + spin_unlock_irq(&adapter->work_lock); +} + +static void db_full_task(struct work_struct *work) +{ + struct adapter *adapter = container_of(work, struct adapter, + db_full_task); + + cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_FULL, 0); +} + +static void db_empty_task(struct work_struct *work) +{ + struct adapter *adapter = container_of(work, struct adapter, + db_empty_task); + + cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_EMPTY, 0); +} + +static void db_drop_task(struct work_struct *work) +{ + struct adapter *adapter = container_of(work, struct adapter, + db_drop_task); + unsigned long delay = 1000; + unsigned short r; + + cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_DROP, 0); + + /* + * Sleep a while before ringing the driver qset dbs. + * The delay is between 1000-2023 usecs. + */ + get_random_bytes(&r, 2); + delay += r & 1023; + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_timeout(usecs_to_jiffies(delay)); + ring_dbs(adapter); +} + +/* + * Processes external (PHY) interrupts in process context. + */ +static void ext_intr_task(struct work_struct *work) +{ + struct adapter *adapter = container_of(work, struct adapter, + ext_intr_handler_task); + int i; + + /* Disable link fault interrupts */ + for_each_port(adapter, i) { + struct net_device *dev = adapter->port[i]; + struct port_info *p = netdev_priv(dev); + + t3_xgm_intr_disable(adapter, i); + t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset); + } + + /* Re-enable link fault interrupts */ + t3_phy_intr_handler(adapter); + + for_each_port(adapter, i) + t3_xgm_intr_enable(adapter, i); + + /* Now reenable external interrupts */ + spin_lock_irq(&adapter->work_lock); + if (adapter->slow_intr_mask) { + adapter->slow_intr_mask |= F_T3DBG; + t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG); + t3_write_reg(adapter, A_PL_INT_ENABLE0, + adapter->slow_intr_mask); + } + spin_unlock_irq(&adapter->work_lock); +} + +/* + * Interrupt-context handler for external (PHY) interrupts. + */ +void t3_os_ext_intr_handler(struct adapter *adapter) +{ + /* + * Schedule a task to handle external interrupts as they may be slow + * and we use a mutex to protect MDIO registers. We disable PHY + * interrupts in the meantime and let the task reenable them when + * it's done. + */ + spin_lock(&adapter->work_lock); + if (adapter->slow_intr_mask) { + adapter->slow_intr_mask &= ~F_T3DBG; + t3_write_reg(adapter, A_PL_INT_ENABLE0, + adapter->slow_intr_mask); + queue_work(cxgb3_wq, &adapter->ext_intr_handler_task); + } + spin_unlock(&adapter->work_lock); +} + +void t3_os_link_fault_handler(struct adapter *adapter, int port_id) +{ + struct net_device *netdev = adapter->port[port_id]; + struct port_info *pi = netdev_priv(netdev); + + spin_lock(&adapter->work_lock); + pi->link_fault = 1; + spin_unlock(&adapter->work_lock); +} + +static int t3_adapter_error(struct adapter *adapter, int reset, int on_wq) +{ + int i, ret = 0; + + if (is_offload(adapter) && + test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) { + cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0); + offload_close(&adapter->tdev); + } + + /* Stop all ports */ + for_each_port(adapter, i) { + struct net_device *netdev = adapter->port[i]; + + if (netif_running(netdev)) + __cxgb_close(netdev, on_wq); + } + + /* Stop SGE timers */ + t3_stop_sge_timers(adapter); + + adapter->flags &= ~FULL_INIT_DONE; + + if (reset) + ret = t3_reset_adapter(adapter); + + pci_disable_device(adapter->pdev); + + return ret; +} + +static int t3_reenable_adapter(struct adapter *adapter) +{ + if (pci_enable_device(adapter->pdev)) { + dev_err(&adapter->pdev->dev, + "Cannot re-enable PCI device after reset.\n"); + goto err; + } + pci_set_master(adapter->pdev); + pci_restore_state(adapter->pdev); + pci_save_state(adapter->pdev); + + /* Free sge resources */ + t3_free_sge_resources(adapter); + + if (t3_replay_prep_adapter(adapter)) + goto err; + + return 0; +err: + return -1; +} + +static void t3_resume_ports(struct adapter *adapter) +{ + int i; + + /* Restart the ports */ + for_each_port(adapter, i) { + struct net_device *netdev = adapter->port[i]; + + if (netif_running(netdev)) { + if (cxgb_open(netdev)) { + dev_err(&adapter->pdev->dev, + "can't bring device back up" + " after reset\n"); + continue; + } + } + } + + if (is_offload(adapter) && !ofld_disable) + cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0); +} + +/* + * processes a fatal error. + * Bring the ports down, reset the chip, bring the ports back up. + */ +static void fatal_error_task(struct work_struct *work) +{ + struct adapter *adapter = container_of(work, struct adapter, + fatal_error_handler_task); + int err = 0; + + rtnl_lock(); + err = t3_adapter_error(adapter, 1, 1); + if (!err) + err = t3_reenable_adapter(adapter); + if (!err) + t3_resume_ports(adapter); + + CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded"); + rtnl_unlock(); +} + +void t3_fatal_err(struct adapter *adapter) +{ + unsigned int fw_status[4]; + + if (adapter->flags & FULL_INIT_DONE) { + t3_sge_stop_dma(adapter); + t3_write_reg(adapter, A_XGM_TX_CTRL, 0); + t3_write_reg(adapter, A_XGM_RX_CTRL, 0); + t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0); + t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0); + + spin_lock(&adapter->work_lock); + t3_intr_disable(adapter); + queue_work(cxgb3_wq, &adapter->fatal_error_handler_task); + spin_unlock(&adapter->work_lock); + } + CH_ALERT(adapter, "encountered fatal error, operation suspended\n"); + if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status)) + CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n", + fw_status[0], fw_status[1], + fw_status[2], fw_status[3]); +} + +/** + * t3_io_error_detected - called when PCI error is detected + * @pdev: Pointer to PCI device + * @state: The current pci connection state + * + * This function is called after a PCI bus error affecting + * this device has been detected. + */ +static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct adapter *adapter = pci_get_drvdata(pdev); + + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + t3_adapter_error(adapter, 0, 0); + + /* Request a slot reset. */ + return PCI_ERS_RESULT_NEED_RESET; +} + +/** + * t3_io_slot_reset - called after the pci bus has been reset. + * @pdev: Pointer to PCI device + * + * Restart the card from scratch, as if from a cold-boot. + */ +static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev) +{ + struct adapter *adapter = pci_get_drvdata(pdev); + + if (!t3_reenable_adapter(adapter)) + return PCI_ERS_RESULT_RECOVERED; + + return PCI_ERS_RESULT_DISCONNECT; +} + +/** + * t3_io_resume - called when traffic can start flowing again. + * @pdev: Pointer to PCI device + * + * This callback is called when the error recovery driver tells us that + * its OK to resume normal operation. + */ +static void t3_io_resume(struct pci_dev *pdev) +{ + struct adapter *adapter = pci_get_drvdata(pdev); + + CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n", + t3_read_reg(adapter, A_PCIE_PEX_ERR)); + + rtnl_lock(); + t3_resume_ports(adapter); + rtnl_unlock(); +} + +static const struct pci_error_handlers t3_err_handler = { + .error_detected = t3_io_error_detected, + .slot_reset = t3_io_slot_reset, + .resume = t3_io_resume, +}; + +/* + * Set the number of qsets based on the number of CPUs and the number of ports, + * not to exceed the number of available qsets, assuming there are enough qsets + * per port in HW. + */ +static void set_nqsets(struct adapter *adap) +{ + int i, j = 0; + int num_cpus = netif_get_num_default_rss_queues(); + int hwports = adap->params.nports; + int nqsets = adap->msix_nvectors - 1; + + if (adap->params.rev > 0 && adap->flags & USING_MSIX) { + if (hwports == 2 && + (hwports * nqsets > SGE_QSETS || + num_cpus >= nqsets / hwports)) + nqsets /= hwports; + if (nqsets > num_cpus) + nqsets = num_cpus; + if (nqsets < 1 || hwports == 4) + nqsets = 1; + } else { + nqsets = 1; + } + + for_each_port(adap, i) { + struct port_info *pi = adap2pinfo(adap, i); + + pi->first_qset = j; + pi->nqsets = nqsets; + j = pi->first_qset + nqsets; + + dev_info(&adap->pdev->dev, + "Port %d using %d queue sets.\n", i, nqsets); + } +} + +static int cxgb_enable_msix(struct adapter *adap) +{ + struct msix_entry entries[SGE_QSETS + 1]; + int vectors; + int i; + + vectors = ARRAY_SIZE(entries); + for (i = 0; i < vectors; ++i) + entries[i].entry = i; + + vectors = pci_enable_msix_range(adap->pdev, entries, + adap->params.nports + 1, vectors); + if (vectors < 0) + return vectors; + + for (i = 0; i < vectors; ++i) + adap->msix_info[i].vec = entries[i].vector; + adap->msix_nvectors = vectors; + + return 0; +} + +static void print_port_info(struct adapter *adap, const struct adapter_info *ai) +{ + static const char *pci_variant[] = { + "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express" + }; + + int i; + char buf[80]; + + if (is_pcie(adap)) + snprintf(buf, sizeof(buf), "%s x%d", + pci_variant[adap->params.pci.variant], + adap->params.pci.width); + else + snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit", + pci_variant[adap->params.pci.variant], + adap->params.pci.speed, adap->params.pci.width); + + for_each_port(adap, i) { + struct net_device *dev = adap->port[i]; + const struct port_info *pi = netdev_priv(dev); + + if (!test_bit(i, &adap->registered_device_map)) + continue; + netdev_info(dev, "%s %s %sNIC (rev %d) %s%s\n", + ai->desc, pi->phy.desc, + is_offload(adap) ? "R" : "", adap->params.rev, buf, + (adap->flags & USING_MSIX) ? " MSI-X" : + (adap->flags & USING_MSI) ? " MSI" : ""); + if (adap->name == dev->name && adap->params.vpd.mclk) + pr_info("%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n", + adap->name, t3_mc7_size(&adap->cm) >> 20, + t3_mc7_size(&adap->pmtx) >> 20, + t3_mc7_size(&adap->pmrx) >> 20, + adap->params.vpd.sn); + } +} + +static const struct net_device_ops cxgb_netdev_ops = { + .ndo_open = cxgb_open, + .ndo_stop = cxgb_close, + .ndo_start_xmit = t3_eth_xmit, + .ndo_get_stats = cxgb_get_stats, + .ndo_validate_addr = eth_validate_addr, + .ndo_set_rx_mode = cxgb_set_rxmode, + .ndo_eth_ioctl = cxgb_ioctl, + .ndo_siocdevprivate = cxgb_siocdevprivate, + .ndo_change_mtu = cxgb_change_mtu, + .ndo_set_mac_address = cxgb_set_mac_addr, + .ndo_fix_features = cxgb_fix_features, + .ndo_set_features = cxgb_set_features, +#ifdef CONFIG_NET_POLL_CONTROLLER + .ndo_poll_controller = cxgb_netpoll, +#endif +}; + +static void cxgb3_init_iscsi_mac(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + + memcpy(pi->iscsic.mac_addr, dev->dev_addr, ETH_ALEN); + pi->iscsic.mac_addr[3] |= 0x80; +} + +#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN) +#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \ + NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA) +static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + int i, err; + resource_size_t mmio_start, mmio_len; + const struct adapter_info *ai; + struct adapter *adapter = NULL; + struct port_info *pi; + + if (!cxgb3_wq) { + cxgb3_wq = create_singlethread_workqueue(DRV_NAME); + if (!cxgb3_wq) { + pr_err("cannot initialize work queue\n"); + return -ENOMEM; + } + } + + err = pci_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, "cannot enable PCI device\n"); + goto out; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + /* Just info, some other driver may have claimed the device. */ + dev_info(&pdev->dev, "cannot obtain PCI resources\n"); + goto out_disable_device; + } + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_err(&pdev->dev, "no usable DMA configuration\n"); + goto out_release_regions; + } + + pci_set_master(pdev); + pci_save_state(pdev); + + mmio_start = pci_resource_start(pdev, 0); + mmio_len = pci_resource_len(pdev, 0); + ai = t3_get_adapter_info(ent->driver_data); + + adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); + if (!adapter) { + err = -ENOMEM; + goto out_release_regions; + } + + adapter->nofail_skb = + alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL); + if (!adapter->nofail_skb) { + dev_err(&pdev->dev, "cannot allocate nofail buffer\n"); + err = -ENOMEM; + goto out_free_adapter; + } + + adapter->regs = ioremap(mmio_start, mmio_len); + if (!adapter->regs) { + dev_err(&pdev->dev, "cannot map device registers\n"); + err = -ENOMEM; + goto out_free_adapter_nofail; + } + + adapter->pdev = pdev; + adapter->name = pci_name(pdev); + adapter->msg_enable = dflt_msg_enable; + adapter->mmio_len = mmio_len; + + mutex_init(&adapter->mdio_lock); + spin_lock_init(&adapter->work_lock); + spin_lock_init(&adapter->stats_lock); + + INIT_LIST_HEAD(&adapter->adapter_list); + INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task); + INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task); + + INIT_WORK(&adapter->db_full_task, db_full_task); + INIT_WORK(&adapter->db_empty_task, db_empty_task); + INIT_WORK(&adapter->db_drop_task, db_drop_task); + + INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task); + + for (i = 0; i < ai->nports0 + ai->nports1; ++i) { + struct net_device *netdev; + + netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS); + if (!netdev) { + err = -ENOMEM; + goto out_free_dev; + } + + SET_NETDEV_DEV(netdev, &pdev->dev); + + adapter->port[i] = netdev; + pi = netdev_priv(netdev); + pi->adapter = adapter; + pi->port_id = i; + netif_carrier_off(netdev); + netdev->irq = pdev->irq; + netdev->mem_start = mmio_start; + netdev->mem_end = mmio_start + mmio_len - 1; + netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | + NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_RX; + netdev->features |= netdev->hw_features | + NETIF_F_HW_VLAN_CTAG_TX; + netdev->vlan_features |= netdev->features & VLAN_FEAT; + + netdev->features |= NETIF_F_HIGHDMA; + + netdev->netdev_ops = &cxgb_netdev_ops; + netdev->ethtool_ops = &cxgb_ethtool_ops; + netdev->min_mtu = 81; + netdev->max_mtu = ETH_MAX_MTU; + netdev->dev_port = pi->port_id; + } + + pci_set_drvdata(pdev, adapter); + if (t3_prep_adapter(adapter, ai, 1) < 0) { + err = -ENODEV; + goto out_free_dev; + } + + /* + * The card is now ready to go. If any errors occur during device + * registration we do not fail the whole card but rather proceed only + * with the ports we manage to register successfully. However we must + * register at least one net device. + */ + for_each_port(adapter, i) { + err = register_netdev(adapter->port[i]); + if (err) + dev_warn(&pdev->dev, + "cannot register net device %s, skipping\n", + adapter->port[i]->name); + else { + /* + * Change the name we use for messages to the name of + * the first successfully registered interface. + */ + if (!adapter->registered_device_map) + adapter->name = adapter->port[i]->name; + + __set_bit(i, &adapter->registered_device_map); + } + } + if (!adapter->registered_device_map) { + dev_err(&pdev->dev, "could not register any net devices\n"); + err = -ENODEV; + goto out_free_dev; + } + + for_each_port(adapter, i) + cxgb3_init_iscsi_mac(adapter->port[i]); + + /* Driver's ready. Reflect it on LEDs */ + t3_led_ready(adapter); + + if (is_offload(adapter)) { + __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map); + cxgb3_adapter_ofld(adapter); + } + + /* See what interrupts we'll be using */ + if (msi > 1 && cxgb_enable_msix(adapter) == 0) + adapter->flags |= USING_MSIX; + else if (msi > 0 && pci_enable_msi(pdev) == 0) + adapter->flags |= USING_MSI; + + set_nqsets(adapter); + + err = sysfs_create_group(&adapter->port[0]->dev.kobj, + &cxgb3_attr_group); + if (err) { + dev_err(&pdev->dev, "cannot create sysfs group\n"); + goto out_close_led; + } + + print_port_info(adapter, ai); + return 0; + +out_close_led: + t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, 0); + +out_free_dev: + iounmap(adapter->regs); + for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i) + if (adapter->port[i]) + free_netdev(adapter->port[i]); + +out_free_adapter_nofail: + kfree_skb(adapter->nofail_skb); + +out_free_adapter: + kfree(adapter); + +out_release_regions: + pci_release_regions(pdev); +out_disable_device: + pci_disable_device(pdev); +out: + return err; +} + +static void remove_one(struct pci_dev *pdev) +{ + struct adapter *adapter = pci_get_drvdata(pdev); + + if (adapter) { + int i; + + t3_sge_stop(adapter); + sysfs_remove_group(&adapter->port[0]->dev.kobj, + &cxgb3_attr_group); + + if (is_offload(adapter)) { + cxgb3_adapter_unofld(adapter); + if (test_bit(OFFLOAD_DEVMAP_BIT, + &adapter->open_device_map)) + offload_close(&adapter->tdev); + } + + for_each_port(adapter, i) + if (test_bit(i, &adapter->registered_device_map)) + unregister_netdev(adapter->port[i]); + + t3_stop_sge_timers(adapter); + t3_free_sge_resources(adapter); + cxgb_disable_msi(adapter); + + for_each_port(adapter, i) + if (adapter->port[i]) + free_netdev(adapter->port[i]); + + iounmap(adapter->regs); + kfree_skb(adapter->nofail_skb); + kfree(adapter); + pci_release_regions(pdev); + pci_disable_device(pdev); + } +} + +static struct pci_driver driver = { + .name = DRV_NAME, + .id_table = cxgb3_pci_tbl, + .probe = init_one, + .remove = remove_one, + .err_handler = &t3_err_handler, +}; + +static int __init cxgb3_init_module(void) +{ + int ret; + + cxgb3_offload_init(); + + ret = pci_register_driver(&driver); + return ret; +} + +static void __exit cxgb3_cleanup_module(void) +{ + pci_unregister_driver(&driver); + if (cxgb3_wq) + destroy_workqueue(cxgb3_wq); +} + +module_init(cxgb3_init_module); +module_exit(cxgb3_cleanup_module); diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_offload.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_offload.c new file mode 100644 index 000000000..89256b866 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_offload.c @@ -0,0 +1,1402 @@ +/* + * Copyright (c) 2006-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/list.h> +#include <linux/slab.h> +#include <net/neighbour.h> +#include <linux/notifier.h> +#include <linux/atomic.h> +#include <linux/proc_fs.h> +#include <linux/if_vlan.h> +#include <net/netevent.h> +#include <linux/highmem.h> +#include <linux/vmalloc.h> +#include <linux/export.h> + +#include "common.h" +#include "regs.h" +#include "cxgb3_ioctl.h" +#include "cxgb3_ctl_defs.h" +#include "cxgb3_defs.h" +#include "l2t.h" +#include "firmware_exports.h" +#include "cxgb3_offload.h" + +static LIST_HEAD(client_list); +static LIST_HEAD(ofld_dev_list); +static DEFINE_MUTEX(cxgb3_db_lock); + +static DEFINE_RWLOCK(adapter_list_lock); +static LIST_HEAD(adapter_list); + +static const unsigned int MAX_ATIDS = 64 * 1024; +static const unsigned int ATID_BASE = 0x10000; + +static void cxgb_neigh_update(struct neighbour *neigh); +static void cxgb_redirect(struct dst_entry *old, struct dst_entry *new, + struct neighbour *neigh, const void *daddr); + +static inline int offload_activated(struct t3cdev *tdev) +{ + const struct adapter *adapter = tdev2adap(tdev); + + return test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map); +} + +/** + * cxgb3_register_client - register an offload client + * @client: the client + * + * Add the client to the client list, + * and call backs the client for each activated offload device + */ +void cxgb3_register_client(struct cxgb3_client *client) +{ + struct t3cdev *tdev; + + mutex_lock(&cxgb3_db_lock); + list_add_tail(&client->client_list, &client_list); + + if (client->add) { + list_for_each_entry(tdev, &ofld_dev_list, ofld_dev_list) { + if (offload_activated(tdev)) + client->add(tdev); + } + } + mutex_unlock(&cxgb3_db_lock); +} + +EXPORT_SYMBOL(cxgb3_register_client); + +/** + * cxgb3_unregister_client - unregister an offload client + * @client: the client + * + * Remove the client to the client list, + * and call backs the client for each activated offload device. + */ +void cxgb3_unregister_client(struct cxgb3_client *client) +{ + struct t3cdev *tdev; + + mutex_lock(&cxgb3_db_lock); + list_del(&client->client_list); + + if (client->remove) { + list_for_each_entry(tdev, &ofld_dev_list, ofld_dev_list) { + if (offload_activated(tdev)) + client->remove(tdev); + } + } + mutex_unlock(&cxgb3_db_lock); +} + +EXPORT_SYMBOL(cxgb3_unregister_client); + +/** + * cxgb3_add_clients - activate registered clients for an offload device + * @tdev: the offload device + * + * Call backs all registered clients once a offload device is activated + */ +void cxgb3_add_clients(struct t3cdev *tdev) +{ + struct cxgb3_client *client; + + mutex_lock(&cxgb3_db_lock); + list_for_each_entry(client, &client_list, client_list) { + if (client->add) + client->add(tdev); + } + mutex_unlock(&cxgb3_db_lock); +} + +/** + * cxgb3_remove_clients - deactivates registered clients + * for an offload device + * @tdev: the offload device + * + * Call backs all registered clients once a offload device is deactivated + */ +void cxgb3_remove_clients(struct t3cdev *tdev) +{ + struct cxgb3_client *client; + + mutex_lock(&cxgb3_db_lock); + list_for_each_entry(client, &client_list, client_list) { + if (client->remove) + client->remove(tdev); + } + mutex_unlock(&cxgb3_db_lock); +} + +void cxgb3_event_notify(struct t3cdev *tdev, u32 event, u32 port) +{ + struct cxgb3_client *client; + + mutex_lock(&cxgb3_db_lock); + list_for_each_entry(client, &client_list, client_list) { + if (client->event_handler) + client->event_handler(tdev, event, port); + } + mutex_unlock(&cxgb3_db_lock); +} + +static struct net_device *get_iff_from_mac(struct adapter *adapter, + const unsigned char *mac, + unsigned int vlan) +{ + int i; + + for_each_port(adapter, i) { + struct net_device *dev = adapter->port[i]; + + if (ether_addr_equal(dev->dev_addr, mac)) { + rcu_read_lock(); + if (vlan && vlan != VLAN_VID_MASK) { + dev = __vlan_find_dev_deep_rcu(dev, htons(ETH_P_8021Q), vlan); + } else if (netif_is_bond_slave(dev)) { + struct net_device *upper_dev; + + while ((upper_dev = + netdev_master_upper_dev_get_rcu(dev))) + dev = upper_dev; + } + rcu_read_unlock(); + return dev; + } + } + return NULL; +} + +static int cxgb_ulp_iscsi_ctl(struct adapter *adapter, unsigned int req, + void *data) +{ + int i; + int ret = 0; + unsigned int val = 0; + struct ulp_iscsi_info *uiip = data; + + switch (req) { + case ULP_ISCSI_GET_PARAMS: + uiip->pdev = adapter->pdev; + uiip->llimit = t3_read_reg(adapter, A_ULPRX_ISCSI_LLIMIT); + uiip->ulimit = t3_read_reg(adapter, A_ULPRX_ISCSI_ULIMIT); + uiip->tagmask = t3_read_reg(adapter, A_ULPRX_ISCSI_TAGMASK); + + val = t3_read_reg(adapter, A_ULPRX_ISCSI_PSZ); + for (i = 0; i < 4; i++, val >>= 8) + uiip->pgsz_factor[i] = val & 0xFF; + + val = t3_read_reg(adapter, A_TP_PARA_REG7); + uiip->max_txsz = + uiip->max_rxsz = min((val >> S_PMMAXXFERLEN0)&M_PMMAXXFERLEN0, + (val >> S_PMMAXXFERLEN1)&M_PMMAXXFERLEN1); + /* + * On tx, the iscsi pdu has to be <= tx page size and has to + * fit into the Tx PM FIFO. + */ + val = min(adapter->params.tp.tx_pg_size, + t3_read_reg(adapter, A_PM1_TX_CFG) >> 17); + uiip->max_txsz = min(val, uiip->max_txsz); + + /* set MaxRxData to 16224 */ + val = t3_read_reg(adapter, A_TP_PARA_REG2); + if ((val >> S_MAXRXDATA) != 0x3f60) { + val &= (M_RXCOALESCESIZE << S_RXCOALESCESIZE); + val |= V_MAXRXDATA(0x3f60); + pr_info("%s, iscsi set MaxRxData to 16224 (0x%x)\n", + adapter->name, val); + t3_write_reg(adapter, A_TP_PARA_REG2, val); + } + + /* + * on rx, the iscsi pdu has to be < rx page size and the + * max rx data length programmed in TP + */ + val = min(adapter->params.tp.rx_pg_size, + ((t3_read_reg(adapter, A_TP_PARA_REG2)) >> + S_MAXRXDATA) & M_MAXRXDATA); + uiip->max_rxsz = min(val, uiip->max_rxsz); + break; + case ULP_ISCSI_SET_PARAMS: + t3_write_reg(adapter, A_ULPRX_ISCSI_TAGMASK, uiip->tagmask); + /* program the ddp page sizes */ + for (i = 0; i < 4; i++) + val |= (uiip->pgsz_factor[i] & 0xF) << (8 * i); + if (val && (val != t3_read_reg(adapter, A_ULPRX_ISCSI_PSZ))) { + pr_info("%s, setting iscsi pgsz 0x%x, %u,%u,%u,%u\n", + adapter->name, val, uiip->pgsz_factor[0], + uiip->pgsz_factor[1], uiip->pgsz_factor[2], + uiip->pgsz_factor[3]); + t3_write_reg(adapter, A_ULPRX_ISCSI_PSZ, val); + } + break; + default: + ret = -EOPNOTSUPP; + } + return ret; +} + +/* Response queue used for RDMA events. */ +#define ASYNC_NOTIF_RSPQ 0 + +static int cxgb_rdma_ctl(struct adapter *adapter, unsigned int req, void *data) +{ + int ret = 0; + + switch (req) { + case RDMA_GET_PARAMS: { + struct rdma_info *rdma = data; + struct pci_dev *pdev = adapter->pdev; + + rdma->udbell_physbase = pci_resource_start(pdev, 2); + rdma->udbell_len = pci_resource_len(pdev, 2); + rdma->tpt_base = + t3_read_reg(adapter, A_ULPTX_TPT_LLIMIT); + rdma->tpt_top = t3_read_reg(adapter, A_ULPTX_TPT_ULIMIT); + rdma->pbl_base = + t3_read_reg(adapter, A_ULPTX_PBL_LLIMIT); + rdma->pbl_top = t3_read_reg(adapter, A_ULPTX_PBL_ULIMIT); + rdma->rqt_base = t3_read_reg(adapter, A_ULPRX_RQ_LLIMIT); + rdma->rqt_top = t3_read_reg(adapter, A_ULPRX_RQ_ULIMIT); + rdma->kdb_addr = adapter->regs + A_SG_KDOORBELL; + rdma->pdev = pdev; + break; + } + case RDMA_CQ_OP:{ + unsigned long flags; + struct rdma_cq_op *rdma = data; + + /* may be called in any context */ + spin_lock_irqsave(&adapter->sge.reg_lock, flags); + ret = t3_sge_cqcntxt_op(adapter, rdma->id, rdma->op, + rdma->credits); + spin_unlock_irqrestore(&adapter->sge.reg_lock, flags); + break; + } + case RDMA_GET_MEM:{ + struct ch_mem_range *t = data; + struct mc7 *mem; + + if ((t->addr & 7) || (t->len & 7)) + return -EINVAL; + if (t->mem_id == MEM_CM) + mem = &adapter->cm; + else if (t->mem_id == MEM_PMRX) + mem = &adapter->pmrx; + else if (t->mem_id == MEM_PMTX) + mem = &adapter->pmtx; + else + return -EINVAL; + + ret = + t3_mc7_bd_read(mem, t->addr / 8, t->len / 8, + (u64 *) t->buf); + if (ret) + return ret; + break; + } + case RDMA_CQ_SETUP:{ + struct rdma_cq_setup *rdma = data; + + spin_lock_irq(&adapter->sge.reg_lock); + ret = + t3_sge_init_cqcntxt(adapter, rdma->id, + rdma->base_addr, rdma->size, + ASYNC_NOTIF_RSPQ, + rdma->ovfl_mode, rdma->credits, + rdma->credit_thres); + spin_unlock_irq(&adapter->sge.reg_lock); + break; + } + case RDMA_CQ_DISABLE: + spin_lock_irq(&adapter->sge.reg_lock); + ret = t3_sge_disable_cqcntxt(adapter, *(unsigned int *)data); + spin_unlock_irq(&adapter->sge.reg_lock); + break; + case RDMA_CTRL_QP_SETUP:{ + struct rdma_ctrlqp_setup *rdma = data; + + spin_lock_irq(&adapter->sge.reg_lock); + ret = t3_sge_init_ecntxt(adapter, FW_RI_SGEEC_START, 0, + SGE_CNTXT_RDMA, + ASYNC_NOTIF_RSPQ, + rdma->base_addr, rdma->size, + FW_RI_TID_START, 1, 0); + spin_unlock_irq(&adapter->sge.reg_lock); + break; + } + case RDMA_GET_MIB: { + spin_lock(&adapter->stats_lock); + t3_tp_get_mib_stats(adapter, (struct tp_mib_stats *)data); + spin_unlock(&adapter->stats_lock); + break; + } + default: + ret = -EOPNOTSUPP; + } + return ret; +} + +static int cxgb_offload_ctl(struct t3cdev *tdev, unsigned int req, void *data) +{ + struct adapter *adapter = tdev2adap(tdev); + struct tid_range *tid; + struct mtutab *mtup; + struct iff_mac *iffmacp; + struct ddp_params *ddpp; + struct adap_ports *ports; + struct ofld_page_info *rx_page_info; + struct tp_params *tp = &adapter->params.tp; + int i; + + switch (req) { + case GET_MAX_OUTSTANDING_WR: + *(unsigned int *)data = FW_WR_NUM; + break; + case GET_WR_LEN: + *(unsigned int *)data = WR_FLITS; + break; + case GET_TX_MAX_CHUNK: + *(unsigned int *)data = 1 << 20; /* 1MB */ + break; + case GET_TID_RANGE: + tid = data; + tid->num = t3_mc5_size(&adapter->mc5) - + adapter->params.mc5.nroutes - + adapter->params.mc5.nfilters - adapter->params.mc5.nservers; + tid->base = 0; + break; + case GET_STID_RANGE: + tid = data; + tid->num = adapter->params.mc5.nservers; + tid->base = t3_mc5_size(&adapter->mc5) - tid->num - + adapter->params.mc5.nfilters - adapter->params.mc5.nroutes; + break; + case GET_L2T_CAPACITY: + *(unsigned int *)data = 2048; + break; + case GET_MTUS: + mtup = data; + mtup->size = NMTUS; + mtup->mtus = adapter->params.mtus; + break; + case GET_IFF_FROM_MAC: + iffmacp = data; + iffmacp->dev = get_iff_from_mac(adapter, iffmacp->mac_addr, + iffmacp->vlan_tag & + VLAN_VID_MASK); + break; + case GET_DDP_PARAMS: + ddpp = data; + ddpp->llimit = t3_read_reg(adapter, A_ULPRX_TDDP_LLIMIT); + ddpp->ulimit = t3_read_reg(adapter, A_ULPRX_TDDP_ULIMIT); + ddpp->tag_mask = t3_read_reg(adapter, A_ULPRX_TDDP_TAGMASK); + break; + case GET_PORTS: + ports = data; + ports->nports = adapter->params.nports; + for_each_port(adapter, i) + ports->lldevs[i] = adapter->port[i]; + break; + case ULP_ISCSI_GET_PARAMS: + case ULP_ISCSI_SET_PARAMS: + if (!offload_running(adapter)) + return -EAGAIN; + return cxgb_ulp_iscsi_ctl(adapter, req, data); + case RDMA_GET_PARAMS: + case RDMA_CQ_OP: + case RDMA_CQ_SETUP: + case RDMA_CQ_DISABLE: + case RDMA_CTRL_QP_SETUP: + case RDMA_GET_MEM: + case RDMA_GET_MIB: + if (!offload_running(adapter)) + return -EAGAIN; + return cxgb_rdma_ctl(adapter, req, data); + case GET_RX_PAGE_INFO: + rx_page_info = data; + rx_page_info->page_size = tp->rx_pg_size; + rx_page_info->num = tp->rx_num_pgs; + break; + case GET_ISCSI_IPV4ADDR: { + struct iscsi_ipv4addr *p = data; + struct port_info *pi = netdev_priv(p->dev); + p->ipv4addr = pi->iscsi_ipv4addr; + break; + } + case GET_EMBEDDED_INFO: { + struct ch_embedded_info *e = data; + + spin_lock(&adapter->stats_lock); + t3_get_fw_version(adapter, &e->fw_vers); + t3_get_tp_version(adapter, &e->tp_vers); + spin_unlock(&adapter->stats_lock); + break; + } + default: + return -EOPNOTSUPP; + } + return 0; +} + +/* + * Dummy handler for Rx offload packets in case we get an offload packet before + * proper processing is setup. This complains and drops the packet as it isn't + * normal to get offload packets at this stage. + */ +static int rx_offload_blackhole(struct t3cdev *dev, struct sk_buff **skbs, + int n) +{ + while (n--) + dev_kfree_skb_any(skbs[n]); + return 0; +} + +static void dummy_neigh_update(struct t3cdev *dev, struct neighbour *neigh) +{ +} + +void cxgb3_set_dummy_ops(struct t3cdev *dev) +{ + dev->recv = rx_offload_blackhole; + dev->neigh_update = dummy_neigh_update; +} + +/* + * Free an active-open TID. + */ +void *cxgb3_free_atid(struct t3cdev *tdev, int atid) +{ + struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; + union active_open_entry *p = atid2entry(t, atid); + void *ctx = p->t3c_tid.ctx; + + spin_lock_bh(&t->atid_lock); + p->next = t->afree; + t->afree = p; + t->atids_in_use--; + spin_unlock_bh(&t->atid_lock); + + return ctx; +} + +EXPORT_SYMBOL(cxgb3_free_atid); + +/* + * Free a server TID and return it to the free pool. + */ +void cxgb3_free_stid(struct t3cdev *tdev, int stid) +{ + struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; + union listen_entry *p = stid2entry(t, stid); + + spin_lock_bh(&t->stid_lock); + p->next = t->sfree; + t->sfree = p; + t->stids_in_use--; + spin_unlock_bh(&t->stid_lock); +} + +EXPORT_SYMBOL(cxgb3_free_stid); + +void cxgb3_insert_tid(struct t3cdev *tdev, struct cxgb3_client *client, + void *ctx, unsigned int tid) +{ + struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; + + t->tid_tab[tid].client = client; + t->tid_tab[tid].ctx = ctx; + atomic_inc(&t->tids_in_use); +} + +EXPORT_SYMBOL(cxgb3_insert_tid); + +/* + * Populate a TID_RELEASE WR. The skb must be already propely sized. + */ +static inline void mk_tid_release(struct sk_buff *skb, unsigned int tid) +{ + struct cpl_tid_release *req; + + skb->priority = CPL_PRIORITY_SETUP; + req = __skb_put(skb, sizeof(*req)); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); +} + +static void t3_process_tid_release_list(struct work_struct *work) +{ + struct t3c_data *td = container_of(work, struct t3c_data, + tid_release_task); + struct sk_buff *skb; + struct t3cdev *tdev = td->dev; + + + spin_lock_bh(&td->tid_release_lock); + while (td->tid_release_list) { + struct t3c_tid_entry *p = td->tid_release_list; + + td->tid_release_list = p->ctx; + spin_unlock_bh(&td->tid_release_lock); + + skb = alloc_skb(sizeof(struct cpl_tid_release), + GFP_KERNEL); + if (!skb) + skb = td->nofail_skb; + if (!skb) { + spin_lock_bh(&td->tid_release_lock); + p->ctx = (void *)td->tid_release_list; + td->tid_release_list = p; + break; + } + mk_tid_release(skb, p - td->tid_maps.tid_tab); + cxgb3_ofld_send(tdev, skb); + p->ctx = NULL; + if (skb == td->nofail_skb) + td->nofail_skb = + alloc_skb(sizeof(struct cpl_tid_release), + GFP_KERNEL); + spin_lock_bh(&td->tid_release_lock); + } + td->release_list_incomplete = (td->tid_release_list == NULL) ? 0 : 1; + spin_unlock_bh(&td->tid_release_lock); + + if (!td->nofail_skb) + td->nofail_skb = + alloc_skb(sizeof(struct cpl_tid_release), + GFP_KERNEL); +} + +/* use ctx as a next pointer in the tid release list */ +void cxgb3_queue_tid_release(struct t3cdev *tdev, unsigned int tid) +{ + struct t3c_data *td = T3C_DATA(tdev); + struct t3c_tid_entry *p = &td->tid_maps.tid_tab[tid]; + + spin_lock_bh(&td->tid_release_lock); + p->ctx = (void *)td->tid_release_list; + p->client = NULL; + td->tid_release_list = p; + if (!p->ctx || td->release_list_incomplete) + schedule_work(&td->tid_release_task); + spin_unlock_bh(&td->tid_release_lock); +} + +EXPORT_SYMBOL(cxgb3_queue_tid_release); + +/* + * Remove a tid from the TID table. A client may defer processing its last + * CPL message if it is locked at the time it arrives, and while the message + * sits in the client's backlog the TID may be reused for another connection. + * To handle this we atomically switch the TID association if it still points + * to the original client context. + */ +void cxgb3_remove_tid(struct t3cdev *tdev, void *ctx, unsigned int tid) +{ + struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; + + BUG_ON(tid >= t->ntids); + if (tdev->type == T3A) + (void)cmpxchg(&t->tid_tab[tid].ctx, ctx, NULL); + else { + struct sk_buff *skb; + + skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC); + if (likely(skb)) { + mk_tid_release(skb, tid); + cxgb3_ofld_send(tdev, skb); + t->tid_tab[tid].ctx = NULL; + } else + cxgb3_queue_tid_release(tdev, tid); + } + atomic_dec(&t->tids_in_use); +} + +EXPORT_SYMBOL(cxgb3_remove_tid); + +int cxgb3_alloc_atid(struct t3cdev *tdev, struct cxgb3_client *client, + void *ctx) +{ + int atid = -1; + struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; + + spin_lock_bh(&t->atid_lock); + if (t->afree && + t->atids_in_use + atomic_read(&t->tids_in_use) + MC5_MIN_TIDS <= + t->ntids) { + union active_open_entry *p = t->afree; + + atid = (p - t->atid_tab) + t->atid_base; + t->afree = p->next; + p->t3c_tid.ctx = ctx; + p->t3c_tid.client = client; + t->atids_in_use++; + } + spin_unlock_bh(&t->atid_lock); + return atid; +} + +EXPORT_SYMBOL(cxgb3_alloc_atid); + +int cxgb3_alloc_stid(struct t3cdev *tdev, struct cxgb3_client *client, + void *ctx) +{ + int stid = -1; + struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; + + spin_lock_bh(&t->stid_lock); + if (t->sfree) { + union listen_entry *p = t->sfree; + + stid = (p - t->stid_tab) + t->stid_base; + t->sfree = p->next; + p->t3c_tid.ctx = ctx; + p->t3c_tid.client = client; + t->stids_in_use++; + } + spin_unlock_bh(&t->stid_lock); + return stid; +} + +EXPORT_SYMBOL(cxgb3_alloc_stid); + +/* Get the t3cdev associated with a net_device */ +struct t3cdev *dev2t3cdev(struct net_device *dev) +{ + const struct port_info *pi = netdev_priv(dev); + + return (struct t3cdev *)pi->adapter; +} + +EXPORT_SYMBOL(dev2t3cdev); + +static int do_smt_write_rpl(struct t3cdev *dev, struct sk_buff *skb) +{ + struct cpl_smt_write_rpl *rpl = cplhdr(skb); + + if (rpl->status != CPL_ERR_NONE) + pr_err("Unexpected SMT_WRITE_RPL status %u for entry %u\n", + rpl->status, GET_TID(rpl)); + + return CPL_RET_BUF_DONE; +} + +static int do_l2t_write_rpl(struct t3cdev *dev, struct sk_buff *skb) +{ + struct cpl_l2t_write_rpl *rpl = cplhdr(skb); + + if (rpl->status != CPL_ERR_NONE) + pr_err("Unexpected L2T_WRITE_RPL status %u for entry %u\n", + rpl->status, GET_TID(rpl)); + + return CPL_RET_BUF_DONE; +} + +static int do_rte_write_rpl(struct t3cdev *dev, struct sk_buff *skb) +{ + struct cpl_rte_write_rpl *rpl = cplhdr(skb); + + if (rpl->status != CPL_ERR_NONE) + pr_err("Unexpected RTE_WRITE_RPL status %u for entry %u\n", + rpl->status, GET_TID(rpl)); + + return CPL_RET_BUF_DONE; +} + +static int do_act_open_rpl(struct t3cdev *dev, struct sk_buff *skb) +{ + struct cpl_act_open_rpl *rpl = cplhdr(skb); + unsigned int atid = G_TID(ntohl(rpl->atid)); + struct t3c_tid_entry *t3c_tid; + + t3c_tid = lookup_atid(&(T3C_DATA(dev))->tid_maps, atid); + if (t3c_tid && t3c_tid->ctx && t3c_tid->client && + t3c_tid->client->handlers && + t3c_tid->client->handlers[CPL_ACT_OPEN_RPL]) { + return t3c_tid->client->handlers[CPL_ACT_OPEN_RPL] (dev, skb, + t3c_tid-> + ctx); + } else { + pr_err("%s: received clientless CPL command 0x%x\n", + dev->name, CPL_ACT_OPEN_RPL); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } +} + +static int do_stid_rpl(struct t3cdev *dev, struct sk_buff *skb) +{ + union opcode_tid *p = cplhdr(skb); + unsigned int stid = G_TID(ntohl(p->opcode_tid)); + struct t3c_tid_entry *t3c_tid; + + t3c_tid = lookup_stid(&(T3C_DATA(dev))->tid_maps, stid); + if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && + t3c_tid->client->handlers[p->opcode]) { + return t3c_tid->client->handlers[p->opcode] (dev, skb, + t3c_tid->ctx); + } else { + pr_err("%s: received clientless CPL command 0x%x\n", + dev->name, p->opcode); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } +} + +static int do_hwtid_rpl(struct t3cdev *dev, struct sk_buff *skb) +{ + union opcode_tid *p = cplhdr(skb); + unsigned int hwtid = G_TID(ntohl(p->opcode_tid)); + struct t3c_tid_entry *t3c_tid; + + t3c_tid = lookup_tid(&(T3C_DATA(dev))->tid_maps, hwtid); + if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && + t3c_tid->client->handlers[p->opcode]) { + return t3c_tid->client->handlers[p->opcode] + (dev, skb, t3c_tid->ctx); + } else { + pr_err("%s: received clientless CPL command 0x%x\n", + dev->name, p->opcode); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } +} + +static int do_cr(struct t3cdev *dev, struct sk_buff *skb) +{ + struct cpl_pass_accept_req *req = cplhdr(skb); + unsigned int stid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); + struct tid_info *t = &(T3C_DATA(dev))->tid_maps; + struct t3c_tid_entry *t3c_tid; + unsigned int tid = GET_TID(req); + + if (unlikely(tid >= t->ntids)) { + printk("%s: passive open TID %u too large\n", + dev->name, tid); + t3_fatal_err(tdev2adap(dev)); + return CPL_RET_BUF_DONE; + } + + t3c_tid = lookup_stid(t, stid); + if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && + t3c_tid->client->handlers[CPL_PASS_ACCEPT_REQ]) { + return t3c_tid->client->handlers[CPL_PASS_ACCEPT_REQ] + (dev, skb, t3c_tid->ctx); + } else { + pr_err("%s: received clientless CPL command 0x%x\n", + dev->name, CPL_PASS_ACCEPT_REQ); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } +} + +/* + * Returns an sk_buff for a reply CPL message of size len. If the input + * sk_buff has no other users it is trimmed and reused, otherwise a new buffer + * is allocated. The input skb must be of size at least len. Note that this + * operation does not destroy the original skb data even if it decides to reuse + * the buffer. + */ +static struct sk_buff *cxgb3_get_cpl_reply_skb(struct sk_buff *skb, size_t len, + gfp_t gfp) +{ + if (likely(!skb_cloned(skb))) { + BUG_ON(skb->len < len); + __skb_trim(skb, len); + skb_get(skb); + } else { + skb = alloc_skb(len, gfp); + if (skb) + __skb_put(skb, len); + } + return skb; +} + +static int do_abort_req_rss(struct t3cdev *dev, struct sk_buff *skb) +{ + union opcode_tid *p = cplhdr(skb); + unsigned int hwtid = G_TID(ntohl(p->opcode_tid)); + struct t3c_tid_entry *t3c_tid; + + t3c_tid = lookup_tid(&(T3C_DATA(dev))->tid_maps, hwtid); + if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && + t3c_tid->client->handlers[p->opcode]) { + return t3c_tid->client->handlers[p->opcode] + (dev, skb, t3c_tid->ctx); + } else { + struct cpl_abort_req_rss *req = cplhdr(skb); + struct cpl_abort_rpl *rpl; + struct sk_buff *reply_skb; + unsigned int tid = GET_TID(req); + u8 cmd = req->status; + + if (req->status == CPL_ERR_RTX_NEG_ADVICE || + req->status == CPL_ERR_PERSIST_NEG_ADVICE) + goto out; + + reply_skb = cxgb3_get_cpl_reply_skb(skb, + sizeof(struct + cpl_abort_rpl), + GFP_ATOMIC); + + if (!reply_skb) { + printk("do_abort_req_rss: couldn't get skb!\n"); + goto out; + } + reply_skb->priority = CPL_PRIORITY_DATA; + __skb_put(reply_skb, sizeof(struct cpl_abort_rpl)); + rpl = cplhdr(reply_skb); + rpl->wr.wr_hi = + htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); + rpl->wr.wr_lo = htonl(V_WR_TID(tid)); + OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, tid)); + rpl->cmd = cmd; + cxgb3_ofld_send(dev, reply_skb); +out: + return CPL_RET_BUF_DONE; + } +} + +static int do_act_establish(struct t3cdev *dev, struct sk_buff *skb) +{ + struct cpl_act_establish *req = cplhdr(skb); + unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); + struct tid_info *t = &(T3C_DATA(dev))->tid_maps; + struct t3c_tid_entry *t3c_tid; + unsigned int tid = GET_TID(req); + + if (unlikely(tid >= t->ntids)) { + printk("%s: active establish TID %u too large\n", + dev->name, tid); + t3_fatal_err(tdev2adap(dev)); + return CPL_RET_BUF_DONE; + } + + t3c_tid = lookup_atid(t, atid); + if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && + t3c_tid->client->handlers[CPL_ACT_ESTABLISH]) { + return t3c_tid->client->handlers[CPL_ACT_ESTABLISH] + (dev, skb, t3c_tid->ctx); + } else { + pr_err("%s: received clientless CPL command 0x%x\n", + dev->name, CPL_ACT_ESTABLISH); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } +} + +static int do_trace(struct t3cdev *dev, struct sk_buff *skb) +{ + struct cpl_trace_pkt *p = cplhdr(skb); + + skb->protocol = htons(0xffff); + skb->dev = dev->lldev; + skb_pull(skb, sizeof(*p)); + skb_reset_mac_header(skb); + netif_receive_skb(skb); + return 0; +} + +/* + * That skb would better have come from process_responses() where we abuse + * ->priority and ->csum to carry our data. NB: if we get to per-arch + * ->csum, the things might get really interesting here. + */ + +static inline u32 get_hwtid(struct sk_buff *skb) +{ + return ntohl((__force __be32)skb->priority) >> 8 & 0xfffff; +} + +static inline u32 get_opcode(struct sk_buff *skb) +{ + return G_OPCODE(ntohl((__force __be32)skb->csum)); +} + +static int do_term(struct t3cdev *dev, struct sk_buff *skb) +{ + unsigned int hwtid = get_hwtid(skb); + unsigned int opcode = get_opcode(skb); + struct t3c_tid_entry *t3c_tid; + + t3c_tid = lookup_tid(&(T3C_DATA(dev))->tid_maps, hwtid); + if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && + t3c_tid->client->handlers[opcode]) { + return t3c_tid->client->handlers[opcode] (dev, skb, + t3c_tid->ctx); + } else { + pr_err("%s: received clientless CPL command 0x%x\n", + dev->name, opcode); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; + } +} + +static int nb_callback(struct notifier_block *self, unsigned long event, + void *ctx) +{ + switch (event) { + case (NETEVENT_NEIGH_UPDATE):{ + cxgb_neigh_update((struct neighbour *)ctx); + break; + } + case (NETEVENT_REDIRECT):{ + struct netevent_redirect *nr = ctx; + cxgb_redirect(nr->old, nr->new, nr->neigh, + nr->daddr); + cxgb_neigh_update(nr->neigh); + break; + } + default: + break; + } + return 0; +} + +static struct notifier_block nb = { + .notifier_call = nb_callback +}; + +/* + * Process a received packet with an unknown/unexpected CPL opcode. + */ +static int do_bad_cpl(struct t3cdev *dev, struct sk_buff *skb) +{ + pr_err("%s: received bad CPL command 0x%x\n", dev->name, *skb->data); + return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; +} + +/* + * Handlers for each CPL opcode + */ +static cpl_handler_func cpl_handlers[NUM_CPL_CMDS]; + +/* + * Add a new handler to the CPL dispatch table. A NULL handler may be supplied + * to unregister an existing handler. + */ +void t3_register_cpl_handler(unsigned int opcode, cpl_handler_func h) +{ + if (opcode < NUM_CPL_CMDS) + cpl_handlers[opcode] = h ? h : do_bad_cpl; + else + pr_err("T3C: handler registration for opcode %x failed\n", + opcode); +} + +EXPORT_SYMBOL(t3_register_cpl_handler); + +/* + * T3CDEV's receive method. + */ +static int process_rx(struct t3cdev *dev, struct sk_buff **skbs, int n) +{ + while (n--) { + struct sk_buff *skb = *skbs++; + unsigned int opcode = get_opcode(skb); + int ret = cpl_handlers[opcode] (dev, skb); + +#if VALIDATE_TID + if (ret & CPL_RET_UNKNOWN_TID) { + union opcode_tid *p = cplhdr(skb); + + pr_err("%s: CPL message (opcode %u) had unknown TID %u\n", + dev->name, opcode, G_TID(ntohl(p->opcode_tid))); + } +#endif + if (ret & CPL_RET_BUF_DONE) + kfree_skb(skb); + } + return 0; +} + +/* + * Sends an sk_buff to a T3C driver after dealing with any active network taps. + */ +int cxgb3_ofld_send(struct t3cdev *dev, struct sk_buff *skb) +{ + int r; + + local_bh_disable(); + r = dev->send(dev, skb); + local_bh_enable(); + return r; +} + +EXPORT_SYMBOL(cxgb3_ofld_send); + +static int is_offloading(struct net_device *dev) +{ + struct adapter *adapter; + int i; + + read_lock_bh(&adapter_list_lock); + list_for_each_entry(adapter, &adapter_list, adapter_list) { + for_each_port(adapter, i) { + if (dev == adapter->port[i]) { + read_unlock_bh(&adapter_list_lock); + return 1; + } + } + } + read_unlock_bh(&adapter_list_lock); + return 0; +} + +static void cxgb_neigh_update(struct neighbour *neigh) +{ + struct net_device *dev; + + if (!neigh) + return; + dev = neigh->dev; + if (dev && (is_offloading(dev))) { + struct t3cdev *tdev = dev2t3cdev(dev); + + BUG_ON(!tdev); + t3_l2t_update(tdev, neigh); + } +} + +static void set_l2t_ix(struct t3cdev *tdev, u32 tid, struct l2t_entry *e) +{ + struct sk_buff *skb; + struct cpl_set_tcb_field *req; + + skb = alloc_skb(sizeof(*req), GFP_ATOMIC); + if (!skb) { + pr_err("%s: cannot allocate skb!\n", __func__); + return; + } + skb->priority = CPL_PRIORITY_CONTROL; + req = skb_put(skb, sizeof(*req)); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); + req->reply = 0; + req->cpu_idx = 0; + req->word = htons(W_TCB_L2T_IX); + req->mask = cpu_to_be64(V_TCB_L2T_IX(M_TCB_L2T_IX)); + req->val = cpu_to_be64(V_TCB_L2T_IX(e->idx)); + tdev->send(tdev, skb); +} + +static void cxgb_redirect(struct dst_entry *old, struct dst_entry *new, + struct neighbour *neigh, + const void *daddr) +{ + struct net_device *dev; + struct tid_info *ti; + struct t3cdev *tdev; + u32 tid; + int update_tcb; + struct l2t_entry *e; + struct t3c_tid_entry *te; + + dev = neigh->dev; + + if (!is_offloading(dev)) + return; + tdev = dev2t3cdev(dev); + BUG_ON(!tdev); + + /* Add new L2T entry */ + e = t3_l2t_get(tdev, new, dev, daddr); + if (!e) { + pr_err("%s: couldn't allocate new l2t entry!\n", __func__); + return; + } + + /* Walk tid table and notify clients of dst change. */ + ti = &(T3C_DATA(tdev))->tid_maps; + for (tid = 0; tid < ti->ntids; tid++) { + te = lookup_tid(ti, tid); + BUG_ON(!te); + if (te && te->ctx && te->client && te->client->redirect) { + update_tcb = te->client->redirect(te->ctx, old, new, e); + if (update_tcb) { + rcu_read_lock(); + l2t_hold(L2DATA(tdev), e); + rcu_read_unlock(); + set_l2t_ix(tdev, tid, e); + } + } + } + l2t_release(tdev, e); +} + +/* + * Allocate and initialize the TID tables. Returns 0 on success. + */ +static int init_tid_tabs(struct tid_info *t, unsigned int ntids, + unsigned int natids, unsigned int nstids, + unsigned int atid_base, unsigned int stid_base) +{ + unsigned long size = ntids * sizeof(*t->tid_tab) + + natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab); + + t->tid_tab = kvzalloc(size, GFP_KERNEL); + if (!t->tid_tab) + return -ENOMEM; + + t->stid_tab = (union listen_entry *)&t->tid_tab[ntids]; + t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids]; + t->ntids = ntids; + t->nstids = nstids; + t->stid_base = stid_base; + t->sfree = NULL; + t->natids = natids; + t->atid_base = atid_base; + t->afree = NULL; + t->stids_in_use = t->atids_in_use = 0; + atomic_set(&t->tids_in_use, 0); + spin_lock_init(&t->stid_lock); + spin_lock_init(&t->atid_lock); + + /* + * Setup the free lists for stid_tab and atid_tab. + */ + if (nstids) { + while (--nstids) + t->stid_tab[nstids - 1].next = &t->stid_tab[nstids]; + t->sfree = t->stid_tab; + } + if (natids) { + while (--natids) + t->atid_tab[natids - 1].next = &t->atid_tab[natids]; + t->afree = t->atid_tab; + } + return 0; +} + +static void free_tid_maps(struct tid_info *t) +{ + kvfree(t->tid_tab); +} + +static inline void add_adapter(struct adapter *adap) +{ + write_lock_bh(&adapter_list_lock); + list_add_tail(&adap->adapter_list, &adapter_list); + write_unlock_bh(&adapter_list_lock); +} + +static inline void remove_adapter(struct adapter *adap) +{ + write_lock_bh(&adapter_list_lock); + list_del(&adap->adapter_list); + write_unlock_bh(&adapter_list_lock); +} + +int cxgb3_offload_activate(struct adapter *adapter) +{ + struct t3cdev *dev = &adapter->tdev; + int natids, err; + struct t3c_data *t; + struct tid_range stid_range, tid_range; + struct mtutab mtutab; + unsigned int l2t_capacity; + struct l2t_data *l2td; + + t = kzalloc(sizeof(*t), GFP_KERNEL); + if (!t) + return -ENOMEM; + + err = -EOPNOTSUPP; + if (dev->ctl(dev, GET_TX_MAX_CHUNK, &t->tx_max_chunk) < 0 || + dev->ctl(dev, GET_MAX_OUTSTANDING_WR, &t->max_wrs) < 0 || + dev->ctl(dev, GET_L2T_CAPACITY, &l2t_capacity) < 0 || + dev->ctl(dev, GET_MTUS, &mtutab) < 0 || + dev->ctl(dev, GET_TID_RANGE, &tid_range) < 0 || + dev->ctl(dev, GET_STID_RANGE, &stid_range) < 0) + goto out_free; + + err = -ENOMEM; + l2td = t3_init_l2t(l2t_capacity); + if (!l2td) + goto out_free; + + natids = min(tid_range.num / 2, MAX_ATIDS); + err = init_tid_tabs(&t->tid_maps, tid_range.num, natids, + stid_range.num, ATID_BASE, stid_range.base); + if (err) + goto out_free_l2t; + + t->mtus = mtutab.mtus; + t->nmtus = mtutab.size; + + INIT_WORK(&t->tid_release_task, t3_process_tid_release_list); + spin_lock_init(&t->tid_release_lock); + INIT_LIST_HEAD(&t->list_node); + t->dev = dev; + + RCU_INIT_POINTER(dev->l2opt, l2td); + T3C_DATA(dev) = t; + dev->recv = process_rx; + dev->neigh_update = t3_l2t_update; + + /* Register netevent handler once */ + if (list_empty(&adapter_list)) + register_netevent_notifier(&nb); + + t->nofail_skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_KERNEL); + t->release_list_incomplete = 0; + + add_adapter(adapter); + return 0; + +out_free_l2t: + kvfree(l2td); +out_free: + kfree(t); + return err; +} + +static void clean_l2_data(struct rcu_head *head) +{ + struct l2t_data *d = container_of(head, struct l2t_data, rcu_head); + kvfree(d); +} + + +void cxgb3_offload_deactivate(struct adapter *adapter) +{ + struct t3cdev *tdev = &adapter->tdev; + struct t3c_data *t = T3C_DATA(tdev); + struct l2t_data *d; + + remove_adapter(adapter); + if (list_empty(&adapter_list)) + unregister_netevent_notifier(&nb); + + free_tid_maps(&t->tid_maps); + T3C_DATA(tdev) = NULL; + rcu_read_lock(); + d = L2DATA(tdev); + rcu_read_unlock(); + RCU_INIT_POINTER(tdev->l2opt, NULL); + call_rcu(&d->rcu_head, clean_l2_data); + kfree_skb(t->nofail_skb); + kfree(t); +} + +static inline void register_tdev(struct t3cdev *tdev) +{ + static int unit; + + mutex_lock(&cxgb3_db_lock); + snprintf(tdev->name, sizeof(tdev->name), "ofld_dev%d", unit++); + list_add_tail(&tdev->ofld_dev_list, &ofld_dev_list); + mutex_unlock(&cxgb3_db_lock); +} + +static inline void unregister_tdev(struct t3cdev *tdev) +{ + mutex_lock(&cxgb3_db_lock); + list_del(&tdev->ofld_dev_list); + mutex_unlock(&cxgb3_db_lock); +} + +static inline int adap2type(struct adapter *adapter) +{ + int type = 0; + + switch (adapter->params.rev) { + case T3_REV_A: + type = T3A; + break; + case T3_REV_B: + case T3_REV_B2: + type = T3B; + break; + case T3_REV_C: + type = T3C; + break; + } + return type; +} + +void cxgb3_adapter_ofld(struct adapter *adapter) +{ + struct t3cdev *tdev = &adapter->tdev; + + INIT_LIST_HEAD(&tdev->ofld_dev_list); + + cxgb3_set_dummy_ops(tdev); + tdev->send = t3_offload_tx; + tdev->ctl = cxgb_offload_ctl; + tdev->type = adap2type(adapter); + + register_tdev(tdev); +} + +void cxgb3_adapter_unofld(struct adapter *adapter) +{ + struct t3cdev *tdev = &adapter->tdev; + + tdev->recv = NULL; + tdev->neigh_update = NULL; + + unregister_tdev(tdev); +} + +void __init cxgb3_offload_init(void) +{ + int i; + + for (i = 0; i < NUM_CPL_CMDS; ++i) + cpl_handlers[i] = do_bad_cpl; + + t3_register_cpl_handler(CPL_SMT_WRITE_RPL, do_smt_write_rpl); + t3_register_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl); + t3_register_cpl_handler(CPL_RTE_WRITE_RPL, do_rte_write_rpl); + t3_register_cpl_handler(CPL_PASS_OPEN_RPL, do_stid_rpl); + t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_stid_rpl); + t3_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_cr); + t3_register_cpl_handler(CPL_PASS_ESTABLISH, do_hwtid_rpl); + t3_register_cpl_handler(CPL_ABORT_RPL_RSS, do_hwtid_rpl); + t3_register_cpl_handler(CPL_ABORT_RPL, do_hwtid_rpl); + t3_register_cpl_handler(CPL_RX_URG_NOTIFY, do_hwtid_rpl); + t3_register_cpl_handler(CPL_RX_DATA, do_hwtid_rpl); + t3_register_cpl_handler(CPL_TX_DATA_ACK, do_hwtid_rpl); + t3_register_cpl_handler(CPL_TX_DMA_ACK, do_hwtid_rpl); + t3_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl); + t3_register_cpl_handler(CPL_PEER_CLOSE, do_hwtid_rpl); + t3_register_cpl_handler(CPL_CLOSE_CON_RPL, do_hwtid_rpl); + t3_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req_rss); + t3_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish); + t3_register_cpl_handler(CPL_SET_TCB_RPL, do_hwtid_rpl); + t3_register_cpl_handler(CPL_GET_TCB_RPL, do_hwtid_rpl); + t3_register_cpl_handler(CPL_RDMA_TERMINATE, do_term); + t3_register_cpl_handler(CPL_RDMA_EC_STATUS, do_hwtid_rpl); + t3_register_cpl_handler(CPL_TRACE_PKT, do_trace); + t3_register_cpl_handler(CPL_RX_DATA_DDP, do_hwtid_rpl); + t3_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_hwtid_rpl); + t3_register_cpl_handler(CPL_ISCSI_HDR, do_hwtid_rpl); +} diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_offload.h b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_offload.h new file mode 100644 index 000000000..929c29811 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_offload.h @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2006-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef _CXGB3_OFFLOAD_H +#define _CXGB3_OFFLOAD_H + +#include <linux/list.h> +#include <linux/skbuff.h> + +#include "l2t.h" + +#include "t3cdev.h" +#include "t3_cpl.h" + +struct adapter; + +void cxgb3_offload_init(void); + +void cxgb3_adapter_ofld(struct adapter *adapter); +void cxgb3_adapter_unofld(struct adapter *adapter); +int cxgb3_offload_activate(struct adapter *adapter); +void cxgb3_offload_deactivate(struct adapter *adapter); + +void cxgb3_set_dummy_ops(struct t3cdev *dev); + +struct t3cdev *dev2t3cdev(struct net_device *dev); + +/* + * Client registration. Users of T3 driver must register themselves. + * The T3 driver will call the add function of every client for each T3 + * adapter activated, passing up the t3cdev ptr. Each client fills out an + * array of callback functions to process CPL messages. + */ + +void cxgb3_register_client(struct cxgb3_client *client); +void cxgb3_unregister_client(struct cxgb3_client *client); +void cxgb3_add_clients(struct t3cdev *tdev); +void cxgb3_remove_clients(struct t3cdev *tdev); +void cxgb3_event_notify(struct t3cdev *tdev, u32 event, u32 port); + +typedef int (*cxgb3_cpl_handler_func)(struct t3cdev *dev, + struct sk_buff *skb, void *ctx); + +enum { + OFFLOAD_STATUS_UP, + OFFLOAD_STATUS_DOWN, + OFFLOAD_PORT_DOWN, + OFFLOAD_PORT_UP, + OFFLOAD_DB_FULL, + OFFLOAD_DB_EMPTY, + OFFLOAD_DB_DROP +}; + +struct cxgb3_client { + char *name; + void (*add) (struct t3cdev *); + void (*remove) (struct t3cdev *); + cxgb3_cpl_handler_func *handlers; + int (*redirect)(void *ctx, struct dst_entry *old, + struct dst_entry *new, struct l2t_entry *l2t); + struct list_head client_list; + void (*event_handler)(struct t3cdev *tdev, u32 event, u32 port); +}; + +/* + * TID allocation services. + */ +int cxgb3_alloc_atid(struct t3cdev *dev, struct cxgb3_client *client, + void *ctx); +int cxgb3_alloc_stid(struct t3cdev *dev, struct cxgb3_client *client, + void *ctx); +void *cxgb3_free_atid(struct t3cdev *dev, int atid); +void cxgb3_free_stid(struct t3cdev *dev, int stid); +void cxgb3_insert_tid(struct t3cdev *dev, struct cxgb3_client *client, + void *ctx, unsigned int tid); +void cxgb3_queue_tid_release(struct t3cdev *dev, unsigned int tid); +void cxgb3_remove_tid(struct t3cdev *dev, void *ctx, unsigned int tid); + +struct t3c_tid_entry { + struct cxgb3_client *client; + void *ctx; +}; + +/* CPL message priority levels */ +enum { + CPL_PRIORITY_DATA = 0, /* data messages */ + CPL_PRIORITY_SETUP = 1, /* connection setup messages */ + CPL_PRIORITY_TEARDOWN = 0, /* connection teardown messages */ + CPL_PRIORITY_LISTEN = 1, /* listen start/stop messages */ + CPL_PRIORITY_ACK = 1, /* RX ACK messages */ + CPL_PRIORITY_CONTROL = 1 /* offload control messages */ +}; + +/* Flags for return value of CPL message handlers */ +enum { + CPL_RET_BUF_DONE = 1, /* buffer processing done, buffer may be freed */ + CPL_RET_BAD_MSG = 2, /* bad CPL message (e.g., unknown opcode) */ + CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */ +}; + +typedef int (*cpl_handler_func)(struct t3cdev *dev, struct sk_buff *skb); + +/* + * Returns a pointer to the first byte of the CPL header in an sk_buff that + * contains a CPL message. + */ +static inline void *cplhdr(struct sk_buff *skb) +{ + return skb->data; +} + +void t3_register_cpl_handler(unsigned int opcode, cpl_handler_func h); + +union listen_entry { + struct t3c_tid_entry t3c_tid; + union listen_entry *next; +}; + +union active_open_entry { + struct t3c_tid_entry t3c_tid; + union active_open_entry *next; +}; + +/* + * Holds the size, base address, free list start, etc of the TID, server TID, + * and active-open TID tables for a offload device. + * The tables themselves are allocated dynamically. + */ +struct tid_info { + struct t3c_tid_entry *tid_tab; + unsigned int ntids; + atomic_t tids_in_use; + + union listen_entry *stid_tab; + unsigned int nstids; + unsigned int stid_base; + + union active_open_entry *atid_tab; + unsigned int natids; + unsigned int atid_base; + + /* + * The following members are accessed R/W so we put them in their own + * cache lines. + * + * XXX We could combine the atid fields above with the lock here since + * atids are use once (unlike other tids). OTOH the above fields are + * usually in cache due to tid_tab. + */ + spinlock_t atid_lock ____cacheline_aligned_in_smp; + union active_open_entry *afree; + unsigned int atids_in_use; + + spinlock_t stid_lock ____cacheline_aligned; + union listen_entry *sfree; + unsigned int stids_in_use; +}; + +struct t3c_data { + struct list_head list_node; + struct t3cdev *dev; + unsigned int tx_max_chunk; /* max payload for TX_DATA */ + unsigned int max_wrs; /* max in-flight WRs per connection */ + unsigned int nmtus; + const unsigned short *mtus; + struct tid_info tid_maps; + + struct t3c_tid_entry *tid_release_list; + spinlock_t tid_release_lock; + struct work_struct tid_release_task; + + struct sk_buff *nofail_skb; + unsigned int release_list_incomplete; +}; + +/* + * t3cdev -> t3c_data accessor + */ +#define T3C_DATA(dev) (*(struct t3c_data **)&(dev)->l4opt) + +#endif diff --git a/drivers/net/ethernet/chelsio/cxgb3/firmware_exports.h b/drivers/net/ethernet/chelsio/cxgb3/firmware_exports.h new file mode 100644 index 000000000..0d9b0e6dc --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/firmware_exports.h @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2004-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef _FIRMWARE_EXPORTS_H_ +#define _FIRMWARE_EXPORTS_H_ + +/* WR OPCODES supported by the firmware. + */ +#define FW_WROPCODE_FORWARD 0x01 +#define FW_WROPCODE_BYPASS 0x05 + +#define FW_WROPCODE_TUNNEL_TX_PKT 0x03 + +#define FW_WROPOCDE_ULPTX_DATA_SGL 0x00 +#define FW_WROPCODE_ULPTX_MEM_READ 0x02 +#define FW_WROPCODE_ULPTX_PKT 0x04 +#define FW_WROPCODE_ULPTX_INVALIDATE 0x06 + +#define FW_WROPCODE_TUNNEL_RX_PKT 0x07 + +#define FW_WROPCODE_OFLD_GETTCB_RPL 0x08 +#define FW_WROPCODE_OFLD_CLOSE_CON 0x09 +#define FW_WROPCODE_OFLD_TP_ABORT_CON_REQ 0x0A +#define FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL 0x0F +#define FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ 0x0B +#define FW_WROPCODE_OFLD_TP_ABORT_CON_RPL 0x0C +#define FW_WROPCODE_OFLD_TX_DATA 0x0D +#define FW_WROPCODE_OFLD_TX_DATA_ACK 0x0E + +#define FW_WROPCODE_RI_RDMA_INIT 0x10 +#define FW_WROPCODE_RI_RDMA_WRITE 0x11 +#define FW_WROPCODE_RI_RDMA_READ_REQ 0x12 +#define FW_WROPCODE_RI_RDMA_READ_RESP 0x13 +#define FW_WROPCODE_RI_SEND 0x14 +#define FW_WROPCODE_RI_TERMINATE 0x15 +#define FW_WROPCODE_RI_RDMA_READ 0x16 +#define FW_WROPCODE_RI_RECEIVE 0x17 +#define FW_WROPCODE_RI_BIND_MW 0x18 +#define FW_WROPCODE_RI_FASTREGISTER_MR 0x19 +#define FW_WROPCODE_RI_LOCAL_INV 0x1A +#define FW_WROPCODE_RI_MODIFY_QP 0x1B +#define FW_WROPCODE_RI_BYPASS 0x1C + +#define FW_WROPOCDE_RSVD 0x1E + +#define FW_WROPCODE_SGE_EGRESSCONTEXT_RR 0x1F + +#define FW_WROPCODE_MNGT 0x1D +#define FW_MNGTOPCODE_PKTSCHED_SET 0x00 + +/* Maximum size of a WR sent from the host, limited by the SGE. + * + * Note: WR coming from ULP or TP are only limited by CIM. + */ +#define FW_WR_SIZE 128 + +/* Maximum number of outstanding WRs sent from the host. Value must be + * programmed in the CTRL/TUNNEL/QP SGE Egress Context and used by + * offload modules to limit the number of WRs per connection. + */ +#define FW_T3_WR_NUM 16 +#define FW_N3_WR_NUM 7 + +#ifndef N3 +# define FW_WR_NUM FW_T3_WR_NUM +#else +# define FW_WR_NUM FW_N3_WR_NUM +#endif + +/* FW_TUNNEL_NUM corresponds to the number of supported TUNNEL Queues. These + * queues must start at SGE Egress Context FW_TUNNEL_SGEEC_START and must + * start at 'TID' (or 'uP Token') FW_TUNNEL_TID_START. + * + * Ingress Traffic (e.g. DMA completion credit) for TUNNEL Queue[i] is sent + * to RESP Queue[i]. + */ +#define FW_TUNNEL_NUM 8 +#define FW_TUNNEL_SGEEC_START 8 +#define FW_TUNNEL_TID_START 65544 + +/* FW_CTRL_NUM corresponds to the number of supported CTRL Queues. These queues + * must start at SGE Egress Context FW_CTRL_SGEEC_START and must start at 'TID' + * (or 'uP Token') FW_CTRL_TID_START. + * + * Ingress Traffic for CTRL Queue[i] is sent to RESP Queue[i]. + */ +#define FW_CTRL_NUM 8 +#define FW_CTRL_SGEEC_START 65528 +#define FW_CTRL_TID_START 65536 + +/* FW_OFLD_NUM corresponds to the number of supported OFFLOAD Queues. These + * queues must start at SGE Egress Context FW_OFLD_SGEEC_START. + * + * Note: the 'uP Token' in the SGE Egress Context fields is irrelevant for + * OFFLOAD Queues, as the host is responsible for providing the correct TID in + * every WR. + * + * Ingress Trafffic for OFFLOAD Queue[i] is sent to RESP Queue[i]. + */ +#define FW_OFLD_NUM 8 +#define FW_OFLD_SGEEC_START 0 + +/* + * + */ +#define FW_RI_NUM 1 +#define FW_RI_SGEEC_START 65527 +#define FW_RI_TID_START 65552 + +/* + * The RX_PKT_TID + */ +#define FW_RX_PKT_NUM 1 +#define FW_RX_PKT_TID_START 65553 + +/* FW_WRC_NUM corresponds to the number of Work Request Context that supported + * by the firmware. + */ +#define FW_WRC_NUM \ + (65536 + FW_TUNNEL_NUM + FW_CTRL_NUM + FW_RI_NUM + FW_RX_PKT_NUM) + +/* + * FW type and version. + */ +#define S_FW_VERSION_TYPE 28 +#define M_FW_VERSION_TYPE 0xF +#define V_FW_VERSION_TYPE(x) ((x) << S_FW_VERSION_TYPE) +#define G_FW_VERSION_TYPE(x) \ + (((x) >> S_FW_VERSION_TYPE) & M_FW_VERSION_TYPE) + +#define S_FW_VERSION_MAJOR 16 +#define M_FW_VERSION_MAJOR 0xFFF +#define V_FW_VERSION_MAJOR(x) ((x) << S_FW_VERSION_MAJOR) +#define G_FW_VERSION_MAJOR(x) \ + (((x) >> S_FW_VERSION_MAJOR) & M_FW_VERSION_MAJOR) + +#define S_FW_VERSION_MINOR 8 +#define M_FW_VERSION_MINOR 0xFF +#define V_FW_VERSION_MINOR(x) ((x) << S_FW_VERSION_MINOR) +#define G_FW_VERSION_MINOR(x) \ + (((x) >> S_FW_VERSION_MINOR) & M_FW_VERSION_MINOR) + +#define S_FW_VERSION_MICRO 0 +#define M_FW_VERSION_MICRO 0xFF +#define V_FW_VERSION_MICRO(x) ((x) << S_FW_VERSION_MICRO) +#define G_FW_VERSION_MICRO(x) \ + (((x) >> S_FW_VERSION_MICRO) & M_FW_VERSION_MICRO) + +#endif /* _FIRMWARE_EXPORTS_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb3/l2t.c b/drivers/net/ethernet/chelsio/cxgb3/l2t.c new file mode 100644 index 000000000..9749d1239 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/l2t.c @@ -0,0 +1,465 @@ +/* + * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include <linux/skbuff.h> +#include <linux/netdevice.h> +#include <linux/if.h> +#include <linux/if_vlan.h> +#include <linux/jhash.h> +#include <linux/slab.h> +#include <linux/export.h> +#include <net/neighbour.h> +#include "common.h" +#include "t3cdev.h" +#include "cxgb3_defs.h" +#include "l2t.h" +#include "t3_cpl.h" +#include "firmware_exports.h" + +#define VLAN_NONE 0xfff + +/* + * Module locking notes: There is a RW lock protecting the L2 table as a + * whole plus a spinlock per L2T entry. Entry lookups and allocations happen + * under the protection of the table lock, individual entry changes happen + * while holding that entry's spinlock. The table lock nests outside the + * entry locks. Allocations of new entries take the table lock as writers so + * no other lookups can happen while allocating new entries. Entry updates + * take the table lock as readers so multiple entries can be updated in + * parallel. An L2T entry can be dropped by decrementing its reference count + * and therefore can happen in parallel with entry allocation but no entry + * can change state or increment its ref count during allocation as both of + * these perform lookups. + */ + +static inline unsigned int vlan_prio(const struct l2t_entry *e) +{ + return e->vlan >> 13; +} + +static inline unsigned int arp_hash(u32 key, int ifindex, + const struct l2t_data *d) +{ + return jhash_2words(key, ifindex, 0) & (d->nentries - 1); +} + +static inline void neigh_replace(struct l2t_entry *e, struct neighbour *n) +{ + neigh_hold(n); + if (e->neigh) + neigh_release(e->neigh); + e->neigh = n; +} + +/* + * Set up an L2T entry and send any packets waiting in the arp queue. The + * supplied skb is used for the CPL_L2T_WRITE_REQ. Must be called with the + * entry locked. + */ +static int setup_l2e_send_pending(struct t3cdev *dev, struct sk_buff *skb, + struct l2t_entry *e) +{ + struct cpl_l2t_write_req *req; + struct sk_buff *tmp; + + if (!skb) { + skb = alloc_skb(sizeof(*req), GFP_ATOMIC); + if (!skb) + return -ENOMEM; + } + + req = __skb_put(skb, sizeof(*req)); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, e->idx)); + req->params = htonl(V_L2T_W_IDX(e->idx) | V_L2T_W_IFF(e->smt_idx) | + V_L2T_W_VLAN(e->vlan & VLAN_VID_MASK) | + V_L2T_W_PRIO(vlan_prio(e))); + memcpy(e->dmac, e->neigh->ha, sizeof(e->dmac)); + memcpy(req->dst_mac, e->dmac, sizeof(req->dst_mac)); + skb->priority = CPL_PRIORITY_CONTROL; + cxgb3_ofld_send(dev, skb); + + skb_queue_walk_safe(&e->arpq, skb, tmp) { + __skb_unlink(skb, &e->arpq); + cxgb3_ofld_send(dev, skb); + } + e->state = L2T_STATE_VALID; + + return 0; +} + +/* + * Add a packet to the an L2T entry's queue of packets awaiting resolution. + * Must be called with the entry's lock held. + */ +static inline void arpq_enqueue(struct l2t_entry *e, struct sk_buff *skb) +{ + __skb_queue_tail(&e->arpq, skb); +} + +int t3_l2t_send_slow(struct t3cdev *dev, struct sk_buff *skb, + struct l2t_entry *e) +{ +again: + switch (e->state) { + case L2T_STATE_STALE: /* entry is stale, kick off revalidation */ + neigh_event_send(e->neigh, NULL); + spin_lock_bh(&e->lock); + if (e->state == L2T_STATE_STALE) + e->state = L2T_STATE_VALID; + spin_unlock_bh(&e->lock); + fallthrough; + case L2T_STATE_VALID: /* fast-path, send the packet on */ + return cxgb3_ofld_send(dev, skb); + case L2T_STATE_RESOLVING: + spin_lock_bh(&e->lock); + if (e->state != L2T_STATE_RESOLVING) { + /* ARP already completed */ + spin_unlock_bh(&e->lock); + goto again; + } + arpq_enqueue(e, skb); + spin_unlock_bh(&e->lock); + + /* + * Only the first packet added to the arpq should kick off + * resolution. However, because the alloc_skb below can fail, + * we allow each packet added to the arpq to retry resolution + * as a way of recovering from transient memory exhaustion. + * A better way would be to use a work request to retry L2T + * entries when there's no memory. + */ + if (!neigh_event_send(e->neigh, NULL)) { + skb = alloc_skb(sizeof(struct cpl_l2t_write_req), + GFP_ATOMIC); + if (!skb) + break; + + spin_lock_bh(&e->lock); + if (!skb_queue_empty(&e->arpq)) + setup_l2e_send_pending(dev, skb, e); + else /* we lost the race */ + __kfree_skb(skb); + spin_unlock_bh(&e->lock); + } + } + return 0; +} + +EXPORT_SYMBOL(t3_l2t_send_slow); + +void t3_l2t_send_event(struct t3cdev *dev, struct l2t_entry *e) +{ +again: + switch (e->state) { + case L2T_STATE_STALE: /* entry is stale, kick off revalidation */ + neigh_event_send(e->neigh, NULL); + spin_lock_bh(&e->lock); + if (e->state == L2T_STATE_STALE) { + e->state = L2T_STATE_VALID; + } + spin_unlock_bh(&e->lock); + return; + case L2T_STATE_VALID: /* fast-path, send the packet on */ + return; + case L2T_STATE_RESOLVING: + spin_lock_bh(&e->lock); + if (e->state != L2T_STATE_RESOLVING) { + /* ARP already completed */ + spin_unlock_bh(&e->lock); + goto again; + } + spin_unlock_bh(&e->lock); + + /* + * Only the first packet added to the arpq should kick off + * resolution. However, because the alloc_skb below can fail, + * we allow each packet added to the arpq to retry resolution + * as a way of recovering from transient memory exhaustion. + * A better way would be to use a work request to retry L2T + * entries when there's no memory. + */ + neigh_event_send(e->neigh, NULL); + } +} + +EXPORT_SYMBOL(t3_l2t_send_event); + +/* + * Allocate a free L2T entry. Must be called with l2t_data.lock held. + */ +static struct l2t_entry *alloc_l2e(struct l2t_data *d) +{ + struct l2t_entry *end, *e, **p; + + if (!atomic_read(&d->nfree)) + return NULL; + + /* there's definitely a free entry */ + for (e = d->rover, end = &d->l2tab[d->nentries]; e != end; ++e) + if (atomic_read(&e->refcnt) == 0) + goto found; + + for (e = &d->l2tab[1]; atomic_read(&e->refcnt); ++e) ; +found: + d->rover = e + 1; + atomic_dec(&d->nfree); + + /* + * The entry we found may be an inactive entry that is + * presently in the hash table. We need to remove it. + */ + if (e->state != L2T_STATE_UNUSED) { + int hash = arp_hash(e->addr, e->ifindex, d); + + for (p = &d->l2tab[hash].first; *p; p = &(*p)->next) + if (*p == e) { + *p = e->next; + break; + } + e->state = L2T_STATE_UNUSED; + } + return e; +} + +/* + * Called when an L2T entry has no more users. The entry is left in the hash + * table since it is likely to be reused but we also bump nfree to indicate + * that the entry can be reallocated for a different neighbor. We also drop + * the existing neighbor reference in case the neighbor is going away and is + * waiting on our reference. + * + * Because entries can be reallocated to other neighbors once their ref count + * drops to 0 we need to take the entry's lock to avoid races with a new + * incarnation. + */ +void t3_l2e_free(struct l2t_data *d, struct l2t_entry *e) +{ + spin_lock_bh(&e->lock); + if (atomic_read(&e->refcnt) == 0) { /* hasn't been recycled */ + if (e->neigh) { + neigh_release(e->neigh); + e->neigh = NULL; + } + } + spin_unlock_bh(&e->lock); + atomic_inc(&d->nfree); +} + +EXPORT_SYMBOL(t3_l2e_free); + +/* + * Update an L2T entry that was previously used for the same next hop as neigh. + * Must be called with softirqs disabled. + */ +static inline void reuse_entry(struct l2t_entry *e, struct neighbour *neigh) +{ + unsigned int nud_state; + + spin_lock(&e->lock); /* avoid race with t3_l2t_free */ + + if (neigh != e->neigh) + neigh_replace(e, neigh); + nud_state = neigh->nud_state; + if (memcmp(e->dmac, neigh->ha, sizeof(e->dmac)) || + !(nud_state & NUD_VALID)) + e->state = L2T_STATE_RESOLVING; + else if (nud_state & NUD_CONNECTED) + e->state = L2T_STATE_VALID; + else + e->state = L2T_STATE_STALE; + spin_unlock(&e->lock); +} + +struct l2t_entry *t3_l2t_get(struct t3cdev *cdev, struct dst_entry *dst, + struct net_device *dev, const void *daddr) +{ + struct l2t_entry *e = NULL; + struct neighbour *neigh; + struct port_info *p; + struct l2t_data *d; + int hash; + u32 addr; + int ifidx; + int smt_idx; + + rcu_read_lock(); + neigh = dst_neigh_lookup(dst, daddr); + if (!neigh) + goto done_rcu; + + addr = *(u32 *) neigh->primary_key; + ifidx = neigh->dev->ifindex; + + if (!dev) + dev = neigh->dev; + p = netdev_priv(dev); + smt_idx = p->port_id; + + d = L2DATA(cdev); + if (!d) + goto done_rcu; + + hash = arp_hash(addr, ifidx, d); + + write_lock_bh(&d->lock); + for (e = d->l2tab[hash].first; e; e = e->next) + if (e->addr == addr && e->ifindex == ifidx && + e->smt_idx == smt_idx) { + l2t_hold(d, e); + if (atomic_read(&e->refcnt) == 1) + reuse_entry(e, neigh); + goto done_unlock; + } + + /* Need to allocate a new entry */ + e = alloc_l2e(d); + if (e) { + spin_lock(&e->lock); /* avoid race with t3_l2t_free */ + e->next = d->l2tab[hash].first; + d->l2tab[hash].first = e; + e->state = L2T_STATE_RESOLVING; + e->addr = addr; + e->ifindex = ifidx; + e->smt_idx = smt_idx; + atomic_set(&e->refcnt, 1); + neigh_replace(e, neigh); + if (is_vlan_dev(neigh->dev)) + e->vlan = vlan_dev_vlan_id(neigh->dev); + else + e->vlan = VLAN_NONE; + spin_unlock(&e->lock); + } +done_unlock: + write_unlock_bh(&d->lock); +done_rcu: + if (neigh) + neigh_release(neigh); + rcu_read_unlock(); + return e; +} + +EXPORT_SYMBOL(t3_l2t_get); + +/* + * Called when address resolution fails for an L2T entry to handle packets + * on the arpq head. If a packet specifies a failure handler it is invoked, + * otherwise the packets is sent to the offload device. + * + * XXX: maybe we should abandon the latter behavior and just require a failure + * handler. + */ +static void handle_failed_resolution(struct t3cdev *dev, struct sk_buff_head *arpq) +{ + struct sk_buff *skb, *tmp; + + skb_queue_walk_safe(arpq, skb, tmp) { + struct l2t_skb_cb *cb = L2T_SKB_CB(skb); + + __skb_unlink(skb, arpq); + if (cb->arp_failure_handler) + cb->arp_failure_handler(dev, skb); + else + cxgb3_ofld_send(dev, skb); + } +} + +/* + * Called when the host's ARP layer makes a change to some entry that is + * loaded into the HW L2 table. + */ +void t3_l2t_update(struct t3cdev *dev, struct neighbour *neigh) +{ + struct sk_buff_head arpq; + struct l2t_entry *e; + struct l2t_data *d = L2DATA(dev); + u32 addr = *(u32 *) neigh->primary_key; + int ifidx = neigh->dev->ifindex; + int hash = arp_hash(addr, ifidx, d); + + read_lock_bh(&d->lock); + for (e = d->l2tab[hash].first; e; e = e->next) + if (e->addr == addr && e->ifindex == ifidx) { + spin_lock(&e->lock); + goto found; + } + read_unlock_bh(&d->lock); + return; + +found: + __skb_queue_head_init(&arpq); + + read_unlock(&d->lock); + if (atomic_read(&e->refcnt)) { + if (neigh != e->neigh) + neigh_replace(e, neigh); + + if (e->state == L2T_STATE_RESOLVING) { + if (neigh->nud_state & NUD_FAILED) { + skb_queue_splice_init(&e->arpq, &arpq); + } else if (neigh->nud_state & (NUD_CONNECTED|NUD_STALE)) + setup_l2e_send_pending(dev, NULL, e); + } else { + e->state = neigh->nud_state & NUD_CONNECTED ? + L2T_STATE_VALID : L2T_STATE_STALE; + if (!ether_addr_equal(e->dmac, neigh->ha)) + setup_l2e_send_pending(dev, NULL, e); + } + } + spin_unlock_bh(&e->lock); + + if (!skb_queue_empty(&arpq)) + handle_failed_resolution(dev, &arpq); +} + +struct l2t_data *t3_init_l2t(unsigned int l2t_capacity) +{ + struct l2t_data *d; + int i; + + d = kvzalloc(struct_size(d, l2tab, l2t_capacity), GFP_KERNEL); + if (!d) + return NULL; + + d->nentries = l2t_capacity; + d->rover = &d->l2tab[1]; /* entry 0 is not used */ + atomic_set(&d->nfree, l2t_capacity - 1); + rwlock_init(&d->lock); + + for (i = 0; i < l2t_capacity; ++i) { + d->l2tab[i].idx = i; + d->l2tab[i].state = L2T_STATE_UNUSED; + __skb_queue_head_init(&d->l2tab[i].arpq); + spin_lock_init(&d->l2tab[i].lock); + atomic_set(&d->l2tab[i].refcnt, 0); + } + return d; +} diff --git a/drivers/net/ethernet/chelsio/cxgb3/l2t.h b/drivers/net/ethernet/chelsio/cxgb3/l2t.h new file mode 100644 index 000000000..ea75f2750 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/l2t.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef _CHELSIO_L2T_H +#define _CHELSIO_L2T_H + +#include <linux/spinlock.h> +#include "t3cdev.h" +#include <linux/atomic.h> + +enum { + L2T_STATE_VALID, /* entry is up to date */ + L2T_STATE_STALE, /* entry may be used but needs revalidation */ + L2T_STATE_RESOLVING, /* entry needs address resolution */ + L2T_STATE_UNUSED /* entry not in use */ +}; + +struct neighbour; +struct sk_buff; + +/* + * Each L2T entry plays multiple roles. First of all, it keeps state for the + * corresponding entry of the HW L2 table and maintains a queue of offload + * packets awaiting address resolution. Second, it is a node of a hash table + * chain, where the nodes of the chain are linked together through their next + * pointer. Finally, each node is a bucket of a hash table, pointing to the + * first element in its chain through its first pointer. + */ +struct l2t_entry { + u16 state; /* entry state */ + u16 idx; /* entry index */ + u32 addr; /* dest IP address */ + int ifindex; /* neighbor's net_device's ifindex */ + u16 smt_idx; /* SMT index */ + u16 vlan; /* VLAN TCI (id: bits 0-11, prio: 13-15 */ + struct neighbour *neigh; /* associated neighbour */ + struct l2t_entry *first; /* start of hash chain */ + struct l2t_entry *next; /* next l2t_entry on chain */ + struct sk_buff_head arpq; /* queue of packets awaiting resolution */ + spinlock_t lock; + atomic_t refcnt; /* entry reference count */ + u8 dmac[6]; /* neighbour's MAC address */ +}; + +struct l2t_data { + unsigned int nentries; /* number of entries */ + struct l2t_entry *rover; /* starting point for next allocation */ + atomic_t nfree; /* number of free entries */ + rwlock_t lock; + struct rcu_head rcu_head; /* to handle rcu cleanup */ + struct l2t_entry l2tab[]; +}; + +typedef void (*arp_failure_handler_func)(struct t3cdev * dev, + struct sk_buff * skb); + +/* + * Callback stored in an skb to handle address resolution failure. + */ +struct l2t_skb_cb { + arp_failure_handler_func arp_failure_handler; +}; + +#define L2T_SKB_CB(skb) ((struct l2t_skb_cb *)(skb)->cb) + +static inline void set_arp_failure_handler(struct sk_buff *skb, + arp_failure_handler_func hnd) +{ + L2T_SKB_CB(skb)->arp_failure_handler = hnd; +} + +/* + * Getting to the L2 data from an offload device. + */ +#define L2DATA(cdev) (rcu_dereference((cdev)->l2opt)) + +#define W_TCB_L2T_IX 0 +#define S_TCB_L2T_IX 7 +#define M_TCB_L2T_IX 0x7ffULL +#define V_TCB_L2T_IX(x) ((x) << S_TCB_L2T_IX) + +void t3_l2e_free(struct l2t_data *d, struct l2t_entry *e); +void t3_l2t_update(struct t3cdev *dev, struct neighbour *neigh); +struct l2t_entry *t3_l2t_get(struct t3cdev *cdev, struct dst_entry *dst, + struct net_device *dev, const void *daddr); +int t3_l2t_send_slow(struct t3cdev *dev, struct sk_buff *skb, + struct l2t_entry *e); +void t3_l2t_send_event(struct t3cdev *dev, struct l2t_entry *e); +struct l2t_data *t3_init_l2t(unsigned int l2t_capacity); + +int cxgb3_ofld_send(struct t3cdev *dev, struct sk_buff *skb); + +static inline int l2t_send(struct t3cdev *dev, struct sk_buff *skb, + struct l2t_entry *e) +{ + if (likely(e->state == L2T_STATE_VALID)) + return cxgb3_ofld_send(dev, skb); + return t3_l2t_send_slow(dev, skb, e); +} + +static inline void l2t_release(struct t3cdev *t, struct l2t_entry *e) +{ + struct l2t_data *d; + + rcu_read_lock(); + d = L2DATA(t); + + if (atomic_dec_and_test(&e->refcnt) && d) + t3_l2e_free(d, e); + + rcu_read_unlock(); +} + +static inline void l2t_hold(struct l2t_data *d, struct l2t_entry *e) +{ + if (d && atomic_add_return(1, &e->refcnt) == 1) /* 0 -> 1 transition */ + atomic_dec(&d->nfree); +} + +#endif diff --git a/drivers/net/ethernet/chelsio/cxgb3/mc5.c b/drivers/net/ethernet/chelsio/cxgb3/mc5.c new file mode 100644 index 000000000..338301b11 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/mc5.c @@ -0,0 +1,422 @@ +/* + * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "common.h" +#include "regs.h" + +enum { + IDT75P52100 = 4, + IDT75N43102 = 5 +}; + +/* DBGI command mode */ +enum { + DBGI_MODE_MBUS = 0, + DBGI_MODE_IDT52100 = 5 +}; + +/* IDT 75P52100 commands */ +#define IDT_CMD_READ 0 +#define IDT_CMD_WRITE 1 +#define IDT_CMD_SEARCH 2 +#define IDT_CMD_LEARN 3 + +/* IDT LAR register address and value for 144-bit mode (low 32 bits) */ +#define IDT_LAR_ADR0 0x180006 +#define IDT_LAR_MODE144 0xffff0000 + +/* IDT SCR and SSR addresses (low 32 bits) */ +#define IDT_SCR_ADR0 0x180000 +#define IDT_SSR0_ADR0 0x180002 +#define IDT_SSR1_ADR0 0x180004 + +/* IDT GMR base address (low 32 bits) */ +#define IDT_GMR_BASE_ADR0 0x180020 + +/* IDT data and mask array base addresses (low 32 bits) */ +#define IDT_DATARY_BASE_ADR0 0 +#define IDT_MSKARY_BASE_ADR0 0x80000 + +/* IDT 75N43102 commands */ +#define IDT4_CMD_SEARCH144 3 +#define IDT4_CMD_WRITE 4 +#define IDT4_CMD_READ 5 + +/* IDT 75N43102 SCR address (low 32 bits) */ +#define IDT4_SCR_ADR0 0x3 + +/* IDT 75N43102 GMR base addresses (low 32 bits) */ +#define IDT4_GMR_BASE0 0x10 +#define IDT4_GMR_BASE1 0x20 +#define IDT4_GMR_BASE2 0x30 + +/* IDT 75N43102 data and mask array base addresses (low 32 bits) */ +#define IDT4_DATARY_BASE_ADR0 0x1000000 +#define IDT4_MSKARY_BASE_ADR0 0x2000000 + +#define MAX_WRITE_ATTEMPTS 5 + +#define MAX_ROUTES 2048 + +/* + * Issue a command to the TCAM and wait for its completion. The address and + * any data required by the command must have been setup by the caller. + */ +static int mc5_cmd_write(struct adapter *adapter, u32 cmd) +{ + t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_CMD, cmd); + return t3_wait_op_done(adapter, A_MC5_DB_DBGI_RSP_STATUS, + F_DBGIRSPVALID, 1, MAX_WRITE_ATTEMPTS, 1); +} + +static inline void dbgi_wr_data3(struct adapter *adapter, u32 v1, u32 v2, + u32 v3) +{ + t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_DATA0, v1); + t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_DATA1, v2); + t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_DATA2, v3); +} + +/* + * Write data to the TCAM register at address (0, 0, addr_lo) using the TCAM + * command cmd. The data to be written must have been set up by the caller. + * Returns -1 on failure, 0 on success. + */ +static int mc5_write(struct adapter *adapter, u32 addr_lo, u32 cmd) +{ + t3_write_reg(adapter, A_MC5_DB_DBGI_REQ_ADDR0, addr_lo); + if (mc5_cmd_write(adapter, cmd) == 0) + return 0; + CH_ERR(adapter, "MC5 timeout writing to TCAM address 0x%x\n", + addr_lo); + return -1; +} + +static int init_mask_data_array(struct mc5 *mc5, u32 mask_array_base, + u32 data_array_base, u32 write_cmd, + int addr_shift) +{ + unsigned int i; + struct adapter *adap = mc5->adapter; + + /* + * We need the size of the TCAM data and mask arrays in terms of + * 72-bit entries. + */ + unsigned int size72 = mc5->tcam_size; + unsigned int server_base = t3_read_reg(adap, A_MC5_DB_SERVER_INDEX); + + if (mc5->mode == MC5_MODE_144_BIT) { + size72 *= 2; /* 1 144-bit entry is 2 72-bit entries */ + server_base *= 2; + } + + /* Clear the data array */ + dbgi_wr_data3(adap, 0, 0, 0); + for (i = 0; i < size72; i++) + if (mc5_write(adap, data_array_base + (i << addr_shift), + write_cmd)) + return -1; + + /* Initialize the mask array. */ + dbgi_wr_data3(adap, 0xffffffff, 0xffffffff, 0xff); + for (i = 0; i < size72; i++) { + if (i == server_base) /* entering server or routing region */ + t3_write_reg(adap, A_MC5_DB_DBGI_REQ_DATA0, + mc5->mode == MC5_MODE_144_BIT ? + 0xfffffff9 : 0xfffffffd); + if (mc5_write(adap, mask_array_base + (i << addr_shift), + write_cmd)) + return -1; + } + return 0; +} + +static int init_idt52100(struct mc5 *mc5) +{ + int i; + struct adapter *adap = mc5->adapter; + + t3_write_reg(adap, A_MC5_DB_RSP_LATENCY, + V_RDLAT(0x15) | V_LRNLAT(0x15) | V_SRCHLAT(0x15)); + t3_write_reg(adap, A_MC5_DB_PART_ID_INDEX, 2); + + /* + * Use GMRs 14-15 for ELOOKUP, GMRs 12-13 for SYN lookups, and + * GMRs 8-9 for ACK- and AOPEN searches. + */ + t3_write_reg(adap, A_MC5_DB_POPEN_DATA_WR_CMD, IDT_CMD_WRITE); + t3_write_reg(adap, A_MC5_DB_POPEN_MASK_WR_CMD, IDT_CMD_WRITE); + t3_write_reg(adap, A_MC5_DB_AOPEN_SRCH_CMD, IDT_CMD_SEARCH); + t3_write_reg(adap, A_MC5_DB_AOPEN_LRN_CMD, IDT_CMD_LEARN); + t3_write_reg(adap, A_MC5_DB_SYN_SRCH_CMD, IDT_CMD_SEARCH | 0x6000); + t3_write_reg(adap, A_MC5_DB_SYN_LRN_CMD, IDT_CMD_LEARN); + t3_write_reg(adap, A_MC5_DB_ACK_SRCH_CMD, IDT_CMD_SEARCH); + t3_write_reg(adap, A_MC5_DB_ACK_LRN_CMD, IDT_CMD_LEARN); + t3_write_reg(adap, A_MC5_DB_ILOOKUP_CMD, IDT_CMD_SEARCH); + t3_write_reg(adap, A_MC5_DB_ELOOKUP_CMD, IDT_CMD_SEARCH | 0x7000); + t3_write_reg(adap, A_MC5_DB_DATA_WRITE_CMD, IDT_CMD_WRITE); + t3_write_reg(adap, A_MC5_DB_DATA_READ_CMD, IDT_CMD_READ); + + /* Set DBGI command mode for IDT TCAM. */ + t3_write_reg(adap, A_MC5_DB_DBGI_CONFIG, DBGI_MODE_IDT52100); + + /* Set up LAR */ + dbgi_wr_data3(adap, IDT_LAR_MODE144, 0, 0); + if (mc5_write(adap, IDT_LAR_ADR0, IDT_CMD_WRITE)) + goto err; + + /* Set up SSRs */ + dbgi_wr_data3(adap, 0xffffffff, 0xffffffff, 0); + if (mc5_write(adap, IDT_SSR0_ADR0, IDT_CMD_WRITE) || + mc5_write(adap, IDT_SSR1_ADR0, IDT_CMD_WRITE)) + goto err; + + /* Set up GMRs */ + for (i = 0; i < 32; ++i) { + if (i >= 12 && i < 15) + dbgi_wr_data3(adap, 0xfffffff9, 0xffffffff, 0xff); + else if (i == 15) + dbgi_wr_data3(adap, 0xfffffff9, 0xffff8007, 0xff); + else + dbgi_wr_data3(adap, 0xffffffff, 0xffffffff, 0xff); + + if (mc5_write(adap, IDT_GMR_BASE_ADR0 + i, IDT_CMD_WRITE)) + goto err; + } + + /* Set up SCR */ + dbgi_wr_data3(adap, 1, 0, 0); + if (mc5_write(adap, IDT_SCR_ADR0, IDT_CMD_WRITE)) + goto err; + + return init_mask_data_array(mc5, IDT_MSKARY_BASE_ADR0, + IDT_DATARY_BASE_ADR0, IDT_CMD_WRITE, 0); +err: + return -EIO; +} + +static int init_idt43102(struct mc5 *mc5) +{ + int i; + struct adapter *adap = mc5->adapter; + + t3_write_reg(adap, A_MC5_DB_RSP_LATENCY, + adap->params.rev == 0 ? V_RDLAT(0xd) | V_SRCHLAT(0x11) : + V_RDLAT(0xd) | V_SRCHLAT(0x12)); + + /* + * Use GMRs 24-25 for ELOOKUP, GMRs 20-21 for SYN lookups, and no mask + * for ACK- and AOPEN searches. + */ + t3_write_reg(adap, A_MC5_DB_POPEN_DATA_WR_CMD, IDT4_CMD_WRITE); + t3_write_reg(adap, A_MC5_DB_POPEN_MASK_WR_CMD, IDT4_CMD_WRITE); + t3_write_reg(adap, A_MC5_DB_AOPEN_SRCH_CMD, + IDT4_CMD_SEARCH144 | 0x3800); + t3_write_reg(adap, A_MC5_DB_SYN_SRCH_CMD, IDT4_CMD_SEARCH144); + t3_write_reg(adap, A_MC5_DB_ACK_SRCH_CMD, IDT4_CMD_SEARCH144 | 0x3800); + t3_write_reg(adap, A_MC5_DB_ILOOKUP_CMD, IDT4_CMD_SEARCH144 | 0x3800); + t3_write_reg(adap, A_MC5_DB_ELOOKUP_CMD, IDT4_CMD_SEARCH144 | 0x800); + t3_write_reg(adap, A_MC5_DB_DATA_WRITE_CMD, IDT4_CMD_WRITE); + t3_write_reg(adap, A_MC5_DB_DATA_READ_CMD, IDT4_CMD_READ); + + t3_write_reg(adap, A_MC5_DB_PART_ID_INDEX, 3); + + /* Set DBGI command mode for IDT TCAM. */ + t3_write_reg(adap, A_MC5_DB_DBGI_CONFIG, DBGI_MODE_IDT52100); + + /* Set up GMRs */ + dbgi_wr_data3(adap, 0xffffffff, 0xffffffff, 0xff); + for (i = 0; i < 7; ++i) + if (mc5_write(adap, IDT4_GMR_BASE0 + i, IDT4_CMD_WRITE)) + goto err; + + for (i = 0; i < 4; ++i) + if (mc5_write(adap, IDT4_GMR_BASE2 + i, IDT4_CMD_WRITE)) + goto err; + + dbgi_wr_data3(adap, 0xfffffff9, 0xffffffff, 0xff); + if (mc5_write(adap, IDT4_GMR_BASE1, IDT4_CMD_WRITE) || + mc5_write(adap, IDT4_GMR_BASE1 + 1, IDT4_CMD_WRITE) || + mc5_write(adap, IDT4_GMR_BASE1 + 4, IDT4_CMD_WRITE)) + goto err; + + dbgi_wr_data3(adap, 0xfffffff9, 0xffff8007, 0xff); + if (mc5_write(adap, IDT4_GMR_BASE1 + 5, IDT4_CMD_WRITE)) + goto err; + + /* Set up SCR */ + dbgi_wr_data3(adap, 0xf0000000, 0, 0); + if (mc5_write(adap, IDT4_SCR_ADR0, IDT4_CMD_WRITE)) + goto err; + + return init_mask_data_array(mc5, IDT4_MSKARY_BASE_ADR0, + IDT4_DATARY_BASE_ADR0, IDT4_CMD_WRITE, 1); +err: + return -EIO; +} + +/* Put MC5 in DBGI mode. */ +static inline void mc5_dbgi_mode_enable(const struct mc5 *mc5) +{ + t3_write_reg(mc5->adapter, A_MC5_DB_CONFIG, + V_TMMODE(mc5->mode == MC5_MODE_72_BIT) | F_DBGIEN); +} + +/* Put MC5 in M-Bus mode. */ +static void mc5_dbgi_mode_disable(const struct mc5 *mc5) +{ + t3_write_reg(mc5->adapter, A_MC5_DB_CONFIG, + V_TMMODE(mc5->mode == MC5_MODE_72_BIT) | + V_COMPEN(mc5->mode == MC5_MODE_72_BIT) | + V_PRTYEN(mc5->parity_enabled) | F_MBUSEN); +} + +/* + * Initialization that requires the OS and protocol layers to already + * be initialized goes here. + */ +int t3_mc5_init(struct mc5 *mc5, unsigned int nservers, unsigned int nfilters, + unsigned int nroutes) +{ + u32 cfg; + int err; + unsigned int tcam_size = mc5->tcam_size; + struct adapter *adap = mc5->adapter; + + if (!tcam_size) + return 0; + + if (nroutes > MAX_ROUTES || nroutes + nservers + nfilters > tcam_size) + return -EINVAL; + + /* Reset the TCAM */ + cfg = t3_read_reg(adap, A_MC5_DB_CONFIG) & ~F_TMMODE; + cfg |= V_TMMODE(mc5->mode == MC5_MODE_72_BIT) | F_TMRST; + t3_write_reg(adap, A_MC5_DB_CONFIG, cfg); + if (t3_wait_op_done(adap, A_MC5_DB_CONFIG, F_TMRDY, 1, 500, 0)) { + CH_ERR(adap, "TCAM reset timed out\n"); + return -1; + } + + t3_write_reg(adap, A_MC5_DB_ROUTING_TABLE_INDEX, tcam_size - nroutes); + t3_write_reg(adap, A_MC5_DB_FILTER_TABLE, + tcam_size - nroutes - nfilters); + t3_write_reg(adap, A_MC5_DB_SERVER_INDEX, + tcam_size - nroutes - nfilters - nservers); + + mc5->parity_enabled = 1; + + /* All the TCAM addresses we access have only the low 32 bits non 0 */ + t3_write_reg(adap, A_MC5_DB_DBGI_REQ_ADDR1, 0); + t3_write_reg(adap, A_MC5_DB_DBGI_REQ_ADDR2, 0); + + mc5_dbgi_mode_enable(mc5); + + switch (mc5->part_type) { + case IDT75P52100: + err = init_idt52100(mc5); + break; + case IDT75N43102: + err = init_idt43102(mc5); + break; + default: + CH_ERR(adap, "Unsupported TCAM type %d\n", mc5->part_type); + err = -EINVAL; + break; + } + + mc5_dbgi_mode_disable(mc5); + return err; +} + + +#define MC5_INT_FATAL (F_PARITYERR | F_REQQPARERR | F_DISPQPARERR) + +/* + * MC5 interrupt handler + */ +void t3_mc5_intr_handler(struct mc5 *mc5) +{ + struct adapter *adap = mc5->adapter; + u32 cause = t3_read_reg(adap, A_MC5_DB_INT_CAUSE); + + if ((cause & F_PARITYERR) && mc5->parity_enabled) { + CH_ALERT(adap, "MC5 parity error\n"); + mc5->stats.parity_err++; + } + + if (cause & F_REQQPARERR) { + CH_ALERT(adap, "MC5 request queue parity error\n"); + mc5->stats.reqq_parity_err++; + } + + if (cause & F_DISPQPARERR) { + CH_ALERT(adap, "MC5 dispatch queue parity error\n"); + mc5->stats.dispq_parity_err++; + } + + if (cause & F_ACTRGNFULL) + mc5->stats.active_rgn_full++; + if (cause & F_NFASRCHFAIL) + mc5->stats.nfa_srch_err++; + if (cause & F_UNKNOWNCMD) + mc5->stats.unknown_cmd++; + if (cause & F_DELACTEMPTY) + mc5->stats.del_act_empty++; + if (cause & MC5_INT_FATAL) + t3_fatal_err(adap); + + t3_write_reg(adap, A_MC5_DB_INT_CAUSE, cause); +} + +void t3_mc5_prep(struct adapter *adapter, struct mc5 *mc5, int mode) +{ +#define K * 1024 + + static unsigned int tcam_part_size[] = { /* in K 72-bit entries */ + 64 K, 128 K, 256 K, 32 K + }; + +#undef K + + u32 cfg = t3_read_reg(adapter, A_MC5_DB_CONFIG); + + mc5->adapter = adapter; + mc5->mode = (unsigned char)mode; + mc5->part_type = (unsigned char)G_TMTYPE(cfg); + if (cfg & F_TMTYPEHI) + mc5->part_type |= 4; + + mc5->tcam_size = tcam_part_size[G_TMPARTSIZE(cfg)]; + if (mode == MC5_MODE_144_BIT) + mc5->tcam_size /= 2; +} diff --git a/drivers/net/ethernet/chelsio/cxgb3/regs.h b/drivers/net/ethernet/chelsio/cxgb3/regs.h new file mode 100644 index 000000000..174eb4510 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/regs.h @@ -0,0 +1,2564 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#define A_SG_CONTROL 0x0 + +#define S_CONGMODE 29 +#define V_CONGMODE(x) ((x) << S_CONGMODE) +#define F_CONGMODE V_CONGMODE(1U) + +#define S_TNLFLMODE 28 +#define V_TNLFLMODE(x) ((x) << S_TNLFLMODE) +#define F_TNLFLMODE V_TNLFLMODE(1U) + +#define S_FATLPERREN 27 +#define V_FATLPERREN(x) ((x) << S_FATLPERREN) +#define F_FATLPERREN V_FATLPERREN(1U) + +#define S_DROPPKT 20 +#define V_DROPPKT(x) ((x) << S_DROPPKT) +#define F_DROPPKT V_DROPPKT(1U) + +#define S_EGRGENCTRL 19 +#define V_EGRGENCTRL(x) ((x) << S_EGRGENCTRL) +#define F_EGRGENCTRL V_EGRGENCTRL(1U) + +#define S_USERSPACESIZE 14 +#define M_USERSPACESIZE 0x1f +#define V_USERSPACESIZE(x) ((x) << S_USERSPACESIZE) + +#define S_HOSTPAGESIZE 11 +#define M_HOSTPAGESIZE 0x7 +#define V_HOSTPAGESIZE(x) ((x) << S_HOSTPAGESIZE) + +#define S_FLMODE 9 +#define V_FLMODE(x) ((x) << S_FLMODE) +#define F_FLMODE V_FLMODE(1U) + +#define S_PKTSHIFT 6 +#define M_PKTSHIFT 0x7 +#define V_PKTSHIFT(x) ((x) << S_PKTSHIFT) + +#define S_ONEINTMULTQ 5 +#define V_ONEINTMULTQ(x) ((x) << S_ONEINTMULTQ) +#define F_ONEINTMULTQ V_ONEINTMULTQ(1U) + +#define S_BIGENDIANINGRESS 2 +#define V_BIGENDIANINGRESS(x) ((x) << S_BIGENDIANINGRESS) +#define F_BIGENDIANINGRESS V_BIGENDIANINGRESS(1U) + +#define S_ISCSICOALESCING 1 +#define V_ISCSICOALESCING(x) ((x) << S_ISCSICOALESCING) +#define F_ISCSICOALESCING V_ISCSICOALESCING(1U) + +#define S_GLOBALENABLE 0 +#define V_GLOBALENABLE(x) ((x) << S_GLOBALENABLE) +#define F_GLOBALENABLE V_GLOBALENABLE(1U) + +#define S_AVOIDCQOVFL 24 +#define V_AVOIDCQOVFL(x) ((x) << S_AVOIDCQOVFL) +#define F_AVOIDCQOVFL V_AVOIDCQOVFL(1U) + +#define S_OPTONEINTMULTQ 23 +#define V_OPTONEINTMULTQ(x) ((x) << S_OPTONEINTMULTQ) +#define F_OPTONEINTMULTQ V_OPTONEINTMULTQ(1U) + +#define S_CQCRDTCTRL 22 +#define V_CQCRDTCTRL(x) ((x) << S_CQCRDTCTRL) +#define F_CQCRDTCTRL V_CQCRDTCTRL(1U) + +#define A_SG_KDOORBELL 0x4 + +#define S_SELEGRCNTX 31 +#define V_SELEGRCNTX(x) ((x) << S_SELEGRCNTX) +#define F_SELEGRCNTX V_SELEGRCNTX(1U) + +#define S_EGRCNTX 0 +#define M_EGRCNTX 0xffff +#define V_EGRCNTX(x) ((x) << S_EGRCNTX) + +#define A_SG_GTS 0x8 + +#define S_RSPQ 29 +#define M_RSPQ 0x7 +#define V_RSPQ(x) ((x) << S_RSPQ) +#define G_RSPQ(x) (((x) >> S_RSPQ) & M_RSPQ) + +#define S_NEWTIMER 16 +#define M_NEWTIMER 0x1fff +#define V_NEWTIMER(x) ((x) << S_NEWTIMER) + +#define S_NEWINDEX 0 +#define M_NEWINDEX 0xffff +#define V_NEWINDEX(x) ((x) << S_NEWINDEX) + +#define A_SG_CONTEXT_CMD 0xc + +#define S_CONTEXT_CMD_OPCODE 28 +#define M_CONTEXT_CMD_OPCODE 0xf +#define V_CONTEXT_CMD_OPCODE(x) ((x) << S_CONTEXT_CMD_OPCODE) + +#define S_CONTEXT_CMD_BUSY 27 +#define V_CONTEXT_CMD_BUSY(x) ((x) << S_CONTEXT_CMD_BUSY) +#define F_CONTEXT_CMD_BUSY V_CONTEXT_CMD_BUSY(1U) + +#define S_CQ_CREDIT 20 + +#define M_CQ_CREDIT 0x7f + +#define V_CQ_CREDIT(x) ((x) << S_CQ_CREDIT) + +#define G_CQ_CREDIT(x) (((x) >> S_CQ_CREDIT) & M_CQ_CREDIT) + +#define S_CQ 19 + +#define V_CQ(x) ((x) << S_CQ) +#define F_CQ V_CQ(1U) + +#define S_RESPONSEQ 18 +#define V_RESPONSEQ(x) ((x) << S_RESPONSEQ) +#define F_RESPONSEQ V_RESPONSEQ(1U) + +#define S_EGRESS 17 +#define V_EGRESS(x) ((x) << S_EGRESS) +#define F_EGRESS V_EGRESS(1U) + +#define S_FREELIST 16 +#define V_FREELIST(x) ((x) << S_FREELIST) +#define F_FREELIST V_FREELIST(1U) + +#define S_CONTEXT 0 +#define M_CONTEXT 0xffff +#define V_CONTEXT(x) ((x) << S_CONTEXT) + +#define G_CONTEXT(x) (((x) >> S_CONTEXT) & M_CONTEXT) + +#define A_SG_CONTEXT_DATA0 0x10 + +#define A_SG_CONTEXT_DATA1 0x14 + +#define A_SG_CONTEXT_DATA2 0x18 + +#define A_SG_CONTEXT_DATA3 0x1c + +#define A_SG_CONTEXT_MASK0 0x20 + +#define A_SG_CONTEXT_MASK1 0x24 + +#define A_SG_CONTEXT_MASK2 0x28 + +#define A_SG_CONTEXT_MASK3 0x2c + +#define A_SG_RSPQ_CREDIT_RETURN 0x30 + +#define S_CREDITS 0 +#define M_CREDITS 0xffff +#define V_CREDITS(x) ((x) << S_CREDITS) + +#define A_SG_DATA_INTR 0x34 + +#define S_ERRINTR 31 +#define V_ERRINTR(x) ((x) << S_ERRINTR) +#define F_ERRINTR V_ERRINTR(1U) + +#define A_SG_HI_DRB_HI_THRSH 0x38 + +#define A_SG_HI_DRB_LO_THRSH 0x3c + +#define A_SG_LO_DRB_HI_THRSH 0x40 + +#define A_SG_LO_DRB_LO_THRSH 0x44 + +#define A_SG_RSPQ_FL_STATUS 0x4c + +#define S_RSPQ0DISABLED 8 + +#define S_FL0EMPTY 16 +#define V_FL0EMPTY(x) ((x) << S_FL0EMPTY) +#define F_FL0EMPTY V_FL0EMPTY(1U) + +#define A_SG_EGR_RCQ_DRB_THRSH 0x54 + +#define S_HIRCQDRBTHRSH 16 +#define M_HIRCQDRBTHRSH 0x7ff +#define V_HIRCQDRBTHRSH(x) ((x) << S_HIRCQDRBTHRSH) + +#define S_LORCQDRBTHRSH 0 +#define M_LORCQDRBTHRSH 0x7ff +#define V_LORCQDRBTHRSH(x) ((x) << S_LORCQDRBTHRSH) + +#define A_SG_EGR_CNTX_BADDR 0x58 + +#define A_SG_INT_CAUSE 0x5c + +#define S_HIRCQPARITYERROR 31 +#define V_HIRCQPARITYERROR(x) ((x) << S_HIRCQPARITYERROR) +#define F_HIRCQPARITYERROR V_HIRCQPARITYERROR(1U) + +#define S_LORCQPARITYERROR 30 +#define V_LORCQPARITYERROR(x) ((x) << S_LORCQPARITYERROR) +#define F_LORCQPARITYERROR V_LORCQPARITYERROR(1U) + +#define S_HIDRBPARITYERROR 29 +#define V_HIDRBPARITYERROR(x) ((x) << S_HIDRBPARITYERROR) +#define F_HIDRBPARITYERROR V_HIDRBPARITYERROR(1U) + +#define S_LODRBPARITYERROR 28 +#define V_LODRBPARITYERROR(x) ((x) << S_LODRBPARITYERROR) +#define F_LODRBPARITYERROR V_LODRBPARITYERROR(1U) + +#define S_FLPARITYERROR 22 +#define M_FLPARITYERROR 0x3f +#define V_FLPARITYERROR(x) ((x) << S_FLPARITYERROR) +#define G_FLPARITYERROR(x) (((x) >> S_FLPARITYERROR) & M_FLPARITYERROR) + +#define S_ITPARITYERROR 20 +#define M_ITPARITYERROR 0x3 +#define V_ITPARITYERROR(x) ((x) << S_ITPARITYERROR) +#define G_ITPARITYERROR(x) (((x) >> S_ITPARITYERROR) & M_ITPARITYERROR) + +#define S_IRPARITYERROR 19 +#define V_IRPARITYERROR(x) ((x) << S_IRPARITYERROR) +#define F_IRPARITYERROR V_IRPARITYERROR(1U) + +#define S_RCPARITYERROR 18 +#define V_RCPARITYERROR(x) ((x) << S_RCPARITYERROR) +#define F_RCPARITYERROR V_RCPARITYERROR(1U) + +#define S_OCPARITYERROR 17 +#define V_OCPARITYERROR(x) ((x) << S_OCPARITYERROR) +#define F_OCPARITYERROR V_OCPARITYERROR(1U) + +#define S_CPPARITYERROR 16 +#define V_CPPARITYERROR(x) ((x) << S_CPPARITYERROR) +#define F_CPPARITYERROR V_CPPARITYERROR(1U) + +#define S_R_REQ_FRAMINGERROR 15 +#define V_R_REQ_FRAMINGERROR(x) ((x) << S_R_REQ_FRAMINGERROR) +#define F_R_REQ_FRAMINGERROR V_R_REQ_FRAMINGERROR(1U) + +#define S_UC_REQ_FRAMINGERROR 14 +#define V_UC_REQ_FRAMINGERROR(x) ((x) << S_UC_REQ_FRAMINGERROR) +#define F_UC_REQ_FRAMINGERROR V_UC_REQ_FRAMINGERROR(1U) + +#define S_HICTLDRBDROPERR 13 +#define V_HICTLDRBDROPERR(x) ((x) << S_HICTLDRBDROPERR) +#define F_HICTLDRBDROPERR V_HICTLDRBDROPERR(1U) + +#define S_LOCTLDRBDROPERR 12 +#define V_LOCTLDRBDROPERR(x) ((x) << S_LOCTLDRBDROPERR) +#define F_LOCTLDRBDROPERR V_LOCTLDRBDROPERR(1U) + +#define S_HIPIODRBDROPERR 11 +#define V_HIPIODRBDROPERR(x) ((x) << S_HIPIODRBDROPERR) +#define F_HIPIODRBDROPERR V_HIPIODRBDROPERR(1U) + +#define S_LOPIODRBDROPERR 10 +#define V_LOPIODRBDROPERR(x) ((x) << S_LOPIODRBDROPERR) +#define F_LOPIODRBDROPERR V_LOPIODRBDROPERR(1U) + +#define S_HIPRIORITYDBFULL 7 +#define V_HIPRIORITYDBFULL(x) ((x) << S_HIPRIORITYDBFULL) +#define F_HIPRIORITYDBFULL V_HIPRIORITYDBFULL(1U) + +#define S_HIPRIORITYDBEMPTY 6 +#define V_HIPRIORITYDBEMPTY(x) ((x) << S_HIPRIORITYDBEMPTY) +#define F_HIPRIORITYDBEMPTY V_HIPRIORITYDBEMPTY(1U) + +#define S_LOPRIORITYDBFULL 5 +#define V_LOPRIORITYDBFULL(x) ((x) << S_LOPRIORITYDBFULL) +#define F_LOPRIORITYDBFULL V_LOPRIORITYDBFULL(1U) + +#define S_LOPRIORITYDBEMPTY 4 +#define V_LOPRIORITYDBEMPTY(x) ((x) << S_LOPRIORITYDBEMPTY) +#define F_LOPRIORITYDBEMPTY V_LOPRIORITYDBEMPTY(1U) + +#define S_RSPQDISABLED 3 +#define V_RSPQDISABLED(x) ((x) << S_RSPQDISABLED) +#define F_RSPQDISABLED V_RSPQDISABLED(1U) + +#define S_RSPQCREDITOVERFOW 2 +#define V_RSPQCREDITOVERFOW(x) ((x) << S_RSPQCREDITOVERFOW) +#define F_RSPQCREDITOVERFOW V_RSPQCREDITOVERFOW(1U) + +#define S_FLEMPTY 1 +#define V_FLEMPTY(x) ((x) << S_FLEMPTY) +#define F_FLEMPTY V_FLEMPTY(1U) + +#define A_SG_INT_ENABLE 0x60 + +#define A_SG_CMDQ_CREDIT_TH 0x64 + +#define S_TIMEOUT 8 +#define M_TIMEOUT 0xffffff +#define V_TIMEOUT(x) ((x) << S_TIMEOUT) + +#define S_THRESHOLD 0 +#define M_THRESHOLD 0xff +#define V_THRESHOLD(x) ((x) << S_THRESHOLD) + +#define A_SG_TIMER_TICK 0x68 + +#define A_SG_CQ_CONTEXT_BADDR 0x6c + +#define A_SG_OCO_BASE 0x70 + +#define S_BASE1 16 +#define M_BASE1 0xffff +#define V_BASE1(x) ((x) << S_BASE1) + +#define A_SG_DRB_PRI_THRESH 0x74 + +#define A_PCIX_INT_ENABLE 0x80 + +#define S_MSIXPARERR 22 +#define M_MSIXPARERR 0x7 + +#define V_MSIXPARERR(x) ((x) << S_MSIXPARERR) + +#define S_CFPARERR 18 +#define M_CFPARERR 0xf + +#define V_CFPARERR(x) ((x) << S_CFPARERR) + +#define S_RFPARERR 14 +#define M_RFPARERR 0xf + +#define V_RFPARERR(x) ((x) << S_RFPARERR) + +#define S_WFPARERR 12 +#define M_WFPARERR 0x3 + +#define V_WFPARERR(x) ((x) << S_WFPARERR) + +#define S_PIOPARERR 11 +#define V_PIOPARERR(x) ((x) << S_PIOPARERR) +#define F_PIOPARERR V_PIOPARERR(1U) + +#define S_DETUNCECCERR 10 +#define V_DETUNCECCERR(x) ((x) << S_DETUNCECCERR) +#define F_DETUNCECCERR V_DETUNCECCERR(1U) + +#define S_DETCORECCERR 9 +#define V_DETCORECCERR(x) ((x) << S_DETCORECCERR) +#define F_DETCORECCERR V_DETCORECCERR(1U) + +#define S_RCVSPLCMPERR 8 +#define V_RCVSPLCMPERR(x) ((x) << S_RCVSPLCMPERR) +#define F_RCVSPLCMPERR V_RCVSPLCMPERR(1U) + +#define S_UNXSPLCMP 7 +#define V_UNXSPLCMP(x) ((x) << S_UNXSPLCMP) +#define F_UNXSPLCMP V_UNXSPLCMP(1U) + +#define S_SPLCMPDIS 6 +#define V_SPLCMPDIS(x) ((x) << S_SPLCMPDIS) +#define F_SPLCMPDIS V_SPLCMPDIS(1U) + +#define S_DETPARERR 5 +#define V_DETPARERR(x) ((x) << S_DETPARERR) +#define F_DETPARERR V_DETPARERR(1U) + +#define S_SIGSYSERR 4 +#define V_SIGSYSERR(x) ((x) << S_SIGSYSERR) +#define F_SIGSYSERR V_SIGSYSERR(1U) + +#define S_RCVMSTABT 3 +#define V_RCVMSTABT(x) ((x) << S_RCVMSTABT) +#define F_RCVMSTABT V_RCVMSTABT(1U) + +#define S_RCVTARABT 2 +#define V_RCVTARABT(x) ((x) << S_RCVTARABT) +#define F_RCVTARABT V_RCVTARABT(1U) + +#define S_SIGTARABT 1 +#define V_SIGTARABT(x) ((x) << S_SIGTARABT) +#define F_SIGTARABT V_SIGTARABT(1U) + +#define S_MSTDETPARERR 0 +#define V_MSTDETPARERR(x) ((x) << S_MSTDETPARERR) +#define F_MSTDETPARERR V_MSTDETPARERR(1U) + +#define A_PCIX_INT_CAUSE 0x84 + +#define A_PCIX_CFG 0x88 + +#define S_DMASTOPEN 19 +#define V_DMASTOPEN(x) ((x) << S_DMASTOPEN) +#define F_DMASTOPEN V_DMASTOPEN(1U) + +#define S_CLIDECEN 18 +#define V_CLIDECEN(x) ((x) << S_CLIDECEN) +#define F_CLIDECEN V_CLIDECEN(1U) + +#define A_PCIX_MODE 0x8c + +#define S_PCLKRANGE 6 +#define M_PCLKRANGE 0x3 +#define V_PCLKRANGE(x) ((x) << S_PCLKRANGE) +#define G_PCLKRANGE(x) (((x) >> S_PCLKRANGE) & M_PCLKRANGE) + +#define S_PCIXINITPAT 2 +#define M_PCIXINITPAT 0xf +#define V_PCIXINITPAT(x) ((x) << S_PCIXINITPAT) +#define G_PCIXINITPAT(x) (((x) >> S_PCIXINITPAT) & M_PCIXINITPAT) + +#define S_64BIT 0 +#define V_64BIT(x) ((x) << S_64BIT) +#define F_64BIT V_64BIT(1U) + +#define A_PCIE_INT_ENABLE 0x80 + +#define S_BISTERR 15 +#define M_BISTERR 0xff + +#define V_BISTERR(x) ((x) << S_BISTERR) + +#define S_TXPARERR 18 +#define V_TXPARERR(x) ((x) << S_TXPARERR) +#define F_TXPARERR V_TXPARERR(1U) + +#define S_RXPARERR 17 +#define V_RXPARERR(x) ((x) << S_RXPARERR) +#define F_RXPARERR V_RXPARERR(1U) + +#define S_RETRYLUTPARERR 16 +#define V_RETRYLUTPARERR(x) ((x) << S_RETRYLUTPARERR) +#define F_RETRYLUTPARERR V_RETRYLUTPARERR(1U) + +#define S_RETRYBUFPARERR 15 +#define V_RETRYBUFPARERR(x) ((x) << S_RETRYBUFPARERR) +#define F_RETRYBUFPARERR V_RETRYBUFPARERR(1U) + +#define S_PCIE_MSIXPARERR 12 +#define M_PCIE_MSIXPARERR 0x7 + +#define V_PCIE_MSIXPARERR(x) ((x) << S_PCIE_MSIXPARERR) + +#define S_PCIE_CFPARERR 11 +#define V_PCIE_CFPARERR(x) ((x) << S_PCIE_CFPARERR) +#define F_PCIE_CFPARERR V_PCIE_CFPARERR(1U) + +#define S_PCIE_RFPARERR 10 +#define V_PCIE_RFPARERR(x) ((x) << S_PCIE_RFPARERR) +#define F_PCIE_RFPARERR V_PCIE_RFPARERR(1U) + +#define S_PCIE_WFPARERR 9 +#define V_PCIE_WFPARERR(x) ((x) << S_PCIE_WFPARERR) +#define F_PCIE_WFPARERR V_PCIE_WFPARERR(1U) + +#define S_PCIE_PIOPARERR 8 +#define V_PCIE_PIOPARERR(x) ((x) << S_PCIE_PIOPARERR) +#define F_PCIE_PIOPARERR V_PCIE_PIOPARERR(1U) + +#define S_UNXSPLCPLERRC 7 +#define V_UNXSPLCPLERRC(x) ((x) << S_UNXSPLCPLERRC) +#define F_UNXSPLCPLERRC V_UNXSPLCPLERRC(1U) + +#define S_UNXSPLCPLERRR 6 +#define V_UNXSPLCPLERRR(x) ((x) << S_UNXSPLCPLERRR) +#define F_UNXSPLCPLERRR V_UNXSPLCPLERRR(1U) + +#define S_PEXERR 0 +#define V_PEXERR(x) ((x) << S_PEXERR) +#define F_PEXERR V_PEXERR(1U) + +#define A_PCIE_INT_CAUSE 0x84 + +#define S_PCIE_DMASTOPEN 24 +#define V_PCIE_DMASTOPEN(x) ((x) << S_PCIE_DMASTOPEN) +#define F_PCIE_DMASTOPEN V_PCIE_DMASTOPEN(1U) + +#define A_PCIE_CFG 0x88 + +#define S_ENABLELINKDWNDRST 21 +#define V_ENABLELINKDWNDRST(x) ((x) << S_ENABLELINKDWNDRST) +#define F_ENABLELINKDWNDRST V_ENABLELINKDWNDRST(1U) + +#define S_ENABLELINKDOWNRST 20 +#define V_ENABLELINKDOWNRST(x) ((x) << S_ENABLELINKDOWNRST) +#define F_ENABLELINKDOWNRST V_ENABLELINKDOWNRST(1U) + +#define S_PCIE_CLIDECEN 16 +#define V_PCIE_CLIDECEN(x) ((x) << S_PCIE_CLIDECEN) +#define F_PCIE_CLIDECEN V_PCIE_CLIDECEN(1U) + +#define S_CRSTWRMMODE 0 +#define V_CRSTWRMMODE(x) ((x) << S_CRSTWRMMODE) +#define F_CRSTWRMMODE V_CRSTWRMMODE(1U) + +#define A_PCIE_MODE 0x8c + +#define S_NUMFSTTRNSEQRX 10 +#define M_NUMFSTTRNSEQRX 0xff +#define V_NUMFSTTRNSEQRX(x) ((x) << S_NUMFSTTRNSEQRX) +#define G_NUMFSTTRNSEQRX(x) (((x) >> S_NUMFSTTRNSEQRX) & M_NUMFSTTRNSEQRX) + +#define A_PCIE_PEX_CTRL0 0x98 + +#define S_NUMFSTTRNSEQ 22 +#define M_NUMFSTTRNSEQ 0xff +#define V_NUMFSTTRNSEQ(x) ((x) << S_NUMFSTTRNSEQ) +#define G_NUMFSTTRNSEQ(x) (((x) >> S_NUMFSTTRNSEQ) & M_NUMFSTTRNSEQ) + +#define S_REPLAYLMT 2 +#define M_REPLAYLMT 0xfffff + +#define V_REPLAYLMT(x) ((x) << S_REPLAYLMT) + +#define A_PCIE_PEX_CTRL1 0x9c + +#define S_T3A_ACKLAT 0 +#define M_T3A_ACKLAT 0x7ff + +#define V_T3A_ACKLAT(x) ((x) << S_T3A_ACKLAT) + +#define S_ACKLAT 0 +#define M_ACKLAT 0x1fff + +#define V_ACKLAT(x) ((x) << S_ACKLAT) + +#define A_PCIE_PEX_ERR 0xa4 + +#define A_T3DBG_GPIO_EN 0xd0 + +#define S_GPIO11_OEN 27 +#define V_GPIO11_OEN(x) ((x) << S_GPIO11_OEN) +#define F_GPIO11_OEN V_GPIO11_OEN(1U) + +#define S_GPIO10_OEN 26 +#define V_GPIO10_OEN(x) ((x) << S_GPIO10_OEN) +#define F_GPIO10_OEN V_GPIO10_OEN(1U) + +#define S_GPIO7_OEN 23 +#define V_GPIO7_OEN(x) ((x) << S_GPIO7_OEN) +#define F_GPIO7_OEN V_GPIO7_OEN(1U) + +#define S_GPIO6_OEN 22 +#define V_GPIO6_OEN(x) ((x) << S_GPIO6_OEN) +#define F_GPIO6_OEN V_GPIO6_OEN(1U) + +#define S_GPIO5_OEN 21 +#define V_GPIO5_OEN(x) ((x) << S_GPIO5_OEN) +#define F_GPIO5_OEN V_GPIO5_OEN(1U) + +#define S_GPIO4_OEN 20 +#define V_GPIO4_OEN(x) ((x) << S_GPIO4_OEN) +#define F_GPIO4_OEN V_GPIO4_OEN(1U) + +#define S_GPIO2_OEN 18 +#define V_GPIO2_OEN(x) ((x) << S_GPIO2_OEN) +#define F_GPIO2_OEN V_GPIO2_OEN(1U) + +#define S_GPIO1_OEN 17 +#define V_GPIO1_OEN(x) ((x) << S_GPIO1_OEN) +#define F_GPIO1_OEN V_GPIO1_OEN(1U) + +#define S_GPIO0_OEN 16 +#define V_GPIO0_OEN(x) ((x) << S_GPIO0_OEN) +#define F_GPIO0_OEN V_GPIO0_OEN(1U) + +#define S_GPIO10_OUT_VAL 10 +#define V_GPIO10_OUT_VAL(x) ((x) << S_GPIO10_OUT_VAL) +#define F_GPIO10_OUT_VAL V_GPIO10_OUT_VAL(1U) + +#define S_GPIO7_OUT_VAL 7 +#define V_GPIO7_OUT_VAL(x) ((x) << S_GPIO7_OUT_VAL) +#define F_GPIO7_OUT_VAL V_GPIO7_OUT_VAL(1U) + +#define S_GPIO6_OUT_VAL 6 +#define V_GPIO6_OUT_VAL(x) ((x) << S_GPIO6_OUT_VAL) +#define F_GPIO6_OUT_VAL V_GPIO6_OUT_VAL(1U) + +#define S_GPIO5_OUT_VAL 5 +#define V_GPIO5_OUT_VAL(x) ((x) << S_GPIO5_OUT_VAL) +#define F_GPIO5_OUT_VAL V_GPIO5_OUT_VAL(1U) + +#define S_GPIO4_OUT_VAL 4 +#define V_GPIO4_OUT_VAL(x) ((x) << S_GPIO4_OUT_VAL) +#define F_GPIO4_OUT_VAL V_GPIO4_OUT_VAL(1U) + +#define S_GPIO2_OUT_VAL 2 +#define V_GPIO2_OUT_VAL(x) ((x) << S_GPIO2_OUT_VAL) +#define F_GPIO2_OUT_VAL V_GPIO2_OUT_VAL(1U) + +#define S_GPIO1_OUT_VAL 1 +#define V_GPIO1_OUT_VAL(x) ((x) << S_GPIO1_OUT_VAL) +#define F_GPIO1_OUT_VAL V_GPIO1_OUT_VAL(1U) + +#define S_GPIO0_OUT_VAL 0 +#define V_GPIO0_OUT_VAL(x) ((x) << S_GPIO0_OUT_VAL) +#define F_GPIO0_OUT_VAL V_GPIO0_OUT_VAL(1U) + +#define A_T3DBG_INT_ENABLE 0xd8 + +#define S_GPIO11 11 +#define V_GPIO11(x) ((x) << S_GPIO11) +#define F_GPIO11 V_GPIO11(1U) + +#define S_GPIO10 10 +#define V_GPIO10(x) ((x) << S_GPIO10) +#define F_GPIO10 V_GPIO10(1U) + +#define S_GPIO9 9 +#define V_GPIO9(x) ((x) << S_GPIO9) +#define F_GPIO9 V_GPIO9(1U) + +#define S_GPIO7 7 +#define V_GPIO7(x) ((x) << S_GPIO7) +#define F_GPIO7 V_GPIO7(1U) + +#define S_GPIO6 6 +#define V_GPIO6(x) ((x) << S_GPIO6) +#define F_GPIO6 V_GPIO6(1U) + +#define S_GPIO5 5 +#define V_GPIO5(x) ((x) << S_GPIO5) +#define F_GPIO5 V_GPIO5(1U) + +#define S_GPIO4 4 +#define V_GPIO4(x) ((x) << S_GPIO4) +#define F_GPIO4 V_GPIO4(1U) + +#define S_GPIO3 3 +#define V_GPIO3(x) ((x) << S_GPIO3) +#define F_GPIO3 V_GPIO3(1U) + +#define S_GPIO2 2 +#define V_GPIO2(x) ((x) << S_GPIO2) +#define F_GPIO2 V_GPIO2(1U) + +#define S_GPIO1 1 +#define V_GPIO1(x) ((x) << S_GPIO1) +#define F_GPIO1 V_GPIO1(1U) + +#define S_GPIO0 0 +#define V_GPIO0(x) ((x) << S_GPIO0) +#define F_GPIO0 V_GPIO0(1U) + +#define A_T3DBG_INT_CAUSE 0xdc + +#define A_T3DBG_GPIO_ACT_LOW 0xf0 + +#define MC7_PMRX_BASE_ADDR 0x100 + +#define A_MC7_CFG 0x100 + +#define S_IFEN 13 +#define V_IFEN(x) ((x) << S_IFEN) +#define F_IFEN V_IFEN(1U) + +#define S_TERM150 11 +#define V_TERM150(x) ((x) << S_TERM150) +#define F_TERM150 V_TERM150(1U) + +#define S_SLOW 10 +#define V_SLOW(x) ((x) << S_SLOW) +#define F_SLOW V_SLOW(1U) + +#define S_WIDTH 8 +#define M_WIDTH 0x3 +#define V_WIDTH(x) ((x) << S_WIDTH) +#define G_WIDTH(x) (((x) >> S_WIDTH) & M_WIDTH) + +#define S_BKS 6 +#define V_BKS(x) ((x) << S_BKS) +#define F_BKS V_BKS(1U) + +#define S_ORG 5 +#define V_ORG(x) ((x) << S_ORG) +#define F_ORG V_ORG(1U) + +#define S_DEN 2 +#define M_DEN 0x7 +#define V_DEN(x) ((x) << S_DEN) +#define G_DEN(x) (((x) >> S_DEN) & M_DEN) + +#define S_RDY 1 +#define V_RDY(x) ((x) << S_RDY) +#define F_RDY V_RDY(1U) + +#define S_CLKEN 0 +#define V_CLKEN(x) ((x) << S_CLKEN) +#define F_CLKEN V_CLKEN(1U) + +#define A_MC7_MODE 0x104 + +#define S_BUSY 31 +#define V_BUSY(x) ((x) << S_BUSY) +#define F_BUSY V_BUSY(1U) + +#define A_MC7_EXT_MODE1 0x108 + +#define A_MC7_EXT_MODE2 0x10c + +#define A_MC7_EXT_MODE3 0x110 + +#define A_MC7_PRE 0x114 + +#define A_MC7_REF 0x118 + +#define S_PREREFDIV 1 +#define M_PREREFDIV 0x3fff +#define V_PREREFDIV(x) ((x) << S_PREREFDIV) + +#define S_PERREFEN 0 +#define V_PERREFEN(x) ((x) << S_PERREFEN) +#define F_PERREFEN V_PERREFEN(1U) + +#define A_MC7_DLL 0x11c + +#define S_DLLENB 1 +#define V_DLLENB(x) ((x) << S_DLLENB) +#define F_DLLENB V_DLLENB(1U) + +#define S_DLLRST 0 +#define V_DLLRST(x) ((x) << S_DLLRST) +#define F_DLLRST V_DLLRST(1U) + +#define A_MC7_PARM 0x120 + +#define S_ACTTOPREDLY 26 +#define M_ACTTOPREDLY 0xf +#define V_ACTTOPREDLY(x) ((x) << S_ACTTOPREDLY) + +#define S_ACTTORDWRDLY 23 +#define M_ACTTORDWRDLY 0x7 +#define V_ACTTORDWRDLY(x) ((x) << S_ACTTORDWRDLY) + +#define S_PRECYC 20 +#define M_PRECYC 0x7 +#define V_PRECYC(x) ((x) << S_PRECYC) + +#define S_REFCYC 13 +#define M_REFCYC 0x7f +#define V_REFCYC(x) ((x) << S_REFCYC) + +#define S_BKCYC 8 +#define M_BKCYC 0x1f +#define V_BKCYC(x) ((x) << S_BKCYC) + +#define S_WRTORDDLY 4 +#define M_WRTORDDLY 0xf +#define V_WRTORDDLY(x) ((x) << S_WRTORDDLY) + +#define S_RDTOWRDLY 0 +#define M_RDTOWRDLY 0xf +#define V_RDTOWRDLY(x) ((x) << S_RDTOWRDLY) + +#define A_MC7_CAL 0x128 + +#define S_CAL_FAULT 30 +#define V_CAL_FAULT(x) ((x) << S_CAL_FAULT) +#define F_CAL_FAULT V_CAL_FAULT(1U) + +#define S_SGL_CAL_EN 20 +#define V_SGL_CAL_EN(x) ((x) << S_SGL_CAL_EN) +#define F_SGL_CAL_EN V_SGL_CAL_EN(1U) + +#define A_MC7_ERR_ADDR 0x12c + +#define A_MC7_ECC 0x130 + +#define S_ECCCHKEN 1 +#define V_ECCCHKEN(x) ((x) << S_ECCCHKEN) +#define F_ECCCHKEN V_ECCCHKEN(1U) + +#define S_ECCGENEN 0 +#define V_ECCGENEN(x) ((x) << S_ECCGENEN) +#define F_ECCGENEN V_ECCGENEN(1U) + +#define A_MC7_CE_ADDR 0x134 + +#define A_MC7_CE_DATA0 0x138 + +#define A_MC7_CE_DATA1 0x13c + +#define A_MC7_CE_DATA2 0x140 + +#define S_DATA 0 +#define M_DATA 0xff + +#define G_DATA(x) (((x) >> S_DATA) & M_DATA) + +#define A_MC7_UE_ADDR 0x144 + +#define A_MC7_UE_DATA0 0x148 + +#define A_MC7_UE_DATA1 0x14c + +#define A_MC7_UE_DATA2 0x150 + +#define A_MC7_BD_ADDR 0x154 + +#define S_ADDR 3 + +#define M_ADDR 0x1fffffff + +#define A_MC7_BD_DATA0 0x158 + +#define A_MC7_BD_DATA1 0x15c + +#define A_MC7_BD_OP 0x164 + +#define S_OP 0 + +#define V_OP(x) ((x) << S_OP) +#define F_OP V_OP(1U) + +#define A_MC7_BIST_ADDR_BEG 0x168 + +#define A_MC7_BIST_ADDR_END 0x16c + +#define A_MC7_BIST_DATA 0x170 + +#define A_MC7_BIST_OP 0x174 + +#define S_CONT 3 +#define V_CONT(x) ((x) << S_CONT) +#define F_CONT V_CONT(1U) + +#define A_MC7_INT_ENABLE 0x178 + +#define S_AE 17 +#define V_AE(x) ((x) << S_AE) +#define F_AE V_AE(1U) + +#define S_PE 2 +#define M_PE 0x7fff + +#define V_PE(x) ((x) << S_PE) + +#define G_PE(x) (((x) >> S_PE) & M_PE) + +#define S_UE 1 +#define V_UE(x) ((x) << S_UE) +#define F_UE V_UE(1U) + +#define S_CE 0 +#define V_CE(x) ((x) << S_CE) +#define F_CE V_CE(1U) + +#define A_MC7_INT_CAUSE 0x17c + +#define MC7_PMTX_BASE_ADDR 0x180 + +#define MC7_CM_BASE_ADDR 0x200 + +#define A_CIM_BOOT_CFG 0x280 + +#define S_BOOTADDR 2 +#define M_BOOTADDR 0x3fffffff +#define V_BOOTADDR(x) ((x) << S_BOOTADDR) + +#define A_CIM_SDRAM_BASE_ADDR 0x28c + +#define A_CIM_SDRAM_ADDR_SIZE 0x290 + +#define A_CIM_HOST_INT_ENABLE 0x298 + +#define S_DTAGPARERR 28 +#define V_DTAGPARERR(x) ((x) << S_DTAGPARERR) +#define F_DTAGPARERR V_DTAGPARERR(1U) + +#define S_ITAGPARERR 27 +#define V_ITAGPARERR(x) ((x) << S_ITAGPARERR) +#define F_ITAGPARERR V_ITAGPARERR(1U) + +#define S_IBQTPPARERR 26 +#define V_IBQTPPARERR(x) ((x) << S_IBQTPPARERR) +#define F_IBQTPPARERR V_IBQTPPARERR(1U) + +#define S_IBQULPPARERR 25 +#define V_IBQULPPARERR(x) ((x) << S_IBQULPPARERR) +#define F_IBQULPPARERR V_IBQULPPARERR(1U) + +#define S_IBQSGEHIPARERR 24 +#define V_IBQSGEHIPARERR(x) ((x) << S_IBQSGEHIPARERR) +#define F_IBQSGEHIPARERR V_IBQSGEHIPARERR(1U) + +#define S_IBQSGELOPARERR 23 +#define V_IBQSGELOPARERR(x) ((x) << S_IBQSGELOPARERR) +#define F_IBQSGELOPARERR V_IBQSGELOPARERR(1U) + +#define S_OBQULPLOPARERR 22 +#define V_OBQULPLOPARERR(x) ((x) << S_OBQULPLOPARERR) +#define F_OBQULPLOPARERR V_OBQULPLOPARERR(1U) + +#define S_OBQULPHIPARERR 21 +#define V_OBQULPHIPARERR(x) ((x) << S_OBQULPHIPARERR) +#define F_OBQULPHIPARERR V_OBQULPHIPARERR(1U) + +#define S_OBQSGEPARERR 20 +#define V_OBQSGEPARERR(x) ((x) << S_OBQSGEPARERR) +#define F_OBQSGEPARERR V_OBQSGEPARERR(1U) + +#define S_DCACHEPARERR 19 +#define V_DCACHEPARERR(x) ((x) << S_DCACHEPARERR) +#define F_DCACHEPARERR V_DCACHEPARERR(1U) + +#define S_ICACHEPARERR 18 +#define V_ICACHEPARERR(x) ((x) << S_ICACHEPARERR) +#define F_ICACHEPARERR V_ICACHEPARERR(1U) + +#define S_DRAMPARERR 17 +#define V_DRAMPARERR(x) ((x) << S_DRAMPARERR) +#define F_DRAMPARERR V_DRAMPARERR(1U) + +#define A_CIM_HOST_INT_CAUSE 0x29c + +#define S_BLKWRPLINT 12 +#define V_BLKWRPLINT(x) ((x) << S_BLKWRPLINT) +#define F_BLKWRPLINT V_BLKWRPLINT(1U) + +#define S_BLKRDPLINT 11 +#define V_BLKRDPLINT(x) ((x) << S_BLKRDPLINT) +#define F_BLKRDPLINT V_BLKRDPLINT(1U) + +#define S_BLKWRCTLINT 10 +#define V_BLKWRCTLINT(x) ((x) << S_BLKWRCTLINT) +#define F_BLKWRCTLINT V_BLKWRCTLINT(1U) + +#define S_BLKRDCTLINT 9 +#define V_BLKRDCTLINT(x) ((x) << S_BLKRDCTLINT) +#define F_BLKRDCTLINT V_BLKRDCTLINT(1U) + +#define S_BLKWRFLASHINT 8 +#define V_BLKWRFLASHINT(x) ((x) << S_BLKWRFLASHINT) +#define F_BLKWRFLASHINT V_BLKWRFLASHINT(1U) + +#define S_BLKRDFLASHINT 7 +#define V_BLKRDFLASHINT(x) ((x) << S_BLKRDFLASHINT) +#define F_BLKRDFLASHINT V_BLKRDFLASHINT(1U) + +#define S_SGLWRFLASHINT 6 +#define V_SGLWRFLASHINT(x) ((x) << S_SGLWRFLASHINT) +#define F_SGLWRFLASHINT V_SGLWRFLASHINT(1U) + +#define S_WRBLKFLASHINT 5 +#define V_WRBLKFLASHINT(x) ((x) << S_WRBLKFLASHINT) +#define F_WRBLKFLASHINT V_WRBLKFLASHINT(1U) + +#define S_BLKWRBOOTINT 4 +#define V_BLKWRBOOTINT(x) ((x) << S_BLKWRBOOTINT) +#define F_BLKWRBOOTINT V_BLKWRBOOTINT(1U) + +#define S_FLASHRANGEINT 2 +#define V_FLASHRANGEINT(x) ((x) << S_FLASHRANGEINT) +#define F_FLASHRANGEINT V_FLASHRANGEINT(1U) + +#define S_SDRAMRANGEINT 1 +#define V_SDRAMRANGEINT(x) ((x) << S_SDRAMRANGEINT) +#define F_SDRAMRANGEINT V_SDRAMRANGEINT(1U) + +#define S_RSVDSPACEINT 0 +#define V_RSVDSPACEINT(x) ((x) << S_RSVDSPACEINT) +#define F_RSVDSPACEINT V_RSVDSPACEINT(1U) + +#define A_CIM_HOST_ACC_CTRL 0x2b0 + +#define S_HOSTBUSY 17 +#define V_HOSTBUSY(x) ((x) << S_HOSTBUSY) +#define F_HOSTBUSY V_HOSTBUSY(1U) + +#define A_CIM_HOST_ACC_DATA 0x2b4 + +#define A_CIM_IBQ_DBG_CFG 0x2c0 + +#define S_IBQDBGADDR 16 +#define M_IBQDBGADDR 0x1ff +#define V_IBQDBGADDR(x) ((x) << S_IBQDBGADDR) +#define G_IBQDBGADDR(x) (((x) >> S_IBQDBGADDR) & M_IBQDBGADDR) + +#define S_IBQDBGQID 3 +#define M_IBQDBGQID 0x3 +#define V_IBQDBGQID(x) ((x) << S_IBQDBGQID) +#define G_IBQDBGQID(x) (((x) >> S_IBQDBGQID) & M_IBQDBGQID) + +#define S_IBQDBGWR 2 +#define V_IBQDBGWR(x) ((x) << S_IBQDBGWR) +#define F_IBQDBGWR V_IBQDBGWR(1U) + +#define S_IBQDBGBUSY 1 +#define V_IBQDBGBUSY(x) ((x) << S_IBQDBGBUSY) +#define F_IBQDBGBUSY V_IBQDBGBUSY(1U) + +#define S_IBQDBGEN 0 +#define V_IBQDBGEN(x) ((x) << S_IBQDBGEN) +#define F_IBQDBGEN V_IBQDBGEN(1U) + +#define A_CIM_IBQ_DBG_DATA 0x2c8 + +#define A_TP_IN_CONFIG 0x300 + +#define S_RXFBARBPRIO 25 +#define V_RXFBARBPRIO(x) ((x) << S_RXFBARBPRIO) +#define F_RXFBARBPRIO V_RXFBARBPRIO(1U) + +#define S_TXFBARBPRIO 24 +#define V_TXFBARBPRIO(x) ((x) << S_TXFBARBPRIO) +#define F_TXFBARBPRIO V_TXFBARBPRIO(1U) + +#define S_NICMODE 14 +#define V_NICMODE(x) ((x) << S_NICMODE) +#define F_NICMODE V_NICMODE(1U) + +#define S_IPV6ENABLE 15 +#define V_IPV6ENABLE(x) ((x) << S_IPV6ENABLE) +#define F_IPV6ENABLE V_IPV6ENABLE(1U) + +#define A_TP_OUT_CONFIG 0x304 + +#define S_VLANEXTRACTIONENABLE 12 + +#define A_TP_GLOBAL_CONFIG 0x308 + +#define S_TXPACINGENABLE 24 +#define V_TXPACINGENABLE(x) ((x) << S_TXPACINGENABLE) +#define F_TXPACINGENABLE V_TXPACINGENABLE(1U) + +#define S_PATHMTU 15 +#define V_PATHMTU(x) ((x) << S_PATHMTU) +#define F_PATHMTU V_PATHMTU(1U) + +#define S_IPCHECKSUMOFFLOAD 13 +#define V_IPCHECKSUMOFFLOAD(x) ((x) << S_IPCHECKSUMOFFLOAD) +#define F_IPCHECKSUMOFFLOAD V_IPCHECKSUMOFFLOAD(1U) + +#define S_UDPCHECKSUMOFFLOAD 12 +#define V_UDPCHECKSUMOFFLOAD(x) ((x) << S_UDPCHECKSUMOFFLOAD) +#define F_UDPCHECKSUMOFFLOAD V_UDPCHECKSUMOFFLOAD(1U) + +#define S_TCPCHECKSUMOFFLOAD 11 +#define V_TCPCHECKSUMOFFLOAD(x) ((x) << S_TCPCHECKSUMOFFLOAD) +#define F_TCPCHECKSUMOFFLOAD V_TCPCHECKSUMOFFLOAD(1U) + +#define S_IPTTL 0 +#define M_IPTTL 0xff +#define V_IPTTL(x) ((x) << S_IPTTL) + +#define A_TP_CMM_MM_BASE 0x314 + +#define A_TP_CMM_TIMER_BASE 0x318 + +#define S_CMTIMERMAXNUM 28 +#define M_CMTIMERMAXNUM 0x3 +#define V_CMTIMERMAXNUM(x) ((x) << S_CMTIMERMAXNUM) + +#define A_TP_PMM_SIZE 0x31c + +#define A_TP_PMM_TX_BASE 0x320 + +#define A_TP_PMM_RX_BASE 0x328 + +#define A_TP_PMM_RX_PAGE_SIZE 0x32c + +#define A_TP_PMM_RX_MAX_PAGE 0x330 + +#define A_TP_PMM_TX_PAGE_SIZE 0x334 + +#define A_TP_PMM_TX_MAX_PAGE 0x338 + +#define A_TP_TCP_OPTIONS 0x340 + +#define S_MTUDEFAULT 16 +#define M_MTUDEFAULT 0xffff +#define V_MTUDEFAULT(x) ((x) << S_MTUDEFAULT) + +#define S_MTUENABLE 10 +#define V_MTUENABLE(x) ((x) << S_MTUENABLE) +#define F_MTUENABLE V_MTUENABLE(1U) + +#define S_SACKRX 8 +#define V_SACKRX(x) ((x) << S_SACKRX) +#define F_SACKRX V_SACKRX(1U) + +#define S_SACKMODE 4 + +#define M_SACKMODE 0x3 + +#define V_SACKMODE(x) ((x) << S_SACKMODE) + +#define S_WINDOWSCALEMODE 2 +#define M_WINDOWSCALEMODE 0x3 +#define V_WINDOWSCALEMODE(x) ((x) << S_WINDOWSCALEMODE) + +#define S_TIMESTAMPSMODE 0 + +#define M_TIMESTAMPSMODE 0x3 + +#define V_TIMESTAMPSMODE(x) ((x) << S_TIMESTAMPSMODE) + +#define A_TP_DACK_CONFIG 0x344 + +#define S_AUTOSTATE3 30 +#define M_AUTOSTATE3 0x3 +#define V_AUTOSTATE3(x) ((x) << S_AUTOSTATE3) + +#define S_AUTOSTATE2 28 +#define M_AUTOSTATE2 0x3 +#define V_AUTOSTATE2(x) ((x) << S_AUTOSTATE2) + +#define S_AUTOSTATE1 26 +#define M_AUTOSTATE1 0x3 +#define V_AUTOSTATE1(x) ((x) << S_AUTOSTATE1) + +#define S_BYTETHRESHOLD 5 +#define M_BYTETHRESHOLD 0xfffff +#define V_BYTETHRESHOLD(x) ((x) << S_BYTETHRESHOLD) + +#define S_MSSTHRESHOLD 3 +#define M_MSSTHRESHOLD 0x3 +#define V_MSSTHRESHOLD(x) ((x) << S_MSSTHRESHOLD) + +#define S_AUTOCAREFUL 2 +#define V_AUTOCAREFUL(x) ((x) << S_AUTOCAREFUL) +#define F_AUTOCAREFUL V_AUTOCAREFUL(1U) + +#define S_AUTOENABLE 1 +#define V_AUTOENABLE(x) ((x) << S_AUTOENABLE) +#define F_AUTOENABLE V_AUTOENABLE(1U) + +#define S_DACK_MODE 0 +#define V_DACK_MODE(x) ((x) << S_DACK_MODE) +#define F_DACK_MODE V_DACK_MODE(1U) + +#define A_TP_PC_CONFIG 0x348 + +#define S_TXTOSQUEUEMAPMODE 26 +#define V_TXTOSQUEUEMAPMODE(x) ((x) << S_TXTOSQUEUEMAPMODE) +#define F_TXTOSQUEUEMAPMODE V_TXTOSQUEUEMAPMODE(1U) + +#define S_ENABLEEPCMDAFULL 23 +#define V_ENABLEEPCMDAFULL(x) ((x) << S_ENABLEEPCMDAFULL) +#define F_ENABLEEPCMDAFULL V_ENABLEEPCMDAFULL(1U) + +#define S_MODULATEUNIONMODE 22 +#define V_MODULATEUNIONMODE(x) ((x) << S_MODULATEUNIONMODE) +#define F_MODULATEUNIONMODE V_MODULATEUNIONMODE(1U) + +#define S_TXDEFERENABLE 20 +#define V_TXDEFERENABLE(x) ((x) << S_TXDEFERENABLE) +#define F_TXDEFERENABLE V_TXDEFERENABLE(1U) + +#define S_RXCONGESTIONMODE 19 +#define V_RXCONGESTIONMODE(x) ((x) << S_RXCONGESTIONMODE) +#define F_RXCONGESTIONMODE V_RXCONGESTIONMODE(1U) + +#define S_HEARBEATDACK 16 +#define V_HEARBEATDACK(x) ((x) << S_HEARBEATDACK) +#define F_HEARBEATDACK V_HEARBEATDACK(1U) + +#define S_TXCONGESTIONMODE 15 +#define V_TXCONGESTIONMODE(x) ((x) << S_TXCONGESTIONMODE) +#define F_TXCONGESTIONMODE V_TXCONGESTIONMODE(1U) + +#define S_ENABLEOCSPIFULL 30 +#define V_ENABLEOCSPIFULL(x) ((x) << S_ENABLEOCSPIFULL) +#define F_ENABLEOCSPIFULL V_ENABLEOCSPIFULL(1U) + +#define S_LOCKTID 28 +#define V_LOCKTID(x) ((x) << S_LOCKTID) +#define F_LOCKTID V_LOCKTID(1U) + +#define S_TABLELATENCYDELTA 0 +#define M_TABLELATENCYDELTA 0xf +#define V_TABLELATENCYDELTA(x) ((x) << S_TABLELATENCYDELTA) +#define G_TABLELATENCYDELTA(x) \ + (((x) >> S_TABLELATENCYDELTA) & M_TABLELATENCYDELTA) + +#define A_TP_PC_CONFIG2 0x34c + +#define S_DISBLEDAPARBIT0 15 +#define V_DISBLEDAPARBIT0(x) ((x) << S_DISBLEDAPARBIT0) +#define F_DISBLEDAPARBIT0 V_DISBLEDAPARBIT0(1U) + +#define S_ENABLEARPMISS 13 +#define V_ENABLEARPMISS(x) ((x) << S_ENABLEARPMISS) +#define F_ENABLEARPMISS V_ENABLEARPMISS(1U) + +#define S_ENABLENONOFDTNLSYN 12 +#define V_ENABLENONOFDTNLSYN(x) ((x) << S_ENABLENONOFDTNLSYN) +#define F_ENABLENONOFDTNLSYN V_ENABLENONOFDTNLSYN(1U) + +#define S_ENABLEIPV6RSS 11 +#define V_ENABLEIPV6RSS(x) ((x) << S_ENABLEIPV6RSS) +#define F_ENABLEIPV6RSS V_ENABLEIPV6RSS(1U) + +#define S_CHDRAFULL 4 +#define V_CHDRAFULL(x) ((x) << S_CHDRAFULL) +#define F_CHDRAFULL V_CHDRAFULL(1U) + +#define A_TP_TCP_BACKOFF_REG0 0x350 + +#define A_TP_TCP_BACKOFF_REG1 0x354 + +#define A_TP_TCP_BACKOFF_REG2 0x358 + +#define A_TP_TCP_BACKOFF_REG3 0x35c + +#define A_TP_PARA_REG2 0x368 + +#define S_MAXRXDATA 16 +#define M_MAXRXDATA 0xffff +#define V_MAXRXDATA(x) ((x) << S_MAXRXDATA) + +#define S_RXCOALESCESIZE 0 +#define M_RXCOALESCESIZE 0xffff +#define V_RXCOALESCESIZE(x) ((x) << S_RXCOALESCESIZE) + +#define A_TP_PARA_REG3 0x36c + +#define S_TXDATAACKIDX 16 +#define M_TXDATAACKIDX 0xf + +#define V_TXDATAACKIDX(x) ((x) << S_TXDATAACKIDX) + +#define S_TXPACEAUTOSTRICT 10 +#define V_TXPACEAUTOSTRICT(x) ((x) << S_TXPACEAUTOSTRICT) +#define F_TXPACEAUTOSTRICT V_TXPACEAUTOSTRICT(1U) + +#define S_TXPACEFIXED 9 +#define V_TXPACEFIXED(x) ((x) << S_TXPACEFIXED) +#define F_TXPACEFIXED V_TXPACEFIXED(1U) + +#define S_TXPACEAUTO 8 +#define V_TXPACEAUTO(x) ((x) << S_TXPACEAUTO) +#define F_TXPACEAUTO V_TXPACEAUTO(1U) + +#define S_RXCOALESCEENABLE 1 +#define V_RXCOALESCEENABLE(x) ((x) << S_RXCOALESCEENABLE) +#define F_RXCOALESCEENABLE V_RXCOALESCEENABLE(1U) + +#define S_RXCOALESCEPSHEN 0 +#define V_RXCOALESCEPSHEN(x) ((x) << S_RXCOALESCEPSHEN) +#define F_RXCOALESCEPSHEN V_RXCOALESCEPSHEN(1U) + +#define A_TP_PARA_REG4 0x370 + +#define A_TP_PARA_REG5 0x374 + +#define S_RXDDPOFFINIT 3 +#define V_RXDDPOFFINIT(x) ((x) << S_RXDDPOFFINIT) +#define F_RXDDPOFFINIT V_RXDDPOFFINIT(1U) + +#define A_TP_PARA_REG6 0x378 + +#define S_T3A_ENABLEESND 13 +#define V_T3A_ENABLEESND(x) ((x) << S_T3A_ENABLEESND) +#define F_T3A_ENABLEESND V_T3A_ENABLEESND(1U) + +#define S_ENABLEESND 11 +#define V_ENABLEESND(x) ((x) << S_ENABLEESND) +#define F_ENABLEESND V_ENABLEESND(1U) + +#define A_TP_PARA_REG7 0x37c + +#define S_PMMAXXFERLEN1 16 +#define M_PMMAXXFERLEN1 0xffff +#define V_PMMAXXFERLEN1(x) ((x) << S_PMMAXXFERLEN1) + +#define S_PMMAXXFERLEN0 0 +#define M_PMMAXXFERLEN0 0xffff +#define V_PMMAXXFERLEN0(x) ((x) << S_PMMAXXFERLEN0) + +#define A_TP_TIMER_RESOLUTION 0x390 + +#define S_TIMERRESOLUTION 16 +#define M_TIMERRESOLUTION 0xff +#define V_TIMERRESOLUTION(x) ((x) << S_TIMERRESOLUTION) + +#define S_TIMESTAMPRESOLUTION 8 +#define M_TIMESTAMPRESOLUTION 0xff +#define V_TIMESTAMPRESOLUTION(x) ((x) << S_TIMESTAMPRESOLUTION) + +#define S_DELAYEDACKRESOLUTION 0 +#define M_DELAYEDACKRESOLUTION 0xff +#define V_DELAYEDACKRESOLUTION(x) ((x) << S_DELAYEDACKRESOLUTION) + +#define A_TP_MSL 0x394 + +#define A_TP_RXT_MIN 0x398 + +#define A_TP_RXT_MAX 0x39c + +#define A_TP_PERS_MIN 0x3a0 + +#define A_TP_PERS_MAX 0x3a4 + +#define A_TP_KEEP_IDLE 0x3a8 + +#define A_TP_KEEP_INTVL 0x3ac + +#define A_TP_INIT_SRTT 0x3b0 + +#define A_TP_DACK_TIMER 0x3b4 + +#define A_TP_FINWAIT2_TIMER 0x3b8 + +#define A_TP_SHIFT_CNT 0x3c0 + +#define S_SYNSHIFTMAX 24 + +#define M_SYNSHIFTMAX 0xff + +#define V_SYNSHIFTMAX(x) ((x) << S_SYNSHIFTMAX) + +#define S_RXTSHIFTMAXR1 20 + +#define M_RXTSHIFTMAXR1 0xf + +#define V_RXTSHIFTMAXR1(x) ((x) << S_RXTSHIFTMAXR1) + +#define S_RXTSHIFTMAXR2 16 + +#define M_RXTSHIFTMAXR2 0xf + +#define V_RXTSHIFTMAXR2(x) ((x) << S_RXTSHIFTMAXR2) + +#define S_PERSHIFTBACKOFFMAX 12 +#define M_PERSHIFTBACKOFFMAX 0xf +#define V_PERSHIFTBACKOFFMAX(x) ((x) << S_PERSHIFTBACKOFFMAX) + +#define S_PERSHIFTMAX 8 +#define M_PERSHIFTMAX 0xf +#define V_PERSHIFTMAX(x) ((x) << S_PERSHIFTMAX) + +#define S_KEEPALIVEMAX 0 + +#define M_KEEPALIVEMAX 0xff + +#define V_KEEPALIVEMAX(x) ((x) << S_KEEPALIVEMAX) + +#define A_TP_MTU_PORT_TABLE 0x3d0 + +#define A_TP_CCTRL_TABLE 0x3dc + +#define A_TP_MTU_TABLE 0x3e4 + +#define A_TP_RSS_MAP_TABLE 0x3e8 + +#define A_TP_RSS_LKP_TABLE 0x3ec + +#define A_TP_RSS_CONFIG 0x3f0 + +#define S_TNL4TUPEN 29 +#define V_TNL4TUPEN(x) ((x) << S_TNL4TUPEN) +#define F_TNL4TUPEN V_TNL4TUPEN(1U) + +#define S_TNL2TUPEN 28 +#define V_TNL2TUPEN(x) ((x) << S_TNL2TUPEN) +#define F_TNL2TUPEN V_TNL2TUPEN(1U) + +#define S_TNLPRTEN 26 +#define V_TNLPRTEN(x) ((x) << S_TNLPRTEN) +#define F_TNLPRTEN V_TNLPRTEN(1U) + +#define S_TNLMAPEN 25 +#define V_TNLMAPEN(x) ((x) << S_TNLMAPEN) +#define F_TNLMAPEN V_TNLMAPEN(1U) + +#define S_TNLLKPEN 24 +#define V_TNLLKPEN(x) ((x) << S_TNLLKPEN) +#define F_TNLLKPEN V_TNLLKPEN(1U) + +#define S_RRCPLMAPEN 7 +#define V_RRCPLMAPEN(x) ((x) << S_RRCPLMAPEN) +#define F_RRCPLMAPEN V_RRCPLMAPEN(1U) + +#define S_RRCPLCPUSIZE 4 +#define M_RRCPLCPUSIZE 0x7 +#define V_RRCPLCPUSIZE(x) ((x) << S_RRCPLCPUSIZE) + +#define S_RQFEEDBACKENABLE 3 +#define V_RQFEEDBACKENABLE(x) ((x) << S_RQFEEDBACKENABLE) +#define F_RQFEEDBACKENABLE V_RQFEEDBACKENABLE(1U) + +#define S_HASHTOEPLITZ 2 +#define V_HASHTOEPLITZ(x) ((x) << S_HASHTOEPLITZ) +#define F_HASHTOEPLITZ V_HASHTOEPLITZ(1U) + +#define S_DISABLE 0 + +#define A_TP_TM_PIO_ADDR 0x418 + +#define A_TP_TM_PIO_DATA 0x41c + +#define A_TP_TX_MOD_QUE_TABLE 0x420 + +#define A_TP_TX_RESOURCE_LIMIT 0x424 + +#define A_TP_TX_MOD_QUEUE_REQ_MAP 0x428 + +#define S_TX_MOD_QUEUE_REQ_MAP 0 +#define M_TX_MOD_QUEUE_REQ_MAP 0xff +#define V_TX_MOD_QUEUE_REQ_MAP(x) ((x) << S_TX_MOD_QUEUE_REQ_MAP) + +#define A_TP_TX_MOD_QUEUE_WEIGHT1 0x42c + +#define A_TP_TX_MOD_QUEUE_WEIGHT0 0x430 + +#define A_TP_MOD_CHANNEL_WEIGHT 0x434 + +#define A_TP_MOD_RATE_LIMIT 0x438 + +#define A_TP_PIO_ADDR 0x440 + +#define A_TP_PIO_DATA 0x444 + +#define A_TP_RESET 0x44c + +#define S_FLSTINITENABLE 1 +#define V_FLSTINITENABLE(x) ((x) << S_FLSTINITENABLE) +#define F_FLSTINITENABLE V_FLSTINITENABLE(1U) + +#define S_TPRESET 0 +#define V_TPRESET(x) ((x) << S_TPRESET) +#define F_TPRESET V_TPRESET(1U) + +#define A_TP_CMM_MM_RX_FLST_BASE 0x460 + +#define A_TP_CMM_MM_TX_FLST_BASE 0x464 + +#define A_TP_CMM_MM_PS_FLST_BASE 0x468 + +#define A_TP_MIB_INDEX 0x450 + +#define A_TP_MIB_RDATA 0x454 + +#define A_TP_CMM_MM_MAX_PSTRUCT 0x46c + +#define A_TP_INT_ENABLE 0x470 + +#define S_FLMTXFLSTEMPTY 30 +#define V_FLMTXFLSTEMPTY(x) ((x) << S_FLMTXFLSTEMPTY) +#define F_FLMTXFLSTEMPTY V_FLMTXFLSTEMPTY(1U) + +#define S_FLMRXFLSTEMPTY 29 +#define V_FLMRXFLSTEMPTY(x) ((x) << S_FLMRXFLSTEMPTY) +#define F_FLMRXFLSTEMPTY V_FLMRXFLSTEMPTY(1U) + +#define S_ARPLUTPERR 26 +#define V_ARPLUTPERR(x) ((x) << S_ARPLUTPERR) +#define F_ARPLUTPERR V_ARPLUTPERR(1U) + +#define S_CMCACHEPERR 24 +#define V_CMCACHEPERR(x) ((x) << S_CMCACHEPERR) +#define F_CMCACHEPERR V_CMCACHEPERR(1U) + +#define A_TP_INT_CAUSE 0x474 + +#define A_TP_TX_MOD_Q1_Q0_RATE_LIMIT 0x8 + +#define A_TP_TX_DROP_CFG_CH0 0x12b + +#define A_TP_TX_DROP_MODE 0x12f + +#define A_TP_EGRESS_CONFIG 0x145 + +#define S_REWRITEFORCETOSIZE 0 +#define V_REWRITEFORCETOSIZE(x) ((x) << S_REWRITEFORCETOSIZE) +#define F_REWRITEFORCETOSIZE V_REWRITEFORCETOSIZE(1U) + +#define A_TP_TX_TRC_KEY0 0x20 + +#define A_TP_RX_TRC_KEY0 0x120 + +#define A_TP_TX_DROP_CNT_CH0 0x12d + +#define S_TXDROPCNTCH0RCVD 0 +#define M_TXDROPCNTCH0RCVD 0xffff +#define V_TXDROPCNTCH0RCVD(x) ((x) << S_TXDROPCNTCH0RCVD) +#define G_TXDROPCNTCH0RCVD(x) (((x) >> S_TXDROPCNTCH0RCVD) & \ + M_TXDROPCNTCH0RCVD) + +#define A_TP_PROXY_FLOW_CNTL 0x4b0 + +#define A_TP_EMBED_OP_FIELD0 0x4e8 +#define A_TP_EMBED_OP_FIELD1 0x4ec +#define A_TP_EMBED_OP_FIELD2 0x4f0 +#define A_TP_EMBED_OP_FIELD3 0x4f4 +#define A_TP_EMBED_OP_FIELD4 0x4f8 +#define A_TP_EMBED_OP_FIELD5 0x4fc + +#define A_ULPRX_CTL 0x500 + +#define S_ROUND_ROBIN 4 +#define V_ROUND_ROBIN(x) ((x) << S_ROUND_ROBIN) +#define F_ROUND_ROBIN V_ROUND_ROBIN(1U) + +#define A_ULPRX_INT_ENABLE 0x504 + +#define S_DATASELFRAMEERR0 7 +#define V_DATASELFRAMEERR0(x) ((x) << S_DATASELFRAMEERR0) +#define F_DATASELFRAMEERR0 V_DATASELFRAMEERR0(1U) + +#define S_DATASELFRAMEERR1 6 +#define V_DATASELFRAMEERR1(x) ((x) << S_DATASELFRAMEERR1) +#define F_DATASELFRAMEERR1 V_DATASELFRAMEERR1(1U) + +#define S_PCMDMUXPERR 5 +#define V_PCMDMUXPERR(x) ((x) << S_PCMDMUXPERR) +#define F_PCMDMUXPERR V_PCMDMUXPERR(1U) + +#define S_ARBFPERR 4 +#define V_ARBFPERR(x) ((x) << S_ARBFPERR) +#define F_ARBFPERR V_ARBFPERR(1U) + +#define S_ARBPF0PERR 3 +#define V_ARBPF0PERR(x) ((x) << S_ARBPF0PERR) +#define F_ARBPF0PERR V_ARBPF0PERR(1U) + +#define S_ARBPF1PERR 2 +#define V_ARBPF1PERR(x) ((x) << S_ARBPF1PERR) +#define F_ARBPF1PERR V_ARBPF1PERR(1U) + +#define S_PARERRPCMD 1 +#define V_PARERRPCMD(x) ((x) << S_PARERRPCMD) +#define F_PARERRPCMD V_PARERRPCMD(1U) + +#define S_PARERRDATA 0 +#define V_PARERRDATA(x) ((x) << S_PARERRDATA) +#define F_PARERRDATA V_PARERRDATA(1U) + +#define A_ULPRX_INT_CAUSE 0x508 + +#define A_ULPRX_ISCSI_LLIMIT 0x50c + +#define A_ULPRX_ISCSI_ULIMIT 0x510 + +#define A_ULPRX_ISCSI_TAGMASK 0x514 + +#define A_ULPRX_ISCSI_PSZ 0x518 + +#define A_ULPRX_TDDP_LLIMIT 0x51c + +#define A_ULPRX_TDDP_ULIMIT 0x520 +#define A_ULPRX_TDDP_PSZ 0x528 + +#define S_HPZ0 0 +#define M_HPZ0 0xf +#define V_HPZ0(x) ((x) << S_HPZ0) +#define G_HPZ0(x) (((x) >> S_HPZ0) & M_HPZ0) + +#define A_ULPRX_STAG_LLIMIT 0x52c + +#define A_ULPRX_STAG_ULIMIT 0x530 + +#define A_ULPRX_RQ_LLIMIT 0x534 + +#define A_ULPRX_RQ_ULIMIT 0x538 + +#define A_ULPRX_PBL_LLIMIT 0x53c + +#define A_ULPRX_PBL_ULIMIT 0x540 + +#define A_ULPRX_TDDP_TAGMASK 0x524 + +#define A_ULPTX_CONFIG 0x580 + +#define S_CFG_CQE_SOP_MASK 1 +#define V_CFG_CQE_SOP_MASK(x) ((x) << S_CFG_CQE_SOP_MASK) +#define F_CFG_CQE_SOP_MASK V_CFG_CQE_SOP_MASK(1U) + +#define S_CFG_RR_ARB 0 +#define V_CFG_RR_ARB(x) ((x) << S_CFG_RR_ARB) +#define F_CFG_RR_ARB V_CFG_RR_ARB(1U) + +#define A_ULPTX_INT_ENABLE 0x584 + +#define S_PBL_BOUND_ERR_CH1 1 +#define V_PBL_BOUND_ERR_CH1(x) ((x) << S_PBL_BOUND_ERR_CH1) +#define F_PBL_BOUND_ERR_CH1 V_PBL_BOUND_ERR_CH1(1U) + +#define S_PBL_BOUND_ERR_CH0 0 +#define V_PBL_BOUND_ERR_CH0(x) ((x) << S_PBL_BOUND_ERR_CH0) +#define F_PBL_BOUND_ERR_CH0 V_PBL_BOUND_ERR_CH0(1U) + +#define A_ULPTX_INT_CAUSE 0x588 + +#define A_ULPTX_TPT_LLIMIT 0x58c + +#define A_ULPTX_TPT_ULIMIT 0x590 + +#define A_ULPTX_PBL_LLIMIT 0x594 + +#define A_ULPTX_PBL_ULIMIT 0x598 + +#define A_ULPTX_DMA_WEIGHT 0x5ac + +#define S_D1_WEIGHT 16 +#define M_D1_WEIGHT 0xffff +#define V_D1_WEIGHT(x) ((x) << S_D1_WEIGHT) + +#define S_D0_WEIGHT 0 +#define M_D0_WEIGHT 0xffff +#define V_D0_WEIGHT(x) ((x) << S_D0_WEIGHT) + +#define A_PM1_RX_CFG 0x5c0 +#define A_PM1_RX_MODE 0x5c4 + +#define A_PM1_RX_INT_ENABLE 0x5d8 + +#define S_ZERO_E_CMD_ERROR 18 +#define V_ZERO_E_CMD_ERROR(x) ((x) << S_ZERO_E_CMD_ERROR) +#define F_ZERO_E_CMD_ERROR V_ZERO_E_CMD_ERROR(1U) + +#define S_IESPI0_FIFO2X_RX_FRAMING_ERROR 17 +#define V_IESPI0_FIFO2X_RX_FRAMING_ERROR(x) ((x) << S_IESPI0_FIFO2X_RX_FRAMING_ERROR) +#define F_IESPI0_FIFO2X_RX_FRAMING_ERROR V_IESPI0_FIFO2X_RX_FRAMING_ERROR(1U) + +#define S_IESPI1_FIFO2X_RX_FRAMING_ERROR 16 +#define V_IESPI1_FIFO2X_RX_FRAMING_ERROR(x) ((x) << S_IESPI1_FIFO2X_RX_FRAMING_ERROR) +#define F_IESPI1_FIFO2X_RX_FRAMING_ERROR V_IESPI1_FIFO2X_RX_FRAMING_ERROR(1U) + +#define S_IESPI0_RX_FRAMING_ERROR 15 +#define V_IESPI0_RX_FRAMING_ERROR(x) ((x) << S_IESPI0_RX_FRAMING_ERROR) +#define F_IESPI0_RX_FRAMING_ERROR V_IESPI0_RX_FRAMING_ERROR(1U) + +#define S_IESPI1_RX_FRAMING_ERROR 14 +#define V_IESPI1_RX_FRAMING_ERROR(x) ((x) << S_IESPI1_RX_FRAMING_ERROR) +#define F_IESPI1_RX_FRAMING_ERROR V_IESPI1_RX_FRAMING_ERROR(1U) + +#define S_IESPI0_TX_FRAMING_ERROR 13 +#define V_IESPI0_TX_FRAMING_ERROR(x) ((x) << S_IESPI0_TX_FRAMING_ERROR) +#define F_IESPI0_TX_FRAMING_ERROR V_IESPI0_TX_FRAMING_ERROR(1U) + +#define S_IESPI1_TX_FRAMING_ERROR 12 +#define V_IESPI1_TX_FRAMING_ERROR(x) ((x) << S_IESPI1_TX_FRAMING_ERROR) +#define F_IESPI1_TX_FRAMING_ERROR V_IESPI1_TX_FRAMING_ERROR(1U) + +#define S_OCSPI0_RX_FRAMING_ERROR 11 +#define V_OCSPI0_RX_FRAMING_ERROR(x) ((x) << S_OCSPI0_RX_FRAMING_ERROR) +#define F_OCSPI0_RX_FRAMING_ERROR V_OCSPI0_RX_FRAMING_ERROR(1U) + +#define S_OCSPI1_RX_FRAMING_ERROR 10 +#define V_OCSPI1_RX_FRAMING_ERROR(x) ((x) << S_OCSPI1_RX_FRAMING_ERROR) +#define F_OCSPI1_RX_FRAMING_ERROR V_OCSPI1_RX_FRAMING_ERROR(1U) + +#define S_OCSPI0_TX_FRAMING_ERROR 9 +#define V_OCSPI0_TX_FRAMING_ERROR(x) ((x) << S_OCSPI0_TX_FRAMING_ERROR) +#define F_OCSPI0_TX_FRAMING_ERROR V_OCSPI0_TX_FRAMING_ERROR(1U) + +#define S_OCSPI1_TX_FRAMING_ERROR 8 +#define V_OCSPI1_TX_FRAMING_ERROR(x) ((x) << S_OCSPI1_TX_FRAMING_ERROR) +#define F_OCSPI1_TX_FRAMING_ERROR V_OCSPI1_TX_FRAMING_ERROR(1U) + +#define S_OCSPI0_OFIFO2X_TX_FRAMING_ERROR 7 +#define V_OCSPI0_OFIFO2X_TX_FRAMING_ERROR(x) ((x) << S_OCSPI0_OFIFO2X_TX_FRAMING_ERROR) +#define F_OCSPI0_OFIFO2X_TX_FRAMING_ERROR V_OCSPI0_OFIFO2X_TX_FRAMING_ERROR(1U) + +#define S_OCSPI1_OFIFO2X_TX_FRAMING_ERROR 6 +#define V_OCSPI1_OFIFO2X_TX_FRAMING_ERROR(x) ((x) << S_OCSPI1_OFIFO2X_TX_FRAMING_ERROR) +#define F_OCSPI1_OFIFO2X_TX_FRAMING_ERROR V_OCSPI1_OFIFO2X_TX_FRAMING_ERROR(1U) + +#define S_IESPI_PAR_ERROR 3 +#define M_IESPI_PAR_ERROR 0x7 + +#define V_IESPI_PAR_ERROR(x) ((x) << S_IESPI_PAR_ERROR) + +#define S_OCSPI_PAR_ERROR 0 +#define M_OCSPI_PAR_ERROR 0x7 + +#define V_OCSPI_PAR_ERROR(x) ((x) << S_OCSPI_PAR_ERROR) + +#define A_PM1_RX_INT_CAUSE 0x5dc + +#define A_PM1_TX_CFG 0x5e0 +#define A_PM1_TX_MODE 0x5e4 + +#define A_PM1_TX_INT_ENABLE 0x5f8 + +#define S_ZERO_C_CMD_ERROR 18 +#define V_ZERO_C_CMD_ERROR(x) ((x) << S_ZERO_C_CMD_ERROR) +#define F_ZERO_C_CMD_ERROR V_ZERO_C_CMD_ERROR(1U) + +#define S_ICSPI0_FIFO2X_RX_FRAMING_ERROR 17 +#define V_ICSPI0_FIFO2X_RX_FRAMING_ERROR(x) ((x) << S_ICSPI0_FIFO2X_RX_FRAMING_ERROR) +#define F_ICSPI0_FIFO2X_RX_FRAMING_ERROR V_ICSPI0_FIFO2X_RX_FRAMING_ERROR(1U) + +#define S_ICSPI1_FIFO2X_RX_FRAMING_ERROR 16 +#define V_ICSPI1_FIFO2X_RX_FRAMING_ERROR(x) ((x) << S_ICSPI1_FIFO2X_RX_FRAMING_ERROR) +#define F_ICSPI1_FIFO2X_RX_FRAMING_ERROR V_ICSPI1_FIFO2X_RX_FRAMING_ERROR(1U) + +#define S_ICSPI0_RX_FRAMING_ERROR 15 +#define V_ICSPI0_RX_FRAMING_ERROR(x) ((x) << S_ICSPI0_RX_FRAMING_ERROR) +#define F_ICSPI0_RX_FRAMING_ERROR V_ICSPI0_RX_FRAMING_ERROR(1U) + +#define S_ICSPI1_RX_FRAMING_ERROR 14 +#define V_ICSPI1_RX_FRAMING_ERROR(x) ((x) << S_ICSPI1_RX_FRAMING_ERROR) +#define F_ICSPI1_RX_FRAMING_ERROR V_ICSPI1_RX_FRAMING_ERROR(1U) + +#define S_ICSPI0_TX_FRAMING_ERROR 13 +#define V_ICSPI0_TX_FRAMING_ERROR(x) ((x) << S_ICSPI0_TX_FRAMING_ERROR) +#define F_ICSPI0_TX_FRAMING_ERROR V_ICSPI0_TX_FRAMING_ERROR(1U) + +#define S_ICSPI1_TX_FRAMING_ERROR 12 +#define V_ICSPI1_TX_FRAMING_ERROR(x) ((x) << S_ICSPI1_TX_FRAMING_ERROR) +#define F_ICSPI1_TX_FRAMING_ERROR V_ICSPI1_TX_FRAMING_ERROR(1U) + +#define S_OESPI0_RX_FRAMING_ERROR 11 +#define V_OESPI0_RX_FRAMING_ERROR(x) ((x) << S_OESPI0_RX_FRAMING_ERROR) +#define F_OESPI0_RX_FRAMING_ERROR V_OESPI0_RX_FRAMING_ERROR(1U) + +#define S_OESPI1_RX_FRAMING_ERROR 10 +#define V_OESPI1_RX_FRAMING_ERROR(x) ((x) << S_OESPI1_RX_FRAMING_ERROR) +#define F_OESPI1_RX_FRAMING_ERROR V_OESPI1_RX_FRAMING_ERROR(1U) + +#define S_OESPI0_TX_FRAMING_ERROR 9 +#define V_OESPI0_TX_FRAMING_ERROR(x) ((x) << S_OESPI0_TX_FRAMING_ERROR) +#define F_OESPI0_TX_FRAMING_ERROR V_OESPI0_TX_FRAMING_ERROR(1U) + +#define S_OESPI1_TX_FRAMING_ERROR 8 +#define V_OESPI1_TX_FRAMING_ERROR(x) ((x) << S_OESPI1_TX_FRAMING_ERROR) +#define F_OESPI1_TX_FRAMING_ERROR V_OESPI1_TX_FRAMING_ERROR(1U) + +#define S_OESPI0_OFIFO2X_TX_FRAMING_ERROR 7 +#define V_OESPI0_OFIFO2X_TX_FRAMING_ERROR(x) ((x) << S_OESPI0_OFIFO2X_TX_FRAMING_ERROR) +#define F_OESPI0_OFIFO2X_TX_FRAMING_ERROR V_OESPI0_OFIFO2X_TX_FRAMING_ERROR(1U) + +#define S_OESPI1_OFIFO2X_TX_FRAMING_ERROR 6 +#define V_OESPI1_OFIFO2X_TX_FRAMING_ERROR(x) ((x) << S_OESPI1_OFIFO2X_TX_FRAMING_ERROR) +#define F_OESPI1_OFIFO2X_TX_FRAMING_ERROR V_OESPI1_OFIFO2X_TX_FRAMING_ERROR(1U) + +#define S_ICSPI_PAR_ERROR 3 +#define M_ICSPI_PAR_ERROR 0x7 + +#define V_ICSPI_PAR_ERROR(x) ((x) << S_ICSPI_PAR_ERROR) + +#define S_OESPI_PAR_ERROR 0 +#define M_OESPI_PAR_ERROR 0x7 + +#define V_OESPI_PAR_ERROR(x) ((x) << S_OESPI_PAR_ERROR) + +#define A_PM1_TX_INT_CAUSE 0x5fc + +#define A_MPS_CFG 0x600 + +#define S_TPRXPORTEN 4 +#define V_TPRXPORTEN(x) ((x) << S_TPRXPORTEN) +#define F_TPRXPORTEN V_TPRXPORTEN(1U) + +#define S_TPTXPORT1EN 3 +#define V_TPTXPORT1EN(x) ((x) << S_TPTXPORT1EN) +#define F_TPTXPORT1EN V_TPTXPORT1EN(1U) + +#define S_TPTXPORT0EN 2 +#define V_TPTXPORT0EN(x) ((x) << S_TPTXPORT0EN) +#define F_TPTXPORT0EN V_TPTXPORT0EN(1U) + +#define S_PORT1ACTIVE 1 +#define V_PORT1ACTIVE(x) ((x) << S_PORT1ACTIVE) +#define F_PORT1ACTIVE V_PORT1ACTIVE(1U) + +#define S_PORT0ACTIVE 0 +#define V_PORT0ACTIVE(x) ((x) << S_PORT0ACTIVE) +#define F_PORT0ACTIVE V_PORT0ACTIVE(1U) + +#define S_ENFORCEPKT 11 +#define V_ENFORCEPKT(x) ((x) << S_ENFORCEPKT) +#define F_ENFORCEPKT V_ENFORCEPKT(1U) + +#define A_MPS_INT_ENABLE 0x61c + +#define S_MCAPARERRENB 6 +#define M_MCAPARERRENB 0x7 + +#define V_MCAPARERRENB(x) ((x) << S_MCAPARERRENB) + +#define S_RXTPPARERRENB 4 +#define M_RXTPPARERRENB 0x3 + +#define V_RXTPPARERRENB(x) ((x) << S_RXTPPARERRENB) + +#define S_TX1TPPARERRENB 2 +#define M_TX1TPPARERRENB 0x3 + +#define V_TX1TPPARERRENB(x) ((x) << S_TX1TPPARERRENB) + +#define S_TX0TPPARERRENB 0 +#define M_TX0TPPARERRENB 0x3 + +#define V_TX0TPPARERRENB(x) ((x) << S_TX0TPPARERRENB) + +#define A_MPS_INT_CAUSE 0x620 + +#define S_MCAPARERR 6 +#define M_MCAPARERR 0x7 + +#define V_MCAPARERR(x) ((x) << S_MCAPARERR) + +#define S_RXTPPARERR 4 +#define M_RXTPPARERR 0x3 + +#define V_RXTPPARERR(x) ((x) << S_RXTPPARERR) + +#define S_TX1TPPARERR 2 +#define M_TX1TPPARERR 0x3 + +#define V_TX1TPPARERR(x) ((x) << S_TX1TPPARERR) + +#define S_TX0TPPARERR 0 +#define M_TX0TPPARERR 0x3 + +#define V_TX0TPPARERR(x) ((x) << S_TX0TPPARERR) + +#define A_CPL_SWITCH_CNTRL 0x640 + +#define A_CPL_INTR_ENABLE 0x650 + +#define S_CIM_OP_MAP_PERR 5 +#define V_CIM_OP_MAP_PERR(x) ((x) << S_CIM_OP_MAP_PERR) +#define F_CIM_OP_MAP_PERR V_CIM_OP_MAP_PERR(1U) + +#define S_CIM_OVFL_ERROR 4 +#define V_CIM_OVFL_ERROR(x) ((x) << S_CIM_OVFL_ERROR) +#define F_CIM_OVFL_ERROR V_CIM_OVFL_ERROR(1U) + +#define S_TP_FRAMING_ERROR 3 +#define V_TP_FRAMING_ERROR(x) ((x) << S_TP_FRAMING_ERROR) +#define F_TP_FRAMING_ERROR V_TP_FRAMING_ERROR(1U) + +#define S_SGE_FRAMING_ERROR 2 +#define V_SGE_FRAMING_ERROR(x) ((x) << S_SGE_FRAMING_ERROR) +#define F_SGE_FRAMING_ERROR V_SGE_FRAMING_ERROR(1U) + +#define S_CIM_FRAMING_ERROR 1 +#define V_CIM_FRAMING_ERROR(x) ((x) << S_CIM_FRAMING_ERROR) +#define F_CIM_FRAMING_ERROR V_CIM_FRAMING_ERROR(1U) + +#define S_ZERO_SWITCH_ERROR 0 +#define V_ZERO_SWITCH_ERROR(x) ((x) << S_ZERO_SWITCH_ERROR) +#define F_ZERO_SWITCH_ERROR V_ZERO_SWITCH_ERROR(1U) + +#define A_CPL_INTR_CAUSE 0x654 + +#define A_CPL_MAP_TBL_DATA 0x65c + +#define A_SMB_GLOBAL_TIME_CFG 0x660 + +#define A_I2C_CFG 0x6a0 + +#define S_I2C_CLKDIV 0 +#define M_I2C_CLKDIV 0xfff +#define V_I2C_CLKDIV(x) ((x) << S_I2C_CLKDIV) + +#define A_MI1_CFG 0x6b0 + +#define S_CLKDIV 5 +#define M_CLKDIV 0xff +#define V_CLKDIV(x) ((x) << S_CLKDIV) + +#define S_ST 3 + +#define M_ST 0x3 + +#define V_ST(x) ((x) << S_ST) + +#define G_ST(x) (((x) >> S_ST) & M_ST) + +#define S_PREEN 2 +#define V_PREEN(x) ((x) << S_PREEN) +#define F_PREEN V_PREEN(1U) + +#define S_MDIINV 1 +#define V_MDIINV(x) ((x) << S_MDIINV) +#define F_MDIINV V_MDIINV(1U) + +#define S_MDIEN 0 +#define V_MDIEN(x) ((x) << S_MDIEN) +#define F_MDIEN V_MDIEN(1U) + +#define A_MI1_ADDR 0x6b4 + +#define S_PHYADDR 5 +#define M_PHYADDR 0x1f +#define V_PHYADDR(x) ((x) << S_PHYADDR) + +#define S_REGADDR 0 +#define M_REGADDR 0x1f +#define V_REGADDR(x) ((x) << S_REGADDR) + +#define A_MI1_DATA 0x6b8 + +#define A_MI1_OP 0x6bc + +#define S_MDI_OP 0 +#define M_MDI_OP 0x3 +#define V_MDI_OP(x) ((x) << S_MDI_OP) + +#define A_SF_DATA 0x6d8 + +#define A_SF_OP 0x6dc + +#define S_BYTECNT 1 +#define M_BYTECNT 0x3 +#define V_BYTECNT(x) ((x) << S_BYTECNT) + +#define A_PL_INT_ENABLE0 0x6e0 + +#define S_T3DBG 23 +#define V_T3DBG(x) ((x) << S_T3DBG) +#define F_T3DBG V_T3DBG(1U) + +#define S_XGMAC0_1 20 +#define V_XGMAC0_1(x) ((x) << S_XGMAC0_1) +#define F_XGMAC0_1 V_XGMAC0_1(1U) + +#define S_XGMAC0_0 19 +#define V_XGMAC0_0(x) ((x) << S_XGMAC0_0) +#define F_XGMAC0_0 V_XGMAC0_0(1U) + +#define S_MC5A 18 +#define V_MC5A(x) ((x) << S_MC5A) +#define F_MC5A V_MC5A(1U) + +#define S_CPL_SWITCH 12 +#define V_CPL_SWITCH(x) ((x) << S_CPL_SWITCH) +#define F_CPL_SWITCH V_CPL_SWITCH(1U) + +#define S_MPS0 11 +#define V_MPS0(x) ((x) << S_MPS0) +#define F_MPS0 V_MPS0(1U) + +#define S_PM1_TX 10 +#define V_PM1_TX(x) ((x) << S_PM1_TX) +#define F_PM1_TX V_PM1_TX(1U) + +#define S_PM1_RX 9 +#define V_PM1_RX(x) ((x) << S_PM1_RX) +#define F_PM1_RX V_PM1_RX(1U) + +#define S_ULP2_TX 8 +#define V_ULP2_TX(x) ((x) << S_ULP2_TX) +#define F_ULP2_TX V_ULP2_TX(1U) + +#define S_ULP2_RX 7 +#define V_ULP2_RX(x) ((x) << S_ULP2_RX) +#define F_ULP2_RX V_ULP2_RX(1U) + +#define S_TP1 6 +#define V_TP1(x) ((x) << S_TP1) +#define F_TP1 V_TP1(1U) + +#define S_CIM 5 +#define V_CIM(x) ((x) << S_CIM) +#define F_CIM V_CIM(1U) + +#define S_MC7_CM 4 +#define V_MC7_CM(x) ((x) << S_MC7_CM) +#define F_MC7_CM V_MC7_CM(1U) + +#define S_MC7_PMTX 3 +#define V_MC7_PMTX(x) ((x) << S_MC7_PMTX) +#define F_MC7_PMTX V_MC7_PMTX(1U) + +#define S_MC7_PMRX 2 +#define V_MC7_PMRX(x) ((x) << S_MC7_PMRX) +#define F_MC7_PMRX V_MC7_PMRX(1U) + +#define S_PCIM0 1 +#define V_PCIM0(x) ((x) << S_PCIM0) +#define F_PCIM0 V_PCIM0(1U) + +#define S_SGE3 0 +#define V_SGE3(x) ((x) << S_SGE3) +#define F_SGE3 V_SGE3(1U) + +#define A_PL_INT_CAUSE0 0x6e4 + +#define A_PL_RST 0x6f0 + +#define S_FATALPERREN 4 +#define V_FATALPERREN(x) ((x) << S_FATALPERREN) +#define F_FATALPERREN V_FATALPERREN(1U) + +#define S_CRSTWRM 1 +#define V_CRSTWRM(x) ((x) << S_CRSTWRM) +#define F_CRSTWRM V_CRSTWRM(1U) + +#define A_PL_REV 0x6f4 + +#define A_PL_CLI 0x6f8 + +#define A_MC5_DB_CONFIG 0x704 + +#define S_TMTYPEHI 30 +#define V_TMTYPEHI(x) ((x) << S_TMTYPEHI) +#define F_TMTYPEHI V_TMTYPEHI(1U) + +#define S_TMPARTSIZE 28 +#define M_TMPARTSIZE 0x3 +#define V_TMPARTSIZE(x) ((x) << S_TMPARTSIZE) +#define G_TMPARTSIZE(x) (((x) >> S_TMPARTSIZE) & M_TMPARTSIZE) + +#define S_TMTYPE 26 +#define M_TMTYPE 0x3 +#define V_TMTYPE(x) ((x) << S_TMTYPE) +#define G_TMTYPE(x) (((x) >> S_TMTYPE) & M_TMTYPE) + +#define S_COMPEN 17 +#define V_COMPEN(x) ((x) << S_COMPEN) +#define F_COMPEN V_COMPEN(1U) + +#define S_PRTYEN 6 +#define V_PRTYEN(x) ((x) << S_PRTYEN) +#define F_PRTYEN V_PRTYEN(1U) + +#define S_MBUSEN 5 +#define V_MBUSEN(x) ((x) << S_MBUSEN) +#define F_MBUSEN V_MBUSEN(1U) + +#define S_DBGIEN 4 +#define V_DBGIEN(x) ((x) << S_DBGIEN) +#define F_DBGIEN V_DBGIEN(1U) + +#define S_TMRDY 2 +#define V_TMRDY(x) ((x) << S_TMRDY) +#define F_TMRDY V_TMRDY(1U) + +#define S_TMRST 1 +#define V_TMRST(x) ((x) << S_TMRST) +#define F_TMRST V_TMRST(1U) + +#define S_TMMODE 0 +#define V_TMMODE(x) ((x) << S_TMMODE) +#define F_TMMODE V_TMMODE(1U) + +#define A_MC5_DB_ROUTING_TABLE_INDEX 0x70c + +#define A_MC5_DB_FILTER_TABLE 0x710 + +#define A_MC5_DB_SERVER_INDEX 0x714 + +#define A_MC5_DB_RSP_LATENCY 0x720 + +#define S_RDLAT 16 +#define M_RDLAT 0x1f +#define V_RDLAT(x) ((x) << S_RDLAT) + +#define S_LRNLAT 8 +#define M_LRNLAT 0x1f +#define V_LRNLAT(x) ((x) << S_LRNLAT) + +#define S_SRCHLAT 0 +#define M_SRCHLAT 0x1f +#define V_SRCHLAT(x) ((x) << S_SRCHLAT) + +#define A_MC5_DB_PART_ID_INDEX 0x72c + +#define A_MC5_DB_INT_ENABLE 0x740 + +#define S_DELACTEMPTY 18 +#define V_DELACTEMPTY(x) ((x) << S_DELACTEMPTY) +#define F_DELACTEMPTY V_DELACTEMPTY(1U) + +#define S_DISPQPARERR 17 +#define V_DISPQPARERR(x) ((x) << S_DISPQPARERR) +#define F_DISPQPARERR V_DISPQPARERR(1U) + +#define S_REQQPARERR 16 +#define V_REQQPARERR(x) ((x) << S_REQQPARERR) +#define F_REQQPARERR V_REQQPARERR(1U) + +#define S_UNKNOWNCMD 15 +#define V_UNKNOWNCMD(x) ((x) << S_UNKNOWNCMD) +#define F_UNKNOWNCMD V_UNKNOWNCMD(1U) + +#define S_NFASRCHFAIL 8 +#define V_NFASRCHFAIL(x) ((x) << S_NFASRCHFAIL) +#define F_NFASRCHFAIL V_NFASRCHFAIL(1U) + +#define S_ACTRGNFULL 7 +#define V_ACTRGNFULL(x) ((x) << S_ACTRGNFULL) +#define F_ACTRGNFULL V_ACTRGNFULL(1U) + +#define S_PARITYERR 6 +#define V_PARITYERR(x) ((x) << S_PARITYERR) +#define F_PARITYERR V_PARITYERR(1U) + +#define A_MC5_DB_INT_CAUSE 0x744 + +#define A_MC5_DB_DBGI_CONFIG 0x774 + +#define A_MC5_DB_DBGI_REQ_CMD 0x778 + +#define A_MC5_DB_DBGI_REQ_ADDR0 0x77c + +#define A_MC5_DB_DBGI_REQ_ADDR1 0x780 + +#define A_MC5_DB_DBGI_REQ_ADDR2 0x784 + +#define A_MC5_DB_DBGI_REQ_DATA0 0x788 + +#define A_MC5_DB_DBGI_REQ_DATA1 0x78c + +#define A_MC5_DB_DBGI_REQ_DATA2 0x790 + +#define A_MC5_DB_DBGI_RSP_STATUS 0x7b0 + +#define S_DBGIRSPVALID 0 +#define V_DBGIRSPVALID(x) ((x) << S_DBGIRSPVALID) +#define F_DBGIRSPVALID V_DBGIRSPVALID(1U) + +#define A_MC5_DB_DBGI_RSP_DATA0 0x7b4 + +#define A_MC5_DB_DBGI_RSP_DATA1 0x7b8 + +#define A_MC5_DB_DBGI_RSP_DATA2 0x7bc + +#define A_MC5_DB_POPEN_DATA_WR_CMD 0x7cc + +#define A_MC5_DB_POPEN_MASK_WR_CMD 0x7d0 + +#define A_MC5_DB_AOPEN_SRCH_CMD 0x7d4 + +#define A_MC5_DB_AOPEN_LRN_CMD 0x7d8 + +#define A_MC5_DB_SYN_SRCH_CMD 0x7dc + +#define A_MC5_DB_SYN_LRN_CMD 0x7e0 + +#define A_MC5_DB_ACK_SRCH_CMD 0x7e4 + +#define A_MC5_DB_ACK_LRN_CMD 0x7e8 + +#define A_MC5_DB_ILOOKUP_CMD 0x7ec + +#define A_MC5_DB_ELOOKUP_CMD 0x7f0 + +#define A_MC5_DB_DATA_WRITE_CMD 0x7f4 + +#define A_MC5_DB_DATA_READ_CMD 0x7f8 + +#define XGMAC0_0_BASE_ADDR 0x800 + +#define A_XGM_TX_CTRL 0x800 + +#define S_TXEN 0 +#define V_TXEN(x) ((x) << S_TXEN) +#define F_TXEN V_TXEN(1U) + +#define A_XGM_TX_CFG 0x804 + +#define S_TXPAUSEEN 0 +#define V_TXPAUSEEN(x) ((x) << S_TXPAUSEEN) +#define F_TXPAUSEEN V_TXPAUSEEN(1U) + +#define A_XGM_TX_PAUSE_QUANTA 0x808 + +#define A_XGM_RX_CTRL 0x80c + +#define S_RXEN 0 +#define V_RXEN(x) ((x) << S_RXEN) +#define F_RXEN V_RXEN(1U) + +#define A_XGM_RX_CFG 0x810 + +#define S_DISPAUSEFRAMES 9 +#define V_DISPAUSEFRAMES(x) ((x) << S_DISPAUSEFRAMES) +#define F_DISPAUSEFRAMES V_DISPAUSEFRAMES(1U) + +#define S_EN1536BFRAMES 8 +#define V_EN1536BFRAMES(x) ((x) << S_EN1536BFRAMES) +#define F_EN1536BFRAMES V_EN1536BFRAMES(1U) + +#define S_ENJUMBO 7 +#define V_ENJUMBO(x) ((x) << S_ENJUMBO) +#define F_ENJUMBO V_ENJUMBO(1U) + +#define S_RMFCS 6 +#define V_RMFCS(x) ((x) << S_RMFCS) +#define F_RMFCS V_RMFCS(1U) + +#define S_ENHASHMCAST 2 +#define V_ENHASHMCAST(x) ((x) << S_ENHASHMCAST) +#define F_ENHASHMCAST V_ENHASHMCAST(1U) + +#define S_COPYALLFRAMES 0 +#define V_COPYALLFRAMES(x) ((x) << S_COPYALLFRAMES) +#define F_COPYALLFRAMES V_COPYALLFRAMES(1U) + +#define S_DISBCAST 1 +#define V_DISBCAST(x) ((x) << S_DISBCAST) +#define F_DISBCAST V_DISBCAST(1U) + +#define A_XGM_RX_HASH_LOW 0x814 + +#define A_XGM_RX_HASH_HIGH 0x818 + +#define A_XGM_RX_EXACT_MATCH_LOW_1 0x81c + +#define A_XGM_RX_EXACT_MATCH_HIGH_1 0x820 + +#define A_XGM_RX_EXACT_MATCH_LOW_2 0x824 + +#define A_XGM_RX_EXACT_MATCH_LOW_3 0x82c + +#define A_XGM_RX_EXACT_MATCH_LOW_4 0x834 + +#define A_XGM_RX_EXACT_MATCH_LOW_5 0x83c + +#define A_XGM_RX_EXACT_MATCH_LOW_6 0x844 + +#define A_XGM_RX_EXACT_MATCH_LOW_7 0x84c + +#define A_XGM_RX_EXACT_MATCH_LOW_8 0x854 + +#define A_XGM_INT_STATUS 0x86c + +#define S_LINKFAULTCHANGE 9 +#define V_LINKFAULTCHANGE(x) ((x) << S_LINKFAULTCHANGE) +#define F_LINKFAULTCHANGE V_LINKFAULTCHANGE(1U) + +#define A_XGM_XGM_INT_ENABLE 0x874 +#define A_XGM_XGM_INT_DISABLE 0x878 + +#define A_XGM_STAT_CTRL 0x880 + +#define S_CLRSTATS 2 +#define V_CLRSTATS(x) ((x) << S_CLRSTATS) +#define F_CLRSTATS V_CLRSTATS(1U) + +#define A_XGM_RXFIFO_CFG 0x884 + +#define S_RXFIFO_EMPTY 31 +#define V_RXFIFO_EMPTY(x) ((x) << S_RXFIFO_EMPTY) +#define F_RXFIFO_EMPTY V_RXFIFO_EMPTY(1U) + +#define S_RXFIFOPAUSEHWM 17 +#define M_RXFIFOPAUSEHWM 0xfff + +#define V_RXFIFOPAUSEHWM(x) ((x) << S_RXFIFOPAUSEHWM) + +#define G_RXFIFOPAUSEHWM(x) (((x) >> S_RXFIFOPAUSEHWM) & M_RXFIFOPAUSEHWM) + +#define S_RXFIFOPAUSELWM 5 +#define M_RXFIFOPAUSELWM 0xfff + +#define V_RXFIFOPAUSELWM(x) ((x) << S_RXFIFOPAUSELWM) + +#define G_RXFIFOPAUSELWM(x) (((x) >> S_RXFIFOPAUSELWM) & M_RXFIFOPAUSELWM) + +#define S_RXSTRFRWRD 1 +#define V_RXSTRFRWRD(x) ((x) << S_RXSTRFRWRD) +#define F_RXSTRFRWRD V_RXSTRFRWRD(1U) + +#define S_DISERRFRAMES 0 +#define V_DISERRFRAMES(x) ((x) << S_DISERRFRAMES) +#define F_DISERRFRAMES V_DISERRFRAMES(1U) + +#define A_XGM_TXFIFO_CFG 0x888 + +#define S_UNDERUNFIX 22 +#define V_UNDERUNFIX(x) ((x) << S_UNDERUNFIX) +#define F_UNDERUNFIX V_UNDERUNFIX(1U) + +#define S_TXIPG 13 +#define M_TXIPG 0xff +#define V_TXIPG(x) ((x) << S_TXIPG) +#define G_TXIPG(x) (((x) >> S_TXIPG) & M_TXIPG) + +#define S_TXFIFOTHRESH 4 +#define M_TXFIFOTHRESH 0x1ff + +#define V_TXFIFOTHRESH(x) ((x) << S_TXFIFOTHRESH) + +#define S_ENDROPPKT 21 +#define V_ENDROPPKT(x) ((x) << S_ENDROPPKT) +#define F_ENDROPPKT V_ENDROPPKT(1U) + +#define A_XGM_SERDES_CTRL 0x890 +#define A_XGM_SERDES_CTRL0 0x8e0 + +#define S_SERDESRESET_ 24 +#define V_SERDESRESET_(x) ((x) << S_SERDESRESET_) +#define F_SERDESRESET_ V_SERDESRESET_(1U) + +#define S_RXENABLE 4 +#define V_RXENABLE(x) ((x) << S_RXENABLE) +#define F_RXENABLE V_RXENABLE(1U) + +#define S_TXENABLE 3 +#define V_TXENABLE(x) ((x) << S_TXENABLE) +#define F_TXENABLE V_TXENABLE(1U) + +#define A_XGM_PAUSE_TIMER 0x890 + +#define A_XGM_RGMII_IMP 0x89c + +#define S_XGM_IMPSETUPDATE 6 +#define V_XGM_IMPSETUPDATE(x) ((x) << S_XGM_IMPSETUPDATE) +#define F_XGM_IMPSETUPDATE V_XGM_IMPSETUPDATE(1U) + +#define S_RGMIIIMPPD 3 +#define M_RGMIIIMPPD 0x7 +#define V_RGMIIIMPPD(x) ((x) << S_RGMIIIMPPD) + +#define S_RGMIIIMPPU 0 +#define M_RGMIIIMPPU 0x7 +#define V_RGMIIIMPPU(x) ((x) << S_RGMIIIMPPU) + +#define S_CALRESET 8 +#define V_CALRESET(x) ((x) << S_CALRESET) +#define F_CALRESET V_CALRESET(1U) + +#define S_CALUPDATE 7 +#define V_CALUPDATE(x) ((x) << S_CALUPDATE) +#define F_CALUPDATE V_CALUPDATE(1U) + +#define A_XGM_XAUI_IMP 0x8a0 + +#define S_CALBUSY 31 +#define V_CALBUSY(x) ((x) << S_CALBUSY) +#define F_CALBUSY V_CALBUSY(1U) + +#define S_XGM_CALFAULT 29 +#define V_XGM_CALFAULT(x) ((x) << S_XGM_CALFAULT) +#define F_XGM_CALFAULT V_XGM_CALFAULT(1U) + +#define S_CALIMP 24 +#define M_CALIMP 0x1f +#define V_CALIMP(x) ((x) << S_CALIMP) +#define G_CALIMP(x) (((x) >> S_CALIMP) & M_CALIMP) + +#define S_XAUIIMP 0 +#define M_XAUIIMP 0x7 +#define V_XAUIIMP(x) ((x) << S_XAUIIMP) + +#define A_XGM_RX_MAX_PKT_SIZE 0x8a8 + +#define S_RXMAXFRAMERSIZE 17 +#define M_RXMAXFRAMERSIZE 0x3fff +#define V_RXMAXFRAMERSIZE(x) ((x) << S_RXMAXFRAMERSIZE) +#define G_RXMAXFRAMERSIZE(x) (((x) >> S_RXMAXFRAMERSIZE) & M_RXMAXFRAMERSIZE) + +#define S_RXENFRAMER 14 +#define V_RXENFRAMER(x) ((x) << S_RXENFRAMER) +#define F_RXENFRAMER V_RXENFRAMER(1U) + +#define S_RXMAXPKTSIZE 0 +#define M_RXMAXPKTSIZE 0x3fff +#define V_RXMAXPKTSIZE(x) ((x) << S_RXMAXPKTSIZE) +#define G_RXMAXPKTSIZE(x) (((x) >> S_RXMAXPKTSIZE) & M_RXMAXPKTSIZE) + +#define A_XGM_RESET_CTRL 0x8ac + +#define S_XGMAC_STOP_EN 4 +#define V_XGMAC_STOP_EN(x) ((x) << S_XGMAC_STOP_EN) +#define F_XGMAC_STOP_EN V_XGMAC_STOP_EN(1U) + +#define S_XG2G_RESET_ 3 +#define V_XG2G_RESET_(x) ((x) << S_XG2G_RESET_) +#define F_XG2G_RESET_ V_XG2G_RESET_(1U) + +#define S_RGMII_RESET_ 2 +#define V_RGMII_RESET_(x) ((x) << S_RGMII_RESET_) +#define F_RGMII_RESET_ V_RGMII_RESET_(1U) + +#define S_PCS_RESET_ 1 +#define V_PCS_RESET_(x) ((x) << S_PCS_RESET_) +#define F_PCS_RESET_ V_PCS_RESET_(1U) + +#define S_MAC_RESET_ 0 +#define V_MAC_RESET_(x) ((x) << S_MAC_RESET_) +#define F_MAC_RESET_ V_MAC_RESET_(1U) + +#define A_XGM_PORT_CFG 0x8b8 + +#define S_CLKDIVRESET_ 3 +#define V_CLKDIVRESET_(x) ((x) << S_CLKDIVRESET_) +#define F_CLKDIVRESET_ V_CLKDIVRESET_(1U) + +#define S_PORTSPEED 1 +#define M_PORTSPEED 0x3 + +#define V_PORTSPEED(x) ((x) << S_PORTSPEED) + +#define S_ENRGMII 0 +#define V_ENRGMII(x) ((x) << S_ENRGMII) +#define F_ENRGMII V_ENRGMII(1U) + +#define A_XGM_INT_ENABLE 0x8d4 + +#define S_TXFIFO_PRTY_ERR 17 +#define M_TXFIFO_PRTY_ERR 0x7 + +#define V_TXFIFO_PRTY_ERR(x) ((x) << S_TXFIFO_PRTY_ERR) + +#define S_RXFIFO_PRTY_ERR 14 +#define M_RXFIFO_PRTY_ERR 0x7 + +#define V_RXFIFO_PRTY_ERR(x) ((x) << S_RXFIFO_PRTY_ERR) + +#define S_TXFIFO_UNDERRUN 13 +#define V_TXFIFO_UNDERRUN(x) ((x) << S_TXFIFO_UNDERRUN) +#define F_TXFIFO_UNDERRUN V_TXFIFO_UNDERRUN(1U) + +#define S_RXFIFO_OVERFLOW 12 +#define V_RXFIFO_OVERFLOW(x) ((x) << S_RXFIFO_OVERFLOW) +#define F_RXFIFO_OVERFLOW V_RXFIFO_OVERFLOW(1U) + +#define S_SERDES_LOS 4 +#define M_SERDES_LOS 0xf + +#define V_SERDES_LOS(x) ((x) << S_SERDES_LOS) + +#define S_XAUIPCSCTCERR 3 +#define V_XAUIPCSCTCERR(x) ((x) << S_XAUIPCSCTCERR) +#define F_XAUIPCSCTCERR V_XAUIPCSCTCERR(1U) + +#define S_XAUIPCSALIGNCHANGE 2 +#define V_XAUIPCSALIGNCHANGE(x) ((x) << S_XAUIPCSALIGNCHANGE) +#define F_XAUIPCSALIGNCHANGE V_XAUIPCSALIGNCHANGE(1U) + +#define S_XGM_INT 0 +#define V_XGM_INT(x) ((x) << S_XGM_INT) +#define F_XGM_INT V_XGM_INT(1U) + +#define A_XGM_INT_CAUSE 0x8d8 + +#define A_XGM_XAUI_ACT_CTRL 0x8dc + +#define S_TXACTENABLE 1 +#define V_TXACTENABLE(x) ((x) << S_TXACTENABLE) +#define F_TXACTENABLE V_TXACTENABLE(1U) + +#define S_RESET3 23 +#define V_RESET3(x) ((x) << S_RESET3) +#define F_RESET3 V_RESET3(1U) + +#define S_RESET2 22 +#define V_RESET2(x) ((x) << S_RESET2) +#define F_RESET2 V_RESET2(1U) + +#define S_RESET1 21 +#define V_RESET1(x) ((x) << S_RESET1) +#define F_RESET1 V_RESET1(1U) + +#define S_RESET0 20 +#define V_RESET0(x) ((x) << S_RESET0) +#define F_RESET0 V_RESET0(1U) + +#define S_PWRDN3 19 +#define V_PWRDN3(x) ((x) << S_PWRDN3) +#define F_PWRDN3 V_PWRDN3(1U) + +#define S_PWRDN2 18 +#define V_PWRDN2(x) ((x) << S_PWRDN2) +#define F_PWRDN2 V_PWRDN2(1U) + +#define S_PWRDN1 17 +#define V_PWRDN1(x) ((x) << S_PWRDN1) +#define F_PWRDN1 V_PWRDN1(1U) + +#define S_PWRDN0 16 +#define V_PWRDN0(x) ((x) << S_PWRDN0) +#define F_PWRDN0 V_PWRDN0(1U) + +#define S_RESETPLL23 15 +#define V_RESETPLL23(x) ((x) << S_RESETPLL23) +#define F_RESETPLL23 V_RESETPLL23(1U) + +#define S_RESETPLL01 14 +#define V_RESETPLL01(x) ((x) << S_RESETPLL01) +#define F_RESETPLL01 V_RESETPLL01(1U) + +#define A_XGM_SERDES_STAT0 0x8f0 +#define A_XGM_SERDES_STAT1 0x8f4 +#define A_XGM_SERDES_STAT2 0x8f8 + +#define S_LOWSIG0 0 +#define V_LOWSIG0(x) ((x) << S_LOWSIG0) +#define F_LOWSIG0 V_LOWSIG0(1U) + +#define A_XGM_SERDES_STAT3 0x8fc + +#define A_XGM_STAT_TX_BYTE_LOW 0x900 + +#define A_XGM_STAT_TX_BYTE_HIGH 0x904 + +#define A_XGM_STAT_TX_FRAME_LOW 0x908 + +#define A_XGM_STAT_TX_FRAME_HIGH 0x90c + +#define A_XGM_STAT_TX_BCAST 0x910 + +#define A_XGM_STAT_TX_MCAST 0x914 + +#define A_XGM_STAT_TX_PAUSE 0x918 + +#define A_XGM_STAT_TX_64B_FRAMES 0x91c + +#define A_XGM_STAT_TX_65_127B_FRAMES 0x920 + +#define A_XGM_STAT_TX_128_255B_FRAMES 0x924 + +#define A_XGM_STAT_TX_256_511B_FRAMES 0x928 + +#define A_XGM_STAT_TX_512_1023B_FRAMES 0x92c + +#define A_XGM_STAT_TX_1024_1518B_FRAMES 0x930 + +#define A_XGM_STAT_TX_1519_MAXB_FRAMES 0x934 + +#define A_XGM_STAT_TX_ERR_FRAMES 0x938 + +#define A_XGM_STAT_RX_BYTES_LOW 0x93c + +#define A_XGM_STAT_RX_BYTES_HIGH 0x940 + +#define A_XGM_STAT_RX_FRAMES_LOW 0x944 + +#define A_XGM_STAT_RX_FRAMES_HIGH 0x948 + +#define A_XGM_STAT_RX_BCAST_FRAMES 0x94c + +#define A_XGM_STAT_RX_MCAST_FRAMES 0x950 + +#define A_XGM_STAT_RX_PAUSE_FRAMES 0x954 + +#define A_XGM_STAT_RX_64B_FRAMES 0x958 + +#define A_XGM_STAT_RX_65_127B_FRAMES 0x95c + +#define A_XGM_STAT_RX_128_255B_FRAMES 0x960 + +#define A_XGM_STAT_RX_256_511B_FRAMES 0x964 + +#define A_XGM_STAT_RX_512_1023B_FRAMES 0x968 + +#define A_XGM_STAT_RX_1024_1518B_FRAMES 0x96c + +#define A_XGM_STAT_RX_1519_MAXB_FRAMES 0x970 + +#define A_XGM_STAT_RX_SHORT_FRAMES 0x974 + +#define A_XGM_STAT_RX_OVERSIZE_FRAMES 0x978 + +#define A_XGM_STAT_RX_JABBER_FRAMES 0x97c + +#define A_XGM_STAT_RX_CRC_ERR_FRAMES 0x980 + +#define A_XGM_STAT_RX_LENGTH_ERR_FRAMES 0x984 + +#define A_XGM_STAT_RX_SYM_CODE_ERR_FRAMES 0x988 + +#define A_XGM_SERDES_STATUS0 0x98c + +#define A_XGM_SERDES_STATUS1 0x990 + +#define S_CMULOCK 31 +#define V_CMULOCK(x) ((x) << S_CMULOCK) +#define F_CMULOCK V_CMULOCK(1U) + +#define A_XGM_RX_MAX_PKT_SIZE_ERR_CNT 0x9a4 + +#define A_XGM_TX_SPI4_SOP_EOP_CNT 0x9a8 + +#define S_TXSPI4SOPCNT 16 +#define M_TXSPI4SOPCNT 0xffff +#define V_TXSPI4SOPCNT(x) ((x) << S_TXSPI4SOPCNT) +#define G_TXSPI4SOPCNT(x) (((x) >> S_TXSPI4SOPCNT) & M_TXSPI4SOPCNT) + +#define A_XGM_RX_SPI4_SOP_EOP_CNT 0x9ac + +#define XGMAC0_1_BASE_ADDR 0xa00 diff --git a/drivers/net/ethernet/chelsio/cxgb3/sge.c b/drivers/net/ethernet/chelsio/cxgb3/sge.c new file mode 100644 index 000000000..62dfbdd33 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/sge.c @@ -0,0 +1,3389 @@ +/* + * Copyright (c) 2005-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include <linux/skbuff.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> +#include <linux/if_vlan.h> +#include <linux/ip.h> +#include <linux/tcp.h> +#include <linux/dma-mapping.h> +#include <linux/slab.h> +#include <linux/prefetch.h> +#include <net/arp.h> +#include "common.h" +#include "regs.h" +#include "sge_defs.h" +#include "t3_cpl.h" +#include "firmware_exports.h" +#include "cxgb3_offload.h" + +#define USE_GTS 0 + +#define SGE_RX_SM_BUF_SIZE 1536 + +#define SGE_RX_COPY_THRES 256 +#define SGE_RX_PULL_LEN 128 + +#define SGE_PG_RSVD SMP_CACHE_BYTES +/* + * Page chunk size for FL0 buffers if FL0 is to be populated with page chunks. + * It must be a divisor of PAGE_SIZE. If set to 0 FL0 will use sk_buffs + * directly. + */ +#define FL0_PG_CHUNK_SIZE 2048 +#define FL0_PG_ORDER 0 +#define FL0_PG_ALLOC_SIZE (PAGE_SIZE << FL0_PG_ORDER) +#define FL1_PG_CHUNK_SIZE (PAGE_SIZE > 8192 ? 16384 : 8192) +#define FL1_PG_ORDER (PAGE_SIZE > 8192 ? 0 : 1) +#define FL1_PG_ALLOC_SIZE (PAGE_SIZE << FL1_PG_ORDER) + +#define SGE_RX_DROP_THRES 16 +#define RX_RECLAIM_PERIOD (HZ/4) + +/* + * Max number of Rx buffers we replenish at a time. + */ +#define MAX_RX_REFILL 16U +/* + * Period of the Tx buffer reclaim timer. This timer does not need to run + * frequently as Tx buffers are usually reclaimed by new Tx packets. + */ +#define TX_RECLAIM_PERIOD (HZ / 4) +#define TX_RECLAIM_TIMER_CHUNK 64U +#define TX_RECLAIM_CHUNK 16U + +/* WR size in bytes */ +#define WR_LEN (WR_FLITS * 8) + +/* + * Types of Tx queues in each queue set. Order here matters, do not change. + */ +enum { TXQ_ETH, TXQ_OFLD, TXQ_CTRL }; + +/* Values for sge_txq.flags */ +enum { + TXQ_RUNNING = 1 << 0, /* fetch engine is running */ + TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */ +}; + +struct tx_desc { + __be64 flit[TX_DESC_FLITS]; +}; + +struct rx_desc { + __be32 addr_lo; + __be32 len_gen; + __be32 gen2; + __be32 addr_hi; +}; + +struct tx_sw_desc { /* SW state per Tx descriptor */ + struct sk_buff *skb; + u8 eop; /* set if last descriptor for packet */ + u8 addr_idx; /* buffer index of first SGL entry in descriptor */ + u8 fragidx; /* first page fragment associated with descriptor */ + s8 sflit; /* start flit of first SGL entry in descriptor */ +}; + +struct rx_sw_desc { /* SW state per Rx descriptor */ + union { + struct sk_buff *skb; + struct fl_pg_chunk pg_chunk; + }; + DEFINE_DMA_UNMAP_ADDR(dma_addr); +}; + +struct rsp_desc { /* response queue descriptor */ + struct rss_header rss_hdr; + __be32 flags; + __be32 len_cq; + struct_group(immediate, + u8 imm_data[47]; + u8 intr_gen; + ); +}; + +/* + * Holds unmapping information for Tx packets that need deferred unmapping. + * This structure lives at skb->head and must be allocated by callers. + */ +struct deferred_unmap_info { + struct pci_dev *pdev; + dma_addr_t addr[MAX_SKB_FRAGS + 1]; +}; + +/* + * Maps a number of flits to the number of Tx descriptors that can hold them. + * The formula is + * + * desc = 1 + (flits - 2) / (WR_FLITS - 1). + * + * HW allows up to 4 descriptors to be combined into a WR. + */ +static u8 flit_desc_map[] = { + 0, +#if SGE_NUM_GENBITS == 1 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 +#elif SGE_NUM_GENBITS == 2 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, +#else +# error "SGE_NUM_GENBITS must be 1 or 2" +#endif +}; + +static inline struct sge_qset *fl_to_qset(const struct sge_fl *q, int qidx) +{ + return container_of(q, struct sge_qset, fl[qidx]); +} + +static inline struct sge_qset *rspq_to_qset(const struct sge_rspq *q) +{ + return container_of(q, struct sge_qset, rspq); +} + +static inline struct sge_qset *txq_to_qset(const struct sge_txq *q, int qidx) +{ + return container_of(q, struct sge_qset, txq[qidx]); +} + +/** + * refill_rspq - replenish an SGE response queue + * @adapter: the adapter + * @q: the response queue to replenish + * @credits: how many new responses to make available + * + * Replenishes a response queue by making the supplied number of responses + * available to HW. + */ +static inline void refill_rspq(struct adapter *adapter, + const struct sge_rspq *q, unsigned int credits) +{ + rmb(); + t3_write_reg(adapter, A_SG_RSPQ_CREDIT_RETURN, + V_RSPQ(q->cntxt_id) | V_CREDITS(credits)); +} + +/** + * need_skb_unmap - does the platform need unmapping of sk_buffs? + * + * Returns true if the platform needs sk_buff unmapping. The compiler + * optimizes away unnecessary code if this returns true. + */ +static inline int need_skb_unmap(void) +{ +#ifdef CONFIG_NEED_DMA_MAP_STATE + return 1; +#else + return 0; +#endif +} + +/** + * unmap_skb - unmap a packet main body and its page fragments + * @skb: the packet + * @q: the Tx queue containing Tx descriptors for the packet + * @cidx: index of Tx descriptor + * @pdev: the PCI device + * + * Unmap the main body of an sk_buff and its page fragments, if any. + * Because of the fairly complicated structure of our SGLs and the desire + * to conserve space for metadata, the information necessary to unmap an + * sk_buff is spread across the sk_buff itself (buffer lengths), the HW Tx + * descriptors (the physical addresses of the various data buffers), and + * the SW descriptor state (assorted indices). The send functions + * initialize the indices for the first packet descriptor so we can unmap + * the buffers held in the first Tx descriptor here, and we have enough + * information at this point to set the state for the next Tx descriptor. + * + * Note that it is possible to clean up the first descriptor of a packet + * before the send routines have written the next descriptors, but this + * race does not cause any problem. We just end up writing the unmapping + * info for the descriptor first. + */ +static inline void unmap_skb(struct sk_buff *skb, struct sge_txq *q, + unsigned int cidx, struct pci_dev *pdev) +{ + const struct sg_ent *sgp; + struct tx_sw_desc *d = &q->sdesc[cidx]; + int nfrags, frag_idx, curflit, j = d->addr_idx; + + sgp = (struct sg_ent *)&q->desc[cidx].flit[d->sflit]; + frag_idx = d->fragidx; + + if (frag_idx == 0 && skb_headlen(skb)) { + dma_unmap_single(&pdev->dev, be64_to_cpu(sgp->addr[0]), + skb_headlen(skb), DMA_TO_DEVICE); + j = 1; + } + + curflit = d->sflit + 1 + j; + nfrags = skb_shinfo(skb)->nr_frags; + + while (frag_idx < nfrags && curflit < WR_FLITS) { + dma_unmap_page(&pdev->dev, be64_to_cpu(sgp->addr[j]), + skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]), + DMA_TO_DEVICE); + j ^= 1; + if (j == 0) { + sgp++; + curflit++; + } + curflit++; + frag_idx++; + } + + if (frag_idx < nfrags) { /* SGL continues into next Tx descriptor */ + d = cidx + 1 == q->size ? q->sdesc : d + 1; + d->fragidx = frag_idx; + d->addr_idx = j; + d->sflit = curflit - WR_FLITS - j; /* sflit can be -1 */ + } +} + +/** + * free_tx_desc - reclaims Tx descriptors and their buffers + * @adapter: the adapter + * @q: the Tx queue to reclaim descriptors from + * @n: the number of descriptors to reclaim + * + * Reclaims Tx descriptors from an SGE Tx queue and frees the associated + * Tx buffers. Called with the Tx queue lock held. + */ +static void free_tx_desc(struct adapter *adapter, struct sge_txq *q, + unsigned int n) +{ + struct tx_sw_desc *d; + struct pci_dev *pdev = adapter->pdev; + unsigned int cidx = q->cidx; + + const int need_unmap = need_skb_unmap() && + q->cntxt_id >= FW_TUNNEL_SGEEC_START; + + d = &q->sdesc[cidx]; + while (n--) { + if (d->skb) { /* an SGL is present */ + if (need_unmap) + unmap_skb(d->skb, q, cidx, pdev); + if (d->eop) { + dev_consume_skb_any(d->skb); + d->skb = NULL; + } + } + ++d; + if (++cidx == q->size) { + cidx = 0; + d = q->sdesc; + } + } + q->cidx = cidx; +} + +/** + * reclaim_completed_tx - reclaims completed Tx descriptors + * @adapter: the adapter + * @q: the Tx queue to reclaim completed descriptors from + * @chunk: maximum number of descriptors to reclaim + * + * Reclaims Tx descriptors that the SGE has indicated it has processed, + * and frees the associated buffers if possible. Called with the Tx + * queue's lock held. + */ +static inline unsigned int reclaim_completed_tx(struct adapter *adapter, + struct sge_txq *q, + unsigned int chunk) +{ + unsigned int reclaim = q->processed - q->cleaned; + + reclaim = min(chunk, reclaim); + if (reclaim) { + free_tx_desc(adapter, q, reclaim); + q->cleaned += reclaim; + q->in_use -= reclaim; + } + return q->processed - q->cleaned; +} + +/** + * should_restart_tx - are there enough resources to restart a Tx queue? + * @q: the Tx queue + * + * Checks if there are enough descriptors to restart a suspended Tx queue. + */ +static inline int should_restart_tx(const struct sge_txq *q) +{ + unsigned int r = q->processed - q->cleaned; + + return q->in_use - r < (q->size >> 1); +} + +static void clear_rx_desc(struct pci_dev *pdev, const struct sge_fl *q, + struct rx_sw_desc *d) +{ + if (q->use_pages && d->pg_chunk.page) { + (*d->pg_chunk.p_cnt)--; + if (!*d->pg_chunk.p_cnt) + dma_unmap_page(&pdev->dev, d->pg_chunk.mapping, + q->alloc_size, DMA_FROM_DEVICE); + + put_page(d->pg_chunk.page); + d->pg_chunk.page = NULL; + } else { + dma_unmap_single(&pdev->dev, dma_unmap_addr(d, dma_addr), + q->buf_size, DMA_FROM_DEVICE); + kfree_skb(d->skb); + d->skb = NULL; + } +} + +/** + * free_rx_bufs - free the Rx buffers on an SGE free list + * @pdev: the PCI device associated with the adapter + * @q: the SGE free list to clean up + * + * Release the buffers on an SGE free-buffer Rx queue. HW fetching from + * this queue should be stopped before calling this function. + */ +static void free_rx_bufs(struct pci_dev *pdev, struct sge_fl *q) +{ + unsigned int cidx = q->cidx; + + while (q->credits--) { + struct rx_sw_desc *d = &q->sdesc[cidx]; + + + clear_rx_desc(pdev, q, d); + if (++cidx == q->size) + cidx = 0; + } + + if (q->pg_chunk.page) { + __free_pages(q->pg_chunk.page, q->order); + q->pg_chunk.page = NULL; + } +} + +/** + * add_one_rx_buf - add a packet buffer to a free-buffer list + * @va: buffer start VA + * @len: the buffer length + * @d: the HW Rx descriptor to write + * @sd: the SW Rx descriptor to write + * @gen: the generation bit value + * @pdev: the PCI device associated with the adapter + * + * Add a buffer of the given length to the supplied HW and SW Rx + * descriptors. + */ +static inline int add_one_rx_buf(void *va, unsigned int len, + struct rx_desc *d, struct rx_sw_desc *sd, + unsigned int gen, struct pci_dev *pdev) +{ + dma_addr_t mapping; + + mapping = dma_map_single(&pdev->dev, va, len, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(&pdev->dev, mapping))) + return -ENOMEM; + + dma_unmap_addr_set(sd, dma_addr, mapping); + + d->addr_lo = cpu_to_be32(mapping); + d->addr_hi = cpu_to_be32((u64) mapping >> 32); + dma_wmb(); + d->len_gen = cpu_to_be32(V_FLD_GEN1(gen)); + d->gen2 = cpu_to_be32(V_FLD_GEN2(gen)); + return 0; +} + +static inline int add_one_rx_chunk(dma_addr_t mapping, struct rx_desc *d, + unsigned int gen) +{ + d->addr_lo = cpu_to_be32(mapping); + d->addr_hi = cpu_to_be32((u64) mapping >> 32); + dma_wmb(); + d->len_gen = cpu_to_be32(V_FLD_GEN1(gen)); + d->gen2 = cpu_to_be32(V_FLD_GEN2(gen)); + return 0; +} + +static int alloc_pg_chunk(struct adapter *adapter, struct sge_fl *q, + struct rx_sw_desc *sd, gfp_t gfp, + unsigned int order) +{ + if (!q->pg_chunk.page) { + dma_addr_t mapping; + + q->pg_chunk.page = alloc_pages(gfp, order); + if (unlikely(!q->pg_chunk.page)) + return -ENOMEM; + q->pg_chunk.va = page_address(q->pg_chunk.page); + q->pg_chunk.p_cnt = q->pg_chunk.va + (PAGE_SIZE << order) - + SGE_PG_RSVD; + q->pg_chunk.offset = 0; + mapping = dma_map_page(&adapter->pdev->dev, q->pg_chunk.page, + 0, q->alloc_size, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(&adapter->pdev->dev, mapping))) { + __free_pages(q->pg_chunk.page, order); + q->pg_chunk.page = NULL; + return -EIO; + } + q->pg_chunk.mapping = mapping; + } + sd->pg_chunk = q->pg_chunk; + + prefetch(sd->pg_chunk.p_cnt); + + q->pg_chunk.offset += q->buf_size; + if (q->pg_chunk.offset == (PAGE_SIZE << order)) + q->pg_chunk.page = NULL; + else { + q->pg_chunk.va += q->buf_size; + get_page(q->pg_chunk.page); + } + + if (sd->pg_chunk.offset == 0) + *sd->pg_chunk.p_cnt = 1; + else + *sd->pg_chunk.p_cnt += 1; + + return 0; +} + +static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q) +{ + if (q->pend_cred >= q->credits / 4) { + q->pend_cred = 0; + wmb(); + t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); + } +} + +/** + * refill_fl - refill an SGE free-buffer list + * @adap: the adapter + * @q: the free-list to refill + * @n: the number of new buffers to allocate + * @gfp: the gfp flags for allocating new buffers + * + * (Re)populate an SGE free-buffer list with up to @n new packet buffers, + * allocated with the supplied gfp flags. The caller must assure that + * @n does not exceed the queue's capacity. + */ +static int refill_fl(struct adapter *adap, struct sge_fl *q, int n, gfp_t gfp) +{ + struct rx_sw_desc *sd = &q->sdesc[q->pidx]; + struct rx_desc *d = &q->desc[q->pidx]; + unsigned int count = 0; + + while (n--) { + dma_addr_t mapping; + int err; + + if (q->use_pages) { + if (unlikely(alloc_pg_chunk(adap, q, sd, gfp, + q->order))) { +nomem: q->alloc_failed++; + break; + } + mapping = sd->pg_chunk.mapping + sd->pg_chunk.offset; + dma_unmap_addr_set(sd, dma_addr, mapping); + + add_one_rx_chunk(mapping, d, q->gen); + dma_sync_single_for_device(&adap->pdev->dev, mapping, + q->buf_size - SGE_PG_RSVD, + DMA_FROM_DEVICE); + } else { + void *buf_start; + + struct sk_buff *skb = alloc_skb(q->buf_size, gfp); + if (!skb) + goto nomem; + + sd->skb = skb; + buf_start = skb->data; + err = add_one_rx_buf(buf_start, q->buf_size, d, sd, + q->gen, adap->pdev); + if (unlikely(err)) { + clear_rx_desc(adap->pdev, q, sd); + break; + } + } + + d++; + sd++; + if (++q->pidx == q->size) { + q->pidx = 0; + q->gen ^= 1; + sd = q->sdesc; + d = q->desc; + } + count++; + } + + q->credits += count; + q->pend_cred += count; + ring_fl_db(adap, q); + + return count; +} + +static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl) +{ + refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits), + GFP_ATOMIC | __GFP_COMP); +} + +/** + * recycle_rx_buf - recycle a receive buffer + * @adap: the adapter + * @q: the SGE free list + * @idx: index of buffer to recycle + * + * Recycles the specified buffer on the given free list by adding it at + * the next available slot on the list. + */ +static void recycle_rx_buf(struct adapter *adap, struct sge_fl *q, + unsigned int idx) +{ + struct rx_desc *from = &q->desc[idx]; + struct rx_desc *to = &q->desc[q->pidx]; + + q->sdesc[q->pidx] = q->sdesc[idx]; + to->addr_lo = from->addr_lo; /* already big endian */ + to->addr_hi = from->addr_hi; /* likewise */ + dma_wmb(); + to->len_gen = cpu_to_be32(V_FLD_GEN1(q->gen)); + to->gen2 = cpu_to_be32(V_FLD_GEN2(q->gen)); + + if (++q->pidx == q->size) { + q->pidx = 0; + q->gen ^= 1; + } + + q->credits++; + q->pend_cred++; + ring_fl_db(adap, q); +} + +/** + * alloc_ring - allocate resources for an SGE descriptor ring + * @pdev: the PCI device + * @nelem: the number of descriptors + * @elem_size: the size of each descriptor + * @sw_size: the size of the SW state associated with each ring element + * @phys: the physical address of the allocated ring + * @metadata: address of the array holding the SW state for the ring + * + * Allocates resources for an SGE descriptor ring, such as Tx queues, + * free buffer lists, or response queues. Each SGE ring requires + * space for its HW descriptors plus, optionally, space for the SW state + * associated with each HW entry (the metadata). The function returns + * three values: the virtual address for the HW ring (the return value + * of the function), the physical address of the HW ring, and the address + * of the SW ring. + */ +static void *alloc_ring(struct pci_dev *pdev, size_t nelem, size_t elem_size, + size_t sw_size, dma_addr_t * phys, void *metadata) +{ + size_t len = nelem * elem_size; + void *s = NULL; + void *p = dma_alloc_coherent(&pdev->dev, len, phys, GFP_KERNEL); + + if (!p) + return NULL; + if (sw_size && metadata) { + s = kcalloc(nelem, sw_size, GFP_KERNEL); + + if (!s) { + dma_free_coherent(&pdev->dev, len, p, *phys); + return NULL; + } + *(void **)metadata = s; + } + return p; +} + +/** + * t3_reset_qset - reset a sge qset + * @q: the queue set + * + * Reset the qset structure. + * the NAPI structure is preserved in the event of + * the qset's reincarnation, for example during EEH recovery. + */ +static void t3_reset_qset(struct sge_qset *q) +{ + if (q->adap && + !(q->adap->flags & NAPI_INIT)) { + memset(q, 0, sizeof(*q)); + return; + } + + q->adap = NULL; + memset(&q->rspq, 0, sizeof(q->rspq)); + memset(q->fl, 0, sizeof(struct sge_fl) * SGE_RXQ_PER_SET); + memset(q->txq, 0, sizeof(struct sge_txq) * SGE_TXQ_PER_SET); + q->txq_stopped = 0; + q->tx_reclaim_timer.function = NULL; /* for t3_stop_sge_timers() */ + q->rx_reclaim_timer.function = NULL; + q->nomem = 0; + napi_free_frags(&q->napi); +} + + +/** + * t3_free_qset - free the resources of an SGE queue set + * @adapter: the adapter owning the queue set + * @q: the queue set + * + * Release the HW and SW resources associated with an SGE queue set, such + * as HW contexts, packet buffers, and descriptor rings. Traffic to the + * queue set must be quiesced prior to calling this. + */ +static void t3_free_qset(struct adapter *adapter, struct sge_qset *q) +{ + int i; + struct pci_dev *pdev = adapter->pdev; + + for (i = 0; i < SGE_RXQ_PER_SET; ++i) + if (q->fl[i].desc) { + spin_lock_irq(&adapter->sge.reg_lock); + t3_sge_disable_fl(adapter, q->fl[i].cntxt_id); + spin_unlock_irq(&adapter->sge.reg_lock); + free_rx_bufs(pdev, &q->fl[i]); + kfree(q->fl[i].sdesc); + dma_free_coherent(&pdev->dev, + q->fl[i].size * + sizeof(struct rx_desc), q->fl[i].desc, + q->fl[i].phys_addr); + } + + for (i = 0; i < SGE_TXQ_PER_SET; ++i) + if (q->txq[i].desc) { + spin_lock_irq(&adapter->sge.reg_lock); + t3_sge_enable_ecntxt(adapter, q->txq[i].cntxt_id, 0); + spin_unlock_irq(&adapter->sge.reg_lock); + if (q->txq[i].sdesc) { + free_tx_desc(adapter, &q->txq[i], + q->txq[i].in_use); + kfree(q->txq[i].sdesc); + } + dma_free_coherent(&pdev->dev, + q->txq[i].size * + sizeof(struct tx_desc), + q->txq[i].desc, q->txq[i].phys_addr); + __skb_queue_purge(&q->txq[i].sendq); + } + + if (q->rspq.desc) { + spin_lock_irq(&adapter->sge.reg_lock); + t3_sge_disable_rspcntxt(adapter, q->rspq.cntxt_id); + spin_unlock_irq(&adapter->sge.reg_lock); + dma_free_coherent(&pdev->dev, + q->rspq.size * sizeof(struct rsp_desc), + q->rspq.desc, q->rspq.phys_addr); + } + + t3_reset_qset(q); +} + +/** + * init_qset_cntxt - initialize an SGE queue set context info + * @qs: the queue set + * @id: the queue set id + * + * Initializes the TIDs and context ids for the queues of a queue set. + */ +static void init_qset_cntxt(struct sge_qset *qs, unsigned int id) +{ + qs->rspq.cntxt_id = id; + qs->fl[0].cntxt_id = 2 * id; + qs->fl[1].cntxt_id = 2 * id + 1; + qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id; + qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id; + qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id; + qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id; + qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id; +} + +/** + * sgl_len - calculates the size of an SGL of the given capacity + * @n: the number of SGL entries + * + * Calculates the number of flits needed for a scatter/gather list that + * can hold the given number of entries. + */ +static inline unsigned int sgl_len(unsigned int n) +{ + /* alternatively: 3 * (n / 2) + 2 * (n & 1) */ + return (3 * n) / 2 + (n & 1); +} + +/** + * flits_to_desc - returns the num of Tx descriptors for the given flits + * @n: the number of flits + * + * Calculates the number of Tx descriptors needed for the supplied number + * of flits. + */ +static inline unsigned int flits_to_desc(unsigned int n) +{ + BUG_ON(n >= ARRAY_SIZE(flit_desc_map)); + return flit_desc_map[n]; +} + +/** + * get_packet - return the next ingress packet buffer from a free list + * @adap: the adapter that received the packet + * @fl: the SGE free list holding the packet + * @len: the packet length including any SGE padding + * @drop_thres: # of remaining buffers before we start dropping packets + * + * Get the next packet from a free list and complete setup of the + * sk_buff. If the packet is small we make a copy and recycle the + * original buffer, otherwise we use the original buffer itself. If a + * positive drop threshold is supplied packets are dropped and their + * buffers recycled if (a) the number of remaining buffers is under the + * threshold and the packet is too big to copy, or (b) the packet should + * be copied but there is no memory for the copy. + */ +static struct sk_buff *get_packet(struct adapter *adap, struct sge_fl *fl, + unsigned int len, unsigned int drop_thres) +{ + struct sk_buff *skb = NULL; + struct rx_sw_desc *sd = &fl->sdesc[fl->cidx]; + + prefetch(sd->skb->data); + fl->credits--; + + if (len <= SGE_RX_COPY_THRES) { + skb = alloc_skb(len, GFP_ATOMIC); + if (likely(skb != NULL)) { + __skb_put(skb, len); + dma_sync_single_for_cpu(&adap->pdev->dev, + dma_unmap_addr(sd, dma_addr), + len, DMA_FROM_DEVICE); + memcpy(skb->data, sd->skb->data, len); + dma_sync_single_for_device(&adap->pdev->dev, + dma_unmap_addr(sd, dma_addr), + len, DMA_FROM_DEVICE); + } else if (!drop_thres) + goto use_orig_buf; +recycle: + recycle_rx_buf(adap, fl, fl->cidx); + return skb; + } + + if (unlikely(fl->credits < drop_thres) && + refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits - 1), + GFP_ATOMIC | __GFP_COMP) == 0) + goto recycle; + +use_orig_buf: + dma_unmap_single(&adap->pdev->dev, dma_unmap_addr(sd, dma_addr), + fl->buf_size, DMA_FROM_DEVICE); + skb = sd->skb; + skb_put(skb, len); + __refill_fl(adap, fl); + return skb; +} + +/** + * get_packet_pg - return the next ingress packet buffer from a free list + * @adap: the adapter that received the packet + * @fl: the SGE free list holding the packet + * @q: the queue + * @len: the packet length including any SGE padding + * @drop_thres: # of remaining buffers before we start dropping packets + * + * Get the next packet from a free list populated with page chunks. + * If the packet is small we make a copy and recycle the original buffer, + * otherwise we attach the original buffer as a page fragment to a fresh + * sk_buff. If a positive drop threshold is supplied packets are dropped + * and their buffers recycled if (a) the number of remaining buffers is + * under the threshold and the packet is too big to copy, or (b) there's + * no system memory. + * + * Note: this function is similar to @get_packet but deals with Rx buffers + * that are page chunks rather than sk_buffs. + */ +static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl, + struct sge_rspq *q, unsigned int len, + unsigned int drop_thres) +{ + struct sk_buff *newskb, *skb; + struct rx_sw_desc *sd = &fl->sdesc[fl->cidx]; + + dma_addr_t dma_addr = dma_unmap_addr(sd, dma_addr); + + newskb = skb = q->pg_skb; + if (!skb && (len <= SGE_RX_COPY_THRES)) { + newskb = alloc_skb(len, GFP_ATOMIC); + if (likely(newskb != NULL)) { + __skb_put(newskb, len); + dma_sync_single_for_cpu(&adap->pdev->dev, dma_addr, + len, DMA_FROM_DEVICE); + memcpy(newskb->data, sd->pg_chunk.va, len); + dma_sync_single_for_device(&adap->pdev->dev, dma_addr, + len, DMA_FROM_DEVICE); + } else if (!drop_thres) + return NULL; +recycle: + fl->credits--; + recycle_rx_buf(adap, fl, fl->cidx); + q->rx_recycle_buf++; + return newskb; + } + + if (unlikely(q->rx_recycle_buf || (!skb && fl->credits <= drop_thres))) + goto recycle; + + prefetch(sd->pg_chunk.p_cnt); + + if (!skb) + newskb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC); + + if (unlikely(!newskb)) { + if (!drop_thres) + return NULL; + goto recycle; + } + + dma_sync_single_for_cpu(&adap->pdev->dev, dma_addr, len, + DMA_FROM_DEVICE); + (*sd->pg_chunk.p_cnt)--; + if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page) + dma_unmap_page(&adap->pdev->dev, sd->pg_chunk.mapping, + fl->alloc_size, DMA_FROM_DEVICE); + if (!skb) { + __skb_put(newskb, SGE_RX_PULL_LEN); + memcpy(newskb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN); + skb_fill_page_desc(newskb, 0, sd->pg_chunk.page, + sd->pg_chunk.offset + SGE_RX_PULL_LEN, + len - SGE_RX_PULL_LEN); + newskb->len = len; + newskb->data_len = len - SGE_RX_PULL_LEN; + newskb->truesize += newskb->data_len; + } else { + skb_fill_page_desc(newskb, skb_shinfo(newskb)->nr_frags, + sd->pg_chunk.page, + sd->pg_chunk.offset, len); + newskb->len += len; + newskb->data_len += len; + newskb->truesize += len; + } + + fl->credits--; + /* + * We do not refill FLs here, we let the caller do it to overlap a + * prefetch. + */ + return newskb; +} + +/** + * get_imm_packet - return the next ingress packet buffer from a response + * @resp: the response descriptor containing the packet data + * + * Return a packet containing the immediate data of the given response. + */ +static inline struct sk_buff *get_imm_packet(const struct rsp_desc *resp) +{ + struct sk_buff *skb = alloc_skb(IMMED_PKT_SIZE, GFP_ATOMIC); + + if (skb) { + __skb_put(skb, IMMED_PKT_SIZE); + BUILD_BUG_ON(IMMED_PKT_SIZE != sizeof(resp->immediate)); + skb_copy_to_linear_data(skb, &resp->immediate, IMMED_PKT_SIZE); + } + return skb; +} + +/** + * calc_tx_descs - calculate the number of Tx descriptors for a packet + * @skb: the packet + * + * Returns the number of Tx descriptors needed for the given Ethernet + * packet. Ethernet packets require addition of WR and CPL headers. + */ +static inline unsigned int calc_tx_descs(const struct sk_buff *skb) +{ + unsigned int flits; + + if (skb->len <= WR_LEN - sizeof(struct cpl_tx_pkt)) + return 1; + + flits = sgl_len(skb_shinfo(skb)->nr_frags + 1) + 2; + if (skb_shinfo(skb)->gso_size) + flits++; + return flits_to_desc(flits); +} + +/* map_skb - map a packet main body and its page fragments + * @pdev: the PCI device + * @skb: the packet + * @addr: placeholder to save the mapped addresses + * + * map the main body of an sk_buff and its page fragments, if any. + */ +static int map_skb(struct pci_dev *pdev, const struct sk_buff *skb, + dma_addr_t *addr) +{ + const skb_frag_t *fp, *end; + const struct skb_shared_info *si; + + if (skb_headlen(skb)) { + *addr = dma_map_single(&pdev->dev, skb->data, + skb_headlen(skb), DMA_TO_DEVICE); + if (dma_mapping_error(&pdev->dev, *addr)) + goto out_err; + addr++; + } + + si = skb_shinfo(skb); + end = &si->frags[si->nr_frags]; + + for (fp = si->frags; fp < end; fp++) { + *addr = skb_frag_dma_map(&pdev->dev, fp, 0, skb_frag_size(fp), + DMA_TO_DEVICE); + if (dma_mapping_error(&pdev->dev, *addr)) + goto unwind; + addr++; + } + return 0; + +unwind: + while (fp-- > si->frags) + dma_unmap_page(&pdev->dev, *--addr, skb_frag_size(fp), + DMA_TO_DEVICE); + + dma_unmap_single(&pdev->dev, addr[-1], skb_headlen(skb), + DMA_TO_DEVICE); +out_err: + return -ENOMEM; +} + +/** + * write_sgl - populate a scatter/gather list for a packet + * @skb: the packet + * @sgp: the SGL to populate + * @start: start address of skb main body data to include in the SGL + * @len: length of skb main body data to include in the SGL + * @addr: the list of the mapped addresses + * + * Copies the scatter/gather list for the buffers that make up a packet + * and returns the SGL size in 8-byte words. The caller must size the SGL + * appropriately. + */ +static inline unsigned int write_sgl(const struct sk_buff *skb, + struct sg_ent *sgp, unsigned char *start, + unsigned int len, const dma_addr_t *addr) +{ + unsigned int i, j = 0, k = 0, nfrags; + + if (len) { + sgp->len[0] = cpu_to_be32(len); + sgp->addr[j++] = cpu_to_be64(addr[k++]); + } + + nfrags = skb_shinfo(skb)->nr_frags; + for (i = 0; i < nfrags; i++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + + sgp->len[j] = cpu_to_be32(skb_frag_size(frag)); + sgp->addr[j] = cpu_to_be64(addr[k++]); + j ^= 1; + if (j == 0) + ++sgp; + } + if (j) + sgp->len[j] = 0; + return ((nfrags + (len != 0)) * 3) / 2 + j; +} + +/** + * check_ring_tx_db - check and potentially ring a Tx queue's doorbell + * @adap: the adapter + * @q: the Tx queue + * + * Ring the doorbel if a Tx queue is asleep. There is a natural race, + * where the HW is going to sleep just after we checked, however, + * then the interrupt handler will detect the outstanding TX packet + * and ring the doorbell for us. + * + * When GTS is disabled we unconditionally ring the doorbell. + */ +static inline void check_ring_tx_db(struct adapter *adap, struct sge_txq *q) +{ +#if USE_GTS + clear_bit(TXQ_LAST_PKT_DB, &q->flags); + if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) { + set_bit(TXQ_LAST_PKT_DB, &q->flags); + t3_write_reg(adap, A_SG_KDOORBELL, + F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); + } +#else + wmb(); /* write descriptors before telling HW */ + t3_write_reg(adap, A_SG_KDOORBELL, + F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); +#endif +} + +static inline void wr_gen2(struct tx_desc *d, unsigned int gen) +{ +#if SGE_NUM_GENBITS == 2 + d->flit[TX_DESC_FLITS - 1] = cpu_to_be64(gen); +#endif +} + +/** + * write_wr_hdr_sgl - write a WR header and, optionally, SGL + * @ndesc: number of Tx descriptors spanned by the SGL + * @skb: the packet corresponding to the WR + * @d: first Tx descriptor to be written + * @pidx: index of above descriptors + * @q: the SGE Tx queue + * @sgl: the SGL + * @flits: number of flits to the start of the SGL in the first descriptor + * @sgl_flits: the SGL size in flits + * @gen: the Tx descriptor generation + * @wr_hi: top 32 bits of WR header based on WR type (big endian) + * @wr_lo: low 32 bits of WR header based on WR type (big endian) + * + * Write a work request header and an associated SGL. If the SGL is + * small enough to fit into one Tx descriptor it has already been written + * and we just need to write the WR header. Otherwise we distribute the + * SGL across the number of descriptors it spans. + */ +static void write_wr_hdr_sgl(unsigned int ndesc, struct sk_buff *skb, + struct tx_desc *d, unsigned int pidx, + const struct sge_txq *q, + const struct sg_ent *sgl, + unsigned int flits, unsigned int sgl_flits, + unsigned int gen, __be32 wr_hi, + __be32 wr_lo) +{ + struct work_request_hdr *wrp = (struct work_request_hdr *)d; + struct tx_sw_desc *sd = &q->sdesc[pidx]; + + sd->skb = skb; + if (need_skb_unmap()) { + sd->fragidx = 0; + sd->addr_idx = 0; + sd->sflit = flits; + } + + if (likely(ndesc == 1)) { + sd->eop = 1; + wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | + V_WR_SGLSFLT(flits)) | wr_hi; + dma_wmb(); + wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) | + V_WR_GEN(gen)) | wr_lo; + wr_gen2(d, gen); + } else { + unsigned int ogen = gen; + const u64 *fp = (const u64 *)sgl; + struct work_request_hdr *wp = wrp; + + wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) | + V_WR_SGLSFLT(flits)) | wr_hi; + + while (sgl_flits) { + unsigned int avail = WR_FLITS - flits; + + if (avail > sgl_flits) + avail = sgl_flits; + memcpy(&d->flit[flits], fp, avail * sizeof(*fp)); + sgl_flits -= avail; + ndesc--; + if (!sgl_flits) + break; + + fp += avail; + d++; + sd->eop = 0; + sd++; + if (++pidx == q->size) { + pidx = 0; + gen ^= 1; + d = q->desc; + sd = q->sdesc; + } + + sd->skb = skb; + wrp = (struct work_request_hdr *)d; + wrp->wr_hi = htonl(V_WR_DATATYPE(1) | + V_WR_SGLSFLT(1)) | wr_hi; + wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS, + sgl_flits + 1)) | + V_WR_GEN(gen)) | wr_lo; + wr_gen2(d, gen); + flits = 1; + } + sd->eop = 1; + wrp->wr_hi |= htonl(F_WR_EOP); + dma_wmb(); + wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo; + wr_gen2((struct tx_desc *)wp, ogen); + WARN_ON(ndesc != 0); + } +} + +/** + * write_tx_pkt_wr - write a TX_PKT work request + * @adap: the adapter + * @skb: the packet to send + * @pi: the egress interface + * @pidx: index of the first Tx descriptor to write + * @gen: the generation value to use + * @q: the Tx queue + * @ndesc: number of descriptors the packet will occupy + * @compl: the value of the COMPL bit to use + * @addr: address + * + * Generate a TX_PKT work request to send the supplied packet. + */ +static void write_tx_pkt_wr(struct adapter *adap, struct sk_buff *skb, + const struct port_info *pi, + unsigned int pidx, unsigned int gen, + struct sge_txq *q, unsigned int ndesc, + unsigned int compl, const dma_addr_t *addr) +{ + unsigned int flits, sgl_flits, cntrl, tso_info; + struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1]; + struct tx_desc *d = &q->desc[pidx]; + struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)d; + + cpl->len = htonl(skb->len); + cntrl = V_TXPKT_INTF(pi->port_id); + + if (skb_vlan_tag_present(skb)) + cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(skb_vlan_tag_get(skb)); + + tso_info = V_LSO_MSS(skb_shinfo(skb)->gso_size); + if (tso_info) { + int eth_type; + struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)cpl; + + d->flit[2] = 0; + cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO); + hdr->cntrl = htonl(cntrl); + eth_type = skb_network_offset(skb) == ETH_HLEN ? + CPL_ETH_II : CPL_ETH_II_VLAN; + tso_info |= V_LSO_ETH_TYPE(eth_type) | + V_LSO_IPHDR_WORDS(ip_hdr(skb)->ihl) | + V_LSO_TCPHDR_WORDS(tcp_hdr(skb)->doff); + hdr->lso_info = htonl(tso_info); + flits = 3; + } else { + cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); + cntrl |= F_TXPKT_IPCSUM_DIS; /* SW calculates IP csum */ + cntrl |= V_TXPKT_L4CSUM_DIS(skb->ip_summed != CHECKSUM_PARTIAL); + cpl->cntrl = htonl(cntrl); + + if (skb->len <= WR_LEN - sizeof(*cpl)) { + q->sdesc[pidx].skb = NULL; + if (!skb->data_len) + skb_copy_from_linear_data(skb, &d->flit[2], + skb->len); + else + skb_copy_bits(skb, 0, &d->flit[2], skb->len); + + flits = (skb->len + 7) / 8 + 2; + cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(skb->len & 7) | + V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) + | F_WR_SOP | F_WR_EOP | compl); + dma_wmb(); + cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(gen) | + V_WR_TID(q->token)); + wr_gen2(d, gen); + dev_consume_skb_any(skb); + return; + } + + flits = 2; + } + + sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl; + sgl_flits = write_sgl(skb, sgp, skb->data, skb_headlen(skb), addr); + + write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits, gen, + htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | compl), + htonl(V_WR_TID(q->token))); +} + +static inline void t3_stop_tx_queue(struct netdev_queue *txq, + struct sge_qset *qs, struct sge_txq *q) +{ + netif_tx_stop_queue(txq); + set_bit(TXQ_ETH, &qs->txq_stopped); + q->stops++; +} + +/** + * t3_eth_xmit - add a packet to the Ethernet Tx queue + * @skb: the packet + * @dev: the egress net device + * + * Add a packet to an SGE Tx queue. Runs with softirqs disabled. + */ +netdev_tx_t t3_eth_xmit(struct sk_buff *skb, struct net_device *dev) +{ + int qidx; + unsigned int ndesc, pidx, credits, gen, compl; + const struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + struct netdev_queue *txq; + struct sge_qset *qs; + struct sge_txq *q; + dma_addr_t addr[MAX_SKB_FRAGS + 1]; + + /* + * The chip min packet length is 9 octets but play safe and reject + * anything shorter than an Ethernet header. + */ + if (unlikely(skb->len < ETH_HLEN)) { + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + + qidx = skb_get_queue_mapping(skb); + qs = &pi->qs[qidx]; + q = &qs->txq[TXQ_ETH]; + txq = netdev_get_tx_queue(dev, qidx); + + reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK); + + credits = q->size - q->in_use; + ndesc = calc_tx_descs(skb); + + if (unlikely(credits < ndesc)) { + t3_stop_tx_queue(txq, qs, q); + dev_err(&adap->pdev->dev, + "%s: Tx ring %u full while queue awake!\n", + dev->name, q->cntxt_id & 7); + return NETDEV_TX_BUSY; + } + + /* Check if ethernet packet can't be sent as immediate data */ + if (skb->len > (WR_LEN - sizeof(struct cpl_tx_pkt))) { + if (unlikely(map_skb(adap->pdev, skb, addr) < 0)) { + dev_kfree_skb(skb); + return NETDEV_TX_OK; + } + } + + q->in_use += ndesc; + if (unlikely(credits - ndesc < q->stop_thres)) { + t3_stop_tx_queue(txq, qs, q); + + if (should_restart_tx(q) && + test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) { + q->restarts++; + netif_tx_start_queue(txq); + } + } + + gen = q->gen; + q->unacked += ndesc; + compl = (q->unacked & 8) << (S_WR_COMPL - 3); + q->unacked &= 7; + pidx = q->pidx; + q->pidx += ndesc; + if (q->pidx >= q->size) { + q->pidx -= q->size; + q->gen ^= 1; + } + + /* update port statistics */ + if (skb->ip_summed == CHECKSUM_PARTIAL) + qs->port_stats[SGE_PSTAT_TX_CSUM]++; + if (skb_shinfo(skb)->gso_size) + qs->port_stats[SGE_PSTAT_TSO]++; + if (skb_vlan_tag_present(skb)) + qs->port_stats[SGE_PSTAT_VLANINS]++; + + /* + * We do not use Tx completion interrupts to free DMAd Tx packets. + * This is good for performance but means that we rely on new Tx + * packets arriving to run the destructors of completed packets, + * which open up space in their sockets' send queues. Sometimes + * we do not get such new packets causing Tx to stall. A single + * UDP transmitter is a good example of this situation. We have + * a clean up timer that periodically reclaims completed packets + * but it doesn't run often enough (nor do we want it to) to prevent + * lengthy stalls. A solution to this problem is to run the + * destructor early, after the packet is queued but before it's DMAd. + * A cons is that we lie to socket memory accounting, but the amount + * of extra memory is reasonable (limited by the number of Tx + * descriptors), the packets do actually get freed quickly by new + * packets almost always, and for protocols like TCP that wait for + * acks to really free up the data the extra memory is even less. + * On the positive side we run the destructors on the sending CPU + * rather than on a potentially different completing CPU, usually a + * good thing. We also run them without holding our Tx queue lock, + * unlike what reclaim_completed_tx() would otherwise do. + * + * Run the destructor before telling the DMA engine about the packet + * to make sure it doesn't complete and get freed prematurely. + */ + if (likely(!skb_shared(skb))) + skb_orphan(skb); + + write_tx_pkt_wr(adap, skb, pi, pidx, gen, q, ndesc, compl, addr); + check_ring_tx_db(adap, q); + return NETDEV_TX_OK; +} + +/** + * write_imm - write a packet into a Tx descriptor as immediate data + * @d: the Tx descriptor to write + * @skb: the packet + * @len: the length of packet data to write as immediate data + * @gen: the generation bit value to write + * + * Writes a packet as immediate data into a Tx descriptor. The packet + * contains a work request at its beginning. We must write the packet + * carefully so the SGE doesn't read it accidentally before it's written + * in its entirety. + */ +static inline void write_imm(struct tx_desc *d, struct sk_buff *skb, + unsigned int len, unsigned int gen) +{ + struct work_request_hdr *from = (struct work_request_hdr *)skb->data; + struct work_request_hdr *to = (struct work_request_hdr *)d; + + if (likely(!skb->data_len)) + memcpy(&to[1], &from[1], len - sizeof(*from)); + else + skb_copy_bits(skb, sizeof(*from), &to[1], len - sizeof(*from)); + + to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP | + V_WR_BCNTLFLT(len & 7)); + dma_wmb(); + to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) | + V_WR_LEN((len + 7) / 8)); + wr_gen2(d, gen); + kfree_skb(skb); +} + +/** + * check_desc_avail - check descriptor availability on a send queue + * @adap: the adapter + * @q: the send queue + * @skb: the packet needing the descriptors + * @ndesc: the number of Tx descriptors needed + * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL) + * + * Checks if the requested number of Tx descriptors is available on an + * SGE send queue. If the queue is already suspended or not enough + * descriptors are available the packet is queued for later transmission. + * Must be called with the Tx queue locked. + * + * Returns 0 if enough descriptors are available, 1 if there aren't + * enough descriptors and the packet has been queued, and 2 if the caller + * needs to retry because there weren't enough descriptors at the + * beginning of the call but some freed up in the mean time. + */ +static inline int check_desc_avail(struct adapter *adap, struct sge_txq *q, + struct sk_buff *skb, unsigned int ndesc, + unsigned int qid) +{ + if (unlikely(!skb_queue_empty(&q->sendq))) { + addq_exit:__skb_queue_tail(&q->sendq, skb); + return 1; + } + if (unlikely(q->size - q->in_use < ndesc)) { + struct sge_qset *qs = txq_to_qset(q, qid); + + set_bit(qid, &qs->txq_stopped); + smp_mb__after_atomic(); + + if (should_restart_tx(q) && + test_and_clear_bit(qid, &qs->txq_stopped)) + return 2; + + q->stops++; + goto addq_exit; + } + return 0; +} + +/** + * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs + * @q: the SGE control Tx queue + * + * This is a variant of reclaim_completed_tx() that is used for Tx queues + * that send only immediate data (presently just the control queues) and + * thus do not have any sk_buffs to release. + */ +static inline void reclaim_completed_tx_imm(struct sge_txq *q) +{ + unsigned int reclaim = q->processed - q->cleaned; + + q->in_use -= reclaim; + q->cleaned += reclaim; +} + +static inline int immediate(const struct sk_buff *skb) +{ + return skb->len <= WR_LEN; +} + +/** + * ctrl_xmit - send a packet through an SGE control Tx queue + * @adap: the adapter + * @q: the control queue + * @skb: the packet + * + * Send a packet through an SGE control Tx queue. Packets sent through + * a control queue must fit entirely as immediate data in a single Tx + * descriptor and have no page fragments. + */ +static int ctrl_xmit(struct adapter *adap, struct sge_txq *q, + struct sk_buff *skb) +{ + int ret; + struct work_request_hdr *wrp = (struct work_request_hdr *)skb->data; + + if (unlikely(!immediate(skb))) { + WARN_ON(1); + dev_kfree_skb(skb); + return NET_XMIT_SUCCESS; + } + + wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP); + wrp->wr_lo = htonl(V_WR_TID(q->token)); + + spin_lock(&q->lock); + again:reclaim_completed_tx_imm(q); + + ret = check_desc_avail(adap, q, skb, 1, TXQ_CTRL); + if (unlikely(ret)) { + if (ret == 1) { + spin_unlock(&q->lock); + return NET_XMIT_CN; + } + goto again; + } + + write_imm(&q->desc[q->pidx], skb, skb->len, q->gen); + + q->in_use++; + if (++q->pidx >= q->size) { + q->pidx = 0; + q->gen ^= 1; + } + spin_unlock(&q->lock); + wmb(); + t3_write_reg(adap, A_SG_KDOORBELL, + F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); + return NET_XMIT_SUCCESS; +} + +/** + * restart_ctrlq - restart a suspended control queue + * @w: pointer to the work associated with this handler + * + * Resumes transmission on a suspended Tx control queue. + */ +static void restart_ctrlq(struct work_struct *w) +{ + struct sk_buff *skb; + struct sge_qset *qs = container_of(w, struct sge_qset, + txq[TXQ_CTRL].qresume_task); + struct sge_txq *q = &qs->txq[TXQ_CTRL]; + + spin_lock(&q->lock); + again:reclaim_completed_tx_imm(q); + + while (q->in_use < q->size && + (skb = __skb_dequeue(&q->sendq)) != NULL) { + + write_imm(&q->desc[q->pidx], skb, skb->len, q->gen); + + if (++q->pidx >= q->size) { + q->pidx = 0; + q->gen ^= 1; + } + q->in_use++; + } + + if (!skb_queue_empty(&q->sendq)) { + set_bit(TXQ_CTRL, &qs->txq_stopped); + smp_mb__after_atomic(); + + if (should_restart_tx(q) && + test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) + goto again; + q->stops++; + } + + spin_unlock(&q->lock); + wmb(); + t3_write_reg(qs->adap, A_SG_KDOORBELL, + F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); +} + +/* + * Send a management message through control queue 0 + */ +int t3_mgmt_tx(struct adapter *adap, struct sk_buff *skb) +{ + int ret; + local_bh_disable(); + ret = ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], skb); + local_bh_enable(); + + return ret; +} + +/** + * deferred_unmap_destructor - unmap a packet when it is freed + * @skb: the packet + * + * This is the packet destructor used for Tx packets that need to remain + * mapped until they are freed rather than until their Tx descriptors are + * freed. + */ +static void deferred_unmap_destructor(struct sk_buff *skb) +{ + int i; + const dma_addr_t *p; + const struct skb_shared_info *si; + const struct deferred_unmap_info *dui; + + dui = (struct deferred_unmap_info *)skb->head; + p = dui->addr; + + if (skb_tail_pointer(skb) - skb_transport_header(skb)) + dma_unmap_single(&dui->pdev->dev, *p++, + skb_tail_pointer(skb) - skb_transport_header(skb), + DMA_TO_DEVICE); + + si = skb_shinfo(skb); + for (i = 0; i < si->nr_frags; i++) + dma_unmap_page(&dui->pdev->dev, *p++, + skb_frag_size(&si->frags[i]), DMA_TO_DEVICE); +} + +static void setup_deferred_unmapping(struct sk_buff *skb, struct pci_dev *pdev, + const struct sg_ent *sgl, int sgl_flits) +{ + dma_addr_t *p; + struct deferred_unmap_info *dui; + + dui = (struct deferred_unmap_info *)skb->head; + dui->pdev = pdev; + for (p = dui->addr; sgl_flits >= 3; sgl++, sgl_flits -= 3) { + *p++ = be64_to_cpu(sgl->addr[0]); + *p++ = be64_to_cpu(sgl->addr[1]); + } + if (sgl_flits) + *p = be64_to_cpu(sgl->addr[0]); +} + +/** + * write_ofld_wr - write an offload work request + * @adap: the adapter + * @skb: the packet to send + * @q: the Tx queue + * @pidx: index of the first Tx descriptor to write + * @gen: the generation value to use + * @ndesc: number of descriptors the packet will occupy + * @addr: the address + * + * Write an offload work request to send the supplied packet. The packet + * data already carry the work request with most fields populated. + */ +static void write_ofld_wr(struct adapter *adap, struct sk_buff *skb, + struct sge_txq *q, unsigned int pidx, + unsigned int gen, unsigned int ndesc, + const dma_addr_t *addr) +{ + unsigned int sgl_flits, flits; + struct work_request_hdr *from; + struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1]; + struct tx_desc *d = &q->desc[pidx]; + + if (immediate(skb)) { + q->sdesc[pidx].skb = NULL; + write_imm(d, skb, skb->len, gen); + return; + } + + /* Only TX_DATA builds SGLs */ + + from = (struct work_request_hdr *)skb->data; + memcpy(&d->flit[1], &from[1], + skb_transport_offset(skb) - sizeof(*from)); + + flits = skb_transport_offset(skb) / 8; + sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl; + sgl_flits = write_sgl(skb, sgp, skb_transport_header(skb), + skb_tail_pointer(skb) - skb_transport_header(skb), + addr); + if (need_skb_unmap()) { + setup_deferred_unmapping(skb, adap->pdev, sgp, sgl_flits); + skb->destructor = deferred_unmap_destructor; + } + + write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits, + gen, from->wr_hi, from->wr_lo); +} + +/** + * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet + * @skb: the packet + * + * Returns the number of Tx descriptors needed for the given offload + * packet. These packets are already fully constructed. + */ +static inline unsigned int calc_tx_descs_ofld(const struct sk_buff *skb) +{ + unsigned int flits, cnt; + + if (skb->len <= WR_LEN) + return 1; /* packet fits as immediate data */ + + flits = skb_transport_offset(skb) / 8; /* headers */ + cnt = skb_shinfo(skb)->nr_frags; + if (skb_tail_pointer(skb) != skb_transport_header(skb)) + cnt++; + return flits_to_desc(flits + sgl_len(cnt)); +} + +/** + * ofld_xmit - send a packet through an offload queue + * @adap: the adapter + * @q: the Tx offload queue + * @skb: the packet + * + * Send an offload packet through an SGE offload queue. + */ +static int ofld_xmit(struct adapter *adap, struct sge_txq *q, + struct sk_buff *skb) +{ + int ret; + unsigned int ndesc = calc_tx_descs_ofld(skb), pidx, gen; + + spin_lock(&q->lock); +again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK); + + ret = check_desc_avail(adap, q, skb, ndesc, TXQ_OFLD); + if (unlikely(ret)) { + if (ret == 1) { + skb->priority = ndesc; /* save for restart */ + spin_unlock(&q->lock); + return NET_XMIT_CN; + } + goto again; + } + + if (!immediate(skb) && + map_skb(adap->pdev, skb, (dma_addr_t *)skb->head)) { + spin_unlock(&q->lock); + return NET_XMIT_SUCCESS; + } + + gen = q->gen; + q->in_use += ndesc; + pidx = q->pidx; + q->pidx += ndesc; + if (q->pidx >= q->size) { + q->pidx -= q->size; + q->gen ^= 1; + } + spin_unlock(&q->lock); + + write_ofld_wr(adap, skb, q, pidx, gen, ndesc, (dma_addr_t *)skb->head); + check_ring_tx_db(adap, q); + return NET_XMIT_SUCCESS; +} + +/** + * restart_offloadq - restart a suspended offload queue + * @w: pointer to the work associated with this handler + * + * Resumes transmission on a suspended Tx offload queue. + */ +static void restart_offloadq(struct work_struct *w) +{ + struct sk_buff *skb; + struct sge_qset *qs = container_of(w, struct sge_qset, + txq[TXQ_OFLD].qresume_task); + struct sge_txq *q = &qs->txq[TXQ_OFLD]; + const struct port_info *pi = netdev_priv(qs->netdev); + struct adapter *adap = pi->adapter; + unsigned int written = 0; + + spin_lock(&q->lock); +again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK); + + while ((skb = skb_peek(&q->sendq)) != NULL) { + unsigned int gen, pidx; + unsigned int ndesc = skb->priority; + + if (unlikely(q->size - q->in_use < ndesc)) { + set_bit(TXQ_OFLD, &qs->txq_stopped); + smp_mb__after_atomic(); + + if (should_restart_tx(q) && + test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) + goto again; + q->stops++; + break; + } + + if (!immediate(skb) && + map_skb(adap->pdev, skb, (dma_addr_t *)skb->head)) + break; + + gen = q->gen; + q->in_use += ndesc; + pidx = q->pidx; + q->pidx += ndesc; + written += ndesc; + if (q->pidx >= q->size) { + q->pidx -= q->size; + q->gen ^= 1; + } + __skb_unlink(skb, &q->sendq); + spin_unlock(&q->lock); + + write_ofld_wr(adap, skb, q, pidx, gen, ndesc, + (dma_addr_t *)skb->head); + spin_lock(&q->lock); + } + spin_unlock(&q->lock); + +#if USE_GTS + set_bit(TXQ_RUNNING, &q->flags); + set_bit(TXQ_LAST_PKT_DB, &q->flags); +#endif + wmb(); + if (likely(written)) + t3_write_reg(adap, A_SG_KDOORBELL, + F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); +} + +/** + * queue_set - return the queue set a packet should use + * @skb: the packet + * + * Maps a packet to the SGE queue set it should use. The desired queue + * set is carried in bits 1-3 in the packet's priority. + */ +static inline int queue_set(const struct sk_buff *skb) +{ + return skb->priority >> 1; +} + +/** + * is_ctrl_pkt - return whether an offload packet is a control packet + * @skb: the packet + * + * Determines whether an offload packet should use an OFLD or a CTRL + * Tx queue. This is indicated by bit 0 in the packet's priority. + */ +static inline int is_ctrl_pkt(const struct sk_buff *skb) +{ + return skb->priority & 1; +} + +/** + * t3_offload_tx - send an offload packet + * @tdev: the offload device to send to + * @skb: the packet + * + * Sends an offload packet. We use the packet priority to select the + * appropriate Tx queue as follows: bit 0 indicates whether the packet + * should be sent as regular or control, bits 1-3 select the queue set. + */ +int t3_offload_tx(struct t3cdev *tdev, struct sk_buff *skb) +{ + struct adapter *adap = tdev2adap(tdev); + struct sge_qset *qs = &adap->sge.qs[queue_set(skb)]; + + if (unlikely(is_ctrl_pkt(skb))) + return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], skb); + + return ofld_xmit(adap, &qs->txq[TXQ_OFLD], skb); +} + +/** + * offload_enqueue - add an offload packet to an SGE offload receive queue + * @q: the SGE response queue + * @skb: the packet + * + * Add a new offload packet to an SGE response queue's offload packet + * queue. If the packet is the first on the queue it schedules the RX + * softirq to process the queue. + */ +static inline void offload_enqueue(struct sge_rspq *q, struct sk_buff *skb) +{ + int was_empty = skb_queue_empty(&q->rx_queue); + + __skb_queue_tail(&q->rx_queue, skb); + + if (was_empty) { + struct sge_qset *qs = rspq_to_qset(q); + + napi_schedule(&qs->napi); + } +} + +/** + * deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts + * @tdev: the offload device that will be receiving the packets + * @q: the SGE response queue that assembled the bundle + * @skbs: the partial bundle + * @n: the number of packets in the bundle + * + * Delivers a (partial) bundle of Rx offload packets to an offload device. + */ +static inline void deliver_partial_bundle(struct t3cdev *tdev, + struct sge_rspq *q, + struct sk_buff *skbs[], int n) +{ + if (n) { + q->offload_bundles++; + tdev->recv(tdev, skbs, n); + } +} + +/** + * ofld_poll - NAPI handler for offload packets in interrupt mode + * @napi: the network device doing the polling + * @budget: polling budget + * + * The NAPI handler for offload packets when a response queue is serviced + * by the hard interrupt handler, i.e., when it's operating in non-polling + * mode. Creates small packet batches and sends them through the offload + * receive handler. Batches need to be of modest size as we do prefetches + * on the packets in each. + */ +static int ofld_poll(struct napi_struct *napi, int budget) +{ + struct sge_qset *qs = container_of(napi, struct sge_qset, napi); + struct sge_rspq *q = &qs->rspq; + struct adapter *adapter = qs->adap; + int work_done = 0; + + while (work_done < budget) { + struct sk_buff *skb, *tmp, *skbs[RX_BUNDLE_SIZE]; + struct sk_buff_head queue; + int ngathered; + + spin_lock_irq(&q->lock); + __skb_queue_head_init(&queue); + skb_queue_splice_init(&q->rx_queue, &queue); + if (skb_queue_empty(&queue)) { + napi_complete_done(napi, work_done); + spin_unlock_irq(&q->lock); + return work_done; + } + spin_unlock_irq(&q->lock); + + ngathered = 0; + skb_queue_walk_safe(&queue, skb, tmp) { + if (work_done >= budget) + break; + work_done++; + + __skb_unlink(skb, &queue); + prefetch(skb->data); + skbs[ngathered] = skb; + if (++ngathered == RX_BUNDLE_SIZE) { + q->offload_bundles++; + adapter->tdev.recv(&adapter->tdev, skbs, + ngathered); + ngathered = 0; + } + } + if (!skb_queue_empty(&queue)) { + /* splice remaining packets back onto Rx queue */ + spin_lock_irq(&q->lock); + skb_queue_splice(&queue, &q->rx_queue); + spin_unlock_irq(&q->lock); + } + deliver_partial_bundle(&adapter->tdev, q, skbs, ngathered); + } + + return work_done; +} + +/** + * rx_offload - process a received offload packet + * @tdev: the offload device receiving the packet + * @rq: the response queue that received the packet + * @skb: the packet + * @rx_gather: a gather list of packets if we are building a bundle + * @gather_idx: index of the next available slot in the bundle + * + * Process an ingress offload packet and add it to the offload ingress + * queue. Returns the index of the next available slot in the bundle. + */ +static inline int rx_offload(struct t3cdev *tdev, struct sge_rspq *rq, + struct sk_buff *skb, struct sk_buff *rx_gather[], + unsigned int gather_idx) +{ + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + + if (rq->polling) { + rx_gather[gather_idx++] = skb; + if (gather_idx == RX_BUNDLE_SIZE) { + tdev->recv(tdev, rx_gather, RX_BUNDLE_SIZE); + gather_idx = 0; + rq->offload_bundles++; + } + } else + offload_enqueue(rq, skb); + + return gather_idx; +} + +/** + * restart_tx - check whether to restart suspended Tx queues + * @qs: the queue set to resume + * + * Restarts suspended Tx queues of an SGE queue set if they have enough + * free resources to resume operation. + */ +static void restart_tx(struct sge_qset *qs) +{ + if (test_bit(TXQ_ETH, &qs->txq_stopped) && + should_restart_tx(&qs->txq[TXQ_ETH]) && + test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) { + qs->txq[TXQ_ETH].restarts++; + if (netif_running(qs->netdev)) + netif_tx_wake_queue(qs->tx_q); + } + + if (test_bit(TXQ_OFLD, &qs->txq_stopped) && + should_restart_tx(&qs->txq[TXQ_OFLD]) && + test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) { + qs->txq[TXQ_OFLD].restarts++; + + /* The work can be quite lengthy so we use driver's own queue */ + queue_work(cxgb3_wq, &qs->txq[TXQ_OFLD].qresume_task); + } + if (test_bit(TXQ_CTRL, &qs->txq_stopped) && + should_restart_tx(&qs->txq[TXQ_CTRL]) && + test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) { + qs->txq[TXQ_CTRL].restarts++; + + /* The work can be quite lengthy so we use driver's own queue */ + queue_work(cxgb3_wq, &qs->txq[TXQ_CTRL].qresume_task); + } +} + +/** + * cxgb3_arp_process - process an ARP request probing a private IP address + * @pi: the port info + * @skb: the skbuff containing the ARP request + * + * Check if the ARP request is probing the private IP address + * dedicated to iSCSI, generate an ARP reply if so. + */ +static void cxgb3_arp_process(struct port_info *pi, struct sk_buff *skb) +{ + struct net_device *dev = skb->dev; + struct arphdr *arp; + unsigned char *arp_ptr; + unsigned char *sha; + __be32 sip, tip; + + if (!dev) + return; + + skb_reset_network_header(skb); + arp = arp_hdr(skb); + + if (arp->ar_op != htons(ARPOP_REQUEST)) + return; + + arp_ptr = (unsigned char *)(arp + 1); + sha = arp_ptr; + arp_ptr += dev->addr_len; + memcpy(&sip, arp_ptr, sizeof(sip)); + arp_ptr += sizeof(sip); + arp_ptr += dev->addr_len; + memcpy(&tip, arp_ptr, sizeof(tip)); + + if (tip != pi->iscsi_ipv4addr) + return; + + arp_send(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha, + pi->iscsic.mac_addr, sha); + +} + +static inline int is_arp(struct sk_buff *skb) +{ + return skb->protocol == htons(ETH_P_ARP); +} + +static void cxgb3_process_iscsi_prov_pack(struct port_info *pi, + struct sk_buff *skb) +{ + if (is_arp(skb)) { + cxgb3_arp_process(pi, skb); + return; + } + + if (pi->iscsic.recv) + pi->iscsic.recv(pi, skb); + +} + +/** + * rx_eth - process an ingress ethernet packet + * @adap: the adapter + * @rq: the response queue that received the packet + * @skb: the packet + * @pad: padding + * @lro: large receive offload + * + * Process an ingress ethernet packet and deliver it to the stack. + * The padding is 2 if the packet was delivered in an Rx buffer and 0 + * if it was immediate data in a response. + */ +static void rx_eth(struct adapter *adap, struct sge_rspq *rq, + struct sk_buff *skb, int pad, int lro) +{ + struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)(skb->data + pad); + struct sge_qset *qs = rspq_to_qset(rq); + struct port_info *pi; + + skb_pull(skb, sizeof(*p) + pad); + skb->protocol = eth_type_trans(skb, adap->port[p->iff]); + pi = netdev_priv(skb->dev); + if ((skb->dev->features & NETIF_F_RXCSUM) && p->csum_valid && + p->csum == htons(0xffff) && !p->fragment) { + qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++; + skb->ip_summed = CHECKSUM_UNNECESSARY; + } else + skb_checksum_none_assert(skb); + skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]); + + if (p->vlan_valid) { + qs->port_stats[SGE_PSTAT_VLANEX]++; + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(p->vlan)); + } + if (rq->polling) { + if (lro) + napi_gro_receive(&qs->napi, skb); + else { + if (unlikely(pi->iscsic.flags)) + cxgb3_process_iscsi_prov_pack(pi, skb); + netif_receive_skb(skb); + } + } else + netif_rx(skb); +} + +static inline int is_eth_tcp(u32 rss) +{ + return G_HASHTYPE(ntohl(rss)) == RSS_HASH_4_TUPLE; +} + +/** + * lro_add_page - add a page chunk to an LRO session + * @adap: the adapter + * @qs: the associated queue set + * @fl: the free list containing the page chunk to add + * @len: packet length + * @complete: Indicates the last fragment of a frame + * + * Add a received packet contained in a page chunk to an existing LRO + * session. + */ +static void lro_add_page(struct adapter *adap, struct sge_qset *qs, + struct sge_fl *fl, int len, int complete) +{ + struct rx_sw_desc *sd = &fl->sdesc[fl->cidx]; + struct port_info *pi = netdev_priv(qs->netdev); + struct sk_buff *skb = NULL; + struct cpl_rx_pkt *cpl; + skb_frag_t *rx_frag; + int nr_frags; + int offset = 0; + + if (!qs->nomem) { + skb = napi_get_frags(&qs->napi); + qs->nomem = !skb; + } + + fl->credits--; + + dma_sync_single_for_cpu(&adap->pdev->dev, + dma_unmap_addr(sd, dma_addr), + fl->buf_size - SGE_PG_RSVD, DMA_FROM_DEVICE); + + (*sd->pg_chunk.p_cnt)--; + if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page) + dma_unmap_page(&adap->pdev->dev, sd->pg_chunk.mapping, + fl->alloc_size, DMA_FROM_DEVICE); + + if (!skb) { + put_page(sd->pg_chunk.page); + if (complete) + qs->nomem = 0; + return; + } + + rx_frag = skb_shinfo(skb)->frags; + nr_frags = skb_shinfo(skb)->nr_frags; + + if (!nr_frags) { + offset = 2 + sizeof(struct cpl_rx_pkt); + cpl = qs->lro_va = sd->pg_chunk.va + 2; + + if ((qs->netdev->features & NETIF_F_RXCSUM) && + cpl->csum_valid && cpl->csum == htons(0xffff)) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++; + } else + skb->ip_summed = CHECKSUM_NONE; + } else + cpl = qs->lro_va; + + len -= offset; + + rx_frag += nr_frags; + __skb_frag_set_page(rx_frag, sd->pg_chunk.page); + skb_frag_off_set(rx_frag, sd->pg_chunk.offset + offset); + skb_frag_size_set(rx_frag, len); + + skb->len += len; + skb->data_len += len; + skb->truesize += len; + skb_shinfo(skb)->nr_frags++; + + if (!complete) + return; + + skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]); + + if (cpl->vlan_valid) { + qs->port_stats[SGE_PSTAT_VLANEX]++; + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(cpl->vlan)); + } + napi_gro_frags(&qs->napi); +} + +/** + * handle_rsp_cntrl_info - handles control information in a response + * @qs: the queue set corresponding to the response + * @flags: the response control flags + * + * Handles the control information of an SGE response, such as GTS + * indications and completion credits for the queue set's Tx queues. + * HW coalesces credits, we don't do any extra SW coalescing. + */ +static inline void handle_rsp_cntrl_info(struct sge_qset *qs, u32 flags) +{ + unsigned int credits; + +#if USE_GTS + if (flags & F_RSPD_TXQ0_GTS) + clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags); +#endif + + credits = G_RSPD_TXQ0_CR(flags); + if (credits) + qs->txq[TXQ_ETH].processed += credits; + + credits = G_RSPD_TXQ2_CR(flags); + if (credits) + qs->txq[TXQ_CTRL].processed += credits; + +# if USE_GTS + if (flags & F_RSPD_TXQ1_GTS) + clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags); +# endif + credits = G_RSPD_TXQ1_CR(flags); + if (credits) + qs->txq[TXQ_OFLD].processed += credits; +} + +/** + * check_ring_db - check if we need to ring any doorbells + * @adap: the adapter + * @qs: the queue set whose Tx queues are to be examined + * @sleeping: indicates which Tx queue sent GTS + * + * Checks if some of a queue set's Tx queues need to ring their doorbells + * to resume transmission after idling while they still have unprocessed + * descriptors. + */ +static void check_ring_db(struct adapter *adap, struct sge_qset *qs, + unsigned int sleeping) +{ + if (sleeping & F_RSPD_TXQ0_GTS) { + struct sge_txq *txq = &qs->txq[TXQ_ETH]; + + if (txq->cleaned + txq->in_use != txq->processed && + !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) { + set_bit(TXQ_RUNNING, &txq->flags); + t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | + V_EGRCNTX(txq->cntxt_id)); + } + } + + if (sleeping & F_RSPD_TXQ1_GTS) { + struct sge_txq *txq = &qs->txq[TXQ_OFLD]; + + if (txq->cleaned + txq->in_use != txq->processed && + !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) { + set_bit(TXQ_RUNNING, &txq->flags); + t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | + V_EGRCNTX(txq->cntxt_id)); + } + } +} + +/** + * is_new_response - check if a response is newly written + * @r: the response descriptor + * @q: the response queue + * + * Returns true if a response descriptor contains a yet unprocessed + * response. + */ +static inline int is_new_response(const struct rsp_desc *r, + const struct sge_rspq *q) +{ + return (r->intr_gen & F_RSPD_GEN2) == q->gen; +} + +static inline void clear_rspq_bufstate(struct sge_rspq * const q) +{ + q->pg_skb = NULL; + q->rx_recycle_buf = 0; +} + +#define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS) +#define RSPD_CTRL_MASK (RSPD_GTS_MASK | \ + V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \ + V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \ + V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR)) + +/* How long to delay the next interrupt in case of memory shortage, in 0.1us. */ +#define NOMEM_INTR_DELAY 2500 + +/** + * process_responses - process responses from an SGE response queue + * @adap: the adapter + * @qs: the queue set to which the response queue belongs + * @budget: how many responses can be processed in this round + * + * Process responses from an SGE response queue up to the supplied budget. + * Responses include received packets as well as credits and other events + * for the queues that belong to the response queue's queue set. + * A negative budget is effectively unlimited. + * + * Additionally choose the interrupt holdoff time for the next interrupt + * on this queue. If the system is under memory shortage use a fairly + * long delay to help recovery. + */ +static int process_responses(struct adapter *adap, struct sge_qset *qs, + int budget) +{ + struct sge_rspq *q = &qs->rspq; + struct rsp_desc *r = &q->desc[q->cidx]; + int budget_left = budget; + unsigned int sleeping = 0; + struct sk_buff *offload_skbs[RX_BUNDLE_SIZE]; + int ngathered = 0; + + q->next_holdoff = q->holdoff_tmr; + + while (likely(budget_left && is_new_response(r, q))) { + int packet_complete, eth, ethpad = 2; + int lro = !!(qs->netdev->features & NETIF_F_GRO); + struct sk_buff *skb = NULL; + u32 len, flags; + __be32 rss_hi, rss_lo; + + dma_rmb(); + eth = r->rss_hdr.opcode == CPL_RX_PKT; + rss_hi = *(const __be32 *)r; + rss_lo = r->rss_hdr.rss_hash_val; + flags = ntohl(r->flags); + + if (unlikely(flags & F_RSPD_ASYNC_NOTIF)) { + skb = alloc_skb(AN_PKT_SIZE, GFP_ATOMIC); + if (!skb) + goto no_mem; + + __skb_put_data(skb, r, AN_PKT_SIZE); + skb->data[0] = CPL_ASYNC_NOTIF; + rss_hi = htonl(CPL_ASYNC_NOTIF << 24); + q->async_notif++; + } else if (flags & F_RSPD_IMM_DATA_VALID) { + skb = get_imm_packet(r); + if (unlikely(!skb)) { +no_mem: + q->next_holdoff = NOMEM_INTR_DELAY; + q->nomem++; + /* consume one credit since we tried */ + budget_left--; + break; + } + q->imm_data++; + ethpad = 0; + } else if ((len = ntohl(r->len_cq)) != 0) { + struct sge_fl *fl; + + lro &= eth && is_eth_tcp(rss_hi); + + fl = (len & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; + if (fl->use_pages) { + void *addr = fl->sdesc[fl->cidx].pg_chunk.va; + + net_prefetch(addr); + __refill_fl(adap, fl); + if (lro > 0) { + lro_add_page(adap, qs, fl, + G_RSPD_LEN(len), + flags & F_RSPD_EOP); + goto next_fl; + } + + skb = get_packet_pg(adap, fl, q, + G_RSPD_LEN(len), + eth ? + SGE_RX_DROP_THRES : 0); + q->pg_skb = skb; + } else + skb = get_packet(adap, fl, G_RSPD_LEN(len), + eth ? SGE_RX_DROP_THRES : 0); + if (unlikely(!skb)) { + if (!eth) + goto no_mem; + q->rx_drops++; + } else if (unlikely(r->rss_hdr.opcode == CPL_TRACE_PKT)) + __skb_pull(skb, 2); +next_fl: + if (++fl->cidx == fl->size) + fl->cidx = 0; + } else + q->pure_rsps++; + + if (flags & RSPD_CTRL_MASK) { + sleeping |= flags & RSPD_GTS_MASK; + handle_rsp_cntrl_info(qs, flags); + } + + r++; + if (unlikely(++q->cidx == q->size)) { + q->cidx = 0; + q->gen ^= 1; + r = q->desc; + } + prefetch(r); + + if (++q->credits >= (q->size / 4)) { + refill_rspq(adap, q, q->credits); + q->credits = 0; + } + + packet_complete = flags & + (F_RSPD_EOP | F_RSPD_IMM_DATA_VALID | + F_RSPD_ASYNC_NOTIF); + + if (skb != NULL && packet_complete) { + if (eth) + rx_eth(adap, q, skb, ethpad, lro); + else { + q->offload_pkts++; + /* Preserve the RSS info in csum & priority */ + skb->csum = rss_hi; + skb->priority = rss_lo; + ngathered = rx_offload(&adap->tdev, q, skb, + offload_skbs, + ngathered); + } + + if (flags & F_RSPD_EOP) + clear_rspq_bufstate(q); + } + --budget_left; + } + + deliver_partial_bundle(&adap->tdev, q, offload_skbs, ngathered); + + if (sleeping) + check_ring_db(adap, qs, sleeping); + + smp_mb(); /* commit Tx queue .processed updates */ + if (unlikely(qs->txq_stopped != 0)) + restart_tx(qs); + + budget -= budget_left; + return budget; +} + +static inline int is_pure_response(const struct rsp_desc *r) +{ + __be32 n = r->flags & htonl(F_RSPD_ASYNC_NOTIF | F_RSPD_IMM_DATA_VALID); + + return (n | r->len_cq) == 0; +} + +/** + * napi_rx_handler - the NAPI handler for Rx processing + * @napi: the napi instance + * @budget: how many packets we can process in this round + * + * Handler for new data events when using NAPI. + */ +static int napi_rx_handler(struct napi_struct *napi, int budget) +{ + struct sge_qset *qs = container_of(napi, struct sge_qset, napi); + struct adapter *adap = qs->adap; + int work_done = process_responses(adap, qs, budget); + + if (likely(work_done < budget)) { + napi_complete_done(napi, work_done); + + /* + * Because we don't atomically flush the following + * write it is possible that in very rare cases it can + * reach the device in a way that races with a new + * response being written plus an error interrupt + * causing the NAPI interrupt handler below to return + * unhandled status to the OS. To protect against + * this would require flushing the write and doing + * both the write and the flush with interrupts off. + * Way too expensive and unjustifiable given the + * rarity of the race. + * + * The race cannot happen at all with MSI-X. + */ + t3_write_reg(adap, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) | + V_NEWTIMER(qs->rspq.next_holdoff) | + V_NEWINDEX(qs->rspq.cidx)); + } + return work_done; +} + +/* + * Returns true if the device is already scheduled for polling. + */ +static inline int napi_is_scheduled(struct napi_struct *napi) +{ + return test_bit(NAPI_STATE_SCHED, &napi->state); +} + +/** + * process_pure_responses - process pure responses from a response queue + * @adap: the adapter + * @qs: the queue set owning the response queue + * @r: the first pure response to process + * + * A simpler version of process_responses() that handles only pure (i.e., + * non data-carrying) responses. Such respones are too light-weight to + * justify calling a softirq under NAPI, so we handle them specially in + * the interrupt handler. The function is called with a pointer to a + * response, which the caller must ensure is a valid pure response. + * + * Returns 1 if it encounters a valid data-carrying response, 0 otherwise. + */ +static int process_pure_responses(struct adapter *adap, struct sge_qset *qs, + struct rsp_desc *r) +{ + struct sge_rspq *q = &qs->rspq; + unsigned int sleeping = 0; + + do { + u32 flags = ntohl(r->flags); + + r++; + if (unlikely(++q->cidx == q->size)) { + q->cidx = 0; + q->gen ^= 1; + r = q->desc; + } + prefetch(r); + + if (flags & RSPD_CTRL_MASK) { + sleeping |= flags & RSPD_GTS_MASK; + handle_rsp_cntrl_info(qs, flags); + } + + q->pure_rsps++; + if (++q->credits >= (q->size / 4)) { + refill_rspq(adap, q, q->credits); + q->credits = 0; + } + if (!is_new_response(r, q)) + break; + dma_rmb(); + } while (is_pure_response(r)); + + if (sleeping) + check_ring_db(adap, qs, sleeping); + + smp_mb(); /* commit Tx queue .processed updates */ + if (unlikely(qs->txq_stopped != 0)) + restart_tx(qs); + + return is_new_response(r, q); +} + +/** + * handle_responses - decide what to do with new responses in NAPI mode + * @adap: the adapter + * @q: the response queue + * + * This is used by the NAPI interrupt handlers to decide what to do with + * new SGE responses. If there are no new responses it returns -1. If + * there are new responses and they are pure (i.e., non-data carrying) + * it handles them straight in hard interrupt context as they are very + * cheap and don't deliver any packets. Finally, if there are any data + * signaling responses it schedules the NAPI handler. Returns 1 if it + * schedules NAPI, 0 if all new responses were pure. + * + * The caller must ascertain NAPI is not already running. + */ +static inline int handle_responses(struct adapter *adap, struct sge_rspq *q) +{ + struct sge_qset *qs = rspq_to_qset(q); + struct rsp_desc *r = &q->desc[q->cidx]; + + if (!is_new_response(r, q)) + return -1; + dma_rmb(); + if (is_pure_response(r) && process_pure_responses(adap, qs, r) == 0) { + t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) | + V_NEWTIMER(q->holdoff_tmr) | V_NEWINDEX(q->cidx)); + return 0; + } + napi_schedule(&qs->napi); + return 1; +} + +/* + * The MSI-X interrupt handler for an SGE response queue for the non-NAPI case + * (i.e., response queue serviced in hard interrupt). + */ +static irqreturn_t t3_sge_intr_msix(int irq, void *cookie) +{ + struct sge_qset *qs = cookie; + struct adapter *adap = qs->adap; + struct sge_rspq *q = &qs->rspq; + + spin_lock(&q->lock); + if (process_responses(adap, qs, -1) == 0) + q->unhandled_irqs++; + t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) | + V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx)); + spin_unlock(&q->lock); + return IRQ_HANDLED; +} + +/* + * The MSI-X interrupt handler for an SGE response queue for the NAPI case + * (i.e., response queue serviced by NAPI polling). + */ +static irqreturn_t t3_sge_intr_msix_napi(int irq, void *cookie) +{ + struct sge_qset *qs = cookie; + struct sge_rspq *q = &qs->rspq; + + spin_lock(&q->lock); + + if (handle_responses(qs->adap, q) < 0) + q->unhandled_irqs++; + spin_unlock(&q->lock); + return IRQ_HANDLED; +} + +/* + * The non-NAPI MSI interrupt handler. This needs to handle data events from + * SGE response queues as well as error and other async events as they all use + * the same MSI vector. We use one SGE response queue per port in this mode + * and protect all response queues with queue 0's lock. + */ +static irqreturn_t t3_intr_msi(int irq, void *cookie) +{ + int new_packets = 0; + struct adapter *adap = cookie; + struct sge_rspq *q = &adap->sge.qs[0].rspq; + + spin_lock(&q->lock); + + if (process_responses(adap, &adap->sge.qs[0], -1)) { + t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) | + V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx)); + new_packets = 1; + } + + if (adap->params.nports == 2 && + process_responses(adap, &adap->sge.qs[1], -1)) { + struct sge_rspq *q1 = &adap->sge.qs[1].rspq; + + t3_write_reg(adap, A_SG_GTS, V_RSPQ(q1->cntxt_id) | + V_NEWTIMER(q1->next_holdoff) | + V_NEWINDEX(q1->cidx)); + new_packets = 1; + } + + if (!new_packets && t3_slow_intr_handler(adap) == 0) + q->unhandled_irqs++; + + spin_unlock(&q->lock); + return IRQ_HANDLED; +} + +static int rspq_check_napi(struct sge_qset *qs) +{ + struct sge_rspq *q = &qs->rspq; + + if (!napi_is_scheduled(&qs->napi) && + is_new_response(&q->desc[q->cidx], q)) { + napi_schedule(&qs->napi); + return 1; + } + return 0; +} + +/* + * The MSI interrupt handler for the NAPI case (i.e., response queues serviced + * by NAPI polling). Handles data events from SGE response queues as well as + * error and other async events as they all use the same MSI vector. We use + * one SGE response queue per port in this mode and protect all response + * queues with queue 0's lock. + */ +static irqreturn_t t3_intr_msi_napi(int irq, void *cookie) +{ + int new_packets; + struct adapter *adap = cookie; + struct sge_rspq *q = &adap->sge.qs[0].rspq; + + spin_lock(&q->lock); + + new_packets = rspq_check_napi(&adap->sge.qs[0]); + if (adap->params.nports == 2) + new_packets += rspq_check_napi(&adap->sge.qs[1]); + if (!new_packets && t3_slow_intr_handler(adap) == 0) + q->unhandled_irqs++; + + spin_unlock(&q->lock); + return IRQ_HANDLED; +} + +/* + * A helper function that processes responses and issues GTS. + */ +static inline int process_responses_gts(struct adapter *adap, + struct sge_rspq *rq) +{ + int work; + + work = process_responses(adap, rspq_to_qset(rq), -1); + t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) | + V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx)); + return work; +} + +/* + * The legacy INTx interrupt handler. This needs to handle data events from + * SGE response queues as well as error and other async events as they all use + * the same interrupt pin. We use one SGE response queue per port in this mode + * and protect all response queues with queue 0's lock. + */ +static irqreturn_t t3_intr(int irq, void *cookie) +{ + int work_done, w0, w1; + struct adapter *adap = cookie; + struct sge_rspq *q0 = &adap->sge.qs[0].rspq; + struct sge_rspq *q1 = &adap->sge.qs[1].rspq; + + spin_lock(&q0->lock); + + w0 = is_new_response(&q0->desc[q0->cidx], q0); + w1 = adap->params.nports == 2 && + is_new_response(&q1->desc[q1->cidx], q1); + + if (likely(w0 | w1)) { + t3_write_reg(adap, A_PL_CLI, 0); + t3_read_reg(adap, A_PL_CLI); /* flush */ + + if (likely(w0)) + process_responses_gts(adap, q0); + + if (w1) + process_responses_gts(adap, q1); + + work_done = w0 | w1; + } else + work_done = t3_slow_intr_handler(adap); + + spin_unlock(&q0->lock); + return IRQ_RETVAL(work_done != 0); +} + +/* + * Interrupt handler for legacy INTx interrupts for T3B-based cards. + * Handles data events from SGE response queues as well as error and other + * async events as they all use the same interrupt pin. We use one SGE + * response queue per port in this mode and protect all response queues with + * queue 0's lock. + */ +static irqreturn_t t3b_intr(int irq, void *cookie) +{ + u32 map; + struct adapter *adap = cookie; + struct sge_rspq *q0 = &adap->sge.qs[0].rspq; + + t3_write_reg(adap, A_PL_CLI, 0); + map = t3_read_reg(adap, A_SG_DATA_INTR); + + if (unlikely(!map)) /* shared interrupt, most likely */ + return IRQ_NONE; + + spin_lock(&q0->lock); + + if (unlikely(map & F_ERRINTR)) + t3_slow_intr_handler(adap); + + if (likely(map & 1)) + process_responses_gts(adap, q0); + + if (map & 2) + process_responses_gts(adap, &adap->sge.qs[1].rspq); + + spin_unlock(&q0->lock); + return IRQ_HANDLED; +} + +/* + * NAPI interrupt handler for legacy INTx interrupts for T3B-based cards. + * Handles data events from SGE response queues as well as error and other + * async events as they all use the same interrupt pin. We use one SGE + * response queue per port in this mode and protect all response queues with + * queue 0's lock. + */ +static irqreturn_t t3b_intr_napi(int irq, void *cookie) +{ + u32 map; + struct adapter *adap = cookie; + struct sge_qset *qs0 = &adap->sge.qs[0]; + struct sge_rspq *q0 = &qs0->rspq; + + t3_write_reg(adap, A_PL_CLI, 0); + map = t3_read_reg(adap, A_SG_DATA_INTR); + + if (unlikely(!map)) /* shared interrupt, most likely */ + return IRQ_NONE; + + spin_lock(&q0->lock); + + if (unlikely(map & F_ERRINTR)) + t3_slow_intr_handler(adap); + + if (likely(map & 1)) + napi_schedule(&qs0->napi); + + if (map & 2) + napi_schedule(&adap->sge.qs[1].napi); + + spin_unlock(&q0->lock); + return IRQ_HANDLED; +} + +/** + * t3_intr_handler - select the top-level interrupt handler + * @adap: the adapter + * @polling: whether using NAPI to service response queues + * + * Selects the top-level interrupt handler based on the type of interrupts + * (MSI-X, MSI, or legacy) and whether NAPI will be used to service the + * response queues. + */ +irq_handler_t t3_intr_handler(struct adapter *adap, int polling) +{ + if (adap->flags & USING_MSIX) + return polling ? t3_sge_intr_msix_napi : t3_sge_intr_msix; + if (adap->flags & USING_MSI) + return polling ? t3_intr_msi_napi : t3_intr_msi; + if (adap->params.rev > 0) + return polling ? t3b_intr_napi : t3b_intr; + return t3_intr; +} + +#define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \ + F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \ + V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \ + F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \ + F_HIRCQPARITYERROR) +#define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR) +#define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \ + F_RSPQDISABLED) + +/** + * t3_sge_err_intr_handler - SGE async event interrupt handler + * @adapter: the adapter + * + * Interrupt handler for SGE asynchronous (non-data) events. + */ +void t3_sge_err_intr_handler(struct adapter *adapter) +{ + unsigned int v, status = t3_read_reg(adapter, A_SG_INT_CAUSE) & + ~F_FLEMPTY; + + if (status & SGE_PARERR) + CH_ALERT(adapter, "SGE parity error (0x%x)\n", + status & SGE_PARERR); + if (status & SGE_FRAMINGERR) + CH_ALERT(adapter, "SGE framing error (0x%x)\n", + status & SGE_FRAMINGERR); + + if (status & F_RSPQCREDITOVERFOW) + CH_ALERT(adapter, "SGE response queue credit overflow\n"); + + if (status & F_RSPQDISABLED) { + v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS); + + CH_ALERT(adapter, + "packet delivered to disabled response queue " + "(0x%x)\n", (v >> S_RSPQ0DISABLED) & 0xff); + } + + if (status & (F_HIPIODRBDROPERR | F_LOPIODRBDROPERR)) + queue_work(cxgb3_wq, &adapter->db_drop_task); + + if (status & (F_HIPRIORITYDBFULL | F_LOPRIORITYDBFULL)) + queue_work(cxgb3_wq, &adapter->db_full_task); + + if (status & (F_HIPRIORITYDBEMPTY | F_LOPRIORITYDBEMPTY)) + queue_work(cxgb3_wq, &adapter->db_empty_task); + + t3_write_reg(adapter, A_SG_INT_CAUSE, status); + if (status & SGE_FATALERR) + t3_fatal_err(adapter); +} + +/** + * sge_timer_tx - perform periodic maintenance of an SGE qset + * @t: a timer list containing the SGE queue set to maintain + * + * Runs periodically from a timer to perform maintenance of an SGE queue + * set. It performs two tasks: + * + * Cleans up any completed Tx descriptors that may still be pending. + * Normal descriptor cleanup happens when new packets are added to a Tx + * queue so this timer is relatively infrequent and does any cleanup only + * if the Tx queue has not seen any new packets in a while. We make a + * best effort attempt to reclaim descriptors, in that we don't wait + * around if we cannot get a queue's lock (which most likely is because + * someone else is queueing new packets and so will also handle the clean + * up). Since control queues use immediate data exclusively we don't + * bother cleaning them up here. + * + */ +static void sge_timer_tx(struct timer_list *t) +{ + struct sge_qset *qs = from_timer(qs, t, tx_reclaim_timer); + struct port_info *pi = netdev_priv(qs->netdev); + struct adapter *adap = pi->adapter; + unsigned int tbd[SGE_TXQ_PER_SET] = {0, 0}; + unsigned long next_period; + + if (__netif_tx_trylock(qs->tx_q)) { + tbd[TXQ_ETH] = reclaim_completed_tx(adap, &qs->txq[TXQ_ETH], + TX_RECLAIM_TIMER_CHUNK); + __netif_tx_unlock(qs->tx_q); + } + + if (spin_trylock(&qs->txq[TXQ_OFLD].lock)) { + tbd[TXQ_OFLD] = reclaim_completed_tx(adap, &qs->txq[TXQ_OFLD], + TX_RECLAIM_TIMER_CHUNK); + spin_unlock(&qs->txq[TXQ_OFLD].lock); + } + + next_period = TX_RECLAIM_PERIOD >> + (max(tbd[TXQ_ETH], tbd[TXQ_OFLD]) / + TX_RECLAIM_TIMER_CHUNK); + mod_timer(&qs->tx_reclaim_timer, jiffies + next_period); +} + +/** + * sge_timer_rx - perform periodic maintenance of an SGE qset + * @t: the timer list containing the SGE queue set to maintain + * + * a) Replenishes Rx queues that have run out due to memory shortage. + * Normally new Rx buffers are added when existing ones are consumed but + * when out of memory a queue can become empty. We try to add only a few + * buffers here, the queue will be replenished fully as these new buffers + * are used up if memory shortage has subsided. + * + * b) Return coalesced response queue credits in case a response queue is + * starved. + * + */ +static void sge_timer_rx(struct timer_list *t) +{ + spinlock_t *lock; + struct sge_qset *qs = from_timer(qs, t, rx_reclaim_timer); + struct port_info *pi = netdev_priv(qs->netdev); + struct adapter *adap = pi->adapter; + u32 status; + + lock = adap->params.rev > 0 ? + &qs->rspq.lock : &adap->sge.qs[0].rspq.lock; + + if (!spin_trylock_irq(lock)) + goto out; + + if (napi_is_scheduled(&qs->napi)) + goto unlock; + + if (adap->params.rev < 4) { + status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS); + + if (status & (1 << qs->rspq.cntxt_id)) { + qs->rspq.starved++; + if (qs->rspq.credits) { + qs->rspq.credits--; + refill_rspq(adap, &qs->rspq, 1); + qs->rspq.restarted++; + t3_write_reg(adap, A_SG_RSPQ_FL_STATUS, + 1 << qs->rspq.cntxt_id); + } + } + } + + if (qs->fl[0].credits < qs->fl[0].size) + __refill_fl(adap, &qs->fl[0]); + if (qs->fl[1].credits < qs->fl[1].size) + __refill_fl(adap, &qs->fl[1]); + +unlock: + spin_unlock_irq(lock); +out: + mod_timer(&qs->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD); +} + +/** + * t3_update_qset_coalesce - update coalescing settings for a queue set + * @qs: the SGE queue set + * @p: new queue set parameters + * + * Update the coalescing settings for an SGE queue set. Nothing is done + * if the queue set is not initialized yet. + */ +void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p) +{ + qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);/* can't be 0 */ + qs->rspq.polling = p->polling; + qs->napi.poll = p->polling ? napi_rx_handler : ofld_poll; +} + +/** + * t3_sge_alloc_qset - initialize an SGE queue set + * @adapter: the adapter + * @id: the queue set id + * @nports: how many Ethernet ports will be using this queue set + * @irq_vec_idx: the IRQ vector index for response queue interrupts + * @p: configuration parameters for this queue set + * @ntxq: number of Tx queues for the queue set + * @dev: net device associated with this queue set + * @netdevq: net device TX queue associated with this queue set + * + * Allocate resources and initialize an SGE queue set. A queue set + * comprises a response queue, two Rx free-buffer queues, and up to 3 + * Tx queues. The Tx queues are assigned roles in the order Ethernet + * queue, offload queue, and control queue. + */ +int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports, + int irq_vec_idx, const struct qset_params *p, + int ntxq, struct net_device *dev, + struct netdev_queue *netdevq) +{ + int i, avail, ret = -ENOMEM; + struct sge_qset *q = &adapter->sge.qs[id]; + + init_qset_cntxt(q, id); + timer_setup(&q->tx_reclaim_timer, sge_timer_tx, 0); + timer_setup(&q->rx_reclaim_timer, sge_timer_rx, 0); + + q->fl[0].desc = alloc_ring(adapter->pdev, p->fl_size, + sizeof(struct rx_desc), + sizeof(struct rx_sw_desc), + &q->fl[0].phys_addr, &q->fl[0].sdesc); + if (!q->fl[0].desc) + goto err; + + q->fl[1].desc = alloc_ring(adapter->pdev, p->jumbo_size, + sizeof(struct rx_desc), + sizeof(struct rx_sw_desc), + &q->fl[1].phys_addr, &q->fl[1].sdesc); + if (!q->fl[1].desc) + goto err; + + q->rspq.desc = alloc_ring(adapter->pdev, p->rspq_size, + sizeof(struct rsp_desc), 0, + &q->rspq.phys_addr, NULL); + if (!q->rspq.desc) + goto err; + + for (i = 0; i < ntxq; ++i) { + /* + * The control queue always uses immediate data so does not + * need to keep track of any sk_buffs. + */ + size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc); + + q->txq[i].desc = alloc_ring(adapter->pdev, p->txq_size[i], + sizeof(struct tx_desc), sz, + &q->txq[i].phys_addr, + &q->txq[i].sdesc); + if (!q->txq[i].desc) + goto err; + + q->txq[i].gen = 1; + q->txq[i].size = p->txq_size[i]; + spin_lock_init(&q->txq[i].lock); + skb_queue_head_init(&q->txq[i].sendq); + } + + INIT_WORK(&q->txq[TXQ_OFLD].qresume_task, restart_offloadq); + INIT_WORK(&q->txq[TXQ_CTRL].qresume_task, restart_ctrlq); + + q->fl[0].gen = q->fl[1].gen = 1; + q->fl[0].size = p->fl_size; + q->fl[1].size = p->jumbo_size; + + q->rspq.gen = 1; + q->rspq.size = p->rspq_size; + spin_lock_init(&q->rspq.lock); + skb_queue_head_init(&q->rspq.rx_queue); + + q->txq[TXQ_ETH].stop_thres = nports * + flits_to_desc(sgl_len(MAX_SKB_FRAGS + 1) + 3); + +#if FL0_PG_CHUNK_SIZE > 0 + q->fl[0].buf_size = FL0_PG_CHUNK_SIZE; +#else + q->fl[0].buf_size = SGE_RX_SM_BUF_SIZE + sizeof(struct cpl_rx_data); +#endif +#if FL1_PG_CHUNK_SIZE > 0 + q->fl[1].buf_size = FL1_PG_CHUNK_SIZE; +#else + q->fl[1].buf_size = is_offload(adapter) ? + (16 * 1024) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : + MAX_FRAME_SIZE + 2 + sizeof(struct cpl_rx_pkt); +#endif + + q->fl[0].use_pages = FL0_PG_CHUNK_SIZE > 0; + q->fl[1].use_pages = FL1_PG_CHUNK_SIZE > 0; + q->fl[0].order = FL0_PG_ORDER; + q->fl[1].order = FL1_PG_ORDER; + q->fl[0].alloc_size = FL0_PG_ALLOC_SIZE; + q->fl[1].alloc_size = FL1_PG_ALLOC_SIZE; + + spin_lock_irq(&adapter->sge.reg_lock); + + /* FL threshold comparison uses < */ + ret = t3_sge_init_rspcntxt(adapter, q->rspq.cntxt_id, irq_vec_idx, + q->rspq.phys_addr, q->rspq.size, + q->fl[0].buf_size - SGE_PG_RSVD, 1, 0); + if (ret) + goto err_unlock; + + for (i = 0; i < SGE_RXQ_PER_SET; ++i) { + ret = t3_sge_init_flcntxt(adapter, q->fl[i].cntxt_id, 0, + q->fl[i].phys_addr, q->fl[i].size, + q->fl[i].buf_size - SGE_PG_RSVD, + p->cong_thres, 1, 0); + if (ret) + goto err_unlock; + } + + ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_ETH].cntxt_id, USE_GTS, + SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr, + q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token, + 1, 0); + if (ret) + goto err_unlock; + + if (ntxq > 1) { + ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_OFLD].cntxt_id, + USE_GTS, SGE_CNTXT_OFLD, id, + q->txq[TXQ_OFLD].phys_addr, + q->txq[TXQ_OFLD].size, 0, 1, 0); + if (ret) + goto err_unlock; + } + + if (ntxq > 2) { + ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_CTRL].cntxt_id, 0, + SGE_CNTXT_CTRL, id, + q->txq[TXQ_CTRL].phys_addr, + q->txq[TXQ_CTRL].size, + q->txq[TXQ_CTRL].token, 1, 0); + if (ret) + goto err_unlock; + } + + spin_unlock_irq(&adapter->sge.reg_lock); + + q->adap = adapter; + q->netdev = dev; + q->tx_q = netdevq; + t3_update_qset_coalesce(q, p); + + avail = refill_fl(adapter, &q->fl[0], q->fl[0].size, + GFP_KERNEL | __GFP_COMP); + if (!avail) { + CH_ALERT(adapter, "free list queue 0 initialization failed\n"); + ret = -ENOMEM; + goto err; + } + if (avail < q->fl[0].size) + CH_WARN(adapter, "free list queue 0 enabled with %d credits\n", + avail); + + avail = refill_fl(adapter, &q->fl[1], q->fl[1].size, + GFP_KERNEL | __GFP_COMP); + if (avail < q->fl[1].size) + CH_WARN(adapter, "free list queue 1 enabled with %d credits\n", + avail); + refill_rspq(adapter, &q->rspq, q->rspq.size - 1); + + t3_write_reg(adapter, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) | + V_NEWTIMER(q->rspq.holdoff_tmr)); + + return 0; + +err_unlock: + spin_unlock_irq(&adapter->sge.reg_lock); +err: + t3_free_qset(adapter, q); + return ret; +} + +/** + * t3_start_sge_timers - start SGE timer call backs + * @adap: the adapter + * + * Starts each SGE queue set's timer call back + */ +void t3_start_sge_timers(struct adapter *adap) +{ + int i; + + for (i = 0; i < SGE_QSETS; ++i) { + struct sge_qset *q = &adap->sge.qs[i]; + + if (q->tx_reclaim_timer.function) + mod_timer(&q->tx_reclaim_timer, + jiffies + TX_RECLAIM_PERIOD); + + if (q->rx_reclaim_timer.function) + mod_timer(&q->rx_reclaim_timer, + jiffies + RX_RECLAIM_PERIOD); + } +} + +/** + * t3_stop_sge_timers - stop SGE timer call backs + * @adap: the adapter + * + * Stops each SGE queue set's timer call back + */ +void t3_stop_sge_timers(struct adapter *adap) +{ + int i; + + for (i = 0; i < SGE_QSETS; ++i) { + struct sge_qset *q = &adap->sge.qs[i]; + + if (q->tx_reclaim_timer.function) + del_timer_sync(&q->tx_reclaim_timer); + if (q->rx_reclaim_timer.function) + del_timer_sync(&q->rx_reclaim_timer); + } +} + +/** + * t3_free_sge_resources - free SGE resources + * @adap: the adapter + * + * Frees resources used by the SGE queue sets. + */ +void t3_free_sge_resources(struct adapter *adap) +{ + int i; + + for (i = 0; i < SGE_QSETS; ++i) + t3_free_qset(adap, &adap->sge.qs[i]); +} + +/** + * t3_sge_start - enable SGE + * @adap: the adapter + * + * Enables the SGE for DMAs. This is the last step in starting packet + * transfers. + */ +void t3_sge_start(struct adapter *adap) +{ + t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE); +} + +/** + * t3_sge_stop_dma - Disable SGE DMA engine operation + * @adap: the adapter + * + * Can be invoked from interrupt context e.g. error handler. + * + * Note that this function cannot disable the restart of works as + * it cannot wait if called from interrupt context, however the + * works will have no effect since the doorbells are disabled. The + * driver will call tg3_sge_stop() later from process context, at + * which time the works will be stopped if they are still running. + */ +void t3_sge_stop_dma(struct adapter *adap) +{ + t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, 0); +} + +/** + * t3_sge_stop - disable SGE operation completly + * @adap: the adapter + * + * Called from process context. Disables the DMA engine and any + * pending queue restart works. + */ +void t3_sge_stop(struct adapter *adap) +{ + int i; + + t3_sge_stop_dma(adap); + + /* workqueues aren't initialized otherwise */ + if (!(adap->flags & FULL_INIT_DONE)) + return; + for (i = 0; i < SGE_QSETS; ++i) { + struct sge_qset *qs = &adap->sge.qs[i]; + + cancel_work_sync(&qs->txq[TXQ_OFLD].qresume_task); + cancel_work_sync(&qs->txq[TXQ_CTRL].qresume_task); + } +} + +/** + * t3_sge_init - initialize SGE + * @adap: the adapter + * @p: the SGE parameters + * + * Performs SGE initialization needed every time after a chip reset. + * We do not initialize any of the queue sets here, instead the driver + * top-level must request those individually. We also do not enable DMA + * here, that should be done after the queues have been set up. + */ +void t3_sge_init(struct adapter *adap, struct sge_params *p) +{ + unsigned int ctrl, ups = ffs(pci_resource_len(adap->pdev, 2) >> 12); + + ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL | + F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN | + V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS | + V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING; +#if SGE_NUM_GENBITS == 1 + ctrl |= F_EGRGENCTRL; +#endif + if (adap->params.rev > 0) { + if (!(adap->flags & (USING_MSIX | USING_MSI))) + ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ; + } + t3_write_reg(adap, A_SG_CONTROL, ctrl); + t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) | + V_LORCQDRBTHRSH(512)); + t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10); + t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) | + V_TIMEOUT(200 * core_ticks_per_usec(adap))); + t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH, + adap->params.rev < T3_REV_C ? 1000 : 500); + t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256); + t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000); + t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256); + t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff)); + t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024); +} + +/** + * t3_sge_prep - one-time SGE initialization + * @adap: the associated adapter + * @p: SGE parameters + * + * Performs one-time initialization of SGE SW state. Includes determining + * defaults for the assorted SGE parameters, which admins can change until + * they are used to initialize the SGE. + */ +void t3_sge_prep(struct adapter *adap, struct sge_params *p) +{ + int i; + + p->max_pkt_size = (16 * 1024) - sizeof(struct cpl_rx_data) - + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + + for (i = 0; i < SGE_QSETS; ++i) { + struct qset_params *q = p->qset + i; + + q->polling = adap->params.rev > 0; + q->coalesce_usecs = 5; + q->rspq_size = 1024; + q->fl_size = 1024; + q->jumbo_size = 512; + q->txq_size[TXQ_ETH] = 1024; + q->txq_size[TXQ_OFLD] = 1024; + q->txq_size[TXQ_CTRL] = 256; + q->cong_thres = 0; + } + + spin_lock_init(&adap->sge.reg_lock); +} diff --git a/drivers/net/ethernet/chelsio/cxgb3/sge_defs.h b/drivers/net/ethernet/chelsio/cxgb3/sge_defs.h new file mode 100644 index 000000000..c31ce8dc9 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/sge_defs.h @@ -0,0 +1,256 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is automatically generated --- any changes will be lost. + */ + +#ifndef _SGE_DEFS_H +#define _SGE_DEFS_H + +#define S_EC_CREDITS 0 +#define M_EC_CREDITS 0x7FFF +#define V_EC_CREDITS(x) ((x) << S_EC_CREDITS) +#define G_EC_CREDITS(x) (((x) >> S_EC_CREDITS) & M_EC_CREDITS) + +#define S_EC_GTS 15 +#define V_EC_GTS(x) ((x) << S_EC_GTS) +#define F_EC_GTS V_EC_GTS(1U) + +#define S_EC_INDEX 16 +#define M_EC_INDEX 0xFFFF +#define V_EC_INDEX(x) ((x) << S_EC_INDEX) +#define G_EC_INDEX(x) (((x) >> S_EC_INDEX) & M_EC_INDEX) + +#define S_EC_SIZE 0 +#define M_EC_SIZE 0xFFFF +#define V_EC_SIZE(x) ((x) << S_EC_SIZE) +#define G_EC_SIZE(x) (((x) >> S_EC_SIZE) & M_EC_SIZE) + +#define S_EC_BASE_LO 16 +#define M_EC_BASE_LO 0xFFFF +#define V_EC_BASE_LO(x) ((x) << S_EC_BASE_LO) +#define G_EC_BASE_LO(x) (((x) >> S_EC_BASE_LO) & M_EC_BASE_LO) + +#define S_EC_BASE_HI 0 +#define M_EC_BASE_HI 0xF +#define V_EC_BASE_HI(x) ((x) << S_EC_BASE_HI) +#define G_EC_BASE_HI(x) (((x) >> S_EC_BASE_HI) & M_EC_BASE_HI) + +#define S_EC_RESPQ 4 +#define M_EC_RESPQ 0x7 +#define V_EC_RESPQ(x) ((x) << S_EC_RESPQ) +#define G_EC_RESPQ(x) (((x) >> S_EC_RESPQ) & M_EC_RESPQ) + +#define S_EC_TYPE 7 +#define M_EC_TYPE 0x7 +#define V_EC_TYPE(x) ((x) << S_EC_TYPE) +#define G_EC_TYPE(x) (((x) >> S_EC_TYPE) & M_EC_TYPE) + +#define S_EC_GEN 10 +#define V_EC_GEN(x) ((x) << S_EC_GEN) +#define F_EC_GEN V_EC_GEN(1U) + +#define S_EC_UP_TOKEN 11 +#define M_EC_UP_TOKEN 0xFFFFF +#define V_EC_UP_TOKEN(x) ((x) << S_EC_UP_TOKEN) +#define G_EC_UP_TOKEN(x) (((x) >> S_EC_UP_TOKEN) & M_EC_UP_TOKEN) + +#define S_EC_VALID 31 +#define V_EC_VALID(x) ((x) << S_EC_VALID) +#define F_EC_VALID V_EC_VALID(1U) + +#define S_RQ_MSI_VEC 20 +#define M_RQ_MSI_VEC 0x3F +#define V_RQ_MSI_VEC(x) ((x) << S_RQ_MSI_VEC) +#define G_RQ_MSI_VEC(x) (((x) >> S_RQ_MSI_VEC) & M_RQ_MSI_VEC) + +#define S_RQ_INTR_EN 26 +#define V_RQ_INTR_EN(x) ((x) << S_RQ_INTR_EN) +#define F_RQ_INTR_EN V_RQ_INTR_EN(1U) + +#define S_RQ_GEN 28 +#define V_RQ_GEN(x) ((x) << S_RQ_GEN) +#define F_RQ_GEN V_RQ_GEN(1U) + +#define S_CQ_INDEX 0 +#define M_CQ_INDEX 0xFFFF +#define V_CQ_INDEX(x) ((x) << S_CQ_INDEX) +#define G_CQ_INDEX(x) (((x) >> S_CQ_INDEX) & M_CQ_INDEX) + +#define S_CQ_SIZE 16 +#define M_CQ_SIZE 0xFFFF +#define V_CQ_SIZE(x) ((x) << S_CQ_SIZE) +#define G_CQ_SIZE(x) (((x) >> S_CQ_SIZE) & M_CQ_SIZE) + +#define S_CQ_BASE_HI 0 +#define M_CQ_BASE_HI 0xFFFFF +#define V_CQ_BASE_HI(x) ((x) << S_CQ_BASE_HI) +#define G_CQ_BASE_HI(x) (((x) >> S_CQ_BASE_HI) & M_CQ_BASE_HI) + +#define S_CQ_RSPQ 20 +#define M_CQ_RSPQ 0x3F +#define V_CQ_RSPQ(x) ((x) << S_CQ_RSPQ) +#define G_CQ_RSPQ(x) (((x) >> S_CQ_RSPQ) & M_CQ_RSPQ) + +#define S_CQ_ASYNC_NOTIF 26 +#define V_CQ_ASYNC_NOTIF(x) ((x) << S_CQ_ASYNC_NOTIF) +#define F_CQ_ASYNC_NOTIF V_CQ_ASYNC_NOTIF(1U) + +#define S_CQ_ARMED 27 +#define V_CQ_ARMED(x) ((x) << S_CQ_ARMED) +#define F_CQ_ARMED V_CQ_ARMED(1U) + +#define S_CQ_ASYNC_NOTIF_SOL 28 +#define V_CQ_ASYNC_NOTIF_SOL(x) ((x) << S_CQ_ASYNC_NOTIF_SOL) +#define F_CQ_ASYNC_NOTIF_SOL V_CQ_ASYNC_NOTIF_SOL(1U) + +#define S_CQ_GEN 29 +#define V_CQ_GEN(x) ((x) << S_CQ_GEN) +#define F_CQ_GEN V_CQ_GEN(1U) + +#define S_CQ_ERR 30 +#define V_CQ_ERR(x) ((x) << S_CQ_ERR) +#define F_CQ_ERR V_CQ_ERR(1U) + +#define S_CQ_OVERFLOW_MODE 31 +#define V_CQ_OVERFLOW_MODE(x) ((x) << S_CQ_OVERFLOW_MODE) +#define F_CQ_OVERFLOW_MODE V_CQ_OVERFLOW_MODE(1U) + +#define S_CQ_CREDITS 0 +#define M_CQ_CREDITS 0xFFFF +#define V_CQ_CREDITS(x) ((x) << S_CQ_CREDITS) +#define G_CQ_CREDITS(x) (((x) >> S_CQ_CREDITS) & M_CQ_CREDITS) + +#define S_CQ_CREDIT_THRES 16 +#define M_CQ_CREDIT_THRES 0x1FFF +#define V_CQ_CREDIT_THRES(x) ((x) << S_CQ_CREDIT_THRES) +#define G_CQ_CREDIT_THRES(x) (((x) >> S_CQ_CREDIT_THRES) & M_CQ_CREDIT_THRES) + +#define S_FL_BASE_HI 0 +#define M_FL_BASE_HI 0xFFFFF +#define V_FL_BASE_HI(x) ((x) << S_FL_BASE_HI) +#define G_FL_BASE_HI(x) (((x) >> S_FL_BASE_HI) & M_FL_BASE_HI) + +#define S_FL_INDEX_LO 20 +#define M_FL_INDEX_LO 0xFFF +#define V_FL_INDEX_LO(x) ((x) << S_FL_INDEX_LO) +#define G_FL_INDEX_LO(x) (((x) >> S_FL_INDEX_LO) & M_FL_INDEX_LO) + +#define S_FL_INDEX_HI 0 +#define M_FL_INDEX_HI 0xF +#define V_FL_INDEX_HI(x) ((x) << S_FL_INDEX_HI) +#define G_FL_INDEX_HI(x) (((x) >> S_FL_INDEX_HI) & M_FL_INDEX_HI) + +#define S_FL_SIZE 4 +#define M_FL_SIZE 0xFFFF +#define V_FL_SIZE(x) ((x) << S_FL_SIZE) +#define G_FL_SIZE(x) (((x) >> S_FL_SIZE) & M_FL_SIZE) + +#define S_FL_GEN 20 +#define V_FL_GEN(x) ((x) << S_FL_GEN) +#define F_FL_GEN V_FL_GEN(1U) + +#define S_FL_ENTRY_SIZE_LO 21 +#define M_FL_ENTRY_SIZE_LO 0x7FF +#define V_FL_ENTRY_SIZE_LO(x) ((x) << S_FL_ENTRY_SIZE_LO) +#define G_FL_ENTRY_SIZE_LO(x) (((x) >> S_FL_ENTRY_SIZE_LO) & M_FL_ENTRY_SIZE_LO) + +#define S_FL_ENTRY_SIZE_HI 0 +#define M_FL_ENTRY_SIZE_HI 0x1FFFFF +#define V_FL_ENTRY_SIZE_HI(x) ((x) << S_FL_ENTRY_SIZE_HI) +#define G_FL_ENTRY_SIZE_HI(x) (((x) >> S_FL_ENTRY_SIZE_HI) & M_FL_ENTRY_SIZE_HI) + +#define S_FL_CONG_THRES 21 +#define M_FL_CONG_THRES 0x3FF +#define V_FL_CONG_THRES(x) ((x) << S_FL_CONG_THRES) +#define G_FL_CONG_THRES(x) (((x) >> S_FL_CONG_THRES) & M_FL_CONG_THRES) + +#define S_FL_GTS 31 +#define V_FL_GTS(x) ((x) << S_FL_GTS) +#define F_FL_GTS V_FL_GTS(1U) + +#define S_FLD_GEN1 31 +#define V_FLD_GEN1(x) ((x) << S_FLD_GEN1) +#define F_FLD_GEN1 V_FLD_GEN1(1U) + +#define S_FLD_GEN2 0 +#define V_FLD_GEN2(x) ((x) << S_FLD_GEN2) +#define F_FLD_GEN2 V_FLD_GEN2(1U) + +#define S_RSPD_TXQ1_CR 0 +#define M_RSPD_TXQ1_CR 0x7F +#define V_RSPD_TXQ1_CR(x) ((x) << S_RSPD_TXQ1_CR) +#define G_RSPD_TXQ1_CR(x) (((x) >> S_RSPD_TXQ1_CR) & M_RSPD_TXQ1_CR) + +#define S_RSPD_TXQ1_GTS 7 +#define V_RSPD_TXQ1_GTS(x) ((x) << S_RSPD_TXQ1_GTS) +#define F_RSPD_TXQ1_GTS V_RSPD_TXQ1_GTS(1U) + +#define S_RSPD_TXQ2_CR 8 +#define M_RSPD_TXQ2_CR 0x7F +#define V_RSPD_TXQ2_CR(x) ((x) << S_RSPD_TXQ2_CR) +#define G_RSPD_TXQ2_CR(x) (((x) >> S_RSPD_TXQ2_CR) & M_RSPD_TXQ2_CR) + +#define S_RSPD_TXQ2_GTS 15 +#define V_RSPD_TXQ2_GTS(x) ((x) << S_RSPD_TXQ2_GTS) +#define F_RSPD_TXQ2_GTS V_RSPD_TXQ2_GTS(1U) + +#define S_RSPD_TXQ0_CR 16 +#define M_RSPD_TXQ0_CR 0x7F +#define V_RSPD_TXQ0_CR(x) ((x) << S_RSPD_TXQ0_CR) +#define G_RSPD_TXQ0_CR(x) (((x) >> S_RSPD_TXQ0_CR) & M_RSPD_TXQ0_CR) + +#define S_RSPD_TXQ0_GTS 23 +#define V_RSPD_TXQ0_GTS(x) ((x) << S_RSPD_TXQ0_GTS) +#define F_RSPD_TXQ0_GTS V_RSPD_TXQ0_GTS(1U) + +#define S_RSPD_EOP 24 +#define V_RSPD_EOP(x) ((x) << S_RSPD_EOP) +#define F_RSPD_EOP V_RSPD_EOP(1U) + +#define S_RSPD_SOP 25 +#define V_RSPD_SOP(x) ((x) << S_RSPD_SOP) +#define F_RSPD_SOP V_RSPD_SOP(1U) + +#define S_RSPD_ASYNC_NOTIF 26 +#define V_RSPD_ASYNC_NOTIF(x) ((x) << S_RSPD_ASYNC_NOTIF) +#define F_RSPD_ASYNC_NOTIF V_RSPD_ASYNC_NOTIF(1U) + +#define S_RSPD_FL0_GTS 27 +#define V_RSPD_FL0_GTS(x) ((x) << S_RSPD_FL0_GTS) +#define F_RSPD_FL0_GTS V_RSPD_FL0_GTS(1U) + +#define S_RSPD_FL1_GTS 28 +#define V_RSPD_FL1_GTS(x) ((x) << S_RSPD_FL1_GTS) +#define F_RSPD_FL1_GTS V_RSPD_FL1_GTS(1U) + +#define S_RSPD_IMM_DATA_VALID 29 +#define V_RSPD_IMM_DATA_VALID(x) ((x) << S_RSPD_IMM_DATA_VALID) +#define F_RSPD_IMM_DATA_VALID V_RSPD_IMM_DATA_VALID(1U) + +#define S_RSPD_OFFLOAD 30 +#define V_RSPD_OFFLOAD(x) ((x) << S_RSPD_OFFLOAD) +#define F_RSPD_OFFLOAD V_RSPD_OFFLOAD(1U) + +#define S_RSPD_GEN1 31 +#define V_RSPD_GEN1(x) ((x) << S_RSPD_GEN1) +#define F_RSPD_GEN1 V_RSPD_GEN1(1U) + +#define S_RSPD_LEN 0 +#define M_RSPD_LEN 0x7FFFFFFF +#define V_RSPD_LEN(x) ((x) << S_RSPD_LEN) +#define G_RSPD_LEN(x) (((x) >> S_RSPD_LEN) & M_RSPD_LEN) + +#define S_RSPD_FLQ 31 +#define V_RSPD_FLQ(x) ((x) << S_RSPD_FLQ) +#define F_RSPD_FLQ V_RSPD_FLQ(1U) + +#define S_RSPD_GEN2 0 +#define V_RSPD_GEN2(x) ((x) << S_RSPD_GEN2) +#define F_RSPD_GEN2 V_RSPD_GEN2(1U) + +#define S_RSPD_INR_VEC 1 +#define M_RSPD_INR_VEC 0x7F +#define V_RSPD_INR_VEC(x) ((x) << S_RSPD_INR_VEC) +#define G_RSPD_INR_VEC(x) (((x) >> S_RSPD_INR_VEC) & M_RSPD_INR_VEC) + +#endif /* _SGE_DEFS_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb3/t3_cpl.h b/drivers/net/ethernet/chelsio/cxgb3/t3_cpl.h new file mode 100644 index 000000000..68bb5f39f --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/t3_cpl.h @@ -0,0 +1,1495 @@ +/* + * Copyright (c) 2004-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef T3_CPL_H +#define T3_CPL_H + +#if !defined(__LITTLE_ENDIAN_BITFIELD) && !defined(__BIG_ENDIAN_BITFIELD) +# include <asm/byteorder.h> +#endif + +enum CPL_opcode { + CPL_PASS_OPEN_REQ = 0x1, + CPL_PASS_ACCEPT_RPL = 0x2, + CPL_ACT_OPEN_REQ = 0x3, + CPL_SET_TCB = 0x4, + CPL_SET_TCB_FIELD = 0x5, + CPL_GET_TCB = 0x6, + CPL_PCMD = 0x7, + CPL_CLOSE_CON_REQ = 0x8, + CPL_CLOSE_LISTSRV_REQ = 0x9, + CPL_ABORT_REQ = 0xA, + CPL_ABORT_RPL = 0xB, + CPL_TX_DATA = 0xC, + CPL_RX_DATA_ACK = 0xD, + CPL_TX_PKT = 0xE, + CPL_RTE_DELETE_REQ = 0xF, + CPL_RTE_WRITE_REQ = 0x10, + CPL_RTE_READ_REQ = 0x11, + CPL_L2T_WRITE_REQ = 0x12, + CPL_L2T_READ_REQ = 0x13, + CPL_SMT_WRITE_REQ = 0x14, + CPL_SMT_READ_REQ = 0x15, + CPL_TX_PKT_LSO = 0x16, + CPL_PCMD_READ = 0x17, + CPL_BARRIER = 0x18, + CPL_TID_RELEASE = 0x1A, + + CPL_CLOSE_LISTSRV_RPL = 0x20, + CPL_ERROR = 0x21, + CPL_GET_TCB_RPL = 0x22, + CPL_L2T_WRITE_RPL = 0x23, + CPL_PCMD_READ_RPL = 0x24, + CPL_PCMD_RPL = 0x25, + CPL_PEER_CLOSE = 0x26, + CPL_RTE_DELETE_RPL = 0x27, + CPL_RTE_WRITE_RPL = 0x28, + CPL_RX_DDP_COMPLETE = 0x29, + CPL_RX_PHYS_ADDR = 0x2A, + CPL_RX_PKT = 0x2B, + CPL_RX_URG_NOTIFY = 0x2C, + CPL_SET_TCB_RPL = 0x2D, + CPL_SMT_WRITE_RPL = 0x2E, + CPL_TX_DATA_ACK = 0x2F, + + CPL_ABORT_REQ_RSS = 0x30, + CPL_ABORT_RPL_RSS = 0x31, + CPL_CLOSE_CON_RPL = 0x32, + CPL_ISCSI_HDR = 0x33, + CPL_L2T_READ_RPL = 0x34, + CPL_RDMA_CQE = 0x35, + CPL_RDMA_CQE_READ_RSP = 0x36, + CPL_RDMA_CQE_ERR = 0x37, + CPL_RTE_READ_RPL = 0x38, + CPL_RX_DATA = 0x39, + + CPL_ACT_OPEN_RPL = 0x40, + CPL_PASS_OPEN_RPL = 0x41, + CPL_RX_DATA_DDP = 0x42, + CPL_SMT_READ_RPL = 0x43, + + CPL_ACT_ESTABLISH = 0x50, + CPL_PASS_ESTABLISH = 0x51, + + CPL_PASS_ACCEPT_REQ = 0x70, + + CPL_ASYNC_NOTIF = 0x80, /* fake opcode for async notifications */ + + CPL_TX_DMA_ACK = 0xA0, + CPL_RDMA_READ_REQ = 0xA1, + CPL_RDMA_TERMINATE = 0xA2, + CPL_TRACE_PKT = 0xA3, + CPL_RDMA_EC_STATUS = 0xA5, + + NUM_CPL_CMDS /* must be last and previous entries must be sorted */ +}; + +enum CPL_error { + CPL_ERR_NONE = 0, + CPL_ERR_TCAM_PARITY = 1, + CPL_ERR_TCAM_FULL = 3, + CPL_ERR_CONN_RESET = 20, + CPL_ERR_CONN_EXIST = 22, + CPL_ERR_ARP_MISS = 23, + CPL_ERR_BAD_SYN = 24, + CPL_ERR_CONN_TIMEDOUT = 30, + CPL_ERR_XMIT_TIMEDOUT = 31, + CPL_ERR_PERSIST_TIMEDOUT = 32, + CPL_ERR_FINWAIT2_TIMEDOUT = 33, + CPL_ERR_KEEPALIVE_TIMEDOUT = 34, + CPL_ERR_RTX_NEG_ADVICE = 35, + CPL_ERR_PERSIST_NEG_ADVICE = 36, + CPL_ERR_ABORT_FAILED = 42, + CPL_ERR_GENERAL = 99 +}; + +enum { + CPL_CONN_POLICY_AUTO = 0, + CPL_CONN_POLICY_ASK = 1, + CPL_CONN_POLICY_DENY = 3 +}; + +enum { + ULP_MODE_NONE = 0, + ULP_MODE_ISCSI = 2, + ULP_MODE_RDMA = 4, + ULP_MODE_TCPDDP = 5 +}; + +enum { + ULP_CRC_HEADER = 1 << 0, + ULP_CRC_DATA = 1 << 1 +}; + +enum { + CPL_PASS_OPEN_ACCEPT, + CPL_PASS_OPEN_REJECT +}; + +enum { + CPL_ABORT_SEND_RST = 0, + CPL_ABORT_NO_RST, + CPL_ABORT_POST_CLOSE_REQ = 2 +}; + +enum { /* TX_PKT_LSO ethernet types */ + CPL_ETH_II, + CPL_ETH_II_VLAN, + CPL_ETH_802_3, + CPL_ETH_802_3_VLAN +}; + +enum { /* TCP congestion control algorithms */ + CONG_ALG_RENO, + CONG_ALG_TAHOE, + CONG_ALG_NEWRENO, + CONG_ALG_HIGHSPEED +}; + +enum { /* RSS hash type */ + RSS_HASH_NONE = 0, + RSS_HASH_2_TUPLE = 1, + RSS_HASH_4_TUPLE = 2, + RSS_HASH_TCPV6 = 3 +}; + +union opcode_tid { + __be32 opcode_tid; + __u8 opcode; +}; + +#define S_OPCODE 24 +#define V_OPCODE(x) ((x) << S_OPCODE) +#define G_OPCODE(x) (((x) >> S_OPCODE) & 0xFF) +#define G_TID(x) ((x) & 0xFFFFFF) + +#define S_QNUM 0 +#define G_QNUM(x) (((x) >> S_QNUM) & 0xFFFF) + +#define S_HASHTYPE 22 +#define M_HASHTYPE 0x3 +#define G_HASHTYPE(x) (((x) >> S_HASHTYPE) & M_HASHTYPE) + +/* tid is assumed to be 24-bits */ +#define MK_OPCODE_TID(opcode, tid) (V_OPCODE(opcode) | (tid)) + +#define OPCODE_TID(cmd) ((cmd)->ot.opcode_tid) + +/* extract the TID from a CPL command */ +#define GET_TID(cmd) (G_TID(ntohl(OPCODE_TID(cmd)))) + +struct tcp_options { + __be16 mss; + __u8 wsf; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8:5; + __u8 ecn:1; + __u8 sack:1; + __u8 tstamp:1; +#else + __u8 tstamp:1; + __u8 sack:1; + __u8 ecn:1; + __u8:5; +#endif +}; + +struct rss_header { + __u8 opcode; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 cpu_idx:6; + __u8 hash_type:2; +#else + __u8 hash_type:2; + __u8 cpu_idx:6; +#endif + __be16 cq_idx; + __be32 rss_hash_val; +}; + +#ifndef CHELSIO_FW +struct work_request_hdr { + __be32 wr_hi; + __be32 wr_lo; +}; + +/* wr_hi fields */ +#define S_WR_SGE_CREDITS 0 +#define M_WR_SGE_CREDITS 0xFF +#define V_WR_SGE_CREDITS(x) ((x) << S_WR_SGE_CREDITS) +#define G_WR_SGE_CREDITS(x) (((x) >> S_WR_SGE_CREDITS) & M_WR_SGE_CREDITS) + +#define S_WR_SGLSFLT 8 +#define M_WR_SGLSFLT 0xFF +#define V_WR_SGLSFLT(x) ((x) << S_WR_SGLSFLT) +#define G_WR_SGLSFLT(x) (((x) >> S_WR_SGLSFLT) & M_WR_SGLSFLT) + +#define S_WR_BCNTLFLT 16 +#define M_WR_BCNTLFLT 0xF +#define V_WR_BCNTLFLT(x) ((x) << S_WR_BCNTLFLT) +#define G_WR_BCNTLFLT(x) (((x) >> S_WR_BCNTLFLT) & M_WR_BCNTLFLT) + +#define S_WR_DATATYPE 20 +#define V_WR_DATATYPE(x) ((x) << S_WR_DATATYPE) +#define F_WR_DATATYPE V_WR_DATATYPE(1U) + +#define S_WR_COMPL 21 +#define V_WR_COMPL(x) ((x) << S_WR_COMPL) +#define F_WR_COMPL V_WR_COMPL(1U) + +#define S_WR_EOP 22 +#define V_WR_EOP(x) ((x) << S_WR_EOP) +#define F_WR_EOP V_WR_EOP(1U) + +#define S_WR_SOP 23 +#define V_WR_SOP(x) ((x) << S_WR_SOP) +#define F_WR_SOP V_WR_SOP(1U) + +#define S_WR_OP 24 +#define M_WR_OP 0xFF +#define V_WR_OP(x) ((x) << S_WR_OP) +#define G_WR_OP(x) (((x) >> S_WR_OP) & M_WR_OP) + +/* wr_lo fields */ +#define S_WR_LEN 0 +#define M_WR_LEN 0xFF +#define V_WR_LEN(x) ((x) << S_WR_LEN) +#define G_WR_LEN(x) (((x) >> S_WR_LEN) & M_WR_LEN) + +#define S_WR_TID 8 +#define M_WR_TID 0xFFFFF +#define V_WR_TID(x) ((x) << S_WR_TID) +#define G_WR_TID(x) (((x) >> S_WR_TID) & M_WR_TID) + +#define S_WR_CR_FLUSH 30 +#define V_WR_CR_FLUSH(x) ((x) << S_WR_CR_FLUSH) +#define F_WR_CR_FLUSH V_WR_CR_FLUSH(1U) + +#define S_WR_GEN 31 +#define V_WR_GEN(x) ((x) << S_WR_GEN) +#define F_WR_GEN V_WR_GEN(1U) + +# define WR_HDR struct work_request_hdr wr +# define RSS_HDR +#else +# define WR_HDR +# define RSS_HDR struct rss_header rss_hdr; +#endif + +/* option 0 lower-half fields */ +#define S_CPL_STATUS 0 +#define M_CPL_STATUS 0xFF +#define V_CPL_STATUS(x) ((x) << S_CPL_STATUS) +#define G_CPL_STATUS(x) (((x) >> S_CPL_STATUS) & M_CPL_STATUS) + +#define S_INJECT_TIMER 6 +#define V_INJECT_TIMER(x) ((x) << S_INJECT_TIMER) +#define F_INJECT_TIMER V_INJECT_TIMER(1U) + +#define S_NO_OFFLOAD 7 +#define V_NO_OFFLOAD(x) ((x) << S_NO_OFFLOAD) +#define F_NO_OFFLOAD V_NO_OFFLOAD(1U) + +#define S_ULP_MODE 8 +#define M_ULP_MODE 0xF +#define V_ULP_MODE(x) ((x) << S_ULP_MODE) +#define G_ULP_MODE(x) (((x) >> S_ULP_MODE) & M_ULP_MODE) + +#define S_RCV_BUFSIZ 12 +#define M_RCV_BUFSIZ 0x3FFF +#define V_RCV_BUFSIZ(x) ((x) << S_RCV_BUFSIZ) +#define G_RCV_BUFSIZ(x) (((x) >> S_RCV_BUFSIZ) & M_RCV_BUFSIZ) + +#define S_TOS 26 +#define M_TOS 0x3F +#define V_TOS(x) ((x) << S_TOS) +#define G_TOS(x) (((x) >> S_TOS) & M_TOS) + +/* option 0 upper-half fields */ +#define S_DELACK 0 +#define V_DELACK(x) ((x) << S_DELACK) +#define F_DELACK V_DELACK(1U) + +#define S_NO_CONG 1 +#define V_NO_CONG(x) ((x) << S_NO_CONG) +#define F_NO_CONG V_NO_CONG(1U) + +#define S_SRC_MAC_SEL 2 +#define M_SRC_MAC_SEL 0x3 +#define V_SRC_MAC_SEL(x) ((x) << S_SRC_MAC_SEL) +#define G_SRC_MAC_SEL(x) (((x) >> S_SRC_MAC_SEL) & M_SRC_MAC_SEL) + +#define S_L2T_IDX 4 +#define M_L2T_IDX 0x7FF +#define V_L2T_IDX(x) ((x) << S_L2T_IDX) +#define G_L2T_IDX(x) (((x) >> S_L2T_IDX) & M_L2T_IDX) + +#define S_TX_CHANNEL 15 +#define V_TX_CHANNEL(x) ((x) << S_TX_CHANNEL) +#define F_TX_CHANNEL V_TX_CHANNEL(1U) + +#define S_TCAM_BYPASS 16 +#define V_TCAM_BYPASS(x) ((x) << S_TCAM_BYPASS) +#define F_TCAM_BYPASS V_TCAM_BYPASS(1U) + +#define S_NAGLE 17 +#define V_NAGLE(x) ((x) << S_NAGLE) +#define F_NAGLE V_NAGLE(1U) + +#define S_WND_SCALE 18 +#define M_WND_SCALE 0xF +#define V_WND_SCALE(x) ((x) << S_WND_SCALE) +#define G_WND_SCALE(x) (((x) >> S_WND_SCALE) & M_WND_SCALE) + +#define S_KEEP_ALIVE 22 +#define V_KEEP_ALIVE(x) ((x) << S_KEEP_ALIVE) +#define F_KEEP_ALIVE V_KEEP_ALIVE(1U) + +#define S_MAX_RETRANS 23 +#define M_MAX_RETRANS 0xF +#define V_MAX_RETRANS(x) ((x) << S_MAX_RETRANS) +#define G_MAX_RETRANS(x) (((x) >> S_MAX_RETRANS) & M_MAX_RETRANS) + +#define S_MAX_RETRANS_OVERRIDE 27 +#define V_MAX_RETRANS_OVERRIDE(x) ((x) << S_MAX_RETRANS_OVERRIDE) +#define F_MAX_RETRANS_OVERRIDE V_MAX_RETRANS_OVERRIDE(1U) + +#define S_MSS_IDX 28 +#define M_MSS_IDX 0xF +#define V_MSS_IDX(x) ((x) << S_MSS_IDX) +#define G_MSS_IDX(x) (((x) >> S_MSS_IDX) & M_MSS_IDX) + +/* option 1 fields */ +#define S_RSS_ENABLE 0 +#define V_RSS_ENABLE(x) ((x) << S_RSS_ENABLE) +#define F_RSS_ENABLE V_RSS_ENABLE(1U) + +#define S_RSS_MASK_LEN 1 +#define M_RSS_MASK_LEN 0x7 +#define V_RSS_MASK_LEN(x) ((x) << S_RSS_MASK_LEN) +#define G_RSS_MASK_LEN(x) (((x) >> S_RSS_MASK_LEN) & M_RSS_MASK_LEN) + +#define S_CPU_IDX 4 +#define M_CPU_IDX 0x3F +#define V_CPU_IDX(x) ((x) << S_CPU_IDX) +#define G_CPU_IDX(x) (((x) >> S_CPU_IDX) & M_CPU_IDX) + +#define S_MAC_MATCH_VALID 18 +#define V_MAC_MATCH_VALID(x) ((x) << S_MAC_MATCH_VALID) +#define F_MAC_MATCH_VALID V_MAC_MATCH_VALID(1U) + +#define S_CONN_POLICY 19 +#define M_CONN_POLICY 0x3 +#define V_CONN_POLICY(x) ((x) << S_CONN_POLICY) +#define G_CONN_POLICY(x) (((x) >> S_CONN_POLICY) & M_CONN_POLICY) + +#define S_SYN_DEFENSE 21 +#define V_SYN_DEFENSE(x) ((x) << S_SYN_DEFENSE) +#define F_SYN_DEFENSE V_SYN_DEFENSE(1U) + +#define S_VLAN_PRI 22 +#define M_VLAN_PRI 0x3 +#define V_VLAN_PRI(x) ((x) << S_VLAN_PRI) +#define G_VLAN_PRI(x) (((x) >> S_VLAN_PRI) & M_VLAN_PRI) + +#define S_VLAN_PRI_VALID 24 +#define V_VLAN_PRI_VALID(x) ((x) << S_VLAN_PRI_VALID) +#define F_VLAN_PRI_VALID V_VLAN_PRI_VALID(1U) + +#define S_PKT_TYPE 25 +#define M_PKT_TYPE 0x3 +#define V_PKT_TYPE(x) ((x) << S_PKT_TYPE) +#define G_PKT_TYPE(x) (((x) >> S_PKT_TYPE) & M_PKT_TYPE) + +#define S_MAC_MATCH 27 +#define M_MAC_MATCH 0x1F +#define V_MAC_MATCH(x) ((x) << S_MAC_MATCH) +#define G_MAC_MATCH(x) (((x) >> S_MAC_MATCH) & M_MAC_MATCH) + +/* option 2 fields */ +#define S_CPU_INDEX 0 +#define M_CPU_INDEX 0x7F +#define V_CPU_INDEX(x) ((x) << S_CPU_INDEX) +#define G_CPU_INDEX(x) (((x) >> S_CPU_INDEX) & M_CPU_INDEX) + +#define S_CPU_INDEX_VALID 7 +#define V_CPU_INDEX_VALID(x) ((x) << S_CPU_INDEX_VALID) +#define F_CPU_INDEX_VALID V_CPU_INDEX_VALID(1U) + +#define S_RX_COALESCE 8 +#define M_RX_COALESCE 0x3 +#define V_RX_COALESCE(x) ((x) << S_RX_COALESCE) +#define G_RX_COALESCE(x) (((x) >> S_RX_COALESCE) & M_RX_COALESCE) + +#define S_RX_COALESCE_VALID 10 +#define V_RX_COALESCE_VALID(x) ((x) << S_RX_COALESCE_VALID) +#define F_RX_COALESCE_VALID V_RX_COALESCE_VALID(1U) + +#define S_CONG_CONTROL_FLAVOR 11 +#define M_CONG_CONTROL_FLAVOR 0x3 +#define V_CONG_CONTROL_FLAVOR(x) ((x) << S_CONG_CONTROL_FLAVOR) +#define G_CONG_CONTROL_FLAVOR(x) (((x) >> S_CONG_CONTROL_FLAVOR) & M_CONG_CONTROL_FLAVOR) + +#define S_PACING_FLAVOR 13 +#define M_PACING_FLAVOR 0x3 +#define V_PACING_FLAVOR(x) ((x) << S_PACING_FLAVOR) +#define G_PACING_FLAVOR(x) (((x) >> S_PACING_FLAVOR) & M_PACING_FLAVOR) + +#define S_FLAVORS_VALID 15 +#define V_FLAVORS_VALID(x) ((x) << S_FLAVORS_VALID) +#define F_FLAVORS_VALID V_FLAVORS_VALID(1U) + +#define S_RX_FC_DISABLE 16 +#define V_RX_FC_DISABLE(x) ((x) << S_RX_FC_DISABLE) +#define F_RX_FC_DISABLE V_RX_FC_DISABLE(1U) + +#define S_RX_FC_VALID 17 +#define V_RX_FC_VALID(x) ((x) << S_RX_FC_VALID) +#define F_RX_FC_VALID V_RX_FC_VALID(1U) + +struct cpl_pass_open_req { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be32 opt0h; + __be32 opt0l; + __be32 peer_netmask; + __be32 opt1; +}; + +struct cpl_pass_open_rpl { + RSS_HDR union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __u8 resvd[7]; + __u8 status; +}; + +struct cpl_pass_establish { + RSS_HDR union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be32 tos_tid; + __be16 l2t_idx; + __be16 tcp_opt; + __be32 snd_isn; + __be32 rcv_isn; +}; + +/* cpl_pass_establish.tos_tid fields */ +#define S_PASS_OPEN_TID 0 +#define M_PASS_OPEN_TID 0xFFFFFF +#define V_PASS_OPEN_TID(x) ((x) << S_PASS_OPEN_TID) +#define G_PASS_OPEN_TID(x) (((x) >> S_PASS_OPEN_TID) & M_PASS_OPEN_TID) + +#define S_PASS_OPEN_TOS 24 +#define M_PASS_OPEN_TOS 0xFF +#define V_PASS_OPEN_TOS(x) ((x) << S_PASS_OPEN_TOS) +#define G_PASS_OPEN_TOS(x) (((x) >> S_PASS_OPEN_TOS) & M_PASS_OPEN_TOS) + +/* cpl_pass_establish.l2t_idx fields */ +#define S_L2T_IDX16 5 +#define M_L2T_IDX16 0x7FF +#define V_L2T_IDX16(x) ((x) << S_L2T_IDX16) +#define G_L2T_IDX16(x) (((x) >> S_L2T_IDX16) & M_L2T_IDX16) + +/* cpl_pass_establish.tcp_opt fields (also applies act_open_establish) */ +#define G_TCPOPT_WSCALE_OK(x) (((x) >> 5) & 1) +#define G_TCPOPT_SACK(x) (((x) >> 6) & 1) +#define G_TCPOPT_TSTAMP(x) (((x) >> 7) & 1) +#define G_TCPOPT_SND_WSCALE(x) (((x) >> 8) & 0xf) +#define G_TCPOPT_MSS(x) (((x) >> 12) & 0xf) + +struct cpl_pass_accept_req { + RSS_HDR union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be32 tos_tid; + struct tcp_options tcp_options; + __u8 dst_mac[6]; + __be16 vlan_tag; + __u8 src_mac[6]; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8:3; + __u8 addr_idx:3; + __u8 port_idx:1; + __u8 exact_match:1; +#else + __u8 exact_match:1; + __u8 port_idx:1; + __u8 addr_idx:3; + __u8:3; +#endif + __u8 rsvd; + __be32 rcv_isn; + __be32 rsvd2; +}; + +struct cpl_pass_accept_rpl { + WR_HDR; + union opcode_tid ot; + __be32 opt2; + __be32 rsvd; + __be32 peer_ip; + __be32 opt0h; + __be32 opt0l_status; +}; + +struct cpl_act_open_req { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be32 opt0h; + __be32 opt0l; + __be32 params; + __be32 opt2; +}; + +/* cpl_act_open_req.params fields */ +#define S_AOPEN_VLAN_PRI 9 +#define M_AOPEN_VLAN_PRI 0x3 +#define V_AOPEN_VLAN_PRI(x) ((x) << S_AOPEN_VLAN_PRI) +#define G_AOPEN_VLAN_PRI(x) (((x) >> S_AOPEN_VLAN_PRI) & M_AOPEN_VLAN_PRI) + +#define S_AOPEN_VLAN_PRI_VALID 11 +#define V_AOPEN_VLAN_PRI_VALID(x) ((x) << S_AOPEN_VLAN_PRI_VALID) +#define F_AOPEN_VLAN_PRI_VALID V_AOPEN_VLAN_PRI_VALID(1U) + +#define S_AOPEN_PKT_TYPE 12 +#define M_AOPEN_PKT_TYPE 0x3 +#define V_AOPEN_PKT_TYPE(x) ((x) << S_AOPEN_PKT_TYPE) +#define G_AOPEN_PKT_TYPE(x) (((x) >> S_AOPEN_PKT_TYPE) & M_AOPEN_PKT_TYPE) + +#define S_AOPEN_MAC_MATCH 14 +#define M_AOPEN_MAC_MATCH 0x1F +#define V_AOPEN_MAC_MATCH(x) ((x) << S_AOPEN_MAC_MATCH) +#define G_AOPEN_MAC_MATCH(x) (((x) >> S_AOPEN_MAC_MATCH) & M_AOPEN_MAC_MATCH) + +#define S_AOPEN_MAC_MATCH_VALID 19 +#define V_AOPEN_MAC_MATCH_VALID(x) ((x) << S_AOPEN_MAC_MATCH_VALID) +#define F_AOPEN_MAC_MATCH_VALID V_AOPEN_MAC_MATCH_VALID(1U) + +#define S_AOPEN_IFF_VLAN 20 +#define M_AOPEN_IFF_VLAN 0xFFF +#define V_AOPEN_IFF_VLAN(x) ((x) << S_AOPEN_IFF_VLAN) +#define G_AOPEN_IFF_VLAN(x) (((x) >> S_AOPEN_IFF_VLAN) & M_AOPEN_IFF_VLAN) + +struct cpl_act_open_rpl { + RSS_HDR union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be32 atid; + __u8 rsvd[3]; + __u8 status; +}; + +struct cpl_act_establish { + RSS_HDR union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be32 tos_tid; + __be16 l2t_idx; + __be16 tcp_opt; + __be32 snd_isn; + __be32 rcv_isn; +}; + +struct cpl_get_tcb { + WR_HDR; + union opcode_tid ot; + __be16 cpuno; + __be16 rsvd; +}; + +struct cpl_get_tcb_rpl { + RSS_HDR union opcode_tid ot; + __u8 rsvd; + __u8 status; + __be16 len; +}; + +struct cpl_set_tcb { + WR_HDR; + union opcode_tid ot; + __u8 reply; + __u8 cpu_idx; + __be16 len; +}; + +/* cpl_set_tcb.reply fields */ +#define S_NO_REPLY 7 +#define V_NO_REPLY(x) ((x) << S_NO_REPLY) +#define F_NO_REPLY V_NO_REPLY(1U) + +struct cpl_set_tcb_field { + WR_HDR; + union opcode_tid ot; + __u8 reply; + __u8 cpu_idx; + __be16 word; + __be64 mask; + __be64 val; +}; + +struct cpl_set_tcb_rpl { + RSS_HDR union opcode_tid ot; + __u8 rsvd[3]; + __u8 status; +}; + +struct cpl_pcmd { + WR_HDR; + union opcode_tid ot; + __u8 rsvd[3]; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 src:1; + __u8 bundle:1; + __u8 channel:1; + __u8:5; +#else + __u8:5; + __u8 channel:1; + __u8 bundle:1; + __u8 src:1; +#endif + __be32 pcmd_parm[2]; +}; + +struct cpl_pcmd_reply { + RSS_HDR union opcode_tid ot; + __u8 status; + __u8 rsvd; + __be16 len; +}; + +struct cpl_close_con_req { + WR_HDR; + union opcode_tid ot; + __be32 rsvd; +}; + +struct cpl_close_con_rpl { + RSS_HDR union opcode_tid ot; + __u8 rsvd[3]; + __u8 status; + __be32 snd_nxt; + __be32 rcv_nxt; +}; + +struct cpl_close_listserv_req { + WR_HDR; + union opcode_tid ot; + __u8 rsvd0; + __u8 cpu_idx; + __be16 rsvd1; +}; + +struct cpl_close_listserv_rpl { + RSS_HDR union opcode_tid ot; + __u8 rsvd[3]; + __u8 status; +}; + +struct cpl_abort_req_rss { + RSS_HDR union opcode_tid ot; + __be32 rsvd0; + __u8 rsvd1; + __u8 status; + __u8 rsvd2[6]; +}; + +struct cpl_abort_req { + WR_HDR; + union opcode_tid ot; + __be32 rsvd0; + __u8 rsvd1; + __u8 cmd; + __u8 rsvd2[6]; +}; + +struct cpl_abort_rpl_rss { + RSS_HDR union opcode_tid ot; + __be32 rsvd0; + __u8 rsvd1; + __u8 status; + __u8 rsvd2[6]; +}; + +struct cpl_abort_rpl { + WR_HDR; + union opcode_tid ot; + __be32 rsvd0; + __u8 rsvd1; + __u8 cmd; + __u8 rsvd2[6]; +}; + +struct cpl_peer_close { + RSS_HDR union opcode_tid ot; + __be32 rcv_nxt; +}; + +struct tx_data_wr { + __be32 wr_hi; + __be32 wr_lo; + __be32 len; + __be32 flags; + __be32 sndseq; + __be32 param; +}; + +/* tx_data_wr.flags fields */ +#define S_TX_ACK_PAGES 21 +#define M_TX_ACK_PAGES 0x7 +#define V_TX_ACK_PAGES(x) ((x) << S_TX_ACK_PAGES) +#define G_TX_ACK_PAGES(x) (((x) >> S_TX_ACK_PAGES) & M_TX_ACK_PAGES) + +/* tx_data_wr.param fields */ +#define S_TX_PORT 0 +#define M_TX_PORT 0x7 +#define V_TX_PORT(x) ((x) << S_TX_PORT) +#define G_TX_PORT(x) (((x) >> S_TX_PORT) & M_TX_PORT) + +#define S_TX_MSS 4 +#define M_TX_MSS 0xF +#define V_TX_MSS(x) ((x) << S_TX_MSS) +#define G_TX_MSS(x) (((x) >> S_TX_MSS) & M_TX_MSS) + +#define S_TX_QOS 8 +#define M_TX_QOS 0xFF +#define V_TX_QOS(x) ((x) << S_TX_QOS) +#define G_TX_QOS(x) (((x) >> S_TX_QOS) & M_TX_QOS) + +#define S_TX_SNDBUF 16 +#define M_TX_SNDBUF 0xFFFF +#define V_TX_SNDBUF(x) ((x) << S_TX_SNDBUF) +#define G_TX_SNDBUF(x) (((x) >> S_TX_SNDBUF) & M_TX_SNDBUF) + +struct cpl_tx_data { + union opcode_tid ot; + __be32 len; + __be32 rsvd; + __be16 urg; + __be16 flags; +}; + +/* cpl_tx_data.flags fields */ +#define S_TX_ULP_SUBMODE 6 +#define M_TX_ULP_SUBMODE 0xF +#define V_TX_ULP_SUBMODE(x) ((x) << S_TX_ULP_SUBMODE) +#define G_TX_ULP_SUBMODE(x) (((x) >> S_TX_ULP_SUBMODE) & M_TX_ULP_SUBMODE) + +#define S_TX_ULP_MODE 10 +#define M_TX_ULP_MODE 0xF +#define V_TX_ULP_MODE(x) ((x) << S_TX_ULP_MODE) +#define G_TX_ULP_MODE(x) (((x) >> S_TX_ULP_MODE) & M_TX_ULP_MODE) + +#define S_TX_SHOVE 14 +#define V_TX_SHOVE(x) ((x) << S_TX_SHOVE) +#define F_TX_SHOVE V_TX_SHOVE(1U) + +#define S_TX_MORE 15 +#define V_TX_MORE(x) ((x) << S_TX_MORE) +#define F_TX_MORE V_TX_MORE(1U) + +/* additional tx_data_wr.flags fields */ +#define S_TX_CPU_IDX 0 +#define M_TX_CPU_IDX 0x3F +#define V_TX_CPU_IDX(x) ((x) << S_TX_CPU_IDX) +#define G_TX_CPU_IDX(x) (((x) >> S_TX_CPU_IDX) & M_TX_CPU_IDX) + +#define S_TX_URG 16 +#define V_TX_URG(x) ((x) << S_TX_URG) +#define F_TX_URG V_TX_URG(1U) + +#define S_TX_CLOSE 17 +#define V_TX_CLOSE(x) ((x) << S_TX_CLOSE) +#define F_TX_CLOSE V_TX_CLOSE(1U) + +#define S_TX_INIT 18 +#define V_TX_INIT(x) ((x) << S_TX_INIT) +#define F_TX_INIT V_TX_INIT(1U) + +#define S_TX_IMM_ACK 19 +#define V_TX_IMM_ACK(x) ((x) << S_TX_IMM_ACK) +#define F_TX_IMM_ACK V_TX_IMM_ACK(1U) + +#define S_TX_IMM_DMA 20 +#define V_TX_IMM_DMA(x) ((x) << S_TX_IMM_DMA) +#define F_TX_IMM_DMA V_TX_IMM_DMA(1U) + +struct cpl_tx_data_ack { + RSS_HDR union opcode_tid ot; + __be32 ack_seq; +}; + +struct cpl_wr_ack { + RSS_HDR union opcode_tid ot; + __be16 credits; + __be16 rsvd; + __be32 snd_nxt; + __be32 snd_una; +}; + +struct cpl_rdma_ec_status { + RSS_HDR union opcode_tid ot; + __u8 rsvd[3]; + __u8 status; +}; + +struct mngt_pktsched_wr { + __be32 wr_hi; + __be32 wr_lo; + __u8 mngt_opcode; + __u8 rsvd[7]; + __u8 sched; + __u8 idx; + __u8 min; + __u8 max; + __u8 binding; + __u8 rsvd1[3]; +}; + +struct cpl_iscsi_hdr { + RSS_HDR union opcode_tid ot; + __be16 pdu_len_ddp; + __be16 len; + __be32 seq; + __be16 urg; + __u8 rsvd; + __u8 status; +}; + +/* cpl_iscsi_hdr.pdu_len_ddp fields */ +#define S_ISCSI_PDU_LEN 0 +#define M_ISCSI_PDU_LEN 0x7FFF +#define V_ISCSI_PDU_LEN(x) ((x) << S_ISCSI_PDU_LEN) +#define G_ISCSI_PDU_LEN(x) (((x) >> S_ISCSI_PDU_LEN) & M_ISCSI_PDU_LEN) + +#define S_ISCSI_DDP 15 +#define V_ISCSI_DDP(x) ((x) << S_ISCSI_DDP) +#define F_ISCSI_DDP V_ISCSI_DDP(1U) + +struct cpl_rx_data { + RSS_HDR union opcode_tid ot; + __be16 rsvd; + __be16 len; + __be32 seq; + __be16 urg; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 dack_mode:2; + __u8 psh:1; + __u8 heartbeat:1; + __u8:4; +#else + __u8:4; + __u8 heartbeat:1; + __u8 psh:1; + __u8 dack_mode:2; +#endif + __u8 status; +}; + +struct cpl_rx_data_ack { + WR_HDR; + union opcode_tid ot; + __be32 credit_dack; +}; + +/* cpl_rx_data_ack.ack_seq fields */ +#define S_RX_CREDITS 0 +#define M_RX_CREDITS 0x7FFFFFF +#define V_RX_CREDITS(x) ((x) << S_RX_CREDITS) +#define G_RX_CREDITS(x) (((x) >> S_RX_CREDITS) & M_RX_CREDITS) + +#define S_RX_MODULATE 27 +#define V_RX_MODULATE(x) ((x) << S_RX_MODULATE) +#define F_RX_MODULATE V_RX_MODULATE(1U) + +#define S_RX_FORCE_ACK 28 +#define V_RX_FORCE_ACK(x) ((x) << S_RX_FORCE_ACK) +#define F_RX_FORCE_ACK V_RX_FORCE_ACK(1U) + +#define S_RX_DACK_MODE 29 +#define M_RX_DACK_MODE 0x3 +#define V_RX_DACK_MODE(x) ((x) << S_RX_DACK_MODE) +#define G_RX_DACK_MODE(x) (((x) >> S_RX_DACK_MODE) & M_RX_DACK_MODE) + +#define S_RX_DACK_CHANGE 31 +#define V_RX_DACK_CHANGE(x) ((x) << S_RX_DACK_CHANGE) +#define F_RX_DACK_CHANGE V_RX_DACK_CHANGE(1U) + +struct cpl_rx_urg_notify { + RSS_HDR union opcode_tid ot; + __be32 seq; +}; + +struct cpl_rx_ddp_complete { + RSS_HDR union opcode_tid ot; + __be32 ddp_report; +}; + +struct cpl_rx_data_ddp { + RSS_HDR union opcode_tid ot; + __be16 urg; + __be16 len; + __be32 seq; + union { + __be32 nxt_seq; + __be32 ddp_report; + }; + __be32 ulp_crc; + __be32 ddpvld_status; +}; + +/* cpl_rx_data_ddp.ddpvld_status fields */ +#define S_DDP_STATUS 0 +#define M_DDP_STATUS 0xFF +#define V_DDP_STATUS(x) ((x) << S_DDP_STATUS) +#define G_DDP_STATUS(x) (((x) >> S_DDP_STATUS) & M_DDP_STATUS) + +#define S_DDP_VALID 15 +#define M_DDP_VALID 0x1FFFF +#define V_DDP_VALID(x) ((x) << S_DDP_VALID) +#define G_DDP_VALID(x) (((x) >> S_DDP_VALID) & M_DDP_VALID) + +#define S_DDP_PPOD_MISMATCH 15 +#define V_DDP_PPOD_MISMATCH(x) ((x) << S_DDP_PPOD_MISMATCH) +#define F_DDP_PPOD_MISMATCH V_DDP_PPOD_MISMATCH(1U) + +#define S_DDP_PDU 16 +#define V_DDP_PDU(x) ((x) << S_DDP_PDU) +#define F_DDP_PDU V_DDP_PDU(1U) + +#define S_DDP_LLIMIT_ERR 17 +#define V_DDP_LLIMIT_ERR(x) ((x) << S_DDP_LLIMIT_ERR) +#define F_DDP_LLIMIT_ERR V_DDP_LLIMIT_ERR(1U) + +#define S_DDP_PPOD_PARITY_ERR 18 +#define V_DDP_PPOD_PARITY_ERR(x) ((x) << S_DDP_PPOD_PARITY_ERR) +#define F_DDP_PPOD_PARITY_ERR V_DDP_PPOD_PARITY_ERR(1U) + +#define S_DDP_PADDING_ERR 19 +#define V_DDP_PADDING_ERR(x) ((x) << S_DDP_PADDING_ERR) +#define F_DDP_PADDING_ERR V_DDP_PADDING_ERR(1U) + +#define S_DDP_HDRCRC_ERR 20 +#define V_DDP_HDRCRC_ERR(x) ((x) << S_DDP_HDRCRC_ERR) +#define F_DDP_HDRCRC_ERR V_DDP_HDRCRC_ERR(1U) + +#define S_DDP_DATACRC_ERR 21 +#define V_DDP_DATACRC_ERR(x) ((x) << S_DDP_DATACRC_ERR) +#define F_DDP_DATACRC_ERR V_DDP_DATACRC_ERR(1U) + +#define S_DDP_INVALID_TAG 22 +#define V_DDP_INVALID_TAG(x) ((x) << S_DDP_INVALID_TAG) +#define F_DDP_INVALID_TAG V_DDP_INVALID_TAG(1U) + +#define S_DDP_ULIMIT_ERR 23 +#define V_DDP_ULIMIT_ERR(x) ((x) << S_DDP_ULIMIT_ERR) +#define F_DDP_ULIMIT_ERR V_DDP_ULIMIT_ERR(1U) + +#define S_DDP_OFFSET_ERR 24 +#define V_DDP_OFFSET_ERR(x) ((x) << S_DDP_OFFSET_ERR) +#define F_DDP_OFFSET_ERR V_DDP_OFFSET_ERR(1U) + +#define S_DDP_COLOR_ERR 25 +#define V_DDP_COLOR_ERR(x) ((x) << S_DDP_COLOR_ERR) +#define F_DDP_COLOR_ERR V_DDP_COLOR_ERR(1U) + +#define S_DDP_TID_MISMATCH 26 +#define V_DDP_TID_MISMATCH(x) ((x) << S_DDP_TID_MISMATCH) +#define F_DDP_TID_MISMATCH V_DDP_TID_MISMATCH(1U) + +#define S_DDP_INVALID_PPOD 27 +#define V_DDP_INVALID_PPOD(x) ((x) << S_DDP_INVALID_PPOD) +#define F_DDP_INVALID_PPOD V_DDP_INVALID_PPOD(1U) + +#define S_DDP_ULP_MODE 28 +#define M_DDP_ULP_MODE 0xF +#define V_DDP_ULP_MODE(x) ((x) << S_DDP_ULP_MODE) +#define G_DDP_ULP_MODE(x) (((x) >> S_DDP_ULP_MODE) & M_DDP_ULP_MODE) + +/* cpl_rx_data_ddp.ddp_report fields */ +#define S_DDP_OFFSET 0 +#define M_DDP_OFFSET 0x3FFFFF +#define V_DDP_OFFSET(x) ((x) << S_DDP_OFFSET) +#define G_DDP_OFFSET(x) (((x) >> S_DDP_OFFSET) & M_DDP_OFFSET) + +#define S_DDP_URG 24 +#define V_DDP_URG(x) ((x) << S_DDP_URG) +#define F_DDP_URG V_DDP_URG(1U) + +#define S_DDP_PSH 25 +#define V_DDP_PSH(x) ((x) << S_DDP_PSH) +#define F_DDP_PSH V_DDP_PSH(1U) + +#define S_DDP_BUF_COMPLETE 26 +#define V_DDP_BUF_COMPLETE(x) ((x) << S_DDP_BUF_COMPLETE) +#define F_DDP_BUF_COMPLETE V_DDP_BUF_COMPLETE(1U) + +#define S_DDP_BUF_TIMED_OUT 27 +#define V_DDP_BUF_TIMED_OUT(x) ((x) << S_DDP_BUF_TIMED_OUT) +#define F_DDP_BUF_TIMED_OUT V_DDP_BUF_TIMED_OUT(1U) + +#define S_DDP_BUF_IDX 28 +#define V_DDP_BUF_IDX(x) ((x) << S_DDP_BUF_IDX) +#define F_DDP_BUF_IDX V_DDP_BUF_IDX(1U) + +struct cpl_tx_pkt { + WR_HDR; + __be32 cntrl; + __be32 len; +}; + +struct cpl_tx_pkt_lso { + WR_HDR; + __be32 cntrl; + __be32 len; + + __be32 rsvd; + __be32 lso_info; +}; + +/* cpl_tx_pkt*.cntrl fields */ +#define S_TXPKT_VLAN 0 +#define M_TXPKT_VLAN 0xFFFF +#define V_TXPKT_VLAN(x) ((x) << S_TXPKT_VLAN) +#define G_TXPKT_VLAN(x) (((x) >> S_TXPKT_VLAN) & M_TXPKT_VLAN) + +#define S_TXPKT_INTF 16 +#define M_TXPKT_INTF 0xF +#define V_TXPKT_INTF(x) ((x) << S_TXPKT_INTF) +#define G_TXPKT_INTF(x) (((x) >> S_TXPKT_INTF) & M_TXPKT_INTF) + +#define S_TXPKT_IPCSUM_DIS 20 +#define V_TXPKT_IPCSUM_DIS(x) ((x) << S_TXPKT_IPCSUM_DIS) +#define F_TXPKT_IPCSUM_DIS V_TXPKT_IPCSUM_DIS(1U) + +#define S_TXPKT_L4CSUM_DIS 21 +#define V_TXPKT_L4CSUM_DIS(x) ((x) << S_TXPKT_L4CSUM_DIS) +#define F_TXPKT_L4CSUM_DIS V_TXPKT_L4CSUM_DIS(1U) + +#define S_TXPKT_VLAN_VLD 22 +#define V_TXPKT_VLAN_VLD(x) ((x) << S_TXPKT_VLAN_VLD) +#define F_TXPKT_VLAN_VLD V_TXPKT_VLAN_VLD(1U) + +#define S_TXPKT_LOOPBACK 23 +#define V_TXPKT_LOOPBACK(x) ((x) << S_TXPKT_LOOPBACK) +#define F_TXPKT_LOOPBACK V_TXPKT_LOOPBACK(1U) + +#define S_TXPKT_OPCODE 24 +#define M_TXPKT_OPCODE 0xFF +#define V_TXPKT_OPCODE(x) ((x) << S_TXPKT_OPCODE) +#define G_TXPKT_OPCODE(x) (((x) >> S_TXPKT_OPCODE) & M_TXPKT_OPCODE) + +/* cpl_tx_pkt_lso.lso_info fields */ +#define S_LSO_MSS 0 +#define M_LSO_MSS 0x3FFF +#define V_LSO_MSS(x) ((x) << S_LSO_MSS) +#define G_LSO_MSS(x) (((x) >> S_LSO_MSS) & M_LSO_MSS) + +#define S_LSO_ETH_TYPE 14 +#define M_LSO_ETH_TYPE 0x3 +#define V_LSO_ETH_TYPE(x) ((x) << S_LSO_ETH_TYPE) +#define G_LSO_ETH_TYPE(x) (((x) >> S_LSO_ETH_TYPE) & M_LSO_ETH_TYPE) + +#define S_LSO_TCPHDR_WORDS 16 +#define M_LSO_TCPHDR_WORDS 0xF +#define V_LSO_TCPHDR_WORDS(x) ((x) << S_LSO_TCPHDR_WORDS) +#define G_LSO_TCPHDR_WORDS(x) (((x) >> S_LSO_TCPHDR_WORDS) & M_LSO_TCPHDR_WORDS) + +#define S_LSO_IPHDR_WORDS 20 +#define M_LSO_IPHDR_WORDS 0xF +#define V_LSO_IPHDR_WORDS(x) ((x) << S_LSO_IPHDR_WORDS) +#define G_LSO_IPHDR_WORDS(x) (((x) >> S_LSO_IPHDR_WORDS) & M_LSO_IPHDR_WORDS) + +#define S_LSO_IPV6 24 +#define V_LSO_IPV6(x) ((x) << S_LSO_IPV6) +#define F_LSO_IPV6 V_LSO_IPV6(1U) + +struct cpl_trace_pkt { +#ifdef CHELSIO_FW + __u8 rss_opcode; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 err:1; + __u8:7; +#else + __u8:7; + __u8 err:1; +#endif + __u8 rsvd0; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 qid:4; + __u8:4; +#else + __u8:4; + __u8 qid:4; +#endif + __be32 tstamp; +#endif /* CHELSIO_FW */ + + __u8 opcode; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 iff:4; + __u8:4; +#else + __u8:4; + __u8 iff:4; +#endif + __u8 rsvd[4]; + __be16 len; +}; + +struct cpl_rx_pkt { + RSS_HDR __u8 opcode; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 iff:4; + __u8 csum_valid:1; + __u8 ipmi_pkt:1; + __u8 vlan_valid:1; + __u8 fragment:1; +#else + __u8 fragment:1; + __u8 vlan_valid:1; + __u8 ipmi_pkt:1; + __u8 csum_valid:1; + __u8 iff:4; +#endif + __be16 csum; + __be16 vlan; + __be16 len; +}; + +struct cpl_l2t_write_req { + WR_HDR; + union opcode_tid ot; + __be32 params; + __u8 rsvd[2]; + __u8 dst_mac[6]; +}; + +/* cpl_l2t_write_req.params fields */ +#define S_L2T_W_IDX 0 +#define M_L2T_W_IDX 0x7FF +#define V_L2T_W_IDX(x) ((x) << S_L2T_W_IDX) +#define G_L2T_W_IDX(x) (((x) >> S_L2T_W_IDX) & M_L2T_W_IDX) + +#define S_L2T_W_VLAN 11 +#define M_L2T_W_VLAN 0xFFF +#define V_L2T_W_VLAN(x) ((x) << S_L2T_W_VLAN) +#define G_L2T_W_VLAN(x) (((x) >> S_L2T_W_VLAN) & M_L2T_W_VLAN) + +#define S_L2T_W_IFF 23 +#define M_L2T_W_IFF 0xF +#define V_L2T_W_IFF(x) ((x) << S_L2T_W_IFF) +#define G_L2T_W_IFF(x) (((x) >> S_L2T_W_IFF) & M_L2T_W_IFF) + +#define S_L2T_W_PRIO 27 +#define M_L2T_W_PRIO 0x7 +#define V_L2T_W_PRIO(x) ((x) << S_L2T_W_PRIO) +#define G_L2T_W_PRIO(x) (((x) >> S_L2T_W_PRIO) & M_L2T_W_PRIO) + +struct cpl_l2t_write_rpl { + RSS_HDR union opcode_tid ot; + __u8 status; + __u8 rsvd[3]; +}; + +struct cpl_l2t_read_req { + WR_HDR; + union opcode_tid ot; + __be16 rsvd; + __be16 l2t_idx; +}; + +struct cpl_l2t_read_rpl { + RSS_HDR union opcode_tid ot; + __be32 params; + __u8 rsvd[2]; + __u8 dst_mac[6]; +}; + +/* cpl_l2t_read_rpl.params fields */ +#define S_L2T_R_PRIO 0 +#define M_L2T_R_PRIO 0x7 +#define V_L2T_R_PRIO(x) ((x) << S_L2T_R_PRIO) +#define G_L2T_R_PRIO(x) (((x) >> S_L2T_R_PRIO) & M_L2T_R_PRIO) + +#define S_L2T_R_VLAN 8 +#define M_L2T_R_VLAN 0xFFF +#define V_L2T_R_VLAN(x) ((x) << S_L2T_R_VLAN) +#define G_L2T_R_VLAN(x) (((x) >> S_L2T_R_VLAN) & M_L2T_R_VLAN) + +#define S_L2T_R_IFF 20 +#define M_L2T_R_IFF 0xF +#define V_L2T_R_IFF(x) ((x) << S_L2T_R_IFF) +#define G_L2T_R_IFF(x) (((x) >> S_L2T_R_IFF) & M_L2T_R_IFF) + +#define S_L2T_STATUS 24 +#define M_L2T_STATUS 0xFF +#define V_L2T_STATUS(x) ((x) << S_L2T_STATUS) +#define G_L2T_STATUS(x) (((x) >> S_L2T_STATUS) & M_L2T_STATUS) + +struct cpl_smt_write_req { + WR_HDR; + union opcode_tid ot; + __u8 rsvd0; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 mtu_idx:4; + __u8 iff:4; +#else + __u8 iff:4; + __u8 mtu_idx:4; +#endif + __be16 rsvd2; + __be16 rsvd3; + __u8 src_mac1[6]; + __be16 rsvd4; + __u8 src_mac0[6]; +}; + +struct cpl_smt_write_rpl { + RSS_HDR union opcode_tid ot; + __u8 status; + __u8 rsvd[3]; +}; + +struct cpl_smt_read_req { + WR_HDR; + union opcode_tid ot; + __u8 rsvd0; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8:4; + __u8 iff:4; +#else + __u8 iff:4; + __u8:4; +#endif + __be16 rsvd2; +}; + +struct cpl_smt_read_rpl { + RSS_HDR union opcode_tid ot; + __u8 status; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 mtu_idx:4; + __u8:4; +#else + __u8:4; + __u8 mtu_idx:4; +#endif + __be16 rsvd2; + __be16 rsvd3; + __u8 src_mac1[6]; + __be16 rsvd4; + __u8 src_mac0[6]; +}; + +struct cpl_rte_delete_req { + WR_HDR; + union opcode_tid ot; + __be32 params; +}; + +/* { cpl_rte_delete_req, cpl_rte_read_req }.params fields */ +#define S_RTE_REQ_LUT_IX 8 +#define M_RTE_REQ_LUT_IX 0x7FF +#define V_RTE_REQ_LUT_IX(x) ((x) << S_RTE_REQ_LUT_IX) +#define G_RTE_REQ_LUT_IX(x) (((x) >> S_RTE_REQ_LUT_IX) & M_RTE_REQ_LUT_IX) + +#define S_RTE_REQ_LUT_BASE 19 +#define M_RTE_REQ_LUT_BASE 0x7FF +#define V_RTE_REQ_LUT_BASE(x) ((x) << S_RTE_REQ_LUT_BASE) +#define G_RTE_REQ_LUT_BASE(x) (((x) >> S_RTE_REQ_LUT_BASE) & M_RTE_REQ_LUT_BASE) + +#define S_RTE_READ_REQ_SELECT 31 +#define V_RTE_READ_REQ_SELECT(x) ((x) << S_RTE_READ_REQ_SELECT) +#define F_RTE_READ_REQ_SELECT V_RTE_READ_REQ_SELECT(1U) + +struct cpl_rte_delete_rpl { + RSS_HDR union opcode_tid ot; + __u8 status; + __u8 rsvd[3]; +}; + +struct cpl_rte_write_req { + WR_HDR; + union opcode_tid ot; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8:6; + __u8 write_tcam:1; + __u8 write_l2t_lut:1; +#else + __u8 write_l2t_lut:1; + __u8 write_tcam:1; + __u8:6; +#endif + __u8 rsvd[3]; + __be32 lut_params; + __be16 rsvd2; + __be16 l2t_idx; + __be32 netmask; + __be32 faddr; +}; + +/* cpl_rte_write_req.lut_params fields */ +#define S_RTE_WRITE_REQ_LUT_IX 10 +#define M_RTE_WRITE_REQ_LUT_IX 0x7FF +#define V_RTE_WRITE_REQ_LUT_IX(x) ((x) << S_RTE_WRITE_REQ_LUT_IX) +#define G_RTE_WRITE_REQ_LUT_IX(x) (((x) >> S_RTE_WRITE_REQ_LUT_IX) & M_RTE_WRITE_REQ_LUT_IX) + +#define S_RTE_WRITE_REQ_LUT_BASE 21 +#define M_RTE_WRITE_REQ_LUT_BASE 0x7FF +#define V_RTE_WRITE_REQ_LUT_BASE(x) ((x) << S_RTE_WRITE_REQ_LUT_BASE) +#define G_RTE_WRITE_REQ_LUT_BASE(x) (((x) >> S_RTE_WRITE_REQ_LUT_BASE) & M_RTE_WRITE_REQ_LUT_BASE) + +struct cpl_rte_write_rpl { + RSS_HDR union opcode_tid ot; + __u8 status; + __u8 rsvd[3]; +}; + +struct cpl_rte_read_req { + WR_HDR; + union opcode_tid ot; + __be32 params; +}; + +struct cpl_rte_read_rpl { + RSS_HDR union opcode_tid ot; + __u8 status; + __u8 rsvd0; + __be16 l2t_idx; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8:7; + __u8 select:1; +#else + __u8 select:1; + __u8:7; +#endif + __u8 rsvd2[3]; + __be32 addr; +}; + +struct cpl_tid_release { + WR_HDR; + union opcode_tid ot; + __be32 rsvd; +}; + +struct cpl_barrier { + WR_HDR; + __u8 opcode; + __u8 rsvd[7]; +}; + +struct cpl_rdma_read_req { + __u8 opcode; + __u8 rsvd[15]; +}; + +struct cpl_rdma_terminate { +#ifdef CHELSIO_FW + __u8 opcode; + __u8 rsvd[2]; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 rspq:3; + __u8:5; +#else + __u8:5; + __u8 rspq:3; +#endif + __be32 tid_len; +#endif + __be32 msn; + __be32 mo; + __u8 data[]; +}; + +/* cpl_rdma_terminate.tid_len fields */ +#define S_FLIT_CNT 0 +#define M_FLIT_CNT 0xFF +#define V_FLIT_CNT(x) ((x) << S_FLIT_CNT) +#define G_FLIT_CNT(x) (((x) >> S_FLIT_CNT) & M_FLIT_CNT) + +#define S_TERM_TID 8 +#define M_TERM_TID 0xFFFFF +#define V_TERM_TID(x) ((x) << S_TERM_TID) +#define G_TERM_TID(x) (((x) >> S_TERM_TID) & M_TERM_TID) + +/* ULP_TX opcodes */ +enum { ULP_MEM_READ = 2, ULP_MEM_WRITE = 3, ULP_TXPKT = 4 }; + +#define S_ULPTX_CMD 28 +#define M_ULPTX_CMD 0xF +#define V_ULPTX_CMD(x) ((x) << S_ULPTX_CMD) + +#define S_ULPTX_NFLITS 0 +#define M_ULPTX_NFLITS 0xFF +#define V_ULPTX_NFLITS(x) ((x) << S_ULPTX_NFLITS) + +struct ulp_mem_io { + WR_HDR; + __be32 cmd_lock_addr; + __be32 len; +}; + +/* ulp_mem_io.cmd_lock_addr fields */ +#define S_ULP_MEMIO_ADDR 0 +#define M_ULP_MEMIO_ADDR 0x7FFFFFF +#define V_ULP_MEMIO_ADDR(x) ((x) << S_ULP_MEMIO_ADDR) +#define S_ULP_MEMIO_LOCK 27 +#define V_ULP_MEMIO_LOCK(x) ((x) << S_ULP_MEMIO_LOCK) +#define F_ULP_MEMIO_LOCK V_ULP_MEMIO_LOCK(1U) + +/* ulp_mem_io.len fields */ +#define S_ULP_MEMIO_DATA_LEN 28 +#define M_ULP_MEMIO_DATA_LEN 0xF +#define V_ULP_MEMIO_DATA_LEN(x) ((x) << S_ULP_MEMIO_DATA_LEN) + +#endif /* T3_CPL_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb3/t3_hw.c b/drivers/net/ethernet/chelsio/cxgb3/t3_hw.c new file mode 100644 index 000000000..a06003bfa --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/t3_hw.c @@ -0,0 +1,3748 @@ +/* + * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include <linux/etherdevice.h> +#include "common.h" +#include "regs.h" +#include "sge_defs.h" +#include "firmware_exports.h" + +static void t3_port_intr_clear(struct adapter *adapter, int idx); + +/** + * t3_wait_op_done_val - wait until an operation is completed + * @adapter: the adapter performing the operation + * @reg: the register to check for completion + * @mask: a single-bit field within @reg that indicates completion + * @polarity: the value of the field when the operation is completed + * @attempts: number of check iterations + * @delay: delay in usecs between iterations + * @valp: where to store the value of the register at completion time + * + * Wait until an operation is completed by checking a bit in a register + * up to @attempts times. If @valp is not NULL the value of the register + * at the time it indicated completion is stored there. Returns 0 if the + * operation completes and -EAGAIN otherwise. + */ + +int t3_wait_op_done_val(struct adapter *adapter, int reg, u32 mask, + int polarity, int attempts, int delay, u32 *valp) +{ + while (1) { + u32 val = t3_read_reg(adapter, reg); + + if (!!(val & mask) == polarity) { + if (valp) + *valp = val; + return 0; + } + if (--attempts == 0) + return -EAGAIN; + if (delay) + udelay(delay); + } +} + +/** + * t3_write_regs - write a bunch of registers + * @adapter: the adapter to program + * @p: an array of register address/register value pairs + * @n: the number of address/value pairs + * @offset: register address offset + * + * Takes an array of register address/register value pairs and writes each + * value to the corresponding register. Register addresses are adjusted + * by the supplied offset. + */ +void t3_write_regs(struct adapter *adapter, const struct addr_val_pair *p, + int n, unsigned int offset) +{ + while (n--) { + t3_write_reg(adapter, p->reg_addr + offset, p->val); + p++; + } +} + +/** + * t3_set_reg_field - set a register field to a value + * @adapter: the adapter to program + * @addr: the register address + * @mask: specifies the portion of the register to modify + * @val: the new value for the register field + * + * Sets a register field specified by the supplied mask to the + * given value. + */ +void t3_set_reg_field(struct adapter *adapter, unsigned int addr, u32 mask, + u32 val) +{ + u32 v = t3_read_reg(adapter, addr) & ~mask; + + t3_write_reg(adapter, addr, v | val); + t3_read_reg(adapter, addr); /* flush */ +} + +/** + * t3_read_indirect - read indirectly addressed registers + * @adap: the adapter + * @addr_reg: register holding the indirect address + * @data_reg: register holding the value of the indirect register + * @vals: where the read register values are stored + * @start_idx: index of first indirect register to read + * @nregs: how many indirect registers to read + * + * Reads registers that are accessed indirectly through an address/data + * register pair. + */ +static void t3_read_indirect(struct adapter *adap, unsigned int addr_reg, + unsigned int data_reg, u32 *vals, + unsigned int nregs, unsigned int start_idx) +{ + while (nregs--) { + t3_write_reg(adap, addr_reg, start_idx); + *vals++ = t3_read_reg(adap, data_reg); + start_idx++; + } +} + +/** + * t3_mc7_bd_read - read from MC7 through backdoor accesses + * @mc7: identifies MC7 to read from + * @start: index of first 64-bit word to read + * @n: number of 64-bit words to read + * @buf: where to store the read result + * + * Read n 64-bit words from MC7 starting at word start, using backdoor + * accesses. + */ +int t3_mc7_bd_read(struct mc7 *mc7, unsigned int start, unsigned int n, + u64 *buf) +{ + static const int shift[] = { 0, 0, 16, 24 }; + static const int step[] = { 0, 32, 16, 8 }; + + unsigned int size64 = mc7->size / 8; /* # of 64-bit words */ + struct adapter *adap = mc7->adapter; + + if (start >= size64 || start + n > size64) + return -EINVAL; + + start *= (8 << mc7->width); + while (n--) { + int i; + u64 val64 = 0; + + for (i = (1 << mc7->width) - 1; i >= 0; --i) { + int attempts = 10; + u32 val; + + t3_write_reg(adap, mc7->offset + A_MC7_BD_ADDR, start); + t3_write_reg(adap, mc7->offset + A_MC7_BD_OP, 0); + val = t3_read_reg(adap, mc7->offset + A_MC7_BD_OP); + while ((val & F_BUSY) && attempts--) + val = t3_read_reg(adap, + mc7->offset + A_MC7_BD_OP); + if (val & F_BUSY) + return -EIO; + + val = t3_read_reg(adap, mc7->offset + A_MC7_BD_DATA1); + if (mc7->width == 0) { + val64 = t3_read_reg(adap, + mc7->offset + + A_MC7_BD_DATA0); + val64 |= (u64) val << 32; + } else { + if (mc7->width > 1) + val >>= shift[mc7->width]; + val64 |= (u64) val << (step[mc7->width] * i); + } + start += 8; + } + *buf++ = val64; + } + return 0; +} + +/* + * Initialize MI1. + */ +static void mi1_init(struct adapter *adap, const struct adapter_info *ai) +{ + u32 clkdiv = adap->params.vpd.cclk / (2 * adap->params.vpd.mdc) - 1; + u32 val = F_PREEN | V_CLKDIV(clkdiv); + + t3_write_reg(adap, A_MI1_CFG, val); +} + +#define MDIO_ATTEMPTS 20 + +/* + * MI1 read/write operations for clause 22 PHYs. + */ +static int t3_mi1_read(struct net_device *dev, int phy_addr, int mmd_addr, + u16 reg_addr) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + int ret; + u32 addr = V_REGADDR(reg_addr) | V_PHYADDR(phy_addr); + + mutex_lock(&adapter->mdio_lock); + t3_set_reg_field(adapter, A_MI1_CFG, V_ST(M_ST), V_ST(1)); + t3_write_reg(adapter, A_MI1_ADDR, addr); + t3_write_reg(adapter, A_MI1_OP, V_MDI_OP(2)); + ret = t3_wait_op_done(adapter, A_MI1_OP, F_BUSY, 0, MDIO_ATTEMPTS, 10); + if (!ret) + ret = t3_read_reg(adapter, A_MI1_DATA); + mutex_unlock(&adapter->mdio_lock); + return ret; +} + +static int t3_mi1_write(struct net_device *dev, int phy_addr, int mmd_addr, + u16 reg_addr, u16 val) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + int ret; + u32 addr = V_REGADDR(reg_addr) | V_PHYADDR(phy_addr); + + mutex_lock(&adapter->mdio_lock); + t3_set_reg_field(adapter, A_MI1_CFG, V_ST(M_ST), V_ST(1)); + t3_write_reg(adapter, A_MI1_ADDR, addr); + t3_write_reg(adapter, A_MI1_DATA, val); + t3_write_reg(adapter, A_MI1_OP, V_MDI_OP(1)); + ret = t3_wait_op_done(adapter, A_MI1_OP, F_BUSY, 0, MDIO_ATTEMPTS, 10); + mutex_unlock(&adapter->mdio_lock); + return ret; +} + +static const struct mdio_ops mi1_mdio_ops = { + .read = t3_mi1_read, + .write = t3_mi1_write, + .mode_support = MDIO_SUPPORTS_C22 +}; + +/* + * Performs the address cycle for clause 45 PHYs. + * Must be called with the MDIO_LOCK held. + */ +static int mi1_wr_addr(struct adapter *adapter, int phy_addr, int mmd_addr, + int reg_addr) +{ + u32 addr = V_REGADDR(mmd_addr) | V_PHYADDR(phy_addr); + + t3_set_reg_field(adapter, A_MI1_CFG, V_ST(M_ST), 0); + t3_write_reg(adapter, A_MI1_ADDR, addr); + t3_write_reg(adapter, A_MI1_DATA, reg_addr); + t3_write_reg(adapter, A_MI1_OP, V_MDI_OP(0)); + return t3_wait_op_done(adapter, A_MI1_OP, F_BUSY, 0, + MDIO_ATTEMPTS, 10); +} + +/* + * MI1 read/write operations for indirect-addressed PHYs. + */ +static int mi1_ext_read(struct net_device *dev, int phy_addr, int mmd_addr, + u16 reg_addr) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + int ret; + + mutex_lock(&adapter->mdio_lock); + ret = mi1_wr_addr(adapter, phy_addr, mmd_addr, reg_addr); + if (!ret) { + t3_write_reg(adapter, A_MI1_OP, V_MDI_OP(3)); + ret = t3_wait_op_done(adapter, A_MI1_OP, F_BUSY, 0, + MDIO_ATTEMPTS, 10); + if (!ret) + ret = t3_read_reg(adapter, A_MI1_DATA); + } + mutex_unlock(&adapter->mdio_lock); + return ret; +} + +static int mi1_ext_write(struct net_device *dev, int phy_addr, int mmd_addr, + u16 reg_addr, u16 val) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + int ret; + + mutex_lock(&adapter->mdio_lock); + ret = mi1_wr_addr(adapter, phy_addr, mmd_addr, reg_addr); + if (!ret) { + t3_write_reg(adapter, A_MI1_DATA, val); + t3_write_reg(adapter, A_MI1_OP, V_MDI_OP(1)); + ret = t3_wait_op_done(adapter, A_MI1_OP, F_BUSY, 0, + MDIO_ATTEMPTS, 10); + } + mutex_unlock(&adapter->mdio_lock); + return ret; +} + +static const struct mdio_ops mi1_mdio_ext_ops = { + .read = mi1_ext_read, + .write = mi1_ext_write, + .mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22 +}; + +/** + * t3_mdio_change_bits - modify the value of a PHY register + * @phy: the PHY to operate on + * @mmd: the device address + * @reg: the register address + * @clear: what part of the register value to mask off + * @set: what part of the register value to set + * + * Changes the value of a PHY register by applying a mask to its current + * value and ORing the result with a new value. + */ +int t3_mdio_change_bits(struct cphy *phy, int mmd, int reg, unsigned int clear, + unsigned int set) +{ + int ret; + unsigned int val; + + ret = t3_mdio_read(phy, mmd, reg, &val); + if (!ret) { + val &= ~clear; + ret = t3_mdio_write(phy, mmd, reg, val | set); + } + return ret; +} + +/** + * t3_phy_reset - reset a PHY block + * @phy: the PHY to operate on + * @mmd: the device address of the PHY block to reset + * @wait: how long to wait for the reset to complete in 1ms increments + * + * Resets a PHY block and optionally waits for the reset to complete. + * @mmd should be 0 for 10/100/1000 PHYs and the device address to reset + * for 10G PHYs. + */ +int t3_phy_reset(struct cphy *phy, int mmd, int wait) +{ + int err; + unsigned int ctl; + + err = t3_mdio_change_bits(phy, mmd, MDIO_CTRL1, MDIO_CTRL1_LPOWER, + MDIO_CTRL1_RESET); + if (err || !wait) + return err; + + do { + err = t3_mdio_read(phy, mmd, MDIO_CTRL1, &ctl); + if (err) + return err; + ctl &= MDIO_CTRL1_RESET; + if (ctl) + msleep(1); + } while (ctl && --wait); + + return ctl ? -1 : 0; +} + +/** + * t3_phy_advertise - set the PHY advertisement registers for autoneg + * @phy: the PHY to operate on + * @advert: bitmap of capabilities the PHY should advertise + * + * Sets a 10/100/1000 PHY's advertisement registers to advertise the + * requested capabilities. + */ +int t3_phy_advertise(struct cphy *phy, unsigned int advert) +{ + int err; + unsigned int val = 0; + + err = t3_mdio_read(phy, MDIO_DEVAD_NONE, MII_CTRL1000, &val); + if (err) + return err; + + val &= ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL); + if (advert & ADVERTISED_1000baseT_Half) + val |= ADVERTISE_1000HALF; + if (advert & ADVERTISED_1000baseT_Full) + val |= ADVERTISE_1000FULL; + + err = t3_mdio_write(phy, MDIO_DEVAD_NONE, MII_CTRL1000, val); + if (err) + return err; + + val = 1; + if (advert & ADVERTISED_10baseT_Half) + val |= ADVERTISE_10HALF; + if (advert & ADVERTISED_10baseT_Full) + val |= ADVERTISE_10FULL; + if (advert & ADVERTISED_100baseT_Half) + val |= ADVERTISE_100HALF; + if (advert & ADVERTISED_100baseT_Full) + val |= ADVERTISE_100FULL; + if (advert & ADVERTISED_Pause) + val |= ADVERTISE_PAUSE_CAP; + if (advert & ADVERTISED_Asym_Pause) + val |= ADVERTISE_PAUSE_ASYM; + return t3_mdio_write(phy, MDIO_DEVAD_NONE, MII_ADVERTISE, val); +} + +/** + * t3_phy_advertise_fiber - set fiber PHY advertisement register + * @phy: the PHY to operate on + * @advert: bitmap of capabilities the PHY should advertise + * + * Sets a fiber PHY's advertisement register to advertise the + * requested capabilities. + */ +int t3_phy_advertise_fiber(struct cphy *phy, unsigned int advert) +{ + unsigned int val = 0; + + if (advert & ADVERTISED_1000baseT_Half) + val |= ADVERTISE_1000XHALF; + if (advert & ADVERTISED_1000baseT_Full) + val |= ADVERTISE_1000XFULL; + if (advert & ADVERTISED_Pause) + val |= ADVERTISE_1000XPAUSE; + if (advert & ADVERTISED_Asym_Pause) + val |= ADVERTISE_1000XPSE_ASYM; + return t3_mdio_write(phy, MDIO_DEVAD_NONE, MII_ADVERTISE, val); +} + +/** + * t3_set_phy_speed_duplex - force PHY speed and duplex + * @phy: the PHY to operate on + * @speed: requested PHY speed + * @duplex: requested PHY duplex + * + * Force a 10/100/1000 PHY's speed and duplex. This also disables + * auto-negotiation except for GigE, where auto-negotiation is mandatory. + */ +int t3_set_phy_speed_duplex(struct cphy *phy, int speed, int duplex) +{ + int err; + unsigned int ctl; + + err = t3_mdio_read(phy, MDIO_DEVAD_NONE, MII_BMCR, &ctl); + if (err) + return err; + + if (speed >= 0) { + ctl &= ~(BMCR_SPEED100 | BMCR_SPEED1000 | BMCR_ANENABLE); + if (speed == SPEED_100) + ctl |= BMCR_SPEED100; + else if (speed == SPEED_1000) + ctl |= BMCR_SPEED1000; + } + if (duplex >= 0) { + ctl &= ~(BMCR_FULLDPLX | BMCR_ANENABLE); + if (duplex == DUPLEX_FULL) + ctl |= BMCR_FULLDPLX; + } + if (ctl & BMCR_SPEED1000) /* auto-negotiation required for GigE */ + ctl |= BMCR_ANENABLE; + return t3_mdio_write(phy, MDIO_DEVAD_NONE, MII_BMCR, ctl); +} + +int t3_phy_lasi_intr_enable(struct cphy *phy) +{ + return t3_mdio_write(phy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_CTRL, + MDIO_PMA_LASI_LSALARM); +} + +int t3_phy_lasi_intr_disable(struct cphy *phy) +{ + return t3_mdio_write(phy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_CTRL, 0); +} + +int t3_phy_lasi_intr_clear(struct cphy *phy) +{ + u32 val; + + return t3_mdio_read(phy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_STAT, &val); +} + +int t3_phy_lasi_intr_handler(struct cphy *phy) +{ + unsigned int status; + int err = t3_mdio_read(phy, MDIO_MMD_PMAPMD, MDIO_PMA_LASI_STAT, + &status); + + if (err) + return err; + return (status & MDIO_PMA_LASI_LSALARM) ? cphy_cause_link_change : 0; +} + +static const struct adapter_info t3_adap_info[] = { + {1, 1, 0, + F_GPIO2_OEN | F_GPIO4_OEN | + F_GPIO2_OUT_VAL | F_GPIO4_OUT_VAL, { S_GPIO3, S_GPIO5 }, 0, + &mi1_mdio_ops, "Chelsio PE9000"}, + {1, 1, 0, + F_GPIO2_OEN | F_GPIO4_OEN | + F_GPIO2_OUT_VAL | F_GPIO4_OUT_VAL, { S_GPIO3, S_GPIO5 }, 0, + &mi1_mdio_ops, "Chelsio T302"}, + {1, 0, 0, + F_GPIO1_OEN | F_GPIO6_OEN | F_GPIO7_OEN | F_GPIO10_OEN | + F_GPIO11_OEN | F_GPIO1_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL, + { 0 }, SUPPORTED_10000baseT_Full | SUPPORTED_AUI, + &mi1_mdio_ext_ops, "Chelsio T310"}, + {1, 1, 0, + F_GPIO1_OEN | F_GPIO2_OEN | F_GPIO4_OEN | F_GPIO5_OEN | F_GPIO6_OEN | + F_GPIO7_OEN | F_GPIO10_OEN | F_GPIO11_OEN | F_GPIO1_OUT_VAL | + F_GPIO5_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL, + { S_GPIO9, S_GPIO3 }, SUPPORTED_10000baseT_Full | SUPPORTED_AUI, + &mi1_mdio_ext_ops, "Chelsio T320"}, + {}, + {}, + {1, 0, 0, + F_GPIO1_OEN | F_GPIO2_OEN | F_GPIO4_OEN | F_GPIO6_OEN | F_GPIO7_OEN | + F_GPIO10_OEN | F_GPIO1_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL, + { S_GPIO9 }, SUPPORTED_10000baseT_Full | SUPPORTED_AUI, + &mi1_mdio_ext_ops, "Chelsio T310" }, + {1, 0, 0, + F_GPIO1_OEN | F_GPIO6_OEN | F_GPIO7_OEN | + F_GPIO1_OUT_VAL | F_GPIO6_OUT_VAL, + { S_GPIO9 }, SUPPORTED_10000baseT_Full | SUPPORTED_AUI, + &mi1_mdio_ext_ops, "Chelsio N320E-G2" }, +}; + +/* + * Return the adapter_info structure with a given index. Out-of-range indices + * return NULL. + */ +const struct adapter_info *t3_get_adapter_info(unsigned int id) +{ + return id < ARRAY_SIZE(t3_adap_info) ? &t3_adap_info[id] : NULL; +} + +struct port_type_info { + int (*phy_prep)(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *ops); +}; + +static const struct port_type_info port_types[] = { + { NULL }, + { t3_ael1002_phy_prep }, + { t3_vsc8211_phy_prep }, + { NULL}, + { t3_xaui_direct_phy_prep }, + { t3_ael2005_phy_prep }, + { t3_qt2045_phy_prep }, + { t3_ael1006_phy_prep }, + { NULL }, + { t3_aq100x_phy_prep }, + { t3_ael2020_phy_prep }, +}; + +#define VPD_ENTRY(name, len) \ + u8 name##_kword[2]; u8 name##_len; u8 name##_data[len] + +/* + * Partial EEPROM Vital Product Data structure. Includes only the ID and + * VPD-R sections. + */ +struct t3_vpd { + u8 id_tag; + u8 id_len[2]; + u8 id_data[16]; + u8 vpdr_tag; + u8 vpdr_len[2]; + VPD_ENTRY(pn, 16); /* part number */ + VPD_ENTRY(ec, 16); /* EC level */ + VPD_ENTRY(sn, SERNUM_LEN); /* serial number */ + VPD_ENTRY(na, 12); /* MAC address base */ + VPD_ENTRY(cclk, 6); /* core clock */ + VPD_ENTRY(mclk, 6); /* mem clock */ + VPD_ENTRY(uclk, 6); /* uP clk */ + VPD_ENTRY(mdc, 6); /* MDIO clk */ + VPD_ENTRY(mt, 2); /* mem timing */ + VPD_ENTRY(xaui0cfg, 6); /* XAUI0 config */ + VPD_ENTRY(xaui1cfg, 6); /* XAUI1 config */ + VPD_ENTRY(port0, 2); /* PHY0 complex */ + VPD_ENTRY(port1, 2); /* PHY1 complex */ + VPD_ENTRY(port2, 2); /* PHY2 complex */ + VPD_ENTRY(port3, 2); /* PHY3 complex */ + VPD_ENTRY(rv, 1); /* csum */ + u32 pad; /* for multiple-of-4 sizing and alignment */ +}; + +#define EEPROM_STAT_ADDR 0x4000 +#define VPD_BASE 0xc00 + +/** + * t3_seeprom_wp - enable/disable EEPROM write protection + * @adapter: the adapter + * @enable: 1 to enable write protection, 0 to disable it + * + * Enables or disables write protection on the serial EEPROM. + */ +int t3_seeprom_wp(struct adapter *adapter, int enable) +{ + u32 data = enable ? 0xc : 0; + int ret; + + /* EEPROM_STAT_ADDR is outside VPD area, use pci_write_vpd_any() */ + ret = pci_write_vpd_any(adapter->pdev, EEPROM_STAT_ADDR, sizeof(u32), + &data); + + return ret < 0 ? ret : 0; +} + +static int vpdstrtouint(char *s, u8 len, unsigned int base, unsigned int *val) +{ + char tok[256]; + + memcpy(tok, s, len); + tok[len] = 0; + return kstrtouint(strim(tok), base, val); +} + +static int vpdstrtou16(char *s, u8 len, unsigned int base, u16 *val) +{ + char tok[256]; + + memcpy(tok, s, len); + tok[len] = 0; + return kstrtou16(strim(tok), base, val); +} + +/** + * get_vpd_params - read VPD parameters from VPD EEPROM + * @adapter: adapter to read + * @p: where to store the parameters + * + * Reads card parameters stored in VPD EEPROM. + */ +static int get_vpd_params(struct adapter *adapter, struct vpd_params *p) +{ + struct t3_vpd vpd; + u8 base_val = 0; + int addr, ret; + + /* + * Card information is normally at VPD_BASE but some early cards had + * it at 0. + */ + ret = pci_read_vpd(adapter->pdev, VPD_BASE, 1, &base_val); + if (ret < 0) + return ret; + addr = base_val == PCI_VPD_LRDT_ID_STRING ? VPD_BASE : 0; + + ret = pci_read_vpd(adapter->pdev, addr, sizeof(vpd), &vpd); + if (ret < 0) + return ret; + + ret = vpdstrtouint(vpd.cclk_data, vpd.cclk_len, 10, &p->cclk); + if (ret) + return ret; + ret = vpdstrtouint(vpd.mclk_data, vpd.mclk_len, 10, &p->mclk); + if (ret) + return ret; + ret = vpdstrtouint(vpd.uclk_data, vpd.uclk_len, 10, &p->uclk); + if (ret) + return ret; + ret = vpdstrtouint(vpd.mdc_data, vpd.mdc_len, 10, &p->mdc); + if (ret) + return ret; + ret = vpdstrtouint(vpd.mt_data, vpd.mt_len, 10, &p->mem_timing); + if (ret) + return ret; + memcpy(p->sn, vpd.sn_data, SERNUM_LEN); + + /* Old eeproms didn't have port information */ + if (adapter->params.rev == 0 && !vpd.port0_data[0]) { + p->port_type[0] = uses_xaui(adapter) ? 1 : 2; + p->port_type[1] = uses_xaui(adapter) ? 6 : 2; + } else { + p->port_type[0] = hex_to_bin(vpd.port0_data[0]); + p->port_type[1] = hex_to_bin(vpd.port1_data[0]); + ret = vpdstrtou16(vpd.xaui0cfg_data, vpd.xaui0cfg_len, 16, + &p->xauicfg[0]); + if (ret) + return ret; + ret = vpdstrtou16(vpd.xaui1cfg_data, vpd.xaui1cfg_len, 16, + &p->xauicfg[1]); + if (ret) + return ret; + } + + ret = hex2bin(p->eth_base, vpd.na_data, 6); + if (ret < 0) + return -EINVAL; + return 0; +} + +/* serial flash and firmware constants */ +enum { + SF_ATTEMPTS = 5, /* max retries for SF1 operations */ + SF_SEC_SIZE = 64 * 1024, /* serial flash sector size */ + SF_SIZE = SF_SEC_SIZE * 8, /* serial flash size */ + + /* flash command opcodes */ + SF_PROG_PAGE = 2, /* program page */ + SF_WR_DISABLE = 4, /* disable writes */ + SF_RD_STATUS = 5, /* read status register */ + SF_WR_ENABLE = 6, /* enable writes */ + SF_RD_DATA_FAST = 0xb, /* read flash */ + SF_ERASE_SECTOR = 0xd8, /* erase sector */ + + FW_FLASH_BOOT_ADDR = 0x70000, /* start address of FW in flash */ + FW_VERS_ADDR = 0x7fffc, /* flash address holding FW version */ + FW_MIN_SIZE = 8 /* at least version and csum */ +}; + +/** + * sf1_read - read data from the serial flash + * @adapter: the adapter + * @byte_cnt: number of bytes to read + * @cont: whether another operation will be chained + * @valp: where to store the read data + * + * Reads up to 4 bytes of data from the serial flash. The location of + * the read needs to be specified prior to calling this by issuing the + * appropriate commands to the serial flash. + */ +static int sf1_read(struct adapter *adapter, unsigned int byte_cnt, int cont, + u32 *valp) +{ + int ret; + + if (!byte_cnt || byte_cnt > 4) + return -EINVAL; + if (t3_read_reg(adapter, A_SF_OP) & F_BUSY) + return -EBUSY; + t3_write_reg(adapter, A_SF_OP, V_CONT(cont) | V_BYTECNT(byte_cnt - 1)); + ret = t3_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 10); + if (!ret) + *valp = t3_read_reg(adapter, A_SF_DATA); + return ret; +} + +/** + * sf1_write - write data to the serial flash + * @adapter: the adapter + * @byte_cnt: number of bytes to write + * @cont: whether another operation will be chained + * @val: value to write + * + * Writes up to 4 bytes of data to the serial flash. The location of + * the write needs to be specified prior to calling this by issuing the + * appropriate commands to the serial flash. + */ +static int sf1_write(struct adapter *adapter, unsigned int byte_cnt, int cont, + u32 val) +{ + if (!byte_cnt || byte_cnt > 4) + return -EINVAL; + if (t3_read_reg(adapter, A_SF_OP) & F_BUSY) + return -EBUSY; + t3_write_reg(adapter, A_SF_DATA, val); + t3_write_reg(adapter, A_SF_OP, + V_CONT(cont) | V_BYTECNT(byte_cnt - 1) | V_OP(1)); + return t3_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 10); +} + +/** + * flash_wait_op - wait for a flash operation to complete + * @adapter: the adapter + * @attempts: max number of polls of the status register + * @delay: delay between polls in ms + * + * Wait for a flash operation to complete by polling the status register. + */ +static int flash_wait_op(struct adapter *adapter, int attempts, int delay) +{ + int ret; + u32 status; + + while (1) { + if ((ret = sf1_write(adapter, 1, 1, SF_RD_STATUS)) != 0 || + (ret = sf1_read(adapter, 1, 0, &status)) != 0) + return ret; + if (!(status & 1)) + return 0; + if (--attempts == 0) + return -EAGAIN; + if (delay) + msleep(delay); + } +} + +/** + * t3_read_flash - read words from serial flash + * @adapter: the adapter + * @addr: the start address for the read + * @nwords: how many 32-bit words to read + * @data: where to store the read data + * @byte_oriented: whether to store data as bytes or as words + * + * Read the specified number of 32-bit words from the serial flash. + * If @byte_oriented is set the read data is stored as a byte array + * (i.e., big-endian), otherwise as 32-bit words in the platform's + * natural endianness. + */ +static int t3_read_flash(struct adapter *adapter, unsigned int addr, + unsigned int nwords, u32 *data, int byte_oriented) +{ + int ret; + + if (addr + nwords * sizeof(u32) > SF_SIZE || (addr & 3)) + return -EINVAL; + + addr = swab32(addr) | SF_RD_DATA_FAST; + + if ((ret = sf1_write(adapter, 4, 1, addr)) != 0 || + (ret = sf1_read(adapter, 1, 1, data)) != 0) + return ret; + + for (; nwords; nwords--, data++) { + ret = sf1_read(adapter, 4, nwords > 1, data); + if (ret) + return ret; + if (byte_oriented) + *data = htonl(*data); + } + return 0; +} + +/** + * t3_write_flash - write up to a page of data to the serial flash + * @adapter: the adapter + * @addr: the start address to write + * @n: length of data to write + * @data: the data to write + * + * Writes up to a page of data (256 bytes) to the serial flash starting + * at the given address. + */ +static int t3_write_flash(struct adapter *adapter, unsigned int addr, + unsigned int n, const u8 *data) +{ + int ret; + u32 buf[64]; + unsigned int i, c, left, val, offset = addr & 0xff; + + if (addr + n > SF_SIZE || offset + n > 256) + return -EINVAL; + + val = swab32(addr) | SF_PROG_PAGE; + + if ((ret = sf1_write(adapter, 1, 0, SF_WR_ENABLE)) != 0 || + (ret = sf1_write(adapter, 4, 1, val)) != 0) + return ret; + + for (left = n; left; left -= c) { + c = min(left, 4U); + for (val = 0, i = 0; i < c; ++i) + val = (val << 8) + *data++; + + ret = sf1_write(adapter, c, c != left, val); + if (ret) + return ret; + } + if ((ret = flash_wait_op(adapter, 5, 1)) != 0) + return ret; + + /* Read the page to verify the write succeeded */ + ret = t3_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, 1); + if (ret) + return ret; + + if (memcmp(data - n, (u8 *) buf + offset, n)) + return -EIO; + return 0; +} + +/** + * t3_get_tp_version - read the tp sram version + * @adapter: the adapter + * @vers: where to place the version + * + * Reads the protocol sram version from sram. + */ +int t3_get_tp_version(struct adapter *adapter, u32 *vers) +{ + int ret; + + /* Get version loaded in SRAM */ + t3_write_reg(adapter, A_TP_EMBED_OP_FIELD0, 0); + ret = t3_wait_op_done(adapter, A_TP_EMBED_OP_FIELD0, + 1, 1, 5, 1); + if (ret) + return ret; + + *vers = t3_read_reg(adapter, A_TP_EMBED_OP_FIELD1); + + return 0; +} + +/** + * t3_check_tpsram_version - read the tp sram version + * @adapter: the adapter + * + * Reads the protocol sram version from flash. + */ +int t3_check_tpsram_version(struct adapter *adapter) +{ + int ret; + u32 vers; + unsigned int major, minor; + + if (adapter->params.rev == T3_REV_A) + return 0; + + + ret = t3_get_tp_version(adapter, &vers); + if (ret) + return ret; + + major = G_TP_VERSION_MAJOR(vers); + minor = G_TP_VERSION_MINOR(vers); + + if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) + return 0; + else { + CH_ERR(adapter, "found wrong TP version (%u.%u), " + "driver compiled for version %d.%d\n", major, minor, + TP_VERSION_MAJOR, TP_VERSION_MINOR); + } + return -EINVAL; +} + +/** + * t3_check_tpsram - check if provided protocol SRAM + * is compatible with this driver + * @adapter: the adapter + * @tp_sram: the firmware image to write + * @size: image size + * + * Checks if an adapter's tp sram is compatible with the driver. + * Returns 0 if the versions are compatible, a negative error otherwise. + */ +int t3_check_tpsram(struct adapter *adapter, const u8 *tp_sram, + unsigned int size) +{ + u32 csum; + unsigned int i; + const __be32 *p = (const __be32 *)tp_sram; + + /* Verify checksum */ + for (csum = 0, i = 0; i < size / sizeof(csum); i++) + csum += ntohl(p[i]); + if (csum != 0xffffffff) { + CH_ERR(adapter, "corrupted protocol SRAM image, checksum %u\n", + csum); + return -EINVAL; + } + + return 0; +} + +enum fw_version_type { + FW_VERSION_N3, + FW_VERSION_T3 +}; + +/** + * t3_get_fw_version - read the firmware version + * @adapter: the adapter + * @vers: where to place the version + * + * Reads the FW version from flash. + */ +int t3_get_fw_version(struct adapter *adapter, u32 *vers) +{ + return t3_read_flash(adapter, FW_VERS_ADDR, 1, vers, 0); +} + +/** + * t3_check_fw_version - check if the FW is compatible with this driver + * @adapter: the adapter + * + * Checks if an adapter's FW is compatible with the driver. Returns 0 + * if the versions are compatible, a negative error otherwise. + */ +int t3_check_fw_version(struct adapter *adapter) +{ + int ret; + u32 vers; + unsigned int type, major, minor; + + ret = t3_get_fw_version(adapter, &vers); + if (ret) + return ret; + + type = G_FW_VERSION_TYPE(vers); + major = G_FW_VERSION_MAJOR(vers); + minor = G_FW_VERSION_MINOR(vers); + + if (type == FW_VERSION_T3 && major == FW_VERSION_MAJOR && + minor == FW_VERSION_MINOR) + return 0; + else if (major != FW_VERSION_MAJOR || minor < FW_VERSION_MINOR) + CH_WARN(adapter, "found old FW minor version(%u.%u), " + "driver compiled for version %u.%u\n", major, minor, + FW_VERSION_MAJOR, FW_VERSION_MINOR); + else { + CH_WARN(adapter, "found newer FW version(%u.%u), " + "driver compiled for version %u.%u\n", major, minor, + FW_VERSION_MAJOR, FW_VERSION_MINOR); + return 0; + } + return -EINVAL; +} + +/** + * t3_flash_erase_sectors - erase a range of flash sectors + * @adapter: the adapter + * @start: the first sector to erase + * @end: the last sector to erase + * + * Erases the sectors in the given range. + */ +static int t3_flash_erase_sectors(struct adapter *adapter, int start, int end) +{ + while (start <= end) { + int ret; + + if ((ret = sf1_write(adapter, 1, 0, SF_WR_ENABLE)) != 0 || + (ret = sf1_write(adapter, 4, 0, + SF_ERASE_SECTOR | (start << 8))) != 0 || + (ret = flash_wait_op(adapter, 5, 500)) != 0) + return ret; + start++; + } + return 0; +} + +/** + * t3_load_fw - download firmware + * @adapter: the adapter + * @fw_data: the firmware image to write + * @size: image size + * + * Write the supplied firmware image to the card's serial flash. + * The FW image has the following sections: @size - 8 bytes of code and + * data, followed by 4 bytes of FW version, followed by the 32-bit + * 1's complement checksum of the whole image. + */ +int t3_load_fw(struct adapter *adapter, const u8 *fw_data, unsigned int size) +{ + u32 csum; + unsigned int i; + const __be32 *p = (const __be32 *)fw_data; + int ret, addr, fw_sector = FW_FLASH_BOOT_ADDR >> 16; + + if ((size & 3) || size < FW_MIN_SIZE) + return -EINVAL; + if (size > FW_VERS_ADDR + 8 - FW_FLASH_BOOT_ADDR) + return -EFBIG; + + for (csum = 0, i = 0; i < size / sizeof(csum); i++) + csum += ntohl(p[i]); + if (csum != 0xffffffff) { + CH_ERR(adapter, "corrupted firmware image, checksum %u\n", + csum); + return -EINVAL; + } + + ret = t3_flash_erase_sectors(adapter, fw_sector, fw_sector); + if (ret) + goto out; + + size -= 8; /* trim off version and checksum */ + for (addr = FW_FLASH_BOOT_ADDR; size;) { + unsigned int chunk_size = min(size, 256U); + + ret = t3_write_flash(adapter, addr, chunk_size, fw_data); + if (ret) + goto out; + + addr += chunk_size; + fw_data += chunk_size; + size -= chunk_size; + } + + ret = t3_write_flash(adapter, FW_VERS_ADDR, 4, fw_data); +out: + if (ret) + CH_ERR(adapter, "firmware download failed, error %d\n", ret); + return ret; +} + +#define CIM_CTL_BASE 0x2000 + +/** + * t3_cim_ctl_blk_read - read a block from CIM control region + * + * @adap: the adapter + * @addr: the start address within the CIM control region + * @n: number of words to read + * @valp: where to store the result + * + * Reads a block of 4-byte words from the CIM control region. + */ +int t3_cim_ctl_blk_read(struct adapter *adap, unsigned int addr, + unsigned int n, unsigned int *valp) +{ + int ret = 0; + + if (t3_read_reg(adap, A_CIM_HOST_ACC_CTRL) & F_HOSTBUSY) + return -EBUSY; + + for ( ; !ret && n--; addr += 4) { + t3_write_reg(adap, A_CIM_HOST_ACC_CTRL, CIM_CTL_BASE + addr); + ret = t3_wait_op_done(adap, A_CIM_HOST_ACC_CTRL, F_HOSTBUSY, + 0, 5, 2); + if (!ret) + *valp++ = t3_read_reg(adap, A_CIM_HOST_ACC_DATA); + } + return ret; +} + +static void t3_gate_rx_traffic(struct cmac *mac, u32 *rx_cfg, + u32 *rx_hash_high, u32 *rx_hash_low) +{ + /* stop Rx unicast traffic */ + t3_mac_disable_exact_filters(mac); + + /* stop broadcast, multicast, promiscuous mode traffic */ + *rx_cfg = t3_read_reg(mac->adapter, A_XGM_RX_CFG); + t3_set_reg_field(mac->adapter, A_XGM_RX_CFG, + F_ENHASHMCAST | F_DISBCAST | F_COPYALLFRAMES, + F_DISBCAST); + + *rx_hash_high = t3_read_reg(mac->adapter, A_XGM_RX_HASH_HIGH); + t3_write_reg(mac->adapter, A_XGM_RX_HASH_HIGH, 0); + + *rx_hash_low = t3_read_reg(mac->adapter, A_XGM_RX_HASH_LOW); + t3_write_reg(mac->adapter, A_XGM_RX_HASH_LOW, 0); + + /* Leave time to drain max RX fifo */ + msleep(1); +} + +static void t3_open_rx_traffic(struct cmac *mac, u32 rx_cfg, + u32 rx_hash_high, u32 rx_hash_low) +{ + t3_mac_enable_exact_filters(mac); + t3_set_reg_field(mac->adapter, A_XGM_RX_CFG, + F_ENHASHMCAST | F_DISBCAST | F_COPYALLFRAMES, + rx_cfg); + t3_write_reg(mac->adapter, A_XGM_RX_HASH_HIGH, rx_hash_high); + t3_write_reg(mac->adapter, A_XGM_RX_HASH_LOW, rx_hash_low); +} + +/** + * t3_link_changed - handle interface link changes + * @adapter: the adapter + * @port_id: the port index that changed link state + * + * Called when a port's link settings change to propagate the new values + * to the associated PHY and MAC. After performing the common tasks it + * invokes an OS-specific handler. + */ +void t3_link_changed(struct adapter *adapter, int port_id) +{ + int link_ok, speed, duplex, fc; + struct port_info *pi = adap2pinfo(adapter, port_id); + struct cphy *phy = &pi->phy; + struct cmac *mac = &pi->mac; + struct link_config *lc = &pi->link_config; + + phy->ops->get_link_status(phy, &link_ok, &speed, &duplex, &fc); + + if (!lc->link_ok && link_ok) { + u32 rx_cfg, rx_hash_high, rx_hash_low; + u32 status; + + t3_xgm_intr_enable(adapter, port_id); + t3_gate_rx_traffic(mac, &rx_cfg, &rx_hash_high, &rx_hash_low); + t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset, 0); + t3_mac_enable(mac, MAC_DIRECTION_RX); + + status = t3_read_reg(adapter, A_XGM_INT_STATUS + mac->offset); + if (status & F_LINKFAULTCHANGE) { + mac->stats.link_faults++; + pi->link_fault = 1; + } + t3_open_rx_traffic(mac, rx_cfg, rx_hash_high, rx_hash_low); + } + + if (lc->requested_fc & PAUSE_AUTONEG) + fc &= lc->requested_fc; + else + fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); + + if (link_ok == lc->link_ok && speed == lc->speed && + duplex == lc->duplex && fc == lc->fc) + return; /* nothing changed */ + + if (link_ok != lc->link_ok && adapter->params.rev > 0 && + uses_xaui(adapter)) { + if (link_ok) + t3b_pcs_reset(mac); + t3_write_reg(adapter, A_XGM_XAUI_ACT_CTRL + mac->offset, + link_ok ? F_TXACTENABLE | F_RXEN : 0); + } + lc->link_ok = link_ok; + lc->speed = speed < 0 ? SPEED_INVALID : speed; + lc->duplex = duplex < 0 ? DUPLEX_INVALID : duplex; + + if (link_ok && speed >= 0 && lc->autoneg == AUTONEG_ENABLE) { + /* Set MAC speed, duplex, and flow control to match PHY. */ + t3_mac_set_speed_duplex_fc(mac, speed, duplex, fc); + lc->fc = fc; + } + + t3_os_link_changed(adapter, port_id, link_ok && !pi->link_fault, + speed, duplex, fc); +} + +void t3_link_fault(struct adapter *adapter, int port_id) +{ + struct port_info *pi = adap2pinfo(adapter, port_id); + struct cmac *mac = &pi->mac; + struct cphy *phy = &pi->phy; + struct link_config *lc = &pi->link_config; + int link_ok, speed, duplex, fc, link_fault; + u32 rx_cfg, rx_hash_high, rx_hash_low; + + t3_gate_rx_traffic(mac, &rx_cfg, &rx_hash_high, &rx_hash_low); + + if (adapter->params.rev > 0 && uses_xaui(adapter)) + t3_write_reg(adapter, A_XGM_XAUI_ACT_CTRL + mac->offset, 0); + + t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset, 0); + t3_mac_enable(mac, MAC_DIRECTION_RX); + + t3_open_rx_traffic(mac, rx_cfg, rx_hash_high, rx_hash_low); + + link_fault = t3_read_reg(adapter, + A_XGM_INT_STATUS + mac->offset); + link_fault &= F_LINKFAULTCHANGE; + + link_ok = lc->link_ok; + speed = lc->speed; + duplex = lc->duplex; + fc = lc->fc; + + phy->ops->get_link_status(phy, &link_ok, &speed, &duplex, &fc); + + if (link_fault) { + lc->link_ok = 0; + lc->speed = SPEED_INVALID; + lc->duplex = DUPLEX_INVALID; + + t3_os_link_fault(adapter, port_id, 0); + + /* Account link faults only when the phy reports a link up */ + if (link_ok) + mac->stats.link_faults++; + } else { + if (link_ok) + t3_write_reg(adapter, A_XGM_XAUI_ACT_CTRL + mac->offset, + F_TXACTENABLE | F_RXEN); + + pi->link_fault = 0; + lc->link_ok = (unsigned char)link_ok; + lc->speed = speed < 0 ? SPEED_INVALID : speed; + lc->duplex = duplex < 0 ? DUPLEX_INVALID : duplex; + t3_os_link_fault(adapter, port_id, link_ok); + } +} + +/** + * t3_link_start - apply link configuration to MAC/PHY + * @phy: the PHY to setup + * @mac: the MAC to setup + * @lc: the requested link configuration + * + * Set up a port's MAC and PHY according to a desired link configuration. + * - If the PHY can auto-negotiate first decide what to advertise, then + * enable/disable auto-negotiation as desired, and reset. + * - If the PHY does not auto-negotiate just reset it. + * - If auto-negotiation is off set the MAC to the proper speed/duplex/FC, + * otherwise do it later based on the outcome of auto-negotiation. + */ +int t3_link_start(struct cphy *phy, struct cmac *mac, struct link_config *lc) +{ + unsigned int fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX); + + lc->link_ok = 0; + if (lc->supported & SUPPORTED_Autoneg) { + lc->advertising &= ~(ADVERTISED_Asym_Pause | ADVERTISED_Pause); + if (fc) { + lc->advertising |= ADVERTISED_Asym_Pause; + if (fc & PAUSE_RX) + lc->advertising |= ADVERTISED_Pause; + } + phy->ops->advertise(phy, lc->advertising); + + if (lc->autoneg == AUTONEG_DISABLE) { + lc->speed = lc->requested_speed; + lc->duplex = lc->requested_duplex; + lc->fc = (unsigned char)fc; + t3_mac_set_speed_duplex_fc(mac, lc->speed, lc->duplex, + fc); + /* Also disables autoneg */ + phy->ops->set_speed_duplex(phy, lc->speed, lc->duplex); + } else + phy->ops->autoneg_enable(phy); + } else { + t3_mac_set_speed_duplex_fc(mac, -1, -1, fc); + lc->fc = (unsigned char)fc; + phy->ops->reset(phy, 0); + } + return 0; +} + +/** + * t3_set_vlan_accel - control HW VLAN extraction + * @adapter: the adapter + * @ports: bitmap of adapter ports to operate on + * @on: enable (1) or disable (0) HW VLAN extraction + * + * Enables or disables HW extraction of VLAN tags for the given port. + */ +void t3_set_vlan_accel(struct adapter *adapter, unsigned int ports, int on) +{ + t3_set_reg_field(adapter, A_TP_OUT_CONFIG, + ports << S_VLANEXTRACTIONENABLE, + on ? (ports << S_VLANEXTRACTIONENABLE) : 0); +} + +struct intr_info { + unsigned int mask; /* bits to check in interrupt status */ + const char *msg; /* message to print or NULL */ + short stat_idx; /* stat counter to increment or -1 */ + unsigned short fatal; /* whether the condition reported is fatal */ +}; + +/** + * t3_handle_intr_status - table driven interrupt handler + * @adapter: the adapter that generated the interrupt + * @reg: the interrupt status register to process + * @mask: a mask to apply to the interrupt status + * @acts: table of interrupt actions + * @stats: statistics counters tracking interrupt occurrences + * + * A table driven interrupt handler that applies a set of masks to an + * interrupt status word and performs the corresponding actions if the + * interrupts described by the mask have occurred. The actions include + * optionally printing a warning or alert message, and optionally + * incrementing a stat counter. The table is terminated by an entry + * specifying mask 0. Returns the number of fatal interrupt conditions. + */ +static int t3_handle_intr_status(struct adapter *adapter, unsigned int reg, + unsigned int mask, + const struct intr_info *acts, + unsigned long *stats) +{ + int fatal = 0; + unsigned int status = t3_read_reg(adapter, reg) & mask; + + for (; acts->mask; ++acts) { + if (!(status & acts->mask)) + continue; + if (acts->fatal) { + fatal++; + CH_ALERT(adapter, "%s (0x%x)\n", + acts->msg, status & acts->mask); + status &= ~acts->mask; + } else if (acts->msg) + CH_WARN(adapter, "%s (0x%x)\n", + acts->msg, status & acts->mask); + if (acts->stat_idx >= 0) + stats[acts->stat_idx]++; + } + if (status) /* clear processed interrupts */ + t3_write_reg(adapter, reg, status); + return fatal; +} + +#define SGE_INTR_MASK (F_RSPQDISABLED | \ + F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR | \ + F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \ + F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \ + V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \ + F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \ + F_HIRCQPARITYERROR | F_LOPRIORITYDBFULL | \ + F_HIPRIORITYDBFULL | F_LOPRIORITYDBEMPTY | \ + F_HIPRIORITYDBEMPTY | F_HIPIODRBDROPERR | \ + F_LOPIODRBDROPERR) +#define MC5_INTR_MASK (F_PARITYERR | F_ACTRGNFULL | F_UNKNOWNCMD | \ + F_REQQPARERR | F_DISPQPARERR | F_DELACTEMPTY | \ + F_NFASRCHFAIL) +#define MC7_INTR_MASK (F_AE | F_UE | F_CE | V_PE(M_PE)) +#define XGM_INTR_MASK (V_TXFIFO_PRTY_ERR(M_TXFIFO_PRTY_ERR) | \ + V_RXFIFO_PRTY_ERR(M_RXFIFO_PRTY_ERR) | \ + F_TXFIFO_UNDERRUN) +#define PCIX_INTR_MASK (F_MSTDETPARERR | F_SIGTARABT | F_RCVTARABT | \ + F_RCVMSTABT | F_SIGSYSERR | F_DETPARERR | \ + F_SPLCMPDIS | F_UNXSPLCMP | F_RCVSPLCMPERR | \ + F_DETCORECCERR | F_DETUNCECCERR | F_PIOPARERR | \ + V_WFPARERR(M_WFPARERR) | V_RFPARERR(M_RFPARERR) | \ + V_CFPARERR(M_CFPARERR) /* | V_MSIXPARERR(M_MSIXPARERR) */) +#define PCIE_INTR_MASK (F_UNXSPLCPLERRR | F_UNXSPLCPLERRC | F_PCIE_PIOPARERR |\ + F_PCIE_WFPARERR | F_PCIE_RFPARERR | F_PCIE_CFPARERR | \ + /* V_PCIE_MSIXPARERR(M_PCIE_MSIXPARERR) | */ \ + F_RETRYBUFPARERR | F_RETRYLUTPARERR | F_RXPARERR | \ + F_TXPARERR | V_BISTERR(M_BISTERR)) +#define ULPRX_INTR_MASK (F_PARERRDATA | F_PARERRPCMD | F_ARBPF1PERR | \ + F_ARBPF0PERR | F_ARBFPERR | F_PCMDMUXPERR | \ + F_DATASELFRAMEERR1 | F_DATASELFRAMEERR0) +#define ULPTX_INTR_MASK 0xfc +#define CPLSW_INTR_MASK (F_CIM_OP_MAP_PERR | F_TP_FRAMING_ERROR | \ + F_SGE_FRAMING_ERROR | F_CIM_FRAMING_ERROR | \ + F_ZERO_SWITCH_ERROR) +#define CIM_INTR_MASK (F_BLKWRPLINT | F_BLKRDPLINT | F_BLKWRCTLINT | \ + F_BLKRDCTLINT | F_BLKWRFLASHINT | F_BLKRDFLASHINT | \ + F_SGLWRFLASHINT | F_WRBLKFLASHINT | F_BLKWRBOOTINT | \ + F_FLASHRANGEINT | F_SDRAMRANGEINT | F_RSVDSPACEINT | \ + F_DRAMPARERR | F_ICACHEPARERR | F_DCACHEPARERR | \ + F_OBQSGEPARERR | F_OBQULPHIPARERR | F_OBQULPLOPARERR | \ + F_IBQSGELOPARERR | F_IBQSGEHIPARERR | F_IBQULPPARERR | \ + F_IBQTPPARERR | F_ITAGPARERR | F_DTAGPARERR) +#define PMTX_INTR_MASK (F_ZERO_C_CMD_ERROR | ICSPI_FRM_ERR | OESPI_FRM_ERR | \ + V_ICSPI_PAR_ERROR(M_ICSPI_PAR_ERROR) | \ + V_OESPI_PAR_ERROR(M_OESPI_PAR_ERROR)) +#define PMRX_INTR_MASK (F_ZERO_E_CMD_ERROR | IESPI_FRM_ERR | OCSPI_FRM_ERR | \ + V_IESPI_PAR_ERROR(M_IESPI_PAR_ERROR) | \ + V_OCSPI_PAR_ERROR(M_OCSPI_PAR_ERROR)) +#define MPS_INTR_MASK (V_TX0TPPARERRENB(M_TX0TPPARERRENB) | \ + V_TX1TPPARERRENB(M_TX1TPPARERRENB) | \ + V_RXTPPARERRENB(M_RXTPPARERRENB) | \ + V_MCAPARERRENB(M_MCAPARERRENB)) +#define XGM_EXTRA_INTR_MASK (F_LINKFAULTCHANGE) +#define PL_INTR_MASK (F_T3DBG | F_XGMAC0_0 | F_XGMAC0_1 | F_MC5A | F_PM1_TX | \ + F_PM1_RX | F_ULP2_TX | F_ULP2_RX | F_TP1 | F_CIM | \ + F_MC7_CM | F_MC7_PMTX | F_MC7_PMRX | F_SGE3 | F_PCIM0 | \ + F_MPS0 | F_CPL_SWITCH) +/* + * Interrupt handler for the PCIX1 module. + */ +static void pci_intr_handler(struct adapter *adapter) +{ + static const struct intr_info pcix1_intr_info[] = { + {F_MSTDETPARERR, "PCI master detected parity error", -1, 1}, + {F_SIGTARABT, "PCI signaled target abort", -1, 1}, + {F_RCVTARABT, "PCI received target abort", -1, 1}, + {F_RCVMSTABT, "PCI received master abort", -1, 1}, + {F_SIGSYSERR, "PCI signaled system error", -1, 1}, + {F_DETPARERR, "PCI detected parity error", -1, 1}, + {F_SPLCMPDIS, "PCI split completion discarded", -1, 1}, + {F_UNXSPLCMP, "PCI unexpected split completion error", -1, 1}, + {F_RCVSPLCMPERR, "PCI received split completion error", -1, + 1}, + {F_DETCORECCERR, "PCI correctable ECC error", + STAT_PCI_CORR_ECC, 0}, + {F_DETUNCECCERR, "PCI uncorrectable ECC error", -1, 1}, + {F_PIOPARERR, "PCI PIO FIFO parity error", -1, 1}, + {V_WFPARERR(M_WFPARERR), "PCI write FIFO parity error", -1, + 1}, + {V_RFPARERR(M_RFPARERR), "PCI read FIFO parity error", -1, + 1}, + {V_CFPARERR(M_CFPARERR), "PCI command FIFO parity error", -1, + 1}, + {V_MSIXPARERR(M_MSIXPARERR), "PCI MSI-X table/PBA parity " + "error", -1, 1}, + {0} + }; + + if (t3_handle_intr_status(adapter, A_PCIX_INT_CAUSE, PCIX_INTR_MASK, + pcix1_intr_info, adapter->irq_stats)) + t3_fatal_err(adapter); +} + +/* + * Interrupt handler for the PCIE module. + */ +static void pcie_intr_handler(struct adapter *adapter) +{ + static const struct intr_info pcie_intr_info[] = { + {F_PEXERR, "PCI PEX error", -1, 1}, + {F_UNXSPLCPLERRR, + "PCI unexpected split completion DMA read error", -1, 1}, + {F_UNXSPLCPLERRC, + "PCI unexpected split completion DMA command error", -1, 1}, + {F_PCIE_PIOPARERR, "PCI PIO FIFO parity error", -1, 1}, + {F_PCIE_WFPARERR, "PCI write FIFO parity error", -1, 1}, + {F_PCIE_RFPARERR, "PCI read FIFO parity error", -1, 1}, + {F_PCIE_CFPARERR, "PCI command FIFO parity error", -1, 1}, + {V_PCIE_MSIXPARERR(M_PCIE_MSIXPARERR), + "PCI MSI-X table/PBA parity error", -1, 1}, + {F_RETRYBUFPARERR, "PCI retry buffer parity error", -1, 1}, + {F_RETRYLUTPARERR, "PCI retry LUT parity error", -1, 1}, + {F_RXPARERR, "PCI Rx parity error", -1, 1}, + {F_TXPARERR, "PCI Tx parity error", -1, 1}, + {V_BISTERR(M_BISTERR), "PCI BIST error", -1, 1}, + {0} + }; + + if (t3_read_reg(adapter, A_PCIE_INT_CAUSE) & F_PEXERR) + CH_ALERT(adapter, "PEX error code 0x%x\n", + t3_read_reg(adapter, A_PCIE_PEX_ERR)); + + if (t3_handle_intr_status(adapter, A_PCIE_INT_CAUSE, PCIE_INTR_MASK, + pcie_intr_info, adapter->irq_stats)) + t3_fatal_err(adapter); +} + +/* + * TP interrupt handler. + */ +static void tp_intr_handler(struct adapter *adapter) +{ + static const struct intr_info tp_intr_info[] = { + {0xffffff, "TP parity error", -1, 1}, + {0x1000000, "TP out of Rx pages", -1, 1}, + {0x2000000, "TP out of Tx pages", -1, 1}, + {0} + }; + + static const struct intr_info tp_intr_info_t3c[] = { + {0x1fffffff, "TP parity error", -1, 1}, + {F_FLMRXFLSTEMPTY, "TP out of Rx pages", -1, 1}, + {F_FLMTXFLSTEMPTY, "TP out of Tx pages", -1, 1}, + {0} + }; + + if (t3_handle_intr_status(adapter, A_TP_INT_CAUSE, 0xffffffff, + adapter->params.rev < T3_REV_C ? + tp_intr_info : tp_intr_info_t3c, NULL)) + t3_fatal_err(adapter); +} + +/* + * CIM interrupt handler. + */ +static void cim_intr_handler(struct adapter *adapter) +{ + static const struct intr_info cim_intr_info[] = { + {F_RSVDSPACEINT, "CIM reserved space write", -1, 1}, + {F_SDRAMRANGEINT, "CIM SDRAM address out of range", -1, 1}, + {F_FLASHRANGEINT, "CIM flash address out of range", -1, 1}, + {F_BLKWRBOOTINT, "CIM block write to boot space", -1, 1}, + {F_WRBLKFLASHINT, "CIM write to cached flash space", -1, 1}, + {F_SGLWRFLASHINT, "CIM single write to flash space", -1, 1}, + {F_BLKRDFLASHINT, "CIM block read from flash space", -1, 1}, + {F_BLKWRFLASHINT, "CIM block write to flash space", -1, 1}, + {F_BLKRDCTLINT, "CIM block read from CTL space", -1, 1}, + {F_BLKWRCTLINT, "CIM block write to CTL space", -1, 1}, + {F_BLKRDPLINT, "CIM block read from PL space", -1, 1}, + {F_BLKWRPLINT, "CIM block write to PL space", -1, 1}, + {F_DRAMPARERR, "CIM DRAM parity error", -1, 1}, + {F_ICACHEPARERR, "CIM icache parity error", -1, 1}, + {F_DCACHEPARERR, "CIM dcache parity error", -1, 1}, + {F_OBQSGEPARERR, "CIM OBQ SGE parity error", -1, 1}, + {F_OBQULPHIPARERR, "CIM OBQ ULPHI parity error", -1, 1}, + {F_OBQULPLOPARERR, "CIM OBQ ULPLO parity error", -1, 1}, + {F_IBQSGELOPARERR, "CIM IBQ SGELO parity error", -1, 1}, + {F_IBQSGEHIPARERR, "CIM IBQ SGEHI parity error", -1, 1}, + {F_IBQULPPARERR, "CIM IBQ ULP parity error", -1, 1}, + {F_IBQTPPARERR, "CIM IBQ TP parity error", -1, 1}, + {F_ITAGPARERR, "CIM itag parity error", -1, 1}, + {F_DTAGPARERR, "CIM dtag parity error", -1, 1}, + {0} + }; + + if (t3_handle_intr_status(adapter, A_CIM_HOST_INT_CAUSE, 0xffffffff, + cim_intr_info, NULL)) + t3_fatal_err(adapter); +} + +/* + * ULP RX interrupt handler. + */ +static void ulprx_intr_handler(struct adapter *adapter) +{ + static const struct intr_info ulprx_intr_info[] = { + {F_PARERRDATA, "ULP RX data parity error", -1, 1}, + {F_PARERRPCMD, "ULP RX command parity error", -1, 1}, + {F_ARBPF1PERR, "ULP RX ArbPF1 parity error", -1, 1}, + {F_ARBPF0PERR, "ULP RX ArbPF0 parity error", -1, 1}, + {F_ARBFPERR, "ULP RX ArbF parity error", -1, 1}, + {F_PCMDMUXPERR, "ULP RX PCMDMUX parity error", -1, 1}, + {F_DATASELFRAMEERR1, "ULP RX frame error", -1, 1}, + {F_DATASELFRAMEERR0, "ULP RX frame error", -1, 1}, + {0} + }; + + if (t3_handle_intr_status(adapter, A_ULPRX_INT_CAUSE, 0xffffffff, + ulprx_intr_info, NULL)) + t3_fatal_err(adapter); +} + +/* + * ULP TX interrupt handler. + */ +static void ulptx_intr_handler(struct adapter *adapter) +{ + static const struct intr_info ulptx_intr_info[] = { + {F_PBL_BOUND_ERR_CH0, "ULP TX channel 0 PBL out of bounds", + STAT_ULP_CH0_PBL_OOB, 0}, + {F_PBL_BOUND_ERR_CH1, "ULP TX channel 1 PBL out of bounds", + STAT_ULP_CH1_PBL_OOB, 0}, + {0xfc, "ULP TX parity error", -1, 1}, + {0} + }; + + if (t3_handle_intr_status(adapter, A_ULPTX_INT_CAUSE, 0xffffffff, + ulptx_intr_info, adapter->irq_stats)) + t3_fatal_err(adapter); +} + +#define ICSPI_FRM_ERR (F_ICSPI0_FIFO2X_RX_FRAMING_ERROR | \ + F_ICSPI1_FIFO2X_RX_FRAMING_ERROR | F_ICSPI0_RX_FRAMING_ERROR | \ + F_ICSPI1_RX_FRAMING_ERROR | F_ICSPI0_TX_FRAMING_ERROR | \ + F_ICSPI1_TX_FRAMING_ERROR) +#define OESPI_FRM_ERR (F_OESPI0_RX_FRAMING_ERROR | \ + F_OESPI1_RX_FRAMING_ERROR | F_OESPI0_TX_FRAMING_ERROR | \ + F_OESPI1_TX_FRAMING_ERROR | F_OESPI0_OFIFO2X_TX_FRAMING_ERROR | \ + F_OESPI1_OFIFO2X_TX_FRAMING_ERROR) + +/* + * PM TX interrupt handler. + */ +static void pmtx_intr_handler(struct adapter *adapter) +{ + static const struct intr_info pmtx_intr_info[] = { + {F_ZERO_C_CMD_ERROR, "PMTX 0-length pcmd", -1, 1}, + {ICSPI_FRM_ERR, "PMTX ispi framing error", -1, 1}, + {OESPI_FRM_ERR, "PMTX ospi framing error", -1, 1}, + {V_ICSPI_PAR_ERROR(M_ICSPI_PAR_ERROR), + "PMTX ispi parity error", -1, 1}, + {V_OESPI_PAR_ERROR(M_OESPI_PAR_ERROR), + "PMTX ospi parity error", -1, 1}, + {0} + }; + + if (t3_handle_intr_status(adapter, A_PM1_TX_INT_CAUSE, 0xffffffff, + pmtx_intr_info, NULL)) + t3_fatal_err(adapter); +} + +#define IESPI_FRM_ERR (F_IESPI0_FIFO2X_RX_FRAMING_ERROR | \ + F_IESPI1_FIFO2X_RX_FRAMING_ERROR | F_IESPI0_RX_FRAMING_ERROR | \ + F_IESPI1_RX_FRAMING_ERROR | F_IESPI0_TX_FRAMING_ERROR | \ + F_IESPI1_TX_FRAMING_ERROR) +#define OCSPI_FRM_ERR (F_OCSPI0_RX_FRAMING_ERROR | \ + F_OCSPI1_RX_FRAMING_ERROR | F_OCSPI0_TX_FRAMING_ERROR | \ + F_OCSPI1_TX_FRAMING_ERROR | F_OCSPI0_OFIFO2X_TX_FRAMING_ERROR | \ + F_OCSPI1_OFIFO2X_TX_FRAMING_ERROR) + +/* + * PM RX interrupt handler. + */ +static void pmrx_intr_handler(struct adapter *adapter) +{ + static const struct intr_info pmrx_intr_info[] = { + {F_ZERO_E_CMD_ERROR, "PMRX 0-length pcmd", -1, 1}, + {IESPI_FRM_ERR, "PMRX ispi framing error", -1, 1}, + {OCSPI_FRM_ERR, "PMRX ospi framing error", -1, 1}, + {V_IESPI_PAR_ERROR(M_IESPI_PAR_ERROR), + "PMRX ispi parity error", -1, 1}, + {V_OCSPI_PAR_ERROR(M_OCSPI_PAR_ERROR), + "PMRX ospi parity error", -1, 1}, + {0} + }; + + if (t3_handle_intr_status(adapter, A_PM1_RX_INT_CAUSE, 0xffffffff, + pmrx_intr_info, NULL)) + t3_fatal_err(adapter); +} + +/* + * CPL switch interrupt handler. + */ +static void cplsw_intr_handler(struct adapter *adapter) +{ + static const struct intr_info cplsw_intr_info[] = { + {F_CIM_OP_MAP_PERR, "CPL switch CIM parity error", -1, 1}, + {F_CIM_OVFL_ERROR, "CPL switch CIM overflow", -1, 1}, + {F_TP_FRAMING_ERROR, "CPL switch TP framing error", -1, 1}, + {F_SGE_FRAMING_ERROR, "CPL switch SGE framing error", -1, 1}, + {F_CIM_FRAMING_ERROR, "CPL switch CIM framing error", -1, 1}, + {F_ZERO_SWITCH_ERROR, "CPL switch no-switch error", -1, 1}, + {0} + }; + + if (t3_handle_intr_status(adapter, A_CPL_INTR_CAUSE, 0xffffffff, + cplsw_intr_info, NULL)) + t3_fatal_err(adapter); +} + +/* + * MPS interrupt handler. + */ +static void mps_intr_handler(struct adapter *adapter) +{ + static const struct intr_info mps_intr_info[] = { + {0x1ff, "MPS parity error", -1, 1}, + {0} + }; + + if (t3_handle_intr_status(adapter, A_MPS_INT_CAUSE, 0xffffffff, + mps_intr_info, NULL)) + t3_fatal_err(adapter); +} + +#define MC7_INTR_FATAL (F_UE | V_PE(M_PE) | F_AE) + +/* + * MC7 interrupt handler. + */ +static void mc7_intr_handler(struct mc7 *mc7) +{ + struct adapter *adapter = mc7->adapter; + u32 cause = t3_read_reg(adapter, mc7->offset + A_MC7_INT_CAUSE); + + if (cause & F_CE) { + mc7->stats.corr_err++; + CH_WARN(adapter, "%s MC7 correctable error at addr 0x%x, " + "data 0x%x 0x%x 0x%x\n", mc7->name, + t3_read_reg(adapter, mc7->offset + A_MC7_CE_ADDR), + t3_read_reg(adapter, mc7->offset + A_MC7_CE_DATA0), + t3_read_reg(adapter, mc7->offset + A_MC7_CE_DATA1), + t3_read_reg(adapter, mc7->offset + A_MC7_CE_DATA2)); + } + + if (cause & F_UE) { + mc7->stats.uncorr_err++; + CH_ALERT(adapter, "%s MC7 uncorrectable error at addr 0x%x, " + "data 0x%x 0x%x 0x%x\n", mc7->name, + t3_read_reg(adapter, mc7->offset + A_MC7_UE_ADDR), + t3_read_reg(adapter, mc7->offset + A_MC7_UE_DATA0), + t3_read_reg(adapter, mc7->offset + A_MC7_UE_DATA1), + t3_read_reg(adapter, mc7->offset + A_MC7_UE_DATA2)); + } + + if (G_PE(cause)) { + mc7->stats.parity_err++; + CH_ALERT(adapter, "%s MC7 parity error 0x%x\n", + mc7->name, G_PE(cause)); + } + + if (cause & F_AE) { + u32 addr = 0; + + if (adapter->params.rev > 0) + addr = t3_read_reg(adapter, + mc7->offset + A_MC7_ERR_ADDR); + mc7->stats.addr_err++; + CH_ALERT(adapter, "%s MC7 address error: 0x%x\n", + mc7->name, addr); + } + + if (cause & MC7_INTR_FATAL) + t3_fatal_err(adapter); + + t3_write_reg(adapter, mc7->offset + A_MC7_INT_CAUSE, cause); +} + +#define XGM_INTR_FATAL (V_TXFIFO_PRTY_ERR(M_TXFIFO_PRTY_ERR) | \ + V_RXFIFO_PRTY_ERR(M_RXFIFO_PRTY_ERR)) +/* + * XGMAC interrupt handler. + */ +static int mac_intr_handler(struct adapter *adap, unsigned int idx) +{ + struct cmac *mac = &adap2pinfo(adap, idx)->mac; + /* + * We mask out interrupt causes for which we're not taking interrupts. + * This allows us to use polling logic to monitor some of the other + * conditions when taking interrupts would impose too much load on the + * system. + */ + u32 cause = t3_read_reg(adap, A_XGM_INT_CAUSE + mac->offset) & + ~F_RXFIFO_OVERFLOW; + + if (cause & V_TXFIFO_PRTY_ERR(M_TXFIFO_PRTY_ERR)) { + mac->stats.tx_fifo_parity_err++; + CH_ALERT(adap, "port%d: MAC TX FIFO parity error\n", idx); + } + if (cause & V_RXFIFO_PRTY_ERR(M_RXFIFO_PRTY_ERR)) { + mac->stats.rx_fifo_parity_err++; + CH_ALERT(adap, "port%d: MAC RX FIFO parity error\n", idx); + } + if (cause & F_TXFIFO_UNDERRUN) + mac->stats.tx_fifo_urun++; + if (cause & F_RXFIFO_OVERFLOW) + mac->stats.rx_fifo_ovfl++; + if (cause & V_SERDES_LOS(M_SERDES_LOS)) + mac->stats.serdes_signal_loss++; + if (cause & F_XAUIPCSCTCERR) + mac->stats.xaui_pcs_ctc_err++; + if (cause & F_XAUIPCSALIGNCHANGE) + mac->stats.xaui_pcs_align_change++; + if (cause & F_XGM_INT) { + t3_set_reg_field(adap, + A_XGM_INT_ENABLE + mac->offset, + F_XGM_INT, 0); + mac->stats.link_faults++; + + t3_os_link_fault_handler(adap, idx); + } + + if (cause & XGM_INTR_FATAL) + t3_fatal_err(adap); + + t3_write_reg(adap, A_XGM_INT_CAUSE + mac->offset, cause); + return cause != 0; +} + +/* + * Interrupt handler for PHY events. + */ +int t3_phy_intr_handler(struct adapter *adapter) +{ + u32 i, cause = t3_read_reg(adapter, A_T3DBG_INT_CAUSE); + + for_each_port(adapter, i) { + struct port_info *p = adap2pinfo(adapter, i); + + if (!(p->phy.caps & SUPPORTED_IRQ)) + continue; + + if (cause & (1 << adapter_info(adapter)->gpio_intr[i])) { + int phy_cause = p->phy.ops->intr_handler(&p->phy); + + if (phy_cause & cphy_cause_link_change) + t3_link_changed(adapter, i); + if (phy_cause & cphy_cause_fifo_error) + p->phy.fifo_errors++; + if (phy_cause & cphy_cause_module_change) + t3_os_phymod_changed(adapter, i); + } + } + + t3_write_reg(adapter, A_T3DBG_INT_CAUSE, cause); + return 0; +} + +/* + * T3 slow path (non-data) interrupt handler. + */ +int t3_slow_intr_handler(struct adapter *adapter) +{ + u32 cause = t3_read_reg(adapter, A_PL_INT_CAUSE0); + + cause &= adapter->slow_intr_mask; + if (!cause) + return 0; + if (cause & F_PCIM0) { + if (is_pcie(adapter)) + pcie_intr_handler(adapter); + else + pci_intr_handler(adapter); + } + if (cause & F_SGE3) + t3_sge_err_intr_handler(adapter); + if (cause & F_MC7_PMRX) + mc7_intr_handler(&adapter->pmrx); + if (cause & F_MC7_PMTX) + mc7_intr_handler(&adapter->pmtx); + if (cause & F_MC7_CM) + mc7_intr_handler(&adapter->cm); + if (cause & F_CIM) + cim_intr_handler(adapter); + if (cause & F_TP1) + tp_intr_handler(adapter); + if (cause & F_ULP2_RX) + ulprx_intr_handler(adapter); + if (cause & F_ULP2_TX) + ulptx_intr_handler(adapter); + if (cause & F_PM1_RX) + pmrx_intr_handler(adapter); + if (cause & F_PM1_TX) + pmtx_intr_handler(adapter); + if (cause & F_CPL_SWITCH) + cplsw_intr_handler(adapter); + if (cause & F_MPS0) + mps_intr_handler(adapter); + if (cause & F_MC5A) + t3_mc5_intr_handler(&adapter->mc5); + if (cause & F_XGMAC0_0) + mac_intr_handler(adapter, 0); + if (cause & F_XGMAC0_1) + mac_intr_handler(adapter, 1); + if (cause & F_T3DBG) + t3_os_ext_intr_handler(adapter); + + /* Clear the interrupts just processed. */ + t3_write_reg(adapter, A_PL_INT_CAUSE0, cause); + t3_read_reg(adapter, A_PL_INT_CAUSE0); /* flush */ + return 1; +} + +static unsigned int calc_gpio_intr(struct adapter *adap) +{ + unsigned int i, gpi_intr = 0; + + for_each_port(adap, i) + if ((adap2pinfo(adap, i)->phy.caps & SUPPORTED_IRQ) && + adapter_info(adap)->gpio_intr[i]) + gpi_intr |= 1 << adapter_info(adap)->gpio_intr[i]; + return gpi_intr; +} + +/** + * t3_intr_enable - enable interrupts + * @adapter: the adapter whose interrupts should be enabled + * + * Enable interrupts by setting the interrupt enable registers of the + * various HW modules and then enabling the top-level interrupt + * concentrator. + */ +void t3_intr_enable(struct adapter *adapter) +{ + static const struct addr_val_pair intr_en_avp[] = { + {A_SG_INT_ENABLE, SGE_INTR_MASK}, + {A_MC7_INT_ENABLE, MC7_INTR_MASK}, + {A_MC7_INT_ENABLE - MC7_PMRX_BASE_ADDR + MC7_PMTX_BASE_ADDR, + MC7_INTR_MASK}, + {A_MC7_INT_ENABLE - MC7_PMRX_BASE_ADDR + MC7_CM_BASE_ADDR, + MC7_INTR_MASK}, + {A_MC5_DB_INT_ENABLE, MC5_INTR_MASK}, + {A_ULPRX_INT_ENABLE, ULPRX_INTR_MASK}, + {A_PM1_TX_INT_ENABLE, PMTX_INTR_MASK}, + {A_PM1_RX_INT_ENABLE, PMRX_INTR_MASK}, + {A_CIM_HOST_INT_ENABLE, CIM_INTR_MASK}, + {A_MPS_INT_ENABLE, MPS_INTR_MASK}, + }; + + adapter->slow_intr_mask = PL_INTR_MASK; + + t3_write_regs(adapter, intr_en_avp, ARRAY_SIZE(intr_en_avp), 0); + t3_write_reg(adapter, A_TP_INT_ENABLE, + adapter->params.rev >= T3_REV_C ? 0x2bfffff : 0x3bfffff); + + if (adapter->params.rev > 0) { + t3_write_reg(adapter, A_CPL_INTR_ENABLE, + CPLSW_INTR_MASK | F_CIM_OVFL_ERROR); + t3_write_reg(adapter, A_ULPTX_INT_ENABLE, + ULPTX_INTR_MASK | F_PBL_BOUND_ERR_CH0 | + F_PBL_BOUND_ERR_CH1); + } else { + t3_write_reg(adapter, A_CPL_INTR_ENABLE, CPLSW_INTR_MASK); + t3_write_reg(adapter, A_ULPTX_INT_ENABLE, ULPTX_INTR_MASK); + } + + t3_write_reg(adapter, A_T3DBG_INT_ENABLE, calc_gpio_intr(adapter)); + + if (is_pcie(adapter)) + t3_write_reg(adapter, A_PCIE_INT_ENABLE, PCIE_INTR_MASK); + else + t3_write_reg(adapter, A_PCIX_INT_ENABLE, PCIX_INTR_MASK); + t3_write_reg(adapter, A_PL_INT_ENABLE0, adapter->slow_intr_mask); + t3_read_reg(adapter, A_PL_INT_ENABLE0); /* flush */ +} + +/** + * t3_intr_disable - disable a card's interrupts + * @adapter: the adapter whose interrupts should be disabled + * + * Disable interrupts. We only disable the top-level interrupt + * concentrator and the SGE data interrupts. + */ +void t3_intr_disable(struct adapter *adapter) +{ + t3_write_reg(adapter, A_PL_INT_ENABLE0, 0); + t3_read_reg(adapter, A_PL_INT_ENABLE0); /* flush */ + adapter->slow_intr_mask = 0; +} + +/** + * t3_intr_clear - clear all interrupts + * @adapter: the adapter whose interrupts should be cleared + * + * Clears all interrupts. + */ +void t3_intr_clear(struct adapter *adapter) +{ + static const unsigned int cause_reg_addr[] = { + A_SG_INT_CAUSE, + A_SG_RSPQ_FL_STATUS, + A_PCIX_INT_CAUSE, + A_MC7_INT_CAUSE, + A_MC7_INT_CAUSE - MC7_PMRX_BASE_ADDR + MC7_PMTX_BASE_ADDR, + A_MC7_INT_CAUSE - MC7_PMRX_BASE_ADDR + MC7_CM_BASE_ADDR, + A_CIM_HOST_INT_CAUSE, + A_TP_INT_CAUSE, + A_MC5_DB_INT_CAUSE, + A_ULPRX_INT_CAUSE, + A_ULPTX_INT_CAUSE, + A_CPL_INTR_CAUSE, + A_PM1_TX_INT_CAUSE, + A_PM1_RX_INT_CAUSE, + A_MPS_INT_CAUSE, + A_T3DBG_INT_CAUSE, + }; + unsigned int i; + + /* Clear PHY and MAC interrupts for each port. */ + for_each_port(adapter, i) + t3_port_intr_clear(adapter, i); + + for (i = 0; i < ARRAY_SIZE(cause_reg_addr); ++i) + t3_write_reg(adapter, cause_reg_addr[i], 0xffffffff); + + if (is_pcie(adapter)) + t3_write_reg(adapter, A_PCIE_PEX_ERR, 0xffffffff); + t3_write_reg(adapter, A_PL_INT_CAUSE0, 0xffffffff); + t3_read_reg(adapter, A_PL_INT_CAUSE0); /* flush */ +} + +void t3_xgm_intr_enable(struct adapter *adapter, int idx) +{ + struct port_info *pi = adap2pinfo(adapter, idx); + + t3_write_reg(adapter, A_XGM_XGM_INT_ENABLE + pi->mac.offset, + XGM_EXTRA_INTR_MASK); +} + +void t3_xgm_intr_disable(struct adapter *adapter, int idx) +{ + struct port_info *pi = adap2pinfo(adapter, idx); + + t3_write_reg(adapter, A_XGM_XGM_INT_DISABLE + pi->mac.offset, + 0x7ff); +} + +/** + * t3_port_intr_enable - enable port-specific interrupts + * @adapter: associated adapter + * @idx: index of port whose interrupts should be enabled + * + * Enable port-specific (i.e., MAC and PHY) interrupts for the given + * adapter port. + */ +void t3_port_intr_enable(struct adapter *adapter, int idx) +{ + struct cphy *phy = &adap2pinfo(adapter, idx)->phy; + + t3_write_reg(adapter, XGM_REG(A_XGM_INT_ENABLE, idx), XGM_INTR_MASK); + t3_read_reg(adapter, XGM_REG(A_XGM_INT_ENABLE, idx)); /* flush */ + phy->ops->intr_enable(phy); +} + +/** + * t3_port_intr_disable - disable port-specific interrupts + * @adapter: associated adapter + * @idx: index of port whose interrupts should be disabled + * + * Disable port-specific (i.e., MAC and PHY) interrupts for the given + * adapter port. + */ +void t3_port_intr_disable(struct adapter *adapter, int idx) +{ + struct cphy *phy = &adap2pinfo(adapter, idx)->phy; + + t3_write_reg(adapter, XGM_REG(A_XGM_INT_ENABLE, idx), 0); + t3_read_reg(adapter, XGM_REG(A_XGM_INT_ENABLE, idx)); /* flush */ + phy->ops->intr_disable(phy); +} + +/** + * t3_port_intr_clear - clear port-specific interrupts + * @adapter: associated adapter + * @idx: index of port whose interrupts to clear + * + * Clear port-specific (i.e., MAC and PHY) interrupts for the given + * adapter port. + */ +static void t3_port_intr_clear(struct adapter *adapter, int idx) +{ + struct cphy *phy = &adap2pinfo(adapter, idx)->phy; + + t3_write_reg(adapter, XGM_REG(A_XGM_INT_CAUSE, idx), 0xffffffff); + t3_read_reg(adapter, XGM_REG(A_XGM_INT_CAUSE, idx)); /* flush */ + phy->ops->intr_clear(phy); +} + +#define SG_CONTEXT_CMD_ATTEMPTS 100 + +/** + * t3_sge_write_context - write an SGE context + * @adapter: the adapter + * @id: the context id + * @type: the context type + * + * Program an SGE context with the values already loaded in the + * CONTEXT_DATA? registers. + */ +static int t3_sge_write_context(struct adapter *adapter, unsigned int id, + unsigned int type) +{ + if (type == F_RESPONSEQ) { + /* + * Can't write the Response Queue Context bits for + * Interrupt Armed or the Reserve bits after the chip + * has been initialized out of reset. Writing to these + * bits can confuse the hardware. + */ + t3_write_reg(adapter, A_SG_CONTEXT_MASK0, 0xffffffff); + t3_write_reg(adapter, A_SG_CONTEXT_MASK1, 0xffffffff); + t3_write_reg(adapter, A_SG_CONTEXT_MASK2, 0x17ffffff); + t3_write_reg(adapter, A_SG_CONTEXT_MASK3, 0xffffffff); + } else { + t3_write_reg(adapter, A_SG_CONTEXT_MASK0, 0xffffffff); + t3_write_reg(adapter, A_SG_CONTEXT_MASK1, 0xffffffff); + t3_write_reg(adapter, A_SG_CONTEXT_MASK2, 0xffffffff); + t3_write_reg(adapter, A_SG_CONTEXT_MASK3, 0xffffffff); + } + t3_write_reg(adapter, A_SG_CONTEXT_CMD, + V_CONTEXT_CMD_OPCODE(1) | type | V_CONTEXT(id)); + return t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, + 0, SG_CONTEXT_CMD_ATTEMPTS, 1); +} + +/** + * clear_sge_ctxt - completely clear an SGE context + * @adap: the adapter + * @id: the context id + * @type: the context type + * + * Completely clear an SGE context. Used predominantly at post-reset + * initialization. Note in particular that we don't skip writing to any + * "sensitive bits" in the contexts the way that t3_sge_write_context() + * does ... + */ +static int clear_sge_ctxt(struct adapter *adap, unsigned int id, + unsigned int type) +{ + t3_write_reg(adap, A_SG_CONTEXT_DATA0, 0); + t3_write_reg(adap, A_SG_CONTEXT_DATA1, 0); + t3_write_reg(adap, A_SG_CONTEXT_DATA2, 0); + t3_write_reg(adap, A_SG_CONTEXT_DATA3, 0); + t3_write_reg(adap, A_SG_CONTEXT_MASK0, 0xffffffff); + t3_write_reg(adap, A_SG_CONTEXT_MASK1, 0xffffffff); + t3_write_reg(adap, A_SG_CONTEXT_MASK2, 0xffffffff); + t3_write_reg(adap, A_SG_CONTEXT_MASK3, 0xffffffff); + t3_write_reg(adap, A_SG_CONTEXT_CMD, + V_CONTEXT_CMD_OPCODE(1) | type | V_CONTEXT(id)); + return t3_wait_op_done(adap, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, + 0, SG_CONTEXT_CMD_ATTEMPTS, 1); +} + +/** + * t3_sge_init_ecntxt - initialize an SGE egress context + * @adapter: the adapter to configure + * @id: the context id + * @gts_enable: whether to enable GTS for the context + * @type: the egress context type + * @respq: associated response queue + * @base_addr: base address of queue + * @size: number of queue entries + * @token: uP token + * @gen: initial generation value for the context + * @cidx: consumer pointer + * + * Initialize an SGE egress context and make it ready for use. If the + * platform allows concurrent context operations, the caller is + * responsible for appropriate locking. + */ +int t3_sge_init_ecntxt(struct adapter *adapter, unsigned int id, int gts_enable, + enum sge_context_type type, int respq, u64 base_addr, + unsigned int size, unsigned int token, int gen, + unsigned int cidx) +{ + unsigned int credits = type == SGE_CNTXT_OFLD ? 0 : FW_WR_NUM; + + if (base_addr & 0xfff) /* must be 4K aligned */ + return -EINVAL; + if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) + return -EBUSY; + + base_addr >>= 12; + t3_write_reg(adapter, A_SG_CONTEXT_DATA0, V_EC_INDEX(cidx) | + V_EC_CREDITS(credits) | V_EC_GTS(gts_enable)); + t3_write_reg(adapter, A_SG_CONTEXT_DATA1, V_EC_SIZE(size) | + V_EC_BASE_LO(base_addr & 0xffff)); + base_addr >>= 16; + t3_write_reg(adapter, A_SG_CONTEXT_DATA2, base_addr); + base_addr >>= 32; + t3_write_reg(adapter, A_SG_CONTEXT_DATA3, + V_EC_BASE_HI(base_addr & 0xf) | V_EC_RESPQ(respq) | + V_EC_TYPE(type) | V_EC_GEN(gen) | V_EC_UP_TOKEN(token) | + F_EC_VALID); + return t3_sge_write_context(adapter, id, F_EGRESS); +} + +/** + * t3_sge_init_flcntxt - initialize an SGE free-buffer list context + * @adapter: the adapter to configure + * @id: the context id + * @gts_enable: whether to enable GTS for the context + * @base_addr: base address of queue + * @size: number of queue entries + * @bsize: size of each buffer for this queue + * @cong_thres: threshold to signal congestion to upstream producers + * @gen: initial generation value for the context + * @cidx: consumer pointer + * + * Initialize an SGE free list context and make it ready for use. The + * caller is responsible for ensuring only one context operation occurs + * at a time. + */ +int t3_sge_init_flcntxt(struct adapter *adapter, unsigned int id, + int gts_enable, u64 base_addr, unsigned int size, + unsigned int bsize, unsigned int cong_thres, int gen, + unsigned int cidx) +{ + if (base_addr & 0xfff) /* must be 4K aligned */ + return -EINVAL; + if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) + return -EBUSY; + + base_addr >>= 12; + t3_write_reg(adapter, A_SG_CONTEXT_DATA0, base_addr); + base_addr >>= 32; + t3_write_reg(adapter, A_SG_CONTEXT_DATA1, + V_FL_BASE_HI((u32) base_addr) | + V_FL_INDEX_LO(cidx & M_FL_INDEX_LO)); + t3_write_reg(adapter, A_SG_CONTEXT_DATA2, V_FL_SIZE(size) | + V_FL_GEN(gen) | V_FL_INDEX_HI(cidx >> 12) | + V_FL_ENTRY_SIZE_LO(bsize & M_FL_ENTRY_SIZE_LO)); + t3_write_reg(adapter, A_SG_CONTEXT_DATA3, + V_FL_ENTRY_SIZE_HI(bsize >> (32 - S_FL_ENTRY_SIZE_LO)) | + V_FL_CONG_THRES(cong_thres) | V_FL_GTS(gts_enable)); + return t3_sge_write_context(adapter, id, F_FREELIST); +} + +/** + * t3_sge_init_rspcntxt - initialize an SGE response queue context + * @adapter: the adapter to configure + * @id: the context id + * @irq_vec_idx: MSI-X interrupt vector index, 0 if no MSI-X, -1 if no IRQ + * @base_addr: base address of queue + * @size: number of queue entries + * @fl_thres: threshold for selecting the normal or jumbo free list + * @gen: initial generation value for the context + * @cidx: consumer pointer + * + * Initialize an SGE response queue context and make it ready for use. + * The caller is responsible for ensuring only one context operation + * occurs at a time. + */ +int t3_sge_init_rspcntxt(struct adapter *adapter, unsigned int id, + int irq_vec_idx, u64 base_addr, unsigned int size, + unsigned int fl_thres, int gen, unsigned int cidx) +{ + unsigned int intr = 0; + + if (base_addr & 0xfff) /* must be 4K aligned */ + return -EINVAL; + if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) + return -EBUSY; + + base_addr >>= 12; + t3_write_reg(adapter, A_SG_CONTEXT_DATA0, V_CQ_SIZE(size) | + V_CQ_INDEX(cidx)); + t3_write_reg(adapter, A_SG_CONTEXT_DATA1, base_addr); + base_addr >>= 32; + if (irq_vec_idx >= 0) + intr = V_RQ_MSI_VEC(irq_vec_idx) | F_RQ_INTR_EN; + t3_write_reg(adapter, A_SG_CONTEXT_DATA2, + V_CQ_BASE_HI((u32) base_addr) | intr | V_RQ_GEN(gen)); + t3_write_reg(adapter, A_SG_CONTEXT_DATA3, fl_thres); + return t3_sge_write_context(adapter, id, F_RESPONSEQ); +} + +/** + * t3_sge_init_cqcntxt - initialize an SGE completion queue context + * @adapter: the adapter to configure + * @id: the context id + * @base_addr: base address of queue + * @size: number of queue entries + * @rspq: response queue for async notifications + * @ovfl_mode: CQ overflow mode + * @credits: completion queue credits + * @credit_thres: the credit threshold + * + * Initialize an SGE completion queue context and make it ready for use. + * The caller is responsible for ensuring only one context operation + * occurs at a time. + */ +int t3_sge_init_cqcntxt(struct adapter *adapter, unsigned int id, u64 base_addr, + unsigned int size, int rspq, int ovfl_mode, + unsigned int credits, unsigned int credit_thres) +{ + if (base_addr & 0xfff) /* must be 4K aligned */ + return -EINVAL; + if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) + return -EBUSY; + + base_addr >>= 12; + t3_write_reg(adapter, A_SG_CONTEXT_DATA0, V_CQ_SIZE(size)); + t3_write_reg(adapter, A_SG_CONTEXT_DATA1, base_addr); + base_addr >>= 32; + t3_write_reg(adapter, A_SG_CONTEXT_DATA2, + V_CQ_BASE_HI((u32) base_addr) | V_CQ_RSPQ(rspq) | + V_CQ_GEN(1) | V_CQ_OVERFLOW_MODE(ovfl_mode) | + V_CQ_ERR(ovfl_mode)); + t3_write_reg(adapter, A_SG_CONTEXT_DATA3, V_CQ_CREDITS(credits) | + V_CQ_CREDIT_THRES(credit_thres)); + return t3_sge_write_context(adapter, id, F_CQ); +} + +/** + * t3_sge_enable_ecntxt - enable/disable an SGE egress context + * @adapter: the adapter + * @id: the egress context id + * @enable: enable (1) or disable (0) the context + * + * Enable or disable an SGE egress context. The caller is responsible for + * ensuring only one context operation occurs at a time. + */ +int t3_sge_enable_ecntxt(struct adapter *adapter, unsigned int id, int enable) +{ + if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) + return -EBUSY; + + t3_write_reg(adapter, A_SG_CONTEXT_MASK0, 0); + t3_write_reg(adapter, A_SG_CONTEXT_MASK1, 0); + t3_write_reg(adapter, A_SG_CONTEXT_MASK2, 0); + t3_write_reg(adapter, A_SG_CONTEXT_MASK3, F_EC_VALID); + t3_write_reg(adapter, A_SG_CONTEXT_DATA3, V_EC_VALID(enable)); + t3_write_reg(adapter, A_SG_CONTEXT_CMD, + V_CONTEXT_CMD_OPCODE(1) | F_EGRESS | V_CONTEXT(id)); + return t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, + 0, SG_CONTEXT_CMD_ATTEMPTS, 1); +} + +/** + * t3_sge_disable_fl - disable an SGE free-buffer list + * @adapter: the adapter + * @id: the free list context id + * + * Disable an SGE free-buffer list. The caller is responsible for + * ensuring only one context operation occurs at a time. + */ +int t3_sge_disable_fl(struct adapter *adapter, unsigned int id) +{ + if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) + return -EBUSY; + + t3_write_reg(adapter, A_SG_CONTEXT_MASK0, 0); + t3_write_reg(adapter, A_SG_CONTEXT_MASK1, 0); + t3_write_reg(adapter, A_SG_CONTEXT_MASK2, V_FL_SIZE(M_FL_SIZE)); + t3_write_reg(adapter, A_SG_CONTEXT_MASK3, 0); + t3_write_reg(adapter, A_SG_CONTEXT_DATA2, 0); + t3_write_reg(adapter, A_SG_CONTEXT_CMD, + V_CONTEXT_CMD_OPCODE(1) | F_FREELIST | V_CONTEXT(id)); + return t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, + 0, SG_CONTEXT_CMD_ATTEMPTS, 1); +} + +/** + * t3_sge_disable_rspcntxt - disable an SGE response queue + * @adapter: the adapter + * @id: the response queue context id + * + * Disable an SGE response queue. The caller is responsible for + * ensuring only one context operation occurs at a time. + */ +int t3_sge_disable_rspcntxt(struct adapter *adapter, unsigned int id) +{ + if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) + return -EBUSY; + + t3_write_reg(adapter, A_SG_CONTEXT_MASK0, V_CQ_SIZE(M_CQ_SIZE)); + t3_write_reg(adapter, A_SG_CONTEXT_MASK1, 0); + t3_write_reg(adapter, A_SG_CONTEXT_MASK2, 0); + t3_write_reg(adapter, A_SG_CONTEXT_MASK3, 0); + t3_write_reg(adapter, A_SG_CONTEXT_DATA0, 0); + t3_write_reg(adapter, A_SG_CONTEXT_CMD, + V_CONTEXT_CMD_OPCODE(1) | F_RESPONSEQ | V_CONTEXT(id)); + return t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, + 0, SG_CONTEXT_CMD_ATTEMPTS, 1); +} + +/** + * t3_sge_disable_cqcntxt - disable an SGE completion queue + * @adapter: the adapter + * @id: the completion queue context id + * + * Disable an SGE completion queue. The caller is responsible for + * ensuring only one context operation occurs at a time. + */ +int t3_sge_disable_cqcntxt(struct adapter *adapter, unsigned int id) +{ + if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) + return -EBUSY; + + t3_write_reg(adapter, A_SG_CONTEXT_MASK0, V_CQ_SIZE(M_CQ_SIZE)); + t3_write_reg(adapter, A_SG_CONTEXT_MASK1, 0); + t3_write_reg(adapter, A_SG_CONTEXT_MASK2, 0); + t3_write_reg(adapter, A_SG_CONTEXT_MASK3, 0); + t3_write_reg(adapter, A_SG_CONTEXT_DATA0, 0); + t3_write_reg(adapter, A_SG_CONTEXT_CMD, + V_CONTEXT_CMD_OPCODE(1) | F_CQ | V_CONTEXT(id)); + return t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, + 0, SG_CONTEXT_CMD_ATTEMPTS, 1); +} + +/** + * t3_sge_cqcntxt_op - perform an operation on a completion queue context + * @adapter: the adapter + * @id: the context id + * @op: the operation to perform + * @credits: credit value to write + * + * Perform the selected operation on an SGE completion queue context. + * The caller is responsible for ensuring only one context operation + * occurs at a time. + */ +int t3_sge_cqcntxt_op(struct adapter *adapter, unsigned int id, unsigned int op, + unsigned int credits) +{ + u32 val; + + if (t3_read_reg(adapter, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) + return -EBUSY; + + t3_write_reg(adapter, A_SG_CONTEXT_DATA0, credits << 16); + t3_write_reg(adapter, A_SG_CONTEXT_CMD, V_CONTEXT_CMD_OPCODE(op) | + V_CONTEXT(id) | F_CQ); + if (t3_wait_op_done_val(adapter, A_SG_CONTEXT_CMD, F_CONTEXT_CMD_BUSY, + 0, SG_CONTEXT_CMD_ATTEMPTS, 1, &val)) + return -EIO; + + if (op >= 2 && op < 7) { + if (adapter->params.rev > 0) + return G_CQ_INDEX(val); + + t3_write_reg(adapter, A_SG_CONTEXT_CMD, + V_CONTEXT_CMD_OPCODE(0) | F_CQ | V_CONTEXT(id)); + if (t3_wait_op_done(adapter, A_SG_CONTEXT_CMD, + F_CONTEXT_CMD_BUSY, 0, + SG_CONTEXT_CMD_ATTEMPTS, 1)) + return -EIO; + return G_CQ_INDEX(t3_read_reg(adapter, A_SG_CONTEXT_DATA0)); + } + return 0; +} + +/** + * t3_config_rss - configure Rx packet steering + * @adapter: the adapter + * @rss_config: RSS settings (written to TP_RSS_CONFIG) + * @cpus: values for the CPU lookup table (0xff terminated) + * @rspq: values for the response queue lookup table (0xffff terminated) + * + * Programs the receive packet steering logic. @cpus and @rspq provide + * the values for the CPU and response queue lookup tables. If they + * provide fewer values than the size of the tables the supplied values + * are used repeatedly until the tables are fully populated. + */ +void t3_config_rss(struct adapter *adapter, unsigned int rss_config, + const u8 * cpus, const u16 *rspq) +{ + int i, j, cpu_idx = 0, q_idx = 0; + + if (cpus) + for (i = 0; i < RSS_TABLE_SIZE; ++i) { + u32 val = i << 16; + + for (j = 0; j < 2; ++j) { + val |= (cpus[cpu_idx++] & 0x3f) << (8 * j); + if (cpus[cpu_idx] == 0xff) + cpu_idx = 0; + } + t3_write_reg(adapter, A_TP_RSS_LKP_TABLE, val); + } + + if (rspq) + for (i = 0; i < RSS_TABLE_SIZE; ++i) { + t3_write_reg(adapter, A_TP_RSS_MAP_TABLE, + (i << 16) | rspq[q_idx++]); + if (rspq[q_idx] == 0xffff) + q_idx = 0; + } + + t3_write_reg(adapter, A_TP_RSS_CONFIG, rss_config); +} + +/** + * t3_tp_set_offload_mode - put TP in NIC/offload mode + * @adap: the adapter + * @enable: 1 to select offload mode, 0 for regular NIC + * + * Switches TP to NIC/offload mode. + */ +void t3_tp_set_offload_mode(struct adapter *adap, int enable) +{ + if (is_offload(adap) || !enable) + t3_set_reg_field(adap, A_TP_IN_CONFIG, F_NICMODE, + V_NICMODE(!enable)); +} + +/** + * pm_num_pages - calculate the number of pages of the payload memory + * @mem_size: the size of the payload memory + * @pg_size: the size of each payload memory page + * + * Calculate the number of pages, each of the given size, that fit in a + * memory of the specified size, respecting the HW requirement that the + * number of pages must be a multiple of 24. + */ +static inline unsigned int pm_num_pages(unsigned int mem_size, + unsigned int pg_size) +{ + unsigned int n = mem_size / pg_size; + + return n - n % 24; +} + +#define mem_region(adap, start, size, reg) \ + t3_write_reg((adap), A_ ## reg, (start)); \ + start += size + +/** + * partition_mem - partition memory and configure TP memory settings + * @adap: the adapter + * @p: the TP parameters + * + * Partitions context and payload memory and configures TP's memory + * registers. + */ +static void partition_mem(struct adapter *adap, const struct tp_params *p) +{ + unsigned int m, pstructs, tids = t3_mc5_size(&adap->mc5); + unsigned int timers = 0, timers_shift = 22; + + if (adap->params.rev > 0) { + if (tids <= 16 * 1024) { + timers = 1; + timers_shift = 16; + } else if (tids <= 64 * 1024) { + timers = 2; + timers_shift = 18; + } else if (tids <= 256 * 1024) { + timers = 3; + timers_shift = 20; + } + } + + t3_write_reg(adap, A_TP_PMM_SIZE, + p->chan_rx_size | (p->chan_tx_size >> 16)); + + t3_write_reg(adap, A_TP_PMM_TX_BASE, 0); + t3_write_reg(adap, A_TP_PMM_TX_PAGE_SIZE, p->tx_pg_size); + t3_write_reg(adap, A_TP_PMM_TX_MAX_PAGE, p->tx_num_pgs); + t3_set_reg_field(adap, A_TP_PARA_REG3, V_TXDATAACKIDX(M_TXDATAACKIDX), + V_TXDATAACKIDX(fls(p->tx_pg_size) - 12)); + + t3_write_reg(adap, A_TP_PMM_RX_BASE, 0); + t3_write_reg(adap, A_TP_PMM_RX_PAGE_SIZE, p->rx_pg_size); + t3_write_reg(adap, A_TP_PMM_RX_MAX_PAGE, p->rx_num_pgs); + + pstructs = p->rx_num_pgs + p->tx_num_pgs; + /* Add a bit of headroom and make multiple of 24 */ + pstructs += 48; + pstructs -= pstructs % 24; + t3_write_reg(adap, A_TP_CMM_MM_MAX_PSTRUCT, pstructs); + + m = tids * TCB_SIZE; + mem_region(adap, m, (64 << 10) * 64, SG_EGR_CNTX_BADDR); + mem_region(adap, m, (64 << 10) * 64, SG_CQ_CONTEXT_BADDR); + t3_write_reg(adap, A_TP_CMM_TIMER_BASE, V_CMTIMERMAXNUM(timers) | m); + m += ((p->ntimer_qs - 1) << timers_shift) + (1 << 22); + mem_region(adap, m, pstructs * 64, TP_CMM_MM_BASE); + mem_region(adap, m, 64 * (pstructs / 24), TP_CMM_MM_PS_FLST_BASE); + mem_region(adap, m, 64 * (p->rx_num_pgs / 24), TP_CMM_MM_RX_FLST_BASE); + mem_region(adap, m, 64 * (p->tx_num_pgs / 24), TP_CMM_MM_TX_FLST_BASE); + + m = (m + 4095) & ~0xfff; + t3_write_reg(adap, A_CIM_SDRAM_BASE_ADDR, m); + t3_write_reg(adap, A_CIM_SDRAM_ADDR_SIZE, p->cm_size - m); + + tids = (p->cm_size - m - (3 << 20)) / 3072 - 32; + m = t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers - + adap->params.mc5.nfilters - adap->params.mc5.nroutes; + if (tids < m) + adap->params.mc5.nservers += m - tids; +} + +static inline void tp_wr_indirect(struct adapter *adap, unsigned int addr, + u32 val) +{ + t3_write_reg(adap, A_TP_PIO_ADDR, addr); + t3_write_reg(adap, A_TP_PIO_DATA, val); +} + +static void tp_config(struct adapter *adap, const struct tp_params *p) +{ + t3_write_reg(adap, A_TP_GLOBAL_CONFIG, F_TXPACINGENABLE | F_PATHMTU | + F_IPCHECKSUMOFFLOAD | F_UDPCHECKSUMOFFLOAD | + F_TCPCHECKSUMOFFLOAD | V_IPTTL(64)); + t3_write_reg(adap, A_TP_TCP_OPTIONS, V_MTUDEFAULT(576) | + F_MTUENABLE | V_WINDOWSCALEMODE(1) | + V_TIMESTAMPSMODE(1) | V_SACKMODE(1) | V_SACKRX(1)); + t3_write_reg(adap, A_TP_DACK_CONFIG, V_AUTOSTATE3(1) | + V_AUTOSTATE2(1) | V_AUTOSTATE1(0) | + V_BYTETHRESHOLD(26880) | V_MSSTHRESHOLD(2) | + F_AUTOCAREFUL | F_AUTOENABLE | V_DACK_MODE(1)); + t3_set_reg_field(adap, A_TP_IN_CONFIG, F_RXFBARBPRIO | F_TXFBARBPRIO, + F_IPV6ENABLE | F_NICMODE); + t3_write_reg(adap, A_TP_TX_RESOURCE_LIMIT, 0x18141814); + t3_write_reg(adap, A_TP_PARA_REG4, 0x5050105); + t3_set_reg_field(adap, A_TP_PARA_REG6, 0, + adap->params.rev > 0 ? F_ENABLEESND : + F_T3A_ENABLEESND); + + t3_set_reg_field(adap, A_TP_PC_CONFIG, + F_ENABLEEPCMDAFULL, + F_ENABLEOCSPIFULL |F_TXDEFERENABLE | F_HEARBEATDACK | + F_TXCONGESTIONMODE | F_RXCONGESTIONMODE); + t3_set_reg_field(adap, A_TP_PC_CONFIG2, F_CHDRAFULL, + F_ENABLEIPV6RSS | F_ENABLENONOFDTNLSYN | + F_ENABLEARPMISS | F_DISBLEDAPARBIT0); + t3_write_reg(adap, A_TP_PROXY_FLOW_CNTL, 1080); + t3_write_reg(adap, A_TP_PROXY_FLOW_CNTL, 1000); + + if (adap->params.rev > 0) { + tp_wr_indirect(adap, A_TP_EGRESS_CONFIG, F_REWRITEFORCETOSIZE); + t3_set_reg_field(adap, A_TP_PARA_REG3, F_TXPACEAUTO, + F_TXPACEAUTO); + t3_set_reg_field(adap, A_TP_PC_CONFIG, F_LOCKTID, F_LOCKTID); + t3_set_reg_field(adap, A_TP_PARA_REG3, 0, F_TXPACEAUTOSTRICT); + } else + t3_set_reg_field(adap, A_TP_PARA_REG3, 0, F_TXPACEFIXED); + + if (adap->params.rev == T3_REV_C) + t3_set_reg_field(adap, A_TP_PC_CONFIG, + V_TABLELATENCYDELTA(M_TABLELATENCYDELTA), + V_TABLELATENCYDELTA(4)); + + t3_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT1, 0); + t3_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0, 0); + t3_write_reg(adap, A_TP_MOD_CHANNEL_WEIGHT, 0); + t3_write_reg(adap, A_TP_MOD_RATE_LIMIT, 0xf2200000); +} + +/* Desired TP timer resolution in usec */ +#define TP_TMR_RES 50 + +/* TCP timer values in ms */ +#define TP_DACK_TIMER 50 +#define TP_RTO_MIN 250 + +/** + * tp_set_timers - set TP timing parameters + * @adap: the adapter to set + * @core_clk: the core clock frequency in Hz + * + * Set TP's timing parameters, such as the various timer resolutions and + * the TCP timer values. + */ +static void tp_set_timers(struct adapter *adap, unsigned int core_clk) +{ + unsigned int tre = fls(core_clk / (1000000 / TP_TMR_RES)) - 1; + unsigned int dack_re = fls(core_clk / 5000) - 1; /* 200us */ + unsigned int tstamp_re = fls(core_clk / 1000); /* 1ms, at least */ + unsigned int tps = core_clk >> tre; + + t3_write_reg(adap, A_TP_TIMER_RESOLUTION, V_TIMERRESOLUTION(tre) | + V_DELAYEDACKRESOLUTION(dack_re) | + V_TIMESTAMPRESOLUTION(tstamp_re)); + t3_write_reg(adap, A_TP_DACK_TIMER, + (core_clk >> dack_re) / (1000 / TP_DACK_TIMER)); + t3_write_reg(adap, A_TP_TCP_BACKOFF_REG0, 0x3020100); + t3_write_reg(adap, A_TP_TCP_BACKOFF_REG1, 0x7060504); + t3_write_reg(adap, A_TP_TCP_BACKOFF_REG2, 0xb0a0908); + t3_write_reg(adap, A_TP_TCP_BACKOFF_REG3, 0xf0e0d0c); + t3_write_reg(adap, A_TP_SHIFT_CNT, V_SYNSHIFTMAX(6) | + V_RXTSHIFTMAXR1(4) | V_RXTSHIFTMAXR2(15) | + V_PERSHIFTBACKOFFMAX(8) | V_PERSHIFTMAX(8) | + V_KEEPALIVEMAX(9)); + +#define SECONDS * tps + + t3_write_reg(adap, A_TP_MSL, adap->params.rev > 0 ? 0 : 2 SECONDS); + t3_write_reg(adap, A_TP_RXT_MIN, tps / (1000 / TP_RTO_MIN)); + t3_write_reg(adap, A_TP_RXT_MAX, 64 SECONDS); + t3_write_reg(adap, A_TP_PERS_MIN, 5 SECONDS); + t3_write_reg(adap, A_TP_PERS_MAX, 64 SECONDS); + t3_write_reg(adap, A_TP_KEEP_IDLE, 7200 SECONDS); + t3_write_reg(adap, A_TP_KEEP_INTVL, 75 SECONDS); + t3_write_reg(adap, A_TP_INIT_SRTT, 3 SECONDS); + t3_write_reg(adap, A_TP_FINWAIT2_TIMER, 600 SECONDS); + +#undef SECONDS +} + +/** + * t3_tp_set_coalescing_size - set receive coalescing size + * @adap: the adapter + * @size: the receive coalescing size + * @psh: whether a set PSH bit should deliver coalesced data + * + * Set the receive coalescing size and PSH bit handling. + */ +static int t3_tp_set_coalescing_size(struct adapter *adap, + unsigned int size, int psh) +{ + u32 val; + + if (size > MAX_RX_COALESCING_LEN) + return -EINVAL; + + val = t3_read_reg(adap, A_TP_PARA_REG3); + val &= ~(F_RXCOALESCEENABLE | F_RXCOALESCEPSHEN); + + if (size) { + val |= F_RXCOALESCEENABLE; + if (psh) + val |= F_RXCOALESCEPSHEN; + size = min(MAX_RX_COALESCING_LEN, size); + t3_write_reg(adap, A_TP_PARA_REG2, V_RXCOALESCESIZE(size) | + V_MAXRXDATA(MAX_RX_COALESCING_LEN)); + } + t3_write_reg(adap, A_TP_PARA_REG3, val); + return 0; +} + +/** + * t3_tp_set_max_rxsize - set the max receive size + * @adap: the adapter + * @size: the max receive size + * + * Set TP's max receive size. This is the limit that applies when + * receive coalescing is disabled. + */ +static void t3_tp_set_max_rxsize(struct adapter *adap, unsigned int size) +{ + t3_write_reg(adap, A_TP_PARA_REG7, + V_PMMAXXFERLEN0(size) | V_PMMAXXFERLEN1(size)); +} + +static void init_mtus(unsigned short mtus[]) +{ + /* + * See draft-mathis-plpmtud-00.txt for the values. The min is 88 so + * it can accommodate max size TCP/IP headers when SACK and timestamps + * are enabled and still have at least 8 bytes of payload. + */ + mtus[0] = 88; + mtus[1] = 88; + mtus[2] = 256; + mtus[3] = 512; + mtus[4] = 576; + mtus[5] = 1024; + mtus[6] = 1280; + mtus[7] = 1492; + mtus[8] = 1500; + mtus[9] = 2002; + mtus[10] = 2048; + mtus[11] = 4096; + mtus[12] = 4352; + mtus[13] = 8192; + mtus[14] = 9000; + mtus[15] = 9600; +} + +/* + * Initial congestion control parameters. + */ +static void init_cong_ctrl(unsigned short *a, unsigned short *b) +{ + a[0] = a[1] = a[2] = a[3] = a[4] = a[5] = a[6] = a[7] = a[8] = 1; + a[9] = 2; + a[10] = 3; + a[11] = 4; + a[12] = 5; + a[13] = 6; + a[14] = 7; + a[15] = 8; + a[16] = 9; + a[17] = 10; + a[18] = 14; + a[19] = 17; + a[20] = 21; + a[21] = 25; + a[22] = 30; + a[23] = 35; + a[24] = 45; + a[25] = 60; + a[26] = 80; + a[27] = 100; + a[28] = 200; + a[29] = 300; + a[30] = 400; + a[31] = 500; + + b[0] = b[1] = b[2] = b[3] = b[4] = b[5] = b[6] = b[7] = b[8] = 0; + b[9] = b[10] = 1; + b[11] = b[12] = 2; + b[13] = b[14] = b[15] = b[16] = 3; + b[17] = b[18] = b[19] = b[20] = b[21] = 4; + b[22] = b[23] = b[24] = b[25] = b[26] = b[27] = 5; + b[28] = b[29] = 6; + b[30] = b[31] = 7; +} + +/* The minimum additive increment value for the congestion control table */ +#define CC_MIN_INCR 2U + +/** + * t3_load_mtus - write the MTU and congestion control HW tables + * @adap: the adapter + * @mtus: the unrestricted values for the MTU table + * @alpha: the values for the congestion control alpha parameter + * @beta: the values for the congestion control beta parameter + * @mtu_cap: the maximum permitted effective MTU + * + * Write the MTU table with the supplied MTUs capping each at &mtu_cap. + * Update the high-speed congestion control table with the supplied alpha, + * beta, and MTUs. + */ +void t3_load_mtus(struct adapter *adap, unsigned short mtus[NMTUS], + unsigned short alpha[NCCTRL_WIN], + unsigned short beta[NCCTRL_WIN], unsigned short mtu_cap) +{ + static const unsigned int avg_pkts[NCCTRL_WIN] = { + 2, 6, 10, 14, 20, 28, 40, 56, 80, 112, 160, 224, 320, 448, 640, + 896, 1281, 1792, 2560, 3584, 5120, 7168, 10240, 14336, 20480, + 28672, 40960, 57344, 81920, 114688, 163840, 229376 + }; + + unsigned int i, w; + + for (i = 0; i < NMTUS; ++i) { + unsigned int mtu = min(mtus[i], mtu_cap); + unsigned int log2 = fls(mtu); + + if (!(mtu & ((1 << log2) >> 2))) /* round */ + log2--; + t3_write_reg(adap, A_TP_MTU_TABLE, + (i << 24) | (log2 << 16) | mtu); + + for (w = 0; w < NCCTRL_WIN; ++w) { + unsigned int inc; + + inc = max(((mtu - 40) * alpha[w]) / avg_pkts[w], + CC_MIN_INCR); + + t3_write_reg(adap, A_TP_CCTRL_TABLE, (i << 21) | + (w << 16) | (beta[w] << 13) | inc); + } + } +} + +/** + * t3_tp_get_mib_stats - read TP's MIB counters + * @adap: the adapter + * @tps: holds the returned counter values + * + * Returns the values of TP's MIB counters. + */ +void t3_tp_get_mib_stats(struct adapter *adap, struct tp_mib_stats *tps) +{ + t3_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_RDATA, (u32 *) tps, + sizeof(*tps) / sizeof(u32), 0); +} + +#define ulp_region(adap, name, start, len) \ + t3_write_reg((adap), A_ULPRX_ ## name ## _LLIMIT, (start)); \ + t3_write_reg((adap), A_ULPRX_ ## name ## _ULIMIT, \ + (start) + (len) - 1); \ + start += len + +#define ulptx_region(adap, name, start, len) \ + t3_write_reg((adap), A_ULPTX_ ## name ## _LLIMIT, (start)); \ + t3_write_reg((adap), A_ULPTX_ ## name ## _ULIMIT, \ + (start) + (len) - 1) + +static void ulp_config(struct adapter *adap, const struct tp_params *p) +{ + unsigned int m = p->chan_rx_size; + + ulp_region(adap, ISCSI, m, p->chan_rx_size / 8); + ulp_region(adap, TDDP, m, p->chan_rx_size / 8); + ulptx_region(adap, TPT, m, p->chan_rx_size / 4); + ulp_region(adap, STAG, m, p->chan_rx_size / 4); + ulp_region(adap, RQ, m, p->chan_rx_size / 4); + ulptx_region(adap, PBL, m, p->chan_rx_size / 4); + ulp_region(adap, PBL, m, p->chan_rx_size / 4); + t3_write_reg(adap, A_ULPRX_TDDP_TAGMASK, 0xffffffff); +} + +/** + * t3_set_proto_sram - set the contents of the protocol sram + * @adap: the adapter + * @data: the protocol image + * + * Write the contents of the protocol SRAM. + */ +int t3_set_proto_sram(struct adapter *adap, const u8 *data) +{ + int i; + const __be32 *buf = (const __be32 *)data; + + for (i = 0; i < PROTO_SRAM_LINES; i++) { + t3_write_reg(adap, A_TP_EMBED_OP_FIELD5, be32_to_cpu(*buf++)); + t3_write_reg(adap, A_TP_EMBED_OP_FIELD4, be32_to_cpu(*buf++)); + t3_write_reg(adap, A_TP_EMBED_OP_FIELD3, be32_to_cpu(*buf++)); + t3_write_reg(adap, A_TP_EMBED_OP_FIELD2, be32_to_cpu(*buf++)); + t3_write_reg(adap, A_TP_EMBED_OP_FIELD1, be32_to_cpu(*buf++)); + + t3_write_reg(adap, A_TP_EMBED_OP_FIELD0, i << 1 | 1 << 31); + if (t3_wait_op_done(adap, A_TP_EMBED_OP_FIELD0, 1, 1, 5, 1)) + return -EIO; + } + t3_write_reg(adap, A_TP_EMBED_OP_FIELD0, 0); + + return 0; +} + +void t3_config_trace_filter(struct adapter *adapter, + const struct trace_params *tp, int filter_index, + int invert, int enable) +{ + u32 addr, key[4], mask[4]; + + key[0] = tp->sport | (tp->sip << 16); + key[1] = (tp->sip >> 16) | (tp->dport << 16); + key[2] = tp->dip; + key[3] = tp->proto | (tp->vlan << 8) | (tp->intf << 20); + + mask[0] = tp->sport_mask | (tp->sip_mask << 16); + mask[1] = (tp->sip_mask >> 16) | (tp->dport_mask << 16); + mask[2] = tp->dip_mask; + mask[3] = tp->proto_mask | (tp->vlan_mask << 8) | (tp->intf_mask << 20); + + if (invert) + key[3] |= (1 << 29); + if (enable) + key[3] |= (1 << 28); + + addr = filter_index ? A_TP_RX_TRC_KEY0 : A_TP_TX_TRC_KEY0; + tp_wr_indirect(adapter, addr++, key[0]); + tp_wr_indirect(adapter, addr++, mask[0]); + tp_wr_indirect(adapter, addr++, key[1]); + tp_wr_indirect(adapter, addr++, mask[1]); + tp_wr_indirect(adapter, addr++, key[2]); + tp_wr_indirect(adapter, addr++, mask[2]); + tp_wr_indirect(adapter, addr++, key[3]); + tp_wr_indirect(adapter, addr, mask[3]); + t3_read_reg(adapter, A_TP_PIO_DATA); +} + +/** + * t3_config_sched - configure a HW traffic scheduler + * @adap: the adapter + * @kbps: target rate in Kbps + * @sched: the scheduler index + * + * Configure a HW scheduler for the target rate + */ +int t3_config_sched(struct adapter *adap, unsigned int kbps, int sched) +{ + unsigned int v, tps, cpt, bpt, delta, mindelta = ~0; + unsigned int clk = adap->params.vpd.cclk * 1000; + unsigned int selected_cpt = 0, selected_bpt = 0; + + if (kbps > 0) { + kbps *= 125; /* -> bytes */ + for (cpt = 1; cpt <= 255; cpt++) { + tps = clk / cpt; + bpt = (kbps + tps / 2) / tps; + if (bpt > 0 && bpt <= 255) { + v = bpt * tps; + delta = v >= kbps ? v - kbps : kbps - v; + if (delta <= mindelta) { + mindelta = delta; + selected_cpt = cpt; + selected_bpt = bpt; + } + } else if (selected_cpt) + break; + } + if (!selected_cpt) + return -EINVAL; + } + t3_write_reg(adap, A_TP_TM_PIO_ADDR, + A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2); + v = t3_read_reg(adap, A_TP_TM_PIO_DATA); + if (sched & 1) + v = (v & 0xffff) | (selected_cpt << 16) | (selected_bpt << 24); + else + v = (v & 0xffff0000) | selected_cpt | (selected_bpt << 8); + t3_write_reg(adap, A_TP_TM_PIO_DATA, v); + return 0; +} + +static int tp_init(struct adapter *adap, const struct tp_params *p) +{ + int busy = 0; + + tp_config(adap, p); + t3_set_vlan_accel(adap, 3, 0); + + if (is_offload(adap)) { + tp_set_timers(adap, adap->params.vpd.cclk * 1000); + t3_write_reg(adap, A_TP_RESET, F_FLSTINITENABLE); + busy = t3_wait_op_done(adap, A_TP_RESET, F_FLSTINITENABLE, + 0, 1000, 5); + if (busy) + CH_ERR(adap, "TP initialization timed out\n"); + } + + if (!busy) + t3_write_reg(adap, A_TP_RESET, F_TPRESET); + return busy; +} + +/* + * Perform the bits of HW initialization that are dependent on the Tx + * channels being used. + */ +static void chan_init_hw(struct adapter *adap, unsigned int chan_map) +{ + int i; + + if (chan_map != 3) { /* one channel */ + t3_set_reg_field(adap, A_ULPRX_CTL, F_ROUND_ROBIN, 0); + t3_set_reg_field(adap, A_ULPTX_CONFIG, F_CFG_RR_ARB, 0); + t3_write_reg(adap, A_MPS_CFG, F_TPRXPORTEN | F_ENFORCEPKT | + (chan_map == 1 ? F_TPTXPORT0EN | F_PORT0ACTIVE : + F_TPTXPORT1EN | F_PORT1ACTIVE)); + t3_write_reg(adap, A_PM1_TX_CFG, + chan_map == 1 ? 0xffffffff : 0); + } else { /* two channels */ + t3_set_reg_field(adap, A_ULPRX_CTL, 0, F_ROUND_ROBIN); + t3_set_reg_field(adap, A_ULPTX_CONFIG, 0, F_CFG_RR_ARB); + t3_write_reg(adap, A_ULPTX_DMA_WEIGHT, + V_D1_WEIGHT(16) | V_D0_WEIGHT(16)); + t3_write_reg(adap, A_MPS_CFG, F_TPTXPORT0EN | F_TPTXPORT1EN | + F_TPRXPORTEN | F_PORT0ACTIVE | F_PORT1ACTIVE | + F_ENFORCEPKT); + t3_write_reg(adap, A_PM1_TX_CFG, 0x80008000); + t3_set_reg_field(adap, A_TP_PC_CONFIG, 0, F_TXTOSQUEUEMAPMODE); + t3_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP, + V_TX_MOD_QUEUE_REQ_MAP(0xaa)); + for (i = 0; i < 16; i++) + t3_write_reg(adap, A_TP_TX_MOD_QUE_TABLE, + (i << 16) | 0x1010); + } +} + +static int calibrate_xgm(struct adapter *adapter) +{ + if (uses_xaui(adapter)) { + unsigned int v, i; + + for (i = 0; i < 5; ++i) { + t3_write_reg(adapter, A_XGM_XAUI_IMP, 0); + t3_read_reg(adapter, A_XGM_XAUI_IMP); + msleep(1); + v = t3_read_reg(adapter, A_XGM_XAUI_IMP); + if (!(v & (F_XGM_CALFAULT | F_CALBUSY))) { + t3_write_reg(adapter, A_XGM_XAUI_IMP, + V_XAUIIMP(G_CALIMP(v) >> 2)); + return 0; + } + } + CH_ERR(adapter, "MAC calibration failed\n"); + return -1; + } else { + t3_write_reg(adapter, A_XGM_RGMII_IMP, + V_RGMIIIMPPD(2) | V_RGMIIIMPPU(3)); + t3_set_reg_field(adapter, A_XGM_RGMII_IMP, F_XGM_IMPSETUPDATE, + F_XGM_IMPSETUPDATE); + } + return 0; +} + +static void calibrate_xgm_t3b(struct adapter *adapter) +{ + if (!uses_xaui(adapter)) { + t3_write_reg(adapter, A_XGM_RGMII_IMP, F_CALRESET | + F_CALUPDATE | V_RGMIIIMPPD(2) | V_RGMIIIMPPU(3)); + t3_set_reg_field(adapter, A_XGM_RGMII_IMP, F_CALRESET, 0); + t3_set_reg_field(adapter, A_XGM_RGMII_IMP, 0, + F_XGM_IMPSETUPDATE); + t3_set_reg_field(adapter, A_XGM_RGMII_IMP, F_XGM_IMPSETUPDATE, + 0); + t3_set_reg_field(adapter, A_XGM_RGMII_IMP, F_CALUPDATE, 0); + t3_set_reg_field(adapter, A_XGM_RGMII_IMP, 0, F_CALUPDATE); + } +} + +struct mc7_timing_params { + unsigned char ActToPreDly; + unsigned char ActToRdWrDly; + unsigned char PreCyc; + unsigned char RefCyc[5]; + unsigned char BkCyc; + unsigned char WrToRdDly; + unsigned char RdToWrDly; +}; + +/* + * Write a value to a register and check that the write completed. These + * writes normally complete in a cycle or two, so one read should suffice. + * The very first read exists to flush the posted write to the device. + */ +static int wrreg_wait(struct adapter *adapter, unsigned int addr, u32 val) +{ + t3_write_reg(adapter, addr, val); + t3_read_reg(adapter, addr); /* flush */ + if (!(t3_read_reg(adapter, addr) & F_BUSY)) + return 0; + CH_ERR(adapter, "write to MC7 register 0x%x timed out\n", addr); + return -EIO; +} + +static int mc7_init(struct mc7 *mc7, unsigned int mc7_clock, int mem_type) +{ + static const unsigned int mc7_mode[] = { + 0x632, 0x642, 0x652, 0x432, 0x442 + }; + static const struct mc7_timing_params mc7_timings[] = { + {12, 3, 4, {20, 28, 34, 52, 0}, 15, 6, 4}, + {12, 4, 5, {20, 28, 34, 52, 0}, 16, 7, 4}, + {12, 5, 6, {20, 28, 34, 52, 0}, 17, 8, 4}, + {9, 3, 4, {15, 21, 26, 39, 0}, 12, 6, 4}, + {9, 4, 5, {15, 21, 26, 39, 0}, 13, 7, 4} + }; + + u32 val; + unsigned int width, density, slow, attempts; + struct adapter *adapter = mc7->adapter; + const struct mc7_timing_params *p = &mc7_timings[mem_type]; + + if (!mc7->size) + return 0; + + val = t3_read_reg(adapter, mc7->offset + A_MC7_CFG); + slow = val & F_SLOW; + width = G_WIDTH(val); + density = G_DEN(val); + + t3_write_reg(adapter, mc7->offset + A_MC7_CFG, val | F_IFEN); + val = t3_read_reg(adapter, mc7->offset + A_MC7_CFG); /* flush */ + msleep(1); + + if (!slow) { + t3_write_reg(adapter, mc7->offset + A_MC7_CAL, F_SGL_CAL_EN); + t3_read_reg(adapter, mc7->offset + A_MC7_CAL); + msleep(1); + if (t3_read_reg(adapter, mc7->offset + A_MC7_CAL) & + (F_BUSY | F_SGL_CAL_EN | F_CAL_FAULT)) { + CH_ERR(adapter, "%s MC7 calibration timed out\n", + mc7->name); + goto out_fail; + } + } + + t3_write_reg(adapter, mc7->offset + A_MC7_PARM, + V_ACTTOPREDLY(p->ActToPreDly) | + V_ACTTORDWRDLY(p->ActToRdWrDly) | V_PRECYC(p->PreCyc) | + V_REFCYC(p->RefCyc[density]) | V_BKCYC(p->BkCyc) | + V_WRTORDDLY(p->WrToRdDly) | V_RDTOWRDLY(p->RdToWrDly)); + + t3_write_reg(adapter, mc7->offset + A_MC7_CFG, + val | F_CLKEN | F_TERM150); + t3_read_reg(adapter, mc7->offset + A_MC7_CFG); /* flush */ + + if (!slow) + t3_set_reg_field(adapter, mc7->offset + A_MC7_DLL, F_DLLENB, + F_DLLENB); + udelay(1); + + val = slow ? 3 : 6; + if (wrreg_wait(adapter, mc7->offset + A_MC7_PRE, 0) || + wrreg_wait(adapter, mc7->offset + A_MC7_EXT_MODE2, 0) || + wrreg_wait(adapter, mc7->offset + A_MC7_EXT_MODE3, 0) || + wrreg_wait(adapter, mc7->offset + A_MC7_EXT_MODE1, val)) + goto out_fail; + + if (!slow) { + t3_write_reg(adapter, mc7->offset + A_MC7_MODE, 0x100); + t3_set_reg_field(adapter, mc7->offset + A_MC7_DLL, F_DLLRST, 0); + udelay(5); + } + + if (wrreg_wait(adapter, mc7->offset + A_MC7_PRE, 0) || + wrreg_wait(adapter, mc7->offset + A_MC7_REF, 0) || + wrreg_wait(adapter, mc7->offset + A_MC7_REF, 0) || + wrreg_wait(adapter, mc7->offset + A_MC7_MODE, + mc7_mode[mem_type]) || + wrreg_wait(adapter, mc7->offset + A_MC7_EXT_MODE1, val | 0x380) || + wrreg_wait(adapter, mc7->offset + A_MC7_EXT_MODE1, val)) + goto out_fail; + + /* clock value is in KHz */ + mc7_clock = mc7_clock * 7812 + mc7_clock / 2; /* ns */ + mc7_clock /= 1000000; /* KHz->MHz, ns->us */ + + t3_write_reg(adapter, mc7->offset + A_MC7_REF, + F_PERREFEN | V_PREREFDIV(mc7_clock)); + t3_read_reg(adapter, mc7->offset + A_MC7_REF); /* flush */ + + t3_write_reg(adapter, mc7->offset + A_MC7_ECC, F_ECCGENEN | F_ECCCHKEN); + t3_write_reg(adapter, mc7->offset + A_MC7_BIST_DATA, 0); + t3_write_reg(adapter, mc7->offset + A_MC7_BIST_ADDR_BEG, 0); + t3_write_reg(adapter, mc7->offset + A_MC7_BIST_ADDR_END, + (mc7->size << width) - 1); + t3_write_reg(adapter, mc7->offset + A_MC7_BIST_OP, V_OP(1)); + t3_read_reg(adapter, mc7->offset + A_MC7_BIST_OP); /* flush */ + + attempts = 50; + do { + msleep(250); + val = t3_read_reg(adapter, mc7->offset + A_MC7_BIST_OP); + } while ((val & F_BUSY) && --attempts); + if (val & F_BUSY) { + CH_ERR(adapter, "%s MC7 BIST timed out\n", mc7->name); + goto out_fail; + } + + /* Enable normal memory accesses. */ + t3_set_reg_field(adapter, mc7->offset + A_MC7_CFG, 0, F_RDY); + return 0; + +out_fail: + return -1; +} + +static void config_pcie(struct adapter *adap) +{ + static const u16 ack_lat[4][6] = { + {237, 416, 559, 1071, 2095, 4143}, + {128, 217, 289, 545, 1057, 2081}, + {73, 118, 154, 282, 538, 1050}, + {67, 107, 86, 150, 278, 534} + }; + static const u16 rpl_tmr[4][6] = { + {711, 1248, 1677, 3213, 6285, 12429}, + {384, 651, 867, 1635, 3171, 6243}, + {219, 354, 462, 846, 1614, 3150}, + {201, 321, 258, 450, 834, 1602} + }; + + u16 val, devid; + unsigned int log2_width, pldsize; + unsigned int fst_trn_rx, fst_trn_tx, acklat, rpllmt; + + pcie_capability_read_word(adap->pdev, PCI_EXP_DEVCTL, &val); + pldsize = (val & PCI_EXP_DEVCTL_PAYLOAD) >> 5; + + pci_read_config_word(adap->pdev, 0x2, &devid); + if (devid == 0x37) { + pcie_capability_write_word(adap->pdev, PCI_EXP_DEVCTL, + val & ~PCI_EXP_DEVCTL_READRQ & + ~PCI_EXP_DEVCTL_PAYLOAD); + pldsize = 0; + } + + pcie_capability_read_word(adap->pdev, PCI_EXP_LNKCTL, &val); + + fst_trn_tx = G_NUMFSTTRNSEQ(t3_read_reg(adap, A_PCIE_PEX_CTRL0)); + fst_trn_rx = adap->params.rev == 0 ? fst_trn_tx : + G_NUMFSTTRNSEQRX(t3_read_reg(adap, A_PCIE_MODE)); + log2_width = fls(adap->params.pci.width) - 1; + acklat = ack_lat[log2_width][pldsize]; + if (val & PCI_EXP_LNKCTL_ASPM_L0S) /* check LOsEnable */ + acklat += fst_trn_tx * 4; + rpllmt = rpl_tmr[log2_width][pldsize] + fst_trn_rx * 4; + + if (adap->params.rev == 0) + t3_set_reg_field(adap, A_PCIE_PEX_CTRL1, + V_T3A_ACKLAT(M_T3A_ACKLAT), + V_T3A_ACKLAT(acklat)); + else + t3_set_reg_field(adap, A_PCIE_PEX_CTRL1, V_ACKLAT(M_ACKLAT), + V_ACKLAT(acklat)); + + t3_set_reg_field(adap, A_PCIE_PEX_CTRL0, V_REPLAYLMT(M_REPLAYLMT), + V_REPLAYLMT(rpllmt)); + + t3_write_reg(adap, A_PCIE_PEX_ERR, 0xffffffff); + t3_set_reg_field(adap, A_PCIE_CFG, 0, + F_ENABLELINKDWNDRST | F_ENABLELINKDOWNRST | + F_PCIE_DMASTOPEN | F_PCIE_CLIDECEN); +} + +/* + * Initialize and configure T3 HW modules. This performs the + * initialization steps that need to be done once after a card is reset. + * MAC and PHY initialization is handled separarely whenever a port is enabled. + * + * fw_params are passed to FW and their value is platform dependent. Only the + * top 8 bits are available for use, the rest must be 0. + */ +int t3_init_hw(struct adapter *adapter, u32 fw_params) +{ + int err = -EIO, attempts, i; + const struct vpd_params *vpd = &adapter->params.vpd; + + if (adapter->params.rev > 0) + calibrate_xgm_t3b(adapter); + else if (calibrate_xgm(adapter)) + goto out_err; + + if (vpd->mclk) { + partition_mem(adapter, &adapter->params.tp); + + if (mc7_init(&adapter->pmrx, vpd->mclk, vpd->mem_timing) || + mc7_init(&adapter->pmtx, vpd->mclk, vpd->mem_timing) || + mc7_init(&adapter->cm, vpd->mclk, vpd->mem_timing) || + t3_mc5_init(&adapter->mc5, adapter->params.mc5.nservers, + adapter->params.mc5.nfilters, + adapter->params.mc5.nroutes)) + goto out_err; + + for (i = 0; i < 32; i++) + if (clear_sge_ctxt(adapter, i, F_CQ)) + goto out_err; + } + + if (tp_init(adapter, &adapter->params.tp)) + goto out_err; + + t3_tp_set_coalescing_size(adapter, + min(adapter->params.sge.max_pkt_size, + MAX_RX_COALESCING_LEN), 1); + t3_tp_set_max_rxsize(adapter, + min(adapter->params.sge.max_pkt_size, 16384U)); + ulp_config(adapter, &adapter->params.tp); + + if (is_pcie(adapter)) + config_pcie(adapter); + else + t3_set_reg_field(adapter, A_PCIX_CFG, 0, + F_DMASTOPEN | F_CLIDECEN); + + if (adapter->params.rev == T3_REV_C) + t3_set_reg_field(adapter, A_ULPTX_CONFIG, 0, + F_CFG_CQE_SOP_MASK); + + t3_write_reg(adapter, A_PM1_RX_CFG, 0xffffffff); + t3_write_reg(adapter, A_PM1_RX_MODE, 0); + t3_write_reg(adapter, A_PM1_TX_MODE, 0); + chan_init_hw(adapter, adapter->params.chan_map); + t3_sge_init(adapter, &adapter->params.sge); + t3_set_reg_field(adapter, A_PL_RST, 0, F_FATALPERREN); + + t3_write_reg(adapter, A_T3DBG_GPIO_ACT_LOW, calc_gpio_intr(adapter)); + + t3_write_reg(adapter, A_CIM_HOST_ACC_DATA, vpd->uclk | fw_params); + t3_write_reg(adapter, A_CIM_BOOT_CFG, + V_BOOTADDR(FW_FLASH_BOOT_ADDR >> 2)); + t3_read_reg(adapter, A_CIM_BOOT_CFG); /* flush */ + + attempts = 100; + do { /* wait for uP to initialize */ + msleep(20); + } while (t3_read_reg(adapter, A_CIM_HOST_ACC_DATA) && --attempts); + if (!attempts) { + CH_ERR(adapter, "uP initialization timed out\n"); + goto out_err; + } + + err = 0; +out_err: + return err; +} + +/** + * get_pci_mode - determine a card's PCI mode + * @adapter: the adapter + * @p: where to store the PCI settings + * + * Determines a card's PCI mode and associated parameters, such as speed + * and width. + */ +static void get_pci_mode(struct adapter *adapter, struct pci_params *p) +{ + static unsigned short speed_map[] = { 33, 66, 100, 133 }; + u32 pci_mode; + + if (pci_is_pcie(adapter->pdev)) { + u16 val; + + p->variant = PCI_VARIANT_PCIE; + pcie_capability_read_word(adapter->pdev, PCI_EXP_LNKSTA, &val); + p->width = (val >> 4) & 0x3f; + return; + } + + pci_mode = t3_read_reg(adapter, A_PCIX_MODE); + p->speed = speed_map[G_PCLKRANGE(pci_mode)]; + p->width = (pci_mode & F_64BIT) ? 64 : 32; + pci_mode = G_PCIXINITPAT(pci_mode); + if (pci_mode == 0) + p->variant = PCI_VARIANT_PCI; + else if (pci_mode < 4) + p->variant = PCI_VARIANT_PCIX_MODE1_PARITY; + else if (pci_mode < 8) + p->variant = PCI_VARIANT_PCIX_MODE1_ECC; + else + p->variant = PCI_VARIANT_PCIX_266_MODE2; +} + +/** + * init_link_config - initialize a link's SW state + * @lc: structure holding the link state + * @caps: information about the current card + * + * Initializes the SW state maintained for each link, including the link's + * capabilities and default speed/duplex/flow-control/autonegotiation + * settings. + */ +static void init_link_config(struct link_config *lc, unsigned int caps) +{ + lc->supported = caps; + lc->requested_speed = lc->speed = SPEED_INVALID; + lc->requested_duplex = lc->duplex = DUPLEX_INVALID; + lc->requested_fc = lc->fc = PAUSE_RX | PAUSE_TX; + if (lc->supported & SUPPORTED_Autoneg) { + lc->advertising = lc->supported; + lc->autoneg = AUTONEG_ENABLE; + lc->requested_fc |= PAUSE_AUTONEG; + } else { + lc->advertising = 0; + lc->autoneg = AUTONEG_DISABLE; + } +} + +/** + * mc7_calc_size - calculate MC7 memory size + * @cfg: the MC7 configuration + * + * Calculates the size of an MC7 memory in bytes from the value of its + * configuration register. + */ +static unsigned int mc7_calc_size(u32 cfg) +{ + unsigned int width = G_WIDTH(cfg); + unsigned int banks = !!(cfg & F_BKS) + 1; + unsigned int org = !!(cfg & F_ORG) + 1; + unsigned int density = G_DEN(cfg); + unsigned int MBs = ((256 << density) * banks) / (org << width); + + return MBs << 20; +} + +static void mc7_prep(struct adapter *adapter, struct mc7 *mc7, + unsigned int base_addr, const char *name) +{ + u32 cfg; + + mc7->adapter = adapter; + mc7->name = name; + mc7->offset = base_addr - MC7_PMRX_BASE_ADDR; + cfg = t3_read_reg(adapter, mc7->offset + A_MC7_CFG); + mc7->size = G_DEN(cfg) == M_DEN ? 0 : mc7_calc_size(cfg); + mc7->width = G_WIDTH(cfg); +} + +static void mac_prep(struct cmac *mac, struct adapter *adapter, int index) +{ + u16 devid; + + mac->adapter = adapter; + pci_read_config_word(adapter->pdev, 0x2, &devid); + + if (devid == 0x37 && !adapter->params.vpd.xauicfg[1]) + index = 0; + mac->offset = (XGMAC0_1_BASE_ADDR - XGMAC0_0_BASE_ADDR) * index; + mac->nucast = 1; + + if (adapter->params.rev == 0 && uses_xaui(adapter)) { + t3_write_reg(adapter, A_XGM_SERDES_CTRL + mac->offset, + is_10G(adapter) ? 0x2901c04 : 0x2301c04); + t3_set_reg_field(adapter, A_XGM_PORT_CFG + mac->offset, + F_ENRGMII, 0); + } +} + +static void early_hw_init(struct adapter *adapter, + const struct adapter_info *ai) +{ + u32 val = V_PORTSPEED(is_10G(adapter) ? 3 : 2); + + mi1_init(adapter, ai); + t3_write_reg(adapter, A_I2C_CFG, /* set for 80KHz */ + V_I2C_CLKDIV(adapter->params.vpd.cclk / 80 - 1)); + t3_write_reg(adapter, A_T3DBG_GPIO_EN, + ai->gpio_out | F_GPIO0_OEN | F_GPIO0_OUT_VAL); + t3_write_reg(adapter, A_MC5_DB_SERVER_INDEX, 0); + t3_write_reg(adapter, A_SG_OCO_BASE, V_BASE1(0xfff)); + + if (adapter->params.rev == 0 || !uses_xaui(adapter)) + val |= F_ENRGMII; + + /* Enable MAC clocks so we can access the registers */ + t3_write_reg(adapter, A_XGM_PORT_CFG, val); + t3_read_reg(adapter, A_XGM_PORT_CFG); + + val |= F_CLKDIVRESET_; + t3_write_reg(adapter, A_XGM_PORT_CFG, val); + t3_read_reg(adapter, A_XGM_PORT_CFG); + t3_write_reg(adapter, XGM_REG(A_XGM_PORT_CFG, 1), val); + t3_read_reg(adapter, A_XGM_PORT_CFG); +} + +/* + * Reset the adapter. + * Older PCIe cards lose their config space during reset, PCI-X + * ones don't. + */ +int t3_reset_adapter(struct adapter *adapter) +{ + int i, save_and_restore_pcie = + adapter->params.rev < T3_REV_B2 && is_pcie(adapter); + uint16_t devid = 0; + + if (save_and_restore_pcie) + pci_save_state(adapter->pdev); + t3_write_reg(adapter, A_PL_RST, F_CRSTWRM | F_CRSTWRMMODE); + + /* + * Delay. Give Some time to device to reset fully. + * XXX The delay time should be modified. + */ + for (i = 0; i < 10; i++) { + msleep(50); + pci_read_config_word(adapter->pdev, 0x00, &devid); + if (devid == 0x1425) + break; + } + + if (devid != 0x1425) + return -1; + + if (save_and_restore_pcie) + pci_restore_state(adapter->pdev); + return 0; +} + +static int init_parity(struct adapter *adap) +{ + int i, err, addr; + + if (t3_read_reg(adap, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY) + return -EBUSY; + + for (err = i = 0; !err && i < 16; i++) + err = clear_sge_ctxt(adap, i, F_EGRESS); + for (i = 0xfff0; !err && i <= 0xffff; i++) + err = clear_sge_ctxt(adap, i, F_EGRESS); + for (i = 0; !err && i < SGE_QSETS; i++) + err = clear_sge_ctxt(adap, i, F_RESPONSEQ); + if (err) + return err; + + t3_write_reg(adap, A_CIM_IBQ_DBG_DATA, 0); + for (i = 0; i < 4; i++) + for (addr = 0; addr <= M_IBQDBGADDR; addr++) { + t3_write_reg(adap, A_CIM_IBQ_DBG_CFG, F_IBQDBGEN | + F_IBQDBGWR | V_IBQDBGQID(i) | + V_IBQDBGADDR(addr)); + err = t3_wait_op_done(adap, A_CIM_IBQ_DBG_CFG, + F_IBQDBGBUSY, 0, 2, 1); + if (err) + return err; + } + return 0; +} + +/* + * Initialize adapter SW state for the various HW modules, set initial values + * for some adapter tunables, take PHYs out of reset, and initialize the MDIO + * interface. + */ +int t3_prep_adapter(struct adapter *adapter, const struct adapter_info *ai, + int reset) +{ + int ret; + unsigned int i, j = -1; + + get_pci_mode(adapter, &adapter->params.pci); + + adapter->params.info = ai; + adapter->params.nports = ai->nports0 + ai->nports1; + adapter->params.chan_map = (!!ai->nports0) | (!!ai->nports1 << 1); + adapter->params.rev = t3_read_reg(adapter, A_PL_REV); + /* + * We used to only run the "adapter check task" once a second if + * we had PHYs which didn't support interrupts (we would check + * their link status once a second). Now we check other conditions + * in that routine which could potentially impose a very high + * interrupt load on the system. As such, we now always scan the + * adapter state once a second ... + */ + adapter->params.linkpoll_period = 10; + adapter->params.stats_update_period = is_10G(adapter) ? + MAC_STATS_ACCUM_SECS : (MAC_STATS_ACCUM_SECS * 10); + adapter->params.pci.vpd_cap_addr = + pci_find_capability(adapter->pdev, PCI_CAP_ID_VPD); + if (!adapter->params.pci.vpd_cap_addr) + return -ENODEV; + ret = get_vpd_params(adapter, &adapter->params.vpd); + if (ret < 0) + return ret; + + if (reset && t3_reset_adapter(adapter)) + return -1; + + t3_sge_prep(adapter, &adapter->params.sge); + + if (adapter->params.vpd.mclk) { + struct tp_params *p = &adapter->params.tp; + + mc7_prep(adapter, &adapter->pmrx, MC7_PMRX_BASE_ADDR, "PMRX"); + mc7_prep(adapter, &adapter->pmtx, MC7_PMTX_BASE_ADDR, "PMTX"); + mc7_prep(adapter, &adapter->cm, MC7_CM_BASE_ADDR, "CM"); + + p->nchan = adapter->params.chan_map == 3 ? 2 : 1; + p->pmrx_size = t3_mc7_size(&adapter->pmrx); + p->pmtx_size = t3_mc7_size(&adapter->pmtx); + p->cm_size = t3_mc7_size(&adapter->cm); + p->chan_rx_size = p->pmrx_size / 2; /* only 1 Rx channel */ + p->chan_tx_size = p->pmtx_size / p->nchan; + p->rx_pg_size = 64 * 1024; + p->tx_pg_size = is_10G(adapter) ? 64 * 1024 : 16 * 1024; + p->rx_num_pgs = pm_num_pages(p->chan_rx_size, p->rx_pg_size); + p->tx_num_pgs = pm_num_pages(p->chan_tx_size, p->tx_pg_size); + p->ntimer_qs = p->cm_size >= (128 << 20) || + adapter->params.rev > 0 ? 12 : 6; + } + + adapter->params.offload = t3_mc7_size(&adapter->pmrx) && + t3_mc7_size(&adapter->pmtx) && + t3_mc7_size(&adapter->cm); + + if (is_offload(adapter)) { + adapter->params.mc5.nservers = DEFAULT_NSERVERS; + adapter->params.mc5.nfilters = adapter->params.rev > 0 ? + DEFAULT_NFILTERS : 0; + adapter->params.mc5.nroutes = 0; + t3_mc5_prep(adapter, &adapter->mc5, MC5_MODE_144_BIT); + + init_mtus(adapter->params.mtus); + init_cong_ctrl(adapter->params.a_wnd, adapter->params.b_wnd); + } + + early_hw_init(adapter, ai); + ret = init_parity(adapter); + if (ret) + return ret; + + for_each_port(adapter, i) { + u8 hw_addr[6]; + const struct port_type_info *pti; + struct port_info *p = adap2pinfo(adapter, i); + + while (!adapter->params.vpd.port_type[++j]) + ; + + pti = &port_types[adapter->params.vpd.port_type[j]]; + if (!pti->phy_prep) { + CH_ALERT(adapter, "Invalid port type index %d\n", + adapter->params.vpd.port_type[j]); + return -EINVAL; + } + + p->phy.mdio.dev = adapter->port[i]; + ret = pti->phy_prep(&p->phy, adapter, ai->phy_base_addr + j, + ai->mdio_ops); + if (ret) + return ret; + mac_prep(&p->mac, adapter, j); + + /* + * The VPD EEPROM stores the base Ethernet address for the + * card. A port's address is derived from the base by adding + * the port's index to the base's low octet. + */ + memcpy(hw_addr, adapter->params.vpd.eth_base, 5); + hw_addr[5] = adapter->params.vpd.eth_base[5] + i; + + eth_hw_addr_set(adapter->port[i], hw_addr); + init_link_config(&p->link_config, p->phy.caps); + p->phy.ops->power_down(&p->phy, 1); + + /* + * If the PHY doesn't support interrupts for link status + * changes, schedule a scan of the adapter links at least + * once a second. + */ + if (!(p->phy.caps & SUPPORTED_IRQ) && + adapter->params.linkpoll_period > 10) + adapter->params.linkpoll_period = 10; + } + + return 0; +} + +void t3_led_ready(struct adapter *adapter) +{ + t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, + F_GPIO0_OUT_VAL); +} + +int t3_replay_prep_adapter(struct adapter *adapter) +{ + const struct adapter_info *ai = adapter->params.info; + unsigned int i, j = -1; + int ret; + + early_hw_init(adapter, ai); + ret = init_parity(adapter); + if (ret) + return ret; + + for_each_port(adapter, i) { + const struct port_type_info *pti; + struct port_info *p = adap2pinfo(adapter, i); + + while (!adapter->params.vpd.port_type[++j]) + ; + + pti = &port_types[adapter->params.vpd.port_type[j]]; + ret = pti->phy_prep(&p->phy, adapter, p->phy.mdio.prtad, NULL); + if (ret) + return ret; + p->phy.ops->power_down(&p->phy, 1); + } + + return 0; +} + diff --git a/drivers/net/ethernet/chelsio/cxgb3/t3cdev.h b/drivers/net/ethernet/chelsio/cxgb3/t3cdev.h new file mode 100644 index 000000000..3c3e6cf6a --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/t3cdev.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2006-2008 Chelsio Communications. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef _T3CDEV_H_ +#define _T3CDEV_H_ + +#include <linux/list.h> +#include <linux/atomic.h> +#include <linux/netdevice.h> +#include <linux/proc_fs.h> +#include <linux/skbuff.h> +#include <net/neighbour.h> + +#define T3CNAMSIZ 16 + +struct cxgb3_client; + +enum t3ctype { + T3A = 0, + T3B, + T3C, +}; + +struct t3cdev { + char name[T3CNAMSIZ]; /* T3C device name */ + enum t3ctype type; + struct list_head ofld_dev_list; /* for list linking */ + struct net_device *lldev; /* LL dev associated with T3C messages */ + struct proc_dir_entry *proc_dir; /* root of proc dir for this T3C */ + int (*send)(struct t3cdev *dev, struct sk_buff *skb); + int (*recv)(struct t3cdev *dev, struct sk_buff **skb, int n); + int (*ctl)(struct t3cdev *dev, unsigned int req, void *data); + void (*neigh_update)(struct t3cdev *dev, struct neighbour *neigh); + void *priv; /* driver private data */ + void __rcu *l2opt; /* optional layer 2 data */ + void *l3opt; /* optional layer 3 data */ + void *l4opt; /* optional layer 4 data */ + void *ulp; /* ulp stuff */ + void *ulp_iscsi; /* ulp iscsi */ +}; + +#endif /* _T3CDEV_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb3/version.h b/drivers/net/ethernet/chelsio/cxgb3/version.h new file mode 100644 index 000000000..b4b2547ef --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/version.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +/* $Date: 2006/10/31 18:57:51 $ $RCSfile: version.h,v $ $Revision: 1.3 $ */ +#ifndef __CHELSIO_VERSION_H +#define __CHELSIO_VERSION_H +#define DRV_DESC "Chelsio T3 Network Driver" +#define DRV_NAME "cxgb3" + +/* Firmware version */ +#define FW_VERSION_MAJOR 7 +#define FW_VERSION_MINOR 12 +#define FW_VERSION_MICRO 0 +#endif /* __CHELSIO_VERSION_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb3/vsc8211.c b/drivers/net/ethernet/chelsio/cxgb3/vsc8211.c new file mode 100644 index 000000000..8638ad42b --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/vsc8211.c @@ -0,0 +1,416 @@ +/* + * Copyright (c) 2005-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "common.h" + +/* VSC8211 PHY specific registers. */ +enum { + VSC8211_SIGDET_CTRL = 19, + VSC8211_EXT_CTRL = 23, + VSC8211_INTR_ENABLE = 25, + VSC8211_INTR_STATUS = 26, + VSC8211_LED_CTRL = 27, + VSC8211_AUX_CTRL_STAT = 28, + VSC8211_EXT_PAGE_AXS = 31, +}; + +enum { + VSC_INTR_RX_ERR = 1 << 0, + VSC_INTR_MS_ERR = 1 << 1, /* master/slave resolution error */ + VSC_INTR_CABLE = 1 << 2, /* cable impairment */ + VSC_INTR_FALSE_CARR = 1 << 3, /* false carrier */ + VSC_INTR_MEDIA_CHG = 1 << 4, /* AMS media change */ + VSC_INTR_RX_FIFO = 1 << 5, /* Rx FIFO over/underflow */ + VSC_INTR_TX_FIFO = 1 << 6, /* Tx FIFO over/underflow */ + VSC_INTR_DESCRAMBL = 1 << 7, /* descrambler lock-lost */ + VSC_INTR_SYMBOL_ERR = 1 << 8, /* symbol error */ + VSC_INTR_NEG_DONE = 1 << 10, /* autoneg done */ + VSC_INTR_NEG_ERR = 1 << 11, /* autoneg error */ + VSC_INTR_DPLX_CHG = 1 << 12, /* duplex change */ + VSC_INTR_LINK_CHG = 1 << 13, /* link change */ + VSC_INTR_SPD_CHG = 1 << 14, /* speed change */ + VSC_INTR_ENABLE = 1 << 15, /* interrupt enable */ +}; + +enum { + VSC_CTRL_CLAUSE37_VIEW = 1 << 4, /* Switch to Clause 37 view */ + VSC_CTRL_MEDIA_MODE_HI = 0xf000 /* High part of media mode select */ +}; + +#define CFG_CHG_INTR_MASK (VSC_INTR_LINK_CHG | VSC_INTR_NEG_ERR | \ + VSC_INTR_DPLX_CHG | VSC_INTR_SPD_CHG | \ + VSC_INTR_NEG_DONE) +#define INTR_MASK (CFG_CHG_INTR_MASK | VSC_INTR_TX_FIFO | VSC_INTR_RX_FIFO | \ + VSC_INTR_ENABLE) + +/* PHY specific auxiliary control & status register fields */ +#define S_ACSR_ACTIPHY_TMR 0 +#define M_ACSR_ACTIPHY_TMR 0x3 +#define V_ACSR_ACTIPHY_TMR(x) ((x) << S_ACSR_ACTIPHY_TMR) + +#define S_ACSR_SPEED 3 +#define M_ACSR_SPEED 0x3 +#define G_ACSR_SPEED(x) (((x) >> S_ACSR_SPEED) & M_ACSR_SPEED) + +#define S_ACSR_DUPLEX 5 +#define F_ACSR_DUPLEX (1 << S_ACSR_DUPLEX) + +#define S_ACSR_ACTIPHY 6 +#define F_ACSR_ACTIPHY (1 << S_ACSR_ACTIPHY) + +/* + * Reset the PHY. This PHY completes reset immediately so we never wait. + */ +static int vsc8211_reset(struct cphy *cphy, int wait) +{ + return t3_phy_reset(cphy, MDIO_DEVAD_NONE, 0); +} + +static int vsc8211_intr_enable(struct cphy *cphy) +{ + return t3_mdio_write(cphy, MDIO_DEVAD_NONE, VSC8211_INTR_ENABLE, + INTR_MASK); +} + +static int vsc8211_intr_disable(struct cphy *cphy) +{ + return t3_mdio_write(cphy, MDIO_DEVAD_NONE, VSC8211_INTR_ENABLE, 0); +} + +static int vsc8211_intr_clear(struct cphy *cphy) +{ + u32 val; + + /* Clear PHY interrupts by reading the register. */ + return t3_mdio_read(cphy, MDIO_DEVAD_NONE, VSC8211_INTR_STATUS, &val); +} + +static int vsc8211_autoneg_enable(struct cphy *cphy) +{ + return t3_mdio_change_bits(cphy, MDIO_DEVAD_NONE, MII_BMCR, + BMCR_PDOWN | BMCR_ISOLATE, + BMCR_ANENABLE | BMCR_ANRESTART); +} + +static int vsc8211_autoneg_restart(struct cphy *cphy) +{ + return t3_mdio_change_bits(cphy, MDIO_DEVAD_NONE, MII_BMCR, + BMCR_PDOWN | BMCR_ISOLATE, + BMCR_ANRESTART); +} + +static int vsc8211_get_link_status(struct cphy *cphy, int *link_ok, + int *speed, int *duplex, int *fc) +{ + unsigned int bmcr, status, lpa, adv; + int err, sp = -1, dplx = -1, pause = 0; + + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, MII_BMCR, &bmcr); + if (!err) + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, MII_BMSR, &status); + if (err) + return err; + + if (link_ok) { + /* + * BMSR_LSTATUS is latch-low, so if it is 0 we need to read it + * once more to get the current link state. + */ + if (!(status & BMSR_LSTATUS)) + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, MII_BMSR, + &status); + if (err) + return err; + *link_ok = (status & BMSR_LSTATUS) != 0; + } + if (!(bmcr & BMCR_ANENABLE)) { + dplx = (bmcr & BMCR_FULLDPLX) ? DUPLEX_FULL : DUPLEX_HALF; + if (bmcr & BMCR_SPEED1000) + sp = SPEED_1000; + else if (bmcr & BMCR_SPEED100) + sp = SPEED_100; + else + sp = SPEED_10; + } else if (status & BMSR_ANEGCOMPLETE) { + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, VSC8211_AUX_CTRL_STAT, + &status); + if (err) + return err; + + dplx = (status & F_ACSR_DUPLEX) ? DUPLEX_FULL : DUPLEX_HALF; + sp = G_ACSR_SPEED(status); + if (sp == 0) + sp = SPEED_10; + else if (sp == 1) + sp = SPEED_100; + else + sp = SPEED_1000; + + if (fc && dplx == DUPLEX_FULL) { + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, MII_LPA, + &lpa); + if (!err) + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, + MII_ADVERTISE, &adv); + if (err) + return err; + + if (lpa & adv & ADVERTISE_PAUSE_CAP) + pause = PAUSE_RX | PAUSE_TX; + else if ((lpa & ADVERTISE_PAUSE_CAP) && + (lpa & ADVERTISE_PAUSE_ASYM) && + (adv & ADVERTISE_PAUSE_ASYM)) + pause = PAUSE_TX; + else if ((lpa & ADVERTISE_PAUSE_ASYM) && + (adv & ADVERTISE_PAUSE_CAP)) + pause = PAUSE_RX; + } + } + if (speed) + *speed = sp; + if (duplex) + *duplex = dplx; + if (fc) + *fc = pause; + return 0; +} + +static int vsc8211_get_link_status_fiber(struct cphy *cphy, int *link_ok, + int *speed, int *duplex, int *fc) +{ + unsigned int bmcr, status, lpa, adv; + int err, sp = -1, dplx = -1, pause = 0; + + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, MII_BMCR, &bmcr); + if (!err) + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, MII_BMSR, &status); + if (err) + return err; + + if (link_ok) { + /* + * BMSR_LSTATUS is latch-low, so if it is 0 we need to read it + * once more to get the current link state. + */ + if (!(status & BMSR_LSTATUS)) + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, MII_BMSR, + &status); + if (err) + return err; + *link_ok = (status & BMSR_LSTATUS) != 0; + } + if (!(bmcr & BMCR_ANENABLE)) { + dplx = (bmcr & BMCR_FULLDPLX) ? DUPLEX_FULL : DUPLEX_HALF; + if (bmcr & BMCR_SPEED1000) + sp = SPEED_1000; + else if (bmcr & BMCR_SPEED100) + sp = SPEED_100; + else + sp = SPEED_10; + } else if (status & BMSR_ANEGCOMPLETE) { + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, MII_LPA, &lpa); + if (!err) + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, MII_ADVERTISE, + &adv); + if (err) + return err; + + if (adv & lpa & ADVERTISE_1000XFULL) { + dplx = DUPLEX_FULL; + sp = SPEED_1000; + } else if (adv & lpa & ADVERTISE_1000XHALF) { + dplx = DUPLEX_HALF; + sp = SPEED_1000; + } + + if (fc && dplx == DUPLEX_FULL) { + if (lpa & adv & ADVERTISE_1000XPAUSE) + pause = PAUSE_RX | PAUSE_TX; + else if ((lpa & ADVERTISE_1000XPAUSE) && + (adv & lpa & ADVERTISE_1000XPSE_ASYM)) + pause = PAUSE_TX; + else if ((lpa & ADVERTISE_1000XPSE_ASYM) && + (adv & ADVERTISE_1000XPAUSE)) + pause = PAUSE_RX; + } + } + if (speed) + *speed = sp; + if (duplex) + *duplex = dplx; + if (fc) + *fc = pause; + return 0; +} + +#ifdef UNUSED +/* + * Enable/disable auto MDI/MDI-X in forced link speed mode. + */ +static int vsc8211_set_automdi(struct cphy *phy, int enable) +{ + int err; + + err = t3_mdio_write(phy, MDIO_DEVAD_NONE, VSC8211_EXT_PAGE_AXS, 0x52b5); + if (err) + return err; + + err = t3_mdio_write(phy, MDIO_DEVAD_NONE, 18, 0x12); + if (err) + return err; + + err = t3_mdio_write(phy, MDIO_DEVAD_NONE, 17, enable ? 0x2803 : 0x3003); + if (err) + return err; + + err = t3_mdio_write(phy, MDIO_DEVAD_NONE, 16, 0x87fa); + if (err) + return err; + + err = t3_mdio_write(phy, MDIO_DEVAD_NONE, VSC8211_EXT_PAGE_AXS, 0); + if (err) + return err; + + return 0; +} + +int vsc8211_set_speed_duplex(struct cphy *phy, int speed, int duplex) +{ + int err; + + err = t3_set_phy_speed_duplex(phy, speed, duplex); + if (!err) + err = vsc8211_set_automdi(phy, 1); + return err; +} +#endif /* UNUSED */ + +static int vsc8211_power_down(struct cphy *cphy, int enable) +{ + return t3_mdio_change_bits(cphy, 0, MII_BMCR, BMCR_PDOWN, + enable ? BMCR_PDOWN : 0); +} + +static int vsc8211_intr_handler(struct cphy *cphy) +{ + unsigned int cause; + int err, cphy_cause = 0; + + err = t3_mdio_read(cphy, MDIO_DEVAD_NONE, VSC8211_INTR_STATUS, &cause); + if (err) + return err; + + cause &= INTR_MASK; + if (cause & CFG_CHG_INTR_MASK) + cphy_cause |= cphy_cause_link_change; + if (cause & (VSC_INTR_RX_FIFO | VSC_INTR_TX_FIFO)) + cphy_cause |= cphy_cause_fifo_error; + return cphy_cause; +} + +static const struct cphy_ops vsc8211_ops = { + .reset = vsc8211_reset, + .intr_enable = vsc8211_intr_enable, + .intr_disable = vsc8211_intr_disable, + .intr_clear = vsc8211_intr_clear, + .intr_handler = vsc8211_intr_handler, + .autoneg_enable = vsc8211_autoneg_enable, + .autoneg_restart = vsc8211_autoneg_restart, + .advertise = t3_phy_advertise, + .set_speed_duplex = t3_set_phy_speed_duplex, + .get_link_status = vsc8211_get_link_status, + .power_down = vsc8211_power_down, +}; + +static const struct cphy_ops vsc8211_fiber_ops = { + .reset = vsc8211_reset, + .intr_enable = vsc8211_intr_enable, + .intr_disable = vsc8211_intr_disable, + .intr_clear = vsc8211_intr_clear, + .intr_handler = vsc8211_intr_handler, + .autoneg_enable = vsc8211_autoneg_enable, + .autoneg_restart = vsc8211_autoneg_restart, + .advertise = t3_phy_advertise_fiber, + .set_speed_duplex = t3_set_phy_speed_duplex, + .get_link_status = vsc8211_get_link_status_fiber, + .power_down = vsc8211_power_down, +}; + +int t3_vsc8211_phy_prep(struct cphy *phy, struct adapter *adapter, + int phy_addr, const struct mdio_ops *mdio_ops) +{ + int err; + unsigned int val; + + cphy_init(phy, adapter, phy_addr, &vsc8211_ops, mdio_ops, + SUPPORTED_10baseT_Full | SUPPORTED_100baseT_Full | + SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | SUPPORTED_MII | + SUPPORTED_TP | SUPPORTED_IRQ, "10/100/1000BASE-T"); + msleep(20); /* PHY needs ~10ms to start responding to MDIO */ + + err = t3_mdio_read(phy, MDIO_DEVAD_NONE, VSC8211_EXT_CTRL, &val); + if (err) + return err; + if (val & VSC_CTRL_MEDIA_MODE_HI) { + /* copper interface, just need to configure the LEDs */ + return t3_mdio_write(phy, MDIO_DEVAD_NONE, VSC8211_LED_CTRL, + 0x100); + } + + phy->caps = SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | + SUPPORTED_MII | SUPPORTED_FIBRE | SUPPORTED_IRQ; + phy->desc = "1000BASE-X"; + phy->ops = &vsc8211_fiber_ops; + + err = t3_mdio_write(phy, MDIO_DEVAD_NONE, VSC8211_EXT_PAGE_AXS, 1); + if (err) + return err; + + err = t3_mdio_write(phy, MDIO_DEVAD_NONE, VSC8211_SIGDET_CTRL, 1); + if (err) + return err; + + err = t3_mdio_write(phy, MDIO_DEVAD_NONE, VSC8211_EXT_PAGE_AXS, 0); + if (err) + return err; + + err = t3_mdio_write(phy, MDIO_DEVAD_NONE, VSC8211_EXT_CTRL, + val | VSC_CTRL_CLAUSE37_VIEW); + if (err) + return err; + + err = vsc8211_reset(phy, 0); + if (err) + return err; + + udelay(5); /* delay after reset before next SMI */ + return 0; +} diff --git a/drivers/net/ethernet/chelsio/cxgb3/xgmac.c b/drivers/net/ethernet/chelsio/cxgb3/xgmac.c new file mode 100644 index 000000000..1bdc6cad1 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb3/xgmac.c @@ -0,0 +1,657 @@ +/* + * Copyright (c) 2005-2008 Chelsio, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "common.h" +#include "regs.h" + +/* + * # of exact address filters. The first one is used for the station address, + * the rest are available for multicast addresses. + */ +#define EXACT_ADDR_FILTERS 8 + +static inline int macidx(const struct cmac *mac) +{ + return mac->offset / (XGMAC0_1_BASE_ADDR - XGMAC0_0_BASE_ADDR); +} + +static void xaui_serdes_reset(struct cmac *mac) +{ + static const unsigned int clear[] = { + F_PWRDN0 | F_PWRDN1, F_RESETPLL01, F_RESET0 | F_RESET1, + F_PWRDN2 | F_PWRDN3, F_RESETPLL23, F_RESET2 | F_RESET3 + }; + + int i; + struct adapter *adap = mac->adapter; + u32 ctrl = A_XGM_SERDES_CTRL0 + mac->offset; + + t3_write_reg(adap, ctrl, adap->params.vpd.xauicfg[macidx(mac)] | + F_RESET3 | F_RESET2 | F_RESET1 | F_RESET0 | + F_PWRDN3 | F_PWRDN2 | F_PWRDN1 | F_PWRDN0 | + F_RESETPLL23 | F_RESETPLL01); + t3_read_reg(adap, ctrl); + udelay(15); + + for (i = 0; i < ARRAY_SIZE(clear); i++) { + t3_set_reg_field(adap, ctrl, clear[i], 0); + udelay(15); + } +} + +void t3b_pcs_reset(struct cmac *mac) +{ + t3_set_reg_field(mac->adapter, A_XGM_RESET_CTRL + mac->offset, + F_PCS_RESET_, 0); + udelay(20); + t3_set_reg_field(mac->adapter, A_XGM_RESET_CTRL + mac->offset, 0, + F_PCS_RESET_); +} + +int t3_mac_reset(struct cmac *mac) +{ + static const struct addr_val_pair mac_reset_avp[] = { + {A_XGM_TX_CTRL, 0}, + {A_XGM_RX_CTRL, 0}, + {A_XGM_RX_CFG, F_DISPAUSEFRAMES | F_EN1536BFRAMES | + F_RMFCS | F_ENJUMBO | F_ENHASHMCAST}, + {A_XGM_RX_HASH_LOW, 0}, + {A_XGM_RX_HASH_HIGH, 0}, + {A_XGM_RX_EXACT_MATCH_LOW_1, 0}, + {A_XGM_RX_EXACT_MATCH_LOW_2, 0}, + {A_XGM_RX_EXACT_MATCH_LOW_3, 0}, + {A_XGM_RX_EXACT_MATCH_LOW_4, 0}, + {A_XGM_RX_EXACT_MATCH_LOW_5, 0}, + {A_XGM_RX_EXACT_MATCH_LOW_6, 0}, + {A_XGM_RX_EXACT_MATCH_LOW_7, 0}, + {A_XGM_RX_EXACT_MATCH_LOW_8, 0}, + {A_XGM_STAT_CTRL, F_CLRSTATS} + }; + u32 val; + struct adapter *adap = mac->adapter; + unsigned int oft = mac->offset; + + t3_write_reg(adap, A_XGM_RESET_CTRL + oft, F_MAC_RESET_); + t3_read_reg(adap, A_XGM_RESET_CTRL + oft); /* flush */ + + t3_write_regs(adap, mac_reset_avp, ARRAY_SIZE(mac_reset_avp), oft); + t3_set_reg_field(adap, A_XGM_RXFIFO_CFG + oft, + F_RXSTRFRWRD | F_DISERRFRAMES, + uses_xaui(adap) ? 0 : F_RXSTRFRWRD); + t3_set_reg_field(adap, A_XGM_TXFIFO_CFG + oft, 0, F_UNDERUNFIX); + + if (uses_xaui(adap)) { + if (adap->params.rev == 0) { + t3_set_reg_field(adap, A_XGM_SERDES_CTRL + oft, 0, + F_RXENABLE | F_TXENABLE); + if (t3_wait_op_done(adap, A_XGM_SERDES_STATUS1 + oft, + F_CMULOCK, 1, 5, 2)) { + CH_ERR(adap, + "MAC %d XAUI SERDES CMU lock failed\n", + macidx(mac)); + return -1; + } + t3_set_reg_field(adap, A_XGM_SERDES_CTRL + oft, 0, + F_SERDESRESET_); + } else + xaui_serdes_reset(mac); + } + + t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + oft, + V_RXMAXFRAMERSIZE(M_RXMAXFRAMERSIZE), + V_RXMAXFRAMERSIZE(MAX_FRAME_SIZE) | F_RXENFRAMER); + val = F_MAC_RESET_ | F_XGMAC_STOP_EN; + + if (is_10G(adap)) + val |= F_PCS_RESET_; + else if (uses_xaui(adap)) + val |= F_PCS_RESET_ | F_XG2G_RESET_; + else + val |= F_RGMII_RESET_ | F_XG2G_RESET_; + t3_write_reg(adap, A_XGM_RESET_CTRL + oft, val); + t3_read_reg(adap, A_XGM_RESET_CTRL + oft); /* flush */ + if ((val & F_PCS_RESET_) && adap->params.rev) { + msleep(1); + t3b_pcs_reset(mac); + } + + memset(&mac->stats, 0, sizeof(mac->stats)); + return 0; +} + +static int t3b2_mac_reset(struct cmac *mac) +{ + struct adapter *adap = mac->adapter; + unsigned int oft = mac->offset, store; + int idx = macidx(mac); + u32 val; + + if (!macidx(mac)) + t3_set_reg_field(adap, A_MPS_CFG, F_PORT0ACTIVE, 0); + else + t3_set_reg_field(adap, A_MPS_CFG, F_PORT1ACTIVE, 0); + + /* Stop NIC traffic to reduce the number of TXTOGGLES */ + t3_set_reg_field(adap, A_MPS_CFG, F_ENFORCEPKT, 0); + /* Ensure TX drains */ + t3_set_reg_field(adap, A_XGM_TX_CFG + oft, F_TXPAUSEEN, 0); + + t3_write_reg(adap, A_XGM_RESET_CTRL + oft, F_MAC_RESET_); + t3_read_reg(adap, A_XGM_RESET_CTRL + oft); /* flush */ + + /* Store A_TP_TX_DROP_CFG_CH0 */ + t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CFG_CH0 + idx); + store = t3_read_reg(adap, A_TP_TX_DROP_CFG_CH0 + idx); + + msleep(10); + + /* Change DROP_CFG to 0xc0000011 */ + t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CFG_CH0 + idx); + t3_write_reg(adap, A_TP_PIO_DATA, 0xc0000011); + + /* Check for xgm Rx fifo empty */ + /* Increased loop count to 1000 from 5 cover 1G and 100Mbps case */ + if (t3_wait_op_done(adap, A_XGM_RX_MAX_PKT_SIZE_ERR_CNT + oft, + 0x80000000, 1, 1000, 2)) { + CH_ERR(adap, "MAC %d Rx fifo drain failed\n", + macidx(mac)); + return -1; + } + + t3_write_reg(adap, A_XGM_RESET_CTRL + oft, 0); + t3_read_reg(adap, A_XGM_RESET_CTRL + oft); /* flush */ + + val = F_MAC_RESET_; + if (is_10G(adap)) + val |= F_PCS_RESET_; + else if (uses_xaui(adap)) + val |= F_PCS_RESET_ | F_XG2G_RESET_; + else + val |= F_RGMII_RESET_ | F_XG2G_RESET_; + t3_write_reg(adap, A_XGM_RESET_CTRL + oft, val); + t3_read_reg(adap, A_XGM_RESET_CTRL + oft); /* flush */ + if ((val & F_PCS_RESET_) && adap->params.rev) { + msleep(1); + t3b_pcs_reset(mac); + } + t3_write_reg(adap, A_XGM_RX_CFG + oft, + F_DISPAUSEFRAMES | F_EN1536BFRAMES | + F_RMFCS | F_ENJUMBO | F_ENHASHMCAST); + + /* Restore the DROP_CFG */ + t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CFG_CH0 + idx); + t3_write_reg(adap, A_TP_PIO_DATA, store); + + if (!idx) + t3_set_reg_field(adap, A_MPS_CFG, 0, F_PORT0ACTIVE); + else + t3_set_reg_field(adap, A_MPS_CFG, 0, F_PORT1ACTIVE); + + /* re-enable nic traffic */ + t3_set_reg_field(adap, A_MPS_CFG, F_ENFORCEPKT, 1); + + /* Set: re-enable NIC traffic */ + t3_set_reg_field(adap, A_MPS_CFG, F_ENFORCEPKT, 1); + + return 0; +} + +/* + * Set the exact match register 'idx' to recognize the given Ethernet address. + */ +static void set_addr_filter(struct cmac *mac, int idx, const u8 * addr) +{ + u32 addr_lo, addr_hi; + unsigned int oft = mac->offset + idx * 8; + + addr_lo = (addr[3] << 24) | (addr[2] << 16) | (addr[1] << 8) | addr[0]; + addr_hi = (addr[5] << 8) | addr[4]; + + t3_write_reg(mac->adapter, A_XGM_RX_EXACT_MATCH_LOW_1 + oft, addr_lo); + t3_write_reg(mac->adapter, A_XGM_RX_EXACT_MATCH_HIGH_1 + oft, addr_hi); +} + +/* Set one of the station's unicast MAC addresses. */ +int t3_mac_set_address(struct cmac *mac, unsigned int idx, const u8 addr[6]) +{ + if (idx >= mac->nucast) + return -EINVAL; + set_addr_filter(mac, idx, addr); + return 0; +} + +/* + * Specify the number of exact address filters that should be reserved for + * unicast addresses. Caller should reload the unicast and multicast addresses + * after calling this. + */ +int t3_mac_set_num_ucast(struct cmac *mac, int n) +{ + if (n > EXACT_ADDR_FILTERS) + return -EINVAL; + mac->nucast = n; + return 0; +} + +void t3_mac_disable_exact_filters(struct cmac *mac) +{ + unsigned int i, reg = mac->offset + A_XGM_RX_EXACT_MATCH_LOW_1; + + for (i = 0; i < EXACT_ADDR_FILTERS; i++, reg += 8) { + u32 v = t3_read_reg(mac->adapter, reg); + t3_write_reg(mac->adapter, reg, v); + } + t3_read_reg(mac->adapter, A_XGM_RX_EXACT_MATCH_LOW_1); /* flush */ +} + +void t3_mac_enable_exact_filters(struct cmac *mac) +{ + unsigned int i, reg = mac->offset + A_XGM_RX_EXACT_MATCH_HIGH_1; + + for (i = 0; i < EXACT_ADDR_FILTERS; i++, reg += 8) { + u32 v = t3_read_reg(mac->adapter, reg); + t3_write_reg(mac->adapter, reg, v); + } + t3_read_reg(mac->adapter, A_XGM_RX_EXACT_MATCH_LOW_1); /* flush */ +} + +/* Calculate the RX hash filter index of an Ethernet address */ +static int hash_hw_addr(const u8 * addr) +{ + int hash = 0, octet, bit, i = 0, c; + + for (octet = 0; octet < 6; ++octet) + for (c = addr[octet], bit = 0; bit < 8; c >>= 1, ++bit) { + hash ^= (c & 1) << i; + if (++i == 6) + i = 0; + } + return hash; +} + +int t3_mac_set_rx_mode(struct cmac *mac, struct net_device *dev) +{ + u32 val, hash_lo, hash_hi; + struct adapter *adap = mac->adapter; + unsigned int oft = mac->offset; + + val = t3_read_reg(adap, A_XGM_RX_CFG + oft) & ~F_COPYALLFRAMES; + if (dev->flags & IFF_PROMISC) + val |= F_COPYALLFRAMES; + t3_write_reg(adap, A_XGM_RX_CFG + oft, val); + + if (dev->flags & IFF_ALLMULTI) + hash_lo = hash_hi = 0xffffffff; + else { + struct netdev_hw_addr *ha; + int exact_addr_idx = mac->nucast; + + hash_lo = hash_hi = 0; + netdev_for_each_mc_addr(ha, dev) + if (exact_addr_idx < EXACT_ADDR_FILTERS) + set_addr_filter(mac, exact_addr_idx++, + ha->addr); + else { + int hash = hash_hw_addr(ha->addr); + + if (hash < 32) + hash_lo |= (1 << hash); + else + hash_hi |= (1 << (hash - 32)); + } + } + + t3_write_reg(adap, A_XGM_RX_HASH_LOW + oft, hash_lo); + t3_write_reg(adap, A_XGM_RX_HASH_HIGH + oft, hash_hi); + return 0; +} + +static int rx_fifo_hwm(int mtu) +{ + int hwm; + + hwm = max(MAC_RXFIFO_SIZE - 3 * mtu, (MAC_RXFIFO_SIZE * 38) / 100); + return min(hwm, MAC_RXFIFO_SIZE - 8192); +} + +int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu) +{ + int hwm, lwm, divisor; + int ipg; + unsigned int thres, v, reg; + struct adapter *adap = mac->adapter; + + /* + * MAX_FRAME_SIZE inludes header + FCS, mtu doesn't. The HW max + * packet size register includes header, but not FCS. + */ + mtu += 14; + if (mtu > 1536) + mtu += 4; + + if (mtu > MAX_FRAME_SIZE - 4) + return -EINVAL; + t3_write_reg(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset, mtu); + + if (adap->params.rev >= T3_REV_B2 && + (t3_read_reg(adap, A_XGM_RX_CTRL + mac->offset) & F_RXEN)) { + t3_mac_disable_exact_filters(mac); + v = t3_read_reg(adap, A_XGM_RX_CFG + mac->offset); + t3_set_reg_field(adap, A_XGM_RX_CFG + mac->offset, + F_ENHASHMCAST | F_COPYALLFRAMES, F_DISBCAST); + + reg = adap->params.rev == T3_REV_B2 ? + A_XGM_RX_MAX_PKT_SIZE_ERR_CNT : A_XGM_RXFIFO_CFG; + + /* drain RX FIFO */ + if (t3_wait_op_done(adap, reg + mac->offset, + F_RXFIFO_EMPTY, 1, 20, 5)) { + t3_write_reg(adap, A_XGM_RX_CFG + mac->offset, v); + t3_mac_enable_exact_filters(mac); + return -EIO; + } + t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset, + V_RXMAXPKTSIZE(M_RXMAXPKTSIZE), + V_RXMAXPKTSIZE(mtu)); + t3_write_reg(adap, A_XGM_RX_CFG + mac->offset, v); + t3_mac_enable_exact_filters(mac); + } else + t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset, + V_RXMAXPKTSIZE(M_RXMAXPKTSIZE), + V_RXMAXPKTSIZE(mtu)); + + /* + * Adjust the PAUSE frame watermarks. We always set the LWM, and the + * HWM only if flow-control is enabled. + */ + hwm = rx_fifo_hwm(mtu); + lwm = min(3 * (int)mtu, MAC_RXFIFO_SIZE / 4); + v = t3_read_reg(adap, A_XGM_RXFIFO_CFG + mac->offset); + v &= ~V_RXFIFOPAUSELWM(M_RXFIFOPAUSELWM); + v |= V_RXFIFOPAUSELWM(lwm / 8); + if (G_RXFIFOPAUSEHWM(v)) + v = (v & ~V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM)) | + V_RXFIFOPAUSEHWM(hwm / 8); + + t3_write_reg(adap, A_XGM_RXFIFO_CFG + mac->offset, v); + + /* Adjust the TX FIFO threshold based on the MTU */ + thres = (adap->params.vpd.cclk * 1000) / 15625; + thres = (thres * mtu) / 1000; + if (is_10G(adap)) + thres /= 10; + thres = mtu > thres ? (mtu - thres + 7) / 8 : 0; + thres = max(thres, 8U); /* need at least 8 */ + ipg = (adap->params.rev == T3_REV_C) ? 0 : 1; + t3_set_reg_field(adap, A_XGM_TXFIFO_CFG + mac->offset, + V_TXFIFOTHRESH(M_TXFIFOTHRESH) | V_TXIPG(M_TXIPG), + V_TXFIFOTHRESH(thres) | V_TXIPG(ipg)); + + if (adap->params.rev > 0) { + divisor = (adap->params.rev == T3_REV_C) ? 64 : 8; + t3_write_reg(adap, A_XGM_PAUSE_TIMER + mac->offset, + (hwm - lwm) * 4 / divisor); + } + t3_write_reg(adap, A_XGM_TX_PAUSE_QUANTA + mac->offset, + MAC_RXFIFO_SIZE * 4 * 8 / 512); + return 0; +} + +int t3_mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, int fc) +{ + u32 val; + struct adapter *adap = mac->adapter; + unsigned int oft = mac->offset; + + if (duplex >= 0 && duplex != DUPLEX_FULL) + return -EINVAL; + if (speed >= 0) { + if (speed == SPEED_10) + val = V_PORTSPEED(0); + else if (speed == SPEED_100) + val = V_PORTSPEED(1); + else if (speed == SPEED_1000) + val = V_PORTSPEED(2); + else if (speed == SPEED_10000) + val = V_PORTSPEED(3); + else + return -EINVAL; + + t3_set_reg_field(adap, A_XGM_PORT_CFG + oft, + V_PORTSPEED(M_PORTSPEED), val); + } + + val = t3_read_reg(adap, A_XGM_RXFIFO_CFG + oft); + val &= ~V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM); + if (fc & PAUSE_TX) { + u32 rx_max_pkt_size = + G_RXMAXPKTSIZE(t3_read_reg(adap, + A_XGM_RX_MAX_PKT_SIZE + oft)); + val |= V_RXFIFOPAUSEHWM(rx_fifo_hwm(rx_max_pkt_size) / 8); + } + t3_write_reg(adap, A_XGM_RXFIFO_CFG + oft, val); + + t3_set_reg_field(adap, A_XGM_TX_CFG + oft, F_TXPAUSEEN, + (fc & PAUSE_RX) ? F_TXPAUSEEN : 0); + return 0; +} + +int t3_mac_enable(struct cmac *mac, int which) +{ + int idx = macidx(mac); + struct adapter *adap = mac->adapter; + unsigned int oft = mac->offset; + struct mac_stats *s = &mac->stats; + + if (which & MAC_DIRECTION_TX) { + t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CFG_CH0 + idx); + t3_write_reg(adap, A_TP_PIO_DATA, + adap->params.rev == T3_REV_C ? + 0xc4ffff01 : 0xc0ede401); + t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_MODE); + t3_set_reg_field(adap, A_TP_PIO_DATA, 1 << idx, + adap->params.rev == T3_REV_C ? 0 : 1 << idx); + + t3_write_reg(adap, A_XGM_TX_CTRL + oft, F_TXEN); + + t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CNT_CH0 + idx); + mac->tx_mcnt = s->tx_frames; + mac->tx_tcnt = (G_TXDROPCNTCH0RCVD(t3_read_reg(adap, + A_TP_PIO_DATA))); + mac->tx_xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap, + A_XGM_TX_SPI4_SOP_EOP_CNT + + oft))); + mac->rx_mcnt = s->rx_frames; + mac->rx_pause = s->rx_pause; + mac->rx_xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap, + A_XGM_RX_SPI4_SOP_EOP_CNT + + oft))); + mac->rx_ocnt = s->rx_fifo_ovfl; + mac->txen = F_TXEN; + mac->toggle_cnt = 0; + } + if (which & MAC_DIRECTION_RX) + t3_write_reg(adap, A_XGM_RX_CTRL + oft, F_RXEN); + return 0; +} + +int t3_mac_disable(struct cmac *mac, int which) +{ + struct adapter *adap = mac->adapter; + + if (which & MAC_DIRECTION_TX) { + t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, 0); + mac->txen = 0; + } + if (which & MAC_DIRECTION_RX) { + int val = F_MAC_RESET_; + + t3_set_reg_field(mac->adapter, A_XGM_RESET_CTRL + mac->offset, + F_PCS_RESET_, 0); + msleep(100); + t3_write_reg(adap, A_XGM_RX_CTRL + mac->offset, 0); + if (is_10G(adap)) + val |= F_PCS_RESET_; + else if (uses_xaui(adap)) + val |= F_PCS_RESET_ | F_XG2G_RESET_; + else + val |= F_RGMII_RESET_ | F_XG2G_RESET_; + t3_write_reg(mac->adapter, A_XGM_RESET_CTRL + mac->offset, val); + } + return 0; +} + +int t3b2_mac_watchdog_task(struct cmac *mac) +{ + struct adapter *adap = mac->adapter; + struct mac_stats *s = &mac->stats; + unsigned int tx_tcnt, tx_xcnt; + u64 tx_mcnt = s->tx_frames; + int status; + + status = 0; + tx_xcnt = 1; /* By default tx_xcnt is making progress */ + tx_tcnt = mac->tx_tcnt; /* If tx_mcnt is progressing ignore tx_tcnt */ + if (tx_mcnt == mac->tx_mcnt && mac->rx_pause == s->rx_pause) { + tx_xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap, + A_XGM_TX_SPI4_SOP_EOP_CNT + + mac->offset))); + if (tx_xcnt == 0) { + t3_write_reg(adap, A_TP_PIO_ADDR, + A_TP_TX_DROP_CNT_CH0 + macidx(mac)); + tx_tcnt = (G_TXDROPCNTCH0RCVD(t3_read_reg(adap, + A_TP_PIO_DATA))); + } else { + goto out; + } + } else { + mac->toggle_cnt = 0; + goto out; + } + + if ((tx_tcnt != mac->tx_tcnt) && (mac->tx_xcnt == 0)) { + if (mac->toggle_cnt > 4) { + status = 2; + goto out; + } else { + status = 1; + goto out; + } + } else { + mac->toggle_cnt = 0; + goto out; + } + +out: + mac->tx_tcnt = tx_tcnt; + mac->tx_xcnt = tx_xcnt; + mac->tx_mcnt = s->tx_frames; + mac->rx_pause = s->rx_pause; + if (status == 1) { + t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, 0); + t3_read_reg(adap, A_XGM_TX_CTRL + mac->offset); /* flush */ + t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, mac->txen); + t3_read_reg(adap, A_XGM_TX_CTRL + mac->offset); /* flush */ + mac->toggle_cnt++; + } else if (status == 2) { + t3b2_mac_reset(mac); + mac->toggle_cnt = 0; + } + return status; +} + +/* + * This function is called periodically to accumulate the current values of the + * RMON counters into the port statistics. Since the packet counters are only + * 32 bits they can overflow in ~286 secs at 10G, so the function should be + * called more frequently than that. The byte counters are 45-bit wide, they + * would overflow in ~7.8 hours. + */ +const struct mac_stats *t3_mac_update_stats(struct cmac *mac) +{ +#define RMON_READ(mac, addr) t3_read_reg(mac->adapter, addr + mac->offset) +#define RMON_UPDATE(mac, name, reg) \ + (mac)->stats.name += (u64)RMON_READ(mac, A_XGM_STAT_##reg) +#define RMON_UPDATE64(mac, name, reg_lo, reg_hi) \ + (mac)->stats.name += RMON_READ(mac, A_XGM_STAT_##reg_lo) + \ + ((u64)RMON_READ(mac, A_XGM_STAT_##reg_hi) << 32) + + u32 v, lo; + + RMON_UPDATE64(mac, rx_octets, RX_BYTES_LOW, RX_BYTES_HIGH); + RMON_UPDATE64(mac, rx_frames, RX_FRAMES_LOW, RX_FRAMES_HIGH); + RMON_UPDATE(mac, rx_mcast_frames, RX_MCAST_FRAMES); + RMON_UPDATE(mac, rx_bcast_frames, RX_BCAST_FRAMES); + RMON_UPDATE(mac, rx_fcs_errs, RX_CRC_ERR_FRAMES); + RMON_UPDATE(mac, rx_pause, RX_PAUSE_FRAMES); + RMON_UPDATE(mac, rx_jabber, RX_JABBER_FRAMES); + RMON_UPDATE(mac, rx_short, RX_SHORT_FRAMES); + RMON_UPDATE(mac, rx_symbol_errs, RX_SYM_CODE_ERR_FRAMES); + + RMON_UPDATE(mac, rx_too_long, RX_OVERSIZE_FRAMES); + + v = RMON_READ(mac, A_XGM_RX_MAX_PKT_SIZE_ERR_CNT); + if (mac->adapter->params.rev == T3_REV_B2) + v &= 0x7fffffff; + mac->stats.rx_too_long += v; + + RMON_UPDATE(mac, rx_frames_64, RX_64B_FRAMES); + RMON_UPDATE(mac, rx_frames_65_127, RX_65_127B_FRAMES); + RMON_UPDATE(mac, rx_frames_128_255, RX_128_255B_FRAMES); + RMON_UPDATE(mac, rx_frames_256_511, RX_256_511B_FRAMES); + RMON_UPDATE(mac, rx_frames_512_1023, RX_512_1023B_FRAMES); + RMON_UPDATE(mac, rx_frames_1024_1518, RX_1024_1518B_FRAMES); + RMON_UPDATE(mac, rx_frames_1519_max, RX_1519_MAXB_FRAMES); + + RMON_UPDATE64(mac, tx_octets, TX_BYTE_LOW, TX_BYTE_HIGH); + RMON_UPDATE64(mac, tx_frames, TX_FRAME_LOW, TX_FRAME_HIGH); + RMON_UPDATE(mac, tx_mcast_frames, TX_MCAST); + RMON_UPDATE(mac, tx_bcast_frames, TX_BCAST); + RMON_UPDATE(mac, tx_pause, TX_PAUSE); + /* This counts error frames in general (bad FCS, underrun, etc). */ + RMON_UPDATE(mac, tx_underrun, TX_ERR_FRAMES); + + RMON_UPDATE(mac, tx_frames_64, TX_64B_FRAMES); + RMON_UPDATE(mac, tx_frames_65_127, TX_65_127B_FRAMES); + RMON_UPDATE(mac, tx_frames_128_255, TX_128_255B_FRAMES); + RMON_UPDATE(mac, tx_frames_256_511, TX_256_511B_FRAMES); + RMON_UPDATE(mac, tx_frames_512_1023, TX_512_1023B_FRAMES); + RMON_UPDATE(mac, tx_frames_1024_1518, TX_1024_1518B_FRAMES); + RMON_UPDATE(mac, tx_frames_1519_max, TX_1519_MAXB_FRAMES); + + /* The next stat isn't clear-on-read. */ + t3_write_reg(mac->adapter, A_TP_MIB_INDEX, mac->offset ? 51 : 50); + v = t3_read_reg(mac->adapter, A_TP_MIB_RDATA); + lo = (u32) mac->stats.rx_cong_drops; + mac->stats.rx_cong_drops += (u64) (v - lo); + + return &mac->stats; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/Makefile b/drivers/net/ethernet/chelsio/cxgb4/Makefile new file mode 100644 index 000000000..a4b4d475a --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/Makefile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Chelsio T4 driver +# + +obj-$(CONFIG_CHELSIO_T4) += cxgb4.o + +cxgb4-objs := cxgb4_main.o l2t.o smt.o t4_hw.o sge.o clip_tbl.o cxgb4_ethtool.o \ + cxgb4_uld.o srq.o sched.o cxgb4_filter.o cxgb4_tc_u32.o \ + cxgb4_ptp.o cxgb4_tc_flower.o cxgb4_cudbg.o cxgb4_mps.o \ + cudbg_common.o cudbg_lib.o cudbg_zlib.o cxgb4_tc_mqprio.o \ + cxgb4_tc_matchall.o +cxgb4-$(CONFIG_CHELSIO_T4_DCB) += cxgb4_dcb.o +cxgb4-$(CONFIG_CHELSIO_T4_FCOE) += cxgb4_fcoe.o +cxgb4-$(CONFIG_DEBUG_FS) += cxgb4_debugfs.o +cxgb4-$(CONFIG_THERMAL) += cxgb4_thermal.o diff --git a/drivers/net/ethernet/chelsio/cxgb4/clip_tbl.c b/drivers/net/ethernet/chelsio/cxgb4/clip_tbl.c new file mode 100644 index 000000000..163efab27 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/clip_tbl.c @@ -0,0 +1,328 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * Copyright (C) 2003-2014 Chelsio Communications. All rights reserved. + * + * Written by Deepak (deepak.s@chelsio.com) + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this + * release for licensing terms and conditions. + */ + +#include <linux/module.h> +#include <linux/netdevice.h> +#include <linux/jhash.h> +#include <linux/if_vlan.h> +#include <net/addrconf.h> +#include "cxgb4.h" +#include "clip_tbl.h" + +static inline unsigned int ipv4_clip_hash(struct clip_tbl *c, const u32 *key) +{ + unsigned int clipt_size_half = c->clipt_size / 2; + + return jhash_1word(*key, 0) % clipt_size_half; +} + +static inline unsigned int ipv6_clip_hash(struct clip_tbl *d, const u32 *key) +{ + unsigned int clipt_size_half = d->clipt_size / 2; + u32 xor = key[0] ^ key[1] ^ key[2] ^ key[3]; + + return clipt_size_half + + (jhash_1word(xor, 0) % clipt_size_half); +} + +static unsigned int clip_addr_hash(struct clip_tbl *ctbl, const u32 *addr, + u8 v6) +{ + return v6 ? ipv6_clip_hash(ctbl, addr) : + ipv4_clip_hash(ctbl, addr); +} + +static int clip6_get_mbox(const struct net_device *dev, + const struct in6_addr *lip) +{ + struct adapter *adap = netdev2adap(dev); + struct fw_clip_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_write = htonl(FW_CMD_OP_V(FW_CLIP_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F); + c.alloc_to_len16 = htonl(FW_CLIP_CMD_ALLOC_F | FW_LEN16(c)); + *(__be64 *)&c.ip_hi = *(__be64 *)(lip->s6_addr); + *(__be64 *)&c.ip_lo = *(__be64 *)(lip->s6_addr + 8); + return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false); +} + +static int clip6_release_mbox(const struct net_device *dev, + const struct in6_addr *lip) +{ + struct adapter *adap = netdev2adap(dev); + struct fw_clip_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_write = htonl(FW_CMD_OP_V(FW_CLIP_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F); + c.alloc_to_len16 = htonl(FW_CLIP_CMD_FREE_F | FW_LEN16(c)); + *(__be64 *)&c.ip_hi = *(__be64 *)(lip->s6_addr); + *(__be64 *)&c.ip_lo = *(__be64 *)(lip->s6_addr + 8); + return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false); +} + +int cxgb4_clip_get(const struct net_device *dev, const u32 *lip, u8 v6) +{ + struct adapter *adap = netdev2adap(dev); + struct clip_tbl *ctbl = adap->clipt; + struct clip_entry *ce, *cte; + u32 *addr = (u32 *)lip; + int hash; + int ret = -1; + + if (!ctbl) + return 0; + + hash = clip_addr_hash(ctbl, addr, v6); + + read_lock_bh(&ctbl->lock); + list_for_each_entry(cte, &ctbl->hash_list[hash], list) { + if (cte->addr6.sin6_family == AF_INET6 && v6) + ret = memcmp(lip, cte->addr6.sin6_addr.s6_addr, + sizeof(struct in6_addr)); + else if (cte->addr.sin_family == AF_INET && !v6) + ret = memcmp(lip, (char *)(&cte->addr.sin_addr), + sizeof(struct in_addr)); + if (!ret) { + ce = cte; + read_unlock_bh(&ctbl->lock); + refcount_inc(&ce->refcnt); + return 0; + } + } + read_unlock_bh(&ctbl->lock); + + write_lock_bh(&ctbl->lock); + if (!list_empty(&ctbl->ce_free_head)) { + ce = list_first_entry(&ctbl->ce_free_head, + struct clip_entry, list); + list_del_init(&ce->list); + spin_lock_init(&ce->lock); + refcount_set(&ce->refcnt, 0); + atomic_dec(&ctbl->nfree); + list_add_tail(&ce->list, &ctbl->hash_list[hash]); + if (v6) { + ce->addr6.sin6_family = AF_INET6; + memcpy(ce->addr6.sin6_addr.s6_addr, + lip, sizeof(struct in6_addr)); + ret = clip6_get_mbox(dev, (const struct in6_addr *)lip); + if (ret) { + write_unlock_bh(&ctbl->lock); + dev_err(adap->pdev_dev, + "CLIP FW cmd failed with error %d, " + "Connections using %pI6c wont be " + "offloaded", + ret, ce->addr6.sin6_addr.s6_addr); + return ret; + } + } else { + ce->addr.sin_family = AF_INET; + memcpy((char *)(&ce->addr.sin_addr), lip, + sizeof(struct in_addr)); + } + } else { + write_unlock_bh(&ctbl->lock); + dev_info(adap->pdev_dev, "CLIP table overflow, " + "Connections using %pI6c wont be offloaded", + (void *)lip); + return -ENOMEM; + } + write_unlock_bh(&ctbl->lock); + refcount_set(&ce->refcnt, 1); + return 0; +} +EXPORT_SYMBOL(cxgb4_clip_get); + +void cxgb4_clip_release(const struct net_device *dev, const u32 *lip, u8 v6) +{ + struct adapter *adap = netdev2adap(dev); + struct clip_tbl *ctbl = adap->clipt; + struct clip_entry *ce, *cte; + u32 *addr = (u32 *)lip; + int hash; + int ret = -1; + + if (!ctbl) + return; + + hash = clip_addr_hash(ctbl, addr, v6); + + read_lock_bh(&ctbl->lock); + list_for_each_entry(cte, &ctbl->hash_list[hash], list) { + if (cte->addr6.sin6_family == AF_INET6 && v6) + ret = memcmp(lip, cte->addr6.sin6_addr.s6_addr, + sizeof(struct in6_addr)); + else if (cte->addr.sin_family == AF_INET && !v6) + ret = memcmp(lip, (char *)(&cte->addr.sin_addr), + sizeof(struct in_addr)); + if (!ret) { + ce = cte; + read_unlock_bh(&ctbl->lock); + goto found; + } + } + read_unlock_bh(&ctbl->lock); + + return; +found: + write_lock_bh(&ctbl->lock); + spin_lock_bh(&ce->lock); + if (refcount_dec_and_test(&ce->refcnt)) { + list_del_init(&ce->list); + list_add_tail(&ce->list, &ctbl->ce_free_head); + atomic_inc(&ctbl->nfree); + if (v6) + clip6_release_mbox(dev, (const struct in6_addr *)lip); + } + spin_unlock_bh(&ce->lock); + write_unlock_bh(&ctbl->lock); +} +EXPORT_SYMBOL(cxgb4_clip_release); + +/* Retrieves IPv6 addresses from a root device (bond, vlan) associated with + * a physical device. + * The physical device reference is needed to send the actul CLIP command. + */ +static int cxgb4_update_dev_clip(struct net_device *root_dev, + struct net_device *dev) +{ + struct inet6_dev *idev = NULL; + struct inet6_ifaddr *ifa; + int ret = 0; + + idev = __in6_dev_get(root_dev); + if (!idev) + return ret; + + read_lock_bh(&idev->lock); + list_for_each_entry(ifa, &idev->addr_list, if_list) { + ret = cxgb4_clip_get(dev, (const u32 *)ifa->addr.s6_addr, 1); + if (ret < 0) + break; + } + read_unlock_bh(&idev->lock); + + return ret; +} + +int cxgb4_update_root_dev_clip(struct net_device *dev) +{ + struct net_device *root_dev = NULL; + int i, ret = 0; + + /* First populate the real net device's IPv6 addresses */ + ret = cxgb4_update_dev_clip(dev, dev); + if (ret) + return ret; + + /* Parse all bond and vlan devices layered on top of the physical dev */ + root_dev = netdev_master_upper_dev_get_rcu(dev); + if (root_dev) { + ret = cxgb4_update_dev_clip(root_dev, dev); + if (ret) + return ret; + } + + for (i = 0; i < VLAN_N_VID; i++) { + root_dev = __vlan_find_dev_deep_rcu(dev, htons(ETH_P_8021Q), i); + if (!root_dev) + continue; + + ret = cxgb4_update_dev_clip(root_dev, dev); + if (ret) + break; + } + + return ret; +} +EXPORT_SYMBOL(cxgb4_update_root_dev_clip); + +int clip_tbl_show(struct seq_file *seq, void *v) +{ + struct adapter *adapter = seq->private; + struct clip_tbl *ctbl = adapter->clipt; + struct clip_entry *ce; + char ip[60]; + int i; + + read_lock_bh(&ctbl->lock); + + seq_puts(seq, "IP Address Users\n"); + for (i = 0 ; i < ctbl->clipt_size; ++i) { + list_for_each_entry(ce, &ctbl->hash_list[i], list) { + ip[0] = '\0'; + sprintf(ip, "%pISc", &ce->addr); + seq_printf(seq, "%-25s %u\n", ip, + refcount_read(&ce->refcnt)); + } + } + seq_printf(seq, "Free clip entries : %d\n", atomic_read(&ctbl->nfree)); + + read_unlock_bh(&ctbl->lock); + + return 0; +} + +struct clip_tbl *t4_init_clip_tbl(unsigned int clipt_start, + unsigned int clipt_end) +{ + struct clip_entry *cl_list; + struct clip_tbl *ctbl; + unsigned int clipt_size; + int i; + + if (clipt_start >= clipt_end) + return NULL; + clipt_size = clipt_end - clipt_start + 1; + if (clipt_size < CLIPT_MIN_HASH_BUCKETS) + return NULL; + + ctbl = kvzalloc(struct_size(ctbl, hash_list, clipt_size), GFP_KERNEL); + if (!ctbl) + return NULL; + + ctbl->clipt_start = clipt_start; + ctbl->clipt_size = clipt_size; + INIT_LIST_HEAD(&ctbl->ce_free_head); + + atomic_set(&ctbl->nfree, clipt_size); + rwlock_init(&ctbl->lock); + + for (i = 0; i < ctbl->clipt_size; ++i) + INIT_LIST_HEAD(&ctbl->hash_list[i]); + + cl_list = kvcalloc(clipt_size, sizeof(struct clip_entry), GFP_KERNEL); + if (!cl_list) { + kvfree(ctbl); + return NULL; + } + ctbl->cl_list = (void *)cl_list; + + for (i = 0; i < clipt_size; i++) { + INIT_LIST_HEAD(&cl_list[i].list); + list_add_tail(&cl_list[i].list, &ctbl->ce_free_head); + } + + return ctbl; +} + +void t4_cleanup_clip_tbl(struct adapter *adap) +{ + struct clip_tbl *ctbl = adap->clipt; + + if (ctbl) { + kvfree(ctbl->cl_list); + kvfree(ctbl); + } +} +EXPORT_SYMBOL(t4_cleanup_clip_tbl); diff --git a/drivers/net/ethernet/chelsio/cxgb4/clip_tbl.h b/drivers/net/ethernet/chelsio/cxgb4/clip_tbl.h new file mode 100644 index 000000000..290c10580 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/clip_tbl.h @@ -0,0 +1,45 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * Copyright (C) 2003-2014 Chelsio Communications. All rights reserved. + * + * Written by Deepak (deepak.s@chelsio.com) + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this + * release for licensing terms and conditions. + */ + +#include <linux/refcount.h> + +struct clip_entry { + spinlock_t lock; /* Hold while modifying clip reference */ + refcount_t refcnt; + struct list_head list; + union { + struct sockaddr_in addr; + struct sockaddr_in6 addr6; + }; +}; + +struct clip_tbl { + unsigned int clipt_start; + unsigned int clipt_size; + rwlock_t lock; + atomic_t nfree; + struct list_head ce_free_head; + void *cl_list; + struct list_head hash_list[]; +}; + +enum { + CLIPT_MIN_HASH_BUCKETS = 2, +}; + +struct clip_tbl *t4_init_clip_tbl(unsigned int clipt_start, + unsigned int clipt_end); +int cxgb4_clip_get(const struct net_device *dev, const u32 *lip, u8 v6); +void cxgb4_clip_release(const struct net_device *dev, const u32 *lip, u8 v6); +int clip_tbl_show(struct seq_file *seq, void *v); +int cxgb4_update_root_dev_clip(struct net_device *dev); +void t4_cleanup_clip_tbl(struct adapter *adap); diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_common.c b/drivers/net/ethernet/chelsio/cxgb4/cudbg_common.c new file mode 100644 index 000000000..175e1a675 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_common.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017 Chelsio Communications. All rights reserved. + */ + +#include "cxgb4.h" +#include "cudbg_if.h" +#include "cudbg_lib_common.h" + +int cudbg_get_buff(struct cudbg_init *pdbg_init, + struct cudbg_buffer *pdbg_buff, u32 size, + struct cudbg_buffer *pin_buff) +{ + u32 offset; + + offset = pdbg_buff->offset; + if (offset + size > pdbg_buff->size) + return CUDBG_STATUS_NO_MEM; + + if (pdbg_init->compress_type != CUDBG_COMPRESSION_NONE) { + if (size > pdbg_init->compress_buff_size) + return CUDBG_STATUS_NO_MEM; + + pin_buff->data = (char *)pdbg_init->compress_buff; + pin_buff->offset = 0; + pin_buff->size = size; + return 0; + } + + pin_buff->data = (char *)pdbg_buff->data + offset; + pin_buff->offset = offset; + pin_buff->size = size; + return 0; +} + +void cudbg_put_buff(struct cudbg_init *pdbg_init, + struct cudbg_buffer *pin_buff) +{ + /* Clear compression buffer for re-use */ + if (pdbg_init->compress_type != CUDBG_COMPRESSION_NONE) + memset(pdbg_init->compress_buff, 0, + pdbg_init->compress_buff_size); + + pin_buff->data = NULL; + pin_buff->offset = 0; + pin_buff->size = 0; +} + +void cudbg_update_buff(struct cudbg_buffer *pin_buff, + struct cudbg_buffer *pout_buff) +{ + /* We already write to buffer provided by ethool, so just + * increment offset to next free space. + */ + pout_buff->offset += pin_buff->size; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h new file mode 100644 index 000000000..d5218e742 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h @@ -0,0 +1,352 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017 Chelsio Communications. All rights reserved. + */ + +#ifndef __CUDBG_ENTITY_H__ +#define __CUDBG_ENTITY_H__ + +#define EDC0_FLAG 0 +#define EDC1_FLAG 1 +#define MC_FLAG 2 +#define MC0_FLAG 3 +#define MC1_FLAG 4 +#define HMA_FLAG 5 + +#define CUDBG_ENTITY_SIGNATURE 0xCCEDB001 + +struct cudbg_mbox_log { + struct mbox_cmd entry; + u32 hi[MBOX_LEN / 8]; + u32 lo[MBOX_LEN / 8]; +}; + +struct cudbg_cim_qcfg { + u8 chip; + u16 base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; + u16 size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; + u16 thres[CIM_NUM_IBQ]; + u32 obq_wr[2 * CIM_NUM_OBQ_T5]; + u32 stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)]; +}; + +struct cudbg_rss_vf_conf { + u32 rss_vf_vfl; + u32 rss_vf_vfh; +}; + +struct cudbg_pm_stats { + u32 tx_cnt[T6_PM_NSTATS]; + u32 rx_cnt[T6_PM_NSTATS]; + u64 tx_cyc[T6_PM_NSTATS]; + u64 rx_cyc[T6_PM_NSTATS]; +}; + +struct cudbg_hw_sched { + u32 kbps[NTX_SCHED]; + u32 ipg[NTX_SCHED]; + u32 pace_tab[NTX_SCHED]; + u32 mode; + u32 map; +}; + +#define SGE_QBASE_DATA_REG_NUM 4 + +struct sge_qbase_reg_field { + u32 reg_addr; + u32 reg_data[SGE_QBASE_DATA_REG_NUM]; + /* Max supported PFs */ + u32 pf_data_value[PCIE_FW_MASTER_M + 1][SGE_QBASE_DATA_REG_NUM]; + /* Max supported VFs */ + u32 vf_data_value[T6_VF_M + 1][SGE_QBASE_DATA_REG_NUM]; + u32 vfcount; /* Actual number of max vfs in current configuration */ +}; + +struct ireg_field { + u32 ireg_addr; + u32 ireg_data; + u32 ireg_local_offset; + u32 ireg_offset_range; +}; + +struct ireg_buf { + struct ireg_field tp_pio; + u32 outbuf[32]; +}; + +struct cudbg_ulprx_la { + u32 data[ULPRX_LA_SIZE * 8]; + u32 size; +}; + +struct cudbg_tp_la { + u32 size; + u32 mode; + u8 data[]; +}; + +static const char * const cudbg_region[] = { + "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", + "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", + "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", + "TDDP region:", "TPT region:", "STAG region:", "RQ region:", + "RQUDP region:", "PBL region:", "TXPBL region:", + "DBVFIFO region:", "ULPRX state:", "ULPTX state:", + "On-chip queues:" +}; + +/* Memory region info relative to current memory (i.e. wrt 0). */ +struct cudbg_region_info { + bool exist; /* Does region exists in current memory? */ + u32 start; /* Start wrt 0 */ + u32 end; /* End wrt 0 */ +}; + +struct cudbg_mem_desc { + u32 base; + u32 limit; + u32 idx; +}; + +#define CUDBG_MEMINFO_REV 1 + +struct cudbg_meminfo { + struct cudbg_mem_desc avail[4]; + struct cudbg_mem_desc mem[ARRAY_SIZE(cudbg_region) + 3]; + u32 avail_c; + u32 mem_c; + u32 up_ram_lo; + u32 up_ram_hi; + u32 up_extmem2_lo; + u32 up_extmem2_hi; + u32 rx_pages_data[3]; + u32 tx_pages_data[4]; + u32 p_structs; + u32 reserved[12]; + u32 port_used[4]; + u32 port_alloc[4]; + u32 loopback_used[NCHAN]; + u32 loopback_alloc[NCHAN]; + u32 p_structs_free_cnt; + u32 free_rx_cnt; + u32 free_tx_cnt; +}; + +struct cudbg_cim_pif_la { + int size; + u8 data[]; +}; + +struct cudbg_clk_info { + u64 retransmit_min; + u64 retransmit_max; + u64 persist_timer_min; + u64 persist_timer_max; + u64 keepalive_idle_timer; + u64 keepalive_interval; + u64 initial_srtt; + u64 finwait2_timer; + u32 dack_timer; + u32 res; + u32 cclk_ps; + u32 tre; + u32 dack_re; +}; + +struct cudbg_tid_info_region { + u32 ntids; + u32 nstids; + u32 stid_base; + u32 hash_base; + + u32 natids; + u32 nftids; + u32 ftid_base; + u32 aftid_base; + u32 aftid_end; + + u32 sftid_base; + u32 nsftids; + + u32 uotid_base; + u32 nuotids; + + u32 sb; + u32 flags; + u32 le_db_conf; + u32 ip_users; + u32 ipv6_users; + + u32 hpftid_base; + u32 nhpftids; +}; + +#define CUDBG_TID_INFO_REV 1 + +struct cudbg_tid_info_region_rev1 { + struct cudbg_ver_hdr ver_hdr; + struct cudbg_tid_info_region tid; + u32 tid_start; + u32 reserved[16]; +}; + +#define CUDBG_LOWMEM_MAX_CTXT_QIDS 256 +#define CUDBG_MAX_FL_QIDS 1024 + +struct cudbg_ch_cntxt { + u32 cntxt_type; + u32 cntxt_id; + u32 data[SGE_CTXT_SIZE / 4]; +}; + +#define CUDBG_MAX_RPLC_SIZE 128 + +struct cudbg_mps_tcam { + u64 mask; + u32 rplc[8]; + u32 idx; + u32 cls_lo; + u32 cls_hi; + u32 rplc_size; + u32 vniy; + u32 vnix; + u32 dip_hit; + u32 vlan_vld; + u32 repli; + u16 ivlan; + u8 addr[ETH_ALEN]; + u8 lookup_type; + u8 port_num; + u8 reserved[2]; +}; + +#define CUDBG_VPD_VER_ADDR 0x18c7 +#define CUDBG_VPD_VER_LEN 2 + +struct cudbg_vpd_data { + u8 sn[SERNUM_LEN + 1]; + u8 bn[PN_LEN + 1]; + u8 na[MACADDR_LEN + 1]; + u8 mn[ID_LEN + 1]; + u16 fw_major; + u16 fw_minor; + u16 fw_micro; + u16 fw_build; + u32 scfg_vers; + u32 vpd_vers; +}; + +#define CUDBG_MAX_TCAM_TID 0x800 +#define CUDBG_T6_CLIP 1536 +#define CUDBG_MAX_TID_COMP_EN 6144 +#define CUDBG_MAX_TID_COMP_DIS 3072 + +enum cudbg_le_entry_types { + LE_ET_UNKNOWN = 0, + LE_ET_TCAM_CON = 1, + LE_ET_TCAM_SERVER = 2, + LE_ET_TCAM_FILTER = 3, + LE_ET_TCAM_CLIP = 4, + LE_ET_TCAM_ROUTING = 5, + LE_ET_HASH_CON = 6, + LE_ET_INVALID_TID = 8, +}; + +struct cudbg_tcam { + u32 filter_start; + u32 server_start; + u32 clip_start; + u32 routing_start; + u32 tid_hash_base; + u32 max_tid; +}; + +struct cudbg_tid_data { + u32 tid; + u32 dbig_cmd; + u32 dbig_conf; + u32 dbig_rsp_stat; + u32 data[NUM_LE_DB_DBGI_RSP_DATA_INSTANCES]; +}; + +#define CUDBG_NUM_ULPTX 11 +#define CUDBG_NUM_ULPTX_READ 512 +#define CUDBG_NUM_ULPTX_ASIC 6 +#define CUDBG_NUM_ULPTX_ASIC_READ 128 + +#define CUDBG_ULPTX_LA_REV 1 + +struct cudbg_ulptx_la { + u32 rdptr[CUDBG_NUM_ULPTX]; + u32 wrptr[CUDBG_NUM_ULPTX]; + u32 rddata[CUDBG_NUM_ULPTX]; + u32 rd_data[CUDBG_NUM_ULPTX][CUDBG_NUM_ULPTX_READ]; + u32 rdptr_asic[CUDBG_NUM_ULPTX_ASIC_READ]; + u32 rddata_asic[CUDBG_NUM_ULPTX_ASIC_READ][CUDBG_NUM_ULPTX_ASIC]; +}; + +#define CUDBG_CHAC_PBT_ADDR 0x2800 +#define CUDBG_CHAC_PBT_LRF 0x3000 +#define CUDBG_CHAC_PBT_DATA 0x3800 +#define CUDBG_PBT_DYNAMIC_ENTRIES 8 +#define CUDBG_PBT_STATIC_ENTRIES 16 +#define CUDBG_LRF_ENTRIES 8 +#define CUDBG_PBT_DATA_ENTRIES 512 + +struct cudbg_pbt_tables { + u32 pbt_dynamic[CUDBG_PBT_DYNAMIC_ENTRIES]; + u32 pbt_static[CUDBG_PBT_STATIC_ENTRIES]; + u32 lrf_table[CUDBG_LRF_ENTRIES]; + u32 pbt_data[CUDBG_PBT_DATA_ENTRIES]; +}; + +enum cudbg_qdesc_qtype { + CUDBG_QTYPE_UNKNOWN = 0, + CUDBG_QTYPE_NIC_TXQ, + CUDBG_QTYPE_NIC_RXQ, + CUDBG_QTYPE_NIC_FLQ, + CUDBG_QTYPE_CTRLQ, + CUDBG_QTYPE_FWEVTQ, + CUDBG_QTYPE_INTRQ, + CUDBG_QTYPE_PTP_TXQ, + CUDBG_QTYPE_OFLD_TXQ, + CUDBG_QTYPE_RDMA_RXQ, + CUDBG_QTYPE_RDMA_FLQ, + CUDBG_QTYPE_RDMA_CIQ, + CUDBG_QTYPE_ISCSI_RXQ, + CUDBG_QTYPE_ISCSI_FLQ, + CUDBG_QTYPE_ISCSIT_RXQ, + CUDBG_QTYPE_ISCSIT_FLQ, + CUDBG_QTYPE_CRYPTO_TXQ, + CUDBG_QTYPE_CRYPTO_RXQ, + CUDBG_QTYPE_CRYPTO_FLQ, + CUDBG_QTYPE_TLS_RXQ, + CUDBG_QTYPE_TLS_FLQ, + CUDBG_QTYPE_ETHOFLD_TXQ, + CUDBG_QTYPE_ETHOFLD_RXQ, + CUDBG_QTYPE_ETHOFLD_FLQ, + CUDBG_QTYPE_MAX, +}; + +#define CUDBG_QDESC_REV 1 + +struct cudbg_qdesc_entry { + u32 data_size; + u32 qtype; + u32 qid; + u32 desc_size; + u32 num_desc; + u8 data[]; /* Must be last */ +}; + +struct cudbg_qdesc_info { + u32 qdesc_entry_size; + u32 num_queues; + u8 data[]; /* Must be last */ +}; + +#define IREG_NUM_ELEM 4 + +#define CUDBG_NUM_PCIE_CONFIG_REGS 0x61 + +#endif /* __CUDBG_ENTITY_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h new file mode 100644 index 000000000..c84719e3c --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017 Chelsio Communications. All rights reserved. + */ + +#ifndef __CUDBG_IF_H__ +#define __CUDBG_IF_H__ + +/* Error codes */ +#define CUDBG_STATUS_NO_MEM -19 +#define CUDBG_STATUS_ENTITY_NOT_FOUND -24 +#define CUDBG_STATUS_NOT_IMPLEMENTED -28 +#define CUDBG_SYSTEM_ERROR -29 +#define CUDBG_STATUS_CCLK_NOT_DEFINED -32 +#define CUDBG_STATUS_PARTIAL_DATA -41 + +#define CUDBG_MAJOR_VERSION 1 +#define CUDBG_MINOR_VERSION 14 + +enum cudbg_dbg_entity_type { + CUDBG_REG_DUMP = 1, + CUDBG_DEV_LOG = 2, + CUDBG_CIM_LA = 3, + CUDBG_CIM_MA_LA = 4, + CUDBG_CIM_QCFG = 5, + CUDBG_CIM_IBQ_TP0 = 6, + CUDBG_CIM_IBQ_TP1 = 7, + CUDBG_CIM_IBQ_ULP = 8, + CUDBG_CIM_IBQ_SGE0 = 9, + CUDBG_CIM_IBQ_SGE1 = 10, + CUDBG_CIM_IBQ_NCSI = 11, + CUDBG_CIM_OBQ_ULP0 = 12, + CUDBG_CIM_OBQ_ULP1 = 13, + CUDBG_CIM_OBQ_ULP2 = 14, + CUDBG_CIM_OBQ_ULP3 = 15, + CUDBG_CIM_OBQ_SGE = 16, + CUDBG_CIM_OBQ_NCSI = 17, + CUDBG_EDC0 = 18, + CUDBG_EDC1 = 19, + CUDBG_MC0 = 20, + CUDBG_MC1 = 21, + CUDBG_RSS = 22, + CUDBG_RSS_VF_CONF = 25, + CUDBG_PATH_MTU = 27, + CUDBG_PM_STATS = 30, + CUDBG_HW_SCHED = 31, + CUDBG_TP_INDIRECT = 36, + CUDBG_SGE_INDIRECT = 37, + CUDBG_ULPRX_LA = 41, + CUDBG_TP_LA = 43, + CUDBG_MEMINFO = 44, + CUDBG_CIM_PIF_LA = 45, + CUDBG_CLK = 46, + CUDBG_CIM_OBQ_RXQ0 = 47, + CUDBG_CIM_OBQ_RXQ1 = 48, + CUDBG_PCIE_INDIRECT = 50, + CUDBG_PM_INDIRECT = 51, + CUDBG_TID_INFO = 54, + CUDBG_PCIE_CONFIG = 55, + CUDBG_DUMP_CONTEXT = 56, + CUDBG_MPS_TCAM = 57, + CUDBG_VPD_DATA = 58, + CUDBG_LE_TCAM = 59, + CUDBG_CCTRL = 60, + CUDBG_MA_INDIRECT = 61, + CUDBG_ULPTX_LA = 62, + CUDBG_UP_CIM_INDIRECT = 64, + CUDBG_PBT_TABLE = 65, + CUDBG_MBOX_LOG = 66, + CUDBG_HMA_INDIRECT = 67, + CUDBG_HMA = 68, + CUDBG_QDESC = 70, + CUDBG_FLASH = 71, + CUDBG_MAX_ENTITY = 72, +}; + +struct cudbg_init { + struct adapter *adap; /* Pointer to adapter structure */ + void *outbuf; /* Output buffer */ + u32 outbuf_size; /* Output buffer size */ + u8 compress_type; /* Type of compression to use */ + void *compress_buff; /* Compression buffer */ + u32 compress_buff_size; /* Compression buffer size */ + void *workspace; /* Workspace for zlib */ +}; + +static inline unsigned int cudbg_mbytes_to_bytes(unsigned int size) +{ + return size * 1024 * 1024; +} +#endif /* __CUDBG_IF_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c new file mode 100644 index 000000000..557c591a6 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c @@ -0,0 +1,3609 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017 Chelsio Communications. All rights reserved. + */ + +#include <linux/sort.h> +#include <linux/string.h> + +#include "t4_regs.h" +#include "cxgb4.h" +#include "cxgb4_cudbg.h" +#include "cudbg_if.h" +#include "cudbg_lib_common.h" +#include "cudbg_entity.h" +#include "cudbg_lib.h" +#include "cudbg_zlib.h" +#include "cxgb4_tc_mqprio.h" + +static const u32 t6_tp_pio_array[][IREG_NUM_ELEM] = { + {0x7e40, 0x7e44, 0x020, 28}, /* t6_tp_pio_regs_20_to_3b */ + {0x7e40, 0x7e44, 0x040, 10}, /* t6_tp_pio_regs_40_to_49 */ + {0x7e40, 0x7e44, 0x050, 10}, /* t6_tp_pio_regs_50_to_59 */ + {0x7e40, 0x7e44, 0x060, 14}, /* t6_tp_pio_regs_60_to_6d */ + {0x7e40, 0x7e44, 0x06F, 1}, /* t6_tp_pio_regs_6f */ + {0x7e40, 0x7e44, 0x070, 6}, /* t6_tp_pio_regs_70_to_75 */ + {0x7e40, 0x7e44, 0x130, 18}, /* t6_tp_pio_regs_130_to_141 */ + {0x7e40, 0x7e44, 0x145, 19}, /* t6_tp_pio_regs_145_to_157 */ + {0x7e40, 0x7e44, 0x160, 1}, /* t6_tp_pio_regs_160 */ + {0x7e40, 0x7e44, 0x230, 25}, /* t6_tp_pio_regs_230_to_248 */ + {0x7e40, 0x7e44, 0x24a, 3}, /* t6_tp_pio_regs_24c */ + {0x7e40, 0x7e44, 0x8C0, 1} /* t6_tp_pio_regs_8c0 */ +}; + +static const u32 t5_tp_pio_array[][IREG_NUM_ELEM] = { + {0x7e40, 0x7e44, 0x020, 28}, /* t5_tp_pio_regs_20_to_3b */ + {0x7e40, 0x7e44, 0x040, 19}, /* t5_tp_pio_regs_40_to_52 */ + {0x7e40, 0x7e44, 0x054, 2}, /* t5_tp_pio_regs_54_to_55 */ + {0x7e40, 0x7e44, 0x060, 13}, /* t5_tp_pio_regs_60_to_6c */ + {0x7e40, 0x7e44, 0x06F, 1}, /* t5_tp_pio_regs_6f */ + {0x7e40, 0x7e44, 0x120, 4}, /* t5_tp_pio_regs_120_to_123 */ + {0x7e40, 0x7e44, 0x12b, 2}, /* t5_tp_pio_regs_12b_to_12c */ + {0x7e40, 0x7e44, 0x12f, 21}, /* t5_tp_pio_regs_12f_to_143 */ + {0x7e40, 0x7e44, 0x145, 19}, /* t5_tp_pio_regs_145_to_157 */ + {0x7e40, 0x7e44, 0x230, 25}, /* t5_tp_pio_regs_230_to_248 */ + {0x7e40, 0x7e44, 0x8C0, 1} /* t5_tp_pio_regs_8c0 */ +}; + +static const u32 t6_tp_tm_pio_array[][IREG_NUM_ELEM] = { + {0x7e18, 0x7e1c, 0x0, 12} +}; + +static const u32 t5_tp_tm_pio_array[][IREG_NUM_ELEM] = { + {0x7e18, 0x7e1c, 0x0, 12} +}; + +static const u32 t6_tp_mib_index_array[6][IREG_NUM_ELEM] = { + {0x7e50, 0x7e54, 0x0, 13}, + {0x7e50, 0x7e54, 0x10, 6}, + {0x7e50, 0x7e54, 0x18, 21}, + {0x7e50, 0x7e54, 0x30, 32}, + {0x7e50, 0x7e54, 0x50, 22}, + {0x7e50, 0x7e54, 0x68, 12} +}; + +static const u32 t5_tp_mib_index_array[9][IREG_NUM_ELEM] = { + {0x7e50, 0x7e54, 0x0, 13}, + {0x7e50, 0x7e54, 0x10, 6}, + {0x7e50, 0x7e54, 0x18, 8}, + {0x7e50, 0x7e54, 0x20, 13}, + {0x7e50, 0x7e54, 0x30, 16}, + {0x7e50, 0x7e54, 0x40, 16}, + {0x7e50, 0x7e54, 0x50, 16}, + {0x7e50, 0x7e54, 0x60, 6}, + {0x7e50, 0x7e54, 0x68, 4} +}; + +static const u32 t5_sge_dbg_index_array[2][IREG_NUM_ELEM] = { + {0x10cc, 0x10d0, 0x0, 16}, + {0x10cc, 0x10d4, 0x0, 16}, +}; + +static const u32 t6_sge_qbase_index_array[] = { + /* 1 addr reg SGE_QBASE_INDEX and 4 data reg SGE_QBASE_MAP[0-3] */ + 0x1250, 0x1240, 0x1244, 0x1248, 0x124c, +}; + +static const u32 t5_pcie_pdbg_array[][IREG_NUM_ELEM] = { + {0x5a04, 0x5a0c, 0x00, 0x20}, /* t5_pcie_pdbg_regs_00_to_20 */ + {0x5a04, 0x5a0c, 0x21, 0x20}, /* t5_pcie_pdbg_regs_21_to_40 */ + {0x5a04, 0x5a0c, 0x41, 0x10}, /* t5_pcie_pdbg_regs_41_to_50 */ +}; + +static const u32 t5_pcie_cdbg_array[][IREG_NUM_ELEM] = { + {0x5a10, 0x5a18, 0x00, 0x20}, /* t5_pcie_cdbg_regs_00_to_20 */ + {0x5a10, 0x5a18, 0x21, 0x18}, /* t5_pcie_cdbg_regs_21_to_37 */ +}; + +static const u32 t5_pm_rx_array[][IREG_NUM_ELEM] = { + {0x8FD0, 0x8FD4, 0x10000, 0x20}, /* t5_pm_rx_regs_10000_to_10020 */ + {0x8FD0, 0x8FD4, 0x10021, 0x0D}, /* t5_pm_rx_regs_10021_to_1002c */ +}; + +static const u32 t5_pm_tx_array[][IREG_NUM_ELEM] = { + {0x8FF0, 0x8FF4, 0x10000, 0x20}, /* t5_pm_tx_regs_10000_to_10020 */ + {0x8FF0, 0x8FF4, 0x10021, 0x1D}, /* t5_pm_tx_regs_10021_to_1003c */ +}; + +static const u32 t5_pcie_config_array[][2] = { + {0x0, 0x34}, + {0x3c, 0x40}, + {0x50, 0x64}, + {0x70, 0x80}, + {0x94, 0xa0}, + {0xb0, 0xb8}, + {0xd0, 0xd4}, + {0x100, 0x128}, + {0x140, 0x148}, + {0x150, 0x164}, + {0x170, 0x178}, + {0x180, 0x194}, + {0x1a0, 0x1b8}, + {0x1c0, 0x208}, +}; + +static const u32 t6_ma_ireg_array[][IREG_NUM_ELEM] = { + {0x78f8, 0x78fc, 0xa000, 23}, /* t6_ma_regs_a000_to_a016 */ + {0x78f8, 0x78fc, 0xa400, 30}, /* t6_ma_regs_a400_to_a41e */ + {0x78f8, 0x78fc, 0xa800, 20} /* t6_ma_regs_a800_to_a813 */ +}; + +static const u32 t6_ma_ireg_array2[][IREG_NUM_ELEM] = { + {0x78f8, 0x78fc, 0xe400, 17}, /* t6_ma_regs_e400_to_e600 */ + {0x78f8, 0x78fc, 0xe640, 13} /* t6_ma_regs_e640_to_e7c0 */ +}; + +static const u32 t6_up_cim_reg_array[][IREG_NUM_ELEM + 1] = { + {0x7b50, 0x7b54, 0x2000, 0x20, 0}, /* up_cim_2000_to_207c */ + {0x7b50, 0x7b54, 0x2080, 0x1d, 0}, /* up_cim_2080_to_20fc */ + {0x7b50, 0x7b54, 0x00, 0x20, 0}, /* up_cim_00_to_7c */ + {0x7b50, 0x7b54, 0x80, 0x20, 0}, /* up_cim_80_to_fc */ + {0x7b50, 0x7b54, 0x100, 0x11, 0}, /* up_cim_100_to_14c */ + {0x7b50, 0x7b54, 0x200, 0x10, 0}, /* up_cim_200_to_23c */ + {0x7b50, 0x7b54, 0x240, 0x2, 0}, /* up_cim_240_to_244 */ + {0x7b50, 0x7b54, 0x250, 0x2, 0}, /* up_cim_250_to_254 */ + {0x7b50, 0x7b54, 0x260, 0x2, 0}, /* up_cim_260_to_264 */ + {0x7b50, 0x7b54, 0x270, 0x2, 0}, /* up_cim_270_to_274 */ + {0x7b50, 0x7b54, 0x280, 0x20, 0}, /* up_cim_280_to_2fc */ + {0x7b50, 0x7b54, 0x300, 0x20, 0}, /* up_cim_300_to_37c */ + {0x7b50, 0x7b54, 0x380, 0x14, 0}, /* up_cim_380_to_3cc */ + {0x7b50, 0x7b54, 0x4900, 0x4, 0x4}, /* up_cim_4900_to_4c60 */ + {0x7b50, 0x7b54, 0x4904, 0x4, 0x4}, /* up_cim_4904_to_4c64 */ + {0x7b50, 0x7b54, 0x4908, 0x4, 0x4}, /* up_cim_4908_to_4c68 */ + {0x7b50, 0x7b54, 0x4910, 0x4, 0x4}, /* up_cim_4910_to_4c70 */ + {0x7b50, 0x7b54, 0x4914, 0x4, 0x4}, /* up_cim_4914_to_4c74 */ + {0x7b50, 0x7b54, 0x4920, 0x10, 0x10}, /* up_cim_4920_to_4a10 */ + {0x7b50, 0x7b54, 0x4924, 0x10, 0x10}, /* up_cim_4924_to_4a14 */ + {0x7b50, 0x7b54, 0x4928, 0x10, 0x10}, /* up_cim_4928_to_4a18 */ + {0x7b50, 0x7b54, 0x492c, 0x10, 0x10}, /* up_cim_492c_to_4a1c */ +}; + +static const u32 t5_up_cim_reg_array[][IREG_NUM_ELEM + 1] = { + {0x7b50, 0x7b54, 0x2000, 0x20, 0}, /* up_cim_2000_to_207c */ + {0x7b50, 0x7b54, 0x2080, 0x19, 0}, /* up_cim_2080_to_20ec */ + {0x7b50, 0x7b54, 0x00, 0x20, 0}, /* up_cim_00_to_7c */ + {0x7b50, 0x7b54, 0x80, 0x20, 0}, /* up_cim_80_to_fc */ + {0x7b50, 0x7b54, 0x100, 0x11, 0}, /* up_cim_100_to_14c */ + {0x7b50, 0x7b54, 0x200, 0x10, 0}, /* up_cim_200_to_23c */ + {0x7b50, 0x7b54, 0x240, 0x2, 0}, /* up_cim_240_to_244 */ + {0x7b50, 0x7b54, 0x250, 0x2, 0}, /* up_cim_250_to_254 */ + {0x7b50, 0x7b54, 0x260, 0x2, 0}, /* up_cim_260_to_264 */ + {0x7b50, 0x7b54, 0x270, 0x2, 0}, /* up_cim_270_to_274 */ + {0x7b50, 0x7b54, 0x280, 0x20, 0}, /* up_cim_280_to_2fc */ + {0x7b50, 0x7b54, 0x300, 0x20, 0}, /* up_cim_300_to_37c */ + {0x7b50, 0x7b54, 0x380, 0x14, 0}, /* up_cim_380_to_3cc */ +}; + +static const u32 t6_hma_ireg_array[][IREG_NUM_ELEM] = { + {0x51320, 0x51324, 0xa000, 32} /* t6_hma_regs_a000_to_a01f */ +}; + +u32 cudbg_get_entity_length(struct adapter *adap, u32 entity) +{ + struct cudbg_tcam tcam_region = { 0 }; + u32 value, n = 0, len = 0; + + switch (entity) { + case CUDBG_REG_DUMP: + switch (CHELSIO_CHIP_VERSION(adap->params.chip)) { + case CHELSIO_T4: + len = T4_REGMAP_SIZE; + break; + case CHELSIO_T5: + case CHELSIO_T6: + len = T5_REGMAP_SIZE; + break; + default: + break; + } + break; + case CUDBG_DEV_LOG: + len = adap->params.devlog.size; + break; + case CUDBG_CIM_LA: + if (is_t6(adap->params.chip)) { + len = adap->params.cim_la_size / 10 + 1; + len *= 10 * sizeof(u32); + } else { + len = adap->params.cim_la_size / 8; + len *= 8 * sizeof(u32); + } + len += sizeof(u32); /* for reading CIM LA configuration */ + break; + case CUDBG_CIM_MA_LA: + len = 2 * CIM_MALA_SIZE * 5 * sizeof(u32); + break; + case CUDBG_CIM_QCFG: + len = sizeof(struct cudbg_cim_qcfg); + break; + case CUDBG_CIM_IBQ_TP0: + case CUDBG_CIM_IBQ_TP1: + case CUDBG_CIM_IBQ_ULP: + case CUDBG_CIM_IBQ_SGE0: + case CUDBG_CIM_IBQ_SGE1: + case CUDBG_CIM_IBQ_NCSI: + len = CIM_IBQ_SIZE * 4 * sizeof(u32); + break; + case CUDBG_CIM_OBQ_ULP0: + len = cudbg_cim_obq_size(adap, 0); + break; + case CUDBG_CIM_OBQ_ULP1: + len = cudbg_cim_obq_size(adap, 1); + break; + case CUDBG_CIM_OBQ_ULP2: + len = cudbg_cim_obq_size(adap, 2); + break; + case CUDBG_CIM_OBQ_ULP3: + len = cudbg_cim_obq_size(adap, 3); + break; + case CUDBG_CIM_OBQ_SGE: + len = cudbg_cim_obq_size(adap, 4); + break; + case CUDBG_CIM_OBQ_NCSI: + len = cudbg_cim_obq_size(adap, 5); + break; + case CUDBG_CIM_OBQ_RXQ0: + len = cudbg_cim_obq_size(adap, 6); + break; + case CUDBG_CIM_OBQ_RXQ1: + len = cudbg_cim_obq_size(adap, 7); + break; + case CUDBG_EDC0: + value = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A); + if (value & EDRAM0_ENABLE_F) { + value = t4_read_reg(adap, MA_EDRAM0_BAR_A); + len = EDRAM0_SIZE_G(value); + } + len = cudbg_mbytes_to_bytes(len); + break; + case CUDBG_EDC1: + value = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A); + if (value & EDRAM1_ENABLE_F) { + value = t4_read_reg(adap, MA_EDRAM1_BAR_A); + len = EDRAM1_SIZE_G(value); + } + len = cudbg_mbytes_to_bytes(len); + break; + case CUDBG_MC0: + value = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A); + if (value & EXT_MEM0_ENABLE_F) { + value = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A); + len = EXT_MEM0_SIZE_G(value); + } + len = cudbg_mbytes_to_bytes(len); + break; + case CUDBG_MC1: + value = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A); + if (value & EXT_MEM1_ENABLE_F) { + value = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); + len = EXT_MEM1_SIZE_G(value); + } + len = cudbg_mbytes_to_bytes(len); + break; + case CUDBG_RSS: + len = t4_chip_rss_size(adap) * sizeof(u16); + break; + case CUDBG_RSS_VF_CONF: + len = adap->params.arch.vfcount * + sizeof(struct cudbg_rss_vf_conf); + break; + case CUDBG_PATH_MTU: + len = NMTUS * sizeof(u16); + break; + case CUDBG_PM_STATS: + len = sizeof(struct cudbg_pm_stats); + break; + case CUDBG_HW_SCHED: + len = sizeof(struct cudbg_hw_sched); + break; + case CUDBG_TP_INDIRECT: + switch (CHELSIO_CHIP_VERSION(adap->params.chip)) { + case CHELSIO_T5: + n = sizeof(t5_tp_pio_array) + + sizeof(t5_tp_tm_pio_array) + + sizeof(t5_tp_mib_index_array); + break; + case CHELSIO_T6: + n = sizeof(t6_tp_pio_array) + + sizeof(t6_tp_tm_pio_array) + + sizeof(t6_tp_mib_index_array); + break; + default: + break; + } + n = n / (IREG_NUM_ELEM * sizeof(u32)); + len = sizeof(struct ireg_buf) * n; + break; + case CUDBG_SGE_INDIRECT: + len = sizeof(struct ireg_buf) * 2 + + sizeof(struct sge_qbase_reg_field); + break; + case CUDBG_ULPRX_LA: + len = sizeof(struct cudbg_ulprx_la); + break; + case CUDBG_TP_LA: + len = sizeof(struct cudbg_tp_la) + TPLA_SIZE * sizeof(u64); + break; + case CUDBG_MEMINFO: + len = sizeof(struct cudbg_ver_hdr) + + sizeof(struct cudbg_meminfo); + break; + case CUDBG_CIM_PIF_LA: + len = sizeof(struct cudbg_cim_pif_la); + len += 2 * CIM_PIFLA_SIZE * 6 * sizeof(u32); + break; + case CUDBG_CLK: + len = sizeof(struct cudbg_clk_info); + break; + case CUDBG_PCIE_INDIRECT: + n = sizeof(t5_pcie_pdbg_array) / (IREG_NUM_ELEM * sizeof(u32)); + len = sizeof(struct ireg_buf) * n * 2; + break; + case CUDBG_PM_INDIRECT: + n = sizeof(t5_pm_rx_array) / (IREG_NUM_ELEM * sizeof(u32)); + len = sizeof(struct ireg_buf) * n * 2; + break; + case CUDBG_TID_INFO: + len = sizeof(struct cudbg_tid_info_region_rev1); + break; + case CUDBG_PCIE_CONFIG: + len = sizeof(u32) * CUDBG_NUM_PCIE_CONFIG_REGS; + break; + case CUDBG_DUMP_CONTEXT: + len = cudbg_dump_context_size(adap); + break; + case CUDBG_MPS_TCAM: + len = sizeof(struct cudbg_mps_tcam) * + adap->params.arch.mps_tcam_size; + break; + case CUDBG_VPD_DATA: + len = sizeof(struct cudbg_vpd_data); + break; + case CUDBG_LE_TCAM: + cudbg_fill_le_tcam_info(adap, &tcam_region); + len = sizeof(struct cudbg_tcam) + + sizeof(struct cudbg_tid_data) * tcam_region.max_tid; + break; + case CUDBG_CCTRL: + len = sizeof(u16) * NMTUS * NCCTRL_WIN; + break; + case CUDBG_MA_INDIRECT: + if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) { + n = sizeof(t6_ma_ireg_array) / + (IREG_NUM_ELEM * sizeof(u32)); + len = sizeof(struct ireg_buf) * n * 2; + } + break; + case CUDBG_ULPTX_LA: + len = sizeof(struct cudbg_ver_hdr) + + sizeof(struct cudbg_ulptx_la); + break; + case CUDBG_UP_CIM_INDIRECT: + n = 0; + if (is_t5(adap->params.chip)) + n = sizeof(t5_up_cim_reg_array) / + ((IREG_NUM_ELEM + 1) * sizeof(u32)); + else if (is_t6(adap->params.chip)) + n = sizeof(t6_up_cim_reg_array) / + ((IREG_NUM_ELEM + 1) * sizeof(u32)); + len = sizeof(struct ireg_buf) * n; + break; + case CUDBG_PBT_TABLE: + len = sizeof(struct cudbg_pbt_tables); + break; + case CUDBG_MBOX_LOG: + len = sizeof(struct cudbg_mbox_log) * adap->mbox_log->size; + break; + case CUDBG_HMA_INDIRECT: + if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) { + n = sizeof(t6_hma_ireg_array) / + (IREG_NUM_ELEM * sizeof(u32)); + len = sizeof(struct ireg_buf) * n; + } + break; + case CUDBG_HMA: + value = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A); + if (value & HMA_MUX_F) { + /* In T6, there's no MC1. So, HMA shares MC1 + * address space. + */ + value = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); + len = EXT_MEM1_SIZE_G(value); + } + len = cudbg_mbytes_to_bytes(len); + break; + case CUDBG_QDESC: + cudbg_fill_qdesc_num_and_size(adap, NULL, &len); + break; + default: + break; + } + + return len; +} + +static int cudbg_do_compression(struct cudbg_init *pdbg_init, + struct cudbg_buffer *pin_buff, + struct cudbg_buffer *dbg_buff) +{ + struct cudbg_buffer temp_in_buff = { 0 }; + int bytes_left, bytes_read, bytes; + u32 offset = dbg_buff->offset; + int rc; + + temp_in_buff.offset = pin_buff->offset; + temp_in_buff.data = pin_buff->data; + temp_in_buff.size = pin_buff->size; + + bytes_left = pin_buff->size; + bytes_read = 0; + while (bytes_left > 0) { + /* Do compression in smaller chunks */ + bytes = min_t(unsigned long, bytes_left, + (unsigned long)CUDBG_CHUNK_SIZE); + temp_in_buff.data = (char *)pin_buff->data + bytes_read; + temp_in_buff.size = bytes; + rc = cudbg_compress_buff(pdbg_init, &temp_in_buff, dbg_buff); + if (rc) + return rc; + bytes_left -= bytes; + bytes_read += bytes; + } + + pin_buff->size = dbg_buff->offset - offset; + return 0; +} + +static int cudbg_write_and_release_buff(struct cudbg_init *pdbg_init, + struct cudbg_buffer *pin_buff, + struct cudbg_buffer *dbg_buff) +{ + int rc = 0; + + if (pdbg_init->compress_type == CUDBG_COMPRESSION_NONE) { + cudbg_update_buff(pin_buff, dbg_buff); + } else { + rc = cudbg_do_compression(pdbg_init, pin_buff, dbg_buff); + if (rc) + goto out; + } + +out: + cudbg_put_buff(pdbg_init, pin_buff); + return rc; +} + +static int is_fw_attached(struct cudbg_init *pdbg_init) +{ + struct adapter *padap = pdbg_init->adap; + + if (!(padap->flags & CXGB4_FW_OK) || padap->use_bd) + return 0; + + return 1; +} + +/* This function will add additional padding bytes into debug_buffer to make it + * 4 byte aligned. + */ +void cudbg_align_debug_buffer(struct cudbg_buffer *dbg_buff, + struct cudbg_entity_hdr *entity_hdr) +{ + u8 zero_buf[4] = {0}; + u8 padding, remain; + + remain = (dbg_buff->offset - entity_hdr->start_offset) % 4; + padding = 4 - remain; + if (remain) { + memcpy(((u8 *)dbg_buff->data) + dbg_buff->offset, &zero_buf, + padding); + dbg_buff->offset += padding; + entity_hdr->num_pad = padding; + } + entity_hdr->size = dbg_buff->offset - entity_hdr->start_offset; +} + +struct cudbg_entity_hdr *cudbg_get_entity_hdr(void *outbuf, int i) +{ + struct cudbg_hdr *cudbg_hdr = (struct cudbg_hdr *)outbuf; + + return (struct cudbg_entity_hdr *) + ((char *)outbuf + cudbg_hdr->hdr_len + + (sizeof(struct cudbg_entity_hdr) * (i - 1))); +} + +static int cudbg_read_vpd_reg(struct adapter *padap, u32 addr, u32 len, + void *dest) +{ + int vaddr, rc; + + vaddr = t4_eeprom_ptov(addr, padap->pf, EEPROMPFSIZE); + if (vaddr < 0) + return vaddr; + + rc = pci_read_vpd(padap->pdev, vaddr, len, dest); + if (rc < 0) + return rc; + + return 0; +} + +static int cudbg_mem_desc_cmp(const void *a, const void *b) +{ + return ((const struct cudbg_mem_desc *)a)->base - + ((const struct cudbg_mem_desc *)b)->base; +} + +int cudbg_fill_meminfo(struct adapter *padap, + struct cudbg_meminfo *meminfo_buff) +{ + struct cudbg_mem_desc *md; + u32 lo, hi, used, alloc; + int n, i; + + memset(meminfo_buff->avail, 0, + ARRAY_SIZE(meminfo_buff->avail) * + sizeof(struct cudbg_mem_desc)); + memset(meminfo_buff->mem, 0, + (ARRAY_SIZE(cudbg_region) + 3) * sizeof(struct cudbg_mem_desc)); + md = meminfo_buff->mem; + + for (i = 0; i < ARRAY_SIZE(meminfo_buff->mem); i++) { + meminfo_buff->mem[i].limit = 0; + meminfo_buff->mem[i].idx = i; + } + + /* Find and sort the populated memory ranges */ + i = 0; + lo = t4_read_reg(padap, MA_TARGET_MEM_ENABLE_A); + if (lo & EDRAM0_ENABLE_F) { + hi = t4_read_reg(padap, MA_EDRAM0_BAR_A); + meminfo_buff->avail[i].base = + cudbg_mbytes_to_bytes(EDRAM0_BASE_G(hi)); + meminfo_buff->avail[i].limit = + meminfo_buff->avail[i].base + + cudbg_mbytes_to_bytes(EDRAM0_SIZE_G(hi)); + meminfo_buff->avail[i].idx = 0; + i++; + } + + if (lo & EDRAM1_ENABLE_F) { + hi = t4_read_reg(padap, MA_EDRAM1_BAR_A); + meminfo_buff->avail[i].base = + cudbg_mbytes_to_bytes(EDRAM1_BASE_G(hi)); + meminfo_buff->avail[i].limit = + meminfo_buff->avail[i].base + + cudbg_mbytes_to_bytes(EDRAM1_SIZE_G(hi)); + meminfo_buff->avail[i].idx = 1; + i++; + } + + if (is_t5(padap->params.chip)) { + if (lo & EXT_MEM0_ENABLE_F) { + hi = t4_read_reg(padap, MA_EXT_MEMORY0_BAR_A); + meminfo_buff->avail[i].base = + cudbg_mbytes_to_bytes(EXT_MEM_BASE_G(hi)); + meminfo_buff->avail[i].limit = + meminfo_buff->avail[i].base + + cudbg_mbytes_to_bytes(EXT_MEM_SIZE_G(hi)); + meminfo_buff->avail[i].idx = 3; + i++; + } + + if (lo & EXT_MEM1_ENABLE_F) { + hi = t4_read_reg(padap, MA_EXT_MEMORY1_BAR_A); + meminfo_buff->avail[i].base = + cudbg_mbytes_to_bytes(EXT_MEM1_BASE_G(hi)); + meminfo_buff->avail[i].limit = + meminfo_buff->avail[i].base + + cudbg_mbytes_to_bytes(EXT_MEM1_SIZE_G(hi)); + meminfo_buff->avail[i].idx = 4; + i++; + } + } else { + if (lo & EXT_MEM_ENABLE_F) { + hi = t4_read_reg(padap, MA_EXT_MEMORY_BAR_A); + meminfo_buff->avail[i].base = + cudbg_mbytes_to_bytes(EXT_MEM_BASE_G(hi)); + meminfo_buff->avail[i].limit = + meminfo_buff->avail[i].base + + cudbg_mbytes_to_bytes(EXT_MEM_SIZE_G(hi)); + meminfo_buff->avail[i].idx = 2; + i++; + } + + if (lo & HMA_MUX_F) { + hi = t4_read_reg(padap, MA_EXT_MEMORY1_BAR_A); + meminfo_buff->avail[i].base = + cudbg_mbytes_to_bytes(EXT_MEM1_BASE_G(hi)); + meminfo_buff->avail[i].limit = + meminfo_buff->avail[i].base + + cudbg_mbytes_to_bytes(EXT_MEM1_SIZE_G(hi)); + meminfo_buff->avail[i].idx = 5; + i++; + } + } + + if (!i) /* no memory available */ + return CUDBG_STATUS_ENTITY_NOT_FOUND; + + meminfo_buff->avail_c = i; + sort(meminfo_buff->avail, i, sizeof(struct cudbg_mem_desc), + cudbg_mem_desc_cmp, NULL); + (md++)->base = t4_read_reg(padap, SGE_DBQ_CTXT_BADDR_A); + (md++)->base = t4_read_reg(padap, SGE_IMSG_CTXT_BADDR_A); + (md++)->base = t4_read_reg(padap, SGE_FLM_CACHE_BADDR_A); + (md++)->base = t4_read_reg(padap, TP_CMM_TCB_BASE_A); + (md++)->base = t4_read_reg(padap, TP_CMM_MM_BASE_A); + (md++)->base = t4_read_reg(padap, TP_CMM_TIMER_BASE_A); + (md++)->base = t4_read_reg(padap, TP_CMM_MM_RX_FLST_BASE_A); + (md++)->base = t4_read_reg(padap, TP_CMM_MM_TX_FLST_BASE_A); + (md++)->base = t4_read_reg(padap, TP_CMM_MM_PS_FLST_BASE_A); + + /* the next few have explicit upper bounds */ + md->base = t4_read_reg(padap, TP_PMM_TX_BASE_A); + md->limit = md->base - 1 + + t4_read_reg(padap, TP_PMM_TX_PAGE_SIZE_A) * + PMTXMAXPAGE_G(t4_read_reg(padap, TP_PMM_TX_MAX_PAGE_A)); + md++; + + md->base = t4_read_reg(padap, TP_PMM_RX_BASE_A); + md->limit = md->base - 1 + + t4_read_reg(padap, TP_PMM_RX_PAGE_SIZE_A) * + PMRXMAXPAGE_G(t4_read_reg(padap, TP_PMM_RX_MAX_PAGE_A)); + md++; + + if (t4_read_reg(padap, LE_DB_CONFIG_A) & HASHEN_F) { + if (CHELSIO_CHIP_VERSION(padap->params.chip) <= CHELSIO_T5) { + hi = t4_read_reg(padap, LE_DB_TID_HASHBASE_A) / 4; + md->base = t4_read_reg(padap, LE_DB_HASH_TID_BASE_A); + } else { + hi = t4_read_reg(padap, LE_DB_HASH_TID_BASE_A); + md->base = t4_read_reg(padap, + LE_DB_HASH_TBL_BASE_ADDR_A); + } + md->limit = 0; + } else { + md->base = 0; + md->idx = ARRAY_SIZE(cudbg_region); /* hide it */ + } + md++; + +#define ulp_region(reg) do { \ + md->base = t4_read_reg(padap, ULP_ ## reg ## _LLIMIT_A);\ + (md++)->limit = t4_read_reg(padap, ULP_ ## reg ## _ULIMIT_A);\ +} while (0) + + ulp_region(RX_ISCSI); + ulp_region(RX_TDDP); + ulp_region(TX_TPT); + ulp_region(RX_STAG); + ulp_region(RX_RQ); + ulp_region(RX_RQUDP); + ulp_region(RX_PBL); + ulp_region(TX_PBL); +#undef ulp_region + md->base = 0; + md->idx = ARRAY_SIZE(cudbg_region); + if (!is_t4(padap->params.chip)) { + u32 fifo_size = t4_read_reg(padap, SGE_DBVFIFO_SIZE_A); + u32 sge_ctrl = t4_read_reg(padap, SGE_CONTROL2_A); + u32 size = 0; + + if (is_t5(padap->params.chip)) { + if (sge_ctrl & VFIFO_ENABLE_F) + size = DBVFIFO_SIZE_G(fifo_size); + } else { + size = T6_DBVFIFO_SIZE_G(fifo_size); + } + + if (size) { + md->base = BASEADDR_G(t4_read_reg(padap, + SGE_DBVFIFO_BADDR_A)); + md->limit = md->base + (size << 2) - 1; + } + } + + md++; + + md->base = t4_read_reg(padap, ULP_RX_CTX_BASE_A); + md->limit = 0; + md++; + md->base = t4_read_reg(padap, ULP_TX_ERR_TABLE_BASE_A); + md->limit = 0; + md++; + + md->base = padap->vres.ocq.start; + if (padap->vres.ocq.size) + md->limit = md->base + padap->vres.ocq.size - 1; + else + md->idx = ARRAY_SIZE(cudbg_region); /* hide it */ + md++; + + /* add any address-space holes, there can be up to 3 */ + for (n = 0; n < i - 1; n++) + if (meminfo_buff->avail[n].limit < + meminfo_buff->avail[n + 1].base) + (md++)->base = meminfo_buff->avail[n].limit; + + if (meminfo_buff->avail[n].limit) + (md++)->base = meminfo_buff->avail[n].limit; + + n = md - meminfo_buff->mem; + meminfo_buff->mem_c = n; + + sort(meminfo_buff->mem, n, sizeof(struct cudbg_mem_desc), + cudbg_mem_desc_cmp, NULL); + + lo = t4_read_reg(padap, CIM_SDRAM_BASE_ADDR_A); + hi = t4_read_reg(padap, CIM_SDRAM_ADDR_SIZE_A) + lo - 1; + meminfo_buff->up_ram_lo = lo; + meminfo_buff->up_ram_hi = hi; + + lo = t4_read_reg(padap, CIM_EXTMEM2_BASE_ADDR_A); + hi = t4_read_reg(padap, CIM_EXTMEM2_ADDR_SIZE_A) + lo - 1; + meminfo_buff->up_extmem2_lo = lo; + meminfo_buff->up_extmem2_hi = hi; + + lo = t4_read_reg(padap, TP_PMM_RX_MAX_PAGE_A); + for (i = 0, meminfo_buff->free_rx_cnt = 0; i < 2; i++) + meminfo_buff->free_rx_cnt += + FREERXPAGECOUNT_G(t4_read_reg(padap, + TP_FLM_FREE_RX_CNT_A)); + + meminfo_buff->rx_pages_data[0] = PMRXMAXPAGE_G(lo); + meminfo_buff->rx_pages_data[1] = + t4_read_reg(padap, TP_PMM_RX_PAGE_SIZE_A) >> 10; + meminfo_buff->rx_pages_data[2] = (lo & PMRXNUMCHN_F) ? 2 : 1; + + lo = t4_read_reg(padap, TP_PMM_TX_MAX_PAGE_A); + hi = t4_read_reg(padap, TP_PMM_TX_PAGE_SIZE_A); + for (i = 0, meminfo_buff->free_tx_cnt = 0; i < 4; i++) + meminfo_buff->free_tx_cnt += + FREETXPAGECOUNT_G(t4_read_reg(padap, + TP_FLM_FREE_TX_CNT_A)); + + meminfo_buff->tx_pages_data[0] = PMTXMAXPAGE_G(lo); + meminfo_buff->tx_pages_data[1] = + hi >= (1 << 20) ? (hi >> 20) : (hi >> 10); + meminfo_buff->tx_pages_data[2] = + hi >= (1 << 20) ? 'M' : 'K'; + meminfo_buff->tx_pages_data[3] = 1 << PMTXNUMCHN_G(lo); + + meminfo_buff->p_structs = t4_read_reg(padap, TP_CMM_MM_MAX_PSTRUCT_A); + meminfo_buff->p_structs_free_cnt = + FREEPSTRUCTCOUNT_G(t4_read_reg(padap, TP_FLM_FREE_PS_CNT_A)); + + for (i = 0; i < 4; i++) { + if (CHELSIO_CHIP_VERSION(padap->params.chip) > CHELSIO_T5) + lo = t4_read_reg(padap, + MPS_RX_MAC_BG_PG_CNT0_A + i * 4); + else + lo = t4_read_reg(padap, MPS_RX_PG_RSV0_A + i * 4); + if (is_t5(padap->params.chip)) { + used = T5_USED_G(lo); + alloc = T5_ALLOC_G(lo); + } else { + used = USED_G(lo); + alloc = ALLOC_G(lo); + } + meminfo_buff->port_used[i] = used; + meminfo_buff->port_alloc[i] = alloc; + } + + for (i = 0; i < padap->params.arch.nchan; i++) { + if (CHELSIO_CHIP_VERSION(padap->params.chip) > CHELSIO_T5) + lo = t4_read_reg(padap, + MPS_RX_LPBK_BG_PG_CNT0_A + i * 4); + else + lo = t4_read_reg(padap, MPS_RX_PG_RSV4_A + i * 4); + if (is_t5(padap->params.chip)) { + used = T5_USED_G(lo); + alloc = T5_ALLOC_G(lo); + } else { + used = USED_G(lo); + alloc = ALLOC_G(lo); + } + meminfo_buff->loopback_used[i] = used; + meminfo_buff->loopback_alloc[i] = alloc; + } + + return 0; +} + +int cudbg_collect_reg_dump(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + u32 buf_size = 0; + int rc = 0; + + if (is_t4(padap->params.chip)) + buf_size = T4_REGMAP_SIZE; + else if (is_t5(padap->params.chip) || is_t6(padap->params.chip)) + buf_size = T5_REGMAP_SIZE; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, buf_size, &temp_buff); + if (rc) + return rc; + t4_get_regs(padap, (void *)temp_buff.data, temp_buff.size); + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_fw_devlog(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct devlog_params *dparams; + int rc = 0; + + rc = t4_init_devlog_params(padap); + if (rc < 0) { + cudbg_err->sys_err = rc; + return rc; + } + + dparams = &padap->params.devlog; + rc = cudbg_get_buff(pdbg_init, dbg_buff, dparams->size, &temp_buff); + if (rc) + return rc; + + /* Collect FW devlog */ + if (dparams->start != 0) { + spin_lock(&padap->win0_lock); + rc = t4_memory_rw(padap, padap->params.drv_memwin, + dparams->memtype, dparams->start, + dparams->size, + (__be32 *)(char *)temp_buff.data, + 1); + spin_unlock(&padap->win0_lock); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_cim_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + int size, rc; + u32 cfg = 0; + + if (is_t6(padap->params.chip)) { + size = padap->params.cim_la_size / 10 + 1; + size *= 10 * sizeof(u32); + } else { + size = padap->params.cim_la_size / 8; + size *= 8 * sizeof(u32); + } + + size += sizeof(cfg); + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + rc = t4_cim_read(padap, UP_UP_DBG_LA_CFG_A, 1, &cfg); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + + memcpy((char *)temp_buff.data, &cfg, sizeof(cfg)); + rc = t4_cim_read_la(padap, + (u32 *)((char *)temp_buff.data + sizeof(cfg)), + NULL); + if (rc < 0) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_cim_ma_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + int size, rc; + + size = 2 * CIM_MALA_SIZE * 5 * sizeof(u32); + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + t4_cim_read_ma_la(padap, + (u32 *)temp_buff.data, + (u32 *)((char *)temp_buff.data + + 5 * CIM_MALA_SIZE)); + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_cim_qcfg(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_cim_qcfg *cim_qcfg_data; + int rc; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_cim_qcfg), + &temp_buff); + if (rc) + return rc; + + cim_qcfg_data = (struct cudbg_cim_qcfg *)temp_buff.data; + cim_qcfg_data->chip = padap->params.chip; + rc = t4_cim_read(padap, UP_IBQ_0_RDADDR_A, + ARRAY_SIZE(cim_qcfg_data->stat), cim_qcfg_data->stat); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + + rc = t4_cim_read(padap, UP_OBQ_0_REALADDR_A, + ARRAY_SIZE(cim_qcfg_data->obq_wr), + cim_qcfg_data->obq_wr); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + + t4_read_cimq_cfg(padap, cim_qcfg_data->base, cim_qcfg_data->size, + cim_qcfg_data->thres); + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +static int cudbg_read_cim_ibq(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err, int qid) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + int no_of_read_words, rc = 0; + u32 qsize; + + /* collect CIM IBQ */ + qsize = CIM_IBQ_SIZE * 4 * sizeof(u32); + rc = cudbg_get_buff(pdbg_init, dbg_buff, qsize, &temp_buff); + if (rc) + return rc; + + /* t4_read_cim_ibq will return no. of read words or error */ + no_of_read_words = t4_read_cim_ibq(padap, qid, + (u32 *)temp_buff.data, qsize); + /* no_of_read_words is less than or equal to 0 means error */ + if (no_of_read_words <= 0) { + if (!no_of_read_words) + rc = CUDBG_SYSTEM_ERROR; + else + rc = no_of_read_words; + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_cim_ibq_tp0(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_ibq(pdbg_init, dbg_buff, cudbg_err, 0); +} + +int cudbg_collect_cim_ibq_tp1(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_ibq(pdbg_init, dbg_buff, cudbg_err, 1); +} + +int cudbg_collect_cim_ibq_ulp(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_ibq(pdbg_init, dbg_buff, cudbg_err, 2); +} + +int cudbg_collect_cim_ibq_sge0(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_ibq(pdbg_init, dbg_buff, cudbg_err, 3); +} + +int cudbg_collect_cim_ibq_sge1(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_ibq(pdbg_init, dbg_buff, cudbg_err, 4); +} + +int cudbg_collect_cim_ibq_ncsi(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_ibq(pdbg_init, dbg_buff, cudbg_err, 5); +} + +u32 cudbg_cim_obq_size(struct adapter *padap, int qid) +{ + u32 value; + + t4_write_reg(padap, CIM_QUEUE_CONFIG_REF_A, OBQSELECT_F | + QUENUMSELECT_V(qid)); + value = t4_read_reg(padap, CIM_QUEUE_CONFIG_CTRL_A); + value = CIMQSIZE_G(value) * 64; /* size in number of words */ + return value * sizeof(u32); +} + +static int cudbg_read_cim_obq(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err, int qid) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + int no_of_read_words, rc = 0; + u32 qsize; + + /* collect CIM OBQ */ + qsize = cudbg_cim_obq_size(padap, qid); + rc = cudbg_get_buff(pdbg_init, dbg_buff, qsize, &temp_buff); + if (rc) + return rc; + + /* t4_read_cim_obq will return no. of read words or error */ + no_of_read_words = t4_read_cim_obq(padap, qid, + (u32 *)temp_buff.data, qsize); + /* no_of_read_words is less than or equal to 0 means error */ + if (no_of_read_words <= 0) { + if (!no_of_read_words) + rc = CUDBG_SYSTEM_ERROR; + else + rc = no_of_read_words; + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_cim_obq_ulp0(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_obq(pdbg_init, dbg_buff, cudbg_err, 0); +} + +int cudbg_collect_cim_obq_ulp1(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_obq(pdbg_init, dbg_buff, cudbg_err, 1); +} + +int cudbg_collect_cim_obq_ulp2(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_obq(pdbg_init, dbg_buff, cudbg_err, 2); +} + +int cudbg_collect_cim_obq_ulp3(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_obq(pdbg_init, dbg_buff, cudbg_err, 3); +} + +int cudbg_collect_cim_obq_sge(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_obq(pdbg_init, dbg_buff, cudbg_err, 4); +} + +int cudbg_collect_cim_obq_ncsi(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_obq(pdbg_init, dbg_buff, cudbg_err, 5); +} + +int cudbg_collect_obq_sge_rx_q0(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_obq(pdbg_init, dbg_buff, cudbg_err, 6); +} + +int cudbg_collect_obq_sge_rx_q1(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_read_cim_obq(pdbg_init, dbg_buff, cudbg_err, 7); +} + +static int cudbg_meminfo_get_mem_index(struct adapter *padap, + struct cudbg_meminfo *mem_info, + u8 mem_type, u8 *idx) +{ + u8 i, flag; + + switch (mem_type) { + case MEM_EDC0: + flag = EDC0_FLAG; + break; + case MEM_EDC1: + flag = EDC1_FLAG; + break; + case MEM_MC0: + /* Some T5 cards have both MC0 and MC1. */ + flag = is_t5(padap->params.chip) ? MC0_FLAG : MC_FLAG; + break; + case MEM_MC1: + flag = MC1_FLAG; + break; + case MEM_HMA: + flag = HMA_FLAG; + break; + default: + return CUDBG_STATUS_ENTITY_NOT_FOUND; + } + + for (i = 0; i < mem_info->avail_c; i++) { + if (mem_info->avail[i].idx == flag) { + *idx = i; + return 0; + } + } + + return CUDBG_STATUS_ENTITY_NOT_FOUND; +} + +/* Fetch the @region_name's start and end from @meminfo. */ +static int cudbg_get_mem_region(struct adapter *padap, + struct cudbg_meminfo *meminfo, + u8 mem_type, const char *region_name, + struct cudbg_mem_desc *mem_desc) +{ + u8 mc, found = 0; + u32 idx = 0; + int rc, i; + + rc = cudbg_meminfo_get_mem_index(padap, meminfo, mem_type, &mc); + if (rc) + return rc; + + i = match_string(cudbg_region, ARRAY_SIZE(cudbg_region), region_name); + if (i < 0) + return -EINVAL; + + idx = i; + for (i = 0; i < meminfo->mem_c; i++) { + if (meminfo->mem[i].idx >= ARRAY_SIZE(cudbg_region)) + continue; /* Skip holes */ + + if (!(meminfo->mem[i].limit)) + meminfo->mem[i].limit = + i < meminfo->mem_c - 1 ? + meminfo->mem[i + 1].base - 1 : ~0; + + if (meminfo->mem[i].idx == idx) { + /* Check if the region exists in @mem_type memory */ + if (meminfo->mem[i].base < meminfo->avail[mc].base && + meminfo->mem[i].limit < meminfo->avail[mc].base) + return -EINVAL; + + if (meminfo->mem[i].base > meminfo->avail[mc].limit) + return -EINVAL; + + memcpy(mem_desc, &meminfo->mem[i], + sizeof(struct cudbg_mem_desc)); + found = 1; + break; + } + } + if (!found) + return -EINVAL; + + return 0; +} + +/* Fetch and update the start and end of the requested memory region w.r.t 0 + * in the corresponding EDC/MC/HMA. + */ +static int cudbg_get_mem_relative(struct adapter *padap, + struct cudbg_meminfo *meminfo, + u8 mem_type, u32 *out_base, u32 *out_end) +{ + u8 mc_idx; + int rc; + + rc = cudbg_meminfo_get_mem_index(padap, meminfo, mem_type, &mc_idx); + if (rc) + return rc; + + if (*out_base < meminfo->avail[mc_idx].base) + *out_base = 0; + else + *out_base -= meminfo->avail[mc_idx].base; + + if (*out_end > meminfo->avail[mc_idx].limit) + *out_end = meminfo->avail[mc_idx].limit; + else + *out_end -= meminfo->avail[mc_idx].base; + + return 0; +} + +/* Get TX and RX Payload region */ +static int cudbg_get_payload_range(struct adapter *padap, u8 mem_type, + const char *region_name, + struct cudbg_region_info *payload) +{ + struct cudbg_mem_desc mem_desc = { 0 }; + struct cudbg_meminfo meminfo; + int rc; + + rc = cudbg_fill_meminfo(padap, &meminfo); + if (rc) + return rc; + + rc = cudbg_get_mem_region(padap, &meminfo, mem_type, region_name, + &mem_desc); + if (rc) { + payload->exist = false; + return 0; + } + + payload->exist = true; + payload->start = mem_desc.base; + payload->end = mem_desc.limit; + + return cudbg_get_mem_relative(padap, &meminfo, mem_type, + &payload->start, &payload->end); +} + +static int cudbg_memory_read(struct cudbg_init *pdbg_init, int win, + int mtype, u32 addr, u32 len, void *hbuf) +{ + u32 win_pf, memoffset, mem_aperture, mem_base; + struct adapter *adap = pdbg_init->adap; + u32 pos, offset, resid; + u32 *res_buf; + u64 *buf; + int ret; + + /* Argument sanity checks ... + */ + if (addr & 0x3 || (uintptr_t)hbuf & 0x3) + return -EINVAL; + + buf = (u64 *)hbuf; + + /* Try to do 64-bit reads. Residual will be handled later. */ + resid = len & 0x7; + len -= resid; + + ret = t4_memory_rw_init(adap, win, mtype, &memoffset, &mem_base, + &mem_aperture); + if (ret) + return ret; + + addr = addr + memoffset; + win_pf = is_t4(adap->params.chip) ? 0 : PFNUM_V(adap->pf); + + pos = addr & ~(mem_aperture - 1); + offset = addr - pos; + + /* Set up initial PCI-E Memory Window to cover the start of our + * transfer. + */ + t4_memory_update_win(adap, win, pos | win_pf); + + /* Transfer data from the adapter */ + while (len > 0) { + *buf++ = le64_to_cpu((__force __le64) + t4_read_reg64(adap, mem_base + offset)); + offset += sizeof(u64); + len -= sizeof(u64); + + /* If we've reached the end of our current window aperture, + * move the PCI-E Memory Window on to the next. + */ + if (offset == mem_aperture) { + pos += mem_aperture; + offset = 0; + t4_memory_update_win(adap, win, pos | win_pf); + } + } + + res_buf = (u32 *)buf; + /* Read residual in 32-bit multiples */ + while (resid > sizeof(u32)) { + *res_buf++ = le32_to_cpu((__force __le32) + t4_read_reg(adap, mem_base + offset)); + offset += sizeof(u32); + resid -= sizeof(u32); + + /* If we've reached the end of our current window aperture, + * move the PCI-E Memory Window on to the next. + */ + if (offset == mem_aperture) { + pos += mem_aperture; + offset = 0; + t4_memory_update_win(adap, win, pos | win_pf); + } + } + + /* Transfer residual < 32-bits */ + if (resid) + t4_memory_rw_residual(adap, resid, mem_base + offset, + (u8 *)res_buf, T4_MEMORY_READ); + + return 0; +} + +#define CUDBG_YIELD_ITERATION 256 + +static int cudbg_read_fw_mem(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, u8 mem_type, + unsigned long tot_len, + struct cudbg_error *cudbg_err) +{ + static const char * const region_name[] = { "Tx payload:", + "Rx payload:" }; + unsigned long bytes, bytes_left, bytes_read = 0; + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_region_info payload[2]; + u32 yield_count = 0; + int rc = 0; + u8 i; + + /* Get TX/RX Payload region range if they exist */ + memset(payload, 0, sizeof(payload)); + for (i = 0; i < ARRAY_SIZE(region_name); i++) { + rc = cudbg_get_payload_range(padap, mem_type, region_name[i], + &payload[i]); + if (rc) + return rc; + + if (payload[i].exist) { + /* Align start and end to avoid wrap around */ + payload[i].start = roundup(payload[i].start, + CUDBG_CHUNK_SIZE); + payload[i].end = rounddown(payload[i].end, + CUDBG_CHUNK_SIZE); + } + } + + bytes_left = tot_len; + while (bytes_left > 0) { + /* As MC size is huge and read through PIO access, this + * loop will hold cpu for a longer time. OS may think that + * the process is hanged and will generate CPU stall traces. + * So yield the cpu regularly. + */ + yield_count++; + if (!(yield_count % CUDBG_YIELD_ITERATION)) + schedule(); + + bytes = min_t(unsigned long, bytes_left, + (unsigned long)CUDBG_CHUNK_SIZE); + rc = cudbg_get_buff(pdbg_init, dbg_buff, bytes, &temp_buff); + if (rc) + return rc; + + for (i = 0; i < ARRAY_SIZE(payload); i++) + if (payload[i].exist && + bytes_read >= payload[i].start && + bytes_read + bytes <= payload[i].end) + /* TX and RX Payload regions can't overlap */ + goto skip_read; + + spin_lock(&padap->win0_lock); + rc = cudbg_memory_read(pdbg_init, MEMWIN_NIC, mem_type, + bytes_read, bytes, temp_buff.data); + spin_unlock(&padap->win0_lock); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + +skip_read: + bytes_left -= bytes; + bytes_read += bytes; + rc = cudbg_write_and_release_buff(pdbg_init, &temp_buff, + dbg_buff); + if (rc) { + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + } + return rc; +} + +static void cudbg_t4_fwcache(struct cudbg_init *pdbg_init, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + int rc; + + if (is_fw_attached(pdbg_init)) { + /* Flush uP dcache before reading edcX/mcX */ + rc = t4_fwcache(padap, FW_PARAM_DEV_FWCACHE_FLUSH); + if (rc) + cudbg_err->sys_warn = rc; + } +} + +static int cudbg_mem_region_size(struct cudbg_init *pdbg_init, + struct cudbg_error *cudbg_err, + u8 mem_type, unsigned long *region_size) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_meminfo mem_info; + u8 mc_idx; + int rc; + + memset(&mem_info, 0, sizeof(struct cudbg_meminfo)); + rc = cudbg_fill_meminfo(padap, &mem_info); + if (rc) { + cudbg_err->sys_err = rc; + return rc; + } + + cudbg_t4_fwcache(pdbg_init, cudbg_err); + rc = cudbg_meminfo_get_mem_index(padap, &mem_info, mem_type, &mc_idx); + if (rc) { + cudbg_err->sys_err = rc; + return rc; + } + + if (region_size) + *region_size = mem_info.avail[mc_idx].limit - + mem_info.avail[mc_idx].base; + + return 0; +} + +static int cudbg_collect_mem_region(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err, + u8 mem_type) +{ + unsigned long size = 0; + int rc; + + rc = cudbg_mem_region_size(pdbg_init, cudbg_err, mem_type, &size); + if (rc) + return rc; + + return cudbg_read_fw_mem(pdbg_init, dbg_buff, mem_type, size, + cudbg_err); +} + +int cudbg_collect_edc0_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_collect_mem_region(pdbg_init, dbg_buff, cudbg_err, + MEM_EDC0); +} + +int cudbg_collect_edc1_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_collect_mem_region(pdbg_init, dbg_buff, cudbg_err, + MEM_EDC1); +} + +int cudbg_collect_mc0_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_collect_mem_region(pdbg_init, dbg_buff, cudbg_err, + MEM_MC0); +} + +int cudbg_collect_mc1_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_collect_mem_region(pdbg_init, dbg_buff, cudbg_err, + MEM_MC1); +} + +int cudbg_collect_hma_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + return cudbg_collect_mem_region(pdbg_init, dbg_buff, cudbg_err, + MEM_HMA); +} + +int cudbg_collect_rss(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + int rc, nentries; + + nentries = t4_chip_rss_size(padap); + rc = cudbg_get_buff(pdbg_init, dbg_buff, nentries * sizeof(u16), + &temp_buff); + if (rc) + return rc; + + rc = t4_read_rss(padap, (u16 *)temp_buff.data); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_rss_vf_config(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_rss_vf_conf *vfconf; + int vf, rc, vf_count; + + vf_count = padap->params.arch.vfcount; + rc = cudbg_get_buff(pdbg_init, dbg_buff, + vf_count * sizeof(struct cudbg_rss_vf_conf), + &temp_buff); + if (rc) + return rc; + + vfconf = (struct cudbg_rss_vf_conf *)temp_buff.data; + for (vf = 0; vf < vf_count; vf++) + t4_read_rss_vf_config(padap, vf, &vfconf[vf].rss_vf_vfl, + &vfconf[vf].rss_vf_vfh, true); + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_path_mtu(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + int rc; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, NMTUS * sizeof(u16), + &temp_buff); + if (rc) + return rc; + + t4_read_mtu_tbl(padap, (u16 *)temp_buff.data, NULL); + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_pm_stats(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_pm_stats *pm_stats_buff; + int rc; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_pm_stats), + &temp_buff); + if (rc) + return rc; + + pm_stats_buff = (struct cudbg_pm_stats *)temp_buff.data; + t4_pmtx_get_stats(padap, pm_stats_buff->tx_cnt, pm_stats_buff->tx_cyc); + t4_pmrx_get_stats(padap, pm_stats_buff->rx_cnt, pm_stats_buff->rx_cyc); + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_hw_sched(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_hw_sched *hw_sched_buff; + int i, rc = 0; + + if (!padap->params.vpd.cclk) + return CUDBG_STATUS_CCLK_NOT_DEFINED; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_hw_sched), + &temp_buff); + + if (rc) + return rc; + + hw_sched_buff = (struct cudbg_hw_sched *)temp_buff.data; + hw_sched_buff->map = t4_read_reg(padap, TP_TX_MOD_QUEUE_REQ_MAP_A); + hw_sched_buff->mode = TIMERMODE_G(t4_read_reg(padap, TP_MOD_CONFIG_A)); + t4_read_pace_tbl(padap, hw_sched_buff->pace_tab); + for (i = 0; i < NTX_SCHED; ++i) + t4_get_tx_sched(padap, i, &hw_sched_buff->kbps[i], + &hw_sched_buff->ipg[i], true); + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_tp_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct ireg_buf *ch_tp_pio; + int i, rc, n = 0; + u32 size; + + if (is_t5(padap->params.chip)) + n = sizeof(t5_tp_pio_array) + + sizeof(t5_tp_tm_pio_array) + + sizeof(t5_tp_mib_index_array); + else + n = sizeof(t6_tp_pio_array) + + sizeof(t6_tp_tm_pio_array) + + sizeof(t6_tp_mib_index_array); + + n = n / (IREG_NUM_ELEM * sizeof(u32)); + size = sizeof(struct ireg_buf) * n; + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + ch_tp_pio = (struct ireg_buf *)temp_buff.data; + + /* TP_PIO */ + if (is_t5(padap->params.chip)) + n = sizeof(t5_tp_pio_array) / (IREG_NUM_ELEM * sizeof(u32)); + else if (is_t6(padap->params.chip)) + n = sizeof(t6_tp_pio_array) / (IREG_NUM_ELEM * sizeof(u32)); + + for (i = 0; i < n; i++) { + struct ireg_field *tp_pio = &ch_tp_pio->tp_pio; + u32 *buff = ch_tp_pio->outbuf; + + if (is_t5(padap->params.chip)) { + tp_pio->ireg_addr = t5_tp_pio_array[i][0]; + tp_pio->ireg_data = t5_tp_pio_array[i][1]; + tp_pio->ireg_local_offset = t5_tp_pio_array[i][2]; + tp_pio->ireg_offset_range = t5_tp_pio_array[i][3]; + } else if (is_t6(padap->params.chip)) { + tp_pio->ireg_addr = t6_tp_pio_array[i][0]; + tp_pio->ireg_data = t6_tp_pio_array[i][1]; + tp_pio->ireg_local_offset = t6_tp_pio_array[i][2]; + tp_pio->ireg_offset_range = t6_tp_pio_array[i][3]; + } + t4_tp_pio_read(padap, buff, tp_pio->ireg_offset_range, + tp_pio->ireg_local_offset, true); + ch_tp_pio++; + } + + /* TP_TM_PIO */ + if (is_t5(padap->params.chip)) + n = sizeof(t5_tp_tm_pio_array) / (IREG_NUM_ELEM * sizeof(u32)); + else if (is_t6(padap->params.chip)) + n = sizeof(t6_tp_tm_pio_array) / (IREG_NUM_ELEM * sizeof(u32)); + + for (i = 0; i < n; i++) { + struct ireg_field *tp_pio = &ch_tp_pio->tp_pio; + u32 *buff = ch_tp_pio->outbuf; + + if (is_t5(padap->params.chip)) { + tp_pio->ireg_addr = t5_tp_tm_pio_array[i][0]; + tp_pio->ireg_data = t5_tp_tm_pio_array[i][1]; + tp_pio->ireg_local_offset = t5_tp_tm_pio_array[i][2]; + tp_pio->ireg_offset_range = t5_tp_tm_pio_array[i][3]; + } else if (is_t6(padap->params.chip)) { + tp_pio->ireg_addr = t6_tp_tm_pio_array[i][0]; + tp_pio->ireg_data = t6_tp_tm_pio_array[i][1]; + tp_pio->ireg_local_offset = t6_tp_tm_pio_array[i][2]; + tp_pio->ireg_offset_range = t6_tp_tm_pio_array[i][3]; + } + t4_tp_tm_pio_read(padap, buff, tp_pio->ireg_offset_range, + tp_pio->ireg_local_offset, true); + ch_tp_pio++; + } + + /* TP_MIB_INDEX */ + if (is_t5(padap->params.chip)) + n = sizeof(t5_tp_mib_index_array) / + (IREG_NUM_ELEM * sizeof(u32)); + else if (is_t6(padap->params.chip)) + n = sizeof(t6_tp_mib_index_array) / + (IREG_NUM_ELEM * sizeof(u32)); + + for (i = 0; i < n ; i++) { + struct ireg_field *tp_pio = &ch_tp_pio->tp_pio; + u32 *buff = ch_tp_pio->outbuf; + + if (is_t5(padap->params.chip)) { + tp_pio->ireg_addr = t5_tp_mib_index_array[i][0]; + tp_pio->ireg_data = t5_tp_mib_index_array[i][1]; + tp_pio->ireg_local_offset = + t5_tp_mib_index_array[i][2]; + tp_pio->ireg_offset_range = + t5_tp_mib_index_array[i][3]; + } else if (is_t6(padap->params.chip)) { + tp_pio->ireg_addr = t6_tp_mib_index_array[i][0]; + tp_pio->ireg_data = t6_tp_mib_index_array[i][1]; + tp_pio->ireg_local_offset = + t6_tp_mib_index_array[i][2]; + tp_pio->ireg_offset_range = + t6_tp_mib_index_array[i][3]; + } + t4_tp_mib_read(padap, buff, tp_pio->ireg_offset_range, + tp_pio->ireg_local_offset, true); + ch_tp_pio++; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +static void cudbg_read_sge_qbase_indirect_reg(struct adapter *padap, + struct sge_qbase_reg_field *qbase, + u32 func, bool is_pf) +{ + u32 *buff, i; + + if (is_pf) { + buff = qbase->pf_data_value[func]; + } else { + buff = qbase->vf_data_value[func]; + /* In SGE_QBASE_INDEX, + * Entries 0->7 are PF0->7, Entries 8->263 are VFID0->256. + */ + func += 8; + } + + t4_write_reg(padap, qbase->reg_addr, func); + for (i = 0; i < SGE_QBASE_DATA_REG_NUM; i++, buff++) + *buff = t4_read_reg(padap, qbase->reg_data[i]); +} + +int cudbg_collect_sge_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct sge_qbase_reg_field *sge_qbase; + struct ireg_buf *ch_sge_dbg; + u8 padap_running = 0; + int i, rc; + u32 size; + + /* Accessing SGE_QBASE_MAP[0-3] and SGE_QBASE_INDEX regs can + * lead to SGE missing doorbells under heavy traffic. So, only + * collect them when adapter is idle. + */ + for_each_port(padap, i) { + padap_running = netif_running(padap->port[i]); + if (padap_running) + break; + } + + size = sizeof(*ch_sge_dbg) * 2; + if (!padap_running) + size += sizeof(*sge_qbase); + + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + ch_sge_dbg = (struct ireg_buf *)temp_buff.data; + for (i = 0; i < 2; i++) { + struct ireg_field *sge_pio = &ch_sge_dbg->tp_pio; + u32 *buff = ch_sge_dbg->outbuf; + + sge_pio->ireg_addr = t5_sge_dbg_index_array[i][0]; + sge_pio->ireg_data = t5_sge_dbg_index_array[i][1]; + sge_pio->ireg_local_offset = t5_sge_dbg_index_array[i][2]; + sge_pio->ireg_offset_range = t5_sge_dbg_index_array[i][3]; + t4_read_indirect(padap, + sge_pio->ireg_addr, + sge_pio->ireg_data, + buff, + sge_pio->ireg_offset_range, + sge_pio->ireg_local_offset); + ch_sge_dbg++; + } + + if (CHELSIO_CHIP_VERSION(padap->params.chip) > CHELSIO_T5 && + !padap_running) { + sge_qbase = (struct sge_qbase_reg_field *)ch_sge_dbg; + /* 1 addr reg SGE_QBASE_INDEX and 4 data reg + * SGE_QBASE_MAP[0-3] + */ + sge_qbase->reg_addr = t6_sge_qbase_index_array[0]; + for (i = 0; i < SGE_QBASE_DATA_REG_NUM; i++) + sge_qbase->reg_data[i] = + t6_sge_qbase_index_array[i + 1]; + + for (i = 0; i <= PCIE_FW_MASTER_M; i++) + cudbg_read_sge_qbase_indirect_reg(padap, sge_qbase, + i, true); + + for (i = 0; i < padap->params.arch.vfcount; i++) + cudbg_read_sge_qbase_indirect_reg(padap, sge_qbase, + i, false); + + sge_qbase->vfcount = padap->params.arch.vfcount; + } + + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_ulprx_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_ulprx_la *ulprx_la_buff; + int rc; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_ulprx_la), + &temp_buff); + if (rc) + return rc; + + ulprx_la_buff = (struct cudbg_ulprx_la *)temp_buff.data; + t4_ulprx_read_la(padap, (u32 *)ulprx_la_buff->data); + ulprx_la_buff->size = ULPRX_LA_SIZE; + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_tp_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_tp_la *tp_la_buff; + int size, rc; + + size = sizeof(struct cudbg_tp_la) + TPLA_SIZE * sizeof(u64); + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + tp_la_buff = (struct cudbg_tp_la *)temp_buff.data; + tp_la_buff->mode = DBGLAMODE_G(t4_read_reg(padap, TP_DBG_LA_CONFIG_A)); + t4_tp_read_la(padap, (u64 *)tp_la_buff->data, NULL); + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_meminfo *meminfo_buff; + struct cudbg_ver_hdr *ver_hdr; + int rc; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, + sizeof(struct cudbg_ver_hdr) + + sizeof(struct cudbg_meminfo), + &temp_buff); + if (rc) + return rc; + + ver_hdr = (struct cudbg_ver_hdr *)temp_buff.data; + ver_hdr->signature = CUDBG_ENTITY_SIGNATURE; + ver_hdr->revision = CUDBG_MEMINFO_REV; + ver_hdr->size = sizeof(struct cudbg_meminfo); + + meminfo_buff = (struct cudbg_meminfo *)(temp_buff.data + + sizeof(*ver_hdr)); + rc = cudbg_fill_meminfo(padap, meminfo_buff); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_cim_pif_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct cudbg_cim_pif_la *cim_pif_la_buff; + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + int size, rc; + + size = sizeof(struct cudbg_cim_pif_la) + + 2 * CIM_PIFLA_SIZE * 6 * sizeof(u32); + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + cim_pif_la_buff = (struct cudbg_cim_pif_la *)temp_buff.data; + cim_pif_la_buff->size = CIM_PIFLA_SIZE; + t4_cim_read_pif_la(padap, (u32 *)cim_pif_la_buff->data, + (u32 *)cim_pif_la_buff->data + 6 * CIM_PIFLA_SIZE, + NULL, NULL); + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_clk_info(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_clk_info *clk_info_buff; + u64 tp_tick_us; + int rc; + + if (!padap->params.vpd.cclk) + return CUDBG_STATUS_CCLK_NOT_DEFINED; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_clk_info), + &temp_buff); + if (rc) + return rc; + + clk_info_buff = (struct cudbg_clk_info *)temp_buff.data; + clk_info_buff->cclk_ps = 1000000000 / padap->params.vpd.cclk; /* psec */ + clk_info_buff->res = t4_read_reg(padap, TP_TIMER_RESOLUTION_A); + clk_info_buff->tre = TIMERRESOLUTION_G(clk_info_buff->res); + clk_info_buff->dack_re = DELAYEDACKRESOLUTION_G(clk_info_buff->res); + tp_tick_us = (clk_info_buff->cclk_ps << clk_info_buff->tre) / 1000000; + + clk_info_buff->dack_timer = + (clk_info_buff->cclk_ps << clk_info_buff->dack_re) / 1000000 * + t4_read_reg(padap, TP_DACK_TIMER_A); + clk_info_buff->retransmit_min = + tp_tick_us * t4_read_reg(padap, TP_RXT_MIN_A); + clk_info_buff->retransmit_max = + tp_tick_us * t4_read_reg(padap, TP_RXT_MAX_A); + clk_info_buff->persist_timer_min = + tp_tick_us * t4_read_reg(padap, TP_PERS_MIN_A); + clk_info_buff->persist_timer_max = + tp_tick_us * t4_read_reg(padap, TP_PERS_MAX_A); + clk_info_buff->keepalive_idle_timer = + tp_tick_us * t4_read_reg(padap, TP_KEEP_IDLE_A); + clk_info_buff->keepalive_interval = + tp_tick_us * t4_read_reg(padap, TP_KEEP_INTVL_A); + clk_info_buff->initial_srtt = + tp_tick_us * INITSRTT_G(t4_read_reg(padap, TP_INIT_SRTT_A)); + clk_info_buff->finwait2_timer = + tp_tick_us * t4_read_reg(padap, TP_FINWAIT2_TIMER_A); + + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_pcie_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct ireg_buf *ch_pcie; + int i, rc, n; + u32 size; + + n = sizeof(t5_pcie_pdbg_array) / (IREG_NUM_ELEM * sizeof(u32)); + size = sizeof(struct ireg_buf) * n * 2; + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + ch_pcie = (struct ireg_buf *)temp_buff.data; + /* PCIE_PDBG */ + for (i = 0; i < n; i++) { + struct ireg_field *pcie_pio = &ch_pcie->tp_pio; + u32 *buff = ch_pcie->outbuf; + + pcie_pio->ireg_addr = t5_pcie_pdbg_array[i][0]; + pcie_pio->ireg_data = t5_pcie_pdbg_array[i][1]; + pcie_pio->ireg_local_offset = t5_pcie_pdbg_array[i][2]; + pcie_pio->ireg_offset_range = t5_pcie_pdbg_array[i][3]; + t4_read_indirect(padap, + pcie_pio->ireg_addr, + pcie_pio->ireg_data, + buff, + pcie_pio->ireg_offset_range, + pcie_pio->ireg_local_offset); + ch_pcie++; + } + + /* PCIE_CDBG */ + n = sizeof(t5_pcie_cdbg_array) / (IREG_NUM_ELEM * sizeof(u32)); + for (i = 0; i < n; i++) { + struct ireg_field *pcie_pio = &ch_pcie->tp_pio; + u32 *buff = ch_pcie->outbuf; + + pcie_pio->ireg_addr = t5_pcie_cdbg_array[i][0]; + pcie_pio->ireg_data = t5_pcie_cdbg_array[i][1]; + pcie_pio->ireg_local_offset = t5_pcie_cdbg_array[i][2]; + pcie_pio->ireg_offset_range = t5_pcie_cdbg_array[i][3]; + t4_read_indirect(padap, + pcie_pio->ireg_addr, + pcie_pio->ireg_data, + buff, + pcie_pio->ireg_offset_range, + pcie_pio->ireg_local_offset); + ch_pcie++; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_pm_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct ireg_buf *ch_pm; + int i, rc, n; + u32 size; + + n = sizeof(t5_pm_rx_array) / (IREG_NUM_ELEM * sizeof(u32)); + size = sizeof(struct ireg_buf) * n * 2; + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + ch_pm = (struct ireg_buf *)temp_buff.data; + /* PM_RX */ + for (i = 0; i < n; i++) { + struct ireg_field *pm_pio = &ch_pm->tp_pio; + u32 *buff = ch_pm->outbuf; + + pm_pio->ireg_addr = t5_pm_rx_array[i][0]; + pm_pio->ireg_data = t5_pm_rx_array[i][1]; + pm_pio->ireg_local_offset = t5_pm_rx_array[i][2]; + pm_pio->ireg_offset_range = t5_pm_rx_array[i][3]; + t4_read_indirect(padap, + pm_pio->ireg_addr, + pm_pio->ireg_data, + buff, + pm_pio->ireg_offset_range, + pm_pio->ireg_local_offset); + ch_pm++; + } + + /* PM_TX */ + n = sizeof(t5_pm_tx_array) / (IREG_NUM_ELEM * sizeof(u32)); + for (i = 0; i < n; i++) { + struct ireg_field *pm_pio = &ch_pm->tp_pio; + u32 *buff = ch_pm->outbuf; + + pm_pio->ireg_addr = t5_pm_tx_array[i][0]; + pm_pio->ireg_data = t5_pm_tx_array[i][1]; + pm_pio->ireg_local_offset = t5_pm_tx_array[i][2]; + pm_pio->ireg_offset_range = t5_pm_tx_array[i][3]; + t4_read_indirect(padap, + pm_pio->ireg_addr, + pm_pio->ireg_data, + buff, + pm_pio->ireg_offset_range, + pm_pio->ireg_local_offset); + ch_pm++; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_tid(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_tid_info_region_rev1 *tid1; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_tid_info_region *tid; + u32 para[2], val[2]; + int rc; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, + sizeof(struct cudbg_tid_info_region_rev1), + &temp_buff); + if (rc) + return rc; + + tid1 = (struct cudbg_tid_info_region_rev1 *)temp_buff.data; + tid = &tid1->tid; + tid1->ver_hdr.signature = CUDBG_ENTITY_SIGNATURE; + tid1->ver_hdr.revision = CUDBG_TID_INFO_REV; + tid1->ver_hdr.size = sizeof(struct cudbg_tid_info_region_rev1) - + sizeof(struct cudbg_ver_hdr); + + /* If firmware is not attached/alive, use backdoor register + * access to collect dump. + */ + if (!is_fw_attached(pdbg_init)) + goto fill_tid; + +#define FW_PARAM_PFVF_A(param) \ + (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \ + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param) | \ + FW_PARAMS_PARAM_Y_V(0) | \ + FW_PARAMS_PARAM_Z_V(0)) + + para[0] = FW_PARAM_PFVF_A(ETHOFLD_START); + para[1] = FW_PARAM_PFVF_A(ETHOFLD_END); + rc = t4_query_params(padap, padap->mbox, padap->pf, 0, 2, para, val); + if (rc < 0) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + tid->uotid_base = val[0]; + tid->nuotids = val[1] - val[0] + 1; + + if (is_t5(padap->params.chip)) { + tid->sb = t4_read_reg(padap, LE_DB_SERVER_INDEX_A) / 4; + } else if (is_t6(padap->params.chip)) { + tid1->tid_start = + t4_read_reg(padap, LE_DB_ACTIVE_TABLE_START_INDEX_A); + tid->sb = t4_read_reg(padap, LE_DB_SRVR_START_INDEX_A); + + para[0] = FW_PARAM_PFVF_A(HPFILTER_START); + para[1] = FW_PARAM_PFVF_A(HPFILTER_END); + rc = t4_query_params(padap, padap->mbox, padap->pf, 0, 2, + para, val); + if (rc < 0) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + tid->hpftid_base = val[0]; + tid->nhpftids = val[1] - val[0] + 1; + } + +#undef FW_PARAM_PFVF_A + +fill_tid: + tid->ntids = padap->tids.ntids; + tid->nstids = padap->tids.nstids; + tid->stid_base = padap->tids.stid_base; + tid->hash_base = padap->tids.hash_base; + + tid->natids = padap->tids.natids; + tid->nftids = padap->tids.nftids; + tid->ftid_base = padap->tids.ftid_base; + tid->aftid_base = padap->tids.aftid_base; + tid->aftid_end = padap->tids.aftid_end; + + tid->sftid_base = padap->tids.sftid_base; + tid->nsftids = padap->tids.nsftids; + + tid->flags = padap->flags; + tid->le_db_conf = t4_read_reg(padap, LE_DB_CONFIG_A); + tid->ip_users = t4_read_reg(padap, LE_DB_ACT_CNT_IPV4_A); + tid->ipv6_users = t4_read_reg(padap, LE_DB_ACT_CNT_IPV6_A); + + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_pcie_config(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + u32 size, *value, j; + int i, rc, n; + + size = sizeof(u32) * CUDBG_NUM_PCIE_CONFIG_REGS; + n = sizeof(t5_pcie_config_array) / (2 * sizeof(u32)); + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + value = (u32 *)temp_buff.data; + for (i = 0; i < n; i++) { + for (j = t5_pcie_config_array[i][0]; + j <= t5_pcie_config_array[i][1]; j += 4) { + t4_hw_pci_read_cfg4(padap, j, value); + value++; + } + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +static int cudbg_sge_ctxt_check_valid(u32 *buf, int type) +{ + int index, bit, bit_pos = 0; + + switch (type) { + case CTXT_EGRESS: + bit_pos = 176; + break; + case CTXT_INGRESS: + bit_pos = 141; + break; + case CTXT_FLM: + bit_pos = 89; + break; + } + index = bit_pos / 32; + bit = bit_pos % 32; + return buf[index] & (1U << bit); +} + +static int cudbg_get_ctxt_region_info(struct adapter *padap, + struct cudbg_region_info *ctx_info, + u8 *mem_type) +{ + struct cudbg_mem_desc mem_desc; + struct cudbg_meminfo meminfo; + u32 i, j, value, found; + u8 flq; + int rc; + + rc = cudbg_fill_meminfo(padap, &meminfo); + if (rc) + return rc; + + /* Get EGRESS and INGRESS context region size */ + for (i = CTXT_EGRESS; i <= CTXT_INGRESS; i++) { + found = 0; + memset(&mem_desc, 0, sizeof(struct cudbg_mem_desc)); + for (j = 0; j < ARRAY_SIZE(meminfo.avail); j++) { + rc = cudbg_get_mem_region(padap, &meminfo, j, + cudbg_region[i], + &mem_desc); + if (!rc) { + found = 1; + rc = cudbg_get_mem_relative(padap, &meminfo, j, + &mem_desc.base, + &mem_desc.limit); + if (rc) { + ctx_info[i].exist = false; + break; + } + ctx_info[i].exist = true; + ctx_info[i].start = mem_desc.base; + ctx_info[i].end = mem_desc.limit; + mem_type[i] = j; + break; + } + } + if (!found) + ctx_info[i].exist = false; + } + + /* Get FLM and CNM max qid. */ + value = t4_read_reg(padap, SGE_FLM_CFG_A); + + /* Get number of data freelist queues */ + flq = HDRSTARTFLQ_G(value); + ctx_info[CTXT_FLM].exist = true; + ctx_info[CTXT_FLM].end = (CUDBG_MAX_FL_QIDS >> flq) * SGE_CTXT_SIZE; + + /* The number of CONM contexts are same as number of freelist + * queues. + */ + ctx_info[CTXT_CNM].exist = true; + ctx_info[CTXT_CNM].end = ctx_info[CTXT_FLM].end; + + return 0; +} + +int cudbg_dump_context_size(struct adapter *padap) +{ + struct cudbg_region_info region_info[CTXT_CNM + 1] = { {0} }; + u8 mem_type[CTXT_INGRESS + 1] = { 0 }; + u32 i, size = 0; + int rc; + + /* Get max valid qid for each type of queue */ + rc = cudbg_get_ctxt_region_info(padap, region_info, mem_type); + if (rc) + return rc; + + for (i = 0; i < CTXT_CNM; i++) { + if (!region_info[i].exist) { + if (i == CTXT_EGRESS || i == CTXT_INGRESS) + size += CUDBG_LOWMEM_MAX_CTXT_QIDS * + SGE_CTXT_SIZE; + continue; + } + + size += (region_info[i].end - region_info[i].start + 1) / + SGE_CTXT_SIZE; + } + return size * sizeof(struct cudbg_ch_cntxt); +} + +static void cudbg_read_sge_ctxt(struct cudbg_init *pdbg_init, u32 cid, + enum ctxt_type ctype, u32 *data) +{ + struct adapter *padap = pdbg_init->adap; + int rc = -1; + + /* Under heavy traffic, the SGE Queue contexts registers will be + * frequently accessed by firmware. + * + * To avoid conflicts with firmware, always ask firmware to fetch + * the SGE Queue contexts via mailbox. On failure, fallback to + * accessing hardware registers directly. + */ + if (is_fw_attached(pdbg_init)) + rc = t4_sge_ctxt_rd(padap, padap->mbox, cid, ctype, data); + if (rc) + t4_sge_ctxt_rd_bd(padap, cid, ctype, data); +} + +static void cudbg_get_sge_ctxt_fw(struct cudbg_init *pdbg_init, u32 max_qid, + u8 ctxt_type, + struct cudbg_ch_cntxt **out_buff) +{ + struct cudbg_ch_cntxt *buff = *out_buff; + int rc; + u32 j; + + for (j = 0; j < max_qid; j++) { + cudbg_read_sge_ctxt(pdbg_init, j, ctxt_type, buff->data); + rc = cudbg_sge_ctxt_check_valid(buff->data, ctxt_type); + if (!rc) + continue; + + buff->cntxt_type = ctxt_type; + buff->cntxt_id = j; + buff++; + if (ctxt_type == CTXT_FLM) { + cudbg_read_sge_ctxt(pdbg_init, j, CTXT_CNM, buff->data); + buff->cntxt_type = CTXT_CNM; + buff->cntxt_id = j; + buff++; + } + } + + *out_buff = buff; +} + +int cudbg_collect_dump_context(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct cudbg_region_info region_info[CTXT_CNM + 1] = { {0} }; + struct adapter *padap = pdbg_init->adap; + u32 j, size, max_ctx_size, max_ctx_qid; + u8 mem_type[CTXT_INGRESS + 1] = { 0 }; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_ch_cntxt *buff; + u8 *ctx_buf; + u8 i, k; + int rc; + + /* Get max valid qid for each type of queue */ + rc = cudbg_get_ctxt_region_info(padap, region_info, mem_type); + if (rc) + return rc; + + rc = cudbg_dump_context_size(padap); + if (rc <= 0) + return CUDBG_STATUS_ENTITY_NOT_FOUND; + + size = rc; + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + /* Get buffer with enough space to read the biggest context + * region in memory. + */ + max_ctx_size = max(region_info[CTXT_EGRESS].end - + region_info[CTXT_EGRESS].start + 1, + region_info[CTXT_INGRESS].end - + region_info[CTXT_INGRESS].start + 1); + + ctx_buf = kvzalloc(max_ctx_size, GFP_KERNEL); + if (!ctx_buf) { + cudbg_put_buff(pdbg_init, &temp_buff); + return -ENOMEM; + } + + buff = (struct cudbg_ch_cntxt *)temp_buff.data; + + /* Collect EGRESS and INGRESS context data. + * In case of failures, fallback to collecting via FW or + * backdoor access. + */ + for (i = CTXT_EGRESS; i <= CTXT_INGRESS; i++) { + if (!region_info[i].exist) { + max_ctx_qid = CUDBG_LOWMEM_MAX_CTXT_QIDS; + cudbg_get_sge_ctxt_fw(pdbg_init, max_ctx_qid, i, + &buff); + continue; + } + + max_ctx_size = region_info[i].end - region_info[i].start + 1; + max_ctx_qid = max_ctx_size / SGE_CTXT_SIZE; + + /* If firmware is not attached/alive, use backdoor register + * access to collect dump. + */ + if (is_fw_attached(pdbg_init)) { + t4_sge_ctxt_flush(padap, padap->mbox, i); + + rc = t4_memory_rw(padap, MEMWIN_NIC, mem_type[i], + region_info[i].start, max_ctx_size, + (__be32 *)ctx_buf, 1); + } + + if (rc || !is_fw_attached(pdbg_init)) { + max_ctx_qid = CUDBG_LOWMEM_MAX_CTXT_QIDS; + cudbg_get_sge_ctxt_fw(pdbg_init, max_ctx_qid, i, + &buff); + continue; + } + + for (j = 0; j < max_ctx_qid; j++) { + __be64 *dst_off; + u64 *src_off; + + src_off = (u64 *)(ctx_buf + j * SGE_CTXT_SIZE); + dst_off = (__be64 *)buff->data; + + /* The data is stored in 64-bit cpu order. Convert it + * to big endian before parsing. + */ + for (k = 0; k < SGE_CTXT_SIZE / sizeof(u64); k++) + dst_off[k] = cpu_to_be64(src_off[k]); + + rc = cudbg_sge_ctxt_check_valid(buff->data, i); + if (!rc) + continue; + + buff->cntxt_type = i; + buff->cntxt_id = j; + buff++; + } + } + + kvfree(ctx_buf); + + /* Collect FREELIST and CONGESTION MANAGER contexts */ + max_ctx_size = region_info[CTXT_FLM].end - + region_info[CTXT_FLM].start + 1; + max_ctx_qid = max_ctx_size / SGE_CTXT_SIZE; + /* Since FLM and CONM are 1-to-1 mapped, the below function + * will fetch both FLM and CONM contexts. + */ + cudbg_get_sge_ctxt_fw(pdbg_init, max_ctx_qid, CTXT_FLM, &buff); + + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +static inline void cudbg_tcamxy2valmask(u64 x, u64 y, u8 *addr, u64 *mask) +{ + *mask = x | y; + y = (__force u64)cpu_to_be64(y); + memcpy(addr, (char *)&y + 2, ETH_ALEN); +} + +static void cudbg_mps_rpl_backdoor(struct adapter *padap, + struct fw_ldst_mps_rplc *mps_rplc) +{ + if (is_t5(padap->params.chip)) { + mps_rplc->rplc255_224 = htonl(t4_read_reg(padap, + MPS_VF_RPLCT_MAP3_A)); + mps_rplc->rplc223_192 = htonl(t4_read_reg(padap, + MPS_VF_RPLCT_MAP2_A)); + mps_rplc->rplc191_160 = htonl(t4_read_reg(padap, + MPS_VF_RPLCT_MAP1_A)); + mps_rplc->rplc159_128 = htonl(t4_read_reg(padap, + MPS_VF_RPLCT_MAP0_A)); + } else { + mps_rplc->rplc255_224 = htonl(t4_read_reg(padap, + MPS_VF_RPLCT_MAP7_A)); + mps_rplc->rplc223_192 = htonl(t4_read_reg(padap, + MPS_VF_RPLCT_MAP6_A)); + mps_rplc->rplc191_160 = htonl(t4_read_reg(padap, + MPS_VF_RPLCT_MAP5_A)); + mps_rplc->rplc159_128 = htonl(t4_read_reg(padap, + MPS_VF_RPLCT_MAP4_A)); + } + mps_rplc->rplc127_96 = htonl(t4_read_reg(padap, MPS_VF_RPLCT_MAP3_A)); + mps_rplc->rplc95_64 = htonl(t4_read_reg(padap, MPS_VF_RPLCT_MAP2_A)); + mps_rplc->rplc63_32 = htonl(t4_read_reg(padap, MPS_VF_RPLCT_MAP1_A)); + mps_rplc->rplc31_0 = htonl(t4_read_reg(padap, MPS_VF_RPLCT_MAP0_A)); +} + +static int cudbg_collect_tcam_index(struct cudbg_init *pdbg_init, + struct cudbg_mps_tcam *tcam, u32 idx) +{ + struct adapter *padap = pdbg_init->adap; + u64 tcamy, tcamx, val; + u32 ctl, data2; + int rc = 0; + + if (CHELSIO_CHIP_VERSION(padap->params.chip) >= CHELSIO_T6) { + /* CtlReqID - 1: use Host Driver Requester ID + * CtlCmdType - 0: Read, 1: Write + * CtlTcamSel - 0: TCAM0, 1: TCAM1 + * CtlXYBitSel- 0: Y bit, 1: X bit + */ + + /* Read tcamy */ + ctl = CTLREQID_V(1) | CTLCMDTYPE_V(0) | CTLXYBITSEL_V(0); + if (idx < 256) + ctl |= CTLTCAMINDEX_V(idx) | CTLTCAMSEL_V(0); + else + ctl |= CTLTCAMINDEX_V(idx - 256) | CTLTCAMSEL_V(1); + + t4_write_reg(padap, MPS_CLS_TCAM_DATA2_CTL_A, ctl); + val = t4_read_reg(padap, MPS_CLS_TCAM_RDATA1_REQ_ID1_A); + tcamy = DMACH_G(val) << 32; + tcamy |= t4_read_reg(padap, MPS_CLS_TCAM_RDATA0_REQ_ID1_A); + data2 = t4_read_reg(padap, MPS_CLS_TCAM_RDATA2_REQ_ID1_A); + tcam->lookup_type = DATALKPTYPE_G(data2); + + /* 0 - Outer header, 1 - Inner header + * [71:48] bit locations are overloaded for + * outer vs. inner lookup types. + */ + if (tcam->lookup_type && tcam->lookup_type != DATALKPTYPE_M) { + /* Inner header VNI */ + tcam->vniy = (data2 & DATAVIDH2_F) | DATAVIDH1_G(data2); + tcam->vniy = (tcam->vniy << 16) | VIDL_G(val); + tcam->dip_hit = data2 & DATADIPHIT_F; + } else { + tcam->vlan_vld = data2 & DATAVIDH2_F; + tcam->ivlan = VIDL_G(val); + } + + tcam->port_num = DATAPORTNUM_G(data2); + + /* Read tcamx. Change the control param */ + ctl |= CTLXYBITSEL_V(1); + t4_write_reg(padap, MPS_CLS_TCAM_DATA2_CTL_A, ctl); + val = t4_read_reg(padap, MPS_CLS_TCAM_RDATA1_REQ_ID1_A); + tcamx = DMACH_G(val) << 32; + tcamx |= t4_read_reg(padap, MPS_CLS_TCAM_RDATA0_REQ_ID1_A); + data2 = t4_read_reg(padap, MPS_CLS_TCAM_RDATA2_REQ_ID1_A); + if (tcam->lookup_type && tcam->lookup_type != DATALKPTYPE_M) { + /* Inner header VNI mask */ + tcam->vnix = (data2 & DATAVIDH2_F) | DATAVIDH1_G(data2); + tcam->vnix = (tcam->vnix << 16) | VIDL_G(val); + } + } else { + tcamy = t4_read_reg64(padap, MPS_CLS_TCAM_Y_L(idx)); + tcamx = t4_read_reg64(padap, MPS_CLS_TCAM_X_L(idx)); + } + + /* If no entry, return */ + if (tcamx & tcamy) + return rc; + + tcam->cls_lo = t4_read_reg(padap, MPS_CLS_SRAM_L(idx)); + tcam->cls_hi = t4_read_reg(padap, MPS_CLS_SRAM_H(idx)); + + if (is_t5(padap->params.chip)) + tcam->repli = (tcam->cls_lo & REPLICATE_F); + else if (is_t6(padap->params.chip)) + tcam->repli = (tcam->cls_lo & T6_REPLICATE_F); + + if (tcam->repli) { + struct fw_ldst_cmd ldst_cmd; + struct fw_ldst_mps_rplc mps_rplc; + + memset(&ldst_cmd, 0, sizeof(ldst_cmd)); + ldst_cmd.op_to_addrspace = + htonl(FW_CMD_OP_V(FW_LDST_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F | + FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_MPS)); + ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd)); + ldst_cmd.u.mps.rplc.fid_idx = + htons(FW_LDST_CMD_FID_V(FW_LDST_MPS_RPLC) | + FW_LDST_CMD_IDX_V(idx)); + + /* If firmware is not attached/alive, use backdoor register + * access to collect dump. + */ + if (is_fw_attached(pdbg_init)) + rc = t4_wr_mbox(padap, padap->mbox, &ldst_cmd, + sizeof(ldst_cmd), &ldst_cmd); + + if (rc || !is_fw_attached(pdbg_init)) { + cudbg_mps_rpl_backdoor(padap, &mps_rplc); + /* Ignore error since we collected directly from + * reading registers. + */ + rc = 0; + } else { + mps_rplc = ldst_cmd.u.mps.rplc; + } + + tcam->rplc[0] = ntohl(mps_rplc.rplc31_0); + tcam->rplc[1] = ntohl(mps_rplc.rplc63_32); + tcam->rplc[2] = ntohl(mps_rplc.rplc95_64); + tcam->rplc[3] = ntohl(mps_rplc.rplc127_96); + if (padap->params.arch.mps_rplc_size > CUDBG_MAX_RPLC_SIZE) { + tcam->rplc[4] = ntohl(mps_rplc.rplc159_128); + tcam->rplc[5] = ntohl(mps_rplc.rplc191_160); + tcam->rplc[6] = ntohl(mps_rplc.rplc223_192); + tcam->rplc[7] = ntohl(mps_rplc.rplc255_224); + } + } + cudbg_tcamxy2valmask(tcamx, tcamy, tcam->addr, &tcam->mask); + tcam->idx = idx; + tcam->rplc_size = padap->params.arch.mps_rplc_size; + return rc; +} + +int cudbg_collect_mps_tcam(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + u32 size = 0, i, n, total_size = 0; + struct cudbg_mps_tcam *tcam; + int rc; + + n = padap->params.arch.mps_tcam_size; + size = sizeof(struct cudbg_mps_tcam) * n; + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + tcam = (struct cudbg_mps_tcam *)temp_buff.data; + for (i = 0; i < n; i++) { + rc = cudbg_collect_tcam_index(pdbg_init, tcam, i); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + total_size += sizeof(struct cudbg_mps_tcam); + tcam++; + } + + if (!total_size) { + rc = CUDBG_SYSTEM_ERROR; + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_vpd_data(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + char vpd_str[CUDBG_VPD_VER_LEN + 1]; + struct cudbg_vpd_data *vpd_data; + struct vpd_params vpd = { 0 }; + u32 vpd_vers, fw_vers; + int rc; + + rc = t4_get_raw_vpd_params(padap, &vpd); + if (rc) + return rc; + + rc = t4_get_fw_version(padap, &fw_vers); + if (rc) + return rc; + + rc = cudbg_read_vpd_reg(padap, CUDBG_VPD_VER_ADDR, CUDBG_VPD_VER_LEN, + vpd_str); + if (rc) + return rc; + + vpd_str[CUDBG_VPD_VER_LEN] = '\0'; + rc = kstrtouint(vpd_str, 0, &vpd_vers); + if (rc) + return rc; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_vpd_data), + &temp_buff); + if (rc) + return rc; + + vpd_data = (struct cudbg_vpd_data *)temp_buff.data; + memcpy(vpd_data->sn, vpd.sn, SERNUM_LEN + 1); + memcpy(vpd_data->bn, vpd.pn, PN_LEN + 1); + memcpy(vpd_data->na, vpd.na, MACADDR_LEN + 1); + memcpy(vpd_data->mn, vpd.id, ID_LEN + 1); + vpd_data->scfg_vers = t4_read_reg(padap, PCIE_STATIC_SPARE2_A); + vpd_data->vpd_vers = vpd_vers; + vpd_data->fw_major = FW_HDR_FW_VER_MAJOR_G(fw_vers); + vpd_data->fw_minor = FW_HDR_FW_VER_MINOR_G(fw_vers); + vpd_data->fw_micro = FW_HDR_FW_VER_MICRO_G(fw_vers); + vpd_data->fw_build = FW_HDR_FW_VER_BUILD_G(fw_vers); + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +static int cudbg_read_tid(struct cudbg_init *pdbg_init, u32 tid, + struct cudbg_tid_data *tid_data) +{ + struct adapter *padap = pdbg_init->adap; + int i, cmd_retry = 8; + u32 val; + + /* Fill REQ_DATA regs with 0's */ + for (i = 0; i < NUM_LE_DB_DBGI_REQ_DATA_INSTANCES; i++) + t4_write_reg(padap, LE_DB_DBGI_REQ_DATA_A + (i << 2), 0); + + /* Write DBIG command */ + val = DBGICMD_V(4) | DBGITID_V(tid); + t4_write_reg(padap, LE_DB_DBGI_REQ_TCAM_CMD_A, val); + tid_data->dbig_cmd = val; + + val = DBGICMDSTRT_F | DBGICMDMODE_V(1); /* LE mode */ + t4_write_reg(padap, LE_DB_DBGI_CONFIG_A, val); + tid_data->dbig_conf = val; + + /* Poll the DBGICMDBUSY bit */ + val = 1; + while (val) { + val = t4_read_reg(padap, LE_DB_DBGI_CONFIG_A); + val = val & DBGICMDBUSY_F; + cmd_retry--; + if (!cmd_retry) + return CUDBG_SYSTEM_ERROR; + } + + /* Check RESP status */ + val = t4_read_reg(padap, LE_DB_DBGI_RSP_STATUS_A); + tid_data->dbig_rsp_stat = val; + if (!(val & 1)) + return CUDBG_SYSTEM_ERROR; + + /* Read RESP data */ + for (i = 0; i < NUM_LE_DB_DBGI_RSP_DATA_INSTANCES; i++) + tid_data->data[i] = t4_read_reg(padap, + LE_DB_DBGI_RSP_DATA_A + + (i << 2)); + tid_data->tid = tid; + return 0; +} + +static int cudbg_get_le_type(u32 tid, struct cudbg_tcam tcam_region) +{ + int type = LE_ET_UNKNOWN; + + if (tid < tcam_region.server_start) + type = LE_ET_TCAM_CON; + else if (tid < tcam_region.filter_start) + type = LE_ET_TCAM_SERVER; + else if (tid < tcam_region.clip_start) + type = LE_ET_TCAM_FILTER; + else if (tid < tcam_region.routing_start) + type = LE_ET_TCAM_CLIP; + else if (tid < tcam_region.tid_hash_base) + type = LE_ET_TCAM_ROUTING; + else if (tid < tcam_region.max_tid) + type = LE_ET_HASH_CON; + else + type = LE_ET_INVALID_TID; + + return type; +} + +static int cudbg_is_ipv6_entry(struct cudbg_tid_data *tid_data, + struct cudbg_tcam tcam_region) +{ + int ipv6 = 0; + int le_type; + + le_type = cudbg_get_le_type(tid_data->tid, tcam_region); + if (tid_data->tid & 1) + return 0; + + if (le_type == LE_ET_HASH_CON) { + ipv6 = tid_data->data[16] & 0x8000; + } else if (le_type == LE_ET_TCAM_CON) { + ipv6 = tid_data->data[16] & 0x8000; + if (ipv6) + ipv6 = tid_data->data[9] == 0x00C00000; + } else { + ipv6 = 0; + } + return ipv6; +} + +void cudbg_fill_le_tcam_info(struct adapter *padap, + struct cudbg_tcam *tcam_region) +{ + u32 value; + + /* Get the LE regions */ + value = t4_read_reg(padap, LE_DB_TID_HASHBASE_A); /* hash base index */ + tcam_region->tid_hash_base = value; + + /* Get routing table index */ + value = t4_read_reg(padap, LE_DB_ROUTING_TABLE_INDEX_A); + tcam_region->routing_start = value; + + /* Get clip table index. For T6 there is separate CLIP TCAM */ + if (is_t6(padap->params.chip)) + value = t4_read_reg(padap, LE_DB_CLCAM_TID_BASE_A); + else + value = t4_read_reg(padap, LE_DB_CLIP_TABLE_INDEX_A); + tcam_region->clip_start = value; + + /* Get filter table index */ + value = t4_read_reg(padap, LE_DB_FILTER_TABLE_INDEX_A); + tcam_region->filter_start = value; + + /* Get server table index */ + value = t4_read_reg(padap, LE_DB_SERVER_INDEX_A); + tcam_region->server_start = value; + + /* Check whether hash is enabled and calculate the max tids */ + value = t4_read_reg(padap, LE_DB_CONFIG_A); + if ((value >> HASHEN_S) & 1) { + value = t4_read_reg(padap, LE_DB_HASH_CONFIG_A); + if (CHELSIO_CHIP_VERSION(padap->params.chip) > CHELSIO_T5) { + tcam_region->max_tid = (value & 0xFFFFF) + + tcam_region->tid_hash_base; + } else { + value = HASHTIDSIZE_G(value); + value = 1 << value; + tcam_region->max_tid = value + + tcam_region->tid_hash_base; + } + } else { /* hash not enabled */ + if (is_t6(padap->params.chip)) + tcam_region->max_tid = (value & ASLIPCOMPEN_F) ? + CUDBG_MAX_TID_COMP_EN : + CUDBG_MAX_TID_COMP_DIS; + else + tcam_region->max_tid = CUDBG_MAX_TCAM_TID; + } + + if (is_t6(padap->params.chip)) + tcam_region->max_tid += CUDBG_T6_CLIP; +} + +int cudbg_collect_le_tcam(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_tcam tcam_region = { 0 }; + struct cudbg_tid_data *tid_data; + u32 bytes = 0; + int rc, size; + u32 i; + + cudbg_fill_le_tcam_info(padap, &tcam_region); + + size = sizeof(struct cudbg_tid_data) * tcam_region.max_tid; + size += sizeof(struct cudbg_tcam); + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + memcpy(temp_buff.data, &tcam_region, sizeof(struct cudbg_tcam)); + bytes = sizeof(struct cudbg_tcam); + tid_data = (struct cudbg_tid_data *)(temp_buff.data + bytes); + /* read all tid */ + for (i = 0; i < tcam_region.max_tid; ) { + rc = cudbg_read_tid(pdbg_init, i, tid_data); + if (rc) { + cudbg_err->sys_warn = CUDBG_STATUS_PARTIAL_DATA; + /* Update tcam header and exit */ + tcam_region.max_tid = i; + memcpy(temp_buff.data, &tcam_region, + sizeof(struct cudbg_tcam)); + goto out; + } + + if (cudbg_is_ipv6_entry(tid_data, tcam_region)) { + /* T6 CLIP TCAM: ipv6 takes 4 entries */ + if (is_t6(padap->params.chip) && + i >= tcam_region.clip_start && + i < tcam_region.clip_start + CUDBG_T6_CLIP) + i += 4; + else /* Main TCAM: ipv6 takes two tids */ + i += 2; + } else { + i++; + } + + tid_data++; + bytes += sizeof(struct cudbg_tid_data); + } + +out: + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_cctrl(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + u32 size; + int rc; + + size = sizeof(u16) * NMTUS * NCCTRL_WIN; + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + t4_read_cong_tbl(padap, (void *)temp_buff.data); + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_ma_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct ireg_buf *ma_indr; + int i, rc, n; + u32 size, j; + + if (CHELSIO_CHIP_VERSION(padap->params.chip) < CHELSIO_T6) + return CUDBG_STATUS_ENTITY_NOT_FOUND; + + n = sizeof(t6_ma_ireg_array) / (IREG_NUM_ELEM * sizeof(u32)); + size = sizeof(struct ireg_buf) * n * 2; + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + ma_indr = (struct ireg_buf *)temp_buff.data; + for (i = 0; i < n; i++) { + struct ireg_field *ma_fli = &ma_indr->tp_pio; + u32 *buff = ma_indr->outbuf; + + ma_fli->ireg_addr = t6_ma_ireg_array[i][0]; + ma_fli->ireg_data = t6_ma_ireg_array[i][1]; + ma_fli->ireg_local_offset = t6_ma_ireg_array[i][2]; + ma_fli->ireg_offset_range = t6_ma_ireg_array[i][3]; + t4_read_indirect(padap, ma_fli->ireg_addr, ma_fli->ireg_data, + buff, ma_fli->ireg_offset_range, + ma_fli->ireg_local_offset); + ma_indr++; + } + + n = sizeof(t6_ma_ireg_array2) / (IREG_NUM_ELEM * sizeof(u32)); + for (i = 0; i < n; i++) { + struct ireg_field *ma_fli = &ma_indr->tp_pio; + u32 *buff = ma_indr->outbuf; + + ma_fli->ireg_addr = t6_ma_ireg_array2[i][0]; + ma_fli->ireg_data = t6_ma_ireg_array2[i][1]; + ma_fli->ireg_local_offset = t6_ma_ireg_array2[i][2]; + for (j = 0; j < t6_ma_ireg_array2[i][3]; j++) { + t4_read_indirect(padap, ma_fli->ireg_addr, + ma_fli->ireg_data, buff, 1, + ma_fli->ireg_local_offset); + buff++; + ma_fli->ireg_local_offset += 0x20; + } + ma_indr++; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_ulptx_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_ulptx_la *ulptx_la_buff; + struct cudbg_ver_hdr *ver_hdr; + u32 i, j; + int rc; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, + sizeof(struct cudbg_ver_hdr) + + sizeof(struct cudbg_ulptx_la), + &temp_buff); + if (rc) + return rc; + + ver_hdr = (struct cudbg_ver_hdr *)temp_buff.data; + ver_hdr->signature = CUDBG_ENTITY_SIGNATURE; + ver_hdr->revision = CUDBG_ULPTX_LA_REV; + ver_hdr->size = sizeof(struct cudbg_ulptx_la); + + ulptx_la_buff = (struct cudbg_ulptx_la *)(temp_buff.data + + sizeof(*ver_hdr)); + for (i = 0; i < CUDBG_NUM_ULPTX; i++) { + ulptx_la_buff->rdptr[i] = t4_read_reg(padap, + ULP_TX_LA_RDPTR_0_A + + 0x10 * i); + ulptx_la_buff->wrptr[i] = t4_read_reg(padap, + ULP_TX_LA_WRPTR_0_A + + 0x10 * i); + ulptx_la_buff->rddata[i] = t4_read_reg(padap, + ULP_TX_LA_RDDATA_0_A + + 0x10 * i); + for (j = 0; j < CUDBG_NUM_ULPTX_READ; j++) + ulptx_la_buff->rd_data[i][j] = + t4_read_reg(padap, + ULP_TX_LA_RDDATA_0_A + 0x10 * i); + } + + for (i = 0; i < CUDBG_NUM_ULPTX_ASIC_READ; i++) { + t4_write_reg(padap, ULP_TX_ASIC_DEBUG_CTRL_A, 0x1); + ulptx_la_buff->rdptr_asic[i] = + t4_read_reg(padap, ULP_TX_ASIC_DEBUG_CTRL_A); + ulptx_la_buff->rddata_asic[i][0] = + t4_read_reg(padap, ULP_TX_ASIC_DEBUG_0_A); + ulptx_la_buff->rddata_asic[i][1] = + t4_read_reg(padap, ULP_TX_ASIC_DEBUG_1_A); + ulptx_la_buff->rddata_asic[i][2] = + t4_read_reg(padap, ULP_TX_ASIC_DEBUG_2_A); + ulptx_la_buff->rddata_asic[i][3] = + t4_read_reg(padap, ULP_TX_ASIC_DEBUG_3_A); + ulptx_la_buff->rddata_asic[i][4] = + t4_read_reg(padap, ULP_TX_ASIC_DEBUG_4_A); + ulptx_la_buff->rddata_asic[i][5] = + t4_read_reg(padap, PM_RX_BASE_ADDR); + } + + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_up_cim_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + u32 local_offset, local_range; + struct ireg_buf *up_cim; + u32 size, j, iter; + u32 instance = 0; + int i, rc, n; + + if (is_t5(padap->params.chip)) + n = sizeof(t5_up_cim_reg_array) / + ((IREG_NUM_ELEM + 1) * sizeof(u32)); + else if (is_t6(padap->params.chip)) + n = sizeof(t6_up_cim_reg_array) / + ((IREG_NUM_ELEM + 1) * sizeof(u32)); + else + return CUDBG_STATUS_NOT_IMPLEMENTED; + + size = sizeof(struct ireg_buf) * n; + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + up_cim = (struct ireg_buf *)temp_buff.data; + for (i = 0; i < n; i++) { + struct ireg_field *up_cim_reg = &up_cim->tp_pio; + u32 *buff = up_cim->outbuf; + + if (is_t5(padap->params.chip)) { + up_cim_reg->ireg_addr = t5_up_cim_reg_array[i][0]; + up_cim_reg->ireg_data = t5_up_cim_reg_array[i][1]; + up_cim_reg->ireg_local_offset = + t5_up_cim_reg_array[i][2]; + up_cim_reg->ireg_offset_range = + t5_up_cim_reg_array[i][3]; + instance = t5_up_cim_reg_array[i][4]; + } else if (is_t6(padap->params.chip)) { + up_cim_reg->ireg_addr = t6_up_cim_reg_array[i][0]; + up_cim_reg->ireg_data = t6_up_cim_reg_array[i][1]; + up_cim_reg->ireg_local_offset = + t6_up_cim_reg_array[i][2]; + up_cim_reg->ireg_offset_range = + t6_up_cim_reg_array[i][3]; + instance = t6_up_cim_reg_array[i][4]; + } + + switch (instance) { + case NUM_CIM_CTL_TSCH_CHANNEL_INSTANCES: + iter = up_cim_reg->ireg_offset_range; + local_offset = 0x120; + local_range = 1; + break; + case NUM_CIM_CTL_TSCH_CHANNEL_TSCH_CLASS_INSTANCES: + iter = up_cim_reg->ireg_offset_range; + local_offset = 0x10; + local_range = 1; + break; + default: + iter = 1; + local_offset = 0; + local_range = up_cim_reg->ireg_offset_range; + break; + } + + for (j = 0; j < iter; j++, buff++) { + rc = t4_cim_read(padap, + up_cim_reg->ireg_local_offset + + (j * local_offset), local_range, buff); + if (rc) { + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + } + up_cim++; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_pbt_tables(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_pbt_tables *pbt; + int i, rc; + u32 addr; + + rc = cudbg_get_buff(pdbg_init, dbg_buff, + sizeof(struct cudbg_pbt_tables), + &temp_buff); + if (rc) + return rc; + + pbt = (struct cudbg_pbt_tables *)temp_buff.data; + /* PBT dynamic entries */ + addr = CUDBG_CHAC_PBT_ADDR; + for (i = 0; i < CUDBG_PBT_DYNAMIC_ENTRIES; i++) { + rc = t4_cim_read(padap, addr + (i * 4), 1, + &pbt->pbt_dynamic[i]); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + } + + /* PBT static entries */ + /* static entries start when bit 6 is set */ + addr = CUDBG_CHAC_PBT_ADDR + (1 << 6); + for (i = 0; i < CUDBG_PBT_STATIC_ENTRIES; i++) { + rc = t4_cim_read(padap, addr + (i * 4), 1, + &pbt->pbt_static[i]); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + } + + /* LRF entries */ + addr = CUDBG_CHAC_PBT_LRF; + for (i = 0; i < CUDBG_LRF_ENTRIES; i++) { + rc = t4_cim_read(padap, addr + (i * 4), 1, + &pbt->lrf_table[i]); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + } + + /* PBT data entries */ + addr = CUDBG_CHAC_PBT_DATA; + for (i = 0; i < CUDBG_PBT_DATA_ENTRIES; i++) { + rc = t4_cim_read(padap, addr + (i * 4), 1, + &pbt->pbt_data[i]); + if (rc) { + cudbg_err->sys_err = rc; + cudbg_put_buff(pdbg_init, &temp_buff); + return rc; + } + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_mbox_log(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_mbox_log *mboxlog = NULL; + struct cudbg_buffer temp_buff = { 0 }; + struct mbox_cmd_log *log = NULL; + struct mbox_cmd *entry; + unsigned int entry_idx; + u16 mbox_cmds; + int i, k, rc; + u64 flit; + u32 size; + + log = padap->mbox_log; + mbox_cmds = padap->mbox_log->size; + size = sizeof(struct cudbg_mbox_log) * mbox_cmds; + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + mboxlog = (struct cudbg_mbox_log *)temp_buff.data; + for (k = 0; k < mbox_cmds; k++) { + entry_idx = log->cursor + k; + if (entry_idx >= log->size) + entry_idx -= log->size; + + entry = mbox_cmd_log_entry(log, entry_idx); + /* skip over unused entries */ + if (entry->timestamp == 0) + continue; + + memcpy(&mboxlog->entry, entry, sizeof(struct mbox_cmd)); + for (i = 0; i < MBOX_LEN / 8; i++) { + flit = entry->cmd[i]; + mboxlog->hi[i] = (u32)(flit >> 32); + mboxlog->lo[i] = (u32)flit; + } + mboxlog++; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +int cudbg_collect_hma_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct ireg_buf *hma_indr; + int i, rc, n; + u32 size; + + if (CHELSIO_CHIP_VERSION(padap->params.chip) < CHELSIO_T6) + return CUDBG_STATUS_ENTITY_NOT_FOUND; + + n = sizeof(t6_hma_ireg_array) / (IREG_NUM_ELEM * sizeof(u32)); + size = sizeof(struct ireg_buf) * n; + rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff); + if (rc) + return rc; + + hma_indr = (struct ireg_buf *)temp_buff.data; + for (i = 0; i < n; i++) { + struct ireg_field *hma_fli = &hma_indr->tp_pio; + u32 *buff = hma_indr->outbuf; + + hma_fli->ireg_addr = t6_hma_ireg_array[i][0]; + hma_fli->ireg_data = t6_hma_ireg_array[i][1]; + hma_fli->ireg_local_offset = t6_hma_ireg_array[i][2]; + hma_fli->ireg_offset_range = t6_hma_ireg_array[i][3]; + t4_read_indirect(padap, hma_fli->ireg_addr, hma_fli->ireg_data, + buff, hma_fli->ireg_offset_range, + hma_fli->ireg_local_offset); + hma_indr++; + } + return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff); +} + +void cudbg_fill_qdesc_num_and_size(const struct adapter *padap, + u32 *num, u32 *size) +{ + u32 tot_entries = 0, tot_size = 0; + + /* NIC TXQ, RXQ, FLQ, and CTRLQ */ + tot_entries += MAX_ETH_QSETS * 3; + tot_entries += MAX_CTRL_QUEUES; + + tot_size += MAX_ETH_QSETS * MAX_TXQ_ENTRIES * MAX_TXQ_DESC_SIZE; + tot_size += MAX_ETH_QSETS * MAX_RSPQ_ENTRIES * MAX_RXQ_DESC_SIZE; + tot_size += MAX_ETH_QSETS * MAX_RX_BUFFERS * MAX_FL_DESC_SIZE; + tot_size += MAX_CTRL_QUEUES * MAX_CTRL_TXQ_ENTRIES * + MAX_CTRL_TXQ_DESC_SIZE; + + /* FW_EVTQ and INTRQ */ + tot_entries += INGQ_EXTRAS; + tot_size += INGQ_EXTRAS * MAX_RSPQ_ENTRIES * MAX_RXQ_DESC_SIZE; + + /* PTP_TXQ */ + tot_entries += 1; + tot_size += MAX_TXQ_ENTRIES * MAX_TXQ_DESC_SIZE; + + /* ULD TXQ, RXQ, and FLQ */ + tot_entries += CXGB4_TX_MAX * MAX_OFLD_QSETS; + tot_entries += CXGB4_ULD_MAX * MAX_ULD_QSETS * 2; + + tot_size += CXGB4_TX_MAX * MAX_OFLD_QSETS * MAX_TXQ_ENTRIES * + MAX_TXQ_DESC_SIZE; + tot_size += CXGB4_ULD_MAX * MAX_ULD_QSETS * MAX_RSPQ_ENTRIES * + MAX_RXQ_DESC_SIZE; + tot_size += CXGB4_ULD_MAX * MAX_ULD_QSETS * MAX_RX_BUFFERS * + MAX_FL_DESC_SIZE; + + /* ULD CIQ */ + tot_entries += CXGB4_ULD_MAX * MAX_ULD_QSETS; + tot_size += CXGB4_ULD_MAX * MAX_ULD_QSETS * SGE_MAX_IQ_SIZE * + MAX_RXQ_DESC_SIZE; + + /* ETHOFLD TXQ, RXQ, and FLQ */ + tot_entries += MAX_OFLD_QSETS * 3; + tot_size += MAX_OFLD_QSETS * MAX_TXQ_ENTRIES * MAX_TXQ_DESC_SIZE; + + tot_size += sizeof(struct cudbg_ver_hdr) + + sizeof(struct cudbg_qdesc_info) + + sizeof(struct cudbg_qdesc_entry) * tot_entries; + + if (num) + *num = tot_entries; + + if (size) + *size = tot_size; +} + +int cudbg_collect_qdesc(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + u32 num_queues = 0, tot_entries = 0, size = 0; + struct adapter *padap = pdbg_init->adap; + struct cudbg_buffer temp_buff = { 0 }; + struct cudbg_qdesc_entry *qdesc_entry; + struct cudbg_qdesc_info *qdesc_info; + struct cudbg_ver_hdr *ver_hdr; + struct sge *s = &padap->sge; + u32 i, j, cur_off, tot_len; + u8 *data; + int rc; + + cudbg_fill_qdesc_num_and_size(padap, &tot_entries, &size); + size = min_t(u32, size, CUDBG_DUMP_BUFF_SIZE); + tot_len = size; + data = kvzalloc(size, GFP_KERNEL); + if (!data) + return -ENOMEM; + + ver_hdr = (struct cudbg_ver_hdr *)data; + ver_hdr->signature = CUDBG_ENTITY_SIGNATURE; + ver_hdr->revision = CUDBG_QDESC_REV; + ver_hdr->size = sizeof(struct cudbg_qdesc_info); + size -= sizeof(*ver_hdr); + + qdesc_info = (struct cudbg_qdesc_info *)(data + + sizeof(*ver_hdr)); + size -= sizeof(*qdesc_info); + qdesc_entry = (struct cudbg_qdesc_entry *)qdesc_info->data; + +#define QDESC_GET(q, desc, type, label) do { \ + if (size <= 0) { \ + goto label; \ + } \ + if (desc) { \ + cudbg_fill_qdesc_##q(q, type, qdesc_entry); \ + size -= sizeof(*qdesc_entry) + qdesc_entry->data_size; \ + num_queues++; \ + qdesc_entry = cudbg_next_qdesc(qdesc_entry); \ + } \ +} while (0) + +#define QDESC_GET_TXQ(q, type, label) do { \ + struct sge_txq *txq = (struct sge_txq *)q; \ + QDESC_GET(txq, txq->desc, type, label); \ +} while (0) + +#define QDESC_GET_RXQ(q, type, label) do { \ + struct sge_rspq *rxq = (struct sge_rspq *)q; \ + QDESC_GET(rxq, rxq->desc, type, label); \ +} while (0) + +#define QDESC_GET_FLQ(q, type, label) do { \ + struct sge_fl *flq = (struct sge_fl *)q; \ + QDESC_GET(flq, flq->desc, type, label); \ +} while (0) + + /* NIC TXQ */ + for (i = 0; i < s->ethqsets; i++) + QDESC_GET_TXQ(&s->ethtxq[i].q, CUDBG_QTYPE_NIC_TXQ, out); + + /* NIC RXQ */ + for (i = 0; i < s->ethqsets; i++) + QDESC_GET_RXQ(&s->ethrxq[i].rspq, CUDBG_QTYPE_NIC_RXQ, out); + + /* NIC FLQ */ + for (i = 0; i < s->ethqsets; i++) + QDESC_GET_FLQ(&s->ethrxq[i].fl, CUDBG_QTYPE_NIC_FLQ, out); + + /* NIC CTRLQ */ + for (i = 0; i < padap->params.nports; i++) + QDESC_GET_TXQ(&s->ctrlq[i].q, CUDBG_QTYPE_CTRLQ, out); + + /* FW_EVTQ */ + QDESC_GET_RXQ(&s->fw_evtq, CUDBG_QTYPE_FWEVTQ, out); + + /* INTRQ */ + QDESC_GET_RXQ(&s->intrq, CUDBG_QTYPE_INTRQ, out); + + /* PTP_TXQ */ + QDESC_GET_TXQ(&s->ptptxq.q, CUDBG_QTYPE_PTP_TXQ, out); + + /* ULD Queues */ + mutex_lock(&uld_mutex); + + if (s->uld_txq_info) { + struct sge_uld_txq_info *utxq; + + /* ULD TXQ */ + for (j = 0; j < CXGB4_TX_MAX; j++) { + if (!s->uld_txq_info[j]) + continue; + + utxq = s->uld_txq_info[j]; + for (i = 0; i < utxq->ntxq; i++) + QDESC_GET_TXQ(&utxq->uldtxq[i].q, + cudbg_uld_txq_to_qtype(j), + out_unlock_uld); + } + } + + if (s->uld_rxq_info) { + struct sge_uld_rxq_info *urxq; + u32 base; + + /* ULD RXQ */ + for (j = 0; j < CXGB4_ULD_MAX; j++) { + if (!s->uld_rxq_info[j]) + continue; + + urxq = s->uld_rxq_info[j]; + for (i = 0; i < urxq->nrxq; i++) + QDESC_GET_RXQ(&urxq->uldrxq[i].rspq, + cudbg_uld_rxq_to_qtype(j), + out_unlock_uld); + } + + /* ULD FLQ */ + for (j = 0; j < CXGB4_ULD_MAX; j++) { + if (!s->uld_rxq_info[j]) + continue; + + urxq = s->uld_rxq_info[j]; + for (i = 0; i < urxq->nrxq; i++) + QDESC_GET_FLQ(&urxq->uldrxq[i].fl, + cudbg_uld_flq_to_qtype(j), + out_unlock_uld); + } + + /* ULD CIQ */ + for (j = 0; j < CXGB4_ULD_MAX; j++) { + if (!s->uld_rxq_info[j]) + continue; + + urxq = s->uld_rxq_info[j]; + base = urxq->nrxq; + for (i = 0; i < urxq->nciq; i++) + QDESC_GET_RXQ(&urxq->uldrxq[base + i].rspq, + cudbg_uld_ciq_to_qtype(j), + out_unlock_uld); + } + } + mutex_unlock(&uld_mutex); + + if (!padap->tc_mqprio) + goto out; + + mutex_lock(&padap->tc_mqprio->mqprio_mutex); + /* ETHOFLD TXQ */ + if (s->eohw_txq) + for (i = 0; i < s->eoqsets; i++) + QDESC_GET_TXQ(&s->eohw_txq[i].q, + CUDBG_QTYPE_ETHOFLD_TXQ, out_unlock_mqprio); + + /* ETHOFLD RXQ and FLQ */ + if (s->eohw_rxq) { + for (i = 0; i < s->eoqsets; i++) + QDESC_GET_RXQ(&s->eohw_rxq[i].rspq, + CUDBG_QTYPE_ETHOFLD_RXQ, out_unlock_mqprio); + + for (i = 0; i < s->eoqsets; i++) + QDESC_GET_FLQ(&s->eohw_rxq[i].fl, + CUDBG_QTYPE_ETHOFLD_FLQ, out_unlock_mqprio); + } + +out_unlock_mqprio: + mutex_unlock(&padap->tc_mqprio->mqprio_mutex); + +out: + qdesc_info->qdesc_entry_size = sizeof(*qdesc_entry); + qdesc_info->num_queues = num_queues; + cur_off = 0; + while (tot_len) { + u32 chunk_size = min_t(u32, tot_len, CUDBG_CHUNK_SIZE); + + rc = cudbg_get_buff(pdbg_init, dbg_buff, chunk_size, + &temp_buff); + if (rc) { + cudbg_err->sys_warn = CUDBG_STATUS_PARTIAL_DATA; + goto out_free; + } + + memcpy(temp_buff.data, data + cur_off, chunk_size); + tot_len -= chunk_size; + cur_off += chunk_size; + rc = cudbg_write_and_release_buff(pdbg_init, &temp_buff, + dbg_buff); + if (rc) { + cudbg_put_buff(pdbg_init, &temp_buff); + cudbg_err->sys_warn = CUDBG_STATUS_PARTIAL_DATA; + goto out_free; + } + } + +out_free: + kvfree(data); + +#undef QDESC_GET_FLQ +#undef QDESC_GET_RXQ +#undef QDESC_GET_TXQ +#undef QDESC_GET + + return rc; + +out_unlock_uld: + mutex_unlock(&uld_mutex); + goto out; +} + +int cudbg_collect_flash(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err) +{ + struct adapter *padap = pdbg_init->adap; + u32 count = padap->params.sf_size, n; + struct cudbg_buffer temp_buff = {0}; + u32 addr, i; + int rc; + + addr = FLASH_EXP_ROM_START; + + for (i = 0; i < count; i += SF_PAGE_SIZE) { + n = min_t(u32, count - i, SF_PAGE_SIZE); + + rc = cudbg_get_buff(pdbg_init, dbg_buff, n, &temp_buff); + if (rc) { + cudbg_err->sys_warn = CUDBG_STATUS_PARTIAL_DATA; + goto out; + } + rc = t4_read_flash(padap, addr, n, (u32 *)temp_buff.data, 0); + if (rc) + goto out; + + addr += (n * 4); + rc = cudbg_write_and_release_buff(pdbg_init, &temp_buff, + dbg_buff); + if (rc) { + cudbg_err->sys_warn = CUDBG_STATUS_PARTIAL_DATA; + goto out; + } + } + +out: + return rc; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.h new file mode 100644 index 000000000..d6d6cd298 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.h @@ -0,0 +1,283 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017 Chelsio Communications. All rights reserved. + */ + +#ifndef __CUDBG_LIB_H__ +#define __CUDBG_LIB_H__ + +int cudbg_collect_reg_dump(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_fw_devlog(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_ma_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_qcfg(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_ibq_tp0(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_ibq_tp1(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_ibq_ulp(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_ibq_sge0(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_ibq_sge1(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_ibq_ncsi(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_obq_ulp0(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_obq_ulp1(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_obq_ulp2(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_obq_ulp3(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_obq_sge(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_obq_ncsi(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_edc0_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_edc1_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_mc0_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_mc1_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_rss(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_rss_vf_config(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_tp_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_path_mtu(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_pm_stats(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_hw_sched(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_sge_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_ulprx_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_tp_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cim_pif_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_clk_info(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_obq_sge_rx_q0(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_obq_sge_rx_q1(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_pcie_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_pm_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_tid(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_pcie_config(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_dump_context(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_mps_tcam(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_vpd_data(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_le_tcam(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_cctrl(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_ma_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_ulptx_la(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_up_cim_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_pbt_tables(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_mbox_log(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_hma_indirect(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_hma_meminfo(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_qdesc(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); +int cudbg_collect_flash(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); + +u32 cudbg_get_entity_length(struct adapter *adap, u32 entity); +struct cudbg_entity_hdr *cudbg_get_entity_hdr(void *outbuf, int i); +void cudbg_align_debug_buffer(struct cudbg_buffer *dbg_buff, + struct cudbg_entity_hdr *entity_hdr); +u32 cudbg_cim_obq_size(struct adapter *padap, int qid); +int cudbg_dump_context_size(struct adapter *padap); + +int cudbg_fill_meminfo(struct adapter *padap, + struct cudbg_meminfo *meminfo_buff); +void cudbg_fill_le_tcam_info(struct adapter *padap, + struct cudbg_tcam *tcam_region); +void cudbg_fill_qdesc_num_and_size(const struct adapter *padap, + u32 *num, u32 *size); + +static inline u32 cudbg_uld_txq_to_qtype(u32 uld) +{ + switch (uld) { + case CXGB4_TX_OFLD: + return CUDBG_QTYPE_OFLD_TXQ; + case CXGB4_TX_CRYPTO: + return CUDBG_QTYPE_CRYPTO_TXQ; + } + + return CUDBG_QTYPE_UNKNOWN; +} + +static inline u32 cudbg_uld_rxq_to_qtype(u32 uld) +{ + switch (uld) { + case CXGB4_ULD_RDMA: + return CUDBG_QTYPE_RDMA_RXQ; + case CXGB4_ULD_ISCSI: + return CUDBG_QTYPE_ISCSI_RXQ; + case CXGB4_ULD_ISCSIT: + return CUDBG_QTYPE_ISCSIT_RXQ; + case CXGB4_ULD_CRYPTO: + return CUDBG_QTYPE_CRYPTO_RXQ; + case CXGB4_ULD_TLS: + return CUDBG_QTYPE_TLS_RXQ; + } + + return CUDBG_QTYPE_UNKNOWN; +} + +static inline u32 cudbg_uld_flq_to_qtype(u32 uld) +{ + switch (uld) { + case CXGB4_ULD_RDMA: + return CUDBG_QTYPE_RDMA_FLQ; + case CXGB4_ULD_ISCSI: + return CUDBG_QTYPE_ISCSI_FLQ; + case CXGB4_ULD_ISCSIT: + return CUDBG_QTYPE_ISCSIT_FLQ; + case CXGB4_ULD_CRYPTO: + return CUDBG_QTYPE_CRYPTO_FLQ; + case CXGB4_ULD_TLS: + return CUDBG_QTYPE_TLS_FLQ; + } + + return CUDBG_QTYPE_UNKNOWN; +} + +static inline u32 cudbg_uld_ciq_to_qtype(u32 uld) +{ + switch (uld) { + case CXGB4_ULD_RDMA: + return CUDBG_QTYPE_RDMA_CIQ; + } + + return CUDBG_QTYPE_UNKNOWN; +} + +static inline void cudbg_fill_qdesc_txq(const struct sge_txq *txq, + enum cudbg_qdesc_qtype type, + struct cudbg_qdesc_entry *entry) +{ + entry->qtype = type; + entry->qid = txq->cntxt_id; + entry->desc_size = sizeof(struct tx_desc); + entry->num_desc = txq->size; + entry->data_size = txq->size * sizeof(struct tx_desc); + memcpy(entry->data, txq->desc, entry->data_size); +} + +static inline void cudbg_fill_qdesc_rxq(const struct sge_rspq *rxq, + enum cudbg_qdesc_qtype type, + struct cudbg_qdesc_entry *entry) +{ + entry->qtype = type; + entry->qid = rxq->cntxt_id; + entry->desc_size = rxq->iqe_len; + entry->num_desc = rxq->size; + entry->data_size = rxq->size * rxq->iqe_len; + memcpy(entry->data, rxq->desc, entry->data_size); +} + +static inline void cudbg_fill_qdesc_flq(const struct sge_fl *flq, + enum cudbg_qdesc_qtype type, + struct cudbg_qdesc_entry *entry) +{ + entry->qtype = type; + entry->qid = flq->cntxt_id; + entry->desc_size = sizeof(__be64); + entry->num_desc = flq->size; + entry->data_size = flq->size * sizeof(__be64); + memcpy(entry->data, flq->desc, entry->data_size); +} + +static inline +struct cudbg_qdesc_entry *cudbg_next_qdesc(struct cudbg_qdesc_entry *e) +{ + return (struct cudbg_qdesc_entry *) + ((u8 *)e + sizeof(*e) + e->data_size); +} +#endif /* __CUDBG_LIB_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib_common.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib_common.h new file mode 100644 index 000000000..9fac777b0 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib_common.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017 Chelsio Communications. All rights reserved. + */ + +#ifndef __CUDBG_LIB_COMMON_H__ +#define __CUDBG_LIB_COMMON_H__ + +#define CUDBG_SIGNATURE 67856866 /* CUDB in ascii */ + +enum cudbg_dump_type { + CUDBG_DUMP_TYPE_MINI = 1, +}; + +enum cudbg_compression_type { + CUDBG_COMPRESSION_NONE = 1, + CUDBG_COMPRESSION_ZLIB, +}; + +struct cudbg_hdr { + u32 signature; + u32 hdr_len; + u16 major_ver; + u16 minor_ver; + u32 data_len; + u32 hdr_flags; + u16 max_entities; + u8 chip_ver; + u8 dump_type:3; + u8 reserved1:1; + u8 compress_type:4; + u32 reserved[8]; +}; + +struct cudbg_entity_hdr { + u32 entity_type; + u32 start_offset; + u32 size; + int hdr_flags; + u32 sys_warn; + u32 sys_err; + u8 num_pad; + u8 flag; /* bit 0 is used to indicate ext data */ + u8 reserved1[2]; + u32 next_ext_offset; /* pointer to next extended entity meta data */ + u32 reserved[5]; +}; + +struct cudbg_ver_hdr { + u32 signature; + u16 revision; + u16 size; +}; + +struct cudbg_buffer { + u32 size; + u32 offset; + char *data; +}; + +struct cudbg_error { + int sys_err; + int sys_warn; + int app_err; +}; + +#define CDUMP_MAX_COMP_BUF_SIZE ((64 * 1024) - 1) +#define CUDBG_CHUNK_SIZE ((CDUMP_MAX_COMP_BUF_SIZE / 1024) * 1024) + +int cudbg_get_buff(struct cudbg_init *pdbg_init, + struct cudbg_buffer *pdbg_buff, u32 size, + struct cudbg_buffer *pin_buff); +void cudbg_put_buff(struct cudbg_init *pdbg_init, + struct cudbg_buffer *pin_buff); +void cudbg_update_buff(struct cudbg_buffer *pin_buff, + struct cudbg_buffer *pout_buff); +#endif /* __CUDBG_LIB_COMMON_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.c b/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.c new file mode 100644 index 000000000..aad55fb35 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018 Chelsio Communications. All rights reserved. + */ + +#include <linux/zlib.h> + +#include "cxgb4.h" +#include "cudbg_if.h" +#include "cudbg_lib_common.h" +#include "cudbg_zlib.h" + +static int cudbg_get_compress_hdr(struct cudbg_buffer *pdbg_buff, + struct cudbg_buffer *pin_buff) +{ + if (pdbg_buff->offset + sizeof(struct cudbg_compress_hdr) > + pdbg_buff->size) + return CUDBG_STATUS_NO_MEM; + + pin_buff->data = (char *)pdbg_buff->data + pdbg_buff->offset; + pin_buff->offset = 0; + pin_buff->size = sizeof(struct cudbg_compress_hdr); + pdbg_buff->offset += sizeof(struct cudbg_compress_hdr); + return 0; +} + +int cudbg_compress_buff(struct cudbg_init *pdbg_init, + struct cudbg_buffer *pin_buff, + struct cudbg_buffer *pout_buff) +{ + struct cudbg_buffer temp_buff = { 0 }; + struct z_stream_s compress_stream; + struct cudbg_compress_hdr *c_hdr; + int rc; + + /* Write compression header to output buffer before compression */ + rc = cudbg_get_compress_hdr(pout_buff, &temp_buff); + if (rc) + return rc; + + c_hdr = (struct cudbg_compress_hdr *)temp_buff.data; + c_hdr->compress_id = CUDBG_ZLIB_COMPRESS_ID; + + memset(&compress_stream, 0, sizeof(struct z_stream_s)); + compress_stream.workspace = pdbg_init->workspace; + rc = zlib_deflateInit2(&compress_stream, Z_DEFAULT_COMPRESSION, + Z_DEFLATED, CUDBG_ZLIB_WIN_BITS, + CUDBG_ZLIB_MEM_LVL, Z_DEFAULT_STRATEGY); + if (rc != Z_OK) + return CUDBG_SYSTEM_ERROR; + + compress_stream.next_in = pin_buff->data; + compress_stream.avail_in = pin_buff->size; + compress_stream.next_out = pout_buff->data + pout_buff->offset; + compress_stream.avail_out = pout_buff->size - pout_buff->offset; + + rc = zlib_deflate(&compress_stream, Z_FINISH); + if (rc != Z_STREAM_END) + return CUDBG_SYSTEM_ERROR; + + rc = zlib_deflateEnd(&compress_stream); + if (rc != Z_OK) + return CUDBG_SYSTEM_ERROR; + + c_hdr->compress_size = compress_stream.total_out; + c_hdr->decompress_size = pin_buff->size; + pout_buff->offset += compress_stream.total_out; + + return 0; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.h new file mode 100644 index 000000000..f6d83289f --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2018 Chelsio Communications. All rights reserved. + */ + +#ifndef __CUDBG_ZLIB_H__ +#define __CUDBG_ZLIB_H__ + +#include <linux/zlib.h> + +#define CUDBG_ZLIB_COMPRESS_ID 17 +#define CUDBG_ZLIB_WIN_BITS 12 +#define CUDBG_ZLIB_MEM_LVL 4 + +struct cudbg_compress_hdr { + u32 compress_id; + u64 decompress_size; + u64 compress_size; + u64 rsvd[32]; +}; + +static inline int cudbg_get_workspace_size(void) +{ + return zlib_deflate_workspacesize(CUDBG_ZLIB_WIN_BITS, + CUDBG_ZLIB_MEM_LVL); +} + +int cudbg_compress_buff(struct cudbg_init *pdbg_init, + struct cudbg_buffer *pin_buff, + struct cudbg_buffer *pout_buff); +#endif /* __CUDBG_ZLIB_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h new file mode 100644 index 000000000..5657ac8cf --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h @@ -0,0 +1,2185 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_H__ +#define __CXGB4_H__ + +#include "t4_hw.h" + +#include <linux/bitops.h> +#include <linux/cache.h> +#include <linux/ethtool.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/netdevice.h> +#include <linux/pci.h> +#include <linux/spinlock.h> +#include <linux/timer.h> +#include <linux/vmalloc.h> +#include <linux/rhashtable.h> +#include <linux/etherdevice.h> +#include <linux/net_tstamp.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/ptp_classify.h> +#include <linux/crash_dump.h> +#include <linux/thermal.h> +#include <asm/io.h> +#include "t4_chip_type.h" +#include "cxgb4_uld.h" +#include "t4fw_api.h" + +#define CH_WARN(adap, fmt, ...) dev_warn(adap->pdev_dev, fmt, ## __VA_ARGS__) +extern struct list_head adapter_list; +extern struct list_head uld_list; +extern struct mutex uld_mutex; + +/* Suspend an Ethernet Tx queue with fewer available descriptors than this. + * This is the same as calc_tx_descs() for a TSO packet with + * nr_frags == MAX_SKB_FRAGS. + */ +#define ETHTXQ_STOP_THRES \ + (1 + DIV_ROUND_UP((3 * MAX_SKB_FRAGS) / 2 + (MAX_SKB_FRAGS & 1), 8)) + +#define FW_PARAM_DEV(param) \ + (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \ + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param)) + +#define FW_PARAM_PFVF(param) \ + (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \ + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param) | \ + FW_PARAMS_PARAM_Y_V(0) | \ + FW_PARAMS_PARAM_Z_V(0)) + +enum { + MAX_NPORTS = 4, /* max # of ports */ + SERNUM_LEN = 24, /* Serial # length */ + ID_LEN = 16, /* ID length */ + PN_LEN = 16, /* Part Number length */ + MACADDR_LEN = 12, /* MAC Address length */ +}; + +enum { + T4_REGMAP_SIZE = (160 * 1024), + T5_REGMAP_SIZE = (332 * 1024), +}; + +enum { + MEM_EDC0, + MEM_EDC1, + MEM_MC, + MEM_MC0 = MEM_MC, + MEM_MC1, + MEM_HMA, +}; + +enum { + MEMWIN0_APERTURE = 2048, + MEMWIN0_BASE = 0x1b800, + MEMWIN1_APERTURE = 32768, + MEMWIN1_BASE = 0x28000, + MEMWIN1_BASE_T5 = 0x52000, + MEMWIN2_APERTURE = 65536, + MEMWIN2_BASE = 0x30000, + MEMWIN2_APERTURE_T5 = 131072, + MEMWIN2_BASE_T5 = 0x60000, +}; + +enum dev_master { + MASTER_CANT, + MASTER_MAY, + MASTER_MUST +}; + +enum dev_state { + DEV_STATE_UNINIT, + DEV_STATE_INIT, + DEV_STATE_ERR +}; + +enum cc_pause { + PAUSE_RX = 1 << 0, + PAUSE_TX = 1 << 1, + PAUSE_AUTONEG = 1 << 2 +}; + +enum cc_fec { + FEC_AUTO = 1 << 0, /* IEEE 802.3 "automatic" */ + FEC_RS = 1 << 1, /* Reed-Solomon */ + FEC_BASER_RS = 1 << 2 /* BaseR/Reed-Solomon */ +}; + +enum { + CXGB4_ETHTOOL_FLASH_FW = 1, + CXGB4_ETHTOOL_FLASH_PHY = 2, + CXGB4_ETHTOOL_FLASH_BOOT = 3, + CXGB4_ETHTOOL_FLASH_BOOTCFG = 4 +}; + +enum cxgb4_netdev_tls_ops { + CXGB4_TLSDEV_OPS = 1, + CXGB4_XFRMDEV_OPS +}; + +struct cxgb4_bootcfg_data { + __le16 signature; + __u8 reserved[2]; +}; + +struct cxgb4_pcir_data { + __le32 signature; /* Signature. The string "PCIR" */ + __le16 vendor_id; /* Vendor Identification */ + __le16 device_id; /* Device Identification */ + __u8 vital_product[2]; /* Pointer to Vital Product Data */ + __u8 length[2]; /* PCIR Data Structure Length */ + __u8 revision; /* PCIR Data Structure Revision */ + __u8 class_code[3]; /* Class Code */ + __u8 image_length[2]; /* Image Length. Multiple of 512B */ + __u8 code_revision[2]; /* Revision Level of Code/Data */ + __u8 code_type; + __u8 indicator; + __u8 reserved[2]; +}; + +/* BIOS boot headers */ +struct cxgb4_pci_exp_rom_header { + __le16 signature; /* ROM Signature. Should be 0xaa55 */ + __u8 reserved[22]; /* Reserved per processor Architecture data */ + __le16 pcir_offset; /* Offset to PCI Data Structure */ +}; + +/* Legacy PCI Expansion ROM Header */ +struct legacy_pci_rom_hdr { + __u8 signature[2]; /* ROM Signature. Should be 0xaa55 */ + __u8 size512; /* Current Image Size in units of 512 bytes */ + __u8 initentry_point[4]; + __u8 cksum; /* Checksum computed on the entire Image */ + __u8 reserved[16]; /* Reserved */ + __le16 pcir_offset; /* Offset to PCI Data Struture */ +}; + +#define CXGB4_HDR_CODE1 0x00 +#define CXGB4_HDR_CODE2 0x03 +#define CXGB4_HDR_INDI 0x80 + +/* BOOT constants */ +enum { + BOOT_CFG_SIG = 0x4243, + BOOT_SIZE_INC = 512, + BOOT_SIGNATURE = 0xaa55, + BOOT_MIN_SIZE = sizeof(struct cxgb4_pci_exp_rom_header), + BOOT_MAX_SIZE = 1024 * BOOT_SIZE_INC, + PCIR_SIGNATURE = 0x52494350 +}; + +struct port_stats { + u64 tx_octets; /* total # of octets in good frames */ + u64 tx_frames; /* all good frames */ + u64 tx_bcast_frames; /* all broadcast frames */ + u64 tx_mcast_frames; /* all multicast frames */ + u64 tx_ucast_frames; /* all unicast frames */ + u64 tx_error_frames; /* all error frames */ + + u64 tx_frames_64; /* # of Tx frames in a particular range */ + u64 tx_frames_65_127; + u64 tx_frames_128_255; + u64 tx_frames_256_511; + u64 tx_frames_512_1023; + u64 tx_frames_1024_1518; + u64 tx_frames_1519_max; + + u64 tx_drop; /* # of dropped Tx frames */ + u64 tx_pause; /* # of transmitted pause frames */ + u64 tx_ppp0; /* # of transmitted PPP prio 0 frames */ + u64 tx_ppp1; /* # of transmitted PPP prio 1 frames */ + u64 tx_ppp2; /* # of transmitted PPP prio 2 frames */ + u64 tx_ppp3; /* # of transmitted PPP prio 3 frames */ + u64 tx_ppp4; /* # of transmitted PPP prio 4 frames */ + u64 tx_ppp5; /* # of transmitted PPP prio 5 frames */ + u64 tx_ppp6; /* # of transmitted PPP prio 6 frames */ + u64 tx_ppp7; /* # of transmitted PPP prio 7 frames */ + + u64 rx_octets; /* total # of octets in good frames */ + u64 rx_frames; /* all good frames */ + u64 rx_bcast_frames; /* all broadcast frames */ + u64 rx_mcast_frames; /* all multicast frames */ + u64 rx_ucast_frames; /* all unicast frames */ + u64 rx_too_long; /* # of frames exceeding MTU */ + u64 rx_jabber; /* # of jabber frames */ + u64 rx_fcs_err; /* # of received frames with bad FCS */ + u64 rx_len_err; /* # of received frames with length error */ + u64 rx_symbol_err; /* symbol errors */ + u64 rx_runt; /* # of short frames */ + + u64 rx_frames_64; /* # of Rx frames in a particular range */ + u64 rx_frames_65_127; + u64 rx_frames_128_255; + u64 rx_frames_256_511; + u64 rx_frames_512_1023; + u64 rx_frames_1024_1518; + u64 rx_frames_1519_max; + + u64 rx_pause; /* # of received pause frames */ + u64 rx_ppp0; /* # of received PPP prio 0 frames */ + u64 rx_ppp1; /* # of received PPP prio 1 frames */ + u64 rx_ppp2; /* # of received PPP prio 2 frames */ + u64 rx_ppp3; /* # of received PPP prio 3 frames */ + u64 rx_ppp4; /* # of received PPP prio 4 frames */ + u64 rx_ppp5; /* # of received PPP prio 5 frames */ + u64 rx_ppp6; /* # of received PPP prio 6 frames */ + u64 rx_ppp7; /* # of received PPP prio 7 frames */ + + u64 rx_ovflow0; /* drops due to buffer-group 0 overflows */ + u64 rx_ovflow1; /* drops due to buffer-group 1 overflows */ + u64 rx_ovflow2; /* drops due to buffer-group 2 overflows */ + u64 rx_ovflow3; /* drops due to buffer-group 3 overflows */ + u64 rx_trunc0; /* buffer-group 0 truncated packets */ + u64 rx_trunc1; /* buffer-group 1 truncated packets */ + u64 rx_trunc2; /* buffer-group 2 truncated packets */ + u64 rx_trunc3; /* buffer-group 3 truncated packets */ +}; + +struct lb_port_stats { + u64 octets; + u64 frames; + u64 bcast_frames; + u64 mcast_frames; + u64 ucast_frames; + u64 error_frames; + + u64 frames_64; + u64 frames_65_127; + u64 frames_128_255; + u64 frames_256_511; + u64 frames_512_1023; + u64 frames_1024_1518; + u64 frames_1519_max; + + u64 drop; + + u64 ovflow0; + u64 ovflow1; + u64 ovflow2; + u64 ovflow3; + u64 trunc0; + u64 trunc1; + u64 trunc2; + u64 trunc3; +}; + +struct tp_tcp_stats { + u32 tcp_out_rsts; + u64 tcp_in_segs; + u64 tcp_out_segs; + u64 tcp_retrans_segs; +}; + +struct tp_usm_stats { + u32 frames; + u32 drops; + u64 octets; +}; + +struct tp_fcoe_stats { + u32 frames_ddp; + u32 frames_drop; + u64 octets_ddp; +}; + +struct tp_err_stats { + u32 mac_in_errs[4]; + u32 hdr_in_errs[4]; + u32 tcp_in_errs[4]; + u32 tnl_cong_drops[4]; + u32 ofld_chan_drops[4]; + u32 tnl_tx_drops[4]; + u32 ofld_vlan_drops[4]; + u32 tcp6_in_errs[4]; + u32 ofld_no_neigh; + u32 ofld_cong_defer; +}; + +struct tp_cpl_stats { + u32 req[4]; + u32 rsp[4]; +}; + +struct tp_rdma_stats { + u32 rqe_dfr_pkt; + u32 rqe_dfr_mod; +}; + +struct sge_params { + u32 hps; /* host page size for our PF/VF */ + u32 eq_qpp; /* egress queues/page for our PF/VF */ + u32 iq_qpp; /* egress queues/page for our PF/VF */ +}; + +struct tp_params { + unsigned int tre; /* log2 of core clocks per TP tick */ + unsigned int la_mask; /* what events are recorded by TP LA */ + unsigned short tx_modq_map; /* TX modulation scheduler queue to */ + /* channel map */ + + uint32_t dack_re; /* DACK timer resolution */ + unsigned short tx_modq[NCHAN]; /* channel to modulation queue map */ + + u32 vlan_pri_map; /* cached TP_VLAN_PRI_MAP */ + u32 filter_mask; + u32 ingress_config; /* cached TP_INGRESS_CONFIG */ + + /* cached TP_OUT_CONFIG compressed error vector + * and passing outer header info for encapsulated packets. + */ + int rx_pkt_encap; + + /* TP_VLAN_PRI_MAP Compressed Filter Tuple field offsets. This is a + * subset of the set of fields which may be present in the Compressed + * Filter Tuple portion of filters and TCP TCB connections. The + * fields which are present are controlled by the TP_VLAN_PRI_MAP. + * Since a variable number of fields may or may not be present, their + * shifted field positions within the Compressed Filter Tuple may + * vary, or not even be present if the field isn't selected in + * TP_VLAN_PRI_MAP. Since some of these fields are needed in various + * places we store their offsets here, or a -1 if the field isn't + * present. + */ + int fcoe_shift; + int port_shift; + int vnic_shift; + int vlan_shift; + int tos_shift; + int protocol_shift; + int ethertype_shift; + int macmatch_shift; + int matchtype_shift; + int frag_shift; + + u64 hash_filter_mask; +}; + +struct vpd_params { + unsigned int cclk; + u8 sn[SERNUM_LEN + 1]; + u8 id[ID_LEN + 1]; + u8 pn[PN_LEN + 1]; + u8 na[MACADDR_LEN + 1]; +}; + +/* Maximum resources provisioned for a PCI PF. + */ +struct pf_resources { + unsigned int nvi; /* N virtual interfaces */ + unsigned int neq; /* N egress Qs */ + unsigned int nethctrl; /* N egress ETH or CTRL Qs */ + unsigned int niqflint; /* N ingress Qs/w free list(s) & intr */ + unsigned int niq; /* N ingress Qs */ + unsigned int tc; /* PCI-E traffic class */ + unsigned int pmask; /* port access rights mask */ + unsigned int nexactf; /* N exact MPS filters */ + unsigned int r_caps; /* read capabilities */ + unsigned int wx_caps; /* write/execute capabilities */ +}; + +struct pci_params { + unsigned char speed; + unsigned char width; +}; + +struct devlog_params { + u32 memtype; /* which memory (EDC0, EDC1, MC) */ + u32 start; /* start of log in firmware memory */ + u32 size; /* size of log */ +}; + +/* Stores chip specific parameters */ +struct arch_specific_params { + u8 nchan; + u8 pm_stats_cnt; + u8 cng_ch_bits_log; /* congestion channel map bits width */ + u16 mps_rplc_size; + u16 vfcount; + u32 sge_fl_db; + u16 mps_tcam_size; +}; + +struct adapter_params { + struct sge_params sge; + struct tp_params tp; + struct vpd_params vpd; + struct pf_resources pfres; + struct pci_params pci; + struct devlog_params devlog; + enum pcie_memwin drv_memwin; + + unsigned int cim_la_size; + + unsigned int sf_size; /* serial flash size in bytes */ + unsigned int sf_nsec; /* # of flash sectors */ + + unsigned int fw_vers; /* firmware version */ + unsigned int bs_vers; /* bootstrap version */ + unsigned int tp_vers; /* TP microcode version */ + unsigned int er_vers; /* expansion ROM version */ + unsigned int scfg_vers; /* Serial Configuration version */ + unsigned int vpd_vers; /* VPD Version */ + u8 api_vers[7]; + + unsigned short mtus[NMTUS]; + unsigned short a_wnd[NCCTRL_WIN]; + unsigned short b_wnd[NCCTRL_WIN]; + + unsigned char nports; /* # of ethernet ports */ + unsigned char portvec; + enum chip_type chip; /* chip code */ + struct arch_specific_params arch; /* chip specific params */ + unsigned char offload; + unsigned char crypto; /* HW capability for crypto */ + unsigned char ethofld; /* QoS support */ + + unsigned char bypass; + unsigned char hash_filter; + + unsigned int ofldq_wr_cred; + bool ulptx_memwrite_dsgl; /* use of T5 DSGL allowed */ + + unsigned int nsched_cls; /* number of traffic classes */ + unsigned int max_ordird_qp; /* Max read depth per RDMA QP */ + unsigned int max_ird_adapter; /* Max read depth per adapter */ + bool fr_nsmr_tpte_wr_support; /* FW support for FR_NSMR_TPTE_WR */ + u8 fw_caps_support; /* 32-bit Port Capabilities */ + bool filter2_wr_support; /* FW support for FILTER2_WR */ + unsigned int viid_smt_extn_support:1; /* FW returns vin and smt index */ + + /* MPS Buffer Group Map[per Port]. Bit i is set if buffer group i is + * used by the Port + */ + u8 mps_bg_map[MAX_NPORTS]; /* MPS Buffer Group Map */ + bool write_w_imm_support; /* FW supports WRITE_WITH_IMMEDIATE */ + bool write_cmpl_support; /* FW supports WRITE_CMPL */ +}; + +/* State needed to monitor the forward progress of SGE Ingress DMA activities + * and possible hangs. + */ +struct sge_idma_monitor_state { + unsigned int idma_1s_thresh; /* 1s threshold in Core Clock ticks */ + unsigned int idma_stalled[2]; /* synthesized stalled timers in HZ */ + unsigned int idma_state[2]; /* IDMA Hang detect state */ + unsigned int idma_qid[2]; /* IDMA Hung Ingress Queue ID */ + unsigned int idma_warn[2]; /* time to warning in HZ */ +}; + +/* Firmware Mailbox Command/Reply log. All values are in Host-Endian format. + * The access and execute times are signed in order to accommodate negative + * error returns. + */ +struct mbox_cmd { + u64 cmd[MBOX_LEN / 8]; /* a Firmware Mailbox Command/Reply */ + u64 timestamp; /* OS-dependent timestamp */ + u32 seqno; /* sequence number */ + s16 access; /* time (ms) to access mailbox */ + s16 execute; /* time (ms) to execute */ +}; + +struct mbox_cmd_log { + unsigned int size; /* number of entries in the log */ + unsigned int cursor; /* next position in the log to write */ + u32 seqno; /* next sequence number */ + /* variable length mailbox command log starts here */ +}; + +/* Given a pointer to a Firmware Mailbox Command Log and a log entry index, + * return a pointer to the specified entry. + */ +static inline struct mbox_cmd *mbox_cmd_log_entry(struct mbox_cmd_log *log, + unsigned int entry_idx) +{ + return &((struct mbox_cmd *)&(log)[1])[entry_idx]; +} + +#define FW_VERSION(chip) ( \ + FW_HDR_FW_VER_MAJOR_G(chip##FW_VERSION_MAJOR) | \ + FW_HDR_FW_VER_MINOR_G(chip##FW_VERSION_MINOR) | \ + FW_HDR_FW_VER_MICRO_G(chip##FW_VERSION_MICRO) | \ + FW_HDR_FW_VER_BUILD_G(chip##FW_VERSION_BUILD)) +#define FW_INTFVER(chip, intf) (FW_HDR_INTFVER_##intf) + +struct cxgb4_ethtool_lb_test { + struct completion completion; + int result; + int loopback; +}; + +struct fw_info { + u8 chip; + char *fs_name; + char *fw_mod_name; + struct fw_hdr fw_hdr; +}; + +struct trace_params { + u32 data[TRACE_LEN / 4]; + u32 mask[TRACE_LEN / 4]; + unsigned short snap_len; + unsigned short min_len; + unsigned char skip_ofst; + unsigned char skip_len; + unsigned char invert; + unsigned char port; +}; + +struct cxgb4_fw_data { + __be32 signature; + __u8 reserved[4]; +}; + +/* Firmware Port Capabilities types. */ + +typedef u16 fw_port_cap16_t; /* 16-bit Port Capabilities integral value */ +typedef u32 fw_port_cap32_t; /* 32-bit Port Capabilities integral value */ + +enum fw_caps { + FW_CAPS_UNKNOWN = 0, /* 0'ed out initial state */ + FW_CAPS16 = 1, /* old Firmware: 16-bit Port Capabilities */ + FW_CAPS32 = 2, /* new Firmware: 32-bit Port Capabilities */ +}; + +struct link_config { + fw_port_cap32_t pcaps; /* link capabilities */ + fw_port_cap32_t def_acaps; /* default advertised capabilities */ + fw_port_cap32_t acaps; /* advertised capabilities */ + fw_port_cap32_t lpacaps; /* peer advertised capabilities */ + + fw_port_cap32_t speed_caps; /* speed(s) user has requested */ + unsigned int speed; /* actual link speed (Mb/s) */ + + enum cc_pause requested_fc; /* flow control user has requested */ + enum cc_pause fc; /* actual link flow control */ + enum cc_pause advertised_fc; /* actual advertised flow control */ + + enum cc_fec requested_fec; /* Forward Error Correction: */ + enum cc_fec fec; /* requested and actual in use */ + + unsigned char autoneg; /* autonegotiating? */ + + unsigned char link_ok; /* link up? */ + unsigned char link_down_rc; /* link down reason */ + + bool new_module; /* ->OS Transceiver Module inserted */ + bool redo_l1cfg; /* ->CC redo current "sticky" L1 CFG */ +}; + +#define FW_LEN16(fw_struct) FW_CMD_LEN16_V(sizeof(fw_struct) / 16) + +enum { + MAX_ETH_QSETS = 32, /* # of Ethernet Tx/Rx queue sets */ + MAX_OFLD_QSETS = 16, /* # of offload Tx, iscsi Rx queue sets */ + MAX_CTRL_QUEUES = NCHAN, /* # of control Tx queues */ +}; + +enum { + MAX_TXQ_ENTRIES = 16384, + MAX_CTRL_TXQ_ENTRIES = 1024, + MAX_RSPQ_ENTRIES = 16384, + MAX_RX_BUFFERS = 16384, + MIN_TXQ_ENTRIES = 32, + MIN_CTRL_TXQ_ENTRIES = 32, + MIN_RSPQ_ENTRIES = 128, + MIN_FL_ENTRIES = 16 +}; + +enum { + MAX_TXQ_DESC_SIZE = 64, + MAX_RXQ_DESC_SIZE = 128, + MAX_FL_DESC_SIZE = 8, + MAX_CTRL_TXQ_DESC_SIZE = 64, +}; + +enum { + INGQ_EXTRAS = 2, /* firmware event queue and */ + /* forwarded interrupts */ + MAX_INGQ = MAX_ETH_QSETS + INGQ_EXTRAS, +}; + +enum { + PRIV_FLAG_PORT_TX_VM_BIT, +}; + +#define PRIV_FLAG_PORT_TX_VM BIT(PRIV_FLAG_PORT_TX_VM_BIT) + +#define PRIV_FLAGS_ADAP 0 +#define PRIV_FLAGS_PORT PRIV_FLAG_PORT_TX_VM + +struct adapter; +struct sge_rspq; + +#include "cxgb4_dcb.h" + +#ifdef CONFIG_CHELSIO_T4_FCOE +#include "cxgb4_fcoe.h" +#endif /* CONFIG_CHELSIO_T4_FCOE */ + +struct port_info { + struct adapter *adapter; + u16 viid; + int xact_addr_filt; /* index of exact MAC address filter */ + u16 rss_size; /* size of VI's RSS table slice */ + s8 mdio_addr; + enum fw_port_type port_type; + u8 mod_type; + u8 port_id; + u8 tx_chan; + u8 lport; /* associated offload logical port */ + u8 nqsets; /* # of qsets */ + u8 first_qset; /* index of first qset */ + u8 rss_mode; + struct link_config link_cfg; + u16 *rss; + struct port_stats stats_base; +#ifdef CONFIG_CHELSIO_T4_DCB + struct port_dcb_info dcb; /* Data Center Bridging support */ +#endif +#ifdef CONFIG_CHELSIO_T4_FCOE + struct cxgb_fcoe fcoe; +#endif /* CONFIG_CHELSIO_T4_FCOE */ + bool rxtstamp; /* Enable TS */ + struct hwtstamp_config tstamp_config; + bool ptp_enable; + struct sched_table *sched_tbl; + u32 eth_flags; + + /* viid and smt fields either returned by fw + * or decoded by parsing viid by driver. + */ + u8 vin; + u8 vivld; + u8 smt_idx; + u8 rx_cchan; + + bool tc_block_shared; + + /* Mirror VI information */ + u16 viid_mirror; + u16 nmirrorqsets; + u32 vi_mirror_count; + struct mutex vi_mirror_mutex; /* Sync access to Mirror VI info */ + struct cxgb4_ethtool_lb_test ethtool_lb; +}; + +struct dentry; +struct work_struct; + +enum { /* adapter flags */ + CXGB4_FULL_INIT_DONE = (1 << 0), + CXGB4_DEV_ENABLED = (1 << 1), + CXGB4_USING_MSI = (1 << 2), + CXGB4_USING_MSIX = (1 << 3), + CXGB4_FW_OK = (1 << 4), + CXGB4_RSS_TNLALLLOOKUP = (1 << 5), + CXGB4_USING_SOFT_PARAMS = (1 << 6), + CXGB4_MASTER_PF = (1 << 7), + CXGB4_FW_OFLD_CONN = (1 << 9), + CXGB4_ROOT_NO_RELAXED_ORDERING = (1 << 10), + CXGB4_SHUTTING_DOWN = (1 << 11), + CXGB4_SGE_DBQ_TIMER = (1 << 12), +}; + +enum { + ULP_CRYPTO_LOOKASIDE = 1 << 0, + ULP_CRYPTO_IPSEC_INLINE = 1 << 1, + ULP_CRYPTO_KTLS_INLINE = 1 << 3, +}; + +#define CXGB4_MIRROR_RXQ_DEFAULT_DESC_NUM 1024 +#define CXGB4_MIRROR_RXQ_DEFAULT_DESC_SIZE 64 +#define CXGB4_MIRROR_RXQ_DEFAULT_INTR_USEC 5 +#define CXGB4_MIRROR_RXQ_DEFAULT_PKT_CNT 8 + +#define CXGB4_MIRROR_FLQ_DEFAULT_DESC_NUM 72 + +struct rx_sw_desc; + +struct sge_fl { /* SGE free-buffer queue state */ + unsigned int avail; /* # of available Rx buffers */ + unsigned int pend_cred; /* new buffers since last FL DB ring */ + unsigned int cidx; /* consumer index */ + unsigned int pidx; /* producer index */ + unsigned long alloc_failed; /* # of times buffer allocation failed */ + unsigned long large_alloc_failed; + unsigned long mapping_err; /* # of RX Buffer DMA Mapping failures */ + unsigned long low; /* # of times momentarily starving */ + unsigned long starving; + /* RO fields */ + unsigned int cntxt_id; /* SGE context id for the free list */ + unsigned int size; /* capacity of free list */ + struct rx_sw_desc *sdesc; /* address of SW Rx descriptor ring */ + __be64 *desc; /* address of HW Rx descriptor ring */ + dma_addr_t addr; /* bus address of HW ring start */ + void __iomem *bar2_addr; /* address of BAR2 Queue registers */ + unsigned int bar2_qid; /* Queue ID for BAR2 Queue registers */ +}; + +/* A packet gather list */ +struct pkt_gl { + u64 sgetstamp; /* SGE Time Stamp for Ingress Packet */ + struct page_frag frags[MAX_SKB_FRAGS]; + void *va; /* virtual address of first byte */ + unsigned int nfrags; /* # of fragments */ + unsigned int tot_len; /* total length of fragments */ +}; + +typedef int (*rspq_handler_t)(struct sge_rspq *q, const __be64 *rsp, + const struct pkt_gl *gl); +typedef void (*rspq_flush_handler_t)(struct sge_rspq *q); +/* LRO related declarations for ULD */ +struct t4_lro_mgr { +#define MAX_LRO_SESSIONS 64 + u8 lro_session_cnt; /* # of sessions to aggregate */ + unsigned long lro_pkts; /* # of LRO super packets */ + unsigned long lro_merged; /* # of wire packets merged by LRO */ + struct sk_buff_head lroq; /* list of aggregated sessions */ +}; + +struct sge_rspq { /* state for an SGE response queue */ + struct napi_struct napi; + const __be64 *cur_desc; /* current descriptor in queue */ + unsigned int cidx; /* consumer index */ + u8 gen; /* current generation bit */ + u8 intr_params; /* interrupt holdoff parameters */ + u8 next_intr_params; /* holdoff params for next interrupt */ + u8 adaptive_rx; + u8 pktcnt_idx; /* interrupt packet threshold */ + u8 uld; /* ULD handling this queue */ + u8 idx; /* queue index within its group */ + int offset; /* offset into current Rx buffer */ + u16 cntxt_id; /* SGE context id for the response q */ + u16 abs_id; /* absolute SGE id for the response q */ + __be64 *desc; /* address of HW response ring */ + dma_addr_t phys_addr; /* physical address of the ring */ + void __iomem *bar2_addr; /* address of BAR2 Queue registers */ + unsigned int bar2_qid; /* Queue ID for BAR2 Queue registers */ + unsigned int iqe_len; /* entry size */ + unsigned int size; /* capacity of response queue */ + struct adapter *adap; + struct net_device *netdev; /* associated net device */ + rspq_handler_t handler; + rspq_flush_handler_t flush_handler; + struct t4_lro_mgr lro_mgr; +}; + +struct sge_eth_stats { /* Ethernet queue statistics */ + unsigned long pkts; /* # of ethernet packets */ + unsigned long lro_pkts; /* # of LRO super packets */ + unsigned long lro_merged; /* # of wire packets merged by LRO */ + unsigned long rx_cso; /* # of Rx checksum offloads */ + unsigned long vlan_ex; /* # of Rx VLAN extractions */ + unsigned long rx_drops; /* # of packets dropped due to no mem */ + unsigned long bad_rx_pkts; /* # of packets with err_vec!=0 */ +}; + +struct sge_eth_rxq { /* SW Ethernet Rx queue */ + struct sge_rspq rspq; + struct sge_fl fl; + struct sge_eth_stats stats; + struct msix_info *msix; +} ____cacheline_aligned_in_smp; + +struct sge_ofld_stats { /* offload queue statistics */ + unsigned long pkts; /* # of packets */ + unsigned long imm; /* # of immediate-data packets */ + unsigned long an; /* # of asynchronous notifications */ + unsigned long nomem; /* # of responses deferred due to no mem */ +}; + +struct sge_ofld_rxq { /* SW offload Rx queue */ + struct sge_rspq rspq; + struct sge_fl fl; + struct sge_ofld_stats stats; + struct msix_info *msix; +} ____cacheline_aligned_in_smp; + +struct tx_desc { + __be64 flit[8]; +}; + +struct ulptx_sgl; + +struct tx_sw_desc { + struct sk_buff *skb; /* SKB to free after getting completion */ + dma_addr_t addr[MAX_SKB_FRAGS + 1]; /* DMA mapped addresses */ +}; + +struct sge_txq { + unsigned int in_use; /* # of in-use Tx descriptors */ + unsigned int q_type; /* Q type Eth/Ctrl/Ofld */ + unsigned int size; /* # of descriptors */ + unsigned int cidx; /* SW consumer index */ + unsigned int pidx; /* producer index */ + unsigned long stops; /* # of times q has been stopped */ + unsigned long restarts; /* # of queue restarts */ + unsigned int cntxt_id; /* SGE context id for the Tx q */ + struct tx_desc *desc; /* address of HW Tx descriptor ring */ + struct tx_sw_desc *sdesc; /* address of SW Tx descriptor ring */ + struct sge_qstat *stat; /* queue status entry */ + dma_addr_t phys_addr; /* physical address of the ring */ + spinlock_t db_lock; + int db_disabled; + unsigned short db_pidx; + unsigned short db_pidx_inc; + void __iomem *bar2_addr; /* address of BAR2 Queue registers */ + unsigned int bar2_qid; /* Queue ID for BAR2 Queue registers */ +}; + +struct sge_eth_txq { /* state for an SGE Ethernet Tx queue */ + struct sge_txq q; + struct netdev_queue *txq; /* associated netdev TX queue */ +#ifdef CONFIG_CHELSIO_T4_DCB + u8 dcb_prio; /* DCB Priority bound to queue */ +#endif + u8 dbqt; /* SGE Doorbell Queue Timer in use */ + unsigned int dbqtimerix; /* SGE Doorbell Queue Timer Index */ + unsigned long tso; /* # of TSO requests */ + unsigned long uso; /* # of USO requests */ + unsigned long tx_cso; /* # of Tx checksum offloads */ + unsigned long vlan_ins; /* # of Tx VLAN insertions */ + unsigned long mapping_err; /* # of I/O MMU packet mapping errors */ +} ____cacheline_aligned_in_smp; + +struct sge_uld_txq { /* state for an SGE offload Tx queue */ + struct sge_txq q; + struct adapter *adap; + struct sk_buff_head sendq; /* list of backpressured packets */ + struct tasklet_struct qresume_tsk; /* restarts the queue */ + bool service_ofldq_running; /* service_ofldq() is processing sendq */ + u8 full; /* the Tx ring is full */ + unsigned long mapping_err; /* # of I/O MMU packet mapping errors */ +} ____cacheline_aligned_in_smp; + +struct sge_ctrl_txq { /* state for an SGE control Tx queue */ + struct sge_txq q; + struct adapter *adap; + struct sk_buff_head sendq; /* list of backpressured packets */ + struct tasklet_struct qresume_tsk; /* restarts the queue */ + u8 full; /* the Tx ring is full */ +} ____cacheline_aligned_in_smp; + +struct sge_uld_rxq_info { + char name[IFNAMSIZ]; /* name of ULD driver */ + struct sge_ofld_rxq *uldrxq; /* Rxq's for ULD */ + u16 *rspq_id; /* response queue id's of rxq */ + u16 nrxq; /* # of ingress uld queues */ + u16 nciq; /* # of completion queues */ + u8 uld; /* uld type */ +}; + +struct sge_uld_txq_info { + struct sge_uld_txq *uldtxq; /* Txq's for ULD */ + atomic_t users; /* num users */ + u16 ntxq; /* # of egress uld queues */ +}; + +/* struct to maintain ULD list to reallocate ULD resources on hotplug */ +struct cxgb4_uld_list { + struct cxgb4_uld_info uld_info; + struct list_head list_node; + enum cxgb4_uld uld_type; +}; + +enum sge_eosw_state { + CXGB4_EO_STATE_CLOSED = 0, /* Not ready to accept traffic */ + CXGB4_EO_STATE_FLOWC_OPEN_SEND, /* Send FLOWC open request */ + CXGB4_EO_STATE_FLOWC_OPEN_REPLY, /* Waiting for FLOWC open reply */ + CXGB4_EO_STATE_ACTIVE, /* Ready to accept traffic */ + CXGB4_EO_STATE_FLOWC_CLOSE_SEND, /* Send FLOWC close request */ + CXGB4_EO_STATE_FLOWC_CLOSE_REPLY, /* Waiting for FLOWC close reply */ +}; + +struct sge_eosw_txq { + spinlock_t lock; /* Per queue lock to synchronize completions */ + enum sge_eosw_state state; /* Current ETHOFLD State */ + struct tx_sw_desc *desc; /* Descriptor ring to hold packets */ + u32 ndesc; /* Number of descriptors */ + u32 pidx; /* Current Producer Index */ + u32 last_pidx; /* Last successfully transmitted Producer Index */ + u32 cidx; /* Current Consumer Index */ + u32 last_cidx; /* Last successfully reclaimed Consumer Index */ + u32 flowc_idx; /* Descriptor containing a FLOWC request */ + u32 inuse; /* Number of packets held in ring */ + + u32 cred; /* Current available credits */ + u32 ncompl; /* # of completions posted */ + u32 last_compl; /* # of credits consumed since last completion req */ + + u32 eotid; /* Index into EOTID table in software */ + u32 hwtid; /* Hardware EOTID index */ + + u32 hwqid; /* Underlying hardware queue index */ + struct net_device *netdev; /* Pointer to netdevice */ + struct tasklet_struct qresume_tsk; /* Restarts the queue */ + struct completion completion; /* completion for FLOWC rendezvous */ +}; + +struct sge_eohw_txq { + spinlock_t lock; /* Per queue lock */ + struct sge_txq q; /* HW Txq */ + struct adapter *adap; /* Backpointer to adapter */ + unsigned long tso; /* # of TSO requests */ + unsigned long uso; /* # of USO requests */ + unsigned long tx_cso; /* # of Tx checksum offloads */ + unsigned long vlan_ins; /* # of Tx VLAN insertions */ + unsigned long mapping_err; /* # of I/O MMU packet mapping errors */ +}; + +struct sge { + struct sge_eth_txq ethtxq[MAX_ETH_QSETS]; + struct sge_eth_txq ptptxq; + struct sge_ctrl_txq ctrlq[MAX_CTRL_QUEUES]; + + struct sge_eth_rxq ethrxq[MAX_ETH_QSETS]; + struct sge_rspq fw_evtq ____cacheline_aligned_in_smp; + struct sge_uld_rxq_info **uld_rxq_info; + struct sge_uld_txq_info **uld_txq_info; + + struct sge_rspq intrq ____cacheline_aligned_in_smp; + spinlock_t intrq_lock; + + struct sge_eohw_txq *eohw_txq; + struct sge_ofld_rxq *eohw_rxq; + + struct sge_eth_rxq *mirror_rxq[NCHAN]; + + u16 max_ethqsets; /* # of available Ethernet queue sets */ + u16 ethqsets; /* # of active Ethernet queue sets */ + u16 ethtxq_rover; /* Tx queue to clean up next */ + u16 ofldqsets; /* # of active ofld queue sets */ + u16 nqs_per_uld; /* # of Rx queues per ULD */ + u16 eoqsets; /* # of ETHOFLD queues */ + u16 mirrorqsets; /* # of Mirror queues */ + + u16 timer_val[SGE_NTIMERS]; + u8 counter_val[SGE_NCOUNTERS]; + u16 dbqtimer_tick; + u16 dbqtimer_val[SGE_NDBQTIMERS]; + u32 fl_pg_order; /* large page allocation size */ + u32 stat_len; /* length of status page at ring end */ + u32 pktshift; /* padding between CPL & packet data */ + u32 fl_align; /* response queue message alignment */ + u32 fl_starve_thres; /* Free List starvation threshold */ + + struct sge_idma_monitor_state idma_monitor; + unsigned int egr_start; + unsigned int egr_sz; + unsigned int ingr_start; + unsigned int ingr_sz; + void **egr_map; /* qid->queue egress queue map */ + struct sge_rspq **ingr_map; /* qid->queue ingress queue map */ + unsigned long *starving_fl; + unsigned long *txq_maperr; + unsigned long *blocked_fl; + struct timer_list rx_timer; /* refills starving FLs */ + struct timer_list tx_timer; /* checks Tx queues */ + + int fwevtq_msix_idx; /* Index to firmware event queue MSI-X info */ + int nd_msix_idx; /* Index to non-data interrupts MSI-X info */ +}; + +#define for_each_ethrxq(sge, i) for (i = 0; i < (sge)->ethqsets; i++) +#define for_each_ofldtxq(sge, i) for (i = 0; i < (sge)->ofldqsets; i++) + +struct l2t_data; + +#ifdef CONFIG_PCI_IOV + +/* T4 supports SRIOV on PF0-3 and T5 on PF0-7. However, the Serial + * Configuration initialization for T5 only has SR-IOV functionality enabled + * on PF0-3 in order to simplify everything. + */ +#define NUM_OF_PF_WITH_SRIOV 4 + +#endif + +struct doorbell_stats { + u32 db_drop; + u32 db_empty; + u32 db_full; +}; + +struct hash_mac_addr { + struct list_head list; + u8 addr[ETH_ALEN]; + unsigned int iface_mac; +}; + +struct msix_bmap { + unsigned long *msix_bmap; + unsigned int mapsize; + spinlock_t lock; /* lock for acquiring bitmap */ +}; + +struct msix_info { + unsigned short vec; + char desc[IFNAMSIZ + 10]; + unsigned int idx; + cpumask_var_t aff_mask; +}; + +struct vf_info { + unsigned char vf_mac_addr[ETH_ALEN]; + unsigned int tx_rate; + bool pf_set_mac; + u16 vlan; + int link_state; +}; + +enum { + HMA_DMA_MAPPED_FLAG = 1 +}; + +struct hma_data { + unsigned char flags; + struct sg_table *sgt; + dma_addr_t *phy_addr; /* physical address of the page */ +}; + +struct mbox_list { + struct list_head list; +}; + +#if IS_ENABLED(CONFIG_THERMAL) +struct ch_thermal { + struct thermal_zone_device *tzdev; + int trip_temp; + int trip_type; +}; +#endif + +struct mps_entries_ref { + struct list_head list; + u8 addr[ETH_ALEN]; + u8 mask[ETH_ALEN]; + u16 idx; + refcount_t refcnt; +}; + +struct cxgb4_ethtool_filter_info { + u32 *loc_array; /* Array holding the actual TIDs set to filters */ + unsigned long *bmap; /* Bitmap for managing filters in use */ + u32 in_use; /* # of filters in use */ +}; + +struct cxgb4_ethtool_filter { + u32 nentries; /* Adapter wide number of supported filters */ + struct cxgb4_ethtool_filter_info *port; /* Per port entry */ +}; + +struct adapter { + void __iomem *regs; + void __iomem *bar2; + u32 t4_bar0; + struct pci_dev *pdev; + struct device *pdev_dev; + const char *name; + unsigned int mbox; + unsigned int pf; + unsigned int flags; + unsigned int adap_idx; + enum chip_type chip; + u32 eth_flags; + + int msg_enable; + __be16 vxlan_port; + __be16 geneve_port; + + struct adapter_params params; + struct cxgb4_virt_res vres; + unsigned int swintr; + + /* MSI-X Info for NIC and OFLD queues */ + struct msix_info *msix_info; + struct msix_bmap msix_bmap; + + struct doorbell_stats db_stats; + struct sge sge; + + struct net_device *port[MAX_NPORTS]; + u8 chan_map[NCHAN]; /* channel -> port map */ + + struct vf_info *vfinfo; + u8 num_vfs; + + u32 filter_mode; + unsigned int l2t_start; + unsigned int l2t_end; + struct l2t_data *l2t; + unsigned int clipt_start; + unsigned int clipt_end; + struct clip_tbl *clipt; + unsigned int rawf_start; + unsigned int rawf_cnt; + struct smt_data *smt; + struct cxgb4_uld_info *uld; + void *uld_handle[CXGB4_ULD_MAX]; + unsigned int num_uld; + unsigned int num_ofld_uld; + struct list_head list_node; + struct list_head rcu_node; + struct list_head mac_hlist; /* list of MAC addresses in MPS Hash */ + struct list_head mps_ref; + spinlock_t mps_ref_lock; /* lock for syncing mps ref/def activities */ + + void *iscsi_ppm; + + struct tid_info tids; + void **tid_release_head; + spinlock_t tid_release_lock; + struct workqueue_struct *workq; + struct work_struct tid_release_task; + struct work_struct db_full_task; + struct work_struct db_drop_task; + struct work_struct fatal_err_notify_task; + bool tid_release_task_busy; + + /* lock for mailbox cmd list */ + spinlock_t mbox_lock; + struct mbox_list mlist; + + /* support for mailbox command/reply logging */ +#define T4_OS_LOG_MBOX_CMDS 256 + struct mbox_cmd_log *mbox_log; + + struct mutex uld_mutex; + + struct dentry *debugfs_root; + bool use_bd; /* Use SGE Back Door intfc for reading SGE Contexts */ + bool trace_rss; /* 1 implies that different RSS flit per filter is + * used per filter else if 0 default RSS flit is + * used for all 4 filters. + */ + + struct ptp_clock *ptp_clock; + struct ptp_clock_info ptp_clock_info; + struct sk_buff *ptp_tx_skb; + /* ptp lock */ + spinlock_t ptp_lock; + spinlock_t stats_lock; + spinlock_t win0_lock ____cacheline_aligned_in_smp; + + /* TC u32 offload */ + struct cxgb4_tc_u32_table *tc_u32; + struct chcr_ktls chcr_ktls; + struct chcr_stats_debug chcr_stats; +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + struct ch_ktls_stats_debug ch_ktls_stats; +#endif +#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) + struct ch_ipsec_stats_debug ch_ipsec_stats; +#endif + + /* TC flower offload */ + bool tc_flower_initialized; + struct rhashtable flower_tbl; + struct rhashtable_params flower_ht_params; + struct timer_list flower_stats_timer; + struct work_struct flower_stats_work; + + /* Ethtool Dump */ + struct ethtool_dump eth_dump; + + /* HMA */ + struct hma_data hma; + + struct srq_data *srq; + + /* Dump buffer for collecting logs in kdump kernel */ + struct vmcoredd_data vmcoredd; +#if IS_ENABLED(CONFIG_THERMAL) + struct ch_thermal ch_thermal; +#endif + + /* TC MQPRIO offload */ + struct cxgb4_tc_mqprio *tc_mqprio; + + /* TC MATCHALL classifier offload */ + struct cxgb4_tc_matchall *tc_matchall; + + /* Ethtool n-tuple */ + struct cxgb4_ethtool_filter *ethtool_filters; +}; + +/* Support for "sched-class" command to allow a TX Scheduling Class to be + * programmed with various parameters. + */ +struct ch_sched_params { + u8 type; /* packet or flow */ + union { + struct { + u8 level; /* scheduler hierarchy level */ + u8 mode; /* per-class or per-flow */ + u8 rateunit; /* bit or packet rate */ + u8 ratemode; /* %port relative or kbps absolute */ + u8 channel; /* scheduler channel [0..N] */ + u8 class; /* scheduler class [0..N] */ + u32 minrate; /* minimum rate */ + u32 maxrate; /* maximum rate */ + u16 weight; /* percent weight */ + u16 pktsize; /* average packet size */ + u16 burstsize; /* burst buffer size */ + } params; + } u; +}; + +enum { + SCHED_CLASS_TYPE_PACKET = 0, /* class type */ +}; + +enum { + SCHED_CLASS_LEVEL_CL_RL = 0, /* class rate limiter */ + SCHED_CLASS_LEVEL_CH_RL = 2, /* channel rate limiter */ +}; + +enum { + SCHED_CLASS_MODE_CLASS = 0, /* per-class scheduling */ + SCHED_CLASS_MODE_FLOW, /* per-flow scheduling */ +}; + +enum { + SCHED_CLASS_RATEUNIT_BITS = 0, /* bit rate scheduling */ +}; + +enum { + SCHED_CLASS_RATEMODE_ABS = 1, /* Kb/s */ +}; + +/* Support for "sched_queue" command to allow one or more NIC TX Queues + * to be bound to a TX Scheduling Class. + */ +struct ch_sched_queue { + s8 queue; /* queue index */ + s8 class; /* class index */ +}; + +/* Support for "sched_flowc" command to allow one or more FLOWC + * to be bound to a TX Scheduling Class. + */ +struct ch_sched_flowc { + s32 tid; /* TID to bind */ + s8 class; /* class index */ +}; + +/* Defined bit width of user definable filter tuples + */ +#define ETHTYPE_BITWIDTH 16 +#define FRAG_BITWIDTH 1 +#define MACIDX_BITWIDTH 9 +#define FCOE_BITWIDTH 1 +#define IPORT_BITWIDTH 3 +#define MATCHTYPE_BITWIDTH 3 +#define PROTO_BITWIDTH 8 +#define TOS_BITWIDTH 8 +#define PF_BITWIDTH 8 +#define VF_BITWIDTH 8 +#define IVLAN_BITWIDTH 16 +#define OVLAN_BITWIDTH 16 +#define ENCAP_VNI_BITWIDTH 24 + +/* Filter matching rules. These consist of a set of ingress packet field + * (value, mask) tuples. The associated ingress packet field matches the + * tuple when ((field & mask) == value). (Thus a wildcard "don't care" field + * rule can be constructed by specifying a tuple of (0, 0).) A filter rule + * matches an ingress packet when all of the individual individual field + * matching rules are true. + * + * Partial field masks are always valid, however, while it may be easy to + * understand their meanings for some fields (e.g. IP address to match a + * subnet), for others making sensible partial masks is less intuitive (e.g. + * MPS match type) ... + * + * Most of the following data structures are modeled on T4 capabilities. + * Drivers for earlier chips use the subsets which make sense for those chips. + * We really need to come up with a hardware-independent mechanism to + * represent hardware filter capabilities ... + */ +struct ch_filter_tuple { + /* Compressed header matching field rules. The TP_VLAN_PRI_MAP + * register selects which of these fields will participate in the + * filter match rules -- up to a maximum of 36 bits. Because + * TP_VLAN_PRI_MAP is a global register, all filters must use the same + * set of fields. + */ + uint32_t ethtype:ETHTYPE_BITWIDTH; /* Ethernet type */ + uint32_t frag:FRAG_BITWIDTH; /* IP fragmentation header */ + uint32_t ivlan_vld:1; /* inner VLAN valid */ + uint32_t ovlan_vld:1; /* outer VLAN valid */ + uint32_t pfvf_vld:1; /* PF/VF valid */ + uint32_t encap_vld:1; /* Encapsulation valid */ + uint32_t macidx:MACIDX_BITWIDTH; /* exact match MAC index */ + uint32_t fcoe:FCOE_BITWIDTH; /* FCoE packet */ + uint32_t iport:IPORT_BITWIDTH; /* ingress port */ + uint32_t matchtype:MATCHTYPE_BITWIDTH; /* MPS match type */ + uint32_t proto:PROTO_BITWIDTH; /* protocol type */ + uint32_t tos:TOS_BITWIDTH; /* TOS/Traffic Type */ + uint32_t pf:PF_BITWIDTH; /* PCI-E PF ID */ + uint32_t vf:VF_BITWIDTH; /* PCI-E VF ID */ + uint32_t ivlan:IVLAN_BITWIDTH; /* inner VLAN */ + uint32_t ovlan:OVLAN_BITWIDTH; /* outer VLAN */ + uint32_t vni:ENCAP_VNI_BITWIDTH; /* VNI of tunnel */ + + /* Uncompressed header matching field rules. These are always + * available for field rules. + */ + uint8_t lip[16]; /* local IP address (IPv4 in [3:0]) */ + uint8_t fip[16]; /* foreign IP address (IPv4 in [3:0]) */ + uint16_t lport; /* local port */ + uint16_t fport; /* foreign port */ +}; + +/* A filter ioctl command. + */ +struct ch_filter_specification { + /* Administrative fields for filter. + */ + uint32_t hitcnts:1; /* count filter hits in TCB */ + uint32_t prio:1; /* filter has priority over active/server */ + + /* Fundamental filter typing. This is the one element of filter + * matching that doesn't exist as a (value, mask) tuple. + */ + uint32_t type:1; /* 0 => IPv4, 1 => IPv6 */ + u32 hash:1; /* 0 => wild-card, 1 => exact-match */ + + /* Packet dispatch information. Ingress packets which match the + * filter rules will be dropped, passed to the host or switched back + * out as egress packets. + */ + uint32_t action:2; /* drop, pass, switch */ + + uint32_t rpttid:1; /* report TID in RSS hash field */ + + uint32_t dirsteer:1; /* 0 => RSS, 1 => steer to iq */ + uint32_t iq:10; /* ingress queue */ + + uint32_t maskhash:1; /* dirsteer=0: store RSS hash in TCB */ + uint32_t dirsteerhash:1;/* dirsteer=1: 0 => TCB contains RSS hash */ + /* 1 => TCB contains IQ ID */ + + /* Switch proxy/rewrite fields. An ingress packet which matches a + * filter with "switch" set will be looped back out as an egress + * packet -- potentially with some Ethernet header rewriting. + */ + uint32_t eport:2; /* egress port to switch packet out */ + uint32_t newdmac:1; /* rewrite destination MAC address */ + uint32_t newsmac:1; /* rewrite source MAC address */ + uint32_t newvlan:2; /* rewrite VLAN Tag */ + uint32_t nat_mode:3; /* specify NAT operation mode */ + uint8_t dmac[ETH_ALEN]; /* new destination MAC address */ + uint8_t smac[ETH_ALEN]; /* new source MAC address */ + uint16_t vlan; /* VLAN Tag to insert */ + + u8 nat_lip[16]; /* local IP to use after NAT'ing */ + u8 nat_fip[16]; /* foreign IP to use after NAT'ing */ + u16 nat_lport; /* local port to use after NAT'ing */ + u16 nat_fport; /* foreign port to use after NAT'ing */ + + u32 tc_prio; /* TC's filter priority index */ + u64 tc_cookie; /* Unique cookie identifying TC rules */ + + /* reservation for future additions */ + u8 rsvd[12]; + + /* Filter rule value/mask pairs. + */ + struct ch_filter_tuple val; + struct ch_filter_tuple mask; +}; + +enum { + FILTER_PASS = 0, /* default */ + FILTER_DROP, + FILTER_SWITCH +}; + +enum { + VLAN_NOCHANGE = 0, /* default */ + VLAN_REMOVE, + VLAN_INSERT, + VLAN_REWRITE +}; + +enum { + NAT_MODE_NONE = 0, /* No NAT performed */ + NAT_MODE_DIP, /* NAT on Dst IP */ + NAT_MODE_DIP_DP, /* NAT on Dst IP, Dst Port */ + NAT_MODE_DIP_DP_SIP, /* NAT on Dst IP, Dst Port and Src IP */ + NAT_MODE_DIP_DP_SP, /* NAT on Dst IP, Dst Port and Src Port */ + NAT_MODE_SIP_SP, /* NAT on Src IP and Src Port */ + NAT_MODE_DIP_SIP_SP, /* NAT on Dst IP, Src IP and Src Port */ + NAT_MODE_ALL /* NAT on entire 4-tuple */ +}; + +#define CXGB4_FILTER_TYPE_MAX 2 + +/* Host shadow copy of ingress filter entry. This is in host native format + * and doesn't match the ordering or bit order, etc. of the hardware of the + * firmware command. The use of bit-field structure elements is purely to + * remind ourselves of the field size limitations and save memory in the case + * where the filter table is large. + */ +struct filter_entry { + /* Administrative fields for filter. */ + u32 valid:1; /* filter allocated and valid */ + u32 locked:1; /* filter is administratively locked */ + + u32 pending:1; /* filter action is pending firmware reply */ + struct filter_ctx *ctx; /* Caller's completion hook */ + struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ + struct smt_entry *smt; /* Source Mac Table entry for smac */ + struct net_device *dev; /* Associated net device */ + u32 tid; /* This will store the actual tid */ + + /* The filter itself. Most of this is a straight copy of information + * provided by the extended ioctl(). Some fields are translated to + * internal forms -- for instance the Ingress Queue ID passed in from + * the ioctl() is translated into the Absolute Ingress Queue ID. + */ + struct ch_filter_specification fs; +}; + +static inline int is_offload(const struct adapter *adap) +{ + return adap->params.offload; +} + +static inline int is_hashfilter(const struct adapter *adap) +{ + return adap->params.hash_filter; +} + +static inline int is_pci_uld(const struct adapter *adap) +{ + return adap->params.crypto; +} + +static inline int is_uld(const struct adapter *adap) +{ + return (adap->params.offload || adap->params.crypto); +} + +static inline int is_ethofld(const struct adapter *adap) +{ + return adap->params.ethofld; +} + +static inline u32 t4_read_reg(struct adapter *adap, u32 reg_addr) +{ + return readl(adap->regs + reg_addr); +} + +static inline void t4_write_reg(struct adapter *adap, u32 reg_addr, u32 val) +{ + writel(val, adap->regs + reg_addr); +} + +#ifndef readq +static inline u64 readq(const volatile void __iomem *addr) +{ + return readl(addr) + ((u64)readl(addr + 4) << 32); +} + +static inline void writeq(u64 val, volatile void __iomem *addr) +{ + writel(val, addr); + writel(val >> 32, addr + 4); +} +#endif + +static inline u64 t4_read_reg64(struct adapter *adap, u32 reg_addr) +{ + return readq(adap->regs + reg_addr); +} + +static inline void t4_write_reg64(struct adapter *adap, u32 reg_addr, u64 val) +{ + writeq(val, adap->regs + reg_addr); +} + +/** + * t4_set_hw_addr - store a port's MAC address in SW + * @adapter: the adapter + * @port_idx: the port index + * @hw_addr: the Ethernet address + * + * Store the Ethernet address of the given port in SW. Called by the common + * code when it retrieves a port's Ethernet address from EEPROM. + */ +static inline void t4_set_hw_addr(struct adapter *adapter, int port_idx, + u8 hw_addr[]) +{ + eth_hw_addr_set(adapter->port[port_idx], hw_addr); + ether_addr_copy(adapter->port[port_idx]->perm_addr, hw_addr); +} + +/** + * netdev2pinfo - return the port_info structure associated with a net_device + * @dev: the netdev + * + * Return the struct port_info associated with a net_device + */ +static inline struct port_info *netdev2pinfo(const struct net_device *dev) +{ + return netdev_priv(dev); +} + +/** + * adap2pinfo - return the port_info of a port + * @adap: the adapter + * @idx: the port index + * + * Return the port_info structure for the port of the given index. + */ +static inline struct port_info *adap2pinfo(struct adapter *adap, int idx) +{ + return netdev_priv(adap->port[idx]); +} + +/** + * netdev2adap - return the adapter structure associated with a net_device + * @dev: the netdev + * + * Return the struct adapter associated with a net_device + */ +static inline struct adapter *netdev2adap(const struct net_device *dev) +{ + return netdev2pinfo(dev)->adapter; +} + +/* Return a version number to identify the type of adapter. The scheme is: + * - bits 0..9: chip version + * - bits 10..15: chip revision + * - bits 16..23: register dump version + */ +static inline unsigned int mk_adap_vers(struct adapter *ap) +{ + return CHELSIO_CHIP_VERSION(ap->params.chip) | + (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16); +} + +/* Return a queue's interrupt hold-off time in us. 0 means no timer. */ +static inline unsigned int qtimer_val(const struct adapter *adap, + const struct sge_rspq *q) +{ + unsigned int idx = q->intr_params >> 1; + + return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0; +} + +/* driver name used for ethtool_drvinfo */ +extern char cxgb4_driver_name[]; + +void t4_os_portmod_changed(struct adapter *adap, int port_id); +void t4_os_link_changed(struct adapter *adap, int port_id, int link_stat); + +void t4_free_sge_resources(struct adapter *adap); +void t4_free_ofld_rxqs(struct adapter *adap, int n, struct sge_ofld_rxq *q); +irq_handler_t t4_intr_handler(struct adapter *adap); +netdev_tx_t t4_start_xmit(struct sk_buff *skb, struct net_device *dev); +int cxgb4_selftest_lb_pkt(struct net_device *netdev); +int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, + const struct pkt_gl *gl); +int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb); +int t4_ofld_send(struct adapter *adap, struct sk_buff *skb); +int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, + struct net_device *dev, int intr_idx, + struct sge_fl *fl, rspq_handler_t hnd, + rspq_flush_handler_t flush_handler, int cong); +int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq, + struct net_device *dev, struct netdev_queue *netdevq, + unsigned int iqid, u8 dbqt); +int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq, + struct net_device *dev, unsigned int iqid, + unsigned int cmplqid); +int t4_sge_mod_ctrl_txq(struct adapter *adap, unsigned int eqid, + unsigned int cmplqid); +int t4_sge_alloc_uld_txq(struct adapter *adap, struct sge_uld_txq *txq, + struct net_device *dev, unsigned int iqid, + unsigned int uld_type); +int t4_sge_alloc_ethofld_txq(struct adapter *adap, struct sge_eohw_txq *txq, + struct net_device *dev, u32 iqid); +void t4_sge_free_ethofld_txq(struct adapter *adap, struct sge_eohw_txq *txq); +irqreturn_t t4_sge_intr_msix(int irq, void *cookie); +int t4_sge_init(struct adapter *adap); +void t4_sge_start(struct adapter *adap); +void t4_sge_stop(struct adapter *adap); +int t4_sge_eth_txq_egress_update(struct adapter *adap, struct sge_eth_txq *q, + int maxreclaim); +void cxgb4_set_ethtool_ops(struct net_device *netdev); +int cxgb4_write_rss(const struct port_info *pi, const u16 *queues); +enum cpl_tx_tnl_lso_type cxgb_encap_offload_supported(struct sk_buff *skb); +extern int dbfifo_int_thresh; + +#define for_each_port(adapter, iter) \ + for (iter = 0; iter < (adapter)->params.nports; ++iter) + +static inline int is_bypass(struct adapter *adap) +{ + return adap->params.bypass; +} + +static inline int is_bypass_device(int device) +{ + /* this should be set based upon device capabilities */ + switch (device) { + case 0x440b: + case 0x440c: + return 1; + default: + return 0; + } +} + +static inline int is_10gbt_device(int device) +{ + /* this should be set based upon device capabilities */ + switch (device) { + case 0x4409: + case 0x4486: + return 1; + + default: + return 0; + } +} + +static inline unsigned int core_ticks_per_usec(const struct adapter *adap) +{ + return adap->params.vpd.cclk / 1000; +} + +static inline unsigned int us_to_core_ticks(const struct adapter *adap, + unsigned int us) +{ + return (us * adap->params.vpd.cclk) / 1000; +} + +static inline unsigned int core_ticks_to_us(const struct adapter *adapter, + unsigned int ticks) +{ + /* add Core Clock / 2 to round ticks to nearest uS */ + return ((ticks * 1000 + adapter->params.vpd.cclk/2) / + adapter->params.vpd.cclk); +} + +static inline unsigned int dack_ticks_to_usec(const struct adapter *adap, + unsigned int ticks) +{ + return (ticks << adap->params.tp.dack_re) / core_ticks_per_usec(adap); +} + +void t4_set_reg_field(struct adapter *adap, unsigned int addr, u32 mask, + u32 val); + +int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd, + int size, void *rpl, bool sleep_ok, int timeout); +int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size, + void *rpl, bool sleep_ok); + +static inline int t4_wr_mbox_timeout(struct adapter *adap, int mbox, + const void *cmd, int size, void *rpl, + int timeout) +{ + return t4_wr_mbox_meat_timeout(adap, mbox, cmd, size, rpl, true, + timeout); +} + +static inline int t4_wr_mbox(struct adapter *adap, int mbox, const void *cmd, + int size, void *rpl) +{ + return t4_wr_mbox_meat(adap, mbox, cmd, size, rpl, true); +} + +static inline int t4_wr_mbox_ns(struct adapter *adap, int mbox, const void *cmd, + int size, void *rpl) +{ + return t4_wr_mbox_meat(adap, mbox, cmd, size, rpl, false); +} + +/** + * hash_mac_addr - return the hash value of a MAC address + * @addr: the 48-bit Ethernet MAC address + * + * Hashes a MAC address according to the hash function used by HW inexact + * (hash) address matching. + */ +static inline int hash_mac_addr(const u8 *addr) +{ + u32 a = ((u32)addr[0] << 16) | ((u32)addr[1] << 8) | addr[2]; + u32 b = ((u32)addr[3] << 16) | ((u32)addr[4] << 8) | addr[5]; + + a ^= b; + a ^= (a >> 12); + a ^= (a >> 6); + return a & 0x3f; +} + +int cxgb4_set_rspq_intr_params(struct sge_rspq *q, unsigned int us, + unsigned int cnt); +static inline void init_rspq(struct adapter *adap, struct sge_rspq *q, + unsigned int us, unsigned int cnt, + unsigned int size, unsigned int iqe_size) +{ + q->adap = adap; + cxgb4_set_rspq_intr_params(q, us, cnt); + q->iqe_len = iqe_size; + q->size = size; +} + +/** + * t4_is_inserted_mod_type - is a plugged in Firmware Module Type + * @fw_mod_type: the Firmware Mofule Type + * + * Return whether the Firmware Module Type represents a real Transceiver + * Module/Cable Module Type which has been inserted. + */ +static inline bool t4_is_inserted_mod_type(unsigned int fw_mod_type) +{ + return (fw_mod_type != FW_PORT_MOD_TYPE_NONE && + fw_mod_type != FW_PORT_MOD_TYPE_NOTSUPPORTED && + fw_mod_type != FW_PORT_MOD_TYPE_UNKNOWN && + fw_mod_type != FW_PORT_MOD_TYPE_ERROR); +} + +void t4_write_indirect(struct adapter *adap, unsigned int addr_reg, + unsigned int data_reg, const u32 *vals, + unsigned int nregs, unsigned int start_idx); +void t4_read_indirect(struct adapter *adap, unsigned int addr_reg, + unsigned int data_reg, u32 *vals, unsigned int nregs, + unsigned int start_idx); +void t4_hw_pci_read_cfg4(struct adapter *adapter, int reg, u32 *val); + +struct fw_filter_wr; + +void t4_intr_enable(struct adapter *adapter); +void t4_intr_disable(struct adapter *adapter); +int t4_slow_intr_handler(struct adapter *adapter); + +int t4_wait_dev_ready(void __iomem *regs); + +fw_port_cap32_t t4_link_acaps(struct adapter *adapter, unsigned int port, + struct link_config *lc); +int t4_link_l1cfg_core(struct adapter *adap, unsigned int mbox, + unsigned int port, struct link_config *lc, + u8 sleep_ok, int timeout); + +static inline int t4_link_l1cfg(struct adapter *adapter, unsigned int mbox, + unsigned int port, struct link_config *lc) +{ + return t4_link_l1cfg_core(adapter, mbox, port, lc, + true, FW_CMD_MAX_TIMEOUT); +} + +static inline int t4_link_l1cfg_ns(struct adapter *adapter, unsigned int mbox, + unsigned int port, struct link_config *lc) +{ + return t4_link_l1cfg_core(adapter, mbox, port, lc, + false, FW_CMD_MAX_TIMEOUT); +} + +int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port); + +u32 t4_read_pcie_cfg4(struct adapter *adap, int reg); +u32 t4_get_util_window(struct adapter *adap); +void t4_setup_memwin(struct adapter *adap, u32 memwin_base, u32 window); + +int t4_memory_rw_init(struct adapter *adap, int win, int mtype, u32 *mem_off, + u32 *mem_base, u32 *mem_aperture); +void t4_memory_update_win(struct adapter *adap, int win, u32 addr); +void t4_memory_rw_residual(struct adapter *adap, u32 off, u32 addr, u8 *buf, + int dir); +#define T4_MEMORY_WRITE 0 +#define T4_MEMORY_READ 1 +int t4_memory_rw(struct adapter *adap, int win, int mtype, u32 addr, u32 len, + void *buf, int dir); +static inline int t4_memory_write(struct adapter *adap, int mtype, u32 addr, + u32 len, __be32 *buf) +{ + return t4_memory_rw(adap, 0, mtype, addr, len, buf, 0); +} + +unsigned int t4_get_regs_len(struct adapter *adapter); +void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size); + +int t4_eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz); +int t4_seeprom_wp(struct adapter *adapter, bool enable); +int t4_get_raw_vpd_params(struct adapter *adapter, struct vpd_params *p); +int t4_get_vpd_params(struct adapter *adapter, struct vpd_params *p); +int t4_get_pfres(struct adapter *adapter); +int t4_read_flash(struct adapter *adapter, unsigned int addr, + unsigned int nwords, u32 *data, int byte_oriented); +int t4_load_fw(struct adapter *adapter, const u8 *fw_data, unsigned int size); +int t4_load_phy_fw(struct adapter *adap, int win, + int (*phy_fw_version)(const u8 *, size_t), + const u8 *phy_fw_data, size_t phy_fw_size); +int t4_phy_fw_ver(struct adapter *adap, int *phy_fw_ver); +int t4_fwcache(struct adapter *adap, enum fw_params_param_dev_fwcache op); +int t4_fw_upgrade(struct adapter *adap, unsigned int mbox, + const u8 *fw_data, unsigned int size, int force); +int t4_fl_pkt_align(struct adapter *adap); +unsigned int t4_flash_cfg_addr(struct adapter *adapter); +int t4_check_fw_version(struct adapter *adap); +int t4_load_cfg(struct adapter *adapter, const u8 *cfg_data, unsigned int size); +int t4_get_fw_version(struct adapter *adapter, u32 *vers); +int t4_get_bs_version(struct adapter *adapter, u32 *vers); +int t4_get_tp_version(struct adapter *adapter, u32 *vers); +int t4_get_exprom_version(struct adapter *adapter, u32 *vers); +int t4_get_scfg_version(struct adapter *adapter, u32 *vers); +int t4_get_vpd_version(struct adapter *adapter, u32 *vers); +int t4_get_version_info(struct adapter *adapter); +void t4_dump_version_info(struct adapter *adapter); +int t4_prep_fw(struct adapter *adap, struct fw_info *fw_info, + const u8 *fw_data, unsigned int fw_size, + struct fw_hdr *card_fw, enum dev_state state, int *reset); +int t4_prep_adapter(struct adapter *adapter); +int t4_shutdown_adapter(struct adapter *adapter); + +enum t4_bar2_qtype { T4_BAR2_QTYPE_EGRESS, T4_BAR2_QTYPE_INGRESS }; +int t4_bar2_sge_qregs(struct adapter *adapter, + unsigned int qid, + enum t4_bar2_qtype qtype, + int user, + u64 *pbar2_qoffset, + unsigned int *pbar2_qid); + +unsigned int qtimer_val(const struct adapter *adap, + const struct sge_rspq *q); + +int t4_init_devlog_params(struct adapter *adapter); +int t4_init_sge_params(struct adapter *adapter); +int t4_init_tp_params(struct adapter *adap, bool sleep_ok); +int t4_filter_field_shift(const struct adapter *adap, int filter_sel); +int t4_init_rss_mode(struct adapter *adap, int mbox); +int t4_init_portinfo(struct port_info *pi, int mbox, + int port, int pf, int vf, u8 mac[]); +int t4_port_init(struct adapter *adap, int mbox, int pf, int vf); +int t4_init_port_mirror(struct port_info *pi, u8 mbox, u8 port, u8 pf, u8 vf, + u16 *mirror_viid); +void t4_fatal_err(struct adapter *adapter); +unsigned int t4_chip_rss_size(struct adapter *adapter); +int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid, + int start, int n, const u16 *rspq, unsigned int nrspq); +int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode, + unsigned int flags); +int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid, + unsigned int flags, unsigned int defq); +int t4_read_rss(struct adapter *adapter, u16 *entries); +void t4_read_rss_key(struct adapter *adapter, u32 *key, bool sleep_ok); +void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx, + bool sleep_ok); +void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index, + u32 *valp, bool sleep_ok); +void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index, + u32 *vfl, u32 *vfh, bool sleep_ok); +u32 t4_read_rss_pf_map(struct adapter *adapter, bool sleep_ok); +u32 t4_read_rss_pf_mask(struct adapter *adapter, bool sleep_ok); + +unsigned int t4_get_mps_bg_map(struct adapter *adapter, int pidx); +unsigned int t4_get_tp_ch_map(struct adapter *adapter, int pidx); +void t4_pmtx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[]); +void t4_pmrx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[]); +int t4_read_cim_ibq(struct adapter *adap, unsigned int qid, u32 *data, + size_t n); +int t4_read_cim_obq(struct adapter *adap, unsigned int qid, u32 *data, + size_t n); +int t4_cim_read(struct adapter *adap, unsigned int addr, unsigned int n, + unsigned int *valp); +int t4_cim_write(struct adapter *adap, unsigned int addr, unsigned int n, + const unsigned int *valp); +int t4_cim_read_la(struct adapter *adap, u32 *la_buf, unsigned int *wrptr); +void t4_cim_read_pif_la(struct adapter *adap, u32 *pif_req, u32 *pif_rsp, + unsigned int *pif_req_wrptr, + unsigned int *pif_rsp_wrptr); +void t4_cim_read_ma_la(struct adapter *adap, u32 *ma_req, u32 *ma_rsp); +void t4_read_cimq_cfg(struct adapter *adap, u16 *base, u16 *size, u16 *thres); +const char *t4_get_port_type_description(enum fw_port_type port_type); +void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p); +void t4_get_port_stats_offset(struct adapter *adap, int idx, + struct port_stats *stats, + struct port_stats *offset); +void t4_get_lb_stats(struct adapter *adap, int idx, struct lb_port_stats *p); +void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log); +void t4_read_cong_tbl(struct adapter *adap, u16 incr[NMTUS][NCCTRL_WIN]); +void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr, + unsigned int mask, unsigned int val); +void t4_tp_read_la(struct adapter *adap, u64 *la_buf, unsigned int *wrptr); +void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st, + bool sleep_ok); +void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st, + bool sleep_ok); +void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st, + bool sleep_ok); +void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st, + bool sleep_ok); +void t4_tp_get_tcp_stats(struct adapter *adap, struct tp_tcp_stats *v4, + struct tp_tcp_stats *v6, bool sleep_ok); +void t4_get_fcoe_stats(struct adapter *adap, unsigned int idx, + struct tp_fcoe_stats *st, bool sleep_ok); +void t4_load_mtus(struct adapter *adap, const unsigned short *mtus, + const unsigned short *alpha, const unsigned short *beta); + +void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf); + +void t4_get_chan_txrate(struct adapter *adap, u64 *nic_rate, u64 *ofld_rate); +void t4_mk_filtdelwr(unsigned int ftid, struct fw_filter_wr *wr, int qid); + +void t4_wol_magic_enable(struct adapter *adap, unsigned int port, + const u8 *addr); +int t4_wol_pat_enable(struct adapter *adap, unsigned int port, unsigned int map, + u64 mask0, u64 mask1, unsigned int crc, bool enable); + +int t4_fw_hello(struct adapter *adap, unsigned int mbox, unsigned int evt_mbox, + enum dev_master master, enum dev_state *state); +int t4_fw_bye(struct adapter *adap, unsigned int mbox); +int t4_early_init(struct adapter *adap, unsigned int mbox); +int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset); +int t4_fixup_host_params(struct adapter *adap, unsigned int page_size, + unsigned int cache_line_size); +int t4_fw_initialize(struct adapter *adap, unsigned int mbox); +int t4_query_params(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int nparams, const u32 *params, + u32 *val); +int t4_query_params_ns(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int nparams, const u32 *params, + u32 *val); +int t4_query_params_rw(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int nparams, const u32 *params, + u32 *val, int rw, bool sleep_ok); +int t4_set_params_timeout(struct adapter *adap, unsigned int mbox, + unsigned int pf, unsigned int vf, + unsigned int nparams, const u32 *params, + const u32 *val, int timeout); +int t4_set_params(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int nparams, const u32 *params, + const u32 *val); +int t4_cfg_pfvf(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int txq, unsigned int txq_eth_ctrl, + unsigned int rxqi, unsigned int rxq, unsigned int tc, + unsigned int vi, unsigned int cmask, unsigned int pmask, + unsigned int nexact, unsigned int rcaps, unsigned int wxcaps); +int t4_alloc_vi(struct adapter *adap, unsigned int mbox, unsigned int port, + unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac, + unsigned int *rss_size, u8 *vivld, u8 *vin); +int t4_free_vi(struct adapter *adap, unsigned int mbox, + unsigned int pf, unsigned int vf, + unsigned int viid); +int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid, + unsigned int viid_mirror, int mtu, int promisc, int all_multi, + int bcast, int vlanex, bool sleep_ok); +int t4_free_raw_mac_filt(struct adapter *adap, unsigned int viid, + const u8 *addr, const u8 *mask, unsigned int idx, + u8 lookup_type, u8 port_id, bool sleep_ok); +int t4_free_encap_mac_filt(struct adapter *adap, unsigned int viid, int idx, + bool sleep_ok); +int t4_alloc_encap_mac_filt(struct adapter *adap, unsigned int viid, + const u8 *addr, const u8 *mask, unsigned int vni, + unsigned int vni_mask, u8 dip_hit, u8 lookup_type, + bool sleep_ok); +int t4_alloc_raw_mac_filt(struct adapter *adap, unsigned int viid, + const u8 *addr, const u8 *mask, unsigned int idx, + u8 lookup_type, u8 port_id, bool sleep_ok); +int t4_alloc_mac_filt(struct adapter *adap, unsigned int mbox, + unsigned int viid, bool free, unsigned int naddr, + const u8 **addr, u16 *idx, u64 *hash, bool sleep_ok); +int t4_free_mac_filt(struct adapter *adap, unsigned int mbox, + unsigned int viid, unsigned int naddr, + const u8 **addr, bool sleep_ok); +int t4_change_mac(struct adapter *adap, unsigned int mbox, unsigned int viid, + int idx, const u8 *addr, bool persist, u8 *smt_idx); +int t4_set_addr_hash(struct adapter *adap, unsigned int mbox, unsigned int viid, + bool ucast, u64 vec, bool sleep_ok); +int t4_enable_vi_params(struct adapter *adap, unsigned int mbox, + unsigned int viid, bool rx_en, bool tx_en, bool dcb_en); +int t4_enable_pi_params(struct adapter *adap, unsigned int mbox, + struct port_info *pi, + bool rx_en, bool tx_en, bool dcb_en); +int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid, + bool rx_en, bool tx_en); +int t4_identify_port(struct adapter *adap, unsigned int mbox, unsigned int viid, + unsigned int nblinks); +int t4_mdio_rd(struct adapter *adap, unsigned int mbox, unsigned int phy_addr, + unsigned int mmd, unsigned int reg, u16 *valp); +int t4_mdio_wr(struct adapter *adap, unsigned int mbox, unsigned int phy_addr, + unsigned int mmd, unsigned int reg, u16 val); +int t4_iq_stop(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int iqtype, unsigned int iqid, + unsigned int fl0id, unsigned int fl1id); +int t4_iq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int iqtype, unsigned int iqid, + unsigned int fl0id, unsigned int fl1id); +int t4_eth_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int eqid); +int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int eqid); +int t4_ofld_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int eqid); +int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox, int ctxt_type); +int t4_read_sge_dbqtimers(struct adapter *adap, unsigned int ndbqtimers, + u16 *dbqtimers); +void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl); +int t4_update_port_info(struct port_info *pi); +int t4_get_link_params(struct port_info *pi, unsigned int *link_okp, + unsigned int *speedp, unsigned int *mtup); +int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl); +void t4_db_full(struct adapter *adapter); +void t4_db_dropped(struct adapter *adapter); +int t4_set_trace_filter(struct adapter *adapter, const struct trace_params *tp, + int filter_index, int enable); +void t4_get_trace_filter(struct adapter *adapter, struct trace_params *tp, + int filter_index, int *enabled); +int t4_fwaddrspace_write(struct adapter *adap, unsigned int mbox, + u32 addr, u32 val); +void t4_read_pace_tbl(struct adapter *adap, unsigned int pace_vals[NTX_SCHED]); +void t4_get_tx_sched(struct adapter *adap, unsigned int sched, + unsigned int *kbps, unsigned int *ipg, bool sleep_ok); +int t4_sge_ctxt_rd(struct adapter *adap, unsigned int mbox, unsigned int cid, + enum ctxt_type ctype, u32 *data); +int t4_sge_ctxt_rd_bd(struct adapter *adap, unsigned int cid, + enum ctxt_type ctype, u32 *data); +int t4_sched_params(struct adapter *adapter, u8 type, u8 level, u8 mode, + u8 rateunit, u8 ratemode, u8 channel, u8 class, + u32 minrate, u32 maxrate, u16 weight, u16 pktsize, + u16 burstsize); +void t4_sge_decode_idma_state(struct adapter *adapter, int state); +void t4_idma_monitor_init(struct adapter *adapter, + struct sge_idma_monitor_state *idma); +void t4_idma_monitor(struct adapter *adapter, + struct sge_idma_monitor_state *idma, + int hz, int ticks); +int t4_set_vf_mac_acl(struct adapter *adapter, unsigned int vf, + unsigned int naddr, u8 *addr); +void t4_tp_pio_read(struct adapter *adap, u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok); +void t4_tp_tm_pio_read(struct adapter *adap, u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok); +void t4_tp_mib_read(struct adapter *adap, u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok); + +void t4_uld_mem_free(struct adapter *adap); +int t4_uld_mem_alloc(struct adapter *adap); +void t4_uld_clean_up(struct adapter *adap); +void t4_register_netevent_notifier(void); +int t4_i2c_rd(struct adapter *adap, unsigned int mbox, int port, + unsigned int devid, unsigned int offset, + unsigned int len, u8 *buf); +int t4_load_boot(struct adapter *adap, u8 *boot_data, + unsigned int boot_addr, unsigned int size); +int t4_load_bootcfg(struct adapter *adap, + const u8 *cfg_data, unsigned int size); +void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq, struct sge_fl *fl); +void free_tx_desc(struct adapter *adap, struct sge_txq *q, + unsigned int n, bool unmap); +void cxgb4_eosw_txq_free_desc(struct adapter *adap, struct sge_eosw_txq *txq, + u32 ndesc); +int cxgb4_ethofld_send_flowc(struct net_device *dev, u32 eotid, u32 tc); +void cxgb4_ethofld_restart(struct tasklet_struct *t); +int cxgb4_ethofld_rx_handler(struct sge_rspq *q, const __be64 *rsp, + const struct pkt_gl *si); +void free_txq(struct adapter *adap, struct sge_txq *q); +void cxgb4_reclaim_completed_tx(struct adapter *adap, + struct sge_txq *q, bool unmap); +int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb, + dma_addr_t *addr); +void cxgb4_inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q, + void *pos); +void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q, + struct ulptx_sgl *sgl, u64 *end, unsigned int start, + const dma_addr_t *addr); +void cxgb4_write_partial_sgl(const struct sk_buff *skb, struct sge_txq *q, + struct ulptx_sgl *sgl, u64 *end, + const dma_addr_t *addr, u32 start, u32 send_len); +void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n); +int t4_set_vlan_acl(struct adapter *adap, unsigned int mbox, unsigned int vf, + u16 vlan); +int cxgb4_dcb_enabled(const struct net_device *dev); + +int cxgb4_thermal_init(struct adapter *adap); +int cxgb4_thermal_remove(struct adapter *adap); +int cxgb4_set_msix_aff(struct adapter *adap, unsigned short vec, + cpumask_var_t *aff_mask, int idx); +void cxgb4_clear_msix_aff(unsigned short vec, cpumask_var_t aff_mask); + +int cxgb4_change_mac(struct port_info *pi, unsigned int viid, + int *tcam_idx, const u8 *addr, + bool persistent, u8 *smt_idx); + +int cxgb4_alloc_mac_filt(struct adapter *adap, unsigned int viid, + bool free, unsigned int naddr, + const u8 **addr, u16 *idx, + u64 *hash, bool sleep_ok); +int cxgb4_free_mac_filt(struct adapter *adap, unsigned int viid, + unsigned int naddr, const u8 **addr, bool sleep_ok); +int cxgb4_init_mps_ref_entries(struct adapter *adap); +void cxgb4_free_mps_ref_entries(struct adapter *adap); +int cxgb4_alloc_encap_mac_filt(struct adapter *adap, unsigned int viid, + const u8 *addr, const u8 *mask, + unsigned int vni, unsigned int vni_mask, + u8 dip_hit, u8 lookup_type, bool sleep_ok); +int cxgb4_free_encap_mac_filt(struct adapter *adap, unsigned int viid, + int idx, bool sleep_ok); +int cxgb4_free_raw_mac_filt(struct adapter *adap, + unsigned int viid, + const u8 *addr, + const u8 *mask, + unsigned int idx, + u8 lookup_type, + u8 port_id, + bool sleep_ok); +int cxgb4_alloc_raw_mac_filt(struct adapter *adap, + unsigned int viid, + const u8 *addr, + const u8 *mask, + unsigned int idx, + u8 lookup_type, + u8 port_id, + bool sleep_ok); +int cxgb4_update_mac_filt(struct port_info *pi, unsigned int viid, + int *tcam_idx, const u8 *addr, + bool persistent, u8 *smt_idx); +int cxgb4_get_msix_idx_from_bmap(struct adapter *adap); +void cxgb4_free_msix_idx_in_bmap(struct adapter *adap, u32 msix_idx); +void cxgb4_enable_rx(struct adapter *adap, struct sge_rspq *q); +void cxgb4_quiesce_rx(struct sge_rspq *q); +int cxgb4_port_mirror_alloc(struct net_device *dev); +void cxgb4_port_mirror_free(struct net_device *dev); +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) +int cxgb4_set_ktls_feature(struct adapter *adap, bool enable); +#endif +#endif /* __CXGB4_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c new file mode 100644 index 000000000..dd66b2444 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017 Chelsio Communications. All rights reserved. + */ + +#include "t4_regs.h" +#include "cxgb4.h" +#include "cxgb4_cudbg.h" +#include "cudbg_zlib.h" + +static const struct cxgb4_collect_entity cxgb4_collect_mem_dump[] = { + { CUDBG_EDC0, cudbg_collect_edc0_meminfo }, + { CUDBG_EDC1, cudbg_collect_edc1_meminfo }, + { CUDBG_MC0, cudbg_collect_mc0_meminfo }, + { CUDBG_MC1, cudbg_collect_mc1_meminfo }, + { CUDBG_HMA, cudbg_collect_hma_meminfo }, +}; + +static const struct cxgb4_collect_entity cxgb4_collect_hw_dump[] = { + { CUDBG_MBOX_LOG, cudbg_collect_mbox_log }, + { CUDBG_QDESC, cudbg_collect_qdesc }, + { CUDBG_DEV_LOG, cudbg_collect_fw_devlog }, + { CUDBG_REG_DUMP, cudbg_collect_reg_dump }, + { CUDBG_CIM_LA, cudbg_collect_cim_la }, + { CUDBG_CIM_MA_LA, cudbg_collect_cim_ma_la }, + { CUDBG_CIM_QCFG, cudbg_collect_cim_qcfg }, + { CUDBG_CIM_IBQ_TP0, cudbg_collect_cim_ibq_tp0 }, + { CUDBG_CIM_IBQ_TP1, cudbg_collect_cim_ibq_tp1 }, + { CUDBG_CIM_IBQ_ULP, cudbg_collect_cim_ibq_ulp }, + { CUDBG_CIM_IBQ_SGE0, cudbg_collect_cim_ibq_sge0 }, + { CUDBG_CIM_IBQ_SGE1, cudbg_collect_cim_ibq_sge1 }, + { CUDBG_CIM_IBQ_NCSI, cudbg_collect_cim_ibq_ncsi }, + { CUDBG_CIM_OBQ_ULP0, cudbg_collect_cim_obq_ulp0 }, + { CUDBG_CIM_OBQ_ULP1, cudbg_collect_cim_obq_ulp1 }, + { CUDBG_CIM_OBQ_ULP2, cudbg_collect_cim_obq_ulp2 }, + { CUDBG_CIM_OBQ_ULP3, cudbg_collect_cim_obq_ulp3 }, + { CUDBG_CIM_OBQ_SGE, cudbg_collect_cim_obq_sge }, + { CUDBG_CIM_OBQ_NCSI, cudbg_collect_cim_obq_ncsi }, + { CUDBG_RSS, cudbg_collect_rss }, + { CUDBG_RSS_VF_CONF, cudbg_collect_rss_vf_config }, + { CUDBG_PATH_MTU, cudbg_collect_path_mtu }, + { CUDBG_PM_STATS, cudbg_collect_pm_stats }, + { CUDBG_HW_SCHED, cudbg_collect_hw_sched }, + { CUDBG_TP_INDIRECT, cudbg_collect_tp_indirect }, + { CUDBG_SGE_INDIRECT, cudbg_collect_sge_indirect }, + { CUDBG_ULPRX_LA, cudbg_collect_ulprx_la }, + { CUDBG_TP_LA, cudbg_collect_tp_la }, + { CUDBG_MEMINFO, cudbg_collect_meminfo }, + { CUDBG_CIM_PIF_LA, cudbg_collect_cim_pif_la }, + { CUDBG_CLK, cudbg_collect_clk_info }, + { CUDBG_CIM_OBQ_RXQ0, cudbg_collect_obq_sge_rx_q0 }, + { CUDBG_CIM_OBQ_RXQ1, cudbg_collect_obq_sge_rx_q1 }, + { CUDBG_PCIE_INDIRECT, cudbg_collect_pcie_indirect }, + { CUDBG_PM_INDIRECT, cudbg_collect_pm_indirect }, + { CUDBG_TID_INFO, cudbg_collect_tid }, + { CUDBG_PCIE_CONFIG, cudbg_collect_pcie_config }, + { CUDBG_DUMP_CONTEXT, cudbg_collect_dump_context }, + { CUDBG_MPS_TCAM, cudbg_collect_mps_tcam }, + { CUDBG_VPD_DATA, cudbg_collect_vpd_data }, + { CUDBG_LE_TCAM, cudbg_collect_le_tcam }, + { CUDBG_CCTRL, cudbg_collect_cctrl }, + { CUDBG_MA_INDIRECT, cudbg_collect_ma_indirect }, + { CUDBG_ULPTX_LA, cudbg_collect_ulptx_la }, + { CUDBG_UP_CIM_INDIRECT, cudbg_collect_up_cim_indirect }, + { CUDBG_PBT_TABLE, cudbg_collect_pbt_tables }, + { CUDBG_HMA_INDIRECT, cudbg_collect_hma_indirect }, +}; + +static const struct cxgb4_collect_entity cxgb4_collect_flash_dump[] = { + { CUDBG_FLASH, cudbg_collect_flash }, +}; + +u32 cxgb4_get_dump_length(struct adapter *adap, u32 flag) +{ + u32 i, entity; + u32 len = 0; + u32 wsize; + + if (flag & CXGB4_ETH_DUMP_HW) { + for (i = 0; i < ARRAY_SIZE(cxgb4_collect_hw_dump); i++) { + entity = cxgb4_collect_hw_dump[i].entity; + len += cudbg_get_entity_length(adap, entity); + } + } + + if (flag & CXGB4_ETH_DUMP_MEM) { + for (i = 0; i < ARRAY_SIZE(cxgb4_collect_mem_dump); i++) { + entity = cxgb4_collect_mem_dump[i].entity; + len += cudbg_get_entity_length(adap, entity); + } + } + + if (flag & CXGB4_ETH_DUMP_FLASH) + len += adap->params.sf_size; + + /* If compression is enabled, a smaller destination buffer is enough */ + wsize = cudbg_get_workspace_size(); + if (wsize && len > CUDBG_DUMP_BUFF_SIZE) + len = CUDBG_DUMP_BUFF_SIZE; + + return len; +} + +static void cxgb4_cudbg_collect_entity(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + const struct cxgb4_collect_entity *e_arr, + u32 arr_size, void *buf, u32 *tot_size) +{ + struct cudbg_error cudbg_err = { 0 }; + struct cudbg_entity_hdr *entity_hdr; + u32 i, total_size = 0; + int ret; + + for (i = 0; i < arr_size; i++) { + const struct cxgb4_collect_entity *e = &e_arr[i]; + + entity_hdr = cudbg_get_entity_hdr(buf, e->entity); + entity_hdr->entity_type = e->entity; + entity_hdr->start_offset = dbg_buff->offset; + memset(&cudbg_err, 0, sizeof(struct cudbg_error)); + ret = e->collect_cb(pdbg_init, dbg_buff, &cudbg_err); + if (ret) { + entity_hdr->size = 0; + dbg_buff->offset = entity_hdr->start_offset; + } else { + cudbg_align_debug_buffer(dbg_buff, entity_hdr); + } + + /* Log error and continue with next entity */ + if (cudbg_err.sys_err) + ret = CUDBG_SYSTEM_ERROR; + + entity_hdr->hdr_flags = ret; + entity_hdr->sys_err = cudbg_err.sys_err; + entity_hdr->sys_warn = cudbg_err.sys_warn; + total_size += entity_hdr->size; + } + + *tot_size += total_size; +} + +static int cudbg_alloc_compress_buff(struct cudbg_init *pdbg_init) +{ + u32 workspace_size; + + workspace_size = cudbg_get_workspace_size(); + pdbg_init->compress_buff = vzalloc(CUDBG_COMPRESS_BUFF_SIZE + + workspace_size); + if (!pdbg_init->compress_buff) + return -ENOMEM; + + pdbg_init->compress_buff_size = CUDBG_COMPRESS_BUFF_SIZE; + pdbg_init->workspace = (u8 *)pdbg_init->compress_buff + + CUDBG_COMPRESS_BUFF_SIZE - workspace_size; + return 0; +} + +static void cudbg_free_compress_buff(struct cudbg_init *pdbg_init) +{ + vfree(pdbg_init->compress_buff); +} + +int cxgb4_cudbg_collect(struct adapter *adap, void *buf, u32 *buf_size, + u32 flag) +{ + struct cudbg_buffer dbg_buff = { 0 }; + u32 size, min_size, total_size = 0; + struct cudbg_init cudbg_init; + struct cudbg_hdr *cudbg_hdr; + int rc; + + size = *buf_size; + + memset(&cudbg_init, 0, sizeof(struct cudbg_init)); + cudbg_init.adap = adap; + cudbg_init.outbuf = buf; + cudbg_init.outbuf_size = size; + + dbg_buff.data = buf; + dbg_buff.size = size; + dbg_buff.offset = 0; + + cudbg_hdr = (struct cudbg_hdr *)buf; + cudbg_hdr->signature = CUDBG_SIGNATURE; + cudbg_hdr->hdr_len = sizeof(struct cudbg_hdr); + cudbg_hdr->major_ver = CUDBG_MAJOR_VERSION; + cudbg_hdr->minor_ver = CUDBG_MINOR_VERSION; + cudbg_hdr->max_entities = CUDBG_MAX_ENTITY; + cudbg_hdr->chip_ver = adap->params.chip; + cudbg_hdr->dump_type = CUDBG_DUMP_TYPE_MINI; + + min_size = sizeof(struct cudbg_hdr) + + sizeof(struct cudbg_entity_hdr) * + cudbg_hdr->max_entities; + if (size < min_size) + return -ENOMEM; + + rc = cudbg_get_workspace_size(); + if (rc) { + /* Zlib available. So, use zlib deflate */ + cudbg_init.compress_type = CUDBG_COMPRESSION_ZLIB; + rc = cudbg_alloc_compress_buff(&cudbg_init); + if (rc) { + /* Ignore error and continue without compression. */ + dev_warn(adap->pdev_dev, + "Fail allocating compression buffer ret: %d. Continuing without compression.\n", + rc); + cudbg_init.compress_type = CUDBG_COMPRESSION_NONE; + rc = 0; + } + } else { + cudbg_init.compress_type = CUDBG_COMPRESSION_NONE; + } + + cudbg_hdr->compress_type = cudbg_init.compress_type; + dbg_buff.offset += min_size; + total_size = dbg_buff.offset; + + if (flag & CXGB4_ETH_DUMP_HW) + cxgb4_cudbg_collect_entity(&cudbg_init, &dbg_buff, + cxgb4_collect_hw_dump, + ARRAY_SIZE(cxgb4_collect_hw_dump), + buf, + &total_size); + + if (flag & CXGB4_ETH_DUMP_MEM) + cxgb4_cudbg_collect_entity(&cudbg_init, &dbg_buff, + cxgb4_collect_mem_dump, + ARRAY_SIZE(cxgb4_collect_mem_dump), + buf, + &total_size); + + if (flag & CXGB4_ETH_DUMP_FLASH) + cxgb4_cudbg_collect_entity(&cudbg_init, &dbg_buff, + cxgb4_collect_flash_dump, + ARRAY_SIZE(cxgb4_collect_flash_dump), + buf, + &total_size); + + cudbg_free_compress_buff(&cudbg_init); + cudbg_hdr->data_len = total_size; + if (cudbg_init.compress_type != CUDBG_COMPRESSION_NONE) + *buf_size = size; + else + *buf_size = total_size; + return 0; +} + +void cxgb4_init_ethtool_dump(struct adapter *adapter) +{ + adapter->eth_dump.flag = CXGB4_ETH_DUMP_NONE; + adapter->eth_dump.version = adapter->params.fw_vers; + adapter->eth_dump.len = 0; +} + +static int cxgb4_cudbg_vmcoredd_collect(struct vmcoredd_data *data, void *buf) +{ + struct adapter *adap = container_of(data, struct adapter, vmcoredd); + u32 len = data->size; + + return cxgb4_cudbg_collect(adap, buf, &len, CXGB4_ETH_DUMP_ALL); +} + +int cxgb4_cudbg_vmcore_add_dump(struct adapter *adap) +{ + struct vmcoredd_data *data = &adap->vmcoredd; + u32 len; + + len = sizeof(struct cudbg_hdr) + + sizeof(struct cudbg_entity_hdr) * CUDBG_MAX_ENTITY; + len += CUDBG_DUMP_BUFF_SIZE; + + data->size = len; + snprintf(data->dump_name, sizeof(data->dump_name), "%s_%s", + cxgb4_driver_name, adap->name); + data->vmcoredd_callback = cxgb4_cudbg_vmcoredd_collect; + + return vmcore_add_device_dump(data); +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h new file mode 100644 index 000000000..c04a49b63 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017 Chelsio Communications. All rights reserved. + */ + +#ifndef __CXGB4_CUDBG_H__ +#define __CXGB4_CUDBG_H__ + +#include "cudbg_if.h" +#include "cudbg_lib_common.h" +#include "cudbg_entity.h" +#include "cudbg_lib.h" + +#define CUDBG_DUMP_BUFF_SIZE (32 * 1024 * 1024) /* 32 MB */ +#define CUDBG_COMPRESS_BUFF_SIZE (4 * 1024 * 1024) /* 4 MB */ + +typedef int (*cudbg_collect_callback_t)(struct cudbg_init *pdbg_init, + struct cudbg_buffer *dbg_buff, + struct cudbg_error *cudbg_err); + +struct cxgb4_collect_entity { + enum cudbg_dbg_entity_type entity; + cudbg_collect_callback_t collect_cb; +}; + +enum CXGB4_ETHTOOL_DUMP_FLAGS { + CXGB4_ETH_DUMP_NONE = ETH_FW_DUMP_DISABLE, + CXGB4_ETH_DUMP_MEM = (1 << 0), /* On-Chip Memory Dumps */ + CXGB4_ETH_DUMP_HW = (1 << 1), /* various FW and HW dumps */ + CXGB4_ETH_DUMP_FLASH = (1 << 2), /* Dump flash memory */ +}; + +#define CXGB4_ETH_DUMP_ALL (CXGB4_ETH_DUMP_MEM | CXGB4_ETH_DUMP_HW) + +u32 cxgb4_get_dump_length(struct adapter *adap, u32 flag); +int cxgb4_cudbg_collect(struct adapter *adap, void *buf, u32 *buf_size, + u32 flag); +void cxgb4_init_ethtool_dump(struct adapter *adapter); +int cxgb4_cudbg_vmcore_add_dump(struct adapter *adap); +#endif /* __CXGB4_CUDBG_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_dcb.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_dcb.c new file mode 100644 index 000000000..7d5204834 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_dcb.c @@ -0,0 +1,1281 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2013-2014 Chelsio Communications. All rights reserved. + * + * Written by Anish Bhatt (anish@chelsio.com) + * Casey Leedom (leedom@chelsio.com) + */ + +#include "cxgb4.h" + +/* DCBx version control + */ +const char * const dcb_ver_array[] = { + "Unknown", + "DCBx-CIN", + "DCBx-CEE 1.01", + "DCBx-IEEE", + "", "", "", + "Auto Negotiated" +}; + +static inline bool cxgb4_dcb_state_synced(enum cxgb4_dcb_state state) +{ + if (state == CXGB4_DCB_STATE_FW_ALLSYNCED || + state == CXGB4_DCB_STATE_HOST) + return true; + else + return false; +} + +/* Initialize a port's Data Center Bridging state. + */ +void cxgb4_dcb_state_init(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct port_dcb_info *dcb = &pi->dcb; + int version_temp = dcb->dcb_version; + + memset(dcb, 0, sizeof(struct port_dcb_info)); + dcb->state = CXGB4_DCB_STATE_START; + if (version_temp) + dcb->dcb_version = version_temp; + + netdev_dbg(dev, "%s: Initializing DCB state for port[%d]\n", + __func__, pi->port_id); +} + +void cxgb4_dcb_version_init(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct port_dcb_info *dcb = &pi->dcb; + + /* Any writes here are only done on kernels that exlicitly need + * a specific version, say < 2.6.38 which only support CEE + */ + dcb->dcb_version = FW_PORT_DCB_VER_AUTO; +} + +static void cxgb4_dcb_cleanup_apps(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = pi->adapter; + struct port_dcb_info *dcb = &pi->dcb; + struct dcb_app app; + int i, err; + + /* zero priority implies remove */ + app.priority = 0; + + for (i = 0; i < CXGB4_MAX_DCBX_APP_SUPPORTED; i++) { + /* Check if app list is exhausted */ + if (!dcb->app_priority[i].protocolid) + break; + + app.protocol = dcb->app_priority[i].protocolid; + + if (dcb->dcb_version == FW_PORT_DCB_VER_IEEE) { + app.priority = dcb->app_priority[i].user_prio_map; + app.selector = dcb->app_priority[i].sel_field + 1; + err = dcb_ieee_delapp(dev, &app); + } else { + app.selector = !!(dcb->app_priority[i].sel_field); + err = dcb_setapp(dev, &app); + } + + if (err) { + dev_err(adap->pdev_dev, + "Failed DCB Clear %s Application Priority: sel=%d, prot=%d, err=%d\n", + dcb_ver_array[dcb->dcb_version], app.selector, + app.protocol, -err); + break; + } + } +} + +/* Reset a port's Data Center Bridging state. Typically used after a + * Link Down event. + */ +void cxgb4_dcb_reset(struct net_device *dev) +{ + cxgb4_dcb_cleanup_apps(dev); + cxgb4_dcb_state_init(dev); +} + +/* update the dcb port support, if version is IEEE then set it to + * FW_PORT_DCB_VER_IEEE and if DCB_CAP_DCBX_VER_CEE is already set then + * clear that. and if it is set to CEE then set dcb supported to + * DCB_CAP_DCBX_VER_CEE & if DCB_CAP_DCBX_VER_IEEE is set, clear it + */ +static inline void cxgb4_dcb_update_support(struct port_dcb_info *dcb) +{ + if (dcb->dcb_version == FW_PORT_DCB_VER_IEEE) { + if (dcb->supported & DCB_CAP_DCBX_VER_CEE) + dcb->supported &= ~DCB_CAP_DCBX_VER_CEE; + dcb->supported |= DCB_CAP_DCBX_VER_IEEE; + } else if (dcb->dcb_version == FW_PORT_DCB_VER_CEE1D01) { + if (dcb->supported & DCB_CAP_DCBX_VER_IEEE) + dcb->supported &= ~DCB_CAP_DCBX_VER_IEEE; + dcb->supported |= DCB_CAP_DCBX_VER_CEE; + } +} + +/* Finite State machine for Data Center Bridging. + */ +void cxgb4_dcb_state_fsm(struct net_device *dev, + enum cxgb4_dcb_state_input transition_to) +{ + struct port_info *pi = netdev2pinfo(dev); + struct port_dcb_info *dcb = &pi->dcb; + struct adapter *adap = pi->adapter; + enum cxgb4_dcb_state current_state = dcb->state; + + netdev_dbg(dev, "%s: State change from %d to %d for %s\n", + __func__, dcb->state, transition_to, dev->name); + + switch (current_state) { + case CXGB4_DCB_STATE_START: { + switch (transition_to) { + case CXGB4_DCB_INPUT_FW_DISABLED: { + /* we're going to use Host DCB */ + dcb->state = CXGB4_DCB_STATE_HOST; + dcb->supported = CXGB4_DCBX_HOST_SUPPORT; + break; + } + + case CXGB4_DCB_INPUT_FW_ENABLED: { + /* we're going to use Firmware DCB */ + dcb->state = CXGB4_DCB_STATE_FW_INCOMPLETE; + dcb->supported = DCB_CAP_DCBX_LLD_MANAGED; + if (dcb->dcb_version == FW_PORT_DCB_VER_IEEE) + dcb->supported |= DCB_CAP_DCBX_VER_IEEE; + else + dcb->supported |= DCB_CAP_DCBX_VER_CEE; + break; + } + + case CXGB4_DCB_INPUT_FW_INCOMPLETE: { + /* expected transition */ + break; + } + + case CXGB4_DCB_INPUT_FW_ALLSYNCED: { + dcb->state = CXGB4_DCB_STATE_FW_ALLSYNCED; + break; + } + + default: + goto bad_state_input; + } + break; + } + + case CXGB4_DCB_STATE_FW_INCOMPLETE: { + if (transition_to != CXGB4_DCB_INPUT_FW_DISABLED) { + /* during this CXGB4_DCB_STATE_FW_INCOMPLETE state, + * check if the dcb version is changed (there can be + * mismatch in default config & the negotiated switch + * configuration at FW, so update the dcb support + * accordingly. + */ + cxgb4_dcb_update_support(dcb); + } + switch (transition_to) { + case CXGB4_DCB_INPUT_FW_ENABLED: { + /* we're alreaady in firmware DCB mode */ + break; + } + + case CXGB4_DCB_INPUT_FW_INCOMPLETE: { + /* we're already incomplete */ + break; + } + + case CXGB4_DCB_INPUT_FW_ALLSYNCED: { + dcb->state = CXGB4_DCB_STATE_FW_ALLSYNCED; + dcb->enabled = 1; + linkwatch_fire_event(dev); + break; + } + + default: + goto bad_state_input; + } + break; + } + + case CXGB4_DCB_STATE_FW_ALLSYNCED: { + switch (transition_to) { + case CXGB4_DCB_INPUT_FW_ENABLED: { + /* we're alreaady in firmware DCB mode */ + break; + } + + case CXGB4_DCB_INPUT_FW_INCOMPLETE: { + /* We were successfully running with firmware DCB but + * now it's telling us that it's in an "incomplete + * state. We need to reset back to a ground state + * of incomplete. + */ + cxgb4_dcb_reset(dev); + dcb->state = CXGB4_DCB_STATE_FW_INCOMPLETE; + dcb->supported = CXGB4_DCBX_FW_SUPPORT; + linkwatch_fire_event(dev); + break; + } + + case CXGB4_DCB_INPUT_FW_ALLSYNCED: { + /* we're already all sync'ed + * this is only applicable for IEEE or + * when another VI already completed negotiaton + */ + dcb->enabled = 1; + linkwatch_fire_event(dev); + break; + } + + default: + goto bad_state_input; + } + break; + } + + case CXGB4_DCB_STATE_HOST: { + switch (transition_to) { + case CXGB4_DCB_INPUT_FW_DISABLED: { + /* we're alreaady in Host DCB mode */ + break; + } + + default: + goto bad_state_input; + } + break; + } + + default: + goto bad_state_transition; + } + return; + +bad_state_input: + dev_err(adap->pdev_dev, "cxgb4_dcb_state_fsm: illegal input symbol %d\n", + transition_to); + return; + +bad_state_transition: + dev_err(adap->pdev_dev, "cxgb4_dcb_state_fsm: bad state transition, state = %d, input = %d\n", + current_state, transition_to); +} + +/* Handle a DCB/DCBX update message from the firmware. + */ +void cxgb4_dcb_handle_fw_update(struct adapter *adap, + const struct fw_port_cmd *pcmd) +{ + const union fw_port_dcb *fwdcb = &pcmd->u.dcb; + int port = FW_PORT_CMD_PORTID_G(be32_to_cpu(pcmd->op_to_portid)); + struct net_device *dev = adap->port[adap->chan_map[port]]; + struct port_info *pi = netdev_priv(dev); + struct port_dcb_info *dcb = &pi->dcb; + int dcb_type = pcmd->u.dcb.pgid.type; + int dcb_running_version; + + /* Handle Firmware DCB Control messages separately since they drive + * our state machine. + */ + if (dcb_type == FW_PORT_DCB_TYPE_CONTROL) { + enum cxgb4_dcb_state_input input = + ((pcmd->u.dcb.control.all_syncd_pkd & + FW_PORT_CMD_ALL_SYNCD_F) + ? CXGB4_DCB_INPUT_FW_ALLSYNCED + : CXGB4_DCB_INPUT_FW_INCOMPLETE); + + if (dcb->dcb_version != FW_PORT_DCB_VER_UNKNOWN) { + dcb_running_version = FW_PORT_CMD_DCB_VERSION_G( + be16_to_cpu( + pcmd->u.dcb.control.dcb_version_to_app_state)); + if (dcb_running_version == FW_PORT_DCB_VER_CEE1D01 || + dcb_running_version == FW_PORT_DCB_VER_IEEE) { + dcb->dcb_version = dcb_running_version; + dev_warn(adap->pdev_dev, "Interface %s is running %s\n", + dev->name, + dcb_ver_array[dcb->dcb_version]); + } else { + dev_warn(adap->pdev_dev, + "Something screwed up, requested firmware for %s, but firmware returned %s instead\n", + dcb_ver_array[dcb->dcb_version], + dcb_ver_array[dcb_running_version]); + dcb->dcb_version = FW_PORT_DCB_VER_UNKNOWN; + } + } + + cxgb4_dcb_state_fsm(dev, input); + return; + } + + /* It's weird, and almost certainly an error, to get Firmware DCB + * messages when we either haven't been told whether we're going to be + * doing Host or Firmware DCB; and even worse when we've been told + * that we're doing Host DCB! + */ + if (dcb->state == CXGB4_DCB_STATE_START || + dcb->state == CXGB4_DCB_STATE_HOST) { + dev_err(adap->pdev_dev, "Receiving Firmware DCB messages in State %d\n", + dcb->state); + return; + } + + /* Now handle the general Firmware DCB update messages ... + */ + switch (dcb_type) { + case FW_PORT_DCB_TYPE_PGID: + dcb->pgid = be32_to_cpu(fwdcb->pgid.pgid); + dcb->msgs |= CXGB4_DCB_FW_PGID; + break; + + case FW_PORT_DCB_TYPE_PGRATE: + dcb->pg_num_tcs_supported = fwdcb->pgrate.num_tcs_supported; + memcpy(dcb->pgrate, &fwdcb->pgrate.pgrate, + sizeof(dcb->pgrate)); + memcpy(dcb->tsa, &fwdcb->pgrate.tsa, + sizeof(dcb->tsa)); + dcb->msgs |= CXGB4_DCB_FW_PGRATE; + if (dcb->msgs & CXGB4_DCB_FW_PGID) + IEEE_FAUX_SYNC(dev, dcb); + break; + + case FW_PORT_DCB_TYPE_PRIORATE: + memcpy(dcb->priorate, &fwdcb->priorate.strict_priorate, + sizeof(dcb->priorate)); + dcb->msgs |= CXGB4_DCB_FW_PRIORATE; + break; + + case FW_PORT_DCB_TYPE_PFC: + dcb->pfcen = fwdcb->pfc.pfcen; + dcb->pfc_num_tcs_supported = fwdcb->pfc.max_pfc_tcs; + dcb->msgs |= CXGB4_DCB_FW_PFC; + IEEE_FAUX_SYNC(dev, dcb); + break; + + case FW_PORT_DCB_TYPE_APP_ID: { + const struct fw_port_app_priority *fwap = &fwdcb->app_priority; + int idx = fwap->idx; + struct app_priority *ap = &dcb->app_priority[idx]; + + struct dcb_app app = { + .protocol = be16_to_cpu(fwap->protocolid), + }; + int err; + + /* Convert from firmware format to relevant format + * when using app selector + */ + if (dcb->dcb_version == FW_PORT_DCB_VER_IEEE) { + app.selector = (fwap->sel_field + 1); + app.priority = ffs(fwap->user_prio_map) - 1; + err = dcb_ieee_setapp(dev, &app); + IEEE_FAUX_SYNC(dev, dcb); + } else { + /* Default is CEE */ + app.selector = !!(fwap->sel_field); + app.priority = fwap->user_prio_map; + err = dcb_setapp(dev, &app); + } + + if (err) + dev_err(adap->pdev_dev, + "Failed DCB Set Application Priority: sel=%d, prot=%d, prio=%d, err=%d\n", + app.selector, app.protocol, app.priority, -err); + + ap->user_prio_map = fwap->user_prio_map; + ap->sel_field = fwap->sel_field; + ap->protocolid = be16_to_cpu(fwap->protocolid); + dcb->msgs |= CXGB4_DCB_FW_APP_ID; + break; + } + + default: + dev_err(adap->pdev_dev, "Unknown DCB update type received %x\n", + dcb_type); + break; + } +} + +/* Data Center Bridging netlink operations. + */ + + +/* Get current DCB enabled/disabled state. + */ +static u8 cxgb4_getstate(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + + return pi->dcb.enabled; +} + +/* Set DCB enabled/disabled. + */ +static u8 cxgb4_setstate(struct net_device *dev, u8 enabled) +{ + struct port_info *pi = netdev2pinfo(dev); + + /* If DCBx is host-managed, dcb is enabled by outside lldp agents */ + if (pi->dcb.state == CXGB4_DCB_STATE_HOST) { + pi->dcb.enabled = enabled; + return 0; + } + + /* Firmware doesn't provide any mechanism to control the DCB state. + */ + if (enabled != (pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED)) + return 1; + + return 0; +} + +static void cxgb4_getpgtccfg(struct net_device *dev, int tc, + u8 *prio_type, u8 *pgid, u8 *bw_per, + u8 *up_tc_map, int local) +{ + struct fw_port_cmd pcmd; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = pi->adapter; + int err; + + *prio_type = *pgid = *bw_per = *up_tc_map = 0; + + if (local) + INIT_PORT_DCB_READ_LOCAL_CMD(pcmd, pi->port_id); + else + INIT_PORT_DCB_READ_PEER_CMD(pcmd, pi->port_id); + + pcmd.u.dcb.pgid.type = FW_PORT_DCB_TYPE_PGID; + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB read PGID failed with %d\n", -err); + return; + } + *pgid = (be32_to_cpu(pcmd.u.dcb.pgid.pgid) >> (tc * 4)) & 0xf; + + if (local) + INIT_PORT_DCB_READ_LOCAL_CMD(pcmd, pi->port_id); + else + INIT_PORT_DCB_READ_PEER_CMD(pcmd, pi->port_id); + pcmd.u.dcb.pgrate.type = FW_PORT_DCB_TYPE_PGRATE; + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB read PGRATE failed with %d\n", + -err); + return; + } + + *bw_per = pcmd.u.dcb.pgrate.pgrate[*pgid]; + *up_tc_map = (1 << tc); + + /* prio_type is link strict */ + if (*pgid != 0xF) + *prio_type = 0x2; +} + +static void cxgb4_getpgtccfg_tx(struct net_device *dev, int tc, + u8 *prio_type, u8 *pgid, u8 *bw_per, + u8 *up_tc_map) +{ + /* tc 0 is written at MSB position */ + return cxgb4_getpgtccfg(dev, (7 - tc), prio_type, pgid, bw_per, + up_tc_map, 1); +} + + +static void cxgb4_getpgtccfg_rx(struct net_device *dev, int tc, + u8 *prio_type, u8 *pgid, u8 *bw_per, + u8 *up_tc_map) +{ + /* tc 0 is written at MSB position */ + return cxgb4_getpgtccfg(dev, (7 - tc), prio_type, pgid, bw_per, + up_tc_map, 0); +} + +static void cxgb4_setpgtccfg_tx(struct net_device *dev, int tc, + u8 prio_type, u8 pgid, u8 bw_per, + u8 up_tc_map) +{ + struct fw_port_cmd pcmd; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = pi->adapter; + int fw_tc = 7 - tc; + u32 _pgid; + int err; + + if (pgid == DCB_ATTR_VALUE_UNDEFINED) + return; + if (bw_per == DCB_ATTR_VALUE_UNDEFINED) + return; + + INIT_PORT_DCB_READ_LOCAL_CMD(pcmd, pi->port_id); + pcmd.u.dcb.pgid.type = FW_PORT_DCB_TYPE_PGID; + + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB read PGID failed with %d\n", -err); + return; + } + + _pgid = be32_to_cpu(pcmd.u.dcb.pgid.pgid); + _pgid &= ~(0xF << (fw_tc * 4)); + _pgid |= pgid << (fw_tc * 4); + pcmd.u.dcb.pgid.pgid = cpu_to_be32(_pgid); + + INIT_PORT_DCB_WRITE_CMD(pcmd, pi->port_id); + + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB write PGID failed with %d\n", + -err); + return; + } + + memset(&pcmd, 0, sizeof(struct fw_port_cmd)); + + INIT_PORT_DCB_READ_LOCAL_CMD(pcmd, pi->port_id); + pcmd.u.dcb.pgrate.type = FW_PORT_DCB_TYPE_PGRATE; + + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB read PGRATE failed with %d\n", + -err); + return; + } + + pcmd.u.dcb.pgrate.pgrate[pgid] = bw_per; + + INIT_PORT_DCB_WRITE_CMD(pcmd, pi->port_id); + if (pi->dcb.state == CXGB4_DCB_STATE_HOST) + pcmd.op_to_portid |= cpu_to_be32(FW_PORT_CMD_APPLY_F); + + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) + dev_err(adap->pdev_dev, "DCB write PGRATE failed with %d\n", + -err); +} + +static void cxgb4_getpgbwgcfg(struct net_device *dev, int pgid, u8 *bw_per, + int local) +{ + struct fw_port_cmd pcmd; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = pi->adapter; + int err; + + if (local) + INIT_PORT_DCB_READ_LOCAL_CMD(pcmd, pi->port_id); + else + INIT_PORT_DCB_READ_PEER_CMD(pcmd, pi->port_id); + + pcmd.u.dcb.pgrate.type = FW_PORT_DCB_TYPE_PGRATE; + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB read PGRATE failed with %d\n", + -err); + return; + } + + *bw_per = pcmd.u.dcb.pgrate.pgrate[pgid]; +} + +static void cxgb4_getpgbwgcfg_tx(struct net_device *dev, int pgid, u8 *bw_per) +{ + return cxgb4_getpgbwgcfg(dev, pgid, bw_per, 1); +} + +static void cxgb4_getpgbwgcfg_rx(struct net_device *dev, int pgid, u8 *bw_per) +{ + return cxgb4_getpgbwgcfg(dev, pgid, bw_per, 0); +} + +static void cxgb4_setpgbwgcfg_tx(struct net_device *dev, int pgid, + u8 bw_per) +{ + struct fw_port_cmd pcmd; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = pi->adapter; + int err; + + INIT_PORT_DCB_READ_LOCAL_CMD(pcmd, pi->port_id); + pcmd.u.dcb.pgrate.type = FW_PORT_DCB_TYPE_PGRATE; + + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB read PGRATE failed with %d\n", + -err); + return; + } + + pcmd.u.dcb.pgrate.pgrate[pgid] = bw_per; + + INIT_PORT_DCB_WRITE_CMD(pcmd, pi->port_id); + if (pi->dcb.state == CXGB4_DCB_STATE_HOST) + pcmd.op_to_portid |= cpu_to_be32(FW_PORT_CMD_APPLY_F); + + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + + if (err != FW_PORT_DCB_CFG_SUCCESS) + dev_err(adap->pdev_dev, "DCB write PGRATE failed with %d\n", + -err); +} + +/* Return whether the specified Traffic Class Priority has Priority Pause + * Frames enabled. + */ +static void cxgb4_getpfccfg(struct net_device *dev, int priority, u8 *pfccfg) +{ + struct port_info *pi = netdev2pinfo(dev); + struct port_dcb_info *dcb = &pi->dcb; + + if (!cxgb4_dcb_state_synced(dcb->state) || + priority >= CXGB4_MAX_PRIORITY) + *pfccfg = 0; + else + *pfccfg = (pi->dcb.pfcen >> (7 - priority)) & 1; +} + +/* Enable/disable Priority Pause Frames for the specified Traffic Class + * Priority. + */ +static void cxgb4_setpfccfg(struct net_device *dev, int priority, u8 pfccfg) +{ + struct fw_port_cmd pcmd; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = pi->adapter; + int err; + + if (!cxgb4_dcb_state_synced(pi->dcb.state) || + priority >= CXGB4_MAX_PRIORITY) + return; + + INIT_PORT_DCB_WRITE_CMD(pcmd, pi->port_id); + if (pi->dcb.state == CXGB4_DCB_STATE_HOST) + pcmd.op_to_portid |= cpu_to_be32(FW_PORT_CMD_APPLY_F); + + pcmd.u.dcb.pfc.type = FW_PORT_DCB_TYPE_PFC; + pcmd.u.dcb.pfc.pfcen = pi->dcb.pfcen; + + if (pfccfg) + pcmd.u.dcb.pfc.pfcen |= (1 << (7 - priority)); + else + pcmd.u.dcb.pfc.pfcen &= (~(1 << (7 - priority))); + + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB PFC write failed with %d\n", -err); + return; + } + + pi->dcb.pfcen = pcmd.u.dcb.pfc.pfcen; +} + +static u8 cxgb4_setall(struct net_device *dev) +{ + return 0; +} + +/* Return DCB capabilities. + */ +static u8 cxgb4_getcap(struct net_device *dev, int cap_id, u8 *caps) +{ + struct port_info *pi = netdev2pinfo(dev); + + switch (cap_id) { + case DCB_CAP_ATTR_PG: + case DCB_CAP_ATTR_PFC: + *caps = true; + break; + + case DCB_CAP_ATTR_PG_TCS: + /* 8 priorities for PG represented by bitmap */ + *caps = 0x80; + break; + + case DCB_CAP_ATTR_PFC_TCS: + /* 8 priorities for PFC represented by bitmap */ + *caps = 0x80; + break; + + case DCB_CAP_ATTR_GSP: + *caps = true; + break; + + case DCB_CAP_ATTR_UP2TC: + case DCB_CAP_ATTR_BCN: + *caps = false; + break; + + case DCB_CAP_ATTR_DCBX: + *caps = pi->dcb.supported; + break; + + default: + *caps = false; + } + + return 0; +} + +/* Return the number of Traffic Classes for the indicated Traffic Class ID. + */ +static int cxgb4_getnumtcs(struct net_device *dev, int tcs_id, u8 *num) +{ + struct port_info *pi = netdev2pinfo(dev); + + switch (tcs_id) { + case DCB_NUMTCS_ATTR_PG: + if (pi->dcb.msgs & CXGB4_DCB_FW_PGRATE) + *num = pi->dcb.pg_num_tcs_supported; + else + *num = 0x8; + break; + + case DCB_NUMTCS_ATTR_PFC: + *num = 0x8; + break; + + default: + return -EINVAL; + } + + return 0; +} + +/* Set the number of Traffic Classes supported for the indicated Traffic Class + * ID. + */ +static int cxgb4_setnumtcs(struct net_device *dev, int tcs_id, u8 num) +{ + /* Setting the number of Traffic Classes isn't supported. + */ + return -ENOSYS; +} + +/* Return whether Priority Flow Control is enabled. */ +static u8 cxgb4_getpfcstate(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + + if (!cxgb4_dcb_state_synced(pi->dcb.state)) + return false; + + return pi->dcb.pfcen != 0; +} + +/* Enable/disable Priority Flow Control. */ +static void cxgb4_setpfcstate(struct net_device *dev, u8 state) +{ + /* We can't enable/disable Priority Flow Control but we also can't + * return an error ... + */ +} + +/* Return the Application User Priority Map associated with the specified + * Application ID. + */ +static int __cxgb4_getapp(struct net_device *dev, u8 app_idtype, u16 app_id, + int peer) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = pi->adapter; + int i; + + if (!cxgb4_dcb_state_synced(pi->dcb.state)) + return 0; + + for (i = 0; i < CXGB4_MAX_DCBX_APP_SUPPORTED; i++) { + struct fw_port_cmd pcmd; + int err; + + if (peer) + INIT_PORT_DCB_READ_PEER_CMD(pcmd, pi->port_id); + else + INIT_PORT_DCB_READ_LOCAL_CMD(pcmd, pi->port_id); + + pcmd.u.dcb.app_priority.type = FW_PORT_DCB_TYPE_APP_ID; + pcmd.u.dcb.app_priority.idx = i; + + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB APP read failed with %d\n", + -err); + return err; + } + if (be16_to_cpu(pcmd.u.dcb.app_priority.protocolid) == app_id) + if (pcmd.u.dcb.app_priority.sel_field == app_idtype) + return pcmd.u.dcb.app_priority.user_prio_map; + + /* exhausted app list */ + if (!pcmd.u.dcb.app_priority.protocolid) + break; + } + + return -EEXIST; +} + +/* Return the Application User Priority Map associated with the specified + * Application ID. + */ +static int cxgb4_getapp(struct net_device *dev, u8 app_idtype, u16 app_id) +{ + /* Convert app_idtype to firmware format before querying */ + return __cxgb4_getapp(dev, app_idtype == DCB_APP_IDTYPE_ETHTYPE ? + app_idtype : 3, app_id, 0); +} + +/* Write a new Application User Priority Map for the specified Application ID + */ +static int __cxgb4_setapp(struct net_device *dev, u8 app_idtype, u16 app_id, + u8 app_prio) +{ + struct fw_port_cmd pcmd; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = pi->adapter; + int i, err; + + + if (!cxgb4_dcb_state_synced(pi->dcb.state)) + return -EINVAL; + + /* DCB info gets thrown away on link up */ + if (!netif_carrier_ok(dev)) + return -ENOLINK; + + for (i = 0; i < CXGB4_MAX_DCBX_APP_SUPPORTED; i++) { + INIT_PORT_DCB_READ_LOCAL_CMD(pcmd, pi->port_id); + pcmd.u.dcb.app_priority.type = FW_PORT_DCB_TYPE_APP_ID; + pcmd.u.dcb.app_priority.idx = i; + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB app table read failed with %d\n", + -err); + return err; + } + if (be16_to_cpu(pcmd.u.dcb.app_priority.protocolid) == app_id) { + /* overwrite existing app table */ + pcmd.u.dcb.app_priority.protocolid = 0; + break; + } + /* find first empty slot */ + if (!pcmd.u.dcb.app_priority.protocolid) + break; + } + + if (i == CXGB4_MAX_DCBX_APP_SUPPORTED) { + /* no empty slots available */ + dev_err(adap->pdev_dev, "DCB app table full\n"); + return -EBUSY; + } + + /* write out new app table entry */ + INIT_PORT_DCB_WRITE_CMD(pcmd, pi->port_id); + if (pi->dcb.state == CXGB4_DCB_STATE_HOST) + pcmd.op_to_portid |= cpu_to_be32(FW_PORT_CMD_APPLY_F); + + pcmd.u.dcb.app_priority.type = FW_PORT_DCB_TYPE_APP_ID; + pcmd.u.dcb.app_priority.protocolid = cpu_to_be16(app_id); + pcmd.u.dcb.app_priority.sel_field = app_idtype; + pcmd.u.dcb.app_priority.user_prio_map = app_prio; + pcmd.u.dcb.app_priority.idx = i; + + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB app table write failed with %d\n", + -err); + return err; + } + + return 0; +} + +/* Priority for CEE inside dcb_app is bitmask, with 0 being an invalid value */ +static int cxgb4_setapp(struct net_device *dev, u8 app_idtype, u16 app_id, + u8 app_prio) +{ + int ret; + struct dcb_app app = { + .selector = app_idtype, + .protocol = app_id, + .priority = app_prio, + }; + + if (app_idtype != DCB_APP_IDTYPE_ETHTYPE && + app_idtype != DCB_APP_IDTYPE_PORTNUM) + return -EINVAL; + + /* Convert app_idtype to a format that firmware understands */ + ret = __cxgb4_setapp(dev, app_idtype == DCB_APP_IDTYPE_ETHTYPE ? + app_idtype : 3, app_id, app_prio); + if (ret) + return ret; + + return dcb_setapp(dev, &app); +} + +/* Return whether IEEE Data Center Bridging has been negotiated. + */ +static inline int +cxgb4_ieee_negotiation_complete(struct net_device *dev, + enum cxgb4_dcb_fw_msgs dcb_subtype) +{ + struct port_info *pi = netdev2pinfo(dev); + struct port_dcb_info *dcb = &pi->dcb; + + if (dcb->state == CXGB4_DCB_STATE_FW_ALLSYNCED) + if (dcb_subtype && !(dcb->msgs & dcb_subtype)) + return 0; + + return (cxgb4_dcb_state_synced(dcb->state) && + (dcb->supported & DCB_CAP_DCBX_VER_IEEE)); +} + +static int cxgb4_ieee_read_ets(struct net_device *dev, struct ieee_ets *ets, + int local) +{ + struct port_info *pi = netdev2pinfo(dev); + struct port_dcb_info *dcb = &pi->dcb; + struct adapter *adap = pi->adapter; + uint32_t tc_info; + struct fw_port_cmd pcmd; + int i, bwg, err; + + if (!(dcb->msgs & (CXGB4_DCB_FW_PGID | CXGB4_DCB_FW_PGRATE))) + return 0; + + ets->ets_cap = dcb->pg_num_tcs_supported; + + if (local) { + ets->willing = 1; + INIT_PORT_DCB_READ_LOCAL_CMD(pcmd, pi->port_id); + } else { + INIT_PORT_DCB_READ_PEER_CMD(pcmd, pi->port_id); + } + + pcmd.u.dcb.pgid.type = FW_PORT_DCB_TYPE_PGID; + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB read PGID failed with %d\n", -err); + return err; + } + + tc_info = be32_to_cpu(pcmd.u.dcb.pgid.pgid); + + if (local) + INIT_PORT_DCB_READ_LOCAL_CMD(pcmd, pi->port_id); + else + INIT_PORT_DCB_READ_PEER_CMD(pcmd, pi->port_id); + + pcmd.u.dcb.pgrate.type = FW_PORT_DCB_TYPE_PGRATE; + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB read PGRATE failed with %d\n", + -err); + return err; + } + + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { + bwg = (tc_info >> ((7 - i) * 4)) & 0xF; + ets->prio_tc[i] = bwg; + ets->tc_tx_bw[i] = pcmd.u.dcb.pgrate.pgrate[i]; + ets->tc_rx_bw[i] = ets->tc_tx_bw[i]; + ets->tc_tsa[i] = pcmd.u.dcb.pgrate.tsa[i]; + } + + return 0; +} + +static int cxgb4_ieee_get_ets(struct net_device *dev, struct ieee_ets *ets) +{ + return cxgb4_ieee_read_ets(dev, ets, 1); +} + +/* We reuse this for peer PFC as well, as we can't have it enabled one way */ +static int cxgb4_ieee_get_pfc(struct net_device *dev, struct ieee_pfc *pfc) +{ + struct port_info *pi = netdev2pinfo(dev); + struct port_dcb_info *dcb = &pi->dcb; + + memset(pfc, 0, sizeof(struct ieee_pfc)); + + if (!(dcb->msgs & CXGB4_DCB_FW_PFC)) + return 0; + + pfc->pfc_cap = dcb->pfc_num_tcs_supported; + pfc->pfc_en = bitswap_1(dcb->pfcen); + + return 0; +} + +static int cxgb4_ieee_peer_ets(struct net_device *dev, struct ieee_ets *ets) +{ + return cxgb4_ieee_read_ets(dev, ets, 0); +} + +/* Fill in the Application User Priority Map associated with the + * specified Application. + * Priority for IEEE dcb_app is an integer, with 0 being a valid value + */ +static int cxgb4_ieee_getapp(struct net_device *dev, struct dcb_app *app) +{ + int prio; + + if (!cxgb4_ieee_negotiation_complete(dev, CXGB4_DCB_FW_APP_ID)) + return -EINVAL; + if (!(app->selector && app->protocol)) + return -EINVAL; + + /* Try querying firmware first, use firmware format */ + prio = __cxgb4_getapp(dev, app->selector - 1, app->protocol, 0); + + if (prio < 0) + prio = dcb_ieee_getapp_mask(dev, app); + + app->priority = ffs(prio) - 1; + return 0; +} + +/* Write a new Application User Priority Map for the specified Application ID. + * Priority for IEEE dcb_app is an integer, with 0 being a valid value + */ +static int cxgb4_ieee_setapp(struct net_device *dev, struct dcb_app *app) +{ + int ret; + + if (!cxgb4_ieee_negotiation_complete(dev, CXGB4_DCB_FW_APP_ID)) + return -EINVAL; + if (!(app->selector && app->protocol)) + return -EINVAL; + + if (!(app->selector > IEEE_8021QAZ_APP_SEL_ETHERTYPE && + app->selector < IEEE_8021QAZ_APP_SEL_ANY)) + return -EINVAL; + + /* change selector to a format that firmware understands */ + ret = __cxgb4_setapp(dev, app->selector - 1, app->protocol, + (1 << app->priority)); + if (ret) + return ret; + + return dcb_ieee_setapp(dev, app); +} + +/* Return our DCBX parameters. + */ +static u8 cxgb4_getdcbx(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + + /* This is already set by cxgb4_set_dcb_caps, so just return it */ + return pi->dcb.supported; +} + +/* Set our DCBX parameters. + */ +static u8 cxgb4_setdcbx(struct net_device *dev, u8 dcb_request) +{ + struct port_info *pi = netdev2pinfo(dev); + + /* Filter out requests which exceed our capabilities. + */ + if ((dcb_request & (CXGB4_DCBX_FW_SUPPORT | CXGB4_DCBX_HOST_SUPPORT)) + != dcb_request) + return 1; + + /* Can't enable DCB if we haven't successfully negotiated it. + */ + if (!cxgb4_dcb_state_synced(pi->dcb.state)) + return 1; + + /* There's currently no mechanism to allow for the firmware DCBX + * negotiation to be changed from the Host Driver. If the caller + * requests exactly the same parameters that we already have then + * we'll allow them to be successfully "set" ... + */ + if (dcb_request != pi->dcb.supported) + return 1; + + pi->dcb.supported = dcb_request; + return 0; +} + +static int cxgb4_getpeer_app(struct net_device *dev, + struct dcb_peer_app_info *info, u16 *app_count) +{ + struct fw_port_cmd pcmd; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = pi->adapter; + int i, err = 0; + + if (!cxgb4_dcb_state_synced(pi->dcb.state)) + return 1; + + info->willing = 0; + info->error = 0; + + *app_count = 0; + for (i = 0; i < CXGB4_MAX_DCBX_APP_SUPPORTED; i++) { + INIT_PORT_DCB_READ_PEER_CMD(pcmd, pi->port_id); + pcmd.u.dcb.app_priority.type = FW_PORT_DCB_TYPE_APP_ID; + pcmd.u.dcb.app_priority.idx = *app_count; + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB app table read failed with %d\n", + -err); + return err; + } + + /* find first empty slot */ + if (!pcmd.u.dcb.app_priority.protocolid) + break; + } + *app_count = i; + return err; +} + +static int cxgb4_getpeerapp_tbl(struct net_device *dev, struct dcb_app *table) +{ + struct fw_port_cmd pcmd; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = pi->adapter; + int i, err = 0; + + if (!cxgb4_dcb_state_synced(pi->dcb.state)) + return 1; + + for (i = 0; i < CXGB4_MAX_DCBX_APP_SUPPORTED; i++) { + INIT_PORT_DCB_READ_PEER_CMD(pcmd, pi->port_id); + pcmd.u.dcb.app_priority.type = FW_PORT_DCB_TYPE_APP_ID; + pcmd.u.dcb.app_priority.idx = i; + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB app table read failed with %d\n", + -err); + return err; + } + + /* find first empty slot */ + if (!pcmd.u.dcb.app_priority.protocolid) + break; + + table[i].selector = (pcmd.u.dcb.app_priority.sel_field + 1); + table[i].protocol = + be16_to_cpu(pcmd.u.dcb.app_priority.protocolid); + table[i].priority = + ffs(pcmd.u.dcb.app_priority.user_prio_map) - 1; + } + return err; +} + +/* Return Priority Group information. + */ +static int cxgb4_cee_peer_getpg(struct net_device *dev, struct cee_pg *pg) +{ + struct fw_port_cmd pcmd; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = pi->adapter; + u32 pgid; + int i, err; + + /* We're always "willing" -- the Switch Fabric always dictates the + * DCBX parameters to us. + */ + pg->willing = true; + + INIT_PORT_DCB_READ_PEER_CMD(pcmd, pi->port_id); + pcmd.u.dcb.pgid.type = FW_PORT_DCB_TYPE_PGID; + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB read PGID failed with %d\n", -err); + return err; + } + pgid = be32_to_cpu(pcmd.u.dcb.pgid.pgid); + + for (i = 0; i < CXGB4_MAX_PRIORITY; i++) + pg->prio_pg[7 - i] = (pgid >> (i * 4)) & 0xF; + + INIT_PORT_DCB_READ_PEER_CMD(pcmd, pi->port_id); + pcmd.u.dcb.pgrate.type = FW_PORT_DCB_TYPE_PGRATE; + err = t4_wr_mbox(adap, adap->mbox, &pcmd, sizeof(pcmd), &pcmd); + if (err != FW_PORT_DCB_CFG_SUCCESS) { + dev_err(adap->pdev_dev, "DCB read PGRATE failed with %d\n", + -err); + return err; + } + + for (i = 0; i < CXGB4_MAX_PRIORITY; i++) + pg->pg_bw[i] = pcmd.u.dcb.pgrate.pgrate[i]; + + pg->tcs_supported = pcmd.u.dcb.pgrate.num_tcs_supported; + + return 0; +} + +/* Return Priority Flow Control information. + */ +static int cxgb4_cee_peer_getpfc(struct net_device *dev, struct cee_pfc *pfc) +{ + struct port_info *pi = netdev2pinfo(dev); + + cxgb4_getnumtcs(dev, DCB_NUMTCS_ATTR_PFC, &(pfc->tcs_supported)); + + /* Firmware sends this to us in a formwat that is a bit flipped version + * of spec, correct it before we send it to host. This is taken care of + * by bit shifting in other uses of pfcen + */ + pfc->pfc_en = bitswap_1(pi->dcb.pfcen); + + pfc->tcs_supported = pi->dcb.pfc_num_tcs_supported; + + return 0; +} + +const struct dcbnl_rtnl_ops cxgb4_dcb_ops = { + .ieee_getets = cxgb4_ieee_get_ets, + .ieee_getpfc = cxgb4_ieee_get_pfc, + .ieee_getapp = cxgb4_ieee_getapp, + .ieee_setapp = cxgb4_ieee_setapp, + .ieee_peer_getets = cxgb4_ieee_peer_ets, + .ieee_peer_getpfc = cxgb4_ieee_get_pfc, + + /* CEE std */ + .getstate = cxgb4_getstate, + .setstate = cxgb4_setstate, + .getpgtccfgtx = cxgb4_getpgtccfg_tx, + .getpgbwgcfgtx = cxgb4_getpgbwgcfg_tx, + .getpgtccfgrx = cxgb4_getpgtccfg_rx, + .getpgbwgcfgrx = cxgb4_getpgbwgcfg_rx, + .setpgtccfgtx = cxgb4_setpgtccfg_tx, + .setpgbwgcfgtx = cxgb4_setpgbwgcfg_tx, + .setpfccfg = cxgb4_setpfccfg, + .getpfccfg = cxgb4_getpfccfg, + .setall = cxgb4_setall, + .getcap = cxgb4_getcap, + .getnumtcs = cxgb4_getnumtcs, + .setnumtcs = cxgb4_setnumtcs, + .getpfcstate = cxgb4_getpfcstate, + .setpfcstate = cxgb4_setpfcstate, + .getapp = cxgb4_getapp, + .setapp = cxgb4_setapp, + + /* DCBX configuration */ + .getdcbx = cxgb4_getdcbx, + .setdcbx = cxgb4_setdcbx, + + /* peer apps */ + .peer_getappinfo = cxgb4_getpeer_app, + .peer_getapptable = cxgb4_getpeerapp_tbl, + + /* CEE peer */ + .cee_peer_getpg = cxgb4_cee_peer_getpg, + .cee_peer_getpfc = cxgb4_cee_peer_getpfc, +}; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_dcb.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_dcb.h new file mode 100644 index 000000000..80c6627fe --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_dcb.h @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2013-2014 Chelsio Communications. All rights reserved. + * + * Written by Anish Bhatt (anish@chelsio.com) + */ + +#ifndef __CXGB4_DCB_H +#define __CXGB4_DCB_H + +#include <linux/netdevice.h> +#include <linux/dcbnl.h> +#include <net/dcbnl.h> + +#ifdef CONFIG_CHELSIO_T4_DCB + +#define CXGB4_DCBX_FW_SUPPORT \ + (DCB_CAP_DCBX_VER_CEE | \ + DCB_CAP_DCBX_VER_IEEE | \ + DCB_CAP_DCBX_LLD_MANAGED) +#define CXGB4_DCBX_HOST_SUPPORT \ + (DCB_CAP_DCBX_VER_CEE | \ + DCB_CAP_DCBX_VER_IEEE | \ + DCB_CAP_DCBX_HOST) + +#define CXGB4_MAX_PRIORITY CXGB4_MAX_DCBX_APP_SUPPORTED +#define CXGB4_MAX_TCS CXGB4_MAX_DCBX_APP_SUPPORTED + +#define INIT_PORT_DCB_CMD(__pcmd, __port, __op, __action) \ + do { \ + memset(&(__pcmd), 0, sizeof(__pcmd)); \ + (__pcmd).op_to_portid = \ + cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) | \ + FW_CMD_REQUEST_F | \ + FW_CMD_##__op##_F | \ + FW_PORT_CMD_PORTID_V(__port)); \ + (__pcmd).action_to_len16 = \ + cpu_to_be32(FW_PORT_CMD_ACTION_V(__action) | \ + FW_LEN16(pcmd)); \ + } while (0) + +#define INIT_PORT_DCB_READ_PEER_CMD(__pcmd, __port) \ + INIT_PORT_DCB_CMD(__pcmd, __port, READ, FW_PORT_ACTION_DCB_READ_RECV) + +#define INIT_PORT_DCB_READ_LOCAL_CMD(__pcmd, __port) \ + INIT_PORT_DCB_CMD(__pcmd, __port, READ, FW_PORT_ACTION_DCB_READ_TRANS) + +#define INIT_PORT_DCB_READ_SYNC_CMD(__pcmd, __port) \ + INIT_PORT_DCB_CMD(__pcmd, __port, READ, FW_PORT_ACTION_DCB_READ_DET) + +#define INIT_PORT_DCB_WRITE_CMD(__pcmd, __port) \ + INIT_PORT_DCB_CMD(__pcmd, __port, EXEC, FW_PORT_ACTION_L2_DCB_CFG) + +#define IEEE_FAUX_SYNC(__dev, __dcb) \ + do { \ + if ((__dcb)->dcb_version == FW_PORT_DCB_VER_IEEE) \ + cxgb4_dcb_state_fsm((__dev), \ + CXGB4_DCB_INPUT_FW_ALLSYNCED); \ + } while (0) + +/* States we can be in for a port's Data Center Bridging. + */ +enum cxgb4_dcb_state { + CXGB4_DCB_STATE_START, /* initial unknown state */ + CXGB4_DCB_STATE_HOST, /* we're using Host DCB (if at all) */ + CXGB4_DCB_STATE_FW_INCOMPLETE, /* using firmware DCB, incomplete */ + CXGB4_DCB_STATE_FW_ALLSYNCED, /* using firmware DCB, all sync'ed */ +}; + +/* Data Center Bridging state input for the Finite State Machine. + */ +enum cxgb4_dcb_state_input { + /* Input from the firmware. + */ + CXGB4_DCB_INPUT_FW_DISABLED, /* firmware DCB disabled */ + CXGB4_DCB_INPUT_FW_ENABLED, /* firmware DCB enabled */ + CXGB4_DCB_INPUT_FW_INCOMPLETE, /* firmware reports incomplete DCB */ + CXGB4_DCB_INPUT_FW_ALLSYNCED, /* firmware reports all sync'ed */ + +}; + +/* Firmware DCB messages that we've received so far ... + */ +enum cxgb4_dcb_fw_msgs { + CXGB4_DCB_FW_PGID = 0x01, + CXGB4_DCB_FW_PGRATE = 0x02, + CXGB4_DCB_FW_PRIORATE = 0x04, + CXGB4_DCB_FW_PFC = 0x08, + CXGB4_DCB_FW_APP_ID = 0x10, +}; + +#define CXGB4_MAX_DCBX_APP_SUPPORTED 8 + +/* Data Center Bridging support; + */ +struct port_dcb_info { + enum cxgb4_dcb_state state; /* DCB State Machine */ + enum cxgb4_dcb_fw_msgs msgs; /* DCB Firmware messages received */ + unsigned int supported; /* OS DCB capabilities supported */ + bool enabled; /* OS Enabled state */ + + /* Cached copies of DCB information sent by the firmware (in Host + * Native Endian format). + */ + u32 pgid; /* Priority Group[0..7] */ + u8 dcb_version; /* Running DCBx version */ + u8 pfcen; /* Priority Flow Control[0..7] */ + u8 pg_num_tcs_supported; /* max PG Traffic Classes */ + u8 pfc_num_tcs_supported; /* max PFC Traffic Classes */ + u8 pgrate[8]; /* Priority Group Rate[0..7] */ + u8 priorate[8]; /* Priority Rate[0..7] */ + u8 tsa[8]; /* TSA Algorithm[0..7] */ + struct app_priority { /* Application Information */ + u8 user_prio_map; /* Priority Map bitfield */ + u8 sel_field; /* Protocol ID interpretation */ + u16 protocolid; /* Protocol ID */ + } app_priority[CXGB4_MAX_DCBX_APP_SUPPORTED]; +}; + +void cxgb4_dcb_state_init(struct net_device *); +void cxgb4_dcb_version_init(struct net_device *); +void cxgb4_dcb_reset(struct net_device *dev); +void cxgb4_dcb_state_fsm(struct net_device *, enum cxgb4_dcb_state_input); +void cxgb4_dcb_handle_fw_update(struct adapter *, const struct fw_port_cmd *); +void cxgb4_dcb_set_caps(struct adapter *, const struct fw_port_cmd *); +extern const struct dcbnl_rtnl_ops cxgb4_dcb_ops; + +static inline __u8 bitswap_1(unsigned char val) +{ + return ((val & 0x80) >> 7) | + ((val & 0x40) >> 5) | + ((val & 0x20) >> 3) | + ((val & 0x10) >> 1) | + ((val & 0x08) << 1) | + ((val & 0x04) << 3) | + ((val & 0x02) << 5) | + ((val & 0x01) << 7); +} + +extern const char * const dcb_ver_array[]; + +#define CXGB4_DCB_ENABLED true + +#else /* !CONFIG_CHELSIO_T4_DCB */ + +static inline void cxgb4_dcb_state_init(struct net_device *dev) +{ +} + +#define CXGB4_DCB_ENABLED false + +#endif /* !CONFIG_CHELSIO_T4_DCB */ + +#endif /* __CXGB4_DCB_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c new file mode 100644 index 000000000..14e0d989c --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c @@ -0,0 +1,3893 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/seq_file.h> +#include <linux/debugfs.h> +#include <linux/string_helpers.h> +#include <linux/sort.h> +#include <linux/ctype.h> + +#include "cxgb4.h" +#include "t4_regs.h" +#include "t4_values.h" +#include "t4fw_api.h" +#include "cxgb4_debugfs.h" +#include "clip_tbl.h" +#include "l2t.h" +#include "cudbg_if.h" +#include "cudbg_lib_common.h" +#include "cudbg_entity.h" +#include "cudbg_lib.h" +#include "cxgb4_tc_mqprio.h" + +/* generic seq_file support for showing a table of size rows x width. */ +static void *seq_tab_get_idx(struct seq_tab *tb, loff_t pos) +{ + pos -= tb->skip_first; + return pos >= tb->rows ? NULL : &tb->data[pos * tb->width]; +} + +static void *seq_tab_start(struct seq_file *seq, loff_t *pos) +{ + struct seq_tab *tb = seq->private; + + if (tb->skip_first && *pos == 0) + return SEQ_START_TOKEN; + + return seq_tab_get_idx(tb, *pos); +} + +static void *seq_tab_next(struct seq_file *seq, void *v, loff_t *pos) +{ + v = seq_tab_get_idx(seq->private, *pos + 1); + ++(*pos); + return v; +} + +static void seq_tab_stop(struct seq_file *seq, void *v) +{ +} + +static int seq_tab_show(struct seq_file *seq, void *v) +{ + const struct seq_tab *tb = seq->private; + + return tb->show(seq, v, ((char *)v - tb->data) / tb->width); +} + +static const struct seq_operations seq_tab_ops = { + .start = seq_tab_start, + .next = seq_tab_next, + .stop = seq_tab_stop, + .show = seq_tab_show +}; + +struct seq_tab *seq_open_tab(struct file *f, unsigned int rows, + unsigned int width, unsigned int have_header, + int (*show)(struct seq_file *seq, void *v, int i)) +{ + struct seq_tab *p; + + p = __seq_open_private(f, &seq_tab_ops, sizeof(*p) + rows * width); + if (p) { + p->show = show; + p->rows = rows; + p->width = width; + p->skip_first = have_header != 0; + } + return p; +} + +/* Trim the size of a seq_tab to the supplied number of rows. The operation is + * irreversible. + */ +static int seq_tab_trim(struct seq_tab *p, unsigned int new_rows) +{ + if (new_rows > p->rows) + return -EINVAL; + p->rows = new_rows; + return 0; +} + +static int cim_la_show(struct seq_file *seq, void *v, int idx) +{ + if (v == SEQ_START_TOKEN) + seq_puts(seq, "Status Data PC LS0Stat LS0Addr " + " LS0Data\n"); + else { + const u32 *p = v; + + seq_printf(seq, + " %02x %x%07x %x%07x %08x %08x %08x%08x%08x%08x\n", + (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, + p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], + p[6], p[7]); + } + return 0; +} + +static int cim_la_show_3in1(struct seq_file *seq, void *v, int idx) +{ + if (v == SEQ_START_TOKEN) { + seq_puts(seq, "Status Data PC\n"); + } else { + const u32 *p = v; + + seq_printf(seq, " %02x %08x %08x\n", p[5] & 0xff, p[6], + p[7]); + seq_printf(seq, " %02x %02x%06x %02x%06x\n", + (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, + p[4] & 0xff, p[5] >> 8); + seq_printf(seq, " %02x %x%07x %x%07x\n", (p[0] >> 4) & 0xff, + p[0] & 0xf, p[1] >> 4, p[1] & 0xf, p[2] >> 4); + } + return 0; +} + +static int cim_la_show_t6(struct seq_file *seq, void *v, int idx) +{ + if (v == SEQ_START_TOKEN) { + seq_puts(seq, "Status Inst Data PC LS0Stat " + "LS0Addr LS0Data LS1Stat LS1Addr LS1Data\n"); + } else { + const u32 *p = v; + + seq_printf(seq, " %02x %04x%04x %04x%04x %04x%04x %08x %08x %08x %08x %08x %08x\n", + (p[9] >> 16) & 0xff, /* Status */ + p[9] & 0xffff, p[8] >> 16, /* Inst */ + p[8] & 0xffff, p[7] >> 16, /* Data */ + p[7] & 0xffff, p[6] >> 16, /* PC */ + p[2], p[1], p[0], /* LS0 Stat, Addr and Data */ + p[5], p[4], p[3]); /* LS1 Stat, Addr and Data */ + } + return 0; +} + +static int cim_la_show_pc_t6(struct seq_file *seq, void *v, int idx) +{ + if (v == SEQ_START_TOKEN) { + seq_puts(seq, "Status Inst Data PC\n"); + } else { + const u32 *p = v; + + seq_printf(seq, " %02x %08x %08x %08x\n", + p[3] & 0xff, p[2], p[1], p[0]); + seq_printf(seq, " %02x %02x%06x %02x%06x %02x%06x\n", + (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8, + p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8); + seq_printf(seq, " %02x %04x%04x %04x%04x %04x%04x\n", + (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16, + p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff, + p[6] >> 16); + } + return 0; +} + +static int cim_la_open(struct inode *inode, struct file *file) +{ + int ret; + unsigned int cfg; + struct seq_tab *p; + struct adapter *adap = inode->i_private; + + ret = t4_cim_read(adap, UP_UP_DBG_LA_CFG_A, 1, &cfg); + if (ret) + return ret; + + if (is_t6(adap->params.chip)) { + /* +1 to account for integer division of CIMLA_SIZE/10 */ + p = seq_open_tab(file, (adap->params.cim_la_size / 10) + 1, + 10 * sizeof(u32), 1, + cfg & UPDBGLACAPTPCONLY_F ? + cim_la_show_pc_t6 : cim_la_show_t6); + } else { + p = seq_open_tab(file, adap->params.cim_la_size / 8, + 8 * sizeof(u32), 1, + cfg & UPDBGLACAPTPCONLY_F ? cim_la_show_3in1 : + cim_la_show); + } + if (!p) + return -ENOMEM; + + ret = t4_cim_read_la(adap, (u32 *)p->data, NULL); + if (ret) + seq_release_private(inode, file); + return ret; +} + +static const struct file_operations cim_la_fops = { + .owner = THIS_MODULE, + .open = cim_la_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private +}; + +static int cim_pif_la_show(struct seq_file *seq, void *v, int idx) +{ + const u32 *p = v; + + if (v == SEQ_START_TOKEN) { + seq_puts(seq, "Cntl ID DataBE Addr Data\n"); + } else if (idx < CIM_PIFLA_SIZE) { + seq_printf(seq, " %02x %02x %04x %08x %08x%08x%08x%08x\n", + (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, + p[5] & 0xffff, p[4], p[3], p[2], p[1], p[0]); + } else { + if (idx == CIM_PIFLA_SIZE) + seq_puts(seq, "\nCntl ID Data\n"); + seq_printf(seq, " %02x %02x %08x%08x%08x%08x\n", + (p[4] >> 6) & 0xff, p[4] & 0x3f, + p[3], p[2], p[1], p[0]); + } + return 0; +} + +static int cim_pif_la_open(struct inode *inode, struct file *file) +{ + struct seq_tab *p; + struct adapter *adap = inode->i_private; + + p = seq_open_tab(file, 2 * CIM_PIFLA_SIZE, 6 * sizeof(u32), 1, + cim_pif_la_show); + if (!p) + return -ENOMEM; + + t4_cim_read_pif_la(adap, (u32 *)p->data, + (u32 *)p->data + 6 * CIM_PIFLA_SIZE, NULL, NULL); + return 0; +} + +static const struct file_operations cim_pif_la_fops = { + .owner = THIS_MODULE, + .open = cim_pif_la_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private +}; + +static int cim_ma_la_show(struct seq_file *seq, void *v, int idx) +{ + const u32 *p = v; + + if (v == SEQ_START_TOKEN) { + seq_puts(seq, "\n"); + } else if (idx < CIM_MALA_SIZE) { + seq_printf(seq, "%02x%08x%08x%08x%08x\n", + p[4], p[3], p[2], p[1], p[0]); + } else { + if (idx == CIM_MALA_SIZE) + seq_puts(seq, + "\nCnt ID Tag UE Data RDY VLD\n"); + seq_printf(seq, "%3u %2u %x %u %08x%08x %u %u\n", + (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, + (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, + (p[1] >> 2) | ((p[2] & 3) << 30), + (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, + p[0] & 1); + } + return 0; +} + +static int cim_ma_la_open(struct inode *inode, struct file *file) +{ + struct seq_tab *p; + struct adapter *adap = inode->i_private; + + p = seq_open_tab(file, 2 * CIM_MALA_SIZE, 5 * sizeof(u32), 1, + cim_ma_la_show); + if (!p) + return -ENOMEM; + + t4_cim_read_ma_la(adap, (u32 *)p->data, + (u32 *)p->data + 5 * CIM_MALA_SIZE); + return 0; +} + +static const struct file_operations cim_ma_la_fops = { + .owner = THIS_MODULE, + .open = cim_ma_la_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private +}; + +static int cim_qcfg_show(struct seq_file *seq, void *v) +{ + static const char * const qname[] = { + "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", + "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", + "SGE0-RX", "SGE1-RX" + }; + + int i; + struct adapter *adap = seq->private; + u16 base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; + u16 size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; + u32 stat[(4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5))]; + u16 thres[CIM_NUM_IBQ]; + u32 obq_wr_t4[2 * CIM_NUM_OBQ], *wr; + u32 obq_wr_t5[2 * CIM_NUM_OBQ_T5]; + u32 *p = stat; + int cim_num_obq = is_t4(adap->params.chip) ? + CIM_NUM_OBQ : CIM_NUM_OBQ_T5; + + i = t4_cim_read(adap, is_t4(adap->params.chip) ? UP_IBQ_0_RDADDR_A : + UP_IBQ_0_SHADOW_RDADDR_A, + ARRAY_SIZE(stat), stat); + if (!i) { + if (is_t4(adap->params.chip)) { + i = t4_cim_read(adap, UP_OBQ_0_REALADDR_A, + ARRAY_SIZE(obq_wr_t4), obq_wr_t4); + wr = obq_wr_t4; + } else { + i = t4_cim_read(adap, UP_OBQ_0_SHADOW_REALADDR_A, + ARRAY_SIZE(obq_wr_t5), obq_wr_t5); + wr = obq_wr_t5; + } + } + if (i) + return i; + + t4_read_cimq_cfg(adap, base, size, thres); + + seq_printf(seq, + " Queue Base Size Thres RdPtr WrPtr SOP EOP Avail\n"); + for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) + seq_printf(seq, "%7s %5x %5u %5u %6x %4x %4u %4u %5u\n", + qname[i], base[i], size[i], thres[i], + IBQRDADDR_G(p[0]), IBQWRADDR_G(p[1]), + QUESOPCNT_G(p[3]), QUEEOPCNT_G(p[3]), + QUEREMFLITS_G(p[2]) * 16); + for ( ; i < CIM_NUM_IBQ + cim_num_obq; i++, p += 4, wr += 2) + seq_printf(seq, "%7s %5x %5u %12x %4x %4u %4u %5u\n", + qname[i], base[i], size[i], + QUERDADDR_G(p[0]) & 0x3fff, wr[0] - base[i], + QUESOPCNT_G(p[3]), QUEEOPCNT_G(p[3]), + QUEREMFLITS_G(p[2]) * 16); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(cim_qcfg); + +static int cimq_show(struct seq_file *seq, void *v, int idx) +{ + const u32 *p = v; + + seq_printf(seq, "%#06x: %08x %08x %08x %08x\n", idx * 16, p[0], p[1], + p[2], p[3]); + return 0; +} + +static int cim_ibq_open(struct inode *inode, struct file *file) +{ + int ret; + struct seq_tab *p; + unsigned int qid = (uintptr_t)inode->i_private & 7; + struct adapter *adap = inode->i_private - qid; + + p = seq_open_tab(file, CIM_IBQ_SIZE, 4 * sizeof(u32), 0, cimq_show); + if (!p) + return -ENOMEM; + + ret = t4_read_cim_ibq(adap, qid, (u32 *)p->data, CIM_IBQ_SIZE * 4); + if (ret < 0) + seq_release_private(inode, file); + else + ret = 0; + return ret; +} + +static const struct file_operations cim_ibq_fops = { + .owner = THIS_MODULE, + .open = cim_ibq_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private +}; + +static int cim_obq_open(struct inode *inode, struct file *file) +{ + int ret; + struct seq_tab *p; + unsigned int qid = (uintptr_t)inode->i_private & 7; + struct adapter *adap = inode->i_private - qid; + + p = seq_open_tab(file, 6 * CIM_OBQ_SIZE, 4 * sizeof(u32), 0, cimq_show); + if (!p) + return -ENOMEM; + + ret = t4_read_cim_obq(adap, qid, (u32 *)p->data, 6 * CIM_OBQ_SIZE * 4); + if (ret < 0) { + seq_release_private(inode, file); + } else { + seq_tab_trim(p, ret / 4); + ret = 0; + } + return ret; +} + +static const struct file_operations cim_obq_fops = { + .owner = THIS_MODULE, + .open = cim_obq_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private +}; + +struct field_desc { + const char *name; + unsigned int start; + unsigned int width; +}; + +static void field_desc_show(struct seq_file *seq, u64 v, + const struct field_desc *p) +{ + char buf[32]; + int line_size = 0; + + while (p->name) { + u64 mask = (1ULL << p->width) - 1; + int len = scnprintf(buf, sizeof(buf), "%s: %llu", p->name, + ((unsigned long long)v >> p->start) & mask); + + if (line_size + len >= 79) { + line_size = 8; + seq_puts(seq, "\n "); + } + seq_printf(seq, "%s ", buf); + line_size += len + 1; + p++; + } + seq_putc(seq, '\n'); +} + +static struct field_desc tp_la0[] = { + { "RcfOpCodeOut", 60, 4 }, + { "State", 56, 4 }, + { "WcfState", 52, 4 }, + { "RcfOpcSrcOut", 50, 2 }, + { "CRxError", 49, 1 }, + { "ERxError", 48, 1 }, + { "SanityFailed", 47, 1 }, + { "SpuriousMsg", 46, 1 }, + { "FlushInputMsg", 45, 1 }, + { "FlushInputCpl", 44, 1 }, + { "RssUpBit", 43, 1 }, + { "RssFilterHit", 42, 1 }, + { "Tid", 32, 10 }, + { "InitTcb", 31, 1 }, + { "LineNumber", 24, 7 }, + { "Emsg", 23, 1 }, + { "EdataOut", 22, 1 }, + { "Cmsg", 21, 1 }, + { "CdataOut", 20, 1 }, + { "EreadPdu", 19, 1 }, + { "CreadPdu", 18, 1 }, + { "TunnelPkt", 17, 1 }, + { "RcfPeerFin", 16, 1 }, + { "RcfReasonOut", 12, 4 }, + { "TxCchannel", 10, 2 }, + { "RcfTxChannel", 8, 2 }, + { "RxEchannel", 6, 2 }, + { "RcfRxChannel", 5, 1 }, + { "RcfDataOutSrdy", 4, 1 }, + { "RxDvld", 3, 1 }, + { "RxOoDvld", 2, 1 }, + { "RxCongestion", 1, 1 }, + { "TxCongestion", 0, 1 }, + { NULL } +}; + +static int tp_la_show(struct seq_file *seq, void *v, int idx) +{ + const u64 *p = v; + + field_desc_show(seq, *p, tp_la0); + return 0; +} + +static int tp_la_show2(struct seq_file *seq, void *v, int idx) +{ + const u64 *p = v; + + if (idx) + seq_putc(seq, '\n'); + field_desc_show(seq, p[0], tp_la0); + if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) + field_desc_show(seq, p[1], tp_la0); + return 0; +} + +static int tp_la_show3(struct seq_file *seq, void *v, int idx) +{ + static struct field_desc tp_la1[] = { + { "CplCmdIn", 56, 8 }, + { "CplCmdOut", 48, 8 }, + { "ESynOut", 47, 1 }, + { "EAckOut", 46, 1 }, + { "EFinOut", 45, 1 }, + { "ERstOut", 44, 1 }, + { "SynIn", 43, 1 }, + { "AckIn", 42, 1 }, + { "FinIn", 41, 1 }, + { "RstIn", 40, 1 }, + { "DataIn", 39, 1 }, + { "DataInVld", 38, 1 }, + { "PadIn", 37, 1 }, + { "RxBufEmpty", 36, 1 }, + { "RxDdp", 35, 1 }, + { "RxFbCongestion", 34, 1 }, + { "TxFbCongestion", 33, 1 }, + { "TxPktSumSrdy", 32, 1 }, + { "RcfUlpType", 28, 4 }, + { "Eread", 27, 1 }, + { "Ebypass", 26, 1 }, + { "Esave", 25, 1 }, + { "Static0", 24, 1 }, + { "Cread", 23, 1 }, + { "Cbypass", 22, 1 }, + { "Csave", 21, 1 }, + { "CPktOut", 20, 1 }, + { "RxPagePoolFull", 18, 2 }, + { "RxLpbkPkt", 17, 1 }, + { "TxLpbkPkt", 16, 1 }, + { "RxVfValid", 15, 1 }, + { "SynLearned", 14, 1 }, + { "SetDelEntry", 13, 1 }, + { "SetInvEntry", 12, 1 }, + { "CpcmdDvld", 11, 1 }, + { "CpcmdSave", 10, 1 }, + { "RxPstructsFull", 8, 2 }, + { "EpcmdDvld", 7, 1 }, + { "EpcmdFlush", 6, 1 }, + { "EpcmdTrimPrefix", 5, 1 }, + { "EpcmdTrimPostfix", 4, 1 }, + { "ERssIp4Pkt", 3, 1 }, + { "ERssIp6Pkt", 2, 1 }, + { "ERssTcpUdpPkt", 1, 1 }, + { "ERssFceFipPkt", 0, 1 }, + { NULL } + }; + static struct field_desc tp_la2[] = { + { "CplCmdIn", 56, 8 }, + { "MpsVfVld", 55, 1 }, + { "MpsPf", 52, 3 }, + { "MpsVf", 44, 8 }, + { "SynIn", 43, 1 }, + { "AckIn", 42, 1 }, + { "FinIn", 41, 1 }, + { "RstIn", 40, 1 }, + { "DataIn", 39, 1 }, + { "DataInVld", 38, 1 }, + { "PadIn", 37, 1 }, + { "RxBufEmpty", 36, 1 }, + { "RxDdp", 35, 1 }, + { "RxFbCongestion", 34, 1 }, + { "TxFbCongestion", 33, 1 }, + { "TxPktSumSrdy", 32, 1 }, + { "RcfUlpType", 28, 4 }, + { "Eread", 27, 1 }, + { "Ebypass", 26, 1 }, + { "Esave", 25, 1 }, + { "Static0", 24, 1 }, + { "Cread", 23, 1 }, + { "Cbypass", 22, 1 }, + { "Csave", 21, 1 }, + { "CPktOut", 20, 1 }, + { "RxPagePoolFull", 18, 2 }, + { "RxLpbkPkt", 17, 1 }, + { "TxLpbkPkt", 16, 1 }, + { "RxVfValid", 15, 1 }, + { "SynLearned", 14, 1 }, + { "SetDelEntry", 13, 1 }, + { "SetInvEntry", 12, 1 }, + { "CpcmdDvld", 11, 1 }, + { "CpcmdSave", 10, 1 }, + { "RxPstructsFull", 8, 2 }, + { "EpcmdDvld", 7, 1 }, + { "EpcmdFlush", 6, 1 }, + { "EpcmdTrimPrefix", 5, 1 }, + { "EpcmdTrimPostfix", 4, 1 }, + { "ERssIp4Pkt", 3, 1 }, + { "ERssIp6Pkt", 2, 1 }, + { "ERssTcpUdpPkt", 1, 1 }, + { "ERssFceFipPkt", 0, 1 }, + { NULL } + }; + const u64 *p = v; + + if (idx) + seq_putc(seq, '\n'); + field_desc_show(seq, p[0], tp_la0); + if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) + field_desc_show(seq, p[1], (p[0] & BIT(17)) ? tp_la2 : tp_la1); + return 0; +} + +static int tp_la_open(struct inode *inode, struct file *file) +{ + struct seq_tab *p; + struct adapter *adap = inode->i_private; + + switch (DBGLAMODE_G(t4_read_reg(adap, TP_DBG_LA_CONFIG_A))) { + case 2: + p = seq_open_tab(file, TPLA_SIZE / 2, 2 * sizeof(u64), 0, + tp_la_show2); + break; + case 3: + p = seq_open_tab(file, TPLA_SIZE / 2, 2 * sizeof(u64), 0, + tp_la_show3); + break; + default: + p = seq_open_tab(file, TPLA_SIZE, sizeof(u64), 0, tp_la_show); + } + if (!p) + return -ENOMEM; + + t4_tp_read_la(adap, (u64 *)p->data, NULL); + return 0; +} + +static ssize_t tp_la_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + int err; + char s[32]; + unsigned long val; + size_t size = min(sizeof(s) - 1, count); + struct adapter *adap = file_inode(file)->i_private; + + if (copy_from_user(s, buf, size)) + return -EFAULT; + s[size] = '\0'; + err = kstrtoul(s, 0, &val); + if (err) + return err; + if (val > 0xffff) + return -EINVAL; + adap->params.tp.la_mask = val << 16; + t4_set_reg_field(adap, TP_DBG_LA_CONFIG_A, 0xffff0000U, + adap->params.tp.la_mask); + return count; +} + +static const struct file_operations tp_la_fops = { + .owner = THIS_MODULE, + .open = tp_la_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, + .write = tp_la_write +}; + +static int ulprx_la_show(struct seq_file *seq, void *v, int idx) +{ + const u32 *p = v; + + if (v == SEQ_START_TOKEN) + seq_puts(seq, " Pcmd Type Message" + " Data\n"); + else + seq_printf(seq, "%08x%08x %4x %08x %08x%08x%08x%08x\n", + p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); + return 0; +} + +static int ulprx_la_open(struct inode *inode, struct file *file) +{ + struct seq_tab *p; + struct adapter *adap = inode->i_private; + + p = seq_open_tab(file, ULPRX_LA_SIZE, 8 * sizeof(u32), 1, + ulprx_la_show); + if (!p) + return -ENOMEM; + + t4_ulprx_read_la(adap, (u32 *)p->data); + return 0; +} + +static const struct file_operations ulprx_la_fops = { + .owner = THIS_MODULE, + .open = ulprx_la_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private +}; + +/* Show the PM memory stats. These stats include: + * + * TX: + * Read: memory read operation + * Write Bypass: cut-through + * Bypass + mem: cut-through and save copy + * + * RX: + * Read: memory read + * Write Bypass: cut-through + * Flush: payload trim or drop + */ +static int pm_stats_show(struct seq_file *seq, void *v) +{ + static const char * const tx_pm_stats[] = { + "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" + }; + static const char * const rx_pm_stats[] = { + "Read:", "Write bypass:", "Write mem:", "Flush:" + }; + + int i; + u32 tx_cnt[T6_PM_NSTATS], rx_cnt[T6_PM_NSTATS]; + u64 tx_cyc[T6_PM_NSTATS], rx_cyc[T6_PM_NSTATS]; + struct adapter *adap = seq->private; + + t4_pmtx_get_stats(adap, tx_cnt, tx_cyc); + t4_pmrx_get_stats(adap, rx_cnt, rx_cyc); + + seq_printf(seq, "%13s %10s %20s\n", " ", "Tx pcmds", "Tx bytes"); + for (i = 0; i < PM_NSTATS - 1; i++) + seq_printf(seq, "%-13s %10u %20llu\n", + tx_pm_stats[i], tx_cnt[i], tx_cyc[i]); + + seq_printf(seq, "%13s %10s %20s\n", " ", "Rx pcmds", "Rx bytes"); + for (i = 0; i < PM_NSTATS - 1; i++) + seq_printf(seq, "%-13s %10u %20llu\n", + rx_pm_stats[i], rx_cnt[i], rx_cyc[i]); + + if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) { + /* In T5 the granularity of the total wait is too fine. + * It is not useful as it reaches the max value too fast. + * Hence display this Input FIFO wait for T6 onwards. + */ + seq_printf(seq, "%13s %10s %20s\n", + " ", "Total wait", "Total Occupancy"); + seq_printf(seq, "Tx FIFO wait %10u %20llu\n", + tx_cnt[i], tx_cyc[i]); + seq_printf(seq, "Rx FIFO wait %10u %20llu\n", + rx_cnt[i], rx_cyc[i]); + + /* Skip index 6 as there is nothing useful ihere */ + i += 2; + + /* At index 7, a new stat for read latency (count, total wait) + * is added. + */ + seq_printf(seq, "%13s %10s %20s\n", + " ", "Reads", "Total wait"); + seq_printf(seq, "Tx latency %10u %20llu\n", + tx_cnt[i], tx_cyc[i]); + seq_printf(seq, "Rx latency %10u %20llu\n", + rx_cnt[i], rx_cyc[i]); + } + return 0; +} + +static int pm_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, pm_stats_show, inode->i_private); +} + +static ssize_t pm_stats_clear(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + struct adapter *adap = file_inode(file)->i_private; + + t4_write_reg(adap, PM_RX_STAT_CONFIG_A, 0); + t4_write_reg(adap, PM_TX_STAT_CONFIG_A, 0); + return count; +} + +static const struct file_operations pm_stats_debugfs_fops = { + .owner = THIS_MODULE, + .open = pm_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = pm_stats_clear +}; + +static int tx_rate_show(struct seq_file *seq, void *v) +{ + u64 nrate[NCHAN], orate[NCHAN]; + struct adapter *adap = seq->private; + + t4_get_chan_txrate(adap, nrate, orate); + if (adap->params.arch.nchan == NCHAN) { + seq_puts(seq, " channel 0 channel 1 " + "channel 2 channel 3\n"); + seq_printf(seq, "NIC B/s: %10llu %10llu %10llu %10llu\n", + (unsigned long long)nrate[0], + (unsigned long long)nrate[1], + (unsigned long long)nrate[2], + (unsigned long long)nrate[3]); + seq_printf(seq, "Offload B/s: %10llu %10llu %10llu %10llu\n", + (unsigned long long)orate[0], + (unsigned long long)orate[1], + (unsigned long long)orate[2], + (unsigned long long)orate[3]); + } else { + seq_puts(seq, " channel 0 channel 1\n"); + seq_printf(seq, "NIC B/s: %10llu %10llu\n", + (unsigned long long)nrate[0], + (unsigned long long)nrate[1]); + seq_printf(seq, "Offload B/s: %10llu %10llu\n", + (unsigned long long)orate[0], + (unsigned long long)orate[1]); + } + return 0; +} +DEFINE_SHOW_ATTRIBUTE(tx_rate); + +static int cctrl_tbl_show(struct seq_file *seq, void *v) +{ + static const char * const dec_fac[] = { + "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", + "0.9375" }; + + int i; + u16 (*incr)[NCCTRL_WIN]; + struct adapter *adap = seq->private; + + incr = kmalloc_array(NMTUS, sizeof(*incr), GFP_KERNEL); + if (!incr) + return -ENOMEM; + + t4_read_cong_tbl(adap, incr); + + for (i = 0; i < NCCTRL_WIN; ++i) { + seq_printf(seq, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, + incr[0][i], incr[1][i], incr[2][i], incr[3][i], + incr[4][i], incr[5][i], incr[6][i], incr[7][i]); + seq_printf(seq, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", + incr[8][i], incr[9][i], incr[10][i], incr[11][i], + incr[12][i], incr[13][i], incr[14][i], incr[15][i], + adap->params.a_wnd[i], + dec_fac[adap->params.b_wnd[i]]); + } + + kfree(incr); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(cctrl_tbl); + +/* Format a value in a unit that differs from the value's native unit by the + * given factor. + */ +static char *unit_conv(char *buf, size_t len, unsigned int val, + unsigned int factor) +{ + unsigned int rem = val % factor; + + if (rem == 0) { + snprintf(buf, len, "%u", val / factor); + } else { + while (rem % 10 == 0) + rem /= 10; + snprintf(buf, len, "%u.%u", val / factor, rem); + } + return buf; +} + +static int clk_show(struct seq_file *seq, void *v) +{ + char buf[32]; + struct adapter *adap = seq->private; + unsigned int cclk_ps = 1000000000 / adap->params.vpd.cclk; /* in ps */ + u32 res = t4_read_reg(adap, TP_TIMER_RESOLUTION_A); + unsigned int tre = TIMERRESOLUTION_G(res); + unsigned int dack_re = DELAYEDACKRESOLUTION_G(res); + unsigned long long tp_tick_us = (cclk_ps << tre) / 1000000; /* in us */ + + seq_printf(seq, "Core clock period: %s ns\n", + unit_conv(buf, sizeof(buf), cclk_ps, 1000)); + seq_printf(seq, "TP timer tick: %s us\n", + unit_conv(buf, sizeof(buf), (cclk_ps << tre), 1000000)); + seq_printf(seq, "TCP timestamp tick: %s us\n", + unit_conv(buf, sizeof(buf), + (cclk_ps << TIMESTAMPRESOLUTION_G(res)), 1000000)); + seq_printf(seq, "DACK tick: %s us\n", + unit_conv(buf, sizeof(buf), (cclk_ps << dack_re), 1000000)); + seq_printf(seq, "DACK timer: %u us\n", + ((cclk_ps << dack_re) / 1000000) * + t4_read_reg(adap, TP_DACK_TIMER_A)); + seq_printf(seq, "Retransmit min: %llu us\n", + tp_tick_us * t4_read_reg(adap, TP_RXT_MIN_A)); + seq_printf(seq, "Retransmit max: %llu us\n", + tp_tick_us * t4_read_reg(adap, TP_RXT_MAX_A)); + seq_printf(seq, "Persist timer min: %llu us\n", + tp_tick_us * t4_read_reg(adap, TP_PERS_MIN_A)); + seq_printf(seq, "Persist timer max: %llu us\n", + tp_tick_us * t4_read_reg(adap, TP_PERS_MAX_A)); + seq_printf(seq, "Keepalive idle timer: %llu us\n", + tp_tick_us * t4_read_reg(adap, TP_KEEP_IDLE_A)); + seq_printf(seq, "Keepalive interval: %llu us\n", + tp_tick_us * t4_read_reg(adap, TP_KEEP_INTVL_A)); + seq_printf(seq, "Initial SRTT: %llu us\n", + tp_tick_us * INITSRTT_G(t4_read_reg(adap, TP_INIT_SRTT_A))); + seq_printf(seq, "FINWAIT2 timer: %llu us\n", + tp_tick_us * t4_read_reg(adap, TP_FINWAIT2_TIMER_A)); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(clk); + +/* Firmware Device Log dump. */ +static const char * const devlog_level_strings[] = { + [FW_DEVLOG_LEVEL_EMERG] = "EMERG", + [FW_DEVLOG_LEVEL_CRIT] = "CRIT", + [FW_DEVLOG_LEVEL_ERR] = "ERR", + [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", + [FW_DEVLOG_LEVEL_INFO] = "INFO", + [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" +}; + +static const char * const devlog_facility_strings[] = { + [FW_DEVLOG_FACILITY_CORE] = "CORE", + [FW_DEVLOG_FACILITY_CF] = "CF", + [FW_DEVLOG_FACILITY_SCHED] = "SCHED", + [FW_DEVLOG_FACILITY_TIMER] = "TIMER", + [FW_DEVLOG_FACILITY_RES] = "RES", + [FW_DEVLOG_FACILITY_HW] = "HW", + [FW_DEVLOG_FACILITY_FLR] = "FLR", + [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", + [FW_DEVLOG_FACILITY_PHY] = "PHY", + [FW_DEVLOG_FACILITY_MAC] = "MAC", + [FW_DEVLOG_FACILITY_PORT] = "PORT", + [FW_DEVLOG_FACILITY_VI] = "VI", + [FW_DEVLOG_FACILITY_FILTER] = "FILTER", + [FW_DEVLOG_FACILITY_ACL] = "ACL", + [FW_DEVLOG_FACILITY_TM] = "TM", + [FW_DEVLOG_FACILITY_QFC] = "QFC", + [FW_DEVLOG_FACILITY_DCB] = "DCB", + [FW_DEVLOG_FACILITY_ETH] = "ETH", + [FW_DEVLOG_FACILITY_OFLD] = "OFLD", + [FW_DEVLOG_FACILITY_RI] = "RI", + [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", + [FW_DEVLOG_FACILITY_FCOE] = "FCOE", + [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", + [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" +}; + +/* Information gathered by Device Log Open routine for the display routine. + */ +struct devlog_info { + unsigned int nentries; /* number of entries in log[] */ + unsigned int first; /* first [temporal] entry in log[] */ + struct fw_devlog_e log[]; /* Firmware Device Log */ +}; + +/* Dump a Firmaware Device Log entry. + */ +static int devlog_show(struct seq_file *seq, void *v) +{ + if (v == SEQ_START_TOKEN) + seq_printf(seq, "%10s %15s %8s %8s %s\n", + "Seq#", "Tstamp", "Level", "Facility", "Message"); + else { + struct devlog_info *dinfo = seq->private; + int fidx = (uintptr_t)v - 2; + unsigned long index; + struct fw_devlog_e *e; + + /* Get a pointer to the log entry to display. Skip unused log + * entries. + */ + index = dinfo->first + fidx; + if (index >= dinfo->nentries) + index -= dinfo->nentries; + e = &dinfo->log[index]; + if (e->timestamp == 0) + return 0; + + /* Print the message. This depends on the firmware using + * exactly the same formating strings as the kernel so we may + * eventually have to put a format interpreter in here ... + */ + seq_printf(seq, "%10d %15llu %8s %8s ", + be32_to_cpu(e->seqno), + be64_to_cpu(e->timestamp), + (e->level < ARRAY_SIZE(devlog_level_strings) + ? devlog_level_strings[e->level] + : "UNKNOWN"), + (e->facility < ARRAY_SIZE(devlog_facility_strings) + ? devlog_facility_strings[e->facility] + : "UNKNOWN")); + seq_printf(seq, e->fmt, + be32_to_cpu(e->params[0]), + be32_to_cpu(e->params[1]), + be32_to_cpu(e->params[2]), + be32_to_cpu(e->params[3]), + be32_to_cpu(e->params[4]), + be32_to_cpu(e->params[5]), + be32_to_cpu(e->params[6]), + be32_to_cpu(e->params[7])); + } + return 0; +} + +/* Sequential File Operations for Device Log. + */ +static inline void *devlog_get_idx(struct devlog_info *dinfo, loff_t pos) +{ + if (pos > dinfo->nentries) + return NULL; + + return (void *)(uintptr_t)(pos + 1); +} + +static void *devlog_start(struct seq_file *seq, loff_t *pos) +{ + struct devlog_info *dinfo = seq->private; + + return (*pos + ? devlog_get_idx(dinfo, *pos) + : SEQ_START_TOKEN); +} + +static void *devlog_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct devlog_info *dinfo = seq->private; + + (*pos)++; + return devlog_get_idx(dinfo, *pos); +} + +static void devlog_stop(struct seq_file *seq, void *v) +{ +} + +static const struct seq_operations devlog_seq_ops = { + .start = devlog_start, + .next = devlog_next, + .stop = devlog_stop, + .show = devlog_show +}; + +/* Set up for reading the firmware's device log. We read the entire log here + * and then display it incrementally in devlog_show(). + */ +static int devlog_open(struct inode *inode, struct file *file) +{ + struct adapter *adap = inode->i_private; + struct devlog_params *dparams = &adap->params.devlog; + struct devlog_info *dinfo; + unsigned int index; + u32 fseqno; + int ret; + + /* If we don't know where the log is we can't do anything. + */ + if (dparams->start == 0) + return -ENXIO; + + /* Allocate the space to read in the firmware's device log and set up + * for the iterated call to our display function. + */ + dinfo = __seq_open_private(file, &devlog_seq_ops, + sizeof(*dinfo) + dparams->size); + if (!dinfo) + return -ENOMEM; + + /* Record the basic log buffer information and read in the raw log. + */ + dinfo->nentries = (dparams->size / sizeof(struct fw_devlog_e)); + dinfo->first = 0; + spin_lock(&adap->win0_lock); + ret = t4_memory_rw(adap, adap->params.drv_memwin, dparams->memtype, + dparams->start, dparams->size, (__be32 *)dinfo->log, + T4_MEMORY_READ); + spin_unlock(&adap->win0_lock); + if (ret) { + seq_release_private(inode, file); + return ret; + } + + /* Find the earliest (lowest Sequence Number) log entry in the + * circular Device Log. + */ + for (fseqno = ~((u32)0), index = 0; index < dinfo->nentries; index++) { + struct fw_devlog_e *e = &dinfo->log[index]; + __u32 seqno; + + if (e->timestamp == 0) + continue; + + seqno = be32_to_cpu(e->seqno); + if (seqno < fseqno) { + fseqno = seqno; + dinfo->first = index; + } + } + return 0; +} + +static const struct file_operations devlog_fops = { + .owner = THIS_MODULE, + .open = devlog_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private +}; + +/* Show Firmware Mailbox Command/Reply Log + * + * Note that we don't do any locking when dumping the Firmware Mailbox Log so + * it's possible that we can catch things during a log update and therefore + * see partially corrupted log entries. But it's probably Good Enough(tm). + * If we ever decide that we want to make sure that we're dumping a coherent + * log, we'd need to perform locking in the mailbox logging and in + * mboxlog_open() where we'd need to grab the entire mailbox log in one go + * like we do for the Firmware Device Log. + */ +static int mboxlog_show(struct seq_file *seq, void *v) +{ + struct adapter *adapter = seq->private; + struct mbox_cmd_log *log = adapter->mbox_log; + struct mbox_cmd *entry; + int entry_idx, i; + + if (v == SEQ_START_TOKEN) { + seq_printf(seq, + "%10s %15s %5s %5s %s\n", + "Seq#", "Tstamp", "Atime", "Etime", + "Command/Reply"); + return 0; + } + + entry_idx = log->cursor + ((uintptr_t)v - 2); + if (entry_idx >= log->size) + entry_idx -= log->size; + entry = mbox_cmd_log_entry(log, entry_idx); + + /* skip over unused entries */ + if (entry->timestamp == 0) + return 0; + + seq_printf(seq, "%10u %15llu %5d %5d", + entry->seqno, entry->timestamp, + entry->access, entry->execute); + for (i = 0; i < MBOX_LEN / 8; i++) { + u64 flit = entry->cmd[i]; + u32 hi = (u32)(flit >> 32); + u32 lo = (u32)flit; + + seq_printf(seq, " %08x %08x", hi, lo); + } + seq_puts(seq, "\n"); + return 0; +} + +static inline void *mboxlog_get_idx(struct seq_file *seq, loff_t pos) +{ + struct adapter *adapter = seq->private; + struct mbox_cmd_log *log = adapter->mbox_log; + + return ((pos <= log->size) ? (void *)(uintptr_t)(pos + 1) : NULL); +} + +static void *mboxlog_start(struct seq_file *seq, loff_t *pos) +{ + return *pos ? mboxlog_get_idx(seq, *pos) : SEQ_START_TOKEN; +} + +static void *mboxlog_next(struct seq_file *seq, void *v, loff_t *pos) +{ + ++*pos; + return mboxlog_get_idx(seq, *pos); +} + +static void mboxlog_stop(struct seq_file *seq, void *v) +{ +} + +static const struct seq_operations mboxlog_seq_ops = { + .start = mboxlog_start, + .next = mboxlog_next, + .stop = mboxlog_stop, + .show = mboxlog_show +}; + +static int mboxlog_open(struct inode *inode, struct file *file) +{ + int res = seq_open(file, &mboxlog_seq_ops); + + if (!res) { + struct seq_file *seq = file->private_data; + + seq->private = inode->i_private; + } + return res; +} + +static const struct file_operations mboxlog_fops = { + .owner = THIS_MODULE, + .open = mboxlog_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int mbox_show(struct seq_file *seq, void *v) +{ + static const char * const owner[] = { "none", "FW", "driver", + "unknown", "<unread>" }; + + int i; + unsigned int mbox = (uintptr_t)seq->private & 7; + struct adapter *adap = seq->private - mbox; + void __iomem *addr = adap->regs + PF_REG(mbox, CIM_PF_MAILBOX_DATA_A); + + /* For T4 we don't have a shadow copy of the Mailbox Control register. + * And since reading that real register causes a side effect of + * granting ownership, we're best of simply not reading it at all. + */ + if (is_t4(adap->params.chip)) { + i = 4; /* index of "<unread>" */ + } else { + unsigned int ctrl_reg = CIM_PF_MAILBOX_CTRL_SHADOW_COPY_A; + void __iomem *ctrl = adap->regs + PF_REG(mbox, ctrl_reg); + + i = MBOWNER_G(readl(ctrl)); + } + + seq_printf(seq, "mailbox owned by %s\n\n", owner[i]); + + for (i = 0; i < MBOX_LEN; i += 8) + seq_printf(seq, "%016llx\n", + (unsigned long long)readq(addr + i)); + return 0; +} + +static int mbox_open(struct inode *inode, struct file *file) +{ + return single_open(file, mbox_show, inode->i_private); +} + +static ssize_t mbox_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + int i; + char c = '\n', s[256]; + unsigned long long data[8]; + const struct inode *ino; + unsigned int mbox; + struct adapter *adap; + void __iomem *addr; + void __iomem *ctrl; + + if (count > sizeof(s) - 1 || !count) + return -EINVAL; + if (copy_from_user(s, buf, count)) + return -EFAULT; + s[count] = '\0'; + + if (sscanf(s, "%llx %llx %llx %llx %llx %llx %llx %llx%c", &data[0], + &data[1], &data[2], &data[3], &data[4], &data[5], &data[6], + &data[7], &c) < 8 || c != '\n') + return -EINVAL; + + ino = file_inode(file); + mbox = (uintptr_t)ino->i_private & 7; + adap = ino->i_private - mbox; + addr = adap->regs + PF_REG(mbox, CIM_PF_MAILBOX_DATA_A); + ctrl = addr + MBOX_LEN; + + if (MBOWNER_G(readl(ctrl)) != X_MBOWNER_PL) + return -EBUSY; + + for (i = 0; i < 8; i++) + writeq(data[i], addr + 8 * i); + + writel(MBMSGVALID_F | MBOWNER_V(X_MBOWNER_FW), ctrl); + return count; +} + +static const struct file_operations mbox_debugfs_fops = { + .owner = THIS_MODULE, + .open = mbox_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = mbox_write +}; + +static int mps_trc_show(struct seq_file *seq, void *v) +{ + int enabled, i; + struct trace_params tp; + unsigned int trcidx = (uintptr_t)seq->private & 3; + struct adapter *adap = seq->private - trcidx; + + t4_get_trace_filter(adap, &tp, trcidx, &enabled); + if (!enabled) { + seq_puts(seq, "tracer is disabled\n"); + return 0; + } + + if (tp.skip_ofst * 8 >= TRACE_LEN) { + dev_err(adap->pdev_dev, "illegal trace pattern skip offset\n"); + return -EINVAL; + } + if (tp.port < 8) { + i = adap->chan_map[tp.port & 3]; + if (i >= MAX_NPORTS) { + dev_err(adap->pdev_dev, "tracer %u is assigned " + "to non-existing port\n", trcidx); + return -EINVAL; + } + seq_printf(seq, "tracer is capturing %s %s, ", + adap->port[i]->name, tp.port < 4 ? "Rx" : "Tx"); + } else + seq_printf(seq, "tracer is capturing loopback %d, ", + tp.port - 8); + seq_printf(seq, "snap length: %u, min length: %u\n", tp.snap_len, + tp.min_len); + seq_printf(seq, "packets captured %smatch filter\n", + tp.invert ? "do not " : ""); + + if (tp.skip_ofst) { + seq_puts(seq, "filter pattern: "); + for (i = 0; i < tp.skip_ofst * 2; i += 2) + seq_printf(seq, "%08x%08x", tp.data[i], tp.data[i + 1]); + seq_putc(seq, '/'); + for (i = 0; i < tp.skip_ofst * 2; i += 2) + seq_printf(seq, "%08x%08x", tp.mask[i], tp.mask[i + 1]); + seq_puts(seq, "@0\n"); + } + + seq_puts(seq, "filter pattern: "); + for (i = tp.skip_ofst * 2; i < TRACE_LEN / 4; i += 2) + seq_printf(seq, "%08x%08x", tp.data[i], tp.data[i + 1]); + seq_putc(seq, '/'); + for (i = tp.skip_ofst * 2; i < TRACE_LEN / 4; i += 2) + seq_printf(seq, "%08x%08x", tp.mask[i], tp.mask[i + 1]); + seq_printf(seq, "@%u\n", (tp.skip_ofst + tp.skip_len) * 8); + return 0; +} + +static int mps_trc_open(struct inode *inode, struct file *file) +{ + return single_open(file, mps_trc_show, inode->i_private); +} + +static unsigned int xdigit2int(unsigned char c) +{ + return isdigit(c) ? c - '0' : tolower(c) - 'a' + 10; +} + +#define TRC_PORT_NONE 0xff +#define TRC_RSS_ENABLE 0x33 +#define TRC_RSS_DISABLE 0x13 + +/* Set an MPS trace filter. Syntax is: + * + * disable + * + * to disable tracing, or + * + * interface qid=<qid no> [snaplen=<val>] [minlen=<val>] [not] [<pattern>]... + * + * where interface is one of rxN, txN, or loopbackN, N = 0..3, qid can be one + * of the NIC's response qid obtained from sge_qinfo and pattern has the form + * + * <pattern data>[/<pattern mask>][@<anchor>] + * + * Up to 2 filter patterns can be specified. If 2 are supplied the first one + * must be anchored at 0. An omitted mask is taken as a mask of 1s, an omitted + * anchor is taken as 0. + */ +static ssize_t mps_trc_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + int i, enable, ret; + u32 *data, *mask; + struct trace_params tp; + const struct inode *ino; + unsigned int trcidx; + char *s, *p, *word, *end; + struct adapter *adap; + u32 j; + + ino = file_inode(file); + trcidx = (uintptr_t)ino->i_private & 3; + adap = ino->i_private - trcidx; + + /* Don't accept input more than 1K, can't be anything valid except lots + * of whitespace. Well, use less. + */ + if (count > 1024) + return -EFBIG; + p = s = kzalloc(count + 1, GFP_USER); + if (!s) + return -ENOMEM; + if (copy_from_user(s, buf, count)) { + count = -EFAULT; + goto out; + } + + if (s[count - 1] == '\n') + s[count - 1] = '\0'; + + enable = strcmp("disable", s) != 0; + if (!enable) + goto apply; + + /* enable or disable trace multi rss filter */ + if (adap->trace_rss) + t4_write_reg(adap, MPS_TRC_CFG_A, TRC_RSS_ENABLE); + else + t4_write_reg(adap, MPS_TRC_CFG_A, TRC_RSS_DISABLE); + + memset(&tp, 0, sizeof(tp)); + tp.port = TRC_PORT_NONE; + i = 0; /* counts pattern nibbles */ + + while (p) { + while (isspace(*p)) + p++; + word = strsep(&p, " "); + if (!*word) + break; + + if (!strncmp(word, "qid=", 4)) { + end = (char *)word + 4; + ret = kstrtouint(end, 10, &j); + if (ret) + goto out; + if (!adap->trace_rss) { + t4_write_reg(adap, MPS_T5_TRC_RSS_CONTROL_A, j); + continue; + } + + switch (trcidx) { + case 0: + t4_write_reg(adap, MPS_TRC_RSS_CONTROL_A, j); + break; + case 1: + t4_write_reg(adap, + MPS_TRC_FILTER1_RSS_CONTROL_A, j); + break; + case 2: + t4_write_reg(adap, + MPS_TRC_FILTER2_RSS_CONTROL_A, j); + break; + case 3: + t4_write_reg(adap, + MPS_TRC_FILTER3_RSS_CONTROL_A, j); + break; + } + continue; + } + if (!strncmp(word, "snaplen=", 8)) { + end = (char *)word + 8; + ret = kstrtouint(end, 10, &j); + if (ret || j > 9600) { +inval: count = -EINVAL; + goto out; + } + tp.snap_len = j; + continue; + } + if (!strncmp(word, "minlen=", 7)) { + end = (char *)word + 7; + ret = kstrtouint(end, 10, &j); + if (ret || j > TFMINPKTSIZE_M) + goto inval; + tp.min_len = j; + continue; + } + if (!strcmp(word, "not")) { + tp.invert = !tp.invert; + continue; + } + if (!strncmp(word, "loopback", 8) && tp.port == TRC_PORT_NONE) { + if (word[8] < '0' || word[8] > '3' || word[9]) + goto inval; + tp.port = word[8] - '0' + 8; + continue; + } + if (!strncmp(word, "tx", 2) && tp.port == TRC_PORT_NONE) { + if (word[2] < '0' || word[2] > '3' || word[3]) + goto inval; + tp.port = word[2] - '0' + 4; + if (adap->chan_map[tp.port & 3] >= MAX_NPORTS) + goto inval; + continue; + } + if (!strncmp(word, "rx", 2) && tp.port == TRC_PORT_NONE) { + if (word[2] < '0' || word[2] > '3' || word[3]) + goto inval; + tp.port = word[2] - '0'; + if (adap->chan_map[tp.port] >= MAX_NPORTS) + goto inval; + continue; + } + if (!isxdigit(*word)) + goto inval; + + /* we have found a trace pattern */ + if (i) { /* split pattern */ + if (tp.skip_len) /* too many splits */ + goto inval; + tp.skip_ofst = i / 16; + } + + data = &tp.data[i / 8]; + mask = &tp.mask[i / 8]; + j = i; + + while (isxdigit(*word)) { + if (i >= TRACE_LEN * 2) { + count = -EFBIG; + goto out; + } + *data = (*data << 4) + xdigit2int(*word++); + if (++i % 8 == 0) + data++; + } + if (*word == '/') { + word++; + while (isxdigit(*word)) { + if (j >= i) /* mask longer than data */ + goto inval; + *mask = (*mask << 4) + xdigit2int(*word++); + if (++j % 8 == 0) + mask++; + } + if (i != j) /* mask shorter than data */ + goto inval; + } else { /* no mask, use all 1s */ + for ( ; i - j >= 8; j += 8) + *mask++ = 0xffffffff; + if (i % 8) + *mask = (1 << (i % 8) * 4) - 1; + } + if (*word == '@') { + end = (char *)word + 1; + ret = kstrtouint(end, 10, &j); + if (*end && *end != '\n') + goto inval; + if (j & 7) /* doesn't start at multiple of 8 */ + goto inval; + j /= 8; + if (j < tp.skip_ofst) /* overlaps earlier pattern */ + goto inval; + if (j - tp.skip_ofst > 31) /* skip too big */ + goto inval; + tp.skip_len = j - tp.skip_ofst; + } + if (i % 8) { + *data <<= (8 - i % 8) * 4; + *mask <<= (8 - i % 8) * 4; + i = (i + 15) & ~15; /* 8-byte align */ + } + } + + if (tp.port == TRC_PORT_NONE) + goto inval; + +apply: + i = t4_set_trace_filter(adap, &tp, trcidx, enable); + if (i) + count = i; +out: + kfree(s); + return count; +} + +static const struct file_operations mps_trc_debugfs_fops = { + .owner = THIS_MODULE, + .open = mps_trc_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = mps_trc_write +}; + +static ssize_t flash_read(struct file *file, char __user *buf, size_t count, + loff_t *ppos) +{ + loff_t pos = *ppos; + loff_t avail = file_inode(file)->i_size; + struct adapter *adap = file->private_data; + + if (pos < 0) + return -EINVAL; + if (pos >= avail) + return 0; + if (count > avail - pos) + count = avail - pos; + + while (count) { + size_t len; + int ret, ofst; + u8 data[256]; + + ofst = pos & 3; + len = min(count + ofst, sizeof(data)); + ret = t4_read_flash(adap, pos - ofst, (len + 3) / 4, + (u32 *)data, 1); + if (ret) + return ret; + + len -= ofst; + if (copy_to_user(buf, data + ofst, len)) + return -EFAULT; + + buf += len; + pos += len; + count -= len; + } + count = pos - *ppos; + *ppos = pos; + return count; +} + +static const struct file_operations flash_debugfs_fops = { + .owner = THIS_MODULE, + .open = mem_open, + .read = flash_read, + .llseek = default_llseek, +}; + +static inline void tcamxy2valmask(u64 x, u64 y, u8 *addr, u64 *mask) +{ + *mask = x | y; + y = (__force u64)cpu_to_be64(y); + memcpy(addr, (char *)&y + 2, ETH_ALEN); +} + +static int mps_tcam_show(struct seq_file *seq, void *v) +{ + struct adapter *adap = seq->private; + unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip); + if (v == SEQ_START_TOKEN) { + if (chip_ver > CHELSIO_T5) { + seq_puts(seq, "Idx Ethernet address Mask " + " VNI Mask IVLAN Vld " + "DIP_Hit Lookup Port " + "Vld Ports PF VF " + "Replication " + " P0 P1 P2 P3 ML\n"); + } else { + if (adap->params.arch.mps_rplc_size > 128) + seq_puts(seq, "Idx Ethernet address Mask " + "Vld Ports PF VF " + "Replication " + " P0 P1 P2 P3 ML\n"); + else + seq_puts(seq, "Idx Ethernet address Mask " + "Vld Ports PF VF Replication" + " P0 P1 P2 P3 ML\n"); + } + } else { + u64 mask; + u8 addr[ETH_ALEN]; + bool replicate, dip_hit = false, vlan_vld = false; + unsigned int idx = (uintptr_t)v - 2; + u64 tcamy, tcamx, val; + u32 cls_lo, cls_hi, ctl, data2, vnix = 0, vniy = 0; + u32 rplc[8] = {0}; + u8 lookup_type = 0, port_num = 0; + u16 ivlan = 0; + + if (chip_ver > CHELSIO_T5) { + /* CtlCmdType - 0: Read, 1: Write + * CtlTcamSel - 0: TCAM0, 1: TCAM1 + * CtlXYBitSel- 0: Y bit, 1: X bit + */ + + /* Read tcamy */ + ctl = CTLCMDTYPE_V(0) | CTLXYBITSEL_V(0); + if (idx < 256) + ctl |= CTLTCAMINDEX_V(idx) | CTLTCAMSEL_V(0); + else + ctl |= CTLTCAMINDEX_V(idx - 256) | + CTLTCAMSEL_V(1); + t4_write_reg(adap, MPS_CLS_TCAM_DATA2_CTL_A, ctl); + val = t4_read_reg(adap, MPS_CLS_TCAM_DATA1_A); + tcamy = DMACH_G(val) << 32; + tcamy |= t4_read_reg(adap, MPS_CLS_TCAM_DATA0_A); + data2 = t4_read_reg(adap, MPS_CLS_TCAM_DATA2_CTL_A); + lookup_type = DATALKPTYPE_G(data2); + /* 0 - Outer header, 1 - Inner header + * [71:48] bit locations are overloaded for + * outer vs. inner lookup types. + */ + if (lookup_type && (lookup_type != DATALKPTYPE_M)) { + /* Inner header VNI */ + vniy = (data2 & DATAVIDH2_F) | + (DATAVIDH1_G(data2) << 16) | VIDL_G(val); + dip_hit = data2 & DATADIPHIT_F; + } else { + vlan_vld = data2 & DATAVIDH2_F; + ivlan = VIDL_G(val); + } + port_num = DATAPORTNUM_G(data2); + + /* Read tcamx. Change the control param */ + vnix = 0; + ctl |= CTLXYBITSEL_V(1); + t4_write_reg(adap, MPS_CLS_TCAM_DATA2_CTL_A, ctl); + val = t4_read_reg(adap, MPS_CLS_TCAM_DATA1_A); + tcamx = DMACH_G(val) << 32; + tcamx |= t4_read_reg(adap, MPS_CLS_TCAM_DATA0_A); + data2 = t4_read_reg(adap, MPS_CLS_TCAM_DATA2_CTL_A); + if (lookup_type && (lookup_type != DATALKPTYPE_M)) { + /* Inner header VNI mask */ + vnix = (data2 & DATAVIDH2_F) | + (DATAVIDH1_G(data2) << 16) | VIDL_G(val); + } + } else { + tcamy = t4_read_reg64(adap, MPS_CLS_TCAM_Y_L(idx)); + tcamx = t4_read_reg64(adap, MPS_CLS_TCAM_X_L(idx)); + } + + cls_lo = t4_read_reg(adap, MPS_CLS_SRAM_L(idx)); + cls_hi = t4_read_reg(adap, MPS_CLS_SRAM_H(idx)); + + if (tcamx & tcamy) { + seq_printf(seq, "%3u -\n", idx); + goto out; + } + + rplc[0] = rplc[1] = rplc[2] = rplc[3] = 0; + if (chip_ver > CHELSIO_T5) + replicate = (cls_lo & T6_REPLICATE_F); + else + replicate = (cls_lo & REPLICATE_F); + + if (replicate) { + struct fw_ldst_cmd ldst_cmd; + int ret; + struct fw_ldst_mps_rplc mps_rplc; + u32 ldst_addrspc; + + memset(&ldst_cmd, 0, sizeof(ldst_cmd)); + ldst_addrspc = + FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_MPS); + ldst_cmd.op_to_addrspace = + htonl(FW_CMD_OP_V(FW_LDST_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F | + ldst_addrspc); + ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd)); + ldst_cmd.u.mps.rplc.fid_idx = + htons(FW_LDST_CMD_FID_V(FW_LDST_MPS_RPLC) | + FW_LDST_CMD_IDX_V(idx)); + ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, + sizeof(ldst_cmd), &ldst_cmd); + if (ret) + dev_warn(adap->pdev_dev, "Can't read MPS " + "replication map for idx %d: %d\n", + idx, -ret); + else { + mps_rplc = ldst_cmd.u.mps.rplc; + rplc[0] = ntohl(mps_rplc.rplc31_0); + rplc[1] = ntohl(mps_rplc.rplc63_32); + rplc[2] = ntohl(mps_rplc.rplc95_64); + rplc[3] = ntohl(mps_rplc.rplc127_96); + if (adap->params.arch.mps_rplc_size > 128) { + rplc[4] = ntohl(mps_rplc.rplc159_128); + rplc[5] = ntohl(mps_rplc.rplc191_160); + rplc[6] = ntohl(mps_rplc.rplc223_192); + rplc[7] = ntohl(mps_rplc.rplc255_224); + } + } + } + + tcamxy2valmask(tcamx, tcamy, addr, &mask); + if (chip_ver > CHELSIO_T5) { + /* Inner header lookup */ + if (lookup_type && (lookup_type != DATALKPTYPE_M)) { + seq_printf(seq, + "%3u %pM %012llx %06x %06x - - %3c 'I' %4x %3c %#x%4u%4d", + idx, addr, + (unsigned long long)mask, + vniy, (vnix | vniy), + dip_hit ? 'Y' : 'N', + port_num, + (cls_lo & T6_SRAM_VLD_F) ? 'Y' : 'N', + PORTMAP_G(cls_hi), + T6_PF_G(cls_lo), + (cls_lo & T6_VF_VALID_F) ? + T6_VF_G(cls_lo) : -1); + } else { + seq_printf(seq, + "%3u %pM %012llx - - ", + idx, addr, + (unsigned long long)mask); + + if (vlan_vld) + seq_printf(seq, "%4u Y ", ivlan); + else + seq_puts(seq, " - N "); + + seq_printf(seq, + "- %3c %4x %3c %#x%4u%4d", + lookup_type ? 'I' : 'O', port_num, + (cls_lo & T6_SRAM_VLD_F) ? 'Y' : 'N', + PORTMAP_G(cls_hi), + T6_PF_G(cls_lo), + (cls_lo & T6_VF_VALID_F) ? + T6_VF_G(cls_lo) : -1); + } + } else + seq_printf(seq, "%3u %pM %012llx%3c %#x%4u%4d", + idx, addr, (unsigned long long)mask, + (cls_lo & SRAM_VLD_F) ? 'Y' : 'N', + PORTMAP_G(cls_hi), + PF_G(cls_lo), + (cls_lo & VF_VALID_F) ? VF_G(cls_lo) : -1); + + if (replicate) { + if (adap->params.arch.mps_rplc_size > 128) + seq_printf(seq, " %08x %08x %08x %08x " + "%08x %08x %08x %08x", + rplc[7], rplc[6], rplc[5], rplc[4], + rplc[3], rplc[2], rplc[1], rplc[0]); + else + seq_printf(seq, " %08x %08x %08x %08x", + rplc[3], rplc[2], rplc[1], rplc[0]); + } else { + if (adap->params.arch.mps_rplc_size > 128) + seq_printf(seq, "%72c", ' '); + else + seq_printf(seq, "%36c", ' '); + } + + if (chip_ver > CHELSIO_T5) + seq_printf(seq, "%4u%3u%3u%3u %#x\n", + T6_SRAM_PRIO0_G(cls_lo), + T6_SRAM_PRIO1_G(cls_lo), + T6_SRAM_PRIO2_G(cls_lo), + T6_SRAM_PRIO3_G(cls_lo), + (cls_lo >> T6_MULTILISTEN0_S) & 0xf); + else + seq_printf(seq, "%4u%3u%3u%3u %#x\n", + SRAM_PRIO0_G(cls_lo), SRAM_PRIO1_G(cls_lo), + SRAM_PRIO2_G(cls_lo), SRAM_PRIO3_G(cls_lo), + (cls_lo >> MULTILISTEN0_S) & 0xf); + } +out: return 0; +} + +static inline void *mps_tcam_get_idx(struct seq_file *seq, loff_t pos) +{ + struct adapter *adap = seq->private; + int max_mac_addr = is_t4(adap->params.chip) ? + NUM_MPS_CLS_SRAM_L_INSTANCES : + NUM_MPS_T5_CLS_SRAM_L_INSTANCES; + return ((pos <= max_mac_addr) ? (void *)(uintptr_t)(pos + 1) : NULL); +} + +static void *mps_tcam_start(struct seq_file *seq, loff_t *pos) +{ + return *pos ? mps_tcam_get_idx(seq, *pos) : SEQ_START_TOKEN; +} + +static void *mps_tcam_next(struct seq_file *seq, void *v, loff_t *pos) +{ + ++*pos; + return mps_tcam_get_idx(seq, *pos); +} + +static void mps_tcam_stop(struct seq_file *seq, void *v) +{ +} + +static const struct seq_operations mps_tcam_seq_ops = { + .start = mps_tcam_start, + .next = mps_tcam_next, + .stop = mps_tcam_stop, + .show = mps_tcam_show +}; + +static int mps_tcam_open(struct inode *inode, struct file *file) +{ + int res = seq_open(file, &mps_tcam_seq_ops); + + if (!res) { + struct seq_file *seq = file->private_data; + + seq->private = inode->i_private; + } + return res; +} + +static const struct file_operations mps_tcam_debugfs_fops = { + .owner = THIS_MODULE, + .open = mps_tcam_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +/* Display various sensor information. + */ +static int sensors_show(struct seq_file *seq, void *v) +{ + struct adapter *adap = seq->private; + u32 param[7], val[7]; + int ret; + + /* Note that if the sensors haven't been initialized and turned on + * we'll get values of 0, so treat those as "<unknown>" ... + */ + param[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_DIAG) | + FW_PARAMS_PARAM_Y_V(FW_PARAM_DEV_DIAG_TMP)); + param[1] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_DIAG) | + FW_PARAMS_PARAM_Y_V(FW_PARAM_DEV_DIAG_VDD)); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, + param, val); + + if (ret < 0 || val[0] == 0) + seq_puts(seq, "Temperature: <unknown>\n"); + else + seq_printf(seq, "Temperature: %dC\n", val[0]); + + if (ret < 0 || val[1] == 0) + seq_puts(seq, "Core VDD: <unknown>\n"); + else + seq_printf(seq, "Core VDD: %dmV\n", val[1]); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(sensors); + +#if IS_ENABLED(CONFIG_IPV6) +DEFINE_SHOW_ATTRIBUTE(clip_tbl); +#endif + +/*RSS Table. + */ + +static int rss_show(struct seq_file *seq, void *v, int idx) +{ + u16 *entry = v; + + seq_printf(seq, "%4d: %4u %4u %4u %4u %4u %4u %4u %4u\n", + idx * 8, entry[0], entry[1], entry[2], entry[3], entry[4], + entry[5], entry[6], entry[7]); + return 0; +} + +static int rss_open(struct inode *inode, struct file *file) +{ + struct adapter *adap = inode->i_private; + int ret, nentries; + struct seq_tab *p; + + nentries = t4_chip_rss_size(adap); + p = seq_open_tab(file, nentries / 8, 8 * sizeof(u16), 0, rss_show); + if (!p) + return -ENOMEM; + + ret = t4_read_rss(adap, (u16 *)p->data); + if (ret) + seq_release_private(inode, file); + + return ret; +} + +static const struct file_operations rss_debugfs_fops = { + .owner = THIS_MODULE, + .open = rss_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private +}; + +/* RSS Configuration. + */ + +/* Small utility function to return the strings "yes" or "no" if the supplied + * argument is non-zero. + */ +static const char *yesno(int x) +{ + static const char *yes = "yes"; + static const char *no = "no"; + + return x ? yes : no; +} + +static int rss_config_show(struct seq_file *seq, void *v) +{ + struct adapter *adapter = seq->private; + static const char * const keymode[] = { + "global", + "global and per-VF scramble", + "per-PF and per-VF scramble", + "per-VF and per-VF scramble", + }; + u32 rssconf; + + rssconf = t4_read_reg(adapter, TP_RSS_CONFIG_A); + seq_printf(seq, "TP_RSS_CONFIG: %#x\n", rssconf); + seq_printf(seq, " Tnl4TupEnIpv6: %3s\n", yesno(rssconf & + TNL4TUPENIPV6_F)); + seq_printf(seq, " Tnl2TupEnIpv6: %3s\n", yesno(rssconf & + TNL2TUPENIPV6_F)); + seq_printf(seq, " Tnl4TupEnIpv4: %3s\n", yesno(rssconf & + TNL4TUPENIPV4_F)); + seq_printf(seq, " Tnl2TupEnIpv4: %3s\n", yesno(rssconf & + TNL2TUPENIPV4_F)); + seq_printf(seq, " TnlTcpSel: %3s\n", yesno(rssconf & TNLTCPSEL_F)); + seq_printf(seq, " TnlIp6Sel: %3s\n", yesno(rssconf & TNLIP6SEL_F)); + seq_printf(seq, " TnlVrtSel: %3s\n", yesno(rssconf & TNLVRTSEL_F)); + seq_printf(seq, " TnlMapEn: %3s\n", yesno(rssconf & TNLMAPEN_F)); + seq_printf(seq, " OfdHashSave: %3s\n", yesno(rssconf & + OFDHASHSAVE_F)); + seq_printf(seq, " OfdVrtSel: %3s\n", yesno(rssconf & OFDVRTSEL_F)); + seq_printf(seq, " OfdMapEn: %3s\n", yesno(rssconf & OFDMAPEN_F)); + seq_printf(seq, " OfdLkpEn: %3s\n", yesno(rssconf & OFDLKPEN_F)); + seq_printf(seq, " Syn4TupEnIpv6: %3s\n", yesno(rssconf & + SYN4TUPENIPV6_F)); + seq_printf(seq, " Syn2TupEnIpv6: %3s\n", yesno(rssconf & + SYN2TUPENIPV6_F)); + seq_printf(seq, " Syn4TupEnIpv4: %3s\n", yesno(rssconf & + SYN4TUPENIPV4_F)); + seq_printf(seq, " Syn2TupEnIpv4: %3s\n", yesno(rssconf & + SYN2TUPENIPV4_F)); + seq_printf(seq, " Syn4TupEnIpv6: %3s\n", yesno(rssconf & + SYN4TUPENIPV6_F)); + seq_printf(seq, " SynIp6Sel: %3s\n", yesno(rssconf & SYNIP6SEL_F)); + seq_printf(seq, " SynVrt6Sel: %3s\n", yesno(rssconf & SYNVRTSEL_F)); + seq_printf(seq, " SynMapEn: %3s\n", yesno(rssconf & SYNMAPEN_F)); + seq_printf(seq, " SynLkpEn: %3s\n", yesno(rssconf & SYNLKPEN_F)); + seq_printf(seq, " ChnEn: %3s\n", yesno(rssconf & + CHANNELENABLE_F)); + seq_printf(seq, " PrtEn: %3s\n", yesno(rssconf & + PORTENABLE_F)); + seq_printf(seq, " TnlAllLkp: %3s\n", yesno(rssconf & + TNLALLLOOKUP_F)); + seq_printf(seq, " VrtEn: %3s\n", yesno(rssconf & + VIRTENABLE_F)); + seq_printf(seq, " CngEn: %3s\n", yesno(rssconf & + CONGESTIONENABLE_F)); + seq_printf(seq, " HashToeplitz: %3s\n", yesno(rssconf & + HASHTOEPLITZ_F)); + seq_printf(seq, " Udp4En: %3s\n", yesno(rssconf & UDPENABLE_F)); + seq_printf(seq, " Disable: %3s\n", yesno(rssconf & DISABLE_F)); + + seq_puts(seq, "\n"); + + rssconf = t4_read_reg(adapter, TP_RSS_CONFIG_TNL_A); + seq_printf(seq, "TP_RSS_CONFIG_TNL: %#x\n", rssconf); + seq_printf(seq, " MaskSize: %3d\n", MASKSIZE_G(rssconf)); + seq_printf(seq, " MaskFilter: %3d\n", MASKFILTER_G(rssconf)); + if (CHELSIO_CHIP_VERSION(adapter->params.chip) > CHELSIO_T5) { + seq_printf(seq, " HashAll: %3s\n", + yesno(rssconf & HASHALL_F)); + seq_printf(seq, " HashEth: %3s\n", + yesno(rssconf & HASHETH_F)); + } + seq_printf(seq, " UseWireCh: %3s\n", yesno(rssconf & USEWIRECH_F)); + + seq_puts(seq, "\n"); + + rssconf = t4_read_reg(adapter, TP_RSS_CONFIG_OFD_A); + seq_printf(seq, "TP_RSS_CONFIG_OFD: %#x\n", rssconf); + seq_printf(seq, " MaskSize: %3d\n", MASKSIZE_G(rssconf)); + seq_printf(seq, " RRCplMapEn: %3s\n", yesno(rssconf & + RRCPLMAPEN_F)); + seq_printf(seq, " RRCplQueWidth: %3d\n", RRCPLQUEWIDTH_G(rssconf)); + + seq_puts(seq, "\n"); + + rssconf = t4_read_reg(adapter, TP_RSS_CONFIG_SYN_A); + seq_printf(seq, "TP_RSS_CONFIG_SYN: %#x\n", rssconf); + seq_printf(seq, " MaskSize: %3d\n", MASKSIZE_G(rssconf)); + seq_printf(seq, " UseWireCh: %3s\n", yesno(rssconf & USEWIRECH_F)); + + seq_puts(seq, "\n"); + + rssconf = t4_read_reg(adapter, TP_RSS_CONFIG_VRT_A); + seq_printf(seq, "TP_RSS_CONFIG_VRT: %#x\n", rssconf); + if (CHELSIO_CHIP_VERSION(adapter->params.chip) > CHELSIO_T5) { + seq_printf(seq, " KeyWrAddrX: %3d\n", + KEYWRADDRX_G(rssconf)); + seq_printf(seq, " KeyExtend: %3s\n", + yesno(rssconf & KEYEXTEND_F)); + } + seq_printf(seq, " VfRdRg: %3s\n", yesno(rssconf & VFRDRG_F)); + seq_printf(seq, " VfRdEn: %3s\n", yesno(rssconf & VFRDEN_F)); + seq_printf(seq, " VfPerrEn: %3s\n", yesno(rssconf & VFPERREN_F)); + seq_printf(seq, " KeyPerrEn: %3s\n", yesno(rssconf & KEYPERREN_F)); + seq_printf(seq, " DisVfVlan: %3s\n", yesno(rssconf & + DISABLEVLAN_F)); + seq_printf(seq, " EnUpSwt: %3s\n", yesno(rssconf & ENABLEUP0_F)); + seq_printf(seq, " HashDelay: %3d\n", HASHDELAY_G(rssconf)); + if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) + seq_printf(seq, " VfWrAddr: %3d\n", VFWRADDR_G(rssconf)); + else + seq_printf(seq, " VfWrAddr: %3d\n", + T6_VFWRADDR_G(rssconf)); + seq_printf(seq, " KeyMode: %s\n", keymode[KEYMODE_G(rssconf)]); + seq_printf(seq, " VfWrEn: %3s\n", yesno(rssconf & VFWREN_F)); + seq_printf(seq, " KeyWrEn: %3s\n", yesno(rssconf & KEYWREN_F)); + seq_printf(seq, " KeyWrAddr: %3d\n", KEYWRADDR_G(rssconf)); + + seq_puts(seq, "\n"); + + rssconf = t4_read_reg(adapter, TP_RSS_CONFIG_CNG_A); + seq_printf(seq, "TP_RSS_CONFIG_CNG: %#x\n", rssconf); + seq_printf(seq, " ChnCount3: %3s\n", yesno(rssconf & CHNCOUNT3_F)); + seq_printf(seq, " ChnCount2: %3s\n", yesno(rssconf & CHNCOUNT2_F)); + seq_printf(seq, " ChnCount1: %3s\n", yesno(rssconf & CHNCOUNT1_F)); + seq_printf(seq, " ChnCount0: %3s\n", yesno(rssconf & CHNCOUNT0_F)); + seq_printf(seq, " ChnUndFlow3: %3s\n", yesno(rssconf & + CHNUNDFLOW3_F)); + seq_printf(seq, " ChnUndFlow2: %3s\n", yesno(rssconf & + CHNUNDFLOW2_F)); + seq_printf(seq, " ChnUndFlow1: %3s\n", yesno(rssconf & + CHNUNDFLOW1_F)); + seq_printf(seq, " ChnUndFlow0: %3s\n", yesno(rssconf & + CHNUNDFLOW0_F)); + seq_printf(seq, " RstChn3: %3s\n", yesno(rssconf & RSTCHN3_F)); + seq_printf(seq, " RstChn2: %3s\n", yesno(rssconf & RSTCHN2_F)); + seq_printf(seq, " RstChn1: %3s\n", yesno(rssconf & RSTCHN1_F)); + seq_printf(seq, " RstChn0: %3s\n", yesno(rssconf & RSTCHN0_F)); + seq_printf(seq, " UpdVld: %3s\n", yesno(rssconf & UPDVLD_F)); + seq_printf(seq, " Xoff: %3s\n", yesno(rssconf & XOFF_F)); + seq_printf(seq, " UpdChn3: %3s\n", yesno(rssconf & UPDCHN3_F)); + seq_printf(seq, " UpdChn2: %3s\n", yesno(rssconf & UPDCHN2_F)); + seq_printf(seq, " UpdChn1: %3s\n", yesno(rssconf & UPDCHN1_F)); + seq_printf(seq, " UpdChn0: %3s\n", yesno(rssconf & UPDCHN0_F)); + seq_printf(seq, " Queue: %3d\n", QUEUE_G(rssconf)); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(rss_config); + +/* RSS Secret Key. + */ + +static int rss_key_show(struct seq_file *seq, void *v) +{ + u32 key[10]; + + t4_read_rss_key(seq->private, key, true); + seq_printf(seq, "%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x\n", + key[9], key[8], key[7], key[6], key[5], key[4], key[3], + key[2], key[1], key[0]); + return 0; +} + +static int rss_key_open(struct inode *inode, struct file *file) +{ + return single_open(file, rss_key_show, inode->i_private); +} + +static ssize_t rss_key_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + int i, j; + u32 key[10]; + char s[100], *p; + struct adapter *adap = file_inode(file)->i_private; + + if (count > sizeof(s) - 1) + return -EINVAL; + if (copy_from_user(s, buf, count)) + return -EFAULT; + for (i = count; i > 0 && isspace(s[i - 1]); i--) + ; + s[i] = '\0'; + + for (p = s, i = 9; i >= 0; i--) { + key[i] = 0; + for (j = 0; j < 8; j++, p++) { + if (!isxdigit(*p)) + return -EINVAL; + key[i] = (key[i] << 4) | hex2val(*p); + } + } + + t4_write_rss_key(adap, key, -1, true); + return count; +} + +static const struct file_operations rss_key_debugfs_fops = { + .owner = THIS_MODULE, + .open = rss_key_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = rss_key_write +}; + +/* PF RSS Configuration. + */ + +struct rss_pf_conf { + u32 rss_pf_map; + u32 rss_pf_mask; + u32 rss_pf_config; +}; + +static int rss_pf_config_show(struct seq_file *seq, void *v, int idx) +{ + struct rss_pf_conf *pfconf; + + if (v == SEQ_START_TOKEN) { + /* use the 0th entry to dump the PF Map Index Size */ + pfconf = seq->private + offsetof(struct seq_tab, data); + seq_printf(seq, "PF Map Index Size = %d\n\n", + LKPIDXSIZE_G(pfconf->rss_pf_map)); + + seq_puts(seq, " RSS PF VF Hash Tuple Enable Default\n"); + seq_puts(seq, " Enable IPF Mask Mask IPv6 IPv4 UDP Queue\n"); + seq_puts(seq, " PF Map Chn Prt Map Size Size Four Two Four Two Four Ch1 Ch0\n"); + } else { + #define G_PFnLKPIDX(map, n) \ + (((map) >> PF1LKPIDX_S*(n)) & PF0LKPIDX_M) + #define G_PFnMSKSIZE(mask, n) \ + (((mask) >> PF1MSKSIZE_S*(n)) & PF1MSKSIZE_M) + + pfconf = v; + seq_printf(seq, "%3d %3s %3s %3s %3d %3d %3d %3s %3s %3s %3s %3s %3d %3d\n", + idx, + yesno(pfconf->rss_pf_config & MAPENABLE_F), + yesno(pfconf->rss_pf_config & CHNENABLE_F), + yesno(pfconf->rss_pf_config & PRTENABLE_F), + G_PFnLKPIDX(pfconf->rss_pf_map, idx), + G_PFnMSKSIZE(pfconf->rss_pf_mask, idx), + IVFWIDTH_G(pfconf->rss_pf_config), + yesno(pfconf->rss_pf_config & IP6FOURTUPEN_F), + yesno(pfconf->rss_pf_config & IP6TWOTUPEN_F), + yesno(pfconf->rss_pf_config & IP4FOURTUPEN_F), + yesno(pfconf->rss_pf_config & IP4TWOTUPEN_F), + yesno(pfconf->rss_pf_config & UDPFOURTUPEN_F), + CH1DEFAULTQUEUE_G(pfconf->rss_pf_config), + CH0DEFAULTQUEUE_G(pfconf->rss_pf_config)); + + #undef G_PFnLKPIDX + #undef G_PFnMSKSIZE + } + return 0; +} + +static int rss_pf_config_open(struct inode *inode, struct file *file) +{ + struct adapter *adapter = inode->i_private; + struct seq_tab *p; + u32 rss_pf_map, rss_pf_mask; + struct rss_pf_conf *pfconf; + int pf; + + p = seq_open_tab(file, 8, sizeof(*pfconf), 1, rss_pf_config_show); + if (!p) + return -ENOMEM; + + pfconf = (struct rss_pf_conf *)p->data; + rss_pf_map = t4_read_rss_pf_map(adapter, true); + rss_pf_mask = t4_read_rss_pf_mask(adapter, true); + for (pf = 0; pf < 8; pf++) { + pfconf[pf].rss_pf_map = rss_pf_map; + pfconf[pf].rss_pf_mask = rss_pf_mask; + t4_read_rss_pf_config(adapter, pf, &pfconf[pf].rss_pf_config, + true); + } + return 0; +} + +static const struct file_operations rss_pf_config_debugfs_fops = { + .owner = THIS_MODULE, + .open = rss_pf_config_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private +}; + +/* VF RSS Configuration. + */ + +struct rss_vf_conf { + u32 rss_vf_vfl; + u32 rss_vf_vfh; +}; + +static int rss_vf_config_show(struct seq_file *seq, void *v, int idx) +{ + if (v == SEQ_START_TOKEN) { + seq_puts(seq, " RSS Hash Tuple Enable\n"); + seq_puts(seq, " Enable IVF Dis Enb IPv6 IPv4 UDP Def Secret Key\n"); + seq_puts(seq, " VF Chn Prt Map VLAN uP Four Two Four Two Four Que Idx Hash\n"); + } else { + struct rss_vf_conf *vfconf = v; + + seq_printf(seq, "%3d %3s %3s %3d %3s %3s %3s %3s %3s %3s %3s %4d %3d %#10x\n", + idx, + yesno(vfconf->rss_vf_vfh & VFCHNEN_F), + yesno(vfconf->rss_vf_vfh & VFPRTEN_F), + VFLKPIDX_G(vfconf->rss_vf_vfh), + yesno(vfconf->rss_vf_vfh & VFVLNEX_F), + yesno(vfconf->rss_vf_vfh & VFUPEN_F), + yesno(vfconf->rss_vf_vfh & VFIP4FOURTUPEN_F), + yesno(vfconf->rss_vf_vfh & VFIP6TWOTUPEN_F), + yesno(vfconf->rss_vf_vfh & VFIP4FOURTUPEN_F), + yesno(vfconf->rss_vf_vfh & VFIP4TWOTUPEN_F), + yesno(vfconf->rss_vf_vfh & ENABLEUDPHASH_F), + DEFAULTQUEUE_G(vfconf->rss_vf_vfh), + KEYINDEX_G(vfconf->rss_vf_vfh), + vfconf->rss_vf_vfl); + } + return 0; +} + +static int rss_vf_config_open(struct inode *inode, struct file *file) +{ + struct adapter *adapter = inode->i_private; + struct seq_tab *p; + struct rss_vf_conf *vfconf; + int vf, vfcount = adapter->params.arch.vfcount; + + p = seq_open_tab(file, vfcount, sizeof(*vfconf), 1, rss_vf_config_show); + if (!p) + return -ENOMEM; + + vfconf = (struct rss_vf_conf *)p->data; + for (vf = 0; vf < vfcount; vf++) { + t4_read_rss_vf_config(adapter, vf, &vfconf[vf].rss_vf_vfl, + &vfconf[vf].rss_vf_vfh, true); + } + return 0; +} + +static const struct file_operations rss_vf_config_debugfs_fops = { + .owner = THIS_MODULE, + .open = rss_vf_config_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private +}; + +#ifdef CONFIG_CHELSIO_T4_DCB + +/* Data Center Briging information for each port. + */ +static int dcb_info_show(struct seq_file *seq, void *v) +{ + struct adapter *adap = seq->private; + + if (v == SEQ_START_TOKEN) { + seq_puts(seq, "Data Center Bridging Information\n"); + } else { + int port = (uintptr_t)v - 2; + struct net_device *dev = adap->port[port]; + struct port_info *pi = netdev2pinfo(dev); + struct port_dcb_info *dcb = &pi->dcb; + + seq_puts(seq, "\n"); + seq_printf(seq, "Port: %d (DCB negotiated: %s)\n", + port, + cxgb4_dcb_enabled(dev) ? "yes" : "no"); + + if (cxgb4_dcb_enabled(dev)) + seq_printf(seq, "[ DCBx Version %s ]\n", + dcb_ver_array[dcb->dcb_version]); + + if (dcb->msgs) { + int i; + + seq_puts(seq, "\n Index\t\t\t :\t"); + for (i = 0; i < 8; i++) + seq_printf(seq, " %3d", i); + seq_puts(seq, "\n\n"); + } + + if (dcb->msgs & CXGB4_DCB_FW_PGID) { + int prio, pgid; + + seq_puts(seq, " Priority Group IDs\t :\t"); + for (prio = 0; prio < 8; prio++) { + pgid = (dcb->pgid >> 4 * (7 - prio)) & 0xf; + seq_printf(seq, " %3d", pgid); + } + seq_puts(seq, "\n"); + } + + if (dcb->msgs & CXGB4_DCB_FW_PGRATE) { + int pg; + + seq_puts(seq, " Priority Group BW(%)\t :\t"); + for (pg = 0; pg < 8; pg++) + seq_printf(seq, " %3d", dcb->pgrate[pg]); + seq_puts(seq, "\n"); + + if (dcb->dcb_version == FW_PORT_DCB_VER_IEEE) { + seq_puts(seq, " TSA Algorithm\t\t :\t"); + for (pg = 0; pg < 8; pg++) + seq_printf(seq, " %3d", dcb->tsa[pg]); + seq_puts(seq, "\n"); + } + + seq_printf(seq, " Max PG Traffic Classes [%3d ]\n", + dcb->pg_num_tcs_supported); + + seq_puts(seq, "\n"); + } + + if (dcb->msgs & CXGB4_DCB_FW_PRIORATE) { + int prio; + + seq_puts(seq, " Priority Rate\t:\t"); + for (prio = 0; prio < 8; prio++) + seq_printf(seq, " %3d", dcb->priorate[prio]); + seq_puts(seq, "\n"); + } + + if (dcb->msgs & CXGB4_DCB_FW_PFC) { + int prio; + + seq_puts(seq, " Priority Flow Control :\t"); + for (prio = 0; prio < 8; prio++) { + int pfcen = (dcb->pfcen >> 1 * (7 - prio)) + & 0x1; + seq_printf(seq, " %3d", pfcen); + } + seq_puts(seq, "\n"); + + seq_printf(seq, " Max PFC Traffic Classes [%3d ]\n", + dcb->pfc_num_tcs_supported); + + seq_puts(seq, "\n"); + } + + if (dcb->msgs & CXGB4_DCB_FW_APP_ID) { + int app, napps; + + seq_puts(seq, " Application Information:\n"); + seq_puts(seq, " App Priority Selection Protocol\n"); + seq_puts(seq, " Index Map Field ID\n"); + for (app = 0, napps = 0; + app < CXGB4_MAX_DCBX_APP_SUPPORTED; app++) { + struct app_priority *ap; + static const char * const sel_names[] = { + "Ethertype", + "Socket TCP", + "Socket UDP", + "Socket All", + }; + const char *sel_name; + + ap = &dcb->app_priority[app]; + /* skip empty slots */ + if (ap->protocolid == 0) + continue; + napps++; + + if (ap->sel_field < ARRAY_SIZE(sel_names)) + sel_name = sel_names[ap->sel_field]; + else + sel_name = "UNKNOWN"; + + seq_printf(seq, " %3d %#04x %-10s (%d) %#06x (%d)\n", + app, + ap->user_prio_map, + sel_name, ap->sel_field, + ap->protocolid, ap->protocolid); + } + if (napps == 0) + seq_puts(seq, " --- None ---\n"); + } + } + return 0; +} + +static inline void *dcb_info_get_idx(struct adapter *adap, loff_t pos) +{ + return (pos <= adap->params.nports + ? (void *)((uintptr_t)pos + 1) + : NULL); +} + +static void *dcb_info_start(struct seq_file *seq, loff_t *pos) +{ + struct adapter *adap = seq->private; + + return (*pos + ? dcb_info_get_idx(adap, *pos) + : SEQ_START_TOKEN); +} + +static void dcb_info_stop(struct seq_file *seq, void *v) +{ +} + +static void *dcb_info_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct adapter *adap = seq->private; + + (*pos)++; + return dcb_info_get_idx(adap, *pos); +} + +static const struct seq_operations dcb_info_seq_ops = { + .start = dcb_info_start, + .next = dcb_info_next, + .stop = dcb_info_stop, + .show = dcb_info_show +}; + +static int dcb_info_open(struct inode *inode, struct file *file) +{ + int res = seq_open(file, &dcb_info_seq_ops); + + if (!res) { + struct seq_file *seq = file->private_data; + + seq->private = inode->i_private; + } + return res; +} + +static const struct file_operations dcb_info_debugfs_fops = { + .owner = THIS_MODULE, + .open = dcb_info_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; +#endif /* CONFIG_CHELSIO_T4_DCB */ + +static int resources_show(struct seq_file *seq, void *v) +{ + struct adapter *adapter = seq->private; + struct pf_resources *pfres = &adapter->params.pfres; + + #define S(desc, fmt, var) \ + seq_printf(seq, "%-60s " fmt "\n", \ + desc " (" #var "):", pfres->var) + + S("Virtual Interfaces", "%d", nvi); + S("Egress Queues", "%d", neq); + S("Ethernet Control", "%d", nethctrl); + S("Ingress Queues/w Free Lists/Interrupts", "%d", niqflint); + S("Ingress Queues", "%d", niq); + S("Traffic Class", "%d", tc); + S("Port Access Rights Mask", "%#x", pmask); + S("MAC Address Filters", "%d", nexactf); + S("Firmware Command Read Capabilities", "%#x", r_caps); + S("Firmware Command Write/Execute Capabilities", "%#x", wx_caps); + + #undef S + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(resources); + +/** + * ethqset2pinfo - return port_info of an Ethernet Queue Set + * @adap: the adapter + * @qset: Ethernet Queue Set + */ +static inline struct port_info *ethqset2pinfo(struct adapter *adap, int qset) +{ + int pidx; + + for_each_port(adap, pidx) { + struct port_info *pi = adap2pinfo(adap, pidx); + + if (qset >= pi->first_qset && + qset < pi->first_qset + pi->nqsets) + return pi; + } + + /* should never happen! */ + BUG(); + return NULL; +} + +static int sge_qinfo_uld_txq_entries(const struct adapter *adap, int uld) +{ + const struct sge_uld_txq_info *utxq_info = adap->sge.uld_txq_info[uld]; + + if (!utxq_info) + return 0; + + return DIV_ROUND_UP(utxq_info->ntxq, 4); +} + +static int sge_qinfo_uld_rspq_entries(const struct adapter *adap, int uld, + bool ciq) +{ + const struct sge_uld_rxq_info *urxq_info = adap->sge.uld_rxq_info[uld]; + + if (!urxq_info) + return 0; + + return ciq ? DIV_ROUND_UP(urxq_info->nciq, 4) : + DIV_ROUND_UP(urxq_info->nrxq, 4); +} + +static int sge_qinfo_uld_rxq_entries(const struct adapter *adap, int uld) +{ + return sge_qinfo_uld_rspq_entries(adap, uld, false); +} + +static int sge_qinfo_uld_ciq_entries(const struct adapter *adap, int uld) +{ + return sge_qinfo_uld_rspq_entries(adap, uld, true); +} + +static int sge_qinfo_show(struct seq_file *seq, void *v) +{ + int eth_entries, ctrl_entries, eohw_entries = 0, eosw_entries = 0; + int uld_rxq_entries[CXGB4_ULD_MAX] = { 0 }; + int uld_ciq_entries[CXGB4_ULD_MAX] = { 0 }; + int uld_txq_entries[CXGB4_TX_MAX] = { 0 }; + const struct sge_uld_txq_info *utxq_info; + const struct sge_uld_rxq_info *urxq_info; + struct cxgb4_tc_port_mqprio *port_mqprio; + struct adapter *adap = seq->private; + int i, j, n, r = (uintptr_t)v - 1; + struct sge *s = &adap->sge; + + eth_entries = DIV_ROUND_UP(adap->sge.ethqsets, 4); + ctrl_entries = DIV_ROUND_UP(MAX_CTRL_QUEUES, 4); + + if (r) + seq_putc(seq, '\n'); + +#define S3(fmt_spec, s, v) \ +do { \ + seq_printf(seq, "%-12s", s); \ + for (i = 0; i < n; ++i) \ + seq_printf(seq, " %16" fmt_spec, v); \ + seq_putc(seq, '\n'); \ +} while (0) +#define S(s, v) S3("s", s, v) +#define T3(fmt_spec, s, v) S3(fmt_spec, s, tx[i].v) +#define T(s, v) S3("u", s, tx[i].v) +#define TL(s, v) T3("lu", s, v) +#define R3(fmt_spec, s, v) S3(fmt_spec, s, rx[i].v) +#define R(s, v) S3("u", s, rx[i].v) +#define RL(s, v) R3("lu", s, v) + + if (r < eth_entries) { + int base_qset = r * 4; + const struct sge_eth_rxq *rx = &s->ethrxq[base_qset]; + const struct sge_eth_txq *tx = &s->ethtxq[base_qset]; + + n = min(4, s->ethqsets - 4 * r); + + S("QType:", "Ethernet"); + S("Interface:", + rx[i].rspq.netdev ? rx[i].rspq.netdev->name : "N/A"); + T("TxQ ID:", q.cntxt_id); + T("TxQ size:", q.size); + T("TxQ inuse:", q.in_use); + T("TxQ CIDX:", q.cidx); + T("TxQ PIDX:", q.pidx); +#ifdef CONFIG_CHELSIO_T4_DCB + T("DCB Prio:", dcb_prio); + S3("u", "DCB PGID:", + (ethqset2pinfo(adap, base_qset + i)->dcb.pgid >> + 4*(7-tx[i].dcb_prio)) & 0xf); + S3("u", "DCB PFC:", + (ethqset2pinfo(adap, base_qset + i)->dcb.pfcen >> + 1*(7-tx[i].dcb_prio)) & 0x1); +#endif + R("RspQ ID:", rspq.abs_id); + R("RspQ size:", rspq.size); + R("RspQE size:", rspq.iqe_len); + R("RspQ CIDX:", rspq.cidx); + R("RspQ Gen:", rspq.gen); + S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq)); + S3("u", "Intr pktcnt:", s->counter_val[rx[i].rspq.pktcnt_idx]); + R("FL ID:", fl.cntxt_id); + R("FL size:", fl.size - 8); + R("FL pend:", fl.pend_cred); + R("FL avail:", fl.avail); + R("FL PIDX:", fl.pidx); + R("FL CIDX:", fl.cidx); + RL("RxPackets:", stats.pkts); + RL("RxCSO:", stats.rx_cso); + RL("VLANxtract:", stats.vlan_ex); + RL("LROmerged:", stats.lro_merged); + RL("LROpackets:", stats.lro_pkts); + RL("RxDrops:", stats.rx_drops); + RL("RxBadPkts:", stats.bad_rx_pkts); + TL("TSO:", tso); + TL("USO:", uso); + TL("TxCSO:", tx_cso); + TL("VLANins:", vlan_ins); + TL("TxQFull:", q.stops); + TL("TxQRestarts:", q.restarts); + TL("TxMapErr:", mapping_err); + RL("FLAllocErr:", fl.alloc_failed); + RL("FLLrgAlcErr:", fl.large_alloc_failed); + RL("FLMapErr:", fl.mapping_err); + RL("FLLow:", fl.low); + RL("FLStarving:", fl.starving); + + goto out; + } + + r -= eth_entries; + for_each_port(adap, j) { + struct port_info *pi = adap2pinfo(adap, j); + const struct sge_eth_rxq *rx; + + mutex_lock(&pi->vi_mirror_mutex); + if (!pi->vi_mirror_count) { + mutex_unlock(&pi->vi_mirror_mutex); + continue; + } + + if (r >= DIV_ROUND_UP(pi->nmirrorqsets, 4)) { + r -= DIV_ROUND_UP(pi->nmirrorqsets, 4); + mutex_unlock(&pi->vi_mirror_mutex); + continue; + } + + rx = &s->mirror_rxq[j][r * 4]; + n = min(4, pi->nmirrorqsets - 4 * r); + + S("QType:", "Mirror-Rxq"); + S("Interface:", + rx[i].rspq.netdev ? rx[i].rspq.netdev->name : "N/A"); + R("RspQ ID:", rspq.abs_id); + R("RspQ size:", rspq.size); + R("RspQE size:", rspq.iqe_len); + R("RspQ CIDX:", rspq.cidx); + R("RspQ Gen:", rspq.gen); + S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq)); + S3("u", "Intr pktcnt:", s->counter_val[rx[i].rspq.pktcnt_idx]); + R("FL ID:", fl.cntxt_id); + R("FL size:", fl.size - 8); + R("FL pend:", fl.pend_cred); + R("FL avail:", fl.avail); + R("FL PIDX:", fl.pidx); + R("FL CIDX:", fl.cidx); + RL("RxPackets:", stats.pkts); + RL("RxCSO:", stats.rx_cso); + RL("VLANxtract:", stats.vlan_ex); + RL("LROmerged:", stats.lro_merged); + RL("LROpackets:", stats.lro_pkts); + RL("RxDrops:", stats.rx_drops); + RL("RxBadPkts:", stats.bad_rx_pkts); + RL("FLAllocErr:", fl.alloc_failed); + RL("FLLrgAlcErr:", fl.large_alloc_failed); + RL("FLMapErr:", fl.mapping_err); + RL("FLLow:", fl.low); + RL("FLStarving:", fl.starving); + + mutex_unlock(&pi->vi_mirror_mutex); + goto out; + } + + if (!adap->tc_mqprio) + goto skip_mqprio; + + mutex_lock(&adap->tc_mqprio->mqprio_mutex); + if (!refcount_read(&adap->tc_mqprio->refcnt)) { + mutex_unlock(&adap->tc_mqprio->mqprio_mutex); + goto skip_mqprio; + } + + eohw_entries = DIV_ROUND_UP(adap->sge.eoqsets, 4); + if (r < eohw_entries) { + int base_qset = r * 4; + const struct sge_ofld_rxq *rx = &s->eohw_rxq[base_qset]; + const struct sge_eohw_txq *tx = &s->eohw_txq[base_qset]; + + n = min(4, s->eoqsets - 4 * r); + + S("QType:", "ETHOFLD"); + S("Interface:", + rx[i].rspq.netdev ? rx[i].rspq.netdev->name : "N/A"); + T("TxQ ID:", q.cntxt_id); + T("TxQ size:", q.size); + T("TxQ inuse:", q.in_use); + T("TxQ CIDX:", q.cidx); + T("TxQ PIDX:", q.pidx); + R("RspQ ID:", rspq.abs_id); + R("RspQ size:", rspq.size); + R("RspQE size:", rspq.iqe_len); + R("RspQ CIDX:", rspq.cidx); + R("RspQ Gen:", rspq.gen); + S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq)); + S3("u", "Intr pktcnt:", s->counter_val[rx[i].rspq.pktcnt_idx]); + R("FL ID:", fl.cntxt_id); + S3("u", "FL size:", rx->fl.size ? rx->fl.size - 8 : 0); + R("FL pend:", fl.pend_cred); + R("FL avail:", fl.avail); + R("FL PIDX:", fl.pidx); + R("FL CIDX:", fl.cidx); + RL("RxPackets:", stats.pkts); + RL("RxImm:", stats.imm); + RL("RxAN", stats.an); + RL("RxNoMem", stats.nomem); + TL("TSO:", tso); + TL("USO:", uso); + TL("TxCSO:", tx_cso); + TL("VLANins:", vlan_ins); + TL("TxQFull:", q.stops); + TL("TxQRestarts:", q.restarts); + TL("TxMapErr:", mapping_err); + RL("FLAllocErr:", fl.alloc_failed); + RL("FLLrgAlcErr:", fl.large_alloc_failed); + RL("FLMapErr:", fl.mapping_err); + RL("FLLow:", fl.low); + RL("FLStarving:", fl.starving); + + mutex_unlock(&adap->tc_mqprio->mqprio_mutex); + goto out; + } + + r -= eohw_entries; + for (j = 0; j < adap->params.nports; j++) { + int entries; + u8 tc; + + port_mqprio = &adap->tc_mqprio->port_mqprio[j]; + entries = 0; + for (tc = 0; tc < port_mqprio->mqprio.qopt.num_tc; tc++) + entries += port_mqprio->mqprio.qopt.count[tc]; + + if (!entries) + continue; + + eosw_entries = DIV_ROUND_UP(entries, 4); + if (r < eosw_entries) { + const struct sge_eosw_txq *tx; + + n = min(4, entries - 4 * r); + tx = &port_mqprio->eosw_txq[4 * r]; + + S("QType:", "EOSW-TXQ"); + S("Interface:", + adap->port[j] ? adap->port[j]->name : "N/A"); + T("EOTID:", hwtid); + T("HWQID:", hwqid); + T("State:", state); + T("Size:", ndesc); + T("In-Use:", inuse); + T("Credits:", cred); + T("Compl:", ncompl); + T("Last-Compl:", last_compl); + T("PIDX:", pidx); + T("Last-PIDX:", last_pidx); + T("CIDX:", cidx); + T("Last-CIDX:", last_cidx); + T("FLOWC-IDX:", flowc_idx); + + mutex_unlock(&adap->tc_mqprio->mqprio_mutex); + goto out; + } + + r -= eosw_entries; + } + mutex_unlock(&adap->tc_mqprio->mqprio_mutex); + +skip_mqprio: + if (!is_uld(adap)) + goto skip_uld; + + mutex_lock(&uld_mutex); + if (s->uld_txq_info) + for (i = 0; i < ARRAY_SIZE(uld_txq_entries); i++) + uld_txq_entries[i] = sge_qinfo_uld_txq_entries(adap, i); + + if (s->uld_rxq_info) { + for (i = 0; i < ARRAY_SIZE(uld_rxq_entries); i++) { + uld_rxq_entries[i] = sge_qinfo_uld_rxq_entries(adap, i); + uld_ciq_entries[i] = sge_qinfo_uld_ciq_entries(adap, i); + } + } + + if (r < uld_txq_entries[CXGB4_TX_OFLD]) { + const struct sge_uld_txq *tx; + + utxq_info = s->uld_txq_info[CXGB4_TX_OFLD]; + tx = &utxq_info->uldtxq[r * 4]; + n = min(4, utxq_info->ntxq - 4 * r); + + S("QType:", "OFLD-TXQ"); + T("TxQ ID:", q.cntxt_id); + T("TxQ size:", q.size); + T("TxQ inuse:", q.in_use); + T("TxQ CIDX:", q.cidx); + T("TxQ PIDX:", q.pidx); + + goto unlock; + } + + r -= uld_txq_entries[CXGB4_TX_OFLD]; + if (r < uld_rxq_entries[CXGB4_ULD_RDMA]) { + const struct sge_ofld_rxq *rx; + + urxq_info = s->uld_rxq_info[CXGB4_ULD_RDMA]; + rx = &urxq_info->uldrxq[r * 4]; + n = min(4, urxq_info->nrxq - 4 * r); + + S("QType:", "RDMA-CPL"); + S("Interface:", + rx[i].rspq.netdev ? rx[i].rspq.netdev->name : "N/A"); + R("RspQ ID:", rspq.abs_id); + R("RspQ size:", rspq.size); + R("RspQE size:", rspq.iqe_len); + R("RspQ CIDX:", rspq.cidx); + R("RspQ Gen:", rspq.gen); + S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq)); + S3("u", "Intr pktcnt:", s->counter_val[rx[i].rspq.pktcnt_idx]); + R("FL ID:", fl.cntxt_id); + R("FL size:", fl.size - 8); + R("FL pend:", fl.pend_cred); + R("FL avail:", fl.avail); + R("FL PIDX:", fl.pidx); + R("FL CIDX:", fl.cidx); + + goto unlock; + } + + r -= uld_rxq_entries[CXGB4_ULD_RDMA]; + if (r < uld_ciq_entries[CXGB4_ULD_RDMA]) { + const struct sge_ofld_rxq *rx; + int ciq_idx = 0; + + urxq_info = s->uld_rxq_info[CXGB4_ULD_RDMA]; + ciq_idx = urxq_info->nrxq + (r * 4); + rx = &urxq_info->uldrxq[ciq_idx]; + n = min(4, urxq_info->nciq - 4 * r); + + S("QType:", "RDMA-CIQ"); + S("Interface:", + rx[i].rspq.netdev ? rx[i].rspq.netdev->name : "N/A"); + R("RspQ ID:", rspq.abs_id); + R("RspQ size:", rspq.size); + R("RspQE size:", rspq.iqe_len); + R("RspQ CIDX:", rspq.cidx); + R("RspQ Gen:", rspq.gen); + S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq)); + S3("u", "Intr pktcnt:", s->counter_val[rx[i].rspq.pktcnt_idx]); + + goto unlock; + } + + r -= uld_ciq_entries[CXGB4_ULD_RDMA]; + if (r < uld_rxq_entries[CXGB4_ULD_ISCSI]) { + const struct sge_ofld_rxq *rx; + + urxq_info = s->uld_rxq_info[CXGB4_ULD_ISCSI]; + rx = &urxq_info->uldrxq[r * 4]; + n = min(4, urxq_info->nrxq - 4 * r); + + S("QType:", "iSCSI"); + R("RspQ ID:", rspq.abs_id); + R("RspQ size:", rspq.size); + R("RspQE size:", rspq.iqe_len); + R("RspQ CIDX:", rspq.cidx); + R("RspQ Gen:", rspq.gen); + S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq)); + S3("u", "Intr pktcnt:", s->counter_val[rx[i].rspq.pktcnt_idx]); + R("FL ID:", fl.cntxt_id); + R("FL size:", fl.size - 8); + R("FL pend:", fl.pend_cred); + R("FL avail:", fl.avail); + R("FL PIDX:", fl.pidx); + R("FL CIDX:", fl.cidx); + + goto unlock; + } + + r -= uld_rxq_entries[CXGB4_ULD_ISCSI]; + if (r < uld_rxq_entries[CXGB4_ULD_ISCSIT]) { + const struct sge_ofld_rxq *rx; + + urxq_info = s->uld_rxq_info[CXGB4_ULD_ISCSIT]; + rx = &urxq_info->uldrxq[r * 4]; + n = min(4, urxq_info->nrxq - 4 * r); + + S("QType:", "iSCSIT"); + R("RspQ ID:", rspq.abs_id); + R("RspQ size:", rspq.size); + R("RspQE size:", rspq.iqe_len); + R("RspQ CIDX:", rspq.cidx); + R("RspQ Gen:", rspq.gen); + S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq)); + S3("u", "Intr pktcnt:", s->counter_val[rx[i].rspq.pktcnt_idx]); + R("FL ID:", fl.cntxt_id); + R("FL size:", fl.size - 8); + R("FL pend:", fl.pend_cred); + R("FL avail:", fl.avail); + R("FL PIDX:", fl.pidx); + R("FL CIDX:", fl.cidx); + + goto unlock; + } + + r -= uld_rxq_entries[CXGB4_ULD_ISCSIT]; + if (r < uld_rxq_entries[CXGB4_ULD_TLS]) { + const struct sge_ofld_rxq *rx; + + urxq_info = s->uld_rxq_info[CXGB4_ULD_TLS]; + rx = &urxq_info->uldrxq[r * 4]; + n = min(4, urxq_info->nrxq - 4 * r); + + S("QType:", "TLS"); + R("RspQ ID:", rspq.abs_id); + R("RspQ size:", rspq.size); + R("RspQE size:", rspq.iqe_len); + R("RspQ CIDX:", rspq.cidx); + R("RspQ Gen:", rspq.gen); + S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq)); + S3("u", "Intr pktcnt:", s->counter_val[rx[i].rspq.pktcnt_idx]); + R("FL ID:", fl.cntxt_id); + R("FL size:", fl.size - 8); + R("FL pend:", fl.pend_cred); + R("FL avail:", fl.avail); + R("FL PIDX:", fl.pidx); + R("FL CIDX:", fl.cidx); + + goto unlock; + } + + r -= uld_rxq_entries[CXGB4_ULD_TLS]; + if (r < uld_txq_entries[CXGB4_TX_CRYPTO]) { + const struct sge_ofld_rxq *rx; + const struct sge_uld_txq *tx; + + utxq_info = s->uld_txq_info[CXGB4_TX_CRYPTO]; + urxq_info = s->uld_rxq_info[CXGB4_ULD_CRYPTO]; + tx = &utxq_info->uldtxq[r * 4]; + rx = &urxq_info->uldrxq[r * 4]; + n = min(4, utxq_info->ntxq - 4 * r); + + S("QType:", "Crypto"); + T("TxQ ID:", q.cntxt_id); + T("TxQ size:", q.size); + T("TxQ inuse:", q.in_use); + T("TxQ CIDX:", q.cidx); + T("TxQ PIDX:", q.pidx); + R("RspQ ID:", rspq.abs_id); + R("RspQ size:", rspq.size); + R("RspQE size:", rspq.iqe_len); + R("RspQ CIDX:", rspq.cidx); + R("RspQ Gen:", rspq.gen); + S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq)); + S3("u", "Intr pktcnt:", s->counter_val[rx[i].rspq.pktcnt_idx]); + R("FL ID:", fl.cntxt_id); + R("FL size:", fl.size - 8); + R("FL pend:", fl.pend_cred); + R("FL avail:", fl.avail); + R("FL PIDX:", fl.pidx); + R("FL CIDX:", fl.cidx); + + goto unlock; + } + + r -= uld_txq_entries[CXGB4_TX_CRYPTO]; + mutex_unlock(&uld_mutex); + +skip_uld: + if (r < ctrl_entries) { + const struct sge_ctrl_txq *tx = &s->ctrlq[r * 4]; + + n = min(4, adap->params.nports - 4 * r); + + S("QType:", "Control"); + T("TxQ ID:", q.cntxt_id); + T("TxQ size:", q.size); + T("TxQ inuse:", q.in_use); + T("TxQ CIDX:", q.cidx); + T("TxQ PIDX:", q.pidx); + TL("TxQFull:", q.stops); + TL("TxQRestarts:", q.restarts); + + goto out; + } + + r -= ctrl_entries; + if (r < 1) { + const struct sge_rspq *evtq = &s->fw_evtq; + + seq_printf(seq, "%-12s %16s\n", "QType:", "FW event queue"); + seq_printf(seq, "%-12s %16u\n", "RspQ ID:", evtq->abs_id); + seq_printf(seq, "%-12s %16u\n", "RspQ size:", evtq->size); + seq_printf(seq, "%-12s %16u\n", "RspQE size:", evtq->iqe_len); + seq_printf(seq, "%-12s %16u\n", "RspQ CIDX:", evtq->cidx); + seq_printf(seq, "%-12s %16u\n", "RspQ Gen:", evtq->gen); + seq_printf(seq, "%-12s %16u\n", "Intr delay:", + qtimer_val(adap, evtq)); + seq_printf(seq, "%-12s %16u\n", "Intr pktcnt:", + s->counter_val[evtq->pktcnt_idx]); + + goto out; + } + +#undef R +#undef RL +#undef T +#undef TL +#undef S +#undef R3 +#undef T3 +#undef S3 +out: + return 0; + +unlock: + mutex_unlock(&uld_mutex); + return 0; +} + +static int sge_queue_entries(struct adapter *adap) +{ + int i, tot_uld_entries = 0, eohw_entries = 0, eosw_entries = 0; + int mirror_rxq_entries = 0; + + if (adap->tc_mqprio) { + struct cxgb4_tc_port_mqprio *port_mqprio; + u8 tc; + + mutex_lock(&adap->tc_mqprio->mqprio_mutex); + if (adap->sge.eohw_txq) + eohw_entries = DIV_ROUND_UP(adap->sge.eoqsets, 4); + + for (i = 0; i < adap->params.nports; i++) { + u32 entries = 0; + + port_mqprio = &adap->tc_mqprio->port_mqprio[i]; + for (tc = 0; tc < port_mqprio->mqprio.qopt.num_tc; tc++) + entries += port_mqprio->mqprio.qopt.count[tc]; + + if (entries) + eosw_entries += DIV_ROUND_UP(entries, 4); + } + mutex_unlock(&adap->tc_mqprio->mqprio_mutex); + } + + for_each_port(adap, i) { + struct port_info *pi = adap2pinfo(adap, i); + + mutex_lock(&pi->vi_mirror_mutex); + if (pi->vi_mirror_count) + mirror_rxq_entries += DIV_ROUND_UP(pi->nmirrorqsets, 4); + mutex_unlock(&pi->vi_mirror_mutex); + } + + if (!is_uld(adap)) + goto lld_only; + + mutex_lock(&uld_mutex); + for (i = 0; i < CXGB4_TX_MAX; i++) + tot_uld_entries += sge_qinfo_uld_txq_entries(adap, i); + + for (i = 0; i < CXGB4_ULD_MAX; i++) { + tot_uld_entries += sge_qinfo_uld_rxq_entries(adap, i); + tot_uld_entries += sge_qinfo_uld_ciq_entries(adap, i); + } + mutex_unlock(&uld_mutex); + +lld_only: + return DIV_ROUND_UP(adap->sge.ethqsets, 4) + mirror_rxq_entries + + eohw_entries + eosw_entries + tot_uld_entries + + DIV_ROUND_UP(MAX_CTRL_QUEUES, 4) + 1; +} + +static void *sge_queue_start(struct seq_file *seq, loff_t *pos) +{ + int entries = sge_queue_entries(seq->private); + + return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL; +} + +static void sge_queue_stop(struct seq_file *seq, void *v) +{ +} + +static void *sge_queue_next(struct seq_file *seq, void *v, loff_t *pos) +{ + int entries = sge_queue_entries(seq->private); + + ++*pos; + return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL; +} + +static const struct seq_operations sge_qinfo_seq_ops = { + .start = sge_queue_start, + .next = sge_queue_next, + .stop = sge_queue_stop, + .show = sge_qinfo_show +}; + +static int sge_qinfo_open(struct inode *inode, struct file *file) +{ + int res = seq_open(file, &sge_qinfo_seq_ops); + + if (!res) { + struct seq_file *seq = file->private_data; + + seq->private = inode->i_private; + } + return res; +} + +static const struct file_operations sge_qinfo_debugfs_fops = { + .owner = THIS_MODULE, + .open = sge_qinfo_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +int mem_open(struct inode *inode, struct file *file) +{ + unsigned int mem; + struct adapter *adap; + + file->private_data = inode->i_private; + + mem = (uintptr_t)file->private_data & 0x7; + adap = file->private_data - mem; + + (void)t4_fwcache(adap, FW_PARAM_DEV_FWCACHE_FLUSH); + + return 0; +} + +static ssize_t mem_read(struct file *file, char __user *buf, size_t count, + loff_t *ppos) +{ + loff_t pos = *ppos; + loff_t avail = file_inode(file)->i_size; + unsigned int mem = (uintptr_t)file->private_data & 0x7; + struct adapter *adap = file->private_data - mem; + __be32 *data; + int ret; + + if (pos < 0) + return -EINVAL; + if (pos >= avail) + return 0; + if (count > avail - pos) + count = avail - pos; + + data = kvzalloc(count, GFP_KERNEL); + if (!data) + return -ENOMEM; + + spin_lock(&adap->win0_lock); + ret = t4_memory_rw(adap, 0, mem, pos, count, data, T4_MEMORY_READ); + spin_unlock(&adap->win0_lock); + if (ret) { + kvfree(data); + return ret; + } + ret = copy_to_user(buf, data, count); + + kvfree(data); + if (ret) + return -EFAULT; + + *ppos = pos + count; + return count; +} +static const struct file_operations mem_debugfs_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = mem_read, + .llseek = default_llseek, +}; + +static int tid_info_show(struct seq_file *seq, void *v) +{ + struct adapter *adap = seq->private; + const struct tid_info *t; + enum chip_type chip; + + t = &adap->tids; + chip = CHELSIO_CHIP_VERSION(adap->params.chip); + if (t4_read_reg(adap, LE_DB_CONFIG_A) & HASHEN_F) { + unsigned int sb; + seq_printf(seq, "Connections in use: %u\n", + atomic_read(&t->conns_in_use)); + + if (chip <= CHELSIO_T5) + sb = t4_read_reg(adap, LE_DB_SERVER_INDEX_A) / 4; + else + sb = t4_read_reg(adap, LE_DB_SRVR_START_INDEX_A); + + if (sb) { + seq_printf(seq, "TID range: %u..%u/%u..%u", t->tid_base, + sb - 1, adap->tids.hash_base, + t->tid_base + t->ntids - 1); + seq_printf(seq, ", in use: %u/%u\n", + atomic_read(&t->tids_in_use), + atomic_read(&t->hash_tids_in_use)); + } else if (adap->flags & CXGB4_FW_OFLD_CONN) { + seq_printf(seq, "TID range: %u..%u/%u..%u", + t->aftid_base, + t->aftid_end, + adap->tids.hash_base, + t->tid_base + t->ntids - 1); + seq_printf(seq, ", in use: %u/%u\n", + atomic_read(&t->tids_in_use), + atomic_read(&t->hash_tids_in_use)); + } else { + seq_printf(seq, "TID range: %u..%u", + adap->tids.hash_base, + t->tid_base + t->ntids - 1); + seq_printf(seq, ", in use: %u\n", + atomic_read(&t->hash_tids_in_use)); + } + } else if (t->ntids) { + seq_printf(seq, "Connections in use: %u\n", + atomic_read(&t->conns_in_use)); + + seq_printf(seq, "TID range: %u..%u", t->tid_base, + t->tid_base + t->ntids - 1); + seq_printf(seq, ", in use: %u\n", + atomic_read(&t->tids_in_use)); + } + + if (t->nstids) + seq_printf(seq, "STID range: %u..%u, in use-IPv4/IPv6: %u/%u\n", + (!t->stid_base && + (chip <= CHELSIO_T5)) ? + t->stid_base + 1 : t->stid_base, + t->stid_base + t->nstids - 1, + t->stids_in_use - t->v6_stids_in_use, + t->v6_stids_in_use); + + if (t->natids) + seq_printf(seq, "ATID range: 0..%u, in use: %u\n", + t->natids - 1, t->atids_in_use); + seq_printf(seq, "FTID range: %u..%u\n", t->ftid_base, + t->ftid_base + t->nftids - 1); + if (t->nsftids) + seq_printf(seq, "SFTID range: %u..%u in use: %u\n", + t->sftid_base, t->sftid_base + t->nsftids - 2, + t->sftids_in_use); + if (t->nhpftids) + seq_printf(seq, "HPFTID range: %u..%u\n", t->hpftid_base, + t->hpftid_base + t->nhpftids - 1); + if (t->neotids) + seq_printf(seq, "EOTID range: %u..%u, in use: %u\n", + t->eotid_base, t->eotid_base + t->neotids - 1, + atomic_read(&t->eotids_in_use)); + if (t->ntids) + seq_printf(seq, "HW TID usage: %u IP users, %u IPv6 users\n", + t4_read_reg(adap, LE_DB_ACT_CNT_IPV4_A), + t4_read_reg(adap, LE_DB_ACT_CNT_IPV6_A)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(tid_info); + +static void add_debugfs_mem(struct adapter *adap, const char *name, + unsigned int idx, unsigned int size_mb) +{ + debugfs_create_file_size(name, 0400, adap->debugfs_root, + (void *)adap + idx, &mem_debugfs_fops, + size_mb << 20); +} + +static ssize_t blocked_fl_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *ppos) +{ + int len; + const struct adapter *adap = filp->private_data; + char *buf; + ssize_t size = (adap->sge.egr_sz + 3) / 4 + + adap->sge.egr_sz / 32 + 2; /* includes ,/\n/\0 */ + + buf = kzalloc(size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + len = snprintf(buf, size - 1, "%*pb\n", + adap->sge.egr_sz, adap->sge.blocked_fl); + len += sprintf(buf + len, "\n"); + size = simple_read_from_buffer(ubuf, count, ppos, buf, len); + kfree(buf); + return size; +} + +static ssize_t blocked_fl_write(struct file *filp, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + int err; + unsigned long *t; + struct adapter *adap = filp->private_data; + + t = bitmap_zalloc(adap->sge.egr_sz, GFP_KERNEL); + if (!t) + return -ENOMEM; + + err = bitmap_parse_user(ubuf, count, t, adap->sge.egr_sz); + if (err) { + bitmap_free(t); + return err; + } + + bitmap_copy(adap->sge.blocked_fl, t, adap->sge.egr_sz); + bitmap_free(t); + return count; +} + +static const struct file_operations blocked_fl_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = blocked_fl_read, + .write = blocked_fl_write, + .llseek = generic_file_llseek, +}; + +static void mem_region_show(struct seq_file *seq, const char *name, + unsigned int from, unsigned int to) +{ + char buf[40]; + + string_get_size((u64)to - from + 1, 1, STRING_UNITS_2, buf, + sizeof(buf)); + seq_printf(seq, "%-15s %#x-%#x [%s]\n", name, from, to, buf); +} + +static int meminfo_show(struct seq_file *seq, void *v) +{ + static const char * const memory[] = { "EDC0:", "EDC1:", "MC:", + "MC0:", "MC1:", "HMA:"}; + struct adapter *adap = seq->private; + struct cudbg_meminfo meminfo; + int i, rc; + + memset(&meminfo, 0, sizeof(struct cudbg_meminfo)); + rc = cudbg_fill_meminfo(adap, &meminfo); + if (rc) + return -ENXIO; + + for (i = 0; i < meminfo.avail_c; i++) + mem_region_show(seq, memory[meminfo.avail[i].idx], + meminfo.avail[i].base, + meminfo.avail[i].limit - 1); + + seq_putc(seq, '\n'); + for (i = 0; i < meminfo.mem_c; i++) { + if (meminfo.mem[i].idx >= ARRAY_SIZE(cudbg_region)) + continue; /* skip holes */ + if (!meminfo.mem[i].limit) + meminfo.mem[i].limit = + i < meminfo.mem_c - 1 ? + meminfo.mem[i + 1].base - 1 : ~0; + mem_region_show(seq, cudbg_region[meminfo.mem[i].idx], + meminfo.mem[i].base, meminfo.mem[i].limit); + } + + seq_putc(seq, '\n'); + mem_region_show(seq, "uP RAM:", meminfo.up_ram_lo, meminfo.up_ram_hi); + mem_region_show(seq, "uP Extmem2:", meminfo.up_extmem2_lo, + meminfo.up_extmem2_hi); + + seq_printf(seq, "\n%u Rx pages (%u free) of size %uKiB for %u channels\n", + meminfo.rx_pages_data[0], meminfo.free_rx_cnt, + meminfo.rx_pages_data[1], meminfo.rx_pages_data[2]); + + seq_printf(seq, "%u Tx pages (%u free) of size %u%ciB for %u channels\n", + meminfo.tx_pages_data[0], meminfo.free_tx_cnt, + meminfo.tx_pages_data[1], meminfo.tx_pages_data[2], + meminfo.tx_pages_data[3]); + + seq_printf(seq, "%u p-structs (%u free)\n\n", + meminfo.p_structs, meminfo.p_structs_free_cnt); + + for (i = 0; i < 4; i++) + /* For T6 these are MAC buffer groups */ + seq_printf(seq, "Port %d using %u pages out of %u allocated\n", + i, meminfo.port_used[i], meminfo.port_alloc[i]); + + for (i = 0; i < adap->params.arch.nchan; i++) + /* For T6 these are MAC buffer groups */ + seq_printf(seq, + "Loopback %d using %u pages out of %u allocated\n", + i, meminfo.loopback_used[i], + meminfo.loopback_alloc[i]); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(meminfo); + +static int chcr_stats_show(struct seq_file *seq, void *v) +{ +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + struct ch_ktls_port_stats_debug *ktls_port; + int i = 0; +#endif + struct adapter *adap = seq->private; + + seq_puts(seq, "Chelsio Crypto Accelerator Stats \n"); + seq_printf(seq, "Cipher Ops: %10u \n", + atomic_read(&adap->chcr_stats.cipher_rqst)); + seq_printf(seq, "Digest Ops: %10u \n", + atomic_read(&adap->chcr_stats.digest_rqst)); + seq_printf(seq, "Aead Ops: %10u \n", + atomic_read(&adap->chcr_stats.aead_rqst)); + seq_printf(seq, "Completion: %10u \n", + atomic_read(&adap->chcr_stats.complete)); + seq_printf(seq, "Error: %10u \n", + atomic_read(&adap->chcr_stats.error)); + seq_printf(seq, "Fallback: %10u \n", + atomic_read(&adap->chcr_stats.fallback)); + seq_printf(seq, "TLS PDU Tx: %10u\n", + atomic_read(&adap->chcr_stats.tls_pdu_tx)); + seq_printf(seq, "TLS PDU Rx: %10u\n", + atomic_read(&adap->chcr_stats.tls_pdu_rx)); + seq_printf(seq, "TLS Keys (DDR) Count: %10u\n", + atomic_read(&adap->chcr_stats.tls_key)); +#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) + seq_puts(seq, "\nChelsio Inline IPsec Crypto Accelerator Stats\n"); + seq_printf(seq, "IPSec PDU: %10u\n", + atomic_read(&adap->ch_ipsec_stats.ipsec_cnt)); +#endif +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + seq_puts(seq, "\nChelsio KTLS Crypto Accelerator Stats\n"); + seq_printf(seq, "Tx TLS offload refcount: %20u\n", + refcount_read(&adap->chcr_ktls.ktls_refcount)); + seq_printf(seq, "Tx records send: %20llu\n", + atomic64_read(&adap->ch_ktls_stats.ktls_tx_send_records)); + seq_printf(seq, "Tx partial start of records: %20llu\n", + atomic64_read(&adap->ch_ktls_stats.ktls_tx_start_pkts)); + seq_printf(seq, "Tx partial middle of records: %20llu\n", + atomic64_read(&adap->ch_ktls_stats.ktls_tx_middle_pkts)); + seq_printf(seq, "Tx partial end of record: %20llu\n", + atomic64_read(&adap->ch_ktls_stats.ktls_tx_end_pkts)); + seq_printf(seq, "Tx complete records: %20llu\n", + atomic64_read(&adap->ch_ktls_stats.ktls_tx_complete_pkts)); + seq_printf(seq, "TX trim pkts : %20llu\n", + atomic64_read(&adap->ch_ktls_stats.ktls_tx_trimmed_pkts)); + seq_printf(seq, "TX sw fallback : %20llu\n", + atomic64_read(&adap->ch_ktls_stats.ktls_tx_fallback)); + while (i < MAX_NPORTS) { + ktls_port = &adap->ch_ktls_stats.ktls_port[i]; + seq_printf(seq, "Port %d\n", i); + seq_printf(seq, "Tx connection created: %20llu\n", + atomic64_read(&ktls_port->ktls_tx_connection_open)); + seq_printf(seq, "Tx connection failed: %20llu\n", + atomic64_read(&ktls_port->ktls_tx_connection_fail)); + seq_printf(seq, "Tx connection closed: %20llu\n", + atomic64_read(&ktls_port->ktls_tx_connection_close)); + i++; + } +#endif + return 0; +} +DEFINE_SHOW_ATTRIBUTE(chcr_stats); + +#define PRINT_ADAP_STATS(string, value) \ + seq_printf(seq, "%-25s %-20llu\n", (string), \ + (unsigned long long)(value)) + +#define PRINT_CH_STATS(string, value) \ +do { \ + seq_printf(seq, "%-25s ", (string)); \ + for (i = 0; i < adap->params.arch.nchan; i++) \ + seq_printf(seq, "%-20llu ", \ + (unsigned long long)stats.value[i]); \ + seq_printf(seq, "\n"); \ +} while (0) + +#define PRINT_CH_STATS2(string, value) \ +do { \ + seq_printf(seq, "%-25s ", (string)); \ + for (i = 0; i < adap->params.arch.nchan; i++) \ + seq_printf(seq, "%-20llu ", \ + (unsigned long long)stats[i].value); \ + seq_printf(seq, "\n"); \ +} while (0) + +static void show_tcp_stats(struct seq_file *seq) +{ + struct adapter *adap = seq->private; + struct tp_tcp_stats v4, v6; + + spin_lock(&adap->stats_lock); + t4_tp_get_tcp_stats(adap, &v4, &v6, false); + spin_unlock(&adap->stats_lock); + + PRINT_ADAP_STATS("tcp_ipv4_out_rsts:", v4.tcp_out_rsts); + PRINT_ADAP_STATS("tcp_ipv4_in_segs:", v4.tcp_in_segs); + PRINT_ADAP_STATS("tcp_ipv4_out_segs:", v4.tcp_out_segs); + PRINT_ADAP_STATS("tcp_ipv4_retrans_segs:", v4.tcp_retrans_segs); + PRINT_ADAP_STATS("tcp_ipv6_out_rsts:", v6.tcp_out_rsts); + PRINT_ADAP_STATS("tcp_ipv6_in_segs:", v6.tcp_in_segs); + PRINT_ADAP_STATS("tcp_ipv6_out_segs:", v6.tcp_out_segs); + PRINT_ADAP_STATS("tcp_ipv6_retrans_segs:", v6.tcp_retrans_segs); +} + +static void show_ddp_stats(struct seq_file *seq) +{ + struct adapter *adap = seq->private; + struct tp_usm_stats stats; + + spin_lock(&adap->stats_lock); + t4_get_usm_stats(adap, &stats, false); + spin_unlock(&adap->stats_lock); + + PRINT_ADAP_STATS("usm_ddp_frames:", stats.frames); + PRINT_ADAP_STATS("usm_ddp_octets:", stats.octets); + PRINT_ADAP_STATS("usm_ddp_drops:", stats.drops); +} + +static void show_rdma_stats(struct seq_file *seq) +{ + struct adapter *adap = seq->private; + struct tp_rdma_stats stats; + + spin_lock(&adap->stats_lock); + t4_tp_get_rdma_stats(adap, &stats, false); + spin_unlock(&adap->stats_lock); + + PRINT_ADAP_STATS("rdma_no_rqe_mod_defer:", stats.rqe_dfr_mod); + PRINT_ADAP_STATS("rdma_no_rqe_pkt_defer:", stats.rqe_dfr_pkt); +} + +static void show_tp_err_adapter_stats(struct seq_file *seq) +{ + struct adapter *adap = seq->private; + struct tp_err_stats stats; + + spin_lock(&adap->stats_lock); + t4_tp_get_err_stats(adap, &stats, false); + spin_unlock(&adap->stats_lock); + + PRINT_ADAP_STATS("tp_err_ofld_no_neigh:", stats.ofld_no_neigh); + PRINT_ADAP_STATS("tp_err_ofld_cong_defer:", stats.ofld_cong_defer); +} + +static void show_cpl_stats(struct seq_file *seq) +{ + struct adapter *adap = seq->private; + struct tp_cpl_stats stats; + u8 i; + + spin_lock(&adap->stats_lock); + t4_tp_get_cpl_stats(adap, &stats, false); + spin_unlock(&adap->stats_lock); + + PRINT_CH_STATS("tp_cpl_requests:", req); + PRINT_CH_STATS("tp_cpl_responses:", rsp); +} + +static void show_tp_err_channel_stats(struct seq_file *seq) +{ + struct adapter *adap = seq->private; + struct tp_err_stats stats; + u8 i; + + spin_lock(&adap->stats_lock); + t4_tp_get_err_stats(adap, &stats, false); + spin_unlock(&adap->stats_lock); + + PRINT_CH_STATS("tp_mac_in_errs:", mac_in_errs); + PRINT_CH_STATS("tp_hdr_in_errs:", hdr_in_errs); + PRINT_CH_STATS("tp_tcp_in_errs:", tcp_in_errs); + PRINT_CH_STATS("tp_tcp6_in_errs:", tcp6_in_errs); + PRINT_CH_STATS("tp_tnl_cong_drops:", tnl_cong_drops); + PRINT_CH_STATS("tp_tnl_tx_drops:", tnl_tx_drops); + PRINT_CH_STATS("tp_ofld_vlan_drops:", ofld_vlan_drops); + PRINT_CH_STATS("tp_ofld_chan_drops:", ofld_chan_drops); +} + +static void show_fcoe_stats(struct seq_file *seq) +{ + struct adapter *adap = seq->private; + struct tp_fcoe_stats stats[NCHAN]; + u8 i; + + spin_lock(&adap->stats_lock); + for (i = 0; i < adap->params.arch.nchan; i++) + t4_get_fcoe_stats(adap, i, &stats[i], false); + spin_unlock(&adap->stats_lock); + + PRINT_CH_STATS2("fcoe_octets_ddp", octets_ddp); + PRINT_CH_STATS2("fcoe_frames_ddp", frames_ddp); + PRINT_CH_STATS2("fcoe_frames_drop", frames_drop); +} + +#undef PRINT_CH_STATS2 +#undef PRINT_CH_STATS +#undef PRINT_ADAP_STATS + +static int tp_stats_show(struct seq_file *seq, void *v) +{ + struct adapter *adap = seq->private; + + seq_puts(seq, "\n--------Adapter Stats--------\n"); + show_tcp_stats(seq); + show_ddp_stats(seq); + show_rdma_stats(seq); + show_tp_err_adapter_stats(seq); + + seq_puts(seq, "\n-------- Channel Stats --------\n"); + if (adap->params.arch.nchan == NCHAN) + seq_printf(seq, "%-25s %-20s %-20s %-20s %-20s\n", + " ", "channel 0", "channel 1", + "channel 2", "channel 3"); + else + seq_printf(seq, "%-25s %-20s %-20s\n", + " ", "channel 0", "channel 1"); + show_cpl_stats(seq); + show_tp_err_channel_stats(seq); + show_fcoe_stats(seq); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(tp_stats); + +/* Add an array of Debug FS files. + */ +void add_debugfs_files(struct adapter *adap, + struct t4_debugfs_entry *files, + unsigned int nfiles) +{ + int i; + + /* debugfs support is best effort */ + for (i = 0; i < nfiles; i++) + debugfs_create_file(files[i].name, files[i].mode, + adap->debugfs_root, + (void *)adap + files[i].data, + files[i].ops); +} + +int t4_setup_debugfs(struct adapter *adap) +{ + int i; + u32 size = 0; + + static struct t4_debugfs_entry t4_debugfs_files[] = { + { "cim_la", &cim_la_fops, 0400, 0 }, + { "cim_pif_la", &cim_pif_la_fops, 0400, 0 }, + { "cim_ma_la", &cim_ma_la_fops, 0400, 0 }, + { "cim_qcfg", &cim_qcfg_fops, 0400, 0 }, + { "clk", &clk_fops, 0400, 0 }, + { "devlog", &devlog_fops, 0400, 0 }, + { "mboxlog", &mboxlog_fops, 0400, 0 }, + { "mbox0", &mbox_debugfs_fops, 0600, 0 }, + { "mbox1", &mbox_debugfs_fops, 0600, 1 }, + { "mbox2", &mbox_debugfs_fops, 0600, 2 }, + { "mbox3", &mbox_debugfs_fops, 0600, 3 }, + { "mbox4", &mbox_debugfs_fops, 0600, 4 }, + { "mbox5", &mbox_debugfs_fops, 0600, 5 }, + { "mbox6", &mbox_debugfs_fops, 0600, 6 }, + { "mbox7", &mbox_debugfs_fops, 0600, 7 }, + { "trace0", &mps_trc_debugfs_fops, 0600, 0 }, + { "trace1", &mps_trc_debugfs_fops, 0600, 1 }, + { "trace2", &mps_trc_debugfs_fops, 0600, 2 }, + { "trace3", &mps_trc_debugfs_fops, 0600, 3 }, + { "l2t", &t4_l2t_fops, 0400, 0}, + { "mps_tcam", &mps_tcam_debugfs_fops, 0400, 0 }, + { "rss", &rss_debugfs_fops, 0400, 0 }, + { "rss_config", &rss_config_fops, 0400, 0 }, + { "rss_key", &rss_key_debugfs_fops, 0400, 0 }, + { "rss_pf_config", &rss_pf_config_debugfs_fops, 0400, 0 }, + { "rss_vf_config", &rss_vf_config_debugfs_fops, 0400, 0 }, + { "resources", &resources_fops, 0400, 0 }, +#ifdef CONFIG_CHELSIO_T4_DCB + { "dcb_info", &dcb_info_debugfs_fops, 0400, 0 }, +#endif + { "sge_qinfo", &sge_qinfo_debugfs_fops, 0400, 0 }, + { "ibq_tp0", &cim_ibq_fops, 0400, 0 }, + { "ibq_tp1", &cim_ibq_fops, 0400, 1 }, + { "ibq_ulp", &cim_ibq_fops, 0400, 2 }, + { "ibq_sge0", &cim_ibq_fops, 0400, 3 }, + { "ibq_sge1", &cim_ibq_fops, 0400, 4 }, + { "ibq_ncsi", &cim_ibq_fops, 0400, 5 }, + { "obq_ulp0", &cim_obq_fops, 0400, 0 }, + { "obq_ulp1", &cim_obq_fops, 0400, 1 }, + { "obq_ulp2", &cim_obq_fops, 0400, 2 }, + { "obq_ulp3", &cim_obq_fops, 0400, 3 }, + { "obq_sge", &cim_obq_fops, 0400, 4 }, + { "obq_ncsi", &cim_obq_fops, 0400, 5 }, + { "tp_la", &tp_la_fops, 0400, 0 }, + { "ulprx_la", &ulprx_la_fops, 0400, 0 }, + { "sensors", &sensors_fops, 0400, 0 }, + { "pm_stats", &pm_stats_debugfs_fops, 0400, 0 }, + { "tx_rate", &tx_rate_fops, 0400, 0 }, + { "cctrl", &cctrl_tbl_fops, 0400, 0 }, +#if IS_ENABLED(CONFIG_IPV6) + { "clip_tbl", &clip_tbl_fops, 0400, 0 }, +#endif + { "tids", &tid_info_fops, 0400, 0}, + { "blocked_fl", &blocked_fl_fops, 0600, 0 }, + { "meminfo", &meminfo_fops, 0400, 0 }, + { "crypto", &chcr_stats_fops, 0400, 0 }, + { "tp_stats", &tp_stats_fops, 0400, 0 }, + }; + + /* Debug FS nodes common to all T5 and later adapters. + */ + static struct t4_debugfs_entry t5_debugfs_files[] = { + { "obq_sge_rx_q0", &cim_obq_fops, 0400, 6 }, + { "obq_sge_rx_q1", &cim_obq_fops, 0400, 7 }, + }; + + add_debugfs_files(adap, + t4_debugfs_files, + ARRAY_SIZE(t4_debugfs_files)); + if (!is_t4(adap->params.chip)) + add_debugfs_files(adap, + t5_debugfs_files, + ARRAY_SIZE(t5_debugfs_files)); + + i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A); + if (i & EDRAM0_ENABLE_F) { + size = t4_read_reg(adap, MA_EDRAM0_BAR_A); + add_debugfs_mem(adap, "edc0", MEM_EDC0, EDRAM0_SIZE_G(size)); + } + if (i & EDRAM1_ENABLE_F) { + size = t4_read_reg(adap, MA_EDRAM1_BAR_A); + add_debugfs_mem(adap, "edc1", MEM_EDC1, EDRAM1_SIZE_G(size)); + } + if (is_t5(adap->params.chip)) { + if (i & EXT_MEM0_ENABLE_F) { + size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A); + add_debugfs_mem(adap, "mc0", MEM_MC0, + EXT_MEM0_SIZE_G(size)); + } + if (i & EXT_MEM1_ENABLE_F) { + size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); + add_debugfs_mem(adap, "mc1", MEM_MC1, + EXT_MEM1_SIZE_G(size)); + } + } else { + if (i & EXT_MEM_ENABLE_F) { + size = t4_read_reg(adap, MA_EXT_MEMORY_BAR_A); + add_debugfs_mem(adap, "mc", MEM_MC, + EXT_MEM_SIZE_G(size)); + } + + if (i & HMA_MUX_F) { + size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); + add_debugfs_mem(adap, "hma", MEM_HMA, + EXT_MEM1_SIZE_G(size)); + } + } + + debugfs_create_file_size("flash", 0400, adap->debugfs_root, adap, + &flash_debugfs_fops, adap->params.sf_size); + debugfs_create_bool("use_backdoor", 0600, + adap->debugfs_root, &adap->use_bd); + debugfs_create_bool("trace_rss", 0600, + adap->debugfs_root, &adap->trace_rss); + + return 0; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.h new file mode 100644 index 000000000..1471cf0de --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.h @@ -0,0 +1,70 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_DEBUGFS_H +#define __CXGB4_DEBUGFS_H + +#include <linux/export.h> + +struct t4_debugfs_entry { + const char *name; + const struct file_operations *ops; + umode_t mode; + unsigned char data; +}; + +struct seq_tab { + int (*show)(struct seq_file *seq, void *v, int idx); + unsigned int rows; /* # of entries */ + unsigned char width; /* size in bytes of each entry */ + unsigned char skip_first; /* whether the first line is a header */ + char data[]; /* the table data */ +}; + +static inline unsigned int hex2val(char c) +{ + return isdigit(c) ? c - '0' : tolower(c) - 'a' + 10; +} + +struct seq_tab *seq_open_tab(struct file *f, unsigned int rows, + unsigned int width, unsigned int have_header, + int (*show)(struct seq_file *seq, void *v, int i)); + +int t4_setup_debugfs(struct adapter *adap); +void add_debugfs_files(struct adapter *adap, + struct t4_debugfs_entry *files, + unsigned int nfiles); +int mem_open(struct inode *inode, struct file *file); + +#endif diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c new file mode 100644 index 000000000..8477a93ce --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c @@ -0,0 +1,2299 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2013-2015 Chelsio Communications. All rights reserved. + */ + +#include <linux/firmware.h> +#include <linux/mdio.h> + +#include "cxgb4.h" +#include "t4_regs.h" +#include "t4fw_api.h" +#include "cxgb4_cudbg.h" +#include "cxgb4_filter.h" +#include "cxgb4_tc_flower.h" + +#define EEPROM_MAGIC 0x38E2F10C + +static u32 get_msglevel(struct net_device *dev) +{ + return netdev2adap(dev)->msg_enable; +} + +static void set_msglevel(struct net_device *dev, u32 val) +{ + netdev2adap(dev)->msg_enable = val; +} + +enum cxgb4_ethtool_tests { + CXGB4_ETHTOOL_LB_TEST, + CXGB4_ETHTOOL_MAX_TEST, +}; + +static const char cxgb4_selftest_strings[CXGB4_ETHTOOL_MAX_TEST][ETH_GSTRING_LEN] = { + "Loop back test (offline)", +}; + +static const char * const flash_region_strings[] = { + "All", + "Firmware", + "PHY Firmware", + "Boot", + "Boot CFG", +}; + +static const char stats_strings[][ETH_GSTRING_LEN] = { + "tx_octets_ok ", + "tx_frames_ok ", + "tx_broadcast_frames ", + "tx_multicast_frames ", + "tx_unicast_frames ", + "tx_error_frames ", + + "tx_frames_64 ", + "tx_frames_65_to_127 ", + "tx_frames_128_to_255 ", + "tx_frames_256_to_511 ", + "tx_frames_512_to_1023 ", + "tx_frames_1024_to_1518 ", + "tx_frames_1519_to_max ", + + "tx_frames_dropped ", + "tx_pause_frames ", + "tx_ppp0_frames ", + "tx_ppp1_frames ", + "tx_ppp2_frames ", + "tx_ppp3_frames ", + "tx_ppp4_frames ", + "tx_ppp5_frames ", + "tx_ppp6_frames ", + "tx_ppp7_frames ", + + "rx_octets_ok ", + "rx_frames_ok ", + "rx_broadcast_frames ", + "rx_multicast_frames ", + "rx_unicast_frames ", + + "rx_frames_too_long ", + "rx_jabber_errors ", + "rx_fcs_errors ", + "rx_length_errors ", + "rx_symbol_errors ", + "rx_runt_frames ", + + "rx_frames_64 ", + "rx_frames_65_to_127 ", + "rx_frames_128_to_255 ", + "rx_frames_256_to_511 ", + "rx_frames_512_to_1023 ", + "rx_frames_1024_to_1518 ", + "rx_frames_1519_to_max ", + + "rx_pause_frames ", + "rx_ppp0_frames ", + "rx_ppp1_frames ", + "rx_ppp2_frames ", + "rx_ppp3_frames ", + "rx_ppp4_frames ", + "rx_ppp5_frames ", + "rx_ppp6_frames ", + "rx_ppp7_frames ", + + "rx_bg0_frames_dropped ", + "rx_bg1_frames_dropped ", + "rx_bg2_frames_dropped ", + "rx_bg3_frames_dropped ", + "rx_bg0_frames_trunc ", + "rx_bg1_frames_trunc ", + "rx_bg2_frames_trunc ", + "rx_bg3_frames_trunc ", + + "tso ", + "uso ", + "tx_csum_offload ", + "rx_csum_good ", + "vlan_extractions ", + "vlan_insertions ", + "gro_packets ", + "gro_merged ", +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + "tx_tls_encrypted_packets", + "tx_tls_encrypted_bytes ", + "tx_tls_ctx ", + "tx_tls_ooo ", + "tx_tls_skip_no_sync_data", + "tx_tls_drop_no_sync_data", + "tx_tls_drop_bypass_req ", +#endif +}; + +static char adapter_stats_strings[][ETH_GSTRING_LEN] = { + "db_drop ", + "db_full ", + "db_empty ", + "write_coal_success ", + "write_coal_fail ", +}; + +static char loopback_stats_strings[][ETH_GSTRING_LEN] = { + "-------Loopback----------- ", + "octets_ok ", + "frames_ok ", + "bcast_frames ", + "mcast_frames ", + "ucast_frames ", + "error_frames ", + "frames_64 ", + "frames_65_to_127 ", + "frames_128_to_255 ", + "frames_256_to_511 ", + "frames_512_to_1023 ", + "frames_1024_to_1518 ", + "frames_1519_to_max ", + "frames_dropped ", + "bg0_frames_dropped ", + "bg1_frames_dropped ", + "bg2_frames_dropped ", + "bg3_frames_dropped ", + "bg0_frames_trunc ", + "bg1_frames_trunc ", + "bg2_frames_trunc ", + "bg3_frames_trunc ", +}; + +static const char cxgb4_priv_flags_strings[][ETH_GSTRING_LEN] = { + [PRIV_FLAG_PORT_TX_VM_BIT] = "port_tx_vm_wr", +}; + +static int get_sset_count(struct net_device *dev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return ARRAY_SIZE(stats_strings) + + ARRAY_SIZE(adapter_stats_strings) + + ARRAY_SIZE(loopback_stats_strings); + case ETH_SS_PRIV_FLAGS: + return ARRAY_SIZE(cxgb4_priv_flags_strings); + case ETH_SS_TEST: + return ARRAY_SIZE(cxgb4_selftest_strings); + default: + return -EOPNOTSUPP; + } +} + +static int get_regs_len(struct net_device *dev) +{ + struct adapter *adap = netdev2adap(dev); + + return t4_get_regs_len(adap); +} + +static int get_eeprom_len(struct net_device *dev) +{ + return EEPROMSIZE; +} + +static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) +{ + struct adapter *adapter = netdev2adap(dev); + u32 exprom_vers; + + strscpy(info->driver, cxgb4_driver_name, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(adapter->pdev), + sizeof(info->bus_info)); + info->regdump_len = get_regs_len(dev); + + if (adapter->params.fw_vers) + snprintf(info->fw_version, sizeof(info->fw_version), + "%u.%u.%u.%u, TP %u.%u.%u.%u", + FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers), + FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers), + FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers), + FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers), + FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers), + FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers), + FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers), + FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers)); + + if (!t4_get_exprom_version(adapter, &exprom_vers)) + snprintf(info->erom_version, sizeof(info->erom_version), + "%u.%u.%u.%u", + FW_HDR_FW_VER_MAJOR_G(exprom_vers), + FW_HDR_FW_VER_MINOR_G(exprom_vers), + FW_HDR_FW_VER_MICRO_G(exprom_vers), + FW_HDR_FW_VER_BUILD_G(exprom_vers)); + info->n_priv_flags = ARRAY_SIZE(cxgb4_priv_flags_strings); +} + +static void get_strings(struct net_device *dev, u32 stringset, u8 *data) +{ + if (stringset == ETH_SS_STATS) { + memcpy(data, stats_strings, sizeof(stats_strings)); + data += sizeof(stats_strings); + memcpy(data, adapter_stats_strings, + sizeof(adapter_stats_strings)); + data += sizeof(adapter_stats_strings); + memcpy(data, loopback_stats_strings, + sizeof(loopback_stats_strings)); + } else if (stringset == ETH_SS_PRIV_FLAGS) { + memcpy(data, cxgb4_priv_flags_strings, + sizeof(cxgb4_priv_flags_strings)); + } else if (stringset == ETH_SS_TEST) { + memcpy(data, cxgb4_selftest_strings, + sizeof(cxgb4_selftest_strings)); + } +} + +/* port stats maintained per queue of the port. They should be in the same + * order as in stats_strings above. + */ +struct queue_port_stats { + u64 tso; + u64 uso; + u64 tx_csum; + u64 rx_csum; + u64 vlan_ex; + u64 vlan_ins; + u64 gro_pkts; + u64 gro_merged; +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + u64 tx_tls_encrypted_packets; + u64 tx_tls_encrypted_bytes; + u64 tx_tls_ctx; + u64 tx_tls_ooo; + u64 tx_tls_skip_no_sync_data; + u64 tx_tls_drop_no_sync_data; + u64 tx_tls_drop_bypass_req; +#endif +}; + +struct adapter_stats { + u64 db_drop; + u64 db_full; + u64 db_empty; + u64 wc_success; + u64 wc_fail; +}; + +static void collect_sge_port_stats(const struct adapter *adap, + const struct port_info *p, + struct queue_port_stats *s) +{ + const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset]; + const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset]; +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + const struct ch_ktls_port_stats_debug *ktls_stats; +#endif + struct sge_eohw_txq *eohw_tx; + unsigned int i; + + memset(s, 0, sizeof(*s)); + for (i = 0; i < p->nqsets; i++, rx++, tx++) { + s->tso += tx->tso; + s->uso += tx->uso; + s->tx_csum += tx->tx_cso; + s->rx_csum += rx->stats.rx_cso; + s->vlan_ex += rx->stats.vlan_ex; + s->vlan_ins += tx->vlan_ins; + s->gro_pkts += rx->stats.lro_pkts; + s->gro_merged += rx->stats.lro_merged; + } + + if (adap->sge.eohw_txq) { + eohw_tx = &adap->sge.eohw_txq[p->first_qset]; + for (i = 0; i < p->nqsets; i++, eohw_tx++) { + s->tso += eohw_tx->tso; + s->uso += eohw_tx->uso; + s->tx_csum += eohw_tx->tx_cso; + s->vlan_ins += eohw_tx->vlan_ins; + } + } +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + ktls_stats = &adap->ch_ktls_stats.ktls_port[p->port_id]; + s->tx_tls_encrypted_packets = + atomic64_read(&ktls_stats->ktls_tx_encrypted_packets); + s->tx_tls_encrypted_bytes = + atomic64_read(&ktls_stats->ktls_tx_encrypted_bytes); + s->tx_tls_ctx = atomic64_read(&ktls_stats->ktls_tx_ctx); + s->tx_tls_ooo = atomic64_read(&ktls_stats->ktls_tx_ooo); + s->tx_tls_skip_no_sync_data = + atomic64_read(&ktls_stats->ktls_tx_skip_no_sync_data); + s->tx_tls_drop_no_sync_data = + atomic64_read(&ktls_stats->ktls_tx_drop_no_sync_data); + s->tx_tls_drop_bypass_req = + atomic64_read(&ktls_stats->ktls_tx_drop_bypass_req); +#endif +} + +static void collect_adapter_stats(struct adapter *adap, struct adapter_stats *s) +{ + u64 val1, val2; + + memset(s, 0, sizeof(*s)); + + s->db_drop = adap->db_stats.db_drop; + s->db_full = adap->db_stats.db_full; + s->db_empty = adap->db_stats.db_empty; + + if (!is_t4(adap->params.chip)) { + int v; + + v = t4_read_reg(adap, SGE_STAT_CFG_A); + if (STATSOURCE_T5_G(v) == 7) { + val2 = t4_read_reg(adap, SGE_STAT_MATCH_A); + val1 = t4_read_reg(adap, SGE_STAT_TOTAL_A); + s->wc_success = val1 - val2; + s->wc_fail = val2; + } + } +} + +static void get_stats(struct net_device *dev, struct ethtool_stats *stats, + u64 *data) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + struct lb_port_stats s; + int i; + u64 *p0; + + t4_get_port_stats_offset(adapter, pi->tx_chan, + (struct port_stats *)data, + &pi->stats_base); + + data += sizeof(struct port_stats) / sizeof(u64); + collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data); + data += sizeof(struct queue_port_stats) / sizeof(u64); + collect_adapter_stats(adapter, (struct adapter_stats *)data); + data += sizeof(struct adapter_stats) / sizeof(u64); + + *data++ = (u64)pi->port_id; + memset(&s, 0, sizeof(s)); + t4_get_lb_stats(adapter, pi->port_id, &s); + + p0 = &s.octets; + for (i = 0; i < ARRAY_SIZE(loopback_stats_strings) - 1; i++) + *data++ = (unsigned long long)*p0++; +} + +static void get_regs(struct net_device *dev, struct ethtool_regs *regs, + void *buf) +{ + struct adapter *adap = netdev2adap(dev); + size_t buf_size; + + buf_size = t4_get_regs_len(adap); + regs->version = mk_adap_vers(adap); + t4_get_regs(adap, buf, buf_size); +} + +static int restart_autoneg(struct net_device *dev) +{ + struct port_info *p = netdev_priv(dev); + + if (!netif_running(dev)) + return -EAGAIN; + if (p->link_cfg.autoneg != AUTONEG_ENABLE) + return -EINVAL; + t4_restart_aneg(p->adapter, p->adapter->pf, p->tx_chan); + return 0; +} + +static int identify_port(struct net_device *dev, + enum ethtool_phys_id_state state) +{ + unsigned int val; + struct adapter *adap = netdev2adap(dev); + + if (state == ETHTOOL_ID_ACTIVE) + val = 0xffff; + else if (state == ETHTOOL_ID_INACTIVE) + val = 0; + else + return -EINVAL; + + return t4_identify_port(adap, adap->pf, netdev2pinfo(dev)->viid, val); +} + +/** + * from_fw_port_mod_type - translate Firmware Port/Module type to Ethtool + * @port_type: Firmware Port Type + * @mod_type: Firmware Module Type + * + * Translate Firmware Port/Module type to Ethtool Port Type. + */ +static int from_fw_port_mod_type(enum fw_port_type port_type, + enum fw_port_module_type mod_type) +{ + if (port_type == FW_PORT_TYPE_BT_SGMII || + port_type == FW_PORT_TYPE_BT_XFI || + port_type == FW_PORT_TYPE_BT_XAUI) { + return PORT_TP; + } else if (port_type == FW_PORT_TYPE_FIBER_XFI || + port_type == FW_PORT_TYPE_FIBER_XAUI) { + return PORT_FIBRE; + } else if (port_type == FW_PORT_TYPE_SFP || + port_type == FW_PORT_TYPE_QSFP_10G || + port_type == FW_PORT_TYPE_QSA || + port_type == FW_PORT_TYPE_QSFP || + port_type == FW_PORT_TYPE_CR4_QSFP || + port_type == FW_PORT_TYPE_CR_QSFP || + port_type == FW_PORT_TYPE_CR2_QSFP || + port_type == FW_PORT_TYPE_SFP28) { + if (mod_type == FW_PORT_MOD_TYPE_LR || + mod_type == FW_PORT_MOD_TYPE_SR || + mod_type == FW_PORT_MOD_TYPE_ER || + mod_type == FW_PORT_MOD_TYPE_LRM) + return PORT_FIBRE; + else if (mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE || + mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE) + return PORT_DA; + else + return PORT_OTHER; + } else if (port_type == FW_PORT_TYPE_KR4_100G || + port_type == FW_PORT_TYPE_KR_SFP28 || + port_type == FW_PORT_TYPE_KR_XLAUI) { + return PORT_NONE; + } + + return PORT_OTHER; +} + +/** + * speed_to_fw_caps - translate Port Speed to Firmware Port Capabilities + * @speed: speed in Kb/s + * + * Translates a specific Port Speed into a Firmware Port Capabilities + * value. + */ +static unsigned int speed_to_fw_caps(int speed) +{ + if (speed == 100) + return FW_PORT_CAP32_SPEED_100M; + if (speed == 1000) + return FW_PORT_CAP32_SPEED_1G; + if (speed == 10000) + return FW_PORT_CAP32_SPEED_10G; + if (speed == 25000) + return FW_PORT_CAP32_SPEED_25G; + if (speed == 40000) + return FW_PORT_CAP32_SPEED_40G; + if (speed == 50000) + return FW_PORT_CAP32_SPEED_50G; + if (speed == 100000) + return FW_PORT_CAP32_SPEED_100G; + if (speed == 200000) + return FW_PORT_CAP32_SPEED_200G; + if (speed == 400000) + return FW_PORT_CAP32_SPEED_400G; + return 0; +} + +/** + * fw_caps_to_lmm - translate Firmware to ethtool Link Mode Mask + * @port_type: Firmware Port Type + * @fw_caps: Firmware Port Capabilities + * @link_mode_mask: ethtool Link Mode Mask + * + * Translate a Firmware Port Capabilities specification to an ethtool + * Link Mode Mask. + */ +static void fw_caps_to_lmm(enum fw_port_type port_type, + fw_port_cap32_t fw_caps, + unsigned long *link_mode_mask) +{ + #define SET_LMM(__lmm_name) \ + do { \ + __set_bit(ETHTOOL_LINK_MODE_ ## __lmm_name ## _BIT, \ + link_mode_mask); \ + } while (0) + + #define FW_CAPS_TO_LMM(__fw_name, __lmm_name) \ + do { \ + if (fw_caps & FW_PORT_CAP32_ ## __fw_name) \ + SET_LMM(__lmm_name); \ + } while (0) + + switch (port_type) { + case FW_PORT_TYPE_BT_SGMII: + case FW_PORT_TYPE_BT_XFI: + case FW_PORT_TYPE_BT_XAUI: + SET_LMM(TP); + FW_CAPS_TO_LMM(SPEED_100M, 100baseT_Full); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full); + break; + + case FW_PORT_TYPE_KX4: + case FW_PORT_TYPE_KX: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKX4_Full); + break; + + case FW_PORT_TYPE_KR: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + break; + + case FW_PORT_TYPE_BP_AP: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseR_FEC); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + break; + + case FW_PORT_TYPE_BP4_AP: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseR_FEC); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKX4_Full); + break; + + case FW_PORT_TYPE_FIBER_XFI: + case FW_PORT_TYPE_FIBER_XAUI: + case FW_PORT_TYPE_SFP: + case FW_PORT_TYPE_QSFP_10G: + case FW_PORT_TYPE_QSA: + SET_LMM(FIBRE); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full); + break; + + case FW_PORT_TYPE_BP40_BA: + case FW_PORT_TYPE_QSFP: + SET_LMM(FIBRE); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full); + FW_CAPS_TO_LMM(SPEED_40G, 40000baseSR4_Full); + break; + + case FW_PORT_TYPE_CR_QSFP: + case FW_PORT_TYPE_SFP28: + SET_LMM(FIBRE); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full); + FW_CAPS_TO_LMM(SPEED_25G, 25000baseCR_Full); + break; + + case FW_PORT_TYPE_KR_SFP28: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + FW_CAPS_TO_LMM(SPEED_25G, 25000baseKR_Full); + break; + + case FW_PORT_TYPE_KR_XLAUI: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + FW_CAPS_TO_LMM(SPEED_40G, 40000baseKR4_Full); + break; + + case FW_PORT_TYPE_CR2_QSFP: + SET_LMM(FIBRE); + FW_CAPS_TO_LMM(SPEED_50G, 50000baseSR2_Full); + break; + + case FW_PORT_TYPE_KR4_100G: + case FW_PORT_TYPE_CR4_QSFP: + SET_LMM(FIBRE); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + FW_CAPS_TO_LMM(SPEED_40G, 40000baseSR4_Full); + FW_CAPS_TO_LMM(SPEED_25G, 25000baseCR_Full); + FW_CAPS_TO_LMM(SPEED_50G, 50000baseCR2_Full); + FW_CAPS_TO_LMM(SPEED_100G, 100000baseCR4_Full); + break; + + default: + break; + } + + if (fw_caps & FW_PORT_CAP32_FEC_V(FW_PORT_CAP32_FEC_M)) { + FW_CAPS_TO_LMM(FEC_RS, FEC_RS); + FW_CAPS_TO_LMM(FEC_BASER_RS, FEC_BASER); + } else { + SET_LMM(FEC_NONE); + } + + FW_CAPS_TO_LMM(ANEG, Autoneg); + FW_CAPS_TO_LMM(802_3_PAUSE, Pause); + FW_CAPS_TO_LMM(802_3_ASM_DIR, Asym_Pause); + + #undef FW_CAPS_TO_LMM + #undef SET_LMM +} + +/** + * lmm_to_fw_caps - translate ethtool Link Mode Mask to Firmware + * capabilities + * @link_mode_mask: ethtool Link Mode Mask + * + * Translate ethtool Link Mode Mask into a Firmware Port capabilities + * value. + */ +static unsigned int lmm_to_fw_caps(const unsigned long *link_mode_mask) +{ + unsigned int fw_caps = 0; + + #define LMM_TO_FW_CAPS(__lmm_name, __fw_name) \ + do { \ + if (test_bit(ETHTOOL_LINK_MODE_ ## __lmm_name ## _BIT, \ + link_mode_mask)) \ + fw_caps |= FW_PORT_CAP32_ ## __fw_name; \ + } while (0) + + LMM_TO_FW_CAPS(100baseT_Full, SPEED_100M); + LMM_TO_FW_CAPS(1000baseT_Full, SPEED_1G); + LMM_TO_FW_CAPS(10000baseT_Full, SPEED_10G); + LMM_TO_FW_CAPS(40000baseSR4_Full, SPEED_40G); + LMM_TO_FW_CAPS(25000baseCR_Full, SPEED_25G); + LMM_TO_FW_CAPS(50000baseCR2_Full, SPEED_50G); + LMM_TO_FW_CAPS(100000baseCR4_Full, SPEED_100G); + + #undef LMM_TO_FW_CAPS + + return fw_caps; +} + +static int get_link_ksettings(struct net_device *dev, + struct ethtool_link_ksettings *link_ksettings) +{ + struct port_info *pi = netdev_priv(dev); + struct ethtool_link_settings *base = &link_ksettings->base; + + /* For the nonce, the Firmware doesn't send up Port State changes + * when the Virtual Interface attached to the Port is down. So + * if it's down, let's grab any changes. + */ + if (!netif_running(dev)) + (void)t4_update_port_info(pi); + + ethtool_link_ksettings_zero_link_mode(link_ksettings, supported); + ethtool_link_ksettings_zero_link_mode(link_ksettings, advertising); + ethtool_link_ksettings_zero_link_mode(link_ksettings, lp_advertising); + + base->port = from_fw_port_mod_type(pi->port_type, pi->mod_type); + + if (pi->mdio_addr >= 0) { + base->phy_address = pi->mdio_addr; + base->mdio_support = (pi->port_type == FW_PORT_TYPE_BT_SGMII + ? ETH_MDIO_SUPPORTS_C22 + : ETH_MDIO_SUPPORTS_C45); + } else { + base->phy_address = 255; + base->mdio_support = 0; + } + + fw_caps_to_lmm(pi->port_type, pi->link_cfg.pcaps, + link_ksettings->link_modes.supported); + fw_caps_to_lmm(pi->port_type, + t4_link_acaps(pi->adapter, + pi->lport, + &pi->link_cfg), + link_ksettings->link_modes.advertising); + fw_caps_to_lmm(pi->port_type, pi->link_cfg.lpacaps, + link_ksettings->link_modes.lp_advertising); + + base->speed = (netif_carrier_ok(dev) + ? pi->link_cfg.speed + : SPEED_UNKNOWN); + base->duplex = DUPLEX_FULL; + + base->autoneg = pi->link_cfg.autoneg; + if (pi->link_cfg.pcaps & FW_PORT_CAP32_ANEG) + ethtool_link_ksettings_add_link_mode(link_ksettings, + supported, Autoneg); + if (pi->link_cfg.autoneg) + ethtool_link_ksettings_add_link_mode(link_ksettings, + advertising, Autoneg); + + return 0; +} + +static int set_link_ksettings(struct net_device *dev, + const struct ethtool_link_ksettings *link_ksettings) +{ + struct port_info *pi = netdev_priv(dev); + struct link_config *lc = &pi->link_cfg; + const struct ethtool_link_settings *base = &link_ksettings->base; + struct link_config old_lc; + unsigned int fw_caps; + int ret = 0; + + /* only full-duplex supported */ + if (base->duplex != DUPLEX_FULL) + return -EINVAL; + + old_lc = *lc; + if (!(lc->pcaps & FW_PORT_CAP32_ANEG) || + base->autoneg == AUTONEG_DISABLE) { + fw_caps = speed_to_fw_caps(base->speed); + + /* Speed must be supported by Physical Port Capabilities. */ + if (!(lc->pcaps & fw_caps)) + return -EINVAL; + + lc->speed_caps = fw_caps; + lc->acaps = fw_caps; + } else { + fw_caps = + lmm_to_fw_caps(link_ksettings->link_modes.advertising); + if (!(lc->pcaps & fw_caps)) + return -EINVAL; + lc->speed_caps = 0; + lc->acaps = fw_caps | FW_PORT_CAP32_ANEG; + } + lc->autoneg = base->autoneg; + + /* If the firmware rejects the Link Configuration request, back out + * the changes and report the error. + */ + ret = t4_link_l1cfg(pi->adapter, pi->adapter->mbox, pi->tx_chan, lc); + if (ret) + *lc = old_lc; + + return ret; +} + +/* Translate the Firmware FEC value into the ethtool value. */ +static inline unsigned int fwcap_to_eth_fec(unsigned int fw_fec) +{ + unsigned int eth_fec = 0; + + if (fw_fec & FW_PORT_CAP32_FEC_RS) + eth_fec |= ETHTOOL_FEC_RS; + if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS) + eth_fec |= ETHTOOL_FEC_BASER; + + /* if nothing is set, then FEC is off */ + if (!eth_fec) + eth_fec = ETHTOOL_FEC_OFF; + + return eth_fec; +} + +/* Translate Common Code FEC value into ethtool value. */ +static inline unsigned int cc_to_eth_fec(unsigned int cc_fec) +{ + unsigned int eth_fec = 0; + + if (cc_fec & FEC_AUTO) + eth_fec |= ETHTOOL_FEC_AUTO; + if (cc_fec & FEC_RS) + eth_fec |= ETHTOOL_FEC_RS; + if (cc_fec & FEC_BASER_RS) + eth_fec |= ETHTOOL_FEC_BASER; + + /* if nothing is set, then FEC is off */ + if (!eth_fec) + eth_fec = ETHTOOL_FEC_OFF; + + return eth_fec; +} + +/* Translate ethtool FEC value into Common Code value. */ +static inline unsigned int eth_to_cc_fec(unsigned int eth_fec) +{ + unsigned int cc_fec = 0; + + if (eth_fec & ETHTOOL_FEC_OFF) + return cc_fec; + + if (eth_fec & ETHTOOL_FEC_AUTO) + cc_fec |= FEC_AUTO; + if (eth_fec & ETHTOOL_FEC_RS) + cc_fec |= FEC_RS; + if (eth_fec & ETHTOOL_FEC_BASER) + cc_fec |= FEC_BASER_RS; + + return cc_fec; +} + +static int get_fecparam(struct net_device *dev, struct ethtool_fecparam *fec) +{ + const struct port_info *pi = netdev_priv(dev); + const struct link_config *lc = &pi->link_cfg; + + /* Translate the Firmware FEC Support into the ethtool value. We + * always support IEEE 802.3 "automatic" selection of Link FEC type if + * any FEC is supported. + */ + fec->fec = fwcap_to_eth_fec(lc->pcaps); + if (fec->fec != ETHTOOL_FEC_OFF) + fec->fec |= ETHTOOL_FEC_AUTO; + + /* Translate the current internal FEC parameters into the + * ethtool values. + */ + fec->active_fec = cc_to_eth_fec(lc->fec); + + return 0; +} + +static int set_fecparam(struct net_device *dev, struct ethtool_fecparam *fec) +{ + struct port_info *pi = netdev_priv(dev); + struct link_config *lc = &pi->link_cfg; + struct link_config old_lc; + int ret; + + /* Save old Link Configuration in case the L1 Configure below + * fails. + */ + old_lc = *lc; + + /* Try to perform the L1 Configure and return the result of that + * effort. If it fails, revert the attempted change. + */ + lc->requested_fec = eth_to_cc_fec(fec->fec); + ret = t4_link_l1cfg(pi->adapter, pi->adapter->mbox, + pi->tx_chan, lc); + if (ret) + *lc = old_lc; + return ret; +} + +static void get_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *epause) +{ + struct port_info *p = netdev_priv(dev); + + epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0; + epause->rx_pause = (p->link_cfg.advertised_fc & PAUSE_RX) != 0; + epause->tx_pause = (p->link_cfg.advertised_fc & PAUSE_TX) != 0; +} + +static int set_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *epause) +{ + struct port_info *p = netdev_priv(dev); + struct link_config *lc = &p->link_cfg; + + if (epause->autoneg == AUTONEG_DISABLE) + lc->requested_fc = 0; + else if (lc->pcaps & FW_PORT_CAP32_ANEG) + lc->requested_fc = PAUSE_AUTONEG; + else + return -EINVAL; + + if (epause->rx_pause) + lc->requested_fc |= PAUSE_RX; + if (epause->tx_pause) + lc->requested_fc |= PAUSE_TX; + if (netif_running(dev)) + return t4_link_l1cfg(p->adapter, p->adapter->mbox, p->tx_chan, + lc); + return 0; +} + +static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) +{ + const struct port_info *pi = netdev_priv(dev); + const struct sge *s = &pi->adapter->sge; + + e->rx_max_pending = MAX_RX_BUFFERS; + e->rx_mini_max_pending = MAX_RSPQ_ENTRIES; + e->rx_jumbo_max_pending = 0; + e->tx_max_pending = MAX_TXQ_ENTRIES; + + e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8; + e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size; + e->rx_jumbo_pending = 0; + e->tx_pending = s->ethtxq[pi->first_qset].q.size; +} + +static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) +{ + int i; + const struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + struct sge *s = &adapter->sge; + + if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending || + e->tx_pending > MAX_TXQ_ENTRIES || + e->rx_mini_pending > MAX_RSPQ_ENTRIES || + e->rx_mini_pending < MIN_RSPQ_ENTRIES || + e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES) + return -EINVAL; + + if (adapter->flags & CXGB4_FULL_INIT_DONE) + return -EBUSY; + + for (i = 0; i < pi->nqsets; ++i) { + s->ethtxq[pi->first_qset + i].q.size = e->tx_pending; + s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8; + s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending; + } + return 0; +} + +/** + * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete! + * @dev: the network device + * @us: the hold-off time in us, or 0 to disable timer + * @cnt: the hold-off packet count, or 0 to disable counter + * + * Set the RX interrupt hold-off parameters for a network device. + */ +static int set_rx_intr_params(struct net_device *dev, + unsigned int us, unsigned int cnt) +{ + int i, err; + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset]; + + for (i = 0; i < pi->nqsets; i++, q++) { + err = cxgb4_set_rspq_intr_params(&q->rspq, us, cnt); + if (err) + return err; + } + return 0; +} + +static int set_adaptive_rx_setting(struct net_device *dev, int adaptive_rx) +{ + int i; + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset]; + + for (i = 0; i < pi->nqsets; i++, q++) + q->rspq.adaptive_rx = adaptive_rx; + + return 0; +} + +static int get_adaptive_rx_setting(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset]; + + return q->rspq.adaptive_rx; +} + +/* Return the current global Adapter SGE Doorbell Queue Timer Tick for all + * Ethernet TX Queues. + */ +static int get_dbqtimer_tick(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + + if (!(adap->flags & CXGB4_SGE_DBQ_TIMER)) + return 0; + + return adap->sge.dbqtimer_tick; +} + +/* Return the SGE Doorbell Queue Timer Value for the Ethernet TX Queues + * associated with a Network Device. + */ +static int get_dbqtimer(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + struct sge_eth_txq *txq; + + txq = &adap->sge.ethtxq[pi->first_qset]; + + if (!(adap->flags & CXGB4_SGE_DBQ_TIMER)) + return 0; + + /* all of the TX Queues use the same Timer Index */ + return adap->sge.dbqtimer_val[txq->dbqtimerix]; +} + +/* Set the global Adapter SGE Doorbell Queue Timer Tick for all Ethernet TX + * Queues. This is the fundamental "Tick" that sets the scale of values which + * can be used. Individual Ethernet TX Queues index into a relatively small + * array of Tick Multipliers. Changing the base Tick will thus change all of + * the resulting Timer Values associated with those multipliers for all + * Ethernet TX Queues. + */ +static int set_dbqtimer_tick(struct net_device *dev, int usecs) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + struct sge *s = &adap->sge; + u32 param, val; + int ret; + + if (!(adap->flags & CXGB4_SGE_DBQ_TIMER)) + return 0; + + /* return early if it's the same Timer Tick we're already using */ + if (s->dbqtimer_tick == usecs) + return 0; + + /* attempt to set the new Timer Tick value */ + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_DBQ_TIMERTICK)); + val = usecs; + ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, ¶m, &val); + if (ret) + return ret; + s->dbqtimer_tick = usecs; + + /* if successful, reread resulting dependent Timer values */ + ret = t4_read_sge_dbqtimers(adap, ARRAY_SIZE(s->dbqtimer_val), + s->dbqtimer_val); + return ret; +} + +/* Set the SGE Doorbell Queue Timer Value for the Ethernet TX Queues + * associated with a Network Device. There is a relatively small array of + * possible Timer Values so we need to pick the closest value available. + */ +static int set_dbqtimer(struct net_device *dev, int usecs) +{ + int qix, timerix, min_timerix, delta, min_delta; + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + struct sge *s = &adap->sge; + struct sge_eth_txq *txq; + u32 param, val; + int ret; + + if (!(adap->flags & CXGB4_SGE_DBQ_TIMER)) + return 0; + + /* Find the SGE Doorbell Timer Value that's closest to the requested + * value. + */ + min_delta = INT_MAX; + min_timerix = 0; + for (timerix = 0; timerix < ARRAY_SIZE(s->dbqtimer_val); timerix++) { + delta = s->dbqtimer_val[timerix] - usecs; + if (delta < 0) + delta = -delta; + if (delta < min_delta) { + min_delta = delta; + min_timerix = timerix; + } + } + + /* Return early if it's the same Timer Index we're already using. + * We use the same Timer Index for all of the TX Queues for an + * interface so it's only necessary to check the first one. + */ + txq = &s->ethtxq[pi->first_qset]; + if (txq->dbqtimerix == min_timerix) + return 0; + + for (qix = 0; qix < pi->nqsets; qix++, txq++) { + if (adap->flags & CXGB4_FULL_INIT_DONE) { + param = + (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_EQ_TIMERIX) | + FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id)); + val = min_timerix; + ret = t4_set_params(adap, adap->mbox, adap->pf, 0, + 1, ¶m, &val); + if (ret) + return ret; + } + txq->dbqtimerix = min_timerix; + } + return 0; +} + +/* Set the global Adapter SGE Doorbell Queue Timer Tick for all Ethernet TX + * Queues and the Timer Value for the Ethernet TX Queues associated with a + * Network Device. Since changing the global Tick changes all of the + * available Timer Values, we need to do this first before selecting the + * resulting closest Timer Value. Moreover, since the Tick is global, + * changing it affects the Timer Values for all Network Devices on the + * adapter. So, before changing the Tick, we grab all of the current Timer + * Values for other Network Devices on this Adapter and then attempt to select + * new Timer Values which are close to the old values ... + */ +static int set_dbqtimer_tickval(struct net_device *dev, + int tick_usecs, int timer_usecs) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + int timer[MAX_NPORTS]; + unsigned int port; + int ret; + + /* Grab the other adapter Network Interface current timers and fill in + * the new one for this Network Interface. + */ + for_each_port(adap, port) + if (port == pi->port_id) + timer[port] = timer_usecs; + else + timer[port] = get_dbqtimer(adap->port[port]); + + /* Change the global Tick first ... */ + ret = set_dbqtimer_tick(dev, tick_usecs); + if (ret) + return ret; + + /* ... and then set all of the Network Interface Timer Values ... */ + for_each_port(adap, port) { + ret = set_dbqtimer(adap->port[port], timer[port]); + if (ret) + return ret; + } + + return 0; +} + +static int set_coalesce(struct net_device *dev, + struct ethtool_coalesce *coalesce, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + int ret; + + set_adaptive_rx_setting(dev, coalesce->use_adaptive_rx_coalesce); + + ret = set_rx_intr_params(dev, coalesce->rx_coalesce_usecs, + coalesce->rx_max_coalesced_frames); + if (ret) + return ret; + + return set_dbqtimer_tickval(dev, + coalesce->tx_coalesce_usecs_irq, + coalesce->tx_coalesce_usecs); +} + +static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + const struct port_info *pi = netdev_priv(dev); + const struct adapter *adap = pi->adapter; + const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq; + + c->rx_coalesce_usecs = qtimer_val(adap, rq); + c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN_F) ? + adap->sge.counter_val[rq->pktcnt_idx] : 0; + c->use_adaptive_rx_coalesce = get_adaptive_rx_setting(dev); + c->tx_coalesce_usecs_irq = get_dbqtimer_tick(dev); + c->tx_coalesce_usecs = get_dbqtimer(dev); + return 0; +} + +/* The next two routines implement eeprom read/write from physical addresses. + */ +static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v) +{ + int vaddr = t4_eeprom_ptov(phys_addr, adap->pf, EEPROMPFSIZE); + + if (vaddr >= 0) + vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v); + return vaddr < 0 ? vaddr : 0; +} + +static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v) +{ + int vaddr = t4_eeprom_ptov(phys_addr, adap->pf, EEPROMPFSIZE); + + if (vaddr >= 0) + vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v); + return vaddr < 0 ? vaddr : 0; +} + +#define EEPROM_MAGIC 0x38E2F10C + +static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e, + u8 *data) +{ + int i, err = 0; + struct adapter *adapter = netdev2adap(dev); + u8 *buf = kvzalloc(EEPROMSIZE, GFP_KERNEL); + + if (!buf) + return -ENOMEM; + + e->magic = EEPROM_MAGIC; + for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4) + err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]); + + if (!err) + memcpy(data, buf + e->offset, e->len); + kvfree(buf); + return err; +} + +static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, + u8 *data) +{ + u8 *buf; + int err = 0; + u32 aligned_offset, aligned_len, *p; + struct adapter *adapter = netdev2adap(dev); + + if (eeprom->magic != EEPROM_MAGIC) + return -EINVAL; + + aligned_offset = eeprom->offset & ~3; + aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3; + + if (adapter->pf > 0) { + u32 start = 1024 + adapter->pf * EEPROMPFSIZE; + + if (aligned_offset < start || + aligned_offset + aligned_len > start + EEPROMPFSIZE) + return -EPERM; + } + + if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) { + /* RMW possibly needed for first or last words. + */ + buf = kvzalloc(aligned_len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf); + if (!err && aligned_len > 4) + err = eeprom_rd_phys(adapter, + aligned_offset + aligned_len - 4, + (u32 *)&buf[aligned_len - 4]); + if (err) + goto out; + memcpy(buf + (eeprom->offset & 3), data, eeprom->len); + } else { + buf = data; + } + + err = t4_seeprom_wp(adapter, false); + if (err) + goto out; + + for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { + err = eeprom_wr_phys(adapter, aligned_offset, *p); + aligned_offset += 4; + } + + if (!err) + err = t4_seeprom_wp(adapter, true); +out: + if (buf != data) + kvfree(buf); + return err; +} + +static int cxgb4_ethtool_flash_bootcfg(struct net_device *netdev, + const u8 *data, u32 size) +{ + struct adapter *adap = netdev2adap(netdev); + int ret; + + ret = t4_load_bootcfg(adap, data, size); + if (ret) + dev_err(adap->pdev_dev, "Failed to load boot cfg image\n"); + + return ret; +} + +static int cxgb4_ethtool_flash_boot(struct net_device *netdev, + const u8 *bdata, u32 size) +{ + struct adapter *adap = netdev2adap(netdev); + unsigned int offset; + u8 *data; + int ret; + + data = kmemdup(bdata, size, GFP_KERNEL); + if (!data) + return -ENOMEM; + + offset = OFFSET_G(t4_read_reg(adap, PF_REG(0, PCIE_PF_EXPROM_OFST_A))); + + ret = t4_load_boot(adap, data, offset, size); + if (ret) + dev_err(adap->pdev_dev, "Failed to load boot image\n"); + + kfree(data); + return ret; +} + +#define CXGB4_PHY_SIG 0x130000ea + +static int cxgb4_validate_phy_image(const u8 *data, u32 *size) +{ + struct cxgb4_fw_data *header; + + header = (struct cxgb4_fw_data *)data; + if (be32_to_cpu(header->signature) != CXGB4_PHY_SIG) + return -EINVAL; + + return 0; +} + +static int cxgb4_ethtool_flash_phy(struct net_device *netdev, + const u8 *data, u32 size) +{ + struct adapter *adap = netdev2adap(netdev); + int ret; + + ret = cxgb4_validate_phy_image(data, NULL); + if (ret) { + dev_err(adap->pdev_dev, "PHY signature mismatch\n"); + return ret; + } + + /* We have to RESET the chip/firmware because we need the + * chip in uninitialized state for loading new PHY image. + * Otherwise, the running firmware will only store the PHY + * image in local RAM which will be lost after next reset. + */ + ret = t4_fw_reset(adap, adap->mbox, PIORSTMODE_F | PIORST_F); + if (ret < 0) { + dev_err(adap->pdev_dev, + "Set FW to RESET for flashing PHY FW failed. ret: %d\n", + ret); + return ret; + } + + ret = t4_load_phy_fw(adap, MEMWIN_NIC, NULL, data, size); + if (ret < 0) { + dev_err(adap->pdev_dev, "Failed to load PHY FW. ret: %d\n", + ret); + return ret; + } + + return 0; +} + +static int cxgb4_ethtool_flash_fw(struct net_device *netdev, + const u8 *data, u32 size) +{ + struct adapter *adap = netdev2adap(netdev); + unsigned int mbox = PCIE_FW_MASTER_M + 1; + int ret; + + /* If the adapter has been fully initialized then we'll go ahead and + * try to get the firmware's cooperation in upgrading to the new + * firmware image otherwise we'll try to do the entire job from the + * host ... and we always "force" the operation in this path. + */ + if (adap->flags & CXGB4_FULL_INIT_DONE) + mbox = adap->mbox; + + ret = t4_fw_upgrade(adap, mbox, data, size, 1); + if (ret) + dev_err(adap->pdev_dev, + "Failed to flash firmware\n"); + + return ret; +} + +static int cxgb4_ethtool_flash_region(struct net_device *netdev, + const u8 *data, u32 size, u32 region) +{ + struct adapter *adap = netdev2adap(netdev); + int ret; + + switch (region) { + case CXGB4_ETHTOOL_FLASH_FW: + ret = cxgb4_ethtool_flash_fw(netdev, data, size); + break; + case CXGB4_ETHTOOL_FLASH_PHY: + ret = cxgb4_ethtool_flash_phy(netdev, data, size); + break; + case CXGB4_ETHTOOL_FLASH_BOOT: + ret = cxgb4_ethtool_flash_boot(netdev, data, size); + break; + case CXGB4_ETHTOOL_FLASH_BOOTCFG: + ret = cxgb4_ethtool_flash_bootcfg(netdev, data, size); + break; + default: + ret = -EOPNOTSUPP; + break; + } + + if (!ret) + dev_info(adap->pdev_dev, + "loading %s successful, reload cxgb4 driver\n", + flash_region_strings[region]); + return ret; +} + +#define CXGB4_FW_SIG 0x4368656c +#define CXGB4_FW_SIG_OFFSET 0x160 + +static int cxgb4_validate_fw_image(const u8 *data, u32 *size) +{ + struct cxgb4_fw_data *header; + + header = (struct cxgb4_fw_data *)&data[CXGB4_FW_SIG_OFFSET]; + if (be32_to_cpu(header->signature) != CXGB4_FW_SIG) + return -EINVAL; + + if (size) + *size = be16_to_cpu(((struct fw_hdr *)data)->len512) * 512; + + return 0; +} + +static int cxgb4_validate_bootcfg_image(const u8 *data, u32 *size) +{ + struct cxgb4_bootcfg_data *header; + + header = (struct cxgb4_bootcfg_data *)data; + if (le16_to_cpu(header->signature) != BOOT_CFG_SIG) + return -EINVAL; + + return 0; +} + +static int cxgb4_validate_boot_image(const u8 *data, u32 *size) +{ + struct cxgb4_pci_exp_rom_header *exp_header; + struct cxgb4_pcir_data *pcir_header; + struct legacy_pci_rom_hdr *header; + const u8 *cur_header = data; + u16 pcir_offset; + + exp_header = (struct cxgb4_pci_exp_rom_header *)data; + + if (le16_to_cpu(exp_header->signature) != BOOT_SIGNATURE) + return -EINVAL; + + if (size) { + do { + header = (struct legacy_pci_rom_hdr *)cur_header; + pcir_offset = le16_to_cpu(header->pcir_offset); + pcir_header = (struct cxgb4_pcir_data *)(cur_header + + pcir_offset); + + *size += header->size512 * 512; + cur_header += header->size512 * 512; + } while (!(pcir_header->indicator & CXGB4_HDR_INDI)); + } + + return 0; +} + +static int cxgb4_ethtool_get_flash_region(const u8 *data, u32 *size) +{ + if (!cxgb4_validate_fw_image(data, size)) + return CXGB4_ETHTOOL_FLASH_FW; + if (!cxgb4_validate_boot_image(data, size)) + return CXGB4_ETHTOOL_FLASH_BOOT; + if (!cxgb4_validate_phy_image(data, size)) + return CXGB4_ETHTOOL_FLASH_PHY; + if (!cxgb4_validate_bootcfg_image(data, size)) + return CXGB4_ETHTOOL_FLASH_BOOTCFG; + + return -EOPNOTSUPP; +} + +static int set_flash(struct net_device *netdev, struct ethtool_flash *ef) +{ + struct adapter *adap = netdev2adap(netdev); + const struct firmware *fw; + unsigned int master; + u8 master_vld = 0; + const u8 *fw_data; + size_t fw_size; + u32 size = 0; + u32 pcie_fw; + int region; + int ret; + + pcie_fw = t4_read_reg(adap, PCIE_FW_A); + master = PCIE_FW_MASTER_G(pcie_fw); + if (pcie_fw & PCIE_FW_MASTER_VLD_F) + master_vld = 1; + /* if csiostor is the master return */ + if (master_vld && (master != adap->pf)) { + dev_warn(adap->pdev_dev, + "cxgb4 driver needs to be loaded as MASTER to support FW flash\n"); + return -EOPNOTSUPP; + } + + ef->data[sizeof(ef->data) - 1] = '\0'; + ret = request_firmware(&fw, ef->data, adap->pdev_dev); + if (ret < 0) + return ret; + + fw_data = fw->data; + fw_size = fw->size; + if (ef->region == ETHTOOL_FLASH_ALL_REGIONS) { + while (fw_size > 0) { + size = 0; + region = cxgb4_ethtool_get_flash_region(fw_data, &size); + if (region < 0 || !size) { + ret = region; + goto out_free_fw; + } + + ret = cxgb4_ethtool_flash_region(netdev, fw_data, size, + region); + if (ret) + goto out_free_fw; + + fw_data += size; + fw_size -= size; + } + } else { + ret = cxgb4_ethtool_flash_region(netdev, fw_data, fw_size, + ef->region); + } + +out_free_fw: + release_firmware(fw); + return ret; +} + +static int get_ts_info(struct net_device *dev, struct ethtool_ts_info *ts_info) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + ts_info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE; + + ts_info->so_timestamping |= SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + ts_info->tx_types = (1 << HWTSTAMP_TX_OFF) | + (1 << HWTSTAMP_TX_ON); + + ts_info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) | + (1 << HWTSTAMP_FILTER_PTP_V2_L4_EVENT) | + (1 << HWTSTAMP_FILTER_PTP_V1_L4_SYNC) | + (1 << HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ) | + (1 << HWTSTAMP_FILTER_PTP_V2_L4_SYNC) | + (1 << HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ); + + if (adapter->ptp_clock) + ts_info->phc_index = ptp_clock_index(adapter->ptp_clock); + else + ts_info->phc_index = -1; + + return 0; +} + +static u32 get_rss_table_size(struct net_device *dev) +{ + const struct port_info *pi = netdev_priv(dev); + + return pi->rss_size; +} + +static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc) +{ + const struct port_info *pi = netdev_priv(dev); + unsigned int n = pi->rss_size; + + if (hfunc) + *hfunc = ETH_RSS_HASH_TOP; + if (!p) + return 0; + while (n--) + p[n] = pi->rss[n]; + return 0; +} + +static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key, + const u8 hfunc) +{ + unsigned int i; + struct port_info *pi = netdev_priv(dev); + + /* We require at least one supported parameter to be changed and no + * change in any of the unsupported parameters + */ + if (key || + (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) + return -EOPNOTSUPP; + if (!p) + return 0; + + /* Interface must be brought up atleast once */ + if (pi->adapter->flags & CXGB4_FULL_INIT_DONE) { + for (i = 0; i < pi->rss_size; i++) + pi->rss[i] = p[i]; + + return cxgb4_write_rss(pi, pi->rss); + } + + return -EPERM; +} + +static struct filter_entry *cxgb4_get_filter_entry(struct adapter *adap, + u32 ftid) +{ + struct tid_info *t = &adap->tids; + + if (ftid >= t->hpftid_base && ftid < t->hpftid_base + t->nhpftids) + return &t->hpftid_tab[ftid - t->hpftid_base]; + + if (ftid >= t->ftid_base && ftid < t->ftid_base + t->nftids) + return &t->ftid_tab[ftid - t->ftid_base]; + + return lookup_tid(t, ftid); +} + +static void cxgb4_fill_filter_rule(struct ethtool_rx_flow_spec *fs, + struct ch_filter_specification *dfs) +{ + switch (dfs->val.proto) { + case IPPROTO_TCP: + if (dfs->type) + fs->flow_type = TCP_V6_FLOW; + else + fs->flow_type = TCP_V4_FLOW; + break; + case IPPROTO_UDP: + if (dfs->type) + fs->flow_type = UDP_V6_FLOW; + else + fs->flow_type = UDP_V4_FLOW; + break; + } + + if (dfs->type) { + fs->h_u.tcp_ip6_spec.psrc = cpu_to_be16(dfs->val.fport); + fs->m_u.tcp_ip6_spec.psrc = cpu_to_be16(dfs->mask.fport); + fs->h_u.tcp_ip6_spec.pdst = cpu_to_be16(dfs->val.lport); + fs->m_u.tcp_ip6_spec.pdst = cpu_to_be16(dfs->mask.lport); + memcpy(&fs->h_u.tcp_ip6_spec.ip6src, &dfs->val.fip[0], + sizeof(fs->h_u.tcp_ip6_spec.ip6src)); + memcpy(&fs->m_u.tcp_ip6_spec.ip6src, &dfs->mask.fip[0], + sizeof(fs->m_u.tcp_ip6_spec.ip6src)); + memcpy(&fs->h_u.tcp_ip6_spec.ip6dst, &dfs->val.lip[0], + sizeof(fs->h_u.tcp_ip6_spec.ip6dst)); + memcpy(&fs->m_u.tcp_ip6_spec.ip6dst, &dfs->mask.lip[0], + sizeof(fs->m_u.tcp_ip6_spec.ip6dst)); + fs->h_u.tcp_ip6_spec.tclass = dfs->val.tos; + fs->m_u.tcp_ip6_spec.tclass = dfs->mask.tos; + } else { + fs->h_u.tcp_ip4_spec.psrc = cpu_to_be16(dfs->val.fport); + fs->m_u.tcp_ip4_spec.psrc = cpu_to_be16(dfs->mask.fport); + fs->h_u.tcp_ip4_spec.pdst = cpu_to_be16(dfs->val.lport); + fs->m_u.tcp_ip4_spec.pdst = cpu_to_be16(dfs->mask.lport); + memcpy(&fs->h_u.tcp_ip4_spec.ip4src, &dfs->val.fip[0], + sizeof(fs->h_u.tcp_ip4_spec.ip4src)); + memcpy(&fs->m_u.tcp_ip4_spec.ip4src, &dfs->mask.fip[0], + sizeof(fs->m_u.tcp_ip4_spec.ip4src)); + memcpy(&fs->h_u.tcp_ip4_spec.ip4dst, &dfs->val.lip[0], + sizeof(fs->h_u.tcp_ip4_spec.ip4dst)); + memcpy(&fs->m_u.tcp_ip4_spec.ip4dst, &dfs->mask.lip[0], + sizeof(fs->m_u.tcp_ip4_spec.ip4dst)); + fs->h_u.tcp_ip4_spec.tos = dfs->val.tos; + fs->m_u.tcp_ip4_spec.tos = dfs->mask.tos; + } + fs->h_ext.vlan_tci = cpu_to_be16(dfs->val.ivlan); + fs->m_ext.vlan_tci = cpu_to_be16(dfs->mask.ivlan); + fs->flow_type |= FLOW_EXT; + + if (dfs->action == FILTER_DROP) + fs->ring_cookie = RX_CLS_FLOW_DISC; + else + fs->ring_cookie = dfs->iq; +} + +static int cxgb4_ntuple_get_filter(struct net_device *dev, + struct ethtool_rxnfc *cmd, + unsigned int loc) +{ + const struct port_info *pi = netdev_priv(dev); + struct adapter *adap = netdev2adap(dev); + struct filter_entry *f; + int ftid; + + if (!(adap->flags & CXGB4_FULL_INIT_DONE)) + return -EAGAIN; + + /* Check for maximum filter range */ + if (!adap->ethtool_filters) + return -EOPNOTSUPP; + + if (loc >= adap->ethtool_filters->nentries) + return -ERANGE; + + if (!test_bit(loc, adap->ethtool_filters->port[pi->port_id].bmap)) + return -ENOENT; + + ftid = adap->ethtool_filters->port[pi->port_id].loc_array[loc]; + + /* Fetch filter_entry */ + f = cxgb4_get_filter_entry(adap, ftid); + + cxgb4_fill_filter_rule(&cmd->fs, &f->fs); + + return 0; +} + +static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, + u32 *rules) +{ + const struct port_info *pi = netdev_priv(dev); + struct adapter *adap = netdev2adap(dev); + unsigned int count = 0, index = 0; + int ret = 0; + + switch (info->cmd) { + case ETHTOOL_GRXFH: { + unsigned int v = pi->rss_mode; + + info->data = 0; + switch (info->flow_type) { + case TCP_V4_FLOW: + if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) + info->data = RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3; + else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F) + info->data = RXH_IP_SRC | RXH_IP_DST; + break; + case UDP_V4_FLOW: + if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) && + (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F)) + info->data = RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3; + else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F) + info->data = RXH_IP_SRC | RXH_IP_DST; + break; + case SCTP_V4_FLOW: + case AH_ESP_V4_FLOW: + case IPV4_FLOW: + if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F) + info->data = RXH_IP_SRC | RXH_IP_DST; + break; + case TCP_V6_FLOW: + if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) + info->data = RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3; + else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F) + info->data = RXH_IP_SRC | RXH_IP_DST; + break; + case UDP_V6_FLOW: + if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) && + (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F)) + info->data = RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3; + else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F) + info->data = RXH_IP_SRC | RXH_IP_DST; + break; + case SCTP_V6_FLOW: + case AH_ESP_V6_FLOW: + case IPV6_FLOW: + if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F) + info->data = RXH_IP_SRC | RXH_IP_DST; + break; + } + return 0; + } + case ETHTOOL_GRXRINGS: + info->data = pi->nqsets; + return 0; + case ETHTOOL_GRXCLSRLCNT: + info->rule_cnt = + adap->ethtool_filters->port[pi->port_id].in_use; + return 0; + case ETHTOOL_GRXCLSRULE: + return cxgb4_ntuple_get_filter(dev, info, info->fs.location); + case ETHTOOL_GRXCLSRLALL: + info->data = adap->ethtool_filters->nentries; + while (count < info->rule_cnt) { + ret = cxgb4_ntuple_get_filter(dev, info, index); + if (!ret) + rules[count++] = index; + index++; + } + return 0; + } + + return -EOPNOTSUPP; +} + +static int cxgb4_ntuple_del_filter(struct net_device *dev, + struct ethtool_rxnfc *cmd) +{ + struct cxgb4_ethtool_filter_info *filter_info; + struct adapter *adapter = netdev2adap(dev); + struct port_info *pi = netdev_priv(dev); + struct filter_entry *f; + u32 filter_id; + int ret; + + if (!(adapter->flags & CXGB4_FULL_INIT_DONE)) + return -EAGAIN; /* can still change nfilters */ + + if (!adapter->ethtool_filters) + return -EOPNOTSUPP; + + if (cmd->fs.location >= adapter->ethtool_filters->nentries) { + dev_err(adapter->pdev_dev, + "Location must be < %u", + adapter->ethtool_filters->nentries); + return -ERANGE; + } + + filter_info = &adapter->ethtool_filters->port[pi->port_id]; + + if (!test_bit(cmd->fs.location, filter_info->bmap)) + return -ENOENT; + + filter_id = filter_info->loc_array[cmd->fs.location]; + f = cxgb4_get_filter_entry(adapter, filter_id); + + if (f->fs.prio) + filter_id -= adapter->tids.hpftid_base; + else if (!f->fs.hash) + filter_id -= (adapter->tids.ftid_base - adapter->tids.nhpftids); + + ret = cxgb4_flow_rule_destroy(dev, f->fs.tc_prio, &f->fs, filter_id); + if (ret) + goto err; + + clear_bit(cmd->fs.location, filter_info->bmap); + filter_info->in_use--; + +err: + return ret; +} + +/* Add Ethtool n-tuple filters. */ +static int cxgb4_ntuple_set_filter(struct net_device *netdev, + struct ethtool_rxnfc *cmd) +{ + struct ethtool_rx_flow_spec_input input = {}; + struct cxgb4_ethtool_filter_info *filter_info; + struct adapter *adapter = netdev2adap(netdev); + struct port_info *pi = netdev_priv(netdev); + struct ch_filter_specification fs; + struct ethtool_rx_flow_rule *flow; + u32 tid; + int ret; + + if (!(adapter->flags & CXGB4_FULL_INIT_DONE)) + return -EAGAIN; /* can still change nfilters */ + + if (!adapter->ethtool_filters) + return -EOPNOTSUPP; + + if (cmd->fs.location >= adapter->ethtool_filters->nentries) { + dev_err(adapter->pdev_dev, + "Location must be < %u", + adapter->ethtool_filters->nentries); + return -ERANGE; + } + + if (test_bit(cmd->fs.location, + adapter->ethtool_filters->port[pi->port_id].bmap)) + return -EEXIST; + + memset(&fs, 0, sizeof(fs)); + + input.fs = &cmd->fs; + flow = ethtool_rx_flow_rule_create(&input); + if (IS_ERR(flow)) { + ret = PTR_ERR(flow); + goto exit; + } + + fs.hitcnts = 1; + + ret = cxgb4_flow_rule_replace(netdev, flow->rule, cmd->fs.location, + NULL, &fs, &tid); + if (ret) + goto free; + + filter_info = &adapter->ethtool_filters->port[pi->port_id]; + + if (fs.prio) + tid += adapter->tids.hpftid_base; + else if (!fs.hash) + tid += (adapter->tids.ftid_base - adapter->tids.nhpftids); + + filter_info->loc_array[cmd->fs.location] = tid; + set_bit(cmd->fs.location, filter_info->bmap); + filter_info->in_use++; + +free: + ethtool_rx_flow_rule_destroy(flow); +exit: + return ret; +} + +static int set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd) +{ + int ret = -EOPNOTSUPP; + + switch (cmd->cmd) { + case ETHTOOL_SRXCLSRLINS: + ret = cxgb4_ntuple_set_filter(dev, cmd); + break; + case ETHTOOL_SRXCLSRLDEL: + ret = cxgb4_ntuple_del_filter(dev, cmd); + break; + default: + break; + } + + return ret; +} + +static int set_dump(struct net_device *dev, struct ethtool_dump *eth_dump) +{ + struct adapter *adapter = netdev2adap(dev); + u32 len = 0; + + len = sizeof(struct cudbg_hdr) + + sizeof(struct cudbg_entity_hdr) * CUDBG_MAX_ENTITY; + len += cxgb4_get_dump_length(adapter, eth_dump->flag); + + adapter->eth_dump.flag = eth_dump->flag; + adapter->eth_dump.len = len; + return 0; +} + +static int get_dump_flag(struct net_device *dev, struct ethtool_dump *eth_dump) +{ + struct adapter *adapter = netdev2adap(dev); + + eth_dump->flag = adapter->eth_dump.flag; + eth_dump->len = adapter->eth_dump.len; + eth_dump->version = adapter->eth_dump.version; + return 0; +} + +static int get_dump_data(struct net_device *dev, struct ethtool_dump *eth_dump, + void *buf) +{ + struct adapter *adapter = netdev2adap(dev); + u32 len = 0; + int ret = 0; + + if (adapter->eth_dump.flag == CXGB4_ETH_DUMP_NONE) + return -ENOENT; + + len = sizeof(struct cudbg_hdr) + + sizeof(struct cudbg_entity_hdr) * CUDBG_MAX_ENTITY; + len += cxgb4_get_dump_length(adapter, adapter->eth_dump.flag); + if (eth_dump->len < len) + return -ENOMEM; + + ret = cxgb4_cudbg_collect(adapter, buf, &len, adapter->eth_dump.flag); + if (ret) + return ret; + + eth_dump->flag = adapter->eth_dump.flag; + eth_dump->len = len; + eth_dump->version = adapter->eth_dump.version; + return 0; +} + +static bool cxgb4_fw_mod_type_info_available(unsigned int fw_mod_type) +{ + /* Read port module EEPROM as long as it is plugged-in and + * safe to read. + */ + return (fw_mod_type != FW_PORT_MOD_TYPE_NONE && + fw_mod_type != FW_PORT_MOD_TYPE_ERROR); +} + +static int cxgb4_get_module_info(struct net_device *dev, + struct ethtool_modinfo *modinfo) +{ + struct port_info *pi = netdev_priv(dev); + u8 sff8472_comp, sff_diag_type, sff_rev; + struct adapter *adapter = pi->adapter; + int ret; + + if (!cxgb4_fw_mod_type_info_available(pi->mod_type)) + return -EINVAL; + + switch (pi->port_type) { + case FW_PORT_TYPE_SFP: + case FW_PORT_TYPE_QSA: + case FW_PORT_TYPE_SFP28: + ret = t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan, + I2C_DEV_ADDR_A0, SFF_8472_COMP_ADDR, + SFF_8472_COMP_LEN, &sff8472_comp); + if (ret) + return ret; + ret = t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan, + I2C_DEV_ADDR_A0, SFP_DIAG_TYPE_ADDR, + SFP_DIAG_TYPE_LEN, &sff_diag_type); + if (ret) + return ret; + + if (!sff8472_comp || (sff_diag_type & SFP_DIAG_ADDRMODE)) { + modinfo->type = ETH_MODULE_SFF_8079; + modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN; + } else { + modinfo->type = ETH_MODULE_SFF_8472; + if (sff_diag_type & SFP_DIAG_IMPLEMENTED) + modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN; + else + modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN / 2; + } + break; + + case FW_PORT_TYPE_QSFP: + case FW_PORT_TYPE_QSFP_10G: + case FW_PORT_TYPE_CR_QSFP: + case FW_PORT_TYPE_CR2_QSFP: + case FW_PORT_TYPE_CR4_QSFP: + ret = t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan, + I2C_DEV_ADDR_A0, SFF_REV_ADDR, + SFF_REV_LEN, &sff_rev); + /* For QSFP type ports, revision value >= 3 + * means the SFP is 8636 compliant. + */ + if (ret) + return ret; + if (sff_rev >= 0x3) { + modinfo->type = ETH_MODULE_SFF_8636; + modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; + } else { + modinfo->type = ETH_MODULE_SFF_8436; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; + } + break; + + default: + return -EINVAL; + } + + return 0; +} + +static int cxgb4_get_module_eeprom(struct net_device *dev, + struct ethtool_eeprom *eprom, u8 *data) +{ + int ret = 0, offset = eprom->offset, len = eprom->len; + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + memset(data, 0, eprom->len); + if (offset + len <= I2C_PAGE_SIZE) + return t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan, + I2C_DEV_ADDR_A0, offset, len, data); + + /* offset + len spans 0xa0 and 0xa1 pages */ + if (offset <= I2C_PAGE_SIZE) { + /* read 0xa0 page */ + len = I2C_PAGE_SIZE - offset; + ret = t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan, + I2C_DEV_ADDR_A0, offset, len, data); + if (ret) + return ret; + offset = I2C_PAGE_SIZE; + /* Remaining bytes to be read from second page = + * Total length - bytes read from first page + */ + len = eprom->len - len; + } + /* Read additional optical diagnostics from page 0xa2 if supported */ + return t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan, I2C_DEV_ADDR_A2, + offset, len, &data[eprom->len - len]); +} + +static u32 cxgb4_get_priv_flags(struct net_device *netdev) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adapter = pi->adapter; + + return (adapter->eth_flags | pi->eth_flags); +} + +/** + * set_flags - set/unset specified flags if passed in new_flags + * @cur_flags: pointer to current flags + * @new_flags: new incoming flags + * @flags: set of flags to set/unset + */ +static inline void set_flags(u32 *cur_flags, u32 new_flags, u32 flags) +{ + *cur_flags = (*cur_flags & ~flags) | (new_flags & flags); +} + +static int cxgb4_set_priv_flags(struct net_device *netdev, u32 flags) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adapter = pi->adapter; + + set_flags(&adapter->eth_flags, flags, PRIV_FLAGS_ADAP); + set_flags(&pi->eth_flags, flags, PRIV_FLAGS_PORT); + + return 0; +} + +static void cxgb4_lb_test(struct net_device *netdev, u64 *lb_status) +{ + int dev_state = netif_running(netdev); + + if (dev_state) { + netif_tx_stop_all_queues(netdev); + netif_carrier_off(netdev); + } + + *lb_status = cxgb4_selftest_lb_pkt(netdev); + + if (dev_state) { + netif_tx_start_all_queues(netdev); + netif_carrier_on(netdev); + } +} + +static void cxgb4_self_test(struct net_device *netdev, + struct ethtool_test *eth_test, u64 *data) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adap = pi->adapter; + + memset(data, 0, sizeof(u64) * CXGB4_ETHTOOL_MAX_TEST); + + if (!(adap->flags & CXGB4_FULL_INIT_DONE) || + !(adap->flags & CXGB4_FW_OK)) { + eth_test->flags |= ETH_TEST_FL_FAILED; + return; + } + + if (eth_test->flags & ETH_TEST_FL_OFFLINE) + cxgb4_lb_test(netdev, &data[CXGB4_ETHTOOL_LB_TEST]); + + if (data[CXGB4_ETHTOOL_LB_TEST]) + eth_test->flags |= ETH_TEST_FL_FAILED; +} + +static const struct ethtool_ops cxgb_ethtool_ops = { + .supported_coalesce_params = ETHTOOL_COALESCE_USECS | + ETHTOOL_COALESCE_RX_MAX_FRAMES | + ETHTOOL_COALESCE_TX_USECS_IRQ | + ETHTOOL_COALESCE_USE_ADAPTIVE_RX, + .get_link_ksettings = get_link_ksettings, + .set_link_ksettings = set_link_ksettings, + .get_fecparam = get_fecparam, + .set_fecparam = set_fecparam, + .get_drvinfo = get_drvinfo, + .get_msglevel = get_msglevel, + .set_msglevel = set_msglevel, + .get_ringparam = get_sge_param, + .set_ringparam = set_sge_param, + .get_coalesce = get_coalesce, + .set_coalesce = set_coalesce, + .get_eeprom_len = get_eeprom_len, + .get_eeprom = get_eeprom, + .set_eeprom = set_eeprom, + .get_pauseparam = get_pauseparam, + .set_pauseparam = set_pauseparam, + .get_link = ethtool_op_get_link, + .get_strings = get_strings, + .set_phys_id = identify_port, + .nway_reset = restart_autoneg, + .get_sset_count = get_sset_count, + .get_ethtool_stats = get_stats, + .get_regs_len = get_regs_len, + .get_regs = get_regs, + .get_rxnfc = get_rxnfc, + .set_rxnfc = set_rxnfc, + .get_rxfh_indir_size = get_rss_table_size, + .get_rxfh = get_rss_table, + .set_rxfh = set_rss_table, + .self_test = cxgb4_self_test, + .flash_device = set_flash, + .get_ts_info = get_ts_info, + .set_dump = set_dump, + .get_dump_flag = get_dump_flag, + .get_dump_data = get_dump_data, + .get_module_info = cxgb4_get_module_info, + .get_module_eeprom = cxgb4_get_module_eeprom, + .get_priv_flags = cxgb4_get_priv_flags, + .set_priv_flags = cxgb4_set_priv_flags, +}; + +void cxgb4_cleanup_ethtool_filters(struct adapter *adap) +{ + struct cxgb4_ethtool_filter_info *eth_filter_info; + u8 i; + + if (!adap->ethtool_filters) + return; + + eth_filter_info = adap->ethtool_filters->port; + + if (eth_filter_info) { + for (i = 0; i < adap->params.nports; i++) { + kvfree(eth_filter_info[i].loc_array); + bitmap_free(eth_filter_info[i].bmap); + } + kfree(eth_filter_info); + } + + kfree(adap->ethtool_filters); +} + +int cxgb4_init_ethtool_filters(struct adapter *adap) +{ + struct cxgb4_ethtool_filter_info *eth_filter_info; + struct cxgb4_ethtool_filter *eth_filter; + struct tid_info *tids = &adap->tids; + u32 nentries, i; + int ret; + + eth_filter = kzalloc(sizeof(*eth_filter), GFP_KERNEL); + if (!eth_filter) + return -ENOMEM; + + eth_filter_info = kcalloc(adap->params.nports, + sizeof(*eth_filter_info), + GFP_KERNEL); + if (!eth_filter_info) { + ret = -ENOMEM; + goto free_eth_filter; + } + + eth_filter->port = eth_filter_info; + + nentries = tids->nhpftids + tids->nftids; + if (is_hashfilter(adap)) + nentries += tids->nhash + + (adap->tids.stid_base - adap->tids.tid_base); + eth_filter->nentries = nentries; + + for (i = 0; i < adap->params.nports; i++) { + eth_filter->port[i].loc_array = kvzalloc(nentries, GFP_KERNEL); + if (!eth_filter->port[i].loc_array) { + ret = -ENOMEM; + goto free_eth_finfo; + } + + eth_filter->port[i].bmap = bitmap_zalloc(nentries, GFP_KERNEL); + if (!eth_filter->port[i].bmap) { + ret = -ENOMEM; + goto free_eth_finfo; + } + } + + adap->ethtool_filters = eth_filter; + return 0; + +free_eth_finfo: + while (i-- > 0) { + bitmap_free(eth_filter->port[i].bmap); + kvfree(eth_filter->port[i].loc_array); + } + kfree(eth_filter_info); + +free_eth_filter: + kfree(eth_filter); + + return ret; +} + +void cxgb4_set_ethtool_ops(struct net_device *netdev) +{ + netdev->ethtool_ops = &cxgb_ethtool_ops; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_fcoe.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_fcoe.c new file mode 100644 index 000000000..33b2c0c45 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_fcoe.c @@ -0,0 +1,122 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2015 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifdef CONFIG_CHELSIO_T4_FCOE + +#include <scsi/fc/fc_fs.h> +#include <scsi/libfcoe.h> +#include "cxgb4.h" + +bool cxgb_fcoe_sof_eof_supported(struct adapter *adap, struct sk_buff *skb) +{ + struct fcoe_hdr *fcoeh = (struct fcoe_hdr *)skb_network_header(skb); + u8 sof = fcoeh->fcoe_sof; + u8 eof = 0; + + if ((sof != FC_SOF_I3) && (sof != FC_SOF_N3)) { + dev_err(adap->pdev_dev, "Unsupported SOF 0x%x\n", sof); + return false; + } + + skb_copy_bits(skb, skb->len - 4, &eof, 1); + + if ((eof != FC_EOF_N) && (eof != FC_EOF_T)) { + dev_err(adap->pdev_dev, "Unsupported EOF 0x%x\n", eof); + return false; + } + + return true; +} + +/** + * cxgb_fcoe_enable - enable FCoE offload features + * @netdev: net device + * + * Returns 0 on success or -EINVAL on failure. + */ +int cxgb_fcoe_enable(struct net_device *netdev) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adap = pi->adapter; + struct cxgb_fcoe *fcoe = &pi->fcoe; + + if (is_t4(adap->params.chip)) + return -EINVAL; + + if (!(adap->flags & CXGB4_FULL_INIT_DONE)) + return -EINVAL; + + dev_info(adap->pdev_dev, "Enabling FCoE offload features\n"); + + netdev->features |= NETIF_F_FCOE_CRC; + netdev->vlan_features |= NETIF_F_FCOE_CRC; + netdev->features |= NETIF_F_FCOE_MTU; + netdev->vlan_features |= NETIF_F_FCOE_MTU; + + netdev_features_change(netdev); + + fcoe->flags |= CXGB_FCOE_ENABLED; + + return 0; +} + +/** + * cxgb_fcoe_disable - disable FCoE offload + * @netdev: net device + * + * Returns 0 on success or -EINVAL on failure. + */ +int cxgb_fcoe_disable(struct net_device *netdev) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adap = pi->adapter; + struct cxgb_fcoe *fcoe = &pi->fcoe; + + if (!(fcoe->flags & CXGB_FCOE_ENABLED)) + return -EINVAL; + + dev_info(adap->pdev_dev, "Disabling FCoE offload features\n"); + + fcoe->flags &= ~CXGB_FCOE_ENABLED; + + netdev->features &= ~NETIF_F_FCOE_CRC; + netdev->vlan_features &= ~NETIF_F_FCOE_CRC; + netdev->features &= ~NETIF_F_FCOE_MTU; + netdev->vlan_features &= ~NETIF_F_FCOE_MTU; + + netdev_features_change(netdev); + + return 0; +} +#endif /* CONFIG_CHELSIO_T4_FCOE */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_fcoe.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_fcoe.h new file mode 100644 index 000000000..bf9258a56 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_fcoe.h @@ -0,0 +1,57 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2015 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_FCOE_H__ +#define __CXGB4_FCOE_H__ + +#ifdef CONFIG_CHELSIO_T4_FCOE + +#define CXGB_FCOE_TXPKT_CSUM_START 28 +#define CXGB_FCOE_TXPKT_CSUM_END 8 + +/* fcoe flags */ +enum { + CXGB_FCOE_ENABLED = (1 << 0), +}; + +struct cxgb_fcoe { + u8 flags; +}; + +int cxgb_fcoe_enable(struct net_device *); +int cxgb_fcoe_disable(struct net_device *); +bool cxgb_fcoe_sof_eof_supported(struct adapter *, struct sk_buff *); + +#endif /* CONFIG_CHELSIO_T4_FCOE */ +#endif /* __CXGB4_FCOE_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c new file mode 100644 index 000000000..786ceae34 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c @@ -0,0 +1,2174 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include <net/ipv6.h> + +#include "cxgb4.h" +#include "t4_regs.h" +#include "t4_tcb.h" +#include "t4_values.h" +#include "clip_tbl.h" +#include "l2t.h" +#include "smt.h" +#include "t4fw_api.h" +#include "cxgb4_filter.h" + +static inline bool is_field_set(u32 val, u32 mask) +{ + return val || mask; +} + +static inline bool unsupported(u32 conf, u32 conf_mask, u32 val, u32 mask) +{ + return !(conf & conf_mask) && is_field_set(val, mask); +} + +static int set_tcb_field(struct adapter *adap, struct filter_entry *f, + unsigned int ftid, u16 word, u64 mask, u64 val, + int no_reply) +{ + struct cpl_set_tcb_field *req; + struct sk_buff *skb; + + skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC); + if (!skb) + return -ENOMEM; + + req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req)); + INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, ftid); + req->reply_ctrl = htons(REPLY_CHAN_V(0) | + QUEUENO_V(adap->sge.fw_evtq.abs_id) | + NO_REPLY_V(no_reply)); + req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(ftid)); + req->mask = cpu_to_be64(mask); + req->val = cpu_to_be64(val); + set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3); + t4_ofld_send(adap, skb); + return 0; +} + +/* Set one of the t_flags bits in the TCB. + */ +static int set_tcb_tflag(struct adapter *adap, struct filter_entry *f, + unsigned int ftid, unsigned int bit_pos, + unsigned int val, int no_reply) +{ + return set_tcb_field(adap, f, ftid, TCB_T_FLAGS_W, 1ULL << bit_pos, + (unsigned long long)val << bit_pos, no_reply); +} + +static void mk_abort_req_ulp(struct cpl_abort_req *abort_req, unsigned int tid) +{ + struct ulp_txpkt *txpkt = (struct ulp_txpkt *)abort_req; + struct ulptx_idata *sc = (struct ulptx_idata *)(txpkt + 1); + + txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0)); + txpkt->len = htonl(DIV_ROUND_UP(sizeof(*abort_req), 16)); + sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); + sc->len = htonl(sizeof(*abort_req) - sizeof(struct work_request_hdr)); + OPCODE_TID(abort_req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid)); + abort_req->rsvd0 = htonl(0); + abort_req->rsvd1 = 0; + abort_req->cmd = CPL_ABORT_NO_RST; +} + +static void mk_abort_rpl_ulp(struct cpl_abort_rpl *abort_rpl, unsigned int tid) +{ + struct ulp_txpkt *txpkt = (struct ulp_txpkt *)abort_rpl; + struct ulptx_idata *sc = (struct ulptx_idata *)(txpkt + 1); + + txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0)); + txpkt->len = htonl(DIV_ROUND_UP(sizeof(*abort_rpl), 16)); + sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); + sc->len = htonl(sizeof(*abort_rpl) - sizeof(struct work_request_hdr)); + OPCODE_TID(abort_rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, tid)); + abort_rpl->rsvd0 = htonl(0); + abort_rpl->rsvd1 = 0; + abort_rpl->cmd = CPL_ABORT_NO_RST; +} + +static void mk_set_tcb_ulp(struct filter_entry *f, + struct cpl_set_tcb_field *req, + unsigned int word, u64 mask, u64 val, + u8 cookie, int no_reply) +{ + struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; + struct ulptx_idata *sc = (struct ulptx_idata *)(txpkt + 1); + + txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0)); + txpkt->len = htonl(DIV_ROUND_UP(sizeof(*req), 16)); + sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); + sc->len = htonl(sizeof(*req) - sizeof(struct work_request_hdr)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, f->tid)); + req->reply_ctrl = htons(NO_REPLY_V(no_reply) | REPLY_CHAN_V(0) | + QUEUENO_V(0)); + req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie)); + req->mask = cpu_to_be64(mask); + req->val = cpu_to_be64(val); + sc = (struct ulptx_idata *)(req + 1); + sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); + sc->len = htonl(0); +} + +static int configure_filter_smac(struct adapter *adap, struct filter_entry *f) +{ + int err; + + /* do a set-tcb for smac-sel and CWR bit.. */ + err = set_tcb_field(adap, f, f->tid, TCB_SMAC_SEL_W, + TCB_SMAC_SEL_V(TCB_SMAC_SEL_M), + TCB_SMAC_SEL_V(f->smt->idx), 1); + if (err) + goto smac_err; + + err = set_tcb_tflag(adap, f, f->tid, TF_CCTRL_CWR_S, 1, 1); + if (!err) + return 0; + +smac_err: + dev_err(adap->pdev_dev, "filter %u smac config failed with error %u\n", + f->tid, err); + return err; +} + +static void set_nat_params(struct adapter *adap, struct filter_entry *f, + unsigned int tid, bool dip, bool sip, bool dp, + bool sp) +{ + u8 *nat_lp = (u8 *)&f->fs.nat_lport; + u8 *nat_fp = (u8 *)&f->fs.nat_fport; + + if (dip) { + if (f->fs.type) { + set_tcb_field(adap, f, tid, TCB_SND_UNA_RAW_W, + WORD_MASK, f->fs.nat_lip[15] | + f->fs.nat_lip[14] << 8 | + f->fs.nat_lip[13] << 16 | + (u64)f->fs.nat_lip[12] << 24, 1); + + set_tcb_field(adap, f, tid, TCB_SND_UNA_RAW_W + 1, + WORD_MASK, f->fs.nat_lip[11] | + f->fs.nat_lip[10] << 8 | + f->fs.nat_lip[9] << 16 | + (u64)f->fs.nat_lip[8] << 24, 1); + + set_tcb_field(adap, f, tid, TCB_SND_UNA_RAW_W + 2, + WORD_MASK, f->fs.nat_lip[7] | + f->fs.nat_lip[6] << 8 | + f->fs.nat_lip[5] << 16 | + (u64)f->fs.nat_lip[4] << 24, 1); + + set_tcb_field(adap, f, tid, TCB_SND_UNA_RAW_W + 3, + WORD_MASK, f->fs.nat_lip[3] | + f->fs.nat_lip[2] << 8 | + f->fs.nat_lip[1] << 16 | + (u64)f->fs.nat_lip[0] << 24, 1); + } else { + set_tcb_field(adap, f, tid, TCB_RX_FRAG3_LEN_RAW_W, + WORD_MASK, f->fs.nat_lip[3] | + f->fs.nat_lip[2] << 8 | + f->fs.nat_lip[1] << 16 | + (u64)f->fs.nat_lip[0] << 24, 1); + } + } + + if (sip) { + if (f->fs.type) { + set_tcb_field(adap, f, tid, TCB_RX_FRAG2_PTR_RAW_W, + WORD_MASK, f->fs.nat_fip[15] | + f->fs.nat_fip[14] << 8 | + f->fs.nat_fip[13] << 16 | + (u64)f->fs.nat_fip[12] << 24, 1); + + set_tcb_field(adap, f, tid, TCB_RX_FRAG2_PTR_RAW_W + 1, + WORD_MASK, f->fs.nat_fip[11] | + f->fs.nat_fip[10] << 8 | + f->fs.nat_fip[9] << 16 | + (u64)f->fs.nat_fip[8] << 24, 1); + + set_tcb_field(adap, f, tid, TCB_RX_FRAG2_PTR_RAW_W + 2, + WORD_MASK, f->fs.nat_fip[7] | + f->fs.nat_fip[6] << 8 | + f->fs.nat_fip[5] << 16 | + (u64)f->fs.nat_fip[4] << 24, 1); + + set_tcb_field(adap, f, tid, TCB_RX_FRAG2_PTR_RAW_W + 3, + WORD_MASK, f->fs.nat_fip[3] | + f->fs.nat_fip[2] << 8 | + f->fs.nat_fip[1] << 16 | + (u64)f->fs.nat_fip[0] << 24, 1); + + } else { + set_tcb_field(adap, f, tid, + TCB_RX_FRAG3_START_IDX_OFFSET_RAW_W, + WORD_MASK, f->fs.nat_fip[3] | + f->fs.nat_fip[2] << 8 | + f->fs.nat_fip[1] << 16 | + (u64)f->fs.nat_fip[0] << 24, 1); + } + } + + set_tcb_field(adap, f, tid, TCB_PDU_HDR_LEN_W, WORD_MASK, + (dp ? (nat_lp[1] | nat_lp[0] << 8) : 0) | + (sp ? (nat_fp[1] << 16 | (u64)nat_fp[0] << 24) : 0), + 1); +} + +/* Validate filter spec against configuration done on the card. */ +static int validate_filter(struct net_device *dev, + struct ch_filter_specification *fs) +{ + struct adapter *adapter = netdev2adap(dev); + u32 fconf, iconf; + + /* Check for unconfigured fields being used. */ + iconf = adapter->params.tp.ingress_config; + fconf = fs->hash ? adapter->params.tp.filter_mask : + adapter->params.tp.vlan_pri_map; + + if (unsupported(fconf, FCOE_F, fs->val.fcoe, fs->mask.fcoe) || + unsupported(fconf, PORT_F, fs->val.iport, fs->mask.iport) || + unsupported(fconf, TOS_F, fs->val.tos, fs->mask.tos) || + unsupported(fconf, ETHERTYPE_F, fs->val.ethtype, + fs->mask.ethtype) || + unsupported(fconf, MACMATCH_F, fs->val.macidx, fs->mask.macidx) || + unsupported(fconf, MPSHITTYPE_F, fs->val.matchtype, + fs->mask.matchtype) || + unsupported(fconf, FRAGMENTATION_F, fs->val.frag, fs->mask.frag) || + unsupported(fconf, PROTOCOL_F, fs->val.proto, fs->mask.proto) || + unsupported(fconf, VNIC_ID_F, fs->val.pfvf_vld, + fs->mask.pfvf_vld) || + unsupported(fconf, VNIC_ID_F, fs->val.ovlan_vld, + fs->mask.ovlan_vld) || + unsupported(fconf, VNIC_ID_F, fs->val.encap_vld, + fs->mask.encap_vld) || + unsupported(fconf, VLAN_F, fs->val.ivlan_vld, fs->mask.ivlan_vld)) + return -EOPNOTSUPP; + + /* T4 inconveniently uses the same FT_VNIC_ID_W bits for both the Outer + * VLAN Tag and PF/VF/VFvld fields based on VNIC_F being set + * in TP_INGRESS_CONFIG. Hense the somewhat crazy checks + * below. Additionally, since the T4 firmware interface also + * carries that overlap, we need to translate any PF/VF + * specification into that internal format below. + */ + if ((is_field_set(fs->val.pfvf_vld, fs->mask.pfvf_vld) && + is_field_set(fs->val.ovlan_vld, fs->mask.ovlan_vld)) || + (is_field_set(fs->val.pfvf_vld, fs->mask.pfvf_vld) && + is_field_set(fs->val.encap_vld, fs->mask.encap_vld)) || + (is_field_set(fs->val.ovlan_vld, fs->mask.ovlan_vld) && + is_field_set(fs->val.encap_vld, fs->mask.encap_vld))) + return -EOPNOTSUPP; + if (unsupported(iconf, VNIC_F, fs->val.pfvf_vld, fs->mask.pfvf_vld) || + (is_field_set(fs->val.ovlan_vld, fs->mask.ovlan_vld) && + (iconf & VNIC_F))) + return -EOPNOTSUPP; + if (fs->val.pf > 0x7 || fs->val.vf > 0x7f) + return -ERANGE; + fs->mask.pf &= 0x7; + fs->mask.vf &= 0x7f; + + /* If the user is requesting that the filter action loop + * matching packets back out one of our ports, make sure that + * the egress port is in range. + */ + if (fs->action == FILTER_SWITCH && + fs->eport >= adapter->params.nports) + return -ERANGE; + + /* Don't allow various trivially obvious bogus out-of-range values... */ + if (fs->val.iport >= adapter->params.nports) + return -ERANGE; + + /* T4 doesn't support removing VLAN Tags for loop back filters. */ + if (is_t4(adapter->params.chip) && + fs->action == FILTER_SWITCH && + (fs->newvlan == VLAN_REMOVE || + fs->newvlan == VLAN_REWRITE)) + return -EOPNOTSUPP; + + if (fs->val.encap_vld && + CHELSIO_CHIP_VERSION(adapter->params.chip) < CHELSIO_T6) + return -EOPNOTSUPP; + return 0; +} + +static int get_filter_steerq(struct net_device *dev, + struct ch_filter_specification *fs) +{ + struct adapter *adapter = netdev2adap(dev); + int iq; + + /* If the user has requested steering matching Ingress Packets + * to a specific Queue Set, we need to make sure it's in range + * for the port and map that into the Absolute Queue ID of the + * Queue Set's Response Queue. + */ + if (!fs->dirsteer) { + if (fs->iq) + return -EINVAL; + iq = 0; + } else { + struct port_info *pi = netdev_priv(dev); + + /* If the iq id is greater than the number of qsets, + * then assume it is an absolute qid. + */ + if (fs->iq < pi->nqsets) + iq = adapter->sge.ethrxq[pi->first_qset + + fs->iq].rspq.abs_id; + else + iq = fs->iq; + } + + return iq; +} + +static int get_filter_count(struct adapter *adapter, unsigned int fidx, + u64 *pkts, u64 *bytes, bool hash) +{ + unsigned int tcb_base, tcbaddr; + unsigned int word_offset; + struct filter_entry *f; + __be64 be64_byte_count; + int ret; + + tcb_base = t4_read_reg(adapter, TP_CMM_TCB_BASE_A); + if (is_hashfilter(adapter) && hash) { + if (tid_out_of_range(&adapter->tids, fidx)) + return -E2BIG; + f = adapter->tids.tid_tab[fidx - adapter->tids.tid_base]; + if (!f) + return -EINVAL; + } else { + if ((fidx != (adapter->tids.nftids + adapter->tids.nsftids + + adapter->tids.nhpftids - 1)) && + fidx >= (adapter->tids.nftids + adapter->tids.nhpftids)) + return -E2BIG; + + if (fidx < adapter->tids.nhpftids) + f = &adapter->tids.hpftid_tab[fidx]; + else + f = &adapter->tids.ftid_tab[fidx - + adapter->tids.nhpftids]; + if (!f->valid) + return -EINVAL; + } + tcbaddr = tcb_base + f->tid * TCB_SIZE; + + spin_lock(&adapter->win0_lock); + if (is_t4(adapter->params.chip)) { + __be64 be64_count; + + /* T4 doesn't maintain byte counts in hw */ + *bytes = 0; + + /* Get pkts */ + word_offset = 4; + ret = t4_memory_rw(adapter, MEMWIN_NIC, MEM_EDC0, + tcbaddr + (word_offset * sizeof(__be32)), + sizeof(be64_count), + (__be32 *)&be64_count, + T4_MEMORY_READ); + if (ret < 0) + goto out; + *pkts = be64_to_cpu(be64_count); + } else { + __be32 be32_count; + + /* Get bytes */ + word_offset = 4; + ret = t4_memory_rw(adapter, MEMWIN_NIC, MEM_EDC0, + tcbaddr + (word_offset * sizeof(__be32)), + sizeof(be64_byte_count), + &be64_byte_count, + T4_MEMORY_READ); + if (ret < 0) + goto out; + *bytes = be64_to_cpu(be64_byte_count); + + /* Get pkts */ + word_offset = 6; + ret = t4_memory_rw(adapter, MEMWIN_NIC, MEM_EDC0, + tcbaddr + (word_offset * sizeof(__be32)), + sizeof(be32_count), + &be32_count, + T4_MEMORY_READ); + if (ret < 0) + goto out; + *pkts = (u64)be32_to_cpu(be32_count); + } + +out: + spin_unlock(&adapter->win0_lock); + return ret; +} + +int cxgb4_get_filter_counters(struct net_device *dev, unsigned int fidx, + u64 *hitcnt, u64 *bytecnt, bool hash) +{ + struct adapter *adapter = netdev2adap(dev); + + return get_filter_count(adapter, fidx, hitcnt, bytecnt, hash); +} + +static bool cxgb4_filter_prio_in_range(struct tid_info *t, u32 idx, u8 nslots, + u32 prio) +{ + struct filter_entry *prev_tab, *next_tab, *prev_fe, *next_fe; + u32 prev_ftid, next_ftid; + + /* Only insert the rule if both of the following conditions + * are met: + * 1. The immediate previous rule has priority <= @prio. + * 2. The immediate next rule has priority >= @prio. + */ + + /* High Priority (HPFILTER) region always has higher priority + * than normal FILTER region. So, all rules in HPFILTER region + * must have prio value <= rules in normal FILTER region. + */ + if (idx < t->nhpftids) { + /* Don't insert if there's a rule already present at @idx + * in HPFILTER region. + */ + if (test_bit(idx, t->hpftid_bmap)) + return false; + + next_tab = t->hpftid_tab; + next_ftid = find_next_bit(t->hpftid_bmap, t->nhpftids, idx); + if (next_ftid >= t->nhpftids) { + /* No next entry found in HPFILTER region. + * See if there's any next entry in normal + * FILTER region. + */ + next_ftid = find_first_bit(t->ftid_bmap, t->nftids); + if (next_ftid >= t->nftids) + next_ftid = idx; + else + next_tab = t->ftid_tab; + } + + /* Search for the closest previous filter entry in HPFILTER + * region. No need to search in normal FILTER region because + * there can never be any entry in normal FILTER region whose + * prio value is < last entry in HPFILTER region. + */ + prev_ftid = find_last_bit(t->hpftid_bmap, idx); + if (prev_ftid >= idx) + prev_ftid = idx; + + prev_tab = t->hpftid_tab; + } else { + idx -= t->nhpftids; + + /* Don't insert if there's a rule already present at @idx + * in normal FILTER region. + */ + if (test_bit(idx, t->ftid_bmap)) + return false; + + prev_tab = t->ftid_tab; + prev_ftid = find_last_bit(t->ftid_bmap, idx); + if (prev_ftid >= idx) { + /* No previous entry found in normal FILTER + * region. See if there's any previous entry + * in HPFILTER region. + */ + prev_ftid = find_last_bit(t->hpftid_bmap, t->nhpftids); + if (prev_ftid >= t->nhpftids) + prev_ftid = idx; + else + prev_tab = t->hpftid_tab; + } + + /* Search for the closest next filter entry in normal + * FILTER region. No need to search in HPFILTER region + * because there can never be any entry in HPFILTER + * region whose prio value is > first entry in normal + * FILTER region. + */ + next_ftid = find_next_bit(t->ftid_bmap, t->nftids, idx); + if (next_ftid >= t->nftids) + next_ftid = idx; + + next_tab = t->ftid_tab; + } + + next_fe = &next_tab[next_ftid]; + + /* See if the filter entry belongs to an IPv6 rule, which + * occupy 4 slots on T5 and 2 slots on T6. Adjust the + * reference to the previously inserted filter entry + * accordingly. + */ + prev_fe = &prev_tab[prev_ftid & ~(nslots - 1)]; + if (!prev_fe->fs.type) + prev_fe = &prev_tab[prev_ftid]; + + if ((prev_fe->valid && prev_fe->fs.tc_prio > prio) || + (next_fe->valid && next_fe->fs.tc_prio < prio)) + return false; + + return true; +} + +int cxgb4_get_free_ftid(struct net_device *dev, u8 family, bool hash_en, + u32 tc_prio) +{ + struct adapter *adap = netdev2adap(dev); + struct tid_info *t = &adap->tids; + u32 bmap_ftid, max_ftid; + struct filter_entry *f; + unsigned long *bmap; + bool found = false; + u8 i, cnt, n; + int ftid = 0; + + /* IPv4 occupy 1 slot. IPv6 occupy 2 slots on T6 and 4 slots + * on T5. + */ + n = 1; + if (family == PF_INET6) { + n++; + if (CHELSIO_CHIP_VERSION(adap->params.chip) < CHELSIO_T6) + n += 2; + } + + /* There are 3 filter regions available in hardware in + * following order of priority: + * + * 1. High Priority (HPFILTER) region (Highest Priority). + * 2. HASH region. + * 3. Normal FILTER region (Lowest Priority). + * + * Entries in HPFILTER and normal FILTER region have index + * 0 as the highest priority and the rules will be scanned + * in ascending order until either a rule hits or end of + * the region is reached. + * + * All HASH region entries have same priority. The set of + * fields to match in headers are pre-determined. The same + * set of header match fields must be compulsorily specified + * in all the rules wanting to get inserted in HASH region. + * Hence, HASH region is an exact-match region. A HASH is + * generated for a rule based on the values in the + * pre-determined set of header match fields. The generated + * HASH serves as an index into the HASH region. There can + * never be 2 rules having the same HASH. Hardware will + * compute a HASH for every incoming packet based on the + * values in the pre-determined set of header match fields + * and uses it as an index to check if there's a rule + * inserted in the HASH region at the specified index. If + * there's a rule inserted, then it's considered as a filter + * hit. Otherwise, it's a filter miss and normal FILTER region + * is scanned afterwards. + */ + + spin_lock_bh(&t->ftid_lock); + + ftid = (tc_prio <= t->nhpftids) ? 0 : t->nhpftids; + max_ftid = t->nftids + t->nhpftids; + while (ftid < max_ftid) { + if (ftid < t->nhpftids) { + /* If the new rule wants to get inserted into + * HPFILTER region, but its prio is greater + * than the rule with the highest prio in HASH + * region, or if there's not enough slots + * available in HPFILTER region, then skip + * trying to insert this rule into HPFILTER + * region and directly go to the next region. + */ + if ((t->tc_hash_tids_max_prio && + tc_prio > t->tc_hash_tids_max_prio) || + (ftid + n) > t->nhpftids) { + ftid = t->nhpftids; + continue; + } + + bmap = t->hpftid_bmap; + bmap_ftid = ftid; + } else if (hash_en) { + /* Ensure priority is >= last rule in HPFILTER + * region. + */ + ftid = find_last_bit(t->hpftid_bmap, t->nhpftids); + if (ftid < t->nhpftids) { + f = &t->hpftid_tab[ftid]; + if (f->valid && tc_prio < f->fs.tc_prio) + break; + } + + /* Ensure priority is <= first rule in normal + * FILTER region. + */ + ftid = find_first_bit(t->ftid_bmap, t->nftids); + if (ftid < t->nftids) { + f = &t->ftid_tab[ftid]; + if (f->valid && tc_prio > f->fs.tc_prio) + break; + } + + found = true; + ftid = t->nhpftids; + goto out_unlock; + } else { + /* If the new rule wants to get inserted into + * normal FILTER region, but its prio is less + * than the rule with the highest prio in HASH + * region, then reject the rule. + */ + if (t->tc_hash_tids_max_prio && + tc_prio < t->tc_hash_tids_max_prio) + break; + + if (ftid + n > max_ftid) + break; + + bmap = t->ftid_bmap; + bmap_ftid = ftid - t->nhpftids; + } + + cnt = 0; + for (i = 0; i < n; i++) { + if (test_bit(bmap_ftid + i, bmap)) + break; + cnt++; + } + + if (cnt == n) { + /* Ensure the new rule's prio doesn't conflict + * with existing rules. + */ + if (cxgb4_filter_prio_in_range(t, ftid, n, + tc_prio)) { + ftid &= ~(n - 1); + found = true; + break; + } + } + + ftid += n; + } + +out_unlock: + spin_unlock_bh(&t->ftid_lock); + return found ? ftid : -ENOMEM; +} + +static int cxgb4_set_ftid(struct tid_info *t, int fidx, int family, + unsigned int chip_ver) +{ + spin_lock_bh(&t->ftid_lock); + + if (test_bit(fidx, t->ftid_bmap)) { + spin_unlock_bh(&t->ftid_lock); + return -EBUSY; + } + + if (family == PF_INET) { + __set_bit(fidx, t->ftid_bmap); + } else { + if (chip_ver < CHELSIO_T6) + bitmap_allocate_region(t->ftid_bmap, fidx, 2); + else + bitmap_allocate_region(t->ftid_bmap, fidx, 1); + } + + spin_unlock_bh(&t->ftid_lock); + return 0; +} + +static int cxgb4_set_hpftid(struct tid_info *t, int fidx, int family) +{ + spin_lock_bh(&t->ftid_lock); + + if (test_bit(fidx, t->hpftid_bmap)) { + spin_unlock_bh(&t->ftid_lock); + return -EBUSY; + } + + if (family == PF_INET) + __set_bit(fidx, t->hpftid_bmap); + else + bitmap_allocate_region(t->hpftid_bmap, fidx, 1); + + spin_unlock_bh(&t->ftid_lock); + return 0; +} + +static void cxgb4_clear_ftid(struct tid_info *t, int fidx, int family, + unsigned int chip_ver) +{ + spin_lock_bh(&t->ftid_lock); + if (family == PF_INET) { + __clear_bit(fidx, t->ftid_bmap); + } else { + if (chip_ver < CHELSIO_T6) + bitmap_release_region(t->ftid_bmap, fidx, 2); + else + bitmap_release_region(t->ftid_bmap, fidx, 1); + } + spin_unlock_bh(&t->ftid_lock); +} + +static void cxgb4_clear_hpftid(struct tid_info *t, int fidx, int family) +{ + spin_lock_bh(&t->ftid_lock); + + if (family == PF_INET) + __clear_bit(fidx, t->hpftid_bmap); + else + bitmap_release_region(t->hpftid_bmap, fidx, 1); + + spin_unlock_bh(&t->ftid_lock); +} + +/* Delete the filter at a specified index. */ +static int del_filter_wr(struct adapter *adapter, int fidx) +{ + struct fw_filter_wr *fwr; + struct filter_entry *f; + struct sk_buff *skb; + unsigned int len; + + if (fidx < adapter->tids.nhpftids) + f = &adapter->tids.hpftid_tab[fidx]; + else + f = &adapter->tids.ftid_tab[fidx - adapter->tids.nhpftids]; + + len = sizeof(*fwr); + + skb = alloc_skb(len, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + fwr = __skb_put(skb, len); + t4_mk_filtdelwr(f->tid, fwr, adapter->sge.fw_evtq.abs_id); + + /* Mark the filter as "pending" and ship off the Filter Work Request. + * When we get the Work Request Reply we'll clear the pending status. + */ + f->pending = 1; + t4_mgmt_tx(adapter, skb); + return 0; +} + +/* Send a Work Request to write the filter at a specified index. We construct + * a Firmware Filter Work Request to have the work done and put the indicated + * filter into "pending" mode which will prevent any further actions against + * it till we get a reply from the firmware on the completion status of the + * request. + */ +int set_filter_wr(struct adapter *adapter, int fidx) +{ + struct fw_filter2_wr *fwr; + struct filter_entry *f; + struct sk_buff *skb; + + if (fidx < adapter->tids.nhpftids) + f = &adapter->tids.hpftid_tab[fidx]; + else + f = &adapter->tids.ftid_tab[fidx - adapter->tids.nhpftids]; + + skb = alloc_skb(sizeof(*fwr), GFP_KERNEL); + if (!skb) + return -ENOMEM; + + /* If the new filter requires loopback Destination MAC and/or VLAN + * rewriting then we need to allocate a Layer 2 Table (L2T) entry for + * the filter. + */ + if (f->fs.newdmac || f->fs.newvlan) { + /* allocate L2T entry for new filter */ + f->l2t = t4_l2t_alloc_switching(adapter, f->fs.vlan, + f->fs.eport, f->fs.dmac); + if (!f->l2t) { + kfree_skb(skb); + return -ENOMEM; + } + } + + /* If the new filter requires loopback Source MAC rewriting then + * we need to allocate a SMT entry for the filter. + */ + if (f->fs.newsmac) { + f->smt = cxgb4_smt_alloc_switching(f->dev, f->fs.smac); + if (!f->smt) { + if (f->l2t) { + cxgb4_l2t_release(f->l2t); + f->l2t = NULL; + } + kfree_skb(skb); + return -ENOMEM; + } + } + + fwr = __skb_put_zero(skb, sizeof(*fwr)); + + /* It would be nice to put most of the following in t4_hw.c but most + * of the work is translating the cxgbtool ch_filter_specification + * into the Work Request and the definition of that structure is + * currently in cxgbtool.h which isn't appropriate to pull into the + * common code. We may eventually try to come up with a more neutral + * filter specification structure but for now it's easiest to simply + * put this fairly direct code in line ... + */ + if (adapter->params.filter2_wr_support) + fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER2_WR)); + else + fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR)); + fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr) / 16)); + fwr->tid_to_iq = + htonl(FW_FILTER_WR_TID_V(f->tid) | + FW_FILTER_WR_RQTYPE_V(f->fs.type) | + FW_FILTER_WR_NOREPLY_V(0) | + FW_FILTER_WR_IQ_V(f->fs.iq)); + fwr->del_filter_to_l2tix = + htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) | + FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) | + FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) | + FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) | + FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) | + FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) | + FW_FILTER_WR_DMAC_V(f->fs.newdmac) | + FW_FILTER_WR_SMAC_V(f->fs.newsmac) | + FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT || + f->fs.newvlan == VLAN_REWRITE) | + FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE || + f->fs.newvlan == VLAN_REWRITE) | + FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) | + FW_FILTER_WR_TXCHAN_V(f->fs.eport) | + FW_FILTER_WR_PRIO_V(f->fs.prio) | + FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0)); + fwr->ethtype = htons(f->fs.val.ethtype); + fwr->ethtypem = htons(f->fs.mask.ethtype); + fwr->frag_to_ovlan_vldm = + (FW_FILTER_WR_FRAG_V(f->fs.val.frag) | + FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) | + FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) | + FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) | + FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) | + FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld)); + if (f->fs.newsmac) + fwr->smac_sel = f->smt->idx; + fwr->rx_chan_rx_rpl_iq = + htons(FW_FILTER_WR_RX_CHAN_V(0) | + FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id)); + fwr->maci_to_matchtypem = + htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) | + FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) | + FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) | + FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) | + FW_FILTER_WR_PORT_V(f->fs.val.iport) | + FW_FILTER_WR_PORTM_V(f->fs.mask.iport) | + FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) | + FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype)); + fwr->ptcl = f->fs.val.proto; + fwr->ptclm = f->fs.mask.proto; + fwr->ttyp = f->fs.val.tos; + fwr->ttypm = f->fs.mask.tos; + fwr->ivlan = htons(f->fs.val.ivlan); + fwr->ivlanm = htons(f->fs.mask.ivlan); + fwr->ovlan = htons(f->fs.val.ovlan); + fwr->ovlanm = htons(f->fs.mask.ovlan); + memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip)); + memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm)); + memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip)); + memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm)); + fwr->lp = htons(f->fs.val.lport); + fwr->lpm = htons(f->fs.mask.lport); + fwr->fp = htons(f->fs.val.fport); + fwr->fpm = htons(f->fs.mask.fport); + + if (adapter->params.filter2_wr_support) { + u8 *nat_lp = (u8 *)&f->fs.nat_lport; + u8 *nat_fp = (u8 *)&f->fs.nat_fport; + + fwr->natmode_to_ulp_type = + FW_FILTER2_WR_ULP_TYPE_V(f->fs.nat_mode ? + ULP_MODE_TCPDDP : + ULP_MODE_NONE) | + FW_FILTER2_WR_NATMODE_V(f->fs.nat_mode); + memcpy(fwr->newlip, f->fs.nat_lip, sizeof(fwr->newlip)); + memcpy(fwr->newfip, f->fs.nat_fip, sizeof(fwr->newfip)); + fwr->newlport = htons(nat_lp[1] | nat_lp[0] << 8); + fwr->newfport = htons(nat_fp[1] | nat_fp[0] << 8); + } + + /* Mark the filter as "pending" and ship off the Filter Work Request. + * When we get the Work Request Reply we'll clear the pending status. + */ + f->pending = 1; + set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3); + t4_ofld_send(adapter, skb); + return 0; +} + +/* Return an error number if the indicated filter isn't writable ... */ +int writable_filter(struct filter_entry *f) +{ + if (f->locked) + return -EPERM; + if (f->pending) + return -EBUSY; + + return 0; +} + +/* Delete the filter at the specified index (if valid). The checks for all + * the common problems with doing this like the filter being locked, currently + * pending in another operation, etc. + */ +int delete_filter(struct adapter *adapter, unsigned int fidx) +{ + struct filter_entry *f; + int ret; + + if (fidx >= adapter->tids.nftids + adapter->tids.nsftids + + adapter->tids.nhpftids) + return -EINVAL; + + if (fidx < adapter->tids.nhpftids) + f = &adapter->tids.hpftid_tab[fidx]; + else + f = &adapter->tids.ftid_tab[fidx - adapter->tids.nhpftids]; + ret = writable_filter(f); + if (ret) + return ret; + if (f->valid) + return del_filter_wr(adapter, fidx); + + return 0; +} + +/* Clear a filter and release any of its resources that we own. This also + * clears the filter's "pending" status. + */ +void clear_filter(struct adapter *adap, struct filter_entry *f) +{ + struct port_info *pi = netdev_priv(f->dev); + + /* If the new or old filter have loopback rewriting rules then we'll + * need to free any existing L2T, SMT, CLIP entries of filter + * rule. + */ + if (f->l2t) + cxgb4_l2t_release(f->l2t); + + if (f->smt) + cxgb4_smt_release(f->smt); + + if (f->fs.val.encap_vld && f->fs.val.ovlan_vld) + t4_free_encap_mac_filt(adap, pi->viid, + f->fs.val.ovlan & 0x1ff, 0); + + if ((f->fs.hash || is_t6(adap->params.chip)) && f->fs.type) + cxgb4_clip_release(f->dev, (const u32 *)&f->fs.val.lip, 1); + + /* The zeroing of the filter rule below clears the filter valid, + * pending, locked flags, l2t pointer, etc. so it's all we need for + * this operation. + */ + memset(f, 0, sizeof(*f)); +} + +void clear_all_filters(struct adapter *adapter) +{ + struct net_device *dev = adapter->port[0]; + unsigned int i; + + if (adapter->tids.hpftid_tab) { + struct filter_entry *f = &adapter->tids.hpftid_tab[0]; + + for (i = 0; i < adapter->tids.nhpftids; i++, f++) + if (f->valid || f->pending) + cxgb4_del_filter(dev, i, &f->fs); + } + + if (adapter->tids.ftid_tab) { + struct filter_entry *f = &adapter->tids.ftid_tab[0]; + unsigned int max_ftid = adapter->tids.nftids + + adapter->tids.nsftids + + adapter->tids.nhpftids; + + /* Clear all TCAM filters */ + for (i = adapter->tids.nhpftids; i < max_ftid; i++, f++) + if (f->valid || f->pending) + cxgb4_del_filter(dev, i, &f->fs); + } + + /* Clear all hash filters */ + if (is_hashfilter(adapter) && adapter->tids.tid_tab) { + struct filter_entry *f; + unsigned int sb; + + for (i = adapter->tids.hash_base; + i <= adapter->tids.ntids; i++) { + f = (struct filter_entry *) + adapter->tids.tid_tab[i]; + + if (f && (f->valid || f->pending)) + cxgb4_del_filter(dev, f->tid, &f->fs); + } + + sb = adapter->tids.stid_base; + for (i = 0; i < sb; i++) { + f = (struct filter_entry *)adapter->tids.tid_tab[i]; + + if (f && (f->valid || f->pending)) + cxgb4_del_filter(dev, f->tid, &f->fs); + } + } +} + +/* Fill up default masks for set match fields. */ +static void fill_default_mask(struct ch_filter_specification *fs) +{ + unsigned int lip = 0, lip_mask = 0; + unsigned int fip = 0, fip_mask = 0; + unsigned int i; + + if (fs->val.iport && !fs->mask.iport) + fs->mask.iport |= ~0; + if (fs->val.fcoe && !fs->mask.fcoe) + fs->mask.fcoe |= ~0; + if (fs->val.matchtype && !fs->mask.matchtype) + fs->mask.matchtype |= ~0; + if (fs->val.macidx && !fs->mask.macidx) + fs->mask.macidx |= ~0; + if (fs->val.ethtype && !fs->mask.ethtype) + fs->mask.ethtype |= ~0; + if (fs->val.ivlan && !fs->mask.ivlan) + fs->mask.ivlan |= ~0; + if (fs->val.ovlan && !fs->mask.ovlan) + fs->mask.ovlan |= ~0; + if (fs->val.frag && !fs->mask.frag) + fs->mask.frag |= ~0; + if (fs->val.tos && !fs->mask.tos) + fs->mask.tos |= ~0; + if (fs->val.proto && !fs->mask.proto) + fs->mask.proto |= ~0; + if (fs->val.pfvf_vld && !fs->mask.pfvf_vld) + fs->mask.pfvf_vld |= ~0; + if (fs->val.pf && !fs->mask.pf) + fs->mask.pf |= ~0; + if (fs->val.vf && !fs->mask.vf) + fs->mask.vf |= ~0; + + for (i = 0; i < ARRAY_SIZE(fs->val.lip); i++) { + lip |= fs->val.lip[i]; + lip_mask |= fs->mask.lip[i]; + fip |= fs->val.fip[i]; + fip_mask |= fs->mask.fip[i]; + } + + if (lip && !lip_mask) + memset(fs->mask.lip, ~0, sizeof(fs->mask.lip)); + + if (fip && !fip_mask) + memset(fs->mask.fip, ~0, sizeof(fs->mask.lip)); + + if (fs->val.lport && !fs->mask.lport) + fs->mask.lport = ~0; + if (fs->val.fport && !fs->mask.fport) + fs->mask.fport = ~0; +} + +static bool is_addr_all_mask(u8 *ipmask, int family) +{ + if (family == AF_INET) { + struct in_addr *addr; + + addr = (struct in_addr *)ipmask; + if (addr->s_addr == htonl(0xffffffff)) + return true; + } else if (family == AF_INET6) { + struct in6_addr *addr6; + + addr6 = (struct in6_addr *)ipmask; + if (addr6->s6_addr32[0] == htonl(0xffffffff) && + addr6->s6_addr32[1] == htonl(0xffffffff) && + addr6->s6_addr32[2] == htonl(0xffffffff) && + addr6->s6_addr32[3] == htonl(0xffffffff)) + return true; + } + return false; +} + +static bool is_inaddr_any(u8 *ip, int family) +{ + int addr_type; + + if (family == AF_INET) { + struct in_addr *addr; + + addr = (struct in_addr *)ip; + if (addr->s_addr == htonl(INADDR_ANY)) + return true; + } else if (family == AF_INET6) { + struct in6_addr *addr6; + + addr6 = (struct in6_addr *)ip; + addr_type = ipv6_addr_type((const struct in6_addr *) + &addr6); + if (addr_type == IPV6_ADDR_ANY) + return true; + } + return false; +} + +bool is_filter_exact_match(struct adapter *adap, + struct ch_filter_specification *fs) +{ + struct tp_params *tp = &adap->params.tp; + u64 hash_filter_mask = tp->hash_filter_mask; + u64 ntuple_mask = 0; + + if (!is_hashfilter(adap)) + return false; + + if ((atomic_read(&adap->tids.hash_tids_in_use) + + atomic_read(&adap->tids.tids_in_use)) >= + (adap->tids.nhash + (adap->tids.stid_base - adap->tids.tid_base))) + return false; + + /* Keep tunnel VNI match disabled for hash-filters for now */ + if (fs->mask.encap_vld) + return false; + + if (fs->type) { + if (is_inaddr_any(fs->val.fip, AF_INET6) || + !is_addr_all_mask(fs->mask.fip, AF_INET6)) + return false; + + if (is_inaddr_any(fs->val.lip, AF_INET6) || + !is_addr_all_mask(fs->mask.lip, AF_INET6)) + return false; + } else { + if (is_inaddr_any(fs->val.fip, AF_INET) || + !is_addr_all_mask(fs->mask.fip, AF_INET)) + return false; + + if (is_inaddr_any(fs->val.lip, AF_INET) || + !is_addr_all_mask(fs->mask.lip, AF_INET)) + return false; + } + + if (!fs->val.lport || fs->mask.lport != 0xffff) + return false; + + if (!fs->val.fport || fs->mask.fport != 0xffff) + return false; + + /* calculate tuple mask and compare with mask configured in hw */ + if (tp->fcoe_shift >= 0) + ntuple_mask |= (u64)fs->mask.fcoe << tp->fcoe_shift; + + if (tp->port_shift >= 0) + ntuple_mask |= (u64)fs->mask.iport << tp->port_shift; + + if (tp->vnic_shift >= 0) { + if ((adap->params.tp.ingress_config & VNIC_F)) + ntuple_mask |= (u64)fs->mask.pfvf_vld << tp->vnic_shift; + else + ntuple_mask |= (u64)fs->mask.ovlan_vld << + tp->vnic_shift; + } + + if (tp->vlan_shift >= 0) + ntuple_mask |= (u64)fs->mask.ivlan << tp->vlan_shift; + + if (tp->tos_shift >= 0) + ntuple_mask |= (u64)fs->mask.tos << tp->tos_shift; + + if (tp->protocol_shift >= 0) + ntuple_mask |= (u64)fs->mask.proto << tp->protocol_shift; + + if (tp->ethertype_shift >= 0) + ntuple_mask |= (u64)fs->mask.ethtype << tp->ethertype_shift; + + if (tp->macmatch_shift >= 0) + ntuple_mask |= (u64)fs->mask.macidx << tp->macmatch_shift; + + if (tp->matchtype_shift >= 0) + ntuple_mask |= (u64)fs->mask.matchtype << tp->matchtype_shift; + + if (tp->frag_shift >= 0) + ntuple_mask |= (u64)fs->mask.frag << tp->frag_shift; + + if (ntuple_mask != hash_filter_mask) + return false; + + return true; +} + +static u64 hash_filter_ntuple(struct ch_filter_specification *fs, + struct net_device *dev) +{ + struct adapter *adap = netdev2adap(dev); + struct tp_params *tp = &adap->params.tp; + u64 ntuple = 0; + + /* Initialize each of the fields which we care about which are present + * in the Compressed Filter Tuple. + */ + if (tp->vlan_shift >= 0 && fs->mask.ivlan) + ntuple |= (FT_VLAN_VLD_F | fs->val.ivlan) << tp->vlan_shift; + + if (tp->port_shift >= 0 && fs->mask.iport) + ntuple |= (u64)fs->val.iport << tp->port_shift; + + if (tp->protocol_shift >= 0) { + if (!fs->val.proto) + ntuple |= (u64)IPPROTO_TCP << tp->protocol_shift; + else + ntuple |= (u64)fs->val.proto << tp->protocol_shift; + } + + if (tp->tos_shift >= 0 && fs->mask.tos) + ntuple |= (u64)(fs->val.tos) << tp->tos_shift; + + if (tp->vnic_shift >= 0) { + if ((adap->params.tp.ingress_config & USE_ENC_IDX_F) && + fs->mask.encap_vld) + ntuple |= (u64)((fs->val.encap_vld << 16) | + (fs->val.ovlan)) << tp->vnic_shift; + else if ((adap->params.tp.ingress_config & VNIC_F) && + fs->mask.pfvf_vld) + ntuple |= (u64)((fs->val.pfvf_vld << 16) | + (fs->val.pf << 13) | + (fs->val.vf)) << tp->vnic_shift; + else + ntuple |= (u64)((fs->val.ovlan_vld << 16) | + (fs->val.ovlan)) << tp->vnic_shift; + } + + if (tp->macmatch_shift >= 0 && fs->mask.macidx) + ntuple |= (u64)(fs->val.macidx) << tp->macmatch_shift; + + if (tp->ethertype_shift >= 0 && fs->mask.ethtype) + ntuple |= (u64)(fs->val.ethtype) << tp->ethertype_shift; + + if (tp->matchtype_shift >= 0 && fs->mask.matchtype) + ntuple |= (u64)(fs->val.matchtype) << tp->matchtype_shift; + + if (tp->frag_shift >= 0 && fs->mask.frag) + ntuple |= (u64)(fs->val.frag) << tp->frag_shift; + + if (tp->fcoe_shift >= 0 && fs->mask.fcoe) + ntuple |= (u64)(fs->val.fcoe) << tp->fcoe_shift; + return ntuple; +} + +static void mk_act_open_req6(struct filter_entry *f, struct sk_buff *skb, + unsigned int qid_filterid, struct adapter *adap) +{ + struct cpl_t6_act_open_req6 *t6req = NULL; + struct cpl_act_open_req6 *req = NULL; + + t6req = (struct cpl_t6_act_open_req6 *)__skb_put(skb, sizeof(*t6req)); + INIT_TP_WR(t6req, 0); + req = (struct cpl_act_open_req6 *)t6req; + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_filterid)); + req->local_port = cpu_to_be16(f->fs.val.lport); + req->peer_port = cpu_to_be16(f->fs.val.fport); + req->local_ip_hi = *(__be64 *)(&f->fs.val.lip); + req->local_ip_lo = *(((__be64 *)&f->fs.val.lip) + 1); + req->peer_ip_hi = *(__be64 *)(&f->fs.val.fip); + req->peer_ip_lo = *(((__be64 *)&f->fs.val.fip) + 1); + req->opt0 = cpu_to_be64(NAGLE_V(f->fs.newvlan == VLAN_REMOVE || + f->fs.newvlan == VLAN_REWRITE) | + DELACK_V(f->fs.hitcnts) | + L2T_IDX_V(f->l2t ? f->l2t->idx : 0) | + SMAC_SEL_V((cxgb4_port_viid(f->dev) & + 0x7F) << 1) | + TX_CHAN_V(f->fs.eport) | + NO_CONG_V(f->fs.rpttid) | + ULP_MODE_V(f->fs.nat_mode ? + ULP_MODE_TCPDDP : ULP_MODE_NONE) | + TCAM_BYPASS_F | NON_OFFLOAD_F); + t6req->params = cpu_to_be64(FILTER_TUPLE_V(hash_filter_ntuple(&f->fs, + f->dev))); + t6req->opt2 = htonl(RSS_QUEUE_VALID_F | + RSS_QUEUE_V(f->fs.iq) | + TX_QUEUE_V(f->fs.nat_mode) | + T5_OPT_2_VALID_F | + RX_CHANNEL_V(cxgb4_port_e2cchan(f->dev)) | + PACE_V((f->fs.maskhash) | + ((f->fs.dirsteerhash) << 1))); +} + +static void mk_act_open_req(struct filter_entry *f, struct sk_buff *skb, + unsigned int qid_filterid, struct adapter *adap) +{ + struct cpl_t6_act_open_req *t6req = NULL; + struct cpl_act_open_req *req = NULL; + + t6req = (struct cpl_t6_act_open_req *)__skb_put(skb, sizeof(*t6req)); + INIT_TP_WR(t6req, 0); + req = (struct cpl_act_open_req *)t6req; + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_filterid)); + req->local_port = cpu_to_be16(f->fs.val.lport); + req->peer_port = cpu_to_be16(f->fs.val.fport); + memcpy(&req->local_ip, f->fs.val.lip, 4); + memcpy(&req->peer_ip, f->fs.val.fip, 4); + req->opt0 = cpu_to_be64(NAGLE_V(f->fs.newvlan == VLAN_REMOVE || + f->fs.newvlan == VLAN_REWRITE) | + DELACK_V(f->fs.hitcnts) | + L2T_IDX_V(f->l2t ? f->l2t->idx : 0) | + SMAC_SEL_V((cxgb4_port_viid(f->dev) & + 0x7F) << 1) | + TX_CHAN_V(f->fs.eport) | + NO_CONG_V(f->fs.rpttid) | + ULP_MODE_V(f->fs.nat_mode ? + ULP_MODE_TCPDDP : ULP_MODE_NONE) | + TCAM_BYPASS_F | NON_OFFLOAD_F); + + t6req->params = cpu_to_be64(FILTER_TUPLE_V(hash_filter_ntuple(&f->fs, + f->dev))); + t6req->opt2 = htonl(RSS_QUEUE_VALID_F | + RSS_QUEUE_V(f->fs.iq) | + TX_QUEUE_V(f->fs.nat_mode) | + T5_OPT_2_VALID_F | + RX_CHANNEL_V(cxgb4_port_e2cchan(f->dev)) | + PACE_V((f->fs.maskhash) | + ((f->fs.dirsteerhash) << 1))); +} + +static int cxgb4_set_hash_filter(struct net_device *dev, + struct ch_filter_specification *fs, + struct filter_ctx *ctx) +{ + struct adapter *adapter = netdev2adap(dev); + struct port_info *pi = netdev_priv(dev); + struct tid_info *t = &adapter->tids; + struct filter_entry *f; + struct sk_buff *skb; + int iq, atid, size; + int ret = 0; + u32 iconf; + + fill_default_mask(fs); + ret = validate_filter(dev, fs); + if (ret) + return ret; + + iq = get_filter_steerq(dev, fs); + if (iq < 0) + return iq; + + f = kzalloc(sizeof(*f), GFP_KERNEL); + if (!f) + return -ENOMEM; + + f->fs = *fs; + f->ctx = ctx; + f->dev = dev; + f->fs.iq = iq; + + /* If the new filter requires loopback Destination MAC and/or VLAN + * rewriting then we need to allocate a Layer 2 Table (L2T) entry for + * the filter. + */ + if (f->fs.newdmac || f->fs.newvlan) { + /* allocate L2T entry for new filter */ + f->l2t = t4_l2t_alloc_switching(adapter, f->fs.vlan, + f->fs.eport, f->fs.dmac); + if (!f->l2t) { + ret = -ENOMEM; + goto out_err; + } + } + + /* If the new filter requires loopback Source MAC rewriting then + * we need to allocate a SMT entry for the filter. + */ + if (f->fs.newsmac) { + f->smt = cxgb4_smt_alloc_switching(f->dev, f->fs.smac); + if (!f->smt) { + if (f->l2t) { + cxgb4_l2t_release(f->l2t); + f->l2t = NULL; + } + ret = -ENOMEM; + goto free_l2t; + } + } + + atid = cxgb4_alloc_atid(t, f); + if (atid < 0) { + ret = atid; + goto free_smt; + } + + iconf = adapter->params.tp.ingress_config; + if (iconf & VNIC_F) { + f->fs.val.ovlan = (fs->val.pf << 13) | fs->val.vf; + f->fs.mask.ovlan = (fs->mask.pf << 13) | fs->mask.vf; + f->fs.val.ovlan_vld = fs->val.pfvf_vld; + f->fs.mask.ovlan_vld = fs->mask.pfvf_vld; + } else if (iconf & USE_ENC_IDX_F) { + if (f->fs.val.encap_vld) { + struct port_info *pi = netdev_priv(f->dev); + static const u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 }; + + /* allocate MPS TCAM entry */ + ret = t4_alloc_encap_mac_filt(adapter, pi->viid, + match_all_mac, + match_all_mac, + f->fs.val.vni, + f->fs.mask.vni, + 0, 1, 1); + if (ret < 0) + goto free_atid; + + f->fs.val.ovlan = ret; + f->fs.mask.ovlan = 0xffff; + f->fs.val.ovlan_vld = 1; + f->fs.mask.ovlan_vld = 1; + } + } + + size = sizeof(struct cpl_t6_act_open_req); + if (f->fs.type) { + ret = cxgb4_clip_get(f->dev, (const u32 *)&f->fs.val.lip, 1); + if (ret) + goto free_mps; + + skb = alloc_skb(size, GFP_KERNEL); + if (!skb) { + ret = -ENOMEM; + goto free_clip; + } + + mk_act_open_req6(f, skb, + ((adapter->sge.fw_evtq.abs_id << 14) | atid), + adapter); + } else { + skb = alloc_skb(size, GFP_KERNEL); + if (!skb) { + ret = -ENOMEM; + goto free_mps; + } + + mk_act_open_req(f, skb, + ((adapter->sge.fw_evtq.abs_id << 14) | atid), + adapter); + } + + f->pending = 1; + set_wr_txq(skb, CPL_PRIORITY_SETUP, f->fs.val.iport & 0x3); + t4_ofld_send(adapter, skb); + return 0; + +free_clip: + cxgb4_clip_release(f->dev, (const u32 *)&f->fs.val.lip, 1); + +free_mps: + if (f->fs.val.encap_vld && f->fs.val.ovlan_vld) + t4_free_encap_mac_filt(adapter, pi->viid, f->fs.val.ovlan, 1); + +free_atid: + cxgb4_free_atid(t, atid); + +free_smt: + if (f->smt) { + cxgb4_smt_release(f->smt); + f->smt = NULL; + } + +free_l2t: + if (f->l2t) { + cxgb4_l2t_release(f->l2t); + f->l2t = NULL; + } + +out_err: + kfree(f); + return ret; +} + +/* Check a Chelsio Filter Request for validity, convert it into our internal + * format and send it to the hardware. Return 0 on success, an error number + * otherwise. We attach any provided filter operation context to the internal + * filter specification in order to facilitate signaling completion of the + * operation. + */ +int __cxgb4_set_filter(struct net_device *dev, int ftid, + struct ch_filter_specification *fs, + struct filter_ctx *ctx) +{ + struct adapter *adapter = netdev2adap(dev); + unsigned int max_fidx, fidx, chip_ver; + int iq, ret, filter_id = ftid; + struct filter_entry *f, *tab; + u32 iconf; + + chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip); + if (fs->hash) { + if (is_hashfilter(adapter)) + return cxgb4_set_hash_filter(dev, fs, ctx); + netdev_err(dev, "%s: Exact-match filters only supported with Hash Filter configuration\n", + __func__); + return -EINVAL; + } + + max_fidx = adapter->tids.nftids + adapter->tids.nhpftids; + if (filter_id != (max_fidx + adapter->tids.nsftids - 1) && + filter_id >= max_fidx) + return -E2BIG; + + fill_default_mask(fs); + + ret = validate_filter(dev, fs); + if (ret) + return ret; + + iq = get_filter_steerq(dev, fs); + if (iq < 0) + return iq; + + if (fs->prio) { + tab = &adapter->tids.hpftid_tab[0]; + } else { + tab = &adapter->tids.ftid_tab[0]; + filter_id = ftid - adapter->tids.nhpftids; + } + + /* IPv6 filters occupy four slots and must be aligned on + * four-slot boundaries. IPv4 filters only occupy a single + * slot and have no alignment requirements but writing a new + * IPv4 filter into the middle of an existing IPv6 filter + * requires clearing the old IPv6 filter and hence we prevent + * insertion. + */ + if (fs->type == 0) { /* IPv4 */ + /* For T6, If our IPv4 filter isn't being written to a + * multiple of two filter index and there's an IPv6 + * filter at the multiple of 2 base slot, then we need + * to delete that IPv6 filter ... + * For adapters below T6, IPv6 filter occupies 4 entries. + * Hence we need to delete the filter in multiple of 4 slot. + */ + if (chip_ver < CHELSIO_T6) + fidx = filter_id & ~0x3; + else + fidx = filter_id & ~0x1; + + if (fidx != filter_id && tab[fidx].fs.type) { + f = &tab[fidx]; + if (f->valid) { + dev_err(adapter->pdev_dev, + "Invalid location. IPv6 requires 4 slots and is occupying slots %u to %u\n", + fidx, fidx + 3); + return -EINVAL; + } + } + } else { /* IPv6 */ + if (chip_ver < CHELSIO_T6) { + /* Ensure that the IPv6 filter is aligned on a + * multiple of 4 boundary. + */ + if (filter_id & 0x3) { + dev_err(adapter->pdev_dev, + "Invalid location. IPv6 must be aligned on a 4-slot boundary\n"); + return -EINVAL; + } + + /* Check all except the base overlapping IPv4 filter + * slots. + */ + for (fidx = filter_id + 1; fidx < filter_id + 4; + fidx++) { + f = &tab[fidx]; + if (f->valid) { + dev_err(adapter->pdev_dev, + "Invalid location. IPv6 requires 4 slots and an IPv4 filter exists at %u\n", + fidx); + return -EBUSY; + } + } + } else { + /* For T6, CLIP being enabled, IPv6 filter would occupy + * 2 entries. + */ + if (filter_id & 0x1) + return -EINVAL; + /* Check overlapping IPv4 filter slot */ + fidx = filter_id + 1; + f = &tab[fidx]; + if (f->valid) { + pr_err("%s: IPv6 filter requires 2 indices. IPv4 filter already present at %d. Please remove IPv4 filter first.\n", + __func__, fidx); + return -EBUSY; + } + } + } + + /* Check to make sure that provided filter index is not + * already in use by someone else + */ + f = &tab[filter_id]; + if (f->valid) + return -EBUSY; + + if (fs->prio) { + fidx = filter_id + adapter->tids.hpftid_base; + ret = cxgb4_set_hpftid(&adapter->tids, filter_id, + fs->type ? PF_INET6 : PF_INET); + } else { + fidx = filter_id + adapter->tids.ftid_base; + ret = cxgb4_set_ftid(&adapter->tids, filter_id, + fs->type ? PF_INET6 : PF_INET, + chip_ver); + } + + if (ret) + return ret; + + /* Check t make sure the filter requested is writable ... */ + ret = writable_filter(f); + if (ret) + goto free_tid; + + if (is_t6(adapter->params.chip) && fs->type && + ipv6_addr_type((const struct in6_addr *)fs->val.lip) != + IPV6_ADDR_ANY) { + ret = cxgb4_clip_get(dev, (const u32 *)&fs->val.lip, 1); + if (ret) + goto free_tid; + } + + /* Convert the filter specification into our internal format. + * We copy the PF/VF specification into the Outer VLAN field + * here so the rest of the code -- including the interface to + * the firmware -- doesn't have to constantly do these checks. + */ + f->fs = *fs; + f->fs.iq = iq; + f->dev = dev; + + iconf = adapter->params.tp.ingress_config; + if (iconf & VNIC_F) { + f->fs.val.ovlan = (fs->val.pf << 13) | fs->val.vf; + f->fs.mask.ovlan = (fs->mask.pf << 13) | fs->mask.vf; + f->fs.val.ovlan_vld = fs->val.pfvf_vld; + f->fs.mask.ovlan_vld = fs->mask.pfvf_vld; + } else if (iconf & USE_ENC_IDX_F) { + if (f->fs.val.encap_vld) { + struct port_info *pi = netdev_priv(f->dev); + static const u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 }; + + /* allocate MPS TCAM entry */ + ret = t4_alloc_encap_mac_filt(adapter, pi->viid, + match_all_mac, + match_all_mac, + f->fs.val.vni, + f->fs.mask.vni, + 0, 1, 1); + if (ret < 0) + goto free_tid; + + f->fs.val.ovlan = ret; + f->fs.mask.ovlan = 0x1ff; + f->fs.val.ovlan_vld = 1; + f->fs.mask.ovlan_vld = 1; + } + } + + /* Attempt to set the filter. If we don't succeed, we clear + * it and return the failure. + */ + f->ctx = ctx; + f->tid = fidx; /* Save the actual tid */ + ret = set_filter_wr(adapter, ftid); + if (ret) + goto free_tid; + + return ret; + +free_tid: + if (f->fs.prio) + cxgb4_clear_hpftid(&adapter->tids, filter_id, + fs->type ? PF_INET6 : PF_INET); + else + cxgb4_clear_ftid(&adapter->tids, filter_id, + fs->type ? PF_INET6 : PF_INET, + chip_ver); + + clear_filter(adapter, f); + return ret; +} + +static int cxgb4_del_hash_filter(struct net_device *dev, int filter_id, + struct filter_ctx *ctx) +{ + struct adapter *adapter = netdev2adap(dev); + struct tid_info *t = &adapter->tids; + struct cpl_abort_req *abort_req; + struct cpl_abort_rpl *abort_rpl; + struct cpl_set_tcb_field *req; + struct ulptx_idata *aligner; + struct work_request_hdr *wr; + struct filter_entry *f; + struct sk_buff *skb; + unsigned int wrlen; + int ret; + + netdev_dbg(dev, "%s: filter_id = %d ; nftids = %d\n", + __func__, filter_id, adapter->tids.nftids); + + if (tid_out_of_range(t, filter_id)) + return -E2BIG; + + f = lookup_tid(t, filter_id); + if (!f) { + netdev_err(dev, "%s: no filter entry for filter_id = %d", + __func__, filter_id); + return -EINVAL; + } + + ret = writable_filter(f); + if (ret) + return ret; + + if (!f->valid) + return -EINVAL; + + f->ctx = ctx; + f->pending = 1; + wrlen = roundup(sizeof(*wr) + (sizeof(*req) + sizeof(*aligner)) + + sizeof(*abort_req) + sizeof(*abort_rpl), 16); + skb = alloc_skb(wrlen, GFP_KERNEL); + if (!skb) { + netdev_err(dev, "%s: could not allocate skb ..\n", __func__); + return -ENOMEM; + } + set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3); + req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen); + INIT_ULPTX_WR(req, wrlen, 0, 0); + wr = (struct work_request_hdr *)req; + wr++; + req = (struct cpl_set_tcb_field *)wr; + mk_set_tcb_ulp(f, req, TCB_RSS_INFO_W, TCB_RSS_INFO_V(TCB_RSS_INFO_M), + TCB_RSS_INFO_V(adapter->sge.fw_evtq.abs_id), 0, 1); + aligner = (struct ulptx_idata *)(req + 1); + abort_req = (struct cpl_abort_req *)(aligner + 1); + mk_abort_req_ulp(abort_req, f->tid); + abort_rpl = (struct cpl_abort_rpl *)(abort_req + 1); + mk_abort_rpl_ulp(abort_rpl, f->tid); + t4_ofld_send(adapter, skb); + return 0; +} + +/* Check a delete filter request for validity and send it to the hardware. + * Return 0 on success, an error number otherwise. We attach any provided + * filter operation context to the internal filter specification in order to + * facilitate signaling completion of the operation. + */ +int __cxgb4_del_filter(struct net_device *dev, int filter_id, + struct ch_filter_specification *fs, + struct filter_ctx *ctx) +{ + struct adapter *adapter = netdev2adap(dev); + unsigned int max_fidx, chip_ver; + struct filter_entry *f; + int ret; + + chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip); + if (fs && fs->hash) { + if (is_hashfilter(adapter)) + return cxgb4_del_hash_filter(dev, filter_id, ctx); + netdev_err(dev, "%s: Exact-match filters only supported with Hash Filter configuration\n", + __func__); + return -EINVAL; + } + + max_fidx = adapter->tids.nftids + adapter->tids.nhpftids; + if (filter_id != (max_fidx + adapter->tids.nsftids - 1) && + filter_id >= max_fidx) + return -E2BIG; + + if (filter_id < adapter->tids.nhpftids) + f = &adapter->tids.hpftid_tab[filter_id]; + else + f = &adapter->tids.ftid_tab[filter_id - adapter->tids.nhpftids]; + + ret = writable_filter(f); + if (ret) + return ret; + + if (f->valid) { + f->ctx = ctx; + if (f->fs.prio) + cxgb4_clear_hpftid(&adapter->tids, + f->tid - adapter->tids.hpftid_base, + f->fs.type ? PF_INET6 : PF_INET); + else + cxgb4_clear_ftid(&adapter->tids, + f->tid - adapter->tids.ftid_base, + f->fs.type ? PF_INET6 : PF_INET, + chip_ver); + return del_filter_wr(adapter, filter_id); + } + + /* If the caller has passed in a Completion Context then we need to + * mark it as a successful completion so they don't stall waiting + * for it. + */ + if (ctx) { + ctx->result = 0; + complete(&ctx->completion); + } + return ret; +} + +int cxgb4_set_filter(struct net_device *dev, int filter_id, + struct ch_filter_specification *fs) +{ + struct filter_ctx ctx; + int ret; + + init_completion(&ctx.completion); + + ret = __cxgb4_set_filter(dev, filter_id, fs, &ctx); + if (ret) + goto out; + + /* Wait for reply */ + ret = wait_for_completion_timeout(&ctx.completion, 10 * HZ); + if (!ret) + return -ETIMEDOUT; + + ret = ctx.result; +out: + return ret; +} + +int cxgb4_del_filter(struct net_device *dev, int filter_id, + struct ch_filter_specification *fs) +{ + struct filter_ctx ctx; + int ret; + + if (netdev2adap(dev)->flags & CXGB4_SHUTTING_DOWN) + return 0; + + init_completion(&ctx.completion); + + ret = __cxgb4_del_filter(dev, filter_id, fs, &ctx); + if (ret) + goto out; + + /* Wait for reply */ + ret = wait_for_completion_timeout(&ctx.completion, 10 * HZ); + if (!ret) + return -ETIMEDOUT; + + ret = ctx.result; +out: + return ret; +} + +static int configure_filter_tcb(struct adapter *adap, unsigned int tid, + struct filter_entry *f) +{ + if (f->fs.hitcnts) { + set_tcb_field(adap, f, tid, TCB_TIMESTAMP_W, + TCB_TIMESTAMP_V(TCB_TIMESTAMP_M), + TCB_TIMESTAMP_V(0ULL), + 1); + set_tcb_field(adap, f, tid, TCB_RTT_TS_RECENT_AGE_W, + TCB_RTT_TS_RECENT_AGE_V(TCB_RTT_TS_RECENT_AGE_M), + TCB_RTT_TS_RECENT_AGE_V(0ULL), + 1); + } + + if (f->fs.newdmac) + set_tcb_tflag(adap, f, tid, TF_CCTRL_ECE_S, 1, + 1); + + if (f->fs.newvlan == VLAN_INSERT || + f->fs.newvlan == VLAN_REWRITE) + set_tcb_tflag(adap, f, tid, TF_CCTRL_RFR_S, 1, + 1); + if (f->fs.newsmac) + configure_filter_smac(adap, f); + + if (f->fs.nat_mode) { + switch (f->fs.nat_mode) { + case NAT_MODE_DIP: + set_nat_params(adap, f, tid, true, false, false, false); + break; + + case NAT_MODE_DIP_DP: + set_nat_params(adap, f, tid, true, false, true, false); + break; + + case NAT_MODE_DIP_DP_SIP: + set_nat_params(adap, f, tid, true, true, true, false); + break; + case NAT_MODE_DIP_DP_SP: + set_nat_params(adap, f, tid, true, false, true, true); + break; + + case NAT_MODE_SIP_SP: + set_nat_params(adap, f, tid, false, true, false, true); + break; + + case NAT_MODE_DIP_SIP_SP: + set_nat_params(adap, f, tid, true, true, false, true); + break; + + case NAT_MODE_ALL: + set_nat_params(adap, f, tid, true, true, true, true); + break; + + default: + pr_err("%s: Invalid NAT mode: %d\n", + __func__, f->fs.nat_mode); + return -EINVAL; + } + } + return 0; +} + +void hash_del_filter_rpl(struct adapter *adap, + const struct cpl_abort_rpl_rss *rpl) +{ + unsigned int status = rpl->status; + struct tid_info *t = &adap->tids; + unsigned int tid = GET_TID(rpl); + struct filter_ctx *ctx = NULL; + struct filter_entry *f; + + dev_dbg(adap->pdev_dev, "%s: status = %u; tid = %u\n", + __func__, status, tid); + + f = lookup_tid(t, tid); + if (!f) { + dev_err(adap->pdev_dev, "%s:could not find filter entry", + __func__); + return; + } + ctx = f->ctx; + f->ctx = NULL; + clear_filter(adap, f); + cxgb4_remove_tid(t, 0, tid, 0); + kfree(f); + if (ctx) { + ctx->result = 0; + complete(&ctx->completion); + } +} + +void hash_filter_rpl(struct adapter *adap, const struct cpl_act_open_rpl *rpl) +{ + unsigned int ftid = TID_TID_G(AOPEN_ATID_G(ntohl(rpl->atid_status))); + unsigned int status = AOPEN_STATUS_G(ntohl(rpl->atid_status)); + struct tid_info *t = &adap->tids; + unsigned int tid = GET_TID(rpl); + struct filter_ctx *ctx = NULL; + struct filter_entry *f; + + dev_dbg(adap->pdev_dev, "%s: tid = %u; atid = %u; status = %u\n", + __func__, tid, ftid, status); + + f = lookup_atid(t, ftid); + if (!f) { + dev_err(adap->pdev_dev, "%s:could not find filter entry", + __func__); + return; + } + ctx = f->ctx; + f->ctx = NULL; + + switch (status) { + case CPL_ERR_NONE: + f->tid = tid; + f->pending = 0; + f->valid = 1; + cxgb4_insert_tid(t, f, f->tid, 0); + cxgb4_free_atid(t, ftid); + if (ctx) { + ctx->tid = f->tid; + ctx->result = 0; + } + if (configure_filter_tcb(adap, tid, f)) { + clear_filter(adap, f); + cxgb4_remove_tid(t, 0, tid, 0); + kfree(f); + if (ctx) { + ctx->result = -EINVAL; + complete(&ctx->completion); + } + return; + } + switch (f->fs.action) { + case FILTER_PASS: + if (f->fs.dirsteer) + set_tcb_tflag(adap, f, tid, + TF_DIRECT_STEER_S, 1, 1); + break; + case FILTER_DROP: + set_tcb_tflag(adap, f, tid, TF_DROP_S, 1, 1); + break; + case FILTER_SWITCH: + set_tcb_tflag(adap, f, tid, TF_LPBK_S, 1, 1); + break; + } + + break; + + default: + if (status != CPL_ERR_TCAM_FULL) + dev_err(adap->pdev_dev, "%s: filter creation PROBLEM; status = %u\n", + __func__, status); + + if (ctx) { + if (status == CPL_ERR_TCAM_FULL) + ctx->result = -ENOSPC; + else + ctx->result = -EINVAL; + } + clear_filter(adap, f); + cxgb4_free_atid(t, ftid); + kfree(f); + } + if (ctx) + complete(&ctx->completion); +} + +/* Handle a filter write/deletion reply. */ +void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl) +{ + unsigned int tid = GET_TID(rpl); + struct filter_entry *f = NULL; + unsigned int max_fidx; + int idx; + + max_fidx = adap->tids.nftids + adap->tids.nsftids; + /* Get the corresponding filter entry for this tid */ + if (adap->tids.ftid_tab) { + idx = tid - adap->tids.hpftid_base; + if (idx < adap->tids.nhpftids) { + f = &adap->tids.hpftid_tab[idx]; + } else { + /* Check this in normal filter region */ + idx = tid - adap->tids.ftid_base; + if (idx >= max_fidx) + return; + f = &adap->tids.ftid_tab[idx]; + idx += adap->tids.nhpftids; + } + + if (f->tid != tid) + return; + } + + /* We found the filter entry for this tid */ + if (f) { + unsigned int ret = TCB_COOKIE_G(rpl->cookie); + struct filter_ctx *ctx; + + /* Pull off any filter operation context attached to the + * filter. + */ + ctx = f->ctx; + f->ctx = NULL; + + if (ret == FW_FILTER_WR_FLT_DELETED) { + /* Clear the filter when we get confirmation from the + * hardware that the filter has been deleted. + */ + clear_filter(adap, f); + if (ctx) + ctx->result = 0; + } else if (ret == FW_FILTER_WR_FLT_ADDED) { + f->pending = 0; /* async setup completed */ + f->valid = 1; + if (ctx) { + ctx->result = 0; + ctx->tid = idx; + } + } else { + /* Something went wrong. Issue a warning about the + * problem and clear everything out. + */ + dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n", + idx, ret); + clear_filter(adap, f); + if (ctx) + ctx->result = -EINVAL; + } + if (ctx) + complete(&ctx->completion); + } +} + +void init_hash_filter(struct adapter *adap) +{ + u32 reg; + + /* On T6, verify the necessary register configs and warn the user in + * case of improper config + */ + if (is_t6(adap->params.chip)) { + if (is_offload(adap)) { + if (!(t4_read_reg(adap, TP_GLOBAL_CONFIG_A) + & ACTIVEFILTERCOUNTS_F)) { + dev_err(adap->pdev_dev, "Invalid hash filter + ofld config\n"); + return; + } + } else { + reg = t4_read_reg(adap, LE_DB_RSP_CODE_0_A); + if (TCAM_ACTV_HIT_G(reg) != 4) { + dev_err(adap->pdev_dev, "Invalid hash filter config\n"); + return; + } + + reg = t4_read_reg(adap, LE_DB_RSP_CODE_1_A); + if (HASH_ACTV_HIT_G(reg) != 4) { + dev_err(adap->pdev_dev, "Invalid hash filter config\n"); + return; + } + } + + } else { + dev_err(adap->pdev_dev, "Hash filter supported only on T6\n"); + return; + } + + adap->params.hash_filter = 1; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.h new file mode 100644 index 000000000..807a8dafe --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.h @@ -0,0 +1,58 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_FILTER_H +#define __CXGB4_FILTER_H + +#include "t4_msg.h" + +#define WORD_MASK 0xffffffff + +void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl); +void hash_filter_rpl(struct adapter *adap, const struct cpl_act_open_rpl *rpl); +void hash_del_filter_rpl(struct adapter *adap, + const struct cpl_abort_rpl_rss *rpl); +void clear_filter(struct adapter *adap, struct filter_entry *f); + +int set_filter_wr(struct adapter *adapter, int fidx); +int delete_filter(struct adapter *adapter, unsigned int fidx); + +int writable_filter(struct filter_entry *f); +void clear_all_filters(struct adapter *adapter); +void init_hash_filter(struct adapter *adap); +bool is_filter_exact_match(struct adapter *adap, + struct ch_filter_specification *fs); +void cxgb4_cleanup_ethtool_filters(struct adapter *adap); +int cxgb4_init_ethtool_filters(struct adapter *adap); +#endif /* __CXGB4_FILTER_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c new file mode 100644 index 000000000..9cbce1faa --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -0,0 +1,7284 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/bitmap.h> +#include <linux/crc32.h> +#include <linux/ctype.h> +#include <linux/debugfs.h> +#include <linux/err.h> +#include <linux/etherdevice.h> +#include <linux/firmware.h> +#include <linux/if.h> +#include <linux/if_vlan.h> +#include <linux/init.h> +#include <linux/log2.h> +#include <linux/mdio.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/mutex.h> +#include <linux/netdevice.h> +#include <linux/pci.h> +#include <linux/aer.h> +#include <linux/rtnetlink.h> +#include <linux/sched.h> +#include <linux/seq_file.h> +#include <linux/sockios.h> +#include <linux/vmalloc.h> +#include <linux/workqueue.h> +#include <net/neighbour.h> +#include <net/netevent.h> +#include <net/addrconf.h> +#include <net/bonding.h> +#include <linux/uaccess.h> +#include <linux/crash_dump.h> +#include <net/udp_tunnel.h> +#include <net/xfrm.h> +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) +#include <net/tls.h> +#endif + +#include "cxgb4.h" +#include "cxgb4_filter.h" +#include "t4_regs.h" +#include "t4_values.h" +#include "t4_msg.h" +#include "t4fw_api.h" +#include "t4fw_version.h" +#include "cxgb4_dcb.h" +#include "srq.h" +#include "cxgb4_debugfs.h" +#include "clip_tbl.h" +#include "l2t.h" +#include "smt.h" +#include "sched.h" +#include "cxgb4_tc_u32.h" +#include "cxgb4_tc_flower.h" +#include "cxgb4_tc_mqprio.h" +#include "cxgb4_tc_matchall.h" +#include "cxgb4_ptp.h" +#include "cxgb4_cudbg.h" + +char cxgb4_driver_name[] = KBUILD_MODNAME; + +#define DRV_DESC "Chelsio T4/T5/T6 Network Driver" + +#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \ + NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\ + NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) + +/* Macros needed to support the PCI Device ID Table ... + */ +#define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ + static const struct pci_device_id cxgb4_pci_tbl[] = { +#define CXGB4_UNIFIED_PF 0x4 + +#define CH_PCI_DEVICE_ID_FUNCTION CXGB4_UNIFIED_PF + +/* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is + * called for both. + */ +#define CH_PCI_DEVICE_ID_FUNCTION2 0x0 + +#define CH_PCI_ID_TABLE_ENTRY(devid) \ + {PCI_VDEVICE(CHELSIO, (devid)), CXGB4_UNIFIED_PF} + +#define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \ + { 0, } \ + } + +#include "t4_pci_id_tbl.h" + +#define FW4_FNAME "cxgb4/t4fw.bin" +#define FW5_FNAME "cxgb4/t5fw.bin" +#define FW6_FNAME "cxgb4/t6fw.bin" +#define FW4_CFNAME "cxgb4/t4-config.txt" +#define FW5_CFNAME "cxgb4/t5-config.txt" +#define FW6_CFNAME "cxgb4/t6-config.txt" +#define PHY_AQ1202_FIRMWARE "cxgb4/aq1202_fw.cld" +#define PHY_BCM84834_FIRMWARE "cxgb4/bcm8483.bin" +#define PHY_AQ1202_DEVICEID 0x4409 +#define PHY_BCM84834_DEVICEID 0x4486 + +MODULE_DESCRIPTION(DRV_DESC); +MODULE_AUTHOR("Chelsio Communications"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl); +MODULE_FIRMWARE(FW4_FNAME); +MODULE_FIRMWARE(FW5_FNAME); +MODULE_FIRMWARE(FW6_FNAME); + +/* + * The driver uses the best interrupt scheme available on a platform in the + * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which + * of these schemes the driver may consider as follows: + * + * msi = 2: choose from among all three options + * msi = 1: only consider MSI and INTx interrupts + * msi = 0: force INTx interrupts + */ +static int msi = 2; + +module_param(msi, int, 0644); +MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)"); + +/* + * Normally we tell the chip to deliver Ingress Packets into our DMA buffers + * offset by 2 bytes in order to have the IP headers line up on 4-byte + * boundaries. This is a requirement for many architectures which will throw + * a machine check fault if an attempt is made to access one of the 4-byte IP + * header fields on a non-4-byte boundary. And it's a major performance issue + * even on some architectures which allow it like some implementations of the + * x86 ISA. However, some architectures don't mind this and for some very + * edge-case performance sensitive applications (like forwarding large volumes + * of small packets), setting this DMA offset to 0 will decrease the number of + * PCI-E Bus transfers enough to measurably affect performance. + */ +static int rx_dma_offset = 2; + +/* TX Queue select used to determine what algorithm to use for selecting TX + * queue. Select between the kernel provided function (select_queue=0) or user + * cxgb_select_queue function (select_queue=1) + * + * Default: select_queue=0 + */ +static int select_queue; +module_param(select_queue, int, 0644); +MODULE_PARM_DESC(select_queue, + "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method."); + +static struct dentry *cxgb4_debugfs_root; + +LIST_HEAD(adapter_list); +DEFINE_MUTEX(uld_mutex); +LIST_HEAD(uld_list); + +static int cfg_queues(struct adapter *adap); + +static void link_report(struct net_device *dev) +{ + if (!netif_carrier_ok(dev)) + netdev_info(dev, "link down\n"); + else { + static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" }; + + const char *s; + const struct port_info *p = netdev_priv(dev); + + switch (p->link_cfg.speed) { + case 100: + s = "100Mbps"; + break; + case 1000: + s = "1Gbps"; + break; + case 10000: + s = "10Gbps"; + break; + case 25000: + s = "25Gbps"; + break; + case 40000: + s = "40Gbps"; + break; + case 50000: + s = "50Gbps"; + break; + case 100000: + s = "100Gbps"; + break; + default: + pr_info("%s: unsupported speed: %d\n", + dev->name, p->link_cfg.speed); + return; + } + + netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, + fc[p->link_cfg.fc]); + } +} + +#ifdef CONFIG_CHELSIO_T4_DCB +/* Set up/tear down Data Center Bridging Priority mapping for a net device. */ +static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset]; + int i; + + /* We use a simple mapping of Port TX Queue Index to DCB + * Priority when we're enabling DCB. + */ + for (i = 0; i < pi->nqsets; i++, txq++) { + u32 name, value; + int err; + + name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | + FW_PARAMS_PARAM_X_V( + FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) | + FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id)); + value = enable ? i : 0xffffffff; + + /* Since we can be called while atomic (from "interrupt + * level") we need to issue the Set Parameters Commannd + * without sleeping (timeout < 0). + */ + err = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1, + &name, &value, + -FW_CMD_MAX_TIMEOUT); + + if (err) + dev_err(adap->pdev_dev, + "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n", + enable ? "set" : "unset", pi->port_id, i, -err); + else + txq->dcb_prio = enable ? value : 0; + } +} + +int cxgb4_dcb_enabled(const struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + + if (!pi->dcb.enabled) + return 0; + + return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) || + (pi->dcb.state == CXGB4_DCB_STATE_HOST)); +} +#endif /* CONFIG_CHELSIO_T4_DCB */ + +void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat) +{ + struct net_device *dev = adapter->port[port_id]; + + /* Skip changes from disabled ports. */ + if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) { + if (link_stat) + netif_carrier_on(dev); + else { +#ifdef CONFIG_CHELSIO_T4_DCB + if (cxgb4_dcb_enabled(dev)) { + cxgb4_dcb_reset(dev); + dcb_tx_queue_prio_enable(dev, false); + } +#endif /* CONFIG_CHELSIO_T4_DCB */ + netif_carrier_off(dev); + } + + link_report(dev); + } +} + +void t4_os_portmod_changed(struct adapter *adap, int port_id) +{ + static const char *mod_str[] = { + NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM" + }; + + struct net_device *dev = adap->port[port_id]; + struct port_info *pi = netdev_priv(dev); + + if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) + netdev_info(dev, "port module unplugged\n"); + else if (pi->mod_type < ARRAY_SIZE(mod_str)) + netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]); + else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) + netdev_info(dev, "%s: unsupported port module inserted\n", + dev->name); + else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) + netdev_info(dev, "%s: unknown port module inserted\n", + dev->name); + else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR) + netdev_info(dev, "%s: transceiver module error\n", dev->name); + else + netdev_info(dev, "%s: unknown module type %d inserted\n", + dev->name, pi->mod_type); + + /* If the interface is running, then we'll need any "sticky" Link + * Parameters redone with a new Transceiver Module. + */ + pi->link_cfg.redo_l1cfg = netif_running(dev); +} + +int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */ +module_param(dbfifo_int_thresh, int, 0644); +MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold"); + +/* + * usecs to sleep while draining the dbfifo + */ +static int dbfifo_drain_delay = 1000; +module_param(dbfifo_drain_delay, int, 0644); +MODULE_PARM_DESC(dbfifo_drain_delay, + "usecs to sleep while draining the dbfifo"); + +static inline int cxgb4_set_addr_hash(struct port_info *pi) +{ + struct adapter *adap = pi->adapter; + u64 vec = 0; + bool ucast = false; + struct hash_mac_addr *entry; + + /* Calculate the hash vector for the updated list and program it */ + list_for_each_entry(entry, &adap->mac_hlist, list) { + ucast |= is_unicast_ether_addr(entry->addr); + vec |= (1ULL << hash_mac_addr(entry->addr)); + } + return t4_set_addr_hash(adap, adap->mbox, pi->viid, ucast, + vec, false); +} + +static int cxgb4_mac_sync(struct net_device *netdev, const u8 *mac_addr) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adap = pi->adapter; + int ret; + u64 mhash = 0; + u64 uhash = 0; + /* idx stores the index of allocated filters, + * its size should be modified based on the number of + * MAC addresses that we allocate filters for + */ + + u16 idx[1] = {}; + bool free = false; + bool ucast = is_unicast_ether_addr(mac_addr); + const u8 *maclist[1] = {mac_addr}; + struct hash_mac_addr *new_entry; + + ret = cxgb4_alloc_mac_filt(adap, pi->viid, free, 1, maclist, + idx, ucast ? &uhash : &mhash, false); + if (ret < 0) + goto out; + /* if hash != 0, then add the addr to hash addr list + * so on the end we will calculate the hash for the + * list and program it + */ + if (uhash || mhash) { + new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC); + if (!new_entry) + return -ENOMEM; + ether_addr_copy(new_entry->addr, mac_addr); + list_add_tail(&new_entry->list, &adap->mac_hlist); + ret = cxgb4_set_addr_hash(pi); + } +out: + return ret < 0 ? ret : 0; +} + +static int cxgb4_mac_unsync(struct net_device *netdev, const u8 *mac_addr) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adap = pi->adapter; + int ret; + const u8 *maclist[1] = {mac_addr}; + struct hash_mac_addr *entry, *tmp; + + /* If the MAC address to be removed is in the hash addr + * list, delete it from the list and update hash vector + */ + list_for_each_entry_safe(entry, tmp, &adap->mac_hlist, list) { + if (ether_addr_equal(entry->addr, mac_addr)) { + list_del(&entry->list); + kfree(entry); + return cxgb4_set_addr_hash(pi); + } + } + + ret = cxgb4_free_mac_filt(adap, pi->viid, 1, maclist, false); + return ret < 0 ? -EINVAL : 0; +} + +/* + * Set Rx properties of a port, such as promiscruity, address filters, and MTU. + * If @mtu is -1 it is left unchanged. + */ +static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + __dev_uc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); + __dev_mc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); + + return t4_set_rxmode(adapter, adapter->mbox, pi->viid, pi->viid_mirror, + mtu, (dev->flags & IFF_PROMISC) ? 1 : 0, + (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1, + sleep_ok); +} + +/** + * cxgb4_change_mac - Update match filter for a MAC address. + * @pi: the port_info + * @viid: the VI id + * @tcam_idx: TCAM index of existing filter for old value of MAC address, + * or -1 + * @addr: the new MAC address value + * @persist: whether a new MAC allocation should be persistent + * @smt_idx: the destination to store the new SMT index. + * + * Modifies an MPS filter and sets it to the new MAC address if + * @tcam_idx >= 0, or adds the MAC address to a new filter if + * @tcam_idx < 0. In the latter case the address is added persistently + * if @persist is %true. + * Addresses are programmed to hash region, if tcam runs out of entries. + * + */ +int cxgb4_change_mac(struct port_info *pi, unsigned int viid, + int *tcam_idx, const u8 *addr, bool persist, + u8 *smt_idx) +{ + struct adapter *adapter = pi->adapter; + struct hash_mac_addr *entry, *new_entry; + int ret; + + ret = t4_change_mac(adapter, adapter->mbox, viid, + *tcam_idx, addr, persist, smt_idx); + /* We ran out of TCAM entries. try programming hash region. */ + if (ret == -ENOMEM) { + /* If the MAC address to be updated is in the hash addr + * list, update it from the list + */ + list_for_each_entry(entry, &adapter->mac_hlist, list) { + if (entry->iface_mac) { + ether_addr_copy(entry->addr, addr); + goto set_hash; + } + } + new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL); + if (!new_entry) + return -ENOMEM; + ether_addr_copy(new_entry->addr, addr); + new_entry->iface_mac = true; + list_add_tail(&new_entry->list, &adapter->mac_hlist); +set_hash: + ret = cxgb4_set_addr_hash(pi); + } else if (ret >= 0) { + *tcam_idx = ret; + ret = 0; + } + + return ret; +} + +/* + * link_start - enable a port + * @dev: the port to enable + * + * Performs the MAC and PHY actions needed to enable a port. + */ +static int link_start(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + unsigned int mb = pi->adapter->mbox; + int ret; + + /* + * We do not set address filters and promiscuity here, the stack does + * that step explicitly. + */ + ret = t4_set_rxmode(pi->adapter, mb, pi->viid, pi->viid_mirror, + dev->mtu, -1, -1, -1, + !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true); + if (ret == 0) + ret = cxgb4_update_mac_filt(pi, pi->viid, &pi->xact_addr_filt, + dev->dev_addr, true, &pi->smt_idx); + if (ret == 0) + ret = t4_link_l1cfg(pi->adapter, mb, pi->tx_chan, + &pi->link_cfg); + if (ret == 0) { + local_bh_disable(); + ret = t4_enable_pi_params(pi->adapter, mb, pi, true, + true, CXGB4_DCB_ENABLED); + local_bh_enable(); + } + + return ret; +} + +#ifdef CONFIG_CHELSIO_T4_DCB +/* Handle a Data Center Bridging update message from the firmware. */ +static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd) +{ + int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid)); + struct net_device *dev = adap->port[adap->chan_map[port]]; + int old_dcb_enabled = cxgb4_dcb_enabled(dev); + int new_dcb_enabled; + + cxgb4_dcb_handle_fw_update(adap, pcmd); + new_dcb_enabled = cxgb4_dcb_enabled(dev); + + /* If the DCB has become enabled or disabled on the port then we're + * going to need to set up/tear down DCB Priority parameters for the + * TX Queues associated with the port. + */ + if (new_dcb_enabled != old_dcb_enabled) + dcb_tx_queue_prio_enable(dev, new_dcb_enabled); +} +#endif /* CONFIG_CHELSIO_T4_DCB */ + +/* Response queue handler for the FW event queue. + */ +static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp, + const struct pkt_gl *gl) +{ + u8 opcode = ((const struct rss_header *)rsp)->opcode; + + rsp++; /* skip RSS header */ + + /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG. + */ + if (unlikely(opcode == CPL_FW4_MSG && + ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) { + rsp++; + opcode = ((const struct rss_header *)rsp)->opcode; + rsp++; + if (opcode != CPL_SGE_EGR_UPDATE) { + dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n" + , opcode); + goto out; + } + } + + if (likely(opcode == CPL_SGE_EGR_UPDATE)) { + const struct cpl_sge_egr_update *p = (void *)rsp; + unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid)); + struct sge_txq *txq; + + txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start]; + txq->restarts++; + if (txq->q_type == CXGB4_TXQ_ETH) { + struct sge_eth_txq *eq; + + eq = container_of(txq, struct sge_eth_txq, q); + t4_sge_eth_txq_egress_update(q->adap, eq, -1); + } else { + struct sge_uld_txq *oq; + + oq = container_of(txq, struct sge_uld_txq, q); + tasklet_schedule(&oq->qresume_tsk); + } + } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) { + const struct cpl_fw6_msg *p = (void *)rsp; + +#ifdef CONFIG_CHELSIO_T4_DCB + const struct fw_port_cmd *pcmd = (const void *)p->data; + unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid)); + unsigned int action = + FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16)); + + if (cmd == FW_PORT_CMD && + (action == FW_PORT_ACTION_GET_PORT_INFO || + action == FW_PORT_ACTION_GET_PORT_INFO32)) { + int port = FW_PORT_CMD_PORTID_G( + be32_to_cpu(pcmd->op_to_portid)); + struct net_device *dev; + int dcbxdis, state_input; + + dev = q->adap->port[q->adap->chan_map[port]]; + dcbxdis = (action == FW_PORT_ACTION_GET_PORT_INFO + ? !!(pcmd->u.info.dcbxdis_pkd & FW_PORT_CMD_DCBXDIS_F) + : !!(be32_to_cpu(pcmd->u.info32.lstatus32_to_cbllen32) + & FW_PORT_CMD_DCBXDIS32_F)); + state_input = (dcbxdis + ? CXGB4_DCB_INPUT_FW_DISABLED + : CXGB4_DCB_INPUT_FW_ENABLED); + + cxgb4_dcb_state_fsm(dev, state_input); + } + + if (cmd == FW_PORT_CMD && + action == FW_PORT_ACTION_L2_DCB_CFG) + dcb_rpl(q->adap, pcmd); + else +#endif + if (p->type == 0) + t4_handle_fw_rpl(q->adap, p->data); + } else if (opcode == CPL_L2T_WRITE_RPL) { + const struct cpl_l2t_write_rpl *p = (void *)rsp; + + do_l2t_write_rpl(q->adap, p); + } else if (opcode == CPL_SMT_WRITE_RPL) { + const struct cpl_smt_write_rpl *p = (void *)rsp; + + do_smt_write_rpl(q->adap, p); + } else if (opcode == CPL_SET_TCB_RPL) { + const struct cpl_set_tcb_rpl *p = (void *)rsp; + + filter_rpl(q->adap, p); + } else if (opcode == CPL_ACT_OPEN_RPL) { + const struct cpl_act_open_rpl *p = (void *)rsp; + + hash_filter_rpl(q->adap, p); + } else if (opcode == CPL_ABORT_RPL_RSS) { + const struct cpl_abort_rpl_rss *p = (void *)rsp; + + hash_del_filter_rpl(q->adap, p); + } else if (opcode == CPL_SRQ_TABLE_RPL) { + const struct cpl_srq_table_rpl *p = (void *)rsp; + + do_srq_table_rpl(q->adap, p); + } else + dev_err(q->adap->pdev_dev, + "unexpected CPL %#x on FW event queue\n", opcode); +out: + return 0; +} + +static void disable_msi(struct adapter *adapter) +{ + if (adapter->flags & CXGB4_USING_MSIX) { + pci_disable_msix(adapter->pdev); + adapter->flags &= ~CXGB4_USING_MSIX; + } else if (adapter->flags & CXGB4_USING_MSI) { + pci_disable_msi(adapter->pdev); + adapter->flags &= ~CXGB4_USING_MSI; + } +} + +/* + * Interrupt handler for non-data events used with MSI-X. + */ +static irqreturn_t t4_nondata_intr(int irq, void *cookie) +{ + struct adapter *adap = cookie; + u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A)); + + if (v & PFSW_F) { + adap->swintr = 1; + t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v); + } + if (adap->flags & CXGB4_MASTER_PF) + t4_slow_intr_handler(adap); + return IRQ_HANDLED; +} + +int cxgb4_set_msix_aff(struct adapter *adap, unsigned short vec, + cpumask_var_t *aff_mask, int idx) +{ + int rv; + + if (!zalloc_cpumask_var(aff_mask, GFP_KERNEL)) { + dev_err(adap->pdev_dev, "alloc_cpumask_var failed\n"); + return -ENOMEM; + } + + cpumask_set_cpu(cpumask_local_spread(idx, dev_to_node(adap->pdev_dev)), + *aff_mask); + + rv = irq_set_affinity_hint(vec, *aff_mask); + if (rv) + dev_warn(adap->pdev_dev, + "irq_set_affinity_hint %u failed %d\n", + vec, rv); + + return 0; +} + +void cxgb4_clear_msix_aff(unsigned short vec, cpumask_var_t aff_mask) +{ + irq_set_affinity_hint(vec, NULL); + free_cpumask_var(aff_mask); +} + +static int request_msix_queue_irqs(struct adapter *adap) +{ + struct sge *s = &adap->sge; + struct msix_info *minfo; + int err, ethqidx; + + if (s->fwevtq_msix_idx < 0) + return -ENOMEM; + + err = request_irq(adap->msix_info[s->fwevtq_msix_idx].vec, + t4_sge_intr_msix, 0, + adap->msix_info[s->fwevtq_msix_idx].desc, + &s->fw_evtq); + if (err) + return err; + + for_each_ethrxq(s, ethqidx) { + minfo = s->ethrxq[ethqidx].msix; + err = request_irq(minfo->vec, + t4_sge_intr_msix, 0, + minfo->desc, + &s->ethrxq[ethqidx].rspq); + if (err) + goto unwind; + + cxgb4_set_msix_aff(adap, minfo->vec, + &minfo->aff_mask, ethqidx); + } + return 0; + +unwind: + while (--ethqidx >= 0) { + minfo = s->ethrxq[ethqidx].msix; + cxgb4_clear_msix_aff(minfo->vec, minfo->aff_mask); + free_irq(minfo->vec, &s->ethrxq[ethqidx].rspq); + } + free_irq(adap->msix_info[s->fwevtq_msix_idx].vec, &s->fw_evtq); + return err; +} + +static void free_msix_queue_irqs(struct adapter *adap) +{ + struct sge *s = &adap->sge; + struct msix_info *minfo; + int i; + + free_irq(adap->msix_info[s->fwevtq_msix_idx].vec, &s->fw_evtq); + for_each_ethrxq(s, i) { + minfo = s->ethrxq[i].msix; + cxgb4_clear_msix_aff(minfo->vec, minfo->aff_mask); + free_irq(minfo->vec, &s->ethrxq[i].rspq); + } +} + +static int setup_ppod_edram(struct adapter *adap) +{ + unsigned int param, val; + int ret; + + /* Driver sends FW_PARAMS_PARAM_DEV_PPOD_EDRAM read command to check + * if firmware supports ppod edram feature or not. If firmware + * returns 1, then driver can enable this feature by sending + * FW_PARAMS_PARAM_DEV_PPOD_EDRAM write command with value 1 to + * enable ppod edram feature. + */ + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PPOD_EDRAM)); + + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, ¶m, &val); + if (ret < 0) { + dev_warn(adap->pdev_dev, + "querying PPOD_EDRAM support failed: %d\n", + ret); + return -1; + } + + if (val != 1) + return -1; + + ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, ¶m, &val); + if (ret < 0) { + dev_err(adap->pdev_dev, + "setting PPOD_EDRAM failed: %d\n", ret); + return -1; + } + return 0; +} + +static void adap_config_hpfilter(struct adapter *adapter) +{ + u32 param, val = 0; + int ret; + + /* Enable HP filter region. Older fw will fail this request and + * it is fine. + */ + param = FW_PARAM_DEV(HPFILTER_REGION_SUPPORT); + ret = t4_set_params(adapter, adapter->mbox, adapter->pf, 0, + 1, ¶m, &val); + + /* An error means FW doesn't know about HP filter support, + * it's not a problem, don't return an error. + */ + if (ret < 0) + dev_err(adapter->pdev_dev, + "HP filter region isn't supported by FW\n"); +} + +static int cxgb4_config_rss(const struct port_info *pi, u16 *rss, + u16 rss_size, u16 viid) +{ + struct adapter *adap = pi->adapter; + int ret; + + ret = t4_config_rss_range(adap, adap->mbox, viid, 0, rss_size, rss, + rss_size); + if (ret) + return ret; + + /* If Tunnel All Lookup isn't specified in the global RSS + * Configuration, then we need to specify a default Ingress + * Queue for any ingress packets which aren't hashed. We'll + * use our first ingress queue ... + */ + return t4_config_vi_rss(adap, adap->mbox, viid, + FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F | + FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F | + FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F | + FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F | + FW_RSS_VI_CONFIG_CMD_UDPEN_F, + rss[0]); +} + +/** + * cxgb4_write_rss - write the RSS table for a given port + * @pi: the port + * @queues: array of queue indices for RSS + * + * Sets up the portion of the HW RSS table for the port's VI to distribute + * packets to the Rx queues in @queues. + * Should never be called before setting up sge eth rx queues + */ +int cxgb4_write_rss(const struct port_info *pi, const u16 *queues) +{ + struct adapter *adapter = pi->adapter; + const struct sge_eth_rxq *rxq; + int i, err; + u16 *rss; + + rxq = &adapter->sge.ethrxq[pi->first_qset]; + rss = kmalloc_array(pi->rss_size, sizeof(u16), GFP_KERNEL); + if (!rss) + return -ENOMEM; + + /* map the queue indices to queue ids */ + for (i = 0; i < pi->rss_size; i++, queues++) + rss[i] = rxq[*queues].rspq.abs_id; + + err = cxgb4_config_rss(pi, rss, pi->rss_size, pi->viid); + kfree(rss); + return err; +} + +/** + * setup_rss - configure RSS + * @adap: the adapter + * + * Sets up RSS for each port. + */ +static int setup_rss(struct adapter *adap) +{ + int i, j, err; + + for_each_port(adap, i) { + const struct port_info *pi = adap2pinfo(adap, i); + + /* Fill default values with equal distribution */ + for (j = 0; j < pi->rss_size; j++) + pi->rss[j] = j % pi->nqsets; + + err = cxgb4_write_rss(pi, pi->rss); + if (err) + return err; + } + return 0; +} + +/* + * Return the channel of the ingress queue with the given qid. + */ +static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid) +{ + qid -= p->ingr_start; + return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan; +} + +void cxgb4_quiesce_rx(struct sge_rspq *q) +{ + if (q->handler) + napi_disable(&q->napi); +} + +/* + * Wait until all NAPI handlers are descheduled. + */ +static void quiesce_rx(struct adapter *adap) +{ + int i; + + for (i = 0; i < adap->sge.ingr_sz; i++) { + struct sge_rspq *q = adap->sge.ingr_map[i]; + + if (!q) + continue; + + cxgb4_quiesce_rx(q); + } +} + +/* Disable interrupt and napi handler */ +static void disable_interrupts(struct adapter *adap) +{ + struct sge *s = &adap->sge; + + if (adap->flags & CXGB4_FULL_INIT_DONE) { + t4_intr_disable(adap); + if (adap->flags & CXGB4_USING_MSIX) { + free_msix_queue_irqs(adap); + free_irq(adap->msix_info[s->nd_msix_idx].vec, + adap); + } else { + free_irq(adap->pdev->irq, adap); + } + quiesce_rx(adap); + } +} + +void cxgb4_enable_rx(struct adapter *adap, struct sge_rspq *q) +{ + if (q->handler) + napi_enable(&q->napi); + + /* 0-increment GTS to start the timer and enable interrupts */ + t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A), + SEINTARM_V(q->intr_params) | + INGRESSQID_V(q->cntxt_id)); +} + +/* + * Enable NAPI scheduling and interrupt generation for all Rx queues. + */ +static void enable_rx(struct adapter *adap) +{ + int i; + + for (i = 0; i < adap->sge.ingr_sz; i++) { + struct sge_rspq *q = adap->sge.ingr_map[i]; + + if (!q) + continue; + + cxgb4_enable_rx(adap, q); + } +} + +static int setup_non_data_intr(struct adapter *adap) +{ + int msix; + + adap->sge.nd_msix_idx = -1; + if (!(adap->flags & CXGB4_USING_MSIX)) + return 0; + + /* Request MSI-X vector for non-data interrupt */ + msix = cxgb4_get_msix_idx_from_bmap(adap); + if (msix < 0) + return -ENOMEM; + + snprintf(adap->msix_info[msix].desc, + sizeof(adap->msix_info[msix].desc), + "%s", adap->port[0]->name); + + adap->sge.nd_msix_idx = msix; + return 0; +} + +static int setup_fw_sge_queues(struct adapter *adap) +{ + struct sge *s = &adap->sge; + int msix, err = 0; + + bitmap_zero(s->starving_fl, s->egr_sz); + bitmap_zero(s->txq_maperr, s->egr_sz); + + if (adap->flags & CXGB4_USING_MSIX) { + s->fwevtq_msix_idx = -1; + msix = cxgb4_get_msix_idx_from_bmap(adap); + if (msix < 0) + return -ENOMEM; + + snprintf(adap->msix_info[msix].desc, + sizeof(adap->msix_info[msix].desc), + "%s-FWeventq", adap->port[0]->name); + } else { + err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0, + NULL, NULL, NULL, -1); + if (err) + return err; + msix = -((int)s->intrq.abs_id + 1); + } + + err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0], + msix, NULL, fwevtq_handler, NULL, -1); + if (err && msix >= 0) + cxgb4_free_msix_idx_in_bmap(adap, msix); + + s->fwevtq_msix_idx = msix; + return err; +} + +/** + * setup_sge_queues - configure SGE Tx/Rx/response queues + * @adap: the adapter + * + * Determines how many sets of SGE queues to use and initializes them. + * We support multiple queue sets per port if we have MSI-X, otherwise + * just one queue set per port. + */ +static int setup_sge_queues(struct adapter *adap) +{ + struct sge_uld_rxq_info *rxq_info = NULL; + struct sge *s = &adap->sge; + unsigned int cmplqid = 0; + int err, i, j, msix = 0; + + if (is_uld(adap)) + rxq_info = s->uld_rxq_info[CXGB4_ULD_RDMA]; + + if (!(adap->flags & CXGB4_USING_MSIX)) + msix = -((int)s->intrq.abs_id + 1); + + for_each_port(adap, i) { + struct net_device *dev = adap->port[i]; + struct port_info *pi = netdev_priv(dev); + struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset]; + struct sge_eth_txq *t = &s->ethtxq[pi->first_qset]; + + for (j = 0; j < pi->nqsets; j++, q++) { + if (msix >= 0) { + msix = cxgb4_get_msix_idx_from_bmap(adap); + if (msix < 0) { + err = msix; + goto freeout; + } + + snprintf(adap->msix_info[msix].desc, + sizeof(adap->msix_info[msix].desc), + "%s-Rx%d", dev->name, j); + q->msix = &adap->msix_info[msix]; + } + + err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, + msix, &q->fl, + t4_ethrx_handler, + NULL, + t4_get_tp_ch_map(adap, + pi->tx_chan)); + if (err) + goto freeout; + q->rspq.idx = j; + memset(&q->stats, 0, sizeof(q->stats)); + } + + q = &s->ethrxq[pi->first_qset]; + for (j = 0; j < pi->nqsets; j++, t++, q++) { + err = t4_sge_alloc_eth_txq(adap, t, dev, + netdev_get_tx_queue(dev, j), + q->rspq.cntxt_id, + !!(adap->flags & CXGB4_SGE_DBQ_TIMER)); + if (err) + goto freeout; + } + } + + for_each_port(adap, i) { + /* Note that cmplqid below is 0 if we don't + * have RDMA queues, and that's the right value. + */ + if (rxq_info) + cmplqid = rxq_info->uldrxq[i].rspq.cntxt_id; + + err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i], + s->fw_evtq.cntxt_id, cmplqid); + if (err) + goto freeout; + } + + if (!is_t4(adap->params.chip)) { + err = t4_sge_alloc_eth_txq(adap, &s->ptptxq, adap->port[0], + netdev_get_tx_queue(adap->port[0], 0) + , s->fw_evtq.cntxt_id, false); + if (err) + goto freeout; + } + + t4_write_reg(adap, is_t4(adap->params.chip) ? + MPS_TRC_RSS_CONTROL_A : + MPS_T5_TRC_RSS_CONTROL_A, + RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) | + QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id)); + return 0; +freeout: + dev_err(adap->pdev_dev, "Can't allocate queues, err=%d\n", -err); + t4_free_sge_resources(adap); + return err; +} + +static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb, + struct net_device *sb_dev) +{ + int txq; + +#ifdef CONFIG_CHELSIO_T4_DCB + /* If a Data Center Bridging has been successfully negotiated on this + * link then we'll use the skb's priority to map it to a TX Queue. + * The skb's priority is determined via the VLAN Tag Priority Code + * Point field. + */ + if (cxgb4_dcb_enabled(dev) && !is_kdump_kernel()) { + u16 vlan_tci; + int err; + + err = vlan_get_tag(skb, &vlan_tci); + if (unlikely(err)) { + if (net_ratelimit()) + netdev_warn(dev, + "TX Packet without VLAN Tag on DCB Link\n"); + txq = 0; + } else { + txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; +#ifdef CONFIG_CHELSIO_T4_FCOE + if (skb->protocol == htons(ETH_P_FCOE)) + txq = skb->priority & 0x7; +#endif /* CONFIG_CHELSIO_T4_FCOE */ + } + return txq; + } +#endif /* CONFIG_CHELSIO_T4_DCB */ + + if (dev->num_tc) { + struct port_info *pi = netdev2pinfo(dev); + u8 ver, proto; + + ver = ip_hdr(skb)->version; + proto = (ver == 6) ? ipv6_hdr(skb)->nexthdr : + ip_hdr(skb)->protocol; + + /* Send unsupported traffic pattern to normal NIC queues. */ + txq = netdev_pick_tx(dev, skb, sb_dev); + if (xfrm_offload(skb) || is_ptp_enabled(skb, dev) || + skb->encapsulation || + cxgb4_is_ktls_skb(skb) || + (proto != IPPROTO_TCP && proto != IPPROTO_UDP)) + txq = txq % pi->nqsets; + + return txq; + } + + if (select_queue) { + txq = (skb_rx_queue_recorded(skb) + ? skb_get_rx_queue(skb) + : smp_processor_id()); + + while (unlikely(txq >= dev->real_num_tx_queues)) + txq -= dev->real_num_tx_queues; + + return txq; + } + + return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; +} + +static int closest_timer(const struct sge *s, int time) +{ + int i, delta, match = 0, min_delta = INT_MAX; + + for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { + delta = time - s->timer_val[i]; + if (delta < 0) + delta = -delta; + if (delta < min_delta) { + min_delta = delta; + match = i; + } + } + return match; +} + +static int closest_thres(const struct sge *s, int thres) +{ + int i, delta, match = 0, min_delta = INT_MAX; + + for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { + delta = thres - s->counter_val[i]; + if (delta < 0) + delta = -delta; + if (delta < min_delta) { + min_delta = delta; + match = i; + } + } + return match; +} + +/** + * cxgb4_set_rspq_intr_params - set a queue's interrupt holdoff parameters + * @q: the Rx queue + * @us: the hold-off time in us, or 0 to disable timer + * @cnt: the hold-off packet count, or 0 to disable counter + * + * Sets an Rx queue's interrupt hold-off time and packet count. At least + * one of the two needs to be enabled for the queue to generate interrupts. + */ +int cxgb4_set_rspq_intr_params(struct sge_rspq *q, + unsigned int us, unsigned int cnt) +{ + struct adapter *adap = q->adap; + + if ((us | cnt) == 0) + cnt = 1; + + if (cnt) { + int err; + u32 v, new_idx; + + new_idx = closest_thres(&adap->sge, cnt); + if (q->desc && q->pktcnt_idx != new_idx) { + /* the queue has already been created, update it */ + v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | + FW_PARAMS_PARAM_X_V( + FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) | + FW_PARAMS_PARAM_YZ_V(q->cntxt_id); + err = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, + &v, &new_idx); + if (err) + return err; + } + q->pktcnt_idx = new_idx; + } + + us = us == 0 ? 6 : closest_timer(&adap->sge, us); + q->intr_params = QINTR_TIMER_IDX_V(us) | QINTR_CNT_EN_V(cnt > 0); + return 0; +} + +static int cxgb_set_features(struct net_device *dev, netdev_features_t features) +{ + netdev_features_t changed = dev->features ^ features; + const struct port_info *pi = netdev_priv(dev); + int err; + + if (!(changed & NETIF_F_HW_VLAN_CTAG_RX)) + return 0; + + err = t4_set_rxmode(pi->adapter, pi->adapter->mbox, pi->viid, + pi->viid_mirror, -1, -1, -1, -1, + !!(features & NETIF_F_HW_VLAN_CTAG_RX), true); + if (unlikely(err)) + dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX; + return err; +} + +static int setup_debugfs(struct adapter *adap) +{ + if (IS_ERR_OR_NULL(adap->debugfs_root)) + return -1; + +#ifdef CONFIG_DEBUG_FS + t4_setup_debugfs(adap); +#endif + return 0; +} + +static void cxgb4_port_mirror_free_rxq(struct adapter *adap, + struct sge_eth_rxq *mirror_rxq) +{ + if ((adap->flags & CXGB4_FULL_INIT_DONE) && + !(adap->flags & CXGB4_SHUTTING_DOWN)) + cxgb4_quiesce_rx(&mirror_rxq->rspq); + + if (adap->flags & CXGB4_USING_MSIX) { + cxgb4_clear_msix_aff(mirror_rxq->msix->vec, + mirror_rxq->msix->aff_mask); + free_irq(mirror_rxq->msix->vec, &mirror_rxq->rspq); + cxgb4_free_msix_idx_in_bmap(adap, mirror_rxq->msix->idx); + } + + free_rspq_fl(adap, &mirror_rxq->rspq, &mirror_rxq->fl); +} + +static int cxgb4_port_mirror_alloc_queues(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + struct sge_eth_rxq *mirror_rxq; + struct sge *s = &adap->sge; + int ret = 0, msix = 0; + u16 i, rxqid; + u16 *rss; + + if (!pi->vi_mirror_count) + return 0; + + if (s->mirror_rxq[pi->port_id]) + return 0; + + mirror_rxq = kcalloc(pi->nmirrorqsets, sizeof(*mirror_rxq), GFP_KERNEL); + if (!mirror_rxq) + return -ENOMEM; + + s->mirror_rxq[pi->port_id] = mirror_rxq; + + if (!(adap->flags & CXGB4_USING_MSIX)) + msix = -((int)adap->sge.intrq.abs_id + 1); + + for (i = 0, rxqid = 0; i < pi->nmirrorqsets; i++, rxqid++) { + mirror_rxq = &s->mirror_rxq[pi->port_id][i]; + + /* Allocate Mirror Rxqs */ + if (msix >= 0) { + msix = cxgb4_get_msix_idx_from_bmap(adap); + if (msix < 0) { + ret = msix; + goto out_free_queues; + } + + mirror_rxq->msix = &adap->msix_info[msix]; + snprintf(mirror_rxq->msix->desc, + sizeof(mirror_rxq->msix->desc), + "%s-mirrorrxq%d", dev->name, i); + } + + init_rspq(adap, &mirror_rxq->rspq, + CXGB4_MIRROR_RXQ_DEFAULT_INTR_USEC, + CXGB4_MIRROR_RXQ_DEFAULT_PKT_CNT, + CXGB4_MIRROR_RXQ_DEFAULT_DESC_NUM, + CXGB4_MIRROR_RXQ_DEFAULT_DESC_SIZE); + + mirror_rxq->fl.size = CXGB4_MIRROR_FLQ_DEFAULT_DESC_NUM; + + ret = t4_sge_alloc_rxq(adap, &mirror_rxq->rspq, false, + dev, msix, &mirror_rxq->fl, + t4_ethrx_handler, NULL, 0); + if (ret) + goto out_free_msix_idx; + + /* Setup MSI-X vectors for Mirror Rxqs */ + if (adap->flags & CXGB4_USING_MSIX) { + ret = request_irq(mirror_rxq->msix->vec, + t4_sge_intr_msix, 0, + mirror_rxq->msix->desc, + &mirror_rxq->rspq); + if (ret) + goto out_free_rxq; + + cxgb4_set_msix_aff(adap, mirror_rxq->msix->vec, + &mirror_rxq->msix->aff_mask, i); + } + + /* Start NAPI for Mirror Rxqs */ + cxgb4_enable_rx(adap, &mirror_rxq->rspq); + } + + /* Setup RSS for Mirror Rxqs */ + rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL); + if (!rss) { + ret = -ENOMEM; + goto out_free_queues; + } + + mirror_rxq = &s->mirror_rxq[pi->port_id][0]; + for (i = 0; i < pi->rss_size; i++) + rss[i] = mirror_rxq[i % pi->nmirrorqsets].rspq.abs_id; + + ret = cxgb4_config_rss(pi, rss, pi->rss_size, pi->viid_mirror); + kfree(rss); + if (ret) + goto out_free_queues; + + return 0; + +out_free_rxq: + free_rspq_fl(adap, &mirror_rxq->rspq, &mirror_rxq->fl); + +out_free_msix_idx: + cxgb4_free_msix_idx_in_bmap(adap, mirror_rxq->msix->idx); + +out_free_queues: + while (rxqid-- > 0) + cxgb4_port_mirror_free_rxq(adap, + &s->mirror_rxq[pi->port_id][rxqid]); + + kfree(s->mirror_rxq[pi->port_id]); + s->mirror_rxq[pi->port_id] = NULL; + return ret; +} + +static void cxgb4_port_mirror_free_queues(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + struct sge *s = &adap->sge; + u16 i; + + if (!pi->vi_mirror_count) + return; + + if (!s->mirror_rxq[pi->port_id]) + return; + + for (i = 0; i < pi->nmirrorqsets; i++) + cxgb4_port_mirror_free_rxq(adap, + &s->mirror_rxq[pi->port_id][i]); + + kfree(s->mirror_rxq[pi->port_id]); + s->mirror_rxq[pi->port_id] = NULL; +} + +static int cxgb4_port_mirror_start(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + int ret, idx = -1; + + if (!pi->vi_mirror_count) + return 0; + + /* Mirror VIs can be created dynamically after stack had + * already setup Rx modes like MTU, promisc, allmulti, etc. + * on main VI. So, parse what the stack had setup on the + * main VI and update the same on the mirror VI. + */ + ret = t4_set_rxmode(adap, adap->mbox, pi->viid, pi->viid_mirror, + dev->mtu, (dev->flags & IFF_PROMISC) ? 1 : 0, + (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, + !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true); + if (ret) { + dev_err(adap->pdev_dev, + "Failed start up Rx mode for Mirror VI 0x%x, ret: %d\n", + pi->viid_mirror, ret); + return ret; + } + + /* Enable replication bit for the device's MAC address + * in MPS TCAM, so that the packets for the main VI are + * replicated to mirror VI. + */ + ret = cxgb4_update_mac_filt(pi, pi->viid_mirror, &idx, + dev->dev_addr, true, NULL); + if (ret) { + dev_err(adap->pdev_dev, + "Failed updating MAC filter for Mirror VI 0x%x, ret: %d\n", + pi->viid_mirror, ret); + return ret; + } + + /* Enabling a Virtual Interface can result in an interrupt + * during the processing of the VI Enable command and, in some + * paths, result in an attempt to issue another command in the + * interrupt context. Thus, we disable interrupts during the + * course of the VI Enable command ... + */ + local_bh_disable(); + ret = t4_enable_vi_params(adap, adap->mbox, pi->viid_mirror, true, true, + false); + local_bh_enable(); + if (ret) + dev_err(adap->pdev_dev, + "Failed starting Mirror VI 0x%x, ret: %d\n", + pi->viid_mirror, ret); + + return ret; +} + +static void cxgb4_port_mirror_stop(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + + if (!pi->vi_mirror_count) + return; + + t4_enable_vi_params(adap, adap->mbox, pi->viid_mirror, false, false, + false); +} + +int cxgb4_port_mirror_alloc(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + int ret = 0; + + if (!pi->nmirrorqsets) + return -EOPNOTSUPP; + + mutex_lock(&pi->vi_mirror_mutex); + if (pi->viid_mirror) { + pi->vi_mirror_count++; + goto out_unlock; + } + + ret = t4_init_port_mirror(pi, adap->mbox, pi->port_id, adap->pf, 0, + &pi->viid_mirror); + if (ret) + goto out_unlock; + + pi->vi_mirror_count = 1; + + if (adap->flags & CXGB4_FULL_INIT_DONE) { + ret = cxgb4_port_mirror_alloc_queues(dev); + if (ret) + goto out_free_vi; + + ret = cxgb4_port_mirror_start(dev); + if (ret) + goto out_free_queues; + } + + mutex_unlock(&pi->vi_mirror_mutex); + return 0; + +out_free_queues: + cxgb4_port_mirror_free_queues(dev); + +out_free_vi: + pi->vi_mirror_count = 0; + t4_free_vi(adap, adap->mbox, adap->pf, 0, pi->viid_mirror); + pi->viid_mirror = 0; + +out_unlock: + mutex_unlock(&pi->vi_mirror_mutex); + return ret; +} + +void cxgb4_port_mirror_free(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + + mutex_lock(&pi->vi_mirror_mutex); + if (!pi->viid_mirror) + goto out_unlock; + + if (pi->vi_mirror_count > 1) { + pi->vi_mirror_count--; + goto out_unlock; + } + + cxgb4_port_mirror_stop(dev); + cxgb4_port_mirror_free_queues(dev); + + pi->vi_mirror_count = 0; + t4_free_vi(adap, adap->mbox, adap->pf, 0, pi->viid_mirror); + pi->viid_mirror = 0; + +out_unlock: + mutex_unlock(&pi->vi_mirror_mutex); +} + +/* + * upper-layer driver support + */ + +/* + * Allocate an active-open TID and set it to the supplied value. + */ +int cxgb4_alloc_atid(struct tid_info *t, void *data) +{ + int atid = -1; + + spin_lock_bh(&t->atid_lock); + if (t->afree) { + union aopen_entry *p = t->afree; + + atid = (p - t->atid_tab) + t->atid_base; + t->afree = p->next; + p->data = data; + t->atids_in_use++; + } + spin_unlock_bh(&t->atid_lock); + return atid; +} +EXPORT_SYMBOL(cxgb4_alloc_atid); + +/* + * Release an active-open TID. + */ +void cxgb4_free_atid(struct tid_info *t, unsigned int atid) +{ + union aopen_entry *p = &t->atid_tab[atid - t->atid_base]; + + spin_lock_bh(&t->atid_lock); + p->next = t->afree; + t->afree = p; + t->atids_in_use--; + spin_unlock_bh(&t->atid_lock); +} +EXPORT_SYMBOL(cxgb4_free_atid); + +/* + * Allocate a server TID and set it to the supplied value. + */ +int cxgb4_alloc_stid(struct tid_info *t, int family, void *data) +{ + int stid; + + spin_lock_bh(&t->stid_lock); + if (family == PF_INET) { + stid = find_first_zero_bit(t->stid_bmap, t->nstids); + if (stid < t->nstids) + __set_bit(stid, t->stid_bmap); + else + stid = -1; + } else { + stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 1); + if (stid < 0) + stid = -1; + } + if (stid >= 0) { + t->stid_tab[stid].data = data; + stid += t->stid_base; + /* IPv6 requires max of 520 bits or 16 cells in TCAM + * This is equivalent to 4 TIDs. With CLIP enabled it + * needs 2 TIDs. + */ + if (family == PF_INET6) { + t->stids_in_use += 2; + t->v6_stids_in_use += 2; + } else { + t->stids_in_use++; + } + } + spin_unlock_bh(&t->stid_lock); + return stid; +} +EXPORT_SYMBOL(cxgb4_alloc_stid); + +/* Allocate a server filter TID and set it to the supplied value. + */ +int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data) +{ + int stid; + + spin_lock_bh(&t->stid_lock); + if (family == PF_INET) { + stid = find_next_zero_bit(t->stid_bmap, + t->nstids + t->nsftids, t->nstids); + if (stid < (t->nstids + t->nsftids)) + __set_bit(stid, t->stid_bmap); + else + stid = -1; + } else { + stid = -1; + } + if (stid >= 0) { + t->stid_tab[stid].data = data; + stid -= t->nstids; + stid += t->sftid_base; + t->sftids_in_use++; + } + spin_unlock_bh(&t->stid_lock); + return stid; +} +EXPORT_SYMBOL(cxgb4_alloc_sftid); + +/* Release a server TID. + */ +void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family) +{ + /* Is it a server filter TID? */ + if (t->nsftids && (stid >= t->sftid_base)) { + stid -= t->sftid_base; + stid += t->nstids; + } else { + stid -= t->stid_base; + } + + spin_lock_bh(&t->stid_lock); + if (family == PF_INET) + __clear_bit(stid, t->stid_bmap); + else + bitmap_release_region(t->stid_bmap, stid, 1); + t->stid_tab[stid].data = NULL; + if (stid < t->nstids) { + if (family == PF_INET6) { + t->stids_in_use -= 2; + t->v6_stids_in_use -= 2; + } else { + t->stids_in_use--; + } + } else { + t->sftids_in_use--; + } + + spin_unlock_bh(&t->stid_lock); +} +EXPORT_SYMBOL(cxgb4_free_stid); + +/* + * Populate a TID_RELEASE WR. Caller must properly size the skb. + */ +static void mk_tid_release(struct sk_buff *skb, unsigned int chan, + unsigned int tid) +{ + struct cpl_tid_release *req; + + set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); + req = __skb_put(skb, sizeof(*req)); + INIT_TP_WR(req, tid); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); +} + +/* + * Queue a TID release request and if necessary schedule a work queue to + * process it. + */ +static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan, + unsigned int tid) +{ + struct adapter *adap = container_of(t, struct adapter, tids); + void **p = &t->tid_tab[tid - t->tid_base]; + + spin_lock_bh(&adap->tid_release_lock); + *p = adap->tid_release_head; + /* Low 2 bits encode the Tx channel number */ + adap->tid_release_head = (void **)((uintptr_t)p | chan); + if (!adap->tid_release_task_busy) { + adap->tid_release_task_busy = true; + queue_work(adap->workq, &adap->tid_release_task); + } + spin_unlock_bh(&adap->tid_release_lock); +} + +/* + * Process the list of pending TID release requests. + */ +static void process_tid_release_list(struct work_struct *work) +{ + struct sk_buff *skb; + struct adapter *adap; + + adap = container_of(work, struct adapter, tid_release_task); + + spin_lock_bh(&adap->tid_release_lock); + while (adap->tid_release_head) { + void **p = adap->tid_release_head; + unsigned int chan = (uintptr_t)p & 3; + p = (void *)p - chan; + + adap->tid_release_head = *p; + *p = NULL; + spin_unlock_bh(&adap->tid_release_lock); + + while (!(skb = alloc_skb(sizeof(struct cpl_tid_release), + GFP_KERNEL))) + schedule_timeout_uninterruptible(1); + + mk_tid_release(skb, chan, p - adap->tids.tid_tab); + t4_ofld_send(adap, skb); + spin_lock_bh(&adap->tid_release_lock); + } + adap->tid_release_task_busy = false; + spin_unlock_bh(&adap->tid_release_lock); +} + +/* + * Release a TID and inform HW. If we are unable to allocate the release + * message we defer to a work queue. + */ +void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid, + unsigned short family) +{ + struct adapter *adap = container_of(t, struct adapter, tids); + struct sk_buff *skb; + + WARN_ON(tid_out_of_range(&adap->tids, tid)); + + if (t->tid_tab[tid - adap->tids.tid_base]) { + t->tid_tab[tid - adap->tids.tid_base] = NULL; + atomic_dec(&t->conns_in_use); + if (t->hash_base && (tid >= t->hash_base)) { + if (family == AF_INET6) + atomic_sub(2, &t->hash_tids_in_use); + else + atomic_dec(&t->hash_tids_in_use); + } else { + if (family == AF_INET6) + atomic_sub(2, &t->tids_in_use); + else + atomic_dec(&t->tids_in_use); + } + } + + skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC); + if (likely(skb)) { + mk_tid_release(skb, chan, tid); + t4_ofld_send(adap, skb); + } else + cxgb4_queue_tid_release(t, chan, tid); +} +EXPORT_SYMBOL(cxgb4_remove_tid); + +/* + * Allocate and initialize the TID tables. Returns 0 on success. + */ +static int tid_init(struct tid_info *t) +{ + struct adapter *adap = container_of(t, struct adapter, tids); + unsigned int max_ftids = t->nftids + t->nsftids; + unsigned int natids = t->natids; + unsigned int hpftid_bmap_size; + unsigned int eotid_bmap_size; + unsigned int stid_bmap_size; + unsigned int ftid_bmap_size; + size_t size; + + stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids); + ftid_bmap_size = BITS_TO_LONGS(t->nftids); + hpftid_bmap_size = BITS_TO_LONGS(t->nhpftids); + eotid_bmap_size = BITS_TO_LONGS(t->neotids); + size = t->ntids * sizeof(*t->tid_tab) + + natids * sizeof(*t->atid_tab) + + t->nstids * sizeof(*t->stid_tab) + + t->nsftids * sizeof(*t->stid_tab) + + stid_bmap_size * sizeof(long) + + t->nhpftids * sizeof(*t->hpftid_tab) + + hpftid_bmap_size * sizeof(long) + + max_ftids * sizeof(*t->ftid_tab) + + ftid_bmap_size * sizeof(long) + + t->neotids * sizeof(*t->eotid_tab) + + eotid_bmap_size * sizeof(long); + + t->tid_tab = kvzalloc(size, GFP_KERNEL); + if (!t->tid_tab) + return -ENOMEM; + + t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids]; + t->stid_tab = (struct serv_entry *)&t->atid_tab[natids]; + t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids]; + t->hpftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size]; + t->hpftid_bmap = (unsigned long *)&t->hpftid_tab[t->nhpftids]; + t->ftid_tab = (struct filter_entry *)&t->hpftid_bmap[hpftid_bmap_size]; + t->ftid_bmap = (unsigned long *)&t->ftid_tab[max_ftids]; + t->eotid_tab = (struct eotid_entry *)&t->ftid_bmap[ftid_bmap_size]; + t->eotid_bmap = (unsigned long *)&t->eotid_tab[t->neotids]; + spin_lock_init(&t->stid_lock); + spin_lock_init(&t->atid_lock); + spin_lock_init(&t->ftid_lock); + + t->stids_in_use = 0; + t->v6_stids_in_use = 0; + t->sftids_in_use = 0; + t->afree = NULL; + t->atids_in_use = 0; + atomic_set(&t->tids_in_use, 0); + atomic_set(&t->conns_in_use, 0); + atomic_set(&t->hash_tids_in_use, 0); + atomic_set(&t->eotids_in_use, 0); + + /* Setup the free list for atid_tab and clear the stid bitmap. */ + if (natids) { + while (--natids) + t->atid_tab[natids - 1].next = &t->atid_tab[natids]; + t->afree = t->atid_tab; + } + + if (is_offload(adap)) { + bitmap_zero(t->stid_bmap, t->nstids + t->nsftids); + /* Reserve stid 0 for T4/T5 adapters */ + if (!t->stid_base && + CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) + __set_bit(0, t->stid_bmap); + + if (t->neotids) + bitmap_zero(t->eotid_bmap, t->neotids); + } + + if (t->nhpftids) + bitmap_zero(t->hpftid_bmap, t->nhpftids); + bitmap_zero(t->ftid_bmap, t->nftids); + return 0; +} + +/** + * cxgb4_create_server - create an IP server + * @dev: the device + * @stid: the server TID + * @sip: local IP address to bind server to + * @sport: the server's TCP port + * @vlan: the VLAN header information + * @queue: queue to direct messages from this server to + * + * Create an IP server for the given port and address. + * Returns <0 on error and one of the %NET_XMIT_* values on success. + */ +int cxgb4_create_server(const struct net_device *dev, unsigned int stid, + __be32 sip, __be16 sport, __be16 vlan, + unsigned int queue) +{ + unsigned int chan; + struct sk_buff *skb; + struct adapter *adap; + struct cpl_pass_open_req *req; + int ret; + + skb = alloc_skb(sizeof(*req), GFP_KERNEL); + if (!skb) + return -ENOMEM; + + adap = netdev2adap(dev); + req = __skb_put(skb, sizeof(*req)); + INIT_TP_WR(req, 0); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid)); + req->local_port = sport; + req->peer_port = htons(0); + req->local_ip = sip; + req->peer_ip = htonl(0); + chan = rxq_to_chan(&adap->sge, queue); + req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); + req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | + SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); + ret = t4_mgmt_tx(adap, skb); + return net_xmit_eval(ret); +} +EXPORT_SYMBOL(cxgb4_create_server); + +/* cxgb4_create_server6 - create an IPv6 server + * @dev: the device + * @stid: the server TID + * @sip: local IPv6 address to bind server to + * @sport: the server's TCP port + * @queue: queue to direct messages from this server to + * + * Create an IPv6 server for the given port and address. + * Returns <0 on error and one of the %NET_XMIT_* values on success. + */ +int cxgb4_create_server6(const struct net_device *dev, unsigned int stid, + const struct in6_addr *sip, __be16 sport, + unsigned int queue) +{ + unsigned int chan; + struct sk_buff *skb; + struct adapter *adap; + struct cpl_pass_open_req6 *req; + int ret; + + skb = alloc_skb(sizeof(*req), GFP_KERNEL); + if (!skb) + return -ENOMEM; + + adap = netdev2adap(dev); + req = __skb_put(skb, sizeof(*req)); + INIT_TP_WR(req, 0); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid)); + req->local_port = sport; + req->peer_port = htons(0); + req->local_ip_hi = *(__be64 *)(sip->s6_addr); + req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8); + req->peer_ip_hi = cpu_to_be64(0); + req->peer_ip_lo = cpu_to_be64(0); + chan = rxq_to_chan(&adap->sge, queue); + req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); + req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | + SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); + ret = t4_mgmt_tx(adap, skb); + return net_xmit_eval(ret); +} +EXPORT_SYMBOL(cxgb4_create_server6); + +int cxgb4_remove_server(const struct net_device *dev, unsigned int stid, + unsigned int queue, bool ipv6) +{ + struct sk_buff *skb; + struct adapter *adap; + struct cpl_close_listsvr_req *req; + int ret; + + adap = netdev2adap(dev); + + skb = alloc_skb(sizeof(*req), GFP_KERNEL); + if (!skb) + return -ENOMEM; + + req = __skb_put(skb, sizeof(*req)); + INIT_TP_WR(req, 0); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid)); + req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) : + LISTSVR_IPV6_V(0)) | QUEUENO_V(queue)); + ret = t4_mgmt_tx(adap, skb); + return net_xmit_eval(ret); +} +EXPORT_SYMBOL(cxgb4_remove_server); + +/** + * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU + * @mtus: the HW MTU table + * @mtu: the target MTU + * @idx: index of selected entry in the MTU table + * + * Returns the index and the value in the HW MTU table that is closest to + * but does not exceed @mtu, unless @mtu is smaller than any value in the + * table, in which case that smallest available value is selected. + */ +unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu, + unsigned int *idx) +{ + unsigned int i = 0; + + while (i < NMTUS - 1 && mtus[i + 1] <= mtu) + ++i; + if (idx) + *idx = i; + return mtus[i]; +} +EXPORT_SYMBOL(cxgb4_best_mtu); + +/** + * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned + * @mtus: the HW MTU table + * @header_size: Header Size + * @data_size_max: maximum Data Segment Size + * @data_size_align: desired Data Segment Size Alignment (2^N) + * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL) + * + * Similar to cxgb4_best_mtu() but instead of searching the Hardware + * MTU Table based solely on a Maximum MTU parameter, we break that + * parameter up into a Header Size and Maximum Data Segment Size, and + * provide a desired Data Segment Size Alignment. If we find an MTU in + * the Hardware MTU Table which will result in a Data Segment Size with + * the requested alignment _and_ that MTU isn't "too far" from the + * closest MTU, then we'll return that rather than the closest MTU. + */ +unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus, + unsigned short header_size, + unsigned short data_size_max, + unsigned short data_size_align, + unsigned int *mtu_idxp) +{ + unsigned short max_mtu = header_size + data_size_max; + unsigned short data_size_align_mask = data_size_align - 1; + int mtu_idx, aligned_mtu_idx; + + /* Scan the MTU Table till we find an MTU which is larger than our + * Maximum MTU or we reach the end of the table. Along the way, + * record the last MTU found, if any, which will result in a Data + * Segment Length matching the requested alignment. + */ + for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) { + unsigned short data_size = mtus[mtu_idx] - header_size; + + /* If this MTU minus the Header Size would result in a + * Data Segment Size of the desired alignment, remember it. + */ + if ((data_size & data_size_align_mask) == 0) + aligned_mtu_idx = mtu_idx; + + /* If we're not at the end of the Hardware MTU Table and the + * next element is larger than our Maximum MTU, drop out of + * the loop. + */ + if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu) + break; + } + + /* If we fell out of the loop because we ran to the end of the table, + * then we just have to use the last [largest] entry. + */ + if (mtu_idx == NMTUS) + mtu_idx--; + + /* If we found an MTU which resulted in the requested Data Segment + * Length alignment and that's "not far" from the largest MTU which is + * less than or equal to the maximum MTU, then use that. + */ + if (aligned_mtu_idx >= 0 && + mtu_idx - aligned_mtu_idx <= 1) + mtu_idx = aligned_mtu_idx; + + /* If the caller has passed in an MTU Index pointer, pass the + * MTU Index back. Return the MTU value. + */ + if (mtu_idxp) + *mtu_idxp = mtu_idx; + return mtus[mtu_idx]; +} +EXPORT_SYMBOL(cxgb4_best_aligned_mtu); + +/** + * cxgb4_port_chan - get the HW channel of a port + * @dev: the net device for the port + * + * Return the HW Tx channel of the given port. + */ +unsigned int cxgb4_port_chan(const struct net_device *dev) +{ + return netdev2pinfo(dev)->tx_chan; +} +EXPORT_SYMBOL(cxgb4_port_chan); + +/** + * cxgb4_port_e2cchan - get the HW c-channel of a port + * @dev: the net device for the port + * + * Return the HW RX c-channel of the given port. + */ +unsigned int cxgb4_port_e2cchan(const struct net_device *dev) +{ + return netdev2pinfo(dev)->rx_cchan; +} +EXPORT_SYMBOL(cxgb4_port_e2cchan); + +unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo) +{ + struct adapter *adap = netdev2adap(dev); + u32 v1, v2, lp_count, hp_count; + + v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); + v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); + if (is_t4(adap->params.chip)) { + lp_count = LP_COUNT_G(v1); + hp_count = HP_COUNT_G(v1); + } else { + lp_count = LP_COUNT_T5_G(v1); + hp_count = HP_COUNT_T5_G(v2); + } + return lpfifo ? lp_count : hp_count; +} +EXPORT_SYMBOL(cxgb4_dbfifo_count); + +/** + * cxgb4_port_viid - get the VI id of a port + * @dev: the net device for the port + * + * Return the VI id of the given port. + */ +unsigned int cxgb4_port_viid(const struct net_device *dev) +{ + return netdev2pinfo(dev)->viid; +} +EXPORT_SYMBOL(cxgb4_port_viid); + +/** + * cxgb4_port_idx - get the index of a port + * @dev: the net device for the port + * + * Return the index of the given port. + */ +unsigned int cxgb4_port_idx(const struct net_device *dev) +{ + return netdev2pinfo(dev)->port_id; +} +EXPORT_SYMBOL(cxgb4_port_idx); + +void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4, + struct tp_tcp_stats *v6) +{ + struct adapter *adap = pci_get_drvdata(pdev); + + spin_lock(&adap->stats_lock); + t4_tp_get_tcp_stats(adap, v4, v6, false); + spin_unlock(&adap->stats_lock); +} +EXPORT_SYMBOL(cxgb4_get_tcp_stats); + +void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask, + const unsigned int *pgsz_order) +{ + struct adapter *adap = netdev2adap(dev); + + t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask); + t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) | + HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) | + HPZ3_V(pgsz_order[3])); +} +EXPORT_SYMBOL(cxgb4_iscsi_init); + +int cxgb4_flush_eq_cache(struct net_device *dev) +{ + struct adapter *adap = netdev2adap(dev); + + return t4_sge_ctxt_flush(adap, adap->mbox, CTXT_EGRESS); +} +EXPORT_SYMBOL(cxgb4_flush_eq_cache); + +static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx) +{ + u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8; + __be64 indices; + int ret; + + spin_lock(&adap->win0_lock); + ret = t4_memory_rw(adap, 0, MEM_EDC0, addr, + sizeof(indices), (__be32 *)&indices, + T4_MEMORY_READ); + spin_unlock(&adap->win0_lock); + if (!ret) { + *cidx = (be64_to_cpu(indices) >> 25) & 0xffff; + *pidx = (be64_to_cpu(indices) >> 9) & 0xffff; + } + return ret; +} + +int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx, + u16 size) +{ + struct adapter *adap = netdev2adap(dev); + u16 hw_pidx, hw_cidx; + int ret; + + ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx); + if (ret) + goto out; + + if (pidx != hw_pidx) { + u16 delta; + u32 val; + + if (pidx >= hw_pidx) + delta = pidx - hw_pidx; + else + delta = size - hw_pidx + pidx; + + if (is_t4(adap->params.chip)) + val = PIDX_V(delta); + else + val = PIDX_T5_V(delta); + wmb(); + t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), + QID_V(qid) | val); + } +out: + return ret; +} +EXPORT_SYMBOL(cxgb4_sync_txq_pidx); + +int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte) +{ + u32 edc0_size, edc1_size, mc0_size, mc1_size, size; + u32 edc0_end, edc1_end, mc0_end, mc1_end; + u32 offset, memtype, memaddr; + struct adapter *adap; + u32 hma_size = 0; + int ret; + + adap = netdev2adap(dev); + + offset = ((stag >> 8) * 32) + adap->vres.stag.start; + + /* Figure out where the offset lands in the Memory Type/Address scheme. + * This code assumes that the memory is laid out starting at offset 0 + * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0 + * and EDC1. Some cards will have neither MC0 nor MC1, most cards have + * MC0, and some have both MC0 and MC1. + */ + size = t4_read_reg(adap, MA_EDRAM0_BAR_A); + edc0_size = EDRAM0_SIZE_G(size) << 20; + size = t4_read_reg(adap, MA_EDRAM1_BAR_A); + edc1_size = EDRAM1_SIZE_G(size) << 20; + size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A); + mc0_size = EXT_MEM0_SIZE_G(size) << 20; + + if (t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A) & HMA_MUX_F) { + size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); + hma_size = EXT_MEM1_SIZE_G(size) << 20; + } + edc0_end = edc0_size; + edc1_end = edc0_end + edc1_size; + mc0_end = edc1_end + mc0_size; + + if (offset < edc0_end) { + memtype = MEM_EDC0; + memaddr = offset; + } else if (offset < edc1_end) { + memtype = MEM_EDC1; + memaddr = offset - edc0_end; + } else { + if (hma_size && (offset < (edc1_end + hma_size))) { + memtype = MEM_HMA; + memaddr = offset - edc1_end; + } else if (offset < mc0_end) { + memtype = MEM_MC0; + memaddr = offset - edc1_end; + } else if (is_t5(adap->params.chip)) { + size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); + mc1_size = EXT_MEM1_SIZE_G(size) << 20; + mc1_end = mc0_end + mc1_size; + if (offset < mc1_end) { + memtype = MEM_MC1; + memaddr = offset - mc0_end; + } else { + /* offset beyond the end of any memory */ + goto err; + } + } else { + /* T4/T6 only has a single memory channel */ + goto err; + } + } + + spin_lock(&adap->win0_lock); + ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ); + spin_unlock(&adap->win0_lock); + return ret; + +err: + dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n", + stag, offset); + return -EINVAL; +} +EXPORT_SYMBOL(cxgb4_read_tpte); + +u64 cxgb4_read_sge_timestamp(struct net_device *dev) +{ + u32 hi, lo; + struct adapter *adap; + + adap = netdev2adap(dev); + lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A); + hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A)); + + return ((u64)hi << 32) | (u64)lo; +} +EXPORT_SYMBOL(cxgb4_read_sge_timestamp); + +int cxgb4_bar2_sge_qregs(struct net_device *dev, + unsigned int qid, + enum cxgb4_bar2_qtype qtype, + int user, + u64 *pbar2_qoffset, + unsigned int *pbar2_qid) +{ + return t4_bar2_sge_qregs(netdev2adap(dev), + qid, + (qtype == CXGB4_BAR2_QTYPE_EGRESS + ? T4_BAR2_QTYPE_EGRESS + : T4_BAR2_QTYPE_INGRESS), + user, + pbar2_qoffset, + pbar2_qid); +} +EXPORT_SYMBOL(cxgb4_bar2_sge_qregs); + +static struct pci_driver cxgb4_driver; + +static void check_neigh_update(struct neighbour *neigh) +{ + const struct device *parent; + const struct net_device *netdev = neigh->dev; + + if (is_vlan_dev(netdev)) + netdev = vlan_dev_real_dev(netdev); + parent = netdev->dev.parent; + if (parent && parent->driver == &cxgb4_driver.driver) + t4_l2t_update(dev_get_drvdata(parent), neigh); +} + +static int netevent_cb(struct notifier_block *nb, unsigned long event, + void *data) +{ + switch (event) { + case NETEVENT_NEIGH_UPDATE: + check_neigh_update(data); + break; + case NETEVENT_REDIRECT: + default: + break; + } + return 0; +} + +static bool netevent_registered; +static struct notifier_block cxgb4_netevent_nb = { + .notifier_call = netevent_cb +}; + +static void drain_db_fifo(struct adapter *adap, int usecs) +{ + u32 v1, v2, lp_count, hp_count; + + do { + v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); + v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); + if (is_t4(adap->params.chip)) { + lp_count = LP_COUNT_G(v1); + hp_count = HP_COUNT_G(v1); + } else { + lp_count = LP_COUNT_T5_G(v1); + hp_count = HP_COUNT_T5_G(v2); + } + + if (lp_count == 0 && hp_count == 0) + break; + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_timeout(usecs_to_jiffies(usecs)); + } while (1); +} + +static void disable_txq_db(struct sge_txq *q) +{ + unsigned long flags; + + spin_lock_irqsave(&q->db_lock, flags); + q->db_disabled = 1; + spin_unlock_irqrestore(&q->db_lock, flags); +} + +static void enable_txq_db(struct adapter *adap, struct sge_txq *q) +{ + spin_lock_irq(&q->db_lock); + if (q->db_pidx_inc) { + /* Make sure that all writes to the TX descriptors + * are committed before we tell HW about them. + */ + wmb(); + t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), + QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc)); + q->db_pidx_inc = 0; + } + q->db_disabled = 0; + spin_unlock_irq(&q->db_lock); +} + +static void disable_dbs(struct adapter *adap) +{ + int i; + + for_each_ethrxq(&adap->sge, i) + disable_txq_db(&adap->sge.ethtxq[i].q); + if (is_offload(adap)) { + struct sge_uld_txq_info *txq_info = + adap->sge.uld_txq_info[CXGB4_TX_OFLD]; + + if (txq_info) { + for_each_ofldtxq(&adap->sge, i) { + struct sge_uld_txq *txq = &txq_info->uldtxq[i]; + + disable_txq_db(&txq->q); + } + } + } + for_each_port(adap, i) + disable_txq_db(&adap->sge.ctrlq[i].q); +} + +static void enable_dbs(struct adapter *adap) +{ + int i; + + for_each_ethrxq(&adap->sge, i) + enable_txq_db(adap, &adap->sge.ethtxq[i].q); + if (is_offload(adap)) { + struct sge_uld_txq_info *txq_info = + adap->sge.uld_txq_info[CXGB4_TX_OFLD]; + + if (txq_info) { + for_each_ofldtxq(&adap->sge, i) { + struct sge_uld_txq *txq = &txq_info->uldtxq[i]; + + enable_txq_db(adap, &txq->q); + } + } + } + for_each_port(adap, i) + enable_txq_db(adap, &adap->sge.ctrlq[i].q); +} + +static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd) +{ + enum cxgb4_uld type = CXGB4_ULD_RDMA; + + if (adap->uld && adap->uld[type].handle) + adap->uld[type].control(adap->uld[type].handle, cmd); +} + +static void process_db_full(struct work_struct *work) +{ + struct adapter *adap; + + adap = container_of(work, struct adapter, db_full_task); + + drain_db_fifo(adap, dbfifo_drain_delay); + enable_dbs(adap); + notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); + if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) + t4_set_reg_field(adap, SGE_INT_ENABLE3_A, + DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, + DBFIFO_HP_INT_F | DBFIFO_LP_INT_F); + else + t4_set_reg_field(adap, SGE_INT_ENABLE3_A, + DBFIFO_LP_INT_F, DBFIFO_LP_INT_F); +} + +static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q) +{ + u16 hw_pidx, hw_cidx; + int ret; + + spin_lock_irq(&q->db_lock); + ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx); + if (ret) + goto out; + if (q->db_pidx != hw_pidx) { + u16 delta; + u32 val; + + if (q->db_pidx >= hw_pidx) + delta = q->db_pidx - hw_pidx; + else + delta = q->size - hw_pidx + q->db_pidx; + + if (is_t4(adap->params.chip)) + val = PIDX_V(delta); + else + val = PIDX_T5_V(delta); + wmb(); + t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), + QID_V(q->cntxt_id) | val); + } +out: + q->db_disabled = 0; + q->db_pidx_inc = 0; + spin_unlock_irq(&q->db_lock); + if (ret) + CH_WARN(adap, "DB drop recovery failed.\n"); +} + +static void recover_all_queues(struct adapter *adap) +{ + int i; + + for_each_ethrxq(&adap->sge, i) + sync_txq_pidx(adap, &adap->sge.ethtxq[i].q); + if (is_offload(adap)) { + struct sge_uld_txq_info *txq_info = + adap->sge.uld_txq_info[CXGB4_TX_OFLD]; + if (txq_info) { + for_each_ofldtxq(&adap->sge, i) { + struct sge_uld_txq *txq = &txq_info->uldtxq[i]; + + sync_txq_pidx(adap, &txq->q); + } + } + } + for_each_port(adap, i) + sync_txq_pidx(adap, &adap->sge.ctrlq[i].q); +} + +static void process_db_drop(struct work_struct *work) +{ + struct adapter *adap; + + adap = container_of(work, struct adapter, db_drop_task); + + if (is_t4(adap->params.chip)) { + drain_db_fifo(adap, dbfifo_drain_delay); + notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP); + drain_db_fifo(adap, dbfifo_drain_delay); + recover_all_queues(adap); + drain_db_fifo(adap, dbfifo_drain_delay); + enable_dbs(adap); + notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); + } else if (is_t5(adap->params.chip)) { + u32 dropped_db = t4_read_reg(adap, 0x010ac); + u16 qid = (dropped_db >> 15) & 0x1ffff; + u16 pidx_inc = dropped_db & 0x1fff; + u64 bar2_qoffset; + unsigned int bar2_qid; + int ret; + + ret = t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS, + 0, &bar2_qoffset, &bar2_qid); + if (ret) + dev_err(adap->pdev_dev, "doorbell drop recovery: " + "qid=%d, pidx_inc=%d\n", qid, pidx_inc); + else + writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid), + adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL); + + /* Re-enable BAR2 WC */ + t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15); + } + + if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) + t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0); +} + +void t4_db_full(struct adapter *adap) +{ + if (is_t4(adap->params.chip)) { + disable_dbs(adap); + notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); + t4_set_reg_field(adap, SGE_INT_ENABLE3_A, + DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0); + queue_work(adap->workq, &adap->db_full_task); + } +} + +void t4_db_dropped(struct adapter *adap) +{ + if (is_t4(adap->params.chip)) { + disable_dbs(adap); + notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); + } + queue_work(adap->workq, &adap->db_drop_task); +} + +void t4_register_netevent_notifier(void) +{ + if (!netevent_registered) { + register_netevent_notifier(&cxgb4_netevent_nb); + netevent_registered = true; + } +} + +static void detach_ulds(struct adapter *adap) +{ + unsigned int i; + + if (!is_uld(adap)) + return; + + mutex_lock(&uld_mutex); + list_del(&adap->list_node); + + for (i = 0; i < CXGB4_ULD_MAX; i++) + if (adap->uld && adap->uld[i].handle) + adap->uld[i].state_change(adap->uld[i].handle, + CXGB4_STATE_DETACH); + + if (netevent_registered && list_empty(&adapter_list)) { + unregister_netevent_notifier(&cxgb4_netevent_nb); + netevent_registered = false; + } + mutex_unlock(&uld_mutex); +} + +static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state) +{ + unsigned int i; + + mutex_lock(&uld_mutex); + for (i = 0; i < CXGB4_ULD_MAX; i++) + if (adap->uld && adap->uld[i].handle) + adap->uld[i].state_change(adap->uld[i].handle, + new_state); + mutex_unlock(&uld_mutex); +} + +#if IS_ENABLED(CONFIG_IPV6) +static int cxgb4_inet6addr_handler(struct notifier_block *this, + unsigned long event, void *data) +{ + struct inet6_ifaddr *ifa = data; + struct net_device *event_dev = ifa->idev->dev; + const struct device *parent = NULL; +#if IS_ENABLED(CONFIG_BONDING) + struct adapter *adap; +#endif + if (is_vlan_dev(event_dev)) + event_dev = vlan_dev_real_dev(event_dev); +#if IS_ENABLED(CONFIG_BONDING) + if (event_dev->flags & IFF_MASTER) { + list_for_each_entry(adap, &adapter_list, list_node) { + switch (event) { + case NETDEV_UP: + cxgb4_clip_get(adap->port[0], + (const u32 *)ifa, 1); + break; + case NETDEV_DOWN: + cxgb4_clip_release(adap->port[0], + (const u32 *)ifa, 1); + break; + default: + break; + } + } + return NOTIFY_OK; + } +#endif + + if (event_dev) + parent = event_dev->dev.parent; + + if (parent && parent->driver == &cxgb4_driver.driver) { + switch (event) { + case NETDEV_UP: + cxgb4_clip_get(event_dev, (const u32 *)ifa, 1); + break; + case NETDEV_DOWN: + cxgb4_clip_release(event_dev, (const u32 *)ifa, 1); + break; + default: + break; + } + } + return NOTIFY_OK; +} + +static bool inet6addr_registered; +static struct notifier_block cxgb4_inet6addr_notifier = { + .notifier_call = cxgb4_inet6addr_handler +}; + +static void update_clip(const struct adapter *adap) +{ + int i; + struct net_device *dev; + int ret; + + rcu_read_lock(); + + for (i = 0; i < MAX_NPORTS; i++) { + dev = adap->port[i]; + ret = 0; + + if (dev) + ret = cxgb4_update_root_dev_clip(dev); + + if (ret < 0) + break; + } + rcu_read_unlock(); +} +#endif /* IS_ENABLED(CONFIG_IPV6) */ + +/** + * cxgb_up - enable the adapter + * @adap: adapter being enabled + * + * Called when the first port is enabled, this function performs the + * actions necessary to make an adapter operational, such as completing + * the initialization of HW modules, and enabling interrupts. + * + * Must be called with the rtnl lock held. + */ +static int cxgb_up(struct adapter *adap) +{ + struct sge *s = &adap->sge; + int err; + + mutex_lock(&uld_mutex); + err = setup_sge_queues(adap); + if (err) + goto rel_lock; + err = setup_rss(adap); + if (err) + goto freeq; + + if (adap->flags & CXGB4_USING_MSIX) { + if (s->nd_msix_idx < 0) { + err = -ENOMEM; + goto irq_err; + } + + err = request_irq(adap->msix_info[s->nd_msix_idx].vec, + t4_nondata_intr, 0, + adap->msix_info[s->nd_msix_idx].desc, adap); + if (err) + goto irq_err; + + err = request_msix_queue_irqs(adap); + if (err) + goto irq_err_free_nd_msix; + } else { + err = request_irq(adap->pdev->irq, t4_intr_handler(adap), + (adap->flags & CXGB4_USING_MSI) ? 0 + : IRQF_SHARED, + adap->port[0]->name, adap); + if (err) + goto irq_err; + } + + enable_rx(adap); + t4_sge_start(adap); + t4_intr_enable(adap); + adap->flags |= CXGB4_FULL_INIT_DONE; + mutex_unlock(&uld_mutex); + + notify_ulds(adap, CXGB4_STATE_UP); +#if IS_ENABLED(CONFIG_IPV6) + update_clip(adap); +#endif + return err; + +irq_err_free_nd_msix: + free_irq(adap->msix_info[s->nd_msix_idx].vec, adap); +irq_err: + dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err); +freeq: + t4_free_sge_resources(adap); +rel_lock: + mutex_unlock(&uld_mutex); + return err; +} + +static void cxgb_down(struct adapter *adapter) +{ + cancel_work_sync(&adapter->tid_release_task); + cancel_work_sync(&adapter->db_full_task); + cancel_work_sync(&adapter->db_drop_task); + adapter->tid_release_task_busy = false; + adapter->tid_release_head = NULL; + + t4_sge_stop(adapter); + t4_free_sge_resources(adapter); + + adapter->flags &= ~CXGB4_FULL_INIT_DONE; +} + +/* + * net_device operations + */ +static int cxgb_open(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + int err; + + netif_carrier_off(dev); + + if (!(adapter->flags & CXGB4_FULL_INIT_DONE)) { + err = cxgb_up(adapter); + if (err < 0) + return err; + } + + /* It's possible that the basic port information could have + * changed since we first read it. + */ + err = t4_update_port_info(pi); + if (err < 0) + return err; + + err = link_start(dev); + if (err) + return err; + + if (pi->nmirrorqsets) { + mutex_lock(&pi->vi_mirror_mutex); + err = cxgb4_port_mirror_alloc_queues(dev); + if (err) + goto out_unlock; + + err = cxgb4_port_mirror_start(dev); + if (err) + goto out_free_queues; + mutex_unlock(&pi->vi_mirror_mutex); + } + + netif_tx_start_all_queues(dev); + return 0; + +out_free_queues: + cxgb4_port_mirror_free_queues(dev); + +out_unlock: + mutex_unlock(&pi->vi_mirror_mutex); + return err; +} + +static int cxgb_close(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + int ret; + + netif_tx_stop_all_queues(dev); + netif_carrier_off(dev); + ret = t4_enable_pi_params(adapter, adapter->pf, pi, + false, false, false); +#ifdef CONFIG_CHELSIO_T4_DCB + cxgb4_dcb_reset(dev); + dcb_tx_queue_prio_enable(dev, false); +#endif + if (ret) + return ret; + + if (pi->nmirrorqsets) { + mutex_lock(&pi->vi_mirror_mutex); + cxgb4_port_mirror_stop(dev); + cxgb4_port_mirror_free_queues(dev); + mutex_unlock(&pi->vi_mirror_mutex); + } + + return 0; +} + +int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid, + __be32 sip, __be16 sport, __be16 vlan, + unsigned int queue, unsigned char port, unsigned char mask) +{ + int ret; + struct filter_entry *f; + struct adapter *adap; + int i; + u8 *val; + + adap = netdev2adap(dev); + + /* Adjust stid to correct filter index */ + stid -= adap->tids.sftid_base; + stid += adap->tids.nftids; + + /* Check to make sure the filter requested is writable ... + */ + f = &adap->tids.ftid_tab[stid]; + ret = writable_filter(f); + if (ret) + return ret; + + /* Clear out any old resources being used by the filter before + * we start constructing the new filter. + */ + if (f->valid) + clear_filter(adap, f); + + /* Clear out filter specifications */ + memset(&f->fs, 0, sizeof(struct ch_filter_specification)); + f->fs.val.lport = be16_to_cpu(sport); + f->fs.mask.lport = ~0; + val = (u8 *)&sip; + if ((val[0] | val[1] | val[2] | val[3]) != 0) { + for (i = 0; i < 4; i++) { + f->fs.val.lip[i] = val[i]; + f->fs.mask.lip[i] = ~0; + } + if (adap->params.tp.vlan_pri_map & PORT_F) { + f->fs.val.iport = port; + f->fs.mask.iport = mask; + } + } + + if (adap->params.tp.vlan_pri_map & PROTOCOL_F) { + f->fs.val.proto = IPPROTO_TCP; + f->fs.mask.proto = ~0; + } + + f->fs.dirsteer = 1; + f->fs.iq = queue; + /* Mark filter as locked */ + f->locked = 1; + f->fs.rpttid = 1; + + /* Save the actual tid. We need this to get the corresponding + * filter entry structure in filter_rpl. + */ + f->tid = stid + adap->tids.ftid_base; + ret = set_filter_wr(adap, stid); + if (ret) { + clear_filter(adap, f); + return ret; + } + + return 0; +} +EXPORT_SYMBOL(cxgb4_create_server_filter); + +int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid, + unsigned int queue, bool ipv6) +{ + struct filter_entry *f; + struct adapter *adap; + + adap = netdev2adap(dev); + + /* Adjust stid to correct filter index */ + stid -= adap->tids.sftid_base; + stid += adap->tids.nftids; + + f = &adap->tids.ftid_tab[stid]; + /* Unlock the filter */ + f->locked = 0; + + return delete_filter(adap, stid); +} +EXPORT_SYMBOL(cxgb4_remove_server_filter); + +static void cxgb_get_stats(struct net_device *dev, + struct rtnl_link_stats64 *ns) +{ + struct port_stats stats; + struct port_info *p = netdev_priv(dev); + struct adapter *adapter = p->adapter; + + /* Block retrieving statistics during EEH error + * recovery. Otherwise, the recovery might fail + * and the PCI device will be removed permanently + */ + spin_lock(&adapter->stats_lock); + if (!netif_device_present(dev)) { + spin_unlock(&adapter->stats_lock); + return; + } + t4_get_port_stats_offset(adapter, p->tx_chan, &stats, + &p->stats_base); + spin_unlock(&adapter->stats_lock); + + ns->tx_bytes = stats.tx_octets; + ns->tx_packets = stats.tx_frames; + ns->rx_bytes = stats.rx_octets; + ns->rx_packets = stats.rx_frames; + ns->multicast = stats.rx_mcast_frames; + + /* detailed rx_errors */ + ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long + + stats.rx_runt; + ns->rx_over_errors = 0; + ns->rx_crc_errors = stats.rx_fcs_err; + ns->rx_frame_errors = stats.rx_symbol_err; + ns->rx_dropped = stats.rx_ovflow0 + stats.rx_ovflow1 + + stats.rx_ovflow2 + stats.rx_ovflow3 + + stats.rx_trunc0 + stats.rx_trunc1 + + stats.rx_trunc2 + stats.rx_trunc3; + ns->rx_missed_errors = 0; + + /* detailed tx_errors */ + ns->tx_aborted_errors = 0; + ns->tx_carrier_errors = 0; + ns->tx_fifo_errors = 0; + ns->tx_heartbeat_errors = 0; + ns->tx_window_errors = 0; + + ns->tx_errors = stats.tx_error_frames; + ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err + + ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors; +} + +static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd) +{ + unsigned int mbox; + int ret = 0, prtad, devad; + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data; + + switch (cmd) { + case SIOCGMIIPHY: + if (pi->mdio_addr < 0) + return -EOPNOTSUPP; + data->phy_id = pi->mdio_addr; + break; + case SIOCGMIIREG: + case SIOCSMIIREG: + if (mdio_phy_id_is_c45(data->phy_id)) { + prtad = mdio_phy_id_prtad(data->phy_id); + devad = mdio_phy_id_devad(data->phy_id); + } else if (data->phy_id < 32) { + prtad = data->phy_id; + devad = 0; + data->reg_num &= 0x1f; + } else + return -EINVAL; + + mbox = pi->adapter->pf; + if (cmd == SIOCGMIIREG) + ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad, + data->reg_num, &data->val_out); + else + ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad, + data->reg_num, data->val_in); + break; + case SIOCGHWTSTAMP: + return copy_to_user(req->ifr_data, &pi->tstamp_config, + sizeof(pi->tstamp_config)) ? + -EFAULT : 0; + case SIOCSHWTSTAMP: + if (copy_from_user(&pi->tstamp_config, req->ifr_data, + sizeof(pi->tstamp_config))) + return -EFAULT; + + if (!is_t4(adapter->params.chip)) { + switch (pi->tstamp_config.tx_type) { + case HWTSTAMP_TX_OFF: + case HWTSTAMP_TX_ON: + break; + default: + return -ERANGE; + } + + switch (pi->tstamp_config.rx_filter) { + case HWTSTAMP_FILTER_NONE: + pi->rxtstamp = false; + break; + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + cxgb4_ptprx_timestamping(pi, pi->port_id, + PTP_TS_L4); + break; + case HWTSTAMP_FILTER_PTP_V2_EVENT: + cxgb4_ptprx_timestamping(pi, pi->port_id, + PTP_TS_L2_L4); + break; + case HWTSTAMP_FILTER_ALL: + case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + pi->rxtstamp = true; + break; + default: + pi->tstamp_config.rx_filter = + HWTSTAMP_FILTER_NONE; + return -ERANGE; + } + + if ((pi->tstamp_config.tx_type == HWTSTAMP_TX_OFF) && + (pi->tstamp_config.rx_filter == + HWTSTAMP_FILTER_NONE)) { + if (cxgb4_ptp_txtype(adapter, pi->port_id) >= 0) + pi->ptp_enable = false; + } + + if (pi->tstamp_config.rx_filter != + HWTSTAMP_FILTER_NONE) { + if (cxgb4_ptp_redirect_rx_packet(adapter, + pi) >= 0) + pi->ptp_enable = true; + } + } else { + /* For T4 Adapters */ + switch (pi->tstamp_config.rx_filter) { + case HWTSTAMP_FILTER_NONE: + pi->rxtstamp = false; + break; + case HWTSTAMP_FILTER_ALL: + pi->rxtstamp = true; + break; + default: + pi->tstamp_config.rx_filter = + HWTSTAMP_FILTER_NONE; + return -ERANGE; + } + } + return copy_to_user(req->ifr_data, &pi->tstamp_config, + sizeof(pi->tstamp_config)) ? + -EFAULT : 0; + default: + return -EOPNOTSUPP; + } + return ret; +} + +static void cxgb_set_rxmode(struct net_device *dev) +{ + /* unfortunately we can't return errors to the stack */ + set_rxmode(dev, -1, false); +} + +static int cxgb_change_mtu(struct net_device *dev, int new_mtu) +{ + struct port_info *pi = netdev_priv(dev); + int ret; + + ret = t4_set_rxmode(pi->adapter, pi->adapter->mbox, pi->viid, + pi->viid_mirror, new_mtu, -1, -1, -1, -1, true); + if (!ret) + dev->mtu = new_mtu; + return ret; +} + +#ifdef CONFIG_PCI_IOV +static int cxgb4_mgmt_open(struct net_device *dev) +{ + /* Turn carrier off since we don't have to transmit anything on this + * interface. + */ + netif_carrier_off(dev); + return 0; +} + +/* Fill MAC address that will be assigned by the FW */ +static void cxgb4_mgmt_fill_vf_station_mac_addr(struct adapter *adap) +{ + u8 hw_addr[ETH_ALEN], macaddr[ETH_ALEN]; + unsigned int i, vf, nvfs; + u16 a, b; + int err; + u8 *na; + + err = t4_get_raw_vpd_params(adap, &adap->params.vpd); + if (err) + return; + + na = adap->params.vpd.na; + for (i = 0; i < ETH_ALEN; i++) + hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 + + hex2val(na[2 * i + 1])); + + a = (hw_addr[0] << 8) | hw_addr[1]; + b = (hw_addr[1] << 8) | hw_addr[2]; + a ^= b; + a |= 0x0200; /* locally assigned Ethernet MAC address */ + a &= ~0x0100; /* not a multicast Ethernet MAC address */ + macaddr[0] = a >> 8; + macaddr[1] = a & 0xff; + + for (i = 2; i < 5; i++) + macaddr[i] = hw_addr[i + 1]; + + for (vf = 0, nvfs = pci_sriov_get_totalvfs(adap->pdev); + vf < nvfs; vf++) { + macaddr[5] = adap->pf * nvfs + vf; + ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, macaddr); + } +} + +static int cxgb4_mgmt_set_vf_mac(struct net_device *dev, int vf, u8 *mac) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + int ret; + + /* verify MAC addr is valid */ + if (!is_valid_ether_addr(mac)) { + dev_err(pi->adapter->pdev_dev, + "Invalid Ethernet address %pM for VF %d\n", + mac, vf); + return -EINVAL; + } + + dev_info(pi->adapter->pdev_dev, + "Setting MAC %pM on VF %d\n", mac, vf); + ret = t4_set_vf_mac_acl(adap, vf + 1, 1, mac); + if (!ret) + ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, mac); + return ret; +} + +static int cxgb4_mgmt_get_vf_config(struct net_device *dev, + int vf, struct ifla_vf_info *ivi) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + struct vf_info *vfinfo; + + if (vf >= adap->num_vfs) + return -EINVAL; + vfinfo = &adap->vfinfo[vf]; + + ivi->vf = vf; + ivi->max_tx_rate = vfinfo->tx_rate; + ivi->min_tx_rate = 0; + ether_addr_copy(ivi->mac, vfinfo->vf_mac_addr); + ivi->vlan = vfinfo->vlan; + ivi->linkstate = vfinfo->link_state; + return 0; +} + +static int cxgb4_mgmt_get_phys_port_id(struct net_device *dev, + struct netdev_phys_item_id *ppid) +{ + struct port_info *pi = netdev_priv(dev); + unsigned int phy_port_id; + + phy_port_id = pi->adapter->adap_idx * 10 + pi->port_id; + ppid->id_len = sizeof(phy_port_id); + memcpy(ppid->id, &phy_port_id, ppid->id_len); + return 0; +} + +static int cxgb4_mgmt_set_vf_rate(struct net_device *dev, int vf, + int min_tx_rate, int max_tx_rate) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + unsigned int link_ok, speed, mtu; + u32 fw_pfvf, fw_class; + int class_id = vf; + int ret; + u16 pktsize; + + if (vf >= adap->num_vfs) + return -EINVAL; + + if (min_tx_rate) { + dev_err(adap->pdev_dev, + "Min tx rate (%d) (> 0) for VF %d is Invalid.\n", + min_tx_rate, vf); + return -EINVAL; + } + + if (max_tx_rate == 0) { + /* unbind VF to to any Traffic Class */ + fw_pfvf = + (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH)); + fw_class = 0xffffffff; + ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1, + &fw_pfvf, &fw_class); + if (ret) { + dev_err(adap->pdev_dev, + "Err %d in unbinding PF %d VF %d from TX Rate Limiting\n", + ret, adap->pf, vf); + return -EINVAL; + } + dev_info(adap->pdev_dev, + "PF %d VF %d is unbound from TX Rate Limiting\n", + adap->pf, vf); + adap->vfinfo[vf].tx_rate = 0; + return 0; + } + + ret = t4_get_link_params(pi, &link_ok, &speed, &mtu); + if (ret != FW_SUCCESS) { + dev_err(adap->pdev_dev, + "Failed to get link information for VF %d\n", vf); + return -EINVAL; + } + + if (!link_ok) { + dev_err(adap->pdev_dev, "Link down for VF %d\n", vf); + return -EINVAL; + } + + if (max_tx_rate > speed) { + dev_err(adap->pdev_dev, + "Max tx rate %d for VF %d can't be > link-speed %u", + max_tx_rate, vf, speed); + return -EINVAL; + } + + pktsize = mtu; + /* subtract ethhdr size and 4 bytes crc since, f/w appends it */ + pktsize = pktsize - sizeof(struct ethhdr) - 4; + /* subtract ipv4 hdr size, tcp hdr size to get typical IPv4 MSS size */ + pktsize = pktsize - sizeof(struct iphdr) - sizeof(struct tcphdr); + /* configure Traffic Class for rate-limiting */ + ret = t4_sched_params(adap, SCHED_CLASS_TYPE_PACKET, + SCHED_CLASS_LEVEL_CL_RL, + SCHED_CLASS_MODE_CLASS, + SCHED_CLASS_RATEUNIT_BITS, + SCHED_CLASS_RATEMODE_ABS, + pi->tx_chan, class_id, 0, + max_tx_rate * 1000, 0, pktsize, 0); + if (ret) { + dev_err(adap->pdev_dev, "Err %d for Traffic Class config\n", + ret); + return -EINVAL; + } + dev_info(adap->pdev_dev, + "Class %d with MSS %u configured with rate %u\n", + class_id, pktsize, max_tx_rate); + + /* bind VF to configured Traffic Class */ + fw_pfvf = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH)); + fw_class = class_id; + ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1, &fw_pfvf, + &fw_class); + if (ret) { + dev_err(adap->pdev_dev, + "Err %d in binding PF %d VF %d to Traffic Class %d\n", + ret, adap->pf, vf, class_id); + return -EINVAL; + } + dev_info(adap->pdev_dev, "PF %d VF %d is bound to Class %d\n", + adap->pf, vf, class_id); + adap->vfinfo[vf].tx_rate = max_tx_rate; + return 0; +} + +static int cxgb4_mgmt_set_vf_vlan(struct net_device *dev, int vf, + u16 vlan, u8 qos, __be16 vlan_proto) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + int ret; + + if (vf >= adap->num_vfs || vlan > 4095 || qos > 7) + return -EINVAL; + + if (vlan_proto != htons(ETH_P_8021Q) || qos != 0) + return -EPROTONOSUPPORT; + + ret = t4_set_vlan_acl(adap, adap->mbox, vf + 1, vlan); + if (!ret) { + adap->vfinfo[vf].vlan = vlan; + return 0; + } + + dev_err(adap->pdev_dev, "Err %d %s VLAN ACL for PF/VF %d/%d\n", + ret, (vlan ? "setting" : "clearing"), adap->pf, vf); + return ret; +} + +static int cxgb4_mgmt_set_vf_link_state(struct net_device *dev, int vf, + int link) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + u32 param, val; + int ret = 0; + + if (vf >= adap->num_vfs) + return -EINVAL; + + switch (link) { + case IFLA_VF_LINK_STATE_AUTO: + val = FW_VF_LINK_STATE_AUTO; + break; + + case IFLA_VF_LINK_STATE_ENABLE: + val = FW_VF_LINK_STATE_ENABLE; + break; + + case IFLA_VF_LINK_STATE_DISABLE: + val = FW_VF_LINK_STATE_DISABLE; + break; + + default: + return -EINVAL; + } + + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_LINK_STATE)); + ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1, + ¶m, &val); + if (ret) { + dev_err(adap->pdev_dev, + "Error %d in setting PF %d VF %d link state\n", + ret, adap->pf, vf); + return -EINVAL; + } + + adap->vfinfo[vf].link_state = link; + return ret; +} +#endif /* CONFIG_PCI_IOV */ + +static int cxgb_set_mac_addr(struct net_device *dev, void *p) +{ + int ret; + struct sockaddr *addr = p; + struct port_info *pi = netdev_priv(dev); + + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + + ret = cxgb4_update_mac_filt(pi, pi->viid, &pi->xact_addr_filt, + addr->sa_data, true, &pi->smt_idx); + if (ret < 0) + return ret; + + eth_hw_addr_set(dev, addr->sa_data); + return 0; +} + +#ifdef CONFIG_NET_POLL_CONTROLLER +static void cxgb_netpoll(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + + if (adap->flags & CXGB4_USING_MSIX) { + int i; + struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset]; + + for (i = pi->nqsets; i; i--, rx++) + t4_sge_intr_msix(0, &rx->rspq); + } else + t4_intr_handler(adap)(0, adap); +} +#endif + +static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + struct ch_sched_queue qe = { 0 }; + struct ch_sched_params p = { 0 }; + struct sched_class *e; + u32 req_rate; + int err = 0; + + if (!can_sched(dev)) + return -ENOTSUPP; + + if (index < 0 || index > pi->nqsets - 1) + return -EINVAL; + + if (!(adap->flags & CXGB4_FULL_INIT_DONE)) { + dev_err(adap->pdev_dev, + "Failed to rate limit on queue %d. Link Down?\n", + index); + return -EINVAL; + } + + qe.queue = index; + e = cxgb4_sched_queue_lookup(dev, &qe); + if (e && e->info.u.params.level != SCHED_CLASS_LEVEL_CL_RL) { + dev_err(adap->pdev_dev, + "Queue %u already bound to class %u of type: %u\n", + index, e->idx, e->info.u.params.level); + return -EBUSY; + } + + /* Convert from Mbps to Kbps */ + req_rate = rate * 1000; + + /* Max rate is 100 Gbps */ + if (req_rate > SCHED_MAX_RATE_KBPS) { + dev_err(adap->pdev_dev, + "Invalid rate %u Mbps, Max rate is %u Mbps\n", + rate, SCHED_MAX_RATE_KBPS / 1000); + return -ERANGE; + } + + /* First unbind the queue from any existing class */ + memset(&qe, 0, sizeof(qe)); + qe.queue = index; + qe.class = SCHED_CLS_NONE; + + err = cxgb4_sched_class_unbind(dev, (void *)(&qe), SCHED_QUEUE); + if (err) { + dev_err(adap->pdev_dev, + "Unbinding Queue %d on port %d fail. Err: %d\n", + index, pi->port_id, err); + return err; + } + + /* Queue already unbound */ + if (!req_rate) + return 0; + + /* Fetch any available unused or matching scheduling class */ + p.type = SCHED_CLASS_TYPE_PACKET; + p.u.params.level = SCHED_CLASS_LEVEL_CL_RL; + p.u.params.mode = SCHED_CLASS_MODE_CLASS; + p.u.params.rateunit = SCHED_CLASS_RATEUNIT_BITS; + p.u.params.ratemode = SCHED_CLASS_RATEMODE_ABS; + p.u.params.channel = pi->tx_chan; + p.u.params.class = SCHED_CLS_NONE; + p.u.params.minrate = 0; + p.u.params.maxrate = req_rate; + p.u.params.weight = 0; + p.u.params.pktsize = dev->mtu; + + e = cxgb4_sched_class_alloc(dev, &p); + if (!e) + return -ENOMEM; + + /* Bind the queue to a scheduling class */ + memset(&qe, 0, sizeof(qe)); + qe.queue = index; + qe.class = e->idx; + + err = cxgb4_sched_class_bind(dev, (void *)(&qe), SCHED_QUEUE); + if (err) + dev_err(adap->pdev_dev, + "Queue rate limiting failed. Err: %d\n", err); + return err; +} + +static int cxgb_setup_tc_flower(struct net_device *dev, + struct flow_cls_offload *cls_flower) +{ + switch (cls_flower->command) { + case FLOW_CLS_REPLACE: + return cxgb4_tc_flower_replace(dev, cls_flower); + case FLOW_CLS_DESTROY: + return cxgb4_tc_flower_destroy(dev, cls_flower); + case FLOW_CLS_STATS: + return cxgb4_tc_flower_stats(dev, cls_flower); + default: + return -EOPNOTSUPP; + } +} + +static int cxgb_setup_tc_cls_u32(struct net_device *dev, + struct tc_cls_u32_offload *cls_u32) +{ + switch (cls_u32->command) { + case TC_CLSU32_NEW_KNODE: + case TC_CLSU32_REPLACE_KNODE: + return cxgb4_config_knode(dev, cls_u32); + case TC_CLSU32_DELETE_KNODE: + return cxgb4_delete_knode(dev, cls_u32); + default: + return -EOPNOTSUPP; + } +} + +static int cxgb_setup_tc_matchall(struct net_device *dev, + struct tc_cls_matchall_offload *cls_matchall, + bool ingress) +{ + struct adapter *adap = netdev2adap(dev); + + if (!adap->tc_matchall) + return -ENOMEM; + + switch (cls_matchall->command) { + case TC_CLSMATCHALL_REPLACE: + return cxgb4_tc_matchall_replace(dev, cls_matchall, ingress); + case TC_CLSMATCHALL_DESTROY: + return cxgb4_tc_matchall_destroy(dev, cls_matchall, ingress); + case TC_CLSMATCHALL_STATS: + if (ingress) + return cxgb4_tc_matchall_stats(dev, cls_matchall); + break; + default: + break; + } + + return -EOPNOTSUPP; +} + +static int cxgb_setup_tc_block_ingress_cb(enum tc_setup_type type, + void *type_data, void *cb_priv) +{ + struct net_device *dev = cb_priv; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + + if (!(adap->flags & CXGB4_FULL_INIT_DONE)) { + dev_err(adap->pdev_dev, + "Failed to setup tc on port %d. Link Down?\n", + pi->port_id); + return -EINVAL; + } + + if (!tc_cls_can_offload_and_chain0(dev, type_data)) + return -EOPNOTSUPP; + + switch (type) { + case TC_SETUP_CLSU32: + return cxgb_setup_tc_cls_u32(dev, type_data); + case TC_SETUP_CLSFLOWER: + return cxgb_setup_tc_flower(dev, type_data); + case TC_SETUP_CLSMATCHALL: + return cxgb_setup_tc_matchall(dev, type_data, true); + default: + return -EOPNOTSUPP; + } +} + +static int cxgb_setup_tc_block_egress_cb(enum tc_setup_type type, + void *type_data, void *cb_priv) +{ + struct net_device *dev = cb_priv; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + + if (!(adap->flags & CXGB4_FULL_INIT_DONE)) { + dev_err(adap->pdev_dev, + "Failed to setup tc on port %d. Link Down?\n", + pi->port_id); + return -EINVAL; + } + + if (!tc_cls_can_offload_and_chain0(dev, type_data)) + return -EOPNOTSUPP; + + switch (type) { + case TC_SETUP_CLSMATCHALL: + return cxgb_setup_tc_matchall(dev, type_data, false); + default: + break; + } + + return -EOPNOTSUPP; +} + +static int cxgb_setup_tc_mqprio(struct net_device *dev, + struct tc_mqprio_qopt_offload *mqprio) +{ + struct adapter *adap = netdev2adap(dev); + + if (!is_ethofld(adap) || !adap->tc_mqprio) + return -ENOMEM; + + return cxgb4_setup_tc_mqprio(dev, mqprio); +} + +static LIST_HEAD(cxgb_block_cb_list); + +static int cxgb_setup_tc_block(struct net_device *dev, + struct flow_block_offload *f) +{ + struct port_info *pi = netdev_priv(dev); + flow_setup_cb_t *cb; + bool ingress_only; + + pi->tc_block_shared = f->block_shared; + if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) { + cb = cxgb_setup_tc_block_egress_cb; + ingress_only = false; + } else { + cb = cxgb_setup_tc_block_ingress_cb; + ingress_only = true; + } + + return flow_block_cb_setup_simple(f, &cxgb_block_cb_list, + cb, pi, dev, ingress_only); +} + +static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type, + void *type_data) +{ + switch (type) { + case TC_SETUP_QDISC_MQPRIO: + return cxgb_setup_tc_mqprio(dev, type_data); + case TC_SETUP_BLOCK: + return cxgb_setup_tc_block(dev, type_data); + default: + return -EOPNOTSUPP; + } +} + +static int cxgb_udp_tunnel_unset_port(struct net_device *netdev, + unsigned int table, unsigned int entry, + struct udp_tunnel_info *ti) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adapter = pi->adapter; + u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 }; + int ret = 0, i; + + switch (ti->type) { + case UDP_TUNNEL_TYPE_VXLAN: + adapter->vxlan_port = 0; + t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A, 0); + break; + case UDP_TUNNEL_TYPE_GENEVE: + adapter->geneve_port = 0; + t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A, 0); + break; + default: + return -EINVAL; + } + + /* Matchall mac entries can be deleted only after all tunnel ports + * are brought down or removed. + */ + if (!adapter->rawf_cnt) + return 0; + for_each_port(adapter, i) { + pi = adap2pinfo(adapter, i); + ret = t4_free_raw_mac_filt(adapter, pi->viid, + match_all_mac, match_all_mac, + adapter->rawf_start + pi->port_id, + 1, pi->port_id, false); + if (ret < 0) { + netdev_info(netdev, "Failed to free mac filter entry, for port %d\n", + i); + return ret; + } + } + + return 0; +} + +static int cxgb_udp_tunnel_set_port(struct net_device *netdev, + unsigned int table, unsigned int entry, + struct udp_tunnel_info *ti) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adapter = pi->adapter; + u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 }; + int i, ret; + + switch (ti->type) { + case UDP_TUNNEL_TYPE_VXLAN: + adapter->vxlan_port = ti->port; + t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A, + VXLAN_V(be16_to_cpu(ti->port)) | VXLAN_EN_F); + break; + case UDP_TUNNEL_TYPE_GENEVE: + adapter->geneve_port = ti->port; + t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A, + GENEVE_V(be16_to_cpu(ti->port)) | GENEVE_EN_F); + break; + default: + return -EINVAL; + } + + /* Create a 'match all' mac filter entry for inner mac, + * if raw mac interface is supported. Once the linux kernel provides + * driver entry points for adding/deleting the inner mac addresses, + * we will remove this 'match all' entry and fallback to adding + * exact match filters. + */ + for_each_port(adapter, i) { + pi = adap2pinfo(adapter, i); + + ret = t4_alloc_raw_mac_filt(adapter, pi->viid, + match_all_mac, + match_all_mac, + adapter->rawf_start + pi->port_id, + 1, pi->port_id, false); + if (ret < 0) { + netdev_info(netdev, "Failed to allocate a mac filter entry, not adding port %d\n", + be16_to_cpu(ti->port)); + return ret; + } + } + + return 0; +} + +static const struct udp_tunnel_nic_info cxgb_udp_tunnels = { + .set_port = cxgb_udp_tunnel_set_port, + .unset_port = cxgb_udp_tunnel_unset_port, + .tables = { + { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, + { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, + }, +}; + +static netdev_features_t cxgb_features_check(struct sk_buff *skb, + struct net_device *dev, + netdev_features_t features) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + if (CHELSIO_CHIP_VERSION(adapter->params.chip) < CHELSIO_T6) + return features; + + /* Check if hw supports offload for this packet */ + if (!skb->encapsulation || cxgb_encap_offload_supported(skb)) + return features; + + /* Offload is not supported for this encapsulated packet */ + return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); +} + +static netdev_features_t cxgb_fix_features(struct net_device *dev, + netdev_features_t features) +{ + /* Disable GRO, if RX_CSUM is disabled */ + if (!(features & NETIF_F_RXCSUM)) + features &= ~NETIF_F_GRO; + + return features; +} + +static const struct net_device_ops cxgb4_netdev_ops = { + .ndo_open = cxgb_open, + .ndo_stop = cxgb_close, + .ndo_start_xmit = t4_start_xmit, + .ndo_select_queue = cxgb_select_queue, + .ndo_get_stats64 = cxgb_get_stats, + .ndo_set_rx_mode = cxgb_set_rxmode, + .ndo_set_mac_address = cxgb_set_mac_addr, + .ndo_set_features = cxgb_set_features, + .ndo_validate_addr = eth_validate_addr, + .ndo_eth_ioctl = cxgb_ioctl, + .ndo_change_mtu = cxgb_change_mtu, +#ifdef CONFIG_NET_POLL_CONTROLLER + .ndo_poll_controller = cxgb_netpoll, +#endif +#ifdef CONFIG_CHELSIO_T4_FCOE + .ndo_fcoe_enable = cxgb_fcoe_enable, + .ndo_fcoe_disable = cxgb_fcoe_disable, +#endif /* CONFIG_CHELSIO_T4_FCOE */ + .ndo_set_tx_maxrate = cxgb_set_tx_maxrate, + .ndo_setup_tc = cxgb_setup_tc, + .ndo_features_check = cxgb_features_check, + .ndo_fix_features = cxgb_fix_features, +}; + +#ifdef CONFIG_PCI_IOV +static const struct net_device_ops cxgb4_mgmt_netdev_ops = { + .ndo_open = cxgb4_mgmt_open, + .ndo_set_vf_mac = cxgb4_mgmt_set_vf_mac, + .ndo_get_vf_config = cxgb4_mgmt_get_vf_config, + .ndo_set_vf_rate = cxgb4_mgmt_set_vf_rate, + .ndo_get_phys_port_id = cxgb4_mgmt_get_phys_port_id, + .ndo_set_vf_vlan = cxgb4_mgmt_set_vf_vlan, + .ndo_set_vf_link_state = cxgb4_mgmt_set_vf_link_state, +}; + +static void cxgb4_mgmt_get_drvinfo(struct net_device *dev, + struct ethtool_drvinfo *info) +{ + struct adapter *adapter = netdev2adap(dev); + + strscpy(info->driver, cxgb4_driver_name, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(adapter->pdev), + sizeof(info->bus_info)); +} + +static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = { + .get_drvinfo = cxgb4_mgmt_get_drvinfo, +}; +#endif + +static void notify_fatal_err(struct work_struct *work) +{ + struct adapter *adap; + + adap = container_of(work, struct adapter, fatal_err_notify_task); + notify_ulds(adap, CXGB4_STATE_FATAL_ERROR); +} + +void t4_fatal_err(struct adapter *adap) +{ + int port; + + if (pci_channel_offline(adap->pdev)) + return; + + /* Disable the SGE since ULDs are going to free resources that + * could be exposed to the adapter. RDMA MWs for example... + */ + t4_shutdown_adapter(adap); + for_each_port(adap, port) { + struct net_device *dev = adap->port[port]; + + /* If we get here in very early initialization the network + * devices may not have been set up yet. + */ + if (!dev) + continue; + + netif_tx_stop_all_queues(dev); + netif_carrier_off(dev); + } + dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n"); + queue_work(adap->workq, &adap->fatal_err_notify_task); +} + +static void setup_memwin(struct adapter *adap) +{ + u32 nic_win_base = t4_get_util_window(adap); + + t4_setup_memwin(adap, nic_win_base, MEMWIN_NIC); +} + +static void setup_memwin_rdma(struct adapter *adap) +{ + if (adap->vres.ocq.size) { + u32 start; + unsigned int sz_kb; + + start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2); + start &= PCI_BASE_ADDRESS_MEM_MASK; + start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres); + sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10; + t4_write_reg(adap, + PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3), + start | BIR_V(1) | WINDOW_V(ilog2(sz_kb))); + t4_write_reg(adap, + PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3), + adap->vres.ocq.start); + t4_read_reg(adap, + PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3)); + } +} + +/* HMA Definitions */ + +/* The maximum number of address that can be send in a single FW cmd */ +#define HMA_MAX_ADDR_IN_CMD 5 + +#define HMA_PAGE_SIZE PAGE_SIZE + +#define HMA_MAX_NO_FW_ADDRESS (16 << 10) /* FW supports 16K addresses */ + +#define HMA_PAGE_ORDER \ + ((HMA_PAGE_SIZE < HMA_MAX_NO_FW_ADDRESS) ? \ + ilog2(HMA_MAX_NO_FW_ADDRESS / HMA_PAGE_SIZE) : 0) + +/* The minimum and maximum possible HMA sizes that can be specified in the FW + * configuration(in units of MB). + */ +#define HMA_MIN_TOTAL_SIZE 1 +#define HMA_MAX_TOTAL_SIZE \ + (((HMA_PAGE_SIZE << HMA_PAGE_ORDER) * \ + HMA_MAX_NO_FW_ADDRESS) >> 20) + +static void adap_free_hma_mem(struct adapter *adapter) +{ + struct scatterlist *iter; + struct page *page; + int i; + + if (!adapter->hma.sgt) + return; + + if (adapter->hma.flags & HMA_DMA_MAPPED_FLAG) { + dma_unmap_sg(adapter->pdev_dev, adapter->hma.sgt->sgl, + adapter->hma.sgt->nents, DMA_BIDIRECTIONAL); + adapter->hma.flags &= ~HMA_DMA_MAPPED_FLAG; + } + + for_each_sg(adapter->hma.sgt->sgl, iter, + adapter->hma.sgt->orig_nents, i) { + page = sg_page(iter); + if (page) + __free_pages(page, HMA_PAGE_ORDER); + } + + kfree(adapter->hma.phy_addr); + sg_free_table(adapter->hma.sgt); + kfree(adapter->hma.sgt); + adapter->hma.sgt = NULL; +} + +static int adap_config_hma(struct adapter *adapter) +{ + struct scatterlist *sgl, *iter; + struct sg_table *sgt; + struct page *newpage; + unsigned int i, j, k; + u32 param, hma_size; + unsigned int ncmds; + size_t page_size; + u32 page_order; + int node, ret; + + /* HMA is supported only for T6+ cards. + * Avoid initializing HMA in kdump kernels. + */ + if (is_kdump_kernel() || + CHELSIO_CHIP_VERSION(adapter->params.chip) < CHELSIO_T6) + return 0; + + /* Get the HMA region size required by fw */ + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_HMA_SIZE)); + ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0, + 1, ¶m, &hma_size); + /* An error means card has its own memory or HMA is not supported by + * the firmware. Return without any errors. + */ + if (ret || !hma_size) + return 0; + + if (hma_size < HMA_MIN_TOTAL_SIZE || + hma_size > HMA_MAX_TOTAL_SIZE) { + dev_err(adapter->pdev_dev, + "HMA size %uMB beyond bounds(%u-%lu)MB\n", + hma_size, HMA_MIN_TOTAL_SIZE, HMA_MAX_TOTAL_SIZE); + return -EINVAL; + } + + page_size = HMA_PAGE_SIZE; + page_order = HMA_PAGE_ORDER; + adapter->hma.sgt = kzalloc(sizeof(*adapter->hma.sgt), GFP_KERNEL); + if (unlikely(!adapter->hma.sgt)) { + dev_err(adapter->pdev_dev, "HMA SG table allocation failed\n"); + return -ENOMEM; + } + sgt = adapter->hma.sgt; + /* FW returned value will be in MB's + */ + sgt->orig_nents = (hma_size << 20) / (page_size << page_order); + if (sg_alloc_table(sgt, sgt->orig_nents, GFP_KERNEL)) { + dev_err(adapter->pdev_dev, "HMA SGL allocation failed\n"); + kfree(adapter->hma.sgt); + adapter->hma.sgt = NULL; + return -ENOMEM; + } + + sgl = adapter->hma.sgt->sgl; + node = dev_to_node(adapter->pdev_dev); + for_each_sg(sgl, iter, sgt->orig_nents, i) { + newpage = alloc_pages_node(node, __GFP_NOWARN | GFP_KERNEL | + __GFP_ZERO, page_order); + if (!newpage) { + dev_err(adapter->pdev_dev, + "Not enough memory for HMA page allocation\n"); + ret = -ENOMEM; + goto free_hma; + } + sg_set_page(iter, newpage, page_size << page_order, 0); + } + + sgt->nents = dma_map_sg(adapter->pdev_dev, sgl, sgt->orig_nents, + DMA_BIDIRECTIONAL); + if (!sgt->nents) { + dev_err(adapter->pdev_dev, + "Not enough memory for HMA DMA mapping"); + ret = -ENOMEM; + goto free_hma; + } + adapter->hma.flags |= HMA_DMA_MAPPED_FLAG; + + adapter->hma.phy_addr = kcalloc(sgt->nents, sizeof(dma_addr_t), + GFP_KERNEL); + if (unlikely(!adapter->hma.phy_addr)) + goto free_hma; + + for_each_sg(sgl, iter, sgt->nents, i) { + newpage = sg_page(iter); + adapter->hma.phy_addr[i] = sg_dma_address(iter); + } + + ncmds = DIV_ROUND_UP(sgt->nents, HMA_MAX_ADDR_IN_CMD); + /* Pass on the addresses to firmware */ + for (i = 0, k = 0; i < ncmds; i++, k += HMA_MAX_ADDR_IN_CMD) { + struct fw_hma_cmd hma_cmd; + u8 naddr = HMA_MAX_ADDR_IN_CMD; + u8 soc = 0, eoc = 0; + u8 hma_mode = 1; /* Presently we support only Page table mode */ + + soc = (i == 0) ? 1 : 0; + eoc = (i == ncmds - 1) ? 1 : 0; + + /* For last cmd, set naddr corresponding to remaining + * addresses + */ + if (i == ncmds - 1) { + naddr = sgt->nents % HMA_MAX_ADDR_IN_CMD; + naddr = naddr ? naddr : HMA_MAX_ADDR_IN_CMD; + } + memset(&hma_cmd, 0, sizeof(hma_cmd)); + hma_cmd.op_pkd = htonl(FW_CMD_OP_V(FW_HMA_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F); + hma_cmd.retval_len16 = htonl(FW_LEN16(hma_cmd)); + + hma_cmd.mode_to_pcie_params = + htonl(FW_HMA_CMD_MODE_V(hma_mode) | + FW_HMA_CMD_SOC_V(soc) | FW_HMA_CMD_EOC_V(eoc)); + + /* HMA cmd size specified in MB's */ + hma_cmd.naddr_size = + htonl(FW_HMA_CMD_SIZE_V(hma_size) | + FW_HMA_CMD_NADDR_V(naddr)); + + /* Total Page size specified in units of 4K */ + hma_cmd.addr_size_pkd = + htonl(FW_HMA_CMD_ADDR_SIZE_V + ((page_size << page_order) >> 12)); + + /* Fill the 5 addresses */ + for (j = 0; j < naddr; j++) { + hma_cmd.phy_address[j] = + cpu_to_be64(adapter->hma.phy_addr[j + k]); + } + ret = t4_wr_mbox(adapter, adapter->mbox, &hma_cmd, + sizeof(hma_cmd), &hma_cmd); + if (ret) { + dev_err(adapter->pdev_dev, + "HMA FW command failed with err %d\n", ret); + goto free_hma; + } + } + + if (!ret) + dev_info(adapter->pdev_dev, + "Reserved %uMB host memory for HMA\n", hma_size); + return ret; + +free_hma: + adap_free_hma_mem(adapter); + return ret; +} + +static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c) +{ + u32 v; + int ret; + + /* Now that we've successfully configured and initialized the adapter + * can ask the Firmware what resources it has provisioned for us. + */ + ret = t4_get_pfres(adap); + if (ret) { + dev_err(adap->pdev_dev, + "Unable to retrieve resource provisioning information\n"); + return ret; + } + + /* get device capabilities */ + memset(c, 0, sizeof(*c)); + c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F); + c->cfvalid_to_len16 = htonl(FW_LEN16(*c)); + ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), c); + if (ret < 0) + return ret; + + c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F); + ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), NULL); + if (ret < 0) + return ret; + + ret = t4_config_glbl_rss(adap, adap->pf, + FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL, + FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F | + FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F); + if (ret < 0) + return ret; + + ret = t4_cfg_pfvf(adap, adap->mbox, adap->pf, 0, adap->sge.egr_sz, 64, + MAX_INGQ, 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, + FW_CMD_CAP_PF); + if (ret < 0) + return ret; + + t4_sge_init(adap); + + /* tweak some settings */ + t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849); + t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12)); + t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A); + v = t4_read_reg(adap, TP_PIO_DATA_A); + t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F); + + /* first 4 Tx modulation queues point to consecutive Tx channels */ + adap->params.tp.tx_modq_map = 0xE4; + t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A, + TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map)); + + /* associate each Tx modulation queue with consecutive Tx channels */ + v = 0x84218421; + t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, + &v, 1, TP_TX_SCHED_HDR_A); + t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, + &v, 1, TP_TX_SCHED_FIFO_A); + t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, + &v, 1, TP_TX_SCHED_PCMD_A); + +#define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */ + if (is_offload(adap)) { + t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A, + TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | + TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | + TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | + TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); + t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A, + TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | + TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | + TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | + TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); + } + + /* get basic stuff going */ + return t4_early_init(adap, adap->pf); +} + +/* + * Max # of ATIDs. The absolute HW max is 16K but we keep it lower. + */ +#define MAX_ATIDS 8192U + +/* + * Phase 0 of initialization: contact FW, obtain config, perform basic init. + * + * If the firmware we're dealing with has Configuration File support, then + * we use that to perform all configuration + */ + +/* + * Tweak configuration based on module parameters, etc. Most of these have + * defaults assigned to them by Firmware Configuration Files (if we're using + * them) but need to be explicitly set if we're using hard-coded + * initialization. But even in the case of using Firmware Configuration + * Files, we'd like to expose the ability to change these via module + * parameters so these are essentially common tweaks/settings for + * Configuration Files and hard-coded initialization ... + */ +static int adap_init0_tweaks(struct adapter *adapter) +{ + /* + * Fix up various Host-Dependent Parameters like Page Size, Cache + * Line Size, etc. The firmware default is for a 4KB Page Size and + * 64B Cache Line Size ... + */ + t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES); + + /* + * Process module parameters which affect early initialization. + */ + if (rx_dma_offset != 2 && rx_dma_offset != 0) { + dev_err(&adapter->pdev->dev, + "Ignoring illegal rx_dma_offset=%d, using 2\n", + rx_dma_offset); + rx_dma_offset = 2; + } + t4_set_reg_field(adapter, SGE_CONTROL_A, + PKTSHIFT_V(PKTSHIFT_M), + PKTSHIFT_V(rx_dma_offset)); + + /* + * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux + * adds the pseudo header itself. + */ + t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A, + CSUM_HAS_PSEUDO_HDR_F, 0); + + return 0; +} + +/* 10Gb/s-BT PHY Support. chip-external 10Gb/s-BT PHYs are complex chips + * unto themselves and they contain their own firmware to perform their + * tasks ... + */ +static int phy_aq1202_version(const u8 *phy_fw_data, + size_t phy_fw_size) +{ + int offset; + + /* At offset 0x8 you're looking for the primary image's + * starting offset which is 3 Bytes wide + * + * At offset 0xa of the primary image, you look for the offset + * of the DRAM segment which is 3 Bytes wide. + * + * The FW version is at offset 0x27e of the DRAM and is 2 Bytes + * wide + */ + #define be16(__p) (((__p)[0] << 8) | (__p)[1]) + #define le16(__p) ((__p)[0] | ((__p)[1] << 8)) + #define le24(__p) (le16(__p) | ((__p)[2] << 16)) + + offset = le24(phy_fw_data + 0x8) << 12; + offset = le24(phy_fw_data + offset + 0xa); + return be16(phy_fw_data + offset + 0x27e); + + #undef be16 + #undef le16 + #undef le24 +} + +static struct info_10gbt_phy_fw { + unsigned int phy_fw_id; /* PCI Device ID */ + char *phy_fw_file; /* /lib/firmware/ PHY Firmware file */ + int (*phy_fw_version)(const u8 *phy_fw_data, size_t phy_fw_size); + int phy_flash; /* Has FLASH for PHY Firmware */ +} phy_info_array[] = { + { + PHY_AQ1202_DEVICEID, + PHY_AQ1202_FIRMWARE, + phy_aq1202_version, + 1, + }, + { + PHY_BCM84834_DEVICEID, + PHY_BCM84834_FIRMWARE, + NULL, + 0, + }, + { 0, NULL, NULL }, +}; + +static struct info_10gbt_phy_fw *find_phy_info(int devid) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(phy_info_array); i++) { + if (phy_info_array[i].phy_fw_id == devid) + return &phy_info_array[i]; + } + return NULL; +} + +/* Handle updating of chip-external 10Gb/s-BT PHY firmware. This needs to + * happen after the FW_RESET_CMD but before the FW_INITIALIZE_CMD. On error + * we return a negative error number. If we transfer new firmware we return 1 + * (from t4_load_phy_fw()). If we don't do anything we return 0. + */ +static int adap_init0_phy(struct adapter *adap) +{ + const struct firmware *phyf; + int ret; + struct info_10gbt_phy_fw *phy_info; + + /* Use the device ID to determine which PHY file to flash. + */ + phy_info = find_phy_info(adap->pdev->device); + if (!phy_info) { + dev_warn(adap->pdev_dev, + "No PHY Firmware file found for this PHY\n"); + return -EOPNOTSUPP; + } + + /* If we have a T4 PHY firmware file under /lib/firmware/cxgb4/, then + * use that. The adapter firmware provides us with a memory buffer + * where we can load a PHY firmware file from the host if we want to + * override the PHY firmware File in flash. + */ + ret = request_firmware_direct(&phyf, phy_info->phy_fw_file, + adap->pdev_dev); + if (ret < 0) { + /* For adapters without FLASH attached to PHY for their + * firmware, it's obviously a fatal error if we can't get the + * firmware to the adapter. For adapters with PHY firmware + * FLASH storage, it's worth a warning if we can't find the + * PHY Firmware but we'll neuter the error ... + */ + dev_err(adap->pdev_dev, "unable to find PHY Firmware image " + "/lib/firmware/%s, error %d\n", + phy_info->phy_fw_file, -ret); + if (phy_info->phy_flash) { + int cur_phy_fw_ver = 0; + + t4_phy_fw_ver(adap, &cur_phy_fw_ver); + dev_warn(adap->pdev_dev, "continuing with, on-adapter " + "FLASH copy, version %#x\n", cur_phy_fw_ver); + ret = 0; + } + + return ret; + } + + /* Load PHY Firmware onto adapter. + */ + ret = t4_load_phy_fw(adap, MEMWIN_NIC, phy_info->phy_fw_version, + (u8 *)phyf->data, phyf->size); + if (ret < 0) + dev_err(adap->pdev_dev, "PHY Firmware transfer error %d\n", + -ret); + else if (ret > 0) { + int new_phy_fw_ver = 0; + + if (phy_info->phy_fw_version) + new_phy_fw_ver = phy_info->phy_fw_version(phyf->data, + phyf->size); + dev_info(adap->pdev_dev, "Successfully transferred PHY " + "Firmware /lib/firmware/%s, version %#x\n", + phy_info->phy_fw_file, new_phy_fw_ver); + } + + release_firmware(phyf); + + return ret; +} + +/* + * Attempt to initialize the adapter via a Firmware Configuration File. + */ +static int adap_init0_config(struct adapter *adapter, int reset) +{ + char *fw_config_file, fw_config_file_path[256]; + u32 finiver, finicsum, cfcsum, param, val; + struct fw_caps_config_cmd caps_cmd; + unsigned long mtype = 0, maddr = 0; + const struct firmware *cf; + char *config_name = NULL; + int config_issued = 0; + int ret; + + /* + * Reset device if necessary. + */ + if (reset) { + ret = t4_fw_reset(adapter, adapter->mbox, + PIORSTMODE_F | PIORST_F); + if (ret < 0) + goto bye; + } + + /* If this is a 10Gb/s-BT adapter make sure the chip-external + * 10Gb/s-BT PHYs have up-to-date firmware. Note that this step needs + * to be performed after any global adapter RESET above since some + * PHYs only have local RAM copies of the PHY firmware. + */ + if (is_10gbt_device(adapter->pdev->device)) { + ret = adap_init0_phy(adapter); + if (ret < 0) + goto bye; + } + /* + * If we have a T4 configuration file under /lib/firmware/cxgb4/, + * then use that. Otherwise, use the configuration file stored + * in the adapter flash ... + */ + switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) { + case CHELSIO_T4: + fw_config_file = FW4_CFNAME; + break; + case CHELSIO_T5: + fw_config_file = FW5_CFNAME; + break; + case CHELSIO_T6: + fw_config_file = FW6_CFNAME; + break; + default: + dev_err(adapter->pdev_dev, "Device %d is not supported\n", + adapter->pdev->device); + ret = -EINVAL; + goto bye; + } + + ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev); + if (ret < 0) { + config_name = "On FLASH"; + mtype = FW_MEMTYPE_CF_FLASH; + maddr = t4_flash_cfg_addr(adapter); + } else { + u32 params[7], val[7]; + + sprintf(fw_config_file_path, + "/lib/firmware/%s", fw_config_file); + config_name = fw_config_file_path; + + if (cf->size >= FLASH_CFG_MAX_SIZE) + ret = -ENOMEM; + else { + params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); + ret = t4_query_params(adapter, adapter->mbox, + adapter->pf, 0, 1, params, val); + if (ret == 0) { + /* + * For t4_memory_rw() below addresses and + * sizes have to be in terms of multiples of 4 + * bytes. So, if the Configuration File isn't + * a multiple of 4 bytes in length we'll have + * to write that out separately since we can't + * guarantee that the bytes following the + * residual byte in the buffer returned by + * request_firmware() are zeroed out ... + */ + size_t resid = cf->size & 0x3; + size_t size = cf->size & ~0x3; + __be32 *data = (__be32 *)cf->data; + + mtype = FW_PARAMS_PARAM_Y_G(val[0]); + maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16; + + spin_lock(&adapter->win0_lock); + ret = t4_memory_rw(adapter, 0, mtype, maddr, + size, data, T4_MEMORY_WRITE); + if (ret == 0 && resid != 0) { + union { + __be32 word; + char buf[4]; + } last; + int i; + + last.word = data[size >> 2]; + for (i = resid; i < 4; i++) + last.buf[i] = 0; + ret = t4_memory_rw(adapter, 0, mtype, + maddr + size, + 4, &last.word, + T4_MEMORY_WRITE); + } + spin_unlock(&adapter->win0_lock); + } + } + + release_firmware(cf); + if (ret) + goto bye; + } + + val = 0; + + /* Ofld + Hash filter is supported. Older fw will fail this request and + * it is fine. + */ + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_HASHFILTER_WITH_OFLD)); + ret = t4_set_params(adapter, adapter->mbox, adapter->pf, 0, + 1, ¶m, &val); + + /* FW doesn't know about Hash filter + ofld support, + * it's not a problem, don't return an error. + */ + if (ret < 0) { + dev_warn(adapter->pdev_dev, + "Hash filter with ofld is not supported by FW\n"); + } + + /* + * Issue a Capability Configuration command to the firmware to get it + * to parse the Configuration File. We don't use t4_fw_config_file() + * because we want the ability to modify various features after we've + * processed the configuration file ... + */ + memset(&caps_cmd, 0, sizeof(caps_cmd)); + caps_cmd.op_to_write = + htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F); + caps_cmd.cfvalid_to_len16 = + htonl(FW_CAPS_CONFIG_CMD_CFVALID_F | + FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) | + FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) | + FW_LEN16(caps_cmd)); + ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), + &caps_cmd); + + /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware + * Configuration File in FLASH), our last gasp effort is to use the + * Firmware Configuration File which is embedded in the firmware. A + * very few early versions of the firmware didn't have one embedded + * but we can ignore those. + */ + if (ret == -ENOENT) { + memset(&caps_cmd, 0, sizeof(caps_cmd)); + caps_cmd.op_to_write = + htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F); + caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); + ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, + sizeof(caps_cmd), &caps_cmd); + config_name = "Firmware Default"; + } + + config_issued = 1; + if (ret < 0) + goto bye; + + finiver = ntohl(caps_cmd.finiver); + finicsum = ntohl(caps_cmd.finicsum); + cfcsum = ntohl(caps_cmd.cfcsum); + if (finicsum != cfcsum) + dev_warn(adapter->pdev_dev, "Configuration File checksum "\ + "mismatch: [fini] csum=%#x, computed csum=%#x\n", + finicsum, cfcsum); + + /* + * And now tell the firmware to use the configuration we just loaded. + */ + caps_cmd.op_to_write = + htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F); + caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); + ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), + NULL); + if (ret < 0) + goto bye; + + /* + * Tweak configuration based on system architecture, module + * parameters, etc. + */ + ret = adap_init0_tweaks(adapter); + if (ret < 0) + goto bye; + + /* We will proceed even if HMA init fails. */ + ret = adap_config_hma(adapter); + if (ret) + dev_err(adapter->pdev_dev, + "HMA configuration failed with error %d\n", ret); + + if (is_t6(adapter->params.chip)) { + adap_config_hpfilter(adapter); + ret = setup_ppod_edram(adapter); + if (!ret) + dev_info(adapter->pdev_dev, "Successfully enabled " + "ppod edram feature\n"); + } + + /* + * And finally tell the firmware to initialize itself using the + * parameters from the Configuration File. + */ + ret = t4_fw_initialize(adapter, adapter->mbox); + if (ret < 0) + goto bye; + + /* Emit Firmware Configuration File information and return + * successfully. + */ + dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\ + "Configuration File \"%s\", version %#x, computed checksum %#x\n", + config_name, finiver, cfcsum); + return 0; + + /* + * Something bad happened. Return the error ... (If the "error" + * is that there's no Configuration File on the adapter we don't + * want to issue a warning since this is fairly common.) + */ +bye: + if (config_issued && ret != -ENOENT) + dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n", + config_name, -ret); + return ret; +} + +static struct fw_info fw_info_array[] = { + { + .chip = CHELSIO_T4, + .fs_name = FW4_CFNAME, + .fw_mod_name = FW4_FNAME, + .fw_hdr = { + .chip = FW_HDR_CHIP_T4, + .fw_ver = __cpu_to_be32(FW_VERSION(T4)), + .intfver_nic = FW_INTFVER(T4, NIC), + .intfver_vnic = FW_INTFVER(T4, VNIC), + .intfver_ri = FW_INTFVER(T4, RI), + .intfver_iscsi = FW_INTFVER(T4, ISCSI), + .intfver_fcoe = FW_INTFVER(T4, FCOE), + }, + }, { + .chip = CHELSIO_T5, + .fs_name = FW5_CFNAME, + .fw_mod_name = FW5_FNAME, + .fw_hdr = { + .chip = FW_HDR_CHIP_T5, + .fw_ver = __cpu_to_be32(FW_VERSION(T5)), + .intfver_nic = FW_INTFVER(T5, NIC), + .intfver_vnic = FW_INTFVER(T5, VNIC), + .intfver_ri = FW_INTFVER(T5, RI), + .intfver_iscsi = FW_INTFVER(T5, ISCSI), + .intfver_fcoe = FW_INTFVER(T5, FCOE), + }, + }, { + .chip = CHELSIO_T6, + .fs_name = FW6_CFNAME, + .fw_mod_name = FW6_FNAME, + .fw_hdr = { + .chip = FW_HDR_CHIP_T6, + .fw_ver = __cpu_to_be32(FW_VERSION(T6)), + .intfver_nic = FW_INTFVER(T6, NIC), + .intfver_vnic = FW_INTFVER(T6, VNIC), + .intfver_ofld = FW_INTFVER(T6, OFLD), + .intfver_ri = FW_INTFVER(T6, RI), + .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU), + .intfver_iscsi = FW_INTFVER(T6, ISCSI), + .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU), + .intfver_fcoe = FW_INTFVER(T6, FCOE), + }, + } + +}; + +static struct fw_info *find_fw_info(int chip) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) { + if (fw_info_array[i].chip == chip) + return &fw_info_array[i]; + } + return NULL; +} + +/* + * Phase 0 of initialization: contact FW, obtain config, perform basic init. + */ +static int adap_init0(struct adapter *adap, int vpd_skip) +{ + struct fw_caps_config_cmd caps_cmd; + u32 params[7], val[7]; + enum dev_state state; + u32 v, port_vec; + int reset = 1; + int ret; + + /* Grab Firmware Device Log parameters as early as possible so we have + * access to it for debugging, etc. + */ + ret = t4_init_devlog_params(adap); + if (ret < 0) + return ret; + + /* Contact FW, advertising Master capability */ + ret = t4_fw_hello(adap, adap->mbox, adap->mbox, + is_kdump_kernel() ? MASTER_MUST : MASTER_MAY, &state); + if (ret < 0) { + dev_err(adap->pdev_dev, "could not connect to FW, error %d\n", + ret); + return ret; + } + if (ret == adap->mbox) + adap->flags |= CXGB4_MASTER_PF; + + /* + * If we're the Master PF Driver and the device is uninitialized, + * then let's consider upgrading the firmware ... (We always want + * to check the firmware version number in order to A. get it for + * later reporting and B. to warn if the currently loaded firmware + * is excessively mismatched relative to the driver.) + */ + + t4_get_version_info(adap); + ret = t4_check_fw_version(adap); + /* If firmware is too old (not supported by driver) force an update. */ + if (ret) + state = DEV_STATE_UNINIT; + if ((adap->flags & CXGB4_MASTER_PF) && state != DEV_STATE_INIT) { + struct fw_info *fw_info; + struct fw_hdr *card_fw; + const struct firmware *fw; + const u8 *fw_data = NULL; + unsigned int fw_size = 0; + + /* This is the firmware whose headers the driver was compiled + * against + */ + fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip)); + if (fw_info == NULL) { + dev_err(adap->pdev_dev, + "unable to get firmware info for chip %d.\n", + CHELSIO_CHIP_VERSION(adap->params.chip)); + return -EINVAL; + } + + /* allocate memory to read the header of the firmware on the + * card + */ + card_fw = kvzalloc(sizeof(*card_fw), GFP_KERNEL); + if (!card_fw) { + ret = -ENOMEM; + goto bye; + } + + /* Get FW from from /lib/firmware/ */ + ret = request_firmware(&fw, fw_info->fw_mod_name, + adap->pdev_dev); + if (ret < 0) { + dev_err(adap->pdev_dev, + "unable to load firmware image %s, error %d\n", + fw_info->fw_mod_name, ret); + } else { + fw_data = fw->data; + fw_size = fw->size; + } + + /* upgrade FW logic */ + ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw, + state, &reset); + + /* Cleaning up */ + release_firmware(fw); + kvfree(card_fw); + + if (ret < 0) + goto bye; + } + + /* If the firmware is initialized already, emit a simply note to that + * effect. Otherwise, it's time to try initializing the adapter. + */ + if (state == DEV_STATE_INIT) { + ret = adap_config_hma(adap); + if (ret) + dev_err(adap->pdev_dev, + "HMA configuration failed with error %d\n", + ret); + dev_info(adap->pdev_dev, "Coming up as %s: "\ + "Adapter already initialized\n", + adap->flags & CXGB4_MASTER_PF ? "MASTER" : "SLAVE"); + } else { + dev_info(adap->pdev_dev, "Coming up as MASTER: "\ + "Initializing adapter\n"); + + /* Find out whether we're dealing with a version of the + * firmware which has configuration file support. + */ + params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, + params, val); + + /* If the firmware doesn't support Configuration Files, + * return an error. + */ + if (ret < 0) { + dev_err(adap->pdev_dev, "firmware doesn't support " + "Firmware Configuration Files\n"); + goto bye; + } + + /* The firmware provides us with a memory buffer where we can + * load a Configuration File from the host if we want to + * override the Configuration File in flash. + */ + ret = adap_init0_config(adap, reset); + if (ret == -ENOENT) { + dev_err(adap->pdev_dev, "no Configuration File " + "present on adapter.\n"); + goto bye; + } + if (ret < 0) { + dev_err(adap->pdev_dev, "could not initialize " + "adapter, error %d\n", -ret); + goto bye; + } + } + + /* Now that we've successfully configured and initialized the adapter + * (or found it already initialized), we can ask the Firmware what + * resources it has provisioned for us. + */ + ret = t4_get_pfres(adap); + if (ret) { + dev_err(adap->pdev_dev, + "Unable to retrieve resource provisioning information\n"); + goto bye; + } + + /* Grab VPD parameters. This should be done after we establish a + * connection to the firmware since some of the VPD parameters + * (notably the Core Clock frequency) are retrieved via requests to + * the firmware. On the other hand, we need these fairly early on + * so we do this right after getting ahold of the firmware. + * + * We need to do this after initializing the adapter because someone + * could have FLASHed a new VPD which won't be read by the firmware + * until we do the RESET ... + */ + if (!vpd_skip) { + ret = t4_get_vpd_params(adap, &adap->params.vpd); + if (ret < 0) + goto bye; + } + + /* Find out what ports are available to us. Note that we need to do + * this before calling adap_init0_no_config() since it needs nports + * and portvec ... + */ + v = + FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec); + if (ret < 0) + goto bye; + + adap->params.nports = hweight32(port_vec); + adap->params.portvec = port_vec; + + /* Give the SGE code a chance to pull in anything that it needs ... + * Note that this must be called after we retrieve our VPD parameters + * in order to know how to convert core ticks to seconds, etc. + */ + ret = t4_sge_init(adap); + if (ret < 0) + goto bye; + + /* Grab the SGE Doorbell Queue Timer values. If successful, that + * indicates that the Firmware and Hardware support this. + */ + params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_DBQ_TIMERTICK)); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, + 1, params, val); + + if (!ret) { + adap->sge.dbqtimer_tick = val[0]; + ret = t4_read_sge_dbqtimers(adap, + ARRAY_SIZE(adap->sge.dbqtimer_val), + adap->sge.dbqtimer_val); + } + + if (!ret) + adap->flags |= CXGB4_SGE_DBQ_TIMER; + + if (is_bypass_device(adap->pdev->device)) + adap->params.bypass = 1; + + /* + * Grab some of our basic fundamental operating parameters. + */ + params[0] = FW_PARAM_PFVF(EQ_START); + params[1] = FW_PARAM_PFVF(L2T_START); + params[2] = FW_PARAM_PFVF(L2T_END); + params[3] = FW_PARAM_PFVF(FILTER_START); + params[4] = FW_PARAM_PFVF(FILTER_END); + params[5] = FW_PARAM_PFVF(IQFLINT_START); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, val); + if (ret < 0) + goto bye; + adap->sge.egr_start = val[0]; + adap->l2t_start = val[1]; + adap->l2t_end = val[2]; + adap->tids.ftid_base = val[3]; + adap->tids.nftids = val[4] - val[3] + 1; + adap->sge.ingr_start = val[5]; + + if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) { + params[0] = FW_PARAM_PFVF(HPFILTER_START); + params[1] = FW_PARAM_PFVF(HPFILTER_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, + params, val); + if (ret < 0) + goto bye; + + adap->tids.hpftid_base = val[0]; + adap->tids.nhpftids = val[1] - val[0] + 1; + + /* Read the raw mps entries. In T6, the last 2 tcam entries + * are reserved for raw mac addresses (rawf = 2, one per port). + */ + params[0] = FW_PARAM_PFVF(RAWF_START); + params[1] = FW_PARAM_PFVF(RAWF_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, + params, val); + if (ret == 0) { + adap->rawf_start = val[0]; + adap->rawf_cnt = val[1] - val[0] + 1; + } + + adap->tids.tid_base = + t4_read_reg(adap, LE_DB_ACTIVE_TABLE_START_INDEX_A); + } + + /* qids (ingress/egress) returned from firmware can be anywhere + * in the range from EQ(IQFLINT)_START to EQ(IQFLINT)_END. + * Hence driver needs to allocate memory for this range to + * store the queue info. Get the highest IQFLINT/EQ index returned + * in FW_EQ_*_CMD.alloc command. + */ + params[0] = FW_PARAM_PFVF(EQ_END); + params[1] = FW_PARAM_PFVF(IQFLINT_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); + if (ret < 0) + goto bye; + adap->sge.egr_sz = val[0] - adap->sge.egr_start + 1; + adap->sge.ingr_sz = val[1] - adap->sge.ingr_start + 1; + + adap->sge.egr_map = kcalloc(adap->sge.egr_sz, + sizeof(*adap->sge.egr_map), GFP_KERNEL); + if (!adap->sge.egr_map) { + ret = -ENOMEM; + goto bye; + } + + adap->sge.ingr_map = kcalloc(adap->sge.ingr_sz, + sizeof(*adap->sge.ingr_map), GFP_KERNEL); + if (!adap->sge.ingr_map) { + ret = -ENOMEM; + goto bye; + } + + /* Allocate the memory for the vaious egress queue bitmaps + * ie starving_fl, txq_maperr and blocked_fl. + */ + adap->sge.starving_fl = bitmap_zalloc(adap->sge.egr_sz, GFP_KERNEL); + if (!adap->sge.starving_fl) { + ret = -ENOMEM; + goto bye; + } + + adap->sge.txq_maperr = bitmap_zalloc(adap->sge.egr_sz, GFP_KERNEL); + if (!adap->sge.txq_maperr) { + ret = -ENOMEM; + goto bye; + } + +#ifdef CONFIG_DEBUG_FS + adap->sge.blocked_fl = bitmap_zalloc(adap->sge.egr_sz, GFP_KERNEL); + if (!adap->sge.blocked_fl) { + ret = -ENOMEM; + goto bye; + } +#endif + + params[0] = FW_PARAM_PFVF(CLIP_START); + params[1] = FW_PARAM_PFVF(CLIP_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); + if (ret < 0) + goto bye; + adap->clipt_start = val[0]; + adap->clipt_end = val[1]; + + /* Get the supported number of traffic classes */ + params[0] = FW_PARAM_DEV(NUM_TM_CLASS); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, params, val); + if (ret < 0) { + /* We couldn't retrieve the number of Traffic Classes + * supported by the hardware/firmware. So we hard + * code it here. + */ + adap->params.nsched_cls = is_t4(adap->params.chip) ? 15 : 16; + } else { + adap->params.nsched_cls = val[0]; + } + + /* query params related to active filter region */ + params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START); + params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); + /* If Active filter size is set we enable establishing + * offload connection through firmware work request + */ + if ((val[0] != val[1]) && (ret >= 0)) { + adap->flags |= CXGB4_FW_OFLD_CONN; + adap->tids.aftid_base = val[0]; + adap->tids.aftid_end = val[1]; + } + + /* If we're running on newer firmware, let it know that we're + * prepared to deal with encapsulated CPL messages. Older + * firmware won't understand this and we'll just get + * unencapsulated messages ... + */ + params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); + val[0] = 1; + (void)t4_set_params(adap, adap->mbox, adap->pf, 0, 1, params, val); + + /* + * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL + * capability. Earlier versions of the firmware didn't have the + * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no + * permission to use ULPTX MEMWRITE DSGL. + */ + if (is_t4(adap->params.chip)) { + adap->params.ulptx_memwrite_dsgl = false; + } else { + params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, + 1, params, val); + adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0); + } + + /* See if FW supports FW_RI_FR_NSMR_TPTE_WR work request */ + params[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, + 1, params, val); + adap->params.fr_nsmr_tpte_wr_support = (ret == 0 && val[0] != 0); + + /* See if FW supports FW_FILTER2 work request */ + if (is_t4(adap->params.chip)) { + adap->params.filter2_wr_support = false; + } else { + params[0] = FW_PARAM_DEV(FILTER2_WR); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, + 1, params, val); + adap->params.filter2_wr_support = (ret == 0 && val[0] != 0); + } + + /* Check if FW supports returning vin and smt index. + * If this is not supported, driver will interpret + * these values from viid. + */ + params[0] = FW_PARAM_DEV(OPAQUE_VIID_SMT_EXTN); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, + 1, params, val); + adap->params.viid_smt_extn_support = (ret == 0 && val[0] != 0); + + /* + * Get device capabilities so we can determine what resources we need + * to manage. + */ + memset(&caps_cmd, 0, sizeof(caps_cmd)); + caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F); + caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); + ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd), + &caps_cmd); + if (ret < 0) + goto bye; + + /* hash filter has some mandatory register settings to be tested and for + * that it needs to test whether offload is enabled or not, hence + * checking and setting it here. + */ + if (caps_cmd.ofldcaps) + adap->params.offload = 1; + + if (caps_cmd.ofldcaps || + (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_HASHFILTER)) || + (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_ETHOFLD))) { + /* query offload-related parameters */ + params[0] = FW_PARAM_DEV(NTID); + params[1] = FW_PARAM_PFVF(SERVER_START); + params[2] = FW_PARAM_PFVF(SERVER_END); + params[3] = FW_PARAM_PFVF(TDDP_START); + params[4] = FW_PARAM_PFVF(TDDP_END); + params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, + params, val); + if (ret < 0) + goto bye; + adap->tids.ntids = val[0]; + adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS); + adap->tids.stid_base = val[1]; + adap->tids.nstids = val[2] - val[1] + 1; + /* + * Setup server filter region. Divide the available filter + * region into two parts. Regular filters get 1/3rd and server + * filters get 2/3rd part. This is only enabled if workarond + * path is enabled. + * 1. For regular filters. + * 2. Server filter: This are special filters which are used + * to redirect SYN packets to offload queue. + */ + if (adap->flags & CXGB4_FW_OFLD_CONN && !is_bypass(adap)) { + adap->tids.sftid_base = adap->tids.ftid_base + + DIV_ROUND_UP(adap->tids.nftids, 3); + adap->tids.nsftids = adap->tids.nftids - + DIV_ROUND_UP(adap->tids.nftids, 3); + adap->tids.nftids = adap->tids.sftid_base - + adap->tids.ftid_base; + } + adap->vres.ddp.start = val[3]; + adap->vres.ddp.size = val[4] - val[3] + 1; + adap->params.ofldq_wr_cred = val[5]; + + if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_HASHFILTER)) { + init_hash_filter(adap); + } else { + adap->num_ofld_uld += 1; + } + + if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_ETHOFLD)) { + params[0] = FW_PARAM_PFVF(ETHOFLD_START); + params[1] = FW_PARAM_PFVF(ETHOFLD_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, + params, val); + if (!ret) { + adap->tids.eotid_base = val[0]; + adap->tids.neotids = min_t(u32, MAX_ATIDS, + val[1] - val[0] + 1); + adap->params.ethofld = 1; + } + } + } + if (caps_cmd.rdmacaps) { + params[0] = FW_PARAM_PFVF(STAG_START); + params[1] = FW_PARAM_PFVF(STAG_END); + params[2] = FW_PARAM_PFVF(RQ_START); + params[3] = FW_PARAM_PFVF(RQ_END); + params[4] = FW_PARAM_PFVF(PBL_START); + params[5] = FW_PARAM_PFVF(PBL_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, + params, val); + if (ret < 0) + goto bye; + adap->vres.stag.start = val[0]; + adap->vres.stag.size = val[1] - val[0] + 1; + adap->vres.rq.start = val[2]; + adap->vres.rq.size = val[3] - val[2] + 1; + adap->vres.pbl.start = val[4]; + adap->vres.pbl.size = val[5] - val[4] + 1; + + params[0] = FW_PARAM_PFVF(SRQ_START); + params[1] = FW_PARAM_PFVF(SRQ_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, + params, val); + if (!ret) { + adap->vres.srq.start = val[0]; + adap->vres.srq.size = val[1] - val[0] + 1; + } + if (adap->vres.srq.size) { + adap->srq = t4_init_srq(adap->vres.srq.size); + if (!adap->srq) + dev_warn(&adap->pdev->dev, "could not allocate SRQ, continuing\n"); + } + + params[0] = FW_PARAM_PFVF(SQRQ_START); + params[1] = FW_PARAM_PFVF(SQRQ_END); + params[2] = FW_PARAM_PFVF(CQ_START); + params[3] = FW_PARAM_PFVF(CQ_END); + params[4] = FW_PARAM_PFVF(OCQ_START); + params[5] = FW_PARAM_PFVF(OCQ_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, + val); + if (ret < 0) + goto bye; + adap->vres.qp.start = val[0]; + adap->vres.qp.size = val[1] - val[0] + 1; + adap->vres.cq.start = val[2]; + adap->vres.cq.size = val[3] - val[2] + 1; + adap->vres.ocq.start = val[4]; + adap->vres.ocq.size = val[5] - val[4] + 1; + + params[0] = FW_PARAM_DEV(MAXORDIRD_QP); + params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, + val); + if (ret < 0) { + adap->params.max_ordird_qp = 8; + adap->params.max_ird_adapter = 32 * adap->tids.ntids; + ret = 0; + } else { + adap->params.max_ordird_qp = val[0]; + adap->params.max_ird_adapter = val[1]; + } + dev_info(adap->pdev_dev, + "max_ordird_qp %d max_ird_adapter %d\n", + adap->params.max_ordird_qp, + adap->params.max_ird_adapter); + + /* Enable write_with_immediate if FW supports it */ + params[0] = FW_PARAM_DEV(RDMA_WRITE_WITH_IMM); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, params, + val); + adap->params.write_w_imm_support = (ret == 0 && val[0] != 0); + + /* Enable write_cmpl if FW supports it */ + params[0] = FW_PARAM_DEV(RI_WRITE_CMPL_WR); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, params, + val); + adap->params.write_cmpl_support = (ret == 0 && val[0] != 0); + adap->num_ofld_uld += 2; + } + if (caps_cmd.iscsicaps) { + params[0] = FW_PARAM_PFVF(ISCSI_START); + params[1] = FW_PARAM_PFVF(ISCSI_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, + params, val); + if (ret < 0) + goto bye; + adap->vres.iscsi.start = val[0]; + adap->vres.iscsi.size = val[1] - val[0] + 1; + if (is_t6(adap->params.chip)) { + params[0] = FW_PARAM_PFVF(PPOD_EDRAM_START); + params[1] = FW_PARAM_PFVF(PPOD_EDRAM_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, + params, val); + if (!ret) { + adap->vres.ppod_edram.start = val[0]; + adap->vres.ppod_edram.size = + val[1] - val[0] + 1; + + dev_info(adap->pdev_dev, + "ppod edram start 0x%x end 0x%x size 0x%x\n", + val[0], val[1], + adap->vres.ppod_edram.size); + } + } + /* LIO target and cxgb4i initiaitor */ + adap->num_ofld_uld += 2; + } + if (caps_cmd.cryptocaps) { + if (ntohs(caps_cmd.cryptocaps) & + FW_CAPS_CONFIG_CRYPTO_LOOKASIDE) { + params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, + 2, params, val); + if (ret < 0) { + if (ret != -EINVAL) + goto bye; + } else { + adap->vres.ncrypto_fc = val[0]; + } + adap->num_ofld_uld += 1; + } + if (ntohs(caps_cmd.cryptocaps) & + FW_CAPS_CONFIG_TLS_INLINE) { + params[0] = FW_PARAM_PFVF(TLS_START); + params[1] = FW_PARAM_PFVF(TLS_END); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, + 2, params, val); + if (ret < 0) + goto bye; + adap->vres.key.start = val[0]; + adap->vres.key.size = val[1] - val[0] + 1; + adap->num_uld += 1; + } + adap->params.crypto = ntohs(caps_cmd.cryptocaps); + } + + /* The MTU/MSS Table is initialized by now, so load their values. If + * we're initializing the adapter, then we'll make any modifications + * we want to the MTU/MSS Table and also initialize the congestion + * parameters. + */ + t4_read_mtu_tbl(adap, adap->params.mtus, NULL); + if (state != DEV_STATE_INIT) { + int i; + + /* The default MTU Table contains values 1492 and 1500. + * However, for TCP, it's better to have two values which are + * a multiple of 8 +/- 4 bytes apart near this popular MTU. + * This allows us to have a TCP Data Payload which is a + * multiple of 8 regardless of what combination of TCP Options + * are in use (always a multiple of 4 bytes) which is + * important for performance reasons. For instance, if no + * options are in use, then we have a 20-byte IP header and a + * 20-byte TCP header. In this case, a 1500-byte MSS would + * result in a TCP Data Payload of 1500 - 40 == 1460 bytes + * which is not a multiple of 8. So using an MSS of 1488 in + * this case results in a TCP Data Payload of 1448 bytes which + * is a multiple of 8. On the other hand, if 12-byte TCP Time + * Stamps have been negotiated, then an MTU of 1500 bytes + * results in a TCP Data Payload of 1448 bytes which, as + * above, is a multiple of 8 bytes ... + */ + for (i = 0; i < NMTUS; i++) + if (adap->params.mtus[i] == 1492) { + adap->params.mtus[i] = 1488; + break; + } + + t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, + adap->params.b_wnd); + } + t4_init_sge_params(adap); + adap->flags |= CXGB4_FW_OK; + t4_init_tp_params(adap, true); + return 0; + + /* + * Something bad happened. If a command timed out or failed with EIO + * FW does not operate within its spec or something catastrophic + * happened to HW/FW, stop issuing commands. + */ +bye: + adap_free_hma_mem(adap); + kfree(adap->sge.egr_map); + kfree(adap->sge.ingr_map); + bitmap_free(adap->sge.starving_fl); + bitmap_free(adap->sge.txq_maperr); +#ifdef CONFIG_DEBUG_FS + bitmap_free(adap->sge.blocked_fl); +#endif + if (ret != -ETIMEDOUT && ret != -EIO) + t4_fw_bye(adap, adap->mbox); + return ret; +} + +/* EEH callbacks */ + +static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + int i; + struct adapter *adap = pci_get_drvdata(pdev); + + if (!adap) + goto out; + + rtnl_lock(); + adap->flags &= ~CXGB4_FW_OK; + notify_ulds(adap, CXGB4_STATE_START_RECOVERY); + spin_lock(&adap->stats_lock); + for_each_port(adap, i) { + struct net_device *dev = adap->port[i]; + if (dev) { + netif_device_detach(dev); + netif_carrier_off(dev); + } + } + spin_unlock(&adap->stats_lock); + disable_interrupts(adap); + if (adap->flags & CXGB4_FULL_INIT_DONE) + cxgb_down(adap); + rtnl_unlock(); + if ((adap->flags & CXGB4_DEV_ENABLED)) { + pci_disable_device(pdev); + adap->flags &= ~CXGB4_DEV_ENABLED; + } +out: return state == pci_channel_io_perm_failure ? + PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev) +{ + int i, ret; + struct fw_caps_config_cmd c; + struct adapter *adap = pci_get_drvdata(pdev); + + if (!adap) { + pci_restore_state(pdev); + pci_save_state(pdev); + return PCI_ERS_RESULT_RECOVERED; + } + + if (!(adap->flags & CXGB4_DEV_ENABLED)) { + if (pci_enable_device(pdev)) { + dev_err(&pdev->dev, "Cannot reenable PCI " + "device after reset\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + adap->flags |= CXGB4_DEV_ENABLED; + } + + pci_set_master(pdev); + pci_restore_state(pdev); + pci_save_state(pdev); + + if (t4_wait_dev_ready(adap->regs) < 0) + return PCI_ERS_RESULT_DISCONNECT; + if (t4_fw_hello(adap, adap->mbox, adap->pf, MASTER_MUST, NULL) < 0) + return PCI_ERS_RESULT_DISCONNECT; + adap->flags |= CXGB4_FW_OK; + if (adap_init1(adap, &c)) + return PCI_ERS_RESULT_DISCONNECT; + + for_each_port(adap, i) { + struct port_info *pi = adap2pinfo(adap, i); + u8 vivld = 0, vin = 0; + + ret = t4_alloc_vi(adap, adap->mbox, pi->tx_chan, adap->pf, 0, 1, + NULL, NULL, &vivld, &vin); + if (ret < 0) + return PCI_ERS_RESULT_DISCONNECT; + pi->viid = ret; + pi->xact_addr_filt = -1; + /* If fw supports returning the VIN as part of FW_VI_CMD, + * save the returned values. + */ + if (adap->params.viid_smt_extn_support) { + pi->vivld = vivld; + pi->vin = vin; + } else { + /* Retrieve the values from VIID */ + pi->vivld = FW_VIID_VIVLD_G(pi->viid); + pi->vin = FW_VIID_VIN_G(pi->viid); + } + } + + t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, + adap->params.b_wnd); + setup_memwin(adap); + if (cxgb_up(adap)) + return PCI_ERS_RESULT_DISCONNECT; + return PCI_ERS_RESULT_RECOVERED; +} + +static void eeh_resume(struct pci_dev *pdev) +{ + int i; + struct adapter *adap = pci_get_drvdata(pdev); + + if (!adap) + return; + + rtnl_lock(); + for_each_port(adap, i) { + struct net_device *dev = adap->port[i]; + if (dev) { + if (netif_running(dev)) { + link_start(dev); + cxgb_set_rxmode(dev); + } + netif_device_attach(dev); + } + } + rtnl_unlock(); +} + +static void eeh_reset_prepare(struct pci_dev *pdev) +{ + struct adapter *adapter = pci_get_drvdata(pdev); + int i; + + if (adapter->pf != 4) + return; + + adapter->flags &= ~CXGB4_FW_OK; + + notify_ulds(adapter, CXGB4_STATE_DOWN); + + for_each_port(adapter, i) + if (adapter->port[i]->reg_state == NETREG_REGISTERED) + cxgb_close(adapter->port[i]); + + disable_interrupts(adapter); + cxgb4_free_mps_ref_entries(adapter); + + adap_free_hma_mem(adapter); + + if (adapter->flags & CXGB4_FULL_INIT_DONE) + cxgb_down(adapter); +} + +static void eeh_reset_done(struct pci_dev *pdev) +{ + struct adapter *adapter = pci_get_drvdata(pdev); + int err, i; + + if (adapter->pf != 4) + return; + + err = t4_wait_dev_ready(adapter->regs); + if (err < 0) { + dev_err(adapter->pdev_dev, + "Device not ready, err %d", err); + return; + } + + setup_memwin(adapter); + + err = adap_init0(adapter, 1); + if (err) { + dev_err(adapter->pdev_dev, + "Adapter init failed, err %d", err); + return; + } + + setup_memwin_rdma(adapter); + + if (adapter->flags & CXGB4_FW_OK) { + err = t4_port_init(adapter, adapter->pf, adapter->pf, 0); + if (err) { + dev_err(adapter->pdev_dev, + "Port init failed, err %d", err); + return; + } + } + + err = cfg_queues(adapter); + if (err) { + dev_err(adapter->pdev_dev, + "Config queues failed, err %d", err); + return; + } + + cxgb4_init_mps_ref_entries(adapter); + + err = setup_fw_sge_queues(adapter); + if (err) { + dev_err(adapter->pdev_dev, + "FW sge queue allocation failed, err %d", err); + return; + } + + for_each_port(adapter, i) + if (adapter->port[i]->reg_state == NETREG_REGISTERED) + cxgb_open(adapter->port[i]); +} + +static const struct pci_error_handlers cxgb4_eeh = { + .error_detected = eeh_err_detected, + .slot_reset = eeh_slot_reset, + .resume = eeh_resume, + .reset_prepare = eeh_reset_prepare, + .reset_done = eeh_reset_done, +}; + +/* Return true if the Link Configuration supports "High Speeds" (those greater + * than 1Gb/s). + */ +static inline bool is_x_10g_port(const struct link_config *lc) +{ + unsigned int speeds, high_speeds; + + speeds = FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_G(lc->pcaps)); + high_speeds = speeds & + ~(FW_PORT_CAP32_SPEED_100M | FW_PORT_CAP32_SPEED_1G); + + return high_speeds != 0; +} + +/* Perform default configuration of DMA queues depending on the number and type + * of ports we found and the number of available CPUs. Most settings can be + * modified by the admin prior to actual use. + */ +static int cfg_queues(struct adapter *adap) +{ + u32 avail_qsets, avail_eth_qsets, avail_uld_qsets; + u32 ncpus = num_online_cpus(); + u32 niqflint, neq, num_ulds; + struct sge *s = &adap->sge; + u32 i, n10g = 0, qidx = 0; + u32 q10g = 0, q1g; + + /* Reduce memory usage in kdump environment, disable all offload. */ + if (is_kdump_kernel() || (is_uld(adap) && t4_uld_mem_alloc(adap))) { + adap->params.offload = 0; + adap->params.crypto = 0; + adap->params.ethofld = 0; + } + + /* Calculate the number of Ethernet Queue Sets available based on + * resources provisioned for us. We always have an Asynchronous + * Firmware Event Ingress Queue. If we're operating in MSI or Legacy + * IRQ Pin Interrupt mode, then we'll also have a Forwarded Interrupt + * Ingress Queue. Meanwhile, we need two Egress Queues for each + * Queue Set: one for the Free List and one for the Ethernet TX Queue. + * + * Note that we should also take into account all of the various + * Offload Queues. But, in any situation where we're operating in + * a Resource Constrained Provisioning environment, doing any Offload + * at all is problematic ... + */ + niqflint = adap->params.pfres.niqflint - 1; + if (!(adap->flags & CXGB4_USING_MSIX)) + niqflint--; + neq = adap->params.pfres.neq / 2; + avail_qsets = min(niqflint, neq); + + if (avail_qsets < adap->params.nports) { + dev_err(adap->pdev_dev, "avail_eth_qsets=%d < nports=%d\n", + avail_qsets, adap->params.nports); + return -ENOMEM; + } + + /* Count the number of 10Gb/s or better ports */ + for_each_port(adap, i) + n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg); + + avail_eth_qsets = min_t(u32, avail_qsets, MAX_ETH_QSETS); + + /* We default to 1 queue per non-10G port and up to # of cores queues + * per 10G port. + */ + if (n10g) + q10g = (avail_eth_qsets - (adap->params.nports - n10g)) / n10g; + +#ifdef CONFIG_CHELSIO_T4_DCB + /* For Data Center Bridging support we need to be able to support up + * to 8 Traffic Priorities; each of which will be assigned to its + * own TX Queue in order to prevent Head-Of-Line Blocking. + */ + q1g = 8; + if (adap->params.nports * 8 > avail_eth_qsets) { + dev_err(adap->pdev_dev, "DCB avail_eth_qsets=%d < %d!\n", + avail_eth_qsets, adap->params.nports * 8); + return -ENOMEM; + } + + if (adap->params.nports * ncpus < avail_eth_qsets) + q10g = max(8U, ncpus); + else + q10g = max(8U, q10g); + + while ((q10g * n10g) > + (avail_eth_qsets - (adap->params.nports - n10g) * q1g)) + q10g--; + +#else /* !CONFIG_CHELSIO_T4_DCB */ + q1g = 1; + q10g = min(q10g, ncpus); +#endif /* !CONFIG_CHELSIO_T4_DCB */ + if (is_kdump_kernel()) { + q10g = 1; + q1g = 1; + } + + for_each_port(adap, i) { + struct port_info *pi = adap2pinfo(adap, i); + + pi->first_qset = qidx; + pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : q1g; + qidx += pi->nqsets; + } + + s->ethqsets = qidx; + s->max_ethqsets = qidx; /* MSI-X may lower it later */ + avail_qsets -= qidx; + + if (is_uld(adap)) { + /* For offload we use 1 queue/channel if all ports are up to 1G, + * otherwise we divide all available queues amongst the channels + * capped by the number of available cores. + */ + num_ulds = adap->num_uld + adap->num_ofld_uld; + i = min_t(u32, MAX_OFLD_QSETS, ncpus); + avail_uld_qsets = roundup(i, adap->params.nports); + if (avail_qsets < num_ulds * adap->params.nports) { + adap->params.offload = 0; + adap->params.crypto = 0; + s->ofldqsets = 0; + } else if (avail_qsets < num_ulds * avail_uld_qsets || !n10g) { + s->ofldqsets = adap->params.nports; + } else { + s->ofldqsets = avail_uld_qsets; + } + + avail_qsets -= num_ulds * s->ofldqsets; + } + + /* ETHOFLD Queues used for QoS offload should follow same + * allocation scheme as normal Ethernet Queues. + */ + if (is_ethofld(adap)) { + if (avail_qsets < s->max_ethqsets) { + adap->params.ethofld = 0; + s->eoqsets = 0; + } else { + s->eoqsets = s->max_ethqsets; + } + avail_qsets -= s->eoqsets; + } + + /* Mirror queues must follow same scheme as normal Ethernet + * Queues, when there are enough queues available. Otherwise, + * allocate at least 1 queue per port. If even 1 queue is not + * available, then disable mirror queues support. + */ + if (avail_qsets >= s->max_ethqsets) + s->mirrorqsets = s->max_ethqsets; + else if (avail_qsets >= adap->params.nports) + s->mirrorqsets = adap->params.nports; + else + s->mirrorqsets = 0; + avail_qsets -= s->mirrorqsets; + + for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) { + struct sge_eth_rxq *r = &s->ethrxq[i]; + + init_rspq(adap, &r->rspq, 5, 10, 1024, 64); + r->fl.size = 72; + } + + for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++) + s->ethtxq[i].q.size = 1024; + + for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) + s->ctrlq[i].q.size = 512; + + if (!is_t4(adap->params.chip)) + s->ptptxq.q.size = 8; + + init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64); + init_rspq(adap, &s->intrq, 0, 1, 512, 64); + + return 0; +} + +/* + * Reduce the number of Ethernet queues across all ports to at most n. + * n provides at least one queue per port. + */ +static void reduce_ethqs(struct adapter *adap, int n) +{ + int i; + struct port_info *pi; + + while (n < adap->sge.ethqsets) + for_each_port(adap, i) { + pi = adap2pinfo(adap, i); + if (pi->nqsets > 1) { + pi->nqsets--; + adap->sge.ethqsets--; + if (adap->sge.ethqsets <= n) + break; + } + } + + n = 0; + for_each_port(adap, i) { + pi = adap2pinfo(adap, i); + pi->first_qset = n; + n += pi->nqsets; + } +} + +static int alloc_msix_info(struct adapter *adap, u32 num_vec) +{ + struct msix_info *msix_info; + + msix_info = kcalloc(num_vec, sizeof(*msix_info), GFP_KERNEL); + if (!msix_info) + return -ENOMEM; + + adap->msix_bmap.msix_bmap = bitmap_zalloc(num_vec, GFP_KERNEL); + if (!adap->msix_bmap.msix_bmap) { + kfree(msix_info); + return -ENOMEM; + } + + spin_lock_init(&adap->msix_bmap.lock); + adap->msix_bmap.mapsize = num_vec; + + adap->msix_info = msix_info; + return 0; +} + +static void free_msix_info(struct adapter *adap) +{ + bitmap_free(adap->msix_bmap.msix_bmap); + kfree(adap->msix_info); +} + +int cxgb4_get_msix_idx_from_bmap(struct adapter *adap) +{ + struct msix_bmap *bmap = &adap->msix_bmap; + unsigned int msix_idx; + unsigned long flags; + + spin_lock_irqsave(&bmap->lock, flags); + msix_idx = find_first_zero_bit(bmap->msix_bmap, bmap->mapsize); + if (msix_idx < bmap->mapsize) { + __set_bit(msix_idx, bmap->msix_bmap); + } else { + spin_unlock_irqrestore(&bmap->lock, flags); + return -ENOSPC; + } + + spin_unlock_irqrestore(&bmap->lock, flags); + return msix_idx; +} + +void cxgb4_free_msix_idx_in_bmap(struct adapter *adap, + unsigned int msix_idx) +{ + struct msix_bmap *bmap = &adap->msix_bmap; + unsigned long flags; + + spin_lock_irqsave(&bmap->lock, flags); + __clear_bit(msix_idx, bmap->msix_bmap); + spin_unlock_irqrestore(&bmap->lock, flags); +} + +/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */ +#define EXTRA_VECS 2 + +static int enable_msix(struct adapter *adap) +{ + u32 eth_need, uld_need = 0, ethofld_need = 0, mirror_need = 0; + u32 ethqsets = 0, ofldqsets = 0, eoqsets = 0, mirrorqsets = 0; + u8 num_uld = 0, nchan = adap->params.nports; + u32 i, want, need, num_vec; + struct sge *s = &adap->sge; + struct msix_entry *entries; + struct port_info *pi; + int allocated, ret; + + want = s->max_ethqsets; +#ifdef CONFIG_CHELSIO_T4_DCB + /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for + * each port. + */ + need = 8 * nchan; +#else + need = nchan; +#endif + eth_need = need; + if (is_uld(adap)) { + num_uld = adap->num_ofld_uld + adap->num_uld; + want += num_uld * s->ofldqsets; + uld_need = num_uld * nchan; + need += uld_need; + } + + if (is_ethofld(adap)) { + want += s->eoqsets; + ethofld_need = eth_need; + need += ethofld_need; + } + + if (s->mirrorqsets) { + want += s->mirrorqsets; + mirror_need = nchan; + need += mirror_need; + } + + want += EXTRA_VECS; + need += EXTRA_VECS; + + entries = kmalloc_array(want, sizeof(*entries), GFP_KERNEL); + if (!entries) + return -ENOMEM; + + for (i = 0; i < want; i++) + entries[i].entry = i; + + allocated = pci_enable_msix_range(adap->pdev, entries, need, want); + if (allocated < 0) { + /* Disable offload and attempt to get vectors for NIC + * only mode. + */ + want = s->max_ethqsets + EXTRA_VECS; + need = eth_need + EXTRA_VECS; + allocated = pci_enable_msix_range(adap->pdev, entries, + need, want); + if (allocated < 0) { + dev_info(adap->pdev_dev, + "Disabling MSI-X due to insufficient MSI-X vectors\n"); + ret = allocated; + goto out_free; + } + + dev_info(adap->pdev_dev, + "Disabling offload due to insufficient MSI-X vectors\n"); + adap->params.offload = 0; + adap->params.crypto = 0; + adap->params.ethofld = 0; + s->ofldqsets = 0; + s->eoqsets = 0; + s->mirrorqsets = 0; + uld_need = 0; + ethofld_need = 0; + mirror_need = 0; + } + + num_vec = allocated; + if (num_vec < want) { + /* Distribute available vectors to the various queue groups. + * Every group gets its minimum requirement and NIC gets top + * priority for leftovers. + */ + ethqsets = eth_need; + if (is_uld(adap)) + ofldqsets = nchan; + if (is_ethofld(adap)) + eoqsets = ethofld_need; + if (s->mirrorqsets) + mirrorqsets = mirror_need; + + num_vec -= need; + while (num_vec) { + if (num_vec < eth_need + ethofld_need || + ethqsets > s->max_ethqsets) + break; + + for_each_port(adap, i) { + pi = adap2pinfo(adap, i); + if (pi->nqsets < 2) + continue; + + ethqsets++; + num_vec--; + if (ethofld_need) { + eoqsets++; + num_vec--; + } + } + } + + if (is_uld(adap)) { + while (num_vec) { + if (num_vec < uld_need || + ofldqsets > s->ofldqsets) + break; + + ofldqsets++; + num_vec -= uld_need; + } + } + + if (s->mirrorqsets) { + while (num_vec) { + if (num_vec < mirror_need || + mirrorqsets > s->mirrorqsets) + break; + + mirrorqsets++; + num_vec -= mirror_need; + } + } + } else { + ethqsets = s->max_ethqsets; + if (is_uld(adap)) + ofldqsets = s->ofldqsets; + if (is_ethofld(adap)) + eoqsets = s->eoqsets; + if (s->mirrorqsets) + mirrorqsets = s->mirrorqsets; + } + + if (ethqsets < s->max_ethqsets) { + s->max_ethqsets = ethqsets; + reduce_ethqs(adap, ethqsets); + } + + if (is_uld(adap)) { + s->ofldqsets = ofldqsets; + s->nqs_per_uld = s->ofldqsets; + } + + if (is_ethofld(adap)) + s->eoqsets = eoqsets; + + if (s->mirrorqsets) { + s->mirrorqsets = mirrorqsets; + for_each_port(adap, i) { + pi = adap2pinfo(adap, i); + pi->nmirrorqsets = s->mirrorqsets / nchan; + mutex_init(&pi->vi_mirror_mutex); + } + } + + /* map for msix */ + ret = alloc_msix_info(adap, allocated); + if (ret) + goto out_disable_msix; + + for (i = 0; i < allocated; i++) { + adap->msix_info[i].vec = entries[i].vector; + adap->msix_info[i].idx = i; + } + + dev_info(adap->pdev_dev, + "%d MSI-X vectors allocated, nic %d eoqsets %d per uld %d mirrorqsets %d\n", + allocated, s->max_ethqsets, s->eoqsets, s->nqs_per_uld, + s->mirrorqsets); + + kfree(entries); + return 0; + +out_disable_msix: + pci_disable_msix(adap->pdev); + +out_free: + kfree(entries); + return ret; +} + +#undef EXTRA_VECS + +static int init_rss(struct adapter *adap) +{ + unsigned int i; + int err; + + err = t4_init_rss_mode(adap, adap->mbox); + if (err) + return err; + + for_each_port(adap, i) { + struct port_info *pi = adap2pinfo(adap, i); + + pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL); + if (!pi->rss) + return -ENOMEM; + } + return 0; +} + +/* Dump basic information about the adapter */ +static void print_adapter_info(struct adapter *adapter) +{ + /* Hardware/Firmware/etc. Version/Revision IDs */ + t4_dump_version_info(adapter); + + /* Software/Hardware configuration */ + dev_info(adapter->pdev_dev, "Configuration: %sNIC %s, %s capable\n", + is_offload(adapter) ? "R" : "", + ((adapter->flags & CXGB4_USING_MSIX) ? "MSI-X" : + (adapter->flags & CXGB4_USING_MSI) ? "MSI" : ""), + is_offload(adapter) ? "Offload" : "non-Offload"); +} + +static void print_port_info(const struct net_device *dev) +{ + char buf[80]; + char *bufp = buf; + const struct port_info *pi = netdev_priv(dev); + const struct adapter *adap = pi->adapter; + + if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100M) + bufp += sprintf(bufp, "100M/"); + if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_1G) + bufp += sprintf(bufp, "1G/"); + if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_10G) + bufp += sprintf(bufp, "10G/"); + if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_25G) + bufp += sprintf(bufp, "25G/"); + if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_40G) + bufp += sprintf(bufp, "40G/"); + if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_50G) + bufp += sprintf(bufp, "50G/"); + if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100G) + bufp += sprintf(bufp, "100G/"); + if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_200G) + bufp += sprintf(bufp, "200G/"); + if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_400G) + bufp += sprintf(bufp, "400G/"); + if (bufp != buf) + --bufp; + sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type)); + + netdev_info(dev, "Chelsio %s %s\n", adap->params.vpd.id, buf); +} + +/* + * Free the following resources: + * - memory used for tables + * - MSI/MSI-X + * - net devices + * - resources FW is holding for us + */ +static void free_some_resources(struct adapter *adapter) +{ + unsigned int i; + + kvfree(adapter->smt); + kvfree(adapter->l2t); + kvfree(adapter->srq); + t4_cleanup_sched(adapter); + kvfree(adapter->tids.tid_tab); + cxgb4_cleanup_tc_matchall(adapter); + cxgb4_cleanup_tc_mqprio(adapter); + cxgb4_cleanup_tc_flower(adapter); + cxgb4_cleanup_tc_u32(adapter); + cxgb4_cleanup_ethtool_filters(adapter); + kfree(adapter->sge.egr_map); + kfree(adapter->sge.ingr_map); + bitmap_free(adapter->sge.starving_fl); + bitmap_free(adapter->sge.txq_maperr); +#ifdef CONFIG_DEBUG_FS + bitmap_free(adapter->sge.blocked_fl); +#endif + disable_msi(adapter); + + for_each_port(adapter, i) + if (adapter->port[i]) { + struct port_info *pi = adap2pinfo(adapter, i); + + if (pi->viid != 0) + t4_free_vi(adapter, adapter->mbox, adapter->pf, + 0, pi->viid); + kfree(adap2pinfo(adapter, i)->rss); + free_netdev(adapter->port[i]); + } + if (adapter->flags & CXGB4_FW_OK) + t4_fw_bye(adapter, adapter->pf); +} + +#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN | \ + NETIF_F_GSO_UDP_L4) +#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \ + NETIF_F_GRO | NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA) +#define SEGMENT_SIZE 128 + +static int t4_get_chip_type(struct adapter *adap, int ver) +{ + u32 pl_rev = REV_G(t4_read_reg(adap, PL_REV_A)); + + switch (ver) { + case CHELSIO_T4: + return CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev); + case CHELSIO_T5: + return CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev); + case CHELSIO_T6: + return CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev); + default: + break; + } + return -EINVAL; +} + +#ifdef CONFIG_PCI_IOV +static void cxgb4_mgmt_setup(struct net_device *dev) +{ + dev->type = ARPHRD_NONE; + dev->mtu = 0; + dev->hard_header_len = 0; + dev->addr_len = 0; + dev->tx_queue_len = 0; + dev->flags |= IFF_NOARP; + dev->priv_flags |= IFF_NO_QUEUE; + + /* Initialize the device structure. */ + dev->netdev_ops = &cxgb4_mgmt_netdev_ops; + dev->ethtool_ops = &cxgb4_mgmt_ethtool_ops; +} + +static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs) +{ + struct adapter *adap = pci_get_drvdata(pdev); + int err = 0; + int current_vfs = pci_num_vf(pdev); + u32 pcie_fw; + + pcie_fw = readl(adap->regs + PCIE_FW_A); + /* Check if fw is initialized */ + if (!(pcie_fw & PCIE_FW_INIT_F)) { + dev_warn(&pdev->dev, "Device not initialized\n"); + return -EOPNOTSUPP; + } + + /* If any of the VF's is already assigned to Guest OS, then + * SRIOV for the same cannot be modified + */ + if (current_vfs && pci_vfs_assigned(pdev)) { + dev_err(&pdev->dev, + "Cannot modify SR-IOV while VFs are assigned\n"); + return current_vfs; + } + /* Note that the upper-level code ensures that we're never called with + * a non-zero "num_vfs" when we already have VFs instantiated. But + * it never hurts to code defensively. + */ + if (num_vfs != 0 && current_vfs != 0) + return -EBUSY; + + /* Nothing to do for no change. */ + if (num_vfs == current_vfs) + return num_vfs; + + /* Disable SRIOV when zero is passed. */ + if (!num_vfs) { + pci_disable_sriov(pdev); + /* free VF Management Interface */ + unregister_netdev(adap->port[0]); + free_netdev(adap->port[0]); + adap->port[0] = NULL; + + /* free VF resources */ + adap->num_vfs = 0; + kfree(adap->vfinfo); + adap->vfinfo = NULL; + return 0; + } + + if (!current_vfs) { + struct fw_pfvf_cmd port_cmd, port_rpl; + struct net_device *netdev; + unsigned int pmask, port; + struct pci_dev *pbridge; + struct port_info *pi; + char name[IFNAMSIZ]; + u32 devcap2; + u16 flags; + + /* If we want to instantiate Virtual Functions, then our + * parent bridge's PCI-E needs to support Alternative Routing + * ID (ARI) because our VFs will show up at function offset 8 + * and above. + */ + pbridge = pdev->bus->self; + pcie_capability_read_word(pbridge, PCI_EXP_FLAGS, &flags); + pcie_capability_read_dword(pbridge, PCI_EXP_DEVCAP2, &devcap2); + + if ((flags & PCI_EXP_FLAGS_VERS) < 2 || + !(devcap2 & PCI_EXP_DEVCAP2_ARI)) { + /* Our parent bridge does not support ARI so issue a + * warning and skip instantiating the VFs. They + * won't be reachable. + */ + dev_warn(&pdev->dev, "Parent bridge %02x:%02x.%x doesn't support ARI; can't instantiate Virtual Functions\n", + pbridge->bus->number, PCI_SLOT(pbridge->devfn), + PCI_FUNC(pbridge->devfn)); + return -ENOTSUPP; + } + memset(&port_cmd, 0, sizeof(port_cmd)); + port_cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PFVF_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F | + FW_PFVF_CMD_PFN_V(adap->pf) | + FW_PFVF_CMD_VFN_V(0)); + port_cmd.retval_len16 = cpu_to_be32(FW_LEN16(port_cmd)); + err = t4_wr_mbox(adap, adap->mbox, &port_cmd, sizeof(port_cmd), + &port_rpl); + if (err) + return err; + pmask = FW_PFVF_CMD_PMASK_G(be32_to_cpu(port_rpl.type_to_neq)); + port = ffs(pmask) - 1; + /* Allocate VF Management Interface. */ + snprintf(name, IFNAMSIZ, "mgmtpf%d,%d", adap->adap_idx, + adap->pf); + netdev = alloc_netdev(sizeof(struct port_info), + name, NET_NAME_UNKNOWN, cxgb4_mgmt_setup); + if (!netdev) + return -ENOMEM; + + pi = netdev_priv(netdev); + pi->adapter = adap; + pi->lport = port; + pi->tx_chan = port; + SET_NETDEV_DEV(netdev, &pdev->dev); + + adap->port[0] = netdev; + pi->port_id = 0; + + err = register_netdev(adap->port[0]); + if (err) { + pr_info("Unable to register VF mgmt netdev %s\n", name); + free_netdev(adap->port[0]); + adap->port[0] = NULL; + return err; + } + /* Allocate and set up VF Information. */ + adap->vfinfo = kcalloc(pci_sriov_get_totalvfs(pdev), + sizeof(struct vf_info), GFP_KERNEL); + if (!adap->vfinfo) { + unregister_netdev(adap->port[0]); + free_netdev(adap->port[0]); + adap->port[0] = NULL; + return -ENOMEM; + } + cxgb4_mgmt_fill_vf_station_mac_addr(adap); + } + /* Instantiate the requested number of VFs. */ + err = pci_enable_sriov(pdev, num_vfs); + if (err) { + pr_info("Unable to instantiate %d VFs\n", num_vfs); + if (!current_vfs) { + unregister_netdev(adap->port[0]); + free_netdev(adap->port[0]); + adap->port[0] = NULL; + kfree(adap->vfinfo); + adap->vfinfo = NULL; + } + return err; + } + + adap->num_vfs = num_vfs; + return num_vfs; +} +#endif /* CONFIG_PCI_IOV */ + +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) || IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) + +static int chcr_offload_state(struct adapter *adap, + enum cxgb4_netdev_tls_ops op_val) +{ + switch (op_val) { +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + case CXGB4_TLSDEV_OPS: + if (!adap->uld[CXGB4_ULD_KTLS].handle) { + dev_dbg(adap->pdev_dev, "ch_ktls driver is not loaded\n"); + return -EOPNOTSUPP; + } + if (!adap->uld[CXGB4_ULD_KTLS].tlsdev_ops) { + dev_dbg(adap->pdev_dev, + "ch_ktls driver has no registered tlsdev_ops\n"); + return -EOPNOTSUPP; + } + break; +#endif /* CONFIG_CHELSIO_TLS_DEVICE */ +#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) + case CXGB4_XFRMDEV_OPS: + if (!adap->uld[CXGB4_ULD_IPSEC].handle) { + dev_dbg(adap->pdev_dev, "chipsec driver is not loaded\n"); + return -EOPNOTSUPP; + } + if (!adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops) { + dev_dbg(adap->pdev_dev, + "chipsec driver has no registered xfrmdev_ops\n"); + return -EOPNOTSUPP; + } + break; +#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ + default: + dev_dbg(adap->pdev_dev, + "driver has no support for offload %d\n", op_val); + return -EOPNOTSUPP; + } + + return 0; +} + +#endif /* CONFIG_CHELSIO_TLS_DEVICE || CONFIG_CHELSIO_IPSEC_INLINE */ + +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + +static int cxgb4_ktls_dev_add(struct net_device *netdev, struct sock *sk, + enum tls_offload_ctx_dir direction, + struct tls_crypto_info *crypto_info, + u32 tcp_sn) +{ + struct adapter *adap = netdev2adap(netdev); + int ret; + + mutex_lock(&uld_mutex); + ret = chcr_offload_state(adap, CXGB4_TLSDEV_OPS); + if (ret) + goto out_unlock; + + ret = cxgb4_set_ktls_feature(adap, FW_PARAMS_PARAM_DEV_KTLS_HW_ENABLE); + if (ret) + goto out_unlock; + + ret = adap->uld[CXGB4_ULD_KTLS].tlsdev_ops->tls_dev_add(netdev, sk, + direction, + crypto_info, + tcp_sn); + /* if there is a failure, clear the refcount */ + if (ret) + cxgb4_set_ktls_feature(adap, + FW_PARAMS_PARAM_DEV_KTLS_HW_DISABLE); +out_unlock: + mutex_unlock(&uld_mutex); + return ret; +} + +static void cxgb4_ktls_dev_del(struct net_device *netdev, + struct tls_context *tls_ctx, + enum tls_offload_ctx_dir direction) +{ + struct adapter *adap = netdev2adap(netdev); + + mutex_lock(&uld_mutex); + if (chcr_offload_state(adap, CXGB4_TLSDEV_OPS)) + goto out_unlock; + + adap->uld[CXGB4_ULD_KTLS].tlsdev_ops->tls_dev_del(netdev, tls_ctx, + direction); + +out_unlock: + cxgb4_set_ktls_feature(adap, FW_PARAMS_PARAM_DEV_KTLS_HW_DISABLE); + mutex_unlock(&uld_mutex); +} + +static const struct tlsdev_ops cxgb4_ktls_ops = { + .tls_dev_add = cxgb4_ktls_dev_add, + .tls_dev_del = cxgb4_ktls_dev_del, +}; +#endif /* CONFIG_CHELSIO_TLS_DEVICE */ + +#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) + +static int cxgb4_xfrm_add_state(struct xfrm_state *x) +{ + struct adapter *adap = netdev2adap(x->xso.dev); + int ret; + + if (!mutex_trylock(&uld_mutex)) { + dev_dbg(adap->pdev_dev, + "crypto uld critical resource is under use\n"); + return -EBUSY; + } + ret = chcr_offload_state(adap, CXGB4_XFRMDEV_OPS); + if (ret) + goto out_unlock; + + ret = adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_add(x); + +out_unlock: + mutex_unlock(&uld_mutex); + + return ret; +} + +static void cxgb4_xfrm_del_state(struct xfrm_state *x) +{ + struct adapter *adap = netdev2adap(x->xso.dev); + + if (!mutex_trylock(&uld_mutex)) { + dev_dbg(adap->pdev_dev, + "crypto uld critical resource is under use\n"); + return; + } + if (chcr_offload_state(adap, CXGB4_XFRMDEV_OPS)) + goto out_unlock; + + adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_delete(x); + +out_unlock: + mutex_unlock(&uld_mutex); +} + +static void cxgb4_xfrm_free_state(struct xfrm_state *x) +{ + struct adapter *adap = netdev2adap(x->xso.dev); + + if (!mutex_trylock(&uld_mutex)) { + dev_dbg(adap->pdev_dev, + "crypto uld critical resource is under use\n"); + return; + } + if (chcr_offload_state(adap, CXGB4_XFRMDEV_OPS)) + goto out_unlock; + + adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_free(x); + +out_unlock: + mutex_unlock(&uld_mutex); +} + +static bool cxgb4_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) +{ + struct adapter *adap = netdev2adap(x->xso.dev); + bool ret = false; + + if (!mutex_trylock(&uld_mutex)) { + dev_dbg(adap->pdev_dev, + "crypto uld critical resource is under use\n"); + return ret; + } + if (chcr_offload_state(adap, CXGB4_XFRMDEV_OPS)) + goto out_unlock; + + ret = adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_offload_ok(skb, x); + +out_unlock: + mutex_unlock(&uld_mutex); + return ret; +} + +static void cxgb4_advance_esn_state(struct xfrm_state *x) +{ + struct adapter *adap = netdev2adap(x->xso.dev); + + if (!mutex_trylock(&uld_mutex)) { + dev_dbg(adap->pdev_dev, + "crypto uld critical resource is under use\n"); + return; + } + if (chcr_offload_state(adap, CXGB4_XFRMDEV_OPS)) + goto out_unlock; + + adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_advance_esn(x); + +out_unlock: + mutex_unlock(&uld_mutex); +} + +static const struct xfrmdev_ops cxgb4_xfrmdev_ops = { + .xdo_dev_state_add = cxgb4_xfrm_add_state, + .xdo_dev_state_delete = cxgb4_xfrm_del_state, + .xdo_dev_state_free = cxgb4_xfrm_free_state, + .xdo_dev_offload_ok = cxgb4_ipsec_offload_ok, + .xdo_dev_state_advance_esn = cxgb4_advance_esn_state, +}; + +#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ + +static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct net_device *netdev; + struct adapter *adapter; + static int adap_idx = 1; + int s_qpp, qpp, num_seg; + struct port_info *pi; + enum chip_type chip; + void __iomem *regs; + int func, chip_ver; + u16 device_id; + int i, err; + u32 whoami; + + err = pci_request_regions(pdev, KBUILD_MODNAME); + if (err) { + /* Just info, some other driver may have claimed the device. */ + dev_info(&pdev->dev, "cannot obtain PCI resources\n"); + return err; + } + + err = pci_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, "cannot enable PCI device\n"); + goto out_release_regions; + } + + regs = pci_ioremap_bar(pdev, 0); + if (!regs) { + dev_err(&pdev->dev, "cannot map device registers\n"); + err = -ENOMEM; + goto out_disable_device; + } + + adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); + if (!adapter) { + err = -ENOMEM; + goto out_unmap_bar0; + } + + adapter->regs = regs; + err = t4_wait_dev_ready(regs); + if (err < 0) + goto out_free_adapter; + + /* We control everything through one PF */ + whoami = t4_read_reg(adapter, PL_WHOAMI_A); + pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id); + chip = t4_get_chip_type(adapter, CHELSIO_PCI_ID_VER(device_id)); + if ((int)chip < 0) { + dev_err(&pdev->dev, "Device %d is not supported\n", device_id); + err = chip; + goto out_free_adapter; + } + chip_ver = CHELSIO_CHIP_VERSION(chip); + func = chip_ver <= CHELSIO_T5 ? + SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami); + + adapter->pdev = pdev; + adapter->pdev_dev = &pdev->dev; + adapter->name = pci_name(pdev); + adapter->mbox = func; + adapter->pf = func; + adapter->params.chip = chip; + adapter->adap_idx = adap_idx; + adapter->msg_enable = DFLT_MSG_ENABLE; + adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) + + (sizeof(struct mbox_cmd) * + T4_OS_LOG_MBOX_CMDS), + GFP_KERNEL); + if (!adapter->mbox_log) { + err = -ENOMEM; + goto out_free_adapter; + } + spin_lock_init(&adapter->mbox_lock); + INIT_LIST_HEAD(&adapter->mlist.list); + adapter->mbox_log->size = T4_OS_LOG_MBOX_CMDS; + pci_set_drvdata(pdev, adapter); + + if (func != ent->driver_data) { + pci_disable_device(pdev); + pci_save_state(pdev); /* to restore SR-IOV later */ + return 0; + } + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_err(&pdev->dev, "no usable DMA configuration\n"); + goto out_free_adapter; + } + + pci_enable_pcie_error_reporting(pdev); + pci_set_master(pdev); + pci_save_state(pdev); + adap_idx++; + adapter->workq = create_singlethread_workqueue("cxgb4"); + if (!adapter->workq) { + err = -ENOMEM; + goto out_free_adapter; + } + + /* PCI device has been enabled */ + adapter->flags |= CXGB4_DEV_ENABLED; + memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map)); + + /* If possible, we use PCIe Relaxed Ordering Attribute to deliver + * Ingress Packet Data to Free List Buffers in order to allow for + * chipset performance optimizations between the Root Complex and + * Memory Controllers. (Messages to the associated Ingress Queue + * notifying new Packet Placement in the Free Lists Buffers will be + * send without the Relaxed Ordering Attribute thus guaranteeing that + * all preceding PCIe Transaction Layer Packets will be processed + * first.) But some Root Complexes have various issues with Upstream + * Transaction Layer Packets with the Relaxed Ordering Attribute set. + * The PCIe devices which under the Root Complexes will be cleared the + * Relaxed Ordering bit in the configuration space, So we check our + * PCIe configuration space to see if it's flagged with advice against + * using Relaxed Ordering. + */ + if (!pcie_relaxed_ordering_enabled(pdev)) + adapter->flags |= CXGB4_ROOT_NO_RELAXED_ORDERING; + + spin_lock_init(&adapter->stats_lock); + spin_lock_init(&adapter->tid_release_lock); + spin_lock_init(&adapter->win0_lock); + + INIT_WORK(&adapter->tid_release_task, process_tid_release_list); + INIT_WORK(&adapter->db_full_task, process_db_full); + INIT_WORK(&adapter->db_drop_task, process_db_drop); + INIT_WORK(&adapter->fatal_err_notify_task, notify_fatal_err); + + err = t4_prep_adapter(adapter); + if (err) + goto out_free_adapter; + + if (is_kdump_kernel()) { + /* Collect hardware state and append to /proc/vmcore */ + err = cxgb4_cudbg_vmcore_add_dump(adapter); + if (err) { + dev_warn(adapter->pdev_dev, + "Fail collecting vmcore device dump, err: %d. Continuing\n", + err); + err = 0; + } + } + + if (!is_t4(adapter->params.chip)) { + s_qpp = (QUEUESPERPAGEPF0_S + + (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) * + adapter->pf); + qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter, + SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp); + num_seg = PAGE_SIZE / SEGMENT_SIZE; + + /* Each segment size is 128B. Write coalescing is enabled only + * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the + * queue is less no of segments that can be accommodated in + * a page size. + */ + if (qpp > num_seg) { + dev_err(&pdev->dev, + "Incorrect number of egress queues per page\n"); + err = -EINVAL; + goto out_free_adapter; + } + adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2), + pci_resource_len(pdev, 2)); + if (!adapter->bar2) { + dev_err(&pdev->dev, "cannot map device bar2 region\n"); + err = -ENOMEM; + goto out_free_adapter; + } + } + + setup_memwin(adapter); + err = adap_init0(adapter, 0); + if (err) + goto out_unmap_bar; + + setup_memwin_rdma(adapter); + + /* configure SGE_STAT_CFG_A to read WC stats */ + if (!is_t4(adapter->params.chip)) + t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7) | + (is_t5(adapter->params.chip) ? STATMODE_V(0) : + T6_STATMODE_V(0))); + + /* Initialize hash mac addr list */ + INIT_LIST_HEAD(&adapter->mac_hlist); + + for_each_port(adapter, i) { + /* For supporting MQPRIO Offload, need some extra + * queues for each ETHOFLD TIDs. Keep it equal to + * MAX_ATIDs for now. Once we connect to firmware + * later and query the EOTID params, we'll come to + * know the actual # of EOTIDs supported. + */ + netdev = alloc_etherdev_mq(sizeof(struct port_info), + MAX_ETH_QSETS + MAX_ATIDS); + if (!netdev) { + err = -ENOMEM; + goto out_free_dev; + } + + SET_NETDEV_DEV(netdev, &pdev->dev); + + adapter->port[i] = netdev; + pi = netdev_priv(netdev); + pi->adapter = adapter; + pi->xact_addr_filt = -1; + pi->port_id = i; + netdev->irq = pdev->irq; + + netdev->hw_features = NETIF_F_SG | TSO_FLAGS | + NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | + NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_GRO | + NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_TC | NETIF_F_NTUPLE | NETIF_F_HIGHDMA; + + if (chip_ver > CHELSIO_T5) { + netdev->hw_enc_features |= NETIF_F_IP_CSUM | + NETIF_F_IPV6_CSUM | + NETIF_F_RXCSUM | + NETIF_F_GSO_UDP_TUNNEL | + NETIF_F_GSO_UDP_TUNNEL_CSUM | + NETIF_F_TSO | NETIF_F_TSO6; + + netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | + NETIF_F_GSO_UDP_TUNNEL_CSUM | + NETIF_F_HW_TLS_RECORD; + + if (adapter->rawf_cnt) + netdev->udp_tunnel_nic_info = &cxgb_udp_tunnels; + } + + netdev->features |= netdev->hw_features; + netdev->vlan_features = netdev->features & VLAN_FEAT; +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + if (pi->adapter->params.crypto & FW_CAPS_CONFIG_TLS_HW) { + netdev->hw_features |= NETIF_F_HW_TLS_TX; + netdev->tlsdev_ops = &cxgb4_ktls_ops; + /* initialize the refcount */ + refcount_set(&pi->adapter->chcr_ktls.ktls_refcount, 0); + } +#endif /* CONFIG_CHELSIO_TLS_DEVICE */ +#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) + if (pi->adapter->params.crypto & FW_CAPS_CONFIG_IPSEC_INLINE) { + netdev->hw_enc_features |= NETIF_F_HW_ESP; + netdev->features |= NETIF_F_HW_ESP; + netdev->xfrmdev_ops = &cxgb4_xfrmdev_ops; + } +#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ + + netdev->priv_flags |= IFF_UNICAST_FLT; + + /* MTU range: 81 - 9600 */ + netdev->min_mtu = 81; /* accommodate SACK */ + netdev->max_mtu = MAX_MTU; + + netdev->netdev_ops = &cxgb4_netdev_ops; +#ifdef CONFIG_CHELSIO_T4_DCB + netdev->dcbnl_ops = &cxgb4_dcb_ops; + cxgb4_dcb_state_init(netdev); + cxgb4_dcb_version_init(netdev); +#endif + cxgb4_set_ethtool_ops(netdev); + } + + cxgb4_init_ethtool_dump(adapter); + + pci_set_drvdata(pdev, adapter); + + if (adapter->flags & CXGB4_FW_OK) { + err = t4_port_init(adapter, func, func, 0); + if (err) + goto out_free_dev; + } else if (adapter->params.nports == 1) { + /* If we don't have a connection to the firmware -- possibly + * because of an error -- grab the raw VPD parameters so we + * can set the proper MAC Address on the debug network + * interface that we've created. + */ + u8 hw_addr[ETH_ALEN]; + u8 *na = adapter->params.vpd.na; + + err = t4_get_raw_vpd_params(adapter, &adapter->params.vpd); + if (!err) { + for (i = 0; i < ETH_ALEN; i++) + hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 + + hex2val(na[2 * i + 1])); + t4_set_hw_addr(adapter, 0, hw_addr); + } + } + + if (!(adapter->flags & CXGB4_FW_OK)) + goto fw_attach_fail; + + /* Configure queues and allocate tables now, they can be needed as + * soon as the first register_netdev completes. + */ + err = cfg_queues(adapter); + if (err) + goto out_free_dev; + + adapter->smt = t4_init_smt(); + if (!adapter->smt) { + /* We tolerate a lack of SMT, giving up some functionality */ + dev_warn(&pdev->dev, "could not allocate SMT, continuing\n"); + } + + adapter->l2t = t4_init_l2t(adapter->l2t_start, adapter->l2t_end); + if (!adapter->l2t) { + /* We tolerate a lack of L2T, giving up some functionality */ + dev_warn(&pdev->dev, "could not allocate L2T, continuing\n"); + adapter->params.offload = 0; + } + +#if IS_ENABLED(CONFIG_IPV6) + if (chip_ver <= CHELSIO_T5 && + (!(t4_read_reg(adapter, LE_DB_CONFIG_A) & ASLIPCOMPEN_F))) { + /* CLIP functionality is not present in hardware, + * hence disable all offload features + */ + dev_warn(&pdev->dev, + "CLIP not enabled in hardware, continuing\n"); + adapter->params.offload = 0; + } else { + adapter->clipt = t4_init_clip_tbl(adapter->clipt_start, + adapter->clipt_end); + if (!adapter->clipt) { + /* We tolerate a lack of clip_table, giving up + * some functionality + */ + dev_warn(&pdev->dev, + "could not allocate Clip table, continuing\n"); + adapter->params.offload = 0; + } + } +#endif + + for_each_port(adapter, i) { + pi = adap2pinfo(adapter, i); + pi->sched_tbl = t4_init_sched(adapter->params.nsched_cls); + if (!pi->sched_tbl) + dev_warn(&pdev->dev, + "could not activate scheduling on port %d\n", + i); + } + + if (is_offload(adapter) || is_hashfilter(adapter)) { + if (t4_read_reg(adapter, LE_DB_CONFIG_A) & HASHEN_F) { + u32 v; + + v = t4_read_reg(adapter, LE_DB_HASH_CONFIG_A); + if (chip_ver <= CHELSIO_T5) { + adapter->tids.nhash = 1 << HASHTIDSIZE_G(v); + v = t4_read_reg(adapter, LE_DB_TID_HASHBASE_A); + adapter->tids.hash_base = v / 4; + } else { + adapter->tids.nhash = HASHTBLSIZE_G(v) << 3; + v = t4_read_reg(adapter, + T6_LE_DB_HASH_TID_BASE_A); + adapter->tids.hash_base = v; + } + } + } + + if (tid_init(&adapter->tids) < 0) { + dev_warn(&pdev->dev, "could not allocate TID table, " + "continuing\n"); + adapter->params.offload = 0; + } else { + adapter->tc_u32 = cxgb4_init_tc_u32(adapter); + if (!adapter->tc_u32) + dev_warn(&pdev->dev, + "could not offload tc u32, continuing\n"); + + if (cxgb4_init_tc_flower(adapter)) + dev_warn(&pdev->dev, + "could not offload tc flower, continuing\n"); + + if (cxgb4_init_tc_mqprio(adapter)) + dev_warn(&pdev->dev, + "could not offload tc mqprio, continuing\n"); + + if (cxgb4_init_tc_matchall(adapter)) + dev_warn(&pdev->dev, + "could not offload tc matchall, continuing\n"); + if (cxgb4_init_ethtool_filters(adapter)) + dev_warn(&pdev->dev, + "could not initialize ethtool filters, continuing\n"); + } + + /* See what interrupts we'll be using */ + if (msi > 1 && enable_msix(adapter) == 0) + adapter->flags |= CXGB4_USING_MSIX; + else if (msi > 0 && pci_enable_msi(pdev) == 0) { + adapter->flags |= CXGB4_USING_MSI; + if (msi > 1) + free_msix_info(adapter); + } + + /* check for PCI Express bandwidth capabiltites */ + pcie_print_link_status(pdev); + + cxgb4_init_mps_ref_entries(adapter); + + err = init_rss(adapter); + if (err) + goto out_free_dev; + + err = setup_non_data_intr(adapter); + if (err) { + dev_err(adapter->pdev_dev, + "Non Data interrupt allocation failed, err: %d\n", err); + goto out_free_dev; + } + + err = setup_fw_sge_queues(adapter); + if (err) { + dev_err(adapter->pdev_dev, + "FW sge queue allocation failed, err %d", err); + goto out_free_dev; + } + +fw_attach_fail: + /* + * The card is now ready to go. If any errors occur during device + * registration we do not fail the whole card but rather proceed only + * with the ports we manage to register successfully. However we must + * register at least one net device. + */ + for_each_port(adapter, i) { + pi = adap2pinfo(adapter, i); + adapter->port[i]->dev_port = pi->lport; + netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets); + netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets); + + netif_carrier_off(adapter->port[i]); + + err = register_netdev(adapter->port[i]); + if (err) + break; + adapter->chan_map[pi->tx_chan] = i; + print_port_info(adapter->port[i]); + } + if (i == 0) { + dev_err(&pdev->dev, "could not register any net devices\n"); + goto out_free_dev; + } + if (err) { + dev_warn(&pdev->dev, "only %d net devices registered\n", i); + err = 0; + } + + if (cxgb4_debugfs_root) { + adapter->debugfs_root = debugfs_create_dir(pci_name(pdev), + cxgb4_debugfs_root); + setup_debugfs(adapter); + } + + /* PCIe EEH recovery on powerpc platforms needs fundamental reset */ + pdev->needs_freset = 1; + + if (is_uld(adapter)) + cxgb4_uld_enable(adapter); + + if (!is_t4(adapter->params.chip)) + cxgb4_ptp_init(adapter); + + if (IS_REACHABLE(CONFIG_THERMAL) && + !is_t4(adapter->params.chip) && (adapter->flags & CXGB4_FW_OK)) + cxgb4_thermal_init(adapter); + + print_adapter_info(adapter); + return 0; + + out_free_dev: + t4_free_sge_resources(adapter); + free_some_resources(adapter); + if (adapter->flags & CXGB4_USING_MSIX) + free_msix_info(adapter); + if (adapter->num_uld || adapter->num_ofld_uld) + t4_uld_mem_free(adapter); + out_unmap_bar: + if (!is_t4(adapter->params.chip)) + iounmap(adapter->bar2); + out_free_adapter: + if (adapter->workq) + destroy_workqueue(adapter->workq); + + kfree(adapter->mbox_log); + kfree(adapter); + out_unmap_bar0: + iounmap(regs); + out_disable_device: + pci_disable_pcie_error_reporting(pdev); + pci_disable_device(pdev); + out_release_regions: + pci_release_regions(pdev); + return err; +} + +static void remove_one(struct pci_dev *pdev) +{ + struct adapter *adapter = pci_get_drvdata(pdev); + struct hash_mac_addr *entry, *tmp; + + if (!adapter) { + pci_release_regions(pdev); + return; + } + + /* If we allocated filters, free up state associated with any + * valid filters ... + */ + clear_all_filters(adapter); + + adapter->flags |= CXGB4_SHUTTING_DOWN; + + if (adapter->pf == 4) { + int i; + + /* Tear down per-adapter Work Queue first since it can contain + * references to our adapter data structure. + */ + destroy_workqueue(adapter->workq); + + detach_ulds(adapter); + + for_each_port(adapter, i) + if (adapter->port[i]->reg_state == NETREG_REGISTERED) + unregister_netdev(adapter->port[i]); + + t4_uld_clean_up(adapter); + + adap_free_hma_mem(adapter); + + disable_interrupts(adapter); + + cxgb4_free_mps_ref_entries(adapter); + + debugfs_remove_recursive(adapter->debugfs_root); + + if (!is_t4(adapter->params.chip)) + cxgb4_ptp_stop(adapter); + if (IS_REACHABLE(CONFIG_THERMAL)) + cxgb4_thermal_remove(adapter); + + if (adapter->flags & CXGB4_FULL_INIT_DONE) + cxgb_down(adapter); + + if (adapter->flags & CXGB4_USING_MSIX) + free_msix_info(adapter); + if (adapter->num_uld || adapter->num_ofld_uld) + t4_uld_mem_free(adapter); + free_some_resources(adapter); + list_for_each_entry_safe(entry, tmp, &adapter->mac_hlist, + list) { + list_del(&entry->list); + kfree(entry); + } + +#if IS_ENABLED(CONFIG_IPV6) + t4_cleanup_clip_tbl(adapter); +#endif + if (!is_t4(adapter->params.chip)) + iounmap(adapter->bar2); + } +#ifdef CONFIG_PCI_IOV + else { + cxgb4_iov_configure(adapter->pdev, 0); + } +#endif + iounmap(adapter->regs); + pci_disable_pcie_error_reporting(pdev); + if ((adapter->flags & CXGB4_DEV_ENABLED)) { + pci_disable_device(pdev); + adapter->flags &= ~CXGB4_DEV_ENABLED; + } + pci_release_regions(pdev); + kfree(adapter->mbox_log); + synchronize_rcu(); + kfree(adapter); +} + +/* "Shutdown" quiesces the device, stopping Ingress Packet and Interrupt + * delivery. This is essentially a stripped down version of the PCI remove() + * function where we do the minimal amount of work necessary to shutdown any + * further activity. + */ +static void shutdown_one(struct pci_dev *pdev) +{ + struct adapter *adapter = pci_get_drvdata(pdev); + + /* As with remove_one() above (see extended comment), we only want do + * do cleanup on PCI Devices which went all the way through init_one() + * ... + */ + if (!adapter) { + pci_release_regions(pdev); + return; + } + + adapter->flags |= CXGB4_SHUTTING_DOWN; + + if (adapter->pf == 4) { + int i; + + for_each_port(adapter, i) + if (adapter->port[i]->reg_state == NETREG_REGISTERED) + cxgb_close(adapter->port[i]); + + rtnl_lock(); + cxgb4_mqprio_stop_offload(adapter); + rtnl_unlock(); + + if (is_uld(adapter)) { + detach_ulds(adapter); + t4_uld_clean_up(adapter); + } + + disable_interrupts(adapter); + disable_msi(adapter); + + t4_sge_stop(adapter); + if (adapter->flags & CXGB4_FW_OK) + t4_fw_bye(adapter, adapter->mbox); + } +} + +static struct pci_driver cxgb4_driver = { + .name = KBUILD_MODNAME, + .id_table = cxgb4_pci_tbl, + .probe = init_one, + .remove = remove_one, + .shutdown = shutdown_one, +#ifdef CONFIG_PCI_IOV + .sriov_configure = cxgb4_iov_configure, +#endif + .err_handler = &cxgb4_eeh, +}; + +static int __init cxgb4_init_module(void) +{ + int ret; + + cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); + + ret = pci_register_driver(&cxgb4_driver); + if (ret < 0) + goto err_pci; + +#if IS_ENABLED(CONFIG_IPV6) + if (!inet6addr_registered) { + ret = register_inet6addr_notifier(&cxgb4_inet6addr_notifier); + if (ret) + pci_unregister_driver(&cxgb4_driver); + else + inet6addr_registered = true; + } +#endif + + if (ret == 0) + return ret; + +err_pci: + debugfs_remove(cxgb4_debugfs_root); + + return ret; +} + +static void __exit cxgb4_cleanup_module(void) +{ +#if IS_ENABLED(CONFIG_IPV6) + if (inet6addr_registered) { + unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier); + inet6addr_registered = false; + } +#endif + pci_unregister_driver(&cxgb4_driver); + debugfs_remove(cxgb4_debugfs_root); /* NULL ok */ +} + +module_init(cxgb4_init_module); +module_exit(cxgb4_cleanup_module); diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_mps.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_mps.c new file mode 100644 index 000000000..a020e8490 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_mps.c @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 Chelsio Communications, Inc. All rights reserved. */ + +#include "cxgb4.h" + +static int cxgb4_mps_ref_dec_by_mac(struct adapter *adap, + const u8 *addr, const u8 *mask) +{ + u8 bitmask[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; + struct mps_entries_ref *mps_entry, *tmp; + int ret = -EINVAL; + + spin_lock_bh(&adap->mps_ref_lock); + list_for_each_entry_safe(mps_entry, tmp, &adap->mps_ref, list) { + if (ether_addr_equal(mps_entry->addr, addr) && + ether_addr_equal(mps_entry->mask, mask ? mask : bitmask)) { + if (!refcount_dec_and_test(&mps_entry->refcnt)) { + spin_unlock_bh(&adap->mps_ref_lock); + return -EBUSY; + } + list_del(&mps_entry->list); + kfree(mps_entry); + ret = 0; + break; + } + } + spin_unlock_bh(&adap->mps_ref_lock); + return ret; +} + +static int cxgb4_mps_ref_dec(struct adapter *adap, u16 idx) +{ + struct mps_entries_ref *mps_entry, *tmp; + int ret = -EINVAL; + + spin_lock(&adap->mps_ref_lock); + list_for_each_entry_safe(mps_entry, tmp, &adap->mps_ref, list) { + if (mps_entry->idx == idx) { + if (!refcount_dec_and_test(&mps_entry->refcnt)) { + spin_unlock(&adap->mps_ref_lock); + return -EBUSY; + } + list_del(&mps_entry->list); + kfree(mps_entry); + ret = 0; + break; + } + } + spin_unlock(&adap->mps_ref_lock); + return ret; +} + +static int cxgb4_mps_ref_inc(struct adapter *adap, const u8 *mac_addr, + u16 idx, const u8 *mask) +{ + u8 bitmask[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; + struct mps_entries_ref *mps_entry; + int ret = 0; + + spin_lock_bh(&adap->mps_ref_lock); + list_for_each_entry(mps_entry, &adap->mps_ref, list) { + if (mps_entry->idx == idx) { + refcount_inc(&mps_entry->refcnt); + goto unlock; + } + } + mps_entry = kzalloc(sizeof(*mps_entry), GFP_ATOMIC); + if (!mps_entry) { + ret = -ENOMEM; + goto unlock; + } + ether_addr_copy(mps_entry->mask, mask ? mask : bitmask); + ether_addr_copy(mps_entry->addr, mac_addr); + mps_entry->idx = idx; + refcount_set(&mps_entry->refcnt, 1); + list_add_tail(&mps_entry->list, &adap->mps_ref); +unlock: + spin_unlock_bh(&adap->mps_ref_lock); + return ret; +} + +int cxgb4_free_mac_filt(struct adapter *adap, unsigned int viid, + unsigned int naddr, const u8 **addr, bool sleep_ok) +{ + int ret, i; + + for (i = 0; i < naddr; i++) { + if (!cxgb4_mps_ref_dec_by_mac(adap, addr[i], NULL)) { + ret = t4_free_mac_filt(adap, adap->mbox, viid, + 1, &addr[i], sleep_ok); + if (ret < 0) + return ret; + } + } + + /* return number of filters freed */ + return naddr; +} + +int cxgb4_alloc_mac_filt(struct adapter *adap, unsigned int viid, + bool free, unsigned int naddr, const u8 **addr, + u16 *idx, u64 *hash, bool sleep_ok) +{ + int ret, i; + + ret = t4_alloc_mac_filt(adap, adap->mbox, viid, free, + naddr, addr, idx, hash, sleep_ok); + if (ret < 0) + return ret; + + for (i = 0; i < naddr; i++) { + if (idx[i] != 0xffff) { + if (cxgb4_mps_ref_inc(adap, addr[i], idx[i], NULL)) { + ret = -ENOMEM; + goto error; + } + } + } + + goto out; +error: + cxgb4_free_mac_filt(adap, viid, naddr, addr, sleep_ok); + +out: + /* Returns a negative error number or the number of filters allocated */ + return ret; +} + +int cxgb4_update_mac_filt(struct port_info *pi, unsigned int viid, + int *tcam_idx, const u8 *addr, + bool persistent, u8 *smt_idx) +{ + int ret; + + ret = cxgb4_change_mac(pi, viid, tcam_idx, + addr, persistent, smt_idx); + if (ret < 0) + return ret; + + cxgb4_mps_ref_inc(pi->adapter, addr, *tcam_idx, NULL); + return ret; +} + +int cxgb4_free_raw_mac_filt(struct adapter *adap, + unsigned int viid, + const u8 *addr, + const u8 *mask, + unsigned int idx, + u8 lookup_type, + u8 port_id, + bool sleep_ok) +{ + int ret = 0; + + if (!cxgb4_mps_ref_dec(adap, idx)) + ret = t4_free_raw_mac_filt(adap, viid, addr, + mask, idx, lookup_type, + port_id, sleep_ok); + + return ret; +} + +int cxgb4_alloc_raw_mac_filt(struct adapter *adap, + unsigned int viid, + const u8 *addr, + const u8 *mask, + unsigned int idx, + u8 lookup_type, + u8 port_id, + bool sleep_ok) +{ + int ret; + + ret = t4_alloc_raw_mac_filt(adap, viid, addr, + mask, idx, lookup_type, + port_id, sleep_ok); + if (ret < 0) + return ret; + + if (cxgb4_mps_ref_inc(adap, addr, ret, mask)) { + ret = -ENOMEM; + t4_free_raw_mac_filt(adap, viid, addr, + mask, idx, lookup_type, + port_id, sleep_ok); + } + + return ret; +} + +int cxgb4_free_encap_mac_filt(struct adapter *adap, unsigned int viid, + int idx, bool sleep_ok) +{ + int ret = 0; + + if (!cxgb4_mps_ref_dec(adap, idx)) + ret = t4_free_encap_mac_filt(adap, viid, idx, sleep_ok); + + return ret; +} + +int cxgb4_alloc_encap_mac_filt(struct adapter *adap, unsigned int viid, + const u8 *addr, const u8 *mask, + unsigned int vni, unsigned int vni_mask, + u8 dip_hit, u8 lookup_type, bool sleep_ok) +{ + int ret; + + ret = t4_alloc_encap_mac_filt(adap, viid, addr, mask, vni, vni_mask, + dip_hit, lookup_type, sleep_ok); + if (ret < 0) + return ret; + + if (cxgb4_mps_ref_inc(adap, addr, ret, mask)) { + ret = -ENOMEM; + t4_free_encap_mac_filt(adap, viid, ret, sleep_ok); + } + return ret; +} + +int cxgb4_init_mps_ref_entries(struct adapter *adap) +{ + spin_lock_init(&adap->mps_ref_lock); + INIT_LIST_HEAD(&adap->mps_ref); + + return 0; +} + +void cxgb4_free_mps_ref_entries(struct adapter *adap) +{ + struct mps_entries_ref *mps_entry, *tmp; + + if (list_empty(&adap->mps_ref)) + return; + + spin_lock(&adap->mps_ref_lock); + list_for_each_entry_safe(mps_entry, tmp, &adap->mps_ref, list) { + list_del(&mps_entry->list); + kfree(mps_entry); + } + spin_unlock(&adap->mps_ref_lock); +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ptp.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ptp.c new file mode 100644 index 000000000..5bf117d21 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ptp.c @@ -0,0 +1,465 @@ +/* + * cxgb4_ptp.c:Chelsio PTP support for T5/T6 + * + * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Written by: Atul Gupta (atul.gupta@chelsio.com) + */ + +#include <linux/module.h> +#include <linux/net_tstamp.h> +#include <linux/skbuff.h> +#include <linux/netdevice.h> +#include <linux/pps_kernel.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/ptp_classify.h> +#include <linux/udp.h> + +#include "cxgb4.h" +#include "t4_hw.h" +#include "t4_regs.h" +#include "t4_msg.h" +#include "t4fw_api.h" +#include "cxgb4_ptp.h" + +/** + * cxgb4_ptp_is_ptp_tx - determine whether TX packet is PTP or not + * @skb: skb of outgoing ptp request + * + */ +bool cxgb4_ptp_is_ptp_tx(struct sk_buff *skb) +{ + struct udphdr *uh; + + uh = udp_hdr(skb); + return skb->len >= PTP_MIN_LENGTH && + skb->len <= PTP_IN_TRANSMIT_PACKET_MAXNUM && + likely(skb->protocol == htons(ETH_P_IP)) && + ip_hdr(skb)->protocol == IPPROTO_UDP && + uh->dest == htons(PTP_EVENT_PORT); +} + +bool is_ptp_enabled(struct sk_buff *skb, struct net_device *dev) +{ + struct port_info *pi; + + pi = netdev_priv(dev); + return (pi->ptp_enable && cxgb4_xmit_with_hwtstamp(skb) && + cxgb4_ptp_is_ptp_tx(skb)); +} + +/** + * cxgb4_ptp_is_ptp_rx - determine whether RX packet is PTP or not + * @skb: skb of incoming ptp request + * + */ +bool cxgb4_ptp_is_ptp_rx(struct sk_buff *skb) +{ + struct udphdr *uh = (struct udphdr *)(skb->data + ETH_HLEN + + IPV4_HLEN(skb->data)); + + return uh->dest == htons(PTP_EVENT_PORT) && + uh->source == htons(PTP_EVENT_PORT); +} + +/** + * cxgb4_ptp_read_hwstamp - read timestamp for TX event PTP message + * @adapter: board private structure + * @pi: port private structure + * + */ +void cxgb4_ptp_read_hwstamp(struct adapter *adapter, struct port_info *pi) +{ + struct skb_shared_hwtstamps *skb_ts = NULL; + u64 tx_ts; + + skb_ts = skb_hwtstamps(adapter->ptp_tx_skb); + + tx_ts = t4_read_reg(adapter, + T5_PORT_REG(pi->port_id, MAC_PORT_TX_TS_VAL_LO)); + + tx_ts |= (u64)t4_read_reg(adapter, + T5_PORT_REG(pi->port_id, + MAC_PORT_TX_TS_VAL_HI)) << 32; + skb_ts->hwtstamp = ns_to_ktime(tx_ts); + skb_tstamp_tx(adapter->ptp_tx_skb, skb_ts); + dev_kfree_skb_any(adapter->ptp_tx_skb); + spin_lock(&adapter->ptp_lock); + adapter->ptp_tx_skb = NULL; + spin_unlock(&adapter->ptp_lock); +} + +/** + * cxgb4_ptprx_timestamping - Enable Timestamp for RX PTP event message + * @pi: port private structure + * @port: pot number + * @mode: RX mode + * + */ +int cxgb4_ptprx_timestamping(struct port_info *pi, u8 port, u16 mode) +{ + struct adapter *adapter = pi->adapter; + struct fw_ptp_cmd c; + int err; + + memset(&c, 0, sizeof(c)); + c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PTP_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_PTP_CMD_PORTID_V(port)); + c.retval_len16 = cpu_to_be32(FW_CMD_LEN16_V(sizeof(c) / 16)); + c.u.init.sc = FW_PTP_SC_RXTIME_STAMP; + c.u.init.mode = cpu_to_be16(mode); + + err = t4_wr_mbox(adapter, adapter->mbox, &c, sizeof(c), NULL); + if (err < 0) + dev_err(adapter->pdev_dev, + "PTP: %s error %d\n", __func__, -err); + return err; +} + +int cxgb4_ptp_txtype(struct adapter *adapter, u8 port) +{ + struct fw_ptp_cmd c; + int err; + + memset(&c, 0, sizeof(c)); + c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PTP_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_PTP_CMD_PORTID_V(port)); + c.retval_len16 = cpu_to_be32(FW_CMD_LEN16_V(sizeof(c) / 16)); + c.u.init.sc = FW_PTP_SC_TX_TYPE; + c.u.init.mode = cpu_to_be16(PTP_TS_NONE); + + err = t4_wr_mbox(adapter, adapter->mbox, &c, sizeof(c), NULL); + if (err < 0) + dev_err(adapter->pdev_dev, + "PTP: %s error %d\n", __func__, -err); + + return err; +} + +int cxgb4_ptp_redirect_rx_packet(struct adapter *adapter, struct port_info *pi) +{ + struct sge *s = &adapter->sge; + struct sge_eth_rxq *receive_q = &s->ethrxq[pi->first_qset]; + struct fw_ptp_cmd c; + int err; + + memset(&c, 0, sizeof(c)); + c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PTP_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_PTP_CMD_PORTID_V(pi->port_id)); + + c.retval_len16 = cpu_to_be32(FW_CMD_LEN16_V(sizeof(c) / 16)); + c.u.init.sc = FW_PTP_SC_RDRX_TYPE; + c.u.init.txchan = pi->tx_chan; + c.u.init.absid = cpu_to_be16(receive_q->rspq.abs_id); + + err = t4_wr_mbox(adapter, adapter->mbox, &c, sizeof(c), NULL); + if (err < 0) + dev_err(adapter->pdev_dev, + "PTP: %s error %d\n", __func__, -err); + return err; +} + +/** + * cxgb4_ptp_adjfreq - Adjust frequency of PHC cycle counter + * @ptp: ptp clock structure + * @ppb: Desired frequency change in parts per billion + * + * Adjust the frequency of the PHC cycle counter by the indicated ppb from + * the base frequency. + */ +static int cxgb4_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb) +{ + struct adapter *adapter = (struct adapter *)container_of(ptp, + struct adapter, ptp_clock_info); + struct fw_ptp_cmd c; + int err; + + memset(&c, 0, sizeof(c)); + c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PTP_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_PTP_CMD_PORTID_V(0)); + c.retval_len16 = cpu_to_be32(FW_CMD_LEN16_V(sizeof(c) / 16)); + c.u.ts.sc = FW_PTP_SC_ADJ_FREQ; + c.u.ts.sign = (ppb < 0) ? 1 : 0; + if (ppb < 0) + ppb = -ppb; + c.u.ts.ppb = cpu_to_be32(ppb); + + err = t4_wr_mbox(adapter, adapter->mbox, &c, sizeof(c), NULL); + if (err < 0) + dev_err(adapter->pdev_dev, + "PTP: %s error %d\n", __func__, -err); + + return err; +} + +/** + * cxgb4_ptp_fineadjtime - Shift the time of the hardware clock + * @adapter: board private structure + * @delta: Desired change in nanoseconds + * + * Adjust the timer by resetting the timecounter structure. + */ +static int cxgb4_ptp_fineadjtime(struct adapter *adapter, s64 delta) +{ + struct fw_ptp_cmd c; + int err; + + memset(&c, 0, sizeof(c)); + c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PTP_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_PTP_CMD_PORTID_V(0)); + c.retval_len16 = cpu_to_be32(FW_CMD_LEN16_V(sizeof(c) / 16)); + c.u.ts.sc = FW_PTP_SC_ADJ_FTIME; + c.u.ts.sign = (delta < 0) ? 1 : 0; + if (delta < 0) + delta = -delta; + c.u.ts.tm = cpu_to_be64(delta); + + err = t4_wr_mbox(adapter, adapter->mbox, &c, sizeof(c), NULL); + if (err < 0) + dev_err(adapter->pdev_dev, + "PTP: %s error %d\n", __func__, -err); + return err; +} + +/** + * cxgb4_ptp_adjtime - Shift the time of the hardware clock + * @ptp: ptp clock structure + * @delta: Desired change in nanoseconds + * + * Adjust the timer by resetting the timecounter structure. + */ +static int cxgb4_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + struct adapter *adapter = + (struct adapter *)container_of(ptp, struct adapter, + ptp_clock_info); + struct fw_ptp_cmd c; + s64 sign = 1; + int err; + + if (delta < 0) + sign = -1; + + if (delta * sign > PTP_CLOCK_MAX_ADJTIME) { + memset(&c, 0, sizeof(c)); + c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PTP_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_PTP_CMD_PORTID_V(0)); + c.retval_len16 = cpu_to_be32(FW_CMD_LEN16_V(sizeof(c) / 16)); + c.u.ts.sc = FW_PTP_SC_ADJ_TIME; + c.u.ts.sign = (delta < 0) ? 1 : 0; + if (delta < 0) + delta = -delta; + c.u.ts.tm = cpu_to_be64(delta); + + err = t4_wr_mbox(adapter, adapter->mbox, &c, sizeof(c), NULL); + if (err < 0) + dev_err(adapter->pdev_dev, + "PTP: %s error %d\n", __func__, -err); + } else { + err = cxgb4_ptp_fineadjtime(adapter, delta); + } + + return err; +} + +/** + * cxgb4_ptp_gettime - Reads the current time from the hardware clock + * @ptp: ptp clock structure + * @ts: timespec structure to hold the current time value + * + * Read the timecounter and return the correct value in ns after converting + * it into a struct timespec. + */ +static int cxgb4_ptp_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts) +{ + struct adapter *adapter = container_of(ptp, struct adapter, + ptp_clock_info); + u64 ns; + + ns = t4_read_reg(adapter, T5_PORT_REG(0, MAC_PORT_PTP_SUM_LO_A)); + ns |= (u64)t4_read_reg(adapter, + T5_PORT_REG(0, MAC_PORT_PTP_SUM_HI_A)) << 32; + + /* convert to timespec*/ + *ts = ns_to_timespec64(ns); + return 0; +} + +/** + * cxgb4_ptp_settime - Set the current time on the hardware clock + * @ptp: ptp clock structure + * @ts: timespec containing the new time for the cycle counter + * + * Reset value to new base value instead of the kernel + * wall timer value. + */ +static int cxgb4_ptp_settime(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + struct adapter *adapter = (struct adapter *)container_of(ptp, + struct adapter, ptp_clock_info); + struct fw_ptp_cmd c; + u64 ns; + int err; + + memset(&c, 0, sizeof(c)); + c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PTP_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_PTP_CMD_PORTID_V(0)); + c.retval_len16 = cpu_to_be32(FW_CMD_LEN16_V(sizeof(c) / 16)); + c.u.ts.sc = FW_PTP_SC_SET_TIME; + + ns = timespec64_to_ns(ts); + c.u.ts.tm = cpu_to_be64(ns); + + err = t4_wr_mbox(adapter, adapter->mbox, &c, sizeof(c), NULL); + if (err < 0) + dev_err(adapter->pdev_dev, + "PTP: %s error %d\n", __func__, -err); + + return err; +} + +static void cxgb4_init_ptp_timer(struct adapter *adapter) +{ + struct fw_ptp_cmd c; + int err; + + memset(&c, 0, sizeof(c)); + c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PTP_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_PTP_CMD_PORTID_V(0)); + c.retval_len16 = cpu_to_be32(FW_CMD_LEN16_V(sizeof(c) / 16)); + c.u.scmd.sc = FW_PTP_SC_INIT_TIMER; + + err = t4_wr_mbox(adapter, adapter->mbox, &c, sizeof(c), NULL); + if (err < 0) + dev_err(adapter->pdev_dev, + "PTP: %s error %d\n", __func__, -err); +} + +/** + * cxgb4_ptp_enable - enable or disable an ancillary feature + * @ptp: ptp clock structure + * @request: Desired resource to enable or disable + * @on: Caller passes one to enable or zero to disable + * + * Enable (or disable) ancillary features of the PHC subsystem. + * Currently, no ancillary features are supported. + */ +static int cxgb4_ptp_enable(struct ptp_clock_info __always_unused *ptp, + struct ptp_clock_request __always_unused *request, + int __always_unused on) +{ + return -ENOTSUPP; +} + +static const struct ptp_clock_info cxgb4_ptp_clock_info = { + .owner = THIS_MODULE, + .name = "cxgb4_clock", + .max_adj = MAX_PTP_FREQ_ADJ, + .n_alarm = 0, + .n_ext_ts = 0, + .n_per_out = 0, + .pps = 0, + .adjfreq = cxgb4_ptp_adjfreq, + .adjtime = cxgb4_ptp_adjtime, + .gettime64 = cxgb4_ptp_gettime, + .settime64 = cxgb4_ptp_settime, + .enable = cxgb4_ptp_enable, +}; + +/** + * cxgb4_ptp_init - initialize PTP for devices which support it + * @adapter: board private structure + * + * This function performs the required steps for enabling PTP support. + */ +void cxgb4_ptp_init(struct adapter *adapter) +{ + struct timespec64 now; + /* no need to create a clock device if we already have one */ + if (!IS_ERR_OR_NULL(adapter->ptp_clock)) + return; + + adapter->ptp_tx_skb = NULL; + adapter->ptp_clock_info = cxgb4_ptp_clock_info; + spin_lock_init(&adapter->ptp_lock); + + adapter->ptp_clock = ptp_clock_register(&adapter->ptp_clock_info, + &adapter->pdev->dev); + if (IS_ERR_OR_NULL(adapter->ptp_clock)) { + adapter->ptp_clock = NULL; + dev_err(adapter->pdev_dev, + "PTP %s Clock registration has failed\n", __func__); + return; + } + + now = ktime_to_timespec64(ktime_get_real()); + cxgb4_init_ptp_timer(adapter); + if (cxgb4_ptp_settime(&adapter->ptp_clock_info, &now) < 0) { + ptp_clock_unregister(adapter->ptp_clock); + adapter->ptp_clock = NULL; + } +} + +/** + * cxgb4_ptp_stop - disable PTP device and stop the overflow check + * @adapter: board private structure + * + * Stop the PTP support. + */ +void cxgb4_ptp_stop(struct adapter *adapter) +{ + if (adapter->ptp_tx_skb) { + dev_kfree_skb_any(adapter->ptp_tx_skb); + adapter->ptp_tx_skb = NULL; + } + + if (adapter->ptp_clock) { + ptp_clock_unregister(adapter->ptp_clock); + adapter->ptp_clock = NULL; + } +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ptp.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ptp.h new file mode 100644 index 000000000..cccfae84b --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ptp.h @@ -0,0 +1,74 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_PTP_H__ +#define __CXGB4_PTP_H__ + +/* Maximum parts-per-billion adjustment that is acceptable */ +#define MAX_PTP_FREQ_ADJ 1000000 +#define PTP_CLOCK_MAX_ADJTIME 10000000 /* 10 ms */ + +#define PTP_MIN_LENGTH 63 +#define PTP_IN_TRANSMIT_PACKET_MAXNUM 240 +#define PTP_EVENT_PORT 319 + +enum ptp_rx_filter_mode { + PTP_TS_NONE = 0, + PTP_TS_L2, + PTP_TS_L4, + PTP_TS_L2_L4 +}; + +struct port_info; + +static inline bool cxgb4_xmit_with_hwtstamp(struct sk_buff *skb) +{ + return skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP; +} + +static inline void cxgb4_xmit_hwtstamp_pending(struct sk_buff *skb) +{ + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; +} + +void cxgb4_ptp_init(struct adapter *adap); +void cxgb4_ptp_stop(struct adapter *adap); +bool cxgb4_ptp_is_ptp_tx(struct sk_buff *skb); +bool cxgb4_ptp_is_ptp_rx(struct sk_buff *skb); +int cxgb4_ptprx_timestamping(struct port_info *pi, u8 port, u16 mode); +int cxgb4_ptp_redirect_rx_packet(struct adapter *adap, struct port_info *pi); +int cxgb4_ptp_txtype(struct adapter *adap, u8 port_id); +void cxgb4_ptp_read_hwstamp(struct adapter *adap, struct port_info *pi); +bool is_ptp_enabled(struct sk_buff *skb, struct net_device *dev); +#endif /* __CXGB4_PTP_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c new file mode 100644 index 000000000..dd9be2298 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c @@ -0,0 +1,1142 @@ +/* + * This file is part of the Chelsio T4/T5/T6 Ethernet driver for Linux. + * + * Copyright (c) 2017 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <net/tc_act/tc_mirred.h> +#include <net/tc_act/tc_pedit.h> +#include <net/tc_act/tc_gact.h> +#include <net/tc_act/tc_vlan.h> + +#include "cxgb4.h" +#include "cxgb4_filter.h" +#include "cxgb4_tc_flower.h" + +#define STATS_CHECK_PERIOD (HZ / 2) + +static struct ch_tc_pedit_fields pedits[] = { + PEDIT_FIELDS(ETH_, DMAC_31_0, 4, dmac, 0), + PEDIT_FIELDS(ETH_, DMAC_47_32, 2, dmac, 4), + PEDIT_FIELDS(ETH_, SMAC_15_0, 2, smac, 0), + PEDIT_FIELDS(ETH_, SMAC_47_16, 4, smac, 2), + PEDIT_FIELDS(IP4_, SRC, 4, nat_fip, 0), + PEDIT_FIELDS(IP4_, DST, 4, nat_lip, 0), + PEDIT_FIELDS(IP6_, SRC_31_0, 4, nat_fip, 0), + PEDIT_FIELDS(IP6_, SRC_63_32, 4, nat_fip, 4), + PEDIT_FIELDS(IP6_, SRC_95_64, 4, nat_fip, 8), + PEDIT_FIELDS(IP6_, SRC_127_96, 4, nat_fip, 12), + PEDIT_FIELDS(IP6_, DST_31_0, 4, nat_lip, 0), + PEDIT_FIELDS(IP6_, DST_63_32, 4, nat_lip, 4), + PEDIT_FIELDS(IP6_, DST_95_64, 4, nat_lip, 8), + PEDIT_FIELDS(IP6_, DST_127_96, 4, nat_lip, 12), +}; + +static const struct cxgb4_natmode_config cxgb4_natmode_config_array[] = { + /* Default supported NAT modes */ + { + .chip = CHELSIO_T5, + .flags = CXGB4_ACTION_NATMODE_NONE, + .natmode = NAT_MODE_NONE, + }, + { + .chip = CHELSIO_T5, + .flags = CXGB4_ACTION_NATMODE_DIP, + .natmode = NAT_MODE_DIP, + }, + { + .chip = CHELSIO_T5, + .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_DPORT, + .natmode = NAT_MODE_DIP_DP, + }, + { + .chip = CHELSIO_T5, + .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_DPORT | + CXGB4_ACTION_NATMODE_SIP, + .natmode = NAT_MODE_DIP_DP_SIP, + }, + { + .chip = CHELSIO_T5, + .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_DPORT | + CXGB4_ACTION_NATMODE_SPORT, + .natmode = NAT_MODE_DIP_DP_SP, + }, + { + .chip = CHELSIO_T5, + .flags = CXGB4_ACTION_NATMODE_SIP | CXGB4_ACTION_NATMODE_SPORT, + .natmode = NAT_MODE_SIP_SP, + }, + { + .chip = CHELSIO_T5, + .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_SIP | + CXGB4_ACTION_NATMODE_SPORT, + .natmode = NAT_MODE_DIP_SIP_SP, + }, + { + .chip = CHELSIO_T5, + .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_SIP | + CXGB4_ACTION_NATMODE_DPORT | + CXGB4_ACTION_NATMODE_SPORT, + .natmode = NAT_MODE_ALL, + }, + /* T6+ can ignore L4 ports when they're disabled. */ + { + .chip = CHELSIO_T6, + .flags = CXGB4_ACTION_NATMODE_SIP, + .natmode = NAT_MODE_SIP_SP, + }, + { + .chip = CHELSIO_T6, + .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_SPORT, + .natmode = NAT_MODE_DIP_DP_SP, + }, + { + .chip = CHELSIO_T6, + .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_SIP, + .natmode = NAT_MODE_ALL, + }, +}; + +static void cxgb4_action_natmode_tweak(struct ch_filter_specification *fs, + u8 natmode_flags) +{ + u8 i = 0; + + /* Translate the enabled NAT 4-tuple fields to one of the + * hardware supported NAT mode configurations. This ensures + * that we pick a valid combination, where the disabled fields + * do not get overwritten to 0. + */ + for (i = 0; i < ARRAY_SIZE(cxgb4_natmode_config_array); i++) { + if (cxgb4_natmode_config_array[i].flags == natmode_flags) { + fs->nat_mode = cxgb4_natmode_config_array[i].natmode; + return; + } + } +} + +static struct ch_tc_flower_entry *allocate_flower_entry(void) +{ + struct ch_tc_flower_entry *new = kzalloc(sizeof(*new), GFP_KERNEL); + if (new) + spin_lock_init(&new->lock); + return new; +} + +/* Must be called with either RTNL or rcu_read_lock */ +static struct ch_tc_flower_entry *ch_flower_lookup(struct adapter *adap, + unsigned long flower_cookie) +{ + return rhashtable_lookup_fast(&adap->flower_tbl, &flower_cookie, + adap->flower_ht_params); +} + +static void cxgb4_process_flow_match(struct net_device *dev, + struct flow_rule *rule, + struct ch_filter_specification *fs) +{ + u16 addr_type = 0; + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { + struct flow_match_control match; + + flow_rule_match_control(rule, &match); + addr_type = match.key->addr_type; + } else if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { + addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; + } else if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { + addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { + struct flow_match_basic match; + u16 ethtype_key, ethtype_mask; + + flow_rule_match_basic(rule, &match); + ethtype_key = ntohs(match.key->n_proto); + ethtype_mask = ntohs(match.mask->n_proto); + + if (ethtype_key == ETH_P_ALL) { + ethtype_key = 0; + ethtype_mask = 0; + } + + if (ethtype_key == ETH_P_IPV6) + fs->type = 1; + + fs->val.ethtype = ethtype_key; + fs->mask.ethtype = ethtype_mask; + fs->val.proto = match.key->ip_proto; + fs->mask.proto = match.mask->ip_proto; + } + + if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { + struct flow_match_ipv4_addrs match; + + flow_rule_match_ipv4_addrs(rule, &match); + fs->type = 0; + memcpy(&fs->val.lip[0], &match.key->dst, sizeof(match.key->dst)); + memcpy(&fs->val.fip[0], &match.key->src, sizeof(match.key->src)); + memcpy(&fs->mask.lip[0], &match.mask->dst, sizeof(match.mask->dst)); + memcpy(&fs->mask.fip[0], &match.mask->src, sizeof(match.mask->src)); + + /* also initialize nat_lip/fip to same values */ + memcpy(&fs->nat_lip[0], &match.key->dst, sizeof(match.key->dst)); + memcpy(&fs->nat_fip[0], &match.key->src, sizeof(match.key->src)); + } + + if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { + struct flow_match_ipv6_addrs match; + + flow_rule_match_ipv6_addrs(rule, &match); + fs->type = 1; + memcpy(&fs->val.lip[0], match.key->dst.s6_addr, + sizeof(match.key->dst)); + memcpy(&fs->val.fip[0], match.key->src.s6_addr, + sizeof(match.key->src)); + memcpy(&fs->mask.lip[0], match.mask->dst.s6_addr, + sizeof(match.mask->dst)); + memcpy(&fs->mask.fip[0], match.mask->src.s6_addr, + sizeof(match.mask->src)); + + /* also initialize nat_lip/fip to same values */ + memcpy(&fs->nat_lip[0], match.key->dst.s6_addr, + sizeof(match.key->dst)); + memcpy(&fs->nat_fip[0], match.key->src.s6_addr, + sizeof(match.key->src)); + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { + struct flow_match_ports match; + + flow_rule_match_ports(rule, &match); + fs->val.lport = be16_to_cpu(match.key->dst); + fs->mask.lport = be16_to_cpu(match.mask->dst); + fs->val.fport = be16_to_cpu(match.key->src); + fs->mask.fport = be16_to_cpu(match.mask->src); + + /* also initialize nat_lport/fport to same values */ + fs->nat_lport = fs->val.lport; + fs->nat_fport = fs->val.fport; + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) { + struct flow_match_ip match; + + flow_rule_match_ip(rule, &match); + fs->val.tos = match.key->tos; + fs->mask.tos = match.mask->tos; + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { + struct flow_match_enc_keyid match; + + flow_rule_match_enc_keyid(rule, &match); + fs->val.vni = be32_to_cpu(match.key->keyid); + fs->mask.vni = be32_to_cpu(match.mask->keyid); + if (fs->mask.vni) { + fs->val.encap_vld = 1; + fs->mask.encap_vld = 1; + } + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { + struct flow_match_vlan match; + u16 vlan_tci, vlan_tci_mask; + + flow_rule_match_vlan(rule, &match); + vlan_tci = match.key->vlan_id | (match.key->vlan_priority << + VLAN_PRIO_SHIFT); + vlan_tci_mask = match.mask->vlan_id | (match.mask->vlan_priority << + VLAN_PRIO_SHIFT); + fs->val.ivlan = vlan_tci; + fs->mask.ivlan = vlan_tci_mask; + + fs->val.ivlan_vld = 1; + fs->mask.ivlan_vld = 1; + + /* Chelsio adapters use ivlan_vld bit to match vlan packets + * as 802.1Q. Also, when vlan tag is present in packets, + * ethtype match is used then to match on ethtype of inner + * header ie. the header following the vlan header. + * So, set the ivlan_vld based on ethtype info supplied by + * TC for vlan packets if its 802.1Q. And then reset the + * ethtype value else, hw will try to match the supplied + * ethtype value with ethtype of inner header. + */ + if (fs->val.ethtype == ETH_P_8021Q) { + fs->val.ethtype = 0; + fs->mask.ethtype = 0; + } + } + + /* Match only packets coming from the ingress port where this + * filter will be created. + */ + fs->val.iport = netdev2pinfo(dev)->port_id; + fs->mask.iport = ~0; +} + +static int cxgb4_validate_flow_match(struct net_device *dev, + struct flow_rule *rule) +{ + struct flow_dissector *dissector = rule->match.dissector; + u16 ethtype_mask = 0; + u16 ethtype_key = 0; + + if (dissector->used_keys & + ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | + BIT(FLOW_DISSECTOR_KEY_BASIC) | + BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT(FLOW_DISSECTOR_KEY_PORTS) | + BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | + BIT(FLOW_DISSECTOR_KEY_VLAN) | + BIT(FLOW_DISSECTOR_KEY_IP))) { + netdev_warn(dev, "Unsupported key used: 0x%x\n", + dissector->used_keys); + return -EOPNOTSUPP; + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { + struct flow_match_basic match; + + flow_rule_match_basic(rule, &match); + ethtype_key = ntohs(match.key->n_proto); + ethtype_mask = ntohs(match.mask->n_proto); + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) { + u16 eth_ip_type = ethtype_key & ethtype_mask; + struct flow_match_ip match; + + if (eth_ip_type != ETH_P_IP && eth_ip_type != ETH_P_IPV6) { + netdev_err(dev, "IP Key supported only with IPv4/v6"); + return -EINVAL; + } + + flow_rule_match_ip(rule, &match); + if (match.mask->ttl) { + netdev_warn(dev, "ttl match unsupported for offload"); + return -EOPNOTSUPP; + } + } + + return 0; +} + +static void offload_pedit(struct ch_filter_specification *fs, u32 val, u32 mask, + u8 field) +{ + u32 set_val = val & ~mask; + u32 offset = 0; + u8 size = 1; + int i; + + for (i = 0; i < ARRAY_SIZE(pedits); i++) { + if (pedits[i].field == field) { + offset = pedits[i].offset; + size = pedits[i].size; + break; + } + } + memcpy((u8 *)fs + offset, &set_val, size); +} + +static void process_pedit_field(struct ch_filter_specification *fs, u32 val, + u32 mask, u32 offset, u8 htype, + u8 *natmode_flags) +{ + switch (htype) { + case FLOW_ACT_MANGLE_HDR_TYPE_ETH: + switch (offset) { + case PEDIT_ETH_DMAC_31_0: + fs->newdmac = 1; + offload_pedit(fs, val, mask, ETH_DMAC_31_0); + break; + case PEDIT_ETH_DMAC_47_32_SMAC_15_0: + if (~mask & PEDIT_ETH_DMAC_MASK) + offload_pedit(fs, val, mask, ETH_DMAC_47_32); + else + offload_pedit(fs, val >> 16, mask >> 16, + ETH_SMAC_15_0); + break; + case PEDIT_ETH_SMAC_47_16: + fs->newsmac = 1; + offload_pedit(fs, val, mask, ETH_SMAC_47_16); + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_IP4: + switch (offset) { + case PEDIT_IP4_SRC: + offload_pedit(fs, val, mask, IP4_SRC); + *natmode_flags |= CXGB4_ACTION_NATMODE_SIP; + break; + case PEDIT_IP4_DST: + offload_pedit(fs, val, mask, IP4_DST); + *natmode_flags |= CXGB4_ACTION_NATMODE_DIP; + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_IP6: + switch (offset) { + case PEDIT_IP6_SRC_31_0: + offload_pedit(fs, val, mask, IP6_SRC_31_0); + *natmode_flags |= CXGB4_ACTION_NATMODE_SIP; + break; + case PEDIT_IP6_SRC_63_32: + offload_pedit(fs, val, mask, IP6_SRC_63_32); + *natmode_flags |= CXGB4_ACTION_NATMODE_SIP; + break; + case PEDIT_IP6_SRC_95_64: + offload_pedit(fs, val, mask, IP6_SRC_95_64); + *natmode_flags |= CXGB4_ACTION_NATMODE_SIP; + break; + case PEDIT_IP6_SRC_127_96: + offload_pedit(fs, val, mask, IP6_SRC_127_96); + *natmode_flags |= CXGB4_ACTION_NATMODE_SIP; + break; + case PEDIT_IP6_DST_31_0: + offload_pedit(fs, val, mask, IP6_DST_31_0); + *natmode_flags |= CXGB4_ACTION_NATMODE_DIP; + break; + case PEDIT_IP6_DST_63_32: + offload_pedit(fs, val, mask, IP6_DST_63_32); + *natmode_flags |= CXGB4_ACTION_NATMODE_DIP; + break; + case PEDIT_IP6_DST_95_64: + offload_pedit(fs, val, mask, IP6_DST_95_64); + *natmode_flags |= CXGB4_ACTION_NATMODE_DIP; + break; + case PEDIT_IP6_DST_127_96: + offload_pedit(fs, val, mask, IP6_DST_127_96); + *natmode_flags |= CXGB4_ACTION_NATMODE_DIP; + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_TCP: + switch (offset) { + case PEDIT_TCP_SPORT_DPORT: + if (~mask & PEDIT_TCP_UDP_SPORT_MASK) { + fs->nat_fport = val; + *natmode_flags |= CXGB4_ACTION_NATMODE_SPORT; + } else { + fs->nat_lport = val >> 16; + *natmode_flags |= CXGB4_ACTION_NATMODE_DPORT; + } + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_UDP: + switch (offset) { + case PEDIT_UDP_SPORT_DPORT: + if (~mask & PEDIT_TCP_UDP_SPORT_MASK) { + fs->nat_fport = val; + *natmode_flags |= CXGB4_ACTION_NATMODE_SPORT; + } else { + fs->nat_lport = val >> 16; + *natmode_flags |= CXGB4_ACTION_NATMODE_DPORT; + } + } + break; + } +} + +static int cxgb4_action_natmode_validate(struct adapter *adap, u8 natmode_flags, + struct netlink_ext_ack *extack) +{ + u8 i = 0; + + /* Extract the NAT mode to enable based on what 4-tuple fields + * are enabled to be overwritten. This ensures that the + * disabled fields don't get overwritten to 0. + */ + for (i = 0; i < ARRAY_SIZE(cxgb4_natmode_config_array); i++) { + const struct cxgb4_natmode_config *c; + + c = &cxgb4_natmode_config_array[i]; + if (CHELSIO_CHIP_VERSION(adap->params.chip) >= c->chip && + natmode_flags == c->flags) + return 0; + } + NL_SET_ERR_MSG_MOD(extack, "Unsupported NAT mode 4-tuple combination"); + return -EOPNOTSUPP; +} + +void cxgb4_process_flow_actions(struct net_device *in, + struct flow_action *actions, + struct ch_filter_specification *fs) +{ + struct flow_action_entry *act; + u8 natmode_flags = 0; + int i; + + flow_action_for_each(i, act, actions) { + switch (act->id) { + case FLOW_ACTION_ACCEPT: + fs->action = FILTER_PASS; + break; + case FLOW_ACTION_DROP: + fs->action = FILTER_DROP; + break; + case FLOW_ACTION_MIRRED: + case FLOW_ACTION_REDIRECT: { + struct net_device *out = act->dev; + struct port_info *pi = netdev_priv(out); + + fs->action = FILTER_SWITCH; + fs->eport = pi->port_id; + } + break; + case FLOW_ACTION_VLAN_POP: + case FLOW_ACTION_VLAN_PUSH: + case FLOW_ACTION_VLAN_MANGLE: { + u8 prio = act->vlan.prio; + u16 vid = act->vlan.vid; + u16 vlan_tci = (prio << VLAN_PRIO_SHIFT) | vid; + switch (act->id) { + case FLOW_ACTION_VLAN_POP: + fs->newvlan |= VLAN_REMOVE; + break; + case FLOW_ACTION_VLAN_PUSH: + fs->newvlan |= VLAN_INSERT; + fs->vlan = vlan_tci; + break; + case FLOW_ACTION_VLAN_MANGLE: + fs->newvlan |= VLAN_REWRITE; + fs->vlan = vlan_tci; + break; + default: + break; + } + } + break; + case FLOW_ACTION_MANGLE: { + u32 mask, val, offset; + u8 htype; + + htype = act->mangle.htype; + mask = act->mangle.mask; + val = act->mangle.val; + offset = act->mangle.offset; + + process_pedit_field(fs, val, mask, offset, htype, + &natmode_flags); + } + break; + case FLOW_ACTION_QUEUE: + fs->action = FILTER_PASS; + fs->dirsteer = 1; + fs->iq = act->queue.index; + break; + default: + break; + } + } + if (natmode_flags) + cxgb4_action_natmode_tweak(fs, natmode_flags); + +} + +static bool valid_l4_mask(u32 mask) +{ + u16 hi, lo; + + /* Either the upper 16-bits (SPORT) OR the lower + * 16-bits (DPORT) can be set, but NOT BOTH. + */ + hi = (mask >> 16) & 0xFFFF; + lo = mask & 0xFFFF; + + return hi && lo ? false : true; +} + +static bool valid_pedit_action(struct net_device *dev, + const struct flow_action_entry *act, + u8 *natmode_flags) +{ + u32 mask, offset; + u8 htype; + + htype = act->mangle.htype; + mask = act->mangle.mask; + offset = act->mangle.offset; + + switch (htype) { + case FLOW_ACT_MANGLE_HDR_TYPE_ETH: + switch (offset) { + case PEDIT_ETH_DMAC_31_0: + case PEDIT_ETH_DMAC_47_32_SMAC_15_0: + case PEDIT_ETH_SMAC_47_16: + break; + default: + netdev_err(dev, "%s: Unsupported pedit field\n", + __func__); + return false; + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_IP4: + switch (offset) { + case PEDIT_IP4_SRC: + *natmode_flags |= CXGB4_ACTION_NATMODE_SIP; + break; + case PEDIT_IP4_DST: + *natmode_flags |= CXGB4_ACTION_NATMODE_DIP; + break; + default: + netdev_err(dev, "%s: Unsupported pedit field\n", + __func__); + return false; + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_IP6: + switch (offset) { + case PEDIT_IP6_SRC_31_0: + case PEDIT_IP6_SRC_63_32: + case PEDIT_IP6_SRC_95_64: + case PEDIT_IP6_SRC_127_96: + *natmode_flags |= CXGB4_ACTION_NATMODE_SIP; + break; + case PEDIT_IP6_DST_31_0: + case PEDIT_IP6_DST_63_32: + case PEDIT_IP6_DST_95_64: + case PEDIT_IP6_DST_127_96: + *natmode_flags |= CXGB4_ACTION_NATMODE_DIP; + break; + default: + netdev_err(dev, "%s: Unsupported pedit field\n", + __func__); + return false; + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_TCP: + switch (offset) { + case PEDIT_TCP_SPORT_DPORT: + if (!valid_l4_mask(~mask)) { + netdev_err(dev, "%s: Unsupported mask for TCP L4 ports\n", + __func__); + return false; + } + if (~mask & PEDIT_TCP_UDP_SPORT_MASK) + *natmode_flags |= CXGB4_ACTION_NATMODE_SPORT; + else + *natmode_flags |= CXGB4_ACTION_NATMODE_DPORT; + break; + default: + netdev_err(dev, "%s: Unsupported pedit field\n", + __func__); + return false; + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_UDP: + switch (offset) { + case PEDIT_UDP_SPORT_DPORT: + if (!valid_l4_mask(~mask)) { + netdev_err(dev, "%s: Unsupported mask for UDP L4 ports\n", + __func__); + return false; + } + if (~mask & PEDIT_TCP_UDP_SPORT_MASK) + *natmode_flags |= CXGB4_ACTION_NATMODE_SPORT; + else + *natmode_flags |= CXGB4_ACTION_NATMODE_DPORT; + break; + default: + netdev_err(dev, "%s: Unsupported pedit field\n", + __func__); + return false; + } + break; + default: + netdev_err(dev, "%s: Unsupported pedit type\n", __func__); + return false; + } + return true; +} + +int cxgb4_validate_flow_actions(struct net_device *dev, + struct flow_action *actions, + struct netlink_ext_ack *extack, + u8 matchall_filter) +{ + struct adapter *adap = netdev2adap(dev); + struct flow_action_entry *act; + bool act_redir = false; + bool act_pedit = false; + bool act_vlan = false; + u8 natmode_flags = 0; + int i; + + if (!flow_action_basic_hw_stats_check(actions, extack)) + return -EOPNOTSUPP; + + flow_action_for_each(i, act, actions) { + switch (act->id) { + case FLOW_ACTION_ACCEPT: + case FLOW_ACTION_DROP: + /* Do nothing */ + break; + case FLOW_ACTION_MIRRED: + case FLOW_ACTION_REDIRECT: { + struct net_device *n_dev, *target_dev; + bool found = false; + unsigned int i; + + if (act->id == FLOW_ACTION_MIRRED && + !matchall_filter) { + NL_SET_ERR_MSG_MOD(extack, + "Egress mirror action is only supported for tc-matchall"); + return -EOPNOTSUPP; + } + + target_dev = act->dev; + for_each_port(adap, i) { + n_dev = adap->port[i]; + if (target_dev == n_dev) { + found = true; + break; + } + } + + /* If interface doesn't belong to our hw, then + * the provided output port is not valid + */ + if (!found) { + netdev_err(dev, "%s: Out port invalid\n", + __func__); + return -EINVAL; + } + act_redir = true; + } + break; + case FLOW_ACTION_VLAN_POP: + case FLOW_ACTION_VLAN_PUSH: + case FLOW_ACTION_VLAN_MANGLE: { + u16 proto = be16_to_cpu(act->vlan.proto); + + switch (act->id) { + case FLOW_ACTION_VLAN_POP: + break; + case FLOW_ACTION_VLAN_PUSH: + case FLOW_ACTION_VLAN_MANGLE: + if (proto != ETH_P_8021Q) { + netdev_err(dev, "%s: Unsupported vlan proto\n", + __func__); + return -EOPNOTSUPP; + } + break; + default: + netdev_err(dev, "%s: Unsupported vlan action\n", + __func__); + return -EOPNOTSUPP; + } + act_vlan = true; + } + break; + case FLOW_ACTION_MANGLE: { + bool pedit_valid = valid_pedit_action(dev, act, + &natmode_flags); + + if (!pedit_valid) + return -EOPNOTSUPP; + act_pedit = true; + } + break; + case FLOW_ACTION_QUEUE: + /* Do nothing. cxgb4_set_filter will validate */ + break; + default: + netdev_err(dev, "%s: Unsupported action\n", __func__); + return -EOPNOTSUPP; + } + } + + if ((act_pedit || act_vlan) && !act_redir) { + netdev_err(dev, "%s: pedit/vlan rewrite invalid without egress redirect\n", + __func__); + return -EINVAL; + } + + if (act_pedit) { + int ret; + + ret = cxgb4_action_natmode_validate(adap, natmode_flags, + extack); + if (ret) + return ret; + } + + return 0; +} + +static void cxgb4_tc_flower_hash_prio_add(struct adapter *adap, u32 tc_prio) +{ + spin_lock_bh(&adap->tids.ftid_lock); + if (adap->tids.tc_hash_tids_max_prio < tc_prio) + adap->tids.tc_hash_tids_max_prio = tc_prio; + spin_unlock_bh(&adap->tids.ftid_lock); +} + +static void cxgb4_tc_flower_hash_prio_del(struct adapter *adap, u32 tc_prio) +{ + struct tid_info *t = &adap->tids; + struct ch_tc_flower_entry *fe; + struct rhashtable_iter iter; + u32 found = 0; + + spin_lock_bh(&t->ftid_lock); + /* Bail if the current rule is not the one with the max + * prio. + */ + if (t->tc_hash_tids_max_prio != tc_prio) + goto out_unlock; + + /* Search for the next rule having the same or next lower + * max prio. + */ + rhashtable_walk_enter(&adap->flower_tbl, &iter); + do { + rhashtable_walk_start(&iter); + + fe = rhashtable_walk_next(&iter); + while (!IS_ERR_OR_NULL(fe)) { + if (fe->fs.hash && + fe->fs.tc_prio <= t->tc_hash_tids_max_prio) { + t->tc_hash_tids_max_prio = fe->fs.tc_prio; + found++; + + /* Bail if we found another rule + * having the same prio as the + * current max one. + */ + if (fe->fs.tc_prio == tc_prio) + break; + } + + fe = rhashtable_walk_next(&iter); + } + + rhashtable_walk_stop(&iter); + } while (fe == ERR_PTR(-EAGAIN)); + rhashtable_walk_exit(&iter); + + if (!found) + t->tc_hash_tids_max_prio = 0; + +out_unlock: + spin_unlock_bh(&t->ftid_lock); +} + +int cxgb4_flow_rule_replace(struct net_device *dev, struct flow_rule *rule, + u32 tc_prio, struct netlink_ext_ack *extack, + struct ch_filter_specification *fs, u32 *tid) +{ + struct adapter *adap = netdev2adap(dev); + struct filter_ctx ctx; + u8 inet_family; + int fidx, ret; + + if (cxgb4_validate_flow_actions(dev, &rule->action, extack, 0)) + return -EOPNOTSUPP; + + if (cxgb4_validate_flow_match(dev, rule)) + return -EOPNOTSUPP; + + cxgb4_process_flow_match(dev, rule, fs); + cxgb4_process_flow_actions(dev, &rule->action, fs); + + fs->hash = is_filter_exact_match(adap, fs); + inet_family = fs->type ? PF_INET6 : PF_INET; + + /* Get a free filter entry TID, where we can insert this new + * rule. Only insert rule if its prio doesn't conflict with + * existing rules. + */ + fidx = cxgb4_get_free_ftid(dev, inet_family, fs->hash, + tc_prio); + if (fidx < 0) { + NL_SET_ERR_MSG_MOD(extack, + "No free LETCAM index available"); + return -ENOMEM; + } + + if (fidx < adap->tids.nhpftids) { + fs->prio = 1; + fs->hash = 0; + } + + /* If the rule can be inserted into HASH region, then ignore + * the index to normal FILTER region. + */ + if (fs->hash) + fidx = 0; + + fs->tc_prio = tc_prio; + + init_completion(&ctx.completion); + ret = __cxgb4_set_filter(dev, fidx, fs, &ctx); + if (ret) { + netdev_err(dev, "%s: filter creation err %d\n", + __func__, ret); + return ret; + } + + /* Wait for reply */ + ret = wait_for_completion_timeout(&ctx.completion, 10 * HZ); + if (!ret) + return -ETIMEDOUT; + + /* Check if hw returned error for filter creation */ + if (ctx.result) + return ctx.result; + + *tid = ctx.tid; + + if (fs->hash) + cxgb4_tc_flower_hash_prio_add(adap, tc_prio); + + return 0; +} + +int cxgb4_tc_flower_replace(struct net_device *dev, + struct flow_cls_offload *cls) +{ + struct flow_rule *rule = flow_cls_offload_flow_rule(cls); + struct netlink_ext_ack *extack = cls->common.extack; + struct adapter *adap = netdev2adap(dev); + struct ch_tc_flower_entry *ch_flower; + struct ch_filter_specification *fs; + int ret; + + ch_flower = allocate_flower_entry(); + if (!ch_flower) { + netdev_err(dev, "%s: ch_flower alloc failed.\n", __func__); + return -ENOMEM; + } + + fs = &ch_flower->fs; + fs->hitcnts = 1; + fs->tc_cookie = cls->cookie; + + ret = cxgb4_flow_rule_replace(dev, rule, cls->common.prio, extack, fs, + &ch_flower->filter_id); + if (ret) + goto free_entry; + + ch_flower->tc_flower_cookie = cls->cookie; + ret = rhashtable_insert_fast(&adap->flower_tbl, &ch_flower->node, + adap->flower_ht_params); + if (ret) + goto del_filter; + + return 0; + +del_filter: + if (fs->hash) + cxgb4_tc_flower_hash_prio_del(adap, cls->common.prio); + + cxgb4_del_filter(dev, ch_flower->filter_id, &ch_flower->fs); + +free_entry: + kfree(ch_flower); + return ret; +} + +int cxgb4_flow_rule_destroy(struct net_device *dev, u32 tc_prio, + struct ch_filter_specification *fs, int tid) +{ + struct adapter *adap = netdev2adap(dev); + u8 hash; + int ret; + + hash = fs->hash; + + ret = cxgb4_del_filter(dev, tid, fs); + if (ret) + return ret; + + if (hash) + cxgb4_tc_flower_hash_prio_del(adap, tc_prio); + + return ret; +} + +int cxgb4_tc_flower_destroy(struct net_device *dev, + struct flow_cls_offload *cls) +{ + struct adapter *adap = netdev2adap(dev); + struct ch_tc_flower_entry *ch_flower; + int ret; + + ch_flower = ch_flower_lookup(adap, cls->cookie); + if (!ch_flower) + return -ENOENT; + + rhashtable_remove_fast(&adap->flower_tbl, &ch_flower->node, + adap->flower_ht_params); + + ret = cxgb4_flow_rule_destroy(dev, ch_flower->fs.tc_prio, + &ch_flower->fs, ch_flower->filter_id); + if (ret) + netdev_err(dev, "Flow rule destroy failed for tid: %u, ret: %d", + ch_flower->filter_id, ret); + + kfree_rcu(ch_flower, rcu); + return ret; +} + +static void ch_flower_stats_handler(struct work_struct *work) +{ + struct adapter *adap = container_of(work, struct adapter, + flower_stats_work); + struct ch_tc_flower_entry *flower_entry; + struct ch_tc_flower_stats *ofld_stats; + struct rhashtable_iter iter; + u64 packets; + u64 bytes; + int ret; + + rhashtable_walk_enter(&adap->flower_tbl, &iter); + do { + rhashtable_walk_start(&iter); + + while ((flower_entry = rhashtable_walk_next(&iter)) && + !IS_ERR(flower_entry)) { + ret = cxgb4_get_filter_counters(adap->port[0], + flower_entry->filter_id, + &packets, &bytes, + flower_entry->fs.hash); + if (!ret) { + spin_lock(&flower_entry->lock); + ofld_stats = &flower_entry->stats; + + if (ofld_stats->prev_packet_count != packets) { + ofld_stats->prev_packet_count = packets; + ofld_stats->last_used = jiffies; + } + spin_unlock(&flower_entry->lock); + } + } + + rhashtable_walk_stop(&iter); + + } while (flower_entry == ERR_PTR(-EAGAIN)); + rhashtable_walk_exit(&iter); + mod_timer(&adap->flower_stats_timer, jiffies + STATS_CHECK_PERIOD); +} + +static void ch_flower_stats_cb(struct timer_list *t) +{ + struct adapter *adap = from_timer(adap, t, flower_stats_timer); + + schedule_work(&adap->flower_stats_work); +} + +int cxgb4_tc_flower_stats(struct net_device *dev, + struct flow_cls_offload *cls) +{ + struct adapter *adap = netdev2adap(dev); + struct ch_tc_flower_stats *ofld_stats; + struct ch_tc_flower_entry *ch_flower; + u64 packets; + u64 bytes; + int ret; + + ch_flower = ch_flower_lookup(adap, cls->cookie); + if (!ch_flower) { + ret = -ENOENT; + goto err; + } + + ret = cxgb4_get_filter_counters(dev, ch_flower->filter_id, + &packets, &bytes, + ch_flower->fs.hash); + if (ret < 0) + goto err; + + spin_lock_bh(&ch_flower->lock); + ofld_stats = &ch_flower->stats; + if (ofld_stats->packet_count != packets) { + if (ofld_stats->prev_packet_count != packets) + ofld_stats->last_used = jiffies; + flow_stats_update(&cls->stats, bytes - ofld_stats->byte_count, + packets - ofld_stats->packet_count, 0, + ofld_stats->last_used, + FLOW_ACTION_HW_STATS_IMMEDIATE); + + ofld_stats->packet_count = packets; + ofld_stats->byte_count = bytes; + ofld_stats->prev_packet_count = packets; + } + spin_unlock_bh(&ch_flower->lock); + return 0; + +err: + return ret; +} + +static const struct rhashtable_params cxgb4_tc_flower_ht_params = { + .nelem_hint = 384, + .head_offset = offsetof(struct ch_tc_flower_entry, node), + .key_offset = offsetof(struct ch_tc_flower_entry, tc_flower_cookie), + .key_len = sizeof(((struct ch_tc_flower_entry *)0)->tc_flower_cookie), + .max_size = 524288, + .min_size = 512, + .automatic_shrinking = true +}; + +int cxgb4_init_tc_flower(struct adapter *adap) +{ + int ret; + + if (adap->tc_flower_initialized) + return -EEXIST; + + adap->flower_ht_params = cxgb4_tc_flower_ht_params; + ret = rhashtable_init(&adap->flower_tbl, &adap->flower_ht_params); + if (ret) + return ret; + + INIT_WORK(&adap->flower_stats_work, ch_flower_stats_handler); + timer_setup(&adap->flower_stats_timer, ch_flower_stats_cb, 0); + mod_timer(&adap->flower_stats_timer, jiffies + STATS_CHECK_PERIOD); + adap->tc_flower_initialized = true; + return 0; +} + +void cxgb4_cleanup_tc_flower(struct adapter *adap) +{ + if (!adap->tc_flower_initialized) + return; + + if (adap->flower_stats_timer.function) + del_timer_sync(&adap->flower_stats_timer); + cancel_work_sync(&adap->flower_stats_work); + rhashtable_destroy(&adap->flower_tbl); + adap->tc_flower_initialized = false; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.h new file mode 100644 index 000000000..3a2fa00c8 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.h @@ -0,0 +1,148 @@ +/* + * This file is part of the Chelsio T4/T5/T6 Ethernet driver for Linux. + * + * Copyright (c) 2017 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_TC_FLOWER_H +#define __CXGB4_TC_FLOWER_H + +#include <net/pkt_cls.h> + +struct ch_tc_flower_stats { + u64 prev_packet_count; + u64 packet_count; + u64 byte_count; + u64 last_used; +}; + +struct ch_tc_flower_entry { + struct ch_filter_specification fs; + struct ch_tc_flower_stats stats; + unsigned long tc_flower_cookie; + struct rhash_head node; + struct rcu_head rcu; + spinlock_t lock; /* lock for stats */ + u32 filter_id; +}; + +enum { + ETH_DMAC_31_0, /* dmac bits 0.. 31 */ + ETH_DMAC_47_32, /* dmac bits 32..47 */ + ETH_SMAC_15_0, /* smac bits 0.. 15 */ + ETH_SMAC_47_16, /* smac bits 16..47 */ + + IP4_SRC, /* 32-bit IPv4 src */ + IP4_DST, /* 32-bit IPv4 dst */ + + IP6_SRC_31_0, /* src bits 0.. 31 */ + IP6_SRC_63_32, /* src bits 63.. 32 */ + IP6_SRC_95_64, /* src bits 95.. 64 */ + IP6_SRC_127_96, /* src bits 127..96 */ + + IP6_DST_31_0, /* dst bits 0.. 31 */ + IP6_DST_63_32, /* dst bits 63.. 32 */ + IP6_DST_95_64, /* dst bits 95.. 64 */ + IP6_DST_127_96, /* dst bits 127..96 */ + + TCP_SPORT, /* 16-bit TCP sport */ + TCP_DPORT, /* 16-bit TCP dport */ + + UDP_SPORT, /* 16-bit UDP sport */ + UDP_DPORT, /* 16-bit UDP dport */ +}; + +struct ch_tc_pedit_fields { + u8 field; + u8 size; + u32 offset; +}; + +#define PEDIT_FIELDS(type, field, size, fs_field, offset) \ + { type## field, size, \ + offsetof(struct ch_filter_specification, fs_field) + (offset) } + +#define PEDIT_ETH_DMAC_MASK 0xffff +#define PEDIT_TCP_UDP_SPORT_MASK 0xffff +#define PEDIT_ETH_DMAC_31_0 0x0 +#define PEDIT_ETH_DMAC_47_32_SMAC_15_0 0x4 +#define PEDIT_ETH_SMAC_47_16 0x8 +#define PEDIT_IP4_SRC 0xC +#define PEDIT_IP4_DST 0x10 +#define PEDIT_IP6_SRC_31_0 0x8 +#define PEDIT_IP6_SRC_63_32 0xC +#define PEDIT_IP6_SRC_95_64 0x10 +#define PEDIT_IP6_SRC_127_96 0x14 +#define PEDIT_IP6_DST_31_0 0x18 +#define PEDIT_IP6_DST_63_32 0x1C +#define PEDIT_IP6_DST_95_64 0x20 +#define PEDIT_IP6_DST_127_96 0x24 +#define PEDIT_TCP_SPORT_DPORT 0x0 +#define PEDIT_UDP_SPORT_DPORT 0x0 + +enum cxgb4_action_natmode_flags { + CXGB4_ACTION_NATMODE_NONE = 0, + CXGB4_ACTION_NATMODE_DIP = (1 << 0), + CXGB4_ACTION_NATMODE_SIP = (1 << 1), + CXGB4_ACTION_NATMODE_DPORT = (1 << 2), + CXGB4_ACTION_NATMODE_SPORT = (1 << 3), +}; + +/* TC PEDIT action to NATMODE translation entry */ +struct cxgb4_natmode_config { + enum chip_type chip; + u8 flags; + u8 natmode; +}; + +void cxgb4_process_flow_actions(struct net_device *in, + struct flow_action *actions, + struct ch_filter_specification *fs); +int cxgb4_validate_flow_actions(struct net_device *dev, + struct flow_action *actions, + struct netlink_ext_ack *extack, + u8 matchall_filter); + +int cxgb4_tc_flower_replace(struct net_device *dev, + struct flow_cls_offload *cls); +int cxgb4_tc_flower_destroy(struct net_device *dev, + struct flow_cls_offload *cls); +int cxgb4_tc_flower_stats(struct net_device *dev, + struct flow_cls_offload *cls); +int cxgb4_flow_rule_replace(struct net_device *dev, struct flow_rule *rule, + u32 tc_prio, struct netlink_ext_ack *extack, + struct ch_filter_specification *fs, u32 *tid); +int cxgb4_flow_rule_destroy(struct net_device *dev, u32 tc_prio, + struct ch_filter_specification *fs, int tid); + +int cxgb4_init_tc_flower(struct adapter *adap); +void cxgb4_cleanup_tc_flower(struct adapter *adap); +#endif /* __CXGB4_TC_FLOWER_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_matchall.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_matchall.c new file mode 100644 index 000000000..1672d3afe --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_matchall.c @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2019 Chelsio Communications. All rights reserved. */ + +#include "cxgb4.h" +#include "cxgb4_tc_matchall.h" +#include "sched.h" +#include "cxgb4_uld.h" +#include "cxgb4_filter.h" +#include "cxgb4_tc_flower.h" + +static int cxgb4_policer_validate(const struct flow_action *action, + const struct flow_action_entry *act, + struct netlink_ext_ack *extack) +{ + if (act->police.exceed.act_id != FLOW_ACTION_DROP) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when exceed action is not drop"); + return -EOPNOTSUPP; + } + + if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && + act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when conform action is not pipe or ok"); + return -EOPNOTSUPP; + } + + if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT && + !flow_action_is_last_entry(action, act)) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when conform action is ok, but action is not last"); + return -EOPNOTSUPP; + } + + if (act->police.peakrate_bytes_ps || + act->police.avrate || act->police.overhead) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when peakrate/avrate/overhead is configured"); + return -EOPNOTSUPP; + } + + if (act->police.rate_pkt_ps) { + NL_SET_ERR_MSG_MOD(extack, + "QoS offload not support packets per second"); + return -EOPNOTSUPP; + } + + return 0; +} + +static int cxgb4_matchall_egress_validate(struct net_device *dev, + struct tc_cls_matchall_offload *cls) +{ + struct netlink_ext_ack *extack = cls->common.extack; + struct flow_action *actions = &cls->rule->action; + struct port_info *pi = netdev2pinfo(dev); + struct flow_action_entry *entry; + struct ch_sched_queue qe; + struct sched_class *e; + u64 max_link_rate; + u32 i, speed; + int ret; + + if (!flow_action_has_entries(actions)) { + NL_SET_ERR_MSG_MOD(extack, + "Egress MATCHALL offload needs at least 1 policing action"); + return -EINVAL; + } else if (!flow_offload_has_one_action(actions)) { + NL_SET_ERR_MSG_MOD(extack, + "Egress MATCHALL offload only supports 1 policing action"); + return -EINVAL; + } else if (pi->tc_block_shared) { + NL_SET_ERR_MSG_MOD(extack, + "Egress MATCHALL offload not supported with shared blocks"); + return -EINVAL; + } + + ret = t4_get_link_params(pi, NULL, &speed, NULL); + if (ret) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to get max speed supported by the link"); + return -EINVAL; + } + + /* Convert from Mbps to bps */ + max_link_rate = (u64)speed * 1000 * 1000; + + flow_action_for_each(i, entry, actions) { + switch (entry->id) { + case FLOW_ACTION_POLICE: + ret = cxgb4_policer_validate(actions, entry, extack); + if (ret) + return ret; + + /* Convert bytes per second to bits per second */ + if (entry->police.rate_bytes_ps * 8 > max_link_rate) { + NL_SET_ERR_MSG_MOD(extack, + "Specified policing max rate is larger than underlying link speed"); + return -ERANGE; + } + break; + default: + NL_SET_ERR_MSG_MOD(extack, + "Only policing action supported with Egress MATCHALL offload"); + return -EOPNOTSUPP; + } + } + + for (i = 0; i < pi->nqsets; i++) { + memset(&qe, 0, sizeof(qe)); + qe.queue = i; + + e = cxgb4_sched_queue_lookup(dev, &qe); + if (e && e->info.u.params.level != SCHED_CLASS_LEVEL_CH_RL) { + NL_SET_ERR_MSG_MOD(extack, + "Some queues are already bound to different class"); + return -EBUSY; + } + } + + return 0; +} + +static int cxgb4_matchall_tc_bind_queues(struct net_device *dev, u32 tc) +{ + struct port_info *pi = netdev2pinfo(dev); + struct ch_sched_queue qe; + int ret; + u32 i; + + for (i = 0; i < pi->nqsets; i++) { + qe.queue = i; + qe.class = tc; + ret = cxgb4_sched_class_bind(dev, &qe, SCHED_QUEUE); + if (ret) + goto out_free; + } + + return 0; + +out_free: + while (i--) { + qe.queue = i; + qe.class = SCHED_CLS_NONE; + cxgb4_sched_class_unbind(dev, &qe, SCHED_QUEUE); + } + + return ret; +} + +static void cxgb4_matchall_tc_unbind_queues(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct ch_sched_queue qe; + u32 i; + + for (i = 0; i < pi->nqsets; i++) { + qe.queue = i; + qe.class = SCHED_CLS_NONE; + cxgb4_sched_class_unbind(dev, &qe, SCHED_QUEUE); + } +} + +static int cxgb4_matchall_alloc_tc(struct net_device *dev, + struct tc_cls_matchall_offload *cls) +{ + struct ch_sched_params p = { + .type = SCHED_CLASS_TYPE_PACKET, + .u.params.level = SCHED_CLASS_LEVEL_CH_RL, + .u.params.mode = SCHED_CLASS_MODE_CLASS, + .u.params.rateunit = SCHED_CLASS_RATEUNIT_BITS, + .u.params.ratemode = SCHED_CLASS_RATEMODE_ABS, + .u.params.class = SCHED_CLS_NONE, + .u.params.minrate = 0, + .u.params.weight = 0, + .u.params.pktsize = dev->mtu, + }; + struct netlink_ext_ack *extack = cls->common.extack; + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + struct flow_action_entry *entry; + struct sched_class *e; + int ret; + u32 i; + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + + flow_action_for_each(i, entry, &cls->rule->action) + if (entry->id == FLOW_ACTION_POLICE) + break; + + ret = cxgb4_policer_validate(&cls->rule->action, entry, extack); + if (ret) + return ret; + + /* Convert from bytes per second to Kbps */ + p.u.params.maxrate = div_u64(entry->police.rate_bytes_ps * 8, 1000); + p.u.params.channel = pi->tx_chan; + e = cxgb4_sched_class_alloc(dev, &p); + if (!e) { + NL_SET_ERR_MSG_MOD(extack, + "No free traffic class available for policing action"); + return -ENOMEM; + } + + ret = cxgb4_matchall_tc_bind_queues(dev, e->idx); + if (ret) { + NL_SET_ERR_MSG_MOD(extack, + "Could not bind queues to traffic class"); + goto out_free; + } + + tc_port_matchall->egress.hwtc = e->idx; + tc_port_matchall->egress.cookie = cls->cookie; + tc_port_matchall->egress.state = CXGB4_MATCHALL_STATE_ENABLED; + return 0; + +out_free: + cxgb4_sched_class_free(dev, e->idx); + return ret; +} + +static void cxgb4_matchall_free_tc(struct net_device *dev) +{ + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + cxgb4_matchall_tc_unbind_queues(dev); + cxgb4_sched_class_free(dev, tc_port_matchall->egress.hwtc); + + tc_port_matchall->egress.hwtc = SCHED_CLS_NONE; + tc_port_matchall->egress.cookie = 0; + tc_port_matchall->egress.state = CXGB4_MATCHALL_STATE_DISABLED; +} + +static int cxgb4_matchall_mirror_alloc(struct net_device *dev, + struct tc_cls_matchall_offload *cls) +{ + struct netlink_ext_ack *extack = cls->common.extack; + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + struct flow_action_entry *act; + int ret; + u32 i; + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + flow_action_for_each(i, act, &cls->rule->action) { + if (act->id == FLOW_ACTION_MIRRED) { + ret = cxgb4_port_mirror_alloc(dev); + if (ret) { + NL_SET_ERR_MSG_MOD(extack, + "Couldn't allocate mirror"); + return ret; + } + + tc_port_matchall->ingress.viid_mirror = pi->viid_mirror; + break; + } + } + + return 0; +} + +static void cxgb4_matchall_mirror_free(struct net_device *dev) +{ + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + if (!tc_port_matchall->ingress.viid_mirror) + return; + + cxgb4_port_mirror_free(dev); + tc_port_matchall->ingress.viid_mirror = 0; +} + +static int cxgb4_matchall_del_filter(struct net_device *dev, u8 filter_type) +{ + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + int ret; + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + ret = cxgb4_del_filter(dev, tc_port_matchall->ingress.tid[filter_type], + &tc_port_matchall->ingress.fs[filter_type]); + if (ret) + return ret; + + tc_port_matchall->ingress.tid[filter_type] = 0; + return 0; +} + +static int cxgb4_matchall_add_filter(struct net_device *dev, + struct tc_cls_matchall_offload *cls, + u8 filter_type) +{ + struct netlink_ext_ack *extack = cls->common.extack; + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + struct ch_filter_specification *fs; + int ret, fidx; + + /* Get a free filter entry TID, where we can insert this new + * rule. Only insert rule if its prio doesn't conflict with + * existing rules. + */ + fidx = cxgb4_get_free_ftid(dev, filter_type ? PF_INET6 : PF_INET, + false, cls->common.prio); + if (fidx < 0) { + NL_SET_ERR_MSG_MOD(extack, + "No free LETCAM index available"); + return -ENOMEM; + } + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + fs = &tc_port_matchall->ingress.fs[filter_type]; + memset(fs, 0, sizeof(*fs)); + + if (fidx < adap->tids.nhpftids) + fs->prio = 1; + fs->tc_prio = cls->common.prio; + fs->tc_cookie = cls->cookie; + fs->type = filter_type; + fs->hitcnts = 1; + + fs->val.pfvf_vld = 1; + fs->val.pf = adap->pf; + fs->val.vf = pi->vin; + + cxgb4_process_flow_actions(dev, &cls->rule->action, fs); + + ret = cxgb4_set_filter(dev, fidx, fs); + if (ret) + return ret; + + tc_port_matchall->ingress.tid[filter_type] = fidx; + return 0; +} + +static int cxgb4_matchall_alloc_filter(struct net_device *dev, + struct tc_cls_matchall_offload *cls) +{ + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + int ret, i; + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + + ret = cxgb4_matchall_mirror_alloc(dev, cls); + if (ret) + return ret; + + for (i = 0; i < CXGB4_FILTER_TYPE_MAX; i++) { + ret = cxgb4_matchall_add_filter(dev, cls, i); + if (ret) + goto out_free; + } + + tc_port_matchall->ingress.state = CXGB4_MATCHALL_STATE_ENABLED; + return 0; + +out_free: + while (i-- > 0) + cxgb4_matchall_del_filter(dev, i); + + cxgb4_matchall_mirror_free(dev); + return ret; +} + +static int cxgb4_matchall_free_filter(struct net_device *dev) +{ + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + int ret; + u8 i; + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + + for (i = 0; i < CXGB4_FILTER_TYPE_MAX; i++) { + ret = cxgb4_matchall_del_filter(dev, i); + if (ret) + return ret; + } + + cxgb4_matchall_mirror_free(dev); + + tc_port_matchall->ingress.packets = 0; + tc_port_matchall->ingress.bytes = 0; + tc_port_matchall->ingress.last_used = 0; + tc_port_matchall->ingress.state = CXGB4_MATCHALL_STATE_DISABLED; + return 0; +} + +int cxgb4_tc_matchall_replace(struct net_device *dev, + struct tc_cls_matchall_offload *cls_matchall, + bool ingress) +{ + struct netlink_ext_ack *extack = cls_matchall->common.extack; + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + int ret; + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + if (ingress) { + if (tc_port_matchall->ingress.state == + CXGB4_MATCHALL_STATE_ENABLED) { + NL_SET_ERR_MSG_MOD(extack, + "Only 1 Ingress MATCHALL can be offloaded"); + return -ENOMEM; + } + + ret = cxgb4_validate_flow_actions(dev, + &cls_matchall->rule->action, + extack, 1); + if (ret) + return ret; + + return cxgb4_matchall_alloc_filter(dev, cls_matchall); + } + + if (tc_port_matchall->egress.state == CXGB4_MATCHALL_STATE_ENABLED) { + NL_SET_ERR_MSG_MOD(extack, + "Only 1 Egress MATCHALL can be offloaded"); + return -ENOMEM; + } + + ret = cxgb4_matchall_egress_validate(dev, cls_matchall); + if (ret) + return ret; + + return cxgb4_matchall_alloc_tc(dev, cls_matchall); +} + +int cxgb4_tc_matchall_destroy(struct net_device *dev, + struct tc_cls_matchall_offload *cls_matchall, + bool ingress) +{ + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + if (ingress) { + /* All the filter types of this matchall rule save the + * same cookie. So, checking for the first one is + * enough. + */ + if (cls_matchall->cookie != + tc_port_matchall->ingress.fs[0].tc_cookie) + return -ENOENT; + + return cxgb4_matchall_free_filter(dev); + } + + if (cls_matchall->cookie != tc_port_matchall->egress.cookie) + return -ENOENT; + + cxgb4_matchall_free_tc(dev); + return 0; +} + +int cxgb4_tc_matchall_stats(struct net_device *dev, + struct tc_cls_matchall_offload *cls_matchall) +{ + u64 tmp_packets, tmp_bytes, packets = 0, bytes = 0; + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct cxgb4_matchall_ingress_entry *ingress; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + int ret; + u8 i; + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + if (tc_port_matchall->ingress.state == CXGB4_MATCHALL_STATE_DISABLED) + return -ENOENT; + + ingress = &tc_port_matchall->ingress; + for (i = 0; i < CXGB4_FILTER_TYPE_MAX; i++) { + ret = cxgb4_get_filter_counters(dev, ingress->tid[i], + &tmp_packets, &tmp_bytes, + ingress->fs[i].hash); + if (ret) + return ret; + + packets += tmp_packets; + bytes += tmp_bytes; + } + + if (tc_port_matchall->ingress.packets != packets) { + flow_stats_update(&cls_matchall->stats, + bytes - tc_port_matchall->ingress.bytes, + packets - tc_port_matchall->ingress.packets, + 0, tc_port_matchall->ingress.last_used, + FLOW_ACTION_HW_STATS_IMMEDIATE); + + tc_port_matchall->ingress.packets = packets; + tc_port_matchall->ingress.bytes = bytes; + tc_port_matchall->ingress.last_used = jiffies; + } + + return 0; +} + +static void cxgb4_matchall_disable_offload(struct net_device *dev) +{ + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + + tc_port_matchall = &adap->tc_matchall->port_matchall[pi->port_id]; + if (tc_port_matchall->egress.state == CXGB4_MATCHALL_STATE_ENABLED) + cxgb4_matchall_free_tc(dev); + + if (tc_port_matchall->ingress.state == CXGB4_MATCHALL_STATE_ENABLED) + cxgb4_matchall_free_filter(dev); +} + +int cxgb4_init_tc_matchall(struct adapter *adap) +{ + struct cxgb4_tc_port_matchall *tc_port_matchall; + struct cxgb4_tc_matchall *tc_matchall; + int ret; + + tc_matchall = kzalloc(sizeof(*tc_matchall), GFP_KERNEL); + if (!tc_matchall) + return -ENOMEM; + + tc_port_matchall = kcalloc(adap->params.nports, + sizeof(*tc_port_matchall), + GFP_KERNEL); + if (!tc_port_matchall) { + ret = -ENOMEM; + goto out_free_matchall; + } + + tc_matchall->port_matchall = tc_port_matchall; + adap->tc_matchall = tc_matchall; + return 0; + +out_free_matchall: + kfree(tc_matchall); + return ret; +} + +void cxgb4_cleanup_tc_matchall(struct adapter *adap) +{ + u8 i; + + if (adap->tc_matchall) { + if (adap->tc_matchall->port_matchall) { + for (i = 0; i < adap->params.nports; i++) { + struct net_device *dev = adap->port[i]; + + if (dev) + cxgb4_matchall_disable_offload(dev); + } + kfree(adap->tc_matchall->port_matchall); + } + kfree(adap->tc_matchall); + } +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_matchall.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_matchall.h new file mode 100644 index 000000000..fe7ec423a --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_matchall.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2019 Chelsio Communications. All rights reserved. */ + +#ifndef __CXGB4_TC_MATCHALL_H__ +#define __CXGB4_TC_MATCHALL_H__ + +#include <net/pkt_cls.h> + +enum cxgb4_matchall_state { + CXGB4_MATCHALL_STATE_DISABLED = 0, + CXGB4_MATCHALL_STATE_ENABLED, +}; + +struct cxgb4_matchall_egress_entry { + enum cxgb4_matchall_state state; /* Current MATCHALL offload state */ + u8 hwtc; /* Traffic class bound to port */ + u64 cookie; /* Used to identify the MATCHALL rule offloaded */ +}; + +struct cxgb4_matchall_ingress_entry { + enum cxgb4_matchall_state state; /* Current MATCHALL offload state */ + u32 tid[CXGB4_FILTER_TYPE_MAX]; /* Index to hardware filter entries */ + /* Filter entries */ + struct ch_filter_specification fs[CXGB4_FILTER_TYPE_MAX]; + u16 viid_mirror; /* Identifier for allocated Mirror VI */ + u64 bytes; /* # of bytes hitting the filter */ + u64 packets; /* # of packets hitting the filter */ + u64 last_used; /* Last updated jiffies time */ +}; + +struct cxgb4_tc_port_matchall { + struct cxgb4_matchall_egress_entry egress; /* Egress offload info */ + struct cxgb4_matchall_ingress_entry ingress; /* Ingress offload info */ +}; + +struct cxgb4_tc_matchall { + struct cxgb4_tc_port_matchall *port_matchall; /* Per port entry */ +}; + +int cxgb4_tc_matchall_replace(struct net_device *dev, + struct tc_cls_matchall_offload *cls_matchall, + bool ingress); +int cxgb4_tc_matchall_destroy(struct net_device *dev, + struct tc_cls_matchall_offload *cls_matchall, + bool ingress); +int cxgb4_tc_matchall_stats(struct net_device *dev, + struct tc_cls_matchall_offload *cls_matchall); + +int cxgb4_init_tc_matchall(struct adapter *adap); +void cxgb4_cleanup_tc_matchall(struct adapter *adap); +#endif /* __CXGB4_TC_MATCHALL_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.c new file mode 100644 index 000000000..338b04f33 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.c @@ -0,0 +1,722 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2019 Chelsio Communications. All rights reserved. */ + +#include "cxgb4.h" +#include "cxgb4_tc_mqprio.h" +#include "sched.h" + +static int cxgb4_mqprio_validate(struct net_device *dev, + struct tc_mqprio_qopt_offload *mqprio) +{ + u64 min_rate = 0, max_rate = 0, max_link_rate; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + u32 speed, qcount = 0, qoffset = 0; + u32 start_a, start_b, end_a, end_b; + int ret; + u8 i, j; + + if (!mqprio->qopt.num_tc) + return 0; + + if (mqprio->qopt.hw != TC_MQPRIO_HW_OFFLOAD_TCS) { + netdev_err(dev, "Only full TC hardware offload is supported\n"); + return -EINVAL; + } else if (mqprio->mode != TC_MQPRIO_MODE_CHANNEL) { + netdev_err(dev, "Only channel mode offload is supported\n"); + return -EINVAL; + } else if (mqprio->shaper != TC_MQPRIO_SHAPER_BW_RATE) { + netdev_err(dev, "Only bandwidth rate shaper supported\n"); + return -EINVAL; + } else if (mqprio->qopt.num_tc > adap->params.nsched_cls) { + netdev_err(dev, + "Only %u traffic classes supported by hardware\n", + adap->params.nsched_cls); + return -ERANGE; + } + + ret = t4_get_link_params(pi, NULL, &speed, NULL); + if (ret) { + netdev_err(dev, "Failed to get link speed, ret: %d\n", ret); + return -EINVAL; + } + + /* Convert from Mbps to bps */ + max_link_rate = (u64)speed * 1000 * 1000; + + for (i = 0; i < mqprio->qopt.num_tc; i++) { + qoffset = max_t(u16, mqprio->qopt.offset[i], qoffset); + qcount += mqprio->qopt.count[i]; + + start_a = mqprio->qopt.offset[i]; + end_a = start_a + mqprio->qopt.count[i] - 1; + for (j = i + 1; j < mqprio->qopt.num_tc; j++) { + start_b = mqprio->qopt.offset[j]; + end_b = start_b + mqprio->qopt.count[j] - 1; + + /* If queue count is 0, then the traffic + * belonging to this class will not use + * ETHOFLD queues. So, no need to validate + * further. + */ + if (!mqprio->qopt.count[i]) + break; + + if (!mqprio->qopt.count[j]) + continue; + + if (max_t(u32, start_a, start_b) <= + min_t(u32, end_a, end_b)) { + netdev_err(dev, + "Queues can't overlap across tc\n"); + return -EINVAL; + } + } + + /* Convert byte per second to bits per second */ + min_rate += (mqprio->min_rate[i] * 8); + max_rate += (mqprio->max_rate[i] * 8); + } + + if (qoffset >= adap->tids.neotids || qcount > adap->tids.neotids) + return -ENOMEM; + + if (min_rate > max_link_rate || max_rate > max_link_rate) { + netdev_err(dev, + "Total Min/Max (%llu/%llu) Rate > supported (%llu)\n", + min_rate, max_rate, max_link_rate); + return -EINVAL; + } + + return 0; +} + +static int cxgb4_init_eosw_txq(struct net_device *dev, + struct sge_eosw_txq *eosw_txq, + u32 eotid, u32 hwqid) +{ + struct adapter *adap = netdev2adap(dev); + struct tx_sw_desc *ring; + + memset(eosw_txq, 0, sizeof(*eosw_txq)); + + ring = kcalloc(CXGB4_EOSW_TXQ_DEFAULT_DESC_NUM, + sizeof(*ring), GFP_KERNEL); + if (!ring) + return -ENOMEM; + + eosw_txq->desc = ring; + eosw_txq->ndesc = CXGB4_EOSW_TXQ_DEFAULT_DESC_NUM; + spin_lock_init(&eosw_txq->lock); + eosw_txq->state = CXGB4_EO_STATE_CLOSED; + eosw_txq->eotid = eotid; + eosw_txq->hwtid = adap->tids.eotid_base + eosw_txq->eotid; + eosw_txq->cred = adap->params.ofldq_wr_cred; + eosw_txq->hwqid = hwqid; + eosw_txq->netdev = dev; + tasklet_setup(&eosw_txq->qresume_tsk, cxgb4_ethofld_restart); + return 0; +} + +static void cxgb4_clean_eosw_txq(struct net_device *dev, + struct sge_eosw_txq *eosw_txq) +{ + struct adapter *adap = netdev2adap(dev); + + cxgb4_eosw_txq_free_desc(adap, eosw_txq, eosw_txq->ndesc); + eosw_txq->pidx = 0; + eosw_txq->last_pidx = 0; + eosw_txq->cidx = 0; + eosw_txq->last_cidx = 0; + eosw_txq->flowc_idx = 0; + eosw_txq->inuse = 0; + eosw_txq->cred = adap->params.ofldq_wr_cred; + eosw_txq->ncompl = 0; + eosw_txq->last_compl = 0; + eosw_txq->state = CXGB4_EO_STATE_CLOSED; +} + +static void cxgb4_free_eosw_txq(struct net_device *dev, + struct sge_eosw_txq *eosw_txq) +{ + spin_lock_bh(&eosw_txq->lock); + cxgb4_clean_eosw_txq(dev, eosw_txq); + kfree(eosw_txq->desc); + spin_unlock_bh(&eosw_txq->lock); + tasklet_kill(&eosw_txq->qresume_tsk); +} + +static int cxgb4_mqprio_alloc_hw_resources(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + struct sge_ofld_rxq *eorxq; + struct sge_eohw_txq *eotxq; + int ret, msix = 0; + u32 i; + + /* Allocate ETHOFLD hardware queue structures if not done already */ + if (!refcount_read(&adap->tc_mqprio->refcnt)) { + adap->sge.eohw_rxq = kcalloc(adap->sge.eoqsets, + sizeof(struct sge_ofld_rxq), + GFP_KERNEL); + if (!adap->sge.eohw_rxq) + return -ENOMEM; + + adap->sge.eohw_txq = kcalloc(adap->sge.eoqsets, + sizeof(struct sge_eohw_txq), + GFP_KERNEL); + if (!adap->sge.eohw_txq) { + kfree(adap->sge.eohw_rxq); + return -ENOMEM; + } + + refcount_set(&adap->tc_mqprio->refcnt, 1); + } else { + refcount_inc(&adap->tc_mqprio->refcnt); + } + + if (!(adap->flags & CXGB4_USING_MSIX)) + msix = -((int)adap->sge.intrq.abs_id + 1); + + for (i = 0; i < pi->nqsets; i++) { + eorxq = &adap->sge.eohw_rxq[pi->first_qset + i]; + eotxq = &adap->sge.eohw_txq[pi->first_qset + i]; + + /* Allocate Rxqs for receiving ETHOFLD Tx completions */ + if (msix >= 0) { + msix = cxgb4_get_msix_idx_from_bmap(adap); + if (msix < 0) { + ret = msix; + goto out_free_queues; + } + + eorxq->msix = &adap->msix_info[msix]; + snprintf(eorxq->msix->desc, + sizeof(eorxq->msix->desc), + "%s-eorxq%d", dev->name, i); + } + + init_rspq(adap, &eorxq->rspq, + CXGB4_EOHW_RXQ_DEFAULT_INTR_USEC, + CXGB4_EOHW_RXQ_DEFAULT_PKT_CNT, + CXGB4_EOHW_RXQ_DEFAULT_DESC_NUM, + CXGB4_EOHW_RXQ_DEFAULT_DESC_SIZE); + + eorxq->fl.size = CXGB4_EOHW_FLQ_DEFAULT_DESC_NUM; + + ret = t4_sge_alloc_rxq(adap, &eorxq->rspq, false, + dev, msix, &eorxq->fl, + cxgb4_ethofld_rx_handler, + NULL, 0); + if (ret) + goto out_free_queues; + + /* Allocate ETHOFLD hardware Txqs */ + eotxq->q.size = CXGB4_EOHW_TXQ_DEFAULT_DESC_NUM; + ret = t4_sge_alloc_ethofld_txq(adap, eotxq, dev, + eorxq->rspq.cntxt_id); + if (ret) + goto out_free_queues; + + /* Allocate IRQs, set IRQ affinity, and start Rx */ + if (adap->flags & CXGB4_USING_MSIX) { + ret = request_irq(eorxq->msix->vec, t4_sge_intr_msix, 0, + eorxq->msix->desc, &eorxq->rspq); + if (ret) + goto out_free_msix; + + cxgb4_set_msix_aff(adap, eorxq->msix->vec, + &eorxq->msix->aff_mask, i); + } + + if (adap->flags & CXGB4_FULL_INIT_DONE) + cxgb4_enable_rx(adap, &eorxq->rspq); + } + + return 0; + +out_free_msix: + while (i-- > 0) { + eorxq = &adap->sge.eohw_rxq[pi->first_qset + i]; + + if (adap->flags & CXGB4_FULL_INIT_DONE) + cxgb4_quiesce_rx(&eorxq->rspq); + + if (adap->flags & CXGB4_USING_MSIX) { + cxgb4_clear_msix_aff(eorxq->msix->vec, + eorxq->msix->aff_mask); + free_irq(eorxq->msix->vec, &eorxq->rspq); + } + } + +out_free_queues: + for (i = 0; i < pi->nqsets; i++) { + eorxq = &adap->sge.eohw_rxq[pi->first_qset + i]; + eotxq = &adap->sge.eohw_txq[pi->first_qset + i]; + + if (eorxq->rspq.desc) + free_rspq_fl(adap, &eorxq->rspq, &eorxq->fl); + if (eorxq->msix) + cxgb4_free_msix_idx_in_bmap(adap, eorxq->msix->idx); + t4_sge_free_ethofld_txq(adap, eotxq); + } + + if (refcount_dec_and_test(&adap->tc_mqprio->refcnt)) { + kfree(adap->sge.eohw_txq); + kfree(adap->sge.eohw_rxq); + } + return ret; +} + +static void cxgb4_mqprio_free_hw_resources(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + struct sge_ofld_rxq *eorxq; + struct sge_eohw_txq *eotxq; + u32 i; + + /* Return if no ETHOFLD structures have been allocated yet */ + if (!refcount_read(&adap->tc_mqprio->refcnt)) + return; + + /* Return if no hardware queues have been allocated */ + if (!adap->sge.eohw_rxq[pi->first_qset].rspq.desc) + return; + + for (i = 0; i < pi->nqsets; i++) { + eorxq = &adap->sge.eohw_rxq[pi->first_qset + i]; + eotxq = &adap->sge.eohw_txq[pi->first_qset + i]; + + /* Device removal path will already disable NAPI + * before unregistering netdevice. So, only disable + * NAPI if we're not in device removal path + */ + if (!(adap->flags & CXGB4_SHUTTING_DOWN)) + cxgb4_quiesce_rx(&eorxq->rspq); + + if (adap->flags & CXGB4_USING_MSIX) { + cxgb4_clear_msix_aff(eorxq->msix->vec, + eorxq->msix->aff_mask); + free_irq(eorxq->msix->vec, &eorxq->rspq); + cxgb4_free_msix_idx_in_bmap(adap, eorxq->msix->idx); + } + + free_rspq_fl(adap, &eorxq->rspq, &eorxq->fl); + t4_sge_free_ethofld_txq(adap, eotxq); + } + + /* Free up ETHOFLD structures if there are no users */ + if (refcount_dec_and_test(&adap->tc_mqprio->refcnt)) { + kfree(adap->sge.eohw_txq); + kfree(adap->sge.eohw_rxq); + } +} + +static int cxgb4_mqprio_alloc_tc(struct net_device *dev, + struct tc_mqprio_qopt_offload *mqprio) +{ + struct ch_sched_params p = { + .type = SCHED_CLASS_TYPE_PACKET, + .u.params.level = SCHED_CLASS_LEVEL_CL_RL, + .u.params.mode = SCHED_CLASS_MODE_FLOW, + .u.params.rateunit = SCHED_CLASS_RATEUNIT_BITS, + .u.params.ratemode = SCHED_CLASS_RATEMODE_ABS, + .u.params.class = SCHED_CLS_NONE, + .u.params.weight = 0, + .u.params.pktsize = dev->mtu, + }; + struct cxgb4_tc_port_mqprio *tc_port_mqprio; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + struct sched_class *e; + int ret; + u8 i; + + tc_port_mqprio = &adap->tc_mqprio->port_mqprio[pi->port_id]; + p.u.params.channel = pi->tx_chan; + for (i = 0; i < mqprio->qopt.num_tc; i++) { + /* Convert from bytes per second to Kbps */ + p.u.params.minrate = div_u64(mqprio->min_rate[i] * 8, 1000); + p.u.params.maxrate = div_u64(mqprio->max_rate[i] * 8, 1000); + + /* Request larger burst buffer for smaller MTU, so + * that hardware can work on more data per burst + * cycle. + */ + if (dev->mtu <= ETH_DATA_LEN) + p.u.params.burstsize = 8 * dev->mtu; + + e = cxgb4_sched_class_alloc(dev, &p); + if (!e) { + ret = -ENOMEM; + goto out_err; + } + + tc_port_mqprio->tc_hwtc_map[i] = e->idx; + } + + return 0; + +out_err: + while (i--) + cxgb4_sched_class_free(dev, tc_port_mqprio->tc_hwtc_map[i]); + + return ret; +} + +static void cxgb4_mqprio_free_tc(struct net_device *dev) +{ + struct cxgb4_tc_port_mqprio *tc_port_mqprio; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + u8 i; + + tc_port_mqprio = &adap->tc_mqprio->port_mqprio[pi->port_id]; + for (i = 0; i < tc_port_mqprio->mqprio.qopt.num_tc; i++) + cxgb4_sched_class_free(dev, tc_port_mqprio->tc_hwtc_map[i]); +} + +static int cxgb4_mqprio_class_bind(struct net_device *dev, + struct sge_eosw_txq *eosw_txq, + u8 tc) +{ + struct ch_sched_flowc fe; + int ret; + + init_completion(&eosw_txq->completion); + + fe.tid = eosw_txq->eotid; + fe.class = tc; + + ret = cxgb4_sched_class_bind(dev, &fe, SCHED_FLOWC); + if (ret) + return ret; + + ret = wait_for_completion_timeout(&eosw_txq->completion, + CXGB4_FLOWC_WAIT_TIMEOUT); + if (!ret) + return -ETIMEDOUT; + + return 0; +} + +static void cxgb4_mqprio_class_unbind(struct net_device *dev, + struct sge_eosw_txq *eosw_txq, + u8 tc) +{ + struct adapter *adap = netdev2adap(dev); + struct ch_sched_flowc fe; + + /* If we're shutting down, interrupts are disabled and no completions + * come back. So, skip waiting for completions in this scenario. + */ + if (!(adap->flags & CXGB4_SHUTTING_DOWN)) + init_completion(&eosw_txq->completion); + + fe.tid = eosw_txq->eotid; + fe.class = tc; + cxgb4_sched_class_unbind(dev, &fe, SCHED_FLOWC); + + if (!(adap->flags & CXGB4_SHUTTING_DOWN)) + wait_for_completion_timeout(&eosw_txq->completion, + CXGB4_FLOWC_WAIT_TIMEOUT); +} + +static int cxgb4_mqprio_enable_offload(struct net_device *dev, + struct tc_mqprio_qopt_offload *mqprio) +{ + struct cxgb4_tc_port_mqprio *tc_port_mqprio; + u32 qoffset, qcount, tot_qcount, qid, hwqid; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + struct sge_eosw_txq *eosw_txq; + int eotid, ret; + u16 i, j; + u8 hwtc; + + ret = cxgb4_mqprio_alloc_hw_resources(dev); + if (ret) + return -ENOMEM; + + tc_port_mqprio = &adap->tc_mqprio->port_mqprio[pi->port_id]; + for (i = 0; i < mqprio->qopt.num_tc; i++) { + qoffset = mqprio->qopt.offset[i]; + qcount = mqprio->qopt.count[i]; + for (j = 0; j < qcount; j++) { + eotid = cxgb4_get_free_eotid(&adap->tids); + if (eotid < 0) { + ret = -ENOMEM; + goto out_free_eotids; + } + + qid = qoffset + j; + hwqid = pi->first_qset + (eotid % pi->nqsets); + eosw_txq = &tc_port_mqprio->eosw_txq[qid]; + ret = cxgb4_init_eosw_txq(dev, eosw_txq, + eotid, hwqid); + if (ret) + goto out_free_eotids; + + cxgb4_alloc_eotid(&adap->tids, eotid, eosw_txq); + + hwtc = tc_port_mqprio->tc_hwtc_map[i]; + ret = cxgb4_mqprio_class_bind(dev, eosw_txq, hwtc); + if (ret) + goto out_free_eotids; + } + } + + memcpy(&tc_port_mqprio->mqprio, mqprio, + sizeof(struct tc_mqprio_qopt_offload)); + + /* Inform the stack about the configured tc params. + * + * Set the correct queue map. If no queue count has been + * specified, then send the traffic through default NIC + * queues; instead of ETHOFLD queues. + */ + ret = netdev_set_num_tc(dev, mqprio->qopt.num_tc); + if (ret) + goto out_free_eotids; + + tot_qcount = pi->nqsets; + for (i = 0; i < mqprio->qopt.num_tc; i++) { + qcount = mqprio->qopt.count[i]; + if (qcount) { + qoffset = mqprio->qopt.offset[i] + pi->nqsets; + } else { + qcount = pi->nqsets; + qoffset = 0; + } + + ret = netdev_set_tc_queue(dev, i, qcount, qoffset); + if (ret) + goto out_reset_tc; + + tot_qcount += mqprio->qopt.count[i]; + } + + ret = netif_set_real_num_tx_queues(dev, tot_qcount); + if (ret) + goto out_reset_tc; + + tc_port_mqprio->state = CXGB4_MQPRIO_STATE_ACTIVE; + return 0; + +out_reset_tc: + netdev_reset_tc(dev); + i = mqprio->qopt.num_tc; + +out_free_eotids: + while (i-- > 0) { + qoffset = mqprio->qopt.offset[i]; + qcount = mqprio->qopt.count[i]; + for (j = 0; j < qcount; j++) { + eosw_txq = &tc_port_mqprio->eosw_txq[qoffset + j]; + + hwtc = tc_port_mqprio->tc_hwtc_map[i]; + cxgb4_mqprio_class_unbind(dev, eosw_txq, hwtc); + + cxgb4_free_eotid(&adap->tids, eosw_txq->eotid); + cxgb4_free_eosw_txq(dev, eosw_txq); + } + } + + cxgb4_mqprio_free_hw_resources(dev); + return ret; +} + +static void cxgb4_mqprio_disable_offload(struct net_device *dev) +{ + struct cxgb4_tc_port_mqprio *tc_port_mqprio; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + struct sge_eosw_txq *eosw_txq; + u32 qoffset, qcount; + u16 i, j; + u8 hwtc; + + tc_port_mqprio = &adap->tc_mqprio->port_mqprio[pi->port_id]; + if (tc_port_mqprio->state != CXGB4_MQPRIO_STATE_ACTIVE) + return; + + netdev_reset_tc(dev); + netif_set_real_num_tx_queues(dev, pi->nqsets); + + for (i = 0; i < tc_port_mqprio->mqprio.qopt.num_tc; i++) { + qoffset = tc_port_mqprio->mqprio.qopt.offset[i]; + qcount = tc_port_mqprio->mqprio.qopt.count[i]; + for (j = 0; j < qcount; j++) { + eosw_txq = &tc_port_mqprio->eosw_txq[qoffset + j]; + + hwtc = tc_port_mqprio->tc_hwtc_map[i]; + cxgb4_mqprio_class_unbind(dev, eosw_txq, hwtc); + + cxgb4_free_eotid(&adap->tids, eosw_txq->eotid); + cxgb4_free_eosw_txq(dev, eosw_txq); + } + } + + cxgb4_mqprio_free_hw_resources(dev); + + /* Free up the traffic classes */ + cxgb4_mqprio_free_tc(dev); + + memset(&tc_port_mqprio->mqprio, 0, + sizeof(struct tc_mqprio_qopt_offload)); + + tc_port_mqprio->state = CXGB4_MQPRIO_STATE_DISABLED; +} + +int cxgb4_setup_tc_mqprio(struct net_device *dev, + struct tc_mqprio_qopt_offload *mqprio) +{ + struct adapter *adap = netdev2adap(dev); + bool needs_bring_up = false; + int ret; + + ret = cxgb4_mqprio_validate(dev, mqprio); + if (ret) + return ret; + + mutex_lock(&adap->tc_mqprio->mqprio_mutex); + + /* To configure tc params, the current allocated EOTIDs must + * be freed up. However, they can't be freed up if there's + * traffic running on the interface. So, ensure interface is + * down before configuring tc params. + */ + if (netif_running(dev)) { + netif_tx_stop_all_queues(dev); + netif_carrier_off(dev); + needs_bring_up = true; + } + + cxgb4_mqprio_disable_offload(dev); + + /* If requested for clear, then just return since resources are + * already freed up by now. + */ + if (!mqprio->qopt.num_tc) + goto out; + + /* Allocate free available traffic classes and configure + * their rate parameters. + */ + ret = cxgb4_mqprio_alloc_tc(dev, mqprio); + if (ret) + goto out; + + ret = cxgb4_mqprio_enable_offload(dev, mqprio); + if (ret) { + cxgb4_mqprio_free_tc(dev); + goto out; + } + +out: + if (needs_bring_up) { + netif_tx_start_all_queues(dev); + netif_carrier_on(dev); + } + + mutex_unlock(&adap->tc_mqprio->mqprio_mutex); + return ret; +} + +void cxgb4_mqprio_stop_offload(struct adapter *adap) +{ + struct cxgb4_tc_port_mqprio *tc_port_mqprio; + struct net_device *dev; + u8 i; + + if (!adap->tc_mqprio || !adap->tc_mqprio->port_mqprio) + return; + + mutex_lock(&adap->tc_mqprio->mqprio_mutex); + for_each_port(adap, i) { + dev = adap->port[i]; + if (!dev) + continue; + + tc_port_mqprio = &adap->tc_mqprio->port_mqprio[i]; + if (!tc_port_mqprio->mqprio.qopt.num_tc) + continue; + + cxgb4_mqprio_disable_offload(dev); + } + mutex_unlock(&adap->tc_mqprio->mqprio_mutex); +} + +int cxgb4_init_tc_mqprio(struct adapter *adap) +{ + struct cxgb4_tc_port_mqprio *tc_port_mqprio, *port_mqprio; + struct cxgb4_tc_mqprio *tc_mqprio; + struct sge_eosw_txq *eosw_txq; + int ret = 0; + u8 i; + + tc_mqprio = kzalloc(sizeof(*tc_mqprio), GFP_KERNEL); + if (!tc_mqprio) + return -ENOMEM; + + tc_port_mqprio = kcalloc(adap->params.nports, sizeof(*tc_port_mqprio), + GFP_KERNEL); + if (!tc_port_mqprio) { + ret = -ENOMEM; + goto out_free_mqprio; + } + + mutex_init(&tc_mqprio->mqprio_mutex); + + tc_mqprio->port_mqprio = tc_port_mqprio; + for (i = 0; i < adap->params.nports; i++) { + port_mqprio = &tc_mqprio->port_mqprio[i]; + eosw_txq = kcalloc(adap->tids.neotids, sizeof(*eosw_txq), + GFP_KERNEL); + if (!eosw_txq) { + ret = -ENOMEM; + goto out_free_ports; + } + port_mqprio->eosw_txq = eosw_txq; + } + + adap->tc_mqprio = tc_mqprio; + refcount_set(&adap->tc_mqprio->refcnt, 0); + return 0; + +out_free_ports: + for (i = 0; i < adap->params.nports; i++) { + port_mqprio = &tc_mqprio->port_mqprio[i]; + kfree(port_mqprio->eosw_txq); + } + kfree(tc_port_mqprio); + +out_free_mqprio: + kfree(tc_mqprio); + return ret; +} + +void cxgb4_cleanup_tc_mqprio(struct adapter *adap) +{ + struct cxgb4_tc_port_mqprio *port_mqprio; + u8 i; + + if (adap->tc_mqprio) { + mutex_lock(&adap->tc_mqprio->mqprio_mutex); + if (adap->tc_mqprio->port_mqprio) { + for (i = 0; i < adap->params.nports; i++) { + struct net_device *dev = adap->port[i]; + + if (dev) + cxgb4_mqprio_disable_offload(dev); + port_mqprio = &adap->tc_mqprio->port_mqprio[i]; + kfree(port_mqprio->eosw_txq); + } + kfree(adap->tc_mqprio->port_mqprio); + } + mutex_unlock(&adap->tc_mqprio->mqprio_mutex); + kfree(adap->tc_mqprio); + } +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.h new file mode 100644 index 000000000..be96f1dc0 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2019 Chelsio Communications. All rights reserved. */ + +#ifndef __CXGB4_TC_MQPRIO_H__ +#define __CXGB4_TC_MQPRIO_H__ + +#include <net/pkt_cls.h> + +#define CXGB4_EOSW_TXQ_DEFAULT_DESC_NUM 128 + +#define CXGB4_EOHW_TXQ_DEFAULT_DESC_NUM 1024 + +#define CXGB4_EOHW_RXQ_DEFAULT_DESC_NUM 1024 +#define CXGB4_EOHW_RXQ_DEFAULT_DESC_SIZE 64 +#define CXGB4_EOHW_RXQ_DEFAULT_INTR_USEC 5 +#define CXGB4_EOHW_RXQ_DEFAULT_PKT_CNT 8 + +#define CXGB4_EOHW_FLQ_DEFAULT_DESC_NUM 72 + +#define CXGB4_FLOWC_WAIT_TIMEOUT (5 * HZ) + +enum cxgb4_mqprio_state { + CXGB4_MQPRIO_STATE_DISABLED = 0, + CXGB4_MQPRIO_STATE_ACTIVE, +}; + +struct cxgb4_tc_port_mqprio { + enum cxgb4_mqprio_state state; /* Current MQPRIO offload state */ + struct tc_mqprio_qopt_offload mqprio; /* MQPRIO offload params */ + struct sge_eosw_txq *eosw_txq; /* Netdev SW Tx queue array */ + u8 tc_hwtc_map[TC_QOPT_MAX_QUEUE]; /* MQPRIO tc to hardware tc map */ +}; + +struct cxgb4_tc_mqprio { + refcount_t refcnt; /* Refcount for adapter-wide resources */ + struct mutex mqprio_mutex; /* Lock for accessing MQPRIO info */ + struct cxgb4_tc_port_mqprio *port_mqprio; /* Per port MQPRIO info */ +}; + +int cxgb4_setup_tc_mqprio(struct net_device *dev, + struct tc_mqprio_qopt_offload *mqprio); +void cxgb4_mqprio_stop_offload(struct adapter *adap); +int cxgb4_init_tc_mqprio(struct adapter *adap); +void cxgb4_cleanup_tc_mqprio(struct adapter *adap); +#endif /* __CXGB4_TC_MQPRIO_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c new file mode 100644 index 000000000..a5d2f84dc --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c @@ -0,0 +1,532 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <net/tc_act/tc_gact.h> +#include <net/tc_act/tc_mirred.h> + +#include "cxgb4.h" +#include "cxgb4_filter.h" +#include "cxgb4_tc_u32_parse.h" +#include "cxgb4_tc_u32.h" + +/* Fill ch_filter_specification with parsed match value/mask pair. */ +static int fill_match_fields(struct adapter *adap, + struct ch_filter_specification *fs, + struct tc_cls_u32_offload *cls, + const struct cxgb4_match_field *entry, + bool next_header) +{ + unsigned int i, j; + __be32 val, mask; + int off, err; + bool found; + + for (i = 0; i < cls->knode.sel->nkeys; i++) { + off = cls->knode.sel->keys[i].off; + val = cls->knode.sel->keys[i].val; + mask = cls->knode.sel->keys[i].mask; + + if (next_header) { + /* For next headers, parse only keys with offmask */ + if (!cls->knode.sel->keys[i].offmask) + continue; + } else { + /* For the remaining, parse only keys without offmask */ + if (cls->knode.sel->keys[i].offmask) + continue; + } + + found = false; + + for (j = 0; entry[j].val; j++) { + if (off == entry[j].off) { + found = true; + err = entry[j].val(fs, val, mask); + if (err) + return err; + break; + } + } + + if (!found) + return -EINVAL; + } + + return 0; +} + +/* Fill ch_filter_specification with parsed action. */ +static int fill_action_fields(struct adapter *adap, + struct ch_filter_specification *fs, + struct tc_cls_u32_offload *cls) +{ + unsigned int num_actions = 0; + const struct tc_action *a; + struct tcf_exts *exts; + int i; + + exts = cls->knode.exts; + if (!tcf_exts_has_actions(exts)) + return -EINVAL; + + tcf_exts_for_each_action(i, a, exts) { + /* Don't allow more than one action per rule. */ + if (num_actions) + return -EINVAL; + + /* Drop in hardware. */ + if (is_tcf_gact_shot(a)) { + fs->action = FILTER_DROP; + num_actions++; + continue; + } + + /* Re-direct to specified port in hardware. */ + if (is_tcf_mirred_egress_redirect(a)) { + struct net_device *n_dev, *target_dev; + bool found = false; + unsigned int i; + + target_dev = tcf_mirred_dev(a); + for_each_port(adap, i) { + n_dev = adap->port[i]; + if (target_dev == n_dev) { + fs->action = FILTER_SWITCH; + fs->eport = i; + found = true; + break; + } + } + + /* Interface doesn't belong to any port of + * the underlying hardware. + */ + if (!found) + return -EINVAL; + + num_actions++; + continue; + } + + /* Un-supported action. */ + return -EINVAL; + } + + return 0; +} + +int cxgb4_config_knode(struct net_device *dev, struct tc_cls_u32_offload *cls) +{ + const struct cxgb4_match_field *start, *link_start = NULL; + struct netlink_ext_ack *extack = cls->common.extack; + struct adapter *adapter = netdev2adap(dev); + __be16 protocol = cls->common.protocol; + struct ch_filter_specification fs; + struct cxgb4_tc_u32_table *t; + struct cxgb4_link *link; + u32 uhtid, link_uhtid; + bool is_ipv6 = false; + u8 inet_family; + int filter_id; + int ret; + + if (!can_tc_u32_offload(dev)) + return -EOPNOTSUPP; + + if (protocol != htons(ETH_P_IP) && protocol != htons(ETH_P_IPV6)) + return -EOPNOTSUPP; + + inet_family = (protocol == htons(ETH_P_IPV6)) ? PF_INET6 : PF_INET; + + /* Get a free filter entry TID, where we can insert this new + * rule. Only insert rule if its prio doesn't conflict with + * existing rules. + */ + filter_id = cxgb4_get_free_ftid(dev, inet_family, false, + TC_U32_NODE(cls->knode.handle)); + if (filter_id < 0) { + NL_SET_ERR_MSG_MOD(extack, + "No free LETCAM index available"); + return -ENOMEM; + } + + t = adapter->tc_u32; + uhtid = TC_U32_USERHTID(cls->knode.handle); + link_uhtid = TC_U32_USERHTID(cls->knode.link_handle); + + /* Ensure that uhtid is either root u32 (i.e. 0x800) + * or a a valid linked bucket. + */ + if (uhtid != 0x800 && uhtid >= t->size) + return -EINVAL; + + /* Ensure link handle uhtid is sane, if specified. */ + if (link_uhtid >= t->size) + return -EINVAL; + + memset(&fs, 0, sizeof(fs)); + + if (filter_id < adapter->tids.nhpftids) + fs.prio = 1; + fs.tc_prio = cls->common.prio; + fs.tc_cookie = cls->knode.handle; + + if (protocol == htons(ETH_P_IPV6)) { + start = cxgb4_ipv6_fields; + is_ipv6 = true; + } else { + start = cxgb4_ipv4_fields; + is_ipv6 = false; + } + + if (uhtid != 0x800) { + /* Link must exist from root node before insertion. */ + if (!t->table[uhtid - 1].link_handle) + return -EINVAL; + + /* Link must have a valid supported next header. */ + link_start = t->table[uhtid - 1].match_field; + if (!link_start) + return -EINVAL; + } + + /* Parse links and record them for subsequent jumps to valid + * next headers. + */ + if (link_uhtid) { + const struct cxgb4_next_header *next; + bool found = false; + unsigned int i, j; + __be32 val, mask; + int off; + + if (t->table[link_uhtid - 1].link_handle) { + dev_err(adapter->pdev_dev, + "Link handle exists for: 0x%x\n", + link_uhtid); + return -EINVAL; + } + + next = is_ipv6 ? cxgb4_ipv6_jumps : cxgb4_ipv4_jumps; + + /* Try to find matches that allow jumps to next header. */ + for (i = 0; next[i].jump; i++) { + if (next[i].sel.offoff != cls->knode.sel->offoff || + next[i].sel.offshift != cls->knode.sel->offshift || + next[i].sel.offmask != cls->knode.sel->offmask || + next[i].sel.off != cls->knode.sel->off) + continue; + + /* Found a possible candidate. Find a key that + * matches the corresponding offset, value, and + * mask to jump to next header. + */ + for (j = 0; j < cls->knode.sel->nkeys; j++) { + off = cls->knode.sel->keys[j].off; + val = cls->knode.sel->keys[j].val; + mask = cls->knode.sel->keys[j].mask; + + if (next[i].key.off == off && + next[i].key.val == val && + next[i].key.mask == mask) { + found = true; + break; + } + } + + if (!found) + continue; /* Try next candidate. */ + + /* Candidate to jump to next header found. + * Translate all keys to internal specification + * and store them in jump table. This spec is copied + * later to set the actual filters. + */ + ret = fill_match_fields(adapter, &fs, cls, + start, false); + if (ret) + goto out; + + link = &t->table[link_uhtid - 1]; + link->match_field = next[i].jump; + link->link_handle = cls->knode.handle; + memcpy(&link->fs, &fs, sizeof(fs)); + break; + } + + /* No candidate found to jump to next header. */ + if (!found) + return -EINVAL; + + return 0; + } + + /* Fill ch_filter_specification match fields to be shipped to hardware. + * Copy the linked spec (if any) first. And then update the spec as + * needed. + */ + if (uhtid != 0x800 && t->table[uhtid - 1].link_handle) { + /* Copy linked ch_filter_specification */ + memcpy(&fs, &t->table[uhtid - 1].fs, sizeof(fs)); + ret = fill_match_fields(adapter, &fs, cls, + link_start, true); + if (ret) + goto out; + } + + ret = fill_match_fields(adapter, &fs, cls, start, false); + if (ret) + goto out; + + /* Fill ch_filter_specification action fields to be shipped to + * hardware. + */ + ret = fill_action_fields(adapter, &fs, cls); + if (ret) + goto out; + + /* The filter spec has been completely built from the info + * provided from u32. We now set some default fields in the + * spec for sanity. + */ + + /* Match only packets coming from the ingress port where this + * filter will be created. + */ + fs.val.iport = netdev2pinfo(dev)->port_id; + fs.mask.iport = ~0; + + /* Enable filter hit counts. */ + fs.hitcnts = 1; + + /* Set type of filter - IPv6 or IPv4 */ + fs.type = is_ipv6 ? 1 : 0; + + /* Set the filter */ + ret = cxgb4_set_filter(dev, filter_id, &fs); + if (ret) + goto out; + + /* If this is a linked bucket, then set the corresponding + * entry in the bitmap to mark it as belonging to this linked + * bucket. + */ + if (uhtid != 0x800 && t->table[uhtid - 1].link_handle) + set_bit(filter_id, t->table[uhtid - 1].tid_map); + +out: + return ret; +} + +int cxgb4_delete_knode(struct net_device *dev, struct tc_cls_u32_offload *cls) +{ + struct adapter *adapter = netdev2adap(dev); + unsigned int filter_id, max_tids, i, j; + struct cxgb4_link *link = NULL; + struct cxgb4_tc_u32_table *t; + struct filter_entry *f; + bool found = false; + u32 handle, uhtid; + u8 nslots; + int ret; + + if (!can_tc_u32_offload(dev)) + return -EOPNOTSUPP; + + /* Fetch the location to delete the filter. */ + max_tids = adapter->tids.nhpftids + adapter->tids.nftids; + + spin_lock_bh(&adapter->tids.ftid_lock); + filter_id = 0; + while (filter_id < max_tids) { + if (filter_id < adapter->tids.nhpftids) { + i = filter_id; + f = &adapter->tids.hpftid_tab[i]; + if (f->valid && f->fs.tc_cookie == cls->knode.handle) { + found = true; + break; + } + + i = find_next_bit(adapter->tids.hpftid_bmap, + adapter->tids.nhpftids, i + 1); + if (i >= adapter->tids.nhpftids) { + filter_id = adapter->tids.nhpftids; + continue; + } + + filter_id = i; + } else { + i = filter_id - adapter->tids.nhpftids; + f = &adapter->tids.ftid_tab[i]; + if (f->valid && f->fs.tc_cookie == cls->knode.handle) { + found = true; + break; + } + + i = find_next_bit(adapter->tids.ftid_bmap, + adapter->tids.nftids, i + 1); + if (i >= adapter->tids.nftids) + break; + + filter_id = i + adapter->tids.nhpftids; + } + + nslots = 0; + if (f->fs.type) { + nslots++; + if (CHELSIO_CHIP_VERSION(adapter->params.chip) < + CHELSIO_T6) + nslots += 2; + } + + filter_id += nslots; + } + spin_unlock_bh(&adapter->tids.ftid_lock); + + if (!found) + return -ERANGE; + + t = adapter->tc_u32; + handle = cls->knode.handle; + uhtid = TC_U32_USERHTID(cls->knode.handle); + + /* Ensure that uhtid is either root u32 (i.e. 0x800) + * or a a valid linked bucket. + */ + if (uhtid != 0x800 && uhtid >= t->size) + return -EINVAL; + + /* Delete the specified filter */ + if (uhtid != 0x800) { + link = &t->table[uhtid - 1]; + if (!link->link_handle) + return -EINVAL; + + if (!test_bit(filter_id, link->tid_map)) + return -EINVAL; + } + + ret = cxgb4_del_filter(dev, filter_id, NULL); + if (ret) + goto out; + + if (link) + clear_bit(filter_id, link->tid_map); + + /* If a link is being deleted, then delete all filters + * associated with the link. + */ + for (i = 0; i < t->size; i++) { + link = &t->table[i]; + + if (link->link_handle == handle) { + for (j = 0; j < max_tids; j++) { + if (!test_bit(j, link->tid_map)) + continue; + + ret = __cxgb4_del_filter(dev, j, NULL, NULL); + if (ret) + goto out; + + clear_bit(j, link->tid_map); + } + + /* Clear the link state */ + link->match_field = NULL; + link->link_handle = 0; + memset(&link->fs, 0, sizeof(link->fs)); + break; + } + } + +out: + return ret; +} + +void cxgb4_cleanup_tc_u32(struct adapter *adap) +{ + struct cxgb4_tc_u32_table *t; + unsigned int i; + + if (!adap->tc_u32) + return; + + /* Free up all allocated memory. */ + t = adap->tc_u32; + for (i = 0; i < t->size; i++) { + struct cxgb4_link *link = &t->table[i]; + + kvfree(link->tid_map); + } + kvfree(adap->tc_u32); +} + +struct cxgb4_tc_u32_table *cxgb4_init_tc_u32(struct adapter *adap) +{ + unsigned int max_tids = adap->tids.nftids + adap->tids.nhpftids; + struct cxgb4_tc_u32_table *t; + unsigned int i; + + if (!max_tids) + return NULL; + + t = kvzalloc(struct_size(t, table, max_tids), GFP_KERNEL); + if (!t) + return NULL; + + t->size = max_tids; + + for (i = 0; i < t->size; i++) { + struct cxgb4_link *link = &t->table[i]; + unsigned int bmap_size; + + bmap_size = BITS_TO_LONGS(max_tids); + link->tid_map = kvcalloc(bmap_size, sizeof(unsigned long), + GFP_KERNEL); + if (!link->tid_map) + goto out_no_mem; + bitmap_zero(link->tid_map, max_tids); + } + + return t; + +out_no_mem: + for (i = 0; i < t->size; i++) { + struct cxgb4_link *link = &t->table[i]; + kvfree(link->tid_map); + } + kvfree(t); + + return NULL; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.h new file mode 100644 index 000000000..70a07b7cc --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.h @@ -0,0 +1,52 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_TC_U32_H +#define __CXGB4_TC_U32_H + +#include <net/pkt_cls.h> + +static inline bool can_tc_u32_offload(struct net_device *dev) +{ + struct adapter *adap = netdev2adap(dev); + + return (dev->features & NETIF_F_HW_TC) && adap->tc_u32 ? true : false; +} + +int cxgb4_config_knode(struct net_device *dev, struct tc_cls_u32_offload *cls); +int cxgb4_delete_knode(struct net_device *dev, struct tc_cls_u32_offload *cls); + +void cxgb4_cleanup_tc_u32(struct adapter *adapter); +struct cxgb4_tc_u32_table *cxgb4_init_tc_u32(struct adapter *adap); +#endif /* __CXGB4_TC_U32_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32_parse.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32_parse.h new file mode 100644 index 000000000..f59dd4b2a --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32_parse.h @@ -0,0 +1,336 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_TC_U32_PARSE_H +#define __CXGB4_TC_U32_PARSE_H + +struct cxgb4_match_field { + int off; /* Offset from the beginning of the header to match */ + /* Fill the value/mask pair in the spec if matched */ + int (*val)(struct ch_filter_specification *f, __be32 val, __be32 mask); +}; + +/* IPv4 match fields */ +static inline int cxgb4_fill_ipv4_tos(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + f->val.tos = (ntohl(val) >> 16) & 0x000000FF; + f->mask.tos = (ntohl(mask) >> 16) & 0x000000FF; + + return 0; +} + +static inline int cxgb4_fill_ipv4_frag(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + u32 mask_val; + u8 frag_val; + + frag_val = (ntohl(val) >> 13) & 0x00000007; + mask_val = ntohl(mask) & 0x0000FFFF; + + if (frag_val == 0x1 && mask_val != 0x3FFF) { /* MF set */ + f->val.frag = 1; + f->mask.frag = 1; + } else if (frag_val == 0x2 && mask_val != 0x3FFF) { /* DF set */ + f->val.frag = 0; + f->mask.frag = 1; + } else { + return -EINVAL; + } + + return 0; +} + +static inline int cxgb4_fill_ipv4_proto(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + f->val.proto = (ntohl(val) >> 16) & 0x000000FF; + f->mask.proto = (ntohl(mask) >> 16) & 0x000000FF; + + return 0; +} + +static inline int cxgb4_fill_ipv4_src_ip(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + memcpy(&f->val.fip[0], &val, sizeof(u32)); + memcpy(&f->mask.fip[0], &mask, sizeof(u32)); + + return 0; +} + +static inline int cxgb4_fill_ipv4_dst_ip(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + memcpy(&f->val.lip[0], &val, sizeof(u32)); + memcpy(&f->mask.lip[0], &mask, sizeof(u32)); + + return 0; +} + +static const struct cxgb4_match_field cxgb4_ipv4_fields[] = { + { .off = 0, .val = cxgb4_fill_ipv4_tos }, + { .off = 4, .val = cxgb4_fill_ipv4_frag }, + { .off = 8, .val = cxgb4_fill_ipv4_proto }, + { .off = 12, .val = cxgb4_fill_ipv4_src_ip }, + { .off = 16, .val = cxgb4_fill_ipv4_dst_ip }, + { .val = NULL } +}; + +/* IPv6 match fields */ +static inline int cxgb4_fill_ipv6_tos(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + f->val.tos = (ntohl(val) >> 20) & 0x000000FF; + f->mask.tos = (ntohl(mask) >> 20) & 0x000000FF; + + return 0; +} + +static inline int cxgb4_fill_ipv6_proto(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + f->val.proto = (ntohl(val) >> 8) & 0x000000FF; + f->mask.proto = (ntohl(mask) >> 8) & 0x000000FF; + + return 0; +} + +static inline int cxgb4_fill_ipv6_src_ip0(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + memcpy(&f->val.fip[0], &val, sizeof(u32)); + memcpy(&f->mask.fip[0], &mask, sizeof(u32)); + + return 0; +} + +static inline int cxgb4_fill_ipv6_src_ip1(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + memcpy(&f->val.fip[4], &val, sizeof(u32)); + memcpy(&f->mask.fip[4], &mask, sizeof(u32)); + + return 0; +} + +static inline int cxgb4_fill_ipv6_src_ip2(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + memcpy(&f->val.fip[8], &val, sizeof(u32)); + memcpy(&f->mask.fip[8], &mask, sizeof(u32)); + + return 0; +} + +static inline int cxgb4_fill_ipv6_src_ip3(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + memcpy(&f->val.fip[12], &val, sizeof(u32)); + memcpy(&f->mask.fip[12], &mask, sizeof(u32)); + + return 0; +} + +static inline int cxgb4_fill_ipv6_dst_ip0(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + memcpy(&f->val.lip[0], &val, sizeof(u32)); + memcpy(&f->mask.lip[0], &mask, sizeof(u32)); + + return 0; +} + +static inline int cxgb4_fill_ipv6_dst_ip1(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + memcpy(&f->val.lip[4], &val, sizeof(u32)); + memcpy(&f->mask.lip[4], &mask, sizeof(u32)); + + return 0; +} + +static inline int cxgb4_fill_ipv6_dst_ip2(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + memcpy(&f->val.lip[8], &val, sizeof(u32)); + memcpy(&f->mask.lip[8], &mask, sizeof(u32)); + + return 0; +} + +static inline int cxgb4_fill_ipv6_dst_ip3(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + memcpy(&f->val.lip[12], &val, sizeof(u32)); + memcpy(&f->mask.lip[12], &mask, sizeof(u32)); + + return 0; +} + +static const struct cxgb4_match_field cxgb4_ipv6_fields[] = { + { .off = 0, .val = cxgb4_fill_ipv6_tos }, + { .off = 4, .val = cxgb4_fill_ipv6_proto }, + { .off = 8, .val = cxgb4_fill_ipv6_src_ip0 }, + { .off = 12, .val = cxgb4_fill_ipv6_src_ip1 }, + { .off = 16, .val = cxgb4_fill_ipv6_src_ip2 }, + { .off = 20, .val = cxgb4_fill_ipv6_src_ip3 }, + { .off = 24, .val = cxgb4_fill_ipv6_dst_ip0 }, + { .off = 28, .val = cxgb4_fill_ipv6_dst_ip1 }, + { .off = 32, .val = cxgb4_fill_ipv6_dst_ip2 }, + { .off = 36, .val = cxgb4_fill_ipv6_dst_ip3 }, + { .val = NULL } +}; + +/* TCP/UDP match */ +static inline int cxgb4_fill_l4_ports(struct ch_filter_specification *f, + __be32 val, __be32 mask) +{ + f->val.fport = ntohl(val) >> 16; + f->mask.fport = ntohl(mask) >> 16; + f->val.lport = ntohl(val) & 0x0000FFFF; + f->mask.lport = ntohl(mask) & 0x0000FFFF; + + return 0; +}; + +static const struct cxgb4_match_field cxgb4_tcp_fields[] = { + { .off = 0, .val = cxgb4_fill_l4_ports }, + { .val = NULL } +}; + +static const struct cxgb4_match_field cxgb4_udp_fields[] = { + { .off = 0, .val = cxgb4_fill_l4_ports }, + { .val = NULL } +}; + +struct cxgb4_next_header { + /* Offset, shift, and mask added to beginning of the header + * to get to next header. Useful when using a header + * field's value to jump to next header such as IHL field + * in IPv4 header. + */ + struct tc_u32_sel sel; + struct tc_u32_key key; + /* location of jump to make */ + const struct cxgb4_match_field *jump; +}; + +/* Accept a rule with a jump to transport layer header based on IHL field in + * IPv4 header. + */ +static const struct cxgb4_next_header cxgb4_ipv4_jumps[] = { + { + /* TCP Jump */ + .sel = { + .off = 0, + .offoff = 0, + .offshift = 6, + .offmask = cpu_to_be16(0x0f00), + }, + .key = { + .off = 8, + .val = cpu_to_be32(0x00060000), + .mask = cpu_to_be32(0x00ff0000), + }, + .jump = cxgb4_tcp_fields, + }, + { + /* UDP Jump */ + .sel = { + .off = 0, + .offoff = 0, + .offshift = 6, + .offmask = cpu_to_be16(0x0f00), + }, + .key = { + .off = 8, + .val = cpu_to_be32(0x00110000), + .mask = cpu_to_be32(0x00ff0000), + }, + .jump = cxgb4_udp_fields, + }, + { .jump = NULL }, +}; + +/* Accept a rule with a jump directly past the 40 Bytes of IPv6 fixed header + * to get to transport layer header. + */ +static const struct cxgb4_next_header cxgb4_ipv6_jumps[] = { + { + /* TCP Jump */ + .sel = { + .off = 40, + .offoff = 0, + .offshift = 0, + .offmask = 0, + }, + .key = { + .off = 4, + .val = cpu_to_be32(0x00000600), + .mask = cpu_to_be32(0x0000ff00), + }, + .jump = cxgb4_tcp_fields, + }, + { + /* UDP Jump */ + .sel = { + .off = 40, + .offoff = 0, + .offshift = 0, + .offmask = 0, + }, + .key = { + .off = 4, + .val = cpu_to_be32(0x00001100), + .mask = cpu_to_be32(0x0000ff00), + }, + .jump = cxgb4_udp_fields, + }, + { .jump = NULL }, +}; + +struct cxgb4_link { + const struct cxgb4_match_field *match_field; /* Next header */ + struct ch_filter_specification fs; /* Match spec associated with link */ + u32 link_handle; /* Knode handle associated with the link */ + unsigned long *tid_map; /* Bitmap for filter tids */ +}; + +struct cxgb4_tc_u32_table { + unsigned int size; /* number of entries in table */ + struct cxgb4_link table[]; /* Jump table */ +}; +#endif /* __CXGB4_TC_U32_PARSE_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c new file mode 100644 index 000000000..9a6d65243 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017 Chelsio Communications. All rights reserved. + * + * Written by: Ganesh Goudar (ganeshgr@chelsio.com) + */ + +#include "cxgb4.h" + +#define CXGB4_NUM_TRIPS 1 + +static int cxgb4_thermal_get_temp(struct thermal_zone_device *tzdev, + int *temp) +{ + struct adapter *adap = tzdev->devdata; + u32 param, val; + int ret; + + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_DIAG) | + FW_PARAMS_PARAM_Y_V(FW_PARAM_DEV_DIAG_TMP)); + + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, + ¶m, &val); + if (ret < 0 || val == 0) + return -1; + + *temp = val * 1000; + return 0; +} + +static int cxgb4_thermal_get_trip_type(struct thermal_zone_device *tzdev, + int trip, enum thermal_trip_type *type) +{ + struct adapter *adap = tzdev->devdata; + + if (!adap->ch_thermal.trip_temp) + return -EINVAL; + + *type = adap->ch_thermal.trip_type; + return 0; +} + +static int cxgb4_thermal_get_trip_temp(struct thermal_zone_device *tzdev, + int trip, int *temp) +{ + struct adapter *adap = tzdev->devdata; + + if (!adap->ch_thermal.trip_temp) + return -EINVAL; + + *temp = adap->ch_thermal.trip_temp; + return 0; +} + +static struct thermal_zone_device_ops cxgb4_thermal_ops = { + .get_temp = cxgb4_thermal_get_temp, + .get_trip_type = cxgb4_thermal_get_trip_type, + .get_trip_temp = cxgb4_thermal_get_trip_temp, +}; + +int cxgb4_thermal_init(struct adapter *adap) +{ + struct ch_thermal *ch_thermal = &adap->ch_thermal; + char ch_tz_name[THERMAL_NAME_LENGTH]; + int num_trip = CXGB4_NUM_TRIPS; + u32 param, val; + int ret; + + /* on older firmwares we may not get the trip temperature, + * set the num of trips to 0. + */ + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_DIAG) | + FW_PARAMS_PARAM_Y_V(FW_PARAM_DEV_DIAG_MAXTMPTHRESH)); + + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, + ¶m, &val); + if (ret < 0) { + num_trip = 0; /* could not get trip temperature */ + } else { + ch_thermal->trip_temp = val * 1000; + ch_thermal->trip_type = THERMAL_TRIP_CRITICAL; + } + + snprintf(ch_tz_name, sizeof(ch_tz_name), "cxgb4_%s", adap->name); + ch_thermal->tzdev = thermal_zone_device_register(ch_tz_name, num_trip, + 0, adap, + &cxgb4_thermal_ops, + NULL, 0, 0); + if (IS_ERR(ch_thermal->tzdev)) { + ret = PTR_ERR(ch_thermal->tzdev); + dev_err(adap->pdev_dev, "Failed to register thermal zone\n"); + ch_thermal->tzdev = NULL; + return ret; + } + + ret = thermal_zone_device_enable(ch_thermal->tzdev); + if (ret) { + dev_err(adap->pdev_dev, "Failed to enable thermal zone\n"); + thermal_zone_device_unregister(adap->ch_thermal.tzdev); + return ret; + } + + return 0; +} + +int cxgb4_thermal_remove(struct adapter *adap) +{ + if (adap->ch_thermal.tzdev) { + thermal_zone_device_unregister(adap->ch_thermal.tzdev); + adap->ch_thermal.tzdev = NULL; + } + return 0; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c new file mode 100644 index 000000000..17faac715 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c @@ -0,0 +1,859 @@ +/* + * cxgb4_uld.c:Chelsio Upper Layer Driver Interface for T4/T5/T6 SGE management + * + * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Written by: Atul Gupta (atul.gupta@chelsio.com) + * Written by: Hariprasad Shenai (hariprasad@chelsio.com) + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/errno.h> +#include <linux/types.h> +#include <linux/debugfs.h> +#include <linux/export.h> +#include <linux/list.h> +#include <linux/skbuff.h> +#include <linux/pci.h> + +#include "cxgb4.h" +#include "cxgb4_uld.h" +#include "t4_regs.h" +#include "t4fw_api.h" +#include "t4_msg.h" + +#define for_each_uldrxq(m, i) for (i = 0; i < ((m)->nrxq + (m)->nciq); i++) + +/* Flush the aggregated lro sessions */ +static void uldrx_flush_handler(struct sge_rspq *q) +{ + struct adapter *adap = q->adap; + + if (adap->uld[q->uld].lro_flush) + adap->uld[q->uld].lro_flush(&q->lro_mgr); +} + +/** + * uldrx_handler - response queue handler for ULD queues + * @q: the response queue that received the packet + * @rsp: the response queue descriptor holding the offload message + * @gl: the gather list of packet fragments + * + * Deliver an ingress offload packet to a ULD. All processing is done by + * the ULD, we just maintain statistics. + */ +static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp, + const struct pkt_gl *gl) +{ + struct adapter *adap = q->adap; + struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq); + int ret; + + /* FW can send CPLs encapsulated in a CPL_FW4_MSG */ + if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG && + ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL) + rsp += 2; + + if (q->flush_handler) + ret = adap->uld[q->uld].lro_rx_handler(adap->uld[q->uld].handle, + rsp, gl, &q->lro_mgr, + &q->napi); + else + ret = adap->uld[q->uld].rx_handler(adap->uld[q->uld].handle, + rsp, gl); + + if (ret) { + rxq->stats.nomem++; + return -1; + } + + if (!gl) + rxq->stats.imm++; + else if (gl == CXGB4_MSG_AN) + rxq->stats.an++; + else + rxq->stats.pkts++; + return 0; +} + +static int alloc_uld_rxqs(struct adapter *adap, + struct sge_uld_rxq_info *rxq_info, bool lro) +{ + unsigned int nq = rxq_info->nrxq + rxq_info->nciq; + struct sge_ofld_rxq *q = rxq_info->uldrxq; + unsigned short *ids = rxq_info->rspq_id; + int i, err, msi_idx, que_idx = 0; + struct sge *s = &adap->sge; + unsigned int per_chan; + + per_chan = rxq_info->nrxq / adap->params.nports; + + if (adap->flags & CXGB4_USING_MSIX) + msi_idx = 1; + else + msi_idx = -((int)s->intrq.abs_id + 1); + + for (i = 0; i < nq; i++, q++) { + if (i == rxq_info->nrxq) { + /* start allocation of concentrator queues */ + per_chan = rxq_info->nciq / adap->params.nports; + que_idx = 0; + } + + if (msi_idx >= 0) { + msi_idx = cxgb4_get_msix_idx_from_bmap(adap); + if (msi_idx < 0) { + err = -ENOSPC; + goto freeout; + } + + snprintf(adap->msix_info[msi_idx].desc, + sizeof(adap->msix_info[msi_idx].desc), + "%s-%s%d", + adap->port[0]->name, rxq_info->name, i); + + q->msix = &adap->msix_info[msi_idx]; + } + err = t4_sge_alloc_rxq(adap, &q->rspq, false, + adap->port[que_idx++ / per_chan], + msi_idx, + q->fl.size ? &q->fl : NULL, + uldrx_handler, + lro ? uldrx_flush_handler : NULL, + 0); + if (err) + goto freeout; + + memset(&q->stats, 0, sizeof(q->stats)); + if (ids) + ids[i] = q->rspq.abs_id; + } + return 0; +freeout: + q = rxq_info->uldrxq; + for ( ; i; i--, q++) { + if (q->rspq.desc) + free_rspq_fl(adap, &q->rspq, + q->fl.size ? &q->fl : NULL); + if (q->msix) + cxgb4_free_msix_idx_in_bmap(adap, q->msix->idx); + } + return err; +} + +static int +setup_sge_queues_uld(struct adapter *adap, unsigned int uld_type, bool lro) +{ + struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type]; + int i, ret; + + ret = alloc_uld_rxqs(adap, rxq_info, lro); + if (ret) + return ret; + + /* Tell uP to route control queue completions to rdma rspq */ + if (adap->flags & CXGB4_FULL_INIT_DONE && uld_type == CXGB4_ULD_RDMA) { + struct sge *s = &adap->sge; + unsigned int cmplqid; + u32 param, cmdop; + + cmdop = FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL; + for_each_port(adap, i) { + cmplqid = rxq_info->uldrxq[i].rspq.cntxt_id; + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | + FW_PARAMS_PARAM_X_V(cmdop) | + FW_PARAMS_PARAM_YZ_V(s->ctrlq[i].q.cntxt_id)); + ret = t4_set_params(adap, adap->mbox, adap->pf, + 0, 1, ¶m, &cmplqid); + } + } + return ret; +} + +static void t4_free_uld_rxqs(struct adapter *adap, int n, + struct sge_ofld_rxq *q) +{ + for ( ; n; n--, q++) { + if (q->rspq.desc) + free_rspq_fl(adap, &q->rspq, + q->fl.size ? &q->fl : NULL); + } +} + +static void free_sge_queues_uld(struct adapter *adap, unsigned int uld_type) +{ + struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type]; + + if (adap->flags & CXGB4_FULL_INIT_DONE && uld_type == CXGB4_ULD_RDMA) { + struct sge *s = &adap->sge; + u32 param, cmdop, cmplqid = 0; + int i; + + cmdop = FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL; + for_each_port(adap, i) { + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | + FW_PARAMS_PARAM_X_V(cmdop) | + FW_PARAMS_PARAM_YZ_V(s->ctrlq[i].q.cntxt_id)); + t4_set_params(adap, adap->mbox, adap->pf, + 0, 1, ¶m, &cmplqid); + } + } + + if (rxq_info->nciq) + t4_free_uld_rxqs(adap, rxq_info->nciq, + rxq_info->uldrxq + rxq_info->nrxq); + t4_free_uld_rxqs(adap, rxq_info->nrxq, rxq_info->uldrxq); +} + +static int cfg_queues_uld(struct adapter *adap, unsigned int uld_type, + const struct cxgb4_uld_info *uld_info) +{ + struct sge *s = &adap->sge; + struct sge_uld_rxq_info *rxq_info; + int i, nrxq, ciq_size; + + rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL); + if (!rxq_info) + return -ENOMEM; + + if (adap->flags & CXGB4_USING_MSIX && uld_info->nrxq > s->nqs_per_uld) { + i = s->nqs_per_uld; + rxq_info->nrxq = roundup(i, adap->params.nports); + } else { + i = min_t(int, uld_info->nrxq, + num_online_cpus()); + rxq_info->nrxq = roundup(i, adap->params.nports); + } + if (!uld_info->ciq) { + rxq_info->nciq = 0; + } else { + if (adap->flags & CXGB4_USING_MSIX) + rxq_info->nciq = min_t(int, s->nqs_per_uld, + num_online_cpus()); + else + rxq_info->nciq = min_t(int, MAX_OFLD_QSETS, + num_online_cpus()); + rxq_info->nciq = ((rxq_info->nciq / adap->params.nports) * + adap->params.nports); + rxq_info->nciq = max_t(int, rxq_info->nciq, + adap->params.nports); + } + + nrxq = rxq_info->nrxq + rxq_info->nciq; /* total rxq's */ + rxq_info->uldrxq = kcalloc(nrxq, sizeof(struct sge_ofld_rxq), + GFP_KERNEL); + if (!rxq_info->uldrxq) { + kfree(rxq_info); + return -ENOMEM; + } + + rxq_info->rspq_id = kcalloc(nrxq, sizeof(unsigned short), GFP_KERNEL); + if (!rxq_info->rspq_id) { + kfree(rxq_info->uldrxq); + kfree(rxq_info); + return -ENOMEM; + } + + for (i = 0; i < rxq_info->nrxq; i++) { + struct sge_ofld_rxq *r = &rxq_info->uldrxq[i]; + + init_rspq(adap, &r->rspq, 5, 1, uld_info->rxq_size, 64); + r->rspq.uld = uld_type; + r->fl.size = 72; + } + + ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids; + if (ciq_size > SGE_MAX_IQ_SIZE) { + dev_warn(adap->pdev_dev, "CIQ size too small for available IQs\n"); + ciq_size = SGE_MAX_IQ_SIZE; + } + + for (i = rxq_info->nrxq; i < nrxq; i++) { + struct sge_ofld_rxq *r = &rxq_info->uldrxq[i]; + + init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64); + r->rspq.uld = uld_type; + } + + memcpy(rxq_info->name, uld_info->name, IFNAMSIZ); + adap->sge.uld_rxq_info[uld_type] = rxq_info; + + return 0; +} + +static void free_queues_uld(struct adapter *adap, unsigned int uld_type) +{ + struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type]; + + adap->sge.uld_rxq_info[uld_type] = NULL; + kfree(rxq_info->rspq_id); + kfree(rxq_info->uldrxq); + kfree(rxq_info); +} + +static int +request_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type) +{ + struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type]; + struct msix_info *minfo; + unsigned int idx; + int err = 0; + + for_each_uldrxq(rxq_info, idx) { + minfo = rxq_info->uldrxq[idx].msix; + err = request_irq(minfo->vec, + t4_sge_intr_msix, 0, + minfo->desc, + &rxq_info->uldrxq[idx].rspq); + if (err) + goto unwind; + + cxgb4_set_msix_aff(adap, minfo->vec, + &minfo->aff_mask, idx); + } + return 0; + +unwind: + while (idx-- > 0) { + minfo = rxq_info->uldrxq[idx].msix; + cxgb4_clear_msix_aff(minfo->vec, minfo->aff_mask); + cxgb4_free_msix_idx_in_bmap(adap, minfo->idx); + free_irq(minfo->vec, &rxq_info->uldrxq[idx].rspq); + } + return err; +} + +static void +free_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type) +{ + struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type]; + struct msix_info *minfo; + unsigned int idx; + + for_each_uldrxq(rxq_info, idx) { + minfo = rxq_info->uldrxq[idx].msix; + cxgb4_clear_msix_aff(minfo->vec, minfo->aff_mask); + cxgb4_free_msix_idx_in_bmap(adap, minfo->idx); + free_irq(minfo->vec, &rxq_info->uldrxq[idx].rspq); + } +} + +static void enable_rx_uld(struct adapter *adap, unsigned int uld_type) +{ + struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type]; + int idx; + + for_each_uldrxq(rxq_info, idx) { + struct sge_rspq *q = &rxq_info->uldrxq[idx].rspq; + + if (!q) + continue; + + cxgb4_enable_rx(adap, q); + } +} + +static void quiesce_rx_uld(struct adapter *adap, unsigned int uld_type) +{ + struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type]; + int idx; + + for_each_uldrxq(rxq_info, idx) { + struct sge_rspq *q = &rxq_info->uldrxq[idx].rspq; + + if (!q) + continue; + + cxgb4_quiesce_rx(q); + } +} + +static void +free_sge_txq_uld(struct adapter *adap, struct sge_uld_txq_info *txq_info) +{ + int nq = txq_info->ntxq; + int i; + + for (i = 0; i < nq; i++) { + struct sge_uld_txq *txq = &txq_info->uldtxq[i]; + + if (txq && txq->q.desc) { + tasklet_kill(&txq->qresume_tsk); + t4_ofld_eq_free(adap, adap->mbox, adap->pf, 0, + txq->q.cntxt_id); + free_tx_desc(adap, &txq->q, txq->q.in_use, false); + kfree(txq->q.sdesc); + __skb_queue_purge(&txq->sendq); + free_txq(adap, &txq->q); + } + } +} + +static int +alloc_sge_txq_uld(struct adapter *adap, struct sge_uld_txq_info *txq_info, + unsigned int uld_type) +{ + struct sge *s = &adap->sge; + int nq = txq_info->ntxq; + int i, j, err; + + j = nq / adap->params.nports; + for (i = 0; i < nq; i++) { + struct sge_uld_txq *txq = &txq_info->uldtxq[i]; + + txq->q.size = 1024; + err = t4_sge_alloc_uld_txq(adap, txq, adap->port[i / j], + s->fw_evtq.cntxt_id, uld_type); + if (err) + goto freeout; + } + return 0; +freeout: + free_sge_txq_uld(adap, txq_info); + return err; +} + +static void +release_sge_txq_uld(struct adapter *adap, unsigned int uld_type) +{ + struct sge_uld_txq_info *txq_info = NULL; + int tx_uld_type = TX_ULD(uld_type); + + txq_info = adap->sge.uld_txq_info[tx_uld_type]; + + if (txq_info && atomic_dec_and_test(&txq_info->users)) { + free_sge_txq_uld(adap, txq_info); + kfree(txq_info->uldtxq); + kfree(txq_info); + adap->sge.uld_txq_info[tx_uld_type] = NULL; + } +} + +static int +setup_sge_txq_uld(struct adapter *adap, unsigned int uld_type, + const struct cxgb4_uld_info *uld_info) +{ + struct sge_uld_txq_info *txq_info = NULL; + int tx_uld_type, i; + + tx_uld_type = TX_ULD(uld_type); + txq_info = adap->sge.uld_txq_info[tx_uld_type]; + + if ((tx_uld_type == CXGB4_TX_OFLD) && txq_info && + (atomic_inc_return(&txq_info->users) > 1)) + return 0; + + txq_info = kzalloc(sizeof(*txq_info), GFP_KERNEL); + if (!txq_info) + return -ENOMEM; + if (uld_type == CXGB4_ULD_CRYPTO) { + i = min_t(int, adap->vres.ncrypto_fc, + num_online_cpus()); + txq_info->ntxq = rounddown(i, adap->params.nports); + if (txq_info->ntxq <= 0) { + dev_warn(adap->pdev_dev, "Crypto Tx Queues can't be zero\n"); + kfree(txq_info); + return -EINVAL; + } + + } else { + i = min_t(int, uld_info->ntxq, num_online_cpus()); + txq_info->ntxq = roundup(i, adap->params.nports); + } + txq_info->uldtxq = kcalloc(txq_info->ntxq, sizeof(struct sge_uld_txq), + GFP_KERNEL); + if (!txq_info->uldtxq) { + kfree(txq_info); + return -ENOMEM; + } + + if (alloc_sge_txq_uld(adap, txq_info, tx_uld_type)) { + kfree(txq_info->uldtxq); + kfree(txq_info); + return -ENOMEM; + } + + atomic_inc(&txq_info->users); + adap->sge.uld_txq_info[tx_uld_type] = txq_info; + return 0; +} + +static void uld_queue_init(struct adapter *adap, unsigned int uld_type, + struct cxgb4_lld_info *lli) +{ + struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type]; + int tx_uld_type = TX_ULD(uld_type); + struct sge_uld_txq_info *txq_info = adap->sge.uld_txq_info[tx_uld_type]; + + lli->rxq_ids = rxq_info->rspq_id; + lli->nrxq = rxq_info->nrxq; + lli->ciq_ids = rxq_info->rspq_id + rxq_info->nrxq; + lli->nciq = rxq_info->nciq; + lli->ntxq = txq_info->ntxq; +} + +int t4_uld_mem_alloc(struct adapter *adap) +{ + struct sge *s = &adap->sge; + + adap->uld = kcalloc(CXGB4_ULD_MAX, sizeof(*adap->uld), GFP_KERNEL); + if (!adap->uld) + return -ENOMEM; + + s->uld_rxq_info = kcalloc(CXGB4_ULD_MAX, + sizeof(struct sge_uld_rxq_info *), + GFP_KERNEL); + if (!s->uld_rxq_info) + goto err_uld; + + s->uld_txq_info = kcalloc(CXGB4_TX_MAX, + sizeof(struct sge_uld_txq_info *), + GFP_KERNEL); + if (!s->uld_txq_info) + goto err_uld_rx; + return 0; + +err_uld_rx: + kfree(s->uld_rxq_info); +err_uld: + kfree(adap->uld); + return -ENOMEM; +} + +void t4_uld_mem_free(struct adapter *adap) +{ + struct sge *s = &adap->sge; + + kfree(s->uld_txq_info); + kfree(s->uld_rxq_info); + kfree(adap->uld); +} + +/* This function should be called with uld_mutex taken. */ +static void cxgb4_shutdown_uld_adapter(struct adapter *adap, enum cxgb4_uld type) +{ + if (adap->uld[type].handle) { + adap->uld[type].handle = NULL; + adap->uld[type].add = NULL; + release_sge_txq_uld(adap, type); + + if (adap->flags & CXGB4_FULL_INIT_DONE) + quiesce_rx_uld(adap, type); + + if (adap->flags & CXGB4_USING_MSIX) + free_msix_queue_irqs_uld(adap, type); + + free_sge_queues_uld(adap, type); + free_queues_uld(adap, type); + } +} + +void t4_uld_clean_up(struct adapter *adap) +{ + unsigned int i; + + if (!is_uld(adap)) + return; + + mutex_lock(&uld_mutex); + for (i = 0; i < CXGB4_ULD_MAX; i++) { + if (!adap->uld[i].handle) + continue; + + cxgb4_shutdown_uld_adapter(adap, i); + } + mutex_unlock(&uld_mutex); +} + +static void uld_init(struct adapter *adap, struct cxgb4_lld_info *lld) +{ + int i; + + lld->pdev = adap->pdev; + lld->pf = adap->pf; + lld->l2t = adap->l2t; + lld->tids = &adap->tids; + lld->ports = adap->port; + lld->vr = &adap->vres; + lld->mtus = adap->params.mtus; + lld->nchan = adap->params.nports; + lld->nports = adap->params.nports; + lld->wr_cred = adap->params.ofldq_wr_cred; + lld->crypto = adap->params.crypto; + lld->iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A)); + lld->iscsi_tagmask = t4_read_reg(adap, ULP_RX_ISCSI_TAGMASK_A); + lld->iscsi_pgsz_order = t4_read_reg(adap, ULP_RX_ISCSI_PSZ_A); + lld->iscsi_llimit = t4_read_reg(adap, ULP_RX_ISCSI_LLIMIT_A); + lld->iscsi_ppm = &adap->iscsi_ppm; + lld->adapter_type = adap->params.chip; + lld->cclk_ps = 1000000000 / adap->params.vpd.cclk; + lld->udb_density = 1 << adap->params.sge.eq_qpp; + lld->ucq_density = 1 << adap->params.sge.iq_qpp; + lld->sge_host_page_size = 1 << (adap->params.sge.hps + 10); + lld->filt_mode = adap->params.tp.vlan_pri_map; + /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */ + for (i = 0; i < NCHAN; i++) + lld->tx_modq[i] = i; + lld->gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A); + lld->db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A); + lld->fw_vers = adap->params.fw_vers; + lld->dbfifo_int_thresh = dbfifo_int_thresh; + lld->sge_ingpadboundary = adap->sge.fl_align; + lld->sge_egrstatuspagesize = adap->sge.stat_len; + lld->sge_pktshift = adap->sge.pktshift; + lld->ulp_crypto = adap->params.crypto; + lld->enable_fw_ofld_conn = adap->flags & CXGB4_FW_OFLD_CONN; + lld->max_ordird_qp = adap->params.max_ordird_qp; + lld->max_ird_adapter = adap->params.max_ird_adapter; + lld->ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl; + lld->nodeid = dev_to_node(adap->pdev_dev); + lld->fr_nsmr_tpte_wr_support = adap->params.fr_nsmr_tpte_wr_support; + lld->write_w_imm_support = adap->params.write_w_imm_support; + lld->write_cmpl_support = adap->params.write_cmpl_support; +} + +static int uld_attach(struct adapter *adap, unsigned int uld) +{ + struct cxgb4_lld_info lli; + void *handle; + + uld_init(adap, &lli); + uld_queue_init(adap, uld, &lli); + + handle = adap->uld[uld].add(&lli); + if (IS_ERR(handle)) { + dev_warn(adap->pdev_dev, + "could not attach to the %s driver, error %ld\n", + adap->uld[uld].name, PTR_ERR(handle)); + return PTR_ERR(handle); + } + + adap->uld[uld].handle = handle; + t4_register_netevent_notifier(); + + if (adap->flags & CXGB4_FULL_INIT_DONE) + adap->uld[uld].state_change(handle, CXGB4_STATE_UP); + + return 0; +} + +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) +static bool cxgb4_uld_in_use(struct adapter *adap) +{ + const struct tid_info *t = &adap->tids; + + return (atomic_read(&t->conns_in_use) || t->stids_in_use); +} + +/* cxgb4_set_ktls_feature: request FW to enable/disable ktls settings. + * @adap: adapter info + * @enable: 1 to enable / 0 to disable ktls settings. + */ +int cxgb4_set_ktls_feature(struct adapter *adap, bool enable) +{ + int ret = 0; + u32 params = + FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_KTLS_HW) | + FW_PARAMS_PARAM_Y_V(enable) | + FW_PARAMS_PARAM_Z_V(FW_PARAMS_PARAM_DEV_KTLS_HW_USER_ENABLE); + + if (enable) { + if (!refcount_read(&adap->chcr_ktls.ktls_refcount)) { + /* At this moment if ULD connection are up means, other + * ULD is/are already active, return failure. + */ + if (cxgb4_uld_in_use(adap)) { + dev_dbg(adap->pdev_dev, + "ULD connections (tid/stid) active. Can't enable kTLS\n"); + return -EINVAL; + } + ret = t4_set_params(adap, adap->mbox, adap->pf, + 0, 1, ¶ms, ¶ms); + if (ret) + return ret; + refcount_set(&adap->chcr_ktls.ktls_refcount, 1); + pr_debug("kTLS has been enabled. Restrictions placed on ULD support\n"); + } else { + /* ktls settings already up, just increment refcount. */ + refcount_inc(&adap->chcr_ktls.ktls_refcount); + } + } else { + /* return failure if refcount is already 0. */ + if (!refcount_read(&adap->chcr_ktls.ktls_refcount)) + return -EINVAL; + /* decrement refcount and test, if 0, disable ktls feature, + * else return command success. + */ + if (refcount_dec_and_test(&adap->chcr_ktls.ktls_refcount)) { + ret = t4_set_params(adap, adap->mbox, adap->pf, + 0, 1, ¶ms, ¶ms); + if (ret) + return ret; + pr_debug("kTLS is disabled. Restrictions on ULD support removed\n"); + } + } + + return ret; +} +#endif + +static void cxgb4_uld_alloc_resources(struct adapter *adap, + enum cxgb4_uld type, + const struct cxgb4_uld_info *p) +{ + int ret = 0; + + if ((type == CXGB4_ULD_CRYPTO && !is_pci_uld(adap)) || + (type != CXGB4_ULD_CRYPTO && !is_offload(adap))) + return; + if (type == CXGB4_ULD_ISCSIT && is_t4(adap->params.chip)) + return; + ret = cfg_queues_uld(adap, type, p); + if (ret) + goto out; + ret = setup_sge_queues_uld(adap, type, p->lro); + if (ret) + goto free_queues; + if (adap->flags & CXGB4_USING_MSIX) { + ret = request_msix_queue_irqs_uld(adap, type); + if (ret) + goto free_rxq; + } + if (adap->flags & CXGB4_FULL_INIT_DONE) + enable_rx_uld(adap, type); + if (adap->uld[type].add) + goto free_irq; + ret = setup_sge_txq_uld(adap, type, p); + if (ret) + goto free_irq; + adap->uld[type] = *p; + ret = uld_attach(adap, type); + if (ret) + goto free_txq; + return; +free_txq: + release_sge_txq_uld(adap, type); +free_irq: + if (adap->flags & CXGB4_FULL_INIT_DONE) + quiesce_rx_uld(adap, type); + if (adap->flags & CXGB4_USING_MSIX) + free_msix_queue_irqs_uld(adap, type); +free_rxq: + free_sge_queues_uld(adap, type); +free_queues: + free_queues_uld(adap, type); +out: + dev_warn(adap->pdev_dev, + "ULD registration failed for uld type %d\n", type); +} + +void cxgb4_uld_enable(struct adapter *adap) +{ + struct cxgb4_uld_list *uld_entry; + + mutex_lock(&uld_mutex); + list_add_tail(&adap->list_node, &adapter_list); + list_for_each_entry(uld_entry, &uld_list, list_node) + cxgb4_uld_alloc_resources(adap, uld_entry->uld_type, + &uld_entry->uld_info); + mutex_unlock(&uld_mutex); +} + +/* cxgb4_register_uld - register an upper-layer driver + * @type: the ULD type + * @p: the ULD methods + * + * Registers an upper-layer driver with this driver and notifies the ULD + * about any presently available devices that support its type. + */ +void cxgb4_register_uld(enum cxgb4_uld type, + const struct cxgb4_uld_info *p) +{ + struct cxgb4_uld_list *uld_entry; + struct adapter *adap; + + if (type >= CXGB4_ULD_MAX) + return; + + uld_entry = kzalloc(sizeof(*uld_entry), GFP_KERNEL); + if (!uld_entry) + return; + + memcpy(&uld_entry->uld_info, p, sizeof(struct cxgb4_uld_info)); + mutex_lock(&uld_mutex); + list_for_each_entry(adap, &adapter_list, list_node) + cxgb4_uld_alloc_resources(adap, type, p); + + uld_entry->uld_type = type; + list_add_tail(&uld_entry->list_node, &uld_list); + mutex_unlock(&uld_mutex); + return; +} +EXPORT_SYMBOL(cxgb4_register_uld); + +/** + * cxgb4_unregister_uld - unregister an upper-layer driver + * @type: the ULD type + * + * Unregisters an existing upper-layer driver. + */ +int cxgb4_unregister_uld(enum cxgb4_uld type) +{ + struct cxgb4_uld_list *uld_entry, *tmp; + struct adapter *adap; + + if (type >= CXGB4_ULD_MAX) + return -EINVAL; + + mutex_lock(&uld_mutex); + list_for_each_entry(adap, &adapter_list, list_node) { + if ((type == CXGB4_ULD_CRYPTO && !is_pci_uld(adap)) || + (type != CXGB4_ULD_CRYPTO && !is_offload(adap))) + continue; + if (type == CXGB4_ULD_ISCSIT && is_t4(adap->params.chip)) + continue; + + cxgb4_shutdown_uld_adapter(adap, type); + } + + list_for_each_entry_safe(uld_entry, tmp, &uld_list, list_node) { + if (uld_entry->uld_type == type) { + list_del(&uld_entry->list_node); + kfree(uld_entry); + } + } + mutex_unlock(&uld_mutex); + + return 0; +} +EXPORT_SYMBOL(cxgb4_unregister_uld); diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h new file mode 100644 index 000000000..34546f531 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h @@ -0,0 +1,544 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_ULD_H +#define __CXGB4_ULD_H + +#include <linux/cache.h> +#include <linux/spinlock.h> +#include <linux/skbuff.h> +#include <linux/inetdevice.h> +#include <linux/atomic.h> +#include <net/tls.h> +#include "cxgb4.h" + +#define MAX_ULD_QSETS 16 +#define MAX_ULD_NPORTS 4 + +/* ulp_mem_io + ulptx_idata + payload + padding */ +#define MAX_IMM_ULPTX_WR_LEN (32 + 8 + 256 + 8) + +/* CPL message priority levels */ +enum { + CPL_PRIORITY_DATA = 0, /* data messages */ + CPL_PRIORITY_SETUP = 1, /* connection setup messages */ + CPL_PRIORITY_TEARDOWN = 0, /* connection teardown messages */ + CPL_PRIORITY_LISTEN = 1, /* listen start/stop messages */ + CPL_PRIORITY_ACK = 1, /* RX ACK messages */ + CPL_PRIORITY_CONTROL = 1 /* control messages */ +}; + +#define INIT_TP_WR(w, tid) do { \ + (w)->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) | \ + FW_WR_IMMDLEN_V(sizeof(*w) - sizeof(w->wr))); \ + (w)->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*w), 16)) | \ + FW_WR_FLOWID_V(tid)); \ + (w)->wr.wr_lo = cpu_to_be64(0); \ +} while (0) + +#define INIT_TP_WR_CPL(w, cpl, tid) do { \ + INIT_TP_WR(w, tid); \ + OPCODE_TID(w) = htonl(MK_OPCODE_TID(cpl, tid)); \ +} while (0) + +#define INIT_ULPTX_WR(w, wrlen, atomic, tid) do { \ + (w)->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) | \ + FW_WR_ATOMIC_V(atomic)); \ + (w)->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(wrlen, 16)) | \ + FW_WR_FLOWID_V(tid)); \ + (w)->wr.wr_lo = cpu_to_be64(0); \ +} while (0) + +/* Special asynchronous notification message */ +#define CXGB4_MSG_AN ((void *)1) +#define TX_ULD(uld)(((uld) != CXGB4_ULD_CRYPTO) ? CXGB4_TX_OFLD :\ + CXGB4_TX_CRYPTO) + +struct serv_entry { + void *data; +}; + +union aopen_entry { + void *data; + union aopen_entry *next; +}; + +struct eotid_entry { + void *data; +}; + +/* + * Holds the size, base address, free list start, etc of the TID, server TID, + * and active-open TID tables. The tables themselves are allocated dynamically. + */ +struct tid_info { + void **tid_tab; + unsigned int tid_base; + unsigned int ntids; + + struct serv_entry *stid_tab; + unsigned long *stid_bmap; + unsigned int nstids; + unsigned int stid_base; + + unsigned int nhash; + unsigned int hash_base; + + union aopen_entry *atid_tab; + unsigned int natids; + unsigned int atid_base; + + struct filter_entry *hpftid_tab; + unsigned long *hpftid_bmap; + unsigned int nhpftids; + unsigned int hpftid_base; + + struct filter_entry *ftid_tab; + unsigned long *ftid_bmap; + unsigned int nftids; + unsigned int ftid_base; + unsigned int aftid_base; + unsigned int aftid_end; + /* Server filter region */ + unsigned int sftid_base; + unsigned int nsftids; + + spinlock_t atid_lock ____cacheline_aligned_in_smp; + union aopen_entry *afree; + unsigned int atids_in_use; + + spinlock_t stid_lock; + unsigned int stids_in_use; + unsigned int v6_stids_in_use; + unsigned int sftids_in_use; + + /* ETHOFLD range */ + struct eotid_entry *eotid_tab; + unsigned long *eotid_bmap; + unsigned int eotid_base; + unsigned int neotids; + + /* TIDs in the TCAM */ + atomic_t tids_in_use; + /* TIDs in the HASH */ + atomic_t hash_tids_in_use; + atomic_t conns_in_use; + /* ETHOFLD TIDs used for rate limiting */ + atomic_t eotids_in_use; + + /* lock for setting/clearing filter bitmap */ + spinlock_t ftid_lock; + + unsigned int tc_hash_tids_max_prio; +}; + +static inline void *lookup_tid(const struct tid_info *t, unsigned int tid) +{ + tid -= t->tid_base; + return tid < t->ntids ? t->tid_tab[tid] : NULL; +} + +static inline bool tid_out_of_range(const struct tid_info *t, unsigned int tid) +{ + return ((tid - t->tid_base) >= t->ntids); +} + +static inline void *lookup_atid(const struct tid_info *t, unsigned int atid) +{ + return atid < t->natids ? t->atid_tab[atid].data : NULL; +} + +static inline void *lookup_stid(const struct tid_info *t, unsigned int stid) +{ + /* Is it a server filter TID? */ + if (t->nsftids && (stid >= t->sftid_base)) { + stid -= t->sftid_base; + stid += t->nstids; + } else { + stid -= t->stid_base; + } + + return stid < (t->nstids + t->nsftids) ? t->stid_tab[stid].data : NULL; +} + +static inline void cxgb4_insert_tid(struct tid_info *t, void *data, + unsigned int tid, unsigned short family) +{ + t->tid_tab[tid - t->tid_base] = data; + if (t->hash_base && (tid >= t->hash_base)) { + if (family == AF_INET6) + atomic_add(2, &t->hash_tids_in_use); + else + atomic_inc(&t->hash_tids_in_use); + } else { + if (family == AF_INET6) + atomic_add(2, &t->tids_in_use); + else + atomic_inc(&t->tids_in_use); + } + atomic_inc(&t->conns_in_use); +} + +static inline struct eotid_entry *cxgb4_lookup_eotid(struct tid_info *t, + u32 eotid) +{ + return eotid < t->neotids ? &t->eotid_tab[eotid] : NULL; +} + +static inline int cxgb4_get_free_eotid(struct tid_info *t) +{ + int eotid; + + eotid = find_first_zero_bit(t->eotid_bmap, t->neotids); + if (eotid >= t->neotids) + eotid = -1; + + return eotid; +} + +static inline void cxgb4_alloc_eotid(struct tid_info *t, u32 eotid, void *data) +{ + set_bit(eotid, t->eotid_bmap); + t->eotid_tab[eotid].data = data; + atomic_inc(&t->eotids_in_use); +} + +static inline void cxgb4_free_eotid(struct tid_info *t, u32 eotid) +{ + clear_bit(eotid, t->eotid_bmap); + t->eotid_tab[eotid].data = NULL; + atomic_dec(&t->eotids_in_use); +} + +int cxgb4_alloc_atid(struct tid_info *t, void *data); +int cxgb4_alloc_stid(struct tid_info *t, int family, void *data); +int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data); +void cxgb4_free_atid(struct tid_info *t, unsigned int atid); +void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family); +void cxgb4_remove_tid(struct tid_info *t, unsigned int qid, unsigned int tid, + unsigned short family); +struct in6_addr; + +int cxgb4_create_server(const struct net_device *dev, unsigned int stid, + __be32 sip, __be16 sport, __be16 vlan, + unsigned int queue); +int cxgb4_create_server6(const struct net_device *dev, unsigned int stid, + const struct in6_addr *sip, __be16 sport, + unsigned int queue); +int cxgb4_remove_server(const struct net_device *dev, unsigned int stid, + unsigned int queue, bool ipv6); +int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid, + __be32 sip, __be16 sport, __be16 vlan, + unsigned int queue, + unsigned char port, unsigned char mask); +int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid, + unsigned int queue, bool ipv6); + +/* Filter operation context to allow callers of cxgb4_set_filter() and + * cxgb4_del_filter() to wait for an asynchronous completion. + */ +struct filter_ctx { + struct completion completion; /* completion rendezvous */ + void *closure; /* caller's opaque information */ + int result; /* result of operation */ + u32 tid; /* to store tid */ +}; + +struct chcr_ktls { + refcount_t ktls_refcount; +}; + +struct ch_filter_specification; + +int cxgb4_get_free_ftid(struct net_device *dev, u8 family, bool hash_en, + u32 tc_prio); +int __cxgb4_set_filter(struct net_device *dev, int filter_id, + struct ch_filter_specification *fs, + struct filter_ctx *ctx); +int __cxgb4_del_filter(struct net_device *dev, int filter_id, + struct ch_filter_specification *fs, + struct filter_ctx *ctx); +int cxgb4_set_filter(struct net_device *dev, int filter_id, + struct ch_filter_specification *fs); +int cxgb4_del_filter(struct net_device *dev, int filter_id, + struct ch_filter_specification *fs); +int cxgb4_get_filter_counters(struct net_device *dev, unsigned int fidx, + u64 *hitcnt, u64 *bytecnt, bool hash); + +static inline void set_wr_txq(struct sk_buff *skb, int prio, int queue) +{ + skb_set_queue_mapping(skb, (queue << 1) | prio); +} + +enum cxgb4_uld { + CXGB4_ULD_INIT, + CXGB4_ULD_RDMA, + CXGB4_ULD_ISCSI, + CXGB4_ULD_ISCSIT, + CXGB4_ULD_CRYPTO, + CXGB4_ULD_IPSEC, + CXGB4_ULD_TLS, + CXGB4_ULD_KTLS, + CXGB4_ULD_MAX +}; + +enum cxgb4_tx_uld { + CXGB4_TX_OFLD, + CXGB4_TX_CRYPTO, + CXGB4_TX_MAX +}; + +enum cxgb4_txq_type { + CXGB4_TXQ_ETH, + CXGB4_TXQ_ULD, + CXGB4_TXQ_CTRL, + CXGB4_TXQ_MAX +}; + +enum cxgb4_state { + CXGB4_STATE_UP, + CXGB4_STATE_START_RECOVERY, + CXGB4_STATE_DOWN, + CXGB4_STATE_DETACH, + CXGB4_STATE_FATAL_ERROR +}; + +enum cxgb4_control { + CXGB4_CONTROL_DB_FULL, + CXGB4_CONTROL_DB_EMPTY, + CXGB4_CONTROL_DB_DROP, +}; + +struct adapter; +struct pci_dev; +struct l2t_data; +struct net_device; +struct pkt_gl; +struct tp_tcp_stats; +struct t4_lro_mgr; + +struct cxgb4_range { + unsigned int start; + unsigned int size; +}; + +struct cxgb4_virt_res { /* virtualized HW resources */ + struct cxgb4_range ddp; + struct cxgb4_range iscsi; + struct cxgb4_range stag; + struct cxgb4_range rq; + struct cxgb4_range srq; + struct cxgb4_range pbl; + struct cxgb4_range qp; + struct cxgb4_range cq; + struct cxgb4_range ocq; + struct cxgb4_range key; + unsigned int ncrypto_fc; + struct cxgb4_range ppod_edram; +}; + +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) +struct ch_ktls_port_stats_debug { + atomic64_t ktls_tx_connection_open; + atomic64_t ktls_tx_connection_fail; + atomic64_t ktls_tx_connection_close; + atomic64_t ktls_tx_encrypted_packets; + atomic64_t ktls_tx_encrypted_bytes; + atomic64_t ktls_tx_ctx; + atomic64_t ktls_tx_ooo; + atomic64_t ktls_tx_skip_no_sync_data; + atomic64_t ktls_tx_drop_no_sync_data; + atomic64_t ktls_tx_drop_bypass_req; +}; + +struct ch_ktls_stats_debug { + struct ch_ktls_port_stats_debug ktls_port[MAX_ULD_NPORTS]; + atomic64_t ktls_tx_send_records; + atomic64_t ktls_tx_end_pkts; + atomic64_t ktls_tx_start_pkts; + atomic64_t ktls_tx_middle_pkts; + atomic64_t ktls_tx_retransmit_pkts; + atomic64_t ktls_tx_complete_pkts; + atomic64_t ktls_tx_trimmed_pkts; + atomic64_t ktls_tx_fallback; +}; +#endif + +struct chcr_stats_debug { + atomic_t cipher_rqst; + atomic_t digest_rqst; + atomic_t aead_rqst; + atomic_t complete; + atomic_t error; + atomic_t fallback; + atomic_t tls_pdu_tx; + atomic_t tls_pdu_rx; + atomic_t tls_key; +}; + +#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) +struct ch_ipsec_stats_debug { + atomic_t ipsec_cnt; +}; +#endif + +#define OCQ_WIN_OFFSET(pdev, vres) \ + (pci_resource_len((pdev), 2) - roundup_pow_of_two((vres)->ocq.size)) + +/* + * Block of information the LLD provides to ULDs attaching to a device. + */ +struct cxgb4_lld_info { + struct pci_dev *pdev; /* associated PCI device */ + struct l2t_data *l2t; /* L2 table */ + struct tid_info *tids; /* TID table */ + struct net_device **ports; /* device ports */ + const struct cxgb4_virt_res *vr; /* assorted HW resources */ + const unsigned short *mtus; /* MTU table */ + const unsigned short *rxq_ids; /* the ULD's Rx queue ids */ + const unsigned short *ciq_ids; /* the ULD's concentrator IQ ids */ + unsigned short nrxq; /* # of Rx queues */ + unsigned short ntxq; /* # of Tx queues */ + unsigned short nciq; /* # of concentrator IQ */ + unsigned char nchan:4; /* # of channels */ + unsigned char nports:4; /* # of ports */ + unsigned char wr_cred; /* WR 16-byte credits */ + unsigned char adapter_type; /* type of adapter */ + unsigned char fw_api_ver; /* FW API version */ + unsigned char crypto; /* crypto support */ + unsigned int fw_vers; /* FW version */ + unsigned int iscsi_iolen; /* iSCSI max I/O length */ + unsigned int cclk_ps; /* Core clock period in psec */ + unsigned short udb_density; /* # of user DB/page */ + unsigned short ucq_density; /* # of user CQs/page */ + unsigned int sge_host_page_size; /* SGE host page size */ + unsigned short filt_mode; /* filter optional components */ + unsigned short tx_modq[NCHAN]; /* maps each tx channel to a */ + /* scheduler queue */ + void __iomem *gts_reg; /* address of GTS register */ + void __iomem *db_reg; /* address of kernel doorbell */ + int dbfifo_int_thresh; /* doorbell fifo int threshold */ + unsigned int sge_ingpadboundary; /* SGE ingress padding boundary */ + unsigned int sge_egrstatuspagesize; /* SGE egress status page size */ + unsigned int sge_pktshift; /* Padding between CPL and */ + /* packet data */ + unsigned int pf; /* Physical Function we're using */ + bool enable_fw_ofld_conn; /* Enable connection through fw */ + /* WR */ + unsigned int max_ordird_qp; /* Max ORD/IRD depth per RDMA QP */ + unsigned int max_ird_adapter; /* Max IRD memory per adapter */ + bool ulptx_memwrite_dsgl; /* use of T5 DSGL allowed */ + unsigned int iscsi_tagmask; /* iscsi ddp tag mask */ + unsigned int iscsi_pgsz_order; /* iscsi ddp page size orders */ + unsigned int iscsi_llimit; /* chip's iscsi region llimit */ + unsigned int ulp_crypto; /* crypto lookaside support */ + void **iscsi_ppm; /* iscsi page pod manager */ + int nodeid; /* device numa node id */ + bool fr_nsmr_tpte_wr_support; /* FW supports FR_NSMR_TPTE_WR */ + bool write_w_imm_support; /* FW supports WRITE_WITH_IMMEDIATE */ + bool write_cmpl_support; /* FW supports WRITE_CMPL WR */ +}; + +struct cxgb4_uld_info { + char name[IFNAMSIZ]; + void *handle; + unsigned int nrxq; + unsigned int rxq_size; + unsigned int ntxq; + bool ciq; + bool lro; + void *(*add)(const struct cxgb4_lld_info *p); + int (*rx_handler)(void *handle, const __be64 *rsp, + const struct pkt_gl *gl); + int (*state_change)(void *handle, enum cxgb4_state new_state); + int (*control)(void *handle, enum cxgb4_control control, ...); + int (*lro_rx_handler)(void *handle, const __be64 *rsp, + const struct pkt_gl *gl, + struct t4_lro_mgr *lro_mgr, + struct napi_struct *napi); + void (*lro_flush)(struct t4_lro_mgr *); + int (*tx_handler)(struct sk_buff *skb, struct net_device *dev); +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + const struct tlsdev_ops *tlsdev_ops; +#endif +#if IS_ENABLED(CONFIG_XFRM_OFFLOAD) + const struct xfrmdev_ops *xfrmdev_ops; +#endif +}; + +static inline bool cxgb4_is_ktls_skb(struct sk_buff *skb) +{ + return skb->sk && tls_is_sk_tx_device_offloaded(skb->sk); +} + +void cxgb4_uld_enable(struct adapter *adap); +void cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p); +int cxgb4_unregister_uld(enum cxgb4_uld type); +int cxgb4_ofld_send(struct net_device *dev, struct sk_buff *skb); +int cxgb4_immdata_send(struct net_device *dev, unsigned int idx, + const void *src, unsigned int len); +int cxgb4_crypto_send(struct net_device *dev, struct sk_buff *skb); +unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo); +unsigned int cxgb4_port_chan(const struct net_device *dev); +unsigned int cxgb4_port_e2cchan(const struct net_device *dev); +unsigned int cxgb4_port_viid(const struct net_device *dev); +unsigned int cxgb4_tp_smt_idx(enum chip_type chip, unsigned int viid); +unsigned int cxgb4_port_idx(const struct net_device *dev); +unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu, + unsigned int *idx); +unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus, + unsigned short header_size, + unsigned short data_size_max, + unsigned short data_size_align, + unsigned int *mtu_idxp); +void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4, + struct tp_tcp_stats *v6); +void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask, + const unsigned int *pgsz_order); +struct sk_buff *cxgb4_pktgl_to_skb(const struct pkt_gl *gl, + unsigned int skb_len, unsigned int pull_len); +int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx, u16 size); +int cxgb4_flush_eq_cache(struct net_device *dev); +int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte); +u64 cxgb4_read_sge_timestamp(struct net_device *dev); + +enum cxgb4_bar2_qtype { CXGB4_BAR2_QTYPE_EGRESS, CXGB4_BAR2_QTYPE_INGRESS }; +int cxgb4_bar2_sge_qregs(struct net_device *dev, + unsigned int qid, + enum cxgb4_bar2_qtype qtype, + int user, + u64 *pbar2_qoffset, + unsigned int *pbar2_qid); + +#endif /* !__CXGB4_ULD_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/l2t.c b/drivers/net/ethernet/chelsio/cxgb4/l2t.c new file mode 100644 index 000000000..a10a6862a --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/l2t.c @@ -0,0 +1,762 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/skbuff.h> +#include <linux/netdevice.h> +#include <linux/if.h> +#include <linux/if_vlan.h> +#include <linux/jhash.h> +#include <linux/module.h> +#include <linux/debugfs.h> +#include <linux/seq_file.h> +#include <net/neighbour.h> +#include "cxgb4.h" +#include "l2t.h" +#include "t4_msg.h" +#include "t4fw_api.h" +#include "t4_regs.h" +#include "t4_values.h" + +/* identifies sync vs async L2T_WRITE_REQs */ +#define SYNC_WR_S 12 +#define SYNC_WR_V(x) ((x) << SYNC_WR_S) +#define SYNC_WR_F SYNC_WR_V(1) + +struct l2t_data { + unsigned int l2t_start; /* start index of our piece of the L2T */ + unsigned int l2t_size; /* number of entries in l2tab */ + rwlock_t lock; + atomic_t nfree; /* number of free entries */ + struct l2t_entry *rover; /* starting point for next allocation */ + struct l2t_entry l2tab[]; /* MUST BE LAST */ +}; + +static inline unsigned int vlan_prio(const struct l2t_entry *e) +{ + return e->vlan >> VLAN_PRIO_SHIFT; +} + +static inline void l2t_hold(struct l2t_data *d, struct l2t_entry *e) +{ + if (atomic_add_return(1, &e->refcnt) == 1) /* 0 -> 1 transition */ + atomic_dec(&d->nfree); +} + +/* + * To avoid having to check address families we do not allow v4 and v6 + * neighbors to be on the same hash chain. We keep v4 entries in the first + * half of available hash buckets and v6 in the second. We need at least two + * entries in our L2T for this scheme to work. + */ +enum { + L2T_MIN_HASH_BUCKETS = 2, +}; + +static inline unsigned int arp_hash(struct l2t_data *d, const u32 *key, + int ifindex) +{ + unsigned int l2t_size_half = d->l2t_size / 2; + + return jhash_2words(*key, ifindex, 0) % l2t_size_half; +} + +static inline unsigned int ipv6_hash(struct l2t_data *d, const u32 *key, + int ifindex) +{ + unsigned int l2t_size_half = d->l2t_size / 2; + u32 xor = key[0] ^ key[1] ^ key[2] ^ key[3]; + + return (l2t_size_half + + (jhash_2words(xor, ifindex, 0) % l2t_size_half)); +} + +static unsigned int addr_hash(struct l2t_data *d, const u32 *addr, + int addr_len, int ifindex) +{ + return addr_len == 4 ? arp_hash(d, addr, ifindex) : + ipv6_hash(d, addr, ifindex); +} + +/* + * Checks if an L2T entry is for the given IP/IPv6 address. It does not check + * whether the L2T entry and the address are of the same address family. + * Callers ensure an address is only checked against L2T entries of the same + * family, something made trivial by the separation of IP and IPv6 hash chains + * mentioned above. Returns 0 if there's a match, + */ +static int addreq(const struct l2t_entry *e, const u32 *addr) +{ + if (e->v6) + return (e->addr[0] ^ addr[0]) | (e->addr[1] ^ addr[1]) | + (e->addr[2] ^ addr[2]) | (e->addr[3] ^ addr[3]); + return e->addr[0] ^ addr[0]; +} + +static void neigh_replace(struct l2t_entry *e, struct neighbour *n) +{ + neigh_hold(n); + if (e->neigh) + neigh_release(e->neigh); + e->neigh = n; +} + +/* + * Write an L2T entry. Must be called with the entry locked. + * The write may be synchronous or asynchronous. + */ +static int write_l2e(struct adapter *adap, struct l2t_entry *e, int sync) +{ + struct l2t_data *d = adap->l2t; + unsigned int l2t_idx = e->idx + d->l2t_start; + struct sk_buff *skb; + struct cpl_l2t_write_req *req; + + skb = alloc_skb(sizeof(*req), GFP_ATOMIC); + if (!skb) + return -ENOMEM; + + req = __skb_put(skb, sizeof(*req)); + INIT_TP_WR(req, 0); + + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, + l2t_idx | (sync ? SYNC_WR_F : 0) | + TID_QID_V(adap->sge.fw_evtq.abs_id))); + req->params = htons(L2T_W_PORT_V(e->lport) | L2T_W_NOREPLY_V(!sync)); + req->l2t_idx = htons(l2t_idx); + req->vlan = htons(e->vlan); + if (e->neigh && !(e->neigh->dev->flags & IFF_LOOPBACK)) + memcpy(e->dmac, e->neigh->ha, sizeof(e->dmac)); + memcpy(req->dst_mac, e->dmac, sizeof(req->dst_mac)); + + t4_mgmt_tx(adap, skb); + + if (sync && e->state != L2T_STATE_SWITCHING) + e->state = L2T_STATE_SYNC_WRITE; + return 0; +} + +/* + * Send packets waiting in an L2T entry's ARP queue. Must be called with the + * entry locked. + */ +static void send_pending(struct adapter *adap, struct l2t_entry *e) +{ + struct sk_buff *skb; + + while ((skb = __skb_dequeue(&e->arpq)) != NULL) + t4_ofld_send(adap, skb); +} + +/* + * Process a CPL_L2T_WRITE_RPL. Wake up the ARP queue if it completes a + * synchronous L2T_WRITE. Note that the TID in the reply is really the L2T + * index it refers to. + */ +void do_l2t_write_rpl(struct adapter *adap, const struct cpl_l2t_write_rpl *rpl) +{ + struct l2t_data *d = adap->l2t; + unsigned int tid = GET_TID(rpl); + unsigned int l2t_idx = tid % L2T_SIZE; + + if (unlikely(rpl->status != CPL_ERR_NONE)) { + dev_err(adap->pdev_dev, + "Unexpected L2T_WRITE_RPL status %u for entry %u\n", + rpl->status, l2t_idx); + return; + } + + if (tid & SYNC_WR_F) { + struct l2t_entry *e = &d->l2tab[l2t_idx - d->l2t_start]; + + spin_lock(&e->lock); + if (e->state != L2T_STATE_SWITCHING) { + send_pending(adap, e); + e->state = (e->neigh->nud_state & NUD_STALE) ? + L2T_STATE_STALE : L2T_STATE_VALID; + } + spin_unlock(&e->lock); + } +} + +/* + * Add a packet to an L2T entry's queue of packets awaiting resolution. + * Must be called with the entry's lock held. + */ +static inline void arpq_enqueue(struct l2t_entry *e, struct sk_buff *skb) +{ + __skb_queue_tail(&e->arpq, skb); +} + +int cxgb4_l2t_send(struct net_device *dev, struct sk_buff *skb, + struct l2t_entry *e) +{ + struct adapter *adap = netdev2adap(dev); + +again: + switch (e->state) { + case L2T_STATE_STALE: /* entry is stale, kick off revalidation */ + neigh_event_send(e->neigh, NULL); + spin_lock_bh(&e->lock); + if (e->state == L2T_STATE_STALE) + e->state = L2T_STATE_VALID; + spin_unlock_bh(&e->lock); + fallthrough; + case L2T_STATE_VALID: /* fast-path, send the packet on */ + return t4_ofld_send(adap, skb); + case L2T_STATE_RESOLVING: + case L2T_STATE_SYNC_WRITE: + spin_lock_bh(&e->lock); + if (e->state != L2T_STATE_SYNC_WRITE && + e->state != L2T_STATE_RESOLVING) { + spin_unlock_bh(&e->lock); + goto again; + } + arpq_enqueue(e, skb); + spin_unlock_bh(&e->lock); + + if (e->state == L2T_STATE_RESOLVING && + !neigh_event_send(e->neigh, NULL)) { + spin_lock_bh(&e->lock); + if (e->state == L2T_STATE_RESOLVING && + !skb_queue_empty(&e->arpq)) + write_l2e(adap, e, 1); + spin_unlock_bh(&e->lock); + } + } + return 0; +} +EXPORT_SYMBOL(cxgb4_l2t_send); + +/* + * Allocate a free L2T entry. Must be called with l2t_data.lock held. + */ +static struct l2t_entry *alloc_l2e(struct l2t_data *d) +{ + struct l2t_entry *end, *e, **p; + + if (!atomic_read(&d->nfree)) + return NULL; + + /* there's definitely a free entry */ + for (e = d->rover, end = &d->l2tab[d->l2t_size]; e != end; ++e) + if (atomic_read(&e->refcnt) == 0) + goto found; + + for (e = d->l2tab; atomic_read(&e->refcnt); ++e) + ; +found: + d->rover = e + 1; + atomic_dec(&d->nfree); + + /* + * The entry we found may be an inactive entry that is + * presently in the hash table. We need to remove it. + */ + if (e->state < L2T_STATE_SWITCHING) + for (p = &d->l2tab[e->hash].first; *p; p = &(*p)->next) + if (*p == e) { + *p = e->next; + e->next = NULL; + break; + } + + e->state = L2T_STATE_UNUSED; + return e; +} + +static struct l2t_entry *find_or_alloc_l2e(struct l2t_data *d, u16 vlan, + u8 port, u8 *dmac) +{ + struct l2t_entry *end, *e, **p; + struct l2t_entry *first_free = NULL; + + for (e = &d->l2tab[0], end = &d->l2tab[d->l2t_size]; e != end; ++e) { + if (atomic_read(&e->refcnt) == 0) { + if (!first_free) + first_free = e; + } else { + if (e->state == L2T_STATE_SWITCHING) { + if (ether_addr_equal(e->dmac, dmac) && + (e->vlan == vlan) && (e->lport == port)) + goto exists; + } + } + } + + if (first_free) { + e = first_free; + goto found; + } + + return NULL; + +found: + /* The entry we found may be an inactive entry that is + * presently in the hash table. We need to remove it. + */ + if (e->state < L2T_STATE_SWITCHING) + for (p = &d->l2tab[e->hash].first; *p; p = &(*p)->next) + if (*p == e) { + *p = e->next; + e->next = NULL; + break; + } + e->state = L2T_STATE_UNUSED; + +exists: + return e; +} + +/* Called when an L2T entry has no more users. The entry is left in the hash + * table since it is likely to be reused but we also bump nfree to indicate + * that the entry can be reallocated for a different neighbor. We also drop + * the existing neighbor reference in case the neighbor is going away and is + * waiting on our reference. + * + * Because entries can be reallocated to other neighbors once their ref count + * drops to 0 we need to take the entry's lock to avoid races with a new + * incarnation. + */ +static void _t4_l2e_free(struct l2t_entry *e) +{ + struct l2t_data *d; + + if (atomic_read(&e->refcnt) == 0) { /* hasn't been recycled */ + if (e->neigh) { + neigh_release(e->neigh); + e->neigh = NULL; + } + __skb_queue_purge(&e->arpq); + } + + d = container_of(e, struct l2t_data, l2tab[e->idx]); + atomic_inc(&d->nfree); +} + +/* Locked version of _t4_l2e_free */ +static void t4_l2e_free(struct l2t_entry *e) +{ + struct l2t_data *d; + + spin_lock_bh(&e->lock); + if (atomic_read(&e->refcnt) == 0) { /* hasn't been recycled */ + if (e->neigh) { + neigh_release(e->neigh); + e->neigh = NULL; + } + __skb_queue_purge(&e->arpq); + } + spin_unlock_bh(&e->lock); + + d = container_of(e, struct l2t_data, l2tab[e->idx]); + atomic_inc(&d->nfree); +} + +void cxgb4_l2t_release(struct l2t_entry *e) +{ + if (atomic_dec_and_test(&e->refcnt)) + t4_l2e_free(e); +} +EXPORT_SYMBOL(cxgb4_l2t_release); + +/* + * Update an L2T entry that was previously used for the same next hop as neigh. + * Must be called with softirqs disabled. + */ +static void reuse_entry(struct l2t_entry *e, struct neighbour *neigh) +{ + unsigned int nud_state; + + spin_lock(&e->lock); /* avoid race with t4_l2t_free */ + if (neigh != e->neigh) + neigh_replace(e, neigh); + nud_state = neigh->nud_state; + if (memcmp(e->dmac, neigh->ha, sizeof(e->dmac)) || + !(nud_state & NUD_VALID)) + e->state = L2T_STATE_RESOLVING; + else if (nud_state & NUD_CONNECTED) + e->state = L2T_STATE_VALID; + else + e->state = L2T_STATE_STALE; + spin_unlock(&e->lock); +} + +struct l2t_entry *cxgb4_l2t_get(struct l2t_data *d, struct neighbour *neigh, + const struct net_device *physdev, + unsigned int priority) +{ + u8 lport; + u16 vlan; + struct l2t_entry *e; + unsigned int addr_len = neigh->tbl->key_len; + u32 *addr = (u32 *)neigh->primary_key; + int ifidx = neigh->dev->ifindex; + int hash = addr_hash(d, addr, addr_len, ifidx); + + if (neigh->dev->flags & IFF_LOOPBACK) + lport = netdev2pinfo(physdev)->tx_chan + 4; + else + lport = netdev2pinfo(physdev)->lport; + + if (is_vlan_dev(neigh->dev)) { + vlan = vlan_dev_vlan_id(neigh->dev); + vlan |= vlan_dev_get_egress_qos_mask(neigh->dev, priority); + } else { + vlan = VLAN_NONE; + } + + write_lock_bh(&d->lock); + for (e = d->l2tab[hash].first; e; e = e->next) + if (!addreq(e, addr) && e->ifindex == ifidx && + e->vlan == vlan && e->lport == lport) { + l2t_hold(d, e); + if (atomic_read(&e->refcnt) == 1) + reuse_entry(e, neigh); + goto done; + } + + /* Need to allocate a new entry */ + e = alloc_l2e(d); + if (e) { + spin_lock(&e->lock); /* avoid race with t4_l2t_free */ + e->state = L2T_STATE_RESOLVING; + if (neigh->dev->flags & IFF_LOOPBACK) + memcpy(e->dmac, physdev->dev_addr, sizeof(e->dmac)); + memcpy(e->addr, addr, addr_len); + e->ifindex = ifidx; + e->hash = hash; + e->lport = lport; + e->v6 = addr_len == 16; + atomic_set(&e->refcnt, 1); + neigh_replace(e, neigh); + e->vlan = vlan; + e->next = d->l2tab[hash].first; + d->l2tab[hash].first = e; + spin_unlock(&e->lock); + } +done: + write_unlock_bh(&d->lock); + return e; +} +EXPORT_SYMBOL(cxgb4_l2t_get); + +u64 cxgb4_select_ntuple(struct net_device *dev, + const struct l2t_entry *l2t) +{ + struct adapter *adap = netdev2adap(dev); + struct tp_params *tp = &adap->params.tp; + u64 ntuple = 0; + + /* Initialize each of the fields which we care about which are present + * in the Compressed Filter Tuple. + */ + if (tp->vlan_shift >= 0 && l2t->vlan != VLAN_NONE) + ntuple |= (u64)(FT_VLAN_VLD_F | l2t->vlan) << tp->vlan_shift; + + if (tp->port_shift >= 0) + ntuple |= (u64)l2t->lport << tp->port_shift; + + if (tp->protocol_shift >= 0) + ntuple |= (u64)IPPROTO_TCP << tp->protocol_shift; + + if (tp->vnic_shift >= 0 && (tp->ingress_config & VNIC_F)) { + struct port_info *pi = (struct port_info *)netdev_priv(dev); + + ntuple |= (u64)(FT_VNID_ID_VF_V(pi->vin) | + FT_VNID_ID_PF_V(adap->pf) | + FT_VNID_ID_VLD_V(pi->vivld)) << tp->vnic_shift; + } + + return ntuple; +} +EXPORT_SYMBOL(cxgb4_select_ntuple); + +/* + * Called when the host's neighbor layer makes a change to some entry that is + * loaded into the HW L2 table. + */ +void t4_l2t_update(struct adapter *adap, struct neighbour *neigh) +{ + unsigned int addr_len = neigh->tbl->key_len; + u32 *addr = (u32 *) neigh->primary_key; + int hash, ifidx = neigh->dev->ifindex; + struct sk_buff_head *arpq = NULL; + struct l2t_data *d = adap->l2t; + struct l2t_entry *e; + + hash = addr_hash(d, addr, addr_len, ifidx); + read_lock_bh(&d->lock); + for (e = d->l2tab[hash].first; e; e = e->next) + if (!addreq(e, addr) && e->ifindex == ifidx) { + spin_lock(&e->lock); + if (atomic_read(&e->refcnt)) + goto found; + spin_unlock(&e->lock); + break; + } + read_unlock_bh(&d->lock); + return; + + found: + read_unlock(&d->lock); + + if (neigh != e->neigh) + neigh_replace(e, neigh); + + if (e->state == L2T_STATE_RESOLVING) { + if (neigh->nud_state & NUD_FAILED) { + arpq = &e->arpq; + } else if ((neigh->nud_state & (NUD_CONNECTED | NUD_STALE)) && + !skb_queue_empty(&e->arpq)) { + write_l2e(adap, e, 1); + } + } else { + e->state = neigh->nud_state & NUD_CONNECTED ? + L2T_STATE_VALID : L2T_STATE_STALE; + if (memcmp(e->dmac, neigh->ha, sizeof(e->dmac))) + write_l2e(adap, e, 0); + } + + if (arpq) { + struct sk_buff *skb; + + /* Called when address resolution fails for an L2T + * entry to handle packets on the arpq head. If a + * packet specifies a failure handler it is invoked, + * otherwise the packet is sent to the device. + */ + while ((skb = __skb_dequeue(&e->arpq)) != NULL) { + const struct l2t_skb_cb *cb = L2T_SKB_CB(skb); + + spin_unlock(&e->lock); + if (cb->arp_err_handler) + cb->arp_err_handler(cb->handle, skb); + else + t4_ofld_send(adap, skb); + spin_lock(&e->lock); + } + } + spin_unlock_bh(&e->lock); +} + +/* Allocate an L2T entry for use by a switching rule. Such need to be + * explicitly freed and while busy they are not on any hash chain, so normal + * address resolution updates do not see them. + */ +struct l2t_entry *t4_l2t_alloc_switching(struct adapter *adap, u16 vlan, + u8 port, u8 *eth_addr) +{ + struct l2t_data *d = adap->l2t; + struct l2t_entry *e; + int ret; + + write_lock_bh(&d->lock); + e = find_or_alloc_l2e(d, vlan, port, eth_addr); + if (e) { + spin_lock(&e->lock); /* avoid race with t4_l2t_free */ + if (!atomic_read(&e->refcnt)) { + e->state = L2T_STATE_SWITCHING; + e->vlan = vlan; + e->lport = port; + ether_addr_copy(e->dmac, eth_addr); + atomic_set(&e->refcnt, 1); + ret = write_l2e(adap, e, 0); + if (ret < 0) { + _t4_l2e_free(e); + spin_unlock(&e->lock); + write_unlock_bh(&d->lock); + return NULL; + } + } else { + atomic_inc(&e->refcnt); + } + + spin_unlock(&e->lock); + } + write_unlock_bh(&d->lock); + return e; +} + +/** + * cxgb4_l2t_alloc_switching - Allocates an L2T entry for switch filters + * @dev: net_device pointer + * @vlan: VLAN Id + * @port: Associated port + * @dmac: Destination MAC address to add to L2T + * Returns pointer to the allocated l2t entry + * + * Allocates an L2T entry for use by switching rule of a filter + */ +struct l2t_entry *cxgb4_l2t_alloc_switching(struct net_device *dev, u16 vlan, + u8 port, u8 *dmac) +{ + struct adapter *adap = netdev2adap(dev); + + return t4_l2t_alloc_switching(adap, vlan, port, dmac); +} +EXPORT_SYMBOL(cxgb4_l2t_alloc_switching); + +struct l2t_data *t4_init_l2t(unsigned int l2t_start, unsigned int l2t_end) +{ + unsigned int l2t_size; + int i; + struct l2t_data *d; + + if (l2t_start >= l2t_end || l2t_end >= L2T_SIZE) + return NULL; + l2t_size = l2t_end - l2t_start + 1; + if (l2t_size < L2T_MIN_HASH_BUCKETS) + return NULL; + + d = kvzalloc(struct_size(d, l2tab, l2t_size), GFP_KERNEL); + if (!d) + return NULL; + + d->l2t_start = l2t_start; + d->l2t_size = l2t_size; + + d->rover = d->l2tab; + atomic_set(&d->nfree, l2t_size); + rwlock_init(&d->lock); + + for (i = 0; i < d->l2t_size; ++i) { + d->l2tab[i].idx = i; + d->l2tab[i].state = L2T_STATE_UNUSED; + spin_lock_init(&d->l2tab[i].lock); + atomic_set(&d->l2tab[i].refcnt, 0); + skb_queue_head_init(&d->l2tab[i].arpq); + } + return d; +} + +static inline void *l2t_get_idx(struct seq_file *seq, loff_t pos) +{ + struct l2t_data *d = seq->private; + + return pos >= d->l2t_size ? NULL : &d->l2tab[pos]; +} + +static void *l2t_seq_start(struct seq_file *seq, loff_t *pos) +{ + return *pos ? l2t_get_idx(seq, *pos - 1) : SEQ_START_TOKEN; +} + +static void *l2t_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + v = l2t_get_idx(seq, *pos); + ++(*pos); + return v; +} + +static void l2t_seq_stop(struct seq_file *seq, void *v) +{ +} + +static char l2e_state(const struct l2t_entry *e) +{ + switch (e->state) { + case L2T_STATE_VALID: return 'V'; + case L2T_STATE_STALE: return 'S'; + case L2T_STATE_SYNC_WRITE: return 'W'; + case L2T_STATE_RESOLVING: + return skb_queue_empty(&e->arpq) ? 'R' : 'A'; + case L2T_STATE_SWITCHING: return 'X'; + default: + return 'U'; + } +} + +bool cxgb4_check_l2t_valid(struct l2t_entry *e) +{ + bool valid; + + spin_lock(&e->lock); + valid = (e->state == L2T_STATE_VALID); + spin_unlock(&e->lock); + return valid; +} +EXPORT_SYMBOL(cxgb4_check_l2t_valid); + +static int l2t_seq_show(struct seq_file *seq, void *v) +{ + if (v == SEQ_START_TOKEN) + seq_puts(seq, " Idx IP address " + "Ethernet address VLAN/P LP State Users Port\n"); + else { + char ip[60]; + struct l2t_data *d = seq->private; + struct l2t_entry *e = v; + + spin_lock_bh(&e->lock); + if (e->state == L2T_STATE_SWITCHING) + ip[0] = '\0'; + else + sprintf(ip, e->v6 ? "%pI6c" : "%pI4", e->addr); + seq_printf(seq, "%4u %-25s %17pM %4d %u %2u %c %5u %s\n", + e->idx + d->l2t_start, ip, e->dmac, + e->vlan & VLAN_VID_MASK, vlan_prio(e), e->lport, + l2e_state(e), atomic_read(&e->refcnt), + e->neigh ? e->neigh->dev->name : ""); + spin_unlock_bh(&e->lock); + } + return 0; +} + +static const struct seq_operations l2t_seq_ops = { + .start = l2t_seq_start, + .next = l2t_seq_next, + .stop = l2t_seq_stop, + .show = l2t_seq_show +}; + +static int l2t_seq_open(struct inode *inode, struct file *file) +{ + int rc = seq_open(file, &l2t_seq_ops); + + if (!rc) { + struct adapter *adap = inode->i_private; + struct seq_file *seq = file->private_data; + + seq->private = adap->l2t; + } + return rc; +} + +const struct file_operations t4_l2t_fops = { + .owner = THIS_MODULE, + .open = l2t_seq_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; diff --git a/drivers/net/ethernet/chelsio/cxgb4/l2t.h b/drivers/net/ethernet/chelsio/cxgb4/l2t.h new file mode 100644 index 000000000..340fecb28 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/l2t.h @@ -0,0 +1,128 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_L2T_H +#define __CXGB4_L2T_H + +#include <linux/spinlock.h> +#include <linux/if_ether.h> +#include <linux/atomic.h> + +#define VLAN_NONE 0xfff + +enum { L2T_SIZE = 4096 }; /* # of L2T entries */ + +enum { + L2T_STATE_VALID, /* entry is up to date */ + L2T_STATE_STALE, /* entry may be used but needs revalidation */ + L2T_STATE_RESOLVING, /* entry needs address resolution */ + L2T_STATE_SYNC_WRITE, /* synchronous write of entry underway */ + L2T_STATE_NOARP, /* Netdev down or removed*/ + + /* when state is one of the below the entry is not hashed */ + L2T_STATE_SWITCHING, /* entry is being used by a switching filter */ + L2T_STATE_UNUSED /* entry not in use */ +}; + +struct adapter; +struct l2t_data; +struct neighbour; +struct net_device; +struct file_operations; +struct cpl_l2t_write_rpl; + +/* + * Each L2T entry plays multiple roles. First of all, it keeps state for the + * corresponding entry of the HW L2 table and maintains a queue of offload + * packets awaiting address resolution. Second, it is a node of a hash table + * chain, where the nodes of the chain are linked together through their next + * pointer. Finally, each node is a bucket of a hash table, pointing to the + * first element in its chain through its first pointer. + */ +struct l2t_entry { + u16 state; /* entry state */ + u16 idx; /* entry index within in-memory table */ + u32 addr[4]; /* next hop IP or IPv6 address */ + int ifindex; /* neighbor's net_device's ifindex */ + struct neighbour *neigh; /* associated neighbour */ + struct l2t_entry *first; /* start of hash chain */ + struct l2t_entry *next; /* next l2t_entry on chain */ + struct sk_buff_head arpq; /* packet queue awaiting resolution */ + spinlock_t lock; + atomic_t refcnt; /* entry reference count */ + u16 hash; /* hash bucket the entry is on */ + u16 vlan; /* VLAN TCI (id: bits 0-11, prio: 13-15 */ + u8 v6; /* whether entry is for IPv6 */ + u8 lport; /* associated offload logical interface */ + u8 dmac[ETH_ALEN]; /* neighbour's MAC address */ +}; + +typedef void (*arp_err_handler_t)(void *handle, struct sk_buff *skb); + +/* + * Callback stored in an skb to handle address resolution failure. + */ +struct l2t_skb_cb { + void *handle; + arp_err_handler_t arp_err_handler; +}; + +#define L2T_SKB_CB(skb) ((struct l2t_skb_cb *)(skb)->cb) + +static inline void t4_set_arp_err_handler(struct sk_buff *skb, void *handle, + arp_err_handler_t handler) +{ + L2T_SKB_CB(skb)->handle = handle; + L2T_SKB_CB(skb)->arp_err_handler = handler; +} + +void cxgb4_l2t_release(struct l2t_entry *e); +int cxgb4_l2t_send(struct net_device *dev, struct sk_buff *skb, + struct l2t_entry *e); +struct l2t_entry *cxgb4_l2t_get(struct l2t_data *d, struct neighbour *neigh, + const struct net_device *physdev, + unsigned int priority); +u64 cxgb4_select_ntuple(struct net_device *dev, + const struct l2t_entry *l2t); +struct l2t_entry *cxgb4_l2t_alloc_switching(struct net_device *dev, u16 vlan, + u8 port, u8 *dmac); +void t4_l2t_update(struct adapter *adap, struct neighbour *neigh); +struct l2t_entry *t4_l2t_alloc_switching(struct adapter *adap, u16 vlan, + u8 port, u8 *dmac); +struct l2t_data *t4_init_l2t(unsigned int l2t_start, unsigned int l2t_end); +void do_l2t_write_rpl(struct adapter *p, const struct cpl_l2t_write_rpl *rpl); +bool cxgb4_check_l2t_valid(struct l2t_entry *e); + +extern const struct file_operations t4_l2t_fops; +#endif /* __CXGB4_L2T_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/sched.c b/drivers/net/ethernet/chelsio/cxgb4/sched.c new file mode 100644 index 000000000..a1b14468d --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/sched.c @@ -0,0 +1,693 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/module.h> +#include <linux/netdevice.h> + +#include "cxgb4.h" +#include "sched.h" + +static int t4_sched_class_fw_cmd(struct port_info *pi, + struct ch_sched_params *p, + enum sched_fw_ops op) +{ + struct adapter *adap = pi->adapter; + struct sched_table *s = pi->sched_tbl; + struct sched_class *e; + int err = 0; + + e = &s->tab[p->u.params.class]; + switch (op) { + case SCHED_FW_OP_ADD: + case SCHED_FW_OP_DEL: + err = t4_sched_params(adap, p->type, + p->u.params.level, p->u.params.mode, + p->u.params.rateunit, + p->u.params.ratemode, + p->u.params.channel, e->idx, + p->u.params.minrate, p->u.params.maxrate, + p->u.params.weight, p->u.params.pktsize, + p->u.params.burstsize); + break; + default: + err = -ENOTSUPP; + break; + } + + return err; +} + +static int t4_sched_bind_unbind_op(struct port_info *pi, void *arg, + enum sched_bind_type type, bool bind) +{ + struct adapter *adap = pi->adapter; + u32 fw_mnem, fw_class, fw_param; + unsigned int pf = adap->pf; + unsigned int vf = 0; + int err = 0; + + switch (type) { + case SCHED_QUEUE: { + struct sched_queue_entry *qe; + + qe = (struct sched_queue_entry *)arg; + + /* Create a template for the FW_PARAMS_CMD mnemonic and + * value (TX Scheduling Class in this case). + */ + fw_mnem = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | + FW_PARAMS_PARAM_X_V( + FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); + fw_class = bind ? qe->param.class : FW_SCHED_CLS_NONE; + fw_param = (fw_mnem | FW_PARAMS_PARAM_YZ_V(qe->cntxt_id)); + + pf = adap->pf; + vf = 0; + + err = t4_set_params(adap, adap->mbox, pf, vf, 1, + &fw_param, &fw_class); + break; + } + case SCHED_FLOWC: { + struct sched_flowc_entry *fe; + + fe = (struct sched_flowc_entry *)arg; + + fw_class = bind ? fe->param.class : FW_SCHED_CLS_NONE; + err = cxgb4_ethofld_send_flowc(adap->port[pi->port_id], + fe->param.tid, fw_class); + break; + } + default: + err = -ENOTSUPP; + break; + } + + return err; +} + +static void *t4_sched_entry_lookup(struct port_info *pi, + enum sched_bind_type type, + const u32 val) +{ + struct sched_table *s = pi->sched_tbl; + struct sched_class *e, *end; + void *found = NULL; + + /* Look for an entry with matching @val */ + end = &s->tab[s->sched_size]; + for (e = &s->tab[0]; e != end; ++e) { + if (e->state == SCHED_STATE_UNUSED || + e->bind_type != type) + continue; + + switch (type) { + case SCHED_QUEUE: { + struct sched_queue_entry *qe; + + list_for_each_entry(qe, &e->entry_list, list) { + if (qe->cntxt_id == val) { + found = qe; + break; + } + } + break; + } + case SCHED_FLOWC: { + struct sched_flowc_entry *fe; + + list_for_each_entry(fe, &e->entry_list, list) { + if (fe->param.tid == val) { + found = fe; + break; + } + } + break; + } + default: + return NULL; + } + + if (found) + break; + } + + return found; +} + +struct sched_class *cxgb4_sched_queue_lookup(struct net_device *dev, + struct ch_sched_queue *p) +{ + struct port_info *pi = netdev2pinfo(dev); + struct sched_queue_entry *qe = NULL; + struct adapter *adap = pi->adapter; + struct sge_eth_txq *txq; + + if (p->queue < 0 || p->queue >= pi->nqsets) + return NULL; + + txq = &adap->sge.ethtxq[pi->first_qset + p->queue]; + qe = t4_sched_entry_lookup(pi, SCHED_QUEUE, txq->q.cntxt_id); + return qe ? &pi->sched_tbl->tab[qe->param.class] : NULL; +} + +static int t4_sched_queue_unbind(struct port_info *pi, struct ch_sched_queue *p) +{ + struct sched_queue_entry *qe = NULL; + struct adapter *adap = pi->adapter; + struct sge_eth_txq *txq; + struct sched_class *e; + int err = 0; + + if (p->queue < 0 || p->queue >= pi->nqsets) + return -ERANGE; + + txq = &adap->sge.ethtxq[pi->first_qset + p->queue]; + + /* Find the existing entry that the queue is bound to */ + qe = t4_sched_entry_lookup(pi, SCHED_QUEUE, txq->q.cntxt_id); + if (qe) { + err = t4_sched_bind_unbind_op(pi, (void *)qe, SCHED_QUEUE, + false); + if (err) + return err; + + e = &pi->sched_tbl->tab[qe->param.class]; + list_del(&qe->list); + kvfree(qe); + if (atomic_dec_and_test(&e->refcnt)) + cxgb4_sched_class_free(adap->port[pi->port_id], e->idx); + } + return err; +} + +static int t4_sched_queue_bind(struct port_info *pi, struct ch_sched_queue *p) +{ + struct sched_table *s = pi->sched_tbl; + struct sched_queue_entry *qe = NULL; + struct adapter *adap = pi->adapter; + struct sge_eth_txq *txq; + struct sched_class *e; + unsigned int qid; + int err = 0; + + if (p->queue < 0 || p->queue >= pi->nqsets) + return -ERANGE; + + qe = kvzalloc(sizeof(struct sched_queue_entry), GFP_KERNEL); + if (!qe) + return -ENOMEM; + + txq = &adap->sge.ethtxq[pi->first_qset + p->queue]; + qid = txq->q.cntxt_id; + + /* Unbind queue from any existing class */ + err = t4_sched_queue_unbind(pi, p); + if (err) + goto out_err; + + /* Bind queue to specified class */ + qe->cntxt_id = qid; + memcpy(&qe->param, p, sizeof(qe->param)); + + e = &s->tab[qe->param.class]; + err = t4_sched_bind_unbind_op(pi, (void *)qe, SCHED_QUEUE, true); + if (err) + goto out_err; + + list_add_tail(&qe->list, &e->entry_list); + e->bind_type = SCHED_QUEUE; + atomic_inc(&e->refcnt); + return err; + +out_err: + kvfree(qe); + return err; +} + +static int t4_sched_flowc_unbind(struct port_info *pi, struct ch_sched_flowc *p) +{ + struct sched_flowc_entry *fe = NULL; + struct adapter *adap = pi->adapter; + struct sched_class *e; + int err = 0; + + if (p->tid < 0 || p->tid >= adap->tids.neotids) + return -ERANGE; + + /* Find the existing entry that the flowc is bound to */ + fe = t4_sched_entry_lookup(pi, SCHED_FLOWC, p->tid); + if (fe) { + err = t4_sched_bind_unbind_op(pi, (void *)fe, SCHED_FLOWC, + false); + if (err) + return err; + + e = &pi->sched_tbl->tab[fe->param.class]; + list_del(&fe->list); + kvfree(fe); + if (atomic_dec_and_test(&e->refcnt)) + cxgb4_sched_class_free(adap->port[pi->port_id], e->idx); + } + return err; +} + +static int t4_sched_flowc_bind(struct port_info *pi, struct ch_sched_flowc *p) +{ + struct sched_table *s = pi->sched_tbl; + struct sched_flowc_entry *fe = NULL; + struct adapter *adap = pi->adapter; + struct sched_class *e; + int err = 0; + + if (p->tid < 0 || p->tid >= adap->tids.neotids) + return -ERANGE; + + fe = kvzalloc(sizeof(*fe), GFP_KERNEL); + if (!fe) + return -ENOMEM; + + /* Unbind flowc from any existing class */ + err = t4_sched_flowc_unbind(pi, p); + if (err) + goto out_err; + + /* Bind flowc to specified class */ + memcpy(&fe->param, p, sizeof(fe->param)); + + e = &s->tab[fe->param.class]; + err = t4_sched_bind_unbind_op(pi, (void *)fe, SCHED_FLOWC, true); + if (err) + goto out_err; + + list_add_tail(&fe->list, &e->entry_list); + e->bind_type = SCHED_FLOWC; + atomic_inc(&e->refcnt); + return err; + +out_err: + kvfree(fe); + return err; +} + +static void t4_sched_class_unbind_all(struct port_info *pi, + struct sched_class *e, + enum sched_bind_type type) +{ + if (!e) + return; + + switch (type) { + case SCHED_QUEUE: { + struct sched_queue_entry *qe; + + list_for_each_entry(qe, &e->entry_list, list) + t4_sched_queue_unbind(pi, &qe->param); + break; + } + case SCHED_FLOWC: { + struct sched_flowc_entry *fe; + + list_for_each_entry(fe, &e->entry_list, list) + t4_sched_flowc_unbind(pi, &fe->param); + break; + } + default: + break; + } +} + +static int t4_sched_class_bind_unbind_op(struct port_info *pi, void *arg, + enum sched_bind_type type, bool bind) +{ + int err = 0; + + if (!arg) + return -EINVAL; + + switch (type) { + case SCHED_QUEUE: { + struct ch_sched_queue *qe = (struct ch_sched_queue *)arg; + + if (bind) + err = t4_sched_queue_bind(pi, qe); + else + err = t4_sched_queue_unbind(pi, qe); + break; + } + case SCHED_FLOWC: { + struct ch_sched_flowc *fe = (struct ch_sched_flowc *)arg; + + if (bind) + err = t4_sched_flowc_bind(pi, fe); + else + err = t4_sched_flowc_unbind(pi, fe); + break; + } + default: + err = -ENOTSUPP; + break; + } + + return err; +} + +/** + * cxgb4_sched_class_bind - Bind an entity to a scheduling class + * @dev: net_device pointer + * @arg: Entity opaque data + * @type: Entity type (Queue) + * + * Binds an entity (queue) to a scheduling class. If the entity + * is bound to another class, it will be unbound from the other class + * and bound to the class specified in @arg. + */ +int cxgb4_sched_class_bind(struct net_device *dev, void *arg, + enum sched_bind_type type) +{ + struct port_info *pi = netdev2pinfo(dev); + u8 class_id; + + if (!can_sched(dev)) + return -ENOTSUPP; + + if (!arg) + return -EINVAL; + + switch (type) { + case SCHED_QUEUE: { + struct ch_sched_queue *qe = (struct ch_sched_queue *)arg; + + class_id = qe->class; + break; + } + case SCHED_FLOWC: { + struct ch_sched_flowc *fe = (struct ch_sched_flowc *)arg; + + class_id = fe->class; + break; + } + default: + return -ENOTSUPP; + } + + if (!valid_class_id(dev, class_id)) + return -EINVAL; + + if (class_id == SCHED_CLS_NONE) + return -ENOTSUPP; + + return t4_sched_class_bind_unbind_op(pi, arg, type, true); + +} + +/** + * cxgb4_sched_class_unbind - Unbind an entity from a scheduling class + * @dev: net_device pointer + * @arg: Entity opaque data + * @type: Entity type (Queue) + * + * Unbinds an entity (queue) from a scheduling class. + */ +int cxgb4_sched_class_unbind(struct net_device *dev, void *arg, + enum sched_bind_type type) +{ + struct port_info *pi = netdev2pinfo(dev); + u8 class_id; + + if (!can_sched(dev)) + return -ENOTSUPP; + + if (!arg) + return -EINVAL; + + switch (type) { + case SCHED_QUEUE: { + struct ch_sched_queue *qe = (struct ch_sched_queue *)arg; + + class_id = qe->class; + break; + } + case SCHED_FLOWC: { + struct ch_sched_flowc *fe = (struct ch_sched_flowc *)arg; + + class_id = fe->class; + break; + } + default: + return -ENOTSUPP; + } + + if (!valid_class_id(dev, class_id)) + return -EINVAL; + + return t4_sched_class_bind_unbind_op(pi, arg, type, false); +} + +/* If @p is NULL, fetch any available unused class */ +static struct sched_class *t4_sched_class_lookup(struct port_info *pi, + const struct ch_sched_params *p) +{ + struct sched_table *s = pi->sched_tbl; + struct sched_class *found = NULL; + struct sched_class *e, *end; + + if (!p) { + /* Get any available unused class */ + end = &s->tab[s->sched_size]; + for (e = &s->tab[0]; e != end; ++e) { + if (e->state == SCHED_STATE_UNUSED) { + found = e; + break; + } + } + } else { + /* Look for a class with matching scheduling parameters */ + struct ch_sched_params info; + struct ch_sched_params tp; + + memcpy(&tp, p, sizeof(tp)); + /* Don't try to match class parameter */ + tp.u.params.class = SCHED_CLS_NONE; + + end = &s->tab[s->sched_size]; + for (e = &s->tab[0]; e != end; ++e) { + if (e->state == SCHED_STATE_UNUSED) + continue; + + memcpy(&info, &e->info, sizeof(info)); + /* Don't try to match class parameter */ + info.u.params.class = SCHED_CLS_NONE; + + if ((info.type == tp.type) && + (!memcmp(&info.u.params, &tp.u.params, + sizeof(info.u.params)))) { + found = e; + break; + } + } + } + + return found; +} + +static struct sched_class *t4_sched_class_alloc(struct port_info *pi, + struct ch_sched_params *p) +{ + struct sched_class *e = NULL; + u8 class_id; + int err; + + if (!p) + return NULL; + + class_id = p->u.params.class; + + /* Only accept search for existing class with matching params + * or allocation of new class with specified params + */ + if (class_id != SCHED_CLS_NONE) + return NULL; + + /* See if there's an exisiting class with same requested sched + * params. Classes can only be shared among FLOWC types. For + * other types, always request a new class. + */ + if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) + e = t4_sched_class_lookup(pi, p); + + if (!e) { + struct ch_sched_params np; + + /* Fetch any available unused class */ + e = t4_sched_class_lookup(pi, NULL); + if (!e) + return NULL; + + memcpy(&np, p, sizeof(np)); + np.u.params.class = e->idx; + /* New class */ + err = t4_sched_class_fw_cmd(pi, &np, SCHED_FW_OP_ADD); + if (err) + return NULL; + memcpy(&e->info, &np, sizeof(e->info)); + atomic_set(&e->refcnt, 0); + e->state = SCHED_STATE_ACTIVE; + } + + return e; +} + +/** + * cxgb4_sched_class_alloc - allocate a scheduling class + * @dev: net_device pointer + * @p: new scheduling class to create. + * + * Returns pointer to the scheduling class created. If @p is NULL, then + * it allocates and returns any available unused scheduling class. If a + * scheduling class with matching @p is found, then the matching class is + * returned. + */ +struct sched_class *cxgb4_sched_class_alloc(struct net_device *dev, + struct ch_sched_params *p) +{ + struct port_info *pi = netdev2pinfo(dev); + u8 class_id; + + if (!can_sched(dev)) + return NULL; + + class_id = p->u.params.class; + if (!valid_class_id(dev, class_id)) + return NULL; + + return t4_sched_class_alloc(pi, p); +} + +/** + * cxgb4_sched_class_free - free a scheduling class + * @dev: net_device pointer + * @classid: scheduling class id to free + * + * Frees a scheduling class if there are no users. + */ +void cxgb4_sched_class_free(struct net_device *dev, u8 classid) +{ + struct port_info *pi = netdev2pinfo(dev); + struct sched_table *s = pi->sched_tbl; + struct ch_sched_params p; + struct sched_class *e; + u32 speed; + int ret; + + e = &s->tab[classid]; + if (!atomic_read(&e->refcnt) && e->state != SCHED_STATE_UNUSED) { + /* Port based rate limiting needs explicit reset back + * to max rate. But, we'll do explicit reset for all + * types, instead of just port based type, to be on + * the safer side. + */ + memcpy(&p, &e->info, sizeof(p)); + /* Always reset mode to 0. Otherwise, FLOWC mode will + * still be enabled even after resetting the traffic + * class. + */ + p.u.params.mode = 0; + p.u.params.minrate = 0; + p.u.params.pktsize = 0; + + ret = t4_get_link_params(pi, NULL, &speed, NULL); + if (!ret) + p.u.params.maxrate = speed * 1000; /* Mbps to Kbps */ + else + p.u.params.maxrate = SCHED_MAX_RATE_KBPS; + + t4_sched_class_fw_cmd(pi, &p, SCHED_FW_OP_DEL); + + e->state = SCHED_STATE_UNUSED; + memset(&e->info, 0, sizeof(e->info)); + } +} + +static void t4_sched_class_free(struct net_device *dev, struct sched_class *e) +{ + struct port_info *pi = netdev2pinfo(dev); + + t4_sched_class_unbind_all(pi, e, e->bind_type); + cxgb4_sched_class_free(dev, e->idx); +} + +struct sched_table *t4_init_sched(unsigned int sched_size) +{ + struct sched_table *s; + unsigned int i; + + s = kvzalloc(struct_size(s, tab, sched_size), GFP_KERNEL); + if (!s) + return NULL; + + s->sched_size = sched_size; + + for (i = 0; i < s->sched_size; i++) { + memset(&s->tab[i], 0, sizeof(struct sched_class)); + s->tab[i].idx = i; + s->tab[i].state = SCHED_STATE_UNUSED; + INIT_LIST_HEAD(&s->tab[i].entry_list); + atomic_set(&s->tab[i].refcnt, 0); + } + return s; +} + +void t4_cleanup_sched(struct adapter *adap) +{ + struct sched_table *s; + unsigned int j, i; + + for_each_port(adap, j) { + struct port_info *pi = netdev2pinfo(adap->port[j]); + + s = pi->sched_tbl; + if (!s) + continue; + + for (i = 0; i < s->sched_size; i++) { + struct sched_class *e; + + e = &s->tab[i]; + if (e->state == SCHED_STATE_ACTIVE) + t4_sched_class_free(adap->port[j], e); + } + kvfree(s); + } +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/sched.h b/drivers/net/ethernet/chelsio/cxgb4/sched.h new file mode 100644 index 000000000..5f8b871d7 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/sched.h @@ -0,0 +1,119 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_SCHED_H +#define __CXGB4_SCHED_H + +#include <linux/spinlock.h> +#include <linux/atomic.h> + +#define SCHED_CLS_NONE 0xff + +#define FW_SCHED_CLS_NONE 0xffffffff + +/* Max rate that can be set to a scheduling class is 100 Gbps */ +#define SCHED_MAX_RATE_KBPS 100000000U + +enum { + SCHED_STATE_ACTIVE, + SCHED_STATE_UNUSED, +}; + +enum sched_fw_ops { + SCHED_FW_OP_ADD, + SCHED_FW_OP_DEL, +}; + +enum sched_bind_type { + SCHED_QUEUE, + SCHED_FLOWC, +}; + +struct sched_queue_entry { + struct list_head list; + unsigned int cntxt_id; + struct ch_sched_queue param; +}; + +struct sched_flowc_entry { + struct list_head list; + struct ch_sched_flowc param; +}; + +struct sched_class { + u8 state; + u8 idx; + struct ch_sched_params info; + enum sched_bind_type bind_type; + struct list_head entry_list; + atomic_t refcnt; +}; + +struct sched_table { /* per port scheduling table */ + u8 sched_size; + struct sched_class tab[]; +}; + +static inline bool can_sched(struct net_device *dev) +{ + struct port_info *pi = netdev2pinfo(dev); + + return !pi->sched_tbl ? false : true; +} + +static inline bool valid_class_id(struct net_device *dev, u8 class_id) +{ + struct port_info *pi = netdev2pinfo(dev); + + if ((class_id > pi->sched_tbl->sched_size - 1) && + (class_id != SCHED_CLS_NONE)) + return false; + + return true; +} + +struct sched_class *cxgb4_sched_queue_lookup(struct net_device *dev, + struct ch_sched_queue *p); +int cxgb4_sched_class_bind(struct net_device *dev, void *arg, + enum sched_bind_type type); +int cxgb4_sched_class_unbind(struct net_device *dev, void *arg, + enum sched_bind_type type); + +struct sched_class *cxgb4_sched_class_alloc(struct net_device *dev, + struct ch_sched_params *p); +void cxgb4_sched_class_free(struct net_device *dev, u8 classid); + +struct sched_table *t4_init_sched(unsigned int size); +void t4_cleanup_sched(struct adapter *adap); +#endif /* __CXGB4_SCHED_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c new file mode 100644 index 000000000..46809e2d9 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -0,0 +1,5234 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/skbuff.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> +#include <linux/if_vlan.h> +#include <linux/ip.h> +#include <linux/dma-mapping.h> +#include <linux/jiffies.h> +#include <linux/prefetch.h> +#include <linux/export.h> +#include <net/xfrm.h> +#include <net/ipv6.h> +#include <net/tcp.h> +#include <net/busy_poll.h> +#ifdef CONFIG_CHELSIO_T4_FCOE +#include <scsi/fc/fc_fcoe.h> +#endif /* CONFIG_CHELSIO_T4_FCOE */ +#include "cxgb4.h" +#include "t4_regs.h" +#include "t4_values.h" +#include "t4_msg.h" +#include "t4fw_api.h" +#include "cxgb4_ptp.h" +#include "cxgb4_uld.h" +#include "cxgb4_tc_mqprio.h" +#include "sched.h" + +/* + * Rx buffer size. We use largish buffers if possible but settle for single + * pages under memory shortage. + */ +#if PAGE_SHIFT >= 16 +# define FL_PG_ORDER 0 +#else +# define FL_PG_ORDER (16 - PAGE_SHIFT) +#endif + +/* RX_PULL_LEN should be <= RX_COPY_THRES */ +#define RX_COPY_THRES 256 +#define RX_PULL_LEN 128 + +/* + * Main body length for sk_buffs used for Rx Ethernet packets with fragments. + * Should be >= RX_PULL_LEN but possibly bigger to give pskb_may_pull some room. + */ +#define RX_PKT_SKB_LEN 512 + +/* + * Max number of Tx descriptors we clean up at a time. Should be modest as + * freeing skbs isn't cheap and it happens while holding locks. We just need + * to free packets faster than they arrive, we eventually catch up and keep + * the amortized cost reasonable. Must be >= 2 * TXQ_STOP_THRES. It should + * also match the CIDX Flush Threshold. + */ +#define MAX_TX_RECLAIM 32 + +/* + * Max number of Rx buffers we replenish at a time. Again keep this modest, + * allocating buffers isn't cheap either. + */ +#define MAX_RX_REFILL 16U + +/* + * Period of the Rx queue check timer. This timer is infrequent as it has + * something to do only when the system experiences severe memory shortage. + */ +#define RX_QCHECK_PERIOD (HZ / 2) + +/* + * Period of the Tx queue check timer. + */ +#define TX_QCHECK_PERIOD (HZ / 2) + +/* + * Max number of Tx descriptors to be reclaimed by the Tx timer. + */ +#define MAX_TIMER_TX_RECLAIM 100 + +/* + * Timer index used when backing off due to memory shortage. + */ +#define NOMEM_TMR_IDX (SGE_NTIMERS - 1) + +/* + * Suspension threshold for non-Ethernet Tx queues. We require enough room + * for a full sized WR. + */ +#define TXQ_STOP_THRES (SGE_MAX_WR_LEN / sizeof(struct tx_desc)) + +/* + * Max Tx descriptor space we allow for an Ethernet packet to be inlined + * into a WR. + */ +#define MAX_IMM_TX_PKT_LEN 256 + +/* + * Max size of a WR sent through a control Tx queue. + */ +#define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN + +struct rx_sw_desc { /* SW state per Rx descriptor */ + struct page *page; + dma_addr_t dma_addr; +}; + +/* + * Rx buffer sizes for "useskbs" Free List buffers (one ingress packet pe skb + * buffer). We currently only support two sizes for 1500- and 9000-byte MTUs. + * We could easily support more but there doesn't seem to be much need for + * that ... + */ +#define FL_MTU_SMALL 1500 +#define FL_MTU_LARGE 9000 + +static inline unsigned int fl_mtu_bufsize(struct adapter *adapter, + unsigned int mtu) +{ + struct sge *s = &adapter->sge; + + return ALIGN(s->pktshift + ETH_HLEN + VLAN_HLEN + mtu, s->fl_align); +} + +#define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL) +#define FL_MTU_LARGE_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_LARGE) + +/* + * Bits 0..3 of rx_sw_desc.dma_addr have special meaning. The hardware uses + * these to specify the buffer size as an index into the SGE Free List Buffer + * Size register array. We also use bit 4, when the buffer has been unmapped + * for DMA, but this is of course never sent to the hardware and is only used + * to prevent double unmappings. All of the above requires that the Free List + * Buffers which we allocate have the bottom 5 bits free (0) -- i.e. are + * 32-byte or or a power of 2 greater in alignment. Since the SGE's minimal + * Free List Buffer alignment is 32 bytes, this works out for us ... + */ +enum { + RX_BUF_FLAGS = 0x1f, /* bottom five bits are special */ + RX_BUF_SIZE = 0x0f, /* bottom three bits are for buf sizes */ + RX_UNMAPPED_BUF = 0x10, /* buffer is not mapped */ + + /* + * XXX We shouldn't depend on being able to use these indices. + * XXX Especially when some other Master PF has initialized the + * XXX adapter or we use the Firmware Configuration File. We + * XXX should really search through the Host Buffer Size register + * XXX array for the appropriately sized buffer indices. + */ + RX_SMALL_PG_BUF = 0x0, /* small (PAGE_SIZE) page buffer */ + RX_LARGE_PG_BUF = 0x1, /* buffer large (FL_PG_ORDER) page buffer */ + + RX_SMALL_MTU_BUF = 0x2, /* small MTU buffer */ + RX_LARGE_MTU_BUF = 0x3, /* large MTU buffer */ +}; + +static int timer_pkt_quota[] = {1, 1, 2, 3, 4, 5}; +#define MIN_NAPI_WORK 1 + +static inline dma_addr_t get_buf_addr(const struct rx_sw_desc *d) +{ + return d->dma_addr & ~(dma_addr_t)RX_BUF_FLAGS; +} + +static inline bool is_buf_mapped(const struct rx_sw_desc *d) +{ + return !(d->dma_addr & RX_UNMAPPED_BUF); +} + +/** + * txq_avail - return the number of available slots in a Tx queue + * @q: the Tx queue + * + * Returns the number of descriptors in a Tx queue available to write new + * packets. + */ +static inline unsigned int txq_avail(const struct sge_txq *q) +{ + return q->size - 1 - q->in_use; +} + +/** + * fl_cap - return the capacity of a free-buffer list + * @fl: the FL + * + * Returns the capacity of a free-buffer list. The capacity is less than + * the size because one descriptor needs to be left unpopulated, otherwise + * HW will think the FL is empty. + */ +static inline unsigned int fl_cap(const struct sge_fl *fl) +{ + return fl->size - 8; /* 1 descriptor = 8 buffers */ +} + +/** + * fl_starving - return whether a Free List is starving. + * @adapter: pointer to the adapter + * @fl: the Free List + * + * Tests specified Free List to see whether the number of buffers + * available to the hardware has falled below our "starvation" + * threshold. + */ +static inline bool fl_starving(const struct adapter *adapter, + const struct sge_fl *fl) +{ + const struct sge *s = &adapter->sge; + + return fl->avail - fl->pend_cred <= s->fl_starve_thres; +} + +int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb, + dma_addr_t *addr) +{ + const skb_frag_t *fp, *end; + const struct skb_shared_info *si; + + *addr = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); + if (dma_mapping_error(dev, *addr)) + goto out_err; + + si = skb_shinfo(skb); + end = &si->frags[si->nr_frags]; + + for (fp = si->frags; fp < end; fp++) { + *++addr = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp), + DMA_TO_DEVICE); + if (dma_mapping_error(dev, *addr)) + goto unwind; + } + return 0; + +unwind: + while (fp-- > si->frags) + dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE); + + dma_unmap_single(dev, addr[-1], skb_headlen(skb), DMA_TO_DEVICE); +out_err: + return -ENOMEM; +} +EXPORT_SYMBOL(cxgb4_map_skb); + +static void unmap_skb(struct device *dev, const struct sk_buff *skb, + const dma_addr_t *addr) +{ + const skb_frag_t *fp, *end; + const struct skb_shared_info *si; + + dma_unmap_single(dev, *addr++, skb_headlen(skb), DMA_TO_DEVICE); + + si = skb_shinfo(skb); + end = &si->frags[si->nr_frags]; + for (fp = si->frags; fp < end; fp++) + dma_unmap_page(dev, *addr++, skb_frag_size(fp), DMA_TO_DEVICE); +} + +#ifdef CONFIG_NEED_DMA_MAP_STATE +/** + * deferred_unmap_destructor - unmap a packet when it is freed + * @skb: the packet + * + * This is the packet destructor used for Tx packets that need to remain + * mapped until they are freed rather than until their Tx descriptors are + * freed. + */ +static void deferred_unmap_destructor(struct sk_buff *skb) +{ + unmap_skb(skb->dev->dev.parent, skb, (dma_addr_t *)skb->head); +} +#endif + +/** + * free_tx_desc - reclaims Tx descriptors and their buffers + * @adap: the adapter + * @q: the Tx queue to reclaim descriptors from + * @n: the number of descriptors to reclaim + * @unmap: whether the buffers should be unmapped for DMA + * + * Reclaims Tx descriptors from an SGE Tx queue and frees the associated + * Tx buffers. Called with the Tx queue lock held. + */ +void free_tx_desc(struct adapter *adap, struct sge_txq *q, + unsigned int n, bool unmap) +{ + unsigned int cidx = q->cidx; + struct tx_sw_desc *d; + + d = &q->sdesc[cidx]; + while (n--) { + if (d->skb) { /* an SGL is present */ + if (unmap && d->addr[0]) { + unmap_skb(adap->pdev_dev, d->skb, d->addr); + memset(d->addr, 0, sizeof(d->addr)); + } + dev_consume_skb_any(d->skb); + d->skb = NULL; + } + ++d; + if (++cidx == q->size) { + cidx = 0; + d = q->sdesc; + } + } + q->cidx = cidx; +} + +/* + * Return the number of reclaimable descriptors in a Tx queue. + */ +static inline int reclaimable(const struct sge_txq *q) +{ + int hw_cidx = ntohs(READ_ONCE(q->stat->cidx)); + hw_cidx -= q->cidx; + return hw_cidx < 0 ? hw_cidx + q->size : hw_cidx; +} + +/** + * reclaim_completed_tx - reclaims completed TX Descriptors + * @adap: the adapter + * @q: the Tx queue to reclaim completed descriptors from + * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1 + * @unmap: whether the buffers should be unmapped for DMA + * + * Reclaims Tx Descriptors that the SGE has indicated it has processed, + * and frees the associated buffers if possible. If @max == -1, then + * we'll use a defaiult maximum. Called with the TX Queue locked. + */ +static inline int reclaim_completed_tx(struct adapter *adap, struct sge_txq *q, + int maxreclaim, bool unmap) +{ + int reclaim = reclaimable(q); + + if (reclaim) { + /* + * Limit the amount of clean up work we do at a time to keep + * the Tx lock hold time O(1). + */ + if (maxreclaim < 0) + maxreclaim = MAX_TX_RECLAIM; + if (reclaim > maxreclaim) + reclaim = maxreclaim; + + free_tx_desc(adap, q, reclaim, unmap); + q->in_use -= reclaim; + } + + return reclaim; +} + +/** + * cxgb4_reclaim_completed_tx - reclaims completed Tx descriptors + * @adap: the adapter + * @q: the Tx queue to reclaim completed descriptors from + * @unmap: whether the buffers should be unmapped for DMA + * + * Reclaims Tx descriptors that the SGE has indicated it has processed, + * and frees the associated buffers if possible. Called with the Tx + * queue locked. + */ +void cxgb4_reclaim_completed_tx(struct adapter *adap, struct sge_txq *q, + bool unmap) +{ + (void)reclaim_completed_tx(adap, q, -1, unmap); +} +EXPORT_SYMBOL(cxgb4_reclaim_completed_tx); + +static inline int get_buf_size(struct adapter *adapter, + const struct rx_sw_desc *d) +{ + struct sge *s = &adapter->sge; + unsigned int rx_buf_size_idx = d->dma_addr & RX_BUF_SIZE; + int buf_size; + + switch (rx_buf_size_idx) { + case RX_SMALL_PG_BUF: + buf_size = PAGE_SIZE; + break; + + case RX_LARGE_PG_BUF: + buf_size = PAGE_SIZE << s->fl_pg_order; + break; + + case RX_SMALL_MTU_BUF: + buf_size = FL_MTU_SMALL_BUFSIZE(adapter); + break; + + case RX_LARGE_MTU_BUF: + buf_size = FL_MTU_LARGE_BUFSIZE(adapter); + break; + + default: + BUG(); + } + + return buf_size; +} + +/** + * free_rx_bufs - free the Rx buffers on an SGE free list + * @adap: the adapter + * @q: the SGE free list to free buffers from + * @n: how many buffers to free + * + * Release the next @n buffers on an SGE free-buffer Rx queue. The + * buffers must be made inaccessible to HW before calling this function. + */ +static void free_rx_bufs(struct adapter *adap, struct sge_fl *q, int n) +{ + while (n--) { + struct rx_sw_desc *d = &q->sdesc[q->cidx]; + + if (is_buf_mapped(d)) + dma_unmap_page(adap->pdev_dev, get_buf_addr(d), + get_buf_size(adap, d), + DMA_FROM_DEVICE); + put_page(d->page); + d->page = NULL; + if (++q->cidx == q->size) + q->cidx = 0; + q->avail--; + } +} + +/** + * unmap_rx_buf - unmap the current Rx buffer on an SGE free list + * @adap: the adapter + * @q: the SGE free list + * + * Unmap the current buffer on an SGE free-buffer Rx queue. The + * buffer must be made inaccessible to HW before calling this function. + * + * This is similar to @free_rx_bufs above but does not free the buffer. + * Do note that the FL still loses any further access to the buffer. + */ +static void unmap_rx_buf(struct adapter *adap, struct sge_fl *q) +{ + struct rx_sw_desc *d = &q->sdesc[q->cidx]; + + if (is_buf_mapped(d)) + dma_unmap_page(adap->pdev_dev, get_buf_addr(d), + get_buf_size(adap, d), DMA_FROM_DEVICE); + d->page = NULL; + if (++q->cidx == q->size) + q->cidx = 0; + q->avail--; +} + +static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q) +{ + if (q->pend_cred >= 8) { + u32 val = adap->params.arch.sge_fl_db; + + if (is_t4(adap->params.chip)) + val |= PIDX_V(q->pend_cred / 8); + else + val |= PIDX_T5_V(q->pend_cred / 8); + + /* Make sure all memory writes to the Free List queue are + * committed before we tell the hardware about them. + */ + wmb(); + + /* If we don't have access to the new User Doorbell (T5+), use + * the old doorbell mechanism; otherwise use the new BAR2 + * mechanism. + */ + if (unlikely(q->bar2_addr == NULL)) { + t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), + val | QID_V(q->cntxt_id)); + } else { + writel(val | QID_V(q->bar2_qid), + q->bar2_addr + SGE_UDB_KDOORBELL); + + /* This Write memory Barrier will force the write to + * the User Doorbell area to be flushed. + */ + wmb(); + } + q->pend_cred &= 7; + } +} + +static inline void set_rx_sw_desc(struct rx_sw_desc *sd, struct page *pg, + dma_addr_t mapping) +{ + sd->page = pg; + sd->dma_addr = mapping; /* includes size low bits */ +} + +/** + * refill_fl - refill an SGE Rx buffer ring + * @adap: the adapter + * @q: the ring to refill + * @n: the number of new buffers to allocate + * @gfp: the gfp flags for the allocations + * + * (Re)populate an SGE free-buffer queue with up to @n new packet buffers, + * allocated with the supplied gfp flags. The caller must assure that + * @n does not exceed the queue's capacity. If afterwards the queue is + * found critically low mark it as starving in the bitmap of starving FLs. + * + * Returns the number of buffers allocated. + */ +static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n, + gfp_t gfp) +{ + struct sge *s = &adap->sge; + struct page *pg; + dma_addr_t mapping; + unsigned int cred = q->avail; + __be64 *d = &q->desc[q->pidx]; + struct rx_sw_desc *sd = &q->sdesc[q->pidx]; + int node; + +#ifdef CONFIG_DEBUG_FS + if (test_bit(q->cntxt_id - adap->sge.egr_start, adap->sge.blocked_fl)) + goto out; +#endif + + gfp |= __GFP_NOWARN; + node = dev_to_node(adap->pdev_dev); + + if (s->fl_pg_order == 0) + goto alloc_small_pages; + + /* + * Prefer large buffers + */ + while (n) { + pg = alloc_pages_node(node, gfp | __GFP_COMP, s->fl_pg_order); + if (unlikely(!pg)) { + q->large_alloc_failed++; + break; /* fall back to single pages */ + } + + mapping = dma_map_page(adap->pdev_dev, pg, 0, + PAGE_SIZE << s->fl_pg_order, + DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) { + __free_pages(pg, s->fl_pg_order); + q->mapping_err++; + goto out; /* do not try small pages for this error */ + } + mapping |= RX_LARGE_PG_BUF; + *d++ = cpu_to_be64(mapping); + + set_rx_sw_desc(sd, pg, mapping); + sd++; + + q->avail++; + if (++q->pidx == q->size) { + q->pidx = 0; + sd = q->sdesc; + d = q->desc; + } + n--; + } + +alloc_small_pages: + while (n--) { + pg = alloc_pages_node(node, gfp, 0); + if (unlikely(!pg)) { + q->alloc_failed++; + break; + } + + mapping = dma_map_page(adap->pdev_dev, pg, 0, PAGE_SIZE, + DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) { + put_page(pg); + q->mapping_err++; + goto out; + } + *d++ = cpu_to_be64(mapping); + + set_rx_sw_desc(sd, pg, mapping); + sd++; + + q->avail++; + if (++q->pidx == q->size) { + q->pidx = 0; + sd = q->sdesc; + d = q->desc; + } + } + +out: cred = q->avail - cred; + q->pend_cred += cred; + ring_fl_db(adap, q); + + if (unlikely(fl_starving(adap, q))) { + smp_wmb(); + q->low++; + set_bit(q->cntxt_id - adap->sge.egr_start, + adap->sge.starving_fl); + } + + return cred; +} + +static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl) +{ + refill_fl(adap, fl, min(MAX_RX_REFILL, fl_cap(fl) - fl->avail), + GFP_ATOMIC); +} + +/** + * alloc_ring - allocate resources for an SGE descriptor ring + * @dev: the PCI device's core device + * @nelem: the number of descriptors + * @elem_size: the size of each descriptor + * @sw_size: the size of the SW state associated with each ring element + * @phys: the physical address of the allocated ring + * @metadata: address of the array holding the SW state for the ring + * @stat_size: extra space in HW ring for status information + * @node: preferred node for memory allocations + * + * Allocates resources for an SGE descriptor ring, such as Tx queues, + * free buffer lists, or response queues. Each SGE ring requires + * space for its HW descriptors plus, optionally, space for the SW state + * associated with each HW entry (the metadata). The function returns + * three values: the virtual address for the HW ring (the return value + * of the function), the bus address of the HW ring, and the address + * of the SW ring. + */ +static void *alloc_ring(struct device *dev, size_t nelem, size_t elem_size, + size_t sw_size, dma_addr_t *phys, void *metadata, + size_t stat_size, int node) +{ + size_t len = nelem * elem_size + stat_size; + void *s = NULL; + void *p = dma_alloc_coherent(dev, len, phys, GFP_KERNEL); + + if (!p) + return NULL; + if (sw_size) { + s = kcalloc_node(sw_size, nelem, GFP_KERNEL, node); + + if (!s) { + dma_free_coherent(dev, len, p, *phys); + return NULL; + } + } + if (metadata) + *(void **)metadata = s; + return p; +} + +/** + * sgl_len - calculates the size of an SGL of the given capacity + * @n: the number of SGL entries + * + * Calculates the number of flits needed for a scatter/gather list that + * can hold the given number of entries. + */ +static inline unsigned int sgl_len(unsigned int n) +{ + /* A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA + * addresses. The DSGL Work Request starts off with a 32-bit DSGL + * ULPTX header, then Length0, then Address0, then, for 1 <= i <= N, + * repeated sequences of { Length[i], Length[i+1], Address[i], + * Address[i+1] } (this ensures that all addresses are on 64-bit + * boundaries). If N is even, then Length[N+1] should be set to 0 and + * Address[N+1] is omitted. + * + * The following calculation incorporates all of the above. It's + * somewhat hard to follow but, briefly: the "+2" accounts for the + * first two flits which include the DSGL header, Length0 and + * Address0; the "(3*(n-1))/2" covers the main body of list entries (3 + * flits for every pair of the remaining N) +1 if (n-1) is odd; and + * finally the "+((n-1)&1)" adds the one remaining flit needed if + * (n-1) is odd ... + */ + n--; + return (3 * n) / 2 + (n & 1) + 2; +} + +/** + * flits_to_desc - returns the num of Tx descriptors for the given flits + * @n: the number of flits + * + * Returns the number of Tx descriptors needed for the supplied number + * of flits. + */ +static inline unsigned int flits_to_desc(unsigned int n) +{ + BUG_ON(n > SGE_MAX_WR_LEN / 8); + return DIV_ROUND_UP(n, 8); +} + +/** + * is_eth_imm - can an Ethernet packet be sent as immediate data? + * @skb: the packet + * @chip_ver: chip version + * + * Returns whether an Ethernet packet is small enough to fit as + * immediate data. Return value corresponds to headroom required. + */ +static inline int is_eth_imm(const struct sk_buff *skb, unsigned int chip_ver) +{ + int hdrlen = 0; + + if (skb->encapsulation && skb_shinfo(skb)->gso_size && + chip_ver > CHELSIO_T5) { + hdrlen = sizeof(struct cpl_tx_tnl_lso); + hdrlen += sizeof(struct cpl_tx_pkt_core); + } else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { + return 0; + } else { + hdrlen = skb_shinfo(skb)->gso_size ? + sizeof(struct cpl_tx_pkt_lso_core) : 0; + hdrlen += sizeof(struct cpl_tx_pkt); + } + if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen) + return hdrlen; + return 0; +} + +/** + * calc_tx_flits - calculate the number of flits for a packet Tx WR + * @skb: the packet + * @chip_ver: chip version + * + * Returns the number of flits needed for a Tx WR for the given Ethernet + * packet, including the needed WR and CPL headers. + */ +static inline unsigned int calc_tx_flits(const struct sk_buff *skb, + unsigned int chip_ver) +{ + unsigned int flits; + int hdrlen = is_eth_imm(skb, chip_ver); + + /* If the skb is small enough, we can pump it out as a work request + * with only immediate data. In that case we just have to have the + * TX Packet header plus the skb data in the Work Request. + */ + + if (hdrlen) + return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64)); + + /* Otherwise, we're going to have to construct a Scatter gather list + * of the skb body and fragments. We also include the flits necessary + * for the TX Packet Work Request and CPL. We always have a firmware + * Write Header (incorporated as part of the cpl_tx_pkt_lso and + * cpl_tx_pkt structures), followed by either a TX Packet Write CPL + * message or, if we're doing a Large Send Offload, an LSO CPL message + * with an embedded TX Packet Write CPL message. + */ + flits = sgl_len(skb_shinfo(skb)->nr_frags + 1); + if (skb_shinfo(skb)->gso_size) { + if (skb->encapsulation && chip_ver > CHELSIO_T5) { + hdrlen = sizeof(struct fw_eth_tx_pkt_wr) + + sizeof(struct cpl_tx_tnl_lso); + } else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { + u32 pkt_hdrlen; + + pkt_hdrlen = eth_get_headlen(skb->dev, skb->data, + skb_headlen(skb)); + hdrlen = sizeof(struct fw_eth_tx_eo_wr) + + round_up(pkt_hdrlen, 16); + } else { + hdrlen = sizeof(struct fw_eth_tx_pkt_wr) + + sizeof(struct cpl_tx_pkt_lso_core); + } + + hdrlen += sizeof(struct cpl_tx_pkt_core); + flits += (hdrlen / sizeof(__be64)); + } else { + flits += (sizeof(struct fw_eth_tx_pkt_wr) + + sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); + } + return flits; +} + +/** + * calc_tx_descs - calculate the number of Tx descriptors for a packet + * @skb: the packet + * @chip_ver: chip version + * + * Returns the number of Tx descriptors needed for the given Ethernet + * packet, including the needed WR and CPL headers. + */ +static inline unsigned int calc_tx_descs(const struct sk_buff *skb, + unsigned int chip_ver) +{ + return flits_to_desc(calc_tx_flits(skb, chip_ver)); +} + +/** + * cxgb4_write_sgl - populate a scatter/gather list for a packet + * @skb: the packet + * @q: the Tx queue we are writing into + * @sgl: starting location for writing the SGL + * @end: points right after the end of the SGL + * @start: start offset into skb main-body data to include in the SGL + * @addr: the list of bus addresses for the SGL elements + * + * Generates a gather list for the buffers that make up a packet. + * The caller must provide adequate space for the SGL that will be written. + * The SGL includes all of the packet's page fragments and the data in its + * main body except for the first @start bytes. @sgl must be 16-byte + * aligned and within a Tx descriptor with available space. @end points + * right after the end of the SGL but does not account for any potential + * wrap around, i.e., @end > @sgl. + */ +void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q, + struct ulptx_sgl *sgl, u64 *end, unsigned int start, + const dma_addr_t *addr) +{ + unsigned int i, len; + struct ulptx_sge_pair *to; + const struct skb_shared_info *si = skb_shinfo(skb); + unsigned int nfrags = si->nr_frags; + struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1]; + + len = skb_headlen(skb) - start; + if (likely(len)) { + sgl->len0 = htonl(len); + sgl->addr0 = cpu_to_be64(addr[0] + start); + nfrags++; + } else { + sgl->len0 = htonl(skb_frag_size(&si->frags[0])); + sgl->addr0 = cpu_to_be64(addr[1]); + } + + sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | + ULPTX_NSGE_V(nfrags)); + if (likely(--nfrags == 0)) + return; + /* + * Most of the complexity below deals with the possibility we hit the + * end of the queue in the middle of writing the SGL. For this case + * only we create the SGL in a temporary buffer and then copy it. + */ + to = (u8 *)end > (u8 *)q->stat ? buf : sgl->sge; + + for (i = (nfrags != si->nr_frags); nfrags >= 2; nfrags -= 2, to++) { + to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i])); + to->len[1] = cpu_to_be32(skb_frag_size(&si->frags[++i])); + to->addr[0] = cpu_to_be64(addr[i]); + to->addr[1] = cpu_to_be64(addr[++i]); + } + if (nfrags) { + to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i])); + to->len[1] = cpu_to_be32(0); + to->addr[0] = cpu_to_be64(addr[i + 1]); + } + if (unlikely((u8 *)end > (u8 *)q->stat)) { + unsigned int part0 = (u8 *)q->stat - (u8 *)sgl->sge, part1; + + if (likely(part0)) + memcpy(sgl->sge, buf, part0); + part1 = (u8 *)end - (u8 *)q->stat; + memcpy(q->desc, (u8 *)buf + part0, part1); + end = (void *)q->desc + part1; + } + if ((uintptr_t)end & 8) /* 0-pad to multiple of 16 */ + *end = 0; +} +EXPORT_SYMBOL(cxgb4_write_sgl); + +/* cxgb4_write_partial_sgl - populate SGL for partial packet + * @skb: the packet + * @q: the Tx queue we are writing into + * @sgl: starting location for writing the SGL + * @end: points right after the end of the SGL + * @addr: the list of bus addresses for the SGL elements + * @start: start offset in the SKB where partial data starts + * @len: length of data from @start to send out + * + * This API will handle sending out partial data of a skb if required. + * Unlike cxgb4_write_sgl, @start can be any offset into the skb data, + * and @len will decide how much data after @start offset to send out. + */ +void cxgb4_write_partial_sgl(const struct sk_buff *skb, struct sge_txq *q, + struct ulptx_sgl *sgl, u64 *end, + const dma_addr_t *addr, u32 start, u32 len) +{ + struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1] = {0}, *to; + u32 frag_size, skb_linear_data_len = skb_headlen(skb); + struct skb_shared_info *si = skb_shinfo(skb); + u8 i = 0, frag_idx = 0, nfrags = 0; + skb_frag_t *frag; + + /* Fill the first SGL either from linear data or from partial + * frag based on @start. + */ + if (unlikely(start < skb_linear_data_len)) { + frag_size = min(len, skb_linear_data_len - start); + sgl->len0 = htonl(frag_size); + sgl->addr0 = cpu_to_be64(addr[0] + start); + len -= frag_size; + nfrags++; + } else { + start -= skb_linear_data_len; + frag = &si->frags[frag_idx]; + frag_size = skb_frag_size(frag); + /* find the first frag */ + while (start >= frag_size) { + start -= frag_size; + frag_idx++; + frag = &si->frags[frag_idx]; + frag_size = skb_frag_size(frag); + } + + frag_size = min(len, skb_frag_size(frag) - start); + sgl->len0 = cpu_to_be32(frag_size); + sgl->addr0 = cpu_to_be64(addr[frag_idx + 1] + start); + len -= frag_size; + nfrags++; + frag_idx++; + } + + /* If the entire partial data fit in one SGL, then send it out + * now. + */ + if (!len) + goto done; + + /* Most of the complexity below deals with the possibility we hit the + * end of the queue in the middle of writing the SGL. For this case + * only we create the SGL in a temporary buffer and then copy it. + */ + to = (u8 *)end > (u8 *)q->stat ? buf : sgl->sge; + + /* If the skb couldn't fit in first SGL completely, fill the + * rest of the frags in subsequent SGLs. Note that each SGL + * pair can store 2 frags. + */ + while (len) { + frag_size = min(len, skb_frag_size(&si->frags[frag_idx])); + to->len[i & 1] = cpu_to_be32(frag_size); + to->addr[i & 1] = cpu_to_be64(addr[frag_idx + 1]); + if (i && (i & 1)) + to++; + nfrags++; + frag_idx++; + i++; + len -= frag_size; + } + + /* If we ended in an odd boundary, then set the second SGL's + * length in the pair to 0. + */ + if (i & 1) + to->len[1] = cpu_to_be32(0); + + /* Copy from temporary buffer to Tx ring, in case we hit the + * end of the queue in the middle of writing the SGL. + */ + if (unlikely((u8 *)end > (u8 *)q->stat)) { + u32 part0 = (u8 *)q->stat - (u8 *)sgl->sge, part1; + + if (likely(part0)) + memcpy(sgl->sge, buf, part0); + part1 = (u8 *)end - (u8 *)q->stat; + memcpy(q->desc, (u8 *)buf + part0, part1); + end = (void *)q->desc + part1; + } + + /* 0-pad to multiple of 16 */ + if ((uintptr_t)end & 8) + *end = 0; +done: + sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | + ULPTX_NSGE_V(nfrags)); +} +EXPORT_SYMBOL(cxgb4_write_partial_sgl); + +/* This function copies 64 byte coalesced work request to + * memory mapped BAR2 space. For coalesced WR SGE fetches + * data from the FIFO instead of from Host. + */ +static void cxgb_pio_copy(u64 __iomem *dst, u64 *src) +{ + int count = 8; + + while (count) { + writeq(*src, dst); + src++; + dst++; + count--; + } +} + +/** + * cxgb4_ring_tx_db - check and potentially ring a Tx queue's doorbell + * @adap: the adapter + * @q: the Tx queue + * @n: number of new descriptors to give to HW + * + * Ring the doorbel for a Tx queue. + */ +inline void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n) +{ + /* Make sure that all writes to the TX Descriptors are committed + * before we tell the hardware about them. + */ + wmb(); + + /* If we don't have access to the new User Doorbell (T5+), use the old + * doorbell mechanism; otherwise use the new BAR2 mechanism. + */ + if (unlikely(q->bar2_addr == NULL)) { + u32 val = PIDX_V(n); + unsigned long flags; + + /* For T4 we need to participate in the Doorbell Recovery + * mechanism. + */ + spin_lock_irqsave(&q->db_lock, flags); + if (!q->db_disabled) + t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), + QID_V(q->cntxt_id) | val); + else + q->db_pidx_inc += n; + q->db_pidx = q->pidx; + spin_unlock_irqrestore(&q->db_lock, flags); + } else { + u32 val = PIDX_T5_V(n); + + /* T4 and later chips share the same PIDX field offset within + * the doorbell, but T5 and later shrank the field in order to + * gain a bit for Doorbell Priority. The field was absurdly + * large in the first place (14 bits) so we just use the T5 + * and later limits and warn if a Queue ID is too large. + */ + WARN_ON(val & DBPRIO_F); + + /* If we're only writing a single TX Descriptor and we can use + * Inferred QID registers, we can use the Write Combining + * Gather Buffer; otherwise we use the simple doorbell. + */ + if (n == 1 && q->bar2_qid == 0) { + int index = (q->pidx + ? (q->pidx - 1) + : (q->size - 1)); + u64 *wr = (u64 *)&q->desc[index]; + + cxgb_pio_copy((u64 __iomem *) + (q->bar2_addr + SGE_UDB_WCDOORBELL), + wr); + } else { + writel(val | QID_V(q->bar2_qid), + q->bar2_addr + SGE_UDB_KDOORBELL); + } + + /* This Write Memory Barrier will force the write to the User + * Doorbell area to be flushed. This is needed to prevent + * writes on different CPUs for the same queue from hitting + * the adapter out of order. This is required when some Work + * Requests take the Write Combine Gather Buffer path (user + * doorbell area offset [SGE_UDB_WCDOORBELL..+63]) and some + * take the traditional path where we simply increment the + * PIDX (User Doorbell area SGE_UDB_KDOORBELL) and have the + * hardware DMA read the actual Work Request. + */ + wmb(); + } +} +EXPORT_SYMBOL(cxgb4_ring_tx_db); + +/** + * cxgb4_inline_tx_skb - inline a packet's data into Tx descriptors + * @skb: the packet + * @q: the Tx queue where the packet will be inlined + * @pos: starting position in the Tx queue where to inline the packet + * + * Inline a packet's contents directly into Tx descriptors, starting at + * the given position within the Tx DMA ring. + * Most of the complexity of this operation is dealing with wrap arounds + * in the middle of the packet we want to inline. + */ +void cxgb4_inline_tx_skb(const struct sk_buff *skb, + const struct sge_txq *q, void *pos) +{ + int left = (void *)q->stat - pos; + u64 *p; + + if (likely(skb->len <= left)) { + if (likely(!skb->data_len)) + skb_copy_from_linear_data(skb, pos, skb->len); + else + skb_copy_bits(skb, 0, pos, skb->len); + pos += skb->len; + } else { + skb_copy_bits(skb, 0, pos, left); + skb_copy_bits(skb, left, q->desc, skb->len - left); + pos = (void *)q->desc + (skb->len - left); + } + + /* 0-pad to multiple of 16 */ + p = PTR_ALIGN(pos, 8); + if ((uintptr_t)p & 8) + *p = 0; +} +EXPORT_SYMBOL(cxgb4_inline_tx_skb); + +static void *inline_tx_skb_header(const struct sk_buff *skb, + const struct sge_txq *q, void *pos, + int length) +{ + u64 *p; + int left = (void *)q->stat - pos; + + if (likely(length <= left)) { + memcpy(pos, skb->data, length); + pos += length; + } else { + memcpy(pos, skb->data, left); + memcpy(q->desc, skb->data + left, length - left); + pos = (void *)q->desc + (length - left); + } + /* 0-pad to multiple of 16 */ + p = PTR_ALIGN(pos, 8); + if ((uintptr_t)p & 8) { + *p = 0; + return p + 1; + } + return p; +} + +/* + * Figure out what HW csum a packet wants and return the appropriate control + * bits. + */ +static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb) +{ + int csum_type; + bool inner_hdr_csum = false; + u16 proto, ver; + + if (skb->encapsulation && + (CHELSIO_CHIP_VERSION(chip) > CHELSIO_T5)) + inner_hdr_csum = true; + + if (inner_hdr_csum) { + ver = inner_ip_hdr(skb)->version; + proto = (ver == 4) ? inner_ip_hdr(skb)->protocol : + inner_ipv6_hdr(skb)->nexthdr; + } else { + ver = ip_hdr(skb)->version; + proto = (ver == 4) ? ip_hdr(skb)->protocol : + ipv6_hdr(skb)->nexthdr; + } + + if (ver == 4) { + if (proto == IPPROTO_TCP) + csum_type = TX_CSUM_TCPIP; + else if (proto == IPPROTO_UDP) + csum_type = TX_CSUM_UDPIP; + else { +nocsum: /* + * unknown protocol, disable HW csum + * and hope a bad packet is detected + */ + return TXPKT_L4CSUM_DIS_F; + } + } else { + /* + * this doesn't work with extension headers + */ + if (proto == IPPROTO_TCP) + csum_type = TX_CSUM_TCPIP6; + else if (proto == IPPROTO_UDP) + csum_type = TX_CSUM_UDPIP6; + else + goto nocsum; + } + + if (likely(csum_type >= TX_CSUM_TCPIP)) { + int eth_hdr_len, l4_len; + u64 hdr_len; + + if (inner_hdr_csum) { + /* This allows checksum offload for all encapsulated + * packets like GRE etc.. + */ + l4_len = skb_inner_network_header_len(skb); + eth_hdr_len = skb_inner_network_offset(skb) - ETH_HLEN; + } else { + l4_len = skb_network_header_len(skb); + eth_hdr_len = skb_network_offset(skb) - ETH_HLEN; + } + hdr_len = TXPKT_IPHDR_LEN_V(l4_len); + + if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5) + hdr_len |= TXPKT_ETHHDR_LEN_V(eth_hdr_len); + else + hdr_len |= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len); + return TXPKT_CSUM_TYPE_V(csum_type) | hdr_len; + } else { + int start = skb_transport_offset(skb); + + return TXPKT_CSUM_TYPE_V(csum_type) | + TXPKT_CSUM_START_V(start) | + TXPKT_CSUM_LOC_V(start + skb->csum_offset); + } +} + +static void eth_txq_stop(struct sge_eth_txq *q) +{ + netif_tx_stop_queue(q->txq); + q->q.stops++; +} + +static inline void txq_advance(struct sge_txq *q, unsigned int n) +{ + q->in_use += n; + q->pidx += n; + if (q->pidx >= q->size) + q->pidx -= q->size; +} + +#ifdef CONFIG_CHELSIO_T4_FCOE +static inline int +cxgb_fcoe_offload(struct sk_buff *skb, struct adapter *adap, + const struct port_info *pi, u64 *cntrl) +{ + const struct cxgb_fcoe *fcoe = &pi->fcoe; + + if (!(fcoe->flags & CXGB_FCOE_ENABLED)) + return 0; + + if (skb->protocol != htons(ETH_P_FCOE)) + return 0; + + skb_reset_mac_header(skb); + skb->mac_len = sizeof(struct ethhdr); + + skb_set_network_header(skb, skb->mac_len); + skb_set_transport_header(skb, skb->mac_len + sizeof(struct fcoe_hdr)); + + if (!cxgb_fcoe_sof_eof_supported(adap, skb)) + return -ENOTSUPP; + + /* FC CRC offload */ + *cntrl = TXPKT_CSUM_TYPE_V(TX_CSUM_FCOE) | + TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F | + TXPKT_CSUM_START_V(CXGB_FCOE_TXPKT_CSUM_START) | + TXPKT_CSUM_END_V(CXGB_FCOE_TXPKT_CSUM_END) | + TXPKT_CSUM_LOC_V(CXGB_FCOE_TXPKT_CSUM_END); + return 0; +} +#endif /* CONFIG_CHELSIO_T4_FCOE */ + +/* Returns tunnel type if hardware supports offloading of the same. + * It is called only for T5 and onwards. + */ +enum cpl_tx_tnl_lso_type cxgb_encap_offload_supported(struct sk_buff *skb) +{ + u8 l4_hdr = 0; + enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE; + struct port_info *pi = netdev_priv(skb->dev); + struct adapter *adapter = pi->adapter; + + if (skb->inner_protocol_type != ENCAP_TYPE_ETHER || + skb->inner_protocol != htons(ETH_P_TEB)) + return tnl_type; + + switch (vlan_get_protocol(skb)) { + case htons(ETH_P_IP): + l4_hdr = ip_hdr(skb)->protocol; + break; + case htons(ETH_P_IPV6): + l4_hdr = ipv6_hdr(skb)->nexthdr; + break; + default: + return tnl_type; + } + + switch (l4_hdr) { + case IPPROTO_UDP: + if (adapter->vxlan_port == udp_hdr(skb)->dest) + tnl_type = TX_TNL_TYPE_VXLAN; + else if (adapter->geneve_port == udp_hdr(skb)->dest) + tnl_type = TX_TNL_TYPE_GENEVE; + break; + default: + return tnl_type; + } + + return tnl_type; +} + +static inline void t6_fill_tnl_lso(struct sk_buff *skb, + struct cpl_tx_tnl_lso *tnl_lso, + enum cpl_tx_tnl_lso_type tnl_type) +{ + u32 val; + int in_eth_xtra_len; + int l3hdr_len = skb_network_header_len(skb); + int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN; + const struct skb_shared_info *ssi = skb_shinfo(skb); + bool v6 = (ip_hdr(skb)->version == 6); + + val = CPL_TX_TNL_LSO_OPCODE_V(CPL_TX_TNL_LSO) | + CPL_TX_TNL_LSO_FIRST_F | + CPL_TX_TNL_LSO_LAST_F | + (v6 ? CPL_TX_TNL_LSO_IPV6OUT_F : 0) | + CPL_TX_TNL_LSO_ETHHDRLENOUT_V(eth_xtra_len / 4) | + CPL_TX_TNL_LSO_IPHDRLENOUT_V(l3hdr_len / 4) | + (v6 ? 0 : CPL_TX_TNL_LSO_IPHDRCHKOUT_F) | + CPL_TX_TNL_LSO_IPLENSETOUT_F | + (v6 ? 0 : CPL_TX_TNL_LSO_IPIDINCOUT_F); + tnl_lso->op_to_IpIdSplitOut = htonl(val); + + tnl_lso->IpIdOffsetOut = 0; + + /* Get the tunnel header length */ + val = skb_inner_mac_header(skb) - skb_mac_header(skb); + in_eth_xtra_len = skb_inner_network_header(skb) - + skb_inner_mac_header(skb) - ETH_HLEN; + + switch (tnl_type) { + case TX_TNL_TYPE_VXLAN: + case TX_TNL_TYPE_GENEVE: + tnl_lso->UdpLenSetOut_to_TnlHdrLen = + htons(CPL_TX_TNL_LSO_UDPCHKCLROUT_F | + CPL_TX_TNL_LSO_UDPLENSETOUT_F); + break; + default: + tnl_lso->UdpLenSetOut_to_TnlHdrLen = 0; + break; + } + + tnl_lso->UdpLenSetOut_to_TnlHdrLen |= + htons(CPL_TX_TNL_LSO_TNLHDRLEN_V(val) | + CPL_TX_TNL_LSO_TNLTYPE_V(tnl_type)); + + tnl_lso->r1 = 0; + + val = CPL_TX_TNL_LSO_ETHHDRLEN_V(in_eth_xtra_len / 4) | + CPL_TX_TNL_LSO_IPV6_V(inner_ip_hdr(skb)->version == 6) | + CPL_TX_TNL_LSO_IPHDRLEN_V(skb_inner_network_header_len(skb) / 4) | + CPL_TX_TNL_LSO_TCPHDRLEN_V(inner_tcp_hdrlen(skb) / 4); + tnl_lso->Flow_to_TcpHdrLen = htonl(val); + + tnl_lso->IpIdOffset = htons(0); + + tnl_lso->IpIdSplit_to_Mss = htons(CPL_TX_TNL_LSO_MSS_V(ssi->gso_size)); + tnl_lso->TCPSeqOffset = htonl(0); + tnl_lso->EthLenOffset_Size = htonl(CPL_TX_TNL_LSO_SIZE_V(skb->len)); +} + +static inline void *write_tso_wr(struct adapter *adap, struct sk_buff *skb, + struct cpl_tx_pkt_lso_core *lso) +{ + int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN; + int l3hdr_len = skb_network_header_len(skb); + const struct skb_shared_info *ssi; + bool ipv6 = false; + + ssi = skb_shinfo(skb); + if (ssi->gso_type & SKB_GSO_TCPV6) + ipv6 = true; + + lso->lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) | + LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F | + LSO_IPV6_V(ipv6) | + LSO_ETHHDR_LEN_V(eth_xtra_len / 4) | + LSO_IPHDR_LEN_V(l3hdr_len / 4) | + LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff)); + lso->ipid_ofst = htons(0); + lso->mss = htons(ssi->gso_size); + lso->seqno_offset = htonl(0); + if (is_t4(adap->params.chip)) + lso->len = htonl(skb->len); + else + lso->len = htonl(LSO_T5_XFER_SIZE_V(skb->len)); + + return (void *)(lso + 1); +} + +/** + * t4_sge_eth_txq_egress_update - handle Ethernet TX Queue update + * @adap: the adapter + * @eq: the Ethernet TX Queue + * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1 + * + * We're typically called here to update the state of an Ethernet TX + * Queue with respect to the hardware's progress in consuming the TX + * Work Requests that we've put on that Egress Queue. This happens + * when we get Egress Queue Update messages and also prophylactically + * in regular timer-based Ethernet TX Queue maintenance. + */ +int t4_sge_eth_txq_egress_update(struct adapter *adap, struct sge_eth_txq *eq, + int maxreclaim) +{ + unsigned int reclaimed, hw_cidx; + struct sge_txq *q = &eq->q; + int hw_in_use; + + if (!q->in_use || !__netif_tx_trylock(eq->txq)) + return 0; + + /* Reclaim pending completed TX Descriptors. */ + reclaimed = reclaim_completed_tx(adap, &eq->q, maxreclaim, true); + + hw_cidx = ntohs(READ_ONCE(q->stat->cidx)); + hw_in_use = q->pidx - hw_cidx; + if (hw_in_use < 0) + hw_in_use += q->size; + + /* If the TX Queue is currently stopped and there's now more than half + * the queue available, restart it. Otherwise bail out since the rest + * of what we want do here is with the possibility of shipping any + * currently buffered Coalesced TX Work Request. + */ + if (netif_tx_queue_stopped(eq->txq) && hw_in_use < (q->size / 2)) { + netif_tx_wake_queue(eq->txq); + eq->q.restarts++; + } + + __netif_tx_unlock(eq->txq); + return reclaimed; +} + +static inline int cxgb4_validate_skb(struct sk_buff *skb, + struct net_device *dev, + u32 min_pkt_len) +{ + u32 max_pkt_len; + + /* The chip min packet length is 10 octets but some firmware + * commands have a minimum packet length requirement. So, play + * safe and reject anything shorter than @min_pkt_len. + */ + if (unlikely(skb->len < min_pkt_len)) + return -EINVAL; + + /* Discard the packet if the length is greater than mtu */ + max_pkt_len = ETH_HLEN + dev->mtu; + + if (skb_vlan_tagged(skb)) + max_pkt_len += VLAN_HLEN; + + if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len))) + return -EINVAL; + + return 0; +} + +static void *write_eo_udp_wr(struct sk_buff *skb, struct fw_eth_tx_eo_wr *wr, + u32 hdr_len) +{ + wr->u.udpseg.type = FW_ETH_TX_EO_TYPE_UDPSEG; + wr->u.udpseg.ethlen = skb_network_offset(skb); + wr->u.udpseg.iplen = cpu_to_be16(skb_network_header_len(skb)); + wr->u.udpseg.udplen = sizeof(struct udphdr); + wr->u.udpseg.rtplen = 0; + wr->u.udpseg.r4 = 0; + if (skb_shinfo(skb)->gso_size) + wr->u.udpseg.mss = cpu_to_be16(skb_shinfo(skb)->gso_size); + else + wr->u.udpseg.mss = cpu_to_be16(skb->len - hdr_len); + wr->u.udpseg.schedpktsize = wr->u.udpseg.mss; + wr->u.udpseg.plen = cpu_to_be32(skb->len - hdr_len); + + return (void *)(wr + 1); +} + +/** + * cxgb4_eth_xmit - add a packet to an Ethernet Tx queue + * @skb: the packet + * @dev: the egress net device + * + * Add a packet to an SGE Ethernet Tx queue. Runs with softirqs disabled. + */ +static netdev_tx_t cxgb4_eth_xmit(struct sk_buff *skb, struct net_device *dev) +{ + enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE; + bool ptp_enabled = is_ptp_enabled(skb, dev); + unsigned int last_desc, flits, ndesc; + u32 wr_mid, ctrl0, op, sgl_off = 0; + const struct skb_shared_info *ssi; + int len, qidx, credits, ret, left; + struct tx_sw_desc *sgl_sdesc; + struct fw_eth_tx_eo_wr *eowr; + struct fw_eth_tx_pkt_wr *wr; + struct cpl_tx_pkt_core *cpl; + const struct port_info *pi; + bool immediate = false; + u64 cntrl, *end, *sgl; + struct sge_eth_txq *q; + unsigned int chip_ver; + struct adapter *adap; + + ret = cxgb4_validate_skb(skb, dev, ETH_HLEN); + if (ret) + goto out_free; + + pi = netdev_priv(dev); + adap = pi->adapter; + ssi = skb_shinfo(skb); +#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) + if (xfrm_offload(skb) && !ssi->gso_size) + return adap->uld[CXGB4_ULD_IPSEC].tx_handler(skb, dev); +#endif /* CHELSIO_IPSEC_INLINE */ + +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + if (cxgb4_is_ktls_skb(skb) && + (skb->len - skb_tcp_all_headers(skb))) + return adap->uld[CXGB4_ULD_KTLS].tx_handler(skb, dev); +#endif /* CHELSIO_TLS_DEVICE */ + + qidx = skb_get_queue_mapping(skb); + if (ptp_enabled) { + if (!(adap->ptp_tx_skb)) { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + adap->ptp_tx_skb = skb_get(skb); + } else { + goto out_free; + } + q = &adap->sge.ptptxq; + } else { + q = &adap->sge.ethtxq[qidx + pi->first_qset]; + } + skb_tx_timestamp(skb); + + reclaim_completed_tx(adap, &q->q, -1, true); + cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; + +#ifdef CONFIG_CHELSIO_T4_FCOE + ret = cxgb_fcoe_offload(skb, adap, pi, &cntrl); + if (unlikely(ret == -EOPNOTSUPP)) + goto out_free; +#endif /* CONFIG_CHELSIO_T4_FCOE */ + + chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip); + flits = calc_tx_flits(skb, chip_ver); + ndesc = flits_to_desc(flits); + credits = txq_avail(&q->q) - ndesc; + + if (unlikely(credits < 0)) { + eth_txq_stop(q); + dev_err(adap->pdev_dev, + "%s: Tx ring %u full while queue awake!\n", + dev->name, qidx); + return NETDEV_TX_BUSY; + } + + if (is_eth_imm(skb, chip_ver)) + immediate = true; + + if (skb->encapsulation && chip_ver > CHELSIO_T5) + tnl_type = cxgb_encap_offload_supported(skb); + + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + sgl_sdesc = &q->q.sdesc[last_desc]; + + if (!immediate && + unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { + memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); + q->mapping_err++; + goto out_free; + } + + wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2)); + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + /* After we're done injecting the Work Request for this + * packet, we'll be below our "stop threshold" so stop the TX + * Queue now and schedule a request for an SGE Egress Queue + * Update message. The queue will get started later on when + * the firmware processes this Work Request and sends us an + * Egress Queue Status Update message indicating that space + * has opened up. + */ + eth_txq_stop(q); + if (chip_ver > CHELSIO_T5) + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + wr = (void *)&q->q.desc[q->q.pidx]; + eowr = (void *)&q->q.desc[q->q.pidx]; + wr->equiq_to_len16 = htonl(wr_mid); + wr->r3 = cpu_to_be64(0); + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) + end = (u64 *)eowr + flits; + else + end = (u64 *)wr + flits; + + len = immediate ? skb->len : 0; + len += sizeof(*cpl); + if (ssi->gso_size && !(ssi->gso_type & SKB_GSO_UDP_L4)) { + struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); + struct cpl_tx_tnl_lso *tnl_lso = (void *)(wr + 1); + + if (tnl_type) + len += sizeof(*tnl_lso); + else + len += sizeof(*lso); + + wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | + FW_WR_IMMDLEN_V(len)); + if (tnl_type) { + struct iphdr *iph = ip_hdr(skb); + + t6_fill_tnl_lso(skb, tnl_lso, tnl_type); + cpl = (void *)(tnl_lso + 1); + /* Driver is expected to compute partial checksum that + * does not include the IP Total Length. + */ + if (iph->version == 4) { + iph->check = 0; + iph->tot_len = 0; + iph->check = ~ip_fast_csum((u8 *)iph, iph->ihl); + } + if (skb->ip_summed == CHECKSUM_PARTIAL) + cntrl = hwcsum(adap->params.chip, skb); + } else { + cpl = write_tso_wr(adap, skb, lso); + cntrl = hwcsum(adap->params.chip, skb); + } + sgl = (u64 *)(cpl + 1); /* sgl start here */ + q->tso++; + q->tx_cso += ssi->gso_segs; + } else if (ssi->gso_size) { + u64 *start; + u32 hdrlen; + + hdrlen = eth_get_headlen(dev, skb->data, skb_headlen(skb)); + len += hdrlen; + wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_EO_WR) | + FW_ETH_TX_EO_WR_IMMDLEN_V(len)); + cpl = write_eo_udp_wr(skb, eowr, hdrlen); + cntrl = hwcsum(adap->params.chip, skb); + + start = (u64 *)(cpl + 1); + sgl = (u64 *)inline_tx_skb_header(skb, &q->q, (void *)start, + hdrlen); + if (unlikely(start > sgl)) { + left = (u8 *)end - (u8 *)q->q.stat; + end = (void *)q->q.desc + left; + } + sgl_off = hdrlen; + q->uso++; + q->tx_cso += ssi->gso_segs; + } else { + if (ptp_enabled) + op = FW_PTP_TX_PKT_WR; + else + op = FW_ETH_TX_PKT_WR; + wr->op_immdlen = htonl(FW_WR_OP_V(op) | + FW_WR_IMMDLEN_V(len)); + cpl = (void *)(wr + 1); + sgl = (u64 *)(cpl + 1); + if (skb->ip_summed == CHECKSUM_PARTIAL) { + cntrl = hwcsum(adap->params.chip, skb) | + TXPKT_IPCSUM_DIS_F; + q->tx_cso++; + } + } + + if (unlikely((u8 *)sgl >= (u8 *)q->q.stat)) { + /* If current position is already at the end of the + * txq, reset the current to point to start of the queue + * and update the end ptr as well. + */ + left = (u8 *)end - (u8 *)q->q.stat; + end = (void *)q->q.desc + left; + sgl = (void *)q->q.desc; + } + + if (skb_vlan_tag_present(skb)) { + q->vlan_ins++; + cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb)); +#ifdef CONFIG_CHELSIO_T4_FCOE + if (skb->protocol == htons(ETH_P_FCOE)) + cntrl |= TXPKT_VLAN_V( + ((skb->priority & 0x7) << VLAN_PRIO_SHIFT)); +#endif /* CONFIG_CHELSIO_T4_FCOE */ + } + + ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) | + TXPKT_PF_V(adap->pf); + if (ptp_enabled) + ctrl0 |= TXPKT_TSTAMP_F; +#ifdef CONFIG_CHELSIO_T4_DCB + if (is_t4(adap->params.chip)) + ctrl0 |= TXPKT_OVLAN_IDX_V(q->dcb_prio); + else + ctrl0 |= TXPKT_T5_OVLAN_IDX_V(q->dcb_prio); +#endif + cpl->ctrl0 = htonl(ctrl0); + cpl->pack = htons(0); + cpl->len = htons(skb->len); + cpl->ctrl1 = cpu_to_be64(cntrl); + + if (immediate) { + cxgb4_inline_tx_skb(skb, &q->q, sgl); + dev_consume_skb_any(skb); + } else { + cxgb4_write_sgl(skb, &q->q, (void *)sgl, end, sgl_off, + sgl_sdesc->addr); + skb_orphan(skb); + sgl_sdesc->skb = skb; + } + + txq_advance(&q->q, ndesc); + + cxgb4_ring_tx_db(adap, &q->q, ndesc); + return NETDEV_TX_OK; + +out_free: + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +} + +/* Constants ... */ +enum { + /* Egress Queue sizes, producer and consumer indices are all in units + * of Egress Context Units bytes. Note that as far as the hardware is + * concerned, the free list is an Egress Queue (the host produces free + * buffers which the hardware consumes) and free list entries are + * 64-bit PCI DMA addresses. + */ + EQ_UNIT = SGE_EQ_IDXSIZE, + FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64), + TXD_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64), + + T4VF_ETHTXQ_MAX_HDR = (sizeof(struct fw_eth_tx_pkt_vm_wr) + + sizeof(struct cpl_tx_pkt_lso_core) + + sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64), +}; + +/** + * t4vf_is_eth_imm - can an Ethernet packet be sent as immediate data? + * @skb: the packet + * + * Returns whether an Ethernet packet is small enough to fit completely as + * immediate data. + */ +static inline int t4vf_is_eth_imm(const struct sk_buff *skb) +{ + /* The VF Driver uses the FW_ETH_TX_PKT_VM_WR firmware Work Request + * which does not accommodate immediate data. We could dike out all + * of the support code for immediate data but that would tie our hands + * too much if we ever want to enhace the firmware. It would also + * create more differences between the PF and VF Drivers. + */ + return false; +} + +/** + * t4vf_calc_tx_flits - calculate the number of flits for a packet TX WR + * @skb: the packet + * + * Returns the number of flits needed for a TX Work Request for the + * given Ethernet packet, including the needed WR and CPL headers. + */ +static inline unsigned int t4vf_calc_tx_flits(const struct sk_buff *skb) +{ + unsigned int flits; + + /* If the skb is small enough, we can pump it out as a work request + * with only immediate data. In that case we just have to have the + * TX Packet header plus the skb data in the Work Request. + */ + if (t4vf_is_eth_imm(skb)) + return DIV_ROUND_UP(skb->len + sizeof(struct cpl_tx_pkt), + sizeof(__be64)); + + /* Otherwise, we're going to have to construct a Scatter gather list + * of the skb body and fragments. We also include the flits necessary + * for the TX Packet Work Request and CPL. We always have a firmware + * Write Header (incorporated as part of the cpl_tx_pkt_lso and + * cpl_tx_pkt structures), followed by either a TX Packet Write CPL + * message or, if we're doing a Large Send Offload, an LSO CPL message + * with an embedded TX Packet Write CPL message. + */ + flits = sgl_len(skb_shinfo(skb)->nr_frags + 1); + if (skb_shinfo(skb)->gso_size) + flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) + + sizeof(struct cpl_tx_pkt_lso_core) + + sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); + else + flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) + + sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); + return flits; +} + +/** + * cxgb4_vf_eth_xmit - add a packet to an Ethernet TX queue + * @skb: the packet + * @dev: the egress net device + * + * Add a packet to an SGE Ethernet TX queue. Runs with softirqs disabled. + */ +static netdev_tx_t cxgb4_vf_eth_xmit(struct sk_buff *skb, + struct net_device *dev) +{ + unsigned int last_desc, flits, ndesc; + const struct skb_shared_info *ssi; + struct fw_eth_tx_pkt_vm_wr *wr; + struct tx_sw_desc *sgl_sdesc; + struct cpl_tx_pkt_core *cpl; + const struct port_info *pi; + struct sge_eth_txq *txq; + struct adapter *adapter; + int qidx, credits, ret; + size_t fw_hdr_copy_len; + unsigned int chip_ver; + u64 cntrl, *end; + u32 wr_mid; + + /* The chip minimum packet length is 10 octets but the firmware + * command that we are using requires that we copy the Ethernet header + * (including the VLAN tag) into the header so we reject anything + * smaller than that ... + */ + BUILD_BUG_ON(sizeof(wr->firmware) != + (sizeof(wr->ethmacdst) + sizeof(wr->ethmacsrc) + + sizeof(wr->ethtype) + sizeof(wr->vlantci))); + fw_hdr_copy_len = sizeof(wr->firmware); + ret = cxgb4_validate_skb(skb, dev, fw_hdr_copy_len); + if (ret) + goto out_free; + + /* Figure out which TX Queue we're going to use. */ + pi = netdev_priv(dev); + adapter = pi->adapter; + qidx = skb_get_queue_mapping(skb); + WARN_ON(qidx >= pi->nqsets); + txq = &adapter->sge.ethtxq[pi->first_qset + qidx]; + + /* Take this opportunity to reclaim any TX Descriptors whose DMA + * transfers have completed. + */ + reclaim_completed_tx(adapter, &txq->q, -1, true); + + /* Calculate the number of flits and TX Descriptors we're going to + * need along with how many TX Descriptors will be left over after + * we inject our Work Request. + */ + flits = t4vf_calc_tx_flits(skb); + ndesc = flits_to_desc(flits); + credits = txq_avail(&txq->q) - ndesc; + + if (unlikely(credits < 0)) { + /* Not enough room for this packet's Work Request. Stop the + * TX Queue and return a "busy" condition. The queue will get + * started later on when the firmware informs us that space + * has opened up. + */ + eth_txq_stop(txq); + dev_err(adapter->pdev_dev, + "%s: TX ring %u full while queue awake!\n", + dev->name, qidx); + return NETDEV_TX_BUSY; + } + + last_desc = txq->q.pidx + ndesc - 1; + if (last_desc >= txq->q.size) + last_desc -= txq->q.size; + sgl_sdesc = &txq->q.sdesc[last_desc]; + + if (!t4vf_is_eth_imm(skb) && + unlikely(cxgb4_map_skb(adapter->pdev_dev, skb, + sgl_sdesc->addr) < 0)) { + /* We need to map the skb into PCI DMA space (because it can't + * be in-lined directly into the Work Request) and the mapping + * operation failed. Record the error and drop the packet. + */ + memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); + txq->mapping_err++; + goto out_free; + } + + chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip); + wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2)); + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + /* After we're done injecting the Work Request for this + * packet, we'll be below our "stop threshold" so stop the TX + * Queue now and schedule a request for an SGE Egress Queue + * Update message. The queue will get started later on when + * the firmware processes this Work Request and sends us an + * Egress Queue Status Update message indicating that space + * has opened up. + */ + eth_txq_stop(txq); + if (chip_ver > CHELSIO_T5) + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + /* Start filling in our Work Request. Note that we do _not_ handle + * the WR Header wrapping around the TX Descriptor Ring. If our + * maximum header size ever exceeds one TX Descriptor, we'll need to + * do something else here. + */ + WARN_ON(DIV_ROUND_UP(T4VF_ETHTXQ_MAX_HDR, TXD_PER_EQ_UNIT) > 1); + wr = (void *)&txq->q.desc[txq->q.pidx]; + wr->equiq_to_len16 = cpu_to_be32(wr_mid); + wr->r3[0] = cpu_to_be32(0); + wr->r3[1] = cpu_to_be32(0); + skb_copy_from_linear_data(skb, &wr->firmware, fw_hdr_copy_len); + end = (u64 *)wr + flits; + + /* If this is a Large Send Offload packet we'll put in an LSO CPL + * message with an encapsulated TX Packet CPL message. Otherwise we + * just use a TX Packet CPL message. + */ + ssi = skb_shinfo(skb); + if (ssi->gso_size) { + struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); + bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0; + int l3hdr_len = skb_network_header_len(skb); + int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN; + + wr->op_immdlen = + cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) | + FW_WR_IMMDLEN_V(sizeof(*lso) + + sizeof(*cpl))); + /* Fill in the LSO CPL message. */ + lso->lso_ctrl = + cpu_to_be32(LSO_OPCODE_V(CPL_TX_PKT_LSO) | + LSO_FIRST_SLICE_F | + LSO_LAST_SLICE_F | + LSO_IPV6_V(v6) | + LSO_ETHHDR_LEN_V(eth_xtra_len / 4) | + LSO_IPHDR_LEN_V(l3hdr_len / 4) | + LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff)); + lso->ipid_ofst = cpu_to_be16(0); + lso->mss = cpu_to_be16(ssi->gso_size); + lso->seqno_offset = cpu_to_be32(0); + if (is_t4(adapter->params.chip)) + lso->len = cpu_to_be32(skb->len); + else + lso->len = cpu_to_be32(LSO_T5_XFER_SIZE_V(skb->len)); + + /* Set up TX Packet CPL pointer, control word and perform + * accounting. + */ + cpl = (void *)(lso + 1); + + if (chip_ver <= CHELSIO_T5) + cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len); + else + cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len); + + cntrl |= TXPKT_CSUM_TYPE_V(v6 ? + TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) | + TXPKT_IPHDR_LEN_V(l3hdr_len); + txq->tso++; + txq->tx_cso += ssi->gso_segs; + } else { + int len; + + len = (t4vf_is_eth_imm(skb) + ? skb->len + sizeof(*cpl) + : sizeof(*cpl)); + wr->op_immdlen = + cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) | + FW_WR_IMMDLEN_V(len)); + + /* Set up TX Packet CPL pointer, control word and perform + * accounting. + */ + cpl = (void *)(wr + 1); + if (skb->ip_summed == CHECKSUM_PARTIAL) { + cntrl = hwcsum(adapter->params.chip, skb) | + TXPKT_IPCSUM_DIS_F; + txq->tx_cso++; + } else { + cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; + } + } + + /* If there's a VLAN tag present, add that to the list of things to + * do in this Work Request. + */ + if (skb_vlan_tag_present(skb)) { + txq->vlan_ins++; + cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb)); + } + + /* Fill in the TX Packet CPL message header. */ + cpl->ctrl0 = cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT) | + TXPKT_INTF_V(pi->port_id) | + TXPKT_PF_V(0)); + cpl->pack = cpu_to_be16(0); + cpl->len = cpu_to_be16(skb->len); + cpl->ctrl1 = cpu_to_be64(cntrl); + + /* Fill in the body of the TX Packet CPL message with either in-lined + * data or a Scatter/Gather List. + */ + if (t4vf_is_eth_imm(skb)) { + /* In-line the packet's data and free the skb since we don't + * need it any longer. + */ + cxgb4_inline_tx_skb(skb, &txq->q, cpl + 1); + dev_consume_skb_any(skb); + } else { + /* Write the skb's Scatter/Gather list into the TX Packet CPL + * message and retain a pointer to the skb so we can free it + * later when its DMA completes. (We store the skb pointer + * in the Software Descriptor corresponding to the last TX + * Descriptor used by the Work Request.) + * + * The retained skb will be freed when the corresponding TX + * Descriptors are reclaimed after their DMAs complete. + * However, this could take quite a while since, in general, + * the hardware is set up to be lazy about sending DMA + * completion notifications to us and we mostly perform TX + * reclaims in the transmit routine. + * + * This is good for performamce but means that we rely on new + * TX packets arriving to run the destructors of completed + * packets, which open up space in their sockets' send queues. + * Sometimes we do not get such new packets causing TX to + * stall. A single UDP transmitter is a good example of this + * situation. We have a clean up timer that periodically + * reclaims completed packets but it doesn't run often enough + * (nor do we want it to) to prevent lengthy stalls. A + * solution to this problem is to run the destructor early, + * after the packet is queued but before it's DMAd. A con is + * that we lie to socket memory accounting, but the amount of + * extra memory is reasonable (limited by the number of TX + * descriptors), the packets do actually get freed quickly by + * new packets almost always, and for protocols like TCP that + * wait for acks to really free up the data the extra memory + * is even less. On the positive side we run the destructors + * on the sending CPU rather than on a potentially different + * completing CPU, usually a good thing. + * + * Run the destructor before telling the DMA engine about the + * packet to make sure it doesn't complete and get freed + * prematurely. + */ + struct ulptx_sgl *sgl = (struct ulptx_sgl *)(cpl + 1); + struct sge_txq *tq = &txq->q; + + /* If the Work Request header was an exact multiple of our TX + * Descriptor length, then it's possible that the starting SGL + * pointer lines up exactly with the end of our TX Descriptor + * ring. If that's the case, wrap around to the beginning + * here ... + */ + if (unlikely((void *)sgl == (void *)tq->stat)) { + sgl = (void *)tq->desc; + end = (void *)((void *)tq->desc + + ((void *)end - (void *)tq->stat)); + } + + cxgb4_write_sgl(skb, tq, sgl, end, 0, sgl_sdesc->addr); + skb_orphan(skb); + sgl_sdesc->skb = skb; + } + + /* Advance our internal TX Queue state, tell the hardware about + * the new TX descriptors and return success. + */ + txq_advance(&txq->q, ndesc); + + cxgb4_ring_tx_db(adapter, &txq->q, ndesc); + return NETDEV_TX_OK; + +out_free: + /* An error of some sort happened. Free the TX skb and tell the + * OS that we've "dealt" with the packet ... + */ + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +} + +/** + * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs + * @q: the SGE control Tx queue + * + * This is a variant of cxgb4_reclaim_completed_tx() that is used + * for Tx queues that send only immediate data (presently just + * the control queues) and thus do not have any sk_buffs to release. + */ +static inline void reclaim_completed_tx_imm(struct sge_txq *q) +{ + int hw_cidx = ntohs(READ_ONCE(q->stat->cidx)); + int reclaim = hw_cidx - q->cidx; + + if (reclaim < 0) + reclaim += q->size; + + q->in_use -= reclaim; + q->cidx = hw_cidx; +} + +static inline void eosw_txq_advance_index(u32 *idx, u32 n, u32 max) +{ + u32 val = *idx + n; + + if (val >= max) + val -= max; + + *idx = val; +} + +void cxgb4_eosw_txq_free_desc(struct adapter *adap, + struct sge_eosw_txq *eosw_txq, u32 ndesc) +{ + struct tx_sw_desc *d; + + d = &eosw_txq->desc[eosw_txq->last_cidx]; + while (ndesc--) { + if (d->skb) { + if (d->addr[0]) { + unmap_skb(adap->pdev_dev, d->skb, d->addr); + memset(d->addr, 0, sizeof(d->addr)); + } + dev_consume_skb_any(d->skb); + d->skb = NULL; + } + eosw_txq_advance_index(&eosw_txq->last_cidx, 1, + eosw_txq->ndesc); + d = &eosw_txq->desc[eosw_txq->last_cidx]; + } +} + +static inline void eosw_txq_advance(struct sge_eosw_txq *eosw_txq, u32 n) +{ + eosw_txq_advance_index(&eosw_txq->pidx, n, eosw_txq->ndesc); + eosw_txq->inuse += n; +} + +static inline int eosw_txq_enqueue(struct sge_eosw_txq *eosw_txq, + struct sk_buff *skb) +{ + if (eosw_txq->inuse == eosw_txq->ndesc) + return -ENOMEM; + + eosw_txq->desc[eosw_txq->pidx].skb = skb; + return 0; +} + +static inline struct sk_buff *eosw_txq_peek(struct sge_eosw_txq *eosw_txq) +{ + return eosw_txq->desc[eosw_txq->last_pidx].skb; +} + +static inline u8 ethofld_calc_tx_flits(struct adapter *adap, + struct sk_buff *skb, u32 hdr_len) +{ + u8 flits, nsgl = 0; + u32 wrlen; + + wrlen = sizeof(struct fw_eth_tx_eo_wr) + sizeof(struct cpl_tx_pkt_core); + if (skb_shinfo(skb)->gso_size && + !(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)) + wrlen += sizeof(struct cpl_tx_pkt_lso_core); + + wrlen += roundup(hdr_len, 16); + + /* Packet headers + WR + CPLs */ + flits = DIV_ROUND_UP(wrlen, 8); + + if (skb_shinfo(skb)->nr_frags > 0) { + if (skb_headlen(skb) - hdr_len) + nsgl = sgl_len(skb_shinfo(skb)->nr_frags + 1); + else + nsgl = sgl_len(skb_shinfo(skb)->nr_frags); + } else if (skb->len - hdr_len) { + nsgl = sgl_len(1); + } + + return flits + nsgl; +} + +static void *write_eo_wr(struct adapter *adap, struct sge_eosw_txq *eosw_txq, + struct sk_buff *skb, struct fw_eth_tx_eo_wr *wr, + u32 hdr_len, u32 wrlen) +{ + const struct skb_shared_info *ssi = skb_shinfo(skb); + struct cpl_tx_pkt_core *cpl; + u32 immd_len, wrlen16; + bool compl = false; + u8 ver, proto; + + ver = ip_hdr(skb)->version; + proto = (ver == 6) ? ipv6_hdr(skb)->nexthdr : ip_hdr(skb)->protocol; + + wrlen16 = DIV_ROUND_UP(wrlen, 16); + immd_len = sizeof(struct cpl_tx_pkt_core); + if (skb_shinfo(skb)->gso_size && + !(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)) + immd_len += sizeof(struct cpl_tx_pkt_lso_core); + immd_len += hdr_len; + + if (!eosw_txq->ncompl || + (eosw_txq->last_compl + wrlen16) >= + (adap->params.ofldq_wr_cred / 2)) { + compl = true; + eosw_txq->ncompl++; + eosw_txq->last_compl = 0; + } + + wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_EO_WR) | + FW_ETH_TX_EO_WR_IMMDLEN_V(immd_len) | + FW_WR_COMPL_V(compl)); + wr->equiq_to_len16 = cpu_to_be32(FW_WR_LEN16_V(wrlen16) | + FW_WR_FLOWID_V(eosw_txq->hwtid)); + wr->r3 = 0; + if (proto == IPPROTO_UDP) { + cpl = write_eo_udp_wr(skb, wr, hdr_len); + } else { + wr->u.tcpseg.type = FW_ETH_TX_EO_TYPE_TCPSEG; + wr->u.tcpseg.ethlen = skb_network_offset(skb); + wr->u.tcpseg.iplen = cpu_to_be16(skb_network_header_len(skb)); + wr->u.tcpseg.tcplen = tcp_hdrlen(skb); + wr->u.tcpseg.tsclk_tsoff = 0; + wr->u.tcpseg.r4 = 0; + wr->u.tcpseg.r5 = 0; + wr->u.tcpseg.plen = cpu_to_be32(skb->len - hdr_len); + + if (ssi->gso_size) { + struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); + + wr->u.tcpseg.mss = cpu_to_be16(ssi->gso_size); + cpl = write_tso_wr(adap, skb, lso); + } else { + wr->u.tcpseg.mss = cpu_to_be16(0xffff); + cpl = (void *)(wr + 1); + } + } + + eosw_txq->cred -= wrlen16; + eosw_txq->last_compl += wrlen16; + return cpl; +} + +static int ethofld_hard_xmit(struct net_device *dev, + struct sge_eosw_txq *eosw_txq) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + u32 wrlen, wrlen16, hdr_len, data_len; + enum sge_eosw_state next_state; + u64 cntrl, *start, *end, *sgl; + struct sge_eohw_txq *eohw_txq; + struct cpl_tx_pkt_core *cpl; + struct fw_eth_tx_eo_wr *wr; + bool skip_eotx_wr = false; + struct tx_sw_desc *d; + struct sk_buff *skb; + int left, ret = 0; + u8 flits, ndesc; + + eohw_txq = &adap->sge.eohw_txq[eosw_txq->hwqid]; + spin_lock(&eohw_txq->lock); + reclaim_completed_tx_imm(&eohw_txq->q); + + d = &eosw_txq->desc[eosw_txq->last_pidx]; + skb = d->skb; + skb_tx_timestamp(skb); + + wr = (struct fw_eth_tx_eo_wr *)&eohw_txq->q.desc[eohw_txq->q.pidx]; + if (unlikely(eosw_txq->state != CXGB4_EO_STATE_ACTIVE && + eosw_txq->last_pidx == eosw_txq->flowc_idx)) { + hdr_len = skb->len; + data_len = 0; + flits = DIV_ROUND_UP(hdr_len, 8); + if (eosw_txq->state == CXGB4_EO_STATE_FLOWC_OPEN_SEND) + next_state = CXGB4_EO_STATE_FLOWC_OPEN_REPLY; + else + next_state = CXGB4_EO_STATE_FLOWC_CLOSE_REPLY; + skip_eotx_wr = true; + } else { + hdr_len = eth_get_headlen(dev, skb->data, skb_headlen(skb)); + data_len = skb->len - hdr_len; + flits = ethofld_calc_tx_flits(adap, skb, hdr_len); + } + ndesc = flits_to_desc(flits); + wrlen = flits * 8; + wrlen16 = DIV_ROUND_UP(wrlen, 16); + + left = txq_avail(&eohw_txq->q) - ndesc; + + /* If there are no descriptors left in hardware queues or no + * CPL credits left in software queues, then wait for them + * to come back and retry again. Note that we always request + * for credits update via interrupt for every half credits + * consumed. So, the interrupt will eventually restore the + * credits and invoke the Tx path again. + */ + if (unlikely(left < 0 || wrlen16 > eosw_txq->cred)) { + ret = -ENOMEM; + goto out_unlock; + } + + if (unlikely(skip_eotx_wr)) { + start = (u64 *)wr; + eosw_txq->state = next_state; + eosw_txq->cred -= wrlen16; + eosw_txq->ncompl++; + eosw_txq->last_compl = 0; + goto write_wr_headers; + } + + cpl = write_eo_wr(adap, eosw_txq, skb, wr, hdr_len, wrlen); + cntrl = hwcsum(adap->params.chip, skb); + if (skb_vlan_tag_present(skb)) + cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb)); + + cpl->ctrl0 = cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT) | + TXPKT_INTF_V(pi->tx_chan) | + TXPKT_PF_V(adap->pf)); + cpl->pack = 0; + cpl->len = cpu_to_be16(skb->len); + cpl->ctrl1 = cpu_to_be64(cntrl); + + start = (u64 *)(cpl + 1); + +write_wr_headers: + sgl = (u64 *)inline_tx_skb_header(skb, &eohw_txq->q, (void *)start, + hdr_len); + if (data_len) { + ret = cxgb4_map_skb(adap->pdev_dev, skb, d->addr); + if (unlikely(ret)) { + memset(d->addr, 0, sizeof(d->addr)); + eohw_txq->mapping_err++; + goto out_unlock; + } + + end = (u64 *)wr + flits; + if (unlikely(start > sgl)) { + left = (u8 *)end - (u8 *)eohw_txq->q.stat; + end = (void *)eohw_txq->q.desc + left; + } + + if (unlikely((u8 *)sgl >= (u8 *)eohw_txq->q.stat)) { + /* If current position is already at the end of the + * txq, reset the current to point to start of the queue + * and update the end ptr as well. + */ + left = (u8 *)end - (u8 *)eohw_txq->q.stat; + + end = (void *)eohw_txq->q.desc + left; + sgl = (void *)eohw_txq->q.desc; + } + + cxgb4_write_sgl(skb, &eohw_txq->q, (void *)sgl, end, hdr_len, + d->addr); + } + + if (skb_shinfo(skb)->gso_size) { + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) + eohw_txq->uso++; + else + eohw_txq->tso++; + eohw_txq->tx_cso += skb_shinfo(skb)->gso_segs; + } else if (skb->ip_summed == CHECKSUM_PARTIAL) { + eohw_txq->tx_cso++; + } + + if (skb_vlan_tag_present(skb)) + eohw_txq->vlan_ins++; + + txq_advance(&eohw_txq->q, ndesc); + cxgb4_ring_tx_db(adap, &eohw_txq->q, ndesc); + eosw_txq_advance_index(&eosw_txq->last_pidx, 1, eosw_txq->ndesc); + +out_unlock: + spin_unlock(&eohw_txq->lock); + return ret; +} + +static void ethofld_xmit(struct net_device *dev, struct sge_eosw_txq *eosw_txq) +{ + struct sk_buff *skb; + int pktcount, ret; + + switch (eosw_txq->state) { + case CXGB4_EO_STATE_ACTIVE: + case CXGB4_EO_STATE_FLOWC_OPEN_SEND: + case CXGB4_EO_STATE_FLOWC_CLOSE_SEND: + pktcount = eosw_txq->pidx - eosw_txq->last_pidx; + if (pktcount < 0) + pktcount += eosw_txq->ndesc; + break; + case CXGB4_EO_STATE_FLOWC_OPEN_REPLY: + case CXGB4_EO_STATE_FLOWC_CLOSE_REPLY: + case CXGB4_EO_STATE_CLOSED: + default: + return; + } + + while (pktcount--) { + skb = eosw_txq_peek(eosw_txq); + if (!skb) { + eosw_txq_advance_index(&eosw_txq->last_pidx, 1, + eosw_txq->ndesc); + continue; + } + + ret = ethofld_hard_xmit(dev, eosw_txq); + if (ret) + break; + } +} + +static netdev_tx_t cxgb4_ethofld_xmit(struct sk_buff *skb, + struct net_device *dev) +{ + struct cxgb4_tc_port_mqprio *tc_port_mqprio; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + struct sge_eosw_txq *eosw_txq; + u32 qid; + int ret; + + ret = cxgb4_validate_skb(skb, dev, ETH_HLEN); + if (ret) + goto out_free; + + tc_port_mqprio = &adap->tc_mqprio->port_mqprio[pi->port_id]; + qid = skb_get_queue_mapping(skb) - pi->nqsets; + eosw_txq = &tc_port_mqprio->eosw_txq[qid]; + spin_lock_bh(&eosw_txq->lock); + if (eosw_txq->state != CXGB4_EO_STATE_ACTIVE) + goto out_unlock; + + ret = eosw_txq_enqueue(eosw_txq, skb); + if (ret) + goto out_unlock; + + /* SKB is queued for processing until credits are available. + * So, call the destructor now and we'll free the skb later + * after it has been successfully transmitted. + */ + skb_orphan(skb); + + eosw_txq_advance(eosw_txq, 1); + ethofld_xmit(dev, eosw_txq); + spin_unlock_bh(&eosw_txq->lock); + return NETDEV_TX_OK; + +out_unlock: + spin_unlock_bh(&eosw_txq->lock); +out_free: + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +} + +netdev_tx_t t4_start_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + u16 qid = skb_get_queue_mapping(skb); + + if (unlikely(pi->eth_flags & PRIV_FLAG_PORT_TX_VM)) + return cxgb4_vf_eth_xmit(skb, dev); + + if (unlikely(qid >= pi->nqsets)) + return cxgb4_ethofld_xmit(skb, dev); + + if (is_ptp_enabled(skb, dev)) { + struct adapter *adap = netdev2adap(dev); + netdev_tx_t ret; + + spin_lock(&adap->ptp_lock); + ret = cxgb4_eth_xmit(skb, dev); + spin_unlock(&adap->ptp_lock); + return ret; + } + + return cxgb4_eth_xmit(skb, dev); +} + +static void eosw_txq_flush_pending_skbs(struct sge_eosw_txq *eosw_txq) +{ + int pktcount = eosw_txq->pidx - eosw_txq->last_pidx; + int pidx = eosw_txq->pidx; + struct sk_buff *skb; + + if (!pktcount) + return; + + if (pktcount < 0) + pktcount += eosw_txq->ndesc; + + while (pktcount--) { + pidx--; + if (pidx < 0) + pidx += eosw_txq->ndesc; + + skb = eosw_txq->desc[pidx].skb; + if (skb) { + dev_consume_skb_any(skb); + eosw_txq->desc[pidx].skb = NULL; + eosw_txq->inuse--; + } + } + + eosw_txq->pidx = eosw_txq->last_pidx + 1; +} + +/** + * cxgb4_ethofld_send_flowc - Send ETHOFLD flowc request to bind eotid to tc. + * @dev: netdevice + * @eotid: ETHOFLD tid to bind/unbind + * @tc: traffic class. If set to FW_SCHED_CLS_NONE, then unbinds the @eotid + * + * Send a FLOWC work request to bind an ETHOFLD TID to a traffic class. + * If @tc is set to FW_SCHED_CLS_NONE, then the @eotid is unbound from + * a traffic class. + */ +int cxgb4_ethofld_send_flowc(struct net_device *dev, u32 eotid, u32 tc) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adap = netdev2adap(dev); + enum sge_eosw_state next_state; + struct sge_eosw_txq *eosw_txq; + u32 len, len16, nparams = 6; + struct fw_flowc_wr *flowc; + struct eotid_entry *entry; + struct sge_ofld_rxq *rxq; + struct sk_buff *skb; + int ret = 0; + + len = struct_size(flowc, mnemval, nparams); + len16 = DIV_ROUND_UP(len, 16); + + entry = cxgb4_lookup_eotid(&adap->tids, eotid); + if (!entry) + return -ENOMEM; + + eosw_txq = (struct sge_eosw_txq *)entry->data; + if (!eosw_txq) + return -ENOMEM; + + if (!(adap->flags & CXGB4_FW_OK)) { + /* Don't stall caller when access to FW is lost */ + complete(&eosw_txq->completion); + return -EIO; + } + + skb = alloc_skb(len, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + spin_lock_bh(&eosw_txq->lock); + if (tc != FW_SCHED_CLS_NONE) { + if (eosw_txq->state != CXGB4_EO_STATE_CLOSED) + goto out_free_skb; + + next_state = CXGB4_EO_STATE_FLOWC_OPEN_SEND; + } else { + if (eosw_txq->state != CXGB4_EO_STATE_ACTIVE) + goto out_free_skb; + + next_state = CXGB4_EO_STATE_FLOWC_CLOSE_SEND; + } + + flowc = __skb_put(skb, len); + memset(flowc, 0, len); + + rxq = &adap->sge.eohw_rxq[eosw_txq->hwqid]; + flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(len16) | + FW_WR_FLOWID_V(eosw_txq->hwtid)); + flowc->op_to_nparams = cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) | + FW_FLOWC_WR_NPARAMS_V(nparams) | + FW_WR_COMPL_V(1)); + flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; + flowc->mnemval[0].val = cpu_to_be32(FW_PFVF_CMD_PFN_V(adap->pf)); + flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; + flowc->mnemval[1].val = cpu_to_be32(pi->tx_chan); + flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; + flowc->mnemval[2].val = cpu_to_be32(pi->tx_chan); + flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; + flowc->mnemval[3].val = cpu_to_be32(rxq->rspq.abs_id); + flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS; + flowc->mnemval[4].val = cpu_to_be32(tc); + flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_EOSTATE; + flowc->mnemval[5].val = cpu_to_be32(tc == FW_SCHED_CLS_NONE ? + FW_FLOWC_MNEM_EOSTATE_CLOSING : + FW_FLOWC_MNEM_EOSTATE_ESTABLISHED); + + /* Free up any pending skbs to ensure there's room for + * termination FLOWC. + */ + if (tc == FW_SCHED_CLS_NONE) + eosw_txq_flush_pending_skbs(eosw_txq); + + ret = eosw_txq_enqueue(eosw_txq, skb); + if (ret) + goto out_free_skb; + + eosw_txq->state = next_state; + eosw_txq->flowc_idx = eosw_txq->pidx; + eosw_txq_advance(eosw_txq, 1); + ethofld_xmit(dev, eosw_txq); + + spin_unlock_bh(&eosw_txq->lock); + return 0; + +out_free_skb: + dev_consume_skb_any(skb); + spin_unlock_bh(&eosw_txq->lock); + return ret; +} + +/** + * is_imm - check whether a packet can be sent as immediate data + * @skb: the packet + * + * Returns true if a packet can be sent as a WR with immediate data. + */ +static inline int is_imm(const struct sk_buff *skb) +{ + return skb->len <= MAX_CTRL_WR_LEN; +} + +/** + * ctrlq_check_stop - check if a control queue is full and should stop + * @q: the queue + * @wr: most recent WR written to the queue + * + * Check if a control queue has become full and should be stopped. + * We clean up control queue descriptors very lazily, only when we are out. + * If the queue is still full after reclaiming any completed descriptors + * we suspend it and have the last WR wake it up. + */ +static void ctrlq_check_stop(struct sge_ctrl_txq *q, struct fw_wr_hdr *wr) +{ + reclaim_completed_tx_imm(&q->q); + if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) { + wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F); + q->q.stops++; + q->full = 1; + } +} + +#define CXGB4_SELFTEST_LB_STR "CHELSIO_SELFTEST" + +int cxgb4_selftest_lb_pkt(struct net_device *netdev) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adap = pi->adapter; + struct cxgb4_ethtool_lb_test *lb; + int ret, i = 0, pkt_len, credits; + struct fw_eth_tx_pkt_wr *wr; + struct cpl_tx_pkt_core *cpl; + u32 ctrl0, ndesc, flits; + struct sge_eth_txq *q; + u8 *sgl; + + pkt_len = ETH_HLEN + sizeof(CXGB4_SELFTEST_LB_STR); + + flits = DIV_ROUND_UP(pkt_len + sizeof(*cpl) + sizeof(*wr), + sizeof(__be64)); + ndesc = flits_to_desc(flits); + + lb = &pi->ethtool_lb; + lb->loopback = 1; + + q = &adap->sge.ethtxq[pi->first_qset]; + __netif_tx_lock(q->txq, smp_processor_id()); + + reclaim_completed_tx(adap, &q->q, -1, true); + credits = txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + __netif_tx_unlock(q->txq); + return -ENOMEM; + } + + wr = (void *)&q->q.desc[q->q.pidx]; + memset(wr, 0, sizeof(struct tx_desc)); + + wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | + FW_WR_IMMDLEN_V(pkt_len + + sizeof(*cpl))); + wr->equiq_to_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2))); + wr->r3 = cpu_to_be64(0); + + cpl = (void *)(wr + 1); + sgl = (u8 *)(cpl + 1); + + ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_PF_V(adap->pf) | + TXPKT_INTF_V(pi->tx_chan + 4); + + cpl->ctrl0 = htonl(ctrl0); + cpl->pack = htons(0); + cpl->len = htons(pkt_len); + cpl->ctrl1 = cpu_to_be64(TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F); + + eth_broadcast_addr(sgl); + i += ETH_ALEN; + ether_addr_copy(&sgl[i], netdev->dev_addr); + i += ETH_ALEN; + + snprintf(&sgl[i], sizeof(CXGB4_SELFTEST_LB_STR), "%s", + CXGB4_SELFTEST_LB_STR); + + init_completion(&lb->completion); + txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(adap, &q->q, ndesc); + __netif_tx_unlock(q->txq); + + /* wait for the pkt to return */ + ret = wait_for_completion_timeout(&lb->completion, 10 * HZ); + if (!ret) + ret = -ETIMEDOUT; + else + ret = lb->result; + + lb->loopback = 0; + + return ret; +} + +/** + * ctrl_xmit - send a packet through an SGE control Tx queue + * @q: the control queue + * @skb: the packet + * + * Send a packet through an SGE control Tx queue. Packets sent through + * a control queue must fit entirely as immediate data. + */ +static int ctrl_xmit(struct sge_ctrl_txq *q, struct sk_buff *skb) +{ + unsigned int ndesc; + struct fw_wr_hdr *wr; + + if (unlikely(!is_imm(skb))) { + WARN_ON(1); + dev_kfree_skb(skb); + return NET_XMIT_DROP; + } + + ndesc = DIV_ROUND_UP(skb->len, sizeof(struct tx_desc)); + spin_lock(&q->sendq.lock); + + if (unlikely(q->full)) { + skb->priority = ndesc; /* save for restart */ + __skb_queue_tail(&q->sendq, skb); + spin_unlock(&q->sendq.lock); + return NET_XMIT_CN; + } + + wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx]; + cxgb4_inline_tx_skb(skb, &q->q, wr); + + txq_advance(&q->q, ndesc); + if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) + ctrlq_check_stop(q, wr); + + cxgb4_ring_tx_db(q->adap, &q->q, ndesc); + spin_unlock(&q->sendq.lock); + + kfree_skb(skb); + return NET_XMIT_SUCCESS; +} + +/** + * restart_ctrlq - restart a suspended control queue + * @t: pointer to the tasklet associated with this handler + * + * Resumes transmission on a suspended Tx control queue. + */ +static void restart_ctrlq(struct tasklet_struct *t) +{ + struct sk_buff *skb; + unsigned int written = 0; + struct sge_ctrl_txq *q = from_tasklet(q, t, qresume_tsk); + + spin_lock(&q->sendq.lock); + reclaim_completed_tx_imm(&q->q); + BUG_ON(txq_avail(&q->q) < TXQ_STOP_THRES); /* q should be empty */ + + while ((skb = __skb_dequeue(&q->sendq)) != NULL) { + struct fw_wr_hdr *wr; + unsigned int ndesc = skb->priority; /* previously saved */ + + written += ndesc; + /* Write descriptors and free skbs outside the lock to limit + * wait times. q->full is still set so new skbs will be queued. + */ + wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx]; + txq_advance(&q->q, ndesc); + spin_unlock(&q->sendq.lock); + + cxgb4_inline_tx_skb(skb, &q->q, wr); + kfree_skb(skb); + + if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) { + unsigned long old = q->q.stops; + + ctrlq_check_stop(q, wr); + if (q->q.stops != old) { /* suspended anew */ + spin_lock(&q->sendq.lock); + goto ringdb; + } + } + if (written > 16) { + cxgb4_ring_tx_db(q->adap, &q->q, written); + written = 0; + } + spin_lock(&q->sendq.lock); + } + q->full = 0; +ringdb: + if (written) + cxgb4_ring_tx_db(q->adap, &q->q, written); + spin_unlock(&q->sendq.lock); +} + +/** + * t4_mgmt_tx - send a management message + * @adap: the adapter + * @skb: the packet containing the management message + * + * Send a management message through control queue 0. + */ +int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb) +{ + int ret; + + local_bh_disable(); + ret = ctrl_xmit(&adap->sge.ctrlq[0], skb); + local_bh_enable(); + return ret; +} + +/** + * is_ofld_imm - check whether a packet can be sent as immediate data + * @skb: the packet + * + * Returns true if a packet can be sent as an offload WR with immediate + * data. + * FW_OFLD_TX_DATA_WR limits the payload to 255 bytes due to 8-bit field. + * However, FW_ULPTX_WR commands have a 256 byte immediate only + * payload limit. + */ +static inline int is_ofld_imm(const struct sk_buff *skb) +{ + struct work_request_hdr *req = (struct work_request_hdr *)skb->data; + unsigned long opcode = FW_WR_OP_G(ntohl(req->wr_hi)); + + if (unlikely(opcode == FW_ULPTX_WR)) + return skb->len <= MAX_IMM_ULPTX_WR_LEN; + else if (opcode == FW_CRYPTO_LOOKASIDE_WR) + return skb->len <= SGE_MAX_WR_LEN; + else + return skb->len <= MAX_IMM_OFLD_TX_DATA_WR_LEN; +} + +/** + * calc_tx_flits_ofld - calculate # of flits for an offload packet + * @skb: the packet + * + * Returns the number of flits needed for the given offload packet. + * These packets are already fully constructed and no additional headers + * will be added. + */ +static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) +{ + unsigned int flits, cnt; + + if (is_ofld_imm(skb)) + return DIV_ROUND_UP(skb->len, 8); + + flits = skb_transport_offset(skb) / 8U; /* headers */ + cnt = skb_shinfo(skb)->nr_frags; + if (skb_tail_pointer(skb) != skb_transport_header(skb)) + cnt++; + return flits + sgl_len(cnt); +} + +/** + * txq_stop_maperr - stop a Tx queue due to I/O MMU exhaustion + * @q: the queue to stop + * + * Mark a Tx queue stopped due to I/O MMU exhaustion and resulting + * inability to map packets. A periodic timer attempts to restart + * queues so marked. + */ +static void txq_stop_maperr(struct sge_uld_txq *q) +{ + q->mapping_err++; + q->q.stops++; + set_bit(q->q.cntxt_id - q->adap->sge.egr_start, + q->adap->sge.txq_maperr); +} + +/** + * ofldtxq_stop - stop an offload Tx queue that has become full + * @q: the queue to stop + * @wr: the Work Request causing the queue to become full + * + * Stops an offload Tx queue that has become full and modifies the packet + * being written to request a wakeup. + */ +static void ofldtxq_stop(struct sge_uld_txq *q, struct fw_wr_hdr *wr) +{ + wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F); + q->q.stops++; + q->full = 1; +} + +/** + * service_ofldq - service/restart a suspended offload queue + * @q: the offload queue + * + * Services an offload Tx queue by moving packets from its Pending Send + * Queue to the Hardware TX ring. The function starts and ends with the + * Send Queue locked, but drops the lock while putting the skb at the + * head of the Send Queue onto the Hardware TX Ring. Dropping the lock + * allows more skbs to be added to the Send Queue by other threads. + * The packet being processed at the head of the Pending Send Queue is + * left on the queue in case we experience DMA Mapping errors, etc. + * and need to give up and restart later. + * + * service_ofldq() can be thought of as a task which opportunistically + * uses other threads execution contexts. We use the Offload Queue + * boolean "service_ofldq_running" to make sure that only one instance + * is ever running at a time ... + */ +static void service_ofldq(struct sge_uld_txq *q) + __must_hold(&q->sendq.lock) +{ + u64 *pos, *before, *end; + int credits; + struct sk_buff *skb; + struct sge_txq *txq; + unsigned int left; + unsigned int written = 0; + unsigned int flits, ndesc; + + /* If another thread is currently in service_ofldq() processing the + * Pending Send Queue then there's nothing to do. Otherwise, flag + * that we're doing the work and continue. Examining/modifying + * the Offload Queue boolean "service_ofldq_running" must be done + * while holding the Pending Send Queue Lock. + */ + if (q->service_ofldq_running) + return; + q->service_ofldq_running = true; + + while ((skb = skb_peek(&q->sendq)) != NULL && !q->full) { + /* We drop the lock while we're working with the skb at the + * head of the Pending Send Queue. This allows more skbs to + * be added to the Pending Send Queue while we're working on + * this one. We don't need to lock to guard the TX Ring + * updates because only one thread of execution is ever + * allowed into service_ofldq() at a time. + */ + spin_unlock(&q->sendq.lock); + + cxgb4_reclaim_completed_tx(q->adap, &q->q, false); + + flits = skb->priority; /* previously saved */ + ndesc = flits_to_desc(flits); + credits = txq_avail(&q->q) - ndesc; + BUG_ON(credits < 0); + if (unlikely(credits < TXQ_STOP_THRES)) + ofldtxq_stop(q, (struct fw_wr_hdr *)skb->data); + + pos = (u64 *)&q->q.desc[q->q.pidx]; + if (is_ofld_imm(skb)) + cxgb4_inline_tx_skb(skb, &q->q, pos); + else if (cxgb4_map_skb(q->adap->pdev_dev, skb, + (dma_addr_t *)skb->head)) { + txq_stop_maperr(q); + spin_lock(&q->sendq.lock); + break; + } else { + int last_desc, hdr_len = skb_transport_offset(skb); + + /* The WR headers may not fit within one descriptor. + * So we need to deal with wrap-around here. + */ + before = (u64 *)pos; + end = (u64 *)pos + flits; + txq = &q->q; + pos = (void *)inline_tx_skb_header(skb, &q->q, + (void *)pos, + hdr_len); + if (before > (u64 *)pos) { + left = (u8 *)end - (u8 *)txq->stat; + end = (void *)txq->desc + left; + } + + /* If current position is already at the end of the + * ofld queue, reset the current to point to + * start of the queue and update the end ptr as well. + */ + if (pos == (u64 *)txq->stat) { + left = (u8 *)end - (u8 *)txq->stat; + end = (void *)txq->desc + left; + pos = (void *)txq->desc; + } + + cxgb4_write_sgl(skb, &q->q, (void *)pos, + end, hdr_len, + (dma_addr_t *)skb->head); +#ifdef CONFIG_NEED_DMA_MAP_STATE + skb->dev = q->adap->port[0]; + skb->destructor = deferred_unmap_destructor; +#endif + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + q->q.sdesc[last_desc].skb = skb; + } + + txq_advance(&q->q, ndesc); + written += ndesc; + if (unlikely(written > 32)) { + cxgb4_ring_tx_db(q->adap, &q->q, written); + written = 0; + } + + /* Reacquire the Pending Send Queue Lock so we can unlink the + * skb we've just successfully transferred to the TX Ring and + * loop for the next skb which may be at the head of the + * Pending Send Queue. + */ + spin_lock(&q->sendq.lock); + __skb_unlink(skb, &q->sendq); + if (is_ofld_imm(skb)) + kfree_skb(skb); + } + if (likely(written)) + cxgb4_ring_tx_db(q->adap, &q->q, written); + + /*Indicate that no thread is processing the Pending Send Queue + * currently. + */ + q->service_ofldq_running = false; +} + +/** + * ofld_xmit - send a packet through an offload queue + * @q: the Tx offload queue + * @skb: the packet + * + * Send an offload packet through an SGE offload queue. + */ +static int ofld_xmit(struct sge_uld_txq *q, struct sk_buff *skb) +{ + skb->priority = calc_tx_flits_ofld(skb); /* save for restart */ + spin_lock(&q->sendq.lock); + + /* Queue the new skb onto the Offload Queue's Pending Send Queue. If + * that results in this new skb being the only one on the queue, start + * servicing it. If there are other skbs already on the list, then + * either the queue is currently being processed or it's been stopped + * for some reason and it'll be restarted at a later time. Restart + * paths are triggered by events like experiencing a DMA Mapping Error + * or filling the Hardware TX Ring. + */ + __skb_queue_tail(&q->sendq, skb); + if (q->sendq.qlen == 1) + service_ofldq(q); + + spin_unlock(&q->sendq.lock); + return NET_XMIT_SUCCESS; +} + +/** + * restart_ofldq - restart a suspended offload queue + * @t: pointer to the tasklet associated with this handler + * + * Resumes transmission on a suspended Tx offload queue. + */ +static void restart_ofldq(struct tasklet_struct *t) +{ + struct sge_uld_txq *q = from_tasklet(q, t, qresume_tsk); + + spin_lock(&q->sendq.lock); + q->full = 0; /* the queue actually is completely empty now */ + service_ofldq(q); + spin_unlock(&q->sendq.lock); +} + +/** + * skb_txq - return the Tx queue an offload packet should use + * @skb: the packet + * + * Returns the Tx queue an offload packet should use as indicated by bits + * 1-15 in the packet's queue_mapping. + */ +static inline unsigned int skb_txq(const struct sk_buff *skb) +{ + return skb->queue_mapping >> 1; +} + +/** + * is_ctrl_pkt - return whether an offload packet is a control packet + * @skb: the packet + * + * Returns whether an offload packet should use an OFLD or a CTRL + * Tx queue as indicated by bit 0 in the packet's queue_mapping. + */ +static inline unsigned int is_ctrl_pkt(const struct sk_buff *skb) +{ + return skb->queue_mapping & 1; +} + +static inline int uld_send(struct adapter *adap, struct sk_buff *skb, + unsigned int tx_uld_type) +{ + struct sge_uld_txq_info *txq_info; + struct sge_uld_txq *txq; + unsigned int idx = skb_txq(skb); + + if (unlikely(is_ctrl_pkt(skb))) { + /* Single ctrl queue is a requirement for LE workaround path */ + if (adap->tids.nsftids) + idx = 0; + return ctrl_xmit(&adap->sge.ctrlq[idx], skb); + } + + txq_info = adap->sge.uld_txq_info[tx_uld_type]; + if (unlikely(!txq_info)) { + WARN_ON(true); + kfree_skb(skb); + return NET_XMIT_DROP; + } + + txq = &txq_info->uldtxq[idx]; + return ofld_xmit(txq, skb); +} + +/** + * t4_ofld_send - send an offload packet + * @adap: the adapter + * @skb: the packet + * + * Sends an offload packet. We use the packet queue_mapping to select the + * appropriate Tx queue as follows: bit 0 indicates whether the packet + * should be sent as regular or control, bits 1-15 select the queue. + */ +int t4_ofld_send(struct adapter *adap, struct sk_buff *skb) +{ + int ret; + + local_bh_disable(); + ret = uld_send(adap, skb, CXGB4_TX_OFLD); + local_bh_enable(); + return ret; +} + +/** + * cxgb4_ofld_send - send an offload packet + * @dev: the net device + * @skb: the packet + * + * Sends an offload packet. This is an exported version of @t4_ofld_send, + * intended for ULDs. + */ +int cxgb4_ofld_send(struct net_device *dev, struct sk_buff *skb) +{ + return t4_ofld_send(netdev2adap(dev), skb); +} +EXPORT_SYMBOL(cxgb4_ofld_send); + +static void *inline_tx_header(const void *src, + const struct sge_txq *q, + void *pos, int length) +{ + int left = (void *)q->stat - pos; + u64 *p; + + if (likely(length <= left)) { + memcpy(pos, src, length); + pos += length; + } else { + memcpy(pos, src, left); + memcpy(q->desc, src + left, length - left); + pos = (void *)q->desc + (length - left); + } + /* 0-pad to multiple of 16 */ + p = PTR_ALIGN(pos, 8); + if ((uintptr_t)p & 8) { + *p = 0; + return p + 1; + } + return p; +} + +/** + * ofld_xmit_direct - copy a WR into offload queue + * @q: the Tx offload queue + * @src: location of WR + * @len: WR length + * + * Copy an immediate WR into an uncontended SGE offload queue. + */ +static int ofld_xmit_direct(struct sge_uld_txq *q, const void *src, + unsigned int len) +{ + unsigned int ndesc; + int credits; + u64 *pos; + + /* Use the lower limit as the cut-off */ + if (len > MAX_IMM_OFLD_TX_DATA_WR_LEN) { + WARN_ON(1); + return NET_XMIT_DROP; + } + + /* Don't return NET_XMIT_CN here as the current + * implementation doesn't queue the request + * using an skb when the following conditions not met + */ + if (!spin_trylock(&q->sendq.lock)) + return NET_XMIT_DROP; + + if (q->full || !skb_queue_empty(&q->sendq) || + q->service_ofldq_running) { + spin_unlock(&q->sendq.lock); + return NET_XMIT_DROP; + } + ndesc = flits_to_desc(DIV_ROUND_UP(len, 8)); + credits = txq_avail(&q->q) - ndesc; + pos = (u64 *)&q->q.desc[q->q.pidx]; + + /* ofldtxq_stop modifies WR header in-situ */ + inline_tx_header(src, &q->q, pos, len); + if (unlikely(credits < TXQ_STOP_THRES)) + ofldtxq_stop(q, (struct fw_wr_hdr *)pos); + txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(q->adap, &q->q, ndesc); + + spin_unlock(&q->sendq.lock); + return NET_XMIT_SUCCESS; +} + +int cxgb4_immdata_send(struct net_device *dev, unsigned int idx, + const void *src, unsigned int len) +{ + struct sge_uld_txq_info *txq_info; + struct sge_uld_txq *txq; + struct adapter *adap; + int ret; + + adap = netdev2adap(dev); + + local_bh_disable(); + txq_info = adap->sge.uld_txq_info[CXGB4_TX_OFLD]; + if (unlikely(!txq_info)) { + WARN_ON(true); + local_bh_enable(); + return NET_XMIT_DROP; + } + txq = &txq_info->uldtxq[idx]; + + ret = ofld_xmit_direct(txq, src, len); + local_bh_enable(); + return net_xmit_eval(ret); +} +EXPORT_SYMBOL(cxgb4_immdata_send); + +/** + * t4_crypto_send - send crypto packet + * @adap: the adapter + * @skb: the packet + * + * Sends crypto packet. We use the packet queue_mapping to select the + * appropriate Tx queue as follows: bit 0 indicates whether the packet + * should be sent as regular or control, bits 1-15 select the queue. + */ +static int t4_crypto_send(struct adapter *adap, struct sk_buff *skb) +{ + int ret; + + local_bh_disable(); + ret = uld_send(adap, skb, CXGB4_TX_CRYPTO); + local_bh_enable(); + return ret; +} + +/** + * cxgb4_crypto_send - send crypto packet + * @dev: the net device + * @skb: the packet + * + * Sends crypto packet. This is an exported version of @t4_crypto_send, + * intended for ULDs. + */ +int cxgb4_crypto_send(struct net_device *dev, struct sk_buff *skb) +{ + return t4_crypto_send(netdev2adap(dev), skb); +} +EXPORT_SYMBOL(cxgb4_crypto_send); + +static inline void copy_frags(struct sk_buff *skb, + const struct pkt_gl *gl, unsigned int offset) +{ + int i; + + /* usually there's just one frag */ + __skb_fill_page_desc(skb, 0, gl->frags[0].page, + gl->frags[0].offset + offset, + gl->frags[0].size - offset); + skb_shinfo(skb)->nr_frags = gl->nfrags; + for (i = 1; i < gl->nfrags; i++) + __skb_fill_page_desc(skb, i, gl->frags[i].page, + gl->frags[i].offset, + gl->frags[i].size); + + /* get a reference to the last page, we don't own it */ + get_page(gl->frags[gl->nfrags - 1].page); +} + +/** + * cxgb4_pktgl_to_skb - build an sk_buff from a packet gather list + * @gl: the gather list + * @skb_len: size of sk_buff main body if it carries fragments + * @pull_len: amount of data to move to the sk_buff's main body + * + * Builds an sk_buff from the given packet gather list. Returns the + * sk_buff or %NULL if sk_buff allocation failed. + */ +struct sk_buff *cxgb4_pktgl_to_skb(const struct pkt_gl *gl, + unsigned int skb_len, unsigned int pull_len) +{ + struct sk_buff *skb; + + /* + * Below we rely on RX_COPY_THRES being less than the smallest Rx buffer + * size, which is expected since buffers are at least PAGE_SIZEd. + * In this case packets up to RX_COPY_THRES have only one fragment. + */ + if (gl->tot_len <= RX_COPY_THRES) { + skb = dev_alloc_skb(gl->tot_len); + if (unlikely(!skb)) + goto out; + __skb_put(skb, gl->tot_len); + skb_copy_to_linear_data(skb, gl->va, gl->tot_len); + } else { + skb = dev_alloc_skb(skb_len); + if (unlikely(!skb)) + goto out; + __skb_put(skb, pull_len); + skb_copy_to_linear_data(skb, gl->va, pull_len); + + copy_frags(skb, gl, pull_len); + skb->len = gl->tot_len; + skb->data_len = skb->len - pull_len; + skb->truesize += skb->data_len; + } +out: return skb; +} +EXPORT_SYMBOL(cxgb4_pktgl_to_skb); + +/** + * t4_pktgl_free - free a packet gather list + * @gl: the gather list + * + * Releases the pages of a packet gather list. We do not own the last + * page on the list and do not free it. + */ +static void t4_pktgl_free(const struct pkt_gl *gl) +{ + int n; + const struct page_frag *p; + + for (p = gl->frags, n = gl->nfrags - 1; n--; p++) + put_page(p->page); +} + +/* + * Process an MPS trace packet. Give it an unused protocol number so it won't + * be delivered to anyone and send it to the stack for capture. + */ +static noinline int handle_trace_pkt(struct adapter *adap, + const struct pkt_gl *gl) +{ + struct sk_buff *skb; + + skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN); + if (unlikely(!skb)) { + t4_pktgl_free(gl); + return 0; + } + + if (is_t4(adap->params.chip)) + __skb_pull(skb, sizeof(struct cpl_trace_pkt)); + else + __skb_pull(skb, sizeof(struct cpl_t5_trace_pkt)); + + skb_reset_mac_header(skb); + skb->protocol = htons(0xffff); + skb->dev = adap->port[0]; + netif_receive_skb(skb); + return 0; +} + +/** + * cxgb4_sgetim_to_hwtstamp - convert sge time stamp to hw time stamp + * @adap: the adapter + * @hwtstamps: time stamp structure to update + * @sgetstamp: 60bit iqe timestamp + * + * Every ingress queue entry has the 60-bit timestamp, convert that timestamp + * which is in Core Clock ticks into ktime_t and assign it + **/ +static void cxgb4_sgetim_to_hwtstamp(struct adapter *adap, + struct skb_shared_hwtstamps *hwtstamps, + u64 sgetstamp) +{ + u64 ns; + u64 tmp = (sgetstamp * 1000 * 1000 + adap->params.vpd.cclk / 2); + + ns = div_u64(tmp, adap->params.vpd.cclk); + + memset(hwtstamps, 0, sizeof(*hwtstamps)); + hwtstamps->hwtstamp = ns_to_ktime(ns); +} + +static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl, + const struct cpl_rx_pkt *pkt, unsigned long tnl_hdr_len) +{ + struct adapter *adapter = rxq->rspq.adap; + struct sge *s = &adapter->sge; + struct port_info *pi; + int ret; + struct sk_buff *skb; + + skb = napi_get_frags(&rxq->rspq.napi); + if (unlikely(!skb)) { + t4_pktgl_free(gl); + rxq->stats.rx_drops++; + return; + } + + copy_frags(skb, gl, s->pktshift); + if (tnl_hdr_len) + skb->csum_level = 1; + skb->len = gl->tot_len - s->pktshift; + skb->data_len = skb->len; + skb->truesize += skb->data_len; + skb->ip_summed = CHECKSUM_UNNECESSARY; + skb_record_rx_queue(skb, rxq->rspq.idx); + pi = netdev_priv(skb->dev); + if (pi->rxtstamp) + cxgb4_sgetim_to_hwtstamp(adapter, skb_hwtstamps(skb), + gl->sgetstamp); + if (rxq->rspq.netdev->features & NETIF_F_RXHASH) + skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val, + PKT_HASH_TYPE_L3); + + if (unlikely(pkt->vlan_ex)) { + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan)); + rxq->stats.vlan_ex++; + } + ret = napi_gro_frags(&rxq->rspq.napi); + if (ret == GRO_HELD) + rxq->stats.lro_pkts++; + else if (ret == GRO_MERGED || ret == GRO_MERGED_FREE) + rxq->stats.lro_merged++; + rxq->stats.pkts++; + rxq->stats.rx_cso++; +} + +enum { + RX_NON_PTP_PKT = 0, + RX_PTP_PKT_SUC = 1, + RX_PTP_PKT_ERR = 2 +}; + +/** + * t4_systim_to_hwstamp - read hardware time stamp + * @adapter: the adapter + * @skb: the packet + * + * Read Time Stamp from MPS packet and insert in skb which + * is forwarded to PTP application + */ +static noinline int t4_systim_to_hwstamp(struct adapter *adapter, + struct sk_buff *skb) +{ + struct skb_shared_hwtstamps *hwtstamps; + struct cpl_rx_mps_pkt *cpl = NULL; + unsigned char *data; + int offset; + + cpl = (struct cpl_rx_mps_pkt *)skb->data; + if (!(CPL_RX_MPS_PKT_TYPE_G(ntohl(cpl->op_to_r1_hi)) & + X_CPL_RX_MPS_PKT_TYPE_PTP)) + return RX_PTP_PKT_ERR; + + data = skb->data + sizeof(*cpl); + skb_pull(skb, 2 * sizeof(u64) + sizeof(struct cpl_rx_mps_pkt)); + offset = ETH_HLEN + IPV4_HLEN(skb->data) + UDP_HLEN; + if (skb->len < offset + OFF_PTP_SEQUENCE_ID + sizeof(short)) + return RX_PTP_PKT_ERR; + + hwtstamps = skb_hwtstamps(skb); + memset(hwtstamps, 0, sizeof(*hwtstamps)); + hwtstamps->hwtstamp = ns_to_ktime(get_unaligned_be64(data)); + + return RX_PTP_PKT_SUC; +} + +/** + * t4_rx_hststamp - Recv PTP Event Message + * @adapter: the adapter + * @rsp: the response queue descriptor holding the RX_PKT message + * @rxq: the response queue holding the RX_PKT message + * @skb: the packet + * + * PTP enabled and MPS packet, read HW timestamp + */ +static int t4_rx_hststamp(struct adapter *adapter, const __be64 *rsp, + struct sge_eth_rxq *rxq, struct sk_buff *skb) +{ + int ret; + + if (unlikely((*(u8 *)rsp == CPL_RX_MPS_PKT) && + !is_t4(adapter->params.chip))) { + ret = t4_systim_to_hwstamp(adapter, skb); + if (ret == RX_PTP_PKT_ERR) { + kfree_skb(skb); + rxq->stats.rx_drops++; + } + return ret; + } + return RX_NON_PTP_PKT; +} + +/** + * t4_tx_hststamp - Loopback PTP Transmit Event Message + * @adapter: the adapter + * @skb: the packet + * @dev: the ingress net device + * + * Read hardware timestamp for the loopback PTP Tx event message + */ +static int t4_tx_hststamp(struct adapter *adapter, struct sk_buff *skb, + struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + + if (!is_t4(adapter->params.chip) && adapter->ptp_tx_skb) { + cxgb4_ptp_read_hwstamp(adapter, pi); + kfree_skb(skb); + return 0; + } + return 1; +} + +/** + * t4_tx_completion_handler - handle CPL_SGE_EGR_UPDATE messages + * @rspq: Ethernet RX Response Queue associated with Ethernet TX Queue + * @rsp: Response Entry pointer into Response Queue + * @gl: Gather List pointer + * + * For adapters which support the SGE Doorbell Queue Timer facility, + * we configure the Ethernet TX Queues to send CIDX Updates to the + * Associated Ethernet RX Response Queue with CPL_SGE_EGR_UPDATE + * messages. This adds a small load to PCIe Link RX bandwidth and, + * potentially, higher CPU Interrupt load, but allows us to respond + * much more quickly to the CIDX Updates. This is important for + * Upper Layer Software which isn't willing to have a large amount + * of TX Data outstanding before receiving DMA Completions. + */ +static void t4_tx_completion_handler(struct sge_rspq *rspq, + const __be64 *rsp, + const struct pkt_gl *gl) +{ + u8 opcode = ((const struct rss_header *)rsp)->opcode; + struct port_info *pi = netdev_priv(rspq->netdev); + struct adapter *adapter = rspq->adap; + struct sge *s = &adapter->sge; + struct sge_eth_txq *txq; + + /* skip RSS header */ + rsp++; + + /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG. + */ + if (unlikely(opcode == CPL_FW4_MSG && + ((const struct cpl_fw4_msg *)rsp)->type == + FW_TYPE_RSSCPL)) { + rsp++; + opcode = ((const struct rss_header *)rsp)->opcode; + rsp++; + } + + if (unlikely(opcode != CPL_SGE_EGR_UPDATE)) { + pr_info("%s: unexpected FW4/CPL %#x on Rx queue\n", + __func__, opcode); + return; + } + + txq = &s->ethtxq[pi->first_qset + rspq->idx]; + + /* We've got the Hardware Consumer Index Update in the Egress Update + * message. These Egress Update messages will be our sole CIDX Updates + * we get since we don't want to chew up PCIe bandwidth for both Ingress + * Messages and Status Page writes. However, The code which manages + * reclaiming successfully DMA'ed TX Work Requests uses the CIDX value + * stored in the Status Page at the end of the TX Queue. It's easiest + * to simply copy the CIDX Update value from the Egress Update message + * to the Status Page. Also note that no Endian issues need to be + * considered here since both are Big Endian and we're just copying + * bytes consistently ... + */ + if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) { + struct cpl_sge_egr_update *egr; + + egr = (struct cpl_sge_egr_update *)rsp; + WRITE_ONCE(txq->q.stat->cidx, egr->cidx); + } + + t4_sge_eth_txq_egress_update(adapter, txq, -1); +} + +static int cxgb4_validate_lb_pkt(struct port_info *pi, const struct pkt_gl *si) +{ + struct adapter *adap = pi->adapter; + struct cxgb4_ethtool_lb_test *lb; + struct sge *s = &adap->sge; + struct net_device *netdev; + u8 *data; + int i; + + netdev = adap->port[pi->port_id]; + lb = &pi->ethtool_lb; + data = si->va + s->pktshift; + + i = ETH_ALEN; + if (!ether_addr_equal(data + i, netdev->dev_addr)) + return -1; + + i += ETH_ALEN; + if (strcmp(&data[i], CXGB4_SELFTEST_LB_STR)) + lb->result = -EIO; + + complete(&lb->completion); + return 0; +} + +/** + * t4_ethrx_handler - process an ingress ethernet packet + * @q: the response queue that received the packet + * @rsp: the response queue descriptor holding the RX_PKT message + * @si: the gather list of packet fragments + * + * Process an ingress ethernet packet and deliver it to the stack. + */ +int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, + const struct pkt_gl *si) +{ + bool csum_ok; + struct sk_buff *skb; + const struct cpl_rx_pkt *pkt; + struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); + struct adapter *adapter = q->adap; + struct sge *s = &q->adap->sge; + int cpl_trace_pkt = is_t4(q->adap->params.chip) ? + CPL_TRACE_PKT : CPL_TRACE_PKT_T5; + u16 err_vec, tnl_hdr_len = 0; + struct port_info *pi; + int ret = 0; + + pi = netdev_priv(q->netdev); + /* If we're looking at TX Queue CIDX Update, handle that separately + * and return. + */ + if (unlikely((*(u8 *)rsp == CPL_FW4_MSG) || + (*(u8 *)rsp == CPL_SGE_EGR_UPDATE))) { + t4_tx_completion_handler(q, rsp, si); + return 0; + } + + if (unlikely(*(u8 *)rsp == cpl_trace_pkt)) + return handle_trace_pkt(q->adap, si); + + pkt = (const struct cpl_rx_pkt *)rsp; + /* Compressed error vector is enabled for T6 only */ + if (q->adap->params.tp.rx_pkt_encap) { + err_vec = T6_COMPR_RXERR_VEC_G(be16_to_cpu(pkt->err_vec)); + tnl_hdr_len = T6_RX_TNLHDR_LEN_G(ntohs(pkt->err_vec)); + } else { + err_vec = be16_to_cpu(pkt->err_vec); + } + + csum_ok = pkt->csum_calc && !err_vec && + (q->netdev->features & NETIF_F_RXCSUM); + + if (err_vec) + rxq->stats.bad_rx_pkts++; + + if (unlikely(pi->ethtool_lb.loopback && pkt->iff >= NCHAN)) { + ret = cxgb4_validate_lb_pkt(pi, si); + if (!ret) + return 0; + } + + if (((pkt->l2info & htonl(RXF_TCP_F)) || + tnl_hdr_len) && + (q->netdev->features & NETIF_F_GRO) && csum_ok && !pkt->ip_frag) { + do_gro(rxq, si, pkt, tnl_hdr_len); + return 0; + } + + skb = cxgb4_pktgl_to_skb(si, RX_PKT_SKB_LEN, RX_PULL_LEN); + if (unlikely(!skb)) { + t4_pktgl_free(si); + rxq->stats.rx_drops++; + return 0; + } + + /* Handle PTP Event Rx packet */ + if (unlikely(pi->ptp_enable)) { + ret = t4_rx_hststamp(adapter, rsp, rxq, skb); + if (ret == RX_PTP_PKT_ERR) + return 0; + } + if (likely(!ret)) + __skb_pull(skb, s->pktshift); /* remove ethernet header pad */ + + /* Handle the PTP Event Tx Loopback packet */ + if (unlikely(pi->ptp_enable && !ret && + (pkt->l2info & htonl(RXF_UDP_F)) && + cxgb4_ptp_is_ptp_rx(skb))) { + if (!t4_tx_hststamp(adapter, skb, q->netdev)) + return 0; + } + + skb->protocol = eth_type_trans(skb, q->netdev); + skb_record_rx_queue(skb, q->idx); + if (skb->dev->features & NETIF_F_RXHASH) + skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val, + PKT_HASH_TYPE_L3); + + rxq->stats.pkts++; + + if (pi->rxtstamp) + cxgb4_sgetim_to_hwtstamp(q->adap, skb_hwtstamps(skb), + si->sgetstamp); + if (csum_ok && (pkt->l2info & htonl(RXF_UDP_F | RXF_TCP_F))) { + if (!pkt->ip_frag) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + rxq->stats.rx_cso++; + } else if (pkt->l2info & htonl(RXF_IP_F)) { + __sum16 c = (__force __sum16)pkt->csum; + skb->csum = csum_unfold(c); + + if (tnl_hdr_len) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + skb->csum_level = 1; + } else { + skb->ip_summed = CHECKSUM_COMPLETE; + } + rxq->stats.rx_cso++; + } + } else { + skb_checksum_none_assert(skb); +#ifdef CONFIG_CHELSIO_T4_FCOE +#define CPL_RX_PKT_FLAGS (RXF_PSH_F | RXF_SYN_F | RXF_UDP_F | \ + RXF_TCP_F | RXF_IP_F | RXF_IP6_F | RXF_LRO_F) + + if (!(pkt->l2info & cpu_to_be32(CPL_RX_PKT_FLAGS))) { + if ((pkt->l2info & cpu_to_be32(RXF_FCOE_F)) && + (pi->fcoe.flags & CXGB_FCOE_ENABLED)) { + if (q->adap->params.tp.rx_pkt_encap) + csum_ok = err_vec & + T6_COMPR_RXERR_SUM_F; + else + csum_ok = err_vec & RXERR_CSUM_F; + if (!csum_ok) + skb->ip_summed = CHECKSUM_UNNECESSARY; + } + } + +#undef CPL_RX_PKT_FLAGS +#endif /* CONFIG_CHELSIO_T4_FCOE */ + } + + if (unlikely(pkt->vlan_ex)) { + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan)); + rxq->stats.vlan_ex++; + } + skb_mark_napi_id(skb, &q->napi); + netif_receive_skb(skb); + return 0; +} + +/** + * restore_rx_bufs - put back a packet's Rx buffers + * @si: the packet gather list + * @q: the SGE free list + * @frags: number of FL buffers to restore + * + * Puts back on an FL the Rx buffers associated with @si. The buffers + * have already been unmapped and are left unmapped, we mark them so to + * prevent further unmapping attempts. + * + * This function undoes a series of @unmap_rx_buf calls when we find out + * that the current packet can't be processed right away afterall and we + * need to come back to it later. This is a very rare event and there's + * no effort to make this particularly efficient. + */ +static void restore_rx_bufs(const struct pkt_gl *si, struct sge_fl *q, + int frags) +{ + struct rx_sw_desc *d; + + while (frags--) { + if (q->cidx == 0) + q->cidx = q->size - 1; + else + q->cidx--; + d = &q->sdesc[q->cidx]; + d->page = si->frags[frags].page; + d->dma_addr |= RX_UNMAPPED_BUF; + q->avail++; + } +} + +/** + * is_new_response - check if a response is newly written + * @r: the response descriptor + * @q: the response queue + * + * Returns true if a response descriptor contains a yet unprocessed + * response. + */ +static inline bool is_new_response(const struct rsp_ctrl *r, + const struct sge_rspq *q) +{ + return (r->type_gen >> RSPD_GEN_S) == q->gen; +} + +/** + * rspq_next - advance to the next entry in a response queue + * @q: the queue + * + * Updates the state of a response queue to advance it to the next entry. + */ +static inline void rspq_next(struct sge_rspq *q) +{ + q->cur_desc = (void *)q->cur_desc + q->iqe_len; + if (unlikely(++q->cidx == q->size)) { + q->cidx = 0; + q->gen ^= 1; + q->cur_desc = q->desc; + } +} + +/** + * process_responses - process responses from an SGE response queue + * @q: the ingress queue to process + * @budget: how many responses can be processed in this round + * + * Process responses from an SGE response queue up to the supplied budget. + * Responses include received packets as well as control messages from FW + * or HW. + * + * Additionally choose the interrupt holdoff time for the next interrupt + * on this queue. If the system is under memory shortage use a fairly + * long delay to help recovery. + */ +static int process_responses(struct sge_rspq *q, int budget) +{ + int ret, rsp_type; + int budget_left = budget; + const struct rsp_ctrl *rc; + struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); + struct adapter *adapter = q->adap; + struct sge *s = &adapter->sge; + + while (likely(budget_left)) { + rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc)); + if (!is_new_response(rc, q)) { + if (q->flush_handler) + q->flush_handler(q); + break; + } + + dma_rmb(); + rsp_type = RSPD_TYPE_G(rc->type_gen); + if (likely(rsp_type == RSPD_TYPE_FLBUF_X)) { + struct page_frag *fp; + struct pkt_gl si; + const struct rx_sw_desc *rsd; + u32 len = ntohl(rc->pldbuflen_qid), bufsz, frags; + + if (len & RSPD_NEWBUF_F) { + if (likely(q->offset > 0)) { + free_rx_bufs(q->adap, &rxq->fl, 1); + q->offset = 0; + } + len = RSPD_LEN_G(len); + } + si.tot_len = len; + + /* gather packet fragments */ + for (frags = 0, fp = si.frags; ; frags++, fp++) { + rsd = &rxq->fl.sdesc[rxq->fl.cidx]; + bufsz = get_buf_size(adapter, rsd); + fp->page = rsd->page; + fp->offset = q->offset; + fp->size = min(bufsz, len); + len -= fp->size; + if (!len) + break; + unmap_rx_buf(q->adap, &rxq->fl); + } + + si.sgetstamp = SGE_TIMESTAMP_G( + be64_to_cpu(rc->last_flit)); + /* + * Last buffer remains mapped so explicitly make it + * coherent for CPU access. + */ + dma_sync_single_for_cpu(q->adap->pdev_dev, + get_buf_addr(rsd), + fp->size, DMA_FROM_DEVICE); + + si.va = page_address(si.frags[0].page) + + si.frags[0].offset; + prefetch(si.va); + + si.nfrags = frags + 1; + ret = q->handler(q, q->cur_desc, &si); + if (likely(ret == 0)) + q->offset += ALIGN(fp->size, s->fl_align); + else + restore_rx_bufs(&si, &rxq->fl, frags); + } else if (likely(rsp_type == RSPD_TYPE_CPL_X)) { + ret = q->handler(q, q->cur_desc, NULL); + } else { + ret = q->handler(q, (const __be64 *)rc, CXGB4_MSG_AN); + } + + if (unlikely(ret)) { + /* couldn't process descriptor, back off for recovery */ + q->next_intr_params = QINTR_TIMER_IDX_V(NOMEM_TMR_IDX); + break; + } + + rspq_next(q); + budget_left--; + } + + if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 16) + __refill_fl(q->adap, &rxq->fl); + return budget - budget_left; +} + +/** + * napi_rx_handler - the NAPI handler for Rx processing + * @napi: the napi instance + * @budget: how many packets we can process in this round + * + * Handler for new data events when using NAPI. This does not need any + * locking or protection from interrupts as data interrupts are off at + * this point and other adapter interrupts do not interfere (the latter + * in not a concern at all with MSI-X as non-data interrupts then have + * a separate handler). + */ +static int napi_rx_handler(struct napi_struct *napi, int budget) +{ + unsigned int params; + struct sge_rspq *q = container_of(napi, struct sge_rspq, napi); + int work_done; + u32 val; + + work_done = process_responses(q, budget); + if (likely(work_done < budget)) { + int timer_index; + + napi_complete_done(napi, work_done); + timer_index = QINTR_TIMER_IDX_G(q->next_intr_params); + + if (q->adaptive_rx) { + if (work_done > max(timer_pkt_quota[timer_index], + MIN_NAPI_WORK)) + timer_index = (timer_index + 1); + else + timer_index = timer_index - 1; + + timer_index = clamp(timer_index, 0, SGE_TIMERREGS - 1); + q->next_intr_params = + QINTR_TIMER_IDX_V(timer_index) | + QINTR_CNT_EN_V(0); + params = q->next_intr_params; + } else { + params = q->next_intr_params; + q->next_intr_params = q->intr_params; + } + } else + params = QINTR_TIMER_IDX_V(7); + + val = CIDXINC_V(work_done) | SEINTARM_V(params); + + /* If we don't have access to the new User GTS (T5+), use the old + * doorbell mechanism; otherwise use the new BAR2 mechanism. + */ + if (unlikely(q->bar2_addr == NULL)) { + t4_write_reg(q->adap, MYPF_REG(SGE_PF_GTS_A), + val | INGRESSQID_V((u32)q->cntxt_id)); + } else { + writel(val | INGRESSQID_V(q->bar2_qid), + q->bar2_addr + SGE_UDB_GTS); + wmb(); + } + return work_done; +} + +void cxgb4_ethofld_restart(struct tasklet_struct *t) +{ + struct sge_eosw_txq *eosw_txq = from_tasklet(eosw_txq, t, + qresume_tsk); + int pktcount; + + spin_lock(&eosw_txq->lock); + pktcount = eosw_txq->cidx - eosw_txq->last_cidx; + if (pktcount < 0) + pktcount += eosw_txq->ndesc; + + if (pktcount) { + cxgb4_eosw_txq_free_desc(netdev2adap(eosw_txq->netdev), + eosw_txq, pktcount); + eosw_txq->inuse -= pktcount; + } + + /* There may be some packets waiting for completions. So, + * attempt to send these packets now. + */ + ethofld_xmit(eosw_txq->netdev, eosw_txq); + spin_unlock(&eosw_txq->lock); +} + +/* cxgb4_ethofld_rx_handler - Process ETHOFLD Tx completions + * @q: the response queue that received the packet + * @rsp: the response queue descriptor holding the CPL message + * @si: the gather list of packet fragments + * + * Process a ETHOFLD Tx completion. Increment the cidx here, but + * free up the descriptors in a tasklet later. + */ +int cxgb4_ethofld_rx_handler(struct sge_rspq *q, const __be64 *rsp, + const struct pkt_gl *si) +{ + u8 opcode = ((const struct rss_header *)rsp)->opcode; + + /* skip RSS header */ + rsp++; + + if (opcode == CPL_FW4_ACK) { + const struct cpl_fw4_ack *cpl; + struct sge_eosw_txq *eosw_txq; + struct eotid_entry *entry; + struct sk_buff *skb; + u32 hdr_len, eotid; + u8 flits, wrlen16; + int credits; + + cpl = (const struct cpl_fw4_ack *)rsp; + eotid = CPL_FW4_ACK_FLOWID_G(ntohl(OPCODE_TID(cpl))) - + q->adap->tids.eotid_base; + entry = cxgb4_lookup_eotid(&q->adap->tids, eotid); + if (!entry) + goto out_done; + + eosw_txq = (struct sge_eosw_txq *)entry->data; + if (!eosw_txq) + goto out_done; + + spin_lock(&eosw_txq->lock); + credits = cpl->credits; + while (credits > 0) { + skb = eosw_txq->desc[eosw_txq->cidx].skb; + if (!skb) + break; + + if (unlikely((eosw_txq->state == + CXGB4_EO_STATE_FLOWC_OPEN_REPLY || + eosw_txq->state == + CXGB4_EO_STATE_FLOWC_CLOSE_REPLY) && + eosw_txq->cidx == eosw_txq->flowc_idx)) { + flits = DIV_ROUND_UP(skb->len, 8); + if (eosw_txq->state == + CXGB4_EO_STATE_FLOWC_OPEN_REPLY) + eosw_txq->state = CXGB4_EO_STATE_ACTIVE; + else + eosw_txq->state = CXGB4_EO_STATE_CLOSED; + complete(&eosw_txq->completion); + } else { + hdr_len = eth_get_headlen(eosw_txq->netdev, + skb->data, + skb_headlen(skb)); + flits = ethofld_calc_tx_flits(q->adap, skb, + hdr_len); + } + eosw_txq_advance_index(&eosw_txq->cidx, 1, + eosw_txq->ndesc); + wrlen16 = DIV_ROUND_UP(flits * 8, 16); + credits -= wrlen16; + } + + eosw_txq->cred += cpl->credits; + eosw_txq->ncompl--; + + spin_unlock(&eosw_txq->lock); + + /* Schedule a tasklet to reclaim SKBs and restart ETHOFLD Tx, + * if there were packets waiting for completion. + */ + tasklet_schedule(&eosw_txq->qresume_tsk); + } + +out_done: + return 0; +} + +/* + * The MSI-X interrupt handler for an SGE response queue. + */ +irqreturn_t t4_sge_intr_msix(int irq, void *cookie) +{ + struct sge_rspq *q = cookie; + + napi_schedule(&q->napi); + return IRQ_HANDLED; +} + +/* + * Process the indirect interrupt entries in the interrupt queue and kick off + * NAPI for each queue that has generated an entry. + */ +static unsigned int process_intrq(struct adapter *adap) +{ + unsigned int credits; + const struct rsp_ctrl *rc; + struct sge_rspq *q = &adap->sge.intrq; + u32 val; + + spin_lock(&adap->sge.intrq_lock); + for (credits = 0; ; credits++) { + rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc)); + if (!is_new_response(rc, q)) + break; + + dma_rmb(); + if (RSPD_TYPE_G(rc->type_gen) == RSPD_TYPE_INTR_X) { + unsigned int qid = ntohl(rc->pldbuflen_qid); + + qid -= adap->sge.ingr_start; + napi_schedule(&adap->sge.ingr_map[qid]->napi); + } + + rspq_next(q); + } + + val = CIDXINC_V(credits) | SEINTARM_V(q->intr_params); + + /* If we don't have access to the new User GTS (T5+), use the old + * doorbell mechanism; otherwise use the new BAR2 mechanism. + */ + if (unlikely(q->bar2_addr == NULL)) { + t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A), + val | INGRESSQID_V(q->cntxt_id)); + } else { + writel(val | INGRESSQID_V(q->bar2_qid), + q->bar2_addr + SGE_UDB_GTS); + wmb(); + } + spin_unlock(&adap->sge.intrq_lock); + return credits; +} + +/* + * The MSI interrupt handler, which handles data events from SGE response queues + * as well as error and other async events as they all use the same MSI vector. + */ +static irqreturn_t t4_intr_msi(int irq, void *cookie) +{ + struct adapter *adap = cookie; + + if (adap->flags & CXGB4_MASTER_PF) + t4_slow_intr_handler(adap); + process_intrq(adap); + return IRQ_HANDLED; +} + +/* + * Interrupt handler for legacy INTx interrupts. + * Handles data events from SGE response queues as well as error and other + * async events as they all use the same interrupt line. + */ +static irqreturn_t t4_intr_intx(int irq, void *cookie) +{ + struct adapter *adap = cookie; + + t4_write_reg(adap, MYPF_REG(PCIE_PF_CLI_A), 0); + if (((adap->flags & CXGB4_MASTER_PF) && t4_slow_intr_handler(adap)) | + process_intrq(adap)) + return IRQ_HANDLED; + return IRQ_NONE; /* probably shared interrupt */ +} + +/** + * t4_intr_handler - select the top-level interrupt handler + * @adap: the adapter + * + * Selects the top-level interrupt handler based on the type of interrupts + * (MSI-X, MSI, or INTx). + */ +irq_handler_t t4_intr_handler(struct adapter *adap) +{ + if (adap->flags & CXGB4_USING_MSIX) + return t4_sge_intr_msix; + if (adap->flags & CXGB4_USING_MSI) + return t4_intr_msi; + return t4_intr_intx; +} + +static void sge_rx_timer_cb(struct timer_list *t) +{ + unsigned long m; + unsigned int i; + struct adapter *adap = from_timer(adap, t, sge.rx_timer); + struct sge *s = &adap->sge; + + for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++) + for (m = s->starving_fl[i]; m; m &= m - 1) { + struct sge_eth_rxq *rxq; + unsigned int id = __ffs(m) + i * BITS_PER_LONG; + struct sge_fl *fl = s->egr_map[id]; + + clear_bit(id, s->starving_fl); + smp_mb__after_atomic(); + + if (fl_starving(adap, fl)) { + rxq = container_of(fl, struct sge_eth_rxq, fl); + if (napi_reschedule(&rxq->rspq.napi)) + fl->starving++; + else + set_bit(id, s->starving_fl); + } + } + /* The remainder of the SGE RX Timer Callback routine is dedicated to + * global Master PF activities like checking for chip ingress stalls, + * etc. + */ + if (!(adap->flags & CXGB4_MASTER_PF)) + goto done; + + t4_idma_monitor(adap, &s->idma_monitor, HZ, RX_QCHECK_PERIOD); + +done: + mod_timer(&s->rx_timer, jiffies + RX_QCHECK_PERIOD); +} + +static void sge_tx_timer_cb(struct timer_list *t) +{ + struct adapter *adap = from_timer(adap, t, sge.tx_timer); + struct sge *s = &adap->sge; + unsigned long m, period; + unsigned int i, budget; + + for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++) + for (m = s->txq_maperr[i]; m; m &= m - 1) { + unsigned long id = __ffs(m) + i * BITS_PER_LONG; + struct sge_uld_txq *txq = s->egr_map[id]; + + clear_bit(id, s->txq_maperr); + tasklet_schedule(&txq->qresume_tsk); + } + + if (!is_t4(adap->params.chip)) { + struct sge_eth_txq *q = &s->ptptxq; + int avail; + + spin_lock(&adap->ptp_lock); + avail = reclaimable(&q->q); + + if (avail) { + free_tx_desc(adap, &q->q, avail, false); + q->q.in_use -= avail; + } + spin_unlock(&adap->ptp_lock); + } + + budget = MAX_TIMER_TX_RECLAIM; + i = s->ethtxq_rover; + do { + budget -= t4_sge_eth_txq_egress_update(adap, &s->ethtxq[i], + budget); + if (!budget) + break; + + if (++i >= s->ethqsets) + i = 0; + } while (i != s->ethtxq_rover); + s->ethtxq_rover = i; + + if (budget == 0) { + /* If we found too many reclaimable packets schedule a timer + * in the near future to continue where we left off. + */ + period = 2; + } else { + /* We reclaimed all reclaimable TX Descriptors, so reschedule + * at the normal period. + */ + period = TX_QCHECK_PERIOD; + } + + mod_timer(&s->tx_timer, jiffies + period); +} + +/** + * bar2_address - return the BAR2 address for an SGE Queue's Registers + * @adapter: the adapter + * @qid: the SGE Queue ID + * @qtype: the SGE Queue Type (Egress or Ingress) + * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues + * + * Returns the BAR2 address for the SGE Queue Registers associated with + * @qid. If BAR2 SGE Registers aren't available, returns NULL. Also + * returns the BAR2 Queue ID to be used with writes to the BAR2 SGE + * Queue Registers. If the BAR2 Queue ID is 0, then "Inferred Queue ID" + * Registers are supported (e.g. the Write Combining Doorbell Buffer). + */ +static void __iomem *bar2_address(struct adapter *adapter, + unsigned int qid, + enum t4_bar2_qtype qtype, + unsigned int *pbar2_qid) +{ + u64 bar2_qoffset; + int ret; + + ret = t4_bar2_sge_qregs(adapter, qid, qtype, 0, + &bar2_qoffset, pbar2_qid); + if (ret) + return NULL; + + return adapter->bar2 + bar2_qoffset; +} + +/* @intr_idx: MSI/MSI-X vector if >=0, -(absolute qid + 1) if < 0 + * @cong: < 0 -> no congestion feedback, >= 0 -> congestion channel map + */ +int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, + struct net_device *dev, int intr_idx, + struct sge_fl *fl, rspq_handler_t hnd, + rspq_flush_handler_t flush_hnd, int cong) +{ + int ret, flsz = 0; + struct fw_iq_cmd c; + struct sge *s = &adap->sge; + struct port_info *pi = netdev_priv(dev); + int relaxed = !(adap->flags & CXGB4_ROOT_NO_RELAXED_ORDERING); + + /* Size needs to be multiple of 16, including status entry. */ + iq->size = roundup(iq->size, 16); + + iq->desc = alloc_ring(adap->pdev_dev, iq->size, iq->iqe_len, 0, + &iq->phys_addr, NULL, 0, + dev_to_node(adap->pdev_dev)); + if (!iq->desc) + return -ENOMEM; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = htonl(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | FW_CMD_EXEC_F | + FW_IQ_CMD_PFN_V(adap->pf) | FW_IQ_CMD_VFN_V(0)); + c.alloc_to_len16 = htonl(FW_IQ_CMD_ALLOC_F | FW_IQ_CMD_IQSTART_F | + FW_LEN16(c)); + c.type_to_iqandstindex = htonl(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP) | + FW_IQ_CMD_IQASYNCH_V(fwevtq) | FW_IQ_CMD_VIID_V(pi->viid) | + FW_IQ_CMD_IQANDST_V(intr_idx < 0) | + FW_IQ_CMD_IQANUD_V(UPDATEDELIVERY_INTERRUPT_X) | + FW_IQ_CMD_IQANDSTINDEX_V(intr_idx >= 0 ? intr_idx : + -intr_idx - 1)); + c.iqdroprss_to_iqesize = htons(FW_IQ_CMD_IQPCIECH_V(pi->tx_chan) | + FW_IQ_CMD_IQGTSMODE_F | + FW_IQ_CMD_IQINTCNTTHRESH_V(iq->pktcnt_idx) | + FW_IQ_CMD_IQESIZE_V(ilog2(iq->iqe_len) - 4)); + c.iqsize = htons(iq->size); + c.iqaddr = cpu_to_be64(iq->phys_addr); + if (cong >= 0) + c.iqns_to_fl0congen = htonl(FW_IQ_CMD_IQFLINTCONGEN_F | + FW_IQ_CMD_IQTYPE_V(cong ? FW_IQ_IQTYPE_NIC + : FW_IQ_IQTYPE_OFLD)); + + if (fl) { + unsigned int chip_ver = + CHELSIO_CHIP_VERSION(adap->params.chip); + + /* Allocate the ring for the hardware free list (with space + * for its status page) along with the associated software + * descriptor ring. The free list size needs to be a multiple + * of the Egress Queue Unit and at least 2 Egress Units larger + * than the SGE's Egress Congrestion Threshold + * (fl_starve_thres - 1). + */ + if (fl->size < s->fl_starve_thres - 1 + 2 * 8) + fl->size = s->fl_starve_thres - 1 + 2 * 8; + fl->size = roundup(fl->size, 8); + fl->desc = alloc_ring(adap->pdev_dev, fl->size, sizeof(__be64), + sizeof(struct rx_sw_desc), &fl->addr, + &fl->sdesc, s->stat_len, + dev_to_node(adap->pdev_dev)); + if (!fl->desc) + goto fl_nomem; + + flsz = fl->size / 8 + s->stat_len / sizeof(struct tx_desc); + c.iqns_to_fl0congen |= htonl(FW_IQ_CMD_FL0PACKEN_F | + FW_IQ_CMD_FL0FETCHRO_V(relaxed) | + FW_IQ_CMD_FL0DATARO_V(relaxed) | + FW_IQ_CMD_FL0PADEN_F); + if (cong >= 0) + c.iqns_to_fl0congen |= + htonl(FW_IQ_CMD_FL0CNGCHMAP_V(cong) | + FW_IQ_CMD_FL0CONGCIF_F | + FW_IQ_CMD_FL0CONGEN_F); + /* In T6, for egress queue type FL there is internal overhead + * of 16B for header going into FLM module. Hence the maximum + * allowed burst size is 448 bytes. For T4/T5, the hardware + * doesn't coalesce fetch requests if more than 64 bytes of + * Free List pointers are provided, so we use a 128-byte Fetch + * Burst Minimum there (T6 implements coalescing so we can use + * the smaller 64-byte value there). + */ + c.fl0dcaen_to_fl0cidxfthresh = + htons(FW_IQ_CMD_FL0FBMIN_V(chip_ver <= CHELSIO_T5 ? + FETCHBURSTMIN_128B_X : + FETCHBURSTMIN_64B_T6_X) | + FW_IQ_CMD_FL0FBMAX_V((chip_ver <= CHELSIO_T5) ? + FETCHBURSTMAX_512B_X : + FETCHBURSTMAX_256B_X)); + c.fl0size = htons(flsz); + c.fl0addr = cpu_to_be64(fl->addr); + } + + ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c); + if (ret) + goto err; + + netif_napi_add(dev, &iq->napi, napi_rx_handler); + iq->cur_desc = iq->desc; + iq->cidx = 0; + iq->gen = 1; + iq->next_intr_params = iq->intr_params; + iq->cntxt_id = ntohs(c.iqid); + iq->abs_id = ntohs(c.physiqid); + iq->bar2_addr = bar2_address(adap, + iq->cntxt_id, + T4_BAR2_QTYPE_INGRESS, + &iq->bar2_qid); + iq->size--; /* subtract status entry */ + iq->netdev = dev; + iq->handler = hnd; + iq->flush_handler = flush_hnd; + + memset(&iq->lro_mgr, 0, sizeof(struct t4_lro_mgr)); + skb_queue_head_init(&iq->lro_mgr.lroq); + + /* set offset to -1 to distinguish ingress queues without FL */ + iq->offset = fl ? 0 : -1; + + adap->sge.ingr_map[iq->cntxt_id - adap->sge.ingr_start] = iq; + + if (fl) { + fl->cntxt_id = ntohs(c.fl0id); + fl->avail = fl->pend_cred = 0; + fl->pidx = fl->cidx = 0; + fl->alloc_failed = fl->large_alloc_failed = fl->starving = 0; + adap->sge.egr_map[fl->cntxt_id - adap->sge.egr_start] = fl; + + /* Note, we must initialize the BAR2 Free List User Doorbell + * information before refilling the Free List! + */ + fl->bar2_addr = bar2_address(adap, + fl->cntxt_id, + T4_BAR2_QTYPE_EGRESS, + &fl->bar2_qid); + refill_fl(adap, fl, fl_cap(fl), GFP_KERNEL); + } + + /* For T5 and later we attempt to set up the Congestion Manager values + * of the new RX Ethernet Queue. This should really be handled by + * firmware because it's more complex than any host driver wants to + * get involved with and it's different per chip and this is almost + * certainly wrong. Firmware would be wrong as well, but it would be + * a lot easier to fix in one place ... For now we do something very + * simple (and hopefully less wrong). + */ + if (!is_t4(adap->params.chip) && cong >= 0) { + u32 param, val, ch_map = 0; + int i; + u16 cng_ch_bits_log = adap->params.arch.cng_ch_bits_log; + + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | + FW_PARAMS_PARAM_YZ_V(iq->cntxt_id)); + if (cong == 0) { + val = CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_QUEUE_X); + } else { + val = + CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_CHANNEL_X); + for (i = 0; i < 4; i++) { + if (cong & (1 << i)) + ch_map |= 1 << (i << cng_ch_bits_log); + } + val |= CONMCTXT_CNGCHMAP_V(ch_map); + } + ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, + ¶m, &val); + if (ret) + dev_warn(adap->pdev_dev, "Failed to set Congestion" + " Manager Context for Ingress Queue %d: %d\n", + iq->cntxt_id, -ret); + } + + return 0; + +fl_nomem: + ret = -ENOMEM; +err: + if (iq->desc) { + dma_free_coherent(adap->pdev_dev, iq->size * iq->iqe_len, + iq->desc, iq->phys_addr); + iq->desc = NULL; + } + if (fl && fl->desc) { + kfree(fl->sdesc); + fl->sdesc = NULL; + dma_free_coherent(adap->pdev_dev, flsz * sizeof(struct tx_desc), + fl->desc, fl->addr); + fl->desc = NULL; + } + return ret; +} + +static void init_txq(struct adapter *adap, struct sge_txq *q, unsigned int id) +{ + q->cntxt_id = id; + q->bar2_addr = bar2_address(adap, + q->cntxt_id, + T4_BAR2_QTYPE_EGRESS, + &q->bar2_qid); + q->in_use = 0; + q->cidx = q->pidx = 0; + q->stops = q->restarts = 0; + q->stat = (void *)&q->desc[q->size]; + spin_lock_init(&q->db_lock); + adap->sge.egr_map[id - adap->sge.egr_start] = q; +} + +/** + * t4_sge_alloc_eth_txq - allocate an Ethernet TX Queue + * @adap: the adapter + * @txq: the SGE Ethernet TX Queue to initialize + * @dev: the Linux Network Device + * @netdevq: the corresponding Linux TX Queue + * @iqid: the Ingress Queue to which to deliver CIDX Update messages + * @dbqt: whether this TX Queue will use the SGE Doorbell Queue Timers + */ +int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq, + struct net_device *dev, struct netdev_queue *netdevq, + unsigned int iqid, u8 dbqt) +{ + unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip); + struct port_info *pi = netdev_priv(dev); + struct sge *s = &adap->sge; + struct fw_eq_eth_cmd c; + int ret, nentries; + + /* Add status entries */ + nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc); + + txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size, + sizeof(struct tx_desc), sizeof(struct tx_sw_desc), + &txq->q.phys_addr, &txq->q.sdesc, s->stat_len, + netdev_queue_numa_node_read(netdevq)); + if (!txq->q.desc) + return -ENOMEM; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_ETH_CMD) | FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | FW_CMD_EXEC_F | + FW_EQ_ETH_CMD_PFN_V(adap->pf) | + FW_EQ_ETH_CMD_VFN_V(0)); + c.alloc_to_len16 = htonl(FW_EQ_ETH_CMD_ALLOC_F | + FW_EQ_ETH_CMD_EQSTART_F | FW_LEN16(c)); + + /* For TX Ethernet Queues using the SGE Doorbell Queue Timer + * mechanism, we use Ingress Queue messages for Hardware Consumer + * Index Updates on the TX Queue. Otherwise we have the Hardware + * write the CIDX Updates into the Status Page at the end of the + * TX Queue. + */ + c.autoequiqe_to_viid = htonl(((chip_ver <= CHELSIO_T5) ? + FW_EQ_ETH_CMD_AUTOEQUIQE_F : + FW_EQ_ETH_CMD_AUTOEQUEQE_F) | + FW_EQ_ETH_CMD_VIID_V(pi->viid)); + + c.fetchszm_to_iqid = + htonl(FW_EQ_ETH_CMD_HOSTFCMODE_V((chip_ver <= CHELSIO_T5) ? + HOSTFCMODE_INGRESS_QUEUE_X : + HOSTFCMODE_STATUS_PAGE_X) | + FW_EQ_ETH_CMD_PCIECHN_V(pi->tx_chan) | + FW_EQ_ETH_CMD_FETCHRO_F | FW_EQ_ETH_CMD_IQID_V(iqid)); + + /* Note that the CIDX Flush Threshold should match MAX_TX_RECLAIM. */ + c.dcaen_to_eqsize = + htonl(FW_EQ_ETH_CMD_FBMIN_V(chip_ver <= CHELSIO_T5 + ? FETCHBURSTMIN_64B_X + : FETCHBURSTMIN_64B_T6_X) | + FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) | + FW_EQ_ETH_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) | + FW_EQ_ETH_CMD_CIDXFTHRESHO_V(chip_ver == CHELSIO_T5) | + FW_EQ_ETH_CMD_EQSIZE_V(nentries)); + + c.eqaddr = cpu_to_be64(txq->q.phys_addr); + + /* If we're using the SGE Doorbell Queue Timer mechanism, pass in the + * currently configured Timer Index. THis can be changed later via an + * ethtool -C tx-usecs {Timer Val} command. Note that the SGE + * Doorbell Queue mode is currently automatically enabled in the + * Firmware by setting either AUTOEQUEQE or AUTOEQUIQE ... + */ + if (dbqt) + c.timeren_timerix = + cpu_to_be32(FW_EQ_ETH_CMD_TIMEREN_F | + FW_EQ_ETH_CMD_TIMERIX_V(txq->dbqtimerix)); + + ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c); + if (ret) { + kfree(txq->q.sdesc); + txq->q.sdesc = NULL; + dma_free_coherent(adap->pdev_dev, + nentries * sizeof(struct tx_desc), + txq->q.desc, txq->q.phys_addr); + txq->q.desc = NULL; + return ret; + } + + txq->q.q_type = CXGB4_TXQ_ETH; + init_txq(adap, &txq->q, FW_EQ_ETH_CMD_EQID_G(ntohl(c.eqid_pkd))); + txq->txq = netdevq; + txq->tso = 0; + txq->uso = 0; + txq->tx_cso = 0; + txq->vlan_ins = 0; + txq->mapping_err = 0; + txq->dbqt = dbqt; + + return 0; +} + +int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq, + struct net_device *dev, unsigned int iqid, + unsigned int cmplqid) +{ + unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip); + struct port_info *pi = netdev_priv(dev); + struct sge *s = &adap->sge; + struct fw_eq_ctrl_cmd c; + int ret, nentries; + + /* Add status entries */ + nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc); + + txq->q.desc = alloc_ring(adap->pdev_dev, nentries, + sizeof(struct tx_desc), 0, &txq->q.phys_addr, + NULL, 0, dev_to_node(adap->pdev_dev)); + if (!txq->q.desc) + return -ENOMEM; + + c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_CTRL_CMD) | FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | FW_CMD_EXEC_F | + FW_EQ_CTRL_CMD_PFN_V(adap->pf) | + FW_EQ_CTRL_CMD_VFN_V(0)); + c.alloc_to_len16 = htonl(FW_EQ_CTRL_CMD_ALLOC_F | + FW_EQ_CTRL_CMD_EQSTART_F | FW_LEN16(c)); + c.cmpliqid_eqid = htonl(FW_EQ_CTRL_CMD_CMPLIQID_V(cmplqid)); + c.physeqid_pkd = htonl(0); + c.fetchszm_to_iqid = + htonl(FW_EQ_CTRL_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) | + FW_EQ_CTRL_CMD_PCIECHN_V(pi->tx_chan) | + FW_EQ_CTRL_CMD_FETCHRO_F | FW_EQ_CTRL_CMD_IQID_V(iqid)); + c.dcaen_to_eqsize = + htonl(FW_EQ_CTRL_CMD_FBMIN_V(chip_ver <= CHELSIO_T5 + ? FETCHBURSTMIN_64B_X + : FETCHBURSTMIN_64B_T6_X) | + FW_EQ_CTRL_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) | + FW_EQ_CTRL_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) | + FW_EQ_CTRL_CMD_EQSIZE_V(nentries)); + c.eqaddr = cpu_to_be64(txq->q.phys_addr); + + ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c); + if (ret) { + dma_free_coherent(adap->pdev_dev, + nentries * sizeof(struct tx_desc), + txq->q.desc, txq->q.phys_addr); + txq->q.desc = NULL; + return ret; + } + + txq->q.q_type = CXGB4_TXQ_CTRL; + init_txq(adap, &txq->q, FW_EQ_CTRL_CMD_EQID_G(ntohl(c.cmpliqid_eqid))); + txq->adap = adap; + skb_queue_head_init(&txq->sendq); + tasklet_setup(&txq->qresume_tsk, restart_ctrlq); + txq->full = 0; + return 0; +} + +int t4_sge_mod_ctrl_txq(struct adapter *adap, unsigned int eqid, + unsigned int cmplqid) +{ + u32 param, val; + + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL) | + FW_PARAMS_PARAM_YZ_V(eqid)); + val = cmplqid; + return t4_set_params(adap, adap->mbox, adap->pf, 0, 1, ¶m, &val); +} + +static int t4_sge_alloc_ofld_txq(struct adapter *adap, struct sge_txq *q, + struct net_device *dev, u32 cmd, u32 iqid) +{ + unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip); + struct port_info *pi = netdev_priv(dev); + struct sge *s = &adap->sge; + struct fw_eq_ofld_cmd c; + u32 fb_min, nentries; + int ret; + + /* Add status entries */ + nentries = q->size + s->stat_len / sizeof(struct tx_desc); + q->desc = alloc_ring(adap->pdev_dev, q->size, sizeof(struct tx_desc), + sizeof(struct tx_sw_desc), &q->phys_addr, + &q->sdesc, s->stat_len, NUMA_NO_NODE); + if (!q->desc) + return -ENOMEM; + + if (chip_ver <= CHELSIO_T5) + fb_min = FETCHBURSTMIN_64B_X; + else + fb_min = FETCHBURSTMIN_64B_T6_X; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = htonl(FW_CMD_OP_V(cmd) | FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | FW_CMD_EXEC_F | + FW_EQ_OFLD_CMD_PFN_V(adap->pf) | + FW_EQ_OFLD_CMD_VFN_V(0)); + c.alloc_to_len16 = htonl(FW_EQ_OFLD_CMD_ALLOC_F | + FW_EQ_OFLD_CMD_EQSTART_F | FW_LEN16(c)); + c.fetchszm_to_iqid = + htonl(FW_EQ_OFLD_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) | + FW_EQ_OFLD_CMD_PCIECHN_V(pi->tx_chan) | + FW_EQ_OFLD_CMD_FETCHRO_F | FW_EQ_OFLD_CMD_IQID_V(iqid)); + c.dcaen_to_eqsize = + htonl(FW_EQ_OFLD_CMD_FBMIN_V(fb_min) | + FW_EQ_OFLD_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) | + FW_EQ_OFLD_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) | + FW_EQ_OFLD_CMD_EQSIZE_V(nentries)); + c.eqaddr = cpu_to_be64(q->phys_addr); + + ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c); + if (ret) { + kfree(q->sdesc); + q->sdesc = NULL; + dma_free_coherent(adap->pdev_dev, + nentries * sizeof(struct tx_desc), + q->desc, q->phys_addr); + q->desc = NULL; + return ret; + } + + init_txq(adap, q, FW_EQ_OFLD_CMD_EQID_G(ntohl(c.eqid_pkd))); + return 0; +} + +int t4_sge_alloc_uld_txq(struct adapter *adap, struct sge_uld_txq *txq, + struct net_device *dev, unsigned int iqid, + unsigned int uld_type) +{ + u32 cmd = FW_EQ_OFLD_CMD; + int ret; + + if (unlikely(uld_type == CXGB4_TX_CRYPTO)) + cmd = FW_EQ_CTRL_CMD; + + ret = t4_sge_alloc_ofld_txq(adap, &txq->q, dev, cmd, iqid); + if (ret) + return ret; + + txq->q.q_type = CXGB4_TXQ_ULD; + txq->adap = adap; + skb_queue_head_init(&txq->sendq); + tasklet_setup(&txq->qresume_tsk, restart_ofldq); + txq->full = 0; + txq->mapping_err = 0; + return 0; +} + +int t4_sge_alloc_ethofld_txq(struct adapter *adap, struct sge_eohw_txq *txq, + struct net_device *dev, u32 iqid) +{ + int ret; + + ret = t4_sge_alloc_ofld_txq(adap, &txq->q, dev, FW_EQ_OFLD_CMD, iqid); + if (ret) + return ret; + + txq->q.q_type = CXGB4_TXQ_ULD; + spin_lock_init(&txq->lock); + txq->adap = adap; + txq->tso = 0; + txq->uso = 0; + txq->tx_cso = 0; + txq->vlan_ins = 0; + txq->mapping_err = 0; + return 0; +} + +void free_txq(struct adapter *adap, struct sge_txq *q) +{ + struct sge *s = &adap->sge; + + dma_free_coherent(adap->pdev_dev, + q->size * sizeof(struct tx_desc) + s->stat_len, + q->desc, q->phys_addr); + q->cntxt_id = 0; + q->sdesc = NULL; + q->desc = NULL; +} + +void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq, + struct sge_fl *fl) +{ + struct sge *s = &adap->sge; + unsigned int fl_id = fl ? fl->cntxt_id : 0xffff; + + adap->sge.ingr_map[rq->cntxt_id - adap->sge.ingr_start] = NULL; + t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP, + rq->cntxt_id, fl_id, 0xffff); + dma_free_coherent(adap->pdev_dev, (rq->size + 1) * rq->iqe_len, + rq->desc, rq->phys_addr); + netif_napi_del(&rq->napi); + rq->netdev = NULL; + rq->cntxt_id = rq->abs_id = 0; + rq->desc = NULL; + + if (fl) { + free_rx_bufs(adap, fl, fl->avail); + dma_free_coherent(adap->pdev_dev, fl->size * 8 + s->stat_len, + fl->desc, fl->addr); + kfree(fl->sdesc); + fl->sdesc = NULL; + fl->cntxt_id = 0; + fl->desc = NULL; + } +} + +/** + * t4_free_ofld_rxqs - free a block of consecutive Rx queues + * @adap: the adapter + * @n: number of queues + * @q: pointer to first queue + * + * Release the resources of a consecutive block of offload Rx queues. + */ +void t4_free_ofld_rxqs(struct adapter *adap, int n, struct sge_ofld_rxq *q) +{ + for ( ; n; n--, q++) + if (q->rspq.desc) + free_rspq_fl(adap, &q->rspq, + q->fl.size ? &q->fl : NULL); +} + +void t4_sge_free_ethofld_txq(struct adapter *adap, struct sge_eohw_txq *txq) +{ + if (txq->q.desc) { + t4_ofld_eq_free(adap, adap->mbox, adap->pf, 0, + txq->q.cntxt_id); + free_tx_desc(adap, &txq->q, txq->q.in_use, false); + kfree(txq->q.sdesc); + free_txq(adap, &txq->q); + } +} + +/** + * t4_free_sge_resources - free SGE resources + * @adap: the adapter + * + * Frees resources used by the SGE queue sets. + */ +void t4_free_sge_resources(struct adapter *adap) +{ + int i; + struct sge_eth_rxq *eq; + struct sge_eth_txq *etq; + + /* stop all Rx queues in order to start them draining */ + for (i = 0; i < adap->sge.ethqsets; i++) { + eq = &adap->sge.ethrxq[i]; + if (eq->rspq.desc) + t4_iq_stop(adap, adap->mbox, adap->pf, 0, + FW_IQ_TYPE_FL_INT_CAP, + eq->rspq.cntxt_id, + eq->fl.size ? eq->fl.cntxt_id : 0xffff, + 0xffff); + } + + /* clean up Ethernet Tx/Rx queues */ + for (i = 0; i < adap->sge.ethqsets; i++) { + eq = &adap->sge.ethrxq[i]; + if (eq->rspq.desc) + free_rspq_fl(adap, &eq->rspq, + eq->fl.size ? &eq->fl : NULL); + if (eq->msix) { + cxgb4_free_msix_idx_in_bmap(adap, eq->msix->idx); + eq->msix = NULL; + } + + etq = &adap->sge.ethtxq[i]; + if (etq->q.desc) { + t4_eth_eq_free(adap, adap->mbox, adap->pf, 0, + etq->q.cntxt_id); + __netif_tx_lock_bh(etq->txq); + free_tx_desc(adap, &etq->q, etq->q.in_use, true); + __netif_tx_unlock_bh(etq->txq); + kfree(etq->q.sdesc); + free_txq(adap, &etq->q); + } + } + + /* clean up control Tx queues */ + for (i = 0; i < ARRAY_SIZE(adap->sge.ctrlq); i++) { + struct sge_ctrl_txq *cq = &adap->sge.ctrlq[i]; + + if (cq->q.desc) { + tasklet_kill(&cq->qresume_tsk); + t4_ctrl_eq_free(adap, adap->mbox, adap->pf, 0, + cq->q.cntxt_id); + __skb_queue_purge(&cq->sendq); + free_txq(adap, &cq->q); + } + } + + if (adap->sge.fw_evtq.desc) { + free_rspq_fl(adap, &adap->sge.fw_evtq, NULL); + if (adap->sge.fwevtq_msix_idx >= 0) + cxgb4_free_msix_idx_in_bmap(adap, + adap->sge.fwevtq_msix_idx); + } + + if (adap->sge.nd_msix_idx >= 0) + cxgb4_free_msix_idx_in_bmap(adap, adap->sge.nd_msix_idx); + + if (adap->sge.intrq.desc) + free_rspq_fl(adap, &adap->sge.intrq, NULL); + + if (!is_t4(adap->params.chip)) { + etq = &adap->sge.ptptxq; + if (etq->q.desc) { + t4_eth_eq_free(adap, adap->mbox, adap->pf, 0, + etq->q.cntxt_id); + spin_lock_bh(&adap->ptp_lock); + free_tx_desc(adap, &etq->q, etq->q.in_use, true); + spin_unlock_bh(&adap->ptp_lock); + kfree(etq->q.sdesc); + free_txq(adap, &etq->q); + } + } + + /* clear the reverse egress queue map */ + memset(adap->sge.egr_map, 0, + adap->sge.egr_sz * sizeof(*adap->sge.egr_map)); +} + +void t4_sge_start(struct adapter *adap) +{ + adap->sge.ethtxq_rover = 0; + mod_timer(&adap->sge.rx_timer, jiffies + RX_QCHECK_PERIOD); + mod_timer(&adap->sge.tx_timer, jiffies + TX_QCHECK_PERIOD); +} + +/** + * t4_sge_stop - disable SGE operation + * @adap: the adapter + * + * Stop tasklets and timers associated with the DMA engine. Note that + * this is effective only if measures have been taken to disable any HW + * events that may restart them. + */ +void t4_sge_stop(struct adapter *adap) +{ + int i; + struct sge *s = &adap->sge; + + if (s->rx_timer.function) + del_timer_sync(&s->rx_timer); + if (s->tx_timer.function) + del_timer_sync(&s->tx_timer); + + if (is_offload(adap)) { + struct sge_uld_txq_info *txq_info; + + txq_info = adap->sge.uld_txq_info[CXGB4_TX_OFLD]; + if (txq_info) { + struct sge_uld_txq *txq = txq_info->uldtxq; + + for_each_ofldtxq(&adap->sge, i) { + if (txq->q.desc) + tasklet_kill(&txq->qresume_tsk); + } + } + } + + if (is_pci_uld(adap)) { + struct sge_uld_txq_info *txq_info; + + txq_info = adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; + if (txq_info) { + struct sge_uld_txq *txq = txq_info->uldtxq; + + for_each_ofldtxq(&adap->sge, i) { + if (txq->q.desc) + tasklet_kill(&txq->qresume_tsk); + } + } + } + + for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) { + struct sge_ctrl_txq *cq = &s->ctrlq[i]; + + if (cq->q.desc) + tasklet_kill(&cq->qresume_tsk); + } +} + +/** + * t4_sge_init_soft - grab core SGE values needed by SGE code + * @adap: the adapter + * + * We need to grab the SGE operating parameters that we need to have + * in order to do our job and make sure we can live with them. + */ + +static int t4_sge_init_soft(struct adapter *adap) +{ + struct sge *s = &adap->sge; + u32 fl_small_pg, fl_large_pg, fl_small_mtu, fl_large_mtu; + u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5; + u32 ingress_rx_threshold; + + /* + * Verify that CPL messages are going to the Ingress Queue for + * process_responses() and that only packet data is going to the + * Free Lists. + */ + if ((t4_read_reg(adap, SGE_CONTROL_A) & RXPKTCPLMODE_F) != + RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X)) { + dev_err(adap->pdev_dev, "bad SGE CPL MODE\n"); + return -EINVAL; + } + + /* + * Validate the Host Buffer Register Array indices that we want to + * use ... + * + * XXX Note that we should really read through the Host Buffer Size + * XXX register array and find the indices of the Buffer Sizes which + * XXX meet our needs! + */ + #define READ_FL_BUF(x) \ + t4_read_reg(adap, SGE_FL_BUFFER_SIZE0_A+(x)*sizeof(u32)) + + fl_small_pg = READ_FL_BUF(RX_SMALL_PG_BUF); + fl_large_pg = READ_FL_BUF(RX_LARGE_PG_BUF); + fl_small_mtu = READ_FL_BUF(RX_SMALL_MTU_BUF); + fl_large_mtu = READ_FL_BUF(RX_LARGE_MTU_BUF); + + /* We only bother using the Large Page logic if the Large Page Buffer + * is larger than our Page Size Buffer. + */ + if (fl_large_pg <= fl_small_pg) + fl_large_pg = 0; + + #undef READ_FL_BUF + + /* The Page Size Buffer must be exactly equal to our Page Size and the + * Large Page Size Buffer should be 0 (per above) or a power of 2. + */ + if (fl_small_pg != PAGE_SIZE || + (fl_large_pg & (fl_large_pg-1)) != 0) { + dev_err(adap->pdev_dev, "bad SGE FL page buffer sizes [%d, %d]\n", + fl_small_pg, fl_large_pg); + return -EINVAL; + } + if (fl_large_pg) + s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT; + + if (fl_small_mtu < FL_MTU_SMALL_BUFSIZE(adap) || + fl_large_mtu < FL_MTU_LARGE_BUFSIZE(adap)) { + dev_err(adap->pdev_dev, "bad SGE FL MTU sizes [%d, %d]\n", + fl_small_mtu, fl_large_mtu); + return -EINVAL; + } + + /* + * Retrieve our RX interrupt holdoff timer values and counter + * threshold values from the SGE parameters. + */ + timer_value_0_and_1 = t4_read_reg(adap, SGE_TIMER_VALUE_0_AND_1_A); + timer_value_2_and_3 = t4_read_reg(adap, SGE_TIMER_VALUE_2_AND_3_A); + timer_value_4_and_5 = t4_read_reg(adap, SGE_TIMER_VALUE_4_AND_5_A); + s->timer_val[0] = core_ticks_to_us(adap, + TIMERVALUE0_G(timer_value_0_and_1)); + s->timer_val[1] = core_ticks_to_us(adap, + TIMERVALUE1_G(timer_value_0_and_1)); + s->timer_val[2] = core_ticks_to_us(adap, + TIMERVALUE2_G(timer_value_2_and_3)); + s->timer_val[3] = core_ticks_to_us(adap, + TIMERVALUE3_G(timer_value_2_and_3)); + s->timer_val[4] = core_ticks_to_us(adap, + TIMERVALUE4_G(timer_value_4_and_5)); + s->timer_val[5] = core_ticks_to_us(adap, + TIMERVALUE5_G(timer_value_4_and_5)); + + ingress_rx_threshold = t4_read_reg(adap, SGE_INGRESS_RX_THRESHOLD_A); + s->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold); + s->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold); + s->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold); + s->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold); + + return 0; +} + +/** + * t4_sge_init - initialize SGE + * @adap: the adapter + * + * Perform low-level SGE code initialization needed every time after a + * chip reset. + */ +int t4_sge_init(struct adapter *adap) +{ + struct sge *s = &adap->sge; + u32 sge_control, sge_conm_ctrl; + int ret, egress_threshold; + + /* + * Ingress Padding Boundary and Egress Status Page Size are set up by + * t4_fixup_host_params(). + */ + sge_control = t4_read_reg(adap, SGE_CONTROL_A); + s->pktshift = PKTSHIFT_G(sge_control); + s->stat_len = (sge_control & EGRSTATUSPAGESIZE_F) ? 128 : 64; + + s->fl_align = t4_fl_pkt_align(adap); + ret = t4_sge_init_soft(adap); + if (ret < 0) + return ret; + + /* + * A FL with <= fl_starve_thres buffers is starving and a periodic + * timer will attempt to refill it. This needs to be larger than the + * SGE's Egress Congestion Threshold. If it isn't, then we can get + * stuck waiting for new packets while the SGE is waiting for us to + * give it more Free List entries. (Note that the SGE's Egress + * Congestion Threshold is in units of 2 Free List pointers.) For T4, + * there was only a single field to control this. For T5 there's the + * original field which now only applies to Unpacked Mode Free List + * buffers and a new field which only applies to Packed Mode Free List + * buffers. + */ + sge_conm_ctrl = t4_read_reg(adap, SGE_CONM_CTRL_A); + switch (CHELSIO_CHIP_VERSION(adap->params.chip)) { + case CHELSIO_T4: + egress_threshold = EGRTHRESHOLD_G(sge_conm_ctrl); + break; + case CHELSIO_T5: + egress_threshold = EGRTHRESHOLDPACKING_G(sge_conm_ctrl); + break; + case CHELSIO_T6: + egress_threshold = T6_EGRTHRESHOLDPACKING_G(sge_conm_ctrl); + break; + default: + dev_err(adap->pdev_dev, "Unsupported Chip version %d\n", + CHELSIO_CHIP_VERSION(adap->params.chip)); + return -EINVAL; + } + s->fl_starve_thres = 2*egress_threshold + 1; + + t4_idma_monitor_init(adap, &s->idma_monitor); + + /* Set up timers used for recuring callbacks to process RX and TX + * administrative tasks. + */ + timer_setup(&s->rx_timer, sge_rx_timer_cb, 0); + timer_setup(&s->tx_timer, sge_tx_timer_cb, 0); + + spin_lock_init(&s->intrq_lock); + + return 0; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/smt.c b/drivers/net/ethernet/chelsio/cxgb4/smt.c new file mode 100644 index 000000000..e617e4aab --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/smt.c @@ -0,0 +1,248 @@ +/* + * This file is part of the Chelsio T4/T5/T6 Ethernet driver for Linux. + * + * Copyright (c) 2017 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "cxgb4.h" +#include "smt.h" +#include "t4_msg.h" +#include "t4fw_api.h" +#include "t4_regs.h" +#include "t4_values.h" + +struct smt_data *t4_init_smt(void) +{ + unsigned int smt_size; + struct smt_data *s; + int i; + + smt_size = SMT_SIZE; + + s = kvzalloc(struct_size(s, smtab, smt_size), GFP_KERNEL); + if (!s) + return NULL; + s->smt_size = smt_size; + rwlock_init(&s->lock); + for (i = 0; i < s->smt_size; ++i) { + s->smtab[i].idx = i; + s->smtab[i].state = SMT_STATE_UNUSED; + eth_zero_addr(s->smtab[i].src_mac); + spin_lock_init(&s->smtab[i].lock); + s->smtab[i].refcnt = 0; + } + return s; +} + +static struct smt_entry *find_or_alloc_smte(struct smt_data *s, u8 *smac) +{ + struct smt_entry *first_free = NULL; + struct smt_entry *e, *end; + + for (e = &s->smtab[0], end = &s->smtab[s->smt_size]; e != end; ++e) { + if (e->refcnt == 0) { + if (!first_free) + first_free = e; + } else { + if (e->state == SMT_STATE_SWITCHING) { + /* This entry is actually in use. See if we can + * re-use it ? + */ + if (memcmp(e->src_mac, smac, ETH_ALEN) == 0) + goto found_reuse; + } + } + } + + if (first_free) { + e = first_free; + goto found; + } + return NULL; + +found: + e->state = SMT_STATE_UNUSED; + +found_reuse: + return e; +} + +static void t4_smte_free(struct smt_entry *e) +{ + if (e->refcnt == 0) { /* hasn't been recycled */ + e->state = SMT_STATE_UNUSED; + } +} + +/** + * cxgb4_smt_release - Release SMT entry + * @e: smt entry to release + * + * Releases ref count and frees up an smt entry from SMT table + */ +void cxgb4_smt_release(struct smt_entry *e) +{ + spin_lock_bh(&e->lock); + if ((--e->refcnt) == 0) + t4_smte_free(e); + spin_unlock_bh(&e->lock); +} +EXPORT_SYMBOL(cxgb4_smt_release); + +void do_smt_write_rpl(struct adapter *adap, const struct cpl_smt_write_rpl *rpl) +{ + unsigned int smtidx = TID_TID_G(GET_TID(rpl)); + struct smt_data *s = adap->smt; + + if (unlikely(rpl->status != CPL_ERR_NONE)) { + struct smt_entry *e = &s->smtab[smtidx]; + + dev_err(adap->pdev_dev, + "Unexpected SMT_WRITE_RPL status %u for entry %u\n", + rpl->status, smtidx); + spin_lock(&e->lock); + e->state = SMT_STATE_ERROR; + spin_unlock(&e->lock); + return; + } +} + +static int write_smt_entry(struct adapter *adapter, struct smt_entry *e) +{ + struct cpl_t6_smt_write_req *t6req; + struct smt_data *s = adapter->smt; + struct cpl_smt_write_req *req; + struct sk_buff *skb; + int size; + u8 row; + + if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) { + size = sizeof(*req); + skb = alloc_skb(size, GFP_ATOMIC); + if (!skb) + return -ENOMEM; + /* Source MAC Table (SMT) contains 256 SMAC entries + * organized in 128 rows of 2 entries each. + */ + req = (struct cpl_smt_write_req *)__skb_put(skb, size); + INIT_TP_WR(req, 0); + + /* Each row contains an SMAC pair. + * LSB selects the SMAC entry within a row + */ + row = (e->idx >> 1); + if (e->idx & 1) { + req->pfvf1 = 0x0; + memcpy(req->src_mac1, e->src_mac, ETH_ALEN); + + /* fill pfvf0/src_mac0 with entry + * at prev index from smt-tab. + */ + req->pfvf0 = 0x0; + memcpy(req->src_mac0, s->smtab[e->idx - 1].src_mac, + ETH_ALEN); + } else { + req->pfvf0 = 0x0; + memcpy(req->src_mac0, e->src_mac, ETH_ALEN); + + /* fill pfvf1/src_mac1 with entry + * at next index from smt-tab + */ + req->pfvf1 = 0x0; + memcpy(req->src_mac1, s->smtab[e->idx + 1].src_mac, + ETH_ALEN); + } + } else { + size = sizeof(*t6req); + skb = alloc_skb(size, GFP_ATOMIC); + if (!skb) + return -ENOMEM; + /* Source MAC Table (SMT) contains 256 SMAC entries */ + t6req = (struct cpl_t6_smt_write_req *)__skb_put(skb, size); + INIT_TP_WR(t6req, 0); + req = (struct cpl_smt_write_req *)t6req; + + /* fill pfvf0/src_mac0 from smt-tab */ + req->pfvf0 = 0x0; + memcpy(req->src_mac0, s->smtab[e->idx].src_mac, ETH_ALEN); + row = e->idx; + } + + OPCODE_TID(req) = + htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, e->idx | + TID_QID_V(adapter->sge.fw_evtq.abs_id))); + req->params = htonl(SMTW_NORPL_V(0) | + SMTW_IDX_V(row) | + SMTW_OVLAN_IDX_V(0)); + t4_mgmt_tx(adapter, skb); + return 0; +} + +static struct smt_entry *t4_smt_alloc_switching(struct adapter *adap, u16 pfvf, + u8 *smac) +{ + struct smt_data *s = adap->smt; + struct smt_entry *e; + + write_lock_bh(&s->lock); + e = find_or_alloc_smte(s, smac); + if (e) { + spin_lock(&e->lock); + if (!e->refcnt) { + e->refcnt = 1; + e->state = SMT_STATE_SWITCHING; + e->pfvf = pfvf; + memcpy(e->src_mac, smac, ETH_ALEN); + write_smt_entry(adap, e); + } else { + ++e->refcnt; + } + spin_unlock(&e->lock); + } + write_unlock_bh(&s->lock); + return e; +} + +/** + * cxgb4_smt_alloc_switching - Allocates an SMT entry for switch filters. + * @dev: net_device pointer + * @smac: MAC address to add to SMT + * Returns pointer to the SMT entry created + * + * Allocates an SMT entry to be used by switching rule of a filter. + */ +struct smt_entry *cxgb4_smt_alloc_switching(struct net_device *dev, u8 *smac) +{ + struct adapter *adap = netdev2adap(dev); + + return t4_smt_alloc_switching(adap, 0x0, smac); +} +EXPORT_SYMBOL(cxgb4_smt_alloc_switching); diff --git a/drivers/net/ethernet/chelsio/cxgb4/smt.h b/drivers/net/ethernet/chelsio/cxgb4/smt.h new file mode 100644 index 000000000..541249d78 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/smt.h @@ -0,0 +1,76 @@ +/* + * This file is part of the Chelsio T4/T5/T6 Ethernet driver for Linux. + * + * Copyright (c) 2017 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_SMT_H +#define __CXGB4_SMT_H + +#include <linux/spinlock.h> +#include <linux/if_ether.h> +#include <linux/atomic.h> + +struct adapter; +struct cpl_smt_write_rpl; + +/* SMT related handling. Heavily adapted based on l2t ops in l2t.h/l2t.c + */ +enum { + SMT_STATE_SWITCHING, + SMT_STATE_UNUSED, + SMT_STATE_ERROR +}; + +enum { + SMT_SIZE = 256 +}; + +struct smt_entry { + u16 state; + u16 idx; + u16 pfvf; + u8 src_mac[ETH_ALEN]; + int refcnt; + spinlock_t lock; /* protect smt entry add,removal */ +}; + +struct smt_data { + unsigned int smt_size; + rwlock_t lock; + struct smt_entry smtab[]; +}; + +struct smt_data *t4_init_smt(void); +struct smt_entry *cxgb4_smt_alloc_switching(struct net_device *dev, u8 *smac); +void cxgb4_smt_release(struct smt_entry *e); +void do_smt_write_rpl(struct adapter *p, const struct cpl_smt_write_rpl *rpl); +#endif /* __CXGB4_SMT_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/srq.c b/drivers/net/ethernet/chelsio/cxgb4/srq.c new file mode 100644 index 000000000..9a54302bb --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/srq.c @@ -0,0 +1,137 @@ +/* + * This file is part of the Chelsio T6 Ethernet driver for Linux. + * + * Copyright (c) 2017-2018 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "cxgb4.h" +#include "t4_msg.h" +#include "srq.h" + +struct srq_data *t4_init_srq(int srq_size) +{ + struct srq_data *s; + + s = kvzalloc(sizeof(*s), GFP_KERNEL); + if (!s) + return NULL; + + s->srq_size = srq_size; + init_completion(&s->comp); + mutex_init(&s->lock); + + return s; +} + +/* cxgb4_get_srq_entry: read the SRQ table entry + * @dev: Pointer to the net_device + * @idx: Index to the srq + * @entryp: pointer to the srq entry + * + * Sends CPL_SRQ_TABLE_REQ message for the given index. + * Contents will be returned in CPL_SRQ_TABLE_RPL message. + * + * Returns zero if the read is successful, else a error + * number will be returned. Caller should not use the srq + * entry if the return value is non-zero. + * + * + */ +int cxgb4_get_srq_entry(struct net_device *dev, + int srq_idx, struct srq_entry *entryp) +{ + struct cpl_srq_table_req *req; + struct adapter *adap; + struct sk_buff *skb; + struct srq_data *s; + int rc = -ENODEV; + + adap = netdev2adap(dev); + s = adap->srq; + + if (!(adap->flags & CXGB4_FULL_INIT_DONE) || !s) + goto out; + + skb = alloc_skb(sizeof(*req), GFP_KERNEL); + if (!skb) + return -ENOMEM; + req = (struct cpl_srq_table_req *) + __skb_put_zero(skb, sizeof(*req)); + INIT_TP_WR(req, 0); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SRQ_TABLE_REQ, + TID_TID_V(srq_idx) | + TID_QID_V(adap->sge.fw_evtq.abs_id))); + req->idx = srq_idx; + + mutex_lock(&s->lock); + + s->entryp = entryp; + t4_mgmt_tx(adap, skb); + + rc = wait_for_completion_timeout(&s->comp, SRQ_WAIT_TO); + if (rc) + rc = 0; + else /* !rc means we timed out */ + rc = -ETIMEDOUT; + + WARN_ON_ONCE(entryp->idx != srq_idx); + mutex_unlock(&s->lock); +out: + return rc; +} +EXPORT_SYMBOL(cxgb4_get_srq_entry); + +void do_srq_table_rpl(struct adapter *adap, + const struct cpl_srq_table_rpl *rpl) +{ + unsigned int idx = TID_TID_G(GET_TID(rpl)); + struct srq_data *s = adap->srq; + struct srq_entry *e; + + if (unlikely(rpl->status != CPL_CONTAINS_READ_RPL)) { + dev_err(adap->pdev_dev, + "Unexpected SRQ_TABLE_RPL status %u for entry %u\n", + rpl->status, idx); + goto out; + } + + /* Store the read entry */ + e = s->entryp; + e->valid = 1; + e->idx = idx; + e->pdid = SRQT_PDID_G(be64_to_cpu(rpl->rsvd_pdid)); + e->qlen = SRQT_QLEN_G(be32_to_cpu(rpl->qlen_qbase)); + e->qbase = SRQT_QBASE_G(be32_to_cpu(rpl->qlen_qbase)); + e->cur_msn = be16_to_cpu(rpl->cur_msn); + e->max_msn = be16_to_cpu(rpl->max_msn); +out: + complete(&s->comp); +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/srq.h b/drivers/net/ethernet/chelsio/cxgb4/srq.h new file mode 100644 index 000000000..ec85cf938 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/srq.h @@ -0,0 +1,65 @@ +/* + * This file is part of the Chelsio T6 Ethernet driver for Linux. + * + * Copyright (c) 2017-2018 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __CXGB4_SRQ_H +#define __CXGB4_SRQ_H + +struct adapter; +struct cpl_srq_table_rpl; + +#define SRQ_WAIT_TO (HZ * 5) + +struct srq_entry { + u8 valid; + u8 idx; + u8 qlen; + u16 pdid; + u16 cur_msn; + u16 max_msn; + u32 qbase; +}; + +struct srq_data { + unsigned int srq_size; + struct srq_entry *entryp; + struct completion comp; + struct mutex lock; /* generic mutex for srq data */ +}; + +struct srq_data *t4_init_srq(int srq_size); +int cxgb4_get_srq_entry(struct net_device *dev, + int srq_idx, struct srq_entry *entryp); +void do_srq_table_rpl(struct adapter *adap, + const struct cpl_srq_table_rpl *rpl); +#endif /* __CXGB4_SRQ_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_chip_type.h b/drivers/net/ethernet/chelsio/cxgb4/t4_chip_type.h new file mode 100644 index 000000000..721c77577 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_chip_type.h @@ -0,0 +1,87 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2015 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef __T4_CHIP_TYPE_H__ +#define __T4_CHIP_TYPE_H__ + +#define CHELSIO_PCI_ID_VER(__DeviceID) ((__DeviceID) >> 12) + +#define CHELSIO_T4 0x4 +#define CHELSIO_T5 0x5 +#define CHELSIO_T6 0x6 + +/* We code the Chelsio T4 Family "Chip Code" as a tuple: + * + * (Chip Version, Chip Revision) + * + * where: + * + * Chip Version: is T4, T5, etc. + * Chip Revision: is the FAB "spin" of the Chip Version. + */ +#define CHELSIO_CHIP_CODE(version, revision) (((version) << 4) | (revision)) +#define CHELSIO_CHIP_VERSION(code) (((code) >> 4) & 0xf) +#define CHELSIO_CHIP_RELEASE(code) ((code) & 0xf) + +enum chip_type { + T4_A1 = CHELSIO_CHIP_CODE(CHELSIO_T4, 1), + T4_A2 = CHELSIO_CHIP_CODE(CHELSIO_T4, 2), + T4_FIRST_REV = T4_A1, + T4_LAST_REV = T4_A2, + + T5_A0 = CHELSIO_CHIP_CODE(CHELSIO_T5, 0), + T5_A1 = CHELSIO_CHIP_CODE(CHELSIO_T5, 1), + T5_FIRST_REV = T5_A0, + T5_LAST_REV = T5_A1, + + T6_A0 = CHELSIO_CHIP_CODE(CHELSIO_T6, 0), + T6_FIRST_REV = T6_A0, + T6_LAST_REV = T6_A0, +}; + +static inline int is_t4(enum chip_type chip) +{ + return (CHELSIO_CHIP_VERSION(chip) == CHELSIO_T4); +} + +static inline int is_t5(enum chip_type chip) +{ + return (CHELSIO_CHIP_VERSION(chip) == CHELSIO_T5); +} + +static inline int is_t6(enum chip_type chip) +{ + return (CHELSIO_CHIP_VERSION(chip) == CHELSIO_T6); +} + +#endif /* __T4_CHIP_TYPE_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c new file mode 100644 index 000000000..76de55306 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c @@ -0,0 +1,10773 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/delay.h> +#include "cxgb4.h" +#include "t4_regs.h" +#include "t4_values.h" +#include "t4fw_api.h" +#include "t4fw_version.h" + +/** + * t4_wait_op_done_val - wait until an operation is completed + * @adapter: the adapter performing the operation + * @reg: the register to check for completion + * @mask: a single-bit field within @reg that indicates completion + * @polarity: the value of the field when the operation is completed + * @attempts: number of check iterations + * @delay: delay in usecs between iterations + * @valp: where to store the value of the register at completion time + * + * Wait until an operation is completed by checking a bit in a register + * up to @attempts times. If @valp is not NULL the value of the register + * at the time it indicated completion is stored there. Returns 0 if the + * operation completes and -EAGAIN otherwise. + */ +static int t4_wait_op_done_val(struct adapter *adapter, int reg, u32 mask, + int polarity, int attempts, int delay, u32 *valp) +{ + while (1) { + u32 val = t4_read_reg(adapter, reg); + + if (!!(val & mask) == polarity) { + if (valp) + *valp = val; + return 0; + } + if (--attempts == 0) + return -EAGAIN; + if (delay) + udelay(delay); + } +} + +static inline int t4_wait_op_done(struct adapter *adapter, int reg, u32 mask, + int polarity, int attempts, int delay) +{ + return t4_wait_op_done_val(adapter, reg, mask, polarity, attempts, + delay, NULL); +} + +/** + * t4_set_reg_field - set a register field to a value + * @adapter: the adapter to program + * @addr: the register address + * @mask: specifies the portion of the register to modify + * @val: the new value for the register field + * + * Sets a register field specified by the supplied mask to the + * given value. + */ +void t4_set_reg_field(struct adapter *adapter, unsigned int addr, u32 mask, + u32 val) +{ + u32 v = t4_read_reg(adapter, addr) & ~mask; + + t4_write_reg(adapter, addr, v | val); + (void) t4_read_reg(adapter, addr); /* flush */ +} + +/** + * t4_read_indirect - read indirectly addressed registers + * @adap: the adapter + * @addr_reg: register holding the indirect address + * @data_reg: register holding the value of the indirect register + * @vals: where the read register values are stored + * @nregs: how many indirect registers to read + * @start_idx: index of first indirect register to read + * + * Reads registers that are accessed indirectly through an address/data + * register pair. + */ +void t4_read_indirect(struct adapter *adap, unsigned int addr_reg, + unsigned int data_reg, u32 *vals, + unsigned int nregs, unsigned int start_idx) +{ + while (nregs--) { + t4_write_reg(adap, addr_reg, start_idx); + *vals++ = t4_read_reg(adap, data_reg); + start_idx++; + } +} + +/** + * t4_write_indirect - write indirectly addressed registers + * @adap: the adapter + * @addr_reg: register holding the indirect addresses + * @data_reg: register holding the value for the indirect registers + * @vals: values to write + * @nregs: how many indirect registers to write + * @start_idx: address of first indirect register to write + * + * Writes a sequential block of registers that are accessed indirectly + * through an address/data register pair. + */ +void t4_write_indirect(struct adapter *adap, unsigned int addr_reg, + unsigned int data_reg, const u32 *vals, + unsigned int nregs, unsigned int start_idx) +{ + while (nregs--) { + t4_write_reg(adap, addr_reg, start_idx++); + t4_write_reg(adap, data_reg, *vals++); + } +} + +/* + * Read a 32-bit PCI Configuration Space register via the PCI-E backdoor + * mechanism. This guarantees that we get the real value even if we're + * operating within a Virtual Machine and the Hypervisor is trapping our + * Configuration Space accesses. + */ +void t4_hw_pci_read_cfg4(struct adapter *adap, int reg, u32 *val) +{ + u32 req = FUNCTION_V(adap->pf) | REGISTER_V(reg); + + if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) + req |= ENABLE_F; + else + req |= T6_ENABLE_F; + + if (is_t4(adap->params.chip)) + req |= LOCALCFG_F; + + t4_write_reg(adap, PCIE_CFG_SPACE_REQ_A, req); + *val = t4_read_reg(adap, PCIE_CFG_SPACE_DATA_A); + + /* Reset ENABLE to 0 so reads of PCIE_CFG_SPACE_DATA won't cause a + * Configuration Space read. (None of the other fields matter when + * ENABLE is 0 so a simple register write is easier than a + * read-modify-write via t4_set_reg_field().) + */ + t4_write_reg(adap, PCIE_CFG_SPACE_REQ_A, 0); +} + +/* + * t4_report_fw_error - report firmware error + * @adap: the adapter + * + * The adapter firmware can indicate error conditions to the host. + * If the firmware has indicated an error, print out the reason for + * the firmware error. + */ +static void t4_report_fw_error(struct adapter *adap) +{ + static const char *const reason[] = { + "Crash", /* PCIE_FW_EVAL_CRASH */ + "During Device Preparation", /* PCIE_FW_EVAL_PREP */ + "During Device Configuration", /* PCIE_FW_EVAL_CONF */ + "During Device Initialization", /* PCIE_FW_EVAL_INIT */ + "Unexpected Event", /* PCIE_FW_EVAL_UNEXPECTEDEVENT */ + "Insufficient Airflow", /* PCIE_FW_EVAL_OVERHEAT */ + "Device Shutdown", /* PCIE_FW_EVAL_DEVICESHUTDOWN */ + "Reserved", /* reserved */ + }; + u32 pcie_fw; + + pcie_fw = t4_read_reg(adap, PCIE_FW_A); + if (pcie_fw & PCIE_FW_ERR_F) { + dev_err(adap->pdev_dev, "Firmware reports adapter error: %s\n", + reason[PCIE_FW_EVAL_G(pcie_fw)]); + adap->flags &= ~CXGB4_FW_OK; + } +} + +/* + * Get the reply to a mailbox command and store it in @rpl in big-endian order. + */ +static void get_mbox_rpl(struct adapter *adap, __be64 *rpl, int nflit, + u32 mbox_addr) +{ + for ( ; nflit; nflit--, mbox_addr += 8) + *rpl++ = cpu_to_be64(t4_read_reg64(adap, mbox_addr)); +} + +/* + * Handle a FW assertion reported in a mailbox. + */ +static void fw_asrt(struct adapter *adap, u32 mbox_addr) +{ + struct fw_debug_cmd asrt; + + get_mbox_rpl(adap, (__be64 *)&asrt, sizeof(asrt) / 8, mbox_addr); + dev_alert(adap->pdev_dev, + "FW assertion at %.16s:%u, val0 %#x, val1 %#x\n", + asrt.u.assert.filename_0_7, be32_to_cpu(asrt.u.assert.line), + be32_to_cpu(asrt.u.assert.x), be32_to_cpu(asrt.u.assert.y)); +} + +/** + * t4_record_mbox - record a Firmware Mailbox Command/Reply in the log + * @adapter: the adapter + * @cmd: the Firmware Mailbox Command or Reply + * @size: command length in bytes + * @access: the time (ms) needed to access the Firmware Mailbox + * @execute: the time (ms) the command spent being executed + */ +static void t4_record_mbox(struct adapter *adapter, + const __be64 *cmd, unsigned int size, + int access, int execute) +{ + struct mbox_cmd_log *log = adapter->mbox_log; + struct mbox_cmd *entry; + int i; + + entry = mbox_cmd_log_entry(log, log->cursor++); + if (log->cursor == log->size) + log->cursor = 0; + + for (i = 0; i < size / 8; i++) + entry->cmd[i] = be64_to_cpu(cmd[i]); + while (i < MBOX_LEN / 8) + entry->cmd[i++] = 0; + entry->timestamp = jiffies; + entry->seqno = log->seqno++; + entry->access = access; + entry->execute = execute; +} + +/** + * t4_wr_mbox_meat_timeout - send a command to FW through the given mailbox + * @adap: the adapter + * @mbox: index of the mailbox to use + * @cmd: the command to write + * @size: command length in bytes + * @rpl: where to optionally store the reply + * @sleep_ok: if true we may sleep while awaiting command completion + * @timeout: time to wait for command to finish before timing out + * + * Sends the given command to FW through the selected mailbox and waits + * for the FW to execute the command. If @rpl is not %NULL it is used to + * store the FW's reply to the command. The command and its optional + * reply are of the same length. FW can take up to %FW_CMD_MAX_TIMEOUT ms + * to respond. @sleep_ok determines whether we may sleep while awaiting + * the response. If sleeping is allowed we use progressive backoff + * otherwise we spin. + * + * The return value is 0 on success or a negative errno on failure. A + * failure can happen either because we are not able to execute the + * command or FW executes it but signals an error. In the latter case + * the return value is the error code indicated by FW (negated). + */ +int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd, + int size, void *rpl, bool sleep_ok, int timeout) +{ + static const int delay[] = { + 1, 1, 3, 5, 10, 10, 20, 50, 100, 200 + }; + + struct mbox_list entry; + u16 access = 0; + u16 execute = 0; + u32 v; + u64 res; + int i, ms, delay_idx, ret; + const __be64 *p = cmd; + u32 data_reg = PF_REG(mbox, CIM_PF_MAILBOX_DATA_A); + u32 ctl_reg = PF_REG(mbox, CIM_PF_MAILBOX_CTRL_A); + __be64 cmd_rpl[MBOX_LEN / 8]; + u32 pcie_fw; + + if ((size & 15) || size > MBOX_LEN) + return -EINVAL; + + /* + * If the device is off-line, as in EEH, commands will time out. + * Fail them early so we don't waste time waiting. + */ + if (adap->pdev->error_state != pci_channel_io_normal) + return -EIO; + + /* If we have a negative timeout, that implies that we can't sleep. */ + if (timeout < 0) { + sleep_ok = false; + timeout = -timeout; + } + + /* Queue ourselves onto the mailbox access list. When our entry is at + * the front of the list, we have rights to access the mailbox. So we + * wait [for a while] till we're at the front [or bail out with an + * EBUSY] ... + */ + spin_lock_bh(&adap->mbox_lock); + list_add_tail(&entry.list, &adap->mlist.list); + spin_unlock_bh(&adap->mbox_lock); + + delay_idx = 0; + ms = delay[0]; + + for (i = 0; ; i += ms) { + /* If we've waited too long, return a busy indication. This + * really ought to be based on our initial position in the + * mailbox access list but this is a start. We very rarely + * contend on access to the mailbox ... + */ + pcie_fw = t4_read_reg(adap, PCIE_FW_A); + if (i > FW_CMD_MAX_TIMEOUT || (pcie_fw & PCIE_FW_ERR_F)) { + spin_lock_bh(&adap->mbox_lock); + list_del(&entry.list); + spin_unlock_bh(&adap->mbox_lock); + ret = (pcie_fw & PCIE_FW_ERR_F) ? -ENXIO : -EBUSY; + t4_record_mbox(adap, cmd, size, access, ret); + return ret; + } + + /* If we're at the head, break out and start the mailbox + * protocol. + */ + if (list_first_entry(&adap->mlist.list, struct mbox_list, + list) == &entry) + break; + + /* Delay for a bit before checking again ... */ + if (sleep_ok) { + ms = delay[delay_idx]; /* last element may repeat */ + if (delay_idx < ARRAY_SIZE(delay) - 1) + delay_idx++; + msleep(ms); + } else { + mdelay(ms); + } + } + + /* Loop trying to get ownership of the mailbox. Return an error + * if we can't gain ownership. + */ + v = MBOWNER_G(t4_read_reg(adap, ctl_reg)); + for (i = 0; v == MBOX_OWNER_NONE && i < 3; i++) + v = MBOWNER_G(t4_read_reg(adap, ctl_reg)); + if (v != MBOX_OWNER_DRV) { + spin_lock_bh(&adap->mbox_lock); + list_del(&entry.list); + spin_unlock_bh(&adap->mbox_lock); + ret = (v == MBOX_OWNER_FW) ? -EBUSY : -ETIMEDOUT; + t4_record_mbox(adap, cmd, size, access, ret); + return ret; + } + + /* Copy in the new mailbox command and send it on its way ... */ + t4_record_mbox(adap, cmd, size, access, 0); + for (i = 0; i < size; i += 8) + t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p++)); + + t4_write_reg(adap, ctl_reg, MBMSGVALID_F | MBOWNER_V(MBOX_OWNER_FW)); + t4_read_reg(adap, ctl_reg); /* flush write */ + + delay_idx = 0; + ms = delay[0]; + + for (i = 0; + !((pcie_fw = t4_read_reg(adap, PCIE_FW_A)) & PCIE_FW_ERR_F) && + i < timeout; + i += ms) { + if (sleep_ok) { + ms = delay[delay_idx]; /* last element may repeat */ + if (delay_idx < ARRAY_SIZE(delay) - 1) + delay_idx++; + msleep(ms); + } else + mdelay(ms); + + v = t4_read_reg(adap, ctl_reg); + if (MBOWNER_G(v) == MBOX_OWNER_DRV) { + if (!(v & MBMSGVALID_F)) { + t4_write_reg(adap, ctl_reg, 0); + continue; + } + + get_mbox_rpl(adap, cmd_rpl, MBOX_LEN / 8, data_reg); + res = be64_to_cpu(cmd_rpl[0]); + + if (FW_CMD_OP_G(res >> 32) == FW_DEBUG_CMD) { + fw_asrt(adap, data_reg); + res = FW_CMD_RETVAL_V(EIO); + } else if (rpl) { + memcpy(rpl, cmd_rpl, size); + } + + t4_write_reg(adap, ctl_reg, 0); + + execute = i + ms; + t4_record_mbox(adap, cmd_rpl, + MBOX_LEN, access, execute); + spin_lock_bh(&adap->mbox_lock); + list_del(&entry.list); + spin_unlock_bh(&adap->mbox_lock); + return -FW_CMD_RETVAL_G((int)res); + } + } + + ret = (pcie_fw & PCIE_FW_ERR_F) ? -ENXIO : -ETIMEDOUT; + t4_record_mbox(adap, cmd, size, access, ret); + dev_err(adap->pdev_dev, "command %#x in mailbox %d timed out\n", + *(const u8 *)cmd, mbox); + t4_report_fw_error(adap); + spin_lock_bh(&adap->mbox_lock); + list_del(&entry.list); + spin_unlock_bh(&adap->mbox_lock); + t4_fatal_err(adap); + return ret; +} + +int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size, + void *rpl, bool sleep_ok) +{ + return t4_wr_mbox_meat_timeout(adap, mbox, cmd, size, rpl, sleep_ok, + FW_CMD_MAX_TIMEOUT); +} + +static int t4_edc_err_read(struct adapter *adap, int idx) +{ + u32 edc_ecc_err_addr_reg; + u32 rdata_reg; + + if (is_t4(adap->params.chip)) { + CH_WARN(adap, "%s: T4 NOT supported.\n", __func__); + return 0; + } + if (idx != 0 && idx != 1) { + CH_WARN(adap, "%s: idx %d NOT supported.\n", __func__, idx); + return 0; + } + + edc_ecc_err_addr_reg = EDC_T5_REG(EDC_H_ECC_ERR_ADDR_A, idx); + rdata_reg = EDC_T5_REG(EDC_H_BIST_STATUS_RDATA_A, idx); + + CH_WARN(adap, + "edc%d err addr 0x%x: 0x%x.\n", + idx, edc_ecc_err_addr_reg, + t4_read_reg(adap, edc_ecc_err_addr_reg)); + CH_WARN(adap, + "bist: 0x%x, status %llx %llx %llx %llx %llx %llx %llx %llx %llx.\n", + rdata_reg, + (unsigned long long)t4_read_reg64(adap, rdata_reg), + (unsigned long long)t4_read_reg64(adap, rdata_reg + 8), + (unsigned long long)t4_read_reg64(adap, rdata_reg + 16), + (unsigned long long)t4_read_reg64(adap, rdata_reg + 24), + (unsigned long long)t4_read_reg64(adap, rdata_reg + 32), + (unsigned long long)t4_read_reg64(adap, rdata_reg + 40), + (unsigned long long)t4_read_reg64(adap, rdata_reg + 48), + (unsigned long long)t4_read_reg64(adap, rdata_reg + 56), + (unsigned long long)t4_read_reg64(adap, rdata_reg + 64)); + + return 0; +} + +/** + * t4_memory_rw_init - Get memory window relative offset, base, and size. + * @adap: the adapter + * @win: PCI-E Memory Window to use + * @mtype: memory type: MEM_EDC0, MEM_EDC1, MEM_HMA or MEM_MC + * @mem_off: memory relative offset with respect to @mtype. + * @mem_base: configured memory base address. + * @mem_aperture: configured memory window aperture. + * + * Get the configured memory window's relative offset, base, and size. + */ +int t4_memory_rw_init(struct adapter *adap, int win, int mtype, u32 *mem_off, + u32 *mem_base, u32 *mem_aperture) +{ + u32 edc_size, mc_size, mem_reg; + + /* Offset into the region of memory which is being accessed + * MEM_EDC0 = 0 + * MEM_EDC1 = 1 + * MEM_MC = 2 -- MEM_MC for chips with only 1 memory controller + * MEM_MC1 = 3 -- for chips with 2 memory controllers (e.g. T5) + * MEM_HMA = 4 + */ + edc_size = EDRAM0_SIZE_G(t4_read_reg(adap, MA_EDRAM0_BAR_A)); + if (mtype == MEM_HMA) { + *mem_off = 2 * (edc_size * 1024 * 1024); + } else if (mtype != MEM_MC1) { + *mem_off = (mtype * (edc_size * 1024 * 1024)); + } else { + mc_size = EXT_MEM0_SIZE_G(t4_read_reg(adap, + MA_EXT_MEMORY0_BAR_A)); + *mem_off = (MEM_MC0 * edc_size + mc_size) * 1024 * 1024; + } + + /* Each PCI-E Memory Window is programmed with a window size -- or + * "aperture" -- which controls the granularity of its mapping onto + * adapter memory. We need to grab that aperture in order to know + * how to use the specified window. The window is also programmed + * with the base address of the Memory Window in BAR0's address + * space. For T4 this is an absolute PCI-E Bus Address. For T5 + * the address is relative to BAR0. + */ + mem_reg = t4_read_reg(adap, + PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, + win)); + /* a dead adapter will return 0xffffffff for PIO reads */ + if (mem_reg == 0xffffffff) + return -ENXIO; + + *mem_aperture = 1 << (WINDOW_G(mem_reg) + WINDOW_SHIFT_X); + *mem_base = PCIEOFST_G(mem_reg) << PCIEOFST_SHIFT_X; + if (is_t4(adap->params.chip)) + *mem_base -= adap->t4_bar0; + + return 0; +} + +/** + * t4_memory_update_win - Move memory window to specified address. + * @adap: the adapter + * @win: PCI-E Memory Window to use + * @addr: location to move. + * + * Move memory window to specified address. + */ +void t4_memory_update_win(struct adapter *adap, int win, u32 addr) +{ + t4_write_reg(adap, + PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, win), + addr); + /* Read it back to ensure that changes propagate before we + * attempt to use the new value. + */ + t4_read_reg(adap, + PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, win)); +} + +/** + * t4_memory_rw_residual - Read/Write residual data. + * @adap: the adapter + * @off: relative offset within residual to start read/write. + * @addr: address within indicated memory type. + * @buf: host memory buffer + * @dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0) + * + * Read/Write residual data less than 32-bits. + */ +void t4_memory_rw_residual(struct adapter *adap, u32 off, u32 addr, u8 *buf, + int dir) +{ + union { + u32 word; + char byte[4]; + } last; + unsigned char *bp; + int i; + + if (dir == T4_MEMORY_READ) { + last.word = le32_to_cpu((__force __le32) + t4_read_reg(adap, addr)); + for (bp = (unsigned char *)buf, i = off; i < 4; i++) + bp[i] = last.byte[i]; + } else { + last.word = *buf; + for (i = off; i < 4; i++) + last.byte[i] = 0; + t4_write_reg(adap, addr, + (__force u32)cpu_to_le32(last.word)); + } +} + +/** + * t4_memory_rw - read/write EDC 0, EDC 1 or MC via PCIE memory window + * @adap: the adapter + * @win: PCI-E Memory Window to use + * @mtype: memory type: MEM_EDC0, MEM_EDC1 or MEM_MC + * @addr: address within indicated memory type + * @len: amount of memory to transfer + * @hbuf: host memory buffer + * @dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0) + * + * Reads/writes an [almost] arbitrary memory region in the firmware: the + * firmware memory address and host buffer must be aligned on 32-bit + * boundaries; the length may be arbitrary. The memory is transferred as + * a raw byte sequence from/to the firmware's memory. If this memory + * contains data structures which contain multi-byte integers, it's the + * caller's responsibility to perform appropriate byte order conversions. + */ +int t4_memory_rw(struct adapter *adap, int win, int mtype, u32 addr, + u32 len, void *hbuf, int dir) +{ + u32 pos, offset, resid, memoffset; + u32 win_pf, mem_aperture, mem_base; + u32 *buf; + int ret; + + /* Argument sanity checks ... + */ + if (addr & 0x3 || (uintptr_t)hbuf & 0x3) + return -EINVAL; + buf = (u32 *)hbuf; + + /* It's convenient to be able to handle lengths which aren't a + * multiple of 32-bits because we often end up transferring files to + * the firmware. So we'll handle that by normalizing the length here + * and then handling any residual transfer at the end. + */ + resid = len & 0x3; + len -= resid; + + ret = t4_memory_rw_init(adap, win, mtype, &memoffset, &mem_base, + &mem_aperture); + if (ret) + return ret; + + /* Determine the PCIE_MEM_ACCESS_OFFSET */ + addr = addr + memoffset; + + win_pf = is_t4(adap->params.chip) ? 0 : PFNUM_V(adap->pf); + + /* Calculate our initial PCI-E Memory Window Position and Offset into + * that Window. + */ + pos = addr & ~(mem_aperture - 1); + offset = addr - pos; + + /* Set up initial PCI-E Memory Window to cover the start of our + * transfer. + */ + t4_memory_update_win(adap, win, pos | win_pf); + + /* Transfer data to/from the adapter as long as there's an integral + * number of 32-bit transfers to complete. + * + * A note on Endianness issues: + * + * The "register" reads and writes below from/to the PCI-E Memory + * Window invoke the standard adapter Big-Endian to PCI-E Link + * Little-Endian "swizzel." As a result, if we have the following + * data in adapter memory: + * + * Memory: ... | b0 | b1 | b2 | b3 | ... + * Address: i+0 i+1 i+2 i+3 + * + * Then a read of the adapter memory via the PCI-E Memory Window + * will yield: + * + * x = readl(i) + * 31 0 + * [ b3 | b2 | b1 | b0 ] + * + * If this value is stored into local memory on a Little-Endian system + * it will show up correctly in local memory as: + * + * ( ..., b0, b1, b2, b3, ... ) + * + * But on a Big-Endian system, the store will show up in memory + * incorrectly swizzled as: + * + * ( ..., b3, b2, b1, b0, ... ) + * + * So we need to account for this in the reads and writes to the + * PCI-E Memory Window below by undoing the register read/write + * swizzels. + */ + while (len > 0) { + if (dir == T4_MEMORY_READ) + *buf++ = le32_to_cpu((__force __le32)t4_read_reg(adap, + mem_base + offset)); + else + t4_write_reg(adap, mem_base + offset, + (__force u32)cpu_to_le32(*buf++)); + offset += sizeof(__be32); + len -= sizeof(__be32); + + /* If we've reached the end of our current window aperture, + * move the PCI-E Memory Window on to the next. Note that + * doing this here after "len" may be 0 allows us to set up + * the PCI-E Memory Window for a possible final residual + * transfer below ... + */ + if (offset == mem_aperture) { + pos += mem_aperture; + offset = 0; + t4_memory_update_win(adap, win, pos | win_pf); + } + } + + /* If the original transfer had a length which wasn't a multiple of + * 32-bits, now's where we need to finish off the transfer of the + * residual amount. The PCI-E Memory Window has already been moved + * above (if necessary) to cover this final transfer. + */ + if (resid) + t4_memory_rw_residual(adap, resid, mem_base + offset, + (u8 *)buf, dir); + + return 0; +} + +/* Return the specified PCI-E Configuration Space register from our Physical + * Function. We try first via a Firmware LDST Command since we prefer to let + * the firmware own all of these registers, but if that fails we go for it + * directly ourselves. + */ +u32 t4_read_pcie_cfg4(struct adapter *adap, int reg) +{ + u32 val, ldst_addrspace; + + /* If fw_attach != 0, construct and send the Firmware LDST Command to + * retrieve the specified PCI-E Configuration Space register. + */ + struct fw_ldst_cmd ldst_cmd; + int ret; + + memset(&ldst_cmd, 0, sizeof(ldst_cmd)); + ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE); + ldst_cmd.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F | + ldst_addrspace); + ldst_cmd.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst_cmd)); + ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1); + ldst_cmd.u.pcie.ctrl_to_fn = + (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->pf)); + ldst_cmd.u.pcie.r = reg; + + /* If the LDST Command succeeds, return the result, otherwise + * fall through to reading it directly ourselves ... + */ + ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd), + &ldst_cmd); + if (ret == 0) + val = be32_to_cpu(ldst_cmd.u.pcie.data[0]); + else + /* Read the desired Configuration Space register via the PCI-E + * Backdoor mechanism. + */ + t4_hw_pci_read_cfg4(adap, reg, &val); + return val; +} + +/* Get the window based on base passed to it. + * Window aperture is currently unhandled, but there is no use case for it + * right now + */ +static u32 t4_get_window(struct adapter *adap, u32 pci_base, u64 pci_mask, + u32 memwin_base) +{ + u32 ret; + + if (is_t4(adap->params.chip)) { + u32 bar0; + + /* Truncation intentional: we only read the bottom 32-bits of + * the 64-bit BAR0/BAR1 ... We use the hardware backdoor + * mechanism to read BAR0 instead of using + * pci_resource_start() because we could be operating from + * within a Virtual Machine which is trapping our accesses to + * our Configuration Space and we need to set up the PCI-E + * Memory Window decoders with the actual addresses which will + * be coming across the PCI-E link. + */ + bar0 = t4_read_pcie_cfg4(adap, pci_base); + bar0 &= pci_mask; + adap->t4_bar0 = bar0; + + ret = bar0 + memwin_base; + } else { + /* For T5, only relative offset inside the PCIe BAR is passed */ + ret = memwin_base; + } + return ret; +} + +/* Get the default utility window (win0) used by everyone */ +u32 t4_get_util_window(struct adapter *adap) +{ + return t4_get_window(adap, PCI_BASE_ADDRESS_0, + PCI_BASE_ADDRESS_MEM_MASK, MEMWIN0_BASE); +} + +/* Set up memory window for accessing adapter memory ranges. (Read + * back MA register to ensure that changes propagate before we attempt + * to use the new values.) + */ +void t4_setup_memwin(struct adapter *adap, u32 memwin_base, u32 window) +{ + t4_write_reg(adap, + PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, window), + memwin_base | BIR_V(0) | + WINDOW_V(ilog2(MEMWIN0_APERTURE) - WINDOW_SHIFT_X)); + t4_read_reg(adap, + PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, window)); +} + +/** + * t4_get_regs_len - return the size of the chips register set + * @adapter: the adapter + * + * Returns the size of the chip's BAR0 register space. + */ +unsigned int t4_get_regs_len(struct adapter *adapter) +{ + unsigned int chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip); + + switch (chip_version) { + case CHELSIO_T4: + return T4_REGMAP_SIZE; + + case CHELSIO_T5: + case CHELSIO_T6: + return T5_REGMAP_SIZE; + } + + dev_err(adapter->pdev_dev, + "Unsupported chip version %d\n", chip_version); + return 0; +} + +/** + * t4_get_regs - read chip registers into provided buffer + * @adap: the adapter + * @buf: register buffer + * @buf_size: size (in bytes) of register buffer + * + * If the provided register buffer isn't large enough for the chip's + * full register range, the register dump will be truncated to the + * register buffer's size. + */ +void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size) +{ + static const unsigned int t4_reg_ranges[] = { + 0x1008, 0x1108, + 0x1180, 0x1184, + 0x1190, 0x1194, + 0x11a0, 0x11a4, + 0x11b0, 0x11b4, + 0x11fc, 0x123c, + 0x1300, 0x173c, + 0x1800, 0x18fc, + 0x3000, 0x30d8, + 0x30e0, 0x30e4, + 0x30ec, 0x5910, + 0x5920, 0x5924, + 0x5960, 0x5960, + 0x5968, 0x5968, + 0x5970, 0x5970, + 0x5978, 0x5978, + 0x5980, 0x5980, + 0x5988, 0x5988, + 0x5990, 0x5990, + 0x5998, 0x5998, + 0x59a0, 0x59d4, + 0x5a00, 0x5ae0, + 0x5ae8, 0x5ae8, + 0x5af0, 0x5af0, + 0x5af8, 0x5af8, + 0x6000, 0x6098, + 0x6100, 0x6150, + 0x6200, 0x6208, + 0x6240, 0x6248, + 0x6280, 0x62b0, + 0x62c0, 0x6338, + 0x6370, 0x638c, + 0x6400, 0x643c, + 0x6500, 0x6524, + 0x6a00, 0x6a04, + 0x6a14, 0x6a38, + 0x6a60, 0x6a70, + 0x6a78, 0x6a78, + 0x6b00, 0x6b0c, + 0x6b1c, 0x6b84, + 0x6bf0, 0x6bf8, + 0x6c00, 0x6c0c, + 0x6c1c, 0x6c84, + 0x6cf0, 0x6cf8, + 0x6d00, 0x6d0c, + 0x6d1c, 0x6d84, + 0x6df0, 0x6df8, + 0x6e00, 0x6e0c, + 0x6e1c, 0x6e84, + 0x6ef0, 0x6ef8, + 0x6f00, 0x6f0c, + 0x6f1c, 0x6f84, + 0x6ff0, 0x6ff8, + 0x7000, 0x700c, + 0x701c, 0x7084, + 0x70f0, 0x70f8, + 0x7100, 0x710c, + 0x711c, 0x7184, + 0x71f0, 0x71f8, + 0x7200, 0x720c, + 0x721c, 0x7284, + 0x72f0, 0x72f8, + 0x7300, 0x730c, + 0x731c, 0x7384, + 0x73f0, 0x73f8, + 0x7400, 0x7450, + 0x7500, 0x7530, + 0x7600, 0x760c, + 0x7614, 0x761c, + 0x7680, 0x76cc, + 0x7700, 0x7798, + 0x77c0, 0x77fc, + 0x7900, 0x79fc, + 0x7b00, 0x7b58, + 0x7b60, 0x7b84, + 0x7b8c, 0x7c38, + 0x7d00, 0x7d38, + 0x7d40, 0x7d80, + 0x7d8c, 0x7ddc, + 0x7de4, 0x7e04, + 0x7e10, 0x7e1c, + 0x7e24, 0x7e38, + 0x7e40, 0x7e44, + 0x7e4c, 0x7e78, + 0x7e80, 0x7ea4, + 0x7eac, 0x7edc, + 0x7ee8, 0x7efc, + 0x8dc0, 0x8e04, + 0x8e10, 0x8e1c, + 0x8e30, 0x8e78, + 0x8ea0, 0x8eb8, + 0x8ec0, 0x8f6c, + 0x8fc0, 0x9008, + 0x9010, 0x9058, + 0x9060, 0x9060, + 0x9068, 0x9074, + 0x90fc, 0x90fc, + 0x9400, 0x9408, + 0x9410, 0x9458, + 0x9600, 0x9600, + 0x9608, 0x9638, + 0x9640, 0x96bc, + 0x9800, 0x9808, + 0x9820, 0x983c, + 0x9850, 0x9864, + 0x9c00, 0x9c6c, + 0x9c80, 0x9cec, + 0x9d00, 0x9d6c, + 0x9d80, 0x9dec, + 0x9e00, 0x9e6c, + 0x9e80, 0x9eec, + 0x9f00, 0x9f6c, + 0x9f80, 0x9fec, + 0xd004, 0xd004, + 0xd010, 0xd03c, + 0xdfc0, 0xdfe0, + 0xe000, 0xea7c, + 0xf000, 0x11110, + 0x11118, 0x11190, + 0x19040, 0x1906c, + 0x19078, 0x19080, + 0x1908c, 0x190e4, + 0x190f0, 0x190f8, + 0x19100, 0x19110, + 0x19120, 0x19124, + 0x19150, 0x19194, + 0x1919c, 0x191b0, + 0x191d0, 0x191e8, + 0x19238, 0x1924c, + 0x193f8, 0x1943c, + 0x1944c, 0x19474, + 0x19490, 0x194e0, + 0x194f0, 0x194f8, + 0x19800, 0x19c08, + 0x19c10, 0x19c90, + 0x19ca0, 0x19ce4, + 0x19cf0, 0x19d40, + 0x19d50, 0x19d94, + 0x19da0, 0x19de8, + 0x19df0, 0x19e40, + 0x19e50, 0x19e90, + 0x19ea0, 0x19f4c, + 0x1a000, 0x1a004, + 0x1a010, 0x1a06c, + 0x1a0b0, 0x1a0e4, + 0x1a0ec, 0x1a0f4, + 0x1a100, 0x1a108, + 0x1a114, 0x1a120, + 0x1a128, 0x1a130, + 0x1a138, 0x1a138, + 0x1a190, 0x1a1c4, + 0x1a1fc, 0x1a1fc, + 0x1e040, 0x1e04c, + 0x1e284, 0x1e28c, + 0x1e2c0, 0x1e2c0, + 0x1e2e0, 0x1e2e0, + 0x1e300, 0x1e384, + 0x1e3c0, 0x1e3c8, + 0x1e440, 0x1e44c, + 0x1e684, 0x1e68c, + 0x1e6c0, 0x1e6c0, + 0x1e6e0, 0x1e6e0, + 0x1e700, 0x1e784, + 0x1e7c0, 0x1e7c8, + 0x1e840, 0x1e84c, + 0x1ea84, 0x1ea8c, + 0x1eac0, 0x1eac0, + 0x1eae0, 0x1eae0, + 0x1eb00, 0x1eb84, + 0x1ebc0, 0x1ebc8, + 0x1ec40, 0x1ec4c, + 0x1ee84, 0x1ee8c, + 0x1eec0, 0x1eec0, + 0x1eee0, 0x1eee0, + 0x1ef00, 0x1ef84, + 0x1efc0, 0x1efc8, + 0x1f040, 0x1f04c, + 0x1f284, 0x1f28c, + 0x1f2c0, 0x1f2c0, + 0x1f2e0, 0x1f2e0, + 0x1f300, 0x1f384, + 0x1f3c0, 0x1f3c8, + 0x1f440, 0x1f44c, + 0x1f684, 0x1f68c, + 0x1f6c0, 0x1f6c0, + 0x1f6e0, 0x1f6e0, + 0x1f700, 0x1f784, + 0x1f7c0, 0x1f7c8, + 0x1f840, 0x1f84c, + 0x1fa84, 0x1fa8c, + 0x1fac0, 0x1fac0, + 0x1fae0, 0x1fae0, + 0x1fb00, 0x1fb84, + 0x1fbc0, 0x1fbc8, + 0x1fc40, 0x1fc4c, + 0x1fe84, 0x1fe8c, + 0x1fec0, 0x1fec0, + 0x1fee0, 0x1fee0, + 0x1ff00, 0x1ff84, + 0x1ffc0, 0x1ffc8, + 0x20000, 0x2002c, + 0x20100, 0x2013c, + 0x20190, 0x201a0, + 0x201a8, 0x201b8, + 0x201c4, 0x201c8, + 0x20200, 0x20318, + 0x20400, 0x204b4, + 0x204c0, 0x20528, + 0x20540, 0x20614, + 0x21000, 0x21040, + 0x2104c, 0x21060, + 0x210c0, 0x210ec, + 0x21200, 0x21268, + 0x21270, 0x21284, + 0x212fc, 0x21388, + 0x21400, 0x21404, + 0x21500, 0x21500, + 0x21510, 0x21518, + 0x2152c, 0x21530, + 0x2153c, 0x2153c, + 0x21550, 0x21554, + 0x21600, 0x21600, + 0x21608, 0x2161c, + 0x21624, 0x21628, + 0x21630, 0x21634, + 0x2163c, 0x2163c, + 0x21700, 0x2171c, + 0x21780, 0x2178c, + 0x21800, 0x21818, + 0x21820, 0x21828, + 0x21830, 0x21848, + 0x21850, 0x21854, + 0x21860, 0x21868, + 0x21870, 0x21870, + 0x21878, 0x21898, + 0x218a0, 0x218a8, + 0x218b0, 0x218c8, + 0x218d0, 0x218d4, + 0x218e0, 0x218e8, + 0x218f0, 0x218f0, + 0x218f8, 0x21a18, + 0x21a20, 0x21a28, + 0x21a30, 0x21a48, + 0x21a50, 0x21a54, + 0x21a60, 0x21a68, + 0x21a70, 0x21a70, + 0x21a78, 0x21a98, + 0x21aa0, 0x21aa8, + 0x21ab0, 0x21ac8, + 0x21ad0, 0x21ad4, + 0x21ae0, 0x21ae8, + 0x21af0, 0x21af0, + 0x21af8, 0x21c18, + 0x21c20, 0x21c20, + 0x21c28, 0x21c30, + 0x21c38, 0x21c38, + 0x21c80, 0x21c98, + 0x21ca0, 0x21ca8, + 0x21cb0, 0x21cc8, + 0x21cd0, 0x21cd4, + 0x21ce0, 0x21ce8, + 0x21cf0, 0x21cf0, + 0x21cf8, 0x21d7c, + 0x21e00, 0x21e04, + 0x22000, 0x2202c, + 0x22100, 0x2213c, + 0x22190, 0x221a0, + 0x221a8, 0x221b8, + 0x221c4, 0x221c8, + 0x22200, 0x22318, + 0x22400, 0x224b4, + 0x224c0, 0x22528, + 0x22540, 0x22614, + 0x23000, 0x23040, + 0x2304c, 0x23060, + 0x230c0, 0x230ec, + 0x23200, 0x23268, + 0x23270, 0x23284, + 0x232fc, 0x23388, + 0x23400, 0x23404, + 0x23500, 0x23500, + 0x23510, 0x23518, + 0x2352c, 0x23530, + 0x2353c, 0x2353c, + 0x23550, 0x23554, + 0x23600, 0x23600, + 0x23608, 0x2361c, + 0x23624, 0x23628, + 0x23630, 0x23634, + 0x2363c, 0x2363c, + 0x23700, 0x2371c, + 0x23780, 0x2378c, + 0x23800, 0x23818, + 0x23820, 0x23828, + 0x23830, 0x23848, + 0x23850, 0x23854, + 0x23860, 0x23868, + 0x23870, 0x23870, + 0x23878, 0x23898, + 0x238a0, 0x238a8, + 0x238b0, 0x238c8, + 0x238d0, 0x238d4, + 0x238e0, 0x238e8, + 0x238f0, 0x238f0, + 0x238f8, 0x23a18, + 0x23a20, 0x23a28, + 0x23a30, 0x23a48, + 0x23a50, 0x23a54, + 0x23a60, 0x23a68, + 0x23a70, 0x23a70, + 0x23a78, 0x23a98, + 0x23aa0, 0x23aa8, + 0x23ab0, 0x23ac8, + 0x23ad0, 0x23ad4, + 0x23ae0, 0x23ae8, + 0x23af0, 0x23af0, + 0x23af8, 0x23c18, + 0x23c20, 0x23c20, + 0x23c28, 0x23c30, + 0x23c38, 0x23c38, + 0x23c80, 0x23c98, + 0x23ca0, 0x23ca8, + 0x23cb0, 0x23cc8, + 0x23cd0, 0x23cd4, + 0x23ce0, 0x23ce8, + 0x23cf0, 0x23cf0, + 0x23cf8, 0x23d7c, + 0x23e00, 0x23e04, + 0x24000, 0x2402c, + 0x24100, 0x2413c, + 0x24190, 0x241a0, + 0x241a8, 0x241b8, + 0x241c4, 0x241c8, + 0x24200, 0x24318, + 0x24400, 0x244b4, + 0x244c0, 0x24528, + 0x24540, 0x24614, + 0x25000, 0x25040, + 0x2504c, 0x25060, + 0x250c0, 0x250ec, + 0x25200, 0x25268, + 0x25270, 0x25284, + 0x252fc, 0x25388, + 0x25400, 0x25404, + 0x25500, 0x25500, + 0x25510, 0x25518, + 0x2552c, 0x25530, + 0x2553c, 0x2553c, + 0x25550, 0x25554, + 0x25600, 0x25600, + 0x25608, 0x2561c, + 0x25624, 0x25628, + 0x25630, 0x25634, + 0x2563c, 0x2563c, + 0x25700, 0x2571c, + 0x25780, 0x2578c, + 0x25800, 0x25818, + 0x25820, 0x25828, + 0x25830, 0x25848, + 0x25850, 0x25854, + 0x25860, 0x25868, + 0x25870, 0x25870, + 0x25878, 0x25898, + 0x258a0, 0x258a8, + 0x258b0, 0x258c8, + 0x258d0, 0x258d4, + 0x258e0, 0x258e8, + 0x258f0, 0x258f0, + 0x258f8, 0x25a18, + 0x25a20, 0x25a28, + 0x25a30, 0x25a48, + 0x25a50, 0x25a54, + 0x25a60, 0x25a68, + 0x25a70, 0x25a70, + 0x25a78, 0x25a98, + 0x25aa0, 0x25aa8, + 0x25ab0, 0x25ac8, + 0x25ad0, 0x25ad4, + 0x25ae0, 0x25ae8, + 0x25af0, 0x25af0, + 0x25af8, 0x25c18, + 0x25c20, 0x25c20, + 0x25c28, 0x25c30, + 0x25c38, 0x25c38, + 0x25c80, 0x25c98, + 0x25ca0, 0x25ca8, + 0x25cb0, 0x25cc8, + 0x25cd0, 0x25cd4, + 0x25ce0, 0x25ce8, + 0x25cf0, 0x25cf0, + 0x25cf8, 0x25d7c, + 0x25e00, 0x25e04, + 0x26000, 0x2602c, + 0x26100, 0x2613c, + 0x26190, 0x261a0, + 0x261a8, 0x261b8, + 0x261c4, 0x261c8, + 0x26200, 0x26318, + 0x26400, 0x264b4, + 0x264c0, 0x26528, + 0x26540, 0x26614, + 0x27000, 0x27040, + 0x2704c, 0x27060, + 0x270c0, 0x270ec, + 0x27200, 0x27268, + 0x27270, 0x27284, + 0x272fc, 0x27388, + 0x27400, 0x27404, + 0x27500, 0x27500, + 0x27510, 0x27518, + 0x2752c, 0x27530, + 0x2753c, 0x2753c, + 0x27550, 0x27554, + 0x27600, 0x27600, + 0x27608, 0x2761c, + 0x27624, 0x27628, + 0x27630, 0x27634, + 0x2763c, 0x2763c, + 0x27700, 0x2771c, + 0x27780, 0x2778c, + 0x27800, 0x27818, + 0x27820, 0x27828, + 0x27830, 0x27848, + 0x27850, 0x27854, + 0x27860, 0x27868, + 0x27870, 0x27870, + 0x27878, 0x27898, + 0x278a0, 0x278a8, + 0x278b0, 0x278c8, + 0x278d0, 0x278d4, + 0x278e0, 0x278e8, + 0x278f0, 0x278f0, + 0x278f8, 0x27a18, + 0x27a20, 0x27a28, + 0x27a30, 0x27a48, + 0x27a50, 0x27a54, + 0x27a60, 0x27a68, + 0x27a70, 0x27a70, + 0x27a78, 0x27a98, + 0x27aa0, 0x27aa8, + 0x27ab0, 0x27ac8, + 0x27ad0, 0x27ad4, + 0x27ae0, 0x27ae8, + 0x27af0, 0x27af0, + 0x27af8, 0x27c18, + 0x27c20, 0x27c20, + 0x27c28, 0x27c30, + 0x27c38, 0x27c38, + 0x27c80, 0x27c98, + 0x27ca0, 0x27ca8, + 0x27cb0, 0x27cc8, + 0x27cd0, 0x27cd4, + 0x27ce0, 0x27ce8, + 0x27cf0, 0x27cf0, + 0x27cf8, 0x27d7c, + 0x27e00, 0x27e04, + }; + + static const unsigned int t5_reg_ranges[] = { + 0x1008, 0x10c0, + 0x10cc, 0x10f8, + 0x1100, 0x1100, + 0x110c, 0x1148, + 0x1180, 0x1184, + 0x1190, 0x1194, + 0x11a0, 0x11a4, + 0x11b0, 0x11b4, + 0x11fc, 0x123c, + 0x1280, 0x173c, + 0x1800, 0x18fc, + 0x3000, 0x3028, + 0x3060, 0x30b0, + 0x30b8, 0x30d8, + 0x30e0, 0x30fc, + 0x3140, 0x357c, + 0x35a8, 0x35cc, + 0x35ec, 0x35ec, + 0x3600, 0x5624, + 0x56cc, 0x56ec, + 0x56f4, 0x5720, + 0x5728, 0x575c, + 0x580c, 0x5814, + 0x5890, 0x589c, + 0x58a4, 0x58ac, + 0x58b8, 0x58bc, + 0x5940, 0x59c8, + 0x59d0, 0x59dc, + 0x59fc, 0x5a18, + 0x5a60, 0x5a70, + 0x5a80, 0x5a9c, + 0x5b94, 0x5bfc, + 0x6000, 0x6020, + 0x6028, 0x6040, + 0x6058, 0x609c, + 0x60a8, 0x614c, + 0x7700, 0x7798, + 0x77c0, 0x78fc, + 0x7b00, 0x7b58, + 0x7b60, 0x7b84, + 0x7b8c, 0x7c54, + 0x7d00, 0x7d38, + 0x7d40, 0x7d80, + 0x7d8c, 0x7ddc, + 0x7de4, 0x7e04, + 0x7e10, 0x7e1c, + 0x7e24, 0x7e38, + 0x7e40, 0x7e44, + 0x7e4c, 0x7e78, + 0x7e80, 0x7edc, + 0x7ee8, 0x7efc, + 0x8dc0, 0x8de0, + 0x8df8, 0x8e04, + 0x8e10, 0x8e84, + 0x8ea0, 0x8f84, + 0x8fc0, 0x9058, + 0x9060, 0x9060, + 0x9068, 0x90f8, + 0x9400, 0x9408, + 0x9410, 0x9470, + 0x9600, 0x9600, + 0x9608, 0x9638, + 0x9640, 0x96f4, + 0x9800, 0x9808, + 0x9810, 0x9864, + 0x9c00, 0x9c6c, + 0x9c80, 0x9cec, + 0x9d00, 0x9d6c, + 0x9d80, 0x9dec, + 0x9e00, 0x9e6c, + 0x9e80, 0x9eec, + 0x9f00, 0x9f6c, + 0x9f80, 0xa020, + 0xd000, 0xd004, + 0xd010, 0xd03c, + 0xdfc0, 0xdfe0, + 0xe000, 0x1106c, + 0x11074, 0x11088, + 0x1109c, 0x1117c, + 0x11190, 0x11204, + 0x19040, 0x1906c, + 0x19078, 0x19080, + 0x1908c, 0x190e8, + 0x190f0, 0x190f8, + 0x19100, 0x19110, + 0x19120, 0x19124, + 0x19150, 0x19194, + 0x1919c, 0x191b0, + 0x191d0, 0x191e8, + 0x19238, 0x19290, + 0x193f8, 0x19428, + 0x19430, 0x19444, + 0x1944c, 0x1946c, + 0x19474, 0x19474, + 0x19490, 0x194cc, + 0x194f0, 0x194f8, + 0x19c00, 0x19c08, + 0x19c10, 0x19c60, + 0x19c94, 0x19ce4, + 0x19cf0, 0x19d40, + 0x19d50, 0x19d94, + 0x19da0, 0x19de8, + 0x19df0, 0x19e10, + 0x19e50, 0x19e90, + 0x19ea0, 0x19f24, + 0x19f34, 0x19f34, + 0x19f40, 0x19f50, + 0x19f90, 0x19fb4, + 0x19fc4, 0x19fe4, + 0x1a000, 0x1a004, + 0x1a010, 0x1a06c, + 0x1a0b0, 0x1a0e4, + 0x1a0ec, 0x1a0f8, + 0x1a100, 0x1a108, + 0x1a114, 0x1a130, + 0x1a138, 0x1a1c4, + 0x1a1fc, 0x1a1fc, + 0x1e008, 0x1e00c, + 0x1e040, 0x1e044, + 0x1e04c, 0x1e04c, + 0x1e284, 0x1e290, + 0x1e2c0, 0x1e2c0, + 0x1e2e0, 0x1e2e0, + 0x1e300, 0x1e384, + 0x1e3c0, 0x1e3c8, + 0x1e408, 0x1e40c, + 0x1e440, 0x1e444, + 0x1e44c, 0x1e44c, + 0x1e684, 0x1e690, + 0x1e6c0, 0x1e6c0, + 0x1e6e0, 0x1e6e0, + 0x1e700, 0x1e784, + 0x1e7c0, 0x1e7c8, + 0x1e808, 0x1e80c, + 0x1e840, 0x1e844, + 0x1e84c, 0x1e84c, + 0x1ea84, 0x1ea90, + 0x1eac0, 0x1eac0, + 0x1eae0, 0x1eae0, + 0x1eb00, 0x1eb84, + 0x1ebc0, 0x1ebc8, + 0x1ec08, 0x1ec0c, + 0x1ec40, 0x1ec44, + 0x1ec4c, 0x1ec4c, + 0x1ee84, 0x1ee90, + 0x1eec0, 0x1eec0, + 0x1eee0, 0x1eee0, + 0x1ef00, 0x1ef84, + 0x1efc0, 0x1efc8, + 0x1f008, 0x1f00c, + 0x1f040, 0x1f044, + 0x1f04c, 0x1f04c, + 0x1f284, 0x1f290, + 0x1f2c0, 0x1f2c0, + 0x1f2e0, 0x1f2e0, + 0x1f300, 0x1f384, + 0x1f3c0, 0x1f3c8, + 0x1f408, 0x1f40c, + 0x1f440, 0x1f444, + 0x1f44c, 0x1f44c, + 0x1f684, 0x1f690, + 0x1f6c0, 0x1f6c0, + 0x1f6e0, 0x1f6e0, + 0x1f700, 0x1f784, + 0x1f7c0, 0x1f7c8, + 0x1f808, 0x1f80c, + 0x1f840, 0x1f844, + 0x1f84c, 0x1f84c, + 0x1fa84, 0x1fa90, + 0x1fac0, 0x1fac0, + 0x1fae0, 0x1fae0, + 0x1fb00, 0x1fb84, + 0x1fbc0, 0x1fbc8, + 0x1fc08, 0x1fc0c, + 0x1fc40, 0x1fc44, + 0x1fc4c, 0x1fc4c, + 0x1fe84, 0x1fe90, + 0x1fec0, 0x1fec0, + 0x1fee0, 0x1fee0, + 0x1ff00, 0x1ff84, + 0x1ffc0, 0x1ffc8, + 0x30000, 0x30030, + 0x30100, 0x30144, + 0x30190, 0x301a0, + 0x301a8, 0x301b8, + 0x301c4, 0x301c8, + 0x301d0, 0x301d0, + 0x30200, 0x30318, + 0x30400, 0x304b4, + 0x304c0, 0x3052c, + 0x30540, 0x3061c, + 0x30800, 0x30828, + 0x30834, 0x30834, + 0x308c0, 0x30908, + 0x30910, 0x309ac, + 0x30a00, 0x30a14, + 0x30a1c, 0x30a2c, + 0x30a44, 0x30a50, + 0x30a74, 0x30a74, + 0x30a7c, 0x30afc, + 0x30b08, 0x30c24, + 0x30d00, 0x30d00, + 0x30d08, 0x30d14, + 0x30d1c, 0x30d20, + 0x30d3c, 0x30d3c, + 0x30d48, 0x30d50, + 0x31200, 0x3120c, + 0x31220, 0x31220, + 0x31240, 0x31240, + 0x31600, 0x3160c, + 0x31a00, 0x31a1c, + 0x31e00, 0x31e20, + 0x31e38, 0x31e3c, + 0x31e80, 0x31e80, + 0x31e88, 0x31ea8, + 0x31eb0, 0x31eb4, + 0x31ec8, 0x31ed4, + 0x31fb8, 0x32004, + 0x32200, 0x32200, + 0x32208, 0x32240, + 0x32248, 0x32280, + 0x32288, 0x322c0, + 0x322c8, 0x322fc, + 0x32600, 0x32630, + 0x32a00, 0x32abc, + 0x32b00, 0x32b10, + 0x32b20, 0x32b30, + 0x32b40, 0x32b50, + 0x32b60, 0x32b70, + 0x33000, 0x33028, + 0x33030, 0x33048, + 0x33060, 0x33068, + 0x33070, 0x3309c, + 0x330f0, 0x33128, + 0x33130, 0x33148, + 0x33160, 0x33168, + 0x33170, 0x3319c, + 0x331f0, 0x33238, + 0x33240, 0x33240, + 0x33248, 0x33250, + 0x3325c, 0x33264, + 0x33270, 0x332b8, + 0x332c0, 0x332e4, + 0x332f8, 0x33338, + 0x33340, 0x33340, + 0x33348, 0x33350, + 0x3335c, 0x33364, + 0x33370, 0x333b8, + 0x333c0, 0x333e4, + 0x333f8, 0x33428, + 0x33430, 0x33448, + 0x33460, 0x33468, + 0x33470, 0x3349c, + 0x334f0, 0x33528, + 0x33530, 0x33548, + 0x33560, 0x33568, + 0x33570, 0x3359c, + 0x335f0, 0x33638, + 0x33640, 0x33640, + 0x33648, 0x33650, + 0x3365c, 0x33664, + 0x33670, 0x336b8, + 0x336c0, 0x336e4, + 0x336f8, 0x33738, + 0x33740, 0x33740, + 0x33748, 0x33750, + 0x3375c, 0x33764, + 0x33770, 0x337b8, + 0x337c0, 0x337e4, + 0x337f8, 0x337fc, + 0x33814, 0x33814, + 0x3382c, 0x3382c, + 0x33880, 0x3388c, + 0x338e8, 0x338ec, + 0x33900, 0x33928, + 0x33930, 0x33948, + 0x33960, 0x33968, + 0x33970, 0x3399c, + 0x339f0, 0x33a38, + 0x33a40, 0x33a40, + 0x33a48, 0x33a50, + 0x33a5c, 0x33a64, + 0x33a70, 0x33ab8, + 0x33ac0, 0x33ae4, + 0x33af8, 0x33b10, + 0x33b28, 0x33b28, + 0x33b3c, 0x33b50, + 0x33bf0, 0x33c10, + 0x33c28, 0x33c28, + 0x33c3c, 0x33c50, + 0x33cf0, 0x33cfc, + 0x34000, 0x34030, + 0x34100, 0x34144, + 0x34190, 0x341a0, + 0x341a8, 0x341b8, + 0x341c4, 0x341c8, + 0x341d0, 0x341d0, + 0x34200, 0x34318, + 0x34400, 0x344b4, + 0x344c0, 0x3452c, + 0x34540, 0x3461c, + 0x34800, 0x34828, + 0x34834, 0x34834, + 0x348c0, 0x34908, + 0x34910, 0x349ac, + 0x34a00, 0x34a14, + 0x34a1c, 0x34a2c, + 0x34a44, 0x34a50, + 0x34a74, 0x34a74, + 0x34a7c, 0x34afc, + 0x34b08, 0x34c24, + 0x34d00, 0x34d00, + 0x34d08, 0x34d14, + 0x34d1c, 0x34d20, + 0x34d3c, 0x34d3c, + 0x34d48, 0x34d50, + 0x35200, 0x3520c, + 0x35220, 0x35220, + 0x35240, 0x35240, + 0x35600, 0x3560c, + 0x35a00, 0x35a1c, + 0x35e00, 0x35e20, + 0x35e38, 0x35e3c, + 0x35e80, 0x35e80, + 0x35e88, 0x35ea8, + 0x35eb0, 0x35eb4, + 0x35ec8, 0x35ed4, + 0x35fb8, 0x36004, + 0x36200, 0x36200, + 0x36208, 0x36240, + 0x36248, 0x36280, + 0x36288, 0x362c0, + 0x362c8, 0x362fc, + 0x36600, 0x36630, + 0x36a00, 0x36abc, + 0x36b00, 0x36b10, + 0x36b20, 0x36b30, + 0x36b40, 0x36b50, + 0x36b60, 0x36b70, + 0x37000, 0x37028, + 0x37030, 0x37048, + 0x37060, 0x37068, + 0x37070, 0x3709c, + 0x370f0, 0x37128, + 0x37130, 0x37148, + 0x37160, 0x37168, + 0x37170, 0x3719c, + 0x371f0, 0x37238, + 0x37240, 0x37240, + 0x37248, 0x37250, + 0x3725c, 0x37264, + 0x37270, 0x372b8, + 0x372c0, 0x372e4, + 0x372f8, 0x37338, + 0x37340, 0x37340, + 0x37348, 0x37350, + 0x3735c, 0x37364, + 0x37370, 0x373b8, + 0x373c0, 0x373e4, + 0x373f8, 0x37428, + 0x37430, 0x37448, + 0x37460, 0x37468, + 0x37470, 0x3749c, + 0x374f0, 0x37528, + 0x37530, 0x37548, + 0x37560, 0x37568, + 0x37570, 0x3759c, + 0x375f0, 0x37638, + 0x37640, 0x37640, + 0x37648, 0x37650, + 0x3765c, 0x37664, + 0x37670, 0x376b8, + 0x376c0, 0x376e4, + 0x376f8, 0x37738, + 0x37740, 0x37740, + 0x37748, 0x37750, + 0x3775c, 0x37764, + 0x37770, 0x377b8, + 0x377c0, 0x377e4, + 0x377f8, 0x377fc, + 0x37814, 0x37814, + 0x3782c, 0x3782c, + 0x37880, 0x3788c, + 0x378e8, 0x378ec, + 0x37900, 0x37928, + 0x37930, 0x37948, + 0x37960, 0x37968, + 0x37970, 0x3799c, + 0x379f0, 0x37a38, + 0x37a40, 0x37a40, + 0x37a48, 0x37a50, + 0x37a5c, 0x37a64, + 0x37a70, 0x37ab8, + 0x37ac0, 0x37ae4, + 0x37af8, 0x37b10, + 0x37b28, 0x37b28, + 0x37b3c, 0x37b50, + 0x37bf0, 0x37c10, + 0x37c28, 0x37c28, + 0x37c3c, 0x37c50, + 0x37cf0, 0x37cfc, + 0x38000, 0x38030, + 0x38100, 0x38144, + 0x38190, 0x381a0, + 0x381a8, 0x381b8, + 0x381c4, 0x381c8, + 0x381d0, 0x381d0, + 0x38200, 0x38318, + 0x38400, 0x384b4, + 0x384c0, 0x3852c, + 0x38540, 0x3861c, + 0x38800, 0x38828, + 0x38834, 0x38834, + 0x388c0, 0x38908, + 0x38910, 0x389ac, + 0x38a00, 0x38a14, + 0x38a1c, 0x38a2c, + 0x38a44, 0x38a50, + 0x38a74, 0x38a74, + 0x38a7c, 0x38afc, + 0x38b08, 0x38c24, + 0x38d00, 0x38d00, + 0x38d08, 0x38d14, + 0x38d1c, 0x38d20, + 0x38d3c, 0x38d3c, + 0x38d48, 0x38d50, + 0x39200, 0x3920c, + 0x39220, 0x39220, + 0x39240, 0x39240, + 0x39600, 0x3960c, + 0x39a00, 0x39a1c, + 0x39e00, 0x39e20, + 0x39e38, 0x39e3c, + 0x39e80, 0x39e80, + 0x39e88, 0x39ea8, + 0x39eb0, 0x39eb4, + 0x39ec8, 0x39ed4, + 0x39fb8, 0x3a004, + 0x3a200, 0x3a200, + 0x3a208, 0x3a240, + 0x3a248, 0x3a280, + 0x3a288, 0x3a2c0, + 0x3a2c8, 0x3a2fc, + 0x3a600, 0x3a630, + 0x3aa00, 0x3aabc, + 0x3ab00, 0x3ab10, + 0x3ab20, 0x3ab30, + 0x3ab40, 0x3ab50, + 0x3ab60, 0x3ab70, + 0x3b000, 0x3b028, + 0x3b030, 0x3b048, + 0x3b060, 0x3b068, + 0x3b070, 0x3b09c, + 0x3b0f0, 0x3b128, + 0x3b130, 0x3b148, + 0x3b160, 0x3b168, + 0x3b170, 0x3b19c, + 0x3b1f0, 0x3b238, + 0x3b240, 0x3b240, + 0x3b248, 0x3b250, + 0x3b25c, 0x3b264, + 0x3b270, 0x3b2b8, + 0x3b2c0, 0x3b2e4, + 0x3b2f8, 0x3b338, + 0x3b340, 0x3b340, + 0x3b348, 0x3b350, + 0x3b35c, 0x3b364, + 0x3b370, 0x3b3b8, + 0x3b3c0, 0x3b3e4, + 0x3b3f8, 0x3b428, + 0x3b430, 0x3b448, + 0x3b460, 0x3b468, + 0x3b470, 0x3b49c, + 0x3b4f0, 0x3b528, + 0x3b530, 0x3b548, + 0x3b560, 0x3b568, + 0x3b570, 0x3b59c, + 0x3b5f0, 0x3b638, + 0x3b640, 0x3b640, + 0x3b648, 0x3b650, + 0x3b65c, 0x3b664, + 0x3b670, 0x3b6b8, + 0x3b6c0, 0x3b6e4, + 0x3b6f8, 0x3b738, + 0x3b740, 0x3b740, + 0x3b748, 0x3b750, + 0x3b75c, 0x3b764, + 0x3b770, 0x3b7b8, + 0x3b7c0, 0x3b7e4, + 0x3b7f8, 0x3b7fc, + 0x3b814, 0x3b814, + 0x3b82c, 0x3b82c, + 0x3b880, 0x3b88c, + 0x3b8e8, 0x3b8ec, + 0x3b900, 0x3b928, + 0x3b930, 0x3b948, + 0x3b960, 0x3b968, + 0x3b970, 0x3b99c, + 0x3b9f0, 0x3ba38, + 0x3ba40, 0x3ba40, + 0x3ba48, 0x3ba50, + 0x3ba5c, 0x3ba64, + 0x3ba70, 0x3bab8, + 0x3bac0, 0x3bae4, + 0x3baf8, 0x3bb10, + 0x3bb28, 0x3bb28, + 0x3bb3c, 0x3bb50, + 0x3bbf0, 0x3bc10, + 0x3bc28, 0x3bc28, + 0x3bc3c, 0x3bc50, + 0x3bcf0, 0x3bcfc, + 0x3c000, 0x3c030, + 0x3c100, 0x3c144, + 0x3c190, 0x3c1a0, + 0x3c1a8, 0x3c1b8, + 0x3c1c4, 0x3c1c8, + 0x3c1d0, 0x3c1d0, + 0x3c200, 0x3c318, + 0x3c400, 0x3c4b4, + 0x3c4c0, 0x3c52c, + 0x3c540, 0x3c61c, + 0x3c800, 0x3c828, + 0x3c834, 0x3c834, + 0x3c8c0, 0x3c908, + 0x3c910, 0x3c9ac, + 0x3ca00, 0x3ca14, + 0x3ca1c, 0x3ca2c, + 0x3ca44, 0x3ca50, + 0x3ca74, 0x3ca74, + 0x3ca7c, 0x3cafc, + 0x3cb08, 0x3cc24, + 0x3cd00, 0x3cd00, + 0x3cd08, 0x3cd14, + 0x3cd1c, 0x3cd20, + 0x3cd3c, 0x3cd3c, + 0x3cd48, 0x3cd50, + 0x3d200, 0x3d20c, + 0x3d220, 0x3d220, + 0x3d240, 0x3d240, + 0x3d600, 0x3d60c, + 0x3da00, 0x3da1c, + 0x3de00, 0x3de20, + 0x3de38, 0x3de3c, + 0x3de80, 0x3de80, + 0x3de88, 0x3dea8, + 0x3deb0, 0x3deb4, + 0x3dec8, 0x3ded4, + 0x3dfb8, 0x3e004, + 0x3e200, 0x3e200, + 0x3e208, 0x3e240, + 0x3e248, 0x3e280, + 0x3e288, 0x3e2c0, + 0x3e2c8, 0x3e2fc, + 0x3e600, 0x3e630, + 0x3ea00, 0x3eabc, + 0x3eb00, 0x3eb10, + 0x3eb20, 0x3eb30, + 0x3eb40, 0x3eb50, + 0x3eb60, 0x3eb70, + 0x3f000, 0x3f028, + 0x3f030, 0x3f048, + 0x3f060, 0x3f068, + 0x3f070, 0x3f09c, + 0x3f0f0, 0x3f128, + 0x3f130, 0x3f148, + 0x3f160, 0x3f168, + 0x3f170, 0x3f19c, + 0x3f1f0, 0x3f238, + 0x3f240, 0x3f240, + 0x3f248, 0x3f250, + 0x3f25c, 0x3f264, + 0x3f270, 0x3f2b8, + 0x3f2c0, 0x3f2e4, + 0x3f2f8, 0x3f338, + 0x3f340, 0x3f340, + 0x3f348, 0x3f350, + 0x3f35c, 0x3f364, + 0x3f370, 0x3f3b8, + 0x3f3c0, 0x3f3e4, + 0x3f3f8, 0x3f428, + 0x3f430, 0x3f448, + 0x3f460, 0x3f468, + 0x3f470, 0x3f49c, + 0x3f4f0, 0x3f528, + 0x3f530, 0x3f548, + 0x3f560, 0x3f568, + 0x3f570, 0x3f59c, + 0x3f5f0, 0x3f638, + 0x3f640, 0x3f640, + 0x3f648, 0x3f650, + 0x3f65c, 0x3f664, + 0x3f670, 0x3f6b8, + 0x3f6c0, 0x3f6e4, + 0x3f6f8, 0x3f738, + 0x3f740, 0x3f740, + 0x3f748, 0x3f750, + 0x3f75c, 0x3f764, + 0x3f770, 0x3f7b8, + 0x3f7c0, 0x3f7e4, + 0x3f7f8, 0x3f7fc, + 0x3f814, 0x3f814, + 0x3f82c, 0x3f82c, + 0x3f880, 0x3f88c, + 0x3f8e8, 0x3f8ec, + 0x3f900, 0x3f928, + 0x3f930, 0x3f948, + 0x3f960, 0x3f968, + 0x3f970, 0x3f99c, + 0x3f9f0, 0x3fa38, + 0x3fa40, 0x3fa40, + 0x3fa48, 0x3fa50, + 0x3fa5c, 0x3fa64, + 0x3fa70, 0x3fab8, + 0x3fac0, 0x3fae4, + 0x3faf8, 0x3fb10, + 0x3fb28, 0x3fb28, + 0x3fb3c, 0x3fb50, + 0x3fbf0, 0x3fc10, + 0x3fc28, 0x3fc28, + 0x3fc3c, 0x3fc50, + 0x3fcf0, 0x3fcfc, + 0x40000, 0x4000c, + 0x40040, 0x40050, + 0x40060, 0x40068, + 0x4007c, 0x4008c, + 0x40094, 0x400b0, + 0x400c0, 0x40144, + 0x40180, 0x4018c, + 0x40200, 0x40254, + 0x40260, 0x40264, + 0x40270, 0x40288, + 0x40290, 0x40298, + 0x402ac, 0x402c8, + 0x402d0, 0x402e0, + 0x402f0, 0x402f0, + 0x40300, 0x4033c, + 0x403f8, 0x403fc, + 0x41304, 0x413c4, + 0x41400, 0x4140c, + 0x41414, 0x4141c, + 0x41480, 0x414d0, + 0x44000, 0x44054, + 0x4405c, 0x44078, + 0x440c0, 0x44174, + 0x44180, 0x441ac, + 0x441b4, 0x441b8, + 0x441c0, 0x44254, + 0x4425c, 0x44278, + 0x442c0, 0x44374, + 0x44380, 0x443ac, + 0x443b4, 0x443b8, + 0x443c0, 0x44454, + 0x4445c, 0x44478, + 0x444c0, 0x44574, + 0x44580, 0x445ac, + 0x445b4, 0x445b8, + 0x445c0, 0x44654, + 0x4465c, 0x44678, + 0x446c0, 0x44774, + 0x44780, 0x447ac, + 0x447b4, 0x447b8, + 0x447c0, 0x44854, + 0x4485c, 0x44878, + 0x448c0, 0x44974, + 0x44980, 0x449ac, + 0x449b4, 0x449b8, + 0x449c0, 0x449fc, + 0x45000, 0x45004, + 0x45010, 0x45030, + 0x45040, 0x45060, + 0x45068, 0x45068, + 0x45080, 0x45084, + 0x450a0, 0x450b0, + 0x45200, 0x45204, + 0x45210, 0x45230, + 0x45240, 0x45260, + 0x45268, 0x45268, + 0x45280, 0x45284, + 0x452a0, 0x452b0, + 0x460c0, 0x460e4, + 0x47000, 0x4703c, + 0x47044, 0x4708c, + 0x47200, 0x47250, + 0x47400, 0x47408, + 0x47414, 0x47420, + 0x47600, 0x47618, + 0x47800, 0x47814, + 0x48000, 0x4800c, + 0x48040, 0x48050, + 0x48060, 0x48068, + 0x4807c, 0x4808c, + 0x48094, 0x480b0, + 0x480c0, 0x48144, + 0x48180, 0x4818c, + 0x48200, 0x48254, + 0x48260, 0x48264, + 0x48270, 0x48288, + 0x48290, 0x48298, + 0x482ac, 0x482c8, + 0x482d0, 0x482e0, + 0x482f0, 0x482f0, + 0x48300, 0x4833c, + 0x483f8, 0x483fc, + 0x49304, 0x493c4, + 0x49400, 0x4940c, + 0x49414, 0x4941c, + 0x49480, 0x494d0, + 0x4c000, 0x4c054, + 0x4c05c, 0x4c078, + 0x4c0c0, 0x4c174, + 0x4c180, 0x4c1ac, + 0x4c1b4, 0x4c1b8, + 0x4c1c0, 0x4c254, + 0x4c25c, 0x4c278, + 0x4c2c0, 0x4c374, + 0x4c380, 0x4c3ac, + 0x4c3b4, 0x4c3b8, + 0x4c3c0, 0x4c454, + 0x4c45c, 0x4c478, + 0x4c4c0, 0x4c574, + 0x4c580, 0x4c5ac, + 0x4c5b4, 0x4c5b8, + 0x4c5c0, 0x4c654, + 0x4c65c, 0x4c678, + 0x4c6c0, 0x4c774, + 0x4c780, 0x4c7ac, + 0x4c7b4, 0x4c7b8, + 0x4c7c0, 0x4c854, + 0x4c85c, 0x4c878, + 0x4c8c0, 0x4c974, + 0x4c980, 0x4c9ac, + 0x4c9b4, 0x4c9b8, + 0x4c9c0, 0x4c9fc, + 0x4d000, 0x4d004, + 0x4d010, 0x4d030, + 0x4d040, 0x4d060, + 0x4d068, 0x4d068, + 0x4d080, 0x4d084, + 0x4d0a0, 0x4d0b0, + 0x4d200, 0x4d204, + 0x4d210, 0x4d230, + 0x4d240, 0x4d260, + 0x4d268, 0x4d268, + 0x4d280, 0x4d284, + 0x4d2a0, 0x4d2b0, + 0x4e0c0, 0x4e0e4, + 0x4f000, 0x4f03c, + 0x4f044, 0x4f08c, + 0x4f200, 0x4f250, + 0x4f400, 0x4f408, + 0x4f414, 0x4f420, + 0x4f600, 0x4f618, + 0x4f800, 0x4f814, + 0x50000, 0x50084, + 0x50090, 0x500cc, + 0x50400, 0x50400, + 0x50800, 0x50884, + 0x50890, 0x508cc, + 0x50c00, 0x50c00, + 0x51000, 0x5101c, + 0x51300, 0x51308, + }; + + static const unsigned int t6_reg_ranges[] = { + 0x1008, 0x101c, + 0x1024, 0x10a8, + 0x10b4, 0x10f8, + 0x1100, 0x1114, + 0x111c, 0x112c, + 0x1138, 0x113c, + 0x1144, 0x114c, + 0x1180, 0x1184, + 0x1190, 0x1194, + 0x11a0, 0x11a4, + 0x11b0, 0x11b4, + 0x11fc, 0x123c, + 0x1254, 0x1274, + 0x1280, 0x133c, + 0x1800, 0x18fc, + 0x3000, 0x302c, + 0x3060, 0x30b0, + 0x30b8, 0x30d8, + 0x30e0, 0x30fc, + 0x3140, 0x357c, + 0x35a8, 0x35cc, + 0x35ec, 0x35ec, + 0x3600, 0x5624, + 0x56cc, 0x56ec, + 0x56f4, 0x5720, + 0x5728, 0x575c, + 0x580c, 0x5814, + 0x5890, 0x589c, + 0x58a4, 0x58ac, + 0x58b8, 0x58bc, + 0x5940, 0x595c, + 0x5980, 0x598c, + 0x59b0, 0x59c8, + 0x59d0, 0x59dc, + 0x59fc, 0x5a18, + 0x5a60, 0x5a6c, + 0x5a80, 0x5a8c, + 0x5a94, 0x5a9c, + 0x5b94, 0x5bfc, + 0x5c10, 0x5e48, + 0x5e50, 0x5e94, + 0x5ea0, 0x5eb0, + 0x5ec0, 0x5ec0, + 0x5ec8, 0x5ed0, + 0x5ee0, 0x5ee0, + 0x5ef0, 0x5ef0, + 0x5f00, 0x5f00, + 0x6000, 0x6020, + 0x6028, 0x6040, + 0x6058, 0x609c, + 0x60a8, 0x619c, + 0x7700, 0x7798, + 0x77c0, 0x7880, + 0x78cc, 0x78fc, + 0x7b00, 0x7b58, + 0x7b60, 0x7b84, + 0x7b8c, 0x7c54, + 0x7d00, 0x7d38, + 0x7d40, 0x7d84, + 0x7d8c, 0x7ddc, + 0x7de4, 0x7e04, + 0x7e10, 0x7e1c, + 0x7e24, 0x7e38, + 0x7e40, 0x7e44, + 0x7e4c, 0x7e78, + 0x7e80, 0x7edc, + 0x7ee8, 0x7efc, + 0x8dc0, 0x8de4, + 0x8df8, 0x8e04, + 0x8e10, 0x8e84, + 0x8ea0, 0x8f88, + 0x8fb8, 0x9058, + 0x9060, 0x9060, + 0x9068, 0x90f8, + 0x9100, 0x9124, + 0x9400, 0x9470, + 0x9600, 0x9600, + 0x9608, 0x9638, + 0x9640, 0x9704, + 0x9710, 0x971c, + 0x9800, 0x9808, + 0x9810, 0x9864, + 0x9c00, 0x9c6c, + 0x9c80, 0x9cec, + 0x9d00, 0x9d6c, + 0x9d80, 0x9dec, + 0x9e00, 0x9e6c, + 0x9e80, 0x9eec, + 0x9f00, 0x9f6c, + 0x9f80, 0xa020, + 0xd000, 0xd03c, + 0xd100, 0xd118, + 0xd200, 0xd214, + 0xd220, 0xd234, + 0xd240, 0xd254, + 0xd260, 0xd274, + 0xd280, 0xd294, + 0xd2a0, 0xd2b4, + 0xd2c0, 0xd2d4, + 0xd2e0, 0xd2f4, + 0xd300, 0xd31c, + 0xdfc0, 0xdfe0, + 0xe000, 0xf008, + 0xf010, 0xf018, + 0xf020, 0xf028, + 0x11000, 0x11014, + 0x11048, 0x1106c, + 0x11074, 0x11088, + 0x11098, 0x11120, + 0x1112c, 0x1117c, + 0x11190, 0x112e0, + 0x11300, 0x1130c, + 0x12000, 0x1206c, + 0x19040, 0x1906c, + 0x19078, 0x19080, + 0x1908c, 0x190e8, + 0x190f0, 0x190f8, + 0x19100, 0x19110, + 0x19120, 0x19124, + 0x19150, 0x19194, + 0x1919c, 0x191b0, + 0x191d0, 0x191e8, + 0x19238, 0x19290, + 0x192a4, 0x192b0, + 0x192bc, 0x192bc, + 0x19348, 0x1934c, + 0x193f8, 0x19418, + 0x19420, 0x19428, + 0x19430, 0x19444, + 0x1944c, 0x1946c, + 0x19474, 0x19474, + 0x19490, 0x194cc, + 0x194f0, 0x194f8, + 0x19c00, 0x19c48, + 0x19c50, 0x19c80, + 0x19c94, 0x19c98, + 0x19ca0, 0x19cbc, + 0x19ce4, 0x19ce4, + 0x19cf0, 0x19cf8, + 0x19d00, 0x19d28, + 0x19d50, 0x19d78, + 0x19d94, 0x19d98, + 0x19da0, 0x19dc8, + 0x19df0, 0x19e10, + 0x19e50, 0x19e6c, + 0x19ea0, 0x19ebc, + 0x19ec4, 0x19ef4, + 0x19f04, 0x19f2c, + 0x19f34, 0x19f34, + 0x19f40, 0x19f50, + 0x19f90, 0x19fac, + 0x19fc4, 0x19fc8, + 0x19fd0, 0x19fe4, + 0x1a000, 0x1a004, + 0x1a010, 0x1a06c, + 0x1a0b0, 0x1a0e4, + 0x1a0ec, 0x1a0f8, + 0x1a100, 0x1a108, + 0x1a114, 0x1a130, + 0x1a138, 0x1a1c4, + 0x1a1fc, 0x1a1fc, + 0x1e008, 0x1e00c, + 0x1e040, 0x1e044, + 0x1e04c, 0x1e04c, + 0x1e284, 0x1e290, + 0x1e2c0, 0x1e2c0, + 0x1e2e0, 0x1e2e0, + 0x1e300, 0x1e384, + 0x1e3c0, 0x1e3c8, + 0x1e408, 0x1e40c, + 0x1e440, 0x1e444, + 0x1e44c, 0x1e44c, + 0x1e684, 0x1e690, + 0x1e6c0, 0x1e6c0, + 0x1e6e0, 0x1e6e0, + 0x1e700, 0x1e784, + 0x1e7c0, 0x1e7c8, + 0x1e808, 0x1e80c, + 0x1e840, 0x1e844, + 0x1e84c, 0x1e84c, + 0x1ea84, 0x1ea90, + 0x1eac0, 0x1eac0, + 0x1eae0, 0x1eae0, + 0x1eb00, 0x1eb84, + 0x1ebc0, 0x1ebc8, + 0x1ec08, 0x1ec0c, + 0x1ec40, 0x1ec44, + 0x1ec4c, 0x1ec4c, + 0x1ee84, 0x1ee90, + 0x1eec0, 0x1eec0, + 0x1eee0, 0x1eee0, + 0x1ef00, 0x1ef84, + 0x1efc0, 0x1efc8, + 0x1f008, 0x1f00c, + 0x1f040, 0x1f044, + 0x1f04c, 0x1f04c, + 0x1f284, 0x1f290, + 0x1f2c0, 0x1f2c0, + 0x1f2e0, 0x1f2e0, + 0x1f300, 0x1f384, + 0x1f3c0, 0x1f3c8, + 0x1f408, 0x1f40c, + 0x1f440, 0x1f444, + 0x1f44c, 0x1f44c, + 0x1f684, 0x1f690, + 0x1f6c0, 0x1f6c0, + 0x1f6e0, 0x1f6e0, + 0x1f700, 0x1f784, + 0x1f7c0, 0x1f7c8, + 0x1f808, 0x1f80c, + 0x1f840, 0x1f844, + 0x1f84c, 0x1f84c, + 0x1fa84, 0x1fa90, + 0x1fac0, 0x1fac0, + 0x1fae0, 0x1fae0, + 0x1fb00, 0x1fb84, + 0x1fbc0, 0x1fbc8, + 0x1fc08, 0x1fc0c, + 0x1fc40, 0x1fc44, + 0x1fc4c, 0x1fc4c, + 0x1fe84, 0x1fe90, + 0x1fec0, 0x1fec0, + 0x1fee0, 0x1fee0, + 0x1ff00, 0x1ff84, + 0x1ffc0, 0x1ffc8, + 0x30000, 0x30030, + 0x30100, 0x30168, + 0x30190, 0x301a0, + 0x301a8, 0x301b8, + 0x301c4, 0x301c8, + 0x301d0, 0x301d0, + 0x30200, 0x30320, + 0x30400, 0x304b4, + 0x304c0, 0x3052c, + 0x30540, 0x3061c, + 0x30800, 0x308a0, + 0x308c0, 0x30908, + 0x30910, 0x309b8, + 0x30a00, 0x30a04, + 0x30a0c, 0x30a14, + 0x30a1c, 0x30a2c, + 0x30a44, 0x30a50, + 0x30a74, 0x30a74, + 0x30a7c, 0x30afc, + 0x30b08, 0x30c24, + 0x30d00, 0x30d14, + 0x30d1c, 0x30d3c, + 0x30d44, 0x30d4c, + 0x30d54, 0x30d74, + 0x30d7c, 0x30d7c, + 0x30de0, 0x30de0, + 0x30e00, 0x30ed4, + 0x30f00, 0x30fa4, + 0x30fc0, 0x30fc4, + 0x31000, 0x31004, + 0x31080, 0x310fc, + 0x31208, 0x31220, + 0x3123c, 0x31254, + 0x31300, 0x31300, + 0x31308, 0x3131c, + 0x31338, 0x3133c, + 0x31380, 0x31380, + 0x31388, 0x313a8, + 0x313b4, 0x313b4, + 0x31400, 0x31420, + 0x31438, 0x3143c, + 0x31480, 0x31480, + 0x314a8, 0x314a8, + 0x314b0, 0x314b4, + 0x314c8, 0x314d4, + 0x31a40, 0x31a4c, + 0x31af0, 0x31b20, + 0x31b38, 0x31b3c, + 0x31b80, 0x31b80, + 0x31ba8, 0x31ba8, + 0x31bb0, 0x31bb4, + 0x31bc8, 0x31bd4, + 0x32140, 0x3218c, + 0x321f0, 0x321f4, + 0x32200, 0x32200, + 0x32218, 0x32218, + 0x32400, 0x32400, + 0x32408, 0x3241c, + 0x32618, 0x32620, + 0x32664, 0x32664, + 0x326a8, 0x326a8, + 0x326ec, 0x326ec, + 0x32a00, 0x32abc, + 0x32b00, 0x32b18, + 0x32b20, 0x32b38, + 0x32b40, 0x32b58, + 0x32b60, 0x32b78, + 0x32c00, 0x32c00, + 0x32c08, 0x32c3c, + 0x33000, 0x3302c, + 0x33034, 0x33050, + 0x33058, 0x33058, + 0x33060, 0x3308c, + 0x3309c, 0x330ac, + 0x330c0, 0x330c0, + 0x330c8, 0x330d0, + 0x330d8, 0x330e0, + 0x330ec, 0x3312c, + 0x33134, 0x33150, + 0x33158, 0x33158, + 0x33160, 0x3318c, + 0x3319c, 0x331ac, + 0x331c0, 0x331c0, + 0x331c8, 0x331d0, + 0x331d8, 0x331e0, + 0x331ec, 0x33290, + 0x33298, 0x332c4, + 0x332e4, 0x33390, + 0x33398, 0x333c4, + 0x333e4, 0x3342c, + 0x33434, 0x33450, + 0x33458, 0x33458, + 0x33460, 0x3348c, + 0x3349c, 0x334ac, + 0x334c0, 0x334c0, + 0x334c8, 0x334d0, + 0x334d8, 0x334e0, + 0x334ec, 0x3352c, + 0x33534, 0x33550, + 0x33558, 0x33558, + 0x33560, 0x3358c, + 0x3359c, 0x335ac, + 0x335c0, 0x335c0, + 0x335c8, 0x335d0, + 0x335d8, 0x335e0, + 0x335ec, 0x33690, + 0x33698, 0x336c4, + 0x336e4, 0x33790, + 0x33798, 0x337c4, + 0x337e4, 0x337fc, + 0x33814, 0x33814, + 0x33854, 0x33868, + 0x33880, 0x3388c, + 0x338c0, 0x338d0, + 0x338e8, 0x338ec, + 0x33900, 0x3392c, + 0x33934, 0x33950, + 0x33958, 0x33958, + 0x33960, 0x3398c, + 0x3399c, 0x339ac, + 0x339c0, 0x339c0, + 0x339c8, 0x339d0, + 0x339d8, 0x339e0, + 0x339ec, 0x33a90, + 0x33a98, 0x33ac4, + 0x33ae4, 0x33b10, + 0x33b24, 0x33b28, + 0x33b38, 0x33b50, + 0x33bf0, 0x33c10, + 0x33c24, 0x33c28, + 0x33c38, 0x33c50, + 0x33cf0, 0x33cfc, + 0x34000, 0x34030, + 0x34100, 0x34168, + 0x34190, 0x341a0, + 0x341a8, 0x341b8, + 0x341c4, 0x341c8, + 0x341d0, 0x341d0, + 0x34200, 0x34320, + 0x34400, 0x344b4, + 0x344c0, 0x3452c, + 0x34540, 0x3461c, + 0x34800, 0x348a0, + 0x348c0, 0x34908, + 0x34910, 0x349b8, + 0x34a00, 0x34a04, + 0x34a0c, 0x34a14, + 0x34a1c, 0x34a2c, + 0x34a44, 0x34a50, + 0x34a74, 0x34a74, + 0x34a7c, 0x34afc, + 0x34b08, 0x34c24, + 0x34d00, 0x34d14, + 0x34d1c, 0x34d3c, + 0x34d44, 0x34d4c, + 0x34d54, 0x34d74, + 0x34d7c, 0x34d7c, + 0x34de0, 0x34de0, + 0x34e00, 0x34ed4, + 0x34f00, 0x34fa4, + 0x34fc0, 0x34fc4, + 0x35000, 0x35004, + 0x35080, 0x350fc, + 0x35208, 0x35220, + 0x3523c, 0x35254, + 0x35300, 0x35300, + 0x35308, 0x3531c, + 0x35338, 0x3533c, + 0x35380, 0x35380, + 0x35388, 0x353a8, + 0x353b4, 0x353b4, + 0x35400, 0x35420, + 0x35438, 0x3543c, + 0x35480, 0x35480, + 0x354a8, 0x354a8, + 0x354b0, 0x354b4, + 0x354c8, 0x354d4, + 0x35a40, 0x35a4c, + 0x35af0, 0x35b20, + 0x35b38, 0x35b3c, + 0x35b80, 0x35b80, + 0x35ba8, 0x35ba8, + 0x35bb0, 0x35bb4, + 0x35bc8, 0x35bd4, + 0x36140, 0x3618c, + 0x361f0, 0x361f4, + 0x36200, 0x36200, + 0x36218, 0x36218, + 0x36400, 0x36400, + 0x36408, 0x3641c, + 0x36618, 0x36620, + 0x36664, 0x36664, + 0x366a8, 0x366a8, + 0x366ec, 0x366ec, + 0x36a00, 0x36abc, + 0x36b00, 0x36b18, + 0x36b20, 0x36b38, + 0x36b40, 0x36b58, + 0x36b60, 0x36b78, + 0x36c00, 0x36c00, + 0x36c08, 0x36c3c, + 0x37000, 0x3702c, + 0x37034, 0x37050, + 0x37058, 0x37058, + 0x37060, 0x3708c, + 0x3709c, 0x370ac, + 0x370c0, 0x370c0, + 0x370c8, 0x370d0, + 0x370d8, 0x370e0, + 0x370ec, 0x3712c, + 0x37134, 0x37150, + 0x37158, 0x37158, + 0x37160, 0x3718c, + 0x3719c, 0x371ac, + 0x371c0, 0x371c0, + 0x371c8, 0x371d0, + 0x371d8, 0x371e0, + 0x371ec, 0x37290, + 0x37298, 0x372c4, + 0x372e4, 0x37390, + 0x37398, 0x373c4, + 0x373e4, 0x3742c, + 0x37434, 0x37450, + 0x37458, 0x37458, + 0x37460, 0x3748c, + 0x3749c, 0x374ac, + 0x374c0, 0x374c0, + 0x374c8, 0x374d0, + 0x374d8, 0x374e0, + 0x374ec, 0x3752c, + 0x37534, 0x37550, + 0x37558, 0x37558, + 0x37560, 0x3758c, + 0x3759c, 0x375ac, + 0x375c0, 0x375c0, + 0x375c8, 0x375d0, + 0x375d8, 0x375e0, + 0x375ec, 0x37690, + 0x37698, 0x376c4, + 0x376e4, 0x37790, + 0x37798, 0x377c4, + 0x377e4, 0x377fc, + 0x37814, 0x37814, + 0x37854, 0x37868, + 0x37880, 0x3788c, + 0x378c0, 0x378d0, + 0x378e8, 0x378ec, + 0x37900, 0x3792c, + 0x37934, 0x37950, + 0x37958, 0x37958, + 0x37960, 0x3798c, + 0x3799c, 0x379ac, + 0x379c0, 0x379c0, + 0x379c8, 0x379d0, + 0x379d8, 0x379e0, + 0x379ec, 0x37a90, + 0x37a98, 0x37ac4, + 0x37ae4, 0x37b10, + 0x37b24, 0x37b28, + 0x37b38, 0x37b50, + 0x37bf0, 0x37c10, + 0x37c24, 0x37c28, + 0x37c38, 0x37c50, + 0x37cf0, 0x37cfc, + 0x40040, 0x40040, + 0x40080, 0x40084, + 0x40100, 0x40100, + 0x40140, 0x401bc, + 0x40200, 0x40214, + 0x40228, 0x40228, + 0x40240, 0x40258, + 0x40280, 0x40280, + 0x40304, 0x40304, + 0x40330, 0x4033c, + 0x41304, 0x413c8, + 0x413d0, 0x413dc, + 0x413f0, 0x413f0, + 0x41400, 0x4140c, + 0x41414, 0x4141c, + 0x41480, 0x414d0, + 0x44000, 0x4407c, + 0x440c0, 0x441ac, + 0x441b4, 0x4427c, + 0x442c0, 0x443ac, + 0x443b4, 0x4447c, + 0x444c0, 0x445ac, + 0x445b4, 0x4467c, + 0x446c0, 0x447ac, + 0x447b4, 0x4487c, + 0x448c0, 0x449ac, + 0x449b4, 0x44a7c, + 0x44ac0, 0x44bac, + 0x44bb4, 0x44c7c, + 0x44cc0, 0x44dac, + 0x44db4, 0x44e7c, + 0x44ec0, 0x44fac, + 0x44fb4, 0x4507c, + 0x450c0, 0x451ac, + 0x451b4, 0x451fc, + 0x45800, 0x45804, + 0x45810, 0x45830, + 0x45840, 0x45860, + 0x45868, 0x45868, + 0x45880, 0x45884, + 0x458a0, 0x458b0, + 0x45a00, 0x45a04, + 0x45a10, 0x45a30, + 0x45a40, 0x45a60, + 0x45a68, 0x45a68, + 0x45a80, 0x45a84, + 0x45aa0, 0x45ab0, + 0x460c0, 0x460e4, + 0x47000, 0x4703c, + 0x47044, 0x4708c, + 0x47200, 0x47250, + 0x47400, 0x47408, + 0x47414, 0x47420, + 0x47600, 0x47618, + 0x47800, 0x47814, + 0x47820, 0x4782c, + 0x50000, 0x50084, + 0x50090, 0x500cc, + 0x50300, 0x50384, + 0x50400, 0x50400, + 0x50800, 0x50884, + 0x50890, 0x508cc, + 0x50b00, 0x50b84, + 0x50c00, 0x50c00, + 0x51000, 0x51020, + 0x51028, 0x510b0, + 0x51300, 0x51324, + }; + + u32 *buf_end = (u32 *)((char *)buf + buf_size); + const unsigned int *reg_ranges; + int reg_ranges_size, range; + unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip); + + /* Select the right set of register ranges to dump depending on the + * adapter chip type. + */ + switch (chip_version) { + case CHELSIO_T4: + reg_ranges = t4_reg_ranges; + reg_ranges_size = ARRAY_SIZE(t4_reg_ranges); + break; + + case CHELSIO_T5: + reg_ranges = t5_reg_ranges; + reg_ranges_size = ARRAY_SIZE(t5_reg_ranges); + break; + + case CHELSIO_T6: + reg_ranges = t6_reg_ranges; + reg_ranges_size = ARRAY_SIZE(t6_reg_ranges); + break; + + default: + dev_err(adap->pdev_dev, + "Unsupported chip version %d\n", chip_version); + return; + } + + /* Clear the register buffer and insert the appropriate register + * values selected by the above register ranges. + */ + memset(buf, 0, buf_size); + for (range = 0; range < reg_ranges_size; range += 2) { + unsigned int reg = reg_ranges[range]; + unsigned int last_reg = reg_ranges[range + 1]; + u32 *bufp = (u32 *)((char *)buf + reg); + + /* Iterate across the register range filling in the register + * buffer but don't write past the end of the register buffer. + */ + while (reg <= last_reg && bufp < buf_end) { + *bufp++ = t4_read_reg(adap, reg); + reg += sizeof(u32); + } + } +} + +#define EEPROM_STAT_ADDR 0x7bfc +#define VPD_BASE 0x400 +#define VPD_BASE_OLD 0 +#define VPD_LEN 1024 + +/** + * t4_eeprom_ptov - translate a physical EEPROM address to virtual + * @phys_addr: the physical EEPROM address + * @fn: the PCI function number + * @sz: size of function-specific area + * + * Translate a physical EEPROM address to virtual. The first 1K is + * accessed through virtual addresses starting at 31K, the rest is + * accessed through virtual addresses starting at 0. + * + * The mapping is as follows: + * [0..1K) -> [31K..32K) + * [1K..1K+A) -> [31K-A..31K) + * [1K+A..ES) -> [0..ES-A-1K) + * + * where A = @fn * @sz, and ES = EEPROM size. + */ +int t4_eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz) +{ + fn *= sz; + if (phys_addr < 1024) + return phys_addr + (31 << 10); + if (phys_addr < 1024 + fn) + return 31744 - fn + phys_addr - 1024; + if (phys_addr < EEPROMSIZE) + return phys_addr - 1024 - fn; + return -EINVAL; +} + +/** + * t4_seeprom_wp - enable/disable EEPROM write protection + * @adapter: the adapter + * @enable: whether to enable or disable write protection + * + * Enables or disables write protection on the serial EEPROM. + */ +int t4_seeprom_wp(struct adapter *adapter, bool enable) +{ + unsigned int v = enable ? 0xc : 0; + int ret = pci_write_vpd(adapter->pdev, EEPROM_STAT_ADDR, 4, &v); + return ret < 0 ? ret : 0; +} + +/** + * t4_get_raw_vpd_params - read VPD parameters from VPD EEPROM + * @adapter: adapter to read + * @p: where to store the parameters + * + * Reads card parameters stored in VPD EEPROM. + */ +int t4_get_raw_vpd_params(struct adapter *adapter, struct vpd_params *p) +{ + unsigned int id_len, pn_len, sn_len, na_len; + int id, sn, pn, na, addr, ret = 0; + u8 *vpd, base_val = 0; + + vpd = vmalloc(VPD_LEN); + if (!vpd) + return -ENOMEM; + + /* Card information normally starts at VPD_BASE but early cards had + * it at 0. + */ + ret = pci_read_vpd(adapter->pdev, VPD_BASE, 1, &base_val); + if (ret < 0) + goto out; + + addr = base_val == PCI_VPD_LRDT_ID_STRING ? VPD_BASE : VPD_BASE_OLD; + + ret = pci_read_vpd(adapter->pdev, addr, VPD_LEN, vpd); + if (ret < 0) + goto out; + + ret = pci_vpd_find_id_string(vpd, VPD_LEN, &id_len); + if (ret < 0) + goto out; + id = ret; + + ret = pci_vpd_check_csum(vpd, VPD_LEN); + if (ret) { + dev_err(adapter->pdev_dev, "VPD checksum incorrect or missing\n"); + ret = -EINVAL; + goto out; + } + + ret = pci_vpd_find_ro_info_keyword(vpd, VPD_LEN, + PCI_VPD_RO_KEYWORD_SERIALNO, &sn_len); + if (ret < 0) + goto out; + sn = ret; + + ret = pci_vpd_find_ro_info_keyword(vpd, VPD_LEN, + PCI_VPD_RO_KEYWORD_PARTNO, &pn_len); + if (ret < 0) + goto out; + pn = ret; + + ret = pci_vpd_find_ro_info_keyword(vpd, VPD_LEN, "NA", &na_len); + if (ret < 0) + goto out; + na = ret; + + memcpy(p->id, vpd + id, min_t(unsigned int, id_len, ID_LEN)); + strim(p->id); + memcpy(p->sn, vpd + sn, min_t(unsigned int, sn_len, SERNUM_LEN)); + strim(p->sn); + memcpy(p->pn, vpd + pn, min_t(unsigned int, pn_len, PN_LEN)); + strim(p->pn); + memcpy(p->na, vpd + na, min_t(unsigned int, na_len, MACADDR_LEN)); + strim(p->na); + +out: + vfree(vpd); + if (ret < 0) { + dev_err(adapter->pdev_dev, "error reading VPD\n"); + return ret; + } + + return 0; +} + +/** + * t4_get_vpd_params - read VPD parameters & retrieve Core Clock + * @adapter: adapter to read + * @p: where to store the parameters + * + * Reads card parameters stored in VPD EEPROM and retrieves the Core + * Clock. This can only be called after a connection to the firmware + * is established. + */ +int t4_get_vpd_params(struct adapter *adapter, struct vpd_params *p) +{ + u32 cclk_param, cclk_val; + int ret; + + /* Grab the raw VPD parameters. + */ + ret = t4_get_raw_vpd_params(adapter, p); + if (ret) + return ret; + + /* Ask firmware for the Core Clock since it knows how to translate the + * Reference Clock ('V2') VPD field into a Core Clock value ... + */ + cclk_param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CCLK)); + ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0, + 1, &cclk_param, &cclk_val); + + if (ret) + return ret; + p->cclk = cclk_val; + + return 0; +} + +/** + * t4_get_pfres - retrieve VF resource limits + * @adapter: the adapter + * + * Retrieves configured resource limits and capabilities for a physical + * function. The results are stored in @adapter->pfres. + */ +int t4_get_pfres(struct adapter *adapter) +{ + struct pf_resources *pfres = &adapter->params.pfres; + struct fw_pfvf_cmd cmd, rpl; + int v; + u32 word; + + /* Execute PFVF Read command to get VF resource limits; bail out early + * with error on command failure. + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PFVF_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F | + FW_PFVF_CMD_PFN_V(adapter->pf) | + FW_PFVF_CMD_VFN_V(0)); + cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); + v = t4_wr_mbox(adapter, adapter->mbox, &cmd, sizeof(cmd), &rpl); + if (v != FW_SUCCESS) + return v; + + /* Extract PF resource limits and return success. + */ + word = be32_to_cpu(rpl.niqflint_niq); + pfres->niqflint = FW_PFVF_CMD_NIQFLINT_G(word); + pfres->niq = FW_PFVF_CMD_NIQ_G(word); + + word = be32_to_cpu(rpl.type_to_neq); + pfres->neq = FW_PFVF_CMD_NEQ_G(word); + pfres->pmask = FW_PFVF_CMD_PMASK_G(word); + + word = be32_to_cpu(rpl.tc_to_nexactf); + pfres->tc = FW_PFVF_CMD_TC_G(word); + pfres->nvi = FW_PFVF_CMD_NVI_G(word); + pfres->nexactf = FW_PFVF_CMD_NEXACTF_G(word); + + word = be32_to_cpu(rpl.r_caps_to_nethctrl); + pfres->r_caps = FW_PFVF_CMD_R_CAPS_G(word); + pfres->wx_caps = FW_PFVF_CMD_WX_CAPS_G(word); + pfres->nethctrl = FW_PFVF_CMD_NETHCTRL_G(word); + + return 0; +} + +/* serial flash and firmware constants */ +enum { + SF_ATTEMPTS = 10, /* max retries for SF operations */ + + /* flash command opcodes */ + SF_PROG_PAGE = 2, /* program page */ + SF_WR_DISABLE = 4, /* disable writes */ + SF_RD_STATUS = 5, /* read status register */ + SF_WR_ENABLE = 6, /* enable writes */ + SF_RD_DATA_FAST = 0xb, /* read flash */ + SF_RD_ID = 0x9f, /* read ID */ + SF_ERASE_SECTOR = 0xd8, /* erase sector */ +}; + +/** + * sf1_read - read data from the serial flash + * @adapter: the adapter + * @byte_cnt: number of bytes to read + * @cont: whether another operation will be chained + * @lock: whether to lock SF for PL access only + * @valp: where to store the read data + * + * Reads up to 4 bytes of data from the serial flash. The location of + * the read needs to be specified prior to calling this by issuing the + * appropriate commands to the serial flash. + */ +static int sf1_read(struct adapter *adapter, unsigned int byte_cnt, int cont, + int lock, u32 *valp) +{ + int ret; + + if (!byte_cnt || byte_cnt > 4) + return -EINVAL; + if (t4_read_reg(adapter, SF_OP_A) & SF_BUSY_F) + return -EBUSY; + t4_write_reg(adapter, SF_OP_A, SF_LOCK_V(lock) | + SF_CONT_V(cont) | BYTECNT_V(byte_cnt - 1)); + ret = t4_wait_op_done(adapter, SF_OP_A, SF_BUSY_F, 0, SF_ATTEMPTS, 5); + if (!ret) + *valp = t4_read_reg(adapter, SF_DATA_A); + return ret; +} + +/** + * sf1_write - write data to the serial flash + * @adapter: the adapter + * @byte_cnt: number of bytes to write + * @cont: whether another operation will be chained + * @lock: whether to lock SF for PL access only + * @val: value to write + * + * Writes up to 4 bytes of data to the serial flash. The location of + * the write needs to be specified prior to calling this by issuing the + * appropriate commands to the serial flash. + */ +static int sf1_write(struct adapter *adapter, unsigned int byte_cnt, int cont, + int lock, u32 val) +{ + if (!byte_cnt || byte_cnt > 4) + return -EINVAL; + if (t4_read_reg(adapter, SF_OP_A) & SF_BUSY_F) + return -EBUSY; + t4_write_reg(adapter, SF_DATA_A, val); + t4_write_reg(adapter, SF_OP_A, SF_LOCK_V(lock) | + SF_CONT_V(cont) | BYTECNT_V(byte_cnt - 1) | OP_V(1)); + return t4_wait_op_done(adapter, SF_OP_A, SF_BUSY_F, 0, SF_ATTEMPTS, 5); +} + +/** + * flash_wait_op - wait for a flash operation to complete + * @adapter: the adapter + * @attempts: max number of polls of the status register + * @delay: delay between polls in ms + * + * Wait for a flash operation to complete by polling the status register. + */ +static int flash_wait_op(struct adapter *adapter, int attempts, int delay) +{ + int ret; + u32 status; + + while (1) { + if ((ret = sf1_write(adapter, 1, 1, 1, SF_RD_STATUS)) != 0 || + (ret = sf1_read(adapter, 1, 0, 1, &status)) != 0) + return ret; + if (!(status & 1)) + return 0; + if (--attempts == 0) + return -EAGAIN; + if (delay) + msleep(delay); + } +} + +/** + * t4_read_flash - read words from serial flash + * @adapter: the adapter + * @addr: the start address for the read + * @nwords: how many 32-bit words to read + * @data: where to store the read data + * @byte_oriented: whether to store data as bytes or as words + * + * Read the specified number of 32-bit words from the serial flash. + * If @byte_oriented is set the read data is stored as a byte array + * (i.e., big-endian), otherwise as 32-bit words in the platform's + * natural endianness. + */ +int t4_read_flash(struct adapter *adapter, unsigned int addr, + unsigned int nwords, u32 *data, int byte_oriented) +{ + int ret; + + if (addr + nwords * sizeof(u32) > adapter->params.sf_size || (addr & 3)) + return -EINVAL; + + addr = swab32(addr) | SF_RD_DATA_FAST; + + if ((ret = sf1_write(adapter, 4, 1, 0, addr)) != 0 || + (ret = sf1_read(adapter, 1, 1, 0, data)) != 0) + return ret; + + for ( ; nwords; nwords--, data++) { + ret = sf1_read(adapter, 4, nwords > 1, nwords == 1, data); + if (nwords == 1) + t4_write_reg(adapter, SF_OP_A, 0); /* unlock SF */ + if (ret) + return ret; + if (byte_oriented) + *data = (__force __u32)(cpu_to_be32(*data)); + } + return 0; +} + +/** + * t4_write_flash - write up to a page of data to the serial flash + * @adapter: the adapter + * @addr: the start address to write + * @n: length of data to write in bytes + * @data: the data to write + * @byte_oriented: whether to store data as bytes or as words + * + * Writes up to a page of data (256 bytes) to the serial flash starting + * at the given address. All the data must be written to the same page. + * If @byte_oriented is set the write data is stored as byte stream + * (i.e. matches what on disk), otherwise in big-endian. + */ +static int t4_write_flash(struct adapter *adapter, unsigned int addr, + unsigned int n, const u8 *data, bool byte_oriented) +{ + unsigned int i, c, left, val, offset = addr & 0xff; + u32 buf[64]; + int ret; + + if (addr >= adapter->params.sf_size || offset + n > SF_PAGE_SIZE) + return -EINVAL; + + val = swab32(addr) | SF_PROG_PAGE; + + if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 || + (ret = sf1_write(adapter, 4, 1, 1, val)) != 0) + goto unlock; + + for (left = n; left; left -= c, data += c) { + c = min(left, 4U); + for (val = 0, i = 0; i < c; ++i) { + if (byte_oriented) + val = (val << 8) + data[i]; + else + val = (val << 8) + data[c - i - 1]; + } + + ret = sf1_write(adapter, c, c != left, 1, val); + if (ret) + goto unlock; + } + ret = flash_wait_op(adapter, 8, 1); + if (ret) + goto unlock; + + t4_write_reg(adapter, SF_OP_A, 0); /* unlock SF */ + + /* Read the page to verify the write succeeded */ + ret = t4_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, + byte_oriented); + if (ret) + return ret; + + if (memcmp(data - n, (u8 *)buf + offset, n)) { + dev_err(adapter->pdev_dev, + "failed to correctly write the flash page at %#x\n", + addr); + return -EIO; + } + return 0; + +unlock: + t4_write_reg(adapter, SF_OP_A, 0); /* unlock SF */ + return ret; +} + +/** + * t4_get_fw_version - read the firmware version + * @adapter: the adapter + * @vers: where to place the version + * + * Reads the FW version from flash. + */ +int t4_get_fw_version(struct adapter *adapter, u32 *vers) +{ + return t4_read_flash(adapter, FLASH_FW_START + + offsetof(struct fw_hdr, fw_ver), 1, + vers, 0); +} + +/** + * t4_get_bs_version - read the firmware bootstrap version + * @adapter: the adapter + * @vers: where to place the version + * + * Reads the FW Bootstrap version from flash. + */ +int t4_get_bs_version(struct adapter *adapter, u32 *vers) +{ + return t4_read_flash(adapter, FLASH_FWBOOTSTRAP_START + + offsetof(struct fw_hdr, fw_ver), 1, + vers, 0); +} + +/** + * t4_get_tp_version - read the TP microcode version + * @adapter: the adapter + * @vers: where to place the version + * + * Reads the TP microcode version from flash. + */ +int t4_get_tp_version(struct adapter *adapter, u32 *vers) +{ + return t4_read_flash(adapter, FLASH_FW_START + + offsetof(struct fw_hdr, tp_microcode_ver), + 1, vers, 0); +} + +/** + * t4_get_exprom_version - return the Expansion ROM version (if any) + * @adap: the adapter + * @vers: where to place the version + * + * Reads the Expansion ROM header from FLASH and returns the version + * number (if present) through the @vers return value pointer. We return + * this in the Firmware Version Format since it's convenient. Return + * 0 on success, -ENOENT if no Expansion ROM is present. + */ +int t4_get_exprom_version(struct adapter *adap, u32 *vers) +{ + struct exprom_header { + unsigned char hdr_arr[16]; /* must start with 0x55aa */ + unsigned char hdr_ver[4]; /* Expansion ROM version */ + } *hdr; + u32 exprom_header_buf[DIV_ROUND_UP(sizeof(struct exprom_header), + sizeof(u32))]; + int ret; + + ret = t4_read_flash(adap, FLASH_EXP_ROM_START, + ARRAY_SIZE(exprom_header_buf), exprom_header_buf, + 0); + if (ret) + return ret; + + hdr = (struct exprom_header *)exprom_header_buf; + if (hdr->hdr_arr[0] != 0x55 || hdr->hdr_arr[1] != 0xaa) + return -ENOENT; + + *vers = (FW_HDR_FW_VER_MAJOR_V(hdr->hdr_ver[0]) | + FW_HDR_FW_VER_MINOR_V(hdr->hdr_ver[1]) | + FW_HDR_FW_VER_MICRO_V(hdr->hdr_ver[2]) | + FW_HDR_FW_VER_BUILD_V(hdr->hdr_ver[3])); + return 0; +} + +/** + * t4_get_vpd_version - return the VPD version + * @adapter: the adapter + * @vers: where to place the version + * + * Reads the VPD via the Firmware interface (thus this can only be called + * once we're ready to issue Firmware commands). The format of the + * VPD version is adapter specific. Returns 0 on success, an error on + * failure. + * + * Note that early versions of the Firmware didn't include the ability + * to retrieve the VPD version, so we zero-out the return-value parameter + * in that case to avoid leaving it with garbage in it. + * + * Also note that the Firmware will return its cached copy of the VPD + * Revision ID, not the actual Revision ID as written in the Serial + * EEPROM. This is only an issue if a new VPD has been written and the + * Firmware/Chip haven't yet gone through a RESET sequence. So it's best + * to defer calling this routine till after a FW_RESET_CMD has been issued + * if the Host Driver will be performing a full adapter initialization. + */ +int t4_get_vpd_version(struct adapter *adapter, u32 *vers) +{ + u32 vpdrev_param; + int ret; + + vpdrev_param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_VPDREV)); + ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0, + 1, &vpdrev_param, vers); + if (ret) + *vers = 0; + return ret; +} + +/** + * t4_get_scfg_version - return the Serial Configuration version + * @adapter: the adapter + * @vers: where to place the version + * + * Reads the Serial Configuration Version via the Firmware interface + * (thus this can only be called once we're ready to issue Firmware + * commands). The format of the Serial Configuration version is + * adapter specific. Returns 0 on success, an error on failure. + * + * Note that early versions of the Firmware didn't include the ability + * to retrieve the Serial Configuration version, so we zero-out the + * return-value parameter in that case to avoid leaving it with + * garbage in it. + * + * Also note that the Firmware will return its cached copy of the Serial + * Initialization Revision ID, not the actual Revision ID as written in + * the Serial EEPROM. This is only an issue if a new VPD has been written + * and the Firmware/Chip haven't yet gone through a RESET sequence. So + * it's best to defer calling this routine till after a FW_RESET_CMD has + * been issued if the Host Driver will be performing a full adapter + * initialization. + */ +int t4_get_scfg_version(struct adapter *adapter, u32 *vers) +{ + u32 scfgrev_param; + int ret; + + scfgrev_param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_SCFGREV)); + ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0, + 1, &scfgrev_param, vers); + if (ret) + *vers = 0; + return ret; +} + +/** + * t4_get_version_info - extract various chip/firmware version information + * @adapter: the adapter + * + * Reads various chip/firmware version numbers and stores them into the + * adapter Adapter Parameters structure. If any of the efforts fails + * the first failure will be returned, but all of the version numbers + * will be read. + */ +int t4_get_version_info(struct adapter *adapter) +{ + int ret = 0; + + #define FIRST_RET(__getvinfo) \ + do { \ + int __ret = __getvinfo; \ + if (__ret && !ret) \ + ret = __ret; \ + } while (0) + + FIRST_RET(t4_get_fw_version(adapter, &adapter->params.fw_vers)); + FIRST_RET(t4_get_bs_version(adapter, &adapter->params.bs_vers)); + FIRST_RET(t4_get_tp_version(adapter, &adapter->params.tp_vers)); + FIRST_RET(t4_get_exprom_version(adapter, &adapter->params.er_vers)); + FIRST_RET(t4_get_scfg_version(adapter, &adapter->params.scfg_vers)); + FIRST_RET(t4_get_vpd_version(adapter, &adapter->params.vpd_vers)); + + #undef FIRST_RET + return ret; +} + +/** + * t4_dump_version_info - dump all of the adapter configuration IDs + * @adapter: the adapter + * + * Dumps all of the various bits of adapter configuration version/revision + * IDs information. This is typically called at some point after + * t4_get_version_info() has been called. + */ +void t4_dump_version_info(struct adapter *adapter) +{ + /* Device information */ + dev_info(adapter->pdev_dev, "Chelsio %s rev %d\n", + adapter->params.vpd.id, + CHELSIO_CHIP_RELEASE(adapter->params.chip)); + dev_info(adapter->pdev_dev, "S/N: %s, P/N: %s\n", + adapter->params.vpd.sn, adapter->params.vpd.pn); + + /* Firmware Version */ + if (!adapter->params.fw_vers) + dev_warn(adapter->pdev_dev, "No firmware loaded\n"); + else + dev_info(adapter->pdev_dev, "Firmware version: %u.%u.%u.%u\n", + FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers), + FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers), + FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers), + FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers)); + + /* Bootstrap Firmware Version. (Some adapters don't have Bootstrap + * Firmware, so dev_info() is more appropriate here.) + */ + if (!adapter->params.bs_vers) + dev_info(adapter->pdev_dev, "No bootstrap loaded\n"); + else + dev_info(adapter->pdev_dev, "Bootstrap version: %u.%u.%u.%u\n", + FW_HDR_FW_VER_MAJOR_G(adapter->params.bs_vers), + FW_HDR_FW_VER_MINOR_G(adapter->params.bs_vers), + FW_HDR_FW_VER_MICRO_G(adapter->params.bs_vers), + FW_HDR_FW_VER_BUILD_G(adapter->params.bs_vers)); + + /* TP Microcode Version */ + if (!adapter->params.tp_vers) + dev_warn(adapter->pdev_dev, "No TP Microcode loaded\n"); + else + dev_info(adapter->pdev_dev, + "TP Microcode version: %u.%u.%u.%u\n", + FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers), + FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers), + FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers), + FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers)); + + /* Expansion ROM version */ + if (!adapter->params.er_vers) + dev_info(adapter->pdev_dev, "No Expansion ROM loaded\n"); + else + dev_info(adapter->pdev_dev, + "Expansion ROM version: %u.%u.%u.%u\n", + FW_HDR_FW_VER_MAJOR_G(adapter->params.er_vers), + FW_HDR_FW_VER_MINOR_G(adapter->params.er_vers), + FW_HDR_FW_VER_MICRO_G(adapter->params.er_vers), + FW_HDR_FW_VER_BUILD_G(adapter->params.er_vers)); + + /* Serial Configuration version */ + dev_info(adapter->pdev_dev, "Serial Configuration version: %#x\n", + adapter->params.scfg_vers); + + /* VPD Version */ + dev_info(adapter->pdev_dev, "VPD version: %#x\n", + adapter->params.vpd_vers); +} + +/** + * t4_check_fw_version - check if the FW is supported with this driver + * @adap: the adapter + * + * Checks if an adapter's FW is compatible with the driver. Returns 0 + * if there's exact match, a negative error if the version could not be + * read or there's a major version mismatch + */ +int t4_check_fw_version(struct adapter *adap) +{ + int i, ret, major, minor, micro; + int exp_major, exp_minor, exp_micro; + unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip); + + ret = t4_get_fw_version(adap, &adap->params.fw_vers); + /* Try multiple times before returning error */ + for (i = 0; (ret == -EBUSY || ret == -EAGAIN) && i < 3; i++) + ret = t4_get_fw_version(adap, &adap->params.fw_vers); + + if (ret) + return ret; + + major = FW_HDR_FW_VER_MAJOR_G(adap->params.fw_vers); + minor = FW_HDR_FW_VER_MINOR_G(adap->params.fw_vers); + micro = FW_HDR_FW_VER_MICRO_G(adap->params.fw_vers); + + switch (chip_version) { + case CHELSIO_T4: + exp_major = T4FW_MIN_VERSION_MAJOR; + exp_minor = T4FW_MIN_VERSION_MINOR; + exp_micro = T4FW_MIN_VERSION_MICRO; + break; + case CHELSIO_T5: + exp_major = T5FW_MIN_VERSION_MAJOR; + exp_minor = T5FW_MIN_VERSION_MINOR; + exp_micro = T5FW_MIN_VERSION_MICRO; + break; + case CHELSIO_T6: + exp_major = T6FW_MIN_VERSION_MAJOR; + exp_minor = T6FW_MIN_VERSION_MINOR; + exp_micro = T6FW_MIN_VERSION_MICRO; + break; + default: + dev_err(adap->pdev_dev, "Unsupported chip type, %x\n", + adap->chip); + return -EINVAL; + } + + if (major < exp_major || (major == exp_major && minor < exp_minor) || + (major == exp_major && minor == exp_minor && micro < exp_micro)) { + dev_err(adap->pdev_dev, + "Card has firmware version %u.%u.%u, minimum " + "supported firmware is %u.%u.%u.\n", major, minor, + micro, exp_major, exp_minor, exp_micro); + return -EFAULT; + } + return 0; +} + +/* Is the given firmware API compatible with the one the driver was compiled + * with? + */ +static int fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) +{ + + /* short circuit if it's the exact same firmware version */ + if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) + return 1; + +#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) + if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && + SAME_INTF(ri) && SAME_INTF(iscsi) && SAME_INTF(fcoe)) + return 1; +#undef SAME_INTF + + return 0; +} + +/* The firmware in the filesystem is usable, but should it be installed? + * This routine explains itself in detail if it indicates the filesystem + * firmware should be installed. + */ +static int should_install_fs_fw(struct adapter *adap, int card_fw_usable, + int k, int c) +{ + const char *reason; + + if (!card_fw_usable) { + reason = "incompatible or unusable"; + goto install; + } + + if (k > c) { + reason = "older than the version supported with this driver"; + goto install; + } + + return 0; + +install: + dev_err(adap->pdev_dev, "firmware on card (%u.%u.%u.%u) is %s, " + "installing firmware %u.%u.%u.%u on card.\n", + FW_HDR_FW_VER_MAJOR_G(c), FW_HDR_FW_VER_MINOR_G(c), + FW_HDR_FW_VER_MICRO_G(c), FW_HDR_FW_VER_BUILD_G(c), reason, + FW_HDR_FW_VER_MAJOR_G(k), FW_HDR_FW_VER_MINOR_G(k), + FW_HDR_FW_VER_MICRO_G(k), FW_HDR_FW_VER_BUILD_G(k)); + + return 1; +} + +int t4_prep_fw(struct adapter *adap, struct fw_info *fw_info, + const u8 *fw_data, unsigned int fw_size, + struct fw_hdr *card_fw, enum dev_state state, + int *reset) +{ + int ret, card_fw_usable, fs_fw_usable; + const struct fw_hdr *fs_fw; + const struct fw_hdr *drv_fw; + + drv_fw = &fw_info->fw_hdr; + + /* Read the header of the firmware on the card */ + ret = t4_read_flash(adap, FLASH_FW_START, + sizeof(*card_fw) / sizeof(uint32_t), + (uint32_t *)card_fw, 1); + if (ret == 0) { + card_fw_usable = fw_compatible(drv_fw, (const void *)card_fw); + } else { + dev_err(adap->pdev_dev, + "Unable to read card's firmware header: %d\n", ret); + card_fw_usable = 0; + } + + if (fw_data != NULL) { + fs_fw = (const void *)fw_data; + fs_fw_usable = fw_compatible(drv_fw, fs_fw); + } else { + fs_fw = NULL; + fs_fw_usable = 0; + } + + if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && + (!fs_fw_usable || fs_fw->fw_ver == drv_fw->fw_ver)) { + /* Common case: the firmware on the card is an exact match and + * the filesystem one is an exact match too, or the filesystem + * one is absent/incompatible. + */ + } else if (fs_fw_usable && state == DEV_STATE_UNINIT && + should_install_fs_fw(adap, card_fw_usable, + be32_to_cpu(fs_fw->fw_ver), + be32_to_cpu(card_fw->fw_ver))) { + ret = t4_fw_upgrade(adap, adap->mbox, fw_data, + fw_size, 0); + if (ret != 0) { + dev_err(adap->pdev_dev, + "failed to install firmware: %d\n", ret); + goto bye; + } + + /* Installed successfully, update the cached header too. */ + *card_fw = *fs_fw; + card_fw_usable = 1; + *reset = 0; /* already reset as part of load_fw */ + } + + if (!card_fw_usable) { + uint32_t d, c, k; + + d = be32_to_cpu(drv_fw->fw_ver); + c = be32_to_cpu(card_fw->fw_ver); + k = fs_fw ? be32_to_cpu(fs_fw->fw_ver) : 0; + + dev_err(adap->pdev_dev, "Cannot find a usable firmware: " + "chip state %d, " + "driver compiled with %d.%d.%d.%d, " + "card has %d.%d.%d.%d, filesystem has %d.%d.%d.%d\n", + state, + FW_HDR_FW_VER_MAJOR_G(d), FW_HDR_FW_VER_MINOR_G(d), + FW_HDR_FW_VER_MICRO_G(d), FW_HDR_FW_VER_BUILD_G(d), + FW_HDR_FW_VER_MAJOR_G(c), FW_HDR_FW_VER_MINOR_G(c), + FW_HDR_FW_VER_MICRO_G(c), FW_HDR_FW_VER_BUILD_G(c), + FW_HDR_FW_VER_MAJOR_G(k), FW_HDR_FW_VER_MINOR_G(k), + FW_HDR_FW_VER_MICRO_G(k), FW_HDR_FW_VER_BUILD_G(k)); + ret = -EINVAL; + goto bye; + } + + /* We're using whatever's on the card and it's known to be good. */ + adap->params.fw_vers = be32_to_cpu(card_fw->fw_ver); + adap->params.tp_vers = be32_to_cpu(card_fw->tp_microcode_ver); + +bye: + return ret; +} + +/** + * t4_flash_erase_sectors - erase a range of flash sectors + * @adapter: the adapter + * @start: the first sector to erase + * @end: the last sector to erase + * + * Erases the sectors in the given inclusive range. + */ +static int t4_flash_erase_sectors(struct adapter *adapter, int start, int end) +{ + int ret = 0; + + if (end >= adapter->params.sf_nsec) + return -EINVAL; + + while (start <= end) { + if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 || + (ret = sf1_write(adapter, 4, 0, 1, + SF_ERASE_SECTOR | (start << 8))) != 0 || + (ret = flash_wait_op(adapter, 14, 500)) != 0) { + dev_err(adapter->pdev_dev, + "erase of flash sector %d failed, error %d\n", + start, ret); + break; + } + start++; + } + t4_write_reg(adapter, SF_OP_A, 0); /* unlock SF */ + return ret; +} + +/** + * t4_flash_cfg_addr - return the address of the flash configuration file + * @adapter: the adapter + * + * Return the address within the flash where the Firmware Configuration + * File is stored. + */ +unsigned int t4_flash_cfg_addr(struct adapter *adapter) +{ + if (adapter->params.sf_size == 0x100000) + return FLASH_FPGA_CFG_START; + else + return FLASH_CFG_START; +} + +/* Return TRUE if the specified firmware matches the adapter. I.e. T4 + * firmware for T4 adapters, T5 firmware for T5 adapters, etc. We go ahead + * and emit an error message for mismatched firmware to save our caller the + * effort ... + */ +static bool t4_fw_matches_chip(const struct adapter *adap, + const struct fw_hdr *hdr) +{ + /* The expression below will return FALSE for any unsupported adapter + * which will keep us "honest" in the future ... + */ + if ((is_t4(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T4) || + (is_t5(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T5) || + (is_t6(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T6)) + return true; + + dev_err(adap->pdev_dev, + "FW image (%d) is not suitable for this adapter (%d)\n", + hdr->chip, CHELSIO_CHIP_VERSION(adap->params.chip)); + return false; +} + +/** + * t4_load_fw - download firmware + * @adap: the adapter + * @fw_data: the firmware image to write + * @size: image size + * + * Write the supplied firmware image to the card's serial flash. + */ +int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size) +{ + u32 csum; + int ret, addr; + unsigned int i; + u8 first_page[SF_PAGE_SIZE]; + const __be32 *p = (const __be32 *)fw_data; + const struct fw_hdr *hdr = (const struct fw_hdr *)fw_data; + unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec; + unsigned int fw_start_sec = FLASH_FW_START_SEC; + unsigned int fw_size = FLASH_FW_MAX_SIZE; + unsigned int fw_start = FLASH_FW_START; + + if (!size) { + dev_err(adap->pdev_dev, "FW image has no data\n"); + return -EINVAL; + } + if (size & 511) { + dev_err(adap->pdev_dev, + "FW image size not multiple of 512 bytes\n"); + return -EINVAL; + } + if ((unsigned int)be16_to_cpu(hdr->len512) * 512 != size) { + dev_err(adap->pdev_dev, + "FW image size differs from size in FW header\n"); + return -EINVAL; + } + if (size > fw_size) { + dev_err(adap->pdev_dev, "FW image too large, max is %u bytes\n", + fw_size); + return -EFBIG; + } + if (!t4_fw_matches_chip(adap, hdr)) + return -EINVAL; + + for (csum = 0, i = 0; i < size / sizeof(csum); i++) + csum += be32_to_cpu(p[i]); + + if (csum != 0xffffffff) { + dev_err(adap->pdev_dev, + "corrupted firmware image, checksum %#x\n", csum); + return -EINVAL; + } + + i = DIV_ROUND_UP(size, sf_sec_size); /* # of sectors spanned */ + ret = t4_flash_erase_sectors(adap, fw_start_sec, fw_start_sec + i - 1); + if (ret) + goto out; + + /* + * We write the correct version at the end so the driver can see a bad + * version if the FW write fails. Start by writing a copy of the + * first page with a bad version. + */ + memcpy(first_page, fw_data, SF_PAGE_SIZE); + ((struct fw_hdr *)first_page)->fw_ver = cpu_to_be32(0xffffffff); + ret = t4_write_flash(adap, fw_start, SF_PAGE_SIZE, first_page, true); + if (ret) + goto out; + + addr = fw_start; + for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) { + addr += SF_PAGE_SIZE; + fw_data += SF_PAGE_SIZE; + ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, fw_data, true); + if (ret) + goto out; + } + + ret = t4_write_flash(adap, fw_start + offsetof(struct fw_hdr, fw_ver), + sizeof(hdr->fw_ver), (const u8 *)&hdr->fw_ver, + true); +out: + if (ret) + dev_err(adap->pdev_dev, "firmware download failed, error %d\n", + ret); + else + ret = t4_get_fw_version(adap, &adap->params.fw_vers); + return ret; +} + +/** + * t4_phy_fw_ver - return current PHY firmware version + * @adap: the adapter + * @phy_fw_ver: return value buffer for PHY firmware version + * + * Returns the current version of external PHY firmware on the + * adapter. + */ +int t4_phy_fw_ver(struct adapter *adap, int *phy_fw_ver) +{ + u32 param, val; + int ret; + + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PHYFW) | + FW_PARAMS_PARAM_Y_V(adap->params.portvec) | + FW_PARAMS_PARAM_Z_V(FW_PARAMS_PARAM_DEV_PHYFW_VERSION)); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, + ¶m, &val); + if (ret) + return ret; + *phy_fw_ver = val; + return 0; +} + +/** + * t4_load_phy_fw - download port PHY firmware + * @adap: the adapter + * @win: the PCI-E Memory Window index to use for t4_memory_rw() + * @phy_fw_version: function to check PHY firmware versions + * @phy_fw_data: the PHY firmware image to write + * @phy_fw_size: image size + * + * Transfer the specified PHY firmware to the adapter. If a non-NULL + * @phy_fw_version is supplied, then it will be used to determine if + * it's necessary to perform the transfer by comparing the version + * of any existing adapter PHY firmware with that of the passed in + * PHY firmware image. + * + * A negative error number will be returned if an error occurs. If + * version number support is available and there's no need to upgrade + * the firmware, 0 will be returned. If firmware is successfully + * transferred to the adapter, 1 will be returned. + * + * NOTE: some adapters only have local RAM to store the PHY firmware. As + * a result, a RESET of the adapter would cause that RAM to lose its + * contents. Thus, loading PHY firmware on such adapters must happen + * after any FW_RESET_CMDs ... + */ +int t4_load_phy_fw(struct adapter *adap, int win, + int (*phy_fw_version)(const u8 *, size_t), + const u8 *phy_fw_data, size_t phy_fw_size) +{ + int cur_phy_fw_ver = 0, new_phy_fw_vers = 0; + unsigned long mtype = 0, maddr = 0; + u32 param, val; + int ret; + + /* If we have version number support, then check to see if the adapter + * already has up-to-date PHY firmware loaded. + */ + if (phy_fw_version) { + new_phy_fw_vers = phy_fw_version(phy_fw_data, phy_fw_size); + ret = t4_phy_fw_ver(adap, &cur_phy_fw_ver); + if (ret < 0) + return ret; + + if (cur_phy_fw_ver >= new_phy_fw_vers) { + CH_WARN(adap, "PHY Firmware already up-to-date, " + "version %#x\n", cur_phy_fw_ver); + return 0; + } + } + + /* Ask the firmware where it wants us to copy the PHY firmware image. + * The size of the file requires a special version of the READ command + * which will pass the file size via the values field in PARAMS_CMD and + * retrieve the return value from firmware and place it in the same + * buffer values + */ + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PHYFW) | + FW_PARAMS_PARAM_Y_V(adap->params.portvec) | + FW_PARAMS_PARAM_Z_V(FW_PARAMS_PARAM_DEV_PHYFW_DOWNLOAD)); + val = phy_fw_size; + ret = t4_query_params_rw(adap, adap->mbox, adap->pf, 0, 1, + ¶m, &val, 1, true); + if (ret < 0) + return ret; + mtype = val >> 8; + maddr = (val & 0xff) << 16; + + /* Copy the supplied PHY Firmware image to the adapter memory location + * allocated by the adapter firmware. + */ + spin_lock_bh(&adap->win0_lock); + ret = t4_memory_rw(adap, win, mtype, maddr, + phy_fw_size, (__be32 *)phy_fw_data, + T4_MEMORY_WRITE); + spin_unlock_bh(&adap->win0_lock); + if (ret) + return ret; + + /* Tell the firmware that the PHY firmware image has been written to + * RAM and it can now start copying it over to the PHYs. The chip + * firmware will RESET the affected PHYs as part of this operation + * leaving them running the new PHY firmware image. + */ + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PHYFW) | + FW_PARAMS_PARAM_Y_V(adap->params.portvec) | + FW_PARAMS_PARAM_Z_V(FW_PARAMS_PARAM_DEV_PHYFW_DOWNLOAD)); + ret = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1, + ¶m, &val, 30000); + if (ret) + return ret; + + /* If we have version number support, then check to see that the new + * firmware got loaded properly. + */ + if (phy_fw_version) { + ret = t4_phy_fw_ver(adap, &cur_phy_fw_ver); + if (ret < 0) + return ret; + + if (cur_phy_fw_ver != new_phy_fw_vers) { + CH_WARN(adap, "PHY Firmware did not update: " + "version on adapter %#x, " + "version flashed %#x\n", + cur_phy_fw_ver, new_phy_fw_vers); + return -ENXIO; + } + } + + return 1; +} + +/** + * t4_fwcache - firmware cache operation + * @adap: the adapter + * @op : the operation (flush or flush and invalidate) + */ +int t4_fwcache(struct adapter *adap, enum fw_params_param_dev_fwcache op) +{ + struct fw_params_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = + cpu_to_be32(FW_CMD_OP_V(FW_PARAMS_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_PARAMS_CMD_PFN_V(adap->pf) | + FW_PARAMS_CMD_VFN_V(0)); + c.retval_len16 = cpu_to_be32(FW_LEN16(c)); + c.param[0].mnem = + cpu_to_be32(FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_FWCACHE)); + c.param[0].val = cpu_to_be32(op); + + return t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), NULL); +} + +void t4_cim_read_pif_la(struct adapter *adap, u32 *pif_req, u32 *pif_rsp, + unsigned int *pif_req_wrptr, + unsigned int *pif_rsp_wrptr) +{ + int i, j; + u32 cfg, val, req, rsp; + + cfg = t4_read_reg(adap, CIM_DEBUGCFG_A); + if (cfg & LADBGEN_F) + t4_write_reg(adap, CIM_DEBUGCFG_A, cfg ^ LADBGEN_F); + + val = t4_read_reg(adap, CIM_DEBUGSTS_A); + req = POLADBGWRPTR_G(val); + rsp = PILADBGWRPTR_G(val); + if (pif_req_wrptr) + *pif_req_wrptr = req; + if (pif_rsp_wrptr) + *pif_rsp_wrptr = rsp; + + for (i = 0; i < CIM_PIFLA_SIZE; i++) { + for (j = 0; j < 6; j++) { + t4_write_reg(adap, CIM_DEBUGCFG_A, POLADBGRDPTR_V(req) | + PILADBGRDPTR_V(rsp)); + *pif_req++ = t4_read_reg(adap, CIM_PO_LA_DEBUGDATA_A); + *pif_rsp++ = t4_read_reg(adap, CIM_PI_LA_DEBUGDATA_A); + req++; + rsp++; + } + req = (req + 2) & POLADBGRDPTR_M; + rsp = (rsp + 2) & PILADBGRDPTR_M; + } + t4_write_reg(adap, CIM_DEBUGCFG_A, cfg); +} + +void t4_cim_read_ma_la(struct adapter *adap, u32 *ma_req, u32 *ma_rsp) +{ + u32 cfg; + int i, j, idx; + + cfg = t4_read_reg(adap, CIM_DEBUGCFG_A); + if (cfg & LADBGEN_F) + t4_write_reg(adap, CIM_DEBUGCFG_A, cfg ^ LADBGEN_F); + + for (i = 0; i < CIM_MALA_SIZE; i++) { + for (j = 0; j < 5; j++) { + idx = 8 * i + j; + t4_write_reg(adap, CIM_DEBUGCFG_A, POLADBGRDPTR_V(idx) | + PILADBGRDPTR_V(idx)); + *ma_req++ = t4_read_reg(adap, CIM_PO_LA_MADEBUGDATA_A); + *ma_rsp++ = t4_read_reg(adap, CIM_PI_LA_MADEBUGDATA_A); + } + } + t4_write_reg(adap, CIM_DEBUGCFG_A, cfg); +} + +void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf) +{ + unsigned int i, j; + + for (i = 0; i < 8; i++) { + u32 *p = la_buf + i; + + t4_write_reg(adap, ULP_RX_LA_CTL_A, i); + j = t4_read_reg(adap, ULP_RX_LA_WRPTR_A); + t4_write_reg(adap, ULP_RX_LA_RDPTR_A, j); + for (j = 0; j < ULPRX_LA_SIZE; j++, p += 8) + *p = t4_read_reg(adap, ULP_RX_LA_RDDATA_A); + } +} + +/* The ADVERT_MASK is used to mask out all of the Advertised Firmware Port + * Capabilities which we control with separate controls -- see, for instance, + * Pause Frames and Forward Error Correction. In order to determine what the + * full set of Advertised Port Capabilities are, the base Advertised Port + * Capabilities (masked by ADVERT_MASK) must be combined with the Advertised + * Port Capabilities associated with those other controls. See + * t4_link_acaps() for how this is done. + */ +#define ADVERT_MASK (FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_M) | \ + FW_PORT_CAP32_ANEG) + +/** + * fwcaps16_to_caps32 - convert 16-bit Port Capabilities to 32-bits + * @caps16: a 16-bit Port Capabilities value + * + * Returns the equivalent 32-bit Port Capabilities value. + */ +static fw_port_cap32_t fwcaps16_to_caps32(fw_port_cap16_t caps16) +{ + fw_port_cap32_t caps32 = 0; + + #define CAP16_TO_CAP32(__cap) \ + do { \ + if (caps16 & FW_PORT_CAP_##__cap) \ + caps32 |= FW_PORT_CAP32_##__cap; \ + } while (0) + + CAP16_TO_CAP32(SPEED_100M); + CAP16_TO_CAP32(SPEED_1G); + CAP16_TO_CAP32(SPEED_25G); + CAP16_TO_CAP32(SPEED_10G); + CAP16_TO_CAP32(SPEED_40G); + CAP16_TO_CAP32(SPEED_100G); + CAP16_TO_CAP32(FC_RX); + CAP16_TO_CAP32(FC_TX); + CAP16_TO_CAP32(ANEG); + CAP16_TO_CAP32(FORCE_PAUSE); + CAP16_TO_CAP32(MDIAUTO); + CAP16_TO_CAP32(MDISTRAIGHT); + CAP16_TO_CAP32(FEC_RS); + CAP16_TO_CAP32(FEC_BASER_RS); + CAP16_TO_CAP32(802_3_PAUSE); + CAP16_TO_CAP32(802_3_ASM_DIR); + + #undef CAP16_TO_CAP32 + + return caps32; +} + +/** + * fwcaps32_to_caps16 - convert 32-bit Port Capabilities to 16-bits + * @caps32: a 32-bit Port Capabilities value + * + * Returns the equivalent 16-bit Port Capabilities value. Note that + * not all 32-bit Port Capabilities can be represented in the 16-bit + * Port Capabilities and some fields/values may not make it. + */ +static fw_port_cap16_t fwcaps32_to_caps16(fw_port_cap32_t caps32) +{ + fw_port_cap16_t caps16 = 0; + + #define CAP32_TO_CAP16(__cap) \ + do { \ + if (caps32 & FW_PORT_CAP32_##__cap) \ + caps16 |= FW_PORT_CAP_##__cap; \ + } while (0) + + CAP32_TO_CAP16(SPEED_100M); + CAP32_TO_CAP16(SPEED_1G); + CAP32_TO_CAP16(SPEED_10G); + CAP32_TO_CAP16(SPEED_25G); + CAP32_TO_CAP16(SPEED_40G); + CAP32_TO_CAP16(SPEED_100G); + CAP32_TO_CAP16(FC_RX); + CAP32_TO_CAP16(FC_TX); + CAP32_TO_CAP16(802_3_PAUSE); + CAP32_TO_CAP16(802_3_ASM_DIR); + CAP32_TO_CAP16(ANEG); + CAP32_TO_CAP16(FORCE_PAUSE); + CAP32_TO_CAP16(MDIAUTO); + CAP32_TO_CAP16(MDISTRAIGHT); + CAP32_TO_CAP16(FEC_RS); + CAP32_TO_CAP16(FEC_BASER_RS); + + #undef CAP32_TO_CAP16 + + return caps16; +} + +/* Translate Firmware Port Capabilities Pause specification to Common Code */ +static inline enum cc_pause fwcap_to_cc_pause(fw_port_cap32_t fw_pause) +{ + enum cc_pause cc_pause = 0; + + if (fw_pause & FW_PORT_CAP32_FC_RX) + cc_pause |= PAUSE_RX; + if (fw_pause & FW_PORT_CAP32_FC_TX) + cc_pause |= PAUSE_TX; + + return cc_pause; +} + +/* Translate Common Code Pause specification into Firmware Port Capabilities */ +static inline fw_port_cap32_t cc_to_fwcap_pause(enum cc_pause cc_pause) +{ + /* Translate orthogonal RX/TX Pause Controls for L1 Configure + * commands, etc. + */ + fw_port_cap32_t fw_pause = 0; + + if (cc_pause & PAUSE_RX) + fw_pause |= FW_PORT_CAP32_FC_RX; + if (cc_pause & PAUSE_TX) + fw_pause |= FW_PORT_CAP32_FC_TX; + if (!(cc_pause & PAUSE_AUTONEG)) + fw_pause |= FW_PORT_CAP32_FORCE_PAUSE; + + /* Translate orthogonal Pause controls into IEEE 802.3 Pause, + * Asymmetrical Pause for use in reporting to upper layer OS code, etc. + * Note that these bits are ignored in L1 Configure commands. + */ + if (cc_pause & PAUSE_RX) { + if (cc_pause & PAUSE_TX) + fw_pause |= FW_PORT_CAP32_802_3_PAUSE; + else + fw_pause |= FW_PORT_CAP32_802_3_ASM_DIR | + FW_PORT_CAP32_802_3_PAUSE; + } else if (cc_pause & PAUSE_TX) { + fw_pause |= FW_PORT_CAP32_802_3_ASM_DIR; + } + + return fw_pause; +} + +/* Translate Firmware Forward Error Correction specification to Common Code */ +static inline enum cc_fec fwcap_to_cc_fec(fw_port_cap32_t fw_fec) +{ + enum cc_fec cc_fec = 0; + + if (fw_fec & FW_PORT_CAP32_FEC_RS) + cc_fec |= FEC_RS; + if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS) + cc_fec |= FEC_BASER_RS; + + return cc_fec; +} + +/* Translate Common Code Forward Error Correction specification to Firmware */ +static inline fw_port_cap32_t cc_to_fwcap_fec(enum cc_fec cc_fec) +{ + fw_port_cap32_t fw_fec = 0; + + if (cc_fec & FEC_RS) + fw_fec |= FW_PORT_CAP32_FEC_RS; + if (cc_fec & FEC_BASER_RS) + fw_fec |= FW_PORT_CAP32_FEC_BASER_RS; + + return fw_fec; +} + +/** + * t4_link_acaps - compute Link Advertised Port Capabilities + * @adapter: the adapter + * @port: the Port ID + * @lc: the Port's Link Configuration + * + * Synthesize the Advertised Port Capabilities we'll be using based on + * the base Advertised Port Capabilities (which have been filtered by + * ADVERT_MASK) plus the individual controls for things like Pause + * Frames, Forward Error Correction, MDI, etc. + */ +fw_port_cap32_t t4_link_acaps(struct adapter *adapter, unsigned int port, + struct link_config *lc) +{ + fw_port_cap32_t fw_fc, fw_fec, acaps; + unsigned int fw_mdi; + char cc_fec; + + fw_mdi = (FW_PORT_CAP32_MDI_V(FW_PORT_CAP32_MDI_AUTO) & lc->pcaps); + + /* Convert driver coding of Pause Frame Flow Control settings into the + * Firmware's API. + */ + fw_fc = cc_to_fwcap_pause(lc->requested_fc); + + /* Convert Common Code Forward Error Control settings into the + * Firmware's API. If the current Requested FEC has "Automatic" + * (IEEE 802.3) specified, then we use whatever the Firmware + * sent us as part of its IEEE 802.3-based interpretation of + * the Transceiver Module EPROM FEC parameters. Otherwise we + * use whatever is in the current Requested FEC settings. + */ + if (lc->requested_fec & FEC_AUTO) + cc_fec = fwcap_to_cc_fec(lc->def_acaps); + else + cc_fec = lc->requested_fec; + fw_fec = cc_to_fwcap_fec(cc_fec); + + /* Figure out what our Requested Port Capabilities are going to be. + * Note parallel structure in t4_handle_get_port_info() and + * init_link_config(). + */ + if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) { + acaps = lc->acaps | fw_fc | fw_fec; + lc->fc = lc->requested_fc & ~PAUSE_AUTONEG; + lc->fec = cc_fec; + } else if (lc->autoneg == AUTONEG_DISABLE) { + acaps = lc->speed_caps | fw_fc | fw_fec | fw_mdi; + lc->fc = lc->requested_fc & ~PAUSE_AUTONEG; + lc->fec = cc_fec; + } else { + acaps = lc->acaps | fw_fc | fw_fec | fw_mdi; + } + + /* Some Requested Port Capabilities are trivially wrong if they exceed + * the Physical Port Capabilities. We can check that here and provide + * moderately useful feedback in the system log. + * + * Note that older Firmware doesn't have FW_PORT_CAP32_FORCE_PAUSE, so + * we need to exclude this from this check in order to maintain + * compatibility ... + */ + if ((acaps & ~lc->pcaps) & ~FW_PORT_CAP32_FORCE_PAUSE) { + dev_err(adapter->pdev_dev, "Requested Port Capabilities %#x exceed Physical Port Capabilities %#x\n", + acaps, lc->pcaps); + return -EINVAL; + } + + return acaps; +} + +/** + * t4_link_l1cfg_core - apply link configuration to MAC/PHY + * @adapter: the adapter + * @mbox: the Firmware Mailbox to use + * @port: the Port ID + * @lc: the Port's Link Configuration + * @sleep_ok: if true we may sleep while awaiting command completion + * @timeout: time to wait for command to finish before timing out + * (negative implies @sleep_ok=false) + * + * Set up a port's MAC and PHY according to a desired link configuration. + * - If the PHY can auto-negotiate first decide what to advertise, then + * enable/disable auto-negotiation as desired, and reset. + * - If the PHY does not auto-negotiate just reset it. + * - If auto-negotiation is off set the MAC to the proper speed/duplex/FC, + * otherwise do it later based on the outcome of auto-negotiation. + */ +int t4_link_l1cfg_core(struct adapter *adapter, unsigned int mbox, + unsigned int port, struct link_config *lc, + u8 sleep_ok, int timeout) +{ + unsigned int fw_caps = adapter->params.fw_caps_support; + struct fw_port_cmd cmd; + fw_port_cap32_t rcap; + int ret; + + if (!(lc->pcaps & FW_PORT_CAP32_ANEG) && + lc->autoneg == AUTONEG_ENABLE) { + return -EINVAL; + } + + /* Compute our Requested Port Capabilities and send that on to the + * Firmware. + */ + rcap = t4_link_acaps(adapter, port, lc); + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) | + FW_CMD_REQUEST_F | FW_CMD_EXEC_F | + FW_PORT_CMD_PORTID_V(port)); + cmd.action_to_len16 = + cpu_to_be32(FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16 + ? FW_PORT_ACTION_L1_CFG + : FW_PORT_ACTION_L1_CFG32) | + FW_LEN16(cmd)); + if (fw_caps == FW_CAPS16) + cmd.u.l1cfg.rcap = cpu_to_be32(fwcaps32_to_caps16(rcap)); + else + cmd.u.l1cfg32.rcap32 = cpu_to_be32(rcap); + + ret = t4_wr_mbox_meat_timeout(adapter, mbox, &cmd, sizeof(cmd), NULL, + sleep_ok, timeout); + + /* Unfortunately, even if the Requested Port Capabilities "fit" within + * the Physical Port Capabilities, some combinations of features may + * still not be legal. For example, 40Gb/s and Reed-Solomon Forward + * Error Correction. So if the Firmware rejects the L1 Configure + * request, flag that here. + */ + if (ret) { + dev_err(adapter->pdev_dev, + "Requested Port Capabilities %#x rejected, error %d\n", + rcap, -ret); + return ret; + } + return 0; +} + +/** + * t4_restart_aneg - restart autonegotiation + * @adap: the adapter + * @mbox: mbox to use for the FW command + * @port: the port id + * + * Restarts autonegotiation for the selected port. + */ +int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port) +{ + unsigned int fw_caps = adap->params.fw_caps_support; + struct fw_port_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) | + FW_CMD_REQUEST_F | FW_CMD_EXEC_F | + FW_PORT_CMD_PORTID_V(port)); + c.action_to_len16 = + cpu_to_be32(FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16 + ? FW_PORT_ACTION_L1_CFG + : FW_PORT_ACTION_L1_CFG32) | + FW_LEN16(c)); + if (fw_caps == FW_CAPS16) + c.u.l1cfg.rcap = cpu_to_be32(FW_PORT_CAP_ANEG); + else + c.u.l1cfg32.rcap32 = cpu_to_be32(FW_PORT_CAP32_ANEG); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +typedef void (*int_handler_t)(struct adapter *adap); + +struct intr_info { + unsigned int mask; /* bits to check in interrupt status */ + const char *msg; /* message to print or NULL */ + short stat_idx; /* stat counter to increment or -1 */ + unsigned short fatal; /* whether the condition reported is fatal */ + int_handler_t int_handler; /* platform-specific int handler */ +}; + +/** + * t4_handle_intr_status - table driven interrupt handler + * @adapter: the adapter that generated the interrupt + * @reg: the interrupt status register to process + * @acts: table of interrupt actions + * + * A table driven interrupt handler that applies a set of masks to an + * interrupt status word and performs the corresponding actions if the + * interrupts described by the mask have occurred. The actions include + * optionally emitting a warning or alert message. The table is terminated + * by an entry specifying mask 0. Returns the number of fatal interrupt + * conditions. + */ +static int t4_handle_intr_status(struct adapter *adapter, unsigned int reg, + const struct intr_info *acts) +{ + int fatal = 0; + unsigned int mask = 0; + unsigned int status = t4_read_reg(adapter, reg); + + for ( ; acts->mask; ++acts) { + if (!(status & acts->mask)) + continue; + if (acts->fatal) { + fatal++; + dev_alert(adapter->pdev_dev, "%s (0x%x)\n", acts->msg, + status & acts->mask); + } else if (acts->msg && printk_ratelimit()) + dev_warn(adapter->pdev_dev, "%s (0x%x)\n", acts->msg, + status & acts->mask); + if (acts->int_handler) + acts->int_handler(adapter); + mask |= acts->mask; + } + status &= mask; + if (status) /* clear processed interrupts */ + t4_write_reg(adapter, reg, status); + return fatal; +} + +/* + * Interrupt handler for the PCIE module. + */ +static void pcie_intr_handler(struct adapter *adapter) +{ + static const struct intr_info sysbus_intr_info[] = { + { RNPP_F, "RXNP array parity error", -1, 1 }, + { RPCP_F, "RXPC array parity error", -1, 1 }, + { RCIP_F, "RXCIF array parity error", -1, 1 }, + { RCCP_F, "Rx completions control array parity error", -1, 1 }, + { RFTP_F, "RXFT array parity error", -1, 1 }, + { 0 } + }; + static const struct intr_info pcie_port_intr_info[] = { + { TPCP_F, "TXPC array parity error", -1, 1 }, + { TNPP_F, "TXNP array parity error", -1, 1 }, + { TFTP_F, "TXFT array parity error", -1, 1 }, + { TCAP_F, "TXCA array parity error", -1, 1 }, + { TCIP_F, "TXCIF array parity error", -1, 1 }, + { RCAP_F, "RXCA array parity error", -1, 1 }, + { OTDD_F, "outbound request TLP discarded", -1, 1 }, + { RDPE_F, "Rx data parity error", -1, 1 }, + { TDUE_F, "Tx uncorrectable data error", -1, 1 }, + { 0 } + }; + static const struct intr_info pcie_intr_info[] = { + { MSIADDRLPERR_F, "MSI AddrL parity error", -1, 1 }, + { MSIADDRHPERR_F, "MSI AddrH parity error", -1, 1 }, + { MSIDATAPERR_F, "MSI data parity error", -1, 1 }, + { MSIXADDRLPERR_F, "MSI-X AddrL parity error", -1, 1 }, + { MSIXADDRHPERR_F, "MSI-X AddrH parity error", -1, 1 }, + { MSIXDATAPERR_F, "MSI-X data parity error", -1, 1 }, + { MSIXDIPERR_F, "MSI-X DI parity error", -1, 1 }, + { PIOCPLPERR_F, "PCI PIO completion FIFO parity error", -1, 1 }, + { PIOREQPERR_F, "PCI PIO request FIFO parity error", -1, 1 }, + { TARTAGPERR_F, "PCI PCI target tag FIFO parity error", -1, 1 }, + { CCNTPERR_F, "PCI CMD channel count parity error", -1, 1 }, + { CREQPERR_F, "PCI CMD channel request parity error", -1, 1 }, + { CRSPPERR_F, "PCI CMD channel response parity error", -1, 1 }, + { DCNTPERR_F, "PCI DMA channel count parity error", -1, 1 }, + { DREQPERR_F, "PCI DMA channel request parity error", -1, 1 }, + { DRSPPERR_F, "PCI DMA channel response parity error", -1, 1 }, + { HCNTPERR_F, "PCI HMA channel count parity error", -1, 1 }, + { HREQPERR_F, "PCI HMA channel request parity error", -1, 1 }, + { HRSPPERR_F, "PCI HMA channel response parity error", -1, 1 }, + { CFGSNPPERR_F, "PCI config snoop FIFO parity error", -1, 1 }, + { FIDPERR_F, "PCI FID parity error", -1, 1 }, + { INTXCLRPERR_F, "PCI INTx clear parity error", -1, 1 }, + { MATAGPERR_F, "PCI MA tag parity error", -1, 1 }, + { PIOTAGPERR_F, "PCI PIO tag parity error", -1, 1 }, + { RXCPLPERR_F, "PCI Rx completion parity error", -1, 1 }, + { RXWRPERR_F, "PCI Rx write parity error", -1, 1 }, + { RPLPERR_F, "PCI replay buffer parity error", -1, 1 }, + { PCIESINT_F, "PCI core secondary fault", -1, 1 }, + { PCIEPINT_F, "PCI core primary fault", -1, 1 }, + { UNXSPLCPLERR_F, "PCI unexpected split completion error", + -1, 0 }, + { 0 } + }; + + static struct intr_info t5_pcie_intr_info[] = { + { MSTGRPPERR_F, "Master Response Read Queue parity error", + -1, 1 }, + { MSTTIMEOUTPERR_F, "Master Timeout FIFO parity error", -1, 1 }, + { MSIXSTIPERR_F, "MSI-X STI SRAM parity error", -1, 1 }, + { MSIXADDRLPERR_F, "MSI-X AddrL parity error", -1, 1 }, + { MSIXADDRHPERR_F, "MSI-X AddrH parity error", -1, 1 }, + { MSIXDATAPERR_F, "MSI-X data parity error", -1, 1 }, + { MSIXDIPERR_F, "MSI-X DI parity error", -1, 1 }, + { PIOCPLGRPPERR_F, "PCI PIO completion Group FIFO parity error", + -1, 1 }, + { PIOREQGRPPERR_F, "PCI PIO request Group FIFO parity error", + -1, 1 }, + { TARTAGPERR_F, "PCI PCI target tag FIFO parity error", -1, 1 }, + { MSTTAGQPERR_F, "PCI master tag queue parity error", -1, 1 }, + { CREQPERR_F, "PCI CMD channel request parity error", -1, 1 }, + { CRSPPERR_F, "PCI CMD channel response parity error", -1, 1 }, + { DREQWRPERR_F, "PCI DMA channel write request parity error", + -1, 1 }, + { DREQPERR_F, "PCI DMA channel request parity error", -1, 1 }, + { DRSPPERR_F, "PCI DMA channel response parity error", -1, 1 }, + { HREQWRPERR_F, "PCI HMA channel count parity error", -1, 1 }, + { HREQPERR_F, "PCI HMA channel request parity error", -1, 1 }, + { HRSPPERR_F, "PCI HMA channel response parity error", -1, 1 }, + { CFGSNPPERR_F, "PCI config snoop FIFO parity error", -1, 1 }, + { FIDPERR_F, "PCI FID parity error", -1, 1 }, + { VFIDPERR_F, "PCI INTx clear parity error", -1, 1 }, + { MAGRPPERR_F, "PCI MA group FIFO parity error", -1, 1 }, + { PIOTAGPERR_F, "PCI PIO tag parity error", -1, 1 }, + { IPRXHDRGRPPERR_F, "PCI IP Rx header group parity error", + -1, 1 }, + { IPRXDATAGRPPERR_F, "PCI IP Rx data group parity error", + -1, 1 }, + { RPLPERR_F, "PCI IP replay buffer parity error", -1, 1 }, + { IPSOTPERR_F, "PCI IP SOT buffer parity error", -1, 1 }, + { TRGT1GRPPERR_F, "PCI TRGT1 group FIFOs parity error", -1, 1 }, + { READRSPERR_F, "Outbound read error", -1, 0 }, + { 0 } + }; + + int fat; + + if (is_t4(adapter->params.chip)) + fat = t4_handle_intr_status(adapter, + PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS_A, + sysbus_intr_info) + + t4_handle_intr_status(adapter, + PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS_A, + pcie_port_intr_info) + + t4_handle_intr_status(adapter, PCIE_INT_CAUSE_A, + pcie_intr_info); + else + fat = t4_handle_intr_status(adapter, PCIE_INT_CAUSE_A, + t5_pcie_intr_info); + + if (fat) + t4_fatal_err(adapter); +} + +/* + * TP interrupt handler. + */ +static void tp_intr_handler(struct adapter *adapter) +{ + static const struct intr_info tp_intr_info[] = { + { 0x3fffffff, "TP parity error", -1, 1 }, + { FLMTXFLSTEMPTY_F, "TP out of Tx pages", -1, 1 }, + { 0 } + }; + + if (t4_handle_intr_status(adapter, TP_INT_CAUSE_A, tp_intr_info)) + t4_fatal_err(adapter); +} + +/* + * SGE interrupt handler. + */ +static void sge_intr_handler(struct adapter *adapter) +{ + u32 v = 0, perr; + u32 err; + + static const struct intr_info sge_intr_info[] = { + { ERR_CPL_EXCEED_IQE_SIZE_F, + "SGE received CPL exceeding IQE size", -1, 1 }, + { ERR_INVALID_CIDX_INC_F, + "SGE GTS CIDX increment too large", -1, 0 }, + { ERR_CPL_OPCODE_0_F, "SGE received 0-length CPL", -1, 0 }, + { DBFIFO_LP_INT_F, NULL, -1, 0, t4_db_full }, + { ERR_DATA_CPL_ON_HIGH_QID1_F | ERR_DATA_CPL_ON_HIGH_QID0_F, + "SGE IQID > 1023 received CPL for FL", -1, 0 }, + { ERR_BAD_DB_PIDX3_F, "SGE DBP 3 pidx increment too large", -1, + 0 }, + { ERR_BAD_DB_PIDX2_F, "SGE DBP 2 pidx increment too large", -1, + 0 }, + { ERR_BAD_DB_PIDX1_F, "SGE DBP 1 pidx increment too large", -1, + 0 }, + { ERR_BAD_DB_PIDX0_F, "SGE DBP 0 pidx increment too large", -1, + 0 }, + { ERR_ING_CTXT_PRIO_F, + "SGE too many priority ingress contexts", -1, 0 }, + { INGRESS_SIZE_ERR_F, "SGE illegal ingress QID", -1, 0 }, + { EGRESS_SIZE_ERR_F, "SGE illegal egress QID", -1, 0 }, + { 0 } + }; + + static struct intr_info t4t5_sge_intr_info[] = { + { ERR_DROPPED_DB_F, NULL, -1, 0, t4_db_dropped }, + { DBFIFO_HP_INT_F, NULL, -1, 0, t4_db_full }, + { ERR_EGR_CTXT_PRIO_F, + "SGE too many priority egress contexts", -1, 0 }, + { 0 } + }; + + perr = t4_read_reg(adapter, SGE_INT_CAUSE1_A); + if (perr) { + v |= perr; + dev_alert(adapter->pdev_dev, "SGE Cause1 Parity Error %#x\n", + perr); + } + + perr = t4_read_reg(adapter, SGE_INT_CAUSE2_A); + if (perr) { + v |= perr; + dev_alert(adapter->pdev_dev, "SGE Cause2 Parity Error %#x\n", + perr); + } + + if (CHELSIO_CHIP_VERSION(adapter->params.chip) >= CHELSIO_T5) { + perr = t4_read_reg(adapter, SGE_INT_CAUSE5_A); + /* Parity error (CRC) for err_T_RxCRC is trivial, ignore it */ + perr &= ~ERR_T_RXCRC_F; + if (perr) { + v |= perr; + dev_alert(adapter->pdev_dev, + "SGE Cause5 Parity Error %#x\n", perr); + } + } + + v |= t4_handle_intr_status(adapter, SGE_INT_CAUSE3_A, sge_intr_info); + if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) + v |= t4_handle_intr_status(adapter, SGE_INT_CAUSE3_A, + t4t5_sge_intr_info); + + err = t4_read_reg(adapter, SGE_ERROR_STATS_A); + if (err & ERROR_QID_VALID_F) { + dev_err(adapter->pdev_dev, "SGE error for queue %u\n", + ERROR_QID_G(err)); + if (err & UNCAPTURED_ERROR_F) + dev_err(adapter->pdev_dev, + "SGE UNCAPTURED_ERROR set (clearing)\n"); + t4_write_reg(adapter, SGE_ERROR_STATS_A, ERROR_QID_VALID_F | + UNCAPTURED_ERROR_F); + } + + if (v != 0) + t4_fatal_err(adapter); +} + +#define CIM_OBQ_INTR (OBQULP0PARERR_F | OBQULP1PARERR_F | OBQULP2PARERR_F |\ + OBQULP3PARERR_F | OBQSGEPARERR_F | OBQNCSIPARERR_F) +#define CIM_IBQ_INTR (IBQTP0PARERR_F | IBQTP1PARERR_F | IBQULPPARERR_F |\ + IBQSGEHIPARERR_F | IBQSGELOPARERR_F | IBQNCSIPARERR_F) + +/* + * CIM interrupt handler. + */ +static void cim_intr_handler(struct adapter *adapter) +{ + static const struct intr_info cim_intr_info[] = { + { PREFDROPINT_F, "CIM control register prefetch drop", -1, 1 }, + { CIM_OBQ_INTR, "CIM OBQ parity error", -1, 1 }, + { CIM_IBQ_INTR, "CIM IBQ parity error", -1, 1 }, + { MBUPPARERR_F, "CIM mailbox uP parity error", -1, 1 }, + { MBHOSTPARERR_F, "CIM mailbox host parity error", -1, 1 }, + { TIEQINPARERRINT_F, "CIM TIEQ outgoing parity error", -1, 1 }, + { TIEQOUTPARERRINT_F, "CIM TIEQ incoming parity error", -1, 1 }, + { TIMER0INT_F, "CIM TIMER0 interrupt", -1, 1 }, + { 0 } + }; + static const struct intr_info cim_upintr_info[] = { + { RSVDSPACEINT_F, "CIM reserved space access", -1, 1 }, + { ILLTRANSINT_F, "CIM illegal transaction", -1, 1 }, + { ILLWRINT_F, "CIM illegal write", -1, 1 }, + { ILLRDINT_F, "CIM illegal read", -1, 1 }, + { ILLRDBEINT_F, "CIM illegal read BE", -1, 1 }, + { ILLWRBEINT_F, "CIM illegal write BE", -1, 1 }, + { SGLRDBOOTINT_F, "CIM single read from boot space", -1, 1 }, + { SGLWRBOOTINT_F, "CIM single write to boot space", -1, 1 }, + { BLKWRBOOTINT_F, "CIM block write to boot space", -1, 1 }, + { SGLRDFLASHINT_F, "CIM single read from flash space", -1, 1 }, + { SGLWRFLASHINT_F, "CIM single write to flash space", -1, 1 }, + { BLKWRFLASHINT_F, "CIM block write to flash space", -1, 1 }, + { SGLRDEEPROMINT_F, "CIM single EEPROM read", -1, 1 }, + { SGLWREEPROMINT_F, "CIM single EEPROM write", -1, 1 }, + { BLKRDEEPROMINT_F, "CIM block EEPROM read", -1, 1 }, + { BLKWREEPROMINT_F, "CIM block EEPROM write", -1, 1 }, + { SGLRDCTLINT_F, "CIM single read from CTL space", -1, 1 }, + { SGLWRCTLINT_F, "CIM single write to CTL space", -1, 1 }, + { BLKRDCTLINT_F, "CIM block read from CTL space", -1, 1 }, + { BLKWRCTLINT_F, "CIM block write to CTL space", -1, 1 }, + { SGLRDPLINT_F, "CIM single read from PL space", -1, 1 }, + { SGLWRPLINT_F, "CIM single write to PL space", -1, 1 }, + { BLKRDPLINT_F, "CIM block read from PL space", -1, 1 }, + { BLKWRPLINT_F, "CIM block write to PL space", -1, 1 }, + { REQOVRLOOKUPINT_F, "CIM request FIFO overwrite", -1, 1 }, + { RSPOVRLOOKUPINT_F, "CIM response FIFO overwrite", -1, 1 }, + { TIMEOUTINT_F, "CIM PIF timeout", -1, 1 }, + { TIMEOUTMAINT_F, "CIM PIF MA timeout", -1, 1 }, + { 0 } + }; + + u32 val, fw_err; + int fat; + + fw_err = t4_read_reg(adapter, PCIE_FW_A); + if (fw_err & PCIE_FW_ERR_F) + t4_report_fw_error(adapter); + + /* When the Firmware detects an internal error which normally + * wouldn't raise a Host Interrupt, it forces a CIM Timer0 interrupt + * in order to make sure the Host sees the Firmware Crash. So + * if we have a Timer0 interrupt and don't see a Firmware Crash, + * ignore the Timer0 interrupt. + */ + + val = t4_read_reg(adapter, CIM_HOST_INT_CAUSE_A); + if (val & TIMER0INT_F) + if (!(fw_err & PCIE_FW_ERR_F) || + (PCIE_FW_EVAL_G(fw_err) != PCIE_FW_EVAL_CRASH)) + t4_write_reg(adapter, CIM_HOST_INT_CAUSE_A, + TIMER0INT_F); + + fat = t4_handle_intr_status(adapter, CIM_HOST_INT_CAUSE_A, + cim_intr_info) + + t4_handle_intr_status(adapter, CIM_HOST_UPACC_INT_CAUSE_A, + cim_upintr_info); + if (fat) + t4_fatal_err(adapter); +} + +/* + * ULP RX interrupt handler. + */ +static void ulprx_intr_handler(struct adapter *adapter) +{ + static const struct intr_info ulprx_intr_info[] = { + { 0x1800000, "ULPRX context error", -1, 1 }, + { 0x7fffff, "ULPRX parity error", -1, 1 }, + { 0 } + }; + + if (t4_handle_intr_status(adapter, ULP_RX_INT_CAUSE_A, ulprx_intr_info)) + t4_fatal_err(adapter); +} + +/* + * ULP TX interrupt handler. + */ +static void ulptx_intr_handler(struct adapter *adapter) +{ + static const struct intr_info ulptx_intr_info[] = { + { PBL_BOUND_ERR_CH3_F, "ULPTX channel 3 PBL out of bounds", -1, + 0 }, + { PBL_BOUND_ERR_CH2_F, "ULPTX channel 2 PBL out of bounds", -1, + 0 }, + { PBL_BOUND_ERR_CH1_F, "ULPTX channel 1 PBL out of bounds", -1, + 0 }, + { PBL_BOUND_ERR_CH0_F, "ULPTX channel 0 PBL out of bounds", -1, + 0 }, + { 0xfffffff, "ULPTX parity error", -1, 1 }, + { 0 } + }; + + if (t4_handle_intr_status(adapter, ULP_TX_INT_CAUSE_A, ulptx_intr_info)) + t4_fatal_err(adapter); +} + +/* + * PM TX interrupt handler. + */ +static void pmtx_intr_handler(struct adapter *adapter) +{ + static const struct intr_info pmtx_intr_info[] = { + { PCMD_LEN_OVFL0_F, "PMTX channel 0 pcmd too large", -1, 1 }, + { PCMD_LEN_OVFL1_F, "PMTX channel 1 pcmd too large", -1, 1 }, + { PCMD_LEN_OVFL2_F, "PMTX channel 2 pcmd too large", -1, 1 }, + { ZERO_C_CMD_ERROR_F, "PMTX 0-length pcmd", -1, 1 }, + { PMTX_FRAMING_ERROR_F, "PMTX framing error", -1, 1 }, + { OESPI_PAR_ERROR_F, "PMTX oespi parity error", -1, 1 }, + { DB_OPTIONS_PAR_ERROR_F, "PMTX db_options parity error", + -1, 1 }, + { ICSPI_PAR_ERROR_F, "PMTX icspi parity error", -1, 1 }, + { PMTX_C_PCMD_PAR_ERROR_F, "PMTX c_pcmd parity error", -1, 1}, + { 0 } + }; + + if (t4_handle_intr_status(adapter, PM_TX_INT_CAUSE_A, pmtx_intr_info)) + t4_fatal_err(adapter); +} + +/* + * PM RX interrupt handler. + */ +static void pmrx_intr_handler(struct adapter *adapter) +{ + static const struct intr_info pmrx_intr_info[] = { + { ZERO_E_CMD_ERROR_F, "PMRX 0-length pcmd", -1, 1 }, + { PMRX_FRAMING_ERROR_F, "PMRX framing error", -1, 1 }, + { OCSPI_PAR_ERROR_F, "PMRX ocspi parity error", -1, 1 }, + { DB_OPTIONS_PAR_ERROR_F, "PMRX db_options parity error", + -1, 1 }, + { IESPI_PAR_ERROR_F, "PMRX iespi parity error", -1, 1 }, + { PMRX_E_PCMD_PAR_ERROR_F, "PMRX e_pcmd parity error", -1, 1}, + { 0 } + }; + + if (t4_handle_intr_status(adapter, PM_RX_INT_CAUSE_A, pmrx_intr_info)) + t4_fatal_err(adapter); +} + +/* + * CPL switch interrupt handler. + */ +static void cplsw_intr_handler(struct adapter *adapter) +{ + static const struct intr_info cplsw_intr_info[] = { + { CIM_OP_MAP_PERR_F, "CPLSW CIM op_map parity error", -1, 1 }, + { CIM_OVFL_ERROR_F, "CPLSW CIM overflow", -1, 1 }, + { TP_FRAMING_ERROR_F, "CPLSW TP framing error", -1, 1 }, + { SGE_FRAMING_ERROR_F, "CPLSW SGE framing error", -1, 1 }, + { CIM_FRAMING_ERROR_F, "CPLSW CIM framing error", -1, 1 }, + { ZERO_SWITCH_ERROR_F, "CPLSW no-switch error", -1, 1 }, + { 0 } + }; + + if (t4_handle_intr_status(adapter, CPL_INTR_CAUSE_A, cplsw_intr_info)) + t4_fatal_err(adapter); +} + +/* + * LE interrupt handler. + */ +static void le_intr_handler(struct adapter *adap) +{ + enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip); + static const struct intr_info le_intr_info[] = { + { LIPMISS_F, "LE LIP miss", -1, 0 }, + { LIP0_F, "LE 0 LIP error", -1, 0 }, + { PARITYERR_F, "LE parity error", -1, 1 }, + { UNKNOWNCMD_F, "LE unknown command", -1, 1 }, + { REQQPARERR_F, "LE request queue parity error", -1, 1 }, + { 0 } + }; + + static struct intr_info t6_le_intr_info[] = { + { T6_LIPMISS_F, "LE LIP miss", -1, 0 }, + { T6_LIP0_F, "LE 0 LIP error", -1, 0 }, + { CMDTIDERR_F, "LE cmd tid error", -1, 1 }, + { TCAMINTPERR_F, "LE parity error", -1, 1 }, + { T6_UNKNOWNCMD_F, "LE unknown command", -1, 1 }, + { SSRAMINTPERR_F, "LE request queue parity error", -1, 1 }, + { HASHTBLMEMCRCERR_F, "LE hash table mem crc error", -1, 0 }, + { 0 } + }; + + if (t4_handle_intr_status(adap, LE_DB_INT_CAUSE_A, + (chip <= CHELSIO_T5) ? + le_intr_info : t6_le_intr_info)) + t4_fatal_err(adap); +} + +/* + * MPS interrupt handler. + */ +static void mps_intr_handler(struct adapter *adapter) +{ + static const struct intr_info mps_rx_intr_info[] = { + { 0xffffff, "MPS Rx parity error", -1, 1 }, + { 0 } + }; + static const struct intr_info mps_tx_intr_info[] = { + { TPFIFO_V(TPFIFO_M), "MPS Tx TP FIFO parity error", -1, 1 }, + { NCSIFIFO_F, "MPS Tx NC-SI FIFO parity error", -1, 1 }, + { TXDATAFIFO_V(TXDATAFIFO_M), "MPS Tx data FIFO parity error", + -1, 1 }, + { TXDESCFIFO_V(TXDESCFIFO_M), "MPS Tx desc FIFO parity error", + -1, 1 }, + { BUBBLE_F, "MPS Tx underflow", -1, 1 }, + { SECNTERR_F, "MPS Tx SOP/EOP error", -1, 1 }, + { FRMERR_F, "MPS Tx framing error", -1, 1 }, + { 0 } + }; + static const struct intr_info t6_mps_tx_intr_info[] = { + { TPFIFO_V(TPFIFO_M), "MPS Tx TP FIFO parity error", -1, 1 }, + { NCSIFIFO_F, "MPS Tx NC-SI FIFO parity error", -1, 1 }, + { TXDATAFIFO_V(TXDATAFIFO_M), "MPS Tx data FIFO parity error", + -1, 1 }, + { TXDESCFIFO_V(TXDESCFIFO_M), "MPS Tx desc FIFO parity error", + -1, 1 }, + /* MPS Tx Bubble is normal for T6 */ + { SECNTERR_F, "MPS Tx SOP/EOP error", -1, 1 }, + { FRMERR_F, "MPS Tx framing error", -1, 1 }, + { 0 } + }; + static const struct intr_info mps_trc_intr_info[] = { + { FILTMEM_V(FILTMEM_M), "MPS TRC filter parity error", -1, 1 }, + { PKTFIFO_V(PKTFIFO_M), "MPS TRC packet FIFO parity error", + -1, 1 }, + { MISCPERR_F, "MPS TRC misc parity error", -1, 1 }, + { 0 } + }; + static const struct intr_info mps_stat_sram_intr_info[] = { + { 0x1fffff, "MPS statistics SRAM parity error", -1, 1 }, + { 0 } + }; + static const struct intr_info mps_stat_tx_intr_info[] = { + { 0xfffff, "MPS statistics Tx FIFO parity error", -1, 1 }, + { 0 } + }; + static const struct intr_info mps_stat_rx_intr_info[] = { + { 0xffffff, "MPS statistics Rx FIFO parity error", -1, 1 }, + { 0 } + }; + static const struct intr_info mps_cls_intr_info[] = { + { MATCHSRAM_F, "MPS match SRAM parity error", -1, 1 }, + { MATCHTCAM_F, "MPS match TCAM parity error", -1, 1 }, + { HASHSRAM_F, "MPS hash SRAM parity error", -1, 1 }, + { 0 } + }; + + int fat; + + fat = t4_handle_intr_status(adapter, MPS_RX_PERR_INT_CAUSE_A, + mps_rx_intr_info) + + t4_handle_intr_status(adapter, MPS_TX_INT_CAUSE_A, + is_t6(adapter->params.chip) + ? t6_mps_tx_intr_info + : mps_tx_intr_info) + + t4_handle_intr_status(adapter, MPS_TRC_INT_CAUSE_A, + mps_trc_intr_info) + + t4_handle_intr_status(adapter, MPS_STAT_PERR_INT_CAUSE_SRAM_A, + mps_stat_sram_intr_info) + + t4_handle_intr_status(adapter, MPS_STAT_PERR_INT_CAUSE_TX_FIFO_A, + mps_stat_tx_intr_info) + + t4_handle_intr_status(adapter, MPS_STAT_PERR_INT_CAUSE_RX_FIFO_A, + mps_stat_rx_intr_info) + + t4_handle_intr_status(adapter, MPS_CLS_INT_CAUSE_A, + mps_cls_intr_info); + + t4_write_reg(adapter, MPS_INT_CAUSE_A, 0); + t4_read_reg(adapter, MPS_INT_CAUSE_A); /* flush */ + if (fat) + t4_fatal_err(adapter); +} + +#define MEM_INT_MASK (PERR_INT_CAUSE_F | ECC_CE_INT_CAUSE_F | \ + ECC_UE_INT_CAUSE_F) + +/* + * EDC/MC interrupt handler. + */ +static void mem_intr_handler(struct adapter *adapter, int idx) +{ + static const char name[4][7] = { "EDC0", "EDC1", "MC/MC0", "MC1" }; + + unsigned int addr, cnt_addr, v; + + if (idx <= MEM_EDC1) { + addr = EDC_REG(EDC_INT_CAUSE_A, idx); + cnt_addr = EDC_REG(EDC_ECC_STATUS_A, idx); + } else if (idx == MEM_MC) { + if (is_t4(adapter->params.chip)) { + addr = MC_INT_CAUSE_A; + cnt_addr = MC_ECC_STATUS_A; + } else { + addr = MC_P_INT_CAUSE_A; + cnt_addr = MC_P_ECC_STATUS_A; + } + } else { + addr = MC_REG(MC_P_INT_CAUSE_A, 1); + cnt_addr = MC_REG(MC_P_ECC_STATUS_A, 1); + } + + v = t4_read_reg(adapter, addr) & MEM_INT_MASK; + if (v & PERR_INT_CAUSE_F) + dev_alert(adapter->pdev_dev, "%s FIFO parity error\n", + name[idx]); + if (v & ECC_CE_INT_CAUSE_F) { + u32 cnt = ECC_CECNT_G(t4_read_reg(adapter, cnt_addr)); + + t4_edc_err_read(adapter, idx); + + t4_write_reg(adapter, cnt_addr, ECC_CECNT_V(ECC_CECNT_M)); + if (printk_ratelimit()) + dev_warn(adapter->pdev_dev, + "%u %s correctable ECC data error%s\n", + cnt, name[idx], cnt > 1 ? "s" : ""); + } + if (v & ECC_UE_INT_CAUSE_F) + dev_alert(adapter->pdev_dev, + "%s uncorrectable ECC data error\n", name[idx]); + + t4_write_reg(adapter, addr, v); + if (v & (PERR_INT_CAUSE_F | ECC_UE_INT_CAUSE_F)) + t4_fatal_err(adapter); +} + +/* + * MA interrupt handler. + */ +static void ma_intr_handler(struct adapter *adap) +{ + u32 v, status = t4_read_reg(adap, MA_INT_CAUSE_A); + + if (status & MEM_PERR_INT_CAUSE_F) { + dev_alert(adap->pdev_dev, + "MA parity error, parity status %#x\n", + t4_read_reg(adap, MA_PARITY_ERROR_STATUS1_A)); + if (is_t5(adap->params.chip)) + dev_alert(adap->pdev_dev, + "MA parity error, parity status %#x\n", + t4_read_reg(adap, + MA_PARITY_ERROR_STATUS2_A)); + } + if (status & MEM_WRAP_INT_CAUSE_F) { + v = t4_read_reg(adap, MA_INT_WRAP_STATUS_A); + dev_alert(adap->pdev_dev, "MA address wrap-around error by " + "client %u to address %#x\n", + MEM_WRAP_CLIENT_NUM_G(v), + MEM_WRAP_ADDRESS_G(v) << 4); + } + t4_write_reg(adap, MA_INT_CAUSE_A, status); + t4_fatal_err(adap); +} + +/* + * SMB interrupt handler. + */ +static void smb_intr_handler(struct adapter *adap) +{ + static const struct intr_info smb_intr_info[] = { + { MSTTXFIFOPARINT_F, "SMB master Tx FIFO parity error", -1, 1 }, + { MSTRXFIFOPARINT_F, "SMB master Rx FIFO parity error", -1, 1 }, + { SLVFIFOPARINT_F, "SMB slave FIFO parity error", -1, 1 }, + { 0 } + }; + + if (t4_handle_intr_status(adap, SMB_INT_CAUSE_A, smb_intr_info)) + t4_fatal_err(adap); +} + +/* + * NC-SI interrupt handler. + */ +static void ncsi_intr_handler(struct adapter *adap) +{ + static const struct intr_info ncsi_intr_info[] = { + { CIM_DM_PRTY_ERR_F, "NC-SI CIM parity error", -1, 1 }, + { MPS_DM_PRTY_ERR_F, "NC-SI MPS parity error", -1, 1 }, + { TXFIFO_PRTY_ERR_F, "NC-SI Tx FIFO parity error", -1, 1 }, + { RXFIFO_PRTY_ERR_F, "NC-SI Rx FIFO parity error", -1, 1 }, + { 0 } + }; + + if (t4_handle_intr_status(adap, NCSI_INT_CAUSE_A, ncsi_intr_info)) + t4_fatal_err(adap); +} + +/* + * XGMAC interrupt handler. + */ +static void xgmac_intr_handler(struct adapter *adap, int port) +{ + u32 v, int_cause_reg; + + if (is_t4(adap->params.chip)) + int_cause_reg = PORT_REG(port, XGMAC_PORT_INT_CAUSE_A); + else + int_cause_reg = T5_PORT_REG(port, MAC_PORT_INT_CAUSE_A); + + v = t4_read_reg(adap, int_cause_reg); + + v &= TXFIFO_PRTY_ERR_F | RXFIFO_PRTY_ERR_F; + if (!v) + return; + + if (v & TXFIFO_PRTY_ERR_F) + dev_alert(adap->pdev_dev, "XGMAC %d Tx FIFO parity error\n", + port); + if (v & RXFIFO_PRTY_ERR_F) + dev_alert(adap->pdev_dev, "XGMAC %d Rx FIFO parity error\n", + port); + t4_write_reg(adap, PORT_REG(port, XGMAC_PORT_INT_CAUSE_A), v); + t4_fatal_err(adap); +} + +/* + * PL interrupt handler. + */ +static void pl_intr_handler(struct adapter *adap) +{ + static const struct intr_info pl_intr_info[] = { + { FATALPERR_F, "T4 fatal parity error", -1, 1 }, + { PERRVFID_F, "PL VFID_MAP parity error", -1, 1 }, + { 0 } + }; + + if (t4_handle_intr_status(adap, PL_PL_INT_CAUSE_A, pl_intr_info)) + t4_fatal_err(adap); +} + +#define PF_INTR_MASK (PFSW_F) +#define GLBL_INTR_MASK (CIM_F | MPS_F | PL_F | PCIE_F | MC_F | EDC0_F | \ + EDC1_F | LE_F | TP_F | MA_F | PM_TX_F | PM_RX_F | ULP_RX_F | \ + CPL_SWITCH_F | SGE_F | ULP_TX_F | SF_F) + +/** + * t4_slow_intr_handler - control path interrupt handler + * @adapter: the adapter + * + * T4 interrupt handler for non-data global interrupt events, e.g., errors. + * The designation 'slow' is because it involves register reads, while + * data interrupts typically don't involve any MMIOs. + */ +int t4_slow_intr_handler(struct adapter *adapter) +{ + /* There are rare cases where a PL_INT_CAUSE bit may end up getting + * set when the corresponding PL_INT_ENABLE bit isn't set. It's + * easiest just to mask that case here. + */ + u32 raw_cause = t4_read_reg(adapter, PL_INT_CAUSE_A); + u32 enable = t4_read_reg(adapter, PL_INT_ENABLE_A); + u32 cause = raw_cause & enable; + + if (!(cause & GLBL_INTR_MASK)) + return 0; + if (cause & CIM_F) + cim_intr_handler(adapter); + if (cause & MPS_F) + mps_intr_handler(adapter); + if (cause & NCSI_F) + ncsi_intr_handler(adapter); + if (cause & PL_F) + pl_intr_handler(adapter); + if (cause & SMB_F) + smb_intr_handler(adapter); + if (cause & XGMAC0_F) + xgmac_intr_handler(adapter, 0); + if (cause & XGMAC1_F) + xgmac_intr_handler(adapter, 1); + if (cause & XGMAC_KR0_F) + xgmac_intr_handler(adapter, 2); + if (cause & XGMAC_KR1_F) + xgmac_intr_handler(adapter, 3); + if (cause & PCIE_F) + pcie_intr_handler(adapter); + if (cause & MC_F) + mem_intr_handler(adapter, MEM_MC); + if (is_t5(adapter->params.chip) && (cause & MC1_F)) + mem_intr_handler(adapter, MEM_MC1); + if (cause & EDC0_F) + mem_intr_handler(adapter, MEM_EDC0); + if (cause & EDC1_F) + mem_intr_handler(adapter, MEM_EDC1); + if (cause & LE_F) + le_intr_handler(adapter); + if (cause & TP_F) + tp_intr_handler(adapter); + if (cause & MA_F) + ma_intr_handler(adapter); + if (cause & PM_TX_F) + pmtx_intr_handler(adapter); + if (cause & PM_RX_F) + pmrx_intr_handler(adapter); + if (cause & ULP_RX_F) + ulprx_intr_handler(adapter); + if (cause & CPL_SWITCH_F) + cplsw_intr_handler(adapter); + if (cause & SGE_F) + sge_intr_handler(adapter); + if (cause & ULP_TX_F) + ulptx_intr_handler(adapter); + + /* Clear the interrupts just processed for which we are the master. */ + t4_write_reg(adapter, PL_INT_CAUSE_A, raw_cause & GLBL_INTR_MASK); + (void)t4_read_reg(adapter, PL_INT_CAUSE_A); /* flush */ + return 1; +} + +/** + * t4_intr_enable - enable interrupts + * @adapter: the adapter whose interrupts should be enabled + * + * Enable PF-specific interrupts for the calling function and the top-level + * interrupt concentrator for global interrupts. Interrupts are already + * enabled at each module, here we just enable the roots of the interrupt + * hierarchies. + * + * Note: this function should be called only when the driver manages + * non PF-specific interrupts from the various HW modules. Only one PCI + * function at a time should be doing this. + */ +void t4_intr_enable(struct adapter *adapter) +{ + u32 val = 0; + u32 whoami = t4_read_reg(adapter, PL_WHOAMI_A); + u32 pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ? + SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami); + + if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) + val = ERR_DROPPED_DB_F | ERR_EGR_CTXT_PRIO_F | DBFIFO_HP_INT_F; + t4_write_reg(adapter, SGE_INT_ENABLE3_A, ERR_CPL_EXCEED_IQE_SIZE_F | + ERR_INVALID_CIDX_INC_F | ERR_CPL_OPCODE_0_F | + ERR_DATA_CPL_ON_HIGH_QID1_F | INGRESS_SIZE_ERR_F | + ERR_DATA_CPL_ON_HIGH_QID0_F | ERR_BAD_DB_PIDX3_F | + ERR_BAD_DB_PIDX2_F | ERR_BAD_DB_PIDX1_F | + ERR_BAD_DB_PIDX0_F | ERR_ING_CTXT_PRIO_F | + DBFIFO_LP_INT_F | EGRESS_SIZE_ERR_F | val); + t4_write_reg(adapter, MYPF_REG(PL_PF_INT_ENABLE_A), PF_INTR_MASK); + t4_set_reg_field(adapter, PL_INT_MAP0_A, 0, 1 << pf); +} + +/** + * t4_intr_disable - disable interrupts + * @adapter: the adapter whose interrupts should be disabled + * + * Disable interrupts. We only disable the top-level interrupt + * concentrators. The caller must be a PCI function managing global + * interrupts. + */ +void t4_intr_disable(struct adapter *adapter) +{ + u32 whoami, pf; + + if (pci_channel_offline(adapter->pdev)) + return; + + whoami = t4_read_reg(adapter, PL_WHOAMI_A); + pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ? + SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami); + + t4_write_reg(adapter, MYPF_REG(PL_PF_INT_ENABLE_A), 0); + t4_set_reg_field(adapter, PL_INT_MAP0_A, 1 << pf, 0); +} + +unsigned int t4_chip_rss_size(struct adapter *adap) +{ + if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) + return RSS_NENTRIES; + else + return T6_RSS_NENTRIES; +} + +/** + * t4_config_rss_range - configure a portion of the RSS mapping table + * @adapter: the adapter + * @mbox: mbox to use for the FW command + * @viid: virtual interface whose RSS subtable is to be written + * @start: start entry in the table to write + * @n: how many table entries to write + * @rspq: values for the response queue lookup table + * @nrspq: number of values in @rspq + * + * Programs the selected part of the VI's RSS mapping table with the + * provided values. If @nrspq < @n the supplied values are used repeatedly + * until the full table range is populated. + * + * The caller must ensure the values in @rspq are in the range allowed for + * @viid. + */ +int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid, + int start, int n, const u16 *rspq, unsigned int nrspq) +{ + int ret; + const u16 *rsp = rspq; + const u16 *rsp_end = rspq + nrspq; + struct fw_rss_ind_tbl_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_RSS_IND_TBL_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_RSS_IND_TBL_CMD_VIID_V(viid)); + cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); + + /* each fw_rss_ind_tbl_cmd takes up to 32 entries */ + while (n > 0) { + int nq = min(n, 32); + __be32 *qp = &cmd.iq0_to_iq2; + + cmd.niqid = cpu_to_be16(nq); + cmd.startidx = cpu_to_be16(start); + + start += nq; + n -= nq; + + while (nq > 0) { + unsigned int v; + + v = FW_RSS_IND_TBL_CMD_IQ0_V(*rsp); + if (++rsp >= rsp_end) + rsp = rspq; + v |= FW_RSS_IND_TBL_CMD_IQ1_V(*rsp); + if (++rsp >= rsp_end) + rsp = rspq; + v |= FW_RSS_IND_TBL_CMD_IQ2_V(*rsp); + if (++rsp >= rsp_end) + rsp = rspq; + + *qp++ = cpu_to_be32(v); + nq -= 3; + } + + ret = t4_wr_mbox(adapter, mbox, &cmd, sizeof(cmd), NULL); + if (ret) + return ret; + } + return 0; +} + +/** + * t4_config_glbl_rss - configure the global RSS mode + * @adapter: the adapter + * @mbox: mbox to use for the FW command + * @mode: global RSS mode + * @flags: mode-specific flags + * + * Sets the global RSS mode. + */ +int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode, + unsigned int flags) +{ + struct fw_rss_glb_config_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_RSS_GLB_CONFIG_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F); + c.retval_len16 = cpu_to_be32(FW_LEN16(c)); + if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_MANUAL) { + c.u.manual.mode_pkd = + cpu_to_be32(FW_RSS_GLB_CONFIG_CMD_MODE_V(mode)); + } else if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) { + c.u.basicvirtual.mode_pkd = + cpu_to_be32(FW_RSS_GLB_CONFIG_CMD_MODE_V(mode)); + c.u.basicvirtual.synmapen_to_hashtoeplitz = cpu_to_be32(flags); + } else + return -EINVAL; + return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_config_vi_rss - configure per VI RSS settings + * @adapter: the adapter + * @mbox: mbox to use for the FW command + * @viid: the VI id + * @flags: RSS flags + * @defq: id of the default RSS queue for the VI. + * + * Configures VI-specific RSS properties. + */ +int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid, + unsigned int flags, unsigned int defq) +{ + struct fw_rss_vi_config_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_RSS_VI_CONFIG_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_RSS_VI_CONFIG_CMD_VIID_V(viid)); + c.retval_len16 = cpu_to_be32(FW_LEN16(c)); + c.u.basicvirtual.defaultq_to_udpen = cpu_to_be32(flags | + FW_RSS_VI_CONFIG_CMD_DEFAULTQ_V(defq)); + return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL); +} + +/* Read an RSS table row */ +static int rd_rss_row(struct adapter *adap, int row, u32 *val) +{ + t4_write_reg(adap, TP_RSS_LKP_TABLE_A, 0xfff00000 | row); + return t4_wait_op_done_val(adap, TP_RSS_LKP_TABLE_A, LKPTBLROWVLD_F, 1, + 5, 0, val); +} + +/** + * t4_read_rss - read the contents of the RSS mapping table + * @adapter: the adapter + * @map: holds the contents of the RSS mapping table + * + * Reads the contents of the RSS hash->queue mapping table. + */ +int t4_read_rss(struct adapter *adapter, u16 *map) +{ + int i, ret, nentries; + u32 val; + + nentries = t4_chip_rss_size(adapter); + for (i = 0; i < nentries / 2; ++i) { + ret = rd_rss_row(adapter, i, &val); + if (ret) + return ret; + *map++ = LKPTBLQUEUE0_G(val); + *map++ = LKPTBLQUEUE1_G(val); + } + return 0; +} + +static unsigned int t4_use_ldst(struct adapter *adap) +{ + return (adap->flags & CXGB4_FW_OK) && !adap->use_bd; +} + +/** + * t4_tp_fw_ldst_rw - Access TP indirect register through LDST + * @adap: the adapter + * @cmd: TP fw ldst address space type + * @vals: where the indirect register values are stored/written + * @nregs: how many indirect registers to read/write + * @start_index: index of first indirect register to read/write + * @rw: Read (1) or Write (0) + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Access TP indirect registers through LDST + */ +static int t4_tp_fw_ldst_rw(struct adapter *adap, int cmd, u32 *vals, + unsigned int nregs, unsigned int start_index, + unsigned int rw, bool sleep_ok) +{ + int ret = 0; + unsigned int i; + struct fw_ldst_cmd c; + + for (i = 0; i < nregs; i++) { + memset(&c, 0, sizeof(c)); + c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) | + FW_CMD_REQUEST_F | + (rw ? FW_CMD_READ_F : + FW_CMD_WRITE_F) | + FW_LDST_CMD_ADDRSPACE_V(cmd)); + c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); + + c.u.addrval.addr = cpu_to_be32(start_index + i); + c.u.addrval.val = rw ? 0 : cpu_to_be32(vals[i]); + ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, + sleep_ok); + if (ret) + return ret; + + if (rw) + vals[i] = be32_to_cpu(c.u.addrval.val); + } + return 0; +} + +/** + * t4_tp_indirect_rw - Read/Write TP indirect register through LDST or backdoor + * @adap: the adapter + * @reg_addr: Address Register + * @reg_data: Data register + * @buff: where the indirect register values are stored/written + * @nregs: how many indirect registers to read/write + * @start_index: index of first indirect register to read/write + * @rw: READ(1) or WRITE(0) + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Read/Write TP indirect registers through LDST if possible. + * Else, use backdoor access + **/ +static void t4_tp_indirect_rw(struct adapter *adap, u32 reg_addr, u32 reg_data, + u32 *buff, u32 nregs, u32 start_index, int rw, + bool sleep_ok) +{ + int rc = -EINVAL; + int cmd; + + switch (reg_addr) { + case TP_PIO_ADDR_A: + cmd = FW_LDST_ADDRSPC_TP_PIO; + break; + case TP_TM_PIO_ADDR_A: + cmd = FW_LDST_ADDRSPC_TP_TM_PIO; + break; + case TP_MIB_INDEX_A: + cmd = FW_LDST_ADDRSPC_TP_MIB; + break; + default: + goto indirect_access; + } + + if (t4_use_ldst(adap)) + rc = t4_tp_fw_ldst_rw(adap, cmd, buff, nregs, start_index, rw, + sleep_ok); + +indirect_access: + + if (rc) { + if (rw) + t4_read_indirect(adap, reg_addr, reg_data, buff, nregs, + start_index); + else + t4_write_indirect(adap, reg_addr, reg_data, buff, nregs, + start_index); + } +} + +/** + * t4_tp_pio_read - Read TP PIO registers + * @adap: the adapter + * @buff: where the indirect register values are written + * @nregs: how many indirect registers to read + * @start_index: index of first indirect register to read + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Read TP PIO Registers + **/ +void t4_tp_pio_read(struct adapter *adap, u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok) +{ + t4_tp_indirect_rw(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, buff, nregs, + start_index, 1, sleep_ok); +} + +/** + * t4_tp_pio_write - Write TP PIO registers + * @adap: the adapter + * @buff: where the indirect register values are stored + * @nregs: how many indirect registers to write + * @start_index: index of first indirect register to write + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Write TP PIO Registers + **/ +static void t4_tp_pio_write(struct adapter *adap, u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok) +{ + t4_tp_indirect_rw(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, buff, nregs, + start_index, 0, sleep_ok); +} + +/** + * t4_tp_tm_pio_read - Read TP TM PIO registers + * @adap: the adapter + * @buff: where the indirect register values are written + * @nregs: how many indirect registers to read + * @start_index: index of first indirect register to read + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Read TP TM PIO Registers + **/ +void t4_tp_tm_pio_read(struct adapter *adap, u32 *buff, u32 nregs, + u32 start_index, bool sleep_ok) +{ + t4_tp_indirect_rw(adap, TP_TM_PIO_ADDR_A, TP_TM_PIO_DATA_A, buff, + nregs, start_index, 1, sleep_ok); +} + +/** + * t4_tp_mib_read - Read TP MIB registers + * @adap: the adapter + * @buff: where the indirect register values are written + * @nregs: how many indirect registers to read + * @start_index: index of first indirect register to read + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Read TP MIB Registers + **/ +void t4_tp_mib_read(struct adapter *adap, u32 *buff, u32 nregs, u32 start_index, + bool sleep_ok) +{ + t4_tp_indirect_rw(adap, TP_MIB_INDEX_A, TP_MIB_DATA_A, buff, nregs, + start_index, 1, sleep_ok); +} + +/** + * t4_read_rss_key - read the global RSS key + * @adap: the adapter + * @key: 10-entry array holding the 320-bit RSS key + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Reads the global 320-bit RSS key. + */ +void t4_read_rss_key(struct adapter *adap, u32 *key, bool sleep_ok) +{ + t4_tp_pio_read(adap, key, 10, TP_RSS_SECRET_KEY0_A, sleep_ok); +} + +/** + * t4_write_rss_key - program one of the RSS keys + * @adap: the adapter + * @key: 10-entry array holding the 320-bit RSS key + * @idx: which RSS key to write + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Writes one of the RSS keys with the given 320-bit value. If @idx is + * 0..15 the corresponding entry in the RSS key table is written, + * otherwise the global RSS key is written. + */ +void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx, + bool sleep_ok) +{ + u8 rss_key_addr_cnt = 16; + u32 vrt = t4_read_reg(adap, TP_RSS_CONFIG_VRT_A); + + /* T6 and later: for KeyMode 3 (per-vf and per-vf scramble), + * allows access to key addresses 16-63 by using KeyWrAddrX + * as index[5:4](upper 2) into key table + */ + if ((CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) && + (vrt & KEYEXTEND_F) && (KEYMODE_G(vrt) == 3)) + rss_key_addr_cnt = 32; + + t4_tp_pio_write(adap, (void *)key, 10, TP_RSS_SECRET_KEY0_A, sleep_ok); + + if (idx >= 0 && idx < rss_key_addr_cnt) { + if (rss_key_addr_cnt > 16) + t4_write_reg(adap, TP_RSS_CONFIG_VRT_A, + KEYWRADDRX_V(idx >> 4) | + T6_VFWRADDR_V(idx) | KEYWREN_F); + else + t4_write_reg(adap, TP_RSS_CONFIG_VRT_A, + KEYWRADDR_V(idx) | KEYWREN_F); + } +} + +/** + * t4_read_rss_pf_config - read PF RSS Configuration Table + * @adapter: the adapter + * @index: the entry in the PF RSS table to read + * @valp: where to store the returned value + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Reads the PF RSS Configuration Table at the specified index and returns + * the value found there. + */ +void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index, + u32 *valp, bool sleep_ok) +{ + t4_tp_pio_read(adapter, valp, 1, TP_RSS_PF0_CONFIG_A + index, sleep_ok); +} + +/** + * t4_read_rss_vf_config - read VF RSS Configuration Table + * @adapter: the adapter + * @index: the entry in the VF RSS table to read + * @vfl: where to store the returned VFL + * @vfh: where to store the returned VFH + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Reads the VF RSS Configuration Table at the specified index and returns + * the (VFL, VFH) values found there. + */ +void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index, + u32 *vfl, u32 *vfh, bool sleep_ok) +{ + u32 vrt, mask, data; + + if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) { + mask = VFWRADDR_V(VFWRADDR_M); + data = VFWRADDR_V(index); + } else { + mask = T6_VFWRADDR_V(T6_VFWRADDR_M); + data = T6_VFWRADDR_V(index); + } + + /* Request that the index'th VF Table values be read into VFL/VFH. + */ + vrt = t4_read_reg(adapter, TP_RSS_CONFIG_VRT_A); + vrt &= ~(VFRDRG_F | VFWREN_F | KEYWREN_F | mask); + vrt |= data | VFRDEN_F; + t4_write_reg(adapter, TP_RSS_CONFIG_VRT_A, vrt); + + /* Grab the VFL/VFH values ... + */ + t4_tp_pio_read(adapter, vfl, 1, TP_RSS_VFL_CONFIG_A, sleep_ok); + t4_tp_pio_read(adapter, vfh, 1, TP_RSS_VFH_CONFIG_A, sleep_ok); +} + +/** + * t4_read_rss_pf_map - read PF RSS Map + * @adapter: the adapter + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Reads the PF RSS Map register and returns its value. + */ +u32 t4_read_rss_pf_map(struct adapter *adapter, bool sleep_ok) +{ + u32 pfmap; + + t4_tp_pio_read(adapter, &pfmap, 1, TP_RSS_PF_MAP_A, sleep_ok); + return pfmap; +} + +/** + * t4_read_rss_pf_mask - read PF RSS Mask + * @adapter: the adapter + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Reads the PF RSS Mask register and returns its value. + */ +u32 t4_read_rss_pf_mask(struct adapter *adapter, bool sleep_ok) +{ + u32 pfmask; + + t4_tp_pio_read(adapter, &pfmask, 1, TP_RSS_PF_MSK_A, sleep_ok); + return pfmask; +} + +/** + * t4_tp_get_tcp_stats - read TP's TCP MIB counters + * @adap: the adapter + * @v4: holds the TCP/IP counter values + * @v6: holds the TCP/IPv6 counter values + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Returns the values of TP's TCP/IP and TCP/IPv6 MIB counters. + * Either @v4 or @v6 may be %NULL to skip the corresponding stats. + */ +void t4_tp_get_tcp_stats(struct adapter *adap, struct tp_tcp_stats *v4, + struct tp_tcp_stats *v6, bool sleep_ok) +{ + u32 val[TP_MIB_TCP_RXT_SEG_LO_A - TP_MIB_TCP_OUT_RST_A + 1]; + +#define STAT_IDX(x) ((TP_MIB_TCP_##x##_A) - TP_MIB_TCP_OUT_RST_A) +#define STAT(x) val[STAT_IDX(x)] +#define STAT64(x) (((u64)STAT(x##_HI) << 32) | STAT(x##_LO)) + + if (v4) { + t4_tp_mib_read(adap, val, ARRAY_SIZE(val), + TP_MIB_TCP_OUT_RST_A, sleep_ok); + v4->tcp_out_rsts = STAT(OUT_RST); + v4->tcp_in_segs = STAT64(IN_SEG); + v4->tcp_out_segs = STAT64(OUT_SEG); + v4->tcp_retrans_segs = STAT64(RXT_SEG); + } + if (v6) { + t4_tp_mib_read(adap, val, ARRAY_SIZE(val), + TP_MIB_TCP_V6OUT_RST_A, sleep_ok); + v6->tcp_out_rsts = STAT(OUT_RST); + v6->tcp_in_segs = STAT64(IN_SEG); + v6->tcp_out_segs = STAT64(OUT_SEG); + v6->tcp_retrans_segs = STAT64(RXT_SEG); + } +#undef STAT64 +#undef STAT +#undef STAT_IDX +} + +/** + * t4_tp_get_err_stats - read TP's error MIB counters + * @adap: the adapter + * @st: holds the counter values + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Returns the values of TP's error counters. + */ +void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st, + bool sleep_ok) +{ + int nchan = adap->params.arch.nchan; + + t4_tp_mib_read(adap, st->mac_in_errs, nchan, TP_MIB_MAC_IN_ERR_0_A, + sleep_ok); + t4_tp_mib_read(adap, st->hdr_in_errs, nchan, TP_MIB_HDR_IN_ERR_0_A, + sleep_ok); + t4_tp_mib_read(adap, st->tcp_in_errs, nchan, TP_MIB_TCP_IN_ERR_0_A, + sleep_ok); + t4_tp_mib_read(adap, st->tnl_cong_drops, nchan, + TP_MIB_TNL_CNG_DROP_0_A, sleep_ok); + t4_tp_mib_read(adap, st->ofld_chan_drops, nchan, + TP_MIB_OFD_CHN_DROP_0_A, sleep_ok); + t4_tp_mib_read(adap, st->tnl_tx_drops, nchan, TP_MIB_TNL_DROP_0_A, + sleep_ok); + t4_tp_mib_read(adap, st->ofld_vlan_drops, nchan, + TP_MIB_OFD_VLN_DROP_0_A, sleep_ok); + t4_tp_mib_read(adap, st->tcp6_in_errs, nchan, + TP_MIB_TCP_V6IN_ERR_0_A, sleep_ok); + t4_tp_mib_read(adap, &st->ofld_no_neigh, 2, TP_MIB_OFD_ARP_DROP_A, + sleep_ok); +} + +/** + * t4_tp_get_cpl_stats - read TP's CPL MIB counters + * @adap: the adapter + * @st: holds the counter values + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Returns the values of TP's CPL counters. + */ +void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st, + bool sleep_ok) +{ + int nchan = adap->params.arch.nchan; + + t4_tp_mib_read(adap, st->req, nchan, TP_MIB_CPL_IN_REQ_0_A, sleep_ok); + + t4_tp_mib_read(adap, st->rsp, nchan, TP_MIB_CPL_OUT_RSP_0_A, sleep_ok); +} + +/** + * t4_tp_get_rdma_stats - read TP's RDMA MIB counters + * @adap: the adapter + * @st: holds the counter values + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Returns the values of TP's RDMA counters. + */ +void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st, + bool sleep_ok) +{ + t4_tp_mib_read(adap, &st->rqe_dfr_pkt, 2, TP_MIB_RQE_DFR_PKT_A, + sleep_ok); +} + +/** + * t4_get_fcoe_stats - read TP's FCoE MIB counters for a port + * @adap: the adapter + * @idx: the port index + * @st: holds the counter values + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Returns the values of TP's FCoE counters for the selected port. + */ +void t4_get_fcoe_stats(struct adapter *adap, unsigned int idx, + struct tp_fcoe_stats *st, bool sleep_ok) +{ + u32 val[2]; + + t4_tp_mib_read(adap, &st->frames_ddp, 1, TP_MIB_FCOE_DDP_0_A + idx, + sleep_ok); + + t4_tp_mib_read(adap, &st->frames_drop, 1, + TP_MIB_FCOE_DROP_0_A + idx, sleep_ok); + + t4_tp_mib_read(adap, val, 2, TP_MIB_FCOE_BYTE_0_HI_A + 2 * idx, + sleep_ok); + + st->octets_ddp = ((u64)val[0] << 32) | val[1]; +} + +/** + * t4_get_usm_stats - read TP's non-TCP DDP MIB counters + * @adap: the adapter + * @st: holds the counter values + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Returns the values of TP's counters for non-TCP directly-placed packets. + */ +void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st, + bool sleep_ok) +{ + u32 val[4]; + + t4_tp_mib_read(adap, val, 4, TP_MIB_USM_PKTS_A, sleep_ok); + st->frames = val[0]; + st->drops = val[1]; + st->octets = ((u64)val[2] << 32) | val[3]; +} + +/** + * t4_read_mtu_tbl - returns the values in the HW path MTU table + * @adap: the adapter + * @mtus: where to store the MTU values + * @mtu_log: where to store the MTU base-2 log (may be %NULL) + * + * Reads the HW path MTU table. + */ +void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log) +{ + u32 v; + int i; + + for (i = 0; i < NMTUS; ++i) { + t4_write_reg(adap, TP_MTU_TABLE_A, + MTUINDEX_V(0xff) | MTUVALUE_V(i)); + v = t4_read_reg(adap, TP_MTU_TABLE_A); + mtus[i] = MTUVALUE_G(v); + if (mtu_log) + mtu_log[i] = MTUWIDTH_G(v); + } +} + +/** + * t4_read_cong_tbl - reads the congestion control table + * @adap: the adapter + * @incr: where to store the alpha values + * + * Reads the additive increments programmed into the HW congestion + * control table. + */ +void t4_read_cong_tbl(struct adapter *adap, u16 incr[NMTUS][NCCTRL_WIN]) +{ + unsigned int mtu, w; + + for (mtu = 0; mtu < NMTUS; ++mtu) + for (w = 0; w < NCCTRL_WIN; ++w) { + t4_write_reg(adap, TP_CCTRL_TABLE_A, + ROWINDEX_V(0xffff) | (mtu << 5) | w); + incr[mtu][w] = (u16)t4_read_reg(adap, + TP_CCTRL_TABLE_A) & 0x1fff; + } +} + +/** + * t4_tp_wr_bits_indirect - set/clear bits in an indirect TP register + * @adap: the adapter + * @addr: the indirect TP register address + * @mask: specifies the field within the register to modify + * @val: new value for the field + * + * Sets a field of an indirect TP register to the given value. + */ +void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr, + unsigned int mask, unsigned int val) +{ + t4_write_reg(adap, TP_PIO_ADDR_A, addr); + val |= t4_read_reg(adap, TP_PIO_DATA_A) & ~mask; + t4_write_reg(adap, TP_PIO_DATA_A, val); +} + +/** + * init_cong_ctrl - initialize congestion control parameters + * @a: the alpha values for congestion control + * @b: the beta values for congestion control + * + * Initialize the congestion control parameters. + */ +static void init_cong_ctrl(unsigned short *a, unsigned short *b) +{ + a[0] = a[1] = a[2] = a[3] = a[4] = a[5] = a[6] = a[7] = a[8] = 1; + a[9] = 2; + a[10] = 3; + a[11] = 4; + a[12] = 5; + a[13] = 6; + a[14] = 7; + a[15] = 8; + a[16] = 9; + a[17] = 10; + a[18] = 14; + a[19] = 17; + a[20] = 21; + a[21] = 25; + a[22] = 30; + a[23] = 35; + a[24] = 45; + a[25] = 60; + a[26] = 80; + a[27] = 100; + a[28] = 200; + a[29] = 300; + a[30] = 400; + a[31] = 500; + + b[0] = b[1] = b[2] = b[3] = b[4] = b[5] = b[6] = b[7] = b[8] = 0; + b[9] = b[10] = 1; + b[11] = b[12] = 2; + b[13] = b[14] = b[15] = b[16] = 3; + b[17] = b[18] = b[19] = b[20] = b[21] = 4; + b[22] = b[23] = b[24] = b[25] = b[26] = b[27] = 5; + b[28] = b[29] = 6; + b[30] = b[31] = 7; +} + +/* The minimum additive increment value for the congestion control table */ +#define CC_MIN_INCR 2U + +/** + * t4_load_mtus - write the MTU and congestion control HW tables + * @adap: the adapter + * @mtus: the values for the MTU table + * @alpha: the values for the congestion control alpha parameter + * @beta: the values for the congestion control beta parameter + * + * Write the HW MTU table with the supplied MTUs and the high-speed + * congestion control table with the supplied alpha, beta, and MTUs. + * We write the two tables together because the additive increments + * depend on the MTUs. + */ +void t4_load_mtus(struct adapter *adap, const unsigned short *mtus, + const unsigned short *alpha, const unsigned short *beta) +{ + static const unsigned int avg_pkts[NCCTRL_WIN] = { + 2, 6, 10, 14, 20, 28, 40, 56, 80, 112, 160, 224, 320, 448, 640, + 896, 1281, 1792, 2560, 3584, 5120, 7168, 10240, 14336, 20480, + 28672, 40960, 57344, 81920, 114688, 163840, 229376 + }; + + unsigned int i, w; + + for (i = 0; i < NMTUS; ++i) { + unsigned int mtu = mtus[i]; + unsigned int log2 = fls(mtu); + + if (!(mtu & ((1 << log2) >> 2))) /* round */ + log2--; + t4_write_reg(adap, TP_MTU_TABLE_A, MTUINDEX_V(i) | + MTUWIDTH_V(log2) | MTUVALUE_V(mtu)); + + for (w = 0; w < NCCTRL_WIN; ++w) { + unsigned int inc; + + inc = max(((mtu - 40) * alpha[w]) / avg_pkts[w], + CC_MIN_INCR); + + t4_write_reg(adap, TP_CCTRL_TABLE_A, (i << 21) | + (w << 16) | (beta[w] << 13) | inc); + } + } +} + +/* Calculates a rate in bytes/s given the number of 256-byte units per 4K core + * clocks. The formula is + * + * bytes/s = bytes256 * 256 * ClkFreq / 4096 + * + * which is equivalent to + * + * bytes/s = 62.5 * bytes256 * ClkFreq_ms + */ +static u64 chan_rate(struct adapter *adap, unsigned int bytes256) +{ + u64 v = bytes256 * adap->params.vpd.cclk; + + return v * 62 + v / 2; +} + +/** + * t4_get_chan_txrate - get the current per channel Tx rates + * @adap: the adapter + * @nic_rate: rates for NIC traffic + * @ofld_rate: rates for offloaded traffic + * + * Return the current Tx rates in bytes/s for NIC and offloaded traffic + * for each channel. + */ +void t4_get_chan_txrate(struct adapter *adap, u64 *nic_rate, u64 *ofld_rate) +{ + u32 v; + + v = t4_read_reg(adap, TP_TX_TRATE_A); + nic_rate[0] = chan_rate(adap, TNLRATE0_G(v)); + nic_rate[1] = chan_rate(adap, TNLRATE1_G(v)); + if (adap->params.arch.nchan == NCHAN) { + nic_rate[2] = chan_rate(adap, TNLRATE2_G(v)); + nic_rate[3] = chan_rate(adap, TNLRATE3_G(v)); + } + + v = t4_read_reg(adap, TP_TX_ORATE_A); + ofld_rate[0] = chan_rate(adap, OFDRATE0_G(v)); + ofld_rate[1] = chan_rate(adap, OFDRATE1_G(v)); + if (adap->params.arch.nchan == NCHAN) { + ofld_rate[2] = chan_rate(adap, OFDRATE2_G(v)); + ofld_rate[3] = chan_rate(adap, OFDRATE3_G(v)); + } +} + +/** + * t4_set_trace_filter - configure one of the tracing filters + * @adap: the adapter + * @tp: the desired trace filter parameters + * @idx: which filter to configure + * @enable: whether to enable or disable the filter + * + * Configures one of the tracing filters available in HW. If @enable is + * %0 @tp is not examined and may be %NULL. The user is responsible to + * set the single/multiple trace mode by writing to MPS_TRC_CFG_A register + */ +int t4_set_trace_filter(struct adapter *adap, const struct trace_params *tp, + int idx, int enable) +{ + int i, ofst = idx * 4; + u32 data_reg, mask_reg, cfg; + + if (!enable) { + t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst, 0); + return 0; + } + + cfg = t4_read_reg(adap, MPS_TRC_CFG_A); + if (cfg & TRCMULTIFILTER_F) { + /* If multiple tracers are enabled, then maximum + * capture size is 2.5KB (FIFO size of a single channel) + * minus 2 flits for CPL_TRACE_PKT header. + */ + if (tp->snap_len > ((10 * 1024 / 4) - (2 * 8))) + return -EINVAL; + } else { + /* If multiple tracers are disabled, to avoid deadlocks + * maximum packet capture size of 9600 bytes is recommended. + * Also in this mode, only trace0 can be enabled and running. + */ + if (tp->snap_len > 9600 || idx) + return -EINVAL; + } + + if (tp->port > (is_t4(adap->params.chip) ? 11 : 19) || tp->invert > 1 || + tp->skip_len > TFLENGTH_M || tp->skip_ofst > TFOFFSET_M || + tp->min_len > TFMINPKTSIZE_M) + return -EINVAL; + + /* stop the tracer we'll be changing */ + t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst, 0); + + idx *= (MPS_TRC_FILTER1_MATCH_A - MPS_TRC_FILTER0_MATCH_A); + data_reg = MPS_TRC_FILTER0_MATCH_A + idx; + mask_reg = MPS_TRC_FILTER0_DONT_CARE_A + idx; + + for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) { + t4_write_reg(adap, data_reg, tp->data[i]); + t4_write_reg(adap, mask_reg, ~tp->mask[i]); + } + t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_B_A + ofst, + TFCAPTUREMAX_V(tp->snap_len) | + TFMINPKTSIZE_V(tp->min_len)); + t4_write_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst, + TFOFFSET_V(tp->skip_ofst) | TFLENGTH_V(tp->skip_len) | + (is_t4(adap->params.chip) ? + TFPORT_V(tp->port) | TFEN_F | TFINVERTMATCH_V(tp->invert) : + T5_TFPORT_V(tp->port) | T5_TFEN_F | + T5_TFINVERTMATCH_V(tp->invert))); + + return 0; +} + +/** + * t4_get_trace_filter - query one of the tracing filters + * @adap: the adapter + * @tp: the current trace filter parameters + * @idx: which trace filter to query + * @enabled: non-zero if the filter is enabled + * + * Returns the current settings of one of the HW tracing filters. + */ +void t4_get_trace_filter(struct adapter *adap, struct trace_params *tp, int idx, + int *enabled) +{ + u32 ctla, ctlb; + int i, ofst = idx * 4; + u32 data_reg, mask_reg; + + ctla = t4_read_reg(adap, MPS_TRC_FILTER_MATCH_CTL_A_A + ofst); + ctlb = t4_read_reg(adap, MPS_TRC_FILTER_MATCH_CTL_B_A + ofst); + + if (is_t4(adap->params.chip)) { + *enabled = !!(ctla & TFEN_F); + tp->port = TFPORT_G(ctla); + tp->invert = !!(ctla & TFINVERTMATCH_F); + } else { + *enabled = !!(ctla & T5_TFEN_F); + tp->port = T5_TFPORT_G(ctla); + tp->invert = !!(ctla & T5_TFINVERTMATCH_F); + } + tp->snap_len = TFCAPTUREMAX_G(ctlb); + tp->min_len = TFMINPKTSIZE_G(ctlb); + tp->skip_ofst = TFOFFSET_G(ctla); + tp->skip_len = TFLENGTH_G(ctla); + + ofst = (MPS_TRC_FILTER1_MATCH_A - MPS_TRC_FILTER0_MATCH_A) * idx; + data_reg = MPS_TRC_FILTER0_MATCH_A + ofst; + mask_reg = MPS_TRC_FILTER0_DONT_CARE_A + ofst; + + for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) { + tp->mask[i] = ~t4_read_reg(adap, mask_reg); + tp->data[i] = t4_read_reg(adap, data_reg) & tp->mask[i]; + } +} + +/** + * t4_pmtx_get_stats - returns the HW stats from PMTX + * @adap: the adapter + * @cnt: where to store the count statistics + * @cycles: where to store the cycle statistics + * + * Returns performance statistics from PMTX. + */ +void t4_pmtx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[]) +{ + int i; + u32 data[2]; + + for (i = 0; i < adap->params.arch.pm_stats_cnt; i++) { + t4_write_reg(adap, PM_TX_STAT_CONFIG_A, i + 1); + cnt[i] = t4_read_reg(adap, PM_TX_STAT_COUNT_A); + if (is_t4(adap->params.chip)) { + cycles[i] = t4_read_reg64(adap, PM_TX_STAT_LSB_A); + } else { + t4_read_indirect(adap, PM_TX_DBG_CTRL_A, + PM_TX_DBG_DATA_A, data, 2, + PM_TX_DBG_STAT_MSB_A); + cycles[i] = (((u64)data[0] << 32) | data[1]); + } + } +} + +/** + * t4_pmrx_get_stats - returns the HW stats from PMRX + * @adap: the adapter + * @cnt: where to store the count statistics + * @cycles: where to store the cycle statistics + * + * Returns performance statistics from PMRX. + */ +void t4_pmrx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[]) +{ + int i; + u32 data[2]; + + for (i = 0; i < adap->params.arch.pm_stats_cnt; i++) { + t4_write_reg(adap, PM_RX_STAT_CONFIG_A, i + 1); + cnt[i] = t4_read_reg(adap, PM_RX_STAT_COUNT_A); + if (is_t4(adap->params.chip)) { + cycles[i] = t4_read_reg64(adap, PM_RX_STAT_LSB_A); + } else { + t4_read_indirect(adap, PM_RX_DBG_CTRL_A, + PM_RX_DBG_DATA_A, data, 2, + PM_RX_DBG_STAT_MSB_A); + cycles[i] = (((u64)data[0] << 32) | data[1]); + } + } +} + +/** + * compute_mps_bg_map - compute the MPS Buffer Group Map for a Port + * @adapter: the adapter + * @pidx: the port index + * + * Computes and returns a bitmap indicating which MPS buffer groups are + * associated with the given Port. Bit i is set if buffer group i is + * used by the Port. + */ +static inline unsigned int compute_mps_bg_map(struct adapter *adapter, + int pidx) +{ + unsigned int chip_version, nports; + + chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip); + nports = 1 << NUMPORTS_G(t4_read_reg(adapter, MPS_CMN_CTL_A)); + + switch (chip_version) { + case CHELSIO_T4: + case CHELSIO_T5: + switch (nports) { + case 1: return 0xf; + case 2: return 3 << (2 * pidx); + case 4: return 1 << pidx; + } + break; + + case CHELSIO_T6: + switch (nports) { + case 2: return 1 << (2 * pidx); + } + break; + } + + dev_err(adapter->pdev_dev, "Need MPS Buffer Group Map for Chip %0x, Nports %d\n", + chip_version, nports); + + return 0; +} + +/** + * t4_get_mps_bg_map - return the buffer groups associated with a port + * @adapter: the adapter + * @pidx: the port index + * + * Returns a bitmap indicating which MPS buffer groups are associated + * with the given Port. Bit i is set if buffer group i is used by the + * Port. + */ +unsigned int t4_get_mps_bg_map(struct adapter *adapter, int pidx) +{ + u8 *mps_bg_map; + unsigned int nports; + + nports = 1 << NUMPORTS_G(t4_read_reg(adapter, MPS_CMN_CTL_A)); + if (pidx >= nports) { + CH_WARN(adapter, "MPS Port Index %d >= Nports %d\n", + pidx, nports); + return 0; + } + + /* If we've already retrieved/computed this, just return the result. + */ + mps_bg_map = adapter->params.mps_bg_map; + if (mps_bg_map[pidx]) + return mps_bg_map[pidx]; + + /* Newer Firmware can tell us what the MPS Buffer Group Map is. + * If we're talking to such Firmware, let it tell us. If the new + * API isn't supported, revert back to old hardcoded way. The value + * obtained from Firmware is encoded in below format: + * + * val = (( MPSBGMAP[Port 3] << 24 ) | + * ( MPSBGMAP[Port 2] << 16 ) | + * ( MPSBGMAP[Port 1] << 8 ) | + * ( MPSBGMAP[Port 0] << 0 )) + */ + if (adapter->flags & CXGB4_FW_OK) { + u32 param, val; + int ret; + + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_MPSBGMAP)); + ret = t4_query_params_ns(adapter, adapter->mbox, adapter->pf, + 0, 1, ¶m, &val); + if (!ret) { + int p; + + /* Store the BG Map for all of the Ports in order to + * avoid more calls to the Firmware in the future. + */ + for (p = 0; p < MAX_NPORTS; p++, val >>= 8) + mps_bg_map[p] = val & 0xff; + + return mps_bg_map[pidx]; + } + } + + /* Either we're not talking to the Firmware or we're dealing with + * older Firmware which doesn't support the new API to get the MPS + * Buffer Group Map. Fall back to computing it ourselves. + */ + mps_bg_map[pidx] = compute_mps_bg_map(adapter, pidx); + return mps_bg_map[pidx]; +} + +/** + * t4_get_tp_e2c_map - return the E2C channel map associated with a port + * @adapter: the adapter + * @pidx: the port index + */ +static unsigned int t4_get_tp_e2c_map(struct adapter *adapter, int pidx) +{ + unsigned int nports; + u32 param, val = 0; + int ret; + + nports = 1 << NUMPORTS_G(t4_read_reg(adapter, MPS_CMN_CTL_A)); + if (pidx >= nports) { + CH_WARN(adapter, "TP E2C Channel Port Index %d >= Nports %d\n", + pidx, nports); + return 0; + } + + /* FW version >= 1.16.44.0 can determine E2C channel map using + * FW_PARAMS_PARAM_DEV_TPCHMAP API. + */ + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_TPCHMAP)); + ret = t4_query_params_ns(adapter, adapter->mbox, adapter->pf, + 0, 1, ¶m, &val); + if (!ret) + return (val >> (8 * pidx)) & 0xff; + + return 0; +} + +/** + * t4_get_tp_ch_map - return TP ingress channels associated with a port + * @adap: the adapter + * @pidx: the port index + * + * Returns a bitmap indicating which TP Ingress Channels are associated + * with a given Port. Bit i is set if TP Ingress Channel i is used by + * the Port. + */ +unsigned int t4_get_tp_ch_map(struct adapter *adap, int pidx) +{ + unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip); + unsigned int nports = 1 << NUMPORTS_G(t4_read_reg(adap, MPS_CMN_CTL_A)); + + if (pidx >= nports) { + dev_warn(adap->pdev_dev, "TP Port Index %d >= Nports %d\n", + pidx, nports); + return 0; + } + + switch (chip_version) { + case CHELSIO_T4: + case CHELSIO_T5: + /* Note that this happens to be the same values as the MPS + * Buffer Group Map for these Chips. But we replicate the code + * here because they're really separate concepts. + */ + switch (nports) { + case 1: return 0xf; + case 2: return 3 << (2 * pidx); + case 4: return 1 << pidx; + } + break; + + case CHELSIO_T6: + switch (nports) { + case 1: + case 2: return 1 << pidx; + } + break; + } + + dev_err(adap->pdev_dev, "Need TP Channel Map for Chip %0x, Nports %d\n", + chip_version, nports); + return 0; +} + +/** + * t4_get_port_type_description - return Port Type string description + * @port_type: firmware Port Type enumeration + */ +const char *t4_get_port_type_description(enum fw_port_type port_type) +{ + static const char *const port_type_description[] = { + "Fiber_XFI", + "Fiber_XAUI", + "BT_SGMII", + "BT_XFI", + "BT_XAUI", + "KX4", + "CX4", + "KX", + "KR", + "SFP", + "BP_AP", + "BP4_AP", + "QSFP_10G", + "QSA", + "QSFP", + "BP40_BA", + "KR4_100G", + "CR4_QSFP", + "CR_QSFP", + "CR2_QSFP", + "SFP28", + "KR_SFP28", + "KR_XLAUI" + }; + + if (port_type < ARRAY_SIZE(port_type_description)) + return port_type_description[port_type]; + return "UNKNOWN"; +} + +/** + * t4_get_port_stats_offset - collect port stats relative to a previous + * snapshot + * @adap: The adapter + * @idx: The port + * @stats: Current stats to fill + * @offset: Previous stats snapshot + */ +void t4_get_port_stats_offset(struct adapter *adap, int idx, + struct port_stats *stats, + struct port_stats *offset) +{ + u64 *s, *o; + int i; + + t4_get_port_stats(adap, idx, stats); + for (i = 0, s = (u64 *)stats, o = (u64 *)offset; + i < (sizeof(struct port_stats) / sizeof(u64)); + i++, s++, o++) + *s -= *o; +} + +/** + * t4_get_port_stats - collect port statistics + * @adap: the adapter + * @idx: the port index + * @p: the stats structure to fill + * + * Collect statistics related to the given port from HW. + */ +void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p) +{ + u32 bgmap = t4_get_mps_bg_map(adap, idx); + u32 stat_ctl = t4_read_reg(adap, MPS_STAT_CTL_A); + +#define GET_STAT(name) \ + t4_read_reg64(adap, \ + (is_t4(adap->params.chip) ? PORT_REG(idx, MPS_PORT_STAT_##name##_L) : \ + T5_PORT_REG(idx, MPS_PORT_STAT_##name##_L))) +#define GET_STAT_COM(name) t4_read_reg64(adap, MPS_STAT_##name##_L) + + p->tx_octets = GET_STAT(TX_PORT_BYTES); + p->tx_frames = GET_STAT(TX_PORT_FRAMES); + p->tx_bcast_frames = GET_STAT(TX_PORT_BCAST); + p->tx_mcast_frames = GET_STAT(TX_PORT_MCAST); + p->tx_ucast_frames = GET_STAT(TX_PORT_UCAST); + p->tx_error_frames = GET_STAT(TX_PORT_ERROR); + p->tx_frames_64 = GET_STAT(TX_PORT_64B); + p->tx_frames_65_127 = GET_STAT(TX_PORT_65B_127B); + p->tx_frames_128_255 = GET_STAT(TX_PORT_128B_255B); + p->tx_frames_256_511 = GET_STAT(TX_PORT_256B_511B); + p->tx_frames_512_1023 = GET_STAT(TX_PORT_512B_1023B); + p->tx_frames_1024_1518 = GET_STAT(TX_PORT_1024B_1518B); + p->tx_frames_1519_max = GET_STAT(TX_PORT_1519B_MAX); + p->tx_drop = GET_STAT(TX_PORT_DROP); + p->tx_pause = GET_STAT(TX_PORT_PAUSE); + p->tx_ppp0 = GET_STAT(TX_PORT_PPP0); + p->tx_ppp1 = GET_STAT(TX_PORT_PPP1); + p->tx_ppp2 = GET_STAT(TX_PORT_PPP2); + p->tx_ppp3 = GET_STAT(TX_PORT_PPP3); + p->tx_ppp4 = GET_STAT(TX_PORT_PPP4); + p->tx_ppp5 = GET_STAT(TX_PORT_PPP5); + p->tx_ppp6 = GET_STAT(TX_PORT_PPP6); + p->tx_ppp7 = GET_STAT(TX_PORT_PPP7); + + if (CHELSIO_CHIP_VERSION(adap->params.chip) >= CHELSIO_T5) { + if (stat_ctl & COUNTPAUSESTATTX_F) + p->tx_frames_64 -= p->tx_pause; + if (stat_ctl & COUNTPAUSEMCTX_F) + p->tx_mcast_frames -= p->tx_pause; + } + p->rx_octets = GET_STAT(RX_PORT_BYTES); + p->rx_frames = GET_STAT(RX_PORT_FRAMES); + p->rx_bcast_frames = GET_STAT(RX_PORT_BCAST); + p->rx_mcast_frames = GET_STAT(RX_PORT_MCAST); + p->rx_ucast_frames = GET_STAT(RX_PORT_UCAST); + p->rx_too_long = GET_STAT(RX_PORT_MTU_ERROR); + p->rx_jabber = GET_STAT(RX_PORT_MTU_CRC_ERROR); + p->rx_fcs_err = GET_STAT(RX_PORT_CRC_ERROR); + p->rx_len_err = GET_STAT(RX_PORT_LEN_ERROR); + p->rx_symbol_err = GET_STAT(RX_PORT_SYM_ERROR); + p->rx_runt = GET_STAT(RX_PORT_LESS_64B); + p->rx_frames_64 = GET_STAT(RX_PORT_64B); + p->rx_frames_65_127 = GET_STAT(RX_PORT_65B_127B); + p->rx_frames_128_255 = GET_STAT(RX_PORT_128B_255B); + p->rx_frames_256_511 = GET_STAT(RX_PORT_256B_511B); + p->rx_frames_512_1023 = GET_STAT(RX_PORT_512B_1023B); + p->rx_frames_1024_1518 = GET_STAT(RX_PORT_1024B_1518B); + p->rx_frames_1519_max = GET_STAT(RX_PORT_1519B_MAX); + p->rx_pause = GET_STAT(RX_PORT_PAUSE); + p->rx_ppp0 = GET_STAT(RX_PORT_PPP0); + p->rx_ppp1 = GET_STAT(RX_PORT_PPP1); + p->rx_ppp2 = GET_STAT(RX_PORT_PPP2); + p->rx_ppp3 = GET_STAT(RX_PORT_PPP3); + p->rx_ppp4 = GET_STAT(RX_PORT_PPP4); + p->rx_ppp5 = GET_STAT(RX_PORT_PPP5); + p->rx_ppp6 = GET_STAT(RX_PORT_PPP6); + p->rx_ppp7 = GET_STAT(RX_PORT_PPP7); + + if (CHELSIO_CHIP_VERSION(adap->params.chip) >= CHELSIO_T5) { + if (stat_ctl & COUNTPAUSESTATRX_F) + p->rx_frames_64 -= p->rx_pause; + if (stat_ctl & COUNTPAUSEMCRX_F) + p->rx_mcast_frames -= p->rx_pause; + } + + p->rx_ovflow0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0; + p->rx_ovflow1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0; + p->rx_ovflow2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0; + p->rx_ovflow3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0; + p->rx_trunc0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0; + p->rx_trunc1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0; + p->rx_trunc2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0; + p->rx_trunc3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0; + +#undef GET_STAT +#undef GET_STAT_COM +} + +/** + * t4_get_lb_stats - collect loopback port statistics + * @adap: the adapter + * @idx: the loopback port index + * @p: the stats structure to fill + * + * Return HW statistics for the given loopback port. + */ +void t4_get_lb_stats(struct adapter *adap, int idx, struct lb_port_stats *p) +{ + u32 bgmap = t4_get_mps_bg_map(adap, idx); + +#define GET_STAT(name) \ + t4_read_reg64(adap, \ + (is_t4(adap->params.chip) ? \ + PORT_REG(idx, MPS_PORT_STAT_LB_PORT_##name##_L) : \ + T5_PORT_REG(idx, MPS_PORT_STAT_LB_PORT_##name##_L))) +#define GET_STAT_COM(name) t4_read_reg64(adap, MPS_STAT_##name##_L) + + p->octets = GET_STAT(BYTES); + p->frames = GET_STAT(FRAMES); + p->bcast_frames = GET_STAT(BCAST); + p->mcast_frames = GET_STAT(MCAST); + p->ucast_frames = GET_STAT(UCAST); + p->error_frames = GET_STAT(ERROR); + + p->frames_64 = GET_STAT(64B); + p->frames_65_127 = GET_STAT(65B_127B); + p->frames_128_255 = GET_STAT(128B_255B); + p->frames_256_511 = GET_STAT(256B_511B); + p->frames_512_1023 = GET_STAT(512B_1023B); + p->frames_1024_1518 = GET_STAT(1024B_1518B); + p->frames_1519_max = GET_STAT(1519B_MAX); + p->drop = GET_STAT(DROP_FRAMES); + + p->ovflow0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_LB_DROP_FRAME) : 0; + p->ovflow1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_LB_DROP_FRAME) : 0; + p->ovflow2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_LB_DROP_FRAME) : 0; + p->ovflow3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_LB_DROP_FRAME) : 0; + p->trunc0 = (bgmap & 1) ? GET_STAT_COM(RX_BG_0_LB_TRUNC_FRAME) : 0; + p->trunc1 = (bgmap & 2) ? GET_STAT_COM(RX_BG_1_LB_TRUNC_FRAME) : 0; + p->trunc2 = (bgmap & 4) ? GET_STAT_COM(RX_BG_2_LB_TRUNC_FRAME) : 0; + p->trunc3 = (bgmap & 8) ? GET_STAT_COM(RX_BG_3_LB_TRUNC_FRAME) : 0; + +#undef GET_STAT +#undef GET_STAT_COM +} + +/* t4_mk_filtdelwr - create a delete filter WR + * @ftid: the filter ID + * @wr: the filter work request to populate + * @qid: ingress queue to receive the delete notification + * + * Creates a filter work request to delete the supplied filter. If @qid is + * negative the delete notification is suppressed. + */ +void t4_mk_filtdelwr(unsigned int ftid, struct fw_filter_wr *wr, int qid) +{ + memset(wr, 0, sizeof(*wr)); + wr->op_pkd = cpu_to_be32(FW_WR_OP_V(FW_FILTER_WR)); + wr->len16_pkd = cpu_to_be32(FW_WR_LEN16_V(sizeof(*wr) / 16)); + wr->tid_to_iq = cpu_to_be32(FW_FILTER_WR_TID_V(ftid) | + FW_FILTER_WR_NOREPLY_V(qid < 0)); + wr->del_filter_to_l2tix = cpu_to_be32(FW_FILTER_WR_DEL_FILTER_F); + if (qid >= 0) + wr->rx_chan_rx_rpl_iq = + cpu_to_be16(FW_FILTER_WR_RX_RPL_IQ_V(qid)); +} + +#define INIT_CMD(var, cmd, rd_wr) do { \ + (var).op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_##cmd##_CMD) | \ + FW_CMD_REQUEST_F | \ + FW_CMD_##rd_wr##_F); \ + (var).retval_len16 = cpu_to_be32(FW_LEN16(var)); \ +} while (0) + +int t4_fwaddrspace_write(struct adapter *adap, unsigned int mbox, + u32 addr, u32 val) +{ + u32 ldst_addrspace; + struct fw_ldst_cmd c; + + memset(&c, 0, sizeof(c)); + ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FIRMWARE); + c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + ldst_addrspace); + c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); + c.u.addrval.addr = cpu_to_be32(addr); + c.u.addrval.val = cpu_to_be32(val); + + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_mdio_rd - read a PHY register through MDIO + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @phy_addr: the PHY address + * @mmd: the PHY MMD to access (0 for clause 22 PHYs) + * @reg: the register to read + * @valp: where to store the value + * + * Issues a FW command through the given mailbox to read a PHY register. + */ +int t4_mdio_rd(struct adapter *adap, unsigned int mbox, unsigned int phy_addr, + unsigned int mmd, unsigned int reg, u16 *valp) +{ + int ret; + u32 ldst_addrspace; + struct fw_ldst_cmd c; + + memset(&c, 0, sizeof(c)); + ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_MDIO); + c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F | + ldst_addrspace); + c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); + c.u.mdio.paddr_mmd = cpu_to_be16(FW_LDST_CMD_PADDR_V(phy_addr) | + FW_LDST_CMD_MMD_V(mmd)); + c.u.mdio.raddr = cpu_to_be16(reg); + + ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); + if (ret == 0) + *valp = be16_to_cpu(c.u.mdio.rval); + return ret; +} + +/** + * t4_mdio_wr - write a PHY register through MDIO + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @phy_addr: the PHY address + * @mmd: the PHY MMD to access (0 for clause 22 PHYs) + * @reg: the register to write + * @val: value to write + * + * Issues a FW command through the given mailbox to write a PHY register. + */ +int t4_mdio_wr(struct adapter *adap, unsigned int mbox, unsigned int phy_addr, + unsigned int mmd, unsigned int reg, u16 val) +{ + u32 ldst_addrspace; + struct fw_ldst_cmd c; + + memset(&c, 0, sizeof(c)); + ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_MDIO); + c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + ldst_addrspace); + c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); + c.u.mdio.paddr_mmd = cpu_to_be16(FW_LDST_CMD_PADDR_V(phy_addr) | + FW_LDST_CMD_MMD_V(mmd)); + c.u.mdio.raddr = cpu_to_be16(reg); + c.u.mdio.rval = cpu_to_be16(val); + + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_sge_decode_idma_state - decode the idma state + * @adapter: the adapter + * @state: the state idma is stuck in + */ +void t4_sge_decode_idma_state(struct adapter *adapter, int state) +{ + static const char * const t4_decode[] = { + "IDMA_IDLE", + "IDMA_PUSH_MORE_CPL_FIFO", + "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO", + "Not used", + "IDMA_PHYSADDR_SEND_PCIEHDR", + "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST", + "IDMA_PHYSADDR_SEND_PAYLOAD", + "IDMA_SEND_FIFO_TO_IMSG", + "IDMA_FL_REQ_DATA_FL_PREP", + "IDMA_FL_REQ_DATA_FL", + "IDMA_FL_DROP", + "IDMA_FL_H_REQ_HEADER_FL", + "IDMA_FL_H_SEND_PCIEHDR", + "IDMA_FL_H_PUSH_CPL_FIFO", + "IDMA_FL_H_SEND_CPL", + "IDMA_FL_H_SEND_IP_HDR_FIRST", + "IDMA_FL_H_SEND_IP_HDR", + "IDMA_FL_H_REQ_NEXT_HEADER_FL", + "IDMA_FL_H_SEND_NEXT_PCIEHDR", + "IDMA_FL_H_SEND_IP_HDR_PADDING", + "IDMA_FL_D_SEND_PCIEHDR", + "IDMA_FL_D_SEND_CPL_AND_IP_HDR", + "IDMA_FL_D_REQ_NEXT_DATA_FL", + "IDMA_FL_SEND_PCIEHDR", + "IDMA_FL_PUSH_CPL_FIFO", + "IDMA_FL_SEND_CPL", + "IDMA_FL_SEND_PAYLOAD_FIRST", + "IDMA_FL_SEND_PAYLOAD", + "IDMA_FL_REQ_NEXT_DATA_FL", + "IDMA_FL_SEND_NEXT_PCIEHDR", + "IDMA_FL_SEND_PADDING", + "IDMA_FL_SEND_COMPLETION_TO_IMSG", + "IDMA_FL_SEND_FIFO_TO_IMSG", + "IDMA_FL_REQ_DATAFL_DONE", + "IDMA_FL_REQ_HEADERFL_DONE", + }; + static const char * const t5_decode[] = { + "IDMA_IDLE", + "IDMA_ALMOST_IDLE", + "IDMA_PUSH_MORE_CPL_FIFO", + "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO", + "IDMA_SGEFLRFLUSH_SEND_PCIEHDR", + "IDMA_PHYSADDR_SEND_PCIEHDR", + "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST", + "IDMA_PHYSADDR_SEND_PAYLOAD", + "IDMA_SEND_FIFO_TO_IMSG", + "IDMA_FL_REQ_DATA_FL", + "IDMA_FL_DROP", + "IDMA_FL_DROP_SEND_INC", + "IDMA_FL_H_REQ_HEADER_FL", + "IDMA_FL_H_SEND_PCIEHDR", + "IDMA_FL_H_PUSH_CPL_FIFO", + "IDMA_FL_H_SEND_CPL", + "IDMA_FL_H_SEND_IP_HDR_FIRST", + "IDMA_FL_H_SEND_IP_HDR", + "IDMA_FL_H_REQ_NEXT_HEADER_FL", + "IDMA_FL_H_SEND_NEXT_PCIEHDR", + "IDMA_FL_H_SEND_IP_HDR_PADDING", + "IDMA_FL_D_SEND_PCIEHDR", + "IDMA_FL_D_SEND_CPL_AND_IP_HDR", + "IDMA_FL_D_REQ_NEXT_DATA_FL", + "IDMA_FL_SEND_PCIEHDR", + "IDMA_FL_PUSH_CPL_FIFO", + "IDMA_FL_SEND_CPL", + "IDMA_FL_SEND_PAYLOAD_FIRST", + "IDMA_FL_SEND_PAYLOAD", + "IDMA_FL_REQ_NEXT_DATA_FL", + "IDMA_FL_SEND_NEXT_PCIEHDR", + "IDMA_FL_SEND_PADDING", + "IDMA_FL_SEND_COMPLETION_TO_IMSG", + }; + static const char * const t6_decode[] = { + "IDMA_IDLE", + "IDMA_PUSH_MORE_CPL_FIFO", + "IDMA_PUSH_CPL_MSG_HEADER_TO_FIFO", + "IDMA_SGEFLRFLUSH_SEND_PCIEHDR", + "IDMA_PHYSADDR_SEND_PCIEHDR", + "IDMA_PHYSADDR_SEND_PAYLOAD_FIRST", + "IDMA_PHYSADDR_SEND_PAYLOAD", + "IDMA_FL_REQ_DATA_FL", + "IDMA_FL_DROP", + "IDMA_FL_DROP_SEND_INC", + "IDMA_FL_H_REQ_HEADER_FL", + "IDMA_FL_H_SEND_PCIEHDR", + "IDMA_FL_H_PUSH_CPL_FIFO", + "IDMA_FL_H_SEND_CPL", + "IDMA_FL_H_SEND_IP_HDR_FIRST", + "IDMA_FL_H_SEND_IP_HDR", + "IDMA_FL_H_REQ_NEXT_HEADER_FL", + "IDMA_FL_H_SEND_NEXT_PCIEHDR", + "IDMA_FL_H_SEND_IP_HDR_PADDING", + "IDMA_FL_D_SEND_PCIEHDR", + "IDMA_FL_D_SEND_CPL_AND_IP_HDR", + "IDMA_FL_D_REQ_NEXT_DATA_FL", + "IDMA_FL_SEND_PCIEHDR", + "IDMA_FL_PUSH_CPL_FIFO", + "IDMA_FL_SEND_CPL", + "IDMA_FL_SEND_PAYLOAD_FIRST", + "IDMA_FL_SEND_PAYLOAD", + "IDMA_FL_REQ_NEXT_DATA_FL", + "IDMA_FL_SEND_NEXT_PCIEHDR", + "IDMA_FL_SEND_PADDING", + "IDMA_FL_SEND_COMPLETION_TO_IMSG", + }; + static const u32 sge_regs[] = { + SGE_DEBUG_DATA_LOW_INDEX_2_A, + SGE_DEBUG_DATA_LOW_INDEX_3_A, + SGE_DEBUG_DATA_HIGH_INDEX_10_A, + }; + const char **sge_idma_decode; + int sge_idma_decode_nstates; + int i; + unsigned int chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip); + + /* Select the right set of decode strings to dump depending on the + * adapter chip type. + */ + switch (chip_version) { + case CHELSIO_T4: + sge_idma_decode = (const char **)t4_decode; + sge_idma_decode_nstates = ARRAY_SIZE(t4_decode); + break; + + case CHELSIO_T5: + sge_idma_decode = (const char **)t5_decode; + sge_idma_decode_nstates = ARRAY_SIZE(t5_decode); + break; + + case CHELSIO_T6: + sge_idma_decode = (const char **)t6_decode; + sge_idma_decode_nstates = ARRAY_SIZE(t6_decode); + break; + + default: + dev_err(adapter->pdev_dev, + "Unsupported chip version %d\n", chip_version); + return; + } + + if (is_t4(adapter->params.chip)) { + sge_idma_decode = (const char **)t4_decode; + sge_idma_decode_nstates = ARRAY_SIZE(t4_decode); + } else { + sge_idma_decode = (const char **)t5_decode; + sge_idma_decode_nstates = ARRAY_SIZE(t5_decode); + } + + if (state < sge_idma_decode_nstates) + CH_WARN(adapter, "idma state %s\n", sge_idma_decode[state]); + else + CH_WARN(adapter, "idma state %d unknown\n", state); + + for (i = 0; i < ARRAY_SIZE(sge_regs); i++) + CH_WARN(adapter, "SGE register %#x value %#x\n", + sge_regs[i], t4_read_reg(adapter, sge_regs[i])); +} + +/** + * t4_sge_ctxt_flush - flush the SGE context cache + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @ctxt_type: Egress or Ingress + * + * Issues a FW command through the given mailbox to flush the + * SGE context cache. + */ +int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox, int ctxt_type) +{ + int ret; + u32 ldst_addrspace; + struct fw_ldst_cmd c; + + memset(&c, 0, sizeof(c)); + ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(ctxt_type == CTXT_EGRESS ? + FW_LDST_ADDRSPC_SGE_EGRC : + FW_LDST_ADDRSPC_SGE_INGC); + c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F | + ldst_addrspace); + c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); + c.u.idctxt.msg_ctxtflush = cpu_to_be32(FW_LDST_CMD_CTXTFLUSH_F); + + ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); + return ret; +} + +/** + * t4_read_sge_dbqtimers - read SGE Doorbell Queue Timer values + * @adap: the adapter + * @ndbqtimers: size of the provided SGE Doorbell Queue Timer table + * @dbqtimers: SGE Doorbell Queue Timer table + * + * Reads the SGE Doorbell Queue Timer values into the provided table. + * Returns 0 on success (Firmware and Hardware support this feature), + * an error on failure. + */ +int t4_read_sge_dbqtimers(struct adapter *adap, unsigned int ndbqtimers, + u16 *dbqtimers) +{ + int ret, dbqtimerix; + + ret = 0; + dbqtimerix = 0; + while (dbqtimerix < ndbqtimers) { + int nparams, param; + u32 params[7], vals[7]; + + nparams = ndbqtimers - dbqtimerix; + if (nparams > ARRAY_SIZE(params)) + nparams = ARRAY_SIZE(params); + + for (param = 0; param < nparams; param++) + params[param] = + (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_DBQ_TIMER) | + FW_PARAMS_PARAM_Y_V(dbqtimerix + param)); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, + nparams, params, vals); + if (ret) + break; + + for (param = 0; param < nparams; param++) + dbqtimers[dbqtimerix++] = vals[param]; + } + return ret; +} + +/** + * t4_fw_hello - establish communication with FW + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @evt_mbox: mailbox to receive async FW events + * @master: specifies the caller's willingness to be the device master + * @state: returns the current device state (if non-NULL) + * + * Issues a command to establish communication with FW. Returns either + * an error (negative integer) or the mailbox of the Master PF. + */ +int t4_fw_hello(struct adapter *adap, unsigned int mbox, unsigned int evt_mbox, + enum dev_master master, enum dev_state *state) +{ + int ret; + struct fw_hello_cmd c; + u32 v; + unsigned int master_mbox; + int retries = FW_CMD_HELLO_RETRIES; + +retry: + memset(&c, 0, sizeof(c)); + INIT_CMD(c, HELLO, WRITE); + c.err_to_clearinit = cpu_to_be32( + FW_HELLO_CMD_MASTERDIS_V(master == MASTER_CANT) | + FW_HELLO_CMD_MASTERFORCE_V(master == MASTER_MUST) | + FW_HELLO_CMD_MBMASTER_V(master == MASTER_MUST ? + mbox : FW_HELLO_CMD_MBMASTER_M) | + FW_HELLO_CMD_MBASYNCNOT_V(evt_mbox) | + FW_HELLO_CMD_STAGE_V(fw_hello_cmd_stage_os) | + FW_HELLO_CMD_CLEARINIT_F); + + /* + * Issue the HELLO command to the firmware. If it's not successful + * but indicates that we got a "busy" or "timeout" condition, retry + * the HELLO until we exhaust our retry limit. If we do exceed our + * retry limit, check to see if the firmware left us any error + * information and report that if so. + */ + ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); + if (ret < 0) { + if ((ret == -EBUSY || ret == -ETIMEDOUT) && retries-- > 0) + goto retry; + if (t4_read_reg(adap, PCIE_FW_A) & PCIE_FW_ERR_F) + t4_report_fw_error(adap); + return ret; + } + + v = be32_to_cpu(c.err_to_clearinit); + master_mbox = FW_HELLO_CMD_MBMASTER_G(v); + if (state) { + if (v & FW_HELLO_CMD_ERR_F) + *state = DEV_STATE_ERR; + else if (v & FW_HELLO_CMD_INIT_F) + *state = DEV_STATE_INIT; + else + *state = DEV_STATE_UNINIT; + } + + /* + * If we're not the Master PF then we need to wait around for the + * Master PF Driver to finish setting up the adapter. + * + * Note that we also do this wait if we're a non-Master-capable PF and + * there is no current Master PF; a Master PF may show up momentarily + * and we wouldn't want to fail pointlessly. (This can happen when an + * OS loads lots of different drivers rapidly at the same time). In + * this case, the Master PF returned by the firmware will be + * PCIE_FW_MASTER_M so the test below will work ... + */ + if ((v & (FW_HELLO_CMD_ERR_F|FW_HELLO_CMD_INIT_F)) == 0 && + master_mbox != mbox) { + int waiting = FW_CMD_HELLO_TIMEOUT; + + /* + * Wait for the firmware to either indicate an error or + * initialized state. If we see either of these we bail out + * and report the issue to the caller. If we exhaust the + * "hello timeout" and we haven't exhausted our retries, try + * again. Otherwise bail with a timeout error. + */ + for (;;) { + u32 pcie_fw; + + msleep(50); + waiting -= 50; + + /* + * If neither Error nor Initialized are indicated + * by the firmware keep waiting till we exhaust our + * timeout ... and then retry if we haven't exhausted + * our retries ... + */ + pcie_fw = t4_read_reg(adap, PCIE_FW_A); + if (!(pcie_fw & (PCIE_FW_ERR_F|PCIE_FW_INIT_F))) { + if (waiting <= 0) { + if (retries-- > 0) + goto retry; + + return -ETIMEDOUT; + } + continue; + } + + /* + * We either have an Error or Initialized condition + * report errors preferentially. + */ + if (state) { + if (pcie_fw & PCIE_FW_ERR_F) + *state = DEV_STATE_ERR; + else if (pcie_fw & PCIE_FW_INIT_F) + *state = DEV_STATE_INIT; + } + + /* + * If we arrived before a Master PF was selected and + * there's not a valid Master PF, grab its identity + * for our caller. + */ + if (master_mbox == PCIE_FW_MASTER_M && + (pcie_fw & PCIE_FW_MASTER_VLD_F)) + master_mbox = PCIE_FW_MASTER_G(pcie_fw); + break; + } + } + + return master_mbox; +} + +/** + * t4_fw_bye - end communication with FW + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * + * Issues a command to terminate communication with FW. + */ +int t4_fw_bye(struct adapter *adap, unsigned int mbox) +{ + struct fw_bye_cmd c; + + memset(&c, 0, sizeof(c)); + INIT_CMD(c, BYE, WRITE); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_early_init - ask FW to initialize the device + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * + * Issues a command to FW to partially initialize the device. This + * performs initialization that generally doesn't depend on user input. + */ +int t4_early_init(struct adapter *adap, unsigned int mbox) +{ + struct fw_initialize_cmd c; + + memset(&c, 0, sizeof(c)); + INIT_CMD(c, INITIALIZE, WRITE); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_fw_reset - issue a reset to FW + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @reset: specifies the type of reset to perform + * + * Issues a reset command of the specified type to FW. + */ +int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset) +{ + struct fw_reset_cmd c; + + memset(&c, 0, sizeof(c)); + INIT_CMD(c, RESET, WRITE); + c.val = cpu_to_be32(reset); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_fw_halt - issue a reset/halt to FW and put uP into RESET + * @adap: the adapter + * @mbox: mailbox to use for the FW RESET command (if desired) + * @force: force uP into RESET even if FW RESET command fails + * + * Issues a RESET command to firmware (if desired) with a HALT indication + * and then puts the microprocessor into RESET state. The RESET command + * will only be issued if a legitimate mailbox is provided (mbox <= + * PCIE_FW_MASTER_M). + * + * This is generally used in order for the host to safely manipulate the + * adapter without fear of conflicting with whatever the firmware might + * be doing. The only way out of this state is to RESTART the firmware + * ... + */ +static int t4_fw_halt(struct adapter *adap, unsigned int mbox, int force) +{ + int ret = 0; + + /* + * If a legitimate mailbox is provided, issue a RESET command + * with a HALT indication. + */ + if (mbox <= PCIE_FW_MASTER_M) { + struct fw_reset_cmd c; + + memset(&c, 0, sizeof(c)); + INIT_CMD(c, RESET, WRITE); + c.val = cpu_to_be32(PIORST_F | PIORSTMODE_F); + c.halt_pkd = cpu_to_be32(FW_RESET_CMD_HALT_F); + ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); + } + + /* + * Normally we won't complete the operation if the firmware RESET + * command fails but if our caller insists we'll go ahead and put the + * uP into RESET. This can be useful if the firmware is hung or even + * missing ... We'll have to take the risk of putting the uP into + * RESET without the cooperation of firmware in that case. + * + * We also force the firmware's HALT flag to be on in case we bypassed + * the firmware RESET command above or we're dealing with old firmware + * which doesn't have the HALT capability. This will serve as a flag + * for the incoming firmware to know that it's coming out of a HALT + * rather than a RESET ... if it's new enough to understand that ... + */ + if (ret == 0 || force) { + t4_set_reg_field(adap, CIM_BOOT_CFG_A, UPCRST_F, UPCRST_F); + t4_set_reg_field(adap, PCIE_FW_A, PCIE_FW_HALT_F, + PCIE_FW_HALT_F); + } + + /* + * And we always return the result of the firmware RESET command + * even when we force the uP into RESET ... + */ + return ret; +} + +/** + * t4_fw_restart - restart the firmware by taking the uP out of RESET + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @reset: if we want to do a RESET to restart things + * + * Restart firmware previously halted by t4_fw_halt(). On successful + * return the previous PF Master remains as the new PF Master and there + * is no need to issue a new HELLO command, etc. + * + * We do this in two ways: + * + * 1. If we're dealing with newer firmware we'll simply want to take + * the chip's microprocessor out of RESET. This will cause the + * firmware to start up from its start vector. And then we'll loop + * until the firmware indicates it's started again (PCIE_FW.HALT + * reset to 0) or we timeout. + * + * 2. If we're dealing with older firmware then we'll need to RESET + * the chip since older firmware won't recognize the PCIE_FW.HALT + * flag and automatically RESET itself on startup. + */ +static int t4_fw_restart(struct adapter *adap, unsigned int mbox, int reset) +{ + if (reset) { + /* + * Since we're directing the RESET instead of the firmware + * doing it automatically, we need to clear the PCIE_FW.HALT + * bit. + */ + t4_set_reg_field(adap, PCIE_FW_A, PCIE_FW_HALT_F, 0); + + /* + * If we've been given a valid mailbox, first try to get the + * firmware to do the RESET. If that works, great and we can + * return success. Otherwise, if we haven't been given a + * valid mailbox or the RESET command failed, fall back to + * hitting the chip with a hammer. + */ + if (mbox <= PCIE_FW_MASTER_M) { + t4_set_reg_field(adap, CIM_BOOT_CFG_A, UPCRST_F, 0); + msleep(100); + if (t4_fw_reset(adap, mbox, + PIORST_F | PIORSTMODE_F) == 0) + return 0; + } + + t4_write_reg(adap, PL_RST_A, PIORST_F | PIORSTMODE_F); + msleep(2000); + } else { + int ms; + + t4_set_reg_field(adap, CIM_BOOT_CFG_A, UPCRST_F, 0); + for (ms = 0; ms < FW_CMD_MAX_TIMEOUT; ) { + if (!(t4_read_reg(adap, PCIE_FW_A) & PCIE_FW_HALT_F)) + return 0; + msleep(100); + ms += 100; + } + return -ETIMEDOUT; + } + return 0; +} + +/** + * t4_fw_upgrade - perform all of the steps necessary to upgrade FW + * @adap: the adapter + * @mbox: mailbox to use for the FW RESET command (if desired) + * @fw_data: the firmware image to write + * @size: image size + * @force: force upgrade even if firmware doesn't cooperate + * + * Perform all of the steps necessary for upgrading an adapter's + * firmware image. Normally this requires the cooperation of the + * existing firmware in order to halt all existing activities + * but if an invalid mailbox token is passed in we skip that step + * (though we'll still put the adapter microprocessor into RESET in + * that case). + * + * On successful return the new firmware will have been loaded and + * the adapter will have been fully RESET losing all previous setup + * state. On unsuccessful return the adapter may be completely hosed ... + * positive errno indicates that the adapter is ~probably~ intact, a + * negative errno indicates that things are looking bad ... + */ +int t4_fw_upgrade(struct adapter *adap, unsigned int mbox, + const u8 *fw_data, unsigned int size, int force) +{ + const struct fw_hdr *fw_hdr = (const struct fw_hdr *)fw_data; + int reset, ret; + + if (!t4_fw_matches_chip(adap, fw_hdr)) + return -EINVAL; + + /* Disable CXGB4_FW_OK flag so that mbox commands with CXGB4_FW_OK flag + * set wont be sent when we are flashing FW. + */ + adap->flags &= ~CXGB4_FW_OK; + + ret = t4_fw_halt(adap, mbox, force); + if (ret < 0 && !force) + goto out; + + ret = t4_load_fw(adap, fw_data, size); + if (ret < 0) + goto out; + + /* + * If there was a Firmware Configuration File stored in FLASH, + * there's a good chance that it won't be compatible with the new + * Firmware. In order to prevent difficult to diagnose adapter + * initialization issues, we clear out the Firmware Configuration File + * portion of the FLASH . The user will need to re-FLASH a new + * Firmware Configuration File which is compatible with the new + * Firmware if that's desired. + */ + (void)t4_load_cfg(adap, NULL, 0); + + /* + * Older versions of the firmware don't understand the new + * PCIE_FW.HALT flag and so won't know to perform a RESET when they + * restart. So for newly loaded older firmware we'll have to do the + * RESET for it so it starts up on a clean slate. We can tell if + * the newly loaded firmware will handle this right by checking + * its header flags to see if it advertises the capability. + */ + reset = ((be32_to_cpu(fw_hdr->flags) & FW_HDR_FLAGS_RESET_HALT) == 0); + ret = t4_fw_restart(adap, mbox, reset); + + /* Grab potentially new Firmware Device Log parameters so we can see + * how healthy the new Firmware is. It's okay to contact the new + * Firmware for these parameters even though, as far as it's + * concerned, we've never said "HELLO" to it ... + */ + (void)t4_init_devlog_params(adap); +out: + adap->flags |= CXGB4_FW_OK; + return ret; +} + +/** + * t4_fl_pkt_align - return the fl packet alignment + * @adap: the adapter + * + * T4 has a single field to specify the packing and padding boundary. + * T5 onwards has separate fields for this and hence the alignment for + * next packet offset is maximum of these two. + * + */ +int t4_fl_pkt_align(struct adapter *adap) +{ + u32 sge_control, sge_control2; + unsigned int ingpadboundary, ingpackboundary, fl_align, ingpad_shift; + + sge_control = t4_read_reg(adap, SGE_CONTROL_A); + + /* T4 uses a single control field to specify both the PCIe Padding and + * Packing Boundary. T5 introduced the ability to specify these + * separately. The actual Ingress Packet Data alignment boundary + * within Packed Buffer Mode is the maximum of these two + * specifications. (Note that it makes no real practical sense to + * have the Padding Boundary be larger than the Packing Boundary but you + * could set the chip up that way and, in fact, legacy T4 code would + * end doing this because it would initialize the Padding Boundary and + * leave the Packing Boundary initialized to 0 (16 bytes).) + * Padding Boundary values in T6 starts from 8B, + * where as it is 32B for T4 and T5. + */ + if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) + ingpad_shift = INGPADBOUNDARY_SHIFT_X; + else + ingpad_shift = T6_INGPADBOUNDARY_SHIFT_X; + + ingpadboundary = 1 << (INGPADBOUNDARY_G(sge_control) + ingpad_shift); + + fl_align = ingpadboundary; + if (!is_t4(adap->params.chip)) { + /* T5 has a weird interpretation of one of the PCIe Packing + * Boundary values. No idea why ... + */ + sge_control2 = t4_read_reg(adap, SGE_CONTROL2_A); + ingpackboundary = INGPACKBOUNDARY_G(sge_control2); + if (ingpackboundary == INGPACKBOUNDARY_16B_X) + ingpackboundary = 16; + else + ingpackboundary = 1 << (ingpackboundary + + INGPACKBOUNDARY_SHIFT_X); + + fl_align = max(ingpadboundary, ingpackboundary); + } + return fl_align; +} + +/** + * t4_fixup_host_params - fix up host-dependent parameters + * @adap: the adapter + * @page_size: the host's Base Page Size + * @cache_line_size: the host's Cache Line Size + * + * Various registers in T4 contain values which are dependent on the + * host's Base Page and Cache Line Sizes. This function will fix all of + * those registers with the appropriate values as passed in ... + */ +int t4_fixup_host_params(struct adapter *adap, unsigned int page_size, + unsigned int cache_line_size) +{ + unsigned int page_shift = fls(page_size) - 1; + unsigned int sge_hps = page_shift - 10; + unsigned int stat_len = cache_line_size > 64 ? 128 : 64; + unsigned int fl_align = cache_line_size < 32 ? 32 : cache_line_size; + unsigned int fl_align_log = fls(fl_align) - 1; + + t4_write_reg(adap, SGE_HOST_PAGE_SIZE_A, + HOSTPAGESIZEPF0_V(sge_hps) | + HOSTPAGESIZEPF1_V(sge_hps) | + HOSTPAGESIZEPF2_V(sge_hps) | + HOSTPAGESIZEPF3_V(sge_hps) | + HOSTPAGESIZEPF4_V(sge_hps) | + HOSTPAGESIZEPF5_V(sge_hps) | + HOSTPAGESIZEPF6_V(sge_hps) | + HOSTPAGESIZEPF7_V(sge_hps)); + + if (is_t4(adap->params.chip)) { + t4_set_reg_field(adap, SGE_CONTROL_A, + INGPADBOUNDARY_V(INGPADBOUNDARY_M) | + EGRSTATUSPAGESIZE_F, + INGPADBOUNDARY_V(fl_align_log - + INGPADBOUNDARY_SHIFT_X) | + EGRSTATUSPAGESIZE_V(stat_len != 64)); + } else { + unsigned int pack_align; + unsigned int ingpad, ingpack; + + /* T5 introduced the separation of the Free List Padding and + * Packing Boundaries. Thus, we can select a smaller Padding + * Boundary to avoid uselessly chewing up PCIe Link and Memory + * Bandwidth, and use a Packing Boundary which is large enough + * to avoid false sharing between CPUs, etc. + * + * For the PCI Link, the smaller the Padding Boundary the + * better. For the Memory Controller, a smaller Padding + * Boundary is better until we cross under the Memory Line + * Size (the minimum unit of transfer to/from Memory). If we + * have a Padding Boundary which is smaller than the Memory + * Line Size, that'll involve a Read-Modify-Write cycle on the + * Memory Controller which is never good. + */ + + /* We want the Packing Boundary to be based on the Cache Line + * Size in order to help avoid False Sharing performance + * issues between CPUs, etc. We also want the Packing + * Boundary to incorporate the PCI-E Maximum Payload Size. We + * get best performance when the Packing Boundary is a + * multiple of the Maximum Payload Size. + */ + pack_align = fl_align; + if (pci_is_pcie(adap->pdev)) { + unsigned int mps, mps_log; + u16 devctl; + + /* The PCIe Device Control Maximum Payload Size field + * [bits 7:5] encodes sizes as powers of 2 starting at + * 128 bytes. + */ + pcie_capability_read_word(adap->pdev, PCI_EXP_DEVCTL, + &devctl); + mps_log = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5) + 7; + mps = 1 << mps_log; + if (mps > pack_align) + pack_align = mps; + } + + /* N.B. T5/T6 have a crazy special interpretation of the "0" + * value for the Packing Boundary. This corresponds to 16 + * bytes instead of the expected 32 bytes. So if we want 32 + * bytes, the best we can really do is 64 bytes ... + */ + if (pack_align <= 16) { + ingpack = INGPACKBOUNDARY_16B_X; + fl_align = 16; + } else if (pack_align == 32) { + ingpack = INGPACKBOUNDARY_64B_X; + fl_align = 64; + } else { + unsigned int pack_align_log = fls(pack_align) - 1; + + ingpack = pack_align_log - INGPACKBOUNDARY_SHIFT_X; + fl_align = pack_align; + } + + /* Use the smallest Ingress Padding which isn't smaller than + * the Memory Controller Read/Write Size. We'll take that as + * being 8 bytes since we don't know of any system with a + * wider Memory Controller Bus Width. + */ + if (is_t5(adap->params.chip)) + ingpad = INGPADBOUNDARY_32B_X; + else + ingpad = T6_INGPADBOUNDARY_8B_X; + + t4_set_reg_field(adap, SGE_CONTROL_A, + INGPADBOUNDARY_V(INGPADBOUNDARY_M) | + EGRSTATUSPAGESIZE_F, + INGPADBOUNDARY_V(ingpad) | + EGRSTATUSPAGESIZE_V(stat_len != 64)); + t4_set_reg_field(adap, SGE_CONTROL2_A, + INGPACKBOUNDARY_V(INGPACKBOUNDARY_M), + INGPACKBOUNDARY_V(ingpack)); + } + /* + * Adjust various SGE Free List Host Buffer Sizes. + * + * This is something of a crock since we're using fixed indices into + * the array which are also known by the sge.c code and the T4 + * Firmware Configuration File. We need to come up with a much better + * approach to managing this array. For now, the first four entries + * are: + * + * 0: Host Page Size + * 1: 64KB + * 2: Buffer size corresponding to 1500 byte MTU (unpacked mode) + * 3: Buffer size corresponding to 9000 byte MTU (unpacked mode) + * + * For the single-MTU buffers in unpacked mode we need to include + * space for the SGE Control Packet Shift, 14 byte Ethernet header, + * possible 4 byte VLAN tag, all rounded up to the next Ingress Packet + * Padding boundary. All of these are accommodated in the Factory + * Default Firmware Configuration File but we need to adjust it for + * this host's cache line size. + */ + t4_write_reg(adap, SGE_FL_BUFFER_SIZE0_A, page_size); + t4_write_reg(adap, SGE_FL_BUFFER_SIZE2_A, + (t4_read_reg(adap, SGE_FL_BUFFER_SIZE2_A) + fl_align-1) + & ~(fl_align-1)); + t4_write_reg(adap, SGE_FL_BUFFER_SIZE3_A, + (t4_read_reg(adap, SGE_FL_BUFFER_SIZE3_A) + fl_align-1) + & ~(fl_align-1)); + + t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(page_shift - 12)); + + return 0; +} + +/** + * t4_fw_initialize - ask FW to initialize the device + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * + * Issues a command to FW to partially initialize the device. This + * performs initialization that generally doesn't depend on user input. + */ +int t4_fw_initialize(struct adapter *adap, unsigned int mbox) +{ + struct fw_initialize_cmd c; + + memset(&c, 0, sizeof(c)); + INIT_CMD(c, INITIALIZE, WRITE); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_query_params_rw - query FW or device parameters + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @pf: the PF + * @vf: the VF + * @nparams: the number of parameters + * @params: the parameter names + * @val: the parameter values + * @rw: Write and read flag + * @sleep_ok: if true, we may sleep awaiting mbox cmd completion + * + * Reads the value of FW or device parameters. Up to 7 parameters can be + * queried at once. + */ +int t4_query_params_rw(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int nparams, const u32 *params, + u32 *val, int rw, bool sleep_ok) +{ + int i, ret; + struct fw_params_cmd c; + __be32 *p = &c.param[0].mnem; + + if (nparams > 7) + return -EINVAL; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PARAMS_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F | + FW_PARAMS_CMD_PFN_V(pf) | + FW_PARAMS_CMD_VFN_V(vf)); + c.retval_len16 = cpu_to_be32(FW_LEN16(c)); + + for (i = 0; i < nparams; i++) { + *p++ = cpu_to_be32(*params++); + if (rw) + *p = cpu_to_be32(*(val + i)); + p++; + } + + ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), &c, sleep_ok); + if (ret == 0) + for (i = 0, p = &c.param[0].val; i < nparams; i++, p += 2) + *val++ = be32_to_cpu(*p); + return ret; +} + +int t4_query_params(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int nparams, const u32 *params, + u32 *val) +{ + return t4_query_params_rw(adap, mbox, pf, vf, nparams, params, val, 0, + true); +} + +int t4_query_params_ns(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int nparams, const u32 *params, + u32 *val) +{ + return t4_query_params_rw(adap, mbox, pf, vf, nparams, params, val, 0, + false); +} + +/** + * t4_set_params_timeout - sets FW or device parameters + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @pf: the PF + * @vf: the VF + * @nparams: the number of parameters + * @params: the parameter names + * @val: the parameter values + * @timeout: the timeout time + * + * Sets the value of FW or device parameters. Up to 7 parameters can be + * specified at once. + */ +int t4_set_params_timeout(struct adapter *adap, unsigned int mbox, + unsigned int pf, unsigned int vf, + unsigned int nparams, const u32 *params, + const u32 *val, int timeout) +{ + struct fw_params_cmd c; + __be32 *p = &c.param[0].mnem; + + if (nparams > 7) + return -EINVAL; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PARAMS_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_PARAMS_CMD_PFN_V(pf) | + FW_PARAMS_CMD_VFN_V(vf)); + c.retval_len16 = cpu_to_be32(FW_LEN16(c)); + + while (nparams--) { + *p++ = cpu_to_be32(*params++); + *p++ = cpu_to_be32(*val++); + } + + return t4_wr_mbox_timeout(adap, mbox, &c, sizeof(c), NULL, timeout); +} + +/** + * t4_set_params - sets FW or device parameters + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @pf: the PF + * @vf: the VF + * @nparams: the number of parameters + * @params: the parameter names + * @val: the parameter values + * + * Sets the value of FW or device parameters. Up to 7 parameters can be + * specified at once. + */ +int t4_set_params(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int nparams, const u32 *params, + const u32 *val) +{ + return t4_set_params_timeout(adap, mbox, pf, vf, nparams, params, val, + FW_CMD_MAX_TIMEOUT); +} + +/** + * t4_cfg_pfvf - configure PF/VF resource limits + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @pf: the PF being configured + * @vf: the VF being configured + * @txq: the max number of egress queues + * @txq_eth_ctrl: the max number of egress Ethernet or control queues + * @rxqi: the max number of interrupt-capable ingress queues + * @rxq: the max number of interruptless ingress queues + * @tc: the PCI traffic class + * @vi: the max number of virtual interfaces + * @cmask: the channel access rights mask for the PF/VF + * @pmask: the port access rights mask for the PF/VF + * @nexact: the maximum number of exact MPS filters + * @rcaps: read capabilities + * @wxcaps: write/execute capabilities + * + * Configures resource limits and capabilities for a physical or virtual + * function. + */ +int t4_cfg_pfvf(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int txq, unsigned int txq_eth_ctrl, + unsigned int rxqi, unsigned int rxq, unsigned int tc, + unsigned int vi, unsigned int cmask, unsigned int pmask, + unsigned int nexact, unsigned int rcaps, unsigned int wxcaps) +{ + struct fw_pfvf_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PFVF_CMD) | FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | FW_PFVF_CMD_PFN_V(pf) | + FW_PFVF_CMD_VFN_V(vf)); + c.retval_len16 = cpu_to_be32(FW_LEN16(c)); + c.niqflint_niq = cpu_to_be32(FW_PFVF_CMD_NIQFLINT_V(rxqi) | + FW_PFVF_CMD_NIQ_V(rxq)); + c.type_to_neq = cpu_to_be32(FW_PFVF_CMD_CMASK_V(cmask) | + FW_PFVF_CMD_PMASK_V(pmask) | + FW_PFVF_CMD_NEQ_V(txq)); + c.tc_to_nexactf = cpu_to_be32(FW_PFVF_CMD_TC_V(tc) | + FW_PFVF_CMD_NVI_V(vi) | + FW_PFVF_CMD_NEXACTF_V(nexact)); + c.r_caps_to_nethctrl = cpu_to_be32(FW_PFVF_CMD_R_CAPS_V(rcaps) | + FW_PFVF_CMD_WX_CAPS_V(wxcaps) | + FW_PFVF_CMD_NETHCTRL_V(txq_eth_ctrl)); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_alloc_vi - allocate a virtual interface + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @port: physical port associated with the VI + * @pf: the PF owning the VI + * @vf: the VF owning the VI + * @nmac: number of MAC addresses needed (1 to 5) + * @mac: the MAC addresses of the VI + * @rss_size: size of RSS table slice associated with this VI + * @vivld: the destination to store the VI Valid value. + * @vin: the destination to store the VIN value. + * + * Allocates a virtual interface for the given physical port. If @mac is + * not %NULL it contains the MAC addresses of the VI as assigned by FW. + * @mac should be large enough to hold @nmac Ethernet addresses, they are + * stored consecutively so the space needed is @nmac * 6 bytes. + * Returns a negative error number or the non-negative VI id. + */ +int t4_alloc_vi(struct adapter *adap, unsigned int mbox, unsigned int port, + unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac, + unsigned int *rss_size, u8 *vivld, u8 *vin) +{ + int ret; + struct fw_vi_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_VI_CMD) | FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | FW_CMD_EXEC_F | + FW_VI_CMD_PFN_V(pf) | FW_VI_CMD_VFN_V(vf)); + c.alloc_to_len16 = cpu_to_be32(FW_VI_CMD_ALLOC_F | FW_LEN16(c)); + c.portid_pkd = FW_VI_CMD_PORTID_V(port); + c.nmac = nmac - 1; + + ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); + if (ret) + return ret; + + if (mac) { + memcpy(mac, c.mac, sizeof(c.mac)); + switch (nmac) { + case 5: + memcpy(mac + 24, c.nmac3, sizeof(c.nmac3)); + fallthrough; + case 4: + memcpy(mac + 18, c.nmac2, sizeof(c.nmac2)); + fallthrough; + case 3: + memcpy(mac + 12, c.nmac1, sizeof(c.nmac1)); + fallthrough; + case 2: + memcpy(mac + 6, c.nmac0, sizeof(c.nmac0)); + } + } + if (rss_size) + *rss_size = FW_VI_CMD_RSSSIZE_G(be16_to_cpu(c.rsssize_pkd)); + + if (vivld) + *vivld = FW_VI_CMD_VFVLD_G(be32_to_cpu(c.alloc_to_len16)); + + if (vin) + *vin = FW_VI_CMD_VIN_G(be32_to_cpu(c.alloc_to_len16)); + + return FW_VI_CMD_VIID_G(be16_to_cpu(c.type_viid)); +} + +/** + * t4_free_vi - free a virtual interface + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @pf: the PF owning the VI + * @vf: the VF owning the VI + * @viid: virtual interface identifiler + * + * Free a previously allocated virtual interface. + */ +int t4_free_vi(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int viid) +{ + struct fw_vi_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_VI_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_EXEC_F | + FW_VI_CMD_PFN_V(pf) | + FW_VI_CMD_VFN_V(vf)); + c.alloc_to_len16 = cpu_to_be32(FW_VI_CMD_FREE_F | FW_LEN16(c)); + c.type_viid = cpu_to_be16(FW_VI_CMD_VIID_V(viid)); + + return t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); +} + +/** + * t4_set_rxmode - set Rx properties of a virtual interface + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @viid: the VI id + * @viid_mirror: the mirror VI id + * @mtu: the new MTU or -1 + * @promisc: 1 to enable promiscuous mode, 0 to disable it, -1 no change + * @all_multi: 1 to enable all-multi mode, 0 to disable it, -1 no change + * @bcast: 1 to enable broadcast Rx, 0 to disable it, -1 no change + * @vlanex: 1 to enable HW VLAN extraction, 0 to disable it, -1 no change + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Sets Rx properties of a virtual interface. + */ +int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid, + unsigned int viid_mirror, int mtu, int promisc, int all_multi, + int bcast, int vlanex, bool sleep_ok) +{ + struct fw_vi_rxmode_cmd c, c_mirror; + int ret; + + /* convert to FW values */ + if (mtu < 0) + mtu = FW_RXMODE_MTU_NO_CHG; + if (promisc < 0) + promisc = FW_VI_RXMODE_CMD_PROMISCEN_M; + if (all_multi < 0) + all_multi = FW_VI_RXMODE_CMD_ALLMULTIEN_M; + if (bcast < 0) + bcast = FW_VI_RXMODE_CMD_BROADCASTEN_M; + if (vlanex < 0) + vlanex = FW_VI_RXMODE_CMD_VLANEXEN_M; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_RXMODE_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_VI_RXMODE_CMD_VIID_V(viid)); + c.retval_len16 = cpu_to_be32(FW_LEN16(c)); + c.mtu_to_vlanexen = + cpu_to_be32(FW_VI_RXMODE_CMD_MTU_V(mtu) | + FW_VI_RXMODE_CMD_PROMISCEN_V(promisc) | + FW_VI_RXMODE_CMD_ALLMULTIEN_V(all_multi) | + FW_VI_RXMODE_CMD_BROADCASTEN_V(bcast) | + FW_VI_RXMODE_CMD_VLANEXEN_V(vlanex)); + + if (viid_mirror) { + memcpy(&c_mirror, &c, sizeof(c_mirror)); + c_mirror.op_to_viid = + cpu_to_be32(FW_CMD_OP_V(FW_VI_RXMODE_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_VI_RXMODE_CMD_VIID_V(viid_mirror)); + } + + ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok); + if (ret) + return ret; + + if (viid_mirror) + ret = t4_wr_mbox_meat(adap, mbox, &c_mirror, sizeof(c_mirror), + NULL, sleep_ok); + + return ret; +} + +/** + * t4_free_encap_mac_filt - frees MPS entry at given index + * @adap: the adapter + * @viid: the VI id + * @idx: index of MPS entry to be freed + * @sleep_ok: call is allowed to sleep + * + * Frees the MPS entry at supplied index + * + * Returns a negative error number or zero on success + */ +int t4_free_encap_mac_filt(struct adapter *adap, unsigned int viid, + int idx, bool sleep_ok) +{ + struct fw_vi_mac_exact *p; + struct fw_vi_mac_cmd c; + int ret = 0; + u32 exact; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_CMD_EXEC_V(0) | + FW_VI_MAC_CMD_VIID_V(viid)); + exact = FW_VI_MAC_CMD_ENTRY_TYPE_V(FW_VI_MAC_TYPE_EXACTMAC); + c.freemacs_to_len16 = cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(0) | + exact | + FW_CMD_LEN16_V(1)); + p = c.u.exact; + p->valid_to_idx = cpu_to_be16(FW_VI_MAC_CMD_VALID_F | + FW_VI_MAC_CMD_IDX_V(idx)); + eth_zero_addr(p->macaddr); + ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok); + return ret; +} + +/** + * t4_free_raw_mac_filt - Frees a raw mac entry in mps tcam + * @adap: the adapter + * @viid: the VI id + * @addr: the MAC address + * @mask: the mask + * @idx: index of the entry in mps tcam + * @lookup_type: MAC address for inner (1) or outer (0) header + * @port_id: the port index + * @sleep_ok: call is allowed to sleep + * + * Removes the mac entry at the specified index using raw mac interface. + * + * Returns a negative error number on failure. + */ +int t4_free_raw_mac_filt(struct adapter *adap, unsigned int viid, + const u8 *addr, const u8 *mask, unsigned int idx, + u8 lookup_type, u8 port_id, bool sleep_ok) +{ + struct fw_vi_mac_cmd c; + struct fw_vi_mac_raw *p = &c.u.raw; + u32 val; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_CMD_EXEC_V(0) | + FW_VI_MAC_CMD_VIID_V(viid)); + val = FW_CMD_LEN16_V(1) | + FW_VI_MAC_CMD_ENTRY_TYPE_V(FW_VI_MAC_TYPE_RAW); + c.freemacs_to_len16 = cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(0) | + FW_CMD_LEN16_V(val)); + + p->raw_idx_pkd = cpu_to_be32(FW_VI_MAC_CMD_RAW_IDX_V(idx) | + FW_VI_MAC_ID_BASED_FREE); + + /* Lookup Type. Outer header: 0, Inner header: 1 */ + p->data0_pkd = cpu_to_be32(DATALKPTYPE_V(lookup_type) | + DATAPORTNUM_V(port_id)); + /* Lookup mask and port mask */ + p->data0m_pkd = cpu_to_be64(DATALKPTYPE_V(DATALKPTYPE_M) | + DATAPORTNUM_V(DATAPORTNUM_M)); + + /* Copy the address and the mask */ + memcpy((u8 *)&p->data1[0] + 2, addr, ETH_ALEN); + memcpy((u8 *)&p->data1m[0] + 2, mask, ETH_ALEN); + + return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok); +} + +/** + * t4_alloc_encap_mac_filt - Adds a mac entry in mps tcam with VNI support + * @adap: the adapter + * @viid: the VI id + * @addr: the MAC address + * @mask: the mask + * @vni: the VNI id for the tunnel protocol + * @vni_mask: mask for the VNI id + * @dip_hit: to enable DIP match for the MPS entry + * @lookup_type: MAC address for inner (1) or outer (0) header + * @sleep_ok: call is allowed to sleep + * + * Allocates an MPS entry with specified MAC address and VNI value. + * + * Returns a negative error number or the allocated index for this mac. + */ +int t4_alloc_encap_mac_filt(struct adapter *adap, unsigned int viid, + const u8 *addr, const u8 *mask, unsigned int vni, + unsigned int vni_mask, u8 dip_hit, u8 lookup_type, + bool sleep_ok) +{ + struct fw_vi_mac_cmd c; + struct fw_vi_mac_vni *p = c.u.exact_vni; + int ret = 0; + u32 val; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_VI_MAC_CMD_VIID_V(viid)); + val = FW_CMD_LEN16_V(1) | + FW_VI_MAC_CMD_ENTRY_TYPE_V(FW_VI_MAC_TYPE_EXACTMAC_VNI); + c.freemacs_to_len16 = cpu_to_be32(val); + p->valid_to_idx = cpu_to_be16(FW_VI_MAC_CMD_VALID_F | + FW_VI_MAC_CMD_IDX_V(FW_VI_MAC_ADD_MAC)); + memcpy(p->macaddr, addr, sizeof(p->macaddr)); + memcpy(p->macaddr_mask, mask, sizeof(p->macaddr_mask)); + + p->lookup_type_to_vni = + cpu_to_be32(FW_VI_MAC_CMD_VNI_V(vni) | + FW_VI_MAC_CMD_DIP_HIT_V(dip_hit) | + FW_VI_MAC_CMD_LOOKUP_TYPE_V(lookup_type)); + p->vni_mask_pkd = cpu_to_be32(FW_VI_MAC_CMD_VNI_MASK_V(vni_mask)); + ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok); + if (ret == 0) + ret = FW_VI_MAC_CMD_IDX_G(be16_to_cpu(p->valid_to_idx)); + return ret; +} + +/** + * t4_alloc_raw_mac_filt - Adds a mac entry in mps tcam + * @adap: the adapter + * @viid: the VI id + * @addr: the MAC address + * @mask: the mask + * @idx: index at which to add this entry + * @lookup_type: MAC address for inner (1) or outer (0) header + * @port_id: the port index + * @sleep_ok: call is allowed to sleep + * + * Adds the mac entry at the specified index using raw mac interface. + * + * Returns a negative error number or the allocated index for this mac. + */ +int t4_alloc_raw_mac_filt(struct adapter *adap, unsigned int viid, + const u8 *addr, const u8 *mask, unsigned int idx, + u8 lookup_type, u8 port_id, bool sleep_ok) +{ + int ret = 0; + struct fw_vi_mac_cmd c; + struct fw_vi_mac_raw *p = &c.u.raw; + u32 val; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_VI_MAC_CMD_VIID_V(viid)); + val = FW_CMD_LEN16_V(1) | + FW_VI_MAC_CMD_ENTRY_TYPE_V(FW_VI_MAC_TYPE_RAW); + c.freemacs_to_len16 = cpu_to_be32(val); + + /* Specify that this is an inner mac address */ + p->raw_idx_pkd = cpu_to_be32(FW_VI_MAC_CMD_RAW_IDX_V(idx)); + + /* Lookup Type. Outer header: 0, Inner header: 1 */ + p->data0_pkd = cpu_to_be32(DATALKPTYPE_V(lookup_type) | + DATAPORTNUM_V(port_id)); + /* Lookup mask and port mask */ + p->data0m_pkd = cpu_to_be64(DATALKPTYPE_V(DATALKPTYPE_M) | + DATAPORTNUM_V(DATAPORTNUM_M)); + + /* Copy the address and the mask */ + memcpy((u8 *)&p->data1[0] + 2, addr, ETH_ALEN); + memcpy((u8 *)&p->data1m[0] + 2, mask, ETH_ALEN); + + ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok); + if (ret == 0) { + ret = FW_VI_MAC_CMD_RAW_IDX_G(be32_to_cpu(p->raw_idx_pkd)); + if (ret != idx) + ret = -ENOMEM; + } + + return ret; +} + +/** + * t4_alloc_mac_filt - allocates exact-match filters for MAC addresses + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @viid: the VI id + * @free: if true any existing filters for this VI id are first removed + * @naddr: the number of MAC addresses to allocate filters for (up to 7) + * @addr: the MAC address(es) + * @idx: where to store the index of each allocated filter + * @hash: pointer to hash address filter bitmap + * @sleep_ok: call is allowed to sleep + * + * Allocates an exact-match filter for each of the supplied addresses and + * sets it to the corresponding address. If @idx is not %NULL it should + * have at least @naddr entries, each of which will be set to the index of + * the filter allocated for the corresponding MAC address. If a filter + * could not be allocated for an address its index is set to 0xffff. + * If @hash is not %NULL addresses that fail to allocate an exact filter + * are hashed and update the hash filter bitmap pointed at by @hash. + * + * Returns a negative error number or the number of filters allocated. + */ +int t4_alloc_mac_filt(struct adapter *adap, unsigned int mbox, + unsigned int viid, bool free, unsigned int naddr, + const u8 **addr, u16 *idx, u64 *hash, bool sleep_ok) +{ + int offset, ret = 0; + struct fw_vi_mac_cmd c; + unsigned int nfilters = 0; + unsigned int max_naddr = adap->params.arch.mps_tcam_size; + unsigned int rem = naddr; + + if (naddr > max_naddr) + return -EINVAL; + + for (offset = 0; offset < naddr ; /**/) { + unsigned int fw_naddr = (rem < ARRAY_SIZE(c.u.exact) ? + rem : ARRAY_SIZE(c.u.exact)); + size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd, + u.exact[fw_naddr]), 16); + struct fw_vi_mac_exact *p; + int i; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_CMD_EXEC_V(free) | + FW_VI_MAC_CMD_VIID_V(viid)); + c.freemacs_to_len16 = + cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(free) | + FW_CMD_LEN16_V(len16)); + + for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) { + p->valid_to_idx = + cpu_to_be16(FW_VI_MAC_CMD_VALID_F | + FW_VI_MAC_CMD_IDX_V( + FW_VI_MAC_ADD_MAC)); + memcpy(p->macaddr, addr[offset + i], + sizeof(p->macaddr)); + } + + /* It's okay if we run out of space in our MAC address arena. + * Some of the addresses we submit may get stored so we need + * to run through the reply to see what the results were ... + */ + ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), &c, sleep_ok); + if (ret && ret != -FW_ENOMEM) + break; + + for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) { + u16 index = FW_VI_MAC_CMD_IDX_G( + be16_to_cpu(p->valid_to_idx)); + + if (idx) + idx[offset + i] = (index >= max_naddr ? + 0xffff : index); + if (index < max_naddr) + nfilters++; + else if (hash) + *hash |= (1ULL << + hash_mac_addr(addr[offset + i])); + } + + free = false; + offset += fw_naddr; + rem -= fw_naddr; + } + + if (ret == 0 || ret == -FW_ENOMEM) + ret = nfilters; + return ret; +} + +/** + * t4_free_mac_filt - frees exact-match filters of given MAC addresses + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @viid: the VI id + * @naddr: the number of MAC addresses to allocate filters for (up to 7) + * @addr: the MAC address(es) + * @sleep_ok: call is allowed to sleep + * + * Frees the exact-match filter for each of the supplied addresses + * + * Returns a negative error number or the number of filters freed. + */ +int t4_free_mac_filt(struct adapter *adap, unsigned int mbox, + unsigned int viid, unsigned int naddr, + const u8 **addr, bool sleep_ok) +{ + int offset, ret = 0; + struct fw_vi_mac_cmd c; + unsigned int nfilters = 0; + unsigned int max_naddr = is_t4(adap->params.chip) ? + NUM_MPS_CLS_SRAM_L_INSTANCES : + NUM_MPS_T5_CLS_SRAM_L_INSTANCES; + unsigned int rem = naddr; + + if (naddr > max_naddr) + return -EINVAL; + + for (offset = 0; offset < (int)naddr ; /**/) { + unsigned int fw_naddr = (rem < ARRAY_SIZE(c.u.exact) + ? rem + : ARRAY_SIZE(c.u.exact)); + size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd, + u.exact[fw_naddr]), 16); + struct fw_vi_mac_exact *p; + int i; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_CMD_EXEC_V(0) | + FW_VI_MAC_CMD_VIID_V(viid)); + c.freemacs_to_len16 = + cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(0) | + FW_CMD_LEN16_V(len16)); + + for (i = 0, p = c.u.exact; i < (int)fw_naddr; i++, p++) { + p->valid_to_idx = cpu_to_be16( + FW_VI_MAC_CMD_VALID_F | + FW_VI_MAC_CMD_IDX_V(FW_VI_MAC_MAC_BASED_FREE)); + memcpy(p->macaddr, addr[offset+i], sizeof(p->macaddr)); + } + + ret = t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), &c, sleep_ok); + if (ret) + break; + + for (i = 0, p = c.u.exact; i < fw_naddr; i++, p++) { + u16 index = FW_VI_MAC_CMD_IDX_G( + be16_to_cpu(p->valid_to_idx)); + + if (index < max_naddr) + nfilters++; + } + + offset += fw_naddr; + rem -= fw_naddr; + } + + if (ret == 0) + ret = nfilters; + return ret; +} + +/** + * t4_change_mac - modifies the exact-match filter for a MAC address + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @viid: the VI id + * @idx: index of existing filter for old value of MAC address, or -1 + * @addr: the new MAC address value + * @persist: whether a new MAC allocation should be persistent + * @smt_idx: the destination to store the new SMT index. + * + * Modifies an exact-match filter and sets it to the new MAC address. + * Note that in general it is not possible to modify the value of a given + * filter so the generic way to modify an address filter is to free the one + * being used by the old address value and allocate a new filter for the + * new address value. @idx can be -1 if the address is a new addition. + * + * Returns a negative error number or the index of the filter with the new + * MAC value. + */ +int t4_change_mac(struct adapter *adap, unsigned int mbox, unsigned int viid, + int idx, const u8 *addr, bool persist, u8 *smt_idx) +{ + int ret, mode; + struct fw_vi_mac_cmd c; + struct fw_vi_mac_exact *p = c.u.exact; + unsigned int max_mac_addr = adap->params.arch.mps_tcam_size; + + if (idx < 0) /* new allocation */ + idx = persist ? FW_VI_MAC_ADD_PERSIST_MAC : FW_VI_MAC_ADD_MAC; + mode = smt_idx ? FW_VI_MAC_SMT_AND_MPSTCAM : FW_VI_MAC_MPS_TCAM_ENTRY; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_VI_MAC_CMD_VIID_V(viid)); + c.freemacs_to_len16 = cpu_to_be32(FW_CMD_LEN16_V(1)); + p->valid_to_idx = cpu_to_be16(FW_VI_MAC_CMD_VALID_F | + FW_VI_MAC_CMD_SMAC_RESULT_V(mode) | + FW_VI_MAC_CMD_IDX_V(idx)); + memcpy(p->macaddr, addr, sizeof(p->macaddr)); + + ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); + if (ret == 0) { + ret = FW_VI_MAC_CMD_IDX_G(be16_to_cpu(p->valid_to_idx)); + if (ret >= max_mac_addr) + ret = -ENOMEM; + if (smt_idx) { + if (adap->params.viid_smt_extn_support) { + *smt_idx = FW_VI_MAC_CMD_SMTID_G + (be32_to_cpu(c.op_to_viid)); + } else { + /* In T4/T5, SMT contains 256 SMAC entries + * organized in 128 rows of 2 entries each. + * In T6, SMT contains 256 SMAC entries in + * 256 rows. + */ + if (CHELSIO_CHIP_VERSION(adap->params.chip) <= + CHELSIO_T5) + *smt_idx = (viid & FW_VIID_VIN_M) << 1; + else + *smt_idx = (viid & FW_VIID_VIN_M); + } + } + } + return ret; +} + +/** + * t4_set_addr_hash - program the MAC inexact-match hash filter + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @viid: the VI id + * @ucast: whether the hash filter should also match unicast addresses + * @vec: the value to be written to the hash filter + * @sleep_ok: call is allowed to sleep + * + * Sets the 64-bit inexact-match hash filter for a virtual interface. + */ +int t4_set_addr_hash(struct adapter *adap, unsigned int mbox, unsigned int viid, + bool ucast, u64 vec, bool sleep_ok) +{ + struct fw_vi_mac_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | FW_CMD_WRITE_F | + FW_VI_ENABLE_CMD_VIID_V(viid)); + c.freemacs_to_len16 = cpu_to_be32(FW_VI_MAC_CMD_HASHVECEN_F | + FW_VI_MAC_CMD_HASHUNIEN_V(ucast) | + FW_CMD_LEN16_V(1)); + c.u.hash.hashvec = cpu_to_be64(vec); + return t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok); +} + +/** + * t4_enable_vi_params - enable/disable a virtual interface + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @viid: the VI id + * @rx_en: 1=enable Rx, 0=disable Rx + * @tx_en: 1=enable Tx, 0=disable Tx + * @dcb_en: 1=enable delivery of Data Center Bridging messages. + * + * Enables/disables a virtual interface. Note that setting DCB Enable + * only makes sense when enabling a Virtual Interface ... + */ +int t4_enable_vi_params(struct adapter *adap, unsigned int mbox, + unsigned int viid, bool rx_en, bool tx_en, bool dcb_en) +{ + struct fw_vi_enable_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_ENABLE_CMD) | + FW_CMD_REQUEST_F | FW_CMD_EXEC_F | + FW_VI_ENABLE_CMD_VIID_V(viid)); + c.ien_to_len16 = cpu_to_be32(FW_VI_ENABLE_CMD_IEN_V(rx_en) | + FW_VI_ENABLE_CMD_EEN_V(tx_en) | + FW_VI_ENABLE_CMD_DCB_INFO_V(dcb_en) | + FW_LEN16(c)); + return t4_wr_mbox_ns(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_enable_vi - enable/disable a virtual interface + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @viid: the VI id + * @rx_en: 1=enable Rx, 0=disable Rx + * @tx_en: 1=enable Tx, 0=disable Tx + * + * Enables/disables a virtual interface. + */ +int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid, + bool rx_en, bool tx_en) +{ + return t4_enable_vi_params(adap, mbox, viid, rx_en, tx_en, 0); +} + +/** + * t4_enable_pi_params - enable/disable a Port's Virtual Interface + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @pi: the Port Information structure + * @rx_en: 1=enable Rx, 0=disable Rx + * @tx_en: 1=enable Tx, 0=disable Tx + * @dcb_en: 1=enable delivery of Data Center Bridging messages. + * + * Enables/disables a Port's Virtual Interface. Note that setting DCB + * Enable only makes sense when enabling a Virtual Interface ... + * If the Virtual Interface enable/disable operation is successful, + * we notify the OS-specific code of a potential Link Status change + * via the OS Contract API t4_os_link_changed(). + */ +int t4_enable_pi_params(struct adapter *adap, unsigned int mbox, + struct port_info *pi, + bool rx_en, bool tx_en, bool dcb_en) +{ + int ret = t4_enable_vi_params(adap, mbox, pi->viid, + rx_en, tx_en, dcb_en); + if (ret) + return ret; + t4_os_link_changed(adap, pi->port_id, + rx_en && tx_en && pi->link_cfg.link_ok); + return 0; +} + +/** + * t4_identify_port - identify a VI's port by blinking its LED + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @viid: the VI id + * @nblinks: how many times to blink LED at 2.5 Hz + * + * Identifies a VI's port by blinking its LED. + */ +int t4_identify_port(struct adapter *adap, unsigned int mbox, unsigned int viid, + unsigned int nblinks) +{ + struct fw_vi_enable_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_ENABLE_CMD) | + FW_CMD_REQUEST_F | FW_CMD_EXEC_F | + FW_VI_ENABLE_CMD_VIID_V(viid)); + c.ien_to_len16 = cpu_to_be32(FW_VI_ENABLE_CMD_LED_F | FW_LEN16(c)); + c.blinkdur = cpu_to_be16(nblinks); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_iq_stop - stop an ingress queue and its FLs + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @pf: the PF owning the queues + * @vf: the VF owning the queues + * @iqtype: the ingress queue type (FW_IQ_TYPE_FL_INT_CAP, etc.) + * @iqid: ingress queue id + * @fl0id: FL0 queue id or 0xffff if no attached FL0 + * @fl1id: FL1 queue id or 0xffff if no attached FL1 + * + * Stops an ingress queue and its associated FLs, if any. This causes + * any current or future data/messages destined for these queues to be + * tossed. + */ +int t4_iq_stop(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int iqtype, unsigned int iqid, + unsigned int fl0id, unsigned int fl1id) +{ + struct fw_iq_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F | + FW_CMD_EXEC_F | FW_IQ_CMD_PFN_V(pf) | + FW_IQ_CMD_VFN_V(vf)); + c.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_IQSTOP_F | FW_LEN16(c)); + c.type_to_iqandstindex = cpu_to_be32(FW_IQ_CMD_TYPE_V(iqtype)); + c.iqid = cpu_to_be16(iqid); + c.fl0id = cpu_to_be16(fl0id); + c.fl1id = cpu_to_be16(fl1id); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_iq_free - free an ingress queue and its FLs + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @pf: the PF owning the queues + * @vf: the VF owning the queues + * @iqtype: the ingress queue type + * @iqid: ingress queue id + * @fl0id: FL0 queue id or 0xffff if no attached FL0 + * @fl1id: FL1 queue id or 0xffff if no attached FL1 + * + * Frees an ingress queue and its associated FLs, if any. + */ +int t4_iq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int iqtype, unsigned int iqid, + unsigned int fl0id, unsigned int fl1id) +{ + struct fw_iq_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F | + FW_CMD_EXEC_F | FW_IQ_CMD_PFN_V(pf) | + FW_IQ_CMD_VFN_V(vf)); + c.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_FREE_F | FW_LEN16(c)); + c.type_to_iqandstindex = cpu_to_be32(FW_IQ_CMD_TYPE_V(iqtype)); + c.iqid = cpu_to_be16(iqid); + c.fl0id = cpu_to_be16(fl0id); + c.fl1id = cpu_to_be16(fl1id); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_eth_eq_free - free an Ethernet egress queue + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @pf: the PF owning the queue + * @vf: the VF owning the queue + * @eqid: egress queue id + * + * Frees an Ethernet egress queue. + */ +int t4_eth_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int eqid) +{ + struct fw_eq_eth_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_ETH_CMD) | + FW_CMD_REQUEST_F | FW_CMD_EXEC_F | + FW_EQ_ETH_CMD_PFN_V(pf) | + FW_EQ_ETH_CMD_VFN_V(vf)); + c.alloc_to_len16 = cpu_to_be32(FW_EQ_ETH_CMD_FREE_F | FW_LEN16(c)); + c.eqid_pkd = cpu_to_be32(FW_EQ_ETH_CMD_EQID_V(eqid)); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_ctrl_eq_free - free a control egress queue + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @pf: the PF owning the queue + * @vf: the VF owning the queue + * @eqid: egress queue id + * + * Frees a control egress queue. + */ +int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int eqid) +{ + struct fw_eq_ctrl_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_CTRL_CMD) | + FW_CMD_REQUEST_F | FW_CMD_EXEC_F | + FW_EQ_CTRL_CMD_PFN_V(pf) | + FW_EQ_CTRL_CMD_VFN_V(vf)); + c.alloc_to_len16 = cpu_to_be32(FW_EQ_CTRL_CMD_FREE_F | FW_LEN16(c)); + c.cmpliqid_eqid = cpu_to_be32(FW_EQ_CTRL_CMD_EQID_V(eqid)); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_ofld_eq_free - free an offload egress queue + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @pf: the PF owning the queue + * @vf: the VF owning the queue + * @eqid: egress queue id + * + * Frees a control egress queue. + */ +int t4_ofld_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, + unsigned int vf, unsigned int eqid) +{ + struct fw_eq_ofld_cmd c; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_OFLD_CMD) | + FW_CMD_REQUEST_F | FW_CMD_EXEC_F | + FW_EQ_OFLD_CMD_PFN_V(pf) | + FW_EQ_OFLD_CMD_VFN_V(vf)); + c.alloc_to_len16 = cpu_to_be32(FW_EQ_OFLD_CMD_FREE_F | FW_LEN16(c)); + c.eqid_pkd = cpu_to_be32(FW_EQ_OFLD_CMD_EQID_V(eqid)); + return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL); +} + +/** + * t4_link_down_rc_str - return a string for a Link Down Reason Code + * @link_down_rc: Link Down Reason Code + * + * Returns a string representation of the Link Down Reason Code. + */ +static const char *t4_link_down_rc_str(unsigned char link_down_rc) +{ + static const char * const reason[] = { + "Link Down", + "Remote Fault", + "Auto-negotiation Failure", + "Reserved", + "Insufficient Airflow", + "Unable To Determine Reason", + "No RX Signal Detected", + "Reserved", + }; + + if (link_down_rc >= ARRAY_SIZE(reason)) + return "Bad Reason Code"; + + return reason[link_down_rc]; +} + +/* Return the highest speed set in the port capabilities, in Mb/s. */ +static unsigned int fwcap_to_speed(fw_port_cap32_t caps) +{ + #define TEST_SPEED_RETURN(__caps_speed, __speed) \ + do { \ + if (caps & FW_PORT_CAP32_SPEED_##__caps_speed) \ + return __speed; \ + } while (0) + + TEST_SPEED_RETURN(400G, 400000); + TEST_SPEED_RETURN(200G, 200000); + TEST_SPEED_RETURN(100G, 100000); + TEST_SPEED_RETURN(50G, 50000); + TEST_SPEED_RETURN(40G, 40000); + TEST_SPEED_RETURN(25G, 25000); + TEST_SPEED_RETURN(10G, 10000); + TEST_SPEED_RETURN(1G, 1000); + TEST_SPEED_RETURN(100M, 100); + + #undef TEST_SPEED_RETURN + + return 0; +} + +/** + * fwcap_to_fwspeed - return highest speed in Port Capabilities + * @acaps: advertised Port Capabilities + * + * Get the highest speed for the port from the advertised Port + * Capabilities. It will be either the highest speed from the list of + * speeds or whatever user has set using ethtool. + */ +static fw_port_cap32_t fwcap_to_fwspeed(fw_port_cap32_t acaps) +{ + #define TEST_SPEED_RETURN(__caps_speed) \ + do { \ + if (acaps & FW_PORT_CAP32_SPEED_##__caps_speed) \ + return FW_PORT_CAP32_SPEED_##__caps_speed; \ + } while (0) + + TEST_SPEED_RETURN(400G); + TEST_SPEED_RETURN(200G); + TEST_SPEED_RETURN(100G); + TEST_SPEED_RETURN(50G); + TEST_SPEED_RETURN(40G); + TEST_SPEED_RETURN(25G); + TEST_SPEED_RETURN(10G); + TEST_SPEED_RETURN(1G); + TEST_SPEED_RETURN(100M); + + #undef TEST_SPEED_RETURN + + return 0; +} + +/** + * lstatus_to_fwcap - translate old lstatus to 32-bit Port Capabilities + * @lstatus: old FW_PORT_ACTION_GET_PORT_INFO lstatus value + * + * Translates old FW_PORT_ACTION_GET_PORT_INFO lstatus field into new + * 32-bit Port Capabilities value. + */ +static fw_port_cap32_t lstatus_to_fwcap(u32 lstatus) +{ + fw_port_cap32_t linkattr = 0; + + /* Unfortunately the format of the Link Status in the old + * 16-bit Port Information message isn't the same as the + * 16-bit Port Capabilities bitfield used everywhere else ... + */ + if (lstatus & FW_PORT_CMD_RXPAUSE_F) + linkattr |= FW_PORT_CAP32_FC_RX; + if (lstatus & FW_PORT_CMD_TXPAUSE_F) + linkattr |= FW_PORT_CAP32_FC_TX; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100M)) + linkattr |= FW_PORT_CAP32_SPEED_100M; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_1G)) + linkattr |= FW_PORT_CAP32_SPEED_1G; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_10G)) + linkattr |= FW_PORT_CAP32_SPEED_10G; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_25G)) + linkattr |= FW_PORT_CAP32_SPEED_25G; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_40G)) + linkattr |= FW_PORT_CAP32_SPEED_40G; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100G)) + linkattr |= FW_PORT_CAP32_SPEED_100G; + + return linkattr; +} + +/** + * t4_handle_get_port_info - process a FW reply message + * @pi: the port info + * @rpl: start of the FW message + * + * Processes a GET_PORT_INFO FW reply message. + */ +void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl) +{ + const struct fw_port_cmd *cmd = (const void *)rpl; + fw_port_cap32_t pcaps, acaps, lpacaps, linkattr; + struct link_config *lc = &pi->link_cfg; + struct adapter *adapter = pi->adapter; + unsigned int speed, fc, fec, adv_fc; + enum fw_port_module_type mod_type; + int action, link_ok, linkdnrc; + enum fw_port_type port_type; + + /* Extract the various fields from the Port Information message. + */ + action = FW_PORT_CMD_ACTION_G(be32_to_cpu(cmd->action_to_len16)); + switch (action) { + case FW_PORT_ACTION_GET_PORT_INFO: { + u32 lstatus = be32_to_cpu(cmd->u.info.lstatus_to_modtype); + + link_ok = (lstatus & FW_PORT_CMD_LSTATUS_F) != 0; + linkdnrc = FW_PORT_CMD_LINKDNRC_G(lstatus); + port_type = FW_PORT_CMD_PTYPE_G(lstatus); + mod_type = FW_PORT_CMD_MODTYPE_G(lstatus); + pcaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.pcap)); + acaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.acap)); + lpacaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.lpacap)); + linkattr = lstatus_to_fwcap(lstatus); + break; + } + + case FW_PORT_ACTION_GET_PORT_INFO32: { + u32 lstatus32; + + lstatus32 = be32_to_cpu(cmd->u.info32.lstatus32_to_cbllen32); + link_ok = (lstatus32 & FW_PORT_CMD_LSTATUS32_F) != 0; + linkdnrc = FW_PORT_CMD_LINKDNRC32_G(lstatus32); + port_type = FW_PORT_CMD_PORTTYPE32_G(lstatus32); + mod_type = FW_PORT_CMD_MODTYPE32_G(lstatus32); + pcaps = be32_to_cpu(cmd->u.info32.pcaps32); + acaps = be32_to_cpu(cmd->u.info32.acaps32); + lpacaps = be32_to_cpu(cmd->u.info32.lpacaps32); + linkattr = be32_to_cpu(cmd->u.info32.linkattr32); + break; + } + + default: + dev_err(adapter->pdev_dev, "Handle Port Information: Bad Command/Action %#x\n", + be32_to_cpu(cmd->action_to_len16)); + return; + } + + fec = fwcap_to_cc_fec(acaps); + adv_fc = fwcap_to_cc_pause(acaps); + fc = fwcap_to_cc_pause(linkattr); + speed = fwcap_to_speed(linkattr); + + /* Reset state for communicating new Transceiver Module status and + * whether the OS-dependent layer wants us to redo the current + * "sticky" L1 Configure Link Parameters. + */ + lc->new_module = false; + lc->redo_l1cfg = false; + + if (mod_type != pi->mod_type) { + /* With the newer SFP28 and QSFP28 Transceiver Module Types, + * various fundamental Port Capabilities which used to be + * immutable can now change radically. We can now have + * Speeds, Auto-Negotiation, Forward Error Correction, etc. + * all change based on what Transceiver Module is inserted. + * So we need to record the Physical "Port" Capabilities on + * every Transceiver Module change. + */ + lc->pcaps = pcaps; + + /* When a new Transceiver Module is inserted, the Firmware + * will examine its i2c EPROM to determine its type and + * general operating parameters including things like Forward + * Error Control, etc. Various IEEE 802.3 standards dictate + * how to interpret these i2c values to determine default + * "sutomatic" settings. We record these for future use when + * the user explicitly requests these standards-based values. + */ + lc->def_acaps = acaps; + + /* Some versions of the early T6 Firmware "cheated" when + * handling different Transceiver Modules by changing the + * underlaying Port Type reported to the Host Drivers. As + * such we need to capture whatever Port Type the Firmware + * sends us and record it in case it's different from what we + * were told earlier. Unfortunately, since Firmware is + * forever, we'll need to keep this code here forever, but in + * later T6 Firmware it should just be an assignment of the + * same value already recorded. + */ + pi->port_type = port_type; + + /* Record new Module Type information. + */ + pi->mod_type = mod_type; + + /* Let the OS-dependent layer know if we have a new + * Transceiver Module inserted. + */ + lc->new_module = t4_is_inserted_mod_type(mod_type); + + t4_os_portmod_changed(adapter, pi->port_id); + } + + if (link_ok != lc->link_ok || speed != lc->speed || + fc != lc->fc || adv_fc != lc->advertised_fc || + fec != lc->fec) { + /* something changed */ + if (!link_ok && lc->link_ok) { + lc->link_down_rc = linkdnrc; + dev_warn_ratelimited(adapter->pdev_dev, + "Port %d link down, reason: %s\n", + pi->tx_chan, + t4_link_down_rc_str(linkdnrc)); + } + lc->link_ok = link_ok; + lc->speed = speed; + lc->advertised_fc = adv_fc; + lc->fc = fc; + lc->fec = fec; + + lc->lpacaps = lpacaps; + lc->acaps = acaps & ADVERT_MASK; + + /* If we're not physically capable of Auto-Negotiation, note + * this as Auto-Negotiation disabled. Otherwise, we track + * what Auto-Negotiation settings we have. Note parallel + * structure in t4_link_l1cfg_core() and init_link_config(). + */ + if (!(lc->acaps & FW_PORT_CAP32_ANEG)) { + lc->autoneg = AUTONEG_DISABLE; + } else if (lc->acaps & FW_PORT_CAP32_ANEG) { + lc->autoneg = AUTONEG_ENABLE; + } else { + /* When Autoneg is disabled, user needs to set + * single speed. + * Similar to cxgb4_ethtool.c: set_link_ksettings + */ + lc->acaps = 0; + lc->speed_caps = fwcap_to_fwspeed(acaps); + lc->autoneg = AUTONEG_DISABLE; + } + + t4_os_link_changed(adapter, pi->port_id, link_ok); + } + + /* If we have a new Transceiver Module and the OS-dependent code has + * told us that it wants us to redo whatever "sticky" L1 Configuration + * Link Parameters are set, do that now. + */ + if (lc->new_module && lc->redo_l1cfg) { + struct link_config old_lc; + int ret; + + /* Save the current L1 Configuration and restore it if an + * error occurs. We probably should fix the l1_cfg*() + * routines not to change the link_config when an error + * occurs ... + */ + old_lc = *lc; + ret = t4_link_l1cfg_ns(adapter, adapter->mbox, pi->lport, lc); + if (ret) { + *lc = old_lc; + dev_warn(adapter->pdev_dev, + "Attempt to update new Transceiver Module settings failed\n"); + } + } + lc->new_module = false; + lc->redo_l1cfg = false; +} + +/** + * t4_update_port_info - retrieve and update port information if changed + * @pi: the port_info + * + * We issue a Get Port Information Command to the Firmware and, if + * successful, we check to see if anything is different from what we + * last recorded and update things accordingly. + */ +int t4_update_port_info(struct port_info *pi) +{ + unsigned int fw_caps = pi->adapter->params.fw_caps_support; + struct fw_port_cmd port_cmd; + int ret; + + memset(&port_cmd, 0, sizeof(port_cmd)); + port_cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F | + FW_PORT_CMD_PORTID_V(pi->tx_chan)); + port_cmd.action_to_len16 = cpu_to_be32( + FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16 + ? FW_PORT_ACTION_GET_PORT_INFO + : FW_PORT_ACTION_GET_PORT_INFO32) | + FW_LEN16(port_cmd)); + ret = t4_wr_mbox(pi->adapter, pi->adapter->mbox, + &port_cmd, sizeof(port_cmd), &port_cmd); + if (ret) + return ret; + + t4_handle_get_port_info(pi, (__be64 *)&port_cmd); + return 0; +} + +/** + * t4_get_link_params - retrieve basic link parameters for given port + * @pi: the port + * @link_okp: value return pointer for link up/down + * @speedp: value return pointer for speed (Mb/s) + * @mtup: value return pointer for mtu + * + * Retrieves basic link parameters for a port: link up/down, speed (Mb/s), + * and MTU for a specified port. A negative error is returned on + * failure; 0 on success. + */ +int t4_get_link_params(struct port_info *pi, unsigned int *link_okp, + unsigned int *speedp, unsigned int *mtup) +{ + unsigned int fw_caps = pi->adapter->params.fw_caps_support; + unsigned int action, link_ok, mtu; + struct fw_port_cmd port_cmd; + fw_port_cap32_t linkattr; + int ret; + + memset(&port_cmd, 0, sizeof(port_cmd)); + port_cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F | + FW_PORT_CMD_PORTID_V(pi->tx_chan)); + action = (fw_caps == FW_CAPS16 + ? FW_PORT_ACTION_GET_PORT_INFO + : FW_PORT_ACTION_GET_PORT_INFO32); + port_cmd.action_to_len16 = cpu_to_be32( + FW_PORT_CMD_ACTION_V(action) | + FW_LEN16(port_cmd)); + ret = t4_wr_mbox(pi->adapter, pi->adapter->mbox, + &port_cmd, sizeof(port_cmd), &port_cmd); + if (ret) + return ret; + + if (action == FW_PORT_ACTION_GET_PORT_INFO) { + u32 lstatus = be32_to_cpu(port_cmd.u.info.lstatus_to_modtype); + + link_ok = !!(lstatus & FW_PORT_CMD_LSTATUS_F); + linkattr = lstatus_to_fwcap(lstatus); + mtu = be16_to_cpu(port_cmd.u.info.mtu); + } else { + u32 lstatus32 = + be32_to_cpu(port_cmd.u.info32.lstatus32_to_cbllen32); + + link_ok = !!(lstatus32 & FW_PORT_CMD_LSTATUS32_F); + linkattr = be32_to_cpu(port_cmd.u.info32.linkattr32); + mtu = FW_PORT_CMD_MTU32_G( + be32_to_cpu(port_cmd.u.info32.auxlinfo32_mtu32)); + } + + if (link_okp) + *link_okp = link_ok; + if (speedp) + *speedp = fwcap_to_speed(linkattr); + if (mtup) + *mtup = mtu; + + return 0; +} + +/** + * t4_handle_fw_rpl - process a FW reply message + * @adap: the adapter + * @rpl: start of the FW message + * + * Processes a FW message, such as link state change messages. + */ +int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl) +{ + u8 opcode = *(const u8 *)rpl; + + /* This might be a port command ... this simplifies the following + * conditionals ... We can get away with pre-dereferencing + * action_to_len16 because it's in the first 16 bytes and all messages + * will be at least that long. + */ + const struct fw_port_cmd *p = (const void *)rpl; + unsigned int action = + FW_PORT_CMD_ACTION_G(be32_to_cpu(p->action_to_len16)); + + if (opcode == FW_PORT_CMD && + (action == FW_PORT_ACTION_GET_PORT_INFO || + action == FW_PORT_ACTION_GET_PORT_INFO32)) { + int i; + int chan = FW_PORT_CMD_PORTID_G(be32_to_cpu(p->op_to_portid)); + struct port_info *pi = NULL; + + for_each_port(adap, i) { + pi = adap2pinfo(adap, i); + if (pi->tx_chan == chan) + break; + } + + t4_handle_get_port_info(pi, rpl); + } else { + dev_warn(adap->pdev_dev, "Unknown firmware reply %d\n", + opcode); + return -EINVAL; + } + return 0; +} + +static void get_pci_mode(struct adapter *adapter, struct pci_params *p) +{ + u16 val; + + if (pci_is_pcie(adapter->pdev)) { + pcie_capability_read_word(adapter->pdev, PCI_EXP_LNKSTA, &val); + p->speed = val & PCI_EXP_LNKSTA_CLS; + p->width = (val & PCI_EXP_LNKSTA_NLW) >> 4; + } +} + +/** + * init_link_config - initialize a link's SW state + * @lc: pointer to structure holding the link state + * @pcaps: link Port Capabilities + * @acaps: link current Advertised Port Capabilities + * + * Initializes the SW state maintained for each link, including the link's + * capabilities and default speed/flow-control/autonegotiation settings. + */ +static void init_link_config(struct link_config *lc, fw_port_cap32_t pcaps, + fw_port_cap32_t acaps) +{ + lc->pcaps = pcaps; + lc->def_acaps = acaps; + lc->lpacaps = 0; + lc->speed_caps = 0; + lc->speed = 0; + lc->requested_fc = lc->fc = PAUSE_RX | PAUSE_TX; + + /* For Forward Error Control, we default to whatever the Firmware + * tells us the Link is currently advertising. + */ + lc->requested_fec = FEC_AUTO; + lc->fec = fwcap_to_cc_fec(lc->def_acaps); + + /* If the Port is capable of Auto-Negtotiation, initialize it as + * "enabled" and copy over all of the Physical Port Capabilities + * to the Advertised Port Capabilities. Otherwise mark it as + * Auto-Negotiate disabled and select the highest supported speed + * for the link. Note parallel structure in t4_link_l1cfg_core() + * and t4_handle_get_port_info(). + */ + if (lc->pcaps & FW_PORT_CAP32_ANEG) { + lc->acaps = lc->pcaps & ADVERT_MASK; + lc->autoneg = AUTONEG_ENABLE; + lc->requested_fc |= PAUSE_AUTONEG; + } else { + lc->acaps = 0; + lc->autoneg = AUTONEG_DISABLE; + lc->speed_caps = fwcap_to_fwspeed(acaps); + } +} + +#define CIM_PF_NOACCESS 0xeeeeeeee + +int t4_wait_dev_ready(void __iomem *regs) +{ + u32 whoami; + + whoami = readl(regs + PL_WHOAMI_A); + if (whoami != 0xffffffff && whoami != CIM_PF_NOACCESS) + return 0; + + msleep(500); + whoami = readl(regs + PL_WHOAMI_A); + return (whoami != 0xffffffff && whoami != CIM_PF_NOACCESS ? 0 : -EIO); +} + +struct flash_desc { + u32 vendor_and_model_id; + u32 size_mb; +}; + +static int t4_get_flash_params(struct adapter *adap) +{ + /* Table for non-Numonix supported flash parts. Numonix parts are left + * to the preexisting code. All flash parts have 64KB sectors. + */ + static struct flash_desc supported_flash[] = { + { 0x150201, 4 << 20 }, /* Spansion 4MB S25FL032P */ + }; + + unsigned int part, manufacturer; + unsigned int density, size = 0; + u32 flashid = 0; + int ret; + + /* Issue a Read ID Command to the Flash part. We decode supported + * Flash parts and their sizes from this. There's a newer Query + * Command which can retrieve detailed geometry information but many + * Flash parts don't support it. + */ + + ret = sf1_write(adap, 1, 1, 0, SF_RD_ID); + if (!ret) + ret = sf1_read(adap, 3, 0, 1, &flashid); + t4_write_reg(adap, SF_OP_A, 0); /* unlock SF */ + if (ret) + return ret; + + /* Check to see if it's one of our non-standard supported Flash parts. + */ + for (part = 0; part < ARRAY_SIZE(supported_flash); part++) + if (supported_flash[part].vendor_and_model_id == flashid) { + adap->params.sf_size = supported_flash[part].size_mb; + adap->params.sf_nsec = + adap->params.sf_size / SF_SEC_SIZE; + goto found; + } + + /* Decode Flash part size. The code below looks repetitive with + * common encodings, but that's not guaranteed in the JEDEC + * specification for the Read JEDEC ID command. The only thing that + * we're guaranteed by the JEDEC specification is where the + * Manufacturer ID is in the returned result. After that each + * Manufacturer ~could~ encode things completely differently. + * Note, all Flash parts must have 64KB sectors. + */ + manufacturer = flashid & 0xff; + switch (manufacturer) { + case 0x20: { /* Micron/Numonix */ + /* This Density -> Size decoding table is taken from Micron + * Data Sheets. + */ + density = (flashid >> 16) & 0xff; + switch (density) { + case 0x14: /* 1MB */ + size = 1 << 20; + break; + case 0x15: /* 2MB */ + size = 1 << 21; + break; + case 0x16: /* 4MB */ + size = 1 << 22; + break; + case 0x17: /* 8MB */ + size = 1 << 23; + break; + case 0x18: /* 16MB */ + size = 1 << 24; + break; + case 0x19: /* 32MB */ + size = 1 << 25; + break; + case 0x20: /* 64MB */ + size = 1 << 26; + break; + case 0x21: /* 128MB */ + size = 1 << 27; + break; + case 0x22: /* 256MB */ + size = 1 << 28; + break; + } + break; + } + case 0x9d: { /* ISSI -- Integrated Silicon Solution, Inc. */ + /* This Density -> Size decoding table is taken from ISSI + * Data Sheets. + */ + density = (flashid >> 16) & 0xff; + switch (density) { + case 0x16: /* 32 MB */ + size = 1 << 25; + break; + case 0x17: /* 64MB */ + size = 1 << 26; + break; + } + break; + } + case 0xc2: { /* Macronix */ + /* This Density -> Size decoding table is taken from Macronix + * Data Sheets. + */ + density = (flashid >> 16) & 0xff; + switch (density) { + case 0x17: /* 8MB */ + size = 1 << 23; + break; + case 0x18: /* 16MB */ + size = 1 << 24; + break; + } + break; + } + case 0xef: { /* Winbond */ + /* This Density -> Size decoding table is taken from Winbond + * Data Sheets. + */ + density = (flashid >> 16) & 0xff; + switch (density) { + case 0x17: /* 8MB */ + size = 1 << 23; + break; + case 0x18: /* 16MB */ + size = 1 << 24; + break; + } + break; + } + } + + /* If we didn't recognize the FLASH part, that's no real issue: the + * Hardware/Software contract says that Hardware will _*ALWAYS*_ + * use a FLASH part which is at least 4MB in size and has 64KB + * sectors. The unrecognized FLASH part is likely to be much larger + * than 4MB, but that's all we really need. + */ + if (size == 0) { + dev_warn(adap->pdev_dev, "Unknown Flash Part, ID = %#x, assuming 4MB\n", + flashid); + size = 1 << 22; + } + + /* Store decoded Flash size and fall through into vetting code. */ + adap->params.sf_size = size; + adap->params.sf_nsec = size / SF_SEC_SIZE; + +found: + if (adap->params.sf_size < FLASH_MIN_SIZE) + dev_warn(adap->pdev_dev, "WARNING: Flash Part ID %#x, size %#x < %#x\n", + flashid, adap->params.sf_size, FLASH_MIN_SIZE); + return 0; +} + +/** + * t4_prep_adapter - prepare SW and HW for operation + * @adapter: the adapter + * + * Initialize adapter SW state for the various HW modules, set initial + * values for some adapter tunables, take PHYs out of reset, and + * initialize the MDIO interface. + */ +int t4_prep_adapter(struct adapter *adapter) +{ + int ret, ver; + uint16_t device_id; + u32 pl_rev; + + get_pci_mode(adapter, &adapter->params.pci); + pl_rev = REV_G(t4_read_reg(adapter, PL_REV_A)); + + ret = t4_get_flash_params(adapter); + if (ret < 0) { + dev_err(adapter->pdev_dev, "error %d identifying flash\n", ret); + return ret; + } + + /* Retrieve adapter's device ID + */ + pci_read_config_word(adapter->pdev, PCI_DEVICE_ID, &device_id); + ver = device_id >> 12; + adapter->params.chip = 0; + switch (ver) { + case CHELSIO_T4: + adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev); + adapter->params.arch.sge_fl_db = DBPRIO_F; + adapter->params.arch.mps_tcam_size = + NUM_MPS_CLS_SRAM_L_INSTANCES; + adapter->params.arch.mps_rplc_size = 128; + adapter->params.arch.nchan = NCHAN; + adapter->params.arch.pm_stats_cnt = PM_NSTATS; + adapter->params.arch.vfcount = 128; + /* Congestion map is for 4 channels so that + * MPS can have 4 priority per port. + */ + adapter->params.arch.cng_ch_bits_log = 2; + break; + case CHELSIO_T5: + adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev); + adapter->params.arch.sge_fl_db = DBPRIO_F | DBTYPE_F; + adapter->params.arch.mps_tcam_size = + NUM_MPS_T5_CLS_SRAM_L_INSTANCES; + adapter->params.arch.mps_rplc_size = 128; + adapter->params.arch.nchan = NCHAN; + adapter->params.arch.pm_stats_cnt = PM_NSTATS; + adapter->params.arch.vfcount = 128; + adapter->params.arch.cng_ch_bits_log = 2; + break; + case CHELSIO_T6: + adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev); + adapter->params.arch.sge_fl_db = 0; + adapter->params.arch.mps_tcam_size = + NUM_MPS_T5_CLS_SRAM_L_INSTANCES; + adapter->params.arch.mps_rplc_size = 256; + adapter->params.arch.nchan = 2; + adapter->params.arch.pm_stats_cnt = T6_PM_NSTATS; + adapter->params.arch.vfcount = 256; + /* Congestion map will be for 2 channels so that + * MPS can have 8 priority per port. + */ + adapter->params.arch.cng_ch_bits_log = 3; + break; + default: + dev_err(adapter->pdev_dev, "Device %d is not supported\n", + device_id); + return -EINVAL; + } + + adapter->params.cim_la_size = CIMLA_SIZE; + init_cong_ctrl(adapter->params.a_wnd, adapter->params.b_wnd); + + /* + * Default port for debugging in case we can't reach FW. + */ + adapter->params.nports = 1; + adapter->params.portvec = 1; + adapter->params.vpd.cclk = 50000; + + /* Set PCIe completion timeout to 4 seconds. */ + pcie_capability_clear_and_set_word(adapter->pdev, PCI_EXP_DEVCTL2, + PCI_EXP_DEVCTL2_COMP_TIMEOUT, 0xd); + return 0; +} + +/** + * t4_shutdown_adapter - shut down adapter, host & wire + * @adapter: the adapter + * + * Perform an emergency shutdown of the adapter and stop it from + * continuing any further communication on the ports or DMA to the + * host. This is typically used when the adapter and/or firmware + * have crashed and we want to prevent any further accidental + * communication with the rest of the world. This will also force + * the port Link Status to go down -- if register writes work -- + * which should help our peers figure out that we're down. + */ +int t4_shutdown_adapter(struct adapter *adapter) +{ + int port; + + t4_intr_disable(adapter); + t4_write_reg(adapter, DBG_GPIO_EN_A, 0); + for_each_port(adapter, port) { + u32 a_port_cfg = is_t4(adapter->params.chip) ? + PORT_REG(port, XGMAC_PORT_CFG_A) : + T5_PORT_REG(port, MAC_PORT_CFG_A); + + t4_write_reg(adapter, a_port_cfg, + t4_read_reg(adapter, a_port_cfg) + & ~SIGNAL_DET_V(1)); + } + t4_set_reg_field(adapter, SGE_CONTROL_A, GLOBALENABLE_F, 0); + + return 0; +} + +/** + * t4_bar2_sge_qregs - return BAR2 SGE Queue register information + * @adapter: the adapter + * @qid: the Queue ID + * @qtype: the Ingress or Egress type for @qid + * @user: true if this request is for a user mode queue + * @pbar2_qoffset: BAR2 Queue Offset + * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues + * + * Returns the BAR2 SGE Queue Registers information associated with the + * indicated Absolute Queue ID. These are passed back in return value + * pointers. @qtype should be T4_BAR2_QTYPE_EGRESS for Egress Queue + * and T4_BAR2_QTYPE_INGRESS for Ingress Queues. + * + * This may return an error which indicates that BAR2 SGE Queue + * registers aren't available. If an error is not returned, then the + * following values are returned: + * + * *@pbar2_qoffset: the BAR2 Offset of the @qid Registers + * *@pbar2_qid: the BAR2 SGE Queue ID or 0 of @qid + * + * If the returned BAR2 Queue ID is 0, then BAR2 SGE registers which + * require the "Inferred Queue ID" ability may be used. E.g. the + * Write Combining Doorbell Buffer. If the BAR2 Queue ID is not 0, + * then these "Inferred Queue ID" register may not be used. + */ +int t4_bar2_sge_qregs(struct adapter *adapter, + unsigned int qid, + enum t4_bar2_qtype qtype, + int user, + u64 *pbar2_qoffset, + unsigned int *pbar2_qid) +{ + unsigned int page_shift, page_size, qpp_shift, qpp_mask; + u64 bar2_page_offset, bar2_qoffset; + unsigned int bar2_qid, bar2_qid_offset, bar2_qinferred; + + /* T4 doesn't support BAR2 SGE Queue registers for kernel mode queues */ + if (!user && is_t4(adapter->params.chip)) + return -EINVAL; + + /* Get our SGE Page Size parameters. + */ + page_shift = adapter->params.sge.hps + 10; + page_size = 1 << page_shift; + + /* Get the right Queues per Page parameters for our Queue. + */ + qpp_shift = (qtype == T4_BAR2_QTYPE_EGRESS + ? adapter->params.sge.eq_qpp + : adapter->params.sge.iq_qpp); + qpp_mask = (1 << qpp_shift) - 1; + + /* Calculate the basics of the BAR2 SGE Queue register area: + * o The BAR2 page the Queue registers will be in. + * o The BAR2 Queue ID. + * o The BAR2 Queue ID Offset into the BAR2 page. + */ + bar2_page_offset = ((u64)(qid >> qpp_shift) << page_shift); + bar2_qid = qid & qpp_mask; + bar2_qid_offset = bar2_qid * SGE_UDB_SIZE; + + /* If the BAR2 Queue ID Offset is less than the Page Size, then the + * hardware will infer the Absolute Queue ID simply from the writes to + * the BAR2 Queue ID Offset within the BAR2 Page (and we need to use a + * BAR2 Queue ID of 0 for those writes). Otherwise, we'll simply + * write to the first BAR2 SGE Queue Area within the BAR2 Page with + * the BAR2 Queue ID and the hardware will infer the Absolute Queue ID + * from the BAR2 Page and BAR2 Queue ID. + * + * One important censequence of this is that some BAR2 SGE registers + * have a "Queue ID" field and we can write the BAR2 SGE Queue ID + * there. But other registers synthesize the SGE Queue ID purely + * from the writes to the registers -- the Write Combined Doorbell + * Buffer is a good example. These BAR2 SGE Registers are only + * available for those BAR2 SGE Register areas where the SGE Absolute + * Queue ID can be inferred from simple writes. + */ + bar2_qoffset = bar2_page_offset; + bar2_qinferred = (bar2_qid_offset < page_size); + if (bar2_qinferred) { + bar2_qoffset += bar2_qid_offset; + bar2_qid = 0; + } + + *pbar2_qoffset = bar2_qoffset; + *pbar2_qid = bar2_qid; + return 0; +} + +/** + * t4_init_devlog_params - initialize adapter->params.devlog + * @adap: the adapter + * + * Initialize various fields of the adapter's Firmware Device Log + * Parameters structure. + */ +int t4_init_devlog_params(struct adapter *adap) +{ + struct devlog_params *dparams = &adap->params.devlog; + u32 pf_dparams; + unsigned int devlog_meminfo; + struct fw_devlog_cmd devlog_cmd; + int ret; + + /* If we're dealing with newer firmware, the Device Log Parameters + * are stored in a designated register which allows us to access the + * Device Log even if we can't talk to the firmware. + */ + pf_dparams = + t4_read_reg(adap, PCIE_FW_REG(PCIE_FW_PF_A, PCIE_FW_PF_DEVLOG)); + if (pf_dparams) { + unsigned int nentries, nentries128; + + dparams->memtype = PCIE_FW_PF_DEVLOG_MEMTYPE_G(pf_dparams); + dparams->start = PCIE_FW_PF_DEVLOG_ADDR16_G(pf_dparams) << 4; + + nentries128 = PCIE_FW_PF_DEVLOG_NENTRIES128_G(pf_dparams); + nentries = (nentries128 + 1) * 128; + dparams->size = nentries * sizeof(struct fw_devlog_e); + + return 0; + } + + /* Otherwise, ask the firmware for it's Device Log Parameters. + */ + memset(&devlog_cmd, 0, sizeof(devlog_cmd)); + devlog_cmd.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_DEVLOG_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F); + devlog_cmd.retval_len16 = cpu_to_be32(FW_LEN16(devlog_cmd)); + ret = t4_wr_mbox(adap, adap->mbox, &devlog_cmd, sizeof(devlog_cmd), + &devlog_cmd); + if (ret) + return ret; + + devlog_meminfo = + be32_to_cpu(devlog_cmd.memtype_devlog_memaddr16_devlog); + dparams->memtype = FW_DEVLOG_CMD_MEMTYPE_DEVLOG_G(devlog_meminfo); + dparams->start = FW_DEVLOG_CMD_MEMADDR16_DEVLOG_G(devlog_meminfo) << 4; + dparams->size = be32_to_cpu(devlog_cmd.memsize_devlog); + + return 0; +} + +/** + * t4_init_sge_params - initialize adap->params.sge + * @adapter: the adapter + * + * Initialize various fields of the adapter's SGE Parameters structure. + */ +int t4_init_sge_params(struct adapter *adapter) +{ + struct sge_params *sge_params = &adapter->params.sge; + u32 hps, qpp; + unsigned int s_hps, s_qpp; + + /* Extract the SGE Page Size for our PF. + */ + hps = t4_read_reg(adapter, SGE_HOST_PAGE_SIZE_A); + s_hps = (HOSTPAGESIZEPF0_S + + (HOSTPAGESIZEPF1_S - HOSTPAGESIZEPF0_S) * adapter->pf); + sge_params->hps = ((hps >> s_hps) & HOSTPAGESIZEPF0_M); + + /* Extract the SGE Egress and Ingess Queues Per Page for our PF. + */ + s_qpp = (QUEUESPERPAGEPF0_S + + (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) * adapter->pf); + qpp = t4_read_reg(adapter, SGE_EGRESS_QUEUES_PER_PAGE_PF_A); + sge_params->eq_qpp = ((qpp >> s_qpp) & QUEUESPERPAGEPF0_M); + qpp = t4_read_reg(adapter, SGE_INGRESS_QUEUES_PER_PAGE_PF_A); + sge_params->iq_qpp = ((qpp >> s_qpp) & QUEUESPERPAGEPF0_M); + + return 0; +} + +/** + * t4_init_tp_params - initialize adap->params.tp + * @adap: the adapter + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Initialize various fields of the adapter's TP Parameters structure. + */ +int t4_init_tp_params(struct adapter *adap, bool sleep_ok) +{ + u32 param, val, v; + int chan, ret; + + + v = t4_read_reg(adap, TP_TIMER_RESOLUTION_A); + adap->params.tp.tre = TIMERRESOLUTION_G(v); + adap->params.tp.dack_re = DELAYEDACKRESOLUTION_G(v); + + /* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */ + for (chan = 0; chan < NCHAN; chan++) + adap->params.tp.tx_modq[chan] = chan; + + /* Cache the adapter's Compressed Filter Mode/Mask and global Ingress + * Configuration. + */ + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_FILTER) | + FW_PARAMS_PARAM_Y_V(FW_PARAM_DEV_FILTER_MODE_MASK)); + + /* Read current value */ + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, + ¶m, &val); + if (ret == 0) { + dev_info(adap->pdev_dev, + "Current filter mode/mask 0x%x:0x%x\n", + FW_PARAMS_PARAM_FILTER_MODE_G(val), + FW_PARAMS_PARAM_FILTER_MASK_G(val)); + adap->params.tp.vlan_pri_map = + FW_PARAMS_PARAM_FILTER_MODE_G(val); + adap->params.tp.filter_mask = + FW_PARAMS_PARAM_FILTER_MASK_G(val); + } else { + dev_info(adap->pdev_dev, + "Failed to read filter mode/mask via fw api, using indirect-reg-read\n"); + + /* Incase of older-fw (which doesn't expose the api + * FW_PARAM_DEV_FILTER_MODE_MASK) and newer-driver (which uses + * the fw api) combination, fall-back to older method of reading + * the filter mode from indirect-register + */ + t4_tp_pio_read(adap, &adap->params.tp.vlan_pri_map, 1, + TP_VLAN_PRI_MAP_A, sleep_ok); + + /* With the older-fw and newer-driver combination we might run + * into an issue when user wants to use hash filter region but + * the filter_mask is zero, in this case filter_mask validation + * is tough. To avoid that we set the filter_mask same as filter + * mode, which will behave exactly as the older way of ignoring + * the filter mask validation. + */ + adap->params.tp.filter_mask = adap->params.tp.vlan_pri_map; + } + + t4_tp_pio_read(adap, &adap->params.tp.ingress_config, 1, + TP_INGRESS_CONFIG_A, sleep_ok); + + /* For T6, cache the adapter's compressed error vector + * and passing outer header info for encapsulated packets. + */ + if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) { + v = t4_read_reg(adap, TP_OUT_CONFIG_A); + adap->params.tp.rx_pkt_encap = (v & CRXPKTENC_F) ? 1 : 0; + } + + /* Now that we have TP_VLAN_PRI_MAP cached, we can calculate the field + * shift positions of several elements of the Compressed Filter Tuple + * for this adapter which we need frequently ... + */ + adap->params.tp.fcoe_shift = t4_filter_field_shift(adap, FCOE_F); + adap->params.tp.port_shift = t4_filter_field_shift(adap, PORT_F); + adap->params.tp.vnic_shift = t4_filter_field_shift(adap, VNIC_ID_F); + adap->params.tp.vlan_shift = t4_filter_field_shift(adap, VLAN_F); + adap->params.tp.tos_shift = t4_filter_field_shift(adap, TOS_F); + adap->params.tp.protocol_shift = t4_filter_field_shift(adap, + PROTOCOL_F); + adap->params.tp.ethertype_shift = t4_filter_field_shift(adap, + ETHERTYPE_F); + adap->params.tp.macmatch_shift = t4_filter_field_shift(adap, + MACMATCH_F); + adap->params.tp.matchtype_shift = t4_filter_field_shift(adap, + MPSHITTYPE_F); + adap->params.tp.frag_shift = t4_filter_field_shift(adap, + FRAGMENTATION_F); + + /* If TP_INGRESS_CONFIG.VNID == 0, then TP_VLAN_PRI_MAP.VNIC_ID + * represents the presence of an Outer VLAN instead of a VNIC ID. + */ + if ((adap->params.tp.ingress_config & VNIC_F) == 0) + adap->params.tp.vnic_shift = -1; + + v = t4_read_reg(adap, LE_3_DB_HASH_MASK_GEN_IPV4_T6_A); + adap->params.tp.hash_filter_mask = v; + v = t4_read_reg(adap, LE_4_DB_HASH_MASK_GEN_IPV4_T6_A); + adap->params.tp.hash_filter_mask |= ((u64)v << 32); + return 0; +} + +/** + * t4_filter_field_shift - calculate filter field shift + * @adap: the adapter + * @filter_sel: the desired field (from TP_VLAN_PRI_MAP bits) + * + * Return the shift position of a filter field within the Compressed + * Filter Tuple. The filter field is specified via its selection bit + * within TP_VLAN_PRI_MAL (filter mode). E.g. F_VLAN. + */ +int t4_filter_field_shift(const struct adapter *adap, int filter_sel) +{ + unsigned int filter_mode = adap->params.tp.vlan_pri_map; + unsigned int sel; + int field_shift; + + if ((filter_mode & filter_sel) == 0) + return -1; + + for (sel = 1, field_shift = 0; sel < filter_sel; sel <<= 1) { + switch (filter_mode & sel) { + case FCOE_F: + field_shift += FT_FCOE_W; + break; + case PORT_F: + field_shift += FT_PORT_W; + break; + case VNIC_ID_F: + field_shift += FT_VNIC_ID_W; + break; + case VLAN_F: + field_shift += FT_VLAN_W; + break; + case TOS_F: + field_shift += FT_TOS_W; + break; + case PROTOCOL_F: + field_shift += FT_PROTOCOL_W; + break; + case ETHERTYPE_F: + field_shift += FT_ETHERTYPE_W; + break; + case MACMATCH_F: + field_shift += FT_MACMATCH_W; + break; + case MPSHITTYPE_F: + field_shift += FT_MPSHITTYPE_W; + break; + case FRAGMENTATION_F: + field_shift += FT_FRAGMENTATION_W; + break; + } + } + return field_shift; +} + +int t4_init_rss_mode(struct adapter *adap, int mbox) +{ + int i, ret; + struct fw_rss_vi_config_cmd rvc; + + memset(&rvc, 0, sizeof(rvc)); + + for_each_port(adap, i) { + struct port_info *p = adap2pinfo(adap, i); + + rvc.op_to_viid = + cpu_to_be32(FW_CMD_OP_V(FW_RSS_VI_CONFIG_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F | + FW_RSS_VI_CONFIG_CMD_VIID_V(p->viid)); + rvc.retval_len16 = cpu_to_be32(FW_LEN16(rvc)); + ret = t4_wr_mbox(adap, mbox, &rvc, sizeof(rvc), &rvc); + if (ret) + return ret; + p->rss_mode = be32_to_cpu(rvc.u.basicvirtual.defaultq_to_udpen); + } + return 0; +} + +/** + * t4_init_portinfo - allocate a virtual interface and initialize port_info + * @pi: the port_info + * @mbox: mailbox to use for the FW command + * @port: physical port associated with the VI + * @pf: the PF owning the VI + * @vf: the VF owning the VI + * @mac: the MAC address of the VI + * + * Allocates a virtual interface for the given physical port. If @mac is + * not %NULL it contains the MAC address of the VI as assigned by FW. + * @mac should be large enough to hold an Ethernet address. + * Returns < 0 on error. + */ +int t4_init_portinfo(struct port_info *pi, int mbox, + int port, int pf, int vf, u8 mac[]) +{ + struct adapter *adapter = pi->adapter; + unsigned int fw_caps = adapter->params.fw_caps_support; + struct fw_port_cmd cmd; + unsigned int rss_size; + enum fw_port_type port_type; + int mdio_addr; + fw_port_cap32_t pcaps, acaps; + u8 vivld = 0, vin = 0; + int ret; + + /* If we haven't yet determined whether we're talking to Firmware + * which knows the new 32-bit Port Capabilities, it's time to find + * out now. This will also tell new Firmware to send us Port Status + * Updates using the new 32-bit Port Capabilities version of the + * Port Information message. + */ + if (fw_caps == FW_CAPS_UNKNOWN) { + u32 param, val; + + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_PORT_CAPS32)); + val = 1; + ret = t4_set_params(adapter, mbox, pf, vf, 1, ¶m, &val); + fw_caps = (ret == 0 ? FW_CAPS32 : FW_CAPS16); + adapter->params.fw_caps_support = fw_caps; + } + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F | + FW_PORT_CMD_PORTID_V(port)); + cmd.action_to_len16 = cpu_to_be32( + FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16 + ? FW_PORT_ACTION_GET_PORT_INFO + : FW_PORT_ACTION_GET_PORT_INFO32) | + FW_LEN16(cmd)); + ret = t4_wr_mbox(pi->adapter, mbox, &cmd, sizeof(cmd), &cmd); + if (ret) + return ret; + + /* Extract the various fields from the Port Information message. + */ + if (fw_caps == FW_CAPS16) { + u32 lstatus = be32_to_cpu(cmd.u.info.lstatus_to_modtype); + + port_type = FW_PORT_CMD_PTYPE_G(lstatus); + mdio_addr = ((lstatus & FW_PORT_CMD_MDIOCAP_F) + ? FW_PORT_CMD_MDIOADDR_G(lstatus) + : -1); + pcaps = fwcaps16_to_caps32(be16_to_cpu(cmd.u.info.pcap)); + acaps = fwcaps16_to_caps32(be16_to_cpu(cmd.u.info.acap)); + } else { + u32 lstatus32 = be32_to_cpu(cmd.u.info32.lstatus32_to_cbllen32); + + port_type = FW_PORT_CMD_PORTTYPE32_G(lstatus32); + mdio_addr = ((lstatus32 & FW_PORT_CMD_MDIOCAP32_F) + ? FW_PORT_CMD_MDIOADDR32_G(lstatus32) + : -1); + pcaps = be32_to_cpu(cmd.u.info32.pcaps32); + acaps = be32_to_cpu(cmd.u.info32.acaps32); + } + + ret = t4_alloc_vi(pi->adapter, mbox, port, pf, vf, 1, mac, &rss_size, + &vivld, &vin); + if (ret < 0) + return ret; + + pi->viid = ret; + pi->tx_chan = port; + pi->lport = port; + pi->rss_size = rss_size; + pi->rx_cchan = t4_get_tp_e2c_map(pi->adapter, port); + + /* If fw supports returning the VIN as part of FW_VI_CMD, + * save the returned values. + */ + if (adapter->params.viid_smt_extn_support) { + pi->vivld = vivld; + pi->vin = vin; + } else { + /* Retrieve the values from VIID */ + pi->vivld = FW_VIID_VIVLD_G(pi->viid); + pi->vin = FW_VIID_VIN_G(pi->viid); + } + + pi->port_type = port_type; + pi->mdio_addr = mdio_addr; + pi->mod_type = FW_PORT_MOD_TYPE_NA; + + init_link_config(&pi->link_cfg, pcaps, acaps); + return 0; +} + +int t4_port_init(struct adapter *adap, int mbox, int pf, int vf) +{ + u8 addr[6]; + int ret, i, j = 0; + + for_each_port(adap, i) { + struct port_info *pi = adap2pinfo(adap, i); + + while ((adap->params.portvec & (1 << j)) == 0) + j++; + + ret = t4_init_portinfo(pi, mbox, j, pf, vf, addr); + if (ret) + return ret; + + eth_hw_addr_set(adap->port[i], addr); + j++; + } + return 0; +} + +int t4_init_port_mirror(struct port_info *pi, u8 mbox, u8 port, u8 pf, u8 vf, + u16 *mirror_viid) +{ + int ret; + + ret = t4_alloc_vi(pi->adapter, mbox, port, pf, vf, 1, NULL, NULL, + NULL, NULL); + if (ret < 0) + return ret; + + if (mirror_viid) + *mirror_viid = ret; + + return 0; +} + +/** + * t4_read_cimq_cfg - read CIM queue configuration + * @adap: the adapter + * @base: holds the queue base addresses in bytes + * @size: holds the queue sizes in bytes + * @thres: holds the queue full thresholds in bytes + * + * Returns the current configuration of the CIM queues, starting with + * the IBQs, then the OBQs. + */ +void t4_read_cimq_cfg(struct adapter *adap, u16 *base, u16 *size, u16 *thres) +{ + unsigned int i, v; + int cim_num_obq = is_t4(adap->params.chip) ? + CIM_NUM_OBQ : CIM_NUM_OBQ_T5; + + for (i = 0; i < CIM_NUM_IBQ; i++) { + t4_write_reg(adap, CIM_QUEUE_CONFIG_REF_A, IBQSELECT_F | + QUENUMSELECT_V(i)); + v = t4_read_reg(adap, CIM_QUEUE_CONFIG_CTRL_A); + /* value is in 256-byte units */ + *base++ = CIMQBASE_G(v) * 256; + *size++ = CIMQSIZE_G(v) * 256; + *thres++ = QUEFULLTHRSH_G(v) * 8; /* 8-byte unit */ + } + for (i = 0; i < cim_num_obq; i++) { + t4_write_reg(adap, CIM_QUEUE_CONFIG_REF_A, OBQSELECT_F | + QUENUMSELECT_V(i)); + v = t4_read_reg(adap, CIM_QUEUE_CONFIG_CTRL_A); + /* value is in 256-byte units */ + *base++ = CIMQBASE_G(v) * 256; + *size++ = CIMQSIZE_G(v) * 256; + } +} + +/** + * t4_read_cim_ibq - read the contents of a CIM inbound queue + * @adap: the adapter + * @qid: the queue index + * @data: where to store the queue contents + * @n: capacity of @data in 32-bit words + * + * Reads the contents of the selected CIM queue starting at address 0 up + * to the capacity of @data. @n must be a multiple of 4. Returns < 0 on + * error and the number of 32-bit words actually read on success. + */ +int t4_read_cim_ibq(struct adapter *adap, unsigned int qid, u32 *data, size_t n) +{ + int i, err, attempts; + unsigned int addr; + const unsigned int nwords = CIM_IBQ_SIZE * 4; + + if (qid > 5 || (n & 3)) + return -EINVAL; + + addr = qid * nwords; + if (n > nwords) + n = nwords; + + /* It might take 3-10ms before the IBQ debug read access is allowed. + * Wait for 1 Sec with a delay of 1 usec. + */ + attempts = 1000000; + + for (i = 0; i < n; i++, addr++) { + t4_write_reg(adap, CIM_IBQ_DBG_CFG_A, IBQDBGADDR_V(addr) | + IBQDBGEN_F); + err = t4_wait_op_done(adap, CIM_IBQ_DBG_CFG_A, IBQDBGBUSY_F, 0, + attempts, 1); + if (err) + return err; + *data++ = t4_read_reg(adap, CIM_IBQ_DBG_DATA_A); + } + t4_write_reg(adap, CIM_IBQ_DBG_CFG_A, 0); + return i; +} + +/** + * t4_read_cim_obq - read the contents of a CIM outbound queue + * @adap: the adapter + * @qid: the queue index + * @data: where to store the queue contents + * @n: capacity of @data in 32-bit words + * + * Reads the contents of the selected CIM queue starting at address 0 up + * to the capacity of @data. @n must be a multiple of 4. Returns < 0 on + * error and the number of 32-bit words actually read on success. + */ +int t4_read_cim_obq(struct adapter *adap, unsigned int qid, u32 *data, size_t n) +{ + int i, err; + unsigned int addr, v, nwords; + int cim_num_obq = is_t4(adap->params.chip) ? + CIM_NUM_OBQ : CIM_NUM_OBQ_T5; + + if ((qid > (cim_num_obq - 1)) || (n & 3)) + return -EINVAL; + + t4_write_reg(adap, CIM_QUEUE_CONFIG_REF_A, OBQSELECT_F | + QUENUMSELECT_V(qid)); + v = t4_read_reg(adap, CIM_QUEUE_CONFIG_CTRL_A); + + addr = CIMQBASE_G(v) * 64; /* muliple of 256 -> muliple of 4 */ + nwords = CIMQSIZE_G(v) * 64; /* same */ + if (n > nwords) + n = nwords; + + for (i = 0; i < n; i++, addr++) { + t4_write_reg(adap, CIM_OBQ_DBG_CFG_A, OBQDBGADDR_V(addr) | + OBQDBGEN_F); + err = t4_wait_op_done(adap, CIM_OBQ_DBG_CFG_A, OBQDBGBUSY_F, 0, + 2, 1); + if (err) + return err; + *data++ = t4_read_reg(adap, CIM_OBQ_DBG_DATA_A); + } + t4_write_reg(adap, CIM_OBQ_DBG_CFG_A, 0); + return i; +} + +/** + * t4_cim_read - read a block from CIM internal address space + * @adap: the adapter + * @addr: the start address within the CIM address space + * @n: number of words to read + * @valp: where to store the result + * + * Reads a block of 4-byte words from the CIM intenal address space. + */ +int t4_cim_read(struct adapter *adap, unsigned int addr, unsigned int n, + unsigned int *valp) +{ + int ret = 0; + + if (t4_read_reg(adap, CIM_HOST_ACC_CTRL_A) & HOSTBUSY_F) + return -EBUSY; + + for ( ; !ret && n--; addr += 4) { + t4_write_reg(adap, CIM_HOST_ACC_CTRL_A, addr); + ret = t4_wait_op_done(adap, CIM_HOST_ACC_CTRL_A, HOSTBUSY_F, + 0, 5, 2); + if (!ret) + *valp++ = t4_read_reg(adap, CIM_HOST_ACC_DATA_A); + } + return ret; +} + +/** + * t4_cim_write - write a block into CIM internal address space + * @adap: the adapter + * @addr: the start address within the CIM address space + * @n: number of words to write + * @valp: set of values to write + * + * Writes a block of 4-byte words into the CIM intenal address space. + */ +int t4_cim_write(struct adapter *adap, unsigned int addr, unsigned int n, + const unsigned int *valp) +{ + int ret = 0; + + if (t4_read_reg(adap, CIM_HOST_ACC_CTRL_A) & HOSTBUSY_F) + return -EBUSY; + + for ( ; !ret && n--; addr += 4) { + t4_write_reg(adap, CIM_HOST_ACC_DATA_A, *valp++); + t4_write_reg(adap, CIM_HOST_ACC_CTRL_A, addr | HOSTWRITE_F); + ret = t4_wait_op_done(adap, CIM_HOST_ACC_CTRL_A, HOSTBUSY_F, + 0, 5, 2); + } + return ret; +} + +static int t4_cim_write1(struct adapter *adap, unsigned int addr, + unsigned int val) +{ + return t4_cim_write(adap, addr, 1, &val); +} + +/** + * t4_cim_read_la - read CIM LA capture buffer + * @adap: the adapter + * @la_buf: where to store the LA data + * @wrptr: the HW write pointer within the capture buffer + * + * Reads the contents of the CIM LA buffer with the most recent entry at + * the end of the returned data and with the entry at @wrptr first. + * We try to leave the LA in the running state we find it in. + */ +int t4_cim_read_la(struct adapter *adap, u32 *la_buf, unsigned int *wrptr) +{ + int i, ret; + unsigned int cfg, val, idx; + + ret = t4_cim_read(adap, UP_UP_DBG_LA_CFG_A, 1, &cfg); + if (ret) + return ret; + + if (cfg & UPDBGLAEN_F) { /* LA is running, freeze it */ + ret = t4_cim_write1(adap, UP_UP_DBG_LA_CFG_A, 0); + if (ret) + return ret; + } + + ret = t4_cim_read(adap, UP_UP_DBG_LA_CFG_A, 1, &val); + if (ret) + goto restart; + + idx = UPDBGLAWRPTR_G(val); + if (wrptr) + *wrptr = idx; + + for (i = 0; i < adap->params.cim_la_size; i++) { + ret = t4_cim_write1(adap, UP_UP_DBG_LA_CFG_A, + UPDBGLARDPTR_V(idx) | UPDBGLARDEN_F); + if (ret) + break; + ret = t4_cim_read(adap, UP_UP_DBG_LA_CFG_A, 1, &val); + if (ret) + break; + if (val & UPDBGLARDEN_F) { + ret = -ETIMEDOUT; + break; + } + ret = t4_cim_read(adap, UP_UP_DBG_LA_DATA_A, 1, &la_buf[i]); + if (ret) + break; + + /* Bits 0-3 of UpDbgLaRdPtr can be between 0000 to 1001 to + * identify the 32-bit portion of the full 312-bit data + */ + if (is_t6(adap->params.chip) && (idx & 0xf) >= 9) + idx = (idx & 0xff0) + 0x10; + else + idx++; + /* address can't exceed 0xfff */ + idx &= UPDBGLARDPTR_M; + } +restart: + if (cfg & UPDBGLAEN_F) { + int r = t4_cim_write1(adap, UP_UP_DBG_LA_CFG_A, + cfg & ~UPDBGLARDEN_F); + if (!ret) + ret = r; + } + return ret; +} + +/** + * t4_tp_read_la - read TP LA capture buffer + * @adap: the adapter + * @la_buf: where to store the LA data + * @wrptr: the HW write pointer within the capture buffer + * + * Reads the contents of the TP LA buffer with the most recent entry at + * the end of the returned data and with the entry at @wrptr first. + * We leave the LA in the running state we find it in. + */ +void t4_tp_read_la(struct adapter *adap, u64 *la_buf, unsigned int *wrptr) +{ + bool last_incomplete; + unsigned int i, cfg, val, idx; + + cfg = t4_read_reg(adap, TP_DBG_LA_CONFIG_A) & 0xffff; + if (cfg & DBGLAENABLE_F) /* freeze LA */ + t4_write_reg(adap, TP_DBG_LA_CONFIG_A, + adap->params.tp.la_mask | (cfg ^ DBGLAENABLE_F)); + + val = t4_read_reg(adap, TP_DBG_LA_CONFIG_A); + idx = DBGLAWPTR_G(val); + last_incomplete = DBGLAMODE_G(val) >= 2 && (val & DBGLAWHLF_F) == 0; + if (last_incomplete) + idx = (idx + 1) & DBGLARPTR_M; + if (wrptr) + *wrptr = idx; + + val &= 0xffff; + val &= ~DBGLARPTR_V(DBGLARPTR_M); + val |= adap->params.tp.la_mask; + + for (i = 0; i < TPLA_SIZE; i++) { + t4_write_reg(adap, TP_DBG_LA_CONFIG_A, DBGLARPTR_V(idx) | val); + la_buf[i] = t4_read_reg64(adap, TP_DBG_LA_DATAL_A); + idx = (idx + 1) & DBGLARPTR_M; + } + + /* Wipe out last entry if it isn't valid */ + if (last_incomplete) + la_buf[TPLA_SIZE - 1] = ~0ULL; + + if (cfg & DBGLAENABLE_F) /* restore running state */ + t4_write_reg(adap, TP_DBG_LA_CONFIG_A, + cfg | adap->params.tp.la_mask); +} + +/* SGE Hung Ingress DMA Warning Threshold time and Warning Repeat Rate (in + * seconds). If we find one of the SGE Ingress DMA State Machines in the same + * state for more than the Warning Threshold then we'll issue a warning about + * a potential hang. We'll repeat the warning as the SGE Ingress DMA Channel + * appears to be hung every Warning Repeat second till the situation clears. + * If the situation clears, we'll note that as well. + */ +#define SGE_IDMA_WARN_THRESH 1 +#define SGE_IDMA_WARN_REPEAT 300 + +/** + * t4_idma_monitor_init - initialize SGE Ingress DMA Monitor + * @adapter: the adapter + * @idma: the adapter IDMA Monitor state + * + * Initialize the state of an SGE Ingress DMA Monitor. + */ +void t4_idma_monitor_init(struct adapter *adapter, + struct sge_idma_monitor_state *idma) +{ + /* Initialize the state variables for detecting an SGE Ingress DMA + * hang. The SGE has internal counters which count up on each clock + * tick whenever the SGE finds its Ingress DMA State Engines in the + * same state they were on the previous clock tick. The clock used is + * the Core Clock so we have a limit on the maximum "time" they can + * record; typically a very small number of seconds. For instance, + * with a 600MHz Core Clock, we can only count up to a bit more than + * 7s. So we'll synthesize a larger counter in order to not run the + * risk of having the "timers" overflow and give us the flexibility to + * maintain a Hung SGE State Machine of our own which operates across + * a longer time frame. + */ + idma->idma_1s_thresh = core_ticks_per_usec(adapter) * 1000000; /* 1s */ + idma->idma_stalled[0] = 0; + idma->idma_stalled[1] = 0; +} + +/** + * t4_idma_monitor - monitor SGE Ingress DMA state + * @adapter: the adapter + * @idma: the adapter IDMA Monitor state + * @hz: number of ticks/second + * @ticks: number of ticks since the last IDMA Monitor call + */ +void t4_idma_monitor(struct adapter *adapter, + struct sge_idma_monitor_state *idma, + int hz, int ticks) +{ + int i, idma_same_state_cnt[2]; + + /* Read the SGE Debug Ingress DMA Same State Count registers. These + * are counters inside the SGE which count up on each clock when the + * SGE finds its Ingress DMA State Engines in the same states they + * were in the previous clock. The counters will peg out at + * 0xffffffff without wrapping around so once they pass the 1s + * threshold they'll stay above that till the IDMA state changes. + */ + t4_write_reg(adapter, SGE_DEBUG_INDEX_A, 13); + idma_same_state_cnt[0] = t4_read_reg(adapter, SGE_DEBUG_DATA_HIGH_A); + idma_same_state_cnt[1] = t4_read_reg(adapter, SGE_DEBUG_DATA_LOW_A); + + for (i = 0; i < 2; i++) { + u32 debug0, debug11; + + /* If the Ingress DMA Same State Counter ("timer") is less + * than 1s, then we can reset our synthesized Stall Timer and + * continue. If we have previously emitted warnings about a + * potential stalled Ingress Queue, issue a note indicating + * that the Ingress Queue has resumed forward progress. + */ + if (idma_same_state_cnt[i] < idma->idma_1s_thresh) { + if (idma->idma_stalled[i] >= SGE_IDMA_WARN_THRESH * hz) + dev_warn(adapter->pdev_dev, "SGE idma%d, queue %u, " + "resumed after %d seconds\n", + i, idma->idma_qid[i], + idma->idma_stalled[i] / hz); + idma->idma_stalled[i] = 0; + continue; + } + + /* Synthesize an SGE Ingress DMA Same State Timer in the Hz + * domain. The first time we get here it'll be because we + * passed the 1s Threshold; each additional time it'll be + * because the RX Timer Callback is being fired on its regular + * schedule. + * + * If the stall is below our Potential Hung Ingress Queue + * Warning Threshold, continue. + */ + if (idma->idma_stalled[i] == 0) { + idma->idma_stalled[i] = hz; + idma->idma_warn[i] = 0; + } else { + idma->idma_stalled[i] += ticks; + idma->idma_warn[i] -= ticks; + } + + if (idma->idma_stalled[i] < SGE_IDMA_WARN_THRESH * hz) + continue; + + /* We'll issue a warning every SGE_IDMA_WARN_REPEAT seconds. + */ + if (idma->idma_warn[i] > 0) + continue; + idma->idma_warn[i] = SGE_IDMA_WARN_REPEAT * hz; + + /* Read and save the SGE IDMA State and Queue ID information. + * We do this every time in case it changes across time ... + * can't be too careful ... + */ + t4_write_reg(adapter, SGE_DEBUG_INDEX_A, 0); + debug0 = t4_read_reg(adapter, SGE_DEBUG_DATA_LOW_A); + idma->idma_state[i] = (debug0 >> (i * 9)) & 0x3f; + + t4_write_reg(adapter, SGE_DEBUG_INDEX_A, 11); + debug11 = t4_read_reg(adapter, SGE_DEBUG_DATA_LOW_A); + idma->idma_qid[i] = (debug11 >> (i * 16)) & 0xffff; + + dev_warn(adapter->pdev_dev, "SGE idma%u, queue %u, potentially stuck in " + "state %u for %d seconds (debug0=%#x, debug11=%#x)\n", + i, idma->idma_qid[i], idma->idma_state[i], + idma->idma_stalled[i] / hz, + debug0, debug11); + t4_sge_decode_idma_state(adapter, idma->idma_state[i]); + } +} + +/** + * t4_load_cfg - download config file + * @adap: the adapter + * @cfg_data: the cfg text file to write + * @size: text file size + * + * Write the supplied config text file to the card's serial flash. + */ +int t4_load_cfg(struct adapter *adap, const u8 *cfg_data, unsigned int size) +{ + int ret, i, n, cfg_addr; + unsigned int addr; + unsigned int flash_cfg_start_sec; + unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec; + + cfg_addr = t4_flash_cfg_addr(adap); + if (cfg_addr < 0) + return cfg_addr; + + addr = cfg_addr; + flash_cfg_start_sec = addr / SF_SEC_SIZE; + + if (size > FLASH_CFG_MAX_SIZE) { + dev_err(adap->pdev_dev, "cfg file too large, max is %u bytes\n", + FLASH_CFG_MAX_SIZE); + return -EFBIG; + } + + i = DIV_ROUND_UP(FLASH_CFG_MAX_SIZE, /* # of sectors spanned */ + sf_sec_size); + ret = t4_flash_erase_sectors(adap, flash_cfg_start_sec, + flash_cfg_start_sec + i - 1); + /* If size == 0 then we're simply erasing the FLASH sectors associated + * with the on-adapter Firmware Configuration File. + */ + if (ret || size == 0) + goto out; + + /* this will write to the flash up to SF_PAGE_SIZE at a time */ + for (i = 0; i < size; i += SF_PAGE_SIZE) { + if ((size - i) < SF_PAGE_SIZE) + n = size - i; + else + n = SF_PAGE_SIZE; + ret = t4_write_flash(adap, addr, n, cfg_data, true); + if (ret) + goto out; + + addr += SF_PAGE_SIZE; + cfg_data += SF_PAGE_SIZE; + } + +out: + if (ret) + dev_err(adap->pdev_dev, "config file %s failed %d\n", + (size == 0 ? "clear" : "download"), ret); + return ret; +} + +/** + * t4_set_vf_mac_acl - Set MAC address for the specified VF + * @adapter: The adapter + * @vf: one of the VFs instantiated by the specified PF + * @naddr: the number of MAC addresses + * @addr: the MAC address(es) to be set to the specified VF + */ +int t4_set_vf_mac_acl(struct adapter *adapter, unsigned int vf, + unsigned int naddr, u8 *addr) +{ + struct fw_acl_mac_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_ACL_MAC_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_ACL_MAC_CMD_PFN_V(adapter->pf) | + FW_ACL_MAC_CMD_VFN_V(vf)); + + /* Note: Do not enable the ACL */ + cmd.en_to_len16 = cpu_to_be32((unsigned int)FW_LEN16(cmd)); + cmd.nmac = naddr; + + switch (adapter->pf) { + case 3: + memcpy(cmd.macaddr3, addr, sizeof(cmd.macaddr3)); + break; + case 2: + memcpy(cmd.macaddr2, addr, sizeof(cmd.macaddr2)); + break; + case 1: + memcpy(cmd.macaddr1, addr, sizeof(cmd.macaddr1)); + break; + case 0: + memcpy(cmd.macaddr0, addr, sizeof(cmd.macaddr0)); + break; + } + + return t4_wr_mbox(adapter, adapter->mbox, &cmd, sizeof(cmd), &cmd); +} + +/** + * t4_read_pace_tbl - read the pace table + * @adap: the adapter + * @pace_vals: holds the returned values + * + * Returns the values of TP's pace table in microseconds. + */ +void t4_read_pace_tbl(struct adapter *adap, unsigned int pace_vals[NTX_SCHED]) +{ + unsigned int i, v; + + for (i = 0; i < NTX_SCHED; i++) { + t4_write_reg(adap, TP_PACE_TABLE_A, 0xffff0000 + i); + v = t4_read_reg(adap, TP_PACE_TABLE_A); + pace_vals[i] = dack_ticks_to_usec(adap, v); + } +} + +/** + * t4_get_tx_sched - get the configuration of a Tx HW traffic scheduler + * @adap: the adapter + * @sched: the scheduler index + * @kbps: the byte rate in Kbps + * @ipg: the interpacket delay in tenths of nanoseconds + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Return the current configuration of a HW Tx scheduler. + */ +void t4_get_tx_sched(struct adapter *adap, unsigned int sched, + unsigned int *kbps, unsigned int *ipg, bool sleep_ok) +{ + unsigned int v, addr, bpt, cpt; + + if (kbps) { + addr = TP_TX_MOD_Q1_Q0_RATE_LIMIT_A - sched / 2; + t4_tp_tm_pio_read(adap, &v, 1, addr, sleep_ok); + if (sched & 1) + v >>= 16; + bpt = (v >> 8) & 0xff; + cpt = v & 0xff; + if (!cpt) { + *kbps = 0; /* scheduler disabled */ + } else { + v = (adap->params.vpd.cclk * 1000) / cpt; /* ticks/s */ + *kbps = (v * bpt) / 125; + } + } + if (ipg) { + addr = TP_TX_MOD_Q1_Q0_TIMER_SEPARATOR_A - sched / 2; + t4_tp_tm_pio_read(adap, &v, 1, addr, sleep_ok); + if (sched & 1) + v >>= 16; + v &= 0xffff; + *ipg = (10000 * v) / core_ticks_per_usec(adap); + } +} + +/* t4_sge_ctxt_rd - read an SGE context through FW + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @cid: the context id + * @ctype: the context type + * @data: where to store the context data + * + * Issues a FW command through the given mailbox to read an SGE context. + */ +int t4_sge_ctxt_rd(struct adapter *adap, unsigned int mbox, unsigned int cid, + enum ctxt_type ctype, u32 *data) +{ + struct fw_ldst_cmd c; + int ret; + + if (ctype == CTXT_FLM) + ret = FW_LDST_ADDRSPC_SGE_FLMC; + else + ret = FW_LDST_ADDRSPC_SGE_CONMC; + + memset(&c, 0, sizeof(c)); + c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F | + FW_LDST_CMD_ADDRSPACE_V(ret)); + c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c)); + c.u.idctxt.physid = cpu_to_be32(cid); + + ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c); + if (ret == 0) { + data[0] = be32_to_cpu(c.u.idctxt.ctxt_data0); + data[1] = be32_to_cpu(c.u.idctxt.ctxt_data1); + data[2] = be32_to_cpu(c.u.idctxt.ctxt_data2); + data[3] = be32_to_cpu(c.u.idctxt.ctxt_data3); + data[4] = be32_to_cpu(c.u.idctxt.ctxt_data4); + data[5] = be32_to_cpu(c.u.idctxt.ctxt_data5); + } + return ret; +} + +/** + * t4_sge_ctxt_rd_bd - read an SGE context bypassing FW + * @adap: the adapter + * @cid: the context id + * @ctype: the context type + * @data: where to store the context data + * + * Reads an SGE context directly, bypassing FW. This is only for + * debugging when FW is unavailable. + */ +int t4_sge_ctxt_rd_bd(struct adapter *adap, unsigned int cid, + enum ctxt_type ctype, u32 *data) +{ + int i, ret; + + t4_write_reg(adap, SGE_CTXT_CMD_A, CTXTQID_V(cid) | CTXTTYPE_V(ctype)); + ret = t4_wait_op_done(adap, SGE_CTXT_CMD_A, BUSY_F, 0, 3, 1); + if (!ret) + for (i = SGE_CTXT_DATA0_A; i <= SGE_CTXT_DATA5_A; i += 4) + *data++ = t4_read_reg(adap, i); + return ret; +} + +int t4_sched_params(struct adapter *adapter, u8 type, u8 level, u8 mode, + u8 rateunit, u8 ratemode, u8 channel, u8 class, + u32 minrate, u32 maxrate, u16 weight, u16 pktsize, + u16 burstsize) +{ + struct fw_sched_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_SCHED_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F); + cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); + + cmd.u.params.sc = FW_SCHED_SC_PARAMS; + cmd.u.params.type = type; + cmd.u.params.level = level; + cmd.u.params.mode = mode; + cmd.u.params.ch = channel; + cmd.u.params.cl = class; + cmd.u.params.unit = rateunit; + cmd.u.params.rate = ratemode; + cmd.u.params.min = cpu_to_be32(minrate); + cmd.u.params.max = cpu_to_be32(maxrate); + cmd.u.params.weight = cpu_to_be16(weight); + cmd.u.params.pktsize = cpu_to_be16(pktsize); + cmd.u.params.burstsize = cpu_to_be16(burstsize); + + return t4_wr_mbox_meat(adapter, adapter->mbox, &cmd, sizeof(cmd), + NULL, 1); +} + +/** + * t4_i2c_rd - read I2C data from adapter + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @port: Port number if per-port device; <0 if not + * @devid: per-port device ID or absolute device ID + * @offset: byte offset into device I2C space + * @len: byte length of I2C space data + * @buf: buffer in which to return I2C data + * + * Reads the I2C data from the indicated device and location. + */ +int t4_i2c_rd(struct adapter *adap, unsigned int mbox, int port, + unsigned int devid, unsigned int offset, + unsigned int len, u8 *buf) +{ + struct fw_ldst_cmd ldst_cmd, ldst_rpl; + unsigned int i2c_max = sizeof(ldst_cmd.u.i2c.data); + int ret = 0; + + if (len > I2C_PAGE_SIZE) + return -EINVAL; + + /* Dont allow reads that spans multiple pages */ + if (offset < I2C_PAGE_SIZE && offset + len > I2C_PAGE_SIZE) + return -EINVAL; + + memset(&ldst_cmd, 0, sizeof(ldst_cmd)); + ldst_cmd.op_to_addrspace = + cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F | + FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_I2C)); + ldst_cmd.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst_cmd)); + ldst_cmd.u.i2c.pid = (port < 0 ? 0xff : port); + ldst_cmd.u.i2c.did = devid; + + while (len > 0) { + unsigned int i2c_len = (len < i2c_max) ? len : i2c_max; + + ldst_cmd.u.i2c.boffset = offset; + ldst_cmd.u.i2c.blen = i2c_len; + + ret = t4_wr_mbox(adap, mbox, &ldst_cmd, sizeof(ldst_cmd), + &ldst_rpl); + if (ret) + break; + + memcpy(buf, ldst_rpl.u.i2c.data, i2c_len); + offset += i2c_len; + buf += i2c_len; + len -= i2c_len; + } + + return ret; +} + +/** + * t4_set_vlan_acl - Set a VLAN id for the specified VF + * @adap: the adapter + * @mbox: mailbox to use for the FW command + * @vf: one of the VFs instantiated by the specified PF + * @vlan: The vlanid to be set + */ +int t4_set_vlan_acl(struct adapter *adap, unsigned int mbox, unsigned int vf, + u16 vlan) +{ + struct fw_acl_vlan_cmd vlan_cmd; + unsigned int enable; + + enable = (vlan ? FW_ACL_VLAN_CMD_EN_F : 0); + memset(&vlan_cmd, 0, sizeof(vlan_cmd)); + vlan_cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_ACL_VLAN_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_CMD_EXEC_F | + FW_ACL_VLAN_CMD_PFN_V(adap->pf) | + FW_ACL_VLAN_CMD_VFN_V(vf)); + vlan_cmd.en_to_len16 = cpu_to_be32(enable | FW_LEN16(vlan_cmd)); + /* Drop all packets that donot match vlan id */ + vlan_cmd.dropnovlan_fm = (enable + ? (FW_ACL_VLAN_CMD_DROPNOVLAN_F | + FW_ACL_VLAN_CMD_FM_F) : 0); + if (enable != 0) { + vlan_cmd.nvlan = 1; + vlan_cmd.vlanid[0] = cpu_to_be16(vlan); + } + + return t4_wr_mbox(adap, adap->mbox, &vlan_cmd, sizeof(vlan_cmd), NULL); +} + +/** + * modify_device_id - Modifies the device ID of the Boot BIOS image + * @device_id: the device ID to write. + * @boot_data: the boot image to modify. + * + * Write the supplied device ID to the boot BIOS image. + */ +static void modify_device_id(int device_id, u8 *boot_data) +{ + struct cxgb4_pcir_data *pcir_header; + struct legacy_pci_rom_hdr *header; + u8 *cur_header = boot_data; + u16 pcir_offset; + + /* Loop through all chained images and change the device ID's */ + do { + header = (struct legacy_pci_rom_hdr *)cur_header; + pcir_offset = le16_to_cpu(header->pcir_offset); + pcir_header = (struct cxgb4_pcir_data *)(cur_header + + pcir_offset); + + /** + * Only modify the Device ID if code type is Legacy or HP. + * 0x00: Okay to modify + * 0x01: FCODE. Do not modify + * 0x03: Okay to modify + * 0x04-0xFF: Do not modify + */ + if (pcir_header->code_type == CXGB4_HDR_CODE1) { + u8 csum = 0; + int i; + + /** + * Modify Device ID to match current adatper + */ + pcir_header->device_id = cpu_to_le16(device_id); + + /** + * Set checksum temporarily to 0. + * We will recalculate it later. + */ + header->cksum = 0x0; + + /** + * Calculate and update checksum + */ + for (i = 0; i < (header->size512 * 512); i++) + csum += cur_header[i]; + + /** + * Invert summed value to create the checksum + * Writing new checksum value directly to the boot data + */ + cur_header[7] = -csum; + + } else if (pcir_header->code_type == CXGB4_HDR_CODE2) { + /** + * Modify Device ID to match current adatper + */ + pcir_header->device_id = cpu_to_le16(device_id); + } + + /** + * Move header pointer up to the next image in the ROM. + */ + cur_header += header->size512 * 512; + } while (!(pcir_header->indicator & CXGB4_HDR_INDI)); +} + +/** + * t4_load_boot - download boot flash + * @adap: the adapter + * @boot_data: the boot image to write + * @boot_addr: offset in flash to write boot_data + * @size: image size + * + * Write the supplied boot image to the card's serial flash. + * The boot image has the following sections: a 28-byte header and the + * boot image. + */ +int t4_load_boot(struct adapter *adap, u8 *boot_data, + unsigned int boot_addr, unsigned int size) +{ + unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec; + unsigned int boot_sector = (boot_addr * 1024); + struct cxgb4_pci_exp_rom_header *header; + struct cxgb4_pcir_data *pcir_header; + int pcir_offset; + unsigned int i; + u16 device_id; + int ret, addr; + + /** + * Make sure the boot image does not encroach on the firmware region + */ + if ((boot_sector + size) >> 16 > FLASH_FW_START_SEC) { + dev_err(adap->pdev_dev, "boot image encroaching on firmware region\n"); + return -EFBIG; + } + + /* Get boot header */ + header = (struct cxgb4_pci_exp_rom_header *)boot_data; + pcir_offset = le16_to_cpu(header->pcir_offset); + /* PCIR Data Structure */ + pcir_header = (struct cxgb4_pcir_data *)&boot_data[pcir_offset]; + + /** + * Perform some primitive sanity testing to avoid accidentally + * writing garbage over the boot sectors. We ought to check for + * more but it's not worth it for now ... + */ + if (size < BOOT_MIN_SIZE || size > BOOT_MAX_SIZE) { + dev_err(adap->pdev_dev, "boot image too small/large\n"); + return -EFBIG; + } + + if (le16_to_cpu(header->signature) != BOOT_SIGNATURE) { + dev_err(adap->pdev_dev, "Boot image missing signature\n"); + return -EINVAL; + } + + /* Check PCI header signature */ + if (le32_to_cpu(pcir_header->signature) != PCIR_SIGNATURE) { + dev_err(adap->pdev_dev, "PCI header missing signature\n"); + return -EINVAL; + } + + /* Check Vendor ID matches Chelsio ID*/ + if (le16_to_cpu(pcir_header->vendor_id) != PCI_VENDOR_ID_CHELSIO) { + dev_err(adap->pdev_dev, "Vendor ID missing signature\n"); + return -EINVAL; + } + + /** + * The boot sector is comprised of the Expansion-ROM boot, iSCSI boot, + * and Boot configuration data sections. These 3 boot sections span + * sectors 0 to 7 in flash and live right before the FW image location. + */ + i = DIV_ROUND_UP(size ? size : FLASH_FW_START, sf_sec_size); + ret = t4_flash_erase_sectors(adap, boot_sector >> 16, + (boot_sector >> 16) + i - 1); + + /** + * If size == 0 then we're simply erasing the FLASH sectors associated + * with the on-adapter option ROM file + */ + if (ret || size == 0) + goto out; + /* Retrieve adapter's device ID */ + pci_read_config_word(adap->pdev, PCI_DEVICE_ID, &device_id); + /* Want to deal with PF 0 so I strip off PF 4 indicator */ + device_id = device_id & 0xf0ff; + + /* Check PCIE Device ID */ + if (le16_to_cpu(pcir_header->device_id) != device_id) { + /** + * Change the device ID in the Boot BIOS image to match + * the Device ID of the current adapter. + */ + modify_device_id(device_id, boot_data); + } + + /** + * Skip over the first SF_PAGE_SIZE worth of data and write it after + * we finish copying the rest of the boot image. This will ensure + * that the BIOS boot header will only be written if the boot image + * was written in full. + */ + addr = boot_sector; + for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) { + addr += SF_PAGE_SIZE; + boot_data += SF_PAGE_SIZE; + ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, boot_data, + false); + if (ret) + goto out; + } + + ret = t4_write_flash(adap, boot_sector, SF_PAGE_SIZE, + (const u8 *)header, false); + +out: + if (ret) + dev_err(adap->pdev_dev, "boot image load failed, error %d\n", + ret); + return ret; +} + +/** + * t4_flash_bootcfg_addr - return the address of the flash + * optionrom configuration + * @adapter: the adapter + * + * Return the address within the flash where the OptionROM Configuration + * is stored, or an error if the device FLASH is too small to contain + * a OptionROM Configuration. + */ +static int t4_flash_bootcfg_addr(struct adapter *adapter) +{ + /** + * If the device FLASH isn't large enough to hold a Firmware + * Configuration File, return an error. + */ + if (adapter->params.sf_size < + FLASH_BOOTCFG_START + FLASH_BOOTCFG_MAX_SIZE) + return -ENOSPC; + + return FLASH_BOOTCFG_START; +} + +int t4_load_bootcfg(struct adapter *adap, const u8 *cfg_data, unsigned int size) +{ + unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec; + struct cxgb4_bootcfg_data *header; + unsigned int flash_cfg_start_sec; + unsigned int addr, npad; + int ret, i, n, cfg_addr; + + cfg_addr = t4_flash_bootcfg_addr(adap); + if (cfg_addr < 0) + return cfg_addr; + + addr = cfg_addr; + flash_cfg_start_sec = addr / SF_SEC_SIZE; + + if (size > FLASH_BOOTCFG_MAX_SIZE) { + dev_err(adap->pdev_dev, "bootcfg file too large, max is %u bytes\n", + FLASH_BOOTCFG_MAX_SIZE); + return -EFBIG; + } + + header = (struct cxgb4_bootcfg_data *)cfg_data; + if (le16_to_cpu(header->signature) != BOOT_CFG_SIG) { + dev_err(adap->pdev_dev, "Wrong bootcfg signature\n"); + ret = -EINVAL; + goto out; + } + + i = DIV_ROUND_UP(FLASH_BOOTCFG_MAX_SIZE, + sf_sec_size); + ret = t4_flash_erase_sectors(adap, flash_cfg_start_sec, + flash_cfg_start_sec + i - 1); + + /** + * If size == 0 then we're simply erasing the FLASH sectors associated + * with the on-adapter OptionROM Configuration File. + */ + if (ret || size == 0) + goto out; + + /* this will write to the flash up to SF_PAGE_SIZE at a time */ + for (i = 0; i < size; i += SF_PAGE_SIZE) { + n = min_t(u32, size - i, SF_PAGE_SIZE); + + ret = t4_write_flash(adap, addr, n, cfg_data, false); + if (ret) + goto out; + + addr += SF_PAGE_SIZE; + cfg_data += SF_PAGE_SIZE; + } + + npad = ((size + 4 - 1) & ~3) - size; + for (i = 0; i < npad; i++) { + u8 data = 0; + + ret = t4_write_flash(adap, cfg_addr + size + i, 1, &data, + false); + if (ret) + goto out; + } + +out: + if (ret) + dev_err(adap->pdev_dev, "boot config data %s failed %d\n", + (size == 0 ? "clear" : "download"), ret); + return ret; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.h b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.h new file mode 100644 index 000000000..63bc956d2 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.h @@ -0,0 +1,303 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __T4_HW_H +#define __T4_HW_H + +#include <linux/types.h> + +enum { + NCHAN = 4, /* # of HW channels */ + MAX_MTU = 9600, /* max MAC MTU, excluding header + FCS */ + EEPROMSIZE = 17408,/* Serial EEPROM physical size */ + EEPROMVSIZE = 32768,/* Serial EEPROM virtual address space size */ + EEPROMPFSIZE = 1024, /* EEPROM writable area size for PFn, n>0 */ + RSS_NENTRIES = 2048, /* # of entries in RSS mapping table */ + T6_RSS_NENTRIES = 4096, /* # of entries in RSS mapping table */ + TCB_SIZE = 128, /* TCB size */ + NMTUS = 16, /* size of MTU table */ + NCCTRL_WIN = 32, /* # of congestion control windows */ + NTX_SCHED = 8, /* # of HW Tx scheduling queues */ + PM_NSTATS = 5, /* # of PM stats */ + T6_PM_NSTATS = 7, /* # of PM stats in T6 */ + MBOX_LEN = 64, /* mailbox size in bytes */ + TRACE_LEN = 112, /* length of trace data and mask */ + FILTER_OPT_LEN = 36, /* filter tuple width for optional components */ +}; + +enum { + CIM_NUM_IBQ = 6, /* # of CIM IBQs */ + CIM_NUM_OBQ = 6, /* # of CIM OBQs */ + CIM_NUM_OBQ_T5 = 8, /* # of CIM OBQs for T5 adapter */ + CIMLA_SIZE = 2048, /* # of 32-bit words in CIM LA */ + CIM_PIFLA_SIZE = 64, /* # of 192-bit words in CIM PIF LA */ + CIM_MALA_SIZE = 64, /* # of 160-bit words in CIM MA LA */ + CIM_IBQ_SIZE = 128, /* # of 128-bit words in a CIM IBQ */ + CIM_OBQ_SIZE = 128, /* # of 128-bit words in a CIM OBQ */ + TPLA_SIZE = 128, /* # of 64-bit words in TP LA */ + ULPRX_LA_SIZE = 512, /* # of 256-bit words in ULP_RX LA */ +}; + +/* SGE context types */ +enum ctxt_type { + CTXT_EGRESS, + CTXT_INGRESS, + CTXT_FLM, + CTXT_CNM, +}; + +enum { + SF_PAGE_SIZE = 256, /* serial flash page size */ + SF_SEC_SIZE = 64 * 1024, /* serial flash sector size */ +}; + +enum { RSP_TYPE_FLBUF, RSP_TYPE_CPL, RSP_TYPE_INTR }; /* response entry types */ + +enum { MBOX_OWNER_NONE, MBOX_OWNER_FW, MBOX_OWNER_DRV }; /* mailbox owners */ + +enum { + SGE_MAX_WR_LEN = 512, /* max WR size in bytes */ + SGE_CTXT_SIZE = 24, /* size of SGE context */ + SGE_NTIMERS = 6, /* # of interrupt holdoff timer values */ + SGE_NCOUNTERS = 4, /* # of interrupt packet counter values */ + SGE_NDBQTIMERS = 8, /* # of Doorbell Queue Timer values */ + SGE_MAX_IQ_SIZE = 65520, + + SGE_TIMER_RSTRT_CNTR = 6, /* restart RX packet threshold counter */ + SGE_TIMER_UPD_CIDX = 7, /* update cidx only */ + + SGE_EQ_IDXSIZE = 64, /* egress queue pidx/cidx unit size */ + + SGE_INTRDST_PCI = 0, /* interrupt destination is PCI-E */ + SGE_INTRDST_IQ = 1, /* destination is an ingress queue */ + + SGE_UPDATEDEL_NONE = 0, /* ingress queue pidx update delivery */ + SGE_UPDATEDEL_INTR = 1, /* interrupt */ + SGE_UPDATEDEL_STPG = 2, /* status page */ + SGE_UPDATEDEL_BOTH = 3, /* interrupt and status page */ + + SGE_HOSTFCMODE_NONE = 0, /* egress queue cidx updates */ + SGE_HOSTFCMODE_IQ = 1, /* sent to ingress queue */ + SGE_HOSTFCMODE_STPG = 2, /* sent to status page */ + SGE_HOSTFCMODE_BOTH = 3, /* ingress queue and status page */ + + SGE_FETCHBURSTMIN_16B = 0,/* egress queue descriptor fetch minimum */ + SGE_FETCHBURSTMIN_32B = 1, + SGE_FETCHBURSTMIN_64B = 2, + SGE_FETCHBURSTMIN_128B = 3, + + SGE_FETCHBURSTMAX_64B = 0,/* egress queue descriptor fetch maximum */ + SGE_FETCHBURSTMAX_128B = 1, + SGE_FETCHBURSTMAX_256B = 2, + SGE_FETCHBURSTMAX_512B = 3, + + SGE_CIDXFLUSHTHRESH_1 = 0,/* egress queue cidx flush threshold */ + SGE_CIDXFLUSHTHRESH_2 = 1, + SGE_CIDXFLUSHTHRESH_4 = 2, + SGE_CIDXFLUSHTHRESH_8 = 3, + SGE_CIDXFLUSHTHRESH_16 = 4, + SGE_CIDXFLUSHTHRESH_32 = 5, + SGE_CIDXFLUSHTHRESH_64 = 6, + SGE_CIDXFLUSHTHRESH_128 = 7, + + SGE_INGPADBOUNDARY_SHIFT = 5,/* ingress queue pad boundary */ +}; + +/* PCI-e memory window access */ +enum pcie_memwin { + MEMWIN_NIC = 0, + MEMWIN_RSVD1 = 1, + MEMWIN_RSVD2 = 2, + MEMWIN_RDMA = 3, + MEMWIN_RSVD4 = 4, + MEMWIN_FOISCSI = 5, + MEMWIN_CSIOSTOR = 6, + MEMWIN_RSVD7 = 7, +}; + +struct sge_qstat { /* data written to SGE queue status entries */ + __be32 qid; + __be16 cidx; + __be16 pidx; +}; + +/* + * Structure for last 128 bits of response descriptors + */ +struct rsp_ctrl { + __be32 hdrbuflen_pidx; + __be32 pldbuflen_qid; + union { + u8 type_gen; + __be64 last_flit; + }; +}; + +#define RSPD_NEWBUF_S 31 +#define RSPD_NEWBUF_V(x) ((x) << RSPD_NEWBUF_S) +#define RSPD_NEWBUF_F RSPD_NEWBUF_V(1U) + +#define RSPD_LEN_S 0 +#define RSPD_LEN_M 0x7fffffff +#define RSPD_LEN_G(x) (((x) >> RSPD_LEN_S) & RSPD_LEN_M) + +#define RSPD_QID_S RSPD_LEN_S +#define RSPD_QID_M RSPD_LEN_M +#define RSPD_QID_G(x) RSPD_LEN_G(x) + +#define RSPD_GEN_S 7 + +#define RSPD_TYPE_S 4 +#define RSPD_TYPE_M 0x3 +#define RSPD_TYPE_G(x) (((x) >> RSPD_TYPE_S) & RSPD_TYPE_M) + +/* Rx queue interrupt deferral fields: counter enable and timer index */ +#define QINTR_CNT_EN_S 0 +#define QINTR_CNT_EN_V(x) ((x) << QINTR_CNT_EN_S) +#define QINTR_CNT_EN_F QINTR_CNT_EN_V(1U) + +#define QINTR_TIMER_IDX_S 1 +#define QINTR_TIMER_IDX_M 0x7 +#define QINTR_TIMER_IDX_V(x) ((x) << QINTR_TIMER_IDX_S) +#define QINTR_TIMER_IDX_G(x) (((x) >> QINTR_TIMER_IDX_S) & QINTR_TIMER_IDX_M) + +/* + * Flash layout. + */ +#define FLASH_START(start) ((start) * SF_SEC_SIZE) +#define FLASH_MAX_SIZE(nsecs) ((nsecs) * SF_SEC_SIZE) + +enum { + /* + * Various Expansion-ROM boot images, etc. + */ + FLASH_EXP_ROM_START_SEC = 0, + FLASH_EXP_ROM_NSECS = 6, + FLASH_EXP_ROM_START = FLASH_START(FLASH_EXP_ROM_START_SEC), + FLASH_EXP_ROM_MAX_SIZE = FLASH_MAX_SIZE(FLASH_EXP_ROM_NSECS), + + /* + * iSCSI Boot Firmware Table (iBFT) and other driver-related + * parameters ... + */ + FLASH_IBFT_START_SEC = 6, + FLASH_IBFT_NSECS = 1, + FLASH_IBFT_START = FLASH_START(FLASH_IBFT_START_SEC), + FLASH_IBFT_MAX_SIZE = FLASH_MAX_SIZE(FLASH_IBFT_NSECS), + + /* + * Boot configuration data. + */ + FLASH_BOOTCFG_START_SEC = 7, + FLASH_BOOTCFG_NSECS = 1, + FLASH_BOOTCFG_START = FLASH_START(FLASH_BOOTCFG_START_SEC), + FLASH_BOOTCFG_MAX_SIZE = FLASH_MAX_SIZE(FLASH_BOOTCFG_NSECS), + + /* + * Location of firmware image in FLASH. + */ + FLASH_FW_START_SEC = 8, + FLASH_FW_NSECS = 16, + FLASH_FW_START = FLASH_START(FLASH_FW_START_SEC), + FLASH_FW_MAX_SIZE = FLASH_MAX_SIZE(FLASH_FW_NSECS), + + /* Location of bootstrap firmware image in FLASH. + */ + FLASH_FWBOOTSTRAP_START_SEC = 27, + FLASH_FWBOOTSTRAP_NSECS = 1, + FLASH_FWBOOTSTRAP_START = FLASH_START(FLASH_FWBOOTSTRAP_START_SEC), + FLASH_FWBOOTSTRAP_MAX_SIZE = FLASH_MAX_SIZE(FLASH_FWBOOTSTRAP_NSECS), + + /* + * iSCSI persistent/crash information. + */ + FLASH_ISCSI_CRASH_START_SEC = 29, + FLASH_ISCSI_CRASH_NSECS = 1, + FLASH_ISCSI_CRASH_START = FLASH_START(FLASH_ISCSI_CRASH_START_SEC), + FLASH_ISCSI_CRASH_MAX_SIZE = FLASH_MAX_SIZE(FLASH_ISCSI_CRASH_NSECS), + + /* + * FCoE persistent/crash information. + */ + FLASH_FCOE_CRASH_START_SEC = 30, + FLASH_FCOE_CRASH_NSECS = 1, + FLASH_FCOE_CRASH_START = FLASH_START(FLASH_FCOE_CRASH_START_SEC), + FLASH_FCOE_CRASH_MAX_SIZE = FLASH_MAX_SIZE(FLASH_FCOE_CRASH_NSECS), + + /* + * Location of Firmware Configuration File in FLASH. Since the FPGA + * "FLASH" is smaller we need to store the Configuration File in a + * different location -- which will overlap the end of the firmware + * image if firmware ever gets that large ... + */ + FLASH_CFG_START_SEC = 31, + FLASH_CFG_NSECS = 1, + FLASH_CFG_START = FLASH_START(FLASH_CFG_START_SEC), + FLASH_CFG_MAX_SIZE = FLASH_MAX_SIZE(FLASH_CFG_NSECS), + + /* We don't support FLASH devices which can't support the full + * standard set of sections which we need for normal + * operations. + */ + FLASH_MIN_SIZE = FLASH_CFG_START + FLASH_CFG_MAX_SIZE, + + FLASH_FPGA_CFG_START_SEC = 15, + FLASH_FPGA_CFG_START = FLASH_START(FLASH_FPGA_CFG_START_SEC), + + /* + * Sectors 32-63 are reserved for FLASH failover. + */ +}; + +#undef FLASH_START +#undef FLASH_MAX_SIZE + +#define SGE_TIMESTAMP_S 0 +#define SGE_TIMESTAMP_M 0xfffffffffffffffULL +#define SGE_TIMESTAMP_V(x) ((__u64)(x) << SGE_TIMESTAMP_S) +#define SGE_TIMESTAMP_G(x) (((__u64)(x) >> SGE_TIMESTAMP_S) & SGE_TIMESTAMP_M) + +#define I2C_DEV_ADDR_A0 0xa0 +#define I2C_DEV_ADDR_A2 0xa2 +#define I2C_PAGE_SIZE 0x100 +#define SFP_DIAG_TYPE_ADDR 0x5c +#define SFP_DIAG_TYPE_LEN 0x1 +#define SFP_DIAG_ADDRMODE BIT(2) +#define SFP_DIAG_IMPLEMENTED BIT(6) +#define SFF_8472_COMP_ADDR 0x5e +#define SFF_8472_COMP_LEN 0x1 +#define SFF_REV_ADDR 0x1 +#define SFF_REV_LEN 0x1 + +#endif /* __T4_HW_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h new file mode 100644 index 000000000..fed5f93bf --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h @@ -0,0 +1,2352 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __T4_MSG_H +#define __T4_MSG_H + +#include <linux/types.h> + +enum { + CPL_PASS_OPEN_REQ = 0x1, + CPL_PASS_ACCEPT_RPL = 0x2, + CPL_ACT_OPEN_REQ = 0x3, + CPL_SET_TCB_FIELD = 0x5, + CPL_GET_TCB = 0x6, + CPL_CLOSE_CON_REQ = 0x8, + CPL_CLOSE_LISTSRV_REQ = 0x9, + CPL_ABORT_REQ = 0xA, + CPL_ABORT_RPL = 0xB, + CPL_TX_DATA = 0xC, + CPL_RX_DATA_ACK = 0xD, + CPL_TX_PKT = 0xE, + CPL_L2T_WRITE_REQ = 0x12, + CPL_SMT_WRITE_REQ = 0x14, + CPL_TID_RELEASE = 0x1A, + CPL_SRQ_TABLE_REQ = 0x1C, + CPL_TX_DATA_ISO = 0x1F, + + CPL_CLOSE_LISTSRV_RPL = 0x20, + CPL_GET_TCB_RPL = 0x22, + CPL_L2T_WRITE_RPL = 0x23, + CPL_PASS_OPEN_RPL = 0x24, + CPL_ACT_OPEN_RPL = 0x25, + CPL_PEER_CLOSE = 0x26, + CPL_ABORT_REQ_RSS = 0x2B, + CPL_ABORT_RPL_RSS = 0x2D, + CPL_SMT_WRITE_RPL = 0x2E, + + CPL_RX_PHYS_ADDR = 0x30, + CPL_CLOSE_CON_RPL = 0x32, + CPL_ISCSI_HDR = 0x33, + CPL_RDMA_CQE = 0x35, + CPL_RDMA_CQE_READ_RSP = 0x36, + CPL_RDMA_CQE_ERR = 0x37, + CPL_RX_DATA = 0x39, + CPL_SET_TCB_RPL = 0x3A, + CPL_RX_PKT = 0x3B, + CPL_RX_DDP_COMPLETE = 0x3F, + + CPL_ACT_ESTABLISH = 0x40, + CPL_PASS_ESTABLISH = 0x41, + CPL_RX_DATA_DDP = 0x42, + CPL_PASS_ACCEPT_REQ = 0x44, + CPL_RX_ISCSI_CMP = 0x45, + CPL_TRACE_PKT_T5 = 0x48, + CPL_RX_ISCSI_DDP = 0x49, + CPL_RX_TLS_CMP = 0x4E, + + CPL_RDMA_READ_REQ = 0x60, + + CPL_PASS_OPEN_REQ6 = 0x81, + CPL_ACT_OPEN_REQ6 = 0x83, + + CPL_TX_TLS_PDU = 0x88, + CPL_TX_TLS_SFO = 0x89, + CPL_TX_SEC_PDU = 0x8A, + CPL_TX_TLS_ACK = 0x8B, + + CPL_RDMA_TERMINATE = 0xA2, + CPL_RDMA_WRITE = 0xA4, + CPL_SGE_EGR_UPDATE = 0xA5, + CPL_RX_MPS_PKT = 0xAF, + + CPL_TRACE_PKT = 0xB0, + CPL_TLS_DATA = 0xB1, + CPL_ISCSI_DATA = 0xB2, + + CPL_FW4_MSG = 0xC0, + CPL_FW4_PLD = 0xC1, + CPL_FW4_ACK = 0xC3, + CPL_SRQ_TABLE_RPL = 0xCC, + + CPL_RX_PHYS_DSGL = 0xD0, + + CPL_FW6_MSG = 0xE0, + CPL_FW6_PLD = 0xE1, + CPL_TX_TNL_LSO = 0xEC, + CPL_TX_PKT_LSO = 0xED, + CPL_TX_PKT_XT = 0xEE, + + NUM_CPL_CMDS +}; + +enum CPL_error { + CPL_ERR_NONE = 0, + CPL_ERR_TCAM_PARITY = 1, + CPL_ERR_TCAM_MISS = 2, + CPL_ERR_TCAM_FULL = 3, + CPL_ERR_BAD_LENGTH = 15, + CPL_ERR_BAD_ROUTE = 18, + CPL_ERR_CONN_RESET = 20, + CPL_ERR_CONN_EXIST_SYNRECV = 21, + CPL_ERR_CONN_EXIST = 22, + CPL_ERR_ARP_MISS = 23, + CPL_ERR_BAD_SYN = 24, + CPL_ERR_CONN_TIMEDOUT = 30, + CPL_ERR_XMIT_TIMEDOUT = 31, + CPL_ERR_PERSIST_TIMEDOUT = 32, + CPL_ERR_FINWAIT2_TIMEDOUT = 33, + CPL_ERR_KEEPALIVE_TIMEDOUT = 34, + CPL_ERR_RTX_NEG_ADVICE = 35, + CPL_ERR_PERSIST_NEG_ADVICE = 36, + CPL_ERR_KEEPALV_NEG_ADVICE = 37, + CPL_ERR_ABORT_FAILED = 42, + CPL_ERR_IWARP_FLM = 50, + CPL_CONTAINS_READ_RPL = 60, + CPL_CONTAINS_WRITE_RPL = 61, +}; + +enum { + CPL_CONN_POLICY_AUTO = 0, + CPL_CONN_POLICY_ASK = 1, + CPL_CONN_POLICY_FILTER = 2, + CPL_CONN_POLICY_DENY = 3 +}; + +enum { + ULP_MODE_NONE = 0, + ULP_MODE_ISCSI = 2, + ULP_MODE_RDMA = 4, + ULP_MODE_TCPDDP = 5, + ULP_MODE_FCOE = 6, + ULP_MODE_TLS = 8, +}; + +enum { + ULP_CRC_HEADER = 1 << 0, + ULP_CRC_DATA = 1 << 1 +}; + +enum { + CPL_ABORT_SEND_RST = 0, + CPL_ABORT_NO_RST, +}; + +enum { /* TX_PKT_XT checksum types */ + TX_CSUM_TCP = 0, + TX_CSUM_UDP = 1, + TX_CSUM_CRC16 = 4, + TX_CSUM_CRC32 = 5, + TX_CSUM_CRC32C = 6, + TX_CSUM_FCOE = 7, + TX_CSUM_TCPIP = 8, + TX_CSUM_UDPIP = 9, + TX_CSUM_TCPIP6 = 10, + TX_CSUM_UDPIP6 = 11, + TX_CSUM_IP = 12, +}; + +union opcode_tid { + __be32 opcode_tid; + u8 opcode; +}; + +#define CPL_OPCODE_S 24 +#define CPL_OPCODE_V(x) ((x) << CPL_OPCODE_S) +#define CPL_OPCODE_G(x) (((x) >> CPL_OPCODE_S) & 0xFF) +#define TID_G(x) ((x) & 0xFFFFFF) + +/* tid is assumed to be 24-bits */ +#define MK_OPCODE_TID(opcode, tid) (CPL_OPCODE_V(opcode) | (tid)) + +#define OPCODE_TID(cmd) ((cmd)->ot.opcode_tid) + +/* extract the TID from a CPL command */ +#define GET_TID(cmd) (TID_G(be32_to_cpu(OPCODE_TID(cmd)))) + +/* partitioning of TID fields that also carry a queue id */ +#define TID_TID_S 0 +#define TID_TID_M 0x3fff +#define TID_TID_V(x) ((x) << TID_TID_S) +#define TID_TID_G(x) (((x) >> TID_TID_S) & TID_TID_M) + +#define TID_QID_S 14 +#define TID_QID_M 0x3ff +#define TID_QID_V(x) ((x) << TID_QID_S) +#define TID_QID_G(x) (((x) >> TID_QID_S) & TID_QID_M) + +struct rss_header { + u8 opcode; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 channel:2; + u8 filter_hit:1; + u8 filter_tid:1; + u8 hash_type:2; + u8 ipv6:1; + u8 send2fw:1; +#else + u8 send2fw:1; + u8 ipv6:1; + u8 hash_type:2; + u8 filter_tid:1; + u8 filter_hit:1; + u8 channel:2; +#endif + __be16 qid; + __be32 hash_val; +}; + +struct work_request_hdr { + __be32 wr_hi; + __be32 wr_mid; + __be64 wr_lo; +}; + +/* wr_hi fields */ +#define WR_OP_S 24 +#define WR_OP_V(x) ((__u64)(x) << WR_OP_S) + +#define WR_HDR struct work_request_hdr wr + +/* option 0 fields */ +#define TX_CHAN_S 2 +#define TX_CHAN_V(x) ((x) << TX_CHAN_S) + +#define ULP_MODE_S 8 +#define ULP_MODE_V(x) ((x) << ULP_MODE_S) + +#define RCV_BUFSIZ_S 12 +#define RCV_BUFSIZ_M 0x3FFU +#define RCV_BUFSIZ_V(x) ((x) << RCV_BUFSIZ_S) + +#define SMAC_SEL_S 28 +#define SMAC_SEL_V(x) ((__u64)(x) << SMAC_SEL_S) + +#define L2T_IDX_S 36 +#define L2T_IDX_V(x) ((__u64)(x) << L2T_IDX_S) + +#define WND_SCALE_S 50 +#define WND_SCALE_V(x) ((__u64)(x) << WND_SCALE_S) + +#define KEEP_ALIVE_S 54 +#define KEEP_ALIVE_V(x) ((__u64)(x) << KEEP_ALIVE_S) +#define KEEP_ALIVE_F KEEP_ALIVE_V(1ULL) + +#define MSS_IDX_S 60 +#define MSS_IDX_M 0xF +#define MSS_IDX_V(x) ((__u64)(x) << MSS_IDX_S) +#define MSS_IDX_G(x) (((x) >> MSS_IDX_S) & MSS_IDX_M) + +/* option 2 fields */ +#define RSS_QUEUE_S 0 +#define RSS_QUEUE_M 0x3FF +#define RSS_QUEUE_V(x) ((x) << RSS_QUEUE_S) +#define RSS_QUEUE_G(x) (((x) >> RSS_QUEUE_S) & RSS_QUEUE_M) + +#define RSS_QUEUE_VALID_S 10 +#define RSS_QUEUE_VALID_V(x) ((x) << RSS_QUEUE_VALID_S) +#define RSS_QUEUE_VALID_F RSS_QUEUE_VALID_V(1U) + +#define RX_FC_DISABLE_S 20 +#define RX_FC_DISABLE_V(x) ((x) << RX_FC_DISABLE_S) +#define RX_FC_DISABLE_F RX_FC_DISABLE_V(1U) + +#define RX_FC_VALID_S 22 +#define RX_FC_VALID_V(x) ((x) << RX_FC_VALID_S) +#define RX_FC_VALID_F RX_FC_VALID_V(1U) + +#define RX_CHANNEL_S 26 +#define RX_CHANNEL_V(x) ((x) << RX_CHANNEL_S) +#define RX_CHANNEL_F RX_CHANNEL_V(1U) + +#define WND_SCALE_EN_S 28 +#define WND_SCALE_EN_V(x) ((x) << WND_SCALE_EN_S) +#define WND_SCALE_EN_F WND_SCALE_EN_V(1U) + +#define T5_OPT_2_VALID_S 31 +#define T5_OPT_2_VALID_V(x) ((x) << T5_OPT_2_VALID_S) +#define T5_OPT_2_VALID_F T5_OPT_2_VALID_V(1U) + +struct cpl_pass_open_req { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be64 opt0; + __be64 opt1; +}; + +/* option 0 fields */ +#define NO_CONG_S 4 +#define NO_CONG_V(x) ((x) << NO_CONG_S) +#define NO_CONG_F NO_CONG_V(1U) + +#define DELACK_S 5 +#define DELACK_V(x) ((x) << DELACK_S) +#define DELACK_F DELACK_V(1U) + +#define NON_OFFLOAD_S 7 +#define NON_OFFLOAD_V(x) ((x) << NON_OFFLOAD_S) +#define NON_OFFLOAD_F NON_OFFLOAD_V(1U) + +#define DSCP_S 22 +#define DSCP_M 0x3F +#define DSCP_V(x) ((x) << DSCP_S) +#define DSCP_G(x) (((x) >> DSCP_S) & DSCP_M) + +#define TCAM_BYPASS_S 48 +#define TCAM_BYPASS_V(x) ((__u64)(x) << TCAM_BYPASS_S) +#define TCAM_BYPASS_F TCAM_BYPASS_V(1ULL) + +#define NAGLE_S 49 +#define NAGLE_V(x) ((__u64)(x) << NAGLE_S) +#define NAGLE_F NAGLE_V(1ULL) + +/* option 1 fields */ +#define SYN_RSS_ENABLE_S 0 +#define SYN_RSS_ENABLE_V(x) ((x) << SYN_RSS_ENABLE_S) +#define SYN_RSS_ENABLE_F SYN_RSS_ENABLE_V(1U) + +#define SYN_RSS_QUEUE_S 2 +#define SYN_RSS_QUEUE_V(x) ((x) << SYN_RSS_QUEUE_S) + +#define CONN_POLICY_S 22 +#define CONN_POLICY_V(x) ((x) << CONN_POLICY_S) + +struct cpl_pass_open_req6 { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be64 local_ip_hi; + __be64 local_ip_lo; + __be64 peer_ip_hi; + __be64 peer_ip_lo; + __be64 opt0; + __be64 opt1; +}; + +struct cpl_pass_open_rpl { + union opcode_tid ot; + u8 rsvd[3]; + u8 status; +}; + +struct tcp_options { + __be16 mss; + __u8 wsf; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8:4; + __u8 unknown:1; + __u8:1; + __u8 sack:1; + __u8 tstamp:1; +#else + __u8 tstamp:1; + __u8 sack:1; + __u8:1; + __u8 unknown:1; + __u8:4; +#endif +}; + +struct cpl_pass_accept_req { + union opcode_tid ot; + __be16 rsvd; + __be16 len; + __be32 hdr_len; + __be16 vlan; + __be16 l2info; + __be32 tos_stid; + struct tcp_options tcpopt; +}; + +/* cpl_pass_accept_req.hdr_len fields */ +#define SYN_RX_CHAN_S 0 +#define SYN_RX_CHAN_M 0xF +#define SYN_RX_CHAN_V(x) ((x) << SYN_RX_CHAN_S) +#define SYN_RX_CHAN_G(x) (((x) >> SYN_RX_CHAN_S) & SYN_RX_CHAN_M) + +#define TCP_HDR_LEN_S 10 +#define TCP_HDR_LEN_M 0x3F +#define TCP_HDR_LEN_V(x) ((x) << TCP_HDR_LEN_S) +#define TCP_HDR_LEN_G(x) (((x) >> TCP_HDR_LEN_S) & TCP_HDR_LEN_M) + +#define IP_HDR_LEN_S 16 +#define IP_HDR_LEN_M 0x3FF +#define IP_HDR_LEN_V(x) ((x) << IP_HDR_LEN_S) +#define IP_HDR_LEN_G(x) (((x) >> IP_HDR_LEN_S) & IP_HDR_LEN_M) + +#define ETH_HDR_LEN_S 26 +#define ETH_HDR_LEN_M 0x1F +#define ETH_HDR_LEN_V(x) ((x) << ETH_HDR_LEN_S) +#define ETH_HDR_LEN_G(x) (((x) >> ETH_HDR_LEN_S) & ETH_HDR_LEN_M) + +/* cpl_pass_accept_req.l2info fields */ +#define SYN_MAC_IDX_S 0 +#define SYN_MAC_IDX_M 0x1FF +#define SYN_MAC_IDX_V(x) ((x) << SYN_MAC_IDX_S) +#define SYN_MAC_IDX_G(x) (((x) >> SYN_MAC_IDX_S) & SYN_MAC_IDX_M) + +#define SYN_XACT_MATCH_S 9 +#define SYN_XACT_MATCH_V(x) ((x) << SYN_XACT_MATCH_S) +#define SYN_XACT_MATCH_F SYN_XACT_MATCH_V(1U) + +#define SYN_INTF_S 12 +#define SYN_INTF_M 0xF +#define SYN_INTF_V(x) ((x) << SYN_INTF_S) +#define SYN_INTF_G(x) (((x) >> SYN_INTF_S) & SYN_INTF_M) + +enum { /* TCP congestion control algorithms */ + CONG_ALG_RENO, + CONG_ALG_TAHOE, + CONG_ALG_NEWRENO, + CONG_ALG_HIGHSPEED +}; + +#define CONG_CNTRL_S 14 +#define CONG_CNTRL_M 0x3 +#define CONG_CNTRL_V(x) ((x) << CONG_CNTRL_S) +#define CONG_CNTRL_G(x) (((x) >> CONG_CNTRL_S) & CONG_CNTRL_M) + +#define T5_ISS_S 18 +#define T5_ISS_V(x) ((x) << T5_ISS_S) +#define T5_ISS_F T5_ISS_V(1U) + +struct cpl_pass_accept_rpl { + WR_HDR; + union opcode_tid ot; + __be32 opt2; + __be64 opt0; +}; + +/* option 2 fields */ +#define RX_COALESCE_VALID_S 11 +#define RX_COALESCE_VALID_V(x) ((x) << RX_COALESCE_VALID_S) +#define RX_COALESCE_VALID_F RX_COALESCE_VALID_V(1U) + +#define RX_COALESCE_S 12 +#define RX_COALESCE_V(x) ((x) << RX_COALESCE_S) + +#define PACE_S 16 +#define PACE_V(x) ((x) << PACE_S) + +#define TX_QUEUE_S 23 +#define TX_QUEUE_M 0x7 +#define TX_QUEUE_V(x) ((x) << TX_QUEUE_S) +#define TX_QUEUE_G(x) (((x) >> TX_QUEUE_S) & TX_QUEUE_M) + +#define CCTRL_ECN_S 27 +#define CCTRL_ECN_V(x) ((x) << CCTRL_ECN_S) +#define CCTRL_ECN_F CCTRL_ECN_V(1U) + +#define TSTAMPS_EN_S 29 +#define TSTAMPS_EN_V(x) ((x) << TSTAMPS_EN_S) +#define TSTAMPS_EN_F TSTAMPS_EN_V(1U) + +#define SACK_EN_S 30 +#define SACK_EN_V(x) ((x) << SACK_EN_S) +#define SACK_EN_F SACK_EN_V(1U) + +struct cpl_t5_pass_accept_rpl { + WR_HDR; + union opcode_tid ot; + __be32 opt2; + __be64 opt0; + __be32 iss; + __be32 rsvd; +}; + +struct cpl_act_open_req { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be64 opt0; + __be32 params; + __be32 opt2; +}; + +#define FILTER_TUPLE_S 24 +#define FILTER_TUPLE_M 0xFFFFFFFFFF +#define FILTER_TUPLE_V(x) ((x) << FILTER_TUPLE_S) +#define FILTER_TUPLE_G(x) (((x) >> FILTER_TUPLE_S) & FILTER_TUPLE_M) +struct cpl_t5_act_open_req { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be64 opt0; + __be32 rsvd; + __be32 opt2; + __be64 params; +}; + +struct cpl_t6_act_open_req { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be64 opt0; + __be32 rsvd; + __be32 opt2; + __be64 params; + __be32 rsvd2; + __be32 opt3; +}; + +struct cpl_act_open_req6 { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be64 local_ip_hi; + __be64 local_ip_lo; + __be64 peer_ip_hi; + __be64 peer_ip_lo; + __be64 opt0; + __be32 params; + __be32 opt2; +}; + +struct cpl_t5_act_open_req6 { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be64 local_ip_hi; + __be64 local_ip_lo; + __be64 peer_ip_hi; + __be64 peer_ip_lo; + __be64 opt0; + __be32 rsvd; + __be32 opt2; + __be64 params; +}; + +struct cpl_t6_act_open_req6 { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be64 local_ip_hi; + __be64 local_ip_lo; + __be64 peer_ip_hi; + __be64 peer_ip_lo; + __be64 opt0; + __be32 rsvd; + __be32 opt2; + __be64 params; + __be32 rsvd2; + __be32 opt3; +}; + +struct cpl_act_open_rpl { + union opcode_tid ot; + __be32 atid_status; +}; + +/* cpl_act_open_rpl.atid_status fields */ +#define AOPEN_STATUS_S 0 +#define AOPEN_STATUS_M 0xFF +#define AOPEN_STATUS_G(x) (((x) >> AOPEN_STATUS_S) & AOPEN_STATUS_M) + +#define AOPEN_ATID_S 8 +#define AOPEN_ATID_M 0xFFFFFF +#define AOPEN_ATID_G(x) (((x) >> AOPEN_ATID_S) & AOPEN_ATID_M) + +struct cpl_pass_establish { + union opcode_tid ot; + __be32 rsvd; + __be32 tos_stid; + __be16 mac_idx; + __be16 tcp_opt; + __be32 snd_isn; + __be32 rcv_isn; +}; + +/* cpl_pass_establish.tos_stid fields */ +#define PASS_OPEN_TID_S 0 +#define PASS_OPEN_TID_M 0xFFFFFF +#define PASS_OPEN_TID_V(x) ((x) << PASS_OPEN_TID_S) +#define PASS_OPEN_TID_G(x) (((x) >> PASS_OPEN_TID_S) & PASS_OPEN_TID_M) + +#define PASS_OPEN_TOS_S 24 +#define PASS_OPEN_TOS_M 0xFF +#define PASS_OPEN_TOS_V(x) ((x) << PASS_OPEN_TOS_S) +#define PASS_OPEN_TOS_G(x) (((x) >> PASS_OPEN_TOS_S) & PASS_OPEN_TOS_M) + +/* cpl_pass_establish.tcp_opt fields (also applies to act_open_establish) */ +#define TCPOPT_WSCALE_OK_S 5 +#define TCPOPT_WSCALE_OK_M 0x1 +#define TCPOPT_WSCALE_OK_G(x) \ + (((x) >> TCPOPT_WSCALE_OK_S) & TCPOPT_WSCALE_OK_M) + +#define TCPOPT_SACK_S 6 +#define TCPOPT_SACK_M 0x1 +#define TCPOPT_SACK_G(x) (((x) >> TCPOPT_SACK_S) & TCPOPT_SACK_M) + +#define TCPOPT_TSTAMP_S 7 +#define TCPOPT_TSTAMP_M 0x1 +#define TCPOPT_TSTAMP_G(x) (((x) >> TCPOPT_TSTAMP_S) & TCPOPT_TSTAMP_M) + +#define TCPOPT_SND_WSCALE_S 8 +#define TCPOPT_SND_WSCALE_M 0xF +#define TCPOPT_SND_WSCALE_G(x) \ + (((x) >> TCPOPT_SND_WSCALE_S) & TCPOPT_SND_WSCALE_M) + +#define TCPOPT_MSS_S 12 +#define TCPOPT_MSS_M 0xF +#define TCPOPT_MSS_G(x) (((x) >> TCPOPT_MSS_S) & TCPOPT_MSS_M) + +#define T6_TCP_HDR_LEN_S 8 +#define T6_TCP_HDR_LEN_V(x) ((x) << T6_TCP_HDR_LEN_S) +#define T6_TCP_HDR_LEN_G(x) (((x) >> T6_TCP_HDR_LEN_S) & TCP_HDR_LEN_M) + +#define T6_IP_HDR_LEN_S 14 +#define T6_IP_HDR_LEN_V(x) ((x) << T6_IP_HDR_LEN_S) +#define T6_IP_HDR_LEN_G(x) (((x) >> T6_IP_HDR_LEN_S) & IP_HDR_LEN_M) + +#define T6_ETH_HDR_LEN_S 24 +#define T6_ETH_HDR_LEN_M 0xFF +#define T6_ETH_HDR_LEN_V(x) ((x) << T6_ETH_HDR_LEN_S) +#define T6_ETH_HDR_LEN_G(x) (((x) >> T6_ETH_HDR_LEN_S) & T6_ETH_HDR_LEN_M) + +struct cpl_act_establish { + union opcode_tid ot; + __be32 rsvd; + __be32 tos_atid; + __be16 mac_idx; + __be16 tcp_opt; + __be32 snd_isn; + __be32 rcv_isn; +}; + +struct cpl_get_tcb { + WR_HDR; + union opcode_tid ot; + __be16 reply_ctrl; + __be16 cookie; +}; + +/* cpl_get_tcb.reply_ctrl fields */ +#define QUEUENO_S 0 +#define QUEUENO_V(x) ((x) << QUEUENO_S) + +#define REPLY_CHAN_S 14 +#define REPLY_CHAN_V(x) ((x) << REPLY_CHAN_S) +#define REPLY_CHAN_F REPLY_CHAN_V(1U) + +#define NO_REPLY_S 15 +#define NO_REPLY_V(x) ((x) << NO_REPLY_S) +#define NO_REPLY_F NO_REPLY_V(1U) + +struct cpl_get_tcb_rpl { + union opcode_tid ot; + __u8 cookie; + __u8 status; + __be16 len; +}; + +struct cpl_set_tcb_field { + WR_HDR; + union opcode_tid ot; + __be16 reply_ctrl; + __be16 word_cookie; + __be64 mask; + __be64 val; +}; + +struct cpl_set_tcb_field_core { + union opcode_tid ot; + __be16 reply_ctrl; + __be16 word_cookie; + __be64 mask; + __be64 val; +}; + +/* cpl_set_tcb_field.word_cookie fields */ +#define TCB_WORD_S 0 +#define TCB_WORD_V(x) ((x) << TCB_WORD_S) + +#define TCB_COOKIE_S 5 +#define TCB_COOKIE_M 0x7 +#define TCB_COOKIE_V(x) ((x) << TCB_COOKIE_S) +#define TCB_COOKIE_G(x) (((x) >> TCB_COOKIE_S) & TCB_COOKIE_M) + +struct cpl_set_tcb_rpl { + union opcode_tid ot; + __be16 rsvd; + u8 cookie; + u8 status; + __be64 oldval; +}; + +struct cpl_close_con_req { + WR_HDR; + union opcode_tid ot; + __be32 rsvd; +}; + +struct cpl_close_con_rpl { + union opcode_tid ot; + u8 rsvd[3]; + u8 status; + __be32 snd_nxt; + __be32 rcv_nxt; +}; + +struct cpl_close_listsvr_req { + WR_HDR; + union opcode_tid ot; + __be16 reply_ctrl; + __be16 rsvd; +}; + +/* additional cpl_close_listsvr_req.reply_ctrl field */ +#define LISTSVR_IPV6_S 14 +#define LISTSVR_IPV6_V(x) ((x) << LISTSVR_IPV6_S) +#define LISTSVR_IPV6_F LISTSVR_IPV6_V(1U) + +struct cpl_close_listsvr_rpl { + union opcode_tid ot; + u8 rsvd[3]; + u8 status; +}; + +struct cpl_abort_req_rss { + union opcode_tid ot; + u8 rsvd[3]; + u8 status; +}; + +struct cpl_abort_req_rss6 { + union opcode_tid ot; + __be32 srqidx_status; +}; + +#define ABORT_RSS_STATUS_S 0 +#define ABORT_RSS_STATUS_M 0xff +#define ABORT_RSS_STATUS_V(x) ((x) << ABORT_RSS_STATUS_S) +#define ABORT_RSS_STATUS_G(x) (((x) >> ABORT_RSS_STATUS_S) & ABORT_RSS_STATUS_M) + +#define ABORT_RSS_SRQIDX_S 8 +#define ABORT_RSS_SRQIDX_M 0xffffff +#define ABORT_RSS_SRQIDX_V(x) ((x) << ABORT_RSS_SRQIDX_S) +#define ABORT_RSS_SRQIDX_G(x) (((x) >> ABORT_RSS_SRQIDX_S) & ABORT_RSS_SRQIDX_M) + +struct cpl_abort_req { + WR_HDR; + union opcode_tid ot; + __be32 rsvd0; + u8 rsvd1; + u8 cmd; + u8 rsvd2[6]; +}; + +struct cpl_abort_rpl_rss { + union opcode_tid ot; + u8 rsvd[3]; + u8 status; +}; + +struct cpl_abort_rpl_rss6 { + union opcode_tid ot; + __be32 srqidx_status; +}; + +struct cpl_abort_rpl { + WR_HDR; + union opcode_tid ot; + __be32 rsvd0; + u8 rsvd1; + u8 cmd; + u8 rsvd2[6]; +}; + +struct cpl_peer_close { + union opcode_tid ot; + __be32 rcv_nxt; +}; + +struct cpl_tid_release { + WR_HDR; + union opcode_tid ot; + __be32 rsvd; +}; + +struct cpl_tx_pkt_core { + __be32 ctrl0; + __be16 pack; + __be16 len; + __be64 ctrl1; +}; + +struct cpl_tx_pkt { + WR_HDR; + struct cpl_tx_pkt_core c; +}; + +#define cpl_tx_pkt_xt cpl_tx_pkt + +/* cpl_tx_pkt_core.ctrl0 fields */ +#define TXPKT_VF_S 0 +#define TXPKT_VF_V(x) ((x) << TXPKT_VF_S) + +#define TXPKT_PF_S 8 +#define TXPKT_PF_V(x) ((x) << TXPKT_PF_S) + +#define TXPKT_VF_VLD_S 11 +#define TXPKT_VF_VLD_V(x) ((x) << TXPKT_VF_VLD_S) +#define TXPKT_VF_VLD_F TXPKT_VF_VLD_V(1U) + +#define TXPKT_OVLAN_IDX_S 12 +#define TXPKT_OVLAN_IDX_V(x) ((x) << TXPKT_OVLAN_IDX_S) + +#define TXPKT_T5_OVLAN_IDX_S 12 +#define TXPKT_T5_OVLAN_IDX_V(x) ((x) << TXPKT_T5_OVLAN_IDX_S) + +#define TXPKT_INTF_S 16 +#define TXPKT_INTF_V(x) ((x) << TXPKT_INTF_S) + +#define TXPKT_INS_OVLAN_S 21 +#define TXPKT_INS_OVLAN_V(x) ((x) << TXPKT_INS_OVLAN_S) +#define TXPKT_INS_OVLAN_F TXPKT_INS_OVLAN_V(1U) + +#define TXPKT_TSTAMP_S 23 +#define TXPKT_TSTAMP_V(x) ((x) << TXPKT_TSTAMP_S) +#define TXPKT_TSTAMP_F TXPKT_TSTAMP_V(1ULL) + +#define TXPKT_OPCODE_S 24 +#define TXPKT_OPCODE_V(x) ((x) << TXPKT_OPCODE_S) + +/* cpl_tx_pkt_core.ctrl1 fields */ +#define TXPKT_CSUM_END_S 12 +#define TXPKT_CSUM_END_V(x) ((x) << TXPKT_CSUM_END_S) + +#define TXPKT_CSUM_START_S 20 +#define TXPKT_CSUM_START_V(x) ((x) << TXPKT_CSUM_START_S) + +#define TXPKT_IPHDR_LEN_S 20 +#define TXPKT_IPHDR_LEN_V(x) ((__u64)(x) << TXPKT_IPHDR_LEN_S) + +#define TXPKT_CSUM_LOC_S 30 +#define TXPKT_CSUM_LOC_V(x) ((__u64)(x) << TXPKT_CSUM_LOC_S) + +#define TXPKT_ETHHDR_LEN_S 34 +#define TXPKT_ETHHDR_LEN_V(x) ((__u64)(x) << TXPKT_ETHHDR_LEN_S) + +#define T6_TXPKT_ETHHDR_LEN_S 32 +#define T6_TXPKT_ETHHDR_LEN_V(x) ((__u64)(x) << T6_TXPKT_ETHHDR_LEN_S) + +#define TXPKT_CSUM_TYPE_S 40 +#define TXPKT_CSUM_TYPE_V(x) ((__u64)(x) << TXPKT_CSUM_TYPE_S) + +#define TXPKT_VLAN_S 44 +#define TXPKT_VLAN_V(x) ((__u64)(x) << TXPKT_VLAN_S) + +#define TXPKT_VLAN_VLD_S 60 +#define TXPKT_VLAN_VLD_V(x) ((__u64)(x) << TXPKT_VLAN_VLD_S) +#define TXPKT_VLAN_VLD_F TXPKT_VLAN_VLD_V(1ULL) + +#define TXPKT_IPCSUM_DIS_S 62 +#define TXPKT_IPCSUM_DIS_V(x) ((__u64)(x) << TXPKT_IPCSUM_DIS_S) +#define TXPKT_IPCSUM_DIS_F TXPKT_IPCSUM_DIS_V(1ULL) + +#define TXPKT_L4CSUM_DIS_S 63 +#define TXPKT_L4CSUM_DIS_V(x) ((__u64)(x) << TXPKT_L4CSUM_DIS_S) +#define TXPKT_L4CSUM_DIS_F TXPKT_L4CSUM_DIS_V(1ULL) + +struct cpl_tx_pkt_lso_core { + __be32 lso_ctrl; + __be16 ipid_ofst; + __be16 mss; + __be32 seqno_offset; + __be32 len; + /* encapsulated CPL (TX_PKT, TX_PKT_XT or TX_DATA) follows here */ +}; + +/* cpl_tx_pkt_lso_core.lso_ctrl fields */ +#define LSO_TCPHDR_LEN_S 0 +#define LSO_TCPHDR_LEN_V(x) ((x) << LSO_TCPHDR_LEN_S) + +#define LSO_IPHDR_LEN_S 4 +#define LSO_IPHDR_LEN_V(x) ((x) << LSO_IPHDR_LEN_S) + +#define LSO_ETHHDR_LEN_S 16 +#define LSO_ETHHDR_LEN_V(x) ((x) << LSO_ETHHDR_LEN_S) + +#define LSO_IPV6_S 20 +#define LSO_IPV6_V(x) ((x) << LSO_IPV6_S) +#define LSO_IPV6_F LSO_IPV6_V(1U) + +#define LSO_LAST_SLICE_S 22 +#define LSO_LAST_SLICE_V(x) ((x) << LSO_LAST_SLICE_S) +#define LSO_LAST_SLICE_F LSO_LAST_SLICE_V(1U) + +#define LSO_FIRST_SLICE_S 23 +#define LSO_FIRST_SLICE_V(x) ((x) << LSO_FIRST_SLICE_S) +#define LSO_FIRST_SLICE_F LSO_FIRST_SLICE_V(1U) + +#define LSO_OPCODE_S 24 +#define LSO_OPCODE_V(x) ((x) << LSO_OPCODE_S) + +#define LSO_T5_XFER_SIZE_S 0 +#define LSO_T5_XFER_SIZE_V(x) ((x) << LSO_T5_XFER_SIZE_S) + +struct cpl_tx_pkt_lso { + WR_HDR; + struct cpl_tx_pkt_lso_core c; + /* encapsulated CPL (TX_PKT, TX_PKT_XT or TX_DATA) follows here */ +}; + +struct cpl_iscsi_hdr { + union opcode_tid ot; + __be16 pdu_len_ddp; + __be16 len; + __be32 seq; + __be16 urg; + u8 rsvd; + u8 status; +}; + +/* cpl_iscsi_hdr.pdu_len_ddp fields */ +#define ISCSI_PDU_LEN_S 0 +#define ISCSI_PDU_LEN_M 0x7FFF +#define ISCSI_PDU_LEN_V(x) ((x) << ISCSI_PDU_LEN_S) +#define ISCSI_PDU_LEN_G(x) (((x) >> ISCSI_PDU_LEN_S) & ISCSI_PDU_LEN_M) + +#define ISCSI_DDP_S 15 +#define ISCSI_DDP_V(x) ((x) << ISCSI_DDP_S) +#define ISCSI_DDP_F ISCSI_DDP_V(1U) + +struct cpl_rx_data_ddp { + union opcode_tid ot; + __be16 urg; + __be16 len; + __be32 seq; + union { + __be32 nxt_seq; + __be32 ddp_report; + }; + __be32 ulp_crc; + __be32 ddpvld; +}; + +#define cpl_rx_iscsi_ddp cpl_rx_data_ddp + +struct cpl_iscsi_data { + union opcode_tid ot; + __u8 rsvd0[2]; + __be16 len; + __be32 seq; + __be16 urg; + __u8 rsvd1; + __u8 status; +}; + +struct cpl_rx_iscsi_cmp { + union opcode_tid ot; + __be16 pdu_len_ddp; + __be16 len; + __be32 seq; + __be16 urg; + __u8 rsvd; + __u8 status; + __be32 ulp_crc; + __be32 ddpvld; +}; + +struct cpl_tx_data_iso { + __be32 op_to_scsi; + __u8 reserved1; + __u8 ahs_len; + __be16 mpdu; + __be32 burst_size; + __be32 len; + __be32 reserved2_seglen_offset; + __be32 datasn_offset; + __be32 buffer_offset; + __be32 reserved3; + + /* encapsulated CPL_TX_DATA follows here */ +}; + +/* cpl_tx_data_iso.op_to_scsi fields */ +#define CPL_TX_DATA_ISO_OP_S 24 +#define CPL_TX_DATA_ISO_OP_M 0xff +#define CPL_TX_DATA_ISO_OP_V(x) ((x) << CPL_TX_DATA_ISO_OP_S) +#define CPL_TX_DATA_ISO_OP_G(x) \ + (((x) >> CPL_TX_DATA_ISO_OP_S) & CPL_TX_DATA_ISO_OP_M) + +#define CPL_TX_DATA_ISO_FIRST_S 23 +#define CPL_TX_DATA_ISO_FIRST_M 0x1 +#define CPL_TX_DATA_ISO_FIRST_V(x) ((x) << CPL_TX_DATA_ISO_FIRST_S) +#define CPL_TX_DATA_ISO_FIRST_G(x) \ + (((x) >> CPL_TX_DATA_ISO_FIRST_S) & CPL_TX_DATA_ISO_FIRST_M) +#define CPL_TX_DATA_ISO_FIRST_F CPL_TX_DATA_ISO_FIRST_V(1U) + +#define CPL_TX_DATA_ISO_LAST_S 22 +#define CPL_TX_DATA_ISO_LAST_M 0x1 +#define CPL_TX_DATA_ISO_LAST_V(x) ((x) << CPL_TX_DATA_ISO_LAST_S) +#define CPL_TX_DATA_ISO_LAST_G(x) \ + (((x) >> CPL_TX_DATA_ISO_LAST_S) & CPL_TX_DATA_ISO_LAST_M) +#define CPL_TX_DATA_ISO_LAST_F CPL_TX_DATA_ISO_LAST_V(1U) + +#define CPL_TX_DATA_ISO_CPLHDRLEN_S 21 +#define CPL_TX_DATA_ISO_CPLHDRLEN_M 0x1 +#define CPL_TX_DATA_ISO_CPLHDRLEN_V(x) ((x) << CPL_TX_DATA_ISO_CPLHDRLEN_S) +#define CPL_TX_DATA_ISO_CPLHDRLEN_G(x) \ + (((x) >> CPL_TX_DATA_ISO_CPLHDRLEN_S) & CPL_TX_DATA_ISO_CPLHDRLEN_M) +#define CPL_TX_DATA_ISO_CPLHDRLEN_F CPL_TX_DATA_ISO_CPLHDRLEN_V(1U) + +#define CPL_TX_DATA_ISO_HDRCRC_S 20 +#define CPL_TX_DATA_ISO_HDRCRC_M 0x1 +#define CPL_TX_DATA_ISO_HDRCRC_V(x) ((x) << CPL_TX_DATA_ISO_HDRCRC_S) +#define CPL_TX_DATA_ISO_HDRCRC_G(x) \ + (((x) >> CPL_TX_DATA_ISO_HDRCRC_S) & CPL_TX_DATA_ISO_HDRCRC_M) +#define CPL_TX_DATA_ISO_HDRCRC_F CPL_TX_DATA_ISO_HDRCRC_V(1U) + +#define CPL_TX_DATA_ISO_PLDCRC_S 19 +#define CPL_TX_DATA_ISO_PLDCRC_M 0x1 +#define CPL_TX_DATA_ISO_PLDCRC_V(x) ((x) << CPL_TX_DATA_ISO_PLDCRC_S) +#define CPL_TX_DATA_ISO_PLDCRC_G(x) \ + (((x) >> CPL_TX_DATA_ISO_PLDCRC_S) & CPL_TX_DATA_ISO_PLDCRC_M) +#define CPL_TX_DATA_ISO_PLDCRC_F CPL_TX_DATA_ISO_PLDCRC_V(1U) + +#define CPL_TX_DATA_ISO_IMMEDIATE_S 18 +#define CPL_TX_DATA_ISO_IMMEDIATE_M 0x1 +#define CPL_TX_DATA_ISO_IMMEDIATE_V(x) ((x) << CPL_TX_DATA_ISO_IMMEDIATE_S) +#define CPL_TX_DATA_ISO_IMMEDIATE_G(x) \ + (((x) >> CPL_TX_DATA_ISO_IMMEDIATE_S) & CPL_TX_DATA_ISO_IMMEDIATE_M) +#define CPL_TX_DATA_ISO_IMMEDIATE_F CPL_TX_DATA_ISO_IMMEDIATE_V(1U) + +#define CPL_TX_DATA_ISO_SCSI_S 16 +#define CPL_TX_DATA_ISO_SCSI_M 0x3 +#define CPL_TX_DATA_ISO_SCSI_V(x) ((x) << CPL_TX_DATA_ISO_SCSI_S) +#define CPL_TX_DATA_ISO_SCSI_G(x) \ + (((x) >> CPL_TX_DATA_ISO_SCSI_S) & CPL_TX_DATA_ISO_SCSI_M) + +/* cpl_tx_data_iso.reserved2_seglen_offset fields */ +#define CPL_TX_DATA_ISO_SEGLEN_OFFSET_S 0 +#define CPL_TX_DATA_ISO_SEGLEN_OFFSET_M 0xffffff +#define CPL_TX_DATA_ISO_SEGLEN_OFFSET_V(x) \ + ((x) << CPL_TX_DATA_ISO_SEGLEN_OFFSET_S) +#define CPL_TX_DATA_ISO_SEGLEN_OFFSET_G(x) \ + (((x) >> CPL_TX_DATA_ISO_SEGLEN_OFFSET_S) & \ + CPL_TX_DATA_ISO_SEGLEN_OFFSET_M) + +struct cpl_rx_data { + union opcode_tid ot; + __be16 rsvd; + __be16 len; + __be32 seq; + __be16 urg; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 dack_mode:2; + u8 psh:1; + u8 heartbeat:1; + u8 ddp_off:1; + u8 :3; +#else + u8 :3; + u8 ddp_off:1; + u8 heartbeat:1; + u8 psh:1; + u8 dack_mode:2; +#endif + u8 status; +}; + +struct cpl_rx_data_ack { + WR_HDR; + union opcode_tid ot; + __be32 credit_dack; +}; + +/* cpl_rx_data_ack.ack_seq fields */ +#define RX_CREDITS_S 0 +#define RX_CREDITS_V(x) ((x) << RX_CREDITS_S) + +#define RX_FORCE_ACK_S 28 +#define RX_FORCE_ACK_V(x) ((x) << RX_FORCE_ACK_S) +#define RX_FORCE_ACK_F RX_FORCE_ACK_V(1U) + +#define RX_DACK_MODE_S 29 +#define RX_DACK_MODE_M 0x3 +#define RX_DACK_MODE_V(x) ((x) << RX_DACK_MODE_S) +#define RX_DACK_MODE_G(x) (((x) >> RX_DACK_MODE_S) & RX_DACK_MODE_M) + +#define RX_DACK_CHANGE_S 31 +#define RX_DACK_CHANGE_V(x) ((x) << RX_DACK_CHANGE_S) +#define RX_DACK_CHANGE_F RX_DACK_CHANGE_V(1U) + +struct cpl_rx_pkt { + struct rss_header rsshdr; + u8 opcode; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 iff:4; + u8 csum_calc:1; + u8 ipmi_pkt:1; + u8 vlan_ex:1; + u8 ip_frag:1; +#else + u8 ip_frag:1; + u8 vlan_ex:1; + u8 ipmi_pkt:1; + u8 csum_calc:1; + u8 iff:4; +#endif + __be16 csum; + __be16 vlan; + __be16 len; + __be32 l2info; + __be16 hdr_len; + __be16 err_vec; +}; + +#define RX_T6_ETHHDR_LEN_M 0xFF +#define RX_T6_ETHHDR_LEN_G(x) (((x) >> RX_ETHHDR_LEN_S) & RX_T6_ETHHDR_LEN_M) + +#define RXF_PSH_S 20 +#define RXF_PSH_V(x) ((x) << RXF_PSH_S) +#define RXF_PSH_F RXF_PSH_V(1U) + +#define RXF_SYN_S 21 +#define RXF_SYN_V(x) ((x) << RXF_SYN_S) +#define RXF_SYN_F RXF_SYN_V(1U) + +#define RXF_UDP_S 22 +#define RXF_UDP_V(x) ((x) << RXF_UDP_S) +#define RXF_UDP_F RXF_UDP_V(1U) + +#define RXF_TCP_S 23 +#define RXF_TCP_V(x) ((x) << RXF_TCP_S) +#define RXF_TCP_F RXF_TCP_V(1U) + +#define RXF_IP_S 24 +#define RXF_IP_V(x) ((x) << RXF_IP_S) +#define RXF_IP_F RXF_IP_V(1U) + +#define RXF_IP6_S 25 +#define RXF_IP6_V(x) ((x) << RXF_IP6_S) +#define RXF_IP6_F RXF_IP6_V(1U) + +#define RXF_SYN_COOKIE_S 26 +#define RXF_SYN_COOKIE_V(x) ((x) << RXF_SYN_COOKIE_S) +#define RXF_SYN_COOKIE_F RXF_SYN_COOKIE_V(1U) + +#define RXF_FCOE_S 26 +#define RXF_FCOE_V(x) ((x) << RXF_FCOE_S) +#define RXF_FCOE_F RXF_FCOE_V(1U) + +#define RXF_LRO_S 27 +#define RXF_LRO_V(x) ((x) << RXF_LRO_S) +#define RXF_LRO_F RXF_LRO_V(1U) + +/* rx_pkt.l2info fields */ +#define RX_ETHHDR_LEN_S 0 +#define RX_ETHHDR_LEN_M 0x1F +#define RX_ETHHDR_LEN_V(x) ((x) << RX_ETHHDR_LEN_S) +#define RX_ETHHDR_LEN_G(x) (((x) >> RX_ETHHDR_LEN_S) & RX_ETHHDR_LEN_M) + +#define RX_T5_ETHHDR_LEN_S 0 +#define RX_T5_ETHHDR_LEN_M 0x3F +#define RX_T5_ETHHDR_LEN_V(x) ((x) << RX_T5_ETHHDR_LEN_S) +#define RX_T5_ETHHDR_LEN_G(x) (((x) >> RX_T5_ETHHDR_LEN_S) & RX_T5_ETHHDR_LEN_M) + +#define RX_MACIDX_S 8 +#define RX_MACIDX_M 0x1FF +#define RX_MACIDX_V(x) ((x) << RX_MACIDX_S) +#define RX_MACIDX_G(x) (((x) >> RX_MACIDX_S) & RX_MACIDX_M) + +#define RXF_SYN_S 21 +#define RXF_SYN_V(x) ((x) << RXF_SYN_S) +#define RXF_SYN_F RXF_SYN_V(1U) + +#define RX_CHAN_S 28 +#define RX_CHAN_M 0xF +#define RX_CHAN_V(x) ((x) << RX_CHAN_S) +#define RX_CHAN_G(x) (((x) >> RX_CHAN_S) & RX_CHAN_M) + +/* rx_pkt.hdr_len fields */ +#define RX_TCPHDR_LEN_S 0 +#define RX_TCPHDR_LEN_M 0x3F +#define RX_TCPHDR_LEN_V(x) ((x) << RX_TCPHDR_LEN_S) +#define RX_TCPHDR_LEN_G(x) (((x) >> RX_TCPHDR_LEN_S) & RX_TCPHDR_LEN_M) + +#define RX_IPHDR_LEN_S 6 +#define RX_IPHDR_LEN_M 0x3FF +#define RX_IPHDR_LEN_V(x) ((x) << RX_IPHDR_LEN_S) +#define RX_IPHDR_LEN_G(x) (((x) >> RX_IPHDR_LEN_S) & RX_IPHDR_LEN_M) + +/* rx_pkt.err_vec fields */ +#define RXERR_CSUM_S 13 +#define RXERR_CSUM_V(x) ((x) << RXERR_CSUM_S) +#define RXERR_CSUM_F RXERR_CSUM_V(1U) + +#define T6_COMPR_RXERR_LEN_S 1 +#define T6_COMPR_RXERR_LEN_V(x) ((x) << T6_COMPR_RXERR_LEN_S) +#define T6_COMPR_RXERR_LEN_F T6_COMPR_RXERR_LEN_V(1U) + +#define T6_COMPR_RXERR_VEC_S 0 +#define T6_COMPR_RXERR_VEC_M 0x3F +#define T6_COMPR_RXERR_VEC_V(x) ((x) << T6_COMPR_RXERR_LEN_S) +#define T6_COMPR_RXERR_VEC_G(x) \ + (((x) >> T6_COMPR_RXERR_VEC_S) & T6_COMPR_RXERR_VEC_M) + +/* Logical OR of RX_ERROR_CSUM, RX_ERROR_CSIP */ +#define T6_COMPR_RXERR_SUM_S 4 +#define T6_COMPR_RXERR_SUM_V(x) ((x) << T6_COMPR_RXERR_SUM_S) +#define T6_COMPR_RXERR_SUM_F T6_COMPR_RXERR_SUM_V(1U) + +#define T6_RX_TNLHDR_LEN_S 8 +#define T6_RX_TNLHDR_LEN_M 0xFF +#define T6_RX_TNLHDR_LEN_V(x) ((x) << T6_RX_TNLHDR_LEN_S) +#define T6_RX_TNLHDR_LEN_G(x) (((x) >> T6_RX_TNLHDR_LEN_S) & T6_RX_TNLHDR_LEN_M) + +struct cpl_trace_pkt { + u8 opcode; + u8 intf; +#if defined(__LITTLE_ENDIAN_BITFIELD) + u8 runt:4; + u8 filter_hit:4; + u8 :6; + u8 err:1; + u8 trunc:1; +#else + u8 filter_hit:4; + u8 runt:4; + u8 trunc:1; + u8 err:1; + u8 :6; +#endif + __be16 rsvd; + __be16 len; + __be64 tstamp; +}; + +struct cpl_t5_trace_pkt { + __u8 opcode; + __u8 intf; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 runt:4; + __u8 filter_hit:4; + __u8:6; + __u8 err:1; + __u8 trunc:1; +#else + __u8 filter_hit:4; + __u8 runt:4; + __u8 trunc:1; + __u8 err:1; + __u8:6; +#endif + __be16 rsvd; + __be16 len; + __be64 tstamp; + __be64 rsvd1; +}; + +struct cpl_l2t_write_req { + WR_HDR; + union opcode_tid ot; + __be16 params; + __be16 l2t_idx; + __be16 vlan; + u8 dst_mac[6]; +}; + +/* cpl_l2t_write_req.params fields */ +#define L2T_W_INFO_S 2 +#define L2T_W_INFO_V(x) ((x) << L2T_W_INFO_S) + +#define L2T_W_PORT_S 8 +#define L2T_W_PORT_V(x) ((x) << L2T_W_PORT_S) + +#define L2T_W_NOREPLY_S 15 +#define L2T_W_NOREPLY_V(x) ((x) << L2T_W_NOREPLY_S) +#define L2T_W_NOREPLY_F L2T_W_NOREPLY_V(1U) + +#define CPL_L2T_VLAN_NONE 0xfff + +struct cpl_l2t_write_rpl { + union opcode_tid ot; + u8 status; + u8 rsvd[3]; +}; + +struct cpl_smt_write_req { + WR_HDR; + union opcode_tid ot; + __be32 params; + __be16 pfvf1; + u8 src_mac1[6]; + __be16 pfvf0; + u8 src_mac0[6]; +}; + +struct cpl_t6_smt_write_req { + WR_HDR; + union opcode_tid ot; + __be32 params; + __be64 tag; + __be16 pfvf0; + u8 src_mac0[6]; + __be32 local_ip; + __be32 rsvd; +}; + +struct cpl_smt_write_rpl { + union opcode_tid ot; + u8 status; + u8 rsvd[3]; +}; + +/* cpl_smt_{read,write}_req.params fields */ +#define SMTW_OVLAN_IDX_S 16 +#define SMTW_OVLAN_IDX_V(x) ((x) << SMTW_OVLAN_IDX_S) + +#define SMTW_IDX_S 20 +#define SMTW_IDX_V(x) ((x) << SMTW_IDX_S) + +#define SMTW_NORPL_S 31 +#define SMTW_NORPL_V(x) ((x) << SMTW_NORPL_S) +#define SMTW_NORPL_F SMTW_NORPL_V(1U) + +struct cpl_rdma_terminate { + union opcode_tid ot; + __be16 rsvd; + __be16 len; +}; + +struct cpl_sge_egr_update { + __be32 opcode_qid; + __be16 cidx; + __be16 pidx; +}; + +/* cpl_sge_egr_update.ot fields */ +#define EGR_QID_S 0 +#define EGR_QID_M 0x1FFFF +#define EGR_QID_G(x) (((x) >> EGR_QID_S) & EGR_QID_M) + +/* cpl_fw*.type values */ +enum { + FW_TYPE_CMD_RPL = 0, + FW_TYPE_WR_RPL = 1, + FW_TYPE_CQE = 2, + FW_TYPE_OFLD_CONNECTION_WR_RPL = 3, + FW_TYPE_RSSCPL = 4, +}; + +struct cpl_fw4_pld { + u8 opcode; + u8 rsvd0[3]; + u8 type; + u8 rsvd1; + __be16 len; + __be64 data; + __be64 rsvd2; +}; + +struct cpl_fw6_pld { + u8 opcode; + u8 rsvd[5]; + __be16 len; + __be64 data[4]; +}; + +struct cpl_fw4_msg { + u8 opcode; + u8 type; + __be16 rsvd0; + __be32 rsvd1; + __be64 data[2]; +}; + +struct cpl_fw4_ack { + union opcode_tid ot; + u8 credits; + u8 rsvd0[2]; + u8 seq_vld; + __be32 snd_nxt; + __be32 snd_una; + __be64 rsvd1; +}; + +enum { + CPL_FW4_ACK_FLAGS_SEQVAL = 0x1, /* seqn valid */ + CPL_FW4_ACK_FLAGS_CH = 0x2, /* channel change complete */ + CPL_FW4_ACK_FLAGS_FLOWC = 0x4, /* fw_flowc_wr complete */ +}; + +#define CPL_FW4_ACK_FLOWID_S 0 +#define CPL_FW4_ACK_FLOWID_M 0xffffff +#define CPL_FW4_ACK_FLOWID_G(x) \ + (((x) >> CPL_FW4_ACK_FLOWID_S) & CPL_FW4_ACK_FLOWID_M) + +struct cpl_fw6_msg { + u8 opcode; + u8 type; + __be16 rsvd0; + __be32 rsvd1; + __be64 data[4]; +}; + +/* cpl_fw6_msg.type values */ +enum { + FW6_TYPE_CMD_RPL = 0, + FW6_TYPE_WR_RPL = 1, + FW6_TYPE_CQE = 2, + FW6_TYPE_OFLD_CONNECTION_WR_RPL = 3, + FW6_TYPE_RSSCPL = FW_TYPE_RSSCPL, +}; + +struct cpl_fw6_msg_ofld_connection_wr_rpl { + __u64 cookie; + __be32 tid; /* or atid in case of active failure */ + __u8 t_state; + __u8 retval; + __u8 rsvd[2]; +}; + +struct cpl_tx_data { + union opcode_tid ot; + __be32 len; + __be32 rsvd; + __be32 flags; +}; + +/* cpl_tx_data.flags field */ +#define TX_FORCE_S 13 +#define TX_FORCE_V(x) ((x) << TX_FORCE_S) + +#define TX_DATA_MSS_S 16 +#define TX_DATA_MSS_M 0xFFFF +#define TX_DATA_MSS_V(x) ((x) << TX_DATA_MSS_S) +#define TX_DATA_MSS_G(x) (((x) >> TX_DATA_MSS_S) & TX_DATA_MSS_M) + +#define TX_LENGTH_S 0 +#define TX_LENGTH_M 0xFFFF +#define TX_LENGTH_V(x) ((x) << TX_LENGTH_S) +#define TX_LENGTH_G(x) (((x) >> TX_LENGTH_S) & TX_LENGTH_M) + +#define T6_TX_FORCE_S 20 +#define T6_TX_FORCE_V(x) ((x) << T6_TX_FORCE_S) +#define T6_TX_FORCE_F T6_TX_FORCE_V(1U) + +#define TX_URG_S 16 +#define TX_URG_V(x) ((x) << TX_URG_S) + +#define TX_SHOVE_S 14 +#define TX_SHOVE_V(x) ((x) << TX_SHOVE_S) +#define TX_SHOVE_F TX_SHOVE_V(1U) + +#define TX_BYPASS_S 21 +#define TX_BYPASS_V(x) ((x) << TX_BYPASS_S) +#define TX_BYPASS_F TX_BYPASS_V(1U) + +#define TX_PUSH_S 22 +#define TX_PUSH_V(x) ((x) << TX_PUSH_S) +#define TX_PUSH_F TX_PUSH_V(1U) + +#define TX_ULP_MODE_S 10 +#define TX_ULP_MODE_M 0x7 +#define TX_ULP_MODE_V(x) ((x) << TX_ULP_MODE_S) +#define TX_ULP_MODE_G(x) (((x) >> TX_ULP_MODE_S) & TX_ULP_MODE_M) + +enum { + ULP_TX_MEM_READ = 2, + ULP_TX_MEM_WRITE = 3, + ULP_TX_PKT = 4 +}; + +enum { + ULP_TX_SC_NOOP = 0x80, + ULP_TX_SC_IMM = 0x81, + ULP_TX_SC_DSGL = 0x82, + ULP_TX_SC_ISGL = 0x83, + ULP_TX_SC_MEMRD = 0x86 +}; + +#define ULPTX_CMD_S 24 +#define ULPTX_CMD_V(x) ((x) << ULPTX_CMD_S) + +#define ULPTX_LEN16_S 0 +#define ULPTX_LEN16_M 0xFF +#define ULPTX_LEN16_V(x) ((x) << ULPTX_LEN16_S) + +#define ULP_TX_SC_MORE_S 23 +#define ULP_TX_SC_MORE_V(x) ((x) << ULP_TX_SC_MORE_S) +#define ULP_TX_SC_MORE_F ULP_TX_SC_MORE_V(1U) + +struct ulptx_sge_pair { + __be32 len[2]; + __be64 addr[2]; +}; + +struct ulptx_sgl { + __be32 cmd_nsge; + __be32 len0; + __be64 addr0; + struct ulptx_sge_pair sge[]; +}; + +struct ulptx_idata { + __be32 cmd_more; + __be32 len; +}; + +struct ulp_txpkt { + __be32 cmd_dest; + __be32 len; +}; + +#define ULPTX_CMD_S 24 +#define ULPTX_CMD_M 0xFF +#define ULPTX_CMD_V(x) ((x) << ULPTX_CMD_S) + +#define ULPTX_NSGE_S 0 +#define ULPTX_NSGE_V(x) ((x) << ULPTX_NSGE_S) + +#define ULPTX_MORE_S 23 +#define ULPTX_MORE_V(x) ((x) << ULPTX_MORE_S) +#define ULPTX_MORE_F ULPTX_MORE_V(1U) + +#define ULP_TXPKT_DEST_S 16 +#define ULP_TXPKT_DEST_M 0x3 +#define ULP_TXPKT_DEST_V(x) ((x) << ULP_TXPKT_DEST_S) + +#define ULP_TXPKT_FID_S 4 +#define ULP_TXPKT_FID_M 0x7ff +#define ULP_TXPKT_FID_V(x) ((x) << ULP_TXPKT_FID_S) + +#define ULP_TXPKT_RO_S 3 +#define ULP_TXPKT_RO_V(x) ((x) << ULP_TXPKT_RO_S) +#define ULP_TXPKT_RO_F ULP_TXPKT_RO_V(1U) + +enum cpl_tx_tnl_lso_type { + TX_TNL_TYPE_OPAQUE, + TX_TNL_TYPE_NVGRE, + TX_TNL_TYPE_VXLAN, + TX_TNL_TYPE_GENEVE, +}; + +struct cpl_tx_tnl_lso { + __be32 op_to_IpIdSplitOut; + __be16 IpIdOffsetOut; + __be16 UdpLenSetOut_to_TnlHdrLen; + __be64 r1; + __be32 Flow_to_TcpHdrLen; + __be16 IpIdOffset; + __be16 IpIdSplit_to_Mss; + __be32 TCPSeqOffset; + __be32 EthLenOffset_Size; + /* encapsulated CPL (TX_PKT_XT) follows here */ +}; + +#define CPL_TX_TNL_LSO_OPCODE_S 24 +#define CPL_TX_TNL_LSO_OPCODE_M 0xff +#define CPL_TX_TNL_LSO_OPCODE_V(x) ((x) << CPL_TX_TNL_LSO_OPCODE_S) +#define CPL_TX_TNL_LSO_OPCODE_G(x) \ + (((x) >> CPL_TX_TNL_LSO_OPCODE_S) & CPL_TX_TNL_LSO_OPCODE_M) + +#define CPL_TX_TNL_LSO_FIRST_S 23 +#define CPL_TX_TNL_LSO_FIRST_M 0x1 +#define CPL_TX_TNL_LSO_FIRST_V(x) ((x) << CPL_TX_TNL_LSO_FIRST_S) +#define CPL_TX_TNL_LSO_FIRST_G(x) \ + (((x) >> CPL_TX_TNL_LSO_FIRST_S) & CPL_TX_TNL_LSO_FIRST_M) +#define CPL_TX_TNL_LSO_FIRST_F CPL_TX_TNL_LSO_FIRST_V(1U) + +#define CPL_TX_TNL_LSO_LAST_S 22 +#define CPL_TX_TNL_LSO_LAST_M 0x1 +#define CPL_TX_TNL_LSO_LAST_V(x) ((x) << CPL_TX_TNL_LSO_LAST_S) +#define CPL_TX_TNL_LSO_LAST_G(x) \ + (((x) >> CPL_TX_TNL_LSO_LAST_S) & CPL_TX_TNL_LSO_LAST_M) +#define CPL_TX_TNL_LSO_LAST_F CPL_TX_TNL_LSO_LAST_V(1U) + +#define CPL_TX_TNL_LSO_ETHHDRLENXOUT_S 21 +#define CPL_TX_TNL_LSO_ETHHDRLENXOUT_M 0x1 +#define CPL_TX_TNL_LSO_ETHHDRLENXOUT_V(x) \ + ((x) << CPL_TX_TNL_LSO_ETHHDRLENXOUT_S) +#define CPL_TX_TNL_LSO_ETHHDRLENXOUT_G(x) \ + (((x) >> CPL_TX_TNL_LSO_ETHHDRLENXOUT_S) & \ + CPL_TX_TNL_LSO_ETHHDRLENXOUT_M) +#define CPL_TX_TNL_LSO_ETHHDRLENXOUT_F CPL_TX_TNL_LSO_ETHHDRLENXOUT_V(1U) + +#define CPL_TX_TNL_LSO_IPV6OUT_S 20 +#define CPL_TX_TNL_LSO_IPV6OUT_M 0x1 +#define CPL_TX_TNL_LSO_IPV6OUT_V(x) ((x) << CPL_TX_TNL_LSO_IPV6OUT_S) +#define CPL_TX_TNL_LSO_IPV6OUT_G(x) \ + (((x) >> CPL_TX_TNL_LSO_IPV6OUT_S) & CPL_TX_TNL_LSO_IPV6OUT_M) +#define CPL_TX_TNL_LSO_IPV6OUT_F CPL_TX_TNL_LSO_IPV6OUT_V(1U) + +#define CPL_TX_TNL_LSO_ETHHDRLEN_S 16 +#define CPL_TX_TNL_LSO_ETHHDRLEN_M 0xf +#define CPL_TX_TNL_LSO_ETHHDRLEN_V(x) ((x) << CPL_TX_TNL_LSO_ETHHDRLEN_S) +#define CPL_TX_TNL_LSO_ETHHDRLEN_G(x) \ + (((x) >> CPL_TX_TNL_LSO_ETHHDRLEN_S) & CPL_TX_TNL_LSO_ETHHDRLEN_M) + +#define CPL_TX_TNL_LSO_IPHDRLEN_S 4 +#define CPL_TX_TNL_LSO_IPHDRLEN_M 0xfff +#define CPL_TX_TNL_LSO_IPHDRLEN_V(x) ((x) << CPL_TX_TNL_LSO_IPHDRLEN_S) +#define CPL_TX_TNL_LSO_IPHDRLEN_G(x) \ + (((x) >> CPL_TX_TNL_LSO_IPHDRLEN_S) & CPL_TX_TNL_LSO_IPHDRLEN_M) + +#define CPL_TX_TNL_LSO_TCPHDRLEN_S 0 +#define CPL_TX_TNL_LSO_TCPHDRLEN_M 0xf +#define CPL_TX_TNL_LSO_TCPHDRLEN_V(x) ((x) << CPL_TX_TNL_LSO_TCPHDRLEN_S) +#define CPL_TX_TNL_LSO_TCPHDRLEN_G(x) \ + (((x) >> CPL_TX_TNL_LSO_TCPHDRLEN_S) & CPL_TX_TNL_LSO_TCPHDRLEN_M) + +#define CPL_TX_TNL_LSO_MSS_S 0 +#define CPL_TX_TNL_LSO_MSS_M 0x3fff +#define CPL_TX_TNL_LSO_MSS_V(x) ((x) << CPL_TX_TNL_LSO_MSS_S) +#define CPL_TX_TNL_LSO_MSS_G(x) \ + (((x) >> CPL_TX_TNL_LSO_MSS_S) & CPL_TX_TNL_LSO_MSS_M) + +#define CPL_TX_TNL_LSO_SIZE_S 0 +#define CPL_TX_TNL_LSO_SIZE_M 0xfffffff +#define CPL_TX_TNL_LSO_SIZE_V(x) ((x) << CPL_TX_TNL_LSO_SIZE_S) +#define CPL_TX_TNL_LSO_SIZE_G(x) \ + (((x) >> CPL_TX_TNL_LSO_SIZE_S) & CPL_TX_TNL_LSO_SIZE_M) + +#define CPL_TX_TNL_LSO_ETHHDRLENOUT_S 16 +#define CPL_TX_TNL_LSO_ETHHDRLENOUT_M 0xf +#define CPL_TX_TNL_LSO_ETHHDRLENOUT_V(x) \ + ((x) << CPL_TX_TNL_LSO_ETHHDRLENOUT_S) +#define CPL_TX_TNL_LSO_ETHHDRLENOUT_G(x) \ + (((x) >> CPL_TX_TNL_LSO_ETHHDRLENOUT_S) & CPL_TX_TNL_LSO_ETHHDRLENOUT_M) + +#define CPL_TX_TNL_LSO_IPHDRLENOUT_S 4 +#define CPL_TX_TNL_LSO_IPHDRLENOUT_M 0xfff +#define CPL_TX_TNL_LSO_IPHDRLENOUT_V(x) ((x) << CPL_TX_TNL_LSO_IPHDRLENOUT_S) +#define CPL_TX_TNL_LSO_IPHDRLENOUT_G(x) \ + (((x) >> CPL_TX_TNL_LSO_IPHDRLENOUT_S) & CPL_TX_TNL_LSO_IPHDRLENOUT_M) + +#define CPL_TX_TNL_LSO_IPHDRCHKOUT_S 3 +#define CPL_TX_TNL_LSO_IPHDRCHKOUT_M 0x1 +#define CPL_TX_TNL_LSO_IPHDRCHKOUT_V(x) ((x) << CPL_TX_TNL_LSO_IPHDRCHKOUT_S) +#define CPL_TX_TNL_LSO_IPHDRCHKOUT_G(x) \ + (((x) >> CPL_TX_TNL_LSO_IPHDRCHKOUT_S) & CPL_TX_TNL_LSO_IPHDRCHKOUT_M) +#define CPL_TX_TNL_LSO_IPHDRCHKOUT_F CPL_TX_TNL_LSO_IPHDRCHKOUT_V(1U) + +#define CPL_TX_TNL_LSO_IPLENSETOUT_S 2 +#define CPL_TX_TNL_LSO_IPLENSETOUT_M 0x1 +#define CPL_TX_TNL_LSO_IPLENSETOUT_V(x) ((x) << CPL_TX_TNL_LSO_IPLENSETOUT_S) +#define CPL_TX_TNL_LSO_IPLENSETOUT_G(x) \ + (((x) >> CPL_TX_TNL_LSO_IPLENSETOUT_S) & CPL_TX_TNL_LSO_IPLENSETOUT_M) +#define CPL_TX_TNL_LSO_IPLENSETOUT_F CPL_TX_TNL_LSO_IPLENSETOUT_V(1U) + +#define CPL_TX_TNL_LSO_IPIDINCOUT_S 1 +#define CPL_TX_TNL_LSO_IPIDINCOUT_M 0x1 +#define CPL_TX_TNL_LSO_IPIDINCOUT_V(x) ((x) << CPL_TX_TNL_LSO_IPIDINCOUT_S) +#define CPL_TX_TNL_LSO_IPIDINCOUT_G(x) \ + (((x) >> CPL_TX_TNL_LSO_IPIDINCOUT_S) & CPL_TX_TNL_LSO_IPIDINCOUT_M) +#define CPL_TX_TNL_LSO_IPIDINCOUT_F CPL_TX_TNL_LSO_IPIDINCOUT_V(1U) + +#define CPL_TX_TNL_LSO_UDPCHKCLROUT_S 14 +#define CPL_TX_TNL_LSO_UDPCHKCLROUT_M 0x1 +#define CPL_TX_TNL_LSO_UDPCHKCLROUT_V(x) \ + ((x) << CPL_TX_TNL_LSO_UDPCHKCLROUT_S) +#define CPL_TX_TNL_LSO_UDPCHKCLROUT_G(x) \ + (((x) >> CPL_TX_TNL_LSO_UDPCHKCLROUT_S) & \ + CPL_TX_TNL_LSO_UDPCHKCLROUT_M) +#define CPL_TX_TNL_LSO_UDPCHKCLROUT_F CPL_TX_TNL_LSO_UDPCHKCLROUT_V(1U) + +#define CPL_TX_TNL_LSO_UDPLENSETOUT_S 15 +#define CPL_TX_TNL_LSO_UDPLENSETOUT_M 0x1 +#define CPL_TX_TNL_LSO_UDPLENSETOUT_V(x) \ + ((x) << CPL_TX_TNL_LSO_UDPLENSETOUT_S) +#define CPL_TX_TNL_LSO_UDPLENSETOUT_G(x) \ + (((x) >> CPL_TX_TNL_LSO_UDPLENSETOUT_S) & \ + CPL_TX_TNL_LSO_UDPLENSETOUT_M) +#define CPL_TX_TNL_LSO_UDPLENSETOUT_F CPL_TX_TNL_LSO_UDPLENSETOUT_V(1U) + +#define CPL_TX_TNL_LSO_TNLTYPE_S 12 +#define CPL_TX_TNL_LSO_TNLTYPE_M 0x3 +#define CPL_TX_TNL_LSO_TNLTYPE_V(x) ((x) << CPL_TX_TNL_LSO_TNLTYPE_S) +#define CPL_TX_TNL_LSO_TNLTYPE_G(x) \ + (((x) >> CPL_TX_TNL_LSO_TNLTYPE_S) & CPL_TX_TNL_LSO_TNLTYPE_M) + +#define S_CPL_TX_TNL_LSO_ETHHDRLEN 16 +#define M_CPL_TX_TNL_LSO_ETHHDRLEN 0xf +#define V_CPL_TX_TNL_LSO_ETHHDRLEN(x) ((x) << S_CPL_TX_TNL_LSO_ETHHDRLEN) +#define G_CPL_TX_TNL_LSO_ETHHDRLEN(x) \ + (((x) >> S_CPL_TX_TNL_LSO_ETHHDRLEN) & M_CPL_TX_TNL_LSO_ETHHDRLEN) + +#define CPL_TX_TNL_LSO_TNLHDRLEN_S 0 +#define CPL_TX_TNL_LSO_TNLHDRLEN_M 0xfff +#define CPL_TX_TNL_LSO_TNLHDRLEN_V(x) ((x) << CPL_TX_TNL_LSO_TNLHDRLEN_S) +#define CPL_TX_TNL_LSO_TNLHDRLEN_G(x) \ + (((x) >> CPL_TX_TNL_LSO_TNLHDRLEN_S) & CPL_TX_TNL_LSO_TNLHDRLEN_M) + +#define CPL_TX_TNL_LSO_IPV6_S 20 +#define CPL_TX_TNL_LSO_IPV6_M 0x1 +#define CPL_TX_TNL_LSO_IPV6_V(x) ((x) << CPL_TX_TNL_LSO_IPV6_S) +#define CPL_TX_TNL_LSO_IPV6_G(x) \ + (((x) >> CPL_TX_TNL_LSO_IPV6_S) & CPL_TX_TNL_LSO_IPV6_M) +#define CPL_TX_TNL_LSO_IPV6_F CPL_TX_TNL_LSO_IPV6_V(1U) + +#define ULP_TX_SC_MORE_S 23 +#define ULP_TX_SC_MORE_V(x) ((x) << ULP_TX_SC_MORE_S) +#define ULP_TX_SC_MORE_F ULP_TX_SC_MORE_V(1U) + +struct ulp_mem_io { + WR_HDR; + __be32 cmd; + __be32 len16; /* command length */ + __be32 dlen; /* data length in 32-byte units */ + __be32 lock_addr; +}; + +#define ULP_MEMIO_LOCK_S 31 +#define ULP_MEMIO_LOCK_V(x) ((x) << ULP_MEMIO_LOCK_S) +#define ULP_MEMIO_LOCK_F ULP_MEMIO_LOCK_V(1U) + +/* additional ulp_mem_io.cmd fields */ +#define ULP_MEMIO_ORDER_S 23 +#define ULP_MEMIO_ORDER_V(x) ((x) << ULP_MEMIO_ORDER_S) +#define ULP_MEMIO_ORDER_F ULP_MEMIO_ORDER_V(1U) + +#define T5_ULP_MEMIO_IMM_S 23 +#define T5_ULP_MEMIO_IMM_V(x) ((x) << T5_ULP_MEMIO_IMM_S) +#define T5_ULP_MEMIO_IMM_F T5_ULP_MEMIO_IMM_V(1U) + +#define T5_ULP_MEMIO_ORDER_S 22 +#define T5_ULP_MEMIO_ORDER_V(x) ((x) << T5_ULP_MEMIO_ORDER_S) +#define T5_ULP_MEMIO_ORDER_F T5_ULP_MEMIO_ORDER_V(1U) + +#define T5_ULP_MEMIO_FID_S 4 +#define T5_ULP_MEMIO_FID_M 0x7ff +#define T5_ULP_MEMIO_FID_V(x) ((x) << T5_ULP_MEMIO_FID_S) + +/* ulp_mem_io.lock_addr fields */ +#define ULP_MEMIO_ADDR_S 0 +#define ULP_MEMIO_ADDR_V(x) ((x) << ULP_MEMIO_ADDR_S) + +/* ulp_mem_io.dlen fields */ +#define ULP_MEMIO_DATA_LEN_S 0 +#define ULP_MEMIO_DATA_LEN_V(x) ((x) << ULP_MEMIO_DATA_LEN_S) + +#define ULPTX_NSGE_S 0 +#define ULPTX_NSGE_M 0xFFFF +#define ULPTX_NSGE_V(x) ((x) << ULPTX_NSGE_S) +#define ULPTX_NSGE_G(x) (((x) >> ULPTX_NSGE_S) & ULPTX_NSGE_M) + +struct ulptx_sc_memrd { + __be32 cmd_to_len; + __be32 addr; +}; + +#define ULP_TXPKT_DATAMODIFY_S 23 +#define ULP_TXPKT_DATAMODIFY_M 0x1 +#define ULP_TXPKT_DATAMODIFY_V(x) ((x) << ULP_TXPKT_DATAMODIFY_S) +#define ULP_TXPKT_DATAMODIFY_G(x) \ + (((x) >> ULP_TXPKT_DATAMODIFY_S) & ULP_TXPKT_DATAMODIFY__M) +#define ULP_TXPKT_DATAMODIFY_F ULP_TXPKT_DATAMODIFY_V(1U) + +#define ULP_TXPKT_CHANNELID_S 22 +#define ULP_TXPKT_CHANNELID_M 0x1 +#define ULP_TXPKT_CHANNELID_V(x) ((x) << ULP_TXPKT_CHANNELID_S) +#define ULP_TXPKT_CHANNELID_G(x) \ + (((x) >> ULP_TXPKT_CHANNELID_S) & ULP_TXPKT_CHANNELID_M) +#define ULP_TXPKT_CHANNELID_F ULP_TXPKT_CHANNELID_V(1U) + +#define SCMD_SEQ_NO_CTRL_S 29 +#define SCMD_SEQ_NO_CTRL_M 0x3 +#define SCMD_SEQ_NO_CTRL_V(x) ((x) << SCMD_SEQ_NO_CTRL_S) +#define SCMD_SEQ_NO_CTRL_G(x) \ + (((x) >> SCMD_SEQ_NO_CTRL_S) & SCMD_SEQ_NO_CTRL_M) + +/* StsFieldPrsnt- Status field at the end of the TLS PDU */ +#define SCMD_STATUS_PRESENT_S 28 +#define SCMD_STATUS_PRESENT_M 0x1 +#define SCMD_STATUS_PRESENT_V(x) ((x) << SCMD_STATUS_PRESENT_S) +#define SCMD_STATUS_PRESENT_G(x) \ + (((x) >> SCMD_STATUS_PRESENT_S) & SCMD_STATUS_PRESENT_M) +#define SCMD_STATUS_PRESENT_F SCMD_STATUS_PRESENT_V(1U) + +/* ProtoVersion - Protocol Version 0: 1.2, 1:1.1, 2:DTLS, 3:Generic, + * 3-15: Reserved. + */ +#define SCMD_PROTO_VERSION_S 24 +#define SCMD_PROTO_VERSION_M 0xf +#define SCMD_PROTO_VERSION_V(x) ((x) << SCMD_PROTO_VERSION_S) +#define SCMD_PROTO_VERSION_G(x) \ + (((x) >> SCMD_PROTO_VERSION_S) & SCMD_PROTO_VERSION_M) + +/* EncDecCtrl - Encryption/Decryption Control. 0: Encrypt, 1: Decrypt */ +#define SCMD_ENC_DEC_CTRL_S 23 +#define SCMD_ENC_DEC_CTRL_M 0x1 +#define SCMD_ENC_DEC_CTRL_V(x) ((x) << SCMD_ENC_DEC_CTRL_S) +#define SCMD_ENC_DEC_CTRL_G(x) \ + (((x) >> SCMD_ENC_DEC_CTRL_S) & SCMD_ENC_DEC_CTRL_M) +#define SCMD_ENC_DEC_CTRL_F SCMD_ENC_DEC_CTRL_V(1U) + +/* CipherAuthSeqCtrl - Cipher Authentication Sequence Control. */ +#define SCMD_CIPH_AUTH_SEQ_CTRL_S 22 +#define SCMD_CIPH_AUTH_SEQ_CTRL_M 0x1 +#define SCMD_CIPH_AUTH_SEQ_CTRL_V(x) \ + ((x) << SCMD_CIPH_AUTH_SEQ_CTRL_S) +#define SCMD_CIPH_AUTH_SEQ_CTRL_G(x) \ + (((x) >> SCMD_CIPH_AUTH_SEQ_CTRL_S) & SCMD_CIPH_AUTH_SEQ_CTRL_M) +#define SCMD_CIPH_AUTH_SEQ_CTRL_F SCMD_CIPH_AUTH_SEQ_CTRL_V(1U) + +/* CiphMode - Cipher Mode. 0: NOP, 1:AES-CBC, 2:AES-GCM, 3:AES-CTR, + * 4:Generic-AES, 5-15: Reserved. + */ +#define SCMD_CIPH_MODE_S 18 +#define SCMD_CIPH_MODE_M 0xf +#define SCMD_CIPH_MODE_V(x) ((x) << SCMD_CIPH_MODE_S) +#define SCMD_CIPH_MODE_G(x) \ + (((x) >> SCMD_CIPH_MODE_S) & SCMD_CIPH_MODE_M) + +/* AuthMode - Auth Mode. 0: NOP, 1:SHA1, 2:SHA2-224, 3:SHA2-256 + * 4-15: Reserved + */ +#define SCMD_AUTH_MODE_S 14 +#define SCMD_AUTH_MODE_M 0xf +#define SCMD_AUTH_MODE_V(x) ((x) << SCMD_AUTH_MODE_S) +#define SCMD_AUTH_MODE_G(x) \ + (((x) >> SCMD_AUTH_MODE_S) & SCMD_AUTH_MODE_M) + +/* HmacCtrl - HMAC Control. 0:NOP, 1:No truncation, 2:Support HMAC Truncation + * per RFC 4366, 3:IPSec 96 bits, 4-7:Reserved + */ +#define SCMD_HMAC_CTRL_S 11 +#define SCMD_HMAC_CTRL_M 0x7 +#define SCMD_HMAC_CTRL_V(x) ((x) << SCMD_HMAC_CTRL_S) +#define SCMD_HMAC_CTRL_G(x) \ + (((x) >> SCMD_HMAC_CTRL_S) & SCMD_HMAC_CTRL_M) + +/* IvSize - IV size in units of 2 bytes */ +#define SCMD_IV_SIZE_S 7 +#define SCMD_IV_SIZE_M 0xf +#define SCMD_IV_SIZE_V(x) ((x) << SCMD_IV_SIZE_S) +#define SCMD_IV_SIZE_G(x) \ + (((x) >> SCMD_IV_SIZE_S) & SCMD_IV_SIZE_M) + +/* NumIVs - Number of IVs */ +#define SCMD_NUM_IVS_S 0 +#define SCMD_NUM_IVS_M 0x7f +#define SCMD_NUM_IVS_V(x) ((x) << SCMD_NUM_IVS_S) +#define SCMD_NUM_IVS_G(x) \ + (((x) >> SCMD_NUM_IVS_S) & SCMD_NUM_IVS_M) + +/* EnbDbgId - If this is enabled upper 20 (63:44) bits if SeqNumber + * (below) are used as Cid (connection id for debug status), these + * bits are padded to zero for forming the 64 bit + * sequence number for TLS + */ +#define SCMD_ENB_DBGID_S 31 +#define SCMD_ENB_DBGID_M 0x1 +#define SCMD_ENB_DBGID_V(x) ((x) << SCMD_ENB_DBGID_S) +#define SCMD_ENB_DBGID_G(x) \ + (((x) >> SCMD_ENB_DBGID_S) & SCMD_ENB_DBGID_M) + +/* IV generation in SW. */ +#define SCMD_IV_GEN_CTRL_S 30 +#define SCMD_IV_GEN_CTRL_M 0x1 +#define SCMD_IV_GEN_CTRL_V(x) ((x) << SCMD_IV_GEN_CTRL_S) +#define SCMD_IV_GEN_CTRL_G(x) \ + (((x) >> SCMD_IV_GEN_CTRL_S) & SCMD_IV_GEN_CTRL_M) +#define SCMD_IV_GEN_CTRL_F SCMD_IV_GEN_CTRL_V(1U) + +/* More frags */ +#define SCMD_MORE_FRAGS_S 20 +#define SCMD_MORE_FRAGS_M 0x1 +#define SCMD_MORE_FRAGS_V(x) ((x) << SCMD_MORE_FRAGS_S) +#define SCMD_MORE_FRAGS_G(x) (((x) >> SCMD_MORE_FRAGS_S) & SCMD_MORE_FRAGS_M) + +/*last frag */ +#define SCMD_LAST_FRAG_S 19 +#define SCMD_LAST_FRAG_M 0x1 +#define SCMD_LAST_FRAG_V(x) ((x) << SCMD_LAST_FRAG_S) +#define SCMD_LAST_FRAG_G(x) (((x) >> SCMD_LAST_FRAG_S) & SCMD_LAST_FRAG_M) + +/* TlsCompPdu */ +#define SCMD_TLS_COMPPDU_S 18 +#define SCMD_TLS_COMPPDU_M 0x1 +#define SCMD_TLS_COMPPDU_V(x) ((x) << SCMD_TLS_COMPPDU_S) +#define SCMD_TLS_COMPPDU_G(x) (((x) >> SCMD_TLS_COMPPDU_S) & SCMD_TLS_COMPPDU_M) + +/* KeyCntxtInline - Key context inline after the scmd OR PayloadOnly*/ +#define SCMD_KEY_CTX_INLINE_S 17 +#define SCMD_KEY_CTX_INLINE_M 0x1 +#define SCMD_KEY_CTX_INLINE_V(x) ((x) << SCMD_KEY_CTX_INLINE_S) +#define SCMD_KEY_CTX_INLINE_G(x) \ + (((x) >> SCMD_KEY_CTX_INLINE_S) & SCMD_KEY_CTX_INLINE_M) +#define SCMD_KEY_CTX_INLINE_F SCMD_KEY_CTX_INLINE_V(1U) + +/* TLSFragEnable - 0: Host created TLS PDUs, 1: TLS Framgmentation in ASIC */ +#define SCMD_TLS_FRAG_ENABLE_S 16 +#define SCMD_TLS_FRAG_ENABLE_M 0x1 +#define SCMD_TLS_FRAG_ENABLE_V(x) ((x) << SCMD_TLS_FRAG_ENABLE_S) +#define SCMD_TLS_FRAG_ENABLE_G(x) \ + (((x) >> SCMD_TLS_FRAG_ENABLE_S) & SCMD_TLS_FRAG_ENABLE_M) +#define SCMD_TLS_FRAG_ENABLE_F SCMD_TLS_FRAG_ENABLE_V(1U) + +/* MacOnly - Only send the MAC and discard PDU. This is valid for hash only + * modes, in this case TLS_TX will drop the PDU and only + * send back the MAC bytes. + */ +#define SCMD_MAC_ONLY_S 15 +#define SCMD_MAC_ONLY_M 0x1 +#define SCMD_MAC_ONLY_V(x) ((x) << SCMD_MAC_ONLY_S) +#define SCMD_MAC_ONLY_G(x) \ + (((x) >> SCMD_MAC_ONLY_S) & SCMD_MAC_ONLY_M) +#define SCMD_MAC_ONLY_F SCMD_MAC_ONLY_V(1U) + +/* AadIVDrop - Drop the AAD and IV fields. Useful in protocols + * which have complex AAD and IV formations Eg:AES-CCM + */ +#define SCMD_AADIVDROP_S 14 +#define SCMD_AADIVDROP_M 0x1 +#define SCMD_AADIVDROP_V(x) ((x) << SCMD_AADIVDROP_S) +#define SCMD_AADIVDROP_G(x) \ + (((x) >> SCMD_AADIVDROP_S) & SCMD_AADIVDROP_M) +#define SCMD_AADIVDROP_F SCMD_AADIVDROP_V(1U) + +/* HdrLength - Length of all headers excluding TLS header + * present before start of crypto PDU/payload. + */ +#define SCMD_HDR_LEN_S 0 +#define SCMD_HDR_LEN_M 0x3fff +#define SCMD_HDR_LEN_V(x) ((x) << SCMD_HDR_LEN_S) +#define SCMD_HDR_LEN_G(x) \ + (((x) >> SCMD_HDR_LEN_S) & SCMD_HDR_LEN_M) + +struct cpl_tx_sec_pdu { + __be32 op_ivinsrtofst; + __be32 pldlen; + __be32 aadstart_cipherstop_hi; + __be32 cipherstop_lo_authinsert; + __be32 seqno_numivs; + __be32 ivgen_hdrlen; + __be64 scmd1; +}; + +#define CPL_TX_SEC_PDU_OPCODE_S 24 +#define CPL_TX_SEC_PDU_OPCODE_M 0xff +#define CPL_TX_SEC_PDU_OPCODE_V(x) ((x) << CPL_TX_SEC_PDU_OPCODE_S) +#define CPL_TX_SEC_PDU_OPCODE_G(x) \ + (((x) >> CPL_TX_SEC_PDU_OPCODE_S) & CPL_TX_SEC_PDU_OPCODE_M) + +/* RX Channel Id */ +#define CPL_TX_SEC_PDU_RXCHID_S 22 +#define CPL_TX_SEC_PDU_RXCHID_M 0x1 +#define CPL_TX_SEC_PDU_RXCHID_V(x) ((x) << CPL_TX_SEC_PDU_RXCHID_S) +#define CPL_TX_SEC_PDU_RXCHID_G(x) \ + (((x) >> CPL_TX_SEC_PDU_RXCHID_S) & CPL_TX_SEC_PDU_RXCHID_M) +#define CPL_TX_SEC_PDU_RXCHID_F CPL_TX_SEC_PDU_RXCHID_V(1U) + +/* Ack Follows */ +#define CPL_TX_SEC_PDU_ACKFOLLOWS_S 21 +#define CPL_TX_SEC_PDU_ACKFOLLOWS_M 0x1 +#define CPL_TX_SEC_PDU_ACKFOLLOWS_V(x) ((x) << CPL_TX_SEC_PDU_ACKFOLLOWS_S) +#define CPL_TX_SEC_PDU_ACKFOLLOWS_G(x) \ + (((x) >> CPL_TX_SEC_PDU_ACKFOLLOWS_S) & CPL_TX_SEC_PDU_ACKFOLLOWS_M) +#define CPL_TX_SEC_PDU_ACKFOLLOWS_F CPL_TX_SEC_PDU_ACKFOLLOWS_V(1U) + +/* Loopback bit in cpl_tx_sec_pdu */ +#define CPL_TX_SEC_PDU_ULPTXLPBK_S 20 +#define CPL_TX_SEC_PDU_ULPTXLPBK_M 0x1 +#define CPL_TX_SEC_PDU_ULPTXLPBK_V(x) ((x) << CPL_TX_SEC_PDU_ULPTXLPBK_S) +#define CPL_TX_SEC_PDU_ULPTXLPBK_G(x) \ + (((x) >> CPL_TX_SEC_PDU_ULPTXLPBK_S) & CPL_TX_SEC_PDU_ULPTXLPBK_M) +#define CPL_TX_SEC_PDU_ULPTXLPBK_F CPL_TX_SEC_PDU_ULPTXLPBK_V(1U) + +/* Length of cpl header encapsulated */ +#define CPL_TX_SEC_PDU_CPLLEN_S 16 +#define CPL_TX_SEC_PDU_CPLLEN_M 0xf +#define CPL_TX_SEC_PDU_CPLLEN_V(x) ((x) << CPL_TX_SEC_PDU_CPLLEN_S) +#define CPL_TX_SEC_PDU_CPLLEN_G(x) \ + (((x) >> CPL_TX_SEC_PDU_CPLLEN_S) & CPL_TX_SEC_PDU_CPLLEN_M) + +/* PlaceHolder */ +#define CPL_TX_SEC_PDU_PLACEHOLDER_S 10 +#define CPL_TX_SEC_PDU_PLACEHOLDER_M 0x1 +#define CPL_TX_SEC_PDU_PLACEHOLDER_V(x) ((x) << CPL_TX_SEC_PDU_PLACEHOLDER_S) +#define CPL_TX_SEC_PDU_PLACEHOLDER_G(x) \ + (((x) >> CPL_TX_SEC_PDU_PLACEHOLDER_S) & \ + CPL_TX_SEC_PDU_PLACEHOLDER_M) + +/* IvInsrtOffset: Insertion location for IV */ +#define CPL_TX_SEC_PDU_IVINSRTOFST_S 0 +#define CPL_TX_SEC_PDU_IVINSRTOFST_M 0x3ff +#define CPL_TX_SEC_PDU_IVINSRTOFST_V(x) ((x) << CPL_TX_SEC_PDU_IVINSRTOFST_S) +#define CPL_TX_SEC_PDU_IVINSRTOFST_G(x) \ + (((x) >> CPL_TX_SEC_PDU_IVINSRTOFST_S) & \ + CPL_TX_SEC_PDU_IVINSRTOFST_M) + +/* AadStartOffset: Offset in bytes for AAD start from + * the first byte following the pkt headers (0-255 bytes) + */ +#define CPL_TX_SEC_PDU_AADSTART_S 24 +#define CPL_TX_SEC_PDU_AADSTART_M 0xff +#define CPL_TX_SEC_PDU_AADSTART_V(x) ((x) << CPL_TX_SEC_PDU_AADSTART_S) +#define CPL_TX_SEC_PDU_AADSTART_G(x) \ + (((x) >> CPL_TX_SEC_PDU_AADSTART_S) & \ + CPL_TX_SEC_PDU_AADSTART_M) + +/* AadStopOffset: offset in bytes for AAD stop/end from the first byte following + * the pkt headers (0-511 bytes) + */ +#define CPL_TX_SEC_PDU_AADSTOP_S 15 +#define CPL_TX_SEC_PDU_AADSTOP_M 0x1ff +#define CPL_TX_SEC_PDU_AADSTOP_V(x) ((x) << CPL_TX_SEC_PDU_AADSTOP_S) +#define CPL_TX_SEC_PDU_AADSTOP_G(x) \ + (((x) >> CPL_TX_SEC_PDU_AADSTOP_S) & CPL_TX_SEC_PDU_AADSTOP_M) + +/* CipherStartOffset: offset in bytes for encryption/decryption start from the + * first byte following the pkt headers (0-1023 bytes) + */ +#define CPL_TX_SEC_PDU_CIPHERSTART_S 5 +#define CPL_TX_SEC_PDU_CIPHERSTART_M 0x3ff +#define CPL_TX_SEC_PDU_CIPHERSTART_V(x) ((x) << CPL_TX_SEC_PDU_CIPHERSTART_S) +#define CPL_TX_SEC_PDU_CIPHERSTART_G(x) \ + (((x) >> CPL_TX_SEC_PDU_CIPHERSTART_S) & \ + CPL_TX_SEC_PDU_CIPHERSTART_M) + +/* CipherStopOffset: offset in bytes for encryption/decryption end + * from end of the payload of this command (0-511 bytes) + */ +#define CPL_TX_SEC_PDU_CIPHERSTOP_HI_S 0 +#define CPL_TX_SEC_PDU_CIPHERSTOP_HI_M 0x1f +#define CPL_TX_SEC_PDU_CIPHERSTOP_HI_V(x) \ + ((x) << CPL_TX_SEC_PDU_CIPHERSTOP_HI_S) +#define CPL_TX_SEC_PDU_CIPHERSTOP_HI_G(x) \ + (((x) >> CPL_TX_SEC_PDU_CIPHERSTOP_HI_S) & \ + CPL_TX_SEC_PDU_CIPHERSTOP_HI_M) + +#define CPL_TX_SEC_PDU_CIPHERSTOP_LO_S 28 +#define CPL_TX_SEC_PDU_CIPHERSTOP_LO_M 0xf +#define CPL_TX_SEC_PDU_CIPHERSTOP_LO_V(x) \ + ((x) << CPL_TX_SEC_PDU_CIPHERSTOP_LO_S) +#define CPL_TX_SEC_PDU_CIPHERSTOP_LO_G(x) \ + (((x) >> CPL_TX_SEC_PDU_CIPHERSTOP_LO_S) & \ + CPL_TX_SEC_PDU_CIPHERSTOP_LO_M) + +/* AuthStartOffset: offset in bytes for authentication start from + * the first byte following the pkt headers (0-1023) + */ +#define CPL_TX_SEC_PDU_AUTHSTART_S 18 +#define CPL_TX_SEC_PDU_AUTHSTART_M 0x3ff +#define CPL_TX_SEC_PDU_AUTHSTART_V(x) ((x) << CPL_TX_SEC_PDU_AUTHSTART_S) +#define CPL_TX_SEC_PDU_AUTHSTART_G(x) \ + (((x) >> CPL_TX_SEC_PDU_AUTHSTART_S) & \ + CPL_TX_SEC_PDU_AUTHSTART_M) + +/* AuthStopOffset: offset in bytes for authentication + * end from end of the payload of this command (0-511 Bytes) + */ +#define CPL_TX_SEC_PDU_AUTHSTOP_S 9 +#define CPL_TX_SEC_PDU_AUTHSTOP_M 0x1ff +#define CPL_TX_SEC_PDU_AUTHSTOP_V(x) ((x) << CPL_TX_SEC_PDU_AUTHSTOP_S) +#define CPL_TX_SEC_PDU_AUTHSTOP_G(x) \ + (((x) >> CPL_TX_SEC_PDU_AUTHSTOP_S) & \ + CPL_TX_SEC_PDU_AUTHSTOP_M) + +/* AuthInsrtOffset: offset in bytes for authentication insertion + * from end of the payload of this command (0-511 bytes) + */ +#define CPL_TX_SEC_PDU_AUTHINSERT_S 0 +#define CPL_TX_SEC_PDU_AUTHINSERT_M 0x1ff +#define CPL_TX_SEC_PDU_AUTHINSERT_V(x) ((x) << CPL_TX_SEC_PDU_AUTHINSERT_S) +#define CPL_TX_SEC_PDU_AUTHINSERT_G(x) \ + (((x) >> CPL_TX_SEC_PDU_AUTHINSERT_S) & \ + CPL_TX_SEC_PDU_AUTHINSERT_M) + +struct cpl_rx_phys_dsgl { + __be32 op_to_tid; + __be32 pcirlxorder_to_noofsgentr; + struct rss_header rss_hdr_int; +}; + +#define CPL_RX_PHYS_DSGL_OPCODE_S 24 +#define CPL_RX_PHYS_DSGL_OPCODE_M 0xff +#define CPL_RX_PHYS_DSGL_OPCODE_V(x) ((x) << CPL_RX_PHYS_DSGL_OPCODE_S) +#define CPL_RX_PHYS_DSGL_OPCODE_G(x) \ + (((x) >> CPL_RX_PHYS_DSGL_OPCODE_S) & CPL_RX_PHYS_DSGL_OPCODE_M) + +#define CPL_RX_PHYS_DSGL_ISRDMA_S 23 +#define CPL_RX_PHYS_DSGL_ISRDMA_M 0x1 +#define CPL_RX_PHYS_DSGL_ISRDMA_V(x) ((x) << CPL_RX_PHYS_DSGL_ISRDMA_S) +#define CPL_RX_PHYS_DSGL_ISRDMA_G(x) \ + (((x) >> CPL_RX_PHYS_DSGL_ISRDMA_S) & CPL_RX_PHYS_DSGL_ISRDMA_M) +#define CPL_RX_PHYS_DSGL_ISRDMA_F CPL_RX_PHYS_DSGL_ISRDMA_V(1U) + +#define CPL_RX_PHYS_DSGL_RSVD1_S 20 +#define CPL_RX_PHYS_DSGL_RSVD1_M 0x7 +#define CPL_RX_PHYS_DSGL_RSVD1_V(x) ((x) << CPL_RX_PHYS_DSGL_RSVD1_S) +#define CPL_RX_PHYS_DSGL_RSVD1_G(x) \ + (((x) >> CPL_RX_PHYS_DSGL_RSVD1_S) & \ + CPL_RX_PHYS_DSGL_RSVD1_M) + +#define CPL_RX_PHYS_DSGL_PCIRLXORDER_S 31 +#define CPL_RX_PHYS_DSGL_PCIRLXORDER_M 0x1 +#define CPL_RX_PHYS_DSGL_PCIRLXORDER_V(x) \ + ((x) << CPL_RX_PHYS_DSGL_PCIRLXORDER_S) +#define CPL_RX_PHYS_DSGL_PCIRLXORDER_G(x) \ + (((x) >> CPL_RX_PHYS_DSGL_PCIRLXORDER_S) & \ + CPL_RX_PHYS_DSGL_PCIRLXORDER_M) +#define CPL_RX_PHYS_DSGL_PCIRLXORDER_F CPL_RX_PHYS_DSGL_PCIRLXORDER_V(1U) + +#define CPL_RX_PHYS_DSGL_PCINOSNOOP_S 30 +#define CPL_RX_PHYS_DSGL_PCINOSNOOP_M 0x1 +#define CPL_RX_PHYS_DSGL_PCINOSNOOP_V(x) \ + ((x) << CPL_RX_PHYS_DSGL_PCINOSNOOP_S) +#define CPL_RX_PHYS_DSGL_PCINOSNOOP_G(x) \ + (((x) >> CPL_RX_PHYS_DSGL_PCINOSNOOP_S) & \ + CPL_RX_PHYS_DSGL_PCINOSNOOP_M) + +#define CPL_RX_PHYS_DSGL_PCINOSNOOP_F CPL_RX_PHYS_DSGL_PCINOSNOOP_V(1U) + +#define CPL_RX_PHYS_DSGL_PCITPHNTENB_S 29 +#define CPL_RX_PHYS_DSGL_PCITPHNTENB_M 0x1 +#define CPL_RX_PHYS_DSGL_PCITPHNTENB_V(x) \ + ((x) << CPL_RX_PHYS_DSGL_PCITPHNTENB_S) +#define CPL_RX_PHYS_DSGL_PCITPHNTENB_G(x) \ + (((x) >> CPL_RX_PHYS_DSGL_PCITPHNTENB_S) & \ + CPL_RX_PHYS_DSGL_PCITPHNTENB_M) +#define CPL_RX_PHYS_DSGL_PCITPHNTENB_F CPL_RX_PHYS_DSGL_PCITPHNTENB_V(1U) + +#define CPL_RX_PHYS_DSGL_PCITPHNT_S 27 +#define CPL_RX_PHYS_DSGL_PCITPHNT_M 0x3 +#define CPL_RX_PHYS_DSGL_PCITPHNT_V(x) ((x) << CPL_RX_PHYS_DSGL_PCITPHNT_S) +#define CPL_RX_PHYS_DSGL_PCITPHNT_G(x) \ + (((x) >> CPL_RX_PHYS_DSGL_PCITPHNT_S) & \ + CPL_RX_PHYS_DSGL_PCITPHNT_M) + +#define CPL_RX_PHYS_DSGL_DCAID_S 16 +#define CPL_RX_PHYS_DSGL_DCAID_M 0x7ff +#define CPL_RX_PHYS_DSGL_DCAID_V(x) ((x) << CPL_RX_PHYS_DSGL_DCAID_S) +#define CPL_RX_PHYS_DSGL_DCAID_G(x) \ + (((x) >> CPL_RX_PHYS_DSGL_DCAID_S) & \ + CPL_RX_PHYS_DSGL_DCAID_M) + +#define CPL_RX_PHYS_DSGL_NOOFSGENTR_S 0 +#define CPL_RX_PHYS_DSGL_NOOFSGENTR_M 0xffff +#define CPL_RX_PHYS_DSGL_NOOFSGENTR_V(x) \ + ((x) << CPL_RX_PHYS_DSGL_NOOFSGENTR_S) +#define CPL_RX_PHYS_DSGL_NOOFSGENTR_G(x) \ + (((x) >> CPL_RX_PHYS_DSGL_NOOFSGENTR_S) & \ + CPL_RX_PHYS_DSGL_NOOFSGENTR_M) + +struct cpl_rx_mps_pkt { + __be32 op_to_r1_hi; + __be32 r1_lo_length; +}; + +#define CPL_RX_MPS_PKT_OP_S 24 +#define CPL_RX_MPS_PKT_OP_M 0xff +#define CPL_RX_MPS_PKT_OP_V(x) ((x) << CPL_RX_MPS_PKT_OP_S) +#define CPL_RX_MPS_PKT_OP_G(x) \ + (((x) >> CPL_RX_MPS_PKT_OP_S) & CPL_RX_MPS_PKT_OP_M) + +#define CPL_RX_MPS_PKT_TYPE_S 20 +#define CPL_RX_MPS_PKT_TYPE_M 0xf +#define CPL_RX_MPS_PKT_TYPE_V(x) ((x) << CPL_RX_MPS_PKT_TYPE_S) +#define CPL_RX_MPS_PKT_TYPE_G(x) \ + (((x) >> CPL_RX_MPS_PKT_TYPE_S) & CPL_RX_MPS_PKT_TYPE_M) + +enum { + X_CPL_RX_MPS_PKT_TYPE_PAUSE = 1 << 0, + X_CPL_RX_MPS_PKT_TYPE_PPP = 1 << 1, + X_CPL_RX_MPS_PKT_TYPE_QFC = 1 << 2, + X_CPL_RX_MPS_PKT_TYPE_PTP = 1 << 3 +}; + +struct cpl_srq_table_req { + WR_HDR; + union opcode_tid ot; + __u8 status; + __u8 rsvd[2]; + __u8 idx; + __be64 rsvd_pdid; + __be32 qlen_qbase; + __be16 cur_msn; + __be16 max_msn; +}; + +struct cpl_srq_table_rpl { + union opcode_tid ot; + __u8 status; + __u8 rsvd[2]; + __u8 idx; + __be64 rsvd_pdid; + __be32 qlen_qbase; + __be16 cur_msn; + __be16 max_msn; +}; + +/* cpl_srq_table_{req,rpl}.params fields */ +#define SRQT_QLEN_S 28 +#define SRQT_QLEN_M 0xF +#define SRQT_QLEN_V(x) ((x) << SRQT_QLEN_S) +#define SRQT_QLEN_G(x) (((x) >> SRQT_QLEN_S) & SRQT_QLEN_M) + +#define SRQT_QBASE_S 0 +#define SRQT_QBASE_M 0x3FFFFFF +#define SRQT_QBASE_V(x) ((x) << SRQT_QBASE_S) +#define SRQT_QBASE_G(x) (((x) >> SRQT_QBASE_S) & SRQT_QBASE_M) + +#define SRQT_PDID_S 0 +#define SRQT_PDID_M 0xFF +#define SRQT_PDID_V(x) ((x) << SRQT_PDID_S) +#define SRQT_PDID_G(x) (((x) >> SRQT_PDID_S) & SRQT_PDID_M) + +#define SRQT_IDX_S 0 +#define SRQT_IDX_M 0xF +#define SRQT_IDX_V(x) ((x) << SRQT_IDX_S) +#define SRQT_IDX_G(x) (((x) >> SRQT_IDX_S) & SRQT_IDX_M) + +struct cpl_tx_tls_sfo { + __be32 op_to_seg_len; + __be32 pld_len; + __be32 type_protover; + __be32 r1_lo; + __be32 seqno_numivs; + __be32 ivgen_hdrlen; + __be64 scmd1; +}; + +/* cpl_tx_tls_sfo macros */ +#define CPL_TX_TLS_SFO_OPCODE_S 24 +#define CPL_TX_TLS_SFO_OPCODE_V(x) ((x) << CPL_TX_TLS_SFO_OPCODE_S) + +#define CPL_TX_TLS_SFO_DATA_TYPE_S 20 +#define CPL_TX_TLS_SFO_DATA_TYPE_V(x) ((x) << CPL_TX_TLS_SFO_DATA_TYPE_S) + +#define CPL_TX_TLS_SFO_CPL_LEN_S 16 +#define CPL_TX_TLS_SFO_CPL_LEN_V(x) ((x) << CPL_TX_TLS_SFO_CPL_LEN_S) + +#define CPL_TX_TLS_SFO_SEG_LEN_S 0 +#define CPL_TX_TLS_SFO_SEG_LEN_M 0xffff +#define CPL_TX_TLS_SFO_SEG_LEN_V(x) ((x) << CPL_TX_TLS_SFO_SEG_LEN_S) +#define CPL_TX_TLS_SFO_SEG_LEN_G(x) \ + (((x) >> CPL_TX_TLS_SFO_SEG_LEN_S) & CPL_TX_TLS_SFO_SEG_LEN_M) + +#define CPL_TX_TLS_SFO_TYPE_S 24 +#define CPL_TX_TLS_SFO_TYPE_M 0xff +#define CPL_TX_TLS_SFO_TYPE_V(x) ((x) << CPL_TX_TLS_SFO_TYPE_S) +#define CPL_TX_TLS_SFO_TYPE_G(x) \ + (((x) >> CPL_TX_TLS_SFO_TYPE_S) & CPL_TX_TLS_SFO_TYPE_M) + +#define CPL_TX_TLS_SFO_PROTOVER_S 8 +#define CPL_TX_TLS_SFO_PROTOVER_M 0xffff +#define CPL_TX_TLS_SFO_PROTOVER_V(x) ((x) << CPL_TX_TLS_SFO_PROTOVER_S) +#define CPL_TX_TLS_SFO_PROTOVER_G(x) \ + (((x) >> CPL_TX_TLS_SFO_PROTOVER_S) & CPL_TX_TLS_SFO_PROTOVER_M) + +struct cpl_tls_data { + struct rss_header rsshdr; + union opcode_tid ot; + __be32 length_pkd; + __be32 seq; + __be32 r1; +}; + +#define CPL_TLS_DATA_OPCODE_S 24 +#define CPL_TLS_DATA_OPCODE_M 0xff +#define CPL_TLS_DATA_OPCODE_V(x) ((x) << CPL_TLS_DATA_OPCODE_S) +#define CPL_TLS_DATA_OPCODE_G(x) \ + (((x) >> CPL_TLS_DATA_OPCODE_S) & CPL_TLS_DATA_OPCODE_M) + +#define CPL_TLS_DATA_TID_S 0 +#define CPL_TLS_DATA_TID_M 0xffffff +#define CPL_TLS_DATA_TID_V(x) ((x) << CPL_TLS_DATA_TID_S) +#define CPL_TLS_DATA_TID_G(x) \ + (((x) >> CPL_TLS_DATA_TID_S) & CPL_TLS_DATA_TID_M) + +#define CPL_TLS_DATA_LENGTH_S 0 +#define CPL_TLS_DATA_LENGTH_M 0xffff +#define CPL_TLS_DATA_LENGTH_V(x) ((x) << CPL_TLS_DATA_LENGTH_S) +#define CPL_TLS_DATA_LENGTH_G(x) \ + (((x) >> CPL_TLS_DATA_LENGTH_S) & CPL_TLS_DATA_LENGTH_M) + +struct cpl_rx_tls_cmp { + struct rss_header rsshdr; + union opcode_tid ot; + __be32 pdulength_length; + __be32 seq; + __be32 ddp_report; + __be32 r; + __be32 ddp_valid; +}; + +#define CPL_RX_TLS_CMP_OPCODE_S 24 +#define CPL_RX_TLS_CMP_OPCODE_M 0xff +#define CPL_RX_TLS_CMP_OPCODE_V(x) ((x) << CPL_RX_TLS_CMP_OPCODE_S) +#define CPL_RX_TLS_CMP_OPCODE_G(x) \ + (((x) >> CPL_RX_TLS_CMP_OPCODE_S) & CPL_RX_TLS_CMP_OPCODE_M) + +#define CPL_RX_TLS_CMP_TID_S 0 +#define CPL_RX_TLS_CMP_TID_M 0xffffff +#define CPL_RX_TLS_CMP_TID_V(x) ((x) << CPL_RX_TLS_CMP_TID_S) +#define CPL_RX_TLS_CMP_TID_G(x) \ + (((x) >> CPL_RX_TLS_CMP_TID_S) & CPL_RX_TLS_CMP_TID_M) + +#define CPL_RX_TLS_CMP_PDULENGTH_S 16 +#define CPL_RX_TLS_CMP_PDULENGTH_M 0xffff +#define CPL_RX_TLS_CMP_PDULENGTH_V(x) ((x) << CPL_RX_TLS_CMP_PDULENGTH_S) +#define CPL_RX_TLS_CMP_PDULENGTH_G(x) \ + (((x) >> CPL_RX_TLS_CMP_PDULENGTH_S) & CPL_RX_TLS_CMP_PDULENGTH_M) + +#define CPL_RX_TLS_CMP_LENGTH_S 0 +#define CPL_RX_TLS_CMP_LENGTH_M 0xffff +#define CPL_RX_TLS_CMP_LENGTH_V(x) ((x) << CPL_RX_TLS_CMP_LENGTH_S) +#define CPL_RX_TLS_CMP_LENGTH_G(x) \ + (((x) >> CPL_RX_TLS_CMP_LENGTH_S) & CPL_RX_TLS_CMP_LENGTH_M) +#endif /* __T4_MSG_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h b/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h new file mode 100644 index 000000000..0b1b5f9c6 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h @@ -0,0 +1,225 @@ +/* + * This file is part of the Chelsio T4/T5 Ethernet driver for Linux. + * + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef __T4_PCI_ID_TBL_H__ +#define __T4_PCI_ID_TBL_H__ + +/* The code can defined cpp macros for creating a PCI Device ID Table. This is + * useful because it allows the PCI ID Table to be maintained in a single place. + * + * The macros are: + * + * CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN + * -- Used to start the definition of the PCI ID Table. + * + * CH_PCI_DEVICE_ID_FUNCTION + * -- The PCI Function Number to use in the PCI Device ID Table. "0" + * -- for drivers attaching to PF0-3, "4" for drivers attaching to PF4, + * -- "8" for drivers attaching to SR-IOV Virtual Functions, etc. + * + * CH_PCI_DEVICE_ID_FUNCTION2 [optional] + * -- If defined, create a PCI Device ID Table with both + * -- CH_PCI_DEVICE_ID_FUNCTION and CH_PCI_DEVICE_ID_FUNCTION2 populated. + * + * CH_PCI_ID_TABLE_ENTRY(DeviceID) + * -- Used for the individual PCI Device ID entries. Note that we will + * -- be adding a trailing comma (",") after all of the entries (and + * -- between the pairs of entries if CH_PCI_DEVICE_ID_FUNCTION2 is defined). + * + * CH_PCI_DEVICE_ID_TABLE_DEFINE_END + * -- Used to finish the definition of the PCI ID Table. Note that we + * -- will be adding a trailing semi-colon (";") here. + */ +#ifndef CH_PCI_DEVICE_ID_FUNCTION +#error CH_PCI_DEVICE_ID_FUNCTION not defined! +#endif +#ifndef CH_PCI_ID_TABLE_ENTRY +#error CH_PCI_ID_TABLE_ENTRY not defined! +#endif +#ifndef CH_PCI_DEVICE_ID_TABLE_DEFINE_END +#error CH_PCI_DEVICE_ID_TABLE_DEFINE_END not defined! +#endif + +/* T4 and later ASICs use a PCI Device ID scheme of 0xVFPP where: + * + * V = "4" for T4; "5" for T5, etc. + * F = "0" for PF 0..3; "4".."7" for PF4..7; and "8" for VFs + * PP = adapter product designation + * + * We use this consistency in order to create the proper PCI Device IDs + * for the specified CH_PCI_DEVICE_ID_FUNCTION. + */ +#ifndef CH_PCI_DEVICE_ID_FUNCTION2 +#define CH_PCI_ID_TABLE_FENTRY(devid) \ + CH_PCI_ID_TABLE_ENTRY((devid) | \ + ((CH_PCI_DEVICE_ID_FUNCTION) << 8)) +#else +#define CH_PCI_ID_TABLE_FENTRY(devid) \ + CH_PCI_ID_TABLE_ENTRY((devid) | \ + ((CH_PCI_DEVICE_ID_FUNCTION) << 8)), \ + CH_PCI_ID_TABLE_ENTRY((devid) | \ + ((CH_PCI_DEVICE_ID_FUNCTION2) << 8)) +#endif + +CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN + /* T4 adapters: + */ + CH_PCI_ID_TABLE_FENTRY(0x4000), /* T440-dbg */ + CH_PCI_ID_TABLE_FENTRY(0x4001), /* T420-cr */ + CH_PCI_ID_TABLE_FENTRY(0x4002), /* T422-cr */ + CH_PCI_ID_TABLE_FENTRY(0x4003), /* T440-cr */ + CH_PCI_ID_TABLE_FENTRY(0x4004), /* T420-bch */ + CH_PCI_ID_TABLE_FENTRY(0x4005), /* T440-bch */ + CH_PCI_ID_TABLE_FENTRY(0x4006), /* T440-ch */ + CH_PCI_ID_TABLE_FENTRY(0x4007), /* T420-so */ + CH_PCI_ID_TABLE_FENTRY(0x4008), /* T420-cx */ + CH_PCI_ID_TABLE_FENTRY(0x4009), /* T420-bt */ + CH_PCI_ID_TABLE_FENTRY(0x400a), /* T404-bt */ + CH_PCI_ID_TABLE_FENTRY(0x400b), /* B420-sr */ + CH_PCI_ID_TABLE_FENTRY(0x400c), /* B404-bt */ + CH_PCI_ID_TABLE_FENTRY(0x400d), /* T480-cr */ + CH_PCI_ID_TABLE_FENTRY(0x400e), /* T440-LP-cr */ + CH_PCI_ID_TABLE_FENTRY(0x4080), /* Custom T480-cr */ + CH_PCI_ID_TABLE_FENTRY(0x4081), /* Custom T440-cr */ + CH_PCI_ID_TABLE_FENTRY(0x4082), /* Custom T420-cr */ + CH_PCI_ID_TABLE_FENTRY(0x4083), /* Custom T420-xaui */ + CH_PCI_ID_TABLE_FENTRY(0x4084), /* Custom T440-cr */ + CH_PCI_ID_TABLE_FENTRY(0x4085), /* Custom T420-cr */ + CH_PCI_ID_TABLE_FENTRY(0x4086), /* Custom T440-bt */ + CH_PCI_ID_TABLE_FENTRY(0x4087), /* Custom T440-cr */ + CH_PCI_ID_TABLE_FENTRY(0x4088), /* Custom T440 2-xaui, 2-xfi */ + + /* T5 adapters: + */ + CH_PCI_ID_TABLE_FENTRY(0x5000), /* T580-dbg */ + CH_PCI_ID_TABLE_FENTRY(0x5001), /* T520-cr */ + CH_PCI_ID_TABLE_FENTRY(0x5002), /* T522-cr */ + CH_PCI_ID_TABLE_FENTRY(0x5003), /* T540-cr */ + CH_PCI_ID_TABLE_FENTRY(0x5004), /* T520-bch */ + CH_PCI_ID_TABLE_FENTRY(0x5005), /* T540-bch */ + CH_PCI_ID_TABLE_FENTRY(0x5006), /* T540-ch */ + CH_PCI_ID_TABLE_FENTRY(0x5007), /* T520-so */ + CH_PCI_ID_TABLE_FENTRY(0x5008), /* T520-cx */ + CH_PCI_ID_TABLE_FENTRY(0x5009), /* T520-bt */ + CH_PCI_ID_TABLE_FENTRY(0x500a), /* T504-bt */ + CH_PCI_ID_TABLE_FENTRY(0x500b), /* B520-sr */ + CH_PCI_ID_TABLE_FENTRY(0x500c), /* B504-bt */ + CH_PCI_ID_TABLE_FENTRY(0x500d), /* T580-cr */ + CH_PCI_ID_TABLE_FENTRY(0x500e), /* T540-LP-cr */ + CH_PCI_ID_TABLE_FENTRY(0x5010), /* T580-LP-cr */ + CH_PCI_ID_TABLE_FENTRY(0x5011), /* T520-LL-cr */ + CH_PCI_ID_TABLE_FENTRY(0x5012), /* T560-cr */ + CH_PCI_ID_TABLE_FENTRY(0x5013), /* T580-chr */ + CH_PCI_ID_TABLE_FENTRY(0x5014), /* T580-so */ + CH_PCI_ID_TABLE_FENTRY(0x5015), /* T502-bt */ + CH_PCI_ID_TABLE_FENTRY(0x5016), /* T580-OCP-SO */ + CH_PCI_ID_TABLE_FENTRY(0x5017), /* T520-OCP-SO */ + CH_PCI_ID_TABLE_FENTRY(0x5018), /* T540-BT */ + CH_PCI_ID_TABLE_FENTRY(0x5019), /* T540-LP-BT */ + CH_PCI_ID_TABLE_FENTRY(0x501a), /* T540-SO-BT */ + CH_PCI_ID_TABLE_FENTRY(0x501b), /* T540-SO-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5080), /* Custom T540-cr */ + CH_PCI_ID_TABLE_FENTRY(0x5081), /* Custom T540-LL-cr */ + CH_PCI_ID_TABLE_FENTRY(0x5082), /* Custom T504-cr */ + CH_PCI_ID_TABLE_FENTRY(0x5083), /* Custom T540-LP-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5084), /* Custom T580-cr */ + CH_PCI_ID_TABLE_FENTRY(0x5085), /* Custom 3x T580-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5086), /* Custom 2x T580-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5087), /* Custom T580-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5088), /* Custom T570-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5089), /* Custom T520-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5090), /* Custom T540-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5091), /* Custom T522-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5092), /* Custom T520-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5093), /* Custom T580-LP-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5094), /* Custom T540-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5095), /* Custom T540-CR-SO */ + CH_PCI_ID_TABLE_FENTRY(0x5096), /* Custom T580-CR */ + CH_PCI_ID_TABLE_FENTRY(0x5097), /* Custom T520-KR */ + CH_PCI_ID_TABLE_FENTRY(0x5098), /* Custom 2x40G QSFP */ + CH_PCI_ID_TABLE_FENTRY(0x5099), /* Custom 2x40G QSFP */ + CH_PCI_ID_TABLE_FENTRY(0x509a), /* Custom T520-CR */ + CH_PCI_ID_TABLE_FENTRY(0x509b), /* Custom T540-CR LOM */ + CH_PCI_ID_TABLE_FENTRY(0x509c), /* Custom T520-CR*/ + CH_PCI_ID_TABLE_FENTRY(0x509d), /* Custom T540-CR*/ + CH_PCI_ID_TABLE_FENTRY(0x509e), /* Custom T520-CR */ + CH_PCI_ID_TABLE_FENTRY(0x509f), /* Custom T540-CR */ + CH_PCI_ID_TABLE_FENTRY(0x50a0), /* Custom T540-CR */ + CH_PCI_ID_TABLE_FENTRY(0x50a1), /* Custom T540-CR */ + CH_PCI_ID_TABLE_FENTRY(0x50a2), /* Custom T540-KR4 */ + CH_PCI_ID_TABLE_FENTRY(0x50a3), /* Custom T580-KR4 */ + CH_PCI_ID_TABLE_FENTRY(0x50a4), /* Custom 2x T540-CR */ + CH_PCI_ID_TABLE_FENTRY(0x50a5), /* Custom T522-BT */ + CH_PCI_ID_TABLE_FENTRY(0x50a6), /* Custom T522-BT-SO */ + CH_PCI_ID_TABLE_FENTRY(0x50a7), /* Custom T580-CR */ + CH_PCI_ID_TABLE_FENTRY(0x50a8), /* Custom T580-KR */ + CH_PCI_ID_TABLE_FENTRY(0x50a9), /* Custom T580-KR */ + CH_PCI_ID_TABLE_FENTRY(0x50aa), /* Custom T580-CR */ + CH_PCI_ID_TABLE_FENTRY(0x50ab), /* Custom T520-CR */ + CH_PCI_ID_TABLE_FENTRY(0x50ac), /* Custom T540-BT */ + CH_PCI_ID_TABLE_FENTRY(0x50ad), /* Custom T520-CR */ + CH_PCI_ID_TABLE_FENTRY(0x50ae), /* Custom T540-XL-SO */ + CH_PCI_ID_TABLE_FENTRY(0x50af), /* Custom T580-KR-SO */ + CH_PCI_ID_TABLE_FENTRY(0x50b0), /* Custom T520-CR-LOM */ + + /* T6 adapters: + */ + CH_PCI_ID_TABLE_FENTRY(0x6001), + CH_PCI_ID_TABLE_FENTRY(0x6002), + CH_PCI_ID_TABLE_FENTRY(0x6003), + CH_PCI_ID_TABLE_FENTRY(0x6004), + CH_PCI_ID_TABLE_FENTRY(0x6005), + CH_PCI_ID_TABLE_FENTRY(0x6006), + CH_PCI_ID_TABLE_FENTRY(0x6007), + CH_PCI_ID_TABLE_FENTRY(0x6008), + CH_PCI_ID_TABLE_FENTRY(0x6009), + CH_PCI_ID_TABLE_FENTRY(0x600d), + CH_PCI_ID_TABLE_FENTRY(0x6011), + CH_PCI_ID_TABLE_FENTRY(0x6014), + CH_PCI_ID_TABLE_FENTRY(0x6015), + CH_PCI_ID_TABLE_FENTRY(0x6080), + CH_PCI_ID_TABLE_FENTRY(0x6081), + CH_PCI_ID_TABLE_FENTRY(0x6082), /* Custom T6225-CR SFP28 */ + CH_PCI_ID_TABLE_FENTRY(0x6083), /* Custom T62100-CR QSFP28 */ + CH_PCI_ID_TABLE_FENTRY(0x6084), /* Custom T64100-CR QSFP28 */ + CH_PCI_ID_TABLE_FENTRY(0x6085), /* Custom T6240-SO */ + CH_PCI_ID_TABLE_FENTRY(0x6086), /* Custom T6225-SO-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6087), /* Custom T6225-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6088), /* Custom T62100-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6089), /* Custom T62100-KR */ + CH_PCI_ID_TABLE_FENTRY(0x608a), /* Custom T62100-CR */ + CH_PCI_ID_TABLE_FENTRY(0x608b), /* Custom T6225-CR */ + CH_PCI_ID_TABLE_FENTRY(0x6092), /* Custom T62100-CR-LOM */ +CH_PCI_DEVICE_ID_TABLE_DEFINE_END; + +#endif /* __T4_PCI_ID_TBL_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h b/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h new file mode 100644 index 000000000..695916ba0 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h @@ -0,0 +1,3386 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __T4_REGS_H +#define __T4_REGS_H + +#define MYPF_BASE 0x1b000 +#define MYPF_REG(reg_addr) (MYPF_BASE + (reg_addr)) + +#define PF0_BASE 0x1e000 +#define PF0_REG(reg_addr) (PF0_BASE + (reg_addr)) + +#define PF_STRIDE 0x400 +#define PF_BASE(idx) (PF0_BASE + (idx) * PF_STRIDE) +#define PF_REG(idx, reg) (PF_BASE(idx) + (reg)) + +#define NUM_CIM_CTL_TSCH_CHANNEL_INSTANCES 4 +#define NUM_CIM_CTL_TSCH_CHANNEL_TSCH_CLASS_INSTANCES 16 + +#define MYPORT_BASE 0x1c000 +#define MYPORT_REG(reg_addr) (MYPORT_BASE + (reg_addr)) + +#define PORT0_BASE 0x20000 +#define PORT0_REG(reg_addr) (PORT0_BASE + (reg_addr)) + +#define PORT_STRIDE 0x2000 +#define PORT_BASE(idx) (PORT0_BASE + (idx) * PORT_STRIDE) +#define PORT_REG(idx, reg) (PORT_BASE(idx) + (reg)) + +#define EDC_STRIDE (EDC_1_BASE_ADDR - EDC_0_BASE_ADDR) +#define EDC_REG(reg, idx) (reg + EDC_STRIDE * idx) + +#define PCIE_MEM_ACCESS_REG(reg_addr, idx) ((reg_addr) + (idx) * 8) +#define PCIE_MAILBOX_REG(reg_addr, idx) ((reg_addr) + (idx) * 8) +#define MC_BIST_STATUS_REG(reg_addr, idx) ((reg_addr) + (idx) * 4) +#define EDC_BIST_STATUS_REG(reg_addr, idx) ((reg_addr) + (idx) * 4) + +#define PCIE_FW_REG(reg_addr, idx) ((reg_addr) + (idx) * 4) + +#define NUM_LE_DB_DBGI_REQ_DATA_INSTANCES 17 +#define NUM_LE_DB_DBGI_RSP_DATA_INSTANCES 17 + +#define SGE_PF_KDOORBELL_A 0x0 + +#define QID_S 15 +#define QID_V(x) ((x) << QID_S) + +#define DBPRIO_S 14 +#define DBPRIO_V(x) ((x) << DBPRIO_S) +#define DBPRIO_F DBPRIO_V(1U) + +#define PIDX_S 0 +#define PIDX_V(x) ((x) << PIDX_S) + +#define SGE_VF_KDOORBELL_A 0x0 + +#define DBTYPE_S 13 +#define DBTYPE_V(x) ((x) << DBTYPE_S) +#define DBTYPE_F DBTYPE_V(1U) + +#define PIDX_T5_S 0 +#define PIDX_T5_M 0x1fffU +#define PIDX_T5_V(x) ((x) << PIDX_T5_S) +#define PIDX_T5_G(x) (((x) >> PIDX_T5_S) & PIDX_T5_M) + +#define SGE_PF_GTS_A 0x4 + +#define INGRESSQID_S 16 +#define INGRESSQID_V(x) ((x) << INGRESSQID_S) + +#define TIMERREG_S 13 +#define TIMERREG_V(x) ((x) << TIMERREG_S) + +#define SEINTARM_S 12 +#define SEINTARM_V(x) ((x) << SEINTARM_S) + +#define CIDXINC_S 0 +#define CIDXINC_M 0xfffU +#define CIDXINC_V(x) ((x) << CIDXINC_S) + +#define SGE_CONTROL_A 0x1008 +#define SGE_CONTROL2_A 0x1124 + +#define RXPKTCPLMODE_S 18 +#define RXPKTCPLMODE_V(x) ((x) << RXPKTCPLMODE_S) +#define RXPKTCPLMODE_F RXPKTCPLMODE_V(1U) + +#define EGRSTATUSPAGESIZE_S 17 +#define EGRSTATUSPAGESIZE_V(x) ((x) << EGRSTATUSPAGESIZE_S) +#define EGRSTATUSPAGESIZE_F EGRSTATUSPAGESIZE_V(1U) + +#define PKTSHIFT_S 10 +#define PKTSHIFT_M 0x7U +#define PKTSHIFT_V(x) ((x) << PKTSHIFT_S) +#define PKTSHIFT_G(x) (((x) >> PKTSHIFT_S) & PKTSHIFT_M) + +#define INGPCIEBOUNDARY_S 7 +#define INGPCIEBOUNDARY_V(x) ((x) << INGPCIEBOUNDARY_S) + +#define INGPADBOUNDARY_S 4 +#define INGPADBOUNDARY_M 0x7U +#define INGPADBOUNDARY_V(x) ((x) << INGPADBOUNDARY_S) +#define INGPADBOUNDARY_G(x) (((x) >> INGPADBOUNDARY_S) & INGPADBOUNDARY_M) + +#define EGRPCIEBOUNDARY_S 1 +#define EGRPCIEBOUNDARY_V(x) ((x) << EGRPCIEBOUNDARY_S) + +#define INGPACKBOUNDARY_S 16 +#define INGPACKBOUNDARY_M 0x7U +#define INGPACKBOUNDARY_V(x) ((x) << INGPACKBOUNDARY_S) +#define INGPACKBOUNDARY_G(x) (((x) >> INGPACKBOUNDARY_S) \ + & INGPACKBOUNDARY_M) + +#define VFIFO_ENABLE_S 10 +#define VFIFO_ENABLE_V(x) ((x) << VFIFO_ENABLE_S) +#define VFIFO_ENABLE_F VFIFO_ENABLE_V(1U) + +#define SGE_DBVFIFO_BADDR_A 0x1138 + +#define DBVFIFO_SIZE_S 6 +#define DBVFIFO_SIZE_M 0xfffU +#define DBVFIFO_SIZE_G(x) (((x) >> DBVFIFO_SIZE_S) & DBVFIFO_SIZE_M) + +#define T6_DBVFIFO_SIZE_S 0 +#define T6_DBVFIFO_SIZE_M 0x1fffU +#define T6_DBVFIFO_SIZE_G(x) (((x) >> T6_DBVFIFO_SIZE_S) & T6_DBVFIFO_SIZE_M) + +#define SGE_CTXT_CMD_A 0x11fc + +#define BUSY_S 31 +#define BUSY_V(x) ((x) << BUSY_S) +#define BUSY_F BUSY_V(1U) + +#define CTXTTYPE_S 24 +#define CTXTTYPE_M 0x3U +#define CTXTTYPE_V(x) ((x) << CTXTTYPE_S) + +#define CTXTQID_S 0 +#define CTXTQID_M 0x1ffffU +#define CTXTQID_V(x) ((x) << CTXTQID_S) + +#define SGE_CTXT_DATA0_A 0x1200 +#define SGE_CTXT_DATA5_A 0x1214 + +#define GLOBALENABLE_S 0 +#define GLOBALENABLE_V(x) ((x) << GLOBALENABLE_S) +#define GLOBALENABLE_F GLOBALENABLE_V(1U) + +#define SGE_HOST_PAGE_SIZE_A 0x100c + +#define HOSTPAGESIZEPF7_S 28 +#define HOSTPAGESIZEPF7_M 0xfU +#define HOSTPAGESIZEPF7_V(x) ((x) << HOSTPAGESIZEPF7_S) +#define HOSTPAGESIZEPF7_G(x) (((x) >> HOSTPAGESIZEPF7_S) & HOSTPAGESIZEPF7_M) + +#define HOSTPAGESIZEPF6_S 24 +#define HOSTPAGESIZEPF6_M 0xfU +#define HOSTPAGESIZEPF6_V(x) ((x) << HOSTPAGESIZEPF6_S) +#define HOSTPAGESIZEPF6_G(x) (((x) >> HOSTPAGESIZEPF6_S) & HOSTPAGESIZEPF6_M) + +#define HOSTPAGESIZEPF5_S 20 +#define HOSTPAGESIZEPF5_M 0xfU +#define HOSTPAGESIZEPF5_V(x) ((x) << HOSTPAGESIZEPF5_S) +#define HOSTPAGESIZEPF5_G(x) (((x) >> HOSTPAGESIZEPF5_S) & HOSTPAGESIZEPF5_M) + +#define HOSTPAGESIZEPF4_S 16 +#define HOSTPAGESIZEPF4_M 0xfU +#define HOSTPAGESIZEPF4_V(x) ((x) << HOSTPAGESIZEPF4_S) +#define HOSTPAGESIZEPF4_G(x) (((x) >> HOSTPAGESIZEPF4_S) & HOSTPAGESIZEPF4_M) + +#define HOSTPAGESIZEPF3_S 12 +#define HOSTPAGESIZEPF3_M 0xfU +#define HOSTPAGESIZEPF3_V(x) ((x) << HOSTPAGESIZEPF3_S) +#define HOSTPAGESIZEPF3_G(x) (((x) >> HOSTPAGESIZEPF3_S) & HOSTPAGESIZEPF3_M) + +#define HOSTPAGESIZEPF2_S 8 +#define HOSTPAGESIZEPF2_M 0xfU +#define HOSTPAGESIZEPF2_V(x) ((x) << HOSTPAGESIZEPF2_S) +#define HOSTPAGESIZEPF2_G(x) (((x) >> HOSTPAGESIZEPF2_S) & HOSTPAGESIZEPF2_M) + +#define HOSTPAGESIZEPF1_S 4 +#define HOSTPAGESIZEPF1_M 0xfU +#define HOSTPAGESIZEPF1_V(x) ((x) << HOSTPAGESIZEPF1_S) +#define HOSTPAGESIZEPF1_G(x) (((x) >> HOSTPAGESIZEPF1_S) & HOSTPAGESIZEPF1_M) + +#define HOSTPAGESIZEPF0_S 0 +#define HOSTPAGESIZEPF0_M 0xfU +#define HOSTPAGESIZEPF0_V(x) ((x) << HOSTPAGESIZEPF0_S) +#define HOSTPAGESIZEPF0_G(x) (((x) >> HOSTPAGESIZEPF0_S) & HOSTPAGESIZEPF0_M) + +#define SGE_EGRESS_QUEUES_PER_PAGE_PF_A 0x1010 +#define SGE_EGRESS_QUEUES_PER_PAGE_VF_A 0x1014 + +#define QUEUESPERPAGEPF1_S 4 + +#define QUEUESPERPAGEPF0_S 0 +#define QUEUESPERPAGEPF0_M 0xfU +#define QUEUESPERPAGEPF0_V(x) ((x) << QUEUESPERPAGEPF0_S) +#define QUEUESPERPAGEPF0_G(x) (((x) >> QUEUESPERPAGEPF0_S) & QUEUESPERPAGEPF0_M) + +#define SGE_INT_CAUSE1_A 0x1024 +#define SGE_INT_CAUSE2_A 0x1030 +#define SGE_INT_CAUSE3_A 0x103c + +#define ERR_FLM_DBP_S 31 +#define ERR_FLM_DBP_V(x) ((x) << ERR_FLM_DBP_S) +#define ERR_FLM_DBP_F ERR_FLM_DBP_V(1U) + +#define ERR_FLM_IDMA1_S 30 +#define ERR_FLM_IDMA1_V(x) ((x) << ERR_FLM_IDMA1_S) +#define ERR_FLM_IDMA1_F ERR_FLM_IDMA1_V(1U) + +#define ERR_FLM_IDMA0_S 29 +#define ERR_FLM_IDMA0_V(x) ((x) << ERR_FLM_IDMA0_S) +#define ERR_FLM_IDMA0_F ERR_FLM_IDMA0_V(1U) + +#define ERR_FLM_HINT_S 28 +#define ERR_FLM_HINT_V(x) ((x) << ERR_FLM_HINT_S) +#define ERR_FLM_HINT_F ERR_FLM_HINT_V(1U) + +#define ERR_PCIE_ERROR3_S 27 +#define ERR_PCIE_ERROR3_V(x) ((x) << ERR_PCIE_ERROR3_S) +#define ERR_PCIE_ERROR3_F ERR_PCIE_ERROR3_V(1U) + +#define ERR_PCIE_ERROR2_S 26 +#define ERR_PCIE_ERROR2_V(x) ((x) << ERR_PCIE_ERROR2_S) +#define ERR_PCIE_ERROR2_F ERR_PCIE_ERROR2_V(1U) + +#define ERR_PCIE_ERROR1_S 25 +#define ERR_PCIE_ERROR1_V(x) ((x) << ERR_PCIE_ERROR1_S) +#define ERR_PCIE_ERROR1_F ERR_PCIE_ERROR1_V(1U) + +#define ERR_PCIE_ERROR0_S 24 +#define ERR_PCIE_ERROR0_V(x) ((x) << ERR_PCIE_ERROR0_S) +#define ERR_PCIE_ERROR0_F ERR_PCIE_ERROR0_V(1U) + +#define ERR_CPL_EXCEED_IQE_SIZE_S 22 +#define ERR_CPL_EXCEED_IQE_SIZE_V(x) ((x) << ERR_CPL_EXCEED_IQE_SIZE_S) +#define ERR_CPL_EXCEED_IQE_SIZE_F ERR_CPL_EXCEED_IQE_SIZE_V(1U) + +#define ERR_INVALID_CIDX_INC_S 21 +#define ERR_INVALID_CIDX_INC_V(x) ((x) << ERR_INVALID_CIDX_INC_S) +#define ERR_INVALID_CIDX_INC_F ERR_INVALID_CIDX_INC_V(1U) + +#define ERR_CPL_OPCODE_0_S 19 +#define ERR_CPL_OPCODE_0_V(x) ((x) << ERR_CPL_OPCODE_0_S) +#define ERR_CPL_OPCODE_0_F ERR_CPL_OPCODE_0_V(1U) + +#define ERR_DROPPED_DB_S 18 +#define ERR_DROPPED_DB_V(x) ((x) << ERR_DROPPED_DB_S) +#define ERR_DROPPED_DB_F ERR_DROPPED_DB_V(1U) + +#define ERR_DATA_CPL_ON_HIGH_QID1_S 17 +#define ERR_DATA_CPL_ON_HIGH_QID1_V(x) ((x) << ERR_DATA_CPL_ON_HIGH_QID1_S) +#define ERR_DATA_CPL_ON_HIGH_QID1_F ERR_DATA_CPL_ON_HIGH_QID1_V(1U) + +#define ERR_DATA_CPL_ON_HIGH_QID0_S 16 +#define ERR_DATA_CPL_ON_HIGH_QID0_V(x) ((x) << ERR_DATA_CPL_ON_HIGH_QID0_S) +#define ERR_DATA_CPL_ON_HIGH_QID0_F ERR_DATA_CPL_ON_HIGH_QID0_V(1U) + +#define ERR_BAD_DB_PIDX3_S 15 +#define ERR_BAD_DB_PIDX3_V(x) ((x) << ERR_BAD_DB_PIDX3_S) +#define ERR_BAD_DB_PIDX3_F ERR_BAD_DB_PIDX3_V(1U) + +#define ERR_BAD_DB_PIDX2_S 14 +#define ERR_BAD_DB_PIDX2_V(x) ((x) << ERR_BAD_DB_PIDX2_S) +#define ERR_BAD_DB_PIDX2_F ERR_BAD_DB_PIDX2_V(1U) + +#define ERR_BAD_DB_PIDX1_S 13 +#define ERR_BAD_DB_PIDX1_V(x) ((x) << ERR_BAD_DB_PIDX1_S) +#define ERR_BAD_DB_PIDX1_F ERR_BAD_DB_PIDX1_V(1U) + +#define ERR_BAD_DB_PIDX0_S 12 +#define ERR_BAD_DB_PIDX0_V(x) ((x) << ERR_BAD_DB_PIDX0_S) +#define ERR_BAD_DB_PIDX0_F ERR_BAD_DB_PIDX0_V(1U) + +#define ERR_ING_CTXT_PRIO_S 10 +#define ERR_ING_CTXT_PRIO_V(x) ((x) << ERR_ING_CTXT_PRIO_S) +#define ERR_ING_CTXT_PRIO_F ERR_ING_CTXT_PRIO_V(1U) + +#define ERR_EGR_CTXT_PRIO_S 9 +#define ERR_EGR_CTXT_PRIO_V(x) ((x) << ERR_EGR_CTXT_PRIO_S) +#define ERR_EGR_CTXT_PRIO_F ERR_EGR_CTXT_PRIO_V(1U) + +#define DBFIFO_HP_INT_S 8 +#define DBFIFO_HP_INT_V(x) ((x) << DBFIFO_HP_INT_S) +#define DBFIFO_HP_INT_F DBFIFO_HP_INT_V(1U) + +#define DBFIFO_LP_INT_S 7 +#define DBFIFO_LP_INT_V(x) ((x) << DBFIFO_LP_INT_S) +#define DBFIFO_LP_INT_F DBFIFO_LP_INT_V(1U) + +#define INGRESS_SIZE_ERR_S 5 +#define INGRESS_SIZE_ERR_V(x) ((x) << INGRESS_SIZE_ERR_S) +#define INGRESS_SIZE_ERR_F INGRESS_SIZE_ERR_V(1U) + +#define EGRESS_SIZE_ERR_S 4 +#define EGRESS_SIZE_ERR_V(x) ((x) << EGRESS_SIZE_ERR_S) +#define EGRESS_SIZE_ERR_F EGRESS_SIZE_ERR_V(1U) + +#define SGE_INT_ENABLE3_A 0x1040 +#define SGE_FL_BUFFER_SIZE0_A 0x1044 +#define SGE_FL_BUFFER_SIZE1_A 0x1048 +#define SGE_FL_BUFFER_SIZE2_A 0x104c +#define SGE_FL_BUFFER_SIZE3_A 0x1050 +#define SGE_FL_BUFFER_SIZE4_A 0x1054 +#define SGE_FL_BUFFER_SIZE5_A 0x1058 +#define SGE_FL_BUFFER_SIZE6_A 0x105c +#define SGE_FL_BUFFER_SIZE7_A 0x1060 +#define SGE_FL_BUFFER_SIZE8_A 0x1064 + +#define SGE_IMSG_CTXT_BADDR_A 0x1088 +#define SGE_FLM_CACHE_BADDR_A 0x108c +#define SGE_FLM_CFG_A 0x1090 + +#define NOHDR_S 18 +#define NOHDR_V(x) ((x) << NOHDR_S) +#define NOHDR_F NOHDR_V(1U) + +#define HDRSTARTFLQ_S 11 +#define HDRSTARTFLQ_M 0x7U +#define HDRSTARTFLQ_G(x) (((x) >> HDRSTARTFLQ_S) & HDRSTARTFLQ_M) + +#define SGE_INGRESS_RX_THRESHOLD_A 0x10a0 + +#define THRESHOLD_0_S 24 +#define THRESHOLD_0_M 0x3fU +#define THRESHOLD_0_V(x) ((x) << THRESHOLD_0_S) +#define THRESHOLD_0_G(x) (((x) >> THRESHOLD_0_S) & THRESHOLD_0_M) + +#define THRESHOLD_1_S 16 +#define THRESHOLD_1_M 0x3fU +#define THRESHOLD_1_V(x) ((x) << THRESHOLD_1_S) +#define THRESHOLD_1_G(x) (((x) >> THRESHOLD_1_S) & THRESHOLD_1_M) + +#define THRESHOLD_2_S 8 +#define THRESHOLD_2_M 0x3fU +#define THRESHOLD_2_V(x) ((x) << THRESHOLD_2_S) +#define THRESHOLD_2_G(x) (((x) >> THRESHOLD_2_S) & THRESHOLD_2_M) + +#define THRESHOLD_3_S 0 +#define THRESHOLD_3_M 0x3fU +#define THRESHOLD_3_V(x) ((x) << THRESHOLD_3_S) +#define THRESHOLD_3_G(x) (((x) >> THRESHOLD_3_S) & THRESHOLD_3_M) + +#define SGE_CONM_CTRL_A 0x1094 + +#define EGRTHRESHOLD_S 8 +#define EGRTHRESHOLD_M 0x3fU +#define EGRTHRESHOLD_V(x) ((x) << EGRTHRESHOLD_S) +#define EGRTHRESHOLD_G(x) (((x) >> EGRTHRESHOLD_S) & EGRTHRESHOLD_M) + +#define EGRTHRESHOLDPACKING_S 14 +#define EGRTHRESHOLDPACKING_M 0x3fU +#define EGRTHRESHOLDPACKING_V(x) ((x) << EGRTHRESHOLDPACKING_S) +#define EGRTHRESHOLDPACKING_G(x) \ + (((x) >> EGRTHRESHOLDPACKING_S) & EGRTHRESHOLDPACKING_M) + +#define T6_EGRTHRESHOLDPACKING_S 16 +#define T6_EGRTHRESHOLDPACKING_M 0xffU +#define T6_EGRTHRESHOLDPACKING_G(x) \ + (((x) >> T6_EGRTHRESHOLDPACKING_S) & T6_EGRTHRESHOLDPACKING_M) + +#define SGE_TIMESTAMP_LO_A 0x1098 +#define SGE_TIMESTAMP_HI_A 0x109c + +#define TSOP_S 28 +#define TSOP_M 0x3U +#define TSOP_V(x) ((x) << TSOP_S) +#define TSOP_G(x) (((x) >> TSOP_S) & TSOP_M) + +#define TSVAL_S 0 +#define TSVAL_M 0xfffffffU +#define TSVAL_V(x) ((x) << TSVAL_S) +#define TSVAL_G(x) (((x) >> TSVAL_S) & TSVAL_M) + +#define SGE_DBFIFO_STATUS_A 0x10a4 +#define SGE_DBVFIFO_SIZE_A 0x113c + +#define HP_INT_THRESH_S 28 +#define HP_INT_THRESH_M 0xfU +#define HP_INT_THRESH_V(x) ((x) << HP_INT_THRESH_S) + +#define LP_INT_THRESH_S 12 +#define LP_INT_THRESH_M 0xfU +#define LP_INT_THRESH_V(x) ((x) << LP_INT_THRESH_S) + +#define SGE_DOORBELL_CONTROL_A 0x10a8 + +#define NOCOALESCE_S 26 +#define NOCOALESCE_V(x) ((x) << NOCOALESCE_S) +#define NOCOALESCE_F NOCOALESCE_V(1U) + +#define ENABLE_DROP_S 13 +#define ENABLE_DROP_V(x) ((x) << ENABLE_DROP_S) +#define ENABLE_DROP_F ENABLE_DROP_V(1U) + +#define SGE_TIMER_VALUE_0_AND_1_A 0x10b8 + +#define TIMERVALUE0_S 16 +#define TIMERVALUE0_M 0xffffU +#define TIMERVALUE0_V(x) ((x) << TIMERVALUE0_S) +#define TIMERVALUE0_G(x) (((x) >> TIMERVALUE0_S) & TIMERVALUE0_M) + +#define TIMERVALUE1_S 0 +#define TIMERVALUE1_M 0xffffU +#define TIMERVALUE1_V(x) ((x) << TIMERVALUE1_S) +#define TIMERVALUE1_G(x) (((x) >> TIMERVALUE1_S) & TIMERVALUE1_M) + +#define SGE_TIMER_VALUE_2_AND_3_A 0x10bc + +#define TIMERVALUE2_S 16 +#define TIMERVALUE2_M 0xffffU +#define TIMERVALUE2_V(x) ((x) << TIMERVALUE2_S) +#define TIMERVALUE2_G(x) (((x) >> TIMERVALUE2_S) & TIMERVALUE2_M) + +#define TIMERVALUE3_S 0 +#define TIMERVALUE3_M 0xffffU +#define TIMERVALUE3_V(x) ((x) << TIMERVALUE3_S) +#define TIMERVALUE3_G(x) (((x) >> TIMERVALUE3_S) & TIMERVALUE3_M) + +#define SGE_TIMER_VALUE_4_AND_5_A 0x10c0 + +#define TIMERVALUE4_S 16 +#define TIMERVALUE4_M 0xffffU +#define TIMERVALUE4_V(x) ((x) << TIMERVALUE4_S) +#define TIMERVALUE4_G(x) (((x) >> TIMERVALUE4_S) & TIMERVALUE4_M) + +#define TIMERVALUE5_S 0 +#define TIMERVALUE5_M 0xffffU +#define TIMERVALUE5_V(x) ((x) << TIMERVALUE5_S) +#define TIMERVALUE5_G(x) (((x) >> TIMERVALUE5_S) & TIMERVALUE5_M) + +#define SGE_DEBUG_INDEX_A 0x10cc +#define SGE_DEBUG_DATA_HIGH_A 0x10d0 +#define SGE_DEBUG_DATA_LOW_A 0x10d4 + +#define SGE_DEBUG_DATA_LOW_INDEX_2_A 0x12c8 +#define SGE_DEBUG_DATA_LOW_INDEX_3_A 0x12cc +#define SGE_DEBUG_DATA_HIGH_INDEX_10_A 0x12a8 + +#define SGE_INGRESS_QUEUES_PER_PAGE_PF_A 0x10f4 +#define SGE_INGRESS_QUEUES_PER_PAGE_VF_A 0x10f8 + +#define SGE_ERROR_STATS_A 0x1100 + +#define UNCAPTURED_ERROR_S 18 +#define UNCAPTURED_ERROR_V(x) ((x) << UNCAPTURED_ERROR_S) +#define UNCAPTURED_ERROR_F UNCAPTURED_ERROR_V(1U) + +#define ERROR_QID_VALID_S 17 +#define ERROR_QID_VALID_V(x) ((x) << ERROR_QID_VALID_S) +#define ERROR_QID_VALID_F ERROR_QID_VALID_V(1U) + +#define ERROR_QID_S 0 +#define ERROR_QID_M 0x1ffffU +#define ERROR_QID_G(x) (((x) >> ERROR_QID_S) & ERROR_QID_M) + +#define SGE_INT_CAUSE5_A 0x110c + +#define ERR_T_RXCRC_S 31 +#define ERR_T_RXCRC_V(x) ((x) << ERR_T_RXCRC_S) +#define ERR_T_RXCRC_F ERR_T_RXCRC_V(1U) + +#define HP_INT_THRESH_S 28 +#define HP_INT_THRESH_M 0xfU +#define HP_INT_THRESH_V(x) ((x) << HP_INT_THRESH_S) + +#define HP_COUNT_S 16 +#define HP_COUNT_M 0x7ffU +#define HP_COUNT_G(x) (((x) >> HP_COUNT_S) & HP_COUNT_M) + +#define LP_INT_THRESH_S 12 +#define LP_INT_THRESH_M 0xfU +#define LP_INT_THRESH_V(x) ((x) << LP_INT_THRESH_S) + +#define LP_COUNT_S 0 +#define LP_COUNT_M 0x7ffU +#define LP_COUNT_G(x) (((x) >> LP_COUNT_S) & LP_COUNT_M) + +#define LP_INT_THRESH_T5_S 18 +#define LP_INT_THRESH_T5_M 0xfffU +#define LP_INT_THRESH_T5_V(x) ((x) << LP_INT_THRESH_T5_S) + +#define LP_COUNT_T5_S 0 +#define LP_COUNT_T5_M 0x3ffffU +#define LP_COUNT_T5_G(x) (((x) >> LP_COUNT_T5_S) & LP_COUNT_T5_M) + +#define SGE_DOORBELL_CONTROL_A 0x10a8 + +#define SGE_STAT_TOTAL_A 0x10e4 +#define SGE_STAT_MATCH_A 0x10e8 +#define SGE_STAT_CFG_A 0x10ec + +#define STATMODE_S 2 +#define STATMODE_V(x) ((x) << STATMODE_S) + +#define STATSOURCE_T5_S 9 +#define STATSOURCE_T5_M 0xfU +#define STATSOURCE_T5_V(x) ((x) << STATSOURCE_T5_S) +#define STATSOURCE_T5_G(x) (((x) >> STATSOURCE_T5_S) & STATSOURCE_T5_M) + +#define T6_STATMODE_S 0 +#define T6_STATMODE_V(x) ((x) << T6_STATMODE_S) + +#define SGE_DBFIFO_STATUS2_A 0x1118 + +#define HP_INT_THRESH_T5_S 10 +#define HP_INT_THRESH_T5_M 0xfU +#define HP_INT_THRESH_T5_V(x) ((x) << HP_INT_THRESH_T5_S) + +#define HP_COUNT_T5_S 0 +#define HP_COUNT_T5_M 0x3ffU +#define HP_COUNT_T5_G(x) (((x) >> HP_COUNT_T5_S) & HP_COUNT_T5_M) + +#define ENABLE_DROP_S 13 +#define ENABLE_DROP_V(x) ((x) << ENABLE_DROP_S) +#define ENABLE_DROP_F ENABLE_DROP_V(1U) + +#define DROPPED_DB_S 0 +#define DROPPED_DB_V(x) ((x) << DROPPED_DB_S) +#define DROPPED_DB_F DROPPED_DB_V(1U) + +#define SGE_CTXT_CMD_A 0x11fc +#define SGE_DBQ_CTXT_BADDR_A 0x1084 + +/* registers for module PCIE */ +#define PCIE_PF_CFG_A 0x40 + +#define AIVEC_S 4 +#define AIVEC_M 0x3ffU +#define AIVEC_V(x) ((x) << AIVEC_S) + +#define PCIE_PF_CLI_A 0x44 + +#define PCIE_PF_EXPROM_OFST_A 0x4c +#define OFFSET_S 10 +#define OFFSET_M 0x3fffU +#define OFFSET_G(x) (((x) >> OFFSET_S) & OFFSET_M) + +#define PCIE_INT_CAUSE_A 0x3004 + +#define UNXSPLCPLERR_S 29 +#define UNXSPLCPLERR_V(x) ((x) << UNXSPLCPLERR_S) +#define UNXSPLCPLERR_F UNXSPLCPLERR_V(1U) + +#define PCIEPINT_S 28 +#define PCIEPINT_V(x) ((x) << PCIEPINT_S) +#define PCIEPINT_F PCIEPINT_V(1U) + +#define PCIESINT_S 27 +#define PCIESINT_V(x) ((x) << PCIESINT_S) +#define PCIESINT_F PCIESINT_V(1U) + +#define RPLPERR_S 26 +#define RPLPERR_V(x) ((x) << RPLPERR_S) +#define RPLPERR_F RPLPERR_V(1U) + +#define RXWRPERR_S 25 +#define RXWRPERR_V(x) ((x) << RXWRPERR_S) +#define RXWRPERR_F RXWRPERR_V(1U) + +#define RXCPLPERR_S 24 +#define RXCPLPERR_V(x) ((x) << RXCPLPERR_S) +#define RXCPLPERR_F RXCPLPERR_V(1U) + +#define PIOTAGPERR_S 23 +#define PIOTAGPERR_V(x) ((x) << PIOTAGPERR_S) +#define PIOTAGPERR_F PIOTAGPERR_V(1U) + +#define MATAGPERR_S 22 +#define MATAGPERR_V(x) ((x) << MATAGPERR_S) +#define MATAGPERR_F MATAGPERR_V(1U) + +#define INTXCLRPERR_S 21 +#define INTXCLRPERR_V(x) ((x) << INTXCLRPERR_S) +#define INTXCLRPERR_F INTXCLRPERR_V(1U) + +#define FIDPERR_S 20 +#define FIDPERR_V(x) ((x) << FIDPERR_S) +#define FIDPERR_F FIDPERR_V(1U) + +#define CFGSNPPERR_S 19 +#define CFGSNPPERR_V(x) ((x) << CFGSNPPERR_S) +#define CFGSNPPERR_F CFGSNPPERR_V(1U) + +#define HRSPPERR_S 18 +#define HRSPPERR_V(x) ((x) << HRSPPERR_S) +#define HRSPPERR_F HRSPPERR_V(1U) + +#define HREQPERR_S 17 +#define HREQPERR_V(x) ((x) << HREQPERR_S) +#define HREQPERR_F HREQPERR_V(1U) + +#define HCNTPERR_S 16 +#define HCNTPERR_V(x) ((x) << HCNTPERR_S) +#define HCNTPERR_F HCNTPERR_V(1U) + +#define DRSPPERR_S 15 +#define DRSPPERR_V(x) ((x) << DRSPPERR_S) +#define DRSPPERR_F DRSPPERR_V(1U) + +#define DREQPERR_S 14 +#define DREQPERR_V(x) ((x) << DREQPERR_S) +#define DREQPERR_F DREQPERR_V(1U) + +#define DCNTPERR_S 13 +#define DCNTPERR_V(x) ((x) << DCNTPERR_S) +#define DCNTPERR_F DCNTPERR_V(1U) + +#define CRSPPERR_S 12 +#define CRSPPERR_V(x) ((x) << CRSPPERR_S) +#define CRSPPERR_F CRSPPERR_V(1U) + +#define CREQPERR_S 11 +#define CREQPERR_V(x) ((x) << CREQPERR_S) +#define CREQPERR_F CREQPERR_V(1U) + +#define CCNTPERR_S 10 +#define CCNTPERR_V(x) ((x) << CCNTPERR_S) +#define CCNTPERR_F CCNTPERR_V(1U) + +#define TARTAGPERR_S 9 +#define TARTAGPERR_V(x) ((x) << TARTAGPERR_S) +#define TARTAGPERR_F TARTAGPERR_V(1U) + +#define PIOREQPERR_S 8 +#define PIOREQPERR_V(x) ((x) << PIOREQPERR_S) +#define PIOREQPERR_F PIOREQPERR_V(1U) + +#define PIOCPLPERR_S 7 +#define PIOCPLPERR_V(x) ((x) << PIOCPLPERR_S) +#define PIOCPLPERR_F PIOCPLPERR_V(1U) + +#define MSIXDIPERR_S 6 +#define MSIXDIPERR_V(x) ((x) << MSIXDIPERR_S) +#define MSIXDIPERR_F MSIXDIPERR_V(1U) + +#define MSIXDATAPERR_S 5 +#define MSIXDATAPERR_V(x) ((x) << MSIXDATAPERR_S) +#define MSIXDATAPERR_F MSIXDATAPERR_V(1U) + +#define MSIXADDRHPERR_S 4 +#define MSIXADDRHPERR_V(x) ((x) << MSIXADDRHPERR_S) +#define MSIXADDRHPERR_F MSIXADDRHPERR_V(1U) + +#define MSIXADDRLPERR_S 3 +#define MSIXADDRLPERR_V(x) ((x) << MSIXADDRLPERR_S) +#define MSIXADDRLPERR_F MSIXADDRLPERR_V(1U) + +#define MSIDATAPERR_S 2 +#define MSIDATAPERR_V(x) ((x) << MSIDATAPERR_S) +#define MSIDATAPERR_F MSIDATAPERR_V(1U) + +#define MSIADDRHPERR_S 1 +#define MSIADDRHPERR_V(x) ((x) << MSIADDRHPERR_S) +#define MSIADDRHPERR_F MSIADDRHPERR_V(1U) + +#define MSIADDRLPERR_S 0 +#define MSIADDRLPERR_V(x) ((x) << MSIADDRLPERR_S) +#define MSIADDRLPERR_F MSIADDRLPERR_V(1U) + +#define READRSPERR_S 29 +#define READRSPERR_V(x) ((x) << READRSPERR_S) +#define READRSPERR_F READRSPERR_V(1U) + +#define TRGT1GRPPERR_S 28 +#define TRGT1GRPPERR_V(x) ((x) << TRGT1GRPPERR_S) +#define TRGT1GRPPERR_F TRGT1GRPPERR_V(1U) + +#define IPSOTPERR_S 27 +#define IPSOTPERR_V(x) ((x) << IPSOTPERR_S) +#define IPSOTPERR_F IPSOTPERR_V(1U) + +#define IPRETRYPERR_S 26 +#define IPRETRYPERR_V(x) ((x) << IPRETRYPERR_S) +#define IPRETRYPERR_F IPRETRYPERR_V(1U) + +#define IPRXDATAGRPPERR_S 25 +#define IPRXDATAGRPPERR_V(x) ((x) << IPRXDATAGRPPERR_S) +#define IPRXDATAGRPPERR_F IPRXDATAGRPPERR_V(1U) + +#define IPRXHDRGRPPERR_S 24 +#define IPRXHDRGRPPERR_V(x) ((x) << IPRXHDRGRPPERR_S) +#define IPRXHDRGRPPERR_F IPRXHDRGRPPERR_V(1U) + +#define MAGRPPERR_S 22 +#define MAGRPPERR_V(x) ((x) << MAGRPPERR_S) +#define MAGRPPERR_F MAGRPPERR_V(1U) + +#define VFIDPERR_S 21 +#define VFIDPERR_V(x) ((x) << VFIDPERR_S) +#define VFIDPERR_F VFIDPERR_V(1U) + +#define HREQWRPERR_S 16 +#define HREQWRPERR_V(x) ((x) << HREQWRPERR_S) +#define HREQWRPERR_F HREQWRPERR_V(1U) + +#define DREQWRPERR_S 13 +#define DREQWRPERR_V(x) ((x) << DREQWRPERR_S) +#define DREQWRPERR_F DREQWRPERR_V(1U) + +#define CREQRDPERR_S 11 +#define CREQRDPERR_V(x) ((x) << CREQRDPERR_S) +#define CREQRDPERR_F CREQRDPERR_V(1U) + +#define MSTTAGQPERR_S 10 +#define MSTTAGQPERR_V(x) ((x) << MSTTAGQPERR_S) +#define MSTTAGQPERR_F MSTTAGQPERR_V(1U) + +#define PIOREQGRPPERR_S 8 +#define PIOREQGRPPERR_V(x) ((x) << PIOREQGRPPERR_S) +#define PIOREQGRPPERR_F PIOREQGRPPERR_V(1U) + +#define PIOCPLGRPPERR_S 7 +#define PIOCPLGRPPERR_V(x) ((x) << PIOCPLGRPPERR_S) +#define PIOCPLGRPPERR_F PIOCPLGRPPERR_V(1U) + +#define MSIXSTIPERR_S 2 +#define MSIXSTIPERR_V(x) ((x) << MSIXSTIPERR_S) +#define MSIXSTIPERR_F MSIXSTIPERR_V(1U) + +#define MSTTIMEOUTPERR_S 1 +#define MSTTIMEOUTPERR_V(x) ((x) << MSTTIMEOUTPERR_S) +#define MSTTIMEOUTPERR_F MSTTIMEOUTPERR_V(1U) + +#define MSTGRPPERR_S 0 +#define MSTGRPPERR_V(x) ((x) << MSTGRPPERR_S) +#define MSTGRPPERR_F MSTGRPPERR_V(1U) + +#define PCIE_NONFAT_ERR_A 0x3010 +#define PCIE_CFG_SPACE_REQ_A 0x3060 +#define PCIE_CFG_SPACE_DATA_A 0x3064 +#define PCIE_MEM_ACCESS_BASE_WIN_A 0x3068 + +#define PCIEOFST_S 10 +#define PCIEOFST_M 0x3fffffU +#define PCIEOFST_G(x) (((x) >> PCIEOFST_S) & PCIEOFST_M) + +#define BIR_S 8 +#define BIR_M 0x3U +#define BIR_V(x) ((x) << BIR_S) +#define BIR_G(x) (((x) >> BIR_S) & BIR_M) + +#define WINDOW_S 0 +#define WINDOW_M 0xffU +#define WINDOW_V(x) ((x) << WINDOW_S) +#define WINDOW_G(x) (((x) >> WINDOW_S) & WINDOW_M) + +#define PCIE_MEM_ACCESS_OFFSET_A 0x306c + +#define ENABLE_S 30 +#define ENABLE_V(x) ((x) << ENABLE_S) +#define ENABLE_F ENABLE_V(1U) + +#define LOCALCFG_S 28 +#define LOCALCFG_V(x) ((x) << LOCALCFG_S) +#define LOCALCFG_F LOCALCFG_V(1U) + +#define FUNCTION_S 12 +#define FUNCTION_V(x) ((x) << FUNCTION_S) + +#define REGISTER_S 0 +#define REGISTER_V(x) ((x) << REGISTER_S) + +#define T6_ENABLE_S 31 +#define T6_ENABLE_V(x) ((x) << T6_ENABLE_S) +#define T6_ENABLE_F T6_ENABLE_V(1U) + +#define PFNUM_S 0 +#define PFNUM_V(x) ((x) << PFNUM_S) + +#define PCIE_FW_A 0x30b8 +#define PCIE_FW_PF_A 0x30bc + +#define PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS_A 0x5908 + +#define RNPP_S 31 +#define RNPP_V(x) ((x) << RNPP_S) +#define RNPP_F RNPP_V(1U) + +#define RPCP_S 29 +#define RPCP_V(x) ((x) << RPCP_S) +#define RPCP_F RPCP_V(1U) + +#define RCIP_S 27 +#define RCIP_V(x) ((x) << RCIP_S) +#define RCIP_F RCIP_V(1U) + +#define RCCP_S 26 +#define RCCP_V(x) ((x) << RCCP_S) +#define RCCP_F RCCP_V(1U) + +#define RFTP_S 23 +#define RFTP_V(x) ((x) << RFTP_S) +#define RFTP_F RFTP_V(1U) + +#define PTRP_S 20 +#define PTRP_V(x) ((x) << PTRP_S) +#define PTRP_F PTRP_V(1U) + +#define PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS_A 0x59a4 + +#define TPCP_S 30 +#define TPCP_V(x) ((x) << TPCP_S) +#define TPCP_F TPCP_V(1U) + +#define TNPP_S 29 +#define TNPP_V(x) ((x) << TNPP_S) +#define TNPP_F TNPP_V(1U) + +#define TFTP_S 28 +#define TFTP_V(x) ((x) << TFTP_S) +#define TFTP_F TFTP_V(1U) + +#define TCAP_S 27 +#define TCAP_V(x) ((x) << TCAP_S) +#define TCAP_F TCAP_V(1U) + +#define TCIP_S 26 +#define TCIP_V(x) ((x) << TCIP_S) +#define TCIP_F TCIP_V(1U) + +#define RCAP_S 25 +#define RCAP_V(x) ((x) << RCAP_S) +#define RCAP_F RCAP_V(1U) + +#define PLUP_S 23 +#define PLUP_V(x) ((x) << PLUP_S) +#define PLUP_F PLUP_V(1U) + +#define PLDN_S 22 +#define PLDN_V(x) ((x) << PLDN_S) +#define PLDN_F PLDN_V(1U) + +#define OTDD_S 21 +#define OTDD_V(x) ((x) << OTDD_S) +#define OTDD_F OTDD_V(1U) + +#define GTRP_S 20 +#define GTRP_V(x) ((x) << GTRP_S) +#define GTRP_F GTRP_V(1U) + +#define RDPE_S 18 +#define RDPE_V(x) ((x) << RDPE_S) +#define RDPE_F RDPE_V(1U) + +#define TDCE_S 17 +#define TDCE_V(x) ((x) << TDCE_S) +#define TDCE_F TDCE_V(1U) + +#define TDUE_S 16 +#define TDUE_V(x) ((x) << TDUE_S) +#define TDUE_F TDUE_V(1U) + +/* SPARE2 register contains 32-bit value at offset 0x6 in Serial INIT + * Configuration flashed on EEPROM. This value corresponds to 32-bit + * Serial Configuration Version information. + */ +#define PCIE_STATIC_SPARE2_A 0x5bfc + +/* registers for module MC */ +#define MC_INT_CAUSE_A 0x7518 +#define MC_P_INT_CAUSE_A 0x41318 + +#define ECC_UE_INT_CAUSE_S 2 +#define ECC_UE_INT_CAUSE_V(x) ((x) << ECC_UE_INT_CAUSE_S) +#define ECC_UE_INT_CAUSE_F ECC_UE_INT_CAUSE_V(1U) + +#define ECC_CE_INT_CAUSE_S 1 +#define ECC_CE_INT_CAUSE_V(x) ((x) << ECC_CE_INT_CAUSE_S) +#define ECC_CE_INT_CAUSE_F ECC_CE_INT_CAUSE_V(1U) + +#define PERR_INT_CAUSE_S 0 +#define PERR_INT_CAUSE_V(x) ((x) << PERR_INT_CAUSE_S) +#define PERR_INT_CAUSE_F PERR_INT_CAUSE_V(1U) + +#define DBG_GPIO_EN_A 0x6010 +#define XGMAC_PORT_CFG_A 0x1000 +#define MAC_PORT_CFG_A 0x800 + +#define SIGNAL_DET_S 14 +#define SIGNAL_DET_V(x) ((x) << SIGNAL_DET_S) +#define SIGNAL_DET_F SIGNAL_DET_V(1U) + +#define MC_ECC_STATUS_A 0x751c +#define MC_P_ECC_STATUS_A 0x4131c + +#define ECC_CECNT_S 16 +#define ECC_CECNT_M 0xffffU +#define ECC_CECNT_V(x) ((x) << ECC_CECNT_S) +#define ECC_CECNT_G(x) (((x) >> ECC_CECNT_S) & ECC_CECNT_M) + +#define ECC_UECNT_S 0 +#define ECC_UECNT_M 0xffffU +#define ECC_UECNT_V(x) ((x) << ECC_UECNT_S) +#define ECC_UECNT_G(x) (((x) >> ECC_UECNT_S) & ECC_UECNT_M) + +#define MC_BIST_CMD_A 0x7600 + +#define START_BIST_S 31 +#define START_BIST_V(x) ((x) << START_BIST_S) +#define START_BIST_F START_BIST_V(1U) + +#define BIST_CMD_GAP_S 8 +#define BIST_CMD_GAP_V(x) ((x) << BIST_CMD_GAP_S) + +#define BIST_OPCODE_S 0 +#define BIST_OPCODE_V(x) ((x) << BIST_OPCODE_S) + +#define MC_BIST_CMD_ADDR_A 0x7604 +#define MC_BIST_CMD_LEN_A 0x7608 +#define MC_BIST_DATA_PATTERN_A 0x760c + +#define MC_BIST_STATUS_RDATA_A 0x7688 + +/* registers for module MA */ +#define MA_EDRAM0_BAR_A 0x77c0 + +#define EDRAM0_BASE_S 16 +#define EDRAM0_BASE_M 0xfffU +#define EDRAM0_BASE_G(x) (((x) >> EDRAM0_BASE_S) & EDRAM0_BASE_M) + +#define EDRAM0_SIZE_S 0 +#define EDRAM0_SIZE_M 0xfffU +#define EDRAM0_SIZE_V(x) ((x) << EDRAM0_SIZE_S) +#define EDRAM0_SIZE_G(x) (((x) >> EDRAM0_SIZE_S) & EDRAM0_SIZE_M) + +#define MA_EDRAM1_BAR_A 0x77c4 + +#define EDRAM1_BASE_S 16 +#define EDRAM1_BASE_M 0xfffU +#define EDRAM1_BASE_G(x) (((x) >> EDRAM1_BASE_S) & EDRAM1_BASE_M) + +#define EDRAM1_SIZE_S 0 +#define EDRAM1_SIZE_M 0xfffU +#define EDRAM1_SIZE_V(x) ((x) << EDRAM1_SIZE_S) +#define EDRAM1_SIZE_G(x) (((x) >> EDRAM1_SIZE_S) & EDRAM1_SIZE_M) + +#define MA_EXT_MEMORY_BAR_A 0x77c8 + +#define EXT_MEM_BASE_S 16 +#define EXT_MEM_BASE_M 0xfffU +#define EXT_MEM_BASE_V(x) ((x) << EXT_MEM_BASE_S) +#define EXT_MEM_BASE_G(x) (((x) >> EXT_MEM_BASE_S) & EXT_MEM_BASE_M) + +#define EXT_MEM_SIZE_S 0 +#define EXT_MEM_SIZE_M 0xfffU +#define EXT_MEM_SIZE_V(x) ((x) << EXT_MEM_SIZE_S) +#define EXT_MEM_SIZE_G(x) (((x) >> EXT_MEM_SIZE_S) & EXT_MEM_SIZE_M) + +#define MA_EXT_MEMORY1_BAR_A 0x7808 + +#define HMA_MUX_S 5 +#define HMA_MUX_V(x) ((x) << HMA_MUX_S) +#define HMA_MUX_F HMA_MUX_V(1U) + +#define EXT_MEM1_BASE_S 16 +#define EXT_MEM1_BASE_M 0xfffU +#define EXT_MEM1_BASE_G(x) (((x) >> EXT_MEM1_BASE_S) & EXT_MEM1_BASE_M) + +#define EXT_MEM1_SIZE_S 0 +#define EXT_MEM1_SIZE_M 0xfffU +#define EXT_MEM1_SIZE_V(x) ((x) << EXT_MEM1_SIZE_S) +#define EXT_MEM1_SIZE_G(x) (((x) >> EXT_MEM1_SIZE_S) & EXT_MEM1_SIZE_M) + +#define MA_EXT_MEMORY0_BAR_A 0x77c8 + +#define EXT_MEM0_BASE_S 16 +#define EXT_MEM0_BASE_M 0xfffU +#define EXT_MEM0_BASE_G(x) (((x) >> EXT_MEM0_BASE_S) & EXT_MEM0_BASE_M) + +#define EXT_MEM0_SIZE_S 0 +#define EXT_MEM0_SIZE_M 0xfffU +#define EXT_MEM0_SIZE_V(x) ((x) << EXT_MEM0_SIZE_S) +#define EXT_MEM0_SIZE_G(x) (((x) >> EXT_MEM0_SIZE_S) & EXT_MEM0_SIZE_M) + +#define MA_TARGET_MEM_ENABLE_A 0x77d8 + +#define EXT_MEM_ENABLE_S 2 +#define EXT_MEM_ENABLE_V(x) ((x) << EXT_MEM_ENABLE_S) +#define EXT_MEM_ENABLE_F EXT_MEM_ENABLE_V(1U) + +#define EDRAM1_ENABLE_S 1 +#define EDRAM1_ENABLE_V(x) ((x) << EDRAM1_ENABLE_S) +#define EDRAM1_ENABLE_F EDRAM1_ENABLE_V(1U) + +#define EDRAM0_ENABLE_S 0 +#define EDRAM0_ENABLE_V(x) ((x) << EDRAM0_ENABLE_S) +#define EDRAM0_ENABLE_F EDRAM0_ENABLE_V(1U) + +#define EXT_MEM1_ENABLE_S 4 +#define EXT_MEM1_ENABLE_V(x) ((x) << EXT_MEM1_ENABLE_S) +#define EXT_MEM1_ENABLE_F EXT_MEM1_ENABLE_V(1U) + +#define EXT_MEM0_ENABLE_S 2 +#define EXT_MEM0_ENABLE_V(x) ((x) << EXT_MEM0_ENABLE_S) +#define EXT_MEM0_ENABLE_F EXT_MEM0_ENABLE_V(1U) + +#define MA_INT_CAUSE_A 0x77e0 + +#define MEM_PERR_INT_CAUSE_S 1 +#define MEM_PERR_INT_CAUSE_V(x) ((x) << MEM_PERR_INT_CAUSE_S) +#define MEM_PERR_INT_CAUSE_F MEM_PERR_INT_CAUSE_V(1U) + +#define MEM_WRAP_INT_CAUSE_S 0 +#define MEM_WRAP_INT_CAUSE_V(x) ((x) << MEM_WRAP_INT_CAUSE_S) +#define MEM_WRAP_INT_CAUSE_F MEM_WRAP_INT_CAUSE_V(1U) + +#define MA_INT_WRAP_STATUS_A 0x77e4 + +#define MEM_WRAP_ADDRESS_S 4 +#define MEM_WRAP_ADDRESS_M 0xfffffffU +#define MEM_WRAP_ADDRESS_G(x) (((x) >> MEM_WRAP_ADDRESS_S) & MEM_WRAP_ADDRESS_M) + +#define MEM_WRAP_CLIENT_NUM_S 0 +#define MEM_WRAP_CLIENT_NUM_M 0xfU +#define MEM_WRAP_CLIENT_NUM_G(x) \ + (((x) >> MEM_WRAP_CLIENT_NUM_S) & MEM_WRAP_CLIENT_NUM_M) + +#define MA_PARITY_ERROR_STATUS_A 0x77f4 +#define MA_PARITY_ERROR_STATUS1_A 0x77f4 +#define MA_PARITY_ERROR_STATUS2_A 0x7804 + +/* registers for module EDC_0 */ +#define EDC_0_BASE_ADDR 0x7900 + +#define EDC_BIST_CMD_A 0x7904 +#define EDC_BIST_CMD_ADDR_A 0x7908 +#define EDC_BIST_CMD_LEN_A 0x790c +#define EDC_BIST_DATA_PATTERN_A 0x7910 +#define EDC_BIST_STATUS_RDATA_A 0x7928 +#define EDC_INT_CAUSE_A 0x7978 + +#define ECC_UE_PAR_S 5 +#define ECC_UE_PAR_V(x) ((x) << ECC_UE_PAR_S) +#define ECC_UE_PAR_F ECC_UE_PAR_V(1U) + +#define ECC_CE_PAR_S 4 +#define ECC_CE_PAR_V(x) ((x) << ECC_CE_PAR_S) +#define ECC_CE_PAR_F ECC_CE_PAR_V(1U) + +#define PERR_PAR_CAUSE_S 3 +#define PERR_PAR_CAUSE_V(x) ((x) << PERR_PAR_CAUSE_S) +#define PERR_PAR_CAUSE_F PERR_PAR_CAUSE_V(1U) + +#define EDC_ECC_STATUS_A 0x797c + +/* registers for module EDC_1 */ +#define EDC_1_BASE_ADDR 0x7980 + +/* registers for module CIM */ +#define CIM_BOOT_CFG_A 0x7b00 +#define CIM_SDRAM_BASE_ADDR_A 0x7b14 +#define CIM_SDRAM_ADDR_SIZE_A 0x7b18 +#define CIM_EXTMEM2_BASE_ADDR_A 0x7b1c +#define CIM_EXTMEM2_ADDR_SIZE_A 0x7b20 +#define CIM_PF_MAILBOX_CTRL_SHADOW_COPY_A 0x290 + +#define BOOTADDR_M 0xffffff00U + +#define UPCRST_S 0 +#define UPCRST_V(x) ((x) << UPCRST_S) +#define UPCRST_F UPCRST_V(1U) + +#define CIM_PF_MAILBOX_DATA_A 0x240 +#define CIM_PF_MAILBOX_CTRL_A 0x280 + +#define MBMSGVALID_S 3 +#define MBMSGVALID_V(x) ((x) << MBMSGVALID_S) +#define MBMSGVALID_F MBMSGVALID_V(1U) + +#define MBINTREQ_S 2 +#define MBINTREQ_V(x) ((x) << MBINTREQ_S) +#define MBINTREQ_F MBINTREQ_V(1U) + +#define MBOWNER_S 0 +#define MBOWNER_M 0x3U +#define MBOWNER_V(x) ((x) << MBOWNER_S) +#define MBOWNER_G(x) (((x) >> MBOWNER_S) & MBOWNER_M) + +#define CIM_PF_HOST_INT_ENABLE_A 0x288 + +#define MBMSGRDYINTEN_S 19 +#define MBMSGRDYINTEN_V(x) ((x) << MBMSGRDYINTEN_S) +#define MBMSGRDYINTEN_F MBMSGRDYINTEN_V(1U) + +#define CIM_PF_HOST_INT_CAUSE_A 0x28c + +#define MBMSGRDYINT_S 19 +#define MBMSGRDYINT_V(x) ((x) << MBMSGRDYINT_S) +#define MBMSGRDYINT_F MBMSGRDYINT_V(1U) + +#define CIM_HOST_INT_CAUSE_A 0x7b2c + +#define TIEQOUTPARERRINT_S 20 +#define TIEQOUTPARERRINT_V(x) ((x) << TIEQOUTPARERRINT_S) +#define TIEQOUTPARERRINT_F TIEQOUTPARERRINT_V(1U) + +#define TIEQINPARERRINT_S 19 +#define TIEQINPARERRINT_V(x) ((x) << TIEQINPARERRINT_S) +#define TIEQINPARERRINT_F TIEQINPARERRINT_V(1U) + +#define TIMER0INT_S 2 +#define TIMER0INT_V(x) ((x) << TIMER0INT_S) +#define TIMER0INT_F TIMER0INT_V(1U) + +#define PREFDROPINT_S 1 +#define PREFDROPINT_V(x) ((x) << PREFDROPINT_S) +#define PREFDROPINT_F PREFDROPINT_V(1U) + +#define UPACCNONZERO_S 0 +#define UPACCNONZERO_V(x) ((x) << UPACCNONZERO_S) +#define UPACCNONZERO_F UPACCNONZERO_V(1U) + +#define MBHOSTPARERR_S 18 +#define MBHOSTPARERR_V(x) ((x) << MBHOSTPARERR_S) +#define MBHOSTPARERR_F MBHOSTPARERR_V(1U) + +#define MBUPPARERR_S 17 +#define MBUPPARERR_V(x) ((x) << MBUPPARERR_S) +#define MBUPPARERR_F MBUPPARERR_V(1U) + +#define IBQTP0PARERR_S 16 +#define IBQTP0PARERR_V(x) ((x) << IBQTP0PARERR_S) +#define IBQTP0PARERR_F IBQTP0PARERR_V(1U) + +#define IBQTP1PARERR_S 15 +#define IBQTP1PARERR_V(x) ((x) << IBQTP1PARERR_S) +#define IBQTP1PARERR_F IBQTP1PARERR_V(1U) + +#define IBQULPPARERR_S 14 +#define IBQULPPARERR_V(x) ((x) << IBQULPPARERR_S) +#define IBQULPPARERR_F IBQULPPARERR_V(1U) + +#define IBQSGELOPARERR_S 13 +#define IBQSGELOPARERR_V(x) ((x) << IBQSGELOPARERR_S) +#define IBQSGELOPARERR_F IBQSGELOPARERR_V(1U) + +#define IBQSGEHIPARERR_S 12 +#define IBQSGEHIPARERR_V(x) ((x) << IBQSGEHIPARERR_S) +#define IBQSGEHIPARERR_F IBQSGEHIPARERR_V(1U) + +#define IBQNCSIPARERR_S 11 +#define IBQNCSIPARERR_V(x) ((x) << IBQNCSIPARERR_S) +#define IBQNCSIPARERR_F IBQNCSIPARERR_V(1U) + +#define OBQULP0PARERR_S 10 +#define OBQULP0PARERR_V(x) ((x) << OBQULP0PARERR_S) +#define OBQULP0PARERR_F OBQULP0PARERR_V(1U) + +#define OBQULP1PARERR_S 9 +#define OBQULP1PARERR_V(x) ((x) << OBQULP1PARERR_S) +#define OBQULP1PARERR_F OBQULP1PARERR_V(1U) + +#define OBQULP2PARERR_S 8 +#define OBQULP2PARERR_V(x) ((x) << OBQULP2PARERR_S) +#define OBQULP2PARERR_F OBQULP2PARERR_V(1U) + +#define OBQULP3PARERR_S 7 +#define OBQULP3PARERR_V(x) ((x) << OBQULP3PARERR_S) +#define OBQULP3PARERR_F OBQULP3PARERR_V(1U) + +#define OBQSGEPARERR_S 6 +#define OBQSGEPARERR_V(x) ((x) << OBQSGEPARERR_S) +#define OBQSGEPARERR_F OBQSGEPARERR_V(1U) + +#define OBQNCSIPARERR_S 5 +#define OBQNCSIPARERR_V(x) ((x) << OBQNCSIPARERR_S) +#define OBQNCSIPARERR_F OBQNCSIPARERR_V(1U) + +#define CIM_HOST_UPACC_INT_CAUSE_A 0x7b34 + +#define EEPROMWRINT_S 30 +#define EEPROMWRINT_V(x) ((x) << EEPROMWRINT_S) +#define EEPROMWRINT_F EEPROMWRINT_V(1U) + +#define TIMEOUTMAINT_S 29 +#define TIMEOUTMAINT_V(x) ((x) << TIMEOUTMAINT_S) +#define TIMEOUTMAINT_F TIMEOUTMAINT_V(1U) + +#define TIMEOUTINT_S 28 +#define TIMEOUTINT_V(x) ((x) << TIMEOUTINT_S) +#define TIMEOUTINT_F TIMEOUTINT_V(1U) + +#define RSPOVRLOOKUPINT_S 27 +#define RSPOVRLOOKUPINT_V(x) ((x) << RSPOVRLOOKUPINT_S) +#define RSPOVRLOOKUPINT_F RSPOVRLOOKUPINT_V(1U) + +#define REQOVRLOOKUPINT_S 26 +#define REQOVRLOOKUPINT_V(x) ((x) << REQOVRLOOKUPINT_S) +#define REQOVRLOOKUPINT_F REQOVRLOOKUPINT_V(1U) + +#define BLKWRPLINT_S 25 +#define BLKWRPLINT_V(x) ((x) << BLKWRPLINT_S) +#define BLKWRPLINT_F BLKWRPLINT_V(1U) + +#define BLKRDPLINT_S 24 +#define BLKRDPLINT_V(x) ((x) << BLKRDPLINT_S) +#define BLKRDPLINT_F BLKRDPLINT_V(1U) + +#define SGLWRPLINT_S 23 +#define SGLWRPLINT_V(x) ((x) << SGLWRPLINT_S) +#define SGLWRPLINT_F SGLWRPLINT_V(1U) + +#define SGLRDPLINT_S 22 +#define SGLRDPLINT_V(x) ((x) << SGLRDPLINT_S) +#define SGLRDPLINT_F SGLRDPLINT_V(1U) + +#define BLKWRCTLINT_S 21 +#define BLKWRCTLINT_V(x) ((x) << BLKWRCTLINT_S) +#define BLKWRCTLINT_F BLKWRCTLINT_V(1U) + +#define BLKRDCTLINT_S 20 +#define BLKRDCTLINT_V(x) ((x) << BLKRDCTLINT_S) +#define BLKRDCTLINT_F BLKRDCTLINT_V(1U) + +#define SGLWRCTLINT_S 19 +#define SGLWRCTLINT_V(x) ((x) << SGLWRCTLINT_S) +#define SGLWRCTLINT_F SGLWRCTLINT_V(1U) + +#define SGLRDCTLINT_S 18 +#define SGLRDCTLINT_V(x) ((x) << SGLRDCTLINT_S) +#define SGLRDCTLINT_F SGLRDCTLINT_V(1U) + +#define BLKWREEPROMINT_S 17 +#define BLKWREEPROMINT_V(x) ((x) << BLKWREEPROMINT_S) +#define BLKWREEPROMINT_F BLKWREEPROMINT_V(1U) + +#define BLKRDEEPROMINT_S 16 +#define BLKRDEEPROMINT_V(x) ((x) << BLKRDEEPROMINT_S) +#define BLKRDEEPROMINT_F BLKRDEEPROMINT_V(1U) + +#define SGLWREEPROMINT_S 15 +#define SGLWREEPROMINT_V(x) ((x) << SGLWREEPROMINT_S) +#define SGLWREEPROMINT_F SGLWREEPROMINT_V(1U) + +#define SGLRDEEPROMINT_S 14 +#define SGLRDEEPROMINT_V(x) ((x) << SGLRDEEPROMINT_S) +#define SGLRDEEPROMINT_F SGLRDEEPROMINT_V(1U) + +#define BLKWRFLASHINT_S 13 +#define BLKWRFLASHINT_V(x) ((x) << BLKWRFLASHINT_S) +#define BLKWRFLASHINT_F BLKWRFLASHINT_V(1U) + +#define BLKRDFLASHINT_S 12 +#define BLKRDFLASHINT_V(x) ((x) << BLKRDFLASHINT_S) +#define BLKRDFLASHINT_F BLKRDFLASHINT_V(1U) + +#define SGLWRFLASHINT_S 11 +#define SGLWRFLASHINT_V(x) ((x) << SGLWRFLASHINT_S) +#define SGLWRFLASHINT_F SGLWRFLASHINT_V(1U) + +#define SGLRDFLASHINT_S 10 +#define SGLRDFLASHINT_V(x) ((x) << SGLRDFLASHINT_S) +#define SGLRDFLASHINT_F SGLRDFLASHINT_V(1U) + +#define BLKWRBOOTINT_S 9 +#define BLKWRBOOTINT_V(x) ((x) << BLKWRBOOTINT_S) +#define BLKWRBOOTINT_F BLKWRBOOTINT_V(1U) + +#define BLKRDBOOTINT_S 8 +#define BLKRDBOOTINT_V(x) ((x) << BLKRDBOOTINT_S) +#define BLKRDBOOTINT_F BLKRDBOOTINT_V(1U) + +#define SGLWRBOOTINT_S 7 +#define SGLWRBOOTINT_V(x) ((x) << SGLWRBOOTINT_S) +#define SGLWRBOOTINT_F SGLWRBOOTINT_V(1U) + +#define SGLRDBOOTINT_S 6 +#define SGLRDBOOTINT_V(x) ((x) << SGLRDBOOTINT_S) +#define SGLRDBOOTINT_F SGLRDBOOTINT_V(1U) + +#define ILLWRBEINT_S 5 +#define ILLWRBEINT_V(x) ((x) << ILLWRBEINT_S) +#define ILLWRBEINT_F ILLWRBEINT_V(1U) + +#define ILLRDBEINT_S 4 +#define ILLRDBEINT_V(x) ((x) << ILLRDBEINT_S) +#define ILLRDBEINT_F ILLRDBEINT_V(1U) + +#define ILLRDINT_S 3 +#define ILLRDINT_V(x) ((x) << ILLRDINT_S) +#define ILLRDINT_F ILLRDINT_V(1U) + +#define ILLWRINT_S 2 +#define ILLWRINT_V(x) ((x) << ILLWRINT_S) +#define ILLWRINT_F ILLWRINT_V(1U) + +#define ILLTRANSINT_S 1 +#define ILLTRANSINT_V(x) ((x) << ILLTRANSINT_S) +#define ILLTRANSINT_F ILLTRANSINT_V(1U) + +#define RSVDSPACEINT_S 0 +#define RSVDSPACEINT_V(x) ((x) << RSVDSPACEINT_S) +#define RSVDSPACEINT_F RSVDSPACEINT_V(1U) + +/* registers for module TP */ +#define DBGLAWHLF_S 23 +#define DBGLAWHLF_V(x) ((x) << DBGLAWHLF_S) +#define DBGLAWHLF_F DBGLAWHLF_V(1U) + +#define DBGLAWPTR_S 16 +#define DBGLAWPTR_M 0x7fU +#define DBGLAWPTR_G(x) (((x) >> DBGLAWPTR_S) & DBGLAWPTR_M) + +#define DBGLAENABLE_S 12 +#define DBGLAENABLE_V(x) ((x) << DBGLAENABLE_S) +#define DBGLAENABLE_F DBGLAENABLE_V(1U) + +#define DBGLARPTR_S 0 +#define DBGLARPTR_M 0x7fU +#define DBGLARPTR_V(x) ((x) << DBGLARPTR_S) + +#define CRXPKTENC_S 3 +#define CRXPKTENC_V(x) ((x) << CRXPKTENC_S) +#define CRXPKTENC_F CRXPKTENC_V(1U) + +#define TP_DBG_LA_DATAL_A 0x7ed8 +#define TP_DBG_LA_CONFIG_A 0x7ed4 +#define TP_OUT_CONFIG_A 0x7d04 +#define TP_GLOBAL_CONFIG_A 0x7d08 + +#define ACTIVEFILTERCOUNTS_S 22 +#define ACTIVEFILTERCOUNTS_V(x) ((x) << ACTIVEFILTERCOUNTS_S) +#define ACTIVEFILTERCOUNTS_F ACTIVEFILTERCOUNTS_V(1U) + +#define TP_CMM_TCB_BASE_A 0x7d10 +#define TP_CMM_MM_BASE_A 0x7d14 +#define TP_CMM_TIMER_BASE_A 0x7d18 +#define TP_PMM_TX_BASE_A 0x7d20 +#define TP_PMM_RX_BASE_A 0x7d28 +#define TP_PMM_RX_PAGE_SIZE_A 0x7d2c +#define TP_PMM_RX_MAX_PAGE_A 0x7d30 +#define TP_PMM_TX_PAGE_SIZE_A 0x7d34 +#define TP_PMM_TX_MAX_PAGE_A 0x7d38 +#define TP_CMM_MM_MAX_PSTRUCT_A 0x7e6c + +#define PMRXNUMCHN_S 31 +#define PMRXNUMCHN_V(x) ((x) << PMRXNUMCHN_S) +#define PMRXNUMCHN_F PMRXNUMCHN_V(1U) + +#define PMTXNUMCHN_S 30 +#define PMTXNUMCHN_M 0x3U +#define PMTXNUMCHN_G(x) (((x) >> PMTXNUMCHN_S) & PMTXNUMCHN_M) + +#define PMTXMAXPAGE_S 0 +#define PMTXMAXPAGE_M 0x1fffffU +#define PMTXMAXPAGE_G(x) (((x) >> PMTXMAXPAGE_S) & PMTXMAXPAGE_M) + +#define PMRXMAXPAGE_S 0 +#define PMRXMAXPAGE_M 0x1fffffU +#define PMRXMAXPAGE_G(x) (((x) >> PMRXMAXPAGE_S) & PMRXMAXPAGE_M) + +#define DBGLAMODE_S 14 +#define DBGLAMODE_M 0x3U +#define DBGLAMODE_G(x) (((x) >> DBGLAMODE_S) & DBGLAMODE_M) + +#define FIVETUPLELOOKUP_S 17 +#define FIVETUPLELOOKUP_M 0x3U +#define FIVETUPLELOOKUP_V(x) ((x) << FIVETUPLELOOKUP_S) +#define FIVETUPLELOOKUP_G(x) (((x) >> FIVETUPLELOOKUP_S) & FIVETUPLELOOKUP_M) + +#define TP_PARA_REG2_A 0x7d68 + +#define MAXRXDATA_S 16 +#define MAXRXDATA_M 0xffffU +#define MAXRXDATA_G(x) (((x) >> MAXRXDATA_S) & MAXRXDATA_M) + +#define TP_TIMER_RESOLUTION_A 0x7d90 + +#define TIMERRESOLUTION_S 16 +#define TIMERRESOLUTION_M 0xffU +#define TIMERRESOLUTION_G(x) (((x) >> TIMERRESOLUTION_S) & TIMERRESOLUTION_M) + +#define TIMESTAMPRESOLUTION_S 8 +#define TIMESTAMPRESOLUTION_M 0xffU +#define TIMESTAMPRESOLUTION_G(x) \ + (((x) >> TIMESTAMPRESOLUTION_S) & TIMESTAMPRESOLUTION_M) + +#define DELAYEDACKRESOLUTION_S 0 +#define DELAYEDACKRESOLUTION_M 0xffU +#define DELAYEDACKRESOLUTION_G(x) \ + (((x) >> DELAYEDACKRESOLUTION_S) & DELAYEDACKRESOLUTION_M) + +#define TP_SHIFT_CNT_A 0x7dc0 +#define TP_RXT_MIN_A 0x7d98 +#define TP_RXT_MAX_A 0x7d9c +#define TP_PERS_MIN_A 0x7da0 +#define TP_PERS_MAX_A 0x7da4 +#define TP_KEEP_IDLE_A 0x7da8 +#define TP_KEEP_INTVL_A 0x7dac +#define TP_INIT_SRTT_A 0x7db0 +#define TP_DACK_TIMER_A 0x7db4 +#define TP_FINWAIT2_TIMER_A 0x7db8 + +#define INITSRTT_S 0 +#define INITSRTT_M 0xffffU +#define INITSRTT_G(x) (((x) >> INITSRTT_S) & INITSRTT_M) + +#define PERSMAX_S 0 +#define PERSMAX_M 0x3fffffffU +#define PERSMAX_V(x) ((x) << PERSMAX_S) +#define PERSMAX_G(x) (((x) >> PERSMAX_S) & PERSMAX_M) + +#define SYNSHIFTMAX_S 24 +#define SYNSHIFTMAX_M 0xffU +#define SYNSHIFTMAX_V(x) ((x) << SYNSHIFTMAX_S) +#define SYNSHIFTMAX_G(x) (((x) >> SYNSHIFTMAX_S) & SYNSHIFTMAX_M) + +#define RXTSHIFTMAXR1_S 20 +#define RXTSHIFTMAXR1_M 0xfU +#define RXTSHIFTMAXR1_V(x) ((x) << RXTSHIFTMAXR1_S) +#define RXTSHIFTMAXR1_G(x) (((x) >> RXTSHIFTMAXR1_S) & RXTSHIFTMAXR1_M) + +#define RXTSHIFTMAXR2_S 16 +#define RXTSHIFTMAXR2_M 0xfU +#define RXTSHIFTMAXR2_V(x) ((x) << RXTSHIFTMAXR2_S) +#define RXTSHIFTMAXR2_G(x) (((x) >> RXTSHIFTMAXR2_S) & RXTSHIFTMAXR2_M) + +#define PERSHIFTBACKOFFMAX_S 12 +#define PERSHIFTBACKOFFMAX_M 0xfU +#define PERSHIFTBACKOFFMAX_V(x) ((x) << PERSHIFTBACKOFFMAX_S) +#define PERSHIFTBACKOFFMAX_G(x) \ + (((x) >> PERSHIFTBACKOFFMAX_S) & PERSHIFTBACKOFFMAX_M) + +#define PERSHIFTMAX_S 8 +#define PERSHIFTMAX_M 0xfU +#define PERSHIFTMAX_V(x) ((x) << PERSHIFTMAX_S) +#define PERSHIFTMAX_G(x) (((x) >> PERSHIFTMAX_S) & PERSHIFTMAX_M) + +#define KEEPALIVEMAXR1_S 4 +#define KEEPALIVEMAXR1_M 0xfU +#define KEEPALIVEMAXR1_V(x) ((x) << KEEPALIVEMAXR1_S) +#define KEEPALIVEMAXR1_G(x) (((x) >> KEEPALIVEMAXR1_S) & KEEPALIVEMAXR1_M) + +#define KEEPALIVEMAXR2_S 0 +#define KEEPALIVEMAXR2_M 0xfU +#define KEEPALIVEMAXR2_V(x) ((x) << KEEPALIVEMAXR2_S) +#define KEEPALIVEMAXR2_G(x) (((x) >> KEEPALIVEMAXR2_S) & KEEPALIVEMAXR2_M) + +#define ROWINDEX_S 16 +#define ROWINDEX_V(x) ((x) << ROWINDEX_S) + +#define TP_CCTRL_TABLE_A 0x7ddc +#define TP_PACE_TABLE_A 0x7dd8 +#define TP_MTU_TABLE_A 0x7de4 + +#define MTUINDEX_S 24 +#define MTUINDEX_V(x) ((x) << MTUINDEX_S) + +#define MTUWIDTH_S 16 +#define MTUWIDTH_M 0xfU +#define MTUWIDTH_V(x) ((x) << MTUWIDTH_S) +#define MTUWIDTH_G(x) (((x) >> MTUWIDTH_S) & MTUWIDTH_M) + +#define MTUVALUE_S 0 +#define MTUVALUE_M 0x3fffU +#define MTUVALUE_V(x) ((x) << MTUVALUE_S) +#define MTUVALUE_G(x) (((x) >> MTUVALUE_S) & MTUVALUE_M) + +#define TP_RSS_LKP_TABLE_A 0x7dec +#define TP_CMM_MM_RX_FLST_BASE_A 0x7e60 +#define TP_CMM_MM_TX_FLST_BASE_A 0x7e64 +#define TP_CMM_MM_PS_FLST_BASE_A 0x7e68 + +#define LKPTBLROWVLD_S 31 +#define LKPTBLROWVLD_V(x) ((x) << LKPTBLROWVLD_S) +#define LKPTBLROWVLD_F LKPTBLROWVLD_V(1U) + +#define LKPTBLQUEUE1_S 10 +#define LKPTBLQUEUE1_M 0x3ffU +#define LKPTBLQUEUE1_G(x) (((x) >> LKPTBLQUEUE1_S) & LKPTBLQUEUE1_M) + +#define LKPTBLQUEUE0_S 0 +#define LKPTBLQUEUE0_M 0x3ffU +#define LKPTBLQUEUE0_G(x) (((x) >> LKPTBLQUEUE0_S) & LKPTBLQUEUE0_M) + +#define TP_TM_PIO_ADDR_A 0x7e18 +#define TP_TM_PIO_DATA_A 0x7e1c +#define TP_MOD_CONFIG_A 0x7e24 + +#define TIMERMODE_S 8 +#define TIMERMODE_M 0xffU +#define TIMERMODE_G(x) (((x) >> TIMERMODE_S) & TIMERMODE_M) + +#define TP_TX_MOD_Q1_Q0_TIMER_SEPARATOR_A 0x3 +#define TP_TX_MOD_Q1_Q0_RATE_LIMIT_A 0x8 + +#define TP_PIO_ADDR_A 0x7e40 +#define TP_PIO_DATA_A 0x7e44 +#define TP_MIB_INDEX_A 0x7e50 +#define TP_MIB_DATA_A 0x7e54 +#define TP_INT_CAUSE_A 0x7e74 + +#define TP_FLM_FREE_PS_CNT_A 0x7e80 +#define TP_FLM_FREE_RX_CNT_A 0x7e84 + +#define FREEPSTRUCTCOUNT_S 0 +#define FREEPSTRUCTCOUNT_M 0x1fffffU +#define FREEPSTRUCTCOUNT_G(x) (((x) >> FREEPSTRUCTCOUNT_S) & FREEPSTRUCTCOUNT_M) + +#define FREERXPAGECOUNT_S 0 +#define FREERXPAGECOUNT_M 0x1fffffU +#define FREERXPAGECOUNT_V(x) ((x) << FREERXPAGECOUNT_S) +#define FREERXPAGECOUNT_G(x) (((x) >> FREERXPAGECOUNT_S) & FREERXPAGECOUNT_M) + +#define TP_FLM_FREE_TX_CNT_A 0x7e88 + +#define FREETXPAGECOUNT_S 0 +#define FREETXPAGECOUNT_M 0x1fffffU +#define FREETXPAGECOUNT_V(x) ((x) << FREETXPAGECOUNT_S) +#define FREETXPAGECOUNT_G(x) (((x) >> FREETXPAGECOUNT_S) & FREETXPAGECOUNT_M) + +#define FLMTXFLSTEMPTY_S 30 +#define FLMTXFLSTEMPTY_V(x) ((x) << FLMTXFLSTEMPTY_S) +#define FLMTXFLSTEMPTY_F FLMTXFLSTEMPTY_V(1U) + +#define TP_TX_ORATE_A 0x7ebc + +#define OFDRATE3_S 24 +#define OFDRATE3_M 0xffU +#define OFDRATE3_G(x) (((x) >> OFDRATE3_S) & OFDRATE3_M) + +#define OFDRATE2_S 16 +#define OFDRATE2_M 0xffU +#define OFDRATE2_G(x) (((x) >> OFDRATE2_S) & OFDRATE2_M) + +#define OFDRATE1_S 8 +#define OFDRATE1_M 0xffU +#define OFDRATE1_G(x) (((x) >> OFDRATE1_S) & OFDRATE1_M) + +#define OFDRATE0_S 0 +#define OFDRATE0_M 0xffU +#define OFDRATE0_G(x) (((x) >> OFDRATE0_S) & OFDRATE0_M) + +#define TP_TX_TRATE_A 0x7ed0 + +#define TNLRATE3_S 24 +#define TNLRATE3_M 0xffU +#define TNLRATE3_G(x) (((x) >> TNLRATE3_S) & TNLRATE3_M) + +#define TNLRATE2_S 16 +#define TNLRATE2_M 0xffU +#define TNLRATE2_G(x) (((x) >> TNLRATE2_S) & TNLRATE2_M) + +#define TNLRATE1_S 8 +#define TNLRATE1_M 0xffU +#define TNLRATE1_G(x) (((x) >> TNLRATE1_S) & TNLRATE1_M) + +#define TNLRATE0_S 0 +#define TNLRATE0_M 0xffU +#define TNLRATE0_G(x) (((x) >> TNLRATE0_S) & TNLRATE0_M) + +#define TP_VLAN_PRI_MAP_A 0x140 + +#define FRAGMENTATION_S 9 +#define FRAGMENTATION_V(x) ((x) << FRAGMENTATION_S) +#define FRAGMENTATION_F FRAGMENTATION_V(1U) + +#define MPSHITTYPE_S 8 +#define MPSHITTYPE_V(x) ((x) << MPSHITTYPE_S) +#define MPSHITTYPE_F MPSHITTYPE_V(1U) + +#define MACMATCH_S 7 +#define MACMATCH_V(x) ((x) << MACMATCH_S) +#define MACMATCH_F MACMATCH_V(1U) + +#define ETHERTYPE_S 6 +#define ETHERTYPE_V(x) ((x) << ETHERTYPE_S) +#define ETHERTYPE_F ETHERTYPE_V(1U) + +#define PROTOCOL_S 5 +#define PROTOCOL_V(x) ((x) << PROTOCOL_S) +#define PROTOCOL_F PROTOCOL_V(1U) + +#define TOS_S 4 +#define TOS_V(x) ((x) << TOS_S) +#define TOS_F TOS_V(1U) + +#define VLAN_S 3 +#define VLAN_V(x) ((x) << VLAN_S) +#define VLAN_F VLAN_V(1U) + +#define VNIC_ID_S 2 +#define VNIC_ID_V(x) ((x) << VNIC_ID_S) +#define VNIC_ID_F VNIC_ID_V(1U) + +#define PORT_S 1 +#define PORT_V(x) ((x) << PORT_S) +#define PORT_F PORT_V(1U) + +#define FCOE_S 0 +#define FCOE_V(x) ((x) << FCOE_S) +#define FCOE_F FCOE_V(1U) + +#define FILTERMODE_S 15 +#define FILTERMODE_V(x) ((x) << FILTERMODE_S) +#define FILTERMODE_F FILTERMODE_V(1U) + +#define FCOEMASK_S 14 +#define FCOEMASK_V(x) ((x) << FCOEMASK_S) +#define FCOEMASK_F FCOEMASK_V(1U) + +#define TP_INGRESS_CONFIG_A 0x141 + +#define VNIC_S 11 +#define VNIC_V(x) ((x) << VNIC_S) +#define VNIC_F VNIC_V(1U) + +#define USE_ENC_IDX_S 13 +#define USE_ENC_IDX_V(x) ((x) << USE_ENC_IDX_S) +#define USE_ENC_IDX_F USE_ENC_IDX_V(1U) + +#define CSUM_HAS_PSEUDO_HDR_S 10 +#define CSUM_HAS_PSEUDO_HDR_V(x) ((x) << CSUM_HAS_PSEUDO_HDR_S) +#define CSUM_HAS_PSEUDO_HDR_F CSUM_HAS_PSEUDO_HDR_V(1U) + +#define TP_MIB_MAC_IN_ERR_0_A 0x0 +#define TP_MIB_HDR_IN_ERR_0_A 0x4 +#define TP_MIB_TCP_IN_ERR_0_A 0x8 +#define TP_MIB_TCP_OUT_RST_A 0xc +#define TP_MIB_TCP_IN_SEG_HI_A 0x10 +#define TP_MIB_TCP_IN_SEG_LO_A 0x11 +#define TP_MIB_TCP_OUT_SEG_HI_A 0x12 +#define TP_MIB_TCP_OUT_SEG_LO_A 0x13 +#define TP_MIB_TCP_RXT_SEG_HI_A 0x14 +#define TP_MIB_TCP_RXT_SEG_LO_A 0x15 +#define TP_MIB_TNL_CNG_DROP_0_A 0x18 +#define TP_MIB_OFD_CHN_DROP_0_A 0x1c +#define TP_MIB_TCP_V6IN_ERR_0_A 0x28 +#define TP_MIB_TCP_V6OUT_RST_A 0x2c +#define TP_MIB_OFD_ARP_DROP_A 0x36 +#define TP_MIB_CPL_IN_REQ_0_A 0x38 +#define TP_MIB_CPL_OUT_RSP_0_A 0x3c +#define TP_MIB_TNL_DROP_0_A 0x44 +#define TP_MIB_FCOE_DDP_0_A 0x48 +#define TP_MIB_FCOE_DROP_0_A 0x4c +#define TP_MIB_FCOE_BYTE_0_HI_A 0x50 +#define TP_MIB_OFD_VLN_DROP_0_A 0x58 +#define TP_MIB_USM_PKTS_A 0x5c +#define TP_MIB_RQE_DFR_PKT_A 0x64 + +#define ULP_TX_INT_CAUSE_A 0x8dcc +#define ULP_TX_TPT_LLIMIT_A 0x8dd4 +#define ULP_TX_TPT_ULIMIT_A 0x8dd8 +#define ULP_TX_PBL_LLIMIT_A 0x8ddc +#define ULP_TX_PBL_ULIMIT_A 0x8de0 +#define ULP_TX_ERR_TABLE_BASE_A 0x8e04 + +#define PBL_BOUND_ERR_CH3_S 31 +#define PBL_BOUND_ERR_CH3_V(x) ((x) << PBL_BOUND_ERR_CH3_S) +#define PBL_BOUND_ERR_CH3_F PBL_BOUND_ERR_CH3_V(1U) + +#define PBL_BOUND_ERR_CH2_S 30 +#define PBL_BOUND_ERR_CH2_V(x) ((x) << PBL_BOUND_ERR_CH2_S) +#define PBL_BOUND_ERR_CH2_F PBL_BOUND_ERR_CH2_V(1U) + +#define PBL_BOUND_ERR_CH1_S 29 +#define PBL_BOUND_ERR_CH1_V(x) ((x) << PBL_BOUND_ERR_CH1_S) +#define PBL_BOUND_ERR_CH1_F PBL_BOUND_ERR_CH1_V(1U) + +#define PBL_BOUND_ERR_CH0_S 28 +#define PBL_BOUND_ERR_CH0_V(x) ((x) << PBL_BOUND_ERR_CH0_S) +#define PBL_BOUND_ERR_CH0_F PBL_BOUND_ERR_CH0_V(1U) + +#define PM_RX_INT_CAUSE_A 0x8fdc +#define PM_RX_STAT_CONFIG_A 0x8fc8 +#define PM_RX_STAT_COUNT_A 0x8fcc +#define PM_RX_STAT_LSB_A 0x8fd0 +#define PM_RX_DBG_CTRL_A 0x8fd0 +#define PM_RX_DBG_DATA_A 0x8fd4 +#define PM_RX_DBG_STAT_MSB_A 0x10013 + +#define PMRX_FRAMING_ERROR_F 0x003ffff0U + +#define ZERO_E_CMD_ERROR_S 22 +#define ZERO_E_CMD_ERROR_V(x) ((x) << ZERO_E_CMD_ERROR_S) +#define ZERO_E_CMD_ERROR_F ZERO_E_CMD_ERROR_V(1U) + +#define OCSPI_PAR_ERROR_S 3 +#define OCSPI_PAR_ERROR_V(x) ((x) << OCSPI_PAR_ERROR_S) +#define OCSPI_PAR_ERROR_F OCSPI_PAR_ERROR_V(1U) + +#define DB_OPTIONS_PAR_ERROR_S 2 +#define DB_OPTIONS_PAR_ERROR_V(x) ((x) << DB_OPTIONS_PAR_ERROR_S) +#define DB_OPTIONS_PAR_ERROR_F DB_OPTIONS_PAR_ERROR_V(1U) + +#define IESPI_PAR_ERROR_S 1 +#define IESPI_PAR_ERROR_V(x) ((x) << IESPI_PAR_ERROR_S) +#define IESPI_PAR_ERROR_F IESPI_PAR_ERROR_V(1U) + +#define ULP_TX_LA_RDPTR_0_A 0x8ec0 +#define ULP_TX_LA_RDDATA_0_A 0x8ec4 +#define ULP_TX_LA_WRPTR_0_A 0x8ec8 +#define ULP_TX_ASIC_DEBUG_CTRL_A 0x8f70 + +#define ULP_TX_ASIC_DEBUG_0_A 0x8f74 +#define ULP_TX_ASIC_DEBUG_1_A 0x8f78 +#define ULP_TX_ASIC_DEBUG_2_A 0x8f7c +#define ULP_TX_ASIC_DEBUG_3_A 0x8f80 +#define ULP_TX_ASIC_DEBUG_4_A 0x8f84 + +/* registers for module PM_RX */ +#define PM_RX_BASE_ADDR 0x8fc0 + +#define PMRX_E_PCMD_PAR_ERROR_S 0 +#define PMRX_E_PCMD_PAR_ERROR_V(x) ((x) << PMRX_E_PCMD_PAR_ERROR_S) +#define PMRX_E_PCMD_PAR_ERROR_F PMRX_E_PCMD_PAR_ERROR_V(1U) + +#define PM_TX_INT_CAUSE_A 0x8ffc +#define PM_TX_STAT_CONFIG_A 0x8fe8 +#define PM_TX_STAT_COUNT_A 0x8fec +#define PM_TX_STAT_LSB_A 0x8ff0 +#define PM_TX_DBG_CTRL_A 0x8ff0 +#define PM_TX_DBG_DATA_A 0x8ff4 +#define PM_TX_DBG_STAT_MSB_A 0x1001a + +#define PCMD_LEN_OVFL0_S 31 +#define PCMD_LEN_OVFL0_V(x) ((x) << PCMD_LEN_OVFL0_S) +#define PCMD_LEN_OVFL0_F PCMD_LEN_OVFL0_V(1U) + +#define PCMD_LEN_OVFL1_S 30 +#define PCMD_LEN_OVFL1_V(x) ((x) << PCMD_LEN_OVFL1_S) +#define PCMD_LEN_OVFL1_F PCMD_LEN_OVFL1_V(1U) + +#define PCMD_LEN_OVFL2_S 29 +#define PCMD_LEN_OVFL2_V(x) ((x) << PCMD_LEN_OVFL2_S) +#define PCMD_LEN_OVFL2_F PCMD_LEN_OVFL2_V(1U) + +#define ZERO_C_CMD_ERROR_S 28 +#define ZERO_C_CMD_ERROR_V(x) ((x) << ZERO_C_CMD_ERROR_S) +#define ZERO_C_CMD_ERROR_F ZERO_C_CMD_ERROR_V(1U) + +#define PMTX_FRAMING_ERROR_F 0x0ffffff0U + +#define OESPI_PAR_ERROR_S 3 +#define OESPI_PAR_ERROR_V(x) ((x) << OESPI_PAR_ERROR_S) +#define OESPI_PAR_ERROR_F OESPI_PAR_ERROR_V(1U) + +#define ICSPI_PAR_ERROR_S 1 +#define ICSPI_PAR_ERROR_V(x) ((x) << ICSPI_PAR_ERROR_S) +#define ICSPI_PAR_ERROR_F ICSPI_PAR_ERROR_V(1U) + +#define PMTX_C_PCMD_PAR_ERROR_S 0 +#define PMTX_C_PCMD_PAR_ERROR_V(x) ((x) << PMTX_C_PCMD_PAR_ERROR_S) +#define PMTX_C_PCMD_PAR_ERROR_F PMTX_C_PCMD_PAR_ERROR_V(1U) + +#define MPS_PORT_STAT_TX_PORT_BYTES_L 0x400 +#define MPS_PORT_STAT_TX_PORT_BYTES_H 0x404 +#define MPS_PORT_STAT_TX_PORT_FRAMES_L 0x408 +#define MPS_PORT_STAT_TX_PORT_FRAMES_H 0x40c +#define MPS_PORT_STAT_TX_PORT_BCAST_L 0x410 +#define MPS_PORT_STAT_TX_PORT_BCAST_H 0x414 +#define MPS_PORT_STAT_TX_PORT_MCAST_L 0x418 +#define MPS_PORT_STAT_TX_PORT_MCAST_H 0x41c +#define MPS_PORT_STAT_TX_PORT_UCAST_L 0x420 +#define MPS_PORT_STAT_TX_PORT_UCAST_H 0x424 +#define MPS_PORT_STAT_TX_PORT_ERROR_L 0x428 +#define MPS_PORT_STAT_TX_PORT_ERROR_H 0x42c +#define MPS_PORT_STAT_TX_PORT_64B_L 0x430 +#define MPS_PORT_STAT_TX_PORT_64B_H 0x434 +#define MPS_PORT_STAT_TX_PORT_65B_127B_L 0x438 +#define MPS_PORT_STAT_TX_PORT_65B_127B_H 0x43c +#define MPS_PORT_STAT_TX_PORT_128B_255B_L 0x440 +#define MPS_PORT_STAT_TX_PORT_128B_255B_H 0x444 +#define MPS_PORT_STAT_TX_PORT_256B_511B_L 0x448 +#define MPS_PORT_STAT_TX_PORT_256B_511B_H 0x44c +#define MPS_PORT_STAT_TX_PORT_512B_1023B_L 0x450 +#define MPS_PORT_STAT_TX_PORT_512B_1023B_H 0x454 +#define MPS_PORT_STAT_TX_PORT_1024B_1518B_L 0x458 +#define MPS_PORT_STAT_TX_PORT_1024B_1518B_H 0x45c +#define MPS_PORT_STAT_TX_PORT_1519B_MAX_L 0x460 +#define MPS_PORT_STAT_TX_PORT_1519B_MAX_H 0x464 +#define MPS_PORT_STAT_TX_PORT_DROP_L 0x468 +#define MPS_PORT_STAT_TX_PORT_DROP_H 0x46c +#define MPS_PORT_STAT_TX_PORT_PAUSE_L 0x470 +#define MPS_PORT_STAT_TX_PORT_PAUSE_H 0x474 +#define MPS_PORT_STAT_TX_PORT_PPP0_L 0x478 +#define MPS_PORT_STAT_TX_PORT_PPP0_H 0x47c +#define MPS_PORT_STAT_TX_PORT_PPP1_L 0x480 +#define MPS_PORT_STAT_TX_PORT_PPP1_H 0x484 +#define MPS_PORT_STAT_TX_PORT_PPP2_L 0x488 +#define MPS_PORT_STAT_TX_PORT_PPP2_H 0x48c +#define MPS_PORT_STAT_TX_PORT_PPP3_L 0x490 +#define MPS_PORT_STAT_TX_PORT_PPP3_H 0x494 +#define MPS_PORT_STAT_TX_PORT_PPP4_L 0x498 +#define MPS_PORT_STAT_TX_PORT_PPP4_H 0x49c +#define MPS_PORT_STAT_TX_PORT_PPP5_L 0x4a0 +#define MPS_PORT_STAT_TX_PORT_PPP5_H 0x4a4 +#define MPS_PORT_STAT_TX_PORT_PPP6_L 0x4a8 +#define MPS_PORT_STAT_TX_PORT_PPP6_H 0x4ac +#define MPS_PORT_STAT_TX_PORT_PPP7_L 0x4b0 +#define MPS_PORT_STAT_TX_PORT_PPP7_H 0x4b4 +#define MPS_PORT_STAT_LB_PORT_BYTES_L 0x4c0 +#define MPS_PORT_STAT_LB_PORT_BYTES_H 0x4c4 +#define MPS_PORT_STAT_LB_PORT_FRAMES_L 0x4c8 +#define MPS_PORT_STAT_LB_PORT_FRAMES_H 0x4cc +#define MPS_PORT_STAT_LB_PORT_BCAST_L 0x4d0 +#define MPS_PORT_STAT_LB_PORT_BCAST_H 0x4d4 +#define MPS_PORT_STAT_LB_PORT_MCAST_L 0x4d8 +#define MPS_PORT_STAT_LB_PORT_MCAST_H 0x4dc +#define MPS_PORT_STAT_LB_PORT_UCAST_L 0x4e0 +#define MPS_PORT_STAT_LB_PORT_UCAST_H 0x4e4 +#define MPS_PORT_STAT_LB_PORT_ERROR_L 0x4e8 +#define MPS_PORT_STAT_LB_PORT_ERROR_H 0x4ec +#define MPS_PORT_STAT_LB_PORT_64B_L 0x4f0 +#define MPS_PORT_STAT_LB_PORT_64B_H 0x4f4 +#define MPS_PORT_STAT_LB_PORT_65B_127B_L 0x4f8 +#define MPS_PORT_STAT_LB_PORT_65B_127B_H 0x4fc +#define MPS_PORT_STAT_LB_PORT_128B_255B_L 0x500 +#define MPS_PORT_STAT_LB_PORT_128B_255B_H 0x504 +#define MPS_PORT_STAT_LB_PORT_256B_511B_L 0x508 +#define MPS_PORT_STAT_LB_PORT_256B_511B_H 0x50c +#define MPS_PORT_STAT_LB_PORT_512B_1023B_L 0x510 +#define MPS_PORT_STAT_LB_PORT_512B_1023B_H 0x514 +#define MPS_PORT_STAT_LB_PORT_1024B_1518B_L 0x518 +#define MPS_PORT_STAT_LB_PORT_1024B_1518B_H 0x51c +#define MPS_PORT_STAT_LB_PORT_1519B_MAX_L 0x520 +#define MPS_PORT_STAT_LB_PORT_1519B_MAX_H 0x524 +#define MPS_PORT_STAT_LB_PORT_DROP_FRAMES 0x528 +#define MPS_PORT_STAT_LB_PORT_DROP_FRAMES_L 0x528 +#define MPS_PORT_STAT_RX_PORT_BYTES_L 0x540 +#define MPS_PORT_STAT_RX_PORT_BYTES_H 0x544 +#define MPS_PORT_STAT_RX_PORT_FRAMES_L 0x548 +#define MPS_PORT_STAT_RX_PORT_FRAMES_H 0x54c +#define MPS_PORT_STAT_RX_PORT_BCAST_L 0x550 +#define MPS_PORT_STAT_RX_PORT_BCAST_H 0x554 +#define MPS_PORT_STAT_RX_PORT_MCAST_L 0x558 +#define MPS_PORT_STAT_RX_PORT_MCAST_H 0x55c +#define MPS_PORT_STAT_RX_PORT_UCAST_L 0x560 +#define MPS_PORT_STAT_RX_PORT_UCAST_H 0x564 +#define MPS_PORT_STAT_RX_PORT_MTU_ERROR_L 0x568 +#define MPS_PORT_STAT_RX_PORT_MTU_ERROR_H 0x56c +#define MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L 0x570 +#define MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_H 0x574 +#define MPS_PORT_STAT_RX_PORT_CRC_ERROR_L 0x578 +#define MPS_PORT_STAT_RX_PORT_CRC_ERROR_H 0x57c +#define MPS_PORT_STAT_RX_PORT_LEN_ERROR_L 0x580 +#define MPS_PORT_STAT_RX_PORT_LEN_ERROR_H 0x584 +#define MPS_PORT_STAT_RX_PORT_SYM_ERROR_L 0x588 +#define MPS_PORT_STAT_RX_PORT_SYM_ERROR_H 0x58c +#define MPS_PORT_STAT_RX_PORT_64B_L 0x590 +#define MPS_PORT_STAT_RX_PORT_64B_H 0x594 +#define MPS_PORT_STAT_RX_PORT_65B_127B_L 0x598 +#define MPS_PORT_STAT_RX_PORT_65B_127B_H 0x59c +#define MPS_PORT_STAT_RX_PORT_128B_255B_L 0x5a0 +#define MPS_PORT_STAT_RX_PORT_128B_255B_H 0x5a4 +#define MPS_PORT_STAT_RX_PORT_256B_511B_L 0x5a8 +#define MPS_PORT_STAT_RX_PORT_256B_511B_H 0x5ac +#define MPS_PORT_STAT_RX_PORT_512B_1023B_L 0x5b0 +#define MPS_PORT_STAT_RX_PORT_512B_1023B_H 0x5b4 +#define MPS_PORT_STAT_RX_PORT_1024B_1518B_L 0x5b8 +#define MPS_PORT_STAT_RX_PORT_1024B_1518B_H 0x5bc +#define MPS_PORT_STAT_RX_PORT_1519B_MAX_L 0x5c0 +#define MPS_PORT_STAT_RX_PORT_1519B_MAX_H 0x5c4 +#define MPS_PORT_STAT_RX_PORT_PAUSE_L 0x5c8 +#define MPS_PORT_STAT_RX_PORT_PAUSE_H 0x5cc +#define MPS_PORT_STAT_RX_PORT_PPP0_L 0x5d0 +#define MPS_PORT_STAT_RX_PORT_PPP0_H 0x5d4 +#define MPS_PORT_STAT_RX_PORT_PPP1_L 0x5d8 +#define MPS_PORT_STAT_RX_PORT_PPP1_H 0x5dc +#define MPS_PORT_STAT_RX_PORT_PPP2_L 0x5e0 +#define MPS_PORT_STAT_RX_PORT_PPP2_H 0x5e4 +#define MPS_PORT_STAT_RX_PORT_PPP3_L 0x5e8 +#define MPS_PORT_STAT_RX_PORT_PPP3_H 0x5ec +#define MPS_PORT_STAT_RX_PORT_PPP4_L 0x5f0 +#define MPS_PORT_STAT_RX_PORT_PPP4_H 0x5f4 +#define MPS_PORT_STAT_RX_PORT_PPP5_L 0x5f8 +#define MPS_PORT_STAT_RX_PORT_PPP5_H 0x5fc +#define MPS_PORT_STAT_RX_PORT_PPP6_L 0x600 +#define MPS_PORT_STAT_RX_PORT_PPP6_H 0x604 +#define MPS_PORT_STAT_RX_PORT_PPP7_L 0x608 +#define MPS_PORT_STAT_RX_PORT_PPP7_H 0x60c +#define MPS_PORT_STAT_RX_PORT_LESS_64B_L 0x610 +#define MPS_PORT_STAT_RX_PORT_LESS_64B_H 0x614 +#define MAC_PORT_MAGIC_MACID_LO 0x824 +#define MAC_PORT_MAGIC_MACID_HI 0x828 +#define MAC_PORT_TX_TS_VAL_LO 0x928 +#define MAC_PORT_TX_TS_VAL_HI 0x92c + +#define MAC_PORT_EPIO_DATA0_A 0x8c0 +#define MAC_PORT_EPIO_DATA1_A 0x8c4 +#define MAC_PORT_EPIO_DATA2_A 0x8c8 +#define MAC_PORT_EPIO_DATA3_A 0x8cc +#define MAC_PORT_EPIO_OP_A 0x8d0 + +#define MAC_PORT_CFG2_A 0x818 + +#define MAC_PORT_PTP_SUM_LO_A 0x990 +#define MAC_PORT_PTP_SUM_HI_A 0x994 + +#define MPS_CMN_CTL_A 0x9000 + +#define COUNTPAUSEMCRX_S 5 +#define COUNTPAUSEMCRX_V(x) ((x) << COUNTPAUSEMCRX_S) +#define COUNTPAUSEMCRX_F COUNTPAUSEMCRX_V(1U) + +#define COUNTPAUSESTATRX_S 4 +#define COUNTPAUSESTATRX_V(x) ((x) << COUNTPAUSESTATRX_S) +#define COUNTPAUSESTATRX_F COUNTPAUSESTATRX_V(1U) + +#define COUNTPAUSEMCTX_S 3 +#define COUNTPAUSEMCTX_V(x) ((x) << COUNTPAUSEMCTX_S) +#define COUNTPAUSEMCTX_F COUNTPAUSEMCTX_V(1U) + +#define COUNTPAUSESTATTX_S 2 +#define COUNTPAUSESTATTX_V(x) ((x) << COUNTPAUSESTATTX_S) +#define COUNTPAUSESTATTX_F COUNTPAUSESTATTX_V(1U) + +#define NUMPORTS_S 0 +#define NUMPORTS_M 0x3U +#define NUMPORTS_G(x) (((x) >> NUMPORTS_S) & NUMPORTS_M) + +#define MPS_INT_CAUSE_A 0x9008 +#define MPS_TX_INT_CAUSE_A 0x9408 +#define MPS_STAT_CTL_A 0x9600 + +#define FRMERR_S 15 +#define FRMERR_V(x) ((x) << FRMERR_S) +#define FRMERR_F FRMERR_V(1U) + +#define SECNTERR_S 14 +#define SECNTERR_V(x) ((x) << SECNTERR_S) +#define SECNTERR_F SECNTERR_V(1U) + +#define BUBBLE_S 13 +#define BUBBLE_V(x) ((x) << BUBBLE_S) +#define BUBBLE_F BUBBLE_V(1U) + +#define TXDESCFIFO_S 9 +#define TXDESCFIFO_M 0xfU +#define TXDESCFIFO_V(x) ((x) << TXDESCFIFO_S) + +#define TXDATAFIFO_S 5 +#define TXDATAFIFO_M 0xfU +#define TXDATAFIFO_V(x) ((x) << TXDATAFIFO_S) + +#define NCSIFIFO_S 4 +#define NCSIFIFO_V(x) ((x) << NCSIFIFO_S) +#define NCSIFIFO_F NCSIFIFO_V(1U) + +#define TPFIFO_S 0 +#define TPFIFO_M 0xfU +#define TPFIFO_V(x) ((x) << TPFIFO_S) + +#define MPS_STAT_PERR_INT_CAUSE_SRAM_A 0x9614 +#define MPS_STAT_PERR_INT_CAUSE_TX_FIFO_A 0x9620 +#define MPS_STAT_PERR_INT_CAUSE_RX_FIFO_A 0x962c + +#define MPS_STAT_RX_BG_0_MAC_DROP_FRAME_L 0x9640 +#define MPS_STAT_RX_BG_0_MAC_DROP_FRAME_H 0x9644 +#define MPS_STAT_RX_BG_1_MAC_DROP_FRAME_L 0x9648 +#define MPS_STAT_RX_BG_1_MAC_DROP_FRAME_H 0x964c +#define MPS_STAT_RX_BG_2_MAC_DROP_FRAME_L 0x9650 +#define MPS_STAT_RX_BG_2_MAC_DROP_FRAME_H 0x9654 +#define MPS_STAT_RX_BG_3_MAC_DROP_FRAME_L 0x9658 +#define MPS_STAT_RX_BG_3_MAC_DROP_FRAME_H 0x965c +#define MPS_STAT_RX_BG_0_LB_DROP_FRAME_L 0x9660 +#define MPS_STAT_RX_BG_0_LB_DROP_FRAME_H 0x9664 +#define MPS_STAT_RX_BG_1_LB_DROP_FRAME_L 0x9668 +#define MPS_STAT_RX_BG_1_LB_DROP_FRAME_H 0x966c +#define MPS_STAT_RX_BG_2_LB_DROP_FRAME_L 0x9670 +#define MPS_STAT_RX_BG_2_LB_DROP_FRAME_H 0x9674 +#define MPS_STAT_RX_BG_3_LB_DROP_FRAME_L 0x9678 +#define MPS_STAT_RX_BG_3_LB_DROP_FRAME_H 0x967c +#define MPS_STAT_RX_BG_0_MAC_TRUNC_FRAME_L 0x9680 +#define MPS_STAT_RX_BG_0_MAC_TRUNC_FRAME_H 0x9684 +#define MPS_STAT_RX_BG_1_MAC_TRUNC_FRAME_L 0x9688 +#define MPS_STAT_RX_BG_1_MAC_TRUNC_FRAME_H 0x968c +#define MPS_STAT_RX_BG_2_MAC_TRUNC_FRAME_L 0x9690 +#define MPS_STAT_RX_BG_2_MAC_TRUNC_FRAME_H 0x9694 +#define MPS_STAT_RX_BG_3_MAC_TRUNC_FRAME_L 0x9698 +#define MPS_STAT_RX_BG_3_MAC_TRUNC_FRAME_H 0x969c +#define MPS_STAT_RX_BG_0_LB_TRUNC_FRAME_L 0x96a0 +#define MPS_STAT_RX_BG_0_LB_TRUNC_FRAME_H 0x96a4 +#define MPS_STAT_RX_BG_1_LB_TRUNC_FRAME_L 0x96a8 +#define MPS_STAT_RX_BG_1_LB_TRUNC_FRAME_H 0x96ac +#define MPS_STAT_RX_BG_2_LB_TRUNC_FRAME_L 0x96b0 +#define MPS_STAT_RX_BG_2_LB_TRUNC_FRAME_H 0x96b4 +#define MPS_STAT_RX_BG_3_LB_TRUNC_FRAME_L 0x96b8 +#define MPS_STAT_RX_BG_3_LB_TRUNC_FRAME_H 0x96bc + +#define MPS_TRC_CFG_A 0x9800 + +#define TRCFIFOEMPTY_S 4 +#define TRCFIFOEMPTY_V(x) ((x) << TRCFIFOEMPTY_S) +#define TRCFIFOEMPTY_F TRCFIFOEMPTY_V(1U) + +#define TRCIGNOREDROPINPUT_S 3 +#define TRCIGNOREDROPINPUT_V(x) ((x) << TRCIGNOREDROPINPUT_S) +#define TRCIGNOREDROPINPUT_F TRCIGNOREDROPINPUT_V(1U) + +#define TRCKEEPDUPLICATES_S 2 +#define TRCKEEPDUPLICATES_V(x) ((x) << TRCKEEPDUPLICATES_S) +#define TRCKEEPDUPLICATES_F TRCKEEPDUPLICATES_V(1U) + +#define TRCEN_S 1 +#define TRCEN_V(x) ((x) << TRCEN_S) +#define TRCEN_F TRCEN_V(1U) + +#define TRCMULTIFILTER_S 0 +#define TRCMULTIFILTER_V(x) ((x) << TRCMULTIFILTER_S) +#define TRCMULTIFILTER_F TRCMULTIFILTER_V(1U) + +#define MPS_TRC_RSS_CONTROL_A 0x9808 +#define MPS_TRC_FILTER1_RSS_CONTROL_A 0x9ff4 +#define MPS_TRC_FILTER2_RSS_CONTROL_A 0x9ffc +#define MPS_TRC_FILTER3_RSS_CONTROL_A 0xa004 +#define MPS_T5_TRC_RSS_CONTROL_A 0xa00c + +#define RSSCONTROL_S 16 +#define RSSCONTROL_V(x) ((x) << RSSCONTROL_S) + +#define QUEUENUMBER_S 0 +#define QUEUENUMBER_V(x) ((x) << QUEUENUMBER_S) + +#define TFINVERTMATCH_S 24 +#define TFINVERTMATCH_V(x) ((x) << TFINVERTMATCH_S) +#define TFINVERTMATCH_F TFINVERTMATCH_V(1U) + +#define TFEN_S 22 +#define TFEN_V(x) ((x) << TFEN_S) +#define TFEN_F TFEN_V(1U) + +#define TFPORT_S 18 +#define TFPORT_M 0xfU +#define TFPORT_V(x) ((x) << TFPORT_S) +#define TFPORT_G(x) (((x) >> TFPORT_S) & TFPORT_M) + +#define TFLENGTH_S 8 +#define TFLENGTH_M 0x1fU +#define TFLENGTH_V(x) ((x) << TFLENGTH_S) +#define TFLENGTH_G(x) (((x) >> TFLENGTH_S) & TFLENGTH_M) + +#define TFOFFSET_S 0 +#define TFOFFSET_M 0x1fU +#define TFOFFSET_V(x) ((x) << TFOFFSET_S) +#define TFOFFSET_G(x) (((x) >> TFOFFSET_S) & TFOFFSET_M) + +#define T5_TFINVERTMATCH_S 25 +#define T5_TFINVERTMATCH_V(x) ((x) << T5_TFINVERTMATCH_S) +#define T5_TFINVERTMATCH_F T5_TFINVERTMATCH_V(1U) + +#define T5_TFEN_S 23 +#define T5_TFEN_V(x) ((x) << T5_TFEN_S) +#define T5_TFEN_F T5_TFEN_V(1U) + +#define T5_TFPORT_S 18 +#define T5_TFPORT_M 0x1fU +#define T5_TFPORT_V(x) ((x) << T5_TFPORT_S) +#define T5_TFPORT_G(x) (((x) >> T5_TFPORT_S) & T5_TFPORT_M) + +#define MPS_TRC_FILTER_MATCH_CTL_A_A 0x9810 +#define MPS_TRC_FILTER_MATCH_CTL_B_A 0x9820 + +#define TFMINPKTSIZE_S 16 +#define TFMINPKTSIZE_M 0x1ffU +#define TFMINPKTSIZE_V(x) ((x) << TFMINPKTSIZE_S) +#define TFMINPKTSIZE_G(x) (((x) >> TFMINPKTSIZE_S) & TFMINPKTSIZE_M) + +#define TFCAPTUREMAX_S 0 +#define TFCAPTUREMAX_M 0x3fffU +#define TFCAPTUREMAX_V(x) ((x) << TFCAPTUREMAX_S) +#define TFCAPTUREMAX_G(x) (((x) >> TFCAPTUREMAX_S) & TFCAPTUREMAX_M) + +#define MPS_TRC_FILTER0_MATCH_A 0x9c00 +#define MPS_TRC_FILTER0_DONT_CARE_A 0x9c80 +#define MPS_TRC_FILTER1_MATCH_A 0x9d00 + +#define TP_RSS_CONFIG_A 0x7df0 + +#define TNL4TUPENIPV6_S 31 +#define TNL4TUPENIPV6_V(x) ((x) << TNL4TUPENIPV6_S) +#define TNL4TUPENIPV6_F TNL4TUPENIPV6_V(1U) + +#define TNL2TUPENIPV6_S 30 +#define TNL2TUPENIPV6_V(x) ((x) << TNL2TUPENIPV6_S) +#define TNL2TUPENIPV6_F TNL2TUPENIPV6_V(1U) + +#define TNL4TUPENIPV4_S 29 +#define TNL4TUPENIPV4_V(x) ((x) << TNL4TUPENIPV4_S) +#define TNL4TUPENIPV4_F TNL4TUPENIPV4_V(1U) + +#define TNL2TUPENIPV4_S 28 +#define TNL2TUPENIPV4_V(x) ((x) << TNL2TUPENIPV4_S) +#define TNL2TUPENIPV4_F TNL2TUPENIPV4_V(1U) + +#define TNLTCPSEL_S 27 +#define TNLTCPSEL_V(x) ((x) << TNLTCPSEL_S) +#define TNLTCPSEL_F TNLTCPSEL_V(1U) + +#define TNLIP6SEL_S 26 +#define TNLIP6SEL_V(x) ((x) << TNLIP6SEL_S) +#define TNLIP6SEL_F TNLIP6SEL_V(1U) + +#define TNLVRTSEL_S 25 +#define TNLVRTSEL_V(x) ((x) << TNLVRTSEL_S) +#define TNLVRTSEL_F TNLVRTSEL_V(1U) + +#define TNLMAPEN_S 24 +#define TNLMAPEN_V(x) ((x) << TNLMAPEN_S) +#define TNLMAPEN_F TNLMAPEN_V(1U) + +#define OFDHASHSAVE_S 19 +#define OFDHASHSAVE_V(x) ((x) << OFDHASHSAVE_S) +#define OFDHASHSAVE_F OFDHASHSAVE_V(1U) + +#define OFDVRTSEL_S 18 +#define OFDVRTSEL_V(x) ((x) << OFDVRTSEL_S) +#define OFDVRTSEL_F OFDVRTSEL_V(1U) + +#define OFDMAPEN_S 17 +#define OFDMAPEN_V(x) ((x) << OFDMAPEN_S) +#define OFDMAPEN_F OFDMAPEN_V(1U) + +#define OFDLKPEN_S 16 +#define OFDLKPEN_V(x) ((x) << OFDLKPEN_S) +#define OFDLKPEN_F OFDLKPEN_V(1U) + +#define SYN4TUPENIPV6_S 15 +#define SYN4TUPENIPV6_V(x) ((x) << SYN4TUPENIPV6_S) +#define SYN4TUPENIPV6_F SYN4TUPENIPV6_V(1U) + +#define SYN2TUPENIPV6_S 14 +#define SYN2TUPENIPV6_V(x) ((x) << SYN2TUPENIPV6_S) +#define SYN2TUPENIPV6_F SYN2TUPENIPV6_V(1U) + +#define SYN4TUPENIPV4_S 13 +#define SYN4TUPENIPV4_V(x) ((x) << SYN4TUPENIPV4_S) +#define SYN4TUPENIPV4_F SYN4TUPENIPV4_V(1U) + +#define SYN2TUPENIPV4_S 12 +#define SYN2TUPENIPV4_V(x) ((x) << SYN2TUPENIPV4_S) +#define SYN2TUPENIPV4_F SYN2TUPENIPV4_V(1U) + +#define SYNIP6SEL_S 11 +#define SYNIP6SEL_V(x) ((x) << SYNIP6SEL_S) +#define SYNIP6SEL_F SYNIP6SEL_V(1U) + +#define SYNVRTSEL_S 10 +#define SYNVRTSEL_V(x) ((x) << SYNVRTSEL_S) +#define SYNVRTSEL_F SYNVRTSEL_V(1U) + +#define SYNMAPEN_S 9 +#define SYNMAPEN_V(x) ((x) << SYNMAPEN_S) +#define SYNMAPEN_F SYNMAPEN_V(1U) + +#define SYNLKPEN_S 8 +#define SYNLKPEN_V(x) ((x) << SYNLKPEN_S) +#define SYNLKPEN_F SYNLKPEN_V(1U) + +#define CHANNELENABLE_S 7 +#define CHANNELENABLE_V(x) ((x) << CHANNELENABLE_S) +#define CHANNELENABLE_F CHANNELENABLE_V(1U) + +#define PORTENABLE_S 6 +#define PORTENABLE_V(x) ((x) << PORTENABLE_S) +#define PORTENABLE_F PORTENABLE_V(1U) + +#define TNLALLLOOKUP_S 5 +#define TNLALLLOOKUP_V(x) ((x) << TNLALLLOOKUP_S) +#define TNLALLLOOKUP_F TNLALLLOOKUP_V(1U) + +#define VIRTENABLE_S 4 +#define VIRTENABLE_V(x) ((x) << VIRTENABLE_S) +#define VIRTENABLE_F VIRTENABLE_V(1U) + +#define CONGESTIONENABLE_S 3 +#define CONGESTIONENABLE_V(x) ((x) << CONGESTIONENABLE_S) +#define CONGESTIONENABLE_F CONGESTIONENABLE_V(1U) + +#define HASHTOEPLITZ_S 2 +#define HASHTOEPLITZ_V(x) ((x) << HASHTOEPLITZ_S) +#define HASHTOEPLITZ_F HASHTOEPLITZ_V(1U) + +#define UDPENABLE_S 1 +#define UDPENABLE_V(x) ((x) << UDPENABLE_S) +#define UDPENABLE_F UDPENABLE_V(1U) + +#define DISABLE_S 0 +#define DISABLE_V(x) ((x) << DISABLE_S) +#define DISABLE_F DISABLE_V(1U) + +#define TP_RSS_CONFIG_TNL_A 0x7df4 + +#define MASKSIZE_S 28 +#define MASKSIZE_M 0xfU +#define MASKSIZE_V(x) ((x) << MASKSIZE_S) +#define MASKSIZE_G(x) (((x) >> MASKSIZE_S) & MASKSIZE_M) + +#define MASKFILTER_S 16 +#define MASKFILTER_M 0x7ffU +#define MASKFILTER_V(x) ((x) << MASKFILTER_S) +#define MASKFILTER_G(x) (((x) >> MASKFILTER_S) & MASKFILTER_M) + +#define USEWIRECH_S 0 +#define USEWIRECH_V(x) ((x) << USEWIRECH_S) +#define USEWIRECH_F USEWIRECH_V(1U) + +#define HASHALL_S 2 +#define HASHALL_V(x) ((x) << HASHALL_S) +#define HASHALL_F HASHALL_V(1U) + +#define HASHETH_S 1 +#define HASHETH_V(x) ((x) << HASHETH_S) +#define HASHETH_F HASHETH_V(1U) + +#define TP_RSS_CONFIG_OFD_A 0x7df8 + +#define RRCPLMAPEN_S 20 +#define RRCPLMAPEN_V(x) ((x) << RRCPLMAPEN_S) +#define RRCPLMAPEN_F RRCPLMAPEN_V(1U) + +#define RRCPLQUEWIDTH_S 16 +#define RRCPLQUEWIDTH_M 0xfU +#define RRCPLQUEWIDTH_V(x) ((x) << RRCPLQUEWIDTH_S) +#define RRCPLQUEWIDTH_G(x) (((x) >> RRCPLQUEWIDTH_S) & RRCPLQUEWIDTH_M) + +#define TP_RSS_CONFIG_SYN_A 0x7dfc +#define TP_RSS_CONFIG_VRT_A 0x7e00 + +#define VFRDRG_S 25 +#define VFRDRG_V(x) ((x) << VFRDRG_S) +#define VFRDRG_F VFRDRG_V(1U) + +#define VFRDEN_S 24 +#define VFRDEN_V(x) ((x) << VFRDEN_S) +#define VFRDEN_F VFRDEN_V(1U) + +#define VFPERREN_S 23 +#define VFPERREN_V(x) ((x) << VFPERREN_S) +#define VFPERREN_F VFPERREN_V(1U) + +#define KEYPERREN_S 22 +#define KEYPERREN_V(x) ((x) << KEYPERREN_S) +#define KEYPERREN_F KEYPERREN_V(1U) + +#define DISABLEVLAN_S 21 +#define DISABLEVLAN_V(x) ((x) << DISABLEVLAN_S) +#define DISABLEVLAN_F DISABLEVLAN_V(1U) + +#define ENABLEUP0_S 20 +#define ENABLEUP0_V(x) ((x) << ENABLEUP0_S) +#define ENABLEUP0_F ENABLEUP0_V(1U) + +#define HASHDELAY_S 16 +#define HASHDELAY_M 0xfU +#define HASHDELAY_V(x) ((x) << HASHDELAY_S) +#define HASHDELAY_G(x) (((x) >> HASHDELAY_S) & HASHDELAY_M) + +#define VFWRADDR_S 8 +#define VFWRADDR_M 0x7fU +#define VFWRADDR_V(x) ((x) << VFWRADDR_S) +#define VFWRADDR_G(x) (((x) >> VFWRADDR_S) & VFWRADDR_M) + +#define KEYMODE_S 6 +#define KEYMODE_M 0x3U +#define KEYMODE_V(x) ((x) << KEYMODE_S) +#define KEYMODE_G(x) (((x) >> KEYMODE_S) & KEYMODE_M) + +#define VFWREN_S 5 +#define VFWREN_V(x) ((x) << VFWREN_S) +#define VFWREN_F VFWREN_V(1U) + +#define KEYWREN_S 4 +#define KEYWREN_V(x) ((x) << KEYWREN_S) +#define KEYWREN_F KEYWREN_V(1U) + +#define KEYWRADDR_S 0 +#define KEYWRADDR_M 0xfU +#define KEYWRADDR_V(x) ((x) << KEYWRADDR_S) +#define KEYWRADDR_G(x) (((x) >> KEYWRADDR_S) & KEYWRADDR_M) + +#define KEYWRADDRX_S 30 +#define KEYWRADDRX_M 0x3U +#define KEYWRADDRX_V(x) ((x) << KEYWRADDRX_S) +#define KEYWRADDRX_G(x) (((x) >> KEYWRADDRX_S) & KEYWRADDRX_M) + +#define KEYEXTEND_S 26 +#define KEYEXTEND_V(x) ((x) << KEYEXTEND_S) +#define KEYEXTEND_F KEYEXTEND_V(1U) + +#define LKPIDXSIZE_S 24 +#define LKPIDXSIZE_M 0x3U +#define LKPIDXSIZE_V(x) ((x) << LKPIDXSIZE_S) +#define LKPIDXSIZE_G(x) (((x) >> LKPIDXSIZE_S) & LKPIDXSIZE_M) + +#define TP_RSS_VFL_CONFIG_A 0x3a +#define TP_RSS_VFH_CONFIG_A 0x3b + +#define ENABLEUDPHASH_S 31 +#define ENABLEUDPHASH_V(x) ((x) << ENABLEUDPHASH_S) +#define ENABLEUDPHASH_F ENABLEUDPHASH_V(1U) + +#define VFUPEN_S 30 +#define VFUPEN_V(x) ((x) << VFUPEN_S) +#define VFUPEN_F VFUPEN_V(1U) + +#define VFVLNEX_S 28 +#define VFVLNEX_V(x) ((x) << VFVLNEX_S) +#define VFVLNEX_F VFVLNEX_V(1U) + +#define VFPRTEN_S 27 +#define VFPRTEN_V(x) ((x) << VFPRTEN_S) +#define VFPRTEN_F VFPRTEN_V(1U) + +#define VFCHNEN_S 26 +#define VFCHNEN_V(x) ((x) << VFCHNEN_S) +#define VFCHNEN_F VFCHNEN_V(1U) + +#define DEFAULTQUEUE_S 16 +#define DEFAULTQUEUE_M 0x3ffU +#define DEFAULTQUEUE_G(x) (((x) >> DEFAULTQUEUE_S) & DEFAULTQUEUE_M) + +#define VFIP6TWOTUPEN_S 6 +#define VFIP6TWOTUPEN_V(x) ((x) << VFIP6TWOTUPEN_S) +#define VFIP6TWOTUPEN_F VFIP6TWOTUPEN_V(1U) + +#define VFIP4FOURTUPEN_S 5 +#define VFIP4FOURTUPEN_V(x) ((x) << VFIP4FOURTUPEN_S) +#define VFIP4FOURTUPEN_F VFIP4FOURTUPEN_V(1U) + +#define VFIP4TWOTUPEN_S 4 +#define VFIP4TWOTUPEN_V(x) ((x) << VFIP4TWOTUPEN_S) +#define VFIP4TWOTUPEN_F VFIP4TWOTUPEN_V(1U) + +#define KEYINDEX_S 0 +#define KEYINDEX_M 0xfU +#define KEYINDEX_G(x) (((x) >> KEYINDEX_S) & KEYINDEX_M) + +#define MAPENABLE_S 31 +#define MAPENABLE_V(x) ((x) << MAPENABLE_S) +#define MAPENABLE_F MAPENABLE_V(1U) + +#define CHNENABLE_S 30 +#define CHNENABLE_V(x) ((x) << CHNENABLE_S) +#define CHNENABLE_F CHNENABLE_V(1U) + +#define LE_DB_DBGI_CONFIG_A 0x19cf0 + +#define DBGICMDBUSY_S 3 +#define DBGICMDBUSY_V(x) ((x) << DBGICMDBUSY_S) +#define DBGICMDBUSY_F DBGICMDBUSY_V(1U) + +#define DBGICMDSTRT_S 2 +#define DBGICMDSTRT_V(x) ((x) << DBGICMDSTRT_S) +#define DBGICMDSTRT_F DBGICMDSTRT_V(1U) + +#define DBGICMDMODE_S 0 +#define DBGICMDMODE_M 0x3U +#define DBGICMDMODE_V(x) ((x) << DBGICMDMODE_S) + +#define LE_DB_DBGI_REQ_TCAM_CMD_A 0x19cf4 + +#define DBGICMD_S 20 +#define DBGICMD_M 0xfU +#define DBGICMD_V(x) ((x) << DBGICMD_S) + +#define DBGITID_S 0 +#define DBGITID_M 0xfffffU +#define DBGITID_V(x) ((x) << DBGITID_S) + +#define LE_DB_DBGI_REQ_DATA_A 0x19d00 +#define LE_DB_DBGI_RSP_STATUS_A 0x19d94 + +#define LE_DB_DBGI_RSP_DATA_A 0x19da0 + +#define PRTENABLE_S 29 +#define PRTENABLE_V(x) ((x) << PRTENABLE_S) +#define PRTENABLE_F PRTENABLE_V(1U) + +#define UDPFOURTUPEN_S 28 +#define UDPFOURTUPEN_V(x) ((x) << UDPFOURTUPEN_S) +#define UDPFOURTUPEN_F UDPFOURTUPEN_V(1U) + +#define IP6FOURTUPEN_S 27 +#define IP6FOURTUPEN_V(x) ((x) << IP6FOURTUPEN_S) +#define IP6FOURTUPEN_F IP6FOURTUPEN_V(1U) + +#define IP6TWOTUPEN_S 26 +#define IP6TWOTUPEN_V(x) ((x) << IP6TWOTUPEN_S) +#define IP6TWOTUPEN_F IP6TWOTUPEN_V(1U) + +#define IP4FOURTUPEN_S 25 +#define IP4FOURTUPEN_V(x) ((x) << IP4FOURTUPEN_S) +#define IP4FOURTUPEN_F IP4FOURTUPEN_V(1U) + +#define IP4TWOTUPEN_S 24 +#define IP4TWOTUPEN_V(x) ((x) << IP4TWOTUPEN_S) +#define IP4TWOTUPEN_F IP4TWOTUPEN_V(1U) + +#define IVFWIDTH_S 20 +#define IVFWIDTH_M 0xfU +#define IVFWIDTH_V(x) ((x) << IVFWIDTH_S) +#define IVFWIDTH_G(x) (((x) >> IVFWIDTH_S) & IVFWIDTH_M) + +#define CH1DEFAULTQUEUE_S 10 +#define CH1DEFAULTQUEUE_M 0x3ffU +#define CH1DEFAULTQUEUE_V(x) ((x) << CH1DEFAULTQUEUE_S) +#define CH1DEFAULTQUEUE_G(x) (((x) >> CH1DEFAULTQUEUE_S) & CH1DEFAULTQUEUE_M) + +#define CH0DEFAULTQUEUE_S 0 +#define CH0DEFAULTQUEUE_M 0x3ffU +#define CH0DEFAULTQUEUE_V(x) ((x) << CH0DEFAULTQUEUE_S) +#define CH0DEFAULTQUEUE_G(x) (((x) >> CH0DEFAULTQUEUE_S) & CH0DEFAULTQUEUE_M) + +#define VFLKPIDX_S 8 +#define VFLKPIDX_M 0xffU +#define VFLKPIDX_G(x) (((x) >> VFLKPIDX_S) & VFLKPIDX_M) + +#define T6_VFWRADDR_S 8 +#define T6_VFWRADDR_M 0xffU +#define T6_VFWRADDR_V(x) ((x) << T6_VFWRADDR_S) +#define T6_VFWRADDR_G(x) (((x) >> T6_VFWRADDR_S) & T6_VFWRADDR_M) + +#define TP_RSS_CONFIG_CNG_A 0x7e04 +#define TP_RSS_SECRET_KEY0_A 0x40 +#define TP_RSS_PF0_CONFIG_A 0x30 +#define TP_RSS_PF_MAP_A 0x38 +#define TP_RSS_PF_MSK_A 0x39 + +#define PF1LKPIDX_S 3 + +#define PF0LKPIDX_M 0x7U + +#define PF1MSKSIZE_S 4 +#define PF1MSKSIZE_M 0xfU + +#define CHNCOUNT3_S 31 +#define CHNCOUNT3_V(x) ((x) << CHNCOUNT3_S) +#define CHNCOUNT3_F CHNCOUNT3_V(1U) + +#define CHNCOUNT2_S 30 +#define CHNCOUNT2_V(x) ((x) << CHNCOUNT2_S) +#define CHNCOUNT2_F CHNCOUNT2_V(1U) + +#define CHNCOUNT1_S 29 +#define CHNCOUNT1_V(x) ((x) << CHNCOUNT1_S) +#define CHNCOUNT1_F CHNCOUNT1_V(1U) + +#define CHNCOUNT0_S 28 +#define CHNCOUNT0_V(x) ((x) << CHNCOUNT0_S) +#define CHNCOUNT0_F CHNCOUNT0_V(1U) + +#define CHNUNDFLOW3_S 27 +#define CHNUNDFLOW3_V(x) ((x) << CHNUNDFLOW3_S) +#define CHNUNDFLOW3_F CHNUNDFLOW3_V(1U) + +#define CHNUNDFLOW2_S 26 +#define CHNUNDFLOW2_V(x) ((x) << CHNUNDFLOW2_S) +#define CHNUNDFLOW2_F CHNUNDFLOW2_V(1U) + +#define CHNUNDFLOW1_S 25 +#define CHNUNDFLOW1_V(x) ((x) << CHNUNDFLOW1_S) +#define CHNUNDFLOW1_F CHNUNDFLOW1_V(1U) + +#define CHNUNDFLOW0_S 24 +#define CHNUNDFLOW0_V(x) ((x) << CHNUNDFLOW0_S) +#define CHNUNDFLOW0_F CHNUNDFLOW0_V(1U) + +#define RSTCHN3_S 19 +#define RSTCHN3_V(x) ((x) << RSTCHN3_S) +#define RSTCHN3_F RSTCHN3_V(1U) + +#define RSTCHN2_S 18 +#define RSTCHN2_V(x) ((x) << RSTCHN2_S) +#define RSTCHN2_F RSTCHN2_V(1U) + +#define RSTCHN1_S 17 +#define RSTCHN1_V(x) ((x) << RSTCHN1_S) +#define RSTCHN1_F RSTCHN1_V(1U) + +#define RSTCHN0_S 16 +#define RSTCHN0_V(x) ((x) << RSTCHN0_S) +#define RSTCHN0_F RSTCHN0_V(1U) + +#define UPDVLD_S 15 +#define UPDVLD_V(x) ((x) << UPDVLD_S) +#define UPDVLD_F UPDVLD_V(1U) + +#define XOFF_S 14 +#define XOFF_V(x) ((x) << XOFF_S) +#define XOFF_F XOFF_V(1U) + +#define UPDCHN3_S 13 +#define UPDCHN3_V(x) ((x) << UPDCHN3_S) +#define UPDCHN3_F UPDCHN3_V(1U) + +#define UPDCHN2_S 12 +#define UPDCHN2_V(x) ((x) << UPDCHN2_S) +#define UPDCHN2_F UPDCHN2_V(1U) + +#define UPDCHN1_S 11 +#define UPDCHN1_V(x) ((x) << UPDCHN1_S) +#define UPDCHN1_F UPDCHN1_V(1U) + +#define UPDCHN0_S 10 +#define UPDCHN0_V(x) ((x) << UPDCHN0_S) +#define UPDCHN0_F UPDCHN0_V(1U) + +#define QUEUE_S 0 +#define QUEUE_M 0x3ffU +#define QUEUE_V(x) ((x) << QUEUE_S) +#define QUEUE_G(x) (((x) >> QUEUE_S) & QUEUE_M) + +#define MPS_TRC_INT_CAUSE_A 0x985c + +#define MISCPERR_S 8 +#define MISCPERR_V(x) ((x) << MISCPERR_S) +#define MISCPERR_F MISCPERR_V(1U) + +#define PKTFIFO_S 4 +#define PKTFIFO_M 0xfU +#define PKTFIFO_V(x) ((x) << PKTFIFO_S) + +#define FILTMEM_S 0 +#define FILTMEM_M 0xfU +#define FILTMEM_V(x) ((x) << FILTMEM_S) + +#define MPS_CLS_INT_CAUSE_A 0xd028 + +#define HASHSRAM_S 2 +#define HASHSRAM_V(x) ((x) << HASHSRAM_S) +#define HASHSRAM_F HASHSRAM_V(1U) + +#define MATCHTCAM_S 1 +#define MATCHTCAM_V(x) ((x) << MATCHTCAM_S) +#define MATCHTCAM_F MATCHTCAM_V(1U) + +#define MATCHSRAM_S 0 +#define MATCHSRAM_V(x) ((x) << MATCHSRAM_S) +#define MATCHSRAM_F MATCHSRAM_V(1U) + +#define MPS_RX_PG_RSV0_A 0x11010 +#define MPS_RX_PG_RSV4_A 0x11020 +#define MPS_RX_PERR_INT_CAUSE_A 0x11074 +#define MPS_RX_MAC_BG_PG_CNT0_A 0x11208 +#define MPS_RX_LPBK_BG_PG_CNT0_A 0x11218 + +#define MPS_RX_VXLAN_TYPE_A 0x11234 + +#define VXLAN_EN_S 16 +#define VXLAN_EN_V(x) ((x) << VXLAN_EN_S) +#define VXLAN_EN_F VXLAN_EN_V(1U) + +#define VXLAN_S 0 +#define VXLAN_M 0xffffU +#define VXLAN_V(x) ((x) << VXLAN_S) +#define VXLAN_G(x) (((x) >> VXLAN_S) & VXLAN_M) + +#define MPS_RX_GENEVE_TYPE_A 0x11238 + +#define GENEVE_EN_S 16 +#define GENEVE_EN_V(x) ((x) << GENEVE_EN_S) +#define GENEVE_EN_F GENEVE_EN_V(1U) + +#define GENEVE_S 0 +#define GENEVE_M 0xffffU +#define GENEVE_V(x) ((x) << GENEVE_S) +#define GENEVE_G(x) (((x) >> GENEVE_S) & GENEVE_M) + +#define MPS_CLS_TCAM_Y_L_A 0xf000 +#define MPS_CLS_TCAM_DATA0_A 0xf000 +#define MPS_CLS_TCAM_DATA1_A 0xf004 + +#define CTLREQID_S 30 +#define CTLREQID_V(x) ((x) << CTLREQID_S) + +#define MPS_VF_RPLCT_MAP0_A 0x1111c +#define MPS_VF_RPLCT_MAP1_A 0x11120 +#define MPS_VF_RPLCT_MAP2_A 0x11124 +#define MPS_VF_RPLCT_MAP3_A 0x11128 +#define MPS_VF_RPLCT_MAP4_A 0x11300 +#define MPS_VF_RPLCT_MAP5_A 0x11304 +#define MPS_VF_RPLCT_MAP6_A 0x11308 +#define MPS_VF_RPLCT_MAP7_A 0x1130c + +#define VIDL_S 16 +#define VIDL_M 0xffffU +#define VIDL_G(x) (((x) >> VIDL_S) & VIDL_M) + +#define DATALKPTYPE_S 10 +#define DATALKPTYPE_M 0x3U +#define DATALKPTYPE_G(x) (((x) >> DATALKPTYPE_S) & DATALKPTYPE_M) + +#define DATAPORTNUM_S 12 +#define DATAPORTNUM_M 0xfU +#define DATAPORTNUM_V(x) ((x) << DATAPORTNUM_S) +#define DATAPORTNUM_G(x) (((x) >> DATAPORTNUM_S) & DATAPORTNUM_M) + +#define DATALKPTYPE_S 10 +#define DATALKPTYPE_M 0x3U +#define DATALKPTYPE_V(x) ((x) << DATALKPTYPE_S) +#define DATALKPTYPE_G(x) (((x) >> DATALKPTYPE_S) & DATALKPTYPE_M) + +#define DATADIPHIT_S 8 +#define DATADIPHIT_V(x) ((x) << DATADIPHIT_S) +#define DATADIPHIT_F DATADIPHIT_V(1U) + +#define DATAVIDH2_S 7 +#define DATAVIDH2_V(x) ((x) << DATAVIDH2_S) +#define DATAVIDH2_F DATAVIDH2_V(1U) + +#define DATAVIDH1_S 0 +#define DATAVIDH1_M 0x7fU +#define DATAVIDH1_G(x) (((x) >> DATAVIDH1_S) & DATAVIDH1_M) + +#define MPS_CLS_TCAM_RDATA0_REQ_ID1_A 0xf020 +#define MPS_CLS_TCAM_RDATA1_REQ_ID1_A 0xf024 +#define MPS_CLS_TCAM_RDATA2_REQ_ID1_A 0xf028 + +#define USED_S 16 +#define USED_M 0x7ffU +#define USED_G(x) (((x) >> USED_S) & USED_M) + +#define ALLOC_S 0 +#define ALLOC_M 0x7ffU +#define ALLOC_G(x) (((x) >> ALLOC_S) & ALLOC_M) + +#define T5_USED_S 16 +#define T5_USED_M 0xfffU +#define T5_USED_G(x) (((x) >> T5_USED_S) & T5_USED_M) + +#define T5_ALLOC_S 0 +#define T5_ALLOC_M 0xfffU +#define T5_ALLOC_G(x) (((x) >> T5_ALLOC_S) & T5_ALLOC_M) + +#define DMACH_S 0 +#define DMACH_M 0xffffU +#define DMACH_G(x) (((x) >> DMACH_S) & DMACH_M) + +#define MPS_CLS_TCAM_X_L_A 0xf008 +#define MPS_CLS_TCAM_DATA2_CTL_A 0xf008 + +#define CTLCMDTYPE_S 31 +#define CTLCMDTYPE_V(x) ((x) << CTLCMDTYPE_S) +#define CTLCMDTYPE_F CTLCMDTYPE_V(1U) + +#define CTLTCAMSEL_S 25 +#define CTLTCAMSEL_V(x) ((x) << CTLTCAMSEL_S) + +#define CTLTCAMINDEX_S 17 +#define CTLTCAMINDEX_V(x) ((x) << CTLTCAMINDEX_S) + +#define CTLXYBITSEL_S 16 +#define CTLXYBITSEL_V(x) ((x) << CTLXYBITSEL_S) + +#define MPS_CLS_TCAM_Y_L(idx) (MPS_CLS_TCAM_Y_L_A + (idx) * 16) +#define NUM_MPS_CLS_TCAM_Y_L_INSTANCES 512 + +#define MPS_CLS_TCAM_X_L(idx) (MPS_CLS_TCAM_X_L_A + (idx) * 16) +#define NUM_MPS_CLS_TCAM_X_L_INSTANCES 512 + +#define MPS_CLS_SRAM_L_A 0xe000 + +#define T6_MULTILISTEN0_S 26 + +#define T6_SRAM_PRIO3_S 23 +#define T6_SRAM_PRIO3_M 0x7U +#define T6_SRAM_PRIO3_G(x) (((x) >> T6_SRAM_PRIO3_S) & T6_SRAM_PRIO3_M) + +#define T6_SRAM_PRIO2_S 20 +#define T6_SRAM_PRIO2_M 0x7U +#define T6_SRAM_PRIO2_G(x) (((x) >> T6_SRAM_PRIO2_S) & T6_SRAM_PRIO2_M) + +#define T6_SRAM_PRIO1_S 17 +#define T6_SRAM_PRIO1_M 0x7U +#define T6_SRAM_PRIO1_G(x) (((x) >> T6_SRAM_PRIO1_S) & T6_SRAM_PRIO1_M) + +#define T6_SRAM_PRIO0_S 14 +#define T6_SRAM_PRIO0_M 0x7U +#define T6_SRAM_PRIO0_G(x) (((x) >> T6_SRAM_PRIO0_S) & T6_SRAM_PRIO0_M) + +#define T6_SRAM_VLD_S 13 +#define T6_SRAM_VLD_V(x) ((x) << T6_SRAM_VLD_S) +#define T6_SRAM_VLD_F T6_SRAM_VLD_V(1U) + +#define T6_REPLICATE_S 12 +#define T6_REPLICATE_V(x) ((x) << T6_REPLICATE_S) +#define T6_REPLICATE_F T6_REPLICATE_V(1U) + +#define T6_PF_S 9 +#define T6_PF_M 0x7U +#define T6_PF_G(x) (((x) >> T6_PF_S) & T6_PF_M) + +#define T6_VF_VALID_S 8 +#define T6_VF_VALID_V(x) ((x) << T6_VF_VALID_S) +#define T6_VF_VALID_F T6_VF_VALID_V(1U) + +#define T6_VF_S 0 +#define T6_VF_M 0xffU +#define T6_VF_G(x) (((x) >> T6_VF_S) & T6_VF_M) + +#define MPS_CLS_SRAM_H_A 0xe004 + +#define MPS_CLS_SRAM_L(idx) (MPS_CLS_SRAM_L_A + (idx) * 8) +#define NUM_MPS_CLS_SRAM_L_INSTANCES 336 + +#define MPS_CLS_SRAM_H(idx) (MPS_CLS_SRAM_H_A + (idx) * 8) +#define NUM_MPS_CLS_SRAM_H_INSTANCES 336 + +#define MULTILISTEN0_S 25 + +#define REPLICATE_S 11 +#define REPLICATE_V(x) ((x) << REPLICATE_S) +#define REPLICATE_F REPLICATE_V(1U) + +#define PF_S 8 +#define PF_M 0x7U +#define PF_G(x) (((x) >> PF_S) & PF_M) + +#define VF_VALID_S 7 +#define VF_VALID_V(x) ((x) << VF_VALID_S) +#define VF_VALID_F VF_VALID_V(1U) + +#define VF_S 0 +#define VF_M 0x7fU +#define VF_G(x) (((x) >> VF_S) & VF_M) + +#define SRAM_PRIO3_S 22 +#define SRAM_PRIO3_M 0x7U +#define SRAM_PRIO3_G(x) (((x) >> SRAM_PRIO3_S) & SRAM_PRIO3_M) + +#define SRAM_PRIO2_S 19 +#define SRAM_PRIO2_M 0x7U +#define SRAM_PRIO2_G(x) (((x) >> SRAM_PRIO2_S) & SRAM_PRIO2_M) + +#define SRAM_PRIO1_S 16 +#define SRAM_PRIO1_M 0x7U +#define SRAM_PRIO1_G(x) (((x) >> SRAM_PRIO1_S) & SRAM_PRIO1_M) + +#define SRAM_PRIO0_S 13 +#define SRAM_PRIO0_M 0x7U +#define SRAM_PRIO0_G(x) (((x) >> SRAM_PRIO0_S) & SRAM_PRIO0_M) + +#define SRAM_VLD_S 12 +#define SRAM_VLD_V(x) ((x) << SRAM_VLD_S) +#define SRAM_VLD_F SRAM_VLD_V(1U) + +#define PORTMAP_S 0 +#define PORTMAP_M 0xfU +#define PORTMAP_G(x) (((x) >> PORTMAP_S) & PORTMAP_M) + +#define CPL_INTR_CAUSE_A 0x19054 + +#define CIM_OP_MAP_PERR_S 5 +#define CIM_OP_MAP_PERR_V(x) ((x) << CIM_OP_MAP_PERR_S) +#define CIM_OP_MAP_PERR_F CIM_OP_MAP_PERR_V(1U) + +#define CIM_OVFL_ERROR_S 4 +#define CIM_OVFL_ERROR_V(x) ((x) << CIM_OVFL_ERROR_S) +#define CIM_OVFL_ERROR_F CIM_OVFL_ERROR_V(1U) + +#define TP_FRAMING_ERROR_S 3 +#define TP_FRAMING_ERROR_V(x) ((x) << TP_FRAMING_ERROR_S) +#define TP_FRAMING_ERROR_F TP_FRAMING_ERROR_V(1U) + +#define SGE_FRAMING_ERROR_S 2 +#define SGE_FRAMING_ERROR_V(x) ((x) << SGE_FRAMING_ERROR_S) +#define SGE_FRAMING_ERROR_F SGE_FRAMING_ERROR_V(1U) + +#define CIM_FRAMING_ERROR_S 1 +#define CIM_FRAMING_ERROR_V(x) ((x) << CIM_FRAMING_ERROR_S) +#define CIM_FRAMING_ERROR_F CIM_FRAMING_ERROR_V(1U) + +#define ZERO_SWITCH_ERROR_S 0 +#define ZERO_SWITCH_ERROR_V(x) ((x) << ZERO_SWITCH_ERROR_S) +#define ZERO_SWITCH_ERROR_F ZERO_SWITCH_ERROR_V(1U) + +#define SMB_INT_CAUSE_A 0x19090 + +#define MSTTXFIFOPARINT_S 21 +#define MSTTXFIFOPARINT_V(x) ((x) << MSTTXFIFOPARINT_S) +#define MSTTXFIFOPARINT_F MSTTXFIFOPARINT_V(1U) + +#define MSTRXFIFOPARINT_S 20 +#define MSTRXFIFOPARINT_V(x) ((x) << MSTRXFIFOPARINT_S) +#define MSTRXFIFOPARINT_F MSTRXFIFOPARINT_V(1U) + +#define SLVFIFOPARINT_S 19 +#define SLVFIFOPARINT_V(x) ((x) << SLVFIFOPARINT_S) +#define SLVFIFOPARINT_F SLVFIFOPARINT_V(1U) + +#define ULP_RX_INT_CAUSE_A 0x19158 +#define ULP_RX_ISCSI_LLIMIT_A 0x1915c +#define ULP_RX_ISCSI_ULIMIT_A 0x19160 +#define ULP_RX_ISCSI_TAGMASK_A 0x19164 +#define ULP_RX_ISCSI_PSZ_A 0x19168 +#define ULP_RX_TDDP_LLIMIT_A 0x1916c +#define ULP_RX_TDDP_ULIMIT_A 0x19170 +#define ULP_RX_STAG_LLIMIT_A 0x1917c +#define ULP_RX_STAG_ULIMIT_A 0x19180 +#define ULP_RX_RQ_LLIMIT_A 0x19184 +#define ULP_RX_RQ_ULIMIT_A 0x19188 +#define ULP_RX_PBL_LLIMIT_A 0x1918c +#define ULP_RX_PBL_ULIMIT_A 0x19190 +#define ULP_RX_CTX_BASE_A 0x19194 +#define ULP_RX_RQUDP_LLIMIT_A 0x191a4 +#define ULP_RX_RQUDP_ULIMIT_A 0x191a8 +#define ULP_RX_LA_CTL_A 0x1923c +#define ULP_RX_LA_RDPTR_A 0x19240 +#define ULP_RX_LA_RDDATA_A 0x19244 +#define ULP_RX_LA_WRPTR_A 0x19248 +#define ULP_RX_TLS_KEY_LLIMIT_A 0x192ac +#define ULP_RX_TLS_KEY_ULIMIT_A 0x192b0 + +#define HPZ3_S 24 +#define HPZ3_V(x) ((x) << HPZ3_S) + +#define HPZ2_S 16 +#define HPZ2_V(x) ((x) << HPZ2_S) + +#define HPZ1_S 8 +#define HPZ1_V(x) ((x) << HPZ1_S) + +#define HPZ0_S 0 +#define HPZ0_V(x) ((x) << HPZ0_S) + +#define ULP_RX_TDDP_PSZ_A 0x19178 + +/* registers for module SF */ +#define SF_DATA_A 0x193f8 +#define SF_OP_A 0x193fc + +#define SF_BUSY_S 31 +#define SF_BUSY_V(x) ((x) << SF_BUSY_S) +#define SF_BUSY_F SF_BUSY_V(1U) + +#define SF_LOCK_S 4 +#define SF_LOCK_V(x) ((x) << SF_LOCK_S) +#define SF_LOCK_F SF_LOCK_V(1U) + +#define SF_CONT_S 3 +#define SF_CONT_V(x) ((x) << SF_CONT_S) +#define SF_CONT_F SF_CONT_V(1U) + +#define BYTECNT_S 1 +#define BYTECNT_V(x) ((x) << BYTECNT_S) + +#define OP_S 0 +#define OP_V(x) ((x) << OP_S) +#define OP_F OP_V(1U) + +#define PL_PF_INT_CAUSE_A 0x3c0 + +#define PFSW_S 3 +#define PFSW_V(x) ((x) << PFSW_S) +#define PFSW_F PFSW_V(1U) + +#define PFCIM_S 1 +#define PFCIM_V(x) ((x) << PFCIM_S) +#define PFCIM_F PFCIM_V(1U) + +#define PL_PF_INT_ENABLE_A 0x3c4 +#define PL_PF_CTL_A 0x3c8 + +#define PL_WHOAMI_A 0x19400 + +#define SOURCEPF_S 8 +#define SOURCEPF_M 0x7U +#define SOURCEPF_G(x) (((x) >> SOURCEPF_S) & SOURCEPF_M) + +#define T6_SOURCEPF_S 9 +#define T6_SOURCEPF_M 0x7U +#define T6_SOURCEPF_G(x) (((x) >> T6_SOURCEPF_S) & T6_SOURCEPF_M) + +#define PL_INT_CAUSE_A 0x1940c + +#define ULP_TX_S 27 +#define ULP_TX_V(x) ((x) << ULP_TX_S) +#define ULP_TX_F ULP_TX_V(1U) + +#define SGE_S 26 +#define SGE_V(x) ((x) << SGE_S) +#define SGE_F SGE_V(1U) + +#define CPL_SWITCH_S 24 +#define CPL_SWITCH_V(x) ((x) << CPL_SWITCH_S) +#define CPL_SWITCH_F CPL_SWITCH_V(1U) + +#define ULP_RX_S 23 +#define ULP_RX_V(x) ((x) << ULP_RX_S) +#define ULP_RX_F ULP_RX_V(1U) + +#define PM_RX_S 22 +#define PM_RX_V(x) ((x) << PM_RX_S) +#define PM_RX_F PM_RX_V(1U) + +#define PM_TX_S 21 +#define PM_TX_V(x) ((x) << PM_TX_S) +#define PM_TX_F PM_TX_V(1U) + +#define MA_S 20 +#define MA_V(x) ((x) << MA_S) +#define MA_F MA_V(1U) + +#define TP_S 19 +#define TP_V(x) ((x) << TP_S) +#define TP_F TP_V(1U) + +#define LE_S 18 +#define LE_V(x) ((x) << LE_S) +#define LE_F LE_V(1U) + +#define EDC1_S 17 +#define EDC1_V(x) ((x) << EDC1_S) +#define EDC1_F EDC1_V(1U) + +#define EDC0_S 16 +#define EDC0_V(x) ((x) << EDC0_S) +#define EDC0_F EDC0_V(1U) + +#define MC_S 15 +#define MC_V(x) ((x) << MC_S) +#define MC_F MC_V(1U) + +#define PCIE_S 14 +#define PCIE_V(x) ((x) << PCIE_S) +#define PCIE_F PCIE_V(1U) + +#define XGMAC_KR1_S 12 +#define XGMAC_KR1_V(x) ((x) << XGMAC_KR1_S) +#define XGMAC_KR1_F XGMAC_KR1_V(1U) + +#define XGMAC_KR0_S 11 +#define XGMAC_KR0_V(x) ((x) << XGMAC_KR0_S) +#define XGMAC_KR0_F XGMAC_KR0_V(1U) + +#define XGMAC1_S 10 +#define XGMAC1_V(x) ((x) << XGMAC1_S) +#define XGMAC1_F XGMAC1_V(1U) + +#define XGMAC0_S 9 +#define XGMAC0_V(x) ((x) << XGMAC0_S) +#define XGMAC0_F XGMAC0_V(1U) + +#define SMB_S 8 +#define SMB_V(x) ((x) << SMB_S) +#define SMB_F SMB_V(1U) + +#define SF_S 7 +#define SF_V(x) ((x) << SF_S) +#define SF_F SF_V(1U) + +#define PL_S 6 +#define PL_V(x) ((x) << PL_S) +#define PL_F PL_V(1U) + +#define NCSI_S 5 +#define NCSI_V(x) ((x) << NCSI_S) +#define NCSI_F NCSI_V(1U) + +#define MPS_S 4 +#define MPS_V(x) ((x) << MPS_S) +#define MPS_F MPS_V(1U) + +#define CIM_S 0 +#define CIM_V(x) ((x) << CIM_S) +#define CIM_F CIM_V(1U) + +#define MC1_S 31 +#define MC1_V(x) ((x) << MC1_S) +#define MC1_F MC1_V(1U) + +#define PL_INT_ENABLE_A 0x19410 +#define PL_INT_MAP0_A 0x19414 +#define PL_RST_A 0x19428 + +#define PIORST_S 1 +#define PIORST_V(x) ((x) << PIORST_S) +#define PIORST_F PIORST_V(1U) + +#define PIORSTMODE_S 0 +#define PIORSTMODE_V(x) ((x) << PIORSTMODE_S) +#define PIORSTMODE_F PIORSTMODE_V(1U) + +#define PL_PL_INT_CAUSE_A 0x19430 + +#define FATALPERR_S 4 +#define FATALPERR_V(x) ((x) << FATALPERR_S) +#define FATALPERR_F FATALPERR_V(1U) + +#define PERRVFID_S 0 +#define PERRVFID_V(x) ((x) << PERRVFID_S) +#define PERRVFID_F PERRVFID_V(1U) + +#define PL_REV_A 0x1943c + +#define REV_S 0 +#define REV_M 0xfU +#define REV_V(x) ((x) << REV_S) +#define REV_G(x) (((x) >> REV_S) & REV_M) + +#define HASHTBLMEMCRCERR_S 27 +#define HASHTBLMEMCRCERR_V(x) ((x) << HASHTBLMEMCRCERR_S) +#define HASHTBLMEMCRCERR_F HASHTBLMEMCRCERR_V(1U) + +#define CMDTIDERR_S 22 +#define CMDTIDERR_V(x) ((x) << CMDTIDERR_S) +#define CMDTIDERR_F CMDTIDERR_V(1U) + +#define T6_UNKNOWNCMD_S 3 +#define T6_UNKNOWNCMD_V(x) ((x) << T6_UNKNOWNCMD_S) +#define T6_UNKNOWNCMD_F T6_UNKNOWNCMD_V(1U) + +#define T6_LIP0_S 2 +#define T6_LIP0_V(x) ((x) << T6_LIP0_S) +#define T6_LIP0_F T6_LIP0_V(1U) + +#define T6_LIPMISS_S 1 +#define T6_LIPMISS_V(x) ((x) << T6_LIPMISS_S) +#define T6_LIPMISS_F T6_LIPMISS_V(1U) + +#define LE_DB_CONFIG_A 0x19c04 +#define LE_DB_ROUTING_TABLE_INDEX_A 0x19c10 +#define LE_DB_ACTIVE_TABLE_START_INDEX_A 0x19c10 +#define LE_DB_FILTER_TABLE_INDEX_A 0x19c14 +#define LE_DB_SERVER_INDEX_A 0x19c18 +#define LE_DB_SRVR_START_INDEX_A 0x19c18 +#define LE_DB_CLIP_TABLE_INDEX_A 0x19c1c +#define LE_DB_ACT_CNT_IPV4_A 0x19c20 +#define LE_DB_ACT_CNT_IPV6_A 0x19c24 +#define LE_DB_HASH_CONFIG_A 0x19c28 + +#define HASHTIDSIZE_S 16 +#define HASHTIDSIZE_M 0x3fU +#define HASHTIDSIZE_G(x) (((x) >> HASHTIDSIZE_S) & HASHTIDSIZE_M) + +#define HASHTBLSIZE_S 3 +#define HASHTBLSIZE_M 0x1ffffU +#define HASHTBLSIZE_G(x) (((x) >> HASHTBLSIZE_S) & HASHTBLSIZE_M) + +#define LE_DB_HASH_TID_BASE_A 0x19c30 +#define LE_DB_HASH_TBL_BASE_ADDR_A 0x19c30 +#define LE_DB_INT_CAUSE_A 0x19c3c +#define LE_DB_CLCAM_TID_BASE_A 0x19df4 +#define LE_DB_TID_HASHBASE_A 0x19df8 +#define T6_LE_DB_HASH_TID_BASE_A 0x19df8 + +#define HASHEN_S 20 +#define HASHEN_V(x) ((x) << HASHEN_S) +#define HASHEN_F HASHEN_V(1U) + +#define ASLIPCOMPEN_S 17 +#define ASLIPCOMPEN_V(x) ((x) << ASLIPCOMPEN_S) +#define ASLIPCOMPEN_F ASLIPCOMPEN_V(1U) + +#define REQQPARERR_S 16 +#define REQQPARERR_V(x) ((x) << REQQPARERR_S) +#define REQQPARERR_F REQQPARERR_V(1U) + +#define UNKNOWNCMD_S 15 +#define UNKNOWNCMD_V(x) ((x) << UNKNOWNCMD_S) +#define UNKNOWNCMD_F UNKNOWNCMD_V(1U) + +#define PARITYERR_S 6 +#define PARITYERR_V(x) ((x) << PARITYERR_S) +#define PARITYERR_F PARITYERR_V(1U) + +#define LIPMISS_S 5 +#define LIPMISS_V(x) ((x) << LIPMISS_S) +#define LIPMISS_F LIPMISS_V(1U) + +#define LIP0_S 4 +#define LIP0_V(x) ((x) << LIP0_S) +#define LIP0_F LIP0_V(1U) + +#define BASEADDR_S 3 +#define BASEADDR_M 0x1fffffffU +#define BASEADDR_G(x) (((x) >> BASEADDR_S) & BASEADDR_M) + +#define TCAMINTPERR_S 13 +#define TCAMINTPERR_V(x) ((x) << TCAMINTPERR_S) +#define TCAMINTPERR_F TCAMINTPERR_V(1U) + +#define SSRAMINTPERR_S 10 +#define SSRAMINTPERR_V(x) ((x) << SSRAMINTPERR_S) +#define SSRAMINTPERR_F SSRAMINTPERR_V(1U) + +#define LE_DB_RSP_CODE_0_A 0x19c74 + +#define TCAM_ACTV_HIT_S 0 +#define TCAM_ACTV_HIT_M 0x1fU +#define TCAM_ACTV_HIT_V(x) ((x) << TCAM_ACTV_HIT_S) +#define TCAM_ACTV_HIT_G(x) (((x) >> TCAM_ACTV_HIT_S) & TCAM_ACTV_HIT_M) + +#define LE_DB_RSP_CODE_1_A 0x19c78 + +#define HASH_ACTV_HIT_S 25 +#define HASH_ACTV_HIT_M 0x1fU +#define HASH_ACTV_HIT_V(x) ((x) << HASH_ACTV_HIT_S) +#define HASH_ACTV_HIT_G(x) (((x) >> HASH_ACTV_HIT_S) & HASH_ACTV_HIT_M) + +#define LE_3_DB_HASH_MASK_GEN_IPV4_T6_A 0x19eac +#define LE_4_DB_HASH_MASK_GEN_IPV4_T6_A 0x19eb0 + +#define NCSI_INT_CAUSE_A 0x1a0d8 + +#define CIM_DM_PRTY_ERR_S 8 +#define CIM_DM_PRTY_ERR_V(x) ((x) << CIM_DM_PRTY_ERR_S) +#define CIM_DM_PRTY_ERR_F CIM_DM_PRTY_ERR_V(1U) + +#define MPS_DM_PRTY_ERR_S 7 +#define MPS_DM_PRTY_ERR_V(x) ((x) << MPS_DM_PRTY_ERR_S) +#define MPS_DM_PRTY_ERR_F MPS_DM_PRTY_ERR_V(1U) + +#define TXFIFO_PRTY_ERR_S 1 +#define TXFIFO_PRTY_ERR_V(x) ((x) << TXFIFO_PRTY_ERR_S) +#define TXFIFO_PRTY_ERR_F TXFIFO_PRTY_ERR_V(1U) + +#define RXFIFO_PRTY_ERR_S 0 +#define RXFIFO_PRTY_ERR_V(x) ((x) << RXFIFO_PRTY_ERR_S) +#define RXFIFO_PRTY_ERR_F RXFIFO_PRTY_ERR_V(1U) + +#define XGMAC_PORT_CFG2_A 0x1018 + +#define PATEN_S 18 +#define PATEN_V(x) ((x) << PATEN_S) +#define PATEN_F PATEN_V(1U) + +#define MAGICEN_S 17 +#define MAGICEN_V(x) ((x) << MAGICEN_S) +#define MAGICEN_F MAGICEN_V(1U) + +#define XGMAC_PORT_MAGIC_MACID_LO 0x1024 +#define XGMAC_PORT_MAGIC_MACID_HI 0x1028 + +#define XGMAC_PORT_EPIO_DATA0_A 0x10c0 +#define XGMAC_PORT_EPIO_DATA1_A 0x10c4 +#define XGMAC_PORT_EPIO_DATA2_A 0x10c8 +#define XGMAC_PORT_EPIO_DATA3_A 0x10cc +#define XGMAC_PORT_EPIO_OP_A 0x10d0 + +#define EPIOWR_S 8 +#define EPIOWR_V(x) ((x) << EPIOWR_S) +#define EPIOWR_F EPIOWR_V(1U) + +#define ADDRESS_S 0 +#define ADDRESS_V(x) ((x) << ADDRESS_S) + +#define MAC_PORT_INT_CAUSE_A 0x8dc +#define XGMAC_PORT_INT_CAUSE_A 0x10dc + +#define TP_TX_MOD_QUEUE_REQ_MAP_A 0x7e28 + +#define TP_TX_MOD_QUEUE_WEIGHT0_A 0x7e30 +#define TP_TX_MOD_CHANNEL_WEIGHT_A 0x7e34 + +#define TX_MOD_QUEUE_REQ_MAP_S 0 +#define TX_MOD_QUEUE_REQ_MAP_V(x) ((x) << TX_MOD_QUEUE_REQ_MAP_S) + +#define TX_MODQ_WEIGHT3_S 24 +#define TX_MODQ_WEIGHT3_V(x) ((x) << TX_MODQ_WEIGHT3_S) + +#define TX_MODQ_WEIGHT2_S 16 +#define TX_MODQ_WEIGHT2_V(x) ((x) << TX_MODQ_WEIGHT2_S) + +#define TX_MODQ_WEIGHT1_S 8 +#define TX_MODQ_WEIGHT1_V(x) ((x) << TX_MODQ_WEIGHT1_S) + +#define TX_MODQ_WEIGHT0_S 0 +#define TX_MODQ_WEIGHT0_V(x) ((x) << TX_MODQ_WEIGHT0_S) + +#define TP_TX_SCHED_HDR_A 0x23 +#define TP_TX_SCHED_FIFO_A 0x24 +#define TP_TX_SCHED_PCMD_A 0x25 + +#define NUM_MPS_CLS_SRAM_L_INSTANCES 336 +#define NUM_MPS_T5_CLS_SRAM_L_INSTANCES 512 + +#define T5_PORT0_BASE 0x30000 +#define T5_PORT_STRIDE 0x4000 +#define T5_PORT_BASE(idx) (T5_PORT0_BASE + (idx) * T5_PORT_STRIDE) +#define T5_PORT_REG(idx, reg) (T5_PORT_BASE(idx) + (reg)) + +#define MC_0_BASE_ADDR 0x40000 +#define MC_1_BASE_ADDR 0x48000 +#define MC_STRIDE (MC_1_BASE_ADDR - MC_0_BASE_ADDR) +#define MC_REG(reg, idx) (reg + MC_STRIDE * idx) + +#define MC_P_BIST_CMD_A 0x41400 +#define MC_P_BIST_CMD_ADDR_A 0x41404 +#define MC_P_BIST_CMD_LEN_A 0x41408 +#define MC_P_BIST_DATA_PATTERN_A 0x4140c +#define MC_P_BIST_STATUS_RDATA_A 0x41488 + +#define EDC_T50_BASE_ADDR 0x50000 + +#define EDC_H_BIST_CMD_A 0x50004 +#define EDC_H_BIST_CMD_ADDR_A 0x50008 +#define EDC_H_BIST_CMD_LEN_A 0x5000c +#define EDC_H_BIST_DATA_PATTERN_A 0x50010 +#define EDC_H_BIST_STATUS_RDATA_A 0x50028 + +#define EDC_H_ECC_ERR_ADDR_A 0x50084 +#define EDC_T51_BASE_ADDR 0x50800 + +#define EDC_T5_STRIDE (EDC_T51_BASE_ADDR - EDC_T50_BASE_ADDR) +#define EDC_T5_REG(reg, idx) (reg + EDC_T5_STRIDE * idx) + +#define PL_VF_REV_A 0x4 +#define PL_VF_WHOAMI_A 0x0 +#define PL_VF_REVISION_A 0x8 + +/* registers for module CIM */ +#define CIM_HOST_ACC_CTRL_A 0x7b50 +#define CIM_HOST_ACC_DATA_A 0x7b54 +#define UP_UP_DBG_LA_CFG_A 0x140 +#define UP_UP_DBG_LA_DATA_A 0x144 + +#define HOSTBUSY_S 17 +#define HOSTBUSY_V(x) ((x) << HOSTBUSY_S) +#define HOSTBUSY_F HOSTBUSY_V(1U) + +#define HOSTWRITE_S 16 +#define HOSTWRITE_V(x) ((x) << HOSTWRITE_S) +#define HOSTWRITE_F HOSTWRITE_V(1U) + +#define CIM_IBQ_DBG_CFG_A 0x7b60 + +#define IBQDBGADDR_S 16 +#define IBQDBGADDR_M 0xfffU +#define IBQDBGADDR_V(x) ((x) << IBQDBGADDR_S) +#define IBQDBGADDR_G(x) (((x) >> IBQDBGADDR_S) & IBQDBGADDR_M) + +#define IBQDBGBUSY_S 1 +#define IBQDBGBUSY_V(x) ((x) << IBQDBGBUSY_S) +#define IBQDBGBUSY_F IBQDBGBUSY_V(1U) + +#define IBQDBGEN_S 0 +#define IBQDBGEN_V(x) ((x) << IBQDBGEN_S) +#define IBQDBGEN_F IBQDBGEN_V(1U) + +#define CIM_OBQ_DBG_CFG_A 0x7b64 + +#define OBQDBGADDR_S 16 +#define OBQDBGADDR_M 0xfffU +#define OBQDBGADDR_V(x) ((x) << OBQDBGADDR_S) +#define OBQDBGADDR_G(x) (((x) >> OBQDBGADDR_S) & OBQDBGADDR_M) + +#define OBQDBGBUSY_S 1 +#define OBQDBGBUSY_V(x) ((x) << OBQDBGBUSY_S) +#define OBQDBGBUSY_F OBQDBGBUSY_V(1U) + +#define OBQDBGEN_S 0 +#define OBQDBGEN_V(x) ((x) << OBQDBGEN_S) +#define OBQDBGEN_F OBQDBGEN_V(1U) + +#define CIM_IBQ_DBG_DATA_A 0x7b68 +#define CIM_OBQ_DBG_DATA_A 0x7b6c +#define CIM_DEBUGCFG_A 0x7b70 +#define CIM_DEBUGSTS_A 0x7b74 + +#define POLADBGRDPTR_S 23 +#define POLADBGRDPTR_M 0x1ffU +#define POLADBGRDPTR_V(x) ((x) << POLADBGRDPTR_S) + +#define POLADBGWRPTR_S 16 +#define POLADBGWRPTR_M 0x1ffU +#define POLADBGWRPTR_G(x) (((x) >> POLADBGWRPTR_S) & POLADBGWRPTR_M) + +#define PILADBGRDPTR_S 14 +#define PILADBGRDPTR_M 0x1ffU +#define PILADBGRDPTR_V(x) ((x) << PILADBGRDPTR_S) + +#define PILADBGWRPTR_S 0 +#define PILADBGWRPTR_M 0x1ffU +#define PILADBGWRPTR_G(x) (((x) >> PILADBGWRPTR_S) & PILADBGWRPTR_M) + +#define LADBGEN_S 12 +#define LADBGEN_V(x) ((x) << LADBGEN_S) +#define LADBGEN_F LADBGEN_V(1U) + +#define CIM_PO_LA_DEBUGDATA_A 0x7b78 +#define CIM_PI_LA_DEBUGDATA_A 0x7b7c +#define CIM_PO_LA_MADEBUGDATA_A 0x7b80 +#define CIM_PI_LA_MADEBUGDATA_A 0x7b84 + +#define UPDBGLARDEN_S 1 +#define UPDBGLARDEN_V(x) ((x) << UPDBGLARDEN_S) +#define UPDBGLARDEN_F UPDBGLARDEN_V(1U) + +#define UPDBGLAEN_S 0 +#define UPDBGLAEN_V(x) ((x) << UPDBGLAEN_S) +#define UPDBGLAEN_F UPDBGLAEN_V(1U) + +#define UPDBGLARDPTR_S 2 +#define UPDBGLARDPTR_M 0xfffU +#define UPDBGLARDPTR_V(x) ((x) << UPDBGLARDPTR_S) + +#define UPDBGLAWRPTR_S 16 +#define UPDBGLAWRPTR_M 0xfffU +#define UPDBGLAWRPTR_G(x) (((x) >> UPDBGLAWRPTR_S) & UPDBGLAWRPTR_M) + +#define UPDBGLACAPTPCONLY_S 30 +#define UPDBGLACAPTPCONLY_V(x) ((x) << UPDBGLACAPTPCONLY_S) +#define UPDBGLACAPTPCONLY_F UPDBGLACAPTPCONLY_V(1U) + +#define CIM_QUEUE_CONFIG_REF_A 0x7b48 +#define CIM_QUEUE_CONFIG_CTRL_A 0x7b4c + +#define CIMQSIZE_S 24 +#define CIMQSIZE_M 0x3fU +#define CIMQSIZE_G(x) (((x) >> CIMQSIZE_S) & CIMQSIZE_M) + +#define CIMQBASE_S 16 +#define CIMQBASE_M 0x3fU +#define CIMQBASE_G(x) (((x) >> CIMQBASE_S) & CIMQBASE_M) + +#define QUEFULLTHRSH_S 0 +#define QUEFULLTHRSH_M 0x1ffU +#define QUEFULLTHRSH_G(x) (((x) >> QUEFULLTHRSH_S) & QUEFULLTHRSH_M) + +#define UP_IBQ_0_RDADDR_A 0x10 +#define UP_IBQ_0_SHADOW_RDADDR_A 0x280 +#define UP_OBQ_0_REALADDR_A 0x104 +#define UP_OBQ_0_SHADOW_REALADDR_A 0x394 + +#define IBQRDADDR_S 0 +#define IBQRDADDR_M 0x1fffU +#define IBQRDADDR_G(x) (((x) >> IBQRDADDR_S) & IBQRDADDR_M) + +#define IBQWRADDR_S 0 +#define IBQWRADDR_M 0x1fffU +#define IBQWRADDR_G(x) (((x) >> IBQWRADDR_S) & IBQWRADDR_M) + +#define QUERDADDR_S 0 +#define QUERDADDR_M 0x7fffU +#define QUERDADDR_G(x) (((x) >> QUERDADDR_S) & QUERDADDR_M) + +#define QUEREMFLITS_S 0 +#define QUEREMFLITS_M 0x7ffU +#define QUEREMFLITS_G(x) (((x) >> QUEREMFLITS_S) & QUEREMFLITS_M) + +#define QUEEOPCNT_S 16 +#define QUEEOPCNT_M 0xfffU +#define QUEEOPCNT_G(x) (((x) >> QUEEOPCNT_S) & QUEEOPCNT_M) + +#define QUESOPCNT_S 0 +#define QUESOPCNT_M 0xfffU +#define QUESOPCNT_G(x) (((x) >> QUESOPCNT_S) & QUESOPCNT_M) + +#define OBQSELECT_S 4 +#define OBQSELECT_V(x) ((x) << OBQSELECT_S) +#define OBQSELECT_F OBQSELECT_V(1U) + +#define IBQSELECT_S 3 +#define IBQSELECT_V(x) ((x) << IBQSELECT_S) +#define IBQSELECT_F IBQSELECT_V(1U) + +#define QUENUMSELECT_S 0 +#define QUENUMSELECT_V(x) ((x) << QUENUMSELECT_S) + +#endif /* __T4_REGS_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_tcb.h b/drivers/net/ethernet/chelsio/cxgb4/t4_tcb.h new file mode 100644 index 000000000..22a022012 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_tcb.h @@ -0,0 +1,144 @@ +/* + * This file is part of the Chelsio T4/T5/T6 Ethernet driver for Linux. + * + * Copyright (c) 2017 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __T4_TCB_H +#define __T4_TCB_H + +#define TCB_L2T_IX_W 0 +#define TCB_L2T_IX_S 12 +#define TCB_L2T_IX_M 0xfffULL +#define TCB_L2T_IX_V(x) ((x) << TCB_L2T_IX_S) + +#define TCB_T_FLAGS_W 1 +#define TCB_T_FLAGS_S 0 +#define TCB_T_FLAGS_M 0xffffffffffffffffULL +#define TCB_T_FLAGS_V(x) ((__u64)(x) << TCB_T_FLAGS_S) + +#define TCB_FIELD_COOKIE_TFLAG 1 + +#define TCB_SMAC_SEL_W 0 +#define TCB_SMAC_SEL_S 24 +#define TCB_SMAC_SEL_M 0xffULL +#define TCB_SMAC_SEL_V(x) ((x) << TCB_SMAC_SEL_S) + +#define TCB_T_FLAGS_W 1 +#define TCB_T_FLAGS_S 0 +#define TCB_T_FLAGS_M 0xffffffffffffffffULL +#define TCB_T_FLAGS_V(x) ((__u64)(x) << TCB_T_FLAGS_S) + +#define TF_DROP_S 22 +#define TF_DIRECT_STEER_S 23 +#define TF_LPBK_S 59 + +#define TF_CCTRL_ECE_S 60 +#define TF_CCTRL_CWR_S 61 +#define TF_CCTRL_RFR_S 62 + +#define TCB_RSS_INFO_W 3 +#define TCB_RSS_INFO_S 0 +#define TCB_RSS_INFO_M 0x3ffULL +#define TCB_RSS_INFO_V(x) ((x) << TCB_RSS_INFO_S) + +#define TCB_T_STATE_W 3 +#define TCB_T_STATE_S 16 +#define TCB_T_STATE_M 0xfULL +#define TCB_T_STATE_V(x) ((x) << TCB_T_STATE_S) + +#define TCB_TIMESTAMP_W 5 +#define TCB_TIMESTAMP_S 0 +#define TCB_TIMESTAMP_M 0xffffffffULL +#define TCB_TIMESTAMP_V(x) ((x) << TCB_TIMESTAMP_S) + +#define TCB_RTT_TS_RECENT_AGE_W 6 +#define TCB_RTT_TS_RECENT_AGE_S 0 +#define TCB_RTT_TS_RECENT_AGE_M 0xffffffffULL +#define TCB_RTT_TS_RECENT_AGE_V(x) ((x) << TCB_RTT_TS_RECENT_AGE_S) + +#define TCB_T_RTSEQ_RECENT_W 7 +#define TCB_T_RTSEQ_RECENT_S 0 +#define TCB_T_RTSEQ_RECENT_M 0xffffffffULL +#define TCB_T_RTSEQ_RECENT_V(x) ((x) << TCB_T_RTSEQ_RECENT_S) + +#define TCB_TX_MAX_W 9 +#define TCB_TX_MAX_S 0 +#define TCB_TX_MAX_M 0xffffffffULL +#define TCB_TX_MAX_V(x) ((x) << TCB_TX_MAX_S) + +#define TCB_SND_UNA_RAW_W 10 +#define TCB_SND_UNA_RAW_S 0 +#define TCB_SND_UNA_RAW_M 0xfffffffULL +#define TCB_SND_UNA_RAW_V(x) ((x) << TCB_SND_UNA_RAW_S) + +#define TCB_SND_NXT_RAW_W 10 +#define TCB_SND_NXT_RAW_S 28 +#define TCB_SND_NXT_RAW_M 0xfffffffULL +#define TCB_SND_NXT_RAW_V(x) ((x) << TCB_SND_NXT_RAW_S) + +#define TCB_SND_MAX_RAW_W 11 +#define TCB_SND_MAX_RAW_S 24 +#define TCB_SND_MAX_RAW_M 0xfffffffULL +#define TCB_SND_MAX_RAW_V(x) ((x) << TCB_SND_MAX_RAW_S) + +#define TCB_RCV_NXT_W 16 +#define TCB_RCV_NXT_S 10 +#define TCB_RCV_NXT_M 0xffffffffULL +#define TCB_RCV_NXT_V(x) ((x) << TCB_RCV_NXT_S) + +#define TCB_RCV_WND_W 17 +#define TCB_RCV_WND_S 10 +#define TCB_RCV_WND_M 0xffffffULL +#define TCB_RCV_WND_V(x) ((x) << TCB_RCV_WND_S) + +#define TCB_RX_FRAG2_PTR_RAW_W 27 +#define TCB_RX_FRAG3_LEN_RAW_W 29 +#define TCB_RX_FRAG3_START_IDX_OFFSET_RAW_W 30 +#define TCB_PDU_HDR_LEN_W 31 + +#define TCB_RQ_START_W 30 +#define TCB_RQ_START_S 0 +#define TCB_RQ_START_M 0x3ffffffULL +#define TCB_RQ_START_V(x) ((x) << TCB_RQ_START_S) + +#define TF_RX_PDU_OUT_S 49 +#define TF_RX_PDU_OUT_V(x) ((__u64)(x) << TF_RX_PDU_OUT_S) + +#define TF_CORE_BYPASS_S 63 +#define TF_CORE_BYPASS_V(x) ((__u64)(x) << TF_CORE_BYPASS_S) +#define TF_CORE_BYPASS_F TF_CORE_BYPASS_V(1) + +#define TF_NON_OFFLOAD_S 1 +#define TF_NON_OFFLOAD_V(x) ((x) << TF_NON_OFFLOAD_S) +#define TF_NON_OFFLOAD_F TF_NON_OFFLOAD_V(1) + +#endif /* __T4_TCB_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_values.h b/drivers/net/ethernet/chelsio/cxgb4/t4_values.h new file mode 100644 index 000000000..eb1aa8214 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_values.h @@ -0,0 +1,162 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __T4_VALUES_H__ +#define __T4_VALUES_H__ + +/* This file contains definitions for various T4 register value hardware + * constants. The types of values encoded here are predominantly those for + * register fields which control "modal" behavior. For the most part, we do + * not include definitions for register fields which are simple numeric + * metrics, etc. + */ + +/* SGE register field values. + */ + +/* CONTROL1 register */ +#define RXPKTCPLMODE_SPLIT_X 1 + +#define INGPCIEBOUNDARY_SHIFT_X 5 +#define INGPCIEBOUNDARY_32B_X 0 + +#define INGPADBOUNDARY_SHIFT_X 5 + +#define T6_INGPADBOUNDARY_SHIFT_X 3 +#define T6_INGPADBOUNDARY_8B_X 0 +#define T6_INGPADBOUNDARY_32B_X 2 + +#define INGPADBOUNDARY_32B_X 0 + +/* CONTROL2 register */ +#define INGPACKBOUNDARY_SHIFT_X 5 +#define INGPACKBOUNDARY_16B_X 0 +#define INGPACKBOUNDARY_64B_X 1 + +/* GTS register */ +#define SGE_TIMERREGS 6 +#define TIMERREG_COUNTER0_X 0 + +#define FETCHBURSTMIN_64B_X 2 +#define FETCHBURSTMIN_128B_X 3 + +/* T6 and later use a single-bit encoding for FetchBurstMin */ +#define FETCHBURSTMIN_64B_T6_X 0 +#define FETCHBURSTMIN_128B_T6_X 1 + +#define FETCHBURSTMAX_256B_X 2 +#define FETCHBURSTMAX_512B_X 3 + +#define HOSTFCMODE_INGRESS_QUEUE_X 1 +#define HOSTFCMODE_STATUS_PAGE_X 2 + +#define CIDXFLUSHTHRESH_32_X 5 +#define CIDXFLUSHTHRESH_128_X 7 + +#define UPDATEDELIVERY_INTERRUPT_X 1 + +#define RSPD_TYPE_FLBUF_X 0 +#define RSPD_TYPE_CPL_X 1 +#define RSPD_TYPE_INTR_X 2 + +/* Congestion Manager Definitions. + */ +#define CONMCTXT_CNGTPMODE_S 19 +#define CONMCTXT_CNGTPMODE_V(x) ((x) << CONMCTXT_CNGTPMODE_S) +#define CONMCTXT_CNGCHMAP_S 0 +#define CONMCTXT_CNGCHMAP_V(x) ((x) << CONMCTXT_CNGCHMAP_S) +#define CONMCTXT_CNGTPMODE_CHANNEL_X 2 +#define CONMCTXT_CNGTPMODE_QUEUE_X 1 + +/* T5 and later support a new BAR2-based doorbell mechanism for Egress Queues. + * The User Doorbells are each 128 bytes in length with a Simple Doorbell at + * offsets 8x and a Write Combining single 64-byte Egress Queue Unit + * (IDXSIZE_UNIT_X) Gather Buffer interface at offset 64. For Ingress Queues, + * we have a Going To Sleep register at offsets 8x+4. + * + * As noted above, we have many instances of the Simple Doorbell and Going To + * Sleep registers at offsets 8x and 8x+4, respectively. We want to use a + * non-64-byte aligned offset for the Simple Doorbell in order to attempt to + * avoid buffering of the writes to the Simple Doorbell and we want to use a + * non-contiguous offset for the Going To Sleep writes in order to avoid + * possible combining between them. + */ +#define SGE_UDB_SIZE 128 +#define SGE_UDB_KDOORBELL 8 +#define SGE_UDB_GTS 20 +#define SGE_UDB_WCDOORBELL 64 + +/* CIM register field values. + */ +#define X_MBOWNER_FW 1 +#define X_MBOWNER_PL 2 + +/* PCI-E definitions */ +#define WINDOW_SHIFT_X 10 +#define PCIEOFST_SHIFT_X 10 + +/* TP_VLAN_PRI_MAP controls which subset of fields will be present in the + * Compressed Filter Tuple for LE filters. Each bit set in TP_VLAN_PRI_MAP + * selects for a particular field being present. These fields, when present + * in the Compressed Filter Tuple, have the following widths in bits. + */ +#define FT_FCOE_W 1 +#define FT_PORT_W 3 +#define FT_VNIC_ID_W 17 +#define FT_VLAN_W 17 +#define FT_TOS_W 8 +#define FT_PROTOCOL_W 8 +#define FT_ETHERTYPE_W 16 +#define FT_MACMATCH_W 9 +#define FT_MPSHITTYPE_W 3 +#define FT_FRAGMENTATION_W 1 + +/* Some of the Compressed Filter Tuple fields have internal structure. These + * bit shifts/masks describe those structures. All shifts are relative to the + * base position of the fields within the Compressed Filter Tuple + */ +#define FT_VLAN_VLD_S 16 +#define FT_VLAN_VLD_V(x) ((x) << FT_VLAN_VLD_S) +#define FT_VLAN_VLD_F FT_VLAN_VLD_V(1U) + +#define FT_VNID_ID_VF_S 0 +#define FT_VNID_ID_VF_V(x) ((x) << FT_VNID_ID_VF_S) + +#define FT_VNID_ID_PF_S 7 +#define FT_VNID_ID_PF_V(x) ((x) << FT_VNID_ID_PF_S) + +#define FT_VNID_ID_VLD_S 16 +#define FT_VNID_ID_VLD_V(x) ((x) << FT_VNID_ID_VLD_S) + +#endif /* __T4_VALUES_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h new file mode 100644 index 000000000..2419459a0 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h @@ -0,0 +1,4187 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2009-2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _T4FW_INTERFACE_H_ +#define _T4FW_INTERFACE_H_ + +enum fw_retval { + FW_SUCCESS = 0, /* completed successfully */ + FW_EPERM = 1, /* operation not permitted */ + FW_ENOENT = 2, /* no such file or directory */ + FW_EIO = 5, /* input/output error; hw bad */ + FW_ENOEXEC = 8, /* exec format error; inv microcode */ + FW_EAGAIN = 11, /* try again */ + FW_ENOMEM = 12, /* out of memory */ + FW_EFAULT = 14, /* bad address; fw bad */ + FW_EBUSY = 16, /* resource busy */ + FW_EEXIST = 17, /* file exists */ + FW_ENODEV = 19, /* no such device */ + FW_EINVAL = 22, /* invalid argument */ + FW_ENOSPC = 28, /* no space left on device */ + FW_ENOSYS = 38, /* functionality not implemented */ + FW_ENODATA = 61, /* no data available */ + FW_EPROTO = 71, /* protocol error */ + FW_EADDRINUSE = 98, /* address already in use */ + FW_EADDRNOTAVAIL = 99, /* cannot assigned requested address */ + FW_ENETDOWN = 100, /* network is down */ + FW_ENETUNREACH = 101, /* network is unreachable */ + FW_ENOBUFS = 105, /* no buffer space available */ + FW_ETIMEDOUT = 110, /* timeout */ + FW_EINPROGRESS = 115, /* fw internal */ + FW_SCSI_ABORT_REQUESTED = 128, /* */ + FW_SCSI_ABORT_TIMEDOUT = 129, /* */ + FW_SCSI_ABORTED = 130, /* */ + FW_SCSI_CLOSE_REQUESTED = 131, /* */ + FW_ERR_LINK_DOWN = 132, /* */ + FW_RDEV_NOT_READY = 133, /* */ + FW_ERR_RDEV_LOST = 134, /* */ + FW_ERR_RDEV_LOGO = 135, /* */ + FW_FCOE_NO_XCHG = 136, /* */ + FW_SCSI_RSP_ERR = 137, /* */ + FW_ERR_RDEV_IMPL_LOGO = 138, /* */ + FW_SCSI_UNDER_FLOW_ERR = 139, /* */ + FW_SCSI_OVER_FLOW_ERR = 140, /* */ + FW_SCSI_DDP_ERR = 141, /* DDP error*/ + FW_SCSI_TASK_ERR = 142, /* No SCSI tasks available */ +}; + +#define FW_T4VF_SGE_BASE_ADDR 0x0000 +#define FW_T4VF_MPS_BASE_ADDR 0x0100 +#define FW_T4VF_PL_BASE_ADDR 0x0200 +#define FW_T4VF_MBDATA_BASE_ADDR 0x0240 +#define FW_T4VF_CIM_BASE_ADDR 0x0300 + +enum fw_wr_opcodes { + FW_FILTER_WR = 0x02, + FW_ULPTX_WR = 0x04, + FW_TP_WR = 0x05, + FW_ETH_TX_PKT_WR = 0x08, + FW_ETH_TX_EO_WR = 0x1c, + FW_OFLD_CONNECTION_WR = 0x2f, + FW_FLOWC_WR = 0x0a, + FW_OFLD_TX_DATA_WR = 0x0b, + FW_CMD_WR = 0x10, + FW_ETH_TX_PKT_VM_WR = 0x11, + FW_RI_RES_WR = 0x0c, + FW_RI_INIT_WR = 0x0d, + FW_RI_RDMA_WRITE_WR = 0x14, + FW_RI_SEND_WR = 0x15, + FW_RI_RDMA_READ_WR = 0x16, + FW_RI_RECV_WR = 0x17, + FW_RI_BIND_MW_WR = 0x18, + FW_RI_FR_NSMR_WR = 0x19, + FW_RI_FR_NSMR_TPTE_WR = 0x20, + FW_RI_RDMA_WRITE_CMPL_WR = 0x21, + FW_RI_INV_LSTAG_WR = 0x1a, + FW_ISCSI_TX_DATA_WR = 0x45, + FW_PTP_TX_PKT_WR = 0x46, + FW_TLSTX_DATA_WR = 0x68, + FW_CRYPTO_LOOKASIDE_WR = 0X6d, + FW_LASTC2E_WR = 0x70, + FW_FILTER2_WR = 0x77 +}; + +struct fw_wr_hdr { + __be32 hi; + __be32 lo; +}; + +/* work request opcode (hi) */ +#define FW_WR_OP_S 24 +#define FW_WR_OP_M 0xff +#define FW_WR_OP_V(x) ((x) << FW_WR_OP_S) +#define FW_WR_OP_G(x) (((x) >> FW_WR_OP_S) & FW_WR_OP_M) + +/* atomic flag (hi) - firmware encapsulates CPLs in CPL_BARRIER */ +#define FW_WR_ATOMIC_S 23 +#define FW_WR_ATOMIC_V(x) ((x) << FW_WR_ATOMIC_S) + +/* flush flag (hi) - firmware flushes flushable work request buffered + * in the flow context. + */ +#define FW_WR_FLUSH_S 22 +#define FW_WR_FLUSH_V(x) ((x) << FW_WR_FLUSH_S) + +/* completion flag (hi) - firmware generates a cpl_fw6_ack */ +#define FW_WR_COMPL_S 21 +#define FW_WR_COMPL_V(x) ((x) << FW_WR_COMPL_S) +#define FW_WR_COMPL_F FW_WR_COMPL_V(1U) + +/* work request immediate data length (hi) */ +#define FW_WR_IMMDLEN_S 0 +#define FW_WR_IMMDLEN_M 0xff +#define FW_WR_IMMDLEN_V(x) ((x) << FW_WR_IMMDLEN_S) + +/* egress queue status update to associated ingress queue entry (lo) */ +#define FW_WR_EQUIQ_S 31 +#define FW_WR_EQUIQ_V(x) ((x) << FW_WR_EQUIQ_S) +#define FW_WR_EQUIQ_F FW_WR_EQUIQ_V(1U) + +/* egress queue status update to egress queue status entry (lo) */ +#define FW_WR_EQUEQ_S 30 +#define FW_WR_EQUEQ_V(x) ((x) << FW_WR_EQUEQ_S) +#define FW_WR_EQUEQ_F FW_WR_EQUEQ_V(1U) + +/* flow context identifier (lo) */ +#define FW_WR_FLOWID_S 8 +#define FW_WR_FLOWID_V(x) ((x) << FW_WR_FLOWID_S) + +/* length in units of 16-bytes (lo) */ +#define FW_WR_LEN16_S 0 +#define FW_WR_LEN16_V(x) ((x) << FW_WR_LEN16_S) + +#define HW_TPL_FR_MT_PR_IV_P_FC 0X32B +#define HW_TPL_FR_MT_PR_OV_P_FC 0X327 + +/* filter wr reply code in cookie in CPL_SET_TCB_RPL */ +enum fw_filter_wr_cookie { + FW_FILTER_WR_SUCCESS, + FW_FILTER_WR_FLT_ADDED, + FW_FILTER_WR_FLT_DELETED, + FW_FILTER_WR_SMT_TBL_FULL, + FW_FILTER_WR_EINVAL, +}; + +struct fw_filter_wr { + __be32 op_pkd; + __be32 len16_pkd; + __be64 r3; + __be32 tid_to_iq; + __be32 del_filter_to_l2tix; + __be16 ethtype; + __be16 ethtypem; + __u8 frag_to_ovlan_vldm; + __u8 smac_sel; + __be16 rx_chan_rx_rpl_iq; + __be32 maci_to_matchtypem; + __u8 ptcl; + __u8 ptclm; + __u8 ttyp; + __u8 ttypm; + __be16 ivlan; + __be16 ivlanm; + __be16 ovlan; + __be16 ovlanm; + __u8 lip[16]; + __u8 lipm[16]; + __u8 fip[16]; + __u8 fipm[16]; + __be16 lp; + __be16 lpm; + __be16 fp; + __be16 fpm; + __be16 r7; + __u8 sma[6]; +}; + +struct fw_filter2_wr { + __be32 op_pkd; + __be32 len16_pkd; + __be64 r3; + __be32 tid_to_iq; + __be32 del_filter_to_l2tix; + __be16 ethtype; + __be16 ethtypem; + __u8 frag_to_ovlan_vldm; + __u8 smac_sel; + __be16 rx_chan_rx_rpl_iq; + __be32 maci_to_matchtypem; + __u8 ptcl; + __u8 ptclm; + __u8 ttyp; + __u8 ttypm; + __be16 ivlan; + __be16 ivlanm; + __be16 ovlan; + __be16 ovlanm; + __u8 lip[16]; + __u8 lipm[16]; + __u8 fip[16]; + __u8 fipm[16]; + __be16 lp; + __be16 lpm; + __be16 fp; + __be16 fpm; + __be16 r7; + __u8 sma[6]; + __be16 r8; + __u8 filter_type_swapmac; + __u8 natmode_to_ulp_type; + __be16 newlport; + __be16 newfport; + __u8 newlip[16]; + __u8 newfip[16]; + __be32 natseqcheck; + __be32 r9; + __be64 r10; + __be64 r11; + __be64 r12; + __be64 r13; +}; + +#define FW_FILTER_WR_TID_S 12 +#define FW_FILTER_WR_TID_M 0xfffff +#define FW_FILTER_WR_TID_V(x) ((x) << FW_FILTER_WR_TID_S) +#define FW_FILTER_WR_TID_G(x) \ + (((x) >> FW_FILTER_WR_TID_S) & FW_FILTER_WR_TID_M) + +#define FW_FILTER_WR_RQTYPE_S 11 +#define FW_FILTER_WR_RQTYPE_M 0x1 +#define FW_FILTER_WR_RQTYPE_V(x) ((x) << FW_FILTER_WR_RQTYPE_S) +#define FW_FILTER_WR_RQTYPE_G(x) \ + (((x) >> FW_FILTER_WR_RQTYPE_S) & FW_FILTER_WR_RQTYPE_M) +#define FW_FILTER_WR_RQTYPE_F FW_FILTER_WR_RQTYPE_V(1U) + +#define FW_FILTER_WR_NOREPLY_S 10 +#define FW_FILTER_WR_NOREPLY_M 0x1 +#define FW_FILTER_WR_NOREPLY_V(x) ((x) << FW_FILTER_WR_NOREPLY_S) +#define FW_FILTER_WR_NOREPLY_G(x) \ + (((x) >> FW_FILTER_WR_NOREPLY_S) & FW_FILTER_WR_NOREPLY_M) +#define FW_FILTER_WR_NOREPLY_F FW_FILTER_WR_NOREPLY_V(1U) + +#define FW_FILTER_WR_IQ_S 0 +#define FW_FILTER_WR_IQ_M 0x3ff +#define FW_FILTER_WR_IQ_V(x) ((x) << FW_FILTER_WR_IQ_S) +#define FW_FILTER_WR_IQ_G(x) \ + (((x) >> FW_FILTER_WR_IQ_S) & FW_FILTER_WR_IQ_M) + +#define FW_FILTER_WR_DEL_FILTER_S 31 +#define FW_FILTER_WR_DEL_FILTER_M 0x1 +#define FW_FILTER_WR_DEL_FILTER_V(x) ((x) << FW_FILTER_WR_DEL_FILTER_S) +#define FW_FILTER_WR_DEL_FILTER_G(x) \ + (((x) >> FW_FILTER_WR_DEL_FILTER_S) & FW_FILTER_WR_DEL_FILTER_M) +#define FW_FILTER_WR_DEL_FILTER_F FW_FILTER_WR_DEL_FILTER_V(1U) + +#define FW_FILTER_WR_RPTTID_S 25 +#define FW_FILTER_WR_RPTTID_M 0x1 +#define FW_FILTER_WR_RPTTID_V(x) ((x) << FW_FILTER_WR_RPTTID_S) +#define FW_FILTER_WR_RPTTID_G(x) \ + (((x) >> FW_FILTER_WR_RPTTID_S) & FW_FILTER_WR_RPTTID_M) +#define FW_FILTER_WR_RPTTID_F FW_FILTER_WR_RPTTID_V(1U) + +#define FW_FILTER_WR_DROP_S 24 +#define FW_FILTER_WR_DROP_M 0x1 +#define FW_FILTER_WR_DROP_V(x) ((x) << FW_FILTER_WR_DROP_S) +#define FW_FILTER_WR_DROP_G(x) \ + (((x) >> FW_FILTER_WR_DROP_S) & FW_FILTER_WR_DROP_M) +#define FW_FILTER_WR_DROP_F FW_FILTER_WR_DROP_V(1U) + +#define FW_FILTER_WR_DIRSTEER_S 23 +#define FW_FILTER_WR_DIRSTEER_M 0x1 +#define FW_FILTER_WR_DIRSTEER_V(x) ((x) << FW_FILTER_WR_DIRSTEER_S) +#define FW_FILTER_WR_DIRSTEER_G(x) \ + (((x) >> FW_FILTER_WR_DIRSTEER_S) & FW_FILTER_WR_DIRSTEER_M) +#define FW_FILTER_WR_DIRSTEER_F FW_FILTER_WR_DIRSTEER_V(1U) + +#define FW_FILTER_WR_MASKHASH_S 22 +#define FW_FILTER_WR_MASKHASH_M 0x1 +#define FW_FILTER_WR_MASKHASH_V(x) ((x) << FW_FILTER_WR_MASKHASH_S) +#define FW_FILTER_WR_MASKHASH_G(x) \ + (((x) >> FW_FILTER_WR_MASKHASH_S) & FW_FILTER_WR_MASKHASH_M) +#define FW_FILTER_WR_MASKHASH_F FW_FILTER_WR_MASKHASH_V(1U) + +#define FW_FILTER_WR_DIRSTEERHASH_S 21 +#define FW_FILTER_WR_DIRSTEERHASH_M 0x1 +#define FW_FILTER_WR_DIRSTEERHASH_V(x) ((x) << FW_FILTER_WR_DIRSTEERHASH_S) +#define FW_FILTER_WR_DIRSTEERHASH_G(x) \ + (((x) >> FW_FILTER_WR_DIRSTEERHASH_S) & FW_FILTER_WR_DIRSTEERHASH_M) +#define FW_FILTER_WR_DIRSTEERHASH_F FW_FILTER_WR_DIRSTEERHASH_V(1U) + +#define FW_FILTER_WR_LPBK_S 20 +#define FW_FILTER_WR_LPBK_M 0x1 +#define FW_FILTER_WR_LPBK_V(x) ((x) << FW_FILTER_WR_LPBK_S) +#define FW_FILTER_WR_LPBK_G(x) \ + (((x) >> FW_FILTER_WR_LPBK_S) & FW_FILTER_WR_LPBK_M) +#define FW_FILTER_WR_LPBK_F FW_FILTER_WR_LPBK_V(1U) + +#define FW_FILTER_WR_DMAC_S 19 +#define FW_FILTER_WR_DMAC_M 0x1 +#define FW_FILTER_WR_DMAC_V(x) ((x) << FW_FILTER_WR_DMAC_S) +#define FW_FILTER_WR_DMAC_G(x) \ + (((x) >> FW_FILTER_WR_DMAC_S) & FW_FILTER_WR_DMAC_M) +#define FW_FILTER_WR_DMAC_F FW_FILTER_WR_DMAC_V(1U) + +#define FW_FILTER_WR_SMAC_S 18 +#define FW_FILTER_WR_SMAC_M 0x1 +#define FW_FILTER_WR_SMAC_V(x) ((x) << FW_FILTER_WR_SMAC_S) +#define FW_FILTER_WR_SMAC_G(x) \ + (((x) >> FW_FILTER_WR_SMAC_S) & FW_FILTER_WR_SMAC_M) +#define FW_FILTER_WR_SMAC_F FW_FILTER_WR_SMAC_V(1U) + +#define FW_FILTER_WR_INSVLAN_S 17 +#define FW_FILTER_WR_INSVLAN_M 0x1 +#define FW_FILTER_WR_INSVLAN_V(x) ((x) << FW_FILTER_WR_INSVLAN_S) +#define FW_FILTER_WR_INSVLAN_G(x) \ + (((x) >> FW_FILTER_WR_INSVLAN_S) & FW_FILTER_WR_INSVLAN_M) +#define FW_FILTER_WR_INSVLAN_F FW_FILTER_WR_INSVLAN_V(1U) + +#define FW_FILTER_WR_RMVLAN_S 16 +#define FW_FILTER_WR_RMVLAN_M 0x1 +#define FW_FILTER_WR_RMVLAN_V(x) ((x) << FW_FILTER_WR_RMVLAN_S) +#define FW_FILTER_WR_RMVLAN_G(x) \ + (((x) >> FW_FILTER_WR_RMVLAN_S) & FW_FILTER_WR_RMVLAN_M) +#define FW_FILTER_WR_RMVLAN_F FW_FILTER_WR_RMVLAN_V(1U) + +#define FW_FILTER_WR_HITCNTS_S 15 +#define FW_FILTER_WR_HITCNTS_M 0x1 +#define FW_FILTER_WR_HITCNTS_V(x) ((x) << FW_FILTER_WR_HITCNTS_S) +#define FW_FILTER_WR_HITCNTS_G(x) \ + (((x) >> FW_FILTER_WR_HITCNTS_S) & FW_FILTER_WR_HITCNTS_M) +#define FW_FILTER_WR_HITCNTS_F FW_FILTER_WR_HITCNTS_V(1U) + +#define FW_FILTER_WR_TXCHAN_S 13 +#define FW_FILTER_WR_TXCHAN_M 0x3 +#define FW_FILTER_WR_TXCHAN_V(x) ((x) << FW_FILTER_WR_TXCHAN_S) +#define FW_FILTER_WR_TXCHAN_G(x) \ + (((x) >> FW_FILTER_WR_TXCHAN_S) & FW_FILTER_WR_TXCHAN_M) + +#define FW_FILTER_WR_PRIO_S 12 +#define FW_FILTER_WR_PRIO_M 0x1 +#define FW_FILTER_WR_PRIO_V(x) ((x) << FW_FILTER_WR_PRIO_S) +#define FW_FILTER_WR_PRIO_G(x) \ + (((x) >> FW_FILTER_WR_PRIO_S) & FW_FILTER_WR_PRIO_M) +#define FW_FILTER_WR_PRIO_F FW_FILTER_WR_PRIO_V(1U) + +#define FW_FILTER_WR_L2TIX_S 0 +#define FW_FILTER_WR_L2TIX_M 0xfff +#define FW_FILTER_WR_L2TIX_V(x) ((x) << FW_FILTER_WR_L2TIX_S) +#define FW_FILTER_WR_L2TIX_G(x) \ + (((x) >> FW_FILTER_WR_L2TIX_S) & FW_FILTER_WR_L2TIX_M) + +#define FW_FILTER_WR_FRAG_S 7 +#define FW_FILTER_WR_FRAG_M 0x1 +#define FW_FILTER_WR_FRAG_V(x) ((x) << FW_FILTER_WR_FRAG_S) +#define FW_FILTER_WR_FRAG_G(x) \ + (((x) >> FW_FILTER_WR_FRAG_S) & FW_FILTER_WR_FRAG_M) +#define FW_FILTER_WR_FRAG_F FW_FILTER_WR_FRAG_V(1U) + +#define FW_FILTER_WR_FRAGM_S 6 +#define FW_FILTER_WR_FRAGM_M 0x1 +#define FW_FILTER_WR_FRAGM_V(x) ((x) << FW_FILTER_WR_FRAGM_S) +#define FW_FILTER_WR_FRAGM_G(x) \ + (((x) >> FW_FILTER_WR_FRAGM_S) & FW_FILTER_WR_FRAGM_M) +#define FW_FILTER_WR_FRAGM_F FW_FILTER_WR_FRAGM_V(1U) + +#define FW_FILTER_WR_IVLAN_VLD_S 5 +#define FW_FILTER_WR_IVLAN_VLD_M 0x1 +#define FW_FILTER_WR_IVLAN_VLD_V(x) ((x) << FW_FILTER_WR_IVLAN_VLD_S) +#define FW_FILTER_WR_IVLAN_VLD_G(x) \ + (((x) >> FW_FILTER_WR_IVLAN_VLD_S) & FW_FILTER_WR_IVLAN_VLD_M) +#define FW_FILTER_WR_IVLAN_VLD_F FW_FILTER_WR_IVLAN_VLD_V(1U) + +#define FW_FILTER_WR_OVLAN_VLD_S 4 +#define FW_FILTER_WR_OVLAN_VLD_M 0x1 +#define FW_FILTER_WR_OVLAN_VLD_V(x) ((x) << FW_FILTER_WR_OVLAN_VLD_S) +#define FW_FILTER_WR_OVLAN_VLD_G(x) \ + (((x) >> FW_FILTER_WR_OVLAN_VLD_S) & FW_FILTER_WR_OVLAN_VLD_M) +#define FW_FILTER_WR_OVLAN_VLD_F FW_FILTER_WR_OVLAN_VLD_V(1U) + +#define FW_FILTER_WR_IVLAN_VLDM_S 3 +#define FW_FILTER_WR_IVLAN_VLDM_M 0x1 +#define FW_FILTER_WR_IVLAN_VLDM_V(x) ((x) << FW_FILTER_WR_IVLAN_VLDM_S) +#define FW_FILTER_WR_IVLAN_VLDM_G(x) \ + (((x) >> FW_FILTER_WR_IVLAN_VLDM_S) & FW_FILTER_WR_IVLAN_VLDM_M) +#define FW_FILTER_WR_IVLAN_VLDM_F FW_FILTER_WR_IVLAN_VLDM_V(1U) + +#define FW_FILTER_WR_OVLAN_VLDM_S 2 +#define FW_FILTER_WR_OVLAN_VLDM_M 0x1 +#define FW_FILTER_WR_OVLAN_VLDM_V(x) ((x) << FW_FILTER_WR_OVLAN_VLDM_S) +#define FW_FILTER_WR_OVLAN_VLDM_G(x) \ + (((x) >> FW_FILTER_WR_OVLAN_VLDM_S) & FW_FILTER_WR_OVLAN_VLDM_M) +#define FW_FILTER_WR_OVLAN_VLDM_F FW_FILTER_WR_OVLAN_VLDM_V(1U) + +#define FW_FILTER_WR_RX_CHAN_S 15 +#define FW_FILTER_WR_RX_CHAN_M 0x1 +#define FW_FILTER_WR_RX_CHAN_V(x) ((x) << FW_FILTER_WR_RX_CHAN_S) +#define FW_FILTER_WR_RX_CHAN_G(x) \ + (((x) >> FW_FILTER_WR_RX_CHAN_S) & FW_FILTER_WR_RX_CHAN_M) +#define FW_FILTER_WR_RX_CHAN_F FW_FILTER_WR_RX_CHAN_V(1U) + +#define FW_FILTER_WR_RX_RPL_IQ_S 0 +#define FW_FILTER_WR_RX_RPL_IQ_M 0x3ff +#define FW_FILTER_WR_RX_RPL_IQ_V(x) ((x) << FW_FILTER_WR_RX_RPL_IQ_S) +#define FW_FILTER_WR_RX_RPL_IQ_G(x) \ + (((x) >> FW_FILTER_WR_RX_RPL_IQ_S) & FW_FILTER_WR_RX_RPL_IQ_M) + +#define FW_FILTER2_WR_FILTER_TYPE_S 1 +#define FW_FILTER2_WR_FILTER_TYPE_M 0x1 +#define FW_FILTER2_WR_FILTER_TYPE_V(x) ((x) << FW_FILTER2_WR_FILTER_TYPE_S) +#define FW_FILTER2_WR_FILTER_TYPE_G(x) \ + (((x) >> FW_FILTER2_WR_FILTER_TYPE_S) & FW_FILTER2_WR_FILTER_TYPE_M) +#define FW_FILTER2_WR_FILTER_TYPE_F FW_FILTER2_WR_FILTER_TYPE_V(1U) + +#define FW_FILTER2_WR_NATMODE_S 5 +#define FW_FILTER2_WR_NATMODE_M 0x7 +#define FW_FILTER2_WR_NATMODE_V(x) ((x) << FW_FILTER2_WR_NATMODE_S) +#define FW_FILTER2_WR_NATMODE_G(x) \ + (((x) >> FW_FILTER2_WR_NATMODE_S) & FW_FILTER2_WR_NATMODE_M) + +#define FW_FILTER2_WR_NATFLAGCHECK_S 4 +#define FW_FILTER2_WR_NATFLAGCHECK_M 0x1 +#define FW_FILTER2_WR_NATFLAGCHECK_V(x) ((x) << FW_FILTER2_WR_NATFLAGCHECK_S) +#define FW_FILTER2_WR_NATFLAGCHECK_G(x) \ + (((x) >> FW_FILTER2_WR_NATFLAGCHECK_S) & FW_FILTER2_WR_NATFLAGCHECK_M) +#define FW_FILTER2_WR_NATFLAGCHECK_F FW_FILTER2_WR_NATFLAGCHECK_V(1U) + +#define FW_FILTER2_WR_ULP_TYPE_S 0 +#define FW_FILTER2_WR_ULP_TYPE_M 0xf +#define FW_FILTER2_WR_ULP_TYPE_V(x) ((x) << FW_FILTER2_WR_ULP_TYPE_S) +#define FW_FILTER2_WR_ULP_TYPE_G(x) \ + (((x) >> FW_FILTER2_WR_ULP_TYPE_S) & FW_FILTER2_WR_ULP_TYPE_M) + +#define FW_FILTER_WR_MACI_S 23 +#define FW_FILTER_WR_MACI_M 0x1ff +#define FW_FILTER_WR_MACI_V(x) ((x) << FW_FILTER_WR_MACI_S) +#define FW_FILTER_WR_MACI_G(x) \ + (((x) >> FW_FILTER_WR_MACI_S) & FW_FILTER_WR_MACI_M) + +#define FW_FILTER_WR_MACIM_S 14 +#define FW_FILTER_WR_MACIM_M 0x1ff +#define FW_FILTER_WR_MACIM_V(x) ((x) << FW_FILTER_WR_MACIM_S) +#define FW_FILTER_WR_MACIM_G(x) \ + (((x) >> FW_FILTER_WR_MACIM_S) & FW_FILTER_WR_MACIM_M) + +#define FW_FILTER_WR_FCOE_S 13 +#define FW_FILTER_WR_FCOE_M 0x1 +#define FW_FILTER_WR_FCOE_V(x) ((x) << FW_FILTER_WR_FCOE_S) +#define FW_FILTER_WR_FCOE_G(x) \ + (((x) >> FW_FILTER_WR_FCOE_S) & FW_FILTER_WR_FCOE_M) +#define FW_FILTER_WR_FCOE_F FW_FILTER_WR_FCOE_V(1U) + +#define FW_FILTER_WR_FCOEM_S 12 +#define FW_FILTER_WR_FCOEM_M 0x1 +#define FW_FILTER_WR_FCOEM_V(x) ((x) << FW_FILTER_WR_FCOEM_S) +#define FW_FILTER_WR_FCOEM_G(x) \ + (((x) >> FW_FILTER_WR_FCOEM_S) & FW_FILTER_WR_FCOEM_M) +#define FW_FILTER_WR_FCOEM_F FW_FILTER_WR_FCOEM_V(1U) + +#define FW_FILTER_WR_PORT_S 9 +#define FW_FILTER_WR_PORT_M 0x7 +#define FW_FILTER_WR_PORT_V(x) ((x) << FW_FILTER_WR_PORT_S) +#define FW_FILTER_WR_PORT_G(x) \ + (((x) >> FW_FILTER_WR_PORT_S) & FW_FILTER_WR_PORT_M) + +#define FW_FILTER_WR_PORTM_S 6 +#define FW_FILTER_WR_PORTM_M 0x7 +#define FW_FILTER_WR_PORTM_V(x) ((x) << FW_FILTER_WR_PORTM_S) +#define FW_FILTER_WR_PORTM_G(x) \ + (((x) >> FW_FILTER_WR_PORTM_S) & FW_FILTER_WR_PORTM_M) + +#define FW_FILTER_WR_MATCHTYPE_S 3 +#define FW_FILTER_WR_MATCHTYPE_M 0x7 +#define FW_FILTER_WR_MATCHTYPE_V(x) ((x) << FW_FILTER_WR_MATCHTYPE_S) +#define FW_FILTER_WR_MATCHTYPE_G(x) \ + (((x) >> FW_FILTER_WR_MATCHTYPE_S) & FW_FILTER_WR_MATCHTYPE_M) + +#define FW_FILTER_WR_MATCHTYPEM_S 0 +#define FW_FILTER_WR_MATCHTYPEM_M 0x7 +#define FW_FILTER_WR_MATCHTYPEM_V(x) ((x) << FW_FILTER_WR_MATCHTYPEM_S) +#define FW_FILTER_WR_MATCHTYPEM_G(x) \ + (((x) >> FW_FILTER_WR_MATCHTYPEM_S) & FW_FILTER_WR_MATCHTYPEM_M) + +struct fw_ulptx_wr { + __be32 op_to_compl; + __be32 flowid_len16; + u64 cookie; +}; + +#define FW_ULPTX_WR_DATA_S 28 +#define FW_ULPTX_WR_DATA_M 0x1 +#define FW_ULPTX_WR_DATA_V(x) ((x) << FW_ULPTX_WR_DATA_S) +#define FW_ULPTX_WR_DATA_G(x) \ + (((x) >> FW_ULPTX_WR_DATA_S) & FW_ULPTX_WR_DATA_M) +#define FW_ULPTX_WR_DATA_F FW_ULPTX_WR_DATA_V(1U) + +struct fw_tp_wr { + __be32 op_to_immdlen; + __be32 flowid_len16; + u64 cookie; +}; + +struct fw_eth_tx_pkt_wr { + __be32 op_immdlen; + __be32 equiq_to_len16; + __be64 r3; +}; + +enum fw_eth_tx_eo_type { + FW_ETH_TX_EO_TYPE_UDPSEG = 0, + FW_ETH_TX_EO_TYPE_TCPSEG, +}; + +struct fw_eth_tx_eo_wr { + __be32 op_immdlen; + __be32 equiq_to_len16; + __be64 r3; + union fw_eth_tx_eo { + struct fw_eth_tx_eo_udpseg { + __u8 type; + __u8 ethlen; + __be16 iplen; + __u8 udplen; + __u8 rtplen; + __be16 r4; + __be16 mss; + __be16 schedpktsize; + __be32 plen; + } udpseg; + struct fw_eth_tx_eo_tcpseg { + __u8 type; + __u8 ethlen; + __be16 iplen; + __u8 tcplen; + __u8 tsclk_tsoff; + __be16 r4; + __be16 mss; + __be16 r5; + __be32 plen; + } tcpseg; + } u; +}; + +#define FW_ETH_TX_EO_WR_IMMDLEN_S 0 +#define FW_ETH_TX_EO_WR_IMMDLEN_M 0x1ff +#define FW_ETH_TX_EO_WR_IMMDLEN_V(x) ((x) << FW_ETH_TX_EO_WR_IMMDLEN_S) +#define FW_ETH_TX_EO_WR_IMMDLEN_G(x) \ + (((x) >> FW_ETH_TX_EO_WR_IMMDLEN_S) & FW_ETH_TX_EO_WR_IMMDLEN_M) + +struct fw_ofld_connection_wr { + __be32 op_compl; + __be32 len16_pkd; + __u64 cookie; + __be64 r2; + __be64 r3; + struct fw_ofld_connection_le { + __be32 version_cpl; + __be32 filter; + __be32 r1; + __be16 lport; + __be16 pport; + union fw_ofld_connection_leip { + struct fw_ofld_connection_le_ipv4 { + __be32 pip; + __be32 lip; + __be64 r0; + __be64 r1; + __be64 r2; + } ipv4; + struct fw_ofld_connection_le_ipv6 { + __be64 pip_hi; + __be64 pip_lo; + __be64 lip_hi; + __be64 lip_lo; + } ipv6; + } u; + } le; + struct fw_ofld_connection_tcb { + __be32 t_state_to_astid; + __be16 cplrxdataack_cplpassacceptrpl; + __be16 rcv_adv; + __be32 rcv_nxt; + __be32 tx_max; + __be64 opt0; + __be32 opt2; + __be32 r1; + __be64 r2; + __be64 r3; + } tcb; +}; + +#define FW_OFLD_CONNECTION_WR_VERSION_S 31 +#define FW_OFLD_CONNECTION_WR_VERSION_M 0x1 +#define FW_OFLD_CONNECTION_WR_VERSION_V(x) \ + ((x) << FW_OFLD_CONNECTION_WR_VERSION_S) +#define FW_OFLD_CONNECTION_WR_VERSION_G(x) \ + (((x) >> FW_OFLD_CONNECTION_WR_VERSION_S) & \ + FW_OFLD_CONNECTION_WR_VERSION_M) +#define FW_OFLD_CONNECTION_WR_VERSION_F \ + FW_OFLD_CONNECTION_WR_VERSION_V(1U) + +#define FW_OFLD_CONNECTION_WR_CPL_S 30 +#define FW_OFLD_CONNECTION_WR_CPL_M 0x1 +#define FW_OFLD_CONNECTION_WR_CPL_V(x) ((x) << FW_OFLD_CONNECTION_WR_CPL_S) +#define FW_OFLD_CONNECTION_WR_CPL_G(x) \ + (((x) >> FW_OFLD_CONNECTION_WR_CPL_S) & FW_OFLD_CONNECTION_WR_CPL_M) +#define FW_OFLD_CONNECTION_WR_CPL_F FW_OFLD_CONNECTION_WR_CPL_V(1U) + +#define FW_OFLD_CONNECTION_WR_T_STATE_S 28 +#define FW_OFLD_CONNECTION_WR_T_STATE_M 0xf +#define FW_OFLD_CONNECTION_WR_T_STATE_V(x) \ + ((x) << FW_OFLD_CONNECTION_WR_T_STATE_S) +#define FW_OFLD_CONNECTION_WR_T_STATE_G(x) \ + (((x) >> FW_OFLD_CONNECTION_WR_T_STATE_S) & \ + FW_OFLD_CONNECTION_WR_T_STATE_M) + +#define FW_OFLD_CONNECTION_WR_RCV_SCALE_S 24 +#define FW_OFLD_CONNECTION_WR_RCV_SCALE_M 0xf +#define FW_OFLD_CONNECTION_WR_RCV_SCALE_V(x) \ + ((x) << FW_OFLD_CONNECTION_WR_RCV_SCALE_S) +#define FW_OFLD_CONNECTION_WR_RCV_SCALE_G(x) \ + (((x) >> FW_OFLD_CONNECTION_WR_RCV_SCALE_S) & \ + FW_OFLD_CONNECTION_WR_RCV_SCALE_M) + +#define FW_OFLD_CONNECTION_WR_ASTID_S 0 +#define FW_OFLD_CONNECTION_WR_ASTID_M 0xffffff +#define FW_OFLD_CONNECTION_WR_ASTID_V(x) \ + ((x) << FW_OFLD_CONNECTION_WR_ASTID_S) +#define FW_OFLD_CONNECTION_WR_ASTID_G(x) \ + (((x) >> FW_OFLD_CONNECTION_WR_ASTID_S) & FW_OFLD_CONNECTION_WR_ASTID_M) + +#define FW_OFLD_CONNECTION_WR_CPLRXDATAACK_S 15 +#define FW_OFLD_CONNECTION_WR_CPLRXDATAACK_M 0x1 +#define FW_OFLD_CONNECTION_WR_CPLRXDATAACK_V(x) \ + ((x) << FW_OFLD_CONNECTION_WR_CPLRXDATAACK_S) +#define FW_OFLD_CONNECTION_WR_CPLRXDATAACK_G(x) \ + (((x) >> FW_OFLD_CONNECTION_WR_CPLRXDATAACK_S) & \ + FW_OFLD_CONNECTION_WR_CPLRXDATAACK_M) +#define FW_OFLD_CONNECTION_WR_CPLRXDATAACK_F \ + FW_OFLD_CONNECTION_WR_CPLRXDATAACK_V(1U) + +#define FW_OFLD_CONNECTION_WR_CPLPASSACCEPTRPL_S 14 +#define FW_OFLD_CONNECTION_WR_CPLPASSACCEPTRPL_M 0x1 +#define FW_OFLD_CONNECTION_WR_CPLPASSACCEPTRPL_V(x) \ + ((x) << FW_OFLD_CONNECTION_WR_CPLPASSACCEPTRPL_S) +#define FW_OFLD_CONNECTION_WR_CPLPASSACCEPTRPL_G(x) \ + (((x) >> FW_OFLD_CONNECTION_WR_CPLPASSACCEPTRPL_S) & \ + FW_OFLD_CONNECTION_WR_CPLPASSACCEPTRPL_M) +#define FW_OFLD_CONNECTION_WR_CPLPASSACCEPTRPL_F \ + FW_OFLD_CONNECTION_WR_CPLPASSACCEPTRPL_V(1U) + +enum fw_flowc_mnem_tcpstate { + FW_FLOWC_MNEM_TCPSTATE_CLOSED = 0, /* illegal */ + FW_FLOWC_MNEM_TCPSTATE_LISTEN = 1, /* illegal */ + FW_FLOWC_MNEM_TCPSTATE_SYNSENT = 2, /* illegal */ + FW_FLOWC_MNEM_TCPSTATE_SYNRECEIVED = 3, /* illegal */ + FW_FLOWC_MNEM_TCPSTATE_ESTABLISHED = 4, /* default */ + FW_FLOWC_MNEM_TCPSTATE_CLOSEWAIT = 5, /* got peer close already */ + FW_FLOWC_MNEM_TCPSTATE_FINWAIT1 = 6, /* haven't gotten ACK for FIN and + * will resend FIN - equiv ESTAB + */ + FW_FLOWC_MNEM_TCPSTATE_CLOSING = 7, /* haven't gotten ACK for FIN and + * will resend FIN but have + * received FIN + */ + FW_FLOWC_MNEM_TCPSTATE_LASTACK = 8, /* haven't gotten ACK for FIN and + * will resend FIN but have + * received FIN + */ + FW_FLOWC_MNEM_TCPSTATE_FINWAIT2 = 9, /* sent FIN and got FIN + ACK, + * waiting for FIN + */ + FW_FLOWC_MNEM_TCPSTATE_TIMEWAIT = 10, /* not expected */ +}; + +enum fw_flowc_mnem_eostate { + FW_FLOWC_MNEM_EOSTATE_ESTABLISHED = 1, /* default */ + /* graceful close, after sending outstanding payload */ + FW_FLOWC_MNEM_EOSTATE_CLOSING = 2, +}; + +enum fw_flowc_mnem { + FW_FLOWC_MNEM_PFNVFN, /* PFN [15:8] VFN [7:0] */ + FW_FLOWC_MNEM_CH, + FW_FLOWC_MNEM_PORT, + FW_FLOWC_MNEM_IQID, + FW_FLOWC_MNEM_SNDNXT, + FW_FLOWC_MNEM_RCVNXT, + FW_FLOWC_MNEM_SNDBUF, + FW_FLOWC_MNEM_MSS, + FW_FLOWC_MNEM_TXDATAPLEN_MAX, + FW_FLOWC_MNEM_TCPSTATE, + FW_FLOWC_MNEM_EOSTATE, + FW_FLOWC_MNEM_SCHEDCLASS, + FW_FLOWC_MNEM_DCBPRIO, + FW_FLOWC_MNEM_SND_SCALE, + FW_FLOWC_MNEM_RCV_SCALE, + FW_FLOWC_MNEM_ULD_MODE, + FW_FLOWC_MNEM_MAX, +}; + +struct fw_flowc_mnemval { + u8 mnemonic; + u8 r4[3]; + __be32 val; +}; + +struct fw_flowc_wr { + __be32 op_to_nparams; + __be32 flowid_len16; + struct fw_flowc_mnemval mnemval[]; +}; + +#define FW_FLOWC_WR_NPARAMS_S 0 +#define FW_FLOWC_WR_NPARAMS_V(x) ((x) << FW_FLOWC_WR_NPARAMS_S) + +struct fw_ofld_tx_data_wr { + __be32 op_to_immdlen; + __be32 flowid_len16; + __be32 plen; + __be32 tunnel_to_proxy; +}; + +#define FW_OFLD_TX_DATA_WR_ALIGNPLD_S 30 +#define FW_OFLD_TX_DATA_WR_ALIGNPLD_V(x) ((x) << FW_OFLD_TX_DATA_WR_ALIGNPLD_S) +#define FW_OFLD_TX_DATA_WR_ALIGNPLD_F FW_OFLD_TX_DATA_WR_ALIGNPLD_V(1U) + +#define FW_OFLD_TX_DATA_WR_SHOVE_S 29 +#define FW_OFLD_TX_DATA_WR_SHOVE_V(x) ((x) << FW_OFLD_TX_DATA_WR_SHOVE_S) +#define FW_OFLD_TX_DATA_WR_SHOVE_F FW_OFLD_TX_DATA_WR_SHOVE_V(1U) + +#define FW_OFLD_TX_DATA_WR_TUNNEL_S 19 +#define FW_OFLD_TX_DATA_WR_TUNNEL_V(x) ((x) << FW_OFLD_TX_DATA_WR_TUNNEL_S) + +#define FW_OFLD_TX_DATA_WR_SAVE_S 18 +#define FW_OFLD_TX_DATA_WR_SAVE_V(x) ((x) << FW_OFLD_TX_DATA_WR_SAVE_S) + +#define FW_OFLD_TX_DATA_WR_FLUSH_S 17 +#define FW_OFLD_TX_DATA_WR_FLUSH_V(x) ((x) << FW_OFLD_TX_DATA_WR_FLUSH_S) +#define FW_OFLD_TX_DATA_WR_FLUSH_F FW_OFLD_TX_DATA_WR_FLUSH_V(1U) + +#define FW_OFLD_TX_DATA_WR_URGENT_S 16 +#define FW_OFLD_TX_DATA_WR_URGENT_V(x) ((x) << FW_OFLD_TX_DATA_WR_URGENT_S) + +#define FW_OFLD_TX_DATA_WR_MORE_S 15 +#define FW_OFLD_TX_DATA_WR_MORE_V(x) ((x) << FW_OFLD_TX_DATA_WR_MORE_S) + +#define FW_OFLD_TX_DATA_WR_ULPMODE_S 10 +#define FW_OFLD_TX_DATA_WR_ULPMODE_V(x) ((x) << FW_OFLD_TX_DATA_WR_ULPMODE_S) + +#define FW_OFLD_TX_DATA_WR_ULPSUBMODE_S 6 +#define FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(x) \ + ((x) << FW_OFLD_TX_DATA_WR_ULPSUBMODE_S) + +struct fw_cmd_wr { + __be32 op_dma; + __be32 len16_pkd; + __be64 cookie_daddr; +}; + +#define FW_CMD_WR_DMA_S 17 +#define FW_CMD_WR_DMA_V(x) ((x) << FW_CMD_WR_DMA_S) + +struct fw_eth_tx_pkt_vm_wr { + __be32 op_immdlen; + __be32 equiq_to_len16; + __be32 r3[2]; + struct_group(firmware, + u8 ethmacdst[ETH_ALEN]; + u8 ethmacsrc[ETH_ALEN]; + __be16 ethtype; + __be16 vlantci; + ); +}; + +#define FW_CMD_MAX_TIMEOUT 10000 + +/* + * If a host driver does a HELLO and discovers that there's already a MASTER + * selected, we may have to wait for that MASTER to finish issuing RESET, + * configuration and INITIALIZE commands. Also, there's a possibility that + * our own HELLO may get lost if it happens right as the MASTER is issuign a + * RESET command, so we need to be willing to make a few retries of our HELLO. + */ +#define FW_CMD_HELLO_TIMEOUT (3 * FW_CMD_MAX_TIMEOUT) +#define FW_CMD_HELLO_RETRIES 3 + + +enum fw_cmd_opcodes { + FW_LDST_CMD = 0x01, + FW_RESET_CMD = 0x03, + FW_HELLO_CMD = 0x04, + FW_BYE_CMD = 0x05, + FW_INITIALIZE_CMD = 0x06, + FW_CAPS_CONFIG_CMD = 0x07, + FW_PARAMS_CMD = 0x08, + FW_PFVF_CMD = 0x09, + FW_IQ_CMD = 0x10, + FW_EQ_MNGT_CMD = 0x11, + FW_EQ_ETH_CMD = 0x12, + FW_EQ_CTRL_CMD = 0x13, + FW_EQ_OFLD_CMD = 0x21, + FW_VI_CMD = 0x14, + FW_VI_MAC_CMD = 0x15, + FW_VI_RXMODE_CMD = 0x16, + FW_VI_ENABLE_CMD = 0x17, + FW_ACL_MAC_CMD = 0x18, + FW_ACL_VLAN_CMD = 0x19, + FW_VI_STATS_CMD = 0x1a, + FW_PORT_CMD = 0x1b, + FW_PORT_STATS_CMD = 0x1c, + FW_PORT_LB_STATS_CMD = 0x1d, + FW_PORT_TRACE_CMD = 0x1e, + FW_PORT_TRACE_MMAP_CMD = 0x1f, + FW_RSS_IND_TBL_CMD = 0x20, + FW_RSS_GLB_CONFIG_CMD = 0x22, + FW_RSS_VI_CONFIG_CMD = 0x23, + FW_SCHED_CMD = 0x24, + FW_DEVLOG_CMD = 0x25, + FW_CLIP_CMD = 0x28, + FW_PTP_CMD = 0x3e, + FW_HMA_CMD = 0x3f, + FW_LASTC2E_CMD = 0x40, + FW_ERROR_CMD = 0x80, + FW_DEBUG_CMD = 0x81, +}; + +enum fw_cmd_cap { + FW_CMD_CAP_PF = 0x01, + FW_CMD_CAP_DMAQ = 0x02, + FW_CMD_CAP_PORT = 0x04, + FW_CMD_CAP_PORTPROMISC = 0x08, + FW_CMD_CAP_PORTSTATS = 0x10, + FW_CMD_CAP_VF = 0x80, +}; + +/* + * Generic command header flit0 + */ +struct fw_cmd_hdr { + __be32 hi; + __be32 lo; +}; + +#define FW_CMD_OP_S 24 +#define FW_CMD_OP_M 0xff +#define FW_CMD_OP_V(x) ((x) << FW_CMD_OP_S) +#define FW_CMD_OP_G(x) (((x) >> FW_CMD_OP_S) & FW_CMD_OP_M) + +#define FW_CMD_REQUEST_S 23 +#define FW_CMD_REQUEST_V(x) ((x) << FW_CMD_REQUEST_S) +#define FW_CMD_REQUEST_F FW_CMD_REQUEST_V(1U) + +#define FW_CMD_READ_S 22 +#define FW_CMD_READ_V(x) ((x) << FW_CMD_READ_S) +#define FW_CMD_READ_F FW_CMD_READ_V(1U) + +#define FW_CMD_WRITE_S 21 +#define FW_CMD_WRITE_V(x) ((x) << FW_CMD_WRITE_S) +#define FW_CMD_WRITE_F FW_CMD_WRITE_V(1U) + +#define FW_CMD_EXEC_S 20 +#define FW_CMD_EXEC_V(x) ((x) << FW_CMD_EXEC_S) +#define FW_CMD_EXEC_F FW_CMD_EXEC_V(1U) + +#define FW_CMD_RAMASK_S 20 +#define FW_CMD_RAMASK_V(x) ((x) << FW_CMD_RAMASK_S) + +#define FW_CMD_RETVAL_S 8 +#define FW_CMD_RETVAL_M 0xff +#define FW_CMD_RETVAL_V(x) ((x) << FW_CMD_RETVAL_S) +#define FW_CMD_RETVAL_G(x) (((x) >> FW_CMD_RETVAL_S) & FW_CMD_RETVAL_M) + +#define FW_CMD_LEN16_S 0 +#define FW_CMD_LEN16_V(x) ((x) << FW_CMD_LEN16_S) + +#define FW_LEN16(fw_struct) FW_CMD_LEN16_V(sizeof(fw_struct) / 16) + +enum fw_ldst_addrspc { + FW_LDST_ADDRSPC_FIRMWARE = 0x0001, + FW_LDST_ADDRSPC_SGE_EGRC = 0x0008, + FW_LDST_ADDRSPC_SGE_INGC = 0x0009, + FW_LDST_ADDRSPC_SGE_FLMC = 0x000a, + FW_LDST_ADDRSPC_SGE_CONMC = 0x000b, + FW_LDST_ADDRSPC_TP_PIO = 0x0010, + FW_LDST_ADDRSPC_TP_TM_PIO = 0x0011, + FW_LDST_ADDRSPC_TP_MIB = 0x0012, + FW_LDST_ADDRSPC_MDIO = 0x0018, + FW_LDST_ADDRSPC_MPS = 0x0020, + FW_LDST_ADDRSPC_FUNC = 0x0028, + FW_LDST_ADDRSPC_FUNC_PCIE = 0x0029, + FW_LDST_ADDRSPC_I2C = 0x0038, +}; + +enum fw_ldst_mps_fid { + FW_LDST_MPS_ATRB, + FW_LDST_MPS_RPLC +}; + +enum fw_ldst_func_access_ctl { + FW_LDST_FUNC_ACC_CTL_VIID, + FW_LDST_FUNC_ACC_CTL_FID +}; + +enum fw_ldst_func_mod_index { + FW_LDST_FUNC_MPS +}; + +struct fw_ldst_cmd { + __be32 op_to_addrspace; + __be32 cycles_to_len16; + union fw_ldst { + struct fw_ldst_addrval { + __be32 addr; + __be32 val; + } addrval; + struct fw_ldst_idctxt { + __be32 physid; + __be32 msg_ctxtflush; + __be32 ctxt_data7; + __be32 ctxt_data6; + __be32 ctxt_data5; + __be32 ctxt_data4; + __be32 ctxt_data3; + __be32 ctxt_data2; + __be32 ctxt_data1; + __be32 ctxt_data0; + } idctxt; + struct fw_ldst_mdio { + __be16 paddr_mmd; + __be16 raddr; + __be16 vctl; + __be16 rval; + } mdio; + struct fw_ldst_cim_rq { + u8 req_first64[8]; + u8 req_second64[8]; + u8 resp_first64[8]; + u8 resp_second64[8]; + __be32 r3[2]; + } cim_rq; + union fw_ldst_mps { + struct fw_ldst_mps_rplc { + __be16 fid_idx; + __be16 rplcpf_pkd; + __be32 rplc255_224; + __be32 rplc223_192; + __be32 rplc191_160; + __be32 rplc159_128; + __be32 rplc127_96; + __be32 rplc95_64; + __be32 rplc63_32; + __be32 rplc31_0; + } rplc; + struct fw_ldst_mps_atrb { + __be16 fid_mpsid; + __be16 r2[3]; + __be32 r3[2]; + __be32 r4; + __be32 atrb; + __be16 vlan[16]; + } atrb; + } mps; + struct fw_ldst_func { + u8 access_ctl; + u8 mod_index; + __be16 ctl_id; + __be32 offset; + __be64 data0; + __be64 data1; + } func; + struct fw_ldst_pcie { + u8 ctrl_to_fn; + u8 bnum; + u8 r; + u8 ext_r; + u8 select_naccess; + u8 pcie_fn; + __be16 nset_pkd; + __be32 data[12]; + } pcie; + struct fw_ldst_i2c_deprecated { + u8 pid_pkd; + u8 base; + u8 boffset; + u8 data; + __be32 r9; + } i2c_deprecated; + struct fw_ldst_i2c { + u8 pid; + u8 did; + u8 boffset; + u8 blen; + __be32 r9; + __u8 data[48]; + } i2c; + struct fw_ldst_le { + __be32 index; + __be32 r9; + u8 val[33]; + u8 r11[7]; + } le; + } u; +}; + +#define FW_LDST_CMD_ADDRSPACE_S 0 +#define FW_LDST_CMD_ADDRSPACE_V(x) ((x) << FW_LDST_CMD_ADDRSPACE_S) + +#define FW_LDST_CMD_MSG_S 31 +#define FW_LDST_CMD_MSG_V(x) ((x) << FW_LDST_CMD_MSG_S) + +#define FW_LDST_CMD_CTXTFLUSH_S 30 +#define FW_LDST_CMD_CTXTFLUSH_V(x) ((x) << FW_LDST_CMD_CTXTFLUSH_S) +#define FW_LDST_CMD_CTXTFLUSH_F FW_LDST_CMD_CTXTFLUSH_V(1U) + +#define FW_LDST_CMD_PADDR_S 8 +#define FW_LDST_CMD_PADDR_V(x) ((x) << FW_LDST_CMD_PADDR_S) + +#define FW_LDST_CMD_MMD_S 0 +#define FW_LDST_CMD_MMD_V(x) ((x) << FW_LDST_CMD_MMD_S) + +#define FW_LDST_CMD_FID_S 15 +#define FW_LDST_CMD_FID_V(x) ((x) << FW_LDST_CMD_FID_S) + +#define FW_LDST_CMD_IDX_S 0 +#define FW_LDST_CMD_IDX_V(x) ((x) << FW_LDST_CMD_IDX_S) + +#define FW_LDST_CMD_RPLCPF_S 0 +#define FW_LDST_CMD_RPLCPF_V(x) ((x) << FW_LDST_CMD_RPLCPF_S) + +#define FW_LDST_CMD_LC_S 4 +#define FW_LDST_CMD_LC_V(x) ((x) << FW_LDST_CMD_LC_S) +#define FW_LDST_CMD_LC_F FW_LDST_CMD_LC_V(1U) + +#define FW_LDST_CMD_FN_S 0 +#define FW_LDST_CMD_FN_V(x) ((x) << FW_LDST_CMD_FN_S) + +#define FW_LDST_CMD_NACCESS_S 0 +#define FW_LDST_CMD_NACCESS_V(x) ((x) << FW_LDST_CMD_NACCESS_S) + +struct fw_reset_cmd { + __be32 op_to_write; + __be32 retval_len16; + __be32 val; + __be32 halt_pkd; +}; + +#define FW_RESET_CMD_HALT_S 31 +#define FW_RESET_CMD_HALT_M 0x1 +#define FW_RESET_CMD_HALT_V(x) ((x) << FW_RESET_CMD_HALT_S) +#define FW_RESET_CMD_HALT_G(x) \ + (((x) >> FW_RESET_CMD_HALT_S) & FW_RESET_CMD_HALT_M) +#define FW_RESET_CMD_HALT_F FW_RESET_CMD_HALT_V(1U) + +enum fw_hellow_cmd { + fw_hello_cmd_stage_os = 0x0 +}; + +struct fw_hello_cmd { + __be32 op_to_write; + __be32 retval_len16; + __be32 err_to_clearinit; + __be32 fwrev; +}; + +#define FW_HELLO_CMD_ERR_S 31 +#define FW_HELLO_CMD_ERR_V(x) ((x) << FW_HELLO_CMD_ERR_S) +#define FW_HELLO_CMD_ERR_F FW_HELLO_CMD_ERR_V(1U) + +#define FW_HELLO_CMD_INIT_S 30 +#define FW_HELLO_CMD_INIT_V(x) ((x) << FW_HELLO_CMD_INIT_S) +#define FW_HELLO_CMD_INIT_F FW_HELLO_CMD_INIT_V(1U) + +#define FW_HELLO_CMD_MASTERDIS_S 29 +#define FW_HELLO_CMD_MASTERDIS_V(x) ((x) << FW_HELLO_CMD_MASTERDIS_S) + +#define FW_HELLO_CMD_MASTERFORCE_S 28 +#define FW_HELLO_CMD_MASTERFORCE_V(x) ((x) << FW_HELLO_CMD_MASTERFORCE_S) + +#define FW_HELLO_CMD_MBMASTER_S 24 +#define FW_HELLO_CMD_MBMASTER_M 0xfU +#define FW_HELLO_CMD_MBMASTER_V(x) ((x) << FW_HELLO_CMD_MBMASTER_S) +#define FW_HELLO_CMD_MBMASTER_G(x) \ + (((x) >> FW_HELLO_CMD_MBMASTER_S) & FW_HELLO_CMD_MBMASTER_M) + +#define FW_HELLO_CMD_MBASYNCNOTINT_S 23 +#define FW_HELLO_CMD_MBASYNCNOTINT_V(x) ((x) << FW_HELLO_CMD_MBASYNCNOTINT_S) + +#define FW_HELLO_CMD_MBASYNCNOT_S 20 +#define FW_HELLO_CMD_MBASYNCNOT_V(x) ((x) << FW_HELLO_CMD_MBASYNCNOT_S) + +#define FW_HELLO_CMD_STAGE_S 17 +#define FW_HELLO_CMD_STAGE_V(x) ((x) << FW_HELLO_CMD_STAGE_S) + +#define FW_HELLO_CMD_CLEARINIT_S 16 +#define FW_HELLO_CMD_CLEARINIT_V(x) ((x) << FW_HELLO_CMD_CLEARINIT_S) +#define FW_HELLO_CMD_CLEARINIT_F FW_HELLO_CMD_CLEARINIT_V(1U) + +struct fw_bye_cmd { + __be32 op_to_write; + __be32 retval_len16; + __be64 r3; +}; + +struct fw_initialize_cmd { + __be32 op_to_write; + __be32 retval_len16; + __be64 r3; +}; + +enum fw_caps_config_hm { + FW_CAPS_CONFIG_HM_PCIE = 0x00000001, + FW_CAPS_CONFIG_HM_PL = 0x00000002, + FW_CAPS_CONFIG_HM_SGE = 0x00000004, + FW_CAPS_CONFIG_HM_CIM = 0x00000008, + FW_CAPS_CONFIG_HM_ULPTX = 0x00000010, + FW_CAPS_CONFIG_HM_TP = 0x00000020, + FW_CAPS_CONFIG_HM_ULPRX = 0x00000040, + FW_CAPS_CONFIG_HM_PMRX = 0x00000080, + FW_CAPS_CONFIG_HM_PMTX = 0x00000100, + FW_CAPS_CONFIG_HM_MC = 0x00000200, + FW_CAPS_CONFIG_HM_LE = 0x00000400, + FW_CAPS_CONFIG_HM_MPS = 0x00000800, + FW_CAPS_CONFIG_HM_XGMAC = 0x00001000, + FW_CAPS_CONFIG_HM_CPLSWITCH = 0x00002000, + FW_CAPS_CONFIG_HM_T4DBG = 0x00004000, + FW_CAPS_CONFIG_HM_MI = 0x00008000, + FW_CAPS_CONFIG_HM_I2CM = 0x00010000, + FW_CAPS_CONFIG_HM_NCSI = 0x00020000, + FW_CAPS_CONFIG_HM_SMB = 0x00040000, + FW_CAPS_CONFIG_HM_MA = 0x00080000, + FW_CAPS_CONFIG_HM_EDRAM = 0x00100000, + FW_CAPS_CONFIG_HM_PMU = 0x00200000, + FW_CAPS_CONFIG_HM_UART = 0x00400000, + FW_CAPS_CONFIG_HM_SF = 0x00800000, +}; + +enum fw_caps_config_nbm { + FW_CAPS_CONFIG_NBM_IPMI = 0x00000001, + FW_CAPS_CONFIG_NBM_NCSI = 0x00000002, +}; + +enum fw_caps_config_link { + FW_CAPS_CONFIG_LINK_PPP = 0x00000001, + FW_CAPS_CONFIG_LINK_QFC = 0x00000002, + FW_CAPS_CONFIG_LINK_DCBX = 0x00000004, +}; + +enum fw_caps_config_switch { + FW_CAPS_CONFIG_SWITCH_INGRESS = 0x00000001, + FW_CAPS_CONFIG_SWITCH_EGRESS = 0x00000002, +}; + +enum fw_caps_config_nic { + FW_CAPS_CONFIG_NIC = 0x00000001, + FW_CAPS_CONFIG_NIC_VM = 0x00000002, + FW_CAPS_CONFIG_NIC_HASHFILTER = 0x00000020, + FW_CAPS_CONFIG_NIC_ETHOFLD = 0x00000040, +}; + +enum fw_caps_config_ofld { + FW_CAPS_CONFIG_OFLD = 0x00000001, +}; + +enum fw_caps_config_rdma { + FW_CAPS_CONFIG_RDMA_RDDP = 0x00000001, + FW_CAPS_CONFIG_RDMA_RDMAC = 0x00000002, +}; + +enum fw_caps_config_iscsi { + FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU = 0x00000001, + FW_CAPS_CONFIG_ISCSI_TARGET_PDU = 0x00000002, + FW_CAPS_CONFIG_ISCSI_INITIATOR_CNXOFLD = 0x00000004, + FW_CAPS_CONFIG_ISCSI_TARGET_CNXOFLD = 0x00000008, +}; + +enum fw_caps_config_crypto { + FW_CAPS_CONFIG_CRYPTO_LOOKASIDE = 0x00000001, + FW_CAPS_CONFIG_TLS_INLINE = 0x00000002, + FW_CAPS_CONFIG_IPSEC_INLINE = 0x00000004, + FW_CAPS_CONFIG_TLS_HW = 0x00000008, +}; + +enum fw_caps_config_fcoe { + FW_CAPS_CONFIG_FCOE_INITIATOR = 0x00000001, + FW_CAPS_CONFIG_FCOE_TARGET = 0x00000002, + FW_CAPS_CONFIG_FCOE_CTRL_OFLD = 0x00000004, +}; + +enum fw_memtype_cf { + FW_MEMTYPE_CF_EDC0 = 0x0, + FW_MEMTYPE_CF_EDC1 = 0x1, + FW_MEMTYPE_CF_EXTMEM = 0x2, + FW_MEMTYPE_CF_FLASH = 0x4, + FW_MEMTYPE_CF_INTERNAL = 0x5, + FW_MEMTYPE_CF_EXTMEM1 = 0x6, + FW_MEMTYPE_CF_HMA = 0x7, +}; + +struct fw_caps_config_cmd { + __be32 op_to_write; + __be32 cfvalid_to_len16; + __be32 r2; + __be32 hwmbitmap; + __be16 nbmcaps; + __be16 linkcaps; + __be16 switchcaps; + __be16 r3; + __be16 niccaps; + __be16 ofldcaps; + __be16 rdmacaps; + __be16 cryptocaps; + __be16 iscsicaps; + __be16 fcoecaps; + __be32 cfcsum; + __be32 finiver; + __be32 finicsum; +}; + +#define FW_CAPS_CONFIG_CMD_CFVALID_S 27 +#define FW_CAPS_CONFIG_CMD_CFVALID_V(x) ((x) << FW_CAPS_CONFIG_CMD_CFVALID_S) +#define FW_CAPS_CONFIG_CMD_CFVALID_F FW_CAPS_CONFIG_CMD_CFVALID_V(1U) + +#define FW_CAPS_CONFIG_CMD_MEMTYPE_CF_S 24 +#define FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(x) \ + ((x) << FW_CAPS_CONFIG_CMD_MEMTYPE_CF_S) + +#define FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_S 16 +#define FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(x) \ + ((x) << FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_S) + +/* + * params command mnemonics + */ +enum fw_params_mnem { + FW_PARAMS_MNEM_DEV = 1, /* device params */ + FW_PARAMS_MNEM_PFVF = 2, /* function params */ + FW_PARAMS_MNEM_REG = 3, /* limited register access */ + FW_PARAMS_MNEM_DMAQ = 4, /* dma queue params */ + FW_PARAMS_MNEM_CHNET = 5, /* chnet params */ + FW_PARAMS_MNEM_LAST +}; + +/* + * device parameters + */ + +#define FW_PARAMS_PARAM_FILTER_MODE_S 16 +#define FW_PARAMS_PARAM_FILTER_MODE_M 0xffff +#define FW_PARAMS_PARAM_FILTER_MODE_V(x) \ + ((x) << FW_PARAMS_PARAM_FILTER_MODE_S) +#define FW_PARAMS_PARAM_FILTER_MODE_G(x) \ + (((x) >> FW_PARAMS_PARAM_FILTER_MODE_S) & \ + FW_PARAMS_PARAM_FILTER_MODE_M) + +#define FW_PARAMS_PARAM_FILTER_MASK_S 0 +#define FW_PARAMS_PARAM_FILTER_MASK_M 0xffff +#define FW_PARAMS_PARAM_FILTER_MASK_V(x) \ + ((x) << FW_PARAMS_PARAM_FILTER_MASK_S) +#define FW_PARAMS_PARAM_FILTER_MASK_G(x) \ + (((x) >> FW_PARAMS_PARAM_FILTER_MASK_S) & \ + FW_PARAMS_PARAM_FILTER_MASK_M) + +enum fw_params_param_dev { + FW_PARAMS_PARAM_DEV_CCLK = 0x00, /* chip core clock in khz */ + FW_PARAMS_PARAM_DEV_PORTVEC = 0x01, /* the port vector */ + FW_PARAMS_PARAM_DEV_NTID = 0x02, /* reads the number of TIDs + * allocated by the device's + * Lookup Engine + */ + FW_PARAMS_PARAM_DEV_FLOWC_BUFFIFO_SZ = 0x03, + FW_PARAMS_PARAM_DEV_INTVER_NIC = 0x04, + FW_PARAMS_PARAM_DEV_INTVER_VNIC = 0x05, + FW_PARAMS_PARAM_DEV_INTVER_OFLD = 0x06, + FW_PARAMS_PARAM_DEV_INTVER_RI = 0x07, + FW_PARAMS_PARAM_DEV_INTVER_ISCSIPDU = 0x08, + FW_PARAMS_PARAM_DEV_INTVER_ISCSI = 0x09, + FW_PARAMS_PARAM_DEV_INTVER_FCOE = 0x0A, + FW_PARAMS_PARAM_DEV_FWREV = 0x0B, + FW_PARAMS_PARAM_DEV_TPREV = 0x0C, + FW_PARAMS_PARAM_DEV_CF = 0x0D, + FW_PARAMS_PARAM_DEV_PHYFW = 0x0F, + FW_PARAMS_PARAM_DEV_DIAG = 0x11, + FW_PARAMS_PARAM_DEV_MAXORDIRD_QP = 0x13, /* max supported QP IRD/ORD */ + FW_PARAMS_PARAM_DEV_MAXIRD_ADAPTER = 0x14, /* max supported adap IRD */ + FW_PARAMS_PARAM_DEV_ULPTX_MEMWRITE_DSGL = 0x17, + FW_PARAMS_PARAM_DEV_FWCACHE = 0x18, + FW_PARAMS_PARAM_DEV_SCFGREV = 0x1A, + FW_PARAMS_PARAM_DEV_VPDREV = 0x1B, + FW_PARAMS_PARAM_DEV_RI_FR_NSMR_TPTE_WR = 0x1C, + FW_PARAMS_PARAM_DEV_FILTER2_WR = 0x1D, + FW_PARAMS_PARAM_DEV_MPSBGMAP = 0x1E, + FW_PARAMS_PARAM_DEV_TPCHMAP = 0x1F, + FW_PARAMS_PARAM_DEV_HMA_SIZE = 0x20, + FW_PARAMS_PARAM_DEV_RDMA_WRITE_WITH_IMM = 0x21, + FW_PARAMS_PARAM_DEV_PPOD_EDRAM = 0x23, + FW_PARAMS_PARAM_DEV_RI_WRITE_CMPL_WR = 0x24, + FW_PARAMS_PARAM_DEV_HPFILTER_REGION_SUPPORT = 0x26, + FW_PARAMS_PARAM_DEV_OPAQUE_VIID_SMT_EXTN = 0x27, + FW_PARAMS_PARAM_DEV_HASHFILTER_WITH_OFLD = 0x28, + FW_PARAMS_PARAM_DEV_DBQ_TIMER = 0x29, + FW_PARAMS_PARAM_DEV_DBQ_TIMERTICK = 0x2A, + FW_PARAMS_PARAM_DEV_NUM_TM_CLASS = 0x2B, + FW_PARAMS_PARAM_DEV_FILTER = 0x2E, + FW_PARAMS_PARAM_DEV_KTLS_HW = 0x31, +}; + +/* + * physical and virtual function parameters + */ +enum fw_params_param_pfvf { + FW_PARAMS_PARAM_PFVF_RWXCAPS = 0x00, + FW_PARAMS_PARAM_PFVF_ROUTE_START = 0x01, + FW_PARAMS_PARAM_PFVF_ROUTE_END = 0x02, + FW_PARAMS_PARAM_PFVF_CLIP_START = 0x03, + FW_PARAMS_PARAM_PFVF_CLIP_END = 0x04, + FW_PARAMS_PARAM_PFVF_FILTER_START = 0x05, + FW_PARAMS_PARAM_PFVF_FILTER_END = 0x06, + FW_PARAMS_PARAM_PFVF_SERVER_START = 0x07, + FW_PARAMS_PARAM_PFVF_SERVER_END = 0x08, + FW_PARAMS_PARAM_PFVF_TDDP_START = 0x09, + FW_PARAMS_PARAM_PFVF_TDDP_END = 0x0A, + FW_PARAMS_PARAM_PFVF_ISCSI_START = 0x0B, + FW_PARAMS_PARAM_PFVF_ISCSI_END = 0x0C, + FW_PARAMS_PARAM_PFVF_STAG_START = 0x0D, + FW_PARAMS_PARAM_PFVF_STAG_END = 0x0E, + FW_PARAMS_PARAM_PFVF_RQ_START = 0x1F, + FW_PARAMS_PARAM_PFVF_RQ_END = 0x10, + FW_PARAMS_PARAM_PFVF_PBL_START = 0x11, + FW_PARAMS_PARAM_PFVF_PBL_END = 0x12, + FW_PARAMS_PARAM_PFVF_L2T_START = 0x13, + FW_PARAMS_PARAM_PFVF_L2T_END = 0x14, + FW_PARAMS_PARAM_PFVF_SQRQ_START = 0x15, + FW_PARAMS_PARAM_PFVF_SQRQ_END = 0x16, + FW_PARAMS_PARAM_PFVF_CQ_START = 0x17, + FW_PARAMS_PARAM_PFVF_CQ_END = 0x18, + FW_PARAMS_PARAM_PFVF_SRQ_START = 0x19, + FW_PARAMS_PARAM_PFVF_SRQ_END = 0x1A, + FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH = 0x20, + FW_PARAMS_PARAM_PFVF_VIID = 0x24, + FW_PARAMS_PARAM_PFVF_CPMASK = 0x25, + FW_PARAMS_PARAM_PFVF_OCQ_START = 0x26, + FW_PARAMS_PARAM_PFVF_OCQ_END = 0x27, + FW_PARAMS_PARAM_PFVF_CONM_MAP = 0x28, + FW_PARAMS_PARAM_PFVF_IQFLINT_START = 0x29, + FW_PARAMS_PARAM_PFVF_IQFLINT_END = 0x2A, + FW_PARAMS_PARAM_PFVF_EQ_START = 0x2B, + FW_PARAMS_PARAM_PFVF_EQ_END = 0x2C, + FW_PARAMS_PARAM_PFVF_ACTIVE_FILTER_START = 0x2D, + FW_PARAMS_PARAM_PFVF_ACTIVE_FILTER_END = 0x2E, + FW_PARAMS_PARAM_PFVF_ETHOFLD_START = 0x2F, + FW_PARAMS_PARAM_PFVF_ETHOFLD_END = 0x30, + FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP = 0x31, + FW_PARAMS_PARAM_PFVF_HPFILTER_START = 0x32, + FW_PARAMS_PARAM_PFVF_HPFILTER_END = 0x33, + FW_PARAMS_PARAM_PFVF_TLS_START = 0x34, + FW_PARAMS_PARAM_PFVF_TLS_END = 0x35, + FW_PARAMS_PARAM_PFVF_RAWF_START = 0x36, + FW_PARAMS_PARAM_PFVF_RAWF_END = 0x37, + FW_PARAMS_PARAM_PFVF_NCRYPTO_LOOKASIDE = 0x39, + FW_PARAMS_PARAM_PFVF_PORT_CAPS32 = 0x3A, + FW_PARAMS_PARAM_PFVF_PPOD_EDRAM_START = 0x3B, + FW_PARAMS_PARAM_PFVF_PPOD_EDRAM_END = 0x3C, + FW_PARAMS_PARAM_PFVF_LINK_STATE = 0x40, +}; + +/* Virtual link state as seen by the specified VF */ +enum vf_link_states { + FW_VF_LINK_STATE_AUTO = 0x00, + FW_VF_LINK_STATE_ENABLE = 0x01, + FW_VF_LINK_STATE_DISABLE = 0x02, +}; + +/* + * dma queue parameters + */ +enum fw_params_param_dmaq { + FW_PARAMS_PARAM_DMAQ_IQ_DCAEN_DCACPU = 0x00, + FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH = 0x01, + FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_MNGT = 0x10, + FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL = 0x11, + FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH = 0x12, + FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH = 0x13, + FW_PARAMS_PARAM_DMAQ_EQ_TIMERIX = 0x15, + FW_PARAMS_PARAM_DMAQ_CONM_CTXT = 0x20, +}; + +enum fw_params_param_dev_ktls_hw { + FW_PARAMS_PARAM_DEV_KTLS_HW_DISABLE = 0x00, + FW_PARAMS_PARAM_DEV_KTLS_HW_ENABLE = 0x01, + FW_PARAMS_PARAM_DEV_KTLS_HW_USER_ENABLE = 0x01, +}; + +enum fw_params_param_dev_phyfw { + FW_PARAMS_PARAM_DEV_PHYFW_DOWNLOAD = 0x00, + FW_PARAMS_PARAM_DEV_PHYFW_VERSION = 0x01, +}; + +enum fw_params_param_dev_diag { + FW_PARAM_DEV_DIAG_TMP = 0x00, + FW_PARAM_DEV_DIAG_VDD = 0x01, + FW_PARAM_DEV_DIAG_MAXTMPTHRESH = 0x02, +}; + +enum fw_params_param_dev_filter { + FW_PARAM_DEV_FILTER_VNIC_MODE = 0x00, + FW_PARAM_DEV_FILTER_MODE_MASK = 0x01, +}; + +enum fw_params_param_dev_fwcache { + FW_PARAM_DEV_FWCACHE_FLUSH = 0x00, + FW_PARAM_DEV_FWCACHE_FLUSHINV = 0x01, +}; + +#define FW_PARAMS_MNEM_S 24 +#define FW_PARAMS_MNEM_V(x) ((x) << FW_PARAMS_MNEM_S) + +#define FW_PARAMS_PARAM_X_S 16 +#define FW_PARAMS_PARAM_X_V(x) ((x) << FW_PARAMS_PARAM_X_S) + +#define FW_PARAMS_PARAM_Y_S 8 +#define FW_PARAMS_PARAM_Y_M 0xffU +#define FW_PARAMS_PARAM_Y_V(x) ((x) << FW_PARAMS_PARAM_Y_S) +#define FW_PARAMS_PARAM_Y_G(x) (((x) >> FW_PARAMS_PARAM_Y_S) &\ + FW_PARAMS_PARAM_Y_M) + +#define FW_PARAMS_PARAM_Z_S 0 +#define FW_PARAMS_PARAM_Z_M 0xffu +#define FW_PARAMS_PARAM_Z_V(x) ((x) << FW_PARAMS_PARAM_Z_S) +#define FW_PARAMS_PARAM_Z_G(x) (((x) >> FW_PARAMS_PARAM_Z_S) &\ + FW_PARAMS_PARAM_Z_M) + +#define FW_PARAMS_PARAM_XYZ_S 0 +#define FW_PARAMS_PARAM_XYZ_V(x) ((x) << FW_PARAMS_PARAM_XYZ_S) + +#define FW_PARAMS_PARAM_YZ_S 0 +#define FW_PARAMS_PARAM_YZ_V(x) ((x) << FW_PARAMS_PARAM_YZ_S) + +struct fw_params_cmd { + __be32 op_to_vfn; + __be32 retval_len16; + struct fw_params_param { + __be32 mnem; + __be32 val; + } param[7]; +}; + +#define FW_PARAMS_CMD_PFN_S 8 +#define FW_PARAMS_CMD_PFN_V(x) ((x) << FW_PARAMS_CMD_PFN_S) + +#define FW_PARAMS_CMD_VFN_S 0 +#define FW_PARAMS_CMD_VFN_V(x) ((x) << FW_PARAMS_CMD_VFN_S) + +struct fw_pfvf_cmd { + __be32 op_to_vfn; + __be32 retval_len16; + __be32 niqflint_niq; + __be32 type_to_neq; + __be32 tc_to_nexactf; + __be32 r_caps_to_nethctrl; + __be16 nricq; + __be16 nriqp; + __be32 r4; +}; + +#define FW_PFVF_CMD_PFN_S 8 +#define FW_PFVF_CMD_PFN_V(x) ((x) << FW_PFVF_CMD_PFN_S) + +#define FW_PFVF_CMD_VFN_S 0 +#define FW_PFVF_CMD_VFN_V(x) ((x) << FW_PFVF_CMD_VFN_S) + +#define FW_PFVF_CMD_NIQFLINT_S 20 +#define FW_PFVF_CMD_NIQFLINT_M 0xfff +#define FW_PFVF_CMD_NIQFLINT_V(x) ((x) << FW_PFVF_CMD_NIQFLINT_S) +#define FW_PFVF_CMD_NIQFLINT_G(x) \ + (((x) >> FW_PFVF_CMD_NIQFLINT_S) & FW_PFVF_CMD_NIQFLINT_M) + +#define FW_PFVF_CMD_NIQ_S 0 +#define FW_PFVF_CMD_NIQ_M 0xfffff +#define FW_PFVF_CMD_NIQ_V(x) ((x) << FW_PFVF_CMD_NIQ_S) +#define FW_PFVF_CMD_NIQ_G(x) \ + (((x) >> FW_PFVF_CMD_NIQ_S) & FW_PFVF_CMD_NIQ_M) + +#define FW_PFVF_CMD_TYPE_S 31 +#define FW_PFVF_CMD_TYPE_M 0x1 +#define FW_PFVF_CMD_TYPE_V(x) ((x) << FW_PFVF_CMD_TYPE_S) +#define FW_PFVF_CMD_TYPE_G(x) \ + (((x) >> FW_PFVF_CMD_TYPE_S) & FW_PFVF_CMD_TYPE_M) +#define FW_PFVF_CMD_TYPE_F FW_PFVF_CMD_TYPE_V(1U) + +#define FW_PFVF_CMD_CMASK_S 24 +#define FW_PFVF_CMD_CMASK_M 0xf +#define FW_PFVF_CMD_CMASK_V(x) ((x) << FW_PFVF_CMD_CMASK_S) +#define FW_PFVF_CMD_CMASK_G(x) \ + (((x) >> FW_PFVF_CMD_CMASK_S) & FW_PFVF_CMD_CMASK_M) + +#define FW_PFVF_CMD_PMASK_S 20 +#define FW_PFVF_CMD_PMASK_M 0xf +#define FW_PFVF_CMD_PMASK_V(x) ((x) << FW_PFVF_CMD_PMASK_S) +#define FW_PFVF_CMD_PMASK_G(x) \ + (((x) >> FW_PFVF_CMD_PMASK_S) & FW_PFVF_CMD_PMASK_M) + +#define FW_PFVF_CMD_NEQ_S 0 +#define FW_PFVF_CMD_NEQ_M 0xfffff +#define FW_PFVF_CMD_NEQ_V(x) ((x) << FW_PFVF_CMD_NEQ_S) +#define FW_PFVF_CMD_NEQ_G(x) \ + (((x) >> FW_PFVF_CMD_NEQ_S) & FW_PFVF_CMD_NEQ_M) + +#define FW_PFVF_CMD_TC_S 24 +#define FW_PFVF_CMD_TC_M 0xff +#define FW_PFVF_CMD_TC_V(x) ((x) << FW_PFVF_CMD_TC_S) +#define FW_PFVF_CMD_TC_G(x) (((x) >> FW_PFVF_CMD_TC_S) & FW_PFVF_CMD_TC_M) + +#define FW_PFVF_CMD_NVI_S 16 +#define FW_PFVF_CMD_NVI_M 0xff +#define FW_PFVF_CMD_NVI_V(x) ((x) << FW_PFVF_CMD_NVI_S) +#define FW_PFVF_CMD_NVI_G(x) (((x) >> FW_PFVF_CMD_NVI_S) & FW_PFVF_CMD_NVI_M) + +#define FW_PFVF_CMD_NEXACTF_S 0 +#define FW_PFVF_CMD_NEXACTF_M 0xffff +#define FW_PFVF_CMD_NEXACTF_V(x) ((x) << FW_PFVF_CMD_NEXACTF_S) +#define FW_PFVF_CMD_NEXACTF_G(x) \ + (((x) >> FW_PFVF_CMD_NEXACTF_S) & FW_PFVF_CMD_NEXACTF_M) + +#define FW_PFVF_CMD_R_CAPS_S 24 +#define FW_PFVF_CMD_R_CAPS_M 0xff +#define FW_PFVF_CMD_R_CAPS_V(x) ((x) << FW_PFVF_CMD_R_CAPS_S) +#define FW_PFVF_CMD_R_CAPS_G(x) \ + (((x) >> FW_PFVF_CMD_R_CAPS_S) & FW_PFVF_CMD_R_CAPS_M) + +#define FW_PFVF_CMD_WX_CAPS_S 16 +#define FW_PFVF_CMD_WX_CAPS_M 0xff +#define FW_PFVF_CMD_WX_CAPS_V(x) ((x) << FW_PFVF_CMD_WX_CAPS_S) +#define FW_PFVF_CMD_WX_CAPS_G(x) \ + (((x) >> FW_PFVF_CMD_WX_CAPS_S) & FW_PFVF_CMD_WX_CAPS_M) + +#define FW_PFVF_CMD_NETHCTRL_S 0 +#define FW_PFVF_CMD_NETHCTRL_M 0xffff +#define FW_PFVF_CMD_NETHCTRL_V(x) ((x) << FW_PFVF_CMD_NETHCTRL_S) +#define FW_PFVF_CMD_NETHCTRL_G(x) \ + (((x) >> FW_PFVF_CMD_NETHCTRL_S) & FW_PFVF_CMD_NETHCTRL_M) + +enum fw_iq_type { + FW_IQ_TYPE_FL_INT_CAP, + FW_IQ_TYPE_NO_FL_INT_CAP +}; + +enum fw_iq_iqtype { + FW_IQ_IQTYPE_OTHER, + FW_IQ_IQTYPE_NIC, + FW_IQ_IQTYPE_OFLD, +}; + +struct fw_iq_cmd { + __be32 op_to_vfn; + __be32 alloc_to_len16; + __be16 physiqid; + __be16 iqid; + __be16 fl0id; + __be16 fl1id; + __be32 type_to_iqandstindex; + __be16 iqdroprss_to_iqesize; + __be16 iqsize; + __be64 iqaddr; + __be32 iqns_to_fl0congen; + __be16 fl0dcaen_to_fl0cidxfthresh; + __be16 fl0size; + __be64 fl0addr; + __be32 fl1cngchmap_to_fl1congen; + __be16 fl1dcaen_to_fl1cidxfthresh; + __be16 fl1size; + __be64 fl1addr; +}; + +#define FW_IQ_CMD_PFN_S 8 +#define FW_IQ_CMD_PFN_V(x) ((x) << FW_IQ_CMD_PFN_S) + +#define FW_IQ_CMD_VFN_S 0 +#define FW_IQ_CMD_VFN_V(x) ((x) << FW_IQ_CMD_VFN_S) + +#define FW_IQ_CMD_ALLOC_S 31 +#define FW_IQ_CMD_ALLOC_V(x) ((x) << FW_IQ_CMD_ALLOC_S) +#define FW_IQ_CMD_ALLOC_F FW_IQ_CMD_ALLOC_V(1U) + +#define FW_IQ_CMD_FREE_S 30 +#define FW_IQ_CMD_FREE_V(x) ((x) << FW_IQ_CMD_FREE_S) +#define FW_IQ_CMD_FREE_F FW_IQ_CMD_FREE_V(1U) + +#define FW_IQ_CMD_MODIFY_S 29 +#define FW_IQ_CMD_MODIFY_V(x) ((x) << FW_IQ_CMD_MODIFY_S) +#define FW_IQ_CMD_MODIFY_F FW_IQ_CMD_MODIFY_V(1U) + +#define FW_IQ_CMD_IQSTART_S 28 +#define FW_IQ_CMD_IQSTART_V(x) ((x) << FW_IQ_CMD_IQSTART_S) +#define FW_IQ_CMD_IQSTART_F FW_IQ_CMD_IQSTART_V(1U) + +#define FW_IQ_CMD_IQSTOP_S 27 +#define FW_IQ_CMD_IQSTOP_V(x) ((x) << FW_IQ_CMD_IQSTOP_S) +#define FW_IQ_CMD_IQSTOP_F FW_IQ_CMD_IQSTOP_V(1U) + +#define FW_IQ_CMD_TYPE_S 29 +#define FW_IQ_CMD_TYPE_V(x) ((x) << FW_IQ_CMD_TYPE_S) + +#define FW_IQ_CMD_IQASYNCH_S 28 +#define FW_IQ_CMD_IQASYNCH_V(x) ((x) << FW_IQ_CMD_IQASYNCH_S) + +#define FW_IQ_CMD_VIID_S 16 +#define FW_IQ_CMD_VIID_V(x) ((x) << FW_IQ_CMD_VIID_S) + +#define FW_IQ_CMD_IQANDST_S 15 +#define FW_IQ_CMD_IQANDST_V(x) ((x) << FW_IQ_CMD_IQANDST_S) + +#define FW_IQ_CMD_IQANUS_S 14 +#define FW_IQ_CMD_IQANUS_V(x) ((x) << FW_IQ_CMD_IQANUS_S) + +#define FW_IQ_CMD_IQANUD_S 12 +#define FW_IQ_CMD_IQANUD_V(x) ((x) << FW_IQ_CMD_IQANUD_S) + +#define FW_IQ_CMD_IQANDSTINDEX_S 0 +#define FW_IQ_CMD_IQANDSTINDEX_V(x) ((x) << FW_IQ_CMD_IQANDSTINDEX_S) + +#define FW_IQ_CMD_IQDROPRSS_S 15 +#define FW_IQ_CMD_IQDROPRSS_V(x) ((x) << FW_IQ_CMD_IQDROPRSS_S) +#define FW_IQ_CMD_IQDROPRSS_F FW_IQ_CMD_IQDROPRSS_V(1U) + +#define FW_IQ_CMD_IQGTSMODE_S 14 +#define FW_IQ_CMD_IQGTSMODE_V(x) ((x) << FW_IQ_CMD_IQGTSMODE_S) +#define FW_IQ_CMD_IQGTSMODE_F FW_IQ_CMD_IQGTSMODE_V(1U) + +#define FW_IQ_CMD_IQPCIECH_S 12 +#define FW_IQ_CMD_IQPCIECH_V(x) ((x) << FW_IQ_CMD_IQPCIECH_S) + +#define FW_IQ_CMD_IQDCAEN_S 11 +#define FW_IQ_CMD_IQDCAEN_V(x) ((x) << FW_IQ_CMD_IQDCAEN_S) + +#define FW_IQ_CMD_IQDCACPU_S 6 +#define FW_IQ_CMD_IQDCACPU_V(x) ((x) << FW_IQ_CMD_IQDCACPU_S) + +#define FW_IQ_CMD_IQINTCNTTHRESH_S 4 +#define FW_IQ_CMD_IQINTCNTTHRESH_V(x) ((x) << FW_IQ_CMD_IQINTCNTTHRESH_S) + +#define FW_IQ_CMD_IQO_S 3 +#define FW_IQ_CMD_IQO_V(x) ((x) << FW_IQ_CMD_IQO_S) +#define FW_IQ_CMD_IQO_F FW_IQ_CMD_IQO_V(1U) + +#define FW_IQ_CMD_IQCPRIO_S 2 +#define FW_IQ_CMD_IQCPRIO_V(x) ((x) << FW_IQ_CMD_IQCPRIO_S) + +#define FW_IQ_CMD_IQESIZE_S 0 +#define FW_IQ_CMD_IQESIZE_V(x) ((x) << FW_IQ_CMD_IQESIZE_S) + +#define FW_IQ_CMD_IQNS_S 31 +#define FW_IQ_CMD_IQNS_V(x) ((x) << FW_IQ_CMD_IQNS_S) + +#define FW_IQ_CMD_IQRO_S 30 +#define FW_IQ_CMD_IQRO_V(x) ((x) << FW_IQ_CMD_IQRO_S) + +#define FW_IQ_CMD_IQFLINTIQHSEN_S 28 +#define FW_IQ_CMD_IQFLINTIQHSEN_V(x) ((x) << FW_IQ_CMD_IQFLINTIQHSEN_S) + +#define FW_IQ_CMD_IQFLINTCONGEN_S 27 +#define FW_IQ_CMD_IQFLINTCONGEN_V(x) ((x) << FW_IQ_CMD_IQFLINTCONGEN_S) +#define FW_IQ_CMD_IQFLINTCONGEN_F FW_IQ_CMD_IQFLINTCONGEN_V(1U) + +#define FW_IQ_CMD_IQFLINTISCSIC_S 26 +#define FW_IQ_CMD_IQFLINTISCSIC_V(x) ((x) << FW_IQ_CMD_IQFLINTISCSIC_S) + +#define FW_IQ_CMD_IQTYPE_S 24 +#define FW_IQ_CMD_IQTYPE_M 0x3 +#define FW_IQ_CMD_IQTYPE_V(x) ((x) << FW_IQ_CMD_IQTYPE_S) +#define FW_IQ_CMD_IQTYPE_G(x) \ + (((x) >> FW_IQ_CMD_IQTYPE_S) & FW_IQ_CMD_IQTYPE_M) + +#define FW_IQ_CMD_FL0CNGCHMAP_S 20 +#define FW_IQ_CMD_FL0CNGCHMAP_V(x) ((x) << FW_IQ_CMD_FL0CNGCHMAP_S) + +#define FW_IQ_CMD_FL0CACHELOCK_S 15 +#define FW_IQ_CMD_FL0CACHELOCK_V(x) ((x) << FW_IQ_CMD_FL0CACHELOCK_S) + +#define FW_IQ_CMD_FL0DBP_S 14 +#define FW_IQ_CMD_FL0DBP_V(x) ((x) << FW_IQ_CMD_FL0DBP_S) + +#define FW_IQ_CMD_FL0DATANS_S 13 +#define FW_IQ_CMD_FL0DATANS_V(x) ((x) << FW_IQ_CMD_FL0DATANS_S) + +#define FW_IQ_CMD_FL0DATARO_S 12 +#define FW_IQ_CMD_FL0DATARO_V(x) ((x) << FW_IQ_CMD_FL0DATARO_S) +#define FW_IQ_CMD_FL0DATARO_F FW_IQ_CMD_FL0DATARO_V(1U) + +#define FW_IQ_CMD_FL0CONGCIF_S 11 +#define FW_IQ_CMD_FL0CONGCIF_V(x) ((x) << FW_IQ_CMD_FL0CONGCIF_S) +#define FW_IQ_CMD_FL0CONGCIF_F FW_IQ_CMD_FL0CONGCIF_V(1U) + +#define FW_IQ_CMD_FL0ONCHIP_S 10 +#define FW_IQ_CMD_FL0ONCHIP_V(x) ((x) << FW_IQ_CMD_FL0ONCHIP_S) + +#define FW_IQ_CMD_FL0STATUSPGNS_S 9 +#define FW_IQ_CMD_FL0STATUSPGNS_V(x) ((x) << FW_IQ_CMD_FL0STATUSPGNS_S) + +#define FW_IQ_CMD_FL0STATUSPGRO_S 8 +#define FW_IQ_CMD_FL0STATUSPGRO_V(x) ((x) << FW_IQ_CMD_FL0STATUSPGRO_S) + +#define FW_IQ_CMD_FL0FETCHNS_S 7 +#define FW_IQ_CMD_FL0FETCHNS_V(x) ((x) << FW_IQ_CMD_FL0FETCHNS_S) + +#define FW_IQ_CMD_FL0FETCHRO_S 6 +#define FW_IQ_CMD_FL0FETCHRO_V(x) ((x) << FW_IQ_CMD_FL0FETCHRO_S) +#define FW_IQ_CMD_FL0FETCHRO_F FW_IQ_CMD_FL0FETCHRO_V(1U) + +#define FW_IQ_CMD_FL0HOSTFCMODE_S 4 +#define FW_IQ_CMD_FL0HOSTFCMODE_V(x) ((x) << FW_IQ_CMD_FL0HOSTFCMODE_S) + +#define FW_IQ_CMD_FL0CPRIO_S 3 +#define FW_IQ_CMD_FL0CPRIO_V(x) ((x) << FW_IQ_CMD_FL0CPRIO_S) + +#define FW_IQ_CMD_FL0PADEN_S 2 +#define FW_IQ_CMD_FL0PADEN_V(x) ((x) << FW_IQ_CMD_FL0PADEN_S) +#define FW_IQ_CMD_FL0PADEN_F FW_IQ_CMD_FL0PADEN_V(1U) + +#define FW_IQ_CMD_FL0PACKEN_S 1 +#define FW_IQ_CMD_FL0PACKEN_V(x) ((x) << FW_IQ_CMD_FL0PACKEN_S) +#define FW_IQ_CMD_FL0PACKEN_F FW_IQ_CMD_FL0PACKEN_V(1U) + +#define FW_IQ_CMD_FL0CONGEN_S 0 +#define FW_IQ_CMD_FL0CONGEN_V(x) ((x) << FW_IQ_CMD_FL0CONGEN_S) +#define FW_IQ_CMD_FL0CONGEN_F FW_IQ_CMD_FL0CONGEN_V(1U) + +#define FW_IQ_CMD_FL0DCAEN_S 15 +#define FW_IQ_CMD_FL0DCAEN_V(x) ((x) << FW_IQ_CMD_FL0DCAEN_S) + +#define FW_IQ_CMD_FL0DCACPU_S 10 +#define FW_IQ_CMD_FL0DCACPU_V(x) ((x) << FW_IQ_CMD_FL0DCACPU_S) + +#define FW_IQ_CMD_FL0FBMIN_S 7 +#define FW_IQ_CMD_FL0FBMIN_V(x) ((x) << FW_IQ_CMD_FL0FBMIN_S) + +#define FW_IQ_CMD_FL0FBMAX_S 4 +#define FW_IQ_CMD_FL0FBMAX_V(x) ((x) << FW_IQ_CMD_FL0FBMAX_S) + +#define FW_IQ_CMD_FL0CIDXFTHRESHO_S 3 +#define FW_IQ_CMD_FL0CIDXFTHRESHO_V(x) ((x) << FW_IQ_CMD_FL0CIDXFTHRESHO_S) +#define FW_IQ_CMD_FL0CIDXFTHRESHO_F FW_IQ_CMD_FL0CIDXFTHRESHO_V(1U) + +#define FW_IQ_CMD_FL0CIDXFTHRESH_S 0 +#define FW_IQ_CMD_FL0CIDXFTHRESH_V(x) ((x) << FW_IQ_CMD_FL0CIDXFTHRESH_S) + +#define FW_IQ_CMD_FL1CNGCHMAP_S 20 +#define FW_IQ_CMD_FL1CNGCHMAP_V(x) ((x) << FW_IQ_CMD_FL1CNGCHMAP_S) + +#define FW_IQ_CMD_FL1CACHELOCK_S 15 +#define FW_IQ_CMD_FL1CACHELOCK_V(x) ((x) << FW_IQ_CMD_FL1CACHELOCK_S) + +#define FW_IQ_CMD_FL1DBP_S 14 +#define FW_IQ_CMD_FL1DBP_V(x) ((x) << FW_IQ_CMD_FL1DBP_S) + +#define FW_IQ_CMD_FL1DATANS_S 13 +#define FW_IQ_CMD_FL1DATANS_V(x) ((x) << FW_IQ_CMD_FL1DATANS_S) + +#define FW_IQ_CMD_FL1DATARO_S 12 +#define FW_IQ_CMD_FL1DATARO_V(x) ((x) << FW_IQ_CMD_FL1DATARO_S) + +#define FW_IQ_CMD_FL1CONGCIF_S 11 +#define FW_IQ_CMD_FL1CONGCIF_V(x) ((x) << FW_IQ_CMD_FL1CONGCIF_S) + +#define FW_IQ_CMD_FL1ONCHIP_S 10 +#define FW_IQ_CMD_FL1ONCHIP_V(x) ((x) << FW_IQ_CMD_FL1ONCHIP_S) + +#define FW_IQ_CMD_FL1STATUSPGNS_S 9 +#define FW_IQ_CMD_FL1STATUSPGNS_V(x) ((x) << FW_IQ_CMD_FL1STATUSPGNS_S) + +#define FW_IQ_CMD_FL1STATUSPGRO_S 8 +#define FW_IQ_CMD_FL1STATUSPGRO_V(x) ((x) << FW_IQ_CMD_FL1STATUSPGRO_S) + +#define FW_IQ_CMD_FL1FETCHNS_S 7 +#define FW_IQ_CMD_FL1FETCHNS_V(x) ((x) << FW_IQ_CMD_FL1FETCHNS_S) + +#define FW_IQ_CMD_FL1FETCHRO_S 6 +#define FW_IQ_CMD_FL1FETCHRO_V(x) ((x) << FW_IQ_CMD_FL1FETCHRO_S) + +#define FW_IQ_CMD_FL1HOSTFCMODE_S 4 +#define FW_IQ_CMD_FL1HOSTFCMODE_V(x) ((x) << FW_IQ_CMD_FL1HOSTFCMODE_S) + +#define FW_IQ_CMD_FL1CPRIO_S 3 +#define FW_IQ_CMD_FL1CPRIO_V(x) ((x) << FW_IQ_CMD_FL1CPRIO_S) + +#define FW_IQ_CMD_FL1PADEN_S 2 +#define FW_IQ_CMD_FL1PADEN_V(x) ((x) << FW_IQ_CMD_FL1PADEN_S) +#define FW_IQ_CMD_FL1PADEN_F FW_IQ_CMD_FL1PADEN_V(1U) + +#define FW_IQ_CMD_FL1PACKEN_S 1 +#define FW_IQ_CMD_FL1PACKEN_V(x) ((x) << FW_IQ_CMD_FL1PACKEN_S) +#define FW_IQ_CMD_FL1PACKEN_F FW_IQ_CMD_FL1PACKEN_V(1U) + +#define FW_IQ_CMD_FL1CONGEN_S 0 +#define FW_IQ_CMD_FL1CONGEN_V(x) ((x) << FW_IQ_CMD_FL1CONGEN_S) +#define FW_IQ_CMD_FL1CONGEN_F FW_IQ_CMD_FL1CONGEN_V(1U) + +#define FW_IQ_CMD_FL1DCAEN_S 15 +#define FW_IQ_CMD_FL1DCAEN_V(x) ((x) << FW_IQ_CMD_FL1DCAEN_S) + +#define FW_IQ_CMD_FL1DCACPU_S 10 +#define FW_IQ_CMD_FL1DCACPU_V(x) ((x) << FW_IQ_CMD_FL1DCACPU_S) + +#define FW_IQ_CMD_FL1FBMIN_S 7 +#define FW_IQ_CMD_FL1FBMIN_V(x) ((x) << FW_IQ_CMD_FL1FBMIN_S) + +#define FW_IQ_CMD_FL1FBMAX_S 4 +#define FW_IQ_CMD_FL1FBMAX_V(x) ((x) << FW_IQ_CMD_FL1FBMAX_S) + +#define FW_IQ_CMD_FL1CIDXFTHRESHO_S 3 +#define FW_IQ_CMD_FL1CIDXFTHRESHO_V(x) ((x) << FW_IQ_CMD_FL1CIDXFTHRESHO_S) +#define FW_IQ_CMD_FL1CIDXFTHRESHO_F FW_IQ_CMD_FL1CIDXFTHRESHO_V(1U) + +#define FW_IQ_CMD_FL1CIDXFTHRESH_S 0 +#define FW_IQ_CMD_FL1CIDXFTHRESH_V(x) ((x) << FW_IQ_CMD_FL1CIDXFTHRESH_S) + +struct fw_eq_eth_cmd { + __be32 op_to_vfn; + __be32 alloc_to_len16; + __be32 eqid_pkd; + __be32 physeqid_pkd; + __be32 fetchszm_to_iqid; + __be32 dcaen_to_eqsize; + __be64 eqaddr; + __be32 autoequiqe_to_viid; + __be32 timeren_timerix; + __be64 r9; +}; + +#define FW_EQ_ETH_CMD_PFN_S 8 +#define FW_EQ_ETH_CMD_PFN_V(x) ((x) << FW_EQ_ETH_CMD_PFN_S) + +#define FW_EQ_ETH_CMD_VFN_S 0 +#define FW_EQ_ETH_CMD_VFN_V(x) ((x) << FW_EQ_ETH_CMD_VFN_S) + +#define FW_EQ_ETH_CMD_ALLOC_S 31 +#define FW_EQ_ETH_CMD_ALLOC_V(x) ((x) << FW_EQ_ETH_CMD_ALLOC_S) +#define FW_EQ_ETH_CMD_ALLOC_F FW_EQ_ETH_CMD_ALLOC_V(1U) + +#define FW_EQ_ETH_CMD_FREE_S 30 +#define FW_EQ_ETH_CMD_FREE_V(x) ((x) << FW_EQ_ETH_CMD_FREE_S) +#define FW_EQ_ETH_CMD_FREE_F FW_EQ_ETH_CMD_FREE_V(1U) + +#define FW_EQ_ETH_CMD_MODIFY_S 29 +#define FW_EQ_ETH_CMD_MODIFY_V(x) ((x) << FW_EQ_ETH_CMD_MODIFY_S) +#define FW_EQ_ETH_CMD_MODIFY_F FW_EQ_ETH_CMD_MODIFY_V(1U) + +#define FW_EQ_ETH_CMD_EQSTART_S 28 +#define FW_EQ_ETH_CMD_EQSTART_V(x) ((x) << FW_EQ_ETH_CMD_EQSTART_S) +#define FW_EQ_ETH_CMD_EQSTART_F FW_EQ_ETH_CMD_EQSTART_V(1U) + +#define FW_EQ_ETH_CMD_EQSTOP_S 27 +#define FW_EQ_ETH_CMD_EQSTOP_V(x) ((x) << FW_EQ_ETH_CMD_EQSTOP_S) +#define FW_EQ_ETH_CMD_EQSTOP_F FW_EQ_ETH_CMD_EQSTOP_V(1U) + +#define FW_EQ_ETH_CMD_EQID_S 0 +#define FW_EQ_ETH_CMD_EQID_M 0xfffff +#define FW_EQ_ETH_CMD_EQID_V(x) ((x) << FW_EQ_ETH_CMD_EQID_S) +#define FW_EQ_ETH_CMD_EQID_G(x) \ + (((x) >> FW_EQ_ETH_CMD_EQID_S) & FW_EQ_ETH_CMD_EQID_M) + +#define FW_EQ_ETH_CMD_PHYSEQID_S 0 +#define FW_EQ_ETH_CMD_PHYSEQID_M 0xfffff +#define FW_EQ_ETH_CMD_PHYSEQID_V(x) ((x) << FW_EQ_ETH_CMD_PHYSEQID_S) +#define FW_EQ_ETH_CMD_PHYSEQID_G(x) \ + (((x) >> FW_EQ_ETH_CMD_PHYSEQID_S) & FW_EQ_ETH_CMD_PHYSEQID_M) + +#define FW_EQ_ETH_CMD_FETCHSZM_S 26 +#define FW_EQ_ETH_CMD_FETCHSZM_V(x) ((x) << FW_EQ_ETH_CMD_FETCHSZM_S) +#define FW_EQ_ETH_CMD_FETCHSZM_F FW_EQ_ETH_CMD_FETCHSZM_V(1U) + +#define FW_EQ_ETH_CMD_STATUSPGNS_S 25 +#define FW_EQ_ETH_CMD_STATUSPGNS_V(x) ((x) << FW_EQ_ETH_CMD_STATUSPGNS_S) + +#define FW_EQ_ETH_CMD_STATUSPGRO_S 24 +#define FW_EQ_ETH_CMD_STATUSPGRO_V(x) ((x) << FW_EQ_ETH_CMD_STATUSPGRO_S) + +#define FW_EQ_ETH_CMD_FETCHNS_S 23 +#define FW_EQ_ETH_CMD_FETCHNS_V(x) ((x) << FW_EQ_ETH_CMD_FETCHNS_S) + +#define FW_EQ_ETH_CMD_FETCHRO_S 22 +#define FW_EQ_ETH_CMD_FETCHRO_V(x) ((x) << FW_EQ_ETH_CMD_FETCHRO_S) +#define FW_EQ_ETH_CMD_FETCHRO_F FW_EQ_ETH_CMD_FETCHRO_V(1U) + +#define FW_EQ_ETH_CMD_HOSTFCMODE_S 20 +#define FW_EQ_ETH_CMD_HOSTFCMODE_V(x) ((x) << FW_EQ_ETH_CMD_HOSTFCMODE_S) + +#define FW_EQ_ETH_CMD_CPRIO_S 19 +#define FW_EQ_ETH_CMD_CPRIO_V(x) ((x) << FW_EQ_ETH_CMD_CPRIO_S) + +#define FW_EQ_ETH_CMD_ONCHIP_S 18 +#define FW_EQ_ETH_CMD_ONCHIP_V(x) ((x) << FW_EQ_ETH_CMD_ONCHIP_S) + +#define FW_EQ_ETH_CMD_PCIECHN_S 16 +#define FW_EQ_ETH_CMD_PCIECHN_V(x) ((x) << FW_EQ_ETH_CMD_PCIECHN_S) + +#define FW_EQ_ETH_CMD_IQID_S 0 +#define FW_EQ_ETH_CMD_IQID_V(x) ((x) << FW_EQ_ETH_CMD_IQID_S) + +#define FW_EQ_ETH_CMD_DCAEN_S 31 +#define FW_EQ_ETH_CMD_DCAEN_V(x) ((x) << FW_EQ_ETH_CMD_DCAEN_S) + +#define FW_EQ_ETH_CMD_DCACPU_S 26 +#define FW_EQ_ETH_CMD_DCACPU_V(x) ((x) << FW_EQ_ETH_CMD_DCACPU_S) + +#define FW_EQ_ETH_CMD_FBMIN_S 23 +#define FW_EQ_ETH_CMD_FBMIN_V(x) ((x) << FW_EQ_ETH_CMD_FBMIN_S) + +#define FW_EQ_ETH_CMD_FBMAX_S 20 +#define FW_EQ_ETH_CMD_FBMAX_V(x) ((x) << FW_EQ_ETH_CMD_FBMAX_S) + +#define FW_EQ_ETH_CMD_CIDXFTHRESHO_S 19 +#define FW_EQ_ETH_CMD_CIDXFTHRESHO_V(x) ((x) << FW_EQ_ETH_CMD_CIDXFTHRESHO_S) + +#define FW_EQ_ETH_CMD_CIDXFTHRESH_S 16 +#define FW_EQ_ETH_CMD_CIDXFTHRESH_V(x) ((x) << FW_EQ_ETH_CMD_CIDXFTHRESH_S) + +#define FW_EQ_ETH_CMD_EQSIZE_S 0 +#define FW_EQ_ETH_CMD_EQSIZE_V(x) ((x) << FW_EQ_ETH_CMD_EQSIZE_S) + +#define FW_EQ_ETH_CMD_AUTOEQUIQE_S 31 +#define FW_EQ_ETH_CMD_AUTOEQUIQE_V(x) ((x) << FW_EQ_ETH_CMD_AUTOEQUIQE_S) +#define FW_EQ_ETH_CMD_AUTOEQUIQE_F FW_EQ_ETH_CMD_AUTOEQUIQE_V(1U) + +#define FW_EQ_ETH_CMD_AUTOEQUEQE_S 30 +#define FW_EQ_ETH_CMD_AUTOEQUEQE_V(x) ((x) << FW_EQ_ETH_CMD_AUTOEQUEQE_S) +#define FW_EQ_ETH_CMD_AUTOEQUEQE_F FW_EQ_ETH_CMD_AUTOEQUEQE_V(1U) + +#define FW_EQ_ETH_CMD_VIID_S 16 +#define FW_EQ_ETH_CMD_VIID_V(x) ((x) << FW_EQ_ETH_CMD_VIID_S) + +#define FW_EQ_ETH_CMD_TIMEREN_S 3 +#define FW_EQ_ETH_CMD_TIMEREN_M 0x1 +#define FW_EQ_ETH_CMD_TIMEREN_V(x) ((x) << FW_EQ_ETH_CMD_TIMEREN_S) +#define FW_EQ_ETH_CMD_TIMEREN_G(x) \ + (((x) >> FW_EQ_ETH_CMD_TIMEREN_S) & FW_EQ_ETH_CMD_TIMEREN_M) +#define FW_EQ_ETH_CMD_TIMEREN_F FW_EQ_ETH_CMD_TIMEREN_V(1U) + +#define FW_EQ_ETH_CMD_TIMERIX_S 0 +#define FW_EQ_ETH_CMD_TIMERIX_M 0x7 +#define FW_EQ_ETH_CMD_TIMERIX_V(x) ((x) << FW_EQ_ETH_CMD_TIMERIX_S) +#define FW_EQ_ETH_CMD_TIMERIX_G(x) \ + (((x) >> FW_EQ_ETH_CMD_TIMERIX_S) & FW_EQ_ETH_CMD_TIMERIX_M) + +struct fw_eq_ctrl_cmd { + __be32 op_to_vfn; + __be32 alloc_to_len16; + __be32 cmpliqid_eqid; + __be32 physeqid_pkd; + __be32 fetchszm_to_iqid; + __be32 dcaen_to_eqsize; + __be64 eqaddr; +}; + +#define FW_EQ_CTRL_CMD_PFN_S 8 +#define FW_EQ_CTRL_CMD_PFN_V(x) ((x) << FW_EQ_CTRL_CMD_PFN_S) + +#define FW_EQ_CTRL_CMD_VFN_S 0 +#define FW_EQ_CTRL_CMD_VFN_V(x) ((x) << FW_EQ_CTRL_CMD_VFN_S) + +#define FW_EQ_CTRL_CMD_ALLOC_S 31 +#define FW_EQ_CTRL_CMD_ALLOC_V(x) ((x) << FW_EQ_CTRL_CMD_ALLOC_S) +#define FW_EQ_CTRL_CMD_ALLOC_F FW_EQ_CTRL_CMD_ALLOC_V(1U) + +#define FW_EQ_CTRL_CMD_FREE_S 30 +#define FW_EQ_CTRL_CMD_FREE_V(x) ((x) << FW_EQ_CTRL_CMD_FREE_S) +#define FW_EQ_CTRL_CMD_FREE_F FW_EQ_CTRL_CMD_FREE_V(1U) + +#define FW_EQ_CTRL_CMD_MODIFY_S 29 +#define FW_EQ_CTRL_CMD_MODIFY_V(x) ((x) << FW_EQ_CTRL_CMD_MODIFY_S) +#define FW_EQ_CTRL_CMD_MODIFY_F FW_EQ_CTRL_CMD_MODIFY_V(1U) + +#define FW_EQ_CTRL_CMD_EQSTART_S 28 +#define FW_EQ_CTRL_CMD_EQSTART_V(x) ((x) << FW_EQ_CTRL_CMD_EQSTART_S) +#define FW_EQ_CTRL_CMD_EQSTART_F FW_EQ_CTRL_CMD_EQSTART_V(1U) + +#define FW_EQ_CTRL_CMD_EQSTOP_S 27 +#define FW_EQ_CTRL_CMD_EQSTOP_V(x) ((x) << FW_EQ_CTRL_CMD_EQSTOP_S) +#define FW_EQ_CTRL_CMD_EQSTOP_F FW_EQ_CTRL_CMD_EQSTOP_V(1U) + +#define FW_EQ_CTRL_CMD_CMPLIQID_S 20 +#define FW_EQ_CTRL_CMD_CMPLIQID_V(x) ((x) << FW_EQ_CTRL_CMD_CMPLIQID_S) + +#define FW_EQ_CTRL_CMD_EQID_S 0 +#define FW_EQ_CTRL_CMD_EQID_M 0xfffff +#define FW_EQ_CTRL_CMD_EQID_V(x) ((x) << FW_EQ_CTRL_CMD_EQID_S) +#define FW_EQ_CTRL_CMD_EQID_G(x) \ + (((x) >> FW_EQ_CTRL_CMD_EQID_S) & FW_EQ_CTRL_CMD_EQID_M) + +#define FW_EQ_CTRL_CMD_PHYSEQID_S 0 +#define FW_EQ_CTRL_CMD_PHYSEQID_M 0xfffff +#define FW_EQ_CTRL_CMD_PHYSEQID_G(x) \ + (((x) >> FW_EQ_CTRL_CMD_PHYSEQID_S) & FW_EQ_CTRL_CMD_PHYSEQID_M) + +#define FW_EQ_CTRL_CMD_FETCHSZM_S 26 +#define FW_EQ_CTRL_CMD_FETCHSZM_V(x) ((x) << FW_EQ_CTRL_CMD_FETCHSZM_S) +#define FW_EQ_CTRL_CMD_FETCHSZM_F FW_EQ_CTRL_CMD_FETCHSZM_V(1U) + +#define FW_EQ_CTRL_CMD_STATUSPGNS_S 25 +#define FW_EQ_CTRL_CMD_STATUSPGNS_V(x) ((x) << FW_EQ_CTRL_CMD_STATUSPGNS_S) +#define FW_EQ_CTRL_CMD_STATUSPGNS_F FW_EQ_CTRL_CMD_STATUSPGNS_V(1U) + +#define FW_EQ_CTRL_CMD_STATUSPGRO_S 24 +#define FW_EQ_CTRL_CMD_STATUSPGRO_V(x) ((x) << FW_EQ_CTRL_CMD_STATUSPGRO_S) +#define FW_EQ_CTRL_CMD_STATUSPGRO_F FW_EQ_CTRL_CMD_STATUSPGRO_V(1U) + +#define FW_EQ_CTRL_CMD_FETCHNS_S 23 +#define FW_EQ_CTRL_CMD_FETCHNS_V(x) ((x) << FW_EQ_CTRL_CMD_FETCHNS_S) +#define FW_EQ_CTRL_CMD_FETCHNS_F FW_EQ_CTRL_CMD_FETCHNS_V(1U) + +#define FW_EQ_CTRL_CMD_FETCHRO_S 22 +#define FW_EQ_CTRL_CMD_FETCHRO_V(x) ((x) << FW_EQ_CTRL_CMD_FETCHRO_S) +#define FW_EQ_CTRL_CMD_FETCHRO_F FW_EQ_CTRL_CMD_FETCHRO_V(1U) + +#define FW_EQ_CTRL_CMD_HOSTFCMODE_S 20 +#define FW_EQ_CTRL_CMD_HOSTFCMODE_V(x) ((x) << FW_EQ_CTRL_CMD_HOSTFCMODE_S) + +#define FW_EQ_CTRL_CMD_CPRIO_S 19 +#define FW_EQ_CTRL_CMD_CPRIO_V(x) ((x) << FW_EQ_CTRL_CMD_CPRIO_S) + +#define FW_EQ_CTRL_CMD_ONCHIP_S 18 +#define FW_EQ_CTRL_CMD_ONCHIP_V(x) ((x) << FW_EQ_CTRL_CMD_ONCHIP_S) + +#define FW_EQ_CTRL_CMD_PCIECHN_S 16 +#define FW_EQ_CTRL_CMD_PCIECHN_V(x) ((x) << FW_EQ_CTRL_CMD_PCIECHN_S) + +#define FW_EQ_CTRL_CMD_IQID_S 0 +#define FW_EQ_CTRL_CMD_IQID_V(x) ((x) << FW_EQ_CTRL_CMD_IQID_S) + +#define FW_EQ_CTRL_CMD_DCAEN_S 31 +#define FW_EQ_CTRL_CMD_DCAEN_V(x) ((x) << FW_EQ_CTRL_CMD_DCAEN_S) + +#define FW_EQ_CTRL_CMD_DCACPU_S 26 +#define FW_EQ_CTRL_CMD_DCACPU_V(x) ((x) << FW_EQ_CTRL_CMD_DCACPU_S) + +#define FW_EQ_CTRL_CMD_FBMIN_S 23 +#define FW_EQ_CTRL_CMD_FBMIN_V(x) ((x) << FW_EQ_CTRL_CMD_FBMIN_S) + +#define FW_EQ_CTRL_CMD_FBMAX_S 20 +#define FW_EQ_CTRL_CMD_FBMAX_V(x) ((x) << FW_EQ_CTRL_CMD_FBMAX_S) + +#define FW_EQ_CTRL_CMD_CIDXFTHRESHO_S 19 +#define FW_EQ_CTRL_CMD_CIDXFTHRESHO_V(x) \ + ((x) << FW_EQ_CTRL_CMD_CIDXFTHRESHO_S) + +#define FW_EQ_CTRL_CMD_CIDXFTHRESH_S 16 +#define FW_EQ_CTRL_CMD_CIDXFTHRESH_V(x) ((x) << FW_EQ_CTRL_CMD_CIDXFTHRESH_S) + +#define FW_EQ_CTRL_CMD_EQSIZE_S 0 +#define FW_EQ_CTRL_CMD_EQSIZE_V(x) ((x) << FW_EQ_CTRL_CMD_EQSIZE_S) + +struct fw_eq_ofld_cmd { + __be32 op_to_vfn; + __be32 alloc_to_len16; + __be32 eqid_pkd; + __be32 physeqid_pkd; + __be32 fetchszm_to_iqid; + __be32 dcaen_to_eqsize; + __be64 eqaddr; +}; + +#define FW_EQ_OFLD_CMD_PFN_S 8 +#define FW_EQ_OFLD_CMD_PFN_V(x) ((x) << FW_EQ_OFLD_CMD_PFN_S) + +#define FW_EQ_OFLD_CMD_VFN_S 0 +#define FW_EQ_OFLD_CMD_VFN_V(x) ((x) << FW_EQ_OFLD_CMD_VFN_S) + +#define FW_EQ_OFLD_CMD_ALLOC_S 31 +#define FW_EQ_OFLD_CMD_ALLOC_V(x) ((x) << FW_EQ_OFLD_CMD_ALLOC_S) +#define FW_EQ_OFLD_CMD_ALLOC_F FW_EQ_OFLD_CMD_ALLOC_V(1U) + +#define FW_EQ_OFLD_CMD_FREE_S 30 +#define FW_EQ_OFLD_CMD_FREE_V(x) ((x) << FW_EQ_OFLD_CMD_FREE_S) +#define FW_EQ_OFLD_CMD_FREE_F FW_EQ_OFLD_CMD_FREE_V(1U) + +#define FW_EQ_OFLD_CMD_MODIFY_S 29 +#define FW_EQ_OFLD_CMD_MODIFY_V(x) ((x) << FW_EQ_OFLD_CMD_MODIFY_S) +#define FW_EQ_OFLD_CMD_MODIFY_F FW_EQ_OFLD_CMD_MODIFY_V(1U) + +#define FW_EQ_OFLD_CMD_EQSTART_S 28 +#define FW_EQ_OFLD_CMD_EQSTART_V(x) ((x) << FW_EQ_OFLD_CMD_EQSTART_S) +#define FW_EQ_OFLD_CMD_EQSTART_F FW_EQ_OFLD_CMD_EQSTART_V(1U) + +#define FW_EQ_OFLD_CMD_EQSTOP_S 27 +#define FW_EQ_OFLD_CMD_EQSTOP_V(x) ((x) << FW_EQ_OFLD_CMD_EQSTOP_S) +#define FW_EQ_OFLD_CMD_EQSTOP_F FW_EQ_OFLD_CMD_EQSTOP_V(1U) + +#define FW_EQ_OFLD_CMD_EQID_S 0 +#define FW_EQ_OFLD_CMD_EQID_M 0xfffff +#define FW_EQ_OFLD_CMD_EQID_V(x) ((x) << FW_EQ_OFLD_CMD_EQID_S) +#define FW_EQ_OFLD_CMD_EQID_G(x) \ + (((x) >> FW_EQ_OFLD_CMD_EQID_S) & FW_EQ_OFLD_CMD_EQID_M) + +#define FW_EQ_OFLD_CMD_PHYSEQID_S 0 +#define FW_EQ_OFLD_CMD_PHYSEQID_M 0xfffff +#define FW_EQ_OFLD_CMD_PHYSEQID_G(x) \ + (((x) >> FW_EQ_OFLD_CMD_PHYSEQID_S) & FW_EQ_OFLD_CMD_PHYSEQID_M) + +#define FW_EQ_OFLD_CMD_FETCHSZM_S 26 +#define FW_EQ_OFLD_CMD_FETCHSZM_V(x) ((x) << FW_EQ_OFLD_CMD_FETCHSZM_S) + +#define FW_EQ_OFLD_CMD_STATUSPGNS_S 25 +#define FW_EQ_OFLD_CMD_STATUSPGNS_V(x) ((x) << FW_EQ_OFLD_CMD_STATUSPGNS_S) + +#define FW_EQ_OFLD_CMD_STATUSPGRO_S 24 +#define FW_EQ_OFLD_CMD_STATUSPGRO_V(x) ((x) << FW_EQ_OFLD_CMD_STATUSPGRO_S) + +#define FW_EQ_OFLD_CMD_FETCHNS_S 23 +#define FW_EQ_OFLD_CMD_FETCHNS_V(x) ((x) << FW_EQ_OFLD_CMD_FETCHNS_S) + +#define FW_EQ_OFLD_CMD_FETCHRO_S 22 +#define FW_EQ_OFLD_CMD_FETCHRO_V(x) ((x) << FW_EQ_OFLD_CMD_FETCHRO_S) +#define FW_EQ_OFLD_CMD_FETCHRO_F FW_EQ_OFLD_CMD_FETCHRO_V(1U) + +#define FW_EQ_OFLD_CMD_HOSTFCMODE_S 20 +#define FW_EQ_OFLD_CMD_HOSTFCMODE_V(x) ((x) << FW_EQ_OFLD_CMD_HOSTFCMODE_S) + +#define FW_EQ_OFLD_CMD_CPRIO_S 19 +#define FW_EQ_OFLD_CMD_CPRIO_V(x) ((x) << FW_EQ_OFLD_CMD_CPRIO_S) + +#define FW_EQ_OFLD_CMD_ONCHIP_S 18 +#define FW_EQ_OFLD_CMD_ONCHIP_V(x) ((x) << FW_EQ_OFLD_CMD_ONCHIP_S) + +#define FW_EQ_OFLD_CMD_PCIECHN_S 16 +#define FW_EQ_OFLD_CMD_PCIECHN_V(x) ((x) << FW_EQ_OFLD_CMD_PCIECHN_S) + +#define FW_EQ_OFLD_CMD_IQID_S 0 +#define FW_EQ_OFLD_CMD_IQID_V(x) ((x) << FW_EQ_OFLD_CMD_IQID_S) + +#define FW_EQ_OFLD_CMD_DCAEN_S 31 +#define FW_EQ_OFLD_CMD_DCAEN_V(x) ((x) << FW_EQ_OFLD_CMD_DCAEN_S) + +#define FW_EQ_OFLD_CMD_DCACPU_S 26 +#define FW_EQ_OFLD_CMD_DCACPU_V(x) ((x) << FW_EQ_OFLD_CMD_DCACPU_S) + +#define FW_EQ_OFLD_CMD_FBMIN_S 23 +#define FW_EQ_OFLD_CMD_FBMIN_V(x) ((x) << FW_EQ_OFLD_CMD_FBMIN_S) + +#define FW_EQ_OFLD_CMD_FBMAX_S 20 +#define FW_EQ_OFLD_CMD_FBMAX_V(x) ((x) << FW_EQ_OFLD_CMD_FBMAX_S) + +#define FW_EQ_OFLD_CMD_CIDXFTHRESHO_S 19 +#define FW_EQ_OFLD_CMD_CIDXFTHRESHO_V(x) \ + ((x) << FW_EQ_OFLD_CMD_CIDXFTHRESHO_S) + +#define FW_EQ_OFLD_CMD_CIDXFTHRESH_S 16 +#define FW_EQ_OFLD_CMD_CIDXFTHRESH_V(x) ((x) << FW_EQ_OFLD_CMD_CIDXFTHRESH_S) + +#define FW_EQ_OFLD_CMD_EQSIZE_S 0 +#define FW_EQ_OFLD_CMD_EQSIZE_V(x) ((x) << FW_EQ_OFLD_CMD_EQSIZE_S) + +/* + * Macros for VIID parsing: + * VIID - [10:8] PFN, [7] VI Valid, [6:0] VI number + */ + +#define FW_VIID_PFN_S 8 +#define FW_VIID_PFN_M 0x7 +#define FW_VIID_PFN_G(x) (((x) >> FW_VIID_PFN_S) & FW_VIID_PFN_M) + +#define FW_VIID_VIVLD_S 7 +#define FW_VIID_VIVLD_M 0x1 +#define FW_VIID_VIVLD_G(x) (((x) >> FW_VIID_VIVLD_S) & FW_VIID_VIVLD_M) + +#define FW_VIID_VIN_S 0 +#define FW_VIID_VIN_M 0x7F +#define FW_VIID_VIN_G(x) (((x) >> FW_VIID_VIN_S) & FW_VIID_VIN_M) + +struct fw_vi_cmd { + __be32 op_to_vfn; + __be32 alloc_to_len16; + __be16 type_viid; + u8 mac[6]; + u8 portid_pkd; + u8 nmac; + u8 nmac0[6]; + __be16 rsssize_pkd; + u8 nmac1[6]; + __be16 idsiiq_pkd; + u8 nmac2[6]; + __be16 idseiq_pkd; + u8 nmac3[6]; + __be64 r9; + __be64 r10; +}; + +#define FW_VI_CMD_PFN_S 8 +#define FW_VI_CMD_PFN_V(x) ((x) << FW_VI_CMD_PFN_S) + +#define FW_VI_CMD_VFN_S 0 +#define FW_VI_CMD_VFN_V(x) ((x) << FW_VI_CMD_VFN_S) + +#define FW_VI_CMD_ALLOC_S 31 +#define FW_VI_CMD_ALLOC_V(x) ((x) << FW_VI_CMD_ALLOC_S) +#define FW_VI_CMD_ALLOC_F FW_VI_CMD_ALLOC_V(1U) + +#define FW_VI_CMD_FREE_S 30 +#define FW_VI_CMD_FREE_V(x) ((x) << FW_VI_CMD_FREE_S) +#define FW_VI_CMD_FREE_F FW_VI_CMD_FREE_V(1U) + +#define FW_VI_CMD_VFVLD_S 24 +#define FW_VI_CMD_VFVLD_M 0x1 +#define FW_VI_CMD_VFVLD_V(x) ((x) << FW_VI_CMD_VFVLD_S) +#define FW_VI_CMD_VFVLD_G(x) \ + (((x) >> FW_VI_CMD_VFVLD_S) & FW_VI_CMD_VFVLD_M) +#define FW_VI_CMD_VFVLD_F FW_VI_CMD_VFVLD_V(1U) + +#define FW_VI_CMD_VIN_S 16 +#define FW_VI_CMD_VIN_M 0xff +#define FW_VI_CMD_VIN_V(x) ((x) << FW_VI_CMD_VIN_S) +#define FW_VI_CMD_VIN_G(x) \ + (((x) >> FW_VI_CMD_VIN_S) & FW_VI_CMD_VIN_M) + +#define FW_VI_CMD_VIID_S 0 +#define FW_VI_CMD_VIID_M 0xfff +#define FW_VI_CMD_VIID_V(x) ((x) << FW_VI_CMD_VIID_S) +#define FW_VI_CMD_VIID_G(x) (((x) >> FW_VI_CMD_VIID_S) & FW_VI_CMD_VIID_M) + +#define FW_VI_CMD_PORTID_S 4 +#define FW_VI_CMD_PORTID_M 0xf +#define FW_VI_CMD_PORTID_V(x) ((x) << FW_VI_CMD_PORTID_S) +#define FW_VI_CMD_PORTID_G(x) \ + (((x) >> FW_VI_CMD_PORTID_S) & FW_VI_CMD_PORTID_M) + +#define FW_VI_CMD_RSSSIZE_S 0 +#define FW_VI_CMD_RSSSIZE_M 0x7ff +#define FW_VI_CMD_RSSSIZE_G(x) \ + (((x) >> FW_VI_CMD_RSSSIZE_S) & FW_VI_CMD_RSSSIZE_M) + +/* Special VI_MAC command index ids */ +#define FW_VI_MAC_ADD_MAC 0x3FF +#define FW_VI_MAC_ADD_PERSIST_MAC 0x3FE +#define FW_VI_MAC_MAC_BASED_FREE 0x3FD +#define FW_VI_MAC_ID_BASED_FREE 0x3FC +#define FW_CLS_TCAM_NUM_ENTRIES 336 + +enum fw_vi_mac_smac { + FW_VI_MAC_MPS_TCAM_ENTRY, + FW_VI_MAC_MPS_TCAM_ONLY, + FW_VI_MAC_SMT_ONLY, + FW_VI_MAC_SMT_AND_MPSTCAM +}; + +enum fw_vi_mac_result { + FW_VI_MAC_R_SUCCESS, + FW_VI_MAC_R_F_NONEXISTENT_NOMEM, + FW_VI_MAC_R_SMAC_FAIL, + FW_VI_MAC_R_F_ACL_CHECK +}; + +enum fw_vi_mac_entry_types { + FW_VI_MAC_TYPE_EXACTMAC, + FW_VI_MAC_TYPE_HASHVEC, + FW_VI_MAC_TYPE_RAW, + FW_VI_MAC_TYPE_EXACTMAC_VNI, +}; + +struct fw_vi_mac_cmd { + __be32 op_to_viid; + __be32 freemacs_to_len16; + union fw_vi_mac { + struct fw_vi_mac_exact { + __be16 valid_to_idx; + u8 macaddr[6]; + } exact[7]; + struct fw_vi_mac_hash { + __be64 hashvec; + } hash; + struct fw_vi_mac_raw { + __be32 raw_idx_pkd; + __be32 data0_pkd; + __be32 data1[2]; + __be64 data0m_pkd; + __be32 data1m[2]; + } raw; + struct fw_vi_mac_vni { + __be16 valid_to_idx; + __u8 macaddr[6]; + __be16 r7; + __u8 macaddr_mask[6]; + __be32 lookup_type_to_vni; + __be32 vni_mask_pkd; + } exact_vni[2]; + } u; +}; + +#define FW_VI_MAC_CMD_SMTID_S 12 +#define FW_VI_MAC_CMD_SMTID_M 0xff +#define FW_VI_MAC_CMD_SMTID_V(x) ((x) << FW_VI_MAC_CMD_SMTID_S) +#define FW_VI_MAC_CMD_SMTID_G(x) \ + (((x) >> FW_VI_MAC_CMD_SMTID_S) & FW_VI_MAC_CMD_SMTID_M) + +#define FW_VI_MAC_CMD_VIID_S 0 +#define FW_VI_MAC_CMD_VIID_V(x) ((x) << FW_VI_MAC_CMD_VIID_S) + +#define FW_VI_MAC_CMD_FREEMACS_S 31 +#define FW_VI_MAC_CMD_FREEMACS_V(x) ((x) << FW_VI_MAC_CMD_FREEMACS_S) + +#define FW_VI_MAC_CMD_ENTRY_TYPE_S 23 +#define FW_VI_MAC_CMD_ENTRY_TYPE_M 0x7 +#define FW_VI_MAC_CMD_ENTRY_TYPE_V(x) ((x) << FW_VI_MAC_CMD_ENTRY_TYPE_S) +#define FW_VI_MAC_CMD_ENTRY_TYPE_G(x) \ + (((x) >> FW_VI_MAC_CMD_ENTRY_TYPE_S) & FW_VI_MAC_CMD_ENTRY_TYPE_M) + +#define FW_VI_MAC_CMD_HASHVECEN_S 23 +#define FW_VI_MAC_CMD_HASHVECEN_V(x) ((x) << FW_VI_MAC_CMD_HASHVECEN_S) +#define FW_VI_MAC_CMD_HASHVECEN_F FW_VI_MAC_CMD_HASHVECEN_V(1U) + +#define FW_VI_MAC_CMD_HASHUNIEN_S 22 +#define FW_VI_MAC_CMD_HASHUNIEN_V(x) ((x) << FW_VI_MAC_CMD_HASHUNIEN_S) + +#define FW_VI_MAC_CMD_VALID_S 15 +#define FW_VI_MAC_CMD_VALID_V(x) ((x) << FW_VI_MAC_CMD_VALID_S) +#define FW_VI_MAC_CMD_VALID_F FW_VI_MAC_CMD_VALID_V(1U) + +#define FW_VI_MAC_CMD_PRIO_S 12 +#define FW_VI_MAC_CMD_PRIO_V(x) ((x) << FW_VI_MAC_CMD_PRIO_S) + +#define FW_VI_MAC_CMD_SMAC_RESULT_S 10 +#define FW_VI_MAC_CMD_SMAC_RESULT_M 0x3 +#define FW_VI_MAC_CMD_SMAC_RESULT_V(x) ((x) << FW_VI_MAC_CMD_SMAC_RESULT_S) +#define FW_VI_MAC_CMD_SMAC_RESULT_G(x) \ + (((x) >> FW_VI_MAC_CMD_SMAC_RESULT_S) & FW_VI_MAC_CMD_SMAC_RESULT_M) + +#define FW_VI_MAC_CMD_IDX_S 0 +#define FW_VI_MAC_CMD_IDX_M 0x3ff +#define FW_VI_MAC_CMD_IDX_V(x) ((x) << FW_VI_MAC_CMD_IDX_S) +#define FW_VI_MAC_CMD_IDX_G(x) \ + (((x) >> FW_VI_MAC_CMD_IDX_S) & FW_VI_MAC_CMD_IDX_M) + +#define FW_VI_MAC_CMD_RAW_IDX_S 16 +#define FW_VI_MAC_CMD_RAW_IDX_M 0xffff +#define FW_VI_MAC_CMD_RAW_IDX_V(x) ((x) << FW_VI_MAC_CMD_RAW_IDX_S) +#define FW_VI_MAC_CMD_RAW_IDX_G(x) \ + (((x) >> FW_VI_MAC_CMD_RAW_IDX_S) & FW_VI_MAC_CMD_RAW_IDX_M) + +#define FW_VI_MAC_CMD_LOOKUP_TYPE_S 31 +#define FW_VI_MAC_CMD_LOOKUP_TYPE_M 0x1 +#define FW_VI_MAC_CMD_LOOKUP_TYPE_V(x) ((x) << FW_VI_MAC_CMD_LOOKUP_TYPE_S) +#define FW_VI_MAC_CMD_LOOKUP_TYPE_G(x) \ + (((x) >> FW_VI_MAC_CMD_LOOKUP_TYPE_S) & FW_VI_MAC_CMD_LOOKUP_TYPE_M) +#define FW_VI_MAC_CMD_LOOKUP_TYPE_F FW_VI_MAC_CMD_LOOKUP_TYPE_V(1U) + +#define FW_VI_MAC_CMD_DIP_HIT_S 30 +#define FW_VI_MAC_CMD_DIP_HIT_M 0x1 +#define FW_VI_MAC_CMD_DIP_HIT_V(x) ((x) << FW_VI_MAC_CMD_DIP_HIT_S) +#define FW_VI_MAC_CMD_DIP_HIT_G(x) \ + (((x) >> FW_VI_MAC_CMD_DIP_HIT_S) & FW_VI_MAC_CMD_DIP_HIT_M) +#define FW_VI_MAC_CMD_DIP_HIT_F FW_VI_MAC_CMD_DIP_HIT_V(1U) + +#define FW_VI_MAC_CMD_VNI_S 0 +#define FW_VI_MAC_CMD_VNI_M 0xffffff +#define FW_VI_MAC_CMD_VNI_V(x) ((x) << FW_VI_MAC_CMD_VNI_S) +#define FW_VI_MAC_CMD_VNI_G(x) \ + (((x) >> FW_VI_MAC_CMD_VNI_S) & FW_VI_MAC_CMD_VNI_M) + +#define FW_VI_MAC_CMD_VNI_MASK_S 0 +#define FW_VI_MAC_CMD_VNI_MASK_M 0xffffff +#define FW_VI_MAC_CMD_VNI_MASK_V(x) ((x) << FW_VI_MAC_CMD_VNI_MASK_S) +#define FW_VI_MAC_CMD_VNI_MASK_G(x) \ + (((x) >> FW_VI_MAC_CMD_VNI_MASK_S) & FW_VI_MAC_CMD_VNI_MASK_M) + +#define FW_RXMODE_MTU_NO_CHG 65535 + +struct fw_vi_rxmode_cmd { + __be32 op_to_viid; + __be32 retval_len16; + __be32 mtu_to_vlanexen; + __be32 r4_lo; +}; + +#define FW_VI_RXMODE_CMD_VIID_S 0 +#define FW_VI_RXMODE_CMD_VIID_V(x) ((x) << FW_VI_RXMODE_CMD_VIID_S) + +#define FW_VI_RXMODE_CMD_MTU_S 16 +#define FW_VI_RXMODE_CMD_MTU_M 0xffff +#define FW_VI_RXMODE_CMD_MTU_V(x) ((x) << FW_VI_RXMODE_CMD_MTU_S) + +#define FW_VI_RXMODE_CMD_PROMISCEN_S 14 +#define FW_VI_RXMODE_CMD_PROMISCEN_M 0x3 +#define FW_VI_RXMODE_CMD_PROMISCEN_V(x) ((x) << FW_VI_RXMODE_CMD_PROMISCEN_S) + +#define FW_VI_RXMODE_CMD_ALLMULTIEN_S 12 +#define FW_VI_RXMODE_CMD_ALLMULTIEN_M 0x3 +#define FW_VI_RXMODE_CMD_ALLMULTIEN_V(x) \ + ((x) << FW_VI_RXMODE_CMD_ALLMULTIEN_S) + +#define FW_VI_RXMODE_CMD_BROADCASTEN_S 10 +#define FW_VI_RXMODE_CMD_BROADCASTEN_M 0x3 +#define FW_VI_RXMODE_CMD_BROADCASTEN_V(x) \ + ((x) << FW_VI_RXMODE_CMD_BROADCASTEN_S) + +#define FW_VI_RXMODE_CMD_VLANEXEN_S 8 +#define FW_VI_RXMODE_CMD_VLANEXEN_M 0x3 +#define FW_VI_RXMODE_CMD_VLANEXEN_V(x) ((x) << FW_VI_RXMODE_CMD_VLANEXEN_S) + +struct fw_vi_enable_cmd { + __be32 op_to_viid; + __be32 ien_to_len16; + __be16 blinkdur; + __be16 r3; + __be32 r4; +}; + +#define FW_VI_ENABLE_CMD_VIID_S 0 +#define FW_VI_ENABLE_CMD_VIID_V(x) ((x) << FW_VI_ENABLE_CMD_VIID_S) + +#define FW_VI_ENABLE_CMD_IEN_S 31 +#define FW_VI_ENABLE_CMD_IEN_V(x) ((x) << FW_VI_ENABLE_CMD_IEN_S) + +#define FW_VI_ENABLE_CMD_EEN_S 30 +#define FW_VI_ENABLE_CMD_EEN_V(x) ((x) << FW_VI_ENABLE_CMD_EEN_S) + +#define FW_VI_ENABLE_CMD_LED_S 29 +#define FW_VI_ENABLE_CMD_LED_V(x) ((x) << FW_VI_ENABLE_CMD_LED_S) +#define FW_VI_ENABLE_CMD_LED_F FW_VI_ENABLE_CMD_LED_V(1U) + +#define FW_VI_ENABLE_CMD_DCB_INFO_S 28 +#define FW_VI_ENABLE_CMD_DCB_INFO_V(x) ((x) << FW_VI_ENABLE_CMD_DCB_INFO_S) + +/* VI VF stats offset definitions */ +#define VI_VF_NUM_STATS 16 +enum fw_vi_stats_vf_index { + FW_VI_VF_STAT_TX_BCAST_BYTES_IX, + FW_VI_VF_STAT_TX_BCAST_FRAMES_IX, + FW_VI_VF_STAT_TX_MCAST_BYTES_IX, + FW_VI_VF_STAT_TX_MCAST_FRAMES_IX, + FW_VI_VF_STAT_TX_UCAST_BYTES_IX, + FW_VI_VF_STAT_TX_UCAST_FRAMES_IX, + FW_VI_VF_STAT_TX_DROP_FRAMES_IX, + FW_VI_VF_STAT_TX_OFLD_BYTES_IX, + FW_VI_VF_STAT_TX_OFLD_FRAMES_IX, + FW_VI_VF_STAT_RX_BCAST_BYTES_IX, + FW_VI_VF_STAT_RX_BCAST_FRAMES_IX, + FW_VI_VF_STAT_RX_MCAST_BYTES_IX, + FW_VI_VF_STAT_RX_MCAST_FRAMES_IX, + FW_VI_VF_STAT_RX_UCAST_BYTES_IX, + FW_VI_VF_STAT_RX_UCAST_FRAMES_IX, + FW_VI_VF_STAT_RX_ERR_FRAMES_IX +}; + +/* VI PF stats offset definitions */ +#define VI_PF_NUM_STATS 17 +enum fw_vi_stats_pf_index { + FW_VI_PF_STAT_TX_BCAST_BYTES_IX, + FW_VI_PF_STAT_TX_BCAST_FRAMES_IX, + FW_VI_PF_STAT_TX_MCAST_BYTES_IX, + FW_VI_PF_STAT_TX_MCAST_FRAMES_IX, + FW_VI_PF_STAT_TX_UCAST_BYTES_IX, + FW_VI_PF_STAT_TX_UCAST_FRAMES_IX, + FW_VI_PF_STAT_TX_OFLD_BYTES_IX, + FW_VI_PF_STAT_TX_OFLD_FRAMES_IX, + FW_VI_PF_STAT_RX_BYTES_IX, + FW_VI_PF_STAT_RX_FRAMES_IX, + FW_VI_PF_STAT_RX_BCAST_BYTES_IX, + FW_VI_PF_STAT_RX_BCAST_FRAMES_IX, + FW_VI_PF_STAT_RX_MCAST_BYTES_IX, + FW_VI_PF_STAT_RX_MCAST_FRAMES_IX, + FW_VI_PF_STAT_RX_UCAST_BYTES_IX, + FW_VI_PF_STAT_RX_UCAST_FRAMES_IX, + FW_VI_PF_STAT_RX_ERR_FRAMES_IX +}; + +struct fw_vi_stats_cmd { + __be32 op_to_viid; + __be32 retval_len16; + union fw_vi_stats { + struct fw_vi_stats_ctl { + __be16 nstats_ix; + __be16 r6; + __be32 r7; + __be64 stat0; + __be64 stat1; + __be64 stat2; + __be64 stat3; + __be64 stat4; + __be64 stat5; + } ctl; + struct fw_vi_stats_pf { + __be64 tx_bcast_bytes; + __be64 tx_bcast_frames; + __be64 tx_mcast_bytes; + __be64 tx_mcast_frames; + __be64 tx_ucast_bytes; + __be64 tx_ucast_frames; + __be64 tx_offload_bytes; + __be64 tx_offload_frames; + __be64 rx_pf_bytes; + __be64 rx_pf_frames; + __be64 rx_bcast_bytes; + __be64 rx_bcast_frames; + __be64 rx_mcast_bytes; + __be64 rx_mcast_frames; + __be64 rx_ucast_bytes; + __be64 rx_ucast_frames; + __be64 rx_err_frames; + } pf; + struct fw_vi_stats_vf { + __be64 tx_bcast_bytes; + __be64 tx_bcast_frames; + __be64 tx_mcast_bytes; + __be64 tx_mcast_frames; + __be64 tx_ucast_bytes; + __be64 tx_ucast_frames; + __be64 tx_drop_frames; + __be64 tx_offload_bytes; + __be64 tx_offload_frames; + __be64 rx_bcast_bytes; + __be64 rx_bcast_frames; + __be64 rx_mcast_bytes; + __be64 rx_mcast_frames; + __be64 rx_ucast_bytes; + __be64 rx_ucast_frames; + __be64 rx_err_frames; + } vf; + } u; +}; + +#define FW_VI_STATS_CMD_VIID_S 0 +#define FW_VI_STATS_CMD_VIID_V(x) ((x) << FW_VI_STATS_CMD_VIID_S) + +#define FW_VI_STATS_CMD_NSTATS_S 12 +#define FW_VI_STATS_CMD_NSTATS_V(x) ((x) << FW_VI_STATS_CMD_NSTATS_S) + +#define FW_VI_STATS_CMD_IX_S 0 +#define FW_VI_STATS_CMD_IX_V(x) ((x) << FW_VI_STATS_CMD_IX_S) + +struct fw_acl_mac_cmd { + __be32 op_to_vfn; + __be32 en_to_len16; + u8 nmac; + u8 r3[7]; + __be16 r4; + u8 macaddr0[6]; + __be16 r5; + u8 macaddr1[6]; + __be16 r6; + u8 macaddr2[6]; + __be16 r7; + u8 macaddr3[6]; +}; + +#define FW_ACL_MAC_CMD_PFN_S 8 +#define FW_ACL_MAC_CMD_PFN_V(x) ((x) << FW_ACL_MAC_CMD_PFN_S) + +#define FW_ACL_MAC_CMD_VFN_S 0 +#define FW_ACL_MAC_CMD_VFN_V(x) ((x) << FW_ACL_MAC_CMD_VFN_S) + +#define FW_ACL_MAC_CMD_EN_S 31 +#define FW_ACL_MAC_CMD_EN_V(x) ((x) << FW_ACL_MAC_CMD_EN_S) + +struct fw_acl_vlan_cmd { + __be32 op_to_vfn; + __be32 en_to_len16; + u8 nvlan; + u8 dropnovlan_fm; + u8 r3_lo[6]; + __be16 vlanid[16]; +}; + +#define FW_ACL_VLAN_CMD_PFN_S 8 +#define FW_ACL_VLAN_CMD_PFN_V(x) ((x) << FW_ACL_VLAN_CMD_PFN_S) + +#define FW_ACL_VLAN_CMD_VFN_S 0 +#define FW_ACL_VLAN_CMD_VFN_V(x) ((x) << FW_ACL_VLAN_CMD_VFN_S) + +#define FW_ACL_VLAN_CMD_EN_S 31 +#define FW_ACL_VLAN_CMD_EN_M 0x1 +#define FW_ACL_VLAN_CMD_EN_V(x) ((x) << FW_ACL_VLAN_CMD_EN_S) +#define FW_ACL_VLAN_CMD_EN_G(x) \ + (((x) >> S_FW_ACL_VLAN_CMD_EN_S) & FW_ACL_VLAN_CMD_EN_M) +#define FW_ACL_VLAN_CMD_EN_F FW_ACL_VLAN_CMD_EN_V(1U) + +#define FW_ACL_VLAN_CMD_DROPNOVLAN_S 7 +#define FW_ACL_VLAN_CMD_DROPNOVLAN_V(x) ((x) << FW_ACL_VLAN_CMD_DROPNOVLAN_S) +#define FW_ACL_VLAN_CMD_DROPNOVLAN_F FW_ACL_VLAN_CMD_DROPNOVLAN_V(1U) + +#define FW_ACL_VLAN_CMD_FM_S 6 +#define FW_ACL_VLAN_CMD_FM_M 0x1 +#define FW_ACL_VLAN_CMD_FM_V(x) ((x) << FW_ACL_VLAN_CMD_FM_S) +#define FW_ACL_VLAN_CMD_FM_G(x) \ + (((x) >> FW_ACL_VLAN_CMD_FM_S) & FW_ACL_VLAN_CMD_FM_M) +#define FW_ACL_VLAN_CMD_FM_F FW_ACL_VLAN_CMD_FM_V(1U) + +/* old 16-bit port capabilities bitmap (fw_port_cap16_t) */ +enum fw_port_cap { + FW_PORT_CAP_SPEED_100M = 0x0001, + FW_PORT_CAP_SPEED_1G = 0x0002, + FW_PORT_CAP_SPEED_25G = 0x0004, + FW_PORT_CAP_SPEED_10G = 0x0008, + FW_PORT_CAP_SPEED_40G = 0x0010, + FW_PORT_CAP_SPEED_100G = 0x0020, + FW_PORT_CAP_FC_RX = 0x0040, + FW_PORT_CAP_FC_TX = 0x0080, + FW_PORT_CAP_ANEG = 0x0100, + FW_PORT_CAP_MDIAUTO = 0x0200, + FW_PORT_CAP_MDISTRAIGHT = 0x0400, + FW_PORT_CAP_FEC_RS = 0x0800, + FW_PORT_CAP_FEC_BASER_RS = 0x1000, + FW_PORT_CAP_FORCE_PAUSE = 0x2000, + FW_PORT_CAP_802_3_PAUSE = 0x4000, + FW_PORT_CAP_802_3_ASM_DIR = 0x8000, +}; + +#define FW_PORT_CAP_SPEED_S 0 +#define FW_PORT_CAP_SPEED_M 0x3f +#define FW_PORT_CAP_SPEED_V(x) ((x) << FW_PORT_CAP_SPEED_S) +#define FW_PORT_CAP_SPEED_G(x) \ + (((x) >> FW_PORT_CAP_SPEED_S) & FW_PORT_CAP_SPEED_M) + +enum fw_port_mdi { + FW_PORT_CAP_MDI_UNCHANGED, + FW_PORT_CAP_MDI_AUTO, + FW_PORT_CAP_MDI_F_STRAIGHT, + FW_PORT_CAP_MDI_F_CROSSOVER +}; + +#define FW_PORT_CAP_MDI_S 9 +#define FW_PORT_CAP_MDI_V(x) ((x) << FW_PORT_CAP_MDI_S) + +/* new 32-bit port capabilities bitmap (fw_port_cap32_t) */ +#define FW_PORT_CAP32_SPEED_100M 0x00000001UL +#define FW_PORT_CAP32_SPEED_1G 0x00000002UL +#define FW_PORT_CAP32_SPEED_10G 0x00000004UL +#define FW_PORT_CAP32_SPEED_25G 0x00000008UL +#define FW_PORT_CAP32_SPEED_40G 0x00000010UL +#define FW_PORT_CAP32_SPEED_50G 0x00000020UL +#define FW_PORT_CAP32_SPEED_100G 0x00000040UL +#define FW_PORT_CAP32_SPEED_200G 0x00000080UL +#define FW_PORT_CAP32_SPEED_400G 0x00000100UL +#define FW_PORT_CAP32_SPEED_RESERVED1 0x00000200UL +#define FW_PORT_CAP32_SPEED_RESERVED2 0x00000400UL +#define FW_PORT_CAP32_SPEED_RESERVED3 0x00000800UL +#define FW_PORT_CAP32_RESERVED1 0x0000f000UL +#define FW_PORT_CAP32_FC_RX 0x00010000UL +#define FW_PORT_CAP32_FC_TX 0x00020000UL +#define FW_PORT_CAP32_802_3_PAUSE 0x00040000UL +#define FW_PORT_CAP32_802_3_ASM_DIR 0x00080000UL +#define FW_PORT_CAP32_ANEG 0x00100000UL +#define FW_PORT_CAP32_MDIAUTO 0x00200000UL +#define FW_PORT_CAP32_MDISTRAIGHT 0x00400000UL +#define FW_PORT_CAP32_FEC_RS 0x00800000UL +#define FW_PORT_CAP32_FEC_BASER_RS 0x01000000UL +#define FW_PORT_CAP32_FEC_RESERVED1 0x02000000UL +#define FW_PORT_CAP32_FEC_RESERVED2 0x04000000UL +#define FW_PORT_CAP32_FEC_RESERVED3 0x08000000UL +#define FW_PORT_CAP32_FORCE_PAUSE 0x10000000UL +#define FW_PORT_CAP32_RESERVED2 0xe0000000UL + +#define FW_PORT_CAP32_SPEED_S 0 +#define FW_PORT_CAP32_SPEED_M 0xfff +#define FW_PORT_CAP32_SPEED_V(x) ((x) << FW_PORT_CAP32_SPEED_S) +#define FW_PORT_CAP32_SPEED_G(x) \ + (((x) >> FW_PORT_CAP32_SPEED_S) & FW_PORT_CAP32_SPEED_M) + +#define FW_PORT_CAP32_FC_S 16 +#define FW_PORT_CAP32_FC_M 0x3 +#define FW_PORT_CAP32_FC_V(x) ((x) << FW_PORT_CAP32_FC_S) +#define FW_PORT_CAP32_FC_G(x) \ + (((x) >> FW_PORT_CAP32_FC_S) & FW_PORT_CAP32_FC_M) + +#define FW_PORT_CAP32_802_3_S 18 +#define FW_PORT_CAP32_802_3_M 0x3 +#define FW_PORT_CAP32_802_3_V(x) ((x) << FW_PORT_CAP32_802_3_S) +#define FW_PORT_CAP32_802_3_G(x) \ + (((x) >> FW_PORT_CAP32_802_3_S) & FW_PORT_CAP32_802_3_M) + +#define FW_PORT_CAP32_ANEG_S 20 +#define FW_PORT_CAP32_ANEG_M 0x1 +#define FW_PORT_CAP32_ANEG_V(x) ((x) << FW_PORT_CAP32_ANEG_S) +#define FW_PORT_CAP32_ANEG_G(x) \ + (((x) >> FW_PORT_CAP32_ANEG_S) & FW_PORT_CAP32_ANEG_M) + +enum fw_port_mdi32 { + FW_PORT_CAP32_MDI_UNCHANGED, + FW_PORT_CAP32_MDI_AUTO, + FW_PORT_CAP32_MDI_F_STRAIGHT, + FW_PORT_CAP32_MDI_F_CROSSOVER +}; + +#define FW_PORT_CAP32_MDI_S 21 +#define FW_PORT_CAP32_MDI_M 3 +#define FW_PORT_CAP32_MDI_V(x) ((x) << FW_PORT_CAP32_MDI_S) +#define FW_PORT_CAP32_MDI_G(x) \ + (((x) >> FW_PORT_CAP32_MDI_S) & FW_PORT_CAP32_MDI_M) + +#define FW_PORT_CAP32_FEC_S 23 +#define FW_PORT_CAP32_FEC_M 0x1f +#define FW_PORT_CAP32_FEC_V(x) ((x) << FW_PORT_CAP32_FEC_S) +#define FW_PORT_CAP32_FEC_G(x) \ + (((x) >> FW_PORT_CAP32_FEC_S) & FW_PORT_CAP32_FEC_M) + +/* macros to isolate various 32-bit Port Capabilities sub-fields */ +#define CAP32_SPEED(__cap32) \ + (FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_M) & __cap32) + +#define CAP32_FEC(__cap32) \ + (FW_PORT_CAP32_FEC_V(FW_PORT_CAP32_FEC_M) & __cap32) + +enum fw_port_action { + FW_PORT_ACTION_L1_CFG = 0x0001, + FW_PORT_ACTION_L2_CFG = 0x0002, + FW_PORT_ACTION_GET_PORT_INFO = 0x0003, + FW_PORT_ACTION_L2_PPP_CFG = 0x0004, + FW_PORT_ACTION_L2_DCB_CFG = 0x0005, + FW_PORT_ACTION_DCB_READ_TRANS = 0x0006, + FW_PORT_ACTION_DCB_READ_RECV = 0x0007, + FW_PORT_ACTION_DCB_READ_DET = 0x0008, + FW_PORT_ACTION_L1_CFG32 = 0x0009, + FW_PORT_ACTION_GET_PORT_INFO32 = 0x000a, + FW_PORT_ACTION_LOW_PWR_TO_NORMAL = 0x0010, + FW_PORT_ACTION_L1_LOW_PWR_EN = 0x0011, + FW_PORT_ACTION_L2_WOL_MODE_EN = 0x0012, + FW_PORT_ACTION_LPBK_TO_NORMAL = 0x0020, + FW_PORT_ACTION_L1_LPBK = 0x0021, + FW_PORT_ACTION_L1_PMA_LPBK = 0x0022, + FW_PORT_ACTION_L1_PCS_LPBK = 0x0023, + FW_PORT_ACTION_L1_PHYXS_CSIDE_LPBK = 0x0024, + FW_PORT_ACTION_L1_PHYXS_ESIDE_LPBK = 0x0025, + FW_PORT_ACTION_PHY_RESET = 0x0040, + FW_PORT_ACTION_PMA_RESET = 0x0041, + FW_PORT_ACTION_PCS_RESET = 0x0042, + FW_PORT_ACTION_PHYXS_RESET = 0x0043, + FW_PORT_ACTION_DTEXS_REEST = 0x0044, + FW_PORT_ACTION_AN_RESET = 0x0045 +}; + +enum fw_port_l2cfg_ctlbf { + FW_PORT_L2_CTLBF_OVLAN0 = 0x01, + FW_PORT_L2_CTLBF_OVLAN1 = 0x02, + FW_PORT_L2_CTLBF_OVLAN2 = 0x04, + FW_PORT_L2_CTLBF_OVLAN3 = 0x08, + FW_PORT_L2_CTLBF_IVLAN = 0x10, + FW_PORT_L2_CTLBF_TXIPG = 0x20 +}; + +enum fw_port_dcb_versions { + FW_PORT_DCB_VER_UNKNOWN, + FW_PORT_DCB_VER_CEE1D0, + FW_PORT_DCB_VER_CEE1D01, + FW_PORT_DCB_VER_IEEE, + FW_PORT_DCB_VER_AUTO = 7 +}; + +enum fw_port_dcb_cfg { + FW_PORT_DCB_CFG_PG = 0x01, + FW_PORT_DCB_CFG_PFC = 0x02, + FW_PORT_DCB_CFG_APPL = 0x04 +}; + +enum fw_port_dcb_cfg_rc { + FW_PORT_DCB_CFG_SUCCESS = 0x0, + FW_PORT_DCB_CFG_ERROR = 0x1 +}; + +enum fw_port_dcb_type { + FW_PORT_DCB_TYPE_PGID = 0x00, + FW_PORT_DCB_TYPE_PGRATE = 0x01, + FW_PORT_DCB_TYPE_PRIORATE = 0x02, + FW_PORT_DCB_TYPE_PFC = 0x03, + FW_PORT_DCB_TYPE_APP_ID = 0x04, + FW_PORT_DCB_TYPE_CONTROL = 0x05, +}; + +enum fw_port_dcb_feature_state { + FW_PORT_DCB_FEATURE_STATE_PENDING = 0x0, + FW_PORT_DCB_FEATURE_STATE_SUCCESS = 0x1, + FW_PORT_DCB_FEATURE_STATE_ERROR = 0x2, + FW_PORT_DCB_FEATURE_STATE_TIMEOUT = 0x3, +}; + +struct fw_port_cmd { + __be32 op_to_portid; + __be32 action_to_len16; + union fw_port { + struct fw_port_l1cfg { + __be32 rcap; + __be32 r; + } l1cfg; + struct fw_port_l2cfg { + __u8 ctlbf; + __u8 ovlan3_to_ivlan0; + __be16 ivlantype; + __be16 txipg_force_pinfo; + __be16 mtu; + __be16 ovlan0mask; + __be16 ovlan0type; + __be16 ovlan1mask; + __be16 ovlan1type; + __be16 ovlan2mask; + __be16 ovlan2type; + __be16 ovlan3mask; + __be16 ovlan3type; + } l2cfg; + struct fw_port_info { + __be32 lstatus_to_modtype; + __be16 pcap; + __be16 acap; + __be16 mtu; + __u8 cbllen; + __u8 auxlinfo; + __u8 dcbxdis_pkd; + __u8 r8_lo; + __be16 lpacap; + __be64 r9; + } info; + struct fw_port_diags { + __u8 diagop; + __u8 r[3]; + __be32 diagval; + } diags; + union fw_port_dcb { + struct fw_port_dcb_pgid { + __u8 type; + __u8 apply_pkd; + __u8 r10_lo[2]; + __be32 pgid; + __be64 r11; + } pgid; + struct fw_port_dcb_pgrate { + __u8 type; + __u8 apply_pkd; + __u8 r10_lo[5]; + __u8 num_tcs_supported; + __u8 pgrate[8]; + __u8 tsa[8]; + } pgrate; + struct fw_port_dcb_priorate { + __u8 type; + __u8 apply_pkd; + __u8 r10_lo[6]; + __u8 strict_priorate[8]; + } priorate; + struct fw_port_dcb_pfc { + __u8 type; + __u8 pfcen; + __u8 r10[5]; + __u8 max_pfc_tcs; + __be64 r11; + } pfc; + struct fw_port_app_priority { + __u8 type; + __u8 r10[2]; + __u8 idx; + __u8 user_prio_map; + __u8 sel_field; + __be16 protocolid; + __be64 r12; + } app_priority; + struct fw_port_dcb_control { + __u8 type; + __u8 all_syncd_pkd; + __be16 dcb_version_to_app_state; + __be32 r11; + __be64 r12; + } control; + } dcb; + struct fw_port_l1cfg32 { + __be32 rcap32; + __be32 r; + } l1cfg32; + struct fw_port_info32 { + __be32 lstatus32_to_cbllen32; + __be32 auxlinfo32_mtu32; + __be32 linkattr32; + __be32 pcaps32; + __be32 acaps32; + __be32 lpacaps32; + } info32; + } u; +}; + +#define FW_PORT_CMD_READ_S 22 +#define FW_PORT_CMD_READ_V(x) ((x) << FW_PORT_CMD_READ_S) +#define FW_PORT_CMD_READ_F FW_PORT_CMD_READ_V(1U) + +#define FW_PORT_CMD_PORTID_S 0 +#define FW_PORT_CMD_PORTID_M 0xf +#define FW_PORT_CMD_PORTID_V(x) ((x) << FW_PORT_CMD_PORTID_S) +#define FW_PORT_CMD_PORTID_G(x) \ + (((x) >> FW_PORT_CMD_PORTID_S) & FW_PORT_CMD_PORTID_M) + +#define FW_PORT_CMD_ACTION_S 16 +#define FW_PORT_CMD_ACTION_M 0xffff +#define FW_PORT_CMD_ACTION_V(x) ((x) << FW_PORT_CMD_ACTION_S) +#define FW_PORT_CMD_ACTION_G(x) \ + (((x) >> FW_PORT_CMD_ACTION_S) & FW_PORT_CMD_ACTION_M) + +#define FW_PORT_CMD_OVLAN3_S 7 +#define FW_PORT_CMD_OVLAN3_V(x) ((x) << FW_PORT_CMD_OVLAN3_S) + +#define FW_PORT_CMD_OVLAN2_S 6 +#define FW_PORT_CMD_OVLAN2_V(x) ((x) << FW_PORT_CMD_OVLAN2_S) + +#define FW_PORT_CMD_OVLAN1_S 5 +#define FW_PORT_CMD_OVLAN1_V(x) ((x) << FW_PORT_CMD_OVLAN1_S) + +#define FW_PORT_CMD_OVLAN0_S 4 +#define FW_PORT_CMD_OVLAN0_V(x) ((x) << FW_PORT_CMD_OVLAN0_S) + +#define FW_PORT_CMD_IVLAN0_S 3 +#define FW_PORT_CMD_IVLAN0_V(x) ((x) << FW_PORT_CMD_IVLAN0_S) + +#define FW_PORT_CMD_TXIPG_S 3 +#define FW_PORT_CMD_TXIPG_V(x) ((x) << FW_PORT_CMD_TXIPG_S) + +#define FW_PORT_CMD_LSTATUS_S 31 +#define FW_PORT_CMD_LSTATUS_M 0x1 +#define FW_PORT_CMD_LSTATUS_V(x) ((x) << FW_PORT_CMD_LSTATUS_S) +#define FW_PORT_CMD_LSTATUS_G(x) \ + (((x) >> FW_PORT_CMD_LSTATUS_S) & FW_PORT_CMD_LSTATUS_M) +#define FW_PORT_CMD_LSTATUS_F FW_PORT_CMD_LSTATUS_V(1U) + +#define FW_PORT_CMD_LSPEED_S 24 +#define FW_PORT_CMD_LSPEED_M 0x3f +#define FW_PORT_CMD_LSPEED_V(x) ((x) << FW_PORT_CMD_LSPEED_S) +#define FW_PORT_CMD_LSPEED_G(x) \ + (((x) >> FW_PORT_CMD_LSPEED_S) & FW_PORT_CMD_LSPEED_M) + +#define FW_PORT_CMD_TXPAUSE_S 23 +#define FW_PORT_CMD_TXPAUSE_V(x) ((x) << FW_PORT_CMD_TXPAUSE_S) +#define FW_PORT_CMD_TXPAUSE_F FW_PORT_CMD_TXPAUSE_V(1U) + +#define FW_PORT_CMD_RXPAUSE_S 22 +#define FW_PORT_CMD_RXPAUSE_V(x) ((x) << FW_PORT_CMD_RXPAUSE_S) +#define FW_PORT_CMD_RXPAUSE_F FW_PORT_CMD_RXPAUSE_V(1U) + +#define FW_PORT_CMD_MDIOCAP_S 21 +#define FW_PORT_CMD_MDIOCAP_V(x) ((x) << FW_PORT_CMD_MDIOCAP_S) +#define FW_PORT_CMD_MDIOCAP_F FW_PORT_CMD_MDIOCAP_V(1U) + +#define FW_PORT_CMD_MDIOADDR_S 16 +#define FW_PORT_CMD_MDIOADDR_M 0x1f +#define FW_PORT_CMD_MDIOADDR_G(x) \ + (((x) >> FW_PORT_CMD_MDIOADDR_S) & FW_PORT_CMD_MDIOADDR_M) + +#define FW_PORT_CMD_LPTXPAUSE_S 15 +#define FW_PORT_CMD_LPTXPAUSE_V(x) ((x) << FW_PORT_CMD_LPTXPAUSE_S) +#define FW_PORT_CMD_LPTXPAUSE_F FW_PORT_CMD_LPTXPAUSE_V(1U) + +#define FW_PORT_CMD_LPRXPAUSE_S 14 +#define FW_PORT_CMD_LPRXPAUSE_V(x) ((x) << FW_PORT_CMD_LPRXPAUSE_S) +#define FW_PORT_CMD_LPRXPAUSE_F FW_PORT_CMD_LPRXPAUSE_V(1U) + +#define FW_PORT_CMD_PTYPE_S 8 +#define FW_PORT_CMD_PTYPE_M 0x1f +#define FW_PORT_CMD_PTYPE_G(x) \ + (((x) >> FW_PORT_CMD_PTYPE_S) & FW_PORT_CMD_PTYPE_M) + +#define FW_PORT_CMD_LINKDNRC_S 5 +#define FW_PORT_CMD_LINKDNRC_M 0x7 +#define FW_PORT_CMD_LINKDNRC_G(x) \ + (((x) >> FW_PORT_CMD_LINKDNRC_S) & FW_PORT_CMD_LINKDNRC_M) + +#define FW_PORT_CMD_MODTYPE_S 0 +#define FW_PORT_CMD_MODTYPE_M 0x1f +#define FW_PORT_CMD_MODTYPE_V(x) ((x) << FW_PORT_CMD_MODTYPE_S) +#define FW_PORT_CMD_MODTYPE_G(x) \ + (((x) >> FW_PORT_CMD_MODTYPE_S) & FW_PORT_CMD_MODTYPE_M) + +#define FW_PORT_CMD_DCBXDIS_S 7 +#define FW_PORT_CMD_DCBXDIS_V(x) ((x) << FW_PORT_CMD_DCBXDIS_S) +#define FW_PORT_CMD_DCBXDIS_F FW_PORT_CMD_DCBXDIS_V(1U) + +#define FW_PORT_CMD_APPLY_S 7 +#define FW_PORT_CMD_APPLY_V(x) ((x) << FW_PORT_CMD_APPLY_S) +#define FW_PORT_CMD_APPLY_F FW_PORT_CMD_APPLY_V(1U) + +#define FW_PORT_CMD_ALL_SYNCD_S 7 +#define FW_PORT_CMD_ALL_SYNCD_V(x) ((x) << FW_PORT_CMD_ALL_SYNCD_S) +#define FW_PORT_CMD_ALL_SYNCD_F FW_PORT_CMD_ALL_SYNCD_V(1U) + +#define FW_PORT_CMD_DCB_VERSION_S 12 +#define FW_PORT_CMD_DCB_VERSION_M 0x7 +#define FW_PORT_CMD_DCB_VERSION_G(x) \ + (((x) >> FW_PORT_CMD_DCB_VERSION_S) & FW_PORT_CMD_DCB_VERSION_M) + +#define FW_PORT_CMD_LSTATUS32_S 31 +#define FW_PORT_CMD_LSTATUS32_M 0x1 +#define FW_PORT_CMD_LSTATUS32_V(x) ((x) << FW_PORT_CMD_LSTATUS32_S) +#define FW_PORT_CMD_LSTATUS32_G(x) \ + (((x) >> FW_PORT_CMD_LSTATUS32_S) & FW_PORT_CMD_LSTATUS32_M) +#define FW_PORT_CMD_LSTATUS32_F FW_PORT_CMD_LSTATUS32_V(1U) + +#define FW_PORT_CMD_LINKDNRC32_S 28 +#define FW_PORT_CMD_LINKDNRC32_M 0x7 +#define FW_PORT_CMD_LINKDNRC32_V(x) ((x) << FW_PORT_CMD_LINKDNRC32_S) +#define FW_PORT_CMD_LINKDNRC32_G(x) \ + (((x) >> FW_PORT_CMD_LINKDNRC32_S) & FW_PORT_CMD_LINKDNRC32_M) + +#define FW_PORT_CMD_DCBXDIS32_S 27 +#define FW_PORT_CMD_DCBXDIS32_M 0x1 +#define FW_PORT_CMD_DCBXDIS32_V(x) ((x) << FW_PORT_CMD_DCBXDIS32_S) +#define FW_PORT_CMD_DCBXDIS32_G(x) \ + (((x) >> FW_PORT_CMD_DCBXDIS32_S) & FW_PORT_CMD_DCBXDIS32_M) +#define FW_PORT_CMD_DCBXDIS32_F FW_PORT_CMD_DCBXDIS32_V(1U) + +#define FW_PORT_CMD_MDIOCAP32_S 26 +#define FW_PORT_CMD_MDIOCAP32_M 0x1 +#define FW_PORT_CMD_MDIOCAP32_V(x) ((x) << FW_PORT_CMD_MDIOCAP32_S) +#define FW_PORT_CMD_MDIOCAP32_G(x) \ + (((x) >> FW_PORT_CMD_MDIOCAP32_S) & FW_PORT_CMD_MDIOCAP32_M) +#define FW_PORT_CMD_MDIOCAP32_F FW_PORT_CMD_MDIOCAP32_V(1U) + +#define FW_PORT_CMD_MDIOADDR32_S 21 +#define FW_PORT_CMD_MDIOADDR32_M 0x1f +#define FW_PORT_CMD_MDIOADDR32_V(x) ((x) << FW_PORT_CMD_MDIOADDR32_S) +#define FW_PORT_CMD_MDIOADDR32_G(x) \ + (((x) >> FW_PORT_CMD_MDIOADDR32_S) & FW_PORT_CMD_MDIOADDR32_M) + +#define FW_PORT_CMD_PORTTYPE32_S 13 +#define FW_PORT_CMD_PORTTYPE32_M 0xff +#define FW_PORT_CMD_PORTTYPE32_V(x) ((x) << FW_PORT_CMD_PORTTYPE32_S) +#define FW_PORT_CMD_PORTTYPE32_G(x) \ + (((x) >> FW_PORT_CMD_PORTTYPE32_S) & FW_PORT_CMD_PORTTYPE32_M) + +#define FW_PORT_CMD_MODTYPE32_S 8 +#define FW_PORT_CMD_MODTYPE32_M 0x1f +#define FW_PORT_CMD_MODTYPE32_V(x) ((x) << FW_PORT_CMD_MODTYPE32_S) +#define FW_PORT_CMD_MODTYPE32_G(x) \ + (((x) >> FW_PORT_CMD_MODTYPE32_S) & FW_PORT_CMD_MODTYPE32_M) + +#define FW_PORT_CMD_CBLLEN32_S 0 +#define FW_PORT_CMD_CBLLEN32_M 0xff +#define FW_PORT_CMD_CBLLEN32_V(x) ((x) << FW_PORT_CMD_CBLLEN32_S) +#define FW_PORT_CMD_CBLLEN32_G(x) \ + (((x) >> FW_PORT_CMD_CBLLEN32_S) & FW_PORT_CMD_CBLLEN32_M) + +#define FW_PORT_CMD_AUXLINFO32_S 24 +#define FW_PORT_CMD_AUXLINFO32_M 0xff +#define FW_PORT_CMD_AUXLINFO32_V(x) ((x) << FW_PORT_CMD_AUXLINFO32_S) +#define FW_PORT_CMD_AUXLINFO32_G(x) \ + (((x) >> FW_PORT_CMD_AUXLINFO32_S) & FW_PORT_CMD_AUXLINFO32_M) + +#define FW_PORT_AUXLINFO32_KX4_S 2 +#define FW_PORT_AUXLINFO32_KX4_M 0x1 +#define FW_PORT_AUXLINFO32_KX4_V(x) \ + ((x) << FW_PORT_AUXLINFO32_KX4_S) +#define FW_PORT_AUXLINFO32_KX4_G(x) \ + (((x) >> FW_PORT_AUXLINFO32_KX4_S) & FW_PORT_AUXLINFO32_KX4_M) +#define FW_PORT_AUXLINFO32_KX4_F FW_PORT_AUXLINFO32_KX4_V(1U) + +#define FW_PORT_AUXLINFO32_KR_S 1 +#define FW_PORT_AUXLINFO32_KR_M 0x1 +#define FW_PORT_AUXLINFO32_KR_V(x) \ + ((x) << FW_PORT_AUXLINFO32_KR_S) +#define FW_PORT_AUXLINFO32_KR_G(x) \ + (((x) >> FW_PORT_AUXLINFO32_KR_S) & FW_PORT_AUXLINFO32_KR_M) +#define FW_PORT_AUXLINFO32_KR_F FW_PORT_AUXLINFO32_KR_V(1U) + +#define FW_PORT_CMD_MTU32_S 0 +#define FW_PORT_CMD_MTU32_M 0xffff +#define FW_PORT_CMD_MTU32_V(x) ((x) << FW_PORT_CMD_MTU32_S) +#define FW_PORT_CMD_MTU32_G(x) \ + (((x) >> FW_PORT_CMD_MTU32_S) & FW_PORT_CMD_MTU32_M) + +enum fw_port_type { + FW_PORT_TYPE_FIBER_XFI, + FW_PORT_TYPE_FIBER_XAUI, + FW_PORT_TYPE_BT_SGMII, + FW_PORT_TYPE_BT_XFI, + FW_PORT_TYPE_BT_XAUI, + FW_PORT_TYPE_KX4, + FW_PORT_TYPE_CX4, + FW_PORT_TYPE_KX, + FW_PORT_TYPE_KR, + FW_PORT_TYPE_SFP, + FW_PORT_TYPE_BP_AP, + FW_PORT_TYPE_BP4_AP, + FW_PORT_TYPE_QSFP_10G, + FW_PORT_TYPE_QSA, + FW_PORT_TYPE_QSFP, + FW_PORT_TYPE_BP40_BA, + FW_PORT_TYPE_KR4_100G, + FW_PORT_TYPE_CR4_QSFP, + FW_PORT_TYPE_CR_QSFP, + FW_PORT_TYPE_CR2_QSFP, + FW_PORT_TYPE_SFP28, + FW_PORT_TYPE_KR_SFP28, + FW_PORT_TYPE_KR_XLAUI, + + FW_PORT_TYPE_NONE = FW_PORT_CMD_PTYPE_M +}; + +enum fw_port_module_type { + FW_PORT_MOD_TYPE_NA, + FW_PORT_MOD_TYPE_LR, + FW_PORT_MOD_TYPE_SR, + FW_PORT_MOD_TYPE_ER, + FW_PORT_MOD_TYPE_TWINAX_PASSIVE, + FW_PORT_MOD_TYPE_TWINAX_ACTIVE, + FW_PORT_MOD_TYPE_LRM, + FW_PORT_MOD_TYPE_ERROR = FW_PORT_CMD_MODTYPE_M - 3, + FW_PORT_MOD_TYPE_UNKNOWN = FW_PORT_CMD_MODTYPE_M - 2, + FW_PORT_MOD_TYPE_NOTSUPPORTED = FW_PORT_CMD_MODTYPE_M - 1, + + FW_PORT_MOD_TYPE_NONE = FW_PORT_CMD_MODTYPE_M +}; + +enum fw_port_mod_sub_type { + FW_PORT_MOD_SUB_TYPE_NA, + FW_PORT_MOD_SUB_TYPE_MV88E114X = 0x1, + FW_PORT_MOD_SUB_TYPE_TN8022 = 0x2, + FW_PORT_MOD_SUB_TYPE_AQ1202 = 0x3, + FW_PORT_MOD_SUB_TYPE_88x3120 = 0x4, + FW_PORT_MOD_SUB_TYPE_BCM84834 = 0x5, + FW_PORT_MOD_SUB_TYPE_BT_VSC8634 = 0x8, + + /* The following will never been in the VPD. They are TWINAX cable + * lengths decoded from SFP+ module i2c PROMs. These should + * almost certainly go somewhere else ... + */ + FW_PORT_MOD_SUB_TYPE_TWINAX_1 = 0x9, + FW_PORT_MOD_SUB_TYPE_TWINAX_3 = 0xA, + FW_PORT_MOD_SUB_TYPE_TWINAX_5 = 0xB, + FW_PORT_MOD_SUB_TYPE_TWINAX_7 = 0xC, +}; + +enum fw_port_stats_tx_index { + FW_STAT_TX_PORT_BYTES_IX = 0, + FW_STAT_TX_PORT_FRAMES_IX, + FW_STAT_TX_PORT_BCAST_IX, + FW_STAT_TX_PORT_MCAST_IX, + FW_STAT_TX_PORT_UCAST_IX, + FW_STAT_TX_PORT_ERROR_IX, + FW_STAT_TX_PORT_64B_IX, + FW_STAT_TX_PORT_65B_127B_IX, + FW_STAT_TX_PORT_128B_255B_IX, + FW_STAT_TX_PORT_256B_511B_IX, + FW_STAT_TX_PORT_512B_1023B_IX, + FW_STAT_TX_PORT_1024B_1518B_IX, + FW_STAT_TX_PORT_1519B_MAX_IX, + FW_STAT_TX_PORT_DROP_IX, + FW_STAT_TX_PORT_PAUSE_IX, + FW_STAT_TX_PORT_PPP0_IX, + FW_STAT_TX_PORT_PPP1_IX, + FW_STAT_TX_PORT_PPP2_IX, + FW_STAT_TX_PORT_PPP3_IX, + FW_STAT_TX_PORT_PPP4_IX, + FW_STAT_TX_PORT_PPP5_IX, + FW_STAT_TX_PORT_PPP6_IX, + FW_STAT_TX_PORT_PPP7_IX, + FW_NUM_PORT_TX_STATS +}; + +enum fw_port_stat_rx_index { + FW_STAT_RX_PORT_BYTES_IX = 0, + FW_STAT_RX_PORT_FRAMES_IX, + FW_STAT_RX_PORT_BCAST_IX, + FW_STAT_RX_PORT_MCAST_IX, + FW_STAT_RX_PORT_UCAST_IX, + FW_STAT_RX_PORT_MTU_ERROR_IX, + FW_STAT_RX_PORT_MTU_CRC_ERROR_IX, + FW_STAT_RX_PORT_CRC_ERROR_IX, + FW_STAT_RX_PORT_LEN_ERROR_IX, + FW_STAT_RX_PORT_SYM_ERROR_IX, + FW_STAT_RX_PORT_64B_IX, + FW_STAT_RX_PORT_65B_127B_IX, + FW_STAT_RX_PORT_128B_255B_IX, + FW_STAT_RX_PORT_256B_511B_IX, + FW_STAT_RX_PORT_512B_1023B_IX, + FW_STAT_RX_PORT_1024B_1518B_IX, + FW_STAT_RX_PORT_1519B_MAX_IX, + FW_STAT_RX_PORT_PAUSE_IX, + FW_STAT_RX_PORT_PPP0_IX, + FW_STAT_RX_PORT_PPP1_IX, + FW_STAT_RX_PORT_PPP2_IX, + FW_STAT_RX_PORT_PPP3_IX, + FW_STAT_RX_PORT_PPP4_IX, + FW_STAT_RX_PORT_PPP5_IX, + FW_STAT_RX_PORT_PPP6_IX, + FW_STAT_RX_PORT_PPP7_IX, + FW_STAT_RX_PORT_LESS_64B_IX, + FW_STAT_RX_PORT_MAC_ERROR_IX, + FW_NUM_PORT_RX_STATS +}; + +/* port stats */ +#define FW_NUM_PORT_STATS (FW_NUM_PORT_TX_STATS + FW_NUM_PORT_RX_STATS) + +struct fw_port_stats_cmd { + __be32 op_to_portid; + __be32 retval_len16; + union fw_port_stats { + struct fw_port_stats_ctl { + u8 nstats_bg_bm; + u8 tx_ix; + __be16 r6; + __be32 r7; + __be64 stat0; + __be64 stat1; + __be64 stat2; + __be64 stat3; + __be64 stat4; + __be64 stat5; + } ctl; + struct fw_port_stats_all { + __be64 tx_bytes; + __be64 tx_frames; + __be64 tx_bcast; + __be64 tx_mcast; + __be64 tx_ucast; + __be64 tx_error; + __be64 tx_64b; + __be64 tx_65b_127b; + __be64 tx_128b_255b; + __be64 tx_256b_511b; + __be64 tx_512b_1023b; + __be64 tx_1024b_1518b; + __be64 tx_1519b_max; + __be64 tx_drop; + __be64 tx_pause; + __be64 tx_ppp0; + __be64 tx_ppp1; + __be64 tx_ppp2; + __be64 tx_ppp3; + __be64 tx_ppp4; + __be64 tx_ppp5; + __be64 tx_ppp6; + __be64 tx_ppp7; + __be64 rx_bytes; + __be64 rx_frames; + __be64 rx_bcast; + __be64 rx_mcast; + __be64 rx_ucast; + __be64 rx_mtu_error; + __be64 rx_mtu_crc_error; + __be64 rx_crc_error; + __be64 rx_len_error; + __be64 rx_sym_error; + __be64 rx_64b; + __be64 rx_65b_127b; + __be64 rx_128b_255b; + __be64 rx_256b_511b; + __be64 rx_512b_1023b; + __be64 rx_1024b_1518b; + __be64 rx_1519b_max; + __be64 rx_pause; + __be64 rx_ppp0; + __be64 rx_ppp1; + __be64 rx_ppp2; + __be64 rx_ppp3; + __be64 rx_ppp4; + __be64 rx_ppp5; + __be64 rx_ppp6; + __be64 rx_ppp7; + __be64 rx_less_64b; + __be64 rx_bg_drop; + __be64 rx_bg_trunc; + } all; + } u; +}; + +/* port loopback stats */ +#define FW_NUM_LB_STATS 16 +enum fw_port_lb_stats_index { + FW_STAT_LB_PORT_BYTES_IX, + FW_STAT_LB_PORT_FRAMES_IX, + FW_STAT_LB_PORT_BCAST_IX, + FW_STAT_LB_PORT_MCAST_IX, + FW_STAT_LB_PORT_UCAST_IX, + FW_STAT_LB_PORT_ERROR_IX, + FW_STAT_LB_PORT_64B_IX, + FW_STAT_LB_PORT_65B_127B_IX, + FW_STAT_LB_PORT_128B_255B_IX, + FW_STAT_LB_PORT_256B_511B_IX, + FW_STAT_LB_PORT_512B_1023B_IX, + FW_STAT_LB_PORT_1024B_1518B_IX, + FW_STAT_LB_PORT_1519B_MAX_IX, + FW_STAT_LB_PORT_DROP_FRAMES_IX +}; + +struct fw_port_lb_stats_cmd { + __be32 op_to_lbport; + __be32 retval_len16; + union fw_port_lb_stats { + struct fw_port_lb_stats_ctl { + u8 nstats_bg_bm; + u8 ix_pkd; + __be16 r6; + __be32 r7; + __be64 stat0; + __be64 stat1; + __be64 stat2; + __be64 stat3; + __be64 stat4; + __be64 stat5; + } ctl; + struct fw_port_lb_stats_all { + __be64 tx_bytes; + __be64 tx_frames; + __be64 tx_bcast; + __be64 tx_mcast; + __be64 tx_ucast; + __be64 tx_error; + __be64 tx_64b; + __be64 tx_65b_127b; + __be64 tx_128b_255b; + __be64 tx_256b_511b; + __be64 tx_512b_1023b; + __be64 tx_1024b_1518b; + __be64 tx_1519b_max; + __be64 rx_lb_drop; + __be64 rx_lb_trunc; + } all; + } u; +}; + +enum fw_ptp_subop { + /* none */ + FW_PTP_SC_INIT_TIMER = 0x00, + FW_PTP_SC_TX_TYPE = 0x01, + /* init */ + FW_PTP_SC_RXTIME_STAMP = 0x08, + FW_PTP_SC_RDRX_TYPE = 0x09, + /* ts */ + FW_PTP_SC_ADJ_FREQ = 0x10, + FW_PTP_SC_ADJ_TIME = 0x11, + FW_PTP_SC_ADJ_FTIME = 0x12, + FW_PTP_SC_WALL_CLOCK = 0x13, + FW_PTP_SC_GET_TIME = 0x14, + FW_PTP_SC_SET_TIME = 0x15, +}; + +struct fw_ptp_cmd { + __be32 op_to_portid; + __be32 retval_len16; + union fw_ptp { + struct fw_ptp_sc { + __u8 sc; + __u8 r3[7]; + } scmd; + struct fw_ptp_init { + __u8 sc; + __u8 txchan; + __be16 absid; + __be16 mode; + __be16 r3; + } init; + struct fw_ptp_ts { + __u8 sc; + __u8 sign; + __be16 r3; + __be32 ppb; + __be64 tm; + } ts; + } u; + __be64 r3; +}; + +#define FW_PTP_CMD_PORTID_S 0 +#define FW_PTP_CMD_PORTID_M 0xf +#define FW_PTP_CMD_PORTID_V(x) ((x) << FW_PTP_CMD_PORTID_S) +#define FW_PTP_CMD_PORTID_G(x) \ + (((x) >> FW_PTP_CMD_PORTID_S) & FW_PTP_CMD_PORTID_M) + +struct fw_rss_ind_tbl_cmd { + __be32 op_to_viid; + __be32 retval_len16; + __be16 niqid; + __be16 startidx; + __be32 r3; + __be32 iq0_to_iq2; + __be32 iq3_to_iq5; + __be32 iq6_to_iq8; + __be32 iq9_to_iq11; + __be32 iq12_to_iq14; + __be32 iq15_to_iq17; + __be32 iq18_to_iq20; + __be32 iq21_to_iq23; + __be32 iq24_to_iq26; + __be32 iq27_to_iq29; + __be32 iq30_iq31; + __be32 r15_lo; +}; + +#define FW_RSS_IND_TBL_CMD_VIID_S 0 +#define FW_RSS_IND_TBL_CMD_VIID_V(x) ((x) << FW_RSS_IND_TBL_CMD_VIID_S) + +#define FW_RSS_IND_TBL_CMD_IQ0_S 20 +#define FW_RSS_IND_TBL_CMD_IQ0_V(x) ((x) << FW_RSS_IND_TBL_CMD_IQ0_S) + +#define FW_RSS_IND_TBL_CMD_IQ1_S 10 +#define FW_RSS_IND_TBL_CMD_IQ1_V(x) ((x) << FW_RSS_IND_TBL_CMD_IQ1_S) + +#define FW_RSS_IND_TBL_CMD_IQ2_S 0 +#define FW_RSS_IND_TBL_CMD_IQ2_V(x) ((x) << FW_RSS_IND_TBL_CMD_IQ2_S) + +struct fw_rss_glb_config_cmd { + __be32 op_to_write; + __be32 retval_len16; + union fw_rss_glb_config { + struct fw_rss_glb_config_manual { + __be32 mode_pkd; + __be32 r3; + __be64 r4; + __be64 r5; + } manual; + struct fw_rss_glb_config_basicvirtual { + __be32 mode_pkd; + __be32 synmapen_to_hashtoeplitz; + __be64 r8; + __be64 r9; + } basicvirtual; + } u; +}; + +#define FW_RSS_GLB_CONFIG_CMD_MODE_S 28 +#define FW_RSS_GLB_CONFIG_CMD_MODE_M 0xf +#define FW_RSS_GLB_CONFIG_CMD_MODE_V(x) ((x) << FW_RSS_GLB_CONFIG_CMD_MODE_S) +#define FW_RSS_GLB_CONFIG_CMD_MODE_G(x) \ + (((x) >> FW_RSS_GLB_CONFIG_CMD_MODE_S) & FW_RSS_GLB_CONFIG_CMD_MODE_M) + +#define FW_RSS_GLB_CONFIG_CMD_MODE_MANUAL 0 +#define FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL 1 + +#define FW_RSS_GLB_CONFIG_CMD_SYNMAPEN_S 8 +#define FW_RSS_GLB_CONFIG_CMD_SYNMAPEN_V(x) \ + ((x) << FW_RSS_GLB_CONFIG_CMD_SYNMAPEN_S) +#define FW_RSS_GLB_CONFIG_CMD_SYNMAPEN_F \ + FW_RSS_GLB_CONFIG_CMD_SYNMAPEN_V(1U) + +#define FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV6_S 7 +#define FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV6_V(x) \ + ((x) << FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV6_S) +#define FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV6_F \ + FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV6_V(1U) + +#define FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV6_S 6 +#define FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV6_V(x) \ + ((x) << FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV6_S) +#define FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV6_F \ + FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV6_V(1U) + +#define FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV4_S 5 +#define FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV4_V(x) \ + ((x) << FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV4_S) +#define FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV4_F \ + FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV4_V(1U) + +#define FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV4_S 4 +#define FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV4_V(x) \ + ((x) << FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV4_S) +#define FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV4_F \ + FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV4_V(1U) + +#define FW_RSS_GLB_CONFIG_CMD_OFDMAPEN_S 3 +#define FW_RSS_GLB_CONFIG_CMD_OFDMAPEN_V(x) \ + ((x) << FW_RSS_GLB_CONFIG_CMD_OFDMAPEN_S) +#define FW_RSS_GLB_CONFIG_CMD_OFDMAPEN_F \ + FW_RSS_GLB_CONFIG_CMD_OFDMAPEN_V(1U) + +#define FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_S 2 +#define FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_V(x) \ + ((x) << FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_S) +#define FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F \ + FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_V(1U) + +#define FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_S 1 +#define FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_V(x) \ + ((x) << FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_S) +#define FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F \ + FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_V(1U) + +#define FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ_S 0 +#define FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ_V(x) \ + ((x) << FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ_S) +#define FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ_F \ + FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ_V(1U) + +struct fw_rss_vi_config_cmd { + __be32 op_to_viid; +#define FW_RSS_VI_CONFIG_CMD_VIID(x) ((x) << 0) + __be32 retval_len16; + union fw_rss_vi_config { + struct fw_rss_vi_config_manual { + __be64 r3; + __be64 r4; + __be64 r5; + } manual; + struct fw_rss_vi_config_basicvirtual { + __be32 r6; + __be32 defaultq_to_udpen; + __be64 r9; + __be64 r10; + } basicvirtual; + } u; +}; + +#define FW_RSS_VI_CONFIG_CMD_VIID_S 0 +#define FW_RSS_VI_CONFIG_CMD_VIID_V(x) ((x) << FW_RSS_VI_CONFIG_CMD_VIID_S) + +#define FW_RSS_VI_CONFIG_CMD_DEFAULTQ_S 16 +#define FW_RSS_VI_CONFIG_CMD_DEFAULTQ_M 0x3ff +#define FW_RSS_VI_CONFIG_CMD_DEFAULTQ_V(x) \ + ((x) << FW_RSS_VI_CONFIG_CMD_DEFAULTQ_S) +#define FW_RSS_VI_CONFIG_CMD_DEFAULTQ_G(x) \ + (((x) >> FW_RSS_VI_CONFIG_CMD_DEFAULTQ_S) & \ + FW_RSS_VI_CONFIG_CMD_DEFAULTQ_M) + +#define FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_S 4 +#define FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_V(x) \ + ((x) << FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_S) +#define FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F \ + FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_V(1U) + +#define FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_S 3 +#define FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_V(x) \ + ((x) << FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_S) +#define FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F \ + FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_V(1U) + +#define FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_S 2 +#define FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_V(x) \ + ((x) << FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_S) +#define FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F \ + FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_V(1U) + +#define FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_S 1 +#define FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_V(x) \ + ((x) << FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_S) +#define FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F \ + FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_V(1U) + +#define FW_RSS_VI_CONFIG_CMD_UDPEN_S 0 +#define FW_RSS_VI_CONFIG_CMD_UDPEN_V(x) ((x) << FW_RSS_VI_CONFIG_CMD_UDPEN_S) +#define FW_RSS_VI_CONFIG_CMD_UDPEN_F FW_RSS_VI_CONFIG_CMD_UDPEN_V(1U) + +enum fw_sched_sc { + FW_SCHED_SC_PARAMS = 1, +}; + +struct fw_sched_cmd { + __be32 op_to_write; + __be32 retval_len16; + union fw_sched { + struct fw_sched_config { + __u8 sc; + __u8 type; + __u8 minmaxen; + __u8 r3[5]; + __u8 nclasses[4]; + __be32 r4; + } config; + struct fw_sched_params { + __u8 sc; + __u8 type; + __u8 level; + __u8 mode; + __u8 unit; + __u8 rate; + __u8 ch; + __u8 cl; + __be32 min; + __be32 max; + __be16 weight; + __be16 pktsize; + __be16 burstsize; + __be16 r4; + } params; + } u; +}; + +struct fw_clip_cmd { + __be32 op_to_write; + __be32 alloc_to_len16; + __be64 ip_hi; + __be64 ip_lo; + __be32 r4[2]; +}; + +#define FW_CLIP_CMD_ALLOC_S 31 +#define FW_CLIP_CMD_ALLOC_V(x) ((x) << FW_CLIP_CMD_ALLOC_S) +#define FW_CLIP_CMD_ALLOC_F FW_CLIP_CMD_ALLOC_V(1U) + +#define FW_CLIP_CMD_FREE_S 30 +#define FW_CLIP_CMD_FREE_V(x) ((x) << FW_CLIP_CMD_FREE_S) +#define FW_CLIP_CMD_FREE_F FW_CLIP_CMD_FREE_V(1U) + +enum fw_error_type { + FW_ERROR_TYPE_EXCEPTION = 0x0, + FW_ERROR_TYPE_HWMODULE = 0x1, + FW_ERROR_TYPE_WR = 0x2, + FW_ERROR_TYPE_ACL = 0x3, +}; + +struct fw_error_cmd { + __be32 op_to_type; + __be32 len16_pkd; + union fw_error { + struct fw_error_exception { + __be32 info[6]; + } exception; + struct fw_error_hwmodule { + __be32 regaddr; + __be32 regval; + } hwmodule; + struct fw_error_wr { + __be16 cidx; + __be16 pfn_vfn; + __be32 eqid; + u8 wrhdr[16]; + } wr; + struct fw_error_acl { + __be16 cidx; + __be16 pfn_vfn; + __be32 eqid; + __be16 mv_pkd; + u8 val[6]; + __be64 r4; + } acl; + } u; +}; + +struct fw_debug_cmd { + __be32 op_type; + __be32 len16_pkd; + union fw_debug { + struct fw_debug_assert { + __be32 fcid; + __be32 line; + __be32 x; + __be32 y; + u8 filename_0_7[8]; + u8 filename_8_15[8]; + __be64 r3; + } assert; + struct fw_debug_prt { + __be16 dprtstridx; + __be16 r3[3]; + __be32 dprtstrparam0; + __be32 dprtstrparam1; + __be32 dprtstrparam2; + __be32 dprtstrparam3; + } prt; + } u; +}; + +#define FW_DEBUG_CMD_TYPE_S 0 +#define FW_DEBUG_CMD_TYPE_M 0xff +#define FW_DEBUG_CMD_TYPE_G(x) \ + (((x) >> FW_DEBUG_CMD_TYPE_S) & FW_DEBUG_CMD_TYPE_M) + +struct fw_hma_cmd { + __be32 op_pkd; + __be32 retval_len16; + __be32 mode_to_pcie_params; + __be32 naddr_size; + __be32 addr_size_pkd; + __be32 r6; + __be64 phy_address[5]; +}; + +#define FW_HMA_CMD_MODE_S 31 +#define FW_HMA_CMD_MODE_M 0x1 +#define FW_HMA_CMD_MODE_V(x) ((x) << FW_HMA_CMD_MODE_S) +#define FW_HMA_CMD_MODE_G(x) \ + (((x) >> FW_HMA_CMD_MODE_S) & FW_HMA_CMD_MODE_M) +#define FW_HMA_CMD_MODE_F FW_HMA_CMD_MODE_V(1U) + +#define FW_HMA_CMD_SOC_S 30 +#define FW_HMA_CMD_SOC_M 0x1 +#define FW_HMA_CMD_SOC_V(x) ((x) << FW_HMA_CMD_SOC_S) +#define FW_HMA_CMD_SOC_G(x) (((x) >> FW_HMA_CMD_SOC_S) & FW_HMA_CMD_SOC_M) +#define FW_HMA_CMD_SOC_F FW_HMA_CMD_SOC_V(1U) + +#define FW_HMA_CMD_EOC_S 29 +#define FW_HMA_CMD_EOC_M 0x1 +#define FW_HMA_CMD_EOC_V(x) ((x) << FW_HMA_CMD_EOC_S) +#define FW_HMA_CMD_EOC_G(x) (((x) >> FW_HMA_CMD_EOC_S) & FW_HMA_CMD_EOC_M) +#define FW_HMA_CMD_EOC_F FW_HMA_CMD_EOC_V(1U) + +#define FW_HMA_CMD_PCIE_PARAMS_S 0 +#define FW_HMA_CMD_PCIE_PARAMS_M 0x7ffffff +#define FW_HMA_CMD_PCIE_PARAMS_V(x) ((x) << FW_HMA_CMD_PCIE_PARAMS_S) +#define FW_HMA_CMD_PCIE_PARAMS_G(x) \ + (((x) >> FW_HMA_CMD_PCIE_PARAMS_S) & FW_HMA_CMD_PCIE_PARAMS_M) + +#define FW_HMA_CMD_NADDR_S 12 +#define FW_HMA_CMD_NADDR_M 0x3f +#define FW_HMA_CMD_NADDR_V(x) ((x) << FW_HMA_CMD_NADDR_S) +#define FW_HMA_CMD_NADDR_G(x) \ + (((x) >> FW_HMA_CMD_NADDR_S) & FW_HMA_CMD_NADDR_M) + +#define FW_HMA_CMD_SIZE_S 0 +#define FW_HMA_CMD_SIZE_M 0xfff +#define FW_HMA_CMD_SIZE_V(x) ((x) << FW_HMA_CMD_SIZE_S) +#define FW_HMA_CMD_SIZE_G(x) \ + (((x) >> FW_HMA_CMD_SIZE_S) & FW_HMA_CMD_SIZE_M) + +#define FW_HMA_CMD_ADDR_SIZE_S 11 +#define FW_HMA_CMD_ADDR_SIZE_M 0x1fffff +#define FW_HMA_CMD_ADDR_SIZE_V(x) ((x) << FW_HMA_CMD_ADDR_SIZE_S) +#define FW_HMA_CMD_ADDR_SIZE_G(x) \ + (((x) >> FW_HMA_CMD_ADDR_SIZE_S) & FW_HMA_CMD_ADDR_SIZE_M) + +enum pcie_fw_eval { + PCIE_FW_EVAL_CRASH = 0, +}; + +#define PCIE_FW_ERR_S 31 +#define PCIE_FW_ERR_V(x) ((x) << PCIE_FW_ERR_S) +#define PCIE_FW_ERR_F PCIE_FW_ERR_V(1U) + +#define PCIE_FW_INIT_S 30 +#define PCIE_FW_INIT_V(x) ((x) << PCIE_FW_INIT_S) +#define PCIE_FW_INIT_F PCIE_FW_INIT_V(1U) + +#define PCIE_FW_HALT_S 29 +#define PCIE_FW_HALT_V(x) ((x) << PCIE_FW_HALT_S) +#define PCIE_FW_HALT_F PCIE_FW_HALT_V(1U) + +#define PCIE_FW_EVAL_S 24 +#define PCIE_FW_EVAL_M 0x7 +#define PCIE_FW_EVAL_G(x) (((x) >> PCIE_FW_EVAL_S) & PCIE_FW_EVAL_M) + +#define PCIE_FW_MASTER_VLD_S 15 +#define PCIE_FW_MASTER_VLD_V(x) ((x) << PCIE_FW_MASTER_VLD_S) +#define PCIE_FW_MASTER_VLD_F PCIE_FW_MASTER_VLD_V(1U) + +#define PCIE_FW_MASTER_S 12 +#define PCIE_FW_MASTER_M 0x7 +#define PCIE_FW_MASTER_V(x) ((x) << PCIE_FW_MASTER_S) +#define PCIE_FW_MASTER_G(x) (((x) >> PCIE_FW_MASTER_S) & PCIE_FW_MASTER_M) + +struct fw_hdr { + u8 ver; + u8 chip; /* terminator chip type */ + __be16 len512; /* bin length in units of 512-bytes */ + __be32 fw_ver; /* firmware version */ + __be32 tp_microcode_ver; + u8 intfver_nic; + u8 intfver_vnic; + u8 intfver_ofld; + u8 intfver_ri; + u8 intfver_iscsipdu; + u8 intfver_iscsi; + u8 intfver_fcoepdu; + u8 intfver_fcoe; + __u32 reserved2; + __u32 reserved3; + __u32 reserved4; + __be32 flags; + __be32 reserved6[23]; +}; + +enum fw_hdr_chip { + FW_HDR_CHIP_T4, + FW_HDR_CHIP_T5, + FW_HDR_CHIP_T6 +}; + +#define FW_HDR_FW_VER_MAJOR_S 24 +#define FW_HDR_FW_VER_MAJOR_M 0xff +#define FW_HDR_FW_VER_MAJOR_V(x) \ + ((x) << FW_HDR_FW_VER_MAJOR_S) +#define FW_HDR_FW_VER_MAJOR_G(x) \ + (((x) >> FW_HDR_FW_VER_MAJOR_S) & FW_HDR_FW_VER_MAJOR_M) + +#define FW_HDR_FW_VER_MINOR_S 16 +#define FW_HDR_FW_VER_MINOR_M 0xff +#define FW_HDR_FW_VER_MINOR_V(x) \ + ((x) << FW_HDR_FW_VER_MINOR_S) +#define FW_HDR_FW_VER_MINOR_G(x) \ + (((x) >> FW_HDR_FW_VER_MINOR_S) & FW_HDR_FW_VER_MINOR_M) + +#define FW_HDR_FW_VER_MICRO_S 8 +#define FW_HDR_FW_VER_MICRO_M 0xff +#define FW_HDR_FW_VER_MICRO_V(x) \ + ((x) << FW_HDR_FW_VER_MICRO_S) +#define FW_HDR_FW_VER_MICRO_G(x) \ + (((x) >> FW_HDR_FW_VER_MICRO_S) & FW_HDR_FW_VER_MICRO_M) + +#define FW_HDR_FW_VER_BUILD_S 0 +#define FW_HDR_FW_VER_BUILD_M 0xff +#define FW_HDR_FW_VER_BUILD_V(x) \ + ((x) << FW_HDR_FW_VER_BUILD_S) +#define FW_HDR_FW_VER_BUILD_G(x) \ + (((x) >> FW_HDR_FW_VER_BUILD_S) & FW_HDR_FW_VER_BUILD_M) + +enum fw_hdr_intfver { + FW_HDR_INTFVER_NIC = 0x00, + FW_HDR_INTFVER_VNIC = 0x00, + FW_HDR_INTFVER_OFLD = 0x00, + FW_HDR_INTFVER_RI = 0x00, + FW_HDR_INTFVER_ISCSIPDU = 0x00, + FW_HDR_INTFVER_ISCSI = 0x00, + FW_HDR_INTFVER_FCOEPDU = 0x00, + FW_HDR_INTFVER_FCOE = 0x00, +}; + +enum fw_hdr_flags { + FW_HDR_FLAGS_RESET_HALT = 0x00000001, +}; + +/* length of the formatting string */ +#define FW_DEVLOG_FMT_LEN 192 + +/* maximum number of the formatting string parameters */ +#define FW_DEVLOG_FMT_PARAMS_NUM 8 + +/* priority levels */ +enum fw_devlog_level { + FW_DEVLOG_LEVEL_EMERG = 0x0, + FW_DEVLOG_LEVEL_CRIT = 0x1, + FW_DEVLOG_LEVEL_ERR = 0x2, + FW_DEVLOG_LEVEL_NOTICE = 0x3, + FW_DEVLOG_LEVEL_INFO = 0x4, + FW_DEVLOG_LEVEL_DEBUG = 0x5, + FW_DEVLOG_LEVEL_MAX = 0x5, +}; + +/* facilities that may send a log message */ +enum fw_devlog_facility { + FW_DEVLOG_FACILITY_CORE = 0x00, + FW_DEVLOG_FACILITY_CF = 0x01, + FW_DEVLOG_FACILITY_SCHED = 0x02, + FW_DEVLOG_FACILITY_TIMER = 0x04, + FW_DEVLOG_FACILITY_RES = 0x06, + FW_DEVLOG_FACILITY_HW = 0x08, + FW_DEVLOG_FACILITY_FLR = 0x10, + FW_DEVLOG_FACILITY_DMAQ = 0x12, + FW_DEVLOG_FACILITY_PHY = 0x14, + FW_DEVLOG_FACILITY_MAC = 0x16, + FW_DEVLOG_FACILITY_PORT = 0x18, + FW_DEVLOG_FACILITY_VI = 0x1A, + FW_DEVLOG_FACILITY_FILTER = 0x1C, + FW_DEVLOG_FACILITY_ACL = 0x1E, + FW_DEVLOG_FACILITY_TM = 0x20, + FW_DEVLOG_FACILITY_QFC = 0x22, + FW_DEVLOG_FACILITY_DCB = 0x24, + FW_DEVLOG_FACILITY_ETH = 0x26, + FW_DEVLOG_FACILITY_OFLD = 0x28, + FW_DEVLOG_FACILITY_RI = 0x2A, + FW_DEVLOG_FACILITY_ISCSI = 0x2C, + FW_DEVLOG_FACILITY_FCOE = 0x2E, + FW_DEVLOG_FACILITY_FOISCSI = 0x30, + FW_DEVLOG_FACILITY_FOFCOE = 0x32, + FW_DEVLOG_FACILITY_CHNET = 0x34, + FW_DEVLOG_FACILITY_MAX = 0x34, +}; + +/* log message format */ +struct fw_devlog_e { + __be64 timestamp; + __be32 seqno; + __be16 reserved1; + __u8 level; + __u8 facility; + __u8 fmt[FW_DEVLOG_FMT_LEN]; + __be32 params[FW_DEVLOG_FMT_PARAMS_NUM]; + __be32 reserved3[4]; +}; + +struct fw_devlog_cmd { + __be32 op_to_write; + __be32 retval_len16; + __u8 level; + __u8 r2[7]; + __be32 memtype_devlog_memaddr16_devlog; + __be32 memsize_devlog; + __be32 r3[2]; +}; + +#define FW_DEVLOG_CMD_MEMTYPE_DEVLOG_S 28 +#define FW_DEVLOG_CMD_MEMTYPE_DEVLOG_M 0xf +#define FW_DEVLOG_CMD_MEMTYPE_DEVLOG_G(x) \ + (((x) >> FW_DEVLOG_CMD_MEMTYPE_DEVLOG_S) & \ + FW_DEVLOG_CMD_MEMTYPE_DEVLOG_M) + +#define FW_DEVLOG_CMD_MEMADDR16_DEVLOG_S 0 +#define FW_DEVLOG_CMD_MEMADDR16_DEVLOG_M 0xfffffff +#define FW_DEVLOG_CMD_MEMADDR16_DEVLOG_G(x) \ + (((x) >> FW_DEVLOG_CMD_MEMADDR16_DEVLOG_S) & \ + FW_DEVLOG_CMD_MEMADDR16_DEVLOG_M) + +/* P C I E F W P F 7 R E G I S T E R */ + +/* PF7 stores the Firmware Device Log parameters which allows Host Drivers to + * access the "devlog" which needing to contact firmware. The encoding is + * mostly the same as that returned by the DEVLOG command except for the size + * which is encoded as the number of entries in multiples-1 of 128 here rather + * than the memory size as is done in the DEVLOG command. Thus, 0 means 128 + * and 15 means 2048. This of course in turn constrains the allowed values + * for the devlog size ... + */ +#define PCIE_FW_PF_DEVLOG 7 + +#define PCIE_FW_PF_DEVLOG_NENTRIES128_S 28 +#define PCIE_FW_PF_DEVLOG_NENTRIES128_M 0xf +#define PCIE_FW_PF_DEVLOG_NENTRIES128_V(x) \ + ((x) << PCIE_FW_PF_DEVLOG_NENTRIES128_S) +#define PCIE_FW_PF_DEVLOG_NENTRIES128_G(x) \ + (((x) >> PCIE_FW_PF_DEVLOG_NENTRIES128_S) & \ + PCIE_FW_PF_DEVLOG_NENTRIES128_M) + +#define PCIE_FW_PF_DEVLOG_ADDR16_S 4 +#define PCIE_FW_PF_DEVLOG_ADDR16_M 0xffffff +#define PCIE_FW_PF_DEVLOG_ADDR16_V(x) ((x) << PCIE_FW_PF_DEVLOG_ADDR16_S) +#define PCIE_FW_PF_DEVLOG_ADDR16_G(x) \ + (((x) >> PCIE_FW_PF_DEVLOG_ADDR16_S) & PCIE_FW_PF_DEVLOG_ADDR16_M) + +#define PCIE_FW_PF_DEVLOG_MEMTYPE_S 0 +#define PCIE_FW_PF_DEVLOG_MEMTYPE_M 0xf +#define PCIE_FW_PF_DEVLOG_MEMTYPE_V(x) ((x) << PCIE_FW_PF_DEVLOG_MEMTYPE_S) +#define PCIE_FW_PF_DEVLOG_MEMTYPE_G(x) \ + (((x) >> PCIE_FW_PF_DEVLOG_MEMTYPE_S) & PCIE_FW_PF_DEVLOG_MEMTYPE_M) + +#define MAX_IMM_OFLD_TX_DATA_WR_LEN (0xff + sizeof(struct fw_ofld_tx_data_wr)) + +struct fw_crypto_lookaside_wr { + __be32 op_to_cctx_size; + __be32 len16_pkd; + __be32 session_id; + __be32 rx_chid_to_rx_q_id; + __be32 key_addr; + __be32 pld_size_hash_size; + __be64 cookie; +}; + +#define FW_CRYPTO_LOOKASIDE_WR_OPCODE_S 24 +#define FW_CRYPTO_LOOKASIDE_WR_OPCODE_M 0xff +#define FW_CRYPTO_LOOKASIDE_WR_OPCODE_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_OPCODE_S) +#define FW_CRYPTO_LOOKASIDE_WR_OPCODE_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_OPCODE_S) & \ + FW_CRYPTO_LOOKASIDE_WR_OPCODE_M) + +#define FW_CRYPTO_LOOKASIDE_WR_COMPL_S 23 +#define FW_CRYPTO_LOOKASIDE_WR_COMPL_M 0x1 +#define FW_CRYPTO_LOOKASIDE_WR_COMPL_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_COMPL_S) +#define FW_CRYPTO_LOOKASIDE_WR_COMPL_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_COMPL_S) & \ + FW_CRYPTO_LOOKASIDE_WR_COMPL_M) +#define FW_CRYPTO_LOOKASIDE_WR_COMPL_F FW_CRYPTO_LOOKASIDE_WR_COMPL_V(1U) + +#define FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_S 15 +#define FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_M 0xff +#define FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_S) +#define FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_S) & \ + FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_M) + +#define FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_S 5 +#define FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_M 0x3 +#define FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_S) +#define FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_S) & \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_M) + +#define FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_S 0 +#define FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_M 0x1f +#define FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_S) +#define FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_S) & \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_M) + +#define FW_CRYPTO_LOOKASIDE_WR_LEN16_S 0 +#define FW_CRYPTO_LOOKASIDE_WR_LEN16_M 0xff +#define FW_CRYPTO_LOOKASIDE_WR_LEN16_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_LEN16_S) +#define FW_CRYPTO_LOOKASIDE_WR_LEN16_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_LEN16_S) & \ + FW_CRYPTO_LOOKASIDE_WR_LEN16_M) + +#define FW_CRYPTO_LOOKASIDE_WR_RX_CHID_S 29 +#define FW_CRYPTO_LOOKASIDE_WR_RX_CHID_M 0x3 +#define FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_RX_CHID_S) +#define FW_CRYPTO_LOOKASIDE_WR_RX_CHID_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_RX_CHID_S) & \ + FW_CRYPTO_LOOKASIDE_WR_RX_CHID_M) + +#define FW_CRYPTO_LOOKASIDE_WR_LCB_S 27 +#define FW_CRYPTO_LOOKASIDE_WR_LCB_M 0x3 +#define FW_CRYPTO_LOOKASIDE_WR_LCB_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_LCB_S) +#define FW_CRYPTO_LOOKASIDE_WR_LCB_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_LCB_S) & FW_CRYPTO_LOOKASIDE_WR_LCB_M) + +#define FW_CRYPTO_LOOKASIDE_WR_PHASH_S 25 +#define FW_CRYPTO_LOOKASIDE_WR_PHASH_M 0x3 +#define FW_CRYPTO_LOOKASIDE_WR_PHASH_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_PHASH_S) +#define FW_CRYPTO_LOOKASIDE_WR_PHASH_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_PHASH_S) & \ + FW_CRYPTO_LOOKASIDE_WR_PHASH_M) + +#define FW_CRYPTO_LOOKASIDE_WR_IV_S 23 +#define FW_CRYPTO_LOOKASIDE_WR_IV_M 0x3 +#define FW_CRYPTO_LOOKASIDE_WR_IV_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_IV_S) +#define FW_CRYPTO_LOOKASIDE_WR_IV_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_IV_S) & FW_CRYPTO_LOOKASIDE_WR_IV_M) + +#define FW_CRYPTO_LOOKASIDE_WR_FQIDX_S 15 +#define FW_CRYPTO_LOOKASIDE_WR_FQIDX_M 0xff +#define FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_FQIDX_S) +#define FW_CRYPTO_LOOKASIDE_WR_FQIDX_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_FQIDX_S) & \ + FW_CRYPTO_LOOKASIDE_WR_FQIDX_M) + +#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_S 10 +#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_M 0x3 +#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_TX_CH_S) +#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_TX_CH_S) & \ + FW_CRYPTO_LOOKASIDE_WR_TX_CH_M) + +#define FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_S 0 +#define FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_M 0x3ff +#define FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_S) +#define FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_S) & \ + FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_M) + +#define FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_S 24 +#define FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_M 0xff +#define FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_S) +#define FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_S) & \ + FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_M) + +#define FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_S 17 +#define FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_M 0x7f +#define FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(x) \ + ((x) << FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_S) +#define FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_G(x) \ + (((x) >> FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_S) & \ + FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_M) + +struct fw_tlstx_data_wr { + __be32 op_to_immdlen; + __be32 flowid_len16; + __be32 plen; + __be32 lsodisable_to_flags; + __be32 r5; + __be32 ctxloc_to_exp; + __be16 mfs; + __be16 adjustedplen_pkd; + __be16 expinplenmax_pkd; + u8 pdusinplenmax_pkd; + u8 r10; +}; + +#define FW_TLSTX_DATA_WR_OPCODE_S 24 +#define FW_TLSTX_DATA_WR_OPCODE_M 0xff +#define FW_TLSTX_DATA_WR_OPCODE_V(x) ((x) << FW_TLSTX_DATA_WR_OPCODE_S) +#define FW_TLSTX_DATA_WR_OPCODE_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_OPCODE_S) & FW_TLSTX_DATA_WR_OPCODE_M) + +#define FW_TLSTX_DATA_WR_COMPL_S 21 +#define FW_TLSTX_DATA_WR_COMPL_M 0x1 +#define FW_TLSTX_DATA_WR_COMPL_V(x) ((x) << FW_TLSTX_DATA_WR_COMPL_S) +#define FW_TLSTX_DATA_WR_COMPL_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_COMPL_S) & FW_TLSTX_DATA_WR_COMPL_M) +#define FW_TLSTX_DATA_WR_COMPL_F FW_TLSTX_DATA_WR_COMPL_V(1U) + +#define FW_TLSTX_DATA_WR_IMMDLEN_S 0 +#define FW_TLSTX_DATA_WR_IMMDLEN_M 0xff +#define FW_TLSTX_DATA_WR_IMMDLEN_V(x) ((x) << FW_TLSTX_DATA_WR_IMMDLEN_S) +#define FW_TLSTX_DATA_WR_IMMDLEN_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_IMMDLEN_S) & FW_TLSTX_DATA_WR_IMMDLEN_M) + +#define FW_TLSTX_DATA_WR_FLOWID_S 8 +#define FW_TLSTX_DATA_WR_FLOWID_M 0xfffff +#define FW_TLSTX_DATA_WR_FLOWID_V(x) ((x) << FW_TLSTX_DATA_WR_FLOWID_S) +#define FW_TLSTX_DATA_WR_FLOWID_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_FLOWID_S) & FW_TLSTX_DATA_WR_FLOWID_M) + +#define FW_TLSTX_DATA_WR_LEN16_S 0 +#define FW_TLSTX_DATA_WR_LEN16_M 0xff +#define FW_TLSTX_DATA_WR_LEN16_V(x) ((x) << FW_TLSTX_DATA_WR_LEN16_S) +#define FW_TLSTX_DATA_WR_LEN16_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_LEN16_S) & FW_TLSTX_DATA_WR_LEN16_M) + +#define FW_TLSTX_DATA_WR_LSODISABLE_S 31 +#define FW_TLSTX_DATA_WR_LSODISABLE_M 0x1 +#define FW_TLSTX_DATA_WR_LSODISABLE_V(x) \ + ((x) << FW_TLSTX_DATA_WR_LSODISABLE_S) +#define FW_TLSTX_DATA_WR_LSODISABLE_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_LSODISABLE_S) & FW_TLSTX_DATA_WR_LSODISABLE_M) +#define FW_TLSTX_DATA_WR_LSODISABLE_F FW_TLSTX_DATA_WR_LSODISABLE_V(1U) + +#define FW_TLSTX_DATA_WR_ALIGNPLD_S 30 +#define FW_TLSTX_DATA_WR_ALIGNPLD_M 0x1 +#define FW_TLSTX_DATA_WR_ALIGNPLD_V(x) ((x) << FW_TLSTX_DATA_WR_ALIGNPLD_S) +#define FW_TLSTX_DATA_WR_ALIGNPLD_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_ALIGNPLD_S) & FW_TLSTX_DATA_WR_ALIGNPLD_M) +#define FW_TLSTX_DATA_WR_ALIGNPLD_F FW_TLSTX_DATA_WR_ALIGNPLD_V(1U) + +#define FW_TLSTX_DATA_WR_ALIGNPLDSHOVE_S 29 +#define FW_TLSTX_DATA_WR_ALIGNPLDSHOVE_M 0x1 +#define FW_TLSTX_DATA_WR_ALIGNPLDSHOVE_V(x) \ + ((x) << FW_TLSTX_DATA_WR_ALIGNPLDSHOVE_S) +#define FW_TLSTX_DATA_WR_ALIGNPLDSHOVE_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_ALIGNPLDSHOVE_S) & \ + FW_TLSTX_DATA_WR_ALIGNPLDSHOVE_M) +#define FW_TLSTX_DATA_WR_ALIGNPLDSHOVE_F FW_TLSTX_DATA_WR_ALIGNPLDSHOVE_V(1U) + +#define FW_TLSTX_DATA_WR_FLAGS_S 0 +#define FW_TLSTX_DATA_WR_FLAGS_M 0xfffffff +#define FW_TLSTX_DATA_WR_FLAGS_V(x) ((x) << FW_TLSTX_DATA_WR_FLAGS_S) +#define FW_TLSTX_DATA_WR_FLAGS_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_FLAGS_S) & FW_TLSTX_DATA_WR_FLAGS_M) + +#define FW_TLSTX_DATA_WR_CTXLOC_S 30 +#define FW_TLSTX_DATA_WR_CTXLOC_M 0x3 +#define FW_TLSTX_DATA_WR_CTXLOC_V(x) ((x) << FW_TLSTX_DATA_WR_CTXLOC_S) +#define FW_TLSTX_DATA_WR_CTXLOC_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_CTXLOC_S) & FW_TLSTX_DATA_WR_CTXLOC_M) + +#define FW_TLSTX_DATA_WR_IVDSGL_S 29 +#define FW_TLSTX_DATA_WR_IVDSGL_M 0x1 +#define FW_TLSTX_DATA_WR_IVDSGL_V(x) ((x) << FW_TLSTX_DATA_WR_IVDSGL_S) +#define FW_TLSTX_DATA_WR_IVDSGL_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_IVDSGL_S) & FW_TLSTX_DATA_WR_IVDSGL_M) +#define FW_TLSTX_DATA_WR_IVDSGL_F FW_TLSTX_DATA_WR_IVDSGL_V(1U) + +#define FW_TLSTX_DATA_WR_KEYSIZE_S 24 +#define FW_TLSTX_DATA_WR_KEYSIZE_M 0x1f +#define FW_TLSTX_DATA_WR_KEYSIZE_V(x) ((x) << FW_TLSTX_DATA_WR_KEYSIZE_S) +#define FW_TLSTX_DATA_WR_KEYSIZE_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_KEYSIZE_S) & FW_TLSTX_DATA_WR_KEYSIZE_M) + +#define FW_TLSTX_DATA_WR_NUMIVS_S 14 +#define FW_TLSTX_DATA_WR_NUMIVS_M 0xff +#define FW_TLSTX_DATA_WR_NUMIVS_V(x) ((x) << FW_TLSTX_DATA_WR_NUMIVS_S) +#define FW_TLSTX_DATA_WR_NUMIVS_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_NUMIVS_S) & FW_TLSTX_DATA_WR_NUMIVS_M) + +#define FW_TLSTX_DATA_WR_EXP_S 0 +#define FW_TLSTX_DATA_WR_EXP_M 0x3fff +#define FW_TLSTX_DATA_WR_EXP_V(x) ((x) << FW_TLSTX_DATA_WR_EXP_S) +#define FW_TLSTX_DATA_WR_EXP_G(x) \ + (((x) >> FW_TLSTX_DATA_WR_EXP_S) & FW_TLSTX_DATA_WR_EXP_M) + +#define FW_TLSTX_DATA_WR_ADJUSTEDPLEN_S 1 +#define FW_TLSTX_DATA_WR_ADJUSTEDPLEN_V(x) \ + ((x) << FW_TLSTX_DATA_WR_ADJUSTEDPLEN_S) + +#define FW_TLSTX_DATA_WR_EXPINPLENMAX_S 4 +#define FW_TLSTX_DATA_WR_EXPINPLENMAX_V(x) \ + ((x) << FW_TLSTX_DATA_WR_EXPINPLENMAX_S) + +#define FW_TLSTX_DATA_WR_PDUSINPLENMAX_S 2 +#define FW_TLSTX_DATA_WR_PDUSINPLENMAX_V(x) \ + ((x) << FW_TLSTX_DATA_WR_PDUSINPLENMAX_S) + +#endif /* _T4FW_INTERFACE_H_ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_version.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_version.h new file mode 100644 index 000000000..a02b1dff4 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_version.h @@ -0,0 +1,64 @@ +/* + * This file is part of the Chelsio T4 Ethernet driver for Linux. + * + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __T4FW_VERSION_H__ +#define __T4FW_VERSION_H__ + +#define T4FW_VERSION_MAJOR 0x01 +#define T4FW_VERSION_MINOR 0x17 +#define T4FW_VERSION_MICRO 0x03 +#define T4FW_VERSION_BUILD 0x00 + +#define T4FW_MIN_VERSION_MAJOR 0x01 +#define T4FW_MIN_VERSION_MINOR 0x04 +#define T4FW_MIN_VERSION_MICRO 0x00 + +#define T5FW_VERSION_MAJOR 0x01 +#define T5FW_VERSION_MINOR 0x17 +#define T5FW_VERSION_MICRO 0x03 +#define T5FW_VERSION_BUILD 0x00 + +#define T5FW_MIN_VERSION_MAJOR 0x00 +#define T5FW_MIN_VERSION_MINOR 0x00 +#define T5FW_MIN_VERSION_MICRO 0x00 + +#define T6FW_VERSION_MAJOR 0x01 +#define T6FW_VERSION_MINOR 0x17 +#define T6FW_VERSION_MICRO 0x03 +#define T6FW_VERSION_BUILD 0x00 + +#define T6FW_MIN_VERSION_MAJOR 0x00 +#define T6FW_MIN_VERSION_MINOR 0x00 +#define T6FW_MIN_VERSION_MICRO 0x00 +#endif diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/Makefile b/drivers/net/ethernet/chelsio/cxgb4vf/Makefile new file mode 100644 index 000000000..f527ab13a --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4vf/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Chelsio T4 SR-IOV Virtual Function Driver +# + +obj-$(CONFIG_CHELSIO_T4VF) += cxgb4vf.o + +cxgb4vf-objs := cxgb4vf_main.o t4vf_hw.o sge.o diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h b/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h new file mode 100644 index 000000000..03cb1410d --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h @@ -0,0 +1,577 @@ +/* + * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet + * driver for Linux. + * + * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* + * This file should not be included directly. Include t4vf_common.h instead. + */ + +#ifndef __CXGB4VF_ADAPTER_H__ +#define __CXGB4VF_ADAPTER_H__ + +#include <linux/etherdevice.h> +#include <linux/interrupt.h> +#include <linux/pci.h> +#include <linux/spinlock.h> +#include <linux/skbuff.h> +#include <linux/if_ether.h> +#include <linux/netdevice.h> + +#include "../cxgb4/t4_hw.h" + +/* + * Constants of the implementation. + */ +enum { + MAX_NPORTS = 1, /* max # of "ports" */ + MAX_PORT_QSETS = 8, /* max # of Queue Sets / "port" */ + MAX_ETH_QSETS = MAX_NPORTS*MAX_PORT_QSETS, + + /* + * MSI-X interrupt index usage. + */ + MSIX_FW = 0, /* MSI-X index for firmware Q */ + MSIX_IQFLINT = 1, /* MSI-X index base for Ingress Qs */ + MSIX_EXTRAS = 1, + MSIX_ENTRIES = MAX_ETH_QSETS + MSIX_EXTRAS, + + /* + * The maximum number of Ingress and Egress Queues is determined by + * the maximum number of "Queue Sets" which we support plus any + * ancillary queues. Each "Queue Set" requires one Ingress Queue + * for RX Packet Ingress Event notifications and two Egress Queues for + * a Free List and an Ethernet TX list. + */ + INGQ_EXTRAS = 2, /* firmware event queue and */ + /* forwarded interrupts */ + MAX_INGQ = MAX_ETH_QSETS+INGQ_EXTRAS, + MAX_EGRQ = MAX_ETH_QSETS*2, +}; + +/* + * Forward structure definition references. + */ +struct adapter; +struct sge_eth_rxq; +struct sge_rspq; + +/* + * Per-"port" information. This is really per-Virtual Interface information + * but the use of the "port" nomanclature makes it easier to go back and forth + * between the PF and VF drivers ... + */ +struct port_info { + struct adapter *adapter; /* our adapter */ + u32 vlan_id; /* vlan id for VST */ + u16 viid; /* virtual interface ID */ + int xact_addr_filt; /* index of our MAC address filter */ + u16 rss_size; /* size of VI's RSS table slice */ + u8 pidx; /* index into adapter port[] */ + s8 mdio_addr; + u8 port_type; /* firmware port type */ + u8 mod_type; /* firmware module type */ + u8 port_id; /* physical port ID */ + u8 nqsets; /* # of "Queue Sets" */ + u8 first_qset; /* index of first "Queue Set" */ + struct link_config link_cfg; /* physical port configuration */ +}; + +/* + * Scatter Gather Engine resources for the "adapter". Our ingress and egress + * queues are organized into "Queue Sets" with one ingress and one egress + * queue per Queue Set. These Queue Sets are aportionable between the "ports" + * (Virtual Interfaces). One extra ingress queue is used to receive + * asynchronous messages from the firmware. Note that the "Queue IDs" that we + * use here are really "Relative Queue IDs" which are returned as part of the + * firmware command to allocate queues. These queue IDs are relative to the + * absolute Queue ID base of the section of the Queue ID space allocated to + * the PF/VF. + */ + +/* + * SGE free-list queue state. + */ +struct rx_sw_desc; +struct sge_fl { + unsigned int avail; /* # of available RX buffers */ + unsigned int pend_cred; /* new buffers since last FL DB ring */ + unsigned int cidx; /* consumer index */ + unsigned int pidx; /* producer index */ + unsigned long alloc_failed; /* # of buffer allocation failures */ + unsigned long large_alloc_failed; + unsigned long starving; /* # of times FL was found starving */ + + /* + * Write-once/infrequently fields. + * ------------------------------- + */ + + unsigned int cntxt_id; /* SGE relative QID for the free list */ + unsigned int abs_id; /* SGE absolute QID for the free list */ + unsigned int size; /* capacity of free list */ + struct rx_sw_desc *sdesc; /* address of SW RX descriptor ring */ + __be64 *desc; /* address of HW RX descriptor ring */ + dma_addr_t addr; /* PCI bus address of hardware ring */ + void __iomem *bar2_addr; /* address of BAR2 Queue registers */ + unsigned int bar2_qid; /* Queue ID for BAR2 Queue registers */ +}; + +/* + * An ingress packet gather list. + */ +struct pkt_gl { + struct page_frag frags[MAX_SKB_FRAGS]; + void *va; /* virtual address of first byte */ + unsigned int nfrags; /* # of fragments */ + unsigned int tot_len; /* total length of fragments */ +}; + +typedef int (*rspq_handler_t)(struct sge_rspq *, const __be64 *, + const struct pkt_gl *); + +/* + * State for an SGE Response Queue. + */ +struct sge_rspq { + struct napi_struct napi; /* NAPI scheduling control */ + const __be64 *cur_desc; /* current descriptor in queue */ + unsigned int cidx; /* consumer index */ + u8 gen; /* current generation bit */ + u8 next_intr_params; /* holdoff params for next interrupt */ + int offset; /* offset into current FL buffer */ + + unsigned int unhandled_irqs; /* bogus interrupts */ + + /* + * Write-once/infrequently fields. + * ------------------------------- + */ + + u8 intr_params; /* interrupt holdoff parameters */ + u8 pktcnt_idx; /* interrupt packet threshold */ + u8 idx; /* queue index within its group */ + u16 cntxt_id; /* SGE rel QID for the response Q */ + u16 abs_id; /* SGE abs QID for the response Q */ + __be64 *desc; /* address of hardware response ring */ + dma_addr_t phys_addr; /* PCI bus address of ring */ + void __iomem *bar2_addr; /* address of BAR2 Queue registers */ + unsigned int bar2_qid; /* Queue ID for BAR2 Queue registers */ + unsigned int iqe_len; /* entry size */ + unsigned int size; /* capcity of response Q */ + struct adapter *adapter; /* our adapter */ + struct net_device *netdev; /* associated net device */ + rspq_handler_t handler; /* the handler for this response Q */ +}; + +/* + * Ethernet queue statistics + */ +struct sge_eth_stats { + unsigned long pkts; /* # of ethernet packets */ + unsigned long lro_pkts; /* # of LRO super packets */ + unsigned long lro_merged; /* # of wire packets merged by LRO */ + unsigned long rx_cso; /* # of Rx checksum offloads */ + unsigned long vlan_ex; /* # of Rx VLAN extractions */ + unsigned long rx_drops; /* # of packets dropped due to no mem */ +}; + +/* + * State for an Ethernet Receive Queue. + */ +struct sge_eth_rxq { + struct sge_rspq rspq; /* Response Queue */ + struct sge_fl fl; /* Free List */ + struct sge_eth_stats stats; /* receive statistics */ +}; + +/* + * SGE Transmit Queue state. This contains all of the resources associated + * with the hardware status of a TX Queue which is a circular ring of hardware + * TX Descriptors. For convenience, it also contains a pointer to a parallel + * "Software Descriptor" array but we don't know anything about it here other + * than its type name. + */ +struct tx_desc { + /* + * Egress Queues are measured in units of SGE_EQ_IDXSIZE by the + * hardware: Sizes, Producer and Consumer indices, etc. + */ + __be64 flit[SGE_EQ_IDXSIZE/sizeof(__be64)]; +}; +struct tx_sw_desc; +struct sge_txq { + unsigned int in_use; /* # of in-use TX descriptors */ + unsigned int size; /* # of descriptors */ + unsigned int cidx; /* SW consumer index */ + unsigned int pidx; /* producer index */ + unsigned long stops; /* # of times queue has been stopped */ + unsigned long restarts; /* # of queue restarts */ + + /* + * Write-once/infrequently fields. + * ------------------------------- + */ + + unsigned int cntxt_id; /* SGE relative QID for the TX Q */ + unsigned int abs_id; /* SGE absolute QID for the TX Q */ + struct tx_desc *desc; /* address of HW TX descriptor ring */ + struct tx_sw_desc *sdesc; /* address of SW TX descriptor ring */ + struct sge_qstat *stat; /* queue status entry */ + dma_addr_t phys_addr; /* PCI bus address of hardware ring */ + void __iomem *bar2_addr; /* address of BAR2 Queue registers */ + unsigned int bar2_qid; /* Queue ID for BAR2 Queue registers */ +}; + +/* + * State for an Ethernet Transmit Queue. + */ +struct sge_eth_txq { + struct sge_txq q; /* SGE TX Queue */ + struct netdev_queue *txq; /* associated netdev TX queue */ + unsigned long tso; /* # of TSO requests */ + unsigned long tx_cso; /* # of TX checksum offloads */ + unsigned long vlan_ins; /* # of TX VLAN insertions */ + unsigned long mapping_err; /* # of I/O MMU packet mapping errors */ +}; + +/* + * The complete set of Scatter/Gather Engine resources. + */ +struct sge { + /* + * Our "Queue Sets" ... + */ + struct sge_eth_txq ethtxq[MAX_ETH_QSETS]; + struct sge_eth_rxq ethrxq[MAX_ETH_QSETS]; + + /* + * Extra ingress queues for asynchronous firmware events and + * forwarded interrupts (when in MSI mode). + */ + struct sge_rspq fw_evtq ____cacheline_aligned_in_smp; + + struct sge_rspq intrq ____cacheline_aligned_in_smp; + spinlock_t intrq_lock; + + /* + * State for managing "starving Free Lists" -- Free Lists which have + * fallen below a certain threshold of buffers available to the + * hardware and attempts to refill them up to that threshold have + * failed. We have a regular "slow tick" timer process which will + * make periodic attempts to refill these starving Free Lists ... + */ + DECLARE_BITMAP(starving_fl, MAX_EGRQ); + struct timer_list rx_timer; + + /* + * State for cleaning up completed TX descriptors. + */ + struct timer_list tx_timer; + + /* + * Write-once/infrequently fields. + * ------------------------------- + */ + + u16 max_ethqsets; /* # of available Ethernet queue sets */ + u16 ethqsets; /* # of active Ethernet queue sets */ + u16 ethtxq_rover; /* Tx queue to clean up next */ + u16 timer_val[SGE_NTIMERS]; /* interrupt holdoff timer array */ + u8 counter_val[SGE_NCOUNTERS]; /* interrupt RX threshold array */ + + /* Decoded Adapter Parameters. + */ + u32 fl_pg_order; /* large page allocation size */ + u32 stat_len; /* length of status page at ring end */ + u32 pktshift; /* padding between CPL & packet data */ + u32 fl_align; /* response queue message alignment */ + u32 fl_starve_thres; /* Free List starvation threshold */ + + /* + * Reverse maps from Absolute Queue IDs to associated queue pointers. + * The absolute Queue IDs are in a compact range which start at a + * [potentially large] Base Queue ID. We perform the reverse map by + * first converting the Absolute Queue ID into a Relative Queue ID by + * subtracting off the Base Queue ID and then use a Relative Queue ID + * indexed table to get the pointer to the corresponding software + * queue structure. + */ + unsigned int egr_base; + unsigned int ingr_base; + void *egr_map[MAX_EGRQ]; + struct sge_rspq *ingr_map[MAX_INGQ]; +}; + +/* + * Utility macros to convert Absolute- to Relative-Queue indices and Egress- + * and Ingress-Queues. The EQ_MAP() and IQ_MAP() macros which provide + * pointers to Ingress- and Egress-Queues can be used as both L- and R-values + */ +#define EQ_IDX(s, abs_id) ((unsigned int)((abs_id) - (s)->egr_base)) +#define IQ_IDX(s, abs_id) ((unsigned int)((abs_id) - (s)->ingr_base)) + +#define EQ_MAP(s, abs_id) ((s)->egr_map[EQ_IDX(s, abs_id)]) +#define IQ_MAP(s, abs_id) ((s)->ingr_map[IQ_IDX(s, abs_id)]) + +/* + * Macro to iterate across Queue Sets ("rxq" is a historic misnomer). + */ +#define for_each_ethrxq(sge, iter) \ + for (iter = 0; iter < (sge)->ethqsets; iter++) + +struct hash_mac_addr { + struct list_head list; + u8 addr[ETH_ALEN]; + unsigned int iface_mac; +}; + +struct mbox_list { + struct list_head list; +}; + +/* + * Per-"adapter" (Virtual Function) information. + */ +struct adapter { + /* PCI resources */ + void __iomem *regs; + void __iomem *bar2; + struct pci_dev *pdev; + struct device *pdev_dev; + + /* "adapter" resources */ + unsigned long registered_device_map; + unsigned long open_device_map; + unsigned long flags; + struct adapter_params params; + + /* queue and interrupt resources */ + struct { + unsigned short vec; + char desc[22]; + } msix_info[MSIX_ENTRIES]; + struct sge sge; + + /* Linux network device resources */ + struct net_device *port[MAX_NPORTS]; + const char *name; + unsigned int msg_enable; + + /* debugfs resources */ + struct dentry *debugfs_root; + + /* various locks */ + spinlock_t stats_lock; + + /* lock for mailbox cmd list */ + spinlock_t mbox_lock; + struct mbox_list mlist; + + /* support for mailbox command/reply logging */ +#define T4VF_OS_LOG_MBOX_CMDS 256 + struct mbox_cmd_log *mbox_log; + + /* list of MAC addresses in MPS Hash */ + struct list_head mac_hlist; +}; + +enum { /* adapter flags */ + CXGB4VF_FULL_INIT_DONE = (1UL << 0), + CXGB4VF_USING_MSI = (1UL << 1), + CXGB4VF_USING_MSIX = (1UL << 2), + CXGB4VF_QUEUES_BOUND = (1UL << 3), + CXGB4VF_ROOT_NO_RELAXED_ORDERING = (1UL << 4), + CXGB4VF_FW_OK = (1UL << 5), +}; + +/* + * The following register read/write routine definitions are required by + * the common code. + */ + +/** + * t4_read_reg - read a HW register + * @adapter: the adapter + * @reg_addr: the register address + * + * Returns the 32-bit value of the given HW register. + */ +static inline u32 t4_read_reg(struct adapter *adapter, u32 reg_addr) +{ + return readl(adapter->regs + reg_addr); +} + +/** + * t4_write_reg - write a HW register + * @adapter: the adapter + * @reg_addr: the register address + * @val: the value to write + * + * Write a 32-bit value into the given HW register. + */ +static inline void t4_write_reg(struct adapter *adapter, u32 reg_addr, u32 val) +{ + writel(val, adapter->regs + reg_addr); +} + +#ifndef readq +static inline u64 readq(const volatile void __iomem *addr) +{ + return readl(addr) + ((u64)readl(addr + 4) << 32); +} + +static inline void writeq(u64 val, volatile void __iomem *addr) +{ + writel(val, addr); + writel(val >> 32, addr + 4); +} +#endif + +/** + * t4_read_reg64 - read a 64-bit HW register + * @adapter: the adapter + * @reg_addr: the register address + * + * Returns the 64-bit value of the given HW register. + */ +static inline u64 t4_read_reg64(struct adapter *adapter, u32 reg_addr) +{ + return readq(adapter->regs + reg_addr); +} + +/** + * t4_write_reg64 - write a 64-bit HW register + * @adapter: the adapter + * @reg_addr: the register address + * @val: the value to write + * + * Write a 64-bit value into the given HW register. + */ +static inline void t4_write_reg64(struct adapter *adapter, u32 reg_addr, + u64 val) +{ + writeq(val, adapter->regs + reg_addr); +} + +/** + * port_name - return the string name of a port + * @adapter: the adapter + * @pidx: the port index + * + * Return the string name of the selected port. + */ +static inline const char *port_name(struct adapter *adapter, int pidx) +{ + return adapter->port[pidx]->name; +} + +/** + * t4_os_set_hw_addr - store a port's MAC address in SW + * @adapter: the adapter + * @pidx: the port index + * @hw_addr: the Ethernet address + * + * Store the Ethernet address of the given port in SW. Called by the common + * code when it retrieves a port's Ethernet address from EEPROM. + */ +static inline void t4_os_set_hw_addr(struct adapter *adapter, int pidx, + u8 hw_addr[]) +{ + eth_hw_addr_set(adapter->port[pidx], hw_addr); +} + +/** + * netdev2pinfo - return the port_info structure associated with a net_device + * @dev: the netdev + * + * Return the struct port_info associated with a net_device + */ +static inline struct port_info *netdev2pinfo(const struct net_device *dev) +{ + return netdev_priv(dev); +} + +/** + * adap2pinfo - return the port_info of a port + * @adap: the adapter + * @pidx: the port index + * + * Return the port_info structure for the adapter. + */ +static inline struct port_info *adap2pinfo(struct adapter *adapter, int pidx) +{ + return netdev_priv(adapter->port[pidx]); +} + +/** + * netdev2adap - return the adapter structure associated with a net_device + * @dev: the netdev + * + * Return the struct adapter associated with a net_device + */ +static inline struct adapter *netdev2adap(const struct net_device *dev) +{ + return netdev2pinfo(dev)->adapter; +} + +/* + * OS "Callback" function declarations. These are functions that the OS code + * is "contracted" to provide for the common code. + */ +void t4vf_os_link_changed(struct adapter *, int, int); +void t4vf_os_portmod_changed(struct adapter *, int); + +/* + * SGE function prototype declarations. + */ +int t4vf_sge_alloc_rxq(struct adapter *, struct sge_rspq *, bool, + struct net_device *, int, + struct sge_fl *, rspq_handler_t); +int t4vf_sge_alloc_eth_txq(struct adapter *, struct sge_eth_txq *, + struct net_device *, struct netdev_queue *, + unsigned int); +void t4vf_free_sge_resources(struct adapter *); + +netdev_tx_t t4vf_eth_xmit(struct sk_buff *, struct net_device *); +int t4vf_ethrx_handler(struct sge_rspq *, const __be64 *, + const struct pkt_gl *); + +irq_handler_t t4vf_intr_handler(struct adapter *); +irqreturn_t t4vf_sge_intr_msix(int, void *); + +int t4vf_sge_init(struct adapter *); +void t4vf_sge_start(struct adapter *); +void t4vf_sge_stop(struct adapter *); + +#endif /* __CXGB4VF_ADAPTER_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c new file mode 100644 index 000000000..63b2bd084 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c @@ -0,0 +1,3447 @@ +/* + * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet + * driver for Linux. + * + * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/init.h> +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> +#include <linux/debugfs.h> +#include <linux/ethtool.h> +#include <linux/mdio.h> + +#include "t4vf_common.h" +#include "t4vf_defs.h" + +#include "../cxgb4/t4_regs.h" +#include "../cxgb4/t4_msg.h" + +/* + * Generic information about the driver. + */ +#define DRV_DESC "Chelsio T4/T5/T6 Virtual Function (VF) Network Driver" + +/* + * Module Parameters. + * ================== + */ + +/* + * Default ethtool "message level" for adapters. + */ +#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \ + NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\ + NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) + +/* + * The driver uses the best interrupt scheme available on a platform in the + * order MSI-X then MSI. This parameter determines which of these schemes the + * driver may consider as follows: + * + * msi = 2: choose from among MSI-X and MSI + * msi = 1: only consider MSI interrupts + * + * Note that unlike the Physical Function driver, this Virtual Function driver + * does _not_ support legacy INTx interrupts (this limitation is mandated by + * the PCI-E SR-IOV standard). + */ +#define MSI_MSIX 2 +#define MSI_MSI 1 +#define MSI_DEFAULT MSI_MSIX + +static int msi = MSI_DEFAULT; + +module_param(msi, int, 0644); +MODULE_PARM_DESC(msi, "whether to use MSI-X or MSI"); + +/* + * Fundamental constants. + * ====================== + */ + +enum { + MAX_TXQ_ENTRIES = 16384, + MAX_RSPQ_ENTRIES = 16384, + MAX_RX_BUFFERS = 16384, + + MIN_TXQ_ENTRIES = 32, + MIN_RSPQ_ENTRIES = 128, + MIN_FL_ENTRIES = 16, + + /* + * For purposes of manipulating the Free List size we need to + * recognize that Free Lists are actually Egress Queues (the host + * produces free buffers which the hardware consumes), Egress Queues + * indices are all in units of Egress Context Units bytes, and free + * list entries are 64-bit PCI DMA addresses. And since the state of + * the Producer Index == the Consumer Index implies an EMPTY list, we + * always have at least one Egress Unit's worth of Free List entries + * unused. See sge.c for more details ... + */ + EQ_UNIT = SGE_EQ_IDXSIZE, + FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64), + MIN_FL_RESID = FL_PER_EQ_UNIT, +}; + +/* + * Global driver state. + * ==================== + */ + +static struct dentry *cxgb4vf_debugfs_root; + +/* + * OS "Callback" functions. + * ======================== + */ + +/* + * The link status has changed on the indicated "port" (Virtual Interface). + */ +void t4vf_os_link_changed(struct adapter *adapter, int pidx, int link_ok) +{ + struct net_device *dev = adapter->port[pidx]; + + /* + * If the port is disabled or the current recorded "link up" + * status matches the new status, just return. + */ + if (!netif_running(dev) || link_ok == netif_carrier_ok(dev)) + return; + + /* + * Tell the OS that the link status has changed and print a short + * informative message on the console about the event. + */ + if (link_ok) { + const char *s; + const char *fc; + const struct port_info *pi = netdev_priv(dev); + + netif_carrier_on(dev); + + switch (pi->link_cfg.speed) { + case 100: + s = "100Mbps"; + break; + case 1000: + s = "1Gbps"; + break; + case 10000: + s = "10Gbps"; + break; + case 25000: + s = "25Gbps"; + break; + case 40000: + s = "40Gbps"; + break; + case 100000: + s = "100Gbps"; + break; + + default: + s = "unknown"; + break; + } + + switch ((int)pi->link_cfg.fc) { + case PAUSE_RX: + fc = "RX"; + break; + + case PAUSE_TX: + fc = "TX"; + break; + + case PAUSE_RX | PAUSE_TX: + fc = "RX/TX"; + break; + + default: + fc = "no"; + break; + } + + netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, fc); + } else { + netif_carrier_off(dev); + netdev_info(dev, "link down\n"); + } +} + +/* + * THe port module type has changed on the indicated "port" (Virtual + * Interface). + */ +void t4vf_os_portmod_changed(struct adapter *adapter, int pidx) +{ + static const char * const mod_str[] = { + NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM" + }; + const struct net_device *dev = adapter->port[pidx]; + const struct port_info *pi = netdev_priv(dev); + + if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) + dev_info(adapter->pdev_dev, "%s: port module unplugged\n", + dev->name); + else if (pi->mod_type < ARRAY_SIZE(mod_str)) + dev_info(adapter->pdev_dev, "%s: %s port module inserted\n", + dev->name, mod_str[pi->mod_type]); + else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) + dev_info(adapter->pdev_dev, "%s: unsupported optical port " + "module inserted\n", dev->name); + else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) + dev_info(adapter->pdev_dev, "%s: unknown port module inserted," + "forcing TWINAX\n", dev->name); + else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR) + dev_info(adapter->pdev_dev, "%s: transceiver module error\n", + dev->name); + else + dev_info(adapter->pdev_dev, "%s: unknown module type %d " + "inserted\n", dev->name, pi->mod_type); +} + +static int cxgb4vf_set_addr_hash(struct port_info *pi) +{ + struct adapter *adapter = pi->adapter; + u64 vec = 0; + bool ucast = false; + struct hash_mac_addr *entry; + + /* Calculate the hash vector for the updated list and program it */ + list_for_each_entry(entry, &adapter->mac_hlist, list) { + ucast |= is_unicast_ether_addr(entry->addr); + vec |= (1ULL << hash_mac_addr(entry->addr)); + } + return t4vf_set_addr_hash(adapter, pi->viid, ucast, vec, false); +} + +/** + * cxgb4vf_change_mac - Update match filter for a MAC address. + * @pi: the port_info + * @viid: the VI id + * @tcam_idx: TCAM index of existing filter for old value of MAC address, + * or -1 + * @addr: the new MAC address value + * @persistent: whether a new MAC allocation should be persistent + * + * Modifies an MPS filter and sets it to the new MAC address if + * @tcam_idx >= 0, or adds the MAC address to a new filter if + * @tcam_idx < 0. In the latter case the address is added persistently + * if @persist is %true. + * Addresses are programmed to hash region, if tcam runs out of entries. + * + */ +static int cxgb4vf_change_mac(struct port_info *pi, unsigned int viid, + int *tcam_idx, const u8 *addr, bool persistent) +{ + struct hash_mac_addr *new_entry, *entry; + struct adapter *adapter = pi->adapter; + int ret; + + ret = t4vf_change_mac(adapter, viid, *tcam_idx, addr, persistent); + /* We ran out of TCAM entries. try programming hash region. */ + if (ret == -ENOMEM) { + /* If the MAC address to be updated is in the hash addr + * list, update it from the list + */ + list_for_each_entry(entry, &adapter->mac_hlist, list) { + if (entry->iface_mac) { + ether_addr_copy(entry->addr, addr); + goto set_hash; + } + } + new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL); + if (!new_entry) + return -ENOMEM; + ether_addr_copy(new_entry->addr, addr); + new_entry->iface_mac = true; + list_add_tail(&new_entry->list, &adapter->mac_hlist); +set_hash: + ret = cxgb4vf_set_addr_hash(pi); + } else if (ret >= 0) { + *tcam_idx = ret; + ret = 0; + } + + return ret; +} + +/* + * Net device operations. + * ====================== + */ + + + + +/* + * Perform the MAC and PHY actions needed to enable a "port" (Virtual + * Interface). + */ +static int link_start(struct net_device *dev) +{ + int ret; + struct port_info *pi = netdev_priv(dev); + + /* + * We do not set address filters and promiscuity here, the stack does + * that step explicitly. Enable vlan accel. + */ + ret = t4vf_set_rxmode(pi->adapter, pi->viid, dev->mtu, -1, -1, -1, 1, + true); + if (ret == 0) + ret = cxgb4vf_change_mac(pi, pi->viid, + &pi->xact_addr_filt, + dev->dev_addr, true); + + /* + * We don't need to actually "start the link" itself since the + * firmware will do that for us when the first Virtual Interface + * is enabled on a port. + */ + if (ret == 0) + ret = t4vf_enable_pi(pi->adapter, pi, true, true); + + return ret; +} + +/* + * Name the MSI-X interrupts. + */ +static void name_msix_vecs(struct adapter *adapter) +{ + int namelen = sizeof(adapter->msix_info[0].desc) - 1; + int pidx; + + /* + * Firmware events. + */ + snprintf(adapter->msix_info[MSIX_FW].desc, namelen, + "%s-FWeventq", adapter->name); + adapter->msix_info[MSIX_FW].desc[namelen] = 0; + + /* + * Ethernet queues. + */ + for_each_port(adapter, pidx) { + struct net_device *dev = adapter->port[pidx]; + const struct port_info *pi = netdev_priv(dev); + int qs, msi; + + for (qs = 0, msi = MSIX_IQFLINT; qs < pi->nqsets; qs++, msi++) { + snprintf(adapter->msix_info[msi].desc, namelen, + "%s-%d", dev->name, qs); + adapter->msix_info[msi].desc[namelen] = 0; + } + } +} + +/* + * Request all of our MSI-X resources. + */ +static int request_msix_queue_irqs(struct adapter *adapter) +{ + struct sge *s = &adapter->sge; + int rxq, msi, err; + + /* + * Firmware events. + */ + err = request_irq(adapter->msix_info[MSIX_FW].vec, t4vf_sge_intr_msix, + 0, adapter->msix_info[MSIX_FW].desc, &s->fw_evtq); + if (err) + return err; + + /* + * Ethernet queues. + */ + msi = MSIX_IQFLINT; + for_each_ethrxq(s, rxq) { + err = request_irq(adapter->msix_info[msi].vec, + t4vf_sge_intr_msix, 0, + adapter->msix_info[msi].desc, + &s->ethrxq[rxq].rspq); + if (err) + goto err_free_irqs; + msi++; + } + return 0; + +err_free_irqs: + while (--rxq >= 0) + free_irq(adapter->msix_info[--msi].vec, &s->ethrxq[rxq].rspq); + free_irq(adapter->msix_info[MSIX_FW].vec, &s->fw_evtq); + return err; +} + +/* + * Free our MSI-X resources. + */ +static void free_msix_queue_irqs(struct adapter *adapter) +{ + struct sge *s = &adapter->sge; + int rxq, msi; + + free_irq(adapter->msix_info[MSIX_FW].vec, &s->fw_evtq); + msi = MSIX_IQFLINT; + for_each_ethrxq(s, rxq) + free_irq(adapter->msix_info[msi++].vec, + &s->ethrxq[rxq].rspq); +} + +/* + * Turn on NAPI and start up interrupts on a response queue. + */ +static void qenable(struct sge_rspq *rspq) +{ + napi_enable(&rspq->napi); + + /* + * 0-increment the Going To Sleep register to start the timer and + * enable interrupts. + */ + t4_write_reg(rspq->adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS, + CIDXINC_V(0) | + SEINTARM_V(rspq->intr_params) | + INGRESSQID_V(rspq->cntxt_id)); +} + +/* + * Enable NAPI scheduling and interrupt generation for all Receive Queues. + */ +static void enable_rx(struct adapter *adapter) +{ + int rxq; + struct sge *s = &adapter->sge; + + for_each_ethrxq(s, rxq) + qenable(&s->ethrxq[rxq].rspq); + qenable(&s->fw_evtq); + + /* + * The interrupt queue doesn't use NAPI so we do the 0-increment of + * its Going To Sleep register here to get it started. + */ + if (adapter->flags & CXGB4VF_USING_MSI) + t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS, + CIDXINC_V(0) | + SEINTARM_V(s->intrq.intr_params) | + INGRESSQID_V(s->intrq.cntxt_id)); + +} + +/* + * Wait until all NAPI handlers are descheduled. + */ +static void quiesce_rx(struct adapter *adapter) +{ + struct sge *s = &adapter->sge; + int rxq; + + for_each_ethrxq(s, rxq) + napi_disable(&s->ethrxq[rxq].rspq.napi); + napi_disable(&s->fw_evtq.napi); +} + +/* + * Response queue handler for the firmware event queue. + */ +static int fwevtq_handler(struct sge_rspq *rspq, const __be64 *rsp, + const struct pkt_gl *gl) +{ + /* + * Extract response opcode and get pointer to CPL message body. + */ + struct adapter *adapter = rspq->adapter; + u8 opcode = ((const struct rss_header *)rsp)->opcode; + void *cpl = (void *)(rsp + 1); + + switch (opcode) { + case CPL_FW6_MSG: { + /* + * We've received an asynchronous message from the firmware. + */ + const struct cpl_fw6_msg *fw_msg = cpl; + if (fw_msg->type == FW6_TYPE_CMD_RPL) + t4vf_handle_fw_rpl(adapter, fw_msg->data); + break; + } + + case CPL_FW4_MSG: { + /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG. + */ + const struct cpl_sge_egr_update *p = (void *)(rsp + 3); + opcode = CPL_OPCODE_G(ntohl(p->opcode_qid)); + if (opcode != CPL_SGE_EGR_UPDATE) { + dev_err(adapter->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n" + , opcode); + break; + } + cpl = (void *)p; + } + fallthrough; + + case CPL_SGE_EGR_UPDATE: { + /* + * We've received an Egress Queue Status Update message. We + * get these, if the SGE is configured to send these when the + * firmware passes certain points in processing our TX + * Ethernet Queue or if we make an explicit request for one. + * We use these updates to determine when we may need to + * restart a TX Ethernet Queue which was stopped for lack of + * free TX Queue Descriptors ... + */ + const struct cpl_sge_egr_update *p = cpl; + unsigned int qid = EGR_QID_G(be32_to_cpu(p->opcode_qid)); + struct sge *s = &adapter->sge; + struct sge_txq *tq; + struct sge_eth_txq *txq; + unsigned int eq_idx; + + /* + * Perform sanity checking on the Queue ID to make sure it + * really refers to one of our TX Ethernet Egress Queues which + * is active and matches the queue's ID. None of these error + * conditions should ever happen so we may want to either make + * them fatal and/or conditionalized under DEBUG. + */ + eq_idx = EQ_IDX(s, qid); + if (unlikely(eq_idx >= MAX_EGRQ)) { + dev_err(adapter->pdev_dev, + "Egress Update QID %d out of range\n", qid); + break; + } + tq = s->egr_map[eq_idx]; + if (unlikely(tq == NULL)) { + dev_err(adapter->pdev_dev, + "Egress Update QID %d TXQ=NULL\n", qid); + break; + } + txq = container_of(tq, struct sge_eth_txq, q); + if (unlikely(tq->abs_id != qid)) { + dev_err(adapter->pdev_dev, + "Egress Update QID %d refers to TXQ %d\n", + qid, tq->abs_id); + break; + } + + /* + * Restart a stopped TX Queue which has less than half of its + * TX ring in use ... + */ + txq->q.restarts++; + netif_tx_wake_queue(txq->txq); + break; + } + + default: + dev_err(adapter->pdev_dev, + "unexpected CPL %#x on FW event queue\n", opcode); + } + + return 0; +} + +/* + * Allocate SGE TX/RX response queues. Determine how many sets of SGE queues + * to use and initializes them. We support multiple "Queue Sets" per port if + * we have MSI-X, otherwise just one queue set per port. + */ +static int setup_sge_queues(struct adapter *adapter) +{ + struct sge *s = &adapter->sge; + int err, pidx, msix; + + /* + * Clear "Queue Set" Free List Starving and TX Queue Mapping Error + * state. + */ + bitmap_zero(s->starving_fl, MAX_EGRQ); + + /* + * If we're using MSI interrupt mode we need to set up a "forwarded + * interrupt" queue which we'll set up with our MSI vector. The rest + * of the ingress queues will be set up to forward their interrupts to + * this queue ... This must be first since t4vf_sge_alloc_rxq() uses + * the intrq's queue ID as the interrupt forwarding queue for the + * subsequent calls ... + */ + if (adapter->flags & CXGB4VF_USING_MSI) { + err = t4vf_sge_alloc_rxq(adapter, &s->intrq, false, + adapter->port[0], 0, NULL, NULL); + if (err) + goto err_free_queues; + } + + /* + * Allocate our ingress queue for asynchronous firmware messages. + */ + err = t4vf_sge_alloc_rxq(adapter, &s->fw_evtq, true, adapter->port[0], + MSIX_FW, NULL, fwevtq_handler); + if (err) + goto err_free_queues; + + /* + * Allocate each "port"'s initial Queue Sets. These can be changed + * later on ... up to the point where any interface on the adapter is + * brought up at which point lots of things get nailed down + * permanently ... + */ + msix = MSIX_IQFLINT; + for_each_port(adapter, pidx) { + struct net_device *dev = adapter->port[pidx]; + struct port_info *pi = netdev_priv(dev); + struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset]; + struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset]; + int qs; + + for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) { + err = t4vf_sge_alloc_rxq(adapter, &rxq->rspq, false, + dev, msix++, + &rxq->fl, t4vf_ethrx_handler); + if (err) + goto err_free_queues; + + err = t4vf_sge_alloc_eth_txq(adapter, txq, dev, + netdev_get_tx_queue(dev, qs), + s->fw_evtq.cntxt_id); + if (err) + goto err_free_queues; + + rxq->rspq.idx = qs; + memset(&rxq->stats, 0, sizeof(rxq->stats)); + } + } + + /* + * Create the reverse mappings for the queues. + */ + s->egr_base = s->ethtxq[0].q.abs_id - s->ethtxq[0].q.cntxt_id; + s->ingr_base = s->ethrxq[0].rspq.abs_id - s->ethrxq[0].rspq.cntxt_id; + IQ_MAP(s, s->fw_evtq.abs_id) = &s->fw_evtq; + for_each_port(adapter, pidx) { + struct net_device *dev = adapter->port[pidx]; + struct port_info *pi = netdev_priv(dev); + struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset]; + struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset]; + int qs; + + for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) { + IQ_MAP(s, rxq->rspq.abs_id) = &rxq->rspq; + EQ_MAP(s, txq->q.abs_id) = &txq->q; + + /* + * The FW_IQ_CMD doesn't return the Absolute Queue IDs + * for Free Lists but since all of the Egress Queues + * (including Free Lists) have Relative Queue IDs + * which are computed as Absolute - Base Queue ID, we + * can synthesize the Absolute Queue IDs for the Free + * Lists. This is useful for debugging purposes when + * we want to dump Queue Contexts via the PF Driver. + */ + rxq->fl.abs_id = rxq->fl.cntxt_id + s->egr_base; + EQ_MAP(s, rxq->fl.abs_id) = &rxq->fl; + } + } + return 0; + +err_free_queues: + t4vf_free_sge_resources(adapter); + return err; +} + +/* + * Set up Receive Side Scaling (RSS) to distribute packets to multiple receive + * queues. We configure the RSS CPU lookup table to distribute to the number + * of HW receive queues, and the response queue lookup table to narrow that + * down to the response queues actually configured for each "port" (Virtual + * Interface). We always configure the RSS mapping for all ports since the + * mapping table has plenty of entries. + */ +static int setup_rss(struct adapter *adapter) +{ + int pidx; + + for_each_port(adapter, pidx) { + struct port_info *pi = adap2pinfo(adapter, pidx); + struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[pi->first_qset]; + u16 rss[MAX_PORT_QSETS]; + int qs, err; + + for (qs = 0; qs < pi->nqsets; qs++) + rss[qs] = rxq[qs].rspq.abs_id; + + err = t4vf_config_rss_range(adapter, pi->viid, + 0, pi->rss_size, rss, pi->nqsets); + if (err) + return err; + + /* + * Perform Global RSS Mode-specific initialization. + */ + switch (adapter->params.rss.mode) { + case FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL: + /* + * If Tunnel All Lookup isn't specified in the global + * RSS Configuration, then we need to specify a + * default Ingress Queue for any ingress packets which + * aren't hashed. We'll use our first ingress queue + * ... + */ + if (!adapter->params.rss.u.basicvirtual.tnlalllookup) { + union rss_vi_config config; + err = t4vf_read_rss_vi_config(adapter, + pi->viid, + &config); + if (err) + return err; + config.basicvirtual.defaultq = + rxq[0].rspq.abs_id; + err = t4vf_write_rss_vi_config(adapter, + pi->viid, + &config); + if (err) + return err; + } + break; + } + } + + return 0; +} + +/* + * Bring the adapter up. Called whenever we go from no "ports" open to having + * one open. This function performs the actions necessary to make an adapter + * operational, such as completing the initialization of HW modules, and + * enabling interrupts. Must be called with the rtnl lock held. (Note that + * this is called "cxgb_up" in the PF Driver.) + */ +static int adapter_up(struct adapter *adapter) +{ + int err; + + /* + * If this is the first time we've been called, perform basic + * adapter setup. Once we've done this, many of our adapter + * parameters can no longer be changed ... + */ + if ((adapter->flags & CXGB4VF_FULL_INIT_DONE) == 0) { + err = setup_sge_queues(adapter); + if (err) + return err; + err = setup_rss(adapter); + if (err) { + t4vf_free_sge_resources(adapter); + return err; + } + + if (adapter->flags & CXGB4VF_USING_MSIX) + name_msix_vecs(adapter); + + adapter->flags |= CXGB4VF_FULL_INIT_DONE; + } + + /* + * Acquire our interrupt resources. We only support MSI-X and MSI. + */ + BUG_ON((adapter->flags & + (CXGB4VF_USING_MSIX | CXGB4VF_USING_MSI)) == 0); + if (adapter->flags & CXGB4VF_USING_MSIX) + err = request_msix_queue_irqs(adapter); + else + err = request_irq(adapter->pdev->irq, + t4vf_intr_handler(adapter), 0, + adapter->name, adapter); + if (err) { + dev_err(adapter->pdev_dev, "request_irq failed, err %d\n", + err); + return err; + } + + /* + * Enable NAPI ingress processing and return success. + */ + enable_rx(adapter); + t4vf_sge_start(adapter); + + return 0; +} + +/* + * Bring the adapter down. Called whenever the last "port" (Virtual + * Interface) closed. (Note that this routine is called "cxgb_down" in the PF + * Driver.) + */ +static void adapter_down(struct adapter *adapter) +{ + /* + * Free interrupt resources. + */ + if (adapter->flags & CXGB4VF_USING_MSIX) + free_msix_queue_irqs(adapter); + else + free_irq(adapter->pdev->irq, adapter); + + /* + * Wait for NAPI handlers to finish. + */ + quiesce_rx(adapter); +} + +/* + * Start up a net device. + */ +static int cxgb4vf_open(struct net_device *dev) +{ + int err; + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + /* + * If we don't have a connection to the firmware there's nothing we + * can do. + */ + if (!(adapter->flags & CXGB4VF_FW_OK)) + return -ENXIO; + + /* + * If this is the first interface that we're opening on the "adapter", + * bring the "adapter" up now. + */ + if (adapter->open_device_map == 0) { + err = adapter_up(adapter); + if (err) + return err; + } + + /* It's possible that the basic port information could have + * changed since we first read it. + */ + err = t4vf_update_port_info(pi); + if (err < 0) + goto err_unwind; + + /* + * Note that this interface is up and start everything up ... + */ + err = link_start(dev); + if (err) + goto err_unwind; + + pi->vlan_id = t4vf_get_vf_vlan_acl(adapter); + + netif_tx_start_all_queues(dev); + set_bit(pi->port_id, &adapter->open_device_map); + return 0; + +err_unwind: + if (adapter->open_device_map == 0) + adapter_down(adapter); + return err; +} + +/* + * Shut down a net device. This routine is called "cxgb_close" in the PF + * Driver ... + */ +static int cxgb4vf_stop(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + netif_tx_stop_all_queues(dev); + netif_carrier_off(dev); + t4vf_enable_pi(adapter, pi, false, false); + + clear_bit(pi->port_id, &adapter->open_device_map); + if (adapter->open_device_map == 0) + adapter_down(adapter); + return 0; +} + +/* + * Translate our basic statistics into the standard "ifconfig" statistics. + */ +static struct net_device_stats *cxgb4vf_get_stats(struct net_device *dev) +{ + struct t4vf_port_stats stats; + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adapter = pi->adapter; + struct net_device_stats *ns = &dev->stats; + int err; + + spin_lock(&adapter->stats_lock); + err = t4vf_get_port_stats(adapter, pi->pidx, &stats); + spin_unlock(&adapter->stats_lock); + + memset(ns, 0, sizeof(*ns)); + if (err) + return ns; + + ns->tx_bytes = (stats.tx_bcast_bytes + stats.tx_mcast_bytes + + stats.tx_ucast_bytes + stats.tx_offload_bytes); + ns->tx_packets = (stats.tx_bcast_frames + stats.tx_mcast_frames + + stats.tx_ucast_frames + stats.tx_offload_frames); + ns->rx_bytes = (stats.rx_bcast_bytes + stats.rx_mcast_bytes + + stats.rx_ucast_bytes); + ns->rx_packets = (stats.rx_bcast_frames + stats.rx_mcast_frames + + stats.rx_ucast_frames); + ns->multicast = stats.rx_mcast_frames; + ns->tx_errors = stats.tx_drop_frames; + ns->rx_errors = stats.rx_err_frames; + + return ns; +} + +static int cxgb4vf_mac_sync(struct net_device *netdev, const u8 *mac_addr) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adapter = pi->adapter; + int ret; + u64 mhash = 0; + u64 uhash = 0; + bool free = false; + bool ucast = is_unicast_ether_addr(mac_addr); + const u8 *maclist[1] = {mac_addr}; + struct hash_mac_addr *new_entry; + + ret = t4vf_alloc_mac_filt(adapter, pi->viid, free, 1, maclist, + NULL, ucast ? &uhash : &mhash, false); + if (ret < 0) + goto out; + /* if hash != 0, then add the addr to hash addr list + * so on the end we will calculate the hash for the + * list and program it + */ + if (uhash || mhash) { + new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC); + if (!new_entry) + return -ENOMEM; + ether_addr_copy(new_entry->addr, mac_addr); + list_add_tail(&new_entry->list, &adapter->mac_hlist); + ret = cxgb4vf_set_addr_hash(pi); + } +out: + return ret < 0 ? ret : 0; +} + +static int cxgb4vf_mac_unsync(struct net_device *netdev, const u8 *mac_addr) +{ + struct port_info *pi = netdev_priv(netdev); + struct adapter *adapter = pi->adapter; + int ret; + const u8 *maclist[1] = {mac_addr}; + struct hash_mac_addr *entry, *tmp; + + /* If the MAC address to be removed is in the hash addr + * list, delete it from the list and update hash vector + */ + list_for_each_entry_safe(entry, tmp, &adapter->mac_hlist, list) { + if (ether_addr_equal(entry->addr, mac_addr)) { + list_del(&entry->list); + kfree(entry); + return cxgb4vf_set_addr_hash(pi); + } + } + + ret = t4vf_free_mac_filt(adapter, pi->viid, 1, maclist, false); + return ret < 0 ? -EINVAL : 0; +} + +/* + * Set RX properties of a port, such as promiscruity, address filters, and MTU. + * If @mtu is -1 it is left unchanged. + */ +static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok) +{ + struct port_info *pi = netdev_priv(dev); + + __dev_uc_sync(dev, cxgb4vf_mac_sync, cxgb4vf_mac_unsync); + __dev_mc_sync(dev, cxgb4vf_mac_sync, cxgb4vf_mac_unsync); + return t4vf_set_rxmode(pi->adapter, pi->viid, -1, + (dev->flags & IFF_PROMISC) != 0, + (dev->flags & IFF_ALLMULTI) != 0, + 1, -1, sleep_ok); +} + +/* + * Set the current receive modes on the device. + */ +static void cxgb4vf_set_rxmode(struct net_device *dev) +{ + /* unfortunately we can't return errors to the stack */ + set_rxmode(dev, -1, false); +} + +/* + * Find the entry in the interrupt holdoff timer value array which comes + * closest to the specified interrupt holdoff value. + */ +static int closest_timer(const struct sge *s, int us) +{ + int i, timer_idx = 0, min_delta = INT_MAX; + + for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { + int delta = us - s->timer_val[i]; + if (delta < 0) + delta = -delta; + if (delta < min_delta) { + min_delta = delta; + timer_idx = i; + } + } + return timer_idx; +} + +static int closest_thres(const struct sge *s, int thres) +{ + int i, delta, pktcnt_idx = 0, min_delta = INT_MAX; + + for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { + delta = thres - s->counter_val[i]; + if (delta < 0) + delta = -delta; + if (delta < min_delta) { + min_delta = delta; + pktcnt_idx = i; + } + } + return pktcnt_idx; +} + +/* + * Return a queue's interrupt hold-off time in us. 0 means no timer. + */ +static unsigned int qtimer_val(const struct adapter *adapter, + const struct sge_rspq *rspq) +{ + unsigned int timer_idx = QINTR_TIMER_IDX_G(rspq->intr_params); + + return timer_idx < SGE_NTIMERS + ? adapter->sge.timer_val[timer_idx] + : 0; +} + +/** + * set_rxq_intr_params - set a queue's interrupt holdoff parameters + * @adapter: the adapter + * @rspq: the RX response queue + * @us: the hold-off time in us, or 0 to disable timer + * @cnt: the hold-off packet count, or 0 to disable counter + * + * Sets an RX response queue's interrupt hold-off time and packet count. + * At least one of the two needs to be enabled for the queue to generate + * interrupts. + */ +static int set_rxq_intr_params(struct adapter *adapter, struct sge_rspq *rspq, + unsigned int us, unsigned int cnt) +{ + unsigned int timer_idx; + + /* + * If both the interrupt holdoff timer and count are specified as + * zero, default to a holdoff count of 1 ... + */ + if ((us | cnt) == 0) + cnt = 1; + + /* + * If an interrupt holdoff count has been specified, then find the + * closest configured holdoff count and use that. If the response + * queue has already been created, then update its queue context + * parameters ... + */ + if (cnt) { + int err; + u32 v, pktcnt_idx; + + pktcnt_idx = closest_thres(&adapter->sge, cnt); + if (rspq->desc && rspq->pktcnt_idx != pktcnt_idx) { + v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | + FW_PARAMS_PARAM_X_V( + FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) | + FW_PARAMS_PARAM_YZ_V(rspq->cntxt_id); + err = t4vf_set_params(adapter, 1, &v, &pktcnt_idx); + if (err) + return err; + } + rspq->pktcnt_idx = pktcnt_idx; + } + + /* + * Compute the closest holdoff timer index from the supplied holdoff + * timer value. + */ + timer_idx = (us == 0 + ? SGE_TIMER_RSTRT_CNTR + : closest_timer(&adapter->sge, us)); + + /* + * Update the response queue's interrupt coalescing parameters and + * return success. + */ + rspq->intr_params = (QINTR_TIMER_IDX_V(timer_idx) | + QINTR_CNT_EN_V(cnt > 0)); + return 0; +} + +/* + * Return a version number to identify the type of adapter. The scheme is: + * - bits 0..9: chip version + * - bits 10..15: chip revision + */ +static inline unsigned int mk_adap_vers(const struct adapter *adapter) +{ + /* + * Chip version 4, revision 0x3f (cxgb4vf). + */ + return CHELSIO_CHIP_VERSION(adapter->params.chip) | (0x3f << 10); +} + +/* + * Execute the specified ioctl command. + */ +static int cxgb4vf_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) +{ + int ret = 0; + + switch (cmd) { + /* + * The VF Driver doesn't have access to any of the other + * common Ethernet device ioctl()'s (like reading/writing + * PHY registers, etc. + */ + + default: + ret = -EOPNOTSUPP; + break; + } + return ret; +} + +/* + * Change the device's MTU. + */ +static int cxgb4vf_change_mtu(struct net_device *dev, int new_mtu) +{ + int ret; + struct port_info *pi = netdev_priv(dev); + + ret = t4vf_set_rxmode(pi->adapter, pi->viid, new_mtu, + -1, -1, -1, -1, true); + if (!ret) + dev->mtu = new_mtu; + return ret; +} + +static netdev_features_t cxgb4vf_fix_features(struct net_device *dev, + netdev_features_t features) +{ + /* + * Since there is no support for separate rx/tx vlan accel + * enable/disable make sure tx flag is always in same state as rx. + */ + if (features & NETIF_F_HW_VLAN_CTAG_RX) + features |= NETIF_F_HW_VLAN_CTAG_TX; + else + features &= ~NETIF_F_HW_VLAN_CTAG_TX; + + return features; +} + +static int cxgb4vf_set_features(struct net_device *dev, + netdev_features_t features) +{ + struct port_info *pi = netdev_priv(dev); + netdev_features_t changed = dev->features ^ features; + + if (changed & NETIF_F_HW_VLAN_CTAG_RX) + t4vf_set_rxmode(pi->adapter, pi->viid, -1, -1, -1, -1, + features & NETIF_F_HW_VLAN_CTAG_TX, 0); + + return 0; +} + +/* + * Change the devices MAC address. + */ +static int cxgb4vf_set_mac_addr(struct net_device *dev, void *_addr) +{ + int ret; + struct sockaddr *addr = _addr; + struct port_info *pi = netdev_priv(dev); + + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + + ret = cxgb4vf_change_mac(pi, pi->viid, &pi->xact_addr_filt, + addr->sa_data, true); + if (ret < 0) + return ret; + + eth_hw_addr_set(dev, addr->sa_data); + return 0; +} + +#ifdef CONFIG_NET_POLL_CONTROLLER +/* + * Poll all of our receive queues. This is called outside of normal interrupt + * context. + */ +static void cxgb4vf_poll_controller(struct net_device *dev) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + if (adapter->flags & CXGB4VF_USING_MSIX) { + struct sge_eth_rxq *rxq; + int nqsets; + + rxq = &adapter->sge.ethrxq[pi->first_qset]; + for (nqsets = pi->nqsets; nqsets; nqsets--) { + t4vf_sge_intr_msix(0, &rxq->rspq); + rxq++; + } + } else + t4vf_intr_handler(adapter)(0, adapter); +} +#endif + +/* + * Ethtool operations. + * =================== + * + * Note that we don't support any ethtool operations which change the physical + * state of the port to which we're linked. + */ + +/** + * from_fw_port_mod_type - translate Firmware Port/Module type to Ethtool + * @port_type: Firmware Port Type + * @mod_type: Firmware Module Type + * + * Translate Firmware Port/Module type to Ethtool Port Type. + */ +static int from_fw_port_mod_type(enum fw_port_type port_type, + enum fw_port_module_type mod_type) +{ + if (port_type == FW_PORT_TYPE_BT_SGMII || + port_type == FW_PORT_TYPE_BT_XFI || + port_type == FW_PORT_TYPE_BT_XAUI) { + return PORT_TP; + } else if (port_type == FW_PORT_TYPE_FIBER_XFI || + port_type == FW_PORT_TYPE_FIBER_XAUI) { + return PORT_FIBRE; + } else if (port_type == FW_PORT_TYPE_SFP || + port_type == FW_PORT_TYPE_QSFP_10G || + port_type == FW_PORT_TYPE_QSA || + port_type == FW_PORT_TYPE_QSFP || + port_type == FW_PORT_TYPE_CR4_QSFP || + port_type == FW_PORT_TYPE_CR_QSFP || + port_type == FW_PORT_TYPE_CR2_QSFP || + port_type == FW_PORT_TYPE_SFP28) { + if (mod_type == FW_PORT_MOD_TYPE_LR || + mod_type == FW_PORT_MOD_TYPE_SR || + mod_type == FW_PORT_MOD_TYPE_ER || + mod_type == FW_PORT_MOD_TYPE_LRM) + return PORT_FIBRE; + else if (mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE || + mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE) + return PORT_DA; + else + return PORT_OTHER; + } else if (port_type == FW_PORT_TYPE_KR4_100G || + port_type == FW_PORT_TYPE_KR_SFP28 || + port_type == FW_PORT_TYPE_KR_XLAUI) { + return PORT_NONE; + } + + return PORT_OTHER; +} + +/** + * fw_caps_to_lmm - translate Firmware to ethtool Link Mode Mask + * @port_type: Firmware Port Type + * @fw_caps: Firmware Port Capabilities + * @link_mode_mask: ethtool Link Mode Mask + * + * Translate a Firmware Port Capabilities specification to an ethtool + * Link Mode Mask. + */ +static void fw_caps_to_lmm(enum fw_port_type port_type, + unsigned int fw_caps, + unsigned long *link_mode_mask) +{ + #define SET_LMM(__lmm_name) \ + __set_bit(ETHTOOL_LINK_MODE_ ## __lmm_name ## _BIT, \ + link_mode_mask) + + #define FW_CAPS_TO_LMM(__fw_name, __lmm_name) \ + do { \ + if (fw_caps & FW_PORT_CAP32_ ## __fw_name) \ + SET_LMM(__lmm_name); \ + } while (0) + + switch (port_type) { + case FW_PORT_TYPE_BT_SGMII: + case FW_PORT_TYPE_BT_XFI: + case FW_PORT_TYPE_BT_XAUI: + SET_LMM(TP); + FW_CAPS_TO_LMM(SPEED_100M, 100baseT_Full); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full); + break; + + case FW_PORT_TYPE_KX4: + case FW_PORT_TYPE_KX: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKX4_Full); + break; + + case FW_PORT_TYPE_KR: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + break; + + case FW_PORT_TYPE_BP_AP: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseR_FEC); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + break; + + case FW_PORT_TYPE_BP4_AP: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseR_FEC); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKX4_Full); + break; + + case FW_PORT_TYPE_FIBER_XFI: + case FW_PORT_TYPE_FIBER_XAUI: + case FW_PORT_TYPE_SFP: + case FW_PORT_TYPE_QSFP_10G: + case FW_PORT_TYPE_QSA: + SET_LMM(FIBRE); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full); + break; + + case FW_PORT_TYPE_BP40_BA: + case FW_PORT_TYPE_QSFP: + SET_LMM(FIBRE); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full); + FW_CAPS_TO_LMM(SPEED_40G, 40000baseSR4_Full); + break; + + case FW_PORT_TYPE_CR_QSFP: + case FW_PORT_TYPE_SFP28: + SET_LMM(FIBRE); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full); + FW_CAPS_TO_LMM(SPEED_25G, 25000baseCR_Full); + break; + + case FW_PORT_TYPE_KR_SFP28: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + FW_CAPS_TO_LMM(SPEED_25G, 25000baseKR_Full); + break; + + case FW_PORT_TYPE_KR_XLAUI: + SET_LMM(Backplane); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + FW_CAPS_TO_LMM(SPEED_40G, 40000baseKR4_Full); + break; + + case FW_PORT_TYPE_CR2_QSFP: + SET_LMM(FIBRE); + FW_CAPS_TO_LMM(SPEED_50G, 50000baseSR2_Full); + break; + + case FW_PORT_TYPE_KR4_100G: + case FW_PORT_TYPE_CR4_QSFP: + SET_LMM(FIBRE); + FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full); + FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full); + FW_CAPS_TO_LMM(SPEED_40G, 40000baseSR4_Full); + FW_CAPS_TO_LMM(SPEED_25G, 25000baseCR_Full); + FW_CAPS_TO_LMM(SPEED_50G, 50000baseCR2_Full); + FW_CAPS_TO_LMM(SPEED_100G, 100000baseCR4_Full); + break; + + default: + break; + } + + if (fw_caps & FW_PORT_CAP32_FEC_V(FW_PORT_CAP32_FEC_M)) { + FW_CAPS_TO_LMM(FEC_RS, FEC_RS); + FW_CAPS_TO_LMM(FEC_BASER_RS, FEC_BASER); + } else { + SET_LMM(FEC_NONE); + } + + FW_CAPS_TO_LMM(ANEG, Autoneg); + FW_CAPS_TO_LMM(802_3_PAUSE, Pause); + FW_CAPS_TO_LMM(802_3_ASM_DIR, Asym_Pause); + + #undef FW_CAPS_TO_LMM + #undef SET_LMM +} + +static int cxgb4vf_get_link_ksettings(struct net_device *dev, + struct ethtool_link_ksettings *link_ksettings) +{ + struct port_info *pi = netdev_priv(dev); + struct ethtool_link_settings *base = &link_ksettings->base; + + /* For the nonce, the Firmware doesn't send up Port State changes + * when the Virtual Interface attached to the Port is down. So + * if it's down, let's grab any changes. + */ + if (!netif_running(dev)) + (void)t4vf_update_port_info(pi); + + ethtool_link_ksettings_zero_link_mode(link_ksettings, supported); + ethtool_link_ksettings_zero_link_mode(link_ksettings, advertising); + ethtool_link_ksettings_zero_link_mode(link_ksettings, lp_advertising); + + base->port = from_fw_port_mod_type(pi->port_type, pi->mod_type); + + if (pi->mdio_addr >= 0) { + base->phy_address = pi->mdio_addr; + base->mdio_support = (pi->port_type == FW_PORT_TYPE_BT_SGMII + ? ETH_MDIO_SUPPORTS_C22 + : ETH_MDIO_SUPPORTS_C45); + } else { + base->phy_address = 255; + base->mdio_support = 0; + } + + fw_caps_to_lmm(pi->port_type, pi->link_cfg.pcaps, + link_ksettings->link_modes.supported); + fw_caps_to_lmm(pi->port_type, pi->link_cfg.acaps, + link_ksettings->link_modes.advertising); + fw_caps_to_lmm(pi->port_type, pi->link_cfg.lpacaps, + link_ksettings->link_modes.lp_advertising); + + if (netif_carrier_ok(dev)) { + base->speed = pi->link_cfg.speed; + base->duplex = DUPLEX_FULL; + } else { + base->speed = SPEED_UNKNOWN; + base->duplex = DUPLEX_UNKNOWN; + } + + base->autoneg = pi->link_cfg.autoneg; + if (pi->link_cfg.pcaps & FW_PORT_CAP32_ANEG) + ethtool_link_ksettings_add_link_mode(link_ksettings, + supported, Autoneg); + if (pi->link_cfg.autoneg) + ethtool_link_ksettings_add_link_mode(link_ksettings, + advertising, Autoneg); + + return 0; +} + +/* Translate the Firmware FEC value into the ethtool value. */ +static inline unsigned int fwcap_to_eth_fec(unsigned int fw_fec) +{ + unsigned int eth_fec = 0; + + if (fw_fec & FW_PORT_CAP32_FEC_RS) + eth_fec |= ETHTOOL_FEC_RS; + if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS) + eth_fec |= ETHTOOL_FEC_BASER; + + /* if nothing is set, then FEC is off */ + if (!eth_fec) + eth_fec = ETHTOOL_FEC_OFF; + + return eth_fec; +} + +/* Translate Common Code FEC value into ethtool value. */ +static inline unsigned int cc_to_eth_fec(unsigned int cc_fec) +{ + unsigned int eth_fec = 0; + + if (cc_fec & FEC_AUTO) + eth_fec |= ETHTOOL_FEC_AUTO; + if (cc_fec & FEC_RS) + eth_fec |= ETHTOOL_FEC_RS; + if (cc_fec & FEC_BASER_RS) + eth_fec |= ETHTOOL_FEC_BASER; + + /* if nothing is set, then FEC is off */ + if (!eth_fec) + eth_fec = ETHTOOL_FEC_OFF; + + return eth_fec; +} + +static int cxgb4vf_get_fecparam(struct net_device *dev, + struct ethtool_fecparam *fec) +{ + const struct port_info *pi = netdev_priv(dev); + const struct link_config *lc = &pi->link_cfg; + + /* Translate the Firmware FEC Support into the ethtool value. We + * always support IEEE 802.3 "automatic" selection of Link FEC type if + * any FEC is supported. + */ + fec->fec = fwcap_to_eth_fec(lc->pcaps); + if (fec->fec != ETHTOOL_FEC_OFF) + fec->fec |= ETHTOOL_FEC_AUTO; + + /* Translate the current internal FEC parameters into the + * ethtool values. + */ + fec->active_fec = cc_to_eth_fec(lc->fec); + return 0; +} + +/* + * Return our driver information. + */ +static void cxgb4vf_get_drvinfo(struct net_device *dev, + struct ethtool_drvinfo *drvinfo) +{ + struct adapter *adapter = netdev2adap(dev); + + strscpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, pci_name(to_pci_dev(dev->dev.parent)), + sizeof(drvinfo->bus_info)); + snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), + "%u.%u.%u.%u, TP %u.%u.%u.%u", + FW_HDR_FW_VER_MAJOR_G(adapter->params.dev.fwrev), + FW_HDR_FW_VER_MINOR_G(adapter->params.dev.fwrev), + FW_HDR_FW_VER_MICRO_G(adapter->params.dev.fwrev), + FW_HDR_FW_VER_BUILD_G(adapter->params.dev.fwrev), + FW_HDR_FW_VER_MAJOR_G(adapter->params.dev.tprev), + FW_HDR_FW_VER_MINOR_G(adapter->params.dev.tprev), + FW_HDR_FW_VER_MICRO_G(adapter->params.dev.tprev), + FW_HDR_FW_VER_BUILD_G(adapter->params.dev.tprev)); +} + +/* + * Return current adapter message level. + */ +static u32 cxgb4vf_get_msglevel(struct net_device *dev) +{ + return netdev2adap(dev)->msg_enable; +} + +/* + * Set current adapter message level. + */ +static void cxgb4vf_set_msglevel(struct net_device *dev, u32 msglevel) +{ + netdev2adap(dev)->msg_enable = msglevel; +} + +/* + * Return the device's current Queue Set ring size parameters along with the + * allowed maximum values. Since ethtool doesn't understand the concept of + * multi-queue devices, we just return the current values associated with the + * first Queue Set. + */ +static void cxgb4vf_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) +{ + const struct port_info *pi = netdev_priv(dev); + const struct sge *s = &pi->adapter->sge; + + rp->rx_max_pending = MAX_RX_BUFFERS; + rp->rx_mini_max_pending = MAX_RSPQ_ENTRIES; + rp->rx_jumbo_max_pending = 0; + rp->tx_max_pending = MAX_TXQ_ENTRIES; + + rp->rx_pending = s->ethrxq[pi->first_qset].fl.size - MIN_FL_RESID; + rp->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size; + rp->rx_jumbo_pending = 0; + rp->tx_pending = s->ethtxq[pi->first_qset].q.size; +} + +/* + * Set the Queue Set ring size parameters for the device. Again, since + * ethtool doesn't allow for the concept of multiple queues per device, we'll + * apply these new values across all of the Queue Sets associated with the + * device -- after vetting them of course! + */ +static int cxgb4vf_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) +{ + const struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + struct sge *s = &adapter->sge; + int qs; + + if (rp->rx_pending > MAX_RX_BUFFERS || + rp->rx_jumbo_pending || + rp->tx_pending > MAX_TXQ_ENTRIES || + rp->rx_mini_pending > MAX_RSPQ_ENTRIES || + rp->rx_mini_pending < MIN_RSPQ_ENTRIES || + rp->rx_pending < MIN_FL_ENTRIES || + rp->tx_pending < MIN_TXQ_ENTRIES) + return -EINVAL; + + if (adapter->flags & CXGB4VF_FULL_INIT_DONE) + return -EBUSY; + + for (qs = pi->first_qset; qs < pi->first_qset + pi->nqsets; qs++) { + s->ethrxq[qs].fl.size = rp->rx_pending + MIN_FL_RESID; + s->ethrxq[qs].rspq.size = rp->rx_mini_pending; + s->ethtxq[qs].q.size = rp->tx_pending; + } + return 0; +} + +/* + * Return the interrupt holdoff timer and count for the first Queue Set on the + * device. Our extension ioctl() (the cxgbtool interface) allows the + * interrupt holdoff timer to be read on all of the device's Queue Sets. + */ +static int cxgb4vf_get_coalesce(struct net_device *dev, + struct ethtool_coalesce *coalesce, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + const struct port_info *pi = netdev_priv(dev); + const struct adapter *adapter = pi->adapter; + const struct sge_rspq *rspq = &adapter->sge.ethrxq[pi->first_qset].rspq; + + coalesce->rx_coalesce_usecs = qtimer_val(adapter, rspq); + coalesce->rx_max_coalesced_frames = + ((rspq->intr_params & QINTR_CNT_EN_F) + ? adapter->sge.counter_val[rspq->pktcnt_idx] + : 0); + return 0; +} + +/* + * Set the RX interrupt holdoff timer and count for the first Queue Set on the + * interface. Our extension ioctl() (the cxgbtool interface) allows us to set + * the interrupt holdoff timer on any of the device's Queue Sets. + */ +static int cxgb4vf_set_coalesce(struct net_device *dev, + struct ethtool_coalesce *coalesce, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + const struct port_info *pi = netdev_priv(dev); + struct adapter *adapter = pi->adapter; + + return set_rxq_intr_params(adapter, + &adapter->sge.ethrxq[pi->first_qset].rspq, + coalesce->rx_coalesce_usecs, + coalesce->rx_max_coalesced_frames); +} + +/* + * Report current port link pause parameter settings. + */ +static void cxgb4vf_get_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *pauseparam) +{ + struct port_info *pi = netdev_priv(dev); + + pauseparam->autoneg = (pi->link_cfg.requested_fc & PAUSE_AUTONEG) != 0; + pauseparam->rx_pause = (pi->link_cfg.advertised_fc & PAUSE_RX) != 0; + pauseparam->tx_pause = (pi->link_cfg.advertised_fc & PAUSE_TX) != 0; +} + +/* + * Identify the port by blinking the port's LED. + */ +static int cxgb4vf_phys_id(struct net_device *dev, + enum ethtool_phys_id_state state) +{ + unsigned int val; + struct port_info *pi = netdev_priv(dev); + + if (state == ETHTOOL_ID_ACTIVE) + val = 0xffff; + else if (state == ETHTOOL_ID_INACTIVE) + val = 0; + else + return -EINVAL; + + return t4vf_identify_port(pi->adapter, pi->viid, val); +} + +/* + * Port stats maintained per queue of the port. + */ +struct queue_port_stats { + u64 tso; + u64 tx_csum; + u64 rx_csum; + u64 vlan_ex; + u64 vlan_ins; + u64 lro_pkts; + u64 lro_merged; +}; + +/* + * Strings for the ETH_SS_STATS statistics set ("ethtool -S"). Note that + * these need to match the order of statistics returned by + * t4vf_get_port_stats(). + */ +static const char stats_strings[][ETH_GSTRING_LEN] = { + /* + * These must match the layout of the t4vf_port_stats structure. + */ + "TxBroadcastBytes ", + "TxBroadcastFrames ", + "TxMulticastBytes ", + "TxMulticastFrames ", + "TxUnicastBytes ", + "TxUnicastFrames ", + "TxDroppedFrames ", + "TxOffloadBytes ", + "TxOffloadFrames ", + "RxBroadcastBytes ", + "RxBroadcastFrames ", + "RxMulticastBytes ", + "RxMulticastFrames ", + "RxUnicastBytes ", + "RxUnicastFrames ", + "RxErrorFrames ", + + /* + * These are accumulated per-queue statistics and must match the + * order of the fields in the queue_port_stats structure. + */ + "TSO ", + "TxCsumOffload ", + "RxCsumGood ", + "VLANextractions ", + "VLANinsertions ", + "GROPackets ", + "GROMerged ", +}; + +/* + * Return the number of statistics in the specified statistics set. + */ +static int cxgb4vf_get_sset_count(struct net_device *dev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return ARRAY_SIZE(stats_strings); + default: + return -EOPNOTSUPP; + } + /*NOTREACHED*/ +} + +/* + * Return the strings for the specified statistics set. + */ +static void cxgb4vf_get_strings(struct net_device *dev, + u32 sset, + u8 *data) +{ + switch (sset) { + case ETH_SS_STATS: + memcpy(data, stats_strings, sizeof(stats_strings)); + break; + } +} + +/* + * Small utility routine to accumulate queue statistics across the queues of + * a "port". + */ +static void collect_sge_port_stats(const struct adapter *adapter, + const struct port_info *pi, + struct queue_port_stats *stats) +{ + const struct sge_eth_txq *txq = &adapter->sge.ethtxq[pi->first_qset]; + const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[pi->first_qset]; + int qs; + + memset(stats, 0, sizeof(*stats)); + for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) { + stats->tso += txq->tso; + stats->tx_csum += txq->tx_cso; + stats->rx_csum += rxq->stats.rx_cso; + stats->vlan_ex += rxq->stats.vlan_ex; + stats->vlan_ins += txq->vlan_ins; + stats->lro_pkts += rxq->stats.lro_pkts; + stats->lro_merged += rxq->stats.lro_merged; + } +} + +/* + * Return the ETH_SS_STATS statistics set. + */ +static void cxgb4vf_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, + u64 *data) +{ + struct port_info *pi = netdev2pinfo(dev); + struct adapter *adapter = pi->adapter; + int err = t4vf_get_port_stats(adapter, pi->pidx, + (struct t4vf_port_stats *)data); + if (err) + memset(data, 0, sizeof(struct t4vf_port_stats)); + + data += sizeof(struct t4vf_port_stats) / sizeof(u64); + collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data); +} + +/* + * Return the size of our register map. + */ +static int cxgb4vf_get_regs_len(struct net_device *dev) +{ + return T4VF_REGMAP_SIZE; +} + +/* + * Dump a block of registers, start to end inclusive, into a buffer. + */ +static void reg_block_dump(struct adapter *adapter, void *regbuf, + unsigned int start, unsigned int end) +{ + u32 *bp = regbuf + start - T4VF_REGMAP_START; + + for ( ; start <= end; start += sizeof(u32)) { + /* + * Avoid reading the Mailbox Control register since that + * can trigger a Mailbox Ownership Arbitration cycle and + * interfere with communication with the firmware. + */ + if (start == T4VF_CIM_BASE_ADDR + CIM_VF_EXT_MAILBOX_CTRL) + *bp++ = 0xffff; + else + *bp++ = t4_read_reg(adapter, start); + } +} + +/* + * Copy our entire register map into the provided buffer. + */ +static void cxgb4vf_get_regs(struct net_device *dev, + struct ethtool_regs *regs, + void *regbuf) +{ + struct adapter *adapter = netdev2adap(dev); + + regs->version = mk_adap_vers(adapter); + + /* + * Fill in register buffer with our register map. + */ + memset(regbuf, 0, T4VF_REGMAP_SIZE); + + reg_block_dump(adapter, regbuf, + T4VF_SGE_BASE_ADDR + T4VF_MOD_MAP_SGE_FIRST, + T4VF_SGE_BASE_ADDR + T4VF_MOD_MAP_SGE_LAST); + reg_block_dump(adapter, regbuf, + T4VF_MPS_BASE_ADDR + T4VF_MOD_MAP_MPS_FIRST, + T4VF_MPS_BASE_ADDR + T4VF_MOD_MAP_MPS_LAST); + + /* T5 adds new registers in the PL Register map. + */ + reg_block_dump(adapter, regbuf, + T4VF_PL_BASE_ADDR + T4VF_MOD_MAP_PL_FIRST, + T4VF_PL_BASE_ADDR + (is_t4(adapter->params.chip) + ? PL_VF_WHOAMI_A : PL_VF_REVISION_A)); + reg_block_dump(adapter, regbuf, + T4VF_CIM_BASE_ADDR + T4VF_MOD_MAP_CIM_FIRST, + T4VF_CIM_BASE_ADDR + T4VF_MOD_MAP_CIM_LAST); + + reg_block_dump(adapter, regbuf, + T4VF_MBDATA_BASE_ADDR + T4VF_MBDATA_FIRST, + T4VF_MBDATA_BASE_ADDR + T4VF_MBDATA_LAST); +} + +/* + * Report current Wake On LAN settings. + */ +static void cxgb4vf_get_wol(struct net_device *dev, + struct ethtool_wolinfo *wol) +{ + wol->supported = 0; + wol->wolopts = 0; + memset(&wol->sopass, 0, sizeof(wol->sopass)); +} + +/* + * TCP Segmentation Offload flags which we support. + */ +#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN) +#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \ + NETIF_F_GRO | NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA) + +static const struct ethtool_ops cxgb4vf_ethtool_ops = { + .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS | + ETHTOOL_COALESCE_RX_MAX_FRAMES, + .get_link_ksettings = cxgb4vf_get_link_ksettings, + .get_fecparam = cxgb4vf_get_fecparam, + .get_drvinfo = cxgb4vf_get_drvinfo, + .get_msglevel = cxgb4vf_get_msglevel, + .set_msglevel = cxgb4vf_set_msglevel, + .get_ringparam = cxgb4vf_get_ringparam, + .set_ringparam = cxgb4vf_set_ringparam, + .get_coalesce = cxgb4vf_get_coalesce, + .set_coalesce = cxgb4vf_set_coalesce, + .get_pauseparam = cxgb4vf_get_pauseparam, + .get_link = ethtool_op_get_link, + .get_strings = cxgb4vf_get_strings, + .set_phys_id = cxgb4vf_phys_id, + .get_sset_count = cxgb4vf_get_sset_count, + .get_ethtool_stats = cxgb4vf_get_ethtool_stats, + .get_regs_len = cxgb4vf_get_regs_len, + .get_regs = cxgb4vf_get_regs, + .get_wol = cxgb4vf_get_wol, +}; + +/* + * /sys/kernel/debug/cxgb4vf support code and data. + * ================================================ + */ + +/* + * Show Firmware Mailbox Command/Reply Log + * + * Note that we don't do any locking when dumping the Firmware Mailbox Log so + * it's possible that we can catch things during a log update and therefore + * see partially corrupted log entries. But i9t's probably Good Enough(tm). + * If we ever decide that we want to make sure that we're dumping a coherent + * log, we'd need to perform locking in the mailbox logging and in + * mboxlog_open() where we'd need to grab the entire mailbox log in one go + * like we do for the Firmware Device Log. But as stated above, meh ... + */ +static int mboxlog_show(struct seq_file *seq, void *v) +{ + struct adapter *adapter = seq->private; + struct mbox_cmd_log *log = adapter->mbox_log; + struct mbox_cmd *entry; + int entry_idx, i; + + if (v == SEQ_START_TOKEN) { + seq_printf(seq, + "%10s %15s %5s %5s %s\n", + "Seq#", "Tstamp", "Atime", "Etime", + "Command/Reply"); + return 0; + } + + entry_idx = log->cursor + ((uintptr_t)v - 2); + if (entry_idx >= log->size) + entry_idx -= log->size; + entry = mbox_cmd_log_entry(log, entry_idx); + + /* skip over unused entries */ + if (entry->timestamp == 0) + return 0; + + seq_printf(seq, "%10u %15llu %5d %5d", + entry->seqno, entry->timestamp, + entry->access, entry->execute); + for (i = 0; i < MBOX_LEN / 8; i++) { + u64 flit = entry->cmd[i]; + u32 hi = (u32)(flit >> 32); + u32 lo = (u32)flit; + + seq_printf(seq, " %08x %08x", hi, lo); + } + seq_puts(seq, "\n"); + return 0; +} + +static inline void *mboxlog_get_idx(struct seq_file *seq, loff_t pos) +{ + struct adapter *adapter = seq->private; + struct mbox_cmd_log *log = adapter->mbox_log; + + return ((pos <= log->size) ? (void *)(uintptr_t)(pos + 1) : NULL); +} + +static void *mboxlog_start(struct seq_file *seq, loff_t *pos) +{ + return *pos ? mboxlog_get_idx(seq, *pos) : SEQ_START_TOKEN; +} + +static void *mboxlog_next(struct seq_file *seq, void *v, loff_t *pos) +{ + ++*pos; + return mboxlog_get_idx(seq, *pos); +} + +static void mboxlog_stop(struct seq_file *seq, void *v) +{ +} + +static const struct seq_operations mboxlog_sops = { + .start = mboxlog_start, + .next = mboxlog_next, + .stop = mboxlog_stop, + .show = mboxlog_show +}; + +DEFINE_SEQ_ATTRIBUTE(mboxlog); +/* + * Show SGE Queue Set information. We display QPL Queues Sets per line. + */ +#define QPL 4 + +static int sge_qinfo_show(struct seq_file *seq, void *v) +{ + struct adapter *adapter = seq->private; + int eth_entries = DIV_ROUND_UP(adapter->sge.ethqsets, QPL); + int qs, r = (uintptr_t)v - 1; + + if (r) + seq_putc(seq, '\n'); + + #define S3(fmt_spec, s, v) \ + do {\ + seq_printf(seq, "%-12s", s); \ + for (qs = 0; qs < n; ++qs) \ + seq_printf(seq, " %16" fmt_spec, v); \ + seq_putc(seq, '\n'); \ + } while (0) + #define S(s, v) S3("s", s, v) + #define T(s, v) S3("u", s, txq[qs].v) + #define R(s, v) S3("u", s, rxq[qs].v) + + if (r < eth_entries) { + const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[r * QPL]; + const struct sge_eth_txq *txq = &adapter->sge.ethtxq[r * QPL]; + int n = min(QPL, adapter->sge.ethqsets - QPL * r); + + S("QType:", "Ethernet"); + S("Interface:", + (rxq[qs].rspq.netdev + ? rxq[qs].rspq.netdev->name + : "N/A")); + S3("d", "Port:", + (rxq[qs].rspq.netdev + ? ((struct port_info *) + netdev_priv(rxq[qs].rspq.netdev))->port_id + : -1)); + T("TxQ ID:", q.abs_id); + T("TxQ size:", q.size); + T("TxQ inuse:", q.in_use); + T("TxQ PIdx:", q.pidx); + T("TxQ CIdx:", q.cidx); + R("RspQ ID:", rspq.abs_id); + R("RspQ size:", rspq.size); + R("RspQE size:", rspq.iqe_len); + S3("u", "Intr delay:", qtimer_val(adapter, &rxq[qs].rspq)); + S3("u", "Intr pktcnt:", + adapter->sge.counter_val[rxq[qs].rspq.pktcnt_idx]); + R("RspQ CIdx:", rspq.cidx); + R("RspQ Gen:", rspq.gen); + R("FL ID:", fl.abs_id); + R("FL size:", fl.size - MIN_FL_RESID); + R("FL avail:", fl.avail); + R("FL PIdx:", fl.pidx); + R("FL CIdx:", fl.cidx); + return 0; + } + + r -= eth_entries; + if (r == 0) { + const struct sge_rspq *evtq = &adapter->sge.fw_evtq; + + seq_printf(seq, "%-12s %16s\n", "QType:", "FW event queue"); + seq_printf(seq, "%-12s %16u\n", "RspQ ID:", evtq->abs_id); + seq_printf(seq, "%-12s %16u\n", "Intr delay:", + qtimer_val(adapter, evtq)); + seq_printf(seq, "%-12s %16u\n", "Intr pktcnt:", + adapter->sge.counter_val[evtq->pktcnt_idx]); + seq_printf(seq, "%-12s %16u\n", "RspQ Cidx:", evtq->cidx); + seq_printf(seq, "%-12s %16u\n", "RspQ Gen:", evtq->gen); + } else if (r == 1) { + const struct sge_rspq *intrq = &adapter->sge.intrq; + + seq_printf(seq, "%-12s %16s\n", "QType:", "Interrupt Queue"); + seq_printf(seq, "%-12s %16u\n", "RspQ ID:", intrq->abs_id); + seq_printf(seq, "%-12s %16u\n", "Intr delay:", + qtimer_val(adapter, intrq)); + seq_printf(seq, "%-12s %16u\n", "Intr pktcnt:", + adapter->sge.counter_val[intrq->pktcnt_idx]); + seq_printf(seq, "%-12s %16u\n", "RspQ Cidx:", intrq->cidx); + seq_printf(seq, "%-12s %16u\n", "RspQ Gen:", intrq->gen); + } + + #undef R + #undef T + #undef S + #undef S3 + + return 0; +} + +/* + * Return the number of "entries" in our "file". We group the multi-Queue + * sections with QPL Queue Sets per "entry". The sections of the output are: + * + * Ethernet RX/TX Queue Sets + * Firmware Event Queue + * Forwarded Interrupt Queue (if in MSI mode) + */ +static int sge_queue_entries(const struct adapter *adapter) +{ + return DIV_ROUND_UP(adapter->sge.ethqsets, QPL) + 1 + + ((adapter->flags & CXGB4VF_USING_MSI) != 0); +} + +static void *sge_queue_start(struct seq_file *seq, loff_t *pos) +{ + int entries = sge_queue_entries(seq->private); + + return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL; +} + +static void sge_queue_stop(struct seq_file *seq, void *v) +{ +} + +static void *sge_queue_next(struct seq_file *seq, void *v, loff_t *pos) +{ + int entries = sge_queue_entries(seq->private); + + ++*pos; + return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL; +} + +static const struct seq_operations sge_qinfo_sops = { + .start = sge_queue_start, + .next = sge_queue_next, + .stop = sge_queue_stop, + .show = sge_qinfo_show +}; + +DEFINE_SEQ_ATTRIBUTE(sge_qinfo); + +/* + * Show SGE Queue Set statistics. We display QPL Queues Sets per line. + */ +#define QPL 4 + +static int sge_qstats_show(struct seq_file *seq, void *v) +{ + struct adapter *adapter = seq->private; + int eth_entries = DIV_ROUND_UP(adapter->sge.ethqsets, QPL); + int qs, r = (uintptr_t)v - 1; + + if (r) + seq_putc(seq, '\n'); + + #define S3(fmt, s, v) \ + do { \ + seq_printf(seq, "%-16s", s); \ + for (qs = 0; qs < n; ++qs) \ + seq_printf(seq, " %8" fmt, v); \ + seq_putc(seq, '\n'); \ + } while (0) + #define S(s, v) S3("s", s, v) + + #define T3(fmt, s, v) S3(fmt, s, txq[qs].v) + #define T(s, v) T3("lu", s, v) + + #define R3(fmt, s, v) S3(fmt, s, rxq[qs].v) + #define R(s, v) R3("lu", s, v) + + if (r < eth_entries) { + const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[r * QPL]; + const struct sge_eth_txq *txq = &adapter->sge.ethtxq[r * QPL]; + int n = min(QPL, adapter->sge.ethqsets - QPL * r); + + S("QType:", "Ethernet"); + S("Interface:", + (rxq[qs].rspq.netdev + ? rxq[qs].rspq.netdev->name + : "N/A")); + R3("u", "RspQNullInts:", rspq.unhandled_irqs); + R("RxPackets:", stats.pkts); + R("RxCSO:", stats.rx_cso); + R("VLANxtract:", stats.vlan_ex); + R("LROmerged:", stats.lro_merged); + R("LROpackets:", stats.lro_pkts); + R("RxDrops:", stats.rx_drops); + T("TSO:", tso); + T("TxCSO:", tx_cso); + T("VLANins:", vlan_ins); + T("TxQFull:", q.stops); + T("TxQRestarts:", q.restarts); + T("TxMapErr:", mapping_err); + R("FLAllocErr:", fl.alloc_failed); + R("FLLrgAlcErr:", fl.large_alloc_failed); + R("FLStarving:", fl.starving); + return 0; + } + + r -= eth_entries; + if (r == 0) { + const struct sge_rspq *evtq = &adapter->sge.fw_evtq; + + seq_printf(seq, "%-8s %16s\n", "QType:", "FW event queue"); + seq_printf(seq, "%-16s %8u\n", "RspQNullInts:", + evtq->unhandled_irqs); + seq_printf(seq, "%-16s %8u\n", "RspQ CIdx:", evtq->cidx); + seq_printf(seq, "%-16s %8u\n", "RspQ Gen:", evtq->gen); + } else if (r == 1) { + const struct sge_rspq *intrq = &adapter->sge.intrq; + + seq_printf(seq, "%-8s %16s\n", "QType:", "Interrupt Queue"); + seq_printf(seq, "%-16s %8u\n", "RspQNullInts:", + intrq->unhandled_irqs); + seq_printf(seq, "%-16s %8u\n", "RspQ CIdx:", intrq->cidx); + seq_printf(seq, "%-16s %8u\n", "RspQ Gen:", intrq->gen); + } + + #undef R + #undef T + #undef S + #undef R3 + #undef T3 + #undef S3 + + return 0; +} + +/* + * Return the number of "entries" in our "file". We group the multi-Queue + * sections with QPL Queue Sets per "entry". The sections of the output are: + * + * Ethernet RX/TX Queue Sets + * Firmware Event Queue + * Forwarded Interrupt Queue (if in MSI mode) + */ +static int sge_qstats_entries(const struct adapter *adapter) +{ + return DIV_ROUND_UP(adapter->sge.ethqsets, QPL) + 1 + + ((adapter->flags & CXGB4VF_USING_MSI) != 0); +} + +static void *sge_qstats_start(struct seq_file *seq, loff_t *pos) +{ + int entries = sge_qstats_entries(seq->private); + + return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL; +} + +static void sge_qstats_stop(struct seq_file *seq, void *v) +{ +} + +static void *sge_qstats_next(struct seq_file *seq, void *v, loff_t *pos) +{ + int entries = sge_qstats_entries(seq->private); + + (*pos)++; + return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL; +} + +static const struct seq_operations sge_qstats_sops = { + .start = sge_qstats_start, + .next = sge_qstats_next, + .stop = sge_qstats_stop, + .show = sge_qstats_show +}; + +DEFINE_SEQ_ATTRIBUTE(sge_qstats); + +/* + * Show PCI-E SR-IOV Virtual Function Resource Limits. + */ +static int resources_show(struct seq_file *seq, void *v) +{ + struct adapter *adapter = seq->private; + struct vf_resources *vfres = &adapter->params.vfres; + + #define S(desc, fmt, var) \ + seq_printf(seq, "%-60s " fmt "\n", \ + desc " (" #var "):", vfres->var) + + S("Virtual Interfaces", "%d", nvi); + S("Egress Queues", "%d", neq); + S("Ethernet Control", "%d", nethctrl); + S("Ingress Queues/w Free Lists/Interrupts", "%d", niqflint); + S("Ingress Queues", "%d", niq); + S("Traffic Class", "%d", tc); + S("Port Access Rights Mask", "%#x", pmask); + S("MAC Address Filters", "%d", nexactf); + S("Firmware Command Read Capabilities", "%#x", r_caps); + S("Firmware Command Write/Execute Capabilities", "%#x", wx_caps); + + #undef S + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(resources); + +/* + * Show Virtual Interfaces. + */ +static int interfaces_show(struct seq_file *seq, void *v) +{ + if (v == SEQ_START_TOKEN) { + seq_puts(seq, "Interface Port VIID\n"); + } else { + struct adapter *adapter = seq->private; + int pidx = (uintptr_t)v - 2; + struct net_device *dev = adapter->port[pidx]; + struct port_info *pi = netdev_priv(dev); + + seq_printf(seq, "%9s %4d %#5x\n", + dev->name, pi->port_id, pi->viid); + } + return 0; +} + +static inline void *interfaces_get_idx(struct adapter *adapter, loff_t pos) +{ + return pos <= adapter->params.nports + ? (void *)(uintptr_t)(pos + 1) + : NULL; +} + +static void *interfaces_start(struct seq_file *seq, loff_t *pos) +{ + return *pos + ? interfaces_get_idx(seq->private, *pos) + : SEQ_START_TOKEN; +} + +static void *interfaces_next(struct seq_file *seq, void *v, loff_t *pos) +{ + (*pos)++; + return interfaces_get_idx(seq->private, *pos); +} + +static void interfaces_stop(struct seq_file *seq, void *v) +{ +} + +static const struct seq_operations interfaces_sops = { + .start = interfaces_start, + .next = interfaces_next, + .stop = interfaces_stop, + .show = interfaces_show +}; + +DEFINE_SEQ_ATTRIBUTE(interfaces); + +/* + * /sys/kernel/debugfs/cxgb4vf/ files list. + */ +struct cxgb4vf_debugfs_entry { + const char *name; /* name of debugfs node */ + umode_t mode; /* file system mode */ + const struct file_operations *fops; +}; + +static struct cxgb4vf_debugfs_entry debugfs_files[] = { + { "mboxlog", 0444, &mboxlog_fops }, + { "sge_qinfo", 0444, &sge_qinfo_fops }, + { "sge_qstats", 0444, &sge_qstats_fops }, + { "resources", 0444, &resources_fops }, + { "interfaces", 0444, &interfaces_fops }, +}; + +/* + * Module and device initialization and cleanup code. + * ================================================== + */ + +/* + * Set up out /sys/kernel/debug/cxgb4vf sub-nodes. We assume that the + * directory (debugfs_root) has already been set up. + */ +static int setup_debugfs(struct adapter *adapter) +{ + int i; + + BUG_ON(IS_ERR_OR_NULL(adapter->debugfs_root)); + + /* + * Debugfs support is best effort. + */ + for (i = 0; i < ARRAY_SIZE(debugfs_files); i++) + debugfs_create_file(debugfs_files[i].name, + debugfs_files[i].mode, + adapter->debugfs_root, adapter, + debugfs_files[i].fops); + + return 0; +} + +/* + * Tear down the /sys/kernel/debug/cxgb4vf sub-nodes created above. We leave + * it to our caller to tear down the directory (debugfs_root). + */ +static void cleanup_debugfs(struct adapter *adapter) +{ + BUG_ON(IS_ERR_OR_NULL(adapter->debugfs_root)); + + /* + * Unlike our sister routine cleanup_proc(), we don't need to remove + * individual entries because a call will be made to + * debugfs_remove_recursive(). We just need to clean up any ancillary + * persistent state. + */ + /* nothing to do */ +} + +/* Figure out how many Ports and Queue Sets we can support. This depends on + * knowing our Virtual Function Resources and may be called a second time if + * we fall back from MSI-X to MSI Interrupt Mode. + */ +static void size_nports_qsets(struct adapter *adapter) +{ + struct vf_resources *vfres = &adapter->params.vfres; + unsigned int ethqsets, pmask_nports; + + /* The number of "ports" which we support is equal to the number of + * Virtual Interfaces with which we've been provisioned. + */ + adapter->params.nports = vfres->nvi; + if (adapter->params.nports > MAX_NPORTS) { + dev_warn(adapter->pdev_dev, "only using %d of %d maximum" + " allowed virtual interfaces\n", MAX_NPORTS, + adapter->params.nports); + adapter->params.nports = MAX_NPORTS; + } + + /* We may have been provisioned with more VIs than the number of + * ports we're allowed to access (our Port Access Rights Mask). + * This is obviously a configuration conflict but we don't want to + * crash the kernel or anything silly just because of that. + */ + pmask_nports = hweight32(adapter->params.vfres.pmask); + if (pmask_nports < adapter->params.nports) { + dev_warn(adapter->pdev_dev, "only using %d of %d provisioned" + " virtual interfaces; limited by Port Access Rights" + " mask %#x\n", pmask_nports, adapter->params.nports, + adapter->params.vfres.pmask); + adapter->params.nports = pmask_nports; + } + + /* We need to reserve an Ingress Queue for the Asynchronous Firmware + * Event Queue. And if we're using MSI Interrupts, we'll also need to + * reserve an Ingress Queue for a Forwarded Interrupts. + * + * The rest of the FL/Intr-capable ingress queues will be matched up + * one-for-one with Ethernet/Control egress queues in order to form + * "Queue Sets" which will be aportioned between the "ports". For + * each Queue Set, we'll need the ability to allocate two Egress + * Contexts -- one for the Ingress Queue Free List and one for the TX + * Ethernet Queue. + * + * Note that even if we're currently configured to use MSI-X + * Interrupts (module variable msi == MSI_MSIX) we may get downgraded + * to MSI Interrupts if we can't get enough MSI-X Interrupts. If that + * happens we'll need to adjust things later. + */ + ethqsets = vfres->niqflint - 1 - (msi == MSI_MSI); + if (vfres->nethctrl != ethqsets) + ethqsets = min(vfres->nethctrl, ethqsets); + if (vfres->neq < ethqsets*2) + ethqsets = vfres->neq/2; + if (ethqsets > MAX_ETH_QSETS) + ethqsets = MAX_ETH_QSETS; + adapter->sge.max_ethqsets = ethqsets; + + if (adapter->sge.max_ethqsets < adapter->params.nports) { + dev_warn(adapter->pdev_dev, "only using %d of %d available" + " virtual interfaces (too few Queue Sets)\n", + adapter->sge.max_ethqsets, adapter->params.nports); + adapter->params.nports = adapter->sge.max_ethqsets; + } +} + +/* + * Perform early "adapter" initialization. This is where we discover what + * adapter parameters we're going to be using and initialize basic adapter + * hardware support. + */ +static int adap_init0(struct adapter *adapter) +{ + struct sge_params *sge_params = &adapter->params.sge; + struct sge *s = &adapter->sge; + int err; + u32 param, val = 0; + + /* + * Some environments do not properly handle PCIE FLRs -- e.g. in Linux + * 2.6.31 and later we can't call pci_reset_function() in order to + * issue an FLR because of a self- deadlock on the device semaphore. + * Meanwhile, the OS infrastructure doesn't issue FLRs in all the + * cases where they're needed -- for instance, some versions of KVM + * fail to reset "Assigned Devices" when the VM reboots. Therefore we + * use the firmware based reset in order to reset any per function + * state. + */ + err = t4vf_fw_reset(adapter); + if (err < 0) { + dev_err(adapter->pdev_dev, "FW reset failed: err=%d\n", err); + return err; + } + + /* + * Grab basic operational parameters. These will predominantly have + * been set up by the Physical Function Driver or will be hard coded + * into the adapter. We just have to live with them ... Note that + * we _must_ get our VPD parameters before our SGE parameters because + * we need to know the adapter's core clock from the VPD in order to + * properly decode the SGE Timer Values. + */ + err = t4vf_get_dev_params(adapter); + if (err) { + dev_err(adapter->pdev_dev, "unable to retrieve adapter" + " device parameters: err=%d\n", err); + return err; + } + err = t4vf_get_vpd_params(adapter); + if (err) { + dev_err(adapter->pdev_dev, "unable to retrieve adapter" + " VPD parameters: err=%d\n", err); + return err; + } + err = t4vf_get_sge_params(adapter); + if (err) { + dev_err(adapter->pdev_dev, "unable to retrieve adapter" + " SGE parameters: err=%d\n", err); + return err; + } + err = t4vf_get_rss_glb_config(adapter); + if (err) { + dev_err(adapter->pdev_dev, "unable to retrieve adapter" + " RSS parameters: err=%d\n", err); + return err; + } + if (adapter->params.rss.mode != + FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) { + dev_err(adapter->pdev_dev, "unable to operate with global RSS" + " mode %d\n", adapter->params.rss.mode); + return -EINVAL; + } + err = t4vf_sge_init(adapter); + if (err) { + dev_err(adapter->pdev_dev, "unable to use adapter parameters:" + " err=%d\n", err); + return err; + } + + /* If we're running on newer firmware, let it know that we're + * prepared to deal with encapsulated CPL messages. Older + * firmware won't understand this and we'll just get + * unencapsulated messages ... + */ + param = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP); + val = 1; + (void) t4vf_set_params(adapter, 1, ¶m, &val); + + /* + * Retrieve our RX interrupt holdoff timer values and counter + * threshold values from the SGE parameters. + */ + s->timer_val[0] = core_ticks_to_us(adapter, + TIMERVALUE0_G(sge_params->sge_timer_value_0_and_1)); + s->timer_val[1] = core_ticks_to_us(adapter, + TIMERVALUE1_G(sge_params->sge_timer_value_0_and_1)); + s->timer_val[2] = core_ticks_to_us(adapter, + TIMERVALUE0_G(sge_params->sge_timer_value_2_and_3)); + s->timer_val[3] = core_ticks_to_us(adapter, + TIMERVALUE1_G(sge_params->sge_timer_value_2_and_3)); + s->timer_val[4] = core_ticks_to_us(adapter, + TIMERVALUE0_G(sge_params->sge_timer_value_4_and_5)); + s->timer_val[5] = core_ticks_to_us(adapter, + TIMERVALUE1_G(sge_params->sge_timer_value_4_and_5)); + + s->counter_val[0] = THRESHOLD_0_G(sge_params->sge_ingress_rx_threshold); + s->counter_val[1] = THRESHOLD_1_G(sge_params->sge_ingress_rx_threshold); + s->counter_val[2] = THRESHOLD_2_G(sge_params->sge_ingress_rx_threshold); + s->counter_val[3] = THRESHOLD_3_G(sge_params->sge_ingress_rx_threshold); + + /* + * Grab our Virtual Interface resource allocation, extract the + * features that we're interested in and do a bit of sanity testing on + * what we discover. + */ + err = t4vf_get_vfres(adapter); + if (err) { + dev_err(adapter->pdev_dev, "unable to get virtual interface" + " resources: err=%d\n", err); + return err; + } + + /* Check for various parameter sanity issues */ + if (adapter->params.vfres.pmask == 0) { + dev_err(adapter->pdev_dev, "no port access configured\n" + "usable!\n"); + return -EINVAL; + } + if (adapter->params.vfres.nvi == 0) { + dev_err(adapter->pdev_dev, "no virtual interfaces configured/" + "usable!\n"); + return -EINVAL; + } + + /* Initialize nports and max_ethqsets now that we have our Virtual + * Function Resources. + */ + size_nports_qsets(adapter); + + adapter->flags |= CXGB4VF_FW_OK; + return 0; +} + +static inline void init_rspq(struct sge_rspq *rspq, u8 timer_idx, + u8 pkt_cnt_idx, unsigned int size, + unsigned int iqe_size) +{ + rspq->intr_params = (QINTR_TIMER_IDX_V(timer_idx) | + (pkt_cnt_idx < SGE_NCOUNTERS ? + QINTR_CNT_EN_F : 0)); + rspq->pktcnt_idx = (pkt_cnt_idx < SGE_NCOUNTERS + ? pkt_cnt_idx + : 0); + rspq->iqe_len = iqe_size; + rspq->size = size; +} + +/* + * Perform default configuration of DMA queues depending on the number and + * type of ports we found and the number of available CPUs. Most settings can + * be modified by the admin via ethtool and cxgbtool prior to the adapter + * being brought up for the first time. + */ +static void cfg_queues(struct adapter *adapter) +{ + struct sge *s = &adapter->sge; + int q10g, n10g, qidx, pidx, qs; + size_t iqe_size; + + /* + * We should not be called till we know how many Queue Sets we can + * support. In particular, this means that we need to know what kind + * of interrupts we'll be using ... + */ + BUG_ON((adapter->flags & + (CXGB4VF_USING_MSIX | CXGB4VF_USING_MSI)) == 0); + + /* + * Count the number of 10GbE Virtual Interfaces that we have. + */ + n10g = 0; + for_each_port(adapter, pidx) + n10g += is_x_10g_port(&adap2pinfo(adapter, pidx)->link_cfg); + + /* + * We default to 1 queue per non-10G port and up to # of cores queues + * per 10G port. + */ + if (n10g == 0) + q10g = 0; + else { + int n1g = (adapter->params.nports - n10g); + q10g = (adapter->sge.max_ethqsets - n1g) / n10g; + if (q10g > num_online_cpus()) + q10g = num_online_cpus(); + } + + /* + * Allocate the "Queue Sets" to the various Virtual Interfaces. + * The layout will be established in setup_sge_queues() when the + * adapter is brough up for the first time. + */ + qidx = 0; + for_each_port(adapter, pidx) { + struct port_info *pi = adap2pinfo(adapter, pidx); + + pi->first_qset = qidx; + pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1; + qidx += pi->nqsets; + } + s->ethqsets = qidx; + + /* + * The Ingress Queue Entry Size for our various Response Queues needs + * to be big enough to accommodate the largest message we can receive + * from the chip/firmware; which is 64 bytes ... + */ + iqe_size = 64; + + /* + * Set up default Queue Set parameters ... Start off with the + * shortest interrupt holdoff timer. + */ + for (qs = 0; qs < s->max_ethqsets; qs++) { + struct sge_eth_rxq *rxq = &s->ethrxq[qs]; + struct sge_eth_txq *txq = &s->ethtxq[qs]; + + init_rspq(&rxq->rspq, 0, 0, 1024, iqe_size); + rxq->fl.size = 72; + txq->q.size = 1024; + } + + /* + * The firmware event queue is used for link state changes and + * notifications of TX DMA completions. + */ + init_rspq(&s->fw_evtq, SGE_TIMER_RSTRT_CNTR, 0, 512, iqe_size); + + /* + * The forwarded interrupt queue is used when we're in MSI interrupt + * mode. In this mode all interrupts associated with RX queues will + * be forwarded to a single queue which we'll associate with our MSI + * interrupt vector. The messages dropped in the forwarded interrupt + * queue will indicate which ingress queue needs servicing ... This + * queue needs to be large enough to accommodate all of the ingress + * queues which are forwarding their interrupt (+1 to prevent the PIDX + * from equalling the CIDX if every ingress queue has an outstanding + * interrupt). The queue doesn't need to be any larger because no + * ingress queue will ever have more than one outstanding interrupt at + * any time ... + */ + init_rspq(&s->intrq, SGE_TIMER_RSTRT_CNTR, 0, MSIX_ENTRIES + 1, + iqe_size); +} + +/* + * Reduce the number of Ethernet queues across all ports to at most n. + * n provides at least one queue per port. + */ +static void reduce_ethqs(struct adapter *adapter, int n) +{ + int i; + struct port_info *pi; + + /* + * While we have too many active Ether Queue Sets, interate across the + * "ports" and reduce their individual Queue Set allocations. + */ + BUG_ON(n < adapter->params.nports); + while (n < adapter->sge.ethqsets) + for_each_port(adapter, i) { + pi = adap2pinfo(adapter, i); + if (pi->nqsets > 1) { + pi->nqsets--; + adapter->sge.ethqsets--; + if (adapter->sge.ethqsets <= n) + break; + } + } + + /* + * Reassign the starting Queue Sets for each of the "ports" ... + */ + n = 0; + for_each_port(adapter, i) { + pi = adap2pinfo(adapter, i); + pi->first_qset = n; + n += pi->nqsets; + } +} + +/* + * We need to grab enough MSI-X vectors to cover our interrupt needs. Ideally + * we get a separate MSI-X vector for every "Queue Set" plus any extras we + * need. Minimally we need one for every Virtual Interface plus those needed + * for our "extras". Note that this process may lower the maximum number of + * allowed Queue Sets ... + */ +static int enable_msix(struct adapter *adapter) +{ + int i, want, need, nqsets; + struct msix_entry entries[MSIX_ENTRIES]; + struct sge *s = &adapter->sge; + + for (i = 0; i < MSIX_ENTRIES; ++i) + entries[i].entry = i; + + /* + * We _want_ enough MSI-X interrupts to cover all of our "Queue Sets" + * plus those needed for our "extras" (for example, the firmware + * message queue). We _need_ at least one "Queue Set" per Virtual + * Interface plus those needed for our "extras". So now we get to see + * if the song is right ... + */ + want = s->max_ethqsets + MSIX_EXTRAS; + need = adapter->params.nports + MSIX_EXTRAS; + + want = pci_enable_msix_range(adapter->pdev, entries, need, want); + if (want < 0) + return want; + + nqsets = want - MSIX_EXTRAS; + if (nqsets < s->max_ethqsets) { + dev_warn(adapter->pdev_dev, "only enough MSI-X vectors" + " for %d Queue Sets\n", nqsets); + s->max_ethqsets = nqsets; + if (nqsets < s->ethqsets) + reduce_ethqs(adapter, nqsets); + } + for (i = 0; i < want; ++i) + adapter->msix_info[i].vec = entries[i].vector; + + return 0; +} + +static const struct net_device_ops cxgb4vf_netdev_ops = { + .ndo_open = cxgb4vf_open, + .ndo_stop = cxgb4vf_stop, + .ndo_start_xmit = t4vf_eth_xmit, + .ndo_get_stats = cxgb4vf_get_stats, + .ndo_set_rx_mode = cxgb4vf_set_rxmode, + .ndo_set_mac_address = cxgb4vf_set_mac_addr, + .ndo_validate_addr = eth_validate_addr, + .ndo_eth_ioctl = cxgb4vf_do_ioctl, + .ndo_change_mtu = cxgb4vf_change_mtu, + .ndo_fix_features = cxgb4vf_fix_features, + .ndo_set_features = cxgb4vf_set_features, +#ifdef CONFIG_NET_POLL_CONTROLLER + .ndo_poll_controller = cxgb4vf_poll_controller, +#endif +}; + +/** + * cxgb4vf_get_port_mask - Get port mask for the VF based on mac + * address stored on the adapter + * @adapter: The adapter + * + * Find the port mask for the VF based on the index of mac + * address stored in the adapter. If no mac address is stored on + * the adapter for the VF, use the port mask received from the + * firmware. + */ +static unsigned int cxgb4vf_get_port_mask(struct adapter *adapter) +{ + unsigned int naddr = 1, pidx = 0; + unsigned int pmask, rmask = 0; + u8 mac[ETH_ALEN]; + int err; + + pmask = adapter->params.vfres.pmask; + while (pmask) { + if (pmask & 1) { + err = t4vf_get_vf_mac_acl(adapter, pidx, &naddr, mac); + if (!err && !is_zero_ether_addr(mac)) + rmask |= (1 << pidx); + } + pmask >>= 1; + pidx++; + } + if (!rmask) + rmask = adapter->params.vfres.pmask; + + return rmask; +} + +/* + * "Probe" a device: initialize a device and construct all kernel and driver + * state needed to manage the device. This routine is called "init_one" in + * the PF Driver ... + */ +static int cxgb4vf_pci_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct adapter *adapter; + struct net_device *netdev; + struct port_info *pi; + unsigned int pmask; + int err, pidx; + + /* + * Initialize generic PCI device state. + */ + err = pci_enable_device(pdev); + if (err) + return dev_err_probe(&pdev->dev, err, "cannot enable PCI device\n"); + + /* + * Reserve PCI resources for the device. If we can't get them some + * other driver may have already claimed the device ... + */ + err = pci_request_regions(pdev, KBUILD_MODNAME); + if (err) { + dev_err(&pdev->dev, "cannot obtain PCI resources\n"); + goto err_disable_device; + } + + /* + * Set up our DMA mask + */ + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_err(&pdev->dev, "no usable DMA configuration\n"); + goto err_release_regions; + } + + /* + * Enable bus mastering for the device ... + */ + pci_set_master(pdev); + + /* + * Allocate our adapter data structure and attach it to the device. + */ + adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); + if (!adapter) { + err = -ENOMEM; + goto err_release_regions; + } + pci_set_drvdata(pdev, adapter); + adapter->pdev = pdev; + adapter->pdev_dev = &pdev->dev; + + adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) + + (sizeof(struct mbox_cmd) * + T4VF_OS_LOG_MBOX_CMDS), + GFP_KERNEL); + if (!adapter->mbox_log) { + err = -ENOMEM; + goto err_free_adapter; + } + adapter->mbox_log->size = T4VF_OS_LOG_MBOX_CMDS; + + /* + * Initialize SMP data synchronization resources. + */ + spin_lock_init(&adapter->stats_lock); + spin_lock_init(&adapter->mbox_lock); + INIT_LIST_HEAD(&adapter->mlist.list); + + /* + * Map our I/O registers in BAR0. + */ + adapter->regs = pci_ioremap_bar(pdev, 0); + if (!adapter->regs) { + dev_err(&pdev->dev, "cannot map device registers\n"); + err = -ENOMEM; + goto err_free_adapter; + } + + /* Wait for the device to become ready before proceeding ... + */ + err = t4vf_prep_adapter(adapter); + if (err) { + dev_err(adapter->pdev_dev, "device didn't become ready:" + " err=%d\n", err); + goto err_unmap_bar0; + } + + /* For T5 and later we want to use the new BAR-based User Doorbells, + * so we need to map BAR2 here ... + */ + if (!is_t4(adapter->params.chip)) { + adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2), + pci_resource_len(pdev, 2)); + if (!adapter->bar2) { + dev_err(adapter->pdev_dev, "cannot map BAR2 doorbells\n"); + err = -ENOMEM; + goto err_unmap_bar0; + } + } + /* + * Initialize adapter level features. + */ + adapter->name = pci_name(pdev); + adapter->msg_enable = DFLT_MSG_ENABLE; + + /* If possible, we use PCIe Relaxed Ordering Attribute to deliver + * Ingress Packet Data to Free List Buffers in order to allow for + * chipset performance optimizations between the Root Complex and + * Memory Controllers. (Messages to the associated Ingress Queue + * notifying new Packet Placement in the Free Lists Buffers will be + * send without the Relaxed Ordering Attribute thus guaranteeing that + * all preceding PCIe Transaction Layer Packets will be processed + * first.) But some Root Complexes have various issues with Upstream + * Transaction Layer Packets with the Relaxed Ordering Attribute set. + * The PCIe devices which under the Root Complexes will be cleared the + * Relaxed Ordering bit in the configuration space, So we check our + * PCIe configuration space to see if it's flagged with advice against + * using Relaxed Ordering. + */ + if (!pcie_relaxed_ordering_enabled(pdev)) + adapter->flags |= CXGB4VF_ROOT_NO_RELAXED_ORDERING; + + err = adap_init0(adapter); + if (err) + dev_err(&pdev->dev, + "Adapter initialization failed, error %d. Continuing in debug mode\n", + err); + + /* Initialize hash mac addr list */ + INIT_LIST_HEAD(&adapter->mac_hlist); + + /* + * Allocate our "adapter ports" and stitch everything together. + */ + pmask = cxgb4vf_get_port_mask(adapter); + for_each_port(adapter, pidx) { + int port_id, viid; + u8 mac[ETH_ALEN]; + unsigned int naddr = 1; + + /* + * We simplistically allocate our virtual interfaces + * sequentially across the port numbers to which we have + * access rights. This should be configurable in some manner + * ... + */ + if (pmask == 0) + break; + port_id = ffs(pmask) - 1; + pmask &= ~(1 << port_id); + + /* + * Allocate our network device and stitch things together. + */ + netdev = alloc_etherdev_mq(sizeof(struct port_info), + MAX_PORT_QSETS); + if (netdev == NULL) { + err = -ENOMEM; + goto err_free_dev; + } + adapter->port[pidx] = netdev; + SET_NETDEV_DEV(netdev, &pdev->dev); + pi = netdev_priv(netdev); + pi->adapter = adapter; + pi->pidx = pidx; + pi->port_id = port_id; + + /* + * Initialize the starting state of our "port" and register + * it. + */ + pi->xact_addr_filt = -1; + netdev->irq = pdev->irq; + + netdev->hw_features = NETIF_F_SG | TSO_FLAGS | NETIF_F_GRO | + NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | + NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; + netdev->features = netdev->hw_features | NETIF_F_HIGHDMA; + netdev->vlan_features = netdev->features & VLAN_FEAT; + + netdev->priv_flags |= IFF_UNICAST_FLT; + netdev->min_mtu = 81; + netdev->max_mtu = ETH_MAX_MTU; + + netdev->netdev_ops = &cxgb4vf_netdev_ops; + netdev->ethtool_ops = &cxgb4vf_ethtool_ops; + netdev->dev_port = pi->port_id; + + /* + * If we haven't been able to contact the firmware, there's + * nothing else we can do for this "port" ... + */ + if (!(adapter->flags & CXGB4VF_FW_OK)) + continue; + + viid = t4vf_alloc_vi(adapter, port_id); + if (viid < 0) { + dev_err(&pdev->dev, + "cannot allocate VI for port %d: err=%d\n", + port_id, viid); + err = viid; + goto err_free_dev; + } + pi->viid = viid; + + /* + * Initialize the hardware/software state for the port. + */ + err = t4vf_port_init(adapter, pidx); + if (err) { + dev_err(&pdev->dev, "cannot initialize port %d\n", + pidx); + goto err_free_dev; + } + + err = t4vf_get_vf_mac_acl(adapter, port_id, &naddr, mac); + if (err) { + dev_err(&pdev->dev, + "unable to determine MAC ACL address, " + "continuing anyway.. (status %d)\n", err); + } else if (naddr && adapter->params.vfres.nvi == 1) { + struct sockaddr addr; + + ether_addr_copy(addr.sa_data, mac); + err = cxgb4vf_set_mac_addr(netdev, &addr); + if (err) { + dev_err(&pdev->dev, + "unable to set MAC address %pM\n", + mac); + goto err_free_dev; + } + dev_info(&pdev->dev, + "Using assigned MAC ACL: %pM\n", mac); + } + } + + /* See what interrupts we'll be using. If we've been configured to + * use MSI-X interrupts, try to enable them but fall back to using + * MSI interrupts if we can't enable MSI-X interrupts. If we can't + * get MSI interrupts we bail with the error. + */ + if (msi == MSI_MSIX && enable_msix(adapter) == 0) + adapter->flags |= CXGB4VF_USING_MSIX; + else { + if (msi == MSI_MSIX) { + dev_info(adapter->pdev_dev, + "Unable to use MSI-X Interrupts; falling " + "back to MSI Interrupts\n"); + + /* We're going to need a Forwarded Interrupt Queue so + * that may cut into how many Queue Sets we can + * support. + */ + msi = MSI_MSI; + size_nports_qsets(adapter); + } + err = pci_enable_msi(pdev); + if (err) { + dev_err(&pdev->dev, "Unable to allocate MSI Interrupts;" + " err=%d\n", err); + goto err_free_dev; + } + adapter->flags |= CXGB4VF_USING_MSI; + } + + /* Now that we know how many "ports" we have and what interrupt + * mechanism we're going to use, we can configure our queue resources. + */ + cfg_queues(adapter); + + /* + * The "card" is now ready to go. If any errors occur during device + * registration we do not fail the whole "card" but rather proceed + * only with the ports we manage to register successfully. However we + * must register at least one net device. + */ + for_each_port(adapter, pidx) { + struct port_info *pi = netdev_priv(adapter->port[pidx]); + netdev = adapter->port[pidx]; + if (netdev == NULL) + continue; + + netif_set_real_num_tx_queues(netdev, pi->nqsets); + netif_set_real_num_rx_queues(netdev, pi->nqsets); + + err = register_netdev(netdev); + if (err) { + dev_warn(&pdev->dev, "cannot register net device %s," + " skipping\n", netdev->name); + continue; + } + + netif_carrier_off(netdev); + set_bit(pidx, &adapter->registered_device_map); + } + if (adapter->registered_device_map == 0) { + dev_err(&pdev->dev, "could not register any net devices\n"); + err = -EINVAL; + goto err_disable_interrupts; + } + + /* + * Set up our debugfs entries. + */ + if (!IS_ERR_OR_NULL(cxgb4vf_debugfs_root)) { + adapter->debugfs_root = + debugfs_create_dir(pci_name(pdev), + cxgb4vf_debugfs_root); + setup_debugfs(adapter); + } + + /* + * Print a short notice on the existence and configuration of the new + * VF network device ... + */ + for_each_port(adapter, pidx) { + dev_info(adapter->pdev_dev, "%s: Chelsio VF NIC PCIe %s\n", + adapter->port[pidx]->name, + (adapter->flags & CXGB4VF_USING_MSIX) ? "MSI-X" : + (adapter->flags & CXGB4VF_USING_MSI) ? "MSI" : ""); + } + + /* + * Return success! + */ + return 0; + + /* + * Error recovery and exit code. Unwind state that's been created + * so far and return the error. + */ +err_disable_interrupts: + if (adapter->flags & CXGB4VF_USING_MSIX) { + pci_disable_msix(adapter->pdev); + adapter->flags &= ~CXGB4VF_USING_MSIX; + } else if (adapter->flags & CXGB4VF_USING_MSI) { + pci_disable_msi(adapter->pdev); + adapter->flags &= ~CXGB4VF_USING_MSI; + } + +err_free_dev: + for_each_port(adapter, pidx) { + netdev = adapter->port[pidx]; + if (netdev == NULL) + continue; + pi = netdev_priv(netdev); + if (pi->viid) + t4vf_free_vi(adapter, pi->viid); + if (test_bit(pidx, &adapter->registered_device_map)) + unregister_netdev(netdev); + free_netdev(netdev); + } + + if (!is_t4(adapter->params.chip)) + iounmap(adapter->bar2); + +err_unmap_bar0: + iounmap(adapter->regs); + +err_free_adapter: + kfree(adapter->mbox_log); + kfree(adapter); + +err_release_regions: + pci_release_regions(pdev); + pci_clear_master(pdev); + +err_disable_device: + pci_disable_device(pdev); + + return err; +} + +/* + * "Remove" a device: tear down all kernel and driver state created in the + * "probe" routine and quiesce the device (disable interrupts, etc.). (Note + * that this is called "remove_one" in the PF Driver.) + */ +static void cxgb4vf_pci_remove(struct pci_dev *pdev) +{ + struct adapter *adapter = pci_get_drvdata(pdev); + struct hash_mac_addr *entry, *tmp; + + /* + * Tear down driver state associated with device. + */ + if (adapter) { + int pidx; + + /* + * Stop all of our activity. Unregister network port, + * disable interrupts, etc. + */ + for_each_port(adapter, pidx) + if (test_bit(pidx, &adapter->registered_device_map)) + unregister_netdev(adapter->port[pidx]); + t4vf_sge_stop(adapter); + if (adapter->flags & CXGB4VF_USING_MSIX) { + pci_disable_msix(adapter->pdev); + adapter->flags &= ~CXGB4VF_USING_MSIX; + } else if (adapter->flags & CXGB4VF_USING_MSI) { + pci_disable_msi(adapter->pdev); + adapter->flags &= ~CXGB4VF_USING_MSI; + } + + /* + * Tear down our debugfs entries. + */ + if (!IS_ERR_OR_NULL(adapter->debugfs_root)) { + cleanup_debugfs(adapter); + debugfs_remove_recursive(adapter->debugfs_root); + } + + /* + * Free all of the various resources which we've acquired ... + */ + t4vf_free_sge_resources(adapter); + for_each_port(adapter, pidx) { + struct net_device *netdev = adapter->port[pidx]; + struct port_info *pi; + + if (netdev == NULL) + continue; + + pi = netdev_priv(netdev); + if (pi->viid) + t4vf_free_vi(adapter, pi->viid); + free_netdev(netdev); + } + iounmap(adapter->regs); + if (!is_t4(adapter->params.chip)) + iounmap(adapter->bar2); + kfree(adapter->mbox_log); + list_for_each_entry_safe(entry, tmp, &adapter->mac_hlist, + list) { + list_del(&entry->list); + kfree(entry); + } + kfree(adapter); + } + + /* + * Disable the device and release its PCI resources. + */ + pci_disable_device(pdev); + pci_clear_master(pdev); + pci_release_regions(pdev); +} + +/* + * "Shutdown" quiesce the device, stopping Ingress Packet and Interrupt + * delivery. + */ +static void cxgb4vf_pci_shutdown(struct pci_dev *pdev) +{ + struct adapter *adapter; + int pidx; + + adapter = pci_get_drvdata(pdev); + if (!adapter) + return; + + /* Disable all Virtual Interfaces. This will shut down the + * delivery of all ingress packets into the chip for these + * Virtual Interfaces. + */ + for_each_port(adapter, pidx) + if (test_bit(pidx, &adapter->registered_device_map)) + unregister_netdev(adapter->port[pidx]); + + /* Free up all Queues which will prevent further DMA and + * Interrupts allowing various internal pathways to drain. + */ + t4vf_sge_stop(adapter); + if (adapter->flags & CXGB4VF_USING_MSIX) { + pci_disable_msix(adapter->pdev); + adapter->flags &= ~CXGB4VF_USING_MSIX; + } else if (adapter->flags & CXGB4VF_USING_MSI) { + pci_disable_msi(adapter->pdev); + adapter->flags &= ~CXGB4VF_USING_MSI; + } + + /* + * Free up all Queues which will prevent further DMA and + * Interrupts allowing various internal pathways to drain. + */ + t4vf_free_sge_resources(adapter); + pci_set_drvdata(pdev, NULL); +} + +/* Macros needed to support the PCI Device ID Table ... + */ +#define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ + static const struct pci_device_id cxgb4vf_pci_tbl[] = { +#define CH_PCI_DEVICE_ID_FUNCTION 0x8 + +#define CH_PCI_ID_TABLE_ENTRY(devid) \ + { PCI_VDEVICE(CHELSIO, (devid)), 0 } + +#define CH_PCI_DEVICE_ID_TABLE_DEFINE_END { 0, } } + +#include "../cxgb4/t4_pci_id_tbl.h" + +MODULE_DESCRIPTION(DRV_DESC); +MODULE_AUTHOR("Chelsio Communications"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DEVICE_TABLE(pci, cxgb4vf_pci_tbl); + +static struct pci_driver cxgb4vf_driver = { + .name = KBUILD_MODNAME, + .id_table = cxgb4vf_pci_tbl, + .probe = cxgb4vf_pci_probe, + .remove = cxgb4vf_pci_remove, + .shutdown = cxgb4vf_pci_shutdown, +}; + +/* + * Initialize global driver state. + */ +static int __init cxgb4vf_module_init(void) +{ + int ret; + + /* + * Vet our module parameters. + */ + if (msi != MSI_MSIX && msi != MSI_MSI) { + pr_warn("bad module parameter msi=%d; must be %d (MSI-X or MSI) or %d (MSI)\n", + msi, MSI_MSIX, MSI_MSI); + return -EINVAL; + } + + /* Debugfs support is optional, debugfs will warn if this fails */ + cxgb4vf_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); + + ret = pci_register_driver(&cxgb4vf_driver); + if (ret < 0) + debugfs_remove(cxgb4vf_debugfs_root); + return ret; +} + +/* + * Tear down global driver state. + */ +static void __exit cxgb4vf_module_exit(void) +{ + pci_unregister_driver(&cxgb4vf_driver); + debugfs_remove(cxgb4vf_debugfs_root); +} + +module_init(cxgb4vf_module_init); +module_exit(cxgb4vf_module_exit); diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c new file mode 100644 index 000000000..2d0cf76fb --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c @@ -0,0 +1,2706 @@ +/* + * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet + * driver for Linux. + * + * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/skbuff.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> +#include <linux/if_vlan.h> +#include <linux/ip.h> +#include <net/ipv6.h> +#include <net/tcp.h> +#include <linux/dma-mapping.h> +#include <linux/prefetch.h> + +#include "t4vf_common.h" +#include "t4vf_defs.h" + +#include "../cxgb4/t4_regs.h" +#include "../cxgb4/t4_values.h" +#include "../cxgb4/t4fw_api.h" +#include "../cxgb4/t4_msg.h" + +/* + * Constants ... + */ +enum { + /* + * Egress Queue sizes, producer and consumer indices are all in units + * of Egress Context Units bytes. Note that as far as the hardware is + * concerned, the free list is an Egress Queue (the host produces free + * buffers which the hardware consumes) and free list entries are + * 64-bit PCI DMA addresses. + */ + EQ_UNIT = SGE_EQ_IDXSIZE, + FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64), + TXD_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64), + + /* + * Max number of TX descriptors we clean up at a time. Should be + * modest as freeing skbs isn't cheap and it happens while holding + * locks. We just need to free packets faster than they arrive, we + * eventually catch up and keep the amortized cost reasonable. + */ + MAX_TX_RECLAIM = 16, + + /* + * Max number of Rx buffers we replenish at a time. Again keep this + * modest, allocating buffers isn't cheap either. + */ + MAX_RX_REFILL = 16, + + /* + * Period of the Rx queue check timer. This timer is infrequent as it + * has something to do only when the system experiences severe memory + * shortage. + */ + RX_QCHECK_PERIOD = (HZ / 2), + + /* + * Period of the TX queue check timer and the maximum number of TX + * descriptors to be reclaimed by the TX timer. + */ + TX_QCHECK_PERIOD = (HZ / 2), + MAX_TIMER_TX_RECLAIM = 100, + + /* + * Suspend an Ethernet TX queue with fewer available descriptors than + * this. We always want to have room for a maximum sized packet: + * inline immediate data + MAX_SKB_FRAGS. This is the same as + * calc_tx_flits() for a TSO packet with nr_frags == MAX_SKB_FRAGS + * (see that function and its helpers for a description of the + * calculation). + */ + ETHTXQ_MAX_FRAGS = MAX_SKB_FRAGS + 1, + ETHTXQ_MAX_SGL_LEN = ((3 * (ETHTXQ_MAX_FRAGS-1))/2 + + ((ETHTXQ_MAX_FRAGS-1) & 1) + + 2), + ETHTXQ_MAX_HDR = (sizeof(struct fw_eth_tx_pkt_vm_wr) + + sizeof(struct cpl_tx_pkt_lso_core) + + sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64), + ETHTXQ_MAX_FLITS = ETHTXQ_MAX_SGL_LEN + ETHTXQ_MAX_HDR, + + ETHTXQ_STOP_THRES = 1 + DIV_ROUND_UP(ETHTXQ_MAX_FLITS, TXD_PER_EQ_UNIT), + + /* + * Max TX descriptor space we allow for an Ethernet packet to be + * inlined into a WR. This is limited by the maximum value which + * we can specify for immediate data in the firmware Ethernet TX + * Work Request. + */ + MAX_IMM_TX_PKT_LEN = FW_WR_IMMDLEN_M, + + /* + * Max size of a WR sent through a control TX queue. + */ + MAX_CTRL_WR_LEN = 256, + + /* + * Maximum amount of data which we'll ever need to inline into a + * TX ring: max(MAX_IMM_TX_PKT_LEN, MAX_CTRL_WR_LEN). + */ + MAX_IMM_TX_LEN = (MAX_IMM_TX_PKT_LEN > MAX_CTRL_WR_LEN + ? MAX_IMM_TX_PKT_LEN + : MAX_CTRL_WR_LEN), + + /* + * For incoming packets less than RX_COPY_THRES, we copy the data into + * an skb rather than referencing the data. We allocate enough + * in-line room in skb's to accommodate pulling in RX_PULL_LEN bytes + * of the data (header). + */ + RX_COPY_THRES = 256, + RX_PULL_LEN = 128, + + /* + * Main body length for sk_buffs used for RX Ethernet packets with + * fragments. Should be >= RX_PULL_LEN but possibly bigger to give + * pskb_may_pull() some room. + */ + RX_SKB_LEN = 512, +}; + +/* + * Software state per TX descriptor. + */ +struct tx_sw_desc { + struct sk_buff *skb; /* socket buffer of TX data source */ + struct ulptx_sgl *sgl; /* scatter/gather list in TX Queue */ +}; + +/* + * Software state per RX Free List descriptor. We keep track of the allocated + * FL page, its size, and its PCI DMA address (if the page is mapped). The FL + * page size and its PCI DMA mapped state are stored in the low bits of the + * PCI DMA address as per below. + */ +struct rx_sw_desc { + struct page *page; /* Free List page buffer */ + dma_addr_t dma_addr; /* PCI DMA address (if mapped) */ + /* and flags (see below) */ +}; + +/* + * The low bits of rx_sw_desc.dma_addr have special meaning. Note that the + * SGE also uses the low 4 bits to determine the size of the buffer. It uses + * those bits to index into the SGE_FL_BUFFER_SIZE[index] register array. + * Since we only use SGE_FL_BUFFER_SIZE0 and SGE_FL_BUFFER_SIZE1, these low 4 + * bits can only contain a 0 or a 1 to indicate which size buffer we're giving + * to the SGE. Thus, our software state of "is the buffer mapped for DMA" is + * maintained in an inverse sense so the hardware never sees that bit high. + */ +enum { + RX_LARGE_BUF = 1 << 0, /* buffer is SGE_FL_BUFFER_SIZE[1] */ + RX_UNMAPPED_BUF = 1 << 1, /* buffer is not mapped */ +}; + +/** + * get_buf_addr - return DMA buffer address of software descriptor + * @sdesc: pointer to the software buffer descriptor + * + * Return the DMA buffer address of a software descriptor (stripping out + * our low-order flag bits). + */ +static inline dma_addr_t get_buf_addr(const struct rx_sw_desc *sdesc) +{ + return sdesc->dma_addr & ~(dma_addr_t)(RX_LARGE_BUF | RX_UNMAPPED_BUF); +} + +/** + * is_buf_mapped - is buffer mapped for DMA? + * @sdesc: pointer to the software buffer descriptor + * + * Determine whether the buffer associated with a software descriptor in + * mapped for DMA or not. + */ +static inline bool is_buf_mapped(const struct rx_sw_desc *sdesc) +{ + return !(sdesc->dma_addr & RX_UNMAPPED_BUF); +} + +/** + * need_skb_unmap - does the platform need unmapping of sk_buffs? + * + * Returns true if the platform needs sk_buff unmapping. The compiler + * optimizes away unnecessary code if this returns true. + */ +static inline int need_skb_unmap(void) +{ +#ifdef CONFIG_NEED_DMA_MAP_STATE + return 1; +#else + return 0; +#endif +} + +/** + * txq_avail - return the number of available slots in a TX queue + * @tq: the TX queue + * + * Returns the number of available descriptors in a TX queue. + */ +static inline unsigned int txq_avail(const struct sge_txq *tq) +{ + return tq->size - 1 - tq->in_use; +} + +/** + * fl_cap - return the capacity of a Free List + * @fl: the Free List + * + * Returns the capacity of a Free List. The capacity is less than the + * size because an Egress Queue Index Unit worth of descriptors needs to + * be left unpopulated, otherwise the Producer and Consumer indices PIDX + * and CIDX will match and the hardware will think the FL is empty. + */ +static inline unsigned int fl_cap(const struct sge_fl *fl) +{ + return fl->size - FL_PER_EQ_UNIT; +} + +/** + * fl_starving - return whether a Free List is starving. + * @adapter: pointer to the adapter + * @fl: the Free List + * + * Tests specified Free List to see whether the number of buffers + * available to the hardware has falled below our "starvation" + * threshold. + */ +static inline bool fl_starving(const struct adapter *adapter, + const struct sge_fl *fl) +{ + const struct sge *s = &adapter->sge; + + return fl->avail - fl->pend_cred <= s->fl_starve_thres; +} + +/** + * map_skb - map an skb for DMA to the device + * @dev: the egress net device + * @skb: the packet to map + * @addr: a pointer to the base of the DMA mapping array + * + * Map an skb for DMA to the device and return an array of DMA addresses. + */ +static int map_skb(struct device *dev, const struct sk_buff *skb, + dma_addr_t *addr) +{ + const skb_frag_t *fp, *end; + const struct skb_shared_info *si; + + *addr = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); + if (dma_mapping_error(dev, *addr)) + goto out_err; + + si = skb_shinfo(skb); + end = &si->frags[si->nr_frags]; + for (fp = si->frags; fp < end; fp++) { + *++addr = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp), + DMA_TO_DEVICE); + if (dma_mapping_error(dev, *addr)) + goto unwind; + } + return 0; + +unwind: + while (fp-- > si->frags) + dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE); + dma_unmap_single(dev, addr[-1], skb_headlen(skb), DMA_TO_DEVICE); + +out_err: + return -ENOMEM; +} + +static void unmap_sgl(struct device *dev, const struct sk_buff *skb, + const struct ulptx_sgl *sgl, const struct sge_txq *tq) +{ + const struct ulptx_sge_pair *p; + unsigned int nfrags = skb_shinfo(skb)->nr_frags; + + if (likely(skb_headlen(skb))) + dma_unmap_single(dev, be64_to_cpu(sgl->addr0), + be32_to_cpu(sgl->len0), DMA_TO_DEVICE); + else { + dma_unmap_page(dev, be64_to_cpu(sgl->addr0), + be32_to_cpu(sgl->len0), DMA_TO_DEVICE); + nfrags--; + } + + /* + * the complexity below is because of the possibility of a wrap-around + * in the middle of an SGL + */ + for (p = sgl->sge; nfrags >= 2; nfrags -= 2) { + if (likely((u8 *)(p + 1) <= (u8 *)tq->stat)) { +unmap: + dma_unmap_page(dev, be64_to_cpu(p->addr[0]), + be32_to_cpu(p->len[0]), DMA_TO_DEVICE); + dma_unmap_page(dev, be64_to_cpu(p->addr[1]), + be32_to_cpu(p->len[1]), DMA_TO_DEVICE); + p++; + } else if ((u8 *)p == (u8 *)tq->stat) { + p = (const struct ulptx_sge_pair *)tq->desc; + goto unmap; + } else if ((u8 *)p + 8 == (u8 *)tq->stat) { + const __be64 *addr = (const __be64 *)tq->desc; + + dma_unmap_page(dev, be64_to_cpu(addr[0]), + be32_to_cpu(p->len[0]), DMA_TO_DEVICE); + dma_unmap_page(dev, be64_to_cpu(addr[1]), + be32_to_cpu(p->len[1]), DMA_TO_DEVICE); + p = (const struct ulptx_sge_pair *)&addr[2]; + } else { + const __be64 *addr = (const __be64 *)tq->desc; + + dma_unmap_page(dev, be64_to_cpu(p->addr[0]), + be32_to_cpu(p->len[0]), DMA_TO_DEVICE); + dma_unmap_page(dev, be64_to_cpu(addr[0]), + be32_to_cpu(p->len[1]), DMA_TO_DEVICE); + p = (const struct ulptx_sge_pair *)&addr[1]; + } + } + if (nfrags) { + __be64 addr; + + if ((u8 *)p == (u8 *)tq->stat) + p = (const struct ulptx_sge_pair *)tq->desc; + addr = ((u8 *)p + 16 <= (u8 *)tq->stat + ? p->addr[0] + : *(const __be64 *)tq->desc); + dma_unmap_page(dev, be64_to_cpu(addr), be32_to_cpu(p->len[0]), + DMA_TO_DEVICE); + } +} + +/** + * free_tx_desc - reclaims TX descriptors and their buffers + * @adapter: the adapter + * @tq: the TX queue to reclaim descriptors from + * @n: the number of descriptors to reclaim + * @unmap: whether the buffers should be unmapped for DMA + * + * Reclaims TX descriptors from an SGE TX queue and frees the associated + * TX buffers. Called with the TX queue lock held. + */ +static void free_tx_desc(struct adapter *adapter, struct sge_txq *tq, + unsigned int n, bool unmap) +{ + struct tx_sw_desc *sdesc; + unsigned int cidx = tq->cidx; + struct device *dev = adapter->pdev_dev; + + const int need_unmap = need_skb_unmap() && unmap; + + sdesc = &tq->sdesc[cidx]; + while (n--) { + /* + * If we kept a reference to the original TX skb, we need to + * unmap it from PCI DMA space (if required) and free it. + */ + if (sdesc->skb) { + if (need_unmap) + unmap_sgl(dev, sdesc->skb, sdesc->sgl, tq); + dev_consume_skb_any(sdesc->skb); + sdesc->skb = NULL; + } + + sdesc++; + if (++cidx == tq->size) { + cidx = 0; + sdesc = tq->sdesc; + } + } + tq->cidx = cidx; +} + +/* + * Return the number of reclaimable descriptors in a TX queue. + */ +static inline int reclaimable(const struct sge_txq *tq) +{ + int hw_cidx = be16_to_cpu(tq->stat->cidx); + int reclaimable = hw_cidx - tq->cidx; + if (reclaimable < 0) + reclaimable += tq->size; + return reclaimable; +} + +/** + * reclaim_completed_tx - reclaims completed TX descriptors + * @adapter: the adapter + * @tq: the TX queue to reclaim completed descriptors from + * @unmap: whether the buffers should be unmapped for DMA + * + * Reclaims TX descriptors that the SGE has indicated it has processed, + * and frees the associated buffers if possible. Called with the TX + * queue locked. + */ +static inline void reclaim_completed_tx(struct adapter *adapter, + struct sge_txq *tq, + bool unmap) +{ + int avail = reclaimable(tq); + + if (avail) { + /* + * Limit the amount of clean up work we do at a time to keep + * the TX lock hold time O(1). + */ + if (avail > MAX_TX_RECLAIM) + avail = MAX_TX_RECLAIM; + + free_tx_desc(adapter, tq, avail, unmap); + tq->in_use -= avail; + } +} + +/** + * get_buf_size - return the size of an RX Free List buffer. + * @adapter: pointer to the associated adapter + * @sdesc: pointer to the software buffer descriptor + */ +static inline int get_buf_size(const struct adapter *adapter, + const struct rx_sw_desc *sdesc) +{ + const struct sge *s = &adapter->sge; + + return (s->fl_pg_order > 0 && (sdesc->dma_addr & RX_LARGE_BUF) + ? (PAGE_SIZE << s->fl_pg_order) : PAGE_SIZE); +} + +/** + * free_rx_bufs - free RX buffers on an SGE Free List + * @adapter: the adapter + * @fl: the SGE Free List to free buffers from + * @n: how many buffers to free + * + * Release the next @n buffers on an SGE Free List RX queue. The + * buffers must be made inaccessible to hardware before calling this + * function. + */ +static void free_rx_bufs(struct adapter *adapter, struct sge_fl *fl, int n) +{ + while (n--) { + struct rx_sw_desc *sdesc = &fl->sdesc[fl->cidx]; + + if (is_buf_mapped(sdesc)) + dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc), + get_buf_size(adapter, sdesc), + DMA_FROM_DEVICE); + put_page(sdesc->page); + sdesc->page = NULL; + if (++fl->cidx == fl->size) + fl->cidx = 0; + fl->avail--; + } +} + +/** + * unmap_rx_buf - unmap the current RX buffer on an SGE Free List + * @adapter: the adapter + * @fl: the SGE Free List + * + * Unmap the current buffer on an SGE Free List RX queue. The + * buffer must be made inaccessible to HW before calling this function. + * + * This is similar to @free_rx_bufs above but does not free the buffer. + * Do note that the FL still loses any further access to the buffer. + * This is used predominantly to "transfer ownership" of an FL buffer + * to another entity (typically an skb's fragment list). + */ +static void unmap_rx_buf(struct adapter *adapter, struct sge_fl *fl) +{ + struct rx_sw_desc *sdesc = &fl->sdesc[fl->cidx]; + + if (is_buf_mapped(sdesc)) + dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc), + get_buf_size(adapter, sdesc), + DMA_FROM_DEVICE); + sdesc->page = NULL; + if (++fl->cidx == fl->size) + fl->cidx = 0; + fl->avail--; +} + +/** + * ring_fl_db - righ doorbell on free list + * @adapter: the adapter + * @fl: the Free List whose doorbell should be rung ... + * + * Tell the Scatter Gather Engine that there are new free list entries + * available. + */ +static inline void ring_fl_db(struct adapter *adapter, struct sge_fl *fl) +{ + u32 val = adapter->params.arch.sge_fl_db; + + /* The SGE keeps track of its Producer and Consumer Indices in terms + * of Egress Queue Units so we can only tell it about integral numbers + * of multiples of Free List Entries per Egress Queue Units ... + */ + if (fl->pend_cred >= FL_PER_EQ_UNIT) { + if (is_t4(adapter->params.chip)) + val |= PIDX_V(fl->pend_cred / FL_PER_EQ_UNIT); + else + val |= PIDX_T5_V(fl->pend_cred / FL_PER_EQ_UNIT); + + /* Make sure all memory writes to the Free List queue are + * committed before we tell the hardware about them. + */ + wmb(); + + /* If we don't have access to the new User Doorbell (T5+), use + * the old doorbell mechanism; otherwise use the new BAR2 + * mechanism. + */ + if (unlikely(fl->bar2_addr == NULL)) { + t4_write_reg(adapter, + T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL, + QID_V(fl->cntxt_id) | val); + } else { + writel(val | QID_V(fl->bar2_qid), + fl->bar2_addr + SGE_UDB_KDOORBELL); + + /* This Write memory Barrier will force the write to + * the User Doorbell area to be flushed. + */ + wmb(); + } + fl->pend_cred %= FL_PER_EQ_UNIT; + } +} + +/** + * set_rx_sw_desc - initialize software RX buffer descriptor + * @sdesc: pointer to the softwore RX buffer descriptor + * @page: pointer to the page data structure backing the RX buffer + * @dma_addr: PCI DMA address (possibly with low-bit flags) + */ +static inline void set_rx_sw_desc(struct rx_sw_desc *sdesc, struct page *page, + dma_addr_t dma_addr) +{ + sdesc->page = page; + sdesc->dma_addr = dma_addr; +} + +/* + * Support for poisoning RX buffers ... + */ +#define POISON_BUF_VAL -1 + +static inline void poison_buf(struct page *page, size_t sz) +{ +#if POISON_BUF_VAL >= 0 + memset(page_address(page), POISON_BUF_VAL, sz); +#endif +} + +/** + * refill_fl - refill an SGE RX buffer ring + * @adapter: the adapter + * @fl: the Free List ring to refill + * @n: the number of new buffers to allocate + * @gfp: the gfp flags for the allocations + * + * (Re)populate an SGE free-buffer queue with up to @n new packet buffers, + * allocated with the supplied gfp flags. The caller must assure that + * @n does not exceed the queue's capacity -- i.e. (cidx == pidx) _IN + * EGRESS QUEUE UNITS_ indicates an empty Free List! Returns the number + * of buffers allocated. If afterwards the queue is found critically low, + * mark it as starving in the bitmap of starving FLs. + */ +static unsigned int refill_fl(struct adapter *adapter, struct sge_fl *fl, + int n, gfp_t gfp) +{ + struct sge *s = &adapter->sge; + struct page *page; + dma_addr_t dma_addr; + unsigned int cred = fl->avail; + __be64 *d = &fl->desc[fl->pidx]; + struct rx_sw_desc *sdesc = &fl->sdesc[fl->pidx]; + + /* + * Sanity: ensure that the result of adding n Free List buffers + * won't result in wrapping the SGE's Producer Index around to + * it's Consumer Index thereby indicating an empty Free List ... + */ + BUG_ON(fl->avail + n > fl->size - FL_PER_EQ_UNIT); + + gfp |= __GFP_NOWARN; + + /* + * If we support large pages, prefer large buffers and fail over to + * small pages if we can't allocate large pages to satisfy the refill. + * If we don't support large pages, drop directly into the small page + * allocation code. + */ + if (s->fl_pg_order == 0) + goto alloc_small_pages; + + while (n) { + page = __dev_alloc_pages(gfp, s->fl_pg_order); + if (unlikely(!page)) { + /* + * We've failed inour attempt to allocate a "large + * page". Fail over to the "small page" allocation + * below. + */ + fl->large_alloc_failed++; + break; + } + poison_buf(page, PAGE_SIZE << s->fl_pg_order); + + dma_addr = dma_map_page(adapter->pdev_dev, page, 0, + PAGE_SIZE << s->fl_pg_order, + DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) { + /* + * We've run out of DMA mapping space. Free up the + * buffer and return with what we've managed to put + * into the free list. We don't want to fail over to + * the small page allocation below in this case + * because DMA mapping resources are typically + * critical resources once they become scarse. + */ + __free_pages(page, s->fl_pg_order); + goto out; + } + dma_addr |= RX_LARGE_BUF; + *d++ = cpu_to_be64(dma_addr); + + set_rx_sw_desc(sdesc, page, dma_addr); + sdesc++; + + fl->avail++; + if (++fl->pidx == fl->size) { + fl->pidx = 0; + sdesc = fl->sdesc; + d = fl->desc; + } + n--; + } + +alloc_small_pages: + while (n--) { + page = __dev_alloc_page(gfp); + if (unlikely(!page)) { + fl->alloc_failed++; + break; + } + poison_buf(page, PAGE_SIZE); + + dma_addr = dma_map_page(adapter->pdev_dev, page, 0, PAGE_SIZE, + DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) { + put_page(page); + break; + } + *d++ = cpu_to_be64(dma_addr); + + set_rx_sw_desc(sdesc, page, dma_addr); + sdesc++; + + fl->avail++; + if (++fl->pidx == fl->size) { + fl->pidx = 0; + sdesc = fl->sdesc; + d = fl->desc; + } + } + +out: + /* + * Update our accounting state to incorporate the new Free List + * buffers, tell the hardware about them and return the number of + * buffers which we were able to allocate. + */ + cred = fl->avail - cred; + fl->pend_cred += cred; + ring_fl_db(adapter, fl); + + if (unlikely(fl_starving(adapter, fl))) { + smp_wmb(); + set_bit(fl->cntxt_id, adapter->sge.starving_fl); + } + + return cred; +} + +/* + * Refill a Free List to its capacity or the Maximum Refill Increment, + * whichever is smaller ... + */ +static inline void __refill_fl(struct adapter *adapter, struct sge_fl *fl) +{ + refill_fl(adapter, fl, + min((unsigned int)MAX_RX_REFILL, fl_cap(fl) - fl->avail), + GFP_ATOMIC); +} + +/** + * alloc_ring - allocate resources for an SGE descriptor ring + * @dev: the PCI device's core device + * @nelem: the number of descriptors + * @hwsize: the size of each hardware descriptor + * @swsize: the size of each software descriptor + * @busaddrp: the physical PCI bus address of the allocated ring + * @swringp: return address pointer for software ring + * @stat_size: extra space in hardware ring for status information + * + * Allocates resources for an SGE descriptor ring, such as TX queues, + * free buffer lists, response queues, etc. Each SGE ring requires + * space for its hardware descriptors plus, optionally, space for software + * state associated with each hardware entry (the metadata). The function + * returns three values: the virtual address for the hardware ring (the + * return value of the function), the PCI bus address of the hardware + * ring (in *busaddrp), and the address of the software ring (in swringp). + * Both the hardware and software rings are returned zeroed out. + */ +static void *alloc_ring(struct device *dev, size_t nelem, size_t hwsize, + size_t swsize, dma_addr_t *busaddrp, void *swringp, + size_t stat_size) +{ + /* + * Allocate the hardware ring and PCI DMA bus address space for said. + */ + size_t hwlen = nelem * hwsize + stat_size; + void *hwring = dma_alloc_coherent(dev, hwlen, busaddrp, GFP_KERNEL); + + if (!hwring) + return NULL; + + /* + * If the caller wants a software ring, allocate it and return a + * pointer to it in *swringp. + */ + BUG_ON((swsize != 0) != (swringp != NULL)); + if (swsize) { + void *swring = kcalloc(nelem, swsize, GFP_KERNEL); + + if (!swring) { + dma_free_coherent(dev, hwlen, hwring, *busaddrp); + return NULL; + } + *(void **)swringp = swring; + } + + return hwring; +} + +/** + * sgl_len - calculates the size of an SGL of the given capacity + * @n: the number of SGL entries + * + * Calculates the number of flits (8-byte units) needed for a Direct + * Scatter/Gather List that can hold the given number of entries. + */ +static inline unsigned int sgl_len(unsigned int n) +{ + /* + * A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA + * addresses. The DSGL Work Request starts off with a 32-bit DSGL + * ULPTX header, then Length0, then Address0, then, for 1 <= i <= N, + * repeated sequences of { Length[i], Length[i+1], Address[i], + * Address[i+1] } (this ensures that all addresses are on 64-bit + * boundaries). If N is even, then Length[N+1] should be set to 0 and + * Address[N+1] is omitted. + * + * The following calculation incorporates all of the above. It's + * somewhat hard to follow but, briefly: the "+2" accounts for the + * first two flits which include the DSGL header, Length0 and + * Address0; the "(3*(n-1))/2" covers the main body of list entries (3 + * flits for every pair of the remaining N) +1 if (n-1) is odd; and + * finally the "+((n-1)&1)" adds the one remaining flit needed if + * (n-1) is odd ... + */ + n--; + return (3 * n) / 2 + (n & 1) + 2; +} + +/** + * flits_to_desc - returns the num of TX descriptors for the given flits + * @flits: the number of flits + * + * Returns the number of TX descriptors needed for the supplied number + * of flits. + */ +static inline unsigned int flits_to_desc(unsigned int flits) +{ + BUG_ON(flits > SGE_MAX_WR_LEN / sizeof(__be64)); + return DIV_ROUND_UP(flits, TXD_PER_EQ_UNIT); +} + +/** + * is_eth_imm - can an Ethernet packet be sent as immediate data? + * @skb: the packet + * + * Returns whether an Ethernet packet is small enough to fit completely as + * immediate data. + */ +static inline int is_eth_imm(const struct sk_buff *skb) +{ + /* + * The VF Driver uses the FW_ETH_TX_PKT_VM_WR firmware Work Request + * which does not accommodate immediate data. We could dike out all + * of the support code for immediate data but that would tie our hands + * too much if we ever want to enhace the firmware. It would also + * create more differences between the PF and VF Drivers. + */ + return false; +} + +/** + * calc_tx_flits - calculate the number of flits for a packet TX WR + * @skb: the packet + * + * Returns the number of flits needed for a TX Work Request for the + * given Ethernet packet, including the needed WR and CPL headers. + */ +static inline unsigned int calc_tx_flits(const struct sk_buff *skb) +{ + unsigned int flits; + + /* + * If the skb is small enough, we can pump it out as a work request + * with only immediate data. In that case we just have to have the + * TX Packet header plus the skb data in the Work Request. + */ + if (is_eth_imm(skb)) + return DIV_ROUND_UP(skb->len + sizeof(struct cpl_tx_pkt), + sizeof(__be64)); + + /* + * Otherwise, we're going to have to construct a Scatter gather list + * of the skb body and fragments. We also include the flits necessary + * for the TX Packet Work Request and CPL. We always have a firmware + * Write Header (incorporated as part of the cpl_tx_pkt_lso and + * cpl_tx_pkt structures), followed by either a TX Packet Write CPL + * message or, if we're doing a Large Send Offload, an LSO CPL message + * with an embedded TX Packet Write CPL message. + */ + flits = sgl_len(skb_shinfo(skb)->nr_frags + 1); + if (skb_shinfo(skb)->gso_size) + flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) + + sizeof(struct cpl_tx_pkt_lso_core) + + sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); + else + flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) + + sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); + return flits; +} + +/** + * write_sgl - populate a Scatter/Gather List for a packet + * @skb: the packet + * @tq: the TX queue we are writing into + * @sgl: starting location for writing the SGL + * @end: points right after the end of the SGL + * @start: start offset into skb main-body data to include in the SGL + * @addr: the list of DMA bus addresses for the SGL elements + * + * Generates a Scatter/Gather List for the buffers that make up a packet. + * The caller must provide adequate space for the SGL that will be written. + * The SGL includes all of the packet's page fragments and the data in its + * main body except for the first @start bytes. @pos must be 16-byte + * aligned and within a TX descriptor with available space. @end points + * write after the end of the SGL but does not account for any potential + * wrap around, i.e., @end > @tq->stat. + */ +static void write_sgl(const struct sk_buff *skb, struct sge_txq *tq, + struct ulptx_sgl *sgl, u64 *end, unsigned int start, + const dma_addr_t *addr) +{ + unsigned int i, len; + struct ulptx_sge_pair *to; + const struct skb_shared_info *si = skb_shinfo(skb); + unsigned int nfrags = si->nr_frags; + struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1]; + + len = skb_headlen(skb) - start; + if (likely(len)) { + sgl->len0 = htonl(len); + sgl->addr0 = cpu_to_be64(addr[0] + start); + nfrags++; + } else { + sgl->len0 = htonl(skb_frag_size(&si->frags[0])); + sgl->addr0 = cpu_to_be64(addr[1]); + } + + sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | + ULPTX_NSGE_V(nfrags)); + if (likely(--nfrags == 0)) + return; + /* + * Most of the complexity below deals with the possibility we hit the + * end of the queue in the middle of writing the SGL. For this case + * only we create the SGL in a temporary buffer and then copy it. + */ + to = (u8 *)end > (u8 *)tq->stat ? buf : sgl->sge; + + for (i = (nfrags != si->nr_frags); nfrags >= 2; nfrags -= 2, to++) { + to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i])); + to->len[1] = cpu_to_be32(skb_frag_size(&si->frags[++i])); + to->addr[0] = cpu_to_be64(addr[i]); + to->addr[1] = cpu_to_be64(addr[++i]); + } + if (nfrags) { + to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i])); + to->len[1] = cpu_to_be32(0); + to->addr[0] = cpu_to_be64(addr[i + 1]); + } + if (unlikely((u8 *)end > (u8 *)tq->stat)) { + unsigned int part0 = (u8 *)tq->stat - (u8 *)sgl->sge, part1; + + if (likely(part0)) + memcpy(sgl->sge, buf, part0); + part1 = (u8 *)end - (u8 *)tq->stat; + memcpy(tq->desc, (u8 *)buf + part0, part1); + end = (void *)tq->desc + part1; + } + if ((uintptr_t)end & 8) /* 0-pad to multiple of 16 */ + *end = 0; +} + +/** + * ring_tx_db - check and potentially ring a TX queue's doorbell + * @adapter: the adapter + * @tq: the TX queue + * @n: number of new descriptors to give to HW + * + * Ring the doorbel for a TX queue. + */ +static inline void ring_tx_db(struct adapter *adapter, struct sge_txq *tq, + int n) +{ + /* Make sure that all writes to the TX Descriptors are committed + * before we tell the hardware about them. + */ + wmb(); + + /* If we don't have access to the new User Doorbell (T5+), use the old + * doorbell mechanism; otherwise use the new BAR2 mechanism. + */ + if (unlikely(tq->bar2_addr == NULL)) { + u32 val = PIDX_V(n); + + t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL, + QID_V(tq->cntxt_id) | val); + } else { + u32 val = PIDX_T5_V(n); + + /* T4 and later chips share the same PIDX field offset within + * the doorbell, but T5 and later shrank the field in order to + * gain a bit for Doorbell Priority. The field was absurdly + * large in the first place (14 bits) so we just use the T5 + * and later limits and warn if a Queue ID is too large. + */ + WARN_ON(val & DBPRIO_F); + + /* If we're only writing a single Egress Unit and the BAR2 + * Queue ID is 0, we can use the Write Combining Doorbell + * Gather Buffer; otherwise we use the simple doorbell. + */ + if (n == 1 && tq->bar2_qid == 0) { + unsigned int index = (tq->pidx + ? (tq->pidx - 1) + : (tq->size - 1)); + __be64 *src = (__be64 *)&tq->desc[index]; + __be64 __iomem *dst = (__be64 __iomem *)(tq->bar2_addr + + SGE_UDB_WCDOORBELL); + unsigned int count = EQ_UNIT / sizeof(__be64); + + /* Copy the TX Descriptor in a tight loop in order to + * try to get it to the adapter in a single Write + * Combined transfer on the PCI-E Bus. If the Write + * Combine fails (say because of an interrupt, etc.) + * the hardware will simply take the last write as a + * simple doorbell write with a PIDX Increment of 1 + * and will fetch the TX Descriptor from memory via + * DMA. + */ + while (count) { + /* the (__force u64) is because the compiler + * doesn't understand the endian swizzling + * going on + */ + writeq((__force u64)*src, dst); + src++; + dst++; + count--; + } + } else + writel(val | QID_V(tq->bar2_qid), + tq->bar2_addr + SGE_UDB_KDOORBELL); + + /* This Write Memory Barrier will force the write to the User + * Doorbell area to be flushed. This is needed to prevent + * writes on different CPUs for the same queue from hitting + * the adapter out of order. This is required when some Work + * Requests take the Write Combine Gather Buffer path (user + * doorbell area offset [SGE_UDB_WCDOORBELL..+63]) and some + * take the traditional path where we simply increment the + * PIDX (User Doorbell area SGE_UDB_KDOORBELL) and have the + * hardware DMA read the actual Work Request. + */ + wmb(); + } +} + +/** + * inline_tx_skb - inline a packet's data into TX descriptors + * @skb: the packet + * @tq: the TX queue where the packet will be inlined + * @pos: starting position in the TX queue to inline the packet + * + * Inline a packet's contents directly into TX descriptors, starting at + * the given position within the TX DMA ring. + * Most of the complexity of this operation is dealing with wrap arounds + * in the middle of the packet we want to inline. + */ +static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *tq, + void *pos) +{ + u64 *p; + int left = (void *)tq->stat - pos; + + if (likely(skb->len <= left)) { + if (likely(!skb->data_len)) + skb_copy_from_linear_data(skb, pos, skb->len); + else + skb_copy_bits(skb, 0, pos, skb->len); + pos += skb->len; + } else { + skb_copy_bits(skb, 0, pos, left); + skb_copy_bits(skb, left, tq->desc, skb->len - left); + pos = (void *)tq->desc + (skb->len - left); + } + + /* 0-pad to multiple of 16 */ + p = PTR_ALIGN(pos, 8); + if ((uintptr_t)p & 8) + *p = 0; +} + +/* + * Figure out what HW csum a packet wants and return the appropriate control + * bits. + */ +static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb) +{ + int csum_type; + const struct iphdr *iph = ip_hdr(skb); + + if (iph->version == 4) { + if (iph->protocol == IPPROTO_TCP) + csum_type = TX_CSUM_TCPIP; + else if (iph->protocol == IPPROTO_UDP) + csum_type = TX_CSUM_UDPIP; + else { +nocsum: + /* + * unknown protocol, disable HW csum + * and hope a bad packet is detected + */ + return TXPKT_L4CSUM_DIS_F; + } + } else { + /* + * this doesn't work with extension headers + */ + const struct ipv6hdr *ip6h = (const struct ipv6hdr *)iph; + + if (ip6h->nexthdr == IPPROTO_TCP) + csum_type = TX_CSUM_TCPIP6; + else if (ip6h->nexthdr == IPPROTO_UDP) + csum_type = TX_CSUM_UDPIP6; + else + goto nocsum; + } + + if (likely(csum_type >= TX_CSUM_TCPIP)) { + u64 hdr_len = TXPKT_IPHDR_LEN_V(skb_network_header_len(skb)); + int eth_hdr_len = skb_network_offset(skb) - ETH_HLEN; + + if (chip <= CHELSIO_T5) + hdr_len |= TXPKT_ETHHDR_LEN_V(eth_hdr_len); + else + hdr_len |= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len); + return TXPKT_CSUM_TYPE_V(csum_type) | hdr_len; + } else { + int start = skb_transport_offset(skb); + + return TXPKT_CSUM_TYPE_V(csum_type) | + TXPKT_CSUM_START_V(start) | + TXPKT_CSUM_LOC_V(start + skb->csum_offset); + } +} + +/* + * Stop an Ethernet TX queue and record that state change. + */ +static void txq_stop(struct sge_eth_txq *txq) +{ + netif_tx_stop_queue(txq->txq); + txq->q.stops++; +} + +/* + * Advance our software state for a TX queue by adding n in use descriptors. + */ +static inline void txq_advance(struct sge_txq *tq, unsigned int n) +{ + tq->in_use += n; + tq->pidx += n; + if (tq->pidx >= tq->size) + tq->pidx -= tq->size; +} + +/** + * t4vf_eth_xmit - add a packet to an Ethernet TX queue + * @skb: the packet + * @dev: the egress net device + * + * Add a packet to an SGE Ethernet TX queue. Runs with softirqs disabled. + */ +netdev_tx_t t4vf_eth_xmit(struct sk_buff *skb, struct net_device *dev) +{ + u32 wr_mid; + u64 cntrl, *end; + int qidx, credits, max_pkt_len; + unsigned int flits, ndesc; + struct adapter *adapter; + struct sge_eth_txq *txq; + const struct port_info *pi; + struct fw_eth_tx_pkt_vm_wr *wr; + struct cpl_tx_pkt_core *cpl; + const struct skb_shared_info *ssi; + dma_addr_t addr[MAX_SKB_FRAGS + 1]; + const size_t fw_hdr_copy_len = sizeof(wr->firmware); + + /* + * The chip minimum packet length is 10 octets but the firmware + * command that we are using requires that we copy the Ethernet header + * (including the VLAN tag) into the header so we reject anything + * smaller than that ... + */ + if (unlikely(skb->len < fw_hdr_copy_len)) + goto out_free; + + /* Discard the packet if the length is greater than mtu */ + max_pkt_len = ETH_HLEN + dev->mtu; + if (skb_vlan_tagged(skb)) + max_pkt_len += VLAN_HLEN; + if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len))) + goto out_free; + + /* + * Figure out which TX Queue we're going to use. + */ + pi = netdev_priv(dev); + adapter = pi->adapter; + qidx = skb_get_queue_mapping(skb); + BUG_ON(qidx >= pi->nqsets); + txq = &adapter->sge.ethtxq[pi->first_qset + qidx]; + + if (pi->vlan_id && !skb_vlan_tag_present(skb)) + __vlan_hwaccel_put_tag(skb, cpu_to_be16(ETH_P_8021Q), + pi->vlan_id); + + /* + * Take this opportunity to reclaim any TX Descriptors whose DMA + * transfers have completed. + */ + reclaim_completed_tx(adapter, &txq->q, true); + + /* + * Calculate the number of flits and TX Descriptors we're going to + * need along with how many TX Descriptors will be left over after + * we inject our Work Request. + */ + flits = calc_tx_flits(skb); + ndesc = flits_to_desc(flits); + credits = txq_avail(&txq->q) - ndesc; + + if (unlikely(credits < 0)) { + /* + * Not enough room for this packet's Work Request. Stop the + * TX Queue and return a "busy" condition. The queue will get + * started later on when the firmware informs us that space + * has opened up. + */ + txq_stop(txq); + dev_err(adapter->pdev_dev, + "%s: TX ring %u full while queue awake!\n", + dev->name, qidx); + return NETDEV_TX_BUSY; + } + + if (!is_eth_imm(skb) && + unlikely(map_skb(adapter->pdev_dev, skb, addr) < 0)) { + /* + * We need to map the skb into PCI DMA space (because it can't + * be in-lined directly into the Work Request) and the mapping + * operation failed. Record the error and drop the packet. + */ + txq->mapping_err++; + goto out_free; + } + + wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2)); + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + /* + * After we're done injecting the Work Request for this + * packet, we'll be below our "stop threshold" so stop the TX + * Queue now and schedule a request for an SGE Egress Queue + * Update message. The queue will get started later on when + * the firmware processes this Work Request and sends us an + * Egress Queue Status Update message indicating that space + * has opened up. + */ + txq_stop(txq); + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + /* + * Start filling in our Work Request. Note that we do _not_ handle + * the WR Header wrapping around the TX Descriptor Ring. If our + * maximum header size ever exceeds one TX Descriptor, we'll need to + * do something else here. + */ + BUG_ON(DIV_ROUND_UP(ETHTXQ_MAX_HDR, TXD_PER_EQ_UNIT) > 1); + wr = (void *)&txq->q.desc[txq->q.pidx]; + wr->equiq_to_len16 = cpu_to_be32(wr_mid); + wr->r3[0] = cpu_to_be32(0); + wr->r3[1] = cpu_to_be32(0); + skb_copy_from_linear_data(skb, &wr->firmware, fw_hdr_copy_len); + end = (u64 *)wr + flits; + + /* + * If this is a Large Send Offload packet we'll put in an LSO CPL + * message with an encapsulated TX Packet CPL message. Otherwise we + * just use a TX Packet CPL message. + */ + ssi = skb_shinfo(skb); + if (ssi->gso_size) { + struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); + bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0; + int l3hdr_len = skb_network_header_len(skb); + int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN; + + wr->op_immdlen = + cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) | + FW_WR_IMMDLEN_V(sizeof(*lso) + + sizeof(*cpl))); + /* + * Fill in the LSO CPL message. + */ + lso->lso_ctrl = + cpu_to_be32(LSO_OPCODE_V(CPL_TX_PKT_LSO) | + LSO_FIRST_SLICE_F | + LSO_LAST_SLICE_F | + LSO_IPV6_V(v6) | + LSO_ETHHDR_LEN_V(eth_xtra_len / 4) | + LSO_IPHDR_LEN_V(l3hdr_len / 4) | + LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff)); + lso->ipid_ofst = cpu_to_be16(0); + lso->mss = cpu_to_be16(ssi->gso_size); + lso->seqno_offset = cpu_to_be32(0); + if (is_t4(adapter->params.chip)) + lso->len = cpu_to_be32(skb->len); + else + lso->len = cpu_to_be32(LSO_T5_XFER_SIZE_V(skb->len)); + + /* + * Set up TX Packet CPL pointer, control word and perform + * accounting. + */ + cpl = (void *)(lso + 1); + + if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) + cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len); + else + cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len); + + cntrl |= TXPKT_CSUM_TYPE_V(v6 ? + TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) | + TXPKT_IPHDR_LEN_V(l3hdr_len); + txq->tso++; + txq->tx_cso += ssi->gso_segs; + } else { + int len; + + len = is_eth_imm(skb) ? skb->len + sizeof(*cpl) : sizeof(*cpl); + wr->op_immdlen = + cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) | + FW_WR_IMMDLEN_V(len)); + + /* + * Set up TX Packet CPL pointer, control word and perform + * accounting. + */ + cpl = (void *)(wr + 1); + if (skb->ip_summed == CHECKSUM_PARTIAL) { + cntrl = hwcsum(adapter->params.chip, skb) | + TXPKT_IPCSUM_DIS_F; + txq->tx_cso++; + } else + cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; + } + + /* + * If there's a VLAN tag present, add that to the list of things to + * do in this Work Request. + */ + if (skb_vlan_tag_present(skb)) { + txq->vlan_ins++; + cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb)); + } + + /* + * Fill in the TX Packet CPL message header. + */ + cpl->ctrl0 = cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT) | + TXPKT_INTF_V(pi->port_id) | + TXPKT_PF_V(0)); + cpl->pack = cpu_to_be16(0); + cpl->len = cpu_to_be16(skb->len); + cpl->ctrl1 = cpu_to_be64(cntrl); + +#ifdef T4_TRACE + T4_TRACE5(adapter->tb[txq->q.cntxt_id & 7], + "eth_xmit: ndesc %u, credits %u, pidx %u, len %u, frags %u", + ndesc, credits, txq->q.pidx, skb->len, ssi->nr_frags); +#endif + + /* + * Fill in the body of the TX Packet CPL message with either in-lined + * data or a Scatter/Gather List. + */ + if (is_eth_imm(skb)) { + /* + * In-line the packet's data and free the skb since we don't + * need it any longer. + */ + inline_tx_skb(skb, &txq->q, cpl + 1); + dev_consume_skb_any(skb); + } else { + /* + * Write the skb's Scatter/Gather list into the TX Packet CPL + * message and retain a pointer to the skb so we can free it + * later when its DMA completes. (We store the skb pointer + * in the Software Descriptor corresponding to the last TX + * Descriptor used by the Work Request.) + * + * The retained skb will be freed when the corresponding TX + * Descriptors are reclaimed after their DMAs complete. + * However, this could take quite a while since, in general, + * the hardware is set up to be lazy about sending DMA + * completion notifications to us and we mostly perform TX + * reclaims in the transmit routine. + * + * This is good for performamce but means that we rely on new + * TX packets arriving to run the destructors of completed + * packets, which open up space in their sockets' send queues. + * Sometimes we do not get such new packets causing TX to + * stall. A single UDP transmitter is a good example of this + * situation. We have a clean up timer that periodically + * reclaims completed packets but it doesn't run often enough + * (nor do we want it to) to prevent lengthy stalls. A + * solution to this problem is to run the destructor early, + * after the packet is queued but before it's DMAd. A con is + * that we lie to socket memory accounting, but the amount of + * extra memory is reasonable (limited by the number of TX + * descriptors), the packets do actually get freed quickly by + * new packets almost always, and for protocols like TCP that + * wait for acks to really free up the data the extra memory + * is even less. On the positive side we run the destructors + * on the sending CPU rather than on a potentially different + * completing CPU, usually a good thing. + * + * Run the destructor before telling the DMA engine about the + * packet to make sure it doesn't complete and get freed + * prematurely. + */ + struct ulptx_sgl *sgl = (struct ulptx_sgl *)(cpl + 1); + struct sge_txq *tq = &txq->q; + int last_desc; + + /* + * If the Work Request header was an exact multiple of our TX + * Descriptor length, then it's possible that the starting SGL + * pointer lines up exactly with the end of our TX Descriptor + * ring. If that's the case, wrap around to the beginning + * here ... + */ + if (unlikely((void *)sgl == (void *)tq->stat)) { + sgl = (void *)tq->desc; + end = ((void *)tq->desc + ((void *)end - (void *)tq->stat)); + } + + write_sgl(skb, tq, sgl, end, 0, addr); + skb_orphan(skb); + + last_desc = tq->pidx + ndesc - 1; + if (last_desc >= tq->size) + last_desc -= tq->size; + tq->sdesc[last_desc].skb = skb; + tq->sdesc[last_desc].sgl = sgl; + } + + /* + * Advance our internal TX Queue state, tell the hardware about + * the new TX descriptors and return success. + */ + txq_advance(&txq->q, ndesc); + netif_trans_update(dev); + ring_tx_db(adapter, &txq->q, ndesc); + return NETDEV_TX_OK; + +out_free: + /* + * An error of some sort happened. Free the TX skb and tell the + * OS that we've "dealt" with the packet ... + */ + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +} + +/** + * copy_frags - copy fragments from gather list into skb_shared_info + * @skb: destination skb + * @gl: source internal packet gather list + * @offset: packet start offset in first page + * + * Copy an internal packet gather list into a Linux skb_shared_info + * structure. + */ +static inline void copy_frags(struct sk_buff *skb, + const struct pkt_gl *gl, + unsigned int offset) +{ + int i; + + /* usually there's just one frag */ + __skb_fill_page_desc(skb, 0, gl->frags[0].page, + gl->frags[0].offset + offset, + gl->frags[0].size - offset); + skb_shinfo(skb)->nr_frags = gl->nfrags; + for (i = 1; i < gl->nfrags; i++) + __skb_fill_page_desc(skb, i, gl->frags[i].page, + gl->frags[i].offset, + gl->frags[i].size); + + /* get a reference to the last page, we don't own it */ + get_page(gl->frags[gl->nfrags - 1].page); +} + +/** + * t4vf_pktgl_to_skb - build an sk_buff from a packet gather list + * @gl: the gather list + * @skb_len: size of sk_buff main body if it carries fragments + * @pull_len: amount of data to move to the sk_buff's main body + * + * Builds an sk_buff from the given packet gather list. Returns the + * sk_buff or %NULL if sk_buff allocation failed. + */ +static struct sk_buff *t4vf_pktgl_to_skb(const struct pkt_gl *gl, + unsigned int skb_len, + unsigned int pull_len) +{ + struct sk_buff *skb; + + /* + * If the ingress packet is small enough, allocate an skb large enough + * for all of the data and copy it inline. Otherwise, allocate an skb + * with enough room to pull in the header and reference the rest of + * the data via the skb fragment list. + * + * Below we rely on RX_COPY_THRES being less than the smallest Rx + * buff! size, which is expected since buffers are at least + * PAGE_SIZEd. In this case packets up to RX_COPY_THRES have only one + * fragment. + */ + if (gl->tot_len <= RX_COPY_THRES) { + /* small packets have only one fragment */ + skb = alloc_skb(gl->tot_len, GFP_ATOMIC); + if (unlikely(!skb)) + goto out; + __skb_put(skb, gl->tot_len); + skb_copy_to_linear_data(skb, gl->va, gl->tot_len); + } else { + skb = alloc_skb(skb_len, GFP_ATOMIC); + if (unlikely(!skb)) + goto out; + __skb_put(skb, pull_len); + skb_copy_to_linear_data(skb, gl->va, pull_len); + + copy_frags(skb, gl, pull_len); + skb->len = gl->tot_len; + skb->data_len = skb->len - pull_len; + skb->truesize += skb->data_len; + } + +out: + return skb; +} + +/** + * t4vf_pktgl_free - free a packet gather list + * @gl: the gather list + * + * Releases the pages of a packet gather list. We do not own the last + * page on the list and do not free it. + */ +static void t4vf_pktgl_free(const struct pkt_gl *gl) +{ + int frag; + + frag = gl->nfrags - 1; + while (frag--) + put_page(gl->frags[frag].page); +} + +/** + * do_gro - perform Generic Receive Offload ingress packet processing + * @rxq: ingress RX Ethernet Queue + * @gl: gather list for ingress packet + * @pkt: CPL header for last packet fragment + * + * Perform Generic Receive Offload (GRO) ingress packet processing. + * We use the standard Linux GRO interfaces for this. + */ +static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl, + const struct cpl_rx_pkt *pkt) +{ + struct adapter *adapter = rxq->rspq.adapter; + struct sge *s = &adapter->sge; + struct port_info *pi; + int ret; + struct sk_buff *skb; + + skb = napi_get_frags(&rxq->rspq.napi); + if (unlikely(!skb)) { + t4vf_pktgl_free(gl); + rxq->stats.rx_drops++; + return; + } + + copy_frags(skb, gl, s->pktshift); + skb->len = gl->tot_len - s->pktshift; + skb->data_len = skb->len; + skb->truesize += skb->data_len; + skb->ip_summed = CHECKSUM_UNNECESSARY; + skb_record_rx_queue(skb, rxq->rspq.idx); + pi = netdev_priv(skb->dev); + + if (pkt->vlan_ex && !pi->vlan_id) { + __vlan_hwaccel_put_tag(skb, cpu_to_be16(ETH_P_8021Q), + be16_to_cpu(pkt->vlan)); + rxq->stats.vlan_ex++; + } + ret = napi_gro_frags(&rxq->rspq.napi); + + if (ret == GRO_HELD) + rxq->stats.lro_pkts++; + else if (ret == GRO_MERGED || ret == GRO_MERGED_FREE) + rxq->stats.lro_merged++; + rxq->stats.pkts++; + rxq->stats.rx_cso++; +} + +/** + * t4vf_ethrx_handler - process an ingress ethernet packet + * @rspq: the response queue that received the packet + * @rsp: the response queue descriptor holding the RX_PKT message + * @gl: the gather list of packet fragments + * + * Process an ingress ethernet packet and deliver it to the stack. + */ +int t4vf_ethrx_handler(struct sge_rspq *rspq, const __be64 *rsp, + const struct pkt_gl *gl) +{ + struct sk_buff *skb; + const struct cpl_rx_pkt *pkt = (void *)rsp; + bool csum_ok = pkt->csum_calc && !pkt->err_vec && + (rspq->netdev->features & NETIF_F_RXCSUM); + struct sge_eth_rxq *rxq = container_of(rspq, struct sge_eth_rxq, rspq); + struct adapter *adapter = rspq->adapter; + struct sge *s = &adapter->sge; + struct port_info *pi; + + /* + * If this is a good TCP packet and we have Generic Receive Offload + * enabled, handle the packet in the GRO path. + */ + if ((pkt->l2info & cpu_to_be32(RXF_TCP_F)) && + (rspq->netdev->features & NETIF_F_GRO) && csum_ok && + !pkt->ip_frag) { + do_gro(rxq, gl, pkt); + return 0; + } + + /* + * Convert the Packet Gather List into an skb. + */ + skb = t4vf_pktgl_to_skb(gl, RX_SKB_LEN, RX_PULL_LEN); + if (unlikely(!skb)) { + t4vf_pktgl_free(gl); + rxq->stats.rx_drops++; + return 0; + } + __skb_pull(skb, s->pktshift); + skb->protocol = eth_type_trans(skb, rspq->netdev); + skb_record_rx_queue(skb, rspq->idx); + pi = netdev_priv(skb->dev); + rxq->stats.pkts++; + + if (csum_ok && !pkt->err_vec && + (be32_to_cpu(pkt->l2info) & (RXF_UDP_F | RXF_TCP_F))) { + if (!pkt->ip_frag) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + rxq->stats.rx_cso++; + } else if (pkt->l2info & htonl(RXF_IP_F)) { + __sum16 c = (__force __sum16)pkt->csum; + skb->csum = csum_unfold(c); + skb->ip_summed = CHECKSUM_COMPLETE; + rxq->stats.rx_cso++; + } + } else + skb_checksum_none_assert(skb); + + if (pkt->vlan_ex && !pi->vlan_id) { + rxq->stats.vlan_ex++; + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), + be16_to_cpu(pkt->vlan)); + } + + netif_receive_skb(skb); + + return 0; +} + +/** + * is_new_response - check if a response is newly written + * @rc: the response control descriptor + * @rspq: the response queue + * + * Returns true if a response descriptor contains a yet unprocessed + * response. + */ +static inline bool is_new_response(const struct rsp_ctrl *rc, + const struct sge_rspq *rspq) +{ + return ((rc->type_gen >> RSPD_GEN_S) & 0x1) == rspq->gen; +} + +/** + * restore_rx_bufs - put back a packet's RX buffers + * @gl: the packet gather list + * @fl: the SGE Free List + * @frags: how many fragments in @si + * + * Called when we find out that the current packet, @si, can't be + * processed right away for some reason. This is a very rare event and + * there's no effort to make this suspension/resumption process + * particularly efficient. + * + * We implement the suspension by putting all of the RX buffers associated + * with the current packet back on the original Free List. The buffers + * have already been unmapped and are left unmapped, we mark them as + * unmapped in order to prevent further unmapping attempts. (Effectively + * this function undoes the series of @unmap_rx_buf calls which were done + * to create the current packet's gather list.) This leaves us ready to + * restart processing of the packet the next time we start processing the + * RX Queue ... + */ +static void restore_rx_bufs(const struct pkt_gl *gl, struct sge_fl *fl, + int frags) +{ + struct rx_sw_desc *sdesc; + + while (frags--) { + if (fl->cidx == 0) + fl->cidx = fl->size - 1; + else + fl->cidx--; + sdesc = &fl->sdesc[fl->cidx]; + sdesc->page = gl->frags[frags].page; + sdesc->dma_addr |= RX_UNMAPPED_BUF; + fl->avail++; + } +} + +/** + * rspq_next - advance to the next entry in a response queue + * @rspq: the queue + * + * Updates the state of a response queue to advance it to the next entry. + */ +static inline void rspq_next(struct sge_rspq *rspq) +{ + rspq->cur_desc = (void *)rspq->cur_desc + rspq->iqe_len; + if (unlikely(++rspq->cidx == rspq->size)) { + rspq->cidx = 0; + rspq->gen ^= 1; + rspq->cur_desc = rspq->desc; + } +} + +/** + * process_responses - process responses from an SGE response queue + * @rspq: the ingress response queue to process + * @budget: how many responses can be processed in this round + * + * Process responses from a Scatter Gather Engine response queue up to + * the supplied budget. Responses include received packets as well as + * control messages from firmware or hardware. + * + * Additionally choose the interrupt holdoff time for the next interrupt + * on this queue. If the system is under memory shortage use a fairly + * long delay to help recovery. + */ +static int process_responses(struct sge_rspq *rspq, int budget) +{ + struct sge_eth_rxq *rxq = container_of(rspq, struct sge_eth_rxq, rspq); + struct adapter *adapter = rspq->adapter; + struct sge *s = &adapter->sge; + int budget_left = budget; + + while (likely(budget_left)) { + int ret, rsp_type; + const struct rsp_ctrl *rc; + + rc = (void *)rspq->cur_desc + (rspq->iqe_len - sizeof(*rc)); + if (!is_new_response(rc, rspq)) + break; + + /* + * Figure out what kind of response we've received from the + * SGE. + */ + dma_rmb(); + rsp_type = RSPD_TYPE_G(rc->type_gen); + if (likely(rsp_type == RSPD_TYPE_FLBUF_X)) { + struct page_frag *fp; + struct pkt_gl gl; + const struct rx_sw_desc *sdesc; + u32 bufsz, frag; + u32 len = be32_to_cpu(rc->pldbuflen_qid); + + /* + * If we get a "new buffer" message from the SGE we + * need to move on to the next Free List buffer. + */ + if (len & RSPD_NEWBUF_F) { + /* + * We get one "new buffer" message when we + * first start up a queue so we need to ignore + * it when our offset into the buffer is 0. + */ + if (likely(rspq->offset > 0)) { + free_rx_bufs(rspq->adapter, &rxq->fl, + 1); + rspq->offset = 0; + } + len = RSPD_LEN_G(len); + } + gl.tot_len = len; + + /* + * Gather packet fragments. + */ + for (frag = 0, fp = gl.frags; /**/; frag++, fp++) { + BUG_ON(frag >= MAX_SKB_FRAGS); + BUG_ON(rxq->fl.avail == 0); + sdesc = &rxq->fl.sdesc[rxq->fl.cidx]; + bufsz = get_buf_size(adapter, sdesc); + fp->page = sdesc->page; + fp->offset = rspq->offset; + fp->size = min(bufsz, len); + len -= fp->size; + if (!len) + break; + unmap_rx_buf(rspq->adapter, &rxq->fl); + } + gl.nfrags = frag+1; + + /* + * Last buffer remains mapped so explicitly make it + * coherent for CPU access and start preloading first + * cache line ... + */ + dma_sync_single_for_cpu(rspq->adapter->pdev_dev, + get_buf_addr(sdesc), + fp->size, DMA_FROM_DEVICE); + gl.va = (page_address(gl.frags[0].page) + + gl.frags[0].offset); + prefetch(gl.va); + + /* + * Hand the new ingress packet to the handler for + * this Response Queue. + */ + ret = rspq->handler(rspq, rspq->cur_desc, &gl); + if (likely(ret == 0)) + rspq->offset += ALIGN(fp->size, s->fl_align); + else + restore_rx_bufs(&gl, &rxq->fl, frag); + } else if (likely(rsp_type == RSPD_TYPE_CPL_X)) { + ret = rspq->handler(rspq, rspq->cur_desc, NULL); + } else { + WARN_ON(rsp_type > RSPD_TYPE_CPL_X); + ret = 0; + } + + if (unlikely(ret)) { + /* + * Couldn't process descriptor, back off for recovery. + * We use the SGE's last timer which has the longest + * interrupt coalescing value ... + */ + const int NOMEM_TIMER_IDX = SGE_NTIMERS-1; + rspq->next_intr_params = + QINTR_TIMER_IDX_V(NOMEM_TIMER_IDX); + break; + } + + rspq_next(rspq); + budget_left--; + } + + /* + * If this is a Response Queue with an associated Free List and + * at least two Egress Queue units available in the Free List + * for new buffer pointers, refill the Free List. + */ + if (rspq->offset >= 0 && + fl_cap(&rxq->fl) - rxq->fl.avail >= 2*FL_PER_EQ_UNIT) + __refill_fl(rspq->adapter, &rxq->fl); + return budget - budget_left; +} + +/** + * napi_rx_handler - the NAPI handler for RX processing + * @napi: the napi instance + * @budget: how many packets we can process in this round + * + * Handler for new data events when using NAPI. This does not need any + * locking or protection from interrupts as data interrupts are off at + * this point and other adapter interrupts do not interfere (the latter + * in not a concern at all with MSI-X as non-data interrupts then have + * a separate handler). + */ +static int napi_rx_handler(struct napi_struct *napi, int budget) +{ + unsigned int intr_params; + struct sge_rspq *rspq = container_of(napi, struct sge_rspq, napi); + int work_done = process_responses(rspq, budget); + u32 val; + + if (likely(work_done < budget)) { + napi_complete_done(napi, work_done); + intr_params = rspq->next_intr_params; + rspq->next_intr_params = rspq->intr_params; + } else + intr_params = QINTR_TIMER_IDX_V(SGE_TIMER_UPD_CIDX); + + if (unlikely(work_done == 0)) + rspq->unhandled_irqs++; + + val = CIDXINC_V(work_done) | SEINTARM_V(intr_params); + /* If we don't have access to the new User GTS (T5+), use the old + * doorbell mechanism; otherwise use the new BAR2 mechanism. + */ + if (unlikely(!rspq->bar2_addr)) { + t4_write_reg(rspq->adapter, + T4VF_SGE_BASE_ADDR + SGE_VF_GTS, + val | INGRESSQID_V((u32)rspq->cntxt_id)); + } else { + writel(val | INGRESSQID_V(rspq->bar2_qid), + rspq->bar2_addr + SGE_UDB_GTS); + wmb(); + } + return work_done; +} + +/* + * The MSI-X interrupt handler for an SGE response queue for the NAPI case + * (i.e., response queue serviced by NAPI polling). + */ +irqreturn_t t4vf_sge_intr_msix(int irq, void *cookie) +{ + struct sge_rspq *rspq = cookie; + + napi_schedule(&rspq->napi); + return IRQ_HANDLED; +} + +/* + * Process the indirect interrupt entries in the interrupt queue and kick off + * NAPI for each queue that has generated an entry. + */ +static unsigned int process_intrq(struct adapter *adapter) +{ + struct sge *s = &adapter->sge; + struct sge_rspq *intrq = &s->intrq; + unsigned int work_done; + u32 val; + + spin_lock(&adapter->sge.intrq_lock); + for (work_done = 0; ; work_done++) { + const struct rsp_ctrl *rc; + unsigned int qid, iq_idx; + struct sge_rspq *rspq; + + /* + * Grab the next response from the interrupt queue and bail + * out if it's not a new response. + */ + rc = (void *)intrq->cur_desc + (intrq->iqe_len - sizeof(*rc)); + if (!is_new_response(rc, intrq)) + break; + + /* + * If the response isn't a forwarded interrupt message issue a + * error and go on to the next response message. This should + * never happen ... + */ + dma_rmb(); + if (unlikely(RSPD_TYPE_G(rc->type_gen) != RSPD_TYPE_INTR_X)) { + dev_err(adapter->pdev_dev, + "Unexpected INTRQ response type %d\n", + RSPD_TYPE_G(rc->type_gen)); + continue; + } + + /* + * Extract the Queue ID from the interrupt message and perform + * sanity checking to make sure it really refers to one of our + * Ingress Queues which is active and matches the queue's ID. + * None of these error conditions should ever happen so we may + * want to either make them fatal and/or conditionalized under + * DEBUG. + */ + qid = RSPD_QID_G(be32_to_cpu(rc->pldbuflen_qid)); + iq_idx = IQ_IDX(s, qid); + if (unlikely(iq_idx >= MAX_INGQ)) { + dev_err(adapter->pdev_dev, + "Ingress QID %d out of range\n", qid); + continue; + } + rspq = s->ingr_map[iq_idx]; + if (unlikely(rspq == NULL)) { + dev_err(adapter->pdev_dev, + "Ingress QID %d RSPQ=NULL\n", qid); + continue; + } + if (unlikely(rspq->abs_id != qid)) { + dev_err(adapter->pdev_dev, + "Ingress QID %d refers to RSPQ %d\n", + qid, rspq->abs_id); + continue; + } + + /* + * Schedule NAPI processing on the indicated Response Queue + * and move on to the next entry in the Forwarded Interrupt + * Queue. + */ + napi_schedule(&rspq->napi); + rspq_next(intrq); + } + + val = CIDXINC_V(work_done) | SEINTARM_V(intrq->intr_params); + /* If we don't have access to the new User GTS (T5+), use the old + * doorbell mechanism; otherwise use the new BAR2 mechanism. + */ + if (unlikely(!intrq->bar2_addr)) { + t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS, + val | INGRESSQID_V(intrq->cntxt_id)); + } else { + writel(val | INGRESSQID_V(intrq->bar2_qid), + intrq->bar2_addr + SGE_UDB_GTS); + wmb(); + } + + spin_unlock(&adapter->sge.intrq_lock); + + return work_done; +} + +/* + * The MSI interrupt handler handles data events from SGE response queues as + * well as error and other async events as they all use the same MSI vector. + */ +static irqreturn_t t4vf_intr_msi(int irq, void *cookie) +{ + struct adapter *adapter = cookie; + + process_intrq(adapter); + return IRQ_HANDLED; +} + +/** + * t4vf_intr_handler - select the top-level interrupt handler + * @adapter: the adapter + * + * Selects the top-level interrupt handler based on the type of interrupts + * (MSI-X or MSI). + */ +irq_handler_t t4vf_intr_handler(struct adapter *adapter) +{ + BUG_ON((adapter->flags & + (CXGB4VF_USING_MSIX | CXGB4VF_USING_MSI)) == 0); + if (adapter->flags & CXGB4VF_USING_MSIX) + return t4vf_sge_intr_msix; + else + return t4vf_intr_msi; +} + +/** + * sge_rx_timer_cb - perform periodic maintenance of SGE RX queues + * @t: Rx timer + * + * Runs periodically from a timer to perform maintenance of SGE RX queues. + * + * a) Replenishes RX queues that have run out due to memory shortage. + * Normally new RX buffers are added when existing ones are consumed but + * when out of memory a queue can become empty. We schedule NAPI to do + * the actual refill. + */ +static void sge_rx_timer_cb(struct timer_list *t) +{ + struct adapter *adapter = from_timer(adapter, t, sge.rx_timer); + struct sge *s = &adapter->sge; + unsigned int i; + + /* + * Scan the "Starving Free Lists" flag array looking for any Free + * Lists in need of more free buffers. If we find one and it's not + * being actively polled, then bump its "starving" counter and attempt + * to refill it. If we're successful in adding enough buffers to push + * the Free List over the starving threshold, then we can clear its + * "starving" status. + */ + for (i = 0; i < ARRAY_SIZE(s->starving_fl); i++) { + unsigned long m; + + for (m = s->starving_fl[i]; m; m &= m - 1) { + unsigned int id = __ffs(m) + i * BITS_PER_LONG; + struct sge_fl *fl = s->egr_map[id]; + + clear_bit(id, s->starving_fl); + smp_mb__after_atomic(); + + /* + * Since we are accessing fl without a lock there's a + * small probability of a false positive where we + * schedule napi but the FL is no longer starving. + * No biggie. + */ + if (fl_starving(adapter, fl)) { + struct sge_eth_rxq *rxq; + + rxq = container_of(fl, struct sge_eth_rxq, fl); + if (napi_reschedule(&rxq->rspq.napi)) + fl->starving++; + else + set_bit(id, s->starving_fl); + } + } + } + + /* + * Reschedule the next scan for starving Free Lists ... + */ + mod_timer(&s->rx_timer, jiffies + RX_QCHECK_PERIOD); +} + +/** + * sge_tx_timer_cb - perform periodic maintenance of SGE Tx queues + * @t: Tx timer + * + * Runs periodically from a timer to perform maintenance of SGE TX queues. + * + * b) Reclaims completed Tx packets for the Ethernet queues. Normally + * packets are cleaned up by new Tx packets, this timer cleans up packets + * when no new packets are being submitted. This is essential for pktgen, + * at least. + */ +static void sge_tx_timer_cb(struct timer_list *t) +{ + struct adapter *adapter = from_timer(adapter, t, sge.tx_timer); + struct sge *s = &adapter->sge; + unsigned int i, budget; + + budget = MAX_TIMER_TX_RECLAIM; + i = s->ethtxq_rover; + do { + struct sge_eth_txq *txq = &s->ethtxq[i]; + + if (reclaimable(&txq->q) && __netif_tx_trylock(txq->txq)) { + int avail = reclaimable(&txq->q); + + if (avail > budget) + avail = budget; + + free_tx_desc(adapter, &txq->q, avail, true); + txq->q.in_use -= avail; + __netif_tx_unlock(txq->txq); + + budget -= avail; + if (!budget) + break; + } + + i++; + if (i >= s->ethqsets) + i = 0; + } while (i != s->ethtxq_rover); + s->ethtxq_rover = i; + + /* + * If we found too many reclaimable packets schedule a timer in the + * near future to continue where we left off. Otherwise the next timer + * will be at its normal interval. + */ + mod_timer(&s->tx_timer, jiffies + (budget ? TX_QCHECK_PERIOD : 2)); +} + +/** + * bar2_address - return the BAR2 address for an SGE Queue's Registers + * @adapter: the adapter + * @qid: the SGE Queue ID + * @qtype: the SGE Queue Type (Egress or Ingress) + * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues + * + * Returns the BAR2 address for the SGE Queue Registers associated with + * @qid. If BAR2 SGE Registers aren't available, returns NULL. Also + * returns the BAR2 Queue ID to be used with writes to the BAR2 SGE + * Queue Registers. If the BAR2 Queue ID is 0, then "Inferred Queue ID" + * Registers are supported (e.g. the Write Combining Doorbell Buffer). + */ +static void __iomem *bar2_address(struct adapter *adapter, + unsigned int qid, + enum t4_bar2_qtype qtype, + unsigned int *pbar2_qid) +{ + u64 bar2_qoffset; + int ret; + + ret = t4vf_bar2_sge_qregs(adapter, qid, qtype, + &bar2_qoffset, pbar2_qid); + if (ret) + return NULL; + + return adapter->bar2 + bar2_qoffset; +} + +/** + * t4vf_sge_alloc_rxq - allocate an SGE RX Queue + * @adapter: the adapter + * @rspq: pointer to to the new rxq's Response Queue to be filled in + * @iqasynch: if 0, a normal rspq; if 1, an asynchronous event queue + * @dev: the network device associated with the new rspq + * @intr_dest: MSI-X vector index (overriden in MSI mode) + * @fl: pointer to the new rxq's Free List to be filled in + * @hnd: the interrupt handler to invoke for the rspq + */ +int t4vf_sge_alloc_rxq(struct adapter *adapter, struct sge_rspq *rspq, + bool iqasynch, struct net_device *dev, + int intr_dest, + struct sge_fl *fl, rspq_handler_t hnd) +{ + struct sge *s = &adapter->sge; + struct port_info *pi = netdev_priv(dev); + struct fw_iq_cmd cmd, rpl; + int ret, iqandst, flsz = 0; + int relaxed = !(adapter->flags & CXGB4VF_ROOT_NO_RELAXED_ORDERING); + + /* + * If we're using MSI interrupts and we're not initializing the + * Forwarded Interrupt Queue itself, then set up this queue for + * indirect interrupts to the Forwarded Interrupt Queue. Obviously + * the Forwarded Interrupt Queue must be set up before any other + * ingress queue ... + */ + if ((adapter->flags & CXGB4VF_USING_MSI) && + rspq != &adapter->sge.intrq) { + iqandst = SGE_INTRDST_IQ; + intr_dest = adapter->sge.intrq.abs_id; + } else + iqandst = SGE_INTRDST_PCI; + + /* + * Allocate the hardware ring for the Response Queue. The size needs + * to be a multiple of 16 which includes the mandatory status entry + * (regardless of whether the Status Page capabilities are enabled or + * not). + */ + rspq->size = roundup(rspq->size, 16); + rspq->desc = alloc_ring(adapter->pdev_dev, rspq->size, rspq->iqe_len, + 0, &rspq->phys_addr, NULL, 0); + if (!rspq->desc) + return -ENOMEM; + + /* + * Fill in the Ingress Queue Command. Note: Ideally this code would + * be in t4vf_hw.c but there are so many parameters and dependencies + * on our Linux SGE state that we would end up having to pass tons of + * parameters. We'll have to think about how this might be migrated + * into OS-independent common code ... + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_IQ_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_CMD_EXEC_F); + cmd.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_ALLOC_F | + FW_IQ_CMD_IQSTART_F | + FW_LEN16(cmd)); + cmd.type_to_iqandstindex = + cpu_to_be32(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP) | + FW_IQ_CMD_IQASYNCH_V(iqasynch) | + FW_IQ_CMD_VIID_V(pi->viid) | + FW_IQ_CMD_IQANDST_V(iqandst) | + FW_IQ_CMD_IQANUS_V(1) | + FW_IQ_CMD_IQANUD_V(SGE_UPDATEDEL_INTR) | + FW_IQ_CMD_IQANDSTINDEX_V(intr_dest)); + cmd.iqdroprss_to_iqesize = + cpu_to_be16(FW_IQ_CMD_IQPCIECH_V(pi->port_id) | + FW_IQ_CMD_IQGTSMODE_F | + FW_IQ_CMD_IQINTCNTTHRESH_V(rspq->pktcnt_idx) | + FW_IQ_CMD_IQESIZE_V(ilog2(rspq->iqe_len) - 4)); + cmd.iqsize = cpu_to_be16(rspq->size); + cmd.iqaddr = cpu_to_be64(rspq->phys_addr); + + if (fl) { + unsigned int chip_ver = + CHELSIO_CHIP_VERSION(adapter->params.chip); + /* + * Allocate the ring for the hardware free list (with space + * for its status page) along with the associated software + * descriptor ring. The free list size needs to be a multiple + * of the Egress Queue Unit and at least 2 Egress Units larger + * than the SGE's Egress Congrestion Threshold + * (fl_starve_thres - 1). + */ + if (fl->size < s->fl_starve_thres - 1 + 2 * FL_PER_EQ_UNIT) + fl->size = s->fl_starve_thres - 1 + 2 * FL_PER_EQ_UNIT; + fl->size = roundup(fl->size, FL_PER_EQ_UNIT); + fl->desc = alloc_ring(adapter->pdev_dev, fl->size, + sizeof(__be64), sizeof(struct rx_sw_desc), + &fl->addr, &fl->sdesc, s->stat_len); + if (!fl->desc) { + ret = -ENOMEM; + goto err; + } + + /* + * Calculate the size of the hardware free list ring plus + * Status Page (which the SGE will place after the end of the + * free list ring) in Egress Queue Units. + */ + flsz = (fl->size / FL_PER_EQ_UNIT + + s->stat_len / EQ_UNIT); + + /* + * Fill in all the relevant firmware Ingress Queue Command + * fields for the free list. + */ + cmd.iqns_to_fl0congen = + cpu_to_be32( + FW_IQ_CMD_FL0HOSTFCMODE_V(SGE_HOSTFCMODE_NONE) | + FW_IQ_CMD_FL0PACKEN_F | + FW_IQ_CMD_FL0FETCHRO_V(relaxed) | + FW_IQ_CMD_FL0DATARO_V(relaxed) | + FW_IQ_CMD_FL0PADEN_F); + + /* In T6, for egress queue type FL there is internal overhead + * of 16B for header going into FLM module. Hence the maximum + * allowed burst size is 448 bytes. For T4/T5, the hardware + * doesn't coalesce fetch requests if more than 64 bytes of + * Free List pointers are provided, so we use a 128-byte Fetch + * Burst Minimum there (T6 implements coalescing so we can use + * the smaller 64-byte value there). + */ + cmd.fl0dcaen_to_fl0cidxfthresh = + cpu_to_be16( + FW_IQ_CMD_FL0FBMIN_V(chip_ver <= CHELSIO_T5 + ? FETCHBURSTMIN_128B_X + : FETCHBURSTMIN_64B_T6_X) | + FW_IQ_CMD_FL0FBMAX_V((chip_ver <= CHELSIO_T5) ? + FETCHBURSTMAX_512B_X : + FETCHBURSTMAX_256B_X)); + cmd.fl0size = cpu_to_be16(flsz); + cmd.fl0addr = cpu_to_be64(fl->addr); + } + + /* + * Issue the firmware Ingress Queue Command and extract the results if + * it completes successfully. + */ + ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl); + if (ret) + goto err; + + netif_napi_add(dev, &rspq->napi, napi_rx_handler); + rspq->cur_desc = rspq->desc; + rspq->cidx = 0; + rspq->gen = 1; + rspq->next_intr_params = rspq->intr_params; + rspq->cntxt_id = be16_to_cpu(rpl.iqid); + rspq->bar2_addr = bar2_address(adapter, + rspq->cntxt_id, + T4_BAR2_QTYPE_INGRESS, + &rspq->bar2_qid); + rspq->abs_id = be16_to_cpu(rpl.physiqid); + rspq->size--; /* subtract status entry */ + rspq->adapter = adapter; + rspq->netdev = dev; + rspq->handler = hnd; + + /* set offset to -1 to distinguish ingress queues without FL */ + rspq->offset = fl ? 0 : -1; + + if (fl) { + fl->cntxt_id = be16_to_cpu(rpl.fl0id); + fl->avail = 0; + fl->pend_cred = 0; + fl->pidx = 0; + fl->cidx = 0; + fl->alloc_failed = 0; + fl->large_alloc_failed = 0; + fl->starving = 0; + + /* Note, we must initialize the BAR2 Free List User Doorbell + * information before refilling the Free List! + */ + fl->bar2_addr = bar2_address(adapter, + fl->cntxt_id, + T4_BAR2_QTYPE_EGRESS, + &fl->bar2_qid); + + refill_fl(adapter, fl, fl_cap(fl), GFP_KERNEL); + } + + return 0; + +err: + /* + * An error occurred. Clean up our partial allocation state and + * return the error. + */ + if (rspq->desc) { + dma_free_coherent(adapter->pdev_dev, rspq->size * rspq->iqe_len, + rspq->desc, rspq->phys_addr); + rspq->desc = NULL; + } + if (fl && fl->desc) { + kfree(fl->sdesc); + fl->sdesc = NULL; + dma_free_coherent(adapter->pdev_dev, flsz * EQ_UNIT, + fl->desc, fl->addr); + fl->desc = NULL; + } + return ret; +} + +/** + * t4vf_sge_alloc_eth_txq - allocate an SGE Ethernet TX Queue + * @adapter: the adapter + * @txq: pointer to the new txq to be filled in + * @dev: the network device + * @devq: the network TX queue associated with the new txq + * @iqid: the relative ingress queue ID to which events relating to + * the new txq should be directed + */ +int t4vf_sge_alloc_eth_txq(struct adapter *adapter, struct sge_eth_txq *txq, + struct net_device *dev, struct netdev_queue *devq, + unsigned int iqid) +{ + unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip); + struct port_info *pi = netdev_priv(dev); + struct fw_eq_eth_cmd cmd, rpl; + struct sge *s = &adapter->sge; + int ret, nentries; + + /* + * Calculate the size of the hardware TX Queue (including the Status + * Page on the end of the TX Queue) in units of TX Descriptors. + */ + nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc); + + /* + * Allocate the hardware ring for the TX ring (with space for its + * status page) along with the associated software descriptor ring. + */ + txq->q.desc = alloc_ring(adapter->pdev_dev, txq->q.size, + sizeof(struct tx_desc), + sizeof(struct tx_sw_desc), + &txq->q.phys_addr, &txq->q.sdesc, s->stat_len); + if (!txq->q.desc) + return -ENOMEM; + + /* + * Fill in the Egress Queue Command. Note: As with the direct use of + * the firmware Ingress Queue COmmand above in our RXQ allocation + * routine, ideally, this code would be in t4vf_hw.c. Again, we'll + * have to see if there's some reasonable way to parameterize it + * into the common code ... + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_ETH_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_CMD_EXEC_F); + cmd.alloc_to_len16 = cpu_to_be32(FW_EQ_ETH_CMD_ALLOC_F | + FW_EQ_ETH_CMD_EQSTART_F | + FW_LEN16(cmd)); + cmd.autoequiqe_to_viid = cpu_to_be32(FW_EQ_ETH_CMD_AUTOEQUEQE_F | + FW_EQ_ETH_CMD_VIID_V(pi->viid)); + cmd.fetchszm_to_iqid = + cpu_to_be32(FW_EQ_ETH_CMD_HOSTFCMODE_V(SGE_HOSTFCMODE_STPG) | + FW_EQ_ETH_CMD_PCIECHN_V(pi->port_id) | + FW_EQ_ETH_CMD_IQID_V(iqid)); + cmd.dcaen_to_eqsize = + cpu_to_be32(FW_EQ_ETH_CMD_FBMIN_V(chip_ver <= CHELSIO_T5 + ? FETCHBURSTMIN_64B_X + : FETCHBURSTMIN_64B_T6_X) | + FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) | + FW_EQ_ETH_CMD_CIDXFTHRESH_V( + CIDXFLUSHTHRESH_32_X) | + FW_EQ_ETH_CMD_EQSIZE_V(nentries)); + cmd.eqaddr = cpu_to_be64(txq->q.phys_addr); + + /* + * Issue the firmware Egress Queue Command and extract the results if + * it completes successfully. + */ + ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl); + if (ret) { + /* + * The girmware Ingress Queue Command failed for some reason. + * Free up our partial allocation state and return the error. + */ + kfree(txq->q.sdesc); + txq->q.sdesc = NULL; + dma_free_coherent(adapter->pdev_dev, + nentries * sizeof(struct tx_desc), + txq->q.desc, txq->q.phys_addr); + txq->q.desc = NULL; + return ret; + } + + txq->q.in_use = 0; + txq->q.cidx = 0; + txq->q.pidx = 0; + txq->q.stat = (void *)&txq->q.desc[txq->q.size]; + txq->q.cntxt_id = FW_EQ_ETH_CMD_EQID_G(be32_to_cpu(rpl.eqid_pkd)); + txq->q.bar2_addr = bar2_address(adapter, + txq->q.cntxt_id, + T4_BAR2_QTYPE_EGRESS, + &txq->q.bar2_qid); + txq->q.abs_id = + FW_EQ_ETH_CMD_PHYSEQID_G(be32_to_cpu(rpl.physeqid_pkd)); + txq->txq = devq; + txq->tso = 0; + txq->tx_cso = 0; + txq->vlan_ins = 0; + txq->q.stops = 0; + txq->q.restarts = 0; + txq->mapping_err = 0; + return 0; +} + +/* + * Free the DMA map resources associated with a TX queue. + */ +static void free_txq(struct adapter *adapter, struct sge_txq *tq) +{ + struct sge *s = &adapter->sge; + + dma_free_coherent(adapter->pdev_dev, + tq->size * sizeof(*tq->desc) + s->stat_len, + tq->desc, tq->phys_addr); + tq->cntxt_id = 0; + tq->sdesc = NULL; + tq->desc = NULL; +} + +/* + * Free the resources associated with a response queue (possibly including a + * free list). + */ +static void free_rspq_fl(struct adapter *adapter, struct sge_rspq *rspq, + struct sge_fl *fl) +{ + struct sge *s = &adapter->sge; + unsigned int flid = fl ? fl->cntxt_id : 0xffff; + + t4vf_iq_free(adapter, FW_IQ_TYPE_FL_INT_CAP, + rspq->cntxt_id, flid, 0xffff); + dma_free_coherent(adapter->pdev_dev, (rspq->size + 1) * rspq->iqe_len, + rspq->desc, rspq->phys_addr); + netif_napi_del(&rspq->napi); + rspq->netdev = NULL; + rspq->cntxt_id = 0; + rspq->abs_id = 0; + rspq->desc = NULL; + + if (fl) { + free_rx_bufs(adapter, fl, fl->avail); + dma_free_coherent(adapter->pdev_dev, + fl->size * sizeof(*fl->desc) + s->stat_len, + fl->desc, fl->addr); + kfree(fl->sdesc); + fl->sdesc = NULL; + fl->cntxt_id = 0; + fl->desc = NULL; + } +} + +/** + * t4vf_free_sge_resources - free SGE resources + * @adapter: the adapter + * + * Frees resources used by the SGE queue sets. + */ +void t4vf_free_sge_resources(struct adapter *adapter) +{ + struct sge *s = &adapter->sge; + struct sge_eth_rxq *rxq = s->ethrxq; + struct sge_eth_txq *txq = s->ethtxq; + struct sge_rspq *evtq = &s->fw_evtq; + struct sge_rspq *intrq = &s->intrq; + int qs; + + for (qs = 0; qs < adapter->sge.ethqsets; qs++, rxq++, txq++) { + if (rxq->rspq.desc) + free_rspq_fl(adapter, &rxq->rspq, &rxq->fl); + if (txq->q.desc) { + t4vf_eth_eq_free(adapter, txq->q.cntxt_id); + free_tx_desc(adapter, &txq->q, txq->q.in_use, true); + kfree(txq->q.sdesc); + free_txq(adapter, &txq->q); + } + } + if (evtq->desc) + free_rspq_fl(adapter, evtq, NULL); + if (intrq->desc) + free_rspq_fl(adapter, intrq, NULL); +} + +/** + * t4vf_sge_start - enable SGE operation + * @adapter: the adapter + * + * Start tasklets and timers associated with the DMA engine. + */ +void t4vf_sge_start(struct adapter *adapter) +{ + adapter->sge.ethtxq_rover = 0; + mod_timer(&adapter->sge.rx_timer, jiffies + RX_QCHECK_PERIOD); + mod_timer(&adapter->sge.tx_timer, jiffies + TX_QCHECK_PERIOD); +} + +/** + * t4vf_sge_stop - disable SGE operation + * @adapter: the adapter + * + * Stop tasklets and timers associated with the DMA engine. Note that + * this is effective only if measures have been taken to disable any HW + * events that may restart them. + */ +void t4vf_sge_stop(struct adapter *adapter) +{ + struct sge *s = &adapter->sge; + + if (s->rx_timer.function) + del_timer_sync(&s->rx_timer); + if (s->tx_timer.function) + del_timer_sync(&s->tx_timer); +} + +/** + * t4vf_sge_init - initialize SGE + * @adapter: the adapter + * + * Performs SGE initialization needed every time after a chip reset. + * We do not initialize any of the queue sets here, instead the driver + * top-level must request those individually. We also do not enable DMA + * here, that should be done after the queues have been set up. + */ +int t4vf_sge_init(struct adapter *adapter) +{ + struct sge_params *sge_params = &adapter->params.sge; + u32 fl_small_pg = sge_params->sge_fl_buffer_size[0]; + u32 fl_large_pg = sge_params->sge_fl_buffer_size[1]; + struct sge *s = &adapter->sge; + + /* + * Start by vetting the basic SGE parameters which have been set up by + * the Physical Function Driver. Ideally we should be able to deal + * with _any_ configuration. Practice is different ... + */ + + /* We only bother using the Large Page logic if the Large Page Buffer + * is larger than our Page Size Buffer. + */ + if (fl_large_pg <= fl_small_pg) + fl_large_pg = 0; + + /* The Page Size Buffer must be exactly equal to our Page Size and the + * Large Page Size Buffer should be 0 (per above) or a power of 2. + */ + if (fl_small_pg != PAGE_SIZE || + (fl_large_pg & (fl_large_pg - 1)) != 0) { + dev_err(adapter->pdev_dev, "bad SGE FL buffer sizes [%d, %d]\n", + fl_small_pg, fl_large_pg); + return -EINVAL; + } + if ((sge_params->sge_control & RXPKTCPLMODE_F) != + RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X)) { + dev_err(adapter->pdev_dev, "bad SGE CPL MODE\n"); + return -EINVAL; + } + + /* + * Now translate the adapter parameters into our internal forms. + */ + if (fl_large_pg) + s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT; + s->stat_len = ((sge_params->sge_control & EGRSTATUSPAGESIZE_F) + ? 128 : 64); + s->pktshift = PKTSHIFT_G(sge_params->sge_control); + s->fl_align = t4vf_fl_pkt_align(adapter); + + /* A FL with <= fl_starve_thres buffers is starving and a periodic + * timer will attempt to refill it. This needs to be larger than the + * SGE's Egress Congestion Threshold. If it isn't, then we can get + * stuck waiting for new packets while the SGE is waiting for us to + * give it more Free List entries. (Note that the SGE's Egress + * Congestion Threshold is in units of 2 Free List pointers.) + */ + switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) { + case CHELSIO_T4: + s->fl_starve_thres = + EGRTHRESHOLD_G(sge_params->sge_congestion_control); + break; + case CHELSIO_T5: + s->fl_starve_thres = + EGRTHRESHOLDPACKING_G(sge_params->sge_congestion_control); + break; + case CHELSIO_T6: + default: + s->fl_starve_thres = + T6_EGRTHRESHOLDPACKING_G(sge_params->sge_congestion_control); + break; + } + s->fl_starve_thres = s->fl_starve_thres * 2 + 1; + + /* + * Set up tasklet timers. + */ + timer_setup(&s->rx_timer, sge_rx_timer_cb, 0); + timer_setup(&s->tx_timer, sge_tx_timer_cb, 0); + + /* + * Initialize Forwarded Interrupt Queue lock. + */ + spin_lock_init(&s->intrq_lock); + + return 0; +} diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h new file mode 100644 index 000000000..03777145e --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h @@ -0,0 +1,422 @@ +/* + * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet + * driver for Linux. + * + * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __T4VF_COMMON_H__ +#define __T4VF_COMMON_H__ + +#include "../cxgb4/t4_hw.h" +#include "../cxgb4/t4fw_api.h" + +#define CHELSIO_CHIP_CODE(version, revision) (((version) << 4) | (revision)) +#define CHELSIO_CHIP_VERSION(code) (((code) >> 4) & 0xf) +#define CHELSIO_CHIP_RELEASE(code) ((code) & 0xf) + +/* All T4 and later chips have their PCI-E Device IDs encoded as 0xVFPP where: + * + * V = "4" for T4; "5" for T5, etc. or + * = "a" for T4 FPGA; "b" for T4 FPGA, etc. + * F = "0" for PF 0..3; "4".."7" for PF4..7; and "8" for VFs + * PP = adapter product designation + */ +#define CHELSIO_T4 0x4 +#define CHELSIO_T5 0x5 +#define CHELSIO_T6 0x6 + +enum chip_type { + T4_A1 = CHELSIO_CHIP_CODE(CHELSIO_T4, 1), + T4_A2 = CHELSIO_CHIP_CODE(CHELSIO_T4, 2), + T4_FIRST_REV = T4_A1, + T4_LAST_REV = T4_A2, + + T5_A0 = CHELSIO_CHIP_CODE(CHELSIO_T5, 0), + T5_A1 = CHELSIO_CHIP_CODE(CHELSIO_T5, 1), + T5_FIRST_REV = T5_A0, + T5_LAST_REV = T5_A1, +}; + +/* + * The "len16" field of a Firmware Command Structure ... + */ +#define FW_LEN16(fw_struct) FW_CMD_LEN16_V(sizeof(fw_struct) / 16) + +/* + * Per-VF statistics. + */ +struct t4vf_port_stats { + /* + * TX statistics. + */ + u64 tx_bcast_bytes; /* broadcast */ + u64 tx_bcast_frames; + u64 tx_mcast_bytes; /* multicast */ + u64 tx_mcast_frames; + u64 tx_ucast_bytes; /* unicast */ + u64 tx_ucast_frames; + u64 tx_drop_frames; /* TX dropped frames */ + u64 tx_offload_bytes; /* offload */ + u64 tx_offload_frames; + + /* + * RX statistics. + */ + u64 rx_bcast_bytes; /* broadcast */ + u64 rx_bcast_frames; + u64 rx_mcast_bytes; /* multicast */ + u64 rx_mcast_frames; + u64 rx_ucast_bytes; + u64 rx_ucast_frames; /* unicast */ + + u64 rx_err_frames; /* RX error frames */ +}; + +/* + * Per-"port" (Virtual Interface) link configuration ... + */ +typedef u16 fw_port_cap16_t; /* 16-bit Port Capabilities integral value */ +typedef u32 fw_port_cap32_t; /* 32-bit Port Capabilities integral value */ + +enum fw_caps { + FW_CAPS_UNKNOWN = 0, /* 0'ed out initial state */ + FW_CAPS16 = 1, /* old Firmware: 16-bit Port Capabilities */ + FW_CAPS32 = 2, /* new Firmware: 32-bit Port Capabilities */ +}; + +enum cc_pause { + PAUSE_RX = 1 << 0, + PAUSE_TX = 1 << 1, + PAUSE_AUTONEG = 1 << 2 +}; + +enum cc_fec { + FEC_AUTO = 1 << 0, /* IEEE 802.3 "automatic" */ + FEC_RS = 1 << 1, /* Reed-Solomon */ + FEC_BASER_RS = 1 << 2, /* BaseR/Reed-Solomon */ +}; + +struct link_config { + fw_port_cap32_t pcaps; /* link capabilities */ + fw_port_cap32_t acaps; /* advertised capabilities */ + fw_port_cap32_t lpacaps; /* peer advertised capabilities */ + + fw_port_cap32_t speed_caps; /* speed(s) user has requested */ + u32 speed; /* actual link speed */ + + enum cc_pause requested_fc; /* flow control user has requested */ + enum cc_pause fc; /* actual link flow control */ + enum cc_pause advertised_fc; /* actual advertised flow control */ + + enum cc_fec auto_fec; /* Forward Error Correction: */ + enum cc_fec requested_fec; /* "automatic" (IEEE 802.3), */ + enum cc_fec fec; /* requested, and actual in use */ + + unsigned char autoneg; /* autonegotiating? */ + + unsigned char link_ok; /* link up? */ + unsigned char link_down_rc; /* link down reason */ +}; + +/* Return true if the Link Configuration supports "High Speeds" (those greater + * than 1Gb/s). + */ +static inline bool is_x_10g_port(const struct link_config *lc) +{ + fw_port_cap32_t speeds, high_speeds; + + speeds = FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_G(lc->pcaps)); + high_speeds = + speeds & ~(FW_PORT_CAP32_SPEED_100M | FW_PORT_CAP32_SPEED_1G); + + return high_speeds != 0; +} + +/* + * General device parameters ... + */ +struct dev_params { + u32 fwrev; /* firmware version */ + u32 tprev; /* TP Microcode Version */ +}; + +/* + * Scatter Gather Engine parameters. These are almost all determined by the + * Physical Function Driver. We just need to grab them to see within which + * environment we're playing ... + */ +struct sge_params { + u32 sge_control; /* padding, boundaries, lengths, etc. */ + u32 sge_control2; /* T5: more of the same */ + u32 sge_host_page_size; /* PF0-7 page sizes */ + u32 sge_egress_queues_per_page; /* PF0-7 egress queues/page */ + u32 sge_ingress_queues_per_page;/* PF0-7 ingress queues/page */ + u32 sge_vf_hps; /* host page size for our vf */ + u32 sge_vf_eq_qpp; /* egress queues/page for our VF */ + u32 sge_vf_iq_qpp; /* ingress queues/page for our VF */ + u32 sge_fl_buffer_size[16]; /* free list buffer sizes */ + u32 sge_ingress_rx_threshold; /* RX counter interrupt threshold[4] */ + u32 sge_congestion_control; /* congestion thresholds, etc. */ + u32 sge_timer_value_0_and_1; /* interrupt coalescing timer values */ + u32 sge_timer_value_2_and_3; + u32 sge_timer_value_4_and_5; +}; + +/* + * Vital Product Data parameters. + */ +struct vpd_params { + u32 cclk; /* Core Clock (KHz) */ +}; + +/* Stores chip specific parameters */ +struct arch_specific_params { + u32 sge_fl_db; + u16 mps_tcam_size; +}; + +/* + * Global Receive Side Scaling (RSS) parameters in host-native format. + */ +struct rss_params { + unsigned int mode; /* RSS mode */ + union { + struct { + unsigned int synmapen:1; /* SYN Map Enable */ + unsigned int syn4tupenipv6:1; /* enable hashing 4-tuple IPv6 SYNs */ + unsigned int syn2tupenipv6:1; /* enable hashing 2-tuple IPv6 SYNs */ + unsigned int syn4tupenipv4:1; /* enable hashing 4-tuple IPv4 SYNs */ + unsigned int syn2tupenipv4:1; /* enable hashing 2-tuple IPv4 SYNs */ + unsigned int ofdmapen:1; /* Offload Map Enable */ + unsigned int tnlmapen:1; /* Tunnel Map Enable */ + unsigned int tnlalllookup:1; /* Tunnel All Lookup */ + unsigned int hashtoeplitz:1; /* use Toeplitz hash */ + } basicvirtual; + } u; +}; + +/* + * Virtual Interface RSS Configuration in host-native format. + */ +union rss_vi_config { + struct { + u16 defaultq; /* Ingress Queue ID for !tnlalllookup */ + unsigned int ip6fourtupen:1; /* hash 4-tuple IPv6 ingress packets */ + unsigned int ip6twotupen:1; /* hash 2-tuple IPv6 ingress packets */ + unsigned int ip4fourtupen:1; /* hash 4-tuple IPv4 ingress packets */ + unsigned int ip4twotupen:1; /* hash 2-tuple IPv4 ingress packets */ + int udpen; /* hash 4-tuple UDP ingress packets */ + } basicvirtual; +}; + +/* + * Maximum resources provisioned for a PCI VF. + */ +struct vf_resources { + unsigned int nvi; /* N virtual interfaces */ + unsigned int neq; /* N egress Qs */ + unsigned int nethctrl; /* N egress ETH or CTRL Qs */ + unsigned int niqflint; /* N ingress Qs/w free list(s) & intr */ + unsigned int niq; /* N ingress Qs */ + unsigned int tc; /* PCI-E traffic class */ + unsigned int pmask; /* port access rights mask */ + unsigned int nexactf; /* N exact MPS filters */ + unsigned int r_caps; /* read capabilities */ + unsigned int wx_caps; /* write/execute capabilities */ +}; + +/* + * Per-"adapter" (Virtual Function) parameters. + */ +struct adapter_params { + struct dev_params dev; /* general device parameters */ + struct sge_params sge; /* Scatter Gather Engine */ + struct vpd_params vpd; /* Vital Product Data */ + struct rss_params rss; /* Receive Side Scaling */ + struct vf_resources vfres; /* Virtual Function Resource limits */ + struct arch_specific_params arch; /* chip specific params */ + enum chip_type chip; /* chip code */ + u8 nports; /* # of Ethernet "ports" */ + u8 fw_caps_support; /* 32-bit Port Capabilities */ +}; + +/* Firmware Mailbox Command/Reply log. All values are in Host-Endian format. + * The access and execute times are signed in order to accommodate negative + * error returns. + */ +struct mbox_cmd { + u64 cmd[MBOX_LEN / 8]; /* a Firmware Mailbox Command/Reply */ + u64 timestamp; /* OS-dependent timestamp */ + u32 seqno; /* sequence number */ + s16 access; /* time (ms) to access mailbox */ + s16 execute; /* time (ms) to execute */ +}; + +struct mbox_cmd_log { + unsigned int size; /* number of entries in the log */ + unsigned int cursor; /* next position in the log to write */ + u32 seqno; /* next sequence number */ + /* variable length mailbox command log starts here */ +}; + +/* Given a pointer to a Firmware Mailbox Command Log and a log entry index, + * return a pointer to the specified entry. + */ +static inline struct mbox_cmd *mbox_cmd_log_entry(struct mbox_cmd_log *log, + unsigned int entry_idx) +{ + return &((struct mbox_cmd *)&(log)[1])[entry_idx]; +} + +#include "adapter.h" + +#ifndef PCI_VENDOR_ID_CHELSIO +# define PCI_VENDOR_ID_CHELSIO 0x1425 +#endif + +#define for_each_port(adapter, iter) \ + for (iter = 0; iter < (adapter)->params.nports; iter++) + +static inline unsigned int core_ticks_per_usec(const struct adapter *adapter) +{ + return adapter->params.vpd.cclk / 1000; +} + +static inline unsigned int us_to_core_ticks(const struct adapter *adapter, + unsigned int us) +{ + return (us * adapter->params.vpd.cclk) / 1000; +} + +static inline unsigned int core_ticks_to_us(const struct adapter *adapter, + unsigned int ticks) +{ + return (ticks * 1000) / adapter->params.vpd.cclk; +} + +int t4vf_wr_mbox_core(struct adapter *, const void *, int, void *, bool); + +static inline int t4vf_wr_mbox(struct adapter *adapter, const void *cmd, + int size, void *rpl) +{ + return t4vf_wr_mbox_core(adapter, cmd, size, rpl, true); +} + +static inline int t4vf_wr_mbox_ns(struct adapter *adapter, const void *cmd, + int size, void *rpl) +{ + return t4vf_wr_mbox_core(adapter, cmd, size, rpl, false); +} + +#define CHELSIO_PCI_ID_VER(dev_id) ((dev_id) >> 12) + +static inline int is_t4(enum chip_type chip) +{ + return CHELSIO_CHIP_VERSION(chip) == CHELSIO_T4; +} + +/** + * hash_mac_addr - return the hash value of a MAC address + * @addr: the 48-bit Ethernet MAC address + * + * Hashes a MAC address according to the hash function used by hardware + * inexact (hash) address matching. + */ +static inline int hash_mac_addr(const u8 *addr) +{ + u32 a = ((u32)addr[0] << 16) | ((u32)addr[1] << 8) | addr[2]; + u32 b = ((u32)addr[3] << 16) | ((u32)addr[4] << 8) | addr[5]; + + a ^= b; + a ^= (a >> 12); + a ^= (a >> 6); + return a & 0x3f; +} + +int t4vf_wait_dev_ready(struct adapter *); +int t4vf_port_init(struct adapter *, int); + +int t4vf_fw_reset(struct adapter *); +int t4vf_set_params(struct adapter *, unsigned int, const u32 *, const u32 *); + +int t4vf_fl_pkt_align(struct adapter *adapter); +enum t4_bar2_qtype { T4_BAR2_QTYPE_EGRESS, T4_BAR2_QTYPE_INGRESS }; +int t4vf_bar2_sge_qregs(struct adapter *adapter, + unsigned int qid, + enum t4_bar2_qtype qtype, + u64 *pbar2_qoffset, + unsigned int *pbar2_qid); + +unsigned int t4vf_get_pf_from_vf(struct adapter *); +int t4vf_get_sge_params(struct adapter *); +int t4vf_get_vpd_params(struct adapter *); +int t4vf_get_dev_params(struct adapter *); +int t4vf_get_rss_glb_config(struct adapter *); +int t4vf_get_vfres(struct adapter *); + +int t4vf_read_rss_vi_config(struct adapter *, unsigned int, + union rss_vi_config *); +int t4vf_write_rss_vi_config(struct adapter *, unsigned int, + union rss_vi_config *); +int t4vf_config_rss_range(struct adapter *, unsigned int, int, int, + const u16 *, int); + +int t4vf_alloc_vi(struct adapter *, int); +int t4vf_free_vi(struct adapter *, int); +int t4vf_enable_vi(struct adapter *adapter, unsigned int viid, bool rx_en, + bool tx_en); +int t4vf_enable_pi(struct adapter *adapter, struct port_info *pi, bool rx_en, + bool tx_en); +int t4vf_identify_port(struct adapter *, unsigned int, unsigned int); + +int t4vf_set_rxmode(struct adapter *, unsigned int, int, int, int, int, int, + bool); +int t4vf_alloc_mac_filt(struct adapter *, unsigned int, bool, unsigned int, + const u8 **, u16 *, u64 *, bool); +int t4vf_free_mac_filt(struct adapter *, unsigned int, unsigned int naddr, + const u8 **, bool); +int t4vf_change_mac(struct adapter *, unsigned int, int, const u8 *, bool); +int t4vf_set_addr_hash(struct adapter *, unsigned int, bool, u64, bool); +int t4vf_get_port_stats(struct adapter *, int, struct t4vf_port_stats *); + +int t4vf_iq_free(struct adapter *, unsigned int, unsigned int, unsigned int, + unsigned int); +int t4vf_eth_eq_free(struct adapter *, unsigned int); + +int t4vf_update_port_info(struct port_info *pi); +int t4vf_handle_fw_rpl(struct adapter *, const __be64 *); +int t4vf_prep_adapter(struct adapter *); +int t4vf_get_vf_mac_acl(struct adapter *adapter, unsigned int port, + unsigned int *naddr, u8 *addr); +int t4vf_get_vf_vlan_acl(struct adapter *adapter); + +#endif /* __T4VF_COMMON_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_defs.h b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_defs.h new file mode 100644 index 000000000..f859db3d2 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_defs.h @@ -0,0 +1,122 @@ +/* + * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet + * driver for Linux. + * + * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __T4VF_DEFS_H__ +#define __T4VF_DEFS_H__ + +#include "../cxgb4/t4_regs.h" + +/* + * The VF Register Map. + * + * The Scatter Gather Engine (SGE), Multiport Support module (MPS), PIO Local + * bus module (PL) and CPU Interface Module (CIM) components are mapped via + * the Slice to Module Map Table (see below) in the Physical Function Register + * Map. The Mail Box Data (MBDATA) range is mapped via the PCI-E Mailbox Base + * and Offset registers in the PF Register Map. The MBDATA base address is + * quite constrained as it determines the Mailbox Data addresses for both PFs + * and VFs, and therefore must fit in both the VF and PF Register Maps without + * overlapping other registers. + */ +#define T4VF_SGE_BASE_ADDR 0x0000 +#define T4VF_MPS_BASE_ADDR 0x0100 +#define T4VF_PL_BASE_ADDR 0x0200 +#define T4VF_MBDATA_BASE_ADDR 0x0240 +#define T6VF_MBDATA_BASE_ADDR 0x0280 +#define T4VF_CIM_BASE_ADDR 0x0300 + +#define T4VF_REGMAP_START 0x0000 +#define T4VF_REGMAP_SIZE 0x0400 + +/* + * There's no hardware limitation which requires that the addresses of the + * Mailbox Data in the fixed CIM PF map and the programmable VF map must + * match. However, it's a useful convention ... + */ +#if T4VF_MBDATA_BASE_ADDR != CIM_PF_MAILBOX_DATA_A +#error T4VF_MBDATA_BASE_ADDR must match CIM_PF_MAILBOX_DATA_A! +#endif + +/* + * Virtual Function "Slice to Module Map Table" definitions. + * + * This table allows us to map subsets of the various module register sets + * into the T4VF Register Map. Each table entry identifies the index of the + * module whose registers are being mapped, the offset within the module's + * register set that the mapping should start at, the limit of the mapping, + * and the offset within the T4VF Register Map to which the module's registers + * are being mapped. All addresses and qualtities are in terms of 32-bit + * words. The "limit" value is also in terms of 32-bit words and is equal to + * the last address mapped in the T4VF Register Map 1 (i.e. it's a "<=" + * relation rather than a "<"). + */ +#define T4VF_MOD_MAP(module, index, first, last) \ + T4VF_MOD_MAP_##module##_INDEX = (index), \ + T4VF_MOD_MAP_##module##_FIRST = (first), \ + T4VF_MOD_MAP_##module##_LAST = (last), \ + T4VF_MOD_MAP_##module##_OFFSET = ((first)/4), \ + T4VF_MOD_MAP_##module##_BASE = \ + (T4VF_##module##_BASE_ADDR/4 + (first)/4), \ + T4VF_MOD_MAP_##module##_LIMIT = \ + (T4VF_##module##_BASE_ADDR/4 + (last)/4), + +#define SGE_VF_KDOORBELL 0x0 +#define SGE_VF_GTS 0x4 +#define MPS_VF_CTL 0x0 +#define MPS_VF_STAT_RX_VF_ERR_FRAMES_H 0xfc +#define PL_VF_WHOAMI 0x0 +#define CIM_VF_EXT_MAILBOX_CTRL 0x0 +#define CIM_VF_EXT_MAILBOX_STATUS 0x4 + +enum { + T4VF_MOD_MAP(SGE, 2, SGE_VF_KDOORBELL, SGE_VF_GTS) + T4VF_MOD_MAP(MPS, 0, MPS_VF_CTL, MPS_VF_STAT_RX_VF_ERR_FRAMES_H) + T4VF_MOD_MAP(PL, 3, PL_VF_WHOAMI, PL_VF_WHOAMI) + T4VF_MOD_MAP(CIM, 1, CIM_VF_EXT_MAILBOX_CTRL, CIM_VF_EXT_MAILBOX_STATUS) +}; + +/* + * There isn't a Slice to Module Map Table entry for the Mailbox Data + * registers, but it's convenient to use similar names as above. There are 8 + * little-endian 64-bit Mailbox Data registers. Note that the "instances" + * value below is in terms of 32-bit words which matches the "word" addressing + * space we use above for the Slice to Module Map Space. + */ +#define NUM_CIM_VF_MAILBOX_DATA_INSTANCES 16 + +#define T4VF_MBDATA_FIRST 0 +#define T4VF_MBDATA_LAST ((NUM_CIM_VF_MAILBOX_DATA_INSTANCES-1)*4) + +#endif /* __T4T4VF_DEFS_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c new file mode 100644 index 000000000..1c52592d3 --- /dev/null +++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c @@ -0,0 +1,2260 @@ +/* + * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet + * driver for Linux. + * + * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/ethtool.h> +#include <linux/pci.h> + +#include "t4vf_common.h" +#include "t4vf_defs.h" + +#include "../cxgb4/t4_regs.h" +#include "../cxgb4/t4_values.h" +#include "../cxgb4/t4fw_api.h" + +/* + * Wait for the device to become ready (signified by our "who am I" register + * returning a value other than all 1's). Return an error if it doesn't + * become ready ... + */ +int t4vf_wait_dev_ready(struct adapter *adapter) +{ + const u32 whoami = T4VF_PL_BASE_ADDR + PL_VF_WHOAMI; + const u32 notready1 = 0xffffffff; + const u32 notready2 = 0xeeeeeeee; + u32 val; + + val = t4_read_reg(adapter, whoami); + if (val != notready1 && val != notready2) + return 0; + msleep(500); + val = t4_read_reg(adapter, whoami); + if (val != notready1 && val != notready2) + return 0; + else + return -EIO; +} + +/* + * Get the reply to a mailbox command and store it in @rpl in big-endian order + * (since the firmware data structures are specified in a big-endian layout). + */ +static void get_mbox_rpl(struct adapter *adapter, __be64 *rpl, int size, + u32 mbox_data) +{ + for ( ; size; size -= 8, mbox_data += 8) + *rpl++ = cpu_to_be64(t4_read_reg64(adapter, mbox_data)); +} + +/** + * t4vf_record_mbox - record a Firmware Mailbox Command/Reply in the log + * @adapter: the adapter + * @cmd: the Firmware Mailbox Command or Reply + * @size: command length in bytes + * @access: the time (ms) needed to access the Firmware Mailbox + * @execute: the time (ms) the command spent being executed + */ +static void t4vf_record_mbox(struct adapter *adapter, const __be64 *cmd, + int size, int access, int execute) +{ + struct mbox_cmd_log *log = adapter->mbox_log; + struct mbox_cmd *entry; + int i; + + entry = mbox_cmd_log_entry(log, log->cursor++); + if (log->cursor == log->size) + log->cursor = 0; + + for (i = 0; i < size / 8; i++) + entry->cmd[i] = be64_to_cpu(cmd[i]); + while (i < MBOX_LEN / 8) + entry->cmd[i++] = 0; + entry->timestamp = jiffies; + entry->seqno = log->seqno++; + entry->access = access; + entry->execute = execute; +} + +/** + * t4vf_wr_mbox_core - send a command to FW through the mailbox + * @adapter: the adapter + * @cmd: the command to write + * @size: command length in bytes + * @rpl: where to optionally store the reply + * @sleep_ok: if true we may sleep while awaiting command completion + * + * Sends the given command to FW through the mailbox and waits for the + * FW to execute the command. If @rpl is not %NULL it is used to store + * the FW's reply to the command. The command and its optional reply + * are of the same length. FW can take up to 500 ms to respond. + * @sleep_ok determines whether we may sleep while awaiting the response. + * If sleeping is allowed we use progressive backoff otherwise we spin. + * + * The return value is 0 on success or a negative errno on failure. A + * failure can happen either because we are not able to execute the + * command or FW executes it but signals an error. In the latter case + * the return value is the error code indicated by FW (negated). + */ +int t4vf_wr_mbox_core(struct adapter *adapter, const void *cmd, int size, + void *rpl, bool sleep_ok) +{ + static const int delay[] = { + 1, 1, 3, 5, 10, 10, 20, 50, 100 + }; + + u16 access = 0, execute = 0; + u32 v, mbox_data; + int i, ms, delay_idx, ret; + const __be64 *p; + u32 mbox_ctl = T4VF_CIM_BASE_ADDR + CIM_VF_EXT_MAILBOX_CTRL; + u32 cmd_op = FW_CMD_OP_G(be32_to_cpu(((struct fw_cmd_hdr *)cmd)->hi)); + __be64 cmd_rpl[MBOX_LEN / 8]; + struct mbox_list entry; + + /* In T6, mailbox size is changed to 128 bytes to avoid + * invalidating the entire prefetch buffer. + */ + if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) + mbox_data = T4VF_MBDATA_BASE_ADDR; + else + mbox_data = T6VF_MBDATA_BASE_ADDR; + + /* + * Commands must be multiples of 16 bytes in length and may not be + * larger than the size of the Mailbox Data register array. + */ + if ((size % 16) != 0 || + size > NUM_CIM_VF_MAILBOX_DATA_INSTANCES * 4) + return -EINVAL; + + /* Queue ourselves onto the mailbox access list. When our entry is at + * the front of the list, we have rights to access the mailbox. So we + * wait [for a while] till we're at the front [or bail out with an + * EBUSY] ... + */ + spin_lock(&adapter->mbox_lock); + list_add_tail(&entry.list, &adapter->mlist.list); + spin_unlock(&adapter->mbox_lock); + + delay_idx = 0; + ms = delay[0]; + + for (i = 0; ; i += ms) { + /* If we've waited too long, return a busy indication. This + * really ought to be based on our initial position in the + * mailbox access list but this is a start. We very rearely + * contend on access to the mailbox ... + */ + if (i > FW_CMD_MAX_TIMEOUT) { + spin_lock(&adapter->mbox_lock); + list_del(&entry.list); + spin_unlock(&adapter->mbox_lock); + ret = -EBUSY; + t4vf_record_mbox(adapter, cmd, size, access, ret); + return ret; + } + + /* If we're at the head, break out and start the mailbox + * protocol. + */ + if (list_first_entry(&adapter->mlist.list, struct mbox_list, + list) == &entry) + break; + + /* Delay for a bit before checking again ... */ + if (sleep_ok) { + ms = delay[delay_idx]; /* last element may repeat */ + if (delay_idx < ARRAY_SIZE(delay) - 1) + delay_idx++; + msleep(ms); + } else { + mdelay(ms); + } + } + + /* + * Loop trying to get ownership of the mailbox. Return an error + * if we can't gain ownership. + */ + v = MBOWNER_G(t4_read_reg(adapter, mbox_ctl)); + for (i = 0; v == MBOX_OWNER_NONE && i < 3; i++) + v = MBOWNER_G(t4_read_reg(adapter, mbox_ctl)); + if (v != MBOX_OWNER_DRV) { + spin_lock(&adapter->mbox_lock); + list_del(&entry.list); + spin_unlock(&adapter->mbox_lock); + ret = (v == MBOX_OWNER_FW) ? -EBUSY : -ETIMEDOUT; + t4vf_record_mbox(adapter, cmd, size, access, ret); + return ret; + } + + /* + * Write the command array into the Mailbox Data register array and + * transfer ownership of the mailbox to the firmware. + * + * For the VFs, the Mailbox Data "registers" are actually backed by + * T4's "MA" interface rather than PL Registers (as is the case for + * the PFs). Because these are in different coherency domains, the + * write to the VF's PL-register-backed Mailbox Control can race in + * front of the writes to the MA-backed VF Mailbox Data "registers". + * So we need to do a read-back on at least one byte of the VF Mailbox + * Data registers before doing the write to the VF Mailbox Control + * register. + */ + if (cmd_op != FW_VI_STATS_CMD) + t4vf_record_mbox(adapter, cmd, size, access, 0); + for (i = 0, p = cmd; i < size; i += 8) + t4_write_reg64(adapter, mbox_data + i, be64_to_cpu(*p++)); + t4_read_reg(adapter, mbox_data); /* flush write */ + + t4_write_reg(adapter, mbox_ctl, + MBMSGVALID_F | MBOWNER_V(MBOX_OWNER_FW)); + t4_read_reg(adapter, mbox_ctl); /* flush write */ + + /* + * Spin waiting for firmware to acknowledge processing our command. + */ + delay_idx = 0; + ms = delay[0]; + + for (i = 0; i < FW_CMD_MAX_TIMEOUT; i += ms) { + if (sleep_ok) { + ms = delay[delay_idx]; + if (delay_idx < ARRAY_SIZE(delay) - 1) + delay_idx++; + msleep(ms); + } else + mdelay(ms); + + /* + * If we're the owner, see if this is the reply we wanted. + */ + v = t4_read_reg(adapter, mbox_ctl); + if (MBOWNER_G(v) == MBOX_OWNER_DRV) { + /* + * If the Message Valid bit isn't on, revoke ownership + * of the mailbox and continue waiting for our reply. + */ + if ((v & MBMSGVALID_F) == 0) { + t4_write_reg(adapter, mbox_ctl, + MBOWNER_V(MBOX_OWNER_NONE)); + continue; + } + + /* + * We now have our reply. Extract the command return + * value, copy the reply back to our caller's buffer + * (if specified) and revoke ownership of the mailbox. + * We return the (negated) firmware command return + * code (this depends on FW_SUCCESS == 0). + */ + get_mbox_rpl(adapter, cmd_rpl, size, mbox_data); + + /* return value in low-order little-endian word */ + v = be64_to_cpu(cmd_rpl[0]); + + if (rpl) { + /* request bit in high-order BE word */ + WARN_ON((be32_to_cpu(*(const __be32 *)cmd) + & FW_CMD_REQUEST_F) == 0); + memcpy(rpl, cmd_rpl, size); + WARN_ON((be32_to_cpu(*(__be32 *)rpl) + & FW_CMD_REQUEST_F) != 0); + } + t4_write_reg(adapter, mbox_ctl, + MBOWNER_V(MBOX_OWNER_NONE)); + execute = i + ms; + if (cmd_op != FW_VI_STATS_CMD) + t4vf_record_mbox(adapter, cmd_rpl, size, access, + execute); + spin_lock(&adapter->mbox_lock); + list_del(&entry.list); + spin_unlock(&adapter->mbox_lock); + return -FW_CMD_RETVAL_G(v); + } + } + + /* We timed out. Return the error ... */ + ret = -ETIMEDOUT; + t4vf_record_mbox(adapter, cmd, size, access, ret); + spin_lock(&adapter->mbox_lock); + list_del(&entry.list); + spin_unlock(&adapter->mbox_lock); + return ret; +} + +/* In the Physical Function Driver Common Code, the ADVERT_MASK is used to + * mask out bits in the Advertised Port Capabilities which are managed via + * separate controls, like Pause Frames and Forward Error Correction. In the + * Virtual Function Common Code, since we never perform L1 Configuration on + * the Link, the only things we really need to filter out are things which + * we decode and report separately like Speed. + */ +#define ADVERT_MASK (FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_M) | \ + FW_PORT_CAP32_802_3_PAUSE | \ + FW_PORT_CAP32_802_3_ASM_DIR | \ + FW_PORT_CAP32_FEC_V(FW_PORT_CAP32_FEC_M) | \ + FW_PORT_CAP32_ANEG) + +/** + * fwcaps16_to_caps32 - convert 16-bit Port Capabilities to 32-bits + * @caps16: a 16-bit Port Capabilities value + * + * Returns the equivalent 32-bit Port Capabilities value. + */ +static fw_port_cap32_t fwcaps16_to_caps32(fw_port_cap16_t caps16) +{ + fw_port_cap32_t caps32 = 0; + + #define CAP16_TO_CAP32(__cap) \ + do { \ + if (caps16 & FW_PORT_CAP_##__cap) \ + caps32 |= FW_PORT_CAP32_##__cap; \ + } while (0) + + CAP16_TO_CAP32(SPEED_100M); + CAP16_TO_CAP32(SPEED_1G); + CAP16_TO_CAP32(SPEED_25G); + CAP16_TO_CAP32(SPEED_10G); + CAP16_TO_CAP32(SPEED_40G); + CAP16_TO_CAP32(SPEED_100G); + CAP16_TO_CAP32(FC_RX); + CAP16_TO_CAP32(FC_TX); + CAP16_TO_CAP32(ANEG); + CAP16_TO_CAP32(MDIAUTO); + CAP16_TO_CAP32(MDISTRAIGHT); + CAP16_TO_CAP32(FEC_RS); + CAP16_TO_CAP32(FEC_BASER_RS); + CAP16_TO_CAP32(802_3_PAUSE); + CAP16_TO_CAP32(802_3_ASM_DIR); + + #undef CAP16_TO_CAP32 + + return caps32; +} + +/* Translate Firmware Pause specification to Common Code */ +static inline enum cc_pause fwcap_to_cc_pause(fw_port_cap32_t fw_pause) +{ + enum cc_pause cc_pause = 0; + + if (fw_pause & FW_PORT_CAP32_FC_RX) + cc_pause |= PAUSE_RX; + if (fw_pause & FW_PORT_CAP32_FC_TX) + cc_pause |= PAUSE_TX; + + return cc_pause; +} + +/* Translate Firmware Forward Error Correction specification to Common Code */ +static inline enum cc_fec fwcap_to_cc_fec(fw_port_cap32_t fw_fec) +{ + enum cc_fec cc_fec = 0; + + if (fw_fec & FW_PORT_CAP32_FEC_RS) + cc_fec |= FEC_RS; + if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS) + cc_fec |= FEC_BASER_RS; + + return cc_fec; +} + +/* Return the highest speed set in the port capabilities, in Mb/s. */ +static unsigned int fwcap_to_speed(fw_port_cap32_t caps) +{ + #define TEST_SPEED_RETURN(__caps_speed, __speed) \ + do { \ + if (caps & FW_PORT_CAP32_SPEED_##__caps_speed) \ + return __speed; \ + } while (0) + + TEST_SPEED_RETURN(400G, 400000); + TEST_SPEED_RETURN(200G, 200000); + TEST_SPEED_RETURN(100G, 100000); + TEST_SPEED_RETURN(50G, 50000); + TEST_SPEED_RETURN(40G, 40000); + TEST_SPEED_RETURN(25G, 25000); + TEST_SPEED_RETURN(10G, 10000); + TEST_SPEED_RETURN(1G, 1000); + TEST_SPEED_RETURN(100M, 100); + + #undef TEST_SPEED_RETURN + + return 0; +} + +/** + * fwcap_to_fwspeed - return highest speed in Port Capabilities + * @acaps: advertised Port Capabilities + * + * Get the highest speed for the port from the advertised Port + * Capabilities. It will be either the highest speed from the list of + * speeds or whatever user has set using ethtool. + */ +static fw_port_cap32_t fwcap_to_fwspeed(fw_port_cap32_t acaps) +{ + #define TEST_SPEED_RETURN(__caps_speed) \ + do { \ + if (acaps & FW_PORT_CAP32_SPEED_##__caps_speed) \ + return FW_PORT_CAP32_SPEED_##__caps_speed; \ + } while (0) + + TEST_SPEED_RETURN(400G); + TEST_SPEED_RETURN(200G); + TEST_SPEED_RETURN(100G); + TEST_SPEED_RETURN(50G); + TEST_SPEED_RETURN(40G); + TEST_SPEED_RETURN(25G); + TEST_SPEED_RETURN(10G); + TEST_SPEED_RETURN(1G); + TEST_SPEED_RETURN(100M); + + #undef TEST_SPEED_RETURN + return 0; +} + +/* + * init_link_config - initialize a link's SW state + * @lc: structure holding the link state + * @pcaps: link Port Capabilities + * @acaps: link current Advertised Port Capabilities + * + * Initializes the SW state maintained for each link, including the link's + * capabilities and default speed/flow-control/autonegotiation settings. + */ +static void init_link_config(struct link_config *lc, + fw_port_cap32_t pcaps, + fw_port_cap32_t acaps) +{ + lc->pcaps = pcaps; + lc->lpacaps = 0; + lc->speed_caps = 0; + lc->speed = 0; + lc->requested_fc = lc->fc = PAUSE_RX | PAUSE_TX; + + /* For Forward Error Control, we default to whatever the Firmware + * tells us the Link is currently advertising. + */ + lc->auto_fec = fwcap_to_cc_fec(acaps); + lc->requested_fec = FEC_AUTO; + lc->fec = lc->auto_fec; + + /* If the Port is capable of Auto-Negtotiation, initialize it as + * "enabled" and copy over all of the Physical Port Capabilities + * to the Advertised Port Capabilities. Otherwise mark it as + * Auto-Negotiate disabled and select the highest supported speed + * for the link. Note parallel structure in t4_link_l1cfg_core() + * and t4_handle_get_port_info(). + */ + if (lc->pcaps & FW_PORT_CAP32_ANEG) { + lc->acaps = acaps & ADVERT_MASK; + lc->autoneg = AUTONEG_ENABLE; + lc->requested_fc |= PAUSE_AUTONEG; + } else { + lc->acaps = 0; + lc->autoneg = AUTONEG_DISABLE; + lc->speed_caps = fwcap_to_fwspeed(acaps); + } +} + +/** + * t4vf_port_init - initialize port hardware/software state + * @adapter: the adapter + * @pidx: the adapter port index + */ +int t4vf_port_init(struct adapter *adapter, int pidx) +{ + struct port_info *pi = adap2pinfo(adapter, pidx); + unsigned int fw_caps = adapter->params.fw_caps_support; + struct fw_vi_cmd vi_cmd, vi_rpl; + struct fw_port_cmd port_cmd, port_rpl; + enum fw_port_type port_type; + int mdio_addr; + fw_port_cap32_t pcaps, acaps; + int ret; + + /* If we haven't yet determined whether we're talking to Firmware + * which knows the new 32-bit Port Capabilities, it's time to find + * out now. This will also tell new Firmware to send us Port Status + * Updates using the new 32-bit Port Capabilities version of the + * Port Information message. + */ + if (fw_caps == FW_CAPS_UNKNOWN) { + u32 param, val; + + param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_PORT_CAPS32)); + val = 1; + ret = t4vf_set_params(adapter, 1, ¶m, &val); + fw_caps = (ret == 0 ? FW_CAPS32 : FW_CAPS16); + adapter->params.fw_caps_support = fw_caps; + } + + /* + * Execute a VI Read command to get our Virtual Interface information + * like MAC address, etc. + */ + memset(&vi_cmd, 0, sizeof(vi_cmd)); + vi_cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_VI_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F); + vi_cmd.alloc_to_len16 = cpu_to_be32(FW_LEN16(vi_cmd)); + vi_cmd.type_viid = cpu_to_be16(FW_VI_CMD_VIID_V(pi->viid)); + ret = t4vf_wr_mbox(adapter, &vi_cmd, sizeof(vi_cmd), &vi_rpl); + if (ret != FW_SUCCESS) + return ret; + + BUG_ON(pi->port_id != FW_VI_CMD_PORTID_G(vi_rpl.portid_pkd)); + pi->rss_size = FW_VI_CMD_RSSSIZE_G(be16_to_cpu(vi_rpl.rsssize_pkd)); + t4_os_set_hw_addr(adapter, pidx, vi_rpl.mac); + + /* + * If we don't have read access to our port information, we're done + * now. Otherwise, execute a PORT Read command to get it ... + */ + if (!(adapter->params.vfres.r_caps & FW_CMD_CAP_PORT)) + return 0; + + memset(&port_cmd, 0, sizeof(port_cmd)); + port_cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F | + FW_PORT_CMD_PORTID_V(pi->port_id)); + port_cmd.action_to_len16 = cpu_to_be32( + FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16 + ? FW_PORT_ACTION_GET_PORT_INFO + : FW_PORT_ACTION_GET_PORT_INFO32) | + FW_LEN16(port_cmd)); + ret = t4vf_wr_mbox(adapter, &port_cmd, sizeof(port_cmd), &port_rpl); + if (ret != FW_SUCCESS) + return ret; + + /* Extract the various fields from the Port Information message. */ + if (fw_caps == FW_CAPS16) { + u32 lstatus = be32_to_cpu(port_rpl.u.info.lstatus_to_modtype); + + port_type = FW_PORT_CMD_PTYPE_G(lstatus); + mdio_addr = ((lstatus & FW_PORT_CMD_MDIOCAP_F) + ? FW_PORT_CMD_MDIOADDR_G(lstatus) + : -1); + pcaps = fwcaps16_to_caps32(be16_to_cpu(port_rpl.u.info.pcap)); + acaps = fwcaps16_to_caps32(be16_to_cpu(port_rpl.u.info.acap)); + } else { + u32 lstatus32 = + be32_to_cpu(port_rpl.u.info32.lstatus32_to_cbllen32); + + port_type = FW_PORT_CMD_PORTTYPE32_G(lstatus32); + mdio_addr = ((lstatus32 & FW_PORT_CMD_MDIOCAP32_F) + ? FW_PORT_CMD_MDIOADDR32_G(lstatus32) + : -1); + pcaps = be32_to_cpu(port_rpl.u.info32.pcaps32); + acaps = be32_to_cpu(port_rpl.u.info32.acaps32); + } + + pi->port_type = port_type; + pi->mdio_addr = mdio_addr; + pi->mod_type = FW_PORT_MOD_TYPE_NA; + + init_link_config(&pi->link_cfg, pcaps, acaps); + return 0; +} + +/** + * t4vf_fw_reset - issue a reset to FW + * @adapter: the adapter + * + * Issues a reset command to FW. For a Physical Function this would + * result in the Firmware resetting all of its state. For a Virtual + * Function this just resets the state associated with the VF. + */ +int t4vf_fw_reset(struct adapter *adapter) +{ + struct fw_reset_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_RESET_CMD) | + FW_CMD_WRITE_F); + cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); + return t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), NULL); +} + +/** + * t4vf_query_params - query FW or device parameters + * @adapter: the adapter + * @nparams: the number of parameters + * @params: the parameter names + * @vals: the parameter values + * + * Reads the values of firmware or device parameters. Up to 7 parameters + * can be queried at once. + */ +static int t4vf_query_params(struct adapter *adapter, unsigned int nparams, + const u32 *params, u32 *vals) +{ + int i, ret; + struct fw_params_cmd cmd, rpl; + struct fw_params_param *p; + size_t len16; + + if (nparams > 7) + return -EINVAL; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PARAMS_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F); + len16 = DIV_ROUND_UP(offsetof(struct fw_params_cmd, + param[nparams].mnem), 16); + cmd.retval_len16 = cpu_to_be32(FW_CMD_LEN16_V(len16)); + for (i = 0, p = &cmd.param[0]; i < nparams; i++, p++) + p->mnem = htonl(*params++); + + ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl); + if (ret == 0) + for (i = 0, p = &rpl.param[0]; i < nparams; i++, p++) + *vals++ = be32_to_cpu(p->val); + return ret; +} + +/** + * t4vf_set_params - sets FW or device parameters + * @adapter: the adapter + * @nparams: the number of parameters + * @params: the parameter names + * @vals: the parameter values + * + * Sets the values of firmware or device parameters. Up to 7 parameters + * can be specified at once. + */ +int t4vf_set_params(struct adapter *adapter, unsigned int nparams, + const u32 *params, const u32 *vals) +{ + int i; + struct fw_params_cmd cmd; + struct fw_params_param *p; + size_t len16; + + if (nparams > 7) + return -EINVAL; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PARAMS_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F); + len16 = DIV_ROUND_UP(offsetof(struct fw_params_cmd, + param[nparams]), 16); + cmd.retval_len16 = cpu_to_be32(FW_CMD_LEN16_V(len16)); + for (i = 0, p = &cmd.param[0]; i < nparams; i++, p++) { + p->mnem = cpu_to_be32(*params++); + p->val = cpu_to_be32(*vals++); + } + + return t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), NULL); +} + +/** + * t4vf_fl_pkt_align - return the fl packet alignment + * @adapter: the adapter + * + * T4 has a single field to specify the packing and padding boundary. + * T5 onwards has separate fields for this and hence the alignment for + * next packet offset is maximum of these two. And T6 changes the + * Ingress Padding Boundary Shift, so it's all a mess and it's best + * if we put this in low-level Common Code ... + * + */ +int t4vf_fl_pkt_align(struct adapter *adapter) +{ + u32 sge_control, sge_control2; + unsigned int ingpadboundary, ingpackboundary, fl_align, ingpad_shift; + + sge_control = adapter->params.sge.sge_control; + + /* T4 uses a single control field to specify both the PCIe Padding and + * Packing Boundary. T5 introduced the ability to specify these + * separately. The actual Ingress Packet Data alignment boundary + * within Packed Buffer Mode is the maximum of these two + * specifications. (Note that it makes no real practical sense to + * have the Pading Boudary be larger than the Packing Boundary but you + * could set the chip up that way and, in fact, legacy T4 code would + * end doing this because it would initialize the Padding Boundary and + * leave the Packing Boundary initialized to 0 (16 bytes).) + * Padding Boundary values in T6 starts from 8B, + * where as it is 32B for T4 and T5. + */ + if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) + ingpad_shift = INGPADBOUNDARY_SHIFT_X; + else + ingpad_shift = T6_INGPADBOUNDARY_SHIFT_X; + + ingpadboundary = 1 << (INGPADBOUNDARY_G(sge_control) + ingpad_shift); + + fl_align = ingpadboundary; + if (!is_t4(adapter->params.chip)) { + /* T5 has a different interpretation of one of the PCIe Packing + * Boundary values. + */ + sge_control2 = adapter->params.sge.sge_control2; + ingpackboundary = INGPACKBOUNDARY_G(sge_control2); + if (ingpackboundary == INGPACKBOUNDARY_16B_X) + ingpackboundary = 16; + else + ingpackboundary = 1 << (ingpackboundary + + INGPACKBOUNDARY_SHIFT_X); + + fl_align = max(ingpadboundary, ingpackboundary); + } + return fl_align; +} + +/** + * t4vf_bar2_sge_qregs - return BAR2 SGE Queue register information + * @adapter: the adapter + * @qid: the Queue ID + * @qtype: the Ingress or Egress type for @qid + * @pbar2_qoffset: BAR2 Queue Offset + * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues + * + * Returns the BAR2 SGE Queue Registers information associated with the + * indicated Absolute Queue ID. These are passed back in return value + * pointers. @qtype should be T4_BAR2_QTYPE_EGRESS for Egress Queue + * and T4_BAR2_QTYPE_INGRESS for Ingress Queues. + * + * This may return an error which indicates that BAR2 SGE Queue + * registers aren't available. If an error is not returned, then the + * following values are returned: + * + * *@pbar2_qoffset: the BAR2 Offset of the @qid Registers + * *@pbar2_qid: the BAR2 SGE Queue ID or 0 of @qid + * + * If the returned BAR2 Queue ID is 0, then BAR2 SGE registers which + * require the "Inferred Queue ID" ability may be used. E.g. the + * Write Combining Doorbell Buffer. If the BAR2 Queue ID is not 0, + * then these "Inferred Queue ID" register may not be used. + */ +int t4vf_bar2_sge_qregs(struct adapter *adapter, + unsigned int qid, + enum t4_bar2_qtype qtype, + u64 *pbar2_qoffset, + unsigned int *pbar2_qid) +{ + unsigned int page_shift, page_size, qpp_shift, qpp_mask; + u64 bar2_page_offset, bar2_qoffset; + unsigned int bar2_qid, bar2_qid_offset, bar2_qinferred; + + /* T4 doesn't support BAR2 SGE Queue registers. + */ + if (is_t4(adapter->params.chip)) + return -EINVAL; + + /* Get our SGE Page Size parameters. + */ + page_shift = adapter->params.sge.sge_vf_hps + 10; + page_size = 1 << page_shift; + + /* Get the right Queues per Page parameters for our Queue. + */ + qpp_shift = (qtype == T4_BAR2_QTYPE_EGRESS + ? adapter->params.sge.sge_vf_eq_qpp + : adapter->params.sge.sge_vf_iq_qpp); + qpp_mask = (1 << qpp_shift) - 1; + + /* Calculate the basics of the BAR2 SGE Queue register area: + * o The BAR2 page the Queue registers will be in. + * o The BAR2 Queue ID. + * o The BAR2 Queue ID Offset into the BAR2 page. + */ + bar2_page_offset = ((u64)(qid >> qpp_shift) << page_shift); + bar2_qid = qid & qpp_mask; + bar2_qid_offset = bar2_qid * SGE_UDB_SIZE; + + /* If the BAR2 Queue ID Offset is less than the Page Size, then the + * hardware will infer the Absolute Queue ID simply from the writes to + * the BAR2 Queue ID Offset within the BAR2 Page (and we need to use a + * BAR2 Queue ID of 0 for those writes). Otherwise, we'll simply + * write to the first BAR2 SGE Queue Area within the BAR2 Page with + * the BAR2 Queue ID and the hardware will infer the Absolute Queue ID + * from the BAR2 Page and BAR2 Queue ID. + * + * One important censequence of this is that some BAR2 SGE registers + * have a "Queue ID" field and we can write the BAR2 SGE Queue ID + * there. But other registers synthesize the SGE Queue ID purely + * from the writes to the registers -- the Write Combined Doorbell + * Buffer is a good example. These BAR2 SGE Registers are only + * available for those BAR2 SGE Register areas where the SGE Absolute + * Queue ID can be inferred from simple writes. + */ + bar2_qoffset = bar2_page_offset; + bar2_qinferred = (bar2_qid_offset < page_size); + if (bar2_qinferred) { + bar2_qoffset += bar2_qid_offset; + bar2_qid = 0; + } + + *pbar2_qoffset = bar2_qoffset; + *pbar2_qid = bar2_qid; + return 0; +} + +unsigned int t4vf_get_pf_from_vf(struct adapter *adapter) +{ + u32 whoami; + + whoami = t4_read_reg(adapter, T4VF_PL_BASE_ADDR + PL_VF_WHOAMI_A); + return (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ? + SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami)); +} + +/** + * t4vf_get_sge_params - retrieve adapter Scatter gather Engine parameters + * @adapter: the adapter + * + * Retrieves various core SGE parameters in the form of hardware SGE + * register values. The caller is responsible for decoding these as + * needed. The SGE parameters are stored in @adapter->params.sge. + */ +int t4vf_get_sge_params(struct adapter *adapter) +{ + struct sge_params *sge_params = &adapter->params.sge; + u32 params[7], vals[7]; + int v; + + params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V(SGE_CONTROL_A)); + params[1] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V(SGE_HOST_PAGE_SIZE_A)); + params[2] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V(SGE_FL_BUFFER_SIZE0_A)); + params[3] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V(SGE_FL_BUFFER_SIZE1_A)); + params[4] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V(SGE_TIMER_VALUE_0_AND_1_A)); + params[5] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V(SGE_TIMER_VALUE_2_AND_3_A)); + params[6] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V(SGE_TIMER_VALUE_4_AND_5_A)); + v = t4vf_query_params(adapter, 7, params, vals); + if (v) + return v; + sge_params->sge_control = vals[0]; + sge_params->sge_host_page_size = vals[1]; + sge_params->sge_fl_buffer_size[0] = vals[2]; + sge_params->sge_fl_buffer_size[1] = vals[3]; + sge_params->sge_timer_value_0_and_1 = vals[4]; + sge_params->sge_timer_value_2_and_3 = vals[5]; + sge_params->sge_timer_value_4_and_5 = vals[6]; + + /* T4 uses a single control field to specify both the PCIe Padding and + * Packing Boundary. T5 introduced the ability to specify these + * separately with the Padding Boundary in SGE_CONTROL and Packing + * Boundary in SGE_CONTROL2. So for T5 and later we need to grab + * SGE_CONTROL in order to determine how ingress packet data will be + * laid out in Packed Buffer Mode. Unfortunately, older versions of + * the firmware won't let us retrieve SGE_CONTROL2 so if we get a + * failure grabbing it we throw an error since we can't figure out the + * right value. + */ + if (!is_t4(adapter->params.chip)) { + params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V(SGE_CONTROL2_A)); + v = t4vf_query_params(adapter, 1, params, vals); + if (v != FW_SUCCESS) { + dev_err(adapter->pdev_dev, + "Unable to get SGE Control2; " + "probably old firmware.\n"); + return v; + } + sge_params->sge_control2 = vals[0]; + } + + params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V(SGE_INGRESS_RX_THRESHOLD_A)); + params[1] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V(SGE_CONM_CTRL_A)); + v = t4vf_query_params(adapter, 2, params, vals); + if (v) + return v; + sge_params->sge_ingress_rx_threshold = vals[0]; + sge_params->sge_congestion_control = vals[1]; + + /* For T5 and later we want to use the new BAR2 Doorbells. + * Unfortunately, older firmware didn't allow the this register to be + * read. + */ + if (!is_t4(adapter->params.chip)) { + unsigned int pf, s_hps, s_qpp; + + params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V( + SGE_EGRESS_QUEUES_PER_PAGE_VF_A)); + params[1] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | + FW_PARAMS_PARAM_XYZ_V( + SGE_INGRESS_QUEUES_PER_PAGE_VF_A)); + v = t4vf_query_params(adapter, 2, params, vals); + if (v != FW_SUCCESS) { + dev_warn(adapter->pdev_dev, + "Unable to get VF SGE Queues/Page; " + "probably old firmware.\n"); + return v; + } + sge_params->sge_egress_queues_per_page = vals[0]; + sge_params->sge_ingress_queues_per_page = vals[1]; + + /* We need the Queues/Page for our VF. This is based on the + * PF from which we're instantiated and is indexed in the + * register we just read. Do it once here so other code in + * the driver can just use it. + */ + pf = t4vf_get_pf_from_vf(adapter); + s_hps = (HOSTPAGESIZEPF0_S + + (HOSTPAGESIZEPF1_S - HOSTPAGESIZEPF0_S) * pf); + sge_params->sge_vf_hps = + ((sge_params->sge_host_page_size >> s_hps) + & HOSTPAGESIZEPF0_M); + + s_qpp = (QUEUESPERPAGEPF0_S + + (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) * pf); + sge_params->sge_vf_eq_qpp = + ((sge_params->sge_egress_queues_per_page >> s_qpp) + & QUEUESPERPAGEPF0_M); + sge_params->sge_vf_iq_qpp = + ((sge_params->sge_ingress_queues_per_page >> s_qpp) + & QUEUESPERPAGEPF0_M); + } + + return 0; +} + +/** + * t4vf_get_vpd_params - retrieve device VPD paremeters + * @adapter: the adapter + * + * Retrives various device Vital Product Data parameters. The parameters + * are stored in @adapter->params.vpd. + */ +int t4vf_get_vpd_params(struct adapter *adapter) +{ + struct vpd_params *vpd_params = &adapter->params.vpd; + u32 params[7], vals[7]; + int v; + + params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CCLK)); + v = t4vf_query_params(adapter, 1, params, vals); + if (v) + return v; + vpd_params->cclk = vals[0]; + + return 0; +} + +/** + * t4vf_get_dev_params - retrieve device paremeters + * @adapter: the adapter + * + * Retrives various device parameters. The parameters are stored in + * @adapter->params.dev. + */ +int t4vf_get_dev_params(struct adapter *adapter) +{ + struct dev_params *dev_params = &adapter->params.dev; + u32 params[7], vals[7]; + int v; + + params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_FWREV)); + params[1] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | + FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_TPREV)); + v = t4vf_query_params(adapter, 2, params, vals); + if (v) + return v; + dev_params->fwrev = vals[0]; + dev_params->tprev = vals[1]; + + return 0; +} + +/** + * t4vf_get_rss_glb_config - retrieve adapter RSS Global Configuration + * @adapter: the adapter + * + * Retrieves global RSS mode and parameters with which we have to live + * and stores them in the @adapter's RSS parameters. + */ +int t4vf_get_rss_glb_config(struct adapter *adapter) +{ + struct rss_params *rss = &adapter->params.rss; + struct fw_rss_glb_config_cmd cmd, rpl; + int v; + + /* + * Execute an RSS Global Configuration read command to retrieve + * our RSS configuration. + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_RSS_GLB_CONFIG_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F); + cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); + v = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl); + if (v) + return v; + + /* + * Transate the big-endian RSS Global Configuration into our + * cpu-endian format based on the RSS mode. We also do first level + * filtering at this point to weed out modes which don't support + * VF Drivers ... + */ + rss->mode = FW_RSS_GLB_CONFIG_CMD_MODE_G( + be32_to_cpu(rpl.u.manual.mode_pkd)); + switch (rss->mode) { + case FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL: { + u32 word = be32_to_cpu( + rpl.u.basicvirtual.synmapen_to_hashtoeplitz); + + rss->u.basicvirtual.synmapen = + ((word & FW_RSS_GLB_CONFIG_CMD_SYNMAPEN_F) != 0); + rss->u.basicvirtual.syn4tupenipv6 = + ((word & FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV6_F) != 0); + rss->u.basicvirtual.syn2tupenipv6 = + ((word & FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV6_F) != 0); + rss->u.basicvirtual.syn4tupenipv4 = + ((word & FW_RSS_GLB_CONFIG_CMD_SYN4TUPENIPV4_F) != 0); + rss->u.basicvirtual.syn2tupenipv4 = + ((word & FW_RSS_GLB_CONFIG_CMD_SYN2TUPENIPV4_F) != 0); + + rss->u.basicvirtual.ofdmapen = + ((word & FW_RSS_GLB_CONFIG_CMD_OFDMAPEN_F) != 0); + + rss->u.basicvirtual.tnlmapen = + ((word & FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F) != 0); + rss->u.basicvirtual.tnlalllookup = + ((word & FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F) != 0); + + rss->u.basicvirtual.hashtoeplitz = + ((word & FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ_F) != 0); + + /* we need at least Tunnel Map Enable to be set */ + if (!rss->u.basicvirtual.tnlmapen) + return -EINVAL; + break; + } + + default: + /* all unknown/unsupported RSS modes result in an error */ + return -EINVAL; + } + + return 0; +} + +/** + * t4vf_get_vfres - retrieve VF resource limits + * @adapter: the adapter + * + * Retrieves configured resource limits and capabilities for a virtual + * function. The results are stored in @adapter->vfres. + */ +int t4vf_get_vfres(struct adapter *adapter) +{ + struct vf_resources *vfres = &adapter->params.vfres; + struct fw_pfvf_cmd cmd, rpl; + int v; + u32 word; + + /* + * Execute PFVF Read command to get VF resource limits; bail out early + * with error on command failure. + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PFVF_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F); + cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); + v = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl); + if (v) + return v; + + /* + * Extract VF resource limits and return success. + */ + word = be32_to_cpu(rpl.niqflint_niq); + vfres->niqflint = FW_PFVF_CMD_NIQFLINT_G(word); + vfres->niq = FW_PFVF_CMD_NIQ_G(word); + + word = be32_to_cpu(rpl.type_to_neq); + vfres->neq = FW_PFVF_CMD_NEQ_G(word); + vfres->pmask = FW_PFVF_CMD_PMASK_G(word); + + word = be32_to_cpu(rpl.tc_to_nexactf); + vfres->tc = FW_PFVF_CMD_TC_G(word); + vfres->nvi = FW_PFVF_CMD_NVI_G(word); + vfres->nexactf = FW_PFVF_CMD_NEXACTF_G(word); + + word = be32_to_cpu(rpl.r_caps_to_nethctrl); + vfres->r_caps = FW_PFVF_CMD_R_CAPS_G(word); + vfres->wx_caps = FW_PFVF_CMD_WX_CAPS_G(word); + vfres->nethctrl = FW_PFVF_CMD_NETHCTRL_G(word); + + return 0; +} + +/** + * t4vf_read_rss_vi_config - read a VI's RSS configuration + * @adapter: the adapter + * @viid: Virtual Interface ID + * @config: pointer to host-native VI RSS Configuration buffer + * + * Reads the Virtual Interface's RSS configuration information and + * translates it into CPU-native format. + */ +int t4vf_read_rss_vi_config(struct adapter *adapter, unsigned int viid, + union rss_vi_config *config) +{ + struct fw_rss_vi_config_cmd cmd, rpl; + int v; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_RSS_VI_CONFIG_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F | + FW_RSS_VI_CONFIG_CMD_VIID(viid)); + cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); + v = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl); + if (v) + return v; + + switch (adapter->params.rss.mode) { + case FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL: { + u32 word = be32_to_cpu(rpl.u.basicvirtual.defaultq_to_udpen); + + config->basicvirtual.ip6fourtupen = + ((word & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) != 0); + config->basicvirtual.ip6twotupen = + ((word & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F) != 0); + config->basicvirtual.ip4fourtupen = + ((word & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) != 0); + config->basicvirtual.ip4twotupen = + ((word & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F) != 0); + config->basicvirtual.udpen = + ((word & FW_RSS_VI_CONFIG_CMD_UDPEN_F) != 0); + config->basicvirtual.defaultq = + FW_RSS_VI_CONFIG_CMD_DEFAULTQ_G(word); + break; + } + + default: + return -EINVAL; + } + + return 0; +} + +/** + * t4vf_write_rss_vi_config - write a VI's RSS configuration + * @adapter: the adapter + * @viid: Virtual Interface ID + * @config: pointer to host-native VI RSS Configuration buffer + * + * Write the Virtual Interface's RSS configuration information + * (translating it into firmware-native format before writing). + */ +int t4vf_write_rss_vi_config(struct adapter *adapter, unsigned int viid, + union rss_vi_config *config) +{ + struct fw_rss_vi_config_cmd cmd, rpl; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_RSS_VI_CONFIG_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_RSS_VI_CONFIG_CMD_VIID(viid)); + cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); + switch (adapter->params.rss.mode) { + case FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL: { + u32 word = 0; + + if (config->basicvirtual.ip6fourtupen) + word |= FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F; + if (config->basicvirtual.ip6twotupen) + word |= FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F; + if (config->basicvirtual.ip4fourtupen) + word |= FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F; + if (config->basicvirtual.ip4twotupen) + word |= FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F; + if (config->basicvirtual.udpen) + word |= FW_RSS_VI_CONFIG_CMD_UDPEN_F; + word |= FW_RSS_VI_CONFIG_CMD_DEFAULTQ_V( + config->basicvirtual.defaultq); + cmd.u.basicvirtual.defaultq_to_udpen = cpu_to_be32(word); + break; + } + + default: + return -EINVAL; + } + + return t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl); +} + +/** + * t4vf_config_rss_range - configure a portion of the RSS mapping table + * @adapter: the adapter + * @viid: Virtual Interface of RSS Table Slice + * @start: starting entry in the table to write + * @n: how many table entries to write + * @rspq: values for the "Response Queue" (Ingress Queue) lookup table + * @nrspq: number of values in @rspq + * + * Programs the selected part of the VI's RSS mapping table with the + * provided values. If @nrspq < @n the supplied values are used repeatedly + * until the full table range is populated. + * + * The caller must ensure the values in @rspq are in the range 0..1023. + */ +int t4vf_config_rss_range(struct adapter *adapter, unsigned int viid, + int start, int n, const u16 *rspq, int nrspq) +{ + const u16 *rsp = rspq; + const u16 *rsp_end = rspq+nrspq; + struct fw_rss_ind_tbl_cmd cmd; + + /* + * Initialize firmware command template to write the RSS table. + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_RSS_IND_TBL_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_RSS_IND_TBL_CMD_VIID_V(viid)); + cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); + + /* + * Each firmware RSS command can accommodate up to 32 RSS Ingress + * Queue Identifiers. These Ingress Queue IDs are packed three to + * a 32-bit word as 10-bit values with the upper remaining 2 bits + * reserved. + */ + while (n > 0) { + __be32 *qp = &cmd.iq0_to_iq2; + int nq = min(n, 32); + int ret; + + /* + * Set up the firmware RSS command header to send the next + * "nq" Ingress Queue IDs to the firmware. + */ + cmd.niqid = cpu_to_be16(nq); + cmd.startidx = cpu_to_be16(start); + + /* + * "nq" more done for the start of the next loop. + */ + start += nq; + n -= nq; + + /* + * While there are still Ingress Queue IDs to stuff into the + * current firmware RSS command, retrieve them from the + * Ingress Queue ID array and insert them into the command. + */ + while (nq > 0) { + /* + * Grab up to the next 3 Ingress Queue IDs (wrapping + * around the Ingress Queue ID array if necessary) and + * insert them into the firmware RSS command at the + * current 3-tuple position within the commad. + */ + u16 qbuf[3]; + u16 *qbp = qbuf; + int nqbuf = min(3, nq); + + nq -= nqbuf; + qbuf[0] = qbuf[1] = qbuf[2] = 0; + while (nqbuf) { + nqbuf--; + *qbp++ = *rsp++; + if (rsp >= rsp_end) + rsp = rspq; + } + *qp++ = cpu_to_be32(FW_RSS_IND_TBL_CMD_IQ0_V(qbuf[0]) | + FW_RSS_IND_TBL_CMD_IQ1_V(qbuf[1]) | + FW_RSS_IND_TBL_CMD_IQ2_V(qbuf[2])); + } + + /* + * Send this portion of the RRS table update to the firmware; + * bail out on any errors. + */ + ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), NULL); + if (ret) + return ret; + } + return 0; +} + +/** + * t4vf_alloc_vi - allocate a virtual interface on a port + * @adapter: the adapter + * @port_id: physical port associated with the VI + * + * Allocate a new Virtual Interface and bind it to the indicated + * physical port. Return the new Virtual Interface Identifier on + * success, or a [negative] error number on failure. + */ +int t4vf_alloc_vi(struct adapter *adapter, int port_id) +{ + struct fw_vi_cmd cmd, rpl; + int v; + + /* + * Execute a VI command to allocate Virtual Interface and return its + * VIID. + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_VI_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_CMD_EXEC_F); + cmd.alloc_to_len16 = cpu_to_be32(FW_LEN16(cmd) | + FW_VI_CMD_ALLOC_F); + cmd.portid_pkd = FW_VI_CMD_PORTID_V(port_id); + v = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl); + if (v) + return v; + + return FW_VI_CMD_VIID_G(be16_to_cpu(rpl.type_viid)); +} + +/** + * t4vf_free_vi -- free a virtual interface + * @adapter: the adapter + * @viid: the virtual interface identifier + * + * Free a previously allocated Virtual Interface. Return an error on + * failure. + */ +int t4vf_free_vi(struct adapter *adapter, int viid) +{ + struct fw_vi_cmd cmd; + + /* + * Execute a VI command to free the Virtual Interface. + */ + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_VI_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_EXEC_F); + cmd.alloc_to_len16 = cpu_to_be32(FW_LEN16(cmd) | + FW_VI_CMD_FREE_F); + cmd.type_viid = cpu_to_be16(FW_VI_CMD_VIID_V(viid)); + return t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), NULL); +} + +/** + * t4vf_enable_vi - enable/disable a virtual interface + * @adapter: the adapter + * @viid: the Virtual Interface ID + * @rx_en: 1=enable Rx, 0=disable Rx + * @tx_en: 1=enable Tx, 0=disable Tx + * + * Enables/disables a virtual interface. + */ +int t4vf_enable_vi(struct adapter *adapter, unsigned int viid, + bool rx_en, bool tx_en) +{ + struct fw_vi_enable_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_ENABLE_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_EXEC_F | + FW_VI_ENABLE_CMD_VIID_V(viid)); + cmd.ien_to_len16 = cpu_to_be32(FW_VI_ENABLE_CMD_IEN_V(rx_en) | + FW_VI_ENABLE_CMD_EEN_V(tx_en) | + FW_LEN16(cmd)); + return t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), NULL); +} + +/** + * t4vf_enable_pi - enable/disable a Port's virtual interface + * @adapter: the adapter + * @pi: the Port Information structure + * @rx_en: 1=enable Rx, 0=disable Rx + * @tx_en: 1=enable Tx, 0=disable Tx + * + * Enables/disables a Port's virtual interface. If the Virtual + * Interface enable/disable operation is successful, we notify the + * OS-specific code of a potential Link Status change via the OS Contract + * API t4vf_os_link_changed(). + */ +int t4vf_enable_pi(struct adapter *adapter, struct port_info *pi, + bool rx_en, bool tx_en) +{ + int ret = t4vf_enable_vi(adapter, pi->viid, rx_en, tx_en); + + if (ret) + return ret; + t4vf_os_link_changed(adapter, pi->pidx, + rx_en && tx_en && pi->link_cfg.link_ok); + return 0; +} + +/** + * t4vf_identify_port - identify a VI's port by blinking its LED + * @adapter: the adapter + * @viid: the Virtual Interface ID + * @nblinks: how many times to blink LED at 2.5 Hz + * + * Identifies a VI's port by blinking its LED. + */ +int t4vf_identify_port(struct adapter *adapter, unsigned int viid, + unsigned int nblinks) +{ + struct fw_vi_enable_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_ENABLE_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_EXEC_F | + FW_VI_ENABLE_CMD_VIID_V(viid)); + cmd.ien_to_len16 = cpu_to_be32(FW_VI_ENABLE_CMD_LED_F | + FW_LEN16(cmd)); + cmd.blinkdur = cpu_to_be16(nblinks); + return t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), NULL); +} + +/** + * t4vf_set_rxmode - set Rx properties of a virtual interface + * @adapter: the adapter + * @viid: the VI id + * @mtu: the new MTU or -1 for no change + * @promisc: 1 to enable promiscuous mode, 0 to disable it, -1 no change + * @all_multi: 1 to enable all-multi mode, 0 to disable it, -1 no change + * @bcast: 1 to enable broadcast Rx, 0 to disable it, -1 no change + * @vlanex: 1 to enable hardware VLAN Tag extraction, 0 to disable it, + * -1 no change + * @sleep_ok: call is allowed to sleep + * + * Sets Rx properties of a virtual interface. + */ +int t4vf_set_rxmode(struct adapter *adapter, unsigned int viid, + int mtu, int promisc, int all_multi, int bcast, int vlanex, + bool sleep_ok) +{ + struct fw_vi_rxmode_cmd cmd; + + /* convert to FW values */ + if (mtu < 0) + mtu = FW_VI_RXMODE_CMD_MTU_M; + if (promisc < 0) + promisc = FW_VI_RXMODE_CMD_PROMISCEN_M; + if (all_multi < 0) + all_multi = FW_VI_RXMODE_CMD_ALLMULTIEN_M; + if (bcast < 0) + bcast = FW_VI_RXMODE_CMD_BROADCASTEN_M; + if (vlanex < 0) + vlanex = FW_VI_RXMODE_CMD_VLANEXEN_M; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_RXMODE_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_VI_RXMODE_CMD_VIID_V(viid)); + cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd)); + cmd.mtu_to_vlanexen = + cpu_to_be32(FW_VI_RXMODE_CMD_MTU_V(mtu) | + FW_VI_RXMODE_CMD_PROMISCEN_V(promisc) | + FW_VI_RXMODE_CMD_ALLMULTIEN_V(all_multi) | + FW_VI_RXMODE_CMD_BROADCASTEN_V(bcast) | + FW_VI_RXMODE_CMD_VLANEXEN_V(vlanex)); + return t4vf_wr_mbox_core(adapter, &cmd, sizeof(cmd), NULL, sleep_ok); +} + +/** + * t4vf_alloc_mac_filt - allocates exact-match filters for MAC addresses + * @adapter: the adapter + * @viid: the Virtual Interface Identifier + * @free: if true any existing filters for this VI id are first removed + * @naddr: the number of MAC addresses to allocate filters for (up to 7) + * @addr: the MAC address(es) + * @idx: where to store the index of each allocated filter + * @hash: pointer to hash address filter bitmap + * @sleep_ok: call is allowed to sleep + * + * Allocates an exact-match filter for each of the supplied addresses and + * sets it to the corresponding address. If @idx is not %NULL it should + * have at least @naddr entries, each of which will be set to the index of + * the filter allocated for the corresponding MAC address. If a filter + * could not be allocated for an address its index is set to 0xffff. + * If @hash is not %NULL addresses that fail to allocate an exact filter + * are hashed and update the hash filter bitmap pointed at by @hash. + * + * Returns a negative error number or the number of filters allocated. + */ +int t4vf_alloc_mac_filt(struct adapter *adapter, unsigned int viid, bool free, + unsigned int naddr, const u8 **addr, u16 *idx, + u64 *hash, bool sleep_ok) +{ + int offset, ret = 0; + unsigned nfilters = 0; + unsigned int rem = naddr; + struct fw_vi_mac_cmd cmd, rpl; + unsigned int max_naddr = adapter->params.arch.mps_tcam_size; + + if (naddr > max_naddr) + return -EINVAL; + + for (offset = 0; offset < naddr; /**/) { + unsigned int fw_naddr = (rem < ARRAY_SIZE(cmd.u.exact) + ? rem + : ARRAY_SIZE(cmd.u.exact)); + size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd, + u.exact[fw_naddr]), 16); + struct fw_vi_mac_exact *p; + int i; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + (free ? FW_CMD_EXEC_F : 0) | + FW_VI_MAC_CMD_VIID_V(viid)); + cmd.freemacs_to_len16 = + cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(free) | + FW_CMD_LEN16_V(len16)); + + for (i = 0, p = cmd.u.exact; i < fw_naddr; i++, p++) { + p->valid_to_idx = cpu_to_be16( + FW_VI_MAC_CMD_VALID_F | + FW_VI_MAC_CMD_IDX_V(FW_VI_MAC_ADD_MAC)); + memcpy(p->macaddr, addr[offset+i], sizeof(p->macaddr)); + } + + + ret = t4vf_wr_mbox_core(adapter, &cmd, sizeof(cmd), &rpl, + sleep_ok); + if (ret && ret != -ENOMEM) + break; + + for (i = 0, p = rpl.u.exact; i < fw_naddr; i++, p++) { + u16 index = FW_VI_MAC_CMD_IDX_G( + be16_to_cpu(p->valid_to_idx)); + + if (idx) + idx[offset+i] = + (index >= max_naddr + ? 0xffff + : index); + if (index < max_naddr) + nfilters++; + else if (hash) + *hash |= (1ULL << hash_mac_addr(addr[offset+i])); + } + + free = false; + offset += fw_naddr; + rem -= fw_naddr; + } + + /* + * If there were no errors or we merely ran out of room in our MAC + * address arena, return the number of filters actually written. + */ + if (ret == 0 || ret == -ENOMEM) + ret = nfilters; + return ret; +} + +/** + * t4vf_free_mac_filt - frees exact-match filters of given MAC addresses + * @adapter: the adapter + * @viid: the VI id + * @naddr: the number of MAC addresses to allocate filters for (up to 7) + * @addr: the MAC address(es) + * @sleep_ok: call is allowed to sleep + * + * Frees the exact-match filter for each of the supplied addresses + * + * Returns a negative error number or the number of filters freed. + */ +int t4vf_free_mac_filt(struct adapter *adapter, unsigned int viid, + unsigned int naddr, const u8 **addr, bool sleep_ok) +{ + int offset, ret = 0; + struct fw_vi_mac_cmd cmd; + unsigned int nfilters = 0; + unsigned int max_naddr = adapter->params.arch.mps_tcam_size; + unsigned int rem = naddr; + + if (naddr > max_naddr) + return -EINVAL; + + for (offset = 0; offset < (int)naddr ; /**/) { + unsigned int fw_naddr = (rem < ARRAY_SIZE(cmd.u.exact) ? + rem : ARRAY_SIZE(cmd.u.exact)); + size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd, + u.exact[fw_naddr]), 16); + struct fw_vi_mac_exact *p; + int i; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_CMD_EXEC_V(0) | + FW_VI_MAC_CMD_VIID_V(viid)); + cmd.freemacs_to_len16 = + cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(0) | + FW_CMD_LEN16_V(len16)); + + for (i = 0, p = cmd.u.exact; i < (int)fw_naddr; i++, p++) { + p->valid_to_idx = cpu_to_be16( + FW_VI_MAC_CMD_VALID_F | + FW_VI_MAC_CMD_IDX_V(FW_VI_MAC_MAC_BASED_FREE)); + memcpy(p->macaddr, addr[offset+i], sizeof(p->macaddr)); + } + + ret = t4vf_wr_mbox_core(adapter, &cmd, sizeof(cmd), &cmd, + sleep_ok); + if (ret) + break; + + for (i = 0, p = cmd.u.exact; i < fw_naddr; i++, p++) { + u16 index = FW_VI_MAC_CMD_IDX_G( + be16_to_cpu(p->valid_to_idx)); + + if (index < max_naddr) + nfilters++; + } + + offset += fw_naddr; + rem -= fw_naddr; + } + + if (ret == 0) + ret = nfilters; + return ret; +} + +/** + * t4vf_change_mac - modifies the exact-match filter for a MAC address + * @adapter: the adapter + * @viid: the Virtual Interface ID + * @idx: index of existing filter for old value of MAC address, or -1 + * @addr: the new MAC address value + * @persist: if idx < 0, the new MAC allocation should be persistent + * + * Modifies an exact-match filter and sets it to the new MAC address. + * Note that in general it is not possible to modify the value of a given + * filter so the generic way to modify an address filter is to free the + * one being used by the old address value and allocate a new filter for + * the new address value. @idx can be -1 if the address is a new + * addition. + * + * Returns a negative error number or the index of the filter with the new + * MAC value. + */ +int t4vf_change_mac(struct adapter *adapter, unsigned int viid, + int idx, const u8 *addr, bool persist) +{ + int ret; + struct fw_vi_mac_cmd cmd, rpl; + struct fw_vi_mac_exact *p = &cmd.u.exact[0]; + size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd, + u.exact[1]), 16); + unsigned int max_mac_addr = adapter->params.arch.mps_tcam_size; + + /* + * If this is a new allocation, determine whether it should be + * persistent (across a "freemacs" operation) or not. + */ + if (idx < 0) + idx = persist ? FW_VI_MAC_ADD_PERSIST_MAC : FW_VI_MAC_ADD_MAC; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_VI_MAC_CMD_VIID_V(viid)); + cmd.freemacs_to_len16 = cpu_to_be32(FW_CMD_LEN16_V(len16)); + p->valid_to_idx = cpu_to_be16(FW_VI_MAC_CMD_VALID_F | + FW_VI_MAC_CMD_IDX_V(idx)); + memcpy(p->macaddr, addr, sizeof(p->macaddr)); + + ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl); + if (ret == 0) { + p = &rpl.u.exact[0]; + ret = FW_VI_MAC_CMD_IDX_G(be16_to_cpu(p->valid_to_idx)); + if (ret >= max_mac_addr) + ret = -ENOMEM; + } + return ret; +} + +/** + * t4vf_set_addr_hash - program the MAC inexact-match hash filter + * @adapter: the adapter + * @viid: the Virtual Interface Identifier + * @ucast: whether the hash filter should also match unicast addresses + * @vec: the value to be written to the hash filter + * @sleep_ok: call is allowed to sleep + * + * Sets the 64-bit inexact-match hash filter for a virtual interface. + */ +int t4vf_set_addr_hash(struct adapter *adapter, unsigned int viid, + bool ucast, u64 vec, bool sleep_ok) +{ + struct fw_vi_mac_cmd cmd; + size_t len16 = DIV_ROUND_UP(offsetof(struct fw_vi_mac_cmd, + u.exact[0]), 16); + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_VI_ENABLE_CMD_VIID_V(viid)); + cmd.freemacs_to_len16 = cpu_to_be32(FW_VI_MAC_CMD_HASHVECEN_F | + FW_VI_MAC_CMD_HASHUNIEN_V(ucast) | + FW_CMD_LEN16_V(len16)); + cmd.u.hash.hashvec = cpu_to_be64(vec); + return t4vf_wr_mbox_core(adapter, &cmd, sizeof(cmd), NULL, sleep_ok); +} + +/** + * t4vf_get_port_stats - collect "port" statistics + * @adapter: the adapter + * @pidx: the port index + * @s: the stats structure to fill + * + * Collect statistics for the "port"'s Virtual Interface. + */ +int t4vf_get_port_stats(struct adapter *adapter, int pidx, + struct t4vf_port_stats *s) +{ + struct port_info *pi = adap2pinfo(adapter, pidx); + struct fw_vi_stats_vf fwstats; + unsigned int rem = VI_VF_NUM_STATS; + __be64 *fwsp = (__be64 *)&fwstats; + + /* + * Grab the Virtual Interface statistics a chunk at a time via mailbox + * commands. We could use a Work Request and get all of them at once + * but that's an asynchronous interface which is awkward to use. + */ + while (rem) { + unsigned int ix = VI_VF_NUM_STATS - rem; + unsigned int nstats = min(6U, rem); + struct fw_vi_stats_cmd cmd, rpl; + size_t len = (offsetof(struct fw_vi_stats_cmd, u) + + sizeof(struct fw_vi_stats_ctl)); + size_t len16 = DIV_ROUND_UP(len, 16); + int ret; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_STATS_CMD) | + FW_VI_STATS_CMD_VIID_V(pi->viid) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F); + cmd.retval_len16 = cpu_to_be32(FW_CMD_LEN16_V(len16)); + cmd.u.ctl.nstats_ix = + cpu_to_be16(FW_VI_STATS_CMD_IX_V(ix) | + FW_VI_STATS_CMD_NSTATS_V(nstats)); + ret = t4vf_wr_mbox_ns(adapter, &cmd, len, &rpl); + if (ret) + return ret; + + memcpy(fwsp, &rpl.u.ctl.stat0, sizeof(__be64) * nstats); + + rem -= nstats; + fwsp += nstats; + } + + /* + * Translate firmware statistics into host native statistics. + */ + s->tx_bcast_bytes = be64_to_cpu(fwstats.tx_bcast_bytes); + s->tx_bcast_frames = be64_to_cpu(fwstats.tx_bcast_frames); + s->tx_mcast_bytes = be64_to_cpu(fwstats.tx_mcast_bytes); + s->tx_mcast_frames = be64_to_cpu(fwstats.tx_mcast_frames); + s->tx_ucast_bytes = be64_to_cpu(fwstats.tx_ucast_bytes); + s->tx_ucast_frames = be64_to_cpu(fwstats.tx_ucast_frames); + s->tx_drop_frames = be64_to_cpu(fwstats.tx_drop_frames); + s->tx_offload_bytes = be64_to_cpu(fwstats.tx_offload_bytes); + s->tx_offload_frames = be64_to_cpu(fwstats.tx_offload_frames); + + s->rx_bcast_bytes = be64_to_cpu(fwstats.rx_bcast_bytes); + s->rx_bcast_frames = be64_to_cpu(fwstats.rx_bcast_frames); + s->rx_mcast_bytes = be64_to_cpu(fwstats.rx_mcast_bytes); + s->rx_mcast_frames = be64_to_cpu(fwstats.rx_mcast_frames); + s->rx_ucast_bytes = be64_to_cpu(fwstats.rx_ucast_bytes); + s->rx_ucast_frames = be64_to_cpu(fwstats.rx_ucast_frames); + + s->rx_err_frames = be64_to_cpu(fwstats.rx_err_frames); + + return 0; +} + +/** + * t4vf_iq_free - free an ingress queue and its free lists + * @adapter: the adapter + * @iqtype: the ingress queue type (FW_IQ_TYPE_FL_INT_CAP, etc.) + * @iqid: ingress queue ID + * @fl0id: FL0 queue ID or 0xffff if no attached FL0 + * @fl1id: FL1 queue ID or 0xffff if no attached FL1 + * + * Frees an ingress queue and its associated free lists, if any. + */ +int t4vf_iq_free(struct adapter *adapter, unsigned int iqtype, + unsigned int iqid, unsigned int fl0id, unsigned int fl1id) +{ + struct fw_iq_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_IQ_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_EXEC_F); + cmd.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_FREE_F | + FW_LEN16(cmd)); + cmd.type_to_iqandstindex = + cpu_to_be32(FW_IQ_CMD_TYPE_V(iqtype)); + + cmd.iqid = cpu_to_be16(iqid); + cmd.fl0id = cpu_to_be16(fl0id); + cmd.fl1id = cpu_to_be16(fl1id); + return t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), NULL); +} + +/** + * t4vf_eth_eq_free - free an Ethernet egress queue + * @adapter: the adapter + * @eqid: egress queue ID + * + * Frees an Ethernet egress queue. + */ +int t4vf_eth_eq_free(struct adapter *adapter, unsigned int eqid) +{ + struct fw_eq_eth_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_ETH_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_EXEC_F); + cmd.alloc_to_len16 = cpu_to_be32(FW_EQ_ETH_CMD_FREE_F | + FW_LEN16(cmd)); + cmd.eqid_pkd = cpu_to_be32(FW_EQ_ETH_CMD_EQID_V(eqid)); + return t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), NULL); +} + +/** + * t4vf_link_down_rc_str - return a string for a Link Down Reason Code + * @link_down_rc: Link Down Reason Code + * + * Returns a string representation of the Link Down Reason Code. + */ +static const char *t4vf_link_down_rc_str(unsigned char link_down_rc) +{ + static const char * const reason[] = { + "Link Down", + "Remote Fault", + "Auto-negotiation Failure", + "Reserved", + "Insufficient Airflow", + "Unable To Determine Reason", + "No RX Signal Detected", + "Reserved", + }; + + if (link_down_rc >= ARRAY_SIZE(reason)) + return "Bad Reason Code"; + + return reason[link_down_rc]; +} + +/** + * t4vf_handle_get_port_info - process a FW reply message + * @pi: the port info + * @cmd: start of the FW message + * + * Processes a GET_PORT_INFO FW reply message. + */ +static void t4vf_handle_get_port_info(struct port_info *pi, + const struct fw_port_cmd *cmd) +{ + fw_port_cap32_t pcaps, acaps, lpacaps, linkattr; + struct link_config *lc = &pi->link_cfg; + struct adapter *adapter = pi->adapter; + unsigned int speed, fc, fec, adv_fc; + enum fw_port_module_type mod_type; + int action, link_ok, linkdnrc; + enum fw_port_type port_type; + + /* Extract the various fields from the Port Information message. */ + action = FW_PORT_CMD_ACTION_G(be32_to_cpu(cmd->action_to_len16)); + switch (action) { + case FW_PORT_ACTION_GET_PORT_INFO: { + u32 lstatus = be32_to_cpu(cmd->u.info.lstatus_to_modtype); + + link_ok = (lstatus & FW_PORT_CMD_LSTATUS_F) != 0; + linkdnrc = FW_PORT_CMD_LINKDNRC_G(lstatus); + port_type = FW_PORT_CMD_PTYPE_G(lstatus); + mod_type = FW_PORT_CMD_MODTYPE_G(lstatus); + pcaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.pcap)); + acaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.acap)); + lpacaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.lpacap)); + + /* Unfortunately the format of the Link Status in the old + * 16-bit Port Information message isn't the same as the + * 16-bit Port Capabilities bitfield used everywhere else ... + */ + linkattr = 0; + if (lstatus & FW_PORT_CMD_RXPAUSE_F) + linkattr |= FW_PORT_CAP32_FC_RX; + if (lstatus & FW_PORT_CMD_TXPAUSE_F) + linkattr |= FW_PORT_CAP32_FC_TX; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100M)) + linkattr |= FW_PORT_CAP32_SPEED_100M; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_1G)) + linkattr |= FW_PORT_CAP32_SPEED_1G; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_10G)) + linkattr |= FW_PORT_CAP32_SPEED_10G; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_25G)) + linkattr |= FW_PORT_CAP32_SPEED_25G; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_40G)) + linkattr |= FW_PORT_CAP32_SPEED_40G; + if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100G)) + linkattr |= FW_PORT_CAP32_SPEED_100G; + + break; + } + + case FW_PORT_ACTION_GET_PORT_INFO32: { + u32 lstatus32; + + lstatus32 = be32_to_cpu(cmd->u.info32.lstatus32_to_cbllen32); + link_ok = (lstatus32 & FW_PORT_CMD_LSTATUS32_F) != 0; + linkdnrc = FW_PORT_CMD_LINKDNRC32_G(lstatus32); + port_type = FW_PORT_CMD_PORTTYPE32_G(lstatus32); + mod_type = FW_PORT_CMD_MODTYPE32_G(lstatus32); + pcaps = be32_to_cpu(cmd->u.info32.pcaps32); + acaps = be32_to_cpu(cmd->u.info32.acaps32); + lpacaps = be32_to_cpu(cmd->u.info32.lpacaps32); + linkattr = be32_to_cpu(cmd->u.info32.linkattr32); + break; + } + + default: + dev_err(adapter->pdev_dev, "Handle Port Information: Bad Command/Action %#x\n", + be32_to_cpu(cmd->action_to_len16)); + return; + } + + fec = fwcap_to_cc_fec(acaps); + adv_fc = fwcap_to_cc_pause(acaps); + fc = fwcap_to_cc_pause(linkattr); + speed = fwcap_to_speed(linkattr); + + if (mod_type != pi->mod_type) { + /* When a new Transceiver Module is inserted, the Firmware + * will examine any Forward Error Correction parameters + * present in the Transceiver Module i2c EPROM and determine + * the supported and recommended FEC settings from those + * based on IEEE 802.3 standards. We always record the + * IEEE 802.3 recommended "automatic" settings. + */ + lc->auto_fec = fec; + + /* Some versions of the early T6 Firmware "cheated" when + * handling different Transceiver Modules by changing the + * underlaying Port Type reported to the Host Drivers. As + * such we need to capture whatever Port Type the Firmware + * sends us and record it in case it's different from what we + * were told earlier. Unfortunately, since Firmware is + * forever, we'll need to keep this code here forever, but in + * later T6 Firmware it should just be an assignment of the + * same value already recorded. + */ + pi->port_type = port_type; + + pi->mod_type = mod_type; + t4vf_os_portmod_changed(adapter, pi->pidx); + } + + if (link_ok != lc->link_ok || speed != lc->speed || + fc != lc->fc || adv_fc != lc->advertised_fc || + fec != lc->fec) { + /* something changed */ + if (!link_ok && lc->link_ok) { + lc->link_down_rc = linkdnrc; + dev_warn_ratelimited(adapter->pdev_dev, + "Port %d link down, reason: %s\n", + pi->port_id, + t4vf_link_down_rc_str(linkdnrc)); + } + lc->link_ok = link_ok; + lc->speed = speed; + lc->advertised_fc = adv_fc; + lc->fc = fc; + lc->fec = fec; + + lc->pcaps = pcaps; + lc->lpacaps = lpacaps; + lc->acaps = acaps & ADVERT_MASK; + + /* If we're not physically capable of Auto-Negotiation, note + * this as Auto-Negotiation disabled. Otherwise, we track + * what Auto-Negotiation settings we have. Note parallel + * structure in init_link_config(). + */ + if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) { + lc->autoneg = AUTONEG_DISABLE; + } else if (lc->acaps & FW_PORT_CAP32_ANEG) { + lc->autoneg = AUTONEG_ENABLE; + } else { + /* When Autoneg is disabled, user needs to set + * single speed. + * Similar to cxgb4_ethtool.c: set_link_ksettings + */ + lc->acaps = 0; + lc->speed_caps = fwcap_to_speed(acaps); + lc->autoneg = AUTONEG_DISABLE; + } + + t4vf_os_link_changed(adapter, pi->pidx, link_ok); + } +} + +/** + * t4vf_update_port_info - retrieve and update port information if changed + * @pi: the port_info + * + * We issue a Get Port Information Command to the Firmware and, if + * successful, we check to see if anything is different from what we + * last recorded and update things accordingly. + */ +int t4vf_update_port_info(struct port_info *pi) +{ + unsigned int fw_caps = pi->adapter->params.fw_caps_support; + struct fw_port_cmd port_cmd; + int ret; + + memset(&port_cmd, 0, sizeof(port_cmd)); + port_cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F | + FW_PORT_CMD_PORTID_V(pi->port_id)); + port_cmd.action_to_len16 = cpu_to_be32( + FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16 + ? FW_PORT_ACTION_GET_PORT_INFO + : FW_PORT_ACTION_GET_PORT_INFO32) | + FW_LEN16(port_cmd)); + ret = t4vf_wr_mbox(pi->adapter, &port_cmd, sizeof(port_cmd), + &port_cmd); + if (ret) + return ret; + t4vf_handle_get_port_info(pi, &port_cmd); + return 0; +} + +/** + * t4vf_handle_fw_rpl - process a firmware reply message + * @adapter: the adapter + * @rpl: start of the firmware message + * + * Processes a firmware message, such as link state change messages. + */ +int t4vf_handle_fw_rpl(struct adapter *adapter, const __be64 *rpl) +{ + const struct fw_cmd_hdr *cmd_hdr = (const struct fw_cmd_hdr *)rpl; + u8 opcode = FW_CMD_OP_G(be32_to_cpu(cmd_hdr->hi)); + + switch (opcode) { + case FW_PORT_CMD: { + /* + * Link/module state change message. + */ + const struct fw_port_cmd *port_cmd = + (const struct fw_port_cmd *)rpl; + int action = FW_PORT_CMD_ACTION_G( + be32_to_cpu(port_cmd->action_to_len16)); + int port_id, pidx; + + if (action != FW_PORT_ACTION_GET_PORT_INFO && + action != FW_PORT_ACTION_GET_PORT_INFO32) { + dev_err(adapter->pdev_dev, + "Unknown firmware PORT reply action %x\n", + action); + break; + } + + port_id = FW_PORT_CMD_PORTID_G( + be32_to_cpu(port_cmd->op_to_portid)); + for_each_port(adapter, pidx) { + struct port_info *pi = adap2pinfo(adapter, pidx); + + if (pi->port_id != port_id) + continue; + t4vf_handle_get_port_info(pi, port_cmd); + } + break; + } + + default: + dev_err(adapter->pdev_dev, "Unknown firmware reply %X\n", + opcode); + } + return 0; +} + +int t4vf_prep_adapter(struct adapter *adapter) +{ + int err; + unsigned int chipid; + + /* Wait for the device to become ready before proceeding ... + */ + err = t4vf_wait_dev_ready(adapter); + if (err) + return err; + + /* Default port and clock for debugging in case we can't reach + * firmware. + */ + adapter->params.nports = 1; + adapter->params.vfres.pmask = 1; + adapter->params.vpd.cclk = 50000; + + adapter->params.chip = 0; + switch (CHELSIO_PCI_ID_VER(adapter->pdev->device)) { + case CHELSIO_T4: + adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T4, 0); + adapter->params.arch.sge_fl_db = DBPRIO_F; + adapter->params.arch.mps_tcam_size = + NUM_MPS_CLS_SRAM_L_INSTANCES; + break; + + case CHELSIO_T5: + chipid = REV_G(t4_read_reg(adapter, PL_VF_REV_A)); + adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T5, chipid); + adapter->params.arch.sge_fl_db = DBPRIO_F | DBTYPE_F; + adapter->params.arch.mps_tcam_size = + NUM_MPS_T5_CLS_SRAM_L_INSTANCES; + break; + + case CHELSIO_T6: + chipid = REV_G(t4_read_reg(adapter, PL_VF_REV_A)); + adapter->params.chip |= CHELSIO_CHIP_CODE(CHELSIO_T6, chipid); + adapter->params.arch.sge_fl_db = 0; + adapter->params.arch.mps_tcam_size = + NUM_MPS_T5_CLS_SRAM_L_INSTANCES; + break; + } + + return 0; +} + +/** + * t4vf_get_vf_mac_acl - Get the MAC address to be set to + * the VI of this VF. + * @adapter: The adapter + * @port: The port associated with vf + * @naddr: the number of ACL MAC addresses returned in addr + * @addr: Placeholder for MAC addresses + * + * Find the MAC address to be set to the VF's VI. The requested MAC address + * is from the host OS via callback in the PF driver. + */ +int t4vf_get_vf_mac_acl(struct adapter *adapter, unsigned int port, + unsigned int *naddr, u8 *addr) +{ + struct fw_acl_mac_cmd cmd; + int ret; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_ACL_MAC_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F); + cmd.en_to_len16 = cpu_to_be32((unsigned int)FW_LEN16(cmd)); + ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &cmd); + if (ret) + return ret; + + if (cmd.nmac < *naddr) + *naddr = cmd.nmac; + + switch (port) { + case 3: + memcpy(addr, cmd.macaddr3, sizeof(cmd.macaddr3)); + break; + case 2: + memcpy(addr, cmd.macaddr2, sizeof(cmd.macaddr2)); + break; + case 1: + memcpy(addr, cmd.macaddr1, sizeof(cmd.macaddr1)); + break; + case 0: + memcpy(addr, cmd.macaddr0, sizeof(cmd.macaddr0)); + break; + } + + return ret; +} + +/** + * t4vf_get_vf_vlan_acl - Get the VLAN ID to be set to + * the VI of this VF. + * @adapter: The adapter + * + * Find the VLAN ID to be set to the VF's VI. The requested VLAN ID + * is from the host OS via callback in the PF driver. + */ +int t4vf_get_vf_vlan_acl(struct adapter *adapter) +{ + struct fw_acl_vlan_cmd cmd; + int vlan = 0; + int ret = 0; + + cmd.op_to_vfn = htonl(FW_CMD_OP_V(FW_ACL_VLAN_CMD) | + FW_CMD_REQUEST_F | FW_CMD_READ_F); + + /* Note: Do not enable the ACL */ + cmd.en_to_len16 = cpu_to_be32((unsigned int)FW_LEN16(cmd)); + + ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &cmd); + + if (!ret) + vlan = be16_to_cpu(cmd.vlanid[0]); + + return vlan; +} diff --git a/drivers/net/ethernet/chelsio/inline_crypto/Kconfig b/drivers/net/ethernet/chelsio/inline_crypto/Kconfig new file mode 100644 index 000000000..521955e1f --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/Kconfig @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Chelsio inline crypto configuration +# + +config CHELSIO_INLINE_CRYPTO + bool "Chelsio Inline Crypto support" + depends on CHELSIO_T4 + default y + help + Enable support for inline crypto. + Allows enable/disable from list of inline crypto drivers. + +if CHELSIO_INLINE_CRYPTO + +config CRYPTO_DEV_CHELSIO_TLS + tristate "Chelsio Crypto Inline TLS Driver" + depends on CHELSIO_T4 + depends on TLS + depends on TLS_TOE + help + Support Chelsio Inline TLS with Chelsio crypto accelerator. + Enable inline TLS support for Tx and Rx. + + To compile this driver as a module, choose M here: the module + will be called chtls. + +config CHELSIO_IPSEC_INLINE + tristate "Chelsio IPSec XFRM Tx crypto offload" + depends on CHELSIO_T4 + depends on XFRM_OFFLOAD + depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD + help + Support Chelsio Inline IPsec with Chelsio crypto accelerator. + Enable inline IPsec support for Tx. + + To compile this driver as a module, choose M here: the module + will be called ch_ipsec. + +config CHELSIO_TLS_DEVICE + tristate "Chelsio Inline KTLS Offload" + depends on CHELSIO_T4 + depends on TLS + depends on TLS_DEVICE + select CRYPTO_LIB_AES + help + This flag enables support for kernel tls offload over Chelsio T6 + crypto accelerator. CONFIG_CHELSIO_TLS_DEVICE flag can be enabled + only if CONFIG_TLS and CONFIG_TLS_DEVICE flags are enabled. + + To compile this driver as a module, choose M here: the module + will be called ch_ktls. + +endif # CHELSIO_INLINE_CRYPTO diff --git a/drivers/net/ethernet/chelsio/inline_crypto/Makefile b/drivers/net/ethernet/chelsio/inline_crypto/Makefile new file mode 100644 index 000000000..27e6d7e2f --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/ +obj-$(CONFIG_CHELSIO_IPSEC_INLINE) += ch_ipsec/ +obj-$(CONFIG_CHELSIO_TLS_DEVICE) += ch_ktls/ diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/Makefile b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/Makefile new file mode 100644 index 000000000..efdcaaebc --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 \ + -I $(srctree)/drivers/crypto/chelsio + +obj-$(CONFIG_CHELSIO_IPSEC_INLINE) += ch_ipsec.o +ch_ipsec-objs := chcr_ipsec.o + + diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c new file mode 100644 index 000000000..585590520 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c @@ -0,0 +1,823 @@ +/* + * This file is part of the Chelsio T6 Crypto driver for Linux. + * + * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Written and Maintained by: + * Atul Gupta (atul.gupta@chelsio.com) + */ + +#define pr_fmt(fmt) "ch_ipsec: " fmt + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/crypto.h> +#include <linux/skbuff.h> +#include <linux/rtnetlink.h> +#include <linux/highmem.h> +#include <linux/if_vlan.h> +#include <linux/ip.h> +#include <linux/netdevice.h> +#include <net/esp.h> +#include <net/xfrm.h> +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <crypto/hash.h> +#include <crypto/sha1.h> +#include <crypto/sha2.h> +#include <crypto/authenc.h> +#include <crypto/internal/aead.h> +#include <crypto/null.h> +#include <crypto/internal/skcipher.h> +#include <crypto/aead.h> +#include <crypto/scatterwalk.h> +#include <crypto/internal/hash.h> + +#include "chcr_ipsec.h" + +/* + * Max Tx descriptor space we allow for an Ethernet packet to be inlined + * into a WR. + */ +#define MAX_IMM_TX_PKT_LEN 256 +#define GCM_ESP_IV_SIZE 8 + +static LIST_HEAD(uld_ctx_list); +static DEFINE_MUTEX(dev_mutex); + +static bool ch_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x); +static int ch_ipsec_uld_state_change(void *handle, enum cxgb4_state new_state); +static int ch_ipsec_xmit(struct sk_buff *skb, struct net_device *dev); +static void *ch_ipsec_uld_add(const struct cxgb4_lld_info *infop); +static void ch_ipsec_advance_esn_state(struct xfrm_state *x); +static void ch_ipsec_xfrm_free_state(struct xfrm_state *x); +static void ch_ipsec_xfrm_del_state(struct xfrm_state *x); +static int ch_ipsec_xfrm_add_state(struct xfrm_state *x); + +static const struct xfrmdev_ops ch_ipsec_xfrmdev_ops = { + .xdo_dev_state_add = ch_ipsec_xfrm_add_state, + .xdo_dev_state_delete = ch_ipsec_xfrm_del_state, + .xdo_dev_state_free = ch_ipsec_xfrm_free_state, + .xdo_dev_offload_ok = ch_ipsec_offload_ok, + .xdo_dev_state_advance_esn = ch_ipsec_advance_esn_state, +}; + +static struct cxgb4_uld_info ch_ipsec_uld_info = { + .name = CHIPSEC_DRV_MODULE_NAME, + .add = ch_ipsec_uld_add, + .state_change = ch_ipsec_uld_state_change, + .tx_handler = ch_ipsec_xmit, + .xfrmdev_ops = &ch_ipsec_xfrmdev_ops, +}; + +static void *ch_ipsec_uld_add(const struct cxgb4_lld_info *infop) +{ + struct ipsec_uld_ctx *u_ctx; + + pr_info_once("%s - version %s\n", CHIPSEC_DRV_DESC, + CHIPSEC_DRV_VERSION); + u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL); + if (!u_ctx) { + u_ctx = ERR_PTR(-ENOMEM); + goto out; + } + u_ctx->lldi = *infop; +out: + return u_ctx; +} + +static int ch_ipsec_uld_state_change(void *handle, enum cxgb4_state new_state) +{ + struct ipsec_uld_ctx *u_ctx = handle; + + pr_debug("new_state %u\n", new_state); + switch (new_state) { + case CXGB4_STATE_UP: + pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev)); + mutex_lock(&dev_mutex); + list_add_tail(&u_ctx->entry, &uld_ctx_list); + mutex_unlock(&dev_mutex); + break; + case CXGB4_STATE_START_RECOVERY: + case CXGB4_STATE_DOWN: + case CXGB4_STATE_DETACH: + pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev)); + list_del(&u_ctx->entry); + break; + default: + break; + } + + return 0; +} + +static int ch_ipsec_setauthsize(struct xfrm_state *x, + struct ipsec_sa_entry *sa_entry) +{ + int hmac_ctrl; + int authsize = x->aead->alg_icv_len / 8; + + sa_entry->authsize = authsize; + + switch (authsize) { + case ICV_8: + hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; + break; + case ICV_12: + hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; + break; + case ICV_16: + hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; + break; + default: + return -EINVAL; + } + return hmac_ctrl; +} + +static int ch_ipsec_setkey(struct xfrm_state *x, + struct ipsec_sa_entry *sa_entry) +{ + int keylen = (x->aead->alg_key_len + 7) / 8; + unsigned char *key = x->aead->alg_key; + int ck_size, key_ctx_size = 0; + unsigned char ghash_h[AEAD_H_SIZE]; + struct crypto_aes_ctx aes; + int ret = 0; + + if (keylen > 3) { + keylen -= 4; /* nonce/salt is present in the last 4 bytes */ + memcpy(sa_entry->salt, key + keylen, 4); + } + + if (keylen == AES_KEYSIZE_128) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + } else if (keylen == AES_KEYSIZE_192) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; + } else if (keylen == AES_KEYSIZE_256) { + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; + } else { + pr_err("GCM: Invalid key length %d\n", keylen); + ret = -EINVAL; + goto out; + } + + memcpy(sa_entry->key, key, keylen); + sa_entry->enckey_len = keylen; + key_ctx_size = sizeof(struct _key_ctx) + + ((DIV_ROUND_UP(keylen, 16)) << 4) + + AEAD_H_SIZE; + + sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, + CHCR_KEYCTX_MAC_KEY_SIZE_128, + 0, 0, + key_ctx_size >> 4); + + /* Calculate the H = CIPH(K, 0 repeated 16 times). + * It will go in key context + */ + ret = aes_expandkey(&aes, key, keylen); + if (ret) { + sa_entry->enckey_len = 0; + goto out; + } + memset(ghash_h, 0, AEAD_H_SIZE); + aes_encrypt(&aes, ghash_h, ghash_h); + memzero_explicit(&aes, sizeof(aes)); + + memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) * + 16), ghash_h, AEAD_H_SIZE); + sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) + + AEAD_H_SIZE; +out: + return ret; +} + +/* + * ch_ipsec_xfrm_add_state + * returns 0 on success, negative error if failed to send message to FPGA + * positive error if FPGA returned a bad response + */ +static int ch_ipsec_xfrm_add_state(struct xfrm_state *x) +{ + struct ipsec_sa_entry *sa_entry; + int res = 0; + + if (x->props.aalgo != SADB_AALG_NONE) { + pr_debug("Cannot offload authenticated xfrm states\n"); + return -EINVAL; + } + if (x->props.calgo != SADB_X_CALG_NONE) { + pr_debug("Cannot offload compressed xfrm states\n"); + return -EINVAL; + } + if (x->props.family != AF_INET && + x->props.family != AF_INET6) { + pr_debug("Only IPv4/6 xfrm state offloaded\n"); + return -EINVAL; + } + if (x->props.mode != XFRM_MODE_TRANSPORT && + x->props.mode != XFRM_MODE_TUNNEL) { + pr_debug("Only transport and tunnel xfrm offload\n"); + return -EINVAL; + } + if (x->id.proto != IPPROTO_ESP) { + pr_debug("Only ESP xfrm state offloaded\n"); + return -EINVAL; + } + if (x->encap) { + pr_debug("Encapsulated xfrm state not offloaded\n"); + return -EINVAL; + } + if (!x->aead) { + pr_debug("Cannot offload xfrm states without aead\n"); + return -EINVAL; + } + if (x->aead->alg_icv_len != 128 && + x->aead->alg_icv_len != 96) { + pr_debug("Cannot offload xfrm states with AEAD ICV length other than 96b & 128b\n"); + return -EINVAL; + } + if ((x->aead->alg_key_len != 128 + 32) && + (x->aead->alg_key_len != 256 + 32)) { + pr_debug("cannot offload xfrm states with AEAD key length other than 128/256 bit\n"); + return -EINVAL; + } + if (x->tfcpad) { + pr_debug("Cannot offload xfrm states with tfc padding\n"); + return -EINVAL; + } + if (!x->geniv) { + pr_debug("Cannot offload xfrm states without geniv\n"); + return -EINVAL; + } + if (strcmp(x->geniv, "seqiv")) { + pr_debug("Cannot offload xfrm states with geniv other than seqiv\n"); + return -EINVAL; + } + + sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); + if (!sa_entry) { + res = -ENOMEM; + goto out; + } + + sa_entry->hmac_ctrl = ch_ipsec_setauthsize(x, sa_entry); + if (x->props.flags & XFRM_STATE_ESN) + sa_entry->esn = 1; + ch_ipsec_setkey(x, sa_entry); + x->xso.offload_handle = (unsigned long)sa_entry; + try_module_get(THIS_MODULE); +out: + return res; +} + +static void ch_ipsec_xfrm_del_state(struct xfrm_state *x) +{ + /* do nothing */ + if (!x->xso.offload_handle) + return; +} + +static void ch_ipsec_xfrm_free_state(struct xfrm_state *x) +{ + struct ipsec_sa_entry *sa_entry; + + if (!x->xso.offload_handle) + return; + + sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle; + kfree(sa_entry); + module_put(THIS_MODULE); +} + +static bool ch_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) +{ + if (x->props.family == AF_INET) { + /* Offload with IP options is not supported yet */ + if (ip_hdr(skb)->ihl > 5) + return false; + } else { + /* Offload with IPv6 extension headers is not support yet */ + if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)) + return false; + } + return true; +} + +static void ch_ipsec_advance_esn_state(struct xfrm_state *x) +{ + /* do nothing */ + if (!x->xso.offload_handle) + return; +} + +static int is_eth_imm(const struct sk_buff *skb, + struct ipsec_sa_entry *sa_entry) +{ + unsigned int kctx_len; + int hdrlen; + + kctx_len = sa_entry->kctx_len; + hdrlen = sizeof(struct fw_ulptx_wr) + + sizeof(struct chcr_ipsec_req) + kctx_len; + + hdrlen += sizeof(struct cpl_tx_pkt); + if (sa_entry->esn) + hdrlen += (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16) + << 4); + if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen) + return hdrlen; + return 0; +} + +static unsigned int calc_tx_sec_flits(const struct sk_buff *skb, + struct ipsec_sa_entry *sa_entry, + bool *immediate) +{ + unsigned int kctx_len; + unsigned int flits; + int aadivlen; + int hdrlen; + + kctx_len = sa_entry->kctx_len; + hdrlen = is_eth_imm(skb, sa_entry); + aadivlen = sa_entry->esn ? DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), + 16) : 0; + aadivlen <<= 4; + + /* If the skb is small enough, we can pump it out as a work request + * with only immediate data. In that case we just have to have the + * TX Packet header plus the skb data in the Work Request. + */ + + if (hdrlen) { + *immediate = true; + return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64)); + } + + flits = sgl_len(skb_shinfo(skb)->nr_frags + 1); + + /* Otherwise, we're going to have to construct a Scatter gather list + * of the skb body and fragments. We also include the flits necessary + * for the TX Packet Work Request and CPL. We always have a firmware + * Write Header (incorporated as part of the cpl_tx_pkt_lso and + * cpl_tx_pkt structures), followed by either a TX Packet Write CPL + * message or, if we're doing a Large Send Offload, an LSO CPL message + * with an embedded TX Packet Write CPL message. + */ + flits += (sizeof(struct fw_ulptx_wr) + + sizeof(struct chcr_ipsec_req) + + kctx_len + + sizeof(struct cpl_tx_pkt_core) + + aadivlen) / sizeof(__be64); + return flits; +} + +static void *copy_esn_pktxt(struct sk_buff *skb, + struct net_device *dev, + void *pos, + struct ipsec_sa_entry *sa_entry) +{ + struct chcr_ipsec_aadiv *aadiv; + struct ulptx_idata *sc_imm; + struct ip_esp_hdr *esphdr; + struct xfrm_offload *xo; + struct sge_eth_txq *q; + struct adapter *adap; + struct port_info *pi; + __be64 seqno; + u32 qidx; + u32 seqlo; + u8 *iv; + int eoq; + int len; + + pi = netdev_priv(dev); + adap = pi->adapter; + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + pi->first_qset]; + + /* end of queue, reset pos to start of queue */ + eoq = (void *)q->q.stat - pos; + if (!eoq) + pos = q->q.desc; + + len = DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16) << 4; + memset(pos, 0, len); + aadiv = (struct chcr_ipsec_aadiv *)pos; + esphdr = (struct ip_esp_hdr *)skb_transport_header(skb); + iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr); + xo = xfrm_offload(skb); + + aadiv->spi = (esphdr->spi); + seqlo = ntohl(esphdr->seq_no); + seqno = cpu_to_be64(seqlo + ((u64)xo->seq.hi << 32)); + memcpy(aadiv->seq_no, &seqno, 8); + iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr); + memcpy(aadiv->iv, iv, 8); + + if (is_eth_imm(skb, sa_entry) && !skb_is_nonlinear(skb)) { + sc_imm = (struct ulptx_idata *)(pos + + (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), + sizeof(__be64)) << 3)); + sc_imm->cmd_more = FILL_CMD_MORE(0); + sc_imm->len = cpu_to_be32(skb->len); + } + pos += len; + return pos; +} + +static void *copy_cpltx_pktxt(struct sk_buff *skb, + struct net_device *dev, + void *pos, + struct ipsec_sa_entry *sa_entry) +{ + struct cpl_tx_pkt_core *cpl; + struct sge_eth_txq *q; + struct adapter *adap; + struct port_info *pi; + u32 ctrl0, qidx; + u64 cntrl = 0; + int left; + + pi = netdev_priv(dev); + adap = pi->adapter; + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + pi->first_qset]; + + left = (void *)q->q.stat - pos; + if (!left) + pos = q->q.desc; + + cpl = (struct cpl_tx_pkt_core *)pos; + + cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F; + ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) | + TXPKT_PF_V(adap->pf); + if (skb_vlan_tag_present(skb)) { + q->vlan_ins++; + cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb)); + } + + cpl->ctrl0 = htonl(ctrl0); + cpl->pack = htons(0); + cpl->len = htons(skb->len); + cpl->ctrl1 = cpu_to_be64(cntrl); + + pos += sizeof(struct cpl_tx_pkt_core); + /* Copy ESN info for HW */ + if (sa_entry->esn) + pos = copy_esn_pktxt(skb, dev, pos, sa_entry); + return pos; +} + +static void *copy_key_cpltx_pktxt(struct sk_buff *skb, + struct net_device *dev, + void *pos, + struct ipsec_sa_entry *sa_entry) +{ + struct _key_ctx *key_ctx; + int left, eoq, key_len; + struct sge_eth_txq *q; + struct adapter *adap; + struct port_info *pi; + unsigned int qidx; + + pi = netdev_priv(dev); + adap = pi->adapter; + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + pi->first_qset]; + key_len = sa_entry->kctx_len; + + /* end of queue, reset pos to start of queue */ + eoq = (void *)q->q.stat - pos; + left = eoq; + if (!eoq) { + pos = q->q.desc; + left = 64 * q->q.size; + } + + /* Copy the Key context header */ + key_ctx = (struct _key_ctx *)pos; + key_ctx->ctx_hdr = sa_entry->key_ctx_hdr; + memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT); + pos += sizeof(struct _key_ctx); + left -= sizeof(struct _key_ctx); + + if (likely(key_len <= left)) { + memcpy(key_ctx->key, sa_entry->key, key_len); + pos += key_len; + } else { + memcpy(pos, sa_entry->key, left); + memcpy(q->q.desc, sa_entry->key + left, + key_len - left); + pos = (u8 *)q->q.desc + (key_len - left); + } + /* Copy CPL TX PKT XT */ + pos = copy_cpltx_pktxt(skb, dev, pos, sa_entry); + + return pos; +} + +static void *ch_ipsec_crypto_wreq(struct sk_buff *skb, + struct net_device *dev, + void *pos, + int credits, + struct ipsec_sa_entry *sa_entry) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + unsigned int ivsize = GCM_ESP_IV_SIZE; + struct chcr_ipsec_wr *wr; + bool immediate = false; + u16 immdatalen = 0; + unsigned int flits; + u32 ivinoffset; + u32 aadstart; + u32 aadstop; + u32 ciphstart; + u16 sc_more = 0; + u32 ivdrop = 0; + u32 esnlen = 0; + u32 wr_mid; + u16 ndesc; + int qidx = skb_get_queue_mapping(skb); + struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset]; + unsigned int kctx_len = sa_entry->kctx_len; + int qid = q->q.cntxt_id; + + atomic_inc(&adap->ch_ipsec_stats.ipsec_cnt); + + flits = calc_tx_sec_flits(skb, sa_entry, &immediate); + ndesc = DIV_ROUND_UP(flits, 2); + if (sa_entry->esn) + ivdrop = 1; + + if (immediate) + immdatalen = skb->len; + + if (sa_entry->esn) { + esnlen = sizeof(struct chcr_ipsec_aadiv); + if (!skb_is_nonlinear(skb)) + sc_more = 1; + } + + /* WR Header */ + wr = (struct chcr_ipsec_wr *)pos; + wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(ndesc); + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + netif_tx_stop_queue(q->txq); + q->q.stops++; + if (!q->dbqt) + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + wr_mid |= FW_ULPTX_WR_DATA_F; + wr->wreq.flowid_len16 = htonl(wr_mid); + + /* ULPTX */ + wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid); + wr->req.ulptx.len = htonl(ndesc - 1); + + /* Sub-command */ + wr->req.sc_imm.cmd_more = FILL_CMD_MORE(!immdatalen || sc_more); + wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + + sizeof(wr->req.key_ctx) + + kctx_len + + sizeof(struct cpl_tx_pkt_core) + + esnlen + + (esnlen ? 0 : immdatalen)); + + /* CPL_SEC_PDU */ + ivinoffset = sa_entry->esn ? (ESN_IV_INSERT_OFFSET + 1) : + (skb_transport_offset(skb) + + sizeof(struct ip_esp_hdr) + 1); + wr->req.sec_cpl.op_ivinsrtofst = htonl( + CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | + CPL_TX_SEC_PDU_CPLLEN_V(2) | + CPL_TX_SEC_PDU_PLACEHOLDER_V(1) | + CPL_TX_SEC_PDU_IVINSRTOFST_V( + ivinoffset)); + + wr->req.sec_cpl.pldlen = htonl(skb->len + esnlen); + aadstart = sa_entry->esn ? 1 : (skb_transport_offset(skb) + 1); + aadstop = sa_entry->esn ? ESN_IV_INSERT_OFFSET : + (skb_transport_offset(skb) + + sizeof(struct ip_esp_hdr)); + ciphstart = skb_transport_offset(skb) + sizeof(struct ip_esp_hdr) + + GCM_ESP_IV_SIZE + 1; + ciphstart += sa_entry->esn ? esnlen : 0; + + wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( + aadstart, + aadstop, + ciphstart, 0); + + wr->req.sec_cpl.cipherstop_lo_authinsert = + FILL_SEC_CPL_AUTHINSERT(0, ciphstart, + sa_entry->authsize, + sa_entry->authsize); + wr->req.sec_cpl.seqno_numivs = + FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1, + CHCR_SCMD_CIPHER_MODE_AES_GCM, + CHCR_SCMD_AUTH_MODE_GHASH, + sa_entry->hmac_ctrl, + ivsize >> 1); + wr->req.sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, + 0, ivdrop, 0); + + pos += sizeof(struct fw_ulptx_wr) + + sizeof(struct ulp_txpkt) + + sizeof(struct ulptx_idata) + + sizeof(struct cpl_tx_sec_pdu); + + pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry); + + return pos; +} + +/** + * flits_to_desc - returns the num of Tx descriptors for the given flits + * @n: the number of flits + * + * Returns the number of Tx descriptors needed for the supplied number + * of flits. + */ +static unsigned int flits_to_desc(unsigned int n) +{ + WARN_ON(n > SGE_MAX_WR_LEN / 8); + return DIV_ROUND_UP(n, 8); +} + +static unsigned int txq_avail(const struct sge_txq *q) +{ + return q->size - 1 - q->in_use; +} + +static void eth_txq_stop(struct sge_eth_txq *q) +{ + netif_tx_stop_queue(q->txq); + q->q.stops++; +} + +static void txq_advance(struct sge_txq *q, unsigned int n) +{ + q->in_use += n; + q->pidx += n; + if (q->pidx >= q->size) + q->pidx -= q->size; +} + +/* + * ch_ipsec_xmit called from ULD Tx handler + */ +int ch_ipsec_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct xfrm_state *x = xfrm_input_state(skb); + unsigned int last_desc, ndesc, flits = 0; + struct ipsec_sa_entry *sa_entry; + u64 *pos, *end, *before, *sgl; + struct tx_sw_desc *sgl_sdesc; + int qidx, left, credits; + bool immediate = false; + struct sge_eth_txq *q; + struct adapter *adap; + struct port_info *pi; + struct sec_path *sp; + + if (!x->xso.offload_handle) + return NETDEV_TX_BUSY; + + sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle; + + sp = skb_sec_path(skb); + if (sp->len != 1) { +out_free: dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + + pi = netdev_priv(dev); + adap = pi->adapter; + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + pi->first_qset]; + + cxgb4_reclaim_completed_tx(adap, &q->q, true); + + flits = calc_tx_sec_flits(skb, sa_entry, &immediate); + ndesc = flits_to_desc(flits); + credits = txq_avail(&q->q) - ndesc; + + if (unlikely(credits < 0)) { + eth_txq_stop(q); + dev_err(adap->pdev_dev, + "%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n", + dev->name, qidx, credits, ndesc, txq_avail(&q->q), + flits); + return NETDEV_TX_BUSY; + } + + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + sgl_sdesc = &q->q.sdesc[last_desc]; + + if (!immediate && + unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { + memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); + q->mapping_err++; + goto out_free; + } + + pos = (u64 *)&q->q.desc[q->q.pidx]; + before = (u64 *)pos; + end = (u64 *)pos + flits; + /* Setup IPSec CPL */ + pos = (void *)ch_ipsec_crypto_wreq(skb, dev, (void *)pos, + credits, sa_entry); + if (before > (u64 *)pos) { + left = (u8 *)end - (u8 *)q->q.stat; + end = (void *)q->q.desc + left; + } + if (pos == (u64 *)q->q.stat) { + left = (u8 *)end - (u8 *)q->q.stat; + end = (void *)q->q.desc + left; + pos = (void *)q->q.desc; + } + + sgl = (void *)pos; + if (immediate) { + cxgb4_inline_tx_skb(skb, &q->q, sgl); + dev_consume_skb_any(skb); + } else { + cxgb4_write_sgl(skb, &q->q, (void *)sgl, end, + 0, sgl_sdesc->addr); + skb_orphan(skb); + sgl_sdesc->skb = skb; + } + txq_advance(&q->q, ndesc); + + cxgb4_ring_tx_db(adap, &q->q, ndesc); + return NETDEV_TX_OK; +} + +static int __init ch_ipsec_init(void) +{ + cxgb4_register_uld(CXGB4_ULD_IPSEC, &ch_ipsec_uld_info); + + return 0; +} + +static void __exit ch_ipsec_exit(void) +{ + struct ipsec_uld_ctx *u_ctx, *tmp; + struct adapter *adap; + + mutex_lock(&dev_mutex); + list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) { + adap = pci_get_drvdata(u_ctx->lldi.pdev); + atomic_set(&adap->ch_ipsec_stats.ipsec_cnt, 0); + list_del(&u_ctx->entry); + kfree(u_ctx); + } + mutex_unlock(&dev_mutex); + cxgb4_unregister_uld(CXGB4_ULD_IPSEC); +} + +module_init(ch_ipsec_init); +module_exit(ch_ipsec_exit); + +MODULE_DESCRIPTION("Crypto IPSEC for Chelsio Terminator cards."); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Chelsio Communications"); +MODULE_VERSION(CHIPSEC_DRV_VERSION); + diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.h b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.h new file mode 100644 index 000000000..1d110d2ed --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2018 Chelsio Communications, Inc. */ + +#ifndef __CHCR_IPSEC_H__ +#define __CHCR_IPSEC_H__ + +#include <crypto/algapi.h> +#include "t4_hw.h" +#include "cxgb4.h" +#include "t4_msg.h" +#include "cxgb4_uld.h" + +#include "chcr_core.h" +#include "chcr_algo.h" +#include "chcr_crypto.h" + +#define CHIPSEC_DRV_MODULE_NAME "ch_ipsec" +#define CHIPSEC_DRV_VERSION "1.0.0.0-ko" +#define CHIPSEC_DRV_DESC "Chelsio T6 Crypto Ipsec offload Driver" + +struct ipsec_uld_ctx { + struct list_head entry; + struct cxgb4_lld_info lldi; +}; + +struct chcr_ipsec_req { + struct ulp_txpkt ulptx; + struct ulptx_idata sc_imm; + struct cpl_tx_sec_pdu sec_cpl; + struct _key_ctx key_ctx; +}; + +struct chcr_ipsec_wr { + struct fw_ulptx_wr wreq; + struct chcr_ipsec_req req; +}; + +#define ESN_IV_INSERT_OFFSET 12 +struct chcr_ipsec_aadiv { + __be32 spi; + u8 seq_no[8]; + u8 iv[8]; +}; + +struct ipsec_sa_entry { + int hmac_ctrl; + u16 esn; + u16 resv; + unsigned int enckey_len; + unsigned int kctx_len; + unsigned int authsize; + __be32 key_ctx_hdr; + char salt[MAX_SALT]; + char key[2 * AES_MAX_KEY_SIZE]; +}; + +#endif /* __CHCR_IPSEC_H__ */ + diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/Makefile b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/Makefile new file mode 100644 index 000000000..5e7d161c3 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 + +obj-$(CONFIG_CHELSIO_TLS_DEVICE) += ch_ktls.o +ch_ktls-objs := chcr_ktls.o diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_common.h b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_common.h new file mode 100644 index 000000000..38319f4c3 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_common.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ + +#ifndef __CHCR_COMMON_H__ +#define __CHCR_COMMON_H__ + +#include "cxgb4.h" + +#define CHCR_MAX_SALT 4 +#define CHCR_KEYCTX_MAC_KEY_SIZE_128 0 +#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128 0 +#define CHCR_SCMD_CIPHER_MODE_AES_GCM 2 +#define CHCR_SCMD_CIPHER_MODE_AES_CTR 3 +#define CHCR_CPL_TX_SEC_PDU_LEN_64BIT 2 +#define CHCR_SCMD_SEQ_NO_CTRL_64BIT 3 +#define CHCR_SCMD_PROTO_VERSION_TLS 0 +#define CHCR_SCMD_PROTO_VERSION_GENERIC 4 +#define CHCR_SCMD_AUTH_MODE_GHASH 4 +#define AES_BLOCK_LEN 16 + +struct ktls_key_ctx { + __be32 ctx_hdr; + u8 salt[CHCR_MAX_SALT]; + __be64 iv_to_auth; + unsigned char key[TLS_CIPHER_AES_GCM_128_KEY_SIZE + + TLS_CIPHER_AES_GCM_256_TAG_SIZE]; +}; + +/* Crypto key context */ +#define KEY_CONTEXT_CTX_LEN_S 24 +#define KEY_CONTEXT_CTX_LEN_V(x) ((x) << KEY_CONTEXT_CTX_LEN_S) + +#define KEY_CONTEXT_SALT_PRESENT_S 10 +#define KEY_CONTEXT_SALT_PRESENT_V(x) ((x) << KEY_CONTEXT_SALT_PRESENT_S) +#define KEY_CONTEXT_SALT_PRESENT_F KEY_CONTEXT_SALT_PRESENT_V(1U) + +#define KEY_CONTEXT_VALID_S 0 +#define KEY_CONTEXT_VALID_V(x) ((x) << KEY_CONTEXT_VALID_S) +#define KEY_CONTEXT_VALID_F KEY_CONTEXT_VALID_V(1U) + +#define KEY_CONTEXT_CK_SIZE_S 6 +#define KEY_CONTEXT_CK_SIZE_V(x) ((x) << KEY_CONTEXT_CK_SIZE_S) + +#define KEY_CONTEXT_MK_SIZE_S 2 +#define KEY_CONTEXT_MK_SIZE_V(x) ((x) << KEY_CONTEXT_MK_SIZE_S) + +#define KEY_CONTEXT_OPAD_PRESENT_S 11 +#define KEY_CONTEXT_OPAD_PRESENT_V(x) ((x) << KEY_CONTEXT_OPAD_PRESENT_S) +#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U) + +#define FILL_KEY_CTX_HDR(ck_size, mk_size, ctx_len) \ + htonl(KEY_CONTEXT_MK_SIZE_V(mk_size) | \ + KEY_CONTEXT_CK_SIZE_V(ck_size) | \ + KEY_CONTEXT_VALID_F | \ + KEY_CONTEXT_SALT_PRESENT_F | \ + KEY_CONTEXT_CTX_LEN_V((ctx_len))) + +static inline void *chcr_copy_to_txd(const void *src, const struct sge_txq *q, + void *pos, int length) +{ + int left = (void *)q->stat - pos; + u64 *p; + + if (likely(length <= left)) { + memcpy(pos, src, length); + pos += length; + } else { + memcpy(pos, src, left); + memcpy(q->desc, src + left, length - left); + pos = (void *)q->desc + (length - left); + } + /* 0-pad to multiple of 16 */ + p = PTR_ALIGN(pos, 8); + if ((uintptr_t)p & 8) { + *p = 0; + return p + 1; + } + return p; +} + +static inline unsigned int chcr_txq_avail(const struct sge_txq *q) +{ + return q->size - 1 - q->in_use; +} + +static inline void chcr_txq_advance(struct sge_txq *q, unsigned int n) +{ + q->in_use += n; + q->pidx += n; + if (q->pidx >= q->size) + q->pidx -= q->size; +} + +static inline void chcr_eth_txq_stop(struct sge_eth_txq *q) +{ + netif_tx_stop_queue(q->txq); + q->q.stops++; +} + +static inline unsigned int chcr_sgl_len(unsigned int n) +{ + n--; + return (3 * n) / 2 + (n & 1) + 2; +} + +static inline unsigned int chcr_flits_to_desc(unsigned int n) +{ + WARN_ON(n > SGE_MAX_WR_LEN / 8); + return DIV_ROUND_UP(n, 8); +} +#endif /* __CHCR_COMMON_H__ */ diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c new file mode 100644 index 000000000..da9973b71 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c @@ -0,0 +1,2277 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/skbuff.h> +#include <linux/module.h> +#include <linux/highmem.h> +#include <linux/ip.h> +#include <net/ipv6.h> +#include <linux/netdevice.h> +#include <crypto/aes.h> +#include "chcr_ktls.h" + +static LIST_HEAD(uld_ctx_list); +static DEFINE_MUTEX(dev_mutex); + +/* chcr_get_nfrags_to_send: get the remaining nfrags after start offset + * @skb: skb + * @start: start offset. + * @len: how much data to send after @start + */ +static int chcr_get_nfrags_to_send(struct sk_buff *skb, u32 start, u32 len) +{ + struct skb_shared_info *si = skb_shinfo(skb); + u32 frag_size, skb_linear_data_len = skb_headlen(skb); + u8 nfrags = 0, frag_idx = 0; + skb_frag_t *frag; + + /* if its a linear skb then return 1 */ + if (!skb_is_nonlinear(skb)) + return 1; + + if (unlikely(start < skb_linear_data_len)) { + frag_size = min(len, skb_linear_data_len - start); + } else { + start -= skb_linear_data_len; + + frag = &si->frags[frag_idx]; + frag_size = skb_frag_size(frag); + while (start >= frag_size) { + start -= frag_size; + frag_idx++; + frag = &si->frags[frag_idx]; + frag_size = skb_frag_size(frag); + } + frag_size = min(len, skb_frag_size(frag) - start); + } + len -= frag_size; + nfrags++; + + while (len) { + frag_size = min(len, skb_frag_size(&si->frags[frag_idx])); + len -= frag_size; + nfrags++; + frag_idx++; + } + return nfrags; +} + +static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info); +static void clear_conn_resources(struct chcr_ktls_info *tx_info); +/* + * chcr_ktls_save_keys: calculate and save crypto keys. + * @tx_info - driver specific tls info. + * @crypto_info - tls crypto information. + * @direction - TX/RX direction. + * return - SUCCESS/FAILURE. + */ +static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info, + struct tls_crypto_info *crypto_info, + enum tls_offload_ctx_dir direction) +{ + int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret; + unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE]; + struct tls12_crypto_info_aes_gcm_128 *info_128_gcm; + struct ktls_key_ctx *kctx = &tx_info->key_ctx; + struct crypto_aes_ctx aes_ctx; + unsigned char *key, *salt; + + switch (crypto_info->cipher_type) { + case TLS_CIPHER_AES_GCM_128: + info_128_gcm = + (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE; + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE; + mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; + tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; + tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv); + + ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; + key = info_128_gcm->key; + salt = info_128_gcm->salt; + tx_info->record_no = *(u64 *)info_128_gcm->rec_seq; + + /* The SCMD fields used when encrypting a full TLS + * record. Its a one time calculation till the + * connection exists. + */ + tx_info->scmd0_seqno_numivs = + SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) | + SCMD_CIPH_AUTH_SEQ_CTRL_F | + SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) | + SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) | + SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) | + SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) | + SCMD_NUM_IVS_V(1); + + /* keys will be sent inline. */ + tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F; + + /* The SCMD fields used when encrypting a partial TLS + * record (no trailer and possibly a truncated payload). + */ + tx_info->scmd0_short_seqno_numivs = + SCMD_CIPH_AUTH_SEQ_CTRL_F | + SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) | + SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) | + SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1); + + tx_info->scmd0_short_ivgen_hdrlen = + tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F; + + break; + + default: + pr_err("GCM: cipher type 0x%x not supported\n", + crypto_info->cipher_type); + ret = -EINVAL; + goto out; + } + + key_ctx_size = CHCR_KTLS_KEY_CTX_LEN + + roundup(keylen, 16) + ghash_size; + /* Calculate the H = CIPH(K, 0 repeated 16 times). + * It will go in key context + */ + + ret = aes_expandkey(&aes_ctx, key, keylen); + if (ret) + goto out; + + memset(ghash_h, 0, ghash_size); + aes_encrypt(&aes_ctx, ghash_h, ghash_h); + memzero_explicit(&aes_ctx, sizeof(aes_ctx)); + + /* fill the Key context */ + if (direction == TLS_OFFLOAD_CTX_DIR_TX) { + kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size, + mac_key_size, + key_ctx_size >> 4); + } else { + ret = -EINVAL; + goto out; + } + + memcpy(kctx->salt, salt, tx_info->salt_size); + memcpy(kctx->key, key, keylen); + memcpy(kctx->key + keylen, ghash_h, ghash_size); + tx_info->key_ctx_len = key_ctx_size; + +out: + return ret; +} + +/* + * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection. + * @sk - tcp socket. + * @tx_info - driver specific tls info. + * @atid - connection active tid. + * return - send success/failure. + */ +static int chcr_ktls_act_open_req(struct sock *sk, + struct chcr_ktls_info *tx_info, + int atid) +{ + struct inet_sock *inet = inet_sk(sk); + struct cpl_t6_act_open_req *cpl6; + struct cpl_act_open_req *cpl; + struct sk_buff *skb; + unsigned int len; + int qid_atid; + u64 options; + + len = sizeof(*cpl6); + skb = alloc_skb(len, GFP_KERNEL); + if (unlikely(!skb)) + return -ENOMEM; + /* mark it a control pkt */ + set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); + + cpl6 = __skb_put_zero(skb, len); + cpl = (struct cpl_act_open_req *)cpl6; + INIT_TP_WR(cpl6, 0); + qid_atid = TID_QID_V(tx_info->rx_qid) | + TID_TID_V(atid); + OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid)); + cpl->local_port = inet->inet_sport; + cpl->peer_port = inet->inet_dport; + cpl->local_ip = inet->inet_rcv_saddr; + cpl->peer_ip = inet->inet_daddr; + + /* fill first 64 bit option field. */ + options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | + SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); + cpl->opt0 = cpu_to_be64(options); + + /* next 64 bit option field. */ + options = + TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); + cpl->opt2 = htonl(options); + + return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); +} + +#if IS_ENABLED(CONFIG_IPV6) +/* + * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection. + * @sk - tcp socket. + * @tx_info - driver specific tls info. + * @atid - connection active tid. + * return - send success/failure. + */ +static int chcr_ktls_act_open_req6(struct sock *sk, + struct chcr_ktls_info *tx_info, + int atid) +{ + struct inet_sock *inet = inet_sk(sk); + struct cpl_t6_act_open_req6 *cpl6; + struct cpl_act_open_req6 *cpl; + struct sk_buff *skb; + unsigned int len; + int qid_atid; + u64 options; + + len = sizeof(*cpl6); + skb = alloc_skb(len, GFP_KERNEL); + if (unlikely(!skb)) + return -ENOMEM; + /* mark it a control pkt */ + set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); + + cpl6 = __skb_put_zero(skb, len); + cpl = (struct cpl_act_open_req6 *)cpl6; + INIT_TP_WR(cpl6, 0); + qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid); + OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid)); + cpl->local_port = inet->inet_sport; + cpl->peer_port = inet->inet_dport; + cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0]; + cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8]; + cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0]; + cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8]; + + /* first 64 bit option field. */ + options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | + SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); + cpl->opt0 = cpu_to_be64(options); + /* next 64 bit option field. */ + options = + TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); + cpl->opt2 = htonl(options); + + return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); +} +#endif /* #if IS_ENABLED(CONFIG_IPV6) */ + +/* + * chcr_setup_connection: create a TCB entry so that TP will form tcp packets. + * @sk - tcp socket. + * @tx_info - driver specific tls info. + * return: NET_TX_OK/NET_XMIT_DROP + */ +static int chcr_setup_connection(struct sock *sk, + struct chcr_ktls_info *tx_info) +{ + struct tid_info *t = &tx_info->adap->tids; + int atid, ret = 0; + + atid = cxgb4_alloc_atid(t, tx_info); + if (atid == -1) + return -EINVAL; + + tx_info->atid = atid; + + if (tx_info->ip_family == AF_INET) { + ret = chcr_ktls_act_open_req(sk, tx_info, atid); +#if IS_ENABLED(CONFIG_IPV6) + } else { + ret = cxgb4_clip_get(tx_info->netdev, (const u32 *) + &sk->sk_v6_rcv_saddr, + 1); + if (ret) + return ret; + ret = chcr_ktls_act_open_req6(sk, tx_info, atid); +#endif + } + + /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret + * success, if any other return type clear atid and return that failure. + */ + if (ret) { + if (ret == NET_XMIT_CN) { + ret = 0; + } else { +#if IS_ENABLED(CONFIG_IPV6) + /* clear clip entry */ + if (tx_info->ip_family == AF_INET6) + cxgb4_clip_release(tx_info->netdev, + (const u32 *) + &sk->sk_v6_rcv_saddr, + 1); +#endif + cxgb4_free_atid(t, atid); + } + } + + return ret; +} + +/* + * chcr_set_tcb_field: update tcb fields. + * @tx_info - driver specific tls info. + * @word - TCB word. + * @mask - TCB word related mask. + * @val - TCB word related value. + * @no_reply - set 1 if not looking for TP response. + */ +static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word, + u64 mask, u64 val, int no_reply) +{ + struct cpl_set_tcb_field *req; + struct sk_buff *skb; + + skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC); + if (!skb) + return -ENOMEM; + + req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req)); + INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid); + req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | + NO_REPLY_V(no_reply)); + req->word_cookie = htons(TCB_WORD_V(word)); + req->mask = cpu_to_be64(mask); + req->val = cpu_to_be64(val); + + set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); + return cxgb4_ofld_send(tx_info->netdev, skb); +} + +/* + * chcr_ktls_dev_del: call back for tls_dev_del. + * Remove the tid and l2t entry and close the connection. + * it per connection basis. + * @netdev - net device. + * @tls_cts - tls context. + * @direction - TX/RX crypto direction + */ +static void chcr_ktls_dev_del(struct net_device *netdev, + struct tls_context *tls_ctx, + enum tls_offload_ctx_dir direction) +{ + struct chcr_ktls_ofld_ctx_tx *tx_ctx = + chcr_get_ktls_tx_context(tls_ctx); + struct chcr_ktls_info *tx_info = tx_ctx->chcr_info; + struct ch_ktls_port_stats_debug *port_stats; + struct chcr_ktls_uld_ctx *u_ctx; + + if (!tx_info) + return; + + u_ctx = tx_info->adap->uld[CXGB4_ULD_KTLS].handle; + if (u_ctx && u_ctx->detach) + return; + /* clear l2t entry */ + if (tx_info->l2te) + cxgb4_l2t_release(tx_info->l2te); + +#if IS_ENABLED(CONFIG_IPV6) + /* clear clip entry */ + if (tx_info->ip_family == AF_INET6) + cxgb4_clip_release(netdev, (const u32 *) + &tx_info->sk->sk_v6_rcv_saddr, + 1); +#endif + + /* clear tid */ + if (tx_info->tid != -1) { + cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, + tx_info->tid, tx_info->ip_family); + + xa_erase(&u_ctx->tid_list, tx_info->tid); + } + + port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id]; + atomic64_inc(&port_stats->ktls_tx_connection_close); + kvfree(tx_info); + tx_ctx->chcr_info = NULL; + /* release module refcount */ + module_put(THIS_MODULE); +} + +/* + * chcr_ktls_dev_add: call back for tls_dev_add. + * Create a tcb entry for TP. Also add l2t entry for the connection. And + * generate keys & save those keys locally. + * @netdev - net device. + * @tls_cts - tls context. + * @direction - TX/RX crypto direction + * return: SUCCESS/FAILURE. + */ +static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, + enum tls_offload_ctx_dir direction, + struct tls_crypto_info *crypto_info, + u32 start_offload_tcp_sn) +{ + struct tls_context *tls_ctx = tls_get_ctx(sk); + struct ch_ktls_port_stats_debug *port_stats; + struct chcr_ktls_ofld_ctx_tx *tx_ctx; + struct chcr_ktls_uld_ctx *u_ctx; + struct chcr_ktls_info *tx_info; + struct dst_entry *dst; + struct adapter *adap; + struct port_info *pi; + struct neighbour *n; + u8 daaddr[16]; + int ret = -1; + + tx_ctx = chcr_get_ktls_tx_context(tls_ctx); + + pi = netdev_priv(netdev); + adap = pi->adapter; + port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id]; + atomic64_inc(&port_stats->ktls_tx_connection_open); + u_ctx = adap->uld[CXGB4_ULD_KTLS].handle; + + if (direction == TLS_OFFLOAD_CTX_DIR_RX) { + pr_err("not expecting for RX direction\n"); + goto out; + } + + if (tx_ctx->chcr_info) + goto out; + + if (u_ctx && u_ctx->detach) + goto out; + + tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL); + if (!tx_info) + goto out; + + tx_info->sk = sk; + spin_lock_init(&tx_info->lock); + /* initialize tid and atid to -1, 0 is a also a valid id. */ + tx_info->tid = -1; + tx_info->atid = -1; + + tx_info->adap = adap; + tx_info->netdev = netdev; + tx_info->first_qset = pi->first_qset; + tx_info->tx_chan = pi->tx_chan; + tx_info->smt_idx = pi->smt_idx; + tx_info->port_id = pi->port_id; + tx_info->prev_ack = 0; + tx_info->prev_win = 0; + + tx_info->rx_qid = chcr_get_first_rx_qid(adap); + if (unlikely(tx_info->rx_qid < 0)) + goto free_tx_info; + + tx_info->prev_seq = start_offload_tcp_sn; + tx_info->tcp_start_seq_number = start_offload_tcp_sn; + + /* save crypto keys */ + ret = chcr_ktls_save_keys(tx_info, crypto_info, direction); + if (ret < 0) + goto free_tx_info; + + /* get peer ip */ + if (sk->sk_family == AF_INET) { + memcpy(daaddr, &sk->sk_daddr, 4); + tx_info->ip_family = AF_INET; +#if IS_ENABLED(CONFIG_IPV6) + } else { + if (!ipv6_only_sock(sk) && + ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { + memcpy(daaddr, &sk->sk_daddr, 4); + tx_info->ip_family = AF_INET; + } else { + memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16); + tx_info->ip_family = AF_INET6; + } +#endif + } + + /* get the l2t index */ + dst = sk_dst_get(sk); + if (!dst) { + pr_err("DST entry not found\n"); + goto free_tx_info; + } + n = dst_neigh_lookup(dst, daaddr); + if (!n || !n->dev) { + pr_err("neighbour not found\n"); + dst_release(dst); + goto free_tx_info; + } + tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0); + + neigh_release(n); + dst_release(dst); + + if (!tx_info->l2te) { + pr_err("l2t entry not found\n"); + goto free_tx_info; + } + + /* Driver shouldn't be removed until any single connection exists */ + if (!try_module_get(THIS_MODULE)) + goto free_l2t; + + init_completion(&tx_info->completion); + /* create a filter and call cxgb4_l2t_send to send the packet out, which + * will take care of updating l2t entry in hw if not already done. + */ + tx_info->open_state = CH_KTLS_OPEN_PENDING; + + if (chcr_setup_connection(sk, tx_info)) + goto put_module; + + /* Wait for reply */ + wait_for_completion_timeout(&tx_info->completion, 30 * HZ); + spin_lock_bh(&tx_info->lock); + if (tx_info->open_state) { + /* need to wait for hw response, can't free tx_info yet. */ + if (tx_info->open_state == CH_KTLS_OPEN_PENDING) + tx_info->pending_close = true; + else + spin_unlock_bh(&tx_info->lock); + /* if in pending close, free the lock after the cleanup */ + goto put_module; + } + spin_unlock_bh(&tx_info->lock); + + /* initialize tcb */ + reinit_completion(&tx_info->completion); + /* mark it pending for hw response */ + tx_info->open_state = CH_KTLS_OPEN_PENDING; + + if (chcr_init_tcb_fields(tx_info)) + goto free_tid; + + /* Wait for reply */ + wait_for_completion_timeout(&tx_info->completion, 30 * HZ); + spin_lock_bh(&tx_info->lock); + if (tx_info->open_state) { + /* need to wait for hw response, can't free tx_info yet. */ + tx_info->pending_close = true; + /* free the lock after cleanup */ + goto free_tid; + } + spin_unlock_bh(&tx_info->lock); + + if (!cxgb4_check_l2t_valid(tx_info->l2te)) + goto free_tid; + + atomic64_inc(&port_stats->ktls_tx_ctx); + tx_ctx->chcr_info = tx_info; + + return 0; + +free_tid: +#if IS_ENABLED(CONFIG_IPV6) + /* clear clip entry */ + if (tx_info->ip_family == AF_INET6) + cxgb4_clip_release(netdev, (const u32 *) + &sk->sk_v6_rcv_saddr, + 1); +#endif + cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, + tx_info->tid, tx_info->ip_family); + + xa_erase(&u_ctx->tid_list, tx_info->tid); + +put_module: + /* release module refcount */ + module_put(THIS_MODULE); +free_l2t: + cxgb4_l2t_release(tx_info->l2te); +free_tx_info: + if (tx_info->pending_close) + spin_unlock_bh(&tx_info->lock); + else + kvfree(tx_info); +out: + atomic64_inc(&port_stats->ktls_tx_connection_fail); + return -1; +} + +/* + * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number + * handling. + * @tx_info - driver specific tls info. + * return: NET_TX_OK/NET_XMIT_DROP + */ +static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info) +{ + int ret = 0; + + /* set tcb in offload and bypass */ + ret = + chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W, + TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F), + TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1); + if (ret) + return ret; + /* reset snd_una and snd_next fields in tcb */ + ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W, + TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) | + TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M), + 0, 1); + if (ret) + return ret; + + /* reset send max */ + ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W, + TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M), + 0, 1); + if (ret) + return ret; + + /* update l2t index and request for tp reply to confirm tcb is + * initialised to handle tx traffic. + */ + ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W, + TCB_L2T_IX_V(TCB_L2T_IX_M), + TCB_L2T_IX_V(tx_info->l2te->idx), 0); + return ret; +} + +/* + * chcr_ktls_cpl_act_open_rpl: connection reply received from TP. + */ +static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, + unsigned char *input) +{ + const struct cpl_act_open_rpl *p = (void *)input; + struct chcr_ktls_info *tx_info = NULL; + struct chcr_ktls_ofld_ctx_tx *tx_ctx; + struct chcr_ktls_uld_ctx *u_ctx; + unsigned int atid, tid, status; + struct tls_context *tls_ctx; + struct tid_info *t; + int ret = 0; + + tid = GET_TID(p); + status = AOPEN_STATUS_G(ntohl(p->atid_status)); + atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status))); + + t = &adap->tids; + tx_info = lookup_atid(t, atid); + + if (!tx_info || tx_info->atid != atid) { + pr_err("%s: incorrect tx_info or atid\n", __func__); + return -1; + } + + cxgb4_free_atid(t, atid); + tx_info->atid = -1; + + spin_lock(&tx_info->lock); + /* HW response is very close, finish pending cleanup */ + if (tx_info->pending_close) { + spin_unlock(&tx_info->lock); + if (!status) { + cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, + tid, tx_info->ip_family); + } + kvfree(tx_info); + return 0; + } + + if (!status) { + tx_info->tid = tid; + cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family); + /* Adding tid */ + tls_ctx = tls_get_ctx(tx_info->sk); + tx_ctx = chcr_get_ktls_tx_context(tls_ctx); + u_ctx = adap->uld[CXGB4_ULD_KTLS].handle; + if (u_ctx) { + ret = xa_insert_bh(&u_ctx->tid_list, tid, tx_ctx, + GFP_NOWAIT); + if (ret < 0) { + pr_err("%s: Failed to allocate tid XA entry = %d\n", + __func__, tx_info->tid); + tx_info->open_state = CH_KTLS_OPEN_FAILURE; + goto out; + } + } + tx_info->open_state = CH_KTLS_OPEN_SUCCESS; + } else { + tx_info->open_state = CH_KTLS_OPEN_FAILURE; + } +out: + spin_unlock(&tx_info->lock); + + complete(&tx_info->completion); + return ret; +} + +/* + * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP. + */ +static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input) +{ + const struct cpl_set_tcb_rpl *p = (void *)input; + struct chcr_ktls_info *tx_info = NULL; + struct tid_info *t; + u32 tid; + + tid = GET_TID(p); + + t = &adap->tids; + tx_info = lookup_tid(t, tid); + + if (!tx_info || tx_info->tid != tid) { + pr_err("%s: incorrect tx_info or tid\n", __func__); + return -1; + } + + spin_lock(&tx_info->lock); + if (tx_info->pending_close) { + spin_unlock(&tx_info->lock); + kvfree(tx_info); + return 0; + } + tx_info->open_state = CH_KTLS_OPEN_SUCCESS; + spin_unlock(&tx_info->lock); + + complete(&tx_info->completion); + return 0; +} + +static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, + u32 tid, void *pos, u16 word, + struct sge_eth_txq *q, u64 mask, + u64 val, u32 reply) +{ + struct cpl_set_tcb_field_core *cpl; + struct ulptx_idata *idata; + struct ulp_txpkt *txpkt; + + /* ULP_TXPKT */ + txpkt = pos; + txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | + ULP_TXPKT_CHANNELID_V(tx_info->port_id) | + ULP_TXPKT_FID_V(q->q.cntxt_id) | + ULP_TXPKT_RO_F); + txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16)); + + /* ULPTX_IDATA sub-command */ + idata = (struct ulptx_idata *)(txpkt + 1); + idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); + idata->len = htonl(sizeof(*cpl)); + pos = idata + 1; + + cpl = pos; + /* CPL_SET_TCB_FIELD */ + OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); + cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | + NO_REPLY_V(!reply)); + cpl->word_cookie = htons(TCB_WORD_V(word)); + cpl->mask = cpu_to_be64(mask); + cpl->val = cpu_to_be64(val); + + /* ULPTX_NOOP */ + idata = (struct ulptx_idata *)(cpl + 1); + idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); + idata->len = htonl(0); + pos = idata + 1; + + return pos; +} + + +/* + * chcr_write_cpl_set_tcb_ulp: update tcb values. + * TCB is responsible to create tcp headers, so all the related values + * should be correctly updated. + * @tx_info - driver specific tls info. + * @q - tx queue on which packet is going out. + * @tid - TCB identifier. + * @pos - current index where should we start writing. + * @word - TCB word. + * @mask - TCB word related mask. + * @val - TCB word related value. + * @reply - set 1 if looking for TP response. + * return - next position to write. + */ +static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, + struct sge_eth_txq *q, u32 tid, + void *pos, u16 word, u64 mask, + u64 val, u32 reply) +{ + int left = (void *)q->q.stat - pos; + + if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) { + if (!left) { + pos = q->q.desc; + } else { + u8 buf[48] = {0}; + + __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, q, + mask, val, reply); + + return chcr_copy_to_txd(buf, &q->q, pos, + CHCR_SET_TCB_FIELD_LEN); + } + } + + pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, q, + mask, val, reply); + + /* check again if we are at the end of the queue */ + if (left == CHCR_SET_TCB_FIELD_LEN) + pos = q->q.desc; + + return pos; +} + +/* + * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header + * with updated values like tcp seq, ack, window etc. + * @tx_info - driver specific tls info. + * @q - TX queue. + * @tcp_seq + * @tcp_ack + * @tcp_win + * return: NETDEV_TX_BUSY/NET_TX_OK. + */ +static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info, + struct sge_eth_txq *q, u64 tcp_seq, + u64 tcp_ack, u64 tcp_win, bool offset) +{ + bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0)); + struct ch_ktls_port_stats_debug *port_stats; + u32 len, cpl = 0, ndesc, wr_len, wr_mid = 0; + struct fw_ulptx_wr *wr; + int credits; + void *pos; + + wr_len = sizeof(*wr); + /* there can be max 4 cpls, check if we have enough credits */ + len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16); + ndesc = DIV_ROUND_UP(len, 64); + + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return NETDEV_TX_BUSY; + } + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + chcr_eth_txq_stop(q); + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + pos = &q->q.desc[q->q.pidx]; + /* make space for WR, we'll fill it later when we know all the cpls + * being sent out and have complete length. + */ + wr = pos; + pos += wr_len; + /* update tx_max if its a re-transmit or the first wr */ + if (first_wr || tcp_seq != tx_info->prev_seq) { + pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, + TCB_TX_MAX_W, + TCB_TX_MAX_V(TCB_TX_MAX_M), + TCB_TX_MAX_V(tcp_seq), 0); + cpl++; + } + /* reset snd una if it's a re-transmit pkt */ + if (tcp_seq != tx_info->prev_seq || offset) { + /* reset snd_una */ + port_stats = + &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id]; + pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, + TCB_SND_UNA_RAW_W, + TCB_SND_UNA_RAW_V + (TCB_SND_UNA_RAW_M), + TCB_SND_UNA_RAW_V(0), 0); + if (tcp_seq != tx_info->prev_seq) + atomic64_inc(&port_stats->ktls_tx_ooo); + cpl++; + } + /* update ack */ + if (first_wr || tx_info->prev_ack != tcp_ack) { + pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, + TCB_RCV_NXT_W, + TCB_RCV_NXT_V(TCB_RCV_NXT_M), + TCB_RCV_NXT_V(tcp_ack), 0); + tx_info->prev_ack = tcp_ack; + cpl++; + } + /* update receive window */ + if (first_wr || tx_info->prev_win != tcp_win) { + chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, + TCB_RCV_WND_W, + TCB_RCV_WND_V(TCB_RCV_WND_M), + TCB_RCV_WND_V(tcp_win), 0); + tx_info->prev_win = tcp_win; + cpl++; + } + + if (cpl) { + /* get the actual length */ + len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16); + /* ULPTX wr */ + wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr->cookie = 0; + /* fill len in wr field */ + wr->flowid_len16 = htonl(wr_mid | + FW_WR_LEN16_V(DIV_ROUND_UP(len, 16))); + + ndesc = DIV_ROUND_UP(len, 64); + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); + } + return 0; +} + +/* + * chcr_ktls_get_tx_flits + * returns number of flits to be sent out, it includes key context length, WR + * size and skb fragments. + */ +static unsigned int +chcr_ktls_get_tx_flits(u32 nr_frags, unsigned int key_ctx_len) +{ + return chcr_sgl_len(nr_frags) + + DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8); +} + +/* + * chcr_ktls_check_tcp_options: To check if there is any TCP option available + * other than timestamp. + * @skb - skb contains partial record.. + * return: 1 / 0 + */ +static int +chcr_ktls_check_tcp_options(struct tcphdr *tcp) +{ + int cnt, opt, optlen; + u_char *cp; + + cp = (u_char *)(tcp + 1); + cnt = (tcp->doff << 2) - sizeof(struct tcphdr); + for (; cnt > 0; cnt -= optlen, cp += optlen) { + opt = cp[0]; + if (opt == TCPOPT_EOL) + break; + if (opt == TCPOPT_NOP) { + optlen = 1; + } else { + if (cnt < 2) + break; + optlen = cp[1]; + if (optlen < 2 || optlen > cnt) + break; + } + switch (opt) { + case TCPOPT_NOP: + break; + default: + return 1; + } + } + return 0; +} + +/* + * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to + * send out separately. + * @tx_info - driver specific tls info. + * @skb - skb contains partial record.. + * @q - TX queue. + * @tx_chan - channel number. + * return: NETDEV_TX_OK/NETDEV_TX_BUSY. + */ +static int +chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb, + struct sge_eth_txq *q, uint32_t tx_chan) +{ + struct fw_eth_tx_pkt_wr *wr; + struct cpl_tx_pkt_core *cpl; + u32 ctrl, iplen, maclen; + struct ipv6hdr *ip6; + unsigned int ndesc; + struct tcphdr *tcp; + int len16, pktlen; + struct iphdr *ip; + u32 wr_mid = 0; + int credits; + u8 buf[150]; + u64 cntrl1; + void *pos; + + iplen = skb_network_header_len(skb); + maclen = skb_mac_header_len(skb); + + /* packet length = eth hdr len + ip hdr len + tcp hdr len + * (including options). + */ + pktlen = skb_tcp_all_headers(skb); + + ctrl = sizeof(*cpl) + pktlen; + len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16); + /* check how many descriptors needed */ + ndesc = DIV_ROUND_UP(len16, 4); + + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return NETDEV_TX_BUSY; + } + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + chcr_eth_txq_stop(q); + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + pos = &q->q.desc[q->q.pidx]; + wr = pos; + + /* Firmware work request header */ + wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | + FW_WR_IMMDLEN_V(ctrl)); + + wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); + wr->r3 = 0; + + cpl = (void *)(wr + 1); + + /* CPL header */ + cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) | + TXPKT_PF_V(tx_info->adap->pf)); + cpl->pack = 0; + cpl->len = htons(pktlen); + + memcpy(buf, skb->data, pktlen); + if (!IS_ENABLED(CONFIG_IPV6) || tx_info->ip_family == AF_INET) { + /* we need to correct ip header len */ + ip = (struct iphdr *)(buf + maclen); + ip->tot_len = htons(pktlen - maclen); + cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP); + } else { + ip6 = (struct ipv6hdr *)(buf + maclen); + ip6->payload_len = htons(pktlen - maclen - iplen); + cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP6); + } + + cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) | + TXPKT_IPHDR_LEN_V(iplen); + /* checksum offload */ + cpl->ctrl1 = cpu_to_be64(cntrl1); + + pos = cpl + 1; + + /* now take care of the tcp header, if fin is not set then clear push + * bit as well, and if fin is set, it will be sent at the last so we + * need to update the tcp sequence number as per the last packet. + */ + tcp = (struct tcphdr *)(buf + maclen + iplen); + + if (!tcp->fin) + tcp->psh = 0; + else + tcp->seq = htonl(tx_info->prev_seq); + + chcr_copy_to_txd(buf, &q->q, pos, pktlen); + + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); + return 0; +} + +/* + * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb + * received has partial end part of the record, send out the complete record, so + * that crypto block will be able to generate TAG/HASH. + * @skb - segment which has complete or partial end part. + * @tx_info - driver specific tls info. + * @q - TX queue. + * @tcp_seq + * @tcp_push - tcp push bit. + * @mss - segment size. + * return: NETDEV_TX_BUSY/NET_TX_OK. + */ +static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb, + struct chcr_ktls_info *tx_info, + struct sge_eth_txq *q, u32 tcp_seq, + bool is_last_wr, u32 data_len, + u32 skb_offset, u32 nfrags, + bool tcp_push, u32 mss) +{ + u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start; + struct adapter *adap = tx_info->adap; + int credits, left, last_desc; + struct tx_sw_desc *sgl_sdesc; + struct cpl_tx_data *tx_data; + struct cpl_tx_sec_pdu *cpl; + struct ulptx_idata *idata; + struct ulp_txpkt *ulptx; + struct fw_ulptx_wr *wr; + void *pos; + u64 *end; + + /* get the number of flits required */ + flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len); + /* number of descriptors */ + ndesc = chcr_flits_to_desc(flits); + /* check if enough credits available */ + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return NETDEV_TX_BUSY; + } + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + /* Credits are below the threshold values, stop the queue after + * injecting the Work Request for this packet. + */ + chcr_eth_txq_stop(q); + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + sgl_sdesc = &q->q.sdesc[last_desc]; + + if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { + memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); + q->mapping_err++; + return NETDEV_TX_BUSY; + } + + if (!is_last_wr) + skb_get(skb); + + pos = &q->q.desc[q->q.pidx]; + end = (u64 *)pos + flits; + /* FW_ULPTX_WR */ + wr = pos; + /* WR will need len16 */ + len16 = DIV_ROUND_UP(flits, 2); + wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); + wr->cookie = 0; + pos += sizeof(*wr); + /* ULP_TXPKT */ + ulptx = pos; + ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | + ULP_TXPKT_CHANNELID_V(tx_info->port_id) | + ULP_TXPKT_FID_V(q->q.cntxt_id) | + ULP_TXPKT_RO_F); + ulptx->len = htonl(len16 - 1); + /* ULPTX_IDATA sub-command */ + idata = (struct ulptx_idata *)(ulptx + 1); + idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); + /* idata length will include cpl_tx_sec_pdu + key context size + + * cpl_tx_data header. + */ + idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len + + sizeof(*tx_data)); + /* SEC CPL */ + cpl = (struct cpl_tx_sec_pdu *)(idata + 1); + cpl->op_ivinsrtofst = + htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | + CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) | + CPL_TX_SEC_PDU_PLACEHOLDER_V(1) | + CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1)); + cpl->pldlen = htonl(data_len); + + /* encryption should start after tls header size + iv size */ + cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1; + + cpl->aadstart_cipherstop_hi = + htonl(CPL_TX_SEC_PDU_AADSTART_V(1) | + CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) | + CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start)); + + /* authentication will also start after tls header + iv size */ + cpl->cipherstop_lo_authinsert = + htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) | + CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) | + CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE)); + + /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ + cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs); + cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen); + cpl->scmd1 = cpu_to_be64(tx_info->record_no); + + pos = cpl + 1; + /* check if space left to fill the keys */ + left = (void *)q->q.stat - pos; + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + + pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos, + tx_info->key_ctx_len); + left = (void *)q->q.stat - pos; + + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + /* CPL_TX_DATA */ + tx_data = (void *)pos; + OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); + tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(data_len)); + + tx_data->rsvd = htonl(tcp_seq); + + tx_data->flags = htonl(TX_BYPASS_F); + if (tcp_push) + tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); + + /* check left again, it might go beyond queue limit */ + pos = tx_data + 1; + left = (void *)q->q.stat - pos; + + /* check the position again */ + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + + /* send the complete packet except the header */ + cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr, + skb_offset, data_len); + sgl_sdesc->skb = skb; + + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(adap, &q->q, ndesc); + atomic64_inc(&adap->ch_ktls_stats.ktls_tx_send_records); + + return 0; +} + +/* + * chcr_ktls_xmit_wr_short: This is to send out partial records. If its + * a middle part of a record, fetch the prior data to make it 16 byte aligned + * and then only send it out. + * + * @skb - skb contains partial record.. + * @tx_info - driver specific tls info. + * @q - TX queue. + * @tcp_seq + * @tcp_push - tcp push bit. + * @mss - segment size. + * @tls_rec_offset - offset from start of the tls record. + * @perior_data - data before the current segment, required to make this record + * 16 byte aligned. + * @prior_data_len - prior_data length (less than 16) + * return: NETDEV_TX_BUSY/NET_TX_OK. + */ +static int chcr_ktls_xmit_wr_short(struct sk_buff *skb, + struct chcr_ktls_info *tx_info, + struct sge_eth_txq *q, + u32 tcp_seq, bool tcp_push, u32 mss, + u32 tls_rec_offset, u8 *prior_data, + u32 prior_data_len, u32 data_len, + u32 skb_offset) +{ + u32 len16, wr_mid = 0, cipher_start, nfrags; + struct adapter *adap = tx_info->adap; + unsigned int flits = 0, ndesc; + int credits, left, last_desc; + struct tx_sw_desc *sgl_sdesc; + struct cpl_tx_data *tx_data; + struct cpl_tx_sec_pdu *cpl; + struct ulptx_idata *idata; + struct ulp_txpkt *ulptx; + struct fw_ulptx_wr *wr; + __be64 iv_record; + void *pos; + u64 *end; + + nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len); + /* get the number of flits required, it's a partial record so 2 flits + * (AES_BLOCK_SIZE) will be added. + */ + flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len) + 2; + /* get the correct 8 byte IV of this record */ + iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no); + /* If it's a middle record and not 16 byte aligned to run AES CTR, need + * to make it 16 byte aligned. So atleadt 2 extra flits of immediate + * data will be added. + */ + if (prior_data_len) + flits += 2; + /* number of descriptors */ + ndesc = chcr_flits_to_desc(flits); + /* check if enough credits available */ + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return NETDEV_TX_BUSY; + } + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + chcr_eth_txq_stop(q); + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + sgl_sdesc = &q->q.sdesc[last_desc]; + + if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { + memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); + q->mapping_err++; + return NETDEV_TX_BUSY; + } + + pos = &q->q.desc[q->q.pidx]; + end = (u64 *)pos + flits; + /* FW_ULPTX_WR */ + wr = pos; + /* WR will need len16 */ + len16 = DIV_ROUND_UP(flits, 2); + wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); + wr->cookie = 0; + pos += sizeof(*wr); + /* ULP_TXPKT */ + ulptx = pos; + ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | + ULP_TXPKT_CHANNELID_V(tx_info->port_id) | + ULP_TXPKT_FID_V(q->q.cntxt_id) | + ULP_TXPKT_RO_F); + ulptx->len = htonl(len16 - 1); + /* ULPTX_IDATA sub-command */ + idata = (struct ulptx_idata *)(ulptx + 1); + idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); + /* idata length will include cpl_tx_sec_pdu + key context size + + * cpl_tx_data header. + */ + idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len + + sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len); + /* SEC CPL */ + cpl = (struct cpl_tx_sec_pdu *)(idata + 1); + /* cipher start will have tls header + iv size extra if its a header + * part of tls record. else only 16 byte IV will be added. + */ + cipher_start = + AES_BLOCK_LEN + 1 + + (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0); + + cpl->op_ivinsrtofst = + htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | + CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) | + CPL_TX_SEC_PDU_IVINSRTOFST_V(1)); + cpl->pldlen = htonl(data_len + AES_BLOCK_LEN + prior_data_len); + cpl->aadstart_cipherstop_hi = + htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start)); + cpl->cipherstop_lo_authinsert = 0; + /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ + cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs); + cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen); + cpl->scmd1 = 0; + + pos = cpl + 1; + /* check if space left to fill the keys */ + left = (void *)q->q.stat - pos; + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + + pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos, + tx_info->key_ctx_len); + left = (void *)q->q.stat - pos; + + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + /* CPL_TX_DATA */ + tx_data = (void *)pos; + OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); + tx_data->len = htonl(TX_DATA_MSS_V(mss) | + TX_LENGTH_V(data_len + prior_data_len)); + tx_data->rsvd = htonl(tcp_seq); + tx_data->flags = htonl(TX_BYPASS_F); + if (tcp_push) + tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); + + /* check left again, it might go beyond queue limit */ + pos = tx_data + 1; + left = (void *)q->q.stat - pos; + + /* check the position again */ + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8 + * bytes of actual IV and 4 bytes of 16 byte-sequence. + */ + memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size); + memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size); + *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) = + htonl(2 + (tls_rec_offset ? ((tls_rec_offset - + (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0)); + + pos += 16; + /* Prior_data_len will always be less than 16 bytes, fill the + * prio_data_len after AES_CTRL_BLOCK and clear the remaining length + * to 0. + */ + if (prior_data_len) + pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16); + /* send the complete packet except the header */ + cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr, + skb_offset, data_len); + sgl_sdesc->skb = skb; + + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(adap, &q->q, ndesc); + + return 0; +} + +/* + * chcr_ktls_tx_plaintxt: This handler will take care of the records which has + * only plain text (only tls header and iv) + * @tx_info - driver specific tls info. + * @skb - skb contains partial record.. + * @tcp_seq + * @mss - segment size. + * @tcp_push - tcp push bit. + * @q - TX queue. + * @port_id : port number + * @perior_data - data before the current segment, required to make this record + * 16 byte aligned. + * @prior_data_len - prior_data length (less than 16) + * return: NETDEV_TX_BUSY/NET_TX_OK. + */ +static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info, + struct sk_buff *skb, u32 tcp_seq, u32 mss, + bool tcp_push, struct sge_eth_txq *q, + u32 port_id, u8 *prior_data, + u32 data_len, u32 skb_offset, + u32 prior_data_len) +{ + int credits, left, len16, last_desc; + unsigned int flits = 0, ndesc; + struct tx_sw_desc *sgl_sdesc; + struct cpl_tx_data *tx_data; + struct ulptx_idata *idata; + struct ulp_txpkt *ulptx; + struct fw_ulptx_wr *wr; + u32 wr_mid = 0, nfrags; + void *pos; + u64 *end; + + flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8); + nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len); + flits += chcr_sgl_len(nfrags); + if (prior_data_len) + flits += 2; + + /* WR will need len16 */ + len16 = DIV_ROUND_UP(flits, 2); + /* check how many descriptors needed */ + ndesc = DIV_ROUND_UP(flits, 8); + + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return NETDEV_TX_BUSY; + } + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + chcr_eth_txq_stop(q); + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + sgl_sdesc = &q->q.sdesc[last_desc]; + + if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb, + sgl_sdesc->addr) < 0)) { + memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); + q->mapping_err++; + return NETDEV_TX_BUSY; + } + + pos = &q->q.desc[q->q.pidx]; + end = (u64 *)pos + flits; + /* FW_ULPTX_WR */ + wr = pos; + wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); + wr->cookie = 0; + /* ULP_TXPKT */ + ulptx = (struct ulp_txpkt *)(wr + 1); + ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | + ULP_TXPKT_DATAMODIFY_V(0) | + ULP_TXPKT_CHANNELID_V(tx_info->port_id) | + ULP_TXPKT_DEST_V(0) | + ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1)); + ulptx->len = htonl(len16 - 1); + /* ULPTX_IDATA sub-command */ + idata = (struct ulptx_idata *)(ulptx + 1); + idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); + idata->len = htonl(sizeof(*tx_data) + prior_data_len); + /* CPL_TX_DATA */ + tx_data = (struct cpl_tx_data *)(idata + 1); + OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); + tx_data->len = htonl(TX_DATA_MSS_V(mss) | + TX_LENGTH_V(data_len + prior_data_len)); + /* set tcp seq number */ + tx_data->rsvd = htonl(tcp_seq); + tx_data->flags = htonl(TX_BYPASS_F); + if (tcp_push) + tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); + + pos = tx_data + 1; + /* apart from prior_data_len, we should set remaining part of 16 bytes + * to be zero. + */ + if (prior_data_len) + pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16); + + /* check left again, it might go beyond queue limit */ + left = (void *)q->q.stat - pos; + + /* check the position again */ + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + /* send the complete packet including the header */ + cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr, + skb_offset, data_len); + sgl_sdesc->skb = skb; + + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); + return 0; +} + +static int chcr_ktls_tunnel_pkt(struct chcr_ktls_info *tx_info, + struct sk_buff *skb, + struct sge_eth_txq *q) +{ + u32 ctrl, iplen, maclen, wr_mid = 0, len16; + struct tx_sw_desc *sgl_sdesc; + struct fw_eth_tx_pkt_wr *wr; + struct cpl_tx_pkt_core *cpl; + unsigned int flits, ndesc; + int credits, last_desc; + u64 cntrl1, *end; + void *pos; + + ctrl = sizeof(*cpl); + flits = DIV_ROUND_UP(sizeof(*wr) + ctrl, 8); + + flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags + 1); + len16 = DIV_ROUND_UP(flits, 2); + /* check how many descriptors needed */ + ndesc = DIV_ROUND_UP(flits, 8); + + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return -ENOMEM; + } + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + chcr_eth_txq_stop(q); + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + sgl_sdesc = &q->q.sdesc[last_desc]; + + if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb, + sgl_sdesc->addr) < 0)) { + memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); + q->mapping_err++; + return -ENOMEM; + } + + iplen = skb_network_header_len(skb); + maclen = skb_mac_header_len(skb); + + pos = &q->q.desc[q->q.pidx]; + end = (u64 *)pos + flits; + wr = pos; + + /* Firmware work request header */ + wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | + FW_WR_IMMDLEN_V(ctrl)); + + wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); + wr->r3 = 0; + + cpl = (void *)(wr + 1); + + /* CPL header */ + cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | + TXPKT_INTF_V(tx_info->tx_chan) | + TXPKT_PF_V(tx_info->adap->pf)); + cpl->pack = 0; + cntrl1 = TXPKT_CSUM_TYPE_V(tx_info->ip_family == AF_INET ? + TX_CSUM_TCPIP : TX_CSUM_TCPIP6); + cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) | + TXPKT_IPHDR_LEN_V(iplen); + /* checksum offload */ + cpl->ctrl1 = cpu_to_be64(cntrl1); + cpl->len = htons(skb->len); + + pos = cpl + 1; + + cxgb4_write_sgl(skb, &q->q, pos, end, 0, sgl_sdesc->addr); + sgl_sdesc->skb = skb; + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); + return 0; +} + +/* + * chcr_ktls_copy_record_in_skb + * @nskb - new skb where the frags to be added. + * @skb - old skb, to copy socket and destructor details. + * @record - specific record which has complete 16k record in frags. + */ +static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb, + struct sk_buff *skb, + struct tls_record_info *record) +{ + int i = 0; + + for (i = 0; i < record->num_frags; i++) { + skb_shinfo(nskb)->frags[i] = record->frags[i]; + /* increase the frag ref count */ + __skb_frag_ref(&skb_shinfo(nskb)->frags[i]); + } + + skb_shinfo(nskb)->nr_frags = record->num_frags; + nskb->data_len = record->len; + nskb->len += record->len; + nskb->truesize += record->len; + nskb->sk = skb->sk; + nskb->destructor = skb->destructor; + refcount_add(nskb->truesize, &nskb->sk->sk_wmem_alloc); +} + +/* + * chcr_end_part_handler: This handler will handle the record which + * is complete or if record's end part is received. T6 adapter has a issue that + * it can't send out TAG with partial record so if its an end part then we have + * to send TAG as well and for which we need to fetch the complete record and + * send it to crypto module. + * @tx_info - driver specific tls info. + * @skb - skb contains partial record. + * @record - complete record of 16K size. + * @tcp_seq + * @mss - segment size in which TP needs to chop a packet. + * @tcp_push_no_fin - tcp push if fin is not set. + * @q - TX queue. + * @tls_end_offset - offset from end of the record. + * @last wr : check if this is the last part of the skb going out. + * return: NETDEV_TX_OK/NETDEV_TX_BUSY. + */ +static int chcr_end_part_handler(struct chcr_ktls_info *tx_info, + struct sk_buff *skb, + struct tls_record_info *record, + u32 tcp_seq, int mss, bool tcp_push_no_fin, + struct sge_eth_txq *q, u32 skb_offset, + u32 tls_end_offset, bool last_wr) +{ + bool free_skb_if_tx_fails = false; + struct sk_buff *nskb = NULL; + + /* check if it is a complete record */ + if (tls_end_offset == record->len) { + nskb = skb; + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_complete_pkts); + } else { + nskb = alloc_skb(0, GFP_ATOMIC); + if (!nskb) { + dev_kfree_skb_any(skb); + return NETDEV_TX_BUSY; + } + + /* copy complete record in skb */ + chcr_ktls_copy_record_in_skb(nskb, skb, record); + /* packet is being sent from the beginning, update the tcp_seq + * accordingly. + */ + tcp_seq = tls_record_start_seq(record); + /* reset skb offset */ + skb_offset = 0; + + if (last_wr) + dev_kfree_skb_any(skb); + else + free_skb_if_tx_fails = true; + + last_wr = true; + + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_end_pkts); + } + + if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq, + last_wr, record->len, skb_offset, + record->num_frags, + (last_wr && tcp_push_no_fin), + mss)) { + if (free_skb_if_tx_fails) + dev_kfree_skb_any(skb); + goto out; + } + tx_info->prev_seq = record->end_seq; + return 0; +out: + dev_kfree_skb_any(nskb); + return NETDEV_TX_BUSY; +} + +/* + * chcr_short_record_handler: This handler will take care of the records which + * doesn't have end part (1st part or the middle part(/s) of a record). In such + * cases, AES CTR will be used in place of AES GCM to send out partial packet. + * This partial record might be the first part of the record, or the middle + * part. In case of middle record we should fetch the prior data to make it 16 + * byte aligned. If it has a partial tls header or iv then get to the start of + * tls header. And if it has partial TAG, then remove the complete TAG and send + * only the payload. + * There is one more possibility that it gets a partial header, send that + * portion as a plaintext. + * @tx_info - driver specific tls info. + * @skb - skb contains partial record.. + * @record - complete record of 16K size. + * @tcp_seq + * @mss - segment size in which TP needs to chop a packet. + * @tcp_push_no_fin - tcp push if fin is not set. + * @q - TX queue. + * @tls_end_offset - offset from end of the record. + * return: NETDEV_TX_OK/NETDEV_TX_BUSY. + */ +static int chcr_short_record_handler(struct chcr_ktls_info *tx_info, + struct sk_buff *skb, + struct tls_record_info *record, + u32 tcp_seq, int mss, bool tcp_push_no_fin, + u32 data_len, u32 skb_offset, + struct sge_eth_txq *q, u32 tls_end_offset) +{ + u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record); + u8 prior_data[16] = {0}; + u32 prior_data_len = 0; + + /* check if the skb is ending in middle of tag/HASH, its a big + * trouble, send the packet before the HASH. + */ + int remaining_record = tls_end_offset - data_len; + + if (remaining_record > 0 && + remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) { + int trimmed_len = 0; + + if (tls_end_offset > TLS_CIPHER_AES_GCM_128_TAG_SIZE) + trimmed_len = data_len - + (TLS_CIPHER_AES_GCM_128_TAG_SIZE - + remaining_record); + if (!trimmed_len) + return FALLBACK; + + WARN_ON(trimmed_len > data_len); + + data_len = trimmed_len; + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_trimmed_pkts); + } + + /* check if it is only the header part. */ + if (tls_rec_offset + data_len <= (TLS_HEADER_SIZE + tx_info->iv_size)) { + if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss, + tcp_push_no_fin, q, + tx_info->port_id, prior_data, + data_len, skb_offset, prior_data_len)) + goto out; + + tx_info->prev_seq = tcp_seq + data_len; + return 0; + } + + /* check if the middle record's start point is 16 byte aligned. CTR + * needs 16 byte aligned start point to start encryption. + */ + if (tls_rec_offset) { + /* there is an offset from start, means its a middle record */ + int remaining = 0; + + if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) { + prior_data_len = tls_rec_offset; + tls_rec_offset = 0; + remaining = 0; + } else { + prior_data_len = + (tls_rec_offset - + (TLS_HEADER_SIZE + tx_info->iv_size)) + % AES_BLOCK_LEN; + remaining = tls_rec_offset - prior_data_len; + } + + /* if prior_data_len is not zero, means we need to fetch prior + * data to make this record 16 byte aligned, or we need to reach + * to start offset. + */ + if (prior_data_len) { + int i = 0; + u8 *data = NULL; + skb_frag_t *f; + u8 *vaddr; + int frag_size = 0, frag_delta = 0; + + while (remaining > 0) { + frag_size = skb_frag_size(&record->frags[i]); + if (remaining < frag_size) + break; + + remaining -= frag_size; + i++; + } + f = &record->frags[i]; + vaddr = kmap_atomic(skb_frag_page(f)); + + data = vaddr + skb_frag_off(f) + remaining; + frag_delta = skb_frag_size(f) - remaining; + + if (frag_delta >= prior_data_len) { + memcpy(prior_data, data, prior_data_len); + kunmap_atomic(vaddr); + } else { + memcpy(prior_data, data, frag_delta); + kunmap_atomic(vaddr); + /* get the next page */ + f = &record->frags[i + 1]; + vaddr = kmap_atomic(skb_frag_page(f)); + data = vaddr + skb_frag_off(f); + memcpy(prior_data + frag_delta, + data, (prior_data_len - frag_delta)); + kunmap_atomic(vaddr); + } + /* reset tcp_seq as per the prior_data_required len */ + tcp_seq -= prior_data_len; + } + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_middle_pkts); + } else { + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_start_pkts); + } + + if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin, + mss, tls_rec_offset, prior_data, + prior_data_len, data_len, skb_offset)) { + goto out; + } + + tx_info->prev_seq = tcp_seq + data_len + prior_data_len; + return 0; +out: + dev_kfree_skb_any(skb); + return NETDEV_TX_BUSY; +} + +static int chcr_ktls_sw_fallback(struct sk_buff *skb, + struct chcr_ktls_info *tx_info, + struct sge_eth_txq *q) +{ + u32 data_len, skb_offset; + struct sk_buff *nskb; + struct tcphdr *th; + + nskb = tls_encrypt_skb(skb); + + if (!nskb) + return 0; + + th = tcp_hdr(nskb); + skb_offset = skb_tcp_all_headers(nskb); + data_len = nskb->len - skb_offset; + skb_tx_timestamp(nskb); + + if (chcr_ktls_tunnel_pkt(tx_info, nskb, q)) + goto out; + + tx_info->prev_seq = ntohl(th->seq) + data_len; + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_fallback); + return 0; +out: + dev_kfree_skb_any(nskb); + return 0; +} +/* nic tls TX handler */ +static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev) +{ + u32 tls_end_offset, tcp_seq, skb_data_len, skb_offset; + struct ch_ktls_port_stats_debug *port_stats; + struct chcr_ktls_ofld_ctx_tx *tx_ctx; + struct ch_ktls_stats_debug *stats; + struct tcphdr *th = tcp_hdr(skb); + int data_len, qidx, ret = 0, mss; + struct tls_record_info *record; + struct chcr_ktls_info *tx_info; + struct net_device *tls_netdev; + struct tls_context *tls_ctx; + struct sge_eth_txq *q; + struct adapter *adap; + unsigned long flags; + + tcp_seq = ntohl(th->seq); + skb_offset = skb_tcp_all_headers(skb); + skb_data_len = skb->len - skb_offset; + data_len = skb_data_len; + + mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : data_len; + + tls_ctx = tls_get_ctx(skb->sk); + tls_netdev = rcu_dereference_bh(tls_ctx->netdev); + /* Don't quit on NULL: if tls_device_down is running in parallel, + * netdev might become NULL, even if tls_is_sk_tx_device_offloaded was + * true. Rather continue processing this packet. + */ + if (unlikely(tls_netdev && tls_netdev != dev)) + goto out; + + tx_ctx = chcr_get_ktls_tx_context(tls_ctx); + tx_info = tx_ctx->chcr_info; + + if (unlikely(!tx_info)) + goto out; + + adap = tx_info->adap; + stats = &adap->ch_ktls_stats; + port_stats = &stats->ktls_port[tx_info->port_id]; + + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + tx_info->first_qset]; + cxgb4_reclaim_completed_tx(adap, &q->q, true); + /* if tcp options are set but finish is not send the options first */ + if (!th->fin && chcr_ktls_check_tcp_options(th)) { + ret = chcr_ktls_write_tcp_options(tx_info, skb, q, + tx_info->tx_chan); + if (ret) + return NETDEV_TX_BUSY; + } + + /* TCP segments can be in received either complete or partial. + * chcr_end_part_handler will handle cases if complete record or end + * part of the record is received. In case of partial end part of record, + * we will send the complete record again. + */ + + spin_lock_irqsave(&tx_ctx->base.lock, flags); + + do { + + cxgb4_reclaim_completed_tx(adap, &q->q, true); + /* fetch the tls record */ + record = tls_get_record(&tx_ctx->base, tcp_seq, + &tx_info->record_no); + /* By the time packet reached to us, ACK is received, and record + * won't be found in that case, handle it gracefully. + */ + if (unlikely(!record)) { + spin_unlock_irqrestore(&tx_ctx->base.lock, flags); + atomic64_inc(&port_stats->ktls_tx_drop_no_sync_data); + goto out; + } + + tls_end_offset = record->end_seq - tcp_seq; + + pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n", + tcp_seq, record->end_seq, tx_info->prev_seq, data_len); + /* update tcb for the skb */ + if (skb_data_len == data_len) { + u32 tx_max = tcp_seq; + + if (!tls_record_is_start_marker(record) && + tls_end_offset < TLS_CIPHER_AES_GCM_128_TAG_SIZE) + tx_max = record->end_seq - + TLS_CIPHER_AES_GCM_128_TAG_SIZE; + + ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, tx_max, + ntohl(th->ack_seq), + ntohs(th->window), + tls_end_offset != + record->len); + if (ret) { + spin_unlock_irqrestore(&tx_ctx->base.lock, + flags); + goto out; + } + + if (th->fin) + skb_get(skb); + } + + if (unlikely(tls_record_is_start_marker(record))) { + atomic64_inc(&port_stats->ktls_tx_skip_no_sync_data); + /* If tls_end_offset < data_len, means there is some + * data after start marker, which needs encryption, send + * plaintext first and take skb refcount. else send out + * complete pkt as plaintext. + */ + if (tls_end_offset < data_len) + skb_get(skb); + else + tls_end_offset = data_len; + + ret = chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss, + (!th->fin && th->psh), q, + tx_info->port_id, NULL, + tls_end_offset, skb_offset, + 0); + + if (ret) { + /* free the refcount taken earlier */ + if (tls_end_offset < data_len) + dev_kfree_skb_any(skb); + spin_unlock_irqrestore(&tx_ctx->base.lock, flags); + goto out; + } + + data_len -= tls_end_offset; + tcp_seq = record->end_seq; + skb_offset += tls_end_offset; + continue; + } + + /* if a tls record is finishing in this SKB */ + if (tls_end_offset <= data_len) { + ret = chcr_end_part_handler(tx_info, skb, record, + tcp_seq, mss, + (!th->fin && th->psh), q, + skb_offset, + tls_end_offset, + skb_offset + + tls_end_offset == skb->len); + + data_len -= tls_end_offset; + /* tcp_seq increment is required to handle next record. + */ + tcp_seq += tls_end_offset; + skb_offset += tls_end_offset; + } else { + ret = chcr_short_record_handler(tx_info, skb, + record, tcp_seq, mss, + (!th->fin && th->psh), + data_len, skb_offset, + q, tls_end_offset); + data_len = 0; + } + + /* if any failure, come out from the loop. */ + if (ret) { + spin_unlock_irqrestore(&tx_ctx->base.lock, flags); + if (th->fin) + dev_kfree_skb_any(skb); + + if (ret == FALLBACK) + return chcr_ktls_sw_fallback(skb, tx_info, q); + + return NETDEV_TX_OK; + } + + /* length should never be less than 0 */ + WARN_ON(data_len < 0); + + } while (data_len > 0); + + spin_unlock_irqrestore(&tx_ctx->base.lock, flags); + atomic64_inc(&port_stats->ktls_tx_encrypted_packets); + atomic64_add(skb_data_len, &port_stats->ktls_tx_encrypted_bytes); + + /* tcp finish is set, send a separate tcp msg including all the options + * as well. + */ + if (th->fin) { + chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan); + dev_kfree_skb_any(skb); + } + + return NETDEV_TX_OK; +out: + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +} + +static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi) +{ + struct chcr_ktls_uld_ctx *u_ctx; + + pr_info_once("%s - version %s\n", CHCR_KTLS_DRV_DESC, + CHCR_KTLS_DRV_VERSION); + u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL); + if (!u_ctx) { + u_ctx = ERR_PTR(-ENOMEM); + goto out; + } + u_ctx->lldi = *lldi; + u_ctx->detach = false; + xa_init_flags(&u_ctx->tid_list, XA_FLAGS_LOCK_BH); +out: + return u_ctx; +} + +static const struct tlsdev_ops chcr_ktls_ops = { + .tls_dev_add = chcr_ktls_dev_add, + .tls_dev_del = chcr_ktls_dev_del, +}; + +static chcr_handler_func work_handlers[NUM_CPL_CMDS] = { + [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl, + [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl, +}; + +static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp, + const struct pkt_gl *pgl) +{ + const struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)rsp; + struct chcr_ktls_uld_ctx *u_ctx = handle; + u8 opcode = rpl->ot.opcode; + struct adapter *adap; + + adap = pci_get_drvdata(u_ctx->lldi.pdev); + + if (!work_handlers[opcode]) { + pr_err("Unsupported opcode %d received\n", opcode); + return 0; + } + + work_handlers[opcode](adap, (unsigned char *)&rsp[1]); + return 0; +} + +static void clear_conn_resources(struct chcr_ktls_info *tx_info) +{ + /* clear l2t entry */ + if (tx_info->l2te) + cxgb4_l2t_release(tx_info->l2te); + +#if IS_ENABLED(CONFIG_IPV6) + /* clear clip entry */ + if (tx_info->ip_family == AF_INET6) + cxgb4_clip_release(tx_info->netdev, (const u32 *) + &tx_info->sk->sk_v6_rcv_saddr, + 1); +#endif + + /* clear tid */ + if (tx_info->tid != -1) + cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, + tx_info->tid, tx_info->ip_family); +} + +static void ch_ktls_reset_all_conn(struct chcr_ktls_uld_ctx *u_ctx) +{ + struct ch_ktls_port_stats_debug *port_stats; + struct chcr_ktls_ofld_ctx_tx *tx_ctx; + struct chcr_ktls_info *tx_info; + unsigned long index; + + xa_for_each(&u_ctx->tid_list, index, tx_ctx) { + tx_info = tx_ctx->chcr_info; + clear_conn_resources(tx_info); + port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id]; + atomic64_inc(&port_stats->ktls_tx_connection_close); + kvfree(tx_info); + tx_ctx->chcr_info = NULL; + /* release module refcount */ + module_put(THIS_MODULE); + } +} + +static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state) +{ + struct chcr_ktls_uld_ctx *u_ctx = handle; + + switch (new_state) { + case CXGB4_STATE_UP: + pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev)); + mutex_lock(&dev_mutex); + list_add_tail(&u_ctx->entry, &uld_ctx_list); + mutex_unlock(&dev_mutex); + break; + case CXGB4_STATE_START_RECOVERY: + case CXGB4_STATE_DOWN: + case CXGB4_STATE_DETACH: + pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev)); + mutex_lock(&dev_mutex); + u_ctx->detach = true; + list_del(&u_ctx->entry); + ch_ktls_reset_all_conn(u_ctx); + xa_destroy(&u_ctx->tid_list); + mutex_unlock(&dev_mutex); + break; + default: + break; + } + + return 0; +} + +static struct cxgb4_uld_info chcr_ktls_uld_info = { + .name = CHCR_KTLS_DRV_MODULE_NAME, + .nrxq = 1, + .rxq_size = 1024, + .add = chcr_ktls_uld_add, + .tx_handler = chcr_ktls_xmit, + .rx_handler = chcr_ktls_uld_rx_handler, + .state_change = chcr_ktls_uld_state_change, + .tlsdev_ops = &chcr_ktls_ops, +}; + +static int __init chcr_ktls_init(void) +{ + cxgb4_register_uld(CXGB4_ULD_KTLS, &chcr_ktls_uld_info); + return 0; +} + +static void __exit chcr_ktls_exit(void) +{ + struct chcr_ktls_uld_ctx *u_ctx, *tmp; + struct adapter *adap; + + pr_info("driver unloaded\n"); + + mutex_lock(&dev_mutex); + list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) { + adap = pci_get_drvdata(u_ctx->lldi.pdev); + memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats)); + list_del(&u_ctx->entry); + xa_destroy(&u_ctx->tid_list); + kfree(u_ctx); + } + mutex_unlock(&dev_mutex); + cxgb4_unregister_uld(CXGB4_ULD_KTLS); +} + +module_init(chcr_ktls_init); +module_exit(chcr_ktls_exit); + +MODULE_DESCRIPTION("Chelsio NIC TLS ULD driver"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Chelsio Communications"); +MODULE_VERSION(CHCR_KTLS_DRV_VERSION); diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h new file mode 100644 index 000000000..10572dc55 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ + +#ifndef __CHCR_KTLS_H__ +#define __CHCR_KTLS_H__ + +#include "cxgb4.h" +#include "t4_msg.h" +#include "t4_tcb.h" +#include "l2t.h" +#include "chcr_common.h" +#include "cxgb4_uld.h" +#include "clip_tbl.h" + +#define CHCR_KTLS_DRV_MODULE_NAME "ch_ktls" +#define CHCR_KTLS_DRV_VERSION "1.0.0.0-ko" +#define CHCR_KTLS_DRV_DESC "Chelsio NIC TLS ULD Driver" + +#define CHCR_TCB_STATE_CLOSED 0 +#define CHCR_KTLS_KEY_CTX_LEN 16 +#define CHCR_SET_TCB_FIELD_LEN sizeof(struct cpl_set_tcb_field) +#define CHCR_PLAIN_TX_DATA_LEN (sizeof(struct fw_ulptx_wr) +\ + sizeof(struct ulp_txpkt) +\ + sizeof(struct ulptx_idata) +\ + sizeof(struct cpl_tx_data)) + +#define CHCR_KTLS_WR_SIZE (CHCR_PLAIN_TX_DATA_LEN +\ + sizeof(struct cpl_tx_sec_pdu)) +#define FALLBACK 35 + +enum ch_ktls_open_state { + CH_KTLS_OPEN_SUCCESS = 0, + CH_KTLS_OPEN_PENDING = 1, + CH_KTLS_OPEN_FAILURE = 2, +}; + +struct chcr_ktls_info { + struct sock *sk; + spinlock_t lock; /* lock for pending_close */ + struct ktls_key_ctx key_ctx; + struct adapter *adap; + struct l2t_entry *l2te; + struct net_device *netdev; + struct completion completion; + u64 iv; + u64 record_no; + int tid; + int atid; + int rx_qid; + u32 iv_size; + u32 prev_seq; + u32 prev_ack; + u32 salt_size; + u32 key_ctx_len; + u32 scmd0_seqno_numivs; + u32 scmd0_ivgen_hdrlen; + u32 tcp_start_seq_number; + u32 scmd0_short_seqno_numivs; + u32 scmd0_short_ivgen_hdrlen; + u16 prev_win; + u8 tx_chan; + u8 smt_idx; + u8 port_id; + u8 ip_family; + u8 first_qset; + enum ch_ktls_open_state open_state; + bool pending_close; +}; + +struct chcr_ktls_ofld_ctx_tx { + struct tls_offload_context_tx base; + struct chcr_ktls_info *chcr_info; +}; + +struct chcr_ktls_uld_ctx { + struct list_head entry; + struct cxgb4_lld_info lldi; + struct xarray tid_list; + bool detach; +}; + +static inline struct chcr_ktls_ofld_ctx_tx * +chcr_get_ktls_tx_context(struct tls_context *tls_ctx) +{ + BUILD_BUG_ON(sizeof(struct chcr_ktls_ofld_ctx_tx) > + TLS_OFFLOAD_CONTEXT_SIZE_TX); + return container_of(tls_offload_ctx_tx(tls_ctx), + struct chcr_ktls_ofld_ctx_tx, + base); +} + +static inline int chcr_get_first_rx_qid(struct adapter *adap) +{ + /* u_ctx is saved in adap, fetch it */ + struct chcr_ktls_uld_ctx *u_ctx = adap->uld[CXGB4_ULD_KTLS].handle; + + if (!u_ctx) + return -1; + return u_ctx->lldi.rxq_ids[0]; +} + +typedef int (*chcr_handler_func)(struct adapter *adap, unsigned char *input); +#endif /* __CHCR_KTLS_H__ */ diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/Makefile b/drivers/net/ethernet/chelsio/inline_crypto/chtls/Makefile new file mode 100644 index 000000000..bc11495ac --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 \ + -I $(srctree)/drivers/crypto/chelsio + +obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o +chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h new file mode 100644 index 000000000..41714203a --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h @@ -0,0 +1,586 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018 Chelsio Communications, Inc. + */ + +#ifndef __CHTLS_H__ +#define __CHTLS_H__ + +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <crypto/hash.h> +#include <crypto/sha1.h> +#include <crypto/sha2.h> +#include <crypto/authenc.h> +#include <crypto/ctr.h> +#include <crypto/gf128mul.h> +#include <crypto/internal/aead.h> +#include <crypto/null.h> +#include <crypto/internal/skcipher.h> +#include <crypto/aead.h> +#include <crypto/scatterwalk.h> +#include <crypto/internal/hash.h> +#include <linux/tls.h> +#include <net/tls.h> +#include <net/tls_toe.h> + +#include "t4fw_api.h" +#include "t4_msg.h" +#include "cxgb4.h" +#include "cxgb4_uld.h" +#include "l2t.h" +#include "chcr_algo.h" +#include "chcr_core.h" +#include "chcr_crypto.h" + +#define CHTLS_DRV_VERSION "1.0.0.0-ko" + +#define TLS_KEYCTX_RXFLIT_CNT_S 24 +#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S) + +#define TLS_KEYCTX_RXPROT_VER_S 20 +#define TLS_KEYCTX_RXPROT_VER_M 0xf +#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S) + +#define TLS_KEYCTX_RXCIPH_MODE_S 16 +#define TLS_KEYCTX_RXCIPH_MODE_M 0xf +#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S) + +#define TLS_KEYCTX_RXAUTH_MODE_S 12 +#define TLS_KEYCTX_RXAUTH_MODE_M 0xf +#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S) + +#define TLS_KEYCTX_RXCIAU_CTRL_S 11 +#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S) + +#define TLS_KEYCTX_RX_SEQCTR_S 9 +#define TLS_KEYCTX_RX_SEQCTR_M 0x3 +#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S) + +#define TLS_KEYCTX_RX_VALID_S 8 +#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S) + +#define TLS_KEYCTX_RXCK_SIZE_S 3 +#define TLS_KEYCTX_RXCK_SIZE_M 0x7 +#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S) + +#define TLS_KEYCTX_RXMK_SIZE_S 0 +#define TLS_KEYCTX_RXMK_SIZE_M 0x7 +#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S) + +#define KEYCTX_TX_WR_IV_S 55 +#define KEYCTX_TX_WR_IV_M 0x1ffULL +#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S) +#define KEYCTX_TX_WR_IV_G(x) \ + (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M) + +#define KEYCTX_TX_WR_AAD_S 47 +#define KEYCTX_TX_WR_AAD_M 0xffULL +#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S) +#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \ + KEYCTX_TX_WR_AAD_M) + +#define KEYCTX_TX_WR_AADST_S 39 +#define KEYCTX_TX_WR_AADST_M 0xffULL +#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S) +#define KEYCTX_TX_WR_AADST_G(x) \ + (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M) + +#define KEYCTX_TX_WR_CIPHER_S 30 +#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL +#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S) +#define KEYCTX_TX_WR_CIPHER_G(x) \ + (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M) + +#define KEYCTX_TX_WR_CIPHERST_S 23 +#define KEYCTX_TX_WR_CIPHERST_M 0x7f +#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S) +#define KEYCTX_TX_WR_CIPHERST_G(x) \ + (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M) + +#define KEYCTX_TX_WR_AUTH_S 14 +#define KEYCTX_TX_WR_AUTH_M 0x1ff +#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S) +#define KEYCTX_TX_WR_AUTH_G(x) \ + (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M) + +#define KEYCTX_TX_WR_AUTHST_S 7 +#define KEYCTX_TX_WR_AUTHST_M 0x7f +#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S) +#define KEYCTX_TX_WR_AUTHST_G(x) \ + (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M) + +#define KEYCTX_TX_WR_AUTHIN_S 0 +#define KEYCTX_TX_WR_AUTHIN_M 0x7f +#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S) +#define KEYCTX_TX_WR_AUTHIN_G(x) \ + (((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTHIN_M) + +struct sge_opaque_hdr { + void *dev; + dma_addr_t addr[MAX_SKB_FRAGS + 1]; +}; + +#define MAX_IVS_PAGE 256 +#define TLS_KEY_CONTEXT_SZ 64 +#define CIPHER_BLOCK_SIZE 16 +#define GCM_TAG_SIZE 16 +#define KEY_ON_MEM_SZ 16 +#define AEAD_EXPLICIT_DATA_SIZE 8 +#define TLS_HEADER_LENGTH 5 +#define SCMD_CIPH_MODE_AES_GCM 2 +/* Any MFS size should work and come from openssl */ +#define TLS_MFS 16384 + +#define RSS_HDR sizeof(struct rss_header) +#define TLS_WR_CPL_LEN \ + (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo)) + +enum { + CHTLS_KEY_CONTEXT_DSGL, + CHTLS_KEY_CONTEXT_IMM, + CHTLS_KEY_CONTEXT_DDR, +}; + +enum { + CHTLS_LISTEN_START, + CHTLS_LISTEN_STOP, +}; + +/* Flags for return value of CPL message handlers */ +enum { + CPL_RET_BUF_DONE = 1, /* buffer processing done */ + CPL_RET_BAD_MSG = 2, /* bad CPL message */ + CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */ +}; + +#define LISTEN_INFO_HASH_SIZE 32 +#define RSPQ_HASH_BITS 5 +struct listen_info { + struct listen_info *next; /* Link to next entry */ + struct sock *sk; /* The listening socket */ + unsigned int stid; /* The server TID */ +}; + +enum { + T4_LISTEN_START_PENDING, + T4_LISTEN_STARTED +}; + +enum csk_flags { + CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */ + CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */ + CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */ + CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */ + CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */ + CSK_ABORT_RPL_PENDING, /* expecting an abort reply */ + CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */ + CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */ + CSK_TX_FAILOVER, /* Tx traffic failing over */ + CSK_UPDATE_RCV_WND, /* Need to update rcv window */ + CSK_RST_ABORTED, /* outgoing RST was aborted */ + CSK_TLS_HANDSHK, /* TLS Handshake */ + CSK_CONN_INLINE, /* Connection on HW */ +}; + +enum chtls_cdev_state { + CHTLS_CDEV_STATE_UP = 1 +}; + +struct listen_ctx { + struct sock *lsk; + struct chtls_dev *cdev; + struct sk_buff_head synq; + u32 state; +}; + +struct key_map { + unsigned long *addr; + unsigned int start; + unsigned int available; + unsigned int size; + spinlock_t lock; /* lock for key id request from map */ +} __packed; + +struct tls_scmd { + u32 seqno_numivs; + u32 ivgen_hdrlen; +}; + +struct chtls_dev { + struct tls_toe_device tlsdev; + struct list_head list; + struct cxgb4_lld_info *lldi; + struct pci_dev *pdev; + struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE]; + spinlock_t listen_lock; /* lock for listen list */ + struct net_device **ports; + struct tid_info *tids; + unsigned int pfvf; + const unsigned short *mtus; + + struct idr hwtid_idr; + struct idr stid_idr; + + spinlock_t idr_lock ____cacheline_aligned_in_smp; + + struct net_device *egr_dev[NCHAN * 2]; + struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS]; + struct sk_buff *askb; + + struct sk_buff_head deferq; + struct work_struct deferq_task; + + struct list_head list_node; + struct list_head rcu_node; + struct list_head na_node; + unsigned int send_page_order; + int max_host_sndbuf; + u32 round_robin_cnt; + struct key_map kmap; + unsigned int cdev_state; +}; + +struct chtls_listen { + struct chtls_dev *cdev; + struct sock *sk; +}; + +struct chtls_hws { + struct sk_buff_head sk_recv_queue; + u8 txqid; + u8 ofld; + u16 type; + u16 rstate; + u16 keyrpl; + u16 pldlen; + u16 rcvpld; + u16 compute; + u16 expansion; + u16 keylen; + u16 pdus; + u16 adjustlen; + u16 ivsize; + u16 txleft; + u32 mfs; + s32 txkey; + s32 rxkey; + u32 fcplenmax; + u32 copied_seq; + u64 tx_seq_no; + struct tls_scmd scmd; + union { + struct tls12_crypto_info_aes_gcm_128 aes_gcm_128; + struct tls12_crypto_info_aes_gcm_256 aes_gcm_256; + } crypto_info; +}; + +struct chtls_sock { + struct sock *sk; + struct chtls_dev *cdev; + struct l2t_entry *l2t_entry; /* pointer to the L2T entry */ + struct net_device *egress_dev; /* TX_CHAN for act open retry */ + + struct sk_buff_head txq; + struct sk_buff *wr_skb_head; + struct sk_buff *wr_skb_tail; + struct sk_buff *ctrl_skb_cache; + struct sk_buff *txdata_skb_cache; /* abort path messages */ + struct kref kref; + unsigned long flags; + u32 opt2; + u32 wr_credits; + u32 wr_unacked; + u32 wr_max_credits; + u32 wr_nondata; + u32 hwtid; /* TCP Control Block ID */ + u32 txq_idx; + u32 rss_qid; + u32 tid; + u32 idr; + u32 mss; + u32 ulp_mode; + u32 tx_chan; + u32 rx_chan; + u32 sndbuf; + u32 txplen_max; + u32 mtu_idx; /* MTU table index */ + u32 smac_idx; + u8 port_id; + u8 tos; + u16 resv2; + u32 delack_mode; + u32 delack_seq; + u32 snd_win; + u32 rcv_win; + + void *passive_reap_next; /* placeholder for passive */ + struct chtls_hws tlshws; + struct synq { + struct sk_buff *next; + struct sk_buff *prev; + } synq; + struct listen_ctx *listen_ctx; +}; + +struct tls_hdr { + u8 type; + u16 version; + u16 length; +} __packed; + +struct tlsrx_cmp_hdr { + u8 type; + u16 version; + u16 length; + + u64 tls_seq; + u16 reserved1; + u8 res_to_mac_error; +} __packed; + +/* res_to_mac_error fields */ +#define TLSRX_HDR_PKT_INT_ERROR_S 4 +#define TLSRX_HDR_PKT_INT_ERROR_M 0x1 +#define TLSRX_HDR_PKT_INT_ERROR_V(x) \ + ((x) << TLSRX_HDR_PKT_INT_ERROR_S) +#define TLSRX_HDR_PKT_INT_ERROR_G(x) \ + (((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M) +#define TLSRX_HDR_PKT_INT_ERROR_F TLSRX_HDR_PKT_INT_ERROR_V(1U) + +#define TLSRX_HDR_PKT_SPP_ERROR_S 3 +#define TLSRX_HDR_PKT_SPP_ERROR_M 0x1 +#define TLSRX_HDR_PKT_SPP_ERROR_V(x) ((x) << TLSRX_HDR_PKT_SPP_ERROR) +#define TLSRX_HDR_PKT_SPP_ERROR_G(x) \ + (((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M) +#define TLSRX_HDR_PKT_SPP_ERROR_F TLSRX_HDR_PKT_SPP_ERROR_V(1U) + +#define TLSRX_HDR_PKT_CCDX_ERROR_S 2 +#define TLSRX_HDR_PKT_CCDX_ERROR_M 0x1 +#define TLSRX_HDR_PKT_CCDX_ERROR_V(x) ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S) +#define TLSRX_HDR_PKT_CCDX_ERROR_G(x) \ + (((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M) +#define TLSRX_HDR_PKT_CCDX_ERROR_F TLSRX_HDR_PKT_CCDX_ERROR_V(1U) + +#define TLSRX_HDR_PKT_PAD_ERROR_S 1 +#define TLSRX_HDR_PKT_PAD_ERROR_M 0x1 +#define TLSRX_HDR_PKT_PAD_ERROR_V(x) ((x) << TLSRX_HDR_PKT_PAD_ERROR_S) +#define TLSRX_HDR_PKT_PAD_ERROR_G(x) \ + (((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M) +#define TLSRX_HDR_PKT_PAD_ERROR_F TLSRX_HDR_PKT_PAD_ERROR_V(1U) + +#define TLSRX_HDR_PKT_MAC_ERROR_S 0 +#define TLSRX_HDR_PKT_MAC_ERROR_M 0x1 +#define TLSRX_HDR_PKT_MAC_ERROR_V(x) ((x) << TLSRX_HDR_PKT_MAC_ERROR) +#define TLSRX_HDR_PKT_MAC_ERROR_G(x) \ + (((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M) +#define TLSRX_HDR_PKT_MAC_ERROR_F TLSRX_HDR_PKT_MAC_ERROR_V(1U) + +#define TLSRX_HDR_PKT_ERROR_M 0x1F +#define CONTENT_TYPE_ERROR 0x7F + +struct ulp_mem_rw { + __be32 cmd; + __be32 len16; /* command length */ + __be32 dlen; /* data length in 32-byte units */ + __be32 lock_addr; +}; + +struct tls_key_wr { + __be32 op_to_compl; + __be32 flowid_len16; + __be32 ftid; + u8 reneg_to_write_rx; + u8 protocol; + __be16 mfs; +}; + +struct tls_key_req { + struct tls_key_wr wr; + struct ulp_mem_rw req; + struct ulptx_idata sc_imm; +}; + +/* + * This lives in skb->cb and is used to chain WRs in a linked list. + */ +struct wr_skb_cb { + struct l2t_skb_cb l2t; /* reserve space for l2t CB */ + struct sk_buff *next_wr; /* next write request */ +}; + +/* Per-skb backlog handler. Run when a socket's backlog is processed. */ +struct blog_skb_cb { + void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb); + struct chtls_dev *cdev; +}; + +/* + * Similar to tcp_skb_cb but with ULP elements added to support TLS, + * etc. + */ +struct ulp_skb_cb { + struct wr_skb_cb wr; /* reserve space for write request */ + u16 flags; /* TCP-like flags */ + u8 psh; + u8 ulp_mode; /* ULP mode/submode of sk_buff */ + u32 seq; /* TCP sequence number */ + union { /* ULP-specific fields */ + struct { + u8 type; + u8 ofld; + u8 iv; + } tls; + } ulp; +}; + +#define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0])) +#define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb) + +/* + * Flags for ulp_skb_cb.flags. + */ +enum { + ULPCB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */ + ULPCB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */ + ULPCB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */ + ULPCB_FLAG_HOLD = 1 << 3, /* skb not ready for Tx yet */ + ULPCB_FLAG_COMPL = 1 << 4, /* request WR completion */ + ULPCB_FLAG_URG = 1 << 5, /* urgent data */ + ULPCB_FLAG_TLS_HDR = 1 << 6, /* payload with tls hdr */ + ULPCB_FLAG_NO_HDR = 1 << 7, /* not a ofld wr */ +}; + +/* The ULP mode/submode of an skbuff */ +#define skb_ulp_mode(skb) (ULP_SKB_CB(skb)->ulp_mode) +#define TCP_PAGE(sk) (sk->sk_frag.page) +#define TCP_OFF(sk) (sk->sk_frag.offset) + +static inline struct chtls_dev *to_chtls_dev(struct tls_toe_device *tlsdev) +{ + return container_of(tlsdev, struct chtls_dev, tlsdev); +} + +static inline void csk_set_flag(struct chtls_sock *csk, + enum csk_flags flag) +{ + __set_bit(flag, &csk->flags); +} + +static inline void csk_reset_flag(struct chtls_sock *csk, + enum csk_flags flag) +{ + __clear_bit(flag, &csk->flags); +} + +static inline bool csk_conn_inline(const struct chtls_sock *csk) +{ + return test_bit(CSK_CONN_INLINE, &csk->flags); +} + +static inline int csk_flag(const struct sock *sk, enum csk_flags flag) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + + if (!csk_conn_inline(csk)) + return 0; + return test_bit(flag, &csk->flags); +} + +static inline int csk_flag_nochk(const struct chtls_sock *csk, + enum csk_flags flag) +{ + return test_bit(flag, &csk->flags); +} + +static inline void *cplhdr(struct sk_buff *skb) +{ + return skb->data; +} + +static inline int is_neg_adv(unsigned int status) +{ + return status == CPL_ERR_RTX_NEG_ADVICE || + status == CPL_ERR_KEEPALV_NEG_ADVICE || + status == CPL_ERR_PERSIST_NEG_ADVICE; +} + +static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *), + struct sock *sk, + struct sk_buff *skb) +{ + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + + bh_lock_sock(sk); + if (unlikely(sock_owned_by_user(sk))) { + BLOG_SKB_CB(skb)->backlog_rcv = fn; + __sk_add_backlog(sk, skb); + } else { + fn(sk, skb); + } + bh_unlock_sock(sk); +} + +static inline void chtls_sock_free(struct kref *ref) +{ + struct chtls_sock *csk = container_of(ref, struct chtls_sock, + kref); + kfree(csk); +} + +static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk) +{ + kref_put(&csk->kref, chtls_sock_free); +} + +static inline void __chtls_sock_get(const char *fn, + struct chtls_sock *csk) +{ + kref_get(&csk->kref); +} + +static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp, + struct sk_buff *skb, int through_l2t) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + + if (through_l2t) { + /* send through L2T */ + cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); + } else { + /* send directly */ + cxgb4_ofld_send(csk->egress_dev, skb); + } +} + +typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *); +extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS]; +void chtls_install_cpl_ops(struct sock *sk); +int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi); +void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk); +int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk); +void chtls_close(struct sock *sk, long timeout); +int chtls_disconnect(struct sock *sk, int flags); +void chtls_shutdown(struct sock *sk, int how); +void chtls_destroy_sock(struct sock *sk); +int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); +int chtls_recvmsg(struct sock *sk, struct msghdr *msg, + size_t len, int flags, int *addr_len); +int chtls_sendpage(struct sock *sk, struct page *page, + int offset, size_t size, int flags); +int send_tx_flowc_wr(struct sock *sk, int compl, + u32 snd_nxt, u32 rcv_nxt); +void chtls_tcp_push(struct sock *sk, int flags); +int chtls_push_frames(struct chtls_sock *csk, int comp); +int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val); +void chtls_set_tcb_field_rpl_skb(struct sock *sk, u16 word, + u64 mask, u64 val, u8 cookie, + int through_l2t); +int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode, int cipher_type); +void chtls_set_quiesce_ctrl(struct sock *sk, int val); +void skb_entail(struct sock *sk, struct sk_buff *skb, int flags); +unsigned int keyid_to_addr(int start_addr, int keyid); +void free_tls_keyid(struct sock *sk); +#endif diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c new file mode 100644 index 000000000..6f6525983 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c @@ -0,0 +1,2334 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018 Chelsio Communications, Inc. + * + * Written by: Atul Gupta (atul.gupta@chelsio.com) + */ + +#include <linux/module.h> +#include <linux/list.h> +#include <linux/workqueue.h> +#include <linux/skbuff.h> +#include <linux/timer.h> +#include <linux/notifier.h> +#include <linux/inetdevice.h> +#include <linux/ip.h> +#include <linux/tcp.h> +#include <linux/sched/signal.h> +#include <linux/kallsyms.h> +#include <linux/kprobes.h> +#include <linux/if_vlan.h> +#include <linux/ipv6.h> +#include <net/ipv6.h> +#include <net/transp_v6.h> +#include <net/ip6_route.h> +#include <net/inet_common.h> +#include <net/tcp.h> +#include <net/dst.h> +#include <net/tls.h> +#include <net/addrconf.h> +#include <net/secure_seq.h> + +#include "chtls.h" +#include "chtls_cm.h" +#include "clip_tbl.h" +#include "t4_tcb.h" + +/* + * State transitions and actions for close. Note that if we are in SYN_SENT + * we remain in that state as we cannot control a connection while it's in + * SYN_SENT; such connections are allowed to establish and are then aborted. + */ +static unsigned char new_state[16] = { + /* current state: new state: action: */ + /* (Invalid) */ TCP_CLOSE, + /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, + /* TCP_SYN_SENT */ TCP_SYN_SENT, + /* TCP_SYN_RECV */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, + /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1, + /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2, + /* TCP_TIME_WAIT */ TCP_CLOSE, + /* TCP_CLOSE */ TCP_CLOSE, + /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN, + /* TCP_LAST_ACK */ TCP_LAST_ACK, + /* TCP_LISTEN */ TCP_CLOSE, + /* TCP_CLOSING */ TCP_CLOSING, +}; + +static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev) +{ + struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC); + + if (!csk) + return NULL; + + csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC); + if (!csk->txdata_skb_cache) { + kfree(csk); + return NULL; + } + + kref_init(&csk->kref); + csk->cdev = cdev; + skb_queue_head_init(&csk->txq); + csk->wr_skb_head = NULL; + csk->wr_skb_tail = NULL; + csk->mss = MAX_MSS; + csk->tlshws.ofld = 1; + csk->tlshws.txkey = -1; + csk->tlshws.rxkey = -1; + csk->tlshws.mfs = TLS_MFS; + skb_queue_head_init(&csk->tlshws.sk_recv_queue); + return csk; +} + +static void chtls_sock_release(struct kref *ref) +{ + struct chtls_sock *csk = + container_of(ref, struct chtls_sock, kref); + + kfree(csk); +} + +static struct net_device *chtls_find_netdev(struct chtls_dev *cdev, + struct sock *sk) +{ + struct adapter *adap = pci_get_drvdata(cdev->pdev); + struct net_device *ndev = cdev->ports[0]; +#if IS_ENABLED(CONFIG_IPV6) + struct net_device *temp; + int addr_type; +#endif + int i; + + switch (sk->sk_family) { + case PF_INET: + if (likely(!inet_sk(sk)->inet_rcv_saddr)) + return ndev; + ndev = __ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr, false); + break; +#if IS_ENABLED(CONFIG_IPV6) + case PF_INET6: + addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr); + if (likely(addr_type == IPV6_ADDR_ANY)) + return ndev; + + for_each_netdev_rcu(&init_net, temp) { + if (ipv6_chk_addr(&init_net, (struct in6_addr *) + &sk->sk_v6_rcv_saddr, temp, 1)) { + ndev = temp; + break; + } + } + break; +#endif + default: + return NULL; + } + + if (!ndev) + return NULL; + + if (is_vlan_dev(ndev)) + ndev = vlan_dev_real_dev(ndev); + + for_each_port(adap, i) + if (cdev->ports[i] == ndev) + return ndev; + return NULL; +} + +static void assign_rxopt(struct sock *sk, unsigned int opt) +{ + const struct chtls_dev *cdev; + struct chtls_sock *csk; + struct tcp_sock *tp; + + csk = rcu_dereference_sk_user_data(sk); + tp = tcp_sk(sk); + + cdev = csk->cdev; + tp->tcp_header_len = sizeof(struct tcphdr); + tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40; + tp->mss_cache = tp->rx_opt.mss_clamp; + tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt); + tp->rx_opt.snd_wscale = TCPOPT_SACK_G(opt); + tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt); + SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt); + if (!tp->rx_opt.wscale_ok) + tp->rx_opt.rcv_wscale = 0; + if (tp->rx_opt.tstamp_ok) { + tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED; + tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED; + } else if (csk->opt2 & TSTAMPS_EN_F) { + csk->opt2 &= ~TSTAMPS_EN_F; + csk->mtu_idx = TCPOPT_MSS_G(opt); + } +} + +static void chtls_purge_receive_queue(struct sock *sk) +{ + struct sk_buff *skb; + + while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { + skb_dst_set(skb, (void *)NULL); + kfree_skb(skb); + } +} + +static void chtls_purge_write_queue(struct sock *sk) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct sk_buff *skb; + + while ((skb = __skb_dequeue(&csk->txq))) { + sk->sk_wmem_queued -= skb->truesize; + __kfree_skb(skb); + } +} + +static void chtls_purge_recv_queue(struct sock *sk) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_hws *tlsk = &csk->tlshws; + struct sk_buff *skb; + + while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) { + skb_dst_set(skb, NULL); + kfree_skb(skb); + } +} + +static void abort_arp_failure(void *handle, struct sk_buff *skb) +{ + struct cpl_abort_req *req = cplhdr(skb); + struct chtls_dev *cdev; + + cdev = (struct chtls_dev *)handle; + req->cmd = CPL_ABORT_NO_RST; + cxgb4_ofld_send(cdev->lldi->ports[0], skb); +} + +static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len) +{ + if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) { + __skb_trim(skb, 0); + refcount_inc(&skb->users); + } else { + skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL); + } + return skb; +} + +static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb) +{ + struct cpl_abort_req *req; + struct chtls_sock *csk; + struct tcp_sock *tp; + + csk = rcu_dereference_sk_user_data(sk); + tp = tcp_sk(sk); + + if (!skb) + skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req)); + + req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req)); + INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid); + skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA); + req->rsvd0 = htonl(tp->snd_nxt); + req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT); + req->cmd = mode; + t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure); + send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST); +} + +static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + + if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) || + !csk->cdev)) { + if (sk->sk_state == TCP_SYN_RECV) + csk_set_flag(csk, CSK_RST_ABORTED); + goto out; + } + + if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) { + struct tcp_sock *tp = tcp_sk(sk); + + if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0) + WARN_ONCE(1, "send tx flowc error"); + csk_set_flag(csk, CSK_TX_DATA_SENT); + } + + csk_set_flag(csk, CSK_ABORT_RPL_PENDING); + chtls_purge_write_queue(sk); + + csk_set_flag(csk, CSK_ABORT_SHUTDOWN); + if (sk->sk_state != TCP_SYN_RECV) + chtls_send_abort(sk, mode, skb); + else + chtls_set_tcb_field_rpl_skb(sk, TCB_T_FLAGS_W, + TCB_T_FLAGS_V(TCB_T_FLAGS_M), 0, + TCB_FIELD_COOKIE_TFLAG, 1); + + return; +out: + kfree_skb(skb); +} + +static void release_tcp_port(struct sock *sk) +{ + if (inet_csk(sk)->icsk_bind_hash) + inet_put_port(sk); +} + +static void tcp_uncork(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + + if (tp->nonagle & TCP_NAGLE_CORK) { + tp->nonagle &= ~TCP_NAGLE_CORK; + chtls_tcp_push(sk, 0); + } +} + +static void chtls_close_conn(struct sock *sk) +{ + struct cpl_close_con_req *req; + struct chtls_sock *csk; + struct sk_buff *skb; + unsigned int tid; + unsigned int len; + + len = roundup(sizeof(struct cpl_close_con_req), 16); + csk = rcu_dereference_sk_user_data(sk); + tid = csk->tid; + + skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL); + req = (struct cpl_close_con_req *)__skb_put(skb, len); + memset(req, 0, len); + req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) | + FW_WR_IMMDLEN_V(sizeof(*req) - + sizeof(req->wr))); + req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) | + FW_WR_FLOWID_V(tid)); + + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); + + tcp_uncork(sk); + skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND); + if (sk->sk_state != TCP_SYN_SENT) + chtls_push_frames(csk, 1); +} + +/* + * Perform a state transition during close and return the actions indicated + * for the transition. Do not make this function inline, the main reason + * it exists at all is to avoid multiple inlining of tcp_set_state. + */ +static int make_close_transition(struct sock *sk) +{ + int next = (int)new_state[sk->sk_state]; + + tcp_set_state(sk, next & TCP_STATE_MASK); + return next & TCP_ACTION_FIN; +} + +void chtls_close(struct sock *sk, long timeout) +{ + int data_lost, prev_state; + struct chtls_sock *csk; + + csk = rcu_dereference_sk_user_data(sk); + + lock_sock(sk); + sk->sk_shutdown |= SHUTDOWN_MASK; + + data_lost = skb_queue_len(&sk->sk_receive_queue); + data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue); + chtls_purge_recv_queue(sk); + chtls_purge_receive_queue(sk); + + if (sk->sk_state == TCP_CLOSE) { + goto wait; + } else if (data_lost || sk->sk_state == TCP_SYN_SENT) { + chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL); + release_tcp_port(sk); + goto unlock; + } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { + sk->sk_prot->disconnect(sk, 0); + } else if (make_close_transition(sk)) { + chtls_close_conn(sk); + } +wait: + if (timeout) + sk_stream_wait_close(sk, timeout); + +unlock: + prev_state = sk->sk_state; + sock_hold(sk); + sock_orphan(sk); + + release_sock(sk); + + local_bh_disable(); + bh_lock_sock(sk); + + if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE) + goto out; + + if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 && + !csk_flag(sk, CSK_ABORT_SHUTDOWN)) { + struct sk_buff *skb; + + skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC); + if (skb) + chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb); + } + + if (sk->sk_state == TCP_CLOSE) + inet_csk_destroy_sock(sk); + +out: + bh_unlock_sock(sk); + local_bh_enable(); + sock_put(sk); +} + +/* + * Wait until a socket enters on of the given states. + */ +static int wait_for_states(struct sock *sk, unsigned int states) +{ + DECLARE_WAITQUEUE(wait, current); + struct socket_wq _sk_wq; + long current_timeo; + int err = 0; + + current_timeo = 200; + + /* + * We want this to work even when there's no associated struct socket. + * In that case we provide a temporary wait_queue_head_t. + */ + if (!sk->sk_wq) { + init_waitqueue_head(&_sk_wq.wait); + _sk_wq.fasync_list = NULL; + init_rcu_head_on_stack(&_sk_wq.rcu); + RCU_INIT_POINTER(sk->sk_wq, &_sk_wq); + } + + add_wait_queue(sk_sleep(sk), &wait); + while (!sk_in_state(sk, states)) { + if (!current_timeo) { + err = -EBUSY; + break; + } + if (signal_pending(current)) { + err = sock_intr_errno(current_timeo); + break; + } + set_current_state(TASK_UNINTERRUPTIBLE); + release_sock(sk); + if (!sk_in_state(sk, states)) + current_timeo = schedule_timeout(current_timeo); + __set_current_state(TASK_RUNNING); + lock_sock(sk); + } + remove_wait_queue(sk_sleep(sk), &wait); + + if (rcu_dereference(sk->sk_wq) == &_sk_wq) + sk->sk_wq = NULL; + return err; +} + +int chtls_disconnect(struct sock *sk, int flags) +{ + struct tcp_sock *tp; + int err; + + tp = tcp_sk(sk); + chtls_purge_recv_queue(sk); + chtls_purge_receive_queue(sk); + chtls_purge_write_queue(sk); + + if (sk->sk_state != TCP_CLOSE) { + sk->sk_err = ECONNRESET; + chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL); + err = wait_for_states(sk, TCPF_CLOSE); + if (err) + return err; + } + chtls_purge_recv_queue(sk); + chtls_purge_receive_queue(sk); + tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale); + return tcp_disconnect(sk, flags); +} + +#define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \ + TCPF_SYN_RECV | TCPF_CLOSE_WAIT) +void chtls_shutdown(struct sock *sk, int how) +{ + if ((how & SEND_SHUTDOWN) && + sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) && + make_close_transition(sk)) + chtls_close_conn(sk); +} + +void chtls_destroy_sock(struct sock *sk) +{ + struct chtls_sock *csk; + + csk = rcu_dereference_sk_user_data(sk); + chtls_purge_recv_queue(sk); + csk->ulp_mode = ULP_MODE_NONE; + chtls_purge_write_queue(sk); + free_tls_keyid(sk); + kref_put(&csk->kref, chtls_sock_release); + if (sk->sk_family == AF_INET) + sk->sk_prot = &tcp_prot; +#if IS_ENABLED(CONFIG_IPV6) + else + sk->sk_prot = &tcpv6_prot; +#endif + sk->sk_prot->destroy(sk); +} + +static void reset_listen_child(struct sock *child) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(child); + struct sk_buff *skb; + + skb = alloc_ctrl_skb(csk->txdata_skb_cache, + sizeof(struct cpl_abort_req)); + + chtls_send_reset(child, CPL_ABORT_SEND_RST, skb); + sock_orphan(child); + INC_ORPHAN_COUNT(child); + if (child->sk_state == TCP_CLOSE) + inet_csk_destroy_sock(child); +} + +static void chtls_disconnect_acceptq(struct sock *listen_sk) +{ + struct request_sock **pprev; + + pprev = ACCEPT_QUEUE(listen_sk); + while (*pprev) { + struct request_sock *req = *pprev; + + if (req->rsk_ops == &chtls_rsk_ops || + req->rsk_ops == &chtls_rsk_opsv6) { + struct sock *child = req->sk; + + *pprev = req->dl_next; + sk_acceptq_removed(listen_sk); + reqsk_put(req); + sock_hold(child); + local_bh_disable(); + bh_lock_sock(child); + release_tcp_port(child); + reset_listen_child(child); + bh_unlock_sock(child); + local_bh_enable(); + sock_put(child); + } else { + pprev = &req->dl_next; + } + } +} + +static int listen_hashfn(const struct sock *sk) +{ + return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1); +} + +static struct listen_info *listen_hash_add(struct chtls_dev *cdev, + struct sock *sk, + unsigned int stid) +{ + struct listen_info *p = kmalloc(sizeof(*p), GFP_KERNEL); + + if (p) { + int key = listen_hashfn(sk); + + p->sk = sk; + p->stid = stid; + spin_lock(&cdev->listen_lock); + p->next = cdev->listen_hash_tab[key]; + cdev->listen_hash_tab[key] = p; + spin_unlock(&cdev->listen_lock); + } + return p; +} + +static int listen_hash_find(struct chtls_dev *cdev, + struct sock *sk) +{ + struct listen_info *p; + int stid = -1; + int key; + + key = listen_hashfn(sk); + + spin_lock(&cdev->listen_lock); + for (p = cdev->listen_hash_tab[key]; p; p = p->next) + if (p->sk == sk) { + stid = p->stid; + break; + } + spin_unlock(&cdev->listen_lock); + return stid; +} + +static int listen_hash_del(struct chtls_dev *cdev, + struct sock *sk) +{ + struct listen_info *p, **prev; + int stid = -1; + int key; + + key = listen_hashfn(sk); + prev = &cdev->listen_hash_tab[key]; + + spin_lock(&cdev->listen_lock); + for (p = *prev; p; prev = &p->next, p = p->next) + if (p->sk == sk) { + stid = p->stid; + *prev = p->next; + kfree(p); + break; + } + spin_unlock(&cdev->listen_lock); + return stid; +} + +static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent) +{ + struct request_sock *req; + struct chtls_sock *csk; + + csk = rcu_dereference_sk_user_data(child); + req = csk->passive_reap_next; + + reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req); + __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq); + chtls_reqsk_free(req); + csk->passive_reap_next = NULL; +} + +static void chtls_reset_synq(struct listen_ctx *listen_ctx) +{ + struct sock *listen_sk = listen_ctx->lsk; + + while (!skb_queue_empty(&listen_ctx->synq)) { + struct chtls_sock *csk = + container_of((struct synq *)skb_peek + (&listen_ctx->synq), struct chtls_sock, synq); + struct sock *child = csk->sk; + + cleanup_syn_rcv_conn(child, listen_sk); + sock_hold(child); + local_bh_disable(); + bh_lock_sock(child); + release_tcp_port(child); + reset_listen_child(child); + bh_unlock_sock(child); + local_bh_enable(); + sock_put(child); + } +} + +int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk) +{ + struct net_device *ndev; +#if IS_ENABLED(CONFIG_IPV6) + bool clip_valid = false; +#endif + struct listen_ctx *ctx; + struct adapter *adap; + struct port_info *pi; + int ret = 0; + int stid; + + rcu_read_lock(); + ndev = chtls_find_netdev(cdev, sk); + rcu_read_unlock(); + if (!ndev) + return -EBADF; + + pi = netdev_priv(ndev); + adap = pi->adapter; + if (!(adap->flags & CXGB4_FULL_INIT_DONE)) + return -EBADF; + + if (listen_hash_find(cdev, sk) >= 0) /* already have it */ + return -EADDRINUSE; + + ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + __module_get(THIS_MODULE); + ctx->lsk = sk; + ctx->cdev = cdev; + ctx->state = T4_LISTEN_START_PENDING; + skb_queue_head_init(&ctx->synq); + + stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx); + if (stid < 0) + goto free_ctx; + + sock_hold(sk); + if (!listen_hash_add(cdev, sk, stid)) + goto free_stid; + + if (sk->sk_family == PF_INET) { + ret = cxgb4_create_server(ndev, stid, + inet_sk(sk)->inet_rcv_saddr, + inet_sk(sk)->inet_sport, 0, + cdev->lldi->rxq_ids[0]); +#if IS_ENABLED(CONFIG_IPV6) + } else { + int addr_type; + + addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr); + if (addr_type != IPV6_ADDR_ANY) { + ret = cxgb4_clip_get(ndev, (const u32 *) + &sk->sk_v6_rcv_saddr, 1); + if (ret) + goto del_hash; + clip_valid = true; + } + ret = cxgb4_create_server6(ndev, stid, + &sk->sk_v6_rcv_saddr, + inet_sk(sk)->inet_sport, + cdev->lldi->rxq_ids[0]); +#endif + } + if (ret > 0) + ret = net_xmit_errno(ret); + if (ret) + goto del_hash; + return 0; +del_hash: +#if IS_ENABLED(CONFIG_IPV6) + if (clip_valid) + cxgb4_clip_release(ndev, (const u32 *)&sk->sk_v6_rcv_saddr, 1); +#endif + listen_hash_del(cdev, sk); +free_stid: + cxgb4_free_stid(cdev->tids, stid, sk->sk_family); + sock_put(sk); +free_ctx: + kfree(ctx); + module_put(THIS_MODULE); + return -EBADF; +} + +void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk) +{ + struct listen_ctx *listen_ctx; + int stid; + + stid = listen_hash_del(cdev, sk); + if (stid < 0) + return; + + listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid); + chtls_reset_synq(listen_ctx); + + cxgb4_remove_server(cdev->lldi->ports[0], stid, + cdev->lldi->rxq_ids[0], sk->sk_family == PF_INET6); + +#if IS_ENABLED(CONFIG_IPV6) + if (sk->sk_family == PF_INET6) { + struct net_device *ndev = chtls_find_netdev(cdev, sk); + int addr_type = 0; + + addr_type = ipv6_addr_type((const struct in6_addr *) + &sk->sk_v6_rcv_saddr); + if (addr_type != IPV6_ADDR_ANY) + cxgb4_clip_release(ndev, (const u32 *) + &sk->sk_v6_rcv_saddr, 1); + } +#endif + chtls_disconnect_acceptq(sk); +} + +static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR; + unsigned int stid = GET_TID(rpl); + struct listen_ctx *listen_ctx; + + listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid); + if (!listen_ctx) + return CPL_RET_BUF_DONE; + + if (listen_ctx->state == T4_LISTEN_START_PENDING) { + listen_ctx->state = T4_LISTEN_STARTED; + return CPL_RET_BUF_DONE; + } + + if (rpl->status != CPL_ERR_NONE) { + pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n", + rpl->status, stid); + } else { + cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family); + sock_put(listen_ctx->lsk); + kfree(listen_ctx); + module_put(THIS_MODULE); + } + return CPL_RET_BUF_DONE; +} + +static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR; + struct listen_ctx *listen_ctx; + unsigned int stid; + void *data; + + stid = GET_TID(rpl); + data = lookup_stid(cdev->tids, stid); + listen_ctx = (struct listen_ctx *)data; + + if (rpl->status != CPL_ERR_NONE) { + pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n", + rpl->status, stid); + } else { + cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family); + sock_put(listen_ctx->lsk); + kfree(listen_ctx); + module_put(THIS_MODULE); + } + return CPL_RET_BUF_DONE; +} + +static void chtls_purge_wr_queue(struct sock *sk) +{ + struct sk_buff *skb; + + while ((skb = dequeue_wr(sk)) != NULL) + kfree_skb(skb); +} + +static void chtls_release_resources(struct sock *sk) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_dev *cdev = csk->cdev; + unsigned int tid = csk->tid; + struct tid_info *tids; + + if (!cdev) + return; + + tids = cdev->tids; + kfree_skb(csk->txdata_skb_cache); + csk->txdata_skb_cache = NULL; + + if (csk->wr_credits != csk->wr_max_credits) { + chtls_purge_wr_queue(sk); + chtls_reset_wr_list(csk); + } + + if (csk->l2t_entry) { + cxgb4_l2t_release(csk->l2t_entry); + csk->l2t_entry = NULL; + } + + if (sk->sk_state != TCP_SYN_SENT) { + cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family); + sock_put(sk); + } +} + +static void chtls_conn_done(struct sock *sk) +{ + if (sock_flag(sk, SOCK_DEAD)) + chtls_purge_receive_queue(sk); + sk_wakeup_sleepers(sk, 0); + tcp_done(sk); +} + +static void do_abort_syn_rcv(struct sock *child, struct sock *parent) +{ + /* + * If the server is still open we clean up the child connection, + * otherwise the server already did the clean up as it was purging + * its SYN queue and the skb was just sitting in its backlog. + */ + if (likely(parent->sk_state == TCP_LISTEN)) { + cleanup_syn_rcv_conn(child, parent); + /* Without the below call to sock_orphan, + * we leak the socket resource with syn_flood test + * as inet_csk_destroy_sock will not be called + * in tcp_done since SOCK_DEAD flag is not set. + * Kernel handles this differently where new socket is + * created only after 3 way handshake is done. + */ + sock_orphan(child); + INC_ORPHAN_COUNT(child); + chtls_release_resources(child); + chtls_conn_done(child); + } else { + if (csk_flag(child, CSK_RST_ABORTED)) { + chtls_release_resources(child); + chtls_conn_done(child); + } + } +} + +static void pass_open_abort(struct sock *child, struct sock *parent, + struct sk_buff *skb) +{ + do_abort_syn_rcv(child, parent); + kfree_skb(skb); +} + +static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb) +{ + pass_open_abort(skb->sk, lsk, skb); +} + +static void chtls_pass_open_arp_failure(struct sock *sk, + struct sk_buff *skb) +{ + const struct request_sock *oreq; + struct chtls_sock *csk; + struct chtls_dev *cdev; + struct sock *parent; + void *data; + + csk = rcu_dereference_sk_user_data(sk); + cdev = csk->cdev; + + /* + * If the connection is being aborted due to the parent listening + * socket going away there's nothing to do, the ABORT_REQ will close + * the connection. + */ + if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) { + kfree_skb(skb); + return; + } + + oreq = csk->passive_reap_next; + data = lookup_stid(cdev->tids, oreq->ts_recent); + parent = ((struct listen_ctx *)data)->lsk; + + bh_lock_sock(parent); + if (!sock_owned_by_user(parent)) { + pass_open_abort(sk, parent, skb); + } else { + BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort; + __sk_add_backlog(parent, skb); + } + bh_unlock_sock(parent); +} + +static void chtls_accept_rpl_arp_failure(void *handle, + struct sk_buff *skb) +{ + struct sock *sk = (struct sock *)handle; + + sock_hold(sk); + process_cpl_msg(chtls_pass_open_arp_failure, sk, skb); + sock_put(sk); +} + +static unsigned int chtls_select_mss(const struct chtls_sock *csk, + unsigned int pmtu, + struct cpl_pass_accept_req *req) +{ + struct chtls_dev *cdev; + struct dst_entry *dst; + unsigned int tcpoptsz; + unsigned int iphdrsz; + unsigned int mtu_idx; + struct tcp_sock *tp; + unsigned int mss; + struct sock *sk; + + mss = ntohs(req->tcpopt.mss); + sk = csk->sk; + dst = __sk_dst_get(sk); + cdev = csk->cdev; + tp = tcp_sk(sk); + tcpoptsz = 0; + +#if IS_ENABLED(CONFIG_IPV6) + if (sk->sk_family == AF_INET6) + iphdrsz = sizeof(struct ipv6hdr) + sizeof(struct tcphdr); + else +#endif + iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr); + if (req->tcpopt.tstamp) + tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4); + + tp->advmss = dst_metric_advmss(dst); + if (USER_MSS(tp) && tp->advmss > USER_MSS(tp)) + tp->advmss = USER_MSS(tp); + if (tp->advmss > pmtu - iphdrsz) + tp->advmss = pmtu - iphdrsz; + if (mss && tp->advmss > mss) + tp->advmss = mss; + + tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus, + iphdrsz + tcpoptsz, + tp->advmss - tcpoptsz, + 8, &mtu_idx); + tp->advmss -= iphdrsz; + + inet_csk(sk)->icsk_pmtu_cookie = pmtu; + return mtu_idx; +} + +static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp) +{ + int wscale = 0; + + if (space > MAX_RCV_WND) + space = MAX_RCV_WND; + if (win_clamp && win_clamp < space) + space = win_clamp; + + if (wscale_ok) { + while (wscale < 14 && (65535 << wscale) < space) + wscale++; + } + return wscale; +} + +static void chtls_pass_accept_rpl(struct sk_buff *skb, + struct cpl_pass_accept_req *req, + unsigned int tid) + +{ + struct cpl_t5_pass_accept_rpl *rpl5; + struct cxgb4_lld_info *lldi; + const struct tcphdr *tcph; + const struct tcp_sock *tp; + struct chtls_sock *csk; + unsigned int len; + struct sock *sk; + u32 opt2, hlen; + u64 opt0; + + sk = skb->sk; + tp = tcp_sk(sk); + csk = sk->sk_user_data; + csk->tid = tid; + lldi = csk->cdev->lldi; + len = roundup(sizeof(*rpl5), 16); + + rpl5 = __skb_put_zero(skb, len); + INIT_TP_WR(rpl5, tid); + + OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, + csk->tid)); + csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)), + req); + opt0 = TCAM_BYPASS_F | + WND_SCALE_V(RCV_WSCALE(tp)) | + MSS_IDX_V(csk->mtu_idx) | + L2T_IDX_V(csk->l2t_entry->idx) | + NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) | + TX_CHAN_V(csk->tx_chan) | + SMAC_SEL_V(csk->smac_idx) | + DSCP_V(csk->tos >> 2) | + ULP_MODE_V(ULP_MODE_TLS) | + RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M)); + + opt2 = RX_CHANNEL_V(0) | + RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid); + + if (!is_t5(lldi->adapter_type)) + opt2 |= RX_FC_DISABLE_F; + if (req->tcpopt.tstamp) + opt2 |= TSTAMPS_EN_F; + if (req->tcpopt.sack) + opt2 |= SACK_EN_F; + hlen = ntohl(req->hdr_len); + + tcph = (struct tcphdr *)((u8 *)(req + 1) + + T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen)); + if (tcph->ece && tcph->cwr) + opt2 |= CCTRL_ECN_V(1); + opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO); + opt2 |= T5_ISS_F; + opt2 |= T5_OPT_2_VALID_F; + opt2 |= WND_SCALE_EN_V(WSCALE_OK(tp)); + rpl5->opt0 = cpu_to_be64(opt0); + rpl5->opt2 = cpu_to_be32(opt2); + rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1); + set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); + t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure); + cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); +} + +static void inet_inherit_port(struct sock *lsk, struct sock *newsk) +{ + local_bh_disable(); + __inet_inherit_port(lsk, newsk); + local_bh_enable(); +} + +static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb) +{ + if (skb->protocol) { + kfree_skb(skb); + return 0; + } + BLOG_SKB_CB(skb)->backlog_rcv(sk, skb); + return 0; +} + +static void chtls_set_tcp_window(struct chtls_sock *csk) +{ + struct net_device *ndev = csk->egress_dev; + struct port_info *pi = netdev_priv(ndev); + unsigned int linkspeed; + u8 scale; + + linkspeed = pi->link_cfg.speed; + scale = linkspeed / SPEED_10000; +#define CHTLS_10G_RCVWIN (256 * 1024) + csk->rcv_win = CHTLS_10G_RCVWIN; + if (scale) + csk->rcv_win *= scale; +#define CHTLS_10G_SNDWIN (256 * 1024) + csk->snd_win = CHTLS_10G_SNDWIN; + if (scale) + csk->snd_win *= scale; +} + +static struct sock *chtls_recv_sock(struct sock *lsk, + struct request_sock *oreq, + void *network_hdr, + const struct cpl_pass_accept_req *req, + struct chtls_dev *cdev) +{ + struct adapter *adap = pci_get_drvdata(cdev->pdev); + struct neighbour *n = NULL; + struct inet_sock *newinet; + const struct iphdr *iph; + struct tls_context *ctx; + struct net_device *ndev; + struct chtls_sock *csk; + struct dst_entry *dst; + struct tcp_sock *tp; + struct sock *newsk; + bool found = false; + u16 port_id; + int rxq_idx; + int step, i; + + iph = (const struct iphdr *)network_hdr; + newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb); + if (!newsk) + goto free_oreq; + + if (lsk->sk_family == AF_INET) { + dst = inet_csk_route_child_sock(lsk, newsk, oreq); + if (!dst) + goto free_sk; + + n = dst_neigh_lookup(dst, &iph->saddr); +#if IS_ENABLED(CONFIG_IPV6) + } else { + const struct ipv6hdr *ip6h; + struct flowi6 fl6; + + ip6h = (const struct ipv6hdr *)network_hdr; + memset(&fl6, 0, sizeof(fl6)); + fl6.flowi6_proto = IPPROTO_TCP; + fl6.saddr = ip6h->daddr; + fl6.daddr = ip6h->saddr; + fl6.fl6_dport = inet_rsk(oreq)->ir_rmt_port; + fl6.fl6_sport = htons(inet_rsk(oreq)->ir_num); + security_req_classify_flow(oreq, flowi6_to_flowi_common(&fl6)); + dst = ip6_dst_lookup_flow(sock_net(lsk), lsk, &fl6, NULL); + if (IS_ERR(dst)) + goto free_sk; + n = dst_neigh_lookup(dst, &ip6h->saddr); +#endif + } + if (!n || !n->dev) + goto free_dst; + + ndev = n->dev; + if (is_vlan_dev(ndev)) + ndev = vlan_dev_real_dev(ndev); + + for_each_port(adap, i) + if (cdev->ports[i] == ndev) + found = true; + + if (!found) + goto free_dst; + + port_id = cxgb4_port_idx(ndev); + + csk = chtls_sock_create(cdev); + if (!csk) + goto free_dst; + + csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0); + if (!csk->l2t_entry) + goto free_csk; + + newsk->sk_user_data = csk; + newsk->sk_backlog_rcv = chtls_backlog_rcv; + + tp = tcp_sk(newsk); + newinet = inet_sk(newsk); + + if (iph->version == 0x4) { + newinet->inet_daddr = iph->saddr; + newinet->inet_rcv_saddr = iph->daddr; + newinet->inet_saddr = iph->daddr; +#if IS_ENABLED(CONFIG_IPV6) + } else { + struct tcp6_sock *newtcp6sk = (struct tcp6_sock *)newsk; + struct inet_request_sock *treq = inet_rsk(oreq); + struct ipv6_pinfo *newnp = inet6_sk(newsk); + struct ipv6_pinfo *np = inet6_sk(lsk); + + inet_sk(newsk)->pinet6 = &newtcp6sk->inet6; + memcpy(newnp, np, sizeof(struct ipv6_pinfo)); + newsk->sk_v6_daddr = treq->ir_v6_rmt_addr; + newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr; + inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr; + newnp->ipv6_fl_list = NULL; + newnp->pktoptions = NULL; + newsk->sk_bound_dev_if = treq->ir_iif; + newinet->inet_opt = NULL; + newinet->inet_daddr = LOOPBACK4_IPV6; + newinet->inet_saddr = LOOPBACK4_IPV6; +#endif + } + + oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid)); + sk_setup_caps(newsk, dst); + ctx = tls_get_ctx(lsk); + newsk->sk_destruct = ctx->sk_destruct; + newsk->sk_prot_creator = lsk->sk_prot_creator; + csk->sk = newsk; + csk->passive_reap_next = oreq; + csk->tx_chan = cxgb4_port_chan(ndev); + csk->port_id = port_id; + csk->egress_dev = ndev; + csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid)); + chtls_set_tcp_window(csk); + tp->rcv_wnd = csk->rcv_win; + csk->sndbuf = csk->snd_win; + csk->ulp_mode = ULP_MODE_TLS; + step = cdev->lldi->nrxq / cdev->lldi->nchan; + rxq_idx = port_id * step; + rxq_idx += cdev->round_robin_cnt++ % step; + csk->rss_qid = cdev->lldi->rxq_ids[rxq_idx]; + csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx : + port_id * step; + csk->sndbuf = newsk->sk_sndbuf; + csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx; + RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk), + READ_ONCE(sock_net(newsk)-> + ipv4.sysctl_tcp_window_scaling), + tp->window_clamp); + neigh_release(n); + inet_inherit_port(lsk, newsk); + csk_set_flag(csk, CSK_CONN_INLINE); + bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */ + + return newsk; +free_csk: + chtls_sock_release(&csk->kref); +free_dst: + if (n) + neigh_release(n); + dst_release(dst); +free_sk: + inet_csk_prepare_forced_close(newsk); + tcp_done(newsk); +free_oreq: + chtls_reqsk_free(oreq); + return NULL; +} + +/* + * Populate a TID_RELEASE WR. The skb must be already propely sized. + */ +static void mk_tid_release(struct sk_buff *skb, + unsigned int chan, unsigned int tid) +{ + struct cpl_tid_release *req; + unsigned int len; + + len = roundup(sizeof(struct cpl_tid_release), 16); + req = (struct cpl_tid_release *)__skb_put(skb, len); + memset(req, 0, len); + set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); + INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid); +} + +static int chtls_get_module(struct sock *sk) +{ + struct inet_connection_sock *icsk = inet_csk(sk); + + if (!try_module_get(icsk->icsk_ulp_ops->owner)) + return -1; + + return 0; +} + +static void chtls_pass_accept_request(struct sock *sk, + struct sk_buff *skb) +{ + struct cpl_t5_pass_accept_rpl *rpl; + struct cpl_pass_accept_req *req; + struct listen_ctx *listen_ctx; + struct vlan_ethhdr *vlan_eh; + struct request_sock *oreq; + struct sk_buff *reply_skb; + struct chtls_sock *csk; + struct chtls_dev *cdev; + struct ipv6hdr *ip6h; + struct tcphdr *tcph; + struct sock *newsk; + struct ethhdr *eh; + struct iphdr *iph; + void *network_hdr; + unsigned int stid; + unsigned int len; + unsigned int tid; + bool th_ecn, ect; + __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ + u16 eth_hdr_len; + bool ecn_ok; + + req = cplhdr(skb) + RSS_HDR; + tid = GET_TID(req); + cdev = BLOG_SKB_CB(skb)->cdev; + newsk = lookup_tid(cdev->tids, tid); + stid = PASS_OPEN_TID_G(ntohl(req->tos_stid)); + if (newsk) { + pr_info("tid (%d) already in use\n", tid); + return; + } + + len = roundup(sizeof(*rpl), 16); + reply_skb = alloc_skb(len, GFP_ATOMIC); + if (!reply_skb) { + cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family); + kfree_skb(skb); + return; + } + + if (sk->sk_state != TCP_LISTEN) + goto reject; + + if (inet_csk_reqsk_queue_is_full(sk)) + goto reject; + + if (sk_acceptq_is_full(sk)) + goto reject; + + + eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len)); + if (eth_hdr_len == ETH_HLEN) { + eh = (struct ethhdr *)(req + 1); + iph = (struct iphdr *)(eh + 1); + ip6h = (struct ipv6hdr *)(eh + 1); + network_hdr = (void *)(eh + 1); + } else { + vlan_eh = (struct vlan_ethhdr *)(req + 1); + iph = (struct iphdr *)(vlan_eh + 1); + ip6h = (struct ipv6hdr *)(vlan_eh + 1); + network_hdr = (void *)(vlan_eh + 1); + } + + if (iph->version == 0x4) { + tcph = (struct tcphdr *)(iph + 1); + skb_set_network_header(skb, (void *)iph - (void *)req); + oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true); + } else { + tcph = (struct tcphdr *)(ip6h + 1); + skb_set_network_header(skb, (void *)ip6h - (void *)req); + oreq = inet_reqsk_alloc(&chtls_rsk_opsv6, sk, false); + } + + if (!oreq) + goto reject; + + oreq->rsk_rcv_wnd = 0; + oreq->rsk_window_clamp = 0; + oreq->syncookie = 0; + oreq->mss = 0; + oreq->ts_recent = 0; + + tcp_rsk(oreq)->tfo_listener = false; + tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq); + chtls_set_req_port(oreq, tcph->source, tcph->dest); + if (iph->version == 0x4) { + chtls_set_req_addr(oreq, iph->daddr, iph->saddr); + ip_dsfield = ipv4_get_dsfield(iph); +#if IS_ENABLED(CONFIG_IPV6) + } else { + inet_rsk(oreq)->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr; + inet_rsk(oreq)->ir_v6_loc_addr = ipv6_hdr(skb)->daddr; + ip_dsfield = ipv6_get_dsfield(ipv6_hdr(skb)); +#endif + } + if (req->tcpopt.wsf <= 14 && + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling)) { + inet_rsk(oreq)->wscale_ok = 1; + inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf; + } + inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if; + th_ecn = tcph->ece && tcph->cwr; + if (th_ecn) { + ect = !INET_ECN_is_not_ect(ip_dsfield); + ecn_ok = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn); + if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk)) + inet_rsk(oreq)->ecn_ok = 1; + } + + newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev); + if (!newsk) + goto reject; + + if (chtls_get_module(newsk)) + goto reject; + inet_csk_reqsk_queue_added(sk); + reply_skb->sk = newsk; + chtls_install_cpl_ops(newsk); + cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family); + csk = rcu_dereference_sk_user_data(newsk); + listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid); + csk->listen_ctx = listen_ctx; + __skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq); + chtls_pass_accept_rpl(reply_skb, req, tid); + kfree_skb(skb); + return; + +reject: + mk_tid_release(reply_skb, 0, tid); + cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb); + kfree_skb(skb); +} + +/* + * Handle a CPL_PASS_ACCEPT_REQ message. + */ +static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR; + struct listen_ctx *ctx; + unsigned int stid; + unsigned int tid; + struct sock *lsk; + void *data; + + stid = PASS_OPEN_TID_G(ntohl(req->tos_stid)); + tid = GET_TID(req); + + data = lookup_stid(cdev->tids, stid); + if (!data) + return 1; + + ctx = (struct listen_ctx *)data; + lsk = ctx->lsk; + + if (unlikely(tid_out_of_range(cdev->tids, tid))) { + pr_info("passive open TID %u too large\n", tid); + return 1; + } + + BLOG_SKB_CB(skb)->cdev = cdev; + process_cpl_msg(chtls_pass_accept_request, lsk, skb); + return 0; +} + +/* + * Completes some final bits of initialization for just established connections + * and changes their state to TCP_ESTABLISHED. + * + * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1. + */ +static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt) +{ + struct tcp_sock *tp = tcp_sk(sk); + + tp->pushed_seq = snd_isn; + tp->write_seq = snd_isn; + tp->snd_nxt = snd_isn; + tp->snd_una = snd_isn; + atomic_set(&inet_sk(sk)->inet_id, get_random_u16()); + assign_rxopt(sk, opt); + + if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10)) + tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10); + + smp_mb(); + tcp_set_state(sk, TCP_ESTABLISHED); +} + +static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb) +{ + struct sk_buff *abort_skb; + + abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC); + if (abort_skb) + chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb); +} + +static struct sock *reap_list; +static DEFINE_SPINLOCK(reap_list_lock); + +/* + * Process the reap list. + */ +DECLARE_TASK_FUNC(process_reap_list, task_param) +{ + spin_lock_bh(&reap_list_lock); + while (reap_list) { + struct sock *sk = reap_list; + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + + reap_list = csk->passive_reap_next; + csk->passive_reap_next = NULL; + spin_unlock(&reap_list_lock); + sock_hold(sk); + + bh_lock_sock(sk); + chtls_abort_conn(sk, NULL); + sock_orphan(sk); + if (sk->sk_state == TCP_CLOSE) + inet_csk_destroy_sock(sk); + bh_unlock_sock(sk); + sock_put(sk); + spin_lock(&reap_list_lock); + } + spin_unlock_bh(&reap_list_lock); +} + +static DECLARE_WORK(reap_task, process_reap_list); + +static void add_to_reap_list(struct sock *sk) +{ + struct chtls_sock *csk = sk->sk_user_data; + + local_bh_disable(); + release_tcp_port(sk); /* release the port immediately */ + + spin_lock(&reap_list_lock); + csk->passive_reap_next = reap_list; + reap_list = sk; + if (!csk->passive_reap_next) + schedule_work(&reap_task); + spin_unlock(&reap_list_lock); + local_bh_enable(); +} + +static void add_pass_open_to_parent(struct sock *child, struct sock *lsk, + struct chtls_dev *cdev) +{ + struct request_sock *oreq; + struct chtls_sock *csk; + + if (lsk->sk_state != TCP_LISTEN) + return; + + csk = child->sk_user_data; + oreq = csk->passive_reap_next; + csk->passive_reap_next = NULL; + + reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq); + __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq); + + if (sk_acceptq_is_full(lsk)) { + chtls_reqsk_free(oreq); + add_to_reap_list(child); + } else { + refcount_set(&oreq->rsk_refcnt, 1); + inet_csk_reqsk_queue_add(lsk, oreq, child); + lsk->sk_data_ready(lsk); + } +} + +static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb) +{ + struct sock *child = skb->sk; + + skb->sk = NULL; + add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev); + kfree_skb(skb); +} + +static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR; + struct chtls_sock *csk; + struct sock *lsk, *sk; + unsigned int hwtid; + + hwtid = GET_TID(req); + sk = lookup_tid(cdev->tids, hwtid); + if (!sk) + return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE); + + bh_lock_sock(sk); + if (unlikely(sock_owned_by_user(sk))) { + kfree_skb(skb); + } else { + unsigned int stid; + void *data; + + csk = sk->sk_user_data; + csk->wr_max_credits = 64; + csk->wr_credits = 64; + csk->wr_unacked = 0; + make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); + stid = PASS_OPEN_TID_G(ntohl(req->tos_stid)); + sk->sk_state_change(sk); + if (unlikely(sk->sk_socket)) + sk_wake_async(sk, 0, POLL_OUT); + + data = lookup_stid(cdev->tids, stid); + if (!data) { + /* listening server close */ + kfree_skb(skb); + goto unlock; + } + lsk = ((struct listen_ctx *)data)->lsk; + + bh_lock_sock(lsk); + if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) { + /* removed from synq */ + bh_unlock_sock(lsk); + kfree_skb(skb); + goto unlock; + } + + if (likely(!sock_owned_by_user(lsk))) { + kfree_skb(skb); + add_pass_open_to_parent(sk, lsk, cdev); + } else { + skb->sk = sk; + BLOG_SKB_CB(skb)->cdev = cdev; + BLOG_SKB_CB(skb)->backlog_rcv = + bl_add_pass_open_to_parent; + __sk_add_backlog(lsk, skb); + } + bh_unlock_sock(lsk); + } +unlock: + bh_unlock_sock(sk); + return 0; +} + +/* + * Handle receipt of an urgent pointer. + */ +static void handle_urg_ptr(struct sock *sk, u32 urg_seq) +{ + struct tcp_sock *tp = tcp_sk(sk); + + urg_seq--; + if (tp->urg_data && !after(urg_seq, tp->urg_seq)) + return; /* duplicate pointer */ + + sk_send_sigurg(sk); + if (tp->urg_seq == tp->copied_seq && tp->urg_data && + !sock_flag(sk, SOCK_URGINLINE) && + tp->copied_seq != tp->rcv_nxt) { + struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); + + tp->copied_seq++; + if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len) + chtls_free_skb(sk, skb); + } + + tp->urg_data = TCP_URG_NOTYET; + tp->urg_seq = urg_seq; +} + +static void check_sk_callbacks(struct chtls_sock *csk) +{ + struct sock *sk = csk->sk; + + if (unlikely(sk->sk_user_data && + !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD))) + csk_set_flag(csk, CSK_CALLBACKS_CHKD); +} + +/* + * Handles Rx data that arrives in a state where the socket isn't accepting + * new data. + */ +static void handle_excess_rx(struct sock *sk, struct sk_buff *skb) +{ + if (!csk_flag(sk, CSK_ABORT_SHUTDOWN)) + chtls_abort_conn(sk, skb); + + kfree_skb(skb); +} + +static void chtls_recv_data(struct sock *sk, struct sk_buff *skb) +{ + struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR; + struct chtls_sock *csk; + struct tcp_sock *tp; + + csk = rcu_dereference_sk_user_data(sk); + tp = tcp_sk(sk); + + if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) { + handle_excess_rx(sk, skb); + return; + } + + ULP_SKB_CB(skb)->seq = ntohl(hdr->seq); + ULP_SKB_CB(skb)->psh = hdr->psh; + skb_ulp_mode(skb) = ULP_MODE_NONE; + + skb_reset_transport_header(skb); + __skb_pull(skb, sizeof(*hdr) + RSS_HDR); + if (!skb->data_len) + __skb_trim(skb, ntohs(hdr->len)); + + if (unlikely(hdr->urg)) + handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg)); + if (unlikely(tp->urg_data == TCP_URG_NOTYET && + tp->urg_seq - tp->rcv_nxt < skb->len)) + tp->urg_data = TCP_URG_VALID | + skb->data[tp->urg_seq - tp->rcv_nxt]; + + if (unlikely(hdr->dack_mode != csk->delack_mode)) { + csk->delack_mode = hdr->dack_mode; + csk->delack_seq = tp->rcv_nxt; + } + + tcp_hdr(skb)->fin = 0; + tp->rcv_nxt += skb->len; + + __skb_queue_tail(&sk->sk_receive_queue, skb); + + if (!sock_flag(sk, SOCK_DEAD)) { + check_sk_callbacks(csk); + sk->sk_data_ready(sk); + } +} + +static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR; + unsigned int hwtid = GET_TID(req); + struct sock *sk; + + sk = lookup_tid(cdev->tids, hwtid); + if (unlikely(!sk)) { + pr_err("can't find conn. for hwtid %u.\n", hwtid); + return -EINVAL; + } + skb_dst_set(skb, NULL); + process_cpl_msg(chtls_recv_data, sk, skb); + return 0; +} + +static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb) +{ + struct cpl_tls_data *hdr = cplhdr(skb); + struct chtls_sock *csk; + struct chtls_hws *tlsk; + struct tcp_sock *tp; + + csk = rcu_dereference_sk_user_data(sk); + tlsk = &csk->tlshws; + tp = tcp_sk(sk); + + if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) { + handle_excess_rx(sk, skb); + return; + } + + ULP_SKB_CB(skb)->seq = ntohl(hdr->seq); + ULP_SKB_CB(skb)->flags = 0; + skb_ulp_mode(skb) = ULP_MODE_TLS; + + skb_reset_transport_header(skb); + __skb_pull(skb, sizeof(*hdr)); + if (!skb->data_len) + __skb_trim(skb, + CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd))); + + if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq - + tp->rcv_nxt < skb->len)) + tp->urg_data = TCP_URG_VALID | + skb->data[tp->urg_seq - tp->rcv_nxt]; + + tcp_hdr(skb)->fin = 0; + tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)); + __skb_queue_tail(&tlsk->sk_recv_queue, skb); +} + +static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct cpl_tls_data *req = cplhdr(skb); + unsigned int hwtid = GET_TID(req); + struct sock *sk; + + sk = lookup_tid(cdev->tids, hwtid); + if (unlikely(!sk)) { + pr_err("can't find conn. for hwtid %u.\n", hwtid); + return -EINVAL; + } + skb_dst_set(skb, NULL); + process_cpl_msg(chtls_recv_pdu, sk, skb); + return 0; +} + +static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen) +{ + struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb); + + skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length); + tls_cmp_hdr->length = ntohs((__force __be16)nlen); +} + +static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb) +{ + struct tlsrx_cmp_hdr *tls_hdr_pkt; + struct cpl_rx_tls_cmp *cmp_cpl; + struct sk_buff *skb_rec; + struct chtls_sock *csk; + struct chtls_hws *tlsk; + struct tcp_sock *tp; + + cmp_cpl = cplhdr(skb); + csk = rcu_dereference_sk_user_data(sk); + tlsk = &csk->tlshws; + tp = tcp_sk(sk); + + ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq); + ULP_SKB_CB(skb)->flags = 0; + + skb_reset_transport_header(skb); + __skb_pull(skb, sizeof(*cmp_cpl)); + tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data; + if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M) + tls_hdr_pkt->type = CONTENT_TYPE_ERROR; + if (!skb->data_len) + __skb_trim(skb, TLS_HEADER_LENGTH); + + tp->rcv_nxt += + CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length)); + + ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR; + skb_rec = __skb_dequeue(&tlsk->sk_recv_queue); + if (!skb_rec) { + __skb_queue_tail(&sk->sk_receive_queue, skb); + } else { + chtls_set_hdrlen(skb, tlsk->pldlen); + tlsk->pldlen = 0; + __skb_queue_tail(&sk->sk_receive_queue, skb); + __skb_queue_tail(&sk->sk_receive_queue, skb_rec); + } + + if (!sock_flag(sk, SOCK_DEAD)) { + check_sk_callbacks(csk); + sk->sk_data_ready(sk); + } +} + +static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct cpl_rx_tls_cmp *req = cplhdr(skb); + unsigned int hwtid = GET_TID(req); + struct sock *sk; + + sk = lookup_tid(cdev->tids, hwtid); + if (unlikely(!sk)) { + pr_err("can't find conn. for hwtid %u.\n", hwtid); + return -EINVAL; + } + skb_dst_set(skb, NULL); + process_cpl_msg(chtls_rx_hdr, sk, skb); + + return 0; +} + +static void chtls_timewait(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + + tp->rcv_nxt++; + tp->rx_opt.ts_recent_stamp = ktime_get_seconds(); + tp->srtt_us = 0; + tcp_time_wait(sk, TCP_TIME_WAIT, 0); +} + +static void chtls_peer_close(struct sock *sk, struct sk_buff *skb) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + + if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) + goto out; + + sk->sk_shutdown |= RCV_SHUTDOWN; + sock_set_flag(sk, SOCK_DONE); + + switch (sk->sk_state) { + case TCP_SYN_RECV: + case TCP_ESTABLISHED: + tcp_set_state(sk, TCP_CLOSE_WAIT); + break; + case TCP_FIN_WAIT1: + tcp_set_state(sk, TCP_CLOSING); + break; + case TCP_FIN_WAIT2: + chtls_release_resources(sk); + if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) + chtls_conn_done(sk); + else + chtls_timewait(sk); + break; + default: + pr_info("cpl_peer_close in bad state %d\n", sk->sk_state); + } + + if (!sock_flag(sk, SOCK_DEAD)) { + sk->sk_state_change(sk); + /* Do not send POLL_HUP for half duplex close. */ + + if ((sk->sk_shutdown & SEND_SHUTDOWN) || + sk->sk_state == TCP_CLOSE) + sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); + else + sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); + } +out: + kfree_skb(skb); +} + +static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb) +{ + struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR; + struct chtls_sock *csk; + struct tcp_sock *tp; + + csk = rcu_dereference_sk_user_data(sk); + + if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) + goto out; + + tp = tcp_sk(sk); + + tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */ + + switch (sk->sk_state) { + case TCP_CLOSING: + chtls_release_resources(sk); + if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) + chtls_conn_done(sk); + else + chtls_timewait(sk); + break; + case TCP_LAST_ACK: + chtls_release_resources(sk); + chtls_conn_done(sk); + break; + case TCP_FIN_WAIT1: + tcp_set_state(sk, TCP_FIN_WAIT2); + sk->sk_shutdown |= SEND_SHUTDOWN; + + if (!sock_flag(sk, SOCK_DEAD)) + sk->sk_state_change(sk); + else if (tcp_sk(sk)->linger2 < 0 && + !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN)) + chtls_abort_conn(sk, skb); + else if (csk_flag_nochk(csk, CSK_TX_DATA_SENT)) + chtls_set_quiesce_ctrl(sk, 0); + break; + default: + pr_info("close_con_rpl in bad state %d\n", sk->sk_state); + } +out: + kfree_skb(skb); +} + +static struct sk_buff *get_cpl_skb(struct sk_buff *skb, + size_t len, gfp_t gfp) +{ + if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) { + WARN_ONCE(skb->len < len, "skb alloc error"); + __skb_trim(skb, len); + skb_get(skb); + } else { + skb = alloc_skb(len, gfp); + if (skb) + __skb_put(skb, len); + } + return skb; +} + +static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid, + int cmd) +{ + struct cpl_abort_rpl *rpl = cplhdr(skb); + + INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid); + rpl->cmd = cmd; +} + +static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct cpl_abort_req_rss *req = cplhdr(skb); + struct sk_buff *reply_skb; + + reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl), + GFP_KERNEL | __GFP_NOFAIL); + __skb_put(reply_skb, sizeof(struct cpl_abort_rpl)); + set_abort_rpl_wr(reply_skb, GET_TID(req), + (req->status & CPL_ABORT_NO_RST)); + set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1); + cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb); + kfree_skb(skb); +} + +/* + * Add an skb to the deferred skb queue for processing from process context. + */ +static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev, + defer_handler_t handler) +{ + DEFERRED_SKB_CB(skb)->handler = handler; + spin_lock_bh(&cdev->deferq.lock); + __skb_queue_tail(&cdev->deferq, skb); + if (skb_queue_len(&cdev->deferq) == 1) + schedule_work(&cdev->deferq_task); + spin_unlock_bh(&cdev->deferq.lock); +} + +static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb, + struct chtls_dev *cdev, + int status, int queue) +{ + struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR; + struct sk_buff *reply_skb; + struct chtls_sock *csk; + unsigned int tid; + + csk = rcu_dereference_sk_user_data(sk); + tid = GET_TID(req); + + reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any()); + if (!reply_skb) { + req->status = (queue << 1) | status; + t4_defer_reply(skb, cdev, send_defer_abort_rpl); + return; + } + + set_abort_rpl_wr(reply_skb, tid, status); + kfree_skb(skb); + set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue); + if (csk_conn_inline(csk)) { + struct l2t_entry *e = csk->l2t_entry; + + if (e && sk->sk_state != TCP_SYN_RECV) { + cxgb4_l2t_send(csk->egress_dev, reply_skb, e); + return; + } + } + cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb); +} + +/* + * This is run from a listener's backlog to abort a child connection in + * SYN_RCV state (i.e., one on the listener's SYN queue). + */ +static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb) +{ + struct chtls_sock *csk; + struct sock *child; + int queue; + + child = skb->sk; + csk = rcu_dereference_sk_user_data(child); + queue = csk->txq_idx; + + skb->sk = NULL; + chtls_send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev, + CPL_ABORT_NO_RST, queue); + do_abort_syn_rcv(child, lsk); +} + +static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb) +{ + const struct request_sock *oreq; + struct listen_ctx *listen_ctx; + struct chtls_sock *csk; + struct chtls_dev *cdev; + struct sock *psk; + void *ctx; + + csk = sk->sk_user_data; + oreq = csk->passive_reap_next; + cdev = csk->cdev; + + if (!oreq) + return -1; + + ctx = lookup_stid(cdev->tids, oreq->ts_recent); + if (!ctx) + return -1; + + listen_ctx = (struct listen_ctx *)ctx; + psk = listen_ctx->lsk; + + bh_lock_sock(psk); + if (!sock_owned_by_user(psk)) { + int queue = csk->txq_idx; + + chtls_send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue); + do_abort_syn_rcv(sk, psk); + } else { + skb->sk = sk; + BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv; + __sk_add_backlog(psk, skb); + } + bh_unlock_sock(psk); + return 0; +} + +static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb) +{ + const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR; + struct chtls_sock *csk = sk->sk_user_data; + int rst_status = CPL_ABORT_NO_RST; + int queue = csk->txq_idx; + + if (is_neg_adv(req->status)) { + kfree_skb(skb); + return; + } + + csk_reset_flag(csk, CSK_ABORT_REQ_RCVD); + + if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) && + !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) { + struct tcp_sock *tp = tcp_sk(sk); + + if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0) + WARN_ONCE(1, "send_tx_flowc error"); + csk_set_flag(csk, CSK_TX_DATA_SENT); + } + + csk_set_flag(csk, CSK_ABORT_SHUTDOWN); + + if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) { + sk->sk_err = ETIMEDOUT; + + if (!sock_flag(sk, SOCK_DEAD)) + sk_error_report(sk); + + if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb)) + return; + + } + + chtls_send_abort_rpl(sk, skb, BLOG_SKB_CB(skb)->cdev, + rst_status, queue); + chtls_release_resources(sk); + chtls_conn_done(sk); +} + +static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb) +{ + struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR; + struct chtls_sock *csk; + struct chtls_dev *cdev; + + csk = rcu_dereference_sk_user_data(sk); + cdev = csk->cdev; + + if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) { + csk_reset_flag(csk, CSK_ABORT_RPL_PENDING); + if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) { + if (sk->sk_state == TCP_SYN_SENT) { + cxgb4_remove_tid(cdev->tids, + csk->port_id, + GET_TID(rpl), + sk->sk_family); + sock_put(sk); + } + chtls_release_resources(sk); + chtls_conn_done(sk); + } + } + kfree_skb(skb); +} + +static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR; + void (*fn)(struct sock *sk, struct sk_buff *skb); + unsigned int hwtid = GET_TID(req); + struct chtls_sock *csk; + struct sock *sk; + u8 opcode; + + opcode = ((const struct rss_header *)cplhdr(skb))->opcode; + + sk = lookup_tid(cdev->tids, hwtid); + if (!sk) + goto rel_skb; + + csk = sk->sk_user_data; + + switch (opcode) { + case CPL_PEER_CLOSE: + fn = chtls_peer_close; + break; + case CPL_CLOSE_CON_RPL: + fn = chtls_close_con_rpl; + break; + case CPL_ABORT_REQ_RSS: + /* + * Save the offload device in the skb, we may process this + * message after the socket has closed. + */ + BLOG_SKB_CB(skb)->cdev = csk->cdev; + fn = chtls_abort_req_rss; + break; + case CPL_ABORT_RPL_RSS: + fn = chtls_abort_rpl_rss; + break; + default: + goto rel_skb; + } + + process_cpl_msg(fn, sk, skb); + return 0; + +rel_skb: + kfree_skb(skb); + return 0; +} + +static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb) +{ + struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR; + struct chtls_sock *csk = sk->sk_user_data; + struct tcp_sock *tp = tcp_sk(sk); + u32 credits = hdr->credits; + u32 snd_una; + + snd_una = ntohl(hdr->snd_una); + csk->wr_credits += credits; + + if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits) + csk->wr_unacked = csk->wr_max_credits - csk->wr_credits; + + while (credits) { + struct sk_buff *pskb = csk->wr_skb_head; + u32 csum; + + if (unlikely(!pskb)) { + if (csk->wr_nondata) + csk->wr_nondata -= credits; + break; + } + csum = (__force u32)pskb->csum; + if (unlikely(credits < csum)) { + pskb->csum = (__force __wsum)(csum - credits); + break; + } + dequeue_wr(sk); + credits -= csum; + kfree_skb(pskb); + } + if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) { + if (unlikely(before(snd_una, tp->snd_una))) { + kfree_skb(skb); + return; + } + + if (tp->snd_una != snd_una) { + tp->snd_una = snd_una; + tp->rcv_tstamp = tcp_jiffies32; + if (tp->snd_una == tp->snd_nxt && + !csk_flag_nochk(csk, CSK_TX_FAILOVER)) + csk_reset_flag(csk, CSK_TX_WAIT_IDLE); + } + } + + if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) { + unsigned int fclen16 = roundup(failover_flowc_wr_len, 16); + + csk->wr_credits -= fclen16; + csk_reset_flag(csk, CSK_TX_WAIT_IDLE); + csk_reset_flag(csk, CSK_TX_FAILOVER); + } + if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0)) + sk->sk_write_space(sk); + + kfree_skb(skb); +} + +static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR; + unsigned int hwtid = GET_TID(rpl); + struct sock *sk; + + sk = lookup_tid(cdev->tids, hwtid); + if (unlikely(!sk)) { + pr_err("can't find conn. for hwtid %u.\n", hwtid); + return -EINVAL; + } + process_cpl_msg(chtls_rx_ack, sk, skb); + + return 0; +} + +static int chtls_set_tcb_rpl(struct chtls_dev *cdev, struct sk_buff *skb) +{ + struct cpl_set_tcb_rpl *rpl = cplhdr(skb) + RSS_HDR; + unsigned int hwtid = GET_TID(rpl); + struct sock *sk; + + sk = lookup_tid(cdev->tids, hwtid); + + /* return EINVAL if socket doesn't exist */ + if (!sk) + return -EINVAL; + + /* Reusing the skb as size of cpl_set_tcb_field structure + * is greater than cpl_abort_req + */ + if (TCB_COOKIE_G(rpl->cookie) == TCB_FIELD_COOKIE_TFLAG) + chtls_send_abort(sk, CPL_ABORT_SEND_RST, NULL); + + kfree_skb(skb); + return 0; +} + +chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = { + [CPL_PASS_OPEN_RPL] = chtls_pass_open_rpl, + [CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl, + [CPL_PASS_ACCEPT_REQ] = chtls_pass_accept_req, + [CPL_PASS_ESTABLISH] = chtls_pass_establish, + [CPL_RX_DATA] = chtls_rx_data, + [CPL_TLS_DATA] = chtls_rx_pdu, + [CPL_RX_TLS_CMP] = chtls_rx_cmp, + [CPL_PEER_CLOSE] = chtls_conn_cpl, + [CPL_CLOSE_CON_RPL] = chtls_conn_cpl, + [CPL_ABORT_REQ_RSS] = chtls_conn_cpl, + [CPL_ABORT_RPL_RSS] = chtls_conn_cpl, + [CPL_FW4_ACK] = chtls_wr_ack, + [CPL_SET_TCB_RPL] = chtls_set_tcb_rpl, +}; diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h new file mode 100644 index 000000000..f61ca6576 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h @@ -0,0 +1,219 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018 Chelsio Communications, Inc. + */ + +#ifndef __CHTLS_CM_H__ +#define __CHTLS_CM_H__ + +/* + * TCB settings + */ +/* 3:0 */ +#define TCB_ULP_TYPE_W 0 +#define TCB_ULP_TYPE_S 0 +#define TCB_ULP_TYPE_M 0xfULL +#define TCB_ULP_TYPE_V(x) ((x) << TCB_ULP_TYPE_S) + +/* 11:4 */ +#define TCB_ULP_RAW_W 0 +#define TCB_ULP_RAW_S 4 +#define TCB_ULP_RAW_M 0xffULL +#define TCB_ULP_RAW_V(x) ((x) << TCB_ULP_RAW_S) + +#define TF_TLS_KEY_SIZE_S 7 +#define TF_TLS_KEY_SIZE_V(x) ((x) << TF_TLS_KEY_SIZE_S) + +#define TF_TLS_CONTROL_S 2 +#define TF_TLS_CONTROL_V(x) ((x) << TF_TLS_CONTROL_S) + +#define TF_TLS_ACTIVE_S 1 +#define TF_TLS_ACTIVE_V(x) ((x) << TF_TLS_ACTIVE_S) + +#define TF_TLS_ENABLE_S 0 +#define TF_TLS_ENABLE_V(x) ((x) << TF_TLS_ENABLE_S) + +#define TF_RX_QUIESCE_S 15 +#define TF_RX_QUIESCE_V(x) ((x) << TF_RX_QUIESCE_S) + +/* + * Max receive window supported by HW in bytes. Only a small part of it can + * be set through option0, the rest needs to be set through RX_DATA_ACK. + */ +#define MAX_RCV_WND ((1U << 27) - 1) +#define MAX_MSS 65536 + +/* + * Min receive window. We want it to be large enough to accommodate receive + * coalescing, handle jumbo frames, and not trigger sender SWS avoidance. + */ +#define MIN_RCV_WND (24 * 1024U) +#define LOOPBACK(x) (((x) & htonl(0xff000000)) == htonl(0x7f000000)) + +/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */ +#define TX_HEADER_LEN \ + (sizeof(struct fw_ofld_tx_data_wr) + sizeof(struct sge_opaque_hdr)) +#define TX_TLSHDR_LEN \ + (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo) + \ + sizeof(struct sge_opaque_hdr)) +#define TXDATA_SKB_LEN 128 + +enum { + CPL_TX_TLS_SFO_TYPE_CCS, + CPL_TX_TLS_SFO_TYPE_ALERT, + CPL_TX_TLS_SFO_TYPE_HANDSHAKE, + CPL_TX_TLS_SFO_TYPE_DATA, + CPL_TX_TLS_SFO_TYPE_HEARTBEAT, +}; + +enum { + TLS_HDR_TYPE_CCS = 20, + TLS_HDR_TYPE_ALERT, + TLS_HDR_TYPE_HANDSHAKE, + TLS_HDR_TYPE_RECORD, + TLS_HDR_TYPE_HEARTBEAT, +}; + +typedef void (*defer_handler_t)(struct chtls_dev *dev, struct sk_buff *skb); +extern struct request_sock_ops chtls_rsk_ops; +extern struct request_sock_ops chtls_rsk_opsv6; + +struct deferred_skb_cb { + defer_handler_t handler; + struct chtls_dev *dev; +}; + +#define DEFERRED_SKB_CB(skb) ((struct deferred_skb_cb *)(skb)->cb) +#define failover_flowc_wr_len offsetof(struct fw_flowc_wr, mnemval[3]) +#define WR_SKB_CB(skb) ((struct wr_skb_cb *)(skb)->cb) +#define ACCEPT_QUEUE(sk) (&inet_csk(sk)->icsk_accept_queue.rskq_accept_head) + +#define SND_WSCALE(tp) ((tp)->rx_opt.snd_wscale) +#define RCV_WSCALE(tp) ((tp)->rx_opt.rcv_wscale) +#define USER_MSS(tp) ((tp)->rx_opt.user_mss) +#define TS_RECENT_STAMP(tp) ((tp)->rx_opt.ts_recent_stamp) +#define WSCALE_OK(tp) ((tp)->rx_opt.wscale_ok) +#define TSTAMP_OK(tp) ((tp)->rx_opt.tstamp_ok) +#define SACK_OK(tp) ((tp)->rx_opt.sack_ok) +#define INC_ORPHAN_COUNT(sk) this_cpu_inc(*(sk)->sk_prot->orphan_count) + +/* TLS SKB */ +#define skb_ulp_tls_inline(skb) (ULP_SKB_CB(skb)->ulp.tls.ofld) +#define skb_ulp_tls_iv_imm(skb) (ULP_SKB_CB(skb)->ulp.tls.iv) + +void chtls_defer_reply(struct sk_buff *skb, struct chtls_dev *dev, + defer_handler_t handler); + +/* + * Returns true if the socket is in one of the supplied states. + */ +static inline unsigned int sk_in_state(const struct sock *sk, + unsigned int states) +{ + return states & (1 << sk->sk_state); +} + +static void chtls_rsk_destructor(struct request_sock *req) +{ + /* do nothing */ +} + +static inline void chtls_init_rsk_ops(struct proto *chtls_tcp_prot, + struct request_sock_ops *chtls_tcp_ops, + struct proto *tcp_prot, int family) +{ + memset(chtls_tcp_ops, 0, sizeof(*chtls_tcp_ops)); + chtls_tcp_ops->family = family; + chtls_tcp_ops->obj_size = sizeof(struct tcp_request_sock); + chtls_tcp_ops->destructor = chtls_rsk_destructor; + chtls_tcp_ops->slab = tcp_prot->rsk_prot->slab; + chtls_tcp_prot->rsk_prot = chtls_tcp_ops; +} + +static inline void chtls_reqsk_free(struct request_sock *req) +{ + if (req->rsk_listener) + sock_put(req->rsk_listener); + kmem_cache_free(req->rsk_ops->slab, req); +} + +#define DECLARE_TASK_FUNC(task, task_param) \ + static void task(struct work_struct *task_param) + +static inline void sk_wakeup_sleepers(struct sock *sk, bool interruptable) +{ + struct socket_wq *wq; + + rcu_read_lock(); + wq = rcu_dereference(sk->sk_wq); + if (skwq_has_sleeper(wq)) { + if (interruptable) + wake_up_interruptible(sk_sleep(sk)); + else + wake_up_all(sk_sleep(sk)); + } + rcu_read_unlock(); +} + +static inline void chtls_set_req_port(struct request_sock *oreq, + __be16 source, __be16 dest) +{ + inet_rsk(oreq)->ir_rmt_port = source; + inet_rsk(oreq)->ir_num = ntohs(dest); +} + +static inline void chtls_set_req_addr(struct request_sock *oreq, + __be32 local_ip, __be32 peer_ip) +{ + inet_rsk(oreq)->ir_loc_addr = local_ip; + inet_rsk(oreq)->ir_rmt_addr = peer_ip; +} + +static inline void chtls_free_skb(struct sock *sk, struct sk_buff *skb) +{ + skb_dst_set(skb, NULL); + __skb_unlink(skb, &sk->sk_receive_queue); + __kfree_skb(skb); +} + +static inline void chtls_kfree_skb(struct sock *sk, struct sk_buff *skb) +{ + skb_dst_set(skb, NULL); + __skb_unlink(skb, &sk->sk_receive_queue); + kfree_skb(skb); +} + +static inline void chtls_reset_wr_list(struct chtls_sock *csk) +{ + csk->wr_skb_head = NULL; + csk->wr_skb_tail = NULL; +} + +static inline void enqueue_wr(struct chtls_sock *csk, struct sk_buff *skb) +{ + WR_SKB_CB(skb)->next_wr = NULL; + + skb_get(skb); + + if (!csk->wr_skb_head) + csk->wr_skb_head = skb; + else + WR_SKB_CB(csk->wr_skb_tail)->next_wr = skb; + csk->wr_skb_tail = skb; +} + +static inline struct sk_buff *dequeue_wr(struct sock *sk) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct sk_buff *skb = NULL; + + skb = csk->wr_skb_head; + + if (likely(skb)) { + /* Don't bother clearing the tail */ + csk->wr_skb_head = WR_SKB_CB(skb)->next_wr; + WR_SKB_CB(skb)->next_wr = NULL; + } + return skb; +} +#endif diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_hw.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_hw.c new file mode 100644 index 000000000..1e67140b0 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_hw.c @@ -0,0 +1,471 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018 Chelsio Communications, Inc. + * + * Written by: Atul Gupta (atul.gupta@chelsio.com) + */ + +#include <linux/module.h> +#include <linux/list.h> +#include <linux/workqueue.h> +#include <linux/skbuff.h> +#include <linux/timer.h> +#include <linux/notifier.h> +#include <linux/inetdevice.h> +#include <linux/ip.h> +#include <linux/tcp.h> +#include <linux/tls.h> +#include <net/tls.h> + +#include "chtls.h" +#include "chtls_cm.h" + +static void __set_tcb_field_direct(struct chtls_sock *csk, + struct cpl_set_tcb_field *req, u16 word, + u64 mask, u64 val, u8 cookie, int no_reply) +{ + struct ulptx_idata *sc; + + INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid); + req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid)); + req->reply_ctrl = htons(NO_REPLY_V(no_reply) | + QUEUENO_V(csk->rss_qid)); + req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie)); + req->mask = cpu_to_be64(mask); + req->val = cpu_to_be64(val); + sc = (struct ulptx_idata *)(req + 1); + sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); + sc->len = htonl(0); +} + +static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word, + u64 mask, u64 val, u8 cookie, int no_reply) +{ + struct cpl_set_tcb_field *req; + struct chtls_sock *csk; + struct ulptx_idata *sc; + unsigned int wrlen; + + wrlen = roundup(sizeof(*req) + sizeof(*sc), 16); + csk = rcu_dereference_sk_user_data(sk); + + req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen); + __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply); + set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); +} + +/* + * Send control message to HW, message go as immediate data and packet + * is freed immediately. + */ +static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val) +{ + struct cpl_set_tcb_field *req; + unsigned int credits_needed; + struct chtls_sock *csk; + struct ulptx_idata *sc; + struct sk_buff *skb; + unsigned int wrlen; + int ret; + + wrlen = roundup(sizeof(*req) + sizeof(*sc), 16); + + skb = alloc_skb(wrlen, GFP_ATOMIC); + if (!skb) + return -ENOMEM; + + credits_needed = DIV_ROUND_UP(wrlen, 16); + csk = rcu_dereference_sk_user_data(sk); + + __set_tcb_field(sk, skb, word, mask, val, 0, 1); + skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA); + csk->wr_credits -= credits_needed; + csk->wr_unacked += credits_needed; + enqueue_wr(csk, skb); + ret = cxgb4_ofld_send(csk->egress_dev, skb); + if (ret < 0) + kfree_skb(skb); + return ret < 0 ? ret : 0; +} + +void chtls_set_tcb_field_rpl_skb(struct sock *sk, u16 word, + u64 mask, u64 val, u8 cookie, + int through_l2t) +{ + struct sk_buff *skb; + unsigned int wrlen; + + wrlen = sizeof(struct cpl_set_tcb_field) + sizeof(struct ulptx_idata); + wrlen = roundup(wrlen, 16); + + skb = alloc_skb(wrlen, GFP_KERNEL | __GFP_NOFAIL); + if (!skb) + return; + + __set_tcb_field(sk, skb, word, mask, val, cookie, 0); + send_or_defer(sk, tcp_sk(sk), skb, through_l2t); +} + +/* + * Set one of the t_flags bits in the TCB. + */ +int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val) +{ + return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos, + (u64)val << bit_pos); +} + +static int chtls_set_tcb_keyid(struct sock *sk, int keyid) +{ + return chtls_set_tcb_field(sk, 31, 0xFFFFFFFFULL, keyid); +} + +static int chtls_set_tcb_seqno(struct sock *sk) +{ + return chtls_set_tcb_field(sk, 28, ~0ULL, 0); +} + +static int chtls_set_tcb_quiesce(struct sock *sk, int val) +{ + return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S), + TF_RX_QUIESCE_V(val)); +} + +void chtls_set_quiesce_ctrl(struct sock *sk, int val) +{ + struct chtls_sock *csk; + struct sk_buff *skb; + unsigned int wrlen; + int ret; + + wrlen = sizeof(struct cpl_set_tcb_field) + sizeof(struct ulptx_idata); + wrlen = roundup(wrlen, 16); + + skb = alloc_skb(wrlen, GFP_ATOMIC); + if (!skb) + return; + + csk = rcu_dereference_sk_user_data(sk); + + __set_tcb_field(sk, skb, 1, TF_RX_QUIESCE_V(1), 0, 0, 1); + set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); + ret = cxgb4_ofld_send(csk->egress_dev, skb); + if (ret < 0) + kfree_skb(skb); +} + +/* TLS Key bitmap processing */ +int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi) +{ + unsigned int num_key_ctx, bsize; + int ksize; + + num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ); + bsize = BITS_TO_LONGS(num_key_ctx); + + cdev->kmap.size = num_key_ctx; + cdev->kmap.available = bsize; + ksize = sizeof(*cdev->kmap.addr) * bsize; + cdev->kmap.addr = kvzalloc(ksize, GFP_KERNEL); + if (!cdev->kmap.addr) + return -ENOMEM; + + cdev->kmap.start = lldi->vr->key.start; + spin_lock_init(&cdev->kmap.lock); + return 0; +} + +static int get_new_keyid(struct chtls_sock *csk, u32 optname) +{ + struct net_device *dev = csk->egress_dev; + struct chtls_dev *cdev = csk->cdev; + struct chtls_hws *hws; + struct adapter *adap; + int keyid; + + adap = netdev2adap(dev); + hws = &csk->tlshws; + + spin_lock_bh(&cdev->kmap.lock); + keyid = find_first_zero_bit(cdev->kmap.addr, cdev->kmap.size); + if (keyid < cdev->kmap.size) { + __set_bit(keyid, cdev->kmap.addr); + if (optname == TLS_RX) + hws->rxkey = keyid; + else + hws->txkey = keyid; + atomic_inc(&adap->chcr_stats.tls_key); + } else { + keyid = -1; + } + spin_unlock_bh(&cdev->kmap.lock); + return keyid; +} + +void free_tls_keyid(struct sock *sk) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct net_device *dev = csk->egress_dev; + struct chtls_dev *cdev = csk->cdev; + struct chtls_hws *hws; + struct adapter *adap; + + if (!cdev->kmap.addr) + return; + + adap = netdev2adap(dev); + hws = &csk->tlshws; + + spin_lock_bh(&cdev->kmap.lock); + if (hws->rxkey >= 0) { + __clear_bit(hws->rxkey, cdev->kmap.addr); + atomic_dec(&adap->chcr_stats.tls_key); + hws->rxkey = -1; + } + if (hws->txkey >= 0) { + __clear_bit(hws->txkey, cdev->kmap.addr); + atomic_dec(&adap->chcr_stats.tls_key); + hws->txkey = -1; + } + spin_unlock_bh(&cdev->kmap.lock); +} + +unsigned int keyid_to_addr(int start_addr, int keyid) +{ + return (start_addr + (keyid * TLS_KEY_CONTEXT_SZ)) >> 5; +} + +static void chtls_rxkey_ivauth(struct _key_ctx *kctx) +{ + kctx->iv_to_auth = cpu_to_be64(KEYCTX_TX_WR_IV_V(6ULL) | + KEYCTX_TX_WR_AAD_V(1ULL) | + KEYCTX_TX_WR_AADST_V(5ULL) | + KEYCTX_TX_WR_CIPHER_V(14ULL) | + KEYCTX_TX_WR_CIPHERST_V(0ULL) | + KEYCTX_TX_WR_AUTH_V(14ULL) | + KEYCTX_TX_WR_AUTHST_V(16ULL) | + KEYCTX_TX_WR_AUTHIN_V(16ULL)); +} + +static int chtls_key_info(struct chtls_sock *csk, + struct _key_ctx *kctx, + u32 keylen, u32 optname, + int cipher_type) +{ + unsigned char key[AES_MAX_KEY_SIZE]; + unsigned char *key_p, *salt; + unsigned char ghash_h[AEAD_H_SIZE]; + int ck_size, key_ctx_size, kctx_mackey_size, salt_size; + struct crypto_aes_ctx aes; + int ret; + + key_ctx_size = sizeof(struct _key_ctx) + + roundup(keylen, 16) + AEAD_H_SIZE; + + /* GCM mode of AES supports 128 and 256 bit encryption, so + * prepare key context base on GCM cipher type + */ + switch (cipher_type) { + case TLS_CIPHER_AES_GCM_128: { + struct tls12_crypto_info_aes_gcm_128 *gcm_ctx_128 = + (struct tls12_crypto_info_aes_gcm_128 *) + &csk->tlshws.crypto_info; + memcpy(key, gcm_ctx_128->key, keylen); + + key_p = gcm_ctx_128->key; + salt = gcm_ctx_128->salt; + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE; + kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; + break; + } + case TLS_CIPHER_AES_GCM_256: { + struct tls12_crypto_info_aes_gcm_256 *gcm_ctx_256 = + (struct tls12_crypto_info_aes_gcm_256 *) + &csk->tlshws.crypto_info; + memcpy(key, gcm_ctx_256->key, keylen); + + key_p = gcm_ctx_256->key; + salt = gcm_ctx_256->salt; + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; + salt_size = TLS_CIPHER_AES_GCM_256_SALT_SIZE; + kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; + break; + } + default: + pr_err("GCM: Invalid key length %d\n", keylen); + return -EINVAL; + } + + /* Calculate the H = CIPH(K, 0 repeated 16 times). + * It will go in key context + */ + ret = aes_expandkey(&aes, key, keylen); + if (ret) + return ret; + + memset(ghash_h, 0, AEAD_H_SIZE); + aes_encrypt(&aes, ghash_h, ghash_h); + memzero_explicit(&aes, sizeof(aes)); + csk->tlshws.keylen = key_ctx_size; + + /* Copy the Key context */ + if (optname == TLS_RX) { + int key_ctx; + + key_ctx = ((key_ctx_size >> 4) << 3); + kctx->ctx_hdr = FILL_KEY_CRX_HDR(ck_size, + kctx_mackey_size, + 0, 0, key_ctx); + chtls_rxkey_ivauth(kctx); + } else { + kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size, + kctx_mackey_size, + 0, 0, key_ctx_size >> 4); + } + + memcpy(kctx->salt, salt, salt_size); + memcpy(kctx->key, key_p, keylen); + memcpy(kctx->key + keylen, ghash_h, AEAD_H_SIZE); + /* erase key info from driver */ + memset(key_p, 0, keylen); + + return 0; +} + +static void chtls_set_scmd(struct chtls_sock *csk) +{ + struct chtls_hws *hws = &csk->tlshws; + + hws->scmd.seqno_numivs = + SCMD_SEQ_NO_CTRL_V(3) | + SCMD_PROTO_VERSION_V(0) | + SCMD_ENC_DEC_CTRL_V(0) | + SCMD_CIPH_AUTH_SEQ_CTRL_V(1) | + SCMD_CIPH_MODE_V(2) | + SCMD_AUTH_MODE_V(4) | + SCMD_HMAC_CTRL_V(0) | + SCMD_IV_SIZE_V(4) | + SCMD_NUM_IVS_V(1); + + hws->scmd.ivgen_hdrlen = + SCMD_IV_GEN_CTRL_V(1) | + SCMD_KEY_CTX_INLINE_V(0) | + SCMD_TLS_FRAG_ENABLE_V(1); +} + +int chtls_setkey(struct chtls_sock *csk, u32 keylen, + u32 optname, int cipher_type) +{ + struct tls_key_req *kwr; + struct chtls_dev *cdev; + struct _key_ctx *kctx; + int wrlen, klen, len; + struct sk_buff *skb; + struct sock *sk; + int keyid; + int kaddr; + int ret; + + cdev = csk->cdev; + sk = csk->sk; + + klen = roundup((keylen + AEAD_H_SIZE) + sizeof(*kctx), 32); + wrlen = roundup(sizeof(*kwr), 16); + len = klen + wrlen; + + /* Flush out-standing data before new key takes effect */ + if (optname == TLS_TX) { + lock_sock(sk); + if (skb_queue_len(&csk->txq)) + chtls_push_frames(csk, 0); + release_sock(sk); + } + + skb = alloc_skb(len, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + keyid = get_new_keyid(csk, optname); + if (keyid < 0) { + ret = -ENOSPC; + goto out_nokey; + } + + kaddr = keyid_to_addr(cdev->kmap.start, keyid); + kwr = (struct tls_key_req *)__skb_put_zero(skb, len); + kwr->wr.op_to_compl = + cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) | FW_WR_COMPL_F | + FW_WR_ATOMIC_V(1U)); + kwr->wr.flowid_len16 = + cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16) | + FW_WR_FLOWID_V(csk->tid))); + kwr->wr.protocol = 0; + kwr->wr.mfs = htons(TLS_MFS); + kwr->wr.reneg_to_write_rx = optname; + + /* ulptx command */ + kwr->req.cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE) | + T5_ULP_MEMIO_ORDER_V(1) | + T5_ULP_MEMIO_IMM_V(1)); + kwr->req.len16 = cpu_to_be32((csk->tid << 8) | + DIV_ROUND_UP(len - sizeof(kwr->wr), 16)); + kwr->req.dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(klen >> 5)); + kwr->req.lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(kaddr)); + + /* sub command */ + kwr->sc_imm.cmd_more = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM)); + kwr->sc_imm.len = cpu_to_be32(klen); + + lock_sock(sk); + /* key info */ + kctx = (struct _key_ctx *)(kwr + 1); + ret = chtls_key_info(csk, kctx, keylen, optname, cipher_type); + if (ret) + goto out_notcb; + + if (unlikely(csk_flag(sk, CSK_ABORT_SHUTDOWN))) + goto out_notcb; + + set_wr_txq(skb, CPL_PRIORITY_DATA, csk->tlshws.txqid); + csk->wr_credits -= DIV_ROUND_UP(len, 16); + csk->wr_unacked += DIV_ROUND_UP(len, 16); + enqueue_wr(csk, skb); + cxgb4_ofld_send(csk->egress_dev, skb); + skb = NULL; + + chtls_set_scmd(csk); + /* Clear quiesce for Rx key */ + if (optname == TLS_RX) { + ret = chtls_set_tcb_keyid(sk, keyid); + if (ret) + goto out_notcb; + ret = chtls_set_tcb_field(sk, 0, + TCB_ULP_RAW_V(TCB_ULP_RAW_M), + TCB_ULP_RAW_V((TF_TLS_KEY_SIZE_V(1) | + TF_TLS_CONTROL_V(1) | + TF_TLS_ACTIVE_V(1) | + TF_TLS_ENABLE_V(1)))); + if (ret) + goto out_notcb; + ret = chtls_set_tcb_seqno(sk); + if (ret) + goto out_notcb; + ret = chtls_set_tcb_quiesce(sk, 0); + if (ret) + goto out_notcb; + csk->tlshws.rxkey = keyid; + } else { + csk->tlshws.tx_seq_no = 0; + csk->tlshws.txkey = keyid; + } + + release_sock(sk); + return ret; +out_notcb: + release_sock(sk); + free_tls_keyid(sk); +out_nokey: + kfree_skb(skb); + return ret; +} diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c new file mode 100644 index 000000000..5e45bef4f --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c @@ -0,0 +1,1933 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018 Chelsio Communications, Inc. + * + * Written by: Atul Gupta (atul.gupta@chelsio.com) + */ + +#include <linux/module.h> +#include <linux/list.h> +#include <linux/workqueue.h> +#include <linux/skbuff.h> +#include <linux/timer.h> +#include <linux/notifier.h> +#include <linux/inetdevice.h> +#include <linux/ip.h> +#include <linux/tcp.h> +#include <linux/sched/signal.h> +#include <net/tcp.h> +#include <net/busy_poll.h> +#include <crypto/aes.h> + +#include "chtls.h" +#include "chtls_cm.h" + +static bool is_tls_tx(struct chtls_sock *csk) +{ + return csk->tlshws.txkey >= 0; +} + +static bool is_tls_rx(struct chtls_sock *csk) +{ + return csk->tlshws.rxkey >= 0; +} + +static int data_sgl_len(const struct sk_buff *skb) +{ + unsigned int cnt; + + cnt = skb_shinfo(skb)->nr_frags; + return sgl_len(cnt) * 8; +} + +static int nos_ivs(struct sock *sk, unsigned int size) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + + return DIV_ROUND_UP(size, csk->tlshws.mfs); +} + +static int set_ivs_imm(struct sock *sk, const struct sk_buff *skb) +{ + int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE; + int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb); + + if ((hlen + KEY_ON_MEM_SZ + ivs_size) < + MAX_IMM_OFLD_TX_DATA_WR_LEN) { + ULP_SKB_CB(skb)->ulp.tls.iv = 1; + return 1; + } + ULP_SKB_CB(skb)->ulp.tls.iv = 0; + return 0; +} + +static int max_ivs_size(struct sock *sk, int size) +{ + return nos_ivs(sk, size) * CIPHER_BLOCK_SIZE; +} + +static int ivs_size(struct sock *sk, const struct sk_buff *skb) +{ + return set_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) * + CIPHER_BLOCK_SIZE) : 0; +} + +static int flowc_wr_credits(int nparams, int *flowclenp) +{ + int flowclen16, flowclen; + + flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]); + flowclen16 = DIV_ROUND_UP(flowclen, 16); + flowclen = flowclen16 * 16; + + if (flowclenp) + *flowclenp = flowclen; + + return flowclen16; +} + +static struct sk_buff *create_flowc_wr_skb(struct sock *sk, + struct fw_flowc_wr *flowc, + int flowclen) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct sk_buff *skb; + + skb = alloc_skb(flowclen, GFP_ATOMIC); + if (!skb) + return NULL; + + __skb_put_data(skb, flowc, flowclen); + skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA); + + return skb; +} + +static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc, + int flowclen) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct tcp_sock *tp = tcp_sk(sk); + struct sk_buff *skb; + int flowclen16; + int ret; + + flowclen16 = flowclen / 16; + + if (csk_flag(sk, CSK_TX_DATA_SENT)) { + skb = create_flowc_wr_skb(sk, flowc, flowclen); + if (!skb) + return -ENOMEM; + + skb_entail(sk, skb, + ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND); + return 0; + } + + ret = cxgb4_immdata_send(csk->egress_dev, + csk->txq_idx, + flowc, flowclen); + if (!ret) + return flowclen16; + skb = create_flowc_wr_skb(sk, flowc, flowclen); + if (!skb) + return -ENOMEM; + send_or_defer(sk, tp, skb, 0); + return flowclen16; +} + +static u8 tcp_state_to_flowc_state(u8 state) +{ + switch (state) { + case TCP_ESTABLISHED: + return FW_FLOWC_MNEM_TCPSTATE_ESTABLISHED; + case TCP_CLOSE_WAIT: + return FW_FLOWC_MNEM_TCPSTATE_CLOSEWAIT; + case TCP_FIN_WAIT1: + return FW_FLOWC_MNEM_TCPSTATE_FINWAIT1; + case TCP_CLOSING: + return FW_FLOWC_MNEM_TCPSTATE_CLOSING; + case TCP_LAST_ACK: + return FW_FLOWC_MNEM_TCPSTATE_LASTACK; + case TCP_FIN_WAIT2: + return FW_FLOWC_MNEM_TCPSTATE_FINWAIT2; + } + + return FW_FLOWC_MNEM_TCPSTATE_ESTABLISHED; +} + +int send_tx_flowc_wr(struct sock *sk, int compl, + u32 snd_nxt, u32 rcv_nxt) +{ + struct flowc_packed { + struct fw_flowc_wr fc; + struct fw_flowc_mnemval mnemval[FW_FLOWC_MNEM_MAX]; + } __packed sflowc; + int nparams, paramidx, flowclen16, flowclen; + struct fw_flowc_wr *flowc; + struct chtls_sock *csk; + struct tcp_sock *tp; + + csk = rcu_dereference_sk_user_data(sk); + tp = tcp_sk(sk); + memset(&sflowc, 0, sizeof(sflowc)); + flowc = &sflowc.fc; + +#define FLOWC_PARAM(__m, __v) \ + do { \ + flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \ + flowc->mnemval[paramidx].val = cpu_to_be32(__v); \ + paramidx++; \ + } while (0) + + paramidx = 0; + + FLOWC_PARAM(PFNVFN, FW_PFVF_CMD_PFN_V(csk->cdev->lldi->pf)); + FLOWC_PARAM(CH, csk->tx_chan); + FLOWC_PARAM(PORT, csk->tx_chan); + FLOWC_PARAM(IQID, csk->rss_qid); + FLOWC_PARAM(SNDNXT, tp->snd_nxt); + FLOWC_PARAM(RCVNXT, tp->rcv_nxt); + FLOWC_PARAM(SNDBUF, csk->sndbuf); + FLOWC_PARAM(MSS, tp->mss_cache); + FLOWC_PARAM(TCPSTATE, tcp_state_to_flowc_state(sk->sk_state)); + + if (SND_WSCALE(tp)) + FLOWC_PARAM(RCV_SCALE, SND_WSCALE(tp)); + + if (csk->ulp_mode == ULP_MODE_TLS) + FLOWC_PARAM(ULD_MODE, ULP_MODE_TLS); + + if (csk->tlshws.fcplenmax) + FLOWC_PARAM(TXDATAPLEN_MAX, csk->tlshws.fcplenmax); + + nparams = paramidx; +#undef FLOWC_PARAM + + flowclen16 = flowc_wr_credits(nparams, &flowclen); + flowc->op_to_nparams = + cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) | + FW_WR_COMPL_V(compl) | + FW_FLOWC_WR_NPARAMS_V(nparams)); + flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(flowclen16) | + FW_WR_FLOWID_V(csk->tid)); + + return send_flowc_wr(sk, flowc, flowclen); +} + +/* Copy IVs to WR */ +static int tls_copy_ivs(struct sock *sk, struct sk_buff *skb) + +{ + struct chtls_sock *csk; + unsigned char *iv_loc; + struct chtls_hws *hws; + unsigned char *ivs; + u16 number_of_ivs; + struct page *page; + int err = 0; + + csk = rcu_dereference_sk_user_data(sk); + hws = &csk->tlshws; + number_of_ivs = nos_ivs(sk, skb->len); + + if (number_of_ivs > MAX_IVS_PAGE) { + pr_warn("MAX IVs in PAGE exceeded %d\n", number_of_ivs); + return -ENOMEM; + } + + /* generate the IVs */ + ivs = kmalloc_array(CIPHER_BLOCK_SIZE, number_of_ivs, GFP_ATOMIC); + if (!ivs) + return -ENOMEM; + get_random_bytes(ivs, number_of_ivs * CIPHER_BLOCK_SIZE); + + if (skb_ulp_tls_iv_imm(skb)) { + /* send the IVs as immediate data in the WR */ + iv_loc = (unsigned char *)__skb_push(skb, number_of_ivs * + CIPHER_BLOCK_SIZE); + if (iv_loc) + memcpy(iv_loc, ivs, number_of_ivs * CIPHER_BLOCK_SIZE); + + hws->ivsize = number_of_ivs * CIPHER_BLOCK_SIZE; + } else { + /* Send the IVs as sgls */ + /* Already accounted IV DSGL for credits */ + skb_shinfo(skb)->nr_frags--; + page = alloc_pages(sk->sk_allocation | __GFP_COMP, 0); + if (!page) { + pr_info("%s : Page allocation for IVs failed\n", + __func__); + err = -ENOMEM; + goto out; + } + memcpy(page_address(page), ivs, number_of_ivs * + CIPHER_BLOCK_SIZE); + skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, page, 0, + number_of_ivs * CIPHER_BLOCK_SIZE); + hws->ivsize = 0; + } +out: + kfree(ivs); + return err; +} + +/* Copy Key to WR */ +static void tls_copy_tx_key(struct sock *sk, struct sk_buff *skb) +{ + struct ulptx_sc_memrd *sc_memrd; + struct chtls_sock *csk; + struct chtls_dev *cdev; + struct ulptx_idata *sc; + struct chtls_hws *hws; + u32 immdlen; + int kaddr; + + csk = rcu_dereference_sk_user_data(sk); + hws = &csk->tlshws; + cdev = csk->cdev; + + immdlen = sizeof(*sc) + sizeof(*sc_memrd); + kaddr = keyid_to_addr(cdev->kmap.start, hws->txkey); + sc = (struct ulptx_idata *)__skb_push(skb, immdlen); + if (sc) { + sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); + sc->len = htonl(0); + sc_memrd = (struct ulptx_sc_memrd *)(sc + 1); + sc_memrd->cmd_to_len = + htonl(ULPTX_CMD_V(ULP_TX_SC_MEMRD) | + ULP_TX_SC_MORE_V(1) | + ULPTX_LEN16_V(hws->keylen >> 4)); + sc_memrd->addr = htonl(kaddr); + } +} + +static u64 tlstx_incr_seqnum(struct chtls_hws *hws) +{ + return hws->tx_seq_no++; +} + +static bool is_sg_request(const struct sk_buff *skb) +{ + return skb->peeked || + (skb->len > MAX_IMM_ULPTX_WR_LEN); +} + +/* + * Returns true if an sk_buff carries urgent data. + */ +static bool skb_urgent(struct sk_buff *skb) +{ + return ULP_SKB_CB(skb)->flags & ULPCB_FLAG_URG; +} + +/* TLS content type for CPL SFO */ +static unsigned char tls_content_type(unsigned char content_type) +{ + switch (content_type) { + case TLS_HDR_TYPE_CCS: + return CPL_TX_TLS_SFO_TYPE_CCS; + case TLS_HDR_TYPE_ALERT: + return CPL_TX_TLS_SFO_TYPE_ALERT; + case TLS_HDR_TYPE_HANDSHAKE: + return CPL_TX_TLS_SFO_TYPE_HANDSHAKE; + case TLS_HDR_TYPE_HEARTBEAT: + return CPL_TX_TLS_SFO_TYPE_HEARTBEAT; + } + return CPL_TX_TLS_SFO_TYPE_DATA; +} + +static void tls_tx_data_wr(struct sock *sk, struct sk_buff *skb, + int dlen, int tls_immd, u32 credits, + int expn, int pdus) +{ + struct fw_tlstx_data_wr *req_wr; + struct cpl_tx_tls_sfo *req_cpl; + unsigned int wr_ulp_mode_force; + struct tls_scmd *updated_scmd; + unsigned char data_type; + struct chtls_sock *csk; + struct net_device *dev; + struct chtls_hws *hws; + struct tls_scmd *scmd; + struct adapter *adap; + unsigned char *req; + int immd_len; + int iv_imm; + int len; + + csk = rcu_dereference_sk_user_data(sk); + iv_imm = skb_ulp_tls_iv_imm(skb); + dev = csk->egress_dev; + adap = netdev2adap(dev); + hws = &csk->tlshws; + scmd = &hws->scmd; + len = dlen + expn; + + dlen = (dlen < hws->mfs) ? dlen : hws->mfs; + atomic_inc(&adap->chcr_stats.tls_pdu_tx); + + updated_scmd = scmd; + updated_scmd->seqno_numivs &= 0xffffff80; + updated_scmd->seqno_numivs |= SCMD_NUM_IVS_V(pdus); + hws->scmd = *updated_scmd; + + req = (unsigned char *)__skb_push(skb, sizeof(struct cpl_tx_tls_sfo)); + req_cpl = (struct cpl_tx_tls_sfo *)req; + req = (unsigned char *)__skb_push(skb, (sizeof(struct + fw_tlstx_data_wr))); + + req_wr = (struct fw_tlstx_data_wr *)req; + immd_len = (tls_immd ? dlen : 0); + req_wr->op_to_immdlen = + htonl(FW_WR_OP_V(FW_TLSTX_DATA_WR) | + FW_TLSTX_DATA_WR_COMPL_V(1) | + FW_TLSTX_DATA_WR_IMMDLEN_V(immd_len)); + req_wr->flowid_len16 = htonl(FW_TLSTX_DATA_WR_FLOWID_V(csk->tid) | + FW_TLSTX_DATA_WR_LEN16_V(credits)); + wr_ulp_mode_force = TX_ULP_MODE_V(ULP_MODE_TLS); + + if (is_sg_request(skb)) + wr_ulp_mode_force |= FW_OFLD_TX_DATA_WR_ALIGNPLD_F | + ((tcp_sk(sk)->nonagle & TCP_NAGLE_OFF) ? 0 : + FW_OFLD_TX_DATA_WR_SHOVE_F); + + req_wr->lsodisable_to_flags = + htonl(TX_ULP_MODE_V(ULP_MODE_TLS) | + TX_URG_V(skb_urgent(skb)) | + T6_TX_FORCE_F | wr_ulp_mode_force | + TX_SHOVE_V((!csk_flag(sk, CSK_TX_MORE_DATA)) && + skb_queue_empty(&csk->txq))); + + req_wr->ctxloc_to_exp = + htonl(FW_TLSTX_DATA_WR_NUMIVS_V(pdus) | + FW_TLSTX_DATA_WR_EXP_V(expn) | + FW_TLSTX_DATA_WR_CTXLOC_V(CHTLS_KEY_CONTEXT_DDR) | + FW_TLSTX_DATA_WR_IVDSGL_V(!iv_imm) | + FW_TLSTX_DATA_WR_KEYSIZE_V(hws->keylen >> 4)); + + /* Fill in the length */ + req_wr->plen = htonl(len); + req_wr->mfs = htons(hws->mfs); + req_wr->adjustedplen_pkd = + htons(FW_TLSTX_DATA_WR_ADJUSTEDPLEN_V(hws->adjustlen)); + req_wr->expinplenmax_pkd = + htons(FW_TLSTX_DATA_WR_EXPINPLENMAX_V(hws->expansion)); + req_wr->pdusinplenmax_pkd = + FW_TLSTX_DATA_WR_PDUSINPLENMAX_V(hws->pdus); + req_wr->r10 = 0; + + data_type = tls_content_type(ULP_SKB_CB(skb)->ulp.tls.type); + req_cpl->op_to_seg_len = htonl(CPL_TX_TLS_SFO_OPCODE_V(CPL_TX_TLS_SFO) | + CPL_TX_TLS_SFO_DATA_TYPE_V(data_type) | + CPL_TX_TLS_SFO_CPL_LEN_V(2) | + CPL_TX_TLS_SFO_SEG_LEN_V(dlen)); + req_cpl->pld_len = htonl(len - expn); + + req_cpl->type_protover = htonl(CPL_TX_TLS_SFO_TYPE_V + ((data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) ? + TLS_HDR_TYPE_HEARTBEAT : 0) | + CPL_TX_TLS_SFO_PROTOVER_V(0)); + + /* create the s-command */ + req_cpl->r1_lo = 0; + req_cpl->seqno_numivs = cpu_to_be32(hws->scmd.seqno_numivs); + req_cpl->ivgen_hdrlen = cpu_to_be32(hws->scmd.ivgen_hdrlen); + req_cpl->scmd1 = cpu_to_be64(tlstx_incr_seqnum(hws)); +} + +/* + * Calculate the TLS data expansion size + */ +static int chtls_expansion_size(struct sock *sk, int data_len, + int fullpdu, + unsigned short *pducnt) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_hws *hws = &csk->tlshws; + struct tls_scmd *scmd = &hws->scmd; + int fragsize = hws->mfs; + int expnsize = 0; + int fragleft; + int fragcnt; + int expppdu; + + if (SCMD_CIPH_MODE_G(scmd->seqno_numivs) == + SCMD_CIPH_MODE_AES_GCM) { + expppdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE + + TLS_HEADER_LENGTH; + + if (fullpdu) { + *pducnt = data_len / (expppdu + fragsize); + if (*pducnt > 32) + *pducnt = 32; + else if (!*pducnt) + *pducnt = 1; + expnsize = (*pducnt) * expppdu; + return expnsize; + } + fragcnt = (data_len / fragsize); + expnsize = fragcnt * expppdu; + fragleft = data_len % fragsize; + if (fragleft > 0) + expnsize += expppdu; + } + return expnsize; +} + +/* WR with IV, KEY and CPL SFO added */ +static void make_tlstx_data_wr(struct sock *sk, struct sk_buff *skb, + int tls_tx_imm, int tls_len, u32 credits) +{ + unsigned short pdus_per_ulp = 0; + struct chtls_sock *csk; + struct chtls_hws *hws; + int expn_sz; + int pdus; + + csk = rcu_dereference_sk_user_data(sk); + hws = &csk->tlshws; + pdus = DIV_ROUND_UP(tls_len, hws->mfs); + expn_sz = chtls_expansion_size(sk, tls_len, 0, NULL); + if (!hws->compute) { + hws->expansion = chtls_expansion_size(sk, + hws->fcplenmax, + 1, &pdus_per_ulp); + hws->pdus = pdus_per_ulp; + hws->adjustlen = hws->pdus * + ((hws->expansion / hws->pdus) + hws->mfs); + hws->compute = 1; + } + if (tls_copy_ivs(sk, skb)) + return; + tls_copy_tx_key(sk, skb); + tls_tx_data_wr(sk, skb, tls_len, tls_tx_imm, credits, expn_sz, pdus); + hws->tx_seq_no += (pdus - 1); +} + +static void make_tx_data_wr(struct sock *sk, struct sk_buff *skb, + unsigned int immdlen, int len, + u32 credits, u32 compl) +{ + struct fw_ofld_tx_data_wr *req; + unsigned int wr_ulp_mode_force; + struct chtls_sock *csk; + unsigned int opcode; + + csk = rcu_dereference_sk_user_data(sk); + opcode = FW_OFLD_TX_DATA_WR; + + req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, sizeof(*req)); + req->op_to_immdlen = htonl(WR_OP_V(opcode) | + FW_WR_COMPL_V(compl) | + FW_WR_IMMDLEN_V(immdlen)); + req->flowid_len16 = htonl(FW_WR_FLOWID_V(csk->tid) | + FW_WR_LEN16_V(credits)); + + wr_ulp_mode_force = TX_ULP_MODE_V(csk->ulp_mode); + if (is_sg_request(skb)) + wr_ulp_mode_force |= FW_OFLD_TX_DATA_WR_ALIGNPLD_F | + ((tcp_sk(sk)->nonagle & TCP_NAGLE_OFF) ? 0 : + FW_OFLD_TX_DATA_WR_SHOVE_F); + + req->tunnel_to_proxy = htonl(wr_ulp_mode_force | + TX_URG_V(skb_urgent(skb)) | + TX_SHOVE_V((!csk_flag(sk, CSK_TX_MORE_DATA)) && + skb_queue_empty(&csk->txq))); + req->plen = htonl(len); +} + +static int chtls_wr_size(struct chtls_sock *csk, const struct sk_buff *skb, + bool size) +{ + int wr_size; + + wr_size = TLS_WR_CPL_LEN; + wr_size += KEY_ON_MEM_SZ; + wr_size += ivs_size(csk->sk, skb); + + if (size) + return wr_size; + + /* frags counted for IV dsgl */ + if (!skb_ulp_tls_iv_imm(skb)) + skb_shinfo(skb)->nr_frags++; + + return wr_size; +} + +static bool is_ofld_imm(struct chtls_sock *csk, const struct sk_buff *skb) +{ + int length = skb->len; + + if (skb->peeked || skb->len > MAX_IMM_ULPTX_WR_LEN) + return false; + + if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) { + /* Check TLS header len for Immediate */ + if (csk->ulp_mode == ULP_MODE_TLS && + skb_ulp_tls_inline(skb)) + length += chtls_wr_size(csk, skb, true); + else + length += sizeof(struct fw_ofld_tx_data_wr); + + return length <= MAX_IMM_OFLD_TX_DATA_WR_LEN; + } + return true; +} + +static unsigned int calc_tx_flits(const struct sk_buff *skb, + unsigned int immdlen) +{ + unsigned int flits, cnt; + + flits = immdlen / 8; /* headers */ + cnt = skb_shinfo(skb)->nr_frags; + if (skb_tail_pointer(skb) != skb_transport_header(skb)) + cnt++; + return flits + sgl_len(cnt); +} + +static void arp_failure_discard(void *handle, struct sk_buff *skb) +{ + kfree_skb(skb); +} + +int chtls_push_frames(struct chtls_sock *csk, int comp) +{ + struct chtls_hws *hws = &csk->tlshws; + struct tcp_sock *tp; + struct sk_buff *skb; + int total_size = 0; + struct sock *sk; + int wr_size; + + wr_size = sizeof(struct fw_ofld_tx_data_wr); + sk = csk->sk; + tp = tcp_sk(sk); + + if (unlikely(sk_in_state(sk, TCPF_SYN_SENT | TCPF_CLOSE))) + return 0; + + if (unlikely(csk_flag(sk, CSK_ABORT_SHUTDOWN))) + return 0; + + while (csk->wr_credits && (skb = skb_peek(&csk->txq)) && + (!(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_HOLD) || + skb_queue_len(&csk->txq) > 1)) { + unsigned int credit_len = skb->len; + unsigned int credits_needed; + unsigned int completion = 0; + int tls_len = skb->len;/* TLS data len before IV/key */ + unsigned int immdlen; + int len = skb->len; /* length [ulp bytes] inserted by hw */ + int flowclen16 = 0; + int tls_tx_imm = 0; + + immdlen = skb->len; + if (!is_ofld_imm(csk, skb)) { + immdlen = skb_transport_offset(skb); + if (skb_ulp_tls_inline(skb)) + wr_size = chtls_wr_size(csk, skb, false); + credit_len = 8 * calc_tx_flits(skb, immdlen); + } else { + if (skb_ulp_tls_inline(skb)) { + wr_size = chtls_wr_size(csk, skb, false); + tls_tx_imm = 1; + } + } + if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) + credit_len += wr_size; + credits_needed = DIV_ROUND_UP(credit_len, 16); + if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) { + flowclen16 = send_tx_flowc_wr(sk, 1, tp->snd_nxt, + tp->rcv_nxt); + if (flowclen16 <= 0) + break; + csk->wr_credits -= flowclen16; + csk->wr_unacked += flowclen16; + csk->wr_nondata += flowclen16; + csk_set_flag(csk, CSK_TX_DATA_SENT); + } + + if (csk->wr_credits < credits_needed) { + if (skb_ulp_tls_inline(skb) && + !skb_ulp_tls_iv_imm(skb)) + skb_shinfo(skb)->nr_frags--; + break; + } + + __skb_unlink(skb, &csk->txq); + skb_set_queue_mapping(skb, (csk->txq_idx << 1) | + CPL_PRIORITY_DATA); + if (hws->ofld) + hws->txqid = (skb->queue_mapping >> 1); + skb->csum = (__force __wsum)(credits_needed + csk->wr_nondata); + csk->wr_credits -= credits_needed; + csk->wr_unacked += credits_needed; + csk->wr_nondata = 0; + enqueue_wr(csk, skb); + + if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) { + if ((comp && csk->wr_unacked == credits_needed) || + (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_COMPL) || + csk->wr_unacked >= csk->wr_max_credits / 2) { + completion = 1; + csk->wr_unacked = 0; + } + if (skb_ulp_tls_inline(skb)) + make_tlstx_data_wr(sk, skb, tls_tx_imm, + tls_len, credits_needed); + else + make_tx_data_wr(sk, skb, immdlen, len, + credits_needed, completion); + tp->snd_nxt += len; + tp->lsndtime = tcp_jiffies32; + if (completion) + ULP_SKB_CB(skb)->flags &= ~ULPCB_FLAG_NEED_HDR; + } else { + struct cpl_close_con_req *req = cplhdr(skb); + unsigned int cmd = CPL_OPCODE_G(ntohl + (OPCODE_TID(req))); + + if (cmd == CPL_CLOSE_CON_REQ) + csk_set_flag(csk, + CSK_CLOSE_CON_REQUESTED); + + if ((ULP_SKB_CB(skb)->flags & ULPCB_FLAG_COMPL) && + (csk->wr_unacked >= csk->wr_max_credits / 2)) { + req->wr.wr_hi |= htonl(FW_WR_COMPL_F); + csk->wr_unacked = 0; + } + } + total_size += skb->truesize; + if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_BARRIER) + csk_set_flag(csk, CSK_TX_WAIT_IDLE); + t4_set_arp_err_handler(skb, NULL, arp_failure_discard); + cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); + } + sk->sk_wmem_queued -= total_size; + return total_size; +} + +static void mark_urg(struct tcp_sock *tp, int flags, + struct sk_buff *skb) +{ + if (unlikely(flags & MSG_OOB)) { + tp->snd_up = tp->write_seq; + ULP_SKB_CB(skb)->flags = ULPCB_FLAG_URG | + ULPCB_FLAG_BARRIER | + ULPCB_FLAG_NO_APPEND | + ULPCB_FLAG_NEED_HDR; + } +} + +/* + * Returns true if a connection should send more data to TCP engine + */ +static bool should_push(struct sock *sk) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_dev *cdev = csk->cdev; + struct tcp_sock *tp = tcp_sk(sk); + + /* + * If we've released our offload resources there's nothing to do ... + */ + if (!cdev) + return false; + + /* + * If there aren't any work requests in flight, or there isn't enough + * data in flight, or Nagle is off then send the current TX_DATA + * otherwise hold it and wait to accumulate more data. + */ + return csk->wr_credits == csk->wr_max_credits || + (tp->nonagle & TCP_NAGLE_OFF); +} + +/* + * Returns true if a TCP socket is corked. + */ +static bool corked(const struct tcp_sock *tp, int flags) +{ + return (flags & MSG_MORE) || (tp->nonagle & TCP_NAGLE_CORK); +} + +/* + * Returns true if a send should try to push new data. + */ +static bool send_should_push(struct sock *sk, int flags) +{ + return should_push(sk) && !corked(tcp_sk(sk), flags); +} + +void chtls_tcp_push(struct sock *sk, int flags) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + int qlen = skb_queue_len(&csk->txq); + + if (likely(qlen)) { + struct sk_buff *skb = skb_peek_tail(&csk->txq); + struct tcp_sock *tp = tcp_sk(sk); + + mark_urg(tp, flags, skb); + + if (!(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) && + corked(tp, flags)) { + ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_HOLD; + return; + } + + ULP_SKB_CB(skb)->flags &= ~ULPCB_FLAG_HOLD; + if (qlen == 1 && + ((ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) || + should_push(sk))) + chtls_push_frames(csk, 1); + } +} + +/* + * Calculate the size for a new send sk_buff. It's maximum size so we can + * pack lots of data into it, unless we plan to send it immediately, in which + * case we size it more tightly. + * + * Note: we don't bother compensating for MSS < PAGE_SIZE because it doesn't + * arise in normal cases and when it does we are just wasting memory. + */ +static int select_size(struct sock *sk, int io_len, int flags, int len) +{ + const int pgbreak = SKB_MAX_HEAD(len); + + /* + * If the data wouldn't fit in the main body anyway, put only the + * header in the main body so it can use immediate data and place all + * the payload in page fragments. + */ + if (io_len > pgbreak) + return 0; + + /* + * If we will be accumulating payload get a large main body. + */ + if (!send_should_push(sk, flags)) + return pgbreak; + + return io_len; +} + +void skb_entail(struct sock *sk, struct sk_buff *skb, int flags) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct tcp_sock *tp = tcp_sk(sk); + + ULP_SKB_CB(skb)->seq = tp->write_seq; + ULP_SKB_CB(skb)->flags = flags; + __skb_queue_tail(&csk->txq, skb); + sk->sk_wmem_queued += skb->truesize; + + if (TCP_PAGE(sk) && TCP_OFF(sk)) { + put_page(TCP_PAGE(sk)); + TCP_PAGE(sk) = NULL; + TCP_OFF(sk) = 0; + } +} + +static struct sk_buff *get_tx_skb(struct sock *sk, int size) +{ + struct sk_buff *skb; + + skb = alloc_skb(size + TX_HEADER_LEN, sk->sk_allocation); + if (likely(skb)) { + skb_reserve(skb, TX_HEADER_LEN); + skb_entail(sk, skb, ULPCB_FLAG_NEED_HDR); + skb_reset_transport_header(skb); + } + return skb; +} + +static struct sk_buff *get_record_skb(struct sock *sk, int size, bool zcopy) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct sk_buff *skb; + + skb = alloc_skb(((zcopy ? 0 : size) + TX_TLSHDR_LEN + + KEY_ON_MEM_SZ + max_ivs_size(sk, size)), + sk->sk_allocation); + if (likely(skb)) { + skb_reserve(skb, (TX_TLSHDR_LEN + + KEY_ON_MEM_SZ + max_ivs_size(sk, size))); + skb_entail(sk, skb, ULPCB_FLAG_NEED_HDR); + skb_reset_transport_header(skb); + ULP_SKB_CB(skb)->ulp.tls.ofld = 1; + ULP_SKB_CB(skb)->ulp.tls.type = csk->tlshws.type; + } + return skb; +} + +static void tx_skb_finalize(struct sk_buff *skb) +{ + struct ulp_skb_cb *cb = ULP_SKB_CB(skb); + + if (!(cb->flags & ULPCB_FLAG_NO_HDR)) + cb->flags = ULPCB_FLAG_NEED_HDR; + cb->flags |= ULPCB_FLAG_NO_APPEND; +} + +static void push_frames_if_head(struct sock *sk) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + + if (skb_queue_len(&csk->txq) == 1) + chtls_push_frames(csk, 1); +} + +static int chtls_skb_copy_to_page_nocache(struct sock *sk, + struct iov_iter *from, + struct sk_buff *skb, + struct page *page, + int off, int copy) +{ + int err; + + err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + + off, copy, skb->len); + if (err) + return err; + + skb->len += copy; + skb->data_len += copy; + skb->truesize += copy; + sk->sk_wmem_queued += copy; + return 0; +} + +static bool csk_mem_free(struct chtls_dev *cdev, struct sock *sk) +{ + return (cdev->max_host_sndbuf - sk->sk_wmem_queued > 0); +} + +static int csk_wait_memory(struct chtls_dev *cdev, + struct sock *sk, long *timeo_p) +{ + DEFINE_WAIT_FUNC(wait, woken_wake_function); + int ret, err = 0; + long current_timeo; + long vm_wait = 0; + bool noblock; + + current_timeo = *timeo_p; + noblock = (*timeo_p ? false : true); + if (csk_mem_free(cdev, sk)) { + current_timeo = prandom_u32_max(HZ / 5) + 2; + vm_wait = prandom_u32_max(HZ / 5) + 2; + } + + add_wait_queue(sk_sleep(sk), &wait); + while (1) { + sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); + + if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) + goto do_error; + if (!*timeo_p) { + if (noblock) + set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); + goto do_nonblock; + } + if (signal_pending(current)) + goto do_interrupted; + sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); + if (csk_mem_free(cdev, sk) && !vm_wait) + break; + + set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); + sk->sk_write_pending++; + ret = sk_wait_event(sk, ¤t_timeo, sk->sk_err || + (sk->sk_shutdown & SEND_SHUTDOWN) || + (csk_mem_free(cdev, sk) && !vm_wait), + &wait); + sk->sk_write_pending--; + if (ret < 0) + goto do_error; + + if (vm_wait) { + vm_wait -= current_timeo; + current_timeo = *timeo_p; + if (current_timeo != MAX_SCHEDULE_TIMEOUT) { + current_timeo -= vm_wait; + if (current_timeo < 0) + current_timeo = 0; + } + vm_wait = 0; + } + *timeo_p = current_timeo; + } +do_rm_wq: + remove_wait_queue(sk_sleep(sk), &wait); + return err; +do_error: + err = -EPIPE; + goto do_rm_wq; +do_nonblock: + err = -EAGAIN; + goto do_rm_wq; +do_interrupted: + err = sock_intr_errno(*timeo_p); + goto do_rm_wq; +} + +static int chtls_proccess_cmsg(struct sock *sk, struct msghdr *msg, + unsigned char *record_type) +{ + struct cmsghdr *cmsg; + int rc = -EINVAL; + + for_each_cmsghdr(cmsg, msg) { + if (!CMSG_OK(msg, cmsg)) + return -EINVAL; + if (cmsg->cmsg_level != SOL_TLS) + continue; + + switch (cmsg->cmsg_type) { + case TLS_SET_RECORD_TYPE: + if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type))) + return -EINVAL; + + if (msg->msg_flags & MSG_MORE) + return -EINVAL; + + *record_type = *(unsigned char *)CMSG_DATA(cmsg); + rc = 0; + break; + default: + return -EINVAL; + } + } + + return rc; +} + +int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_dev *cdev = csk->cdev; + struct tcp_sock *tp = tcp_sk(sk); + struct sk_buff *skb; + int mss, flags, err; + int recordsz = 0; + int copied = 0; + long timeo; + + lock_sock(sk); + flags = msg->msg_flags; + timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); + + if (!sk_in_state(sk, TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { + err = sk_stream_wait_connect(sk, &timeo); + if (err) + goto out_err; + } + + sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); + err = -EPIPE; + if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) + goto out_err; + + mss = csk->mss; + csk_set_flag(csk, CSK_TX_MORE_DATA); + + while (msg_data_left(msg)) { + int copy = 0; + + skb = skb_peek_tail(&csk->txq); + if (skb) { + copy = mss - skb->len; + skb->ip_summed = CHECKSUM_UNNECESSARY; + } + if (!csk_mem_free(cdev, sk)) + goto wait_for_sndbuf; + + if (is_tls_tx(csk) && !csk->tlshws.txleft) { + unsigned char record_type = TLS_RECORD_TYPE_DATA; + + if (unlikely(msg->msg_controllen)) { + err = chtls_proccess_cmsg(sk, msg, + &record_type); + if (err) + goto out_err; + + /* Avoid appending tls handshake, alert to tls data */ + if (skb) + tx_skb_finalize(skb); + } + + recordsz = size; + csk->tlshws.txleft = recordsz; + csk->tlshws.type = record_type; + } + + if (!skb || (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) || + copy <= 0) { +new_buf: + if (skb) { + tx_skb_finalize(skb); + push_frames_if_head(sk); + } + + if (is_tls_tx(csk)) { + skb = get_record_skb(sk, + select_size(sk, + recordsz, + flags, + TX_TLSHDR_LEN), + false); + } else { + skb = get_tx_skb(sk, + select_size(sk, size, flags, + TX_HEADER_LEN)); + } + if (unlikely(!skb)) + goto wait_for_memory; + + skb->ip_summed = CHECKSUM_UNNECESSARY; + copy = mss; + } + if (copy > size) + copy = size; + + if (skb_tailroom(skb) > 0) { + copy = min(copy, skb_tailroom(skb)); + if (is_tls_tx(csk)) + copy = min_t(int, copy, csk->tlshws.txleft); + err = skb_add_data_nocache(sk, skb, + &msg->msg_iter, copy); + if (err) + goto do_fault; + } else { + int i = skb_shinfo(skb)->nr_frags; + struct page *page = TCP_PAGE(sk); + int pg_size = PAGE_SIZE; + int off = TCP_OFF(sk); + bool merge; + + if (page) + pg_size = page_size(page); + if (off < pg_size && + skb_can_coalesce(skb, i, page, off)) { + merge = true; + goto copy; + } + merge = false; + if (i == (is_tls_tx(csk) ? (MAX_SKB_FRAGS - 1) : + MAX_SKB_FRAGS)) + goto new_buf; + + if (page && off == pg_size) { + put_page(page); + TCP_PAGE(sk) = page = NULL; + pg_size = PAGE_SIZE; + } + + if (!page) { + gfp_t gfp = sk->sk_allocation; + int order = cdev->send_page_order; + + if (order) { + page = alloc_pages(gfp | __GFP_COMP | + __GFP_NOWARN | + __GFP_NORETRY, + order); + if (page) + pg_size <<= order; + } + if (!page) { + page = alloc_page(gfp); + pg_size = PAGE_SIZE; + } + if (!page) + goto wait_for_memory; + off = 0; + } +copy: + if (copy > pg_size - off) + copy = pg_size - off; + if (is_tls_tx(csk)) + copy = min_t(int, copy, csk->tlshws.txleft); + + err = chtls_skb_copy_to_page_nocache(sk, &msg->msg_iter, + skb, page, + off, copy); + if (unlikely(err)) { + if (!TCP_PAGE(sk)) { + TCP_PAGE(sk) = page; + TCP_OFF(sk) = 0; + } + goto do_fault; + } + /* Update the skb. */ + if (merge) { + skb_frag_size_add( + &skb_shinfo(skb)->frags[i - 1], + copy); + } else { + skb_fill_page_desc(skb, i, page, off, copy); + if (off + copy < pg_size) { + /* space left keep page */ + get_page(page); + TCP_PAGE(sk) = page; + } else { + TCP_PAGE(sk) = NULL; + } + } + TCP_OFF(sk) = off + copy; + } + if (unlikely(skb->len == mss)) + tx_skb_finalize(skb); + tp->write_seq += copy; + copied += copy; + size -= copy; + + if (is_tls_tx(csk)) + csk->tlshws.txleft -= copy; + + if (corked(tp, flags) && + (sk_stream_wspace(sk) < sk_stream_min_wspace(sk))) + ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_NO_APPEND; + + if (size == 0) + goto out; + + if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) + push_frames_if_head(sk); + continue; +wait_for_sndbuf: + set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); +wait_for_memory: + err = csk_wait_memory(cdev, sk, &timeo); + if (err) + goto do_error; + } +out: + csk_reset_flag(csk, CSK_TX_MORE_DATA); + if (copied) + chtls_tcp_push(sk, flags); +done: + release_sock(sk); + return copied; +do_fault: + if (!skb->len) { + __skb_unlink(skb, &csk->txq); + sk->sk_wmem_queued -= skb->truesize; + __kfree_skb(skb); + } +do_error: + if (copied) + goto out; +out_err: + if (csk_conn_inline(csk)) + csk_reset_flag(csk, CSK_TX_MORE_DATA); + copied = sk_stream_error(sk, flags, err); + goto done; +} + +int chtls_sendpage(struct sock *sk, struct page *page, + int offset, size_t size, int flags) +{ + struct chtls_sock *csk; + struct chtls_dev *cdev; + int mss, err, copied; + struct tcp_sock *tp; + long timeo; + + tp = tcp_sk(sk); + copied = 0; + csk = rcu_dereference_sk_user_data(sk); + cdev = csk->cdev; + lock_sock(sk); + timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); + + err = sk_stream_wait_connect(sk, &timeo); + if (!sk_in_state(sk, TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) && + err != 0) + goto out_err; + + mss = csk->mss; + csk_set_flag(csk, CSK_TX_MORE_DATA); + + while (size > 0) { + struct sk_buff *skb = skb_peek_tail(&csk->txq); + int copy, i; + + if (!skb || (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) || + (copy = mss - skb->len) <= 0) { +new_buf: + if (!csk_mem_free(cdev, sk)) + goto wait_for_sndbuf; + + if (is_tls_tx(csk)) { + skb = get_record_skb(sk, + select_size(sk, size, + flags, + TX_TLSHDR_LEN), + true); + } else { + skb = get_tx_skb(sk, 0); + } + if (!skb) + goto wait_for_memory; + copy = mss; + } + if (copy > size) + copy = size; + + i = skb_shinfo(skb)->nr_frags; + if (skb_can_coalesce(skb, i, page, offset)) { + skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); + } else if (i < MAX_SKB_FRAGS) { + get_page(page); + skb_fill_page_desc(skb, i, page, offset, copy); + } else { + tx_skb_finalize(skb); + push_frames_if_head(sk); + goto new_buf; + } + + skb->len += copy; + if (skb->len == mss) + tx_skb_finalize(skb); + skb->data_len += copy; + skb->truesize += copy; + sk->sk_wmem_queued += copy; + tp->write_seq += copy; + copied += copy; + offset += copy; + size -= copy; + + if (corked(tp, flags) && + (sk_stream_wspace(sk) < sk_stream_min_wspace(sk))) + ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_NO_APPEND; + + if (!size) + break; + + if (unlikely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND)) + push_frames_if_head(sk); + continue; +wait_for_sndbuf: + set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); +wait_for_memory: + err = csk_wait_memory(cdev, sk, &timeo); + if (err) + goto do_error; + } +out: + csk_reset_flag(csk, CSK_TX_MORE_DATA); + if (copied) + chtls_tcp_push(sk, flags); +done: + release_sock(sk); + return copied; + +do_error: + if (copied) + goto out; + +out_err: + if (csk_conn_inline(csk)) + csk_reset_flag(csk, CSK_TX_MORE_DATA); + copied = sk_stream_error(sk, flags, err); + goto done; +} + +static void chtls_select_window(struct sock *sk) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct tcp_sock *tp = tcp_sk(sk); + unsigned int wnd = tp->rcv_wnd; + + wnd = max_t(unsigned int, wnd, tcp_full_space(sk)); + wnd = max_t(unsigned int, MIN_RCV_WND, wnd); + + if (wnd > MAX_RCV_WND) + wnd = MAX_RCV_WND; + +/* + * Check if we need to grow the receive window in response to an increase in + * the socket's receive buffer size. Some applications increase the buffer + * size dynamically and rely on the window to grow accordingly. + */ + + if (wnd > tp->rcv_wnd) { + tp->rcv_wup -= wnd - tp->rcv_wnd; + tp->rcv_wnd = wnd; + /* Mark the receive window as updated */ + csk_reset_flag(csk, CSK_UPDATE_RCV_WND); + } +} + +/* + * Send RX credits through an RX_DATA_ACK CPL message. We are permitted + * to return without sending the message in case we cannot allocate + * an sk_buff. Returns the number of credits sent. + */ +static u32 send_rx_credits(struct chtls_sock *csk, u32 credits) +{ + struct cpl_rx_data_ack *req; + struct sk_buff *skb; + + skb = alloc_skb(sizeof(*req), GFP_ATOMIC); + if (!skb) + return 0; + __skb_put(skb, sizeof(*req)); + req = (struct cpl_rx_data_ack *)skb->head; + + set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id); + INIT_TP_WR(req, csk->tid); + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, + csk->tid)); + req->credit_dack = cpu_to_be32(RX_CREDITS_V(credits) | + RX_FORCE_ACK_F); + cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); + return credits; +} + +#define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | \ + TCPF_FIN_WAIT1 | \ + TCPF_FIN_WAIT2) + +/* + * Called after some received data has been read. It returns RX credits + * to the HW for the amount of data processed. + */ +static void chtls_cleanup_rbuf(struct sock *sk, int copied) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct tcp_sock *tp; + int must_send; + u32 credits; + u32 thres; + + thres = 15 * 1024; + + if (!sk_in_state(sk, CREDIT_RETURN_STATE)) + return; + + chtls_select_window(sk); + tp = tcp_sk(sk); + credits = tp->copied_seq - tp->rcv_wup; + if (unlikely(!credits)) + return; + +/* + * For coalescing to work effectively ensure the receive window has + * at least 16KB left. + */ + must_send = credits + 16384 >= tp->rcv_wnd; + + if (must_send || credits >= thres) + tp->rcv_wup += send_rx_credits(csk, credits); +} + +static int chtls_pt_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, + int flags, int *addr_len) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct chtls_hws *hws = &csk->tlshws; + struct net_device *dev = csk->egress_dev; + struct adapter *adap = netdev2adap(dev); + struct tcp_sock *tp = tcp_sk(sk); + unsigned long avail; + int buffers_freed; + int copied = 0; + int target; + long timeo; + int ret; + + buffers_freed = 0; + + timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); + target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); + + if (unlikely(csk_flag(sk, CSK_UPDATE_RCV_WND))) + chtls_cleanup_rbuf(sk, copied); + + do { + struct sk_buff *skb; + u32 offset = 0; + + if (unlikely(tp->urg_data && + tp->urg_seq == tp->copied_seq)) { + if (copied) + break; + if (signal_pending(current)) { + copied = timeo ? sock_intr_errno(timeo) : + -EAGAIN; + break; + } + } + skb = skb_peek(&sk->sk_receive_queue); + if (skb) + goto found_ok_skb; + if (csk->wr_credits && + skb_queue_len(&csk->txq) && + chtls_push_frames(csk, csk->wr_credits == + csk->wr_max_credits)) + sk->sk_write_space(sk); + + if (copied >= target && !READ_ONCE(sk->sk_backlog.tail)) + break; + + if (copied) { + if (sk->sk_err || sk->sk_state == TCP_CLOSE || + (sk->sk_shutdown & RCV_SHUTDOWN) || + signal_pending(current)) + break; + + if (!timeo) + break; + } else { + if (sock_flag(sk, SOCK_DONE)) + break; + if (sk->sk_err) { + copied = sock_error(sk); + break; + } + if (sk->sk_shutdown & RCV_SHUTDOWN) + break; + if (sk->sk_state == TCP_CLOSE) { + copied = -ENOTCONN; + break; + } + if (!timeo) { + copied = -EAGAIN; + break; + } + if (signal_pending(current)) { + copied = sock_intr_errno(timeo); + break; + } + } + if (READ_ONCE(sk->sk_backlog.tail)) { + release_sock(sk); + lock_sock(sk); + chtls_cleanup_rbuf(sk, copied); + continue; + } + + if (copied >= target) + break; + chtls_cleanup_rbuf(sk, copied); + ret = sk_wait_data(sk, &timeo, NULL); + if (ret < 0) { + copied = copied ? : ret; + goto unlock; + } + continue; +found_ok_skb: + if (!skb->len) { + skb_dst_set(skb, NULL); + __skb_unlink(skb, &sk->sk_receive_queue); + kfree_skb(skb); + + if (!copied && !timeo) { + copied = -EAGAIN; + break; + } + + if (copied < target) { + release_sock(sk); + lock_sock(sk); + continue; + } + break; + } + offset = hws->copied_seq; + avail = skb->len - offset; + if (len < avail) + avail = len; + + if (unlikely(tp->urg_data)) { + u32 urg_offset = tp->urg_seq - tp->copied_seq; + + if (urg_offset < avail) { + if (urg_offset) { + avail = urg_offset; + } else if (!sock_flag(sk, SOCK_URGINLINE)) { + /* First byte is urgent, skip */ + tp->copied_seq++; + offset++; + avail--; + if (!avail) + goto skip_copy; + } + } + } + /* Set record type if not already done. For a non-data record, + * do not proceed if record type could not be copied. + */ + if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_TLS_HDR) { + struct tls_hdr *thdr = (struct tls_hdr *)skb->data; + int cerr = 0; + + cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE, + sizeof(thdr->type), &thdr->type); + + if (cerr && thdr->type != TLS_RECORD_TYPE_DATA) { + copied = -EIO; + break; + } + /* don't send tls header, skip copy */ + goto skip_copy; + } + + if (skb_copy_datagram_msg(skb, offset, msg, avail)) { + if (!copied) { + copied = -EFAULT; + break; + } + } + + copied += avail; + len -= avail; + hws->copied_seq += avail; +skip_copy: + if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) + tp->urg_data = 0; + + if ((avail + offset) >= skb->len) { + struct sk_buff *next_skb; + if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_TLS_HDR) { + tp->copied_seq += skb->len; + hws->rcvpld = skb->hdr_len; + } else { + atomic_inc(&adap->chcr_stats.tls_pdu_rx); + tp->copied_seq += hws->rcvpld; + } + chtls_free_skb(sk, skb); + buffers_freed++; + hws->copied_seq = 0; + next_skb = skb_peek(&sk->sk_receive_queue); + if (copied >= target && !next_skb) + break; + if (ULP_SKB_CB(next_skb)->flags & ULPCB_FLAG_TLS_HDR) + break; + } + } while (len > 0); + + if (buffers_freed) + chtls_cleanup_rbuf(sk, copied); + +unlock: + release_sock(sk); + return copied; +} + +/* + * Peek at data in a socket's receive buffer. + */ +static int peekmsg(struct sock *sk, struct msghdr *msg, + size_t len, int flags) +{ + struct tcp_sock *tp = tcp_sk(sk); + u32 peek_seq, offset; + struct sk_buff *skb; + int copied = 0; + size_t avail; /* amount of available data in current skb */ + long timeo; + int ret; + + lock_sock(sk); + timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); + peek_seq = tp->copied_seq; + + do { + if (unlikely(tp->urg_data && tp->urg_seq == peek_seq)) { + if (copied) + break; + if (signal_pending(current)) { + copied = timeo ? sock_intr_errno(timeo) : + -EAGAIN; + break; + } + } + + skb_queue_walk(&sk->sk_receive_queue, skb) { + offset = peek_seq - ULP_SKB_CB(skb)->seq; + if (offset < skb->len) + goto found_ok_skb; + } + + /* empty receive queue */ + if (copied) + break; + if (sock_flag(sk, SOCK_DONE)) + break; + if (sk->sk_err) { + copied = sock_error(sk); + break; + } + if (sk->sk_shutdown & RCV_SHUTDOWN) + break; + if (sk->sk_state == TCP_CLOSE) { + copied = -ENOTCONN; + break; + } + if (!timeo) { + copied = -EAGAIN; + break; + } + if (signal_pending(current)) { + copied = sock_intr_errno(timeo); + break; + } + + if (READ_ONCE(sk->sk_backlog.tail)) { + /* Do not sleep, just process backlog. */ + release_sock(sk); + lock_sock(sk); + } else { + ret = sk_wait_data(sk, &timeo, NULL); + if (ret < 0) { + /* here 'copied' is 0 due to previous checks */ + copied = ret; + break; + } + } + + if (unlikely(peek_seq != tp->copied_seq)) { + if (net_ratelimit()) + pr_info("TCP(%s:%d), race in MSG_PEEK.\n", + current->comm, current->pid); + peek_seq = tp->copied_seq; + } + continue; + +found_ok_skb: + avail = skb->len - offset; + if (len < avail) + avail = len; + /* + * Do we have urgent data here? We need to skip over the + * urgent byte. + */ + if (unlikely(tp->urg_data)) { + u32 urg_offset = tp->urg_seq - peek_seq; + + if (urg_offset < avail) { + /* + * The amount of data we are preparing to copy + * contains urgent data. + */ + if (!urg_offset) { /* First byte is urgent */ + if (!sock_flag(sk, SOCK_URGINLINE)) { + peek_seq++; + offset++; + avail--; + } + if (!avail) + continue; + } else { + /* stop short of the urgent data */ + avail = urg_offset; + } + } + } + + /* + * If MSG_TRUNC is specified the data is discarded. + */ + if (likely(!(flags & MSG_TRUNC))) + if (skb_copy_datagram_msg(skb, offset, msg, len)) { + if (!copied) { + copied = -EFAULT; + break; + } + } + peek_seq += avail; + copied += avail; + len -= avail; + } while (len > 0); + + release_sock(sk); + return copied; +} + +int chtls_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, + int flags, int *addr_len) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct chtls_sock *csk; + unsigned long avail; /* amount of available data in current skb */ + int buffers_freed; + int copied = 0; + long timeo; + int target; /* Read at least this many bytes */ + int ret; + + buffers_freed = 0; + + if (unlikely(flags & MSG_OOB)) + return tcp_prot.recvmsg(sk, msg, len, flags, addr_len); + + if (unlikely(flags & MSG_PEEK)) + return peekmsg(sk, msg, len, flags); + + if (sk_can_busy_loop(sk) && + skb_queue_empty_lockless(&sk->sk_receive_queue) && + sk->sk_state == TCP_ESTABLISHED) + sk_busy_loop(sk, flags & MSG_DONTWAIT); + + lock_sock(sk); + csk = rcu_dereference_sk_user_data(sk); + + if (is_tls_rx(csk)) + return chtls_pt_recvmsg(sk, msg, len, flags, addr_len); + + timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); + target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); + + if (unlikely(csk_flag(sk, CSK_UPDATE_RCV_WND))) + chtls_cleanup_rbuf(sk, copied); + + do { + struct sk_buff *skb; + u32 offset; + + if (unlikely(tp->urg_data && tp->urg_seq == tp->copied_seq)) { + if (copied) + break; + if (signal_pending(current)) { + copied = timeo ? sock_intr_errno(timeo) : + -EAGAIN; + break; + } + } + + skb = skb_peek(&sk->sk_receive_queue); + if (skb) + goto found_ok_skb; + + if (csk->wr_credits && + skb_queue_len(&csk->txq) && + chtls_push_frames(csk, csk->wr_credits == + csk->wr_max_credits)) + sk->sk_write_space(sk); + + if (copied >= target && !READ_ONCE(sk->sk_backlog.tail)) + break; + + if (copied) { + if (sk->sk_err || sk->sk_state == TCP_CLOSE || + (sk->sk_shutdown & RCV_SHUTDOWN) || + signal_pending(current)) + break; + } else { + if (sock_flag(sk, SOCK_DONE)) + break; + if (sk->sk_err) { + copied = sock_error(sk); + break; + } + if (sk->sk_shutdown & RCV_SHUTDOWN) + break; + if (sk->sk_state == TCP_CLOSE) { + copied = -ENOTCONN; + break; + } + if (!timeo) { + copied = -EAGAIN; + break; + } + if (signal_pending(current)) { + copied = sock_intr_errno(timeo); + break; + } + } + + if (READ_ONCE(sk->sk_backlog.tail)) { + release_sock(sk); + lock_sock(sk); + chtls_cleanup_rbuf(sk, copied); + continue; + } + + if (copied >= target) + break; + chtls_cleanup_rbuf(sk, copied); + ret = sk_wait_data(sk, &timeo, NULL); + if (ret < 0) { + copied = copied ? : ret; + goto unlock; + } + continue; + +found_ok_skb: + if (!skb->len) { + chtls_kfree_skb(sk, skb); + if (!copied && !timeo) { + copied = -EAGAIN; + break; + } + + if (copied < target) + continue; + + break; + } + + offset = tp->copied_seq - ULP_SKB_CB(skb)->seq; + avail = skb->len - offset; + if (len < avail) + avail = len; + + if (unlikely(tp->urg_data)) { + u32 urg_offset = tp->urg_seq - tp->copied_seq; + + if (urg_offset < avail) { + if (urg_offset) { + avail = urg_offset; + } else if (!sock_flag(sk, SOCK_URGINLINE)) { + tp->copied_seq++; + offset++; + avail--; + if (!avail) + goto skip_copy; + } + } + } + + if (likely(!(flags & MSG_TRUNC))) { + if (skb_copy_datagram_msg(skb, offset, + msg, avail)) { + if (!copied) { + copied = -EFAULT; + break; + } + } + } + + tp->copied_seq += avail; + copied += avail; + len -= avail; + +skip_copy: + if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) + tp->urg_data = 0; + + if (avail + offset >= skb->len) { + chtls_free_skb(sk, skb); + buffers_freed++; + + if (copied >= target && + !skb_peek(&sk->sk_receive_queue)) + break; + } + } while (len > 0); + + if (buffers_freed) + chtls_cleanup_rbuf(sk, copied); + +unlock: + release_sock(sk); + return copied; +} diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c new file mode 100644 index 000000000..1e55b12fe --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c @@ -0,0 +1,641 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018 Chelsio Communications, Inc. + * + * Written by: Atul Gupta (atul.gupta@chelsio.com) + */ +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/skbuff.h> +#include <linux/socket.h> +#include <linux/hash.h> +#include <linux/in.h> +#include <linux/net.h> +#include <linux/ip.h> +#include <linux/tcp.h> +#include <net/ipv6.h> +#include <net/transp_v6.h> +#include <net/tcp.h> +#include <net/tls.h> + +#include "chtls.h" +#include "chtls_cm.h" + +#define DRV_NAME "chtls" + +/* + * chtls device management + * maintains a list of the chtls devices + */ +static LIST_HEAD(cdev_list); +static DEFINE_MUTEX(cdev_mutex); + +static DEFINE_MUTEX(notify_mutex); +static RAW_NOTIFIER_HEAD(listen_notify_list); +static struct proto chtls_cpl_prot, chtls_cpl_protv6; +struct request_sock_ops chtls_rsk_ops, chtls_rsk_opsv6; +static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT; + +static void register_listen_notifier(struct notifier_block *nb) +{ + mutex_lock(¬ify_mutex); + raw_notifier_chain_register(&listen_notify_list, nb); + mutex_unlock(¬ify_mutex); +} + +static void unregister_listen_notifier(struct notifier_block *nb) +{ + mutex_lock(¬ify_mutex); + raw_notifier_chain_unregister(&listen_notify_list, nb); + mutex_unlock(¬ify_mutex); +} + +static int listen_notify_handler(struct notifier_block *this, + unsigned long event, void *data) +{ + struct chtls_listen *clisten; + int ret = NOTIFY_DONE; + + clisten = (struct chtls_listen *)data; + + switch (event) { + case CHTLS_LISTEN_START: + ret = chtls_listen_start(clisten->cdev, clisten->sk); + kfree(clisten); + break; + case CHTLS_LISTEN_STOP: + chtls_listen_stop(clisten->cdev, clisten->sk); + kfree(clisten); + break; + } + return ret; +} + +static struct notifier_block listen_notifier = { + .notifier_call = listen_notify_handler +}; + +static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb) +{ + if (likely(skb_transport_header(skb) != skb_network_header(skb))) + return tcp_v4_do_rcv(sk, skb); + BLOG_SKB_CB(skb)->backlog_rcv(sk, skb); + return 0; +} + +static int chtls_start_listen(struct chtls_dev *cdev, struct sock *sk) +{ + struct chtls_listen *clisten; + + if (sk->sk_protocol != IPPROTO_TCP) + return -EPROTONOSUPPORT; + + if (sk->sk_family == PF_INET && + LOOPBACK(inet_sk(sk)->inet_rcv_saddr)) + return -EADDRNOTAVAIL; + + sk->sk_backlog_rcv = listen_backlog_rcv; + clisten = kmalloc(sizeof(*clisten), GFP_KERNEL); + if (!clisten) + return -ENOMEM; + clisten->cdev = cdev; + clisten->sk = sk; + mutex_lock(¬ify_mutex); + raw_notifier_call_chain(&listen_notify_list, + CHTLS_LISTEN_START, clisten); + mutex_unlock(¬ify_mutex); + return 0; +} + +static void chtls_stop_listen(struct chtls_dev *cdev, struct sock *sk) +{ + struct chtls_listen *clisten; + + if (sk->sk_protocol != IPPROTO_TCP) + return; + + clisten = kmalloc(sizeof(*clisten), GFP_KERNEL); + if (!clisten) + return; + clisten->cdev = cdev; + clisten->sk = sk; + mutex_lock(¬ify_mutex); + raw_notifier_call_chain(&listen_notify_list, + CHTLS_LISTEN_STOP, clisten); + mutex_unlock(¬ify_mutex); +} + +static int chtls_inline_feature(struct tls_toe_device *dev) +{ + struct net_device *netdev; + struct chtls_dev *cdev; + int i; + + cdev = to_chtls_dev(dev); + + for (i = 0; i < cdev->lldi->nports; i++) { + netdev = cdev->ports[i]; + if (netdev->features & NETIF_F_HW_TLS_RECORD) + return 1; + } + return 0; +} + +static int chtls_create_hash(struct tls_toe_device *dev, struct sock *sk) +{ + struct chtls_dev *cdev = to_chtls_dev(dev); + + if (sk->sk_state == TCP_LISTEN) + return chtls_start_listen(cdev, sk); + return 0; +} + +static void chtls_destroy_hash(struct tls_toe_device *dev, struct sock *sk) +{ + struct chtls_dev *cdev = to_chtls_dev(dev); + + if (sk->sk_state == TCP_LISTEN) + chtls_stop_listen(cdev, sk); +} + +static void chtls_free_uld(struct chtls_dev *cdev) +{ + int i; + + tls_toe_unregister_device(&cdev->tlsdev); + kvfree(cdev->kmap.addr); + idr_destroy(&cdev->hwtid_idr); + for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) + kfree_skb(cdev->rspq_skb_cache[i]); + kfree(cdev->lldi); + kfree_skb(cdev->askb); + kfree(cdev); +} + +static inline void chtls_dev_release(struct kref *kref) +{ + struct tls_toe_device *dev; + struct chtls_dev *cdev; + struct adapter *adap; + + dev = container_of(kref, struct tls_toe_device, kref); + cdev = to_chtls_dev(dev); + + /* Reset tls rx/tx stats */ + adap = pci_get_drvdata(cdev->pdev); + atomic_set(&adap->chcr_stats.tls_pdu_tx, 0); + atomic_set(&adap->chcr_stats.tls_pdu_rx, 0); + + chtls_free_uld(cdev); +} + +static void chtls_register_dev(struct chtls_dev *cdev) +{ + struct tls_toe_device *tlsdev = &cdev->tlsdev; + + strscpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX); + strlcat(tlsdev->name, cdev->lldi->ports[0]->name, + TLS_TOE_DEVICE_NAME_MAX); + tlsdev->feature = chtls_inline_feature; + tlsdev->hash = chtls_create_hash; + tlsdev->unhash = chtls_destroy_hash; + tlsdev->release = chtls_dev_release; + kref_init(&tlsdev->kref); + tls_toe_register_device(tlsdev); + cdev->cdev_state = CHTLS_CDEV_STATE_UP; +} + +static void process_deferq(struct work_struct *task_param) +{ + struct chtls_dev *cdev = container_of(task_param, + struct chtls_dev, deferq_task); + struct sk_buff *skb; + + spin_lock_bh(&cdev->deferq.lock); + while ((skb = __skb_dequeue(&cdev->deferq)) != NULL) { + spin_unlock_bh(&cdev->deferq.lock); + DEFERRED_SKB_CB(skb)->handler(cdev, skb); + spin_lock_bh(&cdev->deferq.lock); + } + spin_unlock_bh(&cdev->deferq.lock); +} + +static int chtls_get_skb(struct chtls_dev *cdev) +{ + cdev->askb = alloc_skb(sizeof(struct tcphdr), GFP_KERNEL); + if (!cdev->askb) + return -ENOMEM; + + skb_put(cdev->askb, sizeof(struct tcphdr)); + skb_reset_transport_header(cdev->askb); + memset(cdev->askb->data, 0, cdev->askb->len); + return 0; +} + +static void *chtls_uld_add(const struct cxgb4_lld_info *info) +{ + struct cxgb4_lld_info *lldi; + struct chtls_dev *cdev; + int i, j; + + cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); + if (!cdev) + goto out; + + lldi = kzalloc(sizeof(*lldi), GFP_KERNEL); + if (!lldi) + goto out_lldi; + + if (chtls_get_skb(cdev)) + goto out_skb; + + *lldi = *info; + cdev->lldi = lldi; + cdev->pdev = lldi->pdev; + cdev->tids = lldi->tids; + cdev->ports = lldi->ports; + cdev->mtus = lldi->mtus; + cdev->tids = lldi->tids; + cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0])) + << FW_VIID_PFN_S; + + for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) { + unsigned int size = 64 - sizeof(struct rsp_ctrl) - 8; + + cdev->rspq_skb_cache[i] = __alloc_skb(size, + gfp_any(), 0, + lldi->nodeid); + if (unlikely(!cdev->rspq_skb_cache[i])) + goto out_rspq_skb; + } + + idr_init(&cdev->hwtid_idr); + INIT_WORK(&cdev->deferq_task, process_deferq); + spin_lock_init(&cdev->listen_lock); + spin_lock_init(&cdev->idr_lock); + cdev->send_page_order = min_t(uint, get_order(32768), + send_page_order); + cdev->max_host_sndbuf = 48 * 1024; + + if (lldi->vr->key.size) + if (chtls_init_kmap(cdev, lldi)) + goto out_rspq_skb; + + mutex_lock(&cdev_mutex); + list_add_tail(&cdev->list, &cdev_list); + mutex_unlock(&cdev_mutex); + + return cdev; +out_rspq_skb: + for (j = 0; j < i; j++) + kfree_skb(cdev->rspq_skb_cache[j]); + kfree_skb(cdev->askb); +out_skb: + kfree(lldi); +out_lldi: + kfree(cdev); +out: + return NULL; +} + +static void chtls_free_all_uld(void) +{ + struct chtls_dev *cdev, *tmp; + + mutex_lock(&cdev_mutex); + list_for_each_entry_safe(cdev, tmp, &cdev_list, list) { + if (cdev->cdev_state == CHTLS_CDEV_STATE_UP) { + list_del(&cdev->list); + kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release); + } + } + mutex_unlock(&cdev_mutex); +} + +static int chtls_uld_state_change(void *handle, enum cxgb4_state new_state) +{ + struct chtls_dev *cdev = handle; + + switch (new_state) { + case CXGB4_STATE_UP: + chtls_register_dev(cdev); + break; + case CXGB4_STATE_DOWN: + break; + case CXGB4_STATE_START_RECOVERY: + break; + case CXGB4_STATE_DETACH: + mutex_lock(&cdev_mutex); + list_del(&cdev->list); + mutex_unlock(&cdev_mutex); + kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release); + break; + default: + break; + } + return 0; +} + +static struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl, + const __be64 *rsp, + u32 pktshift) +{ + struct sk_buff *skb; + + /* Allocate space for cpl_pass_accpet_req which will be synthesized by + * driver. Once driver synthesizes cpl_pass_accpet_req the skb will go + * through the regular cpl_pass_accept_req processing in TOM. + */ + skb = alloc_skb(gl->tot_len + sizeof(struct cpl_pass_accept_req) + - pktshift, GFP_ATOMIC); + if (unlikely(!skb)) + return NULL; + __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req) + - pktshift); + /* For now we will copy cpl_rx_pkt in the skb */ + skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_rx_pkt)); + skb_copy_to_linear_data_offset(skb, sizeof(struct cpl_pass_accept_req) + , gl->va + pktshift, + gl->tot_len - pktshift); + + return skb; +} + +static int chtls_recv_packet(struct chtls_dev *cdev, + const struct pkt_gl *gl, const __be64 *rsp) +{ + unsigned int opcode = *(u8 *)rsp; + struct sk_buff *skb; + int ret; + + skb = copy_gl_to_skb_pkt(gl, rsp, cdev->lldi->sge_pktshift); + if (!skb) + return -ENOMEM; + + ret = chtls_handlers[opcode](cdev, skb); + if (ret & CPL_RET_BUF_DONE) + kfree_skb(skb); + + return 0; +} + +static int chtls_recv_rsp(struct chtls_dev *cdev, const __be64 *rsp) +{ + unsigned long rspq_bin; + unsigned int opcode; + struct sk_buff *skb; + unsigned int len; + int ret; + + len = 64 - sizeof(struct rsp_ctrl) - 8; + opcode = *(u8 *)rsp; + + rspq_bin = hash_ptr((void *)rsp, RSPQ_HASH_BITS); + skb = cdev->rspq_skb_cache[rspq_bin]; + if (skb && !skb_is_nonlinear(skb) && + !skb_shared(skb) && !skb_cloned(skb)) { + refcount_inc(&skb->users); + if (refcount_read(&skb->users) == 2) { + __skb_trim(skb, 0); + if (skb_tailroom(skb) >= len) + goto copy_out; + } + refcount_dec(&skb->users); + } + skb = alloc_skb(len, GFP_ATOMIC); + if (unlikely(!skb)) + return -ENOMEM; + +copy_out: + __skb_put(skb, len); + skb_copy_to_linear_data(skb, rsp, len); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + ret = chtls_handlers[opcode](cdev, skb); + + if (ret & CPL_RET_BUF_DONE) + kfree_skb(skb); + return 0; +} + +static void chtls_recv(struct chtls_dev *cdev, + struct sk_buff **skbs, const __be64 *rsp) +{ + struct sk_buff *skb = *skbs; + unsigned int opcode; + int ret; + + opcode = *(u8 *)rsp; + + __skb_push(skb, sizeof(struct rss_header)); + skb_copy_to_linear_data(skb, rsp, sizeof(struct rss_header)); + + ret = chtls_handlers[opcode](cdev, skb); + if (ret & CPL_RET_BUF_DONE) + kfree_skb(skb); +} + +static int chtls_uld_rx_handler(void *handle, const __be64 *rsp, + const struct pkt_gl *gl) +{ + struct chtls_dev *cdev = handle; + unsigned int opcode; + struct sk_buff *skb; + + opcode = *(u8 *)rsp; + + if (unlikely(opcode == CPL_RX_PKT)) { + if (chtls_recv_packet(cdev, gl, rsp) < 0) + goto nomem; + return 0; + } + + if (!gl) + return chtls_recv_rsp(cdev, rsp); + +#define RX_PULL_LEN 128 + skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN); + if (unlikely(!skb)) + goto nomem; + chtls_recv(cdev, &skb, rsp); + return 0; + +nomem: + return -ENOMEM; +} + +static int do_chtls_getsockopt(struct sock *sk, char __user *optval, + int __user *optlen) +{ + struct tls_crypto_info crypto_info = { 0 }; + + crypto_info.version = TLS_1_2_VERSION; + if (copy_to_user(optval, &crypto_info, sizeof(struct tls_crypto_info))) + return -EFAULT; + return 0; +} + +static int chtls_getsockopt(struct sock *sk, int level, int optname, + char __user *optval, int __user *optlen) +{ + struct tls_context *ctx = tls_get_ctx(sk); + + if (level != SOL_TLS) + return ctx->sk_proto->getsockopt(sk, level, + optname, optval, optlen); + + return do_chtls_getsockopt(sk, optval, optlen); +} + +static int do_chtls_setsockopt(struct sock *sk, int optname, + sockptr_t optval, unsigned int optlen) +{ + struct tls_crypto_info *crypto_info, tmp_crypto_info; + struct chtls_sock *csk; + int keylen; + int cipher_type; + int rc = 0; + + csk = rcu_dereference_sk_user_data(sk); + + if (sockptr_is_null(optval) || optlen < sizeof(*crypto_info)) { + rc = -EINVAL; + goto out; + } + + rc = copy_from_sockptr(&tmp_crypto_info, optval, sizeof(*crypto_info)); + if (rc) { + rc = -EFAULT; + goto out; + } + + /* check version */ + if (tmp_crypto_info.version != TLS_1_2_VERSION) { + rc = -ENOTSUPP; + goto out; + } + + crypto_info = (struct tls_crypto_info *)&csk->tlshws.crypto_info; + + /* GCM mode of AES supports 128 and 256 bit encryption, so + * copy keys from user based on GCM cipher type. + */ + switch (tmp_crypto_info.cipher_type) { + case TLS_CIPHER_AES_GCM_128: { + /* Obtain version and type from previous copy */ + crypto_info[0] = tmp_crypto_info; + /* Now copy the following data */ + rc = copy_from_sockptr_offset((char *)crypto_info + + sizeof(*crypto_info), + optval, sizeof(*crypto_info), + sizeof(struct tls12_crypto_info_aes_gcm_128) + - sizeof(*crypto_info)); + + if (rc) { + rc = -EFAULT; + goto out; + } + + keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE; + cipher_type = TLS_CIPHER_AES_GCM_128; + break; + } + case TLS_CIPHER_AES_GCM_256: { + crypto_info[0] = tmp_crypto_info; + rc = copy_from_sockptr_offset((char *)crypto_info + + sizeof(*crypto_info), + optval, sizeof(*crypto_info), + sizeof(struct tls12_crypto_info_aes_gcm_256) + - sizeof(*crypto_info)); + + if (rc) { + rc = -EFAULT; + goto out; + } + + keylen = TLS_CIPHER_AES_GCM_256_KEY_SIZE; + cipher_type = TLS_CIPHER_AES_GCM_256; + break; + } + default: + rc = -EINVAL; + goto out; + } + rc = chtls_setkey(csk, keylen, optname, cipher_type); +out: + return rc; +} + +static int chtls_setsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, unsigned int optlen) +{ + struct tls_context *ctx = tls_get_ctx(sk); + + if (level != SOL_TLS) + return ctx->sk_proto->setsockopt(sk, level, + optname, optval, optlen); + + return do_chtls_setsockopt(sk, optname, optval, optlen); +} + +static struct cxgb4_uld_info chtls_uld_info = { + .name = DRV_NAME, + .nrxq = MAX_ULD_QSETS, + .ntxq = MAX_ULD_QSETS, + .rxq_size = 1024, + .add = chtls_uld_add, + .state_change = chtls_uld_state_change, + .rx_handler = chtls_uld_rx_handler, +}; + +void chtls_install_cpl_ops(struct sock *sk) +{ + if (sk->sk_family == AF_INET) + sk->sk_prot = &chtls_cpl_prot; + else + sk->sk_prot = &chtls_cpl_protv6; +} + +static void __init chtls_init_ulp_ops(void) +{ + chtls_cpl_prot = tcp_prot; + chtls_init_rsk_ops(&chtls_cpl_prot, &chtls_rsk_ops, + &tcp_prot, PF_INET); + chtls_cpl_prot.close = chtls_close; + chtls_cpl_prot.disconnect = chtls_disconnect; + chtls_cpl_prot.destroy = chtls_destroy_sock; + chtls_cpl_prot.shutdown = chtls_shutdown; + chtls_cpl_prot.sendmsg = chtls_sendmsg; + chtls_cpl_prot.sendpage = chtls_sendpage; + chtls_cpl_prot.recvmsg = chtls_recvmsg; + chtls_cpl_prot.setsockopt = chtls_setsockopt; + chtls_cpl_prot.getsockopt = chtls_getsockopt; +#if IS_ENABLED(CONFIG_IPV6) + chtls_cpl_protv6 = chtls_cpl_prot; + chtls_init_rsk_ops(&chtls_cpl_protv6, &chtls_rsk_opsv6, + &tcpv6_prot, PF_INET6); +#endif +} + +static int __init chtls_register(void) +{ + chtls_init_ulp_ops(); + register_listen_notifier(&listen_notifier); + cxgb4_register_uld(CXGB4_ULD_TLS, &chtls_uld_info); + return 0; +} + +static void __exit chtls_unregister(void) +{ + unregister_listen_notifier(&listen_notifier); + chtls_free_all_uld(); + cxgb4_unregister_uld(CXGB4_ULD_TLS); +} + +module_init(chtls_register); +module_exit(chtls_unregister); + +MODULE_DESCRIPTION("Chelsio TLS Inline driver"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Chelsio Communications"); +MODULE_VERSION(CHTLS_DRV_VERSION); diff --git a/drivers/net/ethernet/chelsio/libcxgb/Makefile b/drivers/net/ethernet/chelsio/libcxgb/Makefile new file mode 100644 index 000000000..aa79264e7 --- /dev/null +++ b/drivers/net/ethernet/chelsio/libcxgb/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +ccflags-y := -I $(srctree)/$(src)/../cxgb4 + +obj-$(CONFIG_CHELSIO_LIB) += libcxgb.o + +libcxgb-y := libcxgb_ppm.o libcxgb_cm.o diff --git a/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.c b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.c new file mode 100644 index 000000000..da8d10475 --- /dev/null +++ b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.c @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/tcp.h> +#include <linux/ipv6.h> +#include <net/inet_ecn.h> +#include <net/route.h> +#include <net/ip6_route.h> + +#include "libcxgb_cm.h" + +void +cxgb_get_4tuple(struct cpl_pass_accept_req *req, enum chip_type type, + int *iptype, __u8 *local_ip, __u8 *peer_ip, + __be16 *local_port, __be16 *peer_port) +{ + int eth_len = (CHELSIO_CHIP_VERSION(type) <= CHELSIO_T5) ? + ETH_HDR_LEN_G(be32_to_cpu(req->hdr_len)) : + T6_ETH_HDR_LEN_G(be32_to_cpu(req->hdr_len)); + int ip_len = (CHELSIO_CHIP_VERSION(type) <= CHELSIO_T5) ? + IP_HDR_LEN_G(be32_to_cpu(req->hdr_len)) : + T6_IP_HDR_LEN_G(be32_to_cpu(req->hdr_len)); + struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len); + struct ipv6hdr *ip6 = (struct ipv6hdr *)((u8 *)(req + 1) + eth_len); + struct tcphdr *tcp = (struct tcphdr *) + ((u8 *)(req + 1) + eth_len + ip_len); + + if (ip->version == 4) { + pr_debug("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", + __func__, ntohl(ip->saddr), ntohl(ip->daddr), + ntohs(tcp->source), ntohs(tcp->dest)); + *iptype = 4; + memcpy(peer_ip, &ip->saddr, 4); + memcpy(local_ip, &ip->daddr, 4); + } else { + pr_debug("%s saddr %pI6 daddr %pI6 sport %u dport %u\n", + __func__, ip6->saddr.s6_addr, ip6->daddr.s6_addr, + ntohs(tcp->source), ntohs(tcp->dest)); + *iptype = 6; + memcpy(peer_ip, ip6->saddr.s6_addr, 16); + memcpy(local_ip, ip6->daddr.s6_addr, 16); + } + *peer_port = tcp->source; + *local_port = tcp->dest; +} +EXPORT_SYMBOL(cxgb_get_4tuple); + +static bool +cxgb_our_interface(struct cxgb4_lld_info *lldi, + struct net_device *(*get_real_dev)(struct net_device *), + struct net_device *egress_dev) +{ + int i; + + egress_dev = get_real_dev(egress_dev); + for (i = 0; i < lldi->nports; i++) + if (lldi->ports[i] == egress_dev) + return true; + return false; +} + +struct dst_entry * +cxgb_find_route(struct cxgb4_lld_info *lldi, + struct net_device *(*get_real_dev)(struct net_device *), + __be32 local_ip, __be32 peer_ip, __be16 local_port, + __be16 peer_port, u8 tos) +{ + struct rtable *rt; + struct flowi4 fl4; + struct neighbour *n; + + rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip, + peer_port, local_port, IPPROTO_TCP, + tos & ~INET_ECN_MASK, 0); + if (IS_ERR(rt)) + return NULL; + n = dst_neigh_lookup(&rt->dst, &peer_ip); + if (!n) + return NULL; + if (!cxgb_our_interface(lldi, get_real_dev, n->dev) && + !(n->dev->flags & IFF_LOOPBACK)) { + neigh_release(n); + dst_release(&rt->dst); + return NULL; + } + neigh_release(n); + return &rt->dst; +} +EXPORT_SYMBOL(cxgb_find_route); + +struct dst_entry * +cxgb_find_route6(struct cxgb4_lld_info *lldi, + struct net_device *(*get_real_dev)(struct net_device *), + __u8 *local_ip, __u8 *peer_ip, __be16 local_port, + __be16 peer_port, u8 tos, __u32 sin6_scope_id) +{ + struct dst_entry *dst = NULL; + + if (IS_ENABLED(CONFIG_IPV6)) { + struct flowi6 fl6; + + memset(&fl6, 0, sizeof(fl6)); + memcpy(&fl6.daddr, peer_ip, 16); + memcpy(&fl6.saddr, local_ip, 16); + if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) + fl6.flowi6_oif = sin6_scope_id; + dst = ip6_route_output(&init_net, NULL, &fl6); + if (dst->error || + (!cxgb_our_interface(lldi, get_real_dev, + ip6_dst_idev(dst)->dev) && + !(ip6_dst_idev(dst)->dev->flags & IFF_LOOPBACK))) { + dst_release(dst); + return NULL; + } + } + + return dst; +} +EXPORT_SYMBOL(cxgb_find_route6); diff --git a/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.h b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.h new file mode 100644 index 000000000..240ba9d4c --- /dev/null +++ b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.h @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __LIBCXGB_CM_H__ +#define __LIBCXGB_CM_H__ + + +#include <net/tcp.h> + +#include <cxgb4.h> +#include <t4_msg.h> +#include <l2t.h> + +void +cxgb_get_4tuple(struct cpl_pass_accept_req *, enum chip_type, + int *, __u8 *, __u8 *, __be16 *, __be16 *); +struct dst_entry * +cxgb_find_route(struct cxgb4_lld_info *, + struct net_device *(*)(struct net_device *), + __be32, __be32, __be16, __be16, u8); +struct dst_entry * +cxgb_find_route6(struct cxgb4_lld_info *, + struct net_device *(*)(struct net_device *), + __u8 *, __u8 *, __be16, __be16, u8, __u32); + +/* Returns whether a CPL status conveys negative advice. + */ +static inline bool cxgb_is_neg_adv(unsigned int status) +{ + return status == CPL_ERR_RTX_NEG_ADVICE || + status == CPL_ERR_PERSIST_NEG_ADVICE || + status == CPL_ERR_KEEPALV_NEG_ADVICE; +} + +static inline void +cxgb_best_mtu(const unsigned short *mtus, unsigned short mtu, + unsigned int *idx, int use_ts, int ipv6) +{ + unsigned short hdr_size = (ipv6 ? + sizeof(struct ipv6hdr) : + sizeof(struct iphdr)) + + sizeof(struct tcphdr) + + (use_ts ? + round_up(TCPOLEN_TIMESTAMP, 4) : 0); + unsigned short data_size = mtu - hdr_size; + + cxgb4_best_aligned_mtu(mtus, hdr_size, data_size, 8, idx); +} + +static inline u32 cxgb_compute_wscale(u32 win) +{ + u32 wscale = 0; + + while (wscale < 14 && (65535 << wscale) < win) + wscale++; + return wscale; +} + +static inline void +cxgb_mk_tid_release(struct sk_buff *skb, u32 len, u32 tid, u16 chan) +{ + struct cpl_tid_release *req; + + req = __skb_put_zero(skb, len); + + INIT_TP_WR(req, tid); + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); + set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); +} + +static inline void +cxgb_mk_close_con_req(struct sk_buff *skb, u32 len, u32 tid, u16 chan, + void *handle, arp_err_handler_t handler) +{ + struct cpl_close_con_req *req; + + req = __skb_put_zero(skb, len); + + INIT_TP_WR(req, tid); + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); + set_wr_txq(skb, CPL_PRIORITY_DATA, chan); + t4_set_arp_err_handler(skb, handle, handler); +} + +static inline void +cxgb_mk_abort_req(struct sk_buff *skb, u32 len, u32 tid, u16 chan, + void *handle, arp_err_handler_t handler) +{ + struct cpl_abort_req *req; + + req = __skb_put_zero(skb, len); + + INIT_TP_WR(req, tid); + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, tid)); + req->cmd = CPL_ABORT_SEND_RST; + set_wr_txq(skb, CPL_PRIORITY_DATA, chan); + t4_set_arp_err_handler(skb, handle, handler); +} + +static inline void +cxgb_mk_abort_rpl(struct sk_buff *skb, u32 len, u32 tid, u16 chan) +{ + struct cpl_abort_rpl *rpl; + + rpl = __skb_put_zero(skb, len); + + INIT_TP_WR(rpl, tid); + OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, tid)); + rpl->cmd = CPL_ABORT_NO_RST; + set_wr_txq(skb, CPL_PRIORITY_DATA, chan); +} + +static inline void +cxgb_mk_rx_data_ack(struct sk_buff *skb, u32 len, u32 tid, u16 chan, + u32 credit_dack) +{ + struct cpl_rx_data_ack *req; + + req = __skb_put_zero(skb, len); + + INIT_TP_WR(req, tid); + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, tid)); + req->credit_dack = cpu_to_be32(credit_dack); + set_wr_txq(skb, CPL_PRIORITY_ACK, chan); +} +#endif diff --git a/drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.c b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.c new file mode 100644 index 000000000..854d87e11 --- /dev/null +++ b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.c @@ -0,0 +1,532 @@ +/* + * libcxgb_ppm.c: Chelsio common library for T3/T4/T5 iSCSI PagePod Manager + * + * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Written by: Karen Xie (kxie@chelsio.com) + */ + +#define DRV_NAME "libcxgb" +#define pr_fmt(fmt) DRV_NAME ": " fmt + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/errno.h> +#include <linux/types.h> +#include <linux/debugfs.h> +#include <linux/export.h> +#include <linux/list.h> +#include <linux/skbuff.h> +#include <linux/pci.h> +#include <linux/scatterlist.h> + +#include "libcxgb_ppm.h" + +/* Direct Data Placement - + * Directly place the iSCSI Data-In or Data-Out PDU's payload into + * pre-posted final destination host-memory buffers based on the + * Initiator Task Tag (ITT) in Data-In or Target Task Tag (TTT) + * in Data-Out PDUs. The host memory address is programmed into + * h/w in the format of pagepod entries. The location of the + * pagepod entry is encoded into ddp tag which is used as the base + * for ITT/TTT. + */ + +/* Direct-Data Placement page size adjustment + */ +int cxgbi_ppm_find_page_index(struct cxgbi_ppm *ppm, unsigned long pgsz) +{ + struct cxgbi_tag_format *tformat = &ppm->tformat; + int i; + + for (i = 0; i < DDP_PGIDX_MAX; i++) { + if (pgsz == 1UL << (DDP_PGSZ_BASE_SHIFT + + tformat->pgsz_order[i])) { + pr_debug("%s: %s ppm, pgsz %lu -> idx %d.\n", + __func__, ppm->ndev->name, pgsz, i); + return i; + } + } + pr_info("ippm: ddp page size %lu not supported.\n", pgsz); + return DDP_PGIDX_MAX; +} + +/* DDP setup & teardown + */ +static int ppm_find_unused_entries(unsigned long *bmap, + unsigned int max_ppods, + unsigned int start, + unsigned int nr, + unsigned int align_mask) +{ + unsigned long i; + + i = bitmap_find_next_zero_area(bmap, max_ppods, start, nr, align_mask); + + if (unlikely(i >= max_ppods) && (start > nr)) + i = bitmap_find_next_zero_area(bmap, max_ppods, 0, start - 1, + align_mask); + if (unlikely(i >= max_ppods)) + return -ENOSPC; + + bitmap_set(bmap, i, nr); + return (int)i; +} + +static void ppm_mark_entries(struct cxgbi_ppm *ppm, int i, int count, + unsigned long caller_data) +{ + struct cxgbi_ppod_data *pdata = ppm->ppod_data + i; + + pdata->caller_data = caller_data; + pdata->npods = count; + + if (pdata->color == ((1 << PPOD_IDX_SHIFT) - 1)) + pdata->color = 0; + else + pdata->color++; +} + +static int ppm_get_cpu_entries(struct cxgbi_ppm *ppm, unsigned int count, + unsigned long caller_data) +{ + struct cxgbi_ppm_pool *pool; + unsigned int cpu; + int i; + + if (!ppm->pool) + return -EINVAL; + + cpu = get_cpu(); + pool = per_cpu_ptr(ppm->pool, cpu); + spin_lock_bh(&pool->lock); + put_cpu(); + + i = ppm_find_unused_entries(pool->bmap, ppm->pool_index_max, + pool->next, count, 0); + if (i < 0) { + pool->next = 0; + spin_unlock_bh(&pool->lock); + return -ENOSPC; + } + + pool->next = i + count; + if (pool->next >= ppm->pool_index_max) + pool->next = 0; + + spin_unlock_bh(&pool->lock); + + pr_debug("%s: cpu %u, idx %d + %d (%d), next %u.\n", + __func__, cpu, i, count, i + cpu * ppm->pool_index_max, + pool->next); + + i += cpu * ppm->pool_index_max; + ppm_mark_entries(ppm, i, count, caller_data); + + return i; +} + +static int ppm_get_entries(struct cxgbi_ppm *ppm, unsigned int count, + unsigned long caller_data) +{ + int i; + + spin_lock_bh(&ppm->map_lock); + i = ppm_find_unused_entries(ppm->ppod_bmap, ppm->bmap_index_max, + ppm->next, count, 0); + if (i < 0) { + ppm->next = 0; + spin_unlock_bh(&ppm->map_lock); + pr_debug("ippm: NO suitable entries %u available.\n", + count); + return -ENOSPC; + } + + ppm->next = i + count; + if (ppm->max_index_in_edram && (ppm->next >= ppm->max_index_in_edram)) + ppm->next = 0; + else if (ppm->next >= ppm->bmap_index_max) + ppm->next = 0; + + spin_unlock_bh(&ppm->map_lock); + + pr_debug("%s: idx %d + %d (%d), next %u, caller_data 0x%lx.\n", + __func__, i, count, i + ppm->pool_rsvd, ppm->next, + caller_data); + + i += ppm->pool_rsvd; + ppm_mark_entries(ppm, i, count, caller_data); + + return i; +} + +static void ppm_unmark_entries(struct cxgbi_ppm *ppm, int i, int count) +{ + pr_debug("%s: idx %d + %d.\n", __func__, i, count); + + if (i < ppm->pool_rsvd) { + unsigned int cpu; + struct cxgbi_ppm_pool *pool; + + cpu = i / ppm->pool_index_max; + i %= ppm->pool_index_max; + + pool = per_cpu_ptr(ppm->pool, cpu); + spin_lock_bh(&pool->lock); + bitmap_clear(pool->bmap, i, count); + + if (i < pool->next) + pool->next = i; + spin_unlock_bh(&pool->lock); + + pr_debug("%s: cpu %u, idx %d, next %u.\n", + __func__, cpu, i, pool->next); + } else { + spin_lock_bh(&ppm->map_lock); + + i -= ppm->pool_rsvd; + bitmap_clear(ppm->ppod_bmap, i, count); + + if (i < ppm->next) + ppm->next = i; + spin_unlock_bh(&ppm->map_lock); + + pr_debug("%s: idx %d, next %u.\n", __func__, i, ppm->next); + } +} + +void cxgbi_ppm_ppod_release(struct cxgbi_ppm *ppm, u32 idx) +{ + struct cxgbi_ppod_data *pdata; + + if (idx >= ppm->ppmax) { + pr_warn("ippm: idx too big %u > %u.\n", idx, ppm->ppmax); + return; + } + + pdata = ppm->ppod_data + idx; + if (!pdata->npods) { + pr_warn("ippm: idx %u, npods 0.\n", idx); + return; + } + + pr_debug("release idx %u, npods %u.\n", idx, pdata->npods); + ppm_unmark_entries(ppm, idx, pdata->npods); +} +EXPORT_SYMBOL(cxgbi_ppm_ppod_release); + +int cxgbi_ppm_ppods_reserve(struct cxgbi_ppm *ppm, unsigned short nr_pages, + u32 per_tag_pg_idx, u32 *ppod_idx, + u32 *ddp_tag, unsigned long caller_data) +{ + struct cxgbi_ppod_data *pdata; + unsigned int npods; + int idx = -1; + unsigned int hwidx; + u32 tag; + + npods = (nr_pages + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT; + if (!npods) { + pr_warn("%s: pages %u -> npods %u, full.\n", + __func__, nr_pages, npods); + return -EINVAL; + } + + /* grab from cpu pool first */ + idx = ppm_get_cpu_entries(ppm, npods, caller_data); + /* try the general pool */ + if (idx < 0) + idx = ppm_get_entries(ppm, npods, caller_data); + if (idx < 0) { + pr_debug("ippm: pages %u, nospc %u, nxt %u, 0x%lx.\n", + nr_pages, npods, ppm->next, caller_data); + return idx; + } + + pdata = ppm->ppod_data + idx; + hwidx = ppm->base_idx + idx; + + tag = cxgbi_ppm_make_ddp_tag(hwidx, pdata->color); + + if (per_tag_pg_idx) + tag |= (per_tag_pg_idx << 30) & 0xC0000000; + + *ppod_idx = idx; + *ddp_tag = tag; + + pr_debug("ippm: sg %u, tag 0x%x(%u,%u), data 0x%lx.\n", + nr_pages, tag, idx, npods, caller_data); + + return npods; +} +EXPORT_SYMBOL(cxgbi_ppm_ppods_reserve); + +void cxgbi_ppm_make_ppod_hdr(struct cxgbi_ppm *ppm, u32 tag, + unsigned int tid, unsigned int offset, + unsigned int length, + struct cxgbi_pagepod_hdr *hdr) +{ + /* The ddp tag in pagepod should be with bit 31:30 set to 0. + * The ddp Tag on the wire should be with non-zero 31:30 to the peer + */ + tag &= 0x3FFFFFFF; + + hdr->vld_tid = htonl(PPOD_VALID_FLAG | PPOD_TID(tid)); + + hdr->rsvd = 0; + hdr->pgsz_tag_clr = htonl(tag & ppm->tformat.idx_clr_mask); + hdr->max_offset = htonl(length); + hdr->page_offset = htonl(offset); + + pr_debug("ippm: tag 0x%x, tid 0x%x, xfer %u, off %u.\n", + tag, tid, length, offset); +} +EXPORT_SYMBOL(cxgbi_ppm_make_ppod_hdr); + +static void ppm_free(struct cxgbi_ppm *ppm) +{ + vfree(ppm); +} + +static void ppm_destroy(struct kref *kref) +{ + struct cxgbi_ppm *ppm = container_of(kref, + struct cxgbi_ppm, + refcnt); + pr_info("ippm: kref 0, destroy %s ppm 0x%p.\n", + ppm->ndev->name, ppm); + + *ppm->ppm_pp = NULL; + + free_percpu(ppm->pool); + ppm_free(ppm); +} + +int cxgbi_ppm_release(struct cxgbi_ppm *ppm) +{ + if (ppm) { + int rv; + + rv = kref_put(&ppm->refcnt, ppm_destroy); + return rv; + } + return 1; +} +EXPORT_SYMBOL(cxgbi_ppm_release); + +static struct cxgbi_ppm_pool *ppm_alloc_cpu_pool(unsigned int *total, + unsigned int *pcpu_ppmax) +{ + struct cxgbi_ppm_pool *pools; + unsigned int ppmax = (*total) / num_possible_cpus(); + unsigned int max = (PCPU_MIN_UNIT_SIZE - sizeof(*pools)) << 3; + unsigned int bmap; + unsigned int alloc_sz; + unsigned int count = 0; + unsigned int cpu; + + /* make sure per cpu pool fits into PCPU_MIN_UNIT_SIZE */ + if (ppmax > max) + ppmax = max; + + /* pool size must be multiple of unsigned long */ + bmap = ppmax / BITS_PER_TYPE(unsigned long); + if (!bmap) + return NULL; + + ppmax = (bmap * sizeof(unsigned long)) << 3; + + alloc_sz = sizeof(*pools) + sizeof(unsigned long) * bmap; + pools = __alloc_percpu(alloc_sz, __alignof__(struct cxgbi_ppm_pool)); + + if (!pools) + return NULL; + + for_each_possible_cpu(cpu) { + struct cxgbi_ppm_pool *ppool = per_cpu_ptr(pools, cpu); + + memset(ppool, 0, alloc_sz); + spin_lock_init(&ppool->lock); + count += ppmax; + } + + *total = count; + *pcpu_ppmax = ppmax; + + return pools; +} + +int cxgbi_ppm_init(void **ppm_pp, struct net_device *ndev, + struct pci_dev *pdev, void *lldev, + struct cxgbi_tag_format *tformat, unsigned int iscsi_size, + unsigned int llimit, unsigned int start, + unsigned int reserve_factor, unsigned int iscsi_edram_start, + unsigned int iscsi_edram_size) +{ + struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*ppm_pp); + struct cxgbi_ppm_pool *pool = NULL; + unsigned int pool_index_max = 0; + unsigned int ppmax_pool = 0; + unsigned int ppod_bmap_size; + unsigned int alloc_sz; + unsigned int ppmax; + + if (!iscsi_edram_start) + iscsi_edram_size = 0; + + if (iscsi_edram_size && + ((iscsi_edram_start + iscsi_edram_size) != start)) { + pr_err("iscsi ppod region not contiguous: EDRAM start 0x%x " + "size 0x%x DDR start 0x%x\n", + iscsi_edram_start, iscsi_edram_size, start); + return -EINVAL; + } + + if (iscsi_edram_size) { + reserve_factor = 0; + start = iscsi_edram_start; + } + + ppmax = (iscsi_edram_size + iscsi_size) >> PPOD_SIZE_SHIFT; + + if (ppm) { + pr_info("ippm: %s, ppm 0x%p,0x%p already initialized, %u/%u.\n", + ndev->name, ppm_pp, ppm, ppm->ppmax, ppmax); + kref_get(&ppm->refcnt); + return 1; + } + + if (reserve_factor) { + ppmax_pool = ppmax / reserve_factor; + pool = ppm_alloc_cpu_pool(&ppmax_pool, &pool_index_max); + if (!pool) { + ppmax_pool = 0; + reserve_factor = 0; + } + + pr_debug("%s: ppmax %u, cpu total %u, per cpu %u.\n", + ndev->name, ppmax, ppmax_pool, pool_index_max); + } + + ppod_bmap_size = BITS_TO_LONGS(ppmax - ppmax_pool); + alloc_sz = sizeof(struct cxgbi_ppm) + + ppmax * (sizeof(struct cxgbi_ppod_data)) + + ppod_bmap_size * sizeof(unsigned long); + + ppm = vzalloc(alloc_sz); + if (!ppm) + goto release_ppm_pool; + + ppm->ppod_bmap = (unsigned long *)(&ppm->ppod_data[ppmax]); + + if ((ppod_bmap_size >> 3) > (ppmax - ppmax_pool)) { + unsigned int start = ppmax - ppmax_pool; + unsigned int end = ppod_bmap_size >> 3; + + bitmap_set(ppm->ppod_bmap, ppmax, end - start); + pr_info("%s: %u - %u < %u * 8, mask extra bits %u, %u.\n", + __func__, ppmax, ppmax_pool, ppod_bmap_size, start, + end); + } + if (iscsi_edram_size) { + unsigned int first_ddr_idx = + iscsi_edram_size >> PPOD_SIZE_SHIFT; + + ppm->max_index_in_edram = first_ddr_idx - 1; + bitmap_set(ppm->ppod_bmap, first_ddr_idx, 1); + pr_debug("reserved %u ppod in bitmap\n", first_ddr_idx); + } + + spin_lock_init(&ppm->map_lock); + kref_init(&ppm->refcnt); + + memcpy(&ppm->tformat, tformat, sizeof(struct cxgbi_tag_format)); + + ppm->ppm_pp = ppm_pp; + ppm->ndev = ndev; + ppm->pdev = pdev; + ppm->lldev = lldev; + ppm->ppmax = ppmax; + ppm->next = 0; + ppm->llimit = llimit; + ppm->base_idx = start > llimit ? + (start - llimit + 1) >> PPOD_SIZE_SHIFT : 0; + ppm->bmap_index_max = ppmax - ppmax_pool; + + ppm->pool = pool; + ppm->pool_rsvd = ppmax_pool; + ppm->pool_index_max = pool_index_max; + + /* check one more time */ + if (*ppm_pp) { + ppm_free(ppm); + ppm = (struct cxgbi_ppm *)(*ppm_pp); + + pr_info("ippm: %s, ppm 0x%p,0x%p already initialized, %u/%u.\n", + ndev->name, ppm_pp, *ppm_pp, ppm->ppmax, ppmax); + + kref_get(&ppm->refcnt); + return 1; + } + *ppm_pp = ppm; + + ppm->tformat.pgsz_idx_dflt = cxgbi_ppm_find_page_index(ppm, PAGE_SIZE); + + pr_info("ippm %s: ppm 0x%p, 0x%p, base %u/%u, pg %lu,%u, rsvd %u,%u.\n", + ndev->name, ppm_pp, ppm, ppm->base_idx, ppm->ppmax, PAGE_SIZE, + ppm->tformat.pgsz_idx_dflt, ppm->pool_rsvd, + ppm->pool_index_max); + + return 0; + +release_ppm_pool: + free_percpu(pool); + return -ENOMEM; +} +EXPORT_SYMBOL(cxgbi_ppm_init); + +unsigned int cxgbi_tagmask_set(unsigned int ppmax) +{ + unsigned int bits = fls(ppmax); + + if (bits > PPOD_IDX_MAX_SIZE) + bits = PPOD_IDX_MAX_SIZE; + + pr_info("ippm: ppmax %u/0x%x -> bits %u, tagmask 0x%x.\n", + ppmax, ppmax, bits, 1 << (bits + PPOD_IDX_SHIFT)); + + return 1 << (bits + PPOD_IDX_SHIFT); +} +EXPORT_SYMBOL(cxgbi_tagmask_set); + +MODULE_AUTHOR("Chelsio Communications"); +MODULE_DESCRIPTION("Chelsio common library"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.h b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.h new file mode 100644 index 000000000..1b4156461 --- /dev/null +++ b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.h @@ -0,0 +1,335 @@ +/* + * libcxgb_ppm.h: Chelsio common library for T3/T4/T5 iSCSI ddp operation + * + * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Written by: Karen Xie (kxie@chelsio.com) + */ + +#ifndef __LIBCXGB_PPM_H__ +#define __LIBCXGB_PPM_H__ + +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/types.h> +#include <linux/debugfs.h> +#include <linux/list.h> +#include <linux/netdevice.h> +#include <linux/scatterlist.h> +#include <linux/skbuff.h> +#include <linux/vmalloc.h> +#include <linux/bitmap.h> + +struct cxgbi_pagepod_hdr { + u32 vld_tid; + u32 pgsz_tag_clr; + u32 max_offset; + u32 page_offset; + u64 rsvd; +}; + +#define PPOD_PAGES_MAX 4 +struct cxgbi_pagepod { + struct cxgbi_pagepod_hdr hdr; + __be64 addr[PPOD_PAGES_MAX + 1]; +}; + +/* ddp tag format + * for a 32-bit tag: + * bit # + * 31 ..... ..... 0 + * X Y...Y Z...Z, where + * ^ ^^^^^ ^^^^ + * | | |____ when ddp bit = 0: color bits + * | | + * | |____ when ddp bit = 0: idx into the ddp memory region + * | + * |____ ddp bit: 0 - ddp tag, 1 - non-ddp tag + * + * [page selector:2] [sw/free bits] [0] [idx] [color:6] + */ + +#define DDP_PGIDX_MAX 4 +#define DDP_PGSZ_BASE_SHIFT 12 /* base page 4K */ + +struct cxgbi_task_tag_info { + unsigned char flags; +#define CXGBI_PPOD_INFO_FLAG_VALID 0x1 +#define CXGBI_PPOD_INFO_FLAG_MAPPED 0x2 + unsigned char cid; + unsigned short pg_shift; + unsigned int npods; + unsigned int idx; + unsigned int tag; + struct cxgbi_pagepod_hdr hdr; + int nents; + int nr_pages; + struct scatterlist *sgl; +}; + +struct cxgbi_tag_format { + unsigned char pgsz_order[DDP_PGIDX_MAX]; + unsigned char pgsz_idx_dflt; + unsigned char free_bits:4; + unsigned char color_bits:4; + unsigned char idx_bits; + unsigned char rsvd_bits; + unsigned int no_ddp_mask; + unsigned int idx_mask; + unsigned int color_mask; + unsigned int idx_clr_mask; + unsigned int rsvd_mask; +}; + +struct cxgbi_ppod_data { + unsigned char pg_idx:2; + unsigned char color:6; + unsigned char chan_id; + unsigned short npods; + unsigned long caller_data; +}; + +/* per cpu ppm pool */ +struct cxgbi_ppm_pool { + unsigned int base; /* base index */ + unsigned int next; /* next possible free index */ + spinlock_t lock; /* ppm pool lock */ + unsigned long bmap[]; +} ____cacheline_aligned_in_smp; + +struct cxgbi_ppm { + struct kref refcnt; + struct net_device *ndev; /* net_device, 1st port */ + struct pci_dev *pdev; + void *lldev; + void **ppm_pp; + struct cxgbi_tag_format tformat; + unsigned int ppmax; + unsigned int llimit; + unsigned int base_idx; + + unsigned int pool_rsvd; + unsigned int pool_index_max; + struct cxgbi_ppm_pool __percpu *pool; + /* map lock */ + spinlock_t map_lock; /* ppm map lock */ + unsigned int bmap_index_max; + unsigned int next; + unsigned int max_index_in_edram; + unsigned long *ppod_bmap; + struct cxgbi_ppod_data ppod_data[]; +}; + +#define DDP_THRESHOLD 512 + +#define PPOD_PAGES_SHIFT 2 /* 4 pages per pod */ + +#define IPPOD_SIZE sizeof(struct cxgbi_pagepod) /* 64 */ +#define PPOD_SIZE_SHIFT 6 + +/* page pods are allocated in groups of this size (must be power of 2) */ +#define PPOD_CLUSTER_SIZE 16U + +#define ULPMEM_DSGL_MAX_NPPODS 16 /* 1024/PPOD_SIZE */ +#define ULPMEM_IDATA_MAX_NPPODS 3 /* (PPOD_SIZE * 3 + ulptx hdr) < 256B */ +#define PCIE_MEMWIN_MAX_NPPODS 16 /* 1024/PPOD_SIZE */ + +#define PPOD_COLOR_SHIFT 0 +#define PPOD_COLOR(x) ((x) << PPOD_COLOR_SHIFT) + +#define PPOD_IDX_SHIFT 6 +#define PPOD_IDX_MAX_SIZE 24 + +#define PPOD_TID_SHIFT 0 +#define PPOD_TID(x) ((x) << PPOD_TID_SHIFT) + +#define PPOD_TAG_SHIFT 6 +#define PPOD_TAG(x) ((x) << PPOD_TAG_SHIFT) + +#define PPOD_VALID_SHIFT 24 +#define PPOD_VALID(x) ((x) << PPOD_VALID_SHIFT) +#define PPOD_VALID_FLAG PPOD_VALID(1U) + +#define PPOD_PI_EXTRACT_CTL_SHIFT 31 +#define PPOD_PI_EXTRACT_CTL(x) ((x) << PPOD_PI_EXTRACT_CTL_SHIFT) +#define PPOD_PI_EXTRACT_CTL_FLAG V_PPOD_PI_EXTRACT_CTL(1U) + +#define PPOD_PI_TYPE_SHIFT 29 +#define PPOD_PI_TYPE_MASK 0x3 +#define PPOD_PI_TYPE(x) ((x) << PPOD_PI_TYPE_SHIFT) + +#define PPOD_PI_CHECK_CTL_SHIFT 27 +#define PPOD_PI_CHECK_CTL_MASK 0x3 +#define PPOD_PI_CHECK_CTL(x) ((x) << PPOD_PI_CHECK_CTL_SHIFT) + +#define PPOD_PI_REPORT_CTL_SHIFT 25 +#define PPOD_PI_REPORT_CTL_MASK 0x3 +#define PPOD_PI_REPORT_CTL(x) ((x) << PPOD_PI_REPORT_CTL_SHIFT) + +static inline int cxgbi_ppm_is_ddp_tag(struct cxgbi_ppm *ppm, u32 tag) +{ + return !(tag & ppm->tformat.no_ddp_mask); +} + +static inline int cxgbi_ppm_sw_tag_is_usable(struct cxgbi_ppm *ppm, + u32 tag) +{ + /* the sw tag must be using <= 31 bits */ + return !(tag & 0x80000000U); +} + +static inline int cxgbi_ppm_make_non_ddp_tag(struct cxgbi_ppm *ppm, + u32 sw_tag, + u32 *final_tag) +{ + struct cxgbi_tag_format *tformat = &ppm->tformat; + + if (!cxgbi_ppm_sw_tag_is_usable(ppm, sw_tag)) { + pr_info("sw_tag 0x%x NOT usable.\n", sw_tag); + return -EINVAL; + } + + if (!sw_tag) { + *final_tag = tformat->no_ddp_mask; + } else { + unsigned int shift = tformat->idx_bits + tformat->color_bits; + u32 lower = sw_tag & tformat->idx_clr_mask; + u32 upper = (sw_tag >> shift) << (shift + 1); + + *final_tag = upper | tformat->no_ddp_mask | lower; + } + return 0; +} + +static inline u32 cxgbi_ppm_decode_non_ddp_tag(struct cxgbi_ppm *ppm, + u32 tag) +{ + struct cxgbi_tag_format *tformat = &ppm->tformat; + unsigned int shift = tformat->idx_bits + tformat->color_bits; + u32 lower = tag & tformat->idx_clr_mask; + u32 upper = (tag >> tformat->rsvd_bits) << shift; + + return upper | lower; +} + +static inline u32 cxgbi_ppm_ddp_tag_get_idx(struct cxgbi_ppm *ppm, + u32 ddp_tag) +{ + u32 hw_idx = (ddp_tag >> PPOD_IDX_SHIFT) & + ppm->tformat.idx_mask; + + return hw_idx - ppm->base_idx; +} + +static inline u32 cxgbi_ppm_make_ddp_tag(unsigned int hw_idx, + unsigned char color) +{ + return (hw_idx << PPOD_IDX_SHIFT) | ((u32)color); +} + +static inline unsigned long +cxgbi_ppm_get_tag_caller_data(struct cxgbi_ppm *ppm, + u32 ddp_tag) +{ + u32 idx = cxgbi_ppm_ddp_tag_get_idx(ppm, ddp_tag); + + return ppm->ppod_data[idx].caller_data; +} + +/* sw bits are the free bits */ +static inline int cxgbi_ppm_ddp_tag_update_sw_bits(struct cxgbi_ppm *ppm, + u32 val, u32 orig_tag, + u32 *final_tag) +{ + struct cxgbi_tag_format *tformat = &ppm->tformat; + u32 v = val >> tformat->free_bits; + + if (v) { + pr_info("sw_bits 0x%x too large, avail bits %u.\n", + val, tformat->free_bits); + return -EINVAL; + } + if (!cxgbi_ppm_is_ddp_tag(ppm, orig_tag)) + return -EINVAL; + + *final_tag = (val << tformat->rsvd_bits) | + (orig_tag & ppm->tformat.rsvd_mask); + return 0; +} + +static inline void cxgbi_ppm_ppod_clear(struct cxgbi_pagepod *ppod) +{ + ppod->hdr.vld_tid = 0U; +} + +static inline void cxgbi_tagmask_check(unsigned int tagmask, + struct cxgbi_tag_format *tformat) +{ + unsigned int bits = fls(tagmask); + + /* reserve top most 2 bits for page selector */ + tformat->free_bits = 32 - 2 - bits; + tformat->rsvd_bits = bits; + tformat->color_bits = PPOD_IDX_SHIFT; + tformat->idx_bits = bits - 1 - PPOD_IDX_SHIFT; + tformat->no_ddp_mask = 1 << (bits - 1); + tformat->idx_mask = (1 << tformat->idx_bits) - 1; + tformat->color_mask = (1 << PPOD_IDX_SHIFT) - 1; + tformat->idx_clr_mask = (1 << (bits - 1)) - 1; + tformat->rsvd_mask = (1 << bits) - 1; + + pr_info("ippm: tagmask 0x%x, rsvd %u=%u+%u+1, mask 0x%x,0x%x, " + "pg %u,%u,%u,%u.\n", + tagmask, tformat->rsvd_bits, tformat->idx_bits, + tformat->color_bits, tformat->no_ddp_mask, tformat->rsvd_mask, + tformat->pgsz_order[0], tformat->pgsz_order[1], + tformat->pgsz_order[2], tformat->pgsz_order[3]); +} + +int cxgbi_ppm_find_page_index(struct cxgbi_ppm *ppm, unsigned long pgsz); +void cxgbi_ppm_make_ppod_hdr(struct cxgbi_ppm *ppm, u32 tag, + unsigned int tid, unsigned int offset, + unsigned int length, + struct cxgbi_pagepod_hdr *hdr); +void cxgbi_ppm_ppod_release(struct cxgbi_ppm *, u32 idx); +int cxgbi_ppm_ppods_reserve(struct cxgbi_ppm *, unsigned short nr_pages, + u32 per_tag_pg_idx, u32 *ppod_idx, u32 *ddp_tag, + unsigned long caller_data); +int cxgbi_ppm_init(void **ppm_pp, struct net_device *, struct pci_dev *, + void *lldev, struct cxgbi_tag_format *, + unsigned int iscsi_size, unsigned int llimit, + unsigned int start, unsigned int reserve_factor, + unsigned int edram_start, unsigned int edram_size); +int cxgbi_ppm_release(struct cxgbi_ppm *ppm); +void cxgbi_tagmask_check(unsigned int tagmask, struct cxgbi_tag_format *); +unsigned int cxgbi_tagmask_set(unsigned int ppmax); + +#endif /*__LIBCXGB_PPM_H__*/ |