summaryrefslogtreecommitdiffstats
path: root/drivers/net/ethernet/intel/iavf
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/ethernet/intel/iavf')
-rw-r--r--drivers/net/ethernet/intel/iavf/Makefile16
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf.h592
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_adminq.c946
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_adminq.h135
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_adminq_cmd.h528
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_adv_rss.c218
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_adv_rss.h95
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_alloc.h33
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_client.c578
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_client.h169
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_common.c799
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_devids.h12
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_ethtool.c2012
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_fdir.c853
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_fdir.h133
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_main.c5323
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_osdep.h46
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_prototype.h64
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_register.h68
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_status.h78
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_trace.h209
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_txrx.c2537
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_txrx.h522
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_type.h679
-rw-r--r--drivers/net/ethernet/intel/iavf/iavf_virtchnl.c2550
25 files changed, 19195 insertions, 0 deletions
diff --git a/drivers/net/ethernet/intel/iavf/Makefile b/drivers/net/ethernet/intel/iavf/Makefile
new file mode 100644
index 0000000000..9c3e45c54d
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/Makefile
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright(c) 2013 - 2018 Intel Corporation.
+#
+# Makefile for the Intel(R) Ethernet Adaptive Virtual Function (iavf)
+# driver
+#
+#
+
+ccflags-y += -I$(src)
+subdir-ccflags-y += -I$(src)
+
+obj-$(CONFIG_IAVF) += iavf.o
+
+iavf-objs := iavf_main.o iavf_ethtool.o iavf_virtchnl.o iavf_fdir.o \
+ iavf_adv_rss.o \
+ iavf_txrx.o iavf_common.o iavf_adminq.o iavf_client.o
diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h
new file mode 100644
index 0000000000..431d9d62c8
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf.h
@@ -0,0 +1,592 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_H_
+#define _IAVF_H_
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/vmalloc.h>
+#include <linux/interrupt.h>
+#include <linux/ethtool.h>
+#include <linux/if_vlan.h>
+#include <linux/ip.h>
+#include <linux/tcp.h>
+#include <linux/sctp.h>
+#include <linux/ipv6.h>
+#include <linux/kernel.h>
+#include <linux/bitops.h>
+#include <linux/timer.h>
+#include <linux/workqueue.h>
+#include <linux/wait.h>
+#include <linux/delay.h>
+#include <linux/gfp.h>
+#include <linux/skbuff.h>
+#include <linux/dma-mapping.h>
+#include <linux/etherdevice.h>
+#include <linux/socket.h>
+#include <linux/jiffies.h>
+#include <net/ip6_checksum.h>
+#include <net/pkt_cls.h>
+#include <net/pkt_sched.h>
+#include <net/udp.h>
+#include <net/tc_act/tc_gact.h>
+#include <net/tc_act/tc_mirred.h>
+
+#include "iavf_type.h"
+#include <linux/avf/virtchnl.h>
+#include "iavf_txrx.h"
+#include "iavf_fdir.h"
+#include "iavf_adv_rss.h"
+#include <linux/bitmap.h>
+
+#define DEFAULT_DEBUG_LEVEL_SHIFT 3
+#define PFX "iavf: "
+
+int iavf_status_to_errno(enum iavf_status status);
+int virtchnl_status_to_errno(enum virtchnl_status_code v_status);
+
+/* VSI state flags shared with common code */
+enum iavf_vsi_state_t {
+ __IAVF_VSI_DOWN,
+ /* This must be last as it determines the size of the BITMAP */
+ __IAVF_VSI_STATE_SIZE__,
+};
+
+/* dummy struct to make common code less painful */
+struct iavf_vsi {
+ struct iavf_adapter *back;
+ struct net_device *netdev;
+ u16 seid;
+ u16 id;
+ DECLARE_BITMAP(state, __IAVF_VSI_STATE_SIZE__);
+ int base_vector;
+ u16 qs_handle;
+ void *priv; /* client driver data reference. */
+};
+
+/* How many Rx Buffers do we bundle into one write to the hardware ? */
+#define IAVF_RX_BUFFER_WRITE 16 /* Must be power of 2 */
+#define IAVF_DEFAULT_TXD 512
+#define IAVF_DEFAULT_RXD 512
+#define IAVF_MAX_TXD 4096
+#define IAVF_MIN_TXD 64
+#define IAVF_MAX_RXD 4096
+#define IAVF_MIN_RXD 64
+#define IAVF_REQ_DESCRIPTOR_MULTIPLE 32
+#define IAVF_MAX_AQ_BUF_SIZE 4096
+#define IAVF_AQ_LEN 32
+#define IAVF_AQ_MAX_ERR 20 /* times to try before resetting AQ */
+
+#define MAXIMUM_ETHERNET_VLAN_SIZE (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)
+
+#define IAVF_RX_DESC(R, i) (&(((union iavf_32byte_rx_desc *)((R)->desc))[i]))
+#define IAVF_TX_DESC(R, i) (&(((struct iavf_tx_desc *)((R)->desc))[i]))
+#define IAVF_TX_CTXTDESC(R, i) \
+ (&(((struct iavf_tx_context_desc *)((R)->desc))[i]))
+#define IAVF_MAX_REQ_QUEUES 16
+
+#define IAVF_HKEY_ARRAY_SIZE ((IAVF_VFQF_HKEY_MAX_INDEX + 1) * 4)
+#define IAVF_HLUT_ARRAY_SIZE ((IAVF_VFQF_HLUT_MAX_INDEX + 1) * 4)
+#define IAVF_MBPS_DIVISOR 125000 /* divisor to convert to Mbps */
+#define IAVF_MBPS_QUANTA 50
+
+#define IAVF_VIRTCHNL_VF_RESOURCE_SIZE \
+ virtchnl_struct_size((struct virtchnl_vf_resource *)NULL, \
+ vsi_res, IAVF_MAX_VF_VSI)
+
+/* MAX_MSIX_Q_VECTORS of these are allocated,
+ * but we only use one per queue-specific vector.
+ */
+struct iavf_q_vector {
+ struct iavf_adapter *adapter;
+ struct iavf_vsi *vsi;
+ struct napi_struct napi;
+ struct iavf_ring_container rx;
+ struct iavf_ring_container tx;
+ u32 ring_mask;
+ u8 itr_countdown; /* when 0 should adjust adaptive ITR */
+ u8 num_ringpairs; /* total number of ring pairs in vector */
+ u16 v_idx; /* index in the vsi->q_vector array. */
+ u16 reg_idx; /* register index of the interrupt */
+ char name[IFNAMSIZ + 15];
+ bool arm_wb_state;
+ cpumask_t affinity_mask;
+ struct irq_affinity_notify affinity_notify;
+};
+
+/* Helper macros to switch between ints/sec and what the register uses.
+ * And yes, it's the same math going both ways. The lowest value
+ * supported by all of the iavf hardware is 8.
+ */
+#define EITR_INTS_PER_SEC_TO_REG(_eitr) \
+ ((_eitr) ? (1000000000 / ((_eitr) * 256)) : 8)
+#define EITR_REG_TO_INTS_PER_SEC EITR_INTS_PER_SEC_TO_REG
+
+#define IAVF_DESC_UNUSED(R) \
+ ((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->count) + \
+ (R)->next_to_clean - (R)->next_to_use - 1)
+
+#define OTHER_VECTOR 1
+#define NONQ_VECS (OTHER_VECTOR)
+
+#define MIN_MSIX_Q_VECTORS 1
+#define MIN_MSIX_COUNT (MIN_MSIX_Q_VECTORS + NONQ_VECS)
+
+#define IAVF_QUEUE_END_OF_LIST 0x7FF
+#define IAVF_FREE_VECTOR 0x7FFF
+struct iavf_mac_filter {
+ struct list_head list;
+ u8 macaddr[ETH_ALEN];
+ struct {
+ u8 is_new_mac:1; /* filter is new, wait for PF decision */
+ u8 remove:1; /* filter needs to be removed */
+ u8 add:1; /* filter needs to be added */
+ u8 is_primary:1; /* filter is a default VF MAC */
+ u8 add_handled:1; /* received response for filter add */
+ u8 padding:3;
+ };
+};
+
+#define IAVF_VLAN(vid, tpid) ((struct iavf_vlan){ vid, tpid })
+struct iavf_vlan {
+ u16 vid;
+ u16 tpid;
+};
+
+enum iavf_vlan_state_t {
+ IAVF_VLAN_INVALID,
+ IAVF_VLAN_ADD, /* filter needs to be added */
+ IAVF_VLAN_IS_NEW, /* filter is new, wait for PF answer */
+ IAVF_VLAN_ACTIVE, /* filter is accepted by PF */
+ IAVF_VLAN_DISABLE, /* filter needs to be deleted by PF, then marked INACTIVE */
+ IAVF_VLAN_INACTIVE, /* filter is inactive, we are in IFF_DOWN */
+ IAVF_VLAN_REMOVE, /* filter needs to be removed from list */
+};
+
+struct iavf_vlan_filter {
+ struct list_head list;
+ struct iavf_vlan vlan;
+ enum iavf_vlan_state_t state;
+};
+
+#define IAVF_MAX_TRAFFIC_CLASS 4
+/* State of traffic class creation */
+enum iavf_tc_state_t {
+ __IAVF_TC_INVALID, /* no traffic class, default state */
+ __IAVF_TC_RUNNING, /* traffic classes have been created */
+};
+
+/* channel info */
+struct iavf_channel_config {
+ struct virtchnl_channel_info ch_info[IAVF_MAX_TRAFFIC_CLASS];
+ enum iavf_tc_state_t state;
+ u8 total_qps;
+};
+
+/* State of cloud filter */
+enum iavf_cloud_filter_state_t {
+ __IAVF_CF_INVALID, /* cloud filter not added */
+ __IAVF_CF_ADD_PENDING, /* cloud filter pending add by the PF */
+ __IAVF_CF_DEL_PENDING, /* cloud filter pending del by the PF */
+ __IAVF_CF_ACTIVE, /* cloud filter is active */
+};
+
+/* Driver state. The order of these is important! */
+enum iavf_state_t {
+ __IAVF_STARTUP, /* driver loaded, probe complete */
+ __IAVF_REMOVE, /* driver is being unloaded */
+ __IAVF_INIT_VERSION_CHECK, /* aq msg sent, awaiting reply */
+ __IAVF_INIT_GET_RESOURCES, /* aq msg sent, awaiting reply */
+ __IAVF_INIT_EXTENDED_CAPS, /* process extended caps which require aq msg exchange */
+ __IAVF_INIT_CONFIG_ADAPTER,
+ __IAVF_INIT_SW, /* got resources, setting up structs */
+ __IAVF_INIT_FAILED, /* init failed, restarting procedure */
+ __IAVF_RESETTING, /* in reset */
+ __IAVF_COMM_FAILED, /* communication with PF failed */
+ /* Below here, watchdog is running */
+ __IAVF_DOWN, /* ready, can be opened */
+ __IAVF_DOWN_PENDING, /* descending, waiting for watchdog */
+ __IAVF_TESTING, /* in ethtool self-test */
+ __IAVF_RUNNING, /* opened, working */
+};
+
+enum iavf_critical_section_t {
+ __IAVF_IN_REMOVE_TASK, /* device being removed */
+};
+
+#define IAVF_CLOUD_FIELD_OMAC 0x01
+#define IAVF_CLOUD_FIELD_IMAC 0x02
+#define IAVF_CLOUD_FIELD_IVLAN 0x04
+#define IAVF_CLOUD_FIELD_TEN_ID 0x08
+#define IAVF_CLOUD_FIELD_IIP 0x10
+
+#define IAVF_CF_FLAGS_OMAC IAVF_CLOUD_FIELD_OMAC
+#define IAVF_CF_FLAGS_IMAC IAVF_CLOUD_FIELD_IMAC
+#define IAVF_CF_FLAGS_IMAC_IVLAN (IAVF_CLOUD_FIELD_IMAC |\
+ IAVF_CLOUD_FIELD_IVLAN)
+#define IAVF_CF_FLAGS_IMAC_TEN_ID (IAVF_CLOUD_FIELD_IMAC |\
+ IAVF_CLOUD_FIELD_TEN_ID)
+#define IAVF_CF_FLAGS_OMAC_TEN_ID_IMAC (IAVF_CLOUD_FIELD_OMAC |\
+ IAVF_CLOUD_FIELD_IMAC |\
+ IAVF_CLOUD_FIELD_TEN_ID)
+#define IAVF_CF_FLAGS_IMAC_IVLAN_TEN_ID (IAVF_CLOUD_FIELD_IMAC |\
+ IAVF_CLOUD_FIELD_IVLAN |\
+ IAVF_CLOUD_FIELD_TEN_ID)
+#define IAVF_CF_FLAGS_IIP IAVF_CLOUD_FIELD_IIP
+
+/* bookkeeping of cloud filters */
+struct iavf_cloud_filter {
+ enum iavf_cloud_filter_state_t state;
+ struct list_head list;
+ struct virtchnl_filter f;
+ unsigned long cookie;
+ bool del; /* filter needs to be deleted */
+ bool add; /* filter needs to be added */
+};
+
+#define IAVF_RESET_WAIT_MS 10
+#define IAVF_RESET_WAIT_DETECTED_COUNT 500
+#define IAVF_RESET_WAIT_COMPLETE_COUNT 2000
+
+/* board specific private data structure */
+struct iavf_adapter {
+ struct workqueue_struct *wq;
+ struct work_struct reset_task;
+ struct work_struct adminq_task;
+ struct work_struct finish_config;
+ struct delayed_work client_task;
+ wait_queue_head_t down_waitqueue;
+ wait_queue_head_t reset_waitqueue;
+ wait_queue_head_t vc_waitqueue;
+ struct iavf_q_vector *q_vectors;
+ struct list_head vlan_filter_list;
+ int num_vlan_filters;
+ struct list_head mac_filter_list;
+ struct mutex crit_lock;
+ struct mutex client_lock;
+ /* Lock to protect accesses to MAC and VLAN lists */
+ spinlock_t mac_vlan_list_lock;
+ char misc_vector_name[IFNAMSIZ + 9];
+ int num_active_queues;
+ int num_req_queues;
+
+ /* TX */
+ struct iavf_ring *tx_rings;
+ u32 tx_timeout_count;
+ u32 tx_desc_count;
+
+ /* RX */
+ struct iavf_ring *rx_rings;
+ u64 hw_csum_rx_error;
+ u32 rx_desc_count;
+ int num_msix_vectors;
+ int num_rdma_msix;
+ int rdma_base_vector;
+ u32 client_pending;
+ struct iavf_client_instance *cinst;
+ struct msix_entry *msix_entries;
+
+ u32 flags;
+#define IAVF_FLAG_RX_CSUM_ENABLED BIT(0)
+#define IAVF_FLAG_PF_COMMS_FAILED BIT(3)
+#define IAVF_FLAG_RESET_PENDING BIT(4)
+#define IAVF_FLAG_RESET_NEEDED BIT(5)
+#define IAVF_FLAG_WB_ON_ITR_CAPABLE BIT(6)
+#define IAVF_FLAG_SERVICE_CLIENT_REQUESTED BIT(9)
+#define IAVF_FLAG_CLIENT_NEEDS_OPEN BIT(10)
+#define IAVF_FLAG_CLIENT_NEEDS_CLOSE BIT(11)
+#define IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS BIT(12)
+#define IAVF_FLAG_LEGACY_RX BIT(15)
+#define IAVF_FLAG_REINIT_ITR_NEEDED BIT(16)
+#define IAVF_FLAG_QUEUES_DISABLED BIT(17)
+#define IAVF_FLAG_SETUP_NETDEV_FEATURES BIT(18)
+#define IAVF_FLAG_REINIT_MSIX_NEEDED BIT(20)
+#define IAVF_FLAG_FDIR_ENABLED BIT(21)
+/* duplicates for common code */
+#define IAVF_FLAG_DCB_ENABLED 0
+ /* flags for admin queue service task */
+ u64 aq_required;
+#define IAVF_FLAG_AQ_ENABLE_QUEUES BIT_ULL(0)
+#define IAVF_FLAG_AQ_DISABLE_QUEUES BIT_ULL(1)
+#define IAVF_FLAG_AQ_ADD_MAC_FILTER BIT_ULL(2)
+#define IAVF_FLAG_AQ_ADD_VLAN_FILTER BIT_ULL(3)
+#define IAVF_FLAG_AQ_DEL_MAC_FILTER BIT_ULL(4)
+#define IAVF_FLAG_AQ_DEL_VLAN_FILTER BIT_ULL(5)
+#define IAVF_FLAG_AQ_CONFIGURE_QUEUES BIT_ULL(6)
+#define IAVF_FLAG_AQ_MAP_VECTORS BIT_ULL(7)
+#define IAVF_FLAG_AQ_HANDLE_RESET BIT_ULL(8)
+#define IAVF_FLAG_AQ_CONFIGURE_RSS BIT_ULL(9) /* direct AQ config */
+#define IAVF_FLAG_AQ_GET_CONFIG BIT_ULL(10)
+/* Newer style, RSS done by the PF so we can ignore hardware vagaries. */
+#define IAVF_FLAG_AQ_GET_HENA BIT_ULL(11)
+#define IAVF_FLAG_AQ_SET_HENA BIT_ULL(12)
+#define IAVF_FLAG_AQ_SET_RSS_KEY BIT_ULL(13)
+#define IAVF_FLAG_AQ_SET_RSS_LUT BIT_ULL(14)
+#define IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE BIT_ULL(15)
+#define IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING BIT_ULL(19)
+#define IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING BIT_ULL(20)
+#define IAVF_FLAG_AQ_ENABLE_CHANNELS BIT_ULL(21)
+#define IAVF_FLAG_AQ_DISABLE_CHANNELS BIT_ULL(22)
+#define IAVF_FLAG_AQ_ADD_CLOUD_FILTER BIT_ULL(23)
+#define IAVF_FLAG_AQ_DEL_CLOUD_FILTER BIT_ULL(24)
+#define IAVF_FLAG_AQ_ADD_FDIR_FILTER BIT_ULL(25)
+#define IAVF_FLAG_AQ_DEL_FDIR_FILTER BIT_ULL(26)
+#define IAVF_FLAG_AQ_ADD_ADV_RSS_CFG BIT_ULL(27)
+#define IAVF_FLAG_AQ_DEL_ADV_RSS_CFG BIT_ULL(28)
+#define IAVF_FLAG_AQ_REQUEST_STATS BIT_ULL(29)
+#define IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS BIT_ULL(30)
+#define IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING BIT_ULL(31)
+#define IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING BIT_ULL(32)
+#define IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING BIT_ULL(33)
+#define IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING BIT_ULL(34)
+#define IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION BIT_ULL(35)
+#define IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION BIT_ULL(36)
+#define IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION BIT_ULL(37)
+#define IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION BIT_ULL(38)
+
+ /* flags for processing extended capability messages during
+ * __IAVF_INIT_EXTENDED_CAPS. Each capability exchange requires
+ * both a SEND and a RECV step, which must be processed in sequence.
+ *
+ * During the __IAVF_INIT_EXTENDED_CAPS state, the driver will
+ * process one flag at a time during each state loop.
+ */
+ u64 extended_caps;
+#define IAVF_EXTENDED_CAP_SEND_VLAN_V2 BIT_ULL(0)
+#define IAVF_EXTENDED_CAP_RECV_VLAN_V2 BIT_ULL(1)
+
+#define IAVF_EXTENDED_CAPS \
+ (IAVF_EXTENDED_CAP_SEND_VLAN_V2 | \
+ IAVF_EXTENDED_CAP_RECV_VLAN_V2)
+
+ /* Lock to prevent possible clobbering of
+ * current_netdev_promisc_flags
+ */
+ spinlock_t current_netdev_promisc_flags_lock;
+ netdev_features_t current_netdev_promisc_flags;
+
+ /* OS defined structs */
+ struct net_device *netdev;
+ struct pci_dev *pdev;
+
+ struct iavf_hw hw; /* defined in iavf_type.h */
+
+ enum iavf_state_t state;
+ enum iavf_state_t last_state;
+ unsigned long crit_section;
+
+ struct delayed_work watchdog_task;
+ bool netdev_registered;
+ bool link_up;
+ enum virtchnl_link_speed link_speed;
+ /* This is only populated if the VIRTCHNL_VF_CAP_ADV_LINK_SPEED is set
+ * in vf_res->vf_cap_flags. Use ADV_LINK_SUPPORT macro to determine if
+ * this field is valid. This field should be used going forward and the
+ * enum virtchnl_link_speed above should be considered the legacy way of
+ * storing/communicating link speeds.
+ */
+ u32 link_speed_mbps;
+
+ enum virtchnl_ops current_op;
+#define CLIENT_ALLOWED(_a) ((_a)->vf_res ? \
+ (_a)->vf_res->vf_cap_flags & \
+ VIRTCHNL_VF_OFFLOAD_RDMA : \
+ 0)
+#define CLIENT_ENABLED(_a) ((_a)->cinst)
+/* RSS by the PF should be preferred over RSS via other methods. */
+#define RSS_PF(_a) ((_a)->vf_res->vf_cap_flags & \
+ VIRTCHNL_VF_OFFLOAD_RSS_PF)
+#define RSS_AQ(_a) ((_a)->vf_res->vf_cap_flags & \
+ VIRTCHNL_VF_OFFLOAD_RSS_AQ)
+#define RSS_REG(_a) (!((_a)->vf_res->vf_cap_flags & \
+ (VIRTCHNL_VF_OFFLOAD_RSS_AQ | \
+ VIRTCHNL_VF_OFFLOAD_RSS_PF)))
+#define VLAN_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \
+ VIRTCHNL_VF_OFFLOAD_VLAN)
+#define VLAN_V2_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \
+ VIRTCHNL_VF_OFFLOAD_VLAN_V2)
+#define VLAN_V2_FILTERING_ALLOWED(_a) \
+ (VLAN_V2_ALLOWED((_a)) && \
+ ((_a)->vlan_v2_caps.filtering.filtering_support.outer || \
+ (_a)->vlan_v2_caps.filtering.filtering_support.inner))
+#define VLAN_FILTERING_ALLOWED(_a) \
+ (VLAN_ALLOWED((_a)) || VLAN_V2_FILTERING_ALLOWED((_a)))
+#define ADV_LINK_SUPPORT(_a) ((_a)->vf_res->vf_cap_flags & \
+ VIRTCHNL_VF_CAP_ADV_LINK_SPEED)
+#define FDIR_FLTR_SUPPORT(_a) ((_a)->vf_res->vf_cap_flags & \
+ VIRTCHNL_VF_OFFLOAD_FDIR_PF)
+#define ADV_RSS_SUPPORT(_a) ((_a)->vf_res->vf_cap_flags & \
+ VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF)
+ struct virtchnl_vf_resource *vf_res; /* incl. all VSIs */
+ struct virtchnl_vsi_resource *vsi_res; /* our LAN VSI */
+ struct virtchnl_version_info pf_version;
+#define PF_IS_V11(_a) (((_a)->pf_version.major == 1) && \
+ ((_a)->pf_version.minor == 1))
+ struct virtchnl_vlan_caps vlan_v2_caps;
+ u16 msg_enable;
+ struct iavf_eth_stats current_stats;
+ struct iavf_vsi vsi;
+ u32 aq_wait_count;
+ /* RSS stuff */
+ u64 hena;
+ u16 rss_key_size;
+ u16 rss_lut_size;
+ u8 *rss_key;
+ u8 *rss_lut;
+ /* ADQ related members */
+ struct iavf_channel_config ch_config;
+ u8 num_tc;
+ struct list_head cloud_filter_list;
+ /* lock to protect access to the cloud filter list */
+ spinlock_t cloud_filter_list_lock;
+ u16 num_cloud_filters;
+ /* snapshot of "num_active_queues" before setup_tc for qdisc add
+ * is invoked. This information is useful during qdisc del flow,
+ * to restore correct number of queues
+ */
+ int orig_num_active_queues;
+
+#define IAVF_MAX_FDIR_FILTERS 128 /* max allowed Flow Director filters */
+ u16 fdir_active_fltr;
+ struct list_head fdir_list_head;
+ spinlock_t fdir_fltr_lock; /* protect the Flow Director filter list */
+
+ struct list_head adv_rss_list_head;
+ spinlock_t adv_rss_lock; /* protect the RSS management list */
+};
+
+
+/* Ethtool Private Flags */
+
+/* lan device, used by client interface */
+struct iavf_device {
+ struct list_head list;
+ struct iavf_adapter *vf;
+};
+
+/* needed by iavf_ethtool.c */
+extern char iavf_driver_name[];
+
+static inline const char *iavf_state_str(enum iavf_state_t state)
+{
+ switch (state) {
+ case __IAVF_STARTUP:
+ return "__IAVF_STARTUP";
+ case __IAVF_REMOVE:
+ return "__IAVF_REMOVE";
+ case __IAVF_INIT_VERSION_CHECK:
+ return "__IAVF_INIT_VERSION_CHECK";
+ case __IAVF_INIT_GET_RESOURCES:
+ return "__IAVF_INIT_GET_RESOURCES";
+ case __IAVF_INIT_EXTENDED_CAPS:
+ return "__IAVF_INIT_EXTENDED_CAPS";
+ case __IAVF_INIT_CONFIG_ADAPTER:
+ return "__IAVF_INIT_CONFIG_ADAPTER";
+ case __IAVF_INIT_SW:
+ return "__IAVF_INIT_SW";
+ case __IAVF_INIT_FAILED:
+ return "__IAVF_INIT_FAILED";
+ case __IAVF_RESETTING:
+ return "__IAVF_RESETTING";
+ case __IAVF_COMM_FAILED:
+ return "__IAVF_COMM_FAILED";
+ case __IAVF_DOWN:
+ return "__IAVF_DOWN";
+ case __IAVF_DOWN_PENDING:
+ return "__IAVF_DOWN_PENDING";
+ case __IAVF_TESTING:
+ return "__IAVF_TESTING";
+ case __IAVF_RUNNING:
+ return "__IAVF_RUNNING";
+ default:
+ return "__IAVF_UNKNOWN_STATE";
+ }
+}
+
+static inline void iavf_change_state(struct iavf_adapter *adapter,
+ enum iavf_state_t state)
+{
+ if (adapter->state != state) {
+ adapter->last_state = adapter->state;
+ adapter->state = state;
+ }
+ dev_dbg(&adapter->pdev->dev,
+ "state transition from:%s to:%s\n",
+ iavf_state_str(adapter->last_state),
+ iavf_state_str(adapter->state));
+}
+
+int iavf_up(struct iavf_adapter *adapter);
+void iavf_down(struct iavf_adapter *adapter);
+int iavf_process_config(struct iavf_adapter *adapter);
+int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter);
+void iavf_schedule_reset(struct iavf_adapter *adapter, u64 flags);
+void iavf_schedule_aq_request(struct iavf_adapter *adapter, u64 flags);
+void iavf_schedule_finish_config(struct iavf_adapter *adapter);
+void iavf_reset(struct iavf_adapter *adapter);
+void iavf_set_ethtool_ops(struct net_device *netdev);
+void iavf_update_stats(struct iavf_adapter *adapter);
+void iavf_free_all_tx_resources(struct iavf_adapter *adapter);
+void iavf_free_all_rx_resources(struct iavf_adapter *adapter);
+
+void iavf_napi_add_all(struct iavf_adapter *adapter);
+void iavf_napi_del_all(struct iavf_adapter *adapter);
+
+int iavf_send_api_ver(struct iavf_adapter *adapter);
+int iavf_verify_api_ver(struct iavf_adapter *adapter);
+int iavf_send_vf_config_msg(struct iavf_adapter *adapter);
+int iavf_get_vf_config(struct iavf_adapter *adapter);
+int iavf_get_vf_vlan_v2_caps(struct iavf_adapter *adapter);
+int iavf_send_vf_offload_vlan_v2_msg(struct iavf_adapter *adapter);
+void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter);
+u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter);
+void iavf_irq_enable(struct iavf_adapter *adapter, bool flush);
+void iavf_configure_queues(struct iavf_adapter *adapter);
+void iavf_deconfigure_queues(struct iavf_adapter *adapter);
+void iavf_enable_queues(struct iavf_adapter *adapter);
+void iavf_disable_queues(struct iavf_adapter *adapter);
+void iavf_map_queues(struct iavf_adapter *adapter);
+int iavf_request_queues(struct iavf_adapter *adapter, int num);
+void iavf_add_ether_addrs(struct iavf_adapter *adapter);
+void iavf_del_ether_addrs(struct iavf_adapter *adapter);
+void iavf_add_vlans(struct iavf_adapter *adapter);
+void iavf_del_vlans(struct iavf_adapter *adapter);
+void iavf_set_promiscuous(struct iavf_adapter *adapter);
+bool iavf_promiscuous_mode_changed(struct iavf_adapter *adapter);
+void iavf_request_stats(struct iavf_adapter *adapter);
+int iavf_request_reset(struct iavf_adapter *adapter);
+void iavf_get_hena(struct iavf_adapter *adapter);
+void iavf_set_hena(struct iavf_adapter *adapter);
+void iavf_set_rss_key(struct iavf_adapter *adapter);
+void iavf_set_rss_lut(struct iavf_adapter *adapter);
+void iavf_enable_vlan_stripping(struct iavf_adapter *adapter);
+void iavf_disable_vlan_stripping(struct iavf_adapter *adapter);
+void iavf_virtchnl_completion(struct iavf_adapter *adapter,
+ enum virtchnl_ops v_opcode,
+ enum iavf_status v_retval, u8 *msg, u16 msglen);
+int iavf_config_rss(struct iavf_adapter *adapter);
+int iavf_lan_add_device(struct iavf_adapter *adapter);
+int iavf_lan_del_device(struct iavf_adapter *adapter);
+void iavf_client_subtask(struct iavf_adapter *adapter);
+void iavf_notify_client_message(struct iavf_vsi *vsi, u8 *msg, u16 len);
+void iavf_notify_client_l2_params(struct iavf_vsi *vsi);
+void iavf_notify_client_open(struct iavf_vsi *vsi);
+void iavf_notify_client_close(struct iavf_vsi *vsi, bool reset);
+void iavf_enable_channels(struct iavf_adapter *adapter);
+void iavf_disable_channels(struct iavf_adapter *adapter);
+void iavf_add_cloud_filter(struct iavf_adapter *adapter);
+void iavf_del_cloud_filter(struct iavf_adapter *adapter);
+void iavf_enable_vlan_stripping_v2(struct iavf_adapter *adapter, u16 tpid);
+void iavf_disable_vlan_stripping_v2(struct iavf_adapter *adapter, u16 tpid);
+void iavf_enable_vlan_insertion_v2(struct iavf_adapter *adapter, u16 tpid);
+void iavf_disable_vlan_insertion_v2(struct iavf_adapter *adapter, u16 tpid);
+void iavf_add_fdir_filter(struct iavf_adapter *adapter);
+void iavf_del_fdir_filter(struct iavf_adapter *adapter);
+void iavf_add_adv_rss_cfg(struct iavf_adapter *adapter);
+void iavf_del_adv_rss_cfg(struct iavf_adapter *adapter);
+struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
+ const u8 *macaddr);
+int iavf_wait_for_reset(struct iavf_adapter *adapter);
+#endif /* _IAVF_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_adminq.c b/drivers/net/ethernet/intel/iavf/iavf_adminq.c
new file mode 100644
index 0000000000..9ffbd24d83
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_adminq.c
@@ -0,0 +1,946 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#include "iavf_status.h"
+#include "iavf_type.h"
+#include "iavf_register.h"
+#include "iavf_adminq.h"
+#include "iavf_prototype.h"
+
+/**
+ * iavf_adminq_init_regs - Initialize AdminQ registers
+ * @hw: pointer to the hardware structure
+ *
+ * This assumes the alloc_asq and alloc_arq functions have already been called
+ **/
+static void iavf_adminq_init_regs(struct iavf_hw *hw)
+{
+ /* set head and tail registers in our local struct */
+ hw->aq.asq.tail = IAVF_VF_ATQT1;
+ hw->aq.asq.head = IAVF_VF_ATQH1;
+ hw->aq.asq.len = IAVF_VF_ATQLEN1;
+ hw->aq.asq.bal = IAVF_VF_ATQBAL1;
+ hw->aq.asq.bah = IAVF_VF_ATQBAH1;
+ hw->aq.arq.tail = IAVF_VF_ARQT1;
+ hw->aq.arq.head = IAVF_VF_ARQH1;
+ hw->aq.arq.len = IAVF_VF_ARQLEN1;
+ hw->aq.arq.bal = IAVF_VF_ARQBAL1;
+ hw->aq.arq.bah = IAVF_VF_ARQBAH1;
+}
+
+/**
+ * iavf_alloc_adminq_asq_ring - Allocate Admin Queue send rings
+ * @hw: pointer to the hardware structure
+ **/
+static enum iavf_status iavf_alloc_adminq_asq_ring(struct iavf_hw *hw)
+{
+ enum iavf_status ret_code;
+
+ ret_code = iavf_allocate_dma_mem(hw, &hw->aq.asq.desc_buf,
+ iavf_mem_atq_ring,
+ (hw->aq.num_asq_entries *
+ sizeof(struct iavf_aq_desc)),
+ IAVF_ADMINQ_DESC_ALIGNMENT);
+ if (ret_code)
+ return ret_code;
+
+ ret_code = iavf_allocate_virt_mem(hw, &hw->aq.asq.cmd_buf,
+ (hw->aq.num_asq_entries *
+ sizeof(struct iavf_asq_cmd_details)));
+ if (ret_code) {
+ iavf_free_dma_mem(hw, &hw->aq.asq.desc_buf);
+ return ret_code;
+ }
+
+ return ret_code;
+}
+
+/**
+ * iavf_alloc_adminq_arq_ring - Allocate Admin Queue receive rings
+ * @hw: pointer to the hardware structure
+ **/
+static enum iavf_status iavf_alloc_adminq_arq_ring(struct iavf_hw *hw)
+{
+ enum iavf_status ret_code;
+
+ ret_code = iavf_allocate_dma_mem(hw, &hw->aq.arq.desc_buf,
+ iavf_mem_arq_ring,
+ (hw->aq.num_arq_entries *
+ sizeof(struct iavf_aq_desc)),
+ IAVF_ADMINQ_DESC_ALIGNMENT);
+
+ return ret_code;
+}
+
+/**
+ * iavf_free_adminq_asq - Free Admin Queue send rings
+ * @hw: pointer to the hardware structure
+ *
+ * This assumes the posted send buffers have already been cleaned
+ * and de-allocated
+ **/
+static void iavf_free_adminq_asq(struct iavf_hw *hw)
+{
+ iavf_free_dma_mem(hw, &hw->aq.asq.desc_buf);
+}
+
+/**
+ * iavf_free_adminq_arq - Free Admin Queue receive rings
+ * @hw: pointer to the hardware structure
+ *
+ * This assumes the posted receive buffers have already been cleaned
+ * and de-allocated
+ **/
+static void iavf_free_adminq_arq(struct iavf_hw *hw)
+{
+ iavf_free_dma_mem(hw, &hw->aq.arq.desc_buf);
+}
+
+/**
+ * iavf_alloc_arq_bufs - Allocate pre-posted buffers for the receive queue
+ * @hw: pointer to the hardware structure
+ **/
+static enum iavf_status iavf_alloc_arq_bufs(struct iavf_hw *hw)
+{
+ struct iavf_aq_desc *desc;
+ struct iavf_dma_mem *bi;
+ enum iavf_status ret_code;
+ int i;
+
+ /* We'll be allocating the buffer info memory first, then we can
+ * allocate the mapped buffers for the event processing
+ */
+
+ /* buffer_info structures do not need alignment */
+ ret_code = iavf_allocate_virt_mem(hw, &hw->aq.arq.dma_head,
+ (hw->aq.num_arq_entries *
+ sizeof(struct iavf_dma_mem)));
+ if (ret_code)
+ goto alloc_arq_bufs;
+ hw->aq.arq.r.arq_bi = (struct iavf_dma_mem *)hw->aq.arq.dma_head.va;
+
+ /* allocate the mapped buffers */
+ for (i = 0; i < hw->aq.num_arq_entries; i++) {
+ bi = &hw->aq.arq.r.arq_bi[i];
+ ret_code = iavf_allocate_dma_mem(hw, bi,
+ iavf_mem_arq_buf,
+ hw->aq.arq_buf_size,
+ IAVF_ADMINQ_DESC_ALIGNMENT);
+ if (ret_code)
+ goto unwind_alloc_arq_bufs;
+
+ /* now configure the descriptors for use */
+ desc = IAVF_ADMINQ_DESC(hw->aq.arq, i);
+
+ desc->flags = cpu_to_le16(IAVF_AQ_FLAG_BUF);
+ if (hw->aq.arq_buf_size > IAVF_AQ_LARGE_BUF)
+ desc->flags |= cpu_to_le16(IAVF_AQ_FLAG_LB);
+ desc->opcode = 0;
+ /* This is in accordance with Admin queue design, there is no
+ * register for buffer size configuration
+ */
+ desc->datalen = cpu_to_le16((u16)bi->size);
+ desc->retval = 0;
+ desc->cookie_high = 0;
+ desc->cookie_low = 0;
+ desc->params.external.addr_high =
+ cpu_to_le32(upper_32_bits(bi->pa));
+ desc->params.external.addr_low =
+ cpu_to_le32(lower_32_bits(bi->pa));
+ desc->params.external.param0 = 0;
+ desc->params.external.param1 = 0;
+ }
+
+alloc_arq_bufs:
+ return ret_code;
+
+unwind_alloc_arq_bufs:
+ /* don't try to free the one that failed... */
+ i--;
+ for (; i >= 0; i--)
+ iavf_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);
+ iavf_free_virt_mem(hw, &hw->aq.arq.dma_head);
+
+ return ret_code;
+}
+
+/**
+ * iavf_alloc_asq_bufs - Allocate empty buffer structs for the send queue
+ * @hw: pointer to the hardware structure
+ **/
+static enum iavf_status iavf_alloc_asq_bufs(struct iavf_hw *hw)
+{
+ struct iavf_dma_mem *bi;
+ enum iavf_status ret_code;
+ int i;
+
+ /* No mapped memory needed yet, just the buffer info structures */
+ ret_code = iavf_allocate_virt_mem(hw, &hw->aq.asq.dma_head,
+ (hw->aq.num_asq_entries *
+ sizeof(struct iavf_dma_mem)));
+ if (ret_code)
+ goto alloc_asq_bufs;
+ hw->aq.asq.r.asq_bi = (struct iavf_dma_mem *)hw->aq.asq.dma_head.va;
+
+ /* allocate the mapped buffers */
+ for (i = 0; i < hw->aq.num_asq_entries; i++) {
+ bi = &hw->aq.asq.r.asq_bi[i];
+ ret_code = iavf_allocate_dma_mem(hw, bi,
+ iavf_mem_asq_buf,
+ hw->aq.asq_buf_size,
+ IAVF_ADMINQ_DESC_ALIGNMENT);
+ if (ret_code)
+ goto unwind_alloc_asq_bufs;
+ }
+alloc_asq_bufs:
+ return ret_code;
+
+unwind_alloc_asq_bufs:
+ /* don't try to free the one that failed... */
+ i--;
+ for (; i >= 0; i--)
+ iavf_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);
+ iavf_free_virt_mem(hw, &hw->aq.asq.dma_head);
+
+ return ret_code;
+}
+
+/**
+ * iavf_free_arq_bufs - Free receive queue buffer info elements
+ * @hw: pointer to the hardware structure
+ **/
+static void iavf_free_arq_bufs(struct iavf_hw *hw)
+{
+ int i;
+
+ /* free descriptors */
+ for (i = 0; i < hw->aq.num_arq_entries; i++)
+ iavf_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);
+
+ /* free the descriptor memory */
+ iavf_free_dma_mem(hw, &hw->aq.arq.desc_buf);
+
+ /* free the dma header */
+ iavf_free_virt_mem(hw, &hw->aq.arq.dma_head);
+}
+
+/**
+ * iavf_free_asq_bufs - Free send queue buffer info elements
+ * @hw: pointer to the hardware structure
+ **/
+static void iavf_free_asq_bufs(struct iavf_hw *hw)
+{
+ int i;
+
+ /* only unmap if the address is non-NULL */
+ for (i = 0; i < hw->aq.num_asq_entries; i++)
+ if (hw->aq.asq.r.asq_bi[i].pa)
+ iavf_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);
+
+ /* free the buffer info list */
+ iavf_free_virt_mem(hw, &hw->aq.asq.cmd_buf);
+
+ /* free the descriptor memory */
+ iavf_free_dma_mem(hw, &hw->aq.asq.desc_buf);
+
+ /* free the dma header */
+ iavf_free_virt_mem(hw, &hw->aq.asq.dma_head);
+}
+
+/**
+ * iavf_config_asq_regs - configure ASQ registers
+ * @hw: pointer to the hardware structure
+ *
+ * Configure base address and length registers for the transmit queue
+ **/
+static enum iavf_status iavf_config_asq_regs(struct iavf_hw *hw)
+{
+ enum iavf_status ret_code = 0;
+ u32 reg = 0;
+
+ /* Clear Head and Tail */
+ wr32(hw, hw->aq.asq.head, 0);
+ wr32(hw, hw->aq.asq.tail, 0);
+
+ /* set starting point */
+ wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries |
+ IAVF_VF_ATQLEN1_ATQENABLE_MASK));
+ wr32(hw, hw->aq.asq.bal, lower_32_bits(hw->aq.asq.desc_buf.pa));
+ wr32(hw, hw->aq.asq.bah, upper_32_bits(hw->aq.asq.desc_buf.pa));
+
+ /* Check one register to verify that config was applied */
+ reg = rd32(hw, hw->aq.asq.bal);
+ if (reg != lower_32_bits(hw->aq.asq.desc_buf.pa))
+ ret_code = IAVF_ERR_ADMIN_QUEUE_ERROR;
+
+ return ret_code;
+}
+
+/**
+ * iavf_config_arq_regs - ARQ register configuration
+ * @hw: pointer to the hardware structure
+ *
+ * Configure base address and length registers for the receive (event queue)
+ **/
+static enum iavf_status iavf_config_arq_regs(struct iavf_hw *hw)
+{
+ enum iavf_status ret_code = 0;
+ u32 reg = 0;
+
+ /* Clear Head and Tail */
+ wr32(hw, hw->aq.arq.head, 0);
+ wr32(hw, hw->aq.arq.tail, 0);
+
+ /* set starting point */
+ wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries |
+ IAVF_VF_ARQLEN1_ARQENABLE_MASK));
+ wr32(hw, hw->aq.arq.bal, lower_32_bits(hw->aq.arq.desc_buf.pa));
+ wr32(hw, hw->aq.arq.bah, upper_32_bits(hw->aq.arq.desc_buf.pa));
+
+ /* Update tail in the HW to post pre-allocated buffers */
+ wr32(hw, hw->aq.arq.tail, hw->aq.num_arq_entries - 1);
+
+ /* Check one register to verify that config was applied */
+ reg = rd32(hw, hw->aq.arq.bal);
+ if (reg != lower_32_bits(hw->aq.arq.desc_buf.pa))
+ ret_code = IAVF_ERR_ADMIN_QUEUE_ERROR;
+
+ return ret_code;
+}
+
+/**
+ * iavf_init_asq - main initialization routine for ASQ
+ * @hw: pointer to the hardware structure
+ *
+ * This is the main initialization routine for the Admin Send Queue
+ * Prior to calling this function, drivers *MUST* set the following fields
+ * in the hw->aq structure:
+ * - hw->aq.num_asq_entries
+ * - hw->aq.arq_buf_size
+ *
+ * Do *NOT* hold the lock when calling this as the memory allocation routines
+ * called are not going to be atomic context safe
+ **/
+static enum iavf_status iavf_init_asq(struct iavf_hw *hw)
+{
+ enum iavf_status ret_code = 0;
+ int i;
+
+ if (hw->aq.asq.count > 0) {
+ /* queue already initialized */
+ ret_code = IAVF_ERR_NOT_READY;
+ goto init_adminq_exit;
+ }
+
+ /* verify input for valid configuration */
+ if ((hw->aq.num_asq_entries == 0) ||
+ (hw->aq.asq_buf_size == 0)) {
+ ret_code = IAVF_ERR_CONFIG;
+ goto init_adminq_exit;
+ }
+
+ hw->aq.asq.next_to_use = 0;
+ hw->aq.asq.next_to_clean = 0;
+
+ /* allocate the ring memory */
+ ret_code = iavf_alloc_adminq_asq_ring(hw);
+ if (ret_code)
+ goto init_adminq_exit;
+
+ /* allocate buffers in the rings */
+ ret_code = iavf_alloc_asq_bufs(hw);
+ if (ret_code)
+ goto init_adminq_free_rings;
+
+ /* initialize base registers */
+ ret_code = iavf_config_asq_regs(hw);
+ if (ret_code)
+ goto init_free_asq_bufs;
+
+ /* success! */
+ hw->aq.asq.count = hw->aq.num_asq_entries;
+ goto init_adminq_exit;
+
+init_free_asq_bufs:
+ for (i = 0; i < hw->aq.num_asq_entries; i++)
+ iavf_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);
+ iavf_free_virt_mem(hw, &hw->aq.asq.dma_head);
+
+init_adminq_free_rings:
+ iavf_free_adminq_asq(hw);
+
+init_adminq_exit:
+ return ret_code;
+}
+
+/**
+ * iavf_init_arq - initialize ARQ
+ * @hw: pointer to the hardware structure
+ *
+ * The main initialization routine for the Admin Receive (Event) Queue.
+ * Prior to calling this function, drivers *MUST* set the following fields
+ * in the hw->aq structure:
+ * - hw->aq.num_asq_entries
+ * - hw->aq.arq_buf_size
+ *
+ * Do *NOT* hold the lock when calling this as the memory allocation routines
+ * called are not going to be atomic context safe
+ **/
+static enum iavf_status iavf_init_arq(struct iavf_hw *hw)
+{
+ enum iavf_status ret_code = 0;
+ int i;
+
+ if (hw->aq.arq.count > 0) {
+ /* queue already initialized */
+ ret_code = IAVF_ERR_NOT_READY;
+ goto init_adminq_exit;
+ }
+
+ /* verify input for valid configuration */
+ if ((hw->aq.num_arq_entries == 0) ||
+ (hw->aq.arq_buf_size == 0)) {
+ ret_code = IAVF_ERR_CONFIG;
+ goto init_adminq_exit;
+ }
+
+ hw->aq.arq.next_to_use = 0;
+ hw->aq.arq.next_to_clean = 0;
+
+ /* allocate the ring memory */
+ ret_code = iavf_alloc_adminq_arq_ring(hw);
+ if (ret_code)
+ goto init_adminq_exit;
+
+ /* allocate buffers in the rings */
+ ret_code = iavf_alloc_arq_bufs(hw);
+ if (ret_code)
+ goto init_adminq_free_rings;
+
+ /* initialize base registers */
+ ret_code = iavf_config_arq_regs(hw);
+ if (ret_code)
+ goto init_free_arq_bufs;
+
+ /* success! */
+ hw->aq.arq.count = hw->aq.num_arq_entries;
+ goto init_adminq_exit;
+
+init_free_arq_bufs:
+ for (i = 0; i < hw->aq.num_arq_entries; i++)
+ iavf_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);
+ iavf_free_virt_mem(hw, &hw->aq.arq.dma_head);
+init_adminq_free_rings:
+ iavf_free_adminq_arq(hw);
+
+init_adminq_exit:
+ return ret_code;
+}
+
+/**
+ * iavf_shutdown_asq - shutdown the ASQ
+ * @hw: pointer to the hardware structure
+ *
+ * The main shutdown routine for the Admin Send Queue
+ **/
+static enum iavf_status iavf_shutdown_asq(struct iavf_hw *hw)
+{
+ enum iavf_status ret_code = 0;
+
+ mutex_lock(&hw->aq.asq_mutex);
+
+ if (hw->aq.asq.count == 0) {
+ ret_code = IAVF_ERR_NOT_READY;
+ goto shutdown_asq_out;
+ }
+
+ /* Stop firmware AdminQ processing */
+ wr32(hw, hw->aq.asq.head, 0);
+ wr32(hw, hw->aq.asq.tail, 0);
+ wr32(hw, hw->aq.asq.len, 0);
+ wr32(hw, hw->aq.asq.bal, 0);
+ wr32(hw, hw->aq.asq.bah, 0);
+
+ hw->aq.asq.count = 0; /* to indicate uninitialized queue */
+
+ /* free ring buffers */
+ iavf_free_asq_bufs(hw);
+
+shutdown_asq_out:
+ mutex_unlock(&hw->aq.asq_mutex);
+ return ret_code;
+}
+
+/**
+ * iavf_shutdown_arq - shutdown ARQ
+ * @hw: pointer to the hardware structure
+ *
+ * The main shutdown routine for the Admin Receive Queue
+ **/
+static enum iavf_status iavf_shutdown_arq(struct iavf_hw *hw)
+{
+ enum iavf_status ret_code = 0;
+
+ mutex_lock(&hw->aq.arq_mutex);
+
+ if (hw->aq.arq.count == 0) {
+ ret_code = IAVF_ERR_NOT_READY;
+ goto shutdown_arq_out;
+ }
+
+ /* Stop firmware AdminQ processing */
+ wr32(hw, hw->aq.arq.head, 0);
+ wr32(hw, hw->aq.arq.tail, 0);
+ wr32(hw, hw->aq.arq.len, 0);
+ wr32(hw, hw->aq.arq.bal, 0);
+ wr32(hw, hw->aq.arq.bah, 0);
+
+ hw->aq.arq.count = 0; /* to indicate uninitialized queue */
+
+ /* free ring buffers */
+ iavf_free_arq_bufs(hw);
+
+shutdown_arq_out:
+ mutex_unlock(&hw->aq.arq_mutex);
+ return ret_code;
+}
+
+/**
+ * iavf_init_adminq - main initialization routine for Admin Queue
+ * @hw: pointer to the hardware structure
+ *
+ * Prior to calling this function, drivers *MUST* set the following fields
+ * in the hw->aq structure:
+ * - hw->aq.num_asq_entries
+ * - hw->aq.num_arq_entries
+ * - hw->aq.arq_buf_size
+ * - hw->aq.asq_buf_size
+ **/
+enum iavf_status iavf_init_adminq(struct iavf_hw *hw)
+{
+ enum iavf_status ret_code;
+
+ /* verify input for valid configuration */
+ if ((hw->aq.num_arq_entries == 0) ||
+ (hw->aq.num_asq_entries == 0) ||
+ (hw->aq.arq_buf_size == 0) ||
+ (hw->aq.asq_buf_size == 0)) {
+ ret_code = IAVF_ERR_CONFIG;
+ goto init_adminq_exit;
+ }
+
+ /* Set up register offsets */
+ iavf_adminq_init_regs(hw);
+
+ /* setup ASQ command write back timeout */
+ hw->aq.asq_cmd_timeout = IAVF_ASQ_CMD_TIMEOUT;
+
+ /* allocate the ASQ */
+ ret_code = iavf_init_asq(hw);
+ if (ret_code)
+ goto init_adminq_destroy_locks;
+
+ /* allocate the ARQ */
+ ret_code = iavf_init_arq(hw);
+ if (ret_code)
+ goto init_adminq_free_asq;
+
+ /* success! */
+ goto init_adminq_exit;
+
+init_adminq_free_asq:
+ iavf_shutdown_asq(hw);
+init_adminq_destroy_locks:
+
+init_adminq_exit:
+ return ret_code;
+}
+
+/**
+ * iavf_shutdown_adminq - shutdown routine for the Admin Queue
+ * @hw: pointer to the hardware structure
+ **/
+enum iavf_status iavf_shutdown_adminq(struct iavf_hw *hw)
+{
+ if (iavf_check_asq_alive(hw))
+ iavf_aq_queue_shutdown(hw, true);
+
+ iavf_shutdown_asq(hw);
+ iavf_shutdown_arq(hw);
+
+ return 0;
+}
+
+/**
+ * iavf_clean_asq - cleans Admin send queue
+ * @hw: pointer to the hardware structure
+ *
+ * returns the number of free desc
+ **/
+static u16 iavf_clean_asq(struct iavf_hw *hw)
+{
+ struct iavf_adminq_ring *asq = &hw->aq.asq;
+ struct iavf_asq_cmd_details *details;
+ u16 ntc = asq->next_to_clean;
+ struct iavf_aq_desc desc_cb;
+ struct iavf_aq_desc *desc;
+
+ desc = IAVF_ADMINQ_DESC(*asq, ntc);
+ details = IAVF_ADMINQ_DETAILS(*asq, ntc);
+ while (rd32(hw, hw->aq.asq.head) != ntc) {
+ iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
+ "ntc %d head %d.\n", ntc, rd32(hw, hw->aq.asq.head));
+
+ if (details->callback) {
+ IAVF_ADMINQ_CALLBACK cb_func =
+ (IAVF_ADMINQ_CALLBACK)details->callback;
+ desc_cb = *desc;
+ cb_func(hw, &desc_cb);
+ }
+ memset((void *)desc, 0, sizeof(struct iavf_aq_desc));
+ memset((void *)details, 0,
+ sizeof(struct iavf_asq_cmd_details));
+ ntc++;
+ if (ntc == asq->count)
+ ntc = 0;
+ desc = IAVF_ADMINQ_DESC(*asq, ntc);
+ details = IAVF_ADMINQ_DETAILS(*asq, ntc);
+ }
+
+ asq->next_to_clean = ntc;
+
+ return IAVF_DESC_UNUSED(asq);
+}
+
+/**
+ * iavf_asq_done - check if FW has processed the Admin Send Queue
+ * @hw: pointer to the hw struct
+ *
+ * Returns true if the firmware has processed all descriptors on the
+ * admin send queue. Returns false if there are still requests pending.
+ **/
+bool iavf_asq_done(struct iavf_hw *hw)
+{
+ /* AQ designers suggest use of head for better
+ * timing reliability than DD bit
+ */
+ return rd32(hw, hw->aq.asq.head) == hw->aq.asq.next_to_use;
+}
+
+/**
+ * iavf_asq_send_command - send command to Admin Queue
+ * @hw: pointer to the hw struct
+ * @desc: prefilled descriptor describing the command (non DMA mem)
+ * @buff: buffer to use for indirect commands
+ * @buff_size: size of buffer for indirect commands
+ * @cmd_details: pointer to command details structure
+ *
+ * This is the main send command driver routine for the Admin Queue send
+ * queue. It runs the queue, cleans the queue, etc
+ **/
+enum iavf_status iavf_asq_send_command(struct iavf_hw *hw,
+ struct iavf_aq_desc *desc,
+ void *buff, /* can be NULL */
+ u16 buff_size,
+ struct iavf_asq_cmd_details *cmd_details)
+{
+ struct iavf_dma_mem *dma_buff = NULL;
+ struct iavf_asq_cmd_details *details;
+ struct iavf_aq_desc *desc_on_ring;
+ bool cmd_completed = false;
+ enum iavf_status status = 0;
+ u16 retval = 0;
+ u32 val = 0;
+
+ mutex_lock(&hw->aq.asq_mutex);
+
+ if (hw->aq.asq.count == 0) {
+ iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
+ "AQTX: Admin queue not initialized.\n");
+ status = IAVF_ERR_QUEUE_EMPTY;
+ goto asq_send_command_error;
+ }
+
+ hw->aq.asq_last_status = IAVF_AQ_RC_OK;
+
+ val = rd32(hw, hw->aq.asq.head);
+ if (val >= hw->aq.num_asq_entries) {
+ iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
+ "AQTX: head overrun at %d\n", val);
+ status = IAVF_ERR_QUEUE_EMPTY;
+ goto asq_send_command_error;
+ }
+
+ details = IAVF_ADMINQ_DETAILS(hw->aq.asq, hw->aq.asq.next_to_use);
+ if (cmd_details) {
+ *details = *cmd_details;
+
+ /* If the cmd_details are defined copy the cookie. The
+ * cpu_to_le32 is not needed here because the data is ignored
+ * by the FW, only used by the driver
+ */
+ if (details->cookie) {
+ desc->cookie_high =
+ cpu_to_le32(upper_32_bits(details->cookie));
+ desc->cookie_low =
+ cpu_to_le32(lower_32_bits(details->cookie));
+ }
+ } else {
+ memset(details, 0, sizeof(struct iavf_asq_cmd_details));
+ }
+
+ /* clear requested flags and then set additional flags if defined */
+ desc->flags &= ~cpu_to_le16(details->flags_dis);
+ desc->flags |= cpu_to_le16(details->flags_ena);
+
+ if (buff_size > hw->aq.asq_buf_size) {
+ iavf_debug(hw,
+ IAVF_DEBUG_AQ_MESSAGE,
+ "AQTX: Invalid buffer size: %d.\n",
+ buff_size);
+ status = IAVF_ERR_INVALID_SIZE;
+ goto asq_send_command_error;
+ }
+
+ if (details->postpone && !details->async) {
+ iavf_debug(hw,
+ IAVF_DEBUG_AQ_MESSAGE,
+ "AQTX: Async flag not set along with postpone flag");
+ status = IAVF_ERR_PARAM;
+ goto asq_send_command_error;
+ }
+
+ /* call clean and check queue available function to reclaim the
+ * descriptors that were processed by FW, the function returns the
+ * number of desc available
+ */
+ /* the clean function called here could be called in a separate thread
+ * in case of asynchronous completions
+ */
+ if (iavf_clean_asq(hw) == 0) {
+ iavf_debug(hw,
+ IAVF_DEBUG_AQ_MESSAGE,
+ "AQTX: Error queue is full.\n");
+ status = IAVF_ERR_ADMIN_QUEUE_FULL;
+ goto asq_send_command_error;
+ }
+
+ /* initialize the temp desc pointer with the right desc */
+ desc_on_ring = IAVF_ADMINQ_DESC(hw->aq.asq, hw->aq.asq.next_to_use);
+
+ /* if the desc is available copy the temp desc to the right place */
+ *desc_on_ring = *desc;
+
+ /* if buff is not NULL assume indirect command */
+ if (buff) {
+ dma_buff = &hw->aq.asq.r.asq_bi[hw->aq.asq.next_to_use];
+ /* copy the user buff into the respective DMA buff */
+ memcpy(dma_buff->va, buff, buff_size);
+ desc_on_ring->datalen = cpu_to_le16(buff_size);
+
+ /* Update the address values in the desc with the pa value
+ * for respective buffer
+ */
+ desc_on_ring->params.external.addr_high =
+ cpu_to_le32(upper_32_bits(dma_buff->pa));
+ desc_on_ring->params.external.addr_low =
+ cpu_to_le32(lower_32_bits(dma_buff->pa));
+ }
+
+ /* bump the tail */
+ iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE, "AQTX: desc and buffer:\n");
+ iavf_debug_aq(hw, IAVF_DEBUG_AQ_COMMAND, (void *)desc_on_ring,
+ buff, buff_size);
+ (hw->aq.asq.next_to_use)++;
+ if (hw->aq.asq.next_to_use == hw->aq.asq.count)
+ hw->aq.asq.next_to_use = 0;
+ if (!details->postpone)
+ wr32(hw, hw->aq.asq.tail, hw->aq.asq.next_to_use);
+
+ /* if cmd_details are not defined or async flag is not set,
+ * we need to wait for desc write back
+ */
+ if (!details->async && !details->postpone) {
+ u32 total_delay = 0;
+
+ do {
+ /* AQ designers suggest use of head for better
+ * timing reliability than DD bit
+ */
+ if (iavf_asq_done(hw))
+ break;
+ udelay(50);
+ total_delay += 50;
+ } while (total_delay < hw->aq.asq_cmd_timeout);
+ }
+
+ /* if ready, copy the desc back to temp */
+ if (iavf_asq_done(hw)) {
+ *desc = *desc_on_ring;
+ if (buff)
+ memcpy(buff, dma_buff->va, buff_size);
+ retval = le16_to_cpu(desc->retval);
+ if (retval != 0) {
+ iavf_debug(hw,
+ IAVF_DEBUG_AQ_MESSAGE,
+ "AQTX: Command completed with error 0x%X.\n",
+ retval);
+
+ /* strip off FW internal code */
+ retval &= 0xff;
+ }
+ cmd_completed = true;
+ if ((enum iavf_admin_queue_err)retval == IAVF_AQ_RC_OK)
+ status = 0;
+ else if ((enum iavf_admin_queue_err)retval == IAVF_AQ_RC_EBUSY)
+ status = IAVF_ERR_NOT_READY;
+ else
+ status = IAVF_ERR_ADMIN_QUEUE_ERROR;
+ hw->aq.asq_last_status = (enum iavf_admin_queue_err)retval;
+ }
+
+ iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
+ "AQTX: desc and buffer writeback:\n");
+ iavf_debug_aq(hw, IAVF_DEBUG_AQ_COMMAND, (void *)desc, buff, buff_size);
+
+ /* save writeback aq if requested */
+ if (details->wb_desc)
+ *details->wb_desc = *desc_on_ring;
+
+ /* update the error if time out occurred */
+ if ((!cmd_completed) &&
+ (!details->async && !details->postpone)) {
+ if (rd32(hw, hw->aq.asq.len) & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {
+ iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
+ "AQTX: AQ Critical error.\n");
+ status = IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR;
+ } else {
+ iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
+ "AQTX: Writeback timeout.\n");
+ status = IAVF_ERR_ADMIN_QUEUE_TIMEOUT;
+ }
+ }
+
+asq_send_command_error:
+ mutex_unlock(&hw->aq.asq_mutex);
+ return status;
+}
+
+/**
+ * iavf_fill_default_direct_cmd_desc - AQ descriptor helper function
+ * @desc: pointer to the temp descriptor (non DMA mem)
+ * @opcode: the opcode can be used to decide which flags to turn off or on
+ *
+ * Fill the desc with default values
+ **/
+void iavf_fill_default_direct_cmd_desc(struct iavf_aq_desc *desc, u16 opcode)
+{
+ /* zero out the desc */
+ memset((void *)desc, 0, sizeof(struct iavf_aq_desc));
+ desc->opcode = cpu_to_le16(opcode);
+ desc->flags = cpu_to_le16(IAVF_AQ_FLAG_SI);
+}
+
+/**
+ * iavf_clean_arq_element
+ * @hw: pointer to the hw struct
+ * @e: event info from the receive descriptor, includes any buffers
+ * @pending: number of events that could be left to process
+ *
+ * This function cleans one Admin Receive Queue element and returns
+ * the contents through e. It can also return how many events are
+ * left to process through 'pending'
+ **/
+enum iavf_status iavf_clean_arq_element(struct iavf_hw *hw,
+ struct iavf_arq_event_info *e,
+ u16 *pending)
+{
+ u16 ntc = hw->aq.arq.next_to_clean;
+ struct iavf_aq_desc *desc;
+ enum iavf_status ret_code = 0;
+ struct iavf_dma_mem *bi;
+ u16 desc_idx;
+ u16 datalen;
+ u16 flags;
+ u16 ntu;
+
+ /* pre-clean the event info */
+ memset(&e->desc, 0, sizeof(e->desc));
+
+ /* take the lock before we start messing with the ring */
+ mutex_lock(&hw->aq.arq_mutex);
+
+ if (hw->aq.arq.count == 0) {
+ iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
+ "AQRX: Admin queue not initialized.\n");
+ ret_code = IAVF_ERR_QUEUE_EMPTY;
+ goto clean_arq_element_err;
+ }
+
+ /* set next_to_use to head */
+ ntu = rd32(hw, hw->aq.arq.head) & IAVF_VF_ARQH1_ARQH_MASK;
+ if (ntu == ntc) {
+ /* nothing to do - shouldn't need to update ring's values */
+ ret_code = IAVF_ERR_ADMIN_QUEUE_NO_WORK;
+ goto clean_arq_element_out;
+ }
+
+ /* now clean the next descriptor */
+ desc = IAVF_ADMINQ_DESC(hw->aq.arq, ntc);
+ desc_idx = ntc;
+
+ hw->aq.arq_last_status =
+ (enum iavf_admin_queue_err)le16_to_cpu(desc->retval);
+ flags = le16_to_cpu(desc->flags);
+ if (flags & IAVF_AQ_FLAG_ERR) {
+ ret_code = IAVF_ERR_ADMIN_QUEUE_ERROR;
+ iavf_debug(hw,
+ IAVF_DEBUG_AQ_MESSAGE,
+ "AQRX: Event received with error 0x%X.\n",
+ hw->aq.arq_last_status);
+ }
+
+ e->desc = *desc;
+ datalen = le16_to_cpu(desc->datalen);
+ e->msg_len = min(datalen, e->buf_len);
+ if (e->msg_buf && (e->msg_len != 0))
+ memcpy(e->msg_buf, hw->aq.arq.r.arq_bi[desc_idx].va,
+ e->msg_len);
+
+ iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE, "AQRX: desc and buffer:\n");
+ iavf_debug_aq(hw, IAVF_DEBUG_AQ_COMMAND, (void *)desc, e->msg_buf,
+ hw->aq.arq_buf_size);
+
+ /* Restore the original datalen and buffer address in the desc,
+ * FW updates datalen to indicate the event message
+ * size
+ */
+ bi = &hw->aq.arq.r.arq_bi[ntc];
+ memset((void *)desc, 0, sizeof(struct iavf_aq_desc));
+
+ desc->flags = cpu_to_le16(IAVF_AQ_FLAG_BUF);
+ if (hw->aq.arq_buf_size > IAVF_AQ_LARGE_BUF)
+ desc->flags |= cpu_to_le16(IAVF_AQ_FLAG_LB);
+ desc->datalen = cpu_to_le16((u16)bi->size);
+ desc->params.external.addr_high = cpu_to_le32(upper_32_bits(bi->pa));
+ desc->params.external.addr_low = cpu_to_le32(lower_32_bits(bi->pa));
+
+ /* set tail = the last cleaned desc index. */
+ wr32(hw, hw->aq.arq.tail, ntc);
+ /* ntc is updated to tail + 1 */
+ ntc++;
+ if (ntc == hw->aq.num_arq_entries)
+ ntc = 0;
+ hw->aq.arq.next_to_clean = ntc;
+ hw->aq.arq.next_to_use = ntu;
+
+clean_arq_element_out:
+ /* Set pending if needed, unlock and return */
+ if (pending)
+ *pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc);
+
+clean_arq_element_err:
+ mutex_unlock(&hw->aq.arq_mutex);
+
+ return ret_code;
+}
diff --git a/drivers/net/ethernet/intel/iavf/iavf_adminq.h b/drivers/net/ethernet/intel/iavf/iavf_adminq.h
new file mode 100644
index 0000000000..1f60518eb0
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_adminq.h
@@ -0,0 +1,135 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_ADMINQ_H_
+#define _IAVF_ADMINQ_H_
+
+#include "iavf_osdep.h"
+#include "iavf_status.h"
+#include "iavf_adminq_cmd.h"
+
+#define IAVF_ADMINQ_DESC(R, i) \
+ (&(((struct iavf_aq_desc *)((R).desc_buf.va))[i]))
+
+#define IAVF_ADMINQ_DESC_ALIGNMENT 4096
+
+struct iavf_adminq_ring {
+ struct iavf_virt_mem dma_head; /* space for dma structures */
+ struct iavf_dma_mem desc_buf; /* descriptor ring memory */
+ struct iavf_virt_mem cmd_buf; /* command buffer memory */
+
+ union {
+ struct iavf_dma_mem *asq_bi;
+ struct iavf_dma_mem *arq_bi;
+ } r;
+
+ u16 count; /* Number of descriptors */
+ u16 rx_buf_len; /* Admin Receive Queue buffer length */
+
+ /* used for interrupt processing */
+ u16 next_to_use;
+ u16 next_to_clean;
+
+ /* used for queue tracking */
+ u32 head;
+ u32 tail;
+ u32 len;
+ u32 bah;
+ u32 bal;
+};
+
+/* ASQ transaction details */
+struct iavf_asq_cmd_details {
+ void *callback; /* cast from type IAVF_ADMINQ_CALLBACK */
+ u64 cookie;
+ u16 flags_ena;
+ u16 flags_dis;
+ bool async;
+ bool postpone;
+ struct iavf_aq_desc *wb_desc;
+};
+
+#define IAVF_ADMINQ_DETAILS(R, i) \
+ (&(((struct iavf_asq_cmd_details *)((R).cmd_buf.va))[i]))
+
+/* ARQ event information */
+struct iavf_arq_event_info {
+ struct iavf_aq_desc desc;
+ u16 msg_len;
+ u16 buf_len;
+ u8 *msg_buf;
+};
+
+/* Admin Queue information */
+struct iavf_adminq_info {
+ struct iavf_adminq_ring arq; /* receive queue */
+ struct iavf_adminq_ring asq; /* send queue */
+ u32 asq_cmd_timeout; /* send queue cmd write back timeout*/
+ u16 num_arq_entries; /* receive queue depth */
+ u16 num_asq_entries; /* send queue depth */
+ u16 arq_buf_size; /* receive queue buffer size */
+ u16 asq_buf_size; /* send queue buffer size */
+ u16 fw_maj_ver; /* firmware major version */
+ u16 fw_min_ver; /* firmware minor version */
+ u32 fw_build; /* firmware build number */
+ u16 api_maj_ver; /* api major version */
+ u16 api_min_ver; /* api minor version */
+
+ struct mutex asq_mutex; /* Send queue lock */
+ struct mutex arq_mutex; /* Receive queue lock */
+
+ /* last status values on send and receive queues */
+ enum iavf_admin_queue_err asq_last_status;
+ enum iavf_admin_queue_err arq_last_status;
+};
+
+/**
+ * iavf_aq_rc_to_posix - convert errors to user-land codes
+ * @aq_ret: AdminQ handler error code can override aq_rc
+ * @aq_rc: AdminQ firmware error code to convert
+ **/
+static inline int iavf_aq_rc_to_posix(int aq_ret, int aq_rc)
+{
+ int aq_to_posix[] = {
+ 0, /* IAVF_AQ_RC_OK */
+ -EPERM, /* IAVF_AQ_RC_EPERM */
+ -ENOENT, /* IAVF_AQ_RC_ENOENT */
+ -ESRCH, /* IAVF_AQ_RC_ESRCH */
+ -EINTR, /* IAVF_AQ_RC_EINTR */
+ -EIO, /* IAVF_AQ_RC_EIO */
+ -ENXIO, /* IAVF_AQ_RC_ENXIO */
+ -E2BIG, /* IAVF_AQ_RC_E2BIG */
+ -EAGAIN, /* IAVF_AQ_RC_EAGAIN */
+ -ENOMEM, /* IAVF_AQ_RC_ENOMEM */
+ -EACCES, /* IAVF_AQ_RC_EACCES */
+ -EFAULT, /* IAVF_AQ_RC_EFAULT */
+ -EBUSY, /* IAVF_AQ_RC_EBUSY */
+ -EEXIST, /* IAVF_AQ_RC_EEXIST */
+ -EINVAL, /* IAVF_AQ_RC_EINVAL */
+ -ENOTTY, /* IAVF_AQ_RC_ENOTTY */
+ -ENOSPC, /* IAVF_AQ_RC_ENOSPC */
+ -ENOSYS, /* IAVF_AQ_RC_ENOSYS */
+ -ERANGE, /* IAVF_AQ_RC_ERANGE */
+ -EPIPE, /* IAVF_AQ_RC_EFLUSHED */
+ -ESPIPE, /* IAVF_AQ_RC_BAD_ADDR */
+ -EROFS, /* IAVF_AQ_RC_EMODE */
+ -EFBIG, /* IAVF_AQ_RC_EFBIG */
+ };
+
+ /* aq_rc is invalid if AQ timed out */
+ if (aq_ret == IAVF_ERR_ADMIN_QUEUE_TIMEOUT)
+ return -EAGAIN;
+
+ if (!((u32)aq_rc < (sizeof(aq_to_posix) / sizeof((aq_to_posix)[0]))))
+ return -ERANGE;
+
+ return aq_to_posix[aq_rc];
+}
+
+/* general information */
+#define IAVF_AQ_LARGE_BUF 512
+#define IAVF_ASQ_CMD_TIMEOUT 250000 /* usecs */
+
+void iavf_fill_default_direct_cmd_desc(struct iavf_aq_desc *desc, u16 opcode);
+
+#endif /* _IAVF_ADMINQ_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_adminq_cmd.h b/drivers/net/ethernet/intel/iavf/iavf_adminq_cmd.h
new file mode 100644
index 0000000000..bc51230855
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_adminq_cmd.h
@@ -0,0 +1,528 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_ADMINQ_CMD_H_
+#define _IAVF_ADMINQ_CMD_H_
+
+/* This header file defines the iavf Admin Queue commands and is shared between
+ * iavf Firmware and Software.
+ *
+ * This file needs to comply with the Linux Kernel coding style.
+ */
+
+#define IAVF_FW_API_VERSION_MAJOR 0x0001
+#define IAVF_FW_API_VERSION_MINOR_X722 0x0005
+#define IAVF_FW_API_VERSION_MINOR_X710 0x0008
+
+#define IAVF_FW_MINOR_VERSION(_h) ((_h)->mac.type == IAVF_MAC_XL710 ? \
+ IAVF_FW_API_VERSION_MINOR_X710 : \
+ IAVF_FW_API_VERSION_MINOR_X722)
+
+/* API version 1.7 implements additional link and PHY-specific APIs */
+#define IAVF_MINOR_VER_GET_LINK_INFO_XL710 0x0007
+
+struct iavf_aq_desc {
+ __le16 flags;
+ __le16 opcode;
+ __le16 datalen;
+ __le16 retval;
+ __le32 cookie_high;
+ __le32 cookie_low;
+ union {
+ struct {
+ __le32 param0;
+ __le32 param1;
+ __le32 param2;
+ __le32 param3;
+ } internal;
+ struct {
+ __le32 param0;
+ __le32 param1;
+ __le32 addr_high;
+ __le32 addr_low;
+ } external;
+ u8 raw[16];
+ } params;
+};
+
+/* Flags sub-structure
+ * |0 |1 |2 |3 |4 |5 |6 |7 |8 |9 |10 |11 |12 |13 |14 |15 |
+ * |DD |CMP|ERR|VFE| * * RESERVED * * |LB |RD |VFC|BUF|SI |EI |FE |
+ */
+
+/* command flags and offsets*/
+#define IAVF_AQ_FLAG_DD_SHIFT 0
+#define IAVF_AQ_FLAG_CMP_SHIFT 1
+#define IAVF_AQ_FLAG_ERR_SHIFT 2
+#define IAVF_AQ_FLAG_VFE_SHIFT 3
+#define IAVF_AQ_FLAG_LB_SHIFT 9
+#define IAVF_AQ_FLAG_RD_SHIFT 10
+#define IAVF_AQ_FLAG_VFC_SHIFT 11
+#define IAVF_AQ_FLAG_BUF_SHIFT 12
+#define IAVF_AQ_FLAG_SI_SHIFT 13
+#define IAVF_AQ_FLAG_EI_SHIFT 14
+#define IAVF_AQ_FLAG_FE_SHIFT 15
+
+#define IAVF_AQ_FLAG_DD BIT(IAVF_AQ_FLAG_DD_SHIFT) /* 0x1 */
+#define IAVF_AQ_FLAG_CMP BIT(IAVF_AQ_FLAG_CMP_SHIFT) /* 0x2 */
+#define IAVF_AQ_FLAG_ERR BIT(IAVF_AQ_FLAG_ERR_SHIFT) /* 0x4 */
+#define IAVF_AQ_FLAG_VFE BIT(IAVF_AQ_FLAG_VFE_SHIFT) /* 0x8 */
+#define IAVF_AQ_FLAG_LB BIT(IAVF_AQ_FLAG_LB_SHIFT) /* 0x200 */
+#define IAVF_AQ_FLAG_RD BIT(IAVF_AQ_FLAG_RD_SHIFT) /* 0x400 */
+#define IAVF_AQ_FLAG_VFC BIT(IAVF_AQ_FLAG_VFC_SHIFT) /* 0x800 */
+#define IAVF_AQ_FLAG_BUF BIT(IAVF_AQ_FLAG_BUF_SHIFT) /* 0x1000 */
+#define IAVF_AQ_FLAG_SI BIT(IAVF_AQ_FLAG_SI_SHIFT) /* 0x2000 */
+#define IAVF_AQ_FLAG_EI BIT(IAVF_AQ_FLAG_EI_SHIFT) /* 0x4000 */
+#define IAVF_AQ_FLAG_FE BIT(IAVF_AQ_FLAG_FE_SHIFT) /* 0x8000 */
+
+/* error codes */
+enum iavf_admin_queue_err {
+ IAVF_AQ_RC_OK = 0, /* success */
+ IAVF_AQ_RC_EPERM = 1, /* Operation not permitted */
+ IAVF_AQ_RC_ENOENT = 2, /* No such element */
+ IAVF_AQ_RC_ESRCH = 3, /* Bad opcode */
+ IAVF_AQ_RC_EINTR = 4, /* operation interrupted */
+ IAVF_AQ_RC_EIO = 5, /* I/O error */
+ IAVF_AQ_RC_ENXIO = 6, /* No such resource */
+ IAVF_AQ_RC_E2BIG = 7, /* Arg too long */
+ IAVF_AQ_RC_EAGAIN = 8, /* Try again */
+ IAVF_AQ_RC_ENOMEM = 9, /* Out of memory */
+ IAVF_AQ_RC_EACCES = 10, /* Permission denied */
+ IAVF_AQ_RC_EFAULT = 11, /* Bad address */
+ IAVF_AQ_RC_EBUSY = 12, /* Device or resource busy */
+ IAVF_AQ_RC_EEXIST = 13, /* object already exists */
+ IAVF_AQ_RC_EINVAL = 14, /* Invalid argument */
+ IAVF_AQ_RC_ENOTTY = 15, /* Not a typewriter */
+ IAVF_AQ_RC_ENOSPC = 16, /* No space left or alloc failure */
+ IAVF_AQ_RC_ENOSYS = 17, /* Function not implemented */
+ IAVF_AQ_RC_ERANGE = 18, /* Parameter out of range */
+ IAVF_AQ_RC_EFLUSHED = 19, /* Cmd flushed due to prev cmd error */
+ IAVF_AQ_RC_BAD_ADDR = 20, /* Descriptor contains a bad pointer */
+ IAVF_AQ_RC_EMODE = 21, /* Op not allowed in current dev mode */
+ IAVF_AQ_RC_EFBIG = 22, /* File too large */
+};
+
+/* Admin Queue command opcodes */
+enum iavf_admin_queue_opc {
+ /* aq commands */
+ iavf_aqc_opc_get_version = 0x0001,
+ iavf_aqc_opc_driver_version = 0x0002,
+ iavf_aqc_opc_queue_shutdown = 0x0003,
+ iavf_aqc_opc_set_pf_context = 0x0004,
+
+ /* resource ownership */
+ iavf_aqc_opc_request_resource = 0x0008,
+ iavf_aqc_opc_release_resource = 0x0009,
+
+ iavf_aqc_opc_list_func_capabilities = 0x000A,
+ iavf_aqc_opc_list_dev_capabilities = 0x000B,
+
+ /* Proxy commands */
+ iavf_aqc_opc_set_proxy_config = 0x0104,
+ iavf_aqc_opc_set_ns_proxy_table_entry = 0x0105,
+
+ /* LAA */
+ iavf_aqc_opc_mac_address_read = 0x0107,
+ iavf_aqc_opc_mac_address_write = 0x0108,
+
+ /* PXE */
+ iavf_aqc_opc_clear_pxe_mode = 0x0110,
+
+ /* WoL commands */
+ iavf_aqc_opc_set_wol_filter = 0x0120,
+ iavf_aqc_opc_get_wake_reason = 0x0121,
+
+ /* internal switch commands */
+ iavf_aqc_opc_get_switch_config = 0x0200,
+ iavf_aqc_opc_add_statistics = 0x0201,
+ iavf_aqc_opc_remove_statistics = 0x0202,
+ iavf_aqc_opc_set_port_parameters = 0x0203,
+ iavf_aqc_opc_get_switch_resource_alloc = 0x0204,
+ iavf_aqc_opc_set_switch_config = 0x0205,
+ iavf_aqc_opc_rx_ctl_reg_read = 0x0206,
+ iavf_aqc_opc_rx_ctl_reg_write = 0x0207,
+
+ iavf_aqc_opc_add_vsi = 0x0210,
+ iavf_aqc_opc_update_vsi_parameters = 0x0211,
+ iavf_aqc_opc_get_vsi_parameters = 0x0212,
+
+ iavf_aqc_opc_add_pv = 0x0220,
+ iavf_aqc_opc_update_pv_parameters = 0x0221,
+ iavf_aqc_opc_get_pv_parameters = 0x0222,
+
+ iavf_aqc_opc_add_veb = 0x0230,
+ iavf_aqc_opc_update_veb_parameters = 0x0231,
+ iavf_aqc_opc_get_veb_parameters = 0x0232,
+
+ iavf_aqc_opc_delete_element = 0x0243,
+
+ iavf_aqc_opc_add_macvlan = 0x0250,
+ iavf_aqc_opc_remove_macvlan = 0x0251,
+ iavf_aqc_opc_add_vlan = 0x0252,
+ iavf_aqc_opc_remove_vlan = 0x0253,
+ iavf_aqc_opc_set_vsi_promiscuous_modes = 0x0254,
+ iavf_aqc_opc_add_tag = 0x0255,
+ iavf_aqc_opc_remove_tag = 0x0256,
+ iavf_aqc_opc_add_multicast_etag = 0x0257,
+ iavf_aqc_opc_remove_multicast_etag = 0x0258,
+ iavf_aqc_opc_update_tag = 0x0259,
+ iavf_aqc_opc_add_control_packet_filter = 0x025A,
+ iavf_aqc_opc_remove_control_packet_filter = 0x025B,
+ iavf_aqc_opc_add_cloud_filters = 0x025C,
+ iavf_aqc_opc_remove_cloud_filters = 0x025D,
+ iavf_aqc_opc_clear_wol_switch_filters = 0x025E,
+
+ iavf_aqc_opc_add_mirror_rule = 0x0260,
+ iavf_aqc_opc_delete_mirror_rule = 0x0261,
+
+ /* Dynamic Device Personalization */
+ iavf_aqc_opc_write_personalization_profile = 0x0270,
+ iavf_aqc_opc_get_personalization_profile_list = 0x0271,
+
+ /* DCB commands */
+ iavf_aqc_opc_dcb_ignore_pfc = 0x0301,
+ iavf_aqc_opc_dcb_updated = 0x0302,
+ iavf_aqc_opc_set_dcb_parameters = 0x0303,
+
+ /* TX scheduler */
+ iavf_aqc_opc_configure_vsi_bw_limit = 0x0400,
+ iavf_aqc_opc_configure_vsi_ets_sla_bw_limit = 0x0406,
+ iavf_aqc_opc_configure_vsi_tc_bw = 0x0407,
+ iavf_aqc_opc_query_vsi_bw_config = 0x0408,
+ iavf_aqc_opc_query_vsi_ets_sla_config = 0x040A,
+ iavf_aqc_opc_configure_switching_comp_bw_limit = 0x0410,
+
+ iavf_aqc_opc_enable_switching_comp_ets = 0x0413,
+ iavf_aqc_opc_modify_switching_comp_ets = 0x0414,
+ iavf_aqc_opc_disable_switching_comp_ets = 0x0415,
+ iavf_aqc_opc_configure_switching_comp_ets_bw_limit = 0x0416,
+ iavf_aqc_opc_configure_switching_comp_bw_config = 0x0417,
+ iavf_aqc_opc_query_switching_comp_ets_config = 0x0418,
+ iavf_aqc_opc_query_port_ets_config = 0x0419,
+ iavf_aqc_opc_query_switching_comp_bw_config = 0x041A,
+ iavf_aqc_opc_suspend_port_tx = 0x041B,
+ iavf_aqc_opc_resume_port_tx = 0x041C,
+ iavf_aqc_opc_configure_partition_bw = 0x041D,
+ /* hmc */
+ iavf_aqc_opc_query_hmc_resource_profile = 0x0500,
+ iavf_aqc_opc_set_hmc_resource_profile = 0x0501,
+
+ /* phy commands*/
+ iavf_aqc_opc_get_phy_abilities = 0x0600,
+ iavf_aqc_opc_set_phy_config = 0x0601,
+ iavf_aqc_opc_set_mac_config = 0x0603,
+ iavf_aqc_opc_set_link_restart_an = 0x0605,
+ iavf_aqc_opc_get_link_status = 0x0607,
+ iavf_aqc_opc_set_phy_int_mask = 0x0613,
+ iavf_aqc_opc_get_local_advt_reg = 0x0614,
+ iavf_aqc_opc_set_local_advt_reg = 0x0615,
+ iavf_aqc_opc_get_partner_advt = 0x0616,
+ iavf_aqc_opc_set_lb_modes = 0x0618,
+ iavf_aqc_opc_get_phy_wol_caps = 0x0621,
+ iavf_aqc_opc_set_phy_debug = 0x0622,
+ iavf_aqc_opc_upload_ext_phy_fm = 0x0625,
+ iavf_aqc_opc_run_phy_activity = 0x0626,
+ iavf_aqc_opc_set_phy_register = 0x0628,
+ iavf_aqc_opc_get_phy_register = 0x0629,
+
+ /* NVM commands */
+ iavf_aqc_opc_nvm_read = 0x0701,
+ iavf_aqc_opc_nvm_erase = 0x0702,
+ iavf_aqc_opc_nvm_update = 0x0703,
+ iavf_aqc_opc_nvm_config_read = 0x0704,
+ iavf_aqc_opc_nvm_config_write = 0x0705,
+ iavf_aqc_opc_oem_post_update = 0x0720,
+ iavf_aqc_opc_thermal_sensor = 0x0721,
+
+ /* virtualization commands */
+ iavf_aqc_opc_send_msg_to_pf = 0x0801,
+ iavf_aqc_opc_send_msg_to_vf = 0x0802,
+ iavf_aqc_opc_send_msg_to_peer = 0x0803,
+
+ /* alternate structure */
+ iavf_aqc_opc_alternate_write = 0x0900,
+ iavf_aqc_opc_alternate_write_indirect = 0x0901,
+ iavf_aqc_opc_alternate_read = 0x0902,
+ iavf_aqc_opc_alternate_read_indirect = 0x0903,
+ iavf_aqc_opc_alternate_write_done = 0x0904,
+ iavf_aqc_opc_alternate_set_mode = 0x0905,
+ iavf_aqc_opc_alternate_clear_port = 0x0906,
+
+ /* LLDP commands */
+ iavf_aqc_opc_lldp_get_mib = 0x0A00,
+ iavf_aqc_opc_lldp_update_mib = 0x0A01,
+ iavf_aqc_opc_lldp_add_tlv = 0x0A02,
+ iavf_aqc_opc_lldp_update_tlv = 0x0A03,
+ iavf_aqc_opc_lldp_delete_tlv = 0x0A04,
+ iavf_aqc_opc_lldp_stop = 0x0A05,
+ iavf_aqc_opc_lldp_start = 0x0A06,
+
+ /* Tunnel commands */
+ iavf_aqc_opc_add_udp_tunnel = 0x0B00,
+ iavf_aqc_opc_del_udp_tunnel = 0x0B01,
+ iavf_aqc_opc_set_rss_key = 0x0B02,
+ iavf_aqc_opc_set_rss_lut = 0x0B03,
+ iavf_aqc_opc_get_rss_key = 0x0B04,
+ iavf_aqc_opc_get_rss_lut = 0x0B05,
+
+ /* Async Events */
+ iavf_aqc_opc_event_lan_overflow = 0x1001,
+
+ /* OEM commands */
+ iavf_aqc_opc_oem_parameter_change = 0xFE00,
+ iavf_aqc_opc_oem_device_status_change = 0xFE01,
+ iavf_aqc_opc_oem_ocsd_initialize = 0xFE02,
+ iavf_aqc_opc_oem_ocbb_initialize = 0xFE03,
+
+ /* debug commands */
+ iavf_aqc_opc_debug_read_reg = 0xFF03,
+ iavf_aqc_opc_debug_write_reg = 0xFF04,
+ iavf_aqc_opc_debug_modify_reg = 0xFF07,
+ iavf_aqc_opc_debug_dump_internals = 0xFF08,
+};
+
+/* command structures and indirect data structures */
+
+/* Structure naming conventions:
+ * - no suffix for direct command descriptor structures
+ * - _data for indirect sent data
+ * - _resp for indirect return data (data which is both will use _data)
+ * - _completion for direct return data
+ * - _element_ for repeated elements (may also be _data or _resp)
+ *
+ * Command structures are expected to overlay the params.raw member of the basic
+ * descriptor, and as such cannot exceed 16 bytes in length.
+ */
+
+/* This macro is used to generate a compilation error if a structure
+ * is not exactly the correct length. It gives a divide by zero error if the
+ * structure is not of the correct size, otherwise it creates an enum that is
+ * never used.
+ */
+#define IAVF_CHECK_STRUCT_LEN(n, X) enum iavf_static_assert_enum_##X \
+ { iavf_static_assert_##X = (n) / ((sizeof(struct X) == (n)) ? 1 : 0) }
+
+/* This macro is used extensively to ensure that command structures are 16
+ * bytes in length as they have to map to the raw array of that size.
+ */
+#define IAVF_CHECK_CMD_LENGTH(X) IAVF_CHECK_STRUCT_LEN(16, X)
+
+/* Queue Shutdown (direct 0x0003) */
+struct iavf_aqc_queue_shutdown {
+ __le32 driver_unloading;
+#define IAVF_AQ_DRIVER_UNLOADING 0x1
+ u8 reserved[12];
+};
+
+IAVF_CHECK_CMD_LENGTH(iavf_aqc_queue_shutdown);
+
+struct iavf_aqc_vsi_properties_data {
+ /* first 96 byte are written by SW */
+ __le16 valid_sections;
+#define IAVF_AQ_VSI_PROP_SWITCH_VALID 0x0001
+#define IAVF_AQ_VSI_PROP_SECURITY_VALID 0x0002
+#define IAVF_AQ_VSI_PROP_VLAN_VALID 0x0004
+#define IAVF_AQ_VSI_PROP_CAS_PV_VALID 0x0008
+#define IAVF_AQ_VSI_PROP_INGRESS_UP_VALID 0x0010
+#define IAVF_AQ_VSI_PROP_EGRESS_UP_VALID 0x0020
+#define IAVF_AQ_VSI_PROP_QUEUE_MAP_VALID 0x0040
+#define IAVF_AQ_VSI_PROP_QUEUE_OPT_VALID 0x0080
+#define IAVF_AQ_VSI_PROP_OUTER_UP_VALID 0x0100
+#define IAVF_AQ_VSI_PROP_SCHED_VALID 0x0200
+ /* switch section */
+ __le16 switch_id; /* 12bit id combined with flags below */
+#define IAVF_AQ_VSI_SW_ID_SHIFT 0x0000
+#define IAVF_AQ_VSI_SW_ID_MASK (0xFFF << IAVF_AQ_VSI_SW_ID_SHIFT)
+#define IAVF_AQ_VSI_SW_ID_FLAG_NOT_STAG 0x1000
+#define IAVF_AQ_VSI_SW_ID_FLAG_ALLOW_LB 0x2000
+#define IAVF_AQ_VSI_SW_ID_FLAG_LOCAL_LB 0x4000
+ u8 sw_reserved[2];
+ /* security section */
+ u8 sec_flags;
+#define IAVF_AQ_VSI_SEC_FLAG_ALLOW_DEST_OVRD 0x01
+#define IAVF_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK 0x02
+#define IAVF_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK 0x04
+ u8 sec_reserved;
+ /* VLAN section */
+ __le16 pvid; /* VLANS include priority bits */
+ __le16 fcoe_pvid;
+ u8 port_vlan_flags;
+#define IAVF_AQ_VSI_PVLAN_MODE_SHIFT 0x00
+#define IAVF_AQ_VSI_PVLAN_MODE_MASK (0x03 << \
+ IAVF_AQ_VSI_PVLAN_MODE_SHIFT)
+#define IAVF_AQ_VSI_PVLAN_MODE_TAGGED 0x01
+#define IAVF_AQ_VSI_PVLAN_MODE_UNTAGGED 0x02
+#define IAVF_AQ_VSI_PVLAN_MODE_ALL 0x03
+#define IAVF_AQ_VSI_PVLAN_INSERT_PVID 0x04
+#define IAVF_AQ_VSI_PVLAN_EMOD_SHIFT 0x03
+#define IAVF_AQ_VSI_PVLAN_EMOD_MASK (0x3 << \
+ IAVF_AQ_VSI_PVLAN_EMOD_SHIFT)
+#define IAVF_AQ_VSI_PVLAN_EMOD_STR_BOTH 0x0
+#define IAVF_AQ_VSI_PVLAN_EMOD_STR_UP 0x08
+#define IAVF_AQ_VSI_PVLAN_EMOD_STR 0x10
+#define IAVF_AQ_VSI_PVLAN_EMOD_NOTHING 0x18
+ u8 pvlan_reserved[3];
+ /* ingress egress up sections */
+ __le32 ingress_table; /* bitmap, 3 bits per up */
+#define IAVF_AQ_VSI_UP_TABLE_UP0_SHIFT 0
+#define IAVF_AQ_VSI_UP_TABLE_UP0_MASK (0x7 << \
+ IAVF_AQ_VSI_UP_TABLE_UP0_SHIFT)
+#define IAVF_AQ_VSI_UP_TABLE_UP1_SHIFT 3
+#define IAVF_AQ_VSI_UP_TABLE_UP1_MASK (0x7 << \
+ IAVF_AQ_VSI_UP_TABLE_UP1_SHIFT)
+#define IAVF_AQ_VSI_UP_TABLE_UP2_SHIFT 6
+#define IAVF_AQ_VSI_UP_TABLE_UP2_MASK (0x7 << \
+ IAVF_AQ_VSI_UP_TABLE_UP2_SHIFT)
+#define IAVF_AQ_VSI_UP_TABLE_UP3_SHIFT 9
+#define IAVF_AQ_VSI_UP_TABLE_UP3_MASK (0x7 << \
+ IAVF_AQ_VSI_UP_TABLE_UP3_SHIFT)
+#define IAVF_AQ_VSI_UP_TABLE_UP4_SHIFT 12
+#define IAVF_AQ_VSI_UP_TABLE_UP4_MASK (0x7 << \
+ IAVF_AQ_VSI_UP_TABLE_UP4_SHIFT)
+#define IAVF_AQ_VSI_UP_TABLE_UP5_SHIFT 15
+#define IAVF_AQ_VSI_UP_TABLE_UP5_MASK (0x7 << \
+ IAVF_AQ_VSI_UP_TABLE_UP5_SHIFT)
+#define IAVF_AQ_VSI_UP_TABLE_UP6_SHIFT 18
+#define IAVF_AQ_VSI_UP_TABLE_UP6_MASK (0x7 << \
+ IAVF_AQ_VSI_UP_TABLE_UP6_SHIFT)
+#define IAVF_AQ_VSI_UP_TABLE_UP7_SHIFT 21
+#define IAVF_AQ_VSI_UP_TABLE_UP7_MASK (0x7 << \
+ IAVF_AQ_VSI_UP_TABLE_UP7_SHIFT)
+ __le32 egress_table; /* same defines as for ingress table */
+ /* cascaded PV section */
+ __le16 cas_pv_tag;
+ u8 cas_pv_flags;
+#define IAVF_AQ_VSI_CAS_PV_TAGX_SHIFT 0x00
+#define IAVF_AQ_VSI_CAS_PV_TAGX_MASK (0x03 << \
+ IAVF_AQ_VSI_CAS_PV_TAGX_SHIFT)
+#define IAVF_AQ_VSI_CAS_PV_TAGX_LEAVE 0x00
+#define IAVF_AQ_VSI_CAS_PV_TAGX_REMOVE 0x01
+#define IAVF_AQ_VSI_CAS_PV_TAGX_COPY 0x02
+#define IAVF_AQ_VSI_CAS_PV_INSERT_TAG 0x10
+#define IAVF_AQ_VSI_CAS_PV_ETAG_PRUNE 0x20
+#define IAVF_AQ_VSI_CAS_PV_ACCEPT_HOST_TAG 0x40
+ u8 cas_pv_reserved;
+ /* queue mapping section */
+ __le16 mapping_flags;
+#define IAVF_AQ_VSI_QUE_MAP_CONTIG 0x0
+#define IAVF_AQ_VSI_QUE_MAP_NONCONTIG 0x1
+ __le16 queue_mapping[16];
+#define IAVF_AQ_VSI_QUEUE_SHIFT 0x0
+#define IAVF_AQ_VSI_QUEUE_MASK (0x7FF << IAVF_AQ_VSI_QUEUE_SHIFT)
+ __le16 tc_mapping[8];
+#define IAVF_AQ_VSI_TC_QUE_OFFSET_SHIFT 0
+#define IAVF_AQ_VSI_TC_QUE_OFFSET_MASK (0x1FF << \
+ IAVF_AQ_VSI_TC_QUE_OFFSET_SHIFT)
+#define IAVF_AQ_VSI_TC_QUE_NUMBER_SHIFT 9
+#define IAVF_AQ_VSI_TC_QUE_NUMBER_MASK (0x7 << \
+ IAVF_AQ_VSI_TC_QUE_NUMBER_SHIFT)
+ /* queueing option section */
+ u8 queueing_opt_flags;
+#define IAVF_AQ_VSI_QUE_OPT_MULTICAST_UDP_ENA 0x04
+#define IAVF_AQ_VSI_QUE_OPT_UNICAST_UDP_ENA 0x08
+#define IAVF_AQ_VSI_QUE_OPT_TCP_ENA 0x10
+#define IAVF_AQ_VSI_QUE_OPT_FCOE_ENA 0x20
+#define IAVF_AQ_VSI_QUE_OPT_RSS_LUT_PF 0x00
+#define IAVF_AQ_VSI_QUE_OPT_RSS_LUT_VSI 0x40
+ u8 queueing_opt_reserved[3];
+ /* scheduler section */
+ u8 up_enable_bits;
+ u8 sched_reserved;
+ /* outer up section */
+ __le32 outer_up_table; /* same structure and defines as ingress tbl */
+ u8 cmd_reserved[8];
+ /* last 32 bytes are written by FW */
+ __le16 qs_handle[8];
+#define IAVF_AQ_VSI_QS_HANDLE_INVALID 0xFFFF
+ __le16 stat_counter_idx;
+ __le16 sched_id;
+ u8 resp_reserved[12];
+};
+
+IAVF_CHECK_STRUCT_LEN(128, iavf_aqc_vsi_properties_data);
+
+/* Get VEB Parameters (direct 0x0232)
+ * uses iavf_aqc_switch_seid for the descriptor
+ */
+struct iavf_aqc_get_veb_parameters_completion {
+ __le16 seid;
+ __le16 switch_id;
+ __le16 veb_flags; /* only the first/last flags from 0x0230 is valid */
+ __le16 statistic_index;
+ __le16 vebs_used;
+ __le16 vebs_free;
+ u8 reserved[4];
+};
+
+IAVF_CHECK_CMD_LENGTH(iavf_aqc_get_veb_parameters_completion);
+
+#define IAVF_LINK_SPEED_100MB_SHIFT 0x1
+#define IAVF_LINK_SPEED_1000MB_SHIFT 0x2
+#define IAVF_LINK_SPEED_10GB_SHIFT 0x3
+#define IAVF_LINK_SPEED_40GB_SHIFT 0x4
+#define IAVF_LINK_SPEED_20GB_SHIFT 0x5
+#define IAVF_LINK_SPEED_25GB_SHIFT 0x6
+
+enum iavf_aq_link_speed {
+ IAVF_LINK_SPEED_UNKNOWN = 0,
+ IAVF_LINK_SPEED_100MB = BIT(IAVF_LINK_SPEED_100MB_SHIFT),
+ IAVF_LINK_SPEED_1GB = BIT(IAVF_LINK_SPEED_1000MB_SHIFT),
+ IAVF_LINK_SPEED_10GB = BIT(IAVF_LINK_SPEED_10GB_SHIFT),
+ IAVF_LINK_SPEED_40GB = BIT(IAVF_LINK_SPEED_40GB_SHIFT),
+ IAVF_LINK_SPEED_20GB = BIT(IAVF_LINK_SPEED_20GB_SHIFT),
+ IAVF_LINK_SPEED_25GB = BIT(IAVF_LINK_SPEED_25GB_SHIFT),
+};
+
+/* Send to PF command (indirect 0x0801) id is only used by PF
+ * Send to VF command (indirect 0x0802) id is only used by PF
+ * Send to Peer PF command (indirect 0x0803)
+ */
+struct iavf_aqc_pf_vf_message {
+ __le32 id;
+ u8 reserved[4];
+ __le32 addr_high;
+ __le32 addr_low;
+};
+
+IAVF_CHECK_CMD_LENGTH(iavf_aqc_pf_vf_message);
+
+struct iavf_aqc_get_set_rss_key {
+#define IAVF_AQC_SET_RSS_KEY_VSI_VALID BIT(15)
+#define IAVF_AQC_SET_RSS_KEY_VSI_ID_SHIFT 0
+#define IAVF_AQC_SET_RSS_KEY_VSI_ID_MASK (0x3FF << \
+ IAVF_AQC_SET_RSS_KEY_VSI_ID_SHIFT)
+ __le16 vsi_id;
+ u8 reserved[6];
+ __le32 addr_high;
+ __le32 addr_low;
+};
+
+IAVF_CHECK_CMD_LENGTH(iavf_aqc_get_set_rss_key);
+
+struct iavf_aqc_get_set_rss_key_data {
+ u8 standard_rss_key[0x28];
+ u8 extended_hash_key[0xc];
+};
+
+IAVF_CHECK_STRUCT_LEN(0x34, iavf_aqc_get_set_rss_key_data);
+
+struct iavf_aqc_get_set_rss_lut {
+#define IAVF_AQC_SET_RSS_LUT_VSI_VALID BIT(15)
+#define IAVF_AQC_SET_RSS_LUT_VSI_ID_SHIFT 0
+#define IAVF_AQC_SET_RSS_LUT_VSI_ID_MASK (0x3FF << \
+ IAVF_AQC_SET_RSS_LUT_VSI_ID_SHIFT)
+ __le16 vsi_id;
+#define IAVF_AQC_SET_RSS_LUT_TABLE_TYPE_SHIFT 0
+#define IAVF_AQC_SET_RSS_LUT_TABLE_TYPE_MASK \
+ BIT(IAVF_AQC_SET_RSS_LUT_TABLE_TYPE_SHIFT)
+
+#define IAVF_AQC_SET_RSS_LUT_TABLE_TYPE_VSI 0
+#define IAVF_AQC_SET_RSS_LUT_TABLE_TYPE_PF 1
+ __le16 flags;
+ u8 reserved[4];
+ __le32 addr_high;
+ __le32 addr_low;
+};
+
+IAVF_CHECK_CMD_LENGTH(iavf_aqc_get_set_rss_lut);
+#endif /* _IAVF_ADMINQ_CMD_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_adv_rss.c b/drivers/net/ethernet/intel/iavf/iavf_adv_rss.c
new file mode 100644
index 0000000000..6edbf134b7
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_adv_rss.c
@@ -0,0 +1,218 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2021, Intel Corporation. */
+
+/* advanced RSS configuration ethtool support for iavf */
+
+#include "iavf.h"
+
+/**
+ * iavf_fill_adv_rss_ip4_hdr - fill the IPv4 RSS protocol header
+ * @hdr: the virtchnl message protocol header data structure
+ * @hash_flds: the RSS configuration protocol hash fields
+ */
+static void
+iavf_fill_adv_rss_ip4_hdr(struct virtchnl_proto_hdr *hdr, u64 hash_flds)
+{
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, IPV4);
+
+ if (hash_flds & IAVF_ADV_RSS_HASH_FLD_IPV4_SA)
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV4, SRC);
+
+ if (hash_flds & IAVF_ADV_RSS_HASH_FLD_IPV4_DA)
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV4, DST);
+}
+
+/**
+ * iavf_fill_adv_rss_ip6_hdr - fill the IPv6 RSS protocol header
+ * @hdr: the virtchnl message protocol header data structure
+ * @hash_flds: the RSS configuration protocol hash fields
+ */
+static void
+iavf_fill_adv_rss_ip6_hdr(struct virtchnl_proto_hdr *hdr, u64 hash_flds)
+{
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, IPV6);
+
+ if (hash_flds & IAVF_ADV_RSS_HASH_FLD_IPV6_SA)
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV6, SRC);
+
+ if (hash_flds & IAVF_ADV_RSS_HASH_FLD_IPV6_DA)
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV6, DST);
+}
+
+/**
+ * iavf_fill_adv_rss_tcp_hdr - fill the TCP RSS protocol header
+ * @hdr: the virtchnl message protocol header data structure
+ * @hash_flds: the RSS configuration protocol hash fields
+ */
+static void
+iavf_fill_adv_rss_tcp_hdr(struct virtchnl_proto_hdr *hdr, u64 hash_flds)
+{
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, TCP);
+
+ if (hash_flds & IAVF_ADV_RSS_HASH_FLD_TCP_SRC_PORT)
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, TCP, SRC_PORT);
+
+ if (hash_flds & IAVF_ADV_RSS_HASH_FLD_TCP_DST_PORT)
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, TCP, DST_PORT);
+}
+
+/**
+ * iavf_fill_adv_rss_udp_hdr - fill the UDP RSS protocol header
+ * @hdr: the virtchnl message protocol header data structure
+ * @hash_flds: the RSS configuration protocol hash fields
+ */
+static void
+iavf_fill_adv_rss_udp_hdr(struct virtchnl_proto_hdr *hdr, u64 hash_flds)
+{
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, UDP);
+
+ if (hash_flds & IAVF_ADV_RSS_HASH_FLD_UDP_SRC_PORT)
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, UDP, SRC_PORT);
+
+ if (hash_flds & IAVF_ADV_RSS_HASH_FLD_UDP_DST_PORT)
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, UDP, DST_PORT);
+}
+
+/**
+ * iavf_fill_adv_rss_sctp_hdr - fill the SCTP RSS protocol header
+ * @hdr: the virtchnl message protocol header data structure
+ * @hash_flds: the RSS configuration protocol hash fields
+ */
+static void
+iavf_fill_adv_rss_sctp_hdr(struct virtchnl_proto_hdr *hdr, u64 hash_flds)
+{
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, SCTP);
+
+ if (hash_flds & IAVF_ADV_RSS_HASH_FLD_SCTP_SRC_PORT)
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, SCTP, SRC_PORT);
+
+ if (hash_flds & IAVF_ADV_RSS_HASH_FLD_SCTP_DST_PORT)
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, SCTP, DST_PORT);
+}
+
+/**
+ * iavf_fill_adv_rss_cfg_msg - fill the RSS configuration into virtchnl message
+ * @rss_cfg: the virtchnl message to be filled with RSS configuration setting
+ * @packet_hdrs: the RSS configuration protocol header types
+ * @hash_flds: the RSS configuration protocol hash fields
+ *
+ * Returns 0 if the RSS configuration virtchnl message is filled successfully
+ */
+int
+iavf_fill_adv_rss_cfg_msg(struct virtchnl_rss_cfg *rss_cfg,
+ u32 packet_hdrs, u64 hash_flds)
+{
+ struct virtchnl_proto_hdrs *proto_hdrs = &rss_cfg->proto_hdrs;
+ struct virtchnl_proto_hdr *hdr;
+
+ rss_cfg->rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
+
+ proto_hdrs->tunnel_level = 0; /* always outer layer */
+
+ hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ switch (packet_hdrs & IAVF_ADV_RSS_FLOW_SEG_HDR_L3) {
+ case IAVF_ADV_RSS_FLOW_SEG_HDR_IPV4:
+ iavf_fill_adv_rss_ip4_hdr(hdr, hash_flds);
+ break;
+ case IAVF_ADV_RSS_FLOW_SEG_HDR_IPV6:
+ iavf_fill_adv_rss_ip6_hdr(hdr, hash_flds);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ switch (packet_hdrs & IAVF_ADV_RSS_FLOW_SEG_HDR_L4) {
+ case IAVF_ADV_RSS_FLOW_SEG_HDR_TCP:
+ iavf_fill_adv_rss_tcp_hdr(hdr, hash_flds);
+ break;
+ case IAVF_ADV_RSS_FLOW_SEG_HDR_UDP:
+ iavf_fill_adv_rss_udp_hdr(hdr, hash_flds);
+ break;
+ case IAVF_ADV_RSS_FLOW_SEG_HDR_SCTP:
+ iavf_fill_adv_rss_sctp_hdr(hdr, hash_flds);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_find_adv_rss_cfg_by_hdrs - find RSS configuration with header type
+ * @adapter: pointer to the VF adapter structure
+ * @packet_hdrs: protocol header type to find.
+ *
+ * Returns pointer to advance RSS configuration if found or null
+ */
+struct iavf_adv_rss *
+iavf_find_adv_rss_cfg_by_hdrs(struct iavf_adapter *adapter, u32 packet_hdrs)
+{
+ struct iavf_adv_rss *rss;
+
+ list_for_each_entry(rss, &adapter->adv_rss_list_head, list)
+ if (rss->packet_hdrs == packet_hdrs)
+ return rss;
+
+ return NULL;
+}
+
+/**
+ * iavf_print_adv_rss_cfg
+ * @adapter: pointer to the VF adapter structure
+ * @rss: pointer to the advance RSS configuration to print
+ * @action: the string description about how to handle the RSS
+ * @result: the string description about the virtchnl result
+ *
+ * Print the advance RSS configuration
+ **/
+void
+iavf_print_adv_rss_cfg(struct iavf_adapter *adapter, struct iavf_adv_rss *rss,
+ const char *action, const char *result)
+{
+ u32 packet_hdrs = rss->packet_hdrs;
+ u64 hash_flds = rss->hash_flds;
+ static char hash_opt[300];
+ const char *proto;
+
+ if (packet_hdrs & IAVF_ADV_RSS_FLOW_SEG_HDR_TCP)
+ proto = "TCP";
+ else if (packet_hdrs & IAVF_ADV_RSS_FLOW_SEG_HDR_UDP)
+ proto = "UDP";
+ else if (packet_hdrs & IAVF_ADV_RSS_FLOW_SEG_HDR_SCTP)
+ proto = "SCTP";
+ else
+ return;
+
+ memset(hash_opt, 0, sizeof(hash_opt));
+
+ strcat(hash_opt, proto);
+ if (packet_hdrs & IAVF_ADV_RSS_FLOW_SEG_HDR_IPV4)
+ strcat(hash_opt, "v4 ");
+ else
+ strcat(hash_opt, "v6 ");
+
+ if (hash_flds & (IAVF_ADV_RSS_HASH_FLD_IPV4_SA |
+ IAVF_ADV_RSS_HASH_FLD_IPV6_SA))
+ strcat(hash_opt, "IP SA,");
+ if (hash_flds & (IAVF_ADV_RSS_HASH_FLD_IPV4_DA |
+ IAVF_ADV_RSS_HASH_FLD_IPV6_DA))
+ strcat(hash_opt, "IP DA,");
+ if (hash_flds & (IAVF_ADV_RSS_HASH_FLD_TCP_SRC_PORT |
+ IAVF_ADV_RSS_HASH_FLD_UDP_SRC_PORT |
+ IAVF_ADV_RSS_HASH_FLD_SCTP_SRC_PORT))
+ strcat(hash_opt, "src port,");
+ if (hash_flds & (IAVF_ADV_RSS_HASH_FLD_TCP_DST_PORT |
+ IAVF_ADV_RSS_HASH_FLD_UDP_DST_PORT |
+ IAVF_ADV_RSS_HASH_FLD_SCTP_DST_PORT))
+ strcat(hash_opt, "dst port,");
+
+ if (!action)
+ action = "";
+
+ if (!result)
+ result = "";
+
+ dev_info(&adapter->pdev->dev, "%s %s %s\n", action, hash_opt, result);
+}
diff --git a/drivers/net/ethernet/intel/iavf/iavf_adv_rss.h b/drivers/net/ethernet/intel/iavf/iavf_adv_rss.h
new file mode 100644
index 0000000000..4d3be11af7
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_adv_rss.h
@@ -0,0 +1,95 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2021, Intel Corporation. */
+
+#ifndef _IAVF_ADV_RSS_H_
+#define _IAVF_ADV_RSS_H_
+
+struct iavf_adapter;
+
+/* State of advanced RSS configuration */
+enum iavf_adv_rss_state_t {
+ IAVF_ADV_RSS_ADD_REQUEST, /* User requests to add RSS */
+ IAVF_ADV_RSS_ADD_PENDING, /* RSS pending add by the PF */
+ IAVF_ADV_RSS_DEL_REQUEST, /* Driver requests to delete RSS */
+ IAVF_ADV_RSS_DEL_PENDING, /* RSS pending delete by the PF */
+ IAVF_ADV_RSS_ACTIVE, /* RSS configuration is active */
+};
+
+enum iavf_adv_rss_flow_seg_hdr {
+ IAVF_ADV_RSS_FLOW_SEG_HDR_NONE = 0x00000000,
+ IAVF_ADV_RSS_FLOW_SEG_HDR_IPV4 = 0x00000001,
+ IAVF_ADV_RSS_FLOW_SEG_HDR_IPV6 = 0x00000002,
+ IAVF_ADV_RSS_FLOW_SEG_HDR_TCP = 0x00000004,
+ IAVF_ADV_RSS_FLOW_SEG_HDR_UDP = 0x00000008,
+ IAVF_ADV_RSS_FLOW_SEG_HDR_SCTP = 0x00000010,
+};
+
+#define IAVF_ADV_RSS_FLOW_SEG_HDR_L3 \
+ (IAVF_ADV_RSS_FLOW_SEG_HDR_IPV4 | \
+ IAVF_ADV_RSS_FLOW_SEG_HDR_IPV6)
+
+#define IAVF_ADV_RSS_FLOW_SEG_HDR_L4 \
+ (IAVF_ADV_RSS_FLOW_SEG_HDR_TCP | \
+ IAVF_ADV_RSS_FLOW_SEG_HDR_UDP | \
+ IAVF_ADV_RSS_FLOW_SEG_HDR_SCTP)
+
+enum iavf_adv_rss_flow_field {
+ /* L3 */
+ IAVF_ADV_RSS_FLOW_FIELD_IDX_IPV4_SA,
+ IAVF_ADV_RSS_FLOW_FIELD_IDX_IPV4_DA,
+ IAVF_ADV_RSS_FLOW_FIELD_IDX_IPV6_SA,
+ IAVF_ADV_RSS_FLOW_FIELD_IDX_IPV6_DA,
+ /* L4 */
+ IAVF_ADV_RSS_FLOW_FIELD_IDX_TCP_SRC_PORT,
+ IAVF_ADV_RSS_FLOW_FIELD_IDX_TCP_DST_PORT,
+ IAVF_ADV_RSS_FLOW_FIELD_IDX_UDP_SRC_PORT,
+ IAVF_ADV_RSS_FLOW_FIELD_IDX_UDP_DST_PORT,
+ IAVF_ADV_RSS_FLOW_FIELD_IDX_SCTP_SRC_PORT,
+ IAVF_ADV_RSS_FLOW_FIELD_IDX_SCTP_DST_PORT,
+
+ /* The total number of enums must not exceed 64 */
+ IAVF_ADV_RSS_FLOW_FIELD_IDX_MAX
+};
+
+#define IAVF_ADV_RSS_HASH_INVALID 0
+#define IAVF_ADV_RSS_HASH_FLD_IPV4_SA \
+ BIT_ULL(IAVF_ADV_RSS_FLOW_FIELD_IDX_IPV4_SA)
+#define IAVF_ADV_RSS_HASH_FLD_IPV6_SA \
+ BIT_ULL(IAVF_ADV_RSS_FLOW_FIELD_IDX_IPV6_SA)
+#define IAVF_ADV_RSS_HASH_FLD_IPV4_DA \
+ BIT_ULL(IAVF_ADV_RSS_FLOW_FIELD_IDX_IPV4_DA)
+#define IAVF_ADV_RSS_HASH_FLD_IPV6_DA \
+ BIT_ULL(IAVF_ADV_RSS_FLOW_FIELD_IDX_IPV6_DA)
+#define IAVF_ADV_RSS_HASH_FLD_TCP_SRC_PORT \
+ BIT_ULL(IAVF_ADV_RSS_FLOW_FIELD_IDX_TCP_SRC_PORT)
+#define IAVF_ADV_RSS_HASH_FLD_TCP_DST_PORT \
+ BIT_ULL(IAVF_ADV_RSS_FLOW_FIELD_IDX_TCP_DST_PORT)
+#define IAVF_ADV_RSS_HASH_FLD_UDP_SRC_PORT \
+ BIT_ULL(IAVF_ADV_RSS_FLOW_FIELD_IDX_UDP_SRC_PORT)
+#define IAVF_ADV_RSS_HASH_FLD_UDP_DST_PORT \
+ BIT_ULL(IAVF_ADV_RSS_FLOW_FIELD_IDX_UDP_DST_PORT)
+#define IAVF_ADV_RSS_HASH_FLD_SCTP_SRC_PORT \
+ BIT_ULL(IAVF_ADV_RSS_FLOW_FIELD_IDX_SCTP_SRC_PORT)
+#define IAVF_ADV_RSS_HASH_FLD_SCTP_DST_PORT \
+ BIT_ULL(IAVF_ADV_RSS_FLOW_FIELD_IDX_SCTP_DST_PORT)
+
+/* bookkeeping of advanced RSS configuration */
+struct iavf_adv_rss {
+ enum iavf_adv_rss_state_t state;
+ struct list_head list;
+
+ u32 packet_hdrs;
+ u64 hash_flds;
+
+ struct virtchnl_rss_cfg cfg_msg;
+};
+
+int
+iavf_fill_adv_rss_cfg_msg(struct virtchnl_rss_cfg *rss_cfg,
+ u32 packet_hdrs, u64 hash_flds);
+struct iavf_adv_rss *
+iavf_find_adv_rss_cfg_by_hdrs(struct iavf_adapter *adapter, u32 packet_hdrs);
+void
+iavf_print_adv_rss_cfg(struct iavf_adapter *adapter, struct iavf_adv_rss *rss,
+ const char *action, const char *result);
+#endif /* _IAVF_ADV_RSS_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_alloc.h b/drivers/net/ethernet/intel/iavf/iavf_alloc.h
new file mode 100644
index 0000000000..162ea70685
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_alloc.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_ALLOC_H_
+#define _IAVF_ALLOC_H_
+
+struct iavf_hw;
+
+/* Memory allocation types */
+enum iavf_memory_type {
+ iavf_mem_arq_buf = 0, /* ARQ indirect command buffer */
+ iavf_mem_asq_buf = 1,
+ iavf_mem_atq_buf = 2, /* ATQ indirect command buffer */
+ iavf_mem_arq_ring = 3, /* ARQ descriptor ring */
+ iavf_mem_atq_ring = 4, /* ATQ descriptor ring */
+ iavf_mem_pd = 5, /* Page Descriptor */
+ iavf_mem_bp = 6, /* Backing Page - 4KB */
+ iavf_mem_bp_jumbo = 7, /* Backing Page - > 4KB */
+ iavf_mem_reserved
+};
+
+/* prototype for functions used for dynamic memory allocation */
+enum iavf_status iavf_allocate_dma_mem(struct iavf_hw *hw,
+ struct iavf_dma_mem *mem,
+ enum iavf_memory_type type,
+ u64 size, u32 alignment);
+enum iavf_status iavf_free_dma_mem(struct iavf_hw *hw,
+ struct iavf_dma_mem *mem);
+enum iavf_status iavf_allocate_virt_mem(struct iavf_hw *hw,
+ struct iavf_virt_mem *mem, u32 size);
+void iavf_free_virt_mem(struct iavf_hw *hw, struct iavf_virt_mem *mem);
+
+#endif /* _IAVF_ALLOC_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_client.c b/drivers/net/ethernet/intel/iavf/iavf_client.c
new file mode 100644
index 0000000000..e6051b6355
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_client.c
@@ -0,0 +1,578 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#include <linux/list.h>
+#include <linux/errno.h>
+
+#include "iavf.h"
+#include "iavf_prototype.h"
+#include "iavf_client.h"
+
+static
+const char iavf_client_interface_version_str[] = IAVF_CLIENT_VERSION_STR;
+static struct iavf_client *vf_registered_client;
+static LIST_HEAD(iavf_devices);
+static DEFINE_MUTEX(iavf_device_mutex);
+
+static u32 iavf_client_virtchnl_send(struct iavf_info *ldev,
+ struct iavf_client *client,
+ u8 *msg, u16 len);
+
+static int iavf_client_setup_qvlist(struct iavf_info *ldev,
+ struct iavf_client *client,
+ struct iavf_qvlist_info *qvlist_info);
+
+static struct iavf_ops iavf_lan_ops = {
+ .virtchnl_send = iavf_client_virtchnl_send,
+ .setup_qvlist = iavf_client_setup_qvlist,
+};
+
+/**
+ * iavf_client_get_params - retrieve relevant client parameters
+ * @vsi: VSI with parameters
+ * @params: client param struct
+ **/
+static
+void iavf_client_get_params(struct iavf_vsi *vsi, struct iavf_params *params)
+{
+ int i;
+
+ memset(params, 0, sizeof(struct iavf_params));
+ params->mtu = vsi->netdev->mtu;
+ params->link_up = vsi->back->link_up;
+
+ for (i = 0; i < IAVF_MAX_USER_PRIORITY; i++) {
+ params->qos.prio_qos[i].tc = 0;
+ params->qos.prio_qos[i].qs_handle = vsi->qs_handle;
+ }
+}
+
+/**
+ * iavf_notify_client_message - call the client message receive callback
+ * @vsi: the VSI associated with this client
+ * @msg: message buffer
+ * @len: length of message
+ *
+ * If there is a client to this VSI, call the client
+ **/
+void iavf_notify_client_message(struct iavf_vsi *vsi, u8 *msg, u16 len)
+{
+ struct iavf_client_instance *cinst;
+
+ if (!vsi)
+ return;
+
+ cinst = vsi->back->cinst;
+ if (!cinst || !cinst->client || !cinst->client->ops ||
+ !cinst->client->ops->virtchnl_receive) {
+ dev_dbg(&vsi->back->pdev->dev,
+ "Cannot locate client instance virtchnl_receive function\n");
+ return;
+ }
+ cinst->client->ops->virtchnl_receive(&cinst->lan_info, cinst->client,
+ msg, len);
+}
+
+/**
+ * iavf_notify_client_l2_params - call the client notify callback
+ * @vsi: the VSI with l2 param changes
+ *
+ * If there is a client to this VSI, call the client
+ **/
+void iavf_notify_client_l2_params(struct iavf_vsi *vsi)
+{
+ struct iavf_client_instance *cinst;
+ struct iavf_params params;
+
+ if (!vsi)
+ return;
+
+ cinst = vsi->back->cinst;
+
+ if (!cinst || !cinst->client || !cinst->client->ops ||
+ !cinst->client->ops->l2_param_change) {
+ dev_dbg(&vsi->back->pdev->dev,
+ "Cannot locate client instance l2_param_change function\n");
+ return;
+ }
+ iavf_client_get_params(vsi, &params);
+ cinst->lan_info.params = params;
+ cinst->client->ops->l2_param_change(&cinst->lan_info, cinst->client,
+ &params);
+}
+
+/**
+ * iavf_notify_client_open - call the client open callback
+ * @vsi: the VSI with netdev opened
+ *
+ * If there is a client to this netdev, call the client with open
+ **/
+void iavf_notify_client_open(struct iavf_vsi *vsi)
+{
+ struct iavf_adapter *adapter = vsi->back;
+ struct iavf_client_instance *cinst = adapter->cinst;
+ int ret;
+
+ if (!cinst || !cinst->client || !cinst->client->ops ||
+ !cinst->client->ops->open) {
+ dev_dbg(&vsi->back->pdev->dev,
+ "Cannot locate client instance open function\n");
+ return;
+ }
+ if (!(test_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state))) {
+ ret = cinst->client->ops->open(&cinst->lan_info, cinst->client);
+ if (!ret)
+ set_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state);
+ }
+}
+
+/**
+ * iavf_client_release_qvlist - send a message to the PF to release rdma qv map
+ * @ldev: pointer to L2 context.
+ *
+ * Return 0 on success or < 0 on error
+ **/
+static int iavf_client_release_qvlist(struct iavf_info *ldev)
+{
+ struct iavf_adapter *adapter = ldev->vf;
+ enum iavf_status err;
+
+ if (adapter->aq_required)
+ return -EAGAIN;
+
+ err = iavf_aq_send_msg_to_pf(&adapter->hw,
+ VIRTCHNL_OP_RELEASE_RDMA_IRQ_MAP,
+ IAVF_SUCCESS, NULL, 0, NULL);
+
+ if (err)
+ dev_err(&adapter->pdev->dev,
+ "Unable to send RDMA vector release message to PF, error %d, aq status %d\n",
+ err, adapter->hw.aq.asq_last_status);
+
+ return err;
+}
+
+/**
+ * iavf_notify_client_close - call the client close callback
+ * @vsi: the VSI with netdev closed
+ * @reset: true when close called due to reset pending
+ *
+ * If there is a client to this netdev, call the client with close
+ **/
+void iavf_notify_client_close(struct iavf_vsi *vsi, bool reset)
+{
+ struct iavf_adapter *adapter = vsi->back;
+ struct iavf_client_instance *cinst = adapter->cinst;
+
+ if (!cinst || !cinst->client || !cinst->client->ops ||
+ !cinst->client->ops->close) {
+ dev_dbg(&vsi->back->pdev->dev,
+ "Cannot locate client instance close function\n");
+ return;
+ }
+ cinst->client->ops->close(&cinst->lan_info, cinst->client, reset);
+ iavf_client_release_qvlist(&cinst->lan_info);
+ clear_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state);
+}
+
+/**
+ * iavf_client_add_instance - add a client instance to the instance list
+ * @adapter: pointer to the board struct
+ *
+ * Returns cinst ptr on success, NULL on failure
+ **/
+static struct iavf_client_instance *
+iavf_client_add_instance(struct iavf_adapter *adapter)
+{
+ struct iavf_client_instance *cinst = NULL;
+ struct iavf_vsi *vsi = &adapter->vsi;
+ struct netdev_hw_addr *mac = NULL;
+ struct iavf_params params;
+
+ if (!vf_registered_client)
+ goto out;
+
+ if (adapter->cinst) {
+ cinst = adapter->cinst;
+ goto out;
+ }
+
+ cinst = kzalloc(sizeof(*cinst), GFP_KERNEL);
+ if (!cinst)
+ goto out;
+
+ cinst->lan_info.vf = (void *)adapter;
+ cinst->lan_info.netdev = vsi->netdev;
+ cinst->lan_info.pcidev = adapter->pdev;
+ cinst->lan_info.fid = 0;
+ cinst->lan_info.ftype = IAVF_CLIENT_FTYPE_VF;
+ cinst->lan_info.hw_addr = adapter->hw.hw_addr;
+ cinst->lan_info.ops = &iavf_lan_ops;
+ cinst->lan_info.version.major = IAVF_CLIENT_VERSION_MAJOR;
+ cinst->lan_info.version.minor = IAVF_CLIENT_VERSION_MINOR;
+ cinst->lan_info.version.build = IAVF_CLIENT_VERSION_BUILD;
+ iavf_client_get_params(vsi, &params);
+ cinst->lan_info.params = params;
+ set_bit(__IAVF_CLIENT_INSTANCE_NONE, &cinst->state);
+
+ cinst->lan_info.msix_count = adapter->num_rdma_msix;
+ cinst->lan_info.msix_entries =
+ &adapter->msix_entries[adapter->rdma_base_vector];
+
+ mac = list_first_entry(&cinst->lan_info.netdev->dev_addrs.list,
+ struct netdev_hw_addr, list);
+ if (mac)
+ ether_addr_copy(cinst->lan_info.lanmac, mac->addr);
+ else
+ dev_err(&adapter->pdev->dev, "MAC address list is empty!\n");
+
+ cinst->client = vf_registered_client;
+ adapter->cinst = cinst;
+out:
+ return cinst;
+}
+
+/**
+ * iavf_client_del_instance - removes a client instance from the list
+ * @adapter: pointer to the board struct
+ *
+ **/
+static
+void iavf_client_del_instance(struct iavf_adapter *adapter)
+{
+ kfree(adapter->cinst);
+ adapter->cinst = NULL;
+}
+
+/**
+ * iavf_client_subtask - client maintenance work
+ * @adapter: board private structure
+ **/
+void iavf_client_subtask(struct iavf_adapter *adapter)
+{
+ struct iavf_client *client = vf_registered_client;
+ struct iavf_client_instance *cinst;
+ int ret = 0;
+
+ if (adapter->state < __IAVF_DOWN)
+ return;
+
+ /* first check client is registered */
+ if (!client)
+ return;
+
+ /* Add the client instance to the instance list */
+ cinst = iavf_client_add_instance(adapter);
+ if (!cinst)
+ return;
+
+ dev_info(&adapter->pdev->dev, "Added instance of Client %s\n",
+ client->name);
+
+ if (!test_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state)) {
+ /* Send an Open request to the client */
+
+ if (client->ops && client->ops->open)
+ ret = client->ops->open(&cinst->lan_info, client);
+ if (!ret)
+ set_bit(__IAVF_CLIENT_INSTANCE_OPENED,
+ &cinst->state);
+ else
+ /* remove client instance */
+ iavf_client_del_instance(adapter);
+ }
+}
+
+/**
+ * iavf_lan_add_device - add a lan device struct to the list of lan devices
+ * @adapter: pointer to the board struct
+ *
+ * Returns 0 on success or none 0 on error
+ **/
+int iavf_lan_add_device(struct iavf_adapter *adapter)
+{
+ struct iavf_device *ldev;
+ int ret = 0;
+
+ mutex_lock(&iavf_device_mutex);
+ list_for_each_entry(ldev, &iavf_devices, list) {
+ if (ldev->vf == adapter) {
+ ret = -EEXIST;
+ goto out;
+ }
+ }
+ ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
+ if (!ldev) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ ldev->vf = adapter;
+ INIT_LIST_HEAD(&ldev->list);
+ list_add(&ldev->list, &iavf_devices);
+ dev_info(&adapter->pdev->dev, "Added LAN device bus=0x%02x dev=0x%02x func=0x%02x\n",
+ adapter->hw.bus.bus_id, adapter->hw.bus.device,
+ adapter->hw.bus.func);
+
+ /* Since in some cases register may have happened before a device gets
+ * added, we can schedule a subtask to go initiate the clients.
+ */
+ adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
+
+out:
+ mutex_unlock(&iavf_device_mutex);
+ return ret;
+}
+
+/**
+ * iavf_lan_del_device - removes a lan device from the device list
+ * @adapter: pointer to the board struct
+ *
+ * Returns 0 on success or non-0 on error
+ **/
+int iavf_lan_del_device(struct iavf_adapter *adapter)
+{
+ struct iavf_device *ldev, *tmp;
+ int ret = -ENODEV;
+
+ mutex_lock(&iavf_device_mutex);
+ list_for_each_entry_safe(ldev, tmp, &iavf_devices, list) {
+ if (ldev->vf == adapter) {
+ dev_info(&adapter->pdev->dev,
+ "Deleted LAN device bus=0x%02x dev=0x%02x func=0x%02x\n",
+ adapter->hw.bus.bus_id, adapter->hw.bus.device,
+ adapter->hw.bus.func);
+ list_del(&ldev->list);
+ kfree(ldev);
+ ret = 0;
+ break;
+ }
+ }
+
+ mutex_unlock(&iavf_device_mutex);
+ return ret;
+}
+
+/**
+ * iavf_client_release - release client specific resources
+ * @client: pointer to the registered client
+ *
+ **/
+static void iavf_client_release(struct iavf_client *client)
+{
+ struct iavf_client_instance *cinst;
+ struct iavf_device *ldev;
+ struct iavf_adapter *adapter;
+
+ mutex_lock(&iavf_device_mutex);
+ list_for_each_entry(ldev, &iavf_devices, list) {
+ adapter = ldev->vf;
+ cinst = adapter->cinst;
+ if (!cinst)
+ continue;
+ if (test_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state)) {
+ if (client->ops && client->ops->close)
+ client->ops->close(&cinst->lan_info, client,
+ false);
+ iavf_client_release_qvlist(&cinst->lan_info);
+ clear_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state);
+
+ dev_warn(&adapter->pdev->dev,
+ "Client %s instance closed\n", client->name);
+ }
+ /* delete the client instance */
+ iavf_client_del_instance(adapter);
+ dev_info(&adapter->pdev->dev, "Deleted client instance of Client %s\n",
+ client->name);
+ }
+ mutex_unlock(&iavf_device_mutex);
+}
+
+/**
+ * iavf_client_prepare - prepare client specific resources
+ * @client: pointer to the registered client
+ *
+ **/
+static void iavf_client_prepare(struct iavf_client *client)
+{
+ struct iavf_device *ldev;
+ struct iavf_adapter *adapter;
+
+ mutex_lock(&iavf_device_mutex);
+ list_for_each_entry(ldev, &iavf_devices, list) {
+ adapter = ldev->vf;
+ /* Signal the watchdog to service the client */
+ adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
+ }
+ mutex_unlock(&iavf_device_mutex);
+}
+
+/**
+ * iavf_client_virtchnl_send - send a message to the PF instance
+ * @ldev: pointer to L2 context.
+ * @client: Client pointer.
+ * @msg: pointer to message buffer
+ * @len: message length
+ *
+ * Return 0 on success or < 0 on error
+ **/
+static u32 iavf_client_virtchnl_send(struct iavf_info *ldev,
+ struct iavf_client *client,
+ u8 *msg, u16 len)
+{
+ struct iavf_adapter *adapter = ldev->vf;
+ enum iavf_status err;
+
+ if (adapter->aq_required)
+ return -EAGAIN;
+
+ err = iavf_aq_send_msg_to_pf(&adapter->hw, VIRTCHNL_OP_RDMA,
+ IAVF_SUCCESS, msg, len, NULL);
+ if (err)
+ dev_err(&adapter->pdev->dev, "Unable to send RDMA message to PF, error %d, aq status %d\n",
+ err, adapter->hw.aq.asq_last_status);
+
+ return err;
+}
+
+/**
+ * iavf_client_setup_qvlist - send a message to the PF to setup rdma qv map
+ * @ldev: pointer to L2 context.
+ * @client: Client pointer.
+ * @qvlist_info: queue and vector list
+ *
+ * Return 0 on success or < 0 on error
+ **/
+static int iavf_client_setup_qvlist(struct iavf_info *ldev,
+ struct iavf_client *client,
+ struct iavf_qvlist_info *qvlist_info)
+{
+ struct virtchnl_rdma_qvlist_info *v_qvlist_info;
+ struct iavf_adapter *adapter = ldev->vf;
+ struct iavf_qv_info *qv_info;
+ enum iavf_status err;
+ u32 v_idx, i;
+ size_t msg_size;
+
+ if (adapter->aq_required)
+ return -EAGAIN;
+
+ /* A quick check on whether the vectors belong to the client */
+ for (i = 0; i < qvlist_info->num_vectors; i++) {
+ qv_info = &qvlist_info->qv_info[i];
+ if (!qv_info)
+ continue;
+ v_idx = qv_info->v_idx;
+ if ((v_idx >=
+ (adapter->rdma_base_vector + adapter->num_rdma_msix)) ||
+ (v_idx < adapter->rdma_base_vector))
+ return -EINVAL;
+ }
+
+ v_qvlist_info = (struct virtchnl_rdma_qvlist_info *)qvlist_info;
+ msg_size = virtchnl_struct_size(v_qvlist_info, qv_info,
+ v_qvlist_info->num_vectors);
+
+ adapter->client_pending |= BIT(VIRTCHNL_OP_CONFIG_RDMA_IRQ_MAP);
+ err = iavf_aq_send_msg_to_pf(&adapter->hw,
+ VIRTCHNL_OP_CONFIG_RDMA_IRQ_MAP, IAVF_SUCCESS,
+ (u8 *)v_qvlist_info, msg_size, NULL);
+
+ if (err) {
+ dev_err(&adapter->pdev->dev,
+ "Unable to send RDMA vector config message to PF, error %d, aq status %d\n",
+ err, adapter->hw.aq.asq_last_status);
+ goto out;
+ }
+
+ err = -EBUSY;
+ for (i = 0; i < 5; i++) {
+ msleep(100);
+ if (!(adapter->client_pending &
+ BIT(VIRTCHNL_OP_CONFIG_RDMA_IRQ_MAP))) {
+ err = 0;
+ break;
+ }
+ }
+out:
+ return err;
+}
+
+/**
+ * iavf_register_client - Register a iavf client driver with the L2 driver
+ * @client: pointer to the iavf_client struct
+ *
+ * Returns 0 on success or non-0 on error
+ **/
+int iavf_register_client(struct iavf_client *client)
+{
+ int ret = 0;
+
+ if (!client) {
+ ret = -EIO;
+ goto out;
+ }
+
+ if (strlen(client->name) == 0) {
+ pr_info("iavf: Failed to register client with no name\n");
+ ret = -EIO;
+ goto out;
+ }
+
+ if (vf_registered_client) {
+ pr_info("iavf: Client %s has already been registered!\n",
+ client->name);
+ ret = -EEXIST;
+ goto out;
+ }
+
+ if ((client->version.major != IAVF_CLIENT_VERSION_MAJOR) ||
+ (client->version.minor != IAVF_CLIENT_VERSION_MINOR)) {
+ pr_info("iavf: Failed to register client %s due to mismatched client interface version\n",
+ client->name);
+ pr_info("Client is using version: %02d.%02d.%02d while LAN driver supports %s\n",
+ client->version.major, client->version.minor,
+ client->version.build,
+ iavf_client_interface_version_str);
+ ret = -EIO;
+ goto out;
+ }
+
+ vf_registered_client = client;
+
+ iavf_client_prepare(client);
+
+ pr_info("iavf: Registered client %s with return code %d\n",
+ client->name, ret);
+out:
+ return ret;
+}
+EXPORT_SYMBOL(iavf_register_client);
+
+/**
+ * iavf_unregister_client - Unregister a iavf client driver with the L2 driver
+ * @client: pointer to the iavf_client struct
+ *
+ * Returns 0 on success or non-0 on error
+ **/
+int iavf_unregister_client(struct iavf_client *client)
+{
+ int ret = 0;
+
+ /* When a unregister request comes through we would have to send
+ * a close for each of the client instances that were opened.
+ * client_release function is called to handle this.
+ */
+ iavf_client_release(client);
+
+ if (vf_registered_client != client) {
+ pr_info("iavf: Client %s has not been registered\n",
+ client->name);
+ ret = -ENODEV;
+ goto out;
+ }
+ vf_registered_client = NULL;
+ pr_info("iavf: Unregistered client %s\n", client->name);
+out:
+ return ret;
+}
+EXPORT_SYMBOL(iavf_unregister_client);
diff --git a/drivers/net/ethernet/intel/iavf/iavf_client.h b/drivers/net/ethernet/intel/iavf/iavf_client.h
new file mode 100644
index 0000000000..500269bc0f
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_client.h
@@ -0,0 +1,169 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_CLIENT_H_
+#define _IAVF_CLIENT_H_
+
+#define IAVF_CLIENT_STR_LENGTH 10
+
+/* Client interface version should be updated anytime there is a change in the
+ * existing APIs or data structures.
+ */
+#define IAVF_CLIENT_VERSION_MAJOR 0
+#define IAVF_CLIENT_VERSION_MINOR 01
+#define IAVF_CLIENT_VERSION_BUILD 00
+#define IAVF_CLIENT_VERSION_STR \
+ __stringify(IAVF_CLIENT_VERSION_MAJOR) "." \
+ __stringify(IAVF_CLIENT_VERSION_MINOR) "." \
+ __stringify(IAVF_CLIENT_VERSION_BUILD)
+
+struct iavf_client_version {
+ u8 major;
+ u8 minor;
+ u8 build;
+ u8 rsvd;
+};
+
+enum iavf_client_state {
+ __IAVF_CLIENT_NULL,
+ __IAVF_CLIENT_REGISTERED
+};
+
+enum iavf_client_instance_state {
+ __IAVF_CLIENT_INSTANCE_NONE,
+ __IAVF_CLIENT_INSTANCE_OPENED,
+};
+
+struct iavf_ops;
+struct iavf_client;
+
+/* HW does not define a type value for AEQ; only for RX/TX and CEQ.
+ * In order for us to keep the interface simple, SW will define a
+ * unique type value for AEQ.
+ */
+#define IAVF_QUEUE_TYPE_PE_AEQ 0x80
+#define IAVF_QUEUE_INVALID_IDX 0xFFFF
+
+struct iavf_qv_info {
+ u32 v_idx; /* msix_vector */
+ u16 ceq_idx;
+ u16 aeq_idx;
+ u8 itr_idx;
+};
+
+struct iavf_qvlist_info {
+ u32 num_vectors;
+ struct iavf_qv_info qv_info[];
+};
+
+#define IAVF_CLIENT_MSIX_ALL 0xFFFFFFFF
+
+/* set of LAN parameters useful for clients managed by LAN */
+
+/* Struct to hold per priority info */
+struct iavf_prio_qos_params {
+ u16 qs_handle; /* qs handle for prio */
+ u8 tc; /* TC mapped to prio */
+ u8 reserved;
+};
+
+#define IAVF_CLIENT_MAX_USER_PRIORITY 8
+/* Struct to hold Client QoS */
+struct iavf_qos_params {
+ struct iavf_prio_qos_params prio_qos[IAVF_CLIENT_MAX_USER_PRIORITY];
+};
+
+struct iavf_params {
+ struct iavf_qos_params qos;
+ u16 mtu;
+ u16 link_up; /* boolean */
+};
+
+/* Structure to hold LAN device info for a client device */
+struct iavf_info {
+ struct iavf_client_version version;
+ u8 lanmac[6];
+ struct net_device *netdev;
+ struct pci_dev *pcidev;
+ u8 __iomem *hw_addr;
+ u8 fid; /* function id, PF id or VF id */
+#define IAVF_CLIENT_FTYPE_PF 0
+#define IAVF_CLIENT_FTYPE_VF 1
+ u8 ftype; /* function type, PF or VF */
+ void *vf; /* cast to iavf_adapter */
+
+ /* All L2 params that could change during the life span of the device
+ * and needs to be communicated to the client when they change
+ */
+ struct iavf_params params;
+ struct iavf_ops *ops;
+
+ u16 msix_count; /* number of msix vectors*/
+ /* Array down below will be dynamically allocated based on msix_count */
+ struct msix_entry *msix_entries;
+ u16 itr_index; /* Which ITR index the PE driver is suppose to use */
+};
+
+struct iavf_ops {
+ /* setup_q_vector_list enables queues with a particular vector */
+ int (*setup_qvlist)(struct iavf_info *ldev, struct iavf_client *client,
+ struct iavf_qvlist_info *qv_info);
+
+ u32 (*virtchnl_send)(struct iavf_info *ldev, struct iavf_client *client,
+ u8 *msg, u16 len);
+
+ /* If the PE Engine is unresponsive, RDMA driver can request a reset.*/
+ void (*request_reset)(struct iavf_info *ldev,
+ struct iavf_client *client);
+};
+
+struct iavf_client_ops {
+ /* Should be called from register_client() or whenever the driver is
+ * ready to create a specific client instance.
+ */
+ int (*open)(struct iavf_info *ldev, struct iavf_client *client);
+
+ /* Should be closed when netdev is unavailable or when unregister
+ * call comes in. If the close happens due to a reset, set the reset
+ * bit to true.
+ */
+ void (*close)(struct iavf_info *ldev, struct iavf_client *client,
+ bool reset);
+
+ /* called when some l2 managed parameters changes - mss */
+ void (*l2_param_change)(struct iavf_info *ldev,
+ struct iavf_client *client,
+ struct iavf_params *params);
+
+ /* called when a message is received from the PF */
+ int (*virtchnl_receive)(struct iavf_info *ldev,
+ struct iavf_client *client,
+ u8 *msg, u16 len);
+};
+
+/* Client device */
+struct iavf_client_instance {
+ struct list_head list;
+ struct iavf_info lan_info;
+ struct iavf_client *client;
+ unsigned long state;
+};
+
+struct iavf_client {
+ struct list_head list; /* list of registered clients */
+ char name[IAVF_CLIENT_STR_LENGTH];
+ struct iavf_client_version version;
+ unsigned long state; /* client state */
+ atomic_t ref_cnt; /* Count of all the client devices of this kind */
+ u32 flags;
+#define IAVF_CLIENT_FLAGS_LAUNCH_ON_PROBE BIT(0)
+#define IAVF_TX_FLAGS_NOTIFY_OTHER_EVENTS BIT(2)
+ u8 type;
+#define IAVF_CLIENT_RDMA 0
+ struct iavf_client_ops *ops; /* client ops provided by the client */
+};
+
+/* used by clients */
+int iavf_register_client(struct iavf_client *client);
+int iavf_unregister_client(struct iavf_client *client);
+#endif /* _IAVF_CLIENT_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_common.c b/drivers/net/ethernet/intel/iavf/iavf_common.c
new file mode 100644
index 0000000000..1afd761d80
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_common.c
@@ -0,0 +1,799 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#include "iavf_type.h"
+#include "iavf_adminq.h"
+#include "iavf_prototype.h"
+#include <linux/avf/virtchnl.h>
+
+/**
+ * iavf_set_mac_type - Sets MAC type
+ * @hw: pointer to the HW structure
+ *
+ * This function sets the mac type of the adapter based on the
+ * vendor ID and device ID stored in the hw structure.
+ **/
+enum iavf_status iavf_set_mac_type(struct iavf_hw *hw)
+{
+ enum iavf_status status = 0;
+
+ if (hw->vendor_id == PCI_VENDOR_ID_INTEL) {
+ switch (hw->device_id) {
+ case IAVF_DEV_ID_X722_VF:
+ hw->mac.type = IAVF_MAC_X722_VF;
+ break;
+ case IAVF_DEV_ID_VF:
+ case IAVF_DEV_ID_VF_HV:
+ case IAVF_DEV_ID_ADAPTIVE_VF:
+ hw->mac.type = IAVF_MAC_VF;
+ break;
+ default:
+ hw->mac.type = IAVF_MAC_GENERIC;
+ break;
+ }
+ } else {
+ status = IAVF_ERR_DEVICE_NOT_SUPPORTED;
+ }
+
+ return status;
+}
+
+/**
+ * iavf_aq_str - convert AQ err code to a string
+ * @hw: pointer to the HW structure
+ * @aq_err: the AQ error code to convert
+ **/
+const char *iavf_aq_str(struct iavf_hw *hw, enum iavf_admin_queue_err aq_err)
+{
+ switch (aq_err) {
+ case IAVF_AQ_RC_OK:
+ return "OK";
+ case IAVF_AQ_RC_EPERM:
+ return "IAVF_AQ_RC_EPERM";
+ case IAVF_AQ_RC_ENOENT:
+ return "IAVF_AQ_RC_ENOENT";
+ case IAVF_AQ_RC_ESRCH:
+ return "IAVF_AQ_RC_ESRCH";
+ case IAVF_AQ_RC_EINTR:
+ return "IAVF_AQ_RC_EINTR";
+ case IAVF_AQ_RC_EIO:
+ return "IAVF_AQ_RC_EIO";
+ case IAVF_AQ_RC_ENXIO:
+ return "IAVF_AQ_RC_ENXIO";
+ case IAVF_AQ_RC_E2BIG:
+ return "IAVF_AQ_RC_E2BIG";
+ case IAVF_AQ_RC_EAGAIN:
+ return "IAVF_AQ_RC_EAGAIN";
+ case IAVF_AQ_RC_ENOMEM:
+ return "IAVF_AQ_RC_ENOMEM";
+ case IAVF_AQ_RC_EACCES:
+ return "IAVF_AQ_RC_EACCES";
+ case IAVF_AQ_RC_EFAULT:
+ return "IAVF_AQ_RC_EFAULT";
+ case IAVF_AQ_RC_EBUSY:
+ return "IAVF_AQ_RC_EBUSY";
+ case IAVF_AQ_RC_EEXIST:
+ return "IAVF_AQ_RC_EEXIST";
+ case IAVF_AQ_RC_EINVAL:
+ return "IAVF_AQ_RC_EINVAL";
+ case IAVF_AQ_RC_ENOTTY:
+ return "IAVF_AQ_RC_ENOTTY";
+ case IAVF_AQ_RC_ENOSPC:
+ return "IAVF_AQ_RC_ENOSPC";
+ case IAVF_AQ_RC_ENOSYS:
+ return "IAVF_AQ_RC_ENOSYS";
+ case IAVF_AQ_RC_ERANGE:
+ return "IAVF_AQ_RC_ERANGE";
+ case IAVF_AQ_RC_EFLUSHED:
+ return "IAVF_AQ_RC_EFLUSHED";
+ case IAVF_AQ_RC_BAD_ADDR:
+ return "IAVF_AQ_RC_BAD_ADDR";
+ case IAVF_AQ_RC_EMODE:
+ return "IAVF_AQ_RC_EMODE";
+ case IAVF_AQ_RC_EFBIG:
+ return "IAVF_AQ_RC_EFBIG";
+ }
+
+ snprintf(hw->err_str, sizeof(hw->err_str), "%d", aq_err);
+ return hw->err_str;
+}
+
+/**
+ * iavf_stat_str - convert status err code to a string
+ * @hw: pointer to the HW structure
+ * @stat_err: the status error code to convert
+ **/
+const char *iavf_stat_str(struct iavf_hw *hw, enum iavf_status stat_err)
+{
+ switch (stat_err) {
+ case 0:
+ return "OK";
+ case IAVF_ERR_NVM:
+ return "IAVF_ERR_NVM";
+ case IAVF_ERR_NVM_CHECKSUM:
+ return "IAVF_ERR_NVM_CHECKSUM";
+ case IAVF_ERR_PHY:
+ return "IAVF_ERR_PHY";
+ case IAVF_ERR_CONFIG:
+ return "IAVF_ERR_CONFIG";
+ case IAVF_ERR_PARAM:
+ return "IAVF_ERR_PARAM";
+ case IAVF_ERR_MAC_TYPE:
+ return "IAVF_ERR_MAC_TYPE";
+ case IAVF_ERR_UNKNOWN_PHY:
+ return "IAVF_ERR_UNKNOWN_PHY";
+ case IAVF_ERR_LINK_SETUP:
+ return "IAVF_ERR_LINK_SETUP";
+ case IAVF_ERR_ADAPTER_STOPPED:
+ return "IAVF_ERR_ADAPTER_STOPPED";
+ case IAVF_ERR_INVALID_MAC_ADDR:
+ return "IAVF_ERR_INVALID_MAC_ADDR";
+ case IAVF_ERR_DEVICE_NOT_SUPPORTED:
+ return "IAVF_ERR_DEVICE_NOT_SUPPORTED";
+ case IAVF_ERR_PRIMARY_REQUESTS_PENDING:
+ return "IAVF_ERR_PRIMARY_REQUESTS_PENDING";
+ case IAVF_ERR_INVALID_LINK_SETTINGS:
+ return "IAVF_ERR_INVALID_LINK_SETTINGS";
+ case IAVF_ERR_AUTONEG_NOT_COMPLETE:
+ return "IAVF_ERR_AUTONEG_NOT_COMPLETE";
+ case IAVF_ERR_RESET_FAILED:
+ return "IAVF_ERR_RESET_FAILED";
+ case IAVF_ERR_SWFW_SYNC:
+ return "IAVF_ERR_SWFW_SYNC";
+ case IAVF_ERR_NO_AVAILABLE_VSI:
+ return "IAVF_ERR_NO_AVAILABLE_VSI";
+ case IAVF_ERR_NO_MEMORY:
+ return "IAVF_ERR_NO_MEMORY";
+ case IAVF_ERR_BAD_PTR:
+ return "IAVF_ERR_BAD_PTR";
+ case IAVF_ERR_RING_FULL:
+ return "IAVF_ERR_RING_FULL";
+ case IAVF_ERR_INVALID_PD_ID:
+ return "IAVF_ERR_INVALID_PD_ID";
+ case IAVF_ERR_INVALID_QP_ID:
+ return "IAVF_ERR_INVALID_QP_ID";
+ case IAVF_ERR_INVALID_CQ_ID:
+ return "IAVF_ERR_INVALID_CQ_ID";
+ case IAVF_ERR_INVALID_CEQ_ID:
+ return "IAVF_ERR_INVALID_CEQ_ID";
+ case IAVF_ERR_INVALID_AEQ_ID:
+ return "IAVF_ERR_INVALID_AEQ_ID";
+ case IAVF_ERR_INVALID_SIZE:
+ return "IAVF_ERR_INVALID_SIZE";
+ case IAVF_ERR_INVALID_ARP_INDEX:
+ return "IAVF_ERR_INVALID_ARP_INDEX";
+ case IAVF_ERR_INVALID_FPM_FUNC_ID:
+ return "IAVF_ERR_INVALID_FPM_FUNC_ID";
+ case IAVF_ERR_QP_INVALID_MSG_SIZE:
+ return "IAVF_ERR_QP_INVALID_MSG_SIZE";
+ case IAVF_ERR_QP_TOOMANY_WRS_POSTED:
+ return "IAVF_ERR_QP_TOOMANY_WRS_POSTED";
+ case IAVF_ERR_INVALID_FRAG_COUNT:
+ return "IAVF_ERR_INVALID_FRAG_COUNT";
+ case IAVF_ERR_QUEUE_EMPTY:
+ return "IAVF_ERR_QUEUE_EMPTY";
+ case IAVF_ERR_INVALID_ALIGNMENT:
+ return "IAVF_ERR_INVALID_ALIGNMENT";
+ case IAVF_ERR_FLUSHED_QUEUE:
+ return "IAVF_ERR_FLUSHED_QUEUE";
+ case IAVF_ERR_INVALID_PUSH_PAGE_INDEX:
+ return "IAVF_ERR_INVALID_PUSH_PAGE_INDEX";
+ case IAVF_ERR_INVALID_IMM_DATA_SIZE:
+ return "IAVF_ERR_INVALID_IMM_DATA_SIZE";
+ case IAVF_ERR_TIMEOUT:
+ return "IAVF_ERR_TIMEOUT";
+ case IAVF_ERR_OPCODE_MISMATCH:
+ return "IAVF_ERR_OPCODE_MISMATCH";
+ case IAVF_ERR_CQP_COMPL_ERROR:
+ return "IAVF_ERR_CQP_COMPL_ERROR";
+ case IAVF_ERR_INVALID_VF_ID:
+ return "IAVF_ERR_INVALID_VF_ID";
+ case IAVF_ERR_INVALID_HMCFN_ID:
+ return "IAVF_ERR_INVALID_HMCFN_ID";
+ case IAVF_ERR_BACKING_PAGE_ERROR:
+ return "IAVF_ERR_BACKING_PAGE_ERROR";
+ case IAVF_ERR_NO_PBLCHUNKS_AVAILABLE:
+ return "IAVF_ERR_NO_PBLCHUNKS_AVAILABLE";
+ case IAVF_ERR_INVALID_PBLE_INDEX:
+ return "IAVF_ERR_INVALID_PBLE_INDEX";
+ case IAVF_ERR_INVALID_SD_INDEX:
+ return "IAVF_ERR_INVALID_SD_INDEX";
+ case IAVF_ERR_INVALID_PAGE_DESC_INDEX:
+ return "IAVF_ERR_INVALID_PAGE_DESC_INDEX";
+ case IAVF_ERR_INVALID_SD_TYPE:
+ return "IAVF_ERR_INVALID_SD_TYPE";
+ case IAVF_ERR_MEMCPY_FAILED:
+ return "IAVF_ERR_MEMCPY_FAILED";
+ case IAVF_ERR_INVALID_HMC_OBJ_INDEX:
+ return "IAVF_ERR_INVALID_HMC_OBJ_INDEX";
+ case IAVF_ERR_INVALID_HMC_OBJ_COUNT:
+ return "IAVF_ERR_INVALID_HMC_OBJ_COUNT";
+ case IAVF_ERR_INVALID_SRQ_ARM_LIMIT:
+ return "IAVF_ERR_INVALID_SRQ_ARM_LIMIT";
+ case IAVF_ERR_SRQ_ENABLED:
+ return "IAVF_ERR_SRQ_ENABLED";
+ case IAVF_ERR_ADMIN_QUEUE_ERROR:
+ return "IAVF_ERR_ADMIN_QUEUE_ERROR";
+ case IAVF_ERR_ADMIN_QUEUE_TIMEOUT:
+ return "IAVF_ERR_ADMIN_QUEUE_TIMEOUT";
+ case IAVF_ERR_BUF_TOO_SHORT:
+ return "IAVF_ERR_BUF_TOO_SHORT";
+ case IAVF_ERR_ADMIN_QUEUE_FULL:
+ return "IAVF_ERR_ADMIN_QUEUE_FULL";
+ case IAVF_ERR_ADMIN_QUEUE_NO_WORK:
+ return "IAVF_ERR_ADMIN_QUEUE_NO_WORK";
+ case IAVF_ERR_BAD_RDMA_CQE:
+ return "IAVF_ERR_BAD_RDMA_CQE";
+ case IAVF_ERR_NVM_BLANK_MODE:
+ return "IAVF_ERR_NVM_BLANK_MODE";
+ case IAVF_ERR_NOT_IMPLEMENTED:
+ return "IAVF_ERR_NOT_IMPLEMENTED";
+ case IAVF_ERR_PE_DOORBELL_NOT_ENABLED:
+ return "IAVF_ERR_PE_DOORBELL_NOT_ENABLED";
+ case IAVF_ERR_DIAG_TEST_FAILED:
+ return "IAVF_ERR_DIAG_TEST_FAILED";
+ case IAVF_ERR_NOT_READY:
+ return "IAVF_ERR_NOT_READY";
+ case IAVF_NOT_SUPPORTED:
+ return "IAVF_NOT_SUPPORTED";
+ case IAVF_ERR_FIRMWARE_API_VERSION:
+ return "IAVF_ERR_FIRMWARE_API_VERSION";
+ case IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR:
+ return "IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR";
+ }
+
+ snprintf(hw->err_str, sizeof(hw->err_str), "%d", stat_err);
+ return hw->err_str;
+}
+
+/**
+ * iavf_debug_aq
+ * @hw: debug mask related to admin queue
+ * @mask: debug mask
+ * @desc: pointer to admin queue descriptor
+ * @buffer: pointer to command buffer
+ * @buf_len: max length of buffer
+ *
+ * Dumps debug log about adminq command with descriptor contents.
+ **/
+void iavf_debug_aq(struct iavf_hw *hw, enum iavf_debug_mask mask, void *desc,
+ void *buffer, u16 buf_len)
+{
+ struct iavf_aq_desc *aq_desc = (struct iavf_aq_desc *)desc;
+ u8 *buf = (u8 *)buffer;
+
+ if ((!(mask & hw->debug_mask)) || !desc)
+ return;
+
+ iavf_debug(hw, mask,
+ "AQ CMD: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n",
+ le16_to_cpu(aq_desc->opcode),
+ le16_to_cpu(aq_desc->flags),
+ le16_to_cpu(aq_desc->datalen),
+ le16_to_cpu(aq_desc->retval));
+ iavf_debug(hw, mask, "\tcookie (h,l) 0x%08X 0x%08X\n",
+ le32_to_cpu(aq_desc->cookie_high),
+ le32_to_cpu(aq_desc->cookie_low));
+ iavf_debug(hw, mask, "\tparam (0,1) 0x%08X 0x%08X\n",
+ le32_to_cpu(aq_desc->params.internal.param0),
+ le32_to_cpu(aq_desc->params.internal.param1));
+ iavf_debug(hw, mask, "\taddr (h,l) 0x%08X 0x%08X\n",
+ le32_to_cpu(aq_desc->params.external.addr_high),
+ le32_to_cpu(aq_desc->params.external.addr_low));
+
+ if (buffer && aq_desc->datalen) {
+ u16 len = le16_to_cpu(aq_desc->datalen);
+
+ iavf_debug(hw, mask, "AQ CMD Buffer:\n");
+ if (buf_len < len)
+ len = buf_len;
+ /* write the full 16-byte chunks */
+ if (hw->debug_mask & mask) {
+ char prefix[27];
+
+ snprintf(prefix, sizeof(prefix),
+ "iavf %02x:%02x.%x: \t0x",
+ hw->bus.bus_id,
+ hw->bus.device,
+ hw->bus.func);
+
+ print_hex_dump(KERN_INFO, prefix, DUMP_PREFIX_OFFSET,
+ 16, 1, buf, len, false);
+ }
+ }
+}
+
+/**
+ * iavf_check_asq_alive
+ * @hw: pointer to the hw struct
+ *
+ * Returns true if Queue is enabled else false.
+ **/
+bool iavf_check_asq_alive(struct iavf_hw *hw)
+{
+ if (hw->aq.asq.len)
+ return !!(rd32(hw, hw->aq.asq.len) &
+ IAVF_VF_ATQLEN1_ATQENABLE_MASK);
+ else
+ return false;
+}
+
+/**
+ * iavf_aq_queue_shutdown
+ * @hw: pointer to the hw struct
+ * @unloading: is the driver unloading itself
+ *
+ * Tell the Firmware that we're shutting down the AdminQ and whether
+ * or not the driver is unloading as well.
+ **/
+enum iavf_status iavf_aq_queue_shutdown(struct iavf_hw *hw, bool unloading)
+{
+ struct iavf_aq_desc desc;
+ struct iavf_aqc_queue_shutdown *cmd =
+ (struct iavf_aqc_queue_shutdown *)&desc.params.raw;
+ enum iavf_status status;
+
+ iavf_fill_default_direct_cmd_desc(&desc, iavf_aqc_opc_queue_shutdown);
+
+ if (unloading)
+ cmd->driver_unloading = cpu_to_le32(IAVF_AQ_DRIVER_UNLOADING);
+ status = iavf_asq_send_command(hw, &desc, NULL, 0, NULL);
+
+ return status;
+}
+
+/**
+ * iavf_aq_get_set_rss_lut
+ * @hw: pointer to the hardware structure
+ * @vsi_id: vsi fw index
+ * @pf_lut: for PF table set true, for VSI table set false
+ * @lut: pointer to the lut buffer provided by the caller
+ * @lut_size: size of the lut buffer
+ * @set: set true to set the table, false to get the table
+ *
+ * Internal function to get or set RSS look up table
+ **/
+static enum iavf_status iavf_aq_get_set_rss_lut(struct iavf_hw *hw,
+ u16 vsi_id, bool pf_lut,
+ u8 *lut, u16 lut_size,
+ bool set)
+{
+ enum iavf_status status;
+ struct iavf_aq_desc desc;
+ struct iavf_aqc_get_set_rss_lut *cmd_resp =
+ (struct iavf_aqc_get_set_rss_lut *)&desc.params.raw;
+
+ if (set)
+ iavf_fill_default_direct_cmd_desc(&desc,
+ iavf_aqc_opc_set_rss_lut);
+ else
+ iavf_fill_default_direct_cmd_desc(&desc,
+ iavf_aqc_opc_get_rss_lut);
+
+ /* Indirect command */
+ desc.flags |= cpu_to_le16((u16)IAVF_AQ_FLAG_BUF);
+ desc.flags |= cpu_to_le16((u16)IAVF_AQ_FLAG_RD);
+
+ cmd_resp->vsi_id =
+ cpu_to_le16((u16)((vsi_id <<
+ IAVF_AQC_SET_RSS_LUT_VSI_ID_SHIFT) &
+ IAVF_AQC_SET_RSS_LUT_VSI_ID_MASK));
+ cmd_resp->vsi_id |= cpu_to_le16((u16)IAVF_AQC_SET_RSS_LUT_VSI_VALID);
+
+ if (pf_lut)
+ cmd_resp->flags |= cpu_to_le16((u16)
+ ((IAVF_AQC_SET_RSS_LUT_TABLE_TYPE_PF <<
+ IAVF_AQC_SET_RSS_LUT_TABLE_TYPE_SHIFT) &
+ IAVF_AQC_SET_RSS_LUT_TABLE_TYPE_MASK));
+ else
+ cmd_resp->flags |= cpu_to_le16((u16)
+ ((IAVF_AQC_SET_RSS_LUT_TABLE_TYPE_VSI <<
+ IAVF_AQC_SET_RSS_LUT_TABLE_TYPE_SHIFT) &
+ IAVF_AQC_SET_RSS_LUT_TABLE_TYPE_MASK));
+
+ status = iavf_asq_send_command(hw, &desc, lut, lut_size, NULL);
+
+ return status;
+}
+
+/**
+ * iavf_aq_set_rss_lut
+ * @hw: pointer to the hardware structure
+ * @vsi_id: vsi fw index
+ * @pf_lut: for PF table set true, for VSI table set false
+ * @lut: pointer to the lut buffer provided by the caller
+ * @lut_size: size of the lut buffer
+ *
+ * set the RSS lookup table, PF or VSI type
+ **/
+enum iavf_status iavf_aq_set_rss_lut(struct iavf_hw *hw, u16 vsi_id,
+ bool pf_lut, u8 *lut, u16 lut_size)
+{
+ return iavf_aq_get_set_rss_lut(hw, vsi_id, pf_lut, lut, lut_size, true);
+}
+
+/**
+ * iavf_aq_get_set_rss_key
+ * @hw: pointer to the hw struct
+ * @vsi_id: vsi fw index
+ * @key: pointer to key info struct
+ * @set: set true to set the key, false to get the key
+ *
+ * get the RSS key per VSI
+ **/
+static enum
+iavf_status iavf_aq_get_set_rss_key(struct iavf_hw *hw, u16 vsi_id,
+ struct iavf_aqc_get_set_rss_key_data *key,
+ bool set)
+{
+ enum iavf_status status;
+ struct iavf_aq_desc desc;
+ struct iavf_aqc_get_set_rss_key *cmd_resp =
+ (struct iavf_aqc_get_set_rss_key *)&desc.params.raw;
+ u16 key_size = sizeof(struct iavf_aqc_get_set_rss_key_data);
+
+ if (set)
+ iavf_fill_default_direct_cmd_desc(&desc,
+ iavf_aqc_opc_set_rss_key);
+ else
+ iavf_fill_default_direct_cmd_desc(&desc,
+ iavf_aqc_opc_get_rss_key);
+
+ /* Indirect command */
+ desc.flags |= cpu_to_le16((u16)IAVF_AQ_FLAG_BUF);
+ desc.flags |= cpu_to_le16((u16)IAVF_AQ_FLAG_RD);
+
+ cmd_resp->vsi_id =
+ cpu_to_le16((u16)((vsi_id <<
+ IAVF_AQC_SET_RSS_KEY_VSI_ID_SHIFT) &
+ IAVF_AQC_SET_RSS_KEY_VSI_ID_MASK));
+ cmd_resp->vsi_id |= cpu_to_le16((u16)IAVF_AQC_SET_RSS_KEY_VSI_VALID);
+
+ status = iavf_asq_send_command(hw, &desc, key, key_size, NULL);
+
+ return status;
+}
+
+/**
+ * iavf_aq_set_rss_key
+ * @hw: pointer to the hw struct
+ * @vsi_id: vsi fw index
+ * @key: pointer to key info struct
+ *
+ * set the RSS key per VSI
+ **/
+enum iavf_status iavf_aq_set_rss_key(struct iavf_hw *hw, u16 vsi_id,
+ struct iavf_aqc_get_set_rss_key_data *key)
+{
+ return iavf_aq_get_set_rss_key(hw, vsi_id, key, true);
+}
+
+/* The iavf_ptype_lookup table is used to convert from the 8-bit ptype in the
+ * hardware to a bit-field that can be used by SW to more easily determine the
+ * packet type.
+ *
+ * Macros are used to shorten the table lines and make this table human
+ * readable.
+ *
+ * We store the PTYPE in the top byte of the bit field - this is just so that
+ * we can check that the table doesn't have a row missing, as the index into
+ * the table should be the PTYPE.
+ *
+ * Typical work flow:
+ *
+ * IF NOT iavf_ptype_lookup[ptype].known
+ * THEN
+ * Packet is unknown
+ * ELSE IF iavf_ptype_lookup[ptype].outer_ip == IAVF_RX_PTYPE_OUTER_IP
+ * Use the rest of the fields to look at the tunnels, inner protocols, etc
+ * ELSE
+ * Use the enum iavf_rx_l2_ptype to decode the packet type
+ * ENDIF
+ */
+
+/* macro to make the table lines short, use explicit indexing with [PTYPE] */
+#define IAVF_PTT(PTYPE, OUTER_IP, OUTER_IP_VER, OUTER_FRAG, T, TE, TEF, I, PL)\
+ [PTYPE] = { \
+ 1, \
+ IAVF_RX_PTYPE_OUTER_##OUTER_IP, \
+ IAVF_RX_PTYPE_OUTER_##OUTER_IP_VER, \
+ IAVF_RX_PTYPE_##OUTER_FRAG, \
+ IAVF_RX_PTYPE_TUNNEL_##T, \
+ IAVF_RX_PTYPE_TUNNEL_END_##TE, \
+ IAVF_RX_PTYPE_##TEF, \
+ IAVF_RX_PTYPE_INNER_PROT_##I, \
+ IAVF_RX_PTYPE_PAYLOAD_LAYER_##PL }
+
+#define IAVF_PTT_UNUSED_ENTRY(PTYPE) [PTYPE] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+
+/* shorter macros makes the table fit but are terse */
+#define IAVF_RX_PTYPE_NOF IAVF_RX_PTYPE_NOT_FRAG
+#define IAVF_RX_PTYPE_FRG IAVF_RX_PTYPE_FRAG
+#define IAVF_RX_PTYPE_INNER_PROT_TS IAVF_RX_PTYPE_INNER_PROT_TIMESYNC
+
+/* Lookup table mapping the 8-bit HW PTYPE to the bit field for decoding */
+struct iavf_rx_ptype_decoded iavf_ptype_lookup[BIT(8)] = {
+ /* L2 Packet types */
+ IAVF_PTT_UNUSED_ENTRY(0),
+ IAVF_PTT(1, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY2),
+ IAVF_PTT(2, L2, NONE, NOF, NONE, NONE, NOF, TS, PAY2),
+ IAVF_PTT(3, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY2),
+ IAVF_PTT_UNUSED_ENTRY(4),
+ IAVF_PTT_UNUSED_ENTRY(5),
+ IAVF_PTT(6, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY2),
+ IAVF_PTT(7, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY2),
+ IAVF_PTT_UNUSED_ENTRY(8),
+ IAVF_PTT_UNUSED_ENTRY(9),
+ IAVF_PTT(10, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY2),
+ IAVF_PTT(11, L2, NONE, NOF, NONE, NONE, NOF, NONE, NONE),
+ IAVF_PTT(12, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(13, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(14, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(15, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(16, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(17, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(18, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(19, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(20, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(21, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY3),
+
+ /* Non Tunneled IPv4 */
+ IAVF_PTT(22, IP, IPV4, FRG, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(23, IP, IPV4, NOF, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(24, IP, IPV4, NOF, NONE, NONE, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(25),
+ IAVF_PTT(26, IP, IPV4, NOF, NONE, NONE, NOF, TCP, PAY4),
+ IAVF_PTT(27, IP, IPV4, NOF, NONE, NONE, NOF, SCTP, PAY4),
+ IAVF_PTT(28, IP, IPV4, NOF, NONE, NONE, NOF, ICMP, PAY4),
+
+ /* IPv4 --> IPv4 */
+ IAVF_PTT(29, IP, IPV4, NOF, IP_IP, IPV4, FRG, NONE, PAY3),
+ IAVF_PTT(30, IP, IPV4, NOF, IP_IP, IPV4, NOF, NONE, PAY3),
+ IAVF_PTT(31, IP, IPV4, NOF, IP_IP, IPV4, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(32),
+ IAVF_PTT(33, IP, IPV4, NOF, IP_IP, IPV4, NOF, TCP, PAY4),
+ IAVF_PTT(34, IP, IPV4, NOF, IP_IP, IPV4, NOF, SCTP, PAY4),
+ IAVF_PTT(35, IP, IPV4, NOF, IP_IP, IPV4, NOF, ICMP, PAY4),
+
+ /* IPv4 --> IPv6 */
+ IAVF_PTT(36, IP, IPV4, NOF, IP_IP, IPV6, FRG, NONE, PAY3),
+ IAVF_PTT(37, IP, IPV4, NOF, IP_IP, IPV6, NOF, NONE, PAY3),
+ IAVF_PTT(38, IP, IPV4, NOF, IP_IP, IPV6, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(39),
+ IAVF_PTT(40, IP, IPV4, NOF, IP_IP, IPV6, NOF, TCP, PAY4),
+ IAVF_PTT(41, IP, IPV4, NOF, IP_IP, IPV6, NOF, SCTP, PAY4),
+ IAVF_PTT(42, IP, IPV4, NOF, IP_IP, IPV6, NOF, ICMP, PAY4),
+
+ /* IPv4 --> GRE/NAT */
+ IAVF_PTT(43, IP, IPV4, NOF, IP_GRENAT, NONE, NOF, NONE, PAY3),
+
+ /* IPv4 --> GRE/NAT --> IPv4 */
+ IAVF_PTT(44, IP, IPV4, NOF, IP_GRENAT, IPV4, FRG, NONE, PAY3),
+ IAVF_PTT(45, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, NONE, PAY3),
+ IAVF_PTT(46, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(47),
+ IAVF_PTT(48, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, TCP, PAY4),
+ IAVF_PTT(49, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, SCTP, PAY4),
+ IAVF_PTT(50, IP, IPV4, NOF, IP_GRENAT, IPV4, NOF, ICMP, PAY4),
+
+ /* IPv4 --> GRE/NAT --> IPv6 */
+ IAVF_PTT(51, IP, IPV4, NOF, IP_GRENAT, IPV6, FRG, NONE, PAY3),
+ IAVF_PTT(52, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, NONE, PAY3),
+ IAVF_PTT(53, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(54),
+ IAVF_PTT(55, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, TCP, PAY4),
+ IAVF_PTT(56, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, SCTP, PAY4),
+ IAVF_PTT(57, IP, IPV4, NOF, IP_GRENAT, IPV6, NOF, ICMP, PAY4),
+
+ /* IPv4 --> GRE/NAT --> MAC */
+ IAVF_PTT(58, IP, IPV4, NOF, IP_GRENAT_MAC, NONE, NOF, NONE, PAY3),
+
+ /* IPv4 --> GRE/NAT --> MAC --> IPv4 */
+ IAVF_PTT(59, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, FRG, NONE, PAY3),
+ IAVF_PTT(60, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, NONE, PAY3),
+ IAVF_PTT(61, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(62),
+ IAVF_PTT(63, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, TCP, PAY4),
+ IAVF_PTT(64, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, SCTP, PAY4),
+ IAVF_PTT(65, IP, IPV4, NOF, IP_GRENAT_MAC, IPV4, NOF, ICMP, PAY4),
+
+ /* IPv4 --> GRE/NAT -> MAC --> IPv6 */
+ IAVF_PTT(66, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, FRG, NONE, PAY3),
+ IAVF_PTT(67, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, NONE, PAY3),
+ IAVF_PTT(68, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(69),
+ IAVF_PTT(70, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, TCP, PAY4),
+ IAVF_PTT(71, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, SCTP, PAY4),
+ IAVF_PTT(72, IP, IPV4, NOF, IP_GRENAT_MAC, IPV6, NOF, ICMP, PAY4),
+
+ /* IPv4 --> GRE/NAT --> MAC/VLAN */
+ IAVF_PTT(73, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, NONE, NOF, NONE, PAY3),
+
+ /* IPv4 ---> GRE/NAT -> MAC/VLAN --> IPv4 */
+ IAVF_PTT(74, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, FRG, NONE, PAY3),
+ IAVF_PTT(75, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, NONE, PAY3),
+ IAVF_PTT(76, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(77),
+ IAVF_PTT(78, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, TCP, PAY4),
+ IAVF_PTT(79, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, SCTP, PAY4),
+ IAVF_PTT(80, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, ICMP, PAY4),
+
+ /* IPv4 -> GRE/NAT -> MAC/VLAN --> IPv6 */
+ IAVF_PTT(81, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, FRG, NONE, PAY3),
+ IAVF_PTT(82, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, NONE, PAY3),
+ IAVF_PTT(83, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(84),
+ IAVF_PTT(85, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, TCP, PAY4),
+ IAVF_PTT(86, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, SCTP, PAY4),
+ IAVF_PTT(87, IP, IPV4, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, ICMP, PAY4),
+
+ /* Non Tunneled IPv6 */
+ IAVF_PTT(88, IP, IPV6, FRG, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(89, IP, IPV6, NOF, NONE, NONE, NOF, NONE, PAY3),
+ IAVF_PTT(90, IP, IPV6, NOF, NONE, NONE, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(91),
+ IAVF_PTT(92, IP, IPV6, NOF, NONE, NONE, NOF, TCP, PAY4),
+ IAVF_PTT(93, IP, IPV6, NOF, NONE, NONE, NOF, SCTP, PAY4),
+ IAVF_PTT(94, IP, IPV6, NOF, NONE, NONE, NOF, ICMP, PAY4),
+
+ /* IPv6 --> IPv4 */
+ IAVF_PTT(95, IP, IPV6, NOF, IP_IP, IPV4, FRG, NONE, PAY3),
+ IAVF_PTT(96, IP, IPV6, NOF, IP_IP, IPV4, NOF, NONE, PAY3),
+ IAVF_PTT(97, IP, IPV6, NOF, IP_IP, IPV4, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(98),
+ IAVF_PTT(99, IP, IPV6, NOF, IP_IP, IPV4, NOF, TCP, PAY4),
+ IAVF_PTT(100, IP, IPV6, NOF, IP_IP, IPV4, NOF, SCTP, PAY4),
+ IAVF_PTT(101, IP, IPV6, NOF, IP_IP, IPV4, NOF, ICMP, PAY4),
+
+ /* IPv6 --> IPv6 */
+ IAVF_PTT(102, IP, IPV6, NOF, IP_IP, IPV6, FRG, NONE, PAY3),
+ IAVF_PTT(103, IP, IPV6, NOF, IP_IP, IPV6, NOF, NONE, PAY3),
+ IAVF_PTT(104, IP, IPV6, NOF, IP_IP, IPV6, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(105),
+ IAVF_PTT(106, IP, IPV6, NOF, IP_IP, IPV6, NOF, TCP, PAY4),
+ IAVF_PTT(107, IP, IPV6, NOF, IP_IP, IPV6, NOF, SCTP, PAY4),
+ IAVF_PTT(108, IP, IPV6, NOF, IP_IP, IPV6, NOF, ICMP, PAY4),
+
+ /* IPv6 --> GRE/NAT */
+ IAVF_PTT(109, IP, IPV6, NOF, IP_GRENAT, NONE, NOF, NONE, PAY3),
+
+ /* IPv6 --> GRE/NAT -> IPv4 */
+ IAVF_PTT(110, IP, IPV6, NOF, IP_GRENAT, IPV4, FRG, NONE, PAY3),
+ IAVF_PTT(111, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, NONE, PAY3),
+ IAVF_PTT(112, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(113),
+ IAVF_PTT(114, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, TCP, PAY4),
+ IAVF_PTT(115, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, SCTP, PAY4),
+ IAVF_PTT(116, IP, IPV6, NOF, IP_GRENAT, IPV4, NOF, ICMP, PAY4),
+
+ /* IPv6 --> GRE/NAT -> IPv6 */
+ IAVF_PTT(117, IP, IPV6, NOF, IP_GRENAT, IPV6, FRG, NONE, PAY3),
+ IAVF_PTT(118, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, NONE, PAY3),
+ IAVF_PTT(119, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(120),
+ IAVF_PTT(121, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, TCP, PAY4),
+ IAVF_PTT(122, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, SCTP, PAY4),
+ IAVF_PTT(123, IP, IPV6, NOF, IP_GRENAT, IPV6, NOF, ICMP, PAY4),
+
+ /* IPv6 --> GRE/NAT -> MAC */
+ IAVF_PTT(124, IP, IPV6, NOF, IP_GRENAT_MAC, NONE, NOF, NONE, PAY3),
+
+ /* IPv6 --> GRE/NAT -> MAC -> IPv4 */
+ IAVF_PTT(125, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, FRG, NONE, PAY3),
+ IAVF_PTT(126, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, NONE, PAY3),
+ IAVF_PTT(127, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(128),
+ IAVF_PTT(129, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, TCP, PAY4),
+ IAVF_PTT(130, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, SCTP, PAY4),
+ IAVF_PTT(131, IP, IPV6, NOF, IP_GRENAT_MAC, IPV4, NOF, ICMP, PAY4),
+
+ /* IPv6 --> GRE/NAT -> MAC -> IPv6 */
+ IAVF_PTT(132, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, FRG, NONE, PAY3),
+ IAVF_PTT(133, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, NONE, PAY3),
+ IAVF_PTT(134, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(135),
+ IAVF_PTT(136, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, TCP, PAY4),
+ IAVF_PTT(137, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, SCTP, PAY4),
+ IAVF_PTT(138, IP, IPV6, NOF, IP_GRENAT_MAC, IPV6, NOF, ICMP, PAY4),
+
+ /* IPv6 --> GRE/NAT -> MAC/VLAN */
+ IAVF_PTT(139, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, NONE, NOF, NONE, PAY3),
+
+ /* IPv6 --> GRE/NAT -> MAC/VLAN --> IPv4 */
+ IAVF_PTT(140, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, FRG, NONE, PAY3),
+ IAVF_PTT(141, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, NONE, PAY3),
+ IAVF_PTT(142, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(143),
+ IAVF_PTT(144, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, TCP, PAY4),
+ IAVF_PTT(145, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, SCTP, PAY4),
+ IAVF_PTT(146, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV4, NOF, ICMP, PAY4),
+
+ /* IPv6 --> GRE/NAT -> MAC/VLAN --> IPv6 */
+ IAVF_PTT(147, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, FRG, NONE, PAY3),
+ IAVF_PTT(148, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, NONE, PAY3),
+ IAVF_PTT(149, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, UDP, PAY4),
+ IAVF_PTT_UNUSED_ENTRY(150),
+ IAVF_PTT(151, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, TCP, PAY4),
+ IAVF_PTT(152, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, SCTP, PAY4),
+ IAVF_PTT(153, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, ICMP, PAY4),
+
+ /* unused entries */
+ [154 ... 255] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+};
+
+/**
+ * iavf_aq_send_msg_to_pf
+ * @hw: pointer to the hardware structure
+ * @v_opcode: opcodes for VF-PF communication
+ * @v_retval: return error code
+ * @msg: pointer to the msg buffer
+ * @msglen: msg length
+ * @cmd_details: pointer to command details
+ *
+ * Send message to PF driver using admin queue. By default, this message
+ * is sent asynchronously, i.e. iavf_asq_send_command() does not wait for
+ * completion before returning.
+ **/
+enum iavf_status iavf_aq_send_msg_to_pf(struct iavf_hw *hw,
+ enum virtchnl_ops v_opcode,
+ enum iavf_status v_retval,
+ u8 *msg, u16 msglen,
+ struct iavf_asq_cmd_details *cmd_details)
+{
+ struct iavf_asq_cmd_details details;
+ struct iavf_aq_desc desc;
+ enum iavf_status status;
+
+ iavf_fill_default_direct_cmd_desc(&desc, iavf_aqc_opc_send_msg_to_pf);
+ desc.flags |= cpu_to_le16((u16)IAVF_AQ_FLAG_SI);
+ desc.cookie_high = cpu_to_le32(v_opcode);
+ desc.cookie_low = cpu_to_le32(v_retval);
+ if (msglen) {
+ desc.flags |= cpu_to_le16((u16)(IAVF_AQ_FLAG_BUF
+ | IAVF_AQ_FLAG_RD));
+ if (msglen > IAVF_AQ_LARGE_BUF)
+ desc.flags |= cpu_to_le16((u16)IAVF_AQ_FLAG_LB);
+ desc.datalen = cpu_to_le16(msglen);
+ }
+ if (!cmd_details) {
+ memset(&details, 0, sizeof(details));
+ details.async = true;
+ cmd_details = &details;
+ }
+ status = iavf_asq_send_command(hw, &desc, msg, msglen, cmd_details);
+ return status;
+}
+
+/**
+ * iavf_vf_parse_hw_config
+ * @hw: pointer to the hardware structure
+ * @msg: pointer to the virtual channel VF resource structure
+ *
+ * Given a VF resource message from the PF, populate the hw struct
+ * with appropriate information.
+ **/
+void iavf_vf_parse_hw_config(struct iavf_hw *hw,
+ struct virtchnl_vf_resource *msg)
+{
+ struct virtchnl_vsi_resource *vsi_res;
+ int i;
+
+ vsi_res = &msg->vsi_res[0];
+
+ hw->dev_caps.num_vsis = msg->num_vsis;
+ hw->dev_caps.num_rx_qp = msg->num_queue_pairs;
+ hw->dev_caps.num_tx_qp = msg->num_queue_pairs;
+ hw->dev_caps.num_msix_vectors_vf = msg->max_vectors;
+ hw->dev_caps.dcb = msg->vf_cap_flags &
+ VIRTCHNL_VF_OFFLOAD_L2;
+ hw->dev_caps.fcoe = 0;
+ for (i = 0; i < msg->num_vsis; i++) {
+ if (vsi_res->vsi_type == VIRTCHNL_VSI_SRIOV) {
+ ether_addr_copy(hw->mac.perm_addr,
+ vsi_res->default_mac_addr);
+ ether_addr_copy(hw->mac.addr,
+ vsi_res->default_mac_addr);
+ }
+ vsi_res++;
+ }
+}
diff --git a/drivers/net/ethernet/intel/iavf/iavf_devids.h b/drivers/net/ethernet/intel/iavf/iavf_devids.h
new file mode 100644
index 0000000000..8eb7b697e9
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_devids.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_DEVIDS_H_
+#define _IAVF_DEVIDS_H_
+
+/* Device IDs for the VF driver */
+#define IAVF_DEV_ID_VF 0x154C
+#define IAVF_DEV_ID_VF_HV 0x1571
+#define IAVF_DEV_ID_ADAPTIVE_VF 0x1889
+#define IAVF_DEV_ID_X722_VF 0x37CD
+#endif /* _IAVF_DEVIDS_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c
new file mode 100644
index 0000000000..892c6a4f03
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c
@@ -0,0 +1,2012 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+/* ethtool support for iavf */
+#include "iavf.h"
+
+#include <linux/uaccess.h>
+
+/* ethtool statistics helpers */
+
+/**
+ * struct iavf_stats - definition for an ethtool statistic
+ * @stat_string: statistic name to display in ethtool -S output
+ * @sizeof_stat: the sizeof() the stat, must be no greater than sizeof(u64)
+ * @stat_offset: offsetof() the stat from a base pointer
+ *
+ * This structure defines a statistic to be added to the ethtool stats buffer.
+ * It defines a statistic as offset from a common base pointer. Stats should
+ * be defined in constant arrays using the IAVF_STAT macro, with every element
+ * of the array using the same _type for calculating the sizeof_stat and
+ * stat_offset.
+ *
+ * The @sizeof_stat is expected to be sizeof(u8), sizeof(u16), sizeof(u32) or
+ * sizeof(u64). Other sizes are not expected and will produce a WARN_ONCE from
+ * the iavf_add_ethtool_stat() helper function.
+ *
+ * The @stat_string is interpreted as a format string, allowing formatted
+ * values to be inserted while looping over multiple structures for a given
+ * statistics array. Thus, every statistic string in an array should have the
+ * same type and number of format specifiers, to be formatted by variadic
+ * arguments to the iavf_add_stat_string() helper function.
+ **/
+struct iavf_stats {
+ char stat_string[ETH_GSTRING_LEN];
+ int sizeof_stat;
+ int stat_offset;
+};
+
+/* Helper macro to define an iavf_stat structure with proper size and type.
+ * Use this when defining constant statistics arrays. Note that @_type expects
+ * only a type name and is used multiple times.
+ */
+#define IAVF_STAT(_type, _name, _stat) { \
+ .stat_string = _name, \
+ .sizeof_stat = sizeof_field(_type, _stat), \
+ .stat_offset = offsetof(_type, _stat) \
+}
+
+/* Helper macro for defining some statistics related to queues */
+#define IAVF_QUEUE_STAT(_name, _stat) \
+ IAVF_STAT(struct iavf_ring, _name, _stat)
+
+/* Stats associated with a Tx or Rx ring */
+static const struct iavf_stats iavf_gstrings_queue_stats[] = {
+ IAVF_QUEUE_STAT("%s-%u.packets", stats.packets),
+ IAVF_QUEUE_STAT("%s-%u.bytes", stats.bytes),
+};
+
+/**
+ * iavf_add_one_ethtool_stat - copy the stat into the supplied buffer
+ * @data: location to store the stat value
+ * @pointer: basis for where to copy from
+ * @stat: the stat definition
+ *
+ * Copies the stat data defined by the pointer and stat structure pair into
+ * the memory supplied as data. Used to implement iavf_add_ethtool_stats and
+ * iavf_add_queue_stats. If the pointer is null, data will be zero'd.
+ */
+static void
+iavf_add_one_ethtool_stat(u64 *data, void *pointer,
+ const struct iavf_stats *stat)
+{
+ char *p;
+
+ if (!pointer) {
+ /* ensure that the ethtool data buffer is zero'd for any stats
+ * which don't have a valid pointer.
+ */
+ *data = 0;
+ return;
+ }
+
+ p = (char *)pointer + stat->stat_offset;
+ switch (stat->sizeof_stat) {
+ case sizeof(u64):
+ *data = *((u64 *)p);
+ break;
+ case sizeof(u32):
+ *data = *((u32 *)p);
+ break;
+ case sizeof(u16):
+ *data = *((u16 *)p);
+ break;
+ case sizeof(u8):
+ *data = *((u8 *)p);
+ break;
+ default:
+ WARN_ONCE(1, "unexpected stat size for %s",
+ stat->stat_string);
+ *data = 0;
+ }
+}
+
+/**
+ * __iavf_add_ethtool_stats - copy stats into the ethtool supplied buffer
+ * @data: ethtool stats buffer
+ * @pointer: location to copy stats from
+ * @stats: array of stats to copy
+ * @size: the size of the stats definition
+ *
+ * Copy the stats defined by the stats array using the pointer as a base into
+ * the data buffer supplied by ethtool. Updates the data pointer to point to
+ * the next empty location for successive calls to __iavf_add_ethtool_stats.
+ * If pointer is null, set the data values to zero and update the pointer to
+ * skip these stats.
+ **/
+static void
+__iavf_add_ethtool_stats(u64 **data, void *pointer,
+ const struct iavf_stats stats[],
+ const unsigned int size)
+{
+ unsigned int i;
+
+ for (i = 0; i < size; i++)
+ iavf_add_one_ethtool_stat((*data)++, pointer, &stats[i]);
+}
+
+/**
+ * iavf_add_ethtool_stats - copy stats into ethtool supplied buffer
+ * @data: ethtool stats buffer
+ * @pointer: location where stats are stored
+ * @stats: static const array of stat definitions
+ *
+ * Macro to ease the use of __iavf_add_ethtool_stats by taking a static
+ * constant stats array and passing the ARRAY_SIZE(). This avoids typos by
+ * ensuring that we pass the size associated with the given stats array.
+ *
+ * The parameter @stats is evaluated twice, so parameters with side effects
+ * should be avoided.
+ **/
+#define iavf_add_ethtool_stats(data, pointer, stats) \
+ __iavf_add_ethtool_stats(data, pointer, stats, ARRAY_SIZE(stats))
+
+/**
+ * iavf_add_queue_stats - copy queue statistics into supplied buffer
+ * @data: ethtool stats buffer
+ * @ring: the ring to copy
+ *
+ * Queue statistics must be copied while protected by
+ * u64_stats_fetch_begin, so we can't directly use iavf_add_ethtool_stats.
+ * Assumes that queue stats are defined in iavf_gstrings_queue_stats. If the
+ * ring pointer is null, zero out the queue stat values and update the data
+ * pointer. Otherwise safely copy the stats from the ring into the supplied
+ * buffer and update the data pointer when finished.
+ *
+ * This function expects to be called while under rcu_read_lock().
+ **/
+static void
+iavf_add_queue_stats(u64 **data, struct iavf_ring *ring)
+{
+ const unsigned int size = ARRAY_SIZE(iavf_gstrings_queue_stats);
+ const struct iavf_stats *stats = iavf_gstrings_queue_stats;
+ unsigned int start;
+ unsigned int i;
+
+ /* To avoid invalid statistics values, ensure that we keep retrying
+ * the copy until we get a consistent value according to
+ * u64_stats_fetch_retry. But first, make sure our ring is
+ * non-null before attempting to access its syncp.
+ */
+ do {
+ start = !ring ? 0 : u64_stats_fetch_begin(&ring->syncp);
+ for (i = 0; i < size; i++)
+ iavf_add_one_ethtool_stat(&(*data)[i], ring, &stats[i]);
+ } while (ring && u64_stats_fetch_retry(&ring->syncp, start));
+
+ /* Once we successfully copy the stats in, update the data pointer */
+ *data += size;
+}
+
+/**
+ * __iavf_add_stat_strings - copy stat strings into ethtool buffer
+ * @p: ethtool supplied buffer
+ * @stats: stat definitions array
+ * @size: size of the stats array
+ *
+ * Format and copy the strings described by stats into the buffer pointed at
+ * by p.
+ **/
+static void __iavf_add_stat_strings(u8 **p, const struct iavf_stats stats[],
+ const unsigned int size, ...)
+{
+ unsigned int i;
+
+ for (i = 0; i < size; i++) {
+ va_list args;
+
+ va_start(args, size);
+ vsnprintf(*p, ETH_GSTRING_LEN, stats[i].stat_string, args);
+ *p += ETH_GSTRING_LEN;
+ va_end(args);
+ }
+}
+
+/**
+ * iavf_add_stat_strings - copy stat strings into ethtool buffer
+ * @p: ethtool supplied buffer
+ * @stats: stat definitions array
+ *
+ * Format and copy the strings described by the const static stats value into
+ * the buffer pointed at by p.
+ *
+ * The parameter @stats is evaluated twice, so parameters with side effects
+ * should be avoided. Additionally, stats must be an array such that
+ * ARRAY_SIZE can be called on it.
+ **/
+#define iavf_add_stat_strings(p, stats, ...) \
+ __iavf_add_stat_strings(p, stats, ARRAY_SIZE(stats), ## __VA_ARGS__)
+
+#define VF_STAT(_name, _stat) \
+ IAVF_STAT(struct iavf_adapter, _name, _stat)
+
+static const struct iavf_stats iavf_gstrings_stats[] = {
+ VF_STAT("rx_bytes", current_stats.rx_bytes),
+ VF_STAT("rx_unicast", current_stats.rx_unicast),
+ VF_STAT("rx_multicast", current_stats.rx_multicast),
+ VF_STAT("rx_broadcast", current_stats.rx_broadcast),
+ VF_STAT("rx_discards", current_stats.rx_discards),
+ VF_STAT("rx_unknown_protocol", current_stats.rx_unknown_protocol),
+ VF_STAT("tx_bytes", current_stats.tx_bytes),
+ VF_STAT("tx_unicast", current_stats.tx_unicast),
+ VF_STAT("tx_multicast", current_stats.tx_multicast),
+ VF_STAT("tx_broadcast", current_stats.tx_broadcast),
+ VF_STAT("tx_discards", current_stats.tx_discards),
+ VF_STAT("tx_errors", current_stats.tx_errors),
+};
+
+#define IAVF_STATS_LEN ARRAY_SIZE(iavf_gstrings_stats)
+
+#define IAVF_QUEUE_STATS_LEN ARRAY_SIZE(iavf_gstrings_queue_stats)
+
+/* For now we have one and only one private flag and it is only defined
+ * when we have support for the SKIP_CPU_SYNC DMA attribute. Instead
+ * of leaving all this code sitting around empty we will strip it unless
+ * our one private flag is actually available.
+ */
+struct iavf_priv_flags {
+ char flag_string[ETH_GSTRING_LEN];
+ u32 flag;
+ bool read_only;
+};
+
+#define IAVF_PRIV_FLAG(_name, _flag, _read_only) { \
+ .flag_string = _name, \
+ .flag = _flag, \
+ .read_only = _read_only, \
+}
+
+static const struct iavf_priv_flags iavf_gstrings_priv_flags[] = {
+ IAVF_PRIV_FLAG("legacy-rx", IAVF_FLAG_LEGACY_RX, 0),
+};
+
+#define IAVF_PRIV_FLAGS_STR_LEN ARRAY_SIZE(iavf_gstrings_priv_flags)
+
+/**
+ * iavf_get_link_ksettings - Get Link Speed and Duplex settings
+ * @netdev: network interface device structure
+ * @cmd: ethtool command
+ *
+ * Reports speed/duplex settings. Because this is a VF, we don't know what
+ * kind of link we really have, so we fake it.
+ **/
+static int iavf_get_link_ksettings(struct net_device *netdev,
+ struct ethtool_link_ksettings *cmd)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ ethtool_link_ksettings_zero_link_mode(cmd, supported);
+ cmd->base.autoneg = AUTONEG_DISABLE;
+ cmd->base.port = PORT_NONE;
+ cmd->base.duplex = DUPLEX_FULL;
+
+ if (ADV_LINK_SUPPORT(adapter)) {
+ if (adapter->link_speed_mbps &&
+ adapter->link_speed_mbps < U32_MAX)
+ cmd->base.speed = adapter->link_speed_mbps;
+ else
+ cmd->base.speed = SPEED_UNKNOWN;
+
+ return 0;
+ }
+
+ switch (adapter->link_speed) {
+ case VIRTCHNL_LINK_SPEED_40GB:
+ cmd->base.speed = SPEED_40000;
+ break;
+ case VIRTCHNL_LINK_SPEED_25GB:
+ cmd->base.speed = SPEED_25000;
+ break;
+ case VIRTCHNL_LINK_SPEED_20GB:
+ cmd->base.speed = SPEED_20000;
+ break;
+ case VIRTCHNL_LINK_SPEED_10GB:
+ cmd->base.speed = SPEED_10000;
+ break;
+ case VIRTCHNL_LINK_SPEED_5GB:
+ cmd->base.speed = SPEED_5000;
+ break;
+ case VIRTCHNL_LINK_SPEED_2_5GB:
+ cmd->base.speed = SPEED_2500;
+ break;
+ case VIRTCHNL_LINK_SPEED_1GB:
+ cmd->base.speed = SPEED_1000;
+ break;
+ case VIRTCHNL_LINK_SPEED_100MB:
+ cmd->base.speed = SPEED_100;
+ break;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_get_sset_count - Get length of string set
+ * @netdev: network interface device structure
+ * @sset: id of string set
+ *
+ * Reports size of various string tables.
+ **/
+static int iavf_get_sset_count(struct net_device *netdev, int sset)
+{
+ /* Report the maximum number queues, even if not every queue is
+ * currently configured. Since allocation of queues is in pairs,
+ * use netdev->real_num_tx_queues * 2. The real_num_tx_queues is set
+ * at device creation and never changes.
+ */
+
+ if (sset == ETH_SS_STATS)
+ return IAVF_STATS_LEN +
+ (IAVF_QUEUE_STATS_LEN * 2 *
+ netdev->real_num_tx_queues);
+ else if (sset == ETH_SS_PRIV_FLAGS)
+ return IAVF_PRIV_FLAGS_STR_LEN;
+ else
+ return -EINVAL;
+}
+
+/**
+ * iavf_get_ethtool_stats - report device statistics
+ * @netdev: network interface device structure
+ * @stats: ethtool statistics structure
+ * @data: pointer to data buffer
+ *
+ * All statistics are added to the data buffer as an array of u64.
+ **/
+static void iavf_get_ethtool_stats(struct net_device *netdev,
+ struct ethtool_stats *stats, u64 *data)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ unsigned int i;
+
+ /* Explicitly request stats refresh */
+ iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_REQUEST_STATS);
+
+ iavf_add_ethtool_stats(&data, adapter, iavf_gstrings_stats);
+
+ rcu_read_lock();
+ /* As num_active_queues describe both tx and rx queues, we can use
+ * it to iterate over rings' stats.
+ */
+ for (i = 0; i < adapter->num_active_queues; i++) {
+ struct iavf_ring *ring;
+
+ /* Tx rings stats */
+ ring = &adapter->tx_rings[i];
+ iavf_add_queue_stats(&data, ring);
+
+ /* Rx rings stats */
+ ring = &adapter->rx_rings[i];
+ iavf_add_queue_stats(&data, ring);
+ }
+ rcu_read_unlock();
+}
+
+/**
+ * iavf_get_priv_flag_strings - Get private flag strings
+ * @netdev: network interface device structure
+ * @data: buffer for string data
+ *
+ * Builds the private flags string table
+ **/
+static void iavf_get_priv_flag_strings(struct net_device *netdev, u8 *data)
+{
+ unsigned int i;
+
+ for (i = 0; i < IAVF_PRIV_FLAGS_STR_LEN; i++) {
+ snprintf(data, ETH_GSTRING_LEN, "%s",
+ iavf_gstrings_priv_flags[i].flag_string);
+ data += ETH_GSTRING_LEN;
+ }
+}
+
+/**
+ * iavf_get_stat_strings - Get stat strings
+ * @netdev: network interface device structure
+ * @data: buffer for string data
+ *
+ * Builds the statistics string table
+ **/
+static void iavf_get_stat_strings(struct net_device *netdev, u8 *data)
+{
+ unsigned int i;
+
+ iavf_add_stat_strings(&data, iavf_gstrings_stats);
+
+ /* Queues are always allocated in pairs, so we just use
+ * real_num_tx_queues for both Tx and Rx queues.
+ */
+ for (i = 0; i < netdev->real_num_tx_queues; i++) {
+ iavf_add_stat_strings(&data, iavf_gstrings_queue_stats,
+ "tx", i);
+ iavf_add_stat_strings(&data, iavf_gstrings_queue_stats,
+ "rx", i);
+ }
+}
+
+/**
+ * iavf_get_strings - Get string set
+ * @netdev: network interface device structure
+ * @sset: id of string set
+ * @data: buffer for string data
+ *
+ * Builds string tables for various string sets
+ **/
+static void iavf_get_strings(struct net_device *netdev, u32 sset, u8 *data)
+{
+ switch (sset) {
+ case ETH_SS_STATS:
+ iavf_get_stat_strings(netdev, data);
+ break;
+ case ETH_SS_PRIV_FLAGS:
+ iavf_get_priv_flag_strings(netdev, data);
+ break;
+ default:
+ break;
+ }
+}
+
+/**
+ * iavf_get_priv_flags - report device private flags
+ * @netdev: network interface device structure
+ *
+ * The get string set count and the string set should be matched for each
+ * flag returned. Add new strings for each flag to the iavf_gstrings_priv_flags
+ * array.
+ *
+ * Returns a u32 bitmap of flags.
+ **/
+static u32 iavf_get_priv_flags(struct net_device *netdev)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ u32 i, ret_flags = 0;
+
+ for (i = 0; i < IAVF_PRIV_FLAGS_STR_LEN; i++) {
+ const struct iavf_priv_flags *priv_flags;
+
+ priv_flags = &iavf_gstrings_priv_flags[i];
+
+ if (priv_flags->flag & adapter->flags)
+ ret_flags |= BIT(i);
+ }
+
+ return ret_flags;
+}
+
+/**
+ * iavf_set_priv_flags - set private flags
+ * @netdev: network interface device structure
+ * @flags: bit flags to be set
+ **/
+static int iavf_set_priv_flags(struct net_device *netdev, u32 flags)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ u32 orig_flags, new_flags, changed_flags;
+ int ret = 0;
+ u32 i;
+
+ orig_flags = READ_ONCE(adapter->flags);
+ new_flags = orig_flags;
+
+ for (i = 0; i < IAVF_PRIV_FLAGS_STR_LEN; i++) {
+ const struct iavf_priv_flags *priv_flags;
+
+ priv_flags = &iavf_gstrings_priv_flags[i];
+
+ if (flags & BIT(i))
+ new_flags |= priv_flags->flag;
+ else
+ new_flags &= ~(priv_flags->flag);
+
+ if (priv_flags->read_only &&
+ ((orig_flags ^ new_flags) & ~BIT(i)))
+ return -EOPNOTSUPP;
+ }
+
+ /* Before we finalize any flag changes, any checks which we need to
+ * perform to determine if the new flags will be supported should go
+ * here...
+ */
+
+ /* Compare and exchange the new flags into place. If we failed, that
+ * is if cmpxchg returns anything but the old value, this means
+ * something else must have modified the flags variable since we
+ * copied it. We'll just punt with an error and log something in the
+ * message buffer.
+ */
+ if (cmpxchg(&adapter->flags, orig_flags, new_flags) != orig_flags) {
+ dev_warn(&adapter->pdev->dev,
+ "Unable to update adapter->flags as it was modified by another thread...\n");
+ return -EAGAIN;
+ }
+
+ changed_flags = orig_flags ^ new_flags;
+
+ /* Process any additional changes needed as a result of flag changes.
+ * The changed_flags value reflects the list of bits that were changed
+ * in the code above.
+ */
+
+ /* issue a reset to force legacy-rx change to take effect */
+ if (changed_flags & IAVF_FLAG_LEGACY_RX) {
+ if (netif_running(netdev)) {
+ iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
+ ret = iavf_wait_for_reset(adapter);
+ if (ret)
+ netdev_warn(netdev, "Changing private flags timeout or interrupted waiting for reset");
+ }
+ }
+
+ return ret;
+}
+
+/**
+ * iavf_get_msglevel - Get debug message level
+ * @netdev: network interface device structure
+ *
+ * Returns current debug message level.
+ **/
+static u32 iavf_get_msglevel(struct net_device *netdev)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ return adapter->msg_enable;
+}
+
+/**
+ * iavf_set_msglevel - Set debug message level
+ * @netdev: network interface device structure
+ * @data: message level
+ *
+ * Set current debug message level. Higher values cause the driver to
+ * be noisier.
+ **/
+static void iavf_set_msglevel(struct net_device *netdev, u32 data)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ if (IAVF_DEBUG_USER & data)
+ adapter->hw.debug_mask = data;
+ adapter->msg_enable = data;
+}
+
+/**
+ * iavf_get_drvinfo - Get driver info
+ * @netdev: network interface device structure
+ * @drvinfo: ethool driver info structure
+ *
+ * Returns information about the driver and device for display to the user.
+ **/
+static void iavf_get_drvinfo(struct net_device *netdev,
+ struct ethtool_drvinfo *drvinfo)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ strscpy(drvinfo->driver, iavf_driver_name, 32);
+ strscpy(drvinfo->fw_version, "N/A", 4);
+ strscpy(drvinfo->bus_info, pci_name(adapter->pdev), 32);
+ drvinfo->n_priv_flags = IAVF_PRIV_FLAGS_STR_LEN;
+}
+
+/**
+ * iavf_get_ringparam - Get ring parameters
+ * @netdev: network interface device structure
+ * @ring: ethtool ringparam structure
+ * @kernel_ring: ethtool extenal ringparam structure
+ * @extack: netlink extended ACK report struct
+ *
+ * Returns current ring parameters. TX and RX rings are reported separately,
+ * but the number of rings is not reported.
+ **/
+static void iavf_get_ringparam(struct net_device *netdev,
+ struct ethtool_ringparam *ring,
+ struct kernel_ethtool_ringparam *kernel_ring,
+ struct netlink_ext_ack *extack)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ ring->rx_max_pending = IAVF_MAX_RXD;
+ ring->tx_max_pending = IAVF_MAX_TXD;
+ ring->rx_pending = adapter->rx_desc_count;
+ ring->tx_pending = adapter->tx_desc_count;
+}
+
+/**
+ * iavf_set_ringparam - Set ring parameters
+ * @netdev: network interface device structure
+ * @ring: ethtool ringparam structure
+ * @kernel_ring: ethtool external ringparam structure
+ * @extack: netlink extended ACK report struct
+ *
+ * Sets ring parameters. TX and RX rings are controlled separately, but the
+ * number of rings is not specified, so all rings get the same settings.
+ **/
+static int iavf_set_ringparam(struct net_device *netdev,
+ struct ethtool_ringparam *ring,
+ struct kernel_ethtool_ringparam *kernel_ring,
+ struct netlink_ext_ack *extack)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ u32 new_rx_count, new_tx_count;
+ int ret = 0;
+
+ if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))
+ return -EINVAL;
+
+ if (ring->tx_pending > IAVF_MAX_TXD ||
+ ring->tx_pending < IAVF_MIN_TXD ||
+ ring->rx_pending > IAVF_MAX_RXD ||
+ ring->rx_pending < IAVF_MIN_RXD) {
+ netdev_err(netdev, "Descriptors requested (Tx: %d / Rx: %d) out of range [%d-%d] (increment %d)\n",
+ ring->tx_pending, ring->rx_pending, IAVF_MIN_TXD,
+ IAVF_MAX_RXD, IAVF_REQ_DESCRIPTOR_MULTIPLE);
+ return -EINVAL;
+ }
+
+ new_tx_count = ALIGN(ring->tx_pending, IAVF_REQ_DESCRIPTOR_MULTIPLE);
+ if (new_tx_count != ring->tx_pending)
+ netdev_info(netdev, "Requested Tx descriptor count rounded up to %d\n",
+ new_tx_count);
+
+ new_rx_count = ALIGN(ring->rx_pending, IAVF_REQ_DESCRIPTOR_MULTIPLE);
+ if (new_rx_count != ring->rx_pending)
+ netdev_info(netdev, "Requested Rx descriptor count rounded up to %d\n",
+ new_rx_count);
+
+ /* if nothing to do return success */
+ if ((new_tx_count == adapter->tx_desc_count) &&
+ (new_rx_count == adapter->rx_desc_count)) {
+ netdev_dbg(netdev, "Nothing to change, descriptor count is same as requested\n");
+ return 0;
+ }
+
+ if (new_tx_count != adapter->tx_desc_count) {
+ netdev_dbg(netdev, "Changing Tx descriptor count from %d to %d\n",
+ adapter->tx_desc_count, new_tx_count);
+ adapter->tx_desc_count = new_tx_count;
+ }
+
+ if (new_rx_count != adapter->rx_desc_count) {
+ netdev_dbg(netdev, "Changing Rx descriptor count from %d to %d\n",
+ adapter->rx_desc_count, new_rx_count);
+ adapter->rx_desc_count = new_rx_count;
+ }
+
+ if (netif_running(netdev)) {
+ iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
+ ret = iavf_wait_for_reset(adapter);
+ if (ret)
+ netdev_warn(netdev, "Changing ring parameters timeout or interrupted waiting for reset");
+ }
+
+ return ret;
+}
+
+/**
+ * __iavf_get_coalesce - get per-queue coalesce settings
+ * @netdev: the netdev to check
+ * @ec: ethtool coalesce data structure
+ * @queue: which queue to pick
+ *
+ * Gets the per-queue settings for coalescence. Specifically Rx and Tx usecs
+ * are per queue. If queue is <0 then we default to queue 0 as the
+ * representative value.
+ **/
+static int __iavf_get_coalesce(struct net_device *netdev,
+ struct ethtool_coalesce *ec, int queue)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ struct iavf_ring *rx_ring, *tx_ring;
+
+ /* Rx and Tx usecs per queue value. If user doesn't specify the
+ * queue, return queue 0's value to represent.
+ */
+ if (queue < 0)
+ queue = 0;
+ else if (queue >= adapter->num_active_queues)
+ return -EINVAL;
+
+ rx_ring = &adapter->rx_rings[queue];
+ tx_ring = &adapter->tx_rings[queue];
+
+ if (ITR_IS_DYNAMIC(rx_ring->itr_setting))
+ ec->use_adaptive_rx_coalesce = 1;
+
+ if (ITR_IS_DYNAMIC(tx_ring->itr_setting))
+ ec->use_adaptive_tx_coalesce = 1;
+
+ ec->rx_coalesce_usecs = rx_ring->itr_setting & ~IAVF_ITR_DYNAMIC;
+ ec->tx_coalesce_usecs = tx_ring->itr_setting & ~IAVF_ITR_DYNAMIC;
+
+ return 0;
+}
+
+/**
+ * iavf_get_coalesce - Get interrupt coalescing settings
+ * @netdev: network interface device structure
+ * @ec: ethtool coalesce structure
+ * @kernel_coal: ethtool CQE mode setting structure
+ * @extack: extack for reporting error messages
+ *
+ * Returns current coalescing settings. This is referred to elsewhere in the
+ * driver as Interrupt Throttle Rate, as this is how the hardware describes
+ * this functionality. Note that if per-queue settings have been modified this
+ * only represents the settings of queue 0.
+ **/
+static int iavf_get_coalesce(struct net_device *netdev,
+ struct ethtool_coalesce *ec,
+ struct kernel_ethtool_coalesce *kernel_coal,
+ struct netlink_ext_ack *extack)
+{
+ return __iavf_get_coalesce(netdev, ec, -1);
+}
+
+/**
+ * iavf_get_per_queue_coalesce - get coalesce values for specific queue
+ * @netdev: netdev to read
+ * @ec: coalesce settings from ethtool
+ * @queue: the queue to read
+ *
+ * Read specific queue's coalesce settings.
+ **/
+static int iavf_get_per_queue_coalesce(struct net_device *netdev, u32 queue,
+ struct ethtool_coalesce *ec)
+{
+ return __iavf_get_coalesce(netdev, ec, queue);
+}
+
+/**
+ * iavf_set_itr_per_queue - set ITR values for specific queue
+ * @adapter: the VF adapter struct to set values for
+ * @ec: coalesce settings from ethtool
+ * @queue: the queue to modify
+ *
+ * Change the ITR settings for a specific queue.
+ **/
+static int iavf_set_itr_per_queue(struct iavf_adapter *adapter,
+ struct ethtool_coalesce *ec, int queue)
+{
+ struct iavf_ring *rx_ring = &adapter->rx_rings[queue];
+ struct iavf_ring *tx_ring = &adapter->tx_rings[queue];
+ struct iavf_q_vector *q_vector;
+ u16 itr_setting;
+
+ itr_setting = rx_ring->itr_setting & ~IAVF_ITR_DYNAMIC;
+
+ if (ec->rx_coalesce_usecs != itr_setting &&
+ ec->use_adaptive_rx_coalesce) {
+ netif_info(adapter, drv, adapter->netdev,
+ "Rx interrupt throttling cannot be changed if adaptive-rx is enabled\n");
+ return -EINVAL;
+ }
+
+ itr_setting = tx_ring->itr_setting & ~IAVF_ITR_DYNAMIC;
+
+ if (ec->tx_coalesce_usecs != itr_setting &&
+ ec->use_adaptive_tx_coalesce) {
+ netif_info(adapter, drv, adapter->netdev,
+ "Tx interrupt throttling cannot be changed if adaptive-tx is enabled\n");
+ return -EINVAL;
+ }
+
+ rx_ring->itr_setting = ITR_REG_ALIGN(ec->rx_coalesce_usecs);
+ tx_ring->itr_setting = ITR_REG_ALIGN(ec->tx_coalesce_usecs);
+
+ rx_ring->itr_setting |= IAVF_ITR_DYNAMIC;
+ if (!ec->use_adaptive_rx_coalesce)
+ rx_ring->itr_setting ^= IAVF_ITR_DYNAMIC;
+
+ tx_ring->itr_setting |= IAVF_ITR_DYNAMIC;
+ if (!ec->use_adaptive_tx_coalesce)
+ tx_ring->itr_setting ^= IAVF_ITR_DYNAMIC;
+
+ q_vector = rx_ring->q_vector;
+ q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
+
+ q_vector = tx_ring->q_vector;
+ q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
+
+ /* The interrupt handler itself will take care of programming
+ * the Tx and Rx ITR values based on the values we have entered
+ * into the q_vector, no need to write the values now.
+ */
+ return 0;
+}
+
+/**
+ * __iavf_set_coalesce - set coalesce settings for particular queue
+ * @netdev: the netdev to change
+ * @ec: ethtool coalesce settings
+ * @queue: the queue to change
+ *
+ * Sets the coalesce settings for a particular queue.
+ **/
+static int __iavf_set_coalesce(struct net_device *netdev,
+ struct ethtool_coalesce *ec, int queue)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ int i;
+
+ if (ec->rx_coalesce_usecs > IAVF_MAX_ITR) {
+ netif_info(adapter, drv, netdev, "Invalid value, rx-usecs range is 0-8160\n");
+ return -EINVAL;
+ } else if (ec->tx_coalesce_usecs > IAVF_MAX_ITR) {
+ netif_info(adapter, drv, netdev, "Invalid value, tx-usecs range is 0-8160\n");
+ return -EINVAL;
+ }
+
+ /* Rx and Tx usecs has per queue value. If user doesn't specify the
+ * queue, apply to all queues.
+ */
+ if (queue < 0) {
+ for (i = 0; i < adapter->num_active_queues; i++)
+ if (iavf_set_itr_per_queue(adapter, ec, i))
+ return -EINVAL;
+ } else if (queue < adapter->num_active_queues) {
+ if (iavf_set_itr_per_queue(adapter, ec, queue))
+ return -EINVAL;
+ } else {
+ netif_info(adapter, drv, netdev, "Invalid queue value, queue range is 0 - %d\n",
+ adapter->num_active_queues - 1);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_set_coalesce - Set interrupt coalescing settings
+ * @netdev: network interface device structure
+ * @ec: ethtool coalesce structure
+ * @kernel_coal: ethtool CQE mode setting structure
+ * @extack: extack for reporting error messages
+ *
+ * Change current coalescing settings for every queue.
+ **/
+static int iavf_set_coalesce(struct net_device *netdev,
+ struct ethtool_coalesce *ec,
+ struct kernel_ethtool_coalesce *kernel_coal,
+ struct netlink_ext_ack *extack)
+{
+ return __iavf_set_coalesce(netdev, ec, -1);
+}
+
+/**
+ * iavf_set_per_queue_coalesce - set specific queue's coalesce settings
+ * @netdev: the netdev to change
+ * @ec: ethtool's coalesce settings
+ * @queue: the queue to modify
+ *
+ * Modifies a specific queue's coalesce settings.
+ */
+static int iavf_set_per_queue_coalesce(struct net_device *netdev, u32 queue,
+ struct ethtool_coalesce *ec)
+{
+ return __iavf_set_coalesce(netdev, ec, queue);
+}
+
+/**
+ * iavf_fltr_to_ethtool_flow - convert filter type values to ethtool
+ * flow type values
+ * @flow: filter type to be converted
+ *
+ * Returns the corresponding ethtool flow type.
+ */
+static int iavf_fltr_to_ethtool_flow(enum iavf_fdir_flow_type flow)
+{
+ switch (flow) {
+ case IAVF_FDIR_FLOW_IPV4_TCP:
+ return TCP_V4_FLOW;
+ case IAVF_FDIR_FLOW_IPV4_UDP:
+ return UDP_V4_FLOW;
+ case IAVF_FDIR_FLOW_IPV4_SCTP:
+ return SCTP_V4_FLOW;
+ case IAVF_FDIR_FLOW_IPV4_AH:
+ return AH_V4_FLOW;
+ case IAVF_FDIR_FLOW_IPV4_ESP:
+ return ESP_V4_FLOW;
+ case IAVF_FDIR_FLOW_IPV4_OTHER:
+ return IPV4_USER_FLOW;
+ case IAVF_FDIR_FLOW_IPV6_TCP:
+ return TCP_V6_FLOW;
+ case IAVF_FDIR_FLOW_IPV6_UDP:
+ return UDP_V6_FLOW;
+ case IAVF_FDIR_FLOW_IPV6_SCTP:
+ return SCTP_V6_FLOW;
+ case IAVF_FDIR_FLOW_IPV6_AH:
+ return AH_V6_FLOW;
+ case IAVF_FDIR_FLOW_IPV6_ESP:
+ return ESP_V6_FLOW;
+ case IAVF_FDIR_FLOW_IPV6_OTHER:
+ return IPV6_USER_FLOW;
+ case IAVF_FDIR_FLOW_NON_IP_L2:
+ return ETHER_FLOW;
+ default:
+ /* 0 is undefined ethtool flow */
+ return 0;
+ }
+}
+
+/**
+ * iavf_ethtool_flow_to_fltr - convert ethtool flow type to filter enum
+ * @eth: Ethtool flow type to be converted
+ *
+ * Returns flow enum
+ */
+static enum iavf_fdir_flow_type iavf_ethtool_flow_to_fltr(int eth)
+{
+ switch (eth) {
+ case TCP_V4_FLOW:
+ return IAVF_FDIR_FLOW_IPV4_TCP;
+ case UDP_V4_FLOW:
+ return IAVF_FDIR_FLOW_IPV4_UDP;
+ case SCTP_V4_FLOW:
+ return IAVF_FDIR_FLOW_IPV4_SCTP;
+ case AH_V4_FLOW:
+ return IAVF_FDIR_FLOW_IPV4_AH;
+ case ESP_V4_FLOW:
+ return IAVF_FDIR_FLOW_IPV4_ESP;
+ case IPV4_USER_FLOW:
+ return IAVF_FDIR_FLOW_IPV4_OTHER;
+ case TCP_V6_FLOW:
+ return IAVF_FDIR_FLOW_IPV6_TCP;
+ case UDP_V6_FLOW:
+ return IAVF_FDIR_FLOW_IPV6_UDP;
+ case SCTP_V6_FLOW:
+ return IAVF_FDIR_FLOW_IPV6_SCTP;
+ case AH_V6_FLOW:
+ return IAVF_FDIR_FLOW_IPV6_AH;
+ case ESP_V6_FLOW:
+ return IAVF_FDIR_FLOW_IPV6_ESP;
+ case IPV6_USER_FLOW:
+ return IAVF_FDIR_FLOW_IPV6_OTHER;
+ case ETHER_FLOW:
+ return IAVF_FDIR_FLOW_NON_IP_L2;
+ default:
+ return IAVF_FDIR_FLOW_NONE;
+ }
+}
+
+/**
+ * iavf_is_mask_valid - check mask field set
+ * @mask: full mask to check
+ * @field: field for which mask should be valid
+ *
+ * If the mask is fully set return true. If it is not valid for field return
+ * false.
+ */
+static bool iavf_is_mask_valid(u64 mask, u64 field)
+{
+ return (mask & field) == field;
+}
+
+/**
+ * iavf_parse_rx_flow_user_data - deconstruct user-defined data
+ * @fsp: pointer to ethtool Rx flow specification
+ * @fltr: pointer to Flow Director filter for userdef data storage
+ *
+ * Returns 0 on success, negative error value on failure
+ */
+static int
+iavf_parse_rx_flow_user_data(struct ethtool_rx_flow_spec *fsp,
+ struct iavf_fdir_fltr *fltr)
+{
+ struct iavf_flex_word *flex;
+ int i, cnt = 0;
+
+ if (!(fsp->flow_type & FLOW_EXT))
+ return 0;
+
+ for (i = 0; i < IAVF_FLEX_WORD_NUM; i++) {
+#define IAVF_USERDEF_FLEX_WORD_M GENMASK(15, 0)
+#define IAVF_USERDEF_FLEX_OFFS_S 16
+#define IAVF_USERDEF_FLEX_OFFS_M GENMASK(31, IAVF_USERDEF_FLEX_OFFS_S)
+#define IAVF_USERDEF_FLEX_FLTR_M GENMASK(31, 0)
+ u32 value = be32_to_cpu(fsp->h_ext.data[i]);
+ u32 mask = be32_to_cpu(fsp->m_ext.data[i]);
+
+ if (!value || !mask)
+ continue;
+
+ if (!iavf_is_mask_valid(mask, IAVF_USERDEF_FLEX_FLTR_M))
+ return -EINVAL;
+
+ /* 504 is the maximum value for offsets, and offset is measured
+ * from the start of the MAC address.
+ */
+#define IAVF_USERDEF_FLEX_MAX_OFFS_VAL 504
+ flex = &fltr->flex_words[cnt++];
+ flex->word = value & IAVF_USERDEF_FLEX_WORD_M;
+ flex->offset = (value & IAVF_USERDEF_FLEX_OFFS_M) >>
+ IAVF_USERDEF_FLEX_OFFS_S;
+ if (flex->offset > IAVF_USERDEF_FLEX_MAX_OFFS_VAL)
+ return -EINVAL;
+ }
+
+ fltr->flex_cnt = cnt;
+
+ return 0;
+}
+
+/**
+ * iavf_fill_rx_flow_ext_data - fill the additional data
+ * @fsp: pointer to ethtool Rx flow specification
+ * @fltr: pointer to Flow Director filter to get additional data
+ */
+static void
+iavf_fill_rx_flow_ext_data(struct ethtool_rx_flow_spec *fsp,
+ struct iavf_fdir_fltr *fltr)
+{
+ if (!fltr->ext_mask.usr_def[0] && !fltr->ext_mask.usr_def[1])
+ return;
+
+ fsp->flow_type |= FLOW_EXT;
+
+ memcpy(fsp->h_ext.data, fltr->ext_data.usr_def, sizeof(fsp->h_ext.data));
+ memcpy(fsp->m_ext.data, fltr->ext_mask.usr_def, sizeof(fsp->m_ext.data));
+}
+
+/**
+ * iavf_get_ethtool_fdir_entry - fill ethtool structure with Flow Director filter data
+ * @adapter: the VF adapter structure that contains filter list
+ * @cmd: ethtool command data structure to receive the filter data
+ *
+ * Returns 0 as expected for success by ethtool
+ */
+static int
+iavf_get_ethtool_fdir_entry(struct iavf_adapter *adapter,
+ struct ethtool_rxnfc *cmd)
+{
+ struct ethtool_rx_flow_spec *fsp = (struct ethtool_rx_flow_spec *)&cmd->fs;
+ struct iavf_fdir_fltr *rule = NULL;
+ int ret = 0;
+
+ if (!(adapter->flags & IAVF_FLAG_FDIR_ENABLED))
+ return -EOPNOTSUPP;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+
+ rule = iavf_find_fdir_fltr_by_loc(adapter, fsp->location);
+ if (!rule) {
+ ret = -EINVAL;
+ goto release_lock;
+ }
+
+ fsp->flow_type = iavf_fltr_to_ethtool_flow(rule->flow_type);
+
+ memset(&fsp->m_u, 0, sizeof(fsp->m_u));
+ memset(&fsp->m_ext, 0, sizeof(fsp->m_ext));
+
+ switch (fsp->flow_type) {
+ case TCP_V4_FLOW:
+ case UDP_V4_FLOW:
+ case SCTP_V4_FLOW:
+ fsp->h_u.tcp_ip4_spec.ip4src = rule->ip_data.v4_addrs.src_ip;
+ fsp->h_u.tcp_ip4_spec.ip4dst = rule->ip_data.v4_addrs.dst_ip;
+ fsp->h_u.tcp_ip4_spec.psrc = rule->ip_data.src_port;
+ fsp->h_u.tcp_ip4_spec.pdst = rule->ip_data.dst_port;
+ fsp->h_u.tcp_ip4_spec.tos = rule->ip_data.tos;
+ fsp->m_u.tcp_ip4_spec.ip4src = rule->ip_mask.v4_addrs.src_ip;
+ fsp->m_u.tcp_ip4_spec.ip4dst = rule->ip_mask.v4_addrs.dst_ip;
+ fsp->m_u.tcp_ip4_spec.psrc = rule->ip_mask.src_port;
+ fsp->m_u.tcp_ip4_spec.pdst = rule->ip_mask.dst_port;
+ fsp->m_u.tcp_ip4_spec.tos = rule->ip_mask.tos;
+ break;
+ case AH_V4_FLOW:
+ case ESP_V4_FLOW:
+ fsp->h_u.ah_ip4_spec.ip4src = rule->ip_data.v4_addrs.src_ip;
+ fsp->h_u.ah_ip4_spec.ip4dst = rule->ip_data.v4_addrs.dst_ip;
+ fsp->h_u.ah_ip4_spec.spi = rule->ip_data.spi;
+ fsp->h_u.ah_ip4_spec.tos = rule->ip_data.tos;
+ fsp->m_u.ah_ip4_spec.ip4src = rule->ip_mask.v4_addrs.src_ip;
+ fsp->m_u.ah_ip4_spec.ip4dst = rule->ip_mask.v4_addrs.dst_ip;
+ fsp->m_u.ah_ip4_spec.spi = rule->ip_mask.spi;
+ fsp->m_u.ah_ip4_spec.tos = rule->ip_mask.tos;
+ break;
+ case IPV4_USER_FLOW:
+ fsp->h_u.usr_ip4_spec.ip4src = rule->ip_data.v4_addrs.src_ip;
+ fsp->h_u.usr_ip4_spec.ip4dst = rule->ip_data.v4_addrs.dst_ip;
+ fsp->h_u.usr_ip4_spec.l4_4_bytes = rule->ip_data.l4_header;
+ fsp->h_u.usr_ip4_spec.tos = rule->ip_data.tos;
+ fsp->h_u.usr_ip4_spec.ip_ver = ETH_RX_NFC_IP4;
+ fsp->h_u.usr_ip4_spec.proto = rule->ip_data.proto;
+ fsp->m_u.usr_ip4_spec.ip4src = rule->ip_mask.v4_addrs.src_ip;
+ fsp->m_u.usr_ip4_spec.ip4dst = rule->ip_mask.v4_addrs.dst_ip;
+ fsp->m_u.usr_ip4_spec.l4_4_bytes = rule->ip_mask.l4_header;
+ fsp->m_u.usr_ip4_spec.tos = rule->ip_mask.tos;
+ fsp->m_u.usr_ip4_spec.ip_ver = 0xFF;
+ fsp->m_u.usr_ip4_spec.proto = rule->ip_mask.proto;
+ break;
+ case TCP_V6_FLOW:
+ case UDP_V6_FLOW:
+ case SCTP_V6_FLOW:
+ memcpy(fsp->h_u.usr_ip6_spec.ip6src, &rule->ip_data.v6_addrs.src_ip,
+ sizeof(struct in6_addr));
+ memcpy(fsp->h_u.usr_ip6_spec.ip6dst, &rule->ip_data.v6_addrs.dst_ip,
+ sizeof(struct in6_addr));
+ fsp->h_u.tcp_ip6_spec.psrc = rule->ip_data.src_port;
+ fsp->h_u.tcp_ip6_spec.pdst = rule->ip_data.dst_port;
+ fsp->h_u.tcp_ip6_spec.tclass = rule->ip_data.tclass;
+ memcpy(fsp->m_u.usr_ip6_spec.ip6src, &rule->ip_mask.v6_addrs.src_ip,
+ sizeof(struct in6_addr));
+ memcpy(fsp->m_u.usr_ip6_spec.ip6dst, &rule->ip_mask.v6_addrs.dst_ip,
+ sizeof(struct in6_addr));
+ fsp->m_u.tcp_ip6_spec.psrc = rule->ip_mask.src_port;
+ fsp->m_u.tcp_ip6_spec.pdst = rule->ip_mask.dst_port;
+ fsp->m_u.tcp_ip6_spec.tclass = rule->ip_mask.tclass;
+ break;
+ case AH_V6_FLOW:
+ case ESP_V6_FLOW:
+ memcpy(fsp->h_u.ah_ip6_spec.ip6src, &rule->ip_data.v6_addrs.src_ip,
+ sizeof(struct in6_addr));
+ memcpy(fsp->h_u.ah_ip6_spec.ip6dst, &rule->ip_data.v6_addrs.dst_ip,
+ sizeof(struct in6_addr));
+ fsp->h_u.ah_ip6_spec.spi = rule->ip_data.spi;
+ fsp->h_u.ah_ip6_spec.tclass = rule->ip_data.tclass;
+ memcpy(fsp->m_u.ah_ip6_spec.ip6src, &rule->ip_mask.v6_addrs.src_ip,
+ sizeof(struct in6_addr));
+ memcpy(fsp->m_u.ah_ip6_spec.ip6dst, &rule->ip_mask.v6_addrs.dst_ip,
+ sizeof(struct in6_addr));
+ fsp->m_u.ah_ip6_spec.spi = rule->ip_mask.spi;
+ fsp->m_u.ah_ip6_spec.tclass = rule->ip_mask.tclass;
+ break;
+ case IPV6_USER_FLOW:
+ memcpy(fsp->h_u.usr_ip6_spec.ip6src, &rule->ip_data.v6_addrs.src_ip,
+ sizeof(struct in6_addr));
+ memcpy(fsp->h_u.usr_ip6_spec.ip6dst, &rule->ip_data.v6_addrs.dst_ip,
+ sizeof(struct in6_addr));
+ fsp->h_u.usr_ip6_spec.l4_4_bytes = rule->ip_data.l4_header;
+ fsp->h_u.usr_ip6_spec.tclass = rule->ip_data.tclass;
+ fsp->h_u.usr_ip6_spec.l4_proto = rule->ip_data.proto;
+ memcpy(fsp->m_u.usr_ip6_spec.ip6src, &rule->ip_mask.v6_addrs.src_ip,
+ sizeof(struct in6_addr));
+ memcpy(fsp->m_u.usr_ip6_spec.ip6dst, &rule->ip_mask.v6_addrs.dst_ip,
+ sizeof(struct in6_addr));
+ fsp->m_u.usr_ip6_spec.l4_4_bytes = rule->ip_mask.l4_header;
+ fsp->m_u.usr_ip6_spec.tclass = rule->ip_mask.tclass;
+ fsp->m_u.usr_ip6_spec.l4_proto = rule->ip_mask.proto;
+ break;
+ case ETHER_FLOW:
+ fsp->h_u.ether_spec.h_proto = rule->eth_data.etype;
+ fsp->m_u.ether_spec.h_proto = rule->eth_mask.etype;
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+ iavf_fill_rx_flow_ext_data(fsp, rule);
+
+ if (rule->action == VIRTCHNL_ACTION_DROP)
+ fsp->ring_cookie = RX_CLS_FLOW_DISC;
+ else
+ fsp->ring_cookie = rule->q_index;
+
+release_lock:
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+ return ret;
+}
+
+/**
+ * iavf_get_fdir_fltr_ids - fill buffer with filter IDs of active filters
+ * @adapter: the VF adapter structure containing the filter list
+ * @cmd: ethtool command data structure
+ * @rule_locs: ethtool array passed in from OS to receive filter IDs
+ *
+ * Returns 0 as expected for success by ethtool
+ */
+static int
+iavf_get_fdir_fltr_ids(struct iavf_adapter *adapter, struct ethtool_rxnfc *cmd,
+ u32 *rule_locs)
+{
+ struct iavf_fdir_fltr *fltr;
+ unsigned int cnt = 0;
+ int val = 0;
+
+ if (!(adapter->flags & IAVF_FLAG_FDIR_ENABLED))
+ return -EOPNOTSUPP;
+
+ cmd->data = IAVF_MAX_FDIR_FILTERS;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+
+ list_for_each_entry(fltr, &adapter->fdir_list_head, list) {
+ if (cnt == cmd->rule_cnt) {
+ val = -EMSGSIZE;
+ goto release_lock;
+ }
+ rule_locs[cnt] = fltr->loc;
+ cnt++;
+ }
+
+release_lock:
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+ if (!val)
+ cmd->rule_cnt = cnt;
+
+ return val;
+}
+
+/**
+ * iavf_add_fdir_fltr_info - Set the input set for Flow Director filter
+ * @adapter: pointer to the VF adapter structure
+ * @fsp: pointer to ethtool Rx flow specification
+ * @fltr: filter structure
+ */
+static int
+iavf_add_fdir_fltr_info(struct iavf_adapter *adapter, struct ethtool_rx_flow_spec *fsp,
+ struct iavf_fdir_fltr *fltr)
+{
+ u32 flow_type, q_index = 0;
+ enum virtchnl_action act;
+ int err;
+
+ if (fsp->ring_cookie == RX_CLS_FLOW_DISC) {
+ act = VIRTCHNL_ACTION_DROP;
+ } else {
+ q_index = fsp->ring_cookie;
+ if (q_index >= adapter->num_active_queues)
+ return -EINVAL;
+
+ act = VIRTCHNL_ACTION_QUEUE;
+ }
+
+ fltr->action = act;
+ fltr->loc = fsp->location;
+ fltr->q_index = q_index;
+
+ if (fsp->flow_type & FLOW_EXT) {
+ memcpy(fltr->ext_data.usr_def, fsp->h_ext.data,
+ sizeof(fltr->ext_data.usr_def));
+ memcpy(fltr->ext_mask.usr_def, fsp->m_ext.data,
+ sizeof(fltr->ext_mask.usr_def));
+ }
+
+ flow_type = fsp->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT | FLOW_RSS);
+ fltr->flow_type = iavf_ethtool_flow_to_fltr(flow_type);
+
+ switch (flow_type) {
+ case TCP_V4_FLOW:
+ case UDP_V4_FLOW:
+ case SCTP_V4_FLOW:
+ fltr->ip_data.v4_addrs.src_ip = fsp->h_u.tcp_ip4_spec.ip4src;
+ fltr->ip_data.v4_addrs.dst_ip = fsp->h_u.tcp_ip4_spec.ip4dst;
+ fltr->ip_data.src_port = fsp->h_u.tcp_ip4_spec.psrc;
+ fltr->ip_data.dst_port = fsp->h_u.tcp_ip4_spec.pdst;
+ fltr->ip_data.tos = fsp->h_u.tcp_ip4_spec.tos;
+ fltr->ip_mask.v4_addrs.src_ip = fsp->m_u.tcp_ip4_spec.ip4src;
+ fltr->ip_mask.v4_addrs.dst_ip = fsp->m_u.tcp_ip4_spec.ip4dst;
+ fltr->ip_mask.src_port = fsp->m_u.tcp_ip4_spec.psrc;
+ fltr->ip_mask.dst_port = fsp->m_u.tcp_ip4_spec.pdst;
+ fltr->ip_mask.tos = fsp->m_u.tcp_ip4_spec.tos;
+ fltr->ip_ver = 4;
+ break;
+ case AH_V4_FLOW:
+ case ESP_V4_FLOW:
+ fltr->ip_data.v4_addrs.src_ip = fsp->h_u.ah_ip4_spec.ip4src;
+ fltr->ip_data.v4_addrs.dst_ip = fsp->h_u.ah_ip4_spec.ip4dst;
+ fltr->ip_data.spi = fsp->h_u.ah_ip4_spec.spi;
+ fltr->ip_data.tos = fsp->h_u.ah_ip4_spec.tos;
+ fltr->ip_mask.v4_addrs.src_ip = fsp->m_u.ah_ip4_spec.ip4src;
+ fltr->ip_mask.v4_addrs.dst_ip = fsp->m_u.ah_ip4_spec.ip4dst;
+ fltr->ip_mask.spi = fsp->m_u.ah_ip4_spec.spi;
+ fltr->ip_mask.tos = fsp->m_u.ah_ip4_spec.tos;
+ fltr->ip_ver = 4;
+ break;
+ case IPV4_USER_FLOW:
+ fltr->ip_data.v4_addrs.src_ip = fsp->h_u.usr_ip4_spec.ip4src;
+ fltr->ip_data.v4_addrs.dst_ip = fsp->h_u.usr_ip4_spec.ip4dst;
+ fltr->ip_data.l4_header = fsp->h_u.usr_ip4_spec.l4_4_bytes;
+ fltr->ip_data.tos = fsp->h_u.usr_ip4_spec.tos;
+ fltr->ip_data.proto = fsp->h_u.usr_ip4_spec.proto;
+ fltr->ip_mask.v4_addrs.src_ip = fsp->m_u.usr_ip4_spec.ip4src;
+ fltr->ip_mask.v4_addrs.dst_ip = fsp->m_u.usr_ip4_spec.ip4dst;
+ fltr->ip_mask.l4_header = fsp->m_u.usr_ip4_spec.l4_4_bytes;
+ fltr->ip_mask.tos = fsp->m_u.usr_ip4_spec.tos;
+ fltr->ip_mask.proto = fsp->m_u.usr_ip4_spec.proto;
+ fltr->ip_ver = 4;
+ break;
+ case TCP_V6_FLOW:
+ case UDP_V6_FLOW:
+ case SCTP_V6_FLOW:
+ memcpy(&fltr->ip_data.v6_addrs.src_ip, fsp->h_u.usr_ip6_spec.ip6src,
+ sizeof(struct in6_addr));
+ memcpy(&fltr->ip_data.v6_addrs.dst_ip, fsp->h_u.usr_ip6_spec.ip6dst,
+ sizeof(struct in6_addr));
+ fltr->ip_data.src_port = fsp->h_u.tcp_ip6_spec.psrc;
+ fltr->ip_data.dst_port = fsp->h_u.tcp_ip6_spec.pdst;
+ fltr->ip_data.tclass = fsp->h_u.tcp_ip6_spec.tclass;
+ memcpy(&fltr->ip_mask.v6_addrs.src_ip, fsp->m_u.usr_ip6_spec.ip6src,
+ sizeof(struct in6_addr));
+ memcpy(&fltr->ip_mask.v6_addrs.dst_ip, fsp->m_u.usr_ip6_spec.ip6dst,
+ sizeof(struct in6_addr));
+ fltr->ip_mask.src_port = fsp->m_u.tcp_ip6_spec.psrc;
+ fltr->ip_mask.dst_port = fsp->m_u.tcp_ip6_spec.pdst;
+ fltr->ip_mask.tclass = fsp->m_u.tcp_ip6_spec.tclass;
+ fltr->ip_ver = 6;
+ break;
+ case AH_V6_FLOW:
+ case ESP_V6_FLOW:
+ memcpy(&fltr->ip_data.v6_addrs.src_ip, fsp->h_u.ah_ip6_spec.ip6src,
+ sizeof(struct in6_addr));
+ memcpy(&fltr->ip_data.v6_addrs.dst_ip, fsp->h_u.ah_ip6_spec.ip6dst,
+ sizeof(struct in6_addr));
+ fltr->ip_data.spi = fsp->h_u.ah_ip6_spec.spi;
+ fltr->ip_data.tclass = fsp->h_u.ah_ip6_spec.tclass;
+ memcpy(&fltr->ip_mask.v6_addrs.src_ip, fsp->m_u.ah_ip6_spec.ip6src,
+ sizeof(struct in6_addr));
+ memcpy(&fltr->ip_mask.v6_addrs.dst_ip, fsp->m_u.ah_ip6_spec.ip6dst,
+ sizeof(struct in6_addr));
+ fltr->ip_mask.spi = fsp->m_u.ah_ip6_spec.spi;
+ fltr->ip_mask.tclass = fsp->m_u.ah_ip6_spec.tclass;
+ fltr->ip_ver = 6;
+ break;
+ case IPV6_USER_FLOW:
+ memcpy(&fltr->ip_data.v6_addrs.src_ip, fsp->h_u.usr_ip6_spec.ip6src,
+ sizeof(struct in6_addr));
+ memcpy(&fltr->ip_data.v6_addrs.dst_ip, fsp->h_u.usr_ip6_spec.ip6dst,
+ sizeof(struct in6_addr));
+ fltr->ip_data.l4_header = fsp->h_u.usr_ip6_spec.l4_4_bytes;
+ fltr->ip_data.tclass = fsp->h_u.usr_ip6_spec.tclass;
+ fltr->ip_data.proto = fsp->h_u.usr_ip6_spec.l4_proto;
+ memcpy(&fltr->ip_mask.v6_addrs.src_ip, fsp->m_u.usr_ip6_spec.ip6src,
+ sizeof(struct in6_addr));
+ memcpy(&fltr->ip_mask.v6_addrs.dst_ip, fsp->m_u.usr_ip6_spec.ip6dst,
+ sizeof(struct in6_addr));
+ fltr->ip_mask.l4_header = fsp->m_u.usr_ip6_spec.l4_4_bytes;
+ fltr->ip_mask.tclass = fsp->m_u.usr_ip6_spec.tclass;
+ fltr->ip_mask.proto = fsp->m_u.usr_ip6_spec.l4_proto;
+ fltr->ip_ver = 6;
+ break;
+ case ETHER_FLOW:
+ fltr->eth_data.etype = fsp->h_u.ether_spec.h_proto;
+ fltr->eth_mask.etype = fsp->m_u.ether_spec.h_proto;
+ break;
+ default:
+ /* not doing un-parsed flow types */
+ return -EINVAL;
+ }
+
+ err = iavf_validate_fdir_fltr_masks(adapter, fltr);
+ if (err)
+ return err;
+
+ if (iavf_fdir_is_dup_fltr(adapter, fltr))
+ return -EEXIST;
+
+ err = iavf_parse_rx_flow_user_data(fsp, fltr);
+ if (err)
+ return err;
+
+ return iavf_fill_fdir_add_msg(adapter, fltr);
+}
+
+/**
+ * iavf_add_fdir_ethtool - add Flow Director filter
+ * @adapter: pointer to the VF adapter structure
+ * @cmd: command to add Flow Director filter
+ *
+ * Returns 0 on success and negative values for failure
+ */
+static int iavf_add_fdir_ethtool(struct iavf_adapter *adapter, struct ethtool_rxnfc *cmd)
+{
+ struct ethtool_rx_flow_spec *fsp = &cmd->fs;
+ struct iavf_fdir_fltr *fltr;
+ int count = 50;
+ int err;
+
+ if (!(adapter->flags & IAVF_FLAG_FDIR_ENABLED))
+ return -EOPNOTSUPP;
+
+ if (fsp->flow_type & FLOW_MAC_EXT)
+ return -EINVAL;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ if (adapter->fdir_active_fltr >= IAVF_MAX_FDIR_FILTERS) {
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+ dev_err(&adapter->pdev->dev,
+ "Unable to add Flow Director filter because VF reached the limit of max allowed filters (%u)\n",
+ IAVF_MAX_FDIR_FILTERS);
+ return -ENOSPC;
+ }
+
+ if (iavf_find_fdir_fltr_by_loc(adapter, fsp->location)) {
+ dev_err(&adapter->pdev->dev, "Failed to add Flow Director filter, it already exists\n");
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+ return -EEXIST;
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+
+ fltr = kzalloc(sizeof(*fltr), GFP_KERNEL);
+ if (!fltr)
+ return -ENOMEM;
+
+ while (!mutex_trylock(&adapter->crit_lock)) {
+ if (--count == 0) {
+ kfree(fltr);
+ return -EINVAL;
+ }
+ udelay(1);
+ }
+
+ err = iavf_add_fdir_fltr_info(adapter, fsp, fltr);
+ if (err)
+ goto ret;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ iavf_fdir_list_add_fltr(adapter, fltr);
+ adapter->fdir_active_fltr++;
+ if (adapter->link_up) {
+ fltr->state = IAVF_FDIR_FLTR_ADD_REQUEST;
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_FDIR_FILTER;
+ } else {
+ fltr->state = IAVF_FDIR_FLTR_INACTIVE;
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+
+ if (adapter->link_up)
+ mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
+ret:
+ if (err && fltr)
+ kfree(fltr);
+
+ mutex_unlock(&adapter->crit_lock);
+ return err;
+}
+
+/**
+ * iavf_del_fdir_ethtool - delete Flow Director filter
+ * @adapter: pointer to the VF adapter structure
+ * @cmd: command to delete Flow Director filter
+ *
+ * Returns 0 on success and negative values for failure
+ */
+static int iavf_del_fdir_ethtool(struct iavf_adapter *adapter, struct ethtool_rxnfc *cmd)
+{
+ struct ethtool_rx_flow_spec *fsp = (struct ethtool_rx_flow_spec *)&cmd->fs;
+ struct iavf_fdir_fltr *fltr = NULL;
+ int err = 0;
+
+ if (!(adapter->flags & IAVF_FLAG_FDIR_ENABLED))
+ return -EOPNOTSUPP;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ fltr = iavf_find_fdir_fltr_by_loc(adapter, fsp->location);
+ if (fltr) {
+ if (fltr->state == IAVF_FDIR_FLTR_ACTIVE) {
+ fltr->state = IAVF_FDIR_FLTR_DEL_REQUEST;
+ adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
+ } else if (fltr->state == IAVF_FDIR_FLTR_INACTIVE) {
+ list_del(&fltr->list);
+ kfree(fltr);
+ adapter->fdir_active_fltr--;
+ fltr = NULL;
+ } else {
+ err = -EBUSY;
+ }
+ } else if (adapter->fdir_active_fltr) {
+ err = -EINVAL;
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+
+ if (fltr && fltr->state == IAVF_FDIR_FLTR_DEL_REQUEST)
+ mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
+
+ return err;
+}
+
+/**
+ * iavf_adv_rss_parse_hdrs - parses headers from RSS hash input
+ * @cmd: ethtool rxnfc command
+ *
+ * This function parses the rxnfc command and returns intended
+ * header types for RSS configuration
+ */
+static u32 iavf_adv_rss_parse_hdrs(struct ethtool_rxnfc *cmd)
+{
+ u32 hdrs = IAVF_ADV_RSS_FLOW_SEG_HDR_NONE;
+
+ switch (cmd->flow_type) {
+ case TCP_V4_FLOW:
+ hdrs |= IAVF_ADV_RSS_FLOW_SEG_HDR_TCP |
+ IAVF_ADV_RSS_FLOW_SEG_HDR_IPV4;
+ break;
+ case UDP_V4_FLOW:
+ hdrs |= IAVF_ADV_RSS_FLOW_SEG_HDR_UDP |
+ IAVF_ADV_RSS_FLOW_SEG_HDR_IPV4;
+ break;
+ case SCTP_V4_FLOW:
+ hdrs |= IAVF_ADV_RSS_FLOW_SEG_HDR_SCTP |
+ IAVF_ADV_RSS_FLOW_SEG_HDR_IPV4;
+ break;
+ case TCP_V6_FLOW:
+ hdrs |= IAVF_ADV_RSS_FLOW_SEG_HDR_TCP |
+ IAVF_ADV_RSS_FLOW_SEG_HDR_IPV6;
+ break;
+ case UDP_V6_FLOW:
+ hdrs |= IAVF_ADV_RSS_FLOW_SEG_HDR_UDP |
+ IAVF_ADV_RSS_FLOW_SEG_HDR_IPV6;
+ break;
+ case SCTP_V6_FLOW:
+ hdrs |= IAVF_ADV_RSS_FLOW_SEG_HDR_SCTP |
+ IAVF_ADV_RSS_FLOW_SEG_HDR_IPV6;
+ break;
+ default:
+ break;
+ }
+
+ return hdrs;
+}
+
+/**
+ * iavf_adv_rss_parse_hash_flds - parses hash fields from RSS hash input
+ * @cmd: ethtool rxnfc command
+ *
+ * This function parses the rxnfc command and returns intended hash fields for
+ * RSS configuration
+ */
+static u64 iavf_adv_rss_parse_hash_flds(struct ethtool_rxnfc *cmd)
+{
+ u64 hfld = IAVF_ADV_RSS_HASH_INVALID;
+
+ if (cmd->data & RXH_IP_SRC || cmd->data & RXH_IP_DST) {
+ switch (cmd->flow_type) {
+ case TCP_V4_FLOW:
+ case UDP_V4_FLOW:
+ case SCTP_V4_FLOW:
+ if (cmd->data & RXH_IP_SRC)
+ hfld |= IAVF_ADV_RSS_HASH_FLD_IPV4_SA;
+ if (cmd->data & RXH_IP_DST)
+ hfld |= IAVF_ADV_RSS_HASH_FLD_IPV4_DA;
+ break;
+ case TCP_V6_FLOW:
+ case UDP_V6_FLOW:
+ case SCTP_V6_FLOW:
+ if (cmd->data & RXH_IP_SRC)
+ hfld |= IAVF_ADV_RSS_HASH_FLD_IPV6_SA;
+ if (cmd->data & RXH_IP_DST)
+ hfld |= IAVF_ADV_RSS_HASH_FLD_IPV6_DA;
+ break;
+ default:
+ break;
+ }
+ }
+
+ if (cmd->data & RXH_L4_B_0_1 || cmd->data & RXH_L4_B_2_3) {
+ switch (cmd->flow_type) {
+ case TCP_V4_FLOW:
+ case TCP_V6_FLOW:
+ if (cmd->data & RXH_L4_B_0_1)
+ hfld |= IAVF_ADV_RSS_HASH_FLD_TCP_SRC_PORT;
+ if (cmd->data & RXH_L4_B_2_3)
+ hfld |= IAVF_ADV_RSS_HASH_FLD_TCP_DST_PORT;
+ break;
+ case UDP_V4_FLOW:
+ case UDP_V6_FLOW:
+ if (cmd->data & RXH_L4_B_0_1)
+ hfld |= IAVF_ADV_RSS_HASH_FLD_UDP_SRC_PORT;
+ if (cmd->data & RXH_L4_B_2_3)
+ hfld |= IAVF_ADV_RSS_HASH_FLD_UDP_DST_PORT;
+ break;
+ case SCTP_V4_FLOW:
+ case SCTP_V6_FLOW:
+ if (cmd->data & RXH_L4_B_0_1)
+ hfld |= IAVF_ADV_RSS_HASH_FLD_SCTP_SRC_PORT;
+ if (cmd->data & RXH_L4_B_2_3)
+ hfld |= IAVF_ADV_RSS_HASH_FLD_SCTP_DST_PORT;
+ break;
+ default:
+ break;
+ }
+ }
+
+ return hfld;
+}
+
+/**
+ * iavf_set_adv_rss_hash_opt - Enable/Disable flow types for RSS hash
+ * @adapter: pointer to the VF adapter structure
+ * @cmd: ethtool rxnfc command
+ *
+ * Returns Success if the flow input set is supported.
+ */
+static int
+iavf_set_adv_rss_hash_opt(struct iavf_adapter *adapter,
+ struct ethtool_rxnfc *cmd)
+{
+ struct iavf_adv_rss *rss_old, *rss_new;
+ bool rss_new_add = false;
+ int count = 50, err = 0;
+ u64 hash_flds;
+ u32 hdrs;
+
+ if (!ADV_RSS_SUPPORT(adapter))
+ return -EOPNOTSUPP;
+
+ hdrs = iavf_adv_rss_parse_hdrs(cmd);
+ if (hdrs == IAVF_ADV_RSS_FLOW_SEG_HDR_NONE)
+ return -EINVAL;
+
+ hash_flds = iavf_adv_rss_parse_hash_flds(cmd);
+ if (hash_flds == IAVF_ADV_RSS_HASH_INVALID)
+ return -EINVAL;
+
+ rss_new = kzalloc(sizeof(*rss_new), GFP_KERNEL);
+ if (!rss_new)
+ return -ENOMEM;
+
+ if (iavf_fill_adv_rss_cfg_msg(&rss_new->cfg_msg, hdrs, hash_flds)) {
+ kfree(rss_new);
+ return -EINVAL;
+ }
+
+ while (!mutex_trylock(&adapter->crit_lock)) {
+ if (--count == 0) {
+ kfree(rss_new);
+ return -EINVAL;
+ }
+
+ udelay(1);
+ }
+
+ spin_lock_bh(&adapter->adv_rss_lock);
+ rss_old = iavf_find_adv_rss_cfg_by_hdrs(adapter, hdrs);
+ if (rss_old) {
+ if (rss_old->state != IAVF_ADV_RSS_ACTIVE) {
+ err = -EBUSY;
+ } else if (rss_old->hash_flds != hash_flds) {
+ rss_old->state = IAVF_ADV_RSS_ADD_REQUEST;
+ rss_old->hash_flds = hash_flds;
+ memcpy(&rss_old->cfg_msg, &rss_new->cfg_msg,
+ sizeof(rss_new->cfg_msg));
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_ADV_RSS_CFG;
+ } else {
+ err = -EEXIST;
+ }
+ } else {
+ rss_new_add = true;
+ rss_new->state = IAVF_ADV_RSS_ADD_REQUEST;
+ rss_new->packet_hdrs = hdrs;
+ rss_new->hash_flds = hash_flds;
+ list_add_tail(&rss_new->list, &adapter->adv_rss_list_head);
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_ADV_RSS_CFG;
+ }
+ spin_unlock_bh(&adapter->adv_rss_lock);
+
+ if (!err)
+ mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
+
+ mutex_unlock(&adapter->crit_lock);
+
+ if (!rss_new_add)
+ kfree(rss_new);
+
+ return err;
+}
+
+/**
+ * iavf_get_adv_rss_hash_opt - Retrieve hash fields for a given flow-type
+ * @adapter: pointer to the VF adapter structure
+ * @cmd: ethtool rxnfc command
+ *
+ * Returns Success if the flow input set is supported.
+ */
+static int
+iavf_get_adv_rss_hash_opt(struct iavf_adapter *adapter,
+ struct ethtool_rxnfc *cmd)
+{
+ struct iavf_adv_rss *rss;
+ u64 hash_flds;
+ u32 hdrs;
+
+ if (!ADV_RSS_SUPPORT(adapter))
+ return -EOPNOTSUPP;
+
+ cmd->data = 0;
+
+ hdrs = iavf_adv_rss_parse_hdrs(cmd);
+ if (hdrs == IAVF_ADV_RSS_FLOW_SEG_HDR_NONE)
+ return -EINVAL;
+
+ spin_lock_bh(&adapter->adv_rss_lock);
+ rss = iavf_find_adv_rss_cfg_by_hdrs(adapter, hdrs);
+ if (rss)
+ hash_flds = rss->hash_flds;
+ else
+ hash_flds = IAVF_ADV_RSS_HASH_INVALID;
+ spin_unlock_bh(&adapter->adv_rss_lock);
+
+ if (hash_flds == IAVF_ADV_RSS_HASH_INVALID)
+ return -EINVAL;
+
+ if (hash_flds & (IAVF_ADV_RSS_HASH_FLD_IPV4_SA |
+ IAVF_ADV_RSS_HASH_FLD_IPV6_SA))
+ cmd->data |= (u64)RXH_IP_SRC;
+
+ if (hash_flds & (IAVF_ADV_RSS_HASH_FLD_IPV4_DA |
+ IAVF_ADV_RSS_HASH_FLD_IPV6_DA))
+ cmd->data |= (u64)RXH_IP_DST;
+
+ if (hash_flds & (IAVF_ADV_RSS_HASH_FLD_TCP_SRC_PORT |
+ IAVF_ADV_RSS_HASH_FLD_UDP_SRC_PORT |
+ IAVF_ADV_RSS_HASH_FLD_SCTP_SRC_PORT))
+ cmd->data |= (u64)RXH_L4_B_0_1;
+
+ if (hash_flds & (IAVF_ADV_RSS_HASH_FLD_TCP_DST_PORT |
+ IAVF_ADV_RSS_HASH_FLD_UDP_DST_PORT |
+ IAVF_ADV_RSS_HASH_FLD_SCTP_DST_PORT))
+ cmd->data |= (u64)RXH_L4_B_2_3;
+
+ return 0;
+}
+
+/**
+ * iavf_set_rxnfc - command to set Rx flow rules.
+ * @netdev: network interface device structure
+ * @cmd: ethtool rxnfc command
+ *
+ * Returns 0 for success and negative values for errors
+ */
+static int iavf_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ int ret = -EOPNOTSUPP;
+
+ switch (cmd->cmd) {
+ case ETHTOOL_SRXCLSRLINS:
+ ret = iavf_add_fdir_ethtool(adapter, cmd);
+ break;
+ case ETHTOOL_SRXCLSRLDEL:
+ ret = iavf_del_fdir_ethtool(adapter, cmd);
+ break;
+ case ETHTOOL_SRXFH:
+ ret = iavf_set_adv_rss_hash_opt(adapter, cmd);
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+/**
+ * iavf_get_rxnfc - command to get RX flow classification rules
+ * @netdev: network interface device structure
+ * @cmd: ethtool rxnfc command
+ * @rule_locs: pointer to store rule locations
+ *
+ * Returns Success if the command is supported.
+ **/
+static int iavf_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd,
+ u32 *rule_locs)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ int ret = -EOPNOTSUPP;
+
+ switch (cmd->cmd) {
+ case ETHTOOL_GRXRINGS:
+ cmd->data = adapter->num_active_queues;
+ ret = 0;
+ break;
+ case ETHTOOL_GRXCLSRLCNT:
+ if (!(adapter->flags & IAVF_FLAG_FDIR_ENABLED))
+ break;
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ cmd->rule_cnt = adapter->fdir_active_fltr;
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+ cmd->data = IAVF_MAX_FDIR_FILTERS;
+ ret = 0;
+ break;
+ case ETHTOOL_GRXCLSRULE:
+ ret = iavf_get_ethtool_fdir_entry(adapter, cmd);
+ break;
+ case ETHTOOL_GRXCLSRLALL:
+ ret = iavf_get_fdir_fltr_ids(adapter, cmd, (u32 *)rule_locs);
+ break;
+ case ETHTOOL_GRXFH:
+ ret = iavf_get_adv_rss_hash_opt(adapter, cmd);
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+/**
+ * iavf_get_channels: get the number of channels supported by the device
+ * @netdev: network interface device structure
+ * @ch: channel information structure
+ *
+ * For the purposes of our device, we only use combined channels, i.e. a tx/rx
+ * queue pair. Report one extra channel to match our "other" MSI-X vector.
+ **/
+static void iavf_get_channels(struct net_device *netdev,
+ struct ethtool_channels *ch)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ /* Report maximum channels */
+ ch->max_combined = adapter->vsi_res->num_queue_pairs;
+
+ ch->max_other = NONQ_VECS;
+ ch->other_count = NONQ_VECS;
+
+ ch->combined_count = adapter->num_active_queues;
+}
+
+/**
+ * iavf_set_channels: set the new channel count
+ * @netdev: network interface device structure
+ * @ch: channel information structure
+ *
+ * Negotiate a new number of channels with the PF then do a reset. During
+ * reset we'll realloc queues and fix the RSS table. Returns 0 on success,
+ * negative on failure.
+ **/
+static int iavf_set_channels(struct net_device *netdev,
+ struct ethtool_channels *ch)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ u32 num_req = ch->combined_count;
+ int ret = 0;
+
+ if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
+ adapter->num_tc) {
+ dev_info(&adapter->pdev->dev, "Cannot set channels since ADq is enabled.\n");
+ return -EINVAL;
+ }
+
+ /* All of these should have already been checked by ethtool before this
+ * even gets to us, but just to be sure.
+ */
+ if (num_req == 0 || num_req > adapter->vsi_res->num_queue_pairs)
+ return -EINVAL;
+
+ if (num_req == adapter->num_active_queues)
+ return 0;
+
+ if (ch->rx_count || ch->tx_count || ch->other_count != NONQ_VECS)
+ return -EINVAL;
+
+ adapter->num_req_queues = num_req;
+ adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED;
+ iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
+
+ ret = iavf_wait_for_reset(adapter);
+ if (ret)
+ netdev_warn(netdev, "Changing channel count timeout or interrupted waiting for reset");
+
+ return ret;
+}
+
+/**
+ * iavf_get_rxfh_key_size - get the RSS hash key size
+ * @netdev: network interface device structure
+ *
+ * Returns the table size.
+ **/
+static u32 iavf_get_rxfh_key_size(struct net_device *netdev)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ return adapter->rss_key_size;
+}
+
+/**
+ * iavf_get_rxfh_indir_size - get the rx flow hash indirection table size
+ * @netdev: network interface device structure
+ *
+ * Returns the table size.
+ **/
+static u32 iavf_get_rxfh_indir_size(struct net_device *netdev)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ return adapter->rss_lut_size;
+}
+
+/**
+ * iavf_get_rxfh - get the rx flow hash indirection table
+ * @netdev: network interface device structure
+ * @indir: indirection table
+ * @key: hash key
+ * @hfunc: hash function in use
+ *
+ * Reads the indirection table directly from the hardware. Always returns 0.
+ **/
+static int iavf_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key,
+ u8 *hfunc)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ u16 i;
+
+ if (hfunc)
+ *hfunc = ETH_RSS_HASH_TOP;
+ if (key)
+ memcpy(key, adapter->rss_key, adapter->rss_key_size);
+
+ if (indir)
+ /* Each 32 bits pointed by 'indir' is stored with a lut entry */
+ for (i = 0; i < adapter->rss_lut_size; i++)
+ indir[i] = (u32)adapter->rss_lut[i];
+
+ return 0;
+}
+
+/**
+ * iavf_set_rxfh - set the rx flow hash indirection table
+ * @netdev: network interface device structure
+ * @indir: indirection table
+ * @key: hash key
+ * @hfunc: hash function to use
+ *
+ * Returns -EINVAL if the table specifies an invalid queue id, otherwise
+ * returns 0 after programming the table.
+ **/
+static int iavf_set_rxfh(struct net_device *netdev, const u32 *indir,
+ const u8 *key, const u8 hfunc)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ u16 i;
+
+ /* Only support toeplitz hash function */
+ if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
+ return -EOPNOTSUPP;
+
+ if (!key && !indir)
+ return 0;
+
+ if (key)
+ memcpy(adapter->rss_key, key, adapter->rss_key_size);
+
+ if (indir) {
+ /* Each 32 bits pointed by 'indir' is stored with a lut entry */
+ for (i = 0; i < adapter->rss_lut_size; i++)
+ adapter->rss_lut[i] = (u8)(indir[i]);
+ }
+
+ return iavf_config_rss(adapter);
+}
+
+static const struct ethtool_ops iavf_ethtool_ops = {
+ .supported_coalesce_params = ETHTOOL_COALESCE_USECS |
+ ETHTOOL_COALESCE_USE_ADAPTIVE,
+ .get_drvinfo = iavf_get_drvinfo,
+ .get_link = ethtool_op_get_link,
+ .get_ringparam = iavf_get_ringparam,
+ .set_ringparam = iavf_set_ringparam,
+ .get_strings = iavf_get_strings,
+ .get_ethtool_stats = iavf_get_ethtool_stats,
+ .get_sset_count = iavf_get_sset_count,
+ .get_priv_flags = iavf_get_priv_flags,
+ .set_priv_flags = iavf_set_priv_flags,
+ .get_msglevel = iavf_get_msglevel,
+ .set_msglevel = iavf_set_msglevel,
+ .get_coalesce = iavf_get_coalesce,
+ .set_coalesce = iavf_set_coalesce,
+ .get_per_queue_coalesce = iavf_get_per_queue_coalesce,
+ .set_per_queue_coalesce = iavf_set_per_queue_coalesce,
+ .set_rxnfc = iavf_set_rxnfc,
+ .get_rxnfc = iavf_get_rxnfc,
+ .get_rxfh_indir_size = iavf_get_rxfh_indir_size,
+ .get_rxfh = iavf_get_rxfh,
+ .set_rxfh = iavf_set_rxfh,
+ .get_channels = iavf_get_channels,
+ .set_channels = iavf_set_channels,
+ .get_rxfh_key_size = iavf_get_rxfh_key_size,
+ .get_link_ksettings = iavf_get_link_ksettings,
+};
+
+/**
+ * iavf_set_ethtool_ops - Initialize ethtool ops struct
+ * @netdev: network interface device structure
+ *
+ * Sets ethtool ops struct in our netdev so that ethtool can call
+ * our functions.
+ **/
+void iavf_set_ethtool_ops(struct net_device *netdev)
+{
+ netdev->ethtool_ops = &iavf_ethtool_ops;
+}
diff --git a/drivers/net/ethernet/intel/iavf/iavf_fdir.c b/drivers/net/ethernet/intel/iavf/iavf_fdir.c
new file mode 100644
index 0000000000..03e774bd2a
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_fdir.c
@@ -0,0 +1,853 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2020, Intel Corporation. */
+
+/* flow director ethtool support for iavf */
+
+#include "iavf.h"
+
+#define GTPU_PORT 2152
+#define NAT_T_ESP_PORT 4500
+#define PFCP_PORT 8805
+
+static const struct in6_addr ipv6_addr_full_mask = {
+ .in6_u = {
+ .u6_addr8 = {
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ }
+ }
+};
+
+static const struct in6_addr ipv6_addr_zero_mask = {
+ .in6_u = {
+ .u6_addr8 = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ }
+ }
+};
+
+/**
+ * iavf_validate_fdir_fltr_masks - validate Flow Director filter fields masks
+ * @adapter: pointer to the VF adapter structure
+ * @fltr: Flow Director filter data structure
+ *
+ * Returns 0 if all masks of packet fields are either full or empty. Returns
+ * error on at least one partial mask.
+ */
+int iavf_validate_fdir_fltr_masks(struct iavf_adapter *adapter,
+ struct iavf_fdir_fltr *fltr)
+{
+ if (fltr->eth_mask.etype && fltr->eth_mask.etype != htons(U16_MAX))
+ goto partial_mask;
+
+ if (fltr->ip_ver == 4) {
+ if (fltr->ip_mask.v4_addrs.src_ip &&
+ fltr->ip_mask.v4_addrs.src_ip != htonl(U32_MAX))
+ goto partial_mask;
+
+ if (fltr->ip_mask.v4_addrs.dst_ip &&
+ fltr->ip_mask.v4_addrs.dst_ip != htonl(U32_MAX))
+ goto partial_mask;
+
+ if (fltr->ip_mask.tos && fltr->ip_mask.tos != U8_MAX)
+ goto partial_mask;
+ } else if (fltr->ip_ver == 6) {
+ if (memcmp(&fltr->ip_mask.v6_addrs.src_ip, &ipv6_addr_zero_mask,
+ sizeof(struct in6_addr)) &&
+ memcmp(&fltr->ip_mask.v6_addrs.src_ip, &ipv6_addr_full_mask,
+ sizeof(struct in6_addr)))
+ goto partial_mask;
+
+ if (memcmp(&fltr->ip_mask.v6_addrs.dst_ip, &ipv6_addr_zero_mask,
+ sizeof(struct in6_addr)) &&
+ memcmp(&fltr->ip_mask.v6_addrs.dst_ip, &ipv6_addr_full_mask,
+ sizeof(struct in6_addr)))
+ goto partial_mask;
+
+ if (fltr->ip_mask.tclass && fltr->ip_mask.tclass != U8_MAX)
+ goto partial_mask;
+ }
+
+ if (fltr->ip_mask.proto && fltr->ip_mask.proto != U8_MAX)
+ goto partial_mask;
+
+ if (fltr->ip_mask.src_port && fltr->ip_mask.src_port != htons(U16_MAX))
+ goto partial_mask;
+
+ if (fltr->ip_mask.dst_port && fltr->ip_mask.dst_port != htons(U16_MAX))
+ goto partial_mask;
+
+ if (fltr->ip_mask.spi && fltr->ip_mask.spi != htonl(U32_MAX))
+ goto partial_mask;
+
+ if (fltr->ip_mask.l4_header &&
+ fltr->ip_mask.l4_header != htonl(U32_MAX))
+ goto partial_mask;
+
+ return 0;
+
+partial_mask:
+ dev_err(&adapter->pdev->dev, "Failed to add Flow Director filter, partial masks are not supported\n");
+ return -EOPNOTSUPP;
+}
+
+/**
+ * iavf_pkt_udp_no_pay_len - the length of UDP packet without payload
+ * @fltr: Flow Director filter data structure
+ */
+static u16 iavf_pkt_udp_no_pay_len(struct iavf_fdir_fltr *fltr)
+{
+ return sizeof(struct ethhdr) +
+ (fltr->ip_ver == 4 ? sizeof(struct iphdr) : sizeof(struct ipv6hdr)) +
+ sizeof(struct udphdr);
+}
+
+/**
+ * iavf_fill_fdir_gtpu_hdr - fill the GTP-U protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the GTP-U protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_gtpu_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *uhdr = &proto_hdrs->proto_hdr[proto_hdrs->count - 1];
+ struct virtchnl_proto_hdr *ghdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ struct virtchnl_proto_hdr *ehdr = NULL; /* Extension Header if it exists */
+ u16 adj_offs, hdr_offs;
+ int i;
+
+ VIRTCHNL_SET_PROTO_HDR_TYPE(ghdr, GTPU_IP);
+
+ adj_offs = iavf_pkt_udp_no_pay_len(fltr);
+
+ for (i = 0; i < fltr->flex_cnt; i++) {
+#define IAVF_GTPU_HDR_TEID_OFFS0 4
+#define IAVF_GTPU_HDR_TEID_OFFS1 6
+#define IAVF_GTPU_HDR_N_PDU_AND_NEXT_EXTHDR_OFFS 10
+#define IAVF_GTPU_HDR_NEXT_EXTHDR_TYPE_MASK 0x00FF /* skip N_PDU */
+/* PDU Session Container Extension Header (PSC) */
+#define IAVF_GTPU_PSC_EXTHDR_TYPE 0x85
+#define IAVF_GTPU_HDR_PSC_PDU_TYPE_AND_QFI_OFFS 13
+#define IAVF_GTPU_HDR_PSC_PDU_QFI_MASK 0x3F /* skip Type */
+#define IAVF_GTPU_EH_QFI_IDX 1
+
+ if (fltr->flex_words[i].offset < adj_offs)
+ return -EINVAL;
+
+ hdr_offs = fltr->flex_words[i].offset - adj_offs;
+
+ switch (hdr_offs) {
+ case IAVF_GTPU_HDR_TEID_OFFS0:
+ case IAVF_GTPU_HDR_TEID_OFFS1: {
+ __be16 *pay_word = (__be16 *)ghdr->buffer;
+
+ pay_word[hdr_offs >> 1] = htons(fltr->flex_words[i].word);
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(ghdr, GTPU_IP, TEID);
+ }
+ break;
+ case IAVF_GTPU_HDR_N_PDU_AND_NEXT_EXTHDR_OFFS:
+ if ((fltr->flex_words[i].word &
+ IAVF_GTPU_HDR_NEXT_EXTHDR_TYPE_MASK) !=
+ IAVF_GTPU_PSC_EXTHDR_TYPE)
+ return -EOPNOTSUPP;
+ if (!ehdr)
+ ehdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ VIRTCHNL_SET_PROTO_HDR_TYPE(ehdr, GTPU_EH);
+ break;
+ case IAVF_GTPU_HDR_PSC_PDU_TYPE_AND_QFI_OFFS:
+ if (!ehdr)
+ return -EINVAL;
+ ehdr->buffer[IAVF_GTPU_EH_QFI_IDX] =
+ fltr->flex_words[i].word &
+ IAVF_GTPU_HDR_PSC_PDU_QFI_MASK;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(ehdr, GTPU_EH, QFI);
+ break;
+ default:
+ return -EINVAL;
+ }
+ }
+
+ uhdr->field_selector = 0; /* The PF ignores the UDP header fields */
+
+ return 0;
+}
+
+/**
+ * iavf_fill_fdir_pfcp_hdr - fill the PFCP protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the PFCP protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_pfcp_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *uhdr = &proto_hdrs->proto_hdr[proto_hdrs->count - 1];
+ struct virtchnl_proto_hdr *hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ u16 adj_offs, hdr_offs;
+ int i;
+
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, PFCP);
+
+ adj_offs = iavf_pkt_udp_no_pay_len(fltr);
+
+ for (i = 0; i < fltr->flex_cnt; i++) {
+#define IAVF_PFCP_HDR_SFIELD_AND_MSG_TYPE_OFFS 0
+ if (fltr->flex_words[i].offset < adj_offs)
+ return -EINVAL;
+
+ hdr_offs = fltr->flex_words[i].offset - adj_offs;
+
+ switch (hdr_offs) {
+ case IAVF_PFCP_HDR_SFIELD_AND_MSG_TYPE_OFFS:
+ hdr->buffer[0] = (fltr->flex_words[i].word >> 8) & 0xff;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, PFCP, S_FIELD);
+ break;
+ default:
+ return -EINVAL;
+ }
+ }
+
+ uhdr->field_selector = 0; /* The PF ignores the UDP header fields */
+
+ return 0;
+}
+
+/**
+ * iavf_fill_fdir_nat_t_esp_hdr - fill the NAT-T-ESP protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the NAT-T-ESP protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_nat_t_esp_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *uhdr = &proto_hdrs->proto_hdr[proto_hdrs->count - 1];
+ struct virtchnl_proto_hdr *hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ u16 adj_offs, hdr_offs;
+ u32 spi = 0;
+ int i;
+
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, ESP);
+
+ adj_offs = iavf_pkt_udp_no_pay_len(fltr);
+
+ for (i = 0; i < fltr->flex_cnt; i++) {
+#define IAVF_NAT_T_ESP_SPI_OFFS0 0
+#define IAVF_NAT_T_ESP_SPI_OFFS1 2
+ if (fltr->flex_words[i].offset < adj_offs)
+ return -EINVAL;
+
+ hdr_offs = fltr->flex_words[i].offset - adj_offs;
+
+ switch (hdr_offs) {
+ case IAVF_NAT_T_ESP_SPI_OFFS0:
+ spi |= fltr->flex_words[i].word << 16;
+ break;
+ case IAVF_NAT_T_ESP_SPI_OFFS1:
+ spi |= fltr->flex_words[i].word;
+ break;
+ default:
+ return -EINVAL;
+ }
+ }
+
+ if (!spi)
+ return -EOPNOTSUPP; /* Not support IKE Header Format with SPI 0 */
+
+ *(__be32 *)hdr->buffer = htonl(spi);
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, ESP, SPI);
+
+ uhdr->field_selector = 0; /* The PF ignores the UDP header fields */
+
+ return 0;
+}
+
+/**
+ * iavf_fill_fdir_udp_flex_pay_hdr - fill the UDP payload header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the UDP payload defined protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_udp_flex_pay_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ int err;
+
+ switch (ntohs(fltr->ip_data.dst_port)) {
+ case GTPU_PORT:
+ err = iavf_fill_fdir_gtpu_hdr(fltr, proto_hdrs);
+ break;
+ case NAT_T_ESP_PORT:
+ err = iavf_fill_fdir_nat_t_esp_hdr(fltr, proto_hdrs);
+ break;
+ case PFCP_PORT:
+ err = iavf_fill_fdir_pfcp_hdr(fltr, proto_hdrs);
+ break;
+ default:
+ err = -EOPNOTSUPP;
+ break;
+ }
+
+ return err;
+}
+
+/**
+ * iavf_fill_fdir_ip4_hdr - fill the IPv4 protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the IPv4 protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_ip4_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ struct iphdr *iph = (struct iphdr *)hdr->buffer;
+
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, IPV4);
+
+ if (fltr->ip_mask.tos == U8_MAX) {
+ iph->tos = fltr->ip_data.tos;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV4, DSCP);
+ }
+
+ if (fltr->ip_mask.proto == U8_MAX) {
+ iph->protocol = fltr->ip_data.proto;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV4, PROT);
+ }
+
+ if (fltr->ip_mask.v4_addrs.src_ip == htonl(U32_MAX)) {
+ iph->saddr = fltr->ip_data.v4_addrs.src_ip;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV4, SRC);
+ }
+
+ if (fltr->ip_mask.v4_addrs.dst_ip == htonl(U32_MAX)) {
+ iph->daddr = fltr->ip_data.v4_addrs.dst_ip;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV4, DST);
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_fill_fdir_ip6_hdr - fill the IPv6 protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the IPv6 protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_ip6_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ struct ipv6hdr *iph = (struct ipv6hdr *)hdr->buffer;
+
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, IPV6);
+
+ if (fltr->ip_mask.tclass == U8_MAX) {
+ iph->priority = (fltr->ip_data.tclass >> 4) & 0xF;
+ iph->flow_lbl[0] = (fltr->ip_data.tclass << 4) & 0xF0;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV6, TC);
+ }
+
+ if (fltr->ip_mask.proto == U8_MAX) {
+ iph->nexthdr = fltr->ip_data.proto;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV6, PROT);
+ }
+
+ if (!memcmp(&fltr->ip_mask.v6_addrs.src_ip, &ipv6_addr_full_mask,
+ sizeof(struct in6_addr))) {
+ memcpy(&iph->saddr, &fltr->ip_data.v6_addrs.src_ip,
+ sizeof(struct in6_addr));
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV6, SRC);
+ }
+
+ if (!memcmp(&fltr->ip_mask.v6_addrs.dst_ip, &ipv6_addr_full_mask,
+ sizeof(struct in6_addr))) {
+ memcpy(&iph->daddr, &fltr->ip_data.v6_addrs.dst_ip,
+ sizeof(struct in6_addr));
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, IPV6, DST);
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_fill_fdir_tcp_hdr - fill the TCP protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the TCP protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_tcp_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ struct tcphdr *tcph = (struct tcphdr *)hdr->buffer;
+
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, TCP);
+
+ if (fltr->ip_mask.src_port == htons(U16_MAX)) {
+ tcph->source = fltr->ip_data.src_port;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, TCP, SRC_PORT);
+ }
+
+ if (fltr->ip_mask.dst_port == htons(U16_MAX)) {
+ tcph->dest = fltr->ip_data.dst_port;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, TCP, DST_PORT);
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_fill_fdir_udp_hdr - fill the UDP protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the UDP protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_udp_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ struct udphdr *udph = (struct udphdr *)hdr->buffer;
+
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, UDP);
+
+ if (fltr->ip_mask.src_port == htons(U16_MAX)) {
+ udph->source = fltr->ip_data.src_port;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, UDP, SRC_PORT);
+ }
+
+ if (fltr->ip_mask.dst_port == htons(U16_MAX)) {
+ udph->dest = fltr->ip_data.dst_port;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, UDP, DST_PORT);
+ }
+
+ if (!fltr->flex_cnt)
+ return 0;
+
+ return iavf_fill_fdir_udp_flex_pay_hdr(fltr, proto_hdrs);
+}
+
+/**
+ * iavf_fill_fdir_sctp_hdr - fill the SCTP protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the SCTP protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_sctp_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ struct sctphdr *sctph = (struct sctphdr *)hdr->buffer;
+
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, SCTP);
+
+ if (fltr->ip_mask.src_port == htons(U16_MAX)) {
+ sctph->source = fltr->ip_data.src_port;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, SCTP, SRC_PORT);
+ }
+
+ if (fltr->ip_mask.dst_port == htons(U16_MAX)) {
+ sctph->dest = fltr->ip_data.dst_port;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, SCTP, DST_PORT);
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_fill_fdir_ah_hdr - fill the AH protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the AH protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_ah_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ struct ip_auth_hdr *ah = (struct ip_auth_hdr *)hdr->buffer;
+
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, AH);
+
+ if (fltr->ip_mask.spi == htonl(U32_MAX)) {
+ ah->spi = fltr->ip_data.spi;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, AH, SPI);
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_fill_fdir_esp_hdr - fill the ESP protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the ESP protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_esp_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ struct ip_esp_hdr *esph = (struct ip_esp_hdr *)hdr->buffer;
+
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, ESP);
+
+ if (fltr->ip_mask.spi == htonl(U32_MAX)) {
+ esph->spi = fltr->ip_data.spi;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, ESP, SPI);
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_fill_fdir_l4_hdr - fill the L4 protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the L4 protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_l4_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *hdr;
+ __be32 *l4_4_data;
+
+ if (!fltr->ip_mask.proto) /* IPv4/IPv6 header only */
+ return 0;
+
+ hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ l4_4_data = (__be32 *)hdr->buffer;
+
+ /* L2TPv3 over IP with 'Session ID' */
+ if (fltr->ip_data.proto == 115 && fltr->ip_mask.l4_header == htonl(U32_MAX)) {
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, L2TPV3);
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, L2TPV3, SESS_ID);
+
+ *l4_4_data = fltr->ip_data.l4_header;
+ } else {
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_fill_fdir_eth_hdr - fill the Ethernet protocol header
+ * @fltr: Flow Director filter data structure
+ * @proto_hdrs: Flow Director protocol headers data structure
+ *
+ * Returns 0 if the Ethernet protocol header is set successfully
+ */
+static int
+iavf_fill_fdir_eth_hdr(struct iavf_fdir_fltr *fltr,
+ struct virtchnl_proto_hdrs *proto_hdrs)
+{
+ struct virtchnl_proto_hdr *hdr = &proto_hdrs->proto_hdr[proto_hdrs->count++];
+ struct ethhdr *ehdr = (struct ethhdr *)hdr->buffer;
+
+ VIRTCHNL_SET_PROTO_HDR_TYPE(hdr, ETH);
+
+ if (fltr->eth_mask.etype == htons(U16_MAX)) {
+ if (fltr->eth_data.etype == htons(ETH_P_IP) ||
+ fltr->eth_data.etype == htons(ETH_P_IPV6))
+ return -EOPNOTSUPP;
+
+ ehdr->h_proto = fltr->eth_data.etype;
+ VIRTCHNL_ADD_PROTO_HDR_FIELD_BIT(hdr, ETH, ETHERTYPE);
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_fill_fdir_add_msg - fill the Flow Director filter into virtchnl message
+ * @adapter: pointer to the VF adapter structure
+ * @fltr: Flow Director filter data structure
+ *
+ * Returns 0 if the add Flow Director virtchnl message is filled successfully
+ */
+int iavf_fill_fdir_add_msg(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr)
+{
+ struct virtchnl_fdir_add *vc_msg = &fltr->vc_add_msg;
+ struct virtchnl_proto_hdrs *proto_hdrs;
+ int err;
+
+ proto_hdrs = &vc_msg->rule_cfg.proto_hdrs;
+
+ err = iavf_fill_fdir_eth_hdr(fltr, proto_hdrs); /* L2 always exists */
+ if (err)
+ return err;
+
+ switch (fltr->flow_type) {
+ case IAVF_FDIR_FLOW_IPV4_TCP:
+ err = iavf_fill_fdir_ip4_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_tcp_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_IPV4_UDP:
+ err = iavf_fill_fdir_ip4_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_udp_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_IPV4_SCTP:
+ err = iavf_fill_fdir_ip4_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_sctp_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_IPV4_AH:
+ err = iavf_fill_fdir_ip4_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_ah_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_IPV4_ESP:
+ err = iavf_fill_fdir_ip4_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_esp_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_IPV4_OTHER:
+ err = iavf_fill_fdir_ip4_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_l4_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_IPV6_TCP:
+ err = iavf_fill_fdir_ip6_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_tcp_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_IPV6_UDP:
+ err = iavf_fill_fdir_ip6_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_udp_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_IPV6_SCTP:
+ err = iavf_fill_fdir_ip6_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_sctp_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_IPV6_AH:
+ err = iavf_fill_fdir_ip6_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_ah_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_IPV6_ESP:
+ err = iavf_fill_fdir_ip6_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_esp_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_IPV6_OTHER:
+ err = iavf_fill_fdir_ip6_hdr(fltr, proto_hdrs) |
+ iavf_fill_fdir_l4_hdr(fltr, proto_hdrs);
+ break;
+ case IAVF_FDIR_FLOW_NON_IP_L2:
+ break;
+ default:
+ err = -EINVAL;
+ break;
+ }
+
+ if (err)
+ return err;
+
+ vc_msg->vsi_id = adapter->vsi.id;
+ vc_msg->rule_cfg.action_set.count = 1;
+ vc_msg->rule_cfg.action_set.actions[0].type = fltr->action;
+ vc_msg->rule_cfg.action_set.actions[0].act_conf.queue.index = fltr->q_index;
+
+ return 0;
+}
+
+/**
+ * iavf_fdir_flow_proto_name - get the flow protocol name
+ * @flow_type: Flow Director filter flow type
+ **/
+static const char *iavf_fdir_flow_proto_name(enum iavf_fdir_flow_type flow_type)
+{
+ switch (flow_type) {
+ case IAVF_FDIR_FLOW_IPV4_TCP:
+ case IAVF_FDIR_FLOW_IPV6_TCP:
+ return "TCP";
+ case IAVF_FDIR_FLOW_IPV4_UDP:
+ case IAVF_FDIR_FLOW_IPV6_UDP:
+ return "UDP";
+ case IAVF_FDIR_FLOW_IPV4_SCTP:
+ case IAVF_FDIR_FLOW_IPV6_SCTP:
+ return "SCTP";
+ case IAVF_FDIR_FLOW_IPV4_AH:
+ case IAVF_FDIR_FLOW_IPV6_AH:
+ return "AH";
+ case IAVF_FDIR_FLOW_IPV4_ESP:
+ case IAVF_FDIR_FLOW_IPV6_ESP:
+ return "ESP";
+ case IAVF_FDIR_FLOW_IPV4_OTHER:
+ case IAVF_FDIR_FLOW_IPV6_OTHER:
+ return "Other";
+ case IAVF_FDIR_FLOW_NON_IP_L2:
+ return "Ethernet";
+ default:
+ return NULL;
+ }
+}
+
+/**
+ * iavf_print_fdir_fltr
+ * @adapter: adapter structure
+ * @fltr: Flow Director filter to print
+ *
+ * Print the Flow Director filter
+ **/
+void iavf_print_fdir_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr)
+{
+ const char *proto = iavf_fdir_flow_proto_name(fltr->flow_type);
+
+ if (!proto)
+ return;
+
+ switch (fltr->flow_type) {
+ case IAVF_FDIR_FLOW_IPV4_TCP:
+ case IAVF_FDIR_FLOW_IPV4_UDP:
+ case IAVF_FDIR_FLOW_IPV4_SCTP:
+ dev_info(&adapter->pdev->dev, "Rule ID: %u dst_ip: %pI4 src_ip %pI4 %s: dst_port %hu src_port %hu\n",
+ fltr->loc,
+ &fltr->ip_data.v4_addrs.dst_ip,
+ &fltr->ip_data.v4_addrs.src_ip,
+ proto,
+ ntohs(fltr->ip_data.dst_port),
+ ntohs(fltr->ip_data.src_port));
+ break;
+ case IAVF_FDIR_FLOW_IPV4_AH:
+ case IAVF_FDIR_FLOW_IPV4_ESP:
+ dev_info(&adapter->pdev->dev, "Rule ID: %u dst_ip: %pI4 src_ip %pI4 %s: SPI %u\n",
+ fltr->loc,
+ &fltr->ip_data.v4_addrs.dst_ip,
+ &fltr->ip_data.v4_addrs.src_ip,
+ proto,
+ ntohl(fltr->ip_data.spi));
+ break;
+ case IAVF_FDIR_FLOW_IPV4_OTHER:
+ dev_info(&adapter->pdev->dev, "Rule ID: %u dst_ip: %pI4 src_ip %pI4 proto: %u L4_bytes: 0x%x\n",
+ fltr->loc,
+ &fltr->ip_data.v4_addrs.dst_ip,
+ &fltr->ip_data.v4_addrs.src_ip,
+ fltr->ip_data.proto,
+ ntohl(fltr->ip_data.l4_header));
+ break;
+ case IAVF_FDIR_FLOW_IPV6_TCP:
+ case IAVF_FDIR_FLOW_IPV6_UDP:
+ case IAVF_FDIR_FLOW_IPV6_SCTP:
+ dev_info(&adapter->pdev->dev, "Rule ID: %u dst_ip: %pI6 src_ip %pI6 %s: dst_port %hu src_port %hu\n",
+ fltr->loc,
+ &fltr->ip_data.v6_addrs.dst_ip,
+ &fltr->ip_data.v6_addrs.src_ip,
+ proto,
+ ntohs(fltr->ip_data.dst_port),
+ ntohs(fltr->ip_data.src_port));
+ break;
+ case IAVF_FDIR_FLOW_IPV6_AH:
+ case IAVF_FDIR_FLOW_IPV6_ESP:
+ dev_info(&adapter->pdev->dev, "Rule ID: %u dst_ip: %pI6 src_ip %pI6 %s: SPI %u\n",
+ fltr->loc,
+ &fltr->ip_data.v6_addrs.dst_ip,
+ &fltr->ip_data.v6_addrs.src_ip,
+ proto,
+ ntohl(fltr->ip_data.spi));
+ break;
+ case IAVF_FDIR_FLOW_IPV6_OTHER:
+ dev_info(&adapter->pdev->dev, "Rule ID: %u dst_ip: %pI6 src_ip %pI6 proto: %u L4_bytes: 0x%x\n",
+ fltr->loc,
+ &fltr->ip_data.v6_addrs.dst_ip,
+ &fltr->ip_data.v6_addrs.src_ip,
+ fltr->ip_data.proto,
+ ntohl(fltr->ip_data.l4_header));
+ break;
+ case IAVF_FDIR_FLOW_NON_IP_L2:
+ dev_info(&adapter->pdev->dev, "Rule ID: %u eth_type: 0x%x\n",
+ fltr->loc,
+ ntohs(fltr->eth_data.etype));
+ break;
+ default:
+ break;
+ }
+}
+
+/**
+ * iavf_fdir_is_dup_fltr - test if filter is already in list
+ * @adapter: pointer to the VF adapter structure
+ * @fltr: Flow Director filter data structure
+ *
+ * Returns true if the filter is found in the list
+ */
+bool iavf_fdir_is_dup_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr)
+{
+ struct iavf_fdir_fltr *tmp;
+ bool ret = false;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry(tmp, &adapter->fdir_list_head, list) {
+ if (tmp->flow_type != fltr->flow_type)
+ continue;
+
+ if (!memcmp(&tmp->eth_data, &fltr->eth_data,
+ sizeof(fltr->eth_data)) &&
+ !memcmp(&tmp->ip_data, &fltr->ip_data,
+ sizeof(fltr->ip_data)) &&
+ !memcmp(&tmp->ext_data, &fltr->ext_data,
+ sizeof(fltr->ext_data))) {
+ ret = true;
+ break;
+ }
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+
+ return ret;
+}
+
+/**
+ * iavf_find_fdir_fltr_by_loc - find filter with location
+ * @adapter: pointer to the VF adapter structure
+ * @loc: location to find.
+ *
+ * Returns pointer to Flow Director filter if found or null
+ */
+struct iavf_fdir_fltr *iavf_find_fdir_fltr_by_loc(struct iavf_adapter *adapter, u32 loc)
+{
+ struct iavf_fdir_fltr *rule;
+
+ list_for_each_entry(rule, &adapter->fdir_list_head, list)
+ if (rule->loc == loc)
+ return rule;
+
+ return NULL;
+}
+
+/**
+ * iavf_fdir_list_add_fltr - add a new node to the flow director filter list
+ * @adapter: pointer to the VF adapter structure
+ * @fltr: filter node to add to structure
+ */
+void iavf_fdir_list_add_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr)
+{
+ struct iavf_fdir_fltr *rule, *parent = NULL;
+
+ list_for_each_entry(rule, &adapter->fdir_list_head, list) {
+ if (rule->loc >= fltr->loc)
+ break;
+ parent = rule;
+ }
+
+ if (parent)
+ list_add(&fltr->list, &parent->list);
+ else
+ list_add(&fltr->list, &adapter->fdir_list_head);
+}
diff --git a/drivers/net/ethernet/intel/iavf/iavf_fdir.h b/drivers/net/ethernet/intel/iavf/iavf_fdir.h
new file mode 100644
index 0000000000..d31bd923ba
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_fdir.h
@@ -0,0 +1,133 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2021, Intel Corporation. */
+
+#ifndef _IAVF_FDIR_H_
+#define _IAVF_FDIR_H_
+
+struct iavf_adapter;
+
+/* State of Flow Director filter
+ *
+ * *_REQUEST states are used to mark filter to be sent to PF driver to perform
+ * an action (either add or delete filter). *_PENDING states are an indication
+ * that request was sent to PF and the driver is waiting for response.
+ *
+ * Both DELETE and DISABLE states are being used to delete a filter in PF.
+ * The difference is that after a successful response filter in DEL_PENDING
+ * state is being deleted from VF driver as well and filter in DIS_PENDING state
+ * is being changed to INACTIVE state.
+ */
+enum iavf_fdir_fltr_state_t {
+ IAVF_FDIR_FLTR_ADD_REQUEST, /* User requests to add filter */
+ IAVF_FDIR_FLTR_ADD_PENDING, /* Filter pending add by the PF */
+ IAVF_FDIR_FLTR_DEL_REQUEST, /* User requests to delete filter */
+ IAVF_FDIR_FLTR_DEL_PENDING, /* Filter pending delete by the PF */
+ IAVF_FDIR_FLTR_DIS_REQUEST, /* Filter scheduled to be disabled */
+ IAVF_FDIR_FLTR_DIS_PENDING, /* Filter pending disable by the PF */
+ IAVF_FDIR_FLTR_INACTIVE, /* Filter inactive on link down */
+ IAVF_FDIR_FLTR_ACTIVE, /* Filter is active */
+};
+
+enum iavf_fdir_flow_type {
+ /* NONE - used for undef/error */
+ IAVF_FDIR_FLOW_NONE = 0,
+ IAVF_FDIR_FLOW_IPV4_TCP,
+ IAVF_FDIR_FLOW_IPV4_UDP,
+ IAVF_FDIR_FLOW_IPV4_SCTP,
+ IAVF_FDIR_FLOW_IPV4_AH,
+ IAVF_FDIR_FLOW_IPV4_ESP,
+ IAVF_FDIR_FLOW_IPV4_OTHER,
+ IAVF_FDIR_FLOW_IPV6_TCP,
+ IAVF_FDIR_FLOW_IPV6_UDP,
+ IAVF_FDIR_FLOW_IPV6_SCTP,
+ IAVF_FDIR_FLOW_IPV6_AH,
+ IAVF_FDIR_FLOW_IPV6_ESP,
+ IAVF_FDIR_FLOW_IPV6_OTHER,
+ IAVF_FDIR_FLOW_NON_IP_L2,
+ /* MAX - this must be last and add anything new just above it */
+ IAVF_FDIR_FLOW_PTYPE_MAX,
+};
+
+/* Must not exceed the array element number of '__be32 data[2]' in the ethtool
+ * 'struct ethtool_rx_flow_spec.m_ext.data[2]' to express the flex-byte (word).
+ */
+#define IAVF_FLEX_WORD_NUM 2
+
+struct iavf_flex_word {
+ u16 offset;
+ u16 word;
+};
+
+struct iavf_ipv4_addrs {
+ __be32 src_ip;
+ __be32 dst_ip;
+};
+
+struct iavf_ipv6_addrs {
+ struct in6_addr src_ip;
+ struct in6_addr dst_ip;
+};
+
+struct iavf_fdir_eth {
+ __be16 etype;
+};
+
+struct iavf_fdir_ip {
+ union {
+ struct iavf_ipv4_addrs v4_addrs;
+ struct iavf_ipv6_addrs v6_addrs;
+ };
+ __be16 src_port;
+ __be16 dst_port;
+ __be32 l4_header; /* first 4 bytes of the layer 4 header */
+ __be32 spi; /* security parameter index for AH/ESP */
+ union {
+ u8 tos;
+ u8 tclass;
+ };
+ u8 proto;
+};
+
+struct iavf_fdir_extra {
+ u32 usr_def[IAVF_FLEX_WORD_NUM];
+};
+
+/* bookkeeping of Flow Director filters */
+struct iavf_fdir_fltr {
+ enum iavf_fdir_fltr_state_t state;
+ struct list_head list;
+
+ enum iavf_fdir_flow_type flow_type;
+
+ struct iavf_fdir_eth eth_data;
+ struct iavf_fdir_eth eth_mask;
+
+ struct iavf_fdir_ip ip_data;
+ struct iavf_fdir_ip ip_mask;
+
+ struct iavf_fdir_extra ext_data;
+ struct iavf_fdir_extra ext_mask;
+
+ enum virtchnl_action action;
+
+ /* flex byte filter data */
+ u8 ip_ver; /* used to adjust the flex offset, 4 : IPv4, 6 : IPv6 */
+ u8 flex_cnt;
+ struct iavf_flex_word flex_words[IAVF_FLEX_WORD_NUM];
+
+ u32 flow_id;
+
+ u32 loc; /* Rule location inside the flow table */
+ u32 q_index;
+
+ struct virtchnl_fdir_add vc_add_msg;
+};
+
+int iavf_validate_fdir_fltr_masks(struct iavf_adapter *adapter,
+ struct iavf_fdir_fltr *fltr);
+int iavf_fill_fdir_add_msg(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr);
+void iavf_print_fdir_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr);
+bool iavf_fdir_is_dup_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr);
+void iavf_fdir_list_add_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr);
+struct iavf_fdir_fltr *iavf_find_fdir_fltr_by_loc(struct iavf_adapter *adapter, u32 loc);
+#endif /* _IAVF_FDIR_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c
new file mode 100644
index 0000000000..257865647c
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_main.c
@@ -0,0 +1,5323 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#include "iavf.h"
+#include "iavf_prototype.h"
+#include "iavf_client.h"
+/* All iavf tracepoints are defined by the include below, which must
+ * be included exactly once across the whole kernel with
+ * CREATE_TRACE_POINTS defined
+ */
+#define CREATE_TRACE_POINTS
+#include "iavf_trace.h"
+
+static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);
+static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);
+static int iavf_close(struct net_device *netdev);
+static void iavf_init_get_resources(struct iavf_adapter *adapter);
+static int iavf_check_reset_complete(struct iavf_hw *hw);
+
+char iavf_driver_name[] = "iavf";
+static const char iavf_driver_string[] =
+ "Intel(R) Ethernet Adaptive Virtual Function Network Driver";
+
+static const char iavf_copyright[] =
+ "Copyright (c) 2013 - 2018 Intel Corporation.";
+
+/* iavf_pci_tbl - PCI Device ID Table
+ *
+ * Wildcard entries (PCI_ANY_ID) should come last
+ * Last entry must be all 0s
+ *
+ * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
+ * Class, Class Mask, private data (not used) }
+ */
+static const struct pci_device_id iavf_pci_tbl[] = {
+ {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0},
+ {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0},
+ {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0},
+ {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0},
+ /* required last entry */
+ {0, }
+};
+
+MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);
+
+MODULE_ALIAS("i40evf");
+MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
+MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver");
+MODULE_LICENSE("GPL v2");
+
+static const struct net_device_ops iavf_netdev_ops;
+
+int iavf_status_to_errno(enum iavf_status status)
+{
+ switch (status) {
+ case IAVF_SUCCESS:
+ return 0;
+ case IAVF_ERR_PARAM:
+ case IAVF_ERR_MAC_TYPE:
+ case IAVF_ERR_INVALID_MAC_ADDR:
+ case IAVF_ERR_INVALID_LINK_SETTINGS:
+ case IAVF_ERR_INVALID_PD_ID:
+ case IAVF_ERR_INVALID_QP_ID:
+ case IAVF_ERR_INVALID_CQ_ID:
+ case IAVF_ERR_INVALID_CEQ_ID:
+ case IAVF_ERR_INVALID_AEQ_ID:
+ case IAVF_ERR_INVALID_SIZE:
+ case IAVF_ERR_INVALID_ARP_INDEX:
+ case IAVF_ERR_INVALID_FPM_FUNC_ID:
+ case IAVF_ERR_QP_INVALID_MSG_SIZE:
+ case IAVF_ERR_INVALID_FRAG_COUNT:
+ case IAVF_ERR_INVALID_ALIGNMENT:
+ case IAVF_ERR_INVALID_PUSH_PAGE_INDEX:
+ case IAVF_ERR_INVALID_IMM_DATA_SIZE:
+ case IAVF_ERR_INVALID_VF_ID:
+ case IAVF_ERR_INVALID_HMCFN_ID:
+ case IAVF_ERR_INVALID_PBLE_INDEX:
+ case IAVF_ERR_INVALID_SD_INDEX:
+ case IAVF_ERR_INVALID_PAGE_DESC_INDEX:
+ case IAVF_ERR_INVALID_SD_TYPE:
+ case IAVF_ERR_INVALID_HMC_OBJ_INDEX:
+ case IAVF_ERR_INVALID_HMC_OBJ_COUNT:
+ case IAVF_ERR_INVALID_SRQ_ARM_LIMIT:
+ return -EINVAL;
+ case IAVF_ERR_NVM:
+ case IAVF_ERR_NVM_CHECKSUM:
+ case IAVF_ERR_PHY:
+ case IAVF_ERR_CONFIG:
+ case IAVF_ERR_UNKNOWN_PHY:
+ case IAVF_ERR_LINK_SETUP:
+ case IAVF_ERR_ADAPTER_STOPPED:
+ case IAVF_ERR_PRIMARY_REQUESTS_PENDING:
+ case IAVF_ERR_AUTONEG_NOT_COMPLETE:
+ case IAVF_ERR_RESET_FAILED:
+ case IAVF_ERR_BAD_PTR:
+ case IAVF_ERR_SWFW_SYNC:
+ case IAVF_ERR_QP_TOOMANY_WRS_POSTED:
+ case IAVF_ERR_QUEUE_EMPTY:
+ case IAVF_ERR_FLUSHED_QUEUE:
+ case IAVF_ERR_OPCODE_MISMATCH:
+ case IAVF_ERR_CQP_COMPL_ERROR:
+ case IAVF_ERR_BACKING_PAGE_ERROR:
+ case IAVF_ERR_NO_PBLCHUNKS_AVAILABLE:
+ case IAVF_ERR_MEMCPY_FAILED:
+ case IAVF_ERR_SRQ_ENABLED:
+ case IAVF_ERR_ADMIN_QUEUE_ERROR:
+ case IAVF_ERR_ADMIN_QUEUE_FULL:
+ case IAVF_ERR_BAD_RDMA_CQE:
+ case IAVF_ERR_NVM_BLANK_MODE:
+ case IAVF_ERR_PE_DOORBELL_NOT_ENABLED:
+ case IAVF_ERR_DIAG_TEST_FAILED:
+ case IAVF_ERR_FIRMWARE_API_VERSION:
+ case IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR:
+ return -EIO;
+ case IAVF_ERR_DEVICE_NOT_SUPPORTED:
+ return -ENODEV;
+ case IAVF_ERR_NO_AVAILABLE_VSI:
+ case IAVF_ERR_RING_FULL:
+ return -ENOSPC;
+ case IAVF_ERR_NO_MEMORY:
+ return -ENOMEM;
+ case IAVF_ERR_TIMEOUT:
+ case IAVF_ERR_ADMIN_QUEUE_TIMEOUT:
+ return -ETIMEDOUT;
+ case IAVF_ERR_NOT_IMPLEMENTED:
+ case IAVF_NOT_SUPPORTED:
+ return -EOPNOTSUPP;
+ case IAVF_ERR_ADMIN_QUEUE_NO_WORK:
+ return -EALREADY;
+ case IAVF_ERR_NOT_READY:
+ return -EBUSY;
+ case IAVF_ERR_BUF_TOO_SHORT:
+ return -EMSGSIZE;
+ }
+
+ return -EIO;
+}
+
+int virtchnl_status_to_errno(enum virtchnl_status_code v_status)
+{
+ switch (v_status) {
+ case VIRTCHNL_STATUS_SUCCESS:
+ return 0;
+ case VIRTCHNL_STATUS_ERR_PARAM:
+ case VIRTCHNL_STATUS_ERR_INVALID_VF_ID:
+ return -EINVAL;
+ case VIRTCHNL_STATUS_ERR_NO_MEMORY:
+ return -ENOMEM;
+ case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH:
+ case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR:
+ case VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR:
+ return -EIO;
+ case VIRTCHNL_STATUS_ERR_NOT_SUPPORTED:
+ return -EOPNOTSUPP;
+ }
+
+ return -EIO;
+}
+
+/**
+ * iavf_pdev_to_adapter - go from pci_dev to adapter
+ * @pdev: pci_dev pointer
+ */
+static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev)
+{
+ return netdev_priv(pci_get_drvdata(pdev));
+}
+
+/**
+ * iavf_is_reset_in_progress - Check if a reset is in progress
+ * @adapter: board private structure
+ */
+static bool iavf_is_reset_in_progress(struct iavf_adapter *adapter)
+{
+ if (adapter->state == __IAVF_RESETTING ||
+ adapter->flags & (IAVF_FLAG_RESET_PENDING |
+ IAVF_FLAG_RESET_NEEDED))
+ return true;
+
+ return false;
+}
+
+/**
+ * iavf_wait_for_reset - Wait for reset to finish.
+ * @adapter: board private structure
+ *
+ * Returns 0 if reset finished successfully, negative on timeout or interrupt.
+ */
+int iavf_wait_for_reset(struct iavf_adapter *adapter)
+{
+ int ret = wait_event_interruptible_timeout(adapter->reset_waitqueue,
+ !iavf_is_reset_in_progress(adapter),
+ msecs_to_jiffies(5000));
+
+ /* If ret < 0 then it means wait was interrupted.
+ * If ret == 0 then it means we got a timeout while waiting
+ * for reset to finish.
+ * If ret > 0 it means reset has finished.
+ */
+ if (ret > 0)
+ return 0;
+ else if (ret < 0)
+ return -EINTR;
+ else
+ return -EBUSY;
+}
+
+/**
+ * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code
+ * @hw: pointer to the HW structure
+ * @mem: ptr to mem struct to fill out
+ * @size: size of memory requested
+ * @alignment: what to align the allocation to
+ **/
+enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw,
+ struct iavf_dma_mem *mem,
+ u64 size, u32 alignment)
+{
+ struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
+
+ if (!mem)
+ return IAVF_ERR_PARAM;
+
+ mem->size = ALIGN(size, alignment);
+ mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
+ (dma_addr_t *)&mem->pa, GFP_KERNEL);
+ if (mem->va)
+ return 0;
+ else
+ return IAVF_ERR_NO_MEMORY;
+}
+
+/**
+ * iavf_free_dma_mem - wrapper for DMA memory freeing
+ * @hw: pointer to the HW structure
+ * @mem: ptr to mem struct to free
+ **/
+enum iavf_status iavf_free_dma_mem(struct iavf_hw *hw, struct iavf_dma_mem *mem)
+{
+ struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
+
+ if (!mem || !mem->va)
+ return IAVF_ERR_PARAM;
+ dma_free_coherent(&adapter->pdev->dev, mem->size,
+ mem->va, (dma_addr_t)mem->pa);
+ return 0;
+}
+
+/**
+ * iavf_allocate_virt_mem - virt memory alloc wrapper
+ * @hw: pointer to the HW structure
+ * @mem: ptr to mem struct to fill out
+ * @size: size of memory requested
+ **/
+enum iavf_status iavf_allocate_virt_mem(struct iavf_hw *hw,
+ struct iavf_virt_mem *mem, u32 size)
+{
+ if (!mem)
+ return IAVF_ERR_PARAM;
+
+ mem->size = size;
+ mem->va = kzalloc(size, GFP_KERNEL);
+
+ if (mem->va)
+ return 0;
+ else
+ return IAVF_ERR_NO_MEMORY;
+}
+
+/**
+ * iavf_free_virt_mem - virt memory free wrapper
+ * @hw: pointer to the HW structure
+ * @mem: ptr to mem struct to free
+ **/
+void iavf_free_virt_mem(struct iavf_hw *hw, struct iavf_virt_mem *mem)
+{
+ kfree(mem->va);
+}
+
+/**
+ * iavf_schedule_reset - Set the flags and schedule a reset event
+ * @adapter: board private structure
+ * @flags: IAVF_FLAG_RESET_PENDING or IAVF_FLAG_RESET_NEEDED
+ **/
+void iavf_schedule_reset(struct iavf_adapter *adapter, u64 flags)
+{
+ if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section) &&
+ !(adapter->flags &
+ (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) {
+ adapter->flags |= flags;
+ queue_work(adapter->wq, &adapter->reset_task);
+ }
+}
+
+/**
+ * iavf_schedule_aq_request - Set the flags and schedule aq request
+ * @adapter: board private structure
+ * @flags: requested aq flags
+ **/
+void iavf_schedule_aq_request(struct iavf_adapter *adapter, u64 flags)
+{
+ adapter->aq_required |= flags;
+ mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
+}
+
+/**
+ * iavf_tx_timeout - Respond to a Tx Hang
+ * @netdev: network interface device structure
+ * @txqueue: queue number that is timing out
+ **/
+static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ adapter->tx_timeout_count++;
+ iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
+}
+
+/**
+ * iavf_misc_irq_disable - Mask off interrupt generation on the NIC
+ * @adapter: board private structure
+ **/
+static void iavf_misc_irq_disable(struct iavf_adapter *adapter)
+{
+ struct iavf_hw *hw = &adapter->hw;
+
+ if (!adapter->msix_entries)
+ return;
+
+ wr32(hw, IAVF_VFINT_DYN_CTL01, 0);
+
+ iavf_flush(hw);
+
+ synchronize_irq(adapter->msix_entries[0].vector);
+}
+
+/**
+ * iavf_misc_irq_enable - Enable default interrupt generation settings
+ * @adapter: board private structure
+ **/
+static void iavf_misc_irq_enable(struct iavf_adapter *adapter)
+{
+ struct iavf_hw *hw = &adapter->hw;
+
+ wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK |
+ IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
+ wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
+
+ iavf_flush(hw);
+}
+
+/**
+ * iavf_irq_disable - Mask off interrupt generation on the NIC
+ * @adapter: board private structure
+ **/
+static void iavf_irq_disable(struct iavf_adapter *adapter)
+{
+ int i;
+ struct iavf_hw *hw = &adapter->hw;
+
+ if (!adapter->msix_entries)
+ return;
+
+ for (i = 1; i < adapter->num_msix_vectors; i++) {
+ wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0);
+ synchronize_irq(adapter->msix_entries[i].vector);
+ }
+ iavf_flush(hw);
+}
+
+/**
+ * iavf_irq_enable_queues - Enable interrupt for all queues
+ * @adapter: board private structure
+ **/
+static void iavf_irq_enable_queues(struct iavf_adapter *adapter)
+{
+ struct iavf_hw *hw = &adapter->hw;
+ int i;
+
+ for (i = 1; i < adapter->num_msix_vectors; i++) {
+ wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1),
+ IAVF_VFINT_DYN_CTLN1_INTENA_MASK |
+ IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK);
+ }
+}
+
+/**
+ * iavf_irq_enable - Enable default interrupt generation settings
+ * @adapter: board private structure
+ * @flush: boolean value whether to run rd32()
+ **/
+void iavf_irq_enable(struct iavf_adapter *adapter, bool flush)
+{
+ struct iavf_hw *hw = &adapter->hw;
+
+ iavf_misc_irq_enable(adapter);
+ iavf_irq_enable_queues(adapter);
+
+ if (flush)
+ iavf_flush(hw);
+}
+
+/**
+ * iavf_msix_aq - Interrupt handler for vector 0
+ * @irq: interrupt number
+ * @data: pointer to netdev
+ **/
+static irqreturn_t iavf_msix_aq(int irq, void *data)
+{
+ struct net_device *netdev = data;
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ struct iavf_hw *hw = &adapter->hw;
+
+ /* handle non-queue interrupts, these reads clear the registers */
+ rd32(hw, IAVF_VFINT_ICR01);
+ rd32(hw, IAVF_VFINT_ICR0_ENA1);
+
+ if (adapter->state != __IAVF_REMOVE)
+ /* schedule work on the private workqueue */
+ queue_work(adapter->wq, &adapter->adminq_task);
+
+ return IRQ_HANDLED;
+}
+
+/**
+ * iavf_msix_clean_rings - MSIX mode Interrupt Handler
+ * @irq: interrupt number
+ * @data: pointer to a q_vector
+ **/
+static irqreturn_t iavf_msix_clean_rings(int irq, void *data)
+{
+ struct iavf_q_vector *q_vector = data;
+
+ if (!q_vector->tx.ring && !q_vector->rx.ring)
+ return IRQ_HANDLED;
+
+ napi_schedule_irqoff(&q_vector->napi);
+
+ return IRQ_HANDLED;
+}
+
+/**
+ * iavf_map_vector_to_rxq - associate irqs with rx queues
+ * @adapter: board private structure
+ * @v_idx: interrupt number
+ * @r_idx: queue number
+ **/
+static void
+iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx)
+{
+ struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
+ struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx];
+ struct iavf_hw *hw = &adapter->hw;
+
+ rx_ring->q_vector = q_vector;
+ rx_ring->next = q_vector->rx.ring;
+ rx_ring->vsi = &adapter->vsi;
+ q_vector->rx.ring = rx_ring;
+ q_vector->rx.count++;
+ q_vector->rx.next_update = jiffies + 1;
+ q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
+ q_vector->ring_mask |= BIT(r_idx);
+ wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx),
+ q_vector->rx.current_itr >> 1);
+ q_vector->rx.current_itr = q_vector->rx.target_itr;
+}
+
+/**
+ * iavf_map_vector_to_txq - associate irqs with tx queues
+ * @adapter: board private structure
+ * @v_idx: interrupt number
+ * @t_idx: queue number
+ **/
+static void
+iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx)
+{
+ struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
+ struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx];
+ struct iavf_hw *hw = &adapter->hw;
+
+ tx_ring->q_vector = q_vector;
+ tx_ring->next = q_vector->tx.ring;
+ tx_ring->vsi = &adapter->vsi;
+ q_vector->tx.ring = tx_ring;
+ q_vector->tx.count++;
+ q_vector->tx.next_update = jiffies + 1;
+ q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
+ q_vector->num_ringpairs++;
+ wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx),
+ q_vector->tx.target_itr >> 1);
+ q_vector->tx.current_itr = q_vector->tx.target_itr;
+}
+
+/**
+ * iavf_map_rings_to_vectors - Maps descriptor rings to vectors
+ * @adapter: board private structure to initialize
+ *
+ * This function maps descriptor rings to the queue-specific vectors
+ * we were allotted through the MSI-X enabling code. Ideally, we'd have
+ * one vector per ring/queue, but on a constrained vector budget, we
+ * group the rings as "efficiently" as possible. You would add new
+ * mapping configurations in here.
+ **/
+static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter)
+{
+ int rings_remaining = adapter->num_active_queues;
+ int ridx = 0, vidx = 0;
+ int q_vectors;
+
+ q_vectors = adapter->num_msix_vectors - NONQ_VECS;
+
+ for (; ridx < rings_remaining; ridx++) {
+ iavf_map_vector_to_rxq(adapter, vidx, ridx);
+ iavf_map_vector_to_txq(adapter, vidx, ridx);
+
+ /* In the case where we have more queues than vectors, continue
+ * round-robin on vectors until all queues are mapped.
+ */
+ if (++vidx >= q_vectors)
+ vidx = 0;
+ }
+
+ adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
+}
+
+/**
+ * iavf_irq_affinity_notify - Callback for affinity changes
+ * @notify: context as to what irq was changed
+ * @mask: the new affinity mask
+ *
+ * This is a callback function used by the irq_set_affinity_notifier function
+ * so that we may register to receive changes to the irq affinity masks.
+ **/
+static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify,
+ const cpumask_t *mask)
+{
+ struct iavf_q_vector *q_vector =
+ container_of(notify, struct iavf_q_vector, affinity_notify);
+
+ cpumask_copy(&q_vector->affinity_mask, mask);
+}
+
+/**
+ * iavf_irq_affinity_release - Callback for affinity notifier release
+ * @ref: internal core kernel usage
+ *
+ * This is a callback function used by the irq_set_affinity_notifier function
+ * to inform the current notification subscriber that they will no longer
+ * receive notifications.
+ **/
+static void iavf_irq_affinity_release(struct kref *ref) {}
+
+/**
+ * iavf_request_traffic_irqs - Initialize MSI-X interrupts
+ * @adapter: board private structure
+ * @basename: device basename
+ *
+ * Allocates MSI-X vectors for tx and rx handling, and requests
+ * interrupts from the kernel.
+ **/
+static int
+iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename)
+{
+ unsigned int vector, q_vectors;
+ unsigned int rx_int_idx = 0, tx_int_idx = 0;
+ int irq_num, err;
+ int cpu;
+
+ iavf_irq_disable(adapter);
+ /* Decrement for Other and TCP Timer vectors */
+ q_vectors = adapter->num_msix_vectors - NONQ_VECS;
+
+ for (vector = 0; vector < q_vectors; vector++) {
+ struct iavf_q_vector *q_vector = &adapter->q_vectors[vector];
+
+ irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
+
+ if (q_vector->tx.ring && q_vector->rx.ring) {
+ snprintf(q_vector->name, sizeof(q_vector->name),
+ "iavf-%s-TxRx-%u", basename, rx_int_idx++);
+ tx_int_idx++;
+ } else if (q_vector->rx.ring) {
+ snprintf(q_vector->name, sizeof(q_vector->name),
+ "iavf-%s-rx-%u", basename, rx_int_idx++);
+ } else if (q_vector->tx.ring) {
+ snprintf(q_vector->name, sizeof(q_vector->name),
+ "iavf-%s-tx-%u", basename, tx_int_idx++);
+ } else {
+ /* skip this unused q_vector */
+ continue;
+ }
+ err = request_irq(irq_num,
+ iavf_msix_clean_rings,
+ 0,
+ q_vector->name,
+ q_vector);
+ if (err) {
+ dev_info(&adapter->pdev->dev,
+ "Request_irq failed, error: %d\n", err);
+ goto free_queue_irqs;
+ }
+ /* register for affinity change notifications */
+ q_vector->affinity_notify.notify = iavf_irq_affinity_notify;
+ q_vector->affinity_notify.release =
+ iavf_irq_affinity_release;
+ irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
+ /* Spread the IRQ affinity hints across online CPUs. Note that
+ * get_cpu_mask returns a mask with a permanent lifetime so
+ * it's safe to use as a hint for irq_update_affinity_hint.
+ */
+ cpu = cpumask_local_spread(q_vector->v_idx, -1);
+ irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
+ }
+
+ return 0;
+
+free_queue_irqs:
+ while (vector) {
+ vector--;
+ irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
+ irq_set_affinity_notifier(irq_num, NULL);
+ irq_update_affinity_hint(irq_num, NULL);
+ free_irq(irq_num, &adapter->q_vectors[vector]);
+ }
+ return err;
+}
+
+/**
+ * iavf_request_misc_irq - Initialize MSI-X interrupts
+ * @adapter: board private structure
+ *
+ * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
+ * vector is only for the admin queue, and stays active even when the netdev
+ * is closed.
+ **/
+static int iavf_request_misc_irq(struct iavf_adapter *adapter)
+{
+ struct net_device *netdev = adapter->netdev;
+ int err;
+
+ snprintf(adapter->misc_vector_name,
+ sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx",
+ dev_name(&adapter->pdev->dev));
+ err = request_irq(adapter->msix_entries[0].vector,
+ &iavf_msix_aq, 0,
+ adapter->misc_vector_name, netdev);
+ if (err) {
+ dev_err(&adapter->pdev->dev,
+ "request_irq for %s failed: %d\n",
+ adapter->misc_vector_name, err);
+ free_irq(adapter->msix_entries[0].vector, netdev);
+ }
+ return err;
+}
+
+/**
+ * iavf_free_traffic_irqs - Free MSI-X interrupts
+ * @adapter: board private structure
+ *
+ * Frees all MSI-X vectors other than 0.
+ **/
+static void iavf_free_traffic_irqs(struct iavf_adapter *adapter)
+{
+ int vector, irq_num, q_vectors;
+
+ if (!adapter->msix_entries)
+ return;
+
+ q_vectors = adapter->num_msix_vectors - NONQ_VECS;
+
+ for (vector = 0; vector < q_vectors; vector++) {
+ irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
+ irq_set_affinity_notifier(irq_num, NULL);
+ irq_update_affinity_hint(irq_num, NULL);
+ free_irq(irq_num, &adapter->q_vectors[vector]);
+ }
+}
+
+/**
+ * iavf_free_misc_irq - Free MSI-X miscellaneous vector
+ * @adapter: board private structure
+ *
+ * Frees MSI-X vector 0.
+ **/
+static void iavf_free_misc_irq(struct iavf_adapter *adapter)
+{
+ struct net_device *netdev = adapter->netdev;
+
+ if (!adapter->msix_entries)
+ return;
+
+ free_irq(adapter->msix_entries[0].vector, netdev);
+}
+
+/**
+ * iavf_configure_tx - Configure Transmit Unit after Reset
+ * @adapter: board private structure
+ *
+ * Configure the Tx unit of the MAC after a reset.
+ **/
+static void iavf_configure_tx(struct iavf_adapter *adapter)
+{
+ struct iavf_hw *hw = &adapter->hw;
+ int i;
+
+ for (i = 0; i < adapter->num_active_queues; i++)
+ adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i);
+}
+
+/**
+ * iavf_configure_rx - Configure Receive Unit after Reset
+ * @adapter: board private structure
+ *
+ * Configure the Rx unit of the MAC after a reset.
+ **/
+static void iavf_configure_rx(struct iavf_adapter *adapter)
+{
+ unsigned int rx_buf_len = IAVF_RXBUFFER_2048;
+ struct iavf_hw *hw = &adapter->hw;
+ int i;
+
+ /* Legacy Rx will always default to a 2048 buffer size. */
+#if (PAGE_SIZE < 8192)
+ if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) {
+ struct net_device *netdev = adapter->netdev;
+
+ /* For jumbo frames on systems with 4K pages we have to use
+ * an order 1 page, so we might as well increase the size
+ * of our Rx buffer to make better use of the available space
+ */
+ rx_buf_len = IAVF_RXBUFFER_3072;
+
+ /* We use a 1536 buffer size for configurations with
+ * standard Ethernet mtu. On x86 this gives us enough room
+ * for shared info and 192 bytes of padding.
+ */
+ if (!IAVF_2K_TOO_SMALL_WITH_PADDING &&
+ (netdev->mtu <= ETH_DATA_LEN))
+ rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN;
+ }
+#endif
+
+ for (i = 0; i < adapter->num_active_queues; i++) {
+ adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i);
+ adapter->rx_rings[i].rx_buf_len = rx_buf_len;
+
+ if (adapter->flags & IAVF_FLAG_LEGACY_RX)
+ clear_ring_build_skb_enabled(&adapter->rx_rings[i]);
+ else
+ set_ring_build_skb_enabled(&adapter->rx_rings[i]);
+ }
+}
+
+/**
+ * iavf_find_vlan - Search filter list for specific vlan filter
+ * @adapter: board private structure
+ * @vlan: vlan tag
+ *
+ * Returns ptr to the filter object or NULL. Must be called while holding the
+ * mac_vlan_list_lock.
+ **/
+static struct
+iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter,
+ struct iavf_vlan vlan)
+{
+ struct iavf_vlan_filter *f;
+
+ list_for_each_entry(f, &adapter->vlan_filter_list, list) {
+ if (f->vlan.vid == vlan.vid &&
+ f->vlan.tpid == vlan.tpid)
+ return f;
+ }
+
+ return NULL;
+}
+
+/**
+ * iavf_add_vlan - Add a vlan filter to the list
+ * @adapter: board private structure
+ * @vlan: VLAN tag
+ *
+ * Returns ptr to the filter object or NULL when no memory available.
+ **/
+static struct
+iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter,
+ struct iavf_vlan vlan)
+{
+ struct iavf_vlan_filter *f = NULL;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ f = iavf_find_vlan(adapter, vlan);
+ if (!f) {
+ f = kzalloc(sizeof(*f), GFP_ATOMIC);
+ if (!f)
+ goto clearout;
+
+ f->vlan = vlan;
+
+ list_add_tail(&f->list, &adapter->vlan_filter_list);
+ f->state = IAVF_VLAN_ADD;
+ adapter->num_vlan_filters++;
+ iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER);
+ }
+
+clearout:
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return f;
+}
+
+/**
+ * iavf_del_vlan - Remove a vlan filter from the list
+ * @adapter: board private structure
+ * @vlan: VLAN tag
+ **/
+static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan)
+{
+ struct iavf_vlan_filter *f;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ f = iavf_find_vlan(adapter, vlan);
+ if (f) {
+ f->state = IAVF_VLAN_REMOVE;
+ iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DEL_VLAN_FILTER);
+ }
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+}
+
+/**
+ * iavf_restore_filters
+ * @adapter: board private structure
+ *
+ * Restore existing non MAC filters when VF netdev comes back up
+ **/
+static void iavf_restore_filters(struct iavf_adapter *adapter)
+{
+ struct iavf_vlan_filter *f;
+
+ /* re-add all VLAN filters */
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ list_for_each_entry(f, &adapter->vlan_filter_list, list) {
+ if (f->state == IAVF_VLAN_INACTIVE)
+ f->state = IAVF_VLAN_ADD;
+ }
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
+}
+
+/**
+ * iavf_get_num_vlans_added - get number of VLANs added
+ * @adapter: board private structure
+ */
+u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter)
+{
+ return adapter->num_vlan_filters;
+}
+
+/**
+ * iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF
+ * @adapter: board private structure
+ *
+ * This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN,
+ * do not impose a limit as that maintains current behavior and for
+ * VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF.
+ **/
+static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter)
+{
+ /* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has
+ * never been a limit on the VF driver side
+ */
+ if (VLAN_ALLOWED(adapter))
+ return VLAN_N_VID;
+ else if (VLAN_V2_ALLOWED(adapter))
+ return adapter->vlan_v2_caps.filtering.max_filters;
+
+ return 0;
+}
+
+/**
+ * iavf_max_vlans_added - check if maximum VLANs allowed already exist
+ * @adapter: board private structure
+ **/
+static bool iavf_max_vlans_added(struct iavf_adapter *adapter)
+{
+ if (iavf_get_num_vlans_added(adapter) <
+ iavf_get_max_vlans_allowed(adapter))
+ return false;
+
+ return true;
+}
+
+/**
+ * iavf_vlan_rx_add_vid - Add a VLAN filter to a device
+ * @netdev: network device struct
+ * @proto: unused protocol data
+ * @vid: VLAN tag
+ **/
+static int iavf_vlan_rx_add_vid(struct net_device *netdev,
+ __always_unused __be16 proto, u16 vid)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ /* Do not track VLAN 0 filter, always added by the PF on VF init */
+ if (!vid)
+ return 0;
+
+ if (!VLAN_FILTERING_ALLOWED(adapter))
+ return -EIO;
+
+ if (iavf_max_vlans_added(adapter)) {
+ netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n",
+ iavf_get_max_vlans_allowed(adapter));
+ return -EIO;
+ }
+
+ if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto))))
+ return -ENOMEM;
+
+ return 0;
+}
+
+/**
+ * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device
+ * @netdev: network device struct
+ * @proto: unused protocol data
+ * @vid: VLAN tag
+ **/
+static int iavf_vlan_rx_kill_vid(struct net_device *netdev,
+ __always_unused __be16 proto, u16 vid)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ /* We do not track VLAN 0 filter */
+ if (!vid)
+ return 0;
+
+ iavf_del_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)));
+ return 0;
+}
+
+/**
+ * iavf_find_filter - Search filter list for specific mac filter
+ * @adapter: board private structure
+ * @macaddr: the MAC address
+ *
+ * Returns ptr to the filter object or NULL. Must be called while holding the
+ * mac_vlan_list_lock.
+ **/
+static struct
+iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter,
+ const u8 *macaddr)
+{
+ struct iavf_mac_filter *f;
+
+ if (!macaddr)
+ return NULL;
+
+ list_for_each_entry(f, &adapter->mac_filter_list, list) {
+ if (ether_addr_equal(macaddr, f->macaddr))
+ return f;
+ }
+ return NULL;
+}
+
+/**
+ * iavf_add_filter - Add a mac filter to the filter list
+ * @adapter: board private structure
+ * @macaddr: the MAC address
+ *
+ * Returns ptr to the filter object or NULL when no memory available.
+ **/
+struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
+ const u8 *macaddr)
+{
+ struct iavf_mac_filter *f;
+
+ if (!macaddr)
+ return NULL;
+
+ f = iavf_find_filter(adapter, macaddr);
+ if (!f) {
+ f = kzalloc(sizeof(*f), GFP_ATOMIC);
+ if (!f)
+ return f;
+
+ ether_addr_copy(f->macaddr, macaddr);
+
+ list_add_tail(&f->list, &adapter->mac_filter_list);
+ f->add = true;
+ f->add_handled = false;
+ f->is_new_mac = true;
+ f->is_primary = ether_addr_equal(macaddr, adapter->hw.mac.addr);
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
+ } else {
+ f->remove = false;
+ }
+
+ return f;
+}
+
+/**
+ * iavf_replace_primary_mac - Replace current primary address
+ * @adapter: board private structure
+ * @new_mac: new MAC address to be applied
+ *
+ * Replace current dev_addr and send request to PF for removal of previous
+ * primary MAC address filter and addition of new primary MAC filter.
+ * Return 0 for success, -ENOMEM for failure.
+ *
+ * Do not call this with mac_vlan_list_lock!
+ **/
+static int iavf_replace_primary_mac(struct iavf_adapter *adapter,
+ const u8 *new_mac)
+{
+ struct iavf_hw *hw = &adapter->hw;
+ struct iavf_mac_filter *new_f;
+ struct iavf_mac_filter *old_f;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ new_f = iavf_add_filter(adapter, new_mac);
+ if (!new_f) {
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return -ENOMEM;
+ }
+
+ old_f = iavf_find_filter(adapter, hw->mac.addr);
+ if (old_f) {
+ old_f->is_primary = false;
+ old_f->remove = true;
+ adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
+ }
+ /* Always send the request to add if changing primary MAC,
+ * even if filter is already present on the list
+ */
+ new_f->is_primary = true;
+ new_f->add = true;
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
+ ether_addr_copy(hw->mac.addr, new_mac);
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ /* schedule the watchdog task to immediately process the request */
+ mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
+ return 0;
+}
+
+/**
+ * iavf_is_mac_set_handled - wait for a response to set MAC from PF
+ * @netdev: network interface device structure
+ * @macaddr: MAC address to set
+ *
+ * Returns true on success, false on failure
+ */
+static bool iavf_is_mac_set_handled(struct net_device *netdev,
+ const u8 *macaddr)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ struct iavf_mac_filter *f;
+ bool ret = false;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ f = iavf_find_filter(adapter, macaddr);
+
+ if (!f || (!f->add && f->add_handled))
+ ret = true;
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ return ret;
+}
+
+/**
+ * iavf_set_mac - NDO callback to set port MAC address
+ * @netdev: network interface device structure
+ * @p: pointer to an address structure
+ *
+ * Returns 0 on success, negative on failure
+ */
+static int iavf_set_mac(struct net_device *netdev, void *p)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ struct sockaddr *addr = p;
+ int ret;
+
+ if (!is_valid_ether_addr(addr->sa_data))
+ return -EADDRNOTAVAIL;
+
+ ret = iavf_replace_primary_mac(adapter, addr->sa_data);
+
+ if (ret)
+ return ret;
+
+ ret = wait_event_interruptible_timeout(adapter->vc_waitqueue,
+ iavf_is_mac_set_handled(netdev, addr->sa_data),
+ msecs_to_jiffies(2500));
+
+ /* If ret < 0 then it means wait was interrupted.
+ * If ret == 0 then it means we got a timeout.
+ * else it means we got response for set MAC from PF,
+ * check if netdev MAC was updated to requested MAC,
+ * if yes then set MAC succeeded otherwise it failed return -EACCES
+ */
+ if (ret < 0)
+ return ret;
+
+ if (!ret)
+ return -EAGAIN;
+
+ if (!ether_addr_equal(netdev->dev_addr, addr->sa_data))
+ return -EACCES;
+
+ return 0;
+}
+
+/**
+ * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address
+ * @netdev: the netdevice
+ * @addr: address to add
+ *
+ * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
+ * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
+ */
+static int iavf_addr_sync(struct net_device *netdev, const u8 *addr)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ if (iavf_add_filter(adapter, addr))
+ return 0;
+ else
+ return -ENOMEM;
+}
+
+/**
+ * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
+ * @netdev: the netdevice
+ * @addr: address to add
+ *
+ * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
+ * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
+ */
+static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ struct iavf_mac_filter *f;
+
+ /* Under some circumstances, we might receive a request to delete
+ * our own device address from our uc list. Because we store the
+ * device address in the VSI's MAC/VLAN filter list, we need to ignore
+ * such requests and not delete our device address from this list.
+ */
+ if (ether_addr_equal(addr, netdev->dev_addr))
+ return 0;
+
+ f = iavf_find_filter(adapter, addr);
+ if (f) {
+ f->remove = true;
+ adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
+ }
+ return 0;
+}
+
+/**
+ * iavf_promiscuous_mode_changed - check if promiscuous mode bits changed
+ * @adapter: device specific adapter
+ */
+bool iavf_promiscuous_mode_changed(struct iavf_adapter *adapter)
+{
+ return (adapter->current_netdev_promisc_flags ^ adapter->netdev->flags) &
+ (IFF_PROMISC | IFF_ALLMULTI);
+}
+
+/**
+ * iavf_set_rx_mode - NDO callback to set the netdev filters
+ * @netdev: network interface device structure
+ **/
+static void iavf_set_rx_mode(struct net_device *netdev)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+ __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
+ __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ spin_lock_bh(&adapter->current_netdev_promisc_flags_lock);
+ if (iavf_promiscuous_mode_changed(adapter))
+ adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE;
+ spin_unlock_bh(&adapter->current_netdev_promisc_flags_lock);
+}
+
+/**
+ * iavf_napi_enable_all - enable NAPI on all queue vectors
+ * @adapter: board private structure
+ **/
+static void iavf_napi_enable_all(struct iavf_adapter *adapter)
+{
+ int q_idx;
+ struct iavf_q_vector *q_vector;
+ int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
+
+ for (q_idx = 0; q_idx < q_vectors; q_idx++) {
+ struct napi_struct *napi;
+
+ q_vector = &adapter->q_vectors[q_idx];
+ napi = &q_vector->napi;
+ napi_enable(napi);
+ }
+}
+
+/**
+ * iavf_napi_disable_all - disable NAPI on all queue vectors
+ * @adapter: board private structure
+ **/
+static void iavf_napi_disable_all(struct iavf_adapter *adapter)
+{
+ int q_idx;
+ struct iavf_q_vector *q_vector;
+ int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
+
+ for (q_idx = 0; q_idx < q_vectors; q_idx++) {
+ q_vector = &adapter->q_vectors[q_idx];
+ napi_disable(&q_vector->napi);
+ }
+}
+
+/**
+ * iavf_configure - set up transmit and receive data structures
+ * @adapter: board private structure
+ **/
+static void iavf_configure(struct iavf_adapter *adapter)
+{
+ struct net_device *netdev = adapter->netdev;
+ int i;
+
+ iavf_set_rx_mode(netdev);
+
+ iavf_configure_tx(adapter);
+ iavf_configure_rx(adapter);
+ adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES;
+
+ for (i = 0; i < adapter->num_active_queues; i++) {
+ struct iavf_ring *ring = &adapter->rx_rings[i];
+
+ iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring));
+ }
+}
+
+/**
+ * iavf_up_complete - Finish the last steps of bringing up a connection
+ * @adapter: board private structure
+ *
+ * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
+ **/
+static void iavf_up_complete(struct iavf_adapter *adapter)
+{
+ iavf_change_state(adapter, __IAVF_RUNNING);
+ clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
+
+ iavf_napi_enable_all(adapter);
+
+ adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES;
+ if (CLIENT_ENABLED(adapter))
+ adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN;
+ mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
+}
+
+/**
+ * iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF
+ * yet and mark other to be removed.
+ * @adapter: board private structure
+ **/
+static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter)
+{
+ struct iavf_vlan_filter *vlf, *vlftmp;
+ struct iavf_mac_filter *f, *ftmp;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+ /* clear the sync flag on all filters */
+ __dev_uc_unsync(adapter->netdev, NULL);
+ __dev_mc_unsync(adapter->netdev, NULL);
+
+ /* remove all MAC filters */
+ list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list,
+ list) {
+ if (f->add) {
+ list_del(&f->list);
+ kfree(f);
+ } else {
+ f->remove = true;
+ }
+ }
+
+ /* disable all VLAN filters */
+ list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
+ list)
+ vlf->state = IAVF_VLAN_DISABLE;
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+}
+
+/**
+ * iavf_clear_cloud_filters - Remove cloud filters not sent to PF yet and
+ * mark other to be removed.
+ * @adapter: board private structure
+ **/
+static void iavf_clear_cloud_filters(struct iavf_adapter *adapter)
+{
+ struct iavf_cloud_filter *cf, *cftmp;
+
+ /* remove all cloud filters */
+ spin_lock_bh(&adapter->cloud_filter_list_lock);
+ list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
+ list) {
+ if (cf->add) {
+ list_del(&cf->list);
+ kfree(cf);
+ adapter->num_cloud_filters--;
+ } else {
+ cf->del = true;
+ }
+ }
+ spin_unlock_bh(&adapter->cloud_filter_list_lock);
+}
+
+/**
+ * iavf_clear_fdir_filters - Remove fdir filters not sent to PF yet and mark
+ * other to be removed.
+ * @adapter: board private structure
+ **/
+static void iavf_clear_fdir_filters(struct iavf_adapter *adapter)
+{
+ struct iavf_fdir_fltr *fdir;
+
+ /* remove all Flow Director filters */
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry(fdir, &adapter->fdir_list_head, list) {
+ if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST) {
+ /* Cancel a request, keep filter as inactive */
+ fdir->state = IAVF_FDIR_FLTR_INACTIVE;
+ } else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING ||
+ fdir->state == IAVF_FDIR_FLTR_ACTIVE) {
+ /* Disable filters which are active or have a pending
+ * request to PF to be added
+ */
+ fdir->state = IAVF_FDIR_FLTR_DIS_REQUEST;
+ }
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+}
+
+/**
+ * iavf_clear_adv_rss_conf - Remove adv rss conf not sent to PF yet and mark
+ * other to be removed.
+ * @adapter: board private structure
+ **/
+static void iavf_clear_adv_rss_conf(struct iavf_adapter *adapter)
+{
+ struct iavf_adv_rss *rss, *rsstmp;
+
+ /* remove all advance RSS configuration */
+ spin_lock_bh(&adapter->adv_rss_lock);
+ list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
+ list) {
+ if (rss->state == IAVF_ADV_RSS_ADD_REQUEST) {
+ list_del(&rss->list);
+ kfree(rss);
+ } else {
+ rss->state = IAVF_ADV_RSS_DEL_REQUEST;
+ }
+ }
+ spin_unlock_bh(&adapter->adv_rss_lock);
+}
+
+/**
+ * iavf_down - Shutdown the connection processing
+ * @adapter: board private structure
+ *
+ * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
+ **/
+void iavf_down(struct iavf_adapter *adapter)
+{
+ struct net_device *netdev = adapter->netdev;
+
+ if (adapter->state <= __IAVF_DOWN_PENDING)
+ return;
+
+ netif_carrier_off(netdev);
+ netif_tx_disable(netdev);
+ adapter->link_up = false;
+ iavf_napi_disable_all(adapter);
+ iavf_irq_disable(adapter);
+
+ iavf_clear_mac_vlan_filters(adapter);
+ iavf_clear_cloud_filters(adapter);
+ iavf_clear_fdir_filters(adapter);
+ iavf_clear_adv_rss_conf(adapter);
+
+ if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) &&
+ !(test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))) {
+ /* cancel any current operation */
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+ /* Schedule operations to close down the HW. Don't wait
+ * here for this to complete. The watchdog is still running
+ * and it will take care of this.
+ */
+ if (!list_empty(&adapter->mac_filter_list))
+ adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
+ if (!list_empty(&adapter->vlan_filter_list))
+ adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
+ if (!list_empty(&adapter->cloud_filter_list))
+ adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
+ if (!list_empty(&adapter->fdir_list_head))
+ adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
+ if (!list_empty(&adapter->adv_rss_list_head))
+ adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
+ }
+
+ adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES;
+ mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
+}
+
+/**
+ * iavf_acquire_msix_vectors - Setup the MSIX capability
+ * @adapter: board private structure
+ * @vectors: number of vectors to request
+ *
+ * Work with the OS to set up the MSIX vectors needed.
+ *
+ * Returns 0 on success, negative on failure
+ **/
+static int
+iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors)
+{
+ int err, vector_threshold;
+
+ /* We'll want at least 3 (vector_threshold):
+ * 0) Other (Admin Queue and link, mostly)
+ * 1) TxQ[0] Cleanup
+ * 2) RxQ[0] Cleanup
+ */
+ vector_threshold = MIN_MSIX_COUNT;
+
+ /* The more we get, the more we will assign to Tx/Rx Cleanup
+ * for the separate queues...where Rx Cleanup >= Tx Cleanup.
+ * Right now, we simply care about how many we'll get; we'll
+ * set them up later while requesting irq's.
+ */
+ err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
+ vector_threshold, vectors);
+ if (err < 0) {
+ dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");
+ kfree(adapter->msix_entries);
+ adapter->msix_entries = NULL;
+ return err;
+ }
+
+ /* Adjust for only the vectors we'll use, which is minimum
+ * of max_msix_q_vectors + NONQ_VECS, or the number of
+ * vectors we were allocated.
+ */
+ adapter->num_msix_vectors = err;
+ return 0;
+}
+
+/**
+ * iavf_free_queues - Free memory for all rings
+ * @adapter: board private structure to initialize
+ *
+ * Free all of the memory associated with queue pairs.
+ **/
+static void iavf_free_queues(struct iavf_adapter *adapter)
+{
+ if (!adapter->vsi_res)
+ return;
+ adapter->num_active_queues = 0;
+ kfree(adapter->tx_rings);
+ adapter->tx_rings = NULL;
+ kfree(adapter->rx_rings);
+ adapter->rx_rings = NULL;
+}
+
+/**
+ * iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload
+ * @adapter: board private structure
+ *
+ * Based on negotiated capabilities, the VLAN tag needs to be inserted and/or
+ * stripped in certain descriptor fields. Instead of checking the offload
+ * capability bits in the hot path, cache the location the ring specific
+ * flags.
+ */
+void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter)
+{
+ int i;
+
+ for (i = 0; i < adapter->num_active_queues; i++) {
+ struct iavf_ring *tx_ring = &adapter->tx_rings[i];
+ struct iavf_ring *rx_ring = &adapter->rx_rings[i];
+
+ /* prevent multiple L2TAG bits being set after VFR */
+ tx_ring->flags &=
+ ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
+ IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2);
+ rx_ring->flags &=
+ ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
+ IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2);
+
+ if (VLAN_ALLOWED(adapter)) {
+ tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
+ rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
+ } else if (VLAN_V2_ALLOWED(adapter)) {
+ struct virtchnl_vlan_supported_caps *stripping_support;
+ struct virtchnl_vlan_supported_caps *insertion_support;
+
+ stripping_support =
+ &adapter->vlan_v2_caps.offloads.stripping_support;
+ insertion_support =
+ &adapter->vlan_v2_caps.offloads.insertion_support;
+
+ if (stripping_support->outer) {
+ if (stripping_support->outer &
+ VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
+ rx_ring->flags |=
+ IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
+ else if (stripping_support->outer &
+ VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
+ rx_ring->flags |=
+ IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
+ } else if (stripping_support->inner) {
+ if (stripping_support->inner &
+ VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
+ rx_ring->flags |=
+ IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
+ else if (stripping_support->inner &
+ VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
+ rx_ring->flags |=
+ IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
+ }
+
+ if (insertion_support->outer) {
+ if (insertion_support->outer &
+ VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
+ tx_ring->flags |=
+ IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
+ else if (insertion_support->outer &
+ VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
+ tx_ring->flags |=
+ IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
+ } else if (insertion_support->inner) {
+ if (insertion_support->inner &
+ VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
+ tx_ring->flags |=
+ IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
+ else if (insertion_support->inner &
+ VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
+ tx_ring->flags |=
+ IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
+ }
+ }
+ }
+}
+
+/**
+ * iavf_alloc_queues - Allocate memory for all rings
+ * @adapter: board private structure to initialize
+ *
+ * We allocate one ring per queue at run-time since we don't know the
+ * number of queues at compile-time. The polling_netdev array is
+ * intended for Multiqueue, but should work fine with a single queue.
+ **/
+static int iavf_alloc_queues(struct iavf_adapter *adapter)
+{
+ int i, num_active_queues;
+
+ /* If we're in reset reallocating queues we don't actually know yet for
+ * certain the PF gave us the number of queues we asked for but we'll
+ * assume it did. Once basic reset is finished we'll confirm once we
+ * start negotiating config with PF.
+ */
+ if (adapter->num_req_queues)
+ num_active_queues = adapter->num_req_queues;
+ else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
+ adapter->num_tc)
+ num_active_queues = adapter->ch_config.total_qps;
+ else
+ num_active_queues = min_t(int,
+ adapter->vsi_res->num_queue_pairs,
+ (int)(num_online_cpus()));
+
+
+ adapter->tx_rings = kcalloc(num_active_queues,
+ sizeof(struct iavf_ring), GFP_KERNEL);
+ if (!adapter->tx_rings)
+ goto err_out;
+ adapter->rx_rings = kcalloc(num_active_queues,
+ sizeof(struct iavf_ring), GFP_KERNEL);
+ if (!adapter->rx_rings)
+ goto err_out;
+
+ for (i = 0; i < num_active_queues; i++) {
+ struct iavf_ring *tx_ring;
+ struct iavf_ring *rx_ring;
+
+ tx_ring = &adapter->tx_rings[i];
+
+ tx_ring->queue_index = i;
+ tx_ring->netdev = adapter->netdev;
+ tx_ring->dev = &adapter->pdev->dev;
+ tx_ring->count = adapter->tx_desc_count;
+ tx_ring->itr_setting = IAVF_ITR_TX_DEF;
+ if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE)
+ tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR;
+
+ rx_ring = &adapter->rx_rings[i];
+ rx_ring->queue_index = i;
+ rx_ring->netdev = adapter->netdev;
+ rx_ring->dev = &adapter->pdev->dev;
+ rx_ring->count = adapter->rx_desc_count;
+ rx_ring->itr_setting = IAVF_ITR_RX_DEF;
+ }
+
+ adapter->num_active_queues = num_active_queues;
+
+ iavf_set_queue_vlan_tag_loc(adapter);
+
+ return 0;
+
+err_out:
+ iavf_free_queues(adapter);
+ return -ENOMEM;
+}
+
+/**
+ * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported
+ * @adapter: board private structure to initialize
+ *
+ * Attempt to configure the interrupts using the best available
+ * capabilities of the hardware and the kernel.
+ **/
+static int iavf_set_interrupt_capability(struct iavf_adapter *adapter)
+{
+ int vector, v_budget;
+ int pairs = 0;
+ int err = 0;
+
+ if (!adapter->vsi_res) {
+ err = -EIO;
+ goto out;
+ }
+ pairs = adapter->num_active_queues;
+
+ /* It's easy to be greedy for MSI-X vectors, but it really doesn't do
+ * us much good if we have more vectors than CPUs. However, we already
+ * limit the total number of queues by the number of CPUs so we do not
+ * need any further limiting here.
+ */
+ v_budget = min_t(int, pairs + NONQ_VECS,
+ (int)adapter->vf_res->max_vectors);
+
+ adapter->msix_entries = kcalloc(v_budget,
+ sizeof(struct msix_entry), GFP_KERNEL);
+ if (!adapter->msix_entries) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ for (vector = 0; vector < v_budget; vector++)
+ adapter->msix_entries[vector].entry = vector;
+
+ err = iavf_acquire_msix_vectors(adapter, v_budget);
+ if (!err)
+ iavf_schedule_finish_config(adapter);
+
+out:
+ return err;
+}
+
+/**
+ * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands
+ * @adapter: board private structure
+ *
+ * Return 0 on success, negative on failure
+ **/
+static int iavf_config_rss_aq(struct iavf_adapter *adapter)
+{
+ struct iavf_aqc_get_set_rss_key_data *rss_key =
+ (struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key;
+ struct iavf_hw *hw = &adapter->hw;
+ enum iavf_status status;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
+ adapter->current_op);
+ return -EBUSY;
+ }
+
+ status = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key);
+ if (status) {
+ dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",
+ iavf_stat_str(hw, status),
+ iavf_aq_str(hw, hw->aq.asq_last_status));
+ return iavf_status_to_errno(status);
+
+ }
+
+ status = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false,
+ adapter->rss_lut, adapter->rss_lut_size);
+ if (status) {
+ dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",
+ iavf_stat_str(hw, status),
+ iavf_aq_str(hw, hw->aq.asq_last_status));
+ return iavf_status_to_errno(status);
+ }
+
+ return 0;
+
+}
+
+/**
+ * iavf_config_rss_reg - Configure RSS keys and lut by writing registers
+ * @adapter: board private structure
+ *
+ * Returns 0 on success, negative on failure
+ **/
+static int iavf_config_rss_reg(struct iavf_adapter *adapter)
+{
+ struct iavf_hw *hw = &adapter->hw;
+ u32 *dw;
+ u16 i;
+
+ dw = (u32 *)adapter->rss_key;
+ for (i = 0; i <= adapter->rss_key_size / 4; i++)
+ wr32(hw, IAVF_VFQF_HKEY(i), dw[i]);
+
+ dw = (u32 *)adapter->rss_lut;
+ for (i = 0; i <= adapter->rss_lut_size / 4; i++)
+ wr32(hw, IAVF_VFQF_HLUT(i), dw[i]);
+
+ iavf_flush(hw);
+
+ return 0;
+}
+
+/**
+ * iavf_config_rss - Configure RSS keys and lut
+ * @adapter: board private structure
+ *
+ * Returns 0 on success, negative on failure
+ **/
+int iavf_config_rss(struct iavf_adapter *adapter)
+{
+
+ if (RSS_PF(adapter)) {
+ adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT |
+ IAVF_FLAG_AQ_SET_RSS_KEY;
+ return 0;
+ } else if (RSS_AQ(adapter)) {
+ return iavf_config_rss_aq(adapter);
+ } else {
+ return iavf_config_rss_reg(adapter);
+ }
+}
+
+/**
+ * iavf_fill_rss_lut - Fill the lut with default values
+ * @adapter: board private structure
+ **/
+static void iavf_fill_rss_lut(struct iavf_adapter *adapter)
+{
+ u16 i;
+
+ for (i = 0; i < adapter->rss_lut_size; i++)
+ adapter->rss_lut[i] = i % adapter->num_active_queues;
+}
+
+/**
+ * iavf_init_rss - Prepare for RSS
+ * @adapter: board private structure
+ *
+ * Return 0 on success, negative on failure
+ **/
+static int iavf_init_rss(struct iavf_adapter *adapter)
+{
+ struct iavf_hw *hw = &adapter->hw;
+
+ if (!RSS_PF(adapter)) {
+ /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
+ if (adapter->vf_res->vf_cap_flags &
+ VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
+ adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED;
+ else
+ adapter->hena = IAVF_DEFAULT_RSS_HENA;
+
+ wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena);
+ wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32));
+ }
+
+ iavf_fill_rss_lut(adapter);
+ netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size);
+
+ return iavf_config_rss(adapter);
+}
+
+/**
+ * iavf_alloc_q_vectors - Allocate memory for interrupt vectors
+ * @adapter: board private structure to initialize
+ *
+ * We allocate one q_vector per queue interrupt. If allocation fails we
+ * return -ENOMEM.
+ **/
+static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)
+{
+ int q_idx = 0, num_q_vectors;
+ struct iavf_q_vector *q_vector;
+
+ num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
+ adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector),
+ GFP_KERNEL);
+ if (!adapter->q_vectors)
+ return -ENOMEM;
+
+ for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
+ q_vector = &adapter->q_vectors[q_idx];
+ q_vector->adapter = adapter;
+ q_vector->vsi = &adapter->vsi;
+ q_vector->v_idx = q_idx;
+ q_vector->reg_idx = q_idx;
+ cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
+ netif_napi_add(adapter->netdev, &q_vector->napi,
+ iavf_napi_poll);
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_free_q_vectors - Free memory allocated for interrupt vectors
+ * @adapter: board private structure to initialize
+ *
+ * This function frees the memory allocated to the q_vectors. In addition if
+ * NAPI is enabled it will delete any references to the NAPI struct prior
+ * to freeing the q_vector.
+ **/
+static void iavf_free_q_vectors(struct iavf_adapter *adapter)
+{
+ int q_idx, num_q_vectors;
+
+ if (!adapter->q_vectors)
+ return;
+
+ num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
+
+ for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
+ struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx];
+
+ netif_napi_del(&q_vector->napi);
+ }
+ kfree(adapter->q_vectors);
+ adapter->q_vectors = NULL;
+}
+
+/**
+ * iavf_reset_interrupt_capability - Reset MSIX setup
+ * @adapter: board private structure
+ *
+ **/
+static void iavf_reset_interrupt_capability(struct iavf_adapter *adapter)
+{
+ if (!adapter->msix_entries)
+ return;
+
+ pci_disable_msix(adapter->pdev);
+ kfree(adapter->msix_entries);
+ adapter->msix_entries = NULL;
+}
+
+/**
+ * iavf_init_interrupt_scheme - Determine if MSIX is supported and init
+ * @adapter: board private structure to initialize
+ *
+ **/
+static int iavf_init_interrupt_scheme(struct iavf_adapter *adapter)
+{
+ int err;
+
+ err = iavf_alloc_queues(adapter);
+ if (err) {
+ dev_err(&adapter->pdev->dev,
+ "Unable to allocate memory for queues\n");
+ goto err_alloc_queues;
+ }
+
+ err = iavf_set_interrupt_capability(adapter);
+ if (err) {
+ dev_err(&adapter->pdev->dev,
+ "Unable to setup interrupt capabilities\n");
+ goto err_set_interrupt;
+ }
+
+ err = iavf_alloc_q_vectors(adapter);
+ if (err) {
+ dev_err(&adapter->pdev->dev,
+ "Unable to allocate memory for queue vectors\n");
+ goto err_alloc_q_vectors;
+ }
+
+ /* If we've made it so far while ADq flag being ON, then we haven't
+ * bailed out anywhere in middle. And ADq isn't just enabled but actual
+ * resources have been allocated in the reset path.
+ * Now we can truly claim that ADq is enabled.
+ */
+ if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
+ adapter->num_tc)
+ dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",
+ adapter->num_tc);
+
+ dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
+ (adapter->num_active_queues > 1) ? "Enabled" : "Disabled",
+ adapter->num_active_queues);
+
+ return 0;
+err_alloc_q_vectors:
+ iavf_reset_interrupt_capability(adapter);
+err_set_interrupt:
+ iavf_free_queues(adapter);
+err_alloc_queues:
+ return err;
+}
+
+/**
+ * iavf_free_rss - Free memory used by RSS structs
+ * @adapter: board private structure
+ **/
+static void iavf_free_rss(struct iavf_adapter *adapter)
+{
+ kfree(adapter->rss_key);
+ adapter->rss_key = NULL;
+
+ kfree(adapter->rss_lut);
+ adapter->rss_lut = NULL;
+}
+
+/**
+ * iavf_reinit_interrupt_scheme - Reallocate queues and vectors
+ * @adapter: board private structure
+ * @running: true if adapter->state == __IAVF_RUNNING
+ *
+ * Returns 0 on success, negative on failure
+ **/
+static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter, bool running)
+{
+ struct net_device *netdev = adapter->netdev;
+ int err;
+
+ if (running)
+ iavf_free_traffic_irqs(adapter);
+ iavf_free_misc_irq(adapter);
+ iavf_reset_interrupt_capability(adapter);
+ iavf_free_q_vectors(adapter);
+ iavf_free_queues(adapter);
+
+ err = iavf_init_interrupt_scheme(adapter);
+ if (err)
+ goto err;
+
+ netif_tx_stop_all_queues(netdev);
+
+ err = iavf_request_misc_irq(adapter);
+ if (err)
+ goto err;
+
+ set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
+
+ iavf_map_rings_to_vectors(adapter);
+err:
+ return err;
+}
+
+/**
+ * iavf_finish_config - do all netdev work that needs RTNL
+ * @work: our work_struct
+ *
+ * Do work that needs both RTNL and crit_lock.
+ **/
+static void iavf_finish_config(struct work_struct *work)
+{
+ struct iavf_adapter *adapter;
+ int pairs, err;
+
+ adapter = container_of(work, struct iavf_adapter, finish_config);
+
+ /* Always take RTNL first to prevent circular lock dependency */
+ rtnl_lock();
+ mutex_lock(&adapter->crit_lock);
+
+ if ((adapter->flags & IAVF_FLAG_SETUP_NETDEV_FEATURES) &&
+ adapter->netdev_registered &&
+ !test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) {
+ netdev_update_features(adapter->netdev);
+ adapter->flags &= ~IAVF_FLAG_SETUP_NETDEV_FEATURES;
+ }
+
+ switch (adapter->state) {
+ case __IAVF_DOWN:
+ if (!adapter->netdev_registered) {
+ err = register_netdevice(adapter->netdev);
+ if (err) {
+ dev_err(&adapter->pdev->dev, "Unable to register netdev (%d)\n",
+ err);
+
+ /* go back and try again.*/
+ iavf_free_rss(adapter);
+ iavf_free_misc_irq(adapter);
+ iavf_reset_interrupt_capability(adapter);
+ iavf_change_state(adapter,
+ __IAVF_INIT_CONFIG_ADAPTER);
+ goto out;
+ }
+ adapter->netdev_registered = true;
+ }
+
+ /* Set the real number of queues when reset occurs while
+ * state == __IAVF_DOWN
+ */
+ fallthrough;
+ case __IAVF_RUNNING:
+ pairs = adapter->num_active_queues;
+ netif_set_real_num_rx_queues(adapter->netdev, pairs);
+ netif_set_real_num_tx_queues(adapter->netdev, pairs);
+ break;
+
+ default:
+ break;
+ }
+
+out:
+ mutex_unlock(&adapter->crit_lock);
+ rtnl_unlock();
+}
+
+/**
+ * iavf_schedule_finish_config - Set the flags and schedule a reset event
+ * @adapter: board private structure
+ **/
+void iavf_schedule_finish_config(struct iavf_adapter *adapter)
+{
+ if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
+ queue_work(adapter->wq, &adapter->finish_config);
+}
+
+/**
+ * iavf_process_aq_command - process aq_required flags
+ * and sends aq command
+ * @adapter: pointer to iavf adapter structure
+ *
+ * Returns 0 on success
+ * Returns error code if no command was sent
+ * or error code if the command failed.
+ **/
+static int iavf_process_aq_command(struct iavf_adapter *adapter)
+{
+ if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG)
+ return iavf_send_vf_config_msg(adapter);
+ if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS)
+ return iavf_send_vf_offload_vlan_v2_msg(adapter);
+ if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) {
+ iavf_disable_queues(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) {
+ iavf_map_queues(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) {
+ iavf_add_ether_addrs(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) {
+ iavf_add_vlans(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) {
+ iavf_del_ether_addrs(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) {
+ iavf_del_vlans(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
+ iavf_enable_vlan_stripping(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
+ iavf_disable_vlan_stripping(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) {
+ iavf_configure_queues(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) {
+ iavf_enable_queues(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) {
+ /* This message goes straight to the firmware, not the
+ * PF, so we don't have to set current_op as we will
+ * not get a response through the ARQ.
+ */
+ adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS;
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) {
+ iavf_get_hena(adapter);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) {
+ iavf_set_hena(adapter);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) {
+ iavf_set_rss_key(adapter);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) {
+ iavf_set_rss_lut(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE) {
+ iavf_set_promiscuous(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) {
+ iavf_enable_channels(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) {
+ iavf_disable_channels(adapter);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
+ iavf_add_cloud_filter(adapter);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
+ iavf_del_cloud_filter(adapter);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
+ iavf_del_cloud_filter(adapter);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
+ iavf_add_cloud_filter(adapter);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) {
+ iavf_add_fdir_filter(adapter);
+ return IAVF_SUCCESS;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) {
+ iavf_del_fdir_filter(adapter);
+ return IAVF_SUCCESS;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) {
+ iavf_add_adv_rss_cfg(adapter);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) {
+ iavf_del_adv_rss_cfg(adapter);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) {
+ iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) {
+ iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) {
+ iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) {
+ iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) {
+ iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) {
+ iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) {
+ iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q);
+ return 0;
+ }
+ if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) {
+ iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD);
+ return 0;
+ }
+
+ if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) {
+ iavf_request_stats(adapter);
+ return 0;
+ }
+
+ return -EAGAIN;
+}
+
+/**
+ * iavf_set_vlan_offload_features - set VLAN offload configuration
+ * @adapter: board private structure
+ * @prev_features: previous features used for comparison
+ * @features: updated features used for configuration
+ *
+ * Set the aq_required bit(s) based on the requested features passed in to
+ * configure VLAN stripping and/or VLAN insertion if supported. Also, schedule
+ * the watchdog if any changes are requested to expedite the request via
+ * virtchnl.
+ **/
+static void
+iavf_set_vlan_offload_features(struct iavf_adapter *adapter,
+ netdev_features_t prev_features,
+ netdev_features_t features)
+{
+ bool enable_stripping = true, enable_insertion = true;
+ u16 vlan_ethertype = 0;
+ u64 aq_required = 0;
+
+ /* keep cases separate because one ethertype for offloads can be
+ * disabled at the same time as another is disabled, so check for an
+ * enabled ethertype first, then check for disabled. Default to
+ * ETH_P_8021Q so an ethertype is specified if disabling insertion and
+ * stripping.
+ */
+ if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
+ vlan_ethertype = ETH_P_8021AD;
+ else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
+ vlan_ethertype = ETH_P_8021Q;
+ else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
+ vlan_ethertype = ETH_P_8021AD;
+ else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
+ vlan_ethertype = ETH_P_8021Q;
+ else
+ vlan_ethertype = ETH_P_8021Q;
+
+ if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX)))
+ enable_stripping = false;
+ if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX)))
+ enable_insertion = false;
+
+ if (VLAN_ALLOWED(adapter)) {
+ /* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN
+ * stripping via virtchnl. VLAN insertion can be toggled on the
+ * netdev, but it doesn't require a virtchnl message
+ */
+ if (enable_stripping)
+ aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
+ else
+ aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
+
+ } else if (VLAN_V2_ALLOWED(adapter)) {
+ switch (vlan_ethertype) {
+ case ETH_P_8021Q:
+ if (enable_stripping)
+ aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING;
+ else
+ aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING;
+
+ if (enable_insertion)
+ aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION;
+ else
+ aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION;
+ break;
+ case ETH_P_8021AD:
+ if (enable_stripping)
+ aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING;
+ else
+ aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING;
+
+ if (enable_insertion)
+ aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION;
+ else
+ aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION;
+ break;
+ }
+ }
+
+ if (aq_required) {
+ adapter->aq_required |= aq_required;
+ mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
+ }
+}
+
+/**
+ * iavf_startup - first step of driver startup
+ * @adapter: board private structure
+ *
+ * Function process __IAVF_STARTUP driver state.
+ * When success the state is changed to __IAVF_INIT_VERSION_CHECK
+ * when fails the state is changed to __IAVF_INIT_FAILED
+ **/
+static void iavf_startup(struct iavf_adapter *adapter)
+{
+ struct pci_dev *pdev = adapter->pdev;
+ struct iavf_hw *hw = &adapter->hw;
+ enum iavf_status status;
+ int ret;
+
+ WARN_ON(adapter->state != __IAVF_STARTUP);
+
+ /* driver loaded, probe complete */
+ adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
+ adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
+ status = iavf_set_mac_type(hw);
+ if (status) {
+ dev_err(&pdev->dev, "Failed to set MAC type (%d)\n", status);
+ goto err;
+ }
+
+ ret = iavf_check_reset_complete(hw);
+ if (ret) {
+ dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",
+ ret);
+ goto err;
+ }
+ hw->aq.num_arq_entries = IAVF_AQ_LEN;
+ hw->aq.num_asq_entries = IAVF_AQ_LEN;
+ hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
+ hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
+
+ status = iavf_init_adminq(hw);
+ if (status) {
+ dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",
+ status);
+ goto err;
+ }
+ ret = iavf_send_api_ver(adapter);
+ if (ret) {
+ dev_err(&pdev->dev, "Unable to send to PF (%d)\n", ret);
+ iavf_shutdown_adminq(hw);
+ goto err;
+ }
+ iavf_change_state(adapter, __IAVF_INIT_VERSION_CHECK);
+ return;
+err:
+ iavf_change_state(adapter, __IAVF_INIT_FAILED);
+}
+
+/**
+ * iavf_init_version_check - second step of driver startup
+ * @adapter: board private structure
+ *
+ * Function process __IAVF_INIT_VERSION_CHECK driver state.
+ * When success the state is changed to __IAVF_INIT_GET_RESOURCES
+ * when fails the state is changed to __IAVF_INIT_FAILED
+ **/
+static void iavf_init_version_check(struct iavf_adapter *adapter)
+{
+ struct pci_dev *pdev = adapter->pdev;
+ struct iavf_hw *hw = &adapter->hw;
+ int err = -EAGAIN;
+
+ WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK);
+
+ if (!iavf_asq_done(hw)) {
+ dev_err(&pdev->dev, "Admin queue command never completed\n");
+ iavf_shutdown_adminq(hw);
+ iavf_change_state(adapter, __IAVF_STARTUP);
+ goto err;
+ }
+
+ /* aq msg sent, awaiting reply */
+ err = iavf_verify_api_ver(adapter);
+ if (err) {
+ if (err == -EALREADY)
+ err = iavf_send_api_ver(adapter);
+ else
+ dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",
+ adapter->pf_version.major,
+ adapter->pf_version.minor,
+ VIRTCHNL_VERSION_MAJOR,
+ VIRTCHNL_VERSION_MINOR);
+ goto err;
+ }
+ err = iavf_send_vf_config_msg(adapter);
+ if (err) {
+ dev_err(&pdev->dev, "Unable to send config request (%d)\n",
+ err);
+ goto err;
+ }
+ iavf_change_state(adapter, __IAVF_INIT_GET_RESOURCES);
+ return;
+err:
+ iavf_change_state(adapter, __IAVF_INIT_FAILED);
+}
+
+/**
+ * iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES
+ * @adapter: board private structure
+ */
+int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter)
+{
+ int i, num_req_queues = adapter->num_req_queues;
+ struct iavf_vsi *vsi = &adapter->vsi;
+
+ for (i = 0; i < adapter->vf_res->num_vsis; i++) {
+ if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
+ adapter->vsi_res = &adapter->vf_res->vsi_res[i];
+ }
+ if (!adapter->vsi_res) {
+ dev_err(&adapter->pdev->dev, "No LAN VSI found\n");
+ return -ENODEV;
+ }
+
+ if (num_req_queues &&
+ num_req_queues > adapter->vsi_res->num_queue_pairs) {
+ /* Problem. The PF gave us fewer queues than what we had
+ * negotiated in our request. Need a reset to see if we can't
+ * get back to a working state.
+ */
+ dev_err(&adapter->pdev->dev,
+ "Requested %d queues, but PF only gave us %d.\n",
+ num_req_queues,
+ adapter->vsi_res->num_queue_pairs);
+ adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED;
+ adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;
+ iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
+
+ return -EAGAIN;
+ }
+ adapter->num_req_queues = 0;
+ adapter->vsi.id = adapter->vsi_res->vsi_id;
+
+ adapter->vsi.back = adapter;
+ adapter->vsi.base_vector = 1;
+ vsi->netdev = adapter->netdev;
+ vsi->qs_handle = adapter->vsi_res->qset_handle;
+ if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
+ adapter->rss_key_size = adapter->vf_res->rss_key_size;
+ adapter->rss_lut_size = adapter->vf_res->rss_lut_size;
+ } else {
+ adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE;
+ adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE;
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_init_get_resources - third step of driver startup
+ * @adapter: board private structure
+ *
+ * Function process __IAVF_INIT_GET_RESOURCES driver state and
+ * finishes driver initialization procedure.
+ * When success the state is changed to __IAVF_DOWN
+ * when fails the state is changed to __IAVF_INIT_FAILED
+ **/
+static void iavf_init_get_resources(struct iavf_adapter *adapter)
+{
+ struct pci_dev *pdev = adapter->pdev;
+ struct iavf_hw *hw = &adapter->hw;
+ int err;
+
+ WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES);
+ /* aq msg sent, awaiting reply */
+ if (!adapter->vf_res) {
+ adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE,
+ GFP_KERNEL);
+ if (!adapter->vf_res) {
+ err = -ENOMEM;
+ goto err;
+ }
+ }
+ err = iavf_get_vf_config(adapter);
+ if (err == -EALREADY) {
+ err = iavf_send_vf_config_msg(adapter);
+ goto err;
+ } else if (err == -EINVAL) {
+ /* We only get -EINVAL if the device is in a very bad
+ * state or if we've been disabled for previous bad
+ * behavior. Either way, we're done now.
+ */
+ iavf_shutdown_adminq(hw);
+ dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
+ return;
+ }
+ if (err) {
+ dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err);
+ goto err_alloc;
+ }
+
+ err = iavf_parse_vf_resource_msg(adapter);
+ if (err) {
+ dev_err(&pdev->dev, "Failed to parse VF resource message from PF (%d)\n",
+ err);
+ goto err_alloc;
+ }
+ /* Some features require additional messages to negotiate extended
+ * capabilities. These are processed in sequence by the
+ * __IAVF_INIT_EXTENDED_CAPS driver state.
+ */
+ adapter->extended_caps = IAVF_EXTENDED_CAPS;
+
+ iavf_change_state(adapter, __IAVF_INIT_EXTENDED_CAPS);
+ return;
+
+err_alloc:
+ kfree(adapter->vf_res);
+ adapter->vf_res = NULL;
+err:
+ iavf_change_state(adapter, __IAVF_INIT_FAILED);
+}
+
+/**
+ * iavf_init_send_offload_vlan_v2_caps - part of initializing VLAN V2 caps
+ * @adapter: board private structure
+ *
+ * Function processes send of the extended VLAN V2 capability message to the
+ * PF. Must clear IAVF_EXTENDED_CAP_RECV_VLAN_V2 if the message is not sent,
+ * e.g. due to PF not negotiating VIRTCHNL_VF_OFFLOAD_VLAN_V2.
+ */
+static void iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter *adapter)
+{
+ int ret;
+
+ WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2));
+
+ ret = iavf_send_vf_offload_vlan_v2_msg(adapter);
+ if (ret && ret == -EOPNOTSUPP) {
+ /* PF does not support VIRTCHNL_VF_OFFLOAD_V2. In this case,
+ * we did not send the capability exchange message and do not
+ * expect a response.
+ */
+ adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
+ }
+
+ /* We sent the message, so move on to the next step */
+ adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_VLAN_V2;
+}
+
+/**
+ * iavf_init_recv_offload_vlan_v2_caps - part of initializing VLAN V2 caps
+ * @adapter: board private structure
+ *
+ * Function processes receipt of the extended VLAN V2 capability message from
+ * the PF.
+ **/
+static void iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter *adapter)
+{
+ int ret;
+
+ WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2));
+
+ memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps));
+
+ ret = iavf_get_vf_vlan_v2_caps(adapter);
+ if (ret)
+ goto err;
+
+ /* We've processed receipt of the VLAN V2 caps message */
+ adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
+ return;
+err:
+ /* We didn't receive a reply. Make sure we try sending again when
+ * __IAVF_INIT_FAILED attempts to recover.
+ */
+ adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_VLAN_V2;
+ iavf_change_state(adapter, __IAVF_INIT_FAILED);
+}
+
+/**
+ * iavf_init_process_extended_caps - Part of driver startup
+ * @adapter: board private structure
+ *
+ * Function processes __IAVF_INIT_EXTENDED_CAPS driver state. This state
+ * handles negotiating capabilities for features which require an additional
+ * message.
+ *
+ * Once all extended capabilities exchanges are finished, the driver will
+ * transition into __IAVF_INIT_CONFIG_ADAPTER.
+ */
+static void iavf_init_process_extended_caps(struct iavf_adapter *adapter)
+{
+ WARN_ON(adapter->state != __IAVF_INIT_EXTENDED_CAPS);
+
+ /* Process capability exchange for VLAN V2 */
+ if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2) {
+ iavf_init_send_offload_vlan_v2_caps(adapter);
+ return;
+ } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2) {
+ iavf_init_recv_offload_vlan_v2_caps(adapter);
+ return;
+ }
+
+ /* When we reach here, no further extended capabilities exchanges are
+ * necessary, so we finally transition into __IAVF_INIT_CONFIG_ADAPTER
+ */
+ iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER);
+}
+
+/**
+ * iavf_init_config_adapter - last part of driver startup
+ * @adapter: board private structure
+ *
+ * After all the supported capabilities are negotiated, then the
+ * __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization.
+ */
+static void iavf_init_config_adapter(struct iavf_adapter *adapter)
+{
+ struct net_device *netdev = adapter->netdev;
+ struct pci_dev *pdev = adapter->pdev;
+ int err;
+
+ WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER);
+
+ if (iavf_process_config(adapter))
+ goto err;
+
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+
+ adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED;
+
+ netdev->netdev_ops = &iavf_netdev_ops;
+ iavf_set_ethtool_ops(netdev);
+ netdev->watchdog_timeo = 5 * HZ;
+
+ /* MTU range: 68 - 9710 */
+ netdev->min_mtu = ETH_MIN_MTU;
+ netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD;
+
+ if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
+ dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
+ adapter->hw.mac.addr);
+ eth_hw_addr_random(netdev);
+ ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
+ } else {
+ eth_hw_addr_set(netdev, adapter->hw.mac.addr);
+ ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
+ }
+
+ adapter->tx_desc_count = IAVF_DEFAULT_TXD;
+ adapter->rx_desc_count = IAVF_DEFAULT_RXD;
+ err = iavf_init_interrupt_scheme(adapter);
+ if (err)
+ goto err_sw_init;
+ iavf_map_rings_to_vectors(adapter);
+ if (adapter->vf_res->vf_cap_flags &
+ VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
+ adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE;
+
+ err = iavf_request_misc_irq(adapter);
+ if (err)
+ goto err_sw_init;
+
+ netif_carrier_off(netdev);
+ adapter->link_up = false;
+ netif_tx_stop_all_queues(netdev);
+
+ if (CLIENT_ALLOWED(adapter)) {
+ err = iavf_lan_add_device(adapter);
+ if (err)
+ dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n",
+ err);
+ }
+ dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);
+ if (netdev->features & NETIF_F_GRO)
+ dev_info(&pdev->dev, "GRO is enabled\n");
+
+ iavf_change_state(adapter, __IAVF_DOWN);
+ set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
+
+ iavf_misc_irq_enable(adapter);
+ wake_up(&adapter->down_waitqueue);
+
+ adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);
+ adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);
+ if (!adapter->rss_key || !adapter->rss_lut) {
+ err = -ENOMEM;
+ goto err_mem;
+ }
+ if (RSS_AQ(adapter))
+ adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
+ else
+ iavf_init_rss(adapter);
+
+ if (VLAN_V2_ALLOWED(adapter))
+ /* request initial VLAN offload settings */
+ iavf_set_vlan_offload_features(adapter, 0, netdev->features);
+
+ iavf_schedule_finish_config(adapter);
+ return;
+
+err_mem:
+ iavf_free_rss(adapter);
+ iavf_free_misc_irq(adapter);
+err_sw_init:
+ iavf_reset_interrupt_capability(adapter);
+err:
+ iavf_change_state(adapter, __IAVF_INIT_FAILED);
+}
+
+/**
+ * iavf_watchdog_task - Periodic call-back task
+ * @work: pointer to work_struct
+ **/
+static void iavf_watchdog_task(struct work_struct *work)
+{
+ struct iavf_adapter *adapter = container_of(work,
+ struct iavf_adapter,
+ watchdog_task.work);
+ struct iavf_hw *hw = &adapter->hw;
+ u32 reg_val;
+
+ if (!mutex_trylock(&adapter->crit_lock)) {
+ if (adapter->state == __IAVF_REMOVE)
+ return;
+
+ goto restart_watchdog;
+ }
+
+ if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
+ iavf_change_state(adapter, __IAVF_COMM_FAILED);
+
+ switch (adapter->state) {
+ case __IAVF_STARTUP:
+ iavf_startup(adapter);
+ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(adapter->wq, &adapter->watchdog_task,
+ msecs_to_jiffies(30));
+ return;
+ case __IAVF_INIT_VERSION_CHECK:
+ iavf_init_version_check(adapter);
+ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(adapter->wq, &adapter->watchdog_task,
+ msecs_to_jiffies(30));
+ return;
+ case __IAVF_INIT_GET_RESOURCES:
+ iavf_init_get_resources(adapter);
+ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(adapter->wq, &adapter->watchdog_task,
+ msecs_to_jiffies(1));
+ return;
+ case __IAVF_INIT_EXTENDED_CAPS:
+ iavf_init_process_extended_caps(adapter);
+ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(adapter->wq, &adapter->watchdog_task,
+ msecs_to_jiffies(1));
+ return;
+ case __IAVF_INIT_CONFIG_ADAPTER:
+ iavf_init_config_adapter(adapter);
+ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(adapter->wq, &adapter->watchdog_task,
+ msecs_to_jiffies(1));
+ return;
+ case __IAVF_INIT_FAILED:
+ if (test_bit(__IAVF_IN_REMOVE_TASK,
+ &adapter->crit_section)) {
+ /* Do not update the state and do not reschedule
+ * watchdog task, iavf_remove should handle this state
+ * as it can loop forever
+ */
+ mutex_unlock(&adapter->crit_lock);
+ return;
+ }
+ if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
+ dev_err(&adapter->pdev->dev,
+ "Failed to communicate with PF; waiting before retry\n");
+ adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
+ iavf_shutdown_adminq(hw);
+ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(adapter->wq,
+ &adapter->watchdog_task, (5 * HZ));
+ return;
+ }
+ /* Try again from failed step*/
+ iavf_change_state(adapter, adapter->last_state);
+ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(adapter->wq, &adapter->watchdog_task, HZ);
+ return;
+ case __IAVF_COMM_FAILED:
+ if (test_bit(__IAVF_IN_REMOVE_TASK,
+ &adapter->crit_section)) {
+ /* Set state to __IAVF_INIT_FAILED and perform remove
+ * steps. Remove IAVF_FLAG_PF_COMMS_FAILED so the task
+ * doesn't bring the state back to __IAVF_COMM_FAILED.
+ */
+ iavf_change_state(adapter, __IAVF_INIT_FAILED);
+ adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
+ mutex_unlock(&adapter->crit_lock);
+ return;
+ }
+ reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
+ IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
+ if (reg_val == VIRTCHNL_VFR_VFACTIVE ||
+ reg_val == VIRTCHNL_VFR_COMPLETED) {
+ /* A chance for redemption! */
+ dev_err(&adapter->pdev->dev,
+ "Hardware came out of reset. Attempting reinit.\n");
+ /* When init task contacts the PF and
+ * gets everything set up again, it'll restart the
+ * watchdog for us. Down, boy. Sit. Stay. Woof.
+ */
+ iavf_change_state(adapter, __IAVF_STARTUP);
+ adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
+ }
+ adapter->aq_required = 0;
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(adapter->wq,
+ &adapter->watchdog_task,
+ msecs_to_jiffies(10));
+ return;
+ case __IAVF_RESETTING:
+ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(adapter->wq, &adapter->watchdog_task,
+ HZ * 2);
+ return;
+ case __IAVF_DOWN:
+ case __IAVF_DOWN_PENDING:
+ case __IAVF_TESTING:
+ case __IAVF_RUNNING:
+ if (adapter->current_op) {
+ if (!iavf_asq_done(hw)) {
+ dev_dbg(&adapter->pdev->dev,
+ "Admin queue timeout\n");
+ iavf_send_api_ver(adapter);
+ }
+ } else {
+ int ret = iavf_process_aq_command(adapter);
+
+ /* An error will be returned if no commands were
+ * processed; use this opportunity to update stats
+ * if the error isn't -ENOTSUPP
+ */
+ if (ret && ret != -EOPNOTSUPP &&
+ adapter->state == __IAVF_RUNNING)
+ iavf_request_stats(adapter);
+ }
+ if (adapter->state == __IAVF_RUNNING)
+ iavf_detect_recover_hung(&adapter->vsi);
+ break;
+ case __IAVF_REMOVE:
+ default:
+ mutex_unlock(&adapter->crit_lock);
+ return;
+ }
+
+ /* check for hw reset */
+ reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK;
+ if (!reg_val) {
+ adapter->aq_required = 0;
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+ dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
+ iavf_schedule_reset(adapter, IAVF_FLAG_RESET_PENDING);
+ mutex_unlock(&adapter->crit_lock);
+ queue_delayed_work(adapter->wq,
+ &adapter->watchdog_task, HZ * 2);
+ return;
+ }
+
+ schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5));
+ mutex_unlock(&adapter->crit_lock);
+restart_watchdog:
+ if (adapter->state >= __IAVF_DOWN)
+ queue_work(adapter->wq, &adapter->adminq_task);
+ if (adapter->aq_required)
+ queue_delayed_work(adapter->wq, &adapter->watchdog_task,
+ msecs_to_jiffies(20));
+ else
+ queue_delayed_work(adapter->wq, &adapter->watchdog_task,
+ HZ * 2);
+}
+
+/**
+ * iavf_disable_vf - disable VF
+ * @adapter: board private structure
+ *
+ * Set communication failed flag and free all resources.
+ * NOTE: This function is expected to be called with crit_lock being held.
+ **/
+static void iavf_disable_vf(struct iavf_adapter *adapter)
+{
+ struct iavf_mac_filter *f, *ftmp;
+ struct iavf_vlan_filter *fv, *fvtmp;
+ struct iavf_cloud_filter *cf, *cftmp;
+
+ adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
+
+ /* We don't use netif_running() because it may be true prior to
+ * ndo_open() returning, so we can't assume it means all our open
+ * tasks have finished, since we're not holding the rtnl_lock here.
+ */
+ if (adapter->state == __IAVF_RUNNING) {
+ set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
+ netif_carrier_off(adapter->netdev);
+ netif_tx_disable(adapter->netdev);
+ adapter->link_up = false;
+ iavf_napi_disable_all(adapter);
+ iavf_irq_disable(adapter);
+ iavf_free_traffic_irqs(adapter);
+ iavf_free_all_tx_resources(adapter);
+ iavf_free_all_rx_resources(adapter);
+ }
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ /* Delete all of the filters */
+ list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
+ list_del(&f->list);
+ kfree(f);
+ }
+
+ list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {
+ list_del(&fv->list);
+ kfree(fv);
+ }
+ adapter->num_vlan_filters = 0;
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ spin_lock_bh(&adapter->cloud_filter_list_lock);
+ list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
+ list_del(&cf->list);
+ kfree(cf);
+ adapter->num_cloud_filters--;
+ }
+ spin_unlock_bh(&adapter->cloud_filter_list_lock);
+
+ iavf_free_misc_irq(adapter);
+ iavf_reset_interrupt_capability(adapter);
+ iavf_free_q_vectors(adapter);
+ iavf_free_queues(adapter);
+ memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);
+ iavf_shutdown_adminq(&adapter->hw);
+ adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
+ iavf_change_state(adapter, __IAVF_DOWN);
+ wake_up(&adapter->down_waitqueue);
+ dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
+}
+
+/**
+ * iavf_reset_task - Call-back task to handle hardware reset
+ * @work: pointer to work_struct
+ *
+ * During reset we need to shut down and reinitialize the admin queue
+ * before we can use it to communicate with the PF again. We also clear
+ * and reinit the rings because that context is lost as well.
+ **/
+static void iavf_reset_task(struct work_struct *work)
+{
+ struct iavf_adapter *adapter = container_of(work,
+ struct iavf_adapter,
+ reset_task);
+ struct virtchnl_vf_resource *vfres = adapter->vf_res;
+ struct net_device *netdev = adapter->netdev;
+ struct iavf_hw *hw = &adapter->hw;
+ struct iavf_mac_filter *f, *ftmp;
+ struct iavf_cloud_filter *cf;
+ enum iavf_status status;
+ u32 reg_val;
+ int i = 0, err;
+ bool running;
+
+ /* When device is being removed it doesn't make sense to run the reset
+ * task, just return in such a case.
+ */
+ if (!mutex_trylock(&adapter->crit_lock)) {
+ if (adapter->state != __IAVF_REMOVE)
+ queue_work(adapter->wq, &adapter->reset_task);
+
+ return;
+ }
+
+ while (!mutex_trylock(&adapter->client_lock))
+ usleep_range(500, 1000);
+ if (CLIENT_ENABLED(adapter)) {
+ adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN |
+ IAVF_FLAG_CLIENT_NEEDS_CLOSE |
+ IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS |
+ IAVF_FLAG_SERVICE_CLIENT_REQUESTED);
+ cancel_delayed_work_sync(&adapter->client_task);
+ iavf_notify_client_close(&adapter->vsi, true);
+ }
+ iavf_misc_irq_disable(adapter);
+ if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
+ adapter->flags &= ~IAVF_FLAG_RESET_NEEDED;
+ /* Restart the AQ here. If we have been reset but didn't
+ * detect it, or if the PF had to reinit, our AQ will be hosed.
+ */
+ iavf_shutdown_adminq(hw);
+ iavf_init_adminq(hw);
+ iavf_request_reset(adapter);
+ }
+ adapter->flags |= IAVF_FLAG_RESET_PENDING;
+
+ /* poll until we see the reset actually happen */
+ for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) {
+ reg_val = rd32(hw, IAVF_VF_ARQLEN1) &
+ IAVF_VF_ARQLEN1_ARQENABLE_MASK;
+ if (!reg_val)
+ break;
+ usleep_range(5000, 10000);
+ }
+ if (i == IAVF_RESET_WAIT_DETECTED_COUNT) {
+ dev_info(&adapter->pdev->dev, "Never saw reset\n");
+ goto continue_reset; /* act like the reset happened */
+ }
+
+ /* wait until the reset is complete and the PF is responding to us */
+ for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
+ /* sleep first to make sure a minimum wait time is met */
+ msleep(IAVF_RESET_WAIT_MS);
+
+ reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
+ IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
+ if (reg_val == VIRTCHNL_VFR_VFACTIVE)
+ break;
+ }
+
+ pci_set_master(adapter->pdev);
+ pci_restore_msi_state(adapter->pdev);
+
+ if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
+ dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
+ reg_val);
+ iavf_disable_vf(adapter);
+ mutex_unlock(&adapter->client_lock);
+ mutex_unlock(&adapter->crit_lock);
+ return; /* Do not attempt to reinit. It's dead, Jim. */
+ }
+
+continue_reset:
+ /* We don't use netif_running() because it may be true prior to
+ * ndo_open() returning, so we can't assume it means all our open
+ * tasks have finished, since we're not holding the rtnl_lock here.
+ */
+ running = adapter->state == __IAVF_RUNNING;
+
+ if (running) {
+ netif_carrier_off(netdev);
+ netif_tx_stop_all_queues(netdev);
+ adapter->link_up = false;
+ iavf_napi_disable_all(adapter);
+ }
+ iavf_irq_disable(adapter);
+
+ iavf_change_state(adapter, __IAVF_RESETTING);
+ adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
+
+ /* free the Tx/Rx rings and descriptors, might be better to just
+ * re-use them sometime in the future
+ */
+ iavf_free_all_rx_resources(adapter);
+ iavf_free_all_tx_resources(adapter);
+
+ adapter->flags |= IAVF_FLAG_QUEUES_DISABLED;
+ /* kill and reinit the admin queue */
+ iavf_shutdown_adminq(hw);
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+ status = iavf_init_adminq(hw);
+ if (status) {
+ dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",
+ status);
+ goto reset_err;
+ }
+ adapter->aq_required = 0;
+
+ if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
+ (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
+ err = iavf_reinit_interrupt_scheme(adapter, running);
+ if (err)
+ goto reset_err;
+ }
+
+ if (RSS_AQ(adapter)) {
+ adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
+ } else {
+ err = iavf_init_rss(adapter);
+ if (err)
+ goto reset_err;
+ }
+
+ adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG;
+ /* always set since VIRTCHNL_OP_GET_VF_RESOURCES has not been
+ * sent/received yet, so VLAN_V2_ALLOWED() cannot is not reliable here,
+ * however the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS won't be sent until
+ * VIRTCHNL_OP_GET_VF_RESOURCES and VIRTCHNL_VF_OFFLOAD_VLAN_V2 have
+ * been successfully sent and negotiated
+ */
+ adapter->aq_required |= IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS;
+ adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ /* Delete filter for the current MAC address, it could have
+ * been changed by the PF via administratively set MAC.
+ * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES.
+ */
+ list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
+ if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) {
+ list_del(&f->list);
+ kfree(f);
+ }
+ }
+ /* re-add all MAC filters */
+ list_for_each_entry(f, &adapter->mac_filter_list, list) {
+ f->add = true;
+ }
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ /* check if TCs are running and re-add all cloud filters */
+ spin_lock_bh(&adapter->cloud_filter_list_lock);
+ if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
+ adapter->num_tc) {
+ list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
+ cf->add = true;
+ }
+ }
+ spin_unlock_bh(&adapter->cloud_filter_list_lock);
+
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
+ iavf_misc_irq_enable(adapter);
+
+ mod_delayed_work(adapter->wq, &adapter->watchdog_task, 2);
+
+ /* We were running when the reset started, so we need to restore some
+ * state here.
+ */
+ if (running) {
+ /* allocate transmit descriptors */
+ err = iavf_setup_all_tx_resources(adapter);
+ if (err)
+ goto reset_err;
+
+ /* allocate receive descriptors */
+ err = iavf_setup_all_rx_resources(adapter);
+ if (err)
+ goto reset_err;
+
+ if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
+ (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
+ err = iavf_request_traffic_irqs(adapter, netdev->name);
+ if (err)
+ goto reset_err;
+
+ adapter->flags &= ~IAVF_FLAG_REINIT_MSIX_NEEDED;
+ }
+
+ iavf_configure(adapter);
+
+ /* iavf_up_complete() will switch device back
+ * to __IAVF_RUNNING
+ */
+ iavf_up_complete(adapter);
+
+ iavf_irq_enable(adapter, true);
+ } else {
+ iavf_change_state(adapter, __IAVF_DOWN);
+ wake_up(&adapter->down_waitqueue);
+ }
+
+ adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
+
+ wake_up(&adapter->reset_waitqueue);
+ mutex_unlock(&adapter->client_lock);
+ mutex_unlock(&adapter->crit_lock);
+
+ return;
+reset_err:
+ if (running) {
+ set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
+ iavf_free_traffic_irqs(adapter);
+ }
+ iavf_disable_vf(adapter);
+
+ mutex_unlock(&adapter->client_lock);
+ mutex_unlock(&adapter->crit_lock);
+ dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
+}
+
+/**
+ * iavf_adminq_task - worker thread to clean the admin queue
+ * @work: pointer to work_struct containing our data
+ **/
+static void iavf_adminq_task(struct work_struct *work)
+{
+ struct iavf_adapter *adapter =
+ container_of(work, struct iavf_adapter, adminq_task);
+ struct iavf_hw *hw = &adapter->hw;
+ struct iavf_arq_event_info event;
+ enum virtchnl_ops v_op;
+ enum iavf_status ret, v_ret;
+ u32 val, oldval;
+ u16 pending;
+
+ if (!mutex_trylock(&adapter->crit_lock)) {
+ if (adapter->state == __IAVF_REMOVE)
+ return;
+
+ queue_work(adapter->wq, &adapter->adminq_task);
+ goto out;
+ }
+
+ if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
+ goto unlock;
+
+ event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
+ event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
+ if (!event.msg_buf)
+ goto unlock;
+
+ do {
+ ret = iavf_clean_arq_element(hw, &event, &pending);
+ v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
+ v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low);
+
+ if (ret || !v_op)
+ break; /* No event to process or error cleaning ARQ */
+
+ iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,
+ event.msg_len);
+ if (pending != 0)
+ memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE);
+ } while (pending);
+
+ if (iavf_is_reset_in_progress(adapter))
+ goto freedom;
+
+ /* check for error indications */
+ val = rd32(hw, hw->aq.arq.len);
+ if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */
+ goto freedom;
+ oldval = val;
+ if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) {
+ dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");
+ val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK;
+ }
+ if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) {
+ dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");
+ val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK;
+ }
+ if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) {
+ dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");
+ val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK;
+ }
+ if (oldval != val)
+ wr32(hw, hw->aq.arq.len, val);
+
+ val = rd32(hw, hw->aq.asq.len);
+ oldval = val;
+ if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) {
+ dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");
+ val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK;
+ }
+ if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) {
+ dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");
+ val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK;
+ }
+ if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {
+ dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");
+ val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK;
+ }
+ if (oldval != val)
+ wr32(hw, hw->aq.asq.len, val);
+
+freedom:
+ kfree(event.msg_buf);
+unlock:
+ mutex_unlock(&adapter->crit_lock);
+out:
+ /* re-enable Admin queue interrupt cause */
+ iavf_misc_irq_enable(adapter);
+}
+
+/**
+ * iavf_client_task - worker thread to perform client work
+ * @work: pointer to work_struct containing our data
+ *
+ * This task handles client interactions. Because client calls can be
+ * reentrant, we can't handle them in the watchdog.
+ **/
+static void iavf_client_task(struct work_struct *work)
+{
+ struct iavf_adapter *adapter =
+ container_of(work, struct iavf_adapter, client_task.work);
+
+ /* If we can't get the client bit, just give up. We'll be rescheduled
+ * later.
+ */
+
+ if (!mutex_trylock(&adapter->client_lock))
+ return;
+
+ if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) {
+ iavf_client_subtask(adapter);
+ adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
+ goto out;
+ }
+ if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) {
+ iavf_notify_client_l2_params(&adapter->vsi);
+ adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS;
+ goto out;
+ }
+ if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) {
+ iavf_notify_client_close(&adapter->vsi, false);
+ adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE;
+ goto out;
+ }
+ if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) {
+ iavf_notify_client_open(&adapter->vsi);
+ adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN;
+ }
+out:
+ mutex_unlock(&adapter->client_lock);
+}
+
+/**
+ * iavf_free_all_tx_resources - Free Tx Resources for All Queues
+ * @adapter: board private structure
+ *
+ * Free all transmit software resources
+ **/
+void iavf_free_all_tx_resources(struct iavf_adapter *adapter)
+{
+ int i;
+
+ if (!adapter->tx_rings)
+ return;
+
+ for (i = 0; i < adapter->num_active_queues; i++)
+ if (adapter->tx_rings[i].desc)
+ iavf_free_tx_resources(&adapter->tx_rings[i]);
+}
+
+/**
+ * iavf_setup_all_tx_resources - allocate all queues Tx resources
+ * @adapter: board private structure
+ *
+ * If this function returns with an error, then it's possible one or
+ * more of the rings is populated (while the rest are not). It is the
+ * callers duty to clean those orphaned rings.
+ *
+ * Return 0 on success, negative on failure
+ **/
+static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)
+{
+ int i, err = 0;
+
+ for (i = 0; i < adapter->num_active_queues; i++) {
+ adapter->tx_rings[i].count = adapter->tx_desc_count;
+ err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]);
+ if (!err)
+ continue;
+ dev_err(&adapter->pdev->dev,
+ "Allocation for Tx Queue %u failed\n", i);
+ break;
+ }
+
+ return err;
+}
+
+/**
+ * iavf_setup_all_rx_resources - allocate all queues Rx resources
+ * @adapter: board private structure
+ *
+ * If this function returns with an error, then it's possible one or
+ * more of the rings is populated (while the rest are not). It is the
+ * callers duty to clean those orphaned rings.
+ *
+ * Return 0 on success, negative on failure
+ **/
+static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)
+{
+ int i, err = 0;
+
+ for (i = 0; i < adapter->num_active_queues; i++) {
+ adapter->rx_rings[i].count = adapter->rx_desc_count;
+ err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]);
+ if (!err)
+ continue;
+ dev_err(&adapter->pdev->dev,
+ "Allocation for Rx Queue %u failed\n", i);
+ break;
+ }
+ return err;
+}
+
+/**
+ * iavf_free_all_rx_resources - Free Rx Resources for All Queues
+ * @adapter: board private structure
+ *
+ * Free all receive software resources
+ **/
+void iavf_free_all_rx_resources(struct iavf_adapter *adapter)
+{
+ int i;
+
+ if (!adapter->rx_rings)
+ return;
+
+ for (i = 0; i < adapter->num_active_queues; i++)
+ if (adapter->rx_rings[i].desc)
+ iavf_free_rx_resources(&adapter->rx_rings[i]);
+}
+
+/**
+ * iavf_validate_tx_bandwidth - validate the max Tx bandwidth
+ * @adapter: board private structure
+ * @max_tx_rate: max Tx bw for a tc
+ **/
+static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,
+ u64 max_tx_rate)
+{
+ int speed = 0, ret = 0;
+
+ if (ADV_LINK_SUPPORT(adapter)) {
+ if (adapter->link_speed_mbps < U32_MAX) {
+ speed = adapter->link_speed_mbps;
+ goto validate_bw;
+ } else {
+ dev_err(&adapter->pdev->dev, "Unknown link speed\n");
+ return -EINVAL;
+ }
+ }
+
+ switch (adapter->link_speed) {
+ case VIRTCHNL_LINK_SPEED_40GB:
+ speed = SPEED_40000;
+ break;
+ case VIRTCHNL_LINK_SPEED_25GB:
+ speed = SPEED_25000;
+ break;
+ case VIRTCHNL_LINK_SPEED_20GB:
+ speed = SPEED_20000;
+ break;
+ case VIRTCHNL_LINK_SPEED_10GB:
+ speed = SPEED_10000;
+ break;
+ case VIRTCHNL_LINK_SPEED_5GB:
+ speed = SPEED_5000;
+ break;
+ case VIRTCHNL_LINK_SPEED_2_5GB:
+ speed = SPEED_2500;
+ break;
+ case VIRTCHNL_LINK_SPEED_1GB:
+ speed = SPEED_1000;
+ break;
+ case VIRTCHNL_LINK_SPEED_100MB:
+ speed = SPEED_100;
+ break;
+ default:
+ break;
+ }
+
+validate_bw:
+ if (max_tx_rate > speed) {
+ dev_err(&adapter->pdev->dev,
+ "Invalid tx rate specified\n");
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+
+/**
+ * iavf_validate_ch_config - validate queue mapping info
+ * @adapter: board private structure
+ * @mqprio_qopt: queue parameters
+ *
+ * This function validates if the config provided by the user to
+ * configure queue channels is valid or not. Returns 0 on a valid
+ * config.
+ **/
+static int iavf_validate_ch_config(struct iavf_adapter *adapter,
+ struct tc_mqprio_qopt_offload *mqprio_qopt)
+{
+ u64 total_max_rate = 0;
+ u32 tx_rate_rem = 0;
+ int i, num_qps = 0;
+ u64 tx_rate = 0;
+ int ret = 0;
+
+ if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS ||
+ mqprio_qopt->qopt.num_tc < 1)
+ return -EINVAL;
+
+ for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {
+ if (!mqprio_qopt->qopt.count[i] ||
+ mqprio_qopt->qopt.offset[i] != num_qps)
+ return -EINVAL;
+ if (mqprio_qopt->min_rate[i]) {
+ dev_err(&adapter->pdev->dev,
+ "Invalid min tx rate (greater than 0) specified for TC%d\n",
+ i);
+ return -EINVAL;
+ }
+
+ /* convert to Mbps */
+ tx_rate = div_u64(mqprio_qopt->max_rate[i],
+ IAVF_MBPS_DIVISOR);
+
+ if (mqprio_qopt->max_rate[i] &&
+ tx_rate < IAVF_MBPS_QUANTA) {
+ dev_err(&adapter->pdev->dev,
+ "Invalid max tx rate for TC%d, minimum %dMbps\n",
+ i, IAVF_MBPS_QUANTA);
+ return -EINVAL;
+ }
+
+ (void)div_u64_rem(tx_rate, IAVF_MBPS_QUANTA, &tx_rate_rem);
+
+ if (tx_rate_rem != 0) {
+ dev_err(&adapter->pdev->dev,
+ "Invalid max tx rate for TC%d, not divisible by %d\n",
+ i, IAVF_MBPS_QUANTA);
+ return -EINVAL;
+ }
+
+ total_max_rate += tx_rate;
+ num_qps += mqprio_qopt->qopt.count[i];
+ }
+ if (num_qps > adapter->num_active_queues) {
+ dev_err(&adapter->pdev->dev,
+ "Cannot support requested number of queues\n");
+ return -EINVAL;
+ }
+
+ ret = iavf_validate_tx_bandwidth(adapter, total_max_rate);
+ return ret;
+}
+
+/**
+ * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes
+ * @adapter: board private structure
+ **/
+static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter)
+{
+ struct iavf_cloud_filter *cf, *cftmp;
+
+ spin_lock_bh(&adapter->cloud_filter_list_lock);
+ list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
+ list) {
+ list_del(&cf->list);
+ kfree(cf);
+ adapter->num_cloud_filters--;
+ }
+ spin_unlock_bh(&adapter->cloud_filter_list_lock);
+}
+
+/**
+ * __iavf_setup_tc - configure multiple traffic classes
+ * @netdev: network interface device structure
+ * @type_data: tc offload data
+ *
+ * This function processes the config information provided by the
+ * user to configure traffic classes/queue channels and packages the
+ * information to request the PF to setup traffic classes.
+ *
+ * Returns 0 on success.
+ **/
+static int __iavf_setup_tc(struct net_device *netdev, void *type_data)
+{
+ struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ struct virtchnl_vf_resource *vfres = adapter->vf_res;
+ u8 num_tc = 0, total_qps = 0;
+ int ret = 0, netdev_tc = 0;
+ u64 max_tx_rate;
+ u16 mode;
+ int i;
+
+ num_tc = mqprio_qopt->qopt.num_tc;
+ mode = mqprio_qopt->mode;
+
+ /* delete queue_channel */
+ if (!mqprio_qopt->qopt.hw) {
+ if (adapter->ch_config.state == __IAVF_TC_RUNNING) {
+ /* reset the tc configuration */
+ netdev_reset_tc(netdev);
+ adapter->num_tc = 0;
+ netif_tx_stop_all_queues(netdev);
+ netif_tx_disable(netdev);
+ iavf_del_all_cloud_filters(adapter);
+ adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS;
+ total_qps = adapter->orig_num_active_queues;
+ goto exit;
+ } else {
+ return -EINVAL;
+ }
+ }
+
+ /* add queue channel */
+ if (mode == TC_MQPRIO_MODE_CHANNEL) {
+ if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {
+ dev_err(&adapter->pdev->dev, "ADq not supported\n");
+ return -EOPNOTSUPP;
+ }
+ if (adapter->ch_config.state != __IAVF_TC_INVALID) {
+ dev_err(&adapter->pdev->dev, "TC configuration already exists\n");
+ return -EINVAL;
+ }
+
+ ret = iavf_validate_ch_config(adapter, mqprio_qopt);
+ if (ret)
+ return ret;
+ /* Return if same TC config is requested */
+ if (adapter->num_tc == num_tc)
+ return 0;
+ adapter->num_tc = num_tc;
+
+ for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
+ if (i < num_tc) {
+ adapter->ch_config.ch_info[i].count =
+ mqprio_qopt->qopt.count[i];
+ adapter->ch_config.ch_info[i].offset =
+ mqprio_qopt->qopt.offset[i];
+ total_qps += mqprio_qopt->qopt.count[i];
+ max_tx_rate = mqprio_qopt->max_rate[i];
+ /* convert to Mbps */
+ max_tx_rate = div_u64(max_tx_rate,
+ IAVF_MBPS_DIVISOR);
+ adapter->ch_config.ch_info[i].max_tx_rate =
+ max_tx_rate;
+ } else {
+ adapter->ch_config.ch_info[i].count = 1;
+ adapter->ch_config.ch_info[i].offset = 0;
+ }
+ }
+
+ /* Take snapshot of original config such as "num_active_queues"
+ * It is used later when delete ADQ flow is exercised, so that
+ * once delete ADQ flow completes, VF shall go back to its
+ * original queue configuration
+ */
+
+ adapter->orig_num_active_queues = adapter->num_active_queues;
+
+ /* Store queue info based on TC so that VF gets configured
+ * with correct number of queues when VF completes ADQ config
+ * flow
+ */
+ adapter->ch_config.total_qps = total_qps;
+
+ netif_tx_stop_all_queues(netdev);
+ netif_tx_disable(netdev);
+ adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS;
+ netdev_reset_tc(netdev);
+ /* Report the tc mapping up the stack */
+ netdev_set_num_tc(adapter->netdev, num_tc);
+ for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
+ u16 qcount = mqprio_qopt->qopt.count[i];
+ u16 qoffset = mqprio_qopt->qopt.offset[i];
+
+ if (i < num_tc)
+ netdev_set_tc_queue(netdev, netdev_tc++, qcount,
+ qoffset);
+ }
+ }
+exit:
+ if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
+ return 0;
+
+ netif_set_real_num_rx_queues(netdev, total_qps);
+ netif_set_real_num_tx_queues(netdev, total_qps);
+
+ return ret;
+}
+
+/**
+ * iavf_parse_cls_flower - Parse tc flower filters provided by kernel
+ * @adapter: board private structure
+ * @f: pointer to struct flow_cls_offload
+ * @filter: pointer to cloud filter structure
+ */
+static int iavf_parse_cls_flower(struct iavf_adapter *adapter,
+ struct flow_cls_offload *f,
+ struct iavf_cloud_filter *filter)
+{
+ struct flow_rule *rule = flow_cls_offload_flow_rule(f);
+ struct flow_dissector *dissector = rule->match.dissector;
+ u16 n_proto_mask = 0;
+ u16 n_proto_key = 0;
+ u8 field_flags = 0;
+ u16 addr_type = 0;
+ u16 n_proto = 0;
+ int i = 0;
+ struct virtchnl_filter *vf = &filter->f;
+
+ if (dissector->used_keys &
+ ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
+ BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
+ BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
+ BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
+ BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
+ BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
+ BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
+ BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
+ dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%llx\n",
+ dissector->used_keys);
+ return -EOPNOTSUPP;
+ }
+
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
+ struct flow_match_enc_keyid match;
+
+ flow_rule_match_enc_keyid(rule, &match);
+ if (match.mask->keyid != 0)
+ field_flags |= IAVF_CLOUD_FIELD_TEN_ID;
+ }
+
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
+ struct flow_match_basic match;
+
+ flow_rule_match_basic(rule, &match);
+ n_proto_key = ntohs(match.key->n_proto);
+ n_proto_mask = ntohs(match.mask->n_proto);
+
+ if (n_proto_key == ETH_P_ALL) {
+ n_proto_key = 0;
+ n_proto_mask = 0;
+ }
+ n_proto = n_proto_key & n_proto_mask;
+ if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)
+ return -EINVAL;
+ if (n_proto == ETH_P_IPV6) {
+ /* specify flow type as TCP IPv6 */
+ vf->flow_type = VIRTCHNL_TCP_V6_FLOW;
+ }
+
+ if (match.key->ip_proto != IPPROTO_TCP) {
+ dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");
+ return -EINVAL;
+ }
+ }
+
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
+ struct flow_match_eth_addrs match;
+
+ flow_rule_match_eth_addrs(rule, &match);
+
+ /* use is_broadcast and is_zero to check for all 0xf or 0 */
+ if (!is_zero_ether_addr(match.mask->dst)) {
+ if (is_broadcast_ether_addr(match.mask->dst)) {
+ field_flags |= IAVF_CLOUD_FIELD_OMAC;
+ } else {
+ dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",
+ match.mask->dst);
+ return -EINVAL;
+ }
+ }
+
+ if (!is_zero_ether_addr(match.mask->src)) {
+ if (is_broadcast_ether_addr(match.mask->src)) {
+ field_flags |= IAVF_CLOUD_FIELD_IMAC;
+ } else {
+ dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",
+ match.mask->src);
+ return -EINVAL;
+ }
+ }
+
+ if (!is_zero_ether_addr(match.key->dst))
+ if (is_valid_ether_addr(match.key->dst) ||
+ is_multicast_ether_addr(match.key->dst)) {
+ /* set the mask if a valid dst_mac address */
+ for (i = 0; i < ETH_ALEN; i++)
+ vf->mask.tcp_spec.dst_mac[i] |= 0xff;
+ ether_addr_copy(vf->data.tcp_spec.dst_mac,
+ match.key->dst);
+ }
+
+ if (!is_zero_ether_addr(match.key->src))
+ if (is_valid_ether_addr(match.key->src) ||
+ is_multicast_ether_addr(match.key->src)) {
+ /* set the mask if a valid dst_mac address */
+ for (i = 0; i < ETH_ALEN; i++)
+ vf->mask.tcp_spec.src_mac[i] |= 0xff;
+ ether_addr_copy(vf->data.tcp_spec.src_mac,
+ match.key->src);
+ }
+ }
+
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
+ struct flow_match_vlan match;
+
+ flow_rule_match_vlan(rule, &match);
+ if (match.mask->vlan_id) {
+ if (match.mask->vlan_id == VLAN_VID_MASK) {
+ field_flags |= IAVF_CLOUD_FIELD_IVLAN;
+ } else {
+ dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",
+ match.mask->vlan_id);
+ return -EINVAL;
+ }
+ }
+ vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);
+ vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id);
+ }
+
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
+ struct flow_match_control match;
+
+ flow_rule_match_control(rule, &match);
+ addr_type = match.key->addr_type;
+ }
+
+ if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
+ struct flow_match_ipv4_addrs match;
+
+ flow_rule_match_ipv4_addrs(rule, &match);
+ if (match.mask->dst) {
+ if (match.mask->dst == cpu_to_be32(0xffffffff)) {
+ field_flags |= IAVF_CLOUD_FIELD_IIP;
+ } else {
+ dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",
+ be32_to_cpu(match.mask->dst));
+ return -EINVAL;
+ }
+ }
+
+ if (match.mask->src) {
+ if (match.mask->src == cpu_to_be32(0xffffffff)) {
+ field_flags |= IAVF_CLOUD_FIELD_IIP;
+ } else {
+ dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",
+ be32_to_cpu(match.mask->src));
+ return -EINVAL;
+ }
+ }
+
+ if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) {
+ dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");
+ return -EINVAL;
+ }
+ if (match.key->dst) {
+ vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);
+ vf->data.tcp_spec.dst_ip[0] = match.key->dst;
+ }
+ if (match.key->src) {
+ vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);
+ vf->data.tcp_spec.src_ip[0] = match.key->src;
+ }
+ }
+
+ if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
+ struct flow_match_ipv6_addrs match;
+
+ flow_rule_match_ipv6_addrs(rule, &match);
+
+ /* validate mask, make sure it is not IPV6_ADDR_ANY */
+ if (ipv6_addr_any(&match.mask->dst)) {
+ dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",
+ IPV6_ADDR_ANY);
+ return -EINVAL;
+ }
+
+ /* src and dest IPv6 address should not be LOOPBACK
+ * (0:0:0:0:0:0:0:1) which can be represented as ::1
+ */
+ if (ipv6_addr_loopback(&match.key->dst) ||
+ ipv6_addr_loopback(&match.key->src)) {
+ dev_err(&adapter->pdev->dev,
+ "ipv6 addr should not be loopback\n");
+ return -EINVAL;
+ }
+ if (!ipv6_addr_any(&match.mask->dst) ||
+ !ipv6_addr_any(&match.mask->src))
+ field_flags |= IAVF_CLOUD_FIELD_IIP;
+
+ for (i = 0; i < 4; i++)
+ vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);
+ memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32,
+ sizeof(vf->data.tcp_spec.dst_ip));
+ for (i = 0; i < 4; i++)
+ vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);
+ memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32,
+ sizeof(vf->data.tcp_spec.src_ip));
+ }
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
+ struct flow_match_ports match;
+
+ flow_rule_match_ports(rule, &match);
+ if (match.mask->src) {
+ if (match.mask->src == cpu_to_be16(0xffff)) {
+ field_flags |= IAVF_CLOUD_FIELD_IIP;
+ } else {
+ dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",
+ be16_to_cpu(match.mask->src));
+ return -EINVAL;
+ }
+ }
+
+ if (match.mask->dst) {
+ if (match.mask->dst == cpu_to_be16(0xffff)) {
+ field_flags |= IAVF_CLOUD_FIELD_IIP;
+ } else {
+ dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",
+ be16_to_cpu(match.mask->dst));
+ return -EINVAL;
+ }
+ }
+ if (match.key->dst) {
+ vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);
+ vf->data.tcp_spec.dst_port = match.key->dst;
+ }
+
+ if (match.key->src) {
+ vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);
+ vf->data.tcp_spec.src_port = match.key->src;
+ }
+ }
+ vf->field_flags = field_flags;
+
+ return 0;
+}
+
+/**
+ * iavf_handle_tclass - Forward to a traffic class on the device
+ * @adapter: board private structure
+ * @tc: traffic class index on the device
+ * @filter: pointer to cloud filter structure
+ */
+static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc,
+ struct iavf_cloud_filter *filter)
+{
+ if (tc == 0)
+ return 0;
+ if (tc < adapter->num_tc) {
+ if (!filter->f.data.tcp_spec.dst_port) {
+ dev_err(&adapter->pdev->dev,
+ "Specify destination port to redirect to traffic class other than TC0\n");
+ return -EINVAL;
+ }
+ }
+ /* redirect to a traffic class on the same device */
+ filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;
+ filter->f.action_meta = tc;
+ return 0;
+}
+
+/**
+ * iavf_find_cf - Find the cloud filter in the list
+ * @adapter: Board private structure
+ * @cookie: filter specific cookie
+ *
+ * Returns ptr to the filter object or NULL. Must be called while holding the
+ * cloud_filter_list_lock.
+ */
+static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter,
+ unsigned long *cookie)
+{
+ struct iavf_cloud_filter *filter = NULL;
+
+ if (!cookie)
+ return NULL;
+
+ list_for_each_entry(filter, &adapter->cloud_filter_list, list) {
+ if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
+ return filter;
+ }
+ return NULL;
+}
+
+/**
+ * iavf_configure_clsflower - Add tc flower filters
+ * @adapter: board private structure
+ * @cls_flower: Pointer to struct flow_cls_offload
+ */
+static int iavf_configure_clsflower(struct iavf_adapter *adapter,
+ struct flow_cls_offload *cls_flower)
+{
+ int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
+ struct iavf_cloud_filter *filter = NULL;
+ int err = -EINVAL, count = 50;
+
+ if (tc < 0) {
+ dev_err(&adapter->pdev->dev, "Invalid traffic class\n");
+ return -EINVAL;
+ }
+
+ filter = kzalloc(sizeof(*filter), GFP_KERNEL);
+ if (!filter)
+ return -ENOMEM;
+
+ while (!mutex_trylock(&adapter->crit_lock)) {
+ if (--count == 0) {
+ kfree(filter);
+ return err;
+ }
+ udelay(1);
+ }
+
+ filter->cookie = cls_flower->cookie;
+
+ /* bail out here if filter already exists */
+ spin_lock_bh(&adapter->cloud_filter_list_lock);
+ if (iavf_find_cf(adapter, &cls_flower->cookie)) {
+ dev_err(&adapter->pdev->dev, "Failed to add TC Flower filter, it already exists\n");
+ err = -EEXIST;
+ goto spin_unlock;
+ }
+ spin_unlock_bh(&adapter->cloud_filter_list_lock);
+
+ /* set the mask to all zeroes to begin with */
+ memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));
+ /* start out with flow type and eth type IPv4 to begin with */
+ filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
+ err = iavf_parse_cls_flower(adapter, cls_flower, filter);
+ if (err)
+ goto err;
+
+ err = iavf_handle_tclass(adapter, tc, filter);
+ if (err)
+ goto err;
+
+ /* add filter to the list */
+ spin_lock_bh(&adapter->cloud_filter_list_lock);
+ list_add_tail(&filter->list, &adapter->cloud_filter_list);
+ adapter->num_cloud_filters++;
+ filter->add = true;
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
+spin_unlock:
+ spin_unlock_bh(&adapter->cloud_filter_list_lock);
+err:
+ if (err)
+ kfree(filter);
+
+ mutex_unlock(&adapter->crit_lock);
+ return err;
+}
+
+/**
+ * iavf_delete_clsflower - Remove tc flower filters
+ * @adapter: board private structure
+ * @cls_flower: Pointer to struct flow_cls_offload
+ */
+static int iavf_delete_clsflower(struct iavf_adapter *adapter,
+ struct flow_cls_offload *cls_flower)
+{
+ struct iavf_cloud_filter *filter = NULL;
+ int err = 0;
+
+ spin_lock_bh(&adapter->cloud_filter_list_lock);
+ filter = iavf_find_cf(adapter, &cls_flower->cookie);
+ if (filter) {
+ filter->del = true;
+ adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
+ } else {
+ err = -EINVAL;
+ }
+ spin_unlock_bh(&adapter->cloud_filter_list_lock);
+
+ return err;
+}
+
+/**
+ * iavf_setup_tc_cls_flower - flower classifier offloads
+ * @adapter: board private structure
+ * @cls_flower: pointer to flow_cls_offload struct with flow info
+ */
+static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter,
+ struct flow_cls_offload *cls_flower)
+{
+ switch (cls_flower->command) {
+ case FLOW_CLS_REPLACE:
+ return iavf_configure_clsflower(adapter, cls_flower);
+ case FLOW_CLS_DESTROY:
+ return iavf_delete_clsflower(adapter, cls_flower);
+ case FLOW_CLS_STATS:
+ return -EOPNOTSUPP;
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+/**
+ * iavf_setup_tc_block_cb - block callback for tc
+ * @type: type of offload
+ * @type_data: offload data
+ * @cb_priv:
+ *
+ * This function is the block callback for traffic classes
+ **/
+static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
+ void *cb_priv)
+{
+ struct iavf_adapter *adapter = cb_priv;
+
+ if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
+ return -EOPNOTSUPP;
+
+ switch (type) {
+ case TC_SETUP_CLSFLOWER:
+ return iavf_setup_tc_cls_flower(cb_priv, type_data);
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static LIST_HEAD(iavf_block_cb_list);
+
+/**
+ * iavf_setup_tc - configure multiple traffic classes
+ * @netdev: network interface device structure
+ * @type: type of offload
+ * @type_data: tc offload data
+ *
+ * This function is the callback to ndo_setup_tc in the
+ * netdev_ops.
+ *
+ * Returns 0 on success
+ **/
+static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
+ void *type_data)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ switch (type) {
+ case TC_SETUP_QDISC_MQPRIO:
+ return __iavf_setup_tc(netdev, type_data);
+ case TC_SETUP_BLOCK:
+ return flow_block_cb_setup_simple(type_data,
+ &iavf_block_cb_list,
+ iavf_setup_tc_block_cb,
+ adapter, adapter, true);
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+/**
+ * iavf_restore_fdir_filters
+ * @adapter: board private structure
+ *
+ * Restore existing FDIR filters when VF netdev comes back up.
+ **/
+static void iavf_restore_fdir_filters(struct iavf_adapter *adapter)
+{
+ struct iavf_fdir_fltr *f;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry(f, &adapter->fdir_list_head, list) {
+ if (f->state == IAVF_FDIR_FLTR_DIS_REQUEST) {
+ /* Cancel a request, keep filter as active */
+ f->state = IAVF_FDIR_FLTR_ACTIVE;
+ } else if (f->state == IAVF_FDIR_FLTR_DIS_PENDING ||
+ f->state == IAVF_FDIR_FLTR_INACTIVE) {
+ /* Add filters which are inactive or have a pending
+ * request to PF to be deleted
+ */
+ f->state = IAVF_FDIR_FLTR_ADD_REQUEST;
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_FDIR_FILTER;
+ }
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+}
+
+/**
+ * iavf_open - Called when a network interface is made active
+ * @netdev: network interface device structure
+ *
+ * Returns 0 on success, negative value on failure
+ *
+ * The open entry point is called when a network interface is made
+ * active by the system (IFF_UP). At this point all resources needed
+ * for transmit and receive operations are allocated, the interrupt
+ * handler is registered with the OS, the watchdog is started,
+ * and the stack is notified that the interface is ready.
+ **/
+static int iavf_open(struct net_device *netdev)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ int err;
+
+ if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {
+ dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
+ return -EIO;
+ }
+
+ while (!mutex_trylock(&adapter->crit_lock)) {
+ /* If we are in __IAVF_INIT_CONFIG_ADAPTER state the crit_lock
+ * is already taken and iavf_open is called from an upper
+ * device's notifier reacting on NETDEV_REGISTER event.
+ * We have to leave here to avoid dead lock.
+ */
+ if (adapter->state == __IAVF_INIT_CONFIG_ADAPTER)
+ return -EBUSY;
+
+ usleep_range(500, 1000);
+ }
+
+ if (adapter->state != __IAVF_DOWN) {
+ err = -EBUSY;
+ goto err_unlock;
+ }
+
+ if (adapter->state == __IAVF_RUNNING &&
+ !test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) {
+ dev_dbg(&adapter->pdev->dev, "VF is already open.\n");
+ err = 0;
+ goto err_unlock;
+ }
+
+ /* allocate transmit descriptors */
+ err = iavf_setup_all_tx_resources(adapter);
+ if (err)
+ goto err_setup_tx;
+
+ /* allocate receive descriptors */
+ err = iavf_setup_all_rx_resources(adapter);
+ if (err)
+ goto err_setup_rx;
+
+ /* clear any pending interrupts, may auto mask */
+ err = iavf_request_traffic_irqs(adapter, netdev->name);
+ if (err)
+ goto err_req_irq;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ iavf_add_filter(adapter, adapter->hw.mac.addr);
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ /* Restore filters that were removed with IFF_DOWN */
+ iavf_restore_filters(adapter);
+ iavf_restore_fdir_filters(adapter);
+
+ iavf_configure(adapter);
+
+ iavf_up_complete(adapter);
+
+ iavf_irq_enable(adapter, true);
+
+ mutex_unlock(&adapter->crit_lock);
+
+ return 0;
+
+err_req_irq:
+ iavf_down(adapter);
+ iavf_free_traffic_irqs(adapter);
+err_setup_rx:
+ iavf_free_all_rx_resources(adapter);
+err_setup_tx:
+ iavf_free_all_tx_resources(adapter);
+err_unlock:
+ mutex_unlock(&adapter->crit_lock);
+
+ return err;
+}
+
+/**
+ * iavf_close - Disables a network interface
+ * @netdev: network interface device structure
+ *
+ * Returns 0, this is not allowed to fail
+ *
+ * The close entry point is called when an interface is de-activated
+ * by the OS. The hardware is still under the drivers control, but
+ * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
+ * are freed, along with all transmit and receive resources.
+ **/
+static int iavf_close(struct net_device *netdev)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ u64 aq_to_restore;
+ int status;
+
+ mutex_lock(&adapter->crit_lock);
+
+ if (adapter->state <= __IAVF_DOWN_PENDING) {
+ mutex_unlock(&adapter->crit_lock);
+ return 0;
+ }
+
+ set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
+ if (CLIENT_ENABLED(adapter))
+ adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE;
+ /* We cannot send IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS before
+ * IAVF_FLAG_AQ_DISABLE_QUEUES because in such case there is rtnl
+ * deadlock with adminq_task() until iavf_close timeouts. We must send
+ * IAVF_FLAG_AQ_GET_CONFIG before IAVF_FLAG_AQ_DISABLE_QUEUES to make
+ * disable queues possible for vf. Give only necessary flags to
+ * iavf_down and save other to set them right before iavf_close()
+ * returns, when IAVF_FLAG_AQ_DISABLE_QUEUES will be already sent and
+ * iavf will be in DOWN state.
+ */
+ aq_to_restore = adapter->aq_required;
+ adapter->aq_required &= IAVF_FLAG_AQ_GET_CONFIG;
+
+ /* Remove flags which we do not want to send after close or we want to
+ * send before disable queues.
+ */
+ aq_to_restore &= ~(IAVF_FLAG_AQ_GET_CONFIG |
+ IAVF_FLAG_AQ_ENABLE_QUEUES |
+ IAVF_FLAG_AQ_CONFIGURE_QUEUES |
+ IAVF_FLAG_AQ_ADD_VLAN_FILTER |
+ IAVF_FLAG_AQ_ADD_MAC_FILTER |
+ IAVF_FLAG_AQ_ADD_CLOUD_FILTER |
+ IAVF_FLAG_AQ_ADD_FDIR_FILTER |
+ IAVF_FLAG_AQ_ADD_ADV_RSS_CFG);
+
+ iavf_down(adapter);
+ iavf_change_state(adapter, __IAVF_DOWN_PENDING);
+ iavf_free_traffic_irqs(adapter);
+
+ mutex_unlock(&adapter->crit_lock);
+
+ /* We explicitly don't free resources here because the hardware is
+ * still active and can DMA into memory. Resources are cleared in
+ * iavf_virtchnl_completion() after we get confirmation from the PF
+ * driver that the rings have been stopped.
+ *
+ * Also, we wait for state to transition to __IAVF_DOWN before
+ * returning. State change occurs in iavf_virtchnl_completion() after
+ * VF resources are released (which occurs after PF driver processes and
+ * responds to admin queue commands).
+ */
+
+ status = wait_event_timeout(adapter->down_waitqueue,
+ adapter->state == __IAVF_DOWN,
+ msecs_to_jiffies(500));
+ if (!status)
+ netdev_warn(netdev, "Device resources not yet released\n");
+
+ mutex_lock(&adapter->crit_lock);
+ adapter->aq_required |= aq_to_restore;
+ mutex_unlock(&adapter->crit_lock);
+ return 0;
+}
+
+/**
+ * iavf_change_mtu - Change the Maximum Transfer Unit
+ * @netdev: network interface device structure
+ * @new_mtu: new value for maximum frame size
+ *
+ * Returns 0 on success, negative on failure
+ **/
+static int iavf_change_mtu(struct net_device *netdev, int new_mtu)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ int ret = 0;
+
+ netdev_dbg(netdev, "changing MTU from %d to %d\n",
+ netdev->mtu, new_mtu);
+ netdev->mtu = new_mtu;
+ if (CLIENT_ENABLED(adapter)) {
+ iavf_notify_client_l2_params(&adapter->vsi);
+ adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
+ }
+
+ if (netif_running(netdev)) {
+ iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
+ ret = iavf_wait_for_reset(adapter);
+ if (ret < 0)
+ netdev_warn(netdev, "MTU change interrupted waiting for reset");
+ else if (ret)
+ netdev_warn(netdev, "MTU change timed out waiting for reset");
+ }
+
+ return ret;
+}
+
+/**
+ * iavf_disable_fdir - disable Flow Director and clear existing filters
+ * @adapter: board private structure
+ **/
+static void iavf_disable_fdir(struct iavf_adapter *adapter)
+{
+ struct iavf_fdir_fltr *fdir, *fdirtmp;
+ bool del_filters = false;
+
+ adapter->flags &= ~IAVF_FLAG_FDIR_ENABLED;
+
+ /* remove all Flow Director filters */
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head,
+ list) {
+ if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST ||
+ fdir->state == IAVF_FDIR_FLTR_INACTIVE) {
+ /* Delete filters not registered in PF */
+ list_del(&fdir->list);
+ kfree(fdir);
+ adapter->fdir_active_fltr--;
+ } else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING ||
+ fdir->state == IAVF_FDIR_FLTR_DIS_REQUEST ||
+ fdir->state == IAVF_FDIR_FLTR_ACTIVE) {
+ /* Filters registered in PF, schedule their deletion */
+ fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;
+ del_filters = true;
+ } else if (fdir->state == IAVF_FDIR_FLTR_DIS_PENDING) {
+ /* Request to delete filter already sent to PF, change
+ * state to DEL_PENDING to delete filter after PF's
+ * response, not set as INACTIVE
+ */
+ fdir->state = IAVF_FDIR_FLTR_DEL_PENDING;
+ }
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+
+ if (del_filters) {
+ adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
+ mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
+ }
+}
+
+#define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \
+ NETIF_F_HW_VLAN_CTAG_TX | \
+ NETIF_F_HW_VLAN_STAG_RX | \
+ NETIF_F_HW_VLAN_STAG_TX)
+
+/**
+ * iavf_set_features - set the netdev feature flags
+ * @netdev: ptr to the netdev being adjusted
+ * @features: the feature set that the stack is suggesting
+ * Note: expects to be called while under rtnl_lock()
+ **/
+static int iavf_set_features(struct net_device *netdev,
+ netdev_features_t features)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ /* trigger update on any VLAN feature change */
+ if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^
+ (features & NETIF_VLAN_OFFLOAD_FEATURES))
+ iavf_set_vlan_offload_features(adapter, netdev->features,
+ features);
+
+ if ((netdev->features & NETIF_F_NTUPLE) ^ (features & NETIF_F_NTUPLE)) {
+ if (features & NETIF_F_NTUPLE)
+ adapter->flags |= IAVF_FLAG_FDIR_ENABLED;
+ else
+ iavf_disable_fdir(adapter);
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_features_check - Validate encapsulated packet conforms to limits
+ * @skb: skb buff
+ * @dev: This physical port's netdev
+ * @features: Offload features that the stack believes apply
+ **/
+static netdev_features_t iavf_features_check(struct sk_buff *skb,
+ struct net_device *dev,
+ netdev_features_t features)
+{
+ size_t len;
+
+ /* No point in doing any of this if neither checksum nor GSO are
+ * being requested for this frame. We can rule out both by just
+ * checking for CHECKSUM_PARTIAL
+ */
+ if (skb->ip_summed != CHECKSUM_PARTIAL)
+ return features;
+
+ /* We cannot support GSO if the MSS is going to be less than
+ * 64 bytes. If it is then we need to drop support for GSO.
+ */
+ if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
+ features &= ~NETIF_F_GSO_MASK;
+
+ /* MACLEN can support at most 63 words */
+ len = skb_network_header(skb) - skb->data;
+ if (len & ~(63 * 2))
+ goto out_err;
+
+ /* IPLEN and EIPLEN can support at most 127 dwords */
+ len = skb_transport_header(skb) - skb_network_header(skb);
+ if (len & ~(127 * 4))
+ goto out_err;
+
+ if (skb->encapsulation) {
+ /* L4TUNLEN can support 127 words */
+ len = skb_inner_network_header(skb) - skb_transport_header(skb);
+ if (len & ~(127 * 2))
+ goto out_err;
+
+ /* IPLEN can support at most 127 dwords */
+ len = skb_inner_transport_header(skb) -
+ skb_inner_network_header(skb);
+ if (len & ~(127 * 4))
+ goto out_err;
+ }
+
+ /* No need to validate L4LEN as TCP is the only protocol with a
+ * flexible value and we support all possible values supported
+ * by TCP, which is at most 15 dwords
+ */
+
+ return features;
+out_err:
+ return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
+}
+
+/**
+ * iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features that can toggle on/off
+ * @adapter: board private structure
+ *
+ * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
+ * were negotiated determine the VLAN features that can be toggled on and off.
+ **/
+static netdev_features_t
+iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter)
+{
+ netdev_features_t hw_features = 0;
+
+ if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
+ return hw_features;
+
+ /* Enable VLAN features if supported */
+ if (VLAN_ALLOWED(adapter)) {
+ hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |
+ NETIF_F_HW_VLAN_CTAG_RX);
+ } else if (VLAN_V2_ALLOWED(adapter)) {
+ struct virtchnl_vlan_caps *vlan_v2_caps =
+ &adapter->vlan_v2_caps;
+ struct virtchnl_vlan_supported_caps *stripping_support =
+ &vlan_v2_caps->offloads.stripping_support;
+ struct virtchnl_vlan_supported_caps *insertion_support =
+ &vlan_v2_caps->offloads.insertion_support;
+
+ if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
+ stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) {
+ if (stripping_support->outer &
+ VIRTCHNL_VLAN_ETHERTYPE_8100)
+ hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
+ if (stripping_support->outer &
+ VIRTCHNL_VLAN_ETHERTYPE_88A8)
+ hw_features |= NETIF_F_HW_VLAN_STAG_RX;
+ } else if (stripping_support->inner !=
+ VIRTCHNL_VLAN_UNSUPPORTED &&
+ stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) {
+ if (stripping_support->inner &
+ VIRTCHNL_VLAN_ETHERTYPE_8100)
+ hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
+ }
+
+ if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
+ insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) {
+ if (insertion_support->outer &
+ VIRTCHNL_VLAN_ETHERTYPE_8100)
+ hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
+ if (insertion_support->outer &
+ VIRTCHNL_VLAN_ETHERTYPE_88A8)
+ hw_features |= NETIF_F_HW_VLAN_STAG_TX;
+ } else if (insertion_support->inner &&
+ insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) {
+ if (insertion_support->inner &
+ VIRTCHNL_VLAN_ETHERTYPE_8100)
+ hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
+ }
+ }
+
+ return hw_features;
+}
+
+/**
+ * iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures
+ * @adapter: board private structure
+ *
+ * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
+ * were negotiated determine the VLAN features that are enabled by default.
+ **/
+static netdev_features_t
+iavf_get_netdev_vlan_features(struct iavf_adapter *adapter)
+{
+ netdev_features_t features = 0;
+
+ if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
+ return features;
+
+ if (VLAN_ALLOWED(adapter)) {
+ features |= NETIF_F_HW_VLAN_CTAG_FILTER |
+ NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX;
+ } else if (VLAN_V2_ALLOWED(adapter)) {
+ struct virtchnl_vlan_caps *vlan_v2_caps =
+ &adapter->vlan_v2_caps;
+ struct virtchnl_vlan_supported_caps *filtering_support =
+ &vlan_v2_caps->filtering.filtering_support;
+ struct virtchnl_vlan_supported_caps *stripping_support =
+ &vlan_v2_caps->offloads.stripping_support;
+ struct virtchnl_vlan_supported_caps *insertion_support =
+ &vlan_v2_caps->offloads.insertion_support;
+ u32 ethertype_init;
+
+ /* give priority to outer stripping and don't support both outer
+ * and inner stripping
+ */
+ ethertype_init = vlan_v2_caps->offloads.ethertype_init;
+ if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
+ if (stripping_support->outer &
+ VIRTCHNL_VLAN_ETHERTYPE_8100 &&
+ ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
+ features |= NETIF_F_HW_VLAN_CTAG_RX;
+ else if (stripping_support->outer &
+ VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
+ ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
+ features |= NETIF_F_HW_VLAN_STAG_RX;
+ } else if (stripping_support->inner !=
+ VIRTCHNL_VLAN_UNSUPPORTED) {
+ if (stripping_support->inner &
+ VIRTCHNL_VLAN_ETHERTYPE_8100 &&
+ ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
+ features |= NETIF_F_HW_VLAN_CTAG_RX;
+ }
+
+ /* give priority to outer insertion and don't support both outer
+ * and inner insertion
+ */
+ if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
+ if (insertion_support->outer &
+ VIRTCHNL_VLAN_ETHERTYPE_8100 &&
+ ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
+ features |= NETIF_F_HW_VLAN_CTAG_TX;
+ else if (insertion_support->outer &
+ VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
+ ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
+ features |= NETIF_F_HW_VLAN_STAG_TX;
+ } else if (insertion_support->inner !=
+ VIRTCHNL_VLAN_UNSUPPORTED) {
+ if (insertion_support->inner &
+ VIRTCHNL_VLAN_ETHERTYPE_8100 &&
+ ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
+ features |= NETIF_F_HW_VLAN_CTAG_TX;
+ }
+
+ /* give priority to outer filtering and don't bother if both
+ * outer and inner filtering are enabled
+ */
+ ethertype_init = vlan_v2_caps->filtering.ethertype_init;
+ if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
+ if (filtering_support->outer &
+ VIRTCHNL_VLAN_ETHERTYPE_8100 &&
+ ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
+ features |= NETIF_F_HW_VLAN_CTAG_FILTER;
+ if (filtering_support->outer &
+ VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
+ ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
+ features |= NETIF_F_HW_VLAN_STAG_FILTER;
+ } else if (filtering_support->inner !=
+ VIRTCHNL_VLAN_UNSUPPORTED) {
+ if (filtering_support->inner &
+ VIRTCHNL_VLAN_ETHERTYPE_8100 &&
+ ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
+ features |= NETIF_F_HW_VLAN_CTAG_FILTER;
+ if (filtering_support->inner &
+ VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
+ ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
+ features |= NETIF_F_HW_VLAN_STAG_FILTER;
+ }
+ }
+
+ return features;
+}
+
+#define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \
+ (!(((requested) & (feature_bit)) && \
+ !((allowed) & (feature_bit))))
+
+/**
+ * iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support
+ * @adapter: board private structure
+ * @requested_features: stack requested NETDEV features
+ **/
+static netdev_features_t
+iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter,
+ netdev_features_t requested_features)
+{
+ netdev_features_t allowed_features;
+
+ allowed_features = iavf_get_netdev_vlan_hw_features(adapter) |
+ iavf_get_netdev_vlan_features(adapter);
+
+ if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
+ allowed_features,
+ NETIF_F_HW_VLAN_CTAG_TX))
+ requested_features &= ~NETIF_F_HW_VLAN_CTAG_TX;
+
+ if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
+ allowed_features,
+ NETIF_F_HW_VLAN_CTAG_RX))
+ requested_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
+
+ if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
+ allowed_features,
+ NETIF_F_HW_VLAN_STAG_TX))
+ requested_features &= ~NETIF_F_HW_VLAN_STAG_TX;
+ if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
+ allowed_features,
+ NETIF_F_HW_VLAN_STAG_RX))
+ requested_features &= ~NETIF_F_HW_VLAN_STAG_RX;
+
+ if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
+ allowed_features,
+ NETIF_F_HW_VLAN_CTAG_FILTER))
+ requested_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
+
+ if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
+ allowed_features,
+ NETIF_F_HW_VLAN_STAG_FILTER))
+ requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER;
+
+ if ((requested_features &
+ (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) &&
+ (requested_features &
+ (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) &&
+ adapter->vlan_v2_caps.offloads.ethertype_match ==
+ VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION) {
+ netdev_warn(adapter->netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n");
+ requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX |
+ NETIF_F_HW_VLAN_STAG_TX);
+ }
+
+ return requested_features;
+}
+
+/**
+ * iavf_fix_features - fix up the netdev feature bits
+ * @netdev: our net device
+ * @features: desired feature bits
+ *
+ * Returns fixed-up features bits
+ **/
+static netdev_features_t iavf_fix_features(struct net_device *netdev,
+ netdev_features_t features)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ if (!FDIR_FLTR_SUPPORT(adapter))
+ features &= ~NETIF_F_NTUPLE;
+
+ return iavf_fix_netdev_vlan_features(adapter, features);
+}
+
+static const struct net_device_ops iavf_netdev_ops = {
+ .ndo_open = iavf_open,
+ .ndo_stop = iavf_close,
+ .ndo_start_xmit = iavf_xmit_frame,
+ .ndo_set_rx_mode = iavf_set_rx_mode,
+ .ndo_validate_addr = eth_validate_addr,
+ .ndo_set_mac_address = iavf_set_mac,
+ .ndo_change_mtu = iavf_change_mtu,
+ .ndo_tx_timeout = iavf_tx_timeout,
+ .ndo_vlan_rx_add_vid = iavf_vlan_rx_add_vid,
+ .ndo_vlan_rx_kill_vid = iavf_vlan_rx_kill_vid,
+ .ndo_features_check = iavf_features_check,
+ .ndo_fix_features = iavf_fix_features,
+ .ndo_set_features = iavf_set_features,
+ .ndo_setup_tc = iavf_setup_tc,
+};
+
+/**
+ * iavf_check_reset_complete - check that VF reset is complete
+ * @hw: pointer to hw struct
+ *
+ * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
+ **/
+static int iavf_check_reset_complete(struct iavf_hw *hw)
+{
+ u32 rstat;
+ int i;
+
+ for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
+ rstat = rd32(hw, IAVF_VFGEN_RSTAT) &
+ IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
+ if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
+ (rstat == VIRTCHNL_VFR_COMPLETED))
+ return 0;
+ usleep_range(10, 20);
+ }
+ return -EBUSY;
+}
+
+/**
+ * iavf_process_config - Process the config information we got from the PF
+ * @adapter: board private structure
+ *
+ * Verify that we have a valid config struct, and set up our netdev features
+ * and our VSI struct.
+ **/
+int iavf_process_config(struct iavf_adapter *adapter)
+{
+ struct virtchnl_vf_resource *vfres = adapter->vf_res;
+ netdev_features_t hw_vlan_features, vlan_features;
+ struct net_device *netdev = adapter->netdev;
+ netdev_features_t hw_enc_features;
+ netdev_features_t hw_features;
+
+ hw_enc_features = NETIF_F_SG |
+ NETIF_F_IP_CSUM |
+ NETIF_F_IPV6_CSUM |
+ NETIF_F_HIGHDMA |
+ NETIF_F_SOFT_FEATURES |
+ NETIF_F_TSO |
+ NETIF_F_TSO_ECN |
+ NETIF_F_TSO6 |
+ NETIF_F_SCTP_CRC |
+ NETIF_F_RXHASH |
+ NETIF_F_RXCSUM |
+ 0;
+
+ /* advertise to stack only if offloads for encapsulated packets is
+ * supported
+ */
+ if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
+ hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
+ NETIF_F_GSO_GRE |
+ NETIF_F_GSO_GRE_CSUM |
+ NETIF_F_GSO_IPXIP4 |
+ NETIF_F_GSO_IPXIP6 |
+ NETIF_F_GSO_UDP_TUNNEL_CSUM |
+ NETIF_F_GSO_PARTIAL |
+ 0;
+
+ if (!(vfres->vf_cap_flags &
+ VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
+ netdev->gso_partial_features |=
+ NETIF_F_GSO_UDP_TUNNEL_CSUM;
+
+ netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
+ netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
+ netdev->hw_enc_features |= hw_enc_features;
+ }
+ /* record features VLANs can make use of */
+ netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
+
+ /* Write features and hw_features separately to avoid polluting
+ * with, or dropping, features that are set when we registered.
+ */
+ hw_features = hw_enc_features;
+
+ /* get HW VLAN features that can be toggled */
+ hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter);
+
+ /* Enable cloud filter if ADQ is supported */
+ if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)
+ hw_features |= NETIF_F_HW_TC;
+ if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO)
+ hw_features |= NETIF_F_GSO_UDP_L4;
+
+ netdev->hw_features |= hw_features | hw_vlan_features;
+ vlan_features = iavf_get_netdev_vlan_features(adapter);
+
+ netdev->features |= hw_features | vlan_features;
+
+ if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
+ netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
+
+ if (FDIR_FLTR_SUPPORT(adapter)) {
+ netdev->hw_features |= NETIF_F_NTUPLE;
+ netdev->features |= NETIF_F_NTUPLE;
+ adapter->flags |= IAVF_FLAG_FDIR_ENABLED;
+ }
+
+ netdev->priv_flags |= IFF_UNICAST_FLT;
+
+ /* Do not turn on offloads when they are requested to be turned off.
+ * TSO needs minimum 576 bytes to work correctly.
+ */
+ if (netdev->wanted_features) {
+ if (!(netdev->wanted_features & NETIF_F_TSO) ||
+ netdev->mtu < 576)
+ netdev->features &= ~NETIF_F_TSO;
+ if (!(netdev->wanted_features & NETIF_F_TSO6) ||
+ netdev->mtu < 576)
+ netdev->features &= ~NETIF_F_TSO6;
+ if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
+ netdev->features &= ~NETIF_F_TSO_ECN;
+ if (!(netdev->wanted_features & NETIF_F_GRO))
+ netdev->features &= ~NETIF_F_GRO;
+ if (!(netdev->wanted_features & NETIF_F_GSO))
+ netdev->features &= ~NETIF_F_GSO;
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_probe - Device Initialization Routine
+ * @pdev: PCI device information struct
+ * @ent: entry in iavf_pci_tbl
+ *
+ * Returns 0 on success, negative on failure
+ *
+ * iavf_probe initializes an adapter identified by a pci_dev structure.
+ * The OS initialization, configuring of the adapter private structure,
+ * and a hardware reset occur.
+ **/
+static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+ struct net_device *netdev;
+ struct iavf_adapter *adapter = NULL;
+ struct iavf_hw *hw = NULL;
+ int err;
+
+ err = pci_enable_device(pdev);
+ if (err)
+ return err;
+
+ err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+ if (err) {
+ dev_err(&pdev->dev,
+ "DMA configuration failed: 0x%x\n", err);
+ goto err_dma;
+ }
+
+ err = pci_request_regions(pdev, iavf_driver_name);
+ if (err) {
+ dev_err(&pdev->dev,
+ "pci_request_regions failed 0x%x\n", err);
+ goto err_pci_reg;
+ }
+
+ pci_set_master(pdev);
+
+ netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter),
+ IAVF_MAX_REQ_QUEUES);
+ if (!netdev) {
+ err = -ENOMEM;
+ goto err_alloc_etherdev;
+ }
+
+ SET_NETDEV_DEV(netdev, &pdev->dev);
+
+ pci_set_drvdata(pdev, netdev);
+ adapter = netdev_priv(netdev);
+
+ adapter->netdev = netdev;
+ adapter->pdev = pdev;
+
+ hw = &adapter->hw;
+ hw->back = adapter;
+
+ adapter->wq = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM,
+ iavf_driver_name);
+ if (!adapter->wq) {
+ err = -ENOMEM;
+ goto err_alloc_wq;
+ }
+
+ adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
+ iavf_change_state(adapter, __IAVF_STARTUP);
+
+ /* Call save state here because it relies on the adapter struct. */
+ pci_save_state(pdev);
+
+ hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
+ pci_resource_len(pdev, 0));
+ if (!hw->hw_addr) {
+ err = -EIO;
+ goto err_ioremap;
+ }
+ hw->vendor_id = pdev->vendor;
+ hw->device_id = pdev->device;
+ pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
+ hw->subsystem_vendor_id = pdev->subsystem_vendor;
+ hw->subsystem_device_id = pdev->subsystem_device;
+ hw->bus.device = PCI_SLOT(pdev->devfn);
+ hw->bus.func = PCI_FUNC(pdev->devfn);
+ hw->bus.bus_id = pdev->bus->number;
+
+ /* set up the locks for the AQ, do this only once in probe
+ * and destroy them only once in remove
+ */
+ mutex_init(&adapter->crit_lock);
+ mutex_init(&adapter->client_lock);
+ mutex_init(&hw->aq.asq_mutex);
+ mutex_init(&hw->aq.arq_mutex);
+
+ spin_lock_init(&adapter->mac_vlan_list_lock);
+ spin_lock_init(&adapter->cloud_filter_list_lock);
+ spin_lock_init(&adapter->fdir_fltr_lock);
+ spin_lock_init(&adapter->adv_rss_lock);
+ spin_lock_init(&adapter->current_netdev_promisc_flags_lock);
+
+ INIT_LIST_HEAD(&adapter->mac_filter_list);
+ INIT_LIST_HEAD(&adapter->vlan_filter_list);
+ INIT_LIST_HEAD(&adapter->cloud_filter_list);
+ INIT_LIST_HEAD(&adapter->fdir_list_head);
+ INIT_LIST_HEAD(&adapter->adv_rss_list_head);
+
+ INIT_WORK(&adapter->reset_task, iavf_reset_task);
+ INIT_WORK(&adapter->adminq_task, iavf_adminq_task);
+ INIT_WORK(&adapter->finish_config, iavf_finish_config);
+ INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task);
+ INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task);
+
+ /* Setup the wait queue for indicating transition to down status */
+ init_waitqueue_head(&adapter->down_waitqueue);
+
+ /* Setup the wait queue for indicating transition to running state */
+ init_waitqueue_head(&adapter->reset_waitqueue);
+
+ /* Setup the wait queue for indicating virtchannel events */
+ init_waitqueue_head(&adapter->vc_waitqueue);
+
+ queue_delayed_work(adapter->wq, &adapter->watchdog_task,
+ msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
+ /* Initialization goes on in the work. Do not add more of it below. */
+ return 0;
+
+err_ioremap:
+ destroy_workqueue(adapter->wq);
+err_alloc_wq:
+ free_netdev(netdev);
+err_alloc_etherdev:
+ pci_release_regions(pdev);
+err_pci_reg:
+err_dma:
+ pci_disable_device(pdev);
+ return err;
+}
+
+/**
+ * iavf_suspend - Power management suspend routine
+ * @dev_d: device info pointer
+ *
+ * Called when the system (VM) is entering sleep/suspend.
+ **/
+static int __maybe_unused iavf_suspend(struct device *dev_d)
+{
+ struct net_device *netdev = dev_get_drvdata(dev_d);
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+
+ netif_device_detach(netdev);
+
+ while (!mutex_trylock(&adapter->crit_lock))
+ usleep_range(500, 1000);
+
+ if (netif_running(netdev)) {
+ rtnl_lock();
+ iavf_down(adapter);
+ rtnl_unlock();
+ }
+ iavf_free_misc_irq(adapter);
+ iavf_reset_interrupt_capability(adapter);
+
+ mutex_unlock(&adapter->crit_lock);
+
+ return 0;
+}
+
+/**
+ * iavf_resume - Power management resume routine
+ * @dev_d: device info pointer
+ *
+ * Called when the system (VM) is resumed from sleep/suspend.
+ **/
+static int __maybe_unused iavf_resume(struct device *dev_d)
+{
+ struct pci_dev *pdev = to_pci_dev(dev_d);
+ struct iavf_adapter *adapter;
+ u32 err;
+
+ adapter = iavf_pdev_to_adapter(pdev);
+
+ pci_set_master(pdev);
+
+ rtnl_lock();
+ err = iavf_set_interrupt_capability(adapter);
+ if (err) {
+ rtnl_unlock();
+ dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
+ return err;
+ }
+ err = iavf_request_misc_irq(adapter);
+ rtnl_unlock();
+ if (err) {
+ dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
+ return err;
+ }
+
+ queue_work(adapter->wq, &adapter->reset_task);
+
+ netif_device_attach(adapter->netdev);
+
+ return err;
+}
+
+/**
+ * iavf_remove - Device Removal Routine
+ * @pdev: PCI device information struct
+ *
+ * iavf_remove is called by the PCI subsystem to alert the driver
+ * that it should release a PCI device. The could be caused by a
+ * Hot-Plug event, or because the driver is going to be removed from
+ * memory.
+ **/
+static void iavf_remove(struct pci_dev *pdev)
+{
+ struct iavf_fdir_fltr *fdir, *fdirtmp;
+ struct iavf_vlan_filter *vlf, *vlftmp;
+ struct iavf_cloud_filter *cf, *cftmp;
+ struct iavf_adv_rss *rss, *rsstmp;
+ struct iavf_mac_filter *f, *ftmp;
+ struct iavf_adapter *adapter;
+ struct net_device *netdev;
+ struct iavf_hw *hw;
+ int err;
+
+ /* Don't proceed with remove if netdev is already freed */
+ netdev = pci_get_drvdata(pdev);
+ if (!netdev)
+ return;
+
+ adapter = iavf_pdev_to_adapter(pdev);
+ hw = &adapter->hw;
+
+ if (test_and_set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
+ return;
+
+ /* Wait until port initialization is complete.
+ * There are flows where register/unregister netdev may race.
+ */
+ while (1) {
+ mutex_lock(&adapter->crit_lock);
+ if (adapter->state == __IAVF_RUNNING ||
+ adapter->state == __IAVF_DOWN ||
+ adapter->state == __IAVF_INIT_FAILED) {
+ mutex_unlock(&adapter->crit_lock);
+ break;
+ }
+ /* Simply return if we already went through iavf_shutdown */
+ if (adapter->state == __IAVF_REMOVE) {
+ mutex_unlock(&adapter->crit_lock);
+ return;
+ }
+
+ mutex_unlock(&adapter->crit_lock);
+ usleep_range(500, 1000);
+ }
+ cancel_delayed_work_sync(&adapter->watchdog_task);
+ cancel_work_sync(&adapter->finish_config);
+
+ rtnl_lock();
+ if (adapter->netdev_registered) {
+ unregister_netdevice(netdev);
+ adapter->netdev_registered = false;
+ }
+ rtnl_unlock();
+
+ if (CLIENT_ALLOWED(adapter)) {
+ err = iavf_lan_del_device(adapter);
+ if (err)
+ dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
+ err);
+ }
+
+ mutex_lock(&adapter->crit_lock);
+ dev_info(&adapter->pdev->dev, "Removing device\n");
+ iavf_change_state(adapter, __IAVF_REMOVE);
+
+ iavf_request_reset(adapter);
+ msleep(50);
+ /* If the FW isn't responding, kick it once, but only once. */
+ if (!iavf_asq_done(hw)) {
+ iavf_request_reset(adapter);
+ msleep(50);
+ }
+
+ iavf_misc_irq_disable(adapter);
+ /* Shut down all the garbage mashers on the detention level */
+ cancel_work_sync(&adapter->reset_task);
+ cancel_delayed_work_sync(&adapter->watchdog_task);
+ cancel_work_sync(&adapter->adminq_task);
+ cancel_delayed_work_sync(&adapter->client_task);
+
+ adapter->aq_required = 0;
+ adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
+
+ iavf_free_all_tx_resources(adapter);
+ iavf_free_all_rx_resources(adapter);
+ iavf_free_misc_irq(adapter);
+
+ iavf_reset_interrupt_capability(adapter);
+ iavf_free_q_vectors(adapter);
+
+ iavf_free_rss(adapter);
+
+ if (hw->aq.asq.count)
+ iavf_shutdown_adminq(hw);
+
+ /* destroy the locks only once, here */
+ mutex_destroy(&hw->aq.arq_mutex);
+ mutex_destroy(&hw->aq.asq_mutex);
+ mutex_destroy(&adapter->client_lock);
+ mutex_unlock(&adapter->crit_lock);
+ mutex_destroy(&adapter->crit_lock);
+
+ iounmap(hw->hw_addr);
+ pci_release_regions(pdev);
+ iavf_free_queues(adapter);
+ kfree(adapter->vf_res);
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+ /* If we got removed before an up/down sequence, we've got a filter
+ * hanging out there that we need to get rid of.
+ */
+ list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
+ list_del(&f->list);
+ kfree(f);
+ }
+ list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
+ list) {
+ list_del(&vlf->list);
+ kfree(vlf);
+ }
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ spin_lock_bh(&adapter->cloud_filter_list_lock);
+ list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
+ list_del(&cf->list);
+ kfree(cf);
+ }
+ spin_unlock_bh(&adapter->cloud_filter_list_lock);
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) {
+ list_del(&fdir->list);
+ kfree(fdir);
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+
+ spin_lock_bh(&adapter->adv_rss_lock);
+ list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
+ list) {
+ list_del(&rss->list);
+ kfree(rss);
+ }
+ spin_unlock_bh(&adapter->adv_rss_lock);
+
+ destroy_workqueue(adapter->wq);
+
+ pci_set_drvdata(pdev, NULL);
+
+ free_netdev(netdev);
+
+ pci_disable_device(pdev);
+}
+
+/**
+ * iavf_shutdown - Shutdown the device in preparation for a reboot
+ * @pdev: pci device structure
+ **/
+static void iavf_shutdown(struct pci_dev *pdev)
+{
+ iavf_remove(pdev);
+
+ if (system_state == SYSTEM_POWER_OFF)
+ pci_set_power_state(pdev, PCI_D3hot);
+}
+
+static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume);
+
+static struct pci_driver iavf_driver = {
+ .name = iavf_driver_name,
+ .id_table = iavf_pci_tbl,
+ .probe = iavf_probe,
+ .remove = iavf_remove,
+ .driver.pm = &iavf_pm_ops,
+ .shutdown = iavf_shutdown,
+};
+
+/**
+ * iavf_init_module - Driver Registration Routine
+ *
+ * iavf_init_module is the first routine called when the driver is
+ * loaded. All it does is register with the PCI subsystem.
+ **/
+static int __init iavf_init_module(void)
+{
+ pr_info("iavf: %s\n", iavf_driver_string);
+
+ pr_info("%s\n", iavf_copyright);
+
+ return pci_register_driver(&iavf_driver);
+}
+
+module_init(iavf_init_module);
+
+/**
+ * iavf_exit_module - Driver Exit Cleanup Routine
+ *
+ * iavf_exit_module is called just before the driver is removed
+ * from memory.
+ **/
+static void __exit iavf_exit_module(void)
+{
+ pci_unregister_driver(&iavf_driver);
+}
+
+module_exit(iavf_exit_module);
+
+/* iavf_main.c */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_osdep.h b/drivers/net/ethernet/intel/iavf/iavf_osdep.h
new file mode 100644
index 0000000000..77d33deaab
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_osdep.h
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_OSDEP_H_
+#define _IAVF_OSDEP_H_
+
+#include <linux/types.h>
+#include <linux/if_ether.h>
+#include <linux/if_vlan.h>
+#include <linux/tcp.h>
+#include <linux/pci.h>
+
+/* get readq/writeq support for 32 bit kernels, use the low-first version */
+#include <linux/io-64-nonatomic-lo-hi.h>
+
+#define wr32(a, reg, value) writel((value), ((a)->hw_addr + (reg)))
+#define rd32(a, reg) readl((a)->hw_addr + (reg))
+
+#define wr64(a, reg, value) writeq((value), ((a)->hw_addr + (reg)))
+#define rd64(a, reg) readq((a)->hw_addr + (reg))
+#define iavf_flush(a) readl((a)->hw_addr + IAVF_VFGEN_RSTAT)
+
+/* memory allocation tracking */
+struct iavf_dma_mem {
+ void *va;
+ dma_addr_t pa;
+ u32 size;
+};
+
+#define iavf_allocate_dma_mem(h, m, unused, s, a) \
+ iavf_allocate_dma_mem_d(h, m, s, a)
+
+struct iavf_virt_mem {
+ void *va;
+ u32 size;
+};
+
+#define iavf_debug(h, m, s, ...) \
+do { \
+ if (((m) & (h)->debug_mask)) \
+ pr_info("iavf %02x:%02x.%x " s, \
+ (h)->bus.bus_id, (h)->bus.device, \
+ (h)->bus.func, ##__VA_ARGS__); \
+} while (0)
+
+#endif /* _IAVF_OSDEP_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_prototype.h b/drivers/net/ethernet/intel/iavf/iavf_prototype.h
new file mode 100644
index 0000000000..940cb4203f
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_prototype.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_PROTOTYPE_H_
+#define _IAVF_PROTOTYPE_H_
+
+#include "iavf_type.h"
+#include "iavf_alloc.h"
+#include <linux/avf/virtchnl.h>
+
+/* Prototypes for shared code functions that are not in
+ * the standard function pointer structures. These are
+ * mostly because they are needed even before the init
+ * has happened and will assist in the early SW and FW
+ * setup.
+ */
+
+/* adminq functions */
+enum iavf_status iavf_init_adminq(struct iavf_hw *hw);
+enum iavf_status iavf_shutdown_adminq(struct iavf_hw *hw);
+void iavf_adminq_init_ring_data(struct iavf_hw *hw);
+enum iavf_status iavf_clean_arq_element(struct iavf_hw *hw,
+ struct iavf_arq_event_info *e,
+ u16 *events_pending);
+enum iavf_status iavf_asq_send_command(struct iavf_hw *hw,
+ struct iavf_aq_desc *desc,
+ void *buff, /* can be NULL */
+ u16 buff_size,
+ struct iavf_asq_cmd_details *cmd_details);
+bool iavf_asq_done(struct iavf_hw *hw);
+
+/* debug function for adminq */
+void iavf_debug_aq(struct iavf_hw *hw, enum iavf_debug_mask mask,
+ void *desc, void *buffer, u16 buf_len);
+
+void iavf_idle_aq(struct iavf_hw *hw);
+void iavf_resume_aq(struct iavf_hw *hw);
+bool iavf_check_asq_alive(struct iavf_hw *hw);
+enum iavf_status iavf_aq_queue_shutdown(struct iavf_hw *hw, bool unloading);
+const char *iavf_aq_str(struct iavf_hw *hw, enum iavf_admin_queue_err aq_err);
+const char *iavf_stat_str(struct iavf_hw *hw, enum iavf_status stat_err);
+
+enum iavf_status iavf_aq_set_rss_lut(struct iavf_hw *hw, u16 seid,
+ bool pf_lut, u8 *lut, u16 lut_size);
+enum iavf_status iavf_aq_set_rss_key(struct iavf_hw *hw, u16 seid,
+ struct iavf_aqc_get_set_rss_key_data *key);
+
+enum iavf_status iavf_set_mac_type(struct iavf_hw *hw);
+
+extern struct iavf_rx_ptype_decoded iavf_ptype_lookup[];
+
+static inline struct iavf_rx_ptype_decoded decode_rx_desc_ptype(u8 ptype)
+{
+ return iavf_ptype_lookup[ptype];
+}
+
+void iavf_vf_parse_hw_config(struct iavf_hw *hw,
+ struct virtchnl_vf_resource *msg);
+enum iavf_status iavf_aq_send_msg_to_pf(struct iavf_hw *hw,
+ enum virtchnl_ops v_opcode,
+ enum iavf_status v_retval,
+ u8 *msg, u16 msglen,
+ struct iavf_asq_cmd_details *cmd_details);
+#endif /* _IAVF_PROTOTYPE_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_register.h b/drivers/net/ethernet/intel/iavf/iavf_register.h
new file mode 100644
index 0000000000..a19e88898a
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_register.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_REGISTER_H_
+#define _IAVF_REGISTER_H_
+
+#define IAVF_VF_ARQBAH1 0x00006000 /* Reset: EMPR */
+#define IAVF_VF_ARQBAL1 0x00006C00 /* Reset: EMPR */
+#define IAVF_VF_ARQH1 0x00007400 /* Reset: EMPR */
+#define IAVF_VF_ARQH1_ARQH_SHIFT 0
+#define IAVF_VF_ARQH1_ARQH_MASK IAVF_MASK(0x3FF, IAVF_VF_ARQH1_ARQH_SHIFT)
+#define IAVF_VF_ARQLEN1 0x00008000 /* Reset: EMPR */
+#define IAVF_VF_ARQLEN1_ARQVFE_SHIFT 28
+#define IAVF_VF_ARQLEN1_ARQVFE_MASK IAVF_MASK(0x1, IAVF_VF_ARQLEN1_ARQVFE_SHIFT)
+#define IAVF_VF_ARQLEN1_ARQOVFL_SHIFT 29
+#define IAVF_VF_ARQLEN1_ARQOVFL_MASK IAVF_MASK(0x1, IAVF_VF_ARQLEN1_ARQOVFL_SHIFT)
+#define IAVF_VF_ARQLEN1_ARQCRIT_SHIFT 30
+#define IAVF_VF_ARQLEN1_ARQCRIT_MASK IAVF_MASK(0x1, IAVF_VF_ARQLEN1_ARQCRIT_SHIFT)
+#define IAVF_VF_ARQLEN1_ARQENABLE_SHIFT 31
+#define IAVF_VF_ARQLEN1_ARQENABLE_MASK IAVF_MASK(0x1, IAVF_VF_ARQLEN1_ARQENABLE_SHIFT)
+#define IAVF_VF_ARQT1 0x00007000 /* Reset: EMPR */
+#define IAVF_VF_ATQBAH1 0x00007800 /* Reset: EMPR */
+#define IAVF_VF_ATQBAL1 0x00007C00 /* Reset: EMPR */
+#define IAVF_VF_ATQH1 0x00006400 /* Reset: EMPR */
+#define IAVF_VF_ATQLEN1 0x00006800 /* Reset: EMPR */
+#define IAVF_VF_ATQLEN1_ATQVFE_SHIFT 28
+#define IAVF_VF_ATQLEN1_ATQVFE_MASK IAVF_MASK(0x1, IAVF_VF_ATQLEN1_ATQVFE_SHIFT)
+#define IAVF_VF_ATQLEN1_ATQOVFL_SHIFT 29
+#define IAVF_VF_ATQLEN1_ATQOVFL_MASK IAVF_MASK(0x1, IAVF_VF_ATQLEN1_ATQOVFL_SHIFT)
+#define IAVF_VF_ATQLEN1_ATQCRIT_SHIFT 30
+#define IAVF_VF_ATQLEN1_ATQCRIT_MASK IAVF_MASK(0x1, IAVF_VF_ATQLEN1_ATQCRIT_SHIFT)
+#define IAVF_VF_ATQLEN1_ATQENABLE_SHIFT 31
+#define IAVF_VF_ATQLEN1_ATQENABLE_MASK IAVF_MASK(0x1, IAVF_VF_ATQLEN1_ATQENABLE_SHIFT)
+#define IAVF_VF_ATQT1 0x00008400 /* Reset: EMPR */
+#define IAVF_VFGEN_RSTAT 0x00008800 /* Reset: VFR */
+#define IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT 0
+#define IAVF_VFGEN_RSTAT_VFR_STATE_MASK IAVF_MASK(0x3, IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT)
+#define IAVF_VFINT_DYN_CTL01 0x00005C00 /* Reset: VFR */
+#define IAVF_VFINT_DYN_CTL01_INTENA_SHIFT 0
+#define IAVF_VFINT_DYN_CTL01_INTENA_MASK IAVF_MASK(0x1, IAVF_VFINT_DYN_CTL01_INTENA_SHIFT)
+#define IAVF_VFINT_DYN_CTL01_ITR_INDX_SHIFT 3
+#define IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK IAVF_MASK(0x3, IAVF_VFINT_DYN_CTL01_ITR_INDX_SHIFT)
+#define IAVF_VFINT_DYN_CTLN1(_INTVF) (0x00003800 + ((_INTVF) * 4)) /* _i=0...63 */ /* Reset: VFR */
+#define IAVF_VFINT_DYN_CTLN1_INTENA_SHIFT 0
+#define IAVF_VFINT_DYN_CTLN1_INTENA_MASK IAVF_MASK(0x1, IAVF_VFINT_DYN_CTLN1_INTENA_SHIFT)
+#define IAVF_VFINT_DYN_CTLN1_SWINT_TRIG_SHIFT 2
+#define IAVF_VFINT_DYN_CTLN1_SWINT_TRIG_MASK IAVF_MASK(0x1, IAVF_VFINT_DYN_CTLN1_SWINT_TRIG_SHIFT)
+#define IAVF_VFINT_DYN_CTLN1_ITR_INDX_SHIFT 3
+#define IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK IAVF_MASK(0x3, IAVF_VFINT_DYN_CTLN1_ITR_INDX_SHIFT)
+#define IAVF_VFINT_DYN_CTLN1_INTERVAL_SHIFT 5
+#define IAVF_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_SHIFT 24
+#define IAVF_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_MASK IAVF_MASK(0x1, IAVF_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_SHIFT)
+#define IAVF_VFINT_ICR0_ENA1 0x00005000 /* Reset: CORER */
+#define IAVF_VFINT_ICR0_ENA1_ADMINQ_SHIFT 30
+#define IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK IAVF_MASK(0x1, IAVF_VFINT_ICR0_ENA1_ADMINQ_SHIFT)
+#define IAVF_VFINT_ICR0_ENA1_RSVD_SHIFT 31
+#define IAVF_VFINT_ICR01 0x00004800 /* Reset: CORER */
+#define IAVF_VFINT_ITRN1(_i, _INTVF) (0x00002800 + ((_i) * 64 + (_INTVF) * 4)) /* _i=0...2, _INTVF=0...15 */ /* Reset: VFR */
+#define IAVF_QRX_TAIL1(_Q) (0x00002000 + ((_Q) * 4)) /* _i=0...15 */ /* Reset: CORER */
+#define IAVF_QTX_TAIL1(_Q) (0x00000000 + ((_Q) * 4)) /* _i=0...15 */ /* Reset: PFR */
+#define IAVF_VFQF_HENA(_i) (0x0000C400 + ((_i) * 4)) /* _i=0...1 */ /* Reset: CORER */
+#define IAVF_VFQF_HKEY(_i) (0x0000CC00 + ((_i) * 4)) /* _i=0...12 */ /* Reset: CORER */
+#define IAVF_VFQF_HKEY_MAX_INDEX 12
+#define IAVF_VFQF_HLUT(_i) (0x0000D000 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */
+#define IAVF_VFQF_HLUT_MAX_INDEX 15
+#define IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_SHIFT 30
+#define IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_MASK IAVF_MASK(0x1, IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_SHIFT)
+#endif /* _IAVF_REGISTER_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_status.h b/drivers/net/ethernet/intel/iavf/iavf_status.h
new file mode 100644
index 0000000000..0e493ee9e9
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_status.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_STATUS_H_
+#define _IAVF_STATUS_H_
+
+/* Error Codes */
+enum iavf_status {
+ IAVF_SUCCESS = 0,
+ IAVF_ERR_NVM = -1,
+ IAVF_ERR_NVM_CHECKSUM = -2,
+ IAVF_ERR_PHY = -3,
+ IAVF_ERR_CONFIG = -4,
+ IAVF_ERR_PARAM = -5,
+ IAVF_ERR_MAC_TYPE = -6,
+ IAVF_ERR_UNKNOWN_PHY = -7,
+ IAVF_ERR_LINK_SETUP = -8,
+ IAVF_ERR_ADAPTER_STOPPED = -9,
+ IAVF_ERR_INVALID_MAC_ADDR = -10,
+ IAVF_ERR_DEVICE_NOT_SUPPORTED = -11,
+ IAVF_ERR_PRIMARY_REQUESTS_PENDING = -12,
+ IAVF_ERR_INVALID_LINK_SETTINGS = -13,
+ IAVF_ERR_AUTONEG_NOT_COMPLETE = -14,
+ IAVF_ERR_RESET_FAILED = -15,
+ IAVF_ERR_SWFW_SYNC = -16,
+ IAVF_ERR_NO_AVAILABLE_VSI = -17,
+ IAVF_ERR_NO_MEMORY = -18,
+ IAVF_ERR_BAD_PTR = -19,
+ IAVF_ERR_RING_FULL = -20,
+ IAVF_ERR_INVALID_PD_ID = -21,
+ IAVF_ERR_INVALID_QP_ID = -22,
+ IAVF_ERR_INVALID_CQ_ID = -23,
+ IAVF_ERR_INVALID_CEQ_ID = -24,
+ IAVF_ERR_INVALID_AEQ_ID = -25,
+ IAVF_ERR_INVALID_SIZE = -26,
+ IAVF_ERR_INVALID_ARP_INDEX = -27,
+ IAVF_ERR_INVALID_FPM_FUNC_ID = -28,
+ IAVF_ERR_QP_INVALID_MSG_SIZE = -29,
+ IAVF_ERR_QP_TOOMANY_WRS_POSTED = -30,
+ IAVF_ERR_INVALID_FRAG_COUNT = -31,
+ IAVF_ERR_QUEUE_EMPTY = -32,
+ IAVF_ERR_INVALID_ALIGNMENT = -33,
+ IAVF_ERR_FLUSHED_QUEUE = -34,
+ IAVF_ERR_INVALID_PUSH_PAGE_INDEX = -35,
+ IAVF_ERR_INVALID_IMM_DATA_SIZE = -36,
+ IAVF_ERR_TIMEOUT = -37,
+ IAVF_ERR_OPCODE_MISMATCH = -38,
+ IAVF_ERR_CQP_COMPL_ERROR = -39,
+ IAVF_ERR_INVALID_VF_ID = -40,
+ IAVF_ERR_INVALID_HMCFN_ID = -41,
+ IAVF_ERR_BACKING_PAGE_ERROR = -42,
+ IAVF_ERR_NO_PBLCHUNKS_AVAILABLE = -43,
+ IAVF_ERR_INVALID_PBLE_INDEX = -44,
+ IAVF_ERR_INVALID_SD_INDEX = -45,
+ IAVF_ERR_INVALID_PAGE_DESC_INDEX = -46,
+ IAVF_ERR_INVALID_SD_TYPE = -47,
+ IAVF_ERR_MEMCPY_FAILED = -48,
+ IAVF_ERR_INVALID_HMC_OBJ_INDEX = -49,
+ IAVF_ERR_INVALID_HMC_OBJ_COUNT = -50,
+ IAVF_ERR_INVALID_SRQ_ARM_LIMIT = -51,
+ IAVF_ERR_SRQ_ENABLED = -52,
+ IAVF_ERR_ADMIN_QUEUE_ERROR = -53,
+ IAVF_ERR_ADMIN_QUEUE_TIMEOUT = -54,
+ IAVF_ERR_BUF_TOO_SHORT = -55,
+ IAVF_ERR_ADMIN_QUEUE_FULL = -56,
+ IAVF_ERR_ADMIN_QUEUE_NO_WORK = -57,
+ IAVF_ERR_BAD_RDMA_CQE = -58,
+ IAVF_ERR_NVM_BLANK_MODE = -59,
+ IAVF_ERR_NOT_IMPLEMENTED = -60,
+ IAVF_ERR_PE_DOORBELL_NOT_ENABLED = -61,
+ IAVF_ERR_DIAG_TEST_FAILED = -62,
+ IAVF_ERR_NOT_READY = -63,
+ IAVF_NOT_SUPPORTED = -64,
+ IAVF_ERR_FIRMWARE_API_VERSION = -65,
+ IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR = -66,
+};
+
+#endif /* _IAVF_STATUS_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_trace.h b/drivers/net/ethernet/intel/iavf/iavf_trace.h
new file mode 100644
index 0000000000..82fda6f5ab
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_trace.h
@@ -0,0 +1,209 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+/* Modeled on trace-events-sample.h */
+
+/* The trace subsystem name for iavf will be "iavf".
+ *
+ * This file is named iavf_trace.h.
+ *
+ * Since this include file's name is different from the trace
+ * subsystem name, we'll have to define TRACE_INCLUDE_FILE at the end
+ * of this file.
+ */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM iavf
+
+/* See trace-events-sample.h for a detailed description of why this
+ * guard clause is different from most normal include files.
+ */
+#if !defined(_IAVF_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _IAVF_TRACE_H_
+
+#include <linux/tracepoint.h>
+
+/*
+ * iavf_trace() macro enables shared code to refer to trace points
+ * like:
+ *
+ * trace_iavf{,vf}_example(args...)
+ *
+ * ... as:
+ *
+ * iavf_trace(example, args...)
+ *
+ * ... to resolve to the PF or VF version of the tracepoint without
+ * ifdefs, and to allow tracepoints to be disabled entirely at build
+ * time.
+ *
+ * Trace point should always be referred to in the driver via this
+ * macro.
+ *
+ * Similarly, iavf_trace_enabled(trace_name) wraps references to
+ * trace_iavf{,vf}_<trace_name>_enabled() functions.
+ */
+#define _IAVF_TRACE_NAME(trace_name) (trace_ ## iavf ## _ ## trace_name)
+#define IAVF_TRACE_NAME(trace_name) _IAVF_TRACE_NAME(trace_name)
+
+#define iavf_trace(trace_name, args...) IAVF_TRACE_NAME(trace_name)(args)
+
+#define iavf_trace_enabled(trace_name) IAVF_TRACE_NAME(trace_name##_enabled)()
+
+/* Events common to PF and VF. Corresponding versions will be defined
+ * for both, named trace_iavf_* and trace_iavf_*. The iavf_trace()
+ * macro above will select the right trace point name for the driver
+ * being built from shared code.
+ */
+
+/* Events related to a vsi & ring */
+DECLARE_EVENT_CLASS(
+ iavf_tx_template,
+
+ TP_PROTO(struct iavf_ring *ring,
+ struct iavf_tx_desc *desc,
+ struct iavf_tx_buffer *buf),
+
+ TP_ARGS(ring, desc, buf),
+
+ /* The convention here is to make the first fields in the
+ * TP_STRUCT match the TP_PROTO exactly. This enables the use
+ * of the args struct generated by the tplist tool (from the
+ * bcc-tools package) to be used for those fields. To access
+ * fields other than the tracepoint args will require the
+ * tplist output to be adjusted.
+ */
+ TP_STRUCT__entry(
+ __field(void*, ring)
+ __field(void*, desc)
+ __field(void*, buf)
+ __string(devname, ring->netdev->name)
+ ),
+
+ TP_fast_assign(
+ __entry->ring = ring;
+ __entry->desc = desc;
+ __entry->buf = buf;
+ __assign_str(devname, ring->netdev->name);
+ ),
+
+ TP_printk(
+ "netdev: %s ring: %p desc: %p buf %p",
+ __get_str(devname), __entry->ring,
+ __entry->desc, __entry->buf)
+);
+
+DEFINE_EVENT(
+ iavf_tx_template, iavf_clean_tx_irq,
+ TP_PROTO(struct iavf_ring *ring,
+ struct iavf_tx_desc *desc,
+ struct iavf_tx_buffer *buf),
+
+ TP_ARGS(ring, desc, buf));
+
+DEFINE_EVENT(
+ iavf_tx_template, iavf_clean_tx_irq_unmap,
+ TP_PROTO(struct iavf_ring *ring,
+ struct iavf_tx_desc *desc,
+ struct iavf_tx_buffer *buf),
+
+ TP_ARGS(ring, desc, buf));
+
+DECLARE_EVENT_CLASS(
+ iavf_rx_template,
+
+ TP_PROTO(struct iavf_ring *ring,
+ union iavf_32byte_rx_desc *desc,
+ struct sk_buff *skb),
+
+ TP_ARGS(ring, desc, skb),
+
+ TP_STRUCT__entry(
+ __field(void*, ring)
+ __field(void*, desc)
+ __field(void*, skb)
+ __string(devname, ring->netdev->name)
+ ),
+
+ TP_fast_assign(
+ __entry->ring = ring;
+ __entry->desc = desc;
+ __entry->skb = skb;
+ __assign_str(devname, ring->netdev->name);
+ ),
+
+ TP_printk(
+ "netdev: %s ring: %p desc: %p skb %p",
+ __get_str(devname), __entry->ring,
+ __entry->desc, __entry->skb)
+);
+
+DEFINE_EVENT(
+ iavf_rx_template, iavf_clean_rx_irq,
+ TP_PROTO(struct iavf_ring *ring,
+ union iavf_32byte_rx_desc *desc,
+ struct sk_buff *skb),
+
+ TP_ARGS(ring, desc, skb));
+
+DEFINE_EVENT(
+ iavf_rx_template, iavf_clean_rx_irq_rx,
+ TP_PROTO(struct iavf_ring *ring,
+ union iavf_32byte_rx_desc *desc,
+ struct sk_buff *skb),
+
+ TP_ARGS(ring, desc, skb));
+
+DECLARE_EVENT_CLASS(
+ iavf_xmit_template,
+
+ TP_PROTO(struct sk_buff *skb,
+ struct iavf_ring *ring),
+
+ TP_ARGS(skb, ring),
+
+ TP_STRUCT__entry(
+ __field(void*, skb)
+ __field(void*, ring)
+ __string(devname, ring->netdev->name)
+ ),
+
+ TP_fast_assign(
+ __entry->skb = skb;
+ __entry->ring = ring;
+ __assign_str(devname, ring->netdev->name);
+ ),
+
+ TP_printk(
+ "netdev: %s skb: %p ring: %p",
+ __get_str(devname), __entry->skb,
+ __entry->ring)
+);
+
+DEFINE_EVENT(
+ iavf_xmit_template, iavf_xmit_frame_ring,
+ TP_PROTO(struct sk_buff *skb,
+ struct iavf_ring *ring),
+
+ TP_ARGS(skb, ring));
+
+DEFINE_EVENT(
+ iavf_xmit_template, iavf_xmit_frame_ring_drop,
+ TP_PROTO(struct sk_buff *skb,
+ struct iavf_ring *ring),
+
+ TP_ARGS(skb, ring));
+
+/* Events unique to the VF. */
+
+#endif /* _IAVF_TRACE_H_ */
+/* This must be outside ifdef _IAVF_TRACE_H */
+
+/* This trace include file is not located in the .../include/trace
+ * with the kernel tracepoint definitions, because we're a loadable
+ * module.
+ */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE iavf_trace
+#include <trace/define_trace.h>
diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
new file mode 100644
index 0000000000..8c5f6096b0
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
@@ -0,0 +1,2537 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#include <linux/prefetch.h>
+
+#include "iavf.h"
+#include "iavf_trace.h"
+#include "iavf_prototype.h"
+
+static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
+ u32 td_tag)
+{
+ return cpu_to_le64(IAVF_TX_DESC_DTYPE_DATA |
+ ((u64)td_cmd << IAVF_TXD_QW1_CMD_SHIFT) |
+ ((u64)td_offset << IAVF_TXD_QW1_OFFSET_SHIFT) |
+ ((u64)size << IAVF_TXD_QW1_TX_BUF_SZ_SHIFT) |
+ ((u64)td_tag << IAVF_TXD_QW1_L2TAG1_SHIFT));
+}
+
+#define IAVF_TXD_CMD (IAVF_TX_DESC_CMD_EOP | IAVF_TX_DESC_CMD_RS)
+
+/**
+ * iavf_unmap_and_free_tx_resource - Release a Tx buffer
+ * @ring: the ring that owns the buffer
+ * @tx_buffer: the buffer to free
+ **/
+static void iavf_unmap_and_free_tx_resource(struct iavf_ring *ring,
+ struct iavf_tx_buffer *tx_buffer)
+{
+ if (tx_buffer->skb) {
+ if (tx_buffer->tx_flags & IAVF_TX_FLAGS_FD_SB)
+ kfree(tx_buffer->raw_buf);
+ else
+ dev_kfree_skb_any(tx_buffer->skb);
+ if (dma_unmap_len(tx_buffer, len))
+ dma_unmap_single(ring->dev,
+ dma_unmap_addr(tx_buffer, dma),
+ dma_unmap_len(tx_buffer, len),
+ DMA_TO_DEVICE);
+ } else if (dma_unmap_len(tx_buffer, len)) {
+ dma_unmap_page(ring->dev,
+ dma_unmap_addr(tx_buffer, dma),
+ dma_unmap_len(tx_buffer, len),
+ DMA_TO_DEVICE);
+ }
+
+ tx_buffer->next_to_watch = NULL;
+ tx_buffer->skb = NULL;
+ dma_unmap_len_set(tx_buffer, len, 0);
+ /* tx_buffer must be completely set up in the transmit path */
+}
+
+/**
+ * iavf_clean_tx_ring - Free any empty Tx buffers
+ * @tx_ring: ring to be cleaned
+ **/
+static void iavf_clean_tx_ring(struct iavf_ring *tx_ring)
+{
+ unsigned long bi_size;
+ u16 i;
+
+ /* ring already cleared, nothing to do */
+ if (!tx_ring->tx_bi)
+ return;
+
+ /* Free all the Tx ring sk_buffs */
+ for (i = 0; i < tx_ring->count; i++)
+ iavf_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
+
+ bi_size = sizeof(struct iavf_tx_buffer) * tx_ring->count;
+ memset(tx_ring->tx_bi, 0, bi_size);
+
+ /* Zero out the descriptor ring */
+ memset(tx_ring->desc, 0, tx_ring->size);
+
+ tx_ring->next_to_use = 0;
+ tx_ring->next_to_clean = 0;
+
+ if (!tx_ring->netdev)
+ return;
+
+ /* cleanup Tx queue statistics */
+ netdev_tx_reset_queue(txring_txq(tx_ring));
+}
+
+/**
+ * iavf_free_tx_resources - Free Tx resources per queue
+ * @tx_ring: Tx descriptor ring for a specific queue
+ *
+ * Free all transmit software resources
+ **/
+void iavf_free_tx_resources(struct iavf_ring *tx_ring)
+{
+ iavf_clean_tx_ring(tx_ring);
+ kfree(tx_ring->tx_bi);
+ tx_ring->tx_bi = NULL;
+
+ if (tx_ring->desc) {
+ dma_free_coherent(tx_ring->dev, tx_ring->size,
+ tx_ring->desc, tx_ring->dma);
+ tx_ring->desc = NULL;
+ }
+}
+
+/**
+ * iavf_get_tx_pending - how many Tx descriptors not processed
+ * @ring: the ring of descriptors
+ * @in_sw: is tx_pending being checked in SW or HW
+ *
+ * Since there is no access to the ring head register
+ * in XL710, we need to use our local copies
+ **/
+static u32 iavf_get_tx_pending(struct iavf_ring *ring, bool in_sw)
+{
+ u32 head, tail;
+
+ /* underlying hardware might not allow access and/or always return
+ * 0 for the head/tail registers so just use the cached values
+ */
+ head = ring->next_to_clean;
+ tail = ring->next_to_use;
+
+ if (head != tail)
+ return (head < tail) ?
+ tail - head : (tail + ring->count - head);
+
+ return 0;
+}
+
+/**
+ * iavf_force_wb - Issue SW Interrupt so HW does a wb
+ * @vsi: the VSI we care about
+ * @q_vector: the vector on which to force writeback
+ **/
+static void iavf_force_wb(struct iavf_vsi *vsi, struct iavf_q_vector *q_vector)
+{
+ u32 val = IAVF_VFINT_DYN_CTLN1_INTENA_MASK |
+ IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK | /* set noitr */
+ IAVF_VFINT_DYN_CTLN1_SWINT_TRIG_MASK |
+ IAVF_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_MASK
+ /* allow 00 to be written to the index */;
+
+ wr32(&vsi->back->hw,
+ IAVF_VFINT_DYN_CTLN1(q_vector->reg_idx),
+ val);
+}
+
+/**
+ * iavf_detect_recover_hung - Function to detect and recover hung_queues
+ * @vsi: pointer to vsi struct with tx queues
+ *
+ * VSI has netdev and netdev has TX queues. This function is to check each of
+ * those TX queues if they are hung, trigger recovery by issuing SW interrupt.
+ **/
+void iavf_detect_recover_hung(struct iavf_vsi *vsi)
+{
+ struct iavf_ring *tx_ring = NULL;
+ struct net_device *netdev;
+ unsigned int i;
+ int packets;
+
+ if (!vsi)
+ return;
+
+ if (test_bit(__IAVF_VSI_DOWN, vsi->state))
+ return;
+
+ netdev = vsi->netdev;
+ if (!netdev)
+ return;
+
+ if (!netif_carrier_ok(netdev))
+ return;
+
+ for (i = 0; i < vsi->back->num_active_queues; i++) {
+ tx_ring = &vsi->back->tx_rings[i];
+ if (tx_ring && tx_ring->desc) {
+ /* If packet counter has not changed the queue is
+ * likely stalled, so force an interrupt for this
+ * queue.
+ *
+ * prev_pkt_ctr would be negative if there was no
+ * pending work.
+ */
+ packets = tx_ring->stats.packets & INT_MAX;
+ if (tx_ring->tx_stats.prev_pkt_ctr == packets) {
+ iavf_force_wb(vsi, tx_ring->q_vector);
+ continue;
+ }
+
+ /* Memory barrier between read of packet count and call
+ * to iavf_get_tx_pending()
+ */
+ smp_rmb();
+ tx_ring->tx_stats.prev_pkt_ctr =
+ iavf_get_tx_pending(tx_ring, true) ? packets : -1;
+ }
+ }
+}
+
+#define WB_STRIDE 4
+
+/**
+ * iavf_clean_tx_irq - Reclaim resources after transmit completes
+ * @vsi: the VSI we care about
+ * @tx_ring: Tx ring to clean
+ * @napi_budget: Used to determine if we are in netpoll
+ *
+ * Returns true if there's any budget left (e.g. the clean is finished)
+ **/
+static bool iavf_clean_tx_irq(struct iavf_vsi *vsi,
+ struct iavf_ring *tx_ring, int napi_budget)
+{
+ int i = tx_ring->next_to_clean;
+ struct iavf_tx_buffer *tx_buf;
+ struct iavf_tx_desc *tx_desc;
+ unsigned int total_bytes = 0, total_packets = 0;
+ unsigned int budget = IAVF_DEFAULT_IRQ_WORK;
+
+ tx_buf = &tx_ring->tx_bi[i];
+ tx_desc = IAVF_TX_DESC(tx_ring, i);
+ i -= tx_ring->count;
+
+ do {
+ struct iavf_tx_desc *eop_desc = tx_buf->next_to_watch;
+
+ /* if next_to_watch is not set then there is no work pending */
+ if (!eop_desc)
+ break;
+
+ /* prevent any other reads prior to eop_desc */
+ smp_rmb();
+
+ iavf_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf);
+ /* if the descriptor isn't done, no work yet to do */
+ if (!(eop_desc->cmd_type_offset_bsz &
+ cpu_to_le64(IAVF_TX_DESC_DTYPE_DESC_DONE)))
+ break;
+
+ /* clear next_to_watch to prevent false hangs */
+ tx_buf->next_to_watch = NULL;
+
+ /* update the statistics for this packet */
+ total_bytes += tx_buf->bytecount;
+ total_packets += tx_buf->gso_segs;
+
+ /* free the skb */
+ napi_consume_skb(tx_buf->skb, napi_budget);
+
+ /* unmap skb header data */
+ dma_unmap_single(tx_ring->dev,
+ dma_unmap_addr(tx_buf, dma),
+ dma_unmap_len(tx_buf, len),
+ DMA_TO_DEVICE);
+
+ /* clear tx_buffer data */
+ tx_buf->skb = NULL;
+ dma_unmap_len_set(tx_buf, len, 0);
+
+ /* unmap remaining buffers */
+ while (tx_desc != eop_desc) {
+ iavf_trace(clean_tx_irq_unmap,
+ tx_ring, tx_desc, tx_buf);
+
+ tx_buf++;
+ tx_desc++;
+ i++;
+ if (unlikely(!i)) {
+ i -= tx_ring->count;
+ tx_buf = tx_ring->tx_bi;
+ tx_desc = IAVF_TX_DESC(tx_ring, 0);
+ }
+
+ /* unmap any remaining paged data */
+ if (dma_unmap_len(tx_buf, len)) {
+ dma_unmap_page(tx_ring->dev,
+ dma_unmap_addr(tx_buf, dma),
+ dma_unmap_len(tx_buf, len),
+ DMA_TO_DEVICE);
+ dma_unmap_len_set(tx_buf, len, 0);
+ }
+ }
+
+ /* move us one more past the eop_desc for start of next pkt */
+ tx_buf++;
+ tx_desc++;
+ i++;
+ if (unlikely(!i)) {
+ i -= tx_ring->count;
+ tx_buf = tx_ring->tx_bi;
+ tx_desc = IAVF_TX_DESC(tx_ring, 0);
+ }
+
+ prefetch(tx_desc);
+
+ /* update budget accounting */
+ budget--;
+ } while (likely(budget));
+
+ i += tx_ring->count;
+ tx_ring->next_to_clean = i;
+ u64_stats_update_begin(&tx_ring->syncp);
+ tx_ring->stats.bytes += total_bytes;
+ tx_ring->stats.packets += total_packets;
+ u64_stats_update_end(&tx_ring->syncp);
+ tx_ring->q_vector->tx.total_bytes += total_bytes;
+ tx_ring->q_vector->tx.total_packets += total_packets;
+
+ if (tx_ring->flags & IAVF_TXR_FLAGS_WB_ON_ITR) {
+ /* check to see if there are < 4 descriptors
+ * waiting to be written back, then kick the hardware to force
+ * them to be written back in case we stay in NAPI.
+ * In this mode on X722 we do not enable Interrupt.
+ */
+ unsigned int j = iavf_get_tx_pending(tx_ring, false);
+
+ if (budget &&
+ ((j / WB_STRIDE) == 0) && (j > 0) &&
+ !test_bit(__IAVF_VSI_DOWN, vsi->state) &&
+ (IAVF_DESC_UNUSED(tx_ring) != tx_ring->count))
+ tx_ring->arm_wb = true;
+ }
+
+ /* notify netdev of completed buffers */
+ netdev_tx_completed_queue(txring_txq(tx_ring),
+ total_packets, total_bytes);
+
+#define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2))
+ if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
+ (IAVF_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
+ /* Make sure that anybody stopping the queue after this
+ * sees the new next_to_clean.
+ */
+ smp_mb();
+ if (__netif_subqueue_stopped(tx_ring->netdev,
+ tx_ring->queue_index) &&
+ !test_bit(__IAVF_VSI_DOWN, vsi->state)) {
+ netif_wake_subqueue(tx_ring->netdev,
+ tx_ring->queue_index);
+ ++tx_ring->tx_stats.restart_queue;
+ }
+ }
+
+ return !!budget;
+}
+
+/**
+ * iavf_enable_wb_on_itr - Arm hardware to do a wb, interrupts are not enabled
+ * @vsi: the VSI we care about
+ * @q_vector: the vector on which to enable writeback
+ *
+ **/
+static void iavf_enable_wb_on_itr(struct iavf_vsi *vsi,
+ struct iavf_q_vector *q_vector)
+{
+ u16 flags = q_vector->tx.ring[0].flags;
+ u32 val;
+
+ if (!(flags & IAVF_TXR_FLAGS_WB_ON_ITR))
+ return;
+
+ if (q_vector->arm_wb_state)
+ return;
+
+ val = IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_MASK |
+ IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK; /* set noitr */
+
+ wr32(&vsi->back->hw,
+ IAVF_VFINT_DYN_CTLN1(q_vector->reg_idx), val);
+ q_vector->arm_wb_state = true;
+}
+
+static inline bool iavf_container_is_rx(struct iavf_q_vector *q_vector,
+ struct iavf_ring_container *rc)
+{
+ return &q_vector->rx == rc;
+}
+
+#define IAVF_AIM_MULTIPLIER_100G 2560
+#define IAVF_AIM_MULTIPLIER_50G 1280
+#define IAVF_AIM_MULTIPLIER_40G 1024
+#define IAVF_AIM_MULTIPLIER_20G 512
+#define IAVF_AIM_MULTIPLIER_10G 256
+#define IAVF_AIM_MULTIPLIER_1G 32
+
+static unsigned int iavf_mbps_itr_multiplier(u32 speed_mbps)
+{
+ switch (speed_mbps) {
+ case SPEED_100000:
+ return IAVF_AIM_MULTIPLIER_100G;
+ case SPEED_50000:
+ return IAVF_AIM_MULTIPLIER_50G;
+ case SPEED_40000:
+ return IAVF_AIM_MULTIPLIER_40G;
+ case SPEED_25000:
+ case SPEED_20000:
+ return IAVF_AIM_MULTIPLIER_20G;
+ case SPEED_10000:
+ default:
+ return IAVF_AIM_MULTIPLIER_10G;
+ case SPEED_1000:
+ case SPEED_100:
+ return IAVF_AIM_MULTIPLIER_1G;
+ }
+}
+
+static unsigned int
+iavf_virtchnl_itr_multiplier(enum virtchnl_link_speed speed_virtchnl)
+{
+ switch (speed_virtchnl) {
+ case VIRTCHNL_LINK_SPEED_40GB:
+ return IAVF_AIM_MULTIPLIER_40G;
+ case VIRTCHNL_LINK_SPEED_25GB:
+ case VIRTCHNL_LINK_SPEED_20GB:
+ return IAVF_AIM_MULTIPLIER_20G;
+ case VIRTCHNL_LINK_SPEED_10GB:
+ default:
+ return IAVF_AIM_MULTIPLIER_10G;
+ case VIRTCHNL_LINK_SPEED_1GB:
+ case VIRTCHNL_LINK_SPEED_100MB:
+ return IAVF_AIM_MULTIPLIER_1G;
+ }
+}
+
+static unsigned int iavf_itr_divisor(struct iavf_adapter *adapter)
+{
+ if (ADV_LINK_SUPPORT(adapter))
+ return IAVF_ITR_ADAPTIVE_MIN_INC *
+ iavf_mbps_itr_multiplier(adapter->link_speed_mbps);
+ else
+ return IAVF_ITR_ADAPTIVE_MIN_INC *
+ iavf_virtchnl_itr_multiplier(adapter->link_speed);
+}
+
+/**
+ * iavf_update_itr - update the dynamic ITR value based on statistics
+ * @q_vector: structure containing interrupt and ring information
+ * @rc: structure containing ring performance data
+ *
+ * Stores a new ITR value based on packets and byte
+ * counts during the last interrupt. The advantage of per interrupt
+ * computation is faster updates and more accurate ITR for the current
+ * traffic pattern. Constants in this function were computed
+ * based on theoretical maximum wire speed and thresholds were set based
+ * on testing data as well as attempting to minimize response time
+ * while increasing bulk throughput.
+ **/
+static void iavf_update_itr(struct iavf_q_vector *q_vector,
+ struct iavf_ring_container *rc)
+{
+ unsigned int avg_wire_size, packets, bytes, itr;
+ unsigned long next_update = jiffies;
+
+ /* If we don't have any rings just leave ourselves set for maximum
+ * possible latency so we take ourselves out of the equation.
+ */
+ if (!rc->ring || !ITR_IS_DYNAMIC(rc->ring->itr_setting))
+ return;
+
+ /* For Rx we want to push the delay up and default to low latency.
+ * for Tx we want to pull the delay down and default to high latency.
+ */
+ itr = iavf_container_is_rx(q_vector, rc) ?
+ IAVF_ITR_ADAPTIVE_MIN_USECS | IAVF_ITR_ADAPTIVE_LATENCY :
+ IAVF_ITR_ADAPTIVE_MAX_USECS | IAVF_ITR_ADAPTIVE_LATENCY;
+
+ /* If we didn't update within up to 1 - 2 jiffies we can assume
+ * that either packets are coming in so slow there hasn't been
+ * any work, or that there is so much work that NAPI is dealing
+ * with interrupt moderation and we don't need to do anything.
+ */
+ if (time_after(next_update, rc->next_update))
+ goto clear_counts;
+
+ /* If itr_countdown is set it means we programmed an ITR within
+ * the last 4 interrupt cycles. This has a side effect of us
+ * potentially firing an early interrupt. In order to work around
+ * this we need to throw out any data received for a few
+ * interrupts following the update.
+ */
+ if (q_vector->itr_countdown) {
+ itr = rc->target_itr;
+ goto clear_counts;
+ }
+
+ packets = rc->total_packets;
+ bytes = rc->total_bytes;
+
+ if (iavf_container_is_rx(q_vector, rc)) {
+ /* If Rx there are 1 to 4 packets and bytes are less than
+ * 9000 assume insufficient data to use bulk rate limiting
+ * approach unless Tx is already in bulk rate limiting. We
+ * are likely latency driven.
+ */
+ if (packets && packets < 4 && bytes < 9000 &&
+ (q_vector->tx.target_itr & IAVF_ITR_ADAPTIVE_LATENCY)) {
+ itr = IAVF_ITR_ADAPTIVE_LATENCY;
+ goto adjust_by_size;
+ }
+ } else if (packets < 4) {
+ /* If we have Tx and Rx ITR maxed and Tx ITR is running in
+ * bulk mode and we are receiving 4 or fewer packets just
+ * reset the ITR_ADAPTIVE_LATENCY bit for latency mode so
+ * that the Rx can relax.
+ */
+ if (rc->target_itr == IAVF_ITR_ADAPTIVE_MAX_USECS &&
+ (q_vector->rx.target_itr & IAVF_ITR_MASK) ==
+ IAVF_ITR_ADAPTIVE_MAX_USECS)
+ goto clear_counts;
+ } else if (packets > 32) {
+ /* If we have processed over 32 packets in a single interrupt
+ * for Tx assume we need to switch over to "bulk" mode.
+ */
+ rc->target_itr &= ~IAVF_ITR_ADAPTIVE_LATENCY;
+ }
+
+ /* We have no packets to actually measure against. This means
+ * either one of the other queues on this vector is active or
+ * we are a Tx queue doing TSO with too high of an interrupt rate.
+ *
+ * Between 4 and 56 we can assume that our current interrupt delay
+ * is only slightly too low. As such we should increase it by a small
+ * fixed amount.
+ */
+ if (packets < 56) {
+ itr = rc->target_itr + IAVF_ITR_ADAPTIVE_MIN_INC;
+ if ((itr & IAVF_ITR_MASK) > IAVF_ITR_ADAPTIVE_MAX_USECS) {
+ itr &= IAVF_ITR_ADAPTIVE_LATENCY;
+ itr += IAVF_ITR_ADAPTIVE_MAX_USECS;
+ }
+ goto clear_counts;
+ }
+
+ if (packets <= 256) {
+ itr = min(q_vector->tx.current_itr, q_vector->rx.current_itr);
+ itr &= IAVF_ITR_MASK;
+
+ /* Between 56 and 112 is our "goldilocks" zone where we are
+ * working out "just right". Just report that our current
+ * ITR is good for us.
+ */
+ if (packets <= 112)
+ goto clear_counts;
+
+ /* If packet count is 128 or greater we are likely looking
+ * at a slight overrun of the delay we want. Try halving
+ * our delay to see if that will cut the number of packets
+ * in half per interrupt.
+ */
+ itr /= 2;
+ itr &= IAVF_ITR_MASK;
+ if (itr < IAVF_ITR_ADAPTIVE_MIN_USECS)
+ itr = IAVF_ITR_ADAPTIVE_MIN_USECS;
+
+ goto clear_counts;
+ }
+
+ /* The paths below assume we are dealing with a bulk ITR since
+ * number of packets is greater than 256. We are just going to have
+ * to compute a value and try to bring the count under control,
+ * though for smaller packet sizes there isn't much we can do as
+ * NAPI polling will likely be kicking in sooner rather than later.
+ */
+ itr = IAVF_ITR_ADAPTIVE_BULK;
+
+adjust_by_size:
+ /* If packet counts are 256 or greater we can assume we have a gross
+ * overestimation of what the rate should be. Instead of trying to fine
+ * tune it just use the formula below to try and dial in an exact value
+ * give the current packet size of the frame.
+ */
+ avg_wire_size = bytes / packets;
+
+ /* The following is a crude approximation of:
+ * wmem_default / (size + overhead) = desired_pkts_per_int
+ * rate / bits_per_byte / (size + ethernet overhead) = pkt_rate
+ * (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value
+ *
+ * Assuming wmem_default is 212992 and overhead is 640 bytes per
+ * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the
+ * formula down to
+ *
+ * (170 * (size + 24)) / (size + 640) = ITR
+ *
+ * We first do some math on the packet size and then finally bitshift
+ * by 8 after rounding up. We also have to account for PCIe link speed
+ * difference as ITR scales based on this.
+ */
+ if (avg_wire_size <= 60) {
+ /* Start at 250k ints/sec */
+ avg_wire_size = 4096;
+ } else if (avg_wire_size <= 380) {
+ /* 250K ints/sec to 60K ints/sec */
+ avg_wire_size *= 40;
+ avg_wire_size += 1696;
+ } else if (avg_wire_size <= 1084) {
+ /* 60K ints/sec to 36K ints/sec */
+ avg_wire_size *= 15;
+ avg_wire_size += 11452;
+ } else if (avg_wire_size <= 1980) {
+ /* 36K ints/sec to 30K ints/sec */
+ avg_wire_size *= 5;
+ avg_wire_size += 22420;
+ } else {
+ /* plateau at a limit of 30K ints/sec */
+ avg_wire_size = 32256;
+ }
+
+ /* If we are in low latency mode halve our delay which doubles the
+ * rate to somewhere between 100K to 16K ints/sec
+ */
+ if (itr & IAVF_ITR_ADAPTIVE_LATENCY)
+ avg_wire_size /= 2;
+
+ /* Resultant value is 256 times larger than it needs to be. This
+ * gives us room to adjust the value as needed to either increase
+ * or decrease the value based on link speeds of 10G, 2.5G, 1G, etc.
+ *
+ * Use addition as we have already recorded the new latency flag
+ * for the ITR value.
+ */
+ itr += DIV_ROUND_UP(avg_wire_size,
+ iavf_itr_divisor(q_vector->adapter)) *
+ IAVF_ITR_ADAPTIVE_MIN_INC;
+
+ if ((itr & IAVF_ITR_MASK) > IAVF_ITR_ADAPTIVE_MAX_USECS) {
+ itr &= IAVF_ITR_ADAPTIVE_LATENCY;
+ itr += IAVF_ITR_ADAPTIVE_MAX_USECS;
+ }
+
+clear_counts:
+ /* write back value */
+ rc->target_itr = itr;
+
+ /* next update should occur within next jiffy */
+ rc->next_update = next_update + 1;
+
+ rc->total_bytes = 0;
+ rc->total_packets = 0;
+}
+
+/**
+ * iavf_setup_tx_descriptors - Allocate the Tx descriptors
+ * @tx_ring: the tx ring to set up
+ *
+ * Return 0 on success, negative on error
+ **/
+int iavf_setup_tx_descriptors(struct iavf_ring *tx_ring)
+{
+ struct device *dev = tx_ring->dev;
+ int bi_size;
+
+ if (!dev)
+ return -ENOMEM;
+
+ /* warn if we are about to overwrite the pointer */
+ WARN_ON(tx_ring->tx_bi);
+ bi_size = sizeof(struct iavf_tx_buffer) * tx_ring->count;
+ tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
+ if (!tx_ring->tx_bi)
+ goto err;
+
+ /* round up to nearest 4K */
+ tx_ring->size = tx_ring->count * sizeof(struct iavf_tx_desc);
+ tx_ring->size = ALIGN(tx_ring->size, 4096);
+ tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
+ &tx_ring->dma, GFP_KERNEL);
+ if (!tx_ring->desc) {
+ dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
+ tx_ring->size);
+ goto err;
+ }
+
+ tx_ring->next_to_use = 0;
+ tx_ring->next_to_clean = 0;
+ tx_ring->tx_stats.prev_pkt_ctr = -1;
+ return 0;
+
+err:
+ kfree(tx_ring->tx_bi);
+ tx_ring->tx_bi = NULL;
+ return -ENOMEM;
+}
+
+/**
+ * iavf_clean_rx_ring - Free Rx buffers
+ * @rx_ring: ring to be cleaned
+ **/
+static void iavf_clean_rx_ring(struct iavf_ring *rx_ring)
+{
+ unsigned long bi_size;
+ u16 i;
+
+ /* ring already cleared, nothing to do */
+ if (!rx_ring->rx_bi)
+ return;
+
+ if (rx_ring->skb) {
+ dev_kfree_skb(rx_ring->skb);
+ rx_ring->skb = NULL;
+ }
+
+ /* Free all the Rx ring sk_buffs */
+ for (i = 0; i < rx_ring->count; i++) {
+ struct iavf_rx_buffer *rx_bi = &rx_ring->rx_bi[i];
+
+ if (!rx_bi->page)
+ continue;
+
+ /* Invalidate cache lines that may have been written to by
+ * device so that we avoid corrupting memory.
+ */
+ dma_sync_single_range_for_cpu(rx_ring->dev,
+ rx_bi->dma,
+ rx_bi->page_offset,
+ rx_ring->rx_buf_len,
+ DMA_FROM_DEVICE);
+
+ /* free resources associated with mapping */
+ dma_unmap_page_attrs(rx_ring->dev, rx_bi->dma,
+ iavf_rx_pg_size(rx_ring),
+ DMA_FROM_DEVICE,
+ IAVF_RX_DMA_ATTR);
+
+ __page_frag_cache_drain(rx_bi->page, rx_bi->pagecnt_bias);
+
+ rx_bi->page = NULL;
+ rx_bi->page_offset = 0;
+ }
+
+ bi_size = sizeof(struct iavf_rx_buffer) * rx_ring->count;
+ memset(rx_ring->rx_bi, 0, bi_size);
+
+ /* Zero out the descriptor ring */
+ memset(rx_ring->desc, 0, rx_ring->size);
+
+ rx_ring->next_to_alloc = 0;
+ rx_ring->next_to_clean = 0;
+ rx_ring->next_to_use = 0;
+}
+
+/**
+ * iavf_free_rx_resources - Free Rx resources
+ * @rx_ring: ring to clean the resources from
+ *
+ * Free all receive software resources
+ **/
+void iavf_free_rx_resources(struct iavf_ring *rx_ring)
+{
+ iavf_clean_rx_ring(rx_ring);
+ kfree(rx_ring->rx_bi);
+ rx_ring->rx_bi = NULL;
+
+ if (rx_ring->desc) {
+ dma_free_coherent(rx_ring->dev, rx_ring->size,
+ rx_ring->desc, rx_ring->dma);
+ rx_ring->desc = NULL;
+ }
+}
+
+/**
+ * iavf_setup_rx_descriptors - Allocate Rx descriptors
+ * @rx_ring: Rx descriptor ring (for a specific queue) to setup
+ *
+ * Returns 0 on success, negative on failure
+ **/
+int iavf_setup_rx_descriptors(struct iavf_ring *rx_ring)
+{
+ struct device *dev = rx_ring->dev;
+ int bi_size;
+
+ /* warn if we are about to overwrite the pointer */
+ WARN_ON(rx_ring->rx_bi);
+ bi_size = sizeof(struct iavf_rx_buffer) * rx_ring->count;
+ rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
+ if (!rx_ring->rx_bi)
+ goto err;
+
+ u64_stats_init(&rx_ring->syncp);
+
+ /* Round up to nearest 4K */
+ rx_ring->size = rx_ring->count * sizeof(union iavf_32byte_rx_desc);
+ rx_ring->size = ALIGN(rx_ring->size, 4096);
+ rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
+ &rx_ring->dma, GFP_KERNEL);
+
+ if (!rx_ring->desc) {
+ dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
+ rx_ring->size);
+ goto err;
+ }
+
+ rx_ring->next_to_alloc = 0;
+ rx_ring->next_to_clean = 0;
+ rx_ring->next_to_use = 0;
+
+ return 0;
+err:
+ kfree(rx_ring->rx_bi);
+ rx_ring->rx_bi = NULL;
+ return -ENOMEM;
+}
+
+/**
+ * iavf_release_rx_desc - Store the new tail and head values
+ * @rx_ring: ring to bump
+ * @val: new head index
+ **/
+static inline void iavf_release_rx_desc(struct iavf_ring *rx_ring, u32 val)
+{
+ rx_ring->next_to_use = val;
+
+ /* update next to alloc since we have filled the ring */
+ rx_ring->next_to_alloc = val;
+
+ /* Force memory writes to complete before letting h/w
+ * know there are new descriptors to fetch. (Only
+ * applicable for weak-ordered memory model archs,
+ * such as IA-64).
+ */
+ wmb();
+ writel(val, rx_ring->tail);
+}
+
+/**
+ * iavf_rx_offset - Return expected offset into page to access data
+ * @rx_ring: Ring we are requesting offset of
+ *
+ * Returns the offset value for ring into the data buffer.
+ */
+static inline unsigned int iavf_rx_offset(struct iavf_ring *rx_ring)
+{
+ return ring_uses_build_skb(rx_ring) ? IAVF_SKB_PAD : 0;
+}
+
+/**
+ * iavf_alloc_mapped_page - recycle or make a new page
+ * @rx_ring: ring to use
+ * @bi: rx_buffer struct to modify
+ *
+ * Returns true if the page was successfully allocated or
+ * reused.
+ **/
+static bool iavf_alloc_mapped_page(struct iavf_ring *rx_ring,
+ struct iavf_rx_buffer *bi)
+{
+ struct page *page = bi->page;
+ dma_addr_t dma;
+
+ /* since we are recycling buffers we should seldom need to alloc */
+ if (likely(page)) {
+ rx_ring->rx_stats.page_reuse_count++;
+ return true;
+ }
+
+ /* alloc new page for storage */
+ page = dev_alloc_pages(iavf_rx_pg_order(rx_ring));
+ if (unlikely(!page)) {
+ rx_ring->rx_stats.alloc_page_failed++;
+ return false;
+ }
+
+ /* map page for use */
+ dma = dma_map_page_attrs(rx_ring->dev, page, 0,
+ iavf_rx_pg_size(rx_ring),
+ DMA_FROM_DEVICE,
+ IAVF_RX_DMA_ATTR);
+
+ /* if mapping failed free memory back to system since
+ * there isn't much point in holding memory we can't use
+ */
+ if (dma_mapping_error(rx_ring->dev, dma)) {
+ __free_pages(page, iavf_rx_pg_order(rx_ring));
+ rx_ring->rx_stats.alloc_page_failed++;
+ return false;
+ }
+
+ bi->dma = dma;
+ bi->page = page;
+ bi->page_offset = iavf_rx_offset(rx_ring);
+
+ /* initialize pagecnt_bias to 1 representing we fully own page */
+ bi->pagecnt_bias = 1;
+
+ return true;
+}
+
+/**
+ * iavf_receive_skb - Send a completed packet up the stack
+ * @rx_ring: rx ring in play
+ * @skb: packet to send up
+ * @vlan_tag: vlan tag for packet
+ **/
+static void iavf_receive_skb(struct iavf_ring *rx_ring,
+ struct sk_buff *skb, u16 vlan_tag)
+{
+ struct iavf_q_vector *q_vector = rx_ring->q_vector;
+
+ if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
+ (vlan_tag & VLAN_VID_MASK))
+ __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
+ else if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_STAG_RX) &&
+ vlan_tag & VLAN_VID_MASK)
+ __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021AD), vlan_tag);
+
+ napi_gro_receive(&q_vector->napi, skb);
+}
+
+/**
+ * iavf_alloc_rx_buffers - Replace used receive buffers
+ * @rx_ring: ring to place buffers on
+ * @cleaned_count: number of buffers to replace
+ *
+ * Returns false if all allocations were successful, true if any fail
+ **/
+bool iavf_alloc_rx_buffers(struct iavf_ring *rx_ring, u16 cleaned_count)
+{
+ u16 ntu = rx_ring->next_to_use;
+ union iavf_rx_desc *rx_desc;
+ struct iavf_rx_buffer *bi;
+
+ /* do nothing if no valid netdev defined */
+ if (!rx_ring->netdev || !cleaned_count)
+ return false;
+
+ rx_desc = IAVF_RX_DESC(rx_ring, ntu);
+ bi = &rx_ring->rx_bi[ntu];
+
+ do {
+ if (!iavf_alloc_mapped_page(rx_ring, bi))
+ goto no_buffers;
+
+ /* sync the buffer for use by the device */
+ dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
+ bi->page_offset,
+ rx_ring->rx_buf_len,
+ DMA_FROM_DEVICE);
+
+ /* Refresh the desc even if buffer_addrs didn't change
+ * because each write-back erases this info.
+ */
+ rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
+
+ rx_desc++;
+ bi++;
+ ntu++;
+ if (unlikely(ntu == rx_ring->count)) {
+ rx_desc = IAVF_RX_DESC(rx_ring, 0);
+ bi = rx_ring->rx_bi;
+ ntu = 0;
+ }
+
+ /* clear the status bits for the next_to_use descriptor */
+ rx_desc->wb.qword1.status_error_len = 0;
+
+ cleaned_count--;
+ } while (cleaned_count);
+
+ if (rx_ring->next_to_use != ntu)
+ iavf_release_rx_desc(rx_ring, ntu);
+
+ return false;
+
+no_buffers:
+ if (rx_ring->next_to_use != ntu)
+ iavf_release_rx_desc(rx_ring, ntu);
+
+ /* make sure to come back via polling to try again after
+ * allocation failure
+ */
+ return true;
+}
+
+/**
+ * iavf_rx_checksum - Indicate in skb if hw indicated a good cksum
+ * @vsi: the VSI we care about
+ * @skb: skb currently being received and modified
+ * @rx_desc: the receive descriptor
+ **/
+static inline void iavf_rx_checksum(struct iavf_vsi *vsi,
+ struct sk_buff *skb,
+ union iavf_rx_desc *rx_desc)
+{
+ struct iavf_rx_ptype_decoded decoded;
+ u32 rx_error, rx_status;
+ bool ipv4, ipv6;
+ u8 ptype;
+ u64 qword;
+
+ qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
+ ptype = (qword & IAVF_RXD_QW1_PTYPE_MASK) >> IAVF_RXD_QW1_PTYPE_SHIFT;
+ rx_error = (qword & IAVF_RXD_QW1_ERROR_MASK) >>
+ IAVF_RXD_QW1_ERROR_SHIFT;
+ rx_status = (qword & IAVF_RXD_QW1_STATUS_MASK) >>
+ IAVF_RXD_QW1_STATUS_SHIFT;
+ decoded = decode_rx_desc_ptype(ptype);
+
+ skb->ip_summed = CHECKSUM_NONE;
+
+ skb_checksum_none_assert(skb);
+
+ /* Rx csum enabled and ip headers found? */
+ if (!(vsi->netdev->features & NETIF_F_RXCSUM))
+ return;
+
+ /* did the hardware decode the packet and checksum? */
+ if (!(rx_status & BIT(IAVF_RX_DESC_STATUS_L3L4P_SHIFT)))
+ return;
+
+ /* both known and outer_ip must be set for the below code to work */
+ if (!(decoded.known && decoded.outer_ip))
+ return;
+
+ ipv4 = (decoded.outer_ip == IAVF_RX_PTYPE_OUTER_IP) &&
+ (decoded.outer_ip_ver == IAVF_RX_PTYPE_OUTER_IPV4);
+ ipv6 = (decoded.outer_ip == IAVF_RX_PTYPE_OUTER_IP) &&
+ (decoded.outer_ip_ver == IAVF_RX_PTYPE_OUTER_IPV6);
+
+ if (ipv4 &&
+ (rx_error & (BIT(IAVF_RX_DESC_ERROR_IPE_SHIFT) |
+ BIT(IAVF_RX_DESC_ERROR_EIPE_SHIFT))))
+ goto checksum_fail;
+
+ /* likely incorrect csum if alternate IP extension headers found */
+ if (ipv6 &&
+ rx_status & BIT(IAVF_RX_DESC_STATUS_IPV6EXADD_SHIFT))
+ /* don't increment checksum err here, non-fatal err */
+ return;
+
+ /* there was some L4 error, count error and punt packet to the stack */
+ if (rx_error & BIT(IAVF_RX_DESC_ERROR_L4E_SHIFT))
+ goto checksum_fail;
+
+ /* handle packets that were not able to be checksummed due
+ * to arrival speed, in this case the stack can compute
+ * the csum.
+ */
+ if (rx_error & BIT(IAVF_RX_DESC_ERROR_PPRS_SHIFT))
+ return;
+
+ /* Only report checksum unnecessary for TCP, UDP, or SCTP */
+ switch (decoded.inner_prot) {
+ case IAVF_RX_PTYPE_INNER_PROT_TCP:
+ case IAVF_RX_PTYPE_INNER_PROT_UDP:
+ case IAVF_RX_PTYPE_INNER_PROT_SCTP:
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+ fallthrough;
+ default:
+ break;
+ }
+
+ return;
+
+checksum_fail:
+ vsi->back->hw_csum_rx_error++;
+}
+
+/**
+ * iavf_ptype_to_htype - get a hash type
+ * @ptype: the ptype value from the descriptor
+ *
+ * Returns a hash type to be used by skb_set_hash
+ **/
+static inline int iavf_ptype_to_htype(u8 ptype)
+{
+ struct iavf_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
+
+ if (!decoded.known)
+ return PKT_HASH_TYPE_NONE;
+
+ if (decoded.outer_ip == IAVF_RX_PTYPE_OUTER_IP &&
+ decoded.payload_layer == IAVF_RX_PTYPE_PAYLOAD_LAYER_PAY4)
+ return PKT_HASH_TYPE_L4;
+ else if (decoded.outer_ip == IAVF_RX_PTYPE_OUTER_IP &&
+ decoded.payload_layer == IAVF_RX_PTYPE_PAYLOAD_LAYER_PAY3)
+ return PKT_HASH_TYPE_L3;
+ else
+ return PKT_HASH_TYPE_L2;
+}
+
+/**
+ * iavf_rx_hash - set the hash value in the skb
+ * @ring: descriptor ring
+ * @rx_desc: specific descriptor
+ * @skb: skb currently being received and modified
+ * @rx_ptype: Rx packet type
+ **/
+static inline void iavf_rx_hash(struct iavf_ring *ring,
+ union iavf_rx_desc *rx_desc,
+ struct sk_buff *skb,
+ u8 rx_ptype)
+{
+ u32 hash;
+ const __le64 rss_mask =
+ cpu_to_le64((u64)IAVF_RX_DESC_FLTSTAT_RSS_HASH <<
+ IAVF_RX_DESC_STATUS_FLTSTAT_SHIFT);
+
+ if (!(ring->netdev->features & NETIF_F_RXHASH))
+ return;
+
+ if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) {
+ hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
+ skb_set_hash(skb, hash, iavf_ptype_to_htype(rx_ptype));
+ }
+}
+
+/**
+ * iavf_process_skb_fields - Populate skb header fields from Rx descriptor
+ * @rx_ring: rx descriptor ring packet is being transacted on
+ * @rx_desc: pointer to the EOP Rx descriptor
+ * @skb: pointer to current skb being populated
+ * @rx_ptype: the packet type decoded by hardware
+ *
+ * This function checks the ring, descriptor, and packet information in
+ * order to populate the hash, checksum, VLAN, protocol, and
+ * other fields within the skb.
+ **/
+static inline
+void iavf_process_skb_fields(struct iavf_ring *rx_ring,
+ union iavf_rx_desc *rx_desc, struct sk_buff *skb,
+ u8 rx_ptype)
+{
+ iavf_rx_hash(rx_ring, rx_desc, skb, rx_ptype);
+
+ iavf_rx_checksum(rx_ring->vsi, skb, rx_desc);
+
+ skb_record_rx_queue(skb, rx_ring->queue_index);
+
+ /* modifies the skb - consumes the enet header */
+ skb->protocol = eth_type_trans(skb, rx_ring->netdev);
+}
+
+/**
+ * iavf_cleanup_headers - Correct empty headers
+ * @rx_ring: rx descriptor ring packet is being transacted on
+ * @skb: pointer to current skb being fixed
+ *
+ * Also address the case where we are pulling data in on pages only
+ * and as such no data is present in the skb header.
+ *
+ * In addition if skb is not at least 60 bytes we need to pad it so that
+ * it is large enough to qualify as a valid Ethernet frame.
+ *
+ * Returns true if an error was encountered and skb was freed.
+ **/
+static bool iavf_cleanup_headers(struct iavf_ring *rx_ring, struct sk_buff *skb)
+{
+ /* if eth_skb_pad returns an error the skb was freed */
+ if (eth_skb_pad(skb))
+ return true;
+
+ return false;
+}
+
+/**
+ * iavf_reuse_rx_page - page flip buffer and store it back on the ring
+ * @rx_ring: rx descriptor ring to store buffers on
+ * @old_buff: donor buffer to have page reused
+ *
+ * Synchronizes page for reuse by the adapter
+ **/
+static void iavf_reuse_rx_page(struct iavf_ring *rx_ring,
+ struct iavf_rx_buffer *old_buff)
+{
+ struct iavf_rx_buffer *new_buff;
+ u16 nta = rx_ring->next_to_alloc;
+
+ new_buff = &rx_ring->rx_bi[nta];
+
+ /* update, and store next to alloc */
+ nta++;
+ rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
+
+ /* transfer page from old buffer to new buffer */
+ new_buff->dma = old_buff->dma;
+ new_buff->page = old_buff->page;
+ new_buff->page_offset = old_buff->page_offset;
+ new_buff->pagecnt_bias = old_buff->pagecnt_bias;
+}
+
+/**
+ * iavf_can_reuse_rx_page - Determine if this page can be reused by
+ * the adapter for another receive
+ *
+ * @rx_buffer: buffer containing the page
+ *
+ * If page is reusable, rx_buffer->page_offset is adjusted to point to
+ * an unused region in the page.
+ *
+ * For small pages, @truesize will be a constant value, half the size
+ * of the memory at page. We'll attempt to alternate between high and
+ * low halves of the page, with one half ready for use by the hardware
+ * and the other half being consumed by the stack. We use the page
+ * ref count to determine whether the stack has finished consuming the
+ * portion of this page that was passed up with a previous packet. If
+ * the page ref count is >1, we'll assume the "other" half page is
+ * still busy, and this page cannot be reused.
+ *
+ * For larger pages, @truesize will be the actual space used by the
+ * received packet (adjusted upward to an even multiple of the cache
+ * line size). This will advance through the page by the amount
+ * actually consumed by the received packets while there is still
+ * space for a buffer. Each region of larger pages will be used at
+ * most once, after which the page will not be reused.
+ *
+ * In either case, if the page is reusable its refcount is increased.
+ **/
+static bool iavf_can_reuse_rx_page(struct iavf_rx_buffer *rx_buffer)
+{
+ unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
+ struct page *page = rx_buffer->page;
+
+ /* Is any reuse possible? */
+ if (!dev_page_is_reusable(page))
+ return false;
+
+#if (PAGE_SIZE < 8192)
+ /* if we are only owner of page we can reuse it */
+ if (unlikely((page_count(page) - pagecnt_bias) > 1))
+ return false;
+#else
+#define IAVF_LAST_OFFSET \
+ (SKB_WITH_OVERHEAD(PAGE_SIZE) - IAVF_RXBUFFER_2048)
+ if (rx_buffer->page_offset > IAVF_LAST_OFFSET)
+ return false;
+#endif
+
+ /* If we have drained the page fragment pool we need to update
+ * the pagecnt_bias and page count so that we fully restock the
+ * number of references the driver holds.
+ */
+ if (unlikely(!pagecnt_bias)) {
+ page_ref_add(page, USHRT_MAX);
+ rx_buffer->pagecnt_bias = USHRT_MAX;
+ }
+
+ return true;
+}
+
+/**
+ * iavf_add_rx_frag - Add contents of Rx buffer to sk_buff
+ * @rx_ring: rx descriptor ring to transact packets on
+ * @rx_buffer: buffer containing page to add
+ * @skb: sk_buff to place the data into
+ * @size: packet length from rx_desc
+ *
+ * This function will add the data contained in rx_buffer->page to the skb.
+ * It will just attach the page as a frag to the skb.
+ *
+ * The function will then update the page offset.
+ **/
+static void iavf_add_rx_frag(struct iavf_ring *rx_ring,
+ struct iavf_rx_buffer *rx_buffer,
+ struct sk_buff *skb,
+ unsigned int size)
+{
+#if (PAGE_SIZE < 8192)
+ unsigned int truesize = iavf_rx_pg_size(rx_ring) / 2;
+#else
+ unsigned int truesize = SKB_DATA_ALIGN(size + iavf_rx_offset(rx_ring));
+#endif
+
+ if (!size)
+ return;
+
+ skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
+ rx_buffer->page_offset, size, truesize);
+
+ /* page is being used so we must update the page offset */
+#if (PAGE_SIZE < 8192)
+ rx_buffer->page_offset ^= truesize;
+#else
+ rx_buffer->page_offset += truesize;
+#endif
+}
+
+/**
+ * iavf_get_rx_buffer - Fetch Rx buffer and synchronize data for use
+ * @rx_ring: rx descriptor ring to transact packets on
+ * @size: size of buffer to add to skb
+ *
+ * This function will pull an Rx buffer from the ring and synchronize it
+ * for use by the CPU.
+ */
+static struct iavf_rx_buffer *iavf_get_rx_buffer(struct iavf_ring *rx_ring,
+ const unsigned int size)
+{
+ struct iavf_rx_buffer *rx_buffer;
+
+ rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean];
+ prefetchw(rx_buffer->page);
+ if (!size)
+ return rx_buffer;
+
+ /* we are reusing so sync this buffer for CPU use */
+ dma_sync_single_range_for_cpu(rx_ring->dev,
+ rx_buffer->dma,
+ rx_buffer->page_offset,
+ size,
+ DMA_FROM_DEVICE);
+
+ /* We have pulled a buffer for use, so decrement pagecnt_bias */
+ rx_buffer->pagecnt_bias--;
+
+ return rx_buffer;
+}
+
+/**
+ * iavf_construct_skb - Allocate skb and populate it
+ * @rx_ring: rx descriptor ring to transact packets on
+ * @rx_buffer: rx buffer to pull data from
+ * @size: size of buffer to add to skb
+ *
+ * This function allocates an skb. It then populates it with the page
+ * data from the current receive descriptor, taking care to set up the
+ * skb correctly.
+ */
+static struct sk_buff *iavf_construct_skb(struct iavf_ring *rx_ring,
+ struct iavf_rx_buffer *rx_buffer,
+ unsigned int size)
+{
+ void *va;
+#if (PAGE_SIZE < 8192)
+ unsigned int truesize = iavf_rx_pg_size(rx_ring) / 2;
+#else
+ unsigned int truesize = SKB_DATA_ALIGN(size);
+#endif
+ unsigned int headlen;
+ struct sk_buff *skb;
+
+ if (!rx_buffer)
+ return NULL;
+ /* prefetch first cache line of first page */
+ va = page_address(rx_buffer->page) + rx_buffer->page_offset;
+ net_prefetch(va);
+
+ /* allocate a skb to store the frags */
+ skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
+ IAVF_RX_HDR_SIZE,
+ GFP_ATOMIC | __GFP_NOWARN);
+ if (unlikely(!skb))
+ return NULL;
+
+ /* Determine available headroom for copy */
+ headlen = size;
+ if (headlen > IAVF_RX_HDR_SIZE)
+ headlen = eth_get_headlen(skb->dev, va, IAVF_RX_HDR_SIZE);
+
+ /* align pull length to size of long to optimize memcpy performance */
+ memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
+
+ /* update all of the pointers */
+ size -= headlen;
+ if (size) {
+ skb_add_rx_frag(skb, 0, rx_buffer->page,
+ rx_buffer->page_offset + headlen,
+ size, truesize);
+
+ /* buffer is used by skb, update page_offset */
+#if (PAGE_SIZE < 8192)
+ rx_buffer->page_offset ^= truesize;
+#else
+ rx_buffer->page_offset += truesize;
+#endif
+ } else {
+ /* buffer is unused, reset bias back to rx_buffer */
+ rx_buffer->pagecnt_bias++;
+ }
+
+ return skb;
+}
+
+/**
+ * iavf_build_skb - Build skb around an existing buffer
+ * @rx_ring: Rx descriptor ring to transact packets on
+ * @rx_buffer: Rx buffer to pull data from
+ * @size: size of buffer to add to skb
+ *
+ * This function builds an skb around an existing Rx buffer, taking care
+ * to set up the skb correctly and avoid any memcpy overhead.
+ */
+static struct sk_buff *iavf_build_skb(struct iavf_ring *rx_ring,
+ struct iavf_rx_buffer *rx_buffer,
+ unsigned int size)
+{
+ void *va;
+#if (PAGE_SIZE < 8192)
+ unsigned int truesize = iavf_rx_pg_size(rx_ring) / 2;
+#else
+ unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
+ SKB_DATA_ALIGN(IAVF_SKB_PAD + size);
+#endif
+ struct sk_buff *skb;
+
+ if (!rx_buffer || !size)
+ return NULL;
+ /* prefetch first cache line of first page */
+ va = page_address(rx_buffer->page) + rx_buffer->page_offset;
+ net_prefetch(va);
+
+ /* build an skb around the page buffer */
+ skb = napi_build_skb(va - IAVF_SKB_PAD, truesize);
+ if (unlikely(!skb))
+ return NULL;
+
+ /* update pointers within the skb to store the data */
+ skb_reserve(skb, IAVF_SKB_PAD);
+ __skb_put(skb, size);
+
+ /* buffer is used by skb, update page_offset */
+#if (PAGE_SIZE < 8192)
+ rx_buffer->page_offset ^= truesize;
+#else
+ rx_buffer->page_offset += truesize;
+#endif
+
+ return skb;
+}
+
+/**
+ * iavf_put_rx_buffer - Clean up used buffer and either recycle or free
+ * @rx_ring: rx descriptor ring to transact packets on
+ * @rx_buffer: rx buffer to pull data from
+ *
+ * This function will clean up the contents of the rx_buffer. It will
+ * either recycle the buffer or unmap it and free the associated resources.
+ */
+static void iavf_put_rx_buffer(struct iavf_ring *rx_ring,
+ struct iavf_rx_buffer *rx_buffer)
+{
+ if (!rx_buffer)
+ return;
+
+ if (iavf_can_reuse_rx_page(rx_buffer)) {
+ /* hand second half of page back to the ring */
+ iavf_reuse_rx_page(rx_ring, rx_buffer);
+ rx_ring->rx_stats.page_reuse_count++;
+ } else {
+ /* we are not reusing the buffer so unmap it */
+ dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
+ iavf_rx_pg_size(rx_ring),
+ DMA_FROM_DEVICE, IAVF_RX_DMA_ATTR);
+ __page_frag_cache_drain(rx_buffer->page,
+ rx_buffer->pagecnt_bias);
+ }
+
+ /* clear contents of buffer_info */
+ rx_buffer->page = NULL;
+}
+
+/**
+ * iavf_is_non_eop - process handling of non-EOP buffers
+ * @rx_ring: Rx ring being processed
+ * @rx_desc: Rx descriptor for current buffer
+ * @skb: Current socket buffer containing buffer in progress
+ *
+ * This function updates next to clean. If the buffer is an EOP buffer
+ * this function exits returning false, otherwise it will place the
+ * sk_buff in the next buffer to be chained and return true indicating
+ * that this is in fact a non-EOP buffer.
+ **/
+static bool iavf_is_non_eop(struct iavf_ring *rx_ring,
+ union iavf_rx_desc *rx_desc,
+ struct sk_buff *skb)
+{
+ u32 ntc = rx_ring->next_to_clean + 1;
+
+ /* fetch, update, and store next to clean */
+ ntc = (ntc < rx_ring->count) ? ntc : 0;
+ rx_ring->next_to_clean = ntc;
+
+ prefetch(IAVF_RX_DESC(rx_ring, ntc));
+
+ /* if we are the last buffer then there is nothing else to do */
+#define IAVF_RXD_EOF BIT(IAVF_RX_DESC_STATUS_EOF_SHIFT)
+ if (likely(iavf_test_staterr(rx_desc, IAVF_RXD_EOF)))
+ return false;
+
+ rx_ring->rx_stats.non_eop_descs++;
+
+ return true;
+}
+
+/**
+ * iavf_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
+ * @rx_ring: rx descriptor ring to transact packets on
+ * @budget: Total limit on number of packets to process
+ *
+ * This function provides a "bounce buffer" approach to Rx interrupt
+ * processing. The advantage to this is that on systems that have
+ * expensive overhead for IOMMU access this provides a means of avoiding
+ * it by maintaining the mapping of the page to the system.
+ *
+ * Returns amount of work completed
+ **/
+static int iavf_clean_rx_irq(struct iavf_ring *rx_ring, int budget)
+{
+ unsigned int total_rx_bytes = 0, total_rx_packets = 0;
+ struct sk_buff *skb = rx_ring->skb;
+ u16 cleaned_count = IAVF_DESC_UNUSED(rx_ring);
+ bool failure = false;
+
+ while (likely(total_rx_packets < (unsigned int)budget)) {
+ struct iavf_rx_buffer *rx_buffer;
+ union iavf_rx_desc *rx_desc;
+ unsigned int size;
+ u16 vlan_tag = 0;
+ u8 rx_ptype;
+ u64 qword;
+
+ /* return some buffers to hardware, one at a time is too slow */
+ if (cleaned_count >= IAVF_RX_BUFFER_WRITE) {
+ failure = failure ||
+ iavf_alloc_rx_buffers(rx_ring, cleaned_count);
+ cleaned_count = 0;
+ }
+
+ rx_desc = IAVF_RX_DESC(rx_ring, rx_ring->next_to_clean);
+
+ /* status_error_len will always be zero for unused descriptors
+ * because it's cleared in cleanup, and overlaps with hdr_addr
+ * which is always zero because packet split isn't used, if the
+ * hardware wrote DD then the length will be non-zero
+ */
+ qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
+
+ /* This memory barrier is needed to keep us from reading
+ * any other fields out of the rx_desc until we have
+ * verified the descriptor has been written back.
+ */
+ dma_rmb();
+#define IAVF_RXD_DD BIT(IAVF_RX_DESC_STATUS_DD_SHIFT)
+ if (!iavf_test_staterr(rx_desc, IAVF_RXD_DD))
+ break;
+
+ size = (qword & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >>
+ IAVF_RXD_QW1_LENGTH_PBUF_SHIFT;
+
+ iavf_trace(clean_rx_irq, rx_ring, rx_desc, skb);
+ rx_buffer = iavf_get_rx_buffer(rx_ring, size);
+
+ /* retrieve a buffer from the ring */
+ if (skb)
+ iavf_add_rx_frag(rx_ring, rx_buffer, skb, size);
+ else if (ring_uses_build_skb(rx_ring))
+ skb = iavf_build_skb(rx_ring, rx_buffer, size);
+ else
+ skb = iavf_construct_skb(rx_ring, rx_buffer, size);
+
+ /* exit if we failed to retrieve a buffer */
+ if (!skb) {
+ rx_ring->rx_stats.alloc_buff_failed++;
+ if (rx_buffer && size)
+ rx_buffer->pagecnt_bias++;
+ break;
+ }
+
+ iavf_put_rx_buffer(rx_ring, rx_buffer);
+ cleaned_count++;
+
+ if (iavf_is_non_eop(rx_ring, rx_desc, skb))
+ continue;
+
+ /* ERR_MASK will only have valid bits if EOP set, and
+ * what we are doing here is actually checking
+ * IAVF_RX_DESC_ERROR_RXE_SHIFT, since it is the zeroth bit in
+ * the error field
+ */
+ if (unlikely(iavf_test_staterr(rx_desc, BIT(IAVF_RXD_QW1_ERROR_SHIFT)))) {
+ dev_kfree_skb_any(skb);
+ skb = NULL;
+ continue;
+ }
+
+ if (iavf_cleanup_headers(rx_ring, skb)) {
+ skb = NULL;
+ continue;
+ }
+
+ /* probably a little skewed due to removing CRC */
+ total_rx_bytes += skb->len;
+
+ qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
+ rx_ptype = (qword & IAVF_RXD_QW1_PTYPE_MASK) >>
+ IAVF_RXD_QW1_PTYPE_SHIFT;
+
+ /* populate checksum, VLAN, and protocol */
+ iavf_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);
+
+ if (qword & BIT(IAVF_RX_DESC_STATUS_L2TAG1P_SHIFT) &&
+ rx_ring->flags & IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1)
+ vlan_tag = le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1);
+ if (rx_desc->wb.qword2.ext_status &
+ cpu_to_le16(BIT(IAVF_RX_DESC_EXT_STATUS_L2TAG2P_SHIFT)) &&
+ rx_ring->flags & IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2)
+ vlan_tag = le16_to_cpu(rx_desc->wb.qword2.l2tag2_2);
+
+ iavf_trace(clean_rx_irq_rx, rx_ring, rx_desc, skb);
+ iavf_receive_skb(rx_ring, skb, vlan_tag);
+ skb = NULL;
+
+ /* update budget accounting */
+ total_rx_packets++;
+ }
+
+ rx_ring->skb = skb;
+
+ u64_stats_update_begin(&rx_ring->syncp);
+ rx_ring->stats.packets += total_rx_packets;
+ rx_ring->stats.bytes += total_rx_bytes;
+ u64_stats_update_end(&rx_ring->syncp);
+ rx_ring->q_vector->rx.total_packets += total_rx_packets;
+ rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
+
+ /* guarantee a trip back through this routine if there was a failure */
+ return failure ? budget : (int)total_rx_packets;
+}
+
+static inline u32 iavf_buildreg_itr(const int type, u16 itr)
+{
+ u32 val;
+
+ /* We don't bother with setting the CLEARPBA bit as the data sheet
+ * points out doing so is "meaningless since it was already
+ * auto-cleared". The auto-clearing happens when the interrupt is
+ * asserted.
+ *
+ * Hardware errata 28 for also indicates that writing to a
+ * xxINT_DYN_CTLx CSR with INTENA_MSK (bit 31) set to 0 will clear
+ * an event in the PBA anyway so we need to rely on the automask
+ * to hold pending events for us until the interrupt is re-enabled
+ *
+ * The itr value is reported in microseconds, and the register
+ * value is recorded in 2 microsecond units. For this reason we
+ * only need to shift by the interval shift - 1 instead of the
+ * full value.
+ */
+ itr &= IAVF_ITR_MASK;
+
+ val = IAVF_VFINT_DYN_CTLN1_INTENA_MASK |
+ (type << IAVF_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
+ (itr << (IAVF_VFINT_DYN_CTLN1_INTERVAL_SHIFT - 1));
+
+ return val;
+}
+
+/* a small macro to shorten up some long lines */
+#define INTREG IAVF_VFINT_DYN_CTLN1
+
+/* The act of updating the ITR will cause it to immediately trigger. In order
+ * to prevent this from throwing off adaptive update statistics we defer the
+ * update so that it can only happen so often. So after either Tx or Rx are
+ * updated we make the adaptive scheme wait until either the ITR completely
+ * expires via the next_update expiration or we have been through at least
+ * 3 interrupts.
+ */
+#define ITR_COUNTDOWN_START 3
+
+/**
+ * iavf_update_enable_itr - Update itr and re-enable MSIX interrupt
+ * @vsi: the VSI we care about
+ * @q_vector: q_vector for which itr is being updated and interrupt enabled
+ *
+ **/
+static inline void iavf_update_enable_itr(struct iavf_vsi *vsi,
+ struct iavf_q_vector *q_vector)
+{
+ struct iavf_hw *hw = &vsi->back->hw;
+ u32 intval;
+
+ /* These will do nothing if dynamic updates are not enabled */
+ iavf_update_itr(q_vector, &q_vector->tx);
+ iavf_update_itr(q_vector, &q_vector->rx);
+
+ /* This block of logic allows us to get away with only updating
+ * one ITR value with each interrupt. The idea is to perform a
+ * pseudo-lazy update with the following criteria.
+ *
+ * 1. Rx is given higher priority than Tx if both are in same state
+ * 2. If we must reduce an ITR that is given highest priority.
+ * 3. We then give priority to increasing ITR based on amount.
+ */
+ if (q_vector->rx.target_itr < q_vector->rx.current_itr) {
+ /* Rx ITR needs to be reduced, this is highest priority */
+ intval = iavf_buildreg_itr(IAVF_RX_ITR,
+ q_vector->rx.target_itr);
+ q_vector->rx.current_itr = q_vector->rx.target_itr;
+ q_vector->itr_countdown = ITR_COUNTDOWN_START;
+ } else if ((q_vector->tx.target_itr < q_vector->tx.current_itr) ||
+ ((q_vector->rx.target_itr - q_vector->rx.current_itr) <
+ (q_vector->tx.target_itr - q_vector->tx.current_itr))) {
+ /* Tx ITR needs to be reduced, this is second priority
+ * Tx ITR needs to be increased more than Rx, fourth priority
+ */
+ intval = iavf_buildreg_itr(IAVF_TX_ITR,
+ q_vector->tx.target_itr);
+ q_vector->tx.current_itr = q_vector->tx.target_itr;
+ q_vector->itr_countdown = ITR_COUNTDOWN_START;
+ } else if (q_vector->rx.current_itr != q_vector->rx.target_itr) {
+ /* Rx ITR needs to be increased, third priority */
+ intval = iavf_buildreg_itr(IAVF_RX_ITR,
+ q_vector->rx.target_itr);
+ q_vector->rx.current_itr = q_vector->rx.target_itr;
+ q_vector->itr_countdown = ITR_COUNTDOWN_START;
+ } else {
+ /* No ITR update, lowest priority */
+ intval = iavf_buildreg_itr(IAVF_ITR_NONE, 0);
+ if (q_vector->itr_countdown)
+ q_vector->itr_countdown--;
+ }
+
+ if (!test_bit(__IAVF_VSI_DOWN, vsi->state))
+ wr32(hw, INTREG(q_vector->reg_idx), intval);
+}
+
+/**
+ * iavf_napi_poll - NAPI polling Rx/Tx cleanup routine
+ * @napi: napi struct with our devices info in it
+ * @budget: amount of work driver is allowed to do this pass, in packets
+ *
+ * This function will clean all queues associated with a q_vector.
+ *
+ * Returns the amount of work done
+ **/
+int iavf_napi_poll(struct napi_struct *napi, int budget)
+{
+ struct iavf_q_vector *q_vector =
+ container_of(napi, struct iavf_q_vector, napi);
+ struct iavf_vsi *vsi = q_vector->vsi;
+ struct iavf_ring *ring;
+ bool clean_complete = true;
+ bool arm_wb = false;
+ int budget_per_ring;
+ int work_done = 0;
+
+ if (test_bit(__IAVF_VSI_DOWN, vsi->state)) {
+ napi_complete(napi);
+ return 0;
+ }
+
+ /* Since the actual Tx work is minimal, we can give the Tx a larger
+ * budget and be more aggressive about cleaning up the Tx descriptors.
+ */
+ iavf_for_each_ring(ring, q_vector->tx) {
+ if (!iavf_clean_tx_irq(vsi, ring, budget)) {
+ clean_complete = false;
+ continue;
+ }
+ arm_wb |= ring->arm_wb;
+ ring->arm_wb = false;
+ }
+
+ /* Handle case where we are called by netpoll with a budget of 0 */
+ if (budget <= 0)
+ goto tx_only;
+
+ /* We attempt to distribute budget to each Rx queue fairly, but don't
+ * allow the budget to go below 1 because that would exit polling early.
+ */
+ budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
+
+ iavf_for_each_ring(ring, q_vector->rx) {
+ int cleaned = iavf_clean_rx_irq(ring, budget_per_ring);
+
+ work_done += cleaned;
+ /* if we clean as many as budgeted, we must not be done */
+ if (cleaned >= budget_per_ring)
+ clean_complete = false;
+ }
+
+ /* If work not completed, return budget and polling will return */
+ if (!clean_complete) {
+ int cpu_id = smp_processor_id();
+
+ /* It is possible that the interrupt affinity has changed but,
+ * if the cpu is pegged at 100%, polling will never exit while
+ * traffic continues and the interrupt will be stuck on this
+ * cpu. We check to make sure affinity is correct before we
+ * continue to poll, otherwise we must stop polling so the
+ * interrupt can move to the correct cpu.
+ */
+ if (!cpumask_test_cpu(cpu_id, &q_vector->affinity_mask)) {
+ /* Tell napi that we are done polling */
+ napi_complete_done(napi, work_done);
+
+ /* Force an interrupt */
+ iavf_force_wb(vsi, q_vector);
+
+ /* Return budget-1 so that polling stops */
+ return budget - 1;
+ }
+tx_only:
+ if (arm_wb) {
+ q_vector->tx.ring[0].tx_stats.tx_force_wb++;
+ iavf_enable_wb_on_itr(vsi, q_vector);
+ }
+ return budget;
+ }
+
+ if (vsi->back->flags & IAVF_TXR_FLAGS_WB_ON_ITR)
+ q_vector->arm_wb_state = false;
+
+ /* Exit the polling mode, but don't re-enable interrupts if stack might
+ * poll us due to busy-polling
+ */
+ if (likely(napi_complete_done(napi, work_done)))
+ iavf_update_enable_itr(vsi, q_vector);
+
+ return min_t(int, work_done, budget - 1);
+}
+
+/**
+ * iavf_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
+ * @skb: send buffer
+ * @tx_ring: ring to send buffer on
+ * @flags: the tx flags to be set
+ *
+ * Checks the skb and set up correspondingly several generic transmit flags
+ * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
+ *
+ * Returns error code indicate the frame should be dropped upon error and the
+ * otherwise returns 0 to indicate the flags has been set properly.
+ **/
+static void iavf_tx_prepare_vlan_flags(struct sk_buff *skb,
+ struct iavf_ring *tx_ring, u32 *flags)
+{
+ u32 tx_flags = 0;
+
+
+ /* stack will only request hardware VLAN insertion offload for protocols
+ * that the driver supports and has enabled
+ */
+ if (!skb_vlan_tag_present(skb))
+ return;
+
+ tx_flags |= skb_vlan_tag_get(skb) << IAVF_TX_FLAGS_VLAN_SHIFT;
+ if (tx_ring->flags & IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2) {
+ tx_flags |= IAVF_TX_FLAGS_HW_OUTER_SINGLE_VLAN;
+ } else if (tx_ring->flags & IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1) {
+ tx_flags |= IAVF_TX_FLAGS_HW_VLAN;
+ } else {
+ dev_dbg(tx_ring->dev, "Unsupported Tx VLAN tag location requested\n");
+ return;
+ }
+
+ *flags = tx_flags;
+}
+
+/**
+ * iavf_tso - set up the tso context descriptor
+ * @first: pointer to first Tx buffer for xmit
+ * @hdr_len: ptr to the size of the packet header
+ * @cd_type_cmd_tso_mss: Quad Word 1
+ *
+ * Returns 0 if no TSO can happen, 1 if tso is going, or error
+ **/
+static int iavf_tso(struct iavf_tx_buffer *first, u8 *hdr_len,
+ u64 *cd_type_cmd_tso_mss)
+{
+ struct sk_buff *skb = first->skb;
+ u64 cd_cmd, cd_tso_len, cd_mss;
+ union {
+ struct iphdr *v4;
+ struct ipv6hdr *v6;
+ unsigned char *hdr;
+ } ip;
+ union {
+ struct tcphdr *tcp;
+ struct udphdr *udp;
+ unsigned char *hdr;
+ } l4;
+ u32 paylen, l4_offset;
+ u16 gso_segs, gso_size;
+ int err;
+
+ if (skb->ip_summed != CHECKSUM_PARTIAL)
+ return 0;
+
+ if (!skb_is_gso(skb))
+ return 0;
+
+ err = skb_cow_head(skb, 0);
+ if (err < 0)
+ return err;
+
+ ip.hdr = skb_network_header(skb);
+ l4.hdr = skb_transport_header(skb);
+
+ /* initialize outer IP header fields */
+ if (ip.v4->version == 4) {
+ ip.v4->tot_len = 0;
+ ip.v4->check = 0;
+ } else {
+ ip.v6->payload_len = 0;
+ }
+
+ if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
+ SKB_GSO_GRE_CSUM |
+ SKB_GSO_IPXIP4 |
+ SKB_GSO_IPXIP6 |
+ SKB_GSO_UDP_TUNNEL |
+ SKB_GSO_UDP_TUNNEL_CSUM)) {
+ if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
+ (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) {
+ l4.udp->len = 0;
+
+ /* determine offset of outer transport header */
+ l4_offset = l4.hdr - skb->data;
+
+ /* remove payload length from outer checksum */
+ paylen = skb->len - l4_offset;
+ csum_replace_by_diff(&l4.udp->check,
+ (__force __wsum)htonl(paylen));
+ }
+
+ /* reset pointers to inner headers */
+ ip.hdr = skb_inner_network_header(skb);
+ l4.hdr = skb_inner_transport_header(skb);
+
+ /* initialize inner IP header fields */
+ if (ip.v4->version == 4) {
+ ip.v4->tot_len = 0;
+ ip.v4->check = 0;
+ } else {
+ ip.v6->payload_len = 0;
+ }
+ }
+
+ /* determine offset of inner transport header */
+ l4_offset = l4.hdr - skb->data;
+ /* remove payload length from inner checksum */
+ paylen = skb->len - l4_offset;
+
+ if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
+ csum_replace_by_diff(&l4.udp->check,
+ (__force __wsum)htonl(paylen));
+ /* compute length of UDP segmentation header */
+ *hdr_len = (u8)sizeof(l4.udp) + l4_offset;
+ } else {
+ csum_replace_by_diff(&l4.tcp->check,
+ (__force __wsum)htonl(paylen));
+ /* compute length of TCP segmentation header */
+ *hdr_len = (u8)((l4.tcp->doff * 4) + l4_offset);
+ }
+
+ /* pull values out of skb_shinfo */
+ gso_size = skb_shinfo(skb)->gso_size;
+ gso_segs = skb_shinfo(skb)->gso_segs;
+
+ /* update GSO size and bytecount with header size */
+ first->gso_segs = gso_segs;
+ first->bytecount += (first->gso_segs - 1) * *hdr_len;
+
+ /* find the field values */
+ cd_cmd = IAVF_TX_CTX_DESC_TSO;
+ cd_tso_len = skb->len - *hdr_len;
+ cd_mss = gso_size;
+ *cd_type_cmd_tso_mss |= (cd_cmd << IAVF_TXD_CTX_QW1_CMD_SHIFT) |
+ (cd_tso_len << IAVF_TXD_CTX_QW1_TSO_LEN_SHIFT) |
+ (cd_mss << IAVF_TXD_CTX_QW1_MSS_SHIFT);
+ return 1;
+}
+
+/**
+ * iavf_tx_enable_csum - Enable Tx checksum offloads
+ * @skb: send buffer
+ * @tx_flags: pointer to Tx flags currently set
+ * @td_cmd: Tx descriptor command bits to set
+ * @td_offset: Tx descriptor header offsets to set
+ * @tx_ring: Tx descriptor ring
+ * @cd_tunneling: ptr to context desc bits
+ **/
+static int iavf_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags,
+ u32 *td_cmd, u32 *td_offset,
+ struct iavf_ring *tx_ring,
+ u32 *cd_tunneling)
+{
+ union {
+ struct iphdr *v4;
+ struct ipv6hdr *v6;
+ unsigned char *hdr;
+ } ip;
+ union {
+ struct tcphdr *tcp;
+ struct udphdr *udp;
+ unsigned char *hdr;
+ } l4;
+ unsigned char *exthdr;
+ u32 offset, cmd = 0;
+ __be16 frag_off;
+ u8 l4_proto = 0;
+
+ if (skb->ip_summed != CHECKSUM_PARTIAL)
+ return 0;
+
+ ip.hdr = skb_network_header(skb);
+ l4.hdr = skb_transport_header(skb);
+
+ /* compute outer L2 header size */
+ offset = ((ip.hdr - skb->data) / 2) << IAVF_TX_DESC_LENGTH_MACLEN_SHIFT;
+
+ if (skb->encapsulation) {
+ u32 tunnel = 0;
+ /* define outer network header type */
+ if (*tx_flags & IAVF_TX_FLAGS_IPV4) {
+ tunnel |= (*tx_flags & IAVF_TX_FLAGS_TSO) ?
+ IAVF_TX_CTX_EXT_IP_IPV4 :
+ IAVF_TX_CTX_EXT_IP_IPV4_NO_CSUM;
+
+ l4_proto = ip.v4->protocol;
+ } else if (*tx_flags & IAVF_TX_FLAGS_IPV6) {
+ tunnel |= IAVF_TX_CTX_EXT_IP_IPV6;
+
+ exthdr = ip.hdr + sizeof(*ip.v6);
+ l4_proto = ip.v6->nexthdr;
+ if (l4.hdr != exthdr)
+ ipv6_skip_exthdr(skb, exthdr - skb->data,
+ &l4_proto, &frag_off);
+ }
+
+ /* define outer transport */
+ switch (l4_proto) {
+ case IPPROTO_UDP:
+ tunnel |= IAVF_TXD_CTX_UDP_TUNNELING;
+ *tx_flags |= IAVF_TX_FLAGS_VXLAN_TUNNEL;
+ break;
+ case IPPROTO_GRE:
+ tunnel |= IAVF_TXD_CTX_GRE_TUNNELING;
+ *tx_flags |= IAVF_TX_FLAGS_VXLAN_TUNNEL;
+ break;
+ case IPPROTO_IPIP:
+ case IPPROTO_IPV6:
+ *tx_flags |= IAVF_TX_FLAGS_VXLAN_TUNNEL;
+ l4.hdr = skb_inner_network_header(skb);
+ break;
+ default:
+ if (*tx_flags & IAVF_TX_FLAGS_TSO)
+ return -1;
+
+ skb_checksum_help(skb);
+ return 0;
+ }
+
+ /* compute outer L3 header size */
+ tunnel |= ((l4.hdr - ip.hdr) / 4) <<
+ IAVF_TXD_CTX_QW0_EXT_IPLEN_SHIFT;
+
+ /* switch IP header pointer from outer to inner header */
+ ip.hdr = skb_inner_network_header(skb);
+
+ /* compute tunnel header size */
+ tunnel |= ((ip.hdr - l4.hdr) / 2) <<
+ IAVF_TXD_CTX_QW0_NATLEN_SHIFT;
+
+ /* indicate if we need to offload outer UDP header */
+ if ((*tx_flags & IAVF_TX_FLAGS_TSO) &&
+ !(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
+ (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM))
+ tunnel |= IAVF_TXD_CTX_QW0_L4T_CS_MASK;
+
+ /* record tunnel offload values */
+ *cd_tunneling |= tunnel;
+
+ /* switch L4 header pointer from outer to inner */
+ l4.hdr = skb_inner_transport_header(skb);
+ l4_proto = 0;
+
+ /* reset type as we transition from outer to inner headers */
+ *tx_flags &= ~(IAVF_TX_FLAGS_IPV4 | IAVF_TX_FLAGS_IPV6);
+ if (ip.v4->version == 4)
+ *tx_flags |= IAVF_TX_FLAGS_IPV4;
+ if (ip.v6->version == 6)
+ *tx_flags |= IAVF_TX_FLAGS_IPV6;
+ }
+
+ /* Enable IP checksum offloads */
+ if (*tx_flags & IAVF_TX_FLAGS_IPV4) {
+ l4_proto = ip.v4->protocol;
+ /* the stack computes the IP header already, the only time we
+ * need the hardware to recompute it is in the case of TSO.
+ */
+ cmd |= (*tx_flags & IAVF_TX_FLAGS_TSO) ?
+ IAVF_TX_DESC_CMD_IIPT_IPV4_CSUM :
+ IAVF_TX_DESC_CMD_IIPT_IPV4;
+ } else if (*tx_flags & IAVF_TX_FLAGS_IPV6) {
+ cmd |= IAVF_TX_DESC_CMD_IIPT_IPV6;
+
+ exthdr = ip.hdr + sizeof(*ip.v6);
+ l4_proto = ip.v6->nexthdr;
+ if (l4.hdr != exthdr)
+ ipv6_skip_exthdr(skb, exthdr - skb->data,
+ &l4_proto, &frag_off);
+ }
+
+ /* compute inner L3 header size */
+ offset |= ((l4.hdr - ip.hdr) / 4) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT;
+
+ /* Enable L4 checksum offloads */
+ switch (l4_proto) {
+ case IPPROTO_TCP:
+ /* enable checksum offloads */
+ cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP;
+ offset |= l4.tcp->doff << IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
+ break;
+ case IPPROTO_SCTP:
+ /* enable SCTP checksum offload */
+ cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_SCTP;
+ offset |= (sizeof(struct sctphdr) >> 2) <<
+ IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
+ break;
+ case IPPROTO_UDP:
+ /* enable UDP checksum offload */
+ cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_UDP;
+ offset |= (sizeof(struct udphdr) >> 2) <<
+ IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
+ break;
+ default:
+ if (*tx_flags & IAVF_TX_FLAGS_TSO)
+ return -1;
+ skb_checksum_help(skb);
+ return 0;
+ }
+
+ *td_cmd |= cmd;
+ *td_offset |= offset;
+
+ return 1;
+}
+
+/**
+ * iavf_create_tx_ctx - Build the Tx context descriptor
+ * @tx_ring: ring to create the descriptor on
+ * @cd_type_cmd_tso_mss: Quad Word 1
+ * @cd_tunneling: Quad Word 0 - bits 0-31
+ * @cd_l2tag2: Quad Word 0 - bits 32-63
+ **/
+static void iavf_create_tx_ctx(struct iavf_ring *tx_ring,
+ const u64 cd_type_cmd_tso_mss,
+ const u32 cd_tunneling, const u32 cd_l2tag2)
+{
+ struct iavf_tx_context_desc *context_desc;
+ int i = tx_ring->next_to_use;
+
+ if ((cd_type_cmd_tso_mss == IAVF_TX_DESC_DTYPE_CONTEXT) &&
+ !cd_tunneling && !cd_l2tag2)
+ return;
+
+ /* grab the next descriptor */
+ context_desc = IAVF_TX_CTXTDESC(tx_ring, i);
+
+ i++;
+ tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
+
+ /* cpu_to_le32 and assign to struct fields */
+ context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
+ context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
+ context_desc->rsvd = cpu_to_le16(0);
+ context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
+}
+
+/**
+ * __iavf_chk_linearize - Check if there are more than 8 buffers per packet
+ * @skb: send buffer
+ *
+ * Note: Our HW can't DMA more than 8 buffers to build a packet on the wire
+ * and so we need to figure out the cases where we need to linearize the skb.
+ *
+ * For TSO we need to count the TSO header and segment payload separately.
+ * As such we need to check cases where we have 7 fragments or more as we
+ * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for
+ * the segment payload in the first descriptor, and another 7 for the
+ * fragments.
+ **/
+bool __iavf_chk_linearize(struct sk_buff *skb)
+{
+ const skb_frag_t *frag, *stale;
+ int nr_frags, sum;
+
+ /* no need to check if number of frags is less than 7 */
+ nr_frags = skb_shinfo(skb)->nr_frags;
+ if (nr_frags < (IAVF_MAX_BUFFER_TXD - 1))
+ return false;
+
+ /* We need to walk through the list and validate that each group
+ * of 6 fragments totals at least gso_size.
+ */
+ nr_frags -= IAVF_MAX_BUFFER_TXD - 2;
+ frag = &skb_shinfo(skb)->frags[0];
+
+ /* Initialize size to the negative value of gso_size minus 1. We
+ * use this as the worst case scenerio in which the frag ahead
+ * of us only provides one byte which is why we are limited to 6
+ * descriptors for a single transmit as the header and previous
+ * fragment are already consuming 2 descriptors.
+ */
+ sum = 1 - skb_shinfo(skb)->gso_size;
+
+ /* Add size of frags 0 through 4 to create our initial sum */
+ sum += skb_frag_size(frag++);
+ sum += skb_frag_size(frag++);
+ sum += skb_frag_size(frag++);
+ sum += skb_frag_size(frag++);
+ sum += skb_frag_size(frag++);
+
+ /* Walk through fragments adding latest fragment, testing it, and
+ * then removing stale fragments from the sum.
+ */
+ for (stale = &skb_shinfo(skb)->frags[0];; stale++) {
+ int stale_size = skb_frag_size(stale);
+
+ sum += skb_frag_size(frag++);
+
+ /* The stale fragment may present us with a smaller
+ * descriptor than the actual fragment size. To account
+ * for that we need to remove all the data on the front and
+ * figure out what the remainder would be in the last
+ * descriptor associated with the fragment.
+ */
+ if (stale_size > IAVF_MAX_DATA_PER_TXD) {
+ int align_pad = -(skb_frag_off(stale)) &
+ (IAVF_MAX_READ_REQ_SIZE - 1);
+
+ sum -= align_pad;
+ stale_size -= align_pad;
+
+ do {
+ sum -= IAVF_MAX_DATA_PER_TXD_ALIGNED;
+ stale_size -= IAVF_MAX_DATA_PER_TXD_ALIGNED;
+ } while (stale_size > IAVF_MAX_DATA_PER_TXD);
+ }
+
+ /* if sum is negative we failed to make sufficient progress */
+ if (sum < 0)
+ return true;
+
+ if (!nr_frags--)
+ break;
+
+ sum -= stale_size;
+ }
+
+ return false;
+}
+
+/**
+ * __iavf_maybe_stop_tx - 2nd level check for tx stop conditions
+ * @tx_ring: the ring to be checked
+ * @size: the size buffer we want to assure is available
+ *
+ * Returns -EBUSY if a stop is needed, else 0
+ **/
+int __iavf_maybe_stop_tx(struct iavf_ring *tx_ring, int size)
+{
+ netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
+ /* Memory barrier before checking head and tail */
+ smp_mb();
+
+ /* Check again in a case another CPU has just made room available. */
+ if (likely(IAVF_DESC_UNUSED(tx_ring) < size))
+ return -EBUSY;
+
+ /* A reprieve! - use start_queue because it doesn't call schedule */
+ netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
+ ++tx_ring->tx_stats.restart_queue;
+ return 0;
+}
+
+/**
+ * iavf_tx_map - Build the Tx descriptor
+ * @tx_ring: ring to send buffer on
+ * @skb: send buffer
+ * @first: first buffer info buffer to use
+ * @tx_flags: collected send information
+ * @hdr_len: size of the packet header
+ * @td_cmd: the command field in the descriptor
+ * @td_offset: offset for checksum or crc
+ **/
+static inline void iavf_tx_map(struct iavf_ring *tx_ring, struct sk_buff *skb,
+ struct iavf_tx_buffer *first, u32 tx_flags,
+ const u8 hdr_len, u32 td_cmd, u32 td_offset)
+{
+ unsigned int data_len = skb->data_len;
+ unsigned int size = skb_headlen(skb);
+ skb_frag_t *frag;
+ struct iavf_tx_buffer *tx_bi;
+ struct iavf_tx_desc *tx_desc;
+ u16 i = tx_ring->next_to_use;
+ u32 td_tag = 0;
+ dma_addr_t dma;
+
+ if (tx_flags & IAVF_TX_FLAGS_HW_VLAN) {
+ td_cmd |= IAVF_TX_DESC_CMD_IL2TAG1;
+ td_tag = (tx_flags & IAVF_TX_FLAGS_VLAN_MASK) >>
+ IAVF_TX_FLAGS_VLAN_SHIFT;
+ }
+
+ first->tx_flags = tx_flags;
+
+ dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
+
+ tx_desc = IAVF_TX_DESC(tx_ring, i);
+ tx_bi = first;
+
+ for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
+ unsigned int max_data = IAVF_MAX_DATA_PER_TXD_ALIGNED;
+
+ if (dma_mapping_error(tx_ring->dev, dma))
+ goto dma_error;
+
+ /* record length, and DMA address */
+ dma_unmap_len_set(tx_bi, len, size);
+ dma_unmap_addr_set(tx_bi, dma, dma);
+
+ /* align size to end of page */
+ max_data += -dma & (IAVF_MAX_READ_REQ_SIZE - 1);
+ tx_desc->buffer_addr = cpu_to_le64(dma);
+
+ while (unlikely(size > IAVF_MAX_DATA_PER_TXD)) {
+ tx_desc->cmd_type_offset_bsz =
+ build_ctob(td_cmd, td_offset,
+ max_data, td_tag);
+
+ tx_desc++;
+ i++;
+
+ if (i == tx_ring->count) {
+ tx_desc = IAVF_TX_DESC(tx_ring, 0);
+ i = 0;
+ }
+
+ dma += max_data;
+ size -= max_data;
+
+ max_data = IAVF_MAX_DATA_PER_TXD_ALIGNED;
+ tx_desc->buffer_addr = cpu_to_le64(dma);
+ }
+
+ if (likely(!data_len))
+ break;
+
+ tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
+ size, td_tag);
+
+ tx_desc++;
+ i++;
+
+ if (i == tx_ring->count) {
+ tx_desc = IAVF_TX_DESC(tx_ring, 0);
+ i = 0;
+ }
+
+ size = skb_frag_size(frag);
+ data_len -= size;
+
+ dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
+ DMA_TO_DEVICE);
+
+ tx_bi = &tx_ring->tx_bi[i];
+ }
+
+ netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
+
+ i++;
+ if (i == tx_ring->count)
+ i = 0;
+
+ tx_ring->next_to_use = i;
+
+ iavf_maybe_stop_tx(tx_ring, DESC_NEEDED);
+
+ /* write last descriptor with RS and EOP bits */
+ td_cmd |= IAVF_TXD_CMD;
+ tx_desc->cmd_type_offset_bsz =
+ build_ctob(td_cmd, td_offset, size, td_tag);
+
+ skb_tx_timestamp(skb);
+
+ /* Force memory writes to complete before letting h/w know there
+ * are new descriptors to fetch.
+ *
+ * We also use this memory barrier to make certain all of the
+ * status bits have been updated before next_to_watch is written.
+ */
+ wmb();
+
+ /* set next_to_watch value indicating a packet is present */
+ first->next_to_watch = tx_desc;
+
+ /* notify HW of packet */
+ if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
+ writel(i, tx_ring->tail);
+ }
+
+ return;
+
+dma_error:
+ dev_info(tx_ring->dev, "TX DMA map failed\n");
+
+ /* clear dma mappings for failed tx_bi map */
+ for (;;) {
+ tx_bi = &tx_ring->tx_bi[i];
+ iavf_unmap_and_free_tx_resource(tx_ring, tx_bi);
+ if (tx_bi == first)
+ break;
+ if (i == 0)
+ i = tx_ring->count;
+ i--;
+ }
+
+ tx_ring->next_to_use = i;
+}
+
+/**
+ * iavf_xmit_frame_ring - Sends buffer on Tx ring
+ * @skb: send buffer
+ * @tx_ring: ring to send buffer on
+ *
+ * Returns NETDEV_TX_OK if sent, else an error code
+ **/
+static netdev_tx_t iavf_xmit_frame_ring(struct sk_buff *skb,
+ struct iavf_ring *tx_ring)
+{
+ u64 cd_type_cmd_tso_mss = IAVF_TX_DESC_DTYPE_CONTEXT;
+ u32 cd_tunneling = 0, cd_l2tag2 = 0;
+ struct iavf_tx_buffer *first;
+ u32 td_offset = 0;
+ u32 tx_flags = 0;
+ __be16 protocol;
+ u32 td_cmd = 0;
+ u8 hdr_len = 0;
+ int tso, count;
+
+ /* prefetch the data, we'll need it later */
+ prefetch(skb->data);
+
+ iavf_trace(xmit_frame_ring, skb, tx_ring);
+
+ count = iavf_xmit_descriptor_count(skb);
+ if (iavf_chk_linearize(skb, count)) {
+ if (__skb_linearize(skb)) {
+ dev_kfree_skb_any(skb);
+ return NETDEV_TX_OK;
+ }
+ count = iavf_txd_use_count(skb->len);
+ tx_ring->tx_stats.tx_linearize++;
+ }
+
+ /* need: 1 descriptor per page * PAGE_SIZE/IAVF_MAX_DATA_PER_TXD,
+ * + 1 desc for skb_head_len/IAVF_MAX_DATA_PER_TXD,
+ * + 4 desc gap to avoid the cache line where head is,
+ * + 1 desc for context descriptor,
+ * otherwise try next time
+ */
+ if (iavf_maybe_stop_tx(tx_ring, count + 4 + 1)) {
+ tx_ring->tx_stats.tx_busy++;
+ return NETDEV_TX_BUSY;
+ }
+
+ /* record the location of the first descriptor for this packet */
+ first = &tx_ring->tx_bi[tx_ring->next_to_use];
+ first->skb = skb;
+ first->bytecount = skb->len;
+ first->gso_segs = 1;
+
+ /* prepare the xmit flags */
+ iavf_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags);
+ if (tx_flags & IAVF_TX_FLAGS_HW_OUTER_SINGLE_VLAN) {
+ cd_type_cmd_tso_mss |= IAVF_TX_CTX_DESC_IL2TAG2 <<
+ IAVF_TXD_CTX_QW1_CMD_SHIFT;
+ cd_l2tag2 = (tx_flags & IAVF_TX_FLAGS_VLAN_MASK) >>
+ IAVF_TX_FLAGS_VLAN_SHIFT;
+ }
+
+ /* obtain protocol of skb */
+ protocol = vlan_get_protocol(skb);
+
+ /* setup IPv4/IPv6 offloads */
+ if (protocol == htons(ETH_P_IP))
+ tx_flags |= IAVF_TX_FLAGS_IPV4;
+ else if (protocol == htons(ETH_P_IPV6))
+ tx_flags |= IAVF_TX_FLAGS_IPV6;
+
+ tso = iavf_tso(first, &hdr_len, &cd_type_cmd_tso_mss);
+
+ if (tso < 0)
+ goto out_drop;
+ else if (tso)
+ tx_flags |= IAVF_TX_FLAGS_TSO;
+
+ /* Always offload the checksum, since it's in the data descriptor */
+ tso = iavf_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset,
+ tx_ring, &cd_tunneling);
+ if (tso < 0)
+ goto out_drop;
+
+ /* always enable CRC insertion offload */
+ td_cmd |= IAVF_TX_DESC_CMD_ICRC;
+
+ iavf_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
+ cd_tunneling, cd_l2tag2);
+
+ iavf_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
+ td_cmd, td_offset);
+
+ return NETDEV_TX_OK;
+
+out_drop:
+ iavf_trace(xmit_frame_ring_drop, first->skb, tx_ring);
+ dev_kfree_skb_any(first->skb);
+ first->skb = NULL;
+ return NETDEV_TX_OK;
+}
+
+/**
+ * iavf_xmit_frame - Selects the correct VSI and Tx queue to send buffer
+ * @skb: send buffer
+ * @netdev: network interface device structure
+ *
+ * Returns NETDEV_TX_OK if sent, else an error code
+ **/
+netdev_tx_t iavf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
+{
+ struct iavf_adapter *adapter = netdev_priv(netdev);
+ struct iavf_ring *tx_ring = &adapter->tx_rings[skb->queue_mapping];
+
+ /* hardware can't handle really short frames, hardware padding works
+ * beyond this point
+ */
+ if (unlikely(skb->len < IAVF_MIN_TX_LEN)) {
+ if (skb_pad(skb, IAVF_MIN_TX_LEN - skb->len))
+ return NETDEV_TX_OK;
+ skb->len = IAVF_MIN_TX_LEN;
+ skb_set_tail_pointer(skb, IAVF_MIN_TX_LEN);
+ }
+
+ return iavf_xmit_frame_ring(skb, tx_ring);
+}
diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.h b/drivers/net/ethernet/intel/iavf/iavf_txrx.h
new file mode 100644
index 0000000000..10ba36602c
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.h
@@ -0,0 +1,522 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_TXRX_H_
+#define _IAVF_TXRX_H_
+
+/* Interrupt Throttling and Rate Limiting Goodies */
+#define IAVF_DEFAULT_IRQ_WORK 256
+
+/* The datasheet for the X710 and XL710 indicate that the maximum value for
+ * the ITR is 8160usec which is then called out as 0xFF0 with a 2usec
+ * resolution. 8160 is 0x1FE0 when written out in hex. So instead of storing
+ * the register value which is divided by 2 lets use the actual values and
+ * avoid an excessive amount of translation.
+ */
+#define IAVF_ITR_DYNAMIC 0x8000 /* use top bit as a flag */
+#define IAVF_ITR_MASK 0x1FFE /* mask for ITR register value */
+#define IAVF_ITR_100K 10 /* all values below must be even */
+#define IAVF_ITR_50K 20
+#define IAVF_ITR_20K 50
+#define IAVF_ITR_18K 60
+#define IAVF_ITR_8K 122
+#define IAVF_MAX_ITR 8160 /* maximum value as per datasheet */
+#define ITR_TO_REG(setting) ((setting) & ~IAVF_ITR_DYNAMIC)
+#define ITR_REG_ALIGN(setting) __ALIGN_MASK(setting, ~IAVF_ITR_MASK)
+#define ITR_IS_DYNAMIC(setting) (!!((setting) & IAVF_ITR_DYNAMIC))
+
+#define IAVF_ITR_RX_DEF (IAVF_ITR_20K | IAVF_ITR_DYNAMIC)
+#define IAVF_ITR_TX_DEF (IAVF_ITR_20K | IAVF_ITR_DYNAMIC)
+
+/* 0x40 is the enable bit for interrupt rate limiting, and must be set if
+ * the value of the rate limit is non-zero
+ */
+#define INTRL_ENA BIT(6)
+#define IAVF_MAX_INTRL 0x3B /* reg uses 4 usec resolution */
+#define INTRL_REG_TO_USEC(intrl) ((intrl & ~INTRL_ENA) << 2)
+#define INTRL_USEC_TO_REG(set) ((set) ? ((set) >> 2) | INTRL_ENA : 0)
+#define IAVF_INTRL_8K 125 /* 8000 ints/sec */
+#define IAVF_INTRL_62K 16 /* 62500 ints/sec */
+#define IAVF_INTRL_83K 12 /* 83333 ints/sec */
+
+#define IAVF_QUEUE_END_OF_LIST 0x7FF
+
+/* this enum matches hardware bits and is meant to be used by DYN_CTLN
+ * registers and QINT registers or more generally anywhere in the manual
+ * mentioning ITR_INDX, ITR_NONE cannot be used as an index 'n' into any
+ * register but instead is a special value meaning "don't update" ITR0/1/2.
+ */
+enum iavf_dyn_idx_t {
+ IAVF_IDX_ITR0 = 0,
+ IAVF_IDX_ITR1 = 1,
+ IAVF_IDX_ITR2 = 2,
+ IAVF_ITR_NONE = 3 /* ITR_NONE must not be used as an index */
+};
+
+/* these are indexes into ITRN registers */
+#define IAVF_RX_ITR IAVF_IDX_ITR0
+#define IAVF_TX_ITR IAVF_IDX_ITR1
+#define IAVF_PE_ITR IAVF_IDX_ITR2
+
+/* Supported RSS offloads */
+#define IAVF_DEFAULT_RSS_HENA ( \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV4_UDP) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV4_SCTP) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV4_TCP) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV4_OTHER) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_FRAG_IPV4) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV6_UDP) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV6_TCP) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV6_SCTP) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV6_OTHER) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_FRAG_IPV6) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_L2_PAYLOAD))
+
+#define IAVF_DEFAULT_RSS_HENA_EXPANDED (IAVF_DEFAULT_RSS_HENA | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \
+ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP))
+
+/* Supported Rx Buffer Sizes (a multiple of 128) */
+#define IAVF_RXBUFFER_256 256
+#define IAVF_RXBUFFER_1536 1536 /* 128B aligned standard Ethernet frame */
+#define IAVF_RXBUFFER_2048 2048
+#define IAVF_RXBUFFER_3072 3072 /* Used for large frames w/ padding */
+#define IAVF_MAX_RXBUFFER 9728 /* largest size for single descriptor */
+
+/* NOTE: netdev_alloc_skb reserves up to 64 bytes, NET_IP_ALIGN means we
+ * reserve 2 more, and skb_shared_info adds an additional 384 bytes more,
+ * this adds up to 512 bytes of extra data meaning the smallest allocation
+ * we could have is 1K.
+ * i.e. RXBUFFER_256 --> 960 byte skb (size-1024 slab)
+ * i.e. RXBUFFER_512 --> 1216 byte skb (size-2048 slab)
+ */
+#define IAVF_RX_HDR_SIZE IAVF_RXBUFFER_256
+#define IAVF_PACKET_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2))
+#define iavf_rx_desc iavf_32byte_rx_desc
+
+#define IAVF_RX_DMA_ATTR \
+ (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
+
+/* Attempt to maximize the headroom available for incoming frames. We
+ * use a 2K buffer for receives and need 1536/1534 to store the data for
+ * the frame. This leaves us with 512 bytes of room. From that we need
+ * to deduct the space needed for the shared info and the padding needed
+ * to IP align the frame.
+ *
+ * Note: For cache line sizes 256 or larger this value is going to end
+ * up negative. In these cases we should fall back to the legacy
+ * receive path.
+ */
+#if (PAGE_SIZE < 8192)
+#define IAVF_2K_TOO_SMALL_WITH_PADDING \
+((NET_SKB_PAD + IAVF_RXBUFFER_1536) > SKB_WITH_OVERHEAD(IAVF_RXBUFFER_2048))
+
+static inline int iavf_compute_pad(int rx_buf_len)
+{
+ int page_size, pad_size;
+
+ page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2);
+ pad_size = SKB_WITH_OVERHEAD(page_size) - rx_buf_len;
+
+ return pad_size;
+}
+
+static inline int iavf_skb_pad(void)
+{
+ int rx_buf_len;
+
+ /* If a 2K buffer cannot handle a standard Ethernet frame then
+ * optimize padding for a 3K buffer instead of a 1.5K buffer.
+ *
+ * For a 3K buffer we need to add enough padding to allow for
+ * tailroom due to NET_IP_ALIGN possibly shifting us out of
+ * cache-line alignment.
+ */
+ if (IAVF_2K_TOO_SMALL_WITH_PADDING)
+ rx_buf_len = IAVF_RXBUFFER_3072 + SKB_DATA_ALIGN(NET_IP_ALIGN);
+ else
+ rx_buf_len = IAVF_RXBUFFER_1536;
+
+ /* if needed make room for NET_IP_ALIGN */
+ rx_buf_len -= NET_IP_ALIGN;
+
+ return iavf_compute_pad(rx_buf_len);
+}
+
+#define IAVF_SKB_PAD iavf_skb_pad()
+#else
+#define IAVF_2K_TOO_SMALL_WITH_PADDING false
+#define IAVF_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN)
+#endif
+
+/**
+ * iavf_test_staterr - tests bits in Rx descriptor status and error fields
+ * @rx_desc: pointer to receive descriptor (in le64 format)
+ * @stat_err_bits: value to mask
+ *
+ * This function does some fast chicanery in order to return the
+ * value of the mask which is really only used for boolean tests.
+ * The status_error_len doesn't need to be shifted because it begins
+ * at offset zero.
+ */
+static inline bool iavf_test_staterr(union iavf_rx_desc *rx_desc,
+ const u64 stat_err_bits)
+{
+ return !!(rx_desc->wb.qword1.status_error_len &
+ cpu_to_le64(stat_err_bits));
+}
+
+/* How many Rx Buffers do we bundle into one write to the hardware ? */
+#define IAVF_RX_INCREMENT(r, i) \
+ do { \
+ (i)++; \
+ if ((i) == (r)->count) \
+ i = 0; \
+ r->next_to_clean = i; \
+ } while (0)
+
+#define IAVF_RX_NEXT_DESC(r, i, n) \
+ do { \
+ (i)++; \
+ if ((i) == (r)->count) \
+ i = 0; \
+ (n) = IAVF_RX_DESC((r), (i)); \
+ } while (0)
+
+#define IAVF_RX_NEXT_DESC_PREFETCH(r, i, n) \
+ do { \
+ IAVF_RX_NEXT_DESC((r), (i), (n)); \
+ prefetch((n)); \
+ } while (0)
+
+#define IAVF_MAX_BUFFER_TXD 8
+#define IAVF_MIN_TX_LEN 17
+
+/* The size limit for a transmit buffer in a descriptor is (16K - 1).
+ * In order to align with the read requests we will align the value to
+ * the nearest 4K which represents our maximum read request size.
+ */
+#define IAVF_MAX_READ_REQ_SIZE 4096
+#define IAVF_MAX_DATA_PER_TXD (16 * 1024 - 1)
+#define IAVF_MAX_DATA_PER_TXD_ALIGNED \
+ (IAVF_MAX_DATA_PER_TXD & ~(IAVF_MAX_READ_REQ_SIZE - 1))
+
+/**
+ * iavf_txd_use_count - estimate the number of descriptors needed for Tx
+ * @size: transmit request size in bytes
+ *
+ * Due to hardware alignment restrictions (4K alignment), we need to
+ * assume that we can have no more than 12K of data per descriptor, even
+ * though each descriptor can take up to 16K - 1 bytes of aligned memory.
+ * Thus, we need to divide by 12K. But division is slow! Instead,
+ * we decompose the operation into shifts and one relatively cheap
+ * multiply operation.
+ *
+ * To divide by 12K, we first divide by 4K, then divide by 3:
+ * To divide by 4K, shift right by 12 bits
+ * To divide by 3, multiply by 85, then divide by 256
+ * (Divide by 256 is done by shifting right by 8 bits)
+ * Finally, we add one to round up. Because 256 isn't an exact multiple of
+ * 3, we'll underestimate near each multiple of 12K. This is actually more
+ * accurate as we have 4K - 1 of wiggle room that we can fit into the last
+ * segment. For our purposes this is accurate out to 1M which is orders of
+ * magnitude greater than our largest possible GSO size.
+ *
+ * This would then be implemented as:
+ * return (((size >> 12) * 85) >> 8) + 1;
+ *
+ * Since multiplication and division are commutative, we can reorder
+ * operations into:
+ * return ((size * 85) >> 20) + 1;
+ */
+static inline unsigned int iavf_txd_use_count(unsigned int size)
+{
+ return ((size * 85) >> 20) + 1;
+}
+
+/* Tx Descriptors needed, worst case */
+#define DESC_NEEDED (MAX_SKB_FRAGS + 6)
+#define IAVF_MIN_DESC_PENDING 4
+
+#define IAVF_TX_FLAGS_HW_VLAN BIT(1)
+#define IAVF_TX_FLAGS_SW_VLAN BIT(2)
+#define IAVF_TX_FLAGS_TSO BIT(3)
+#define IAVF_TX_FLAGS_IPV4 BIT(4)
+#define IAVF_TX_FLAGS_IPV6 BIT(5)
+#define IAVF_TX_FLAGS_FCCRC BIT(6)
+#define IAVF_TX_FLAGS_FSO BIT(7)
+#define IAVF_TX_FLAGS_FD_SB BIT(9)
+#define IAVF_TX_FLAGS_VXLAN_TUNNEL BIT(10)
+#define IAVF_TX_FLAGS_HW_OUTER_SINGLE_VLAN BIT(11)
+#define IAVF_TX_FLAGS_VLAN_MASK 0xffff0000
+#define IAVF_TX_FLAGS_VLAN_PRIO_MASK 0xe0000000
+#define IAVF_TX_FLAGS_VLAN_PRIO_SHIFT 29
+#define IAVF_TX_FLAGS_VLAN_SHIFT 16
+
+struct iavf_tx_buffer {
+ struct iavf_tx_desc *next_to_watch;
+ union {
+ struct sk_buff *skb;
+ void *raw_buf;
+ };
+ unsigned int bytecount;
+ unsigned short gso_segs;
+
+ DEFINE_DMA_UNMAP_ADDR(dma);
+ DEFINE_DMA_UNMAP_LEN(len);
+ u32 tx_flags;
+};
+
+struct iavf_rx_buffer {
+ dma_addr_t dma;
+ struct page *page;
+#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
+ __u32 page_offset;
+#else
+ __u16 page_offset;
+#endif
+ __u16 pagecnt_bias;
+};
+
+struct iavf_queue_stats {
+ u64 packets;
+ u64 bytes;
+};
+
+struct iavf_tx_queue_stats {
+ u64 restart_queue;
+ u64 tx_busy;
+ u64 tx_done_old;
+ u64 tx_linearize;
+ u64 tx_force_wb;
+ int prev_pkt_ctr;
+ u64 tx_lost_interrupt;
+};
+
+struct iavf_rx_queue_stats {
+ u64 non_eop_descs;
+ u64 alloc_page_failed;
+ u64 alloc_buff_failed;
+ u64 page_reuse_count;
+ u64 realloc_count;
+};
+
+enum iavf_ring_state_t {
+ __IAVF_TX_FDIR_INIT_DONE,
+ __IAVF_TX_XPS_INIT_DONE,
+ __IAVF_RING_STATE_NBITS /* must be last */
+};
+
+/* some useful defines for virtchannel interface, which
+ * is the only remaining user of header split
+ */
+#define IAVF_RX_DTYPE_NO_SPLIT 0
+#define IAVF_RX_DTYPE_HEADER_SPLIT 1
+#define IAVF_RX_DTYPE_SPLIT_ALWAYS 2
+#define IAVF_RX_SPLIT_L2 0x1
+#define IAVF_RX_SPLIT_IP 0x2
+#define IAVF_RX_SPLIT_TCP_UDP 0x4
+#define IAVF_RX_SPLIT_SCTP 0x8
+
+/* struct that defines a descriptor ring, associated with a VSI */
+struct iavf_ring {
+ struct iavf_ring *next; /* pointer to next ring in q_vector */
+ void *desc; /* Descriptor ring memory */
+ struct device *dev; /* Used for DMA mapping */
+ struct net_device *netdev; /* netdev ring maps to */
+ union {
+ struct iavf_tx_buffer *tx_bi;
+ struct iavf_rx_buffer *rx_bi;
+ };
+ DECLARE_BITMAP(state, __IAVF_RING_STATE_NBITS);
+ u16 queue_index; /* Queue number of ring */
+ u8 dcb_tc; /* Traffic class of ring */
+ u8 __iomem *tail;
+
+ /* high bit set means dynamic, use accessors routines to read/write.
+ * hardware only supports 2us resolution for the ITR registers.
+ * these values always store the USER setting, and must be converted
+ * before programming to a register.
+ */
+ u16 itr_setting;
+
+ u16 count; /* Number of descriptors */
+ u16 reg_idx; /* HW register index of the ring */
+ u16 rx_buf_len;
+
+ /* used in interrupt processing */
+ u16 next_to_use;
+ u16 next_to_clean;
+
+ u8 atr_sample_rate;
+ u8 atr_count;
+
+ bool ring_active; /* is ring online or not */
+ bool arm_wb; /* do something to arm write back */
+ u8 packet_stride;
+
+ u16 flags;
+#define IAVF_TXR_FLAGS_WB_ON_ITR BIT(0)
+#define IAVF_RXR_FLAGS_BUILD_SKB_ENABLED BIT(1)
+#define IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 BIT(3)
+#define IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2 BIT(4)
+#define IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2 BIT(5)
+
+ /* stats structs */
+ struct iavf_queue_stats stats;
+ struct u64_stats_sync syncp;
+ union {
+ struct iavf_tx_queue_stats tx_stats;
+ struct iavf_rx_queue_stats rx_stats;
+ };
+
+ unsigned int size; /* length of descriptor ring in bytes */
+ dma_addr_t dma; /* physical address of ring */
+
+ struct iavf_vsi *vsi; /* Backreference to associated VSI */
+ struct iavf_q_vector *q_vector; /* Backreference to associated vector */
+
+ struct rcu_head rcu; /* to avoid race on free */
+ u16 next_to_alloc;
+ struct sk_buff *skb; /* When iavf_clean_rx_ring_irq() must
+ * return before it sees the EOP for
+ * the current packet, we save that skb
+ * here and resume receiving this
+ * packet the next time
+ * iavf_clean_rx_ring_irq() is called
+ * for this ring.
+ */
+} ____cacheline_internodealigned_in_smp;
+
+static inline bool ring_uses_build_skb(struct iavf_ring *ring)
+{
+ return !!(ring->flags & IAVF_RXR_FLAGS_BUILD_SKB_ENABLED);
+}
+
+static inline void set_ring_build_skb_enabled(struct iavf_ring *ring)
+{
+ ring->flags |= IAVF_RXR_FLAGS_BUILD_SKB_ENABLED;
+}
+
+static inline void clear_ring_build_skb_enabled(struct iavf_ring *ring)
+{
+ ring->flags &= ~IAVF_RXR_FLAGS_BUILD_SKB_ENABLED;
+}
+
+#define IAVF_ITR_ADAPTIVE_MIN_INC 0x0002
+#define IAVF_ITR_ADAPTIVE_MIN_USECS 0x0002
+#define IAVF_ITR_ADAPTIVE_MAX_USECS 0x007e
+#define IAVF_ITR_ADAPTIVE_LATENCY 0x8000
+#define IAVF_ITR_ADAPTIVE_BULK 0x0000
+#define ITR_IS_BULK(x) (!((x) & IAVF_ITR_ADAPTIVE_LATENCY))
+
+struct iavf_ring_container {
+ struct iavf_ring *ring; /* pointer to linked list of ring(s) */
+ unsigned long next_update; /* jiffies value of next update */
+ unsigned int total_bytes; /* total bytes processed this int */
+ unsigned int total_packets; /* total packets processed this int */
+ u16 count;
+ u16 target_itr; /* target ITR setting for ring(s) */
+ u16 current_itr; /* current ITR setting for ring(s) */
+};
+
+/* iterator for handling rings in ring container */
+#define iavf_for_each_ring(pos, head) \
+ for (pos = (head).ring; pos != NULL; pos = pos->next)
+
+static inline unsigned int iavf_rx_pg_order(struct iavf_ring *ring)
+{
+#if (PAGE_SIZE < 8192)
+ if (ring->rx_buf_len > (PAGE_SIZE / 2))
+ return 1;
+#endif
+ return 0;
+}
+
+#define iavf_rx_pg_size(_ring) (PAGE_SIZE << iavf_rx_pg_order(_ring))
+
+bool iavf_alloc_rx_buffers(struct iavf_ring *rxr, u16 cleaned_count);
+netdev_tx_t iavf_xmit_frame(struct sk_buff *skb, struct net_device *netdev);
+int iavf_setup_tx_descriptors(struct iavf_ring *tx_ring);
+int iavf_setup_rx_descriptors(struct iavf_ring *rx_ring);
+void iavf_free_tx_resources(struct iavf_ring *tx_ring);
+void iavf_free_rx_resources(struct iavf_ring *rx_ring);
+int iavf_napi_poll(struct napi_struct *napi, int budget);
+void iavf_detect_recover_hung(struct iavf_vsi *vsi);
+int __iavf_maybe_stop_tx(struct iavf_ring *tx_ring, int size);
+bool __iavf_chk_linearize(struct sk_buff *skb);
+
+/**
+ * iavf_xmit_descriptor_count - calculate number of Tx descriptors needed
+ * @skb: send buffer
+ *
+ * Returns number of data descriptors needed for this skb. Returns 0 to indicate
+ * there is not enough descriptors available in this ring since we need at least
+ * one descriptor.
+ **/
+static inline int iavf_xmit_descriptor_count(struct sk_buff *skb)
+{
+ const skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
+ unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
+ int count = 0, size = skb_headlen(skb);
+
+ for (;;) {
+ count += iavf_txd_use_count(size);
+
+ if (!nr_frags--)
+ break;
+
+ size = skb_frag_size(frag++);
+ }
+
+ return count;
+}
+
+/**
+ * iavf_maybe_stop_tx - 1st level check for Tx stop conditions
+ * @tx_ring: the ring to be checked
+ * @size: the size buffer we want to assure is available
+ *
+ * Returns 0 if stop is not needed
+ **/
+static inline int iavf_maybe_stop_tx(struct iavf_ring *tx_ring, int size)
+{
+ if (likely(IAVF_DESC_UNUSED(tx_ring) >= size))
+ return 0;
+ return __iavf_maybe_stop_tx(tx_ring, size);
+}
+
+/**
+ * iavf_chk_linearize - Check if there are more than 8 fragments per packet
+ * @skb: send buffer
+ * @count: number of buffers used
+ *
+ * Note: Our HW can't scatter-gather more than 8 fragments to build
+ * a packet on the wire and so we need to figure out the cases where we
+ * need to linearize the skb.
+ **/
+static inline bool iavf_chk_linearize(struct sk_buff *skb, int count)
+{
+ /* Both TSO and single send will work if count is less than 8 */
+ if (likely(count < IAVF_MAX_BUFFER_TXD))
+ return false;
+
+ if (skb_is_gso(skb))
+ return __iavf_chk_linearize(skb);
+
+ /* we can support up to 8 data buffers for a single send */
+ return count != IAVF_MAX_BUFFER_TXD;
+}
+/**
+ * txring_txq - helper to convert from a ring to a queue
+ * @ring: Tx ring to find the netdev equivalent of
+ **/
+static inline struct netdev_queue *txring_txq(const struct iavf_ring *ring)
+{
+ return netdev_get_tx_queue(ring->netdev, ring->queue_index);
+}
+#endif /* _IAVF_TXRX_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_type.h b/drivers/net/ethernet/intel/iavf/iavf_type.h
new file mode 100644
index 0000000000..9f1f523807
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_type.h
@@ -0,0 +1,679 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#ifndef _IAVF_TYPE_H_
+#define _IAVF_TYPE_H_
+
+#include "iavf_status.h"
+#include "iavf_osdep.h"
+#include "iavf_register.h"
+#include "iavf_adminq.h"
+#include "iavf_devids.h"
+
+#define IAVF_RXQ_CTX_DBUFF_SHIFT 7
+
+/* IAVF_MASK is a macro used on 32 bit registers */
+#define IAVF_MASK(mask, shift) ((u32)(mask) << (shift))
+
+#define IAVF_MAX_VSI_QP 16
+#define IAVF_MAX_VF_VSI 3
+#define IAVF_MAX_CHAINED_RX_BUFFERS 5
+
+/* forward declaration */
+struct iavf_hw;
+typedef void (*IAVF_ADMINQ_CALLBACK)(struct iavf_hw *, struct iavf_aq_desc *);
+
+/* Data type manipulation macros. */
+
+#define IAVF_DESC_UNUSED(R) \
+ ((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->count) + \
+ (R)->next_to_clean - (R)->next_to_use - 1)
+
+/* bitfields for Tx queue mapping in QTX_CTL */
+#define IAVF_QTX_CTL_VF_QUEUE 0x0
+#define IAVF_QTX_CTL_VM_QUEUE 0x1
+#define IAVF_QTX_CTL_PF_QUEUE 0x2
+
+/* debug masks - set these bits in hw->debug_mask to control output */
+enum iavf_debug_mask {
+ IAVF_DEBUG_INIT = 0x00000001,
+ IAVF_DEBUG_RELEASE = 0x00000002,
+
+ IAVF_DEBUG_LINK = 0x00000010,
+ IAVF_DEBUG_PHY = 0x00000020,
+ IAVF_DEBUG_HMC = 0x00000040,
+ IAVF_DEBUG_NVM = 0x00000080,
+ IAVF_DEBUG_LAN = 0x00000100,
+ IAVF_DEBUG_FLOW = 0x00000200,
+ IAVF_DEBUG_DCB = 0x00000400,
+ IAVF_DEBUG_DIAG = 0x00000800,
+ IAVF_DEBUG_FD = 0x00001000,
+ IAVF_DEBUG_PACKAGE = 0x00002000,
+
+ IAVF_DEBUG_AQ_MESSAGE = 0x01000000,
+ IAVF_DEBUG_AQ_DESCRIPTOR = 0x02000000,
+ IAVF_DEBUG_AQ_DESC_BUFFER = 0x04000000,
+ IAVF_DEBUG_AQ_COMMAND = 0x06000000,
+ IAVF_DEBUG_AQ = 0x0F000000,
+
+ IAVF_DEBUG_USER = 0xF0000000,
+
+ IAVF_DEBUG_ALL = 0xFFFFFFFF
+};
+
+/* These are structs for managing the hardware information and the operations.
+ * The structures of function pointers are filled out at init time when we
+ * know for sure exactly which hardware we're working with. This gives us the
+ * flexibility of using the same main driver code but adapting to slightly
+ * different hardware needs as new parts are developed. For this architecture,
+ * the Firmware and AdminQ are intended to insulate the driver from most of the
+ * future changes, but these structures will also do part of the job.
+ */
+enum iavf_mac_type {
+ IAVF_MAC_UNKNOWN = 0,
+ IAVF_MAC_XL710,
+ IAVF_MAC_VF,
+ IAVF_MAC_X722,
+ IAVF_MAC_X722_VF,
+ IAVF_MAC_GENERIC,
+};
+
+enum iavf_vsi_type {
+ IAVF_VSI_MAIN = 0,
+ IAVF_VSI_VMDQ1 = 1,
+ IAVF_VSI_VMDQ2 = 2,
+ IAVF_VSI_CTRL = 3,
+ IAVF_VSI_FCOE = 4,
+ IAVF_VSI_MIRROR = 5,
+ IAVF_VSI_SRIOV = 6,
+ IAVF_VSI_FDIR = 7,
+ IAVF_VSI_TYPE_UNKNOWN
+};
+
+enum iavf_queue_type {
+ IAVF_QUEUE_TYPE_RX = 0,
+ IAVF_QUEUE_TYPE_TX,
+ IAVF_QUEUE_TYPE_PE_CEQ,
+ IAVF_QUEUE_TYPE_UNKNOWN
+};
+
+#define IAVF_HW_CAP_MAX_GPIO 30
+/* Capabilities of a PF or a VF or the whole device */
+struct iavf_hw_capabilities {
+ bool dcb;
+ bool fcoe;
+ u32 num_vsis;
+ u32 num_rx_qp;
+ u32 num_tx_qp;
+ u32 base_queue;
+ u32 num_msix_vectors_vf;
+};
+
+struct iavf_mac_info {
+ enum iavf_mac_type type;
+ u8 addr[ETH_ALEN];
+ u8 perm_addr[ETH_ALEN];
+ u8 san_addr[ETH_ALEN];
+ u16 max_fcoeq;
+};
+
+/* PCI bus types */
+enum iavf_bus_type {
+ iavf_bus_type_unknown = 0,
+ iavf_bus_type_pci,
+ iavf_bus_type_pcix,
+ iavf_bus_type_pci_express,
+ iavf_bus_type_reserved
+};
+
+/* PCI bus speeds */
+enum iavf_bus_speed {
+ iavf_bus_speed_unknown = 0,
+ iavf_bus_speed_33 = 33,
+ iavf_bus_speed_66 = 66,
+ iavf_bus_speed_100 = 100,
+ iavf_bus_speed_120 = 120,
+ iavf_bus_speed_133 = 133,
+ iavf_bus_speed_2500 = 2500,
+ iavf_bus_speed_5000 = 5000,
+ iavf_bus_speed_8000 = 8000,
+ iavf_bus_speed_reserved
+};
+
+/* PCI bus widths */
+enum iavf_bus_width {
+ iavf_bus_width_unknown = 0,
+ iavf_bus_width_pcie_x1 = 1,
+ iavf_bus_width_pcie_x2 = 2,
+ iavf_bus_width_pcie_x4 = 4,
+ iavf_bus_width_pcie_x8 = 8,
+ iavf_bus_width_32 = 32,
+ iavf_bus_width_64 = 64,
+ iavf_bus_width_reserved
+};
+
+/* Bus parameters */
+struct iavf_bus_info {
+ enum iavf_bus_speed speed;
+ enum iavf_bus_width width;
+ enum iavf_bus_type type;
+
+ u16 func;
+ u16 device;
+ u16 lan_id;
+ u16 bus_id;
+};
+
+#define IAVF_MAX_USER_PRIORITY 8
+/* Port hardware description */
+struct iavf_hw {
+ u8 __iomem *hw_addr;
+ void *back;
+
+ /* subsystem structs */
+ struct iavf_mac_info mac;
+ struct iavf_bus_info bus;
+
+ /* pci info */
+ u16 device_id;
+ u16 vendor_id;
+ u16 subsystem_device_id;
+ u16 subsystem_vendor_id;
+ u8 revision_id;
+
+ /* capabilities for entire device and PCI func */
+ struct iavf_hw_capabilities dev_caps;
+
+ /* Admin Queue info */
+ struct iavf_adminq_info aq;
+
+ /* debug mask */
+ u32 debug_mask;
+ char err_str[16];
+};
+
+/* RX Descriptors */
+union iavf_16byte_rx_desc {
+ struct {
+ __le64 pkt_addr; /* Packet buffer address */
+ __le64 hdr_addr; /* Header buffer address */
+ } read;
+ struct {
+ struct {
+ struct {
+ union {
+ __le16 mirroring_status;
+ __le16 fcoe_ctx_id;
+ } mirr_fcoe;
+ __le16 l2tag1;
+ } lo_dword;
+ union {
+ __le32 rss; /* RSS Hash */
+ __le32 fd_id; /* Flow director filter id */
+ __le32 fcoe_param; /* FCoE DDP Context id */
+ } hi_dword;
+ } qword0;
+ struct {
+ /* ext status/error/pktype/length */
+ __le64 status_error_len;
+ } qword1;
+ } wb; /* writeback */
+};
+
+union iavf_32byte_rx_desc {
+ struct {
+ __le64 pkt_addr; /* Packet buffer address */
+ __le64 hdr_addr; /* Header buffer address */
+ /* bit 0 of hdr_buffer_addr is DD bit */
+ __le64 rsvd1;
+ __le64 rsvd2;
+ } read;
+ struct {
+ struct {
+ struct {
+ union {
+ __le16 mirroring_status;
+ __le16 fcoe_ctx_id;
+ } mirr_fcoe;
+ __le16 l2tag1;
+ } lo_dword;
+ union {
+ __le32 rss; /* RSS Hash */
+ __le32 fcoe_param; /* FCoE DDP Context id */
+ /* Flow director filter id in case of
+ * Programming status desc WB
+ */
+ __le32 fd_id;
+ } hi_dword;
+ } qword0;
+ struct {
+ /* status/error/pktype/length */
+ __le64 status_error_len;
+ } qword1;
+ struct {
+ __le16 ext_status; /* extended status */
+ __le16 rsvd;
+ __le16 l2tag2_1;
+ __le16 l2tag2_2;
+ } qword2;
+ struct {
+ union {
+ __le32 flex_bytes_lo;
+ __le32 pe_status;
+ } lo_dword;
+ union {
+ __le32 flex_bytes_hi;
+ __le32 fd_id;
+ } hi_dword;
+ } qword3;
+ } wb; /* writeback */
+};
+
+enum iavf_rx_desc_status_bits {
+ /* Note: These are predefined bit offsets */
+ IAVF_RX_DESC_STATUS_DD_SHIFT = 0,
+ IAVF_RX_DESC_STATUS_EOF_SHIFT = 1,
+ IAVF_RX_DESC_STATUS_L2TAG1P_SHIFT = 2,
+ IAVF_RX_DESC_STATUS_L3L4P_SHIFT = 3,
+ IAVF_RX_DESC_STATUS_CRCP_SHIFT = 4,
+ IAVF_RX_DESC_STATUS_TSYNINDX_SHIFT = 5, /* 2 BITS */
+ IAVF_RX_DESC_STATUS_TSYNVALID_SHIFT = 7,
+ /* Note: Bit 8 is reserved in X710 and XL710 */
+ IAVF_RX_DESC_STATUS_EXT_UDP_0_SHIFT = 8,
+ IAVF_RX_DESC_STATUS_UMBCAST_SHIFT = 9, /* 2 BITS */
+ IAVF_RX_DESC_STATUS_FLM_SHIFT = 11,
+ IAVF_RX_DESC_STATUS_FLTSTAT_SHIFT = 12, /* 2 BITS */
+ IAVF_RX_DESC_STATUS_LPBK_SHIFT = 14,
+ IAVF_RX_DESC_STATUS_IPV6EXADD_SHIFT = 15,
+ IAVF_RX_DESC_STATUS_RESERVED_SHIFT = 16, /* 2 BITS */
+ /* Note: For non-tunnel packets INT_UDP_0 is the right status for
+ * UDP header
+ */
+ IAVF_RX_DESC_STATUS_INT_UDP_0_SHIFT = 18,
+ IAVF_RX_DESC_STATUS_LAST /* this entry must be last!!! */
+};
+
+#define IAVF_RXD_QW1_STATUS_SHIFT 0
+#define IAVF_RXD_QW1_STATUS_MASK ((BIT(IAVF_RX_DESC_STATUS_LAST) - 1) \
+ << IAVF_RXD_QW1_STATUS_SHIFT)
+
+#define IAVF_RXD_QW1_STATUS_TSYNINDX_SHIFT IAVF_RX_DESC_STATUS_TSYNINDX_SHIFT
+#define IAVF_RXD_QW1_STATUS_TSYNINDX_MASK (0x3UL << \
+ IAVF_RXD_QW1_STATUS_TSYNINDX_SHIFT)
+
+#define IAVF_RXD_QW1_STATUS_TSYNVALID_SHIFT IAVF_RX_DESC_STATUS_TSYNVALID_SHIFT
+#define IAVF_RXD_QW1_STATUS_TSYNVALID_MASK \
+ BIT_ULL(IAVF_RXD_QW1_STATUS_TSYNVALID_SHIFT)
+
+enum iavf_rx_desc_fltstat_values {
+ IAVF_RX_DESC_FLTSTAT_NO_DATA = 0,
+ IAVF_RX_DESC_FLTSTAT_RSV_FD_ID = 1, /* 16byte desc? FD_ID : RSV */
+ IAVF_RX_DESC_FLTSTAT_RSV = 2,
+ IAVF_RX_DESC_FLTSTAT_RSS_HASH = 3,
+};
+
+#define IAVF_RXD_QW1_ERROR_SHIFT 19
+#define IAVF_RXD_QW1_ERROR_MASK (0xFFUL << IAVF_RXD_QW1_ERROR_SHIFT)
+
+enum iavf_rx_desc_error_bits {
+ /* Note: These are predefined bit offsets */
+ IAVF_RX_DESC_ERROR_RXE_SHIFT = 0,
+ IAVF_RX_DESC_ERROR_RECIPE_SHIFT = 1,
+ IAVF_RX_DESC_ERROR_HBO_SHIFT = 2,
+ IAVF_RX_DESC_ERROR_L3L4E_SHIFT = 3, /* 3 BITS */
+ IAVF_RX_DESC_ERROR_IPE_SHIFT = 3,
+ IAVF_RX_DESC_ERROR_L4E_SHIFT = 4,
+ IAVF_RX_DESC_ERROR_EIPE_SHIFT = 5,
+ IAVF_RX_DESC_ERROR_OVERSIZE_SHIFT = 6,
+ IAVF_RX_DESC_ERROR_PPRS_SHIFT = 7
+};
+
+enum iavf_rx_desc_error_l3l4e_fcoe_masks {
+ IAVF_RX_DESC_ERROR_L3L4E_NONE = 0,
+ IAVF_RX_DESC_ERROR_L3L4E_PROT = 1,
+ IAVF_RX_DESC_ERROR_L3L4E_FC = 2,
+ IAVF_RX_DESC_ERROR_L3L4E_DMAC_ERR = 3,
+ IAVF_RX_DESC_ERROR_L3L4E_DMAC_WARN = 4
+};
+
+#define IAVF_RXD_QW1_PTYPE_SHIFT 30
+#define IAVF_RXD_QW1_PTYPE_MASK (0xFFULL << IAVF_RXD_QW1_PTYPE_SHIFT)
+
+/* Packet type non-ip values */
+enum iavf_rx_l2_ptype {
+ IAVF_RX_PTYPE_L2_RESERVED = 0,
+ IAVF_RX_PTYPE_L2_MAC_PAY2 = 1,
+ IAVF_RX_PTYPE_L2_TIMESYNC_PAY2 = 2,
+ IAVF_RX_PTYPE_L2_FIP_PAY2 = 3,
+ IAVF_RX_PTYPE_L2_OUI_PAY2 = 4,
+ IAVF_RX_PTYPE_L2_MACCNTRL_PAY2 = 5,
+ IAVF_RX_PTYPE_L2_LLDP_PAY2 = 6,
+ IAVF_RX_PTYPE_L2_ECP_PAY2 = 7,
+ IAVF_RX_PTYPE_L2_EVB_PAY2 = 8,
+ IAVF_RX_PTYPE_L2_QCN_PAY2 = 9,
+ IAVF_RX_PTYPE_L2_EAPOL_PAY2 = 10,
+ IAVF_RX_PTYPE_L2_ARP = 11,
+ IAVF_RX_PTYPE_L2_FCOE_PAY3 = 12,
+ IAVF_RX_PTYPE_L2_FCOE_FCDATA_PAY3 = 13,
+ IAVF_RX_PTYPE_L2_FCOE_FCRDY_PAY3 = 14,
+ IAVF_RX_PTYPE_L2_FCOE_FCRSP_PAY3 = 15,
+ IAVF_RX_PTYPE_L2_FCOE_FCOTHER_PA = 16,
+ IAVF_RX_PTYPE_L2_FCOE_VFT_PAY3 = 17,
+ IAVF_RX_PTYPE_L2_FCOE_VFT_FCDATA = 18,
+ IAVF_RX_PTYPE_L2_FCOE_VFT_FCRDY = 19,
+ IAVF_RX_PTYPE_L2_FCOE_VFT_FCRSP = 20,
+ IAVF_RX_PTYPE_L2_FCOE_VFT_FCOTHER = 21,
+ IAVF_RX_PTYPE_GRENAT4_MAC_PAY3 = 58,
+ IAVF_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4 = 87,
+ IAVF_RX_PTYPE_GRENAT6_MAC_PAY3 = 124,
+ IAVF_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4 = 153
+};
+
+struct iavf_rx_ptype_decoded {
+ u32 known:1;
+ u32 outer_ip:1;
+ u32 outer_ip_ver:1;
+ u32 outer_frag:1;
+ u32 tunnel_type:3;
+ u32 tunnel_end_prot:2;
+ u32 tunnel_end_frag:1;
+ u32 inner_prot:4;
+ u32 payload_layer:3;
+};
+
+enum iavf_rx_ptype_outer_ip {
+ IAVF_RX_PTYPE_OUTER_L2 = 0,
+ IAVF_RX_PTYPE_OUTER_IP = 1
+};
+
+enum iavf_rx_ptype_outer_ip_ver {
+ IAVF_RX_PTYPE_OUTER_NONE = 0,
+ IAVF_RX_PTYPE_OUTER_IPV4 = 0,
+ IAVF_RX_PTYPE_OUTER_IPV6 = 1
+};
+
+enum iavf_rx_ptype_outer_fragmented {
+ IAVF_RX_PTYPE_NOT_FRAG = 0,
+ IAVF_RX_PTYPE_FRAG = 1
+};
+
+enum iavf_rx_ptype_tunnel_type {
+ IAVF_RX_PTYPE_TUNNEL_NONE = 0,
+ IAVF_RX_PTYPE_TUNNEL_IP_IP = 1,
+ IAVF_RX_PTYPE_TUNNEL_IP_GRENAT = 2,
+ IAVF_RX_PTYPE_TUNNEL_IP_GRENAT_MAC = 3,
+ IAVF_RX_PTYPE_TUNNEL_IP_GRENAT_MAC_VLAN = 4,
+};
+
+enum iavf_rx_ptype_tunnel_end_prot {
+ IAVF_RX_PTYPE_TUNNEL_END_NONE = 0,
+ IAVF_RX_PTYPE_TUNNEL_END_IPV4 = 1,
+ IAVF_RX_PTYPE_TUNNEL_END_IPV6 = 2,
+};
+
+enum iavf_rx_ptype_inner_prot {
+ IAVF_RX_PTYPE_INNER_PROT_NONE = 0,
+ IAVF_RX_PTYPE_INNER_PROT_UDP = 1,
+ IAVF_RX_PTYPE_INNER_PROT_TCP = 2,
+ IAVF_RX_PTYPE_INNER_PROT_SCTP = 3,
+ IAVF_RX_PTYPE_INNER_PROT_ICMP = 4,
+ IAVF_RX_PTYPE_INNER_PROT_TIMESYNC = 5
+};
+
+enum iavf_rx_ptype_payload_layer {
+ IAVF_RX_PTYPE_PAYLOAD_LAYER_NONE = 0,
+ IAVF_RX_PTYPE_PAYLOAD_LAYER_PAY2 = 1,
+ IAVF_RX_PTYPE_PAYLOAD_LAYER_PAY3 = 2,
+ IAVF_RX_PTYPE_PAYLOAD_LAYER_PAY4 = 3,
+};
+
+#define IAVF_RXD_QW1_LENGTH_PBUF_SHIFT 38
+#define IAVF_RXD_QW1_LENGTH_PBUF_MASK (0x3FFFULL << \
+ IAVF_RXD_QW1_LENGTH_PBUF_SHIFT)
+
+#define IAVF_RXD_QW1_LENGTH_HBUF_SHIFT 52
+#define IAVF_RXD_QW1_LENGTH_HBUF_MASK (0x7FFULL << \
+ IAVF_RXD_QW1_LENGTH_HBUF_SHIFT)
+
+#define IAVF_RXD_QW1_LENGTH_SPH_SHIFT 63
+#define IAVF_RXD_QW1_LENGTH_SPH_MASK BIT_ULL(IAVF_RXD_QW1_LENGTH_SPH_SHIFT)
+
+enum iavf_rx_desc_ext_status_bits {
+ /* Note: These are predefined bit offsets */
+ IAVF_RX_DESC_EXT_STATUS_L2TAG2P_SHIFT = 0,
+ IAVF_RX_DESC_EXT_STATUS_L2TAG3P_SHIFT = 1,
+ IAVF_RX_DESC_EXT_STATUS_FLEXBL_SHIFT = 2, /* 2 BITS */
+ IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT = 4, /* 2 BITS */
+ IAVF_RX_DESC_EXT_STATUS_FDLONGB_SHIFT = 9,
+ IAVF_RX_DESC_EXT_STATUS_FCOELONGB_SHIFT = 10,
+ IAVF_RX_DESC_EXT_STATUS_PELONGB_SHIFT = 11,
+};
+
+enum iavf_rx_desc_pe_status_bits {
+ /* Note: These are predefined bit offsets */
+ IAVF_RX_DESC_PE_STATUS_QPID_SHIFT = 0, /* 18 BITS */
+ IAVF_RX_DESC_PE_STATUS_L4PORT_SHIFT = 0, /* 16 BITS */
+ IAVF_RX_DESC_PE_STATUS_IPINDEX_SHIFT = 16, /* 8 BITS */
+ IAVF_RX_DESC_PE_STATUS_QPIDHIT_SHIFT = 24,
+ IAVF_RX_DESC_PE_STATUS_APBVTHIT_SHIFT = 25,
+ IAVF_RX_DESC_PE_STATUS_PORTV_SHIFT = 26,
+ IAVF_RX_DESC_PE_STATUS_URG_SHIFT = 27,
+ IAVF_RX_DESC_PE_STATUS_IPFRAG_SHIFT = 28,
+ IAVF_RX_DESC_PE_STATUS_IPOPT_SHIFT = 29
+};
+
+#define IAVF_RX_PROG_STATUS_DESC_LENGTH_SHIFT 38
+#define IAVF_RX_PROG_STATUS_DESC_LENGTH 0x2000000
+
+#define IAVF_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT 2
+#define IAVF_RX_PROG_STATUS_DESC_QW1_PROGID_MASK (0x7UL << \
+ IAVF_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT)
+
+#define IAVF_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT 19
+#define IAVF_RX_PROG_STATUS_DESC_QW1_ERROR_MASK (0x3FUL << \
+ IAVF_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT)
+
+enum iavf_rx_prog_status_desc_status_bits {
+ /* Note: These are predefined bit offsets */
+ IAVF_RX_PROG_STATUS_DESC_DD_SHIFT = 0,
+ IAVF_RX_PROG_STATUS_DESC_PROG_ID_SHIFT = 2 /* 3 BITS */
+};
+
+enum iavf_rx_prog_status_desc_prog_id_masks {
+ IAVF_RX_PROG_STATUS_DESC_FD_FILTER_STATUS = 1,
+ IAVF_RX_PROG_STATUS_DESC_FCOE_CTXT_PROG_STATUS = 2,
+ IAVF_RX_PROG_STATUS_DESC_FCOE_CTXT_INVL_STATUS = 4,
+};
+
+enum iavf_rx_prog_status_desc_error_bits {
+ /* Note: These are predefined bit offsets */
+ IAVF_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT = 0,
+ IAVF_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT = 1,
+ IAVF_RX_PROG_STATUS_DESC_FCOE_TBL_FULL_SHIFT = 2,
+ IAVF_RX_PROG_STATUS_DESC_FCOE_CONFLICT_SHIFT = 3
+};
+
+/* TX Descriptor */
+struct iavf_tx_desc {
+ __le64 buffer_addr; /* Address of descriptor's data buf */
+ __le64 cmd_type_offset_bsz;
+};
+
+#define IAVF_TXD_QW1_DTYPE_SHIFT 0
+#define IAVF_TXD_QW1_DTYPE_MASK (0xFUL << IAVF_TXD_QW1_DTYPE_SHIFT)
+
+enum iavf_tx_desc_dtype_value {
+ IAVF_TX_DESC_DTYPE_DATA = 0x0,
+ IAVF_TX_DESC_DTYPE_NOP = 0x1, /* same as Context desc */
+ IAVF_TX_DESC_DTYPE_CONTEXT = 0x1,
+ IAVF_TX_DESC_DTYPE_FCOE_CTX = 0x2,
+ IAVF_TX_DESC_DTYPE_FILTER_PROG = 0x8,
+ IAVF_TX_DESC_DTYPE_DDP_CTX = 0x9,
+ IAVF_TX_DESC_DTYPE_FLEX_DATA = 0xB,
+ IAVF_TX_DESC_DTYPE_FLEX_CTX_1 = 0xC,
+ IAVF_TX_DESC_DTYPE_FLEX_CTX_2 = 0xD,
+ IAVF_TX_DESC_DTYPE_DESC_DONE = 0xF
+};
+
+#define IAVF_TXD_QW1_CMD_SHIFT 4
+#define IAVF_TXD_QW1_CMD_MASK (0x3FFUL << IAVF_TXD_QW1_CMD_SHIFT)
+
+enum iavf_tx_desc_cmd_bits {
+ IAVF_TX_DESC_CMD_EOP = 0x0001,
+ IAVF_TX_DESC_CMD_RS = 0x0002,
+ IAVF_TX_DESC_CMD_ICRC = 0x0004,
+ IAVF_TX_DESC_CMD_IL2TAG1 = 0x0008,
+ IAVF_TX_DESC_CMD_DUMMY = 0x0010,
+ IAVF_TX_DESC_CMD_IIPT_NONIP = 0x0000, /* 2 BITS */
+ IAVF_TX_DESC_CMD_IIPT_IPV6 = 0x0020, /* 2 BITS */
+ IAVF_TX_DESC_CMD_IIPT_IPV4 = 0x0040, /* 2 BITS */
+ IAVF_TX_DESC_CMD_IIPT_IPV4_CSUM = 0x0060, /* 2 BITS */
+ IAVF_TX_DESC_CMD_FCOET = 0x0080,
+ IAVF_TX_DESC_CMD_L4T_EOFT_UNK = 0x0000, /* 2 BITS */
+ IAVF_TX_DESC_CMD_L4T_EOFT_TCP = 0x0100, /* 2 BITS */
+ IAVF_TX_DESC_CMD_L4T_EOFT_SCTP = 0x0200, /* 2 BITS */
+ IAVF_TX_DESC_CMD_L4T_EOFT_UDP = 0x0300, /* 2 BITS */
+ IAVF_TX_DESC_CMD_L4T_EOFT_EOF_N = 0x0000, /* 2 BITS */
+ IAVF_TX_DESC_CMD_L4T_EOFT_EOF_T = 0x0100, /* 2 BITS */
+ IAVF_TX_DESC_CMD_L4T_EOFT_EOF_NI = 0x0200, /* 2 BITS */
+ IAVF_TX_DESC_CMD_L4T_EOFT_EOF_A = 0x0300, /* 2 BITS */
+};
+
+#define IAVF_TXD_QW1_OFFSET_SHIFT 16
+#define IAVF_TXD_QW1_OFFSET_MASK (0x3FFFFULL << \
+ IAVF_TXD_QW1_OFFSET_SHIFT)
+
+enum iavf_tx_desc_length_fields {
+ /* Note: These are predefined bit offsets */
+ IAVF_TX_DESC_LENGTH_MACLEN_SHIFT = 0, /* 7 BITS */
+ IAVF_TX_DESC_LENGTH_IPLEN_SHIFT = 7, /* 7 BITS */
+ IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT = 14 /* 4 BITS */
+};
+
+#define IAVF_TXD_QW1_TX_BUF_SZ_SHIFT 34
+#define IAVF_TXD_QW1_TX_BUF_SZ_MASK (0x3FFFULL << \
+ IAVF_TXD_QW1_TX_BUF_SZ_SHIFT)
+
+#define IAVF_TXD_QW1_L2TAG1_SHIFT 48
+#define IAVF_TXD_QW1_L2TAG1_MASK (0xFFFFULL << IAVF_TXD_QW1_L2TAG1_SHIFT)
+
+/* Context descriptors */
+struct iavf_tx_context_desc {
+ __le32 tunneling_params;
+ __le16 l2tag2;
+ __le16 rsvd;
+ __le64 type_cmd_tso_mss;
+};
+
+#define IAVF_TXD_CTX_QW1_CMD_SHIFT 4
+#define IAVF_TXD_CTX_QW1_CMD_MASK (0xFFFFUL << IAVF_TXD_CTX_QW1_CMD_SHIFT)
+
+enum iavf_tx_ctx_desc_cmd_bits {
+ IAVF_TX_CTX_DESC_TSO = 0x01,
+ IAVF_TX_CTX_DESC_TSYN = 0x02,
+ IAVF_TX_CTX_DESC_IL2TAG2 = 0x04,
+ IAVF_TX_CTX_DESC_IL2TAG2_IL2H = 0x08,
+ IAVF_TX_CTX_DESC_SWTCH_NOTAG = 0x00,
+ IAVF_TX_CTX_DESC_SWTCH_UPLINK = 0x10,
+ IAVF_TX_CTX_DESC_SWTCH_LOCAL = 0x20,
+ IAVF_TX_CTX_DESC_SWTCH_VSI = 0x30,
+ IAVF_TX_CTX_DESC_SWPE = 0x40
+};
+
+/* Packet Classifier Types for filters */
+enum iavf_filter_pctype {
+ /* Note: Values 0-28 are reserved for future use.
+ * Value 29, 30, 32 are not supported on XL710 and X710.
+ */
+ IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP = 29,
+ IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP = 30,
+ IAVF_FILTER_PCTYPE_NONF_IPV4_UDP = 31,
+ IAVF_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK = 32,
+ IAVF_FILTER_PCTYPE_NONF_IPV4_TCP = 33,
+ IAVF_FILTER_PCTYPE_NONF_IPV4_SCTP = 34,
+ IAVF_FILTER_PCTYPE_NONF_IPV4_OTHER = 35,
+ IAVF_FILTER_PCTYPE_FRAG_IPV4 = 36,
+ /* Note: Values 37-38 are reserved for future use.
+ * Value 39, 40, 42 are not supported on XL710 and X710.
+ */
+ IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP = 39,
+ IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP = 40,
+ IAVF_FILTER_PCTYPE_NONF_IPV6_UDP = 41,
+ IAVF_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK = 42,
+ IAVF_FILTER_PCTYPE_NONF_IPV6_TCP = 43,
+ IAVF_FILTER_PCTYPE_NONF_IPV6_SCTP = 44,
+ IAVF_FILTER_PCTYPE_NONF_IPV6_OTHER = 45,
+ IAVF_FILTER_PCTYPE_FRAG_IPV6 = 46,
+ /* Note: Value 47 is reserved for future use */
+ IAVF_FILTER_PCTYPE_FCOE_OX = 48,
+ IAVF_FILTER_PCTYPE_FCOE_RX = 49,
+ IAVF_FILTER_PCTYPE_FCOE_OTHER = 50,
+ /* Note: Values 51-62 are reserved for future use */
+ IAVF_FILTER_PCTYPE_L2_PAYLOAD = 63,
+};
+
+#define IAVF_TXD_CTX_QW1_TSO_LEN_SHIFT 30
+#define IAVF_TXD_CTX_QW1_TSO_LEN_MASK (0x3FFFFULL << \
+ IAVF_TXD_CTX_QW1_TSO_LEN_SHIFT)
+
+#define IAVF_TXD_CTX_QW1_MSS_SHIFT 50
+#define IAVF_TXD_CTX_QW1_MSS_MASK (0x3FFFULL << \
+ IAVF_TXD_CTX_QW1_MSS_SHIFT)
+
+#define IAVF_TXD_CTX_QW1_VSI_SHIFT 50
+#define IAVF_TXD_CTX_QW1_VSI_MASK (0x1FFULL << IAVF_TXD_CTX_QW1_VSI_SHIFT)
+
+#define IAVF_TXD_CTX_QW0_EXT_IP_SHIFT 0
+#define IAVF_TXD_CTX_QW0_EXT_IP_MASK (0x3ULL << \
+ IAVF_TXD_CTX_QW0_EXT_IP_SHIFT)
+
+enum iavf_tx_ctx_desc_eipt_offload {
+ IAVF_TX_CTX_EXT_IP_NONE = 0x0,
+ IAVF_TX_CTX_EXT_IP_IPV6 = 0x1,
+ IAVF_TX_CTX_EXT_IP_IPV4_NO_CSUM = 0x2,
+ IAVF_TX_CTX_EXT_IP_IPV4 = 0x3
+};
+
+#define IAVF_TXD_CTX_QW0_EXT_IPLEN_SHIFT 2
+#define IAVF_TXD_CTX_QW0_EXT_IPLEN_MASK (0x3FULL << \
+ IAVF_TXD_CTX_QW0_EXT_IPLEN_SHIFT)
+
+#define IAVF_TXD_CTX_QW0_NATT_SHIFT 9
+#define IAVF_TXD_CTX_QW0_NATT_MASK (0x3ULL << IAVF_TXD_CTX_QW0_NATT_SHIFT)
+
+#define IAVF_TXD_CTX_UDP_TUNNELING BIT_ULL(IAVF_TXD_CTX_QW0_NATT_SHIFT)
+#define IAVF_TXD_CTX_GRE_TUNNELING (0x2ULL << IAVF_TXD_CTX_QW0_NATT_SHIFT)
+
+#define IAVF_TXD_CTX_QW0_EIP_NOINC_SHIFT 11
+#define IAVF_TXD_CTX_QW0_EIP_NOINC_MASK \
+ BIT_ULL(IAVF_TXD_CTX_QW0_EIP_NOINC_SHIFT)
+
+#define IAVF_TXD_CTX_EIP_NOINC_IPID_CONST IAVF_TXD_CTX_QW0_EIP_NOINC_MASK
+
+#define IAVF_TXD_CTX_QW0_NATLEN_SHIFT 12
+#define IAVF_TXD_CTX_QW0_NATLEN_MASK (0X7FULL << \
+ IAVF_TXD_CTX_QW0_NATLEN_SHIFT)
+
+#define IAVF_TXD_CTX_QW0_DECTTL_SHIFT 19
+#define IAVF_TXD_CTX_QW0_DECTTL_MASK (0xFULL << \
+ IAVF_TXD_CTX_QW0_DECTTL_SHIFT)
+
+#define IAVF_TXD_CTX_QW0_L4T_CS_SHIFT 23
+#define IAVF_TXD_CTX_QW0_L4T_CS_MASK BIT_ULL(IAVF_TXD_CTX_QW0_L4T_CS_SHIFT)
+
+/* Statistics collected by each port, VSI, VEB, and S-channel */
+struct iavf_eth_stats {
+ u64 rx_bytes; /* gorc */
+ u64 rx_unicast; /* uprc */
+ u64 rx_multicast; /* mprc */
+ u64 rx_broadcast; /* bprc */
+ u64 rx_discards; /* rdpc */
+ u64 rx_unknown_protocol; /* rupp */
+ u64 tx_bytes; /* gotc */
+ u64 tx_unicast; /* uptc */
+ u64 tx_multicast; /* mptc */
+ u64 tx_broadcast; /* bptc */
+ u64 tx_discards; /* tdpc */
+ u64 tx_errors; /* tepc */
+};
+#endif /* _IAVF_TYPE_H_ */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c
new file mode 100644
index 0000000000..b95a4f9032
--- /dev/null
+++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c
@@ -0,0 +1,2550 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2013 - 2018 Intel Corporation. */
+
+#include "iavf.h"
+#include "iavf_prototype.h"
+#include "iavf_client.h"
+
+/**
+ * iavf_send_pf_msg
+ * @adapter: adapter structure
+ * @op: virtual channel opcode
+ * @msg: pointer to message buffer
+ * @len: message length
+ *
+ * Send message to PF and print status if failure.
+ **/
+static int iavf_send_pf_msg(struct iavf_adapter *adapter,
+ enum virtchnl_ops op, u8 *msg, u16 len)
+{
+ struct iavf_hw *hw = &adapter->hw;
+ enum iavf_status status;
+
+ if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
+ return 0; /* nothing to see here, move along */
+
+ status = iavf_aq_send_msg_to_pf(hw, op, 0, msg, len, NULL);
+ if (status)
+ dev_dbg(&adapter->pdev->dev, "Unable to send opcode %d to PF, status %s, aq_err %s\n",
+ op, iavf_stat_str(hw, status),
+ iavf_aq_str(hw, hw->aq.asq_last_status));
+ return iavf_status_to_errno(status);
+}
+
+/**
+ * iavf_send_api_ver
+ * @adapter: adapter structure
+ *
+ * Send API version admin queue message to the PF. The reply is not checked
+ * in this function. Returns 0 if the message was successfully
+ * sent, or one of the IAVF_ADMIN_QUEUE_ERROR_ statuses if not.
+ **/
+int iavf_send_api_ver(struct iavf_adapter *adapter)
+{
+ struct virtchnl_version_info vvi;
+
+ vvi.major = VIRTCHNL_VERSION_MAJOR;
+ vvi.minor = VIRTCHNL_VERSION_MINOR;
+
+ return iavf_send_pf_msg(adapter, VIRTCHNL_OP_VERSION, (u8 *)&vvi,
+ sizeof(vvi));
+}
+
+/**
+ * iavf_poll_virtchnl_msg
+ * @hw: HW configuration structure
+ * @event: event to populate on success
+ * @op_to_poll: requested virtchnl op to poll for
+ *
+ * Initialize poll for virtchnl msg matching the requested_op. Returns 0
+ * if a message of the correct opcode is in the queue or an error code
+ * if no message matching the op code is waiting and other failures.
+ */
+static int
+iavf_poll_virtchnl_msg(struct iavf_hw *hw, struct iavf_arq_event_info *event,
+ enum virtchnl_ops op_to_poll)
+{
+ enum virtchnl_ops received_op;
+ enum iavf_status status;
+ u32 v_retval;
+
+ while (1) {
+ /* When the AQ is empty, iavf_clean_arq_element will return
+ * nonzero and this loop will terminate.
+ */
+ status = iavf_clean_arq_element(hw, event, NULL);
+ if (status != IAVF_SUCCESS)
+ return iavf_status_to_errno(status);
+ received_op =
+ (enum virtchnl_ops)le32_to_cpu(event->desc.cookie_high);
+ if (op_to_poll == received_op)
+ break;
+ }
+
+ v_retval = le32_to_cpu(event->desc.cookie_low);
+ return virtchnl_status_to_errno((enum virtchnl_status_code)v_retval);
+}
+
+/**
+ * iavf_verify_api_ver
+ * @adapter: adapter structure
+ *
+ * Compare API versions with the PF. Must be called after admin queue is
+ * initialized. Returns 0 if API versions match, -EIO if they do not,
+ * IAVF_ERR_ADMIN_QUEUE_NO_WORK if the admin queue is empty, and any errors
+ * from the firmware are propagated.
+ **/
+int iavf_verify_api_ver(struct iavf_adapter *adapter)
+{
+ struct iavf_arq_event_info event;
+ int err;
+
+ event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
+ event.msg_buf = kzalloc(IAVF_MAX_AQ_BUF_SIZE, GFP_KERNEL);
+ if (!event.msg_buf)
+ return -ENOMEM;
+
+ err = iavf_poll_virtchnl_msg(&adapter->hw, &event, VIRTCHNL_OP_VERSION);
+ if (!err) {
+ struct virtchnl_version_info *pf_vvi =
+ (struct virtchnl_version_info *)event.msg_buf;
+ adapter->pf_version = *pf_vvi;
+
+ if (pf_vvi->major > VIRTCHNL_VERSION_MAJOR ||
+ (pf_vvi->major == VIRTCHNL_VERSION_MAJOR &&
+ pf_vvi->minor > VIRTCHNL_VERSION_MINOR))
+ err = -EIO;
+ }
+
+ kfree(event.msg_buf);
+
+ return err;
+}
+
+/**
+ * iavf_send_vf_config_msg
+ * @adapter: adapter structure
+ *
+ * Send VF configuration request admin queue message to the PF. The reply
+ * is not checked in this function. Returns 0 if the message was
+ * successfully sent, or one of the IAVF_ADMIN_QUEUE_ERROR_ statuses if not.
+ **/
+int iavf_send_vf_config_msg(struct iavf_adapter *adapter)
+{
+ u32 caps;
+
+ caps = VIRTCHNL_VF_OFFLOAD_L2 |
+ VIRTCHNL_VF_OFFLOAD_RSS_PF |
+ VIRTCHNL_VF_OFFLOAD_RSS_AQ |
+ VIRTCHNL_VF_OFFLOAD_RSS_REG |
+ VIRTCHNL_VF_OFFLOAD_VLAN |
+ VIRTCHNL_VF_OFFLOAD_WB_ON_ITR |
+ VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2 |
+ VIRTCHNL_VF_OFFLOAD_ENCAP |
+ VIRTCHNL_VF_OFFLOAD_VLAN_V2 |
+ VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM |
+ VIRTCHNL_VF_OFFLOAD_REQ_QUEUES |
+ VIRTCHNL_VF_OFFLOAD_ADQ |
+ VIRTCHNL_VF_OFFLOAD_USO |
+ VIRTCHNL_VF_OFFLOAD_FDIR_PF |
+ VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
+ VIRTCHNL_VF_CAP_ADV_LINK_SPEED;
+
+ adapter->current_op = VIRTCHNL_OP_GET_VF_RESOURCES;
+ adapter->aq_required &= ~IAVF_FLAG_AQ_GET_CONFIG;
+ if (PF_IS_V11(adapter))
+ return iavf_send_pf_msg(adapter, VIRTCHNL_OP_GET_VF_RESOURCES,
+ (u8 *)&caps, sizeof(caps));
+ else
+ return iavf_send_pf_msg(adapter, VIRTCHNL_OP_GET_VF_RESOURCES,
+ NULL, 0);
+}
+
+int iavf_send_vf_offload_vlan_v2_msg(struct iavf_adapter *adapter)
+{
+ adapter->aq_required &= ~IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS;
+
+ if (!VLAN_V2_ALLOWED(adapter))
+ return -EOPNOTSUPP;
+
+ adapter->current_op = VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS;
+
+ return iavf_send_pf_msg(adapter, VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS,
+ NULL, 0);
+}
+
+/**
+ * iavf_validate_num_queues
+ * @adapter: adapter structure
+ *
+ * Validate that the number of queues the PF has sent in
+ * VIRTCHNL_OP_GET_VF_RESOURCES is not larger than the VF can handle.
+ **/
+static void iavf_validate_num_queues(struct iavf_adapter *adapter)
+{
+ if (adapter->vf_res->num_queue_pairs > IAVF_MAX_REQ_QUEUES) {
+ struct virtchnl_vsi_resource *vsi_res;
+ int i;
+
+ dev_info(&adapter->pdev->dev, "Received %d queues, but can only have a max of %d\n",
+ adapter->vf_res->num_queue_pairs,
+ IAVF_MAX_REQ_QUEUES);
+ dev_info(&adapter->pdev->dev, "Fixing by reducing queues to %d\n",
+ IAVF_MAX_REQ_QUEUES);
+ adapter->vf_res->num_queue_pairs = IAVF_MAX_REQ_QUEUES;
+ for (i = 0; i < adapter->vf_res->num_vsis; i++) {
+ vsi_res = &adapter->vf_res->vsi_res[i];
+ vsi_res->num_queue_pairs = IAVF_MAX_REQ_QUEUES;
+ }
+ }
+}
+
+/**
+ * iavf_get_vf_config
+ * @adapter: private adapter structure
+ *
+ * Get VF configuration from PF and populate hw structure. Must be called after
+ * admin queue is initialized. Busy waits until response is received from PF,
+ * with maximum timeout. Response from PF is returned in the buffer for further
+ * processing by the caller.
+ **/
+int iavf_get_vf_config(struct iavf_adapter *adapter)
+{
+ struct iavf_hw *hw = &adapter->hw;
+ struct iavf_arq_event_info event;
+ u16 len;
+ int err;
+
+ len = IAVF_VIRTCHNL_VF_RESOURCE_SIZE;
+ event.buf_len = len;
+ event.msg_buf = kzalloc(len, GFP_KERNEL);
+ if (!event.msg_buf)
+ return -ENOMEM;
+
+ err = iavf_poll_virtchnl_msg(hw, &event, VIRTCHNL_OP_GET_VF_RESOURCES);
+ memcpy(adapter->vf_res, event.msg_buf, min(event.msg_len, len));
+
+ /* some PFs send more queues than we should have so validate that
+ * we aren't getting too many queues
+ */
+ if (!err)
+ iavf_validate_num_queues(adapter);
+ iavf_vf_parse_hw_config(hw, adapter->vf_res);
+
+ kfree(event.msg_buf);
+
+ return err;
+}
+
+int iavf_get_vf_vlan_v2_caps(struct iavf_adapter *adapter)
+{
+ struct iavf_arq_event_info event;
+ int err;
+ u16 len;
+
+ len = sizeof(struct virtchnl_vlan_caps);
+ event.buf_len = len;
+ event.msg_buf = kzalloc(len, GFP_KERNEL);
+ if (!event.msg_buf)
+ return -ENOMEM;
+
+ err = iavf_poll_virtchnl_msg(&adapter->hw, &event,
+ VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS);
+ if (!err)
+ memcpy(&adapter->vlan_v2_caps, event.msg_buf,
+ min(event.msg_len, len));
+
+ kfree(event.msg_buf);
+
+ return err;
+}
+
+/**
+ * iavf_configure_queues
+ * @adapter: adapter structure
+ *
+ * Request that the PF set up our (previously allocated) queues.
+ **/
+void iavf_configure_queues(struct iavf_adapter *adapter)
+{
+ struct virtchnl_vsi_queue_config_info *vqci;
+ int i, max_frame = adapter->vf_res->max_mtu;
+ int pairs = adapter->num_active_queues;
+ struct virtchnl_queue_pair_info *vqpi;
+ size_t len;
+
+ if (max_frame > IAVF_MAX_RXBUFFER || !max_frame)
+ max_frame = IAVF_MAX_RXBUFFER;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot configure queues, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_CONFIG_VSI_QUEUES;
+ len = virtchnl_struct_size(vqci, qpair, pairs);
+ vqci = kzalloc(len, GFP_KERNEL);
+ if (!vqci)
+ return;
+
+ /* Limit maximum frame size when jumbo frames is not enabled */
+ if (!(adapter->flags & IAVF_FLAG_LEGACY_RX) &&
+ (adapter->netdev->mtu <= ETH_DATA_LEN))
+ max_frame = IAVF_RXBUFFER_1536 - NET_IP_ALIGN;
+
+ vqci->vsi_id = adapter->vsi_res->vsi_id;
+ vqci->num_queue_pairs = pairs;
+ vqpi = vqci->qpair;
+ /* Size check is not needed here - HW max is 16 queue pairs, and we
+ * can fit info for 31 of them into the AQ buffer before it overflows.
+ */
+ for (i = 0; i < pairs; i++) {
+ vqpi->txq.vsi_id = vqci->vsi_id;
+ vqpi->txq.queue_id = i;
+ vqpi->txq.ring_len = adapter->tx_rings[i].count;
+ vqpi->txq.dma_ring_addr = adapter->tx_rings[i].dma;
+ vqpi->rxq.vsi_id = vqci->vsi_id;
+ vqpi->rxq.queue_id = i;
+ vqpi->rxq.ring_len = adapter->rx_rings[i].count;
+ vqpi->rxq.dma_ring_addr = adapter->rx_rings[i].dma;
+ vqpi->rxq.max_pkt_size = max_frame;
+ vqpi->rxq.databuffer_size =
+ ALIGN(adapter->rx_rings[i].rx_buf_len,
+ BIT_ULL(IAVF_RXQ_CTX_DBUFF_SHIFT));
+ vqpi++;
+ }
+
+ adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_QUEUES;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_CONFIG_VSI_QUEUES,
+ (u8 *)vqci, len);
+ kfree(vqci);
+}
+
+/**
+ * iavf_enable_queues
+ * @adapter: adapter structure
+ *
+ * Request that the PF enable all of our queues.
+ **/
+void iavf_enable_queues(struct iavf_adapter *adapter)
+{
+ struct virtchnl_queue_select vqs;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot enable queues, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_ENABLE_QUEUES;
+ vqs.vsi_id = adapter->vsi_res->vsi_id;
+ vqs.tx_queues = BIT(adapter->num_active_queues) - 1;
+ vqs.rx_queues = vqs.tx_queues;
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ENABLE_QUEUES;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_ENABLE_QUEUES,
+ (u8 *)&vqs, sizeof(vqs));
+}
+
+/**
+ * iavf_disable_queues
+ * @adapter: adapter structure
+ *
+ * Request that the PF disable all of our queues.
+ **/
+void iavf_disable_queues(struct iavf_adapter *adapter)
+{
+ struct virtchnl_queue_select vqs;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot disable queues, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_DISABLE_QUEUES;
+ vqs.vsi_id = adapter->vsi_res->vsi_id;
+ vqs.tx_queues = BIT(adapter->num_active_queues) - 1;
+ vqs.rx_queues = vqs.tx_queues;
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DISABLE_QUEUES;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_DISABLE_QUEUES,
+ (u8 *)&vqs, sizeof(vqs));
+}
+
+/**
+ * iavf_map_queues
+ * @adapter: adapter structure
+ *
+ * Request that the PF map queues to interrupt vectors. Misc causes, including
+ * admin queue, are always mapped to vector 0.
+ **/
+void iavf_map_queues(struct iavf_adapter *adapter)
+{
+ struct virtchnl_irq_map_info *vimi;
+ struct virtchnl_vector_map *vecmap;
+ struct iavf_q_vector *q_vector;
+ int v_idx, q_vectors;
+ size_t len;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot map queues to vectors, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_CONFIG_IRQ_MAP;
+
+ q_vectors = adapter->num_msix_vectors - NONQ_VECS;
+
+ len = virtchnl_struct_size(vimi, vecmap, adapter->num_msix_vectors);
+ vimi = kzalloc(len, GFP_KERNEL);
+ if (!vimi)
+ return;
+
+ vimi->num_vectors = adapter->num_msix_vectors;
+ /* Queue vectors first */
+ for (v_idx = 0; v_idx < q_vectors; v_idx++) {
+ q_vector = &adapter->q_vectors[v_idx];
+ vecmap = &vimi->vecmap[v_idx];
+
+ vecmap->vsi_id = adapter->vsi_res->vsi_id;
+ vecmap->vector_id = v_idx + NONQ_VECS;
+ vecmap->txq_map = q_vector->ring_mask;
+ vecmap->rxq_map = q_vector->ring_mask;
+ vecmap->rxitr_idx = IAVF_RX_ITR;
+ vecmap->txitr_idx = IAVF_TX_ITR;
+ }
+ /* Misc vector last - this is only for AdminQ messages */
+ vecmap = &vimi->vecmap[v_idx];
+ vecmap->vsi_id = adapter->vsi_res->vsi_id;
+ vecmap->vector_id = 0;
+ vecmap->txq_map = 0;
+ vecmap->rxq_map = 0;
+
+ adapter->aq_required &= ~IAVF_FLAG_AQ_MAP_VECTORS;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_CONFIG_IRQ_MAP,
+ (u8 *)vimi, len);
+ kfree(vimi);
+}
+
+/**
+ * iavf_set_mac_addr_type - Set the correct request type from the filter type
+ * @virtchnl_ether_addr: pointer to requested list element
+ * @filter: pointer to requested filter
+ **/
+static void
+iavf_set_mac_addr_type(struct virtchnl_ether_addr *virtchnl_ether_addr,
+ const struct iavf_mac_filter *filter)
+{
+ virtchnl_ether_addr->type = filter->is_primary ?
+ VIRTCHNL_ETHER_ADDR_PRIMARY :
+ VIRTCHNL_ETHER_ADDR_EXTRA;
+}
+
+/**
+ * iavf_add_ether_addrs
+ * @adapter: adapter structure
+ *
+ * Request that the PF add one or more addresses to our filters.
+ **/
+void iavf_add_ether_addrs(struct iavf_adapter *adapter)
+{
+ struct virtchnl_ether_addr_list *veal;
+ struct iavf_mac_filter *f;
+ int i = 0, count = 0;
+ bool more = false;
+ size_t len;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot add filters, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ list_for_each_entry(f, &adapter->mac_filter_list, list) {
+ if (f->add)
+ count++;
+ }
+ if (!count) {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_MAC_FILTER;
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_ADD_ETH_ADDR;
+
+ len = virtchnl_struct_size(veal, list, count);
+ if (len > IAVF_MAX_AQ_BUF_SIZE) {
+ dev_warn(&adapter->pdev->dev, "Too many add MAC changes in one request\n");
+ while (len > IAVF_MAX_AQ_BUF_SIZE)
+ len = virtchnl_struct_size(veal, list, --count);
+ more = true;
+ }
+
+ veal = kzalloc(len, GFP_ATOMIC);
+ if (!veal) {
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return;
+ }
+
+ veal->vsi_id = adapter->vsi_res->vsi_id;
+ veal->num_elements = count;
+ list_for_each_entry(f, &adapter->mac_filter_list, list) {
+ if (f->add) {
+ ether_addr_copy(veal->list[i].addr, f->macaddr);
+ iavf_set_mac_addr_type(&veal->list[i], f);
+ i++;
+ f->add = false;
+ if (i == count)
+ break;
+ }
+ }
+ if (!more)
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_MAC_FILTER;
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_ETH_ADDR, (u8 *)veal, len);
+ kfree(veal);
+}
+
+/**
+ * iavf_del_ether_addrs
+ * @adapter: adapter structure
+ *
+ * Request that the PF remove one or more addresses from our filters.
+ **/
+void iavf_del_ether_addrs(struct iavf_adapter *adapter)
+{
+ struct virtchnl_ether_addr_list *veal;
+ struct iavf_mac_filter *f, *ftmp;
+ int i = 0, count = 0;
+ bool more = false;
+ size_t len;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot remove filters, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ list_for_each_entry(f, &adapter->mac_filter_list, list) {
+ if (f->remove)
+ count++;
+ }
+ if (!count) {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_MAC_FILTER;
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_DEL_ETH_ADDR;
+
+ len = virtchnl_struct_size(veal, list, count);
+ if (len > IAVF_MAX_AQ_BUF_SIZE) {
+ dev_warn(&adapter->pdev->dev, "Too many delete MAC changes in one request\n");
+ while (len > IAVF_MAX_AQ_BUF_SIZE)
+ len = virtchnl_struct_size(veal, list, --count);
+ more = true;
+ }
+ veal = kzalloc(len, GFP_ATOMIC);
+ if (!veal) {
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return;
+ }
+
+ veal->vsi_id = adapter->vsi_res->vsi_id;
+ veal->num_elements = count;
+ list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
+ if (f->remove) {
+ ether_addr_copy(veal->list[i].addr, f->macaddr);
+ iavf_set_mac_addr_type(&veal->list[i], f);
+ i++;
+ list_del(&f->list);
+ kfree(f);
+ if (i == count)
+ break;
+ }
+ }
+ if (!more)
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_MAC_FILTER;
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_ETH_ADDR, (u8 *)veal, len);
+ kfree(veal);
+}
+
+/**
+ * iavf_mac_add_ok
+ * @adapter: adapter structure
+ *
+ * Submit list of filters based on PF response.
+ **/
+static void iavf_mac_add_ok(struct iavf_adapter *adapter)
+{
+ struct iavf_mac_filter *f, *ftmp;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+ list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
+ f->is_new_mac = false;
+ if (!f->add && !f->add_handled)
+ f->add_handled = true;
+ }
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+}
+
+/**
+ * iavf_mac_add_reject
+ * @adapter: adapter structure
+ *
+ * Remove filters from list based on PF response.
+ **/
+static void iavf_mac_add_reject(struct iavf_adapter *adapter)
+{
+ struct net_device *netdev = adapter->netdev;
+ struct iavf_mac_filter *f, *ftmp;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+ list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
+ if (f->remove && ether_addr_equal(f->macaddr, netdev->dev_addr))
+ f->remove = false;
+
+ if (!f->add && !f->add_handled)
+ f->add_handled = true;
+
+ if (f->is_new_mac) {
+ list_del(&f->list);
+ kfree(f);
+ }
+ }
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+}
+
+/**
+ * iavf_vlan_add_reject
+ * @adapter: adapter structure
+ *
+ * Remove VLAN filters from list based on PF response.
+ **/
+static void iavf_vlan_add_reject(struct iavf_adapter *adapter)
+{
+ struct iavf_vlan_filter *f, *ftmp;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+ list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) {
+ if (f->state == IAVF_VLAN_IS_NEW) {
+ list_del(&f->list);
+ kfree(f);
+ adapter->num_vlan_filters--;
+ }
+ }
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+}
+
+/**
+ * iavf_add_vlans
+ * @adapter: adapter structure
+ *
+ * Request that the PF add one or more VLAN filters to our VSI.
+ **/
+void iavf_add_vlans(struct iavf_adapter *adapter)
+{
+ int len, i = 0, count = 0;
+ struct iavf_vlan_filter *f;
+ bool more = false;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot add VLANs, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ list_for_each_entry(f, &adapter->vlan_filter_list, list) {
+ if (f->state == IAVF_VLAN_ADD)
+ count++;
+ }
+ if (!count || !VLAN_FILTERING_ALLOWED(adapter)) {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_VLAN_FILTER;
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return;
+ }
+
+ if (VLAN_ALLOWED(adapter)) {
+ struct virtchnl_vlan_filter_list *vvfl;
+
+ adapter->current_op = VIRTCHNL_OP_ADD_VLAN;
+
+ len = virtchnl_struct_size(vvfl, vlan_id, count);
+ if (len > IAVF_MAX_AQ_BUF_SIZE) {
+ dev_warn(&adapter->pdev->dev, "Too many add VLAN changes in one request\n");
+ while (len > IAVF_MAX_AQ_BUF_SIZE)
+ len = virtchnl_struct_size(vvfl, vlan_id,
+ --count);
+ more = true;
+ }
+ vvfl = kzalloc(len, GFP_ATOMIC);
+ if (!vvfl) {
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return;
+ }
+
+ vvfl->vsi_id = adapter->vsi_res->vsi_id;
+ vvfl->num_elements = count;
+ list_for_each_entry(f, &adapter->vlan_filter_list, list) {
+ if (f->state == IAVF_VLAN_ADD) {
+ vvfl->vlan_id[i] = f->vlan.vid;
+ i++;
+ f->state = IAVF_VLAN_IS_NEW;
+ if (i == count)
+ break;
+ }
+ }
+ if (!more)
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_VLAN_FILTER;
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_VLAN, (u8 *)vvfl, len);
+ kfree(vvfl);
+ } else {
+ u16 max_vlans = adapter->vlan_v2_caps.filtering.max_filters;
+ u16 current_vlans = iavf_get_num_vlans_added(adapter);
+ struct virtchnl_vlan_filter_list_v2 *vvfl_v2;
+
+ adapter->current_op = VIRTCHNL_OP_ADD_VLAN_V2;
+
+ if ((count + current_vlans) > max_vlans &&
+ current_vlans < max_vlans) {
+ count = max_vlans - iavf_get_num_vlans_added(adapter);
+ more = true;
+ }
+
+ len = virtchnl_struct_size(vvfl_v2, filters, count);
+ if (len > IAVF_MAX_AQ_BUF_SIZE) {
+ dev_warn(&adapter->pdev->dev, "Too many add VLAN changes in one request\n");
+ while (len > IAVF_MAX_AQ_BUF_SIZE)
+ len = virtchnl_struct_size(vvfl_v2, filters,
+ --count);
+ more = true;
+ }
+
+ vvfl_v2 = kzalloc(len, GFP_ATOMIC);
+ if (!vvfl_v2) {
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return;
+ }
+
+ vvfl_v2->vport_id = adapter->vsi_res->vsi_id;
+ vvfl_v2->num_elements = count;
+ list_for_each_entry(f, &adapter->vlan_filter_list, list) {
+ if (f->state == IAVF_VLAN_ADD) {
+ struct virtchnl_vlan_supported_caps *filtering_support =
+ &adapter->vlan_v2_caps.filtering.filtering_support;
+ struct virtchnl_vlan *vlan;
+
+ if (i == count)
+ break;
+
+ /* give priority over outer if it's enabled */
+ if (filtering_support->outer)
+ vlan = &vvfl_v2->filters[i].outer;
+ else
+ vlan = &vvfl_v2->filters[i].inner;
+
+ vlan->tci = f->vlan.vid;
+ vlan->tpid = f->vlan.tpid;
+
+ i++;
+ f->state = IAVF_VLAN_IS_NEW;
+ }
+ }
+
+ if (!more)
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_VLAN_FILTER;
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_VLAN_V2,
+ (u8 *)vvfl_v2, len);
+ kfree(vvfl_v2);
+ }
+}
+
+/**
+ * iavf_del_vlans
+ * @adapter: adapter structure
+ *
+ * Request that the PF remove one or more VLAN filters from our VSI.
+ **/
+void iavf_del_vlans(struct iavf_adapter *adapter)
+{
+ struct iavf_vlan_filter *f, *ftmp;
+ int len, i = 0, count = 0;
+ bool more = false;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot remove VLANs, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) {
+ /* since VLAN capabilities are not allowed, we dont want to send
+ * a VLAN delete request because it will most likely fail and
+ * create unnecessary errors/noise, so just free the VLAN
+ * filters marked for removal to enable bailing out before
+ * sending a virtchnl message
+ */
+ if (f->state == IAVF_VLAN_REMOVE &&
+ !VLAN_FILTERING_ALLOWED(adapter)) {
+ list_del(&f->list);
+ kfree(f);
+ adapter->num_vlan_filters--;
+ } else if (f->state == IAVF_VLAN_DISABLE &&
+ !VLAN_FILTERING_ALLOWED(adapter)) {
+ f->state = IAVF_VLAN_INACTIVE;
+ } else if (f->state == IAVF_VLAN_REMOVE ||
+ f->state == IAVF_VLAN_DISABLE) {
+ count++;
+ }
+ }
+ if (!count || !VLAN_FILTERING_ALLOWED(adapter)) {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_VLAN_FILTER;
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return;
+ }
+
+ if (VLAN_ALLOWED(adapter)) {
+ struct virtchnl_vlan_filter_list *vvfl;
+
+ adapter->current_op = VIRTCHNL_OP_DEL_VLAN;
+
+ len = virtchnl_struct_size(vvfl, vlan_id, count);
+ if (len > IAVF_MAX_AQ_BUF_SIZE) {
+ dev_warn(&adapter->pdev->dev, "Too many delete VLAN changes in one request\n");
+ while (len > IAVF_MAX_AQ_BUF_SIZE)
+ len = virtchnl_struct_size(vvfl, vlan_id,
+ --count);
+ more = true;
+ }
+ vvfl = kzalloc(len, GFP_ATOMIC);
+ if (!vvfl) {
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return;
+ }
+
+ vvfl->vsi_id = adapter->vsi_res->vsi_id;
+ vvfl->num_elements = count;
+ list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) {
+ if (f->state == IAVF_VLAN_DISABLE) {
+ vvfl->vlan_id[i] = f->vlan.vid;
+ f->state = IAVF_VLAN_INACTIVE;
+ i++;
+ if (i == count)
+ break;
+ } else if (f->state == IAVF_VLAN_REMOVE) {
+ vvfl->vlan_id[i] = f->vlan.vid;
+ list_del(&f->list);
+ kfree(f);
+ adapter->num_vlan_filters--;
+ i++;
+ if (i == count)
+ break;
+ }
+ }
+
+ if (!more)
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_VLAN_FILTER;
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_VLAN, (u8 *)vvfl, len);
+ kfree(vvfl);
+ } else {
+ struct virtchnl_vlan_filter_list_v2 *vvfl_v2;
+
+ adapter->current_op = VIRTCHNL_OP_DEL_VLAN_V2;
+
+ len = virtchnl_struct_size(vvfl_v2, filters, count);
+ if (len > IAVF_MAX_AQ_BUF_SIZE) {
+ dev_warn(&adapter->pdev->dev, "Too many add VLAN changes in one request\n");
+ while (len > IAVF_MAX_AQ_BUF_SIZE)
+ len = virtchnl_struct_size(vvfl_v2, filters,
+ --count);
+ more = true;
+ }
+
+ vvfl_v2 = kzalloc(len, GFP_ATOMIC);
+ if (!vvfl_v2) {
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ return;
+ }
+
+ vvfl_v2->vport_id = adapter->vsi_res->vsi_id;
+ vvfl_v2->num_elements = count;
+ list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) {
+ if (f->state == IAVF_VLAN_DISABLE ||
+ f->state == IAVF_VLAN_REMOVE) {
+ struct virtchnl_vlan_supported_caps *filtering_support =
+ &adapter->vlan_v2_caps.filtering.filtering_support;
+ struct virtchnl_vlan *vlan;
+
+ /* give priority over outer if it's enabled */
+ if (filtering_support->outer)
+ vlan = &vvfl_v2->filters[i].outer;
+ else
+ vlan = &vvfl_v2->filters[i].inner;
+
+ vlan->tci = f->vlan.vid;
+ vlan->tpid = f->vlan.tpid;
+
+ if (f->state == IAVF_VLAN_DISABLE) {
+ f->state = IAVF_VLAN_INACTIVE;
+ } else {
+ list_del(&f->list);
+ kfree(f);
+ adapter->num_vlan_filters--;
+ }
+ i++;
+ if (i == count)
+ break;
+ }
+ }
+
+ if (!more)
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_VLAN_FILTER;
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_VLAN_V2,
+ (u8 *)vvfl_v2, len);
+ kfree(vvfl_v2);
+ }
+}
+
+/**
+ * iavf_set_promiscuous
+ * @adapter: adapter structure
+ *
+ * Request that the PF enable promiscuous mode for our VSI.
+ **/
+void iavf_set_promiscuous(struct iavf_adapter *adapter)
+{
+ struct net_device *netdev = adapter->netdev;
+ struct virtchnl_promisc_info vpi;
+ unsigned int flags;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot set promiscuous mode, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+
+ /* prevent changes to promiscuous flags */
+ spin_lock_bh(&adapter->current_netdev_promisc_flags_lock);
+
+ /* sanity check to prevent duplicate AQ calls */
+ if (!iavf_promiscuous_mode_changed(adapter)) {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE;
+ dev_dbg(&adapter->pdev->dev, "No change in promiscuous mode\n");
+ /* allow changes to promiscuous flags */
+ spin_unlock_bh(&adapter->current_netdev_promisc_flags_lock);
+ return;
+ }
+
+ /* there are 2 bits, but only 3 states */
+ if (!(netdev->flags & IFF_PROMISC) &&
+ netdev->flags & IFF_ALLMULTI) {
+ /* State 1 - only multicast promiscuous mode enabled
+ * - !IFF_PROMISC && IFF_ALLMULTI
+ */
+ flags = FLAG_VF_MULTICAST_PROMISC;
+ adapter->current_netdev_promisc_flags |= IFF_ALLMULTI;
+ adapter->current_netdev_promisc_flags &= ~IFF_PROMISC;
+ dev_info(&adapter->pdev->dev, "Entering multicast promiscuous mode\n");
+ } else if (!(netdev->flags & IFF_PROMISC) &&
+ !(netdev->flags & IFF_ALLMULTI)) {
+ /* State 2 - unicast/multicast promiscuous mode disabled
+ * - !IFF_PROMISC && !IFF_ALLMULTI
+ */
+ flags = 0;
+ adapter->current_netdev_promisc_flags &=
+ ~(IFF_PROMISC | IFF_ALLMULTI);
+ dev_info(&adapter->pdev->dev, "Leaving promiscuous mode\n");
+ } else {
+ /* State 3 - unicast/multicast promiscuous mode enabled
+ * - IFF_PROMISC && IFF_ALLMULTI
+ * - IFF_PROMISC && !IFF_ALLMULTI
+ */
+ flags = FLAG_VF_UNICAST_PROMISC | FLAG_VF_MULTICAST_PROMISC;
+ adapter->current_netdev_promisc_flags |= IFF_PROMISC;
+ if (netdev->flags & IFF_ALLMULTI)
+ adapter->current_netdev_promisc_flags |= IFF_ALLMULTI;
+ else
+ adapter->current_netdev_promisc_flags &= ~IFF_ALLMULTI;
+
+ dev_info(&adapter->pdev->dev, "Entering promiscuous mode\n");
+ }
+
+ adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE;
+
+ /* allow changes to promiscuous flags */
+ spin_unlock_bh(&adapter->current_netdev_promisc_flags_lock);
+
+ adapter->current_op = VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
+ vpi.vsi_id = adapter->vsi_res->vsi_id;
+ vpi.flags = flags;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE,
+ (u8 *)&vpi, sizeof(vpi));
+}
+
+/**
+ * iavf_request_stats
+ * @adapter: adapter structure
+ *
+ * Request VSI statistics from PF.
+ **/
+void iavf_request_stats(struct iavf_adapter *adapter)
+{
+ struct virtchnl_queue_select vqs;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* no error message, this isn't crucial */
+ return;
+ }
+
+ adapter->aq_required &= ~IAVF_FLAG_AQ_REQUEST_STATS;
+ adapter->current_op = VIRTCHNL_OP_GET_STATS;
+ vqs.vsi_id = adapter->vsi_res->vsi_id;
+ /* queue maps are ignored for this message - only the vsi is used */
+ if (iavf_send_pf_msg(adapter, VIRTCHNL_OP_GET_STATS, (u8 *)&vqs,
+ sizeof(vqs)))
+ /* if the request failed, don't lock out others */
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+}
+
+/**
+ * iavf_get_hena
+ * @adapter: adapter structure
+ *
+ * Request hash enable capabilities from PF
+ **/
+void iavf_get_hena(struct iavf_adapter *adapter)
+{
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot get RSS hash capabilities, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_GET_RSS_HENA_CAPS;
+ adapter->aq_required &= ~IAVF_FLAG_AQ_GET_HENA;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_GET_RSS_HENA_CAPS, NULL, 0);
+}
+
+/**
+ * iavf_set_hena
+ * @adapter: adapter structure
+ *
+ * Request the PF to set our RSS hash capabilities
+ **/
+void iavf_set_hena(struct iavf_adapter *adapter)
+{
+ struct virtchnl_rss_hena vrh;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot set RSS hash enable, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ vrh.hena = adapter->hena;
+ adapter->current_op = VIRTCHNL_OP_SET_RSS_HENA;
+ adapter->aq_required &= ~IAVF_FLAG_AQ_SET_HENA;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_SET_RSS_HENA, (u8 *)&vrh,
+ sizeof(vrh));
+}
+
+/**
+ * iavf_set_rss_key
+ * @adapter: adapter structure
+ *
+ * Request the PF to set our RSS hash key
+ **/
+void iavf_set_rss_key(struct iavf_adapter *adapter)
+{
+ struct virtchnl_rss_key *vrk;
+ int len;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot set RSS key, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ len = virtchnl_struct_size(vrk, key, adapter->rss_key_size);
+ vrk = kzalloc(len, GFP_KERNEL);
+ if (!vrk)
+ return;
+ vrk->vsi_id = adapter->vsi.id;
+ vrk->key_len = adapter->rss_key_size;
+ memcpy(vrk->key, adapter->rss_key, adapter->rss_key_size);
+
+ adapter->current_op = VIRTCHNL_OP_CONFIG_RSS_KEY;
+ adapter->aq_required &= ~IAVF_FLAG_AQ_SET_RSS_KEY;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_CONFIG_RSS_KEY, (u8 *)vrk, len);
+ kfree(vrk);
+}
+
+/**
+ * iavf_set_rss_lut
+ * @adapter: adapter structure
+ *
+ * Request the PF to set our RSS lookup table
+ **/
+void iavf_set_rss_lut(struct iavf_adapter *adapter)
+{
+ struct virtchnl_rss_lut *vrl;
+ int len;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot set RSS LUT, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ len = virtchnl_struct_size(vrl, lut, adapter->rss_lut_size);
+ vrl = kzalloc(len, GFP_KERNEL);
+ if (!vrl)
+ return;
+ vrl->vsi_id = adapter->vsi.id;
+ vrl->lut_entries = adapter->rss_lut_size;
+ memcpy(vrl->lut, adapter->rss_lut, adapter->rss_lut_size);
+ adapter->current_op = VIRTCHNL_OP_CONFIG_RSS_LUT;
+ adapter->aq_required &= ~IAVF_FLAG_AQ_SET_RSS_LUT;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_CONFIG_RSS_LUT, (u8 *)vrl, len);
+ kfree(vrl);
+}
+
+/**
+ * iavf_enable_vlan_stripping
+ * @adapter: adapter structure
+ *
+ * Request VLAN header stripping to be enabled
+ **/
+void iavf_enable_vlan_stripping(struct iavf_adapter *adapter)
+{
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot enable stripping, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_ENABLE_VLAN_STRIPPING;
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_ENABLE_VLAN_STRIPPING, NULL, 0);
+}
+
+/**
+ * iavf_disable_vlan_stripping
+ * @adapter: adapter structure
+ *
+ * Request VLAN header stripping to be disabled
+ **/
+void iavf_disable_vlan_stripping(struct iavf_adapter *adapter)
+{
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot disable stripping, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_DISABLE_VLAN_STRIPPING;
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_DISABLE_VLAN_STRIPPING, NULL, 0);
+}
+
+/**
+ * iavf_tpid_to_vc_ethertype - transform from VLAN TPID to virtchnl ethertype
+ * @tpid: VLAN TPID (i.e. 0x8100, 0x88a8, etc.)
+ */
+static u32 iavf_tpid_to_vc_ethertype(u16 tpid)
+{
+ switch (tpid) {
+ case ETH_P_8021Q:
+ return VIRTCHNL_VLAN_ETHERTYPE_8100;
+ case ETH_P_8021AD:
+ return VIRTCHNL_VLAN_ETHERTYPE_88A8;
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_set_vc_offload_ethertype - set virtchnl ethertype for offload message
+ * @adapter: adapter structure
+ * @msg: message structure used for updating offloads over virtchnl to update
+ * @tpid: VLAN TPID (i.e. 0x8100, 0x88a8, etc.)
+ * @offload_op: opcode used to determine which support structure to check
+ */
+static int
+iavf_set_vc_offload_ethertype(struct iavf_adapter *adapter,
+ struct virtchnl_vlan_setting *msg, u16 tpid,
+ enum virtchnl_ops offload_op)
+{
+ struct virtchnl_vlan_supported_caps *offload_support;
+ u16 vc_ethertype = iavf_tpid_to_vc_ethertype(tpid);
+
+ /* reference the correct offload support structure */
+ switch (offload_op) {
+ case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2:
+ case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2:
+ offload_support =
+ &adapter->vlan_v2_caps.offloads.stripping_support;
+ break;
+ case VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2:
+ case VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2:
+ offload_support =
+ &adapter->vlan_v2_caps.offloads.insertion_support;
+ break;
+ default:
+ dev_err(&adapter->pdev->dev, "Invalid opcode %d for setting virtchnl ethertype to enable/disable VLAN offloads\n",
+ offload_op);
+ return -EINVAL;
+ }
+
+ /* make sure ethertype is supported */
+ if (offload_support->outer & vc_ethertype &&
+ offload_support->outer & VIRTCHNL_VLAN_TOGGLE) {
+ msg->outer_ethertype_setting = vc_ethertype;
+ } else if (offload_support->inner & vc_ethertype &&
+ offload_support->inner & VIRTCHNL_VLAN_TOGGLE) {
+ msg->inner_ethertype_setting = vc_ethertype;
+ } else {
+ dev_dbg(&adapter->pdev->dev, "opcode %d unsupported for VLAN TPID 0x%04x\n",
+ offload_op, tpid);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/**
+ * iavf_clear_offload_v2_aq_required - clear AQ required bit for offload request
+ * @adapter: adapter structure
+ * @tpid: VLAN TPID
+ * @offload_op: opcode used to determine which AQ required bit to clear
+ */
+static void
+iavf_clear_offload_v2_aq_required(struct iavf_adapter *adapter, u16 tpid,
+ enum virtchnl_ops offload_op)
+{
+ switch (offload_op) {
+ case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2:
+ if (tpid == ETH_P_8021Q)
+ adapter->aq_required &=
+ ~IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING;
+ else if (tpid == ETH_P_8021AD)
+ adapter->aq_required &=
+ ~IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING;
+ break;
+ case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2:
+ if (tpid == ETH_P_8021Q)
+ adapter->aq_required &=
+ ~IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING;
+ else if (tpid == ETH_P_8021AD)
+ adapter->aq_required &=
+ ~IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING;
+ break;
+ case VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2:
+ if (tpid == ETH_P_8021Q)
+ adapter->aq_required &=
+ ~IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION;
+ else if (tpid == ETH_P_8021AD)
+ adapter->aq_required &=
+ ~IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION;
+ break;
+ case VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2:
+ if (tpid == ETH_P_8021Q)
+ adapter->aq_required &=
+ ~IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION;
+ else if (tpid == ETH_P_8021AD)
+ adapter->aq_required &=
+ ~IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION;
+ break;
+ default:
+ dev_err(&adapter->pdev->dev, "Unsupported opcode %d specified for clearing aq_required bits for VIRTCHNL_VF_OFFLOAD_VLAN_V2 offload request\n",
+ offload_op);
+ }
+}
+
+/**
+ * iavf_send_vlan_offload_v2 - send offload enable/disable over virtchnl
+ * @adapter: adapter structure
+ * @tpid: VLAN TPID used for the command (i.e. 0x8100 or 0x88a8)
+ * @offload_op: offload_op used to make the request over virtchnl
+ */
+static void
+iavf_send_vlan_offload_v2(struct iavf_adapter *adapter, u16 tpid,
+ enum virtchnl_ops offload_op)
+{
+ struct virtchnl_vlan_setting *msg;
+ int len = sizeof(*msg);
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot send %d, command %d pending\n",
+ offload_op, adapter->current_op);
+ return;
+ }
+
+ adapter->current_op = offload_op;
+
+ msg = kzalloc(len, GFP_KERNEL);
+ if (!msg)
+ return;
+
+ msg->vport_id = adapter->vsi_res->vsi_id;
+
+ /* always clear to prevent unsupported and endless requests */
+ iavf_clear_offload_v2_aq_required(adapter, tpid, offload_op);
+
+ /* only send valid offload requests */
+ if (!iavf_set_vc_offload_ethertype(adapter, msg, tpid, offload_op))
+ iavf_send_pf_msg(adapter, offload_op, (u8 *)msg, len);
+ else
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+
+ kfree(msg);
+}
+
+/**
+ * iavf_enable_vlan_stripping_v2 - enable VLAN stripping
+ * @adapter: adapter structure
+ * @tpid: VLAN TPID used to enable VLAN stripping
+ */
+void iavf_enable_vlan_stripping_v2(struct iavf_adapter *adapter, u16 tpid)
+{
+ iavf_send_vlan_offload_v2(adapter, tpid,
+ VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2);
+}
+
+/**
+ * iavf_disable_vlan_stripping_v2 - disable VLAN stripping
+ * @adapter: adapter structure
+ * @tpid: VLAN TPID used to disable VLAN stripping
+ */
+void iavf_disable_vlan_stripping_v2(struct iavf_adapter *adapter, u16 tpid)
+{
+ iavf_send_vlan_offload_v2(adapter, tpid,
+ VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2);
+}
+
+/**
+ * iavf_enable_vlan_insertion_v2 - enable VLAN insertion
+ * @adapter: adapter structure
+ * @tpid: VLAN TPID used to enable VLAN insertion
+ */
+void iavf_enable_vlan_insertion_v2(struct iavf_adapter *adapter, u16 tpid)
+{
+ iavf_send_vlan_offload_v2(adapter, tpid,
+ VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2);
+}
+
+/**
+ * iavf_disable_vlan_insertion_v2 - disable VLAN insertion
+ * @adapter: adapter structure
+ * @tpid: VLAN TPID used to disable VLAN insertion
+ */
+void iavf_disable_vlan_insertion_v2(struct iavf_adapter *adapter, u16 tpid)
+{
+ iavf_send_vlan_offload_v2(adapter, tpid,
+ VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2);
+}
+
+#define IAVF_MAX_SPEED_STRLEN 13
+
+/**
+ * iavf_print_link_message - print link up or down
+ * @adapter: adapter structure
+ *
+ * Log a message telling the world of our wonderous link status
+ */
+static void iavf_print_link_message(struct iavf_adapter *adapter)
+{
+ struct net_device *netdev = adapter->netdev;
+ int link_speed_mbps;
+ char *speed;
+
+ if (!adapter->link_up) {
+ netdev_info(netdev, "NIC Link is Down\n");
+ return;
+ }
+
+ speed = kzalloc(IAVF_MAX_SPEED_STRLEN, GFP_KERNEL);
+ if (!speed)
+ return;
+
+ if (ADV_LINK_SUPPORT(adapter)) {
+ link_speed_mbps = adapter->link_speed_mbps;
+ goto print_link_msg;
+ }
+
+ switch (adapter->link_speed) {
+ case VIRTCHNL_LINK_SPEED_40GB:
+ link_speed_mbps = SPEED_40000;
+ break;
+ case VIRTCHNL_LINK_SPEED_25GB:
+ link_speed_mbps = SPEED_25000;
+ break;
+ case VIRTCHNL_LINK_SPEED_20GB:
+ link_speed_mbps = SPEED_20000;
+ break;
+ case VIRTCHNL_LINK_SPEED_10GB:
+ link_speed_mbps = SPEED_10000;
+ break;
+ case VIRTCHNL_LINK_SPEED_5GB:
+ link_speed_mbps = SPEED_5000;
+ break;
+ case VIRTCHNL_LINK_SPEED_2_5GB:
+ link_speed_mbps = SPEED_2500;
+ break;
+ case VIRTCHNL_LINK_SPEED_1GB:
+ link_speed_mbps = SPEED_1000;
+ break;
+ case VIRTCHNL_LINK_SPEED_100MB:
+ link_speed_mbps = SPEED_100;
+ break;
+ default:
+ link_speed_mbps = SPEED_UNKNOWN;
+ break;
+ }
+
+print_link_msg:
+ if (link_speed_mbps > SPEED_1000) {
+ if (link_speed_mbps == SPEED_2500)
+ snprintf(speed, IAVF_MAX_SPEED_STRLEN, "2.5 Gbps");
+ else
+ /* convert to Gbps inline */
+ snprintf(speed, IAVF_MAX_SPEED_STRLEN, "%d %s",
+ link_speed_mbps / 1000, "Gbps");
+ } else if (link_speed_mbps == SPEED_UNKNOWN) {
+ snprintf(speed, IAVF_MAX_SPEED_STRLEN, "%s", "Unknown Mbps");
+ } else {
+ snprintf(speed, IAVF_MAX_SPEED_STRLEN, "%d %s",
+ link_speed_mbps, "Mbps");
+ }
+
+ netdev_info(netdev, "NIC Link is Up Speed is %s Full Duplex\n", speed);
+ kfree(speed);
+}
+
+/**
+ * iavf_get_vpe_link_status
+ * @adapter: adapter structure
+ * @vpe: virtchnl_pf_event structure
+ *
+ * Helper function for determining the link status
+ **/
+static bool
+iavf_get_vpe_link_status(struct iavf_adapter *adapter,
+ struct virtchnl_pf_event *vpe)
+{
+ if (ADV_LINK_SUPPORT(adapter))
+ return vpe->event_data.link_event_adv.link_status;
+ else
+ return vpe->event_data.link_event.link_status;
+}
+
+/**
+ * iavf_set_adapter_link_speed_from_vpe
+ * @adapter: adapter structure for which we are setting the link speed
+ * @vpe: virtchnl_pf_event structure that contains the link speed we are setting
+ *
+ * Helper function for setting iavf_adapter link speed
+ **/
+static void
+iavf_set_adapter_link_speed_from_vpe(struct iavf_adapter *adapter,
+ struct virtchnl_pf_event *vpe)
+{
+ if (ADV_LINK_SUPPORT(adapter))
+ adapter->link_speed_mbps =
+ vpe->event_data.link_event_adv.link_speed;
+ else
+ adapter->link_speed = vpe->event_data.link_event.link_speed;
+}
+
+/**
+ * iavf_enable_channels
+ * @adapter: adapter structure
+ *
+ * Request that the PF enable channels as specified by
+ * the user via tc tool.
+ **/
+void iavf_enable_channels(struct iavf_adapter *adapter)
+{
+ struct virtchnl_tc_info *vti = NULL;
+ size_t len;
+ int i;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot configure mqprio, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+
+ len = virtchnl_struct_size(vti, list, adapter->num_tc);
+ vti = kzalloc(len, GFP_KERNEL);
+ if (!vti)
+ return;
+ vti->num_tc = adapter->num_tc;
+ for (i = 0; i < vti->num_tc; i++) {
+ vti->list[i].count = adapter->ch_config.ch_info[i].count;
+ vti->list[i].offset = adapter->ch_config.ch_info[i].offset;
+ vti->list[i].pad = 0;
+ vti->list[i].max_tx_rate =
+ adapter->ch_config.ch_info[i].max_tx_rate;
+ }
+
+ adapter->ch_config.state = __IAVF_TC_RUNNING;
+ adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED;
+ adapter->current_op = VIRTCHNL_OP_ENABLE_CHANNELS;
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ENABLE_CHANNELS;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_ENABLE_CHANNELS, (u8 *)vti, len);
+ kfree(vti);
+}
+
+/**
+ * iavf_disable_channels
+ * @adapter: adapter structure
+ *
+ * Request that the PF disable channels that are configured
+ **/
+void iavf_disable_channels(struct iavf_adapter *adapter)
+{
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot configure mqprio, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+
+ adapter->ch_config.state = __IAVF_TC_INVALID;
+ adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED;
+ adapter->current_op = VIRTCHNL_OP_DISABLE_CHANNELS;
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DISABLE_CHANNELS;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_DISABLE_CHANNELS, NULL, 0);
+}
+
+/**
+ * iavf_print_cloud_filter
+ * @adapter: adapter structure
+ * @f: cloud filter to print
+ *
+ * Print the cloud filter
+ **/
+static void iavf_print_cloud_filter(struct iavf_adapter *adapter,
+ struct virtchnl_filter *f)
+{
+ switch (f->flow_type) {
+ case VIRTCHNL_TCP_V4_FLOW:
+ dev_info(&adapter->pdev->dev, "dst_mac: %pM src_mac: %pM vlan_id: %hu dst_ip: %pI4 src_ip %pI4 dst_port %hu src_port %hu\n",
+ &f->data.tcp_spec.dst_mac,
+ &f->data.tcp_spec.src_mac,
+ ntohs(f->data.tcp_spec.vlan_id),
+ &f->data.tcp_spec.dst_ip[0],
+ &f->data.tcp_spec.src_ip[0],
+ ntohs(f->data.tcp_spec.dst_port),
+ ntohs(f->data.tcp_spec.src_port));
+ break;
+ case VIRTCHNL_TCP_V6_FLOW:
+ dev_info(&adapter->pdev->dev, "dst_mac: %pM src_mac: %pM vlan_id: %hu dst_ip: %pI6 src_ip %pI6 dst_port %hu src_port %hu\n",
+ &f->data.tcp_spec.dst_mac,
+ &f->data.tcp_spec.src_mac,
+ ntohs(f->data.tcp_spec.vlan_id),
+ &f->data.tcp_spec.dst_ip,
+ &f->data.tcp_spec.src_ip,
+ ntohs(f->data.tcp_spec.dst_port),
+ ntohs(f->data.tcp_spec.src_port));
+ break;
+ }
+}
+
+/**
+ * iavf_add_cloud_filter
+ * @adapter: adapter structure
+ *
+ * Request that the PF add cloud filters as specified
+ * by the user via tc tool.
+ **/
+void iavf_add_cloud_filter(struct iavf_adapter *adapter)
+{
+ struct iavf_cloud_filter *cf;
+ struct virtchnl_filter *f;
+ int len = 0, count = 0;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot add cloud filter, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
+ if (cf->add) {
+ count++;
+ break;
+ }
+ }
+ if (!count) {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_ADD_CLOUD_FILTER;
+
+ len = sizeof(struct virtchnl_filter);
+ f = kzalloc(len, GFP_KERNEL);
+ if (!f)
+ return;
+
+ list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
+ if (cf->add) {
+ memcpy(f, &cf->f, sizeof(struct virtchnl_filter));
+ cf->add = false;
+ cf->state = __IAVF_CF_ADD_PENDING;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_CLOUD_FILTER,
+ (u8 *)f, len);
+ }
+ }
+ kfree(f);
+}
+
+/**
+ * iavf_del_cloud_filter
+ * @adapter: adapter structure
+ *
+ * Request that the PF delete cloud filters as specified
+ * by the user via tc tool.
+ **/
+void iavf_del_cloud_filter(struct iavf_adapter *adapter)
+{
+ struct iavf_cloud_filter *cf, *cftmp;
+ struct virtchnl_filter *f;
+ int len = 0, count = 0;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot remove cloud filter, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+ list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
+ if (cf->del) {
+ count++;
+ break;
+ }
+ }
+ if (!count) {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_DEL_CLOUD_FILTER;
+
+ len = sizeof(struct virtchnl_filter);
+ f = kzalloc(len, GFP_KERNEL);
+ if (!f)
+ return;
+
+ list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
+ if (cf->del) {
+ memcpy(f, &cf->f, sizeof(struct virtchnl_filter));
+ cf->del = false;
+ cf->state = __IAVF_CF_DEL_PENDING;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_CLOUD_FILTER,
+ (u8 *)f, len);
+ }
+ }
+ kfree(f);
+}
+
+/**
+ * iavf_add_fdir_filter
+ * @adapter: the VF adapter structure
+ *
+ * Request that the PF add Flow Director filters as specified
+ * by the user via ethtool.
+ **/
+void iavf_add_fdir_filter(struct iavf_adapter *adapter)
+{
+ struct iavf_fdir_fltr *fdir;
+ struct virtchnl_fdir_add *f;
+ bool process_fltr = false;
+ int len;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot add Flow Director filter, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+
+ len = sizeof(struct virtchnl_fdir_add);
+ f = kzalloc(len, GFP_KERNEL);
+ if (!f)
+ return;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry(fdir, &adapter->fdir_list_head, list) {
+ if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST) {
+ process_fltr = true;
+ fdir->state = IAVF_FDIR_FLTR_ADD_PENDING;
+ memcpy(f, &fdir->vc_add_msg, len);
+ break;
+ }
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+
+ if (!process_fltr) {
+ /* prevent iavf_add_fdir_filter() from being called when there
+ * are no filters to add
+ */
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_FDIR_FILTER;
+ kfree(f);
+ return;
+ }
+ adapter->current_op = VIRTCHNL_OP_ADD_FDIR_FILTER;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_FDIR_FILTER, (u8 *)f, len);
+ kfree(f);
+}
+
+/**
+ * iavf_del_fdir_filter
+ * @adapter: the VF adapter structure
+ *
+ * Request that the PF delete Flow Director filters as specified
+ * by the user via ethtool.
+ **/
+void iavf_del_fdir_filter(struct iavf_adapter *adapter)
+{
+ struct virtchnl_fdir_del f = {};
+ struct iavf_fdir_fltr *fdir;
+ bool process_fltr = false;
+ int len;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot remove Flow Director filter, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+
+ len = sizeof(struct virtchnl_fdir_del);
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry(fdir, &adapter->fdir_list_head, list) {
+ if (fdir->state == IAVF_FDIR_FLTR_DEL_REQUEST) {
+ process_fltr = true;
+ f.vsi_id = fdir->vc_add_msg.vsi_id;
+ f.flow_id = fdir->flow_id;
+ fdir->state = IAVF_FDIR_FLTR_DEL_PENDING;
+ break;
+ } else if (fdir->state == IAVF_FDIR_FLTR_DIS_REQUEST) {
+ process_fltr = true;
+ f.vsi_id = fdir->vc_add_msg.vsi_id;
+ f.flow_id = fdir->flow_id;
+ fdir->state = IAVF_FDIR_FLTR_DIS_PENDING;
+ break;
+ }
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+
+ if (!process_fltr) {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_FDIR_FILTER;
+ return;
+ }
+
+ adapter->current_op = VIRTCHNL_OP_DEL_FDIR_FILTER;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_FDIR_FILTER, (u8 *)&f, len);
+}
+
+/**
+ * iavf_add_adv_rss_cfg
+ * @adapter: the VF adapter structure
+ *
+ * Request that the PF add RSS configuration as specified
+ * by the user via ethtool.
+ **/
+void iavf_add_adv_rss_cfg(struct iavf_adapter *adapter)
+{
+ struct virtchnl_rss_cfg *rss_cfg;
+ struct iavf_adv_rss *rss;
+ bool process_rss = false;
+ int len;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot add RSS configuration, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+
+ len = sizeof(struct virtchnl_rss_cfg);
+ rss_cfg = kzalloc(len, GFP_KERNEL);
+ if (!rss_cfg)
+ return;
+
+ spin_lock_bh(&adapter->adv_rss_lock);
+ list_for_each_entry(rss, &adapter->adv_rss_list_head, list) {
+ if (rss->state == IAVF_ADV_RSS_ADD_REQUEST) {
+ process_rss = true;
+ rss->state = IAVF_ADV_RSS_ADD_PENDING;
+ memcpy(rss_cfg, &rss->cfg_msg, len);
+ iavf_print_adv_rss_cfg(adapter, rss,
+ "Input set change for",
+ "is pending");
+ break;
+ }
+ }
+ spin_unlock_bh(&adapter->adv_rss_lock);
+
+ if (process_rss) {
+ adapter->current_op = VIRTCHNL_OP_ADD_RSS_CFG;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_RSS_CFG,
+ (u8 *)rss_cfg, len);
+ } else {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_ADV_RSS_CFG;
+ }
+
+ kfree(rss_cfg);
+}
+
+/**
+ * iavf_del_adv_rss_cfg
+ * @adapter: the VF adapter structure
+ *
+ * Request that the PF delete RSS configuration as specified
+ * by the user via ethtool.
+ **/
+void iavf_del_adv_rss_cfg(struct iavf_adapter *adapter)
+{
+ struct virtchnl_rss_cfg *rss_cfg;
+ struct iavf_adv_rss *rss;
+ bool process_rss = false;
+ int len;
+
+ if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+ /* bail because we already have a command pending */
+ dev_err(&adapter->pdev->dev, "Cannot remove RSS configuration, command %d pending\n",
+ adapter->current_op);
+ return;
+ }
+
+ len = sizeof(struct virtchnl_rss_cfg);
+ rss_cfg = kzalloc(len, GFP_KERNEL);
+ if (!rss_cfg)
+ return;
+
+ spin_lock_bh(&adapter->adv_rss_lock);
+ list_for_each_entry(rss, &adapter->adv_rss_list_head, list) {
+ if (rss->state == IAVF_ADV_RSS_DEL_REQUEST) {
+ process_rss = true;
+ rss->state = IAVF_ADV_RSS_DEL_PENDING;
+ memcpy(rss_cfg, &rss->cfg_msg, len);
+ break;
+ }
+ }
+ spin_unlock_bh(&adapter->adv_rss_lock);
+
+ if (process_rss) {
+ adapter->current_op = VIRTCHNL_OP_DEL_RSS_CFG;
+ iavf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_RSS_CFG,
+ (u8 *)rss_cfg, len);
+ } else {
+ adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
+ }
+
+ kfree(rss_cfg);
+}
+
+/**
+ * iavf_request_reset
+ * @adapter: adapter structure
+ *
+ * Request that the PF reset this VF. No response is expected.
+ **/
+int iavf_request_reset(struct iavf_adapter *adapter)
+{
+ int err;
+ /* Don't check CURRENT_OP - this is always higher priority */
+ err = iavf_send_pf_msg(adapter, VIRTCHNL_OP_RESET_VF, NULL, 0);
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+ return err;
+}
+
+/**
+ * iavf_netdev_features_vlan_strip_set - update vlan strip status
+ * @netdev: ptr to netdev being adjusted
+ * @enable: enable or disable vlan strip
+ *
+ * Helper function to change vlan strip status in netdev->features.
+ */
+static void iavf_netdev_features_vlan_strip_set(struct net_device *netdev,
+ const bool enable)
+{
+ if (enable)
+ netdev->features |= NETIF_F_HW_VLAN_CTAG_RX;
+ else
+ netdev->features &= ~NETIF_F_HW_VLAN_CTAG_RX;
+}
+
+/**
+ * iavf_activate_fdir_filters - Reactivate all FDIR filters after a reset
+ * @adapter: private adapter structure
+ *
+ * Called after a reset to re-add all FDIR filters and delete some of them
+ * if they were pending to be deleted.
+ */
+static void iavf_activate_fdir_filters(struct iavf_adapter *adapter)
+{
+ struct iavf_fdir_fltr *f, *ftmp;
+ bool add_filters = false;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry_safe(f, ftmp, &adapter->fdir_list_head, list) {
+ if (f->state == IAVF_FDIR_FLTR_ADD_REQUEST ||
+ f->state == IAVF_FDIR_FLTR_ADD_PENDING ||
+ f->state == IAVF_FDIR_FLTR_ACTIVE) {
+ /* All filters and requests have been removed in PF,
+ * restore them
+ */
+ f->state = IAVF_FDIR_FLTR_ADD_REQUEST;
+ add_filters = true;
+ } else if (f->state == IAVF_FDIR_FLTR_DIS_REQUEST ||
+ f->state == IAVF_FDIR_FLTR_DIS_PENDING) {
+ /* Link down state, leave filters as inactive */
+ f->state = IAVF_FDIR_FLTR_INACTIVE;
+ } else if (f->state == IAVF_FDIR_FLTR_DEL_REQUEST ||
+ f->state == IAVF_FDIR_FLTR_DEL_PENDING) {
+ /* Delete filters that were pending to be deleted, the
+ * list on PF is already cleared after a reset
+ */
+ list_del(&f->list);
+ kfree(f);
+ adapter->fdir_active_fltr--;
+ }
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+
+ if (add_filters)
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_FDIR_FILTER;
+}
+
+/**
+ * iavf_virtchnl_completion
+ * @adapter: adapter structure
+ * @v_opcode: opcode sent by PF
+ * @v_retval: retval sent by PF
+ * @msg: message sent by PF
+ * @msglen: message length
+ *
+ * Asynchronous completion function for admin queue messages. Rather than busy
+ * wait, we fire off our requests and assume that no errors will be returned.
+ * This function handles the reply messages.
+ **/
+void iavf_virtchnl_completion(struct iavf_adapter *adapter,
+ enum virtchnl_ops v_opcode,
+ enum iavf_status v_retval, u8 *msg, u16 msglen)
+{
+ struct net_device *netdev = adapter->netdev;
+
+ if (v_opcode == VIRTCHNL_OP_EVENT) {
+ struct virtchnl_pf_event *vpe =
+ (struct virtchnl_pf_event *)msg;
+ bool link_up = iavf_get_vpe_link_status(adapter, vpe);
+
+ switch (vpe->event) {
+ case VIRTCHNL_EVENT_LINK_CHANGE:
+ iavf_set_adapter_link_speed_from_vpe(adapter, vpe);
+
+ /* we've already got the right link status, bail */
+ if (adapter->link_up == link_up)
+ break;
+
+ if (link_up) {
+ /* If we get link up message and start queues
+ * before our queues are configured it will
+ * trigger a TX hang. In that case, just ignore
+ * the link status message,we'll get another one
+ * after we enable queues and actually prepared
+ * to send traffic.
+ */
+ if (adapter->state != __IAVF_RUNNING)
+ break;
+
+ /* For ADq enabled VF, we reconfigure VSIs and
+ * re-allocate queues. Hence wait till all
+ * queues are enabled.
+ */
+ if (adapter->flags &
+ IAVF_FLAG_QUEUES_DISABLED)
+ break;
+ }
+
+ adapter->link_up = link_up;
+ if (link_up) {
+ netif_tx_start_all_queues(netdev);
+ netif_carrier_on(netdev);
+ } else {
+ netif_tx_stop_all_queues(netdev);
+ netif_carrier_off(netdev);
+ }
+ iavf_print_link_message(adapter);
+ break;
+ case VIRTCHNL_EVENT_RESET_IMPENDING:
+ dev_info(&adapter->pdev->dev, "Reset indication received from the PF\n");
+ if (!(adapter->flags & IAVF_FLAG_RESET_PENDING)) {
+ dev_info(&adapter->pdev->dev, "Scheduling reset task\n");
+ iavf_schedule_reset(adapter, IAVF_FLAG_RESET_PENDING);
+ }
+ break;
+ default:
+ dev_err(&adapter->pdev->dev, "Unknown event %d from PF\n",
+ vpe->event);
+ break;
+ }
+ return;
+ }
+ if (v_retval) {
+ switch (v_opcode) {
+ case VIRTCHNL_OP_ADD_VLAN:
+ dev_err(&adapter->pdev->dev, "Failed to add VLAN filter, error %s\n",
+ iavf_stat_str(&adapter->hw, v_retval));
+ break;
+ case VIRTCHNL_OP_ADD_ETH_ADDR:
+ dev_err(&adapter->pdev->dev, "Failed to add MAC filter, error %s\n",
+ iavf_stat_str(&adapter->hw, v_retval));
+ iavf_mac_add_reject(adapter);
+ /* restore administratively set MAC address */
+ ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
+ wake_up(&adapter->vc_waitqueue);
+ break;
+ case VIRTCHNL_OP_DEL_VLAN:
+ dev_err(&adapter->pdev->dev, "Failed to delete VLAN filter, error %s\n",
+ iavf_stat_str(&adapter->hw, v_retval));
+ break;
+ case VIRTCHNL_OP_DEL_ETH_ADDR:
+ dev_err(&adapter->pdev->dev, "Failed to delete MAC filter, error %s\n",
+ iavf_stat_str(&adapter->hw, v_retval));
+ break;
+ case VIRTCHNL_OP_ENABLE_CHANNELS:
+ dev_err(&adapter->pdev->dev, "Failed to configure queue channels, error %s\n",
+ iavf_stat_str(&adapter->hw, v_retval));
+ adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
+ adapter->ch_config.state = __IAVF_TC_INVALID;
+ netdev_reset_tc(netdev);
+ netif_tx_start_all_queues(netdev);
+ break;
+ case VIRTCHNL_OP_DISABLE_CHANNELS:
+ dev_err(&adapter->pdev->dev, "Failed to disable queue channels, error %s\n",
+ iavf_stat_str(&adapter->hw, v_retval));
+ adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
+ adapter->ch_config.state = __IAVF_TC_RUNNING;
+ netif_tx_start_all_queues(netdev);
+ break;
+ case VIRTCHNL_OP_ADD_CLOUD_FILTER: {
+ struct iavf_cloud_filter *cf, *cftmp;
+
+ list_for_each_entry_safe(cf, cftmp,
+ &adapter->cloud_filter_list,
+ list) {
+ if (cf->state == __IAVF_CF_ADD_PENDING) {
+ cf->state = __IAVF_CF_INVALID;
+ dev_info(&adapter->pdev->dev, "Failed to add cloud filter, error %s\n",
+ iavf_stat_str(&adapter->hw,
+ v_retval));
+ iavf_print_cloud_filter(adapter,
+ &cf->f);
+ list_del(&cf->list);
+ kfree(cf);
+ adapter->num_cloud_filters--;
+ }
+ }
+ }
+ break;
+ case VIRTCHNL_OP_DEL_CLOUD_FILTER: {
+ struct iavf_cloud_filter *cf;
+
+ list_for_each_entry(cf, &adapter->cloud_filter_list,
+ list) {
+ if (cf->state == __IAVF_CF_DEL_PENDING) {
+ cf->state = __IAVF_CF_ACTIVE;
+ dev_info(&adapter->pdev->dev, "Failed to del cloud filter, error %s\n",
+ iavf_stat_str(&adapter->hw,
+ v_retval));
+ iavf_print_cloud_filter(adapter,
+ &cf->f);
+ }
+ }
+ }
+ break;
+ case VIRTCHNL_OP_ADD_FDIR_FILTER: {
+ struct iavf_fdir_fltr *fdir, *fdir_tmp;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry_safe(fdir, fdir_tmp,
+ &adapter->fdir_list_head,
+ list) {
+ if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING) {
+ dev_info(&adapter->pdev->dev, "Failed to add Flow Director filter, error %s\n",
+ iavf_stat_str(&adapter->hw,
+ v_retval));
+ iavf_print_fdir_fltr(adapter, fdir);
+ if (msglen)
+ dev_err(&adapter->pdev->dev,
+ "%s\n", msg);
+ list_del(&fdir->list);
+ kfree(fdir);
+ adapter->fdir_active_fltr--;
+ }
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+ }
+ break;
+ case VIRTCHNL_OP_DEL_FDIR_FILTER: {
+ struct iavf_fdir_fltr *fdir;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry(fdir, &adapter->fdir_list_head,
+ list) {
+ if (fdir->state == IAVF_FDIR_FLTR_DEL_PENDING ||
+ fdir->state == IAVF_FDIR_FLTR_DIS_PENDING) {
+ fdir->state = IAVF_FDIR_FLTR_ACTIVE;
+ dev_info(&adapter->pdev->dev, "Failed to del Flow Director filter, error %s\n",
+ iavf_stat_str(&adapter->hw,
+ v_retval));
+ iavf_print_fdir_fltr(adapter, fdir);
+ }
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+ }
+ break;
+ case VIRTCHNL_OP_ADD_RSS_CFG: {
+ struct iavf_adv_rss *rss, *rss_tmp;
+
+ spin_lock_bh(&adapter->adv_rss_lock);
+ list_for_each_entry_safe(rss, rss_tmp,
+ &adapter->adv_rss_list_head,
+ list) {
+ if (rss->state == IAVF_ADV_RSS_ADD_PENDING) {
+ iavf_print_adv_rss_cfg(adapter, rss,
+ "Failed to change the input set for",
+ NULL);
+ list_del(&rss->list);
+ kfree(rss);
+ }
+ }
+ spin_unlock_bh(&adapter->adv_rss_lock);
+ }
+ break;
+ case VIRTCHNL_OP_DEL_RSS_CFG: {
+ struct iavf_adv_rss *rss;
+
+ spin_lock_bh(&adapter->adv_rss_lock);
+ list_for_each_entry(rss, &adapter->adv_rss_list_head,
+ list) {
+ if (rss->state == IAVF_ADV_RSS_DEL_PENDING) {
+ rss->state = IAVF_ADV_RSS_ACTIVE;
+ dev_err(&adapter->pdev->dev, "Failed to delete RSS configuration, error %s\n",
+ iavf_stat_str(&adapter->hw,
+ v_retval));
+ }
+ }
+ spin_unlock_bh(&adapter->adv_rss_lock);
+ }
+ break;
+ case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING:
+ dev_warn(&adapter->pdev->dev, "Changing VLAN Stripping is not allowed when Port VLAN is configured\n");
+ /* Vlan stripping could not be enabled by ethtool.
+ * Disable it in netdev->features.
+ */
+ iavf_netdev_features_vlan_strip_set(netdev, false);
+ break;
+ case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING:
+ dev_warn(&adapter->pdev->dev, "Changing VLAN Stripping is not allowed when Port VLAN is configured\n");
+ /* Vlan stripping could not be disabled by ethtool.
+ * Enable it in netdev->features.
+ */
+ iavf_netdev_features_vlan_strip_set(netdev, true);
+ break;
+ case VIRTCHNL_OP_ADD_VLAN_V2:
+ iavf_vlan_add_reject(adapter);
+ dev_warn(&adapter->pdev->dev, "Failed to add VLAN filter, error %s\n",
+ iavf_stat_str(&adapter->hw, v_retval));
+ break;
+ default:
+ dev_err(&adapter->pdev->dev, "PF returned error %d (%s) to our request %d\n",
+ v_retval, iavf_stat_str(&adapter->hw, v_retval),
+ v_opcode);
+ }
+ }
+ switch (v_opcode) {
+ case VIRTCHNL_OP_ADD_ETH_ADDR:
+ if (!v_retval)
+ iavf_mac_add_ok(adapter);
+ if (!ether_addr_equal(netdev->dev_addr, adapter->hw.mac.addr))
+ if (!ether_addr_equal(netdev->dev_addr,
+ adapter->hw.mac.addr)) {
+ netif_addr_lock_bh(netdev);
+ eth_hw_addr_set(netdev, adapter->hw.mac.addr);
+ netif_addr_unlock_bh(netdev);
+ }
+ wake_up(&adapter->vc_waitqueue);
+ break;
+ case VIRTCHNL_OP_GET_STATS: {
+ struct iavf_eth_stats *stats =
+ (struct iavf_eth_stats *)msg;
+ netdev->stats.rx_packets = stats->rx_unicast +
+ stats->rx_multicast +
+ stats->rx_broadcast;
+ netdev->stats.tx_packets = stats->tx_unicast +
+ stats->tx_multicast +
+ stats->tx_broadcast;
+ netdev->stats.rx_bytes = stats->rx_bytes;
+ netdev->stats.tx_bytes = stats->tx_bytes;
+ netdev->stats.tx_errors = stats->tx_errors;
+ netdev->stats.rx_dropped = stats->rx_discards;
+ netdev->stats.tx_dropped = stats->tx_discards;
+ adapter->current_stats = *stats;
+ }
+ break;
+ case VIRTCHNL_OP_GET_VF_RESOURCES: {
+ u16 len = IAVF_VIRTCHNL_VF_RESOURCE_SIZE;
+
+ memcpy(adapter->vf_res, msg, min(msglen, len));
+ iavf_validate_num_queues(adapter);
+ iavf_vf_parse_hw_config(&adapter->hw, adapter->vf_res);
+ if (is_zero_ether_addr(adapter->hw.mac.addr)) {
+ /* restore current mac address */
+ ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
+ } else {
+ netif_addr_lock_bh(netdev);
+ /* refresh current mac address if changed */
+ ether_addr_copy(netdev->perm_addr,
+ adapter->hw.mac.addr);
+ netif_addr_unlock_bh(netdev);
+ }
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+ iavf_add_filter(adapter, adapter->hw.mac.addr);
+
+ if (VLAN_ALLOWED(adapter)) {
+ if (!list_empty(&adapter->vlan_filter_list)) {
+ struct iavf_vlan_filter *vlf;
+
+ /* re-add all VLAN filters over virtchnl */
+ list_for_each_entry(vlf,
+ &adapter->vlan_filter_list,
+ list)
+ vlf->state = IAVF_VLAN_ADD;
+
+ adapter->aq_required |=
+ IAVF_FLAG_AQ_ADD_VLAN_FILTER;
+ }
+ }
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ iavf_activate_fdir_filters(adapter);
+
+ iavf_parse_vf_resource_msg(adapter);
+
+ /* negotiated VIRTCHNL_VF_OFFLOAD_VLAN_V2, so wait for the
+ * response to VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS to finish
+ * configuration
+ */
+ if (VLAN_V2_ALLOWED(adapter))
+ break;
+ /* fallthrough and finish config if VIRTCHNL_VF_OFFLOAD_VLAN_V2
+ * wasn't successfully negotiated with the PF
+ */
+ }
+ fallthrough;
+ case VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS: {
+ struct iavf_mac_filter *f;
+ bool was_mac_changed;
+ u64 aq_required = 0;
+
+ if (v_opcode == VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS)
+ memcpy(&adapter->vlan_v2_caps, msg,
+ min_t(u16, msglen,
+ sizeof(adapter->vlan_v2_caps)));
+
+ iavf_process_config(adapter);
+ adapter->flags |= IAVF_FLAG_SETUP_NETDEV_FEATURES;
+ iavf_schedule_finish_config(adapter);
+
+ iavf_set_queue_vlan_tag_loc(adapter);
+
+ was_mac_changed = !ether_addr_equal(netdev->dev_addr,
+ adapter->hw.mac.addr);
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+
+ /* re-add all MAC filters */
+ list_for_each_entry(f, &adapter->mac_filter_list, list) {
+ if (was_mac_changed &&
+ ether_addr_equal(netdev->dev_addr, f->macaddr))
+ ether_addr_copy(f->macaddr,
+ adapter->hw.mac.addr);
+
+ f->is_new_mac = true;
+ f->add = true;
+ f->add_handled = false;
+ f->remove = false;
+ }
+
+ /* re-add all VLAN filters */
+ if (VLAN_FILTERING_ALLOWED(adapter)) {
+ struct iavf_vlan_filter *vlf;
+
+ if (!list_empty(&adapter->vlan_filter_list)) {
+ list_for_each_entry(vlf,
+ &adapter->vlan_filter_list,
+ list)
+ vlf->state = IAVF_VLAN_ADD;
+
+ aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
+ }
+ }
+
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
+ netif_addr_lock_bh(netdev);
+ eth_hw_addr_set(netdev, adapter->hw.mac.addr);
+ netif_addr_unlock_bh(netdev);
+
+ adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER |
+ aq_required;
+ }
+ break;
+ case VIRTCHNL_OP_ENABLE_QUEUES:
+ /* enable transmits */
+ iavf_irq_enable(adapter, true);
+ wake_up(&adapter->reset_waitqueue);
+ adapter->flags &= ~IAVF_FLAG_QUEUES_DISABLED;
+ break;
+ case VIRTCHNL_OP_DISABLE_QUEUES:
+ iavf_free_all_tx_resources(adapter);
+ iavf_free_all_rx_resources(adapter);
+ if (adapter->state == __IAVF_DOWN_PENDING) {
+ iavf_change_state(adapter, __IAVF_DOWN);
+ wake_up(&adapter->down_waitqueue);
+ }
+ break;
+ case VIRTCHNL_OP_VERSION:
+ case VIRTCHNL_OP_CONFIG_IRQ_MAP:
+ /* Don't display an error if we get these out of sequence.
+ * If the firmware needed to get kicked, we'll get these and
+ * it's no problem.
+ */
+ if (v_opcode != adapter->current_op)
+ return;
+ break;
+ case VIRTCHNL_OP_RDMA:
+ /* Gobble zero-length replies from the PF. They indicate that
+ * a previous message was received OK, and the client doesn't
+ * care about that.
+ */
+ if (msglen && CLIENT_ENABLED(adapter))
+ iavf_notify_client_message(&adapter->vsi, msg, msglen);
+ break;
+
+ case VIRTCHNL_OP_CONFIG_RDMA_IRQ_MAP:
+ adapter->client_pending &=
+ ~(BIT(VIRTCHNL_OP_CONFIG_RDMA_IRQ_MAP));
+ break;
+ case VIRTCHNL_OP_GET_RSS_HENA_CAPS: {
+ struct virtchnl_rss_hena *vrh = (struct virtchnl_rss_hena *)msg;
+
+ if (msglen == sizeof(*vrh))
+ adapter->hena = vrh->hena;
+ else
+ dev_warn(&adapter->pdev->dev,
+ "Invalid message %d from PF\n", v_opcode);
+ }
+ break;
+ case VIRTCHNL_OP_REQUEST_QUEUES: {
+ struct virtchnl_vf_res_request *vfres =
+ (struct virtchnl_vf_res_request *)msg;
+
+ if (vfres->num_queue_pairs != adapter->num_req_queues) {
+ dev_info(&adapter->pdev->dev,
+ "Requested %d queues, PF can support %d\n",
+ adapter->num_req_queues,
+ vfres->num_queue_pairs);
+ adapter->num_req_queues = 0;
+ adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
+ }
+ }
+ break;
+ case VIRTCHNL_OP_ADD_CLOUD_FILTER: {
+ struct iavf_cloud_filter *cf;
+
+ list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
+ if (cf->state == __IAVF_CF_ADD_PENDING)
+ cf->state = __IAVF_CF_ACTIVE;
+ }
+ }
+ break;
+ case VIRTCHNL_OP_DEL_CLOUD_FILTER: {
+ struct iavf_cloud_filter *cf, *cftmp;
+
+ list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
+ list) {
+ if (cf->state == __IAVF_CF_DEL_PENDING) {
+ cf->state = __IAVF_CF_INVALID;
+ list_del(&cf->list);
+ kfree(cf);
+ adapter->num_cloud_filters--;
+ }
+ }
+ }
+ break;
+ case VIRTCHNL_OP_ADD_FDIR_FILTER: {
+ struct virtchnl_fdir_add *add_fltr = (struct virtchnl_fdir_add *)msg;
+ struct iavf_fdir_fltr *fdir, *fdir_tmp;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry_safe(fdir, fdir_tmp,
+ &adapter->fdir_list_head,
+ list) {
+ if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING) {
+ if (add_fltr->status == VIRTCHNL_FDIR_SUCCESS) {
+ dev_info(&adapter->pdev->dev, "Flow Director filter with location %u is added\n",
+ fdir->loc);
+ fdir->state = IAVF_FDIR_FLTR_ACTIVE;
+ fdir->flow_id = add_fltr->flow_id;
+ } else {
+ dev_info(&adapter->pdev->dev, "Failed to add Flow Director filter with status: %d\n",
+ add_fltr->status);
+ iavf_print_fdir_fltr(adapter, fdir);
+ list_del(&fdir->list);
+ kfree(fdir);
+ adapter->fdir_active_fltr--;
+ }
+ }
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+ }
+ break;
+ case VIRTCHNL_OP_DEL_FDIR_FILTER: {
+ struct virtchnl_fdir_del *del_fltr = (struct virtchnl_fdir_del *)msg;
+ struct iavf_fdir_fltr *fdir, *fdir_tmp;
+
+ spin_lock_bh(&adapter->fdir_fltr_lock);
+ list_for_each_entry_safe(fdir, fdir_tmp, &adapter->fdir_list_head,
+ list) {
+ if (fdir->state == IAVF_FDIR_FLTR_DEL_PENDING) {
+ if (del_fltr->status == VIRTCHNL_FDIR_SUCCESS ||
+ del_fltr->status ==
+ VIRTCHNL_FDIR_FAILURE_RULE_NONEXIST) {
+ dev_info(&adapter->pdev->dev, "Flow Director filter with location %u is deleted\n",
+ fdir->loc);
+ list_del(&fdir->list);
+ kfree(fdir);
+ adapter->fdir_active_fltr--;
+ } else {
+ fdir->state = IAVF_FDIR_FLTR_ACTIVE;
+ dev_info(&adapter->pdev->dev, "Failed to delete Flow Director filter with status: %d\n",
+ del_fltr->status);
+ iavf_print_fdir_fltr(adapter, fdir);
+ }
+ } else if (fdir->state == IAVF_FDIR_FLTR_DIS_PENDING) {
+ if (del_fltr->status == VIRTCHNL_FDIR_SUCCESS ||
+ del_fltr->status ==
+ VIRTCHNL_FDIR_FAILURE_RULE_NONEXIST) {
+ fdir->state = IAVF_FDIR_FLTR_INACTIVE;
+ } else {
+ fdir->state = IAVF_FDIR_FLTR_ACTIVE;
+ dev_info(&adapter->pdev->dev, "Failed to disable Flow Director filter with status: %d\n",
+ del_fltr->status);
+ iavf_print_fdir_fltr(adapter, fdir);
+ }
+ }
+ }
+ spin_unlock_bh(&adapter->fdir_fltr_lock);
+ }
+ break;
+ case VIRTCHNL_OP_ADD_RSS_CFG: {
+ struct iavf_adv_rss *rss;
+
+ spin_lock_bh(&adapter->adv_rss_lock);
+ list_for_each_entry(rss, &adapter->adv_rss_list_head, list) {
+ if (rss->state == IAVF_ADV_RSS_ADD_PENDING) {
+ iavf_print_adv_rss_cfg(adapter, rss,
+ "Input set change for",
+ "successful");
+ rss->state = IAVF_ADV_RSS_ACTIVE;
+ }
+ }
+ spin_unlock_bh(&adapter->adv_rss_lock);
+ }
+ break;
+ case VIRTCHNL_OP_DEL_RSS_CFG: {
+ struct iavf_adv_rss *rss, *rss_tmp;
+
+ spin_lock_bh(&adapter->adv_rss_lock);
+ list_for_each_entry_safe(rss, rss_tmp,
+ &adapter->adv_rss_list_head, list) {
+ if (rss->state == IAVF_ADV_RSS_DEL_PENDING) {
+ list_del(&rss->list);
+ kfree(rss);
+ }
+ }
+ spin_unlock_bh(&adapter->adv_rss_lock);
+ }
+ break;
+ case VIRTCHNL_OP_ADD_VLAN_V2: {
+ struct iavf_vlan_filter *f;
+
+ spin_lock_bh(&adapter->mac_vlan_list_lock);
+ list_for_each_entry(f, &adapter->vlan_filter_list, list) {
+ if (f->state == IAVF_VLAN_IS_NEW)
+ f->state = IAVF_VLAN_ACTIVE;
+ }
+ spin_unlock_bh(&adapter->mac_vlan_list_lock);
+ }
+ break;
+ case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING:
+ /* PF enabled vlan strip on this VF.
+ * Update netdev->features if needed to be in sync with ethtool.
+ */
+ if (!v_retval)
+ iavf_netdev_features_vlan_strip_set(netdev, true);
+ break;
+ case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING:
+ /* PF disabled vlan strip on this VF.
+ * Update netdev->features if needed to be in sync with ethtool.
+ */
+ if (!v_retval)
+ iavf_netdev_features_vlan_strip_set(netdev, false);
+ break;
+ default:
+ if (adapter->current_op && (v_opcode != adapter->current_op))
+ dev_warn(&adapter->pdev->dev, "Expected response %d from PF, received %d\n",
+ adapter->current_op, v_opcode);
+ break;
+ } /* switch v_opcode */
+ adapter->current_op = VIRTCHNL_OP_UNKNOWN;
+}