diff options
Diffstat (limited to 'drivers/net/hyperv')
| -rw-r--r-- | drivers/net/hyperv/Kconfig | 8 | ||||
| -rw-r--r-- | drivers/net/hyperv/Makefile | 4 | ||||
| -rw-r--r-- | drivers/net/hyperv/hyperv_net.h | 1803 | ||||
| -rw-r--r-- | drivers/net/hyperv/netvsc.c | 1856 | ||||
| -rw-r--r-- | drivers/net/hyperv/netvsc_bpf.c | 292 | ||||
| -rw-r--r-- | drivers/net/hyperv/netvsc_drv.c | 2823 | ||||
| -rw-r--r-- | drivers/net/hyperv/netvsc_trace.c | 7 | ||||
| -rw-r--r-- | drivers/net/hyperv/netvsc_trace.h | 182 | ||||
| -rw-r--r-- | drivers/net/hyperv/rndis_filter.c | 1620 |
9 files changed, 8595 insertions, 0 deletions
diff --git a/drivers/net/hyperv/Kconfig b/drivers/net/hyperv/Kconfig new file mode 100644 index 000000000..c8cbd85ad --- /dev/null +++ b/drivers/net/hyperv/Kconfig @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +config HYPERV_NET + tristate "Microsoft Hyper-V virtual network driver" + depends on HYPERV + select UCS2_STRING + select NLS + help + Select this option to enable the Hyper-V virtual network driver. diff --git a/drivers/net/hyperv/Makefile b/drivers/net/hyperv/Makefile new file mode 100644 index 000000000..0db7ccaec --- /dev/null +++ b/drivers/net/hyperv/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_HYPERV_NET) += hv_netvsc.o + +hv_netvsc-y := netvsc_drv.o netvsc.o rndis_filter.o netvsc_trace.o netvsc_bpf.o diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h new file mode 100644 index 000000000..dd5919ec4 --- /dev/null +++ b/drivers/net/hyperv/hyperv_net.h @@ -0,0 +1,1803 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * + * Copyright (c) 2011, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + * K. Y. Srinivasan <kys@microsoft.com> + */ + +#ifndef _HYPERV_NET_H +#define _HYPERV_NET_H + +#include <linux/list.h> +#include <linux/hyperv.h> +#include <linux/rndis.h> +#include <linux/jhash.h> + +/* RSS related */ +#define OID_GEN_RECEIVE_SCALE_CAPABILITIES 0x00010203 /* query only */ +#define OID_GEN_RECEIVE_SCALE_PARAMETERS 0x00010204 /* query and set */ + +#define NDIS_OBJECT_TYPE_RSS_CAPABILITIES 0x88 +#define NDIS_OBJECT_TYPE_RSS_PARAMETERS 0x89 +#define NDIS_OBJECT_TYPE_OFFLOAD 0xa7 + +#define NDIS_RECEIVE_SCALE_CAPABILITIES_REVISION_2 2 +#define NDIS_RECEIVE_SCALE_PARAMETERS_REVISION_2 2 + +struct ndis_obj_header { + u8 type; + u8 rev; + u16 size; +} __packed; + +/* ndis_recv_scale_cap/cap_flag */ +#define NDIS_RSS_CAPS_MESSAGE_SIGNALED_INTERRUPTS 0x01000000 +#define NDIS_RSS_CAPS_CLASSIFICATION_AT_ISR 0x02000000 +#define NDIS_RSS_CAPS_CLASSIFICATION_AT_DPC 0x04000000 +#define NDIS_RSS_CAPS_USING_MSI_X 0x08000000 +#define NDIS_RSS_CAPS_RSS_AVAILABLE_ON_PORTS 0x10000000 +#define NDIS_RSS_CAPS_SUPPORTS_MSI_X 0x20000000 +#define NDIS_RSS_CAPS_HASH_TYPE_TCP_IPV4 0x00000100 +#define NDIS_RSS_CAPS_HASH_TYPE_TCP_IPV6 0x00000200 +#define NDIS_RSS_CAPS_HASH_TYPE_TCP_IPV6_EX 0x00000400 + +struct ndis_recv_scale_cap { /* NDIS_RECEIVE_SCALE_CAPABILITIES */ + struct ndis_obj_header hdr; + u32 cap_flag; + u32 num_int_msg; + u32 num_recv_que; + u16 num_indirect_tabent; +} __packed; + + +/* ndis_recv_scale_param flags */ +#define NDIS_RSS_PARAM_FLAG_BASE_CPU_UNCHANGED 0x0001 +#define NDIS_RSS_PARAM_FLAG_HASH_INFO_UNCHANGED 0x0002 +#define NDIS_RSS_PARAM_FLAG_ITABLE_UNCHANGED 0x0004 +#define NDIS_RSS_PARAM_FLAG_HASH_KEY_UNCHANGED 0x0008 +#define NDIS_RSS_PARAM_FLAG_DISABLE_RSS 0x0010 + +/* Hash info bits */ +#define NDIS_HASH_FUNC_TOEPLITZ 0x00000001 +#define NDIS_HASH_IPV4 0x00000100 +#define NDIS_HASH_TCP_IPV4 0x00000200 +#define NDIS_HASH_IPV6 0x00000400 +#define NDIS_HASH_IPV6_EX 0x00000800 +#define NDIS_HASH_TCP_IPV6 0x00001000 +#define NDIS_HASH_TCP_IPV6_EX 0x00002000 + +#define NDIS_RSS_INDIRECTION_TABLE_MAX_SIZE_REVISION_2 (128 * 4) +#define NDIS_RSS_HASH_SECRET_KEY_MAX_SIZE_REVISION_2 40 + +#define ITAB_NUM 128 + +struct ndis_recv_scale_param { /* NDIS_RECEIVE_SCALE_PARAMETERS */ + struct ndis_obj_header hdr; + + /* Qualifies the rest of the information */ + u16 flag; + + /* The base CPU number to do receive processing. not used */ + u16 base_cpu_number; + + /* This describes the hash function and type being enabled */ + u32 hashinfo; + + /* The size of indirection table array */ + u16 indirect_tabsize; + + /* The offset of the indirection table from the beginning of this + * structure + */ + u32 indirect_taboffset; + + /* The size of the hash secret key */ + u16 hashkey_size; + + /* The offset of the secret key from the beginning of this structure */ + u32 hashkey_offset; + + u32 processor_masks_offset; + u32 num_processor_masks; + u32 processor_masks_entry_size; +}; + +struct ndis_tcp_ip_checksum_info { + union { + struct { + u32 is_ipv4:1; + u32 is_ipv6:1; + u32 tcp_checksum:1; + u32 udp_checksum:1; + u32 ip_header_checksum:1; + u32 reserved:11; + u32 tcp_header_offset:10; + } transmit; + struct { + u32 tcp_checksum_failed:1; + u32 udp_checksum_failed:1; + u32 ip_checksum_failed:1; + u32 tcp_checksum_succeeded:1; + u32 udp_checksum_succeeded:1; + u32 ip_checksum_succeeded:1; + u32 loopback:1; + u32 tcp_checksum_value_invalid:1; + u32 ip_checksum_value_invalid:1; + } receive; + u32 value; + }; +}; + +struct ndis_pkt_8021q_info { + union { + struct { + u32 pri:3; /* User Priority */ + u32 cfi:1; /* Canonical Format ID */ + u32 vlanid:12; /* VLAN ID */ + u32 reserved:16; + }; + u32 value; + }; +}; + +/* + * Represent netvsc packet which contains 1 RNDIS and 1 ethernet frame + * within the RNDIS + * + * The size of this structure is less than 48 bytes and we can now + * place this structure in the skb->cb field. + */ +struct hv_netvsc_packet { + /* Bookkeeping stuff */ + u8 cp_partial; /* partial copy into send buffer */ + + u8 rmsg_size; /* RNDIS header and PPI size */ + u8 rmsg_pgcnt; /* page count of RNDIS header and PPI */ + u8 page_buf_cnt; + + u16 q_idx; + u16 total_packets; + + u32 total_bytes; + u32 send_buf_index; + u32 total_data_buflen; + struct hv_dma_range *dma_range; +}; + +#define NETVSC_HASH_KEYLEN 40 + +struct netvsc_device_info { + unsigned char mac_adr[ETH_ALEN]; + u32 num_chn; + u32 send_sections; + u32 recv_sections; + u32 send_section_size; + u32 recv_section_size; + + struct bpf_prog *bprog; + + u8 rss_key[NETVSC_HASH_KEYLEN]; +}; + +enum rndis_device_state { + RNDIS_DEV_UNINITIALIZED = 0, + RNDIS_DEV_INITIALIZING, + RNDIS_DEV_INITIALIZED, + RNDIS_DEV_DATAINITIALIZED, +}; + +struct rndis_device { + struct net_device *ndev; + + enum rndis_device_state state; + + atomic_t new_req_id; + + spinlock_t request_lock; + struct list_head req_list; + + struct work_struct mcast_work; + u32 filter; + + bool link_state; /* 0 - link up, 1 - link down */ + + u8 hw_mac_adr[ETH_ALEN]; + u8 rss_key[NETVSC_HASH_KEYLEN]; +}; + + +/* Interface */ +struct rndis_message; +struct ndis_offload_params; +struct netvsc_device; +struct netvsc_channel; +struct net_device_context; + +extern u32 netvsc_ring_bytes; + +struct netvsc_device *netvsc_device_add(struct hv_device *device, + const struct netvsc_device_info *info); +int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx); +void netvsc_device_remove(struct hv_device *device); +int netvsc_send(struct net_device *net, + struct hv_netvsc_packet *packet, + struct rndis_message *rndis_msg, + struct hv_page_buffer *page_buffer, + struct sk_buff *skb, + bool xdp_tx); +void netvsc_linkstatus_callback(struct net_device *net, + struct rndis_message *resp, + void *data, u32 data_buflen); +int netvsc_recv_callback(struct net_device *net, + struct netvsc_device *nvdev, + struct netvsc_channel *nvchan); +void netvsc_channel_cb(void *context); +int netvsc_poll(struct napi_struct *napi, int budget); + +void netvsc_xdp_xmit(struct sk_buff *skb, struct net_device *ndev); +u32 netvsc_run_xdp(struct net_device *ndev, struct netvsc_channel *nvchan, + struct xdp_buff *xdp); +unsigned int netvsc_xdp_fraglen(unsigned int len); +struct bpf_prog *netvsc_xdp_get(struct netvsc_device *nvdev); +int netvsc_xdp_set(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack, + struct netvsc_device *nvdev); +int netvsc_vf_setxdp(struct net_device *vf_netdev, struct bpf_prog *prog); +int netvsc_bpf(struct net_device *dev, struct netdev_bpf *bpf); +int netvsc_ndoxdp_xmit(struct net_device *ndev, int n, + struct xdp_frame **frames, u32 flags); + +int rndis_set_subchannel(struct net_device *ndev, + struct netvsc_device *nvdev, + struct netvsc_device_info *dev_info); +int rndis_filter_open(struct netvsc_device *nvdev); +int rndis_filter_close(struct netvsc_device *nvdev); +struct netvsc_device *rndis_filter_device_add(struct hv_device *dev, + struct netvsc_device_info *info); +void rndis_filter_update(struct netvsc_device *nvdev); +void rndis_filter_device_remove(struct hv_device *dev, + struct netvsc_device *nvdev); +int rndis_filter_set_rss_param(struct rndis_device *rdev, + const u8 *key); +int rndis_filter_set_offload_params(struct net_device *ndev, + struct netvsc_device *nvdev, + struct ndis_offload_params *req_offloads); +int rndis_filter_receive(struct net_device *ndev, + struct netvsc_device *net_dev, + struct netvsc_channel *nvchan, + void *data, u32 buflen); + +int rndis_filter_set_device_mac(struct netvsc_device *ndev, + const char *mac); + +int netvsc_switch_datapath(struct net_device *nv_dev, bool vf); + +#define NVSP_INVALID_PROTOCOL_VERSION ((u32)0xFFFFFFFF) + +#define NVSP_PROTOCOL_VERSION_1 2 +#define NVSP_PROTOCOL_VERSION_2 0x30002 +#define NVSP_PROTOCOL_VERSION_4 0x40000 +#define NVSP_PROTOCOL_VERSION_5 0x50000 +#define NVSP_PROTOCOL_VERSION_6 0x60000 +#define NVSP_PROTOCOL_VERSION_61 0x60001 + +enum { + NVSP_MSG_TYPE_NONE = 0, + + /* Init Messages */ + NVSP_MSG_TYPE_INIT = 1, + NVSP_MSG_TYPE_INIT_COMPLETE = 2, + + NVSP_VERSION_MSG_START = 100, + + /* Version 1 Messages */ + NVSP_MSG1_TYPE_SEND_NDIS_VER = NVSP_VERSION_MSG_START, + + NVSP_MSG1_TYPE_SEND_RECV_BUF, + NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE, + NVSP_MSG1_TYPE_REVOKE_RECV_BUF, + + NVSP_MSG1_TYPE_SEND_SEND_BUF, + NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE, + NVSP_MSG1_TYPE_REVOKE_SEND_BUF, + + NVSP_MSG1_TYPE_SEND_RNDIS_PKT, + NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE, + + /* Version 2 messages */ + NVSP_MSG2_TYPE_SEND_CHIMNEY_DELEGATED_BUF, + NVSP_MSG2_TYPE_SEND_CHIMNEY_DELEGATED_BUF_COMP, + NVSP_MSG2_TYPE_REVOKE_CHIMNEY_DELEGATED_BUF, + + NVSP_MSG2_TYPE_RESUME_CHIMNEY_RX_INDICATION, + + NVSP_MSG2_TYPE_TERMINATE_CHIMNEY, + NVSP_MSG2_TYPE_TERMINATE_CHIMNEY_COMP, + + NVSP_MSG2_TYPE_INDICATE_CHIMNEY_EVENT, + + NVSP_MSG2_TYPE_SEND_CHIMNEY_PKT, + NVSP_MSG2_TYPE_SEND_CHIMNEY_PKT_COMP, + + NVSP_MSG2_TYPE_POST_CHIMNEY_RECV_REQ, + NVSP_MSG2_TYPE_POST_CHIMNEY_RECV_REQ_COMP, + + NVSP_MSG2_TYPE_ALLOC_RXBUF, + NVSP_MSG2_TYPE_ALLOC_RXBUF_COMP, + + NVSP_MSG2_TYPE_FREE_RXBUF, + + NVSP_MSG2_TYPE_SEND_VMQ_RNDIS_PKT, + NVSP_MSG2_TYPE_SEND_VMQ_RNDIS_PKT_COMP, + + NVSP_MSG2_TYPE_SEND_NDIS_CONFIG, + + NVSP_MSG2_TYPE_ALLOC_CHIMNEY_HANDLE, + NVSP_MSG2_TYPE_ALLOC_CHIMNEY_HANDLE_COMP, + + NVSP_MSG2_MAX = NVSP_MSG2_TYPE_ALLOC_CHIMNEY_HANDLE_COMP, + + /* Version 4 messages */ + NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION, + NVSP_MSG4_TYPE_SWITCH_DATA_PATH, + NVSP_MSG4_TYPE_UPLINK_CONNECT_STATE_DEPRECATED, + + NVSP_MSG4_MAX = NVSP_MSG4_TYPE_UPLINK_CONNECT_STATE_DEPRECATED, + + /* Version 5 messages */ + NVSP_MSG5_TYPE_OID_QUERY_EX, + NVSP_MSG5_TYPE_OID_QUERY_EX_COMP, + NVSP_MSG5_TYPE_SUBCHANNEL, + NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE, + + NVSP_MSG5_MAX = NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE, + + /* Version 6 messages */ + NVSP_MSG6_TYPE_PD_API, + NVSP_MSG6_TYPE_PD_POST_BATCH, + + NVSP_MSG6_MAX = NVSP_MSG6_TYPE_PD_POST_BATCH +}; + +enum { + NVSP_STAT_NONE = 0, + NVSP_STAT_SUCCESS, + NVSP_STAT_FAIL, + NVSP_STAT_PROTOCOL_TOO_NEW, + NVSP_STAT_PROTOCOL_TOO_OLD, + NVSP_STAT_INVALID_RNDIS_PKT, + NVSP_STAT_BUSY, + NVSP_STAT_PROTOCOL_UNSUPPORTED, + NVSP_STAT_MAX, +}; + +struct nvsp_message_header { + u32 msg_type; +}; + +/* Init Messages */ + +/* + * This message is used by the VSC to initialize the channel after the channels + * has been opened. This message should never include anything other then + * versioning (i.e. this message will be the same for ever). + */ +struct nvsp_message_init { + u32 min_protocol_ver; + u32 max_protocol_ver; +} __packed; + +/* + * This message is used by the VSP to complete the initialization of the + * channel. This message should never include anything other then versioning + * (i.e. this message will be the same for ever). + */ +struct nvsp_message_init_complete { + u32 negotiated_protocol_ver; + u32 max_mdl_chain_len; + u32 status; +} __packed; + +union nvsp_message_init_uber { + struct nvsp_message_init init; + struct nvsp_message_init_complete init_complete; +} __packed; + +/* Version 1 Messages */ + +/* + * This message is used by the VSC to send the NDIS version to the VSP. The VSP + * can use this information when handling OIDs sent by the VSC. + */ +struct nvsp_1_message_send_ndis_version { + u32 ndis_major_ver; + u32 ndis_minor_ver; +} __packed; + +/* + * This message is used by the VSC to send a receive buffer to the VSP. The VSP + * can then use the receive buffer to send data to the VSC. + */ +struct nvsp_1_message_send_receive_buffer { + u32 gpadl_handle; + u16 id; +} __packed; + +struct nvsp_1_receive_buffer_section { + u32 offset; + u32 sub_alloc_size; + u32 num_sub_allocs; + u32 end_offset; +} __packed; + +/* + * This message is used by the VSP to acknowledge a receive buffer send by the + * VSC. This message must be sent by the VSP before the VSP uses the receive + * buffer. + */ +struct nvsp_1_message_send_receive_buffer_complete { + u32 status; + u32 num_sections; + + /* + * The receive buffer is split into two parts, a large suballocation + * section and a small suballocation section. These sections are then + * suballocated by a certain size. + */ + + /* + * For example, the following break up of the receive buffer has 6 + * large suballocations and 10 small suballocations. + */ + + /* + * | Large Section | | Small Section | + * ------------------------------------------------------------ + * | | | | | | | | | | | | | | | | | | + * | | + * LargeOffset SmallOffset + */ + + struct nvsp_1_receive_buffer_section sections[1]; +} __packed; + +/* + * This message is sent by the VSC to revoke the receive buffer. After the VSP + * completes this transaction, the vsp should never use the receive buffer + * again. + */ +struct nvsp_1_message_revoke_receive_buffer { + u16 id; +}; + +/* + * This message is used by the VSC to send a send buffer to the VSP. The VSC + * can then use the send buffer to send data to the VSP. + */ +struct nvsp_1_message_send_send_buffer { + u32 gpadl_handle; + u16 id; +} __packed; + +/* + * This message is used by the VSP to acknowledge a send buffer sent by the + * VSC. This message must be sent by the VSP before the VSP uses the sent + * buffer. + */ +struct nvsp_1_message_send_send_buffer_complete { + u32 status; + + /* + * The VSC gets to choose the size of the send buffer and the VSP gets + * to choose the sections size of the buffer. This was done to enable + * dynamic reconfigurations when the cost of GPA-direct buffers + * decreases. + */ + u32 section_size; +} __packed; + +/* + * This message is sent by the VSC to revoke the send buffer. After the VSP + * completes this transaction, the vsp should never use the send buffer again. + */ +struct nvsp_1_message_revoke_send_buffer { + u16 id; +}; + +/* + * This message is used by both the VSP and the VSC to send a RNDIS message to + * the opposite channel endpoint. + */ +struct nvsp_1_message_send_rndis_packet { + /* + * This field is specified by RNDIS. They assume there's two different + * channels of communication. However, the Network VSP only has one. + * Therefore, the channel travels with the RNDIS packet. + */ + u32 channel_type; + + /* + * This field is used to send part or all of the data through a send + * buffer. This values specifies an index into the send buffer. If the + * index is 0xFFFFFFFF, then the send buffer is not being used and all + * of the data was sent through other VMBus mechanisms. + */ + u32 send_buf_section_index; + u32 send_buf_section_size; +} __packed; + +/* + * This message is used by both the VSP and the VSC to complete a RNDIS message + * to the opposite channel endpoint. At this point, the initiator of this + * message cannot use any resources associated with the original RNDIS packet. + */ +struct nvsp_1_message_send_rndis_packet_complete { + u32 status; +}; + +union nvsp_1_message_uber { + struct nvsp_1_message_send_ndis_version send_ndis_ver; + + struct nvsp_1_message_send_receive_buffer send_recv_buf; + struct nvsp_1_message_send_receive_buffer_complete + send_recv_buf_complete; + struct nvsp_1_message_revoke_receive_buffer revoke_recv_buf; + + struct nvsp_1_message_send_send_buffer send_send_buf; + struct nvsp_1_message_send_send_buffer_complete send_send_buf_complete; + struct nvsp_1_message_revoke_send_buffer revoke_send_buf; + + struct nvsp_1_message_send_rndis_packet send_rndis_pkt; + struct nvsp_1_message_send_rndis_packet_complete + send_rndis_pkt_complete; +} __packed; + + +/* + * Network VSP protocol version 2 messages: + */ +struct nvsp_2_vsc_capability { + union { + u64 data; + struct { + u64 vmq:1; + u64 chimney:1; + u64 sriov:1; + u64 ieee8021q:1; + u64 correlation_id:1; + u64 teaming:1; + u64 vsubnetid:1; + u64 rsc:1; + }; + }; +} __packed; + +struct nvsp_2_send_ndis_config { + u32 mtu; + u32 reserved; + struct nvsp_2_vsc_capability capability; +} __packed; + +/* Allocate receive buffer */ +struct nvsp_2_alloc_rxbuf { + /* Allocation ID to match the allocation request and response */ + u32 alloc_id; + + /* Length of the VM shared memory receive buffer that needs to + * be allocated + */ + u32 len; +} __packed; + +/* Allocate receive buffer complete */ +struct nvsp_2_alloc_rxbuf_comp { + /* The NDIS_STATUS code for buffer allocation */ + u32 status; + + u32 alloc_id; + + /* GPADL handle for the allocated receive buffer */ + u32 gpadl_handle; + + /* Receive buffer ID */ + u64 recv_buf_id; +} __packed; + +struct nvsp_2_free_rxbuf { + u64 recv_buf_id; +} __packed; + +union nvsp_2_message_uber { + struct nvsp_2_send_ndis_config send_ndis_config; + struct nvsp_2_alloc_rxbuf alloc_rxbuf; + struct nvsp_2_alloc_rxbuf_comp alloc_rxbuf_comp; + struct nvsp_2_free_rxbuf free_rxbuf; +} __packed; + +struct nvsp_4_send_vf_association { + /* 1: allocated, serial number is valid. 0: not allocated */ + u32 allocated; + + /* Serial number of the VF to team with */ + u32 serial; +} __packed; + +enum nvsp_vm_datapath { + NVSP_DATAPATH_SYNTHETIC = 0, + NVSP_DATAPATH_VF, + NVSP_DATAPATH_MAX +}; + +struct nvsp_4_sw_datapath { + u32 active_datapath; /* active data path in VM */ +} __packed; + +union nvsp_4_message_uber { + struct nvsp_4_send_vf_association vf_assoc; + struct nvsp_4_sw_datapath active_dp; +} __packed; + +enum nvsp_subchannel_operation { + NVSP_SUBCHANNEL_NONE = 0, + NVSP_SUBCHANNEL_ALLOCATE, + NVSP_SUBCHANNEL_MAX +}; + +struct nvsp_5_subchannel_request { + u32 op; + u32 num_subchannels; +} __packed; + +struct nvsp_5_subchannel_complete { + u32 status; + u32 num_subchannels; /* Actual number of subchannels allocated */ +} __packed; + +struct nvsp_5_send_indirect_table { + /* The number of entries in the send indirection table */ + u32 count; + + /* The offset of the send indirection table from the beginning of + * struct nvsp_message. + * The send indirection table tells which channel to put the send + * traffic on. Each entry is a channel number. + */ + u32 offset; +} __packed; + +union nvsp_5_message_uber { + struct nvsp_5_subchannel_request subchn_req; + struct nvsp_5_subchannel_complete subchn_comp; + struct nvsp_5_send_indirect_table send_table; +} __packed; + +enum nvsp_6_pd_api_op { + PD_API_OP_CONFIG = 1, + PD_API_OP_SW_DATAPATH, /* Switch Datapath */ + PD_API_OP_OPEN_PROVIDER, + PD_API_OP_CLOSE_PROVIDER, + PD_API_OP_CREATE_QUEUE, + PD_API_OP_FLUSH_QUEUE, + PD_API_OP_FREE_QUEUE, + PD_API_OP_ALLOC_COM_BUF, /* Allocate Common Buffer */ + PD_API_OP_FREE_COM_BUF, /* Free Common Buffer */ + PD_API_OP_MAX +}; + +struct grp_affinity { + u64 mask; + u16 grp; + u16 reserved[3]; +} __packed; + +struct nvsp_6_pd_api_req { + u32 op; + + union { + /* MMIO information is sent from the VM to VSP */ + struct __packed { + u64 mmio_pa; /* MMIO Physical Address */ + u32 mmio_len; + + /* Number of PD queues a VM can support */ + u16 num_subchn; + } config; + + /* Switch Datapath */ + struct __packed { + /* Host Datapath Is PacketDirect */ + u8 host_dpath_is_pd; + + /* Guest PacketDirect Is Enabled */ + u8 guest_pd_enabled; + } sw_dpath; + + /* Open Provider*/ + struct __packed { + u32 prov_id; /* Provider id */ + u32 flag; + } open_prov; + + /* Close Provider */ + struct __packed { + u32 prov_id; + } cls_prov; + + /* Create Queue*/ + struct __packed { + u32 prov_id; + u16 q_id; + u16 q_size; + u8 is_recv_q; + u8 is_rss_q; + u32 recv_data_len; + struct grp_affinity affy; + } cr_q; + + /* Delete Queue*/ + struct __packed { + u32 prov_id; + u16 q_id; + } del_q; + + /* Flush Queue */ + struct __packed { + u32 prov_id; + u16 q_id; + } flush_q; + + /* Allocate Common Buffer */ + struct __packed { + u32 len; + u32 pf_node; /* Preferred Node */ + u16 region_id; + } alloc_com_buf; + + /* Free Common Buffer */ + struct __packed { + u32 len; + u64 pa; /* Physical Address */ + u32 pf_node; /* Preferred Node */ + u16 region_id; + u8 cache_type; + } free_com_buf; + } __packed; +} __packed; + +struct nvsp_6_pd_api_comp { + u32 op; + u32 status; + + union { + struct __packed { + /* actual number of PD queues allocated to the VM */ + u16 num_pd_q; + + /* Num Receive Rss PD Queues */ + u8 num_rss_q; + + u8 is_supported; /* Is supported by VSP */ + u8 is_enabled; /* Is enabled by VSP */ + } config; + + /* Open Provider */ + struct __packed { + u32 prov_id; + } open_prov; + + /* Create Queue */ + struct __packed { + u32 prov_id; + u16 q_id; + u16 q_size; + u32 recv_data_len; + struct grp_affinity affy; + } cr_q; + + /* Allocate Common Buffer */ + struct __packed { + u64 pa; /* Physical Address */ + u32 len; + u32 pf_node; /* Preferred Node */ + u16 region_id; + u8 cache_type; + } alloc_com_buf; + } __packed; +} __packed; + +struct nvsp_6_pd_buf { + u32 region_offset; + u16 region_id; + u16 is_partial:1; + u16 reserved:15; +} __packed; + +struct nvsp_6_pd_batch_msg { + struct nvsp_message_header hdr; + u16 count; + u16 guest2host:1; + u16 is_recv:1; + u16 reserved:14; + struct nvsp_6_pd_buf pd_buf[0]; +} __packed; + +union nvsp_6_message_uber { + struct nvsp_6_pd_api_req pd_req; + struct nvsp_6_pd_api_comp pd_comp; +} __packed; + +union nvsp_all_messages { + union nvsp_message_init_uber init_msg; + union nvsp_1_message_uber v1_msg; + union nvsp_2_message_uber v2_msg; + union nvsp_4_message_uber v4_msg; + union nvsp_5_message_uber v5_msg; + union nvsp_6_message_uber v6_msg; +} __packed; + +/* ALL Messages */ +struct nvsp_message { + struct nvsp_message_header hdr; + union nvsp_all_messages msg; +} __packed; + + +#define NETVSC_MTU 65535 +#define NETVSC_MTU_MIN ETH_MIN_MTU + +/* Max buffer sizes allowed by a host */ +#define NETVSC_RECEIVE_BUFFER_SIZE (1024 * 1024 * 31) /* 31MB */ +#define NETVSC_RECEIVE_BUFFER_SIZE_LEGACY (1024 * 1024 * 15) /* 15MB */ +#define NETVSC_RECEIVE_BUFFER_DEFAULT (1024 * 1024 * 16) + +#define NETVSC_SEND_BUFFER_SIZE (1024 * 1024 * 15) /* 15MB */ +#define NETVSC_SEND_BUFFER_DEFAULT (1024 * 1024) + +#define NETVSC_INVALID_INDEX -1 + +#define NETVSC_SEND_SECTION_SIZE 6144 +#define NETVSC_RECV_SECTION_SIZE 1728 + +/* Default size of TX buf: 1MB, RX buf: 16MB */ +#define NETVSC_MIN_TX_SECTIONS 10 +#define NETVSC_DEFAULT_TX (NETVSC_SEND_BUFFER_DEFAULT \ + / NETVSC_SEND_SECTION_SIZE) +#define NETVSC_MIN_RX_SECTIONS 10 +#define NETVSC_DEFAULT_RX (NETVSC_RECEIVE_BUFFER_DEFAULT \ + / NETVSC_RECV_SECTION_SIZE) + +#define NETVSC_RECEIVE_BUFFER_ID 0xcafe +#define NETVSC_SEND_BUFFER_ID 0 + +#define NETVSC_SUPPORTED_HW_FEATURES (NETIF_F_RXCSUM | NETIF_F_IP_CSUM | \ + NETIF_F_TSO | NETIF_F_IPV6_CSUM | \ + NETIF_F_TSO6 | NETIF_F_LRO | \ + NETIF_F_SG | NETIF_F_RXHASH) + +#define VRSS_SEND_TAB_SIZE 16 /* must be power of 2 */ +#define VRSS_CHANNEL_MAX 64 +#define VRSS_CHANNEL_DEFAULT 8 + +#define RNDIS_MAX_PKT_DEFAULT 8 +#define RNDIS_PKT_ALIGN_DEFAULT 8 + +#define NETVSC_XDP_HDRM 256 + +#define NETVSC_MIN_OUT_MSG_SIZE (sizeof(struct vmpacket_descriptor) + \ + sizeof(struct nvsp_message)) +#define NETVSC_MIN_IN_MSG_SIZE sizeof(struct vmpacket_descriptor) + +/* Estimated requestor size: + * out_ring_size/min_out_msg_size + in_ring_size/min_in_msg_size + */ +static inline u32 netvsc_rqstor_size(unsigned long ringbytes) +{ + return ringbytes / NETVSC_MIN_OUT_MSG_SIZE + + ringbytes / NETVSC_MIN_IN_MSG_SIZE; +} + +/* XFER PAGE packets can specify a maximum of 375 ranges for NDIS >= 6.0 + * and a maximum of 64 ranges for NDIS < 6.0 with no RSC; with RSC, this + * limit is raised to 562 (= NVSP_RSC_MAX). + */ +#define NETVSC_MAX_XFER_PAGE_RANGES NVSP_RSC_MAX +#define NETVSC_XFER_HEADER_SIZE(rng_cnt) \ + (offsetof(struct vmtransfer_page_packet_header, ranges) + \ + (rng_cnt) * sizeof(struct vmtransfer_page_range)) +#define NETVSC_MAX_PKT_SIZE (NETVSC_XFER_HEADER_SIZE(NETVSC_MAX_XFER_PAGE_RANGES) + \ + sizeof(struct nvsp_message) + (sizeof(u32) * VRSS_SEND_TAB_SIZE)) + +struct multi_send_data { + struct sk_buff *skb; /* skb containing the pkt */ + struct hv_netvsc_packet *pkt; /* netvsc pkt pending */ + u32 count; /* counter of batched packets */ +}; + +struct recv_comp_data { + u64 tid; /* transaction id */ + u32 status; +}; + +struct multi_recv_comp { + struct recv_comp_data *slots; + u32 first; /* first data entry */ + u32 next; /* next entry for writing */ +}; + +#define NVSP_RSC_MAX 562 /* Max #RSC frags in a vmbus xfer page pkt */ + +struct nvsc_rsc { + struct ndis_pkt_8021q_info vlan; + struct ndis_tcp_ip_checksum_info csum_info; + u32 hash_info; + u8 ppi_flags; /* valid/present bits for the above PPIs */ + u8 is_last; /* last RNDIS msg in a vmtransfer_page */ + u32 cnt; /* #fragments in an RSC packet */ + u32 pktlen; /* Full packet length */ + void *data[NVSP_RSC_MAX]; + u32 len[NVSP_RSC_MAX]; +}; + +#define NVSC_RSC_VLAN BIT(0) /* valid/present bit for 'vlan' */ +#define NVSC_RSC_CSUM_INFO BIT(1) /* valid/present bit for 'csum_info' */ +#define NVSC_RSC_HASH_INFO BIT(2) /* valid/present bit for 'hash_info' */ + +struct netvsc_stats_tx { + u64 packets; + u64 bytes; + u64 xdp_xmit; + struct u64_stats_sync syncp; +}; + +struct netvsc_stats_rx { + u64 packets; + u64 bytes; + u64 broadcast; + u64 multicast; + u64 xdp_drop; + u64 xdp_redirect; + u64 xdp_tx; + struct u64_stats_sync syncp; +}; + +struct netvsc_ethtool_stats { + unsigned long tx_scattered; + unsigned long tx_no_memory; + unsigned long tx_no_space; + unsigned long tx_too_big; + unsigned long tx_busy; + unsigned long tx_send_full; + unsigned long rx_comp_busy; + unsigned long rx_no_memory; + unsigned long stop_queue; + unsigned long wake_queue; + unsigned long vlan_error; +}; + +struct netvsc_ethtool_pcpu_stats { + u64 rx_packets; + u64 rx_bytes; + u64 tx_packets; + u64 tx_bytes; + u64 vf_rx_packets; + u64 vf_rx_bytes; + u64 vf_tx_packets; + u64 vf_tx_bytes; +}; + +struct netvsc_vf_pcpu_stats { + u64 rx_packets; + u64 rx_bytes; + u64 tx_packets; + u64 tx_bytes; + struct u64_stats_sync syncp; + u32 tx_dropped; +}; + +struct netvsc_reconfig { + struct list_head list; + u32 event; +}; + +/* L4 hash bits for different protocols */ +#define HV_TCP4_L4HASH 1 +#define HV_TCP6_L4HASH 2 +#define HV_UDP4_L4HASH 4 +#define HV_UDP6_L4HASH 8 +#define HV_DEFAULT_L4HASH (HV_TCP4_L4HASH | HV_TCP6_L4HASH | HV_UDP4_L4HASH | \ + HV_UDP6_L4HASH) + +/* The context of the netvsc device */ +struct net_device_context { + /* point back to our device context */ + struct hv_device *device_ctx; + /* netvsc_device */ + struct netvsc_device __rcu *nvdev; + /* list of netvsc net_devices */ + struct list_head list; + /* reconfigure work */ + struct delayed_work dwork; + /* last reconfig time */ + unsigned long last_reconfig; + /* reconfig events */ + struct list_head reconfig_events; + /* list protection */ + spinlock_t lock; + + u32 msg_enable; /* debug level */ + + u32 tx_checksum_mask; + + u32 tx_table[VRSS_SEND_TAB_SIZE]; + + u16 rx_table[ITAB_NUM]; + + /* Ethtool settings */ + u8 duplex; + u32 speed; + u32 l4_hash; /* L4 hash settings */ + struct netvsc_ethtool_stats eth_stats; + + /* State to manage the associated VF interface. */ + struct net_device __rcu *vf_netdev; + struct netvsc_vf_pcpu_stats __percpu *vf_stats; + struct delayed_work vf_takeover; + + /* 1: allocated, serial number is valid. 0: not allocated */ + u32 vf_alloc; + /* Serial number of the VF to team with */ + u32 vf_serial; + /* completion variable to confirm vf association */ + struct completion vf_add; + /* Is the current data path through the VF NIC? */ + bool data_path_is_vf; + + /* Used to temporarily save the config info across hibernation */ + struct netvsc_device_info *saved_netvsc_dev_info; +}; + +/* Azure hosts don't support non-TCP port numbers in hashing for fragmented + * packets. We can use ethtool to change UDP hash level when necessary. + */ +static inline u32 netvsc_get_hash(struct sk_buff *skb, + const struct net_device_context *ndc) +{ + struct flow_keys flow; + u32 hash, pkt_proto = 0; + static u32 hashrnd __read_mostly; + + net_get_random_once(&hashrnd, sizeof(hashrnd)); + + if (!skb_flow_dissect_flow_keys(skb, &flow, 0)) + return 0; + + switch (flow.basic.ip_proto) { + case IPPROTO_TCP: + if (flow.basic.n_proto == htons(ETH_P_IP)) + pkt_proto = HV_TCP4_L4HASH; + else if (flow.basic.n_proto == htons(ETH_P_IPV6)) + pkt_proto = HV_TCP6_L4HASH; + + break; + + case IPPROTO_UDP: + if (flow.basic.n_proto == htons(ETH_P_IP)) + pkt_proto = HV_UDP4_L4HASH; + else if (flow.basic.n_proto == htons(ETH_P_IPV6)) + pkt_proto = HV_UDP6_L4HASH; + + break; + } + + if (pkt_proto & ndc->l4_hash) { + return skb_get_hash(skb); + } else { + if (flow.basic.n_proto == htons(ETH_P_IP)) + hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd); + else if (flow.basic.n_proto == htons(ETH_P_IPV6)) + hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd); + else + return 0; + + __skb_set_sw_hash(skb, hash, false); + } + + return hash; +} + +/* Per channel data */ +struct netvsc_channel { + struct vmbus_channel *channel; + struct netvsc_device *net_device; + void *recv_buf; /* buffer to copy packets out from the receive buffer */ + const struct vmpacket_descriptor *desc; + struct napi_struct napi; + struct multi_send_data msd; + struct multi_recv_comp mrc; + atomic_t queue_sends; + struct nvsc_rsc rsc; + + struct bpf_prog __rcu *bpf_prog; + struct xdp_rxq_info xdp_rxq; + bool xdp_flush; + + struct netvsc_stats_tx tx_stats; + struct netvsc_stats_rx rx_stats; +}; + +/* Per netvsc device */ +struct netvsc_device { + u32 nvsp_version; + + wait_queue_head_t wait_drain; + bool destroy; + bool tx_disable; /* if true, do not wake up queue again */ + + /* Receive buffer allocated by us but manages by NetVSP */ + void *recv_buf; + void *recv_original_buf; + u32 recv_buf_size; /* allocated bytes */ + struct vmbus_gpadl recv_buf_gpadl_handle; + u32 recv_section_cnt; + u32 recv_section_size; + u32 recv_completion_cnt; + + /* Send buffer allocated by us */ + void *send_buf; + void *send_original_buf; + u32 send_buf_size; + struct vmbus_gpadl send_buf_gpadl_handle; + u32 send_section_cnt; + u32 send_section_size; + unsigned long *send_section_map; + + /* Used for NetVSP initialization protocol */ + struct completion channel_init_wait; + struct nvsp_message channel_init_pkt; + + struct nvsp_message revoke_packet; + + u32 max_chn; + u32 num_chn; + + atomic_t open_chn; + struct work_struct subchan_work; + wait_queue_head_t subchan_open; + + struct rndis_device *extension; + + u32 max_pkt; /* max number of pkt in one send, e.g. 8 */ + u32 pkt_align; /* alignment bytes, e.g. 8 */ + + struct netvsc_channel chan_table[VRSS_CHANNEL_MAX]; + + struct rcu_head rcu; +}; + +/* NdisInitialize message */ +struct rndis_initialize_request { + u32 req_id; + u32 major_ver; + u32 minor_ver; + u32 max_xfer_size; +}; + +/* Response to NdisInitialize */ +struct rndis_initialize_complete { + u32 req_id; + u32 status; + u32 major_ver; + u32 minor_ver; + u32 dev_flags; + u32 medium; + u32 max_pkt_per_msg; + u32 max_xfer_size; + u32 pkt_alignment_factor; + u32 af_list_offset; + u32 af_list_size; +}; + +/* Call manager devices only: Information about an address family */ +/* supported by the device is appended to the response to NdisInitialize. */ +struct rndis_co_address_family { + u32 address_family; + u32 major_ver; + u32 minor_ver; +}; + +/* NdisHalt message */ +struct rndis_halt_request { + u32 req_id; +}; + +/* NdisQueryRequest message */ +struct rndis_query_request { + u32 req_id; + u32 oid; + u32 info_buflen; + u32 info_buf_offset; + u32 dev_vc_handle; +}; + +/* Response to NdisQueryRequest */ +struct rndis_query_complete { + u32 req_id; + u32 status; + u32 info_buflen; + u32 info_buf_offset; +}; + +/* NdisSetRequest message */ +struct rndis_set_request { + u32 req_id; + u32 oid; + u32 info_buflen; + u32 info_buf_offset; + u32 dev_vc_handle; + u8 info_buf[]; +}; + +/* Response to NdisSetRequest */ +struct rndis_set_complete { + u32 req_id; + u32 status; +}; + +/* NdisReset message */ +struct rndis_reset_request { + u32 reserved; +}; + +/* Response to NdisReset */ +struct rndis_reset_complete { + u32 status; + u32 addressing_reset; +}; + +/* NdisMIndicateStatus message */ +struct rndis_indicate_status { + u32 status; + u32 status_buflen; + u32 status_buf_offset; +}; + +/* Diagnostic information passed as the status buffer in */ +/* struct rndis_indicate_status messages signifying error conditions. */ +struct rndis_diagnostic_info { + u32 diag_status; + u32 error_offset; +}; + +/* NdisKeepAlive message */ +struct rndis_keepalive_request { + u32 req_id; +}; + +/* Response to NdisKeepAlive */ +struct rndis_keepalive_complete { + u32 req_id; + u32 status; +}; + +/* + * Data message. All Offset fields contain byte offsets from the beginning of + * struct rndis_packet. All Length fields are in bytes. VcHandle is set + * to 0 for connectionless data, otherwise it contains the VC handle. + */ +struct rndis_packet { + u32 data_offset; + u32 data_len; + u32 oob_data_offset; + u32 oob_data_len; + u32 num_oob_data_elements; + u32 per_pkt_info_offset; + u32 per_pkt_info_len; + u32 vc_handle; + u32 reserved; +}; + +/* Optional Out of Band data associated with a Data message. */ +struct rndis_oobd { + u32 size; + u32 type; + u32 class_info_offset; +}; + +/* Packet extension field contents associated with a Data message. */ +struct rndis_per_packet_info { + u32 size; + u32 type:31; + u32 internal:1; + u32 ppi_offset; +}; + +enum ndis_per_pkt_info_type { + TCPIP_CHKSUM_PKTINFO, + IPSEC_PKTINFO, + TCP_LARGESEND_PKTINFO, + CLASSIFICATION_HANDLE_PKTINFO, + NDIS_RESERVED, + SG_LIST_PKTINFO, + IEEE_8021Q_INFO, + ORIGINAL_PKTINFO, + PACKET_CANCEL_ID, + NBL_HASH_VALUE = PACKET_CANCEL_ID, + ORIGINAL_NET_BUFLIST, + CACHED_NET_BUFLIST, + SHORT_PKT_PADINFO, + MAX_PER_PKT_INFO +}; + +enum rndis_per_pkt_info_interal_type { + RNDIS_PKTINFO_ID = 1, + /* Add more members here */ + + RNDIS_PKTINFO_MAX +}; + +#define RNDIS_PKTINFO_SUBALLOC BIT(0) +#define RNDIS_PKTINFO_1ST_FRAG BIT(1) +#define RNDIS_PKTINFO_LAST_FRAG BIT(2) + +#define RNDIS_PKTINFO_ID_V1 1 + +struct rndis_pktinfo_id { + u8 ver; + u8 flag; + u16 pkt_id; +}; + +struct ndis_object_header { + u8 type; + u8 revision; + u16 size; +}; + +#define NDIS_OBJECT_TYPE_DEFAULT 0x80 +#define NDIS_OFFLOAD_PARAMETERS_REVISION_3 3 +#define NDIS_OFFLOAD_PARAMETERS_REVISION_2 2 +#define NDIS_OFFLOAD_PARAMETERS_REVISION_1 1 + +#define NDIS_OFFLOAD_PARAMETERS_NO_CHANGE 0 +#define NDIS_OFFLOAD_PARAMETERS_LSOV2_DISABLED 1 +#define NDIS_OFFLOAD_PARAMETERS_LSOV2_ENABLED 2 +#define NDIS_OFFLOAD_PARAMETERS_LSOV1_ENABLED 2 +#define NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED 1 +#define NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED 2 +#define NDIS_OFFLOAD_PARAMETERS_TX_RX_DISABLED 1 +#define NDIS_OFFLOAD_PARAMETERS_TX_ENABLED_RX_DISABLED 2 +#define NDIS_OFFLOAD_PARAMETERS_RX_ENABLED_TX_DISABLED 3 +#define NDIS_OFFLOAD_PARAMETERS_TX_RX_ENABLED 4 + +#define NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE 1 +#define NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4 0 +#define NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6 1 + +#define VERSION_4_OFFLOAD_SIZE 22 +/* + * New offload OIDs for NDIS 6 + */ +#define OID_TCP_OFFLOAD_CURRENT_CONFIG 0xFC01020B /* query only */ +#define OID_TCP_OFFLOAD_PARAMETERS 0xFC01020C /* set only */ +#define OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES 0xFC01020D/* query only */ +#define OID_TCP_CONNECTION_OFFLOAD_CURRENT_CONFIG 0xFC01020E /* query only */ +#define OID_TCP_CONNECTION_OFFLOAD_HARDWARE_CAPABILITIES 0xFC01020F /* query */ +#define OID_OFFLOAD_ENCAPSULATION 0x0101010A /* set/query */ + +/* + * OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES + * ndis_type: NDIS_OBJTYPE_OFFLOAD + */ + +#define NDIS_OFFLOAD_ENCAP_NONE 0x0000 +#define NDIS_OFFLOAD_ENCAP_NULL 0x0001 +#define NDIS_OFFLOAD_ENCAP_8023 0x0002 +#define NDIS_OFFLOAD_ENCAP_8023PQ 0x0004 +#define NDIS_OFFLOAD_ENCAP_8023PQ_OOB 0x0008 +#define NDIS_OFFLOAD_ENCAP_RFC1483 0x0010 + +struct ndis_csum_offload { + u32 ip4_txenc; + u32 ip4_txcsum; +#define NDIS_TXCSUM_CAP_IP4OPT 0x001 +#define NDIS_TXCSUM_CAP_TCP4OPT 0x004 +#define NDIS_TXCSUM_CAP_TCP4 0x010 +#define NDIS_TXCSUM_CAP_UDP4 0x040 +#define NDIS_TXCSUM_CAP_IP4 0x100 + +#define NDIS_TXCSUM_ALL_TCP4 (NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT) + + u32 ip4_rxenc; + u32 ip4_rxcsum; +#define NDIS_RXCSUM_CAP_IP4OPT 0x001 +#define NDIS_RXCSUM_CAP_TCP4OPT 0x004 +#define NDIS_RXCSUM_CAP_TCP4 0x010 +#define NDIS_RXCSUM_CAP_UDP4 0x040 +#define NDIS_RXCSUM_CAP_IP4 0x100 + u32 ip6_txenc; + u32 ip6_txcsum; +#define NDIS_TXCSUM_CAP_IP6EXT 0x001 +#define NDIS_TXCSUM_CAP_TCP6OPT 0x004 +#define NDIS_TXCSUM_CAP_TCP6 0x010 +#define NDIS_TXCSUM_CAP_UDP6 0x040 + u32 ip6_rxenc; + u32 ip6_rxcsum; +#define NDIS_RXCSUM_CAP_IP6EXT 0x001 +#define NDIS_RXCSUM_CAP_TCP6OPT 0x004 +#define NDIS_RXCSUM_CAP_TCP6 0x010 +#define NDIS_RXCSUM_CAP_UDP6 0x040 + +#define NDIS_TXCSUM_ALL_TCP6 (NDIS_TXCSUM_CAP_TCP6 | \ + NDIS_TXCSUM_CAP_TCP6OPT | \ + NDIS_TXCSUM_CAP_IP6EXT) +}; + +struct ndis_lsov1_offload { + u32 encap; + u32 maxsize; + u32 minsegs; + u32 opts; +}; + +struct ndis_ipsecv1_offload { + u32 encap; + u32 ah_esp; + u32 xport_tun; + u32 ip4_opts; + u32 flags; + u32 ip4_ah; + u32 ip4_esp; +}; + +struct ndis_lsov2_offload { + u32 ip4_encap; + u32 ip4_maxsz; + u32 ip4_minsg; + u32 ip6_encap; + u32 ip6_maxsz; + u32 ip6_minsg; + u32 ip6_opts; +#define NDIS_LSOV2_CAP_IP6EXT 0x001 +#define NDIS_LSOV2_CAP_TCP6OPT 0x004 + +#define NDIS_LSOV2_CAP_IP6 (NDIS_LSOV2_CAP_IP6EXT | \ + NDIS_LSOV2_CAP_TCP6OPT) +}; + +struct ndis_ipsecv2_offload { + u32 encap; + u8 ip6; + u8 ip4opt; + u8 ip6ext; + u8 ah; + u8 esp; + u8 ah_esp; + u8 xport; + u8 tun; + u8 xport_tun; + u8 lso; + u8 extseq; + u32 udp_esp; + u32 auth; + u32 crypto; + u32 sa_caps; +}; + +struct ndis_rsc_offload { + u8 ip4; + u8 ip6; +}; + +struct ndis_encap_offload { + u32 flags; + u32 maxhdr; +}; + +struct ndis_offload { + struct ndis_object_header header; + struct ndis_csum_offload csum; + struct ndis_lsov1_offload lsov1; + struct ndis_ipsecv1_offload ipsecv1; + struct ndis_lsov2_offload lsov2; + u32 flags; + /* NDIS >= 6.1 */ + struct ndis_ipsecv2_offload ipsecv2; + /* NDIS >= 6.30 */ + struct ndis_rsc_offload rsc; + struct ndis_encap_offload encap_gre; +}; + +#define NDIS_OFFLOAD_SIZE sizeof(struct ndis_offload) +#define NDIS_OFFLOAD_SIZE_6_0 offsetof(struct ndis_offload, ipsecv2) +#define NDIS_OFFLOAD_SIZE_6_1 offsetof(struct ndis_offload, rsc) + +struct ndis_offload_params { + struct ndis_object_header header; + u8 ip_v4_csum; + u8 tcp_ip_v4_csum; + u8 udp_ip_v4_csum; + u8 tcp_ip_v6_csum; + u8 udp_ip_v6_csum; + u8 lso_v1; + u8 ip_sec_v1; + u8 lso_v2_ipv4; + u8 lso_v2_ipv6; + u8 tcp_connection_ip_v4; + u8 tcp_connection_ip_v6; + u32 flags; + u8 ip_sec_v2; + u8 ip_sec_v2_ip_v4; + struct { + u8 rsc_ip_v4; + u8 rsc_ip_v6; + }; + struct { + u8 encapsulated_packet_task_offload; + u8 encapsulation_types; + }; +}; + +struct ndis_tcp_lso_info { + union { + struct { + u32 unused:30; + u32 type:1; + u32 reserved2:1; + } transmit; + struct { + u32 mss:20; + u32 tcp_header_offset:10; + u32 type:1; + u32 reserved2:1; + } lso_v1_transmit; + struct { + u32 tcp_payload:30; + u32 type:1; + u32 reserved2:1; + } lso_v1_transmit_complete; + struct { + u32 mss:20; + u32 tcp_header_offset:10; + u32 type:1; + u32 ip_version:1; + } lso_v2_transmit; + struct { + u32 reserved:30; + u32 type:1; + u32 reserved2:1; + } lso_v2_transmit_complete; + u32 value; + }; +}; + +#define NDIS_VLAN_PPI_SIZE (sizeof(struct rndis_per_packet_info) + \ + sizeof(struct ndis_pkt_8021q_info)) + +#define NDIS_CSUM_PPI_SIZE (sizeof(struct rndis_per_packet_info) + \ + sizeof(struct ndis_tcp_ip_checksum_info)) + +#define NDIS_LSO_PPI_SIZE (sizeof(struct rndis_per_packet_info) + \ + sizeof(struct ndis_tcp_lso_info)) + +#define NDIS_HASH_PPI_SIZE (sizeof(struct rndis_per_packet_info) + \ + sizeof(u32)) + +/* Total size of all PPI data */ +#define NDIS_ALL_PPI_SIZE (NDIS_VLAN_PPI_SIZE + NDIS_CSUM_PPI_SIZE + \ + NDIS_LSO_PPI_SIZE + NDIS_HASH_PPI_SIZE) + +/* Format of Information buffer passed in a SetRequest for the OID */ +/* OID_GEN_RNDIS_CONFIG_PARAMETER. */ +struct rndis_config_parameter_info { + u32 parameter_name_offset; + u32 parameter_name_length; + u32 parameter_type; + u32 parameter_value_offset; + u32 parameter_value_length; +}; + +/* Values for ParameterType in struct rndis_config_parameter_info */ +#define RNDIS_CONFIG_PARAM_TYPE_INTEGER 0 +#define RNDIS_CONFIG_PARAM_TYPE_STRING 2 + +/* CONDIS Miniport messages for connection oriented devices */ +/* that do not implement a call manager. */ + +/* CoNdisMiniportCreateVc message */ +struct rcondis_mp_create_vc { + u32 req_id; + u32 ndis_vc_handle; +}; + +/* Response to CoNdisMiniportCreateVc */ +struct rcondis_mp_create_vc_complete { + u32 req_id; + u32 dev_vc_handle; + u32 status; +}; + +/* CoNdisMiniportDeleteVc message */ +struct rcondis_mp_delete_vc { + u32 req_id; + u32 dev_vc_handle; +}; + +/* Response to CoNdisMiniportDeleteVc */ +struct rcondis_mp_delete_vc_complete { + u32 req_id; + u32 status; +}; + +/* CoNdisMiniportQueryRequest message */ +struct rcondis_mp_query_request { + u32 req_id; + u32 request_type; + u32 oid; + u32 dev_vc_handle; + u32 info_buflen; + u32 info_buf_offset; +}; + +/* CoNdisMiniportSetRequest message */ +struct rcondis_mp_set_request { + u32 req_id; + u32 request_type; + u32 oid; + u32 dev_vc_handle; + u32 info_buflen; + u32 info_buf_offset; +}; + +/* CoNdisIndicateStatus message */ +struct rcondis_indicate_status { + u32 ndis_vc_handle; + u32 status; + u32 status_buflen; + u32 status_buf_offset; +}; + +/* CONDIS Call/VC parameters */ +struct rcondis_specific_parameters { + u32 parameter_type; + u32 parameter_length; + u32 parameter_lffset; +}; + +struct rcondis_media_parameters { + u32 flags; + u32 reserved1; + u32 reserved2; + struct rcondis_specific_parameters media_specific; +}; + +struct rndis_flowspec { + u32 token_rate; + u32 token_bucket_size; + u32 peak_bandwidth; + u32 latency; + u32 delay_variation; + u32 service_type; + u32 max_sdu_size; + u32 minimum_policed_size; +}; + +struct rcondis_call_manager_parameters { + struct rndis_flowspec transmit; + struct rndis_flowspec receive; + struct rcondis_specific_parameters call_mgr_specific; +}; + +/* CoNdisMiniportActivateVc message */ +struct rcondis_mp_activate_vc_request { + u32 req_id; + u32 flags; + u32 dev_vc_handle; + u32 media_params_offset; + u32 media_params_length; + u32 call_mgr_params_offset; + u32 call_mgr_params_length; +}; + +/* Response to CoNdisMiniportActivateVc */ +struct rcondis_mp_activate_vc_complete { + u32 req_id; + u32 status; +}; + +/* CoNdisMiniportDeactivateVc message */ +struct rcondis_mp_deactivate_vc_request { + u32 req_id; + u32 flags; + u32 dev_vc_handle; +}; + +/* Response to CoNdisMiniportDeactivateVc */ +struct rcondis_mp_deactivate_vc_complete { + u32 req_id; + u32 status; +}; + + +/* union with all of the RNDIS messages */ +union rndis_message_container { + struct rndis_packet pkt; + struct rndis_initialize_request init_req; + struct rndis_halt_request halt_req; + struct rndis_query_request query_req; + struct rndis_set_request set_req; + struct rndis_reset_request reset_req; + struct rndis_keepalive_request keep_alive_req; + struct rndis_indicate_status indicate_status; + struct rndis_initialize_complete init_complete; + struct rndis_query_complete query_complete; + struct rndis_set_complete set_complete; + struct rndis_reset_complete reset_complete; + struct rndis_keepalive_complete keep_alive_complete; + struct rcondis_mp_create_vc co_miniport_create_vc; + struct rcondis_mp_delete_vc co_miniport_delete_vc; + struct rcondis_indicate_status co_indicate_status; + struct rcondis_mp_activate_vc_request co_miniport_activate_vc; + struct rcondis_mp_deactivate_vc_request co_miniport_deactivate_vc; + struct rcondis_mp_create_vc_complete co_miniport_create_vc_complete; + struct rcondis_mp_delete_vc_complete co_miniport_delete_vc_complete; + struct rcondis_mp_activate_vc_complete co_miniport_activate_vc_complete; + struct rcondis_mp_deactivate_vc_complete + co_miniport_deactivate_vc_complete; +}; + +/* Remote NDIS message format */ +struct rndis_message { + u32 ndis_msg_type; + + /* Total length of this message, from the beginning */ + /* of the struct rndis_message, in bytes. */ + u32 msg_len; + + /* Actual message */ + union rndis_message_container msg; +}; + + +/* Handy macros */ + +/* get the size of an RNDIS message. Pass in the message type, */ +/* struct rndis_set_request, struct rndis_packet for example */ +#define RNDIS_MESSAGE_SIZE(msg) \ + (sizeof(msg) + (sizeof(struct rndis_message) - \ + sizeof(union rndis_message_container))) + +#define RNDIS_HEADER_SIZE (sizeof(struct rndis_message) - \ + sizeof(union rndis_message_container)) + +#define RNDIS_AND_PPI_SIZE (sizeof(struct rndis_message) + NDIS_ALL_PPI_SIZE) + +#define NDIS_PACKET_TYPE_DIRECTED 0x00000001 +#define NDIS_PACKET_TYPE_MULTICAST 0x00000002 +#define NDIS_PACKET_TYPE_ALL_MULTICAST 0x00000004 +#define NDIS_PACKET_TYPE_BROADCAST 0x00000008 +#define NDIS_PACKET_TYPE_SOURCE_ROUTING 0x00000010 +#define NDIS_PACKET_TYPE_PROMISCUOUS 0x00000020 +#define NDIS_PACKET_TYPE_SMT 0x00000040 +#define NDIS_PACKET_TYPE_ALL_LOCAL 0x00000080 +#define NDIS_PACKET_TYPE_GROUP 0x00000100 +#define NDIS_PACKET_TYPE_ALL_FUNCTIONAL 0x00000200 +#define NDIS_PACKET_TYPE_FUNCTIONAL 0x00000400 +#define NDIS_PACKET_TYPE_MAC_FRAME 0x00000800 + +#define TRANSPORT_INFO_NOT_IP 0 +#define TRANSPORT_INFO_IPV4_TCP 0x01 +#define TRANSPORT_INFO_IPV4_UDP 0x02 +#define TRANSPORT_INFO_IPV6_TCP 0x10 +#define TRANSPORT_INFO_IPV6_UDP 0x20 + +#define RETRY_US_LO 5000 +#define RETRY_US_HI 10000 +#define RETRY_MAX 2000 /* >10 sec */ + +void netvsc_dma_unmap(struct hv_device *hv_dev, + struct hv_netvsc_packet *packet); +#endif /* _HYPERV_NET_H */ diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c new file mode 100644 index 000000000..da737d959 --- /dev/null +++ b/drivers/net/hyperv/netvsc.c @@ -0,0 +1,1856 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2009, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/kernel.h> +#include <linux/sched.h> +#include <linux/wait.h> +#include <linux/mm.h> +#include <linux/delay.h> +#include <linux/io.h> +#include <linux/slab.h> +#include <linux/netdevice.h> +#include <linux/if_ether.h> +#include <linux/vmalloc.h> +#include <linux/rtnetlink.h> +#include <linux/prefetch.h> +#include <linux/filter.h> + +#include <asm/sync_bitops.h> +#include <asm/mshyperv.h> + +#include "hyperv_net.h" +#include "netvsc_trace.h" + +/* + * Switch the data path from the synthetic interface to the VF + * interface. + */ +int netvsc_switch_datapath(struct net_device *ndev, bool vf) +{ + struct net_device_context *net_device_ctx = netdev_priv(ndev); + struct hv_device *dev = net_device_ctx->device_ctx; + struct netvsc_device *nv_dev = rtnl_dereference(net_device_ctx->nvdev); + struct nvsp_message *init_pkt = &nv_dev->channel_init_pkt; + int ret, retry = 0; + + /* Block sending traffic to VF if it's about to be gone */ + if (!vf) + net_device_ctx->data_path_is_vf = vf; + + memset(init_pkt, 0, sizeof(struct nvsp_message)); + init_pkt->hdr.msg_type = NVSP_MSG4_TYPE_SWITCH_DATA_PATH; + if (vf) + init_pkt->msg.v4_msg.active_dp.active_datapath = + NVSP_DATAPATH_VF; + else + init_pkt->msg.v4_msg.active_dp.active_datapath = + NVSP_DATAPATH_SYNTHETIC; + +again: + trace_nvsp_send(ndev, init_pkt); + + ret = vmbus_sendpacket(dev->channel, init_pkt, + sizeof(struct nvsp_message), + (unsigned long)init_pkt, VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + + /* If failed to switch to/from VF, let data_path_is_vf stay false, + * so we use synthetic path to send data. + */ + if (ret) { + if (ret != -EAGAIN) { + netdev_err(ndev, + "Unable to send sw datapath msg, err: %d\n", + ret); + return ret; + } + + if (retry++ < RETRY_MAX) { + usleep_range(RETRY_US_LO, RETRY_US_HI); + goto again; + } else { + netdev_err( + ndev, + "Retry failed to send sw datapath msg, err: %d\n", + ret); + return ret; + } + } + + wait_for_completion(&nv_dev->channel_init_wait); + net_device_ctx->data_path_is_vf = vf; + + return 0; +} + +/* Worker to setup sub channels on initial setup + * Initial hotplug event occurs in softirq context + * and can't wait for channels. + */ +static void netvsc_subchan_work(struct work_struct *w) +{ + struct netvsc_device *nvdev = + container_of(w, struct netvsc_device, subchan_work); + struct rndis_device *rdev; + int i, ret; + + /* Avoid deadlock with device removal already under RTNL */ + if (!rtnl_trylock()) { + schedule_work(w); + return; + } + + rdev = nvdev->extension; + if (rdev) { + ret = rndis_set_subchannel(rdev->ndev, nvdev, NULL); + if (ret == 0) { + netif_device_attach(rdev->ndev); + } else { + /* fallback to only primary channel */ + for (i = 1; i < nvdev->num_chn; i++) + netif_napi_del(&nvdev->chan_table[i].napi); + + nvdev->max_chn = 1; + nvdev->num_chn = 1; + } + } + + rtnl_unlock(); +} + +static struct netvsc_device *alloc_net_device(void) +{ + struct netvsc_device *net_device; + + net_device = kzalloc(sizeof(struct netvsc_device), GFP_KERNEL); + if (!net_device) + return NULL; + + init_waitqueue_head(&net_device->wait_drain); + net_device->destroy = false; + net_device->tx_disable = true; + + net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT; + net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT; + + init_completion(&net_device->channel_init_wait); + init_waitqueue_head(&net_device->subchan_open); + INIT_WORK(&net_device->subchan_work, netvsc_subchan_work); + + return net_device; +} + +static void free_netvsc_device(struct rcu_head *head) +{ + struct netvsc_device *nvdev + = container_of(head, struct netvsc_device, rcu); + int i; + + kfree(nvdev->extension); + + if (nvdev->recv_original_buf) + vfree(nvdev->recv_original_buf); + else + vfree(nvdev->recv_buf); + + if (nvdev->send_original_buf) + vfree(nvdev->send_original_buf); + else + vfree(nvdev->send_buf); + + bitmap_free(nvdev->send_section_map); + + for (i = 0; i < VRSS_CHANNEL_MAX; i++) { + xdp_rxq_info_unreg(&nvdev->chan_table[i].xdp_rxq); + kfree(nvdev->chan_table[i].recv_buf); + vfree(nvdev->chan_table[i].mrc.slots); + } + + kfree(nvdev); +} + +static void free_netvsc_device_rcu(struct netvsc_device *nvdev) +{ + call_rcu(&nvdev->rcu, free_netvsc_device); +} + +static void netvsc_revoke_recv_buf(struct hv_device *device, + struct netvsc_device *net_device, + struct net_device *ndev) +{ + struct nvsp_message *revoke_packet; + int ret; + + /* + * If we got a section count, it means we received a + * SendReceiveBufferComplete msg (ie sent + * NvspMessage1TypeSendReceiveBuffer msg) therefore, we need + * to send a revoke msg here + */ + if (net_device->recv_section_cnt) { + /* Send the revoke receive buffer */ + revoke_packet = &net_device->revoke_packet; + memset(revoke_packet, 0, sizeof(struct nvsp_message)); + + revoke_packet->hdr.msg_type = + NVSP_MSG1_TYPE_REVOKE_RECV_BUF; + revoke_packet->msg.v1_msg. + revoke_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; + + trace_nvsp_send(ndev, revoke_packet); + + ret = vmbus_sendpacket(device->channel, + revoke_packet, + sizeof(struct nvsp_message), + VMBUS_RQST_ID_NO_RESPONSE, + VM_PKT_DATA_INBAND, 0); + /* If the failure is because the channel is rescinded; + * ignore the failure since we cannot send on a rescinded + * channel. This would allow us to properly cleanup + * even when the channel is rescinded. + */ + if (device->channel->rescind) + ret = 0; + /* + * If we failed here, we might as well return and + * have a leak rather than continue and a bugchk + */ + if (ret != 0) { + netdev_err(ndev, "unable to send " + "revoke receive buffer to netvsp\n"); + return; + } + net_device->recv_section_cnt = 0; + } +} + +static void netvsc_revoke_send_buf(struct hv_device *device, + struct netvsc_device *net_device, + struct net_device *ndev) +{ + struct nvsp_message *revoke_packet; + int ret; + + /* Deal with the send buffer we may have setup. + * If we got a send section size, it means we received a + * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent + * NVSP_MSG1_TYPE_SEND_SEND_BUF msg) therefore, we need + * to send a revoke msg here + */ + if (net_device->send_section_cnt) { + /* Send the revoke receive buffer */ + revoke_packet = &net_device->revoke_packet; + memset(revoke_packet, 0, sizeof(struct nvsp_message)); + + revoke_packet->hdr.msg_type = + NVSP_MSG1_TYPE_REVOKE_SEND_BUF; + revoke_packet->msg.v1_msg.revoke_send_buf.id = + NETVSC_SEND_BUFFER_ID; + + trace_nvsp_send(ndev, revoke_packet); + + ret = vmbus_sendpacket(device->channel, + revoke_packet, + sizeof(struct nvsp_message), + VMBUS_RQST_ID_NO_RESPONSE, + VM_PKT_DATA_INBAND, 0); + + /* If the failure is because the channel is rescinded; + * ignore the failure since we cannot send on a rescinded + * channel. This would allow us to properly cleanup + * even when the channel is rescinded. + */ + if (device->channel->rescind) + ret = 0; + + /* If we failed here, we might as well return and + * have a leak rather than continue and a bugchk + */ + if (ret != 0) { + netdev_err(ndev, "unable to send " + "revoke send buffer to netvsp\n"); + return; + } + net_device->send_section_cnt = 0; + } +} + +static void netvsc_teardown_recv_gpadl(struct hv_device *device, + struct netvsc_device *net_device, + struct net_device *ndev) +{ + int ret; + + if (net_device->recv_buf_gpadl_handle.gpadl_handle) { + ret = vmbus_teardown_gpadl(device->channel, + &net_device->recv_buf_gpadl_handle); + + /* If we failed here, we might as well return and have a leak + * rather than continue and a bugchk + */ + if (ret != 0) { + netdev_err(ndev, + "unable to teardown receive buffer's gpadl\n"); + return; + } + } +} + +static void netvsc_teardown_send_gpadl(struct hv_device *device, + struct netvsc_device *net_device, + struct net_device *ndev) +{ + int ret; + + if (net_device->send_buf_gpadl_handle.gpadl_handle) { + ret = vmbus_teardown_gpadl(device->channel, + &net_device->send_buf_gpadl_handle); + + /* If we failed here, we might as well return and have a leak + * rather than continue and a bugchk + */ + if (ret != 0) { + netdev_err(ndev, + "unable to teardown send buffer's gpadl\n"); + return; + } + } +} + +int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx) +{ + struct netvsc_channel *nvchan = &net_device->chan_table[q_idx]; + int node = cpu_to_node(nvchan->channel->target_cpu); + size_t size; + + size = net_device->recv_completion_cnt * sizeof(struct recv_comp_data); + nvchan->mrc.slots = vzalloc_node(size, node); + if (!nvchan->mrc.slots) + nvchan->mrc.slots = vzalloc(size); + + return nvchan->mrc.slots ? 0 : -ENOMEM; +} + +static int netvsc_init_buf(struct hv_device *device, + struct netvsc_device *net_device, + const struct netvsc_device_info *device_info) +{ + struct nvsp_1_message_send_receive_buffer_complete *resp; + struct net_device *ndev = hv_get_drvdata(device); + struct nvsp_message *init_packet; + unsigned int buf_size; + int i, ret = 0; + void *vaddr; + + /* Get receive buffer area. */ + buf_size = device_info->recv_sections * device_info->recv_section_size; + buf_size = roundup(buf_size, PAGE_SIZE); + + /* Legacy hosts only allow smaller receive buffer */ + if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2) + buf_size = min_t(unsigned int, buf_size, + NETVSC_RECEIVE_BUFFER_SIZE_LEGACY); + + net_device->recv_buf = vzalloc(buf_size); + if (!net_device->recv_buf) { + netdev_err(ndev, + "unable to allocate receive buffer of size %u\n", + buf_size); + ret = -ENOMEM; + goto cleanup; + } + + net_device->recv_buf_size = buf_size; + + /* + * Establish the gpadl handle for this buffer on this + * channel. Note: This call uses the vmbus connection rather + * than the channel to establish the gpadl handle. + */ + ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf, + buf_size, + &net_device->recv_buf_gpadl_handle); + if (ret != 0) { + netdev_err(ndev, + "unable to establish receive buffer's gpadl\n"); + goto cleanup; + } + + if (hv_isolation_type_snp()) { + vaddr = hv_map_memory(net_device->recv_buf, buf_size); + if (!vaddr) { + ret = -ENOMEM; + goto cleanup; + } + + net_device->recv_original_buf = net_device->recv_buf; + net_device->recv_buf = vaddr; + } + + /* Notify the NetVsp of the gpadl handle */ + init_packet = &net_device->channel_init_pkt; + memset(init_packet, 0, sizeof(struct nvsp_message)); + init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_RECV_BUF; + init_packet->msg.v1_msg.send_recv_buf. + gpadl_handle = net_device->recv_buf_gpadl_handle.gpadl_handle; + init_packet->msg.v1_msg. + send_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; + + trace_nvsp_send(ndev, init_packet); + + /* Send the gpadl notification request */ + ret = vmbus_sendpacket(device->channel, init_packet, + sizeof(struct nvsp_message), + (unsigned long)init_packet, + VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (ret != 0) { + netdev_err(ndev, + "unable to send receive buffer's gpadl to netvsp\n"); + goto cleanup; + } + + wait_for_completion(&net_device->channel_init_wait); + + /* Check the response */ + resp = &init_packet->msg.v1_msg.send_recv_buf_complete; + if (resp->status != NVSP_STAT_SUCCESS) { + netdev_err(ndev, + "Unable to complete receive buffer initialization with NetVsp - status %d\n", + resp->status); + ret = -EINVAL; + goto cleanup; + } + + /* Parse the response */ + netdev_dbg(ndev, "Receive sections: %u sub_allocs: size %u count: %u\n", + resp->num_sections, resp->sections[0].sub_alloc_size, + resp->sections[0].num_sub_allocs); + + /* There should only be one section for the entire receive buffer */ + if (resp->num_sections != 1 || resp->sections[0].offset != 0) { + ret = -EINVAL; + goto cleanup; + } + + net_device->recv_section_size = resp->sections[0].sub_alloc_size; + net_device->recv_section_cnt = resp->sections[0].num_sub_allocs; + + /* Ensure buffer will not overflow */ + if (net_device->recv_section_size < NETVSC_MTU_MIN || (u64)net_device->recv_section_size * + (u64)net_device->recv_section_cnt > (u64)buf_size) { + netdev_err(ndev, "invalid recv_section_size %u\n", + net_device->recv_section_size); + ret = -EINVAL; + goto cleanup; + } + + for (i = 0; i < VRSS_CHANNEL_MAX; i++) { + struct netvsc_channel *nvchan = &net_device->chan_table[i]; + + nvchan->recv_buf = kzalloc(net_device->recv_section_size, GFP_KERNEL); + if (nvchan->recv_buf == NULL) { + ret = -ENOMEM; + goto cleanup; + } + } + + /* Setup receive completion ring. + * Add 1 to the recv_section_cnt because at least one entry in a + * ring buffer has to be empty. + */ + net_device->recv_completion_cnt = net_device->recv_section_cnt + 1; + ret = netvsc_alloc_recv_comp_ring(net_device, 0); + if (ret) + goto cleanup; + + /* Now setup the send buffer. */ + buf_size = device_info->send_sections * device_info->send_section_size; + buf_size = round_up(buf_size, PAGE_SIZE); + + net_device->send_buf = vzalloc(buf_size); + if (!net_device->send_buf) { + netdev_err(ndev, "unable to allocate send buffer of size %u\n", + buf_size); + ret = -ENOMEM; + goto cleanup; + } + net_device->send_buf_size = buf_size; + + /* Establish the gpadl handle for this buffer on this + * channel. Note: This call uses the vmbus connection rather + * than the channel to establish the gpadl handle. + */ + ret = vmbus_establish_gpadl(device->channel, net_device->send_buf, + buf_size, + &net_device->send_buf_gpadl_handle); + if (ret != 0) { + netdev_err(ndev, + "unable to establish send buffer's gpadl\n"); + goto cleanup; + } + + if (hv_isolation_type_snp()) { + vaddr = hv_map_memory(net_device->send_buf, buf_size); + if (!vaddr) { + ret = -ENOMEM; + goto cleanup; + } + + net_device->send_original_buf = net_device->send_buf; + net_device->send_buf = vaddr; + } + + /* Notify the NetVsp of the gpadl handle */ + init_packet = &net_device->channel_init_pkt; + memset(init_packet, 0, sizeof(struct nvsp_message)); + init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_SEND_BUF; + init_packet->msg.v1_msg.send_send_buf.gpadl_handle = + net_device->send_buf_gpadl_handle.gpadl_handle; + init_packet->msg.v1_msg.send_send_buf.id = NETVSC_SEND_BUFFER_ID; + + trace_nvsp_send(ndev, init_packet); + + /* Send the gpadl notification request */ + ret = vmbus_sendpacket(device->channel, init_packet, + sizeof(struct nvsp_message), + (unsigned long)init_packet, + VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (ret != 0) { + netdev_err(ndev, + "unable to send send buffer's gpadl to netvsp\n"); + goto cleanup; + } + + wait_for_completion(&net_device->channel_init_wait); + + /* Check the response */ + if (init_packet->msg.v1_msg. + send_send_buf_complete.status != NVSP_STAT_SUCCESS) { + netdev_err(ndev, "Unable to complete send buffer " + "initialization with NetVsp - status %d\n", + init_packet->msg.v1_msg. + send_send_buf_complete.status); + ret = -EINVAL; + goto cleanup; + } + + /* Parse the response */ + net_device->send_section_size = init_packet->msg. + v1_msg.send_send_buf_complete.section_size; + if (net_device->send_section_size < NETVSC_MTU_MIN) { + netdev_err(ndev, "invalid send_section_size %u\n", + net_device->send_section_size); + ret = -EINVAL; + goto cleanup; + } + + /* Section count is simply the size divided by the section size. */ + net_device->send_section_cnt = buf_size / net_device->send_section_size; + + netdev_dbg(ndev, "Send section size: %d, Section count:%d\n", + net_device->send_section_size, net_device->send_section_cnt); + + /* Setup state for managing the send buffer. */ + net_device->send_section_map = bitmap_zalloc(net_device->send_section_cnt, + GFP_KERNEL); + if (!net_device->send_section_map) { + ret = -ENOMEM; + goto cleanup; + } + + goto exit; + +cleanup: + netvsc_revoke_recv_buf(device, net_device, ndev); + netvsc_revoke_send_buf(device, net_device, ndev); + netvsc_teardown_recv_gpadl(device, net_device, ndev); + netvsc_teardown_send_gpadl(device, net_device, ndev); + +exit: + return ret; +} + +/* Negotiate NVSP protocol version */ +static int negotiate_nvsp_ver(struct hv_device *device, + struct netvsc_device *net_device, + struct nvsp_message *init_packet, + u32 nvsp_ver) +{ + struct net_device *ndev = hv_get_drvdata(device); + int ret; + + memset(init_packet, 0, sizeof(struct nvsp_message)); + init_packet->hdr.msg_type = NVSP_MSG_TYPE_INIT; + init_packet->msg.init_msg.init.min_protocol_ver = nvsp_ver; + init_packet->msg.init_msg.init.max_protocol_ver = nvsp_ver; + trace_nvsp_send(ndev, init_packet); + + /* Send the init request */ + ret = vmbus_sendpacket(device->channel, init_packet, + sizeof(struct nvsp_message), + (unsigned long)init_packet, + VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + + if (ret != 0) + return ret; + + wait_for_completion(&net_device->channel_init_wait); + + if (init_packet->msg.init_msg.init_complete.status != + NVSP_STAT_SUCCESS) + return -EINVAL; + + if (nvsp_ver == NVSP_PROTOCOL_VERSION_1) + return 0; + + /* NVSPv2 or later: Send NDIS config */ + memset(init_packet, 0, sizeof(struct nvsp_message)); + init_packet->hdr.msg_type = NVSP_MSG2_TYPE_SEND_NDIS_CONFIG; + init_packet->msg.v2_msg.send_ndis_config.mtu = ndev->mtu + ETH_HLEN; + init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1; + + if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5) { + if (hv_is_isolation_supported()) + netdev_info(ndev, "SR-IOV not advertised by guests on the host supporting isolation\n"); + else + init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1; + + /* Teaming bit is needed to receive link speed updates */ + init_packet->msg.v2_msg.send_ndis_config.capability.teaming = 1; + } + + if (nvsp_ver >= NVSP_PROTOCOL_VERSION_61) + init_packet->msg.v2_msg.send_ndis_config.capability.rsc = 1; + + trace_nvsp_send(ndev, init_packet); + + ret = vmbus_sendpacket(device->channel, init_packet, + sizeof(struct nvsp_message), + VMBUS_RQST_ID_NO_RESPONSE, + VM_PKT_DATA_INBAND, 0); + + return ret; +} + +static int netvsc_connect_vsp(struct hv_device *device, + struct netvsc_device *net_device, + const struct netvsc_device_info *device_info) +{ + struct net_device *ndev = hv_get_drvdata(device); + static const u32 ver_list[] = { + NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2, + NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5, + NVSP_PROTOCOL_VERSION_6, NVSP_PROTOCOL_VERSION_61 + }; + struct nvsp_message *init_packet; + int ndis_version, i, ret; + + init_packet = &net_device->channel_init_pkt; + + /* Negotiate the latest NVSP protocol supported */ + for (i = ARRAY_SIZE(ver_list) - 1; i >= 0; i--) + if (negotiate_nvsp_ver(device, net_device, init_packet, + ver_list[i]) == 0) { + net_device->nvsp_version = ver_list[i]; + break; + } + + if (i < 0) { + ret = -EPROTO; + goto cleanup; + } + + if (hv_is_isolation_supported() && net_device->nvsp_version < NVSP_PROTOCOL_VERSION_61) { + netdev_err(ndev, "Invalid NVSP version 0x%x (expected >= 0x%x) from the host supporting isolation\n", + net_device->nvsp_version, NVSP_PROTOCOL_VERSION_61); + ret = -EPROTO; + goto cleanup; + } + + pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version); + + /* Send the ndis version */ + memset(init_packet, 0, sizeof(struct nvsp_message)); + + if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4) + ndis_version = 0x00060001; + else + ndis_version = 0x0006001e; + + init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_NDIS_VER; + init_packet->msg.v1_msg. + send_ndis_ver.ndis_major_ver = + (ndis_version & 0xFFFF0000) >> 16; + init_packet->msg.v1_msg. + send_ndis_ver.ndis_minor_ver = + ndis_version & 0xFFFF; + + trace_nvsp_send(ndev, init_packet); + + /* Send the init request */ + ret = vmbus_sendpacket(device->channel, init_packet, + sizeof(struct nvsp_message), + VMBUS_RQST_ID_NO_RESPONSE, + VM_PKT_DATA_INBAND, 0); + if (ret != 0) + goto cleanup; + + + ret = netvsc_init_buf(device, net_device, device_info); + +cleanup: + return ret; +} + +/* + * netvsc_device_remove - Callback when the root bus device is removed + */ +void netvsc_device_remove(struct hv_device *device) +{ + struct net_device *ndev = hv_get_drvdata(device); + struct net_device_context *net_device_ctx = netdev_priv(ndev); + struct netvsc_device *net_device + = rtnl_dereference(net_device_ctx->nvdev); + int i; + + /* + * Revoke receive buffer. If host is pre-Win2016 then tear down + * receive buffer GPADL. Do the same for send buffer. + */ + netvsc_revoke_recv_buf(device, net_device, ndev); + if (vmbus_proto_version < VERSION_WIN10) + netvsc_teardown_recv_gpadl(device, net_device, ndev); + + netvsc_revoke_send_buf(device, net_device, ndev); + if (vmbus_proto_version < VERSION_WIN10) + netvsc_teardown_send_gpadl(device, net_device, ndev); + + RCU_INIT_POINTER(net_device_ctx->nvdev, NULL); + + /* Disable NAPI and disassociate its context from the device. */ + for (i = 0; i < net_device->num_chn; i++) { + /* See also vmbus_reset_channel_cb(). */ + napi_disable(&net_device->chan_table[i].napi); + netif_napi_del(&net_device->chan_table[i].napi); + } + + /* + * At this point, no one should be accessing net_device + * except in here + */ + netdev_dbg(ndev, "net device safe to remove\n"); + + /* Now, we can close the channel safely */ + vmbus_close(device->channel); + + /* + * If host is Win2016 or higher then we do the GPADL tear down + * here after VMBus is closed. + */ + if (vmbus_proto_version >= VERSION_WIN10) { + netvsc_teardown_recv_gpadl(device, net_device, ndev); + netvsc_teardown_send_gpadl(device, net_device, ndev); + } + + if (net_device->recv_original_buf) + hv_unmap_memory(net_device->recv_buf); + + if (net_device->send_original_buf) + hv_unmap_memory(net_device->send_buf); + + /* Release all resources */ + free_netvsc_device_rcu(net_device); +} + +#define RING_AVAIL_PERCENT_HIWATER 20 +#define RING_AVAIL_PERCENT_LOWATER 10 + +static inline void netvsc_free_send_slot(struct netvsc_device *net_device, + u32 index) +{ + sync_change_bit(index, net_device->send_section_map); +} + +static void netvsc_send_tx_complete(struct net_device *ndev, + struct netvsc_device *net_device, + struct vmbus_channel *channel, + const struct vmpacket_descriptor *desc, + int budget) +{ + struct net_device_context *ndev_ctx = netdev_priv(ndev); + struct sk_buff *skb; + u16 q_idx = 0; + int queue_sends; + u64 cmd_rqst; + + cmd_rqst = channel->request_addr_callback(channel, desc->trans_id); + if (cmd_rqst == VMBUS_RQST_ERROR) { + netdev_err(ndev, "Invalid transaction ID %llx\n", desc->trans_id); + return; + } + + skb = (struct sk_buff *)(unsigned long)cmd_rqst; + + /* Notify the layer above us */ + if (likely(skb)) { + struct hv_netvsc_packet *packet + = (struct hv_netvsc_packet *)skb->cb; + u32 send_index = packet->send_buf_index; + struct netvsc_stats_tx *tx_stats; + + if (send_index != NETVSC_INVALID_INDEX) + netvsc_free_send_slot(net_device, send_index); + q_idx = packet->q_idx; + + tx_stats = &net_device->chan_table[q_idx].tx_stats; + + u64_stats_update_begin(&tx_stats->syncp); + tx_stats->packets += packet->total_packets; + tx_stats->bytes += packet->total_bytes; + u64_stats_update_end(&tx_stats->syncp); + + netvsc_dma_unmap(ndev_ctx->device_ctx, packet); + napi_consume_skb(skb, budget); + } + + queue_sends = + atomic_dec_return(&net_device->chan_table[q_idx].queue_sends); + + if (unlikely(net_device->destroy)) { + if (queue_sends == 0) + wake_up(&net_device->wait_drain); + } else { + struct netdev_queue *txq = netdev_get_tx_queue(ndev, q_idx); + + if (netif_tx_queue_stopped(txq) && !net_device->tx_disable && + (hv_get_avail_to_write_percent(&channel->outbound) > + RING_AVAIL_PERCENT_HIWATER || queue_sends < 1)) { + netif_tx_wake_queue(txq); + ndev_ctx->eth_stats.wake_queue++; + } + } +} + +static void netvsc_send_completion(struct net_device *ndev, + struct netvsc_device *net_device, + struct vmbus_channel *incoming_channel, + const struct vmpacket_descriptor *desc, + int budget) +{ + const struct nvsp_message *nvsp_packet; + u32 msglen = hv_pkt_datalen(desc); + struct nvsp_message *pkt_rqst; + u64 cmd_rqst; + u32 status; + + /* First check if this is a VMBUS completion without data payload */ + if (!msglen) { + cmd_rqst = incoming_channel->request_addr_callback(incoming_channel, + desc->trans_id); + if (cmd_rqst == VMBUS_RQST_ERROR) { + netdev_err(ndev, "Invalid transaction ID %llx\n", desc->trans_id); + return; + } + + pkt_rqst = (struct nvsp_message *)(uintptr_t)cmd_rqst; + switch (pkt_rqst->hdr.msg_type) { + case NVSP_MSG4_TYPE_SWITCH_DATA_PATH: + complete(&net_device->channel_init_wait); + break; + + default: + netdev_err(ndev, "Unexpected VMBUS completion!!\n"); + } + return; + } + + /* Ensure packet is big enough to read header fields */ + if (msglen < sizeof(struct nvsp_message_header)) { + netdev_err(ndev, "nvsp_message length too small: %u\n", msglen); + return; + } + + nvsp_packet = hv_pkt_data(desc); + switch (nvsp_packet->hdr.msg_type) { + case NVSP_MSG_TYPE_INIT_COMPLETE: + if (msglen < sizeof(struct nvsp_message_header) + + sizeof(struct nvsp_message_init_complete)) { + netdev_err(ndev, "nvsp_msg length too small: %u\n", + msglen); + return; + } + fallthrough; + + case NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE: + if (msglen < sizeof(struct nvsp_message_header) + + sizeof(struct nvsp_1_message_send_receive_buffer_complete)) { + netdev_err(ndev, "nvsp_msg1 length too small: %u\n", + msglen); + return; + } + fallthrough; + + case NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE: + if (msglen < sizeof(struct nvsp_message_header) + + sizeof(struct nvsp_1_message_send_send_buffer_complete)) { + netdev_err(ndev, "nvsp_msg1 length too small: %u\n", + msglen); + return; + } + fallthrough; + + case NVSP_MSG5_TYPE_SUBCHANNEL: + if (msglen < sizeof(struct nvsp_message_header) + + sizeof(struct nvsp_5_subchannel_complete)) { + netdev_err(ndev, "nvsp_msg5 length too small: %u\n", + msglen); + return; + } + /* Copy the response back */ + memcpy(&net_device->channel_init_pkt, nvsp_packet, + sizeof(struct nvsp_message)); + complete(&net_device->channel_init_wait); + break; + + case NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE: + if (msglen < sizeof(struct nvsp_message_header) + + sizeof(struct nvsp_1_message_send_rndis_packet_complete)) { + if (net_ratelimit()) + netdev_err(ndev, "nvsp_rndis_pkt_complete length too small: %u\n", + msglen); + return; + } + + /* If status indicates an error, output a message so we know + * there's a problem. But process the completion anyway so the + * resources are released. + */ + status = nvsp_packet->msg.v1_msg.send_rndis_pkt_complete.status; + if (status != NVSP_STAT_SUCCESS && net_ratelimit()) + netdev_err(ndev, "nvsp_rndis_pkt_complete error status: %x\n", + status); + + netvsc_send_tx_complete(ndev, net_device, incoming_channel, + desc, budget); + break; + + default: + netdev_err(ndev, + "Unknown send completion type %d received!!\n", + nvsp_packet->hdr.msg_type); + } +} + +static u32 netvsc_get_next_send_section(struct netvsc_device *net_device) +{ + unsigned long *map_addr = net_device->send_section_map; + unsigned int i; + + for_each_clear_bit(i, map_addr, net_device->send_section_cnt) { + if (sync_test_and_set_bit(i, map_addr) == 0) + return i; + } + + return NETVSC_INVALID_INDEX; +} + +static void netvsc_copy_to_send_buf(struct netvsc_device *net_device, + unsigned int section_index, + u32 pend_size, + struct hv_netvsc_packet *packet, + struct rndis_message *rndis_msg, + struct hv_page_buffer *pb, + bool xmit_more) +{ + char *start = net_device->send_buf; + char *dest = start + (section_index * net_device->send_section_size) + + pend_size; + int i; + u32 padding = 0; + u32 page_count = packet->cp_partial ? packet->rmsg_pgcnt : + packet->page_buf_cnt; + u32 remain; + + /* Add padding */ + remain = packet->total_data_buflen & (net_device->pkt_align - 1); + if (xmit_more && remain) { + padding = net_device->pkt_align - remain; + rndis_msg->msg_len += padding; + packet->total_data_buflen += padding; + } + + for (i = 0; i < page_count; i++) { + char *src = phys_to_virt(pb[i].pfn << HV_HYP_PAGE_SHIFT); + u32 offset = pb[i].offset; + u32 len = pb[i].len; + + memcpy(dest, (src + offset), len); + dest += len; + } + + if (padding) + memset(dest, 0, padding); +} + +void netvsc_dma_unmap(struct hv_device *hv_dev, + struct hv_netvsc_packet *packet) +{ + int i; + + if (!hv_is_isolation_supported()) + return; + + if (!packet->dma_range) + return; + + for (i = 0; i < packet->page_buf_cnt; i++) + dma_unmap_single(&hv_dev->device, packet->dma_range[i].dma, + packet->dma_range[i].mapping_size, + DMA_TO_DEVICE); + + kfree(packet->dma_range); +} + +/* netvsc_dma_map - Map swiotlb bounce buffer with data page of + * packet sent by vmbus_sendpacket_pagebuffer() in the Isolation + * VM. + * + * In isolation VM, netvsc send buffer has been marked visible to + * host and so the data copied to send buffer doesn't need to use + * bounce buffer. The data pages handled by vmbus_sendpacket_pagebuffer() + * may not be copied to send buffer and so these pages need to be + * mapped with swiotlb bounce buffer. netvsc_dma_map() is to do + * that. The pfns in the struct hv_page_buffer need to be converted + * to bounce buffer's pfn. The loop here is necessary because the + * entries in the page buffer array are not necessarily full + * pages of data. Each entry in the array has a separate offset and + * len that may be non-zero, even for entries in the middle of the + * array. And the entries are not physically contiguous. So each + * entry must be individually mapped rather than as a contiguous unit. + * So not use dma_map_sg() here. + */ +static int netvsc_dma_map(struct hv_device *hv_dev, + struct hv_netvsc_packet *packet, + struct hv_page_buffer *pb) +{ + u32 page_count = packet->page_buf_cnt; + dma_addr_t dma; + int i; + + if (!hv_is_isolation_supported()) + return 0; + + packet->dma_range = kcalloc(page_count, + sizeof(*packet->dma_range), + GFP_ATOMIC); + if (!packet->dma_range) + return -ENOMEM; + + for (i = 0; i < page_count; i++) { + char *src = phys_to_virt((pb[i].pfn << HV_HYP_PAGE_SHIFT) + + pb[i].offset); + u32 len = pb[i].len; + + dma = dma_map_single(&hv_dev->device, src, len, + DMA_TO_DEVICE); + if (dma_mapping_error(&hv_dev->device, dma)) { + kfree(packet->dma_range); + return -ENOMEM; + } + + /* pb[].offset and pb[].len are not changed during dma mapping + * and so not reassign. + */ + packet->dma_range[i].dma = dma; + packet->dma_range[i].mapping_size = len; + pb[i].pfn = dma >> HV_HYP_PAGE_SHIFT; + } + + return 0; +} + +static inline int netvsc_send_pkt( + struct hv_device *device, + struct hv_netvsc_packet *packet, + struct netvsc_device *net_device, + struct hv_page_buffer *pb, + struct sk_buff *skb) +{ + struct nvsp_message nvmsg; + struct nvsp_1_message_send_rndis_packet *rpkt = + &nvmsg.msg.v1_msg.send_rndis_pkt; + struct netvsc_channel * const nvchan = + &net_device->chan_table[packet->q_idx]; + struct vmbus_channel *out_channel = nvchan->channel; + struct net_device *ndev = hv_get_drvdata(device); + struct net_device_context *ndev_ctx = netdev_priv(ndev); + struct netdev_queue *txq = netdev_get_tx_queue(ndev, packet->q_idx); + u64 req_id; + int ret; + u32 ring_avail = hv_get_avail_to_write_percent(&out_channel->outbound); + + memset(&nvmsg, 0, sizeof(struct nvsp_message)); + nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT; + if (skb) + rpkt->channel_type = 0; /* 0 is RMC_DATA */ + else + rpkt->channel_type = 1; /* 1 is RMC_CONTROL */ + + rpkt->send_buf_section_index = packet->send_buf_index; + if (packet->send_buf_index == NETVSC_INVALID_INDEX) + rpkt->send_buf_section_size = 0; + else + rpkt->send_buf_section_size = packet->total_data_buflen; + + req_id = (ulong)skb; + + if (out_channel->rescind) + return -ENODEV; + + trace_nvsp_send_pkt(ndev, out_channel, rpkt); + + packet->dma_range = NULL; + if (packet->page_buf_cnt) { + if (packet->cp_partial) + pb += packet->rmsg_pgcnt; + + ret = netvsc_dma_map(ndev_ctx->device_ctx, packet, pb); + if (ret) { + ret = -EAGAIN; + goto exit; + } + + ret = vmbus_sendpacket_pagebuffer(out_channel, + pb, packet->page_buf_cnt, + &nvmsg, sizeof(nvmsg), + req_id); + + if (ret) + netvsc_dma_unmap(ndev_ctx->device_ctx, packet); + } else { + ret = vmbus_sendpacket(out_channel, + &nvmsg, sizeof(nvmsg), + req_id, VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + } + +exit: + if (ret == 0) { + atomic_inc_return(&nvchan->queue_sends); + + if (ring_avail < RING_AVAIL_PERCENT_LOWATER) { + netif_tx_stop_queue(txq); + ndev_ctx->eth_stats.stop_queue++; + } + } else if (ret == -EAGAIN) { + netif_tx_stop_queue(txq); + ndev_ctx->eth_stats.stop_queue++; + } else { + netdev_err(ndev, + "Unable to send packet pages %u len %u, ret %d\n", + packet->page_buf_cnt, packet->total_data_buflen, + ret); + } + + if (netif_tx_queue_stopped(txq) && + atomic_read(&nvchan->queue_sends) < 1 && + !net_device->tx_disable) { + netif_tx_wake_queue(txq); + ndev_ctx->eth_stats.wake_queue++; + if (ret == -EAGAIN) + ret = -ENOSPC; + } + + return ret; +} + +/* Move packet out of multi send data (msd), and clear msd */ +static inline void move_pkt_msd(struct hv_netvsc_packet **msd_send, + struct sk_buff **msd_skb, + struct multi_send_data *msdp) +{ + *msd_skb = msdp->skb; + *msd_send = msdp->pkt; + msdp->skb = NULL; + msdp->pkt = NULL; + msdp->count = 0; +} + +/* RCU already held by caller */ +/* Batching/bouncing logic is designed to attempt to optimize + * performance. + * + * For small, non-LSO packets we copy the packet to a send buffer + * which is pre-registered with the Hyper-V side. This enables the + * hypervisor to avoid remapping the aperture to access the packet + * descriptor and data. + * + * If we already started using a buffer and the netdev is transmitting + * a burst of packets, keep on copying into the buffer until it is + * full or we are done collecting a burst. If there is an existing + * buffer with space for the RNDIS descriptor but not the packet, copy + * the RNDIS descriptor to the buffer, keeping the packet in place. + * + * If we do batching and send more than one packet using a single + * NetVSC message, free the SKBs of the packets copied, except for the + * last packet. This is done to streamline the handling of the case + * where the last packet only had the RNDIS descriptor copied to the + * send buffer, with the data pointers included in the NetVSC message. + */ +int netvsc_send(struct net_device *ndev, + struct hv_netvsc_packet *packet, + struct rndis_message *rndis_msg, + struct hv_page_buffer *pb, + struct sk_buff *skb, + bool xdp_tx) +{ + struct net_device_context *ndev_ctx = netdev_priv(ndev); + struct netvsc_device *net_device + = rcu_dereference_bh(ndev_ctx->nvdev); + struct hv_device *device = ndev_ctx->device_ctx; + int ret = 0; + struct netvsc_channel *nvchan; + u32 pktlen = packet->total_data_buflen, msd_len = 0; + unsigned int section_index = NETVSC_INVALID_INDEX; + struct multi_send_data *msdp; + struct hv_netvsc_packet *msd_send = NULL, *cur_send = NULL; + struct sk_buff *msd_skb = NULL; + bool try_batch, xmit_more; + + /* If device is rescinded, return error and packet will get dropped. */ + if (unlikely(!net_device || net_device->destroy)) + return -ENODEV; + + nvchan = &net_device->chan_table[packet->q_idx]; + packet->send_buf_index = NETVSC_INVALID_INDEX; + packet->cp_partial = false; + + /* Send a control message or XDP packet directly without accessing + * msd (Multi-Send Data) field which may be changed during data packet + * processing. + */ + if (!skb || xdp_tx) + return netvsc_send_pkt(device, packet, net_device, pb, skb); + + /* batch packets in send buffer if possible */ + msdp = &nvchan->msd; + if (msdp->pkt) + msd_len = msdp->pkt->total_data_buflen; + + try_batch = msd_len > 0 && msdp->count < net_device->max_pkt; + if (try_batch && msd_len + pktlen + net_device->pkt_align < + net_device->send_section_size) { + section_index = msdp->pkt->send_buf_index; + + } else if (try_batch && msd_len + packet->rmsg_size < + net_device->send_section_size) { + section_index = msdp->pkt->send_buf_index; + packet->cp_partial = true; + + } else if (pktlen + net_device->pkt_align < + net_device->send_section_size) { + section_index = netvsc_get_next_send_section(net_device); + if (unlikely(section_index == NETVSC_INVALID_INDEX)) { + ++ndev_ctx->eth_stats.tx_send_full; + } else { + move_pkt_msd(&msd_send, &msd_skb, msdp); + msd_len = 0; + } + } + + /* Keep aggregating only if stack says more data is coming + * and not doing mixed modes send and not flow blocked + */ + xmit_more = netdev_xmit_more() && + !packet->cp_partial && + !netif_xmit_stopped(netdev_get_tx_queue(ndev, packet->q_idx)); + + if (section_index != NETVSC_INVALID_INDEX) { + netvsc_copy_to_send_buf(net_device, + section_index, msd_len, + packet, rndis_msg, pb, xmit_more); + + packet->send_buf_index = section_index; + + if (packet->cp_partial) { + packet->page_buf_cnt -= packet->rmsg_pgcnt; + packet->total_data_buflen = msd_len + packet->rmsg_size; + } else { + packet->page_buf_cnt = 0; + packet->total_data_buflen += msd_len; + } + + if (msdp->pkt) { + packet->total_packets += msdp->pkt->total_packets; + packet->total_bytes += msdp->pkt->total_bytes; + } + + if (msdp->skb) + dev_consume_skb_any(msdp->skb); + + if (xmit_more) { + msdp->skb = skb; + msdp->pkt = packet; + msdp->count++; + } else { + cur_send = packet; + msdp->skb = NULL; + msdp->pkt = NULL; + msdp->count = 0; + } + } else { + move_pkt_msd(&msd_send, &msd_skb, msdp); + cur_send = packet; + } + + if (msd_send) { + int m_ret = netvsc_send_pkt(device, msd_send, net_device, + NULL, msd_skb); + + if (m_ret != 0) { + netvsc_free_send_slot(net_device, + msd_send->send_buf_index); + dev_kfree_skb_any(msd_skb); + } + } + + if (cur_send) + ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb); + + if (ret != 0 && section_index != NETVSC_INVALID_INDEX) + netvsc_free_send_slot(net_device, section_index); + + return ret; +} + +/* Send pending recv completions */ +static int send_recv_completions(struct net_device *ndev, + struct netvsc_device *nvdev, + struct netvsc_channel *nvchan) +{ + struct multi_recv_comp *mrc = &nvchan->mrc; + struct recv_comp_msg { + struct nvsp_message_header hdr; + u32 status; + } __packed; + struct recv_comp_msg msg = { + .hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE, + }; + int ret; + + while (mrc->first != mrc->next) { + const struct recv_comp_data *rcd + = mrc->slots + mrc->first; + + msg.status = rcd->status; + ret = vmbus_sendpacket(nvchan->channel, &msg, sizeof(msg), + rcd->tid, VM_PKT_COMP, 0); + if (unlikely(ret)) { + struct net_device_context *ndev_ctx = netdev_priv(ndev); + + ++ndev_ctx->eth_stats.rx_comp_busy; + return ret; + } + + if (++mrc->first == nvdev->recv_completion_cnt) + mrc->first = 0; + } + + /* receive completion ring has been emptied */ + if (unlikely(nvdev->destroy)) + wake_up(&nvdev->wait_drain); + + return 0; +} + +/* Count how many receive completions are outstanding */ +static void recv_comp_slot_avail(const struct netvsc_device *nvdev, + const struct multi_recv_comp *mrc, + u32 *filled, u32 *avail) +{ + u32 count = nvdev->recv_completion_cnt; + + if (mrc->next >= mrc->first) + *filled = mrc->next - mrc->first; + else + *filled = (count - mrc->first) + mrc->next; + + *avail = count - *filled - 1; +} + +/* Add receive complete to ring to send to host. */ +static void enq_receive_complete(struct net_device *ndev, + struct netvsc_device *nvdev, u16 q_idx, + u64 tid, u32 status) +{ + struct netvsc_channel *nvchan = &nvdev->chan_table[q_idx]; + struct multi_recv_comp *mrc = &nvchan->mrc; + struct recv_comp_data *rcd; + u32 filled, avail; + + recv_comp_slot_avail(nvdev, mrc, &filled, &avail); + + if (unlikely(filled > NAPI_POLL_WEIGHT)) { + send_recv_completions(ndev, nvdev, nvchan); + recv_comp_slot_avail(nvdev, mrc, &filled, &avail); + } + + if (unlikely(!avail)) { + netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n", + q_idx, tid); + return; + } + + rcd = mrc->slots + mrc->next; + rcd->tid = tid; + rcd->status = status; + + if (++mrc->next == nvdev->recv_completion_cnt) + mrc->next = 0; +} + +static int netvsc_receive(struct net_device *ndev, + struct netvsc_device *net_device, + struct netvsc_channel *nvchan, + const struct vmpacket_descriptor *desc) +{ + struct net_device_context *net_device_ctx = netdev_priv(ndev); + struct vmbus_channel *channel = nvchan->channel; + const struct vmtransfer_page_packet_header *vmxferpage_packet + = container_of(desc, const struct vmtransfer_page_packet_header, d); + const struct nvsp_message *nvsp = hv_pkt_data(desc); + u32 msglen = hv_pkt_datalen(desc); + u16 q_idx = channel->offermsg.offer.sub_channel_index; + char *recv_buf = net_device->recv_buf; + u32 status = NVSP_STAT_SUCCESS; + int i; + int count = 0; + + /* Ensure packet is big enough to read header fields */ + if (msglen < sizeof(struct nvsp_message_header)) { + netif_err(net_device_ctx, rx_err, ndev, + "invalid nvsp header, length too small: %u\n", + msglen); + return 0; + } + + /* Make sure this is a valid nvsp packet */ + if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) { + netif_err(net_device_ctx, rx_err, ndev, + "Unknown nvsp packet type received %u\n", + nvsp->hdr.msg_type); + return 0; + } + + /* Validate xfer page pkt header */ + if ((desc->offset8 << 3) < sizeof(struct vmtransfer_page_packet_header)) { + netif_err(net_device_ctx, rx_err, ndev, + "Invalid xfer page pkt, offset too small: %u\n", + desc->offset8 << 3); + return 0; + } + + if (unlikely(vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID)) { + netif_err(net_device_ctx, rx_err, ndev, + "Invalid xfer page set id - expecting %x got %x\n", + NETVSC_RECEIVE_BUFFER_ID, + vmxferpage_packet->xfer_pageset_id); + return 0; + } + + count = vmxferpage_packet->range_cnt; + + /* Check count for a valid value */ + if (NETVSC_XFER_HEADER_SIZE(count) > desc->offset8 << 3) { + netif_err(net_device_ctx, rx_err, ndev, + "Range count is not valid: %d\n", + count); + return 0; + } + + /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */ + for (i = 0; i < count; i++) { + u32 offset = vmxferpage_packet->ranges[i].byte_offset; + u32 buflen = vmxferpage_packet->ranges[i].byte_count; + void *data; + int ret; + + if (unlikely(offset > net_device->recv_buf_size || + buflen > net_device->recv_buf_size - offset)) { + nvchan->rsc.cnt = 0; + status = NVSP_STAT_FAIL; + netif_err(net_device_ctx, rx_err, ndev, + "Packet offset:%u + len:%u too big\n", + offset, buflen); + + continue; + } + + /* We're going to copy (sections of) the packet into nvchan->recv_buf; + * make sure that nvchan->recv_buf is large enough to hold the packet. + */ + if (unlikely(buflen > net_device->recv_section_size)) { + nvchan->rsc.cnt = 0; + status = NVSP_STAT_FAIL; + netif_err(net_device_ctx, rx_err, ndev, + "Packet too big: buflen=%u recv_section_size=%u\n", + buflen, net_device->recv_section_size); + + continue; + } + + data = recv_buf + offset; + + nvchan->rsc.is_last = (i == count - 1); + + trace_rndis_recv(ndev, q_idx, data); + + /* Pass it to the upper layer */ + ret = rndis_filter_receive(ndev, net_device, + nvchan, data, buflen); + + if (unlikely(ret != NVSP_STAT_SUCCESS)) { + /* Drop incomplete packet */ + nvchan->rsc.cnt = 0; + status = NVSP_STAT_FAIL; + } + } + + enq_receive_complete(ndev, net_device, q_idx, + vmxferpage_packet->d.trans_id, status); + + return count; +} + +static void netvsc_send_table(struct net_device *ndev, + struct netvsc_device *nvscdev, + const struct nvsp_message *nvmsg, + u32 msglen) +{ + struct net_device_context *net_device_ctx = netdev_priv(ndev); + u32 count, offset, *tab; + int i; + + /* Ensure packet is big enough to read send_table fields */ + if (msglen < sizeof(struct nvsp_message_header) + + sizeof(struct nvsp_5_send_indirect_table)) { + netdev_err(ndev, "nvsp_v5_msg length too small: %u\n", msglen); + return; + } + + count = nvmsg->msg.v5_msg.send_table.count; + offset = nvmsg->msg.v5_msg.send_table.offset; + + if (count != VRSS_SEND_TAB_SIZE) { + netdev_err(ndev, "Received wrong send-table size:%u\n", count); + return; + } + + /* If negotiated version <= NVSP_PROTOCOL_VERSION_6, the offset may be + * wrong due to a host bug. So fix the offset here. + */ + if (nvscdev->nvsp_version <= NVSP_PROTOCOL_VERSION_6 && + msglen >= sizeof(struct nvsp_message_header) + + sizeof(union nvsp_6_message_uber) + count * sizeof(u32)) + offset = sizeof(struct nvsp_message_header) + + sizeof(union nvsp_6_message_uber); + + /* Boundary check for all versions */ + if (msglen < count * sizeof(u32) || offset > msglen - count * sizeof(u32)) { + netdev_err(ndev, "Received send-table offset too big:%u\n", + offset); + return; + } + + tab = (void *)nvmsg + offset; + + for (i = 0; i < count; i++) + net_device_ctx->tx_table[i] = tab[i]; +} + +static void netvsc_send_vf(struct net_device *ndev, + const struct nvsp_message *nvmsg, + u32 msglen) +{ + struct net_device_context *net_device_ctx = netdev_priv(ndev); + + /* Ensure packet is big enough to read its fields */ + if (msglen < sizeof(struct nvsp_message_header) + + sizeof(struct nvsp_4_send_vf_association)) { + netdev_err(ndev, "nvsp_v4_msg length too small: %u\n", msglen); + return; + } + + net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated; + net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial; + + if (net_device_ctx->vf_alloc) + complete(&net_device_ctx->vf_add); + + netdev_info(ndev, "VF slot %u %s\n", + net_device_ctx->vf_serial, + net_device_ctx->vf_alloc ? "added" : "removed"); +} + +static void netvsc_receive_inband(struct net_device *ndev, + struct netvsc_device *nvscdev, + const struct vmpacket_descriptor *desc) +{ + const struct nvsp_message *nvmsg = hv_pkt_data(desc); + u32 msglen = hv_pkt_datalen(desc); + + /* Ensure packet is big enough to read header fields */ + if (msglen < sizeof(struct nvsp_message_header)) { + netdev_err(ndev, "inband nvsp_message length too small: %u\n", msglen); + return; + } + + switch (nvmsg->hdr.msg_type) { + case NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE: + netvsc_send_table(ndev, nvscdev, nvmsg, msglen); + break; + + case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION: + if (hv_is_isolation_supported()) + netdev_err(ndev, "Ignore VF_ASSOCIATION msg from the host supporting isolation\n"); + else + netvsc_send_vf(ndev, nvmsg, msglen); + break; + } +} + +static int netvsc_process_raw_pkt(struct hv_device *device, + struct netvsc_channel *nvchan, + struct netvsc_device *net_device, + struct net_device *ndev, + const struct vmpacket_descriptor *desc, + int budget) +{ + struct vmbus_channel *channel = nvchan->channel; + const struct nvsp_message *nvmsg = hv_pkt_data(desc); + + trace_nvsp_recv(ndev, channel, nvmsg); + + switch (desc->type) { + case VM_PKT_COMP: + netvsc_send_completion(ndev, net_device, channel, desc, budget); + break; + + case VM_PKT_DATA_USING_XFER_PAGES: + return netvsc_receive(ndev, net_device, nvchan, desc); + + case VM_PKT_DATA_INBAND: + netvsc_receive_inband(ndev, net_device, desc); + break; + + default: + netdev_err(ndev, "unhandled packet type %d, tid %llx\n", + desc->type, desc->trans_id); + break; + } + + return 0; +} + +static struct hv_device *netvsc_channel_to_device(struct vmbus_channel *channel) +{ + struct vmbus_channel *primary = channel->primary_channel; + + return primary ? primary->device_obj : channel->device_obj; +} + +/* Network processing softirq + * Process data in incoming ring buffer from host + * Stops when ring is empty or budget is met or exceeded. + */ +int netvsc_poll(struct napi_struct *napi, int budget) +{ + struct netvsc_channel *nvchan + = container_of(napi, struct netvsc_channel, napi); + struct netvsc_device *net_device = nvchan->net_device; + struct vmbus_channel *channel = nvchan->channel; + struct hv_device *device = netvsc_channel_to_device(channel); + struct net_device *ndev = hv_get_drvdata(device); + int work_done = 0; + int ret; + + /* If starting a new interval */ + if (!nvchan->desc) + nvchan->desc = hv_pkt_iter_first(channel); + + nvchan->xdp_flush = false; + + while (nvchan->desc && work_done < budget) { + work_done += netvsc_process_raw_pkt(device, nvchan, net_device, + ndev, nvchan->desc, budget); + nvchan->desc = hv_pkt_iter_next(channel, nvchan->desc); + } + + if (nvchan->xdp_flush) + xdp_do_flush(); + + /* Send any pending receive completions */ + ret = send_recv_completions(ndev, net_device, nvchan); + + /* If it did not exhaust NAPI budget this time + * and not doing busy poll + * then re-enable host interrupts + * and reschedule if ring is not empty + * or sending receive completion failed. + */ + if (work_done < budget && + napi_complete_done(napi, work_done) && + (ret || hv_end_read(&channel->inbound)) && + napi_schedule_prep(napi)) { + hv_begin_read(&channel->inbound); + __napi_schedule(napi); + } + + /* Driver may overshoot since multiple packets per descriptor */ + return min(work_done, budget); +} + +/* Call back when data is available in host ring buffer. + * Processing is deferred until network softirq (NAPI) + */ +void netvsc_channel_cb(void *context) +{ + struct netvsc_channel *nvchan = context; + struct vmbus_channel *channel = nvchan->channel; + struct hv_ring_buffer_info *rbi = &channel->inbound; + + /* preload first vmpacket descriptor */ + prefetch(hv_get_ring_buffer(rbi) + rbi->priv_read_index); + + if (napi_schedule_prep(&nvchan->napi)) { + /* disable interrupts from host */ + hv_begin_read(rbi); + + __napi_schedule_irqoff(&nvchan->napi); + } +} + +/* + * netvsc_device_add - Callback when the device belonging to this + * driver is added + */ +struct netvsc_device *netvsc_device_add(struct hv_device *device, + const struct netvsc_device_info *device_info) +{ + int i, ret = 0; + struct netvsc_device *net_device; + struct net_device *ndev = hv_get_drvdata(device); + struct net_device_context *net_device_ctx = netdev_priv(ndev); + + net_device = alloc_net_device(); + if (!net_device) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < VRSS_SEND_TAB_SIZE; i++) + net_device_ctx->tx_table[i] = 0; + + /* Because the device uses NAPI, all the interrupt batching and + * control is done via Net softirq, not the channel handling + */ + set_channel_read_mode(device->channel, HV_CALL_ISR); + + /* If we're reopening the device we may have multiple queues, fill the + * chn_table with the default channel to use it before subchannels are + * opened. + * Initialize the channel state before we open; + * we can be interrupted as soon as we open the channel. + */ + + for (i = 0; i < VRSS_CHANNEL_MAX; i++) { + struct netvsc_channel *nvchan = &net_device->chan_table[i]; + + nvchan->channel = device->channel; + nvchan->net_device = net_device; + u64_stats_init(&nvchan->tx_stats.syncp); + u64_stats_init(&nvchan->rx_stats.syncp); + + ret = xdp_rxq_info_reg(&nvchan->xdp_rxq, ndev, i, 0); + + if (ret) { + netdev_err(ndev, "xdp_rxq_info_reg fail: %d\n", ret); + goto cleanup2; + } + + ret = xdp_rxq_info_reg_mem_model(&nvchan->xdp_rxq, + MEM_TYPE_PAGE_SHARED, NULL); + + if (ret) { + netdev_err(ndev, "xdp reg_mem_model fail: %d\n", ret); + goto cleanup2; + } + } + + /* Enable NAPI handler before init callbacks */ + netif_napi_add(ndev, &net_device->chan_table[0].napi, netvsc_poll); + + /* Open the channel */ + device->channel->next_request_id_callback = vmbus_next_request_id; + device->channel->request_addr_callback = vmbus_request_addr; + device->channel->rqstor_size = netvsc_rqstor_size(netvsc_ring_bytes); + device->channel->max_pkt_size = NETVSC_MAX_PKT_SIZE; + + ret = vmbus_open(device->channel, netvsc_ring_bytes, + netvsc_ring_bytes, NULL, 0, + netvsc_channel_cb, net_device->chan_table); + + if (ret != 0) { + netdev_err(ndev, "unable to open channel: %d\n", ret); + goto cleanup; + } + + /* Channel is opened */ + netdev_dbg(ndev, "hv_netvsc channel opened successfully\n"); + + napi_enable(&net_device->chan_table[0].napi); + + /* Connect with the NetVsp */ + ret = netvsc_connect_vsp(device, net_device, device_info); + if (ret != 0) { + netdev_err(ndev, + "unable to connect to NetVSP - %d\n", ret); + goto close; + } + + /* Writing nvdev pointer unlocks netvsc_send(), make sure chn_table is + * populated. + */ + rcu_assign_pointer(net_device_ctx->nvdev, net_device); + + return net_device; + +close: + RCU_INIT_POINTER(net_device_ctx->nvdev, NULL); + napi_disable(&net_device->chan_table[0].napi); + + /* Now, we can close the channel safely */ + vmbus_close(device->channel); + +cleanup: + netif_napi_del(&net_device->chan_table[0].napi); + +cleanup2: + if (net_device->recv_original_buf) + hv_unmap_memory(net_device->recv_buf); + + if (net_device->send_original_buf) + hv_unmap_memory(net_device->send_buf); + + free_netvsc_device(&net_device->rcu); + + return ERR_PTR(ret); +} diff --git a/drivers/net/hyperv/netvsc_bpf.c b/drivers/net/hyperv/netvsc_bpf.c new file mode 100644 index 000000000..4a9522689 --- /dev/null +++ b/drivers/net/hyperv/netvsc_bpf.c @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2019, Microsoft Corporation. + * + * Author: + * Haiyang Zhang <haiyangz@microsoft.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/netdevice.h> +#include <linux/etherdevice.h> +#include <linux/ethtool.h> +#include <linux/netpoll.h> +#include <linux/bpf.h> +#include <linux/bpf_trace.h> +#include <linux/kernel.h> +#include <net/xdp.h> + +#include <linux/mutex.h> +#include <linux/rtnetlink.h> + +#include "hyperv_net.h" + +u32 netvsc_run_xdp(struct net_device *ndev, struct netvsc_channel *nvchan, + struct xdp_buff *xdp) +{ + struct netvsc_stats_rx *rx_stats = &nvchan->rx_stats; + void *data = nvchan->rsc.data[0]; + u32 len = nvchan->rsc.len[0]; + struct page *page = NULL; + struct bpf_prog *prog; + u32 act = XDP_PASS; + bool drop = true; + + xdp->data_hard_start = NULL; + + rcu_read_lock(); + prog = rcu_dereference(nvchan->bpf_prog); + + if (!prog) + goto out; + + /* Ensure that the below memcpy() won't overflow the page buffer. */ + if (len > ndev->mtu + ETH_HLEN) { + act = XDP_DROP; + goto out; + } + + /* allocate page buffer for data */ + page = alloc_page(GFP_ATOMIC); + if (!page) { + act = XDP_DROP; + goto out; + } + + xdp_init_buff(xdp, PAGE_SIZE, &nvchan->xdp_rxq); + xdp_prepare_buff(xdp, page_address(page), NETVSC_XDP_HDRM, len, false); + + memcpy(xdp->data, data, len); + + act = bpf_prog_run_xdp(prog, xdp); + + switch (act) { + case XDP_PASS: + case XDP_TX: + drop = false; + break; + + case XDP_DROP: + break; + + case XDP_REDIRECT: + if (!xdp_do_redirect(ndev, xdp, prog)) { + nvchan->xdp_flush = true; + drop = false; + + u64_stats_update_begin(&rx_stats->syncp); + + rx_stats->xdp_redirect++; + rx_stats->packets++; + rx_stats->bytes += nvchan->rsc.pktlen; + + u64_stats_update_end(&rx_stats->syncp); + + break; + } else { + u64_stats_update_begin(&rx_stats->syncp); + rx_stats->xdp_drop++; + u64_stats_update_end(&rx_stats->syncp); + } + + fallthrough; + + case XDP_ABORTED: + trace_xdp_exception(ndev, prog, act); + break; + + default: + bpf_warn_invalid_xdp_action(ndev, prog, act); + } + +out: + rcu_read_unlock(); + + if (page && drop) { + __free_page(page); + xdp->data_hard_start = NULL; + } + + return act; +} + +unsigned int netvsc_xdp_fraglen(unsigned int len) +{ + return SKB_DATA_ALIGN(len) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); +} + +struct bpf_prog *netvsc_xdp_get(struct netvsc_device *nvdev) +{ + return rtnl_dereference(nvdev->chan_table[0].bpf_prog); +} + +int netvsc_xdp_set(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack, + struct netvsc_device *nvdev) +{ + struct bpf_prog *old_prog; + int buf_max, i; + + old_prog = netvsc_xdp_get(nvdev); + + if (!old_prog && !prog) + return 0; + + buf_max = NETVSC_XDP_HDRM + netvsc_xdp_fraglen(dev->mtu + ETH_HLEN); + if (prog && buf_max > PAGE_SIZE) { + netdev_err(dev, "XDP: mtu:%u too large, buf_max:%u\n", + dev->mtu, buf_max); + NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large"); + + return -EOPNOTSUPP; + } + + if (prog && (dev->features & NETIF_F_LRO)) { + netdev_err(dev, "XDP: not support LRO\n"); + NL_SET_ERR_MSG_MOD(extack, "XDP: not support LRO"); + + return -EOPNOTSUPP; + } + + if (prog) + bpf_prog_add(prog, nvdev->num_chn - 1); + + for (i = 0; i < nvdev->num_chn; i++) + rcu_assign_pointer(nvdev->chan_table[i].bpf_prog, prog); + + if (old_prog) + for (i = 0; i < nvdev->num_chn; i++) + bpf_prog_put(old_prog); + + return 0; +} + +int netvsc_vf_setxdp(struct net_device *vf_netdev, struct bpf_prog *prog) +{ + struct netdev_bpf xdp; + int ret; + + ASSERT_RTNL(); + + if (!vf_netdev) + return 0; + + if (!vf_netdev->netdev_ops->ndo_bpf) + return 0; + + memset(&xdp, 0, sizeof(xdp)); + + if (prog) + bpf_prog_inc(prog); + + xdp.command = XDP_SETUP_PROG; + xdp.prog = prog; + + ret = vf_netdev->netdev_ops->ndo_bpf(vf_netdev, &xdp); + + if (ret && prog) + bpf_prog_put(prog); + + return ret; +} + +int netvsc_bpf(struct net_device *dev, struct netdev_bpf *bpf) +{ + struct net_device_context *ndevctx = netdev_priv(dev); + struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); + struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev); + struct netlink_ext_ack *extack = bpf->extack; + int ret; + + if (!nvdev || nvdev->destroy) { + return -ENODEV; + } + + switch (bpf->command) { + case XDP_SETUP_PROG: + ret = netvsc_xdp_set(dev, bpf->prog, extack, nvdev); + + if (ret) + return ret; + + ret = netvsc_vf_setxdp(vf_netdev, bpf->prog); + + if (ret) { + netdev_err(dev, "vf_setxdp failed:%d\n", ret); + NL_SET_ERR_MSG_MOD(extack, "vf_setxdp failed"); + + netvsc_xdp_set(dev, NULL, extack, nvdev); + } + + return ret; + + default: + return -EINVAL; + } +} + +static int netvsc_ndoxdp_xmit_fm(struct net_device *ndev, + struct xdp_frame *frame, u16 q_idx) +{ + struct sk_buff *skb; + + skb = xdp_build_skb_from_frame(frame, ndev); + if (unlikely(!skb)) + return -ENOMEM; + + netvsc_get_hash(skb, netdev_priv(ndev)); + + skb_record_rx_queue(skb, q_idx); + + netvsc_xdp_xmit(skb, ndev); + + return 0; +} + +int netvsc_ndoxdp_xmit(struct net_device *ndev, int n, + struct xdp_frame **frames, u32 flags) +{ + struct net_device_context *ndev_ctx = netdev_priv(ndev); + const struct net_device_ops *vf_ops; + struct netvsc_stats_tx *tx_stats; + struct netvsc_device *nvsc_dev; + struct net_device *vf_netdev; + int i, count = 0; + u16 q_idx; + + /* Don't transmit if netvsc_device is gone */ + nvsc_dev = rcu_dereference_bh(ndev_ctx->nvdev); + if (unlikely(!nvsc_dev || nvsc_dev->destroy)) + return 0; + + /* If VF is present and up then redirect packets to it. + * Skip the VF if it is marked down or has no carrier. + * If netpoll is in uses, then VF can not be used either. + */ + vf_netdev = rcu_dereference_bh(ndev_ctx->vf_netdev); + if (vf_netdev && netif_running(vf_netdev) && + netif_carrier_ok(vf_netdev) && !netpoll_tx_running(ndev) && + vf_netdev->netdev_ops->ndo_xdp_xmit && + ndev_ctx->data_path_is_vf) { + vf_ops = vf_netdev->netdev_ops; + return vf_ops->ndo_xdp_xmit(vf_netdev, n, frames, flags); + } + + q_idx = smp_processor_id() % ndev->real_num_tx_queues; + + for (i = 0; i < n; i++) { + if (netvsc_ndoxdp_xmit_fm(ndev, frames[i], q_idx)) + break; + + count++; + } + + tx_stats = &nvsc_dev->chan_table[q_idx].tx_stats; + + u64_stats_update_begin(&tx_stats->syncp); + tx_stats->xdp_xmit += count; + u64_stats_update_end(&tx_stats->syncp); + + return count; +} diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c new file mode 100644 index 000000000..c1aac6ceb --- /dev/null +++ b/drivers/net/hyperv/netvsc_drv.c @@ -0,0 +1,2823 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2009, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/init.h> +#include <linux/atomic.h> +#include <linux/ethtool.h> +#include <linux/module.h> +#include <linux/highmem.h> +#include <linux/device.h> +#include <linux/io.h> +#include <linux/delay.h> +#include <linux/netdevice.h> +#include <linux/inetdevice.h> +#include <linux/etherdevice.h> +#include <linux/pci.h> +#include <linux/skbuff.h> +#include <linux/if_vlan.h> +#include <linux/in.h> +#include <linux/slab.h> +#include <linux/rtnetlink.h> +#include <linux/netpoll.h> +#include <linux/bpf.h> + +#include <net/arp.h> +#include <net/route.h> +#include <net/sock.h> +#include <net/pkt_sched.h> +#include <net/checksum.h> +#include <net/ip6_checksum.h> + +#include "hyperv_net.h" + +#define RING_SIZE_MIN 64 + +#define LINKCHANGE_INT (2 * HZ) +#define VF_TAKEOVER_INT (HZ / 10) + +static unsigned int ring_size __ro_after_init = 128; +module_param(ring_size, uint, 0444); +MODULE_PARM_DESC(ring_size, "Ring buffer size (# of 4K pages)"); +unsigned int netvsc_ring_bytes __ro_after_init; + +static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE | + NETIF_MSG_LINK | NETIF_MSG_IFUP | + NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR | + NETIF_MSG_TX_ERR; + +static int debug = -1; +module_param(debug, int, 0444); +MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); + +static LIST_HEAD(netvsc_dev_list); + +static void netvsc_change_rx_flags(struct net_device *net, int change) +{ + struct net_device_context *ndev_ctx = netdev_priv(net); + struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev); + int inc; + + if (!vf_netdev) + return; + + if (change & IFF_PROMISC) { + inc = (net->flags & IFF_PROMISC) ? 1 : -1; + dev_set_promiscuity(vf_netdev, inc); + } + + if (change & IFF_ALLMULTI) { + inc = (net->flags & IFF_ALLMULTI) ? 1 : -1; + dev_set_allmulti(vf_netdev, inc); + } +} + +static void netvsc_set_rx_mode(struct net_device *net) +{ + struct net_device_context *ndev_ctx = netdev_priv(net); + struct net_device *vf_netdev; + struct netvsc_device *nvdev; + + rcu_read_lock(); + vf_netdev = rcu_dereference(ndev_ctx->vf_netdev); + if (vf_netdev) { + dev_uc_sync(vf_netdev, net); + dev_mc_sync(vf_netdev, net); + } + + nvdev = rcu_dereference(ndev_ctx->nvdev); + if (nvdev) + rndis_filter_update(nvdev); + rcu_read_unlock(); +} + +static void netvsc_tx_enable(struct netvsc_device *nvscdev, + struct net_device *ndev) +{ + nvscdev->tx_disable = false; + virt_wmb(); /* ensure queue wake up mechanism is on */ + + netif_tx_wake_all_queues(ndev); +} + +static int netvsc_open(struct net_device *net) +{ + struct net_device_context *ndev_ctx = netdev_priv(net); + struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev); + struct netvsc_device *nvdev = rtnl_dereference(ndev_ctx->nvdev); + struct rndis_device *rdev; + int ret = 0; + + netif_carrier_off(net); + + /* Open up the device */ + ret = rndis_filter_open(nvdev); + if (ret != 0) { + netdev_err(net, "unable to open device (ret %d).\n", ret); + return ret; + } + + rdev = nvdev->extension; + if (!rdev->link_state) { + netif_carrier_on(net); + netvsc_tx_enable(nvdev, net); + } + + if (vf_netdev) { + /* Setting synthetic device up transparently sets + * slave as up. If open fails, then slave will be + * still be offline (and not used). + */ + ret = dev_open(vf_netdev, NULL); + if (ret) + netdev_warn(net, + "unable to open slave: %s: %d\n", + vf_netdev->name, ret); + } + return 0; +} + +static int netvsc_wait_until_empty(struct netvsc_device *nvdev) +{ + unsigned int retry = 0; + int i; + + /* Ensure pending bytes in ring are read */ + for (;;) { + u32 aread = 0; + + for (i = 0; i < nvdev->num_chn; i++) { + struct vmbus_channel *chn + = nvdev->chan_table[i].channel; + + if (!chn) + continue; + + /* make sure receive not running now */ + napi_synchronize(&nvdev->chan_table[i].napi); + + aread = hv_get_bytes_to_read(&chn->inbound); + if (aread) + break; + + aread = hv_get_bytes_to_read(&chn->outbound); + if (aread) + break; + } + + if (aread == 0) + return 0; + + if (++retry > RETRY_MAX) + return -ETIMEDOUT; + + usleep_range(RETRY_US_LO, RETRY_US_HI); + } +} + +static void netvsc_tx_disable(struct netvsc_device *nvscdev, + struct net_device *ndev) +{ + if (nvscdev) { + nvscdev->tx_disable = true; + virt_wmb(); /* ensure txq will not wake up after stop */ + } + + netif_tx_disable(ndev); +} + +static int netvsc_close(struct net_device *net) +{ + struct net_device_context *net_device_ctx = netdev_priv(net); + struct net_device *vf_netdev + = rtnl_dereference(net_device_ctx->vf_netdev); + struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev); + int ret; + + netvsc_tx_disable(nvdev, net); + + /* No need to close rndis filter if it is removed already */ + if (!nvdev) + return 0; + + ret = rndis_filter_close(nvdev); + if (ret != 0) { + netdev_err(net, "unable to close device (ret %d).\n", ret); + return ret; + } + + ret = netvsc_wait_until_empty(nvdev); + if (ret) + netdev_err(net, "Ring buffer not empty after closing rndis\n"); + + if (vf_netdev) + dev_close(vf_netdev); + + return ret; +} + +static inline void *init_ppi_data(struct rndis_message *msg, + u32 ppi_size, u32 pkt_type) +{ + struct rndis_packet *rndis_pkt = &msg->msg.pkt; + struct rndis_per_packet_info *ppi; + + rndis_pkt->data_offset += ppi_size; + ppi = (void *)rndis_pkt + rndis_pkt->per_pkt_info_offset + + rndis_pkt->per_pkt_info_len; + + ppi->size = ppi_size; + ppi->type = pkt_type; + ppi->internal = 0; + ppi->ppi_offset = sizeof(struct rndis_per_packet_info); + + rndis_pkt->per_pkt_info_len += ppi_size; + + return ppi + 1; +} + +static inline int netvsc_get_tx_queue(struct net_device *ndev, + struct sk_buff *skb, int old_idx) +{ + const struct net_device_context *ndc = netdev_priv(ndev); + struct sock *sk = skb->sk; + int q_idx; + + q_idx = ndc->tx_table[netvsc_get_hash(skb, ndc) & + (VRSS_SEND_TAB_SIZE - 1)]; + + /* If queue index changed record the new value */ + if (q_idx != old_idx && + sk && sk_fullsock(sk) && rcu_access_pointer(sk->sk_dst_cache)) + sk_tx_queue_set(sk, q_idx); + + return q_idx; +} + +/* + * Select queue for transmit. + * + * If a valid queue has already been assigned, then use that. + * Otherwise compute tx queue based on hash and the send table. + * + * This is basically similar to default (netdev_pick_tx) with the added step + * of using the host send_table when no other queue has been assigned. + * + * TODO support XPS - but get_xps_queue not exported + */ +static u16 netvsc_pick_tx(struct net_device *ndev, struct sk_buff *skb) +{ + int q_idx = sk_tx_queue_get(skb->sk); + + if (q_idx < 0 || skb->ooo_okay || q_idx >= ndev->real_num_tx_queues) { + /* If forwarding a packet, we use the recorded queue when + * available for better cache locality. + */ + if (skb_rx_queue_recorded(skb)) + q_idx = skb_get_rx_queue(skb); + else + q_idx = netvsc_get_tx_queue(ndev, skb, q_idx); + } + + return q_idx; +} + +static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb, + struct net_device *sb_dev) +{ + struct net_device_context *ndc = netdev_priv(ndev); + struct net_device *vf_netdev; + u16 txq; + + rcu_read_lock(); + vf_netdev = rcu_dereference(ndc->vf_netdev); + if (vf_netdev) { + const struct net_device_ops *vf_ops = vf_netdev->netdev_ops; + + if (vf_ops->ndo_select_queue) + txq = vf_ops->ndo_select_queue(vf_netdev, skb, sb_dev); + else + txq = netdev_pick_tx(vf_netdev, skb, NULL); + + /* Record the queue selected by VF so that it can be + * used for common case where VF has more queues than + * the synthetic device. + */ + qdisc_skb_cb(skb)->slave_dev_queue_mapping = txq; + } else { + txq = netvsc_pick_tx(ndev, skb); + } + rcu_read_unlock(); + + while (txq >= ndev->real_num_tx_queues) + txq -= ndev->real_num_tx_queues; + + return txq; +} + +static u32 fill_pg_buf(unsigned long hvpfn, u32 offset, u32 len, + struct hv_page_buffer *pb) +{ + int j = 0; + + hvpfn += offset >> HV_HYP_PAGE_SHIFT; + offset = offset & ~HV_HYP_PAGE_MASK; + + while (len > 0) { + unsigned long bytes; + + bytes = HV_HYP_PAGE_SIZE - offset; + if (bytes > len) + bytes = len; + pb[j].pfn = hvpfn; + pb[j].offset = offset; + pb[j].len = bytes; + + offset += bytes; + len -= bytes; + + if (offset == HV_HYP_PAGE_SIZE && len) { + hvpfn++; + offset = 0; + j++; + } + } + + return j + 1; +} + +static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb, + struct hv_netvsc_packet *packet, + struct hv_page_buffer *pb) +{ + u32 slots_used = 0; + char *data = skb->data; + int frags = skb_shinfo(skb)->nr_frags; + int i; + + /* The packet is laid out thus: + * 1. hdr: RNDIS header and PPI + * 2. skb linear data + * 3. skb fragment data + */ + slots_used += fill_pg_buf(virt_to_hvpfn(hdr), + offset_in_hvpage(hdr), + len, + &pb[slots_used]); + + packet->rmsg_size = len; + packet->rmsg_pgcnt = slots_used; + + slots_used += fill_pg_buf(virt_to_hvpfn(data), + offset_in_hvpage(data), + skb_headlen(skb), + &pb[slots_used]); + + for (i = 0; i < frags; i++) { + skb_frag_t *frag = skb_shinfo(skb)->frags + i; + + slots_used += fill_pg_buf(page_to_hvpfn(skb_frag_page(frag)), + skb_frag_off(frag), + skb_frag_size(frag), + &pb[slots_used]); + } + return slots_used; +} + +static int count_skb_frag_slots(struct sk_buff *skb) +{ + int i, frags = skb_shinfo(skb)->nr_frags; + int pages = 0; + + for (i = 0; i < frags; i++) { + skb_frag_t *frag = skb_shinfo(skb)->frags + i; + unsigned long size = skb_frag_size(frag); + unsigned long offset = skb_frag_off(frag); + + /* Skip unused frames from start of page */ + offset &= ~HV_HYP_PAGE_MASK; + pages += HVPFN_UP(offset + size); + } + return pages; +} + +static int netvsc_get_slots(struct sk_buff *skb) +{ + char *data = skb->data; + unsigned int offset = offset_in_hvpage(data); + unsigned int len = skb_headlen(skb); + int slots; + int frag_slots; + + slots = DIV_ROUND_UP(offset + len, HV_HYP_PAGE_SIZE); + frag_slots = count_skb_frag_slots(skb); + return slots + frag_slots; +} + +static u32 net_checksum_info(struct sk_buff *skb) +{ + if (skb->protocol == htons(ETH_P_IP)) { + struct iphdr *ip = ip_hdr(skb); + + if (ip->protocol == IPPROTO_TCP) + return TRANSPORT_INFO_IPV4_TCP; + else if (ip->protocol == IPPROTO_UDP) + return TRANSPORT_INFO_IPV4_UDP; + } else { + struct ipv6hdr *ip6 = ipv6_hdr(skb); + + if (ip6->nexthdr == IPPROTO_TCP) + return TRANSPORT_INFO_IPV6_TCP; + else if (ip6->nexthdr == IPPROTO_UDP) + return TRANSPORT_INFO_IPV6_UDP; + } + + return TRANSPORT_INFO_NOT_IP; +} + +/* Send skb on the slave VF device. */ +static int netvsc_vf_xmit(struct net_device *net, struct net_device *vf_netdev, + struct sk_buff *skb) +{ + struct net_device_context *ndev_ctx = netdev_priv(net); + unsigned int len = skb->len; + int rc; + + skb->dev = vf_netdev; + skb_record_rx_queue(skb, qdisc_skb_cb(skb)->slave_dev_queue_mapping); + + rc = dev_queue_xmit(skb); + if (likely(rc == NET_XMIT_SUCCESS || rc == NET_XMIT_CN)) { + struct netvsc_vf_pcpu_stats *pcpu_stats + = this_cpu_ptr(ndev_ctx->vf_stats); + + u64_stats_update_begin(&pcpu_stats->syncp); + pcpu_stats->tx_packets++; + pcpu_stats->tx_bytes += len; + u64_stats_update_end(&pcpu_stats->syncp); + } else { + this_cpu_inc(ndev_ctx->vf_stats->tx_dropped); + } + + return rc; +} + +static int netvsc_xmit(struct sk_buff *skb, struct net_device *net, bool xdp_tx) +{ + struct net_device_context *net_device_ctx = netdev_priv(net); + struct hv_netvsc_packet *packet = NULL; + int ret; + unsigned int num_data_pgs; + struct rndis_message *rndis_msg; + struct net_device *vf_netdev; + u32 rndis_msg_size; + u32 hash; + struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT]; + + /* If VF is present and up then redirect packets to it. + * Skip the VF if it is marked down or has no carrier. + * If netpoll is in uses, then VF can not be used either. + */ + vf_netdev = rcu_dereference_bh(net_device_ctx->vf_netdev); + if (vf_netdev && netif_running(vf_netdev) && + netif_carrier_ok(vf_netdev) && !netpoll_tx_running(net) && + net_device_ctx->data_path_is_vf) + return netvsc_vf_xmit(net, vf_netdev, skb); + + /* We will atmost need two pages to describe the rndis + * header. We can only transmit MAX_PAGE_BUFFER_COUNT number + * of pages in a single packet. If skb is scattered around + * more pages we try linearizing it. + */ + + num_data_pgs = netvsc_get_slots(skb) + 2; + + if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) { + ++net_device_ctx->eth_stats.tx_scattered; + + if (skb_linearize(skb)) + goto no_memory; + + num_data_pgs = netvsc_get_slots(skb) + 2; + if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) { + ++net_device_ctx->eth_stats.tx_too_big; + goto drop; + } + } + + /* + * Place the rndis header in the skb head room and + * the skb->cb will be used for hv_netvsc_packet + * structure. + */ + ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE); + if (ret) + goto no_memory; + + /* Use the skb control buffer for building up the packet */ + BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) > + sizeof_field(struct sk_buff, cb)); + packet = (struct hv_netvsc_packet *)skb->cb; + + packet->q_idx = skb_get_queue_mapping(skb); + + packet->total_data_buflen = skb->len; + packet->total_bytes = skb->len; + packet->total_packets = 1; + + rndis_msg = (struct rndis_message *)skb->head; + + /* Add the rndis header */ + rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET; + rndis_msg->msg_len = packet->total_data_buflen; + + rndis_msg->msg.pkt = (struct rndis_packet) { + .data_offset = sizeof(struct rndis_packet), + .data_len = packet->total_data_buflen, + .per_pkt_info_offset = sizeof(struct rndis_packet), + }; + + rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet); + + hash = skb_get_hash_raw(skb); + if (hash != 0 && net->real_num_tx_queues > 1) { + u32 *hash_info; + + rndis_msg_size += NDIS_HASH_PPI_SIZE; + hash_info = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE, + NBL_HASH_VALUE); + *hash_info = hash; + } + + /* When using AF_PACKET we need to drop VLAN header from + * the frame and update the SKB to allow the HOST OS + * to transmit the 802.1Q packet + */ + if (skb->protocol == htons(ETH_P_8021Q)) { + u16 vlan_tci; + + skb_reset_mac_header(skb); + if (eth_type_vlan(eth_hdr(skb)->h_proto)) { + if (unlikely(__skb_vlan_pop(skb, &vlan_tci) != 0)) { + ++net_device_ctx->eth_stats.vlan_error; + goto drop; + } + + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci); + /* Update the NDIS header pkt lengths */ + packet->total_data_buflen -= VLAN_HLEN; + packet->total_bytes -= VLAN_HLEN; + rndis_msg->msg_len = packet->total_data_buflen; + rndis_msg->msg.pkt.data_len = packet->total_data_buflen; + } + } + + if (skb_vlan_tag_present(skb)) { + struct ndis_pkt_8021q_info *vlan; + + rndis_msg_size += NDIS_VLAN_PPI_SIZE; + vlan = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE, + IEEE_8021Q_INFO); + + vlan->value = 0; + vlan->vlanid = skb_vlan_tag_get_id(skb); + vlan->cfi = skb_vlan_tag_get_cfi(skb); + vlan->pri = skb_vlan_tag_get_prio(skb); + } + + if (skb_is_gso(skb)) { + struct ndis_tcp_lso_info *lso_info; + + rndis_msg_size += NDIS_LSO_PPI_SIZE; + lso_info = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE, + TCP_LARGESEND_PKTINFO); + + lso_info->value = 0; + lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE; + if (skb->protocol == htons(ETH_P_IP)) { + lso_info->lso_v2_transmit.ip_version = + NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4; + ip_hdr(skb)->tot_len = 0; + ip_hdr(skb)->check = 0; + tcp_hdr(skb)->check = + ~csum_tcpudp_magic(ip_hdr(skb)->saddr, + ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); + } else { + lso_info->lso_v2_transmit.ip_version = + NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6; + tcp_v6_gso_csum_prep(skb); + } + lso_info->lso_v2_transmit.tcp_header_offset = skb_transport_offset(skb); + lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size; + } else if (skb->ip_summed == CHECKSUM_PARTIAL) { + if (net_checksum_info(skb) & net_device_ctx->tx_checksum_mask) { + struct ndis_tcp_ip_checksum_info *csum_info; + + rndis_msg_size += NDIS_CSUM_PPI_SIZE; + csum_info = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE, + TCPIP_CHKSUM_PKTINFO); + + csum_info->value = 0; + csum_info->transmit.tcp_header_offset = skb_transport_offset(skb); + + if (skb->protocol == htons(ETH_P_IP)) { + csum_info->transmit.is_ipv4 = 1; + + if (ip_hdr(skb)->protocol == IPPROTO_TCP) + csum_info->transmit.tcp_checksum = 1; + else + csum_info->transmit.udp_checksum = 1; + } else { + csum_info->transmit.is_ipv6 = 1; + + if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) + csum_info->transmit.tcp_checksum = 1; + else + csum_info->transmit.udp_checksum = 1; + } + } else { + /* Can't do offload of this type of checksum */ + if (skb_checksum_help(skb)) + goto drop; + } + } + + /* Start filling in the page buffers with the rndis hdr */ + rndis_msg->msg_len += rndis_msg_size; + packet->total_data_buflen = rndis_msg->msg_len; + packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size, + skb, packet, pb); + + /* timestamp packet in software */ + skb_tx_timestamp(skb); + + ret = netvsc_send(net, packet, rndis_msg, pb, skb, xdp_tx); + if (likely(ret == 0)) + return NETDEV_TX_OK; + + if (ret == -EAGAIN) { + ++net_device_ctx->eth_stats.tx_busy; + return NETDEV_TX_BUSY; + } + + if (ret == -ENOSPC) + ++net_device_ctx->eth_stats.tx_no_space; + +drop: + dev_kfree_skb_any(skb); + net->stats.tx_dropped++; + + return NETDEV_TX_OK; + +no_memory: + ++net_device_ctx->eth_stats.tx_no_memory; + goto drop; +} + +static netdev_tx_t netvsc_start_xmit(struct sk_buff *skb, + struct net_device *ndev) +{ + return netvsc_xmit(skb, ndev, false); +} + +/* + * netvsc_linkstatus_callback - Link up/down notification + */ +void netvsc_linkstatus_callback(struct net_device *net, + struct rndis_message *resp, + void *data, u32 data_buflen) +{ + struct rndis_indicate_status *indicate = &resp->msg.indicate_status; + struct net_device_context *ndev_ctx = netdev_priv(net); + struct netvsc_reconfig *event; + unsigned long flags; + + /* Ensure the packet is big enough to access its fields */ + if (resp->msg_len - RNDIS_HEADER_SIZE < sizeof(struct rndis_indicate_status)) { + netdev_err(net, "invalid rndis_indicate_status packet, len: %u\n", + resp->msg_len); + return; + } + + /* Copy the RNDIS indicate status into nvchan->recv_buf */ + memcpy(indicate, data + RNDIS_HEADER_SIZE, sizeof(*indicate)); + + /* Update the physical link speed when changing to another vSwitch */ + if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) { + u32 speed; + + /* Validate status_buf_offset and status_buflen. + * + * Certain (pre-Fe) implementations of Hyper-V's vSwitch didn't account + * for the status buffer field in resp->msg_len; perform the validation + * using data_buflen (>= resp->msg_len). + */ + if (indicate->status_buflen < sizeof(speed) || + indicate->status_buf_offset < sizeof(*indicate) || + data_buflen - RNDIS_HEADER_SIZE < indicate->status_buf_offset || + data_buflen - RNDIS_HEADER_SIZE - indicate->status_buf_offset + < indicate->status_buflen) { + netdev_err(net, "invalid rndis_indicate_status packet\n"); + return; + } + + speed = *(u32 *)(data + RNDIS_HEADER_SIZE + indicate->status_buf_offset) / 10000; + ndev_ctx->speed = speed; + return; + } + + /* Handle these link change statuses below */ + if (indicate->status != RNDIS_STATUS_NETWORK_CHANGE && + indicate->status != RNDIS_STATUS_MEDIA_CONNECT && + indicate->status != RNDIS_STATUS_MEDIA_DISCONNECT) + return; + + if (net->reg_state != NETREG_REGISTERED) + return; + + event = kzalloc(sizeof(*event), GFP_ATOMIC); + if (!event) + return; + event->event = indicate->status; + + spin_lock_irqsave(&ndev_ctx->lock, flags); + list_add_tail(&event->list, &ndev_ctx->reconfig_events); + spin_unlock_irqrestore(&ndev_ctx->lock, flags); + + schedule_delayed_work(&ndev_ctx->dwork, 0); +} + +/* This function should only be called after skb_record_rx_queue() */ +void netvsc_xdp_xmit(struct sk_buff *skb, struct net_device *ndev) +{ + int rc; + + skb->queue_mapping = skb_get_rx_queue(skb); + __skb_push(skb, ETH_HLEN); + + rc = netvsc_xmit(skb, ndev, true); + + if (dev_xmit_complete(rc)) + return; + + dev_kfree_skb_any(skb); + ndev->stats.tx_dropped++; +} + +static void netvsc_comp_ipcsum(struct sk_buff *skb) +{ + struct iphdr *iph = (struct iphdr *)skb->data; + + iph->check = 0; + iph->check = ip_fast_csum(iph, iph->ihl); +} + +static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net, + struct netvsc_channel *nvchan, + struct xdp_buff *xdp) +{ + struct napi_struct *napi = &nvchan->napi; + const struct ndis_pkt_8021q_info *vlan = &nvchan->rsc.vlan; + const struct ndis_tcp_ip_checksum_info *csum_info = + &nvchan->rsc.csum_info; + const u32 *hash_info = &nvchan->rsc.hash_info; + u8 ppi_flags = nvchan->rsc.ppi_flags; + struct sk_buff *skb; + void *xbuf = xdp->data_hard_start; + int i; + + if (xbuf) { + unsigned int hdroom = xdp->data - xdp->data_hard_start; + unsigned int xlen = xdp->data_end - xdp->data; + unsigned int frag_size = xdp->frame_sz; + + skb = build_skb(xbuf, frag_size); + + if (!skb) { + __free_page(virt_to_page(xbuf)); + return NULL; + } + + skb_reserve(skb, hdroom); + skb_put(skb, xlen); + skb->dev = napi->dev; + } else { + skb = napi_alloc_skb(napi, nvchan->rsc.pktlen); + + if (!skb) + return NULL; + + /* Copy to skb. This copy is needed here since the memory + * pointed by hv_netvsc_packet cannot be deallocated. + */ + for (i = 0; i < nvchan->rsc.cnt; i++) + skb_put_data(skb, nvchan->rsc.data[i], + nvchan->rsc.len[i]); + } + + skb->protocol = eth_type_trans(skb, net); + + /* skb is already created with CHECKSUM_NONE */ + skb_checksum_none_assert(skb); + + /* Incoming packets may have IP header checksum verified by the host. + * They may not have IP header checksum computed after coalescing. + * We compute it here if the flags are set, because on Linux, the IP + * checksum is always checked. + */ + if ((ppi_flags & NVSC_RSC_CSUM_INFO) && csum_info->receive.ip_checksum_value_invalid && + csum_info->receive.ip_checksum_succeeded && + skb->protocol == htons(ETH_P_IP)) { + /* Check that there is enough space to hold the IP header. */ + if (skb_headlen(skb) < sizeof(struct iphdr)) { + kfree_skb(skb); + return NULL; + } + netvsc_comp_ipcsum(skb); + } + + /* Do L4 checksum offload if enabled and present. */ + if ((ppi_flags & NVSC_RSC_CSUM_INFO) && (net->features & NETIF_F_RXCSUM)) { + if (csum_info->receive.tcp_checksum_succeeded || + csum_info->receive.udp_checksum_succeeded) + skb->ip_summed = CHECKSUM_UNNECESSARY; + } + + if ((ppi_flags & NVSC_RSC_HASH_INFO) && (net->features & NETIF_F_RXHASH)) + skb_set_hash(skb, *hash_info, PKT_HASH_TYPE_L4); + + if (ppi_flags & NVSC_RSC_VLAN) { + u16 vlan_tci = vlan->vlanid | (vlan->pri << VLAN_PRIO_SHIFT) | + (vlan->cfi ? VLAN_CFI_MASK : 0); + + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), + vlan_tci); + } + + return skb; +} + +/* + * netvsc_recv_callback - Callback when we receive a packet from the + * "wire" on the specified device. + */ +int netvsc_recv_callback(struct net_device *net, + struct netvsc_device *net_device, + struct netvsc_channel *nvchan) +{ + struct net_device_context *net_device_ctx = netdev_priv(net); + struct vmbus_channel *channel = nvchan->channel; + u16 q_idx = channel->offermsg.offer.sub_channel_index; + struct sk_buff *skb; + struct netvsc_stats_rx *rx_stats = &nvchan->rx_stats; + struct xdp_buff xdp; + u32 act; + + if (net->reg_state != NETREG_REGISTERED) + return NVSP_STAT_FAIL; + + act = netvsc_run_xdp(net, nvchan, &xdp); + + if (act == XDP_REDIRECT) + return NVSP_STAT_SUCCESS; + + if (act != XDP_PASS && act != XDP_TX) { + u64_stats_update_begin(&rx_stats->syncp); + rx_stats->xdp_drop++; + u64_stats_update_end(&rx_stats->syncp); + + return NVSP_STAT_SUCCESS; /* consumed by XDP */ + } + + /* Allocate a skb - TODO direct I/O to pages? */ + skb = netvsc_alloc_recv_skb(net, nvchan, &xdp); + + if (unlikely(!skb)) { + ++net_device_ctx->eth_stats.rx_no_memory; + return NVSP_STAT_FAIL; + } + + skb_record_rx_queue(skb, q_idx); + + /* + * Even if injecting the packet, record the statistics + * on the synthetic device because modifying the VF device + * statistics will not work correctly. + */ + u64_stats_update_begin(&rx_stats->syncp); + if (act == XDP_TX) + rx_stats->xdp_tx++; + + rx_stats->packets++; + rx_stats->bytes += nvchan->rsc.pktlen; + + if (skb->pkt_type == PACKET_BROADCAST) + ++rx_stats->broadcast; + else if (skb->pkt_type == PACKET_MULTICAST) + ++rx_stats->multicast; + u64_stats_update_end(&rx_stats->syncp); + + if (act == XDP_TX) { + netvsc_xdp_xmit(skb, net); + return NVSP_STAT_SUCCESS; + } + + napi_gro_receive(&nvchan->napi, skb); + return NVSP_STAT_SUCCESS; +} + +static void netvsc_get_drvinfo(struct net_device *net, + struct ethtool_drvinfo *info) +{ + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->fw_version, "N/A", sizeof(info->fw_version)); +} + +static void netvsc_get_channels(struct net_device *net, + struct ethtool_channels *channel) +{ + struct net_device_context *net_device_ctx = netdev_priv(net); + struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev); + + if (nvdev) { + channel->max_combined = nvdev->max_chn; + channel->combined_count = nvdev->num_chn; + } +} + +/* Alloc struct netvsc_device_info, and initialize it from either existing + * struct netvsc_device, or from default values. + */ +static +struct netvsc_device_info *netvsc_devinfo_get(struct netvsc_device *nvdev) +{ + struct netvsc_device_info *dev_info; + struct bpf_prog *prog; + + dev_info = kzalloc(sizeof(*dev_info), GFP_ATOMIC); + + if (!dev_info) + return NULL; + + if (nvdev) { + ASSERT_RTNL(); + + dev_info->num_chn = nvdev->num_chn; + dev_info->send_sections = nvdev->send_section_cnt; + dev_info->send_section_size = nvdev->send_section_size; + dev_info->recv_sections = nvdev->recv_section_cnt; + dev_info->recv_section_size = nvdev->recv_section_size; + + memcpy(dev_info->rss_key, nvdev->extension->rss_key, + NETVSC_HASH_KEYLEN); + + prog = netvsc_xdp_get(nvdev); + if (prog) { + bpf_prog_inc(prog); + dev_info->bprog = prog; + } + } else { + dev_info->num_chn = VRSS_CHANNEL_DEFAULT; + dev_info->send_sections = NETVSC_DEFAULT_TX; + dev_info->send_section_size = NETVSC_SEND_SECTION_SIZE; + dev_info->recv_sections = NETVSC_DEFAULT_RX; + dev_info->recv_section_size = NETVSC_RECV_SECTION_SIZE; + } + + return dev_info; +} + +/* Free struct netvsc_device_info */ +static void netvsc_devinfo_put(struct netvsc_device_info *dev_info) +{ + if (dev_info->bprog) { + ASSERT_RTNL(); + bpf_prog_put(dev_info->bprog); + } + + kfree(dev_info); +} + +static int netvsc_detach(struct net_device *ndev, + struct netvsc_device *nvdev) +{ + struct net_device_context *ndev_ctx = netdev_priv(ndev); + struct hv_device *hdev = ndev_ctx->device_ctx; + int ret; + + /* Don't try continuing to try and setup sub channels */ + if (cancel_work_sync(&nvdev->subchan_work)) + nvdev->num_chn = 1; + + netvsc_xdp_set(ndev, NULL, NULL, nvdev); + + /* If device was up (receiving) then shutdown */ + if (netif_running(ndev)) { + netvsc_tx_disable(nvdev, ndev); + + ret = rndis_filter_close(nvdev); + if (ret) { + netdev_err(ndev, + "unable to close device (ret %d).\n", ret); + return ret; + } + + ret = netvsc_wait_until_empty(nvdev); + if (ret) { + netdev_err(ndev, + "Ring buffer not empty after closing rndis\n"); + return ret; + } + } + + netif_device_detach(ndev); + + rndis_filter_device_remove(hdev, nvdev); + + return 0; +} + +static int netvsc_attach(struct net_device *ndev, + struct netvsc_device_info *dev_info) +{ + struct net_device_context *ndev_ctx = netdev_priv(ndev); + struct hv_device *hdev = ndev_ctx->device_ctx; + struct netvsc_device *nvdev; + struct rndis_device *rdev; + struct bpf_prog *prog; + int ret = 0; + + nvdev = rndis_filter_device_add(hdev, dev_info); + if (IS_ERR(nvdev)) + return PTR_ERR(nvdev); + + if (nvdev->num_chn > 1) { + ret = rndis_set_subchannel(ndev, nvdev, dev_info); + + /* if unavailable, just proceed with one queue */ + if (ret) { + nvdev->max_chn = 1; + nvdev->num_chn = 1; + } + } + + prog = dev_info->bprog; + if (prog) { + bpf_prog_inc(prog); + ret = netvsc_xdp_set(ndev, prog, NULL, nvdev); + if (ret) { + bpf_prog_put(prog); + goto err1; + } + } + + /* In any case device is now ready */ + nvdev->tx_disable = false; + netif_device_attach(ndev); + + /* Note: enable and attach happen when sub-channels setup */ + netif_carrier_off(ndev); + + if (netif_running(ndev)) { + ret = rndis_filter_open(nvdev); + if (ret) + goto err2; + + rdev = nvdev->extension; + if (!rdev->link_state) + netif_carrier_on(ndev); + } + + return 0; + +err2: + netif_device_detach(ndev); + +err1: + rndis_filter_device_remove(hdev, nvdev); + + return ret; +} + +static int netvsc_set_channels(struct net_device *net, + struct ethtool_channels *channels) +{ + struct net_device_context *net_device_ctx = netdev_priv(net); + struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev); + unsigned int orig, count = channels->combined_count; + struct netvsc_device_info *device_info; + int ret; + + /* We do not support separate count for rx, tx, or other */ + if (count == 0 || + channels->rx_count || channels->tx_count || channels->other_count) + return -EINVAL; + + if (!nvdev || nvdev->destroy) + return -ENODEV; + + if (nvdev->nvsp_version < NVSP_PROTOCOL_VERSION_5) + return -EINVAL; + + if (count > nvdev->max_chn) + return -EINVAL; + + orig = nvdev->num_chn; + + device_info = netvsc_devinfo_get(nvdev); + + if (!device_info) + return -ENOMEM; + + device_info->num_chn = count; + + ret = netvsc_detach(net, nvdev); + if (ret) + goto out; + + ret = netvsc_attach(net, device_info); + if (ret) { + device_info->num_chn = orig; + if (netvsc_attach(net, device_info)) + netdev_err(net, "restoring channel setting failed\n"); + } + +out: + netvsc_devinfo_put(device_info); + return ret; +} + +static void netvsc_init_settings(struct net_device *dev) +{ + struct net_device_context *ndc = netdev_priv(dev); + + ndc->l4_hash = HV_DEFAULT_L4HASH; + + ndc->speed = SPEED_UNKNOWN; + ndc->duplex = DUPLEX_FULL; + + dev->features = NETIF_F_LRO; +} + +static int netvsc_get_link_ksettings(struct net_device *dev, + struct ethtool_link_ksettings *cmd) +{ + struct net_device_context *ndc = netdev_priv(dev); + struct net_device *vf_netdev; + + vf_netdev = rtnl_dereference(ndc->vf_netdev); + + if (vf_netdev) + return __ethtool_get_link_ksettings(vf_netdev, cmd); + + cmd->base.speed = ndc->speed; + cmd->base.duplex = ndc->duplex; + cmd->base.port = PORT_OTHER; + + return 0; +} + +static int netvsc_set_link_ksettings(struct net_device *dev, + const struct ethtool_link_ksettings *cmd) +{ + struct net_device_context *ndc = netdev_priv(dev); + struct net_device *vf_netdev = rtnl_dereference(ndc->vf_netdev); + + if (vf_netdev) { + if (!vf_netdev->ethtool_ops->set_link_ksettings) + return -EOPNOTSUPP; + + return vf_netdev->ethtool_ops->set_link_ksettings(vf_netdev, + cmd); + } + + return ethtool_virtdev_set_link_ksettings(dev, cmd, + &ndc->speed, &ndc->duplex); +} + +static int netvsc_change_mtu(struct net_device *ndev, int mtu) +{ + struct net_device_context *ndevctx = netdev_priv(ndev); + struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev); + struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); + int orig_mtu = ndev->mtu; + struct netvsc_device_info *device_info; + int ret = 0; + + if (!nvdev || nvdev->destroy) + return -ENODEV; + + device_info = netvsc_devinfo_get(nvdev); + + if (!device_info) + return -ENOMEM; + + /* Change MTU of underlying VF netdev first. */ + if (vf_netdev) { + ret = dev_set_mtu(vf_netdev, mtu); + if (ret) + goto out; + } + + ret = netvsc_detach(ndev, nvdev); + if (ret) + goto rollback_vf; + + ndev->mtu = mtu; + + ret = netvsc_attach(ndev, device_info); + if (!ret) + goto out; + + /* Attempt rollback to original MTU */ + ndev->mtu = orig_mtu; + + if (netvsc_attach(ndev, device_info)) + netdev_err(ndev, "restoring mtu failed\n"); +rollback_vf: + if (vf_netdev) + dev_set_mtu(vf_netdev, orig_mtu); + +out: + netvsc_devinfo_put(device_info); + return ret; +} + +static void netvsc_get_vf_stats(struct net_device *net, + struct netvsc_vf_pcpu_stats *tot) +{ + struct net_device_context *ndev_ctx = netdev_priv(net); + int i; + + memset(tot, 0, sizeof(*tot)); + + for_each_possible_cpu(i) { + const struct netvsc_vf_pcpu_stats *stats + = per_cpu_ptr(ndev_ctx->vf_stats, i); + u64 rx_packets, rx_bytes, tx_packets, tx_bytes; + unsigned int start; + + do { + start = u64_stats_fetch_begin_irq(&stats->syncp); + rx_packets = stats->rx_packets; + tx_packets = stats->tx_packets; + rx_bytes = stats->rx_bytes; + tx_bytes = stats->tx_bytes; + } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); + + tot->rx_packets += rx_packets; + tot->tx_packets += tx_packets; + tot->rx_bytes += rx_bytes; + tot->tx_bytes += tx_bytes; + tot->tx_dropped += stats->tx_dropped; + } +} + +static void netvsc_get_pcpu_stats(struct net_device *net, + struct netvsc_ethtool_pcpu_stats *pcpu_tot) +{ + struct net_device_context *ndev_ctx = netdev_priv(net); + struct netvsc_device *nvdev = rcu_dereference_rtnl(ndev_ctx->nvdev); + int i; + + /* fetch percpu stats of vf */ + for_each_possible_cpu(i) { + const struct netvsc_vf_pcpu_stats *stats = + per_cpu_ptr(ndev_ctx->vf_stats, i); + struct netvsc_ethtool_pcpu_stats *this_tot = &pcpu_tot[i]; + unsigned int start; + + do { + start = u64_stats_fetch_begin_irq(&stats->syncp); + this_tot->vf_rx_packets = stats->rx_packets; + this_tot->vf_tx_packets = stats->tx_packets; + this_tot->vf_rx_bytes = stats->rx_bytes; + this_tot->vf_tx_bytes = stats->tx_bytes; + } while (u64_stats_fetch_retry_irq(&stats->syncp, start)); + this_tot->rx_packets = this_tot->vf_rx_packets; + this_tot->tx_packets = this_tot->vf_tx_packets; + this_tot->rx_bytes = this_tot->vf_rx_bytes; + this_tot->tx_bytes = this_tot->vf_tx_bytes; + } + + /* fetch percpu stats of netvsc */ + for (i = 0; i < nvdev->num_chn; i++) { + const struct netvsc_channel *nvchan = &nvdev->chan_table[i]; + const struct netvsc_stats_tx *tx_stats; + const struct netvsc_stats_rx *rx_stats; + struct netvsc_ethtool_pcpu_stats *this_tot = + &pcpu_tot[nvchan->channel->target_cpu]; + u64 packets, bytes; + unsigned int start; + + tx_stats = &nvchan->tx_stats; + do { + start = u64_stats_fetch_begin_irq(&tx_stats->syncp); + packets = tx_stats->packets; + bytes = tx_stats->bytes; + } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); + + this_tot->tx_bytes += bytes; + this_tot->tx_packets += packets; + + rx_stats = &nvchan->rx_stats; + do { + start = u64_stats_fetch_begin_irq(&rx_stats->syncp); + packets = rx_stats->packets; + bytes = rx_stats->bytes; + } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); + + this_tot->rx_bytes += bytes; + this_tot->rx_packets += packets; + } +} + +static void netvsc_get_stats64(struct net_device *net, + struct rtnl_link_stats64 *t) +{ + struct net_device_context *ndev_ctx = netdev_priv(net); + struct netvsc_device *nvdev; + struct netvsc_vf_pcpu_stats vf_tot; + int i; + + rcu_read_lock(); + + nvdev = rcu_dereference(ndev_ctx->nvdev); + if (!nvdev) + goto out; + + netdev_stats_to_stats64(t, &net->stats); + + netvsc_get_vf_stats(net, &vf_tot); + t->rx_packets += vf_tot.rx_packets; + t->tx_packets += vf_tot.tx_packets; + t->rx_bytes += vf_tot.rx_bytes; + t->tx_bytes += vf_tot.tx_bytes; + t->tx_dropped += vf_tot.tx_dropped; + + for (i = 0; i < nvdev->num_chn; i++) { + const struct netvsc_channel *nvchan = &nvdev->chan_table[i]; + const struct netvsc_stats_tx *tx_stats; + const struct netvsc_stats_rx *rx_stats; + u64 packets, bytes, multicast; + unsigned int start; + + tx_stats = &nvchan->tx_stats; + do { + start = u64_stats_fetch_begin_irq(&tx_stats->syncp); + packets = tx_stats->packets; + bytes = tx_stats->bytes; + } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); + + t->tx_bytes += bytes; + t->tx_packets += packets; + + rx_stats = &nvchan->rx_stats; + do { + start = u64_stats_fetch_begin_irq(&rx_stats->syncp); + packets = rx_stats->packets; + bytes = rx_stats->bytes; + multicast = rx_stats->multicast + rx_stats->broadcast; + } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); + + t->rx_bytes += bytes; + t->rx_packets += packets; + t->multicast += multicast; + } +out: + rcu_read_unlock(); +} + +static int netvsc_set_mac_addr(struct net_device *ndev, void *p) +{ + struct net_device_context *ndc = netdev_priv(ndev); + struct net_device *vf_netdev = rtnl_dereference(ndc->vf_netdev); + struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev); + struct sockaddr *addr = p; + int err; + + err = eth_prepare_mac_addr_change(ndev, p); + if (err) + return err; + + if (!nvdev) + return -ENODEV; + + if (vf_netdev) { + err = dev_set_mac_address(vf_netdev, addr, NULL); + if (err) + return err; + } + + err = rndis_filter_set_device_mac(nvdev, addr->sa_data); + if (!err) { + eth_commit_mac_addr_change(ndev, p); + } else if (vf_netdev) { + /* rollback change on VF */ + memcpy(addr->sa_data, ndev->dev_addr, ETH_ALEN); + dev_set_mac_address(vf_netdev, addr, NULL); + } + + return err; +} + +static const struct { + char name[ETH_GSTRING_LEN]; + u16 offset; +} netvsc_stats[] = { + { "tx_scattered", offsetof(struct netvsc_ethtool_stats, tx_scattered) }, + { "tx_no_memory", offsetof(struct netvsc_ethtool_stats, tx_no_memory) }, + { "tx_no_space", offsetof(struct netvsc_ethtool_stats, tx_no_space) }, + { "tx_too_big", offsetof(struct netvsc_ethtool_stats, tx_too_big) }, + { "tx_busy", offsetof(struct netvsc_ethtool_stats, tx_busy) }, + { "tx_send_full", offsetof(struct netvsc_ethtool_stats, tx_send_full) }, + { "rx_comp_busy", offsetof(struct netvsc_ethtool_stats, rx_comp_busy) }, + { "rx_no_memory", offsetof(struct netvsc_ethtool_stats, rx_no_memory) }, + { "stop_queue", offsetof(struct netvsc_ethtool_stats, stop_queue) }, + { "wake_queue", offsetof(struct netvsc_ethtool_stats, wake_queue) }, + { "vlan_error", offsetof(struct netvsc_ethtool_stats, vlan_error) }, +}, pcpu_stats[] = { + { "cpu%u_rx_packets", + offsetof(struct netvsc_ethtool_pcpu_stats, rx_packets) }, + { "cpu%u_rx_bytes", + offsetof(struct netvsc_ethtool_pcpu_stats, rx_bytes) }, + { "cpu%u_tx_packets", + offsetof(struct netvsc_ethtool_pcpu_stats, tx_packets) }, + { "cpu%u_tx_bytes", + offsetof(struct netvsc_ethtool_pcpu_stats, tx_bytes) }, + { "cpu%u_vf_rx_packets", + offsetof(struct netvsc_ethtool_pcpu_stats, vf_rx_packets) }, + { "cpu%u_vf_rx_bytes", + offsetof(struct netvsc_ethtool_pcpu_stats, vf_rx_bytes) }, + { "cpu%u_vf_tx_packets", + offsetof(struct netvsc_ethtool_pcpu_stats, vf_tx_packets) }, + { "cpu%u_vf_tx_bytes", + offsetof(struct netvsc_ethtool_pcpu_stats, vf_tx_bytes) }, +}, vf_stats[] = { + { "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats, rx_packets) }, + { "vf_rx_bytes", offsetof(struct netvsc_vf_pcpu_stats, rx_bytes) }, + { "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats, tx_packets) }, + { "vf_tx_bytes", offsetof(struct netvsc_vf_pcpu_stats, tx_bytes) }, + { "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats, tx_dropped) }, +}; + +#define NETVSC_GLOBAL_STATS_LEN ARRAY_SIZE(netvsc_stats) +#define NETVSC_VF_STATS_LEN ARRAY_SIZE(vf_stats) + +/* statistics per queue (rx/tx packets/bytes) */ +#define NETVSC_PCPU_STATS_LEN (num_present_cpus() * ARRAY_SIZE(pcpu_stats)) + +/* 8 statistics per queue (rx/tx packets/bytes, XDP actions) */ +#define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 8) + +static int netvsc_get_sset_count(struct net_device *dev, int string_set) +{ + struct net_device_context *ndc = netdev_priv(dev); + struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev); + + if (!nvdev) + return -ENODEV; + + switch (string_set) { + case ETH_SS_STATS: + return NETVSC_GLOBAL_STATS_LEN + + NETVSC_VF_STATS_LEN + + NETVSC_QUEUE_STATS_LEN(nvdev) + + NETVSC_PCPU_STATS_LEN; + default: + return -EINVAL; + } +} + +static void netvsc_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) +{ + struct net_device_context *ndc = netdev_priv(dev); + struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev); + const void *nds = &ndc->eth_stats; + const struct netvsc_stats_tx *tx_stats; + const struct netvsc_stats_rx *rx_stats; + struct netvsc_vf_pcpu_stats sum; + struct netvsc_ethtool_pcpu_stats *pcpu_sum; + unsigned int start; + u64 packets, bytes; + u64 xdp_drop; + u64 xdp_redirect; + u64 xdp_tx; + u64 xdp_xmit; + int i, j, cpu; + + if (!nvdev) + return; + + for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++) + data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset); + + netvsc_get_vf_stats(dev, &sum); + for (j = 0; j < NETVSC_VF_STATS_LEN; j++) + data[i++] = *(u64 *)((void *)&sum + vf_stats[j].offset); + + for (j = 0; j < nvdev->num_chn; j++) { + tx_stats = &nvdev->chan_table[j].tx_stats; + + do { + start = u64_stats_fetch_begin_irq(&tx_stats->syncp); + packets = tx_stats->packets; + bytes = tx_stats->bytes; + xdp_xmit = tx_stats->xdp_xmit; + } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); + data[i++] = packets; + data[i++] = bytes; + data[i++] = xdp_xmit; + + rx_stats = &nvdev->chan_table[j].rx_stats; + do { + start = u64_stats_fetch_begin_irq(&rx_stats->syncp); + packets = rx_stats->packets; + bytes = rx_stats->bytes; + xdp_drop = rx_stats->xdp_drop; + xdp_redirect = rx_stats->xdp_redirect; + xdp_tx = rx_stats->xdp_tx; + } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); + data[i++] = packets; + data[i++] = bytes; + data[i++] = xdp_drop; + data[i++] = xdp_redirect; + data[i++] = xdp_tx; + } + + pcpu_sum = kvmalloc_array(num_possible_cpus(), + sizeof(struct netvsc_ethtool_pcpu_stats), + GFP_KERNEL); + if (!pcpu_sum) + return; + + netvsc_get_pcpu_stats(dev, pcpu_sum); + for_each_present_cpu(cpu) { + struct netvsc_ethtool_pcpu_stats *this_sum = &pcpu_sum[cpu]; + + for (j = 0; j < ARRAY_SIZE(pcpu_stats); j++) + data[i++] = *(u64 *)((void *)this_sum + + pcpu_stats[j].offset); + } + kvfree(pcpu_sum); +} + +static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data) +{ + struct net_device_context *ndc = netdev_priv(dev); + struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev); + u8 *p = data; + int i, cpu; + + if (!nvdev) + return; + + switch (stringset) { + case ETH_SS_STATS: + for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++) + ethtool_sprintf(&p, netvsc_stats[i].name); + + for (i = 0; i < ARRAY_SIZE(vf_stats); i++) + ethtool_sprintf(&p, vf_stats[i].name); + + for (i = 0; i < nvdev->num_chn; i++) { + ethtool_sprintf(&p, "tx_queue_%u_packets", i); + ethtool_sprintf(&p, "tx_queue_%u_bytes", i); + ethtool_sprintf(&p, "tx_queue_%u_xdp_xmit", i); + ethtool_sprintf(&p, "rx_queue_%u_packets", i); + ethtool_sprintf(&p, "rx_queue_%u_bytes", i); + ethtool_sprintf(&p, "rx_queue_%u_xdp_drop", i); + ethtool_sprintf(&p, "rx_queue_%u_xdp_redirect", i); + ethtool_sprintf(&p, "rx_queue_%u_xdp_tx", i); + } + + for_each_present_cpu(cpu) { + for (i = 0; i < ARRAY_SIZE(pcpu_stats); i++) + ethtool_sprintf(&p, pcpu_stats[i].name, cpu); + } + + break; + } +} + +static int +netvsc_get_rss_hash_opts(struct net_device_context *ndc, + struct ethtool_rxnfc *info) +{ + const u32 l4_flag = RXH_L4_B_0_1 | RXH_L4_B_2_3; + + info->data = RXH_IP_SRC | RXH_IP_DST; + + switch (info->flow_type) { + case TCP_V4_FLOW: + if (ndc->l4_hash & HV_TCP4_L4HASH) + info->data |= l4_flag; + + break; + + case TCP_V6_FLOW: + if (ndc->l4_hash & HV_TCP6_L4HASH) + info->data |= l4_flag; + + break; + + case UDP_V4_FLOW: + if (ndc->l4_hash & HV_UDP4_L4HASH) + info->data |= l4_flag; + + break; + + case UDP_V6_FLOW: + if (ndc->l4_hash & HV_UDP6_L4HASH) + info->data |= l4_flag; + + break; + + case IPV4_FLOW: + case IPV6_FLOW: + break; + default: + info->data = 0; + break; + } + + return 0; +} + +static int +netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, + u32 *rules) +{ + struct net_device_context *ndc = netdev_priv(dev); + struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev); + + if (!nvdev) + return -ENODEV; + + switch (info->cmd) { + case ETHTOOL_GRXRINGS: + info->data = nvdev->num_chn; + return 0; + + case ETHTOOL_GRXFH: + return netvsc_get_rss_hash_opts(ndc, info); + } + return -EOPNOTSUPP; +} + +static int netvsc_set_rss_hash_opts(struct net_device_context *ndc, + struct ethtool_rxnfc *info) +{ + if (info->data == (RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3)) { + switch (info->flow_type) { + case TCP_V4_FLOW: + ndc->l4_hash |= HV_TCP4_L4HASH; + break; + + case TCP_V6_FLOW: + ndc->l4_hash |= HV_TCP6_L4HASH; + break; + + case UDP_V4_FLOW: + ndc->l4_hash |= HV_UDP4_L4HASH; + break; + + case UDP_V6_FLOW: + ndc->l4_hash |= HV_UDP6_L4HASH; + break; + + default: + return -EOPNOTSUPP; + } + + return 0; + } + + if (info->data == (RXH_IP_SRC | RXH_IP_DST)) { + switch (info->flow_type) { + case TCP_V4_FLOW: + ndc->l4_hash &= ~HV_TCP4_L4HASH; + break; + + case TCP_V6_FLOW: + ndc->l4_hash &= ~HV_TCP6_L4HASH; + break; + + case UDP_V4_FLOW: + ndc->l4_hash &= ~HV_UDP4_L4HASH; + break; + + case UDP_V6_FLOW: + ndc->l4_hash &= ~HV_UDP6_L4HASH; + break; + + default: + return -EOPNOTSUPP; + } + + return 0; + } + + return -EOPNOTSUPP; +} + +static int +netvsc_set_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *info) +{ + struct net_device_context *ndc = netdev_priv(ndev); + + if (info->cmd == ETHTOOL_SRXFH) + return netvsc_set_rss_hash_opts(ndc, info); + + return -EOPNOTSUPP; +} + +static u32 netvsc_get_rxfh_key_size(struct net_device *dev) +{ + return NETVSC_HASH_KEYLEN; +} + +static u32 netvsc_rss_indir_size(struct net_device *dev) +{ + return ITAB_NUM; +} + +static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, + u8 *hfunc) +{ + struct net_device_context *ndc = netdev_priv(dev); + struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev); + struct rndis_device *rndis_dev; + int i; + + if (!ndev) + return -ENODEV; + + if (hfunc) + *hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */ + + rndis_dev = ndev->extension; + if (indir) { + for (i = 0; i < ITAB_NUM; i++) + indir[i] = ndc->rx_table[i]; + } + + if (key) + memcpy(key, rndis_dev->rss_key, NETVSC_HASH_KEYLEN); + + return 0; +} + +static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir, + const u8 *key, const u8 hfunc) +{ + struct net_device_context *ndc = netdev_priv(dev); + struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev); + struct rndis_device *rndis_dev; + int i; + + if (!ndev) + return -ENODEV; + + if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + return -EOPNOTSUPP; + + rndis_dev = ndev->extension; + if (indir) { + for (i = 0; i < ITAB_NUM; i++) + if (indir[i] >= ndev->num_chn) + return -EINVAL; + + for (i = 0; i < ITAB_NUM; i++) + ndc->rx_table[i] = indir[i]; + } + + if (!key) { + if (!indir) + return 0; + + key = rndis_dev->rss_key; + } + + return rndis_filter_set_rss_param(rndis_dev, key); +} + +/* Hyper-V RNDIS protocol does not have ring in the HW sense. + * It does have pre-allocated receive area which is divided into sections. + */ +static void __netvsc_get_ringparam(struct netvsc_device *nvdev, + struct ethtool_ringparam *ring) +{ + u32 max_buf_size; + + ring->rx_pending = nvdev->recv_section_cnt; + ring->tx_pending = nvdev->send_section_cnt; + + if (nvdev->nvsp_version <= NVSP_PROTOCOL_VERSION_2) + max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY; + else + max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE; + + ring->rx_max_pending = max_buf_size / nvdev->recv_section_size; + ring->tx_max_pending = NETVSC_SEND_BUFFER_SIZE + / nvdev->send_section_size; +} + +static void netvsc_get_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +{ + struct net_device_context *ndevctx = netdev_priv(ndev); + struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); + + if (!nvdev) + return; + + __netvsc_get_ringparam(nvdev, ring); +} + +static int netvsc_set_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +{ + struct net_device_context *ndevctx = netdev_priv(ndev); + struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); + struct netvsc_device_info *device_info; + struct ethtool_ringparam orig; + u32 new_tx, new_rx; + int ret = 0; + + if (!nvdev || nvdev->destroy) + return -ENODEV; + + memset(&orig, 0, sizeof(orig)); + __netvsc_get_ringparam(nvdev, &orig); + + new_tx = clamp_t(u32, ring->tx_pending, + NETVSC_MIN_TX_SECTIONS, orig.tx_max_pending); + new_rx = clamp_t(u32, ring->rx_pending, + NETVSC_MIN_RX_SECTIONS, orig.rx_max_pending); + + if (new_tx == orig.tx_pending && + new_rx == orig.rx_pending) + return 0; /* no change */ + + device_info = netvsc_devinfo_get(nvdev); + + if (!device_info) + return -ENOMEM; + + device_info->send_sections = new_tx; + device_info->recv_sections = new_rx; + + ret = netvsc_detach(ndev, nvdev); + if (ret) + goto out; + + ret = netvsc_attach(ndev, device_info); + if (ret) { + device_info->send_sections = orig.tx_pending; + device_info->recv_sections = orig.rx_pending; + + if (netvsc_attach(ndev, device_info)) + netdev_err(ndev, "restoring ringparam failed"); + } + +out: + netvsc_devinfo_put(device_info); + return ret; +} + +static netdev_features_t netvsc_fix_features(struct net_device *ndev, + netdev_features_t features) +{ + struct net_device_context *ndevctx = netdev_priv(ndev); + struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); + + if (!nvdev || nvdev->destroy) + return features; + + if ((features & NETIF_F_LRO) && netvsc_xdp_get(nvdev)) { + features ^= NETIF_F_LRO; + netdev_info(ndev, "Skip LRO - unsupported with XDP\n"); + } + + return features; +} + +static int netvsc_set_features(struct net_device *ndev, + netdev_features_t features) +{ + netdev_features_t change = features ^ ndev->features; + struct net_device_context *ndevctx = netdev_priv(ndev); + struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); + struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev); + struct ndis_offload_params offloads; + int ret = 0; + + if (!nvdev || nvdev->destroy) + return -ENODEV; + + if (!(change & NETIF_F_LRO)) + goto syncvf; + + memset(&offloads, 0, sizeof(struct ndis_offload_params)); + + if (features & NETIF_F_LRO) { + offloads.rsc_ip_v4 = NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED; + offloads.rsc_ip_v6 = NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED; + } else { + offloads.rsc_ip_v4 = NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED; + offloads.rsc_ip_v6 = NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED; + } + + ret = rndis_filter_set_offload_params(ndev, nvdev, &offloads); + + if (ret) { + features ^= NETIF_F_LRO; + ndev->features = features; + } + +syncvf: + if (!vf_netdev) + return ret; + + vf_netdev->wanted_features = features; + netdev_update_features(vf_netdev); + + return ret; +} + +static int netvsc_get_regs_len(struct net_device *netdev) +{ + return VRSS_SEND_TAB_SIZE * sizeof(u32); +} + +static void netvsc_get_regs(struct net_device *netdev, + struct ethtool_regs *regs, void *p) +{ + struct net_device_context *ndc = netdev_priv(netdev); + u32 *regs_buff = p; + + /* increase the version, if buffer format is changed. */ + regs->version = 1; + + memcpy(regs_buff, ndc->tx_table, VRSS_SEND_TAB_SIZE * sizeof(u32)); +} + +static u32 netvsc_get_msglevel(struct net_device *ndev) +{ + struct net_device_context *ndev_ctx = netdev_priv(ndev); + + return ndev_ctx->msg_enable; +} + +static void netvsc_set_msglevel(struct net_device *ndev, u32 val) +{ + struct net_device_context *ndev_ctx = netdev_priv(ndev); + + ndev_ctx->msg_enable = val; +} + +static const struct ethtool_ops ethtool_ops = { + .get_drvinfo = netvsc_get_drvinfo, + .get_regs_len = netvsc_get_regs_len, + .get_regs = netvsc_get_regs, + .get_msglevel = netvsc_get_msglevel, + .set_msglevel = netvsc_set_msglevel, + .get_link = ethtool_op_get_link, + .get_ethtool_stats = netvsc_get_ethtool_stats, + .get_sset_count = netvsc_get_sset_count, + .get_strings = netvsc_get_strings, + .get_channels = netvsc_get_channels, + .set_channels = netvsc_set_channels, + .get_ts_info = ethtool_op_get_ts_info, + .get_rxnfc = netvsc_get_rxnfc, + .set_rxnfc = netvsc_set_rxnfc, + .get_rxfh_key_size = netvsc_get_rxfh_key_size, + .get_rxfh_indir_size = netvsc_rss_indir_size, + .get_rxfh = netvsc_get_rxfh, + .set_rxfh = netvsc_set_rxfh, + .get_link_ksettings = netvsc_get_link_ksettings, + .set_link_ksettings = netvsc_set_link_ksettings, + .get_ringparam = netvsc_get_ringparam, + .set_ringparam = netvsc_set_ringparam, +}; + +static const struct net_device_ops device_ops = { + .ndo_open = netvsc_open, + .ndo_stop = netvsc_close, + .ndo_start_xmit = netvsc_start_xmit, + .ndo_change_rx_flags = netvsc_change_rx_flags, + .ndo_set_rx_mode = netvsc_set_rx_mode, + .ndo_fix_features = netvsc_fix_features, + .ndo_set_features = netvsc_set_features, + .ndo_change_mtu = netvsc_change_mtu, + .ndo_validate_addr = eth_validate_addr, + .ndo_set_mac_address = netvsc_set_mac_addr, + .ndo_select_queue = netvsc_select_queue, + .ndo_get_stats64 = netvsc_get_stats64, + .ndo_bpf = netvsc_bpf, + .ndo_xdp_xmit = netvsc_ndoxdp_xmit, +}; + +/* + * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link + * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is + * present send GARP packet to network peers with netif_notify_peers(). + */ +static void netvsc_link_change(struct work_struct *w) +{ + struct net_device_context *ndev_ctx = + container_of(w, struct net_device_context, dwork.work); + struct hv_device *device_obj = ndev_ctx->device_ctx; + struct net_device *net = hv_get_drvdata(device_obj); + unsigned long flags, next_reconfig, delay; + struct netvsc_reconfig *event = NULL; + struct netvsc_device *net_device; + struct rndis_device *rdev; + bool reschedule = false; + + /* if changes are happening, comeback later */ + if (!rtnl_trylock()) { + schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT); + return; + } + + net_device = rtnl_dereference(ndev_ctx->nvdev); + if (!net_device) + goto out_unlock; + + rdev = net_device->extension; + + next_reconfig = ndev_ctx->last_reconfig + LINKCHANGE_INT; + if (time_is_after_jiffies(next_reconfig)) { + /* link_watch only sends one notification with current state + * per second, avoid doing reconfig more frequently. Handle + * wrap around. + */ + delay = next_reconfig - jiffies; + delay = delay < LINKCHANGE_INT ? delay : LINKCHANGE_INT; + schedule_delayed_work(&ndev_ctx->dwork, delay); + goto out_unlock; + } + ndev_ctx->last_reconfig = jiffies; + + spin_lock_irqsave(&ndev_ctx->lock, flags); + if (!list_empty(&ndev_ctx->reconfig_events)) { + event = list_first_entry(&ndev_ctx->reconfig_events, + struct netvsc_reconfig, list); + list_del(&event->list); + reschedule = !list_empty(&ndev_ctx->reconfig_events); + } + spin_unlock_irqrestore(&ndev_ctx->lock, flags); + + if (!event) + goto out_unlock; + + switch (event->event) { + /* Only the following events are possible due to the check in + * netvsc_linkstatus_callback() + */ + case RNDIS_STATUS_MEDIA_CONNECT: + if (rdev->link_state) { + rdev->link_state = false; + netif_carrier_on(net); + netvsc_tx_enable(net_device, net); + } else { + __netdev_notify_peers(net); + } + kfree(event); + break; + case RNDIS_STATUS_MEDIA_DISCONNECT: + if (!rdev->link_state) { + rdev->link_state = true; + netif_carrier_off(net); + netvsc_tx_disable(net_device, net); + } + kfree(event); + break; + case RNDIS_STATUS_NETWORK_CHANGE: + /* Only makes sense if carrier is present */ + if (!rdev->link_state) { + rdev->link_state = true; + netif_carrier_off(net); + netvsc_tx_disable(net_device, net); + event->event = RNDIS_STATUS_MEDIA_CONNECT; + spin_lock_irqsave(&ndev_ctx->lock, flags); + list_add(&event->list, &ndev_ctx->reconfig_events); + spin_unlock_irqrestore(&ndev_ctx->lock, flags); + reschedule = true; + } + break; + } + + rtnl_unlock(); + + /* link_watch only sends one notification with current state per + * second, handle next reconfig event in 2 seconds. + */ + if (reschedule) + schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT); + + return; + +out_unlock: + rtnl_unlock(); +} + +static struct net_device *get_netvsc_byref(struct net_device *vf_netdev) +{ + struct net_device_context *net_device_ctx; + struct net_device *dev; + + dev = netdev_master_upper_dev_get(vf_netdev); + if (!dev || dev->netdev_ops != &device_ops) + return NULL; /* not a netvsc device */ + + net_device_ctx = netdev_priv(dev); + if (!rtnl_dereference(net_device_ctx->nvdev)) + return NULL; /* device is removed */ + + return dev; +} + +/* Called when VF is injecting data into network stack. + * Change the associated network device from VF to netvsc. + * note: already called with rcu_read_lock + */ +static rx_handler_result_t netvsc_vf_handle_frame(struct sk_buff **pskb) +{ + struct sk_buff *skb = *pskb; + struct net_device *ndev = rcu_dereference(skb->dev->rx_handler_data); + struct net_device_context *ndev_ctx = netdev_priv(ndev); + struct netvsc_vf_pcpu_stats *pcpu_stats + = this_cpu_ptr(ndev_ctx->vf_stats); + + skb = skb_share_check(skb, GFP_ATOMIC); + if (unlikely(!skb)) + return RX_HANDLER_CONSUMED; + + *pskb = skb; + + skb->dev = ndev; + + u64_stats_update_begin(&pcpu_stats->syncp); + pcpu_stats->rx_packets++; + pcpu_stats->rx_bytes += skb->len; + u64_stats_update_end(&pcpu_stats->syncp); + + return RX_HANDLER_ANOTHER; +} + +static int netvsc_vf_join(struct net_device *vf_netdev, + struct net_device *ndev) +{ + struct net_device_context *ndev_ctx = netdev_priv(ndev); + int ret; + + ret = netdev_rx_handler_register(vf_netdev, + netvsc_vf_handle_frame, ndev); + if (ret != 0) { + netdev_err(vf_netdev, + "can not register netvsc VF receive handler (err = %d)\n", + ret); + goto rx_handler_failed; + } + + ret = netdev_master_upper_dev_link(vf_netdev, ndev, + NULL, NULL, NULL); + if (ret != 0) { + netdev_err(vf_netdev, + "can not set master device %s (err = %d)\n", + ndev->name, ret); + goto upper_link_failed; + } + + schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT); + + call_netdevice_notifiers(NETDEV_JOIN, vf_netdev); + + netdev_info(vf_netdev, "joined to %s\n", ndev->name); + return 0; + +upper_link_failed: + netdev_rx_handler_unregister(vf_netdev); +rx_handler_failed: + return ret; +} + +static void __netvsc_vf_setup(struct net_device *ndev, + struct net_device *vf_netdev) +{ + int ret; + + /* Align MTU of VF with master */ + ret = dev_set_mtu(vf_netdev, ndev->mtu); + if (ret) + netdev_warn(vf_netdev, + "unable to change mtu to %u\n", ndev->mtu); + + /* set multicast etc flags on VF */ + dev_change_flags(vf_netdev, ndev->flags | IFF_SLAVE, NULL); + + /* sync address list from ndev to VF */ + netif_addr_lock_bh(ndev); + dev_uc_sync(vf_netdev, ndev); + dev_mc_sync(vf_netdev, ndev); + netif_addr_unlock_bh(ndev); + + if (netif_running(ndev)) { + ret = dev_open(vf_netdev, NULL); + if (ret) + netdev_warn(vf_netdev, + "unable to open: %d\n", ret); + } +} + +/* Setup VF as slave of the synthetic device. + * Runs in workqueue to avoid recursion in netlink callbacks. + */ +static void netvsc_vf_setup(struct work_struct *w) +{ + struct net_device_context *ndev_ctx + = container_of(w, struct net_device_context, vf_takeover.work); + struct net_device *ndev = hv_get_drvdata(ndev_ctx->device_ctx); + struct net_device *vf_netdev; + + if (!rtnl_trylock()) { + schedule_delayed_work(&ndev_ctx->vf_takeover, 0); + return; + } + + vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev); + if (vf_netdev) + __netvsc_vf_setup(ndev, vf_netdev); + + rtnl_unlock(); +} + +/* Find netvsc by VF serial number. + * The PCI hyperv controller records the serial number as the slot kobj name. + */ +static struct net_device *get_netvsc_byslot(const struct net_device *vf_netdev) +{ + struct device *parent = vf_netdev->dev.parent; + struct net_device_context *ndev_ctx; + struct net_device *ndev; + struct pci_dev *pdev; + u32 serial; + + if (!parent || !dev_is_pci(parent)) + return NULL; /* not a PCI device */ + + pdev = to_pci_dev(parent); + if (!pdev->slot) { + netdev_notice(vf_netdev, "no PCI slot information\n"); + return NULL; + } + + if (kstrtou32(pci_slot_name(pdev->slot), 10, &serial)) { + netdev_notice(vf_netdev, "Invalid vf serial:%s\n", + pci_slot_name(pdev->slot)); + return NULL; + } + + list_for_each_entry(ndev_ctx, &netvsc_dev_list, list) { + if (!ndev_ctx->vf_alloc) + continue; + + if (ndev_ctx->vf_serial != serial) + continue; + + ndev = hv_get_drvdata(ndev_ctx->device_ctx); + if (ndev->addr_len != vf_netdev->addr_len || + memcmp(ndev->perm_addr, vf_netdev->perm_addr, + ndev->addr_len) != 0) + continue; + + return ndev; + + } + + /* Fallback path to check synthetic vf with help of mac addr. + * Because this function can be called before vf_netdev is + * initialized (NETDEV_POST_INIT) when its perm_addr has not been copied + * from dev_addr, also try to match to its dev_addr. + * Note: On Hyper-V and Azure, it's not possible to set a MAC address + * on a VF that matches to the MAC of a unrelated NETVSC device. + */ + list_for_each_entry(ndev_ctx, &netvsc_dev_list, list) { + ndev = hv_get_drvdata(ndev_ctx->device_ctx); + if (ether_addr_equal(vf_netdev->perm_addr, ndev->perm_addr) || + ether_addr_equal(vf_netdev->dev_addr, ndev->perm_addr)) + return ndev; + } + + netdev_notice(vf_netdev, + "no netdev found for vf serial:%u\n", serial); + return NULL; +} + +static int netvsc_prepare_bonding(struct net_device *vf_netdev) +{ + struct net_device *ndev; + + ndev = get_netvsc_byslot(vf_netdev); + if (!ndev) + return NOTIFY_DONE; + + /* set slave flag before open to prevent IPv6 addrconf */ + vf_netdev->flags |= IFF_SLAVE; + return NOTIFY_DONE; +} + +static int netvsc_register_vf(struct net_device *vf_netdev) +{ + struct net_device_context *net_device_ctx; + struct netvsc_device *netvsc_dev; + struct bpf_prog *prog; + struct net_device *ndev; + int ret; + + if (vf_netdev->addr_len != ETH_ALEN) + return NOTIFY_DONE; + + ndev = get_netvsc_byslot(vf_netdev); + if (!ndev) + return NOTIFY_DONE; + + net_device_ctx = netdev_priv(ndev); + netvsc_dev = rtnl_dereference(net_device_ctx->nvdev); + if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev)) + return NOTIFY_DONE; + + /* if synthetic interface is a different namespace, + * then move the VF to that namespace; join will be + * done again in that context. + */ + if (!net_eq(dev_net(ndev), dev_net(vf_netdev))) { + ret = dev_change_net_namespace(vf_netdev, + dev_net(ndev), "eth%d"); + if (ret) + netdev_err(vf_netdev, + "could not move to same namespace as %s: %d\n", + ndev->name, ret); + else + netdev_info(vf_netdev, + "VF moved to namespace with: %s\n", + ndev->name); + return NOTIFY_DONE; + } + + netdev_info(ndev, "VF registering: %s\n", vf_netdev->name); + + if (netvsc_vf_join(vf_netdev, ndev) != 0) + return NOTIFY_DONE; + + dev_hold(vf_netdev); + rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev); + + if (ndev->needed_headroom < vf_netdev->needed_headroom) + ndev->needed_headroom = vf_netdev->needed_headroom; + + vf_netdev->wanted_features = ndev->features; + netdev_update_features(vf_netdev); + + prog = netvsc_xdp_get(netvsc_dev); + netvsc_vf_setxdp(vf_netdev, prog); + + return NOTIFY_OK; +} + +/* Change the data path when VF UP/DOWN/CHANGE are detected. + * + * Typically a UP or DOWN event is followed by a CHANGE event, so + * net_device_ctx->data_path_is_vf is used to cache the current data path + * to avoid the duplicate call of netvsc_switch_datapath() and the duplicate + * message. + * + * During hibernation, if a VF NIC driver (e.g. mlx5) preserves the network + * interface, there is only the CHANGE event and no UP or DOWN event. + */ +static int netvsc_vf_changed(struct net_device *vf_netdev, unsigned long event) +{ + struct net_device_context *net_device_ctx; + struct netvsc_device *netvsc_dev; + struct net_device *ndev; + bool vf_is_up = false; + int ret; + + if (event != NETDEV_GOING_DOWN) + vf_is_up = netif_running(vf_netdev); + + ndev = get_netvsc_byref(vf_netdev); + if (!ndev) + return NOTIFY_DONE; + + net_device_ctx = netdev_priv(ndev); + netvsc_dev = rtnl_dereference(net_device_ctx->nvdev); + if (!netvsc_dev) + return NOTIFY_DONE; + + if (net_device_ctx->data_path_is_vf == vf_is_up) + return NOTIFY_OK; + + if (vf_is_up && !net_device_ctx->vf_alloc) { + netdev_info(ndev, "Waiting for the VF association from host\n"); + wait_for_completion(&net_device_ctx->vf_add); + } + + ret = netvsc_switch_datapath(ndev, vf_is_up); + + if (ret) { + netdev_err(ndev, + "Data path failed to switch %s VF: %s, err: %d\n", + vf_is_up ? "to" : "from", vf_netdev->name, ret); + return NOTIFY_DONE; + } else { + netdev_info(ndev, "Data path switched %s VF: %s\n", + vf_is_up ? "to" : "from", vf_netdev->name); + } + + return NOTIFY_OK; +} + +static int netvsc_unregister_vf(struct net_device *vf_netdev) +{ + struct net_device *ndev; + struct net_device_context *net_device_ctx; + + ndev = get_netvsc_byref(vf_netdev); + if (!ndev) + return NOTIFY_DONE; + + net_device_ctx = netdev_priv(ndev); + cancel_delayed_work_sync(&net_device_ctx->vf_takeover); + + netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name); + + netvsc_vf_setxdp(vf_netdev, NULL); + + reinit_completion(&net_device_ctx->vf_add); + netdev_rx_handler_unregister(vf_netdev); + netdev_upper_dev_unlink(vf_netdev, ndev); + RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL); + dev_put(vf_netdev); + + ndev->needed_headroom = RNDIS_AND_PPI_SIZE; + + return NOTIFY_OK; +} + +static int netvsc_probe(struct hv_device *dev, + const struct hv_vmbus_device_id *dev_id) +{ + struct net_device *net = NULL; + struct net_device_context *net_device_ctx; + struct netvsc_device_info *device_info = NULL; + struct netvsc_device *nvdev; + int ret = -ENOMEM; + + net = alloc_etherdev_mq(sizeof(struct net_device_context), + VRSS_CHANNEL_MAX); + if (!net) + goto no_net; + + netif_carrier_off(net); + + netvsc_init_settings(net); + + net_device_ctx = netdev_priv(net); + net_device_ctx->device_ctx = dev; + net_device_ctx->msg_enable = netif_msg_init(debug, default_msg); + if (netif_msg_probe(net_device_ctx)) + netdev_dbg(net, "netvsc msg_enable: %d\n", + net_device_ctx->msg_enable); + + hv_set_drvdata(dev, net); + + INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change); + + init_completion(&net_device_ctx->vf_add); + spin_lock_init(&net_device_ctx->lock); + INIT_LIST_HEAD(&net_device_ctx->reconfig_events); + INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup); + + net_device_ctx->vf_stats + = netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats); + if (!net_device_ctx->vf_stats) + goto no_stats; + + net->netdev_ops = &device_ops; + net->ethtool_ops = ðtool_ops; + SET_NETDEV_DEV(net, &dev->device); + dma_set_min_align_mask(&dev->device, HV_HYP_PAGE_SIZE - 1); + + /* We always need headroom for rndis header */ + net->needed_headroom = RNDIS_AND_PPI_SIZE; + + /* Initialize the number of queues to be 1, we may change it if more + * channels are offered later. + */ + netif_set_real_num_tx_queues(net, 1); + netif_set_real_num_rx_queues(net, 1); + + /* Notify the netvsc driver of the new device */ + device_info = netvsc_devinfo_get(NULL); + + if (!device_info) { + ret = -ENOMEM; + goto devinfo_failed; + } + + /* We must get rtnl lock before scheduling nvdev->subchan_work, + * otherwise netvsc_subchan_work() can get rtnl lock first and wait + * all subchannels to show up, but that may not happen because + * netvsc_probe() can't get rtnl lock and as a result vmbus_onoffer() + * -> ... -> device_add() -> ... -> __device_attach() can't get + * the device lock, so all the subchannels can't be processed -- + * finally netvsc_subchan_work() hangs forever. + * + * The rtnl lock also needs to be held before rndis_filter_device_add() + * which advertises nvsp_2_vsc_capability / sriov bit, and triggers + * VF NIC offering and registering. If VF NIC finished register_netdev() + * earlier it may cause name based config failure. + */ + rtnl_lock(); + + nvdev = rndis_filter_device_add(dev, device_info); + if (IS_ERR(nvdev)) { + ret = PTR_ERR(nvdev); + netdev_err(net, "unable to add netvsc device (ret %d)\n", ret); + goto rndis_failed; + } + + eth_hw_addr_set(net, device_info->mac_adr); + + if (nvdev->num_chn > 1) + schedule_work(&nvdev->subchan_work); + + /* hw_features computed in rndis_netdev_set_hwcaps() */ + net->features = net->hw_features | + NETIF_F_HIGHDMA | NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_HW_VLAN_CTAG_RX; + net->vlan_features = net->features; + + netdev_lockdep_set_classes(net); + + /* MTU range: 68 - 1500 or 65521 */ + net->min_mtu = NETVSC_MTU_MIN; + if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2) + net->max_mtu = NETVSC_MTU - ETH_HLEN; + else + net->max_mtu = ETH_DATA_LEN; + + nvdev->tx_disable = false; + + ret = register_netdevice(net); + if (ret != 0) { + pr_err("Unable to register netdev.\n"); + goto register_failed; + } + + list_add(&net_device_ctx->list, &netvsc_dev_list); + rtnl_unlock(); + + netvsc_devinfo_put(device_info); + return 0; + +register_failed: + rndis_filter_device_remove(dev, nvdev); +rndis_failed: + rtnl_unlock(); + netvsc_devinfo_put(device_info); +devinfo_failed: + free_percpu(net_device_ctx->vf_stats); +no_stats: + hv_set_drvdata(dev, NULL); + free_netdev(net); +no_net: + return ret; +} + +static int netvsc_remove(struct hv_device *dev) +{ + struct net_device_context *ndev_ctx; + struct net_device *vf_netdev, *net; + struct netvsc_device *nvdev; + + net = hv_get_drvdata(dev); + if (net == NULL) { + dev_err(&dev->device, "No net device to remove\n"); + return 0; + } + + ndev_ctx = netdev_priv(net); + + cancel_delayed_work_sync(&ndev_ctx->dwork); + + rtnl_lock(); + nvdev = rtnl_dereference(ndev_ctx->nvdev); + if (nvdev) { + cancel_work_sync(&nvdev->subchan_work); + netvsc_xdp_set(net, NULL, NULL, nvdev); + } + + /* + * Call to the vsc driver to let it know that the device is being + * removed. Also blocks mtu and channel changes. + */ + vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev); + if (vf_netdev) + netvsc_unregister_vf(vf_netdev); + + if (nvdev) + rndis_filter_device_remove(dev, nvdev); + + unregister_netdevice(net); + list_del(&ndev_ctx->list); + + rtnl_unlock(); + + hv_set_drvdata(dev, NULL); + + free_percpu(ndev_ctx->vf_stats); + free_netdev(net); + return 0; +} + +static int netvsc_suspend(struct hv_device *dev) +{ + struct net_device_context *ndev_ctx; + struct netvsc_device *nvdev; + struct net_device *net; + int ret; + + net = hv_get_drvdata(dev); + + ndev_ctx = netdev_priv(net); + cancel_delayed_work_sync(&ndev_ctx->dwork); + + rtnl_lock(); + + nvdev = rtnl_dereference(ndev_ctx->nvdev); + if (nvdev == NULL) { + ret = -ENODEV; + goto out; + } + + /* Save the current config info */ + ndev_ctx->saved_netvsc_dev_info = netvsc_devinfo_get(nvdev); + if (!ndev_ctx->saved_netvsc_dev_info) { + ret = -ENOMEM; + goto out; + } + ret = netvsc_detach(net, nvdev); +out: + rtnl_unlock(); + + return ret; +} + +static int netvsc_resume(struct hv_device *dev) +{ + struct net_device *net = hv_get_drvdata(dev); + struct net_device_context *net_device_ctx; + struct netvsc_device_info *device_info; + int ret; + + rtnl_lock(); + + net_device_ctx = netdev_priv(net); + + /* Reset the data path to the netvsc NIC before re-opening the vmbus + * channel. Later netvsc_netdev_event() will switch the data path to + * the VF upon the UP or CHANGE event. + */ + net_device_ctx->data_path_is_vf = false; + device_info = net_device_ctx->saved_netvsc_dev_info; + + ret = netvsc_attach(net, device_info); + + netvsc_devinfo_put(device_info); + net_device_ctx->saved_netvsc_dev_info = NULL; + + rtnl_unlock(); + + return ret; +} +static const struct hv_vmbus_device_id id_table[] = { + /* Network guid */ + { HV_NIC_GUID, }, + { }, +}; + +MODULE_DEVICE_TABLE(vmbus, id_table); + +/* The one and only one */ +static struct hv_driver netvsc_drv = { + .name = KBUILD_MODNAME, + .id_table = id_table, + .probe = netvsc_probe, + .remove = netvsc_remove, + .suspend = netvsc_suspend, + .resume = netvsc_resume, + .driver = { + .probe_type = PROBE_FORCE_SYNCHRONOUS, + }, +}; + +/* + * On Hyper-V, every VF interface is matched with a corresponding + * synthetic interface. The synthetic interface is presented first + * to the guest. When the corresponding VF instance is registered, + * we will take care of switching the data path. + */ +static int netvsc_netdev_event(struct notifier_block *this, + unsigned long event, void *ptr) +{ + struct net_device *event_dev = netdev_notifier_info_to_dev(ptr); + + /* Skip our own events */ + if (event_dev->netdev_ops == &device_ops) + return NOTIFY_DONE; + + /* Avoid non-Ethernet type devices */ + if (event_dev->type != ARPHRD_ETHER) + return NOTIFY_DONE; + + /* Avoid Vlan dev with same MAC registering as VF */ + if (is_vlan_dev(event_dev)) + return NOTIFY_DONE; + + /* Avoid Bonding master dev with same MAC registering as VF */ + if (netif_is_bond_master(event_dev)) + return NOTIFY_DONE; + + switch (event) { + case NETDEV_POST_INIT: + return netvsc_prepare_bonding(event_dev); + case NETDEV_REGISTER: + return netvsc_register_vf(event_dev); + case NETDEV_UNREGISTER: + return netvsc_unregister_vf(event_dev); + case NETDEV_UP: + case NETDEV_DOWN: + case NETDEV_CHANGE: + case NETDEV_GOING_DOWN: + return netvsc_vf_changed(event_dev, event); + default: + return NOTIFY_DONE; + } +} + +static struct notifier_block netvsc_netdev_notifier = { + .notifier_call = netvsc_netdev_event, +}; + +static void __exit netvsc_drv_exit(void) +{ + unregister_netdevice_notifier(&netvsc_netdev_notifier); + vmbus_driver_unregister(&netvsc_drv); +} + +static int __init netvsc_drv_init(void) +{ + int ret; + + if (ring_size < RING_SIZE_MIN) { + ring_size = RING_SIZE_MIN; + pr_info("Increased ring_size to %u (min allowed)\n", + ring_size); + } + netvsc_ring_bytes = VMBUS_RING_SIZE(ring_size * 4096); + + register_netdevice_notifier(&netvsc_netdev_notifier); + + ret = vmbus_driver_register(&netvsc_drv); + if (ret) + goto err_vmbus_reg; + + return 0; + +err_vmbus_reg: + unregister_netdevice_notifier(&netvsc_netdev_notifier); + return ret; +} + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Microsoft Hyper-V network driver"); + +module_init(netvsc_drv_init); +module_exit(netvsc_drv_exit); diff --git a/drivers/net/hyperv/netvsc_trace.c b/drivers/net/hyperv/netvsc_trace.c new file mode 100644 index 000000000..bb0ce5a2b --- /dev/null +++ b/drivers/net/hyperv/netvsc_trace.c @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <linux/netdevice.h> + +#include "hyperv_net.h" + +#define CREATE_TRACE_POINTS +#include "netvsc_trace.h" diff --git a/drivers/net/hyperv/netvsc_trace.h b/drivers/net/hyperv/netvsc_trace.h new file mode 100644 index 000000000..f7585563d --- /dev/null +++ b/drivers/net/hyperv/netvsc_trace.h @@ -0,0 +1,182 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#if !defined(_NETVSC_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _NETVSC_TRACE_H + +#include <linux/tracepoint.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM netvsc +#define TRACE_INCLUDE_FILE netvsc_trace + +TRACE_DEFINE_ENUM(RNDIS_MSG_PACKET); +TRACE_DEFINE_ENUM(RNDIS_MSG_INDICATE); +TRACE_DEFINE_ENUM(RNDIS_MSG_INIT); +TRACE_DEFINE_ENUM(RNDIS_MSG_INIT_C); +TRACE_DEFINE_ENUM(RNDIS_MSG_HALT); +TRACE_DEFINE_ENUM(RNDIS_MSG_QUERY); +TRACE_DEFINE_ENUM(RNDIS_MSG_QUERY_C); +TRACE_DEFINE_ENUM(RNDIS_MSG_SET); +TRACE_DEFINE_ENUM(RNDIS_MSG_SET_C); +TRACE_DEFINE_ENUM(RNDIS_MSG_RESET); +TRACE_DEFINE_ENUM(RNDIS_MSG_RESET_C); +TRACE_DEFINE_ENUM(RNDIS_MSG_KEEPALIVE); +TRACE_DEFINE_ENUM(RNDIS_MSG_KEEPALIVE_C); + +#define show_rndis_type(type) \ + __print_symbolic(type, \ + { RNDIS_MSG_PACKET, "PACKET" }, \ + { RNDIS_MSG_INDICATE, "INDICATE", }, \ + { RNDIS_MSG_INIT, "INIT", }, \ + { RNDIS_MSG_INIT_C, "INIT_C", }, \ + { RNDIS_MSG_HALT, "HALT", }, \ + { RNDIS_MSG_QUERY, "QUERY", }, \ + { RNDIS_MSG_QUERY_C, "QUERY_C", }, \ + { RNDIS_MSG_SET, "SET", }, \ + { RNDIS_MSG_SET_C, "SET_C", }, \ + { RNDIS_MSG_RESET, "RESET", }, \ + { RNDIS_MSG_RESET_C, "RESET_C", }, \ + { RNDIS_MSG_KEEPALIVE, "KEEPALIVE", }, \ + { RNDIS_MSG_KEEPALIVE_C, "KEEPALIVE_C", }) + +DECLARE_EVENT_CLASS(rndis_msg_class, + TP_PROTO(const struct net_device *ndev, u16 q, + const struct rndis_message *msg), + TP_ARGS(ndev, q, msg), + TP_STRUCT__entry( + __string( name, ndev->name ) + __field( u16, queue ) + __field( u32, req_id ) + __field( u32, msg_type ) + __field( u32, msg_len ) + ), + TP_fast_assign( + __assign_str(name, ndev->name); + __entry->queue = q; + __entry->req_id = msg->msg.init_req.req_id; + __entry->msg_type = msg->ndis_msg_type; + __entry->msg_len = msg->msg_len; + ), + TP_printk("dev=%s q=%u req=%#x type=%s msg_len=%u", + __get_str(name), __entry->queue, __entry->req_id, + show_rndis_type(__entry->msg_type), __entry->msg_len) +); + +DEFINE_EVENT(rndis_msg_class, rndis_send, + TP_PROTO(const struct net_device *ndev, u16 q, + const struct rndis_message *msg), + TP_ARGS(ndev, q, msg) +); + +DEFINE_EVENT(rndis_msg_class, rndis_recv, + TP_PROTO(const struct net_device *ndev, u16 q, + const struct rndis_message *msg), + TP_ARGS(ndev, q, msg) +); + +TRACE_DEFINE_ENUM(NVSP_MSG_TYPE_INIT); +TRACE_DEFINE_ENUM(NVSP_MSG_TYPE_INIT_COMPLETE); +TRACE_DEFINE_ENUM(NVSP_MSG1_TYPE_SEND_NDIS_VER); +TRACE_DEFINE_ENUM(NVSP_MSG1_TYPE_SEND_RECV_BUF); +TRACE_DEFINE_ENUM(NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE); +TRACE_DEFINE_ENUM(NVSP_MSG1_TYPE_REVOKE_RECV_BUF); +TRACE_DEFINE_ENUM(NVSP_MSG1_TYPE_SEND_SEND_BUF); +TRACE_DEFINE_ENUM(NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE); +TRACE_DEFINE_ENUM(NVSP_MSG1_TYPE_REVOKE_SEND_BUF); +TRACE_DEFINE_ENUM(NVSP_MSG1_TYPE_SEND_RNDIS_PKT); +TRACE_DEFINE_ENUM(NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE); +TRACE_DEFINE_ENUM(NVSP_MSG2_TYPE_SEND_NDIS_CONFIG); + +TRACE_DEFINE_ENUM(NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION); +TRACE_DEFINE_ENUM(NVSP_MSG4_TYPE_SWITCH_DATA_PATH); + +TRACE_DEFINE_ENUM(NVSP_MSG5_TYPE_SUBCHANNEL); +TRACE_DEFINE_ENUM(NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE); + +#define show_nvsp_type(type) \ + __print_symbolic(type, \ + { NVSP_MSG_TYPE_INIT, "INIT" }, \ + { NVSP_MSG_TYPE_INIT_COMPLETE, "INIT_COMPLETE" }, \ + { NVSP_MSG1_TYPE_SEND_NDIS_VER, "SEND_NDIS_VER" }, \ + { NVSP_MSG1_TYPE_SEND_RECV_BUF, "SEND_RECV_BUF" }, \ + { NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE, "SEND_RECV_BUF_COMPLETE" }, \ + { NVSP_MSG1_TYPE_REVOKE_RECV_BUF, "REVOKE_RECV_BUF" }, \ + { NVSP_MSG1_TYPE_SEND_SEND_BUF, "SEND_SEND_BUF" }, \ + { NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE, "SEND_SEND_BUF_COMPLETE" }, \ + { NVSP_MSG1_TYPE_REVOKE_SEND_BUF, "REVOKE_SEND_BUF" }, \ + { NVSP_MSG1_TYPE_SEND_RNDIS_PKT, "SEND_RNDIS_PKT" }, \ + { NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE, "SEND_RNDIS_PKT_COMPLETE" },\ + { NVSP_MSG2_TYPE_SEND_NDIS_CONFIG, "SEND_NDIS_CONFIG" }, \ + { NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION, "SEND_VF_ASSOCIATION" }, \ + { NVSP_MSG4_TYPE_SWITCH_DATA_PATH, "SWITCH_DATA_PATH" }, \ + { NVSP_MSG5_TYPE_SUBCHANNEL, "SUBCHANNEL" }, \ + { NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE, "SEND_INDIRECTION_TABLE" }) + +TRACE_EVENT(nvsp_send, + TP_PROTO(const struct net_device *ndev, + const struct nvsp_message *msg), + TP_ARGS(ndev, msg), + TP_STRUCT__entry( + __string( name, ndev->name ) + __field( u32, msg_type ) + ), + TP_fast_assign( + __assign_str(name, ndev->name); + __entry->msg_type = msg->hdr.msg_type; + ), + TP_printk("dev=%s type=%s", + __get_str(name), + show_nvsp_type(__entry->msg_type)) +); + +TRACE_EVENT(nvsp_send_pkt, + TP_PROTO(const struct net_device *ndev, + const struct vmbus_channel *chan, + const struct nvsp_1_message_send_rndis_packet *rpkt), + TP_ARGS(ndev, chan, rpkt), + TP_STRUCT__entry( + __string( name, ndev->name ) + __field( u16, qid ) + __field( u32, channel_type ) + __field( u32, section_index ) + __field( u32, section_size ) + ), + TP_fast_assign( + __assign_str(name, ndev->name); + __entry->qid = chan->offermsg.offer.sub_channel_index; + __entry->channel_type = rpkt->channel_type; + __entry->section_index = rpkt->send_buf_section_index; + __entry->section_size = rpkt->send_buf_section_size; + ), + TP_printk("dev=%s qid=%u type=%s section=%u size=%d", + __get_str(name), __entry->qid, + __entry->channel_type ? "CONTROL" : "DATA", + __entry->section_index, __entry->section_size) +); + +TRACE_EVENT(nvsp_recv, + TP_PROTO(const struct net_device *ndev, + const struct vmbus_channel *chan, + const struct nvsp_message *msg), + TP_ARGS(ndev, chan, msg), + TP_STRUCT__entry( + __string( name, ndev->name ) + __field( u16, qid ) + __field( u32, msg_type ) + ), + TP_fast_assign( + __assign_str(name, ndev->name); + __entry->qid = chan->offermsg.offer.sub_channel_index; + __entry->msg_type = msg->hdr.msg_type; + ), + TP_printk("dev=%s qid=%u type=%s", + __get_str(name), __entry->qid, + show_nvsp_type(__entry->msg_type)) +); + +#endif /* _NETVSC_TRACE_H */ + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../drivers/net/hyperv +#include <trace/define_trace.h> diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c new file mode 100644 index 000000000..eea777ec2 --- /dev/null +++ b/drivers/net/hyperv/rndis_filter.c @@ -0,0 +1,1620 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2009, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + */ +#include <linux/ethtool.h> +#include <linux/kernel.h> +#include <linux/sched.h> +#include <linux/wait.h> +#include <linux/highmem.h> +#include <linux/slab.h> +#include <linux/io.h> +#include <linux/if_ether.h> +#include <linux/netdevice.h> +#include <linux/if_vlan.h> +#include <linux/nls.h> +#include <linux/vmalloc.h> +#include <linux/rtnetlink.h> +#include <linux/ucs2_string.h> +#include <linux/string.h> + +#include "hyperv_net.h" +#include "netvsc_trace.h" + +static void rndis_set_multicast(struct work_struct *w); + +#define RNDIS_EXT_LEN HV_HYP_PAGE_SIZE +struct rndis_request { + struct list_head list_ent; + struct completion wait_event; + + struct rndis_message response_msg; + /* + * The buffer for extended info after the RNDIS response message. It's + * referenced based on the data offset in the RNDIS message. Its size + * is enough for current needs, and should be sufficient for the near + * future. + */ + u8 response_ext[RNDIS_EXT_LEN]; + + /* Simplify allocation by having a netvsc packet inline */ + struct hv_netvsc_packet pkt; + + struct rndis_message request_msg; + /* + * The buffer for the extended info after the RNDIS request message. + * It is referenced and sized in a similar way as response_ext. + */ + u8 request_ext[RNDIS_EXT_LEN]; +}; + +static const u8 netvsc_hash_key[NETVSC_HASH_KEYLEN] = { + 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, + 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, + 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, + 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, + 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa +}; + +static struct rndis_device *get_rndis_device(void) +{ + struct rndis_device *device; + + device = kzalloc(sizeof(struct rndis_device), GFP_KERNEL); + if (!device) + return NULL; + + spin_lock_init(&device->request_lock); + + INIT_LIST_HEAD(&device->req_list); + INIT_WORK(&device->mcast_work, rndis_set_multicast); + + device->state = RNDIS_DEV_UNINITIALIZED; + + return device; +} + +static struct rndis_request *get_rndis_request(struct rndis_device *dev, + u32 msg_type, + u32 msg_len) +{ + struct rndis_request *request; + struct rndis_message *rndis_msg; + struct rndis_set_request *set; + unsigned long flags; + + request = kzalloc(sizeof(struct rndis_request), GFP_KERNEL); + if (!request) + return NULL; + + init_completion(&request->wait_event); + + rndis_msg = &request->request_msg; + rndis_msg->ndis_msg_type = msg_type; + rndis_msg->msg_len = msg_len; + + request->pkt.q_idx = 0; + + /* + * Set the request id. This field is always after the rndis header for + * request/response packet types so we just used the SetRequest as a + * template + */ + set = &rndis_msg->msg.set_req; + set->req_id = atomic_inc_return(&dev->new_req_id); + + /* Add to the request list */ + spin_lock_irqsave(&dev->request_lock, flags); + list_add_tail(&request->list_ent, &dev->req_list); + spin_unlock_irqrestore(&dev->request_lock, flags); + + return request; +} + +static void put_rndis_request(struct rndis_device *dev, + struct rndis_request *req) +{ + unsigned long flags; + + spin_lock_irqsave(&dev->request_lock, flags); + list_del(&req->list_ent); + spin_unlock_irqrestore(&dev->request_lock, flags); + + kfree(req); +} + +static void dump_rndis_message(struct net_device *netdev, + const struct rndis_message *rndis_msg, + const void *data) +{ + switch (rndis_msg->ndis_msg_type) { + case RNDIS_MSG_PACKET: + if (rndis_msg->msg_len - RNDIS_HEADER_SIZE >= sizeof(struct rndis_packet)) { + const struct rndis_packet *pkt = data + RNDIS_HEADER_SIZE; + netdev_dbg(netdev, "RNDIS_MSG_PACKET (len %u, " + "data offset %u data len %u, # oob %u, " + "oob offset %u, oob len %u, pkt offset %u, " + "pkt len %u\n", + rndis_msg->msg_len, + pkt->data_offset, + pkt->data_len, + pkt->num_oob_data_elements, + pkt->oob_data_offset, + pkt->oob_data_len, + pkt->per_pkt_info_offset, + pkt->per_pkt_info_len); + } + break; + + case RNDIS_MSG_INIT_C: + if (rndis_msg->msg_len - RNDIS_HEADER_SIZE >= + sizeof(struct rndis_initialize_complete)) { + const struct rndis_initialize_complete *init_complete = + data + RNDIS_HEADER_SIZE; + netdev_dbg(netdev, "RNDIS_MSG_INIT_C " + "(len %u, id 0x%x, status 0x%x, major %d, minor %d, " + "device flags %d, max xfer size 0x%x, max pkts %u, " + "pkt aligned %u)\n", + rndis_msg->msg_len, + init_complete->req_id, + init_complete->status, + init_complete->major_ver, + init_complete->minor_ver, + init_complete->dev_flags, + init_complete->max_xfer_size, + init_complete->max_pkt_per_msg, + init_complete->pkt_alignment_factor); + } + break; + + case RNDIS_MSG_QUERY_C: + if (rndis_msg->msg_len - RNDIS_HEADER_SIZE >= + sizeof(struct rndis_query_complete)) { + const struct rndis_query_complete *query_complete = + data + RNDIS_HEADER_SIZE; + netdev_dbg(netdev, "RNDIS_MSG_QUERY_C " + "(len %u, id 0x%x, status 0x%x, buf len %u, " + "buf offset %u)\n", + rndis_msg->msg_len, + query_complete->req_id, + query_complete->status, + query_complete->info_buflen, + query_complete->info_buf_offset); + } + break; + + case RNDIS_MSG_SET_C: + if (rndis_msg->msg_len - RNDIS_HEADER_SIZE + sizeof(struct rndis_set_complete)) { + const struct rndis_set_complete *set_complete = + data + RNDIS_HEADER_SIZE; + netdev_dbg(netdev, + "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)\n", + rndis_msg->msg_len, + set_complete->req_id, + set_complete->status); + } + break; + + case RNDIS_MSG_INDICATE: + if (rndis_msg->msg_len - RNDIS_HEADER_SIZE >= + sizeof(struct rndis_indicate_status)) { + const struct rndis_indicate_status *indicate_status = + data + RNDIS_HEADER_SIZE; + netdev_dbg(netdev, "RNDIS_MSG_INDICATE " + "(len %u, status 0x%x, buf len %u, buf offset %u)\n", + rndis_msg->msg_len, + indicate_status->status, + indicate_status->status_buflen, + indicate_status->status_buf_offset); + } + break; + + default: + netdev_dbg(netdev, "0x%x (len %u)\n", + rndis_msg->ndis_msg_type, + rndis_msg->msg_len); + break; + } +} + +static int rndis_filter_send_request(struct rndis_device *dev, + struct rndis_request *req) +{ + struct hv_netvsc_packet *packet; + struct hv_page_buffer page_buf[2]; + struct hv_page_buffer *pb = page_buf; + int ret; + + /* Setup the packet to send it */ + packet = &req->pkt; + + packet->total_data_buflen = req->request_msg.msg_len; + packet->page_buf_cnt = 1; + + pb[0].pfn = virt_to_phys(&req->request_msg) >> + HV_HYP_PAGE_SHIFT; + pb[0].len = req->request_msg.msg_len; + pb[0].offset = offset_in_hvpage(&req->request_msg); + + /* Add one page_buf when request_msg crossing page boundary */ + if (pb[0].offset + pb[0].len > HV_HYP_PAGE_SIZE) { + packet->page_buf_cnt++; + pb[0].len = HV_HYP_PAGE_SIZE - + pb[0].offset; + pb[1].pfn = virt_to_phys((void *)&req->request_msg + + pb[0].len) >> HV_HYP_PAGE_SHIFT; + pb[1].offset = 0; + pb[1].len = req->request_msg.msg_len - + pb[0].len; + } + + trace_rndis_send(dev->ndev, 0, &req->request_msg); + + rcu_read_lock_bh(); + ret = netvsc_send(dev->ndev, packet, NULL, pb, NULL, false); + rcu_read_unlock_bh(); + + return ret; +} + +static void rndis_set_link_state(struct rndis_device *rdev, + struct rndis_request *request) +{ + u32 link_status; + struct rndis_query_complete *query_complete; + u32 msg_len = request->response_msg.msg_len; + + /* Ensure the packet is big enough to access its fields */ + if (msg_len - RNDIS_HEADER_SIZE < sizeof(struct rndis_query_complete)) + return; + + query_complete = &request->response_msg.msg.query_complete; + + if (query_complete->status == RNDIS_STATUS_SUCCESS && + query_complete->info_buflen >= sizeof(u32) && + query_complete->info_buf_offset >= sizeof(*query_complete) && + msg_len - RNDIS_HEADER_SIZE >= query_complete->info_buf_offset && + msg_len - RNDIS_HEADER_SIZE - query_complete->info_buf_offset + >= query_complete->info_buflen) { + memcpy(&link_status, (void *)((unsigned long)query_complete + + query_complete->info_buf_offset), sizeof(u32)); + rdev->link_state = link_status != 0; + } +} + +static void rndis_filter_receive_response(struct net_device *ndev, + struct netvsc_device *nvdev, + struct rndis_message *resp, + void *data) +{ + u32 *req_id = &resp->msg.init_complete.req_id; + struct rndis_device *dev = nvdev->extension; + struct rndis_request *request = NULL; + bool found = false; + unsigned long flags; + + /* This should never happen, it means control message + * response received after device removed. + */ + if (dev->state == RNDIS_DEV_UNINITIALIZED) { + netdev_err(ndev, + "got rndis message uninitialized\n"); + return; + } + + /* Ensure the packet is big enough to read req_id. Req_id is the 1st + * field in any request/response message, so the payload should have at + * least sizeof(u32) bytes + */ + if (resp->msg_len - RNDIS_HEADER_SIZE < sizeof(u32)) { + netdev_err(ndev, "rndis msg_len too small: %u\n", + resp->msg_len); + return; + } + + /* Copy the request ID into nvchan->recv_buf */ + *req_id = *(u32 *)(data + RNDIS_HEADER_SIZE); + + spin_lock_irqsave(&dev->request_lock, flags); + list_for_each_entry(request, &dev->req_list, list_ent) { + /* + * All request/response message contains RequestId as the 1st + * field + */ + if (request->request_msg.msg.init_req.req_id == *req_id) { + found = true; + break; + } + } + spin_unlock_irqrestore(&dev->request_lock, flags); + + if (found) { + if (resp->msg_len <= + sizeof(struct rndis_message) + RNDIS_EXT_LEN) { + memcpy(&request->response_msg, resp, RNDIS_HEADER_SIZE + sizeof(*req_id)); + unsafe_memcpy((void *)&request->response_msg + RNDIS_HEADER_SIZE + sizeof(*req_id), + data + RNDIS_HEADER_SIZE + sizeof(*req_id), + resp->msg_len - RNDIS_HEADER_SIZE - sizeof(*req_id), + "request->response_msg is followed by a padding of RNDIS_EXT_LEN inside rndis_request"); + if (request->request_msg.ndis_msg_type == + RNDIS_MSG_QUERY && request->request_msg.msg. + query_req.oid == RNDIS_OID_GEN_MEDIA_CONNECT_STATUS) + rndis_set_link_state(dev, request); + } else { + netdev_err(ndev, + "rndis response buffer overflow " + "detected (size %u max %zu)\n", + resp->msg_len, + sizeof(struct rndis_message)); + + if (resp->ndis_msg_type == + RNDIS_MSG_RESET_C) { + /* does not have a request id field */ + request->response_msg.msg.reset_complete. + status = RNDIS_STATUS_BUFFER_OVERFLOW; + } else { + request->response_msg.msg. + init_complete.status = + RNDIS_STATUS_BUFFER_OVERFLOW; + } + } + + netvsc_dma_unmap(((struct net_device_context *) + netdev_priv(ndev))->device_ctx, &request->pkt); + complete(&request->wait_event); + } else { + netdev_err(ndev, + "no rndis request found for this response " + "(id 0x%x res type 0x%x)\n", + *req_id, + resp->ndis_msg_type); + } +} + +/* + * Get the Per-Packet-Info with the specified type + * return NULL if not found. + */ +static inline void *rndis_get_ppi(struct net_device *ndev, + struct rndis_packet *rpkt, + u32 rpkt_len, u32 type, u8 internal, + u32 ppi_size, void *data) +{ + struct rndis_per_packet_info *ppi; + int len; + + if (rpkt->per_pkt_info_offset == 0) + return NULL; + + /* Validate info_offset and info_len */ + if (rpkt->per_pkt_info_offset < sizeof(struct rndis_packet) || + rpkt->per_pkt_info_offset > rpkt_len) { + netdev_err(ndev, "Invalid per_pkt_info_offset: %u\n", + rpkt->per_pkt_info_offset); + return NULL; + } + + if (rpkt->per_pkt_info_len < sizeof(*ppi) || + rpkt->per_pkt_info_len > rpkt_len - rpkt->per_pkt_info_offset) { + netdev_err(ndev, "Invalid per_pkt_info_len: %u\n", + rpkt->per_pkt_info_len); + return NULL; + } + + ppi = (struct rndis_per_packet_info *)((ulong)rpkt + + rpkt->per_pkt_info_offset); + /* Copy the PPIs into nvchan->recv_buf */ + memcpy(ppi, data + RNDIS_HEADER_SIZE + rpkt->per_pkt_info_offset, rpkt->per_pkt_info_len); + len = rpkt->per_pkt_info_len; + + while (len > 0) { + /* Validate ppi_offset and ppi_size */ + if (ppi->size > len) { + netdev_err(ndev, "Invalid ppi size: %u\n", ppi->size); + continue; + } + + if (ppi->ppi_offset >= ppi->size) { + netdev_err(ndev, "Invalid ppi_offset: %u\n", ppi->ppi_offset); + continue; + } + + if (ppi->type == type && ppi->internal == internal) { + /* ppi->size should be big enough to hold the returned object. */ + if (ppi->size - ppi->ppi_offset < ppi_size) { + netdev_err(ndev, "Invalid ppi: size %u ppi_offset %u\n", + ppi->size, ppi->ppi_offset); + continue; + } + return (void *)((ulong)ppi + ppi->ppi_offset); + } + len -= ppi->size; + ppi = (struct rndis_per_packet_info *)((ulong)ppi + ppi->size); + } + + return NULL; +} + +static inline +void rsc_add_data(struct netvsc_channel *nvchan, + const struct ndis_pkt_8021q_info *vlan, + const struct ndis_tcp_ip_checksum_info *csum_info, + const u32 *hash_info, + void *data, u32 len) +{ + u32 cnt = nvchan->rsc.cnt; + + if (cnt) { + nvchan->rsc.pktlen += len; + } else { + /* The data/values pointed by vlan, csum_info and hash_info are shared + * across the different 'fragments' of the RSC packet; store them into + * the packet itself. + */ + if (vlan != NULL) { + memcpy(&nvchan->rsc.vlan, vlan, sizeof(*vlan)); + nvchan->rsc.ppi_flags |= NVSC_RSC_VLAN; + } else { + nvchan->rsc.ppi_flags &= ~NVSC_RSC_VLAN; + } + if (csum_info != NULL) { + memcpy(&nvchan->rsc.csum_info, csum_info, sizeof(*csum_info)); + nvchan->rsc.ppi_flags |= NVSC_RSC_CSUM_INFO; + } else { + nvchan->rsc.ppi_flags &= ~NVSC_RSC_CSUM_INFO; + } + nvchan->rsc.pktlen = len; + if (hash_info != NULL) { + nvchan->rsc.hash_info = *hash_info; + nvchan->rsc.ppi_flags |= NVSC_RSC_HASH_INFO; + } else { + nvchan->rsc.ppi_flags &= ~NVSC_RSC_HASH_INFO; + } + } + + nvchan->rsc.data[cnt] = data; + nvchan->rsc.len[cnt] = len; + nvchan->rsc.cnt++; +} + +static int rndis_filter_receive_data(struct net_device *ndev, + struct netvsc_device *nvdev, + struct netvsc_channel *nvchan, + struct rndis_message *msg, + void *data, u32 data_buflen) +{ + struct rndis_packet *rndis_pkt = &msg->msg.pkt; + const struct ndis_tcp_ip_checksum_info *csum_info; + const struct ndis_pkt_8021q_info *vlan; + const struct rndis_pktinfo_id *pktinfo_id; + const u32 *hash_info; + u32 data_offset, rpkt_len; + bool rsc_more = false; + int ret; + + /* Ensure data_buflen is big enough to read header fields */ + if (data_buflen < RNDIS_HEADER_SIZE + sizeof(struct rndis_packet)) { + netdev_err(ndev, "invalid rndis pkt, data_buflen too small: %u\n", + data_buflen); + return NVSP_STAT_FAIL; + } + + /* Copy the RNDIS packet into nvchan->recv_buf */ + memcpy(rndis_pkt, data + RNDIS_HEADER_SIZE, sizeof(*rndis_pkt)); + + /* Validate rndis_pkt offset */ + if (rndis_pkt->data_offset >= data_buflen - RNDIS_HEADER_SIZE) { + netdev_err(ndev, "invalid rndis packet offset: %u\n", + rndis_pkt->data_offset); + return NVSP_STAT_FAIL; + } + + /* Remove the rndis header and pass it back up the stack */ + data_offset = RNDIS_HEADER_SIZE + rndis_pkt->data_offset; + + rpkt_len = data_buflen - RNDIS_HEADER_SIZE; + data_buflen -= data_offset; + + /* + * Make sure we got a valid RNDIS message, now total_data_buflen + * should be the data packet size plus the trailer padding size + */ + if (unlikely(data_buflen < rndis_pkt->data_len)) { + netdev_err(ndev, "rndis message buffer " + "overflow detected (got %u, min %u)" + "...dropping this message!\n", + data_buflen, rndis_pkt->data_len); + return NVSP_STAT_FAIL; + } + + vlan = rndis_get_ppi(ndev, rndis_pkt, rpkt_len, IEEE_8021Q_INFO, 0, sizeof(*vlan), + data); + + csum_info = rndis_get_ppi(ndev, rndis_pkt, rpkt_len, TCPIP_CHKSUM_PKTINFO, 0, + sizeof(*csum_info), data); + + hash_info = rndis_get_ppi(ndev, rndis_pkt, rpkt_len, NBL_HASH_VALUE, 0, + sizeof(*hash_info), data); + + pktinfo_id = rndis_get_ppi(ndev, rndis_pkt, rpkt_len, RNDIS_PKTINFO_ID, 1, + sizeof(*pktinfo_id), data); + + /* Identify RSC frags, drop erroneous packets */ + if (pktinfo_id && (pktinfo_id->flag & RNDIS_PKTINFO_SUBALLOC)) { + if (pktinfo_id->flag & RNDIS_PKTINFO_1ST_FRAG) + nvchan->rsc.cnt = 0; + else if (nvchan->rsc.cnt == 0) + goto drop; + + rsc_more = true; + + if (pktinfo_id->flag & RNDIS_PKTINFO_LAST_FRAG) + rsc_more = false; + + if (rsc_more && nvchan->rsc.is_last) + goto drop; + } else { + nvchan->rsc.cnt = 0; + } + + if (unlikely(nvchan->rsc.cnt >= NVSP_RSC_MAX)) + goto drop; + + /* Put data into per channel structure. + * Also, remove the rndis trailer padding from rndis packet message + * rndis_pkt->data_len tell us the real data length, we only copy + * the data packet to the stack, without the rndis trailer padding + */ + rsc_add_data(nvchan, vlan, csum_info, hash_info, + data + data_offset, rndis_pkt->data_len); + + if (rsc_more) + return NVSP_STAT_SUCCESS; + + ret = netvsc_recv_callback(ndev, nvdev, nvchan); + nvchan->rsc.cnt = 0; + + return ret; + +drop: + return NVSP_STAT_FAIL; +} + +int rndis_filter_receive(struct net_device *ndev, + struct netvsc_device *net_dev, + struct netvsc_channel *nvchan, + void *data, u32 buflen) +{ + struct net_device_context *net_device_ctx = netdev_priv(ndev); + struct rndis_message *rndis_msg = nvchan->recv_buf; + + if (buflen < RNDIS_HEADER_SIZE) { + netdev_err(ndev, "Invalid rndis_msg (buflen: %u)\n", buflen); + return NVSP_STAT_FAIL; + } + + /* Copy the RNDIS msg header into nvchan->recv_buf */ + memcpy(rndis_msg, data, RNDIS_HEADER_SIZE); + + /* Validate incoming rndis_message packet */ + if (rndis_msg->msg_len < RNDIS_HEADER_SIZE || + buflen < rndis_msg->msg_len) { + netdev_err(ndev, "Invalid rndis_msg (buflen: %u, msg_len: %u)\n", + buflen, rndis_msg->msg_len); + return NVSP_STAT_FAIL; + } + + if (netif_msg_rx_status(net_device_ctx)) + dump_rndis_message(ndev, rndis_msg, data); + + switch (rndis_msg->ndis_msg_type) { + case RNDIS_MSG_PACKET: + return rndis_filter_receive_data(ndev, net_dev, nvchan, + rndis_msg, data, buflen); + case RNDIS_MSG_INIT_C: + case RNDIS_MSG_QUERY_C: + case RNDIS_MSG_SET_C: + /* completion msgs */ + rndis_filter_receive_response(ndev, net_dev, rndis_msg, data); + break; + + case RNDIS_MSG_INDICATE: + /* notification msgs */ + netvsc_linkstatus_callback(ndev, rndis_msg, data, buflen); + break; + default: + netdev_err(ndev, + "unhandled rndis message (type %u len %u)\n", + rndis_msg->ndis_msg_type, + rndis_msg->msg_len); + return NVSP_STAT_FAIL; + } + + return NVSP_STAT_SUCCESS; +} + +static int rndis_filter_query_device(struct rndis_device *dev, + struct netvsc_device *nvdev, + u32 oid, void *result, u32 *result_size) +{ + struct rndis_request *request; + u32 inresult_size = *result_size; + struct rndis_query_request *query; + struct rndis_query_complete *query_complete; + u32 msg_len; + int ret = 0; + + if (!result) + return -EINVAL; + + *result_size = 0; + request = get_rndis_request(dev, RNDIS_MSG_QUERY, + RNDIS_MESSAGE_SIZE(struct rndis_query_request)); + if (!request) { + ret = -ENOMEM; + goto cleanup; + } + + /* Setup the rndis query */ + query = &request->request_msg.msg.query_req; + query->oid = oid; + query->info_buf_offset = sizeof(struct rndis_query_request); + query->info_buflen = 0; + query->dev_vc_handle = 0; + + if (oid == OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES) { + struct ndis_offload *hwcaps; + u32 nvsp_version = nvdev->nvsp_version; + u8 ndis_rev; + size_t size; + + if (nvsp_version >= NVSP_PROTOCOL_VERSION_5) { + ndis_rev = NDIS_OFFLOAD_PARAMETERS_REVISION_3; + size = NDIS_OFFLOAD_SIZE; + } else if (nvsp_version >= NVSP_PROTOCOL_VERSION_4) { + ndis_rev = NDIS_OFFLOAD_PARAMETERS_REVISION_2; + size = NDIS_OFFLOAD_SIZE_6_1; + } else { + ndis_rev = NDIS_OFFLOAD_PARAMETERS_REVISION_1; + size = NDIS_OFFLOAD_SIZE_6_0; + } + + request->request_msg.msg_len += size; + query->info_buflen = size; + hwcaps = (struct ndis_offload *) + ((unsigned long)query + query->info_buf_offset); + + hwcaps->header.type = NDIS_OBJECT_TYPE_OFFLOAD; + hwcaps->header.revision = ndis_rev; + hwcaps->header.size = size; + + } else if (oid == OID_GEN_RECEIVE_SCALE_CAPABILITIES) { + struct ndis_recv_scale_cap *cap; + + request->request_msg.msg_len += + sizeof(struct ndis_recv_scale_cap); + query->info_buflen = sizeof(struct ndis_recv_scale_cap); + cap = (struct ndis_recv_scale_cap *)((unsigned long)query + + query->info_buf_offset); + cap->hdr.type = NDIS_OBJECT_TYPE_RSS_CAPABILITIES; + cap->hdr.rev = NDIS_RECEIVE_SCALE_CAPABILITIES_REVISION_2; + cap->hdr.size = sizeof(struct ndis_recv_scale_cap); + } + + ret = rndis_filter_send_request(dev, request); + if (ret != 0) + goto cleanup; + + wait_for_completion(&request->wait_event); + + /* Copy the response back */ + query_complete = &request->response_msg.msg.query_complete; + msg_len = request->response_msg.msg_len; + + /* Ensure the packet is big enough to access its fields */ + if (msg_len - RNDIS_HEADER_SIZE < sizeof(struct rndis_query_complete)) { + ret = -1; + goto cleanup; + } + + if (query_complete->info_buflen > inresult_size || + query_complete->info_buf_offset < sizeof(*query_complete) || + msg_len - RNDIS_HEADER_SIZE < query_complete->info_buf_offset || + msg_len - RNDIS_HEADER_SIZE - query_complete->info_buf_offset + < query_complete->info_buflen) { + ret = -1; + goto cleanup; + } + + memcpy(result, + (void *)((unsigned long)query_complete + + query_complete->info_buf_offset), + query_complete->info_buflen); + + *result_size = query_complete->info_buflen; + +cleanup: + if (request) + put_rndis_request(dev, request); + + return ret; +} + +/* Get the hardware offload capabilities */ +static int +rndis_query_hwcaps(struct rndis_device *dev, struct netvsc_device *net_device, + struct ndis_offload *caps) +{ + u32 caps_len = sizeof(*caps); + int ret; + + memset(caps, 0, sizeof(*caps)); + + ret = rndis_filter_query_device(dev, net_device, + OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES, + caps, &caps_len); + if (ret) + return ret; + + if (caps->header.type != NDIS_OBJECT_TYPE_OFFLOAD) { + netdev_warn(dev->ndev, "invalid NDIS objtype %#x\n", + caps->header.type); + return -EINVAL; + } + + if (caps->header.revision < NDIS_OFFLOAD_PARAMETERS_REVISION_1) { + netdev_warn(dev->ndev, "invalid NDIS objrev %x\n", + caps->header.revision); + return -EINVAL; + } + + if (caps->header.size > caps_len || + caps->header.size < NDIS_OFFLOAD_SIZE_6_0) { + netdev_warn(dev->ndev, + "invalid NDIS objsize %u, data size %u\n", + caps->header.size, caps_len); + return -EINVAL; + } + + return 0; +} + +static int rndis_filter_query_device_mac(struct rndis_device *dev, + struct netvsc_device *net_device) +{ + u32 size = ETH_ALEN; + + return rndis_filter_query_device(dev, net_device, + RNDIS_OID_802_3_PERMANENT_ADDRESS, + dev->hw_mac_adr, &size); +} + +#define NWADR_STR "NetworkAddress" +#define NWADR_STRLEN 14 + +int rndis_filter_set_device_mac(struct netvsc_device *nvdev, + const char *mac) +{ + struct rndis_device *rdev = nvdev->extension; + struct rndis_request *request; + struct rndis_set_request *set; + struct rndis_config_parameter_info *cpi; + wchar_t *cfg_nwadr, *cfg_mac; + struct rndis_set_complete *set_complete; + char macstr[2*ETH_ALEN+1]; + u32 extlen = sizeof(struct rndis_config_parameter_info) + + 2*NWADR_STRLEN + 4*ETH_ALEN; + int ret; + + request = get_rndis_request(rdev, RNDIS_MSG_SET, + RNDIS_MESSAGE_SIZE(struct rndis_set_request) + extlen); + if (!request) + return -ENOMEM; + + set = &request->request_msg.msg.set_req; + set->oid = RNDIS_OID_GEN_RNDIS_CONFIG_PARAMETER; + set->info_buflen = extlen; + set->info_buf_offset = sizeof(struct rndis_set_request); + set->dev_vc_handle = 0; + + cpi = (struct rndis_config_parameter_info *)((ulong)set + + set->info_buf_offset); + cpi->parameter_name_offset = + sizeof(struct rndis_config_parameter_info); + /* Multiply by 2 because host needs 2 bytes (utf16) for each char */ + cpi->parameter_name_length = 2*NWADR_STRLEN; + cpi->parameter_type = RNDIS_CONFIG_PARAM_TYPE_STRING; + cpi->parameter_value_offset = + cpi->parameter_name_offset + cpi->parameter_name_length; + /* Multiply by 4 because each MAC byte displayed as 2 utf16 chars */ + cpi->parameter_value_length = 4*ETH_ALEN; + + cfg_nwadr = (wchar_t *)((ulong)cpi + cpi->parameter_name_offset); + cfg_mac = (wchar_t *)((ulong)cpi + cpi->parameter_value_offset); + ret = utf8s_to_utf16s(NWADR_STR, NWADR_STRLEN, UTF16_HOST_ENDIAN, + cfg_nwadr, NWADR_STRLEN); + if (ret < 0) + goto cleanup; + snprintf(macstr, 2*ETH_ALEN+1, "%pm", mac); + ret = utf8s_to_utf16s(macstr, 2*ETH_ALEN, UTF16_HOST_ENDIAN, + cfg_mac, 2*ETH_ALEN); + if (ret < 0) + goto cleanup; + + ret = rndis_filter_send_request(rdev, request); + if (ret != 0) + goto cleanup; + + wait_for_completion(&request->wait_event); + + set_complete = &request->response_msg.msg.set_complete; + if (set_complete->status != RNDIS_STATUS_SUCCESS) + ret = -EIO; + +cleanup: + put_rndis_request(rdev, request); + return ret; +} + +int +rndis_filter_set_offload_params(struct net_device *ndev, + struct netvsc_device *nvdev, + struct ndis_offload_params *req_offloads) +{ + struct rndis_device *rdev = nvdev->extension; + struct rndis_request *request; + struct rndis_set_request *set; + struct ndis_offload_params *offload_params; + struct rndis_set_complete *set_complete; + u32 extlen = sizeof(struct ndis_offload_params); + int ret; + u32 vsp_version = nvdev->nvsp_version; + + if (vsp_version <= NVSP_PROTOCOL_VERSION_4) { + extlen = VERSION_4_OFFLOAD_SIZE; + /* On NVSP_PROTOCOL_VERSION_4 and below, we do not support + * UDP checksum offload. + */ + req_offloads->udp_ip_v4_csum = 0; + req_offloads->udp_ip_v6_csum = 0; + } + + request = get_rndis_request(rdev, RNDIS_MSG_SET, + RNDIS_MESSAGE_SIZE(struct rndis_set_request) + extlen); + if (!request) + return -ENOMEM; + + set = &request->request_msg.msg.set_req; + set->oid = OID_TCP_OFFLOAD_PARAMETERS; + set->info_buflen = extlen; + set->info_buf_offset = sizeof(struct rndis_set_request); + set->dev_vc_handle = 0; + + offload_params = (struct ndis_offload_params *)((ulong)set + + set->info_buf_offset); + *offload_params = *req_offloads; + offload_params->header.type = NDIS_OBJECT_TYPE_DEFAULT; + offload_params->header.revision = NDIS_OFFLOAD_PARAMETERS_REVISION_3; + offload_params->header.size = extlen; + + ret = rndis_filter_send_request(rdev, request); + if (ret != 0) + goto cleanup; + + wait_for_completion(&request->wait_event); + set_complete = &request->response_msg.msg.set_complete; + if (set_complete->status != RNDIS_STATUS_SUCCESS) { + netdev_err(ndev, "Fail to set offload on host side:0x%x\n", + set_complete->status); + ret = -EINVAL; + } + +cleanup: + put_rndis_request(rdev, request); + return ret; +} + +static int rndis_set_rss_param_msg(struct rndis_device *rdev, + const u8 *rss_key, u16 flag) +{ + struct net_device *ndev = rdev->ndev; + struct net_device_context *ndc = netdev_priv(ndev); + struct rndis_request *request; + struct rndis_set_request *set; + struct rndis_set_complete *set_complete; + u32 extlen = sizeof(struct ndis_recv_scale_param) + + 4 * ITAB_NUM + NETVSC_HASH_KEYLEN; + struct ndis_recv_scale_param *rssp; + u32 *itab; + u8 *keyp; + int i, ret; + + request = get_rndis_request( + rdev, RNDIS_MSG_SET, + RNDIS_MESSAGE_SIZE(struct rndis_set_request) + extlen); + if (!request) + return -ENOMEM; + + set = &request->request_msg.msg.set_req; + set->oid = OID_GEN_RECEIVE_SCALE_PARAMETERS; + set->info_buflen = extlen; + set->info_buf_offset = sizeof(struct rndis_set_request); + set->dev_vc_handle = 0; + + rssp = (struct ndis_recv_scale_param *)(set + 1); + rssp->hdr.type = NDIS_OBJECT_TYPE_RSS_PARAMETERS; + rssp->hdr.rev = NDIS_RECEIVE_SCALE_PARAMETERS_REVISION_2; + rssp->hdr.size = sizeof(struct ndis_recv_scale_param); + rssp->flag = flag; + rssp->hashinfo = NDIS_HASH_FUNC_TOEPLITZ | NDIS_HASH_IPV4 | + NDIS_HASH_TCP_IPV4 | NDIS_HASH_IPV6 | + NDIS_HASH_TCP_IPV6; + rssp->indirect_tabsize = 4*ITAB_NUM; + rssp->indirect_taboffset = sizeof(struct ndis_recv_scale_param); + rssp->hashkey_size = NETVSC_HASH_KEYLEN; + rssp->hashkey_offset = rssp->indirect_taboffset + + rssp->indirect_tabsize; + + /* Set indirection table entries */ + itab = (u32 *)(rssp + 1); + for (i = 0; i < ITAB_NUM; i++) + itab[i] = ndc->rx_table[i]; + + /* Set hask key values */ + keyp = (u8 *)((unsigned long)rssp + rssp->hashkey_offset); + memcpy(keyp, rss_key, NETVSC_HASH_KEYLEN); + + ret = rndis_filter_send_request(rdev, request); + if (ret != 0) + goto cleanup; + + wait_for_completion(&request->wait_event); + set_complete = &request->response_msg.msg.set_complete; + if (set_complete->status == RNDIS_STATUS_SUCCESS) { + if (!(flag & NDIS_RSS_PARAM_FLAG_DISABLE_RSS) && + !(flag & NDIS_RSS_PARAM_FLAG_HASH_KEY_UNCHANGED)) + memcpy(rdev->rss_key, rss_key, NETVSC_HASH_KEYLEN); + + } else { + netdev_err(ndev, "Fail to set RSS parameters:0x%x\n", + set_complete->status); + ret = -EINVAL; + } + +cleanup: + put_rndis_request(rdev, request); + return ret; +} + +int rndis_filter_set_rss_param(struct rndis_device *rdev, + const u8 *rss_key) +{ + /* Disable RSS before change */ + rndis_set_rss_param_msg(rdev, rss_key, + NDIS_RSS_PARAM_FLAG_DISABLE_RSS); + + return rndis_set_rss_param_msg(rdev, rss_key, 0); +} + +static int rndis_filter_query_device_link_status(struct rndis_device *dev, + struct netvsc_device *net_device) +{ + u32 size = sizeof(u32); + u32 link_status; + + return rndis_filter_query_device(dev, net_device, + RNDIS_OID_GEN_MEDIA_CONNECT_STATUS, + &link_status, &size); +} + +static int rndis_filter_query_link_speed(struct rndis_device *dev, + struct netvsc_device *net_device) +{ + u32 size = sizeof(u32); + u32 link_speed; + struct net_device_context *ndc; + int ret; + + ret = rndis_filter_query_device(dev, net_device, + RNDIS_OID_GEN_LINK_SPEED, + &link_speed, &size); + + if (!ret) { + ndc = netdev_priv(dev->ndev); + + /* The link speed reported from host is in 100bps unit, so + * we convert it to Mbps here. + */ + ndc->speed = link_speed / 10000; + } + + return ret; +} + +static int rndis_filter_set_packet_filter(struct rndis_device *dev, + u32 new_filter) +{ + struct rndis_request *request; + struct rndis_set_request *set; + int ret; + + if (dev->filter == new_filter) + return 0; + + request = get_rndis_request(dev, RNDIS_MSG_SET, + RNDIS_MESSAGE_SIZE(struct rndis_set_request) + + sizeof(u32)); + if (!request) + return -ENOMEM; + + /* Setup the rndis set */ + set = &request->request_msg.msg.set_req; + set->oid = RNDIS_OID_GEN_CURRENT_PACKET_FILTER; + set->info_buflen = sizeof(u32); + set->info_buf_offset = offsetof(typeof(*set), info_buf); + memcpy(set->info_buf, &new_filter, sizeof(u32)); + + ret = rndis_filter_send_request(dev, request); + if (ret == 0) { + wait_for_completion(&request->wait_event); + dev->filter = new_filter; + } + + put_rndis_request(dev, request); + + return ret; +} + +static void rndis_set_multicast(struct work_struct *w) +{ + struct rndis_device *rdev + = container_of(w, struct rndis_device, mcast_work); + u32 filter = NDIS_PACKET_TYPE_DIRECTED; + unsigned int flags = rdev->ndev->flags; + + if (flags & IFF_PROMISC) { + filter = NDIS_PACKET_TYPE_PROMISCUOUS; + } else { + if (!netdev_mc_empty(rdev->ndev) || (flags & IFF_ALLMULTI)) + filter |= NDIS_PACKET_TYPE_ALL_MULTICAST; + if (flags & IFF_BROADCAST) + filter |= NDIS_PACKET_TYPE_BROADCAST; + } + + rndis_filter_set_packet_filter(rdev, filter); +} + +void rndis_filter_update(struct netvsc_device *nvdev) +{ + struct rndis_device *rdev = nvdev->extension; + + schedule_work(&rdev->mcast_work); +} + +static int rndis_filter_init_device(struct rndis_device *dev, + struct netvsc_device *nvdev) +{ + struct rndis_request *request; + struct rndis_initialize_request *init; + struct rndis_initialize_complete *init_complete; + u32 status; + int ret; + + request = get_rndis_request(dev, RNDIS_MSG_INIT, + RNDIS_MESSAGE_SIZE(struct rndis_initialize_request)); + if (!request) { + ret = -ENOMEM; + goto cleanup; + } + + /* Setup the rndis set */ + init = &request->request_msg.msg.init_req; + init->major_ver = RNDIS_MAJOR_VERSION; + init->minor_ver = RNDIS_MINOR_VERSION; + init->max_xfer_size = 0x4000; + + dev->state = RNDIS_DEV_INITIALIZING; + + ret = rndis_filter_send_request(dev, request); + if (ret != 0) { + dev->state = RNDIS_DEV_UNINITIALIZED; + goto cleanup; + } + + wait_for_completion(&request->wait_event); + + init_complete = &request->response_msg.msg.init_complete; + status = init_complete->status; + if (status == RNDIS_STATUS_SUCCESS) { + dev->state = RNDIS_DEV_INITIALIZED; + nvdev->max_pkt = init_complete->max_pkt_per_msg; + nvdev->pkt_align = 1 << init_complete->pkt_alignment_factor; + ret = 0; + } else { + dev->state = RNDIS_DEV_UNINITIALIZED; + ret = -EINVAL; + } + +cleanup: + if (request) + put_rndis_request(dev, request); + + return ret; +} + +static bool netvsc_device_idle(const struct netvsc_device *nvdev) +{ + int i; + + for (i = 0; i < nvdev->num_chn; i++) { + const struct netvsc_channel *nvchan = &nvdev->chan_table[i]; + + if (nvchan->mrc.first != nvchan->mrc.next) + return false; + + if (atomic_read(&nvchan->queue_sends) > 0) + return false; + } + + return true; +} + +static void rndis_filter_halt_device(struct netvsc_device *nvdev, + struct rndis_device *dev) +{ + struct rndis_request *request; + struct rndis_halt_request *halt; + + /* Attempt to do a rndis device halt */ + request = get_rndis_request(dev, RNDIS_MSG_HALT, + RNDIS_MESSAGE_SIZE(struct rndis_halt_request)); + if (!request) + goto cleanup; + + /* Setup the rndis set */ + halt = &request->request_msg.msg.halt_req; + halt->req_id = atomic_inc_return(&dev->new_req_id); + + /* Ignore return since this msg is optional. */ + rndis_filter_send_request(dev, request); + + dev->state = RNDIS_DEV_UNINITIALIZED; + +cleanup: + nvdev->destroy = true; + + /* Force flag to be ordered before waiting */ + wmb(); + + /* Wait for all send completions */ + wait_event(nvdev->wait_drain, netvsc_device_idle(nvdev)); + + if (request) + put_rndis_request(dev, request); +} + +static int rndis_filter_open_device(struct rndis_device *dev) +{ + int ret; + + if (dev->state != RNDIS_DEV_INITIALIZED) + return 0; + + ret = rndis_filter_set_packet_filter(dev, + NDIS_PACKET_TYPE_BROADCAST | + NDIS_PACKET_TYPE_ALL_MULTICAST | + NDIS_PACKET_TYPE_DIRECTED); + if (ret == 0) + dev->state = RNDIS_DEV_DATAINITIALIZED; + + return ret; +} + +static int rndis_filter_close_device(struct rndis_device *dev) +{ + int ret; + + if (dev->state != RNDIS_DEV_DATAINITIALIZED) + return 0; + + /* Make sure rndis_set_multicast doesn't re-enable filter! */ + cancel_work_sync(&dev->mcast_work); + + ret = rndis_filter_set_packet_filter(dev, 0); + if (ret == -ENODEV) + ret = 0; + + if (ret == 0) + dev->state = RNDIS_DEV_INITIALIZED; + + return ret; +} + +static void netvsc_sc_open(struct vmbus_channel *new_sc) +{ + struct net_device *ndev = + hv_get_drvdata(new_sc->primary_channel->device_obj); + struct net_device_context *ndev_ctx = netdev_priv(ndev); + struct netvsc_device *nvscdev; + u16 chn_index = new_sc->offermsg.offer.sub_channel_index; + struct netvsc_channel *nvchan; + int ret; + + /* This is safe because this callback only happens when + * new device is being setup and waiting on the channel_init_wait. + */ + nvscdev = rcu_dereference_raw(ndev_ctx->nvdev); + if (!nvscdev || chn_index >= nvscdev->num_chn) + return; + + nvchan = nvscdev->chan_table + chn_index; + + /* Because the device uses NAPI, all the interrupt batching and + * control is done via Net softirq, not the channel handling + */ + set_channel_read_mode(new_sc, HV_CALL_ISR); + + /* Set the channel before opening.*/ + nvchan->channel = new_sc; + + new_sc->next_request_id_callback = vmbus_next_request_id; + new_sc->request_addr_callback = vmbus_request_addr; + new_sc->rqstor_size = netvsc_rqstor_size(netvsc_ring_bytes); + new_sc->max_pkt_size = NETVSC_MAX_PKT_SIZE; + + ret = vmbus_open(new_sc, netvsc_ring_bytes, + netvsc_ring_bytes, NULL, 0, + netvsc_channel_cb, nvchan); + if (ret == 0) + napi_enable(&nvchan->napi); + else + netdev_notice(ndev, "sub channel open failed: %d\n", ret); + + if (atomic_inc_return(&nvscdev->open_chn) == nvscdev->num_chn) + wake_up(&nvscdev->subchan_open); +} + +/* Open sub-channels after completing the handling of the device probe. + * This breaks overlap of processing the host message for the + * new primary channel with the initialization of sub-channels. + */ +int rndis_set_subchannel(struct net_device *ndev, + struct netvsc_device *nvdev, + struct netvsc_device_info *dev_info) +{ + struct nvsp_message *init_packet = &nvdev->channel_init_pkt; + struct net_device_context *ndev_ctx = netdev_priv(ndev); + struct hv_device *hv_dev = ndev_ctx->device_ctx; + struct rndis_device *rdev = nvdev->extension; + int i, ret; + + ASSERT_RTNL(); + + memset(init_packet, 0, sizeof(struct nvsp_message)); + init_packet->hdr.msg_type = NVSP_MSG5_TYPE_SUBCHANNEL; + init_packet->msg.v5_msg.subchn_req.op = NVSP_SUBCHANNEL_ALLOCATE; + init_packet->msg.v5_msg.subchn_req.num_subchannels = + nvdev->num_chn - 1; + trace_nvsp_send(ndev, init_packet); + + ret = vmbus_sendpacket(hv_dev->channel, init_packet, + sizeof(struct nvsp_message), + (unsigned long)init_packet, + VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (ret) { + netdev_err(ndev, "sub channel allocate send failed: %d\n", ret); + return ret; + } + + wait_for_completion(&nvdev->channel_init_wait); + if (init_packet->msg.v5_msg.subchn_comp.status != NVSP_STAT_SUCCESS) { + netdev_err(ndev, "sub channel request failed\n"); + return -EIO; + } + + /* Check that number of allocated sub channel is within the expected range */ + if (init_packet->msg.v5_msg.subchn_comp.num_subchannels > nvdev->num_chn - 1) { + netdev_err(ndev, "invalid number of allocated sub channel\n"); + return -EINVAL; + } + nvdev->num_chn = 1 + + init_packet->msg.v5_msg.subchn_comp.num_subchannels; + + /* wait for all sub channels to open */ + wait_event(nvdev->subchan_open, + atomic_read(&nvdev->open_chn) == nvdev->num_chn); + + for (i = 0; i < VRSS_SEND_TAB_SIZE; i++) + ndev_ctx->tx_table[i] = i % nvdev->num_chn; + + /* ignore failures from setting rss parameters, still have channels */ + if (dev_info) + rndis_filter_set_rss_param(rdev, dev_info->rss_key); + else + rndis_filter_set_rss_param(rdev, netvsc_hash_key); + + netif_set_real_num_tx_queues(ndev, nvdev->num_chn); + netif_set_real_num_rx_queues(ndev, nvdev->num_chn); + + return 0; +} + +static int rndis_netdev_set_hwcaps(struct rndis_device *rndis_device, + struct netvsc_device *nvdev) +{ + struct net_device *net = rndis_device->ndev; + struct net_device_context *net_device_ctx = netdev_priv(net); + struct ndis_offload hwcaps; + struct ndis_offload_params offloads; + unsigned int gso_max_size = GSO_LEGACY_MAX_SIZE; + int ret; + + /* Find HW offload capabilities */ + ret = rndis_query_hwcaps(rndis_device, nvdev, &hwcaps); + if (ret != 0) + return ret; + + /* A value of zero means "no change"; now turn on what we want. */ + memset(&offloads, 0, sizeof(struct ndis_offload_params)); + + /* Linux does not care about IP checksum, always does in kernel */ + offloads.ip_v4_csum = NDIS_OFFLOAD_PARAMETERS_TX_RX_DISABLED; + + /* Reset previously set hw_features flags */ + net->hw_features &= ~NETVSC_SUPPORTED_HW_FEATURES; + net_device_ctx->tx_checksum_mask = 0; + + /* Compute tx offload settings based on hw capabilities */ + net->hw_features |= NETIF_F_RXCSUM; + net->hw_features |= NETIF_F_SG; + net->hw_features |= NETIF_F_RXHASH; + + if ((hwcaps.csum.ip4_txcsum & NDIS_TXCSUM_ALL_TCP4) == NDIS_TXCSUM_ALL_TCP4) { + /* Can checksum TCP */ + net->hw_features |= NETIF_F_IP_CSUM; + net_device_ctx->tx_checksum_mask |= TRANSPORT_INFO_IPV4_TCP; + + offloads.tcp_ip_v4_csum = NDIS_OFFLOAD_PARAMETERS_TX_RX_ENABLED; + + if (hwcaps.lsov2.ip4_encap & NDIS_OFFLOAD_ENCAP_8023) { + offloads.lso_v2_ipv4 = NDIS_OFFLOAD_PARAMETERS_LSOV2_ENABLED; + net->hw_features |= NETIF_F_TSO; + + if (hwcaps.lsov2.ip4_maxsz < gso_max_size) + gso_max_size = hwcaps.lsov2.ip4_maxsz; + } + + if (hwcaps.csum.ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) { + offloads.udp_ip_v4_csum = NDIS_OFFLOAD_PARAMETERS_TX_RX_ENABLED; + net_device_ctx->tx_checksum_mask |= TRANSPORT_INFO_IPV4_UDP; + } + } + + if ((hwcaps.csum.ip6_txcsum & NDIS_TXCSUM_ALL_TCP6) == NDIS_TXCSUM_ALL_TCP6) { + net->hw_features |= NETIF_F_IPV6_CSUM; + + offloads.tcp_ip_v6_csum = NDIS_OFFLOAD_PARAMETERS_TX_RX_ENABLED; + net_device_ctx->tx_checksum_mask |= TRANSPORT_INFO_IPV6_TCP; + + if ((hwcaps.lsov2.ip6_encap & NDIS_OFFLOAD_ENCAP_8023) && + (hwcaps.lsov2.ip6_opts & NDIS_LSOV2_CAP_IP6) == NDIS_LSOV2_CAP_IP6) { + offloads.lso_v2_ipv6 = NDIS_OFFLOAD_PARAMETERS_LSOV2_ENABLED; + net->hw_features |= NETIF_F_TSO6; + + if (hwcaps.lsov2.ip6_maxsz < gso_max_size) + gso_max_size = hwcaps.lsov2.ip6_maxsz; + } + + if (hwcaps.csum.ip6_txcsum & NDIS_TXCSUM_CAP_UDP6) { + offloads.udp_ip_v6_csum = NDIS_OFFLOAD_PARAMETERS_TX_RX_ENABLED; + net_device_ctx->tx_checksum_mask |= TRANSPORT_INFO_IPV6_UDP; + } + } + + if (hwcaps.rsc.ip4 && hwcaps.rsc.ip6) { + net->hw_features |= NETIF_F_LRO; + + if (net->features & NETIF_F_LRO) { + offloads.rsc_ip_v4 = NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED; + offloads.rsc_ip_v6 = NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED; + } else { + offloads.rsc_ip_v4 = NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED; + offloads.rsc_ip_v6 = NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED; + } + } + + /* In case some hw_features disappeared we need to remove them from + * net->features list as they're no longer supported. + */ + net->features &= ~NETVSC_SUPPORTED_HW_FEATURES | net->hw_features; + + netif_set_tso_max_size(net, gso_max_size); + + ret = rndis_filter_set_offload_params(net, nvdev, &offloads); + + return ret; +} + +static void rndis_get_friendly_name(struct net_device *net, + struct rndis_device *rndis_device, + struct netvsc_device *net_device) +{ + ucs2_char_t wname[256]; + unsigned long len; + u8 ifalias[256]; + u32 size; + + size = sizeof(wname); + if (rndis_filter_query_device(rndis_device, net_device, + RNDIS_OID_GEN_FRIENDLY_NAME, + wname, &size) != 0) + return; /* ignore if host does not support */ + + if (size == 0) + return; /* name not set */ + + /* Convert Windows Unicode string to UTF-8 */ + len = ucs2_as_utf8(ifalias, wname, sizeof(ifalias)); + + /* ignore the default value from host */ + if (strcmp(ifalias, "Network Adapter") != 0) + dev_set_alias(net, ifalias, len); +} + +struct netvsc_device *rndis_filter_device_add(struct hv_device *dev, + struct netvsc_device_info *device_info) +{ + struct net_device *net = hv_get_drvdata(dev); + struct net_device_context *ndc = netdev_priv(net); + struct netvsc_device *net_device; + struct rndis_device *rndis_device; + struct ndis_recv_scale_cap rsscap; + u32 rsscap_size = sizeof(struct ndis_recv_scale_cap); + u32 mtu, size; + u32 num_possible_rss_qs; + int i, ret; + + rndis_device = get_rndis_device(); + if (!rndis_device) + return ERR_PTR(-ENODEV); + + /* Let the inner driver handle this first to create the netvsc channel + * NOTE! Once the channel is created, we may get a receive callback + * (RndisFilterOnReceive()) before this call is completed + */ + net_device = netvsc_device_add(dev, device_info); + if (IS_ERR(net_device)) { + kfree(rndis_device); + return net_device; + } + + /* Initialize the rndis device */ + net_device->max_chn = 1; + net_device->num_chn = 1; + + net_device->extension = rndis_device; + rndis_device->ndev = net; + + /* Send the rndis initialization message */ + ret = rndis_filter_init_device(rndis_device, net_device); + if (ret != 0) + goto err_dev_remv; + + /* Get the MTU from the host */ + size = sizeof(u32); + ret = rndis_filter_query_device(rndis_device, net_device, + RNDIS_OID_GEN_MAXIMUM_FRAME_SIZE, + &mtu, &size); + if (ret == 0 && size == sizeof(u32) && mtu < net->mtu) + net->mtu = mtu; + + /* Get the mac address */ + ret = rndis_filter_query_device_mac(rndis_device, net_device); + if (ret != 0) + goto err_dev_remv; + + memcpy(device_info->mac_adr, rndis_device->hw_mac_adr, ETH_ALEN); + + /* Get friendly name as ifalias*/ + if (!net->ifalias) + rndis_get_friendly_name(net, rndis_device, net_device); + + /* Query and set hardware capabilities */ + ret = rndis_netdev_set_hwcaps(rndis_device, net_device); + if (ret != 0) + goto err_dev_remv; + + rndis_filter_query_device_link_status(rndis_device, net_device); + + netdev_dbg(net, "Device MAC %pM link state %s\n", + rndis_device->hw_mac_adr, + rndis_device->link_state ? "down" : "up"); + + if (net_device->nvsp_version < NVSP_PROTOCOL_VERSION_5) + goto out; + + rndis_filter_query_link_speed(rndis_device, net_device); + + /* vRSS setup */ + memset(&rsscap, 0, rsscap_size); + ret = rndis_filter_query_device(rndis_device, net_device, + OID_GEN_RECEIVE_SCALE_CAPABILITIES, + &rsscap, &rsscap_size); + if (ret || rsscap.num_recv_que < 2) + goto out; + + /* This guarantees that num_possible_rss_qs <= num_online_cpus */ + num_possible_rss_qs = min_t(u32, num_online_cpus(), + rsscap.num_recv_que); + + net_device->max_chn = min_t(u32, VRSS_CHANNEL_MAX, num_possible_rss_qs); + + /* We will use the given number of channels if available. */ + net_device->num_chn = min(net_device->max_chn, device_info->num_chn); + + if (!netif_is_rxfh_configured(net)) { + for (i = 0; i < ITAB_NUM; i++) + ndc->rx_table[i] = ethtool_rxfh_indir_default( + i, net_device->num_chn); + } + + atomic_set(&net_device->open_chn, 1); + vmbus_set_sc_create_callback(dev->channel, netvsc_sc_open); + + for (i = 1; i < net_device->num_chn; i++) { + ret = netvsc_alloc_recv_comp_ring(net_device, i); + if (ret) { + while (--i != 0) + vfree(net_device->chan_table[i].mrc.slots); + goto out; + } + } + + for (i = 1; i < net_device->num_chn; i++) + netif_napi_add(net, &net_device->chan_table[i].napi, + netvsc_poll); + + return net_device; + +out: + /* setting up multiple channels failed */ + net_device->max_chn = 1; + net_device->num_chn = 1; + return net_device; + +err_dev_remv: + rndis_filter_device_remove(dev, net_device); + return ERR_PTR(ret); +} + +void rndis_filter_device_remove(struct hv_device *dev, + struct netvsc_device *net_dev) +{ + struct rndis_device *rndis_dev = net_dev->extension; + + /* Halt and release the rndis device */ + rndis_filter_halt_device(net_dev, rndis_dev); + + netvsc_device_remove(dev); +} + +int rndis_filter_open(struct netvsc_device *nvdev) +{ + if (!nvdev) + return -EINVAL; + + return rndis_filter_open_device(nvdev->extension); +} + +int rndis_filter_close(struct netvsc_device *nvdev) +{ + if (!nvdev) + return -EINVAL; + + return rndis_filter_close_device(nvdev->extension); +} |